materusPC: more vm tuning

configurations/host/materusPC/hardware/boot.nix

@@ -1,4 +1,4 @@
-{ config, pkgs, lib, ... }:
+{ config, pkgs, lib, materusArg, ... }:
 let
   video = [
 
@@ -17,8 +17,8 @@ in
 {
   #Kernel
   boot.kernelPackages = pkgs.linuxPackages_zen;
-  boot.kernelParams = [ "rcu_nocbs=8-15,24-31" "nohz_full=8-15,24-31" "vfio_iommu_type1.allow_unsafe_interrupts=1" "pcie_acs_override=downstream,multifunction" /*"pci-stub.ids=1002:744c"*/ "nox2apic" "nvme_core.default_ps_max_latency_us=0" "nvme_core.io_timeout=255" "nvme_core.max_retries=10" "nvme_core.shutdown_timeout=10" "amd_iommu=on" "iommu=pt" ] ++ video;
+  boot.kernelParams = [ "rcu_nocbs=${materusArg.materusPC.vmCores}" "nohz_full=${materusArg.materusPC.vmCores}" "vfio_iommu_type1.allow_unsafe_interrupts=1" "pcie_acs_override=downstream,multifunction" /*"pci-stub.ids=1002:744c"*/ "nox2apic" "nvme_core.default_ps_max_latency_us=0" "nvme_core.io_timeout=255" "nvme_core.max_retries=10" "nvme_core.shutdown_timeout=10" "amd_iommu=on" "iommu=pt" ] ++ video;
-  boot.kernelModules = [ "pci-stub" "amdgpu" "i2c_dev" "kvm_amd" "vfio" "vfio_iommu_type1" "vfio-pci"];
+  boot.kernelModules = [ "pci-stub" "amdgpu" "i2c_dev" "kvm_amd" "vfio" "vfio_iommu_type1" "vfio-pci" ];
   boot.extraModprobeConfig = ''
     options kvm_amd nested=1 avic=1 npt=1 sev=0
     options vfio_iommu_type1 allow_unsafe_interrupts=1

configurations/host/materusPC/vm/default.nix

@@ -3,7 +3,14 @@
   imports = [
     ./win10
   ];
-
+  materus.materusArg.materusPC = {
+    allCores = "0-31";
+    allCoresMask = "ffffffff";
+    hostCores = "0-7,16-23";
+    hostCoresMask = "00ff00ff";
+    vmCores = "8-15,24-31";
+    vmCoresMask = "ff00ff00";
+  };
   systemd.mounts = [
     {
       where = "/dev/hugepages";
@@ -69,6 +76,7 @@
             bindfs
             qemu-utils
             psmisc
+            procps
           ];
         };
       in

configurations/host/materusPC/vm/win10/default.nix

@@ -1,5 +1,27 @@
-{ config, pkgs, ... }:
+{ config, pkgs, materusArg, ... }:
 let
+  VM_UUID = "ad2632db-0da0-4204-98b3-0592a185ebd0";
+
+  startedHook = ''
+    QEMU_PID=$(ps aux | grep qemu-system-x86_64 | grep "${VM_UUID}" | tr -s ' ' | cut -d " " -f 2)
+
+    for pid in $(cat /sys/fs/cgroup/cpu/machine.slice/machine-qemu*$1.scope/libvirt/vcpu*/tasks); do 
+      renice -n "-15" -p "$pid";
+    done
+    renice -n "-10" -p "$QEMU_PID";
+
+    echo "${materusArg.materusPC.hostCoresMask}" > /proc/irq/default_smp_affinity
+    for irq in /proc/irq/[0-9]*/smp_affinity; do 
+      if [ $(cat $irq) = "${materusArg.materusPC.allCoresMask}" ]; then
+        echo "${materusArg.materusPC.hostCoresMask}" > $irq 2> /dev/null 
+      fi;
+    done;
+    for irq in $(cat /proc/interrupts | grep vfio | cut -d ":" -f 1); do 
+      echo "${materusArg.materusPC.vmCoresMask}" > /proc/irq/$irq/smp_affinity; 
+    done
+
+
+  '';
   startHook = /*''
 
 
@@ -14,32 +36,39 @@ let
     ''
     +*/
     ''
-       # Make sure nothing renders on gpu to prevent "sysfs: cannot create duplicate filename" after rebinding to amdgpu
+      # Make sure nothing renders on gpu to prevent "sysfs: cannot create duplicate filename" after rebinding to amdgpu
-       chmod 0 /dev/dri/renderD128 
+      chmod 0 /dev/dri/renderD128 
-       fuser -k /dev/dri/renderD128
+      fuser -k /dev/dri/renderD128
 
-       # Seems to fix reset bug for 7900 XTX
+      # Seems to fix reset bug for 7900 XTX
-       echo "0" > "/sys/bus/pci/devices/''${VIRSH_GPU_VIDEO}/d3cold_allowed"
+      echo "0" > "/sys/bus/pci/devices/''${VIRSH_GPU_VIDEO}/d3cold_allowed"
 
-       systemctl stop windows-share-mount.service
+      systemctl stop windows-share-mount.service
     
     
-       echo ''$VIRSH_GPU_VIDEO > "/sys/bus/pci/devices/''${VIRSH_GPU_VIDEO}/driver/unbind"
+      echo ''$VIRSH_GPU_VIDEO > "/sys/bus/pci/devices/''${VIRSH_GPU_VIDEO}/driver/unbind"
-       echo ''$VIRSH_GPU_AUDIO > "/sys/bus/pci/devices/''${VIRSH_GPU_AUDIO}/driver/unbind"
+      echo ''$VIRSH_GPU_AUDIO > "/sys/bus/pci/devices/''${VIRSH_GPU_AUDIO}/driver/unbind"
 
-       sleep 1s
+      sleep 1s
 
-       echo "15" > "/sys/bus/pci/devices/''${VIRSH_GPU_VIDEO}/resource0_resize"
+      echo "15" > "/sys/bus/pci/devices/''${VIRSH_GPU_VIDEO}/resource0_resize"
-       echo "8" > "/sys/bus/pci/devices/''${VIRSH_GPU_VIDEO}/resource2_resize"
+      echo "8" > "/sys/bus/pci/devices/''${VIRSH_GPU_VIDEO}/resource2_resize"
 
-       echo "3" > /proc/sys/vm/drop_caches
+      sync
-       echo "1" > /proc/sys/vm/compact_memory
+      echo "3" > /proc/sys/vm/drop_caches
+      sync
+      echo "1" > /proc/sys/vm/compact_memory
 
 
 
-       systemctl set-property --runtime -- user.slice AllowedCPUs=0-7,16-23
+      systemctl set-property --runtime -- user.slice AllowedCPUs=${materusArg.materusPC.hostCores}
-       systemctl set-property --runtime -- system.slice AllowedCPUs=0-7,16-23
+      systemctl set-property --runtime -- system.slice AllowedCPUs=${materusArg.materusPC.hostCores}
-       systemctl set-property --runtime -- init.scope AllowedCPUs=0-7,16-23
+      systemctl set-property --runtime -- init.scope AllowedCPUs=${materusArg.materusPC.hostCores}
+      echo "${materusArg.materusPC.hostCoresMask}" > /sys/bus/workqueue/devices/writeback/cpumask
+      echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor
+
+      sysctl vm.stat_interval=120
+      sysctl -w kernel.watchdog=0
 
 
     '';
@@ -74,10 +103,20 @@ let
 
     systemctl start windows-share-mount.service
 
-    systemctl set-property --runtime -- user.slice AllowedCPUs=0-31
+    systemctl set-property --runtime -- user.slice AllowedCPUs=${materusArg.materusPC.allCores}
-    systemctl set-property --runtime -- system.slice AllowedCPUs=0-31
+    systemctl set-property --runtime -- system.slice AllowedCPUs=${materusArg.materusPC.allCores}
-    systemctl set-property --runtime -- init.scope AllowedCPUs=0-31
+    systemctl set-property --runtime -- init.scope AllowedCPUs=${materusArg.materusPC.allCores}
+    echo "${materusArg.materusPC.allCoresMask}" > /sys/bus/workqueue/devices/writeback/cpumask
+    echo powersave | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor
 
+    sysctl vm.stat_interval=1
+    sysctl -w kernel.watchdog=1
+    echo "${materusArg.materusPC.allCoresMask}" > /proc/irq/default_smp_affinity
+    for irq in /proc/irq/[0-9]*/smp_affinity; do 
+      if [ $(cat $irq) = "${materusArg.materusPC.hostCoresMask}" ] || [ $(cat $irq) = "${materusArg.materusPC.vmCoresMask}" ]; then
+        echo "${materusArg.materusPC.allCoresMask}" > $irq 2> /dev/null 
+      fi;
+    done;
   
 
   '';
@@ -102,6 +141,10 @@ in
           ${startHook}
         fi
 
+        if [ ''$2 = "started" ] && [ ''$3 = "begin" ]; then
+          ${startedHook}
+        fi
+
         if [ ''$2 = "release" ] && [ ''$3 = "end" ]; then
           ${stopHook}
         fi