materusPC: more vm tuning

configurations/host/materusPC/hardware/boot.nix

@@ -1,4 +1,4 @@
-{ config, pkgs, lib, ... }:
+{ config, pkgs, lib, materusArg, ... }:
 let
   video = [
 
@@ -17,8 +17,8 @@ in
 {
   #Kernel
   boot.kernelPackages = pkgs.linuxPackages_zen;
-  boot.kernelParams = [ "rcu_nocbs=8-15,24-31" "nohz_full=8-15,24-31" "vfio_iommu_type1.allow_unsafe_interrupts=1" "pcie_acs_override=downstream,multifunction" /*"pci-stub.ids=1002:744c"*/ "nox2apic" "nvme_core.default_ps_max_latency_us=0" "nvme_core.io_timeout=255" "nvme_core.max_retries=10" "nvme_core.shutdown_timeout=10" "amd_iommu=on" "iommu=pt" ] ++ video;
-  boot.kernelModules = [ "pci-stub" "amdgpu" "i2c_dev" "kvm_amd" "vfio" "vfio_iommu_type1" "vfio-pci"];
+  boot.kernelParams = [ "rcu_nocbs=${materusArg.materusPC.vmCores}" "nohz_full=${materusArg.materusPC.vmCores}" "vfio_iommu_type1.allow_unsafe_interrupts=1" "pcie_acs_override=downstream,multifunction" /*"pci-stub.ids=1002:744c"*/ "nox2apic" "nvme_core.default_ps_max_latency_us=0" "nvme_core.io_timeout=255" "nvme_core.max_retries=10" "nvme_core.shutdown_timeout=10" "amd_iommu=on" "iommu=pt" ] ++ video;
+  boot.kernelModules = [ "pci-stub" "amdgpu" "i2c_dev" "kvm_amd" "vfio" "vfio_iommu_type1" "vfio-pci" ];
   boot.extraModprobeConfig = ''
     options kvm_amd nested=1 avic=1 npt=1 sev=0
     options vfio_iommu_type1 allow_unsafe_interrupts=1

configurations/host/materusPC/tmp.nix

@@ -87,7 +87,7 @@
       "libvirtd"
       "podman"
       "lxd"
-    ]; 
+    ];
     shell = pkgs.zsh;
     description = "Mateusz Słodkowicz";
   };
@@ -120,14 +120,38 @@
 
 
   services.pcscd.enable = true;
+  
   services.samba-wsdd.enable = true;
-
+  services.samba-wsdd.openFirewall = true;
   services.samba = {
     enable = true;
     package = pkgs.sambaFull;
+    securityType = "user";
+    openFirewall = true;
+    extraConfig = ''
+      workgroup = WORKGROUP
+      server string = smbmaterus
+      netbios name = smbmaterus
+      security = user 
+      hosts allow = 192.168.122. 127.0.0.1 localhost
+      hosts deny = 0.0.0.0/0
+      guest account = nobody
+      map to guest = bad user
+    '';
+    shares = {
+      windows = {
+        path = "/materus/data/VM/windows_shared";
+        browseable = "yes";
+        "read only" = "no";
+        "guest ok" = "no";
+        "create mask" = "0644";
+        "directory mask" = "0755";
+        "force user" = "materus";
+        "force group" = "users";
+      };
+    };
   };
 
-
   programs.gnupg.agent = {
     enable = true;
     enableSSHSupport = false;
@@ -155,4 +179,5 @@
   services.davfs2.enable = true;
 
 
+
 }

configurations/host/materusPC/vm/default.nix

@@ -3,7 +3,14 @@
   imports = [
     ./win10
   ];
-
+  materus.materusArg.materusPC = {
+    allCores = "0-31";
+    allCoresMask = "ffffffff";
+    hostCores = "0-7,16-23";
+    hostCoresMask = "00ff00ff";
+    vmCores = "8-15,24-31";
+    vmCoresMask = "ff00ff00";
+  };
   systemd.mounts = [
     {
       where = "/dev/hugepages";
@@ -69,6 +76,7 @@
             bindfs
             qemu-utils
             psmisc
+            procps
           ];
         };
       in

configurations/host/materusPC/vm/win10/default.nix

@@ -1,5 +1,27 @@
-{ config, pkgs, ... }:
+{ config, pkgs, materusArg, ... }:
 let
+  VM_UUID = "ad2632db-0da0-4204-98b3-0592a185ebd0";
+
+  startedHook = ''
+    QEMU_PID=$(ps aux | grep qemu-system-x86_64 | grep "${VM_UUID}" | tr -s ' ' | cut -d " " -f 2)
+
+    for pid in $(cat /sys/fs/cgroup/cpu/machine.slice/machine-qemu*$1.scope/libvirt/vcpu*/tasks); do 
+      renice -n "-15" -p "$pid";
+    done
+    renice -n "-10" -p "$QEMU_PID";
+
+    echo "${materusArg.materusPC.hostCoresMask}" > /proc/irq/default_smp_affinity
+    for irq in /proc/irq/[0-9]*/smp_affinity; do 
+      if [ $(cat $irq) = "${materusArg.materusPC.allCoresMask}" ]; then
+        echo "${materusArg.materusPC.hostCoresMask}" > $irq 2> /dev/null 
+      fi;
+    done;
+    for irq in $(cat /proc/interrupts | grep vfio | cut -d ":" -f 1); do 
+      echo "${materusArg.materusPC.vmCoresMask}" > /proc/irq/$irq/smp_affinity; 
+    done
+
+
+  '';
   startHook = /*''
 
 
@@ -14,33 +36,39 @@ let
     ''
     +*/
     ''
-       # Make sure nothing renders on gpu to prevent "sysfs: cannot create duplicate filename" after rebinding to amdgpu
-       chmod 0 /dev/dri/renderD128 
-       fuser -k /dev/dri/renderD128
+      # Make sure nothing renders on gpu to prevent "sysfs: cannot create duplicate filename" after rebinding to amdgpu
+      chmod 0 /dev/dri/renderD128 
+      fuser -k /dev/dri/renderD128
 
-       # Seems to fix reset bug for 7900 XTX
-       echo "0" > "/sys/bus/pci/devices/''${VIRSH_GPU_VIDEO}/d3cold_allowed"
+      # Seems to fix reset bug for 7900 XTX
+      echo "0" > "/sys/bus/pci/devices/''${VIRSH_GPU_VIDEO}/d3cold_allowed"
 
-       systemctl stop mountWin10Share.service
+      systemctl stop windows-share-mount.service
     
     
-       echo ''$VIRSH_GPU_VIDEO > "/sys/bus/pci/devices/''${VIRSH_GPU_VIDEO}/driver/unbind"
-       echo ''$VIRSH_GPU_AUDIO > "/sys/bus/pci/devices/''${VIRSH_GPU_AUDIO}/driver/unbind"
+      echo ''$VIRSH_GPU_VIDEO > "/sys/bus/pci/devices/''${VIRSH_GPU_VIDEO}/driver/unbind"
+      echo ''$VIRSH_GPU_AUDIO > "/sys/bus/pci/devices/''${VIRSH_GPU_AUDIO}/driver/unbind"
 
-       sleep 1s
+      sleep 1s
 
-       echo "10" > "/sys/bus/pci/devices/''${VIRSH_GPU_VIDEO}/resource0_resize"
-       echo "8" > "/sys/bus/pci/devices/''${VIRSH_GPU_VIDEO}/resource2_resize"
+      echo "15" > "/sys/bus/pci/devices/''${VIRSH_GPU_VIDEO}/resource0_resize"
+      echo "8" > "/sys/bus/pci/devices/''${VIRSH_GPU_VIDEO}/resource2_resize"
 
-       echo "3" > /proc/sys/vm/drop_caches
-       echo "1" > /proc/sys/vm/compact_memory
-      #echo "8192" >  /sys/devices/system/node/node1/hugepages/hugepages-2048kB/nr_hugepages
+      sync
+      echo "3" > /proc/sys/vm/drop_caches
+      sync
+      echo "1" > /proc/sys/vm/compact_memory
 
 
 
-       systemctl set-property --runtime -- user.slice AllowedCPUs=0-7,16-23
-       systemctl set-property --runtime -- system.slice AllowedCPUs=0-7,16-23
-       systemctl set-property --runtime -- init.scope AllowedCPUs=0-7,16-23
+      systemctl set-property --runtime -- user.slice AllowedCPUs=${materusArg.materusPC.hostCores}
+      systemctl set-property --runtime -- system.slice AllowedCPUs=${materusArg.materusPC.hostCores}
+      systemctl set-property --runtime -- init.scope AllowedCPUs=${materusArg.materusPC.hostCores}
+      echo "${materusArg.materusPC.hostCoresMask}" > /sys/bus/workqueue/devices/writeback/cpumask
+      echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor
+
+      sysctl vm.stat_interval=120
+      sysctl -w kernel.watchdog=0
 
 
     '';
@@ -72,14 +100,23 @@ let
     echo ''$VIRSH_GPU_VIDEO > /sys/bus/pci/drivers/amdgpu/bind
     echo ''$VIRSH_GPU_AUDIO > /sys/bus/pci/drivers/snd_hda_intel/bind
     
-   #echo "0" >  /sys/devices/system/node/node1/hugepages/hugepages-2048kB/nr_hugepages
 
-    systemctl start mountWin10Share.service
+    systemctl start windows-share-mount.service
 
-    systemctl set-property --runtime -- user.slice AllowedCPUs=0-31
-    systemctl set-property --runtime -- system.slice AllowedCPUs=0-31
-    systemctl set-property --runtime -- init.scope AllowedCPUs=0-31
+    systemctl set-property --runtime -- user.slice AllowedCPUs=${materusArg.materusPC.allCores}
+    systemctl set-property --runtime -- system.slice AllowedCPUs=${materusArg.materusPC.allCores}
+    systemctl set-property --runtime -- init.scope AllowedCPUs=${materusArg.materusPC.allCores}
+    echo "${materusArg.materusPC.allCoresMask}" > /sys/bus/workqueue/devices/writeback/cpumask
+    echo powersave | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor
 
+    sysctl vm.stat_interval=1
+    sysctl -w kernel.watchdog=1
+    echo "${materusArg.materusPC.allCoresMask}" > /proc/irq/default_smp_affinity
+    for irq in /proc/irq/[0-9]*/smp_affinity; do 
+      if [ $(cat $irq) = "${materusArg.materusPC.hostCoresMask}" ] || [ $(cat $irq) = "${materusArg.materusPC.vmCoresMask}" ]; then
+        echo "${materusArg.materusPC.allCoresMask}" > $irq 2> /dev/null 
+      fi;
+    done;
   
 
   '';
@@ -104,6 +141,10 @@ in
           ${startHook}
         fi
 
+        if [ ''$2 = "started" ] && [ ''$3 = "begin" ]; then
+          ${startedHook}
+        fi
+
         if [ ''$2 = "release" ] && [ ''$3 = "end" ]; then
           ${stopHook}
         fi
@@ -114,21 +155,19 @@ in
     '';
   };
 
-  systemd.services.mountWin10Share = {
+  systemd.services.windows-share-mount = {
     wantedBy = [ "multi-user.target" ];
     path = [ config.virtualisation.libvirtd.qemu.package pkgs.util-linux pkgs.kmod pkgs.coreutils ];
     serviceConfig.Type = "oneshot";
     serviceConfig.RemainAfterExit = true;
     script = ''
-      modprobe nbd max_part=16
-      sleep 1
-      qemu-nbd -c /dev/nbd0 /materus/data/VM/data.qcow2 --cache=unsafe --discard=unmap
-      sleep 1
-      mount /dev/nbd0p1 /materus/data/Windows -o uid=1000,gid=100
+
+      losetup -P /dev/loop6 /materus/data/VM/data.raw
+      mount /dev/loop6p1 /materus/data/Windows -o uid=1000,gid=100
     '';
     preStop = ''
-      umount /materus/data/Windows
-      qemu-nbd -d /dev/nbd0
+      umount -lf /materus/data/Windows
+      losetup -d /dev/loop6
     '';
   };
 }