diff --git a/configurations/host/materusPC/hardware/boot.nix b/configurations/host/materusPC/hardware/boot.nix index c7a97b2..42e40ab 100644 --- a/configurations/host/materusPC/hardware/boot.nix +++ b/configurations/host/materusPC/hardware/boot.nix @@ -1,4 +1,4 @@ -{ config, pkgs, lib, ... }: +{ config, pkgs, lib, materusArg, ... }: let video = [ @@ -17,8 +17,8 @@ in { #Kernel boot.kernelPackages = pkgs.linuxPackages_zen; - boot.kernelParams = [ "rcu_nocbs=8-15,24-31" "nohz_full=8-15,24-31" "vfio_iommu_type1.allow_unsafe_interrupts=1" "pcie_acs_override=downstream,multifunction" /*"pci-stub.ids=1002:744c"*/ "nox2apic" "nvme_core.default_ps_max_latency_us=0" "nvme_core.io_timeout=255" "nvme_core.max_retries=10" "nvme_core.shutdown_timeout=10" "amd_iommu=on" "iommu=pt" ] ++ video; - boot.kernelModules = [ "pci-stub" "amdgpu" "i2c_dev" "kvm_amd" "vfio" "vfio_iommu_type1" "vfio-pci"]; + boot.kernelParams = [ "rcu_nocbs=${materusArg.materusPC.vmCores}" "nohz_full=${materusArg.materusPC.vmCores}" "vfio_iommu_type1.allow_unsafe_interrupts=1" "pcie_acs_override=downstream,multifunction" /*"pci-stub.ids=1002:744c"*/ "nox2apic" "nvme_core.default_ps_max_latency_us=0" "nvme_core.io_timeout=255" "nvme_core.max_retries=10" "nvme_core.shutdown_timeout=10" "amd_iommu=on" "iommu=pt" ] ++ video; + boot.kernelModules = [ "pci-stub" "amdgpu" "i2c_dev" "kvm_amd" "vfio" "vfio_iommu_type1" "vfio-pci" ]; boot.extraModprobeConfig = '' options kvm_amd nested=1 avic=1 npt=1 sev=0 options vfio_iommu_type1 allow_unsafe_interrupts=1 diff --git a/configurations/host/materusPC/vm/default.nix b/configurations/host/materusPC/vm/default.nix index 211d9ef..12cd114 100644 --- a/configurations/host/materusPC/vm/default.nix +++ b/configurations/host/materusPC/vm/default.nix @@ -3,7 +3,14 @@ imports = [ ./win10 ]; - + materus.materusArg.materusPC = { + allCores = "0-31"; + allCoresMask = "ffffffff"; + hostCores = "0-7,16-23"; + hostCoresMask = "00ff00ff"; + vmCores = "8-15,24-31"; + vmCoresMask = "ff00ff00"; + }; systemd.mounts = [ { where = "/dev/hugepages"; @@ -69,6 +76,7 @@ bindfs qemu-utils psmisc + procps ]; }; in diff --git a/configurations/host/materusPC/vm/win10/default.nix b/configurations/host/materusPC/vm/win10/default.nix index f5a8e4d..8044e50 100644 --- a/configurations/host/materusPC/vm/win10/default.nix +++ b/configurations/host/materusPC/vm/win10/default.nix @@ -1,5 +1,27 @@ -{ config, pkgs, ... }: +{ config, pkgs, materusArg, ... }: let + VM_UUID = "ad2632db-0da0-4204-98b3-0592a185ebd0"; + + startedHook = '' + QEMU_PID=$(ps aux | grep qemu-system-x86_64 | grep "${VM_UUID}" | tr -s ' ' | cut -d " " -f 2) + + for pid in $(cat /sys/fs/cgroup/cpu/machine.slice/machine-qemu*$1.scope/libvirt/vcpu*/tasks); do + renice -n "-15" -p "$pid"; + done + renice -n "-10" -p "$QEMU_PID"; + + echo "${materusArg.materusPC.hostCoresMask}" > /proc/irq/default_smp_affinity + for irq in /proc/irq/[0-9]*/smp_affinity; do + if [ $(cat $irq) = "${materusArg.materusPC.allCoresMask}" ]; then + echo "${materusArg.materusPC.hostCoresMask}" > $irq 2> /dev/null + fi; + done; + for irq in $(cat /proc/interrupts | grep vfio | cut -d ":" -f 1); do + echo "${materusArg.materusPC.vmCoresMask}" > /proc/irq/$irq/smp_affinity; + done + + + ''; startHook = /*'' @@ -14,32 +36,39 @@ let '' +*/ '' - # Make sure nothing renders on gpu to prevent "sysfs: cannot create duplicate filename" after rebinding to amdgpu - chmod 0 /dev/dri/renderD128 - fuser -k /dev/dri/renderD128 + # Make sure nothing renders on gpu to prevent "sysfs: cannot create duplicate filename" after rebinding to amdgpu + chmod 0 /dev/dri/renderD128 + fuser -k /dev/dri/renderD128 - # Seems to fix reset bug for 7900 XTX - echo "0" > "/sys/bus/pci/devices/''${VIRSH_GPU_VIDEO}/d3cold_allowed" + # Seems to fix reset bug for 7900 XTX + echo "0" > "/sys/bus/pci/devices/''${VIRSH_GPU_VIDEO}/d3cold_allowed" - systemctl stop windows-share-mount.service + systemctl stop windows-share-mount.service - echo ''$VIRSH_GPU_VIDEO > "/sys/bus/pci/devices/''${VIRSH_GPU_VIDEO}/driver/unbind" - echo ''$VIRSH_GPU_AUDIO > "/sys/bus/pci/devices/''${VIRSH_GPU_AUDIO}/driver/unbind" + echo ''$VIRSH_GPU_VIDEO > "/sys/bus/pci/devices/''${VIRSH_GPU_VIDEO}/driver/unbind" + echo ''$VIRSH_GPU_AUDIO > "/sys/bus/pci/devices/''${VIRSH_GPU_AUDIO}/driver/unbind" - sleep 1s + sleep 1s - echo "15" > "/sys/bus/pci/devices/''${VIRSH_GPU_VIDEO}/resource0_resize" - echo "8" > "/sys/bus/pci/devices/''${VIRSH_GPU_VIDEO}/resource2_resize" + echo "15" > "/sys/bus/pci/devices/''${VIRSH_GPU_VIDEO}/resource0_resize" + echo "8" > "/sys/bus/pci/devices/''${VIRSH_GPU_VIDEO}/resource2_resize" - echo "3" > /proc/sys/vm/drop_caches - echo "1" > /proc/sys/vm/compact_memory + sync + echo "3" > /proc/sys/vm/drop_caches + sync + echo "1" > /proc/sys/vm/compact_memory - systemctl set-property --runtime -- user.slice AllowedCPUs=0-7,16-23 - systemctl set-property --runtime -- system.slice AllowedCPUs=0-7,16-23 - systemctl set-property --runtime -- init.scope AllowedCPUs=0-7,16-23 + systemctl set-property --runtime -- user.slice AllowedCPUs=${materusArg.materusPC.hostCores} + systemctl set-property --runtime -- system.slice AllowedCPUs=${materusArg.materusPC.hostCores} + systemctl set-property --runtime -- init.scope AllowedCPUs=${materusArg.materusPC.hostCores} + echo "${materusArg.materusPC.hostCoresMask}" > /sys/bus/workqueue/devices/writeback/cpumask + echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor + + sysctl vm.stat_interval=120 + sysctl -w kernel.watchdog=0 ''; @@ -74,10 +103,20 @@ let systemctl start windows-share-mount.service - systemctl set-property --runtime -- user.slice AllowedCPUs=0-31 - systemctl set-property --runtime -- system.slice AllowedCPUs=0-31 - systemctl set-property --runtime -- init.scope AllowedCPUs=0-31 + systemctl set-property --runtime -- user.slice AllowedCPUs=${materusArg.materusPC.allCores} + systemctl set-property --runtime -- system.slice AllowedCPUs=${materusArg.materusPC.allCores} + systemctl set-property --runtime -- init.scope AllowedCPUs=${materusArg.materusPC.allCores} + echo "${materusArg.materusPC.allCoresMask}" > /sys/bus/workqueue/devices/writeback/cpumask + echo powersave | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor + sysctl vm.stat_interval=1 + sysctl -w kernel.watchdog=1 + echo "${materusArg.materusPC.allCoresMask}" > /proc/irq/default_smp_affinity + for irq in /proc/irq/[0-9]*/smp_affinity; do + if [ $(cat $irq) = "${materusArg.materusPC.hostCoresMask}" ] || [ $(cat $irq) = "${materusArg.materusPC.vmCoresMask}" ]; then + echo "${materusArg.materusPC.allCoresMask}" > $irq 2> /dev/null + fi; + done; ''; @@ -102,6 +141,10 @@ in ${startHook} fi + if [ ''$2 = "started" ] && [ ''$3 = "begin" ]; then + ${startedHook} + fi + if [ ''$2 = "release" ] && [ ''$3 = "end" ]; then ${stopHook} fi