materusPC: more vm tuning

This commit is contained in:
Mateusz Słodkowicz 2024-04-22 15:33:31 +02:00
parent c6cdd9d8af
commit 400dadba7f
Signed by: materus
GPG Key ID: 28D140BCA60B4FD1
3 changed files with 75 additions and 24 deletions

View File

@ -1,4 +1,4 @@
{ config, pkgs, lib, ... }:
{ config, pkgs, lib, materusArg, ... }:
let
video = [
@ -17,8 +17,8 @@ in
{
#Kernel
boot.kernelPackages = pkgs.linuxPackages_zen;
boot.kernelParams = [ "rcu_nocbs=8-15,24-31" "nohz_full=8-15,24-31" "vfio_iommu_type1.allow_unsafe_interrupts=1" "pcie_acs_override=downstream,multifunction" /*"pci-stub.ids=1002:744c"*/ "nox2apic" "nvme_core.default_ps_max_latency_us=0" "nvme_core.io_timeout=255" "nvme_core.max_retries=10" "nvme_core.shutdown_timeout=10" "amd_iommu=on" "iommu=pt" ] ++ video;
boot.kernelModules = [ "pci-stub" "amdgpu" "i2c_dev" "kvm_amd" "vfio" "vfio_iommu_type1" "vfio-pci"];
boot.kernelParams = [ "rcu_nocbs=${materusArg.materusPC.vmCores}" "nohz_full=${materusArg.materusPC.vmCores}" "vfio_iommu_type1.allow_unsafe_interrupts=1" "pcie_acs_override=downstream,multifunction" /*"pci-stub.ids=1002:744c"*/ "nox2apic" "nvme_core.default_ps_max_latency_us=0" "nvme_core.io_timeout=255" "nvme_core.max_retries=10" "nvme_core.shutdown_timeout=10" "amd_iommu=on" "iommu=pt" ] ++ video;
boot.kernelModules = [ "pci-stub" "amdgpu" "i2c_dev" "kvm_amd" "vfio" "vfio_iommu_type1" "vfio-pci" ];
boot.extraModprobeConfig = ''
options kvm_amd nested=1 avic=1 npt=1 sev=0
options vfio_iommu_type1 allow_unsafe_interrupts=1

View File

@ -3,7 +3,14 @@
imports = [
./win10
];
materus.materusArg.materusPC = {
allCores = "0-31";
allCoresMask = "ffffffff";
hostCores = "0-7,16-23";
hostCoresMask = "00ff00ff";
vmCores = "8-15,24-31";
vmCoresMask = "ff00ff00";
};
systemd.mounts = [
{
where = "/dev/hugepages";
@ -69,6 +76,7 @@
bindfs
qemu-utils
psmisc
procps
];
};
in

View File

@ -1,5 +1,27 @@
{ config, pkgs, ... }:
{ config, pkgs, materusArg, ... }:
let
VM_UUID = "ad2632db-0da0-4204-98b3-0592a185ebd0";
startedHook = ''
QEMU_PID=$(ps aux | grep qemu-system-x86_64 | grep "${VM_UUID}" | tr -s ' ' | cut -d " " -f 2)
for pid in $(cat /sys/fs/cgroup/cpu/machine.slice/machine-qemu*$1.scope/libvirt/vcpu*/tasks); do
renice -n "-15" -p "$pid";
done
renice -n "-10" -p "$QEMU_PID";
echo "${materusArg.materusPC.hostCoresMask}" > /proc/irq/default_smp_affinity
for irq in /proc/irq/[0-9]*/smp_affinity; do
if [ $(cat $irq) = "${materusArg.materusPC.allCoresMask}" ]; then
echo "${materusArg.materusPC.hostCoresMask}" > $irq 2> /dev/null
fi;
done;
for irq in $(cat /proc/interrupts | grep vfio | cut -d ":" -f 1); do
echo "${materusArg.materusPC.vmCoresMask}" > /proc/irq/$irq/smp_affinity;
done
'';
startHook = /*''
@ -14,32 +36,39 @@ let
''
+*/
''
# Make sure nothing renders on gpu to prevent "sysfs: cannot create duplicate filename" after rebinding to amdgpu
chmod 0 /dev/dri/renderD128
fuser -k /dev/dri/renderD128
# Make sure nothing renders on gpu to prevent "sysfs: cannot create duplicate filename" after rebinding to amdgpu
chmod 0 /dev/dri/renderD128
fuser -k /dev/dri/renderD128
# Seems to fix reset bug for 7900 XTX
echo "0" > "/sys/bus/pci/devices/''${VIRSH_GPU_VIDEO}/d3cold_allowed"
# Seems to fix reset bug for 7900 XTX
echo "0" > "/sys/bus/pci/devices/''${VIRSH_GPU_VIDEO}/d3cold_allowed"
systemctl stop windows-share-mount.service
systemctl stop windows-share-mount.service
echo ''$VIRSH_GPU_VIDEO > "/sys/bus/pci/devices/''${VIRSH_GPU_VIDEO}/driver/unbind"
echo ''$VIRSH_GPU_AUDIO > "/sys/bus/pci/devices/''${VIRSH_GPU_AUDIO}/driver/unbind"
echo ''$VIRSH_GPU_VIDEO > "/sys/bus/pci/devices/''${VIRSH_GPU_VIDEO}/driver/unbind"
echo ''$VIRSH_GPU_AUDIO > "/sys/bus/pci/devices/''${VIRSH_GPU_AUDIO}/driver/unbind"
sleep 1s
sleep 1s
echo "15" > "/sys/bus/pci/devices/''${VIRSH_GPU_VIDEO}/resource0_resize"
echo "8" > "/sys/bus/pci/devices/''${VIRSH_GPU_VIDEO}/resource2_resize"
echo "15" > "/sys/bus/pci/devices/''${VIRSH_GPU_VIDEO}/resource0_resize"
echo "8" > "/sys/bus/pci/devices/''${VIRSH_GPU_VIDEO}/resource2_resize"
echo "3" > /proc/sys/vm/drop_caches
echo "1" > /proc/sys/vm/compact_memory
sync
echo "3" > /proc/sys/vm/drop_caches
sync
echo "1" > /proc/sys/vm/compact_memory
systemctl set-property --runtime -- user.slice AllowedCPUs=0-7,16-23
systemctl set-property --runtime -- system.slice AllowedCPUs=0-7,16-23
systemctl set-property --runtime -- init.scope AllowedCPUs=0-7,16-23
systemctl set-property --runtime -- user.slice AllowedCPUs=${materusArg.materusPC.hostCores}
systemctl set-property --runtime -- system.slice AllowedCPUs=${materusArg.materusPC.hostCores}
systemctl set-property --runtime -- init.scope AllowedCPUs=${materusArg.materusPC.hostCores}
echo "${materusArg.materusPC.hostCoresMask}" > /sys/bus/workqueue/devices/writeback/cpumask
echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor
sysctl vm.stat_interval=120
sysctl -w kernel.watchdog=0
'';
@ -74,10 +103,20 @@ let
systemctl start windows-share-mount.service
systemctl set-property --runtime -- user.slice AllowedCPUs=0-31
systemctl set-property --runtime -- system.slice AllowedCPUs=0-31
systemctl set-property --runtime -- init.scope AllowedCPUs=0-31
systemctl set-property --runtime -- user.slice AllowedCPUs=${materusArg.materusPC.allCores}
systemctl set-property --runtime -- system.slice AllowedCPUs=${materusArg.materusPC.allCores}
systemctl set-property --runtime -- init.scope AllowedCPUs=${materusArg.materusPC.allCores}
echo "${materusArg.materusPC.allCoresMask}" > /sys/bus/workqueue/devices/writeback/cpumask
echo powersave | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor
sysctl vm.stat_interval=1
sysctl -w kernel.watchdog=1
echo "${materusArg.materusPC.allCoresMask}" > /proc/irq/default_smp_affinity
for irq in /proc/irq/[0-9]*/smp_affinity; do
if [ $(cat $irq) = "${materusArg.materusPC.hostCoresMask}" ] || [ $(cat $irq) = "${materusArg.materusPC.vmCoresMask}" ]; then
echo "${materusArg.materusPC.allCoresMask}" > $irq 2> /dev/null
fi;
done;
'';
@ -102,6 +141,10 @@ in
${startHook}
fi
if [ ''$2 = "started" ] && [ ''$3 = "begin" ]; then
${startedHook}
fi
if [ ''$2 = "release" ] && [ ''$3 = "end" ]; then
${stopHook}
fi