nixos-config/configurations/host/materusPC/vm/win-vfio/default.nix

{ config, pkgs, materusArg, ... }:
let
  bar0_guest="15";
  bar2_guest="8";
  bar0_host="15";
  bar2_host="8";

  VM_UUID = "ad2632db-0da0-4204-98b3-0592a185ebd0";

  startedHook = ''
    QEMU_PID=$(ps aux | grep qemu-system-x86_64 | grep "${VM_UUID}" | tr -s ' ' | cut -d " " -f 2)

    for pid in $(cat /sys/fs/cgroup/cpu/machine.slice/machine-qemu*$1.scope/libvirt/vcpu*/tasks); do 
      renice -n "-15" -p "$pid";
    done
    renice -n "-10" -p "$QEMU_PID";

    echo "${materusArg.materusPC.hostCoresMask}" > /proc/irq/default_smp_affinity
    for irq in /proc/irq/[0-9]*/smp_affinity; do 
      if [ $(cat $irq) = "${materusArg.materusPC.allCoresMask}" ]; then
        echo "${materusArg.materusPC.hostCoresMask}" > $irq 2> /dev/null 
      fi;
    done;
    for irq in $(cat /proc/interrupts | grep vfio | cut -d ":" -f 1); do 
      echo "${materusArg.materusPC.vmCoresMask}" > /proc/irq/$irq/smp_affinity; 
    done


  '';
  startHook = /*''


    # Debugging
      exec 19>/home/materus/startlogfile
      BASH_XTRACEFD=19
      set -x

      exec 3>&1 4>&2
      trap 'exec 2>&4 1>&3' 0 1 2 3
      exec 1>/home/materus/startlogfile.out 2>&1
    ''
    +*/
    ''
      systemctl stop windows-share-mount.service

      # Make sure nothing renders on gpu to prevent "sysfs: cannot create duplicate filename" after rebinding to amdgpu
      chmod 0 /dev/dri/by-path/pci-$VIRSH_GPU_VIDEO-render 
      chmod 0 /dev/dri/by-path/pci-$VIRSH_GPU_VIDEO-card
      fuser -k /dev/dri/by-path/pci-$VIRSH_GPU_VIDEO-render
      pkill Xwayland

      # Seems to fix reset bug for 7900 XTX
      echo "0" > "/sys/bus/pci/devices/''${VIRSH_GPU_VIDEO}/d3cold_allowed"

      #####################################################################
      # Weird bug on kernel 6.7+, after changing bar sizes and binding to vfio driver, performance after returning to host will be lower than expected
      # binding to amdgpu after changing bar sizes and binding after it to vfio will work as expected.
      # I could skip changing bar sizes since I'm able to use full bar, but keeping it just in case
      echo ''$VIRSH_GPU_VIDEO > "/sys/bus/pci/devices/''${VIRSH_GPU_VIDEO}/driver/unbind"
      sleep 1s
      echo "${bar0_host}" > "/sys/bus/pci/devices/''${VIRSH_GPU_VIDEO}/resource0_resize"
      echo "${bar2_host}" > "/sys/bus/pci/devices/''${VIRSH_GPU_VIDEO}/resource2_resize"

      echo ''$VIRSH_GPU_VIDEO > /sys/bus/pci/drivers/amdgpu/bind
      
      sleep 1s

      chmod 0 /dev/dri/renderD128 
      fuser -k /dev/dri/renderD128
      #####################################################################
      
      echo ''$VIRSH_GPU_VIDEO > "/sys/bus/pci/devices/''${VIRSH_GPU_VIDEO}/driver/unbind"
      echo ''$VIRSH_GPU_AUDIO > "/sys/bus/pci/devices/''${VIRSH_GPU_AUDIO}/driver/unbind"

      echo "${bar0_guest}" > "/sys/bus/pci/devices/''${VIRSH_GPU_VIDEO}/resource0_resize"
      echo "${bar2_guest}" > "/sys/bus/pci/devices/''${VIRSH_GPU_VIDEO}/resource2_resize"

      sync
      echo "3" > /proc/sys/vm/drop_caches
      sync
      echo "1" > /proc/sys/vm/compact_memory
      

      

      systemctl set-property --runtime -- user.slice AllowedCPUs=${materusArg.materusPC.hostCores}
      systemctl set-property --runtime -- system.slice AllowedCPUs=${materusArg.materusPC.hostCores}
      systemctl set-property --runtime -- init.scope AllowedCPUs=${materusArg.materusPC.hostCores}
      echo "${materusArg.materusPC.hostCoresMask}" > /sys/bus/workqueue/devices/writeback/cpumask
      echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor

      sysctl vm.stat_interval=120
      sysctl -w kernel.watchdog=0


      


    '';
  stopHook = ''

    # Debugging
    #  exec 19>/home/materus/stoplogfile
    #  BASH_XTRACEFD=19
    #  set -x

    #  exec 3>&1 4>&2
    #  trap 'exec 2>&4 1>&3' 0 1 2 3
    #  exec 1>/home/materus/stoplogfile.out 2>&1
    echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor

    sysctl vm.stat_interval=1
    sysctl -w kernel.watchdog=1
    echo "${materusArg.materusPC.allCoresMask}" > /proc/irq/default_smp_affinity
    for irq in /proc/irq/[0-9]*/smp_affinity; do 
      if [ $(cat $irq) = "${materusArg.materusPC.hostCoresMask}" ] || [ $(cat $irq) = "${materusArg.materusPC.vmCoresMask}" ]; then
        echo "${materusArg.materusPC.allCoresMask}" > $irq 2> /dev/null 
      fi;
    done;

        
    sleep 1s
    echo ''$VIRSH_GPU_VIDEO > "/sys/bus/pci/devices/''${VIRSH_GPU_VIDEO}/driver/unbind"
    echo ''$VIRSH_GPU_AUDIO > "/sys/bus/pci/devices/''${VIRSH_GPU_AUDIO}/driver/unbind"

        
    

    echo "${bar0_host}" > "/sys/bus/pci/devices/''${VIRSH_GPU_VIDEO}/resource0_resize"
    echo "${bar2_host}" > "/sys/bus/pci/devices/''${VIRSH_GPU_VIDEO}/resource2_resize"
    echo "1" > "/sys/bus/pci/devices/''${VIRSH_GPU_VIDEO}/d3cold_allowed"


    echo ''$VIRSH_GPU_VIDEO > /sys/bus/pci/drivers/amdgpu/bind
    echo ''$VIRSH_GPU_AUDIO > /sys/bus/pci/drivers/snd_hda_intel/bind
    

    systemctl start windows-share-mount.service

    systemctl set-property --runtime -- user.slice AllowedCPUs=${materusArg.materusPC.allCores}
    systemctl set-property --runtime -- system.slice AllowedCPUs=${materusArg.materusPC.allCores}
    systemctl set-property --runtime -- init.scope AllowedCPUs=${materusArg.materusPC.allCores}
    echo "${materusArg.materusPC.allCoresMask}" > /sys/bus/workqueue/devices/writeback/cpumask

  '';
in
{








  virtualisation.libvirtd.hooks.qemu = {
    "windows-vfio" = pkgs.writeShellScript "windows.sh" ''
      VIRSH_GPU_VIDEO="0000:03:00.0"
      VIRSH_GPU_AUDIO="0000:03:00.1"
      VIRSH_USB1="0000:10:00.0"

      if [ ''$1 = "windows-vfio" ]; then
        if [ ''$2 = "prepare" ] && [ ''$3 = "begin" ]; then
          ${startHook}
        fi

        if [ ''$2 = "started" ] && [ ''$3 = "begin" ]; then
          ${startedHook}
        fi

        if [ ''$2 = "release" ] && [ ''$3 = "end" ]; then
          ${stopHook}
        fi

      fi

    
    '';
  };

  systemd.services.windows-share-mount = {
    wantedBy = [ "multi-user.target" ];
    path = [ config.virtualisation.libvirtd.qemu.package pkgs.util-linux pkgs.kmod pkgs.coreutils ];
    serviceConfig.Type = "oneshot";
    serviceConfig.RemainAfterExit = true;
    script = ''
      modprobe nbd max_part=16
      sleep 1
      qemu-nbd -c /dev/nbd10 /materus/data/VM/data.qcow2 --discard=unmap
      sleep 1
      mount /dev/nbd10p1 /materus/data/Windows -o uid=1000,gid=100
    '';
    preStop = ''
      umount -r /dev/nbd10p1
      qemu-nbd -d /dev/nbd10
    '';
  };
}