diff --git a/modules/nixos/cspark-single-gpu-passthru/module.nix b/modules/nixos/cspark-single-gpu-passthru/module.nix new file mode 100644 index 00000000..61a9673e --- /dev/null +++ b/modules/nixos/cspark-single-gpu-passthru/module.nix @@ -0,0 +1,112 @@ +{ lib, config, pkgs, modulesPath, ... }: + +let + cfg = config.cspark-single-gpu-passthru; +in +{ + options.cspark-single-gpu-passthru = { + enable + = lib.mkEnableOption "Enable Single GPU Passthrough Setup"; + + vmName = lib.mkOption { + default = "win11"; + description = '' + Name of the virtual machine + ''; + }; + + gpuPCI = lib.mkOption { + default = "0000:08:00.0"; + description = '' + PCI ID of GPU VGA + ''; + }; + + gpuAudioPCI = lib.mkOption { + default = "0000:08:00.1"; + description = '' + PCI ID of GPU Audio + ''; + }; + }; + + config = lib.mkIf cfg.enable { + # Virtualisation, Virt Manager and Waydroid + virtualisation.libvirtd = { + enable = true; + qemu = { + package = pkgs.qemu_kvm; + runAsRoot = true; + swtpm.enable = true; + ovmf = { + enable = true; + packages = [(pkgs.OVMF.override { + secureBoot = true; + tpmSupport = true; + }).fd]; + }; + }; + }; + programs.virt-manager.enable = true; + systemd.services.libvirtd = { + # Add binaries to path so that hooks can use it + path = let + env = pkgs.buildEnv { + name = "qemu-hook-env"; + paths = with pkgs; [ + bash + libvirt + kmod + systemd + ripgrep + sd + procps + gawk + ]; + }; + in + [ env ]; + + preStart = let + qemuscript_source = builtins.readFile ./resources/libvirt/hooks/qemu; + startscript_source = builtins.readFile ./resources/libvirt/hooks/qemu.d/vm/prepare/begin/start.sh; + revertscript_source = builtins.readFile ./resources/libvirt/hooks/qemu.d/vm/release/end/revert.sh; + qemuscript = pkgs.writeShellScript "cspark-single-gpu-passthru-qemu" qemuscript_source; + startscript = pkgs.writeShellScript "cspark-single-gpu-passthru-start.sh" startscript_source; + revertscript = pkgs.writeShellScript "cspark-single-gpu-passthru-revert.sh" revertscript_source; + gpuvbios = ./resources/libvirt/vgabios/gpuvbios.rom; + in + '' + mkdir -p /var/lib/libvirt/hooks + mkdir -p /var/lib/libvirt/hooks/qemu.d/${cfg.vmName}/prepare/begin + mkdir -p /var/lib/libvirt/hooks/qemu.d/${cfg.vmName}/release/end + mkdir -p /var/lib/libvirt/vgabios + + ln -sf ${qemuscript} /var/lib/libvirt/hooks/qemu + ln -sf ${startscript} /var/lib/libvirt/hooks/qemu.d/${cfg.vmName}/prepare/begin/start.sh + ln -sf ${revertscript} /var/lib/libvirt/hooks/qemu.d/${cfg.vmName}/release/end/revert.sh + ln -sf ${gpuvbios} /var/lib/libvirt/vgabios/gpuvbios.rom + +# chmod +x /var/lib/libvirt/hooks/qemu +# chmod +x /var/lib/libvirt/hooks/qemu.d/${cfg.vmName}/prepare/begin/start.sh +# chmod +x /var/lib/libvirt/hooks/qemu.d/${cfg.vmName}/release/end/revert.sh + ''; + }; + # Vendor Reset Patches for Single GPU Passthrough - https://github.com/gnif/vendor-reset + boot.extraModulePackages = with config.boot.kernelPackages; [ vendor-reset ]; + # Ensures GPU is configured to use vendor reset module + systemd.services.vendor-reset-config = { + description = "Ensures GPU is configured to use vendor reset module"; + wantedBy = [ "multi-user.target" ]; + path = [ pkgs.coreutils ]; + enable = true; + serviceConfig = { + User = "root"; + Group = "root"; + }; + script = ''echo device_specific > "/sys/bus/pci/devices/${cfg.gpuPCI}/reset_method"''; + }; + hardware.opengl.enable = true; + virtualisation.spiceUSBRedirection.enable = true; + }; +} diff --git a/modules/nixos/cspark-single-gpu-passthru/resources/libvirt/bootstrap.sh b/modules/nixos/cspark-single-gpu-passthru/resources/libvirt/bootstrap.sh new file mode 100755 index 00000000..a3b454bb --- /dev/null +++ b/modules/nixos/cspark-single-gpu-passthru/resources/libvirt/bootstrap.sh @@ -0,0 +1,13 @@ +sudo cp ~/libvirt/hooks/qemu /var/lib/libvirt/hooks/qemu +sudo chmod +x /var/lib/libvirt/hooks/qemu + +sudo mkdir -p /var/lib/libvirt/hooks/qemu.d/win11/prepare/begin +sudo cp ~/libvirt/hooks/qemu.d/win11/prepare/begin/start.sh /var/lib/libvirt/hooks/qemu.d/win11/prepare/begin/start.sh +sudo chmod +x /var/lib/libvirt/hooks/qemu.d/win11/prepare/begin/start.sh + +sudo mkdir -p /var/lib/libvirt/hooks/qemu.d/win11/release/end +sudo cp ~/libvirt/hooks/qemu.d/win11/release/end/stop.sh /var/lib/libvirt/hooks/qemu.d/win11/release/end/stop.sh +sudo chmod +x /var/lib/libvirt/hooks/qemu.d/win11/release/end/stop.sh + +sudo mkdir -p /var/lib/libvirt/vgabios +sudo cp ~/libvirt/vgabios/gpuvbios.rom /var/lib/libvirt/vgabios/gpuvbios.rom diff --git a/modules/nixos/cspark-single-gpu-passthru/resources/libvirt/hooks/qemu b/modules/nixos/cspark-single-gpu-passthru/resources/libvirt/hooks/qemu new file mode 100755 index 00000000..cbcd7c59 --- /dev/null +++ b/modules/nixos/cspark-single-gpu-passthru/resources/libvirt/hooks/qemu @@ -0,0 +1,17 @@ +GUEST_NAME="$1" +HOOK_NAME="$2" +STATE_NAME="$3" +MISC="${@:4}" + +BASEDIR="$(dirname $0)" + +HOOKPATH="$BASEDIR/qemu.d/$GUEST_NAME/$HOOK_NAME/$STATE_NAME" +set -e # If a script exits with an error, we should as well. + +if [ -f "$HOOKPATH" ]; then +eval \""$HOOKPATH"\" "$@" +elif [ -d "$HOOKPATH" ]; then +while read file; do + eval \""$file"\" "$@" +done <<< "$(find -L "$HOOKPATH" -maxdepth 1 -type f -executable -print;)" +fi diff --git a/modules/nixos/cspark-single-gpu-passthru/resources/libvirt/hooks/qemu.d/vm/prepare/begin/start.sh b/modules/nixos/cspark-single-gpu-passthru/resources/libvirt/hooks/qemu.d/vm/prepare/begin/start.sh new file mode 100755 index 00000000..dac34249 --- /dev/null +++ b/modules/nixos/cspark-single-gpu-passthru/resources/libvirt/hooks/qemu.d/vm/prepare/begin/start.sh @@ -0,0 +1,25 @@ +set -x + +# Stop desktop environment and display manager +pkill kwin_wayland +systemctl --user -M cspark stop plasma* +systemctl stop display-manager +pkill sddm +pkill sddm-helper + +# Log out of session +session=`loginctl session-status | awk 'NR==1{print $1}'` +loginctl terminate-session $session + +# My audio interface has issues after coming out from a suspended type state, this will reset it to fix it when necessary. +# Unbind audio interface +echo `grep 0a73 /sys/bus/usb/devices/*/idVendor | cut -d '/' -f 6` > /sys/bus/usb/drivers/usb/unbind +# Remove snd_usb_audio once no longer in use +while ! modprobe -r snd_usb_audio; do + sleep 1 +done + +# Remove amdgpu once no longer in use +while ! modprobe -r amdgpu; do + sleep 1 +done diff --git a/modules/nixos/cspark-single-gpu-passthru/resources/libvirt/hooks/qemu.d/vm/release/end/revert.sh b/modules/nixos/cspark-single-gpu-passthru/resources/libvirt/hooks/qemu.d/vm/release/end/revert.sh new file mode 100755 index 00000000..fa5e9139 --- /dev/null +++ b/modules/nixos/cspark-single-gpu-passthru/resources/libvirt/hooks/qemu.d/vm/release/end/revert.sh @@ -0,0 +1,19 @@ +set -x + +# My audio interface has issues after coming out from a suspended type state, this will reset it to fix it when necessary. +modprobe snd_usb_audio +# Rebind audio interface +echo `grep 0a73 /sys/bus/usb/devices/*/idVendor | cut -d '/' -f 6` > /sys/bus/usb/drivers/usb/bind + +# Remove vfio modules +modprobe -r vfio_pci +modprobe -r vfio + +# Re-add amdgpu +modprobe amdgpu + +# Avoid race condition whilst re-initalising amdgpu driver +sleep 5 + +# Restart Display Manager +systemctl start display-manager diff --git a/modules/nixos/cspark-single-gpu-passthru/resources/libvirt/vgabios/gpuvbios.rom b/modules/nixos/cspark-single-gpu-passthru/resources/libvirt/vgabios/gpuvbios.rom new file mode 100644 index 00000000..e1391189 Binary files /dev/null and b/modules/nixos/cspark-single-gpu-passthru/resources/libvirt/vgabios/gpuvbios.rom differ