From 11f1c043dcc6448a0d514760885fed4d357f6f9c Mon Sep 17 00:00:00 2001 From: kat witch Date: Thu, 5 Aug 2021 16:09:17 +0100 Subject: [PATCH] VFIO Profile --- config/modules/nixos/default.nix | 2 +- config/profiles/vfio/default.nix | 77 ++++++++++++++++++++++++- config/profiles/vfio/screenstub.yml | 85 ++++++++++++++++++++++++++++ pkgs/default.nix | 2 + pkgs/kat-vm/default.nix | 88 +++++++++++++++++++++++++++++ 5 files changed, 252 insertions(+), 2 deletions(-) create mode 100644 config/profiles/vfio/screenstub.yml create mode 100644 pkgs/kat-vm/default.nix diff --git a/config/modules/nixos/default.nix b/config/modules/nixos/default.nix index 6958bb71..603b387c 100644 --- a/config/modules/nixos/default.nix +++ b/config/modules/nixos/default.nix @@ -1,7 +1,7 @@ { meta, sources, lib, ... }: { - imports = with (import (sources.nixexprs + "/modules")).nixos; [ base16 base16-shared ] ++ [ + imports = with (import (sources.nixexprs + "/modules")).nixos; [ base16 base16-shared modprobe ] ++ [ ./nftables.nix ./firewall.nix ./deploy.nix diff --git a/config/profiles/vfio/default.nix b/config/profiles/vfio/default.nix index f121b322..6ae0eb05 100644 --- a/config/profiles/vfio/default.nix +++ b/config/profiles/vfio/default.nix @@ -1,9 +1,84 @@ -{ config, pkgs, ... }: +{ config, pkgs, lib, ... }: + +with lib; { deploy.profile.vfio = true; environment.systemPackages = with pkgs; [ screenstub + kat-vm + ddcutil ]; + + boot.extraModulePackages = [ + (pkgs.linuxPackagesFor config.boot.kernelPackages.kernel).vendor-reset + ]; + + users.users.kat.extraGroups = [ "vfio" "input" "uinput" ]; + users.groups = { uinput = { }; vfio = { }; }; + + boot = { + modprobe.modules = mkIf (config.deploy.profile.hardware.ms-7b86) { + vfio-pci = let + vfio-pci-ids = [ + "1002:67df" "1002:aaf0" # RX 580 + "1921:0014" # Renesas USB 3 + "1022:149c" # CPU USB 3 + ]; + in mkIf (vfio-pci-ids != [ ]) { + options.ids = concatStringsSep "," vfio-pci-ids; + }; + }; + kernelPatches = mkIf (config.deploy.profile.hardware.ms-7b86) [ + { + name = "acs-patch.patch"; + patch = (pkgs.fetchpatch { + name = "acs-patch.patch"; + url = "https://gitlab.com/Queuecumber/linux-acs-override/-/raw/cc30d83d05019b84468db3ad009bc02ab1bab240/workspaces/5.6.12/acso.patch?inline=false"; + sha256 = "0qjb66ydbqqypyvhhlq8zwry8zcd8609y8d4a0nidhq1g6cp9vcw"; + }); + } + ]; + initrd.kernelModules = ["vfio" "vfio_iommu_type1" "vfio_pci" "vfio_virqfd"]; + kernelModules = [ "i2c-dev" ]; # i2c-dev is required for DDC/CI for screenstub + kernelParams = [ + "amd_iommu=on" + "pci=noats" + "video=efifb:off" + "pcie_acs_override=downstream,multifunction" + ]; + }; + + environment.etc."qemu/bridge.conf".text = "allow br"; + + security.wrappers = { + qemu-bridge-helper = { + source = "${pkgs.qemu-vfio}/libexec/qemu-bridge-helper"; + }; + }; + + services.udev.extraRules = '' + SUBSYSTEM=="i2c-dev", GROUP="vfio", MODE="0660" + SUBSYSTEM=="usb", ACTION=="add", ATTRS{idVendor}=="1532", ATTRS{idProduct}=="0067", GROUP="vfio" + SUBSYSTEM=="misc", KERNEL=="uinput", OPTIONS+="static_node=uinput", MODE="0660", GROUP="uinput" + SUBSYSTEM=="vfio", OWNER="root", GROUP="vfio" + SUBSYSTEM=="block", ACTION=="add", ATTRS{model}=="HFS256G32TNF-N3A", ATTRS{wwid}=="t10.ATA HFS256G32TNF-N3A0A MJ8BN15091150BM1Z ", OWNER="kat" + ''; + + # TODO: Replace this drive forward with one half of the 1.82TiB drive. + # SUBSYSTEM=="block", ACTION=="add", ATTR{partition}=="2", ATTR{size}=="1953503232", ATTRS{wwid}=="naa.5000039fe6e8614e", OWNER="kat" + + security.pam.loginLimits = [{ + domain = "@vfio"; + type = "-"; + item = "memlock"; + value = "unlimited"; + }]; + + systemd.extraConfig = "DefaultLimitMEMLOCK=infinity"; + services.xserver.deviceSection = lib.mkDefault '' + Option "TearFree" "true" + BusID "PCI:37:0:0" + ''; } diff --git a/config/profiles/vfio/screenstub.yml b/config/profiles/vfio/screenstub.yml new file mode 100644 index 00000000..741168f2 --- /dev/null +++ b/config/profiles/vfio/screenstub.yml @@ -0,0 +1,85 @@ +screens: +- monitor: # fill in with info from `screenstub detect` + manufacturer: BNQ + model: BenQ GW2270 + #serial: "..." + guest_source: # Could be automatically detected, but best to fill in if monitor has more than two inputs + name: HDMI-1 + #value: 0x0f # can also specify raw VCP value + host_source: # Usually automatically detected + name: HDMI-2 + #value: 0x11 + ddc: + #minimal_delay: 100ms # minimum time to wait between switching inputs again + #guest: [] # disable input switching + #host: [] # disable input switching + guest: # configure how to switch to the guest + #- guest_wait # wait until guest agent responds, otherwise might get stranded on other input + - ddc # (default) Use ddc-rs + #exec: ["ddcutil", "-g", "BNQ", "setvcp", "0x60", "{}"] + #- exec: [ddccontrol, -r, "0x60", -w, "{}", /dev/i2c-5] + host: # configure how to switch back from the guest + - ddc # (default) Controls DDC from the host GPU - requires no guest agent but many monitors won't support this + #- exec: ["ddcutil", "-g", "BNQ", "setvcp", "0x60", "{}"] + #- guest_exec: ["C:/ddcset.exe", "setvcp", "60", "{:x}"] # or "0x{:x}" for hex input value + #- guest_exec: ["C:/ScreenBright.exe", "-set", "0x60", "{}"] # "{}" is for decimal input value + #- exec: ["ssh", "user@vm", "ddcutil", "setvcp", "0x60", "{}"] # system commands can also be used + +qemu: + #routing: qmp # (default) does not require extra configuration or dependencies + #routing: spice # no external requirements # CURRENTLY UNIMPLEMENTED + #routing: input-linux # requires uinput + routing: virtio-host # requires uinput, recommended for performance, requires vioinput drivers in guest + #driver: ps2 # use PS/2 in the guest for all input devices (absolute mouse mode unsupported) + #driver: usb # use USB keyboard/mouse/tablet in the guest + #driver: virtio # Recommended but vioinput drivers must be installed in guest + #keyboard_driver: ps2 # (default) can also be set separately per input type, this should rarely be necessary + #relative_driver: usb # (default) + #absolute_driver: usb # (default) + #driver: virtio + #relative_driver: virtio + qmp_socket: /tmp/vfio-qmp # path to QMP socket + ga_socket: /tmp/vfio-qga # path to Guest Agent socket + +key_remap: # Arbitrary keys can be remapped in the guest + # See https://docs.rs/input-linux/*/input_linux/enum.Key.html for a list of key names available (mouse buttons can also be used) + LeftMeta: Reserved # disable the windows key + RightAlt: LeftMeta # remap right alt to trigger the windows key + +hotkeys: # Trigger various events on key combinations + #- toggle_grab:y + # xdevice: # CURRENTLY UNIMPLEMENTED + # devices: # Only grab specific devices from Xorg + # - "..." + #- exec: [echo, hi] # Execute an arbitrary system command + #- show_host # switch to the host display + #- show_guest # switch to the guest display + #- toggle_show # switch the current display + #- unstick_guest # causes all held keys to be released in the guest + #- shutdown # safely shuts the guest system down + #- reboot # reboots the guest + #- exit # quits screenstub +- triggers: [G] + modifiers: [LeftMeta] + events: + - toggle_grab: + x: + mouse: false # Confine input/mouse to window + - toggle_grab: + evdev: # evdev grab is useful for playing games that don't work with absolute mouse events + exclusive: false # grab exclusive access from the device(s) + #new_device_name: "unique-grab-name" # create a new uinput device for this grab + xcore_ignore: [absolute] # which events to ignore from the window (key, button, absolute) + evdev_ignore: [button] # which events to ignore from the evdev device + devices: # List of devices to forward to guest + - /dev/input/by-id/usb-Razer_Razer_Naga_Trinity_00000000001A-event-mouse + - unstick_host # force-depress all Xorg keys (prevents keys getting stuck) +- triggers: [T] + modifiers: [LeftMeta] + on_release: false + events: + - toggle_show + +exit_events: # Events to trigger on window close / exit +- show_host +#- shutdown diff --git a/pkgs/default.nix b/pkgs/default.nix index 726dd728..1ec0bdda 100644 --- a/pkgs/default.nix +++ b/pkgs/default.nix @@ -88,6 +88,8 @@ let kat-tw-export = self.callPackage ./kat-tw-export { }; + kat-vm = self.callPackage ./kat-vm { } ; + kat-scrot = self.callPackage ./kat-scrot { }; } // super.lib.optionalAttrs (builtins.pathExists ../config/trusted/pkgs) diff --git a/pkgs/kat-vm/default.nix b/pkgs/kat-vm/default.nix new file mode 100644 index 00000000..bc69e385 --- /dev/null +++ b/pkgs/kat-vm/default.nix @@ -0,0 +1,88 @@ +{ pkgs, writeShellScriptBin }: + +writeShellScriptBin "kat-vm" '' + cat ${pkgs.OVMF.fd}/FV/OVMF_VARS.fd > /tmp/OVMF_VARS.fd + ${pkgs.qemu-vfio}/bin/qemu-system-x86_64 -name guest=win10,debug-threads=on \ + -blockdev '{"driver":"file","filename":"${pkgs.OVMF.fd}/FV/OVMF_CODE.fd","node-name":"libvirt-pflash0-storage","auto-read-only":true,"discard":"unmap"}' \ + -blockdev '{"node-name":"libvirt-pflash0-format","read-only":true,"driver":"raw","file":"libvirt-pflash0-storage"}' \ + -blockdev '{"driver":"file","filename":"/tmp/OVMF_VARS.fd","node-name":"libvirt-pflash1-storage","auto-read-only":true,"discard":"unmap"}' \ + -blockdev '{"node-name":"libvirt-pflash1-format","read-only":false,"driver":"raw","file":"libvirt-pflash1-storage"}' \ + -machine pc-q35-5.1,accel=kvm,usb=off,vmport=off,dump-guest-core=off,pflash0=libvirt-pflash0-format,pflash1=libvirt-pflash1-format,memory-backend=pc.ram \ + -monitor stdio \ + -cpu host,migratable=on,topoext=on,hv-time,hv-relaxed,hv-vapic,hv-spinlocks=0x1fff,host-cache-info=on,l3-cache=off -m 12288 \ + -vcpu vcpunum=0,affinity=0 \ + -vcpu vcpunum=1,affinity=1 \ + -vcpu vcpunum=2,affinity=2 \ + -vcpu vcpunum=3,affinity=3 \ + -vcpu vcpunum=4,affinity=6 \ + -vcpu vcpunum=5,affinity=7 \ + -vcpu vcpunum=6,affinity=8 \ + -vcpu vcpunum=7,affinity=9 \ + -object memory-backend-ram,id=pc.ram,size=12884901888 -overcommit mem-lock=off \ + -smp 8,sockets=1,dies=1,cores=4,threads=2 \ + -object iothread,id=iothread1 -uuid 96052919-6a83-4e9f-8e9b-628de3e27cc1 \ + -display none \ + -no-user-config \ + -nodefaults \ + -rtc base=localtime,driftfix=slew -global kvm-pit.lost_tick_policy=delay \ + -no-hpet -no-shutdown \ + -global ICH9-LPC.disable_s3=1 \ + -global ICH9-LPC.disable_s4=1 \ + -boot strict=on \ + -device pcie-root-port,port=0x10,chassis=1,id=pci.1,bus=pcie.0,multifunction=on,addr=0x2 \ + -device pcie-root-port,port=0x11,chassis=2,id=pci.2,bus=pcie.0,addr=0x2.0x1 \ + -device pcie-root-port,port=0x12,chassis=3,id=pci.3,bus=pcie.0,addr=0x2.0x2 \ + -device pcie-root-port,port=0x13,chassis=4,id=pci.4,bus=pcie.0,addr=0x2.0x3 \ + -device pcie-root-port,port=0x14,chassis=5,id=pci.5,bus=pcie.0,addr=0x2.0x4 \ + -device pcie-root-port,port=0x15,chassis=6,id=pci.6,bus=pcie.0,addr=0x2.0x5 \ + -device pcie-root-port,port=0x16,chassis=7,id=pci.7,bus=pcie.0,addr=0x2.0x6 \ + -device pcie-pci-bridge,id=pci.8,bus=pci.4,addr=0x0 \ + -device pcie-root-port,port=0x17,chassis=9,id=pci.9,bus=pcie.0,addr=0x2.0x7 \ + -device pcie-root-port,port=0x8,chassis=10,id=pci.10,bus=pcie.0,multifunction=on,addr=0x1 \ + -device pcie-root-port,port=0xa,chassis=11,id=pci.11,bus=pcie.0,addr=0x1.0x1 \ + -device pcie-root-port,port=0xa,chassis=12,id=pci.12,bus=pcie.0,addr=0x1.0x2 \ + -device pcie-root-port,port=0xb,chassis=13,id=pci.13,bus=pcie.0,addr=0x1.0x3 \ + -device pcie-root-port,port=0xc,chassis=14,id=pci.14,bus=pcie.0,addr=0x1.0x4 \ + -device pcie-root-port,port=0xd,chassis=15,id=pci.15,bus=pcie.0,addr=0x1.0x5 \ + -device pcie-root-port,port=0xe,chassis=16,id=pci.16,bus=pcie.0,addr=0x1.0x6 \ + -device pcie-root-port,port=0xf,chassis=17,id=pci.17,bus=pcie.0,addr=0x1.0x7 \ + -device pcie-root-port,port=0x18,chassis=18,id=pci.18,bus=pcie.0,multifunction=on,addr=0x3 \ + -device pcie-root-port,port=0x19,chassis=19,id=pci.19,bus=pcie.0,addr=0x3.0x1 \ + -device pcie-root-port,port=0x1a,chassis=20,id=pci.20,bus=pcie.0,addr=0x3.0x2 \ + -device pcie-root-port,port=0x1b,chassis=21,id=pci.21,bus=pcie.0,addr=0x3.0x3 \ + -device pcie-root-port,port=0x1c,chassis=22,id=pci.22,bus=pcie.0,addr=0x3.0x4 \ + -device pcie-root-port,port=0x1d,chassis=23,id=pci.23,bus=pcie.0,multifunction=on,addr=0x3.0x5 \ + -device pcie-pci-bridge,id=pci.24,bus=pci.10,addr=0x0 \ + -device ich9-usb-ehci1,id=usb -device ich9-usb-uhci1,masterbus=usb.0,firstport=0,multifunction=on -device ich9-usb-uhci2,masterbus=usb.0,firstport=2 -device ich9-usb-uhci3,masterbus=usb.0,firstport=4 \ + -device qemu-xhci,id=usb3,p2=4,p3=8 \ + -device virtio-scsi-pci,id=scsi0,bus=pci.6,addr=0x0 \ + -device virtio-serial-pci,id=virtio-serial0,bus=pci.3,addr=0x0 \ + -device ich9-intel-hda,id=sound0 \ + -device hda-output,audiodev=pa1 \ + -device hda-micro,audiodev=pa1 \ + -audiodev pa,id=pa1,server=/run/user/1000/pulse/native,out.buffer-length=4000,timer-period=1000 \ + -blockdev '{"driver":"host_device","filename":"/dev/disk/by-id/ata-HFS256G32TNF-N3A0A_MJ8BN15091150BM1Z","node-name":"libvirt-2-storage","auto-read-only":true,"discard":"unmap"}' \ + -blockdev '{"node-name":"libvirt-2-format","read-only":false,"discard":"unmap","driver":"raw","file":"libvirt-2-storage"}' \ + -device scsi-hd,bus=scsi0.0,channel=0,scsi-id=0,lun=0,device_id=drive-scsi0-0-0-0,drive=libvirt-2-format,id=scsi0-0-0-0,bootindex=2 \ + -device virtio-net-pci,netdev=hostnet0,id=net0,mac=5b:f2:eb:3c:0b:46 \ + -netdev bridge,id=hostnet0,br=br,helper=$(type -P qemu-bridge-helper) \ + -netdev user,id=smbnet0,restrict=yes,net=10.1.2.0/24,host=10.1.2.1,smb=/home/kat/shared/,smbserver=10.1.2.2 \ + -device virtio-net-pci,netdev=smbnet0,id=net1,mac=2b:c6:c4:ac:67:ba \ + -device vfio-pci,host=0000:29:00.0,id=hostdev0,bus=pci.7,addr=0x0 \ + -device vfio-pci,host=0000:29:00.1,id=hostdev1,bus=pci.9,addr=0x0 \ + -device vfio-pci,host=0000:28:00.0,id=hostdev3,bus=pci.11,addr=0x0 \ + -device vfio-pci,host=0000:2b:00.3,id=hostdev4,bus=pci.19,addr=0x0 \ + -device virtio-balloon-pci,id=balloon0,bus=pci.5,addr=0x0 \ + -chardev socket,path=/tmp/vfio-qmp,server,nowait,id=qmp0 \ + -mon chardev=qmp0,id=qmp,mode=control \ + -chardev socket,path=/tmp/vfio-qga,server,nowait,id=qga0 \ + -device virtserialport,chardev=qga0,name=org.qemu.guest_agent.0 \ + -set device.scsi0-0-0-0.rotation_rate=1 \ + -cpu host,hv_time,kvm=off,hv_vendor_id=null,-hypervisor \ + -msg timestamp=on +'' + +# -blockdev '{"driver":"host_device","filename":"/dev/disk/by-id/ata-TOSHIBA_HDWD130_787VUS4AS-part2","aio":"native","node-name":"libvirt-1-storage","cache":{"direct":true,"no-flush":false},"auto-read-only":true,"discard":"unmap"}' \ +# -device scsi-hd,bus=scsi0.0,channel=0,scsi-id=0,lun=1,device_id=drive-scsi0-0-0-1,drive=libvirt-1-format,id=scsi0-0-0-1,bootindex=3 \ +# -blockdev '{"node-name":"libvirt-1-format","read-only":false,"cache":{"direct":true,"no-flush":false},"driver":"raw","file":"libvirt-1-storage"}' \ +# -device vfio-pci,host=0000:21:00.0,id=hostdev2,bus=pci.12,addr=0x0 \