From c8a5be7b3eb941a1715660b1b3b96b4932a1a0f7 Mon Sep 17 00:00:00 2001 From: Gabriel Simmer Date: Sun, 14 Apr 2024 23:50:37 +0100 Subject: [PATCH] mom i vfio'd up --- flake.lock | 48 +++++++++++++++---------------- flake.nix | 1 + modules/vfio.nix | 56 ++++++++++++++++++++++++++++++++++++ nix/london/configuration.nix | 30 ++++++++++--------- 4 files changed, 98 insertions(+), 37 deletions(-) create mode 100644 modules/vfio.nix diff --git a/flake.lock b/flake.lock index c15eb04..43dcf75 100644 --- a/flake.lock +++ b/flake.lock @@ -239,11 +239,11 @@ ] }, "locked": { - "lastModified": 1712212014, - "narHash": "sha256-s+lbaf3nLRn1++/X2eXwY9mYCA/m9l8AvyG8beeOaXE=", + "lastModified": 1712645849, + "narHash": "sha256-67v20E0gH7nvAaMsah2oRIocnxGO25fATUyzQHIywxQ=", "owner": "nix-community", "repo": "home-manager", - "rev": "7e91f2a0ba4b62b88591279d54f741a13e36245b", + "rev": "40a99619da804a78a0b166e5c6911108c059c3a8", "type": "github" }, "original": { @@ -279,11 +279,11 @@ "nixpkgs-lib": "nixpkgs-lib" }, "locked": { - "lastModified": 1711886936, - "narHash": "sha256-D2WENp9GuaCostvNcQ7vElekk0V5cuMdnFZ7NfRhVrQ=", + "lastModified": 1712491724, + "narHash": "sha256-E5EcBzf/zaR3hD8g1CDtqqwXXebSWtqOvoaR+LDjTME=", "owner": "nix-community", "repo": "lib-aggregate", - "rev": "9c06929b83e57c18d125f1105ba6a423f24083d2", + "rev": "2737d0204685c3274390229a09eb8f7eaa1a9e89", "type": "github" }, "original": { @@ -355,11 +355,11 @@ }, "nixlib": { "locked": { - "lastModified": 1711846064, - "narHash": "sha256-cqfX0QJNEnge3a77VnytM0Q6QZZ0DziFXt6tSCV8ZSc=", + "lastModified": 1712450863, + "narHash": "sha256-K6IkdtMtq9xktmYPj0uaYc8NsIqHuaAoRBaMgu9Fvrw=", "owner": "nix-community", "repo": "nixpkgs.lib", - "rev": "90b1a963ff84dc532db92f678296ff2499a60a87", + "rev": "3c62b6a12571c9a7f65ab037173ee153d539905f", "type": "github" }, "original": { @@ -397,11 +397,11 @@ ] }, "locked": { - "lastModified": 1712191720, - "narHash": "sha256-xXtSSnVHURHsxLQO30dzCKW5NJVGV/umdQPmFjPFMVA=", + "lastModified": 1712537332, + "narHash": "sha256-yYlxv1sg/TNl6hghjAe0ct+/p5PwXiT1mpuaExjhR88=", "owner": "nix-community", "repo": "nixos-generators", - "rev": "0c15e76bed5432d7775a22e8d22059511f59d23a", + "rev": "d942db8df8ee860556a38754f15b8d03bf7e6933", "type": "github" }, "original": { @@ -412,11 +412,11 @@ }, "nixos-hardware": { "locked": { - "lastModified": 1711352745, - "narHash": "sha256-luvqik+i3HTvCbXQZgB6uggvEcxI9uae0nmrgtXJ17U=", + "lastModified": 1712566108, + "narHash": "sha256-c9nT2ZODGqobISP41kUwCQ84Srwg7a/1TmPFQuol2/8=", "owner": "NixOS", "repo": "nixos-hardware", - "rev": "9a763a7acc4cfbb8603bb0231fec3eda864f81c0", + "rev": "1e3b3a35b7083f4152f5a516798cf9b21e686465", "type": "github" }, "original": { @@ -444,11 +444,11 @@ }, "nixpkgs-lib": { "locked": { - "lastModified": 1711846064, - "narHash": "sha256-cqfX0QJNEnge3a77VnytM0Q6QZZ0DziFXt6tSCV8ZSc=", + "lastModified": 1712450863, + "narHash": "sha256-K6IkdtMtq9xktmYPj0uaYc8NsIqHuaAoRBaMgu9Fvrw=", "owner": "nix-community", "repo": "nixpkgs.lib", - "rev": "90b1a963ff84dc532db92f678296ff2499a60a87", + "rev": "3c62b6a12571c9a7f65ab037173ee153d539905f", "type": "github" }, "original": { @@ -467,11 +467,11 @@ ] }, "locked": { - "lastModified": 1712237761, - "narHash": "sha256-NoMBBCADTms3yx5BL+sbc7vfDivNiYULO6t9GBAsPt0=", + "lastModified": 1712675807, + "narHash": "sha256-wt55hgIHa5DF2P20jHoOmT8Ja0utw7Uk4/Sm3rR8mqc=", "owner": "nix-community", "repo": "nixpkgs-wayland", - "rev": "9b77653338f52da4b498abdf4835efb6ff6e453e", + "rev": "00eeba584c4753acae377c421cb496ede04e8bcd", "type": "github" }, "original": { @@ -530,11 +530,11 @@ }, "nixpkgs_5": { "locked": { - "lastModified": 1712192574, - "narHash": "sha256-LbbVOliJKTF4Zl2b9salumvdMXuQBr2kuKP5+ZwbYq4=", + "lastModified": 1712666087, + "narHash": "sha256-WwjUkWsjlU8iUImbivlYxNyMB1L5YVqE8QotQdL9jWc=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "f480f9d09e4b4cf87ee6151eba068197125714de", + "rev": "a76c4553d7e741e17f289224eda135423de0491d", "type": "github" }, "original": { diff --git a/flake.nix b/flake.nix index 38734de..459313d 100644 --- a/flake.nix +++ b/flake.nix @@ -218,6 +218,7 @@ modules = [ (import ./nix/london/configuration.nix) (import ./modules/cloudflare-warp.nix) + (import ./modules/vfio.nix) home-manager.nixosModules.home-manager { home-manager.useUserPackages = true; diff --git a/modules/vfio.nix b/modules/vfio.nix new file mode 100644 index 0000000..87ee8a7 --- /dev/null +++ b/modules/vfio.nix @@ -0,0 +1,56 @@ +let + gpuIDs = [ + "1002:744c" # Graphics + "1002:ab30" # Audio + ]; +in +{ + pkgs, + lib, + config, + ... +}: { + options.vfio.enable = with lib; + mkEnableOption "Configure the machine for VFIO"; + + config = let + cfg = config.vfio; + in { + boot = { + kernelModules = [ "kvm-amd" "vfio_pci" "vfio" "vfio_iommu_type1" "kvmfr" ]; + extraModulePackages = with config.boot.kernelPackages; [ + kvmfr + ]; + extraModprobeConfig = '' + # The memory size is calculates in the same way as VM's shmem. + options kvmfr static_size_mb=64 + ''; + + kernelParams = + [ + "amd_iommu=on" + "pcie_acs_override=downstream,multifunction" + "vfio-pci.ids=1002:744c,1002:ab30" + "pcie_aspm=off" + ]; + }; + services.udev.extraRules = '' + SUBSYSTEM=="kvmfr", OWNER="gsimmer", GROUP="kvm", MODE="0660" + ''; + hardware.opengl.enable = true; + virtualisation.spiceUSBRedirection.enable = true; + virtualisation.libvirtd = { + qemu = { + verbatimConfig = '' + cgroup_device_acl = [ + "/dev/null", "/dev/full", "/dev/zero", + "/dev/random", "/dev/urandom", + "/dev/ptmx", "/dev/kvm", "/dev/kqemu", + "/dev/rtc","/dev/hpet", "/dev/vfio/vfio", + "/dev/kvmfr0" + ] + ''; + }; + }; + }; +} diff --git a/nix/london/configuration.nix b/nix/london/configuration.nix index 921f018..d25e476 100644 --- a/nix/london/configuration.nix +++ b/nix/london/configuration.nix @@ -7,6 +7,7 @@ ./hardware-configuration.nix ]; + vfio.enable = true; # Bootloader boot = { loader = { @@ -20,8 +21,8 @@ extraModulePackages = [ config.boot.kernelPackages.v4l2loopback ]; - kernelPackages = pkgs.linuxPackages_zen; - kernelModules = ["amdgpu" "coretemp" "kvm-amd" "v4l2loopback"]; + kernelPackages = pkgs.linuxPackages_latest; + kernelModules = [ "coretemp" "kvm-amd" "v4l2loopback"]; plymouth = { enable = true; theme = "breeze"; @@ -51,23 +52,28 @@ hostId = "3c26267f"; hostName = "LONDON"; networkmanager.enable = true; + interfaces.enp14s0.useDHCP = true; + interfaces.br0.useDHCP = true; + bridges = { + "br0" = { + interfaces = [ "enp14s0" ]; + }; + }; firewall = { enable = true; allowedUDPPortRanges = [ { - from = 27031; - to = 27036; + from = 3000; + to = 22000; } ]; allowedTCPPortRanges = [ { - from = 27036; - to = 27037; + from = 3000; + to = 22000; } ]; - allowedTCPPorts = [7000 7100 7001 22000 8000 3000 9943 9944]; - allowedUDPPorts = [69 6000 6001 7011 41641 3478 22000 21027 9943 9944]; - trustedInterfaces = ["enp4s0" "tailscale0" "docker0"]; + trustedInterfaces = ["enp14s0" "tailscale0" "docker0"]; checkReversePath = "loose"; }; nftables.enable = true; @@ -140,11 +146,9 @@ xkb.layout = "us"; xkb.variant = ""; enable = true; - displayManager.sddm.enable = true; - }; - desktopManager = { - plasma6.enable = true; }; + desktopManager.plasma6.enable = true; + displayManager.sddm.enable = true; pipewire = { enable = true; alsa.enable = true;