c40f4460e3
persistMode=backing seeds the persistent OS disk as a thin qcow2 overlay (`qemu-img create -b <storeImage>`) instead of a full cp, so many VMs share one base image and each holds only its deltas. Because the overlay reads through a store path that nix does not otherwise pin (the disk lives outside the store), and because even a cp'd Windows disk backs onto the store chain, a per-VM oneshot `vm.vmix-gcroot@<name>` reads the overlay's ACTUAL backing_file at boot (not the config's current image, which drifts after a rebuild) and symlinks it under /nix/var/nix/gcroots. It runs before the VM service and outside its ProtectSystem sandbox. Fires whenever the OS disk is persistent, covering copy too. Default stays copy, so existing VMs are unaffected. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_0117qMyjpuXsjpVAcpJbFD8g
327 lines
17 KiB
Nix
327 lines
17 KiB
Nix
{ config, pkgs, lib, vmixLib, ... }:
|
|
with lib;
|
|
with vmixLib.network;
|
|
let
|
|
vmixCfg = config.vmix;
|
|
mkShortIfaceName = prefix: seed: "${prefix}-${builtins.substring 0 8 (builtins.hashString "sha256" seed)}";
|
|
|
|
mkServices4aVMInNamespace = spaceName: vmName: cfg:
|
|
let
|
|
vmCfg = cfg // { name = vmName; };
|
|
netCfg = vmCfg.networks;
|
|
|
|
mkTap4aLan = lanName: tapCfg:
|
|
let
|
|
tapInterfaceName = mkShortIfaceName "vt" "${spaceName}-${vmCfg.name}-${lanName}";
|
|
lanInterfaceName = "brx-${lanName}";
|
|
in
|
|
{
|
|
name = lanName;
|
|
iface = tapInterfaceName;
|
|
mac = tapCfg.mac;
|
|
create = ''
|
|
ip tuntap add dev ${tapInterfaceName} mode tap
|
|
ip link set dev ${tapInterfaceName} up
|
|
ip link set dev ${tapInterfaceName} master ${lanInterfaceName}
|
|
'' + lib.optionalString (tapCfg.ip != null) ''
|
|
# Make the static ip somehow part of the script so nixOs thinks the service has changed when the IP changes, which will trigger a VM restart.
|
|
# So whenever IP changes, VM will restart automatically
|
|
# The IP is actually bein assigned to VM by dnsmasq, when VM requests it via DHCP
|
|
# Static IP - ${tapCfg.ip}
|
|
'';
|
|
delete = ''
|
|
ip link del ${tapInterfaceName}
|
|
'';
|
|
};
|
|
|
|
mkMacvtap = macvtapName: macvtapVmCfg:
|
|
let
|
|
macvtapNetworkCfg = config.vmix.namespaces.${spaceName}.networks.macvtaps.${macvtapName};
|
|
macvtapInterfaceName = mkShortIfaceName "mt" "${spaceName}-${vmCfg.name}-${macvtapNetworkCfg.uplink.iface}-${macvtapName}";
|
|
uplinkNamespaceArg = lib.optionalString (macvtapNetworkCfg.uplink.namespace != null) "-n ${macvtapNetworkCfg.uplink.namespace}";
|
|
in
|
|
{
|
|
name = macvtapName;
|
|
iface = macvtapInterfaceName;
|
|
mac = macvtapVmCfg.mac;
|
|
|
|
create = ''
|
|
ip ${uplinkNamespaceArg} link add link ${macvtapNetworkCfg.uplink.iface} name ${macvtapInterfaceName} type macvtap mode bridge
|
|
${lib.optionalString (macvtapVmCfg.mac != null) "ip ${uplinkNamespaceArg} link set dev ${macvtapInterfaceName} address ${macvtapVmCfg.mac}"}
|
|
ip ${uplinkNamespaceArg} link set ${macvtapInterfaceName} netns ${spaceName}.vmix
|
|
ip -n ${spaceName}.vmix link set dev ${macvtapInterfaceName} up
|
|
'';
|
|
delete = ''
|
|
ip -n ${spaceName}.vmix link del ${macvtapInterfaceName}
|
|
'';
|
|
};
|
|
|
|
allTaps = (mapAttrsToList mkTap4aLan netCfg.lans);
|
|
allMacvtaps = (mapAttrsToList mkMacvtap netCfg.macvtaps);
|
|
|
|
createTapsforLansScript = pkgs.writeShellScript "${vmCfg.name}-taps-vmix" (
|
|
concatStringsSep "\n" (builtins.map (tap: tap.create) allTaps)
|
|
);
|
|
|
|
deleteTapsforLansScript = pkgs.writeShellScript "${vmCfg.name}-taps-vmix" (
|
|
concatStringsSep "\n" (builtins.map (tap: tap.delete) allTaps)
|
|
);
|
|
|
|
createMacvTapsScript = pkgs.writeShellScript "${vmCfg.name}-taps-vmix" (
|
|
concatStringsSep "\n" (builtins.map (macvtap: macvtap.create) allMacvtaps)
|
|
);
|
|
|
|
deleteMacvTapsScript = pkgs.writeShellScript "${vmCfg.name}-taps-vmix" (
|
|
concatStringsSep "\n" (builtins.map (macvtap: macvtap.delete) allMacvtaps)
|
|
);
|
|
|
|
hasOsDisk = vmCfg.disks.os.file != null;
|
|
|
|
# Auto-detect Windows from _vmixOsType marker on the disk image
|
|
isWindows = vmCfg.windows.enable || (hasOsDisk && (vmCfg.disks.os.file._vmixOsType or "linux") == "windows");
|
|
# Interrupt remapping in the virtual IOMMU only works on a split irqchip,
|
|
# so viommu wins over the full in-kernel irqchip hideVirtualized asks for.
|
|
machineIrqchipArg =
|
|
if vmCfg.pci.viommu.enable then ",kernel-irqchip=split"
|
|
else optionalString vmCfg.cpu.hideVirtualized ",kernel_irqchip=on";
|
|
# Functions of one physical device have to reach the guest as functions
|
|
# of one device too. Giving each address its own root port splits a GPU
|
|
# from its own HDMI audio, and Navi cannot then reset or power-manage
|
|
# either half -- the guest ends up with a card stuck in D3. So group by
|
|
# everything left of the function digit and place each group behind a
|
|
# single root port, multifunction, at the same slot.
|
|
pciDeviceOf = addr: head (splitString "." addr);
|
|
pciFunctionOf = addr: last (splitString "." addr);
|
|
pciGroups = map
|
|
(dev: filter (a: pciDeviceOf a == dev) vmCfg.pci.passthrough)
|
|
(unique (map pciDeviceOf vmCfg.pci.passthrough));
|
|
|
|
|
|
# Linux VMs: apply customizeImage with 9p fstab and machine-id setup
|
|
linuxOsImage = vmixLib.linux.customizeImage vmCfg.disks.os.file {
|
|
name = vmCfg.name;
|
|
commands = ''
|
|
truncate /etc/machine-id
|
|
run-command systemd-machine-id-setup
|
|
run-command ssh-keygen -A
|
|
run ${vmixLib.linux.scriptsNFiles.add-9p-mounts-to-fstab vmCfg.shares}
|
|
'';
|
|
};
|
|
|
|
# Windows VMs: use disk image as-is (customization done at image build time)
|
|
storeImage = if !hasOsDisk then null
|
|
else if isWindows then vmCfg.disks.os.file
|
|
else linuxOsImage;
|
|
|
|
# When persist = true, QEMU needs a mutable disk outside /nix/store.
|
|
# The store image is copied to persistPath on first boot.
|
|
osDiskPath = if !hasOsDisk then null
|
|
else if vmCfg.disks.os.persist then vmCfg.disks.os.persistPath
|
|
else toString storeImage;
|
|
|
|
# Script to seed the persistent disk from the store image on first boot
|
|
seedPersistentDiskScript = pkgs.writeShellScript "${vmCfg.name}-seed-disk-vmix" ''
|
|
PERSIST_PATH="${vmCfg.disks.os.persistPath}"
|
|
if [ ! -f "$PERSIST_PATH" ]; then
|
|
echo "Seeding persistent disk from store image..."
|
|
mkdir -p "$(dirname "$PERSIST_PATH")"
|
|
${if vmCfg.disks.os.persistMode == "backing"
|
|
then ''qemu-img create -f qcow2 -F qcow2 -b "${toString storeImage}" "$PERSIST_PATH"''
|
|
else ''cp --no-preserve=mode "${toString storeImage}" "$PERSIST_PATH"''}
|
|
chmod 600 "$PERSIST_PATH"
|
|
fi
|
|
'';
|
|
|
|
persistExecStartPre = lib.optional (hasOsDisk && vmCfg.disks.os.persist) seedPersistentDiskScript;
|
|
|
|
# A GC root pinning the OS overlay's ACTUAL backing store path, so
|
|
# nix-collect-garbage cannot delete the store image the disk reads through.
|
|
gcrootLink = "/nix/var/nix/gcroots/vmix-${vmCfg.name}-osbacking";
|
|
gcrootScript = pkgs.writeShellScript "${vmCfg.name}-gcroot-vmix" ''
|
|
# Read the live overlay's backing (not the config's current image, which
|
|
# drifts to a new store path after a rebuild while the overlay keeps
|
|
# backing the old one). Pinning the top of the chain transitively keeps
|
|
# the whole chain -- qcow2 backing_file paths are registered nix refs.
|
|
BACK=""
|
|
if [ -f "${vmCfg.disks.os.persistPath}" ]; then
|
|
BACK=$(qemu-img info "${vmCfg.disks.os.persistPath}" 2>/dev/null | awk '/^backing file:/ {print $3; exit}')
|
|
fi
|
|
[ -z "$BACK" ] && BACK="${toString storeImage}"
|
|
if [ -n "$BACK" ]; then
|
|
mkdir -p /nix/var/nix/gcroots
|
|
ln -sfn "$BACK" "${gcrootLink}"
|
|
fi
|
|
'';
|
|
|
|
# QEMU expects single-letter boot codes (e.g. c,d,n), while vmix uses readable names.
|
|
bootOrderQemu =
|
|
let
|
|
bootDeviceAliases = {
|
|
os = "c";
|
|
iso = "d";
|
|
net = "n";
|
|
floppy = "a";
|
|
};
|
|
in
|
|
concatStrings (builtins.map (device: bootDeviceAliases.${device}) vmCfg.boot.order);
|
|
|
|
spiceUsbRedirArgs =
|
|
if vmCfg.spice.enable && vmCfg.spice.usbRedir.enable then
|
|
concatStringsSep " \\\n " ([
|
|
"-device qemu-xhci,id=spice-usb-xhci"
|
|
] ++ (concatMap (i: [
|
|
"-chardev spicevmc,name=usbredir,id=spice-usbredirchardev${toString i}"
|
|
"-device usb-redir,chardev=spice-usbredirchardev${toString i},id=spice-usbredirdev${toString i}"
|
|
]) (range 1 vmCfg.spice.usbRedir.channels)))
|
|
else
|
|
"";
|
|
|
|
vncArgs = concatStringsSep "," (
|
|
[
|
|
"${vmCfg.vnc.addr}:${toString (vmCfg.vnc.port - 5900)}"
|
|
"share=${vmCfg.vnc.sharePolicy}"
|
|
]
|
|
++ optional (vmCfg.vnc.websocketPort != null) "websocket=${toString vmCfg.vnc.websocketPort}"
|
|
++ optional (vmCfg.vnc.passwordFile != null) "password-secret=vnc-pass-${vmCfg.name}"
|
|
);
|
|
|
|
qemuStartVMScript = pkgs.writeShellScript "${vmCfg.name}-qemu-vmix" ''
|
|
${optionalString vmCfg.vnc.enable ''
|
|
${optionalString (vmCfg.vnc.passwordFile != null) ''
|
|
if [ ! -r ${escapeShellArg vmCfg.vnc.passwordFile} ]; then
|
|
echo "VNC password file is not readable: ${vmCfg.vnc.passwordFile}" >&2
|
|
exit 1
|
|
fi
|
|
if [ ! -s ${escapeShellArg vmCfg.vnc.passwordFile} ]; then
|
|
echo "VNC password file is empty: ${vmCfg.vnc.passwordFile}" >&2
|
|
exit 1
|
|
fi
|
|
''}
|
|
''}
|
|
${optionalString vmCfg.spice.enable ''
|
|
${optionalString (vmCfg.spice.passwordFile != null) ''
|
|
if [ ! -r ${escapeShellArg vmCfg.spice.passwordFile} ]; then
|
|
echo "SPICE password file is not readable: ${vmCfg.spice.passwordFile}" >&2
|
|
exit 1
|
|
fi
|
|
if [ ! -s ${escapeShellArg vmCfg.spice.passwordFile} ]; then
|
|
echo "SPICE password file is empty: ${vmCfg.spice.passwordFile}" >&2
|
|
exit 1
|
|
fi
|
|
''}
|
|
''}
|
|
exec qemu-system-${vmCfg.arch} \
|
|
${if vmCfg.nographic && vmCfg.pci.passthrough != [] then "-display none -vga none" else optionalString vmCfg.nographic "-nographic"} \
|
|
${# QEMU realizes devices in command-line order and intel-iommu must
|
|
# exist before anything it translates, so it leads the device list.
|
|
optionalString vmCfg.pci.viommu.enable "-device intel-iommu,intremap=on,caching-mode=on"} \
|
|
${optionalString (vmCfg.vnc.enable && vmCfg.vnc.passwordFile != null) "-object secret,id=vnc-pass-${vmCfg.name},file=${escapeShellArg vmCfg.vnc.passwordFile}"} \
|
|
${optionalString vmCfg.vnc.enable "-vnc ${vncArgs}"} \
|
|
${optionalString (vmCfg.spice.enable && vmCfg.spice.passwordFile != null) "-object secret,id=spice-pass-${vmCfg.name},file=${escapeShellArg vmCfg.spice.passwordFile}"} \
|
|
${optionalString vmCfg.spice.enable "-spice addr=${vmCfg.spice.addr},port=${toString vmCfg.spice.port}${optionalString (vmCfg.spice.passwordFile == null) ",disable-ticketing=on"}${optionalString (vmCfg.spice.passwordFile != null) ",password-secret=spice-pass-${vmCfg.name}"}"} \
|
|
${optionalString vmCfg.spice.enable (if vmCfg.spice.displayDevice == "qxl" && vmCfg.spice.vgamem != null then "-vga none -device qxl-vga,vgamem_mb=${toString vmCfg.spice.vgamem}" else "-vga ${vmCfg.spice.displayDevice}")} \
|
|
${optionalString (vmCfg.spice.enable && vmCfg.spice.agent.enable) "-device virtio-serial-pci -chardev spicevmc,id=vdagent,debug=0,name=vdagent -device virtserialport,chardev=vdagent,name=com.redhat.spice.0"} \
|
|
${# Guest agent channel — prevents qemu-ga from spinning when virtio-win guest tools are installed
|
|
optionalString isWindows "${optionalString (!vmCfg.spice.enable || !vmCfg.spice.agent.enable) "-device virtio-serial-pci"} -chardev socket,path=/tmp/qga-${vmCfg.name}.sock,server=on,wait=off,id=qga0 -device virtserialport,chardev=qga0,name=org.qemu.guest_agent.0"} \
|
|
${spiceUsbRedirArgs} \
|
|
${optionalString vmCfg.kvm "-accel kvm"} \
|
|
-name ${vmCfg.name} \
|
|
-m ${toString vmCfg.mem.size} \
|
|
${optionalString vmCfg.mem.balloon "-device virtio-balloon-pci"} \
|
|
-smp cores=${toString vmCfg.cpu.cores} \
|
|
-cpu ${vmCfg.cpu.model}${optionalString vmCfg.cpu.hideVirtualized ",kvm=off,hv_vendor_id=1234567890ab,-hypervisor"} \
|
|
-machine type=${vmCfg.pc.type}${machineIrqchipArg} \
|
|
${optionalString vmCfg.bios.efi "-bios ${pkgs.OVMF.fd}/FV/OVMF.fd"} \
|
|
${optionalString vmCfg.bios.tpm "-chardev socket,id=chrtpm,path=/tmp/mytpm-sock -tpmdev emulator,id=tpm0,chardev=chrtpm -device tpm-tis,tpmdev=tpm0"} \
|
|
${# Windows: localtime RTC, USB tablet for mouse, disable S3/S4 sleep
|
|
optionalString isWindows ''
|
|
-rtc base=localtime,clock=host \
|
|
-device qemu-xhci -device usb-tablet \
|
|
-global ICH9-LMB.disable_s3=1 -global ICH9-LMB.disable_s4=1 \
|
|
''} \
|
|
${optionalString hasOsDisk "-drive file=${osDiskPath},format=qcow2,if=virtio${optionalString (vmCfg.disks.os.persist == false) ",snapshot=on"}"} \
|
|
${optionalString (vmCfg.disks.iso.file != null) "-drive file=${toString vmCfg.disks.iso.file},media=cdrom,readonly=on"} \
|
|
${concatMapStrings (diskCfg: ''
|
|
-drive file=${toString diskCfg.file},format=${diskCfg.format},if=${vmCfg.disks.bus} \
|
|
'') (attrValues vmCfg.disks.add)} \
|
|
${concatStrings (mapAttrsToList (shareName: shareCfg: ''
|
|
-virtfs local,path=${toString shareCfg.source},security_model=passthrough,mount_tag=${shareName} \
|
|
'') vmCfg.shares)} \
|
|
${optionalString cfg.networks.user.enable "
|
|
-netdev user,id=user \
|
|
-device ${vmCfg.nicModel},netdev=user \
|
|
"} \
|
|
${concatMapStrings (tapCfg: ''
|
|
-device ${vmCfg.nicModel},netdev=lan-${tapCfg.name},mac=${tapCfg.mac} \
|
|
-netdev tap,id=lan-${tapCfg.name},ifname=${tapCfg.iface},script=no,downscript=no \
|
|
'') allTaps} \
|
|
${concatStrings (imap1 (i: macvtap: ''
|
|
-device ${vmCfg.nicModel},netdev=macvtap-${macvtap.name},mac=$(ip l show ${macvtap.iface} | awk '/link\/ether/{print $2}') \
|
|
-netdev tap,id=macvtap-${macvtap.name},fd=${toString (i+2)} ${toString (i+2)}<>/dev/tap$(ip l show ${macvtap.iface} | awk -F':' '/${macvtap.iface}/{print $1}') \
|
|
'') allMacvtaps)} \
|
|
${concatStrings (imap1 (i: group: ''
|
|
-device pcie-root-port,id=pci-passthrough${toString i},chassis=${toString i},slot=${toString i} \
|
|
'' + concatStrings (imap0 (j: pciAddr: ''
|
|
-device vfio-pci,host=${pciAddr},bus=pci-passthrough${toString i},addr=0x0.${pciFunctionOf pciAddr}${optionalString (length group > 1 && j == 0) ",multifunction=on"}${optionalString (i == 1 && j == 0) "${optionalString vmCfg.pci.vgaPassthrough ",x-vga=on"}${optionalString (vmCfg.pci.romFile != null) ",romfile=${vmCfg.pci.romFile}"}"} \
|
|
'') group)) pciGroups)} \
|
|
${concatMapStrings (usbDev: ''
|
|
-device usb-host,vendorid=0x${usbDev.vendorId},productid=0x${usbDev.productId} \
|
|
'') vmCfg.usb.hostDevices} \
|
|
${optionalString (vmCfg.boot.menu == true) "-boot menu=on"} \
|
|
${optionalString (length vmCfg.boot.order > 0) "-boot order=${bootOrderQemu}"} \
|
|
'';
|
|
in
|
|
lib.optionalAttrs (cfg.enable) {
|
|
"vm.vmix@${vmCfg.name}" = rec {
|
|
bindsTo = [ "net.vmix@${spaceName}.target" ] ++ lib.optional (allMacvtaps != []) "macvtaps.vm.vmix@${vmCfg.name}.service";
|
|
unitConfig.JoinsNamespaceOf = "ns.net.vmix@${spaceName}.service";
|
|
after = bindsTo ++ lib.optional (hasOsDisk && vmCfg.disks.os.persist) "vm.vmix-gcroot@${vmCfg.name}.service";
|
|
wants = lib.optional (hasOsDisk && vmCfg.disks.os.persist) "vm.vmix-gcroot@${vmCfg.name}.service";
|
|
path = with pkgs; [ iproute2 qemu gawk coreutils ];
|
|
serviceConfig = {
|
|
ExecStartPre = persistExecStartPre ++ [ createTapsforLansScript ];
|
|
ExecStart = qemuStartVMScript;
|
|
ExecStopPost = deleteTapsforLansScript;
|
|
PrivateTmp = true;
|
|
ProtectSystem = true;
|
|
ProtectHome = true;
|
|
PrivateNetwork = true;
|
|
} // lib.optionalAttrs (vmCfg.pci.passthrough != []) {
|
|
# VFIO passthrough needs raw device access — relax sandboxing
|
|
ProtectSystem = lib.mkForce false;
|
|
SupplementaryGroups = [ "kvm" ];
|
|
};
|
|
wantedBy = lib.mkIf vmCfg.autostart [ "multi-user.target" ];
|
|
};
|
|
} // lib.optionalAttrs (allMacvtaps != []) {
|
|
"macvtaps.vm.vmix@${vmCfg.name}" = rec {
|
|
bindsTo = [ "net.vmix@${spaceName}.target" ];
|
|
after = bindsTo;
|
|
partOf = [ "vm.vmix@${vmCfg.name}.service" ];
|
|
path = with pkgs; [ iproute2 ];
|
|
serviceConfig = {
|
|
Type = "oneshot";
|
|
RemainAfterExit = true;
|
|
ExecStart = createMacvTapsScript;
|
|
ExecStop = deleteMacvTapsScript;
|
|
};
|
|
};
|
|
}
|
|
// lib.optionalAttrs (cfg.enable && hasOsDisk && vmCfg.disks.os.persist) {
|
|
"vm.vmix-gcroot@${vmCfg.name}" = {
|
|
before = [ "vm.vmix@${vmCfg.name}.service" ];
|
|
wantedBy = [ "multi-user.target" ];
|
|
path = with pkgs; [ qemu coreutils ];
|
|
serviceConfig = {
|
|
Type = "oneshot";
|
|
RemainAfterExit = true;
|
|
ExecStart = gcrootScript;
|
|
};
|
|
};
|
|
};
|
|
|
|
vmServices = concatMapAttrs (spaceName: namespaceCfg: (concatMapAttrs (mkServices4aVMInNamespace spaceName) namespaceCfg.vms)) vmixCfg.namespaces;
|
|
in
|
|
{
|
|
config.systemd.services = vmServices;
|
|
}
|