diff --git a/lib/images/windows/helpers/customizeImage.nix b/lib/images/windows/helpers/customizeImage.nix index 19758b9..a1e7be8 100644 --- a/lib/images/windows/helpers/customizeImage.nix +++ b/lib/images/windows/helpers/customizeImage.nix @@ -29,6 +29,12 @@ compact ? false, # QEMU timeout in seconds (default 30 min, increase for Windows Update) qemuTimeout ? 1800, + # Blank disk attached for the Audit Mode boot, e.g. { size = "100G"; }. + # Windows sees it as disk 1, which is what lets a template partition it and + # relocate profiles onto it in that same boot instead of deferring OOBE to + # real hardware. Emitted as the derivation's `data` output, so whatever the + # template writes to it survives the build. + extraDisk ? null, }: let originalImageName = lib.strings.removeSuffix "-vmix" (lib.strings.removeSuffix ".qcow2" originalImage.name); @@ -67,6 +73,11 @@ ]); cdromArgs = lib.concatMapStringsSep " \\\n " (cd: "-drive file=${cd},media=cdrom,readonly=on") cdroms; + extraDiskImg = "./extra.qcow2"; + extraDiskArgs = lib.optionalString (extraDisk != null) + (if isAHCI + then "-drive file=${extraDiskImg},format=qcow2,if=none,id=disk1 -device ide-hd,drive=disk1" + else "-drive file=${extraDiskImg},format=qcow2,if=virtio"); displayArg = if vncDisplay != null then "-vnc ${vncDisplay}" else null; @@ -109,6 +120,7 @@ then "-drive file=${resultImg},format=qcow2,if=none,id=disk0 -device ide-hd,drive=disk0" else "-drive file=${resultImg},format=qcow2,if=virtio"} \ ${cdromArgs} \ + ${extraDiskArgs} \ -nic user,model=${if nicModel != null then nicModel else if isAHCI then "e1000" else "virtio-net-pci"}" timeout ${toString qemuTimeout} qemu-system-x86_64 $VMIX_DISPLAY $QEMU_ARGS || \ @@ -127,6 +139,10 @@ # create resulting image backed by original image qemu-img create -f qcow2 -b ${originalImage} -F qcow2 ${resultImg} [ -n "${diskSize}" ] && qemu-img resize ${resultImg} ${diskSize} + ${lib.optionalString (extraDisk != null) '' + echo "=== vmix: creating extra disk (${extraDisk.size}) ===" + qemu-img create -f qcow2 ${extraDiskImg} ${extraDisk.size} + ''} ${virtWinRegMerge} ${auditBootCommands} ${lib.optionalString compact '' @@ -135,10 +151,14 @@ mv compact.qcow2 ${resultImg} ''} mv ${resultImg} $out + ${lib.optionalString (extraDisk != null) "mv ${extraDiskImg} $data"} ''; builtImage = pkgs.runCommand customImageName ({ nativeBuildInputs = with pkgs; [ qemu perl guestfs-tools ]; requiredSystemFeatures = [ "kvm" ]; - } // lib.optionalAttrs impure { __noChroot = true; }) builderCommand; + } // lib.optionalAttrs impure { __noChroot = true; } + # A second output rather than a directory, so ${image} keeps meaning the OS + # qcow2 for every existing consumer and the fold can still back onto it. + // lib.optionalAttrs (extraDisk != null) { outputs = [ "out" "data" ]; }) builderCommand; in builtImage // { _vmixOsType = "windows"; useAHCI = isAHCI; } diff --git a/lib/images/windows/templates/generalize.nix b/lib/images/windows/templates/generalize.nix index cf1f2e2..65194b2 100644 --- a/lib/images/windows/templates/generalize.nix +++ b/lib/images/windows/templates/generalize.nix @@ -23,6 +23,18 @@ in enableRDP ? false, # NIC model for the build VM (e.g. "e1000" for images without VirtIO drivers) nicModel ? null, + # Static IPv4 for the guest's single NIC, applied from inside Windows: + # { address = "10.10.10.26"; prefixLength = 24; gateway = "10.10.10.1"; + # dns = [ "10.10.10.1" ]; } + staticIP ? null, + # Relocate user profiles, e.g. "D:\\Users". Needs the volume to exist by the + # time specialize runs, which is what dataDisk arranges. + profilesDirectory ? null, + # Partition the non-OS disk and relocate user profiles onto it, e.g. + # { driveLetter = "D"; label = "data"; size = "100G"; }. The disk is attached + # during the build (see extraDisk in the returned set), so this is done and + # verified before the image ever reaches a host. + dataDisk ? null, # delayOobeRun = true: sysprep only, OOBE + activation on real hardware # delayOobeRun = false: sysprep + OOBE + activation in build VM delayOobeRun ? false, @@ -42,6 +54,45 @@ in stripHash = s: lib.removePrefix "#" s; bgRgb = if bgColor != null then hexToRgbStr (stripHash bgColor) else null; + staticDnsList = lib.optionalString (staticIP != null) + (lib.concatMapStringsSep "," (s: "'${s}'") staticIP.dns); + + dataDriveLetter = if dataDisk != null then (dataDisk.driveLetter or "D") else "D"; + dataLabel = if dataDisk != null then (dataDisk.label or "data") else "data"; + + # ProfilesDirectory is only honoured when the volume it names already exists, + # and a freshly created zvol arrives RAW. Initializing the disk here, in the + # same specialize pass, brings it up before oobeSystem creates any profile. + # + # Idempotent, because specialize runs again on every sysprep: a RAW disk gets + # a GPT label, one full-size NTFS partition and the drive letter, while a disk + # that already holds data keeps it and only has its letter re-asserted. The + # OS disk is added to QEMU first and so is always disk 0. + initDataDiskScript = pkgs.writeText "vmix-init-data-disk.cmd" '' + @echo off + powershell -NoProfile -ExecutionPolicy Bypass -Command "$ErrorActionPreference='Stop'; $d = Get-Disk | Where-Object Number -ne 0 | Sort-Object Number | Select-Object -First 1; if (-not $d) { exit 0 }; if ($d.PartitionStyle -eq 'RAW') { Initialize-Disk -Number $d.Number -PartitionStyle GPT -Confirm:$false; $p = New-Partition -DiskNumber $d.Number -UseMaximumSize -DriveLetter ${dataDriveLetter}; Format-Volume -Partition $p -FileSystem NTFS -NewFileSystemLabel '${dataLabel}' -Confirm:$false | Out-Null } else { $p = Get-Partition -DiskNumber $d.Number | Sort-Object Size -Descending | Select-Object -First 1; if ($p -and $p.DriveLetter -ne '${dataDriveLetter}') { Set-Partition -InputObject $p -NewDriveLetter ${dataDriveLetter} } }" + ''; + + folderLocationsXml = lib.optionalString (profilesDirectory != null) '' + + + ${profilesDirectory} + ''; + + dataDiskXml = lib.optionalString (dataDisk != null) '' + + + + + 1 + cmd /c C:\vmix-init-data-disk.cmd + vmix: initialize the data disk + + + ''; + # Post-OOBE script: runs as the created user via FirstLogonCommands. postOobeScript = pkgs.writeText "post-oobe.cmd" '' @echo off @@ -114,6 +165,13 @@ in reg add "HKLM\SYSTEM\CurrentControlSet\Services\TermService" /v Start /t REG_DWORD /d 2 /f ''} + ${lib.optionalString (staticIP != null) '' + :: This VM's only NIC sits on a macvtap bridged to the host's LAN, so its + :: address is a LAN address that nothing hands out -- the guest asserts it. + :: Clearing first makes the command idempotent across re-runs. + powershell -NoProfile -Command "$a = Get-NetAdapter -Physical | Sort-Object ifIndex | Select-Object -First 1; Remove-NetIPAddress -InterfaceIndex $a.ifIndex -AddressFamily IPv4 -Confirm:$false -ErrorAction SilentlyContinue; Remove-NetRoute -InterfaceIndex $a.ifIndex -AddressFamily IPv4 -Confirm:$false -ErrorAction SilentlyContinue; New-NetIPAddress -InterfaceIndex $a.ifIndex -IPAddress '${staticIP.address}' -PrefixLength ${toString staticIP.prefixLength} -DefaultGateway '${staticIP.gateway}' | Out-Null; Set-DnsClientServerAddress -InterfaceIndex $a.ifIndex -ServerAddresses ${staticDnsList}" + ''} + :: Clean up del /q C:\oobe-unattend.xml 2>nul del /q C:\vmix-audit-script.cmd 2>nul @@ -135,7 +193,9 @@ in Automatic +${folderLocationsXml} +${dataDiskXml} @@ -195,11 +255,16 @@ in in { name = if delayOobeRun then "generalize-delay-oobe" else "generalize"; inherit nicModel; + # The blank disk is attached for the Audit Mode boot itself, so the disk-init + # command and the profile relocation both happen under OOBE in the build VM. + # That is what makes delayOobeRun unnecessary: nothing is left to do on real + # hardware. The written disk comes back as this derivation's `data` output. + extraDisk = if dataDisk != null then { size = dataDisk.size or "100G"; } else null; uploads = [ { source = oobeXml; dest = "/oobe-unattend.xml"; } { source = postOobeScript; dest = "/post-oobe.cmd"; } { source = masScript; dest = "/MAS_AIO.cmd"; } - ]; + ] ++ lib.optional (dataDisk != null) { source = initDataDiskScript; dest = "/vmix-init-data-disk.cmd"; }; # delayOobeRun: sysprep + shutdown — OOBE runs on real hardware # generalize: sysprep + reboot into OOBE in the same QEMU session auditScript = '' diff --git a/nixos/vms/config.nix b/nixos/vms/config.nix index 42b0868..b9bac4b 100644 --- a/nixos/vms/config.nix +++ b/nixos/vms/config.nix @@ -79,6 +79,11 @@ let # Auto-detect Windows from _vmixOsType marker on the disk image isWindows = vmCfg.windows.enable || (hasOsDisk && (vmCfg.disks.os.file._vmixOsType or "linux") == "windows"); + # Interrupt remapping in the virtual IOMMU only works on a split irqchip, + # so viommu wins over the full in-kernel irqchip hideVirtualized asks for. + machineIrqchipArg = + if vmCfg.pci.viommu.enable then ",kernel-irqchip=split" + else optionalString vmCfg.cpu.hideVirtualized ",kernel_irqchip=on"; # Linux VMs: apply customizeImage with 9p fstab and machine-id setup linuxOsImage = vmixLib.linux.customizeImage vmCfg.disks.os.file { @@ -174,6 +179,9 @@ let ''} exec qemu-system-${vmCfg.arch} \ ${if vmCfg.nographic && vmCfg.pci.passthrough != [] then "-display none -vga none" else optionalString vmCfg.nographic "-nographic"} \ + ${# QEMU realizes devices in command-line order and intel-iommu must + # exist before anything it translates, so it leads the device list. + optionalString vmCfg.pci.viommu.enable "-device intel-iommu,intremap=on,caching-mode=on"} \ ${optionalString (vmCfg.vnc.enable && vmCfg.vnc.passwordFile != null) "-object secret,id=vnc-pass-${vmCfg.name},file=${escapeShellArg vmCfg.vnc.passwordFile}"} \ ${optionalString vmCfg.vnc.enable "-vnc ${vncArgs}"} \ ${optionalString (vmCfg.spice.enable && vmCfg.spice.passwordFile != null) "-object secret,id=spice-pass-${vmCfg.name},file=${escapeShellArg vmCfg.spice.passwordFile}"} \ @@ -189,7 +197,7 @@ let ${optionalString vmCfg.mem.balloon "-device virtio-balloon-pci"} \ -smp cores=${toString vmCfg.cpu.cores} \ -cpu ${vmCfg.cpu.model}${optionalString vmCfg.cpu.hideVirtualized ",kvm=off,hv_vendor_id=1234567890ab,-hypervisor"} \ - -machine type=${vmCfg.pc.type}${optionalString vmCfg.cpu.hideVirtualized ",kernel_irqchip=on"} \ + -machine type=${vmCfg.pc.type}${machineIrqchipArg} \ ${optionalString vmCfg.bios.efi "-bios ${pkgs.OVMF.fd}/FV/OVMF.fd"} \ ${optionalString vmCfg.bios.tpm "-chardev socket,id=chrtpm,path=/tmp/mytpm-sock -tpmdev emulator,id=tpm0,chardev=chrtpm -device tpm-tis,tpmdev=tpm0"} \ ${# Windows: localtime RTC, USB tablet for mouse, disable S3/S4 sleep @@ -220,7 +228,7 @@ let '') allMacvtaps)} \ ${concatStrings (imap1 (i: pciAddr: '' -device pcie-root-port,id=pci-passthrough${toString i},chassis=${toString i},slot=${toString i} \ - -device vfio-pci,host=${pciAddr},bus=pci-passthrough${toString i}${optionalString (i == 1) ",x-vga=on${optionalString (vmCfg.pci.romFile != null) ",romfile=${vmCfg.pci.romFile}"}"} \ + -device vfio-pci,host=${pciAddr},bus=pci-passthrough${toString i}${optionalString (i == 1) "${optionalString vmCfg.pci.vgaPassthrough ",x-vga=on"}${optionalString (vmCfg.pci.romFile != null) ",romfile=${vmCfg.pci.romFile}"}"} \ '') vmCfg.pci.passthrough)} \ ${concatMapStrings (usbDev: '' -device usb-host,vendorid=0x${usbDev.vendorId},productid=0x${usbDev.productId} \ diff --git a/nixos/vms/submoduleOptions.nix b/nixos/vms/submoduleOptions.nix index 0158c2b..c7cbd1a 100644 --- a/nixos/vms/submoduleOptions.nix +++ b/nixos/vms/submoduleOptions.nix @@ -262,11 +262,36 @@ with lib; default = []; description = "PCI device addresses to passthrough via VFIO (e.g. [\"0000:03:00.0\" \"0000:03:00.1\"])."; }; + pci.vgaPassthrough = mkOption { + type = types.bool; + default = true; + description = '' + Route legacy VGA to the first passthrough device (x-vga=on), which a + guest needs in order to drive that card as its own display. + + Turn it off when the guest only forwards the device onward to a nested + guest: x-vga=on claims the VGA path the emulated adapter wants, and the + nested guest does its own routing anyway. + ''; + }; pci.romFile = mkOption { type = types.nullOr types.path; default = null; description = "GPU VBIOS ROM file for the first passthrough device. Required when GPU PCI ROM BAR doesn't expose the full VBIOS (common with AMD Navi+)."; }; + pci.viommu.enable = mkOption { + type = types.bool; + default = false; + description = '' + Give the guest a virtual Intel IOMMU, so a guest that is itself a + hypervisor can bind a passed-through device to vfio-pci and hand it on + to a nested guest. Without one the guest sees no IOMMU and cannot + re-assign anything it was given. + + Implies kernel-irqchip=split, which interrupt remapping requires and + which replaces the full in-kernel irqchip cpu.hideVirtualized asks for. + ''; + }; usb.hostDevices = mkOption { default = [];