From 0e4619eb3931ea4f90ba2239466cf01a21047aa2 Mon Sep 17 00:00:00 2001 From: Git Sagar Date: Thu, 10 Sep 2026 04:58:54 -0300 Subject: [PATCH] pci: keep multifunction devices together; data disk: diskpart, not Storage cmdlets Two defects found while getting a GPU through Proxmox to a nested guest. Passthrough gave every address its own pcie-root-port, which splits a GPU from its own HDMI audio: 05:00.0 and 05:00.1 arrived in the guest as two devices on two buses instead of functions 0 and 1 of one device. Navi needs both halves on one device to reset or power-manage either, so the guest got a card stuck in D3 and a reset that could not be performed. Addresses are now grouped by everything left of the function digit, and each group goes behind one root port at one slot with multifunction=on on function 0 -- which is also where the VBIOS and the VGA route belong. The data-disk setup used Initialize-Disk/New-Partition/Format-Volume. Only the first of those works that early in specialize; the rest need services that are not up yet, and with ErrorActionPreference=Stop the script gave up straight after writing a GPT header. The result was a 50G disk carrying 24KB of nothing and a ProfilesDirectory pointing at a volume that never existed. diskpart works at that stage. It also tries assigning the letter before laying the disk out, so a disk that already holds a profile is lettered rather than cleaned. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_0117qMyjpuXsjpVAcpJbFD8g --- lib/images/windows/templates/generalize.nix | 28 ++++++++++++++++++++- nixos/vms/config.nix | 19 +++++++++++--- 2 files changed, 43 insertions(+), 4 deletions(-) diff --git a/lib/images/windows/templates/generalize.nix b/lib/images/windows/templates/generalize.nix index c369eaa..3e43a93 100644 --- a/lib/images/windows/templates/generalize.nix +++ b/lib/images/windows/templates/generalize.nix @@ -98,7 +98,33 @@ in # OS disk is added to QEMU first and so is always disk 0. initDataDiskScript = pkgs.writeText "vmix-init-data-disk.cmd" '' @echo off - powershell -NoProfile -ExecutionPolicy Bypass -Command "$ErrorActionPreference='Stop'; $d = Get-Disk | Where-Object Number -ne 0 | Sort-Object Number | Select-Object -First 1; if (-not $d) { exit 0 }; if ($d.PartitionStyle -eq 'RAW') { Initialize-Disk -Number $d.Number -PartitionStyle GPT -Confirm:$false; $p = New-Partition -DiskNumber $d.Number -UseMaximumSize -DriveLetter ${dataDriveLetter}; Format-Volume -Partition $p -FileSystem NTFS -NewFileSystemLabel '${dataLabel}' -Confirm:$false | Out-Null } else { $p = Get-Partition -DiskNumber $d.Number | Sort-Object Size -Descending | Select-Object -First 1; if ($p -and $p.DriveLetter -ne '${dataDriveLetter}') { Set-Partition -InputObject $p -NewDriveLetter ${dataDriveLetter} } }" + :: diskpart rather than the Storage cmdlets. New-Partition and + :: Format-Volume need services that are not up yet this early in + :: specialize, so they fail where Initialize-Disk succeeds -- which left + :: the disk carrying a GPT header and nothing else, and ProfilesDirectory + :: pointing at a volume that never existed. + if exist ${dataDriveLetter}:\ goto :done + + :: The volume may already be laid out and merely unlettered, in which case + :: assigning is enough and cleaning would destroy the profile. + > C:\Windows\Temp\vmix-dd-assign.txt echo select disk 1 + >> C:\Windows\Temp\vmix-dd-assign.txt echo select partition 1 + >> C:\Windows\Temp\vmix-dd-assign.txt echo assign letter=${dataDriveLetter} + diskpart /s C:\Windows\Temp\vmix-dd-assign.txt > nul 2>&1 + if exist ${dataDriveLetter}:\ goto :cleanup + + :: Nothing there to keep, so lay the disk out from scratch. + > C:\Windows\Temp\vmix-dd-init.txt echo select disk 1 + >> C:\Windows\Temp\vmix-dd-init.txt echo clean + >> C:\Windows\Temp\vmix-dd-init.txt echo convert gpt + >> C:\Windows\Temp\vmix-dd-init.txt echo create partition primary + >> C:\Windows\Temp\vmix-dd-init.txt echo format fs=ntfs quick label="${dataLabel}" + >> C:\Windows\Temp\vmix-dd-init.txt echo assign letter=${dataDriveLetter} + diskpart /s C:\Windows\Temp\vmix-dd-init.txt + + :cleanup + del /q C:\Windows\Temp\vmix-dd-assign.txt C:\Windows\Temp\vmix-dd-init.txt 2>nul + :done ''; folderLocationsXml = lib.optionalString (profilesDirectory != null) '' diff --git a/nixos/vms/config.nix b/nixos/vms/config.nix index b9bac4b..4dc86a7 100644 --- a/nixos/vms/config.nix +++ b/nixos/vms/config.nix @@ -84,6 +84,18 @@ let machineIrqchipArg = if vmCfg.pci.viommu.enable then ",kernel-irqchip=split" else optionalString vmCfg.cpu.hideVirtualized ",kernel_irqchip=on"; + # Functions of one physical device have to reach the guest as functions + # of one device too. Giving each address its own root port splits a GPU + # from its own HDMI audio, and Navi cannot then reset or power-manage + # either half -- the guest ends up with a card stuck in D3. So group by + # everything left of the function digit and place each group behind a + # single root port, multifunction, at the same slot. + pciDeviceOf = addr: head (splitString "." addr); + pciFunctionOf = addr: last (splitString "." addr); + pciGroups = map + (dev: filter (a: pciDeviceOf a == dev) vmCfg.pci.passthrough) + (unique (map pciDeviceOf vmCfg.pci.passthrough)); + # Linux VMs: apply customizeImage with 9p fstab and machine-id setup linuxOsImage = vmixLib.linux.customizeImage vmCfg.disks.os.file { @@ -226,10 +238,11 @@ let -device ${vmCfg.nicModel},netdev=macvtap-${macvtap.name},mac=$(ip l show ${macvtap.iface} | awk '/link\/ether/{print $2}') \ -netdev tap,id=macvtap-${macvtap.name},fd=${toString (i+2)} ${toString (i+2)}<>/dev/tap$(ip l show ${macvtap.iface} | awk -F':' '/${macvtap.iface}/{print $1}') \ '') allMacvtaps)} \ - ${concatStrings (imap1 (i: pciAddr: '' + ${concatStrings (imap1 (i: group: '' -device pcie-root-port,id=pci-passthrough${toString i},chassis=${toString i},slot=${toString i} \ - -device vfio-pci,host=${pciAddr},bus=pci-passthrough${toString i}${optionalString (i == 1) "${optionalString vmCfg.pci.vgaPassthrough ",x-vga=on"}${optionalString (vmCfg.pci.romFile != null) ",romfile=${vmCfg.pci.romFile}"}"} \ - '') vmCfg.pci.passthrough)} \ + '' + concatStrings (imap0 (j: pciAddr: '' + -device vfio-pci,host=${pciAddr},bus=pci-passthrough${toString i},addr=0x0.${pciFunctionOf pciAddr}${optionalString (length group > 1 && j == 0) ",multifunction=on"}${optionalString (i == 1 && j == 0) "${optionalString vmCfg.pci.vgaPassthrough ",x-vga=on"}${optionalString (vmCfg.pci.romFile != null) ",romfile=${vmCfg.pci.romFile}"}"} \ + '') group)) pciGroups)} \ ${concatMapStrings (usbDev: '' -device usb-host,vendorid=0x${usbDev.vendorId},productid=0x${usbDev.productId} \ '') vmCfg.usb.hostDevices} \