diff --git a/cli.nix b/cli.nix index e220dbb..b3d9f8f 100644 --- a/cli.nix +++ b/cli.nix @@ -33,7 +33,11 @@ pkgs.writeShellScriptBin "vmix" '' echo " --to-remote-disk SSH:DEV Stream to remote disk via SSH and expand partitions" echo " e.g. root@10.10.10.100:/dev/sda" echo " --ahci Use AHCI storage for vmix run (for laptop images)" - echo " --macos macOS image for vmix run (OpenCore/AppleSMC flags, AHCI)" + echo " --macos macOS image for vmix run (OpenCore/VirtualSMC flags, AHCI)" + echo " --applesmc with --macos: add QEMU's isa-applesmc (images built before 2026-09-09)" + echo " --share DIR with --macos: virtio-fs share (first: /Volumes/My Shared Files); repeatable" + echo " --home FILE with --macos: persistent home volume (qcow2, created+formatted if missing)" + echo " --qga PATH with --macos: guest agent socket path (default /tmp/vmix-qga-.sock)" echo " --vnc DISPLAY VNC instead of SDL for vmix run, e.g. :10 (port 5910) or 0.0.0.0:10" echo " --mac ADDR NIC MAC for vmix run (macOS: read from the image's ESP by default)" echo " -y, --yes Skip disk write confirmation" @@ -94,12 +98,21 @@ pkgs.writeShellScriptBin "vmix" '' RUN_MACOS=false RUN_VNC="" RUN_MAC="" + RUN_SHARES=() + RUN_HOME="" + RUN_HOME_IMAGE="macos.images.tahoe.upstream" + RUN_QGA="" while [[ ''${#} -gt 0 ]]; do case "$1" in --mem) RUN_MEM="$2"; shift 2 ;; --smp) RUN_SMP="$2"; shift 2 ;; --ahci) RUN_AHCI=true; shift ;; --macos) RUN_MACOS=true; shift ;; + --applesmc) RUN_APPLESMC=true; shift ;; + --share) RUN_SHARES+=("$2"); shift 2 ;; + --home) RUN_HOME="$2"; shift 2 ;; + --home-image) RUN_HOME_IMAGE="$2"; shift 2 ;; + --qga) RUN_QGA="$2"; shift 2 ;; --vnc) RUN_VNC="$2"; shift 2 ;; --mac) RUN_MAC="$2"; shift 2 ;; *) echo "Unknown option: $1"; exit 1 ;; @@ -134,10 +147,44 @@ pkgs.writeShellScriptBin "vmix" '' [[ -z "$RUN_MAC" || "$RUN_MAC" == "null" ]] && { RUN_MAC="52:54:00:c9:18:27"; echo "Warning: could not read MAC from image ESP, using $RUN_MAC"; } fi echo "macOS: yes (MAC $RUN_MAC)" + # Apple's built-in QEMU guest agent: guest-exec as root over this socket + [[ -z "$RUN_QGA" ]] && RUN_QGA="/tmp/vmix-qga-$$.sock" + rm -f "$RUN_QGA" + echo "Agent: $RUN_QGA (guest-exec as root)" + # virtio-fs shares: the first one auto-mounts at /Volumes/My Shared Files, the + # others are mounted with: mount -t virtiofs /Volumes/ + MACOS_SHARE_ARGS="" + MACOS_MEM_ARGS="" + i=0 + for SHARE in "''${RUN_SHARES[@]}"; do + i=$((i + 1)); SOCK="/tmp/vmix-vfs-$$-$i.sock"; rm -f "$SOCK" + TAG=$([[ $i -eq 1 ]] && echo "${macosQemu.automountTag}" || echo "share$i") + ${pkgs.virtiofsd}/bin/virtiofsd --socket-path="$SOCK" --shared-dir "$SHARE" --cache auto --sandbox none >/dev/null 2>&1 & + for t in $(seq 1 50); do [[ -S "$SOCK" ]] && break; sleep 0.2; done + MACOS_SHARE_ARGS="$MACOS_SHARE_ARGS -chardev socket,id=vfs$i,path=$SOCK -device vhost-user-fs-pci,chardev=vfs$i,tag=$TAG" + MACOS_MEM_ARGS="-object memory-backend-memfd,id=vmix-mem,size=''${RUN_MEM}M,share=on -numa node,memdev=vmix-mem" + echo "Share: $SHARE -> $([[ $i -eq 1 ]] && echo '/Volumes/My Shared Files' || echo "mount -t virtiofs $TAG ...")" + done + # persistent home volume (virtio-blk); created + formatted APFS by the PE if missing + MACOS_HOME_ARGS="" + if [[ -n "$RUN_HOME" ]]; then + if [[ ! -e "$RUN_HOME" ]]; then + echo "Home: creating $RUN_HOME (64G qcow2) and formatting it as APFS 'vmix-home' via the PE of $RUN_HOME_IMAGE ..." + ${pkgs.qemu}/bin/qemu-img create -q -f qcow2 "$RUN_HOME" 64G + FMT=$(${pkgs.nix}/bin/nix build --no-link --print-out-paths --impure --expr "let l = (builtins.getFlake \"${self}\").lib.${system}; in l.macos.formatVolume { image = l.$RUN_HOME_IMAGE; }") || { echo "Error: could not build the formatter"; exit 1; } + "$FMT" "$RUN_HOME" qcow2 || exit 1 + fi + HOME_FMT=$(${pkgs.qemu}/bin/qemu-img info --output=json "$RUN_HOME" | ${pkgs.jq}/bin/jq -r .format) + MACOS_HOME_ARGS="-drive id=home,if=none,format=$HOME_FMT,file=$RUN_HOME -device virtio-blk-pci,drive=home" + echo "Home: $RUN_HOME (mounted at /Users by images generalized with persistHome)" + fi echo "" exec ${pkgs.qemu}/bin/qemu-system-x86_64 \ + $MACOS_MEM_ARGS $MACOS_SHARE_ARGS $MACOS_HOME_ARGS \ + -device virtio-serial-pci,id=vmix-vser -chardev socket,path="$RUN_QGA",server=on,wait=off,id=vmix-qga -device virtserialport,chardev=vmix-qga,name=org.qemu.guest_agent.0 \ $VMIX_DISPLAY \ ${macosQemu.deviceArgs} ${macosQemu.vgaArgs} \ + $([[ "$RUN_APPLESMC" == true ]] && echo '-device isa-applesmc,osk="${macosQemu.osk}"') \ -accel kvm \ -machine type=q35 \ -cpu ${macosQemu.defaultCpu} \ diff --git a/lib/images/macos/README.md b/lib/images/macos/README.md index 9672842..256fe0e 100644 --- a/lib/images/macos/README.md +++ b/lib/images/macos/README.md @@ -1,138 +1,177 @@ -# vmix macOS images +# macOS images (Tahoe 26) -Unattended macOS (Tahoe / 26) VM images, built the same way as the Windows -images: `makeImage` installs the OS once, templates customize it by booting it, -`.generalize` creates the user and seals the image. +Pre-installed, Apple-ID-capable macOS VM images built the same way as the +Windows ones: `makeImage` (unattended install) → templates → `.generalize` +(user, hostname, fresh SMBIOS identity). Runs on QEMU/KVM with OpenCore. ``` -vmix build --image macos.images.tahoe.basic \ - --generalize username=sagar,password=secret,hostname=MAC,timezone=Europe/Zurich -vmix run ./result --macos --vnc :10 --mem 8192 # VNC on port 5910 +vmix build --image macos.images.tahoe.basic --generalize username=sagar,password=secret,hostname=MAC +vmix run ./result --macos --vnc :10 --mem 8192 ``` -## How it works +Nix: `macos.images.tahoe.{pe,upstream,basic,remote}` and +`.generalize { username; password; hostname; timezone; locale; seed; … }`. -| step | what happens | -|---|---| -| `fetchRecovery` | BaseSystem.dmg from Apple's recovery servers (fixed-output, pinned by sha256) | -| `installerPayload` | takes the App Store `InstallAssistant.pkg` (18 GB, pinned) apart on Linux: the app skeleton (pbzx/cpio) and the byte offset of `SharedSupport.dmg` | -| `makeOpenCore` | OSX-KVM's OpenCore ESP with a config.plist rewritten for this image: SMBIOS model, serial + MLB (`macserial`), UUID and ROM = NIC MAC (derived from a seed), NIC marked built-in | -| `makeImage` | one QEMU session: Recovery boots via OpenCore → `vm-driver.py` opens Terminal with keystrokes (Ctrl-F2 menu navigation, screen-settle detection + OCR of the menu bar) and types `sh /Volumes/VMIX/run.sh` → `vmix-install.sh` erases the disk, rebuilds `Install macOS Tahoe.app` (skeleton + `SharedSupport.dmg` copied from a raw disk mapped straight out of the pkg), runs `startosinstall --installpackage vmix-agent.pkg` → installer reboots through its phases → first boot runs the **vmix agent** which powers off. OpenCore is then copied into the image's EFI partition, so it boots standalone with OVMF | -| `customizeImage` | boots the image with a FAT volume `VMIX`; the agent (LaunchDaemon `ch.vmix.agent`) runs `vmix-run.sh` as root, writes `vmix-run.status`/`.log` back and shuts down | -| `templates.generalize` | user (admin) + auto-login (`/etc/kcpassword`), Setup Assistant suppressed, hostname, timezone, no sleep, APFS grown to the disk, then the agent removes itself; a fresh SMBIOS identity is written to the ESP | +## How it works: the vmix "PE" -The vmix agent replaces Windows' Audit Mode RunOnce; `.AppleSetupDone` replaces -the OOBE unattend. Everything on the host side runs inside `__noChroot` -derivations (KVM + `/tmp`), exactly like the Windows builders. +Apple's Recovery (`BaseSystem.dmg`, a plain journaled HFS+ volume) with **one +LaunchDaemon added** (`makeRecoveryPE`): at boot it mounts a `VMIX` volume and +runs `run.sh` from it as root, records the exit status and powers off. That is +the whole automation surface — the equivalent of Windows PE + Autounattend: -## Generalize options +* **no GUI is driven**: no OCR, no keystrokes, no screen layouts to learn per + macOS version; the hook is a launchd plist, stable across releases (same idea + as AutoNBI/Imagr NetBoot images). +* **observable**: the guest prints `VMIX-*` markers to `/dev/console`, which the + build reads from QEMU's serial log (`boot-args serial=3 -v`). Kernel panics and + reboots show up there too. Screenshots are still taken for debugging. +* **offline**: no NIC during the install, and the guest blackholes Apple's + install/verify endpoints so `startosinstall` never waits on the network. The + only inputs are the pinned `InstallAssistant.pkg` and `BaseSystem.dmg`. +* **everything else happens offline from the PE too**: templates and generalize + mount the image's Data volume (rw) and System volume (ro) and edit them + (`dscl -f` for users, `plutil` for preferences) — the installed macOS is + never booted for customization, so nothing depends on launchd/BTM approval, + first-boot agents or auto-login inside the guest. One PE boot ≈ 30 s. -`username password fullName autoLogon hostname locale timezone delayOobeRun` -as for Windows (`bgColor` is accepted but ignored), plus the SMBIOS identity: -`model serial mlb uuid mac seed`. Anything unset is generated: serial/MLB by -macserial (random per build), MAC and UUID deterministically from `seed` -(default `hostname-username`). `vmix macserial --model MacPro7,1` prints a -ready-to-paste set. +### Pipeline -`delay-oobe-run=true` creates no user and re-arms Setup Assistant for the first -real boot. - -## Apple ID / iMessage - -The image satisfies what Dortania lists for iServices: unique serial + MLB for a -Tahoe-supported model (`MacPro7,1` by default; `iMac20,1/2`, -`MacBookPro16,x` also work), SystemUUID, ROM equal to en0's MAC, and en0 marked -built-in (the NIC is pinned to `PciRoot(0x0)/Pci(0x12,0x0)`). The NixOS module -and `vmix run --macos` use the MAC recorded in the image (`EFI/vmix/vmix.json`). -Give each deployed VM its own generalized image (different `seed`, or explicit -`serial=`/`mlb=`) — two VMs with the same identity will be blocked. - -## Runtime - -* `vmix run --macos [--vnc :N] [--mac ..]` -* NixOS module: `disks.os.file = vmixLib.macos.images.tahoe.basic.generalize {...}` - is auto-detected (`_vmixOsType = "macos"`): Skylake-Client CPU spoof, AppleSMC, - USB keyboard/tablet, AHCI system disk, VMware SVGA, pinned NIC with the image's MAC. - `macos.cpu`, `macos.mac`, `macos.enable` override the defaults. -* `vmix copy` writes the image to a disk but cannot grow APFS from Linux - (`diskutil apfs resizeContainer disk0s2 0` in macOS afterwards). - -## Debugging a build - -Screenshots (`NNN-.png`), `driver.log` and the QMP socket of every VM -session are in `/tmp/vmix-macos//` on the build host. The guest logs -(`install.log`, `vmix-run.log`, `vmix-agent.log`) are printed at the end of the -build. Pass `vncDisplay = ":10"` to `makeImage`/`customizeImage` (or -`--generalize vncDisplay=:10`) to watch live; with a `DISPLAY` an SDL window -is used as for Windows. - -## Updating pins (`upstream.json`) - -* installer: URL + SRI hash of a newer `InstallAssistant.pkg` - (`nix store prefetch-file --name InstallAssistant.pkg `; Mr. Macintosh's - database lists Apple's URLs) -* recovery: Apple serves the current build for the board id, so the sha256 - changes with each point release — copy the "got:" hash from the failed build -* opencore: OSX-KVM `OpenCore.qcow2` at a commit; OpenCorePkg release zip (macserial/ocvalidate) - -## Known limits - -* The Recovery bootstrap depends on keyboard navigation of the Recovery UI - (Ctrl-F2 → Utilities → Terminal). It self-corrects with screenshots + OCR and - falls back to a blind sequence, but a Recovery UI change would need - `vm-driver.py` adjusted. -* Hosts must run KVM with an AVX2-capable CPU (Intel or AMD; the guest sees a - Skylake). `sandbox = relaxed` and the `kvm` system feature, as for Windows. -* Software updates inside the VM are disabled by the `noUpdates` template - (OTA updates in a VM need the RestrictEvents kext). - -## Current status (2026-09-09): working offline install - -`macos.images.tahoe.upstream` builds a bootable, installed macOS Tahoe 26.6.2 -qcow2 **fully offline** on the KVM host — no dependency on Apple's servers at build -time, just the pinned local `InstallAssistant.pkg` and `BaseSystem.dmg`. The -finished image boots standalone (OpenCore from its own ESP) to the macOS -loginwindow. Serial/MLB/UUID/ROM are per-image for Apple ID / iMessage. - -How the install is driven (`vm-driver.py`, all by screenshot + OCR over QMP): - -* The whole `InstallAssistant.pkg` is mapped as a raw disk (it is a "pkgdmg": - xar + koly footer) and `dd`'d byte-exact into the app as `SharedSupport.dmg` — - extracting the bare xar member fails startosinstall with "pkgdmg missing a footer". -* No NIC during install + `/etc/hosts` blackhole of Apple's install/verify - endpoints, so `startosinstall`'s network calls fail fast instead of hanging — - offline prepare, no external dependency. `SecureBootModel=Disabled` lets the - sealed volume install without online personalization. -* The recovery display is kept awake with a tiny mouse jiggle (a lone keypress - does not reset display sleep, and the sleeping display swallows the menu-nav - keystrokes); the settle detector uses a coarse fingerprint so the jiggling - cursor is not seen as a screen change. -* startosinstall prepare is intermittently slow/stalls; a guest watchdog kills and - re-erases/retries an attempt that stalls or runs > 9 min. -* First boot in QEMU intermittently hangs at the Apple logo; a disk-aware watchdog - (`--progress-file`) issues a QMP `system_reset` only when the screen is dark AND - the disk is idle, so a slow-but-working boot is never interrupted. -* The install reaching the (bright) loginwindow is detected by brightness (the - faint gray "password" text does not OCR) and the driver powers the VM down — - the image is installed. macOS `shutdown -h now` halts to black without an ACPI - power-off, so a black+disk-idle screen is also treated as a completed halt. -* OpenCore is then copied into the image's own ESP so it boots standalone with OVMF. +1. `makeRecoveryPE` — BaseSystem.dmg → raw HFS+ image + `ch.vmix.pe` daemon. +2. `makeImage` — QEMU with: OpenCore boot disk (build variant with serial + console), the PE, the empty target disk, the VMIX volume (`vmix-install.sh`, + installer app skeleton) and the whole `InstallAssistant.pkg` mapped as a raw + disk. The guest script erases the target as APFS, unpacks the app and `dd`s + the pkg into it as `SharedSupport.dmg` (it is a "pkgdmg": xar + koly footer; + the bare xar member fails with "pkgdmg is missing a footer"), then runs + `startosinstall`, which reboots itself through the install phases. The + installed system's first boot ends at the loginwindow: the driver detects the + bright screen and powers the VM down. OpenCore is then copied into the image's + own ESP so it boots with plain OVMF. +3. `customizeImage` — boots the PE with the image attached (OpenCore + `ScanPolicy` restricted to HFS+ on SATA, so only the PE can boot) and runs the + template script with `$SYS`/`$DATA` mounted. `pe-lib.sh` has the helpers. +4. `templates/generalize.nix` — user (dscl, admin, home from the user template), + auto-login (`kcpassword`), Setup Assistant suppression, hostname, locale, + timezone, keyboard type, container resize, fresh SMBIOS via a new OpenCore + ESP (`serial`/`mlb` from macserial, MAC + UUID from `seed`). ### Recovery source -`recovery.file` in `upstream.json` points at a content-addressed store path for the -verified Tahoe `BaseSystem.dmg` (Apple's CDN load-balances Sequoia/Tahoe during the -rollout, so a plain fetch is non-deterministic). Reproduce it on any host with -`nix store add-path --name macos-tahoe-BaseSystem.dmg BaseSystem.dmg` (same path -from the same bytes). Set `recovery.sha256` and remove `recovery.file` to fetch it -from Apple instead (subject to the CDN rollout). +`recovery.file` in `upstream.json` points at a content-addressed store path for +the verified Tahoe `BaseSystem.dmg` (Apple's CDN load-balances Sequoia/Tahoe +during the rollout, so a plain fetch is non-deterministic). Reproduce it on any +host with `nix store add-path --name macos-tahoe-BaseSystem.dmg BaseSystem.dmg`. +Drop `recovery.file` to fetch from Apple instead (`fetchRecovery` retries until +the pinned hash matches). -### Not yet done: generalize / user creation +## Reliability -The base image installs and boots to loginwindow. `.generalize` (user creation, -auto-login, hostname) relies on the vmix agent LaunchDaemon running on first boot, -but macOS Ventura+ Background Task Management does not auto-run a headless -third-party daemon, and neither the pkg `launchctl bootstrap` (installer domain -only) nor a cron `@reboot` reliably triggered it. The robust next step is to inject -the user record + settings offline from the agent pkg's postinstall (which runs as -root on the target during install), instead of a first-boot daemon. +Things QEMU does intermittently, and what handles each (all in `vm-driver.py` +and `vmix-install.sh`; every event is logged with a reason): + +* `startosinstall` prepare stalls or crawls — the guest kills and retries it on a + freshly erased target (free-space watchdog + time cap). +* the installer comes back to the PE instead of the install phase — the PE + counts boots and simply re-runs the install (max 3). +* the installed system hangs at the Apple logo on first boot — a `system_reset` + is issued only when the screen is dark and frozen **and** disk and serial + console are idle, so a slow-but-working boot is never interrupted. +* macOS `shutdown -h` halts to a black screen without an ACPI power-off — an + idle black screen counts as a completed halt. +* a kernel panic (seen on the serial console) resets the VM. +* a wedged run fails at the 4 h timeout instead of hanging. + +`tools/soak.sh macos.images.tahoe.upstream 3` rebuilds an image N +times and tabulates outcome, duration, boots, resets, panics and retries. +Measured 2026-09-09 on the build host (Ryzen 7 7840HS, ZFS), Tahoe 26.6.2, +VirtualSMC-only, PE install — 3 of 3 builds completed: + +| run | minutes | kernel boots | prepare tries | panics (self-recovered) | reboot deaths | +|-----|---------|--------------|---------------|-------------------------|---------------| +| 1 | 30 | 8 | 1 | 2 | 0 | +| 2 | 26 | 7 | 1 | 1 | 0 | +| 3 | 26 | 7 | 1 | 1 | 0 | + +What still happens: at roughly one in ten guest-initiated reboots the guest +either panics (GPF in launchd/kernel_task context shortly after `MACH Reboot` +or within the first 15 s of the next boot — tmpfs/APFS/zone corruption +signatures, i.e. memory or register state, not one driver) or never comes back +(dead after `IOPlatformHaltRestartAction`). XNU reboots itself after a panic; +the driver resets a dead guest after 60 s, so builds complete. A device bisect +(`tools`-style 10–30 PE reboots per variant: VMware SVGA vs std VGA, no HDA, +EHCI input, 1 vCPU) showed the rate is independent of the emulated devices and +of SMP; Haswell-noTSX does not boot Tahoe. Host: AMD Zen 4, kvm_amd, Intel +Skylake-Client vCPU model — the FPU-context-switch panic points at XSAVE state +handling on that combination. Not fixed; a `vmix run` VM that hangs on Restart +must be reset from the host. + +## Debugging + +`/tmp/vmix-macos//` on the build host: `driver.log`, `serial.log` +(kernel + `VMIX-*` markers), periodic PNG screenshots, `qmp.sock`. +`vmix-run.log` / `system-install.log` from the VMIX volume are printed at the +end of the build. Add `vncDisplay = ":10"` to watch. + +## QEMU profile + +`helpers/qemu.nix`: q35, `Skylake-Client` CPU spoof (works on AMD), +AppleSMC with the OSK, XHCI keyboard/tablet, AHCI disks, VMware SVGA, +virtio-net pinned to `PciRoot(0x0)/Pci(0x12,0x0)` so OpenCore marks it built-in +(en0, required for Apple ID / iMessage). SMBIOS `MacPro7,1` with four DIMMs +described (avoids the "Memory Modules Misconfigured" warning). + +OpenCore comes from OSX-KVM's proven ESP, with Lilu / VirtualSMC / +WhateverGreen replaced by current releases (`upstream.json` → `opencore.kexts`): +the versions OSX-KVM ships disable themselves on macOS 26, and without +VirtualSMC the guest's restart path panics on QEMU's SMC stub +(`SMCWDT smcWriteKey kSMCBadCommand`, nested panic after `MACH Reboot`). +For the same reason QEMU's `isa-applesmc` is not used any more: its presence +makes VirtualSMC step aside ("multiple devices present"); VirtualSMC carries +the OSK itself. Images built before this change still need the stub: +`vmix run --macos --applesmc`. RestrictEvents (`revpatch=memtab`) silences +MacPro7,1's "Memory Modules Misconfigured" at login. + +## Guest agent, shares, persistent home, online templates + +macOS 13+ ships **Apple's own QEMU guest agent** (`/usr/libexec/AppleQEMUGuestAgent`, +started by launchd when a virtio console port named `org.qemu.guest_agent.0` +appears). It is Apple-signed, needs no approval, and offers `guest-exec` as +root plus `guest-file-*`. vmix uses it everywhere an in-guest agent is needed: + +* `vmix run --macos` and the NixOS module attach it by default + (`/tmp/vmix-qga-.sock`, `/run/vmix/qga-.sock`); talk to it with any + QGA client, e.g. `printf '{"execute":"guest-exec","arguments":{"path":"/usr/bin/id","capture-output":true}}\n' | socat - UNIX-CONNECT:`. +* **online templates** (`bootScript`): `customizeImage` boots the image with the + agent, runs the script as root (network available, `as_user ` runs inside + the logged-in user's session), then shuts down through the agent. + `templates.software.script { name; script; }`, + `templates.software.homebrew { formulae; casks; }`, + `templates.profile.settings { hideWidgets; wallpaper; dockApps; dockAutohide; + darkMode; showHiddenFiles; }` (wallpaper via the pinned `desktoppr`; Apple + Events / `osascript` do not work headless — TCC automation consent). +* **offline software templates** run in the PE: `templates.software.pkg { name; + src; }` (`installer -target`), `templates.software.app { name; src; }`. + +`AppleVirtIO.kext` (x86 Tahoe) drives virtio-fs, 9p, block, console, input, +net, sound, balloon, vsock — QEMU's modern virtio-pci devices work as-is: + +* **shared folders**: virtio-fs (`virtiofsd` + `vhost-user-fs-pci`, shared + memory backend). The tag `com.apple.virtio-fs.automount` is mounted by macOS + itself at `/Volumes/My Shared Files`; further tags are mounted with + `mount -t virtiofs ` — the module does that through the guest agent + for every `shares.` beyond the first. `vmix run --macos --share DIR`. + (9p does not automount on macOS; the Linux `-virtfs` path is not used.) +* **ephemeral OS disk + persistent home**: `generalize { persistHome = true; }` + gives the account its home directory on an APFS volume labelled `vmix-home` + (`NFSHomeDirectory = /Volumes/vmix-home/`; macOS refuses mounts over + `/Users`, which is a firmlink). The host provides a virtio-blk disk + (`macos.homeDisk` in the module, `--home FILE` in the CLI: qcow2/raw file or + zvol) that `formatVolume` formats as APFS `vmix-home` by booting the PE for + ~35 s on first use; diskarbitrationd mounts it before login and loginwindow + creates the home directory there on first login. The OS disk can then run + with `snapshot=on` (`disks.os.persist = false`). +* **SPICE**: `-vga vmware` (or `std`) is kept as the display device — macOS has + no QXL/virtio-gpu driver; USB redirection channels work as for other guests + (`spice.usbRedir`); there is no vdagent for macOS (no clipboard sharing). + virtio keyboard/tablet (`AppleVirtIOInput`) are available as + `qemu.virtioInputArgs` but the USB HID pair is the default. diff --git a/lib/images/macos/default.nix b/lib/images/macos/default.nix index 6b36d2f..c6dd8bd 100644 --- a/lib/images/macos/default.nix +++ b/lib/images/macos/default.nix @@ -9,18 +9,20 @@ let fetchRecovery = import ./helpers/fetchRecovery.nix { inherit pkgs upstream; }; installerPayload = import ./helpers/installerPayload.nix { inherit pkgs lib; }; makeOpenCore = import ./helpers/makeOpenCore.nix { inherit pkgs lib upstream macserial qemu; }; + makeBootDisk = import ./helpers/makeBootDisk.nix { inherit pkgs lib; }; + makeRecoveryPE = import ./helpers/makeRecoveryPE.nix { inherit pkgs lib; }; makeVmixVolume = import ./helpers/makeVmixVolume.nix { inherit pkgs lib; }; - makeAgentPkg = import ./helpers/makeAgentPkg.nix { inherit pkgs lib; }; installBootloader = import ./helpers/installBootloader.nix { inherit pkgs lib; }; vmixReadback = import ./helpers/vmix-readback.nix { inherit pkgs lib; }; vmDriver = ./helpers/vm-driver.py; makeImage = import ./helpers/makeImage.nix { - inherit pkgs lib qemu ident installerPayload makeOpenCore makeVmixVolume makeAgentPkg installBootloader vmixReadback vmDriver; + inherit pkgs lib qemu ident installerPayload makeOpenCore makeBootDisk makeVmixVolume installBootloader vmixReadback vmDriver; }; customizeImage = import ./helpers/customizeImage.nix { - inherit pkgs lib qemu ident makeVmixVolume makeOpenCore installBootloader vmixReadback vmDriver; + inherit pkgs lib qemu ident makeVmixVolume makeOpenCore makeBootDisk installBootloader vmixReadback vmDriver; }; customizeImageFold = builtins.foldl' customizeImage; + formatVolume = import ./helpers/formatVolume.nix { inherit pkgs lib qemu makeVmixVolume makeBootDisk vmDriver; }; templates = import ./templates { inherit pkgs lib; }; }; diff --git a/lib/images/macos/guest/agent.sh b/lib/images/macos/guest/agent.sh deleted file mode 100644 index 3e2ba02..0000000 --- a/lib/images/macos/guest/agent.sh +++ /dev/null @@ -1,42 +0,0 @@ -#!/bin/sh -# vmix agent: LaunchDaemon that runs at every boot as root (installed by the vmix -# agent pkg via startosinstall --installpackage). If a volume named VMIX carrying -# vmix-run.sh is attached, run it, record the result on the volume and power off. -# Without the volume it is a no-op (normal boot). Counterpart of the Windows Audit -# Mode RunOnce script; the generalize step removes it once the image is sealed. -LOG=/var/log/vmix-agent.log -exec >>"$LOG" 2>&1 -echo "=== vmix agent: $(date) ===" -# The agent pkg bootstraps this daemon during the OS install (to approve it past -# Background Task Management, so launchd runs it at first boot). Don't do the job -# in that installer environment — only on the installed system's first boot. -if pgrep -x bootinstalld >/dev/null 2>&1 || pgrep -qx "Installer Progress" 2>/dev/null \ - || [ -d /System/Volumes/Update/mnt1 ]; then - echo "vmix agent: OS installer is running, skipping" - exit 0 -fi -# let DiskArbitration settle so the VMIX volume is mountable -sleep 5 -V=/Volumes/VMIX -i=0 -while [ ! -f "$V/vmix-run.sh" ] && [ $i -lt 30 ]; do - diskutil mount VMIX >/dev/null 2>&1 - sleep 2 - i=$((i + 1)) -done -if [ ! -f "$V/vmix-run.sh" ]; then - echo "vmix agent: no VMIX volume, normal boot" - exit 0 -fi -echo "vmix agent: running vmix-run.sh" -cd "$V" || exit 1 -sh "$V/vmix-run.sh" >"$V/vmix-run.log" 2>&1 -rc=$? -echo "vmix agent: vmix-run.sh exited $rc" -echo "$rc" >"$V/vmix-run.status" -cp "$LOG" "$V/vmix-agent.log" 2>/dev/null -cp /var/log/vmix-agent-install.log "$V/vmix-agent-install.log" 2>/dev/null -sync -sleep 2 -diskutil unmount force "$V" >/dev/null 2>&1 -shutdown -h now diff --git a/lib/images/macos/guest/ch.vmix.agent.plist b/lib/images/macos/guest/ch.vmix.pe.plist similarity index 64% rename from lib/images/macos/guest/ch.vmix.agent.plist rename to lib/images/macos/guest/ch.vmix.pe.plist index c518fa4..a83065a 100644 --- a/lib/images/macos/guest/ch.vmix.agent.plist +++ b/lib/images/macos/guest/ch.vmix.pe.plist @@ -3,17 +3,17 @@ Label - ch.vmix.agent + ch.vmix.pe ProgramArguments - /bin/sh - /Library/vmix/agent.sh + /bin/bash + /usr/libexec/vmix/pe.sh RunAtLoad StandardOutPath - /var/log/vmix-agent.log + /dev/console StandardErrorPath - /var/log/vmix-agent.log + /dev/console diff --git a/lib/images/macos/guest/pe-lib.sh b/lib/images/macos/guest/pe-lib.sh new file mode 100644 index 0000000..08b09c1 --- /dev/null +++ b/lib/images/macos/guest/pe-lib.sh @@ -0,0 +1,51 @@ +# vmix PE helpers, sourced by run.sh scripts running in the recovery. +# Expects V=/Volumes/VMIX (set by pe.sh) and VOLUME_NAME from vmix.conf. +V=${V:-/Volumes/VMIX} +[ -f "$V/vmix.conf" ] && . "$V/vmix.conf" +VOLUME_NAME=${VOLUME_NAME:-Macintosh HD} + +pe_log() { echo "VMIX: $*"; } +pe_fail() { echo "VMIX-FAIL: $*"; exit 1; } + +# Mount the installed system's APFS volume group (System read-only, Data rw) and +# export SYS / DATA mount points plus SYS_ID / DATA_ID device identifiers. +pe_mount_target() { + local list; list=$(diskutil list) + DATA_ID=$(echo "$list" | awk -v n="APFS Volume $VOLUME_NAME - Data" 'index($0, n) {print $NF; exit}') + SYS_ID=$(echo "$list" | awk -v n="APFS Volume $VOLUME_NAME " '!/ - Data/ && index($0, n) {print $NF; exit}') + [ -n "$DATA_ID" ] && [ -n "$SYS_ID" ] || { pe_log "target volumes not found"; echo "$list"; return 1; } + diskutil mount "$SYS_ID" >/dev/null 2>&1 || true + diskutil mount "$DATA_ID" >/dev/null 2>&1 || true + SYS=$(diskutil info "$SYS_ID" | sed -n 's/^ *Mount Point: *//p') + DATA=$(diskutil info "$DATA_ID" | sed -n 's/^ *Mount Point: *//p') + [ -d "$DATA/private/var/db" ] || { pe_log "Data volume not mounted (SYS=[$SYS] DATA=[$DATA])"; return 1; } + pe_log "target mounted: SYS=[$SYS] DATA=[$DATA]" + export SYS DATA SYS_ID DATA_ID +} + +pe_unmount_target() { + sync + diskutil unmount "$DATA_ID" >/dev/null 2>&1 || true + diskutil unmount "$SYS_ID" >/dev/null 2>&1 || true +} + +# plist helpers on files of the (offline) target: create the file if missing. +pe_plist_set() { # FILE KEYPATH TYPE VALUE (TYPE: string|bool|integer|float) + local f=$1 k=$2 t=$3 v=$4 + [ -f "$f" ] || plutil -create xml1 "$f" + plutil -replace "$k" "-$t" "$v" "$f" +} +pe_plist_dict() { # FILE KEYPATH — make sure a dictionary exists at KEYPATH + local f=$1 k=$2 + [ -f "$f" ] || plutil -create xml1 "$f" + plutil -extract "$k" xml1 -o /dev/null "$f" >/dev/null 2>&1 || plutil -insert "$k" -dictionary "$f" +} +# launchd service override on the target (disabled.plist): pe_service LABEL true|false +pe_service_disabled() { + local f="$DATA/private/var/db/com.apple.xpc.launchd/disabled.plist" + mkdir -p "$(dirname "$f")" + pe_plist_set "$f" "$1" bool "$2" +} +# version of the installed system +pe_target_version() { plutil -extract ProductVersion raw -o - "$SYS/System/Library/CoreServices/SystemVersion.plist" 2>/dev/null; } +pe_target_build() { plutil -extract ProductBuildVersion raw -o - "$SYS/System/Library/CoreServices/SystemVersion.plist" 2>/dev/null; } diff --git a/lib/images/macos/guest/pe.sh b/lib/images/macos/guest/pe.sh new file mode 100755 index 0000000..ecb8f29 --- /dev/null +++ b/lib/images/macos/guest/pe.sh @@ -0,0 +1,40 @@ +#!/bin/bash +# vmix PE hook. Runs as root from launchd when the patched Recovery boots +# (injected by makeRecoveryPE). If a VMIX volume is attached it runs +# /Volumes/VMIX/run.sh, records the exit status on the volume and powers off; +# without one it does nothing and the recovery behaves normally. +# Everything printed here goes to /dev/console, i.e. the host's serial log. +exec >/dev/console 2>&1 +echo "VMIX-PE: hook started $(date) uid=$(id -u)" +V=/Volumes/VMIX +i=0 +while [ ! -f "$V/run.sh" ] && [ $i -lt 90 ]; do + diskutil mount VMIX >/dev/null 2>&1 + sleep 2; i=$((i + 1)) +done +if [ ! -f "$V/run.sh" ]; then + echo "VMIX-PE: no VMIX volume, leaving the recovery alone" + exit 0 +fi +echo "VMIX-PE: VMIX mounted after $i retries" +caffeinate -dimsu -t 86400 >/dev/null 2>&1 & +[ -f "$V/vmix.conf" ] && . "$V/vmix.conf" +# certificate checks need a sane clock; a fresh VM RTC can be off +[ -n "${BUILD_DATE:-}" ] && date -u "$BUILD_DATE" >/dev/null 2>&1 && echo "VMIX-PE: clock set to $(date -u)" +export V +cd "$V" +echo "VMIX-PE: running run.sh" +/bin/bash "$V/run.sh" 2>&1 | tee "$V/vmix-run.log" +rc=${PIPESTATUS[0]} +echo "$rc" > "$V/vmix-run.status" +echo "VMIX-PE: run.sh exited $rc" +if [ -f "$V/vmix-reboot" ]; then + rm -f "$V/vmix-reboot"; sync + echo "VMIX-PE: rebooting as requested" + reboot + exit 0 +fi +sync; sleep 1 +diskutil unmount force "$V" >/dev/null 2>&1 +echo "VMIX-PE-DONE rc=$rc" +shutdown -h now diff --git a/lib/images/macos/guest/vmix-install.sh b/lib/images/macos/guest/vmix-install.sh index 486f28d..8374331 100644 --- a/lib/images/macos/guest/vmix-install.sh +++ b/lib/images/macos/guest/vmix-install.sh @@ -1,88 +1,65 @@ -#!/bin/sh -# vmix: automated macOS install. Runs inside macOS Recovery's Terminal, started -# by vm-driver.py which types "sh /Volumes/VMIX/run.sh" for us. -# -# 1. erase the target disk (found by size) as APFS "Macintosh HD" -# 2. rebuild "Install macOS .app": app skeleton from installer-app.tar -# (host-extracted Payload) + SharedSupport.dmg = the WHOLE InstallAssistant.pkg -# dd'd byte-exact from a raw disk (Apple's own postinstall hardlinks the pkg -# there: it is a "pkgdmg" whose koly footer points at the dmg inside; the bare -# xar member fails startosinstall with "pkgdmg is missing a footer") -# 3. startosinstall unattended, with the vmix agent pkg as --installpackage -# 4. startosinstall reboots itself into the install phase; the vmix agent pkg -# installs during that phase and runs on the installed system's first boot -# -# On first boot of the installed system the agent runs /Volumes/VMIX/vmix-run.sh -# and powers off, which ends the QEMU session on the host. - -# macOS Recovery invokes us as `sh` (bash in POSIX mode, no process substitution); -# re-exec once under bash so `>(tee ...)` and other bashisms work. -if [ -z "${VMIX_REEXEC:-}" ]; then VMIX_REEXEC=1 exec bash "$0" "$@"; fi - -V="/Volumes/VMIX" -# tee to the Terminal (visible in host screenshots) and to a log on the volume -exec > >(tee "$V/install.log") 2>&1 +#!/bin/bash +# vmix unattended macOS install, run by the PE hook (pe.sh) as root in the +# Recovery with /Volumes/VMIX mounted (V). Needs vmix.conf: TARGET_BYTES, +# PKG_BYTES, PKG_DISK_BYTES, APP_NAME, VOLUME_NAME. +# 1. find the target disk and the SharedSupport (InstallAssistant.pkg) disk by size +# 2. erase the target as APFS, unpack the installer app, dd the whole pkg into it +# as SharedSupport.dmg (a "pkgdmg", startosinstall checks its koly footer) +# 3. startosinstall prepares, then reboots itself into the install phase; the +# installed system's first boot ends at the loginwindow (the host powers off) +# Never returns on success; a return means failure (the PE records the status). set -x -. "$V/vmix.conf" -# keep the recovery display awake so the host driver can watch the screen -caffeinate -dimsu -t 86400 >/dev/null 2>&1 & -pmset -a displaysleep 0 sleep 0 >/dev/null 2>&1 || true - +. "$V/pe-lib.sh" fail() { - echo "vmix-install: FAIL: $*" - cp /var/log/install.log "$V/system-install.log" 2>/dev/null || true - echo 1 >"$V/install.status" + echo "VMIX-FAIL: $*" + cp /var/log/install.log "$V/system-install.log" 2>/dev/null sync - sleep 2 - shutdown -h now 2>/dev/null || halt 2>/dev/null || true exit 1 } +# each boot into the PE with the install still pending is one attempt +ATTEMPT=$(( $(cat "$V/install.attempt" 2>/dev/null || echo 0) + 1 )) +echo "$ATTEMPT" > "$V/install.attempt"; sync +echo "VMIX-INSTALL: attempt $ATTEMPT (boot into the PE)" +[ "$ATTEMPT" -le 3 ] || fail "the installer keeps coming back to the PE ($ATTEMPT boots)" -# whole-disk identifier (diskN) whose size in bytes is exactly $1 +# --- 1. disks by exact size disk_by_size() { - for d in $(diskutil list | grep -oE '^/dev/disk[0-9]+'); do - s=$(diskutil info "$d" | sed -n 's/.*Disk Size:.*(\([0-9][0-9]*\) Bytes).*/\1/p') - [ "$s" = "$1" ] && { echo "${d#/dev/}"; return 0; } + for d in $(diskutil list | grep -oE '^/dev/disk[0-9]+' | sort -u); do + if [ "$(diskutil info "$d" | sed -n 's/.*Disk Size:.*(\([0-9]*\) Bytes).*/\1/p')" = "$1" ]; then + echo "${d#/dev/}"; return 0 + fi done return 1 } +TARGET=$(disk_by_size "$TARGET_BYTES") || fail "target disk of $TARGET_BYTES bytes not found" +SSDISK=$(disk_by_size "$PKG_DISK_BYTES") || fail "SharedSupport disk of $PKG_DISK_BYTES bytes not found" +echo "VMIX-INSTALL: target=$TARGET sharedsupport=$SSDISK" -echo "vmix-install: $(date) app=$APP_NAME volume=$VOLUME_NAME" -TARGET=$(disk_by_size "$TARGET_BYTES") || fail "target disk ($TARGET_BYTES bytes) not found" -SSDISK=$(disk_by_size "$PKG_DISK_BYTES") || fail "installer pkg disk ($PKG_DISK_BYTES bytes) not found" -echo "vmix-install: target=$TARGET sharedsupport=$SSDISK" - -# --- 1. erase the target disk as an APFS volume -diskutil eraseDisk APFS "$VOLUME_NAME" GPT "$TARGET" || fail "eraseDisk $TARGET" +# --- 2. target volume + installer app (the pkg payload skeleton + SharedSupport.dmg) VOL="/Volumes/$VOLUME_NAME" -[ -d "$VOL" ] || fail "$VOL not mounted" - -# --- 2. rebuild the installer app on the target volume -tar -xf "$V/installer-app.tar" -C "$VOL" || fail "untar installer-app.tar" APP="$VOL/$APP_NAME" -SOI="$APP/Contents/Resources/startosinstall" -[ -x "$SOI" ] || fail "startosinstall missing in $APP" SS="$APP/Contents/SharedSupport/SharedSupport.dmg" -mkdir -p "$APP/Contents/SharedSupport" -FULL=$((PKG_BYTES / 1048576)) -REM=$((PKG_BYTES % 1048576)) -dd if="/dev/r$SSDISK" of="$SS" bs=1048576 count=$FULL || fail "dd SharedSupport.dmg" -if [ "$REM" -gt 0 ]; then - dd if="/dev/r$SSDISK" bs=1048576 skip=$FULL count=1 2>/dev/null | dd bs=1 count=$REM >>"$SS" || fail "dd SharedSupport.dmg tail" -fi -[ "$(stat -f %z "$SS")" = "$PKG_BYTES" ] || fail "SharedSupport.dmg size mismatch: $(stat -f %z "$SS") != $PKG_BYTES" -tail -c 512 "$SS" | grep -qa koly || fail "SharedSupport.dmg has no UDIF koly footer" -chflags -h norestricted "$SS" 2>/dev/null || true -echo "vmix-install: app=$APP SharedSupport.dmg=$(stat -f %z "$SS") bytes" +prepare_target() { + diskutil eraseDisk APFS "$VOLUME_NAME" GPT "$TARGET" || fail "eraseDisk" + [ -d "$VOL" ] || fail "$VOL not mounted after erase" + tar -xf "$V/installer-app.tar" -C "$VOL" || fail "untar installer app" + [ -x "$APP/Contents/Resources/startosinstall" ] || fail "startosinstall missing from $APP" + mkdir -p "$APP/Contents/SharedSupport" + FULL=$(( PKG_BYTES / 1048576 )); REM=$(( PKG_BYTES % 1048576 )) + echo "VMIX-INSTALL: copying SharedSupport.dmg ($PKG_BYTES bytes) from /dev/r$SSDISK" + dd if="/dev/r$SSDISK" of="$SS" bs=1048576 count=$FULL || fail "dd SharedSupport" + [ "$REM" -gt 0 ] && { dd if="/dev/r$SSDISK" bs=1048576 skip=$FULL count=1 | dd bs=1 count=$REM >> "$SS"; } || true + [ "$(stat -f %z "$SS")" = "$PKG_BYTES" ] || fail "SharedSupport.dmg size $(stat -f %z "$SS") != $PKG_BYTES" + tail -c 512 "$SS" | grep -qa koly || fail "SharedSupport.dmg has no koly footer" + chflags -h norestricted "$SS" 2>/dev/null || true + sync +} +prepare_target +SOI="$APP/Contents/Resources/startosinstall" +echo "VMIX-INSTALL: app ready, clock $(date -u)" -# macOS certificate validation needs a sane clock; a fresh VM RTC can be wrong. -echo "vmix-install: guest clock is $(date) (UTC $(date -u))" -if [ -n "${BUILD_DATE:-}" ]; then - date -u "$BUILD_DATE" && echo "vmix-install: set clock to $(date)" -fi - -# Blackhole Apple's install/verify endpoints so osinstallersetupd's network calls -# fail immediately instead of timing out (prepare otherwise crawls). Fully offline. +# Offline install: no NIC is attached. Blackhole Apple's install/verify endpoints +# too, so osinstallersetupd's requests fail immediately instead of timing out. for d in swscan.apple.com swcdn.apple.com swdist.apple.com swquery.apple.com \ gs.apple.com gsa.apple.com gdmf.apple.com mesu.apple.com xp.apple.com \ albert.apple.com captive.apple.com deviceservices-external.apple.com \ @@ -90,50 +67,34 @@ for d in swscan.apple.com swcdn.apple.com swdist.apple.com swquery.apple.com \ ocsp2.apple.com valid.apple.com; do echo "127.0.0.1 $d" >> /etc/hosts done -echo "vmix-install: blackholed Apple install endpoints for a fast offline prepare" -# --- 3. unattended install. startosinstall prepares then reboots the machine -# itself into the install phase. Prepare intermittently stalls (~46% — an online -# verify/personalization step through the VM's NAT), so a watchdog kills and -# retries startosinstall if the target volume makes no write progress for a while. -# The vmix agent pkg installs during the install phase and runs on first boot. -# quote args properly — $VOL contains a space ("Macintosh HD") +# --- 3. startosinstall prepares (~5 min) then reboots the machine itself into the +# install phase; it never returns on success. Prepare is intermittently slow in +# QEMU, so an attempt that stalls or runs too long is killed and retried on a +# freshly erased target. run_soi() { "$SOI" --volume "$VOL" --agreetolicense --nointeraction --rebootdelay 5 "$@"; } free_kb() { df -k "$VOL" 2>/dev/null | awk 'NR==2 {print $4}'; } - -attempt=0 -while [ "$attempt" -lt 10 ]; do - attempt=$((attempt + 1)) - echo "vmix-install: startosinstall attempt $attempt" - if [ "$attempt" -eq 1 ]; then - run_soi --installpackage "$V/vmix-agent.pkg" 2>&1 & - else - # a stalled attempt leaves the volume dirty; re-erase and rebuild for a clean retry - diskutil eraseDisk APFS "$VOLUME_NAME" GPT "$TARGET" || fail "eraseDisk on retry" - tar -xf "$V/installer-app.tar" -C "$VOL" || fail "untar on retry" - mkdir -p "$APP/Contents/SharedSupport" - dd if="/dev/r$SSDISK" of="$SS" bs=1048576 count=$FULL 2>/dev/null - [ "$REM" -gt 0 ] && dd if="/dev/r$SSDISK" bs=1048576 skip=$FULL count=1 2>/dev/null | dd bs=1 count=$REM >>"$SS" 2>/dev/null - chflags -h norestricted "$SS" 2>/dev/null || true - run_soi --installpackage "$V/vmix-agent.pkg" 2>&1 & - fi +try=0 +while [ "$try" -lt 6 ]; do + try=$((try + 1)) + [ "$try" -gt 1 ] && prepare_target + echo "VMIX-INSTALL: startosinstall try $try" + run_soi 2>&1 & SOI_PID=$! - # watchdog: kill startosinstall if free space stalls for ~4 min OR the attempt - # simply takes too long (prepare is intermittently slow; healthy = a few minutes) last=$(free_kb); stalled=0; elapsed=0 while kill -0 "$SOI_PID" 2>/dev/null; do sleep 30; elapsed=$((elapsed + 30)) now=$(free_kb) if [ "$now" = "$last" ]; then stalled=$((stalled + 30)); else stalled=0; last=$now; fi - if [ "$stalled" -ge 240 ] || [ "$elapsed" -ge 540 ]; then - echo "vmix-install: prepare too slow (stalled=${stalled}s elapsed=${elapsed}s), killing to retry" + [ $((elapsed % 120)) -eq 0 ] && echo "VMIX-INSTALL: prepare running ${elapsed}s (stalled ${stalled}s)" + if [ "$stalled" -ge 240 ] || [ "$elapsed" -ge 600 ]; then + echo "VMIX-INSTALL: prepare too slow (stalled=${stalled}s elapsed=${elapsed}s), killing to retry" kill -9 "$SOI_PID" 2>/dev/null; pkill -9 -f startosinstall 2>/dev/null break fi done wait "$SOI_PID" 2>/dev/null - # on success startosinstall reboots the machine and we never get here - echo "vmix-install: startosinstall attempt $attempt ended without rebooting" + echo "VMIX-INSTALL: startosinstall try $try ended without rebooting" sleep 3 done -fail "startosinstall did not complete after $attempt attempts" +fail "startosinstall did not complete after $try tries" diff --git a/lib/images/macos/helpers/customizeImage.nix b/lib/images/macos/helpers/customizeImage.nix index df8df32..6f4dc11 100644 --- a/lib/images/macos/helpers/customizeImage.nix +++ b/lib/images/macos/helpers/customizeImage.nix @@ -1,16 +1,24 @@ -# Customize a macOS image by booting it with a VMIX volume: the vmix agent -# (LaunchDaemon installed by makeImage) runs `script` as root, records the exit -# status on the volume and powers off. Optionally re-installs OpenCore with a new -# SMBIOS identity (`smbios`). Counterpart of the Windows auditScript flow. +# Customize a macOS image offline from the vmix PE: the recovery boots with the +# image and a VMIX volume attached, its hook runs `script` as root with the +# image's System (read-only) and Data (rw) volumes mounted at $SYS / $DATA, then +# powers off. The installed macOS itself is never booted, so nothing depends on +# launchd/BTM approval inside the guest. Counterpart of the Windows +# registry/audit flow. Optionally re-installs OpenCore with a new SMBIOS +# identity (`smbios`). # # Templates provide: -# script — sh script run as root on the booted system -# files — [{ source; name; }] extra files placed next to it on /Volumes/VMIX -# smbios — { model? serial? mlb? uuid? mac? seed? } → fresh OpenCore config in the ESP -{ pkgs, lib, qemu, ident, makeVmixVolume, makeOpenCore, installBootloader, vmixReadback, vmDriver, ... }: +# script — sh script run as root in the PE (pe-lib.sh helpers available) +# bootScript — sh script run as root on the BOOTED image through Apple's QEMU +# guest agent (network, user session available; run after `script`) +# files — [{ source; name; }] extra files placed next to it on /Volumes/VMIX +# smbios — { model? serial? mlb? uuid? mac? seed? } → fresh OpenCore config in the ESP +# network — attach a user-mode NIC for bootScript (default true) +{ pkgs, lib, qemu, ident, makeVmixVolume, makeOpenCore, makeBootDisk, installBootloader, vmixReadback, vmDriver, ... }: originalImage: { name ? "", script ? "", + bootScript ? "", + network ? true, files ? [], smbios ? null, diskSize ? "", @@ -19,14 +27,18 @@ originalImage: { smp ? 4, memSize ? 4096, cpu ? qemu.defaultCpu, - timeout ? 3600, + timeout ? 1800, + machineArgs ? null, # override qemu.machineArgs (device experiments) }: let originalImageName = lib.strings.removeSuffix "-vmix" (lib.strings.removeSuffix ".qcow2" originalImage.name); customImageName = (if name != "" then name else "custom") + "-${originalImageName}-vmix.qcow2"; resultImg = "./disk.qcow2"; hasScript = script != ""; + hasBootScript = bootScript != ""; hasSmbios = smbios != null; + pe = originalImage.pe or (throw "vmix: image ${originalImage.name} carries no PE (built by an older makeImage?)"); + volumeName = originalImage.volumeName or "Macintosh HD"; model = originalImage.model or "MacPro7,1"; seed = if hasSmbios && (smbios.seed or null) != null then smbios.seed else null; @@ -45,61 +57,120 @@ let inherit mac uuid; } // builtins.removeAttrs smbios [ "seed" "mac" "uuid" "model" ]) else originalImage.opencore; + # PE boot disk: serial console, and an OpenCore ScanPolicy that only allows + # HFS+ volumes on SATA (= the PE), so the image's own macOS is never booted. + # 0x10203 = FILE_SYSTEM_LOCK | DEVICE_LOCK | ALLOW_FS_HFS | ALLOW_DEVICE_SATA + bootDisk = makeBootDisk { + name = "${name}-${originalImageName}-pe"; + esp = originalImage.opencore; + bootArgs = "keepsyms=1 serial=3 -v"; + scanPolicy = 66051; + }; - runScript = pkgs.writeText "${name}-vmix-run.sh" '' - #!/bin/sh + runScript = pkgs.writeText "${name}-run.sh" '' + #!/bin/bash + . /Volumes/VMIX/pe-lib.sh echo "=== vmix: ${name} ===" + pe_mount_target || pe_fail "could not mount the target volumes" ${script} + pe_unmount_target ''; vmixVol = makeVmixVolume { name = "${name}-${originalImageName}"; - files = [ { source = runScript; name = "vmix-run.sh"; } ] ++ files; + files = [ + { source = runScript; name = "run.sh"; } + { source = ../guest/pe-lib.sh; name = "pe-lib.sh"; } + ] ++ files; + }; + bootRunScript = pkgs.writeText "${name}-boot.sh" '' + #!/bin/bash + # runs as root on the booted system (guest-exec); VMIX is mounted at $V + V=/Volumes/VMIX + echo "=== vmix (online): ${name} ===" + CONSOLE_USER=$(stat -f %Su /dev/console 2>/dev/null) + CONSOLE_UID=$(id -u "$CONSOLE_USER" 2>/dev/null) + export V CONSOLE_USER CONSOLE_UID + # run something inside the logged-in user's GUI session + as_user() { launchctl asuser "$CONSOLE_UID" sudo -u "$CONSOLE_USER" "$@"; } + ${bootScript} + ''; + bootVol = makeVmixVolume { + name = "${name}-${originalImageName}-boot"; + files = [ { source = bootRunScript; name = "run.sh"; } ] ++ files; }; driverPython = pkgs.python3.withPackages (p: [ p.pillow ]); bootCommands = lib.optionalString hasScript '' cp ${vmixVol} vmix.img chmod +w vmix.img + cat > vmix.conf </dev/null | head -1) - if [ -n "$VMIX_DF" ] && [ "$(stat -c %s "$VMIX_DF")" -lt 256 ] && ! grep -q -P '[^\x20-\x7e\n]' "$VMIX_DF"; then - export DISPLAY=$(tr -d '\n' < "$VMIX_DF") - export HOME=$(mktemp -d) - export XDG_RUNTIME_DIR=$HOME - export SDL_VIDEODRIVER=x11 - VMIX_DISPLAY="-display sdl" - fi - ''} - echo "=== vmix: booting ${originalImageName} for ${name} ===" - python3 ${vmDriver} --mode boot --name "${name}-${originalImageName}" --timeout ${toString timeout} --progress-file ${resultImg} -- \ + echo "=== vmix: running ${name} in the PE against ${originalImageName} ===" + python3 ${vmDriver} --mode pe --name "${name}-${originalImageName}" --timeout ${toString timeout} \ + --serial-log serial.log --progress-file ${resultImg} -- \ qemu-system-x86_64 $VMIX_DISPLAY \ - ${qemu.machineArgs { inherit cpu smp memSize; }} \ + ${if machineArgs != null then machineArgs else qemu.machineArgs { inherit cpu smp memSize; }} \ ${qemu.firmwareArgs "vars.fd"} \ - ${qemu.sataDrive { id = "system"; port = 0; file = resultImg; }} \ - ${qemu.sataDrive { id = "vmix"; port = 1; file = "vmix.img"; format = "raw"; }} \ - ${qemu.netArgs { mac = originalImage.macAddress; }} \ - || { echo "vmix: VM failed during ${name} (see /tmp/vmix-macos/${name}-${originalImageName})"; exit 1; } + ${qemu.serialArgs "serial.log"} \ + ${qemu.sataDrive { id = "opencore"; port = 0; file = "ocboot.qcow2"; }} \ + ${qemu.sataDrive { id = "pe"; port = 1; file = "pe.qcow2"; }} \ + ${qemu.sataDrive { id = "system"; port = 2; file = resultImg; }} \ + ${qemu.sataDrive { id = "vmix"; port = 3; file = "vmix.img"; format = "raw"; }} \ + || { echo "vmix: PE failed during ${name} (see /tmp/vmix-macos/${name}-${originalImageName})"; exit 1; } ${vmixReadback "vmix.img"} [ "$STATUS" = "0" ] || { echo "vmix: ${name} script failed (status '$STATUS')"; exit 1; } echo "=== vmix: ${name} complete ===" ''; + onlineCommands = lib.optionalString hasBootScript '' + cp ${bootVol} vmix-boot.img + chmod +w vmix-boot.img + cp ${pkgs.OVMF.fd}/FV/OVMF_VARS.fd vars-boot.fd + chmod +w vars-boot.fd + VMIX_DISPLAY="-display none" + ${lib.optionalString (vncDisplay != null) ''VMIX_DISPLAY="-display none -vnc ${vncDisplay}"''} + QGA_SOCK=$(mktemp -u /tmp/vmix-qga-XXXXXX.sock) + + echo "=== vmix: booting ${originalImageName} for ${name} (guest agent) ===" + python3 ${vmDriver} --mode qga --name "${name}-${originalImageName}-online" --timeout ${toString timeout} \ + --serial-log serial-boot.log --qga-sock "$QGA_SOCK" \ + --qga-command 'for i in $(seq 1 30); do diskutil mount VMIX >/dev/null 2>&1; [ -f /Volumes/VMIX/run.sh ] && break; sleep 2; done; [ -f /Volumes/VMIX/run.sh ] || { echo "no VMIX volume"; exit 9; }; bash /Volumes/VMIX/run.sh > /Volumes/VMIX/vmix-run.log 2>&1; rc=$?; echo $rc > /Volumes/VMIX/vmix-run.status; sync; cat /Volumes/VMIX/vmix-run.log; diskutil unmount force /Volumes/VMIX >/dev/null 2>&1; exit $rc' -- \ + qemu-system-x86_64 $VMIX_DISPLAY \ + ${if machineArgs != null then machineArgs else qemu.machineArgs { inherit cpu smp memSize; }} \ + ${qemu.firmwareArgs "vars-boot.fd"} \ + ${qemu.serialArgs "serial-boot.log"} \ + ${qemu.guestAgentArgs "$QGA_SOCK"} \ + ${qemu.sataDrive { id = "system"; port = 0; file = resultImg; }} \ + ${qemu.sataDrive { id = "vmix"; port = 1; file = "vmix-boot.img"; format = "raw"; }} \ + ${lib.optionalString network (qemu.netArgs { mac = originalImage.macAddress; })} \ + || { echo "vmix: online step failed during ${name} (see /tmp/vmix-macos/${name}-${originalImageName}-online)"; exit 1; } + rm -f "$QGA_SOCK" + ${vmixReadback "vmix-boot.img"} + [ "$STATUS" = "0" ] || { echo "vmix: ${name} bootScript failed (status '$STATUS')"; exit 1; } + echo "=== vmix: ${name} (online) complete ===" + ''; + builtImage = pkgs.runCommand customImageName ({ - nativeBuildInputs = with pkgs; [ pkgs.qemu mtools driverPython libguestfs-with-appliance ]; + nativeBuildInputs = with pkgs; [ pkgs.qemu driverPython libguestfs-with-appliance ]; requiredSystemFeatures = [ "kvm" ]; } // lib.optionalAttrs impure { __noChroot = true; }) '' qemu-img create -q -f qcow2 -b ${originalImage} -F qcow2 ${resultImg} [ -n "${diskSize}" ] && qemu-img resize ${resultImg} ${diskSize} ${bootCommands} + ${onlineCommands} ${lib.optionalString hasSmbios (installBootloader { inherit esp; image = resultImg; })} mv ${resultImg} $out ''; in - builtImage // { _vmixOsType = "macos"; macAddress = mac; opencore = esp; model = esp.model or model; } + builtImage // { _vmixOsType = "macos"; macAddress = mac; opencore = esp; model = esp.model or model; inherit pe volumeName; } diff --git a/lib/images/macos/helpers/formatVolume.nix b/lib/images/macos/helpers/formatVolume.nix new file mode 100644 index 0000000..d0d9e1a --- /dev/null +++ b/lib/images/macos/helpers/formatVolume.nix @@ -0,0 +1,56 @@ +# Host-side script that formats a blank disk image as an APFS volume with a +# given label by booting the image's PE headless for ~30 s (Linux cannot write +# APFS). Used for the persistent home volume (`generalize { persistHome = true; }` +# mounts LABEL=