macOS: guest agent, online templates, virtio-fs shares, persistent home volume

Apple's built-in QEMU guest agent (AppleQEMUGuestAgent, launched by launchd
when a virtio console port org.qemu.guest_agent.0 appears; guest-exec as root)
is attached by vmix run --macos and the NixOS module. AppleVirtIO.kext on x86
Tahoe drives virtio-fs, block, console, input, net — verified in QEMU.

- customizeImage: `bootScript` — online step through the guest agent (driver
  mode qga): boot the image, run the script as root with the VMIX volume, shut
  down through the agent. `as_user` runs commands in the logged-in session.
- templates.software: pkg/app (offline in the PE), script/homebrew (online).
- templates.profile.settings: widgets, wallpaper (pinned desktoppr — Apple
  Events need TCC consent that a headless session cannot give), dock apps,
  autohide, dark mode, hidden files.
- generalize: persistHome (fstab LABEL=vmix-home /Users), hideWidgets offline.
- formatVolume: formats a blank disk image as APFS by booting the PE (~35 s);
  idempotent.
- NixOS module: macos.guestAgent (/run/vmix/qga-<name>.sock), shares via
  virtiofsd + vhost-user-fs (Apple automount tag for the first share, others
  mounted through the agent), macos.homeDisk (created + formatted on first
  start, virtio-blk), SPICE keeps -vga vmware for macOS.
- CLI: vmix run --macos --share DIR --home FILE --qga PATH.
- qemu.nix helpers; README section.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01XsESshRCoBoUVWV9qKURUF

lib/images/macos/helpers/customizeImage.nix

@@ -7,13 +7,18 @@
 # identity (`smbios`).
 #
 # Templates provide:
-#   script  — sh script run as root in the PE (pe-lib.sh helpers available)
-#   files   — [{ source; name; }] extra files placed next to it on /Volumes/VMIX
-#   smbios  — { model? serial? mlb? uuid? mac? seed? } → fresh OpenCore config in the ESP
+#   script      — sh script run as root in the PE (pe-lib.sh helpers available)
+#   bootScript  — sh script run as root on the BOOTED image through Apple's QEMU
+#                 guest agent (network, user session available; run after `script`)
+#   files       — [{ source; name; }] extra files placed next to it on /Volumes/VMIX
+#   smbios      — { model? serial? mlb? uuid? mac? seed? } → fresh OpenCore config in the ESP
+#   network     — attach a user-mode NIC for bootScript (default true)
 { pkgs, lib, qemu, ident, makeVmixVolume, makeOpenCore, makeBootDisk, installBootloader, vmixReadback, vmDriver, ... }:
 originalImage: {
   name ? "",
   script ? "",
+  bootScript ? "",
+  network ? true,
   files ? [],
   smbios ? null,
   diskSize ? "",
@@ -23,12 +28,14 @@ originalImage: {
   memSize ? 4096,
   cpu ? qemu.defaultCpu,
   timeout ? 1800,
+  machineArgs ? null,   # override qemu.machineArgs (device experiments)
 }:
 let
   originalImageName = lib.strings.removeSuffix "-vmix" (lib.strings.removeSuffix ".qcow2" originalImage.name);
   customImageName = (if name != "" then name else "custom") + "-${originalImageName}-vmix.qcow2";
   resultImg = "./disk.qcow2";
   hasScript = script != "";
+  hasBootScript = bootScript != "";
   hasSmbios = smbios != null;
   pe = originalImage.pe or (throw "vmix: image ${originalImage.name} carries no PE (built by an older makeImage?)");
   volumeName = originalImage.volumeName or "Macintosh HD";
@@ -75,6 +82,22 @@ let
       { source = ../guest/pe-lib.sh; name = "pe-lib.sh"; }
     ] ++ files;
   };
+  bootRunScript = pkgs.writeText "${name}-boot.sh" ''
+    #!/bin/bash
+    # runs as root on the booted system (guest-exec); VMIX is mounted at $V
+    V=/Volumes/VMIX
+    echo "=== vmix (online): ${name} ==="
+    CONSOLE_USER=$(stat -f %Su /dev/console 2>/dev/null)
+    CONSOLE_UID=$(id -u "$CONSOLE_USER" 2>/dev/null)
+    export V CONSOLE_USER CONSOLE_UID
+    # run something inside the logged-in user's GUI session
+    as_user() { launchctl asuser "$CONSOLE_UID" sudo -u "$CONSOLE_USER" "$@"; }
+    ${bootScript}
+  '';
+  bootVol = makeVmixVolume {
+    name = "${name}-${originalImageName}-boot";
+    files = [ { source = bootRunScript; name = "run.sh"; } ] ++ files;
+  };
   driverPython = pkgs.python3.withPackages (p: [ p.pillow ]);
 
   bootCommands = lib.optionalString hasScript ''
@@ -96,7 +119,7 @@ let
     python3 ${vmDriver} --mode pe --name "${name}-${originalImageName}" --timeout ${toString timeout} \
       --serial-log serial.log --progress-file ${resultImg} -- \
       qemu-system-x86_64 $VMIX_DISPLAY \
-      ${qemu.machineArgs { inherit cpu smp memSize; }} \
+      ${if machineArgs != null then machineArgs else qemu.machineArgs { inherit cpu smp memSize; }} \
       ${qemu.firmwareArgs "vars.fd"} \
       ${qemu.serialArgs "serial.log"} \
       ${qemu.sataDrive { id = "opencore"; port = 0; file = "ocboot.qcow2"; }} \
@@ -110,6 +133,34 @@ let
     echo "=== vmix: ${name} complete ==="
   '';
 
+  onlineCommands = lib.optionalString hasBootScript ''
+    cp ${bootVol} vmix-boot.img
+    chmod +w vmix-boot.img
+    cp ${pkgs.OVMF.fd}/FV/OVMF_VARS.fd vars-boot.fd
+    chmod +w vars-boot.fd
+    VMIX_DISPLAY="-display none"
+    ${lib.optionalString (vncDisplay != null) ''VMIX_DISPLAY="-display none -vnc ${vncDisplay}"''}
+    QGA_SOCK=$(mktemp -u /tmp/vmix-qga-XXXXXX.sock)
+
+    echo "=== vmix: booting ${originalImageName} for ${name} (guest agent) ==="
+    python3 ${vmDriver} --mode qga --name "${name}-${originalImageName}-online" --timeout ${toString timeout} \
+      --serial-log serial-boot.log --qga-sock "$QGA_SOCK" \
+      --qga-command 'for i in $(seq 1 30); do diskutil mount VMIX >/dev/null 2>&1; [ -f /Volumes/VMIX/run.sh ] && break; sleep 2; done; [ -f /Volumes/VMIX/run.sh ] || { echo "no VMIX volume"; exit 9; }; bash /Volumes/VMIX/run.sh > /Volumes/VMIX/vmix-run.log 2>&1; rc=$?; echo $rc > /Volumes/VMIX/vmix-run.status; sync; cat /Volumes/VMIX/vmix-run.log; diskutil unmount force /Volumes/VMIX >/dev/null 2>&1; exit $rc' -- \
+      qemu-system-x86_64 $VMIX_DISPLAY \
+      ${if machineArgs != null then machineArgs else qemu.machineArgs { inherit cpu smp memSize; }} \
+      ${qemu.firmwareArgs "vars-boot.fd"} \
+      ${qemu.serialArgs "serial-boot.log"} \
+      ${qemu.guestAgentArgs "$QGA_SOCK"} \
+      ${qemu.sataDrive { id = "system"; port = 0; file = resultImg; }} \
+      ${qemu.sataDrive { id = "vmix"; port = 1; file = "vmix-boot.img"; format = "raw"; }} \
+      ${lib.optionalString network (qemu.netArgs { mac = originalImage.macAddress; })} \
+      || { echo "vmix: online step failed during ${name} (see /tmp/vmix-macos/${name}-${originalImageName}-online)"; exit 1; }
+    rm -f "$QGA_SOCK"
+    ${vmixReadback "vmix-boot.img"}
+    [ "$STATUS" = "0" ] || { echo "vmix: ${name} bootScript failed (status '$STATUS')"; exit 1; }
+    echo "=== vmix: ${name} (online) complete ==="
+  '';
+
   builtImage = pkgs.runCommand customImageName ({
     nativeBuildInputs = with pkgs; [ pkgs.qemu driverPython libguestfs-with-appliance ];
     requiredSystemFeatures = [ "kvm" ];
@@ -117,6 +168,7 @@ let
     qemu-img create -q -f qcow2 -b ${originalImage} -F qcow2 ${resultImg}
     [ -n "${diskSize}" ] && qemu-img resize ${resultImg} ${diskSize}
     ${bootCommands}
+    ${onlineCommands}
     ${lib.optionalString hasSmbios (installBootloader { inherit esp; image = resultImg; })}
     mv ${resultImg} $out
   '';

lib/images/macos/helpers/formatVolume.nix

@@ -0,0 +1,56 @@
+# Host-side script that formats a blank disk image as an APFS volume with a
+# given label by booting the image's PE headless for ~30 s (Linux cannot write
+# APFS). Used for the persistent home volume (`generalize { persistHome = true; }`
+# mounts LABEL=<label> at /Users) by the NixOS module and `vmix run --home`.
+#   ${formatVolume { inherit image; label = "vmix-home"; }} <disk-file> <raw|qcow2>
+# The disk is found inside the PE as the only one without a partition table.
+{ pkgs, lib, qemu, makeVmixVolume, makeBootDisk, vmDriver, ... }:
+{ image, label ? "vmix-home", memSize ? 4096 }:
+let
+  pe = image.pe or (throw "vmix: image ${image.name} carries no PE");
+  bootDisk = makeBootDisk { name = "${label}-format"; esp = image.opencore; bootArgs = "keepsyms=1 serial=3"; scanPolicy = 66051; };
+  runScript = pkgs.writeText "format-${label}.sh" ''
+    . /Volumes/VMIX/pe-lib.sh
+    LIST=$(diskutil list)
+    # idempotent: a run.sh re-run (e.g. after a guest reboot) finds the volume done
+    if echo "$LIST" | grep -q "APFS Volume ${label} "; then echo "vmix: volume '${label}' already exists"; exit 0; fi
+    # blank disk: a whole-disk line followed by no partition entries
+    TARGET=$(echo "$LIST" | awk '/^\/dev\/disk[0-9]+ / {d=$1; n=0; next} /^ +[0-9]+:/ {n++} /^$/ {if (d != "" && n <= 1) print d; d=""} END {if (d != "" && n <= 1) print d}' | grep -vE "synthesized" | head -1)
+    [ -n "$TARGET" ] || { echo "$LIST"; pe_fail "no blank disk found"; }
+    echo "vmix: formatting $TARGET as APFS '${label}'"
+    diskutil eraseDisk APFS "${label}" GPT "$TARGET" || pe_fail "eraseDisk $TARGET"
+    diskutil unmount "/Volumes/${label}" >/dev/null 2>&1 || true
+  '';
+  vmixVol = makeVmixVolume {
+    name = "format-${label}";
+    files = [ { source = runScript; name = "run.sh"; } { source = ../guest/pe-lib.sh; name = "pe-lib.sh"; } ];
+  };
+  driverPython = pkgs.python3.withPackages (p: [ p.pillow ]);
+in
+pkgs.writeShellScript "vmix-format-${label}" ''
+  set -eu
+  DISK="$1"; FMT="''${2:-qcow2}"
+  T=$(mktemp -d /tmp/vmix-format-XXXXXX)
+  cleanup() { if [ "''${OK:-0}" = 1 ]; then rm -rf "$T"; else echo "vmix: logs kept in $T"; fi; }
+  trap cleanup EXIT
+  cp ${vmixVol} "$T/vmix.img"; chmod +w "$T/vmix.img"
+  ${pkgs.qemu}/bin/qemu-img create -q -f qcow2 -F raw -b ${pe} "$T/pe.qcow2"
+  ${pkgs.qemu}/bin/qemu-img create -q -f qcow2 -F raw -b ${bootDisk}/boot.img "$T/ocboot.qcow2"
+  cp ${pkgs.OVMF.fd}/FV/OVMF_VARS.fd "$T/vars.fd"; chmod +w "$T/vars.fd"
+  echo "vmix: formatting $DISK as APFS '${label}' (PE boot)"
+  ${driverPython}/bin/python3 ${vmDriver} --mode pe --name "format-${label}" --debug-dir "$T/debug" --timeout 600 \
+    --serial-log "$T/serial.log" -- \
+    ${pkgs.qemu}/bin/qemu-system-x86_64 -display none \
+    ${qemu.machineArgs { smp = 2; inherit memSize; }} \
+    ${qemu.firmwareArgs "$T/vars.fd"} \
+    ${qemu.serialArgs "$T/serial.log"} \
+    ${qemu.sataDrive { id = "opencore"; port = 0; file = "$T/ocboot.qcow2"; }} \
+    ${qemu.sataDrive { id = "pe"; port = 1; file = "$T/pe.qcow2"; }} \
+    ${qemu.sataDrive { id = "vmix"; port = 2; file = "$T/vmix.img"; format = "raw"; }} \
+    ${qemu.virtioBlkArgs { id = "target"; file = "$DISK"; format = "$FMT"; }} \
+    >/dev/null
+  STATUS=$(${pkgs.libguestfs-with-appliance}/bin/guestfish --ro -a "$T/vmix.img" -m /dev/sda1 cat /vmix-run.status 2>/dev/null | tr -d '[:space:]' || true)
+  [ "$STATUS" = "0" ] || { echo "vmix: formatting failed (status '$STATUS')"; ${pkgs.libguestfs-with-appliance}/bin/guestfish --ro -a "$T/vmix.img" -m /dev/sda1 cat /vmix-run.log 2>/dev/null | tail -20; exit 1; }
+  OK=1
+  echo "vmix: $DISK formatted"
+''

lib/images/macos/helpers/qemu.nix

@@ -39,6 +39,27 @@ rec {
   # XNU logs to COM1 with boot-args serial=3; the build drivers read this file
   serialArgs = file: "-serial file:${file}";
 
+  # Apple's own QEMU guest agent (/usr/libexec/AppleQEMUGuestAgent, macOS 13+)
+  # attaches to a virtio console port named org.qemu.guest_agent.0 and offers
+  # guest-exec (as root), guest-file-* etc. over this unix socket.
+  guestAgentArgs = sock:
+    "-device virtio-serial-pci,id=vmix-vser -chardev socket,path=${sock},server=on,wait=off,id=vmix-qga -device virtserialport,chardev=vmix-qga,name=org.qemu.guest_agent.0";
+
+  # virtio-fs (vhost-user, virtiofsd on the host). macOS auto-mounts the tag
+  # "com.apple.virtio-fs.automount" at /Volumes/My Shared Files; other tags are
+  # mounted with `mount -t virtiofs <tag> <dir>`. Needs a shared memory backend.
+  automountTag = "com.apple.virtio-fs.automount";
+  memBackendArgs = memSize: "-object memory-backend-memfd,id=vmix-mem,size=${toString memSize}M,share=on -numa node,memdev=vmix-mem";
+  virtioFsArgs = { tag, sock, id ? tag }:
+    "-chardev socket,id=vmix-vfs-${id},path=${sock} -device vhost-user-fs-pci,chardev=vmix-vfs-${id},tag=${tag}";
+
+  # virtio-blk data disk (AppleVirtIOBlock), e.g. the persistent home volume
+  virtioBlkArgs = { id, file, format ? "qcow2", extra ? "" }:
+    "-drive id=${id},if=none,format=${format},file=${file}${extra} -device virtio-blk-pci,drive=${id}";
+
+  # virtio keyboard/tablet (AppleVirtIOInput), optional alternative to the USB HID pair
+  virtioInputArgs = "-device virtio-keyboard-pci -device virtio-tablet-pci";
+
   firmwareArgs = varsFile:
     "-drive if=pflash,format=raw,readonly=on,file=${pkgs.OVMF.fd}/FV/OVMF_CODE.fd -drive if=pflash,format=raw,file=${varsFile}";
 }

lib/images/macos/helpers/vm-driver.py

@@ -9,6 +9,9 @@ Modes
            (bright) loginwindow / Setup Assistant — the driver powers it down —
            or halts on its own.
   boot     boot an installed image and wait for it to halt or reach the loginwindow.
+  qga      boot an installed image with Apple's QEMU guest agent attached
+           (--qga-sock), wait for it, run --qga-command as root through it
+           (guest-exec), then shut the guest down. Used for online templates.
 
 Observation is passive: the serial console (boot-args serial=3: the PE's
 "VMIX-*" markers, kernel boots, panics) and screenshots over QMP (mean
@@ -101,6 +104,44 @@ class QMP:
         self.cmd('system_powerdown')
 
 
+class QGA:
+    """Minimal QEMU guest agent client over the unix socket QEMU serves."""
+
+    def __init__(self, path):
+        self.path = path
+
+    def cmd(self, name, timeout=30, **args):
+        s = socket.socket(socket.AF_UNIX)
+        s.settimeout(timeout)
+        try:
+            s.connect(self.path)
+            s.sendall((json.dumps({'execute': name, 'arguments': args}) + '\n').encode())
+            buf = b''
+            while b'\n' not in buf:
+                d = s.recv(65536)
+                if not d:
+                    raise OSError('guest agent closed the connection')
+                buf += d
+        finally:
+            s.close()
+        r = json.loads(buf.split(b'\n', 1)[0])
+        if 'error' in r:
+            raise RuntimeError(r['error'])
+        return r.get('return')
+
+    def ping(self):
+        try:
+            return self.cmd('guest-ping', timeout=5) == {}
+        except Exception:  # noqa: BLE001
+            return False
+
+    def exec_start(self, command):
+        return self.cmd('guest-exec', path='/bin/bash', arg=['-c', command], **{'capture-output': True})['pid']
+
+    def exec_status(self, pid):
+        return self.cmd('guest-exec-status', pid=pid)
+
+
 class Screen:
     """Screenshots over QMP with a coarse change fingerprint (cursor-insensitive)."""
 
@@ -227,6 +268,104 @@ def disk_idle(args):
         return True
 
 
+def run_qga(args, proc, qmp, screen, serial, log):
+    """Online template: wait for the guest agent, run the command, shut down."""
+    import base64
+    qga = QGA(args.qga_sock)
+    start = time.time()
+    last_periodic = 0
+    while not qga.ping():
+        rc = proc.poll()
+        if rc is not None:
+            log(f'QEMU exited with {rc} before the guest agent came up')
+            return rc or 3
+        now = time.time()
+        if now - start > args.start_timeout:
+            try:
+                screen.grab()
+                screen.save('no-agent')
+            except Exception:  # noqa: BLE001
+                pass
+            log(f'guest agent not reachable within {args.start_timeout:.0f}s')
+            proc.kill()
+            return 3
+        for line in serial.poll():
+            if any(m in line for m in PANIC_MARKS):
+                log('serial: ' + line.strip()[:200])
+        if now - last_periodic > args.periodic:
+            last_periodic = now
+            try:
+                screen.grab()
+                screen.save('periodic')
+            except Exception:  # noqa: BLE001
+                pass
+        time.sleep(3)
+    log(f'guest agent up after {time.time() - start:.0f}s: {qga.cmd("guest-info").get("version")}')
+    time.sleep(args.qga_settle)      # let the login session / volumes settle
+    try:
+        pid = qga.exec_start(args.qga_command)
+    except Exception as e:  # noqa: BLE001
+        log(f'guest-exec failed: {e}')
+        proc.kill()
+        return 3
+    log(f'running command as root (pid {pid}): {args.qga_command[:160]}')
+    t0 = time.time()
+    while True:
+        rc = proc.poll()
+        if rc is not None:
+            log(f'QEMU exited with {rc} while the command was running')
+            return rc or 3
+        if time.time() - start > args.timeout:
+            log('timeout reached, killing QEMU')
+            proc.kill()
+            return 124
+        try:
+            st = qga.exec_status(pid)
+        except Exception as e:  # noqa: BLE001
+            log(f'guest-exec-status failed: {e}')
+            time.sleep(5)
+            continue
+        if st.get('exited'):
+            break
+        now = time.time()
+        if now - last_periodic > args.periodic:
+            last_periodic = now
+            try:
+                screen.grab()
+                screen.save('periodic')
+            except Exception:  # noqa: BLE001
+                pass
+        time.sleep(3)
+    out = base64.b64decode(st.get('out-data', '')).decode('utf-8', 'replace')
+    err = base64.b64decode(st.get('err-data', '')).decode('utf-8', 'replace')
+    code = st.get('exitcode', st.get('signal'))
+    log(f'command finished in {time.time() - t0:.0f}s, exit {code}')
+    for line in (out + err).splitlines()[-200:]:
+        log('guest: ' + line[:220])
+    try:
+        screen.grab()
+        screen.save('after-command')
+    except Exception:  # noqa: BLE001
+        pass
+    log('shutting the guest down')
+    try:
+        qga.exec_start('sync; /sbin/shutdown -h now')
+    except Exception as e:  # noqa: BLE001
+        log(f'guest shutdown failed ({e}), ACPI powerdown')
+        try:
+            qmp.system_powerdown()
+        except Exception:  # noqa: BLE001
+            pass
+    for _ in range(180):
+        if proc.poll() is not None:
+            log('QEMU exited')
+            return 0 if code == 0 else 4
+        time.sleep(1)
+    log('guest did not power off, killing QEMU')
+    proc.kill()
+    return 0 if code == 0 else 4
+
+
 def drive(args, proc, qmp, screen, serial, log):
     start = time.time()
     last_periodic = 0
@@ -275,11 +414,12 @@ def drive(args, proc, qmp, screen, serial, log):
                 screen.save('panic')
             except Exception:  # noqa: BLE001
                 pass
-            if args.mode == 'pe' or panics > args.max_resets:
+            if panics > args.max_resets:
                 log(f'kernel panic #{panics}, giving up')
                 proc.kill()
                 return 3
-            # XNU reboots by itself after a panic; only reset if no kernel comes back
+            # XNU reboots by itself after a panic; only reset if no kernel comes back.
+            # (In pe mode the PE then runs run.sh again — templates are idempotent.)
             log(f'kernel panic #{panics}, waiting for the guest to reboot')
             serial.reboot_at = now
             continue
@@ -287,7 +427,7 @@ def drive(args, proc, qmp, screen, serial, log):
         # The guest asked for a reboot but no kernel came back: macOS' restart
         # path panics in QEMU (AppleSMC watchdog keys, see README); reset now
         # instead of waiting for the frozen-screen watchdog.
-        if serial.reboot_at and now - serial.reboot_at > args.reboot_timeout and args.mode != 'pe':
+        if serial.reboot_at and now - serial.reboot_at > args.reboot_timeout:
             resets += 1
             try:
                 screen.grab()
@@ -382,7 +522,10 @@ def drive(args, proc, qmp, screen, serial, log):
 
 def main():
     p = argparse.ArgumentParser()
-    p.add_argument('--mode', choices=['pe', 'install', 'boot'], required=True)
+    p.add_argument('--mode', choices=['pe', 'install', 'boot', 'qga'], required=True)
+    p.add_argument('--qga-sock', default=None, help='guest agent unix socket (mode qga)')
+    p.add_argument('--qga-command', default=None, help='bash command to run as root through the guest agent (mode qga)')
+    p.add_argument('--qga-settle', type=float, default=20.0, help='seconds to wait after the agent answers before running the command')
     p.add_argument('--name', default='macos')
     p.add_argument('--debug-dir', default=None)
     p.add_argument('--serial-log', default=None, help='file QEMU writes the serial console to')
@@ -422,7 +565,10 @@ def main():
     screen = Screen(qmp, debug_dir, log)
     serial = Serial(args.serial_log)
     try:
-        rc = drive(args, proc, qmp, screen, serial, log)
+        if args.mode == 'qga':
+            rc = run_qga(args, proc, qmp, screen, serial, log)
+        else:
+            rc = drive(args, proc, qmp, screen, serial, log)
     finally:
         if args.serial_log and os.path.exists(args.serial_log):
             try: