From c40f4460e3838c849aed5704df8479231b2957fb Mon Sep 17 00:00:00 2001
From: Git Sagar <owner@ipburger.com>
Date: Wed, 16 Sep 2026 12:23:45 -0300
Subject: [PATCH] vms: disks.os.persistMode (copy|backing) + a GC-root for the
 backing chain

persistMode=backing seeds the persistent OS disk as a thin qcow2 overlay
(`qemu-img create -b <storeImage>`) instead of a full cp, so many VMs share one
base image and each holds only its deltas. Because the overlay reads through a
store path that nix does not otherwise pin (the disk lives outside the store),
and because even a cp'd Windows disk backs onto the store chain, a per-VM
oneshot `vm.vmix-gcroot@<name>` reads the overlay's ACTUAL backing_file at boot
(not the config's current image, which drifts after a rebuild) and symlinks it
under /nix/var/nix/gcroots. It runs before the VM service and outside its
ProtectSystem sandbox. Fires whenever the OS disk is persistent, covering copy
too. Default stays copy, so existing VMs are unaffected.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_0117qMyjpuXsjpVAcpJbFD8g
---
 nixos/vms/config.nix           | 38 ++++++++++++++++++++++++++++++++--
 nixos/vms/submoduleOptions.nix | 11 ++++++++++
 2 files changed, 47 insertions(+), 2 deletions(-)

diff --git a/nixos/vms/config.nix b/nixos/vms/config.nix
index 4dc86a7..528671f 100644
--- a/nixos/vms/config.nix
+++ b/nixos/vms/config.nix
@@ -125,13 +125,34 @@ let
         if [ ! -f "$PERSIST_PATH" ]; then
           echo "Seeding persistent disk from store image..."
           mkdir -p "$(dirname "$PERSIST_PATH")"
-          cp --no-preserve=mode "${toString storeImage}" "$PERSIST_PATH"
+          ${if vmCfg.disks.os.persistMode == "backing"
+            then ''qemu-img create -f qcow2 -F qcow2 -b "${toString storeImage}" "$PERSIST_PATH"''
+            else ''cp --no-preserve=mode "${toString storeImage}" "$PERSIST_PATH"''}
           chmod 600 "$PERSIST_PATH"
         fi
       '';
 
       persistExecStartPre = lib.optional (hasOsDisk && vmCfg.disks.os.persist) seedPersistentDiskScript;
 
+      # A GC root pinning the OS overlay's ACTUAL backing store path, so
+      # nix-collect-garbage cannot delete the store image the disk reads through.
+      gcrootLink = "/nix/var/nix/gcroots/vmix-${vmCfg.name}-osbacking";
+      gcrootScript = pkgs.writeShellScript "${vmCfg.name}-gcroot-vmix" ''
+        # Read the live overlay's backing (not the config's current image, which
+        # drifts to a new store path after a rebuild while the overlay keeps
+        # backing the old one). Pinning the top of the chain transitively keeps
+        # the whole chain -- qcow2 backing_file paths are registered nix refs.
+        BACK=""
+        if [ -f "${vmCfg.disks.os.persistPath}" ]; then
+          BACK=$(qemu-img info "${vmCfg.disks.os.persistPath}" 2>/dev/null | awk '/^backing file:/ {print $3; exit}')
+        fi
+        [ -z "$BACK" ] && BACK="${toString storeImage}"
+        if [ -n "$BACK" ]; then
+          mkdir -p /nix/var/nix/gcroots
+          ln -sfn "$BACK" "${gcrootLink}"
+        fi
+      '';
+
       # QEMU expects single-letter boot codes (e.g. c,d,n), while vmix uses readable names.
       bootOrderQemu =
       let
@@ -254,7 +275,8 @@ let
     "vm.vmix@${vmCfg.name}" = rec {
       bindsTo = [ "net.vmix@${spaceName}.target" ] ++ lib.optional (allMacvtaps != []) "macvtaps.vm.vmix@${vmCfg.name}.service";
       unitConfig.JoinsNamespaceOf = "ns.net.vmix@${spaceName}.service";
-      after = bindsTo;
+      after = bindsTo ++ lib.optional (hasOsDisk && vmCfg.disks.os.persist) "vm.vmix-gcroot@${vmCfg.name}.service";
+      wants = lib.optional (hasOsDisk && vmCfg.disks.os.persist) "vm.vmix-gcroot@${vmCfg.name}.service";
       path = with pkgs; [ iproute2 qemu gawk coreutils ];
       serviceConfig = {
         ExecStartPre = persistExecStartPre ++ [ createTapsforLansScript ];
@@ -284,6 +306,18 @@ let
         ExecStop = deleteMacvTapsScript;
       };
     };
+  }
+  // lib.optionalAttrs (cfg.enable && hasOsDisk && vmCfg.disks.os.persist) {
+    "vm.vmix-gcroot@${vmCfg.name}" = {
+      before = [ "vm.vmix@${vmCfg.name}.service" ];
+      wantedBy = [ "multi-user.target" ];
+      path = with pkgs; [ qemu coreutils ];
+      serviceConfig = {
+        Type = "oneshot";
+        RemainAfterExit = true;
+        ExecStart = gcrootScript;
+      };
+    };
   };
 
   vmServices = concatMapAttrs (spaceName: namespaceCfg: (concatMapAttrs (mkServices4aVMInNamespace spaceName) namespaceCfg.vms)) vmixCfg.namespaces;
diff --git a/nixos/vms/submoduleOptions.nix b/nixos/vms/submoduleOptions.nix
index c7cbd1a..45a55c4 100644
--- a/nixos/vms/submoduleOptions.nix
+++ b/nixos/vms/submoduleOptions.nix
@@ -173,6 +173,17 @@ with lib;
       default = "";
       description = "Mutable path for the persistent OS disk (e.g. /storage/vms/myvm/os.qcow2). Required when persist = true.";
     };
+    disks.os.persistMode = mkOption {
+      type = types.enum [ "copy" "backing" ];
+      default = "copy";
+      description = ''
+        How the persistent OS disk is seeded from the store image (persist = true).
+        copy: a full cp of the store image; the mutable disk holds everything.
+        backing: a thin qcow2 overlay backing onto the shared store image, so many
+        VMs share one base and each holds only its own deltas. The store image (and
+        its backing chain) must then survive GC -- vmix pins it via a per-VM gcroot.
+      '';
+    };
     disks.iso.file = mkOption {
       type = types.nullOr (types.either types.path types.str);
       description = "Path to the ISO file. Can be a Nix store path or a string path to a local file.";