fix(template): clean snapshot chain on deploy --force; seal cloud-init; harden first-boot IP

deploy-linux --force reused the VM directory but left any prior snapshot
chain in place, so vmware-vdiskmanager refused the Step 4 resize with
'disk is part of a snapshot chain'. Add _clean_force_artifacts to remove
snapshot deltas (.vmdk), .vmsn, .vmsd and stale .lck before resize.

prepare-linux Step 7d seals the image: drop the stale cloud-init netplan
(leaving only the MAC-agnostic 99-ci-dhcp-allnics), regenerate networkd
config, and disable cloud-init so it never re-runs on linked clones.

deploy user-data sets package_update/upgrade: false so cloud-init does
not hold the dpkg lock at first boot and delay open-vm-tools past the
IP-detection poll; the step-7 IP timeout is raised 300s -> 600s as a
safety net for slow first boots.

Adds unit tests for _clean_force_artifacts (snapshot chain, lock dir,
clean-dir no-op) and updates prepare-linux stubs for the seal step.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

src/ci_orchestrator/commands/template.py

@@ -794,6 +794,37 @@ def _find_vdisk_manager(explicit: str | None) -> str:
     )
 
 
+def _clean_force_artifacts(vm_dir: Path, disk_stem: str) -> list[str]:
+    """Remove snapshot-chain and lock artifacts left in a reused VM directory.
+
+    ``deploy-linux --force`` reuses the existing VM directory. If a previous
+    run (or a prepare-linux snapshot) left a snapshot chain behind, the disk
+    in the .vmx is "part of a snapshot chain" and ``vmware-vdiskmanager``
+    refuses to resize it at Step 4. We delete the snapshot delta disks, the
+    snapshot metadata (.vmsn), the snapshot database (.vmsd) and stale lock
+    files (.lck) so Step 4 sees a plain, chain-free disk.
+
+    The base disk itself is left untouched — Step 4 overwrites it from the
+    cached cloud image. Returns the sorted list of removed file names (for
+    logging and tests).
+    """
+    removed: list[str] = []
+    patterns = (
+        f"{disk_stem}-[0-9]*.vmdk",  # snapshot delta/redo disks
+        "*.vmsn",                    # snapshot memory/state
+        "*.vmsd",                    # snapshot database
+        "*.lck",                     # stale lock files/dirs
+    )
+    for pattern in patterns:
+        for path in vm_dir.glob(pattern):
+            if path.is_dir():
+                shutil.rmtree(path, ignore_errors=True)
+            else:
+                path.unlink(missing_ok=True)
+            removed.append(path.name)
+    return sorted(removed)
+
+
 def _download_vmdk(url: str, dest: Path) -> None:
     click.echo(f"  Downloading {url} ...")
     tmp = dest.with_suffix(dest.suffix + ".part")
@@ -1109,6 +1140,12 @@ def template_deploy_linux(
                     "  Use --force to overwrite."
                 )
             click.echo(f"  [WARN] --force set -- reusing existing VM directory: {vm_dir}")
+            cleaned = _clean_force_artifacts(vm_dir, vmx_path.stem)
+            if cleaned:
+                click.echo(
+                    "  [OK] Cleared stale snapshot/lock artifacts (avoids resize "
+                    f"snapshot-chain error): {', '.join(cleaned)}"
+                )
         else:
             vm_dir.mkdir(parents=True, exist_ok=True)
         click.echo(f"  [OK] VM directory ready: {vm_dir}")
@@ -1149,6 +1186,12 @@ def template_deploy_linux(
         user_data = (
             "#cloud-config\n"
             f"hostname: {guest_hostname}\n"
+            # Suppress cloud-init's first-boot apt update/upgrade. It would hold
+            # the dpkg lock for minutes and delay open-vm-tools startup past the
+            # step-7 IP-detection poll. The real toolchain apt work runs later
+            # in prepare-linux where it is managed.
+            "package_update: false\n"
+            "package_upgrade: false\n"
             "users:\n"
             f"  - name: {cloud_user}\n"
             "    shell: /bin/bash\n"
@@ -1316,10 +1359,10 @@ def template_deploy_linux(
     # Step 7: detect guest IP via vmrun getGuestIPAddress (max 5 min)
     # ──────────────────────────────────────────────────────────────────────────
     _deploy_step(
-        "Step 7: detect guest IP via vmrun getGuestIPAddress (max 5 min)", 7, start_from_step
+        "Step 7: detect guest IP via vmrun getGuestIPAddress (max 10 min)", 7, start_from_step
     )
     if start_from_step <= 7:
-        guest_ip = _poll_guest_ip_deploy(vmrun, str(vmx_path), timeout=300.0)
+        guest_ip = _poll_guest_ip_deploy(vmrun, str(vmx_path), timeout=600.0)
         if not guest_ip:
             raise click.ClickException("[Deploy Step 7] Timed out waiting for guest IP.")
         click.echo(f"  [OK] Guest IP: {guest_ip}")
@@ -1757,6 +1800,44 @@ def template_prepare_linux(
             raise click.ClickException(f"enabling NTP failed (exit {rc_ntp}):\n{err_ntp}")
         click.echo("[prepare-linux] NTP time sync enabled.")
 
+        # ── Step 7d: Seal cloud-init (runs once at deploy, never in clones) ───
+        # cloud-init provisioned the image at deploy time. It must not run
+        # again on the template or on linked clones: the VMX carries the
+        # guestinfo seed (userdata/metadata), which every linked clone
+        # inherits, so cloud-init would re-run on each boot and rewrite
+        # /etc/netplan/50-cloud-init.yaml pinning the deploy-time MAC. That
+        # stale MAC match never matches a clone's NIC (nor a re-MAC'd
+        # template), which wedges systemd-networkd's DHCPv4 — the interface
+        # links up and gets an offer on the wire but no address is ever
+        # applied. We seal the image: drop the stale cloud-init netplan,
+        # leaving only the MAC-agnostic 99-ci-dhcp-allnics.yaml, regenerate
+        # networkd config, then disable cloud-init for all future boots.
+        click.echo("[prepare-linux] Step 7d — Sealing cloud-init (disable for clones) ...")
+        rc_seal, _, err_seal = _pl_run(
+            client,
+            "sudo rm -f /etc/netplan/50-cloud-init.yaml "
+            "/etc/netplan/50-cloud-init.yaml.bak && sudo netplan generate",
+        )
+        if rc_seal != 0:
+            raise click.ClickException(f"cloud-init seal (netplan) failed (exit {rc_seal}):\n{err_seal}")
+        # Disable cloud-init for every subsequent boot, then clean its state
+        # (best-effort: clean may exit non-zero if already clean).
+        _pl_run(client, "sudo touch /etc/cloud/cloud-init.disabled")
+        _pl_run(client, "sudo cloud-init clean --logs 2>/dev/null || true")
+        # Verify the stale netplan is gone and cloud-init is disabled.
+        _, seal_out, _ = _pl_run(
+            client,
+            "test -f /etc/cloud/cloud-init.disabled && echo DISABLED; "
+            "test ! -f /etc/netplan/50-cloud-init.yaml && echo NO50",
+        )
+        if "DISABLED" not in seal_out or "NO50" not in seal_out:
+            raise click.ClickException(
+                f"cloud-init seal verification failed (expected DISABLED+NO50):\n{seal_out}"
+            )
+        click.echo(
+            "[prepare-linux] cloud-init sealed — clones boot with static netplan only."
+        )
+
         # ── Step 8: Graceful shutdown ─────────────────────────────────────────
         if do_snapshot:
             click.echo("[prepare-linux] Step 8 — Sending graceful shutdown ...")

tests/python/test_commands_template.py

@@ -1854,6 +1854,9 @@ def _stub_prepare(
             return 0, "", ""
         if "echo connected" in cmd:
             return 0, "connected", ""
+        # Step 7d cloud-init seal verification.
+        if "cloud-init.disabled" in cmd and "echo DISABLED" in cmd:
+            return 0, "DISABLED\nNO50\n", ""
         # validation script
         return 0, val_output, ""
 
@@ -2063,6 +2066,8 @@ def test_prepare_linux_happy_path_skip_flags(
             return 0, "", ""
         if "echo connected" in cmd:
             return 0, "connected", ""
+        if "cloud-init.disabled" in cmd and "echo DISABLED" in cmd:
+            return 0, "DISABLED\nNO50\n", ""
         return 0, "UserSudo:OK\nSudoNoPass:OK\n", ""
 
     def fake_pl_stream(client: object, cmd: str) -> int:
@@ -2119,3 +2124,55 @@ def test_prepare_linux_existing_snapshot_deleted(
     cmds = [" ".join(c) for c in run_calls]
     assert any("deleteSnapshot" in c for c in cmds)
     assert any("snapshot" in c and "deleteSnapshot" not in c for c in cmds)
+
+
+# ── _clean_force_artifacts (deploy-linux --force snapshot-chain cleanup) ────────
+
+
+def test_clean_force_artifacts_removes_snapshot_chain(tmp_path: Path) -> None:
+    """Snapshot delta/.vmsn/.vmsd/.lck are removed; base disk + vmx kept."""
+    (tmp_path / "LinuxBuild2404.vmdk").write_text("base")
+    (tmp_path / "LinuxBuild2404.vmx").write_text("vmx")
+    (tmp_path / "LinuxBuild2404-000001.vmdk").write_text("delta")
+    (tmp_path / "LinuxBuild2404-000002.vmdk").write_text("delta2")
+    (tmp_path / "LinuxBuild2404-Snapshot7.vmsn").write_text("snap")
+    (tmp_path / "LinuxBuild2404.vmsd").write_text("snapdb")
+    (tmp_path / "LinuxBuild2404.vmdk.lck").write_text("lock")
+
+    removed = tmpl_mod._clean_force_artifacts(tmp_path, "LinuxBuild2404")
+
+    assert set(removed) == {
+        "LinuxBuild2404-000001.vmdk",
+        "LinuxBuild2404-000002.vmdk",
+        "LinuxBuild2404-Snapshot7.vmsn",
+        "LinuxBuild2404.vmsd",
+        "LinuxBuild2404.vmdk.lck",
+    }
+    # Base disk and vmx survive (Step 4/5 overwrite them).
+    assert (tmp_path / "LinuxBuild2404.vmdk").is_file()
+    assert (tmp_path / "LinuxBuild2404.vmx").is_file()
+    # Chain artifacts gone.
+    assert not (tmp_path / "LinuxBuild2404-000001.vmdk").exists()
+    assert not (tmp_path / "LinuxBuild2404.vmsd").exists()
+
+
+def test_clean_force_artifacts_removes_lock_directory(tmp_path: Path) -> None:
+    """Lock entries may be directories (VMware writes .lck dirs)."""
+    lck = tmp_path / "LinuxBuild2404.vmx.lck"
+    lck.mkdir()
+    (lck / "M12345.lck").write_text("host")
+
+    removed = tmpl_mod._clean_force_artifacts(tmp_path, "LinuxBuild2404")
+
+    assert removed == ["LinuxBuild2404.vmx.lck"]
+    assert not lck.exists()
+
+
+def test_clean_force_artifacts_noop_on_clean_dir(tmp_path: Path) -> None:
+    """A clean directory yields no removals and leaves the base disk."""
+    (tmp_path / "LinuxBuild2404.vmdk").write_text("base")
+
+    removed = tmpl_mod._clean_force_artifacts(tmp_path, "LinuxBuild2404")
+
+    assert removed == []
+    assert (tmp_path / "LinuxBuild2404.vmdk").is_file()