Simone/local-ci-cd-system
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(template): clean snapshot chain on deploy --force; seal cloud-init; harden first-boot IP

Browse Source 
deploy-linux --force reused the VM directory but left any prior snapshot
chain in place, so vmware-vdiskmanager refused the Step 4 resize with
'disk is part of a snapshot chain'. Add _clean_force_artifacts to remove
snapshot deltas (.vmdk), .vmsn, .vmsd and stale .lck before resize.

prepare-linux Step 7d seals the image: drop the stale cloud-init netplan
(leaving only the MAC-agnostic 99-ci-dhcp-allnics), regenerate networkd
config, and disable cloud-init so it never re-runs on linked clones.

deploy user-data sets package_update/upgrade: false so cloud-init does
not hold the dpkg lock at first boot and delay open-vm-tools past the
IP-detection poll; the step-7 IP timeout is raised 300s -> 600s as a
safety net for slow first boots.

Adds unit tests for _clean_force_artifacts (snapshot chain, lock dir,
clean-dir no-op) and updates prepare-linux stubs for the seal step.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
Simone
2026-06-06 12:55:34 +02:00
parent 2b0e98774f
commit e91e55f20e
2 changed files with 140 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/ci_orchestrator/commands/template.py
+83 -2
View File
@@ -794,6 +794,37 @@ def _find_vdisk_manager(explicit: str | None) -> str:
)
def _clean_force_artifacts(vm_dir: Path, disk_stem: str) -> list[str]:
"""Remove snapshot-chain and lock artifacts left in a reused VM directory.
``deploy-linux --force`` reuses the existing VM directory. If a previous
run (or a prepare-linux snapshot) left a snapshot chain behind, the disk
in the .vmx is "part of a snapshot chain" and ``vmware-vdiskmanager``
refuses to resize it at Step 4. We delete the snapshot delta disks, the
snapshot metadata (.vmsn), the snapshot database (.vmsd) and stale lock
files (.lck) so Step 4 sees a plain, chain-free disk.
The base disk itself is left untouched — Step 4 overwrites it from the
cached cloud image. Returns the sorted list of removed file names (for
logging and tests).
"""
removed: list[str] = []
patterns = (
f"{disk_stem}-[0-9]*.vmdk", # snapshot delta/redo disks
"*.vmsn", # snapshot memory/state
"*.vmsd", # snapshot database
"*.lck", # stale lock files/dirs
)
for pattern in patterns:
for path in vm_dir.glob(pattern):
if path.is_dir():
shutil.rmtree(path, ignore_errors=True)
else:
path.unlink(missing_ok=True)
removed.append(path.name)
return sorted(removed)
def _download_vmdk(url: str, dest: Path) -> None:
click.echo(f" Downloading {url} ...")
tmp = dest.with_suffix(dest.suffix + ".part")
@@ -1109,6 +1140,12 @@ def template_deploy_linux(
" Use --force to overwrite."
)
click.echo(f" [WARN] --force set -- reusing existing VM directory: {vm_dir}")
cleaned = _clean_force_artifacts(vm_dir, vmx_path.stem)
if cleaned:
click.echo(
" [OK] Cleared stale snapshot/lock artifacts (avoids resize "
f"snapshot-chain error): {', '.join(cleaned)}"
)
else:
vm_dir.mkdir(parents=True, exist_ok=True)
click.echo(f" [OK] VM directory ready: {vm_dir}")
@@ -1149,6 +1186,12 @@ def template_deploy_linux(
user_data = (
"#cloud-config\n"
f"hostname: {guest_hostname}\n"
# Suppress cloud-init's first-boot apt update/upgrade. It would hold
# the dpkg lock for minutes and delay open-vm-tools startup past the
# step-7 IP-detection poll. The real toolchain apt work runs later
# in prepare-linux where it is managed.
"package_update: false\n"
"package_upgrade: false\n"
"users:\n"
f" - name: {cloud_user}\n"
" shell: /bin/bash\n"
@@ -1316,10 +1359,10 @@ def template_deploy_linux(
# Step 7: detect guest IP via vmrun getGuestIPAddress (max 5 min)
# ──────────────────────────────────────────────────────────────────────────
_deploy_step(
"Step 7: detect guest IP via vmrun getGuestIPAddress (max 5 min)", 7, start_from_step
"Step 7: detect guest IP via vmrun getGuestIPAddress (max 10 min)", 7, start_from_step
)
if start_from_step <= 7:
guest_ip = _poll_guest_ip_deploy(vmrun, str(vmx_path), timeout=300.0)
guest_ip = _poll_guest_ip_deploy(vmrun, str(vmx_path), timeout=600.0)
if not guest_ip:
raise click.ClickException("[Deploy Step 7] Timed out waiting for guest IP.")
click.echo(f" [OK] Guest IP: {guest_ip}")
@@ -1757,6 +1800,44 @@ def template_prepare_linux(
raise click.ClickException(f"enabling NTP failed (exit {rc_ntp}):\n{err_ntp}")
click.echo("[prepare-linux] NTP time sync enabled.")
# ── Step 7d: Seal cloud-init (runs once at deploy, never in clones) ───
# cloud-init provisioned the image at deploy time. It must not run
# again on the template or on linked clones: the VMX carries the
# guestinfo seed (userdata/metadata), which every linked clone
# inherits, so cloud-init would re-run on each boot and rewrite
# /etc/netplan/50-cloud-init.yaml pinning the deploy-time MAC. That
# stale MAC match never matches a clone's NIC (nor a re-MAC'd
# template), which wedges systemd-networkd's DHCPv4 — the interface
# links up and gets an offer on the wire but no address is ever
# applied. We seal the image: drop the stale cloud-init netplan,
# leaving only the MAC-agnostic 99-ci-dhcp-allnics.yaml, regenerate
# networkd config, then disable cloud-init for all future boots.
click.echo("[prepare-linux] Step 7d — Sealing cloud-init (disable for clones) ...")
rc_seal, _, err_seal = _pl_run(
client,
"sudo rm -f /etc/netplan/50-cloud-init.yaml "
"/etc/netplan/50-cloud-init.yaml.bak && sudo netplan generate",
)
if rc_seal != 0:
raise click.ClickException(f"cloud-init seal (netplan) failed (exit {rc_seal}):\n{err_seal}")
# Disable cloud-init for every subsequent boot, then clean its state
# (best-effort: clean may exit non-zero if already clean).
_pl_run(client, "sudo touch /etc/cloud/cloud-init.disabled")
_pl_run(client, "sudo cloud-init clean --logs 2>/dev/null || true")
# Verify the stale netplan is gone and cloud-init is disabled.
_, seal_out, _ = _pl_run(
client,
"test -f /etc/cloud/cloud-init.disabled && echo DISABLED; "
"test ! -f /etc/netplan/50-cloud-init.yaml && echo NO50",
)
if "DISABLED" not in seal_out or "NO50" not in seal_out:
raise click.ClickException(
f"cloud-init seal verification failed (expected DISABLED+NO50):\n{seal_out}"
)
click.echo(
"[prepare-linux] cloud-init sealed — clones boot with static netplan only."
)
# ── Step 8: Graceful shutdown ─────────────────────────────────────────
if do_snapshot:
click.echo("[prepare-linux] Step 8 — Sending graceful shutdown ...")
tests/python/test_commands_template.py
+57
View File
@@ -1854,6 +1854,9 @@ def _stub_prepare(
return 0, "", ""
if "echo connected" in cmd:
return 0, "connected", ""
# Step 7d cloud-init seal verification.
if "cloud-init.disabled" in cmd and "echo DISABLED" in cmd:
return 0, "DISABLED\nNO50\n", ""
# validation script
return 0, val_output, ""
@@ -2063,6 +2066,8 @@ def test_prepare_linux_happy_path_skip_flags(
return 0, "", ""
if "echo connected" in cmd:
return 0, "connected", ""
if "cloud-init.disabled" in cmd and "echo DISABLED" in cmd:
return 0, "DISABLED\nNO50\n", ""
return 0, "UserSudo:OK\nSudoNoPass:OK\n", ""
def fake_pl_stream(client: object, cmd: str) -> int:
@@ -2119,3 +2124,55 @@ def test_prepare_linux_existing_snapshot_deleted(
cmds = [" ".join(c) for c in run_calls]
assert any("deleteSnapshot" in c for c in cmds)
assert any("snapshot" in c and "deleteSnapshot" not in c for c in cmds)
# ── _clean_force_artifacts (deploy-linux --force snapshot-chain cleanup) ────────
def test_clean_force_artifacts_removes_snapshot_chain(tmp_path: Path) -> None:
"""Snapshot delta/.vmsn/.vmsd/.lck are removed; base disk + vmx kept."""
(tmp_path / "LinuxBuild2404.vmdk").write_text("base")
(tmp_path / "LinuxBuild2404.vmx").write_text("vmx")
(tmp_path / "LinuxBuild2404-000001.vmdk").write_text("delta")
(tmp_path / "LinuxBuild2404-000002.vmdk").write_text("delta2")
(tmp_path / "LinuxBuild2404-Snapshot7.vmsn").write_text("snap")
(tmp_path / "LinuxBuild2404.vmsd").write_text("snapdb")
(tmp_path / "LinuxBuild2404.vmdk.lck").write_text("lock")
removed = tmpl_mod._clean_force_artifacts(tmp_path, "LinuxBuild2404")
assert set(removed) == {
"LinuxBuild2404-000001.vmdk",
"LinuxBuild2404-000002.vmdk",
"LinuxBuild2404-Snapshot7.vmsn",
"LinuxBuild2404.vmsd",
"LinuxBuild2404.vmdk.lck",
}
# Base disk and vmx survive (Step 4/5 overwrite them).
assert (tmp_path / "LinuxBuild2404.vmdk").is_file()
assert (tmp_path / "LinuxBuild2404.vmx").is_file()
# Chain artifacts gone.
assert not (tmp_path / "LinuxBuild2404-000001.vmdk").exists()
assert not (tmp_path / "LinuxBuild2404.vmsd").exists()
def test_clean_force_artifacts_removes_lock_directory(tmp_path: Path) -> None:
"""Lock entries may be directories (VMware writes .lck dirs)."""
lck = tmp_path / "LinuxBuild2404.vmx.lck"
lck.mkdir()
(lck / "M12345.lck").write_text("host")
removed = tmpl_mod._clean_force_artifacts(tmp_path, "LinuxBuild2404")
assert removed == ["LinuxBuild2404.vmx.lck"]
assert not lck.exists()
def test_clean_force_artifacts_noop_on_clean_dir(tmp_path: Path) -> None:
"""A clean directory yields no removals and leaves the base disk."""
(tmp_path / "LinuxBuild2404.vmdk").write_text("base")
removed = tmpl_mod._clean_force_artifacts(tmp_path, "LinuxBuild2404")
assert removed == []
assert (tmp_path / "LinuxBuild2404.vmdk").is_file()