diff --git a/src/ci_orchestrator/commands/template.py b/src/ci_orchestrator/commands/template.py index 5db4496..51e4e1f 100644 --- a/src/ci_orchestrator/commands/template.py +++ b/src/ci_orchestrator/commands/template.py @@ -794,6 +794,37 @@ def _find_vdisk_manager(explicit: str | None) -> str: ) +def _clean_force_artifacts(vm_dir: Path, disk_stem: str) -> list[str]: + """Remove snapshot-chain and lock artifacts left in a reused VM directory. + + ``deploy-linux --force`` reuses the existing VM directory. If a previous + run (or a prepare-linux snapshot) left a snapshot chain behind, the disk + in the .vmx is "part of a snapshot chain" and ``vmware-vdiskmanager`` + refuses to resize it at Step 4. We delete the snapshot delta disks, the + snapshot metadata (.vmsn), the snapshot database (.vmsd) and stale lock + files (.lck) so Step 4 sees a plain, chain-free disk. + + The base disk itself is left untouched — Step 4 overwrites it from the + cached cloud image. Returns the sorted list of removed file names (for + logging and tests). + """ + removed: list[str] = [] + patterns = ( + f"{disk_stem}-[0-9]*.vmdk", # snapshot delta/redo disks + "*.vmsn", # snapshot memory/state + "*.vmsd", # snapshot database + "*.lck", # stale lock files/dirs + ) + for pattern in patterns: + for path in vm_dir.glob(pattern): + if path.is_dir(): + shutil.rmtree(path, ignore_errors=True) + else: + path.unlink(missing_ok=True) + removed.append(path.name) + return sorted(removed) + + def _download_vmdk(url: str, dest: Path) -> None: click.echo(f" Downloading {url} ...") tmp = dest.with_suffix(dest.suffix + ".part") @@ -1109,6 +1140,12 @@ def template_deploy_linux( " Use --force to overwrite." ) click.echo(f" [WARN] --force set -- reusing existing VM directory: {vm_dir}") + cleaned = _clean_force_artifacts(vm_dir, vmx_path.stem) + if cleaned: + click.echo( + " [OK] Cleared stale snapshot/lock artifacts (avoids resize " + f"snapshot-chain error): {', '.join(cleaned)}" + ) else: vm_dir.mkdir(parents=True, exist_ok=True) click.echo(f" [OK] VM directory ready: {vm_dir}") @@ -1149,6 +1186,12 @@ def template_deploy_linux( user_data = ( "#cloud-config\n" f"hostname: {guest_hostname}\n" + # Suppress cloud-init's first-boot apt update/upgrade. It would hold + # the dpkg lock for minutes and delay open-vm-tools startup past the + # step-7 IP-detection poll. The real toolchain apt work runs later + # in prepare-linux where it is managed. + "package_update: false\n" + "package_upgrade: false\n" "users:\n" f" - name: {cloud_user}\n" " shell: /bin/bash\n" @@ -1316,10 +1359,10 @@ def template_deploy_linux( # Step 7: detect guest IP via vmrun getGuestIPAddress (max 5 min) # ────────────────────────────────────────────────────────────────────────── _deploy_step( - "Step 7: detect guest IP via vmrun getGuestIPAddress (max 5 min)", 7, start_from_step + "Step 7: detect guest IP via vmrun getGuestIPAddress (max 10 min)", 7, start_from_step ) if start_from_step <= 7: - guest_ip = _poll_guest_ip_deploy(vmrun, str(vmx_path), timeout=300.0) + guest_ip = _poll_guest_ip_deploy(vmrun, str(vmx_path), timeout=600.0) if not guest_ip: raise click.ClickException("[Deploy Step 7] Timed out waiting for guest IP.") click.echo(f" [OK] Guest IP: {guest_ip}") @@ -1757,6 +1800,44 @@ def template_prepare_linux( raise click.ClickException(f"enabling NTP failed (exit {rc_ntp}):\n{err_ntp}") click.echo("[prepare-linux] NTP time sync enabled.") + # ── Step 7d: Seal cloud-init (runs once at deploy, never in clones) ─── + # cloud-init provisioned the image at deploy time. It must not run + # again on the template or on linked clones: the VMX carries the + # guestinfo seed (userdata/metadata), which every linked clone + # inherits, so cloud-init would re-run on each boot and rewrite + # /etc/netplan/50-cloud-init.yaml pinning the deploy-time MAC. That + # stale MAC match never matches a clone's NIC (nor a re-MAC'd + # template), which wedges systemd-networkd's DHCPv4 — the interface + # links up and gets an offer on the wire but no address is ever + # applied. We seal the image: drop the stale cloud-init netplan, + # leaving only the MAC-agnostic 99-ci-dhcp-allnics.yaml, regenerate + # networkd config, then disable cloud-init for all future boots. + click.echo("[prepare-linux] Step 7d — Sealing cloud-init (disable for clones) ...") + rc_seal, _, err_seal = _pl_run( + client, + "sudo rm -f /etc/netplan/50-cloud-init.yaml " + "/etc/netplan/50-cloud-init.yaml.bak && sudo netplan generate", + ) + if rc_seal != 0: + raise click.ClickException(f"cloud-init seal (netplan) failed (exit {rc_seal}):\n{err_seal}") + # Disable cloud-init for every subsequent boot, then clean its state + # (best-effort: clean may exit non-zero if already clean). + _pl_run(client, "sudo touch /etc/cloud/cloud-init.disabled") + _pl_run(client, "sudo cloud-init clean --logs 2>/dev/null || true") + # Verify the stale netplan is gone and cloud-init is disabled. + _, seal_out, _ = _pl_run( + client, + "test -f /etc/cloud/cloud-init.disabled && echo DISABLED; " + "test ! -f /etc/netplan/50-cloud-init.yaml && echo NO50", + ) + if "DISABLED" not in seal_out or "NO50" not in seal_out: + raise click.ClickException( + f"cloud-init seal verification failed (expected DISABLED+NO50):\n{seal_out}" + ) + click.echo( + "[prepare-linux] cloud-init sealed — clones boot with static netplan only." + ) + # ── Step 8: Graceful shutdown ───────────────────────────────────────── if do_snapshot: click.echo("[prepare-linux] Step 8 — Sending graceful shutdown ...") diff --git a/tests/python/test_commands_template.py b/tests/python/test_commands_template.py index d65dbe4..bba2bb1 100644 --- a/tests/python/test_commands_template.py +++ b/tests/python/test_commands_template.py @@ -1854,6 +1854,9 @@ def _stub_prepare( return 0, "", "" if "echo connected" in cmd: return 0, "connected", "" + # Step 7d cloud-init seal verification. + if "cloud-init.disabled" in cmd and "echo DISABLED" in cmd: + return 0, "DISABLED\nNO50\n", "" # validation script return 0, val_output, "" @@ -2063,6 +2066,8 @@ def test_prepare_linux_happy_path_skip_flags( return 0, "", "" if "echo connected" in cmd: return 0, "connected", "" + if "cloud-init.disabled" in cmd and "echo DISABLED" in cmd: + return 0, "DISABLED\nNO50\n", "" return 0, "UserSudo:OK\nSudoNoPass:OK\n", "" def fake_pl_stream(client: object, cmd: str) -> int: @@ -2119,3 +2124,55 @@ def test_prepare_linux_existing_snapshot_deleted( cmds = [" ".join(c) for c in run_calls] assert any("deleteSnapshot" in c for c in cmds) assert any("snapshot" in c and "deleteSnapshot" not in c for c in cmds) + + +# ── _clean_force_artifacts (deploy-linux --force snapshot-chain cleanup) ──────── + + +def test_clean_force_artifacts_removes_snapshot_chain(tmp_path: Path) -> None: + """Snapshot delta/.vmsn/.vmsd/.lck are removed; base disk + vmx kept.""" + (tmp_path / "LinuxBuild2404.vmdk").write_text("base") + (tmp_path / "LinuxBuild2404.vmx").write_text("vmx") + (tmp_path / "LinuxBuild2404-000001.vmdk").write_text("delta") + (tmp_path / "LinuxBuild2404-000002.vmdk").write_text("delta2") + (tmp_path / "LinuxBuild2404-Snapshot7.vmsn").write_text("snap") + (tmp_path / "LinuxBuild2404.vmsd").write_text("snapdb") + (tmp_path / "LinuxBuild2404.vmdk.lck").write_text("lock") + + removed = tmpl_mod._clean_force_artifacts(tmp_path, "LinuxBuild2404") + + assert set(removed) == { + "LinuxBuild2404-000001.vmdk", + "LinuxBuild2404-000002.vmdk", + "LinuxBuild2404-Snapshot7.vmsn", + "LinuxBuild2404.vmsd", + "LinuxBuild2404.vmdk.lck", + } + # Base disk and vmx survive (Step 4/5 overwrite them). + assert (tmp_path / "LinuxBuild2404.vmdk").is_file() + assert (tmp_path / "LinuxBuild2404.vmx").is_file() + # Chain artifacts gone. + assert not (tmp_path / "LinuxBuild2404-000001.vmdk").exists() + assert not (tmp_path / "LinuxBuild2404.vmsd").exists() + + +def test_clean_force_artifacts_removes_lock_directory(tmp_path: Path) -> None: + """Lock entries may be directories (VMware writes .lck dirs).""" + lck = tmp_path / "LinuxBuild2404.vmx.lck" + lck.mkdir() + (lck / "M12345.lck").write_text("host") + + removed = tmpl_mod._clean_force_artifacts(tmp_path, "LinuxBuild2404") + + assert removed == ["LinuxBuild2404.vmx.lck"] + assert not lck.exists() + + +def test_clean_force_artifacts_noop_on_clean_dir(tmp_path: Path) -> None: + """A clean directory yields no removals and leaves the base disk.""" + (tmp_path / "LinuxBuild2404.vmdk").write_text("base") + + removed = tmpl_mod._clean_force_artifacts(tmp_path, "LinuxBuild2404") + + assert removed == [] + assert (tmp_path / "LinuxBuild2404.vmdk").is_file()