From 6bce8469531024b51e59fe9f3e91a0c5c24435fa Mon Sep 17 00:00:00 2001 From: Simone Date: Wed, 22 Jul 2026 14:08:56 +0200 Subject: [PATCH] fix(template): make deploy/prepare deterministic and safe to re-run Four defects, each of which could waste a full template rebuild: 1. getGuestIPAddress was called without -wait, so vmrun answered "The VMware Tools are not running" unless the Tools happened to have registered the IP at that exact instant. A guest that was booted, on the network and answering SSH could still burn the entire 10-minute Step 7 budget. This race is the likely cause of every intermittent "no guest IP" incident so far, including ones previously attributed to wedged vmnet DHCP. AGENTS.md #10 already prescribed -wait; the code did not follow it. 2. --force did not check whether the target VM was powered on. It deleted the live .vmdk.lck as a stale artifact and Step 4 rewrote the base VMDK under the running guest, killing it; the damage surfaced ten minutes later as an IP timeout because `vmrun list` still showed the OLD instance. 3. Step 6 accepted "VM appears in vmrun list" as proof of power-on, which a pre-existing instance satisfies. Check the exit code of `vmrun start`. 4. The cloud-init seed ISO was detached while the VM was still running, and VMware rewrote the VMX from its in-memory copy at power-off, restoring ide1:0.present = "TRUE". Every clone would boot with the seed attached (AGENTS.md #6). Detach it in prepare, after power-off and before the snapshot. Also in this change: - --then-prepare chains deploy into the matching prepare in one run, reusing the VM deploy leaves running. The two commands stay separately invocable. The guest toolchain script is resolved and verified BEFORE any VM work, so a wrong working directory costs two seconds instead of failing the hand-off after a 45-minute deploy; --toolchain-script allows running from anywhere. - Corrected the final messages: deploy-rust told the user to run prepare-linux, and both deploy commands reported a snapshot name (PostInstall-Linux) that no command ever creates. The prepare hand-off hint passed a bare filename to --template and omitted --snapshot, so copy-pasting it cloned BaseClean, the Windows snapshot. - template backup --exclude, repeatable, for retired templates. An unknown name or --exclude without --all-templates is an error rather than a silent no-op. test_deploy_linux_full_happy_path declared the VM already running before Step 6, an impossible state that masked the new --force guard; its vmrun mock is now stateful. --- src/ci_orchestrator/commands/template.py | 757 ++++++++++++++++++--- tests/python/test_commands_template.py | 809 ++++++++++++++++++++++- 2 files changed, 1457 insertions(+), 109 deletions(-) diff --git a/src/ci_orchestrator/commands/template.py b/src/ci_orchestrator/commands/template.py index 8047f2d..c3aac53 100644 --- a/src/ci_orchestrator/commands/template.py +++ b/src/ci_orchestrator/commands/template.py @@ -70,6 +70,7 @@ import threading import time import urllib.parse import urllib.request +from collections.abc import Callable from datetime import UTC, datetime from pathlib import Path @@ -105,6 +106,17 @@ def template() -> None: is_flag=True, help="Back up every subdirectory under CI_TEMPLATES in a single pass.", ) +@click.option( + "--exclude", + "excludes", + multiple=True, + metavar="NAME", + help=( + "Template directory name to skip with --all-templates. Repeatable. " + "Use for retired or disposable templates whose archives are not worth " + "the disk (a full Windows template costs ~25 GB per generation)." + ), +) @click.option( "--backup-base-dir", default="/var/lib/ci/backups", @@ -130,6 +142,7 @@ def template() -> None: def template_backup( template_path: str | None, all_templates: bool, + excludes: tuple[str, ...], backup_base_dir: str, keep_count: int, skip_runner_stop: bool, @@ -140,15 +153,37 @@ def template_backup( backup_base = Path(backup_base_dir) timestamp = datetime.now(tz=UTC).strftime("%Y%m%d_%H%M%S") + if excludes and not all_templates: + raise click.ClickException( + "--exclude only applies to --all-templates; with a single " + "--template-path it would be a silent no-op." + ) + if all_templates: tmpl_root = config.paths.templates if not tmpl_root.exists(): raise click.ClickException(f"Templates directory not found: {tmpl_root}") - to_backup = sorted(d for d in tmpl_root.iterdir() if d.is_dir()) - if not to_backup: + found = sorted(d for d in tmpl_root.iterdir() if d.is_dir()) + if not found: raise click.ClickException( f"No template subdirectories found under {tmpl_root}" ) + excluded = {name.strip() for name in excludes if name.strip()} + # Report skips explicitly: a silently missing backup is worse than a + # noisy one, and a typo in --exclude must not look like a skip. + unknown = excluded - {d.name for d in found} + if unknown: + raise click.ClickException( + f"--exclude names no such template: {', '.join(sorted(unknown))}. " + f"Available: {', '.join(d.name for d in found)}" + ) + to_backup = [d for d in found if d.name not in excluded] + for name in sorted(excluded): + click.echo(f"[Backup-CITemplate] Skipping '{name}' (--exclude).") + if not to_backup: + raise click.ClickException( + f"Every template under {tmpl_root} was excluded — nothing to back up." + ) else: p = Path(template_path) if template_path else config.paths.templates / "WinBuild2025" if not p.is_dir(): @@ -1029,6 +1064,15 @@ Register-ScheduledTask -TaskName 'CI-StaticIp' ` f"(target='{credential_target}', user='{build_username}')." ) + click.echo("") + click.echo("[prepare-win] Summary") + click.echo(f" VMX: {vmx_path}") + click.echo(f" Snapshot: {snapshot}") + click.echo( + " Next: ci-orchestrator vm new " + f"--template {vmx_path} --snapshot {snapshot} " + "--guest-os windows --clone-base-dir --job-id " + ) click.echo("[prepare-win] Provisioning complete.") @@ -1164,14 +1208,31 @@ def _poll_guest_ip_deploy(vmrun: str, vmx: str, timeout: float) -> str | None: click.echo(f" ... guest IP from guestinfo.ci-ip: {ip}") return ip try: + # -wait is mandatory (AGENTS.md #10): without it vmrun answers + # "Error: The VMware Tools are not running in the virtual machine" + # unless the Tools happen to have registered the IP at that exact + # instant. That race is what made this poll intermittent — a guest + # that was up, on the network and answering SSH could still burn + # the whole 10-minute budget here. With -wait vmrun blocks until + # the Tools report, so the 30s subprocess cap (not the flag) is + # what keeps this loop responsive. result = subprocess.run( - [vmrun, "getGuestIPAddress", vmx], + [vmrun, "getGuestIPAddress", vmx, "-wait"], capture_output=True, text=True, timeout=30, check=False, ) - ip = result.stdout.strip() + # vmrun can exit 0 with "Error: ..." on stdout — skip those lines + # rather than trusting the exit code. + ip = next( + ( + line.strip() + for line in (result.stdout or "").splitlines() + if line.strip() and not line.strip().lower().startswith("error:") + ), + "", + ) if re.match(r"^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$", ip): return ip except subprocess.TimeoutExpired: @@ -1263,8 +1324,12 @@ def _patch_vmx_key(vmx_path: Path, key: str, value: str) -> None: @click.option("--vcpu-count", default=4, show_default=True, help="vCPU count.") @click.option("--cores-per-socket", default=2, show_default=True, help="Cores per socket.") @click.option( - "--snapshot-name", default="PostInstall-Linux", show_default=True, - help="Snapshot name recorded in summary (taken by prepare-linux).", + "--snapshot-name", default="BaseClean-Linux", show_default=True, + help=( + "Snapshot that prepare-linux will create. deploy-linux takes no " + "snapshot itself — this only feeds the summary and the --then-prepare " + "hand-off, so it must match what prepare-linux is asked to create." + ), ) @click.option( "--ssh-key", "ssh_key_str", default=None, metavar="PATH", @@ -1314,6 +1379,23 @@ def _patch_vmx_key(vmx_path: Path, key: str, value: str) -> None: "5=VMX 6=power-on 7=guest-IP 8=SSH-port 9=cloud-init-verify 10=detach-ISO" ), ) +@click.option( + "--then-prepare", is_flag=True, + help=( + "On success, hand off to the matching prepare-* command in the same " + "run (deploy-linux -> prepare-linux, deploy-rust -> prepare-rust). " + "The two commands stay independent and separately invocable; this " + "only chains them, reusing the VM that deploy left running." + ), +) +@click.option( + "--toolchain-script", "toolchain_script", default=None, metavar="PATH", + help=( + "Explicit path to the guest toolchain script for the chained " + "prepare-*. Only read with --then-prepare; without it the script is " + "looked up under /template/. Checked before the deploy starts." + ), +) def template_deploy_linux( vmx_path_str: str | None, vm_name: str, @@ -1337,6 +1419,15 @@ def template_deploy_linux( diag_password: str, vdisk_manager_str: str | None, start_from_step: int, + then_prepare: bool, + toolchain_script: str | None = None, + # Not click options: set by the deploy-rust wrapper so the log prefix, the + # summary and the --then-prepare hand-off name the command the user + # actually typed. + prog_name: str = "deploy-linux", + prepare_prog: str = "prepare-linux", + rust_channel: str | None = None, + no_cargo_audit: bool = False, ) -> None: """Deploy Ubuntu 24.04 LTS build VM template (Linux VMware Workstation host). @@ -1346,6 +1437,22 @@ def template_deploy_linux( """ _ = timezone # accepted for parity with PS script; set timezone via prepare-linux + # Pre-flight the hand-off BEFORE any VM work. Resolving the toolchain script + # inside prepare means a ~45-minute deploy completes and only then fails on + # a missing file, leaving a half-built template. Resolve it now, and pass + # the absolute path down so the chained prepare no longer depends on CWD. + chained_toolchain: str | None = None + if then_prepare: + toolchain_name = ( + "Install-CIToolchain-LinuxRust2404.sh" + if prepare_prog == "prepare-rust" + else "Install-CIToolchain-Linux2404.sh" + ) + chained_toolchain = str( + _resolve_toolchain_script(toolchain_name, toolchain_script) + ) + click.echo(f"[{prog_name}] --then-prepare pre-flight — toolchain: {chained_toolchain}") + config = load_config() # ── Resolve paths ────────────────────────────────────────────────────────── @@ -1416,6 +1523,22 @@ def template_deploy_linux( f"[PreFlight] VM directory already exists: {vm_dir}\n" " Use --force to overwrite." ) + # Refuse to touch a VM that is powered on. --force deletes the + # disk lock as "stale" and Step 4 rewrites the base VMDK: doing + # that under a live guest destroys it, and the damage only + # surfaces later as a Step 7 IP timeout on a VM that looks fine in + # `vmrun list` (it is the OLD instance being listed). + running = subprocess.run( + [vmrun, "-T", "ws", "list"], capture_output=True, text=True, check=False, + ) + if str(vmx_path) in (running.stdout or ""): + raise click.ClickException( + f"[PreFlight] Refusing --force: the VM is currently powered on.\n" + f" {vmx_path}\n" + " Another deploy/prepare may still be using it. Stop it first:\n" + f" vmrun -T ws stop {vmx_path} hard" + ) + click.echo(" [OK] Target VM is powered off") click.echo(f" [WARN] --force set -- reusing existing VM directory: {vm_dir}") cleaned = _clean_force_artifacts(vm_dir, vmx_path.stem) if cleaned: @@ -1620,9 +1743,21 @@ def template_deploy_linux( _deploy_step("Step 6: power on VM", 6, start_from_step) if start_from_step <= 6: gui_mode = "gui" if show_gui else "nogui" - proc = subprocess.Popen([vmrun, "-T", "ws", "start", str(vmx_path), gui_mode]) - click.echo(f" [Deploy] VM power-on issued (pid={proc.pid}).") - time.sleep(10) + # Presence in `vmrun list` is not proof that OUR power-on worked: if + # another instance of this VMX were already running, the list check + # would pass while our vmrun start died on a busy VMX. The pre-flight + # above rules that out, so here we additionally require the start + # command itself to succeed instead of only inspecting the list. + start_r = subprocess.run( + [vmrun, "-T", "ws", "start", str(vmx_path), gui_mode], + capture_output=True, text=True, check=False, + ) + if start_r.returncode != 0: + detail = ((start_r.stdout or "") + (start_r.stderr or "")).strip() + raise click.ClickException( + f"[Deploy Step 6] vmrun start failed (exit={start_r.returncode}): {detail}" + ) + click.echo(" [Deploy] VM power-on completed.") list_r = subprocess.run( [vmrun, "-T", "ws", "list"], capture_output=True, text=True, check=False, ) @@ -1699,18 +1834,238 @@ def template_deploy_linux( shutil.rmtree(staging_dir, ignore_errors=True) # Summary - click.echo("\n=== Done ===\n") - click.echo("[Deploy] Summary") - click.echo(f" VMX: {vmx_path}") - click.echo(f" Guest IP: {guest_ip or '(not obtained)'}") - click.echo(f" SSH key: {ssh_key_path}") - click.echo(f" Snapshot name: {snapshot_name} (run template prepare-linux next)") - click.echo("") + click.echo(f"\n=== {prog_name} complete ===\n") + click.echo(f"[{prog_name}] Summary") + click.echo(f" VMX: {vmx_path}") + click.echo(f" Guest IP: {guest_ip or '(not obtained)'}") + click.echo(f" SSH key: {ssh_key_path}") + click.echo(f" Snapshot: none yet — {prepare_prog} creates '{snapshot_name}'") if guest_ip: - click.echo(f" SSH: ssh -i {ssh_key_path} {cloud_user}@{guest_ip}") + click.echo(f" SSH: ssh -i {ssh_key_path} {cloud_user}@{guest_ip}") click.echo("") click.echo(" VM is powered on. cloud-init provisioning complete.") + if not then_prepare: + click.echo( + f" Next: ci-orchestrator template {prepare_prog} " + f"--vmx {vmx_path}" + ) + return + + # ── Hand-off to prepare-* ───────────────────────────────────────────────── + # The VM is up and cloud-init has finished, so prepare reuses it instead of + # cold-booting: pass both the VMX (needed for the snapshot) and the IP we + # already resolved (skips a second detection round). + click.echo("") + click.echo(f"[{prog_name}] --then-prepare — handing off to {prepare_prog} ...") + if prepare_prog == "prepare-rust": + toolchain_name = "Install-CIToolchain-LinuxRust2404.sh" + template_dir_name = "LinuxRust2404" + else: + toolchain_name = "Install-CIToolchain-Linux2404.sh" + template_dir_name = "LinuxBuild2404" + _prepare_linux_impl( + prog_name=prepare_prog, + default_toolchain_name=toolchain_name, + default_vmx_name=template_dir_name, + vmx_path_str=str(vmx_path), + guest_ip=guest_ip or "", + cloud_user=cloud_user, + ssh_key_str=str(ssh_key_path), + snapshot=snapshot_name, + # Absolute path resolved during pre-flight: the chained prepare must + # not re-resolve against CWD, which may have changed meaning by now. + toolchain_script_str=chained_toolchain, + skip_update=False, + install_dotnet=False, + no_mingw=False, + ssh_timeout=ssh_timeout, + take_snapshot=True, + rust_channel=rust_channel, + no_cargo_audit=no_cargo_audit, + ) + + +# ═══════════════════════════════════════════════════════════════════════════ +# template deploy-rust command (thin wrapper over deploy-linux) +# ═══════════════════════════════════════════════════════════════════════════ + + +@template.command("deploy-rust") +@click.option( + "--vmx", "vmx_path_str", default=None, metavar="PATH", + help="VMX file to create (default: /LinuxRust2404/LinuxRust2404.vmx).", +) +@click.option("--vm-name", default="LinuxRust2404", show_default=True, help="VM display name.") +@click.option("--cloud-user", default="ci_build", show_default=True, help="cloud-init username.") +@click.option( + "--hostname", "guest_hostname", default="ci-rust-template", show_default=True, + help="Guest hostname.", +) +@click.option("--timezone", default="Europe/Rome", show_default=True, help="IANA timezone (reserved).") +@click.option("--disk-size-gb", default=60, show_default=True, help="Disk size after resize (GB).") +@click.option("--memory-mb", default=4096, show_default=True, help="Guest RAM (MiB).") +@click.option("--vcpu-count", default=4, show_default=True, help="vCPU count.") +@click.option("--cores-per-socket", default=2, show_default=True, help="Cores per socket.") +@click.option( + "--snapshot-name", default="BaseClean-LinuxRust", show_default=True, + help=( + "Snapshot that prepare-rust will create. deploy-rust takes no " + "snapshot itself — this only feeds the summary and the --then-prepare " + "hand-off, so it must match what prepare-rust is asked to create." + ), +) +@click.option( + "--ssh-key", "ssh_key_str", default=None, metavar="PATH", + help="SSH private key (default: from config ssh_key_path or /ci_linux).", +) +@click.option( + "--vmdk-url", + default="https://cloud-images.ubuntu.com/noble/current/noble-server-cloudimg-amd64.vmdk", + show_default=True, + help="Ubuntu cloud VMDK download URL.", +) +@click.option( + "--vmdk-cache", "vmdk_cache_str", default=None, metavar="PATH", + help="Local VMDK cache path (default: /iso/noble-server-cloudimg-amd64.vmdk).", +) +@click.option( + "--vmdk-sha256", "vmdk_sha256", default="", + help="Expected SHA256 of cached VMDK (empty = skip verification).", +) +@click.option( + "--seed-iso", "seed_iso_str", default=None, metavar="PATH", + help="Seed ISO path (default: /cloud-init-seed.iso).", +) +@click.option( + "--ssh-timeout", default=300, show_default=True, help="Seconds to wait for SSH port 22.", +) +@click.option( + "--ssh-poll-interval", default=10, show_default=True, + help="Seconds between SSH port poll attempts.", +) +@click.option("--show-gui", is_flag=True, help="Power on with VMware Workstation GUI visible.") +@click.option("--force", is_flag=True, help="Allow overwriting an existing VM directory.") +@click.option( + "--diag-password", "diag_password", default="", + help="Set plaintext password on ubuntu account for console login (diagnostic only).", +) +@click.option( + "--vdisk-manager", "vdisk_manager_str", default=None, metavar="PATH", + help="Path to vmware-vdiskmanager (auto-detected if omitted).", +) +@click.option( + "--start-from-step", "start_from_step", default=1, + type=click.IntRange(1, 10), show_default=True, + help=( + "Skip to this step (artifacts from skipped steps must exist). " + "Steps: 1=prereqs+vmdk 2=cloud-init 3=seed-ISO 4=vmdk-copy+resize " + "5=VMX 6=power-on 7=guest-IP 8=SSH-port 9=cloud-init-verify 10=detach-ISO" + ), +) +@click.option( + "--then-prepare", is_flag=True, + help=( + "On success, hand off to prepare-rust in the same run, reusing the VM " + "deploy-rust left running. Both commands stay separately invocable." + ), +) +@click.option( + "--rust-channel", default="stable", show_default=True, + help=( + "Rust toolchain baked by the chained prepare-rust. Only read with " + "--then-prepare. Pin an exact version (e.g. 1.97.1) for a " + "deterministic snapshot; 'stable' is a moving target." + ), +) +@click.option( + "--no-cargo-audit", is_flag=True, + help=( + "Skip baking the cargo-audit binary in the chained prepare-rust. " + "Only read with --then-prepare." + ), +) +@click.option( + "--toolchain-script", "toolchain_script", default=None, metavar="PATH", + help=( + "Explicit path to Install-CIToolchain-LinuxRust2404.sh for the chained " + "prepare-rust. Only read with --then-prepare; checked before the " + "deploy starts." + ), +) +def template_deploy_rust( + vmx_path_str: str | None, + vm_name: str, + cloud_user: str, + guest_hostname: str, + timezone: str, + disk_size_gb: int, + memory_mb: int, + vcpu_count: int, + cores_per_socket: int, + snapshot_name: str, + ssh_key_str: str | None, + vmdk_url: str, + vmdk_cache_str: str | None, + vmdk_sha256: str, + seed_iso_str: str | None, + ssh_timeout: int, + ssh_poll_interval: int, + show_gui: bool, + force: bool, + diag_password: str, + vdisk_manager_str: str | None, + start_from_step: int, + then_prepare: bool, + rust_channel: str, + no_cargo_audit: bool, + toolchain_script: str | None, +) -> None: + """Deploy the Rust Linux template VM (LinuxRust2404, Ubuntu 24.04). + + Identical to deploy-linux (same cloud image and VMX layout) but defaults + to the LinuxRust2404 directory/name and a larger disk (60 GB) to hold the + baked Rust toolchain. Run `template prepare-rust` next, or pass + --then-prepare to chain both in one run. + """ + if vmx_path_str is None: + config = load_config() + vmx_path_str = str( + config.paths.templates / "LinuxRust2404" / "LinuxRust2404.vmx" + ) + deploy_linux_cb = template_deploy_linux.callback + assert deploy_linux_cb is not None # click always sets callback for @command + deploy_linux_cb( + vmx_path_str=vmx_path_str, + vm_name=vm_name, + cloud_user=cloud_user, + guest_hostname=guest_hostname, + timezone=timezone, + disk_size_gb=disk_size_gb, + memory_mb=memory_mb, + vcpu_count=vcpu_count, + cores_per_socket=cores_per_socket, + snapshot_name=snapshot_name, + ssh_key_str=ssh_key_str, + vmdk_url=vmdk_url, + vmdk_cache_str=vmdk_cache_str, + vmdk_sha256=vmdk_sha256, + seed_iso_str=seed_iso_str, + ssh_timeout=ssh_timeout, + ssh_poll_interval=ssh_poll_interval, + show_gui=show_gui, + force=force, + diag_password=diag_password, + vdisk_manager_str=vdisk_manager_str, + start_from_step=start_from_step, + then_prepare=then_prepare, + toolchain_script=toolchain_script, + prog_name="deploy-rust", + prepare_prog="prepare-rust", + rust_channel=rust_channel, + no_cargo_audit=no_cargo_audit, + ) + # ═══════════════════════════════════════════════════════════════════════════ # template prepare-linux helpers @@ -1764,6 +2119,39 @@ def _pl_run( return rc, out, err +def _resolve_toolchain_script(default_name: str, explicit: str | None) -> Path: + """Locate the guest-side toolchain script, or fail listing what was tried. + + Resolution order: an explicit ``--toolchain-script``, then ``template/`` + under the current directory, then ``template/`` next to a source checkout of + this package (editable installs only — a non-editable venv has no repo to + walk back to, which is why the systemd units set ``WorkingDirectory``). + + Callers that chain deploy -> prepare must call this **before** starting any + VM work: resolving it late means a 45-minute deploy succeeds and then the + hand-off dies on a missing file. + """ + if explicit is not None: + p = Path(explicit).expanduser() + if not p.is_file(): + raise click.ClickException(f"Toolchain script not found: {p}") + return p + + # Deliberately only /template/. A fallback relative to this module + # would resolve in a source checkout and still fail in the production venv + # (installed non-editable, no repo to walk back to) — dev and prod would + # disagree, which is the failure mode this whole helper exists to remove. + candidate = Path.cwd() / "template" / default_name + if candidate.is_file(): + return candidate + + raise click.ClickException( + f"Toolchain script {default_name!r} not found at {candidate}.\n" + "Run from the repo root (cd /opt/ci/local-ci-cd-system), or pass " + "--toolchain-script with an explicit path." + ) + + def _pl_stream(client: paramiko.SSHClient, cmd: str) -> int: """Run cmd and stream stdout/stderr to console in real time. Returns exit code.""" transport = client.get_transport() @@ -1807,46 +2195,11 @@ def _pl_upload(client: paramiko.SSHClient, local: Path, remote: str) -> None: # ═══════════════════════════════════════════════════════════════════════════ -@template.command("prepare-linux") -@click.option( - "--vmx", "vmx_path_str", default=None, metavar="PATH", - help=( - "Template VM .vmx file. Enables auto power-on, IP detection, " - "and automatic shutdown + snapshot at the end." - ), -) -@click.option( - "--ip", "guest_ip", default="", - help="Guest IP address (auto-detected via vmrun if --vmx is supplied).", -) -@click.option("--cloud-user", default="ci_build", show_default=True, help="SSH username in guest.") -@click.option( - "--ssh-key", "ssh_key_str", default=None, metavar="PATH", - help="SSH private key (default: from config ssh_key_path or /ci_linux).", -) -@click.option( - "--snapshot", default="BaseClean-Linux", show_default=True, - help="Snapshot name (must match the name used by vm new).", -) -@click.option( - "--toolchain-script", "toolchain_script_str", default=None, metavar="PATH", - help=( - "Path to Install-CIToolchain-Linux2404.sh " - "(auto-detected from /template/ if omitted)." - ), -) -@click.option("--skip-update", is_flag=True, help="Pass --skip-update to toolchain script.") -@click.option("--install-dotnet", is_flag=True, help="Pass --dotnet to toolchain script.") -@click.option("--no-mingw", is_flag=True, help="Pass --no-mingw to toolchain script.") -@click.option( - "--ssh-timeout", default=180, show_default=True, - help="Seconds to wait for SSH port 22 to become reachable.", -) -@click.option( - "--take-snapshot", is_flag=True, - help="Shut down and take snapshot after provisioning (always implied when --vmx is set).", -) -def template_prepare_linux( +def _prepare_linux_impl( + *, + prog_name: str, + default_toolchain_name: str, + default_vmx_name: str, vmx_path_str: str | None, guest_ip: str, cloud_user: str, @@ -1858,11 +2211,19 @@ def template_prepare_linux( no_mingw: bool, ssh_timeout: int, take_snapshot: bool, + rust_channel: str | None = None, + no_cargo_audit: bool = False, ) -> None: - """Provision Linux template VM via SSH (ports Prepare-LinuxBuild2404.ps1). + """Shared implementation of ``prepare-linux`` / ``prepare-rust``. - Requires either --vmx (auto power-on + IP detection + snapshot) or - --ip (explicit IP, no auto VM lifecycle), or both. + ``prog_name`` is used for log prefixes (``[{prog_name}]`` / + ``[prepare-rust]``); ``default_toolchain_name`` is the toolchain script + auto-detected under ``/template/`` when ``--toolchain-script`` + is not passed; ``default_vmx_name`` is the template directory name used to + resolve the default VMX (``//.vmx``) when neither + ``--vmx`` nor ``--ip`` is given. Rust-only flags (``rust_channel`` / + ``no_cargo_audit``) are forwarded to the toolchain script only when set + (i.e. by ``prepare-rust``). """ config = load_config() @@ -1875,21 +2236,24 @@ def template_prepare_linux( ssh_key_path = config.paths.keys / "ci_linux" # ── Resolve VMX ─────────────────────────────────────────────────────────── - vmx_path = Path(vmx_path_str).resolve() if vmx_path_str else None + # Default: //.vmx, but + # only when it exists — otherwise fall through to the --vmx/--ip check + # below so the error message still makes sense on hosts without the VM. + vmx_path: Path | None + if vmx_path_str: + vmx_path = Path(vmx_path_str).resolve() + elif not guest_ip: + default_vmx = ( + config.paths.templates / default_vmx_name / f"{default_vmx_name}.vmx" + ) + vmx_path = default_vmx if default_vmx.is_file() else None + else: + vmx_path = None # ── Resolve toolchain script ────────────────────────────────────────────── - if toolchain_script_str is None: - candidate = Path.cwd() / "template" / "Install-CIToolchain-Linux2404.sh" - if not candidate.is_file(): - raise click.ClickException( - f"Toolchain script not found at {candidate}. " - "Run from the repo root or pass --toolchain-script." - ) - toolchain_path = candidate - else: - toolchain_path = Path(toolchain_script_str) - if not toolchain_path.is_file(): - raise click.ClickException(f"Toolchain script not found: {toolchain_path}") + toolchain_path = _resolve_toolchain_script( + default_toolchain_name, toolchain_script_str + ) if vmx_path is None and not guest_ip: raise click.ClickException( @@ -1899,7 +2263,7 @@ def template_prepare_linux( do_snapshot = take_snapshot or vmx_path is not None # ── Step 1: Pre-flight ──────────────────────────────────────────────────── - click.echo("[prepare-linux] Step 1 — Pre-flight validation") + click.echo(f"[{prog_name}] Step 1 — Pre-flight validation") if not ssh_key_path.is_file(): raise click.ClickException(f"SSH private key not found: {ssh_key_path}") click.echo(f" [OK] SSH key: {ssh_key_path}") @@ -1919,36 +2283,36 @@ def template_prepare_linux( handle = VmHandle(identifier=str(vmx_path), name=vmx_path.stem) if not backend.is_running(handle): - click.echo(f"[prepare-linux] Starting VM: {vmx_path.name} ...") + click.echo(f"[{prog_name}] Starting VM: {vmx_path.name} ...") backend.start(handle, headless=True) else: - click.echo(f"[prepare-linux] VM already running: {vmx_path.name}") + click.echo(f"[{prog_name}] VM already running: {vmx_path.name}") # VM process alive but guest may be powered off (e.g. after a # previous prepare run that shut down the guest). Do a quick IP # probe; if nothing comes back, stop and cold-boot the VM. if not guest_ip: - click.echo("[prepare-linux] Quick IP probe (30s) ...") + click.echo(f"[{prog_name}] Quick IP probe (30s) ...") quick_ip = _poll_guest_ip_deploy(vmrun, str(vmx_path), timeout=30.0) if not quick_ip: click.echo( - "[prepare-linux] Guest not responding — stopping and restarting VM ..." + f"[{prog_name}] Guest not responding — stopping and restarting VM ..." ) backend.stop(handle) backend.start(handle, headless=True) if not guest_ip: - click.echo("[prepare-linux] Detecting guest IP (vmrun getGuestIPAddress) ...") + click.echo(f"[{prog_name}] Detecting guest IP (vmrun getGuestIPAddress) ...") detected = _poll_guest_ip_deploy(vmrun, str(vmx_path), timeout=300.0) if not detected: raise click.ClickException( "Could not get guest IP. Ensure open-vm-tools is running in the guest." ) guest_ip = detected - click.echo(f"[prepare-linux] Guest IP: {guest_ip}") + click.echo(f"[{prog_name}] Guest IP: {guest_ip}") # ── Step 3: Wait for SSH port 22 ────────────────────────────────────────── click.echo( - f"[prepare-linux] Step 3 — Waiting for SSH port 22 on {guest_ip} " + f"[{prog_name}] Step 3 — Waiting for SSH port 22 on {guest_ip} " f"(timeout {ssh_timeout}s) ..." ) if not _wait_ssh_port_deploy(guest_ip, 22, float(ssh_timeout), 10.0): @@ -1958,7 +2322,7 @@ def template_prepare_linux( click.echo(f" [OK] TCP/22 reachable on {guest_ip}") # ── Step 4: SSH connectivity test ───────────────────────────────────────── - click.echo("[prepare-linux] Step 4 — Testing SSH connectivity ...") + click.echo(f"[{prog_name}] Step 4 — Testing SSH connectivity ...") try: client = _pl_connect(guest_ip, cloud_user, str(ssh_key_path)) except Exception as exc: @@ -1974,14 +2338,14 @@ def template_prepare_linux( try: # ── Step 5: Upload toolchain script ─────────────────────────────────── click.echo( - f"[prepare-linux] Step 5 — Uploading {toolchain_path.name} to guest /tmp/ ..." + f"[{prog_name}] Step 5 — Uploading {toolchain_path.name} to guest /tmp/ ..." ) raw = toolchain_path.read_bytes() if raw[:3] == b"\xef\xbb\xbf": click.echo(" [WARN] BOM detected — stripping") raw = raw[3:] normalized = raw.replace(b"\r\n", b"\n").replace(b"\r", b"\n") - remote_script = "/tmp/Install-CIToolchain-Linux2404.sh" + remote_script = f"/tmp/{toolchain_path.name}" with tempfile.NamedTemporaryFile(suffix=".sh", delete=False) as tf: tf.write(normalized) @@ -2008,10 +2372,14 @@ def template_prepare_linux( flags.append("--dotnet") if no_mingw: flags.append("--no-mingw") + if rust_channel is not None: + flags.extend(["--rust-channel", rust_channel]) + if no_cargo_audit: + flags.append("--no-cargo-audit") flags_str = " ".join(flags) remote_cmd = f"chmod +x {remote_script} && sudo {remote_script} {flags_str}".strip() - click.echo("[prepare-linux] Step 6 — Running toolchain script (may take several minutes)") + click.echo(f"[{prog_name}] Step 6 — Running toolchain script (may take several minutes)") click.echo(f" Command: {remote_cmd}") click.echo("-" * 60) exit_code = _pl_stream(client, remote_cmd) @@ -2020,12 +2388,12 @@ def template_prepare_linux( raise click.ClickException( f"Toolchain script exited {exit_code} — see output above." ) - click.echo("[prepare-linux] Toolchain script completed (exit 0).") + click.echo(f"[{prog_name}] Toolchain script completed (exit 0).") # ── Step 7: Post-setup validation ───────────────────────────────────── - click.echo("[prepare-linux] Step 7 — Running post-setup validation ...") + click.echo(f"[{prog_name}] Step 7 — Running post-setup validation ...") _, val_out, _ = _pl_run(client, _PREPARE_LINUX_VALIDATION, timeout=120.0) - click.echo("[prepare-linux] Validation results:") + click.echo(f"[{prog_name}] Validation results:") failures: list[str] = [] for line in val_out.splitlines(): line = line.strip() @@ -2039,10 +2407,10 @@ def template_prepare_linux( "Post-setup validation failed:\n" + "\n".join(f" {f}" for f in failures) ) - click.echo("[prepare-linux] All post-setup checks passed.") + click.echo(f"[{prog_name}] All post-setup checks passed.") # ── Step 7b: Clear machine-id (DHCP uniqueness for clones) ─────────── - click.echo("[prepare-linux] Step 7b — Clearing machine-id ...") + click.echo(f"[{prog_name}] Step 7b — Clearing machine-id ...") rc2, _, err2 = _pl_run( client, "sudo truncate -s 0 /etc/machine-id && sudo rm -f /var/lib/dbus/machine-id", @@ -2054,7 +2422,7 @@ def template_prepare_linux( raise click.ClickException( f"/etc/machine-id not empty after truncate (size={size_out2.strip()!r})." ) - click.echo("[prepare-linux] machine-id cleared — clones get unique DHCP ID.") + click.echo(f"[{prog_name}] machine-id cleared — clones get unique DHCP ID.") # ── Step 7c: Enable NTP time sync (clock-skew guard for clones) ────── # Bake persistent NTP into the snapshot so every clone boots and @@ -2062,7 +2430,7 @@ def template_prepare_linux( # drifted behind the host produces source mtimes "in the future" and # make emits "Clock skew detected". This is the clean fix; the build # path also force-syncs + touches sources as a runtime safety net. - click.echo("[prepare-linux] Step 7c — Enabling NTP time sync ...") + click.echo(f"[{prog_name}] Step 7c — Enabling NTP time sync ...") # set-local-rtc 0 forces the RTC to be interpreted as UTC. Without it # a clone can boot hours ahead (RTC in local time read as UTC); NTP # then steps the clock *backward* at runtime, which is what produces @@ -2075,7 +2443,7 @@ def template_prepare_linux( ) if rc_ntp != 0: raise click.ClickException(f"enabling NTP failed (exit {rc_ntp}):\n{err_ntp}") - click.echo("[prepare-linux] NTP time sync enabled.") + click.echo(f"[{prog_name}] NTP time sync enabled.") # ── Step 7d: Seal cloud-init (runs once at deploy, never in clones) ─── # cloud-init provisioned the image at deploy time. It must not run @@ -2089,7 +2457,7 @@ def template_prepare_linux( # applied. We seal the image: drop the stale cloud-init netplan, # leaving only the MAC-agnostic 99-ci-dhcp-allnics.yaml, regenerate # networkd config, then disable cloud-init for all future boots. - click.echo("[prepare-linux] Step 7d — Sealing cloud-init (disable for clones) ...") + click.echo(f"[{prog_name}] Step 7d — Sealing cloud-init (disable for clones) ...") rc_seal, _, err_seal = _pl_run( client, "sudo rm -f /etc/netplan/50-cloud-init.yaml " @@ -2112,17 +2480,17 @@ def template_prepare_linux( f"cloud-init seal verification failed (expected DISABLED+NO50):\n{seal_out}" ) click.echo( - "[prepare-linux] cloud-init sealed — clones boot with static netplan only." + f"[{prog_name}] cloud-init sealed — clones boot with static netplan only." ) # ── Step 8: Graceful shutdown ───────────────────────────────────────── if do_snapshot: - click.echo("[prepare-linux] Step 8 — Sending graceful shutdown ...") + click.echo(f"[{prog_name}] Step 8 — Sending graceful shutdown ...") with contextlib.suppress(Exception): _pl_run(client, "sudo shutdown -h now", timeout=10.0) else: click.echo( - "[prepare-linux] --take-snapshot not set — VM left running. " + f"[{prog_name}] --take-snapshot not set — VM left running. " "Shut down and snapshot manually when ready." ) @@ -2131,7 +2499,7 @@ def template_prepare_linux( # ── Post-SSH: wait for power-off + snapshot ─────────────────────────────── if do_snapshot and vmx_path is not None and backend is not None and handle is not None: - click.echo("[prepare-linux] Waiting for VM to power off (120s) ...") + click.echo(f"[{prog_name}] Waiting for VM to power off (120s) ...") off_deadline = time.monotonic() + 120 while time.monotonic() < off_deadline: if not backend.is_running(handle): @@ -2140,38 +2508,211 @@ def template_prepare_linux( click.echo(" ... waiting for shutdown ...") else: raise click.ClickException("VM did not power off within 120s.") - click.echo("[prepare-linux] VM powered off.") + click.echo(f"[{prog_name}] VM powered off.") + + # Detach the cloud-init seed ISO now, with the VM off and before the + # snapshot. deploy's Step 10 already does this, but it runs while the + # VM is still powered on and VMware rewrites the VMX from its in-memory + # copy at power-off, silently restoring ide1:0.present = "TRUE" (the + # give-away is `cleanShutdown = "TRUE"` appearing in the file). Left + # attached, every clone boots with the seed still present — AGENTS.md + # error #6. Idempotent: harmless when already FALSE. + if any( + line.strip().lower().startswith("ide1:0.present") + and "true" in line.lower() + for line in vmx_path.read_text(encoding="ascii", errors="replace").splitlines() + ): + _patch_vmx_key(vmx_path, "ide1:0.present", "FALSE") + _patch_vmx_key(vmx_path, "ide1:0.startConnected", "FALSE") + click.echo( + f"[{prog_name}] Detached cloud-init seed ISO from VMX " + "(re-attached by VMware at power-off)." + ) existing = backend.list_snapshots(handle) if snapshot in existing: - click.echo(f"[prepare-linux] Deleting existing snapshot '{snapshot}' ...") + click.echo(f"[{prog_name}] Deleting existing snapshot '{snapshot}' ...") subprocess.run( [vmrun, "-T", "ws", "deleteSnapshot", str(vmx_path), snapshot], check=True, ) - click.echo(f"[prepare-linux] Taking snapshot '{snapshot}' ...") + click.echo(f"[{prog_name}] Taking snapshot '{snapshot}' ...") subprocess.run( [vmrun, "-T", "ws", "snapshot", str(vmx_path), snapshot], check=True, ) - click.echo(f"[prepare-linux] Snapshot '{snapshot}' created.") + click.echo(f"[{prog_name}] Snapshot '{snapshot}' created.") # ── Summary ─────────────────────────────────────────────────────────────── click.echo("") - click.echo("[prepare-linux] Summary") + click.echo(f"[{prog_name}] Summary") click.echo(f" Guest IP: {guest_ip}") click.echo(f" SSH key: {ssh_key_path}") if vmx_path: click.echo(f" VMX: {vmx_path}") if do_snapshot and vmx_path is not None: + click.echo(f" Snapshot: {snapshot}") + # --template takes the full VMX path, and --snapshot must be passed + # explicitly: `vm new` defaults to "BaseClean", which is the Windows + # snapshot, not the one prepare-* just created. click.echo( - f" Snapshot: {snapshot} — ready for " - f"`ci-orchestrator vm new --template {vmx_path.name}`" + " Next: ci-orchestrator vm new " + f"--template {vmx_path} --snapshot {snapshot} " + "--guest-os linux --clone-base-dir --job-id " ) else: click.echo(f" Snapshot: not taken — name it '{snapshot}' when done manually.") - click.echo("[prepare-linux] Provisioning complete.") + click.echo(f"[{prog_name}] Provisioning complete.") + + +# ═══════════════════════════════════════════════════════════════════════════ +# template prepare-linux / prepare-rust commands (thin wrappers over the impl) +# ═══════════════════════════════════════════════════════════════════════════ + + +def _prepare_linux_options(f: Callable[..., None]) -> Callable[..., None]: + """Shared click options for prepare-linux / prepare-rust.""" + f = click.option( + "--vmx", "vmx_path_str", default=None, metavar="PATH", + help=( + "Template VM .vmx file. Enables auto power-on, IP detection, " + "and automatic shutdown + snapshot at the end. Defaults to the " + "template VMX under when neither --vmx nor --ip " + "is given." + ), + )(f) + f = click.option( + "--ip", "guest_ip", default="", + help="Guest IP address (auto-detected via vmrun if --vmx is supplied).", + )(f) + f = click.option( + "--cloud-user", default="ci_build", show_default=True, + help="SSH username in guest.", + )(f) + f = click.option( + "--ssh-key", "ssh_key_str", default=None, metavar="PATH", + help="SSH private key (default: from config ssh_key_path or /ci_linux).", + )(f) + f = click.option( + "--snapshot", default=None, + help="Snapshot name (must match the name used by vm new).", + )(f) + f = click.option( + "--toolchain-script", "toolchain_script_str", default=None, metavar="PATH", + help="Path to the toolchain script (auto-detected from /template/ if omitted).", + )(f) + f = click.option( + "--skip-update", is_flag=True, help="Pass --skip-update to toolchain script." + )(f) + f = click.option( + "--install-dotnet", is_flag=True, help="Pass --dotnet to toolchain script." + )(f) + f = click.option( + "--no-mingw", is_flag=True, help="Pass --no-mingw to toolchain script." + )(f) + f = click.option( + "--ssh-timeout", default=180, show_default=True, + help="Seconds to wait for SSH port 22 to become reachable.", + )(f) + f = click.option( + "--take-snapshot", is_flag=True, + help="Shut down and take snapshot after provisioning (always implied when --vmx is set).", + )(f) + return f + + +@template.command("prepare-linux") +@_prepare_linux_options +def template_prepare_linux( + vmx_path_str: str | None, + guest_ip: str, + cloud_user: str, + ssh_key_str: str | None, + snapshot: str | None, + toolchain_script_str: str | None, + skip_update: bool, + install_dotnet: bool, + no_mingw: bool, + ssh_timeout: int, + take_snapshot: bool, +) -> None: + """Provision Linux template VM via SSH (ports Prepare-LinuxBuild2404.ps1). + + Without --vmx/--ip, defaults to the LinuxBuild2404 template VMX under + . Pass --ip to provision a guest with no VM lifecycle. + """ + _prepare_linux_impl( + prog_name="prepare-linux", + default_toolchain_name="Install-CIToolchain-Linux2404.sh", + default_vmx_name="LinuxBuild2404", + vmx_path_str=vmx_path_str, + guest_ip=guest_ip, + cloud_user=cloud_user, + ssh_key_str=ssh_key_str, + snapshot=snapshot or "BaseClean-Linux", + toolchain_script_str=toolchain_script_str, + skip_update=skip_update, + install_dotnet=install_dotnet, + no_mingw=no_mingw, + ssh_timeout=ssh_timeout, + take_snapshot=take_snapshot, + ) + + +@template.command("prepare-rust") +@_prepare_linux_options +@click.option( + "--rust-channel", default="stable", show_default=True, + help=( + "Rust toolchain channel baked into the snapshot. Pin an exact version " + "(e.g. 1.97.1) for a deterministic snapshot; 'stable' is a moving " + "target and rustup re-downloads at runtime when a new stable ships." + ), +) +@click.option( + "--no-cargo-audit", is_flag=True, + help="Skip baking the cargo-audit binary (the advisory DB is never baked).", +) +def template_prepare_rust( + vmx_path_str: str | None, + guest_ip: str, + cloud_user: str, + ssh_key_str: str | None, + snapshot: str | None, + toolchain_script_str: str | None, + skip_update: bool, + install_dotnet: bool, + no_mingw: bool, + ssh_timeout: int, + take_snapshot: bool, + rust_channel: str, + no_cargo_audit: bool, +) -> None: + """Provision the Rust Linux template (LinuxRust2404) via SSH. + + Same flow as prepare-linux but uses Install-CIToolchain-LinuxRust2404.sh + (adds Tauri/webview GTK libs + system-wide rustup toolchain) and defaults + to the BaseClean-LinuxRust snapshot. + """ + _prepare_linux_impl( + prog_name="prepare-rust", + default_toolchain_name="Install-CIToolchain-LinuxRust2404.sh", + default_vmx_name="LinuxRust2404", + vmx_path_str=vmx_path_str, + guest_ip=guest_ip, + cloud_user=cloud_user, + ssh_key_str=ssh_key_str, + snapshot=snapshot or "BaseClean-LinuxRust", + toolchain_script_str=toolchain_script_str, + skip_update=skip_update, + install_dotnet=install_dotnet, + no_mingw=no_mingw, + ssh_timeout=ssh_timeout, + take_snapshot=take_snapshot, + rust_channel=rust_channel, + no_cargo_audit=no_cargo_audit, + ) # ═══════════════════════════════════════════════════════════════════════════ @@ -2617,8 +3158,8 @@ def template_deploy_win( click.echo("\n=== deploy-win complete ===") click.echo(f" VMX: {vmx_path}") - click.echo(f" Snapshot: {snapshot_name}") - click.echo(f" Next: ci template prepare-win --vmx {vmx_path}") + click.echo(f" Snapshot: {snapshot_name} (cold, taken by deploy-win itself)") + click.echo(f" Next: ci-orchestrator template prepare-win --vmx {vmx_path}") # ═══════════════════════════════════════════════════════════════════════════ diff --git a/tests/python/test_commands_template.py b/tests/python/test_commands_template.py index 9e3a627..7f10b68 100644 --- a/tests/python/test_commands_template.py +++ b/tests/python/test_commands_template.py @@ -1710,6 +1710,11 @@ def test_deploy_linux_full_happy_path( vmx = deploy_env["vmx"] + # `vmrun list` must reflect reality: the VM is NOT running before Step 6 + # powers it on. A mock that always reports it running hides the --force + # pre-flight guard (deploy must refuse to rewrite a live VM's disk). + started = {"value": False} + def fake_run(cmd: list[str], **kw: object) -> _sub.CompletedProcess[str]: cmd_str = " ".join(str(c) for c in cmd) if "genisoimage" in cmd_str: @@ -1723,8 +1728,16 @@ def test_deploy_linux_full_happy_path( return _sub.CompletedProcess(cmd, 0, "", "") if "vmware-vdiskmanager" in cmd_str: return _sub.CompletedProcess(cmd, 0, "", "") + if "start" in cmd: + started["value"] = True + return _sub.CompletedProcess(cmd, 0, "", "") if "list" in cmd: - return _sub.CompletedProcess(cmd, 0, f"Total running VMs: 1\n{vmx}\n", "") + body = ( + f"Total running VMs: 1\n{vmx}\n" + if started["value"] + else "Total running VMs: 0\n" + ) + return _sub.CompletedProcess(cmd, 0, body, "") if "getGuestIPAddress" in cmd: return _sub.CompletedProcess(cmd, 0, "192.168.79.200", "") return _sub.CompletedProcess(cmd, 0, "", "") @@ -1913,11 +1926,37 @@ def _stub_prepare( def test_prepare_linux_missing_vmx_and_ip( prepare_env: dict, monkeypatch: pytest.MonkeyPatch ) -> None: + """Without --vmx/--ip AND no default template VMX on disk → error.""" + prepare_env["vmx"].unlink() # default LinuxBuild2404.vmx absent result = CliRunner().invoke(cli, ["template", "prepare-linux"]) assert result.exit_code != 0 assert "--vmx" in result.output or "--ip" in result.output +def test_prepare_linux_default_vmx_used( + prepare_env: dict, monkeypatch: pytest.MonkeyPatch +) -> None: + """Without --vmx/--ip, the default LinuxBuild2404 VMX is picked up.""" + _stub_prepare(monkeypatch, prepare_env) + monkeypatch.setattr(tmpl_mod, "_poll_guest_ip_deploy", lambda *a, **kw: "192.168.1.200") + + backend_m = MagicMock() + backend_m.is_running.side_effect = [False, False] + backend_m.list_snapshots.return_value = [] + monkeypatch.setattr(tmpl_mod, "WorkstationVmrunBackend", lambda **kw: backend_m) + + import subprocess as _sub + monkeypatch.setattr( + tmpl_mod.subprocess, "run", + lambda cmd, **kw: _sub.CompletedProcess(list(cmd), 0, "", ""), + ) + + result = CliRunner().invoke(cli, ["template", "prepare-linux"]) + assert result.exit_code == 0, result.output + assert "LinuxBuild2404.vmx" in result.output + assert "192.168.1.200" in result.output + + def test_prepare_linux_toolchain_not_found( prepare_env: dict, monkeypatch: pytest.MonkeyPatch ) -> None: @@ -2251,3 +2290,771 @@ def test_check_guest_idle_unparseable() -> None: transport.run.return_value = WinRmResult(stdout="not json", stderr="", returncode=0) with pytest.raises(click.ClickException, match="could not parse"): tmpl_mod._sample_guest_idle(transport) + + +# ═══════════════════════════════════════════════════════════════════════════ +# template deploy-rust / prepare-rust — CLI tests +# ═══════════════════════════════════════════════════════════════════════════ + + +def test_deploy_rust_help() -> None: + result = CliRunner().invoke(cli, ["template", "deploy-rust", "--help"]) + assert result.exit_code == 0 + assert "LinuxRust2404" in result.output + + +def test_deploy_rust_forwards_to_deploy_linux( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """deploy-rust calls the deploy-linux callback with the Rust defaults.""" + captured: dict = {} + + def fake_cb(**kwargs: object) -> None: + captured.update(kwargs) + + monkeypatch.setattr(tmpl_mod.template_deploy_linux, "callback", fake_cb) + + result = CliRunner().invoke(cli, ["template", "deploy-rust", "--force"]) + assert result.exit_code == 0, result.output + assert captured["vm_name"] == "LinuxRust2404" + assert captured["guest_hostname"] == "ci-rust-template" + assert captured["disk_size_gb"] == 60 + assert captured["force"] is True + # Default VMX points at the LinuxRust2404 template dir. + vmx = str(captured["vmx_path_str"]) + assert vmx.endswith("LinuxRust2404/LinuxRust2404.vmx") + + +def test_deploy_rust_vmx_override( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """An explicit --vmx is forwarded untouched.""" + captured: dict = {} + monkeypatch.setattr( + tmpl_mod.template_deploy_linux, "callback", + lambda **kw: captured.update(kw), + ) + result = CliRunner().invoke( + cli, ["template", "deploy-rust", "--vmx", "/tmp/custom/Custom.vmx"] + ) + assert result.exit_code == 0, result.output + assert captured["vmx_path_str"] == "/tmp/custom/Custom.vmx" + + +@pytest.fixture() +def prepare_rust_env( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> dict: + """Environment with SSH keys, VMX, and the Rust toolchain script.""" + ssh_key = _make_keys(tmp_path / "keys") + vm_dir = tmp_path / "templates" / "LinuxRust2404" + vm_dir.mkdir(parents=True) + vmx = vm_dir / "LinuxRust2404.vmx" + vmx.write_text('guestOS = "ubuntu-64"\n', encoding="ascii") + + tmpl_dir = tmp_path / "template" + tmpl_dir.mkdir() + toolchain = tmpl_dir / "Install-CIToolchain-LinuxRust2404.sh" + toolchain.write_text("#!/bin/bash\necho done\n") + + config = _deploy_config(tmp_path) + monkeypatch.setattr(tmpl_mod, "load_config", lambda: config) + monkeypatch.setattr(tmpl_mod.time, "sleep", lambda _: None) + monkeypatch.chdir(tmp_path) + + return { + "tmp_path": tmp_path, + "ssh_key": ssh_key, + "vmx": vmx, + "toolchain": toolchain, + } + + +def _stub_prepare_rust( + monkeypatch: pytest.MonkeyPatch, + env: dict, + *, + streamed: list[str] | None = None, +) -> MagicMock: + """Stub SSH/vmrun for prepare-rust; records the streamed toolchain command.""" + client_m = MagicMock() + raw = env["toolchain"].read_bytes() + if raw[:3] == b"\xef\xbb\xbf": + raw = raw[3:] + _toolchain_size = str(len(raw.replace(b"\r\n", b"\n").replace(b"\r", b"\n"))) + + def fake_pl_run( + client: object, cmd: str, timeout: float = 60.0 + ) -> tuple[int, str, str]: + if "stat -c %s /tmp/" in cmd: + return 0, _toolchain_size, "" + if "stat -c %s /etc/machine-id" in cmd: + return 0, "0", "" + if "truncate" in cmd: + return 0, "", "" + if "shutdown" in cmd: + return 0, "", "" + if "echo connected" in cmd: + return 0, "connected", "" + if "cloud-init.disabled" in cmd and "echo DISABLED" in cmd: + return 0, "DISABLED\nNO50\n", "" + return 0, "UserSudo:OK\nSudoNoPass:OK\n", "" + + def fake_pl_stream(client: object, cmd: str) -> int: + if streamed is not None: + streamed.append(cmd) + return 0 + + monkeypatch.setattr(tmpl_mod, "_pl_connect", lambda *a, **kw: client_m) + monkeypatch.setattr(tmpl_mod, "_pl_run", fake_pl_run) + monkeypatch.setattr(tmpl_mod, "_pl_stream", fake_pl_stream) + monkeypatch.setattr(tmpl_mod, "_pl_upload", lambda *a, **kw: None) + monkeypatch.setattr(tmpl_mod, "_wait_ssh_port_deploy", lambda *a, **kw: True) + return client_m + + +def test_prepare_rust_help() -> None: + result = CliRunner().invoke(cli, ["template", "prepare-rust", "--help"]) + assert result.exit_code == 0 + assert "--rust-channel" in result.output + assert "--no-cargo-audit" in result.output + + +def test_prepare_rust_missing_vmx_and_ip(prepare_rust_env: dict) -> None: + """Without --vmx/--ip AND no default template VMX on disk → error.""" + prepare_rust_env["vmx"].unlink() # default LinuxRust2404.vmx absent + result = CliRunner().invoke(cli, ["template", "prepare-rust"]) + assert result.exit_code != 0 + assert "--vmx" in result.output or "--ip" in result.output + + +def test_prepare_rust_default_vmx_used( + prepare_rust_env: dict, monkeypatch: pytest.MonkeyPatch +) -> None: + """Without --vmx/--ip, the default LinuxRust2404 VMX is picked up.""" + _stub_prepare_rust(monkeypatch, prepare_rust_env) + monkeypatch.setattr(tmpl_mod, "_poll_guest_ip_deploy", lambda *a, **kw: "192.168.1.200") + + backend_m = MagicMock() + backend_m.is_running.side_effect = [False, False] + backend_m.list_snapshots.return_value = [] + monkeypatch.setattr(tmpl_mod, "WorkstationVmrunBackend", lambda **kw: backend_m) + + import subprocess as _sub + monkeypatch.setattr( + tmpl_mod.subprocess, "run", + lambda cmd, **kw: _sub.CompletedProcess(list(cmd), 0, "", ""), + ) + + result = CliRunner().invoke(cli, ["template", "prepare-rust"]) + assert result.exit_code == 0, result.output + assert "LinuxRust2404.vmx" in result.output + assert "BaseClean-LinuxRust" in result.output + + +def test_prepare_rust_toolchain_not_found(prepare_rust_env: dict) -> None: + prepare_rust_env["toolchain"].unlink() + result = CliRunner().invoke( + cli, ["template", "prepare-rust", "--ip", "192.168.1.100"] + ) + assert result.exit_code != 0 + assert "not found" in result.output.lower() + assert "LinuxRust2404" in result.output + + +def test_prepare_rust_happy_path_defaults( + prepare_rust_env: dict, monkeypatch: pytest.MonkeyPatch +) -> None: + """Default invocation: Rust toolchain script, BaseClean-LinuxRust, stable channel.""" + streamed: list[str] = [] + _stub_prepare_rust(monkeypatch, prepare_rust_env, streamed=streamed) + + result = CliRunner().invoke( + cli, ["template", "prepare-rust", "--ip", "192.168.1.100"] + ) + assert result.exit_code == 0, result.output + assert "[prepare-rust]" in result.output + assert "BaseClean-LinuxRust" in result.output + assert streamed + assert "Install-CIToolchain-LinuxRust2404.sh" in streamed[0] + assert "--rust-channel stable" in streamed[0] + assert "--no-cargo-audit" not in streamed[0] + + +def test_prepare_rust_forwards_rust_flags( + prepare_rust_env: dict, monkeypatch: pytest.MonkeyPatch +) -> None: + """--rust-channel and --no-cargo-audit reach the toolchain command.""" + streamed: list[str] = [] + _stub_prepare_rust(monkeypatch, prepare_rust_env, streamed=streamed) + + result = CliRunner().invoke( + cli, + [ + "template", "prepare-rust", + "--ip", "192.168.1.100", + "--rust-channel", "1.97.1", + "--no-cargo-audit", + ], + ) + assert result.exit_code == 0, result.output + assert streamed + assert "--rust-channel 1.97.1" in streamed[0] + assert "--no-cargo-audit" in streamed[0] + + +def test_prepare_rust_snapshot_override( + prepare_rust_env: dict, monkeypatch: pytest.MonkeyPatch +) -> None: + """--snapshot overrides the BaseClean-LinuxRust default.""" + _stub_prepare_rust(monkeypatch, prepare_rust_env) + result = CliRunner().invoke( + cli, + [ + "template", "prepare-rust", + "--ip", "192.168.1.100", + "--snapshot", "CustomSnap", + ], + ) + assert result.exit_code == 0, result.output + assert "CustomSnap" in result.output + + +def test_prepare_linux_unchanged_defaults( + prepare_env: dict, monkeypatch: pytest.MonkeyPatch +) -> None: + """Regression: prepare-linux still uses the Linux toolchain + BaseClean-Linux + and never forwards rust flags.""" + streamed: list[str] = [] + client_m = MagicMock() + raw = prepare_env["toolchain"].read_bytes() + _tsize = str(len(raw.replace(b"\r\n", b"\n").replace(b"\r", b"\n"))) + + def fake_pl_run(client: object, cmd: str, timeout: float = 60.0) -> tuple[int, str, str]: + if "stat -c %s /tmp/" in cmd: + return 0, _tsize, "" + if "stat -c %s /etc/machine-id" in cmd: + return 0, "0", "" + if "truncate" in cmd or "shutdown" in cmd: + return 0, "", "" + if "echo connected" in cmd: + return 0, "connected", "" + if "cloud-init.disabled" in cmd and "echo DISABLED" in cmd: + return 0, "DISABLED\nNO50\n", "" + return 0, "UserSudo:OK\nSudoNoPass:OK\n", "" + + monkeypatch.setattr(tmpl_mod, "_pl_connect", lambda *a, **kw: client_m) + monkeypatch.setattr(tmpl_mod, "_pl_run", fake_pl_run) + monkeypatch.setattr( + tmpl_mod, "_pl_stream", + lambda c, cmd: streamed.append(cmd) or 0, + ) + monkeypatch.setattr(tmpl_mod, "_pl_upload", lambda *a, **kw: None) + monkeypatch.setattr(tmpl_mod, "_wait_ssh_port_deploy", lambda *a, **kw: True) + + result = CliRunner().invoke( + cli, ["template", "prepare-linux", "--ip", "192.168.1.100"] + ) + assert result.exit_code == 0, result.output + assert "[prepare-linux]" in result.output + assert "BaseClean-Linux" in result.output + assert streamed + assert "Install-CIToolchain-Linux2404.sh" in streamed[0] + assert "--rust-channel" not in streamed[0] + assert "--no-cargo-audit" not in streamed[0] + + +# ═══════════════════════════════════════════════════════════════════════════ +# Final messages + deploy -> prepare hand-off +# ═══════════════════════════════════════════════════════════════════════════ + + +def _deploy_linux_stub_env(tmp_path: Path, name: str) -> dict: + """Minimal on-disk state for a deploy-* run resumed at the last step.""" + vm_dir = tmp_path / "templates" / name + vm_dir.mkdir(parents=True) + vmx = vm_dir / f"{name}.vmx" + vmx.write_text('ide1:0.present = "TRUE"\n', encoding="ascii") + key = tmp_path / "ci_linux" + key.write_text("PRIVATE KEY", encoding="ascii") + return {"vmx": vmx, "key": key} + + +def test_deploy_rust_summary_points_at_prepare_rust( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """Regression: deploy-rust used to tell the user to run prepare-linux.""" + env = _deploy_linux_stub_env(tmp_path, "LinuxRust2404") + result = CliRunner().invoke( + cli, + [ + "template", "deploy-rust", + "--vmx", str(env["vmx"]), + "--ssh-key", str(env["key"]), + "--start-from-step", "10", + ], + ) + assert result.exit_code == 0, result.output + assert "deploy-rust complete" in result.output + assert "prepare-rust" in result.output + assert "prepare-linux" not in result.output + # The snapshot named in the summary is the one prepare-rust creates. + assert "BaseClean-LinuxRust" in result.output + assert "PostInstall-Linux" not in result.output + + +def test_deploy_linux_summary_points_at_prepare_linux( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + env = _deploy_linux_stub_env(tmp_path, "LinuxBuild2404") + result = CliRunner().invoke( + cli, + [ + "template", "deploy-linux", + "--vmx", str(env["vmx"]), + "--ssh-key", str(env["key"]), + "--start-from-step", "10", + ], + ) + assert result.exit_code == 0, result.output + assert "deploy-linux complete" in result.output + assert "template prepare-linux --vmx" in result.output + assert "BaseClean-Linux" in result.output + + +def test_deploy_rust_then_prepare_hands_off( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """--then-prepare chains into prepare-rust with the Rust defaults.""" + env = _deploy_linux_stub_env(tmp_path, "LinuxRust2404") + captured: dict = {} + monkeypatch.setattr( + tmpl_mod, "_prepare_linux_impl", lambda **kw: captured.update(kw) + ) + result = CliRunner().invoke( + cli, + [ + "template", "deploy-rust", + "--vmx", str(env["vmx"]), + "--ssh-key", str(env["key"]), + "--start-from-step", "10", + "--then-prepare", + "--rust-channel", "1.97.1", + ], + ) + assert result.exit_code == 0, result.output + assert captured["prog_name"] == "prepare-rust" + assert captured["default_vmx_name"] == "LinuxRust2404" + assert captured["default_toolchain_name"] == "Install-CIToolchain-LinuxRust2404.sh" + assert captured["snapshot"] == "BaseClean-LinuxRust" + assert captured["rust_channel"] == "1.97.1" + assert captured["take_snapshot"] is True + assert captured["vmx_path_str"] == str(env["vmx"]) + + +def test_deploy_linux_then_prepare_hands_off( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + env = _deploy_linux_stub_env(tmp_path, "LinuxBuild2404") + captured: dict = {} + monkeypatch.setattr( + tmpl_mod, "_prepare_linux_impl", lambda **kw: captured.update(kw) + ) + result = CliRunner().invoke( + cli, + [ + "template", "deploy-linux", + "--vmx", str(env["vmx"]), + "--ssh-key", str(env["key"]), + "--start-from-step", "10", + "--then-prepare", + ], + ) + assert result.exit_code == 0, result.output + assert captured["prog_name"] == "prepare-linux" + assert captured["default_toolchain_name"] == "Install-CIToolchain-Linux2404.sh" + assert captured["snapshot"] == "BaseClean-Linux" + assert captured["rust_channel"] is None + + +def test_deploy_without_then_prepare_does_not_chain( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """The two commands stay separate: no hand-off unless asked for.""" + env = _deploy_linux_stub_env(tmp_path, "LinuxBuild2404") + calls: list[dict] = [] + monkeypatch.setattr( + tmpl_mod, "_prepare_linux_impl", lambda **kw: calls.append(kw) + ) + result = CliRunner().invoke( + cli, + [ + "template", "deploy-linux", + "--vmx", str(env["vmx"]), + "--ssh-key", str(env["key"]), + "--start-from-step", "10", + ], + ) + assert result.exit_code == 0, result.output + assert calls == [] + + +def test_prepare_rust_next_hint_is_copy_pasteable( + prepare_rust_env: dict, monkeypatch: pytest.MonkeyPatch +) -> None: + """The `vm new` hint needs the full VMX path and an explicit --snapshot. + + `vm new` defaults --snapshot to "BaseClean" (the Windows snapshot), so a + hint that omits it clones the wrong snapshot. + """ + _stub_prepare_rust(monkeypatch, prepare_rust_env) + monkeypatch.setattr( + tmpl_mod, "_poll_guest_ip_deploy", lambda *a, **kw: "192.168.1.200" + ) + backend_m = MagicMock() + backend_m.is_running.side_effect = [False, False] + backend_m.list_snapshots.return_value = [] + monkeypatch.setattr(tmpl_mod, "WorkstationVmrunBackend", lambda **kw: backend_m) + import subprocess as _sub + monkeypatch.setattr( + tmpl_mod.subprocess, "run", + lambda cmd, **kw: _sub.CompletedProcess(list(cmd), 0, "", ""), + ) + + result = CliRunner().invoke(cli, ["template", "prepare-rust"]) + assert result.exit_code == 0, result.output + vmx = str(prepare_rust_env["vmx"]) + assert f"--template {vmx}" in result.output + assert "--snapshot BaseClean-LinuxRust" in result.output + assert "--guest-os linux" in result.output + + +# ── --exclude ───────────────────────────────────────────────────────────────── + + +def _tmpl_root_with(tmp_path: Path, names: tuple[str, ...]) -> Path: + tmpl_root = tmp_path / "templates" + for name in names: + d = tmpl_root / name + d.mkdir(parents=True) + (d / f"{name}.vmx").write_text("vmx") + return tmpl_root + + +def test_exclude_skips_named_template( + tmp_path: Path, backup_dir: Path, monkeypatch: pytest.MonkeyPatch, _no_systemctl: None +) -> None: + """A retired template must not keep costing a backup generation.""" + tmpl_root = _tmpl_root_with( + tmp_path, ("WinBuild2025", "WinBuildTest", "LinuxBuild2404") + ) + monkeypatch.setenv("CI_TEMPLATES", str(tmpl_root)) + + result = CliRunner().invoke( + cli, + [ + "template", "backup", "--all-templates", + "--exclude", "WinBuildTest", + "--backup-base-dir", str(backup_dir), + "--skip-runner-stop", + ], + ) + assert result.exit_code == 0, result.output + names = {f.name for f in backup_dir.iterdir() if f.suffix == ".7z"} + assert any("WinBuild2025" in n for n in names) + assert any("LinuxBuild2404" in n for n in names) + assert not any("WinBuildTest" in n for n in names) + assert "Skipping 'WinBuildTest'" in result.output + + +def test_exclude_is_repeatable( + tmp_path: Path, backup_dir: Path, monkeypatch: pytest.MonkeyPatch, _no_systemctl: None +) -> None: + tmpl_root = _tmpl_root_with( + tmp_path, ("WinBuild2025", "WinBuildTest", "LinuxBuildTest") + ) + monkeypatch.setenv("CI_TEMPLATES", str(tmpl_root)) + + result = CliRunner().invoke( + cli, + [ + "template", "backup", "--all-templates", + "--exclude", "WinBuildTest", + "--exclude", "LinuxBuildTest", + "--backup-base-dir", str(backup_dir), + "--skip-runner-stop", + ], + ) + assert result.exit_code == 0, result.output + names = {f.name for f in backup_dir.iterdir() if f.suffix == ".7z"} + assert names and all("Test" not in n for n in names) + + +def test_exclude_unknown_name_raises( + tmp_path: Path, backup_dir: Path, monkeypatch: pytest.MonkeyPatch, _no_systemctl: None +) -> None: + """A typo must fail loudly, not silently back up everything.""" + tmpl_root = _tmpl_root_with(tmp_path, ("WinBuild2025",)) + monkeypatch.setenv("CI_TEMPLATES", str(tmpl_root)) + + result = CliRunner().invoke( + cli, + [ + "template", "backup", "--all-templates", + "--exclude", "WinBuildTst", + "--backup-base-dir", str(backup_dir), + "--skip-runner-stop", + ], + ) + assert result.exit_code != 0 + assert "no such template" in result.output + + +def test_exclude_everything_raises( + tmp_path: Path, backup_dir: Path, monkeypatch: pytest.MonkeyPatch, _no_systemctl: None +) -> None: + tmpl_root = _tmpl_root_with(tmp_path, ("WinBuild2025",)) + monkeypatch.setenv("CI_TEMPLATES", str(tmpl_root)) + + result = CliRunner().invoke( + cli, + [ + "template", "backup", "--all-templates", + "--exclude", "WinBuild2025", + "--backup-base-dir", str(backup_dir), + "--skip-runner-stop", + ], + ) + assert result.exit_code != 0 + assert "nothing to back up" in result.output + + +def test_exclude_without_all_templates_raises( + tmp_path: Path, backup_dir: Path, monkeypatch: pytest.MonkeyPatch, _no_systemctl: None +) -> None: + """--exclude alone would be a silent no-op; it must error instead.""" + tmpl_root = _tmpl_root_with(tmp_path, ("WinBuild2025",)) + monkeypatch.setenv("CI_TEMPLATES", str(tmpl_root)) + + result = CliRunner().invoke( + cli, + [ + "template", "backup", + "--template-path", str(tmpl_root / "WinBuild2025"), + "--exclude", "WinBuildTest", + "--backup-base-dir", str(backup_dir), + "--skip-runner-stop", + ], + ) + assert result.exit_code != 0 + assert "only applies to --all-templates" in result.output + + +def test_then_prepare_fails_before_deploy_when_toolchain_missing( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """Regression: the toolchain was resolved only after the deploy finished. + + Running deploy-rust --then-prepare from outside the repo root completed a + ~45-minute deploy and only then died on the missing script. The check must + happen before any VM work starts. + """ + env = _deploy_linux_stub_env(tmp_path, "LinuxRust2404") + monkeypatch.chdir(tmp_path) # no template/ dir here + + steps: list[int] = [] + monkeypatch.setattr( + tmpl_mod, "_deploy_step", lambda *a, **kw: steps.append(1) + ) + + result = CliRunner().invoke( + cli, + [ + "template", "deploy-rust", + "--vmx", str(env["vmx"]), + "--ssh-key", str(env["key"]), + "--then-prepare", + ], + ) + assert result.exit_code != 0 + assert "Install-CIToolchain-LinuxRust2404.sh" in result.output + assert "--toolchain-script" in result.output + # Nothing was deployed: the failure landed before the first step ran. + assert steps == [] + + +def test_then_prepare_accepts_explicit_toolchain_script( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """--toolchain-script makes the chain independent of the current directory.""" + env = _deploy_linux_stub_env(tmp_path, "LinuxRust2404") + script = tmp_path / "custom-toolchain.sh" + script.write_text("#!/bin/sh\n", encoding="ascii") + monkeypatch.chdir(tmp_path) + + captured: dict = {} + monkeypatch.setattr( + tmpl_mod, "_prepare_linux_impl", lambda **kw: captured.update(kw) + ) + + result = CliRunner().invoke( + cli, + [ + "template", "deploy-rust", + "--vmx", str(env["vmx"]), + "--ssh-key", str(env["key"]), + "--start-from-step", "10", + "--then-prepare", + "--toolchain-script", str(script), + ], + ) + assert result.exit_code == 0, result.output + assert captured["toolchain_script_str"] == str(script) + + +def test_resolve_toolchain_script_prefers_cwd( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + tdir = tmp_path / "template" + tdir.mkdir() + script = tdir / "Install-CIToolchain-Linux2404.sh" + script.write_text("#!/bin/sh\n", encoding="ascii") + monkeypatch.chdir(tmp_path) + + got = tmpl_mod._resolve_toolchain_script("Install-CIToolchain-Linux2404.sh", None) + assert got == script + + +def test_resolve_toolchain_script_explicit_missing_raises(tmp_path: Path) -> None: + import click as _click + + with pytest.raises(_click.ClickException, match="not found"): + tmpl_mod._resolve_toolchain_script("x.sh", str(tmp_path / "nope.sh")) + + +def test_poll_guest_ip_passes_wait_flag(monkeypatch: pytest.MonkeyPatch) -> None: + """Regression: without -wait vmrun reports "Tools are not running". + + A guest that was booted, on the network and answering SSH still burned the + full 10-minute Step 7 budget because the poll omitted -wait, so vmrun only + answered when the Tools happened to have registered the IP at that instant. + """ + import subprocess as _sub + + seen: list[list[str]] = [] + + def fake_run(cmd, **kw): # type: ignore[no-untyped-def] + seen.append(list(cmd)) + return _sub.CompletedProcess(cmd, 0, "192.168.1.100", "") + + monkeypatch.setattr(tmpl_mod, "_read_guestinfo_ip", lambda *a, **kw: None) + monkeypatch.setattr(tmpl_mod.subprocess, "run", fake_run) + monkeypatch.setattr(tmpl_mod.time, "sleep", lambda _: None) + + assert tmpl_mod._poll_guest_ip_deploy("/vmrun", "/vm.vmx", 60.0) == "192.168.1.100" + assert seen, "vmrun was never invoked" + assert "-wait" in seen[0], f"-wait missing from {seen[0]}" + + +def test_poll_guest_ip_ignores_error_line_on_stdout( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """vmrun can exit 0 with "Error: ..." on stdout (AGENTS.md #10).""" + import subprocess as _sub + + outputs = [ + "Error: The VMware Tools are not running in the virtual machine\n", + "192.168.1.101\n", + ] + + def fake_run(cmd, **kw): # type: ignore[no-untyped-def] + return _sub.CompletedProcess(cmd, 0, outputs.pop(0), "") + + monkeypatch.setattr(tmpl_mod, "_read_guestinfo_ip", lambda *a, **kw: None) + monkeypatch.setattr(tmpl_mod.subprocess, "run", fake_run) + monkeypatch.setattr(tmpl_mod.time, "sleep", lambda _: None) + + # The Error line must not be mistaken for an address; the poll retries. + assert tmpl_mod._poll_guest_ip_deploy("/vmrun", "/vm.vmx", 60.0) == "192.168.1.101" + + +def test_deploy_force_refuses_when_vm_is_powered_on( + deploy_env: dict, monkeypatch: pytest.MonkeyPatch +) -> None: + """Regression: --force against a running VM destroyed the live guest. + + It deleted the live .vmdk.lck as "stale" and Step 4 rewrote the base VMDK + underneath it; the damage only surfaced ~10 minutes later as a Step 7 IP + timeout, because `vmrun list` still showed the OLD instance. + """ + import subprocess as _sub + + vmx = deploy_env["vmx"] + + def fake_run(cmd: list[str], **kw: object) -> _sub.CompletedProcess[str]: + if "list" in cmd: + return _sub.CompletedProcess(cmd, 0, f"Total running VMs: 1\n{vmx}\n", "") + return _sub.CompletedProcess(cmd, 0, "", "") + + monkeypatch.setattr(tmpl_mod.subprocess, "run", fake_run) + + result = CliRunner().invoke( + cli, + ["template", "deploy-linux", "--vmx", str(vmx), "--force"], + ) + assert result.exit_code != 0 + assert "powered on" in result.output + assert "vmrun -T ws stop" in result.output + + +def test_prepare_detaches_seed_iso_before_snapshot( + prepare_rust_env: dict, monkeypatch: pytest.MonkeyPatch +) -> None: + """Regression: the seed ISO ended up attached inside the snapshot. + + deploy Step 10 sets ide1:0.present = FALSE while the VM is still powered + on; VMware rewrites the VMX from its in-memory copy at power-off and + restores TRUE. Clones then boot with cloud-init's seed still attached + (AGENTS.md #6), so prepare must detach it after power-off. + """ + vmx: Path = prepare_rust_env["vmx"] + vmx.write_text( + 'displayName = "LinuxRust2404"\n' + 'ide1:0.present = "TRUE"\n' + 'ide1:0.startConnected = "TRUE"\n' + 'ide1:0.fileName = "cloud-init-seed.iso"\n', + encoding="ascii", + ) + + _stub_prepare_rust(monkeypatch, prepare_rust_env) + monkeypatch.setattr( + tmpl_mod, "_poll_guest_ip_deploy", lambda *a, **kw: "192.168.1.200" + ) + + backend_m = MagicMock() + backend_m.is_running.side_effect = [False, False] + backend_m.list_snapshots.return_value = [] + monkeypatch.setattr(tmpl_mod, "WorkstationVmrunBackend", lambda **kw: backend_m) + + snapshot_calls: list[list[str]] = [] + + import subprocess as _sub + + def fake_run(cmd, **kw): # type: ignore[no-untyped-def] + if "snapshot" in cmd: + # The VMX must already be patched by the time the snapshot is taken. + snapshot_calls.append(vmx.read_text(encoding="ascii").splitlines()) + return _sub.CompletedProcess(list(cmd), 0, "", "") + + monkeypatch.setattr(tmpl_mod.subprocess, "run", fake_run) + + result = CliRunner().invoke(cli, ["template", "prepare-rust"]) + assert result.exit_code == 0, result.output + + text = vmx.read_text(encoding="ascii") + assert 'ide1:0.present = "FALSE"' in text + assert 'ide1:0.startConnected = "FALSE"' in text + assert snapshot_calls, "snapshot was never taken" + assert any('ide1:0.present = "FALSE"' in ln for ln in snapshot_calls[0]), ( + "snapshot captured the VMX with the seed ISO still attached" + )