"""``template`` sub-commands — CI template VM management. Commands -------- template backup Ports ``scripts/Backup-CITemplate.ps1`` to Python. Creates timestamped 7z archives of VMware template directories and prunes old backups beyond ``--keep-count``. Stops ``act-runner.service`` before copying and restarts it afterwards; pass ``--skip-runner-stop`` to bypass this. template prepare-win Provisions a Windows template VM end-to-end via pypsrp (replaces the ``Prepare-WinBuild*.ps1`` PowerShell scripts that required PSWSMan / New-PSSession — unreliable on Linux hosts). Steps performed by ``template prepare-win`` ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 1. Resolves ``Install-CIToolchain-.ps1`` from ``/template/`` (or ``--toolchain-script``). 2. Optionally locates ``template/guest-setup/windows/ci-static-ip.ps1`` for the ip_pool feature. 3. Starts the VM via vmrun (skipped if already running). 4. Auto-detects guest IP via VMware Tools (skipped if ``--ip`` supplied). 5. Waits for TCP/5986 then probes WinRM session readiness. 6. Uploads and executes the toolchain script inside the guest. Handles the Windows Update reboot-required exit code (3010) in a loop (max 10 iterations). 7. Installs the ``CI-StaticIp`` scheduled task (``SYSTEM``, At Startup) so each linked clone applies a pre-assigned static IP from ``guestinfo.ip-assignment`` before WinRM starts. 8. Runs post-setup remote validation (11 guest-state checks via a single ``ConvertTo-Json`` call). 9. Final reboot → re-affirms autologin registry keys → graceful ``Stop-Computer``. 10. Waits for the VM to power off (120 s timeout). 11. Creates the ``--snapshot`` (default ``BaseClean``); optionally deletes and recreates if a snapshot with that name already exists. 12. If ``--store-credential`` is set, writes build-user credentials into the system keyring (Windows → Credential Manager; Linux → SecretService / file backend) using the two-entry scheme read by :class:`~ci_orchestrator.credentials.KeyringCredentialStore`. Run from the repository root so toolchain and guest-setup scripts are auto-discovered under ``template/``. Usage example:: python -m ci_orchestrator template prepare-win \\ --vmx /var/lib/ci/templates/WinBuild2025/WinBuild2025.vmx \\ --admin-password AdminPass1! \\ --build-password CIBuildPass1! \\ --store-credential """ from __future__ import annotations import fnmatch import json import shutil import socket import subprocess import sys import threading import time from datetime import UTC, datetime from pathlib import Path import click from ci_orchestrator.backends.workstation import WorkstationVmrunBackend from ci_orchestrator.backends.protocol import VmHandle from ci_orchestrator.config import load_config from ci_orchestrator.transport.errors import TransportError from ci_orchestrator.transport.winrm import WinRmResult, WinRmTransport @click.group() def template() -> None: """VMware template management.""" @template.command("backup") @click.option( "--template-path", default=None, help=( "Template directory to back up " "(default: /WinBuild2025)." ), ) @click.option( "--all-templates", is_flag=True, help="Back up every subdirectory under CI_TEMPLATES in a single pass.", ) @click.option( "--backup-base-dir", default="/var/lib/ci/backups", show_default=True, help="Parent directory for timestamped backup folders.", ) @click.option( "--keep-count", default=3, show_default=True, help="Number of most-recent backups to retain per template.", ) @click.option( "--skip-runner-stop", is_flag=True, help="Skip stopping/restarting act-runner.service.", ) @click.option( "--what-if", is_flag=True, help="Dry run — log what would happen without copying or deleting.", ) def template_backup( template_path: str | None, all_templates: bool, backup_base_dir: str, keep_count: int, skip_runner_stop: bool, what_if: bool, ) -> None: """Create timestamped backup of CI template VM directories.""" config = load_config() backup_base = Path(backup_base_dir) timestamp = datetime.now(tz=UTC).strftime("%Y%m%d_%H%M%S") if all_templates: tmpl_root = config.paths.templates if not tmpl_root.exists(): raise click.ClickException(f"Templates directory not found: {tmpl_root}") to_backup = sorted(d for d in tmpl_root.iterdir() if d.is_dir()) if not to_backup: raise click.ClickException( f"No template subdirectories found under {tmpl_root}" ) else: p = Path(template_path) if template_path else config.paths.templates / "WinBuild2025" if not p.is_dir(): raise click.ClickException(f"Template directory not found: {p}") to_backup = [p] runner_was_stopped = False try: if not skip_runner_stop: runner_was_stopped = _stop_runner(what_if=what_if) if not what_if: backup_base.mkdir(parents=True, exist_ok=True) for tmpl_dir in to_backup: name_tag = f"{tmpl_dir.name}_{timestamp}" if all_templates else timestamp archive = backup_base / f"Template_{name_tag}.7z" if what_if: click.echo( f"[Backup-CITemplate] WhatIf: would compress {tmpl_dir} -> {archive}" ) else: click.echo(f"[Backup-CITemplate] Compressing {tmpl_dir} -> {archive} ...") _compress_7z(tmpl_dir, archive) size_mb = archive.stat().st_size // (1024 * 1024) click.echo( f"[Backup-CITemplate] Backup complete: {archive} ({size_mb} MB)" ) template_label = tmpl_dir.name if all_templates else None _prune_backups(backup_base, template_label, keep_count, what_if=what_if) finally: if runner_was_stopped: _start_runner(what_if=what_if) # ──────────────────────────────────────────────────────────────── helpers def _stop_runner(*, what_if: bool) -> bool: """Stop act-runner.service. Returns True if it was running.""" result = subprocess.run( ["systemctl", "is-active", "act-runner.service"], capture_output=True, text=True, ) if result.stdout.strip() != "active": return False if what_if: click.echo("[Backup-CITemplate] WhatIf: would stop act-runner.service") return True click.echo("[Backup-CITemplate] Stopping act-runner.service ...") subprocess.run(["systemctl", "stop", "act-runner.service"], check=True) click.echo("[Backup-CITemplate] act-runner.service stopped.") return True def _start_runner(*, what_if: bool) -> None: if what_if: click.echo("[Backup-CITemplate] WhatIf: would start act-runner.service") return click.echo("[Backup-CITemplate] Restarting act-runner.service ...") subprocess.run(["systemctl", "start", "act-runner.service"], check=True) result = subprocess.run( ["systemctl", "is-active", "act-runner.service"], capture_output=True, text=True, ) click.echo( f"[Backup-CITemplate] act-runner.service status: {result.stdout.strip()}" ) def _compress_7z(source: Path, archive: Path) -> None: """Create a 7z archive of ``source`` at ``archive`` using compression level 1.""" subprocess.run( ["7z", "a", "-mx=1", "-mmt=on", str(archive), str(source)], check=True, stdout=subprocess.DEVNULL, ) def _prune_backups( backup_base: Path, template_name: str | None, keep_count: int, *, what_if: bool, ) -> None: if template_name: pattern = f"Template_{template_name}_????????_??????.7z" else: pattern = "Template_????????_??????.7z" existing = sorted( [ f for f in backup_base.iterdir() if f.is_file() and fnmatch.fnmatch(f.name, pattern) ], key=lambda f: f.stat().st_mtime, reverse=True, ) to_remove = existing[keep_count:] for old in to_remove: if what_if: click.echo( f"[Backup-CITemplate] WhatIf: would prune old backup: {old.name}" ) else: click.echo(f"[Backup-CITemplate] Pruning old backup: {old.name}") old.unlink() if to_remove: label = template_name or "template" click.echo(f"[Backup-CITemplate] Retained {keep_count} backup(s) for {label}.") # ═══════════════════════════════════════════════════════════════════════════ # template prepare-win helpers # ═══════════════════════════════════════════════════════════════════════════ def _ps_escape(s: str) -> str: """Escape *s* for use inside a PowerShell single-quoted string.""" return s.replace("'", "''") def _wait_tcp(host: str, port: int, timeout: float) -> bool: """Return True once TCP *port* on *host* accepts a connection within *timeout* s.""" deadline = time.monotonic() + timeout while time.monotonic() < deadline: try: with socket.create_connection((host, port), timeout=5.0): return True except OSError: time.sleep(5) return False def _parse_exit_marker(stdout: str) -> int: """Extract ``CI_EXITCODE:N`` written by the invoke wrapper; default 0.""" for line in reversed(stdout.splitlines()): stripped = line.strip() if stripped.startswith("CI_EXITCODE:"): try: return int(stripped.split(":", 1)[1]) except (IndexError, ValueError): pass return 0 def _wait_winrm(transport: WinRmTransport, timeout: int) -> None: """Poll *transport*.is_ready() until True or *timeout* seconds elapse.""" deadline = time.monotonic() + timeout while time.monotonic() < deadline: if transport.is_ready(): return time.sleep(10) click.echo(" ... waiting for WinRM ...") raise click.ClickException(f"WinRM not ready within {timeout}s.") def _run_with_heartbeat( transport: WinRmTransport, script: str, *, check: bool = True, heartbeat: int = 30, ) -> WinRmResult: """Run *script* via *transport* and print a heartbeat line every *heartbeat* s. pypsrp's execute_ps is fully blocking — no streaming. The heartbeat thread reassures the operator that the command is still running during long-running steps (e.g. Windows Update, which can take 20-40 min). """ result_holder: list[WinRmResult] = [] exc_holder: list[BaseException] = [] stop_event = threading.Event() def _worker() -> None: try: result_holder.append(transport.run(script, check=check)) except BaseException as exc: # noqa: BLE001 exc_holder.append(exc) finally: stop_event.set() worker = threading.Thread(target=_worker, daemon=True) worker.start() elapsed = 0 while not stop_event.wait(timeout=float(heartbeat)): elapsed += heartbeat click.echo(f" ... still running ({elapsed}s elapsed) ...") worker.join() if exc_holder: raise exc_holder[0] return result_holder[0] # ═══════════════════════════════════════════════════════════════════════════ # template prepare-win command # ═══════════════════════════════════════════════════════════════════════════ @template.command("prepare-win") @click.option("--vmx", required=True, metavar="PATH", help="Template VM .vmx file.") @click.option("--ip", "guest_ip", default="", help="Guest IP (auto-detect via VMware Tools if omitted).") @click.option("--admin-username", default="Administrator", show_default=True) @click.option("--admin-password", default="", help="Administrator password (prompted if omitted).") @click.option("--build-username", default="ci_build", show_default=True) @click.option("--build-password", default="", help="CI build user password (prompted if omitted).") @click.option("--snapshot", default="BaseClean", show_default=True, help="Snapshot name.") @click.option("--toolchain-script", "toolchain_script", default=None, metavar="PATH", help="Path to Install-CIToolchain-*.ps1 (auto-detected from VMX stem if omitted).") @click.option("--skip-windows-update", is_flag=True, help="Pass -SkipWindowsUpdate to toolchain script.") @click.option("--skip-tier2", is_flag=True, help="Pass -SkipTier2 to toolchain script.") @click.option("--skip-cleanup", is_flag=True, help="Pass -SkipCleanup to toolchain script.") @click.option("--winrm-timeout", default=300, show_default=True, help="Seconds to wait for WinRM.") @click.option("--auth", default="basic", show_default=True, type=click.Choice(["basic", "ntlm", "negotiate"]), help="WinRM authentication protocol.") @click.option("--store-credential", is_flag=True, help="Store build-user credentials in the system keyring after provisioning.") @click.option("--credential-target", default="BuildVMGuest", show_default=True, help="Keyring target name for the stored credential.") @click.option("--recreate-snapshot", is_flag=True, help="Delete and recreate the snapshot if it already exists (no prompt).") def template_prepare_win( vmx: str, guest_ip: str, admin_username: str, admin_password: str, build_username: str, build_password: str, snapshot: str, toolchain_script: str | None, skip_windows_update: bool, skip_tier2: bool, skip_cleanup: bool, winrm_timeout: int, auth: str, store_credential: bool, credential_target: str, recreate_snapshot: bool, ) -> None: """Provision a Windows template VM via pypsrp (Linux-host compatible). Replaces Prepare-WinBuild*.ps1 for hosts where PSWSMan/New-PSSession is unavailable or unreliable. Run from the repo root so toolchain and guest-setup scripts are auto-discovered under template/. """ vmx_path = Path(vmx).resolve() if not vmx_path.is_file(): raise click.ClickException(f"VMX not found: {vmx_path}") # --- Resolve toolchain script --- if toolchain_script is None: candidate = Path.cwd() / "template" / f"Install-CIToolchain-{vmx_path.stem}.ps1" if not candidate.is_file(): raise click.ClickException( f"Toolchain script not found at {candidate}. " "Run from the repo root or pass --toolchain-script." ) toolchain_path = candidate else: toolchain_path = Path(toolchain_script) if not toolchain_path.is_file(): raise click.ClickException(f"Toolchain script not found: {toolchain_path}") # --- Locate CI-StaticIp guest script --- _sip_candidate = Path.cwd() / "template" / "guest-setup" / "windows" / "ci-static-ip.ps1" static_ip_path: Path | None if _sip_candidate.is_file(): static_ip_path = _sip_candidate else: click.echo( f"[prepare-win] WARNING: {_sip_candidate} not found — " "ip_pool static-IP feature will not work on this template.", err=True, ) static_ip_path = None # --- Prompt for passwords --- if not admin_password: admin_password = click.prompt( f"Template VM {admin_username} password", hide_input=True ) if not build_password: build_password = click.prompt(f"New {build_username} password", hide_input=True) # --- Backend: start VM and detect IP --- config = load_config() backend = WorkstationVmrunBackend(vmrun_path=config.vmrun_path) handle = VmHandle(identifier=str(vmx_path), name=vmx_path.stem) if not backend.is_running(handle): click.echo(f"[prepare-win] Starting VM: {vmx_path.name} ...") backend.start(handle, headless=True) else: click.echo(f"[prepare-win] VM already running: {vmx_path.name}") if not guest_ip: click.echo("[prepare-win] Waiting for guest IP (vmrun getGuestIPAddress -wait) ...") detected = backend.get_ip(handle, timeout=300.0) if not detected: raise click.ClickException( "Could not get guest IP. Ensure VMware Tools is installed." ) guest_ip = detected click.echo(f"[prepare-win] Guest IP: {guest_ip}") # --- Wait for WinRM TCP then probe session --- click.echo(f"[prepare-win] Waiting for TCP/5986 on {guest_ip} ({winrm_timeout}s) ...") if not _wait_tcp(guest_ip, 5986, float(winrm_timeout)): raise click.ClickException(f"TCP/5986 not reachable on {guest_ip}.") transport = WinRmTransport( host=guest_ip, username=admin_username, password=admin_password, port=5986, ssl=True, cert_validation=False, auth=auth, connection_timeout=30, ) click.echo("[prepare-win] Probing WinRM session ...") _wait_winrm(transport, winrm_timeout) click.echo("[prepare-win] WinRM ready.") try: # Ensure C:\CI exists transport.run( "if (-not (Test-Path 'C:\\CI')) " "{ New-Item -ItemType Directory 'C:\\CI' -Force | Out-Null }" ) # Upload toolchain script guest_script = f"C:\\CI\\{toolchain_path.name}" click.echo(f"[prepare-win] Uploading {toolchain_path.name} -> {guest_script} ...") transport.copy(toolchain_path, guest_script) click.echo("[prepare-win] Upload complete.") # Build args string e = _ps_escape args_parts = [ f"-BuildPassword '{e(build_password)}'", f"-BuildUsername '{e(build_username)}'", f"-AdminPassword '{e(admin_password)}'", ] if skip_windows_update: args_parts.append("-SkipWindowsUpdate:$true") if skip_tier2: args_parts.append("-SkipTier2:$true") if skip_cleanup: args_parts.append("-SkipCleanup:$true") args_str = " ".join(args_parts) # Run toolchain script — loop handles Windows Update reboot (exit 3010) _MAX_ITER = 10 exit_code = 0 for iteration in range(1, _MAX_ITER + 1): click.echo( f"[prepare-win] Running {toolchain_path.name} " f"(iteration {iteration}/{_MAX_ITER}) ..." ) invoke = ( "Set-ExecutionPolicy Bypass -Scope Process -Force\n" "$ec = 0\n" "try {\n" f" & '{e(guest_script)}' {args_str}\n" " $ec = if ($null -eq $LASTEXITCODE) { 0 } else { $LASTEXITCODE }\n" "} catch {\n" " Write-Error $_\n" " $ec = 1\n" "}\n" '"CI_EXITCODE:$ec"\n' ) try: result = _run_with_heartbeat(transport, invoke, check=False) exit_code = _parse_exit_marker(result.stdout) if result.stdout.strip(): click.echo(result.stdout.strip()) if result.stderr.strip(): click.echo(result.stderr.strip(), err=True) except TransportError: click.echo( "[prepare-win] Transport error — VM likely rebooted mid-update " "(treating as 3010)." ) exit_code = 3010 click.echo(f"[prepare-win] Toolchain exit code: {exit_code}") if exit_code == 0: break if exit_code != 3010: raise click.ClickException( f"Toolchain script failed (exit {exit_code})." ) if iteration == _MAX_ITER: raise click.ClickException( f"Windows Update did not converge after {_MAX_ITER} iterations." ) click.echo("[prepare-win] Windows Update requires reboot — rebooting ...") try: transport.run("Restart-Computer -Force", check=False) except TransportError: pass transport.close() time.sleep(30) click.echo(f"[prepare-win] Waiting for WinRM after reboot ({winrm_timeout}s) ...") _wait_winrm(transport, winrm_timeout) click.echo("[prepare-win] WinRM ready after reboot.") # Install CI-StaticIp scheduled task if static_ip_path is not None: click.echo("[prepare-win] Installing CI-StaticIp scheduled task ...") transport.copy(static_ip_path, "C:\\CI\\ci-static-ip.ps1") transport.run(r""" $action = New-ScheduledTaskAction -Execute 'powershell.exe' ` -Argument '-NoProfile -NonInteractive -ExecutionPolicy Bypass -File C:\CI\ci-static-ip.ps1' $trigger = New-ScheduledTaskTrigger -AtStartup $principal = New-ScheduledTaskPrincipal -UserId 'SYSTEM' -LogonType ServiceAccount -RunLevel Highest Register-ScheduledTask -TaskName 'CI-StaticIp' ` -Action $action -Trigger $trigger -Principal $principal -Force | Out-Null """) click.echo("[prepare-win] CI-StaticIp task registered.") # Post-setup validation click.echo("[prepare-win] Running post-setup validation ...") esc_u = e(build_username) val_result = transport.run( "$env:PATH = [System.Environment]::GetEnvironmentVariable('PATH','Machine')" " + ';' + [System.Environment]::GetEnvironmentVariable('PATH','User')\n" "$msbuild = 'C:\\Program Files (x86)\\Microsoft Visual Studio\\2026" "\\BuildTools\\MSBuild\\Current\\Bin\\MSBuild.exe'\n" "@{\n" " WinRMRunning = (Get-Service WinRM -EA SilentlyContinue).Status -eq 'Running'\n" f" UserExists = [bool](Get-LocalUser -Name '{esc_u}' -EA SilentlyContinue)\n" f" UserIsAdmin = [bool](net localgroup Administrators 2>&1 | Select-String -SimpleMatch '{esc_u}')\n" " UACDisabled = (Get-ItemProperty 'HKLM:\\SOFTWARE\\Microsoft\\Windows" "\\CurrentVersion\\Policies\\System' -Name EnableLUA -EA SilentlyContinue).EnableLUA -eq 0\n" " FirewallOff = -not (Get-NetFirewallProfile | Where-Object Enabled -eq $true)\n" " DotNetExists = [bool](Get-Command dotnet -EA SilentlyContinue)\n" " PythonExists = Test-Path 'C:\\Python\\python.exe' -PathType Leaf\n" " MSBuildExists = Test-Path $msbuild -PathType Leaf\n" " CIDirsPresent = (Test-Path 'C:\\CI\\build') -and (Test-Path 'C:\\CI\\output')" " -and (Test-Path 'C:\\CI\\scripts')\n" " StaticIpTask = [bool](Get-ScheduledTask -TaskName 'CI-StaticIp' -EA SilentlyContinue)\n" " StaticIpScript = Test-Path 'C:\\CI\\ci-static-ip.ps1' -PathType Leaf\n" "} | ConvertTo-Json -Compress\n", check=False, ) if val_result.ok and val_result.stdout.strip(): try: state: dict[str, object] = json.loads(val_result.stdout.strip()) failed = [k for k, v in state.items() if not v] for k, v in state.items(): mark = "OK " if v else "FAIL" click.echo(f" [{mark}] {k}") if failed: click.echo( f"[prepare-win] WARNING: {len(failed)} check(s) failed: " f"{', '.join(failed)}", err=True, ) else: click.echo("[prepare-win] All post-setup checks passed.") except (json.JSONDecodeError, AttributeError): click.echo(val_result.stdout) else: click.echo( f"[prepare-win] Validation script error:\n{val_result.stderr}", err=True ) # Final reboot to flush installation state click.echo("[prepare-win] Final reboot to flush installation state ...") try: transport.run("Restart-Computer -Force", check=False) except TransportError: pass transport.close() time.sleep(30) click.echo("[prepare-win] Waiting for WinRM after final reboot ...") _wait_winrm(transport, winrm_timeout) # Re-affirm autologin click.echo("[prepare-win] Re-affirming autologin registry settings ...") transport.run( "$wl = 'HKLM:\\SOFTWARE\\Microsoft\\Windows NT\\CurrentVersion\\Winlogon'\n" "Set-ItemProperty $wl AutoAdminLogon '1' -Type String -Force\n" f"Set-ItemProperty $wl DefaultUserName '{e(admin_username)}' -Type String -Force\n" "Set-ItemProperty $wl DefaultDomainName '.' -Type String -Force\n" f"if ('{e(admin_password)}' -ne '') {{\n" f" Set-ItemProperty $wl DefaultPassword '{e(admin_password)}' -Type String -Force\n" "}\n" ) # Graceful shutdown click.echo("[prepare-win] Sending graceful shutdown ...") try: transport.run("Stop-Computer -Force", check=False) except TransportError: pass finally: transport.close() # Wait for VM to power off click.echo("[prepare-win] Waiting for VM to power off (120s) ...") off_deadline = time.monotonic() + 120 while time.monotonic() < off_deadline: if not backend.is_running(handle): break time.sleep(5) click.echo(" ... waiting for shutdown ...") else: raise click.ClickException("VM did not power off within 120s.") click.echo("[prepare-win] VM powered off.") # Create snapshot vmrun_exe = backend.vmrun_path existing = backend.list_snapshots(handle) if snapshot in existing: click.echo(f"[prepare-win] Snapshot '{snapshot}' already exists.") if recreate_snapshot: do_recreate = True else: sys.stdout.write(f"[prepare-win] Delete and recreate '{snapshot}'? [y/N] ") sys.stdout.flush() raw = sys.stdin.readline() do_recreate = raw.strip().lower() in ("y", "yes") if not do_recreate: click.echo("[prepare-win] Keeping existing snapshot. Provisioning complete.") return click.echo(f"[prepare-win] Deleting snapshot '{snapshot}' ...") subprocess.run( [vmrun_exe, "-T", "ws", "deleteSnapshot", str(vmx_path), snapshot], check=True, ) click.echo(f"[prepare-win] Creating snapshot '{snapshot}' ...") subprocess.run( [vmrun_exe, "-T", "ws", "snapshot", str(vmx_path), snapshot], check=True, ) click.echo(f"[prepare-win] Snapshot '{snapshot}' created.") # Store credentials in system keyring (Windows Credential Manager on Windows, # SecretService / file backend on Linux) using the two-entry scheme so # KeyringCredentialStore.get() can retrieve them without knowing the username. if store_credential: try: import keyring except ImportError as exc: raise click.ClickException( "keyring is not installed; run `pip install keyring`." ) from exc keyring.set_password(f"{credential_target}:meta", "username", build_username) keyring.set_password(credential_target, build_username, build_password) click.echo( f"[prepare-win] Credentials stored in keyring " f"(target='{credential_target}', user='{build_username}')." ) click.echo("[prepare-win] Provisioning complete.")