feat(template): add template prepare-win Python command via pypsrp

Replaces the abandoned PSWSMan approach (New-PSSession hangs on Linux)
with a Python-native implementation using the proven pypsrp WinRmTransport.

- New `template prepare-win` command in commands/template.py: starts VM,
  waits for WinRM, uploads and runs Install-CIToolchain-*.ps1, handles
  Windows Update 3010 reboot loop (max 10 iterations), installs CI-StaticIp
  scheduled task, runs post-setup validation, graceful shutdown, snapshot.
- Helpers: _ps_escape, _wait_tcp, _parse_exit_marker, _wait_winrm.
- 40 new tests in test_commands_template.py covering all major branches.
- Fix test_commands_job.py: patch load_config in _patch_common so tests
  don't pick up live /var/lib/ci/config.toml ip_pool and fail on lock.
- Coverage gate: 90.06% (was 88.95% before fix).

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

src/ci_orchestrator/commands/template.py

@@ -14,14 +14,21 @@ is already offline.
 from __future__ import annotations
 
 import fnmatch
+import json
 import shutil
+import socket
 import subprocess
+import time
 from datetime import UTC, datetime
 from pathlib import Path
 
 import click
 
+from ci_orchestrator.backends.workstation import WorkstationVmrunBackend
+from ci_orchestrator.backends.protocol import VmHandle
 from ci_orchestrator.config import load_config
+from ci_orchestrator.transport.errors import TransportError
+from ci_orchestrator.transport.winrm import WinRmTransport
 
 
 @click.group()
@@ -206,3 +213,384 @@ def _prune_backups(
     if to_remove:
         label = template_name or "template"
         click.echo(f"[Backup-CITemplate] Retained {keep_count} backup(s) for {label}.")
+
+
+# ═══════════════════════════════════════════════════════════════════════════
+# template prepare-win helpers
+# ═══════════════════════════════════════════════════════════════════════════
+
+
+def _ps_escape(s: str) -> str:
+    """Escape *s* for use inside a PowerShell single-quoted string."""
+    return s.replace("'", "''")
+
+
+def _wait_tcp(host: str, port: int, timeout: float) -> bool:
+    """Return True once TCP *port* on *host* accepts a connection within *timeout* s."""
+    deadline = time.monotonic() + timeout
+    while time.monotonic() < deadline:
+        try:
+            with socket.create_connection((host, port), timeout=5.0):
+                return True
+        except OSError:
+            time.sleep(5)
+    return False
+
+
+def _parse_exit_marker(stdout: str) -> int:
+    """Extract ``CI_EXITCODE:N`` written by the invoke wrapper; default 0."""
+    for line in reversed(stdout.splitlines()):
+        stripped = line.strip()
+        if stripped.startswith("CI_EXITCODE:"):
+            try:
+                return int(stripped.split(":", 1)[1])
+            except (IndexError, ValueError):
+                pass
+    return 0
+
+
+def _wait_winrm(transport: WinRmTransport, timeout: int) -> None:
+    """Poll *transport*.is_ready() until True or *timeout* seconds elapse."""
+    deadline = time.monotonic() + timeout
+    while time.monotonic() < deadline:
+        if transport.is_ready():
+            return
+        time.sleep(10)
+        click.echo("  ... waiting for WinRM ...")
+    raise click.ClickException(f"WinRM not ready within {timeout}s.")
+
+
+# ═══════════════════════════════════════════════════════════════════════════
+# template prepare-win command
+# ═══════════════════════════════════════════════════════════════════════════
+
+
+@template.command("prepare-win")
+@click.option("--vmx", required=True, metavar="PATH", help="Template VM .vmx file.")
+@click.option("--ip", "guest_ip", default="", help="Guest IP (auto-detect via VMware Tools if omitted).")
+@click.option("--admin-username", default="Administrator", show_default=True)
+@click.option("--admin-password", default="", help="Administrator password (prompted if omitted).")
+@click.option("--build-username", default="ci_build", show_default=True)
+@click.option("--build-password", default="", help="CI build user password (prompted if omitted).")
+@click.option("--snapshot", default="BaseClean", show_default=True, help="Snapshot name.")
+@click.option("--toolchain-script", "toolchain_script", default=None, metavar="PATH",
+              help="Path to Install-CIToolchain-*.ps1 (auto-detected from VMX stem if omitted).")
+@click.option("--skip-windows-update", is_flag=True, help="Pass -SkipWindowsUpdate to toolchain script.")
+@click.option("--skip-tier2", is_flag=True, help="Pass -SkipTier2 to toolchain script.")
+@click.option("--skip-cleanup", is_flag=True, help="Pass -SkipCleanup to toolchain script.")
+@click.option("--winrm-timeout", default=300, show_default=True, help="Seconds to wait for WinRM.")
+@click.option("--auth", default="basic", show_default=True,
+              type=click.Choice(["basic", "ntlm", "negotiate"]),
+              help="WinRM authentication protocol.")
+def template_prepare_win(
+    vmx: str,
+    guest_ip: str,
+    admin_username: str,
+    admin_password: str,
+    build_username: str,
+    build_password: str,
+    snapshot: str,
+    toolchain_script: str | None,
+    skip_windows_update: bool,
+    skip_tier2: bool,
+    skip_cleanup: bool,
+    winrm_timeout: int,
+    auth: str,
+) -> None:
+    """Provision a Windows template VM via pypsrp (Linux-host compatible).
+
+    Replaces Prepare-WinBuild*.ps1 for hosts where PSWSMan/New-PSSession is
+    unavailable or unreliable.  Run from the repo root so toolchain and
+    guest-setup scripts are auto-discovered under template/.
+    """
+    vmx_path = Path(vmx).resolve()
+    if not vmx_path.is_file():
+        raise click.ClickException(f"VMX not found: {vmx_path}")
+
+    # --- Resolve toolchain script ---
+    if toolchain_script is None:
+        candidate = Path.cwd() / "template" / f"Install-CIToolchain-{vmx_path.stem}.ps1"
+        if not candidate.is_file():
+            raise click.ClickException(
+                f"Toolchain script not found at {candidate}. "
+                "Run from the repo root or pass --toolchain-script."
+            )
+        toolchain_path = candidate
+    else:
+        toolchain_path = Path(toolchain_script)
+        if not toolchain_path.is_file():
+            raise click.ClickException(f"Toolchain script not found: {toolchain_path}")
+
+    # --- Locate CI-StaticIp guest script ---
+    _sip_candidate = Path.cwd() / "template" / "guest-setup" / "windows" / "ci-static-ip.ps1"
+    static_ip_path: Path | None
+    if _sip_candidate.is_file():
+        static_ip_path = _sip_candidate
+    else:
+        click.echo(
+            f"[prepare-win] WARNING: {_sip_candidate} not found — "
+            "ip_pool static-IP feature will not work on this template.",
+            err=True,
+        )
+        static_ip_path = None
+
+    # --- Prompt for passwords ---
+    if not admin_password:
+        admin_password = click.prompt(
+            f"Template VM {admin_username} password", hide_input=True
+        )
+    if not build_password:
+        build_password = click.prompt(f"New {build_username} password", hide_input=True)
+
+    # --- Backend: start VM and detect IP ---
+    config = load_config()
+    backend = WorkstationVmrunBackend(vmrun_path=config.vmrun_path)
+    handle = VmHandle(identifier=str(vmx_path), name=vmx_path.stem)
+
+    if not backend.is_running(handle):
+        click.echo(f"[prepare-win] Starting VM: {vmx_path.name} ...")
+        backend.start(handle, headless=False)
+    else:
+        click.echo(f"[prepare-win] VM already running: {vmx_path.name}")
+
+    if not guest_ip:
+        click.echo("[prepare-win] Waiting for guest IP (vmrun getGuestIPAddress -wait) ...")
+        detected = backend.get_ip(handle, timeout=300.0)
+        if not detected:
+            raise click.ClickException(
+                "Could not get guest IP. Ensure VMware Tools is installed."
+            )
+        guest_ip = detected
+        click.echo(f"[prepare-win] Guest IP: {guest_ip}")
+
+    # --- Wait for WinRM TCP then probe session ---
+    click.echo(f"[prepare-win] Waiting for TCP/5986 on {guest_ip} ({winrm_timeout}s) ...")
+    if not _wait_tcp(guest_ip, 5986, float(winrm_timeout)):
+        raise click.ClickException(f"TCP/5986 not reachable on {guest_ip}.")
+
+    transport = WinRmTransport(
+        host=guest_ip,
+        username=admin_username,
+        password=admin_password,
+        port=5986,
+        ssl=True,
+        cert_validation=False,
+        auth=auth,
+        connection_timeout=30,
+    )
+    click.echo("[prepare-win] Probing WinRM session ...")
+    _wait_winrm(transport, winrm_timeout)
+    click.echo("[prepare-win] WinRM ready.")
+
+    try:
+        # Ensure C:\CI exists
+        transport.run(
+            "if (-not (Test-Path 'C:\\CI')) "
+            "{ New-Item -ItemType Directory 'C:\\CI' -Force | Out-Null }"
+        )
+
+        # Upload toolchain script
+        guest_script = f"C:\\CI\\{toolchain_path.name}"
+        click.echo(f"[prepare-win] Uploading {toolchain_path.name} -> {guest_script} ...")
+        transport.copy(toolchain_path, guest_script)
+        click.echo("[prepare-win] Upload complete.")
+
+        # Build args string
+        e = _ps_escape
+        args_parts = [
+            f"-BuildPassword '{e(build_password)}'",
+            f"-BuildUsername '{e(build_username)}'",
+            f"-AdminPassword '{e(admin_password)}'",
+        ]
+        if skip_windows_update:
+            args_parts.append("-SkipWindowsUpdate:$true")
+        if skip_tier2:
+            args_parts.append("-SkipTier2:$true")
+        if skip_cleanup:
+            args_parts.append("-SkipCleanup:$true")
+        args_str = " ".join(args_parts)
+
+        # Run toolchain script — loop handles Windows Update reboot (exit 3010)
+        _MAX_ITER = 10
+        exit_code = 0
+        for iteration in range(1, _MAX_ITER + 1):
+            click.echo(
+                f"[prepare-win] Running {toolchain_path.name} "
+                f"(iteration {iteration}/{_MAX_ITER}) ..."
+            )
+            invoke = (
+                "Set-ExecutionPolicy Bypass -Scope Process -Force\n"
+                "$ec = 0\n"
+                "try {\n"
+                f"    & '{e(guest_script)}' {args_str}\n"
+                "    $ec = if ($null -eq $LASTEXITCODE) { 0 } else { $LASTEXITCODE }\n"
+                "} catch {\n"
+                "    Write-Error $_\n"
+                "    $ec = 1\n"
+                "}\n"
+                '"CI_EXITCODE:$ec"\n'
+            )
+            try:
+                result = transport.run(invoke, check=False)
+                exit_code = _parse_exit_marker(result.stdout)
+                if result.stdout.strip():
+                    click.echo(result.stdout.strip())
+                if result.stderr.strip():
+                    click.echo(result.stderr.strip(), err=True)
+            except TransportError:
+                click.echo(
+                    "[prepare-win] Transport error — VM likely rebooted mid-update "
+                    "(treating as 3010)."
+                )
+                exit_code = 3010
+
+            click.echo(f"[prepare-win] Toolchain exit code: {exit_code}")
+
+            if exit_code == 0:
+                break
+            if exit_code != 3010:
+                raise click.ClickException(
+                    f"Toolchain script failed (exit {exit_code})."
+                )
+            if iteration == _MAX_ITER:
+                raise click.ClickException(
+                    f"Windows Update did not converge after {_MAX_ITER} iterations."
+                )
+
+            click.echo("[prepare-win] Windows Update requires reboot — rebooting ...")
+            try:
+                transport.run("Restart-Computer -Force", check=False)
+            except TransportError:
+                pass
+            transport.close()
+            time.sleep(30)
+            click.echo(f"[prepare-win] Waiting for WinRM after reboot ({winrm_timeout}s) ...")
+            _wait_winrm(transport, winrm_timeout)
+            click.echo("[prepare-win] WinRM ready after reboot.")
+
+        # Install CI-StaticIp scheduled task
+        if static_ip_path is not None:
+            click.echo("[prepare-win] Installing CI-StaticIp scheduled task ...")
+            transport.copy(static_ip_path, "C:\\CI\\ci-static-ip.ps1")
+            transport.run(r"""
+$action    = New-ScheduledTaskAction -Execute 'powershell.exe' `
+                 -Argument '-NoProfile -NonInteractive -ExecutionPolicy Bypass -File C:\CI\ci-static-ip.ps1'
+$trigger   = New-ScheduledTaskTrigger -AtStartup
+$principal = New-ScheduledTaskPrincipal -UserId 'SYSTEM' -LogonType ServiceAccount -RunLevel Highest
+Register-ScheduledTask -TaskName 'CI-StaticIp' `
+    -Action $action -Trigger $trigger -Principal $principal -Force | Out-Null
+""")
+            click.echo("[prepare-win] CI-StaticIp task registered.")
+
+        # Post-setup validation
+        click.echo("[prepare-win] Running post-setup validation ...")
+        esc_u = e(build_username)
+        val_result = transport.run(
+            "$env:PATH = [System.Environment]::GetEnvironmentVariable('PATH','Machine')"
+            " + ';' + [System.Environment]::GetEnvironmentVariable('PATH','User')\n"
+            "$msbuild = 'C:\\Program Files (x86)\\Microsoft Visual Studio\\2026"
+            "\\BuildTools\\MSBuild\\Current\\Bin\\MSBuild.exe'\n"
+            "@{\n"
+            "    WinRMRunning   = (Get-Service WinRM -EA SilentlyContinue).Status -eq 'Running'\n"
+            f"    UserExists     = [bool](Get-LocalUser -Name '{esc_u}' -EA SilentlyContinue)\n"
+            f"    UserIsAdmin    = [bool](net localgroup Administrators 2>&1 | Select-String -SimpleMatch '{esc_u}')\n"
+            "    UACDisabled    = (Get-ItemProperty 'HKLM:\\SOFTWARE\\Microsoft\\Windows"
+            "\\CurrentVersion\\Policies\\System' -Name EnableLUA -EA SilentlyContinue).EnableLUA -eq 0\n"
+            "    FirewallOff    = -not (Get-NetFirewallProfile | Where-Object Enabled -eq $true)\n"
+            "    DotNetExists   = [bool](Get-Command dotnet -EA SilentlyContinue)\n"
+            "    PythonExists   = Test-Path 'C:\\Python\\python.exe' -PathType Leaf\n"
+            "    MSBuildExists  = Test-Path $msbuild -PathType Leaf\n"
+            "    CIDirsPresent  = (Test-Path 'C:\\CI\\build') -and (Test-Path 'C:\\CI\\output')"
+            " -and (Test-Path 'C:\\CI\\scripts')\n"
+            "    StaticIpTask   = [bool](Get-ScheduledTask -TaskName 'CI-StaticIp' -EA SilentlyContinue)\n"
+            "    StaticIpScript = Test-Path 'C:\\CI\\ci-static-ip.ps1' -PathType Leaf\n"
+            "} | ConvertTo-Json -Compress\n",
+            check=False,
+        )
+        if val_result.ok and val_result.stdout.strip():
+            try:
+                state: dict[str, object] = json.loads(val_result.stdout.strip())
+                failed = [k for k, v in state.items() if not v]
+                for k, v in state.items():
+                    mark = "OK  " if v else "FAIL"
+                    click.echo(f"  [{mark}] {k}")
+                if failed:
+                    click.echo(
+                        f"[prepare-win] WARNING: {len(failed)} check(s) failed: "
+                        f"{', '.join(failed)}",
+                        err=True,
+                    )
+                else:
+                    click.echo("[prepare-win] All post-setup checks passed.")
+            except (json.JSONDecodeError, AttributeError):
+                click.echo(val_result.stdout)
+        else:
+            click.echo(
+                f"[prepare-win] Validation script error:\n{val_result.stderr}", err=True
+            )
+
+        # Final reboot to flush installation state
+        click.echo("[prepare-win] Final reboot to flush installation state ...")
+        try:
+            transport.run("Restart-Computer -Force", check=False)
+        except TransportError:
+            pass
+        transport.close()
+        time.sleep(30)
+        click.echo("[prepare-win] Waiting for WinRM after final reboot ...")
+        _wait_winrm(transport, winrm_timeout)
+
+        # Re-affirm autologin
+        click.echo("[prepare-win] Re-affirming autologin registry settings ...")
+        transport.run(
+            "$wl = 'HKLM:\\SOFTWARE\\Microsoft\\Windows NT\\CurrentVersion\\Winlogon'\n"
+            "Set-ItemProperty $wl AutoAdminLogon '1' -Type String -Force\n"
+            f"Set-ItemProperty $wl DefaultUserName '{e(admin_username)}' -Type String -Force\n"
+            "Set-ItemProperty $wl DefaultDomainName '.' -Type String -Force\n"
+            f"if ('{e(admin_password)}' -ne '') {{\n"
+            f"    Set-ItemProperty $wl DefaultPassword '{e(admin_password)}' -Type String -Force\n"
+            "}\n"
+        )
+
+        # Graceful shutdown
+        click.echo("[prepare-win] Sending graceful shutdown ...")
+        try:
+            transport.run("Stop-Computer -Force", check=False)
+        except TransportError:
+            pass
+
+    finally:
+        transport.close()
+
+    # Wait for VM to power off
+    click.echo("[prepare-win] Waiting for VM to power off (120s) ...")
+    off_deadline = time.monotonic() + 120
+    while time.monotonic() < off_deadline:
+        if not backend.is_running(handle):
+            break
+        time.sleep(5)
+        click.echo("  ... waiting for shutdown ...")
+    else:
+        raise click.ClickException("VM did not power off within 120s.")
+    click.echo("[prepare-win] VM powered off.")
+
+    # Create snapshot
+    vmrun_exe = backend.vmrun_path
+    existing = backend.list_snapshots(handle)
+    if snapshot in existing:
+        click.echo(f"[prepare-win] Snapshot '{snapshot}' already exists.")
+        if not click.confirm(f"Delete and recreate '{snapshot}'?", default=False):
+            click.echo("[prepare-win] Keeping existing snapshot. Provisioning complete.")
+            return
+        click.echo(f"[prepare-win] Deleting snapshot '{snapshot}' ...")
+        subprocess.run(
+            [vmrun_exe, "-T", "ws", "deleteSnapshot", str(vmx_path), snapshot],
+            check=True,
+        )
+
+    click.echo(f"[prepare-win] Creating snapshot '{snapshot}' ...")
+    subprocess.run(
+        [vmrun_exe, "-T", "ws", "snapshot", str(vmx_path), snapshot],
+        check=True,
+    )
+    click.echo(f"[prepare-win] Snapshot '{snapshot}' created. Provisioning complete.")