feat(prepare-win): idle check waits for quiet, reports busiest process

The one-shot idle sample fired ~30s after the final reboot, but Windows
Update / CBS (TiWorker) keeps finalizing for minutes — so a still-busy
template was wrongly rejected. Poll until CPU and RAM are both under
threshold (up to --idle-timeout, default 600s) instead of sampling once,
and report the busiest process to aid diagnosis. Adds --idle-timeout.

src/ci_orchestrator/commands/template.py

@@ -379,22 +379,13 @@ def _wait_winrm(transport: WinRmTransport, timeout: int) -> None:
     raise click.ClickException(f"WinRM not ready within {timeout}s.")
 
 
-def _check_guest_idle(
+def _sample_guest_idle(transport: WinRmTransport) -> tuple[float, float, str]:
-    transport: WinRmTransport, *, max_cpu: float, max_ram: float, settle_s: int
+    """Return (cpu%, used-ram%, busiest-process) sampled over ~5s in the guest.
-) -> None:
-    """Sample guest CPU/RAM after a settle period; abort if above thresholds.
 
-    Run just before the snapshot so a busy/leaky template (e.g. a service stuck
+    CPU via Win32_PerfFormattedData (0-100, locale-independent); RAM as
-    spinning after the toolchain install) is caught instead of being frozen into
+    used-physical percent; busiest process by cumulative CPU (for diagnostics).
-    every build clone. CPU via Win32_PerfFormattedData (0-100, locale-independent);
-    RAM as used-physical percent.
     """
-    click.echo(
-        f"[prepare-win] Idle check (settle {settle_s}s; thresholds "
-        f"CPU<={max_cpu}% RAM<={max_ram}%) ..."
-    )
     ps = (
-        f"Start-Sleep -Seconds {settle_s}\n"
         "$s = 0..4 | ForEach-Object { (Get-CimInstance "
         "Win32_PerfFormattedData_PerfOS_Processor -Filter \"Name='_Total'\")"
         ".PercentProcessorTime; Start-Sleep -Milliseconds 1000 }\n"
@@ -402,31 +393,65 @@ def _check_guest_idle(
         "$os = Get-CimInstance Win32_OperatingSystem\n"
         "$ram = [math]::Round((1 - ($os.FreePhysicalMemory / "
         "$os.TotalVisibleMemorySize)) * 100, 1)\n"
-        "[pscustomobject]@{cpu=$cpu; ram=$ram} | ConvertTo-Json -Compress\n"
+        "$top = (Get-Process | Sort-Object CPU -Descending | "
+        "Select-Object -First 1).Name\n"
+        "[pscustomobject]@{cpu=$cpu; ram=$ram; top=$top} | ConvertTo-Json -Compress\n"
     )
     res = transport.run(ps, check=False)
     lines = [ln for ln in res.stdout.splitlines() if ln.strip()]
     try:
         data = json.loads(lines[-1])
-        cpu = float(data["cpu"])
+        return float(data["cpu"]), float(data["ram"]), str(data.get("top", "?"))
-        ram = float(data["ram"])
     except (json.JSONDecodeError, KeyError, ValueError, IndexError) as exc:
         raise click.ClickException(
             f"Idle check: could not parse guest metrics.\n{res.stdout}\n{res.stderr}"
         ) from exc
-    click.echo(f"[prepare-win]   idle CPU {cpu}%  RAM {ram}%")
+
+
+def _check_guest_idle(
+    transport: WinRmTransport,
+    *,
+    max_cpu: float,
+    max_ram: float,
+    settle_s: int,
+    timeout_s: int,
+) -> None:
+    """Wait for the guest to go idle, then proceed; abort if it never settles.
+
+    Run just before the snapshot. Windows Update / CBS (TiWorker) keeps working
+    for minutes after the final reboot, so we poll until CPU and RAM are both
+    under threshold (up to ``timeout_s``) instead of sampling once — otherwise a
+    transiently-busy template would either be wrongly rejected or frozen busy
+    into every clone. The busiest process is reported to aid diagnosis.
+    """
+    click.echo(
+        f"[prepare-win] Idle check (settle {settle_s}s, wait ≤{timeout_s}s; "
+        f"thresholds CPU≤{max_cpu}% RAM≤{max_ram}%) ..."
+    )
+    if settle_s > 0:
+        time.sleep(settle_s)
+    deadline = time.monotonic() + timeout_s
+    cpu = ram = 0.0
+    top = "?"
+    while True:
+        cpu, ram, top = _sample_guest_idle(transport)
+        click.echo(f"[prepare-win]   CPU {cpu}%  RAM {ram}%  (busiest: {top})")
+        if cpu <= max_cpu and ram <= max_ram:
+            click.echo("[prepare-win]   idle OK.")
+            return
+        if time.monotonic() >= deadline:
+            break
+        time.sleep(10)
     problems: list[str] = []
     if cpu > max_cpu:
         problems.append(f"CPU {cpu}% > {max_cpu}%")
     if ram > max_ram:
         problems.append(f"RAM {ram}% > {max_ram}%")
-    if problems:
+    raise click.ClickException(
-        raise click.ClickException(
+        f"Guest did not go idle within {timeout_s}s before snapshot: "
-            "Guest not idle before snapshot: "
+        + "; ".join(problems)
-            + "; ".join(problems)
+        + f" (busiest: {top}). Aborting — snapshot NOT taken; investigate the template."
-            + ". Aborting — snapshot NOT taken; investigate the template."
+    )
-        )
-    click.echo("[prepare-win]   idle OK.")
 
 
 def _run_with_heartbeat(
@@ -517,6 +542,8 @@ def _run_with_heartbeat(
               help="Abort before snapshot if used RAM exceeds this percent.")
 @click.option("--idle-settle", default=30, show_default=True,
               help="Seconds to let the guest settle before sampling idle CPU/RAM.")
+@click.option("--idle-timeout", default=600, show_default=True,
+              help="Max seconds to wait for the guest to go idle before aborting.")
 @click.option("--skip-idle-check", is_flag=True, default=False,
               help="Skip the pre-snapshot idle CPU/RAM check.")
 def template_prepare_win(
@@ -540,6 +567,7 @@ def template_prepare_win(
     max_idle_cpu: float,
     max_idle_ram: float,
     idle_settle: int,
+    idle_timeout: int,
     skip_idle_check: bool,
 ) -> None:
     """Provision a Windows template VM via pypsrp (Linux-host compatible).
@@ -830,6 +858,7 @@ Register-ScheduledTask -TaskName 'CI-StaticIp' `
                 max_cpu=max_idle_cpu,
                 max_ram=max_idle_ram,
                 settle_s=idle_settle,
+                timeout_s=idle_timeout,
             )
 
         # Graceful shutdown

tests/python/test_commands_template.py

@@ -433,7 +433,7 @@ def _make_transport() -> MagicMock:
 
     def _run(script: str = "", **kwargs: object) -> WinRmResult:
         if "PercentProcessorTime" in script:  # pre-snapshot idle check
-            return WinRmResult(stdout='{"cpu":2.0,"ram":21.0}', stderr="", returncode=0)
+            return WinRmResult(stdout='{"cpu":2.0,"ram":21.0,"top":"idle"}', stderr="", returncode=0)
         if "ConvertTo-Json" in script:
             return WinRmResult(stdout=val_json, stderr="", returncode=0)
         return WinRmResult(stdout="CI_EXITCODE:0", stderr="", returncode=0)
@@ -2220,26 +2220,32 @@ def test_clean_force_artifacts_noop_on_clean_dir(tmp_path: Path) -> None:
 
 def test_check_guest_idle_ok() -> None:
     transport = MagicMock()
-    transport.run.return_value = WinRmResult(stdout='{"cpu":3.0,"ram":22.5}', stderr="", returncode=0)
+    transport.run.return_value = WinRmResult(
-    tmpl_mod._check_guest_idle(transport, max_cpu=10.0, max_ram=50.0, settle_s=0)  # no raise
+        stdout='{"cpu":3.0,"ram":22.5,"top":"idle"}', stderr="", returncode=0
+    )
+    tmpl_mod._check_guest_idle(transport, max_cpu=10.0, max_ram=50.0, settle_s=0, timeout_s=5)
 
 
 def test_check_guest_idle_cpu_over() -> None:
     transport = MagicMock()
-    transport.run.return_value = WinRmResult(stdout='{"cpu":42.0,"ram":20.0}', stderr="", returncode=0)
+    transport.run.return_value = WinRmResult(
+        stdout='{"cpu":42.0,"ram":20.0,"top":"TiWorker"}', stderr="", returncode=0
+    )
     with pytest.raises(click.ClickException, match=r"CPU 42\.0% > 10\.0%"):
-        tmpl_mod._check_guest_idle(transport, max_cpu=10.0, max_ram=50.0, settle_s=0)
+        tmpl_mod._check_guest_idle(transport, max_cpu=10.0, max_ram=50.0, settle_s=0, timeout_s=0)
 
 
 def test_check_guest_idle_ram_over() -> None:
     transport = MagicMock()
-    transport.run.return_value = WinRmResult(stdout='{"cpu":1.0,"ram":80.0}', stderr="", returncode=0)
+    transport.run.return_value = WinRmResult(
+        stdout='{"cpu":1.0,"ram":80.0,"top":"x"}', stderr="", returncode=0
+    )
     with pytest.raises(click.ClickException, match=r"RAM 80\.0% > 50\.0%"):
-        tmpl_mod._check_guest_idle(transport, max_cpu=10.0, max_ram=50.0, settle_s=0)
+        tmpl_mod._check_guest_idle(transport, max_cpu=10.0, max_ram=50.0, settle_s=0, timeout_s=0)
 
 
 def test_check_guest_idle_unparseable() -> None:
     transport = MagicMock()
     transport.run.return_value = WinRmResult(stdout="not json", stderr="", returncode=0)
     with pytest.raises(click.ClickException, match="could not parse"):
-        tmpl_mod._check_guest_idle(transport, max_cpu=10.0, max_ram=50.0, settle_s=0)
+        tmpl_mod._sample_guest_idle(transport)