feat(prepare-win): idle check waits for quiet, reports busiest process

The one-shot idle sample fired ~30s after the final reboot, but Windows Update / CBS (TiWorker) keeps finalizing for minutes — so a still-busy template was wrongly rejected. Poll until CPU and RAM are both under threshold (up to --idle-timeout, default 600s) instead of sampling once, and report the busiest process to aid diagnosis. Adds --idle-timeout.
2026-06-09 21:52:29 +02:00
parent 54feb43513
commit c1be0f276f
2 changed files with 67 additions and 32 deletions
@@ -379,22 +379,13 @@ def _wait_winrm(transport: WinRmTransport, timeout: int) -> None:
    raise click.ClickException(f"WinRM not ready within {timeout}s.")


-def _check_guest_idle(
-    transport: WinRmTransport, *, max_cpu: float, max_ram: float, settle_s: int
-) -> None:
-    """Sample guest CPU/RAM after a settle period; abort if above thresholds.
+def _sample_guest_idle(transport: WinRmTransport) -> tuple[float, float, str]:
+    """Return (cpu%, used-ram%, busiest-process) sampled over ~5s in the guest.

-    Run just before the snapshot so a busy/leaky template (e.g. a service stuck
-    spinning after the toolchain install) is caught instead of being frozen into
-    every build clone. CPU via Win32_PerfFormattedData (0-100, locale-independent);
-    RAM as used-physical percent.
+    CPU via Win32_PerfFormattedData (0-100, locale-independent); RAM as
+    used-physical percent; busiest process by cumulative CPU (for diagnostics).
    """
-    click.echo(
-        f"[prepare-win] Idle check (settle {settle_s}s; thresholds "
-        f"CPU<={max_cpu}% RAM<={max_ram}%) ..."
-    )
    ps = (
-        f"Start-Sleep -Seconds {settle_s}\n"
        "$s = 0..4 | ForEach-Object { (Get-CimInstance "
        "Win32_PerfFormattedData_PerfOS_Processor -Filter \"Name='_Total'\")"
        ".PercentProcessorTime; Start-Sleep -Milliseconds 1000 }\n"
@@ -402,31 +393,65 @@ def _check_guest_idle(
        "$os = Get-CimInstance Win32_OperatingSystem\n"
        "$ram = [math]::Round((1 - ($os.FreePhysicalMemory / "
        "$os.TotalVisibleMemorySize)) * 100, 1)\n"
-        "[pscustomobject]@{cpu=$cpu; ram=$ram} | ConvertTo-Json -Compress\n"
+        "$top = (Get-Process | Sort-Object CPU -Descending | "
+        "Select-Object -First 1).Name\n"
+        "[pscustomobject]@{cpu=$cpu; ram=$ram; top=$top} | ConvertTo-Json -Compress\n"
    )
    res = transport.run(ps, check=False)
    lines = [ln for ln in res.stdout.splitlines() if ln.strip()]
    try:
        data = json.loads(lines[-1])
-        cpu = float(data["cpu"])
-        ram = float(data["ram"])
+        return float(data["cpu"]), float(data["ram"]), str(data.get("top", "?"))
    except (json.JSONDecodeError, KeyError, ValueError, IndexError) as exc:
        raise click.ClickException(
            f"Idle check: could not parse guest metrics.\n{res.stdout}\n{res.stderr}"
        ) from exc
-    click.echo(f"[prepare-win]   idle CPU {cpu}%  RAM {ram}%")
+
+
+def _check_guest_idle(
+    transport: WinRmTransport,
+    *,
+    max_cpu: float,
+    max_ram: float,
+    settle_s: int,
+    timeout_s: int,
+) -> None:
+    """Wait for the guest to go idle, then proceed; abort if it never settles.
+
+    Run just before the snapshot. Windows Update / CBS (TiWorker) keeps working
+    for minutes after the final reboot, so we poll until CPU and RAM are both
+    under threshold (up to ``timeout_s``) instead of sampling once — otherwise a
+    transiently-busy template would either be wrongly rejected or frozen busy
+    into every clone. The busiest process is reported to aid diagnosis.
+    """
+    click.echo(
+        f"[prepare-win] Idle check (settle {settle_s}s, wait ≤{timeout_s}s; "
+        f"thresholds CPU≤{max_cpu}% RAM≤{max_ram}%) ..."
+    )
+    if settle_s > 0:
+        time.sleep(settle_s)
+    deadline = time.monotonic() + timeout_s
+    cpu = ram = 0.0
+    top = "?"
+    while True:
+        cpu, ram, top = _sample_guest_idle(transport)
+        click.echo(f"[prepare-win]   CPU {cpu}%  RAM {ram}%  (busiest: {top})")
+        if cpu <= max_cpu and ram <= max_ram:
+            click.echo("[prepare-win]   idle OK.")
+            return
+        if time.monotonic() >= deadline:
+            break
+        time.sleep(10)
    problems: list[str] = []
    if cpu > max_cpu:
        problems.append(f"CPU {cpu}% > {max_cpu}%")
    if ram > max_ram:
        problems.append(f"RAM {ram}% > {max_ram}%")
-    if problems:
-        raise click.ClickException(
-            "Guest not idle before snapshot: "
-            + "; ".join(problems)
-            + ". Aborting — snapshot NOT taken; investigate the template."
-        )
-    click.echo("[prepare-win]   idle OK.")
+    raise click.ClickException(
+        f"Guest did not go idle within {timeout_s}s before snapshot: "
+        + "; ".join(problems)
+        + f" (busiest: {top}). Aborting — snapshot NOT taken; investigate the template."
+    )


 def _run_with_heartbeat(
@@ -517,6 +542,8 @@ def _run_with_heartbeat(
              help="Abort before snapshot if used RAM exceeds this percent.")
@click.option("--idle-settle", default=30, show_default=True,
              help="Seconds to let the guest settle before sampling idle CPU/RAM.")
+@click.option("--idle-timeout", default=600, show_default=True,
+              help="Max seconds to wait for the guest to go idle before aborting.")
@click.option("--skip-idle-check", is_flag=True, default=False,
              help="Skip the pre-snapshot idle CPU/RAM check.")
 def template_prepare_win(
@@ -540,6 +567,7 @@ def template_prepare_win(
    max_idle_cpu: float,
    max_idle_ram: float,
    idle_settle: int,
+    idle_timeout: int,
    skip_idle_check: bool,
 ) -> None:
    """Provision a Windows template VM via pypsrp (Linux-host compatible).
@@ -830,6 +858,7 @@ Register-ScheduledTask -TaskName 'CI-StaticIp' `
                max_cpu=max_idle_cpu,
                max_ram=max_idle_ram,
                settle_s=idle_settle,
+                timeout_s=idle_timeout,
            )

        # Graceful shutdown
@@ -433,7 +433,7 @@ def _make_transport() -> MagicMock:

    def _run(script: str = "", **kwargs: object) -> WinRmResult:
        if "PercentProcessorTime" in script:  # pre-snapshot idle check
-            return WinRmResult(stdout='{"cpu":2.0,"ram":21.0}', stderr="", returncode=0)
+            return WinRmResult(stdout='{"cpu":2.0,"ram":21.0,"top":"idle"}', stderr="", returncode=0)
        if "ConvertTo-Json" in script:
            return WinRmResult(stdout=val_json, stderr="", returncode=0)
        return WinRmResult(stdout="CI_EXITCODE:0", stderr="", returncode=0)
@@ -2220,26 +2220,32 @@ def test_clean_force_artifacts_noop_on_clean_dir(tmp_path: Path) -> None:

 def test_check_guest_idle_ok() -> None:
    transport = MagicMock()
-    transport.run.return_value = WinRmResult(stdout='{"cpu":3.0,"ram":22.5}', stderr="", returncode=0)
-    tmpl_mod._check_guest_idle(transport, max_cpu=10.0, max_ram=50.0, settle_s=0)  # no raise
+    transport.run.return_value = WinRmResult(
+        stdout='{"cpu":3.0,"ram":22.5,"top":"idle"}', stderr="", returncode=0
+    )
+    tmpl_mod._check_guest_idle(transport, max_cpu=10.0, max_ram=50.0, settle_s=0, timeout_s=5)


 def test_check_guest_idle_cpu_over() -> None:
    transport = MagicMock()
-    transport.run.return_value = WinRmResult(stdout='{"cpu":42.0,"ram":20.0}', stderr="", returncode=0)
+    transport.run.return_value = WinRmResult(
+        stdout='{"cpu":42.0,"ram":20.0,"top":"TiWorker"}', stderr="", returncode=0
+    )
    with pytest.raises(click.ClickException, match=r"CPU 42\.0% > 10\.0%"):
-        tmpl_mod._check_guest_idle(transport, max_cpu=10.0, max_ram=50.0, settle_s=0)
+        tmpl_mod._check_guest_idle(transport, max_cpu=10.0, max_ram=50.0, settle_s=0, timeout_s=0)


 def test_check_guest_idle_ram_over() -> None:
    transport = MagicMock()
-    transport.run.return_value = WinRmResult(stdout='{"cpu":1.0,"ram":80.0}', stderr="", returncode=0)
+    transport.run.return_value = WinRmResult(
+        stdout='{"cpu":1.0,"ram":80.0,"top":"x"}', stderr="", returncode=0
+    )
    with pytest.raises(click.ClickException, match=r"RAM 80\.0% > 50\.0%"):
-        tmpl_mod._check_guest_idle(transport, max_cpu=10.0, max_ram=50.0, settle_s=0)
+        tmpl_mod._check_guest_idle(transport, max_cpu=10.0, max_ram=50.0, settle_s=0, timeout_s=0)


 def test_check_guest_idle_unparseable() -> None:
    transport = MagicMock()
    transport.run.return_value = WinRmResult(stdout="not json", stderr="", returncode=0)
    with pytest.raises(click.ClickException, match="could not parse"):
-        tmpl_mod._check_guest_idle(transport, max_cpu=10.0, max_ram=50.0, settle_s=0)
+        tmpl_mod._sample_guest_idle(transport)