diff --git a/src/ci_orchestrator/commands/template.py b/src/ci_orchestrator/commands/template.py index a7fb233..469cd28 100644 --- a/src/ci_orchestrator/commands/template.py +++ b/src/ci_orchestrator/commands/template.py @@ -379,22 +379,13 @@ def _wait_winrm(transport: WinRmTransport, timeout: int) -> None: raise click.ClickException(f"WinRM not ready within {timeout}s.") -def _check_guest_idle( - transport: WinRmTransport, *, max_cpu: float, max_ram: float, settle_s: int -) -> None: - """Sample guest CPU/RAM after a settle period; abort if above thresholds. +def _sample_guest_idle(transport: WinRmTransport) -> tuple[float, float, str]: + """Return (cpu%, used-ram%, busiest-process) sampled over ~5s in the guest. - Run just before the snapshot so a busy/leaky template (e.g. a service stuck - spinning after the toolchain install) is caught instead of being frozen into - every build clone. CPU via Win32_PerfFormattedData (0-100, locale-independent); - RAM as used-physical percent. + CPU via Win32_PerfFormattedData (0-100, locale-independent); RAM as + used-physical percent; busiest process by cumulative CPU (for diagnostics). """ - click.echo( - f"[prepare-win] Idle check (settle {settle_s}s; thresholds " - f"CPU<={max_cpu}% RAM<={max_ram}%) ..." - ) ps = ( - f"Start-Sleep -Seconds {settle_s}\n" "$s = 0..4 | ForEach-Object { (Get-CimInstance " "Win32_PerfFormattedData_PerfOS_Processor -Filter \"Name='_Total'\")" ".PercentProcessorTime; Start-Sleep -Milliseconds 1000 }\n" @@ -402,31 +393,65 @@ def _check_guest_idle( "$os = Get-CimInstance Win32_OperatingSystem\n" "$ram = [math]::Round((1 - ($os.FreePhysicalMemory / " "$os.TotalVisibleMemorySize)) * 100, 1)\n" - "[pscustomobject]@{cpu=$cpu; ram=$ram} | ConvertTo-Json -Compress\n" + "$top = (Get-Process | Sort-Object CPU -Descending | " + "Select-Object -First 1).Name\n" + "[pscustomobject]@{cpu=$cpu; ram=$ram; top=$top} | ConvertTo-Json -Compress\n" ) res = transport.run(ps, check=False) lines = [ln for ln in res.stdout.splitlines() if ln.strip()] try: data = json.loads(lines[-1]) - cpu = float(data["cpu"]) - ram = float(data["ram"]) + return float(data["cpu"]), float(data["ram"]), str(data.get("top", "?")) except (json.JSONDecodeError, KeyError, ValueError, IndexError) as exc: raise click.ClickException( f"Idle check: could not parse guest metrics.\n{res.stdout}\n{res.stderr}" ) from exc - click.echo(f"[prepare-win] idle CPU {cpu}% RAM {ram}%") + + +def _check_guest_idle( + transport: WinRmTransport, + *, + max_cpu: float, + max_ram: float, + settle_s: int, + timeout_s: int, +) -> None: + """Wait for the guest to go idle, then proceed; abort if it never settles. + + Run just before the snapshot. Windows Update / CBS (TiWorker) keeps working + for minutes after the final reboot, so we poll until CPU and RAM are both + under threshold (up to ``timeout_s``) instead of sampling once — otherwise a + transiently-busy template would either be wrongly rejected or frozen busy + into every clone. The busiest process is reported to aid diagnosis. + """ + click.echo( + f"[prepare-win] Idle check (settle {settle_s}s, wait ≤{timeout_s}s; " + f"thresholds CPU≤{max_cpu}% RAM≤{max_ram}%) ..." + ) + if settle_s > 0: + time.sleep(settle_s) + deadline = time.monotonic() + timeout_s + cpu = ram = 0.0 + top = "?" + while True: + cpu, ram, top = _sample_guest_idle(transport) + click.echo(f"[prepare-win] CPU {cpu}% RAM {ram}% (busiest: {top})") + if cpu <= max_cpu and ram <= max_ram: + click.echo("[prepare-win] idle OK.") + return + if time.monotonic() >= deadline: + break + time.sleep(10) problems: list[str] = [] if cpu > max_cpu: problems.append(f"CPU {cpu}% > {max_cpu}%") if ram > max_ram: problems.append(f"RAM {ram}% > {max_ram}%") - if problems: - raise click.ClickException( - "Guest not idle before snapshot: " - + "; ".join(problems) - + ". Aborting — snapshot NOT taken; investigate the template." - ) - click.echo("[prepare-win] idle OK.") + raise click.ClickException( + f"Guest did not go idle within {timeout_s}s before snapshot: " + + "; ".join(problems) + + f" (busiest: {top}). Aborting — snapshot NOT taken; investigate the template." + ) def _run_with_heartbeat( @@ -517,6 +542,8 @@ def _run_with_heartbeat( help="Abort before snapshot if used RAM exceeds this percent.") @click.option("--idle-settle", default=30, show_default=True, help="Seconds to let the guest settle before sampling idle CPU/RAM.") +@click.option("--idle-timeout", default=600, show_default=True, + help="Max seconds to wait for the guest to go idle before aborting.") @click.option("--skip-idle-check", is_flag=True, default=False, help="Skip the pre-snapshot idle CPU/RAM check.") def template_prepare_win( @@ -540,6 +567,7 @@ def template_prepare_win( max_idle_cpu: float, max_idle_ram: float, idle_settle: int, + idle_timeout: int, skip_idle_check: bool, ) -> None: """Provision a Windows template VM via pypsrp (Linux-host compatible). @@ -830,6 +858,7 @@ Register-ScheduledTask -TaskName 'CI-StaticIp' ` max_cpu=max_idle_cpu, max_ram=max_idle_ram, settle_s=idle_settle, + timeout_s=idle_timeout, ) # Graceful shutdown diff --git a/tests/python/test_commands_template.py b/tests/python/test_commands_template.py index 6ee8d96..8ed93e7 100644 --- a/tests/python/test_commands_template.py +++ b/tests/python/test_commands_template.py @@ -433,7 +433,7 @@ def _make_transport() -> MagicMock: def _run(script: str = "", **kwargs: object) -> WinRmResult: if "PercentProcessorTime" in script: # pre-snapshot idle check - return WinRmResult(stdout='{"cpu":2.0,"ram":21.0}', stderr="", returncode=0) + return WinRmResult(stdout='{"cpu":2.0,"ram":21.0,"top":"idle"}', stderr="", returncode=0) if "ConvertTo-Json" in script: return WinRmResult(stdout=val_json, stderr="", returncode=0) return WinRmResult(stdout="CI_EXITCODE:0", stderr="", returncode=0) @@ -2220,26 +2220,32 @@ def test_clean_force_artifacts_noop_on_clean_dir(tmp_path: Path) -> None: def test_check_guest_idle_ok() -> None: transport = MagicMock() - transport.run.return_value = WinRmResult(stdout='{"cpu":3.0,"ram":22.5}', stderr="", returncode=0) - tmpl_mod._check_guest_idle(transport, max_cpu=10.0, max_ram=50.0, settle_s=0) # no raise + transport.run.return_value = WinRmResult( + stdout='{"cpu":3.0,"ram":22.5,"top":"idle"}', stderr="", returncode=0 + ) + tmpl_mod._check_guest_idle(transport, max_cpu=10.0, max_ram=50.0, settle_s=0, timeout_s=5) def test_check_guest_idle_cpu_over() -> None: transport = MagicMock() - transport.run.return_value = WinRmResult(stdout='{"cpu":42.0,"ram":20.0}', stderr="", returncode=0) + transport.run.return_value = WinRmResult( + stdout='{"cpu":42.0,"ram":20.0,"top":"TiWorker"}', stderr="", returncode=0 + ) with pytest.raises(click.ClickException, match=r"CPU 42\.0% > 10\.0%"): - tmpl_mod._check_guest_idle(transport, max_cpu=10.0, max_ram=50.0, settle_s=0) + tmpl_mod._check_guest_idle(transport, max_cpu=10.0, max_ram=50.0, settle_s=0, timeout_s=0) def test_check_guest_idle_ram_over() -> None: transport = MagicMock() - transport.run.return_value = WinRmResult(stdout='{"cpu":1.0,"ram":80.0}', stderr="", returncode=0) + transport.run.return_value = WinRmResult( + stdout='{"cpu":1.0,"ram":80.0,"top":"x"}', stderr="", returncode=0 + ) with pytest.raises(click.ClickException, match=r"RAM 80\.0% > 50\.0%"): - tmpl_mod._check_guest_idle(transport, max_cpu=10.0, max_ram=50.0, settle_s=0) + tmpl_mod._check_guest_idle(transport, max_cpu=10.0, max_ram=50.0, settle_s=0, timeout_s=0) def test_check_guest_idle_unparseable() -> None: transport = MagicMock() transport.run.return_value = WinRmResult(stdout="not json", stderr="", returncode=0) with pytest.raises(click.ClickException, match="could not parse"): - tmpl_mod._check_guest_idle(transport, max_cpu=10.0, max_ram=50.0, settle_s=0) + tmpl_mod._sample_guest_idle(transport)