From 5fe68589f5ec39b5c9997b02607b1bff027c4bbb Mon Sep 17 00:00:00 2001 From: Simone Date: Tue, 9 Jun 2026 21:11:21 +0200 Subject: [PATCH] feat(prepare-win): pre-snapshot idle CPU/RAM check (abort if busy) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit After the final reboot and just before the graceful shutdown for the snapshot, sample the guest's idle CPU and used-RAM percentages (CIM Win32_PerfFormattedData — locale-independent — averaged over ~5s after a settle delay). If either exceeds its threshold, abort prepare without taking the snapshot, so a busy/leaky template isn't frozen into every clone. New options: --max-idle-cpu (10%), --max-idle-ram (50%), --idle-settle (30s), --skip-idle-check. --- src/ci_orchestrator/commands/template.py | 71 ++++++++++++++++++++++++ tests/python/test_commands_template.py | 30 ++++++++++ 2 files changed, 101 insertions(+) diff --git a/src/ci_orchestrator/commands/template.py b/src/ci_orchestrator/commands/template.py index 9bd6f5d..2fe50ea 100644 --- a/src/ci_orchestrator/commands/template.py +++ b/src/ci_orchestrator/commands/template.py @@ -360,6 +360,56 @@ def _wait_winrm(transport: WinRmTransport, timeout: int) -> None: raise click.ClickException(f"WinRM not ready within {timeout}s.") +def _check_guest_idle( + transport: WinRmTransport, *, max_cpu: float, max_ram: float, settle_s: int +) -> None: + """Sample guest CPU/RAM after a settle period; abort if above thresholds. + + Run just before the snapshot so a busy/leaky template (e.g. a service stuck + spinning after the toolchain install) is caught instead of being frozen into + every build clone. CPU via Win32_PerfFormattedData (0-100, locale-independent); + RAM as used-physical percent. + """ + click.echo( + f"[prepare-win] Idle check (settle {settle_s}s; thresholds " + f"CPU<={max_cpu}% RAM<={max_ram}%) ..." + ) + ps = ( + f"Start-Sleep -Seconds {settle_s}\n" + "$s = 0..4 | ForEach-Object { (Get-CimInstance " + "Win32_PerfFormattedData_PerfOS_Processor -Filter \"Name='_Total'\")" + ".PercentProcessorTime; Start-Sleep -Milliseconds 1000 }\n" + "$cpu = [math]::Round((($s | Measure-Object -Average).Average), 1)\n" + "$os = Get-CimInstance Win32_OperatingSystem\n" + "$ram = [math]::Round((1 - ($os.FreePhysicalMemory / " + "$os.TotalVisibleMemorySize)) * 100, 1)\n" + "[pscustomobject]@{cpu=$cpu; ram=$ram} | ConvertTo-Json -Compress\n" + ) + res = transport.run(ps, check=False) + lines = [ln for ln in res.stdout.splitlines() if ln.strip()] + try: + data = json.loads(lines[-1]) + cpu = float(data["cpu"]) + ram = float(data["ram"]) + except (json.JSONDecodeError, KeyError, ValueError, IndexError) as exc: + raise click.ClickException( + f"Idle check: could not parse guest metrics.\n{res.stdout}\n{res.stderr}" + ) from exc + click.echo(f"[prepare-win] idle CPU {cpu}% RAM {ram}%") + problems: list[str] = [] + if cpu > max_cpu: + problems.append(f"CPU {cpu}% > {max_cpu}%") + if ram > max_ram: + problems.append(f"RAM {ram}% > {max_ram}%") + if problems: + raise click.ClickException( + "Guest not idle before snapshot: " + + "; ".join(problems) + + ". Aborting — snapshot NOT taken; investigate the template." + ) + click.echo("[prepare-win] idle OK.") + + def _run_with_heartbeat( transport: WinRmTransport, script: str, @@ -442,6 +492,14 @@ def _run_with_heartbeat( @click.option("--gui/--nogui", "gui", default=False, show_default=True, help="Start the VM with the Workstation window (--gui, needs an X display) " "or headless (--nogui, the default).") +@click.option("--max-idle-cpu", default=10.0, show_default=True, + help="Abort before snapshot if idle CPU exceeds this percent.") +@click.option("--max-idle-ram", default=50.0, show_default=True, + help="Abort before snapshot if used RAM exceeds this percent.") +@click.option("--idle-settle", default=30, show_default=True, + help="Seconds to let the guest settle before sampling idle CPU/RAM.") +@click.option("--skip-idle-check", is_flag=True, default=False, + help="Skip the pre-snapshot idle CPU/RAM check.") def template_prepare_win( vmx: str, guest_ip: str, @@ -460,6 +518,10 @@ def template_prepare_win( credential_target: str, recreate_snapshot: bool, gui: bool, + max_idle_cpu: float, + max_idle_ram: float, + idle_settle: int, + skip_idle_check: bool, ) -> None: """Provision a Windows template VM via pypsrp (Linux-host compatible). @@ -742,6 +804,15 @@ Register-ScheduledTask -TaskName 'CI-StaticIp' ` "}\n" ) + # Pre-snapshot idle check — abort (no snapshot) if the template is busy. + if not skip_idle_check: + _check_guest_idle( + transport, + max_cpu=max_idle_cpu, + max_ram=max_idle_ram, + settle_s=idle_settle, + ) + # Graceful shutdown click.echo("[prepare-win] Sending graceful shutdown ...") with contextlib.suppress(TransportError): diff --git a/tests/python/test_commands_template.py b/tests/python/test_commands_template.py index 19f84d5..e5d111f 100644 --- a/tests/python/test_commands_template.py +++ b/tests/python/test_commands_template.py @@ -448,6 +448,8 @@ def _make_transport() -> MagicMock: ) def _run(script: str = "", **kwargs: object) -> WinRmResult: + if "PercentProcessorTime" in script: # pre-snapshot idle check + return WinRmResult(stdout='{"cpu":2.0,"ram":21.0}', stderr="", returncode=0) if "ConvertTo-Json" in script: return WinRmResult(stdout=val_json, stderr="", returncode=0) return WinRmResult(stdout="CI_EXITCODE:0", stderr="", returncode=0) @@ -530,6 +532,7 @@ def _patch_prepare(monkeypatch: pytest.MonkeyPatch, backend: MagicMock, transpor monkeypatch.setattr(tmpl_mod, "_wait_tcp", lambda *_: True) monkeypatch.setattr(tmpl_mod, "_wait_tcp_down", lambda *_a, **_k: True) monkeypatch.setattr(tmpl_mod, "_wait_winrm", lambda *_: None) + monkeypatch.setattr(tmpl_mod, "_check_guest_idle", lambda *_a, **_k: None) monkeypatch.setattr(tmpl_mod.time, "sleep", lambda _: None) monkeypatch.setattr(tmpl_mod.subprocess, "run", lambda *a, **_: None) @@ -2228,3 +2231,30 @@ def test_clean_force_artifacts_noop_on_clean_dir(tmp_path: Path) -> None: assert removed == [] assert (tmp_path / "LinuxBuild2404.vmdk").is_file() + + +def test_check_guest_idle_ok() -> None: + transport = MagicMock() + transport.run.return_value = WinRmResult(stdout='{"cpu":3.0,"ram":22.5}', stderr="", returncode=0) + tmpl_mod._check_guest_idle(transport, max_cpu=10.0, max_ram=50.0, settle_s=0) # no raise + + +def test_check_guest_idle_cpu_over() -> None: + transport = MagicMock() + transport.run.return_value = WinRmResult(stdout='{"cpu":42.0,"ram":20.0}', stderr="", returncode=0) + with pytest.raises(click.ClickException, match=r"CPU 42\.0% > 10\.0%"): + tmpl_mod._check_guest_idle(transport, max_cpu=10.0, max_ram=50.0, settle_s=0) + + +def test_check_guest_idle_ram_over() -> None: + transport = MagicMock() + transport.run.return_value = WinRmResult(stdout='{"cpu":1.0,"ram":80.0}', stderr="", returncode=0) + with pytest.raises(click.ClickException, match=r"RAM 80\.0% > 50\.0%"): + tmpl_mod._check_guest_idle(transport, max_cpu=10.0, max_ram=50.0, settle_s=0) + + +def test_check_guest_idle_unparseable() -> None: + transport = MagicMock() + transport.run.return_value = WinRmResult(stdout="not json", stderr="", returncode=0) + with pytest.raises(click.ClickException, match="could not parse"): + tmpl_mod._check_guest_idle(transport, max_cpu=10.0, max_ram=50.0, settle_s=0)