feat(prepare-win): pre-snapshot idle CPU/RAM check (abort if busy)

After the final reboot and just before the graceful shutdown for the
snapshot, sample the guest's idle CPU and used-RAM percentages (CIM
Win32_PerfFormattedData — locale-independent — averaged over ~5s after a
settle delay). If either exceeds its threshold, abort prepare without
taking the snapshot, so a busy/leaky template isn't frozen into every clone.

New options: --max-idle-cpu (10%), --max-idle-ram (50%), --idle-settle
(30s), --skip-idle-check.
This commit is contained in:
2026-06-09 21:11:21 +02:00
parent de0f5e6c68
commit 5fe68589f5
2 changed files with 101 additions and 0 deletions
+71
View File
@@ -360,6 +360,56 @@ def _wait_winrm(transport: WinRmTransport, timeout: int) -> None:
raise click.ClickException(f"WinRM not ready within {timeout}s.")
def _check_guest_idle(
transport: WinRmTransport, *, max_cpu: float, max_ram: float, settle_s: int
) -> None:
"""Sample guest CPU/RAM after a settle period; abort if above thresholds.
Run just before the snapshot so a busy/leaky template (e.g. a service stuck
spinning after the toolchain install) is caught instead of being frozen into
every build clone. CPU via Win32_PerfFormattedData (0-100, locale-independent);
RAM as used-physical percent.
"""
click.echo(
f"[prepare-win] Idle check (settle {settle_s}s; thresholds "
f"CPU<={max_cpu}% RAM<={max_ram}%) ..."
)
ps = (
f"Start-Sleep -Seconds {settle_s}\n"
"$s = 0..4 | ForEach-Object { (Get-CimInstance "
"Win32_PerfFormattedData_PerfOS_Processor -Filter \"Name='_Total'\")"
".PercentProcessorTime; Start-Sleep -Milliseconds 1000 }\n"
"$cpu = [math]::Round((($s | Measure-Object -Average).Average), 1)\n"
"$os = Get-CimInstance Win32_OperatingSystem\n"
"$ram = [math]::Round((1 - ($os.FreePhysicalMemory / "
"$os.TotalVisibleMemorySize)) * 100, 1)\n"
"[pscustomobject]@{cpu=$cpu; ram=$ram} | ConvertTo-Json -Compress\n"
)
res = transport.run(ps, check=False)
lines = [ln for ln in res.stdout.splitlines() if ln.strip()]
try:
data = json.loads(lines[-1])
cpu = float(data["cpu"])
ram = float(data["ram"])
except (json.JSONDecodeError, KeyError, ValueError, IndexError) as exc:
raise click.ClickException(
f"Idle check: could not parse guest metrics.\n{res.stdout}\n{res.stderr}"
) from exc
click.echo(f"[prepare-win] idle CPU {cpu}% RAM {ram}%")
problems: list[str] = []
if cpu > max_cpu:
problems.append(f"CPU {cpu}% > {max_cpu}%")
if ram > max_ram:
problems.append(f"RAM {ram}% > {max_ram}%")
if problems:
raise click.ClickException(
"Guest not idle before snapshot: "
+ "; ".join(problems)
+ ". Aborting — snapshot NOT taken; investigate the template."
)
click.echo("[prepare-win] idle OK.")
def _run_with_heartbeat(
transport: WinRmTransport,
script: str,
@@ -442,6 +492,14 @@ def _run_with_heartbeat(
@click.option("--gui/--nogui", "gui", default=False, show_default=True,
help="Start the VM with the Workstation window (--gui, needs an X display) "
"or headless (--nogui, the default).")
@click.option("--max-idle-cpu", default=10.0, show_default=True,
help="Abort before snapshot if idle CPU exceeds this percent.")
@click.option("--max-idle-ram", default=50.0, show_default=True,
help="Abort before snapshot if used RAM exceeds this percent.")
@click.option("--idle-settle", default=30, show_default=True,
help="Seconds to let the guest settle before sampling idle CPU/RAM.")
@click.option("--skip-idle-check", is_flag=True, default=False,
help="Skip the pre-snapshot idle CPU/RAM check.")
def template_prepare_win(
vmx: str,
guest_ip: str,
@@ -460,6 +518,10 @@ def template_prepare_win(
credential_target: str,
recreate_snapshot: bool,
gui: bool,
max_idle_cpu: float,
max_idle_ram: float,
idle_settle: int,
skip_idle_check: bool,
) -> None:
"""Provision a Windows template VM via pypsrp (Linux-host compatible).
@@ -742,6 +804,15 @@ Register-ScheduledTask -TaskName 'CI-StaticIp' `
"}\n"
)
# Pre-snapshot idle check — abort (no snapshot) if the template is busy.
if not skip_idle_check:
_check_guest_idle(
transport,
max_cpu=max_idle_cpu,
max_ram=max_idle_ram,
settle_s=idle_settle,
)
# Graceful shutdown
click.echo("[prepare-win] Sending graceful shutdown ...")
with contextlib.suppress(TransportError):
+30
View File
@@ -448,6 +448,8 @@ def _make_transport() -> MagicMock:
)
def _run(script: str = "", **kwargs: object) -> WinRmResult:
if "PercentProcessorTime" in script: # pre-snapshot idle check
return WinRmResult(stdout='{"cpu":2.0,"ram":21.0}', stderr="", returncode=0)
if "ConvertTo-Json" in script:
return WinRmResult(stdout=val_json, stderr="", returncode=0)
return WinRmResult(stdout="CI_EXITCODE:0", stderr="", returncode=0)
@@ -530,6 +532,7 @@ def _patch_prepare(monkeypatch: pytest.MonkeyPatch, backend: MagicMock, transpor
monkeypatch.setattr(tmpl_mod, "_wait_tcp", lambda *_: True)
monkeypatch.setattr(tmpl_mod, "_wait_tcp_down", lambda *_a, **_k: True)
monkeypatch.setattr(tmpl_mod, "_wait_winrm", lambda *_: None)
monkeypatch.setattr(tmpl_mod, "_check_guest_idle", lambda *_a, **_k: None)
monkeypatch.setattr(tmpl_mod.time, "sleep", lambda _: None)
monkeypatch.setattr(tmpl_mod.subprocess, "run", lambda *a, **_: None)
@@ -2228,3 +2231,30 @@ def test_clean_force_artifacts_noop_on_clean_dir(tmp_path: Path) -> None:
assert removed == []
assert (tmp_path / "LinuxBuild2404.vmdk").is_file()
def test_check_guest_idle_ok() -> None:
transport = MagicMock()
transport.run.return_value = WinRmResult(stdout='{"cpu":3.0,"ram":22.5}', stderr="", returncode=0)
tmpl_mod._check_guest_idle(transport, max_cpu=10.0, max_ram=50.0, settle_s=0) # no raise
def test_check_guest_idle_cpu_over() -> None:
transport = MagicMock()
transport.run.return_value = WinRmResult(stdout='{"cpu":42.0,"ram":20.0}', stderr="", returncode=0)
with pytest.raises(click.ClickException, match=r"CPU 42\.0% > 10\.0%"):
tmpl_mod._check_guest_idle(transport, max_cpu=10.0, max_ram=50.0, settle_s=0)
def test_check_guest_idle_ram_over() -> None:
transport = MagicMock()
transport.run.return_value = WinRmResult(stdout='{"cpu":1.0,"ram":80.0}', stderr="", returncode=0)
with pytest.raises(click.ClickException, match=r"RAM 80\.0% > 50\.0%"):
tmpl_mod._check_guest_idle(transport, max_cpu=10.0, max_ram=50.0, settle_s=0)
def test_check_guest_idle_unparseable() -> None:
transport = MagicMock()
transport.run.return_value = WinRmResult(stdout="not json", stderr="", returncode=0)
with pytest.raises(click.ClickException, match="could not parse"):
tmpl_mod._check_guest_idle(transport, max_cpu=10.0, max_ram=50.0, settle_s=0)