feat(prepare-win): pre-snapshot idle CPU/RAM check (abort if busy)
After the final reboot and just before the graceful shutdown for the snapshot, sample the guest's idle CPU and used-RAM percentages (CIM Win32_PerfFormattedData — locale-independent — averaged over ~5s after a settle delay). If either exceeds its threshold, abort prepare without taking the snapshot, so a busy/leaky template isn't frozen into every clone. New options: --max-idle-cpu (10%), --max-idle-ram (50%), --idle-settle (30s), --skip-idle-check.
This commit is contained in:
@@ -360,6 +360,56 @@ def _wait_winrm(transport: WinRmTransport, timeout: int) -> None:
|
||||
raise click.ClickException(f"WinRM not ready within {timeout}s.")
|
||||
|
||||
|
||||
def _check_guest_idle(
|
||||
transport: WinRmTransport, *, max_cpu: float, max_ram: float, settle_s: int
|
||||
) -> None:
|
||||
"""Sample guest CPU/RAM after a settle period; abort if above thresholds.
|
||||
|
||||
Run just before the snapshot so a busy/leaky template (e.g. a service stuck
|
||||
spinning after the toolchain install) is caught instead of being frozen into
|
||||
every build clone. CPU via Win32_PerfFormattedData (0-100, locale-independent);
|
||||
RAM as used-physical percent.
|
||||
"""
|
||||
click.echo(
|
||||
f"[prepare-win] Idle check (settle {settle_s}s; thresholds "
|
||||
f"CPU<={max_cpu}% RAM<={max_ram}%) ..."
|
||||
)
|
||||
ps = (
|
||||
f"Start-Sleep -Seconds {settle_s}\n"
|
||||
"$s = 0..4 | ForEach-Object { (Get-CimInstance "
|
||||
"Win32_PerfFormattedData_PerfOS_Processor -Filter \"Name='_Total'\")"
|
||||
".PercentProcessorTime; Start-Sleep -Milliseconds 1000 }\n"
|
||||
"$cpu = [math]::Round((($s | Measure-Object -Average).Average), 1)\n"
|
||||
"$os = Get-CimInstance Win32_OperatingSystem\n"
|
||||
"$ram = [math]::Round((1 - ($os.FreePhysicalMemory / "
|
||||
"$os.TotalVisibleMemorySize)) * 100, 1)\n"
|
||||
"[pscustomobject]@{cpu=$cpu; ram=$ram} | ConvertTo-Json -Compress\n"
|
||||
)
|
||||
res = transport.run(ps, check=False)
|
||||
lines = [ln for ln in res.stdout.splitlines() if ln.strip()]
|
||||
try:
|
||||
data = json.loads(lines[-1])
|
||||
cpu = float(data["cpu"])
|
||||
ram = float(data["ram"])
|
||||
except (json.JSONDecodeError, KeyError, ValueError, IndexError) as exc:
|
||||
raise click.ClickException(
|
||||
f"Idle check: could not parse guest metrics.\n{res.stdout}\n{res.stderr}"
|
||||
) from exc
|
||||
click.echo(f"[prepare-win] idle CPU {cpu}% RAM {ram}%")
|
||||
problems: list[str] = []
|
||||
if cpu > max_cpu:
|
||||
problems.append(f"CPU {cpu}% > {max_cpu}%")
|
||||
if ram > max_ram:
|
||||
problems.append(f"RAM {ram}% > {max_ram}%")
|
||||
if problems:
|
||||
raise click.ClickException(
|
||||
"Guest not idle before snapshot: "
|
||||
+ "; ".join(problems)
|
||||
+ ". Aborting — snapshot NOT taken; investigate the template."
|
||||
)
|
||||
click.echo("[prepare-win] idle OK.")
|
||||
|
||||
|
||||
def _run_with_heartbeat(
|
||||
transport: WinRmTransport,
|
||||
script: str,
|
||||
@@ -442,6 +492,14 @@ def _run_with_heartbeat(
|
||||
@click.option("--gui/--nogui", "gui", default=False, show_default=True,
|
||||
help="Start the VM with the Workstation window (--gui, needs an X display) "
|
||||
"or headless (--nogui, the default).")
|
||||
@click.option("--max-idle-cpu", default=10.0, show_default=True,
|
||||
help="Abort before snapshot if idle CPU exceeds this percent.")
|
||||
@click.option("--max-idle-ram", default=50.0, show_default=True,
|
||||
help="Abort before snapshot if used RAM exceeds this percent.")
|
||||
@click.option("--idle-settle", default=30, show_default=True,
|
||||
help="Seconds to let the guest settle before sampling idle CPU/RAM.")
|
||||
@click.option("--skip-idle-check", is_flag=True, default=False,
|
||||
help="Skip the pre-snapshot idle CPU/RAM check.")
|
||||
def template_prepare_win(
|
||||
vmx: str,
|
||||
guest_ip: str,
|
||||
@@ -460,6 +518,10 @@ def template_prepare_win(
|
||||
credential_target: str,
|
||||
recreate_snapshot: bool,
|
||||
gui: bool,
|
||||
max_idle_cpu: float,
|
||||
max_idle_ram: float,
|
||||
idle_settle: int,
|
||||
skip_idle_check: bool,
|
||||
) -> None:
|
||||
"""Provision a Windows template VM via pypsrp (Linux-host compatible).
|
||||
|
||||
@@ -742,6 +804,15 @@ Register-ScheduledTask -TaskName 'CI-StaticIp' `
|
||||
"}\n"
|
||||
)
|
||||
|
||||
# Pre-snapshot idle check — abort (no snapshot) if the template is busy.
|
||||
if not skip_idle_check:
|
||||
_check_guest_idle(
|
||||
transport,
|
||||
max_cpu=max_idle_cpu,
|
||||
max_ram=max_idle_ram,
|
||||
settle_s=idle_settle,
|
||||
)
|
||||
|
||||
# Graceful shutdown
|
||||
click.echo("[prepare-win] Sending graceful shutdown ...")
|
||||
with contextlib.suppress(TransportError):
|
||||
|
||||
@@ -448,6 +448,8 @@ def _make_transport() -> MagicMock:
|
||||
)
|
||||
|
||||
def _run(script: str = "", **kwargs: object) -> WinRmResult:
|
||||
if "PercentProcessorTime" in script: # pre-snapshot idle check
|
||||
return WinRmResult(stdout='{"cpu":2.0,"ram":21.0}', stderr="", returncode=0)
|
||||
if "ConvertTo-Json" in script:
|
||||
return WinRmResult(stdout=val_json, stderr="", returncode=0)
|
||||
return WinRmResult(stdout="CI_EXITCODE:0", stderr="", returncode=0)
|
||||
@@ -530,6 +532,7 @@ def _patch_prepare(monkeypatch: pytest.MonkeyPatch, backend: MagicMock, transpor
|
||||
monkeypatch.setattr(tmpl_mod, "_wait_tcp", lambda *_: True)
|
||||
monkeypatch.setattr(tmpl_mod, "_wait_tcp_down", lambda *_a, **_k: True)
|
||||
monkeypatch.setattr(tmpl_mod, "_wait_winrm", lambda *_: None)
|
||||
monkeypatch.setattr(tmpl_mod, "_check_guest_idle", lambda *_a, **_k: None)
|
||||
monkeypatch.setattr(tmpl_mod.time, "sleep", lambda _: None)
|
||||
monkeypatch.setattr(tmpl_mod.subprocess, "run", lambda *a, **_: None)
|
||||
|
||||
@@ -2228,3 +2231,30 @@ def test_clean_force_artifacts_noop_on_clean_dir(tmp_path: Path) -> None:
|
||||
|
||||
assert removed == []
|
||||
assert (tmp_path / "LinuxBuild2404.vmdk").is_file()
|
||||
|
||||
|
||||
def test_check_guest_idle_ok() -> None:
|
||||
transport = MagicMock()
|
||||
transport.run.return_value = WinRmResult(stdout='{"cpu":3.0,"ram":22.5}', stderr="", returncode=0)
|
||||
tmpl_mod._check_guest_idle(transport, max_cpu=10.0, max_ram=50.0, settle_s=0) # no raise
|
||||
|
||||
|
||||
def test_check_guest_idle_cpu_over() -> None:
|
||||
transport = MagicMock()
|
||||
transport.run.return_value = WinRmResult(stdout='{"cpu":42.0,"ram":20.0}', stderr="", returncode=0)
|
||||
with pytest.raises(click.ClickException, match=r"CPU 42\.0% > 10\.0%"):
|
||||
tmpl_mod._check_guest_idle(transport, max_cpu=10.0, max_ram=50.0, settle_s=0)
|
||||
|
||||
|
||||
def test_check_guest_idle_ram_over() -> None:
|
||||
transport = MagicMock()
|
||||
transport.run.return_value = WinRmResult(stdout='{"cpu":1.0,"ram":80.0}', stderr="", returncode=0)
|
||||
with pytest.raises(click.ClickException, match=r"RAM 80\.0% > 50\.0%"):
|
||||
tmpl_mod._check_guest_idle(transport, max_cpu=10.0, max_ram=50.0, settle_s=0)
|
||||
|
||||
|
||||
def test_check_guest_idle_unparseable() -> None:
|
||||
transport = MagicMock()
|
||||
transport.run.return_value = WinRmResult(stdout="not json", stderr="", returncode=0)
|
||||
with pytest.raises(click.ClickException, match="could not parse"):
|
||||
tmpl_mod._check_guest_idle(transport, max_cpu=10.0, max_ram=50.0, settle_s=0)
|
||||
|
||||
Reference in New Issue
Block a user