feat(prepare-win): pre-snapshot idle CPU/RAM check (abort if busy)
After the final reboot and just before the graceful shutdown for the snapshot, sample the guest's idle CPU and used-RAM percentages (CIM Win32_PerfFormattedData — locale-independent — averaged over ~5s after a settle delay). If either exceeds its threshold, abort prepare without taking the snapshot, so a busy/leaky template isn't frozen into every clone. New options: --max-idle-cpu (10%), --max-idle-ram (50%), --idle-settle (30s), --skip-idle-check.
This commit is contained in:
@@ -360,6 +360,56 @@ def _wait_winrm(transport: WinRmTransport, timeout: int) -> None:
|
||||
raise click.ClickException(f"WinRM not ready within {timeout}s.")
|
||||
|
||||
|
||||
def _check_guest_idle(
|
||||
transport: WinRmTransport, *, max_cpu: float, max_ram: float, settle_s: int
|
||||
) -> None:
|
||||
"""Sample guest CPU/RAM after a settle period; abort if above thresholds.
|
||||
|
||||
Run just before the snapshot so a busy/leaky template (e.g. a service stuck
|
||||
spinning after the toolchain install) is caught instead of being frozen into
|
||||
every build clone. CPU via Win32_PerfFormattedData (0-100, locale-independent);
|
||||
RAM as used-physical percent.
|
||||
"""
|
||||
click.echo(
|
||||
f"[prepare-win] Idle check (settle {settle_s}s; thresholds "
|
||||
f"CPU<={max_cpu}% RAM<={max_ram}%) ..."
|
||||
)
|
||||
ps = (
|
||||
f"Start-Sleep -Seconds {settle_s}\n"
|
||||
"$s = 0..4 | ForEach-Object { (Get-CimInstance "
|
||||
"Win32_PerfFormattedData_PerfOS_Processor -Filter \"Name='_Total'\")"
|
||||
".PercentProcessorTime; Start-Sleep -Milliseconds 1000 }\n"
|
||||
"$cpu = [math]::Round((($s | Measure-Object -Average).Average), 1)\n"
|
||||
"$os = Get-CimInstance Win32_OperatingSystem\n"
|
||||
"$ram = [math]::Round((1 - ($os.FreePhysicalMemory / "
|
||||
"$os.TotalVisibleMemorySize)) * 100, 1)\n"
|
||||
"[pscustomobject]@{cpu=$cpu; ram=$ram} | ConvertTo-Json -Compress\n"
|
||||
)
|
||||
res = transport.run(ps, check=False)
|
||||
lines = [ln for ln in res.stdout.splitlines() if ln.strip()]
|
||||
try:
|
||||
data = json.loads(lines[-1])
|
||||
cpu = float(data["cpu"])
|
||||
ram = float(data["ram"])
|
||||
except (json.JSONDecodeError, KeyError, ValueError, IndexError) as exc:
|
||||
raise click.ClickException(
|
||||
f"Idle check: could not parse guest metrics.\n{res.stdout}\n{res.stderr}"
|
||||
) from exc
|
||||
click.echo(f"[prepare-win] idle CPU {cpu}% RAM {ram}%")
|
||||
problems: list[str] = []
|
||||
if cpu > max_cpu:
|
||||
problems.append(f"CPU {cpu}% > {max_cpu}%")
|
||||
if ram > max_ram:
|
||||
problems.append(f"RAM {ram}% > {max_ram}%")
|
||||
if problems:
|
||||
raise click.ClickException(
|
||||
"Guest not idle before snapshot: "
|
||||
+ "; ".join(problems)
|
||||
+ ". Aborting — snapshot NOT taken; investigate the template."
|
||||
)
|
||||
click.echo("[prepare-win] idle OK.")
|
||||
|
||||
|
||||
def _run_with_heartbeat(
|
||||
transport: WinRmTransport,
|
||||
script: str,
|
||||
@@ -442,6 +492,14 @@ def _run_with_heartbeat(
|
||||
@click.option("--gui/--nogui", "gui", default=False, show_default=True,
|
||||
help="Start the VM with the Workstation window (--gui, needs an X display) "
|
||||
"or headless (--nogui, the default).")
|
||||
@click.option("--max-idle-cpu", default=10.0, show_default=True,
|
||||
help="Abort before snapshot if idle CPU exceeds this percent.")
|
||||
@click.option("--max-idle-ram", default=50.0, show_default=True,
|
||||
help="Abort before snapshot if used RAM exceeds this percent.")
|
||||
@click.option("--idle-settle", default=30, show_default=True,
|
||||
help="Seconds to let the guest settle before sampling idle CPU/RAM.")
|
||||
@click.option("--skip-idle-check", is_flag=True, default=False,
|
||||
help="Skip the pre-snapshot idle CPU/RAM check.")
|
||||
def template_prepare_win(
|
||||
vmx: str,
|
||||
guest_ip: str,
|
||||
@@ -460,6 +518,10 @@ def template_prepare_win(
|
||||
credential_target: str,
|
||||
recreate_snapshot: bool,
|
||||
gui: bool,
|
||||
max_idle_cpu: float,
|
||||
max_idle_ram: float,
|
||||
idle_settle: int,
|
||||
skip_idle_check: bool,
|
||||
) -> None:
|
||||
"""Provision a Windows template VM via pypsrp (Linux-host compatible).
|
||||
|
||||
@@ -742,6 +804,15 @@ Register-ScheduledTask -TaskName 'CI-StaticIp' `
|
||||
"}\n"
|
||||
)
|
||||
|
||||
# Pre-snapshot idle check — abort (no snapshot) if the template is busy.
|
||||
if not skip_idle_check:
|
||||
_check_guest_idle(
|
||||
transport,
|
||||
max_cpu=max_idle_cpu,
|
||||
max_ram=max_idle_ram,
|
||||
settle_s=idle_settle,
|
||||
)
|
||||
|
||||
# Graceful shutdown
|
||||
click.echo("[prepare-win] Sending graceful shutdown ...")
|
||||
with contextlib.suppress(TransportError):
|
||||
|
||||
Reference in New Issue
Block a user