feat(a2): port leaf PS scripts to ci_orchestrator CLI
Implements Phase A2 of plans/implementation-plan-A-B.md: - commands/wait.py -> wait-ready (replaces Wait-VMReady.ps1) - commands/vm.py -> vm remove + vm cleanup (replaces Remove-BuildVM.ps1, Cleanup-OrphanedBuildVMs.ps1); cleanup accepts injected VmBackend (Phase C ESXi hook preserved) - commands/monitor.py -> monitor disk + monitor runner (replaces Watch-DiskSpace.ps1, Watch-RunnerHealth.ps1) - commands/report.py -> report job (replaces Get-CIJobSummary.ps1) Each PS script reduced to a 3-line shim that delegates to the Python CLI and preserves \0. Tests: 69 pytest cases across new test_commands_*.py modules; ruff clean, mypy --strict clean, coverage 74.5% (>=70 gate).
This commit is contained in:
@@ -0,0 +1,357 @@
|
||||
"""``vm`` sub-commands.
|
||||
|
||||
Phase A2 ports the leaf VM management scripts:
|
||||
|
||||
* ``vm remove`` — replaces ``scripts/Remove-BuildVM.ps1``
|
||||
* ``vm cleanup`` — replaces ``scripts/Cleanup-OrphanedBuildVMs.ps1``
|
||||
|
||||
Phase C hook: ``vm cleanup`` accepts an optional :class:`VmBackend`
|
||||
dependency so an ESXi backend can later supply a folder-scoped VM list
|
||||
instead of scanning the local filesystem.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import shutil
|
||||
import time
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import click
|
||||
|
||||
from ci_orchestrator.backends import load_backend
|
||||
from ci_orchestrator.backends.errors import BackendError, BackendNotAvailable
|
||||
from ci_orchestrator.backends.protocol import VmHandle
|
||||
from ci_orchestrator.config import load_config
|
||||
|
||||
if TYPE_CHECKING: # pragma: no cover
|
||||
from ci_orchestrator.backends.protocol import VmBackend
|
||||
|
||||
|
||||
# ────────────────────────────────────────────────────────────────────── helpers
|
||||
|
||||
|
||||
def _make_backend(vmrun_path: str | None) -> VmBackend:
|
||||
"""Build a backend honouring an optional vmrun override."""
|
||||
config = load_config()
|
||||
if vmrun_path is not None:
|
||||
from ci_orchestrator.backends.workstation import WorkstationVmrunBackend
|
||||
|
||||
return WorkstationVmrunBackend(vmrun_path=vmrun_path)
|
||||
return load_backend(config)
|
||||
|
||||
|
||||
def _try_remove_dir(path: Path) -> bool:
|
||||
"""Best-effort recursive directory removal. Returns True if path is gone."""
|
||||
if not path.exists():
|
||||
return True
|
||||
shutil.rmtree(path, ignore_errors=True)
|
||||
return not path.exists()
|
||||
|
||||
|
||||
def _stop_with_fallback(backend: VmBackend, handle: VmHandle) -> None:
|
||||
"""Soft-stop, then hard-stop on failure. Both errors are tolerated."""
|
||||
try:
|
||||
backend.stop(handle, hard=False)
|
||||
except BackendError as exc:
|
||||
click.echo(f"[vm remove] soft stop failed: {exc}", err=True)
|
||||
try:
|
||||
if backend.is_running(handle):
|
||||
click.echo("[vm remove] still running, forcing hard stop...")
|
||||
backend.stop(handle, hard=True)
|
||||
except BackendError as exc:
|
||||
click.echo(f"[vm remove] hard stop failed: {exc}", err=True)
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────────── group
|
||||
|
||||
|
||||
@click.group("vm")
|
||||
def vm() -> None:
|
||||
"""VM lifecycle commands."""
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────────── remove
|
||||
|
||||
|
||||
@vm.command("remove")
|
||||
@click.option(
|
||||
"--vmx",
|
||||
"--vm-path",
|
||||
"vmx",
|
||||
required=True,
|
||||
help="Path to the clone VMX file to destroy.",
|
||||
)
|
||||
@click.option("--vmrun-path", "vmrun_path", default=None, help="Override vmrun.exe path.")
|
||||
@click.option(
|
||||
"--graceful-timeout",
|
||||
"--graceful-timeout-seconds",
|
||||
"graceful_timeout",
|
||||
type=click.IntRange(1, 60),
|
||||
default=15,
|
||||
show_default=True,
|
||||
help="Seconds to wait for soft stop before forcing.",
|
||||
)
|
||||
@click.option(
|
||||
"--force",
|
||||
is_flag=True,
|
||||
default=False,
|
||||
help="Skip soft stop and go straight to hard stop.",
|
||||
)
|
||||
def vm_remove(
|
||||
vmx: str,
|
||||
vmrun_path: str | None,
|
||||
graceful_timeout: int,
|
||||
force: bool,
|
||||
) -> None:
|
||||
"""Stop and permanently delete an ephemeral build VM."""
|
||||
vmx_path = Path(vmx)
|
||||
clone_dir = vmx_path.parent
|
||||
click.echo(f"[vm remove] destroying VM: {vmx}")
|
||||
|
||||
if not vmx_path.is_file():
|
||||
click.echo(
|
||||
f"[vm remove] VMX file not found - nothing to remove: {vmx}",
|
||||
err=True,
|
||||
)
|
||||
# Still try to clean up a partial clone directory.
|
||||
if clone_dir.exists() and clone_dir != vmx_path:
|
||||
_try_remove_dir(clone_dir)
|
||||
return
|
||||
|
||||
try:
|
||||
backend = _make_backend(vmrun_path)
|
||||
except BackendNotAvailable as exc:
|
||||
click.echo(
|
||||
f"[vm remove] vmrun unavailable ({exc}); attempting directory removal only.",
|
||||
err=True,
|
||||
)
|
||||
_try_remove_dir(clone_dir)
|
||||
return
|
||||
|
||||
handle = VmHandle(identifier=vmx)
|
||||
|
||||
if force:
|
||||
try:
|
||||
backend.stop(handle, hard=True)
|
||||
except BackendError as exc:
|
||||
click.echo(f"[vm remove] hard stop failed: {exc}", err=True)
|
||||
else:
|
||||
click.echo(f"[vm remove] sending soft stop (timeout: {graceful_timeout}s)...")
|
||||
_stop_with_fallback(backend, handle)
|
||||
|
||||
# Brief settle: VMware may keep file locks for a moment after stop.
|
||||
deadline = time.monotonic() + 20.0
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
if not backend.is_running(handle):
|
||||
break
|
||||
except BackendError:
|
||||
break
|
||||
time.sleep(2.0)
|
||||
|
||||
click.echo("[vm remove] deleting VM...")
|
||||
deleted = False
|
||||
last_err: str = ""
|
||||
for delay in (0, 3, 6):
|
||||
if delay > 0:
|
||||
click.echo(f"[vm remove] retrying deleteVM in {delay}s...")
|
||||
time.sleep(delay)
|
||||
try:
|
||||
backend.delete(handle)
|
||||
deleted = True
|
||||
break
|
||||
except BackendError as exc:
|
||||
last_err = str(exc)
|
||||
|
||||
if not deleted:
|
||||
click.echo(
|
||||
f"[vm remove] vmrun deleteVM failed after retries: {last_err}", err=True
|
||||
)
|
||||
click.echo("[vm remove] falling back to manual directory removal.")
|
||||
|
||||
if clone_dir.exists():
|
||||
click.echo(f"[vm remove] removing clone directory: {clone_dir}")
|
||||
if _try_remove_dir(clone_dir):
|
||||
click.echo("[vm remove] clone directory removed.")
|
||||
else:
|
||||
click.echo(
|
||||
f"[vm remove] could not fully remove: {clone_dir} - manual cleanup needed.",
|
||||
err=True,
|
||||
)
|
||||
|
||||
click.echo("[vm remove] VM destruction complete.")
|
||||
|
||||
|
||||
# ────────────────────────────────────────────────────────────────────── cleanup
|
||||
|
||||
|
||||
def _list_orphans(clone_base: Path, max_age_hours: int) -> list[Path]:
|
||||
"""Return orphan clone directories older than ``max_age_hours``."""
|
||||
if not clone_base.is_dir():
|
||||
return []
|
||||
cutoff = datetime.now(tz=UTC) - timedelta(hours=max_age_hours)
|
||||
out: list[Path] = []
|
||||
for entry in clone_base.iterdir():
|
||||
if not entry.is_dir():
|
||||
continue
|
||||
try:
|
||||
mtime = datetime.fromtimestamp(entry.stat().st_mtime, tz=UTC)
|
||||
except OSError:
|
||||
continue
|
||||
if mtime < cutoff:
|
||||
out.append(entry)
|
||||
return out
|
||||
|
||||
|
||||
def _find_vmx(clone_dir: Path) -> Path | None:
|
||||
for entry in clone_dir.iterdir():
|
||||
if entry.is_file() and entry.suffix.lower() == ".vmx":
|
||||
return entry
|
||||
return None
|
||||
|
||||
|
||||
def cleanup_orphans(
|
||||
clone_base: Path,
|
||||
max_age_hours: int,
|
||||
backend: VmBackend | None,
|
||||
*,
|
||||
dry_run: bool = False,
|
||||
lock_file: Path | None = None,
|
||||
lock_max_age_minutes: int = 30,
|
||||
) -> int:
|
||||
"""Remove orphaned clone directories. Returns count removed.
|
||||
|
||||
Phase C hook: ``backend`` is used to stop+delete each orphan. If the
|
||||
backend is unavailable, directory removal is still attempted.
|
||||
"""
|
||||
orphans = _list_orphans(clone_base, max_age_hours)
|
||||
if not orphans:
|
||||
click.echo(
|
||||
f"[vm cleanup] no orphaned VMs found (threshold: {max_age_hours} h)."
|
||||
)
|
||||
else:
|
||||
click.echo(
|
||||
f"[vm cleanup] found {len(orphans)} orphaned director"
|
||||
f"{'y' if len(orphans) == 1 else 'ies'} older than {max_age_hours} h."
|
||||
)
|
||||
|
||||
removed = 0
|
||||
now = datetime.now(tz=UTC)
|
||||
for entry in orphans:
|
||||
age_h = int((now - datetime.fromtimestamp(entry.stat().st_mtime, tz=UTC)).total_seconds() // 3600)
|
||||
if dry_run:
|
||||
click.echo(f"[vm cleanup] would destroy {entry} (age: {age_h} h)")
|
||||
continue
|
||||
click.echo(f"[vm cleanup] processing: {entry}")
|
||||
vmx = _find_vmx(entry)
|
||||
if vmx is not None and backend is not None:
|
||||
handle = VmHandle(identifier=str(vmx))
|
||||
try:
|
||||
backend.stop(handle, hard=True)
|
||||
except BackendError as exc:
|
||||
click.echo(f"[vm cleanup] hard stop ignored: {exc}", err=True)
|
||||
time.sleep(2.0)
|
||||
try:
|
||||
backend.delete(handle)
|
||||
except BackendError as exc:
|
||||
click.echo(
|
||||
f"[vm cleanup] vmrun deleteVM failed: {exc}; falling back to dir removal.",
|
||||
err=True,
|
||||
)
|
||||
elif vmx is None:
|
||||
click.echo(
|
||||
f"[vm cleanup] no .vmx in {entry} - removing directory only.", err=True
|
||||
)
|
||||
|
||||
if _try_remove_dir(entry):
|
||||
click.echo(f"[vm cleanup] removed: {entry}")
|
||||
removed += 1
|
||||
else:
|
||||
click.echo(
|
||||
f"[vm cleanup] could not fully remove: {entry} - manual cleanup needed.",
|
||||
err=True,
|
||||
)
|
||||
|
||||
# Stale lock file cleanup (mirrors Cleanup-OrphanedBuildVMs.ps1).
|
||||
if lock_file is not None and lock_file.is_file():
|
||||
cutoff = datetime.now(tz=UTC) - timedelta(minutes=lock_max_age_minutes)
|
||||
mtime = datetime.fromtimestamp(lock_file.stat().st_mtime, tz=UTC)
|
||||
if mtime < cutoff and not dry_run:
|
||||
age_min = int((datetime.now(tz=UTC) - mtime).total_seconds() // 60)
|
||||
try:
|
||||
lock_file.unlink()
|
||||
click.echo(
|
||||
f"[vm cleanup] removed stale lock file (age: {age_min} min): {lock_file}"
|
||||
)
|
||||
except OSError as exc:
|
||||
click.echo(f"[vm cleanup] could not remove lock file: {exc}", err=True)
|
||||
|
||||
click.echo("[vm cleanup] done.")
|
||||
return removed
|
||||
|
||||
|
||||
@vm.command("cleanup")
|
||||
@click.option(
|
||||
"--clone-base-dir",
|
||||
"--clone-base",
|
||||
"clone_base_dir",
|
||||
default=None,
|
||||
help="Directory containing ephemeral VM clones (defaults to config.paths.build_vms).",
|
||||
)
|
||||
@click.option(
|
||||
"--max-age-hours",
|
||||
type=click.IntRange(0, 168),
|
||||
default=4,
|
||||
show_default=True,
|
||||
)
|
||||
@click.option("--vmrun-path", "vmrun_path", default=None, help="Override vmrun.exe path.")
|
||||
@click.option(
|
||||
"--lock-file",
|
||||
default=None,
|
||||
help="Path to a stale vm-start lock file to remove if older than 30 minutes.",
|
||||
)
|
||||
@click.option(
|
||||
"--what-if",
|
||||
"--whatif",
|
||||
"what_if",
|
||||
is_flag=True,
|
||||
default=False,
|
||||
help="List orphans without destroying them.",
|
||||
)
|
||||
def vm_cleanup(
|
||||
clone_base_dir: str | None,
|
||||
max_age_hours: int,
|
||||
vmrun_path: str | None,
|
||||
lock_file: str | None,
|
||||
what_if: bool,
|
||||
) -> None:
|
||||
"""Destroy ephemeral build VMs left behind after a crash or timeout."""
|
||||
config = load_config()
|
||||
base_path = Path(clone_base_dir) if clone_base_dir else config.paths.build_vms
|
||||
if not base_path.exists():
|
||||
click.echo(
|
||||
f"[vm cleanup] clone base dir not found: {base_path} - nothing to do."
|
||||
)
|
||||
return
|
||||
|
||||
try:
|
||||
backend: VmBackend | None = _make_backend(vmrun_path)
|
||||
except BackendNotAvailable as exc:
|
||||
click.echo(
|
||||
f"[vm cleanup] vmrun not available ({exc}); will attempt directory removal only.",
|
||||
err=True,
|
||||
)
|
||||
backend = None
|
||||
|
||||
cleanup_orphans(
|
||||
clone_base=base_path,
|
||||
max_age_hours=max_age_hours,
|
||||
backend=backend,
|
||||
dry_run=what_if,
|
||||
lock_file=Path(lock_file) if lock_file else None,
|
||||
)
|
||||
|
||||
|
||||
__all__ = ["cleanup_orphans", "vm"]
|
||||
Reference in New Issue
Block a user