Simone
80f6661ad5
Implements Phase A2 of plans/implementation-plan-A-B.md: - commands/wait.py -> wait-ready (replaces Wait-VMReady.ps1) - commands/vm.py -> vm remove + vm cleanup (replaces Remove-BuildVM.ps1, Cleanup-OrphanedBuildVMs.ps1); cleanup accepts injected VmBackend (Phase C ESXi hook preserved) - commands/monitor.py -> monitor disk + monitor runner (replaces Watch-DiskSpace.ps1, Watch-RunnerHealth.ps1) - commands/report.py -> report job (replaces Get-CIJobSummary.ps1) Each PS script reduced to a 3-line shim that delegates to the Python CLI and preserves \0. Tests: 69 pytest cases across new test_commands_*.py modules; ruff clean, mypy --strict clean, coverage 74.5% (>=70 gate).
358 lines
12 KiB
Python
358 lines
12 KiB
Python
"""``vm`` sub-commands.
|
|
|
|
Phase A2 ports the leaf VM management scripts:
|
|
|
|
* ``vm remove`` — replaces ``scripts/Remove-BuildVM.ps1``
|
|
* ``vm cleanup`` — replaces ``scripts/Cleanup-OrphanedBuildVMs.ps1``
|
|
|
|
Phase C hook: ``vm cleanup`` accepts an optional :class:`VmBackend`
|
|
dependency so an ESXi backend can later supply a folder-scoped VM list
|
|
instead of scanning the local filesystem.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import shutil
|
|
import time
|
|
from datetime import UTC, datetime, timedelta
|
|
from pathlib import Path
|
|
from typing import TYPE_CHECKING
|
|
|
|
import click
|
|
|
|
from ci_orchestrator.backends import load_backend
|
|
from ci_orchestrator.backends.errors import BackendError, BackendNotAvailable
|
|
from ci_orchestrator.backends.protocol import VmHandle
|
|
from ci_orchestrator.config import load_config
|
|
|
|
if TYPE_CHECKING: # pragma: no cover
|
|
from ci_orchestrator.backends.protocol import VmBackend
|
|
|
|
|
|
# ────────────────────────────────────────────────────────────────────── helpers
|
|
|
|
|
|
def _make_backend(vmrun_path: str | None) -> VmBackend:
|
|
"""Build a backend honouring an optional vmrun override."""
|
|
config = load_config()
|
|
if vmrun_path is not None:
|
|
from ci_orchestrator.backends.workstation import WorkstationVmrunBackend
|
|
|
|
return WorkstationVmrunBackend(vmrun_path=vmrun_path)
|
|
return load_backend(config)
|
|
|
|
|
|
def _try_remove_dir(path: Path) -> bool:
|
|
"""Best-effort recursive directory removal. Returns True if path is gone."""
|
|
if not path.exists():
|
|
return True
|
|
shutil.rmtree(path, ignore_errors=True)
|
|
return not path.exists()
|
|
|
|
|
|
def _stop_with_fallback(backend: VmBackend, handle: VmHandle) -> None:
|
|
"""Soft-stop, then hard-stop on failure. Both errors are tolerated."""
|
|
try:
|
|
backend.stop(handle, hard=False)
|
|
except BackendError as exc:
|
|
click.echo(f"[vm remove] soft stop failed: {exc}", err=True)
|
|
try:
|
|
if backend.is_running(handle):
|
|
click.echo("[vm remove] still running, forcing hard stop...")
|
|
backend.stop(handle, hard=True)
|
|
except BackendError as exc:
|
|
click.echo(f"[vm remove] hard stop failed: {exc}", err=True)
|
|
|
|
|
|
# ─────────────────────────────────────────────────────────────────────── group
|
|
|
|
|
|
@click.group("vm")
|
|
def vm() -> None:
|
|
"""VM lifecycle commands."""
|
|
|
|
|
|
# ─────────────────────────────────────────────────────────────────────── remove
|
|
|
|
|
|
@vm.command("remove")
|
|
@click.option(
|
|
"--vmx",
|
|
"--vm-path",
|
|
"vmx",
|
|
required=True,
|
|
help="Path to the clone VMX file to destroy.",
|
|
)
|
|
@click.option("--vmrun-path", "vmrun_path", default=None, help="Override vmrun.exe path.")
|
|
@click.option(
|
|
"--graceful-timeout",
|
|
"--graceful-timeout-seconds",
|
|
"graceful_timeout",
|
|
type=click.IntRange(1, 60),
|
|
default=15,
|
|
show_default=True,
|
|
help="Seconds to wait for soft stop before forcing.",
|
|
)
|
|
@click.option(
|
|
"--force",
|
|
is_flag=True,
|
|
default=False,
|
|
help="Skip soft stop and go straight to hard stop.",
|
|
)
|
|
def vm_remove(
|
|
vmx: str,
|
|
vmrun_path: str | None,
|
|
graceful_timeout: int,
|
|
force: bool,
|
|
) -> None:
|
|
"""Stop and permanently delete an ephemeral build VM."""
|
|
vmx_path = Path(vmx)
|
|
clone_dir = vmx_path.parent
|
|
click.echo(f"[vm remove] destroying VM: {vmx}")
|
|
|
|
if not vmx_path.is_file():
|
|
click.echo(
|
|
f"[vm remove] VMX file not found - nothing to remove: {vmx}",
|
|
err=True,
|
|
)
|
|
# Still try to clean up a partial clone directory.
|
|
if clone_dir.exists() and clone_dir != vmx_path:
|
|
_try_remove_dir(clone_dir)
|
|
return
|
|
|
|
try:
|
|
backend = _make_backend(vmrun_path)
|
|
except BackendNotAvailable as exc:
|
|
click.echo(
|
|
f"[vm remove] vmrun unavailable ({exc}); attempting directory removal only.",
|
|
err=True,
|
|
)
|
|
_try_remove_dir(clone_dir)
|
|
return
|
|
|
|
handle = VmHandle(identifier=vmx)
|
|
|
|
if force:
|
|
try:
|
|
backend.stop(handle, hard=True)
|
|
except BackendError as exc:
|
|
click.echo(f"[vm remove] hard stop failed: {exc}", err=True)
|
|
else:
|
|
click.echo(f"[vm remove] sending soft stop (timeout: {graceful_timeout}s)...")
|
|
_stop_with_fallback(backend, handle)
|
|
|
|
# Brief settle: VMware may keep file locks for a moment after stop.
|
|
deadline = time.monotonic() + 20.0
|
|
while time.monotonic() < deadline:
|
|
try:
|
|
if not backend.is_running(handle):
|
|
break
|
|
except BackendError:
|
|
break
|
|
time.sleep(2.0)
|
|
|
|
click.echo("[vm remove] deleting VM...")
|
|
deleted = False
|
|
last_err: str = ""
|
|
for delay in (0, 3, 6):
|
|
if delay > 0:
|
|
click.echo(f"[vm remove] retrying deleteVM in {delay}s...")
|
|
time.sleep(delay)
|
|
try:
|
|
backend.delete(handle)
|
|
deleted = True
|
|
break
|
|
except BackendError as exc:
|
|
last_err = str(exc)
|
|
|
|
if not deleted:
|
|
click.echo(
|
|
f"[vm remove] vmrun deleteVM failed after retries: {last_err}", err=True
|
|
)
|
|
click.echo("[vm remove] falling back to manual directory removal.")
|
|
|
|
if clone_dir.exists():
|
|
click.echo(f"[vm remove] removing clone directory: {clone_dir}")
|
|
if _try_remove_dir(clone_dir):
|
|
click.echo("[vm remove] clone directory removed.")
|
|
else:
|
|
click.echo(
|
|
f"[vm remove] could not fully remove: {clone_dir} - manual cleanup needed.",
|
|
err=True,
|
|
)
|
|
|
|
click.echo("[vm remove] VM destruction complete.")
|
|
|
|
|
|
# ────────────────────────────────────────────────────────────────────── cleanup
|
|
|
|
|
|
def _list_orphans(clone_base: Path, max_age_hours: int) -> list[Path]:
|
|
"""Return orphan clone directories older than ``max_age_hours``."""
|
|
if not clone_base.is_dir():
|
|
return []
|
|
cutoff = datetime.now(tz=UTC) - timedelta(hours=max_age_hours)
|
|
out: list[Path] = []
|
|
for entry in clone_base.iterdir():
|
|
if not entry.is_dir():
|
|
continue
|
|
try:
|
|
mtime = datetime.fromtimestamp(entry.stat().st_mtime, tz=UTC)
|
|
except OSError:
|
|
continue
|
|
if mtime < cutoff:
|
|
out.append(entry)
|
|
return out
|
|
|
|
|
|
def _find_vmx(clone_dir: Path) -> Path | None:
|
|
for entry in clone_dir.iterdir():
|
|
if entry.is_file() and entry.suffix.lower() == ".vmx":
|
|
return entry
|
|
return None
|
|
|
|
|
|
def cleanup_orphans(
|
|
clone_base: Path,
|
|
max_age_hours: int,
|
|
backend: VmBackend | None,
|
|
*,
|
|
dry_run: bool = False,
|
|
lock_file: Path | None = None,
|
|
lock_max_age_minutes: int = 30,
|
|
) -> int:
|
|
"""Remove orphaned clone directories. Returns count removed.
|
|
|
|
Phase C hook: ``backend`` is used to stop+delete each orphan. If the
|
|
backend is unavailable, directory removal is still attempted.
|
|
"""
|
|
orphans = _list_orphans(clone_base, max_age_hours)
|
|
if not orphans:
|
|
click.echo(
|
|
f"[vm cleanup] no orphaned VMs found (threshold: {max_age_hours} h)."
|
|
)
|
|
else:
|
|
click.echo(
|
|
f"[vm cleanup] found {len(orphans)} orphaned director"
|
|
f"{'y' if len(orphans) == 1 else 'ies'} older than {max_age_hours} h."
|
|
)
|
|
|
|
removed = 0
|
|
now = datetime.now(tz=UTC)
|
|
for entry in orphans:
|
|
age_h = int((now - datetime.fromtimestamp(entry.stat().st_mtime, tz=UTC)).total_seconds() // 3600)
|
|
if dry_run:
|
|
click.echo(f"[vm cleanup] would destroy {entry} (age: {age_h} h)")
|
|
continue
|
|
click.echo(f"[vm cleanup] processing: {entry}")
|
|
vmx = _find_vmx(entry)
|
|
if vmx is not None and backend is not None:
|
|
handle = VmHandle(identifier=str(vmx))
|
|
try:
|
|
backend.stop(handle, hard=True)
|
|
except BackendError as exc:
|
|
click.echo(f"[vm cleanup] hard stop ignored: {exc}", err=True)
|
|
time.sleep(2.0)
|
|
try:
|
|
backend.delete(handle)
|
|
except BackendError as exc:
|
|
click.echo(
|
|
f"[vm cleanup] vmrun deleteVM failed: {exc}; falling back to dir removal.",
|
|
err=True,
|
|
)
|
|
elif vmx is None:
|
|
click.echo(
|
|
f"[vm cleanup] no .vmx in {entry} - removing directory only.", err=True
|
|
)
|
|
|
|
if _try_remove_dir(entry):
|
|
click.echo(f"[vm cleanup] removed: {entry}")
|
|
removed += 1
|
|
else:
|
|
click.echo(
|
|
f"[vm cleanup] could not fully remove: {entry} - manual cleanup needed.",
|
|
err=True,
|
|
)
|
|
|
|
# Stale lock file cleanup (mirrors Cleanup-OrphanedBuildVMs.ps1).
|
|
if lock_file is not None and lock_file.is_file():
|
|
cutoff = datetime.now(tz=UTC) - timedelta(minutes=lock_max_age_minutes)
|
|
mtime = datetime.fromtimestamp(lock_file.stat().st_mtime, tz=UTC)
|
|
if mtime < cutoff and not dry_run:
|
|
age_min = int((datetime.now(tz=UTC) - mtime).total_seconds() // 60)
|
|
try:
|
|
lock_file.unlink()
|
|
click.echo(
|
|
f"[vm cleanup] removed stale lock file (age: {age_min} min): {lock_file}"
|
|
)
|
|
except OSError as exc:
|
|
click.echo(f"[vm cleanup] could not remove lock file: {exc}", err=True)
|
|
|
|
click.echo("[vm cleanup] done.")
|
|
return removed
|
|
|
|
|
|
@vm.command("cleanup")
|
|
@click.option(
|
|
"--clone-base-dir",
|
|
"--clone-base",
|
|
"clone_base_dir",
|
|
default=None,
|
|
help="Directory containing ephemeral VM clones (defaults to config.paths.build_vms).",
|
|
)
|
|
@click.option(
|
|
"--max-age-hours",
|
|
type=click.IntRange(0, 168),
|
|
default=4,
|
|
show_default=True,
|
|
)
|
|
@click.option("--vmrun-path", "vmrun_path", default=None, help="Override vmrun.exe path.")
|
|
@click.option(
|
|
"--lock-file",
|
|
default=None,
|
|
help="Path to a stale vm-start lock file to remove if older than 30 minutes.",
|
|
)
|
|
@click.option(
|
|
"--what-if",
|
|
"--whatif",
|
|
"what_if",
|
|
is_flag=True,
|
|
default=False,
|
|
help="List orphans without destroying them.",
|
|
)
|
|
def vm_cleanup(
|
|
clone_base_dir: str | None,
|
|
max_age_hours: int,
|
|
vmrun_path: str | None,
|
|
lock_file: str | None,
|
|
what_if: bool,
|
|
) -> None:
|
|
"""Destroy ephemeral build VMs left behind after a crash or timeout."""
|
|
config = load_config()
|
|
base_path = Path(clone_base_dir) if clone_base_dir else config.paths.build_vms
|
|
if not base_path.exists():
|
|
click.echo(
|
|
f"[vm cleanup] clone base dir not found: {base_path} - nothing to do."
|
|
)
|
|
return
|
|
|
|
try:
|
|
backend: VmBackend | None = _make_backend(vmrun_path)
|
|
except BackendNotAvailable as exc:
|
|
click.echo(
|
|
f"[vm cleanup] vmrun not available ({exc}); will attempt directory removal only.",
|
|
err=True,
|
|
)
|
|
backend = None
|
|
|
|
cleanup_orphans(
|
|
clone_base=base_path,
|
|
max_age_hours=max_age_hours,
|
|
backend=backend,
|
|
dry_run=what_if,
|
|
lock_file=Path(lock_file) if lock_file else None,
|
|
)
|
|
|
|
|
|
__all__ = ["cleanup_orphans", "vm"]
|