"""``vm`` sub-commands. Phase A2 ports the leaf VM management scripts: * ``vm remove`` — replaces ``scripts/Remove-BuildVM.ps1`` * ``vm cleanup`` — replaces ``scripts/Cleanup-OrphanedBuildVMs.ps1`` Phase C hook: ``vm cleanup`` accepts an optional :class:`VmBackend` dependency so an ESXi backend can later supply a folder-scoped VM list instead of scanning the local filesystem. """ from __future__ import annotations import shutil import time from datetime import UTC, datetime, timedelta from pathlib import Path from typing import TYPE_CHECKING import click from ci_orchestrator.backends import load_backend from ci_orchestrator.backends.errors import BackendError, BackendNotAvailable from ci_orchestrator.backends.protocol import VmHandle from ci_orchestrator.config import load_config if TYPE_CHECKING: # pragma: no cover from ci_orchestrator.backends.protocol import VmBackend # ────────────────────────────────────────────────────────────────────── helpers def _make_backend(vmrun_path: str | None) -> VmBackend: """Build a backend honouring an optional vmrun override.""" config = load_config() if vmrun_path is not None: from ci_orchestrator.backends.workstation import WorkstationVmrunBackend return WorkstationVmrunBackend(vmrun_path=vmrun_path) return load_backend(config) def _try_remove_dir(path: Path) -> bool: """Best-effort recursive directory removal. Returns True if path is gone.""" if not path.exists(): return True shutil.rmtree(path, ignore_errors=True) return not path.exists() def _stop_with_fallback(backend: VmBackend, handle: VmHandle) -> None: """Soft-stop, then hard-stop on failure. Both errors are tolerated.""" try: backend.stop(handle, hard=False) except BackendError as exc: click.echo(f"[vm remove] soft stop failed: {exc}", err=True) try: if backend.is_running(handle): click.echo("[vm remove] still running, forcing hard stop...") backend.stop(handle, hard=True) except BackendError as exc: click.echo(f"[vm remove] hard stop failed: {exc}", err=True) # ─────────────────────────────────────────────────────────────────────── group @click.group("vm") def vm() -> None: """VM lifecycle commands.""" # ─────────────────────────────────────────────────────────────────────── remove @vm.command("remove") @click.option( "--vmx", "--vm-path", "vmx", required=True, help="Path to the clone VMX file to destroy.", ) @click.option("--vmrun-path", "vmrun_path", default=None, help="Override vmrun.exe path.") @click.option( "--graceful-timeout", "--graceful-timeout-seconds", "graceful_timeout", type=click.IntRange(1, 60), default=15, show_default=True, help="Seconds to wait for soft stop before forcing.", ) @click.option( "--force", is_flag=True, default=False, help="Skip soft stop and go straight to hard stop.", ) def vm_remove( vmx: str, vmrun_path: str | None, graceful_timeout: int, force: bool, ) -> None: """Stop and permanently delete an ephemeral build VM.""" vmx_path = Path(vmx) clone_dir = vmx_path.parent click.echo(f"[vm remove] destroying VM: {vmx}") if not vmx_path.is_file(): click.echo( f"[vm remove] VMX file not found - nothing to remove: {vmx}", err=True, ) # Still try to clean up a partial clone directory. if clone_dir.exists() and clone_dir != vmx_path: _try_remove_dir(clone_dir) return try: backend = _make_backend(vmrun_path) except BackendNotAvailable as exc: click.echo( f"[vm remove] vmrun unavailable ({exc}); attempting directory removal only.", err=True, ) _try_remove_dir(clone_dir) return handle = VmHandle(identifier=vmx) if force: try: backend.stop(handle, hard=True) except BackendError as exc: click.echo(f"[vm remove] hard stop failed: {exc}", err=True) else: click.echo(f"[vm remove] sending soft stop (timeout: {graceful_timeout}s)...") _stop_with_fallback(backend, handle) # Brief settle: VMware may keep file locks for a moment after stop. deadline = time.monotonic() + 20.0 while time.monotonic() < deadline: try: if not backend.is_running(handle): break except BackendError: break time.sleep(2.0) click.echo("[vm remove] deleting VM...") deleted = False last_err: str = "" for delay in (0, 3, 6): if delay > 0: click.echo(f"[vm remove] retrying deleteVM in {delay}s...") time.sleep(delay) try: backend.delete(handle) deleted = True break except BackendError as exc: last_err = str(exc) if not deleted: click.echo( f"[vm remove] vmrun deleteVM failed after retries: {last_err}", err=True ) click.echo("[vm remove] falling back to manual directory removal.") if clone_dir.exists(): click.echo(f"[vm remove] removing clone directory: {clone_dir}") if _try_remove_dir(clone_dir): click.echo("[vm remove] clone directory removed.") else: click.echo( f"[vm remove] could not fully remove: {clone_dir} - manual cleanup needed.", err=True, ) click.echo("[vm remove] VM destruction complete.") # ────────────────────────────────────────────────────────────────────── cleanup def _list_orphans(clone_base: Path, max_age_hours: int) -> list[Path]: """Return orphan clone directories older than ``max_age_hours``.""" if not clone_base.is_dir(): return [] cutoff = datetime.now(tz=UTC) - timedelta(hours=max_age_hours) out: list[Path] = [] for entry in clone_base.iterdir(): if not entry.is_dir(): continue try: mtime = datetime.fromtimestamp(entry.stat().st_mtime, tz=UTC) except OSError: continue if mtime < cutoff: out.append(entry) return out def _find_vmx(clone_dir: Path) -> Path | None: for entry in clone_dir.iterdir(): if entry.is_file() and entry.suffix.lower() == ".vmx": return entry return None def cleanup_orphans( clone_base: Path, max_age_hours: int, backend: VmBackend | None, *, dry_run: bool = False, lock_file: Path | None = None, lock_max_age_minutes: int = 30, ) -> int: """Remove orphaned clone directories. Returns count removed. Phase C hook: ``backend`` is used to stop+delete each orphan. If the backend is unavailable, directory removal is still attempted. """ orphans = _list_orphans(clone_base, max_age_hours) if not orphans: click.echo( f"[vm cleanup] no orphaned VMs found (threshold: {max_age_hours} h)." ) else: click.echo( f"[vm cleanup] found {len(orphans)} orphaned director" f"{'y' if len(orphans) == 1 else 'ies'} older than {max_age_hours} h." ) removed = 0 now = datetime.now(tz=UTC) for entry in orphans: age_h = int((now - datetime.fromtimestamp(entry.stat().st_mtime, tz=UTC)).total_seconds() // 3600) if dry_run: click.echo(f"[vm cleanup] would destroy {entry} (age: {age_h} h)") continue click.echo(f"[vm cleanup] processing: {entry}") vmx = _find_vmx(entry) if vmx is not None and backend is not None: handle = VmHandle(identifier=str(vmx)) try: backend.stop(handle, hard=True) except BackendError as exc: click.echo(f"[vm cleanup] hard stop ignored: {exc}", err=True) time.sleep(2.0) try: backend.delete(handle) except BackendError as exc: click.echo( f"[vm cleanup] vmrun deleteVM failed: {exc}; falling back to dir removal.", err=True, ) elif vmx is None: click.echo( f"[vm cleanup] no .vmx in {entry} - removing directory only.", err=True ) if _try_remove_dir(entry): click.echo(f"[vm cleanup] removed: {entry}") removed += 1 else: click.echo( f"[vm cleanup] could not fully remove: {entry} - manual cleanup needed.", err=True, ) # Stale lock file cleanup (mirrors Cleanup-OrphanedBuildVMs.ps1). if lock_file is not None and lock_file.is_file(): cutoff = datetime.now(tz=UTC) - timedelta(minutes=lock_max_age_minutes) mtime = datetime.fromtimestamp(lock_file.stat().st_mtime, tz=UTC) if mtime < cutoff and not dry_run: age_min = int((datetime.now(tz=UTC) - mtime).total_seconds() // 60) try: lock_file.unlink() click.echo( f"[vm cleanup] removed stale lock file (age: {age_min} min): {lock_file}" ) except OSError as exc: click.echo(f"[vm cleanup] could not remove lock file: {exc}", err=True) click.echo("[vm cleanup] done.") return removed @vm.command("cleanup") @click.option( "--clone-base-dir", "--clone-base", "clone_base_dir", default=None, help="Directory containing ephemeral VM clones (defaults to config.paths.build_vms).", ) @click.option( "--max-age-hours", type=click.IntRange(0, 168), default=4, show_default=True, ) @click.option("--vmrun-path", "vmrun_path", default=None, help="Override vmrun.exe path.") @click.option( "--lock-file", default=None, help="Path to a stale vm-start lock file to remove if older than 30 minutes.", ) @click.option( "--what-if", "--whatif", "what_if", is_flag=True, default=False, help="List orphans without destroying them.", ) def vm_cleanup( clone_base_dir: str | None, max_age_hours: int, vmrun_path: str | None, lock_file: str | None, what_if: bool, ) -> None: """Destroy ephemeral build VMs left behind after a crash or timeout.""" config = load_config() base_path = Path(clone_base_dir) if clone_base_dir else config.paths.build_vms if not base_path.exists(): click.echo( f"[vm cleanup] clone base dir not found: {base_path} - nothing to do." ) return try: backend: VmBackend | None = _make_backend(vmrun_path) except BackendNotAvailable as exc: click.echo( f"[vm cleanup] vmrun not available ({exc}); will attempt directory removal only.", err=True, ) backend = None cleanup_orphans( clone_base=base_path, max_age_hours=max_age_hours, backend=backend, dry_run=what_if, lock_file=Path(lock_file) if lock_file else None, ) __all__ = ["cleanup_orphans", "vm"]