diff --git a/src/ci_orchestrator/commands/vm.py b/src/ci_orchestrator/commands/vm.py index 73ba735..e8913a8 100644 --- a/src/ci_orchestrator/commands/vm.py +++ b/src/ci_orchestrator/commands/vm.py @@ -27,6 +27,7 @@ from ci_orchestrator.backends import load_backend from ci_orchestrator.backends.errors import BackendError, BackendNotAvailable from ci_orchestrator.backends.protocol import VmHandle from ci_orchestrator.config import load_config +from ci_orchestrator.operations import vms as ops_vms if TYPE_CHECKING: # pragma: no cover from ci_orchestrator.backends.protocol import VmBackend @@ -190,111 +191,6 @@ def vm_remove( # ────────────────────────────────────────────────────────────────────── cleanup -def _list_orphans(clone_base: Path, max_age_hours: int) -> list[Path]: - """Return orphan clone directories older than ``max_age_hours``.""" - if not clone_base.is_dir(): - return [] - cutoff = datetime.now(tz=UTC) - timedelta(hours=max_age_hours) - out: list[Path] = [] - for entry in clone_base.iterdir(): - try: - if not entry.is_dir(): - continue - mtime = datetime.fromtimestamp(entry.stat().st_mtime, tz=UTC) - except OSError: - continue - if mtime < cutoff: - out.append(entry) - return out - - -def _find_vmx(clone_dir: Path) -> Path | None: - for entry in clone_dir.iterdir(): - if entry.is_file() and entry.suffix.lower() == ".vmx": - return entry - return None - - -def cleanup_orphans( - clone_base: Path, - max_age_hours: int, - backend: VmBackend | None, - *, - dry_run: bool = False, - lock_file: Path | None = None, - lock_max_age_minutes: int = 30, -) -> int: - """Remove orphaned clone directories. Returns count removed. - - Phase C hook: ``backend`` is used to stop+delete each orphan. If the - backend is unavailable, directory removal is still attempted. - """ - orphans = _list_orphans(clone_base, max_age_hours) - if not orphans: - click.echo( - f"[vm cleanup] no orphaned VMs found (threshold: {max_age_hours} h)." - ) - else: - click.echo( - f"[vm cleanup] found {len(orphans)} orphaned director" - f"{'y' if len(orphans) == 1 else 'ies'} older than {max_age_hours} h." - ) - - removed = 0 - now = datetime.now(tz=UTC) - for entry in orphans: - age_h = int((now - datetime.fromtimestamp(entry.stat().st_mtime, tz=UTC)).total_seconds() // 3600) - if dry_run: - click.echo(f"[vm cleanup] would destroy {entry} (age: {age_h} h)") - continue - click.echo(f"[vm cleanup] processing: {entry}") - vmx = _find_vmx(entry) - if vmx is not None and backend is not None: - handle = VmHandle(identifier=str(vmx)) - try: - backend.stop(handle, hard=True) - except BackendError as exc: - click.echo(f"[vm cleanup] hard stop ignored: {exc}", err=True) - time.sleep(2.0) - try: - backend.delete(handle) - except BackendError as exc: - click.echo( - f"[vm cleanup] vmrun deleteVM failed: {exc}; falling back to dir removal.", - err=True, - ) - elif vmx is None: - click.echo( - f"[vm cleanup] no .vmx in {entry} - removing directory only.", err=True - ) - - if _try_remove_dir(entry): - click.echo(f"[vm cleanup] removed: {entry}") - removed += 1 - else: - click.echo( - f"[vm cleanup] could not fully remove: {entry} - manual cleanup needed.", - err=True, - ) - - # Stale lock file cleanup (mirrors Cleanup-OrphanedBuildVMs.ps1). - if lock_file is not None and lock_file.is_file(): - cutoff = datetime.now(tz=UTC) - timedelta(minutes=lock_max_age_minutes) - mtime = datetime.fromtimestamp(lock_file.stat().st_mtime, tz=UTC) - if mtime < cutoff and not dry_run: - age_min = int((datetime.now(tz=UTC) - mtime).total_seconds() // 60) - try: - lock_file.unlink() - click.echo( - f"[vm cleanup] removed stale lock file (age: {age_min} min): {lock_file}" - ) - except OSError as exc: - click.echo(f"[vm cleanup] could not remove lock file: {exc}", err=True) - - click.echo("[vm cleanup] done.") - return removed - - @vm.command("cleanup") @click.option( "--clone-base-dir", @@ -348,14 +244,71 @@ def vm_cleanup( ) backend = None - cleanup_orphans( + lock_path = Path(lock_file) if lock_file else None + + # Snapshot for messaging: cleanup_orphans() re-derives the same list + # internally (nothing is mutated between the two calls above the + # per-orphan destroy loop), so the ordering matches 1:1. + orphans = ops_vms.list_orphans(base_path, max_age_hours) + if not orphans: + click.echo( + f"[vm cleanup] no orphaned VMs found (threshold: {max_age_hours} h)." + ) + else: + click.echo( + f"[vm cleanup] found {len(orphans)} orphaned director" + f"{'y' if len(orphans) == 1 else 'ies'} older than {max_age_hours} h." + ) + + lock_was_stale = False + lock_age_min = 0 + if lock_path is not None and lock_path.is_file(): + cutoff = datetime.now(tz=UTC) - timedelta(minutes=30) + mtime = datetime.fromtimestamp(lock_path.stat().st_mtime, tz=UTC) + if mtime < cutoff: + lock_was_stale = True + lock_age_min = int((datetime.now(tz=UTC) - mtime).total_seconds() // 60) + + results = ops_vms.cleanup_orphans( clone_base=base_path, max_age_hours=max_age_hours, backend=backend, dry_run=what_if, - lock_file=Path(lock_file) if lock_file else None, + lock_file=lock_path, ) + for orphan, result in zip(orphans, results, strict=True): + if what_if: + click.echo( + f"[vm cleanup] would destroy {orphan.path} (age: {orphan.age_hours} h)" + ) + continue + click.echo(f"[vm cleanup] processing: {orphan.path}") + if orphan.vmx is None: + click.echo( + f"[vm cleanup] no .vmx in {orphan.path} - removing directory only.", + err=True, + ) + elif result.error is not None: + click.echo( + f"[vm cleanup] backend error on {orphan.path}: {result.error}", + err=True, + ) + if result.dir_removed: + click.echo(f"[vm cleanup] removed: {orphan.path}") + else: + click.echo( + f"[vm cleanup] could not fully remove: {orphan.path} - manual cleanup needed.", + err=True, + ) + + if lock_was_stale and not what_if and lock_path is not None and not lock_path.exists(): + click.echo( + f"[vm cleanup] removed stale lock file (age: {lock_age_min} min): {lock_path}" + ) + + click.echo("[vm cleanup] done.") + # ─────────────────────────────────────────────────────────────────────── new @@ -686,4 +639,4 @@ def vmware( ) -__all__ = ["cleanup_orphans", "vm", "vmware"] +__all__ = ["vm", "vmware"] diff --git a/src/ci_orchestrator/operations/vms.py b/src/ci_orchestrator/operations/vms.py new file mode 100644 index 0000000..d31d4ea --- /dev/null +++ b/src/ci_orchestrator/operations/vms.py @@ -0,0 +1,313 @@ +"""Pure VM-lifecycle operations shared by the CLI and (future) web UI. + +Extracted from ``commands/vm.py`` (W1 of the web-UI plan). Zero ``click``, +zero ``print`` — everything here returns a dataclass. Presentation stays in +the CLI command wrappers (and, later, the web routes). +""" + +from __future__ import annotations + +import contextlib +import shutil +import time +from dataclasses import dataclass +from datetime import UTC, datetime, timedelta +from pathlib import Path +from typing import TYPE_CHECKING + +from ci_orchestrator.backends.errors import BackendError +from ci_orchestrator.backends.protocol import VmHandle + +if TYPE_CHECKING: # pragma: no cover + from ci_orchestrator.backends.protocol import VmBackend + from ci_orchestrator.config import Config + +# Matches the default ``vm cleanup --max-age-hours`` threshold; used to flag +# a live VM as an orphan candidate in ``list_live_vms``. +_ORPHAN_AGE_HOURS = 4 + + +@dataclass(frozen=True) +class LiveVm: + """A currently-running VM as reported by the backend.""" + + name: str + vmx: str + ip: str | None + guest_os: str + uptime_sec: int | None + is_orphan: bool + + +@dataclass(frozen=True) +class Orphan: + """A clone directory older than the cleanup threshold.""" + + path: Path + vmx: Path | None + age_hours: int + + +@dataclass(frozen=True) +class RemovalResult: + """Outcome of destroying a single VM (used by both remove and cleanup).""" + + target: str + stopped: bool + deleted: bool + dir_removed: bool + error: str | None + + +# ────────────────────────────────────────────────────────────────── helpers + + +def _try_remove_dir(path: Path) -> bool: + """Best-effort recursive directory removal. Returns True if path is gone.""" + if not path.exists(): + return True + shutil.rmtree(path, ignore_errors=True) + return not path.exists() + + +def _find_vmx(clone_dir: Path) -> Path | None: + for entry in clone_dir.iterdir(): + if entry.is_file() and entry.suffix.lower() == ".vmx": + return entry + return None + + +def _is_under(path: Path, base: Path) -> bool: + """Return True if ``base`` is an ancestor of ``path`` (resolved).""" + try: + return base.resolve() in path.resolve().parents + except OSError: + return False + + +def _read_guest_os(vmx: Path) -> str: + """Best-effort parse of the ``guestOS = "..."`` line from a VMX file.""" + try: + text = vmx.read_text(encoding="utf-8", errors="replace") + except OSError: + return "" + for line in text.splitlines(): + if "=" not in line: + continue + key, _, rhs = line.partition("=") + if key.strip().lower() != "guestos": + continue + rhs = rhs.strip() + if rhs.startswith('"') and rhs.endswith('"') and len(rhs) >= 2: + return rhs[1:-1] + return rhs + return "" + + +# ─────────────────────────────────────────────────────────────── live VMs + + +def list_live_vms(backend: VmBackend, config: Config) -> list[LiveVm]: + """Return every VM the backend reports as running, enriched for display. + + Never raises on a single bad VM — a handle that can't be inspected is + still returned with degraded fields rather than aborting the whole list. + """ + try: + handles = backend.list_running() + except BackendError: + return [] + + now = datetime.now(tz=UTC) + out: list[LiveVm] = [] + for h in handles: + vmx = Path(h.identifier) + name = h.name or vmx.stem + + guest_os = _read_guest_os(vmx) + + try: + ip = backend.get_ip(h, timeout=0) + except BackendError: + ip = None + + under_base = _is_under(vmx, config.paths.build_vms) + + uptime_sec: int | None = None + if under_base: + try: + mtime = datetime.fromtimestamp(vmx.parent.stat().st_mtime, tz=UTC) + uptime_sec = int((now - mtime).total_seconds()) + except OSError: + uptime_sec = None + + is_orphan = False + if under_base and uptime_sec is not None: + age_hours = uptime_sec // 3600 + is_orphan = age_hours >= _ORPHAN_AGE_HOURS + + out.append( + LiveVm( + name=name, + vmx=str(vmx), + ip=ip, + guest_os=guest_os, + uptime_sec=uptime_sec, + is_orphan=is_orphan, + ) + ) + return out + + +# ───────────────────────────────────────────────────────────────── orphans + + +def list_orphans(clone_base: Path, max_age_hours: int) -> list[Orphan]: + """Return orphan clone directories older than ``max_age_hours``.""" + if not clone_base.is_dir(): + return [] + cutoff = datetime.now(tz=UTC) - timedelta(hours=max_age_hours) + now = datetime.now(tz=UTC) + out: list[Orphan] = [] + for entry in clone_base.iterdir(): + try: + if not entry.is_dir(): + continue + mtime = datetime.fromtimestamp(entry.stat().st_mtime, tz=UTC) + except OSError: + continue + if mtime < cutoff: + age_hours = int((now - mtime).total_seconds() // 3600) + out.append(Orphan(path=entry, vmx=_find_vmx(entry), age_hours=age_hours)) + return out + + +# ───────────────────────────────────────────────────────────────── removal + + +def remove_vm(backend: VmBackend, vmx: Path, *, dry_run: bool = False) -> RemovalResult: + """Stop and permanently delete a single VM. Pure extraction of the destroy sequence. + + Swallows :class:`BackendError` from stop/delete (recorded in the result + flags/``error``); never raises. + """ + target = str(vmx) + if dry_run: + return RemovalResult(target=target, stopped=False, deleted=False, dir_removed=False, error=None) + + handle = VmHandle(identifier=target) + first_error: str | None = None + + stopped = False + try: + backend.stop(handle, hard=True) + stopped = True + except BackendError as exc: + first_error = first_error or str(exc) + + time.sleep(2.0) + + deleted = False + last_err: str | None = None + for delay in (0, 3, 6): + if delay > 0: + time.sleep(delay) + try: + backend.delete(handle) + deleted = True + break + except BackendError as exc: + last_err = str(exc) + if not deleted and last_err is not None: + first_error = first_error or last_err + + dir_removed = _try_remove_dir(vmx.parent) + + return RemovalResult( + target=target, + stopped=stopped, + deleted=deleted, + dir_removed=dir_removed, + error=first_error, + ) + + +def cleanup_orphans( + clone_base: Path, + max_age_hours: int, + backend: VmBackend | None, + *, + dry_run: bool = False, + lock_file: Path | None = None, + lock_max_age_minutes: int = 30, +) -> list[RemovalResult]: + """Remove orphaned clone directories. Returns one :class:`RemovalResult` per orphan. + + ``backend`` is used to stop+delete each orphan's VMX before removing its + directory. If the backend is unavailable (``None``), directory removal + is still attempted. + """ + orphans = list_orphans(clone_base, max_age_hours) + + results: list[RemovalResult] = [] + for orphan in orphans: + entry = orphan.path + target = str(orphan.vmx) if orphan.vmx is not None else str(entry) + + if dry_run: + results.append( + RemovalResult(target=target, stopped=False, deleted=False, dir_removed=False, error=None) + ) + continue + + error: str | None = None + stopped = False + deleted = False + + vmx = orphan.vmx + if vmx is not None and backend is not None: + handle = VmHandle(identifier=str(vmx)) + try: + backend.stop(handle, hard=True) + stopped = True + except BackendError as exc: + error = error or str(exc) + time.sleep(2.0) + try: + backend.delete(handle) + deleted = True + except BackendError as exc: + error = error or str(exc) + + dir_removed = _try_remove_dir(entry) + + results.append( + RemovalResult( + target=target, + stopped=stopped, + deleted=deleted, + dir_removed=dir_removed, + error=error, + ) + ) + + # Stale lock file cleanup (mirrors Cleanup-OrphanedBuildVMs.ps1). + if lock_file is not None and lock_file.is_file(): + cutoff = datetime.now(tz=UTC) - timedelta(minutes=lock_max_age_minutes) + mtime = datetime.fromtimestamp(lock_file.stat().st_mtime, tz=UTC) + if mtime < cutoff and not dry_run: + with contextlib.suppress(OSError): + lock_file.unlink() + + return results + + +__all__ = [ + "LiveVm", + "Orphan", + "RemovalResult", + "cleanup_orphans", + "list_live_vms", + "list_orphans", + "remove_vm", +] diff --git a/tests/python/test_commands_vm.py b/tests/python/test_commands_vm.py index 5d2a6c4..4bd5197 100644 --- a/tests/python/test_commands_vm.py +++ b/tests/python/test_commands_vm.py @@ -378,110 +378,6 @@ def test_vm_remove_clone_dir_removal_failure( assert "manual cleanup needed" in result.output -def test_list_orphans_non_dir_base(tmp_path: Path) -> None: - f = tmp_path / "file.txt" - f.write_text("x") - assert vm_module._list_orphans(f, 4) == [] - - -def test_list_orphans_skips_non_dir_and_oserror(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: - base = tmp_path / "b" - base.mkdir() - (base / "a-file").write_text("not a dir") - bad_dir = base / "bad" - bad_dir.mkdir() - _age_dir(bad_dir, hours=10) - - real_stat = Path.stat - - def _stat(self: Path, *a: Any, **kw: Any) -> Any: - if self == bad_dir: - raise OSError("nope") - return real_stat(self, *a, **kw) - - monkeypatch.setattr(Path, "stat", _stat) - assert vm_module._list_orphans(base, 4) == [] - - -def test_find_vmx_returns_none(tmp_path: Path) -> None: - d = tmp_path / "novmx" - d.mkdir() - (d / "readme.txt").write_text("x") - assert vm_module._find_vmx(d) is None - - -def test_cleanup_orphans_dir_without_vmx_logs_and_removes(tmp_path: Path) -> None: - base = tmp_path / "b" - base.mkdir() - no_vmx = base / "no-vmx" - no_vmx.mkdir() - _age_dir(no_vmx, hours=10) - runner_count = vm_module.cleanup_orphans( - clone_base=base, max_age_hours=4, backend=None - ) - assert runner_count == 1 - assert not no_vmx.exists() - - -def test_cleanup_orphans_backend_stop_error_swallowed( - monkeypatch: pytest.MonkeyPatch, tmp_path: Path -) -> None: - from ci_orchestrator.backends.errors import BackendOperationFailed - - base = tmp_path / "b" - base.mkdir() - old, _ = _make_clone_dir(base, "old1") - _age_dir(old, hours=10) - - class _B: - def stop(self, _h: VmHandle, hard: bool = False) -> None: - raise BackendOperationFailed("stop", 1, "no") - - def delete(self, _h: VmHandle) -> None: - raise BackendOperationFailed("delete", 1, "no") - - count = vm_module.cleanup_orphans( - clone_base=base, max_age_hours=4, backend=_B() - ) - assert count == 1 - - -def test_cleanup_orphans_dir_removal_failure_logs( - monkeypatch: pytest.MonkeyPatch, tmp_path: Path -) -> None: - base = tmp_path / "b" - base.mkdir() - old, _ = _make_clone_dir(base, "stuck") - _age_dir(old, hours=10) - monkeypatch.setattr(vm_module, "_try_remove_dir", lambda _p: False) - count = vm_module.cleanup_orphans( - clone_base=base, max_age_hours=4, backend=None - ) - assert count == 0 - - -def test_cleanup_orphans_lock_unlink_oserror( - monkeypatch: pytest.MonkeyPatch, tmp_path: Path -) -> None: - base = tmp_path / "b" - base.mkdir() - lock = tmp_path / "stale.lock" - lock.write_text("x") - _age_dir(lock, hours=1) - - real_unlink = Path.unlink - - def _boom(self: Path, *a: Any, **kw: Any) -> Any: - if self == lock: - raise OSError("locked") - return real_unlink(self, *a, **kw) - - monkeypatch.setattr(Path, "unlink", _boom) - vm_module.cleanup_orphans( - clone_base=base, max_age_hours=4, backend=None, lock_file=lock - ) - - def test_vm_cleanup_backend_unavailable(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: from ci_orchestrator.backends.errors import BackendNotAvailable diff --git a/tests/python/test_operations_vms.py b/tests/python/test_operations_vms.py new file mode 100644 index 0000000..1a3725a --- /dev/null +++ b/tests/python/test_operations_vms.py @@ -0,0 +1,447 @@ +"""Tests for the pure ``operations.vms`` layer (W1 of the web-UI plan). + +Migrated from the direct-call tests that used to live in +``test_commands_vm.py`` (``_list_orphans`` / ``_find_vmx`` / +``cleanup_orphans``), adapted to the new dataclass return types. Adds +coverage for ``list_live_vms`` and ``remove_vm``. +""" + +from __future__ import annotations + +import time +from datetime import datetime, timedelta +from pathlib import Path +from typing import Any + +import pytest + +import ci_orchestrator.operations.vms as vms_module +from ci_orchestrator.backends.errors import BackendOperationFailed +from ci_orchestrator.backends.protocol import VmHandle +from ci_orchestrator.config import BackendConfig, Config, Paths +from ci_orchestrator.operations.vms import ( + Orphan, + RemovalResult, + cleanup_orphans, + list_live_vms, + list_orphans, + remove_vm, +) + + +@pytest.fixture(autouse=True) +def _no_sleep(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(time, "sleep", lambda _s: None) + monkeypatch.setattr(vms_module.time, "sleep", lambda _s: None) + + +def _age_dir(path: Path, hours: float) -> None: + ts = (datetime.now() - timedelta(hours=hours)).timestamp() + import os + + os.utime(path, (ts, ts)) + + +def _make_clone_dir(tmp_path: Path, name: str = "clone") -> tuple[Path, Path]: + clone_dir = tmp_path / name + clone_dir.mkdir() + vmx = clone_dir / f"{name}.vmx" + vmx.write_text('config.version = "8"', encoding="utf-8") + return clone_dir, vmx + + +def _make_config(build_vms: Path) -> Config: + return Config( + paths=Paths( + root=build_vms.parent, + templates=build_vms.parent / "templates", + build_vms=build_vms, + artifacts=build_vms.parent / "artifacts", + keys=build_vms.parent / "keys", + ), + backend=BackendConfig(), + ) + + +class _FakeBackend: + def __init__( + self, + *, + running_after_stop: bool = False, + delete_fails_first: int = 0, + ) -> None: + self.calls: list[tuple[str, Any]] = [] + self._running = True + self._running_after_stop = running_after_stop + self._delete_fails_remaining = delete_fails_first + + def stop(self, handle: VmHandle, hard: bool = False) -> None: + self.calls.append(("stop", hard)) + if not hard or not self._running_after_stop: + self._running = False + + def is_running(self, _h: VmHandle) -> bool: + return self._running + + def delete(self, _h: VmHandle) -> None: + self.calls.append(("delete", None)) + if self._delete_fails_remaining > 0: + self._delete_fails_remaining -= 1 + raise BackendOperationFailed("deleteVM", 1, "transient") + + +# ── list_orphans ──────────────────────────────────────────────────────────── + + +def test_list_orphans_non_dir_base(tmp_path: Path) -> None: + f = tmp_path / "file.txt" + f.write_text("x") + assert list_orphans(f, 4) == [] + + +def test_list_orphans_skips_non_dir_and_oserror( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + base = tmp_path / "b" + base.mkdir() + (base / "a-file").write_text("not a dir") + bad_dir = base / "bad" + bad_dir.mkdir() + _age_dir(bad_dir, hours=10) + + real_stat = Path.stat + + def _stat(self: Path, *a: Any, **kw: Any) -> Any: + if self == bad_dir: + raise OSError("nope") + return real_stat(self, *a, **kw) + + monkeypatch.setattr(Path, "stat", _stat) + assert list_orphans(base, 4) == [] + + +def test_list_orphans_returns_orphan_dataclass(tmp_path: Path) -> None: + base = tmp_path / "b" + base.mkdir() + old, vmx = _make_clone_dir(base, "old") + _age_dir(old, hours=10) + + orphans = list_orphans(base, 4) + assert orphans == [Orphan(path=old, vmx=vmx, age_hours=10)] + + +# ── _find_vmx ──────────────────────────────────────────────────────────── + + +def test_find_vmx_returns_none(tmp_path: Path) -> None: + d = tmp_path / "novmx" + d.mkdir() + (d / "readme.txt").write_text("x") + assert vms_module._find_vmx(d) is None + + +# ── cleanup_orphans ──────────────────────────────────────────────────────── + + +def test_cleanup_orphans_dir_without_vmx_logs_and_removes(tmp_path: Path) -> None: + base = tmp_path / "b" + base.mkdir() + no_vmx = base / "no-vmx" + no_vmx.mkdir() + _age_dir(no_vmx, hours=10) + results = cleanup_orphans(clone_base=base, max_age_hours=4, backend=None) + assert sum(r.dir_removed for r in results) == 1 + assert not no_vmx.exists() + + +def test_cleanup_orphans_backend_stop_error_swallowed(tmp_path: Path) -> None: + base = tmp_path / "b" + base.mkdir() + old, _ = _make_clone_dir(base, "old1") + _age_dir(old, hours=10) + + class _B: + def stop(self, _h: VmHandle, hard: bool = False) -> None: + raise BackendOperationFailed("stop", 1, "no") + + def delete(self, _h: VmHandle) -> None: + raise BackendOperationFailed("delete", 1, "no") + + results = cleanup_orphans(clone_base=base, max_age_hours=4, backend=_B()) + assert sum(r.dir_removed for r in results) == 1 + assert results[0].stopped is False + assert results[0].deleted is False + assert results[0].error is not None + + +def test_cleanup_orphans_dir_removal_failure_logs( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + base = tmp_path / "b" + base.mkdir() + old, _ = _make_clone_dir(base, "stuck") + _age_dir(old, hours=10) + monkeypatch.setattr(vms_module, "_try_remove_dir", lambda _p: False) + results = cleanup_orphans(clone_base=base, max_age_hours=4, backend=None) + assert sum(r.dir_removed for r in results) == 0 + assert len(results) == 1 + + +def test_cleanup_orphans_lock_unlink_oserror( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + base = tmp_path / "b" + base.mkdir() + lock = tmp_path / "stale.lock" + lock.write_text("x") + _age_dir(lock, hours=1) + + real_unlink = Path.unlink + + def _boom(self: Path, *a: Any, **kw: Any) -> Any: + if self == lock: + raise OSError("locked") + return real_unlink(self, *a, **kw) + + monkeypatch.setattr(Path, "unlink", _boom) + # Must not raise even though unlink() fails. + cleanup_orphans(clone_base=base, max_age_hours=4, backend=None, lock_file=lock) + + +def test_cleanup_orphans_dry_run_no_destroy(tmp_path: Path) -> None: + base = tmp_path / "b" + base.mkdir() + old, vmx = _make_clone_dir(base, "old") + _age_dir(old, hours=10) + backend = _FakeBackend() + results = cleanup_orphans(clone_base=base, max_age_hours=4, backend=backend, dry_run=True) + assert results == [RemovalResult(target=str(vmx), stopped=False, deleted=False, dir_removed=False, error=None)] + assert backend.calls == [] + assert old.exists() + + +def test_cleanup_orphans_no_orphans_returns_empty(tmp_path: Path) -> None: + base = tmp_path / "b" + base.mkdir() + assert cleanup_orphans(clone_base=base, max_age_hours=4, backend=None) == [] + + +def test_cleanup_orphans_lock_respects_dry_run(tmp_path: Path) -> None: + base = tmp_path / "b" + base.mkdir() + lock = tmp_path / "stale.lock" + lock.write_text("x") + _age_dir(lock, hours=1) + cleanup_orphans(clone_base=base, max_age_hours=4, backend=None, dry_run=True, lock_file=lock) + assert lock.exists() # dry run never touches the lock file + + +# ── list_live_vms ──────────────────────────────────────────────────────── + + +class _FakeLiveBackend: + def __init__( + self, + handles: list[VmHandle], + *, + ip: str | None = "192.168.1.50", + ip_raises: bool = False, + list_running_raises: bool = False, + ) -> None: + self._handles = handles + self._ip = ip + self._ip_raises = ip_raises + self._list_running_raises = list_running_raises + + def list_running(self) -> list[VmHandle]: + if self._list_running_raises: + raise BackendOperationFailed("list", 1, "boom") + return self._handles + + def get_ip(self, _h: VmHandle, timeout: float = 0.0) -> str | None: + if self._ip_raises: + raise BackendOperationFailed("getGuestIPAddress", 1, "no tools") + return self._ip + + +def test_list_live_vms_orphan_flag_on_old_clone(tmp_path: Path) -> None: + build_vms = tmp_path / "build-vms" + build_vms.mkdir() + clone_dir, vmx = _make_clone_dir(build_vms, "old") + vmx.write_text('guestOS = "ubuntu-64"', encoding="utf-8") + _age_dir(clone_dir, hours=10) + + backend = _FakeLiveBackend([VmHandle(identifier=str(vmx), name="old")]) + config = _make_config(build_vms) + + result = list_live_vms(backend, config) # type: ignore[arg-type] + assert len(result) == 1 + vm = result[0] + assert vm.name == "old" + assert vm.vmx == str(vmx) + assert vm.ip == "192.168.1.50" + assert vm.guest_os == "ubuntu-64" + assert vm.uptime_sec is not None and vm.uptime_sec >= 9 * 3600 + assert vm.is_orphan is True + + +def test_list_live_vms_fresh_clone_not_orphan(tmp_path: Path) -> None: + build_vms = tmp_path / "build-vms" + build_vms.mkdir() + clone_dir, vmx = _make_clone_dir(build_vms, "fresh") + _age_dir(clone_dir, hours=0) + + backend = _FakeLiveBackend([VmHandle(identifier=str(vmx), name=None)]) + config = _make_config(build_vms) + + result = list_live_vms(backend, config) # type: ignore[arg-type] + assert len(result) == 1 + vm = result[0] + assert vm.name == vmx.stem # name fell back to vmx stem + assert vm.is_orphan is False + + +def test_list_live_vms_outside_build_vms_not_orphan(tmp_path: Path) -> None: + build_vms = tmp_path / "build-vms" + build_vms.mkdir() + (tmp_path / "elsewhere").mkdir() + elsewhere, vmx = _make_clone_dir(tmp_path / "elsewhere", "other") + _age_dir(elsewhere, hours=10) + + backend = _FakeLiveBackend([VmHandle(identifier=str(vmx), name="other")]) + config = _make_config(build_vms) + + result = list_live_vms(backend, config) # type: ignore[arg-type] + assert len(result) == 1 + vm = result[0] + assert vm.is_orphan is False + assert vm.uptime_sec is None + + +def test_list_live_vms_get_ip_none(tmp_path: Path) -> None: + build_vms = tmp_path / "build-vms" + build_vms.mkdir() + _, vmx = _make_clone_dir(build_vms, "noip") + + backend = _FakeLiveBackend([VmHandle(identifier=str(vmx))], ip=None) + config = _make_config(build_vms) + + result = list_live_vms(backend, config) # type: ignore[arg-type] + assert result[0].ip is None + + +def test_list_live_vms_get_ip_raises_is_tolerated(tmp_path: Path) -> None: + build_vms = tmp_path / "build-vms" + build_vms.mkdir() + _, vmx = _make_clone_dir(build_vms, "iperr") + + backend = _FakeLiveBackend([VmHandle(identifier=str(vmx))], ip_raises=True) + config = _make_config(build_vms) + + result = list_live_vms(backend, config) # type: ignore[arg-type] + assert result[0].ip is None + + +def test_list_live_vms_list_running_raises_returns_empty(tmp_path: Path) -> None: + build_vms = tmp_path / "build-vms" + build_vms.mkdir() + backend = _FakeLiveBackend([], list_running_raises=True) + config = _make_config(build_vms) + assert list_live_vms(backend, config) == [] # type: ignore[arg-type] + + +def test_list_live_vms_missing_vmx_file_guest_os_empty(tmp_path: Path) -> None: + build_vms = tmp_path / "build-vms" + build_vms.mkdir() + missing_vmx = build_vms / "ghost" / "ghost.vmx" # parent dir doesn't even exist + + backend = _FakeLiveBackend([VmHandle(identifier=str(missing_vmx))]) + config = _make_config(build_vms) + + result = list_live_vms(backend, config) # type: ignore[arg-type] + assert result[0].guest_os == "" + + +def test_list_live_vms_empty_when_nothing_running(tmp_path: Path) -> None: + build_vms = tmp_path / "build-vms" + build_vms.mkdir() + backend = _FakeLiveBackend([]) + config = _make_config(build_vms) + assert list_live_vms(backend, config) == [] # type: ignore[arg-type] + + +# ── remove_vm ──────────────────────────────────────────────────────────── + + +def test_remove_vm_happy_path(tmp_path: Path) -> None: + clone_dir, vmx = _make_clone_dir(tmp_path, "live") + backend = _FakeBackend() + result = remove_vm(backend, vmx) # type: ignore[arg-type] + assert result == RemovalResult( + target=str(vmx), stopped=True, deleted=True, dir_removed=True, error=None + ) + assert not clone_dir.exists() + assert ("delete", None) in backend.calls + + +def test_remove_vm_dry_run_no_op(tmp_path: Path) -> None: + clone_dir, vmx = _make_clone_dir(tmp_path, "dry") + backend = _FakeBackend() + result = remove_vm(backend, vmx, dry_run=True) # type: ignore[arg-type] + assert result == RemovalResult( + target=str(vmx), stopped=False, deleted=False, dir_removed=False, error=None + ) + assert backend.calls == [] + assert clone_dir.exists() + + +def test_remove_vm_delete_retry_succeeds(tmp_path: Path) -> None: + clone_dir, vmx = _make_clone_dir(tmp_path, "retry") + backend = _FakeBackend(delete_fails_first=2) + result = remove_vm(backend, vmx) # type: ignore[arg-type] + delete_calls = [c for c in backend.calls if c[0] == "delete"] + assert len(delete_calls) == 3 # 2 failures + 1 success + assert result.deleted is True + assert not clone_dir.exists() + + +def test_remove_vm_delete_fails_all_retries(tmp_path: Path) -> None: + clone_dir, vmx = _make_clone_dir(tmp_path, "neverdeletes") + + class _B: + def stop(self, _h: VmHandle, hard: bool = False) -> None: + pass + + def delete(self, _h: VmHandle) -> None: + raise BackendOperationFailed("delete", 1, "always") + + result = remove_vm(_B(), vmx) # type: ignore[arg-type] + assert result.deleted is False + assert result.error is not None + # Directory removal is still attempted even when delete fails. + assert not clone_dir.exists() + assert result.dir_removed is True + + +def test_remove_vm_stop_error_is_swallowed(tmp_path: Path) -> None: + _, vmx = _make_clone_dir(tmp_path, "stoperr") + + class _B: + def stop(self, _h: VmHandle, hard: bool = False) -> None: + raise BackendOperationFailed("stop", 1, "no") + + def delete(self, _h: VmHandle) -> None: + pass + + result = remove_vm(_B(), vmx) # type: ignore[arg-type] + assert result.stopped is False + assert result.deleted is True + assert result.error is not None + + +def test_remove_vm_dir_removal_failure(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + _, vmx = _make_clone_dir(tmp_path, "stubborn") + backend = _FakeBackend() + monkeypatch.setattr(vms_module, "_try_remove_dir", lambda _p: False) + result = remove_vm(backend, vmx) # type: ignore[arg-type] + assert result.dir_removed is False