refactor(vms): extract operations/vms.py seam from commands/vm.py (W1)
This commit is contained in:
@@ -27,6 +27,7 @@ from ci_orchestrator.backends import load_backend
|
||||
from ci_orchestrator.backends.errors import BackendError, BackendNotAvailable
|
||||
from ci_orchestrator.backends.protocol import VmHandle
|
||||
from ci_orchestrator.config import load_config
|
||||
from ci_orchestrator.operations import vms as ops_vms
|
||||
|
||||
if TYPE_CHECKING: # pragma: no cover
|
||||
from ci_orchestrator.backends.protocol import VmBackend
|
||||
@@ -190,111 +191,6 @@ def vm_remove(
|
||||
# ────────────────────────────────────────────────────────────────────── cleanup
|
||||
|
||||
|
||||
def _list_orphans(clone_base: Path, max_age_hours: int) -> list[Path]:
|
||||
"""Return orphan clone directories older than ``max_age_hours``."""
|
||||
if not clone_base.is_dir():
|
||||
return []
|
||||
cutoff = datetime.now(tz=UTC) - timedelta(hours=max_age_hours)
|
||||
out: list[Path] = []
|
||||
for entry in clone_base.iterdir():
|
||||
try:
|
||||
if not entry.is_dir():
|
||||
continue
|
||||
mtime = datetime.fromtimestamp(entry.stat().st_mtime, tz=UTC)
|
||||
except OSError:
|
||||
continue
|
||||
if mtime < cutoff:
|
||||
out.append(entry)
|
||||
return out
|
||||
|
||||
|
||||
def _find_vmx(clone_dir: Path) -> Path | None:
|
||||
for entry in clone_dir.iterdir():
|
||||
if entry.is_file() and entry.suffix.lower() == ".vmx":
|
||||
return entry
|
||||
return None
|
||||
|
||||
|
||||
def cleanup_orphans(
|
||||
clone_base: Path,
|
||||
max_age_hours: int,
|
||||
backend: VmBackend | None,
|
||||
*,
|
||||
dry_run: bool = False,
|
||||
lock_file: Path | None = None,
|
||||
lock_max_age_minutes: int = 30,
|
||||
) -> int:
|
||||
"""Remove orphaned clone directories. Returns count removed.
|
||||
|
||||
Phase C hook: ``backend`` is used to stop+delete each orphan. If the
|
||||
backend is unavailable, directory removal is still attempted.
|
||||
"""
|
||||
orphans = _list_orphans(clone_base, max_age_hours)
|
||||
if not orphans:
|
||||
click.echo(
|
||||
f"[vm cleanup] no orphaned VMs found (threshold: {max_age_hours} h)."
|
||||
)
|
||||
else:
|
||||
click.echo(
|
||||
f"[vm cleanup] found {len(orphans)} orphaned director"
|
||||
f"{'y' if len(orphans) == 1 else 'ies'} older than {max_age_hours} h."
|
||||
)
|
||||
|
||||
removed = 0
|
||||
now = datetime.now(tz=UTC)
|
||||
for entry in orphans:
|
||||
age_h = int((now - datetime.fromtimestamp(entry.stat().st_mtime, tz=UTC)).total_seconds() // 3600)
|
||||
if dry_run:
|
||||
click.echo(f"[vm cleanup] would destroy {entry} (age: {age_h} h)")
|
||||
continue
|
||||
click.echo(f"[vm cleanup] processing: {entry}")
|
||||
vmx = _find_vmx(entry)
|
||||
if vmx is not None and backend is not None:
|
||||
handle = VmHandle(identifier=str(vmx))
|
||||
try:
|
||||
backend.stop(handle, hard=True)
|
||||
except BackendError as exc:
|
||||
click.echo(f"[vm cleanup] hard stop ignored: {exc}", err=True)
|
||||
time.sleep(2.0)
|
||||
try:
|
||||
backend.delete(handle)
|
||||
except BackendError as exc:
|
||||
click.echo(
|
||||
f"[vm cleanup] vmrun deleteVM failed: {exc}; falling back to dir removal.",
|
||||
err=True,
|
||||
)
|
||||
elif vmx is None:
|
||||
click.echo(
|
||||
f"[vm cleanup] no .vmx in {entry} - removing directory only.", err=True
|
||||
)
|
||||
|
||||
if _try_remove_dir(entry):
|
||||
click.echo(f"[vm cleanup] removed: {entry}")
|
||||
removed += 1
|
||||
else:
|
||||
click.echo(
|
||||
f"[vm cleanup] could not fully remove: {entry} - manual cleanup needed.",
|
||||
err=True,
|
||||
)
|
||||
|
||||
# Stale lock file cleanup (mirrors Cleanup-OrphanedBuildVMs.ps1).
|
||||
if lock_file is not None and lock_file.is_file():
|
||||
cutoff = datetime.now(tz=UTC) - timedelta(minutes=lock_max_age_minutes)
|
||||
mtime = datetime.fromtimestamp(lock_file.stat().st_mtime, tz=UTC)
|
||||
if mtime < cutoff and not dry_run:
|
||||
age_min = int((datetime.now(tz=UTC) - mtime).total_seconds() // 60)
|
||||
try:
|
||||
lock_file.unlink()
|
||||
click.echo(
|
||||
f"[vm cleanup] removed stale lock file (age: {age_min} min): {lock_file}"
|
||||
)
|
||||
except OSError as exc:
|
||||
click.echo(f"[vm cleanup] could not remove lock file: {exc}", err=True)
|
||||
|
||||
click.echo("[vm cleanup] done.")
|
||||
return removed
|
||||
|
||||
|
||||
@vm.command("cleanup")
|
||||
@click.option(
|
||||
"--clone-base-dir",
|
||||
@@ -348,14 +244,71 @@ def vm_cleanup(
|
||||
)
|
||||
backend = None
|
||||
|
||||
cleanup_orphans(
|
||||
lock_path = Path(lock_file) if lock_file else None
|
||||
|
||||
# Snapshot for messaging: cleanup_orphans() re-derives the same list
|
||||
# internally (nothing is mutated between the two calls above the
|
||||
# per-orphan destroy loop), so the ordering matches 1:1.
|
||||
orphans = ops_vms.list_orphans(base_path, max_age_hours)
|
||||
if not orphans:
|
||||
click.echo(
|
||||
f"[vm cleanup] no orphaned VMs found (threshold: {max_age_hours} h)."
|
||||
)
|
||||
else:
|
||||
click.echo(
|
||||
f"[vm cleanup] found {len(orphans)} orphaned director"
|
||||
f"{'y' if len(orphans) == 1 else 'ies'} older than {max_age_hours} h."
|
||||
)
|
||||
|
||||
lock_was_stale = False
|
||||
lock_age_min = 0
|
||||
if lock_path is not None and lock_path.is_file():
|
||||
cutoff = datetime.now(tz=UTC) - timedelta(minutes=30)
|
||||
mtime = datetime.fromtimestamp(lock_path.stat().st_mtime, tz=UTC)
|
||||
if mtime < cutoff:
|
||||
lock_was_stale = True
|
||||
lock_age_min = int((datetime.now(tz=UTC) - mtime).total_seconds() // 60)
|
||||
|
||||
results = ops_vms.cleanup_orphans(
|
||||
clone_base=base_path,
|
||||
max_age_hours=max_age_hours,
|
||||
backend=backend,
|
||||
dry_run=what_if,
|
||||
lock_file=Path(lock_file) if lock_file else None,
|
||||
lock_file=lock_path,
|
||||
)
|
||||
|
||||
for orphan, result in zip(orphans, results, strict=True):
|
||||
if what_if:
|
||||
click.echo(
|
||||
f"[vm cleanup] would destroy {orphan.path} (age: {orphan.age_hours} h)"
|
||||
)
|
||||
continue
|
||||
click.echo(f"[vm cleanup] processing: {orphan.path}")
|
||||
if orphan.vmx is None:
|
||||
click.echo(
|
||||
f"[vm cleanup] no .vmx in {orphan.path} - removing directory only.",
|
||||
err=True,
|
||||
)
|
||||
elif result.error is not None:
|
||||
click.echo(
|
||||
f"[vm cleanup] backend error on {orphan.path}: {result.error}",
|
||||
err=True,
|
||||
)
|
||||
if result.dir_removed:
|
||||
click.echo(f"[vm cleanup] removed: {orphan.path}")
|
||||
else:
|
||||
click.echo(
|
||||
f"[vm cleanup] could not fully remove: {orphan.path} - manual cleanup needed.",
|
||||
err=True,
|
||||
)
|
||||
|
||||
if lock_was_stale and not what_if and lock_path is not None and not lock_path.exists():
|
||||
click.echo(
|
||||
f"[vm cleanup] removed stale lock file (age: {lock_age_min} min): {lock_path}"
|
||||
)
|
||||
|
||||
click.echo("[vm cleanup] done.")
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────────── new
|
||||
|
||||
@@ -686,4 +639,4 @@ def vmware(
|
||||
)
|
||||
|
||||
|
||||
__all__ = ["cleanup_orphans", "vm", "vmware"]
|
||||
__all__ = ["vm", "vmware"]
|
||||
|
||||
@@ -0,0 +1,313 @@
|
||||
"""Pure VM-lifecycle operations shared by the CLI and (future) web UI.
|
||||
|
||||
Extracted from ``commands/vm.py`` (W1 of the web-UI plan). Zero ``click``,
|
||||
zero ``print`` — everything here returns a dataclass. Presentation stays in
|
||||
the CLI command wrappers (and, later, the web routes).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import contextlib
|
||||
import shutil
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from ci_orchestrator.backends.errors import BackendError
|
||||
from ci_orchestrator.backends.protocol import VmHandle
|
||||
|
||||
if TYPE_CHECKING: # pragma: no cover
|
||||
from ci_orchestrator.backends.protocol import VmBackend
|
||||
from ci_orchestrator.config import Config
|
||||
|
||||
# Matches the default ``vm cleanup --max-age-hours`` threshold; used to flag
|
||||
# a live VM as an orphan candidate in ``list_live_vms``.
|
||||
_ORPHAN_AGE_HOURS = 4
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class LiveVm:
|
||||
"""A currently-running VM as reported by the backend."""
|
||||
|
||||
name: str
|
||||
vmx: str
|
||||
ip: str | None
|
||||
guest_os: str
|
||||
uptime_sec: int | None
|
||||
is_orphan: bool
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Orphan:
|
||||
"""A clone directory older than the cleanup threshold."""
|
||||
|
||||
path: Path
|
||||
vmx: Path | None
|
||||
age_hours: int
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class RemovalResult:
|
||||
"""Outcome of destroying a single VM (used by both remove and cleanup)."""
|
||||
|
||||
target: str
|
||||
stopped: bool
|
||||
deleted: bool
|
||||
dir_removed: bool
|
||||
error: str | None
|
||||
|
||||
|
||||
# ────────────────────────────────────────────────────────────────── helpers
|
||||
|
||||
|
||||
def _try_remove_dir(path: Path) -> bool:
|
||||
"""Best-effort recursive directory removal. Returns True if path is gone."""
|
||||
if not path.exists():
|
||||
return True
|
||||
shutil.rmtree(path, ignore_errors=True)
|
||||
return not path.exists()
|
||||
|
||||
|
||||
def _find_vmx(clone_dir: Path) -> Path | None:
|
||||
for entry in clone_dir.iterdir():
|
||||
if entry.is_file() and entry.suffix.lower() == ".vmx":
|
||||
return entry
|
||||
return None
|
||||
|
||||
|
||||
def _is_under(path: Path, base: Path) -> bool:
|
||||
"""Return True if ``base`` is an ancestor of ``path`` (resolved)."""
|
||||
try:
|
||||
return base.resolve() in path.resolve().parents
|
||||
except OSError:
|
||||
return False
|
||||
|
||||
|
||||
def _read_guest_os(vmx: Path) -> str:
|
||||
"""Best-effort parse of the ``guestOS = "..."`` line from a VMX file."""
|
||||
try:
|
||||
text = vmx.read_text(encoding="utf-8", errors="replace")
|
||||
except OSError:
|
||||
return ""
|
||||
for line in text.splitlines():
|
||||
if "=" not in line:
|
||||
continue
|
||||
key, _, rhs = line.partition("=")
|
||||
if key.strip().lower() != "guestos":
|
||||
continue
|
||||
rhs = rhs.strip()
|
||||
if rhs.startswith('"') and rhs.endswith('"') and len(rhs) >= 2:
|
||||
return rhs[1:-1]
|
||||
return rhs
|
||||
return ""
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────── live VMs
|
||||
|
||||
|
||||
def list_live_vms(backend: VmBackend, config: Config) -> list[LiveVm]:
|
||||
"""Return every VM the backend reports as running, enriched for display.
|
||||
|
||||
Never raises on a single bad VM — a handle that can't be inspected is
|
||||
still returned with degraded fields rather than aborting the whole list.
|
||||
"""
|
||||
try:
|
||||
handles = backend.list_running()
|
||||
except BackendError:
|
||||
return []
|
||||
|
||||
now = datetime.now(tz=UTC)
|
||||
out: list[LiveVm] = []
|
||||
for h in handles:
|
||||
vmx = Path(h.identifier)
|
||||
name = h.name or vmx.stem
|
||||
|
||||
guest_os = _read_guest_os(vmx)
|
||||
|
||||
try:
|
||||
ip = backend.get_ip(h, timeout=0)
|
||||
except BackendError:
|
||||
ip = None
|
||||
|
||||
under_base = _is_under(vmx, config.paths.build_vms)
|
||||
|
||||
uptime_sec: int | None = None
|
||||
if under_base:
|
||||
try:
|
||||
mtime = datetime.fromtimestamp(vmx.parent.stat().st_mtime, tz=UTC)
|
||||
uptime_sec = int((now - mtime).total_seconds())
|
||||
except OSError:
|
||||
uptime_sec = None
|
||||
|
||||
is_orphan = False
|
||||
if under_base and uptime_sec is not None:
|
||||
age_hours = uptime_sec // 3600
|
||||
is_orphan = age_hours >= _ORPHAN_AGE_HOURS
|
||||
|
||||
out.append(
|
||||
LiveVm(
|
||||
name=name,
|
||||
vmx=str(vmx),
|
||||
ip=ip,
|
||||
guest_os=guest_os,
|
||||
uptime_sec=uptime_sec,
|
||||
is_orphan=is_orphan,
|
||||
)
|
||||
)
|
||||
return out
|
||||
|
||||
|
||||
# ───────────────────────────────────────────────────────────────── orphans
|
||||
|
||||
|
||||
def list_orphans(clone_base: Path, max_age_hours: int) -> list[Orphan]:
|
||||
"""Return orphan clone directories older than ``max_age_hours``."""
|
||||
if not clone_base.is_dir():
|
||||
return []
|
||||
cutoff = datetime.now(tz=UTC) - timedelta(hours=max_age_hours)
|
||||
now = datetime.now(tz=UTC)
|
||||
out: list[Orphan] = []
|
||||
for entry in clone_base.iterdir():
|
||||
try:
|
||||
if not entry.is_dir():
|
||||
continue
|
||||
mtime = datetime.fromtimestamp(entry.stat().st_mtime, tz=UTC)
|
||||
except OSError:
|
||||
continue
|
||||
if mtime < cutoff:
|
||||
age_hours = int((now - mtime).total_seconds() // 3600)
|
||||
out.append(Orphan(path=entry, vmx=_find_vmx(entry), age_hours=age_hours))
|
||||
return out
|
||||
|
||||
|
||||
# ───────────────────────────────────────────────────────────────── removal
|
||||
|
||||
|
||||
def remove_vm(backend: VmBackend, vmx: Path, *, dry_run: bool = False) -> RemovalResult:
|
||||
"""Stop and permanently delete a single VM. Pure extraction of the destroy sequence.
|
||||
|
||||
Swallows :class:`BackendError` from stop/delete (recorded in the result
|
||||
flags/``error``); never raises.
|
||||
"""
|
||||
target = str(vmx)
|
||||
if dry_run:
|
||||
return RemovalResult(target=target, stopped=False, deleted=False, dir_removed=False, error=None)
|
||||
|
||||
handle = VmHandle(identifier=target)
|
||||
first_error: str | None = None
|
||||
|
||||
stopped = False
|
||||
try:
|
||||
backend.stop(handle, hard=True)
|
||||
stopped = True
|
||||
except BackendError as exc:
|
||||
first_error = first_error or str(exc)
|
||||
|
||||
time.sleep(2.0)
|
||||
|
||||
deleted = False
|
||||
last_err: str | None = None
|
||||
for delay in (0, 3, 6):
|
||||
if delay > 0:
|
||||
time.sleep(delay)
|
||||
try:
|
||||
backend.delete(handle)
|
||||
deleted = True
|
||||
break
|
||||
except BackendError as exc:
|
||||
last_err = str(exc)
|
||||
if not deleted and last_err is not None:
|
||||
first_error = first_error or last_err
|
||||
|
||||
dir_removed = _try_remove_dir(vmx.parent)
|
||||
|
||||
return RemovalResult(
|
||||
target=target,
|
||||
stopped=stopped,
|
||||
deleted=deleted,
|
||||
dir_removed=dir_removed,
|
||||
error=first_error,
|
||||
)
|
||||
|
||||
|
||||
def cleanup_orphans(
|
||||
clone_base: Path,
|
||||
max_age_hours: int,
|
||||
backend: VmBackend | None,
|
||||
*,
|
||||
dry_run: bool = False,
|
||||
lock_file: Path | None = None,
|
||||
lock_max_age_minutes: int = 30,
|
||||
) -> list[RemovalResult]:
|
||||
"""Remove orphaned clone directories. Returns one :class:`RemovalResult` per orphan.
|
||||
|
||||
``backend`` is used to stop+delete each orphan's VMX before removing its
|
||||
directory. If the backend is unavailable (``None``), directory removal
|
||||
is still attempted.
|
||||
"""
|
||||
orphans = list_orphans(clone_base, max_age_hours)
|
||||
|
||||
results: list[RemovalResult] = []
|
||||
for orphan in orphans:
|
||||
entry = orphan.path
|
||||
target = str(orphan.vmx) if orphan.vmx is not None else str(entry)
|
||||
|
||||
if dry_run:
|
||||
results.append(
|
||||
RemovalResult(target=target, stopped=False, deleted=False, dir_removed=False, error=None)
|
||||
)
|
||||
continue
|
||||
|
||||
error: str | None = None
|
||||
stopped = False
|
||||
deleted = False
|
||||
|
||||
vmx = orphan.vmx
|
||||
if vmx is not None and backend is not None:
|
||||
handle = VmHandle(identifier=str(vmx))
|
||||
try:
|
||||
backend.stop(handle, hard=True)
|
||||
stopped = True
|
||||
except BackendError as exc:
|
||||
error = error or str(exc)
|
||||
time.sleep(2.0)
|
||||
try:
|
||||
backend.delete(handle)
|
||||
deleted = True
|
||||
except BackendError as exc:
|
||||
error = error or str(exc)
|
||||
|
||||
dir_removed = _try_remove_dir(entry)
|
||||
|
||||
results.append(
|
||||
RemovalResult(
|
||||
target=target,
|
||||
stopped=stopped,
|
||||
deleted=deleted,
|
||||
dir_removed=dir_removed,
|
||||
error=error,
|
||||
)
|
||||
)
|
||||
|
||||
# Stale lock file cleanup (mirrors Cleanup-OrphanedBuildVMs.ps1).
|
||||
if lock_file is not None and lock_file.is_file():
|
||||
cutoff = datetime.now(tz=UTC) - timedelta(minutes=lock_max_age_minutes)
|
||||
mtime = datetime.fromtimestamp(lock_file.stat().st_mtime, tz=UTC)
|
||||
if mtime < cutoff and not dry_run:
|
||||
with contextlib.suppress(OSError):
|
||||
lock_file.unlink()
|
||||
|
||||
return results
|
||||
|
||||
|
||||
__all__ = [
|
||||
"LiveVm",
|
||||
"Orphan",
|
||||
"RemovalResult",
|
||||
"cleanup_orphans",
|
||||
"list_live_vms",
|
||||
"list_orphans",
|
||||
"remove_vm",
|
||||
]
|
||||
@@ -378,110 +378,6 @@ def test_vm_remove_clone_dir_removal_failure(
|
||||
assert "manual cleanup needed" in result.output
|
||||
|
||||
|
||||
def test_list_orphans_non_dir_base(tmp_path: Path) -> None:
|
||||
f = tmp_path / "file.txt"
|
||||
f.write_text("x")
|
||||
assert vm_module._list_orphans(f, 4) == []
|
||||
|
||||
|
||||
def test_list_orphans_skips_non_dir_and_oserror(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
|
||||
base = tmp_path / "b"
|
||||
base.mkdir()
|
||||
(base / "a-file").write_text("not a dir")
|
||||
bad_dir = base / "bad"
|
||||
bad_dir.mkdir()
|
||||
_age_dir(bad_dir, hours=10)
|
||||
|
||||
real_stat = Path.stat
|
||||
|
||||
def _stat(self: Path, *a: Any, **kw: Any) -> Any:
|
||||
if self == bad_dir:
|
||||
raise OSError("nope")
|
||||
return real_stat(self, *a, **kw)
|
||||
|
||||
monkeypatch.setattr(Path, "stat", _stat)
|
||||
assert vm_module._list_orphans(base, 4) == []
|
||||
|
||||
|
||||
def test_find_vmx_returns_none(tmp_path: Path) -> None:
|
||||
d = tmp_path / "novmx"
|
||||
d.mkdir()
|
||||
(d / "readme.txt").write_text("x")
|
||||
assert vm_module._find_vmx(d) is None
|
||||
|
||||
|
||||
def test_cleanup_orphans_dir_without_vmx_logs_and_removes(tmp_path: Path) -> None:
|
||||
base = tmp_path / "b"
|
||||
base.mkdir()
|
||||
no_vmx = base / "no-vmx"
|
||||
no_vmx.mkdir()
|
||||
_age_dir(no_vmx, hours=10)
|
||||
runner_count = vm_module.cleanup_orphans(
|
||||
clone_base=base, max_age_hours=4, backend=None
|
||||
)
|
||||
assert runner_count == 1
|
||||
assert not no_vmx.exists()
|
||||
|
||||
|
||||
def test_cleanup_orphans_backend_stop_error_swallowed(
|
||||
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
||||
) -> None:
|
||||
from ci_orchestrator.backends.errors import BackendOperationFailed
|
||||
|
||||
base = tmp_path / "b"
|
||||
base.mkdir()
|
||||
old, _ = _make_clone_dir(base, "old1")
|
||||
_age_dir(old, hours=10)
|
||||
|
||||
class _B:
|
||||
def stop(self, _h: VmHandle, hard: bool = False) -> None:
|
||||
raise BackendOperationFailed("stop", 1, "no")
|
||||
|
||||
def delete(self, _h: VmHandle) -> None:
|
||||
raise BackendOperationFailed("delete", 1, "no")
|
||||
|
||||
count = vm_module.cleanup_orphans(
|
||||
clone_base=base, max_age_hours=4, backend=_B()
|
||||
)
|
||||
assert count == 1
|
||||
|
||||
|
||||
def test_cleanup_orphans_dir_removal_failure_logs(
|
||||
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
||||
) -> None:
|
||||
base = tmp_path / "b"
|
||||
base.mkdir()
|
||||
old, _ = _make_clone_dir(base, "stuck")
|
||||
_age_dir(old, hours=10)
|
||||
monkeypatch.setattr(vm_module, "_try_remove_dir", lambda _p: False)
|
||||
count = vm_module.cleanup_orphans(
|
||||
clone_base=base, max_age_hours=4, backend=None
|
||||
)
|
||||
assert count == 0
|
||||
|
||||
|
||||
def test_cleanup_orphans_lock_unlink_oserror(
|
||||
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
||||
) -> None:
|
||||
base = tmp_path / "b"
|
||||
base.mkdir()
|
||||
lock = tmp_path / "stale.lock"
|
||||
lock.write_text("x")
|
||||
_age_dir(lock, hours=1)
|
||||
|
||||
real_unlink = Path.unlink
|
||||
|
||||
def _boom(self: Path, *a: Any, **kw: Any) -> Any:
|
||||
if self == lock:
|
||||
raise OSError("locked")
|
||||
return real_unlink(self, *a, **kw)
|
||||
|
||||
monkeypatch.setattr(Path, "unlink", _boom)
|
||||
vm_module.cleanup_orphans(
|
||||
clone_base=base, max_age_hours=4, backend=None, lock_file=lock
|
||||
)
|
||||
|
||||
|
||||
def test_vm_cleanup_backend_unavailable(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
|
||||
from ci_orchestrator.backends.errors import BackendNotAvailable
|
||||
|
||||
|
||||
@@ -0,0 +1,447 @@
|
||||
"""Tests for the pure ``operations.vms`` layer (W1 of the web-UI plan).
|
||||
|
||||
Migrated from the direct-call tests that used to live in
|
||||
``test_commands_vm.py`` (``_list_orphans`` / ``_find_vmx`` /
|
||||
``cleanup_orphans``), adapted to the new dataclass return types. Adds
|
||||
coverage for ``list_live_vms`` and ``remove_vm``.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import time
|
||||
from datetime import datetime, timedelta
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
|
||||
import ci_orchestrator.operations.vms as vms_module
|
||||
from ci_orchestrator.backends.errors import BackendOperationFailed
|
||||
from ci_orchestrator.backends.protocol import VmHandle
|
||||
from ci_orchestrator.config import BackendConfig, Config, Paths
|
||||
from ci_orchestrator.operations.vms import (
|
||||
Orphan,
|
||||
RemovalResult,
|
||||
cleanup_orphans,
|
||||
list_live_vms,
|
||||
list_orphans,
|
||||
remove_vm,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _no_sleep(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setattr(time, "sleep", lambda _s: None)
|
||||
monkeypatch.setattr(vms_module.time, "sleep", lambda _s: None)
|
||||
|
||||
|
||||
def _age_dir(path: Path, hours: float) -> None:
|
||||
ts = (datetime.now() - timedelta(hours=hours)).timestamp()
|
||||
import os
|
||||
|
||||
os.utime(path, (ts, ts))
|
||||
|
||||
|
||||
def _make_clone_dir(tmp_path: Path, name: str = "clone") -> tuple[Path, Path]:
|
||||
clone_dir = tmp_path / name
|
||||
clone_dir.mkdir()
|
||||
vmx = clone_dir / f"{name}.vmx"
|
||||
vmx.write_text('config.version = "8"', encoding="utf-8")
|
||||
return clone_dir, vmx
|
||||
|
||||
|
||||
def _make_config(build_vms: Path) -> Config:
|
||||
return Config(
|
||||
paths=Paths(
|
||||
root=build_vms.parent,
|
||||
templates=build_vms.parent / "templates",
|
||||
build_vms=build_vms,
|
||||
artifacts=build_vms.parent / "artifacts",
|
||||
keys=build_vms.parent / "keys",
|
||||
),
|
||||
backend=BackendConfig(),
|
||||
)
|
||||
|
||||
|
||||
class _FakeBackend:
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
running_after_stop: bool = False,
|
||||
delete_fails_first: int = 0,
|
||||
) -> None:
|
||||
self.calls: list[tuple[str, Any]] = []
|
||||
self._running = True
|
||||
self._running_after_stop = running_after_stop
|
||||
self._delete_fails_remaining = delete_fails_first
|
||||
|
||||
def stop(self, handle: VmHandle, hard: bool = False) -> None:
|
||||
self.calls.append(("stop", hard))
|
||||
if not hard or not self._running_after_stop:
|
||||
self._running = False
|
||||
|
||||
def is_running(self, _h: VmHandle) -> bool:
|
||||
return self._running
|
||||
|
||||
def delete(self, _h: VmHandle) -> None:
|
||||
self.calls.append(("delete", None))
|
||||
if self._delete_fails_remaining > 0:
|
||||
self._delete_fails_remaining -= 1
|
||||
raise BackendOperationFailed("deleteVM", 1, "transient")
|
||||
|
||||
|
||||
# ── list_orphans ────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_list_orphans_non_dir_base(tmp_path: Path) -> None:
|
||||
f = tmp_path / "file.txt"
|
||||
f.write_text("x")
|
||||
assert list_orphans(f, 4) == []
|
||||
|
||||
|
||||
def test_list_orphans_skips_non_dir_and_oserror(
|
||||
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
||||
) -> None:
|
||||
base = tmp_path / "b"
|
||||
base.mkdir()
|
||||
(base / "a-file").write_text("not a dir")
|
||||
bad_dir = base / "bad"
|
||||
bad_dir.mkdir()
|
||||
_age_dir(bad_dir, hours=10)
|
||||
|
||||
real_stat = Path.stat
|
||||
|
||||
def _stat(self: Path, *a: Any, **kw: Any) -> Any:
|
||||
if self == bad_dir:
|
||||
raise OSError("nope")
|
||||
return real_stat(self, *a, **kw)
|
||||
|
||||
monkeypatch.setattr(Path, "stat", _stat)
|
||||
assert list_orphans(base, 4) == []
|
||||
|
||||
|
||||
def test_list_orphans_returns_orphan_dataclass(tmp_path: Path) -> None:
|
||||
base = tmp_path / "b"
|
||||
base.mkdir()
|
||||
old, vmx = _make_clone_dir(base, "old")
|
||||
_age_dir(old, hours=10)
|
||||
|
||||
orphans = list_orphans(base, 4)
|
||||
assert orphans == [Orphan(path=old, vmx=vmx, age_hours=10)]
|
||||
|
||||
|
||||
# ── _find_vmx ────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_find_vmx_returns_none(tmp_path: Path) -> None:
|
||||
d = tmp_path / "novmx"
|
||||
d.mkdir()
|
||||
(d / "readme.txt").write_text("x")
|
||||
assert vms_module._find_vmx(d) is None
|
||||
|
||||
|
||||
# ── cleanup_orphans ────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_cleanup_orphans_dir_without_vmx_logs_and_removes(tmp_path: Path) -> None:
|
||||
base = tmp_path / "b"
|
||||
base.mkdir()
|
||||
no_vmx = base / "no-vmx"
|
||||
no_vmx.mkdir()
|
||||
_age_dir(no_vmx, hours=10)
|
||||
results = cleanup_orphans(clone_base=base, max_age_hours=4, backend=None)
|
||||
assert sum(r.dir_removed for r in results) == 1
|
||||
assert not no_vmx.exists()
|
||||
|
||||
|
||||
def test_cleanup_orphans_backend_stop_error_swallowed(tmp_path: Path) -> None:
|
||||
base = tmp_path / "b"
|
||||
base.mkdir()
|
||||
old, _ = _make_clone_dir(base, "old1")
|
||||
_age_dir(old, hours=10)
|
||||
|
||||
class _B:
|
||||
def stop(self, _h: VmHandle, hard: bool = False) -> None:
|
||||
raise BackendOperationFailed("stop", 1, "no")
|
||||
|
||||
def delete(self, _h: VmHandle) -> None:
|
||||
raise BackendOperationFailed("delete", 1, "no")
|
||||
|
||||
results = cleanup_orphans(clone_base=base, max_age_hours=4, backend=_B())
|
||||
assert sum(r.dir_removed for r in results) == 1
|
||||
assert results[0].stopped is False
|
||||
assert results[0].deleted is False
|
||||
assert results[0].error is not None
|
||||
|
||||
|
||||
def test_cleanup_orphans_dir_removal_failure_logs(
|
||||
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
||||
) -> None:
|
||||
base = tmp_path / "b"
|
||||
base.mkdir()
|
||||
old, _ = _make_clone_dir(base, "stuck")
|
||||
_age_dir(old, hours=10)
|
||||
monkeypatch.setattr(vms_module, "_try_remove_dir", lambda _p: False)
|
||||
results = cleanup_orphans(clone_base=base, max_age_hours=4, backend=None)
|
||||
assert sum(r.dir_removed for r in results) == 0
|
||||
assert len(results) == 1
|
||||
|
||||
|
||||
def test_cleanup_orphans_lock_unlink_oserror(
|
||||
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
||||
) -> None:
|
||||
base = tmp_path / "b"
|
||||
base.mkdir()
|
||||
lock = tmp_path / "stale.lock"
|
||||
lock.write_text("x")
|
||||
_age_dir(lock, hours=1)
|
||||
|
||||
real_unlink = Path.unlink
|
||||
|
||||
def _boom(self: Path, *a: Any, **kw: Any) -> Any:
|
||||
if self == lock:
|
||||
raise OSError("locked")
|
||||
return real_unlink(self, *a, **kw)
|
||||
|
||||
monkeypatch.setattr(Path, "unlink", _boom)
|
||||
# Must not raise even though unlink() fails.
|
||||
cleanup_orphans(clone_base=base, max_age_hours=4, backend=None, lock_file=lock)
|
||||
|
||||
|
||||
def test_cleanup_orphans_dry_run_no_destroy(tmp_path: Path) -> None:
|
||||
base = tmp_path / "b"
|
||||
base.mkdir()
|
||||
old, vmx = _make_clone_dir(base, "old")
|
||||
_age_dir(old, hours=10)
|
||||
backend = _FakeBackend()
|
||||
results = cleanup_orphans(clone_base=base, max_age_hours=4, backend=backend, dry_run=True)
|
||||
assert results == [RemovalResult(target=str(vmx), stopped=False, deleted=False, dir_removed=False, error=None)]
|
||||
assert backend.calls == []
|
||||
assert old.exists()
|
||||
|
||||
|
||||
def test_cleanup_orphans_no_orphans_returns_empty(tmp_path: Path) -> None:
|
||||
base = tmp_path / "b"
|
||||
base.mkdir()
|
||||
assert cleanup_orphans(clone_base=base, max_age_hours=4, backend=None) == []
|
||||
|
||||
|
||||
def test_cleanup_orphans_lock_respects_dry_run(tmp_path: Path) -> None:
|
||||
base = tmp_path / "b"
|
||||
base.mkdir()
|
||||
lock = tmp_path / "stale.lock"
|
||||
lock.write_text("x")
|
||||
_age_dir(lock, hours=1)
|
||||
cleanup_orphans(clone_base=base, max_age_hours=4, backend=None, dry_run=True, lock_file=lock)
|
||||
assert lock.exists() # dry run never touches the lock file
|
||||
|
||||
|
||||
# ── list_live_vms ────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
class _FakeLiveBackend:
|
||||
def __init__(
|
||||
self,
|
||||
handles: list[VmHandle],
|
||||
*,
|
||||
ip: str | None = "192.168.1.50",
|
||||
ip_raises: bool = False,
|
||||
list_running_raises: bool = False,
|
||||
) -> None:
|
||||
self._handles = handles
|
||||
self._ip = ip
|
||||
self._ip_raises = ip_raises
|
||||
self._list_running_raises = list_running_raises
|
||||
|
||||
def list_running(self) -> list[VmHandle]:
|
||||
if self._list_running_raises:
|
||||
raise BackendOperationFailed("list", 1, "boom")
|
||||
return self._handles
|
||||
|
||||
def get_ip(self, _h: VmHandle, timeout: float = 0.0) -> str | None:
|
||||
if self._ip_raises:
|
||||
raise BackendOperationFailed("getGuestIPAddress", 1, "no tools")
|
||||
return self._ip
|
||||
|
||||
|
||||
def test_list_live_vms_orphan_flag_on_old_clone(tmp_path: Path) -> None:
|
||||
build_vms = tmp_path / "build-vms"
|
||||
build_vms.mkdir()
|
||||
clone_dir, vmx = _make_clone_dir(build_vms, "old")
|
||||
vmx.write_text('guestOS = "ubuntu-64"', encoding="utf-8")
|
||||
_age_dir(clone_dir, hours=10)
|
||||
|
||||
backend = _FakeLiveBackend([VmHandle(identifier=str(vmx), name="old")])
|
||||
config = _make_config(build_vms)
|
||||
|
||||
result = list_live_vms(backend, config) # type: ignore[arg-type]
|
||||
assert len(result) == 1
|
||||
vm = result[0]
|
||||
assert vm.name == "old"
|
||||
assert vm.vmx == str(vmx)
|
||||
assert vm.ip == "192.168.1.50"
|
||||
assert vm.guest_os == "ubuntu-64"
|
||||
assert vm.uptime_sec is not None and vm.uptime_sec >= 9 * 3600
|
||||
assert vm.is_orphan is True
|
||||
|
||||
|
||||
def test_list_live_vms_fresh_clone_not_orphan(tmp_path: Path) -> None:
|
||||
build_vms = tmp_path / "build-vms"
|
||||
build_vms.mkdir()
|
||||
clone_dir, vmx = _make_clone_dir(build_vms, "fresh")
|
||||
_age_dir(clone_dir, hours=0)
|
||||
|
||||
backend = _FakeLiveBackend([VmHandle(identifier=str(vmx), name=None)])
|
||||
config = _make_config(build_vms)
|
||||
|
||||
result = list_live_vms(backend, config) # type: ignore[arg-type]
|
||||
assert len(result) == 1
|
||||
vm = result[0]
|
||||
assert vm.name == vmx.stem # name fell back to vmx stem
|
||||
assert vm.is_orphan is False
|
||||
|
||||
|
||||
def test_list_live_vms_outside_build_vms_not_orphan(tmp_path: Path) -> None:
|
||||
build_vms = tmp_path / "build-vms"
|
||||
build_vms.mkdir()
|
||||
(tmp_path / "elsewhere").mkdir()
|
||||
elsewhere, vmx = _make_clone_dir(tmp_path / "elsewhere", "other")
|
||||
_age_dir(elsewhere, hours=10)
|
||||
|
||||
backend = _FakeLiveBackend([VmHandle(identifier=str(vmx), name="other")])
|
||||
config = _make_config(build_vms)
|
||||
|
||||
result = list_live_vms(backend, config) # type: ignore[arg-type]
|
||||
assert len(result) == 1
|
||||
vm = result[0]
|
||||
assert vm.is_orphan is False
|
||||
assert vm.uptime_sec is None
|
||||
|
||||
|
||||
def test_list_live_vms_get_ip_none(tmp_path: Path) -> None:
|
||||
build_vms = tmp_path / "build-vms"
|
||||
build_vms.mkdir()
|
||||
_, vmx = _make_clone_dir(build_vms, "noip")
|
||||
|
||||
backend = _FakeLiveBackend([VmHandle(identifier=str(vmx))], ip=None)
|
||||
config = _make_config(build_vms)
|
||||
|
||||
result = list_live_vms(backend, config) # type: ignore[arg-type]
|
||||
assert result[0].ip is None
|
||||
|
||||
|
||||
def test_list_live_vms_get_ip_raises_is_tolerated(tmp_path: Path) -> None:
|
||||
build_vms = tmp_path / "build-vms"
|
||||
build_vms.mkdir()
|
||||
_, vmx = _make_clone_dir(build_vms, "iperr")
|
||||
|
||||
backend = _FakeLiveBackend([VmHandle(identifier=str(vmx))], ip_raises=True)
|
||||
config = _make_config(build_vms)
|
||||
|
||||
result = list_live_vms(backend, config) # type: ignore[arg-type]
|
||||
assert result[0].ip is None
|
||||
|
||||
|
||||
def test_list_live_vms_list_running_raises_returns_empty(tmp_path: Path) -> None:
|
||||
build_vms = tmp_path / "build-vms"
|
||||
build_vms.mkdir()
|
||||
backend = _FakeLiveBackend([], list_running_raises=True)
|
||||
config = _make_config(build_vms)
|
||||
assert list_live_vms(backend, config) == [] # type: ignore[arg-type]
|
||||
|
||||
|
||||
def test_list_live_vms_missing_vmx_file_guest_os_empty(tmp_path: Path) -> None:
|
||||
build_vms = tmp_path / "build-vms"
|
||||
build_vms.mkdir()
|
||||
missing_vmx = build_vms / "ghost" / "ghost.vmx" # parent dir doesn't even exist
|
||||
|
||||
backend = _FakeLiveBackend([VmHandle(identifier=str(missing_vmx))])
|
||||
config = _make_config(build_vms)
|
||||
|
||||
result = list_live_vms(backend, config) # type: ignore[arg-type]
|
||||
assert result[0].guest_os == ""
|
||||
|
||||
|
||||
def test_list_live_vms_empty_when_nothing_running(tmp_path: Path) -> None:
|
||||
build_vms = tmp_path / "build-vms"
|
||||
build_vms.mkdir()
|
||||
backend = _FakeLiveBackend([])
|
||||
config = _make_config(build_vms)
|
||||
assert list_live_vms(backend, config) == [] # type: ignore[arg-type]
|
||||
|
||||
|
||||
# ── remove_vm ────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_remove_vm_happy_path(tmp_path: Path) -> None:
|
||||
clone_dir, vmx = _make_clone_dir(tmp_path, "live")
|
||||
backend = _FakeBackend()
|
||||
result = remove_vm(backend, vmx) # type: ignore[arg-type]
|
||||
assert result == RemovalResult(
|
||||
target=str(vmx), stopped=True, deleted=True, dir_removed=True, error=None
|
||||
)
|
||||
assert not clone_dir.exists()
|
||||
assert ("delete", None) in backend.calls
|
||||
|
||||
|
||||
def test_remove_vm_dry_run_no_op(tmp_path: Path) -> None:
|
||||
clone_dir, vmx = _make_clone_dir(tmp_path, "dry")
|
||||
backend = _FakeBackend()
|
||||
result = remove_vm(backend, vmx, dry_run=True) # type: ignore[arg-type]
|
||||
assert result == RemovalResult(
|
||||
target=str(vmx), stopped=False, deleted=False, dir_removed=False, error=None
|
||||
)
|
||||
assert backend.calls == []
|
||||
assert clone_dir.exists()
|
||||
|
||||
|
||||
def test_remove_vm_delete_retry_succeeds(tmp_path: Path) -> None:
|
||||
clone_dir, vmx = _make_clone_dir(tmp_path, "retry")
|
||||
backend = _FakeBackend(delete_fails_first=2)
|
||||
result = remove_vm(backend, vmx) # type: ignore[arg-type]
|
||||
delete_calls = [c for c in backend.calls if c[0] == "delete"]
|
||||
assert len(delete_calls) == 3 # 2 failures + 1 success
|
||||
assert result.deleted is True
|
||||
assert not clone_dir.exists()
|
||||
|
||||
|
||||
def test_remove_vm_delete_fails_all_retries(tmp_path: Path) -> None:
|
||||
clone_dir, vmx = _make_clone_dir(tmp_path, "neverdeletes")
|
||||
|
||||
class _B:
|
||||
def stop(self, _h: VmHandle, hard: bool = False) -> None:
|
||||
pass
|
||||
|
||||
def delete(self, _h: VmHandle) -> None:
|
||||
raise BackendOperationFailed("delete", 1, "always")
|
||||
|
||||
result = remove_vm(_B(), vmx) # type: ignore[arg-type]
|
||||
assert result.deleted is False
|
||||
assert result.error is not None
|
||||
# Directory removal is still attempted even when delete fails.
|
||||
assert not clone_dir.exists()
|
||||
assert result.dir_removed is True
|
||||
|
||||
|
||||
def test_remove_vm_stop_error_is_swallowed(tmp_path: Path) -> None:
|
||||
_, vmx = _make_clone_dir(tmp_path, "stoperr")
|
||||
|
||||
class _B:
|
||||
def stop(self, _h: VmHandle, hard: bool = False) -> None:
|
||||
raise BackendOperationFailed("stop", 1, "no")
|
||||
|
||||
def delete(self, _h: VmHandle) -> None:
|
||||
pass
|
||||
|
||||
result = remove_vm(_B(), vmx) # type: ignore[arg-type]
|
||||
assert result.stopped is False
|
||||
assert result.deleted is True
|
||||
assert result.error is not None
|
||||
|
||||
|
||||
def test_remove_vm_dir_removal_failure(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
|
||||
_, vmx = _make_clone_dir(tmp_path, "stubborn")
|
||||
backend = _FakeBackend()
|
||||
monkeypatch.setattr(vms_module, "_try_remove_dir", lambda _p: False)
|
||||
result = remove_vm(backend, vmx) # type: ignore[arg-type]
|
||||
assert result.dir_removed is False
|
||||
Reference in New Issue
Block a user