Simone
43a69b82db
- Add `retention run` command (ports Invoke-RetentionPolicy.ps1): purges old artifact/log dirs with aggressive mode when disk space is low. - Add `template backup` command (ports Backup-CITemplate.ps1): 7z -mx=1 compressed archives in /var/lib/ci/backups/, prunes to keep-count=3, stops/restarts act-runner.service around the copy. - Update ci-retention-policy.service and ci-backup-template.service to use Python; pwsh is no longer required on the Linux host. - Fix ci-watch-runner-health.service: pass --service-name act-runner (Linux service name, not Windows act_runner default). - Fix _list_orphans in vm.py: wrap is_dir() inside the OSError try block so a stat() failure on an entry is silently skipped rather than raised. - Mark B5 complete in PhaseB-user-checklist.md. Co-Authored-By: Claude Haiku 4.5 <noreply@anthropic.com>
487 lines
16 KiB
Python
487 lines
16 KiB
Python
"""``vm`` sub-commands.
|
|
|
|
Phase A2 ports the leaf VM management scripts:
|
|
|
|
* ``vm remove`` — replaces ``scripts/Remove-BuildVM.ps1``
|
|
* ``vm cleanup`` — replaces ``scripts/Cleanup-OrphanedBuildVMs.ps1``
|
|
|
|
Phase C hook: ``vm cleanup`` accepts an optional :class:`VmBackend`
|
|
dependency so an ESXi backend can later supply a folder-scoped VM list
|
|
instead of scanning the local filesystem.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import shutil
|
|
import time
|
|
from datetime import UTC, datetime, timedelta
|
|
from pathlib import Path
|
|
from typing import TYPE_CHECKING
|
|
|
|
import click
|
|
|
|
from ci_orchestrator.backends import load_backend
|
|
from ci_orchestrator.backends.errors import BackendError, BackendNotAvailable
|
|
from ci_orchestrator.backends.protocol import VmHandle
|
|
from ci_orchestrator.config import load_config
|
|
|
|
if TYPE_CHECKING: # pragma: no cover
|
|
from ci_orchestrator.backends.protocol import VmBackend
|
|
|
|
|
|
# ────────────────────────────────────────────────────────────────────── helpers
|
|
|
|
|
|
def _make_backend(vmrun_path: str | None) -> VmBackend:
|
|
"""Build a backend honouring an optional vmrun override."""
|
|
config = load_config()
|
|
if vmrun_path is not None:
|
|
from ci_orchestrator.backends.workstation import WorkstationVmrunBackend
|
|
|
|
return WorkstationVmrunBackend(vmrun_path=vmrun_path)
|
|
return load_backend(config)
|
|
|
|
|
|
def _try_remove_dir(path: Path) -> bool:
|
|
"""Best-effort recursive directory removal. Returns True if path is gone."""
|
|
if not path.exists():
|
|
return True
|
|
shutil.rmtree(path, ignore_errors=True)
|
|
return not path.exists()
|
|
|
|
|
|
def _stop_with_fallback(backend: VmBackend, handle: VmHandle) -> None:
|
|
"""Soft-stop, then hard-stop on failure. Both errors are tolerated."""
|
|
try:
|
|
backend.stop(handle, hard=False)
|
|
except BackendError as exc:
|
|
click.echo(f"[vm remove] soft stop failed: {exc}", err=True)
|
|
try:
|
|
if backend.is_running(handle):
|
|
click.echo("[vm remove] still running, forcing hard stop...")
|
|
backend.stop(handle, hard=True)
|
|
except BackendError as exc:
|
|
click.echo(f"[vm remove] hard stop failed: {exc}", err=True)
|
|
|
|
|
|
# ─────────────────────────────────────────────────────────────────────── group
|
|
|
|
|
|
@click.group("vm")
|
|
def vm() -> None:
|
|
"""VM lifecycle commands."""
|
|
|
|
|
|
# ─────────────────────────────────────────────────────────────────────── remove
|
|
|
|
|
|
@vm.command("remove")
|
|
@click.option(
|
|
"--vmx",
|
|
"--vm-path",
|
|
"vmx",
|
|
required=True,
|
|
help="Path to the clone VMX file to destroy.",
|
|
)
|
|
@click.option("--vmrun-path", "vmrun_path", default=None, help="Override vmrun.exe path.")
|
|
@click.option(
|
|
"--graceful-timeout",
|
|
"--graceful-timeout-seconds",
|
|
"graceful_timeout",
|
|
type=click.IntRange(1, 60),
|
|
default=15,
|
|
show_default=True,
|
|
help="Seconds to wait for soft stop before forcing.",
|
|
)
|
|
@click.option(
|
|
"--force",
|
|
is_flag=True,
|
|
default=False,
|
|
help="Skip soft stop and go straight to hard stop.",
|
|
)
|
|
def vm_remove(
|
|
vmx: str,
|
|
vmrun_path: str | None,
|
|
graceful_timeout: int,
|
|
force: bool,
|
|
) -> None:
|
|
"""Stop and permanently delete an ephemeral build VM."""
|
|
vmx_path = Path(vmx)
|
|
clone_dir = vmx_path.parent
|
|
click.echo(f"[vm remove] destroying VM: {vmx}")
|
|
|
|
if not vmx_path.is_file():
|
|
click.echo(
|
|
f"[vm remove] VMX file not found - nothing to remove: {vmx}",
|
|
err=True,
|
|
)
|
|
# Still try to clean up a partial clone directory.
|
|
if clone_dir.exists() and clone_dir != vmx_path:
|
|
_try_remove_dir(clone_dir)
|
|
return
|
|
|
|
try:
|
|
backend = _make_backend(vmrun_path)
|
|
except BackendNotAvailable as exc:
|
|
click.echo(
|
|
f"[vm remove] vmrun unavailable ({exc}); attempting directory removal only.",
|
|
err=True,
|
|
)
|
|
_try_remove_dir(clone_dir)
|
|
return
|
|
|
|
handle = VmHandle(identifier=vmx)
|
|
|
|
if force:
|
|
try:
|
|
backend.stop(handle, hard=True)
|
|
except BackendError as exc:
|
|
click.echo(f"[vm remove] hard stop failed: {exc}", err=True)
|
|
else:
|
|
click.echo(f"[vm remove] sending soft stop (timeout: {graceful_timeout}s)...")
|
|
_stop_with_fallback(backend, handle)
|
|
|
|
# Brief settle: VMware may keep file locks for a moment after stop.
|
|
deadline = time.monotonic() + 20.0
|
|
while time.monotonic() < deadline:
|
|
try:
|
|
if not backend.is_running(handle):
|
|
break
|
|
except BackendError:
|
|
break
|
|
time.sleep(2.0)
|
|
|
|
click.echo("[vm remove] deleting VM...")
|
|
deleted = False
|
|
last_err: str = ""
|
|
for delay in (0, 3, 6):
|
|
if delay > 0:
|
|
click.echo(f"[vm remove] retrying deleteVM in {delay}s...")
|
|
time.sleep(delay)
|
|
try:
|
|
backend.delete(handle)
|
|
deleted = True
|
|
break
|
|
except BackendError as exc:
|
|
last_err = str(exc)
|
|
|
|
if not deleted:
|
|
click.echo(
|
|
f"[vm remove] vmrun deleteVM failed after retries: {last_err}", err=True
|
|
)
|
|
click.echo("[vm remove] falling back to manual directory removal.")
|
|
|
|
if clone_dir.exists():
|
|
click.echo(f"[vm remove] removing clone directory: {clone_dir}")
|
|
if _try_remove_dir(clone_dir):
|
|
click.echo("[vm remove] clone directory removed.")
|
|
else:
|
|
click.echo(
|
|
f"[vm remove] could not fully remove: {clone_dir} - manual cleanup needed.",
|
|
err=True,
|
|
)
|
|
|
|
click.echo("[vm remove] VM destruction complete.")
|
|
|
|
|
|
# ────────────────────────────────────────────────────────────────────── cleanup
|
|
|
|
|
|
def _list_orphans(clone_base: Path, max_age_hours: int) -> list[Path]:
|
|
"""Return orphan clone directories older than ``max_age_hours``."""
|
|
if not clone_base.is_dir():
|
|
return []
|
|
cutoff = datetime.now(tz=UTC) - timedelta(hours=max_age_hours)
|
|
out: list[Path] = []
|
|
for entry in clone_base.iterdir():
|
|
try:
|
|
if not entry.is_dir():
|
|
continue
|
|
mtime = datetime.fromtimestamp(entry.stat().st_mtime, tz=UTC)
|
|
except OSError:
|
|
continue
|
|
if mtime < cutoff:
|
|
out.append(entry)
|
|
return out
|
|
|
|
|
|
def _find_vmx(clone_dir: Path) -> Path | None:
|
|
for entry in clone_dir.iterdir():
|
|
if entry.is_file() and entry.suffix.lower() == ".vmx":
|
|
return entry
|
|
return None
|
|
|
|
|
|
def cleanup_orphans(
|
|
clone_base: Path,
|
|
max_age_hours: int,
|
|
backend: VmBackend | None,
|
|
*,
|
|
dry_run: bool = False,
|
|
lock_file: Path | None = None,
|
|
lock_max_age_minutes: int = 30,
|
|
) -> int:
|
|
"""Remove orphaned clone directories. Returns count removed.
|
|
|
|
Phase C hook: ``backend`` is used to stop+delete each orphan. If the
|
|
backend is unavailable, directory removal is still attempted.
|
|
"""
|
|
orphans = _list_orphans(clone_base, max_age_hours)
|
|
if not orphans:
|
|
click.echo(
|
|
f"[vm cleanup] no orphaned VMs found (threshold: {max_age_hours} h)."
|
|
)
|
|
else:
|
|
click.echo(
|
|
f"[vm cleanup] found {len(orphans)} orphaned director"
|
|
f"{'y' if len(orphans) == 1 else 'ies'} older than {max_age_hours} h."
|
|
)
|
|
|
|
removed = 0
|
|
now = datetime.now(tz=UTC)
|
|
for entry in orphans:
|
|
age_h = int((now - datetime.fromtimestamp(entry.stat().st_mtime, tz=UTC)).total_seconds() // 3600)
|
|
if dry_run:
|
|
click.echo(f"[vm cleanup] would destroy {entry} (age: {age_h} h)")
|
|
continue
|
|
click.echo(f"[vm cleanup] processing: {entry}")
|
|
vmx = _find_vmx(entry)
|
|
if vmx is not None and backend is not None:
|
|
handle = VmHandle(identifier=str(vmx))
|
|
try:
|
|
backend.stop(handle, hard=True)
|
|
except BackendError as exc:
|
|
click.echo(f"[vm cleanup] hard stop ignored: {exc}", err=True)
|
|
time.sleep(2.0)
|
|
try:
|
|
backend.delete(handle)
|
|
except BackendError as exc:
|
|
click.echo(
|
|
f"[vm cleanup] vmrun deleteVM failed: {exc}; falling back to dir removal.",
|
|
err=True,
|
|
)
|
|
elif vmx is None:
|
|
click.echo(
|
|
f"[vm cleanup] no .vmx in {entry} - removing directory only.", err=True
|
|
)
|
|
|
|
if _try_remove_dir(entry):
|
|
click.echo(f"[vm cleanup] removed: {entry}")
|
|
removed += 1
|
|
else:
|
|
click.echo(
|
|
f"[vm cleanup] could not fully remove: {entry} - manual cleanup needed.",
|
|
err=True,
|
|
)
|
|
|
|
# Stale lock file cleanup (mirrors Cleanup-OrphanedBuildVMs.ps1).
|
|
if lock_file is not None and lock_file.is_file():
|
|
cutoff = datetime.now(tz=UTC) - timedelta(minutes=lock_max_age_minutes)
|
|
mtime = datetime.fromtimestamp(lock_file.stat().st_mtime, tz=UTC)
|
|
if mtime < cutoff and not dry_run:
|
|
age_min = int((datetime.now(tz=UTC) - mtime).total_seconds() // 60)
|
|
try:
|
|
lock_file.unlink()
|
|
click.echo(
|
|
f"[vm cleanup] removed stale lock file (age: {age_min} min): {lock_file}"
|
|
)
|
|
except OSError as exc:
|
|
click.echo(f"[vm cleanup] could not remove lock file: {exc}", err=True)
|
|
|
|
click.echo("[vm cleanup] done.")
|
|
return removed
|
|
|
|
|
|
@vm.command("cleanup")
|
|
@click.option(
|
|
"--clone-base-dir",
|
|
"--clone-base",
|
|
"clone_base_dir",
|
|
default=None,
|
|
help="Directory containing ephemeral VM clones (defaults to config.paths.build_vms).",
|
|
)
|
|
@click.option(
|
|
"--max-age-hours",
|
|
type=click.IntRange(0, 168),
|
|
default=4,
|
|
show_default=True,
|
|
)
|
|
@click.option("--vmrun-path", "vmrun_path", default=None, help="Override vmrun.exe path.")
|
|
@click.option(
|
|
"--lock-file",
|
|
default=None,
|
|
help="Path to a stale vm-start lock file to remove if older than 30 minutes.",
|
|
)
|
|
@click.option(
|
|
"--what-if",
|
|
"--whatif",
|
|
"what_if",
|
|
is_flag=True,
|
|
default=False,
|
|
help="List orphans without destroying them.",
|
|
)
|
|
def vm_cleanup(
|
|
clone_base_dir: str | None,
|
|
max_age_hours: int,
|
|
vmrun_path: str | None,
|
|
lock_file: str | None,
|
|
what_if: bool,
|
|
) -> None:
|
|
"""Destroy ephemeral build VMs left behind after a crash or timeout."""
|
|
config = load_config()
|
|
base_path = Path(clone_base_dir) if clone_base_dir else config.paths.build_vms
|
|
if not base_path.exists():
|
|
click.echo(
|
|
f"[vm cleanup] clone base dir not found: {base_path} - nothing to do."
|
|
)
|
|
return
|
|
|
|
try:
|
|
backend: VmBackend | None = _make_backend(vmrun_path)
|
|
except BackendNotAvailable as exc:
|
|
click.echo(
|
|
f"[vm cleanup] vmrun not available ({exc}); will attempt directory removal only.",
|
|
err=True,
|
|
)
|
|
backend = None
|
|
|
|
cleanup_orphans(
|
|
clone_base=base_path,
|
|
max_age_hours=max_age_hours,
|
|
backend=backend,
|
|
dry_run=what_if,
|
|
lock_file=Path(lock_file) if lock_file else None,
|
|
)
|
|
|
|
|
|
# ─────────────────────────────────────────────────────────────────────── new
|
|
|
|
|
|
@vm.command("new")
|
|
@click.option(
|
|
"--template",
|
|
"--template-path",
|
|
"template",
|
|
required=True,
|
|
help="Identifier of the template VM (Workstation: VMX path; ESXi: managed-object ref).",
|
|
)
|
|
@click.option(
|
|
"--snapshot",
|
|
"--snapshot-name",
|
|
"snapshot",
|
|
default="BaseClean",
|
|
show_default=True,
|
|
help="Snapshot name on the template to clone from.",
|
|
)
|
|
@click.option(
|
|
"--clone-base-dir",
|
|
"clone_base_dir",
|
|
required=True,
|
|
help="Directory under which the new clone folder will be created.",
|
|
)
|
|
@click.option(
|
|
"--job-id",
|
|
"job_id",
|
|
required=True,
|
|
help="Unique job identifier (used to name the clone folder).",
|
|
)
|
|
@click.option(
|
|
"--vmrun-path",
|
|
"vmrun_path",
|
|
default=None,
|
|
help="Override vmrun binary path.",
|
|
)
|
|
@click.option(
|
|
"--guest-os",
|
|
type=click.Choice(["windows", "linux"], case_sensitive=False),
|
|
default="windows",
|
|
show_default=True,
|
|
help="Informational; reserved for backend hints in Phase C.",
|
|
)
|
|
@click.option(
|
|
"--start",
|
|
"start_vm",
|
|
is_flag=True,
|
|
default=False,
|
|
help="Start the clone headless immediately after creation.",
|
|
)
|
|
def vm_new(
|
|
template: str,
|
|
snapshot: str,
|
|
clone_base_dir: str,
|
|
job_id: str,
|
|
vmrun_path: str | None,
|
|
guest_os: str,
|
|
start_vm: bool,
|
|
) -> None:
|
|
"""Create a linked clone of the template VM for an ephemeral CI build.
|
|
|
|
Mirrors ``scripts/New-BuildVM.ps1``. Prints the clone identifier
|
|
(Workstation: VMX path) on stdout on success.
|
|
|
|
Phase C hook: ``--template`` is treated as an opaque string; the backend
|
|
is responsible for interpretation. Workstation builds the destination
|
|
path under ``--clone-base-dir`` from ``--job-id`` + timestamp.
|
|
"""
|
|
del guest_os # currently informational only
|
|
# Verify the template identifier looks plausible: for Workstation it is
|
|
# a VMX path. For other backends in Phase C this validation will move
|
|
# into the backend itself.
|
|
template_path = Path(template)
|
|
if not template_path.is_file():
|
|
raise click.ClickException(f"Template VMX not found: {template}")
|
|
|
|
base = Path(clone_base_dir)
|
|
base.mkdir(parents=True, exist_ok=True)
|
|
|
|
timestamp = datetime.now(tz=UTC).strftime("%Y%m%d_%H%M%S")
|
|
clone_name = f"Clone_{job_id}_{timestamp}"
|
|
clone_dir = base / clone_name
|
|
clone_vmx = clone_dir / f"{clone_name}.vmx"
|
|
|
|
click.echo("[vm new] creating linked clone...", err=True)
|
|
click.echo(f" template : {template}", err=True)
|
|
click.echo(f" snapshot : {snapshot}", err=True)
|
|
click.echo(f" clone vmx: {clone_vmx}", err=True)
|
|
|
|
try:
|
|
backend = _make_backend(vmrun_path)
|
|
except BackendNotAvailable as exc:
|
|
raise click.ClickException(f"vmrun unavailable: {exc}") from exc
|
|
|
|
start = time.monotonic()
|
|
try:
|
|
handle = backend.clone_linked(
|
|
template=str(template_path),
|
|
snapshot=snapshot,
|
|
name=clone_name,
|
|
destination=str(clone_vmx),
|
|
)
|
|
except BackendError as exc:
|
|
# Clean up partial clone directory if vmrun left one behind.
|
|
if clone_dir.exists():
|
|
_try_remove_dir(clone_dir)
|
|
raise click.ClickException(f"clone failed: {exc}") from exc
|
|
|
|
if not clone_vmx.is_file():
|
|
raise click.ClickException(
|
|
f"backend reported success but clone VMX is missing: {clone_vmx}"
|
|
)
|
|
|
|
elapsed = time.monotonic() - start
|
|
click.echo(f"[vm new] clone created in {elapsed:.1f}s", err=True)
|
|
|
|
if start_vm:
|
|
click.echo("[vm new] starting VM headless...", err=True)
|
|
try:
|
|
backend.start(handle, headless=True)
|
|
except BackendError as exc:
|
|
raise click.ClickException(f"vmrun start failed: {exc}") from exc
|
|
click.echo("[vm new] VM started.", err=True)
|
|
|
|
# Final stdout line: the identifier itself, so PS callers can capture it.
|
|
click.echo(handle.identifier)
|
|
|
|
|
|
__all__ = ["cleanup_orphans", "vm"]
|