feat(a2): port leaf PS scripts to ci_orchestrator CLI
Implements Phase A2 of plans/implementation-plan-A-B.md: - commands/wait.py -> wait-ready (replaces Wait-VMReady.ps1) - commands/vm.py -> vm remove + vm cleanup (replaces Remove-BuildVM.ps1, Cleanup-OrphanedBuildVMs.ps1); cleanup accepts injected VmBackend (Phase C ESXi hook preserved) - commands/monitor.py -> monitor disk + monitor runner (replaces Watch-DiskSpace.ps1, Watch-RunnerHealth.ps1) - commands/report.py -> report job (replaces Get-CIJobSummary.ps1) Each PS script reduced to a 3-line shim that delegates to the Python CLI and preserves \0. Tests: 69 pytest cases across new test_commands_*.py modules; ruff clean, mypy --strict clean, coverage 74.5% (>=70 gate).
This commit is contained in:
@@ -0,0 +1,10 @@
|
||||
"""Sub-command modules for the ``ci_orchestrator`` CLI.
|
||||
|
||||
Each module exposes one or more :class:`click.Command` instances that the
|
||||
top-level :mod:`ci_orchestrator.__main__` registers on the root group.
|
||||
Phase A2 ports the leaf PowerShell scripts (Wait-VMReady, Remove-BuildVM,
|
||||
Cleanup-OrphanedBuildVMs, Watch-DiskSpace, Watch-RunnerHealth,
|
||||
Get-CIJobSummary).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -0,0 +1,459 @@
|
||||
"""``monitor`` sub-commands.
|
||||
|
||||
Phase A2 ports the maintenance/monitoring leaf scripts:
|
||||
|
||||
* ``monitor disk`` — replaces ``scripts/Watch-DiskSpace.ps1``
|
||||
* ``monitor runner`` — replaces ``scripts/Watch-RunnerHealth.ps1``
|
||||
|
||||
Both commands are designed to run from a scheduler (Windows Task Scheduler
|
||||
in Phase A, systemd timers in Phase B) and emit warnings via the platform
|
||||
event log when available, plus optional Discord/Gitea webhooks via stdlib
|
||||
``urllib`` (no extra dependency).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import click
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
# ──────────────────────────────────────────────────────────────────── helpers
|
||||
|
||||
|
||||
def _post_webhook(url: str, content: str, *, timeout: float = 10.0) -> bool:
|
||||
"""POST a Discord/Gitea-compatible JSON payload. Returns True on success."""
|
||||
if not url:
|
||||
return False
|
||||
payload = json.dumps({"content": content}).encode("utf-8")
|
||||
req = urllib.request.Request(
|
||||
url,
|
||||
data=payload,
|
||||
method="POST",
|
||||
headers={"Content-Type": "application/json"},
|
||||
)
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=timeout) as resp:
|
||||
return bool(200 <= resp.status < 300)
|
||||
except (urllib.error.URLError, TimeoutError, OSError) as exc:
|
||||
logger.warning("Webhook POST failed: %s", exc)
|
||||
return False
|
||||
|
||||
|
||||
def _write_windows_event(
|
||||
source: str, event_id: int, entry_type: str, message: str
|
||||
) -> bool:
|
||||
"""Write a Windows Application Event Log entry via ``eventcreate.exe``.
|
||||
|
||||
Avoids the ``pywin32`` dependency. Silently no-op on non-Windows hosts.
|
||||
Returns True on success.
|
||||
"""
|
||||
if os.name != "nt":
|
||||
return False
|
||||
eventcreate = shutil.which("eventcreate") or shutil.which("eventcreate.exe")
|
||||
if eventcreate is None:
|
||||
return False
|
||||
type_map = {"Information": "INFORMATION", "Warning": "WARNING", "Error": "ERROR"}
|
||||
cmd = [
|
||||
eventcreate,
|
||||
"/T", type_map.get(entry_type, "WARNING"),
|
||||
"/ID", str(event_id),
|
||||
"/L", "APPLICATION",
|
||||
"/SO", source,
|
||||
"/D", message,
|
||||
]
|
||||
try:
|
||||
result = subprocess.run(
|
||||
cmd, capture_output=True, text=True, check=False, timeout=10.0
|
||||
)
|
||||
except (subprocess.TimeoutExpired, OSError) as exc:
|
||||
logger.warning("eventcreate failed: %s", exc)
|
||||
return False
|
||||
if result.returncode != 0:
|
||||
logger.warning("eventcreate returned %d: %s", result.returncode, result.stderr)
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────────── group
|
||||
|
||||
|
||||
@click.group("monitor")
|
||||
def monitor() -> None:
|
||||
"""Host monitoring commands (disk, runner)."""
|
||||
|
||||
|
||||
# ──────────────────────────────────────────────────────────────────────── disk
|
||||
|
||||
|
||||
def _resolve_drive_path(drive: str) -> str:
|
||||
"""Map a single-letter Windows drive (``F``) to a usable path.
|
||||
|
||||
On non-Windows the ``drive`` value is treated as a path itself.
|
||||
"""
|
||||
if os.name == "nt" and len(drive) == 1 and drive.isalpha():
|
||||
return f"{drive.upper()}:\\"
|
||||
return drive
|
||||
|
||||
|
||||
@monitor.command("disk")
|
||||
@click.option(
|
||||
"--min-free-gb",
|
||||
"min_free_gb",
|
||||
type=click.IntRange(1, 2000),
|
||||
default=50,
|
||||
show_default=True,
|
||||
)
|
||||
@click.option(
|
||||
"--drive-letter",
|
||||
"drive_letter",
|
||||
default="F" if os.name == "nt" else "/",
|
||||
show_default=True,
|
||||
help="Drive letter (Windows) or mount path (Linux).",
|
||||
)
|
||||
@click.option("--webhook-url", "webhook_url", default="", help="Optional Discord/Gitea webhook URL.")
|
||||
@click.option(
|
||||
"--json",
|
||||
"as_json",
|
||||
is_flag=True,
|
||||
default=False,
|
||||
help="Emit a single-line JSON object instead of human-readable output.",
|
||||
)
|
||||
def monitor_disk(
|
||||
min_free_gb: int,
|
||||
drive_letter: str,
|
||||
webhook_url: str,
|
||||
as_json: bool,
|
||||
) -> None:
|
||||
"""Check free space on a CI host drive and alert if below threshold.
|
||||
|
||||
Exit codes mirror the original PowerShell script:
|
||||
|
||||
* 0 = drive OK
|
||||
* 1 = below threshold (alert raised)
|
||||
* 2 = drive not found
|
||||
"""
|
||||
target = _resolve_drive_path(drive_letter)
|
||||
try:
|
||||
usage = shutil.disk_usage(target)
|
||||
except (FileNotFoundError, OSError) as exc:
|
||||
click.echo(f"[disk] drive {drive_letter!r} not found: {exc}", err=True)
|
||||
sys.exit(2)
|
||||
|
||||
gb = 1024.0 ** 3
|
||||
free_gb = round(usage.free / gb, 1)
|
||||
total_gb = round(usage.total / gb, 1)
|
||||
pct_free = round(free_gb / total_gb * 100, 0) if total_gb > 0 else 0
|
||||
|
||||
payload = {
|
||||
"drive": drive_letter,
|
||||
"free_gb": free_gb,
|
||||
"total_gb": total_gb,
|
||||
"percent_free": pct_free,
|
||||
"threshold_gb": min_free_gb,
|
||||
"status": "ok" if free_gb >= min_free_gb else "alert",
|
||||
}
|
||||
|
||||
if free_gb >= min_free_gb:
|
||||
if as_json:
|
||||
click.echo(json.dumps(payload))
|
||||
else:
|
||||
click.echo(
|
||||
f"[disk] drive {drive_letter}: {free_gb} GB free / {total_gb} GB "
|
||||
f"({pct_free}%) - OK (threshold: {min_free_gb} GB)"
|
||||
)
|
||||
return
|
||||
|
||||
msg = (
|
||||
f"CI host drive {drive_letter}: free space {free_gb} GB "
|
||||
f"({pct_free}%) is below the {min_free_gb} GB threshold. "
|
||||
"vmrun linked clones may fail silently."
|
||||
)
|
||||
if as_json:
|
||||
payload["message"] = msg
|
||||
click.echo(json.dumps(payload))
|
||||
else:
|
||||
click.echo(f"[disk] WARNING: {msg}", err=True)
|
||||
|
||||
_write_windows_event("CI-DiskSpaceAlert", 1001, "Warning", msg)
|
||||
if webhook_url:
|
||||
_post_webhook(webhook_url, f"[WARNING] CI Disk Alert -- {msg}")
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
# ────────────────────────────────────────────────────────────────────── runner
|
||||
|
||||
|
||||
def _service_status(service_name: str) -> str | None:
|
||||
"""Return ``Running``/``Stopped``/``Unknown``, or ``None`` if not installed."""
|
||||
if os.name == "nt":
|
||||
sc = shutil.which("sc") or r"C:\Windows\System32\sc.exe"
|
||||
try:
|
||||
r = subprocess.run(
|
||||
[sc, "query", service_name],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=False,
|
||||
timeout=10.0,
|
||||
)
|
||||
except (subprocess.TimeoutExpired, OSError):
|
||||
return None
|
||||
if r.returncode != 0:
|
||||
return None
|
||||
out = r.stdout.upper()
|
||||
if "RUNNING" in out:
|
||||
return "Running"
|
||||
if "STOPPED" in out:
|
||||
return "Stopped"
|
||||
return "Unknown"
|
||||
# Linux / systemd
|
||||
systemctl = shutil.which("systemctl")
|
||||
if systemctl is None:
|
||||
return None
|
||||
try:
|
||||
r = subprocess.run(
|
||||
[systemctl, "is-active", service_name],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=False,
|
||||
timeout=10.0,
|
||||
)
|
||||
except (subprocess.TimeoutExpired, OSError):
|
||||
return None
|
||||
state = r.stdout.strip().lower()
|
||||
if state == "active":
|
||||
return "Running"
|
||||
if state in {"inactive", "failed", "deactivating"}:
|
||||
return "Stopped"
|
||||
return None # unknown / not installed
|
||||
|
||||
|
||||
def _restart_service(service_name: str) -> bool:
|
||||
"""Attempt to (re)start ``service_name``. Returns True if now running."""
|
||||
if os.name == "nt":
|
||||
sc = shutil.which("sc") or r"C:\Windows\System32\sc.exe"
|
||||
# `sc start` returns 1056 if already running; ignore.
|
||||
try:
|
||||
subprocess.run(
|
||||
[sc, "start", service_name],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=False,
|
||||
timeout=30.0,
|
||||
)
|
||||
except (subprocess.TimeoutExpired, OSError):
|
||||
return False
|
||||
else:
|
||||
systemctl = shutil.which("systemctl")
|
||||
if systemctl is None:
|
||||
return False
|
||||
try:
|
||||
subprocess.run(
|
||||
[systemctl, "restart", service_name],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=False,
|
||||
timeout=30.0,
|
||||
)
|
||||
except (subprocess.TimeoutExpired, OSError):
|
||||
return False
|
||||
time.sleep(5.0)
|
||||
return _service_status(service_name) == "Running"
|
||||
|
||||
|
||||
def _load_restart_log(path: Path) -> list[str]:
|
||||
if not path.is_file():
|
||||
return []
|
||||
try:
|
||||
raw = path.read_text(encoding="utf-8")
|
||||
except OSError:
|
||||
return []
|
||||
if not raw.strip():
|
||||
return []
|
||||
try:
|
||||
data = json.loads(raw)
|
||||
except json.JSONDecodeError:
|
||||
return []
|
||||
if isinstance(data, list):
|
||||
return [str(x) for x in data]
|
||||
return []
|
||||
|
||||
|
||||
def _save_restart_log(path: Path, entries: list[str]) -> None:
|
||||
try:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(json.dumps(entries), encoding="utf-8")
|
||||
except OSError as exc:
|
||||
logger.warning("Could not persist restart log: %s", exc)
|
||||
|
||||
|
||||
def _gitea_runners_online(api_url: str, token: str, *, timeout: float = 10.0) -> int | None:
|
||||
"""Return number of online runners reported by Gitea, or ``None`` on failure."""
|
||||
api = api_url.rstrip("/") + "/api/v1/admin/runners"
|
||||
req = urllib.request.Request(
|
||||
api, headers={"Authorization": f"token {token}"}
|
||||
)
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=timeout) as resp:
|
||||
data = json.loads(resp.read().decode("utf-8"))
|
||||
except (urllib.error.URLError, TimeoutError, OSError, json.JSONDecodeError) as exc:
|
||||
logger.warning("Gitea API call failed: %s", exc)
|
||||
return None
|
||||
if isinstance(data, list):
|
||||
return sum(1 for r in data if isinstance(r, dict) and r.get("online") is True)
|
||||
return None
|
||||
|
||||
|
||||
@monitor.command("runner")
|
||||
@click.option(
|
||||
"--service-name",
|
||||
"service_name",
|
||||
default="act_runner",
|
||||
show_default=True,
|
||||
)
|
||||
@click.option(
|
||||
"--max-restarts",
|
||||
type=click.IntRange(1, 10),
|
||||
default=3,
|
||||
show_default=True,
|
||||
)
|
||||
@click.option(
|
||||
"--state-dir",
|
||||
"state_dir",
|
||||
default=r"F:\CI\State" if os.name == "nt" else "/var/lib/ci/state",
|
||||
show_default=True,
|
||||
)
|
||||
@click.option("--webhook-url", "webhook_url", default="")
|
||||
@click.option("--gitea-url", "gitea_url", default="")
|
||||
@click.option(
|
||||
"--gitea-credential-target",
|
||||
"gitea_credential_target",
|
||||
default="",
|
||||
help="Credential target name for the Gitea admin PAT.",
|
||||
)
|
||||
def monitor_runner(
|
||||
service_name: str,
|
||||
max_restarts: int,
|
||||
state_dir: str,
|
||||
webhook_url: str,
|
||||
gitea_url: str,
|
||||
gitea_credential_target: str,
|
||||
) -> None:
|
||||
"""Monitor act_runner; auto-restart up to ``--max-restarts`` per hour.
|
||||
|
||||
Exit codes:
|
||||
0 = healthy (or restart succeeded)
|
||||
1 = service down + restart failed/limit reached
|
||||
2 = service not installed
|
||||
"""
|
||||
state_dir_path = Path(state_dir)
|
||||
maintenance_flag = state_dir_path / "runner-maintenance.flag"
|
||||
if maintenance_flag.is_file():
|
||||
click.echo(
|
||||
f"[runner] maintenance mode active ({maintenance_flag}) - skipping."
|
||||
)
|
||||
return
|
||||
|
||||
status = _service_status(service_name)
|
||||
if status is None:
|
||||
click.echo(
|
||||
f"[runner] service {service_name!r} not found - is act_runner installed?",
|
||||
err=True,
|
||||
)
|
||||
sys.exit(2)
|
||||
|
||||
if status == "Running":
|
||||
click.echo(f"[runner] {service_name} is Running - OK")
|
||||
if gitea_url and gitea_credential_target:
|
||||
try:
|
||||
from ci_orchestrator.credentials import KeyringCredentialStore
|
||||
|
||||
cred = KeyringCredentialStore().get(gitea_credential_target)
|
||||
online = _gitea_runners_online(gitea_url, cred.password)
|
||||
except KeyError as exc:
|
||||
click.echo(f"[runner] could not load Gitea PAT: {exc}", err=True)
|
||||
online = None
|
||||
if online is None:
|
||||
click.echo("[runner] Gitea API check skipped/failed.")
|
||||
elif online == 0:
|
||||
msg = (
|
||||
f"Gitea API ({gitea_url}) reports 0 online runners. "
|
||||
f"{service_name} is Running - possible registration issue."
|
||||
)
|
||||
click.echo(f"[runner] WARNING: {msg}", err=True)
|
||||
_write_windows_event("CI-RunnerHealth", 1005, "Warning", msg)
|
||||
if webhook_url:
|
||||
_post_webhook(webhook_url, f"[WARNING] CI Runner Alert -- {msg}")
|
||||
else:
|
||||
click.echo(f"[runner] Gitea API: {online} runner(s) online - OK")
|
||||
return
|
||||
|
||||
# Not running → check rate limit and try to restart.
|
||||
cooldown_file = state_dir_path / "runner-restart-log.json"
|
||||
log = _load_restart_log(cooldown_file)
|
||||
cutoff = datetime.now(tz=UTC) - timedelta(hours=1)
|
||||
pruned: list[str] = []
|
||||
for entry in log:
|
||||
try:
|
||||
ts = datetime.fromisoformat(entry)
|
||||
except ValueError:
|
||||
continue
|
||||
if ts.tzinfo is None:
|
||||
ts = ts.replace(tzinfo=UTC)
|
||||
if ts > cutoff:
|
||||
pruned.append(entry)
|
||||
log = pruned
|
||||
|
||||
state_msg = (
|
||||
f"{service_name} service is '{status}'. "
|
||||
f"Auto-restarts in last hour: {len(log)} / {max_restarts}."
|
||||
)
|
||||
click.echo(f"[runner] WARNING: {state_msg}", err=True)
|
||||
_write_windows_event("CI-RunnerHealth", 1002, "Warning", state_msg)
|
||||
|
||||
if len(log) >= max_restarts:
|
||||
limit_msg = (
|
||||
f"{service_name} auto-restart limit ({max_restarts}/h) reached - "
|
||||
"manual intervention required."
|
||||
)
|
||||
click.echo(f"[runner] {limit_msg}", err=True)
|
||||
_write_windows_event("CI-RunnerHealth", 1004, "Error", limit_msg)
|
||||
if webhook_url:
|
||||
_post_webhook(webhook_url, f"[ALERT] CI Runner Alert -- {limit_msg}")
|
||||
sys.exit(1)
|
||||
|
||||
click.echo(
|
||||
f"[runner] restarting {service_name} (attempt {len(log) + 1} of {max_restarts})..."
|
||||
)
|
||||
ok = _restart_service(service_name)
|
||||
log.append(datetime.now(tz=UTC).isoformat())
|
||||
_save_restart_log(cooldown_file, log)
|
||||
new_status = _service_status(service_name) or "Unknown"
|
||||
restart_msg = f"{service_name} restarted - new status: {new_status}."
|
||||
entry_type = "Information" if ok else "Warning"
|
||||
click.echo(f"[runner] {restart_msg}")
|
||||
_write_windows_event("CI-RunnerHealth", 1003, entry_type, restart_msg)
|
||||
prefix = "[WARNING]" if ok else "[ALERT]"
|
||||
if webhook_url:
|
||||
_post_webhook(webhook_url, f"{prefix} CI Runner Alert -- {restart_msg}")
|
||||
sys.exit(0 if ok else 1)
|
||||
|
||||
|
||||
def _ignored(_name: str, _value: Any) -> None:
|
||||
"""Marker so type checkers see we deliberately drop a value."""
|
||||
|
||||
|
||||
__all__ = ["monitor"]
|
||||
@@ -0,0 +1,258 @@
|
||||
"""``report`` sub-commands.
|
||||
|
||||
Phase A2 ports ``scripts/Get-CIJobSummary.ps1`` to ``report job``. Reads
|
||||
JSONL log files written by the (still-PowerShell) ``Invoke-CIJob.ps1``
|
||||
under ``F:\\CI\\Logs\\<jobId>\\invoke-ci.jsonl`` and prints either a
|
||||
summary table of recent jobs or a per-phase breakdown for one job.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import sys
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
|
||||
import click
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class _Event:
|
||||
phase: str
|
||||
status: str
|
||||
ts: str
|
||||
elapsed_sec: int | None
|
||||
error: str | None
|
||||
raw: dict[str, object]
|
||||
|
||||
|
||||
def _read_events(jsonl_path: Path) -> list[_Event]:
|
||||
out: list[_Event] = []
|
||||
try:
|
||||
text = jsonl_path.read_text(encoding="utf-8")
|
||||
except OSError:
|
||||
return out
|
||||
for line in text.splitlines():
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
try:
|
||||
obj = json.loads(line)
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
if not isinstance(obj, dict):
|
||||
continue
|
||||
data = obj.get("data") if isinstance(obj.get("data"), dict) else {}
|
||||
elapsed: int | None = None
|
||||
if isinstance(data, dict):
|
||||
raw_elapsed = data.get("elapsedSec")
|
||||
if isinstance(raw_elapsed, int | float):
|
||||
elapsed = int(raw_elapsed)
|
||||
err: str | None = None
|
||||
if isinstance(data, dict) and isinstance(data.get("error"), str):
|
||||
err = str(data["error"])
|
||||
out.append(
|
||||
_Event(
|
||||
phase=str(obj.get("phase", "")),
|
||||
status=str(obj.get("status", "")),
|
||||
ts=str(obj.get("ts", "")),
|
||||
elapsed_sec=elapsed,
|
||||
error=err,
|
||||
raw=obj,
|
||||
)
|
||||
)
|
||||
return out
|
||||
|
||||
|
||||
def _format_hms(sec: int) -> str:
|
||||
if sec < 0:
|
||||
return "?"
|
||||
h, rem = divmod(sec, 3600)
|
||||
m, s = divmod(rem, 60)
|
||||
if h > 0:
|
||||
return f"{h}h{m:02d}m{s:02d}s"
|
||||
return f"{m}m{s:02d}s"
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class _Row:
|
||||
job_id: str
|
||||
status: str
|
||||
elapsed: str
|
||||
started: str
|
||||
error: str
|
||||
|
||||
|
||||
def _summarise(events: list[_Event], default_id: str) -> _Row:
|
||||
job_start = next(
|
||||
(e for e in events if e.phase == "job" and e.status == "start"), None
|
||||
)
|
||||
job_success = next(
|
||||
(e for e in events if e.phase == "job" and e.status == "success"), None
|
||||
)
|
||||
job_failure = next(
|
||||
(e for e in events if e.phase == "job" and e.status == "failure"), None
|
||||
)
|
||||
raw_id = events[0].raw.get("jobId") if events else None
|
||||
job_id = str(raw_id) if isinstance(raw_id, str) and raw_id else default_id
|
||||
started = job_start.ts if job_start else ""
|
||||
if job_success:
|
||||
return _Row(
|
||||
job_id=job_id,
|
||||
status="success",
|
||||
elapsed=_format_hms(job_success.elapsed_sec or -1),
|
||||
started=started,
|
||||
error="",
|
||||
)
|
||||
if job_failure:
|
||||
err = job_failure.error or ""
|
||||
if len(err) > 60:
|
||||
err = err[:57] + "..."
|
||||
return _Row(
|
||||
job_id=job_id,
|
||||
status="FAILED",
|
||||
elapsed=_format_hms(job_failure.elapsed_sec or -1),
|
||||
started=started,
|
||||
error=err,
|
||||
)
|
||||
return _Row(
|
||||
job_id=job_id,
|
||||
status="in-progress",
|
||||
elapsed="?",
|
||||
started=started,
|
||||
error="",
|
||||
)
|
||||
|
||||
|
||||
@click.group("report")
|
||||
def report() -> None:
|
||||
"""Reporting commands (job summaries)."""
|
||||
|
||||
|
||||
@report.command("job")
|
||||
@click.option(
|
||||
"--log-dir",
|
||||
"log_dir",
|
||||
default=r"F:\CI\Logs",
|
||||
show_default=True,
|
||||
help="Base directory containing per-job log subdirectories.",
|
||||
)
|
||||
@click.option(
|
||||
"--last",
|
||||
type=click.IntRange(1, 1000),
|
||||
default=20,
|
||||
show_default=True,
|
||||
help="Show only the N most recent jobs.",
|
||||
)
|
||||
@click.option(
|
||||
"--job-id",
|
||||
"job_id",
|
||||
default="",
|
||||
help="Show a phase-by-phase breakdown for a single job.",
|
||||
)
|
||||
@click.option(
|
||||
"--failed",
|
||||
is_flag=True,
|
||||
default=False,
|
||||
help="Filter to failed jobs only.",
|
||||
)
|
||||
@click.option(
|
||||
"--json",
|
||||
"as_json",
|
||||
is_flag=True,
|
||||
default=False,
|
||||
help="Emit JSON instead of a human-readable table.",
|
||||
)
|
||||
def report_job(
|
||||
log_dir: str,
|
||||
last: int,
|
||||
job_id: str,
|
||||
failed: bool,
|
||||
as_json: bool,
|
||||
) -> None:
|
||||
"""Display CI job summaries parsed from invoke-ci.jsonl files."""
|
||||
base = Path(log_dir)
|
||||
if not base.is_dir():
|
||||
click.echo(f"Log directory not found: {log_dir}", err=True)
|
||||
sys.exit(1)
|
||||
|
||||
if job_id:
|
||||
jsonl = base / job_id / "invoke-ci.jsonl"
|
||||
if not jsonl.is_file():
|
||||
click.echo(f"No JSONL log found for job '{job_id}' at: {jsonl}", err=True)
|
||||
sys.exit(1)
|
||||
events = _read_events(jsonl)
|
||||
if not events:
|
||||
click.echo(f"JSONL file is empty or unreadable: {jsonl}", err=True)
|
||||
sys.exit(1)
|
||||
if as_json:
|
||||
click.echo(json.dumps([e.raw for e in events], default=str))
|
||||
return
|
||||
click.echo(f"\nJob: {job_id}")
|
||||
click.echo("=" * 60)
|
||||
click.echo(f"{'Phase':<35} {'Status':<10} {'Timestamp'}")
|
||||
click.echo(f"{'-' * 35:<35} {'-' * 10:<10} {'-' * 24}")
|
||||
for ev in events:
|
||||
click.echo(f"{ev.phase:<35} {ev.status:<10} {ev.ts}")
|
||||
fail = next(
|
||||
(e for e in reversed(events) if e.status == "failure"),
|
||||
None,
|
||||
)
|
||||
if fail and fail.error:
|
||||
click.echo(f"\nError: {fail.error}")
|
||||
return
|
||||
|
||||
files = sorted(
|
||||
base.rglob("invoke-ci.jsonl"),
|
||||
key=lambda p: p.stat().st_mtime,
|
||||
reverse=True,
|
||||
)
|
||||
if not files:
|
||||
click.echo(f"No invoke-ci.jsonl files found under {log_dir}")
|
||||
return
|
||||
|
||||
rows: list[_Row] = []
|
||||
for f in files:
|
||||
events = _read_events(f)
|
||||
if not events:
|
||||
continue
|
||||
default_id = f.parent.name
|
||||
rows.append(_summarise(events, default_id))
|
||||
|
||||
if failed:
|
||||
rows = [r for r in rows if r.status == "FAILED"]
|
||||
rows = rows[:last]
|
||||
if not rows:
|
||||
click.echo("No matching jobs found.")
|
||||
return
|
||||
|
||||
if as_json:
|
||||
click.echo(
|
||||
json.dumps(
|
||||
[
|
||||
{
|
||||
"jobId": r.job_id,
|
||||
"status": r.status,
|
||||
"elapsed": r.elapsed,
|
||||
"started": r.started,
|
||||
"error": r.error,
|
||||
}
|
||||
for r in rows
|
||||
]
|
||||
)
|
||||
)
|
||||
return
|
||||
|
||||
click.echo(f"\nCI Job Summary (last {len(rows)} jobs):")
|
||||
header = f"{'JobId':<40} {'Status':<12} {'Elapsed':<10} {'Started':<26} Error"
|
||||
click.echo(header)
|
||||
click.echo(
|
||||
f"{'-' * 40:<40} {'-' * 12:<12} {'-' * 10:<10} {'-' * 26:<26} {'-' * 30}"
|
||||
)
|
||||
for r in rows:
|
||||
click.echo(
|
||||
f"{r.job_id:<40} {r.status:<12} {r.elapsed:<10} {r.started:<26} {r.error}"
|
||||
)
|
||||
|
||||
|
||||
__all__ = ["report"]
|
||||
@@ -0,0 +1,357 @@
|
||||
"""``vm`` sub-commands.
|
||||
|
||||
Phase A2 ports the leaf VM management scripts:
|
||||
|
||||
* ``vm remove`` — replaces ``scripts/Remove-BuildVM.ps1``
|
||||
* ``vm cleanup`` — replaces ``scripts/Cleanup-OrphanedBuildVMs.ps1``
|
||||
|
||||
Phase C hook: ``vm cleanup`` accepts an optional :class:`VmBackend`
|
||||
dependency so an ESXi backend can later supply a folder-scoped VM list
|
||||
instead of scanning the local filesystem.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import shutil
|
||||
import time
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import click
|
||||
|
||||
from ci_orchestrator.backends import load_backend
|
||||
from ci_orchestrator.backends.errors import BackendError, BackendNotAvailable
|
||||
from ci_orchestrator.backends.protocol import VmHandle
|
||||
from ci_orchestrator.config import load_config
|
||||
|
||||
if TYPE_CHECKING: # pragma: no cover
|
||||
from ci_orchestrator.backends.protocol import VmBackend
|
||||
|
||||
|
||||
# ────────────────────────────────────────────────────────────────────── helpers
|
||||
|
||||
|
||||
def _make_backend(vmrun_path: str | None) -> VmBackend:
|
||||
"""Build a backend honouring an optional vmrun override."""
|
||||
config = load_config()
|
||||
if vmrun_path is not None:
|
||||
from ci_orchestrator.backends.workstation import WorkstationVmrunBackend
|
||||
|
||||
return WorkstationVmrunBackend(vmrun_path=vmrun_path)
|
||||
return load_backend(config)
|
||||
|
||||
|
||||
def _try_remove_dir(path: Path) -> bool:
|
||||
"""Best-effort recursive directory removal. Returns True if path is gone."""
|
||||
if not path.exists():
|
||||
return True
|
||||
shutil.rmtree(path, ignore_errors=True)
|
||||
return not path.exists()
|
||||
|
||||
|
||||
def _stop_with_fallback(backend: VmBackend, handle: VmHandle) -> None:
|
||||
"""Soft-stop, then hard-stop on failure. Both errors are tolerated."""
|
||||
try:
|
||||
backend.stop(handle, hard=False)
|
||||
except BackendError as exc:
|
||||
click.echo(f"[vm remove] soft stop failed: {exc}", err=True)
|
||||
try:
|
||||
if backend.is_running(handle):
|
||||
click.echo("[vm remove] still running, forcing hard stop...")
|
||||
backend.stop(handle, hard=True)
|
||||
except BackendError as exc:
|
||||
click.echo(f"[vm remove] hard stop failed: {exc}", err=True)
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────────── group
|
||||
|
||||
|
||||
@click.group("vm")
|
||||
def vm() -> None:
|
||||
"""VM lifecycle commands."""
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────────── remove
|
||||
|
||||
|
||||
@vm.command("remove")
|
||||
@click.option(
|
||||
"--vmx",
|
||||
"--vm-path",
|
||||
"vmx",
|
||||
required=True,
|
||||
help="Path to the clone VMX file to destroy.",
|
||||
)
|
||||
@click.option("--vmrun-path", "vmrun_path", default=None, help="Override vmrun.exe path.")
|
||||
@click.option(
|
||||
"--graceful-timeout",
|
||||
"--graceful-timeout-seconds",
|
||||
"graceful_timeout",
|
||||
type=click.IntRange(1, 60),
|
||||
default=15,
|
||||
show_default=True,
|
||||
help="Seconds to wait for soft stop before forcing.",
|
||||
)
|
||||
@click.option(
|
||||
"--force",
|
||||
is_flag=True,
|
||||
default=False,
|
||||
help="Skip soft stop and go straight to hard stop.",
|
||||
)
|
||||
def vm_remove(
|
||||
vmx: str,
|
||||
vmrun_path: str | None,
|
||||
graceful_timeout: int,
|
||||
force: bool,
|
||||
) -> None:
|
||||
"""Stop and permanently delete an ephemeral build VM."""
|
||||
vmx_path = Path(vmx)
|
||||
clone_dir = vmx_path.parent
|
||||
click.echo(f"[vm remove] destroying VM: {vmx}")
|
||||
|
||||
if not vmx_path.is_file():
|
||||
click.echo(
|
||||
f"[vm remove] VMX file not found - nothing to remove: {vmx}",
|
||||
err=True,
|
||||
)
|
||||
# Still try to clean up a partial clone directory.
|
||||
if clone_dir.exists() and clone_dir != vmx_path:
|
||||
_try_remove_dir(clone_dir)
|
||||
return
|
||||
|
||||
try:
|
||||
backend = _make_backend(vmrun_path)
|
||||
except BackendNotAvailable as exc:
|
||||
click.echo(
|
||||
f"[vm remove] vmrun unavailable ({exc}); attempting directory removal only.",
|
||||
err=True,
|
||||
)
|
||||
_try_remove_dir(clone_dir)
|
||||
return
|
||||
|
||||
handle = VmHandle(identifier=vmx)
|
||||
|
||||
if force:
|
||||
try:
|
||||
backend.stop(handle, hard=True)
|
||||
except BackendError as exc:
|
||||
click.echo(f"[vm remove] hard stop failed: {exc}", err=True)
|
||||
else:
|
||||
click.echo(f"[vm remove] sending soft stop (timeout: {graceful_timeout}s)...")
|
||||
_stop_with_fallback(backend, handle)
|
||||
|
||||
# Brief settle: VMware may keep file locks for a moment after stop.
|
||||
deadline = time.monotonic() + 20.0
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
if not backend.is_running(handle):
|
||||
break
|
||||
except BackendError:
|
||||
break
|
||||
time.sleep(2.0)
|
||||
|
||||
click.echo("[vm remove] deleting VM...")
|
||||
deleted = False
|
||||
last_err: str = ""
|
||||
for delay in (0, 3, 6):
|
||||
if delay > 0:
|
||||
click.echo(f"[vm remove] retrying deleteVM in {delay}s...")
|
||||
time.sleep(delay)
|
||||
try:
|
||||
backend.delete(handle)
|
||||
deleted = True
|
||||
break
|
||||
except BackendError as exc:
|
||||
last_err = str(exc)
|
||||
|
||||
if not deleted:
|
||||
click.echo(
|
||||
f"[vm remove] vmrun deleteVM failed after retries: {last_err}", err=True
|
||||
)
|
||||
click.echo("[vm remove] falling back to manual directory removal.")
|
||||
|
||||
if clone_dir.exists():
|
||||
click.echo(f"[vm remove] removing clone directory: {clone_dir}")
|
||||
if _try_remove_dir(clone_dir):
|
||||
click.echo("[vm remove] clone directory removed.")
|
||||
else:
|
||||
click.echo(
|
||||
f"[vm remove] could not fully remove: {clone_dir} - manual cleanup needed.",
|
||||
err=True,
|
||||
)
|
||||
|
||||
click.echo("[vm remove] VM destruction complete.")
|
||||
|
||||
|
||||
# ────────────────────────────────────────────────────────────────────── cleanup
|
||||
|
||||
|
||||
def _list_orphans(clone_base: Path, max_age_hours: int) -> list[Path]:
|
||||
"""Return orphan clone directories older than ``max_age_hours``."""
|
||||
if not clone_base.is_dir():
|
||||
return []
|
||||
cutoff = datetime.now(tz=UTC) - timedelta(hours=max_age_hours)
|
||||
out: list[Path] = []
|
||||
for entry in clone_base.iterdir():
|
||||
if not entry.is_dir():
|
||||
continue
|
||||
try:
|
||||
mtime = datetime.fromtimestamp(entry.stat().st_mtime, tz=UTC)
|
||||
except OSError:
|
||||
continue
|
||||
if mtime < cutoff:
|
||||
out.append(entry)
|
||||
return out
|
||||
|
||||
|
||||
def _find_vmx(clone_dir: Path) -> Path | None:
|
||||
for entry in clone_dir.iterdir():
|
||||
if entry.is_file() and entry.suffix.lower() == ".vmx":
|
||||
return entry
|
||||
return None
|
||||
|
||||
|
||||
def cleanup_orphans(
|
||||
clone_base: Path,
|
||||
max_age_hours: int,
|
||||
backend: VmBackend | None,
|
||||
*,
|
||||
dry_run: bool = False,
|
||||
lock_file: Path | None = None,
|
||||
lock_max_age_minutes: int = 30,
|
||||
) -> int:
|
||||
"""Remove orphaned clone directories. Returns count removed.
|
||||
|
||||
Phase C hook: ``backend`` is used to stop+delete each orphan. If the
|
||||
backend is unavailable, directory removal is still attempted.
|
||||
"""
|
||||
orphans = _list_orphans(clone_base, max_age_hours)
|
||||
if not orphans:
|
||||
click.echo(
|
||||
f"[vm cleanup] no orphaned VMs found (threshold: {max_age_hours} h)."
|
||||
)
|
||||
else:
|
||||
click.echo(
|
||||
f"[vm cleanup] found {len(orphans)} orphaned director"
|
||||
f"{'y' if len(orphans) == 1 else 'ies'} older than {max_age_hours} h."
|
||||
)
|
||||
|
||||
removed = 0
|
||||
now = datetime.now(tz=UTC)
|
||||
for entry in orphans:
|
||||
age_h = int((now - datetime.fromtimestamp(entry.stat().st_mtime, tz=UTC)).total_seconds() // 3600)
|
||||
if dry_run:
|
||||
click.echo(f"[vm cleanup] would destroy {entry} (age: {age_h} h)")
|
||||
continue
|
||||
click.echo(f"[vm cleanup] processing: {entry}")
|
||||
vmx = _find_vmx(entry)
|
||||
if vmx is not None and backend is not None:
|
||||
handle = VmHandle(identifier=str(vmx))
|
||||
try:
|
||||
backend.stop(handle, hard=True)
|
||||
except BackendError as exc:
|
||||
click.echo(f"[vm cleanup] hard stop ignored: {exc}", err=True)
|
||||
time.sleep(2.0)
|
||||
try:
|
||||
backend.delete(handle)
|
||||
except BackendError as exc:
|
||||
click.echo(
|
||||
f"[vm cleanup] vmrun deleteVM failed: {exc}; falling back to dir removal.",
|
||||
err=True,
|
||||
)
|
||||
elif vmx is None:
|
||||
click.echo(
|
||||
f"[vm cleanup] no .vmx in {entry} - removing directory only.", err=True
|
||||
)
|
||||
|
||||
if _try_remove_dir(entry):
|
||||
click.echo(f"[vm cleanup] removed: {entry}")
|
||||
removed += 1
|
||||
else:
|
||||
click.echo(
|
||||
f"[vm cleanup] could not fully remove: {entry} - manual cleanup needed.",
|
||||
err=True,
|
||||
)
|
||||
|
||||
# Stale lock file cleanup (mirrors Cleanup-OrphanedBuildVMs.ps1).
|
||||
if lock_file is not None and lock_file.is_file():
|
||||
cutoff = datetime.now(tz=UTC) - timedelta(minutes=lock_max_age_minutes)
|
||||
mtime = datetime.fromtimestamp(lock_file.stat().st_mtime, tz=UTC)
|
||||
if mtime < cutoff and not dry_run:
|
||||
age_min = int((datetime.now(tz=UTC) - mtime).total_seconds() // 60)
|
||||
try:
|
||||
lock_file.unlink()
|
||||
click.echo(
|
||||
f"[vm cleanup] removed stale lock file (age: {age_min} min): {lock_file}"
|
||||
)
|
||||
except OSError as exc:
|
||||
click.echo(f"[vm cleanup] could not remove lock file: {exc}", err=True)
|
||||
|
||||
click.echo("[vm cleanup] done.")
|
||||
return removed
|
||||
|
||||
|
||||
@vm.command("cleanup")
|
||||
@click.option(
|
||||
"--clone-base-dir",
|
||||
"--clone-base",
|
||||
"clone_base_dir",
|
||||
default=None,
|
||||
help="Directory containing ephemeral VM clones (defaults to config.paths.build_vms).",
|
||||
)
|
||||
@click.option(
|
||||
"--max-age-hours",
|
||||
type=click.IntRange(0, 168),
|
||||
default=4,
|
||||
show_default=True,
|
||||
)
|
||||
@click.option("--vmrun-path", "vmrun_path", default=None, help="Override vmrun.exe path.")
|
||||
@click.option(
|
||||
"--lock-file",
|
||||
default=None,
|
||||
help="Path to a stale vm-start lock file to remove if older than 30 minutes.",
|
||||
)
|
||||
@click.option(
|
||||
"--what-if",
|
||||
"--whatif",
|
||||
"what_if",
|
||||
is_flag=True,
|
||||
default=False,
|
||||
help="List orphans without destroying them.",
|
||||
)
|
||||
def vm_cleanup(
|
||||
clone_base_dir: str | None,
|
||||
max_age_hours: int,
|
||||
vmrun_path: str | None,
|
||||
lock_file: str | None,
|
||||
what_if: bool,
|
||||
) -> None:
|
||||
"""Destroy ephemeral build VMs left behind after a crash or timeout."""
|
||||
config = load_config()
|
||||
base_path = Path(clone_base_dir) if clone_base_dir else config.paths.build_vms
|
||||
if not base_path.exists():
|
||||
click.echo(
|
||||
f"[vm cleanup] clone base dir not found: {base_path} - nothing to do."
|
||||
)
|
||||
return
|
||||
|
||||
try:
|
||||
backend: VmBackend | None = _make_backend(vmrun_path)
|
||||
except BackendNotAvailable as exc:
|
||||
click.echo(
|
||||
f"[vm cleanup] vmrun not available ({exc}); will attempt directory removal only.",
|
||||
err=True,
|
||||
)
|
||||
backend = None
|
||||
|
||||
cleanup_orphans(
|
||||
clone_base=base_path,
|
||||
max_age_hours=max_age_hours,
|
||||
backend=backend,
|
||||
dry_run=what_if,
|
||||
lock_file=Path(lock_file) if lock_file else None,
|
||||
)
|
||||
|
||||
|
||||
__all__ = ["cleanup_orphans", "vm"]
|
||||
@@ -0,0 +1,202 @@
|
||||
"""``wait-ready`` sub-command.
|
||||
|
||||
Replaces ``scripts/Wait-VMReady.ps1``. Polls a guest VM until WinRM
|
||||
(Windows) or SSH (Linux) is responsive. Designed to be invoked from the
|
||||
PowerShell shim ``scripts/Wait-VMReady.ps1`` which translates PS-style
|
||||
PascalCase parameters (``-VMPath``, ``-IPAddress``, ``-Transport``,
|
||||
``-SshKeyPath``, ``-SshUser``) into click kebab-case options.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
import time
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import click
|
||||
|
||||
from ci_orchestrator.backends import load_backend
|
||||
from ci_orchestrator.backends.errors import BackendError
|
||||
from ci_orchestrator.backends.protocol import VmHandle
|
||||
from ci_orchestrator.config import load_config
|
||||
from ci_orchestrator.credentials import KeyringCredentialStore
|
||||
from ci_orchestrator.transport.errors import TransportError
|
||||
from ci_orchestrator.transport.ssh import SshTransport
|
||||
from ci_orchestrator.transport.winrm import WinRmTransport
|
||||
|
||||
if TYPE_CHECKING: # pragma: no cover
|
||||
from ci_orchestrator.backends.protocol import VmBackend
|
||||
|
||||
|
||||
# Exit codes (kept in sync with the PowerShell original where possible):
|
||||
# 0 ready, 2 timeout-running, 3 timeout-ip, 4 timeout-transport
|
||||
_EXIT_TIMEOUT_RUNNING = 2
|
||||
_EXIT_TIMEOUT_IP = 3
|
||||
_EXIT_TIMEOUT_TRANSPORT = 4
|
||||
|
||||
|
||||
def _normalise_guest_os(value: str) -> str:
|
||||
"""Map Transport/GuestOS aliases to ``windows``/``linux``."""
|
||||
v = value.strip().lower()
|
||||
if v in {"winrm", "windows", "win"}:
|
||||
return "windows"
|
||||
if v in {"ssh", "linux", "lnx"}:
|
||||
return "linux"
|
||||
raise click.BadParameter(f"Unknown guest-os/transport value: {value!r}")
|
||||
|
||||
|
||||
def _wait_running(
|
||||
backend: VmBackend, handle: VmHandle, deadline: float, poll: float
|
||||
) -> bool:
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
if backend.is_running(handle):
|
||||
return True
|
||||
except BackendError as exc:
|
||||
click.echo(f"[wait-ready] backend probe error: {exc}", err=True)
|
||||
time.sleep(poll)
|
||||
return False
|
||||
|
||||
|
||||
def _wait_for_ip(
|
||||
backend: VmBackend, handle: VmHandle, deadline: float, poll: float
|
||||
) -> str | None:
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
ip = backend.get_ip(handle)
|
||||
except BackendError:
|
||||
ip = None
|
||||
if ip:
|
||||
return ip
|
||||
time.sleep(poll)
|
||||
return None
|
||||
|
||||
|
||||
@click.command("wait-ready")
|
||||
@click.option(
|
||||
"--vmx",
|
||||
"--vm-path",
|
||||
"vmx",
|
||||
required=True,
|
||||
help="Path to the guest VMX file.",
|
||||
)
|
||||
@click.option(
|
||||
"--ip-address",
|
||||
"ip_address",
|
||||
default=None,
|
||||
help="Guest IP address. If omitted the backend is polled for it.",
|
||||
)
|
||||
@click.option(
|
||||
"--guest-os",
|
||||
"--transport",
|
||||
"guest_os",
|
||||
default="windows",
|
||||
show_default=True,
|
||||
callback=lambda _ctx, _p, v: _normalise_guest_os(v),
|
||||
help="windows/WinRM or linux/SSH (case-insensitive).",
|
||||
)
|
||||
@click.option("--timeout", "--timeout-seconds", "timeout", type=float, default=300.0, show_default=True)
|
||||
@click.option(
|
||||
"--poll-interval",
|
||||
"--poll-interval-seconds",
|
||||
"poll_interval",
|
||||
type=float,
|
||||
default=5.0,
|
||||
show_default=True,
|
||||
)
|
||||
@click.option("--vmrun-path", "vmrun_path", default=None, help="Override vmrun.exe path.")
|
||||
@click.option(
|
||||
"--credential-target",
|
||||
default=None,
|
||||
help="Override credential target name (defaults to config.guest_cred_target).",
|
||||
)
|
||||
@click.option("--ssh-user", default="ci_build", show_default=True)
|
||||
@click.option(
|
||||
"--ssh-key-path",
|
||||
default=None,
|
||||
help="SSH private key (defaults to config.ssh_key_path).",
|
||||
)
|
||||
@click.option(
|
||||
"--skip-ping",
|
||||
is_flag=True,
|
||||
default=False,
|
||||
help="Accepted for PS compatibility; ignored (Python uses transport probe directly).",
|
||||
)
|
||||
def wait_ready(
|
||||
vmx: str,
|
||||
ip_address: str | None,
|
||||
guest_os: str,
|
||||
timeout: float,
|
||||
poll_interval: float,
|
||||
vmrun_path: str | None,
|
||||
credential_target: str | None,
|
||||
ssh_user: str,
|
||||
ssh_key_path: str | None,
|
||||
skip_ping: bool,
|
||||
) -> None:
|
||||
"""Poll a guest until WinRM (Windows) or SSH (Linux) is responsive."""
|
||||
del skip_ping # accepted for PS compatibility, no-op
|
||||
config = load_config()
|
||||
if vmrun_path is not None:
|
||||
# Re-load backend with override; cheap, no real connections.
|
||||
from ci_orchestrator.backends.workstation import WorkstationVmrunBackend
|
||||
|
||||
backend: VmBackend = WorkstationVmrunBackend(vmrun_path=vmrun_path)
|
||||
else:
|
||||
backend = load_backend(config)
|
||||
handle = VmHandle(identifier=vmx)
|
||||
deadline = time.monotonic() + timeout
|
||||
|
||||
click.echo(f"[wait-ready] waiting for backend to report VM running: {vmx}")
|
||||
if not _wait_running(backend, handle, deadline, poll_interval):
|
||||
click.echo("[wait-ready] timeout waiting for VM to be running.", err=True)
|
||||
sys.exit(_EXIT_TIMEOUT_RUNNING)
|
||||
|
||||
if ip_address is None:
|
||||
ip = _wait_for_ip(backend, handle, deadline, poll_interval)
|
||||
if not ip:
|
||||
click.echo("[wait-ready] timeout waiting for guest IP.", err=True)
|
||||
sys.exit(_EXIT_TIMEOUT_IP)
|
||||
click.echo(f"[wait-ready] guest IP: {ip}")
|
||||
else:
|
||||
ip = ip_address
|
||||
click.echo(f"[wait-ready] using provided IP: {ip}")
|
||||
|
||||
if guest_os == "windows":
|
||||
store = KeyringCredentialStore()
|
||||
target = credential_target or config.guest_cred_target
|
||||
cred = store.get(target)
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
with WinRmTransport(ip, cred.username, cred.password) as t:
|
||||
if t.is_ready():
|
||||
click.echo("[wait-ready] WinRM ready.")
|
||||
return
|
||||
except TransportError as exc:
|
||||
click.echo(f"[wait-ready] WinRM probe failed: {exc}", err=True)
|
||||
time.sleep(poll_interval)
|
||||
else:
|
||||
key_path: str | None = ssh_key_path
|
||||
if key_path is None and config.ssh_key_path is not None:
|
||||
key_path = str(config.ssh_key_path)
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
with SshTransport(ip, username=ssh_user, key_path=key_path) as t:
|
||||
if t.is_ready():
|
||||
click.echo("[wait-ready] SSH ready.")
|
||||
return
|
||||
except TransportError as exc:
|
||||
click.echo(f"[wait-ready] SSH probe failed: {exc}", err=True)
|
||||
time.sleep(poll_interval)
|
||||
|
||||
click.echo("[wait-ready] timeout waiting for transport to be ready.", err=True)
|
||||
sys.exit(_EXIT_TIMEOUT_TRANSPORT)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"KeyringCredentialStore",
|
||||
"SshTransport",
|
||||
"WinRmTransport",
|
||||
"load_backend",
|
||||
"wait_ready",
|
||||
]
|
||||
Reference in New Issue
Block a user