feat(a2): port leaf PS scripts to ci_orchestrator CLI
Implements Phase A2 of plans/implementation-plan-A-B.md: - commands/wait.py -> wait-ready (replaces Wait-VMReady.ps1) - commands/vm.py -> vm remove + vm cleanup (replaces Remove-BuildVM.ps1, Cleanup-OrphanedBuildVMs.ps1); cleanup accepts injected VmBackend (Phase C ESXi hook preserved) - commands/monitor.py -> monitor disk + monitor runner (replaces Watch-DiskSpace.ps1, Watch-RunnerHealth.ps1) - commands/report.py -> report job (replaces Get-CIJobSummary.ps1) Each PS script reduced to a 3-line shim that delegates to the Python CLI and preserves \0. Tests: 69 pytest cases across new test_commands_*.py modules; ruff clean, mypy --strict clean, coverage 74.5% (>=70 gate).
This commit is contained in:
@@ -0,0 +1,459 @@
|
||||
"""``monitor`` sub-commands.
|
||||
|
||||
Phase A2 ports the maintenance/monitoring leaf scripts:
|
||||
|
||||
* ``monitor disk`` — replaces ``scripts/Watch-DiskSpace.ps1``
|
||||
* ``monitor runner`` — replaces ``scripts/Watch-RunnerHealth.ps1``
|
||||
|
||||
Both commands are designed to run from a scheduler (Windows Task Scheduler
|
||||
in Phase A, systemd timers in Phase B) and emit warnings via the platform
|
||||
event log when available, plus optional Discord/Gitea webhooks via stdlib
|
||||
``urllib`` (no extra dependency).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import click
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
# ──────────────────────────────────────────────────────────────────── helpers
|
||||
|
||||
|
||||
def _post_webhook(url: str, content: str, *, timeout: float = 10.0) -> bool:
|
||||
"""POST a Discord/Gitea-compatible JSON payload. Returns True on success."""
|
||||
if not url:
|
||||
return False
|
||||
payload = json.dumps({"content": content}).encode("utf-8")
|
||||
req = urllib.request.Request(
|
||||
url,
|
||||
data=payload,
|
||||
method="POST",
|
||||
headers={"Content-Type": "application/json"},
|
||||
)
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=timeout) as resp:
|
||||
return bool(200 <= resp.status < 300)
|
||||
except (urllib.error.URLError, TimeoutError, OSError) as exc:
|
||||
logger.warning("Webhook POST failed: %s", exc)
|
||||
return False
|
||||
|
||||
|
||||
def _write_windows_event(
|
||||
source: str, event_id: int, entry_type: str, message: str
|
||||
) -> bool:
|
||||
"""Write a Windows Application Event Log entry via ``eventcreate.exe``.
|
||||
|
||||
Avoids the ``pywin32`` dependency. Silently no-op on non-Windows hosts.
|
||||
Returns True on success.
|
||||
"""
|
||||
if os.name != "nt":
|
||||
return False
|
||||
eventcreate = shutil.which("eventcreate") or shutil.which("eventcreate.exe")
|
||||
if eventcreate is None:
|
||||
return False
|
||||
type_map = {"Information": "INFORMATION", "Warning": "WARNING", "Error": "ERROR"}
|
||||
cmd = [
|
||||
eventcreate,
|
||||
"/T", type_map.get(entry_type, "WARNING"),
|
||||
"/ID", str(event_id),
|
||||
"/L", "APPLICATION",
|
||||
"/SO", source,
|
||||
"/D", message,
|
||||
]
|
||||
try:
|
||||
result = subprocess.run(
|
||||
cmd, capture_output=True, text=True, check=False, timeout=10.0
|
||||
)
|
||||
except (subprocess.TimeoutExpired, OSError) as exc:
|
||||
logger.warning("eventcreate failed: %s", exc)
|
||||
return False
|
||||
if result.returncode != 0:
|
||||
logger.warning("eventcreate returned %d: %s", result.returncode, result.stderr)
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────────── group
|
||||
|
||||
|
||||
@click.group("monitor")
|
||||
def monitor() -> None:
|
||||
"""Host monitoring commands (disk, runner)."""
|
||||
|
||||
|
||||
# ──────────────────────────────────────────────────────────────────────── disk
|
||||
|
||||
|
||||
def _resolve_drive_path(drive: str) -> str:
|
||||
"""Map a single-letter Windows drive (``F``) to a usable path.
|
||||
|
||||
On non-Windows the ``drive`` value is treated as a path itself.
|
||||
"""
|
||||
if os.name == "nt" and len(drive) == 1 and drive.isalpha():
|
||||
return f"{drive.upper()}:\\"
|
||||
return drive
|
||||
|
||||
|
||||
@monitor.command("disk")
|
||||
@click.option(
|
||||
"--min-free-gb",
|
||||
"min_free_gb",
|
||||
type=click.IntRange(1, 2000),
|
||||
default=50,
|
||||
show_default=True,
|
||||
)
|
||||
@click.option(
|
||||
"--drive-letter",
|
||||
"drive_letter",
|
||||
default="F" if os.name == "nt" else "/",
|
||||
show_default=True,
|
||||
help="Drive letter (Windows) or mount path (Linux).",
|
||||
)
|
||||
@click.option("--webhook-url", "webhook_url", default="", help="Optional Discord/Gitea webhook URL.")
|
||||
@click.option(
|
||||
"--json",
|
||||
"as_json",
|
||||
is_flag=True,
|
||||
default=False,
|
||||
help="Emit a single-line JSON object instead of human-readable output.",
|
||||
)
|
||||
def monitor_disk(
|
||||
min_free_gb: int,
|
||||
drive_letter: str,
|
||||
webhook_url: str,
|
||||
as_json: bool,
|
||||
) -> None:
|
||||
"""Check free space on a CI host drive and alert if below threshold.
|
||||
|
||||
Exit codes mirror the original PowerShell script:
|
||||
|
||||
* 0 = drive OK
|
||||
* 1 = below threshold (alert raised)
|
||||
* 2 = drive not found
|
||||
"""
|
||||
target = _resolve_drive_path(drive_letter)
|
||||
try:
|
||||
usage = shutil.disk_usage(target)
|
||||
except (FileNotFoundError, OSError) as exc:
|
||||
click.echo(f"[disk] drive {drive_letter!r} not found: {exc}", err=True)
|
||||
sys.exit(2)
|
||||
|
||||
gb = 1024.0 ** 3
|
||||
free_gb = round(usage.free / gb, 1)
|
||||
total_gb = round(usage.total / gb, 1)
|
||||
pct_free = round(free_gb / total_gb * 100, 0) if total_gb > 0 else 0
|
||||
|
||||
payload = {
|
||||
"drive": drive_letter,
|
||||
"free_gb": free_gb,
|
||||
"total_gb": total_gb,
|
||||
"percent_free": pct_free,
|
||||
"threshold_gb": min_free_gb,
|
||||
"status": "ok" if free_gb >= min_free_gb else "alert",
|
||||
}
|
||||
|
||||
if free_gb >= min_free_gb:
|
||||
if as_json:
|
||||
click.echo(json.dumps(payload))
|
||||
else:
|
||||
click.echo(
|
||||
f"[disk] drive {drive_letter}: {free_gb} GB free / {total_gb} GB "
|
||||
f"({pct_free}%) - OK (threshold: {min_free_gb} GB)"
|
||||
)
|
||||
return
|
||||
|
||||
msg = (
|
||||
f"CI host drive {drive_letter}: free space {free_gb} GB "
|
||||
f"({pct_free}%) is below the {min_free_gb} GB threshold. "
|
||||
"vmrun linked clones may fail silently."
|
||||
)
|
||||
if as_json:
|
||||
payload["message"] = msg
|
||||
click.echo(json.dumps(payload))
|
||||
else:
|
||||
click.echo(f"[disk] WARNING: {msg}", err=True)
|
||||
|
||||
_write_windows_event("CI-DiskSpaceAlert", 1001, "Warning", msg)
|
||||
if webhook_url:
|
||||
_post_webhook(webhook_url, f"[WARNING] CI Disk Alert -- {msg}")
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
# ────────────────────────────────────────────────────────────────────── runner
|
||||
|
||||
|
||||
def _service_status(service_name: str) -> str | None:
|
||||
"""Return ``Running``/``Stopped``/``Unknown``, or ``None`` if not installed."""
|
||||
if os.name == "nt":
|
||||
sc = shutil.which("sc") or r"C:\Windows\System32\sc.exe"
|
||||
try:
|
||||
r = subprocess.run(
|
||||
[sc, "query", service_name],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=False,
|
||||
timeout=10.0,
|
||||
)
|
||||
except (subprocess.TimeoutExpired, OSError):
|
||||
return None
|
||||
if r.returncode != 0:
|
||||
return None
|
||||
out = r.stdout.upper()
|
||||
if "RUNNING" in out:
|
||||
return "Running"
|
||||
if "STOPPED" in out:
|
||||
return "Stopped"
|
||||
return "Unknown"
|
||||
# Linux / systemd
|
||||
systemctl = shutil.which("systemctl")
|
||||
if systemctl is None:
|
||||
return None
|
||||
try:
|
||||
r = subprocess.run(
|
||||
[systemctl, "is-active", service_name],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=False,
|
||||
timeout=10.0,
|
||||
)
|
||||
except (subprocess.TimeoutExpired, OSError):
|
||||
return None
|
||||
state = r.stdout.strip().lower()
|
||||
if state == "active":
|
||||
return "Running"
|
||||
if state in {"inactive", "failed", "deactivating"}:
|
||||
return "Stopped"
|
||||
return None # unknown / not installed
|
||||
|
||||
|
||||
def _restart_service(service_name: str) -> bool:
|
||||
"""Attempt to (re)start ``service_name``. Returns True if now running."""
|
||||
if os.name == "nt":
|
||||
sc = shutil.which("sc") or r"C:\Windows\System32\sc.exe"
|
||||
# `sc start` returns 1056 if already running; ignore.
|
||||
try:
|
||||
subprocess.run(
|
||||
[sc, "start", service_name],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=False,
|
||||
timeout=30.0,
|
||||
)
|
||||
except (subprocess.TimeoutExpired, OSError):
|
||||
return False
|
||||
else:
|
||||
systemctl = shutil.which("systemctl")
|
||||
if systemctl is None:
|
||||
return False
|
||||
try:
|
||||
subprocess.run(
|
||||
[systemctl, "restart", service_name],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=False,
|
||||
timeout=30.0,
|
||||
)
|
||||
except (subprocess.TimeoutExpired, OSError):
|
||||
return False
|
||||
time.sleep(5.0)
|
||||
return _service_status(service_name) == "Running"
|
||||
|
||||
|
||||
def _load_restart_log(path: Path) -> list[str]:
|
||||
if not path.is_file():
|
||||
return []
|
||||
try:
|
||||
raw = path.read_text(encoding="utf-8")
|
||||
except OSError:
|
||||
return []
|
||||
if not raw.strip():
|
||||
return []
|
||||
try:
|
||||
data = json.loads(raw)
|
||||
except json.JSONDecodeError:
|
||||
return []
|
||||
if isinstance(data, list):
|
||||
return [str(x) for x in data]
|
||||
return []
|
||||
|
||||
|
||||
def _save_restart_log(path: Path, entries: list[str]) -> None:
|
||||
try:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(json.dumps(entries), encoding="utf-8")
|
||||
except OSError as exc:
|
||||
logger.warning("Could not persist restart log: %s", exc)
|
||||
|
||||
|
||||
def _gitea_runners_online(api_url: str, token: str, *, timeout: float = 10.0) -> int | None:
|
||||
"""Return number of online runners reported by Gitea, or ``None`` on failure."""
|
||||
api = api_url.rstrip("/") + "/api/v1/admin/runners"
|
||||
req = urllib.request.Request(
|
||||
api, headers={"Authorization": f"token {token}"}
|
||||
)
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=timeout) as resp:
|
||||
data = json.loads(resp.read().decode("utf-8"))
|
||||
except (urllib.error.URLError, TimeoutError, OSError, json.JSONDecodeError) as exc:
|
||||
logger.warning("Gitea API call failed: %s", exc)
|
||||
return None
|
||||
if isinstance(data, list):
|
||||
return sum(1 for r in data if isinstance(r, dict) and r.get("online") is True)
|
||||
return None
|
||||
|
||||
|
||||
@monitor.command("runner")
|
||||
@click.option(
|
||||
"--service-name",
|
||||
"service_name",
|
||||
default="act_runner",
|
||||
show_default=True,
|
||||
)
|
||||
@click.option(
|
||||
"--max-restarts",
|
||||
type=click.IntRange(1, 10),
|
||||
default=3,
|
||||
show_default=True,
|
||||
)
|
||||
@click.option(
|
||||
"--state-dir",
|
||||
"state_dir",
|
||||
default=r"F:\CI\State" if os.name == "nt" else "/var/lib/ci/state",
|
||||
show_default=True,
|
||||
)
|
||||
@click.option("--webhook-url", "webhook_url", default="")
|
||||
@click.option("--gitea-url", "gitea_url", default="")
|
||||
@click.option(
|
||||
"--gitea-credential-target",
|
||||
"gitea_credential_target",
|
||||
default="",
|
||||
help="Credential target name for the Gitea admin PAT.",
|
||||
)
|
||||
def monitor_runner(
|
||||
service_name: str,
|
||||
max_restarts: int,
|
||||
state_dir: str,
|
||||
webhook_url: str,
|
||||
gitea_url: str,
|
||||
gitea_credential_target: str,
|
||||
) -> None:
|
||||
"""Monitor act_runner; auto-restart up to ``--max-restarts`` per hour.
|
||||
|
||||
Exit codes:
|
||||
0 = healthy (or restart succeeded)
|
||||
1 = service down + restart failed/limit reached
|
||||
2 = service not installed
|
||||
"""
|
||||
state_dir_path = Path(state_dir)
|
||||
maintenance_flag = state_dir_path / "runner-maintenance.flag"
|
||||
if maintenance_flag.is_file():
|
||||
click.echo(
|
||||
f"[runner] maintenance mode active ({maintenance_flag}) - skipping."
|
||||
)
|
||||
return
|
||||
|
||||
status = _service_status(service_name)
|
||||
if status is None:
|
||||
click.echo(
|
||||
f"[runner] service {service_name!r} not found - is act_runner installed?",
|
||||
err=True,
|
||||
)
|
||||
sys.exit(2)
|
||||
|
||||
if status == "Running":
|
||||
click.echo(f"[runner] {service_name} is Running - OK")
|
||||
if gitea_url and gitea_credential_target:
|
||||
try:
|
||||
from ci_orchestrator.credentials import KeyringCredentialStore
|
||||
|
||||
cred = KeyringCredentialStore().get(gitea_credential_target)
|
||||
online = _gitea_runners_online(gitea_url, cred.password)
|
||||
except KeyError as exc:
|
||||
click.echo(f"[runner] could not load Gitea PAT: {exc}", err=True)
|
||||
online = None
|
||||
if online is None:
|
||||
click.echo("[runner] Gitea API check skipped/failed.")
|
||||
elif online == 0:
|
||||
msg = (
|
||||
f"Gitea API ({gitea_url}) reports 0 online runners. "
|
||||
f"{service_name} is Running - possible registration issue."
|
||||
)
|
||||
click.echo(f"[runner] WARNING: {msg}", err=True)
|
||||
_write_windows_event("CI-RunnerHealth", 1005, "Warning", msg)
|
||||
if webhook_url:
|
||||
_post_webhook(webhook_url, f"[WARNING] CI Runner Alert -- {msg}")
|
||||
else:
|
||||
click.echo(f"[runner] Gitea API: {online} runner(s) online - OK")
|
||||
return
|
||||
|
||||
# Not running → check rate limit and try to restart.
|
||||
cooldown_file = state_dir_path / "runner-restart-log.json"
|
||||
log = _load_restart_log(cooldown_file)
|
||||
cutoff = datetime.now(tz=UTC) - timedelta(hours=1)
|
||||
pruned: list[str] = []
|
||||
for entry in log:
|
||||
try:
|
||||
ts = datetime.fromisoformat(entry)
|
||||
except ValueError:
|
||||
continue
|
||||
if ts.tzinfo is None:
|
||||
ts = ts.replace(tzinfo=UTC)
|
||||
if ts > cutoff:
|
||||
pruned.append(entry)
|
||||
log = pruned
|
||||
|
||||
state_msg = (
|
||||
f"{service_name} service is '{status}'. "
|
||||
f"Auto-restarts in last hour: {len(log)} / {max_restarts}."
|
||||
)
|
||||
click.echo(f"[runner] WARNING: {state_msg}", err=True)
|
||||
_write_windows_event("CI-RunnerHealth", 1002, "Warning", state_msg)
|
||||
|
||||
if len(log) >= max_restarts:
|
||||
limit_msg = (
|
||||
f"{service_name} auto-restart limit ({max_restarts}/h) reached - "
|
||||
"manual intervention required."
|
||||
)
|
||||
click.echo(f"[runner] {limit_msg}", err=True)
|
||||
_write_windows_event("CI-RunnerHealth", 1004, "Error", limit_msg)
|
||||
if webhook_url:
|
||||
_post_webhook(webhook_url, f"[ALERT] CI Runner Alert -- {limit_msg}")
|
||||
sys.exit(1)
|
||||
|
||||
click.echo(
|
||||
f"[runner] restarting {service_name} (attempt {len(log) + 1} of {max_restarts})..."
|
||||
)
|
||||
ok = _restart_service(service_name)
|
||||
log.append(datetime.now(tz=UTC).isoformat())
|
||||
_save_restart_log(cooldown_file, log)
|
||||
new_status = _service_status(service_name) or "Unknown"
|
||||
restart_msg = f"{service_name} restarted - new status: {new_status}."
|
||||
entry_type = "Information" if ok else "Warning"
|
||||
click.echo(f"[runner] {restart_msg}")
|
||||
_write_windows_event("CI-RunnerHealth", 1003, entry_type, restart_msg)
|
||||
prefix = "[WARNING]" if ok else "[ALERT]"
|
||||
if webhook_url:
|
||||
_post_webhook(webhook_url, f"{prefix} CI Runner Alert -- {restart_msg}")
|
||||
sys.exit(0 if ok else 1)
|
||||
|
||||
|
||||
def _ignored(_name: str, _value: Any) -> None:
|
||||
"""Marker so type checkers see we deliberately drop a value."""
|
||||
|
||||
|
||||
__all__ = ["monitor"]
|
||||
Reference in New Issue
Block a user