Simone
80f6661ad5
Implements Phase A2 of plans/implementation-plan-A-B.md: - commands/wait.py -> wait-ready (replaces Wait-VMReady.ps1) - commands/vm.py -> vm remove + vm cleanup (replaces Remove-BuildVM.ps1, Cleanup-OrphanedBuildVMs.ps1); cleanup accepts injected VmBackend (Phase C ESXi hook preserved) - commands/monitor.py -> monitor disk + monitor runner (replaces Watch-DiskSpace.ps1, Watch-RunnerHealth.ps1) - commands/report.py -> report job (replaces Get-CIJobSummary.ps1) Each PS script reduced to a 3-line shim that delegates to the Python CLI and preserves \0. Tests: 69 pytest cases across new test_commands_*.py modules; ruff clean, mypy --strict clean, coverage 74.5% (>=70 gate).
460 lines
15 KiB
Python
460 lines
15 KiB
Python
"""``monitor`` sub-commands.
|
|
|
|
Phase A2 ports the maintenance/monitoring leaf scripts:
|
|
|
|
* ``monitor disk`` — replaces ``scripts/Watch-DiskSpace.ps1``
|
|
* ``monitor runner`` — replaces ``scripts/Watch-RunnerHealth.ps1``
|
|
|
|
Both commands are designed to run from a scheduler (Windows Task Scheduler
|
|
in Phase A, systemd timers in Phase B) and emit warnings via the platform
|
|
event log when available, plus optional Discord/Gitea webhooks via stdlib
|
|
``urllib`` (no extra dependency).
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import logging
|
|
import os
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
import urllib.error
|
|
import urllib.request
|
|
from datetime import UTC, datetime, timedelta
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
import click
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
# ──────────────────────────────────────────────────────────────────── helpers
|
|
|
|
|
|
def _post_webhook(url: str, content: str, *, timeout: float = 10.0) -> bool:
|
|
"""POST a Discord/Gitea-compatible JSON payload. Returns True on success."""
|
|
if not url:
|
|
return False
|
|
payload = json.dumps({"content": content}).encode("utf-8")
|
|
req = urllib.request.Request(
|
|
url,
|
|
data=payload,
|
|
method="POST",
|
|
headers={"Content-Type": "application/json"},
|
|
)
|
|
try:
|
|
with urllib.request.urlopen(req, timeout=timeout) as resp:
|
|
return bool(200 <= resp.status < 300)
|
|
except (urllib.error.URLError, TimeoutError, OSError) as exc:
|
|
logger.warning("Webhook POST failed: %s", exc)
|
|
return False
|
|
|
|
|
|
def _write_windows_event(
|
|
source: str, event_id: int, entry_type: str, message: str
|
|
) -> bool:
|
|
"""Write a Windows Application Event Log entry via ``eventcreate.exe``.
|
|
|
|
Avoids the ``pywin32`` dependency. Silently no-op on non-Windows hosts.
|
|
Returns True on success.
|
|
"""
|
|
if os.name != "nt":
|
|
return False
|
|
eventcreate = shutil.which("eventcreate") or shutil.which("eventcreate.exe")
|
|
if eventcreate is None:
|
|
return False
|
|
type_map = {"Information": "INFORMATION", "Warning": "WARNING", "Error": "ERROR"}
|
|
cmd = [
|
|
eventcreate,
|
|
"/T", type_map.get(entry_type, "WARNING"),
|
|
"/ID", str(event_id),
|
|
"/L", "APPLICATION",
|
|
"/SO", source,
|
|
"/D", message,
|
|
]
|
|
try:
|
|
result = subprocess.run(
|
|
cmd, capture_output=True, text=True, check=False, timeout=10.0
|
|
)
|
|
except (subprocess.TimeoutExpired, OSError) as exc:
|
|
logger.warning("eventcreate failed: %s", exc)
|
|
return False
|
|
if result.returncode != 0:
|
|
logger.warning("eventcreate returned %d: %s", result.returncode, result.stderr)
|
|
return False
|
|
return True
|
|
|
|
|
|
# ─────────────────────────────────────────────────────────────────────── group
|
|
|
|
|
|
@click.group("monitor")
|
|
def monitor() -> None:
|
|
"""Host monitoring commands (disk, runner)."""
|
|
|
|
|
|
# ──────────────────────────────────────────────────────────────────────── disk
|
|
|
|
|
|
def _resolve_drive_path(drive: str) -> str:
|
|
"""Map a single-letter Windows drive (``F``) to a usable path.
|
|
|
|
On non-Windows the ``drive`` value is treated as a path itself.
|
|
"""
|
|
if os.name == "nt" and len(drive) == 1 and drive.isalpha():
|
|
return f"{drive.upper()}:\\"
|
|
return drive
|
|
|
|
|
|
@monitor.command("disk")
|
|
@click.option(
|
|
"--min-free-gb",
|
|
"min_free_gb",
|
|
type=click.IntRange(1, 2000),
|
|
default=50,
|
|
show_default=True,
|
|
)
|
|
@click.option(
|
|
"--drive-letter",
|
|
"drive_letter",
|
|
default="F" if os.name == "nt" else "/",
|
|
show_default=True,
|
|
help="Drive letter (Windows) or mount path (Linux).",
|
|
)
|
|
@click.option("--webhook-url", "webhook_url", default="", help="Optional Discord/Gitea webhook URL.")
|
|
@click.option(
|
|
"--json",
|
|
"as_json",
|
|
is_flag=True,
|
|
default=False,
|
|
help="Emit a single-line JSON object instead of human-readable output.",
|
|
)
|
|
def monitor_disk(
|
|
min_free_gb: int,
|
|
drive_letter: str,
|
|
webhook_url: str,
|
|
as_json: bool,
|
|
) -> None:
|
|
"""Check free space on a CI host drive and alert if below threshold.
|
|
|
|
Exit codes mirror the original PowerShell script:
|
|
|
|
* 0 = drive OK
|
|
* 1 = below threshold (alert raised)
|
|
* 2 = drive not found
|
|
"""
|
|
target = _resolve_drive_path(drive_letter)
|
|
try:
|
|
usage = shutil.disk_usage(target)
|
|
except (FileNotFoundError, OSError) as exc:
|
|
click.echo(f"[disk] drive {drive_letter!r} not found: {exc}", err=True)
|
|
sys.exit(2)
|
|
|
|
gb = 1024.0 ** 3
|
|
free_gb = round(usage.free / gb, 1)
|
|
total_gb = round(usage.total / gb, 1)
|
|
pct_free = round(free_gb / total_gb * 100, 0) if total_gb > 0 else 0
|
|
|
|
payload = {
|
|
"drive": drive_letter,
|
|
"free_gb": free_gb,
|
|
"total_gb": total_gb,
|
|
"percent_free": pct_free,
|
|
"threshold_gb": min_free_gb,
|
|
"status": "ok" if free_gb >= min_free_gb else "alert",
|
|
}
|
|
|
|
if free_gb >= min_free_gb:
|
|
if as_json:
|
|
click.echo(json.dumps(payload))
|
|
else:
|
|
click.echo(
|
|
f"[disk] drive {drive_letter}: {free_gb} GB free / {total_gb} GB "
|
|
f"({pct_free}%) - OK (threshold: {min_free_gb} GB)"
|
|
)
|
|
return
|
|
|
|
msg = (
|
|
f"CI host drive {drive_letter}: free space {free_gb} GB "
|
|
f"({pct_free}%) is below the {min_free_gb} GB threshold. "
|
|
"vmrun linked clones may fail silently."
|
|
)
|
|
if as_json:
|
|
payload["message"] = msg
|
|
click.echo(json.dumps(payload))
|
|
else:
|
|
click.echo(f"[disk] WARNING: {msg}", err=True)
|
|
|
|
_write_windows_event("CI-DiskSpaceAlert", 1001, "Warning", msg)
|
|
if webhook_url:
|
|
_post_webhook(webhook_url, f"[WARNING] CI Disk Alert -- {msg}")
|
|
sys.exit(1)
|
|
|
|
|
|
# ────────────────────────────────────────────────────────────────────── runner
|
|
|
|
|
|
def _service_status(service_name: str) -> str | None:
|
|
"""Return ``Running``/``Stopped``/``Unknown``, or ``None`` if not installed."""
|
|
if os.name == "nt":
|
|
sc = shutil.which("sc") or r"C:\Windows\System32\sc.exe"
|
|
try:
|
|
r = subprocess.run(
|
|
[sc, "query", service_name],
|
|
capture_output=True,
|
|
text=True,
|
|
check=False,
|
|
timeout=10.0,
|
|
)
|
|
except (subprocess.TimeoutExpired, OSError):
|
|
return None
|
|
if r.returncode != 0:
|
|
return None
|
|
out = r.stdout.upper()
|
|
if "RUNNING" in out:
|
|
return "Running"
|
|
if "STOPPED" in out:
|
|
return "Stopped"
|
|
return "Unknown"
|
|
# Linux / systemd
|
|
systemctl = shutil.which("systemctl")
|
|
if systemctl is None:
|
|
return None
|
|
try:
|
|
r = subprocess.run(
|
|
[systemctl, "is-active", service_name],
|
|
capture_output=True,
|
|
text=True,
|
|
check=False,
|
|
timeout=10.0,
|
|
)
|
|
except (subprocess.TimeoutExpired, OSError):
|
|
return None
|
|
state = r.stdout.strip().lower()
|
|
if state == "active":
|
|
return "Running"
|
|
if state in {"inactive", "failed", "deactivating"}:
|
|
return "Stopped"
|
|
return None # unknown / not installed
|
|
|
|
|
|
def _restart_service(service_name: str) -> bool:
|
|
"""Attempt to (re)start ``service_name``. Returns True if now running."""
|
|
if os.name == "nt":
|
|
sc = shutil.which("sc") or r"C:\Windows\System32\sc.exe"
|
|
# `sc start` returns 1056 if already running; ignore.
|
|
try:
|
|
subprocess.run(
|
|
[sc, "start", service_name],
|
|
capture_output=True,
|
|
text=True,
|
|
check=False,
|
|
timeout=30.0,
|
|
)
|
|
except (subprocess.TimeoutExpired, OSError):
|
|
return False
|
|
else:
|
|
systemctl = shutil.which("systemctl")
|
|
if systemctl is None:
|
|
return False
|
|
try:
|
|
subprocess.run(
|
|
[systemctl, "restart", service_name],
|
|
capture_output=True,
|
|
text=True,
|
|
check=False,
|
|
timeout=30.0,
|
|
)
|
|
except (subprocess.TimeoutExpired, OSError):
|
|
return False
|
|
time.sleep(5.0)
|
|
return _service_status(service_name) == "Running"
|
|
|
|
|
|
def _load_restart_log(path: Path) -> list[str]:
|
|
if not path.is_file():
|
|
return []
|
|
try:
|
|
raw = path.read_text(encoding="utf-8")
|
|
except OSError:
|
|
return []
|
|
if not raw.strip():
|
|
return []
|
|
try:
|
|
data = json.loads(raw)
|
|
except json.JSONDecodeError:
|
|
return []
|
|
if isinstance(data, list):
|
|
return [str(x) for x in data]
|
|
return []
|
|
|
|
|
|
def _save_restart_log(path: Path, entries: list[str]) -> None:
|
|
try:
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
path.write_text(json.dumps(entries), encoding="utf-8")
|
|
except OSError as exc:
|
|
logger.warning("Could not persist restart log: %s", exc)
|
|
|
|
|
|
def _gitea_runners_online(api_url: str, token: str, *, timeout: float = 10.0) -> int | None:
|
|
"""Return number of online runners reported by Gitea, or ``None`` on failure."""
|
|
api = api_url.rstrip("/") + "/api/v1/admin/runners"
|
|
req = urllib.request.Request(
|
|
api, headers={"Authorization": f"token {token}"}
|
|
)
|
|
try:
|
|
with urllib.request.urlopen(req, timeout=timeout) as resp:
|
|
data = json.loads(resp.read().decode("utf-8"))
|
|
except (urllib.error.URLError, TimeoutError, OSError, json.JSONDecodeError) as exc:
|
|
logger.warning("Gitea API call failed: %s", exc)
|
|
return None
|
|
if isinstance(data, list):
|
|
return sum(1 for r in data if isinstance(r, dict) and r.get("online") is True)
|
|
return None
|
|
|
|
|
|
@monitor.command("runner")
|
|
@click.option(
|
|
"--service-name",
|
|
"service_name",
|
|
default="act_runner",
|
|
show_default=True,
|
|
)
|
|
@click.option(
|
|
"--max-restarts",
|
|
type=click.IntRange(1, 10),
|
|
default=3,
|
|
show_default=True,
|
|
)
|
|
@click.option(
|
|
"--state-dir",
|
|
"state_dir",
|
|
default=r"F:\CI\State" if os.name == "nt" else "/var/lib/ci/state",
|
|
show_default=True,
|
|
)
|
|
@click.option("--webhook-url", "webhook_url", default="")
|
|
@click.option("--gitea-url", "gitea_url", default="")
|
|
@click.option(
|
|
"--gitea-credential-target",
|
|
"gitea_credential_target",
|
|
default="",
|
|
help="Credential target name for the Gitea admin PAT.",
|
|
)
|
|
def monitor_runner(
|
|
service_name: str,
|
|
max_restarts: int,
|
|
state_dir: str,
|
|
webhook_url: str,
|
|
gitea_url: str,
|
|
gitea_credential_target: str,
|
|
) -> None:
|
|
"""Monitor act_runner; auto-restart up to ``--max-restarts`` per hour.
|
|
|
|
Exit codes:
|
|
0 = healthy (or restart succeeded)
|
|
1 = service down + restart failed/limit reached
|
|
2 = service not installed
|
|
"""
|
|
state_dir_path = Path(state_dir)
|
|
maintenance_flag = state_dir_path / "runner-maintenance.flag"
|
|
if maintenance_flag.is_file():
|
|
click.echo(
|
|
f"[runner] maintenance mode active ({maintenance_flag}) - skipping."
|
|
)
|
|
return
|
|
|
|
status = _service_status(service_name)
|
|
if status is None:
|
|
click.echo(
|
|
f"[runner] service {service_name!r} not found - is act_runner installed?",
|
|
err=True,
|
|
)
|
|
sys.exit(2)
|
|
|
|
if status == "Running":
|
|
click.echo(f"[runner] {service_name} is Running - OK")
|
|
if gitea_url and gitea_credential_target:
|
|
try:
|
|
from ci_orchestrator.credentials import KeyringCredentialStore
|
|
|
|
cred = KeyringCredentialStore().get(gitea_credential_target)
|
|
online = _gitea_runners_online(gitea_url, cred.password)
|
|
except KeyError as exc:
|
|
click.echo(f"[runner] could not load Gitea PAT: {exc}", err=True)
|
|
online = None
|
|
if online is None:
|
|
click.echo("[runner] Gitea API check skipped/failed.")
|
|
elif online == 0:
|
|
msg = (
|
|
f"Gitea API ({gitea_url}) reports 0 online runners. "
|
|
f"{service_name} is Running - possible registration issue."
|
|
)
|
|
click.echo(f"[runner] WARNING: {msg}", err=True)
|
|
_write_windows_event("CI-RunnerHealth", 1005, "Warning", msg)
|
|
if webhook_url:
|
|
_post_webhook(webhook_url, f"[WARNING] CI Runner Alert -- {msg}")
|
|
else:
|
|
click.echo(f"[runner] Gitea API: {online} runner(s) online - OK")
|
|
return
|
|
|
|
# Not running → check rate limit and try to restart.
|
|
cooldown_file = state_dir_path / "runner-restart-log.json"
|
|
log = _load_restart_log(cooldown_file)
|
|
cutoff = datetime.now(tz=UTC) - timedelta(hours=1)
|
|
pruned: list[str] = []
|
|
for entry in log:
|
|
try:
|
|
ts = datetime.fromisoformat(entry)
|
|
except ValueError:
|
|
continue
|
|
if ts.tzinfo is None:
|
|
ts = ts.replace(tzinfo=UTC)
|
|
if ts > cutoff:
|
|
pruned.append(entry)
|
|
log = pruned
|
|
|
|
state_msg = (
|
|
f"{service_name} service is '{status}'. "
|
|
f"Auto-restarts in last hour: {len(log)} / {max_restarts}."
|
|
)
|
|
click.echo(f"[runner] WARNING: {state_msg}", err=True)
|
|
_write_windows_event("CI-RunnerHealth", 1002, "Warning", state_msg)
|
|
|
|
if len(log) >= max_restarts:
|
|
limit_msg = (
|
|
f"{service_name} auto-restart limit ({max_restarts}/h) reached - "
|
|
"manual intervention required."
|
|
)
|
|
click.echo(f"[runner] {limit_msg}", err=True)
|
|
_write_windows_event("CI-RunnerHealth", 1004, "Error", limit_msg)
|
|
if webhook_url:
|
|
_post_webhook(webhook_url, f"[ALERT] CI Runner Alert -- {limit_msg}")
|
|
sys.exit(1)
|
|
|
|
click.echo(
|
|
f"[runner] restarting {service_name} (attempt {len(log) + 1} of {max_restarts})..."
|
|
)
|
|
ok = _restart_service(service_name)
|
|
log.append(datetime.now(tz=UTC).isoformat())
|
|
_save_restart_log(cooldown_file, log)
|
|
new_status = _service_status(service_name) or "Unknown"
|
|
restart_msg = f"{service_name} restarted - new status: {new_status}."
|
|
entry_type = "Information" if ok else "Warning"
|
|
click.echo(f"[runner] {restart_msg}")
|
|
_write_windows_event("CI-RunnerHealth", 1003, entry_type, restart_msg)
|
|
prefix = "[WARNING]" if ok else "[ALERT]"
|
|
if webhook_url:
|
|
_post_webhook(webhook_url, f"{prefix} CI Runner Alert -- {restart_msg}")
|
|
sys.exit(0 if ok else 1)
|
|
|
|
|
|
def _ignored(_name: str, _value: Any) -> None:
|
|
"""Marker so type checkers see we deliberately drop a value."""
|
|
|
|
|
|
__all__ = ["monitor"]
|