"""``monitor`` sub-commands. Phase A2 ports the maintenance/monitoring leaf scripts: * ``monitor disk`` — replaces ``scripts/Watch-DiskSpace.ps1`` * ``monitor runner`` — replaces ``scripts/Watch-RunnerHealth.ps1`` Both commands are designed to run from a scheduler (Windows Task Scheduler in Phase A, systemd timers in Phase B) and emit warnings via the platform event log when available, plus optional Discord/Gitea webhooks via stdlib ``urllib`` (no extra dependency). """ from __future__ import annotations import json import logging import os import shutil import subprocess import sys import time import urllib.error import urllib.request from datetime import UTC, datetime, timedelta from pathlib import Path from typing import Any import click logger = logging.getLogger(__name__) # ──────────────────────────────────────────────────────────────────── helpers def _post_webhook(url: str, content: str, *, timeout: float = 10.0) -> bool: """POST a Discord/Gitea-compatible JSON payload. Returns True on success.""" if not url: return False payload = json.dumps({"content": content}).encode("utf-8") req = urllib.request.Request( url, data=payload, method="POST", headers={"Content-Type": "application/json"}, ) try: with urllib.request.urlopen(req, timeout=timeout) as resp: return bool(200 <= resp.status < 300) except (urllib.error.URLError, TimeoutError, OSError) as exc: logger.warning("Webhook POST failed: %s", exc) return False def _write_windows_event( source: str, event_id: int, entry_type: str, message: str ) -> bool: """Write a Windows Application Event Log entry via ``eventcreate.exe``. Avoids the ``pywin32`` dependency. Silently no-op on non-Windows hosts. Returns True on success. """ if os.name != "nt": return False eventcreate = shutil.which("eventcreate") or shutil.which("eventcreate.exe") if eventcreate is None: return False type_map = {"Information": "INFORMATION", "Warning": "WARNING", "Error": "ERROR"} cmd = [ eventcreate, "/T", type_map.get(entry_type, "WARNING"), "/ID", str(event_id), "/L", "APPLICATION", "/SO", source, "/D", message, ] try: result = subprocess.run( cmd, capture_output=True, text=True, check=False, timeout=10.0 ) except (subprocess.TimeoutExpired, OSError) as exc: logger.warning("eventcreate failed: %s", exc) return False if result.returncode != 0: logger.warning("eventcreate returned %d: %s", result.returncode, result.stderr) return False return True # ─────────────────────────────────────────────────────────────────────── group @click.group("monitor") def monitor() -> None: """Host monitoring commands (disk, runner).""" # ──────────────────────────────────────────────────────────────────────── disk def _resolve_drive_path(drive: str) -> str: """Map a single-letter Windows drive (``F``) to a usable path. On non-Windows the ``drive`` value is treated as a path itself. """ if os.name == "nt" and len(drive) == 1 and drive.isalpha(): return f"{drive.upper()}:\\" return drive @monitor.command("disk") @click.option( "--min-free-gb", "min_free_gb", type=click.IntRange(1, 2000), default=50, show_default=True, ) @click.option( "--drive-letter", "drive_letter", default="F" if os.name == "nt" else "/", show_default=True, help="Drive letter (Windows) or mount path (Linux).", ) @click.option("--webhook-url", "webhook_url", default="", help="Optional Discord/Gitea webhook URL.") @click.option( "--json", "as_json", is_flag=True, default=False, help="Emit a single-line JSON object instead of human-readable output.", ) def monitor_disk( min_free_gb: int, drive_letter: str, webhook_url: str, as_json: bool, ) -> None: """Check free space on a CI host drive and alert if below threshold. Exit codes mirror the original PowerShell script: * 0 = drive OK * 1 = below threshold (alert raised) * 2 = drive not found """ target = _resolve_drive_path(drive_letter) try: usage = shutil.disk_usage(target) except (FileNotFoundError, OSError) as exc: click.echo(f"[disk] drive {drive_letter!r} not found: {exc}", err=True) sys.exit(2) gb = 1024.0 ** 3 free_gb = round(usage.free / gb, 1) total_gb = round(usage.total / gb, 1) pct_free = round(free_gb / total_gb * 100, 0) if total_gb > 0 else 0 payload = { "drive": drive_letter, "free_gb": free_gb, "total_gb": total_gb, "percent_free": pct_free, "threshold_gb": min_free_gb, "status": "ok" if free_gb >= min_free_gb else "alert", } if free_gb >= min_free_gb: if as_json: click.echo(json.dumps(payload)) else: click.echo( f"[disk] drive {drive_letter}: {free_gb} GB free / {total_gb} GB " f"({pct_free}%) - OK (threshold: {min_free_gb} GB)" ) return msg = ( f"CI host drive {drive_letter}: free space {free_gb} GB " f"({pct_free}%) is below the {min_free_gb} GB threshold. " "vmrun linked clones may fail silently." ) if as_json: payload["message"] = msg click.echo(json.dumps(payload)) else: click.echo(f"[disk] WARNING: {msg}", err=True) _write_windows_event("CI-DiskSpaceAlert", 1001, "Warning", msg) if webhook_url: _post_webhook(webhook_url, f"[WARNING] CI Disk Alert -- {msg}") sys.exit(1) # ────────────────────────────────────────────────────────────────────── runner def _service_status(service_name: str) -> str | None: """Return ``Running``/``Stopped``/``Unknown``, or ``None`` if not installed.""" if os.name == "nt": sc = shutil.which("sc") or r"C:\Windows\System32\sc.exe" try: r = subprocess.run( [sc, "query", service_name], capture_output=True, text=True, check=False, timeout=10.0, ) except (subprocess.TimeoutExpired, OSError): return None if r.returncode != 0: return None out = r.stdout.upper() if "RUNNING" in out: return "Running" if "STOPPED" in out: return "Stopped" return "Unknown" # Linux / systemd systemctl = shutil.which("systemctl") if systemctl is None: return None try: r = subprocess.run( [systemctl, "is-active", service_name], capture_output=True, text=True, check=False, timeout=10.0, ) except (subprocess.TimeoutExpired, OSError): return None state = r.stdout.strip().lower() if state == "active": return "Running" if state in {"inactive", "failed", "deactivating"}: return "Stopped" return None # unknown / not installed def _restart_service(service_name: str) -> bool: """Attempt to (re)start ``service_name``. Returns True if now running.""" if os.name == "nt": sc = shutil.which("sc") or r"C:\Windows\System32\sc.exe" # `sc start` returns 1056 if already running; ignore. try: subprocess.run( [sc, "start", service_name], capture_output=True, text=True, check=False, timeout=30.0, ) except (subprocess.TimeoutExpired, OSError): return False else: systemctl = shutil.which("systemctl") if systemctl is None: return False try: subprocess.run( [systemctl, "restart", service_name], capture_output=True, text=True, check=False, timeout=30.0, ) except (subprocess.TimeoutExpired, OSError): return False time.sleep(5.0) return _service_status(service_name) == "Running" def _load_restart_log(path: Path) -> list[str]: if not path.is_file(): return [] try: raw = path.read_text(encoding="utf-8") except OSError: return [] if not raw.strip(): return [] try: data = json.loads(raw) except json.JSONDecodeError: return [] if isinstance(data, list): return [str(x) for x in data] return [] def _save_restart_log(path: Path, entries: list[str]) -> None: try: path.parent.mkdir(parents=True, exist_ok=True) path.write_text(json.dumps(entries), encoding="utf-8") except OSError as exc: logger.warning("Could not persist restart log: %s", exc) def _gitea_runners_online(api_url: str, token: str, *, timeout: float = 10.0) -> int | None: """Return number of online runners reported by Gitea, or ``None`` on failure.""" api = api_url.rstrip("/") + "/api/v1/admin/runners" req = urllib.request.Request( api, headers={"Authorization": f"token {token}"} ) try: with urllib.request.urlopen(req, timeout=timeout) as resp: data = json.loads(resp.read().decode("utf-8")) except (urllib.error.URLError, TimeoutError, OSError, json.JSONDecodeError) as exc: logger.warning("Gitea API call failed: %s", exc) return None if isinstance(data, list): return sum(1 for r in data if isinstance(r, dict) and r.get("online") is True) return None @monitor.command("runner") @click.option( "--service-name", "service_name", default="act_runner", show_default=True, ) @click.option( "--max-restarts", type=click.IntRange(1, 10), default=3, show_default=True, ) @click.option( "--state-dir", "state_dir", default=r"F:\CI\State" if os.name == "nt" else "/var/lib/ci/state", show_default=True, ) @click.option("--webhook-url", "webhook_url", default="") @click.option("--gitea-url", "gitea_url", default="") @click.option( "--gitea-credential-target", "gitea_credential_target", default="", help="Credential target name for the Gitea admin PAT.", ) def monitor_runner( service_name: str, max_restarts: int, state_dir: str, webhook_url: str, gitea_url: str, gitea_credential_target: str, ) -> None: """Monitor act_runner; auto-restart up to ``--max-restarts`` per hour. Exit codes: 0 = healthy (or restart succeeded) 1 = service down + restart failed/limit reached 2 = service not installed """ state_dir_path = Path(state_dir) maintenance_flag = state_dir_path / "runner-maintenance.flag" if maintenance_flag.is_file(): click.echo( f"[runner] maintenance mode active ({maintenance_flag}) - skipping." ) return status = _service_status(service_name) if status is None: click.echo( f"[runner] service {service_name!r} not found - is act_runner installed?", err=True, ) sys.exit(2) if status == "Running": click.echo(f"[runner] {service_name} is Running - OK") if gitea_url and gitea_credential_target: try: from ci_orchestrator.credentials import KeyringCredentialStore cred = KeyringCredentialStore().get(gitea_credential_target) online = _gitea_runners_online(gitea_url, cred.password) except KeyError as exc: click.echo(f"[runner] could not load Gitea PAT: {exc}", err=True) online = None if online is None: click.echo("[runner] Gitea API check skipped/failed.") elif online == 0: msg = ( f"Gitea API ({gitea_url}) reports 0 online runners. " f"{service_name} is Running - possible registration issue." ) click.echo(f"[runner] WARNING: {msg}", err=True) _write_windows_event("CI-RunnerHealth", 1005, "Warning", msg) if webhook_url: _post_webhook(webhook_url, f"[WARNING] CI Runner Alert -- {msg}") else: click.echo(f"[runner] Gitea API: {online} runner(s) online - OK") return # Not running → check rate limit and try to restart. cooldown_file = state_dir_path / "runner-restart-log.json" log = _load_restart_log(cooldown_file) cutoff = datetime.now(tz=UTC) - timedelta(hours=1) pruned: list[str] = [] for entry in log: try: ts = datetime.fromisoformat(entry) except ValueError: continue if ts.tzinfo is None: ts = ts.replace(tzinfo=UTC) if ts > cutoff: pruned.append(entry) log = pruned state_msg = ( f"{service_name} service is '{status}'. " f"Auto-restarts in last hour: {len(log)} / {max_restarts}." ) click.echo(f"[runner] WARNING: {state_msg}", err=True) _write_windows_event("CI-RunnerHealth", 1002, "Warning", state_msg) if len(log) >= max_restarts: limit_msg = ( f"{service_name} auto-restart limit ({max_restarts}/h) reached - " "manual intervention required." ) click.echo(f"[runner] {limit_msg}", err=True) _write_windows_event("CI-RunnerHealth", 1004, "Error", limit_msg) if webhook_url: _post_webhook(webhook_url, f"[ALERT] CI Runner Alert -- {limit_msg}") sys.exit(1) click.echo( f"[runner] restarting {service_name} (attempt {len(log) + 1} of {max_restarts})..." ) ok = _restart_service(service_name) log.append(datetime.now(tz=UTC).isoformat()) _save_restart_log(cooldown_file, log) new_status = _service_status(service_name) or "Unknown" restart_msg = f"{service_name} restarted - new status: {new_status}." entry_type = "Information" if ok else "Warning" click.echo(f"[runner] {restart_msg}") _write_windows_event("CI-RunnerHealth", 1003, entry_type, restart_msg) prefix = "[WARNING]" if ok else "[ALERT]" if webhook_url: _post_webhook(webhook_url, f"{prefix} CI Runner Alert -- {restart_msg}") sys.exit(0 if ok else 1) def _ignored(_name: str, _value: Any) -> None: """Marker so type checkers see we deliberately drop a value.""" __all__ = ["monitor"]