Simone
8dc7a4fb99
- Implement tests for disk alert JSON output to include a message field when below threshold. - Add tests for runner status when Gitea is online but no webhook URL is provided. - Introduce tests for job report details to ensure no error line appears when there are no failure events. - Add tests to skip empty JSONL files in job report listing.
708 lines
26 KiB
Python
708 lines
26 KiB
Python
"""Tests for ``ci_orchestrator monitor disk`` and ``monitor runner``."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import shutil
|
|
from datetime import UTC, datetime
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
import pytest
|
|
from click.testing import CliRunner
|
|
|
|
import ci_orchestrator.commands.monitor as mon
|
|
from ci_orchestrator.__main__ import cli
|
|
|
|
|
|
@pytest.fixture(autouse=True)
|
|
def _no_event_log_or_webhook(monkeypatch: pytest.MonkeyPatch) -> None:
|
|
monkeypatch.setattr(mon, "_write_windows_event", lambda *_a, **_kw: True)
|
|
monkeypatch.setattr(mon, "_post_webhook", lambda *_a, **_kw: True)
|
|
|
|
|
|
# ── monitor disk ───────────────────────────────────────────────────────────
|
|
|
|
|
|
def _fake_usage(free_gb: float, total_gb: float = 1000.0) -> shutil._ntuple_diskusage:
|
|
gb = 1024 ** 3
|
|
return shutil._ntuple_diskusage(int(total_gb * gb), int((total_gb - free_gb) * gb), int(free_gb * gb))
|
|
|
|
|
|
def test_disk_ok_exits_zero(monkeypatch: pytest.MonkeyPatch) -> None:
|
|
monkeypatch.setattr(mon.shutil, "disk_usage", lambda _p: _fake_usage(free_gb=200))
|
|
result = CliRunner().invoke(cli, ["monitor", "disk", "--min-free-gb", "50"])
|
|
assert result.exit_code == 0, result.output
|
|
assert "OK" in result.output
|
|
|
|
|
|
def test_disk_alert_exits_one(monkeypatch: pytest.MonkeyPatch) -> None:
|
|
monkeypatch.setattr(mon.shutil, "disk_usage", lambda _p: _fake_usage(free_gb=10))
|
|
result = CliRunner().invoke(cli, ["monitor", "disk", "--min-free-gb", "50"])
|
|
assert result.exit_code == 1
|
|
assert "below" in result.output or "WARNING" in result.output
|
|
|
|
|
|
def test_disk_drive_not_found(monkeypatch: pytest.MonkeyPatch) -> None:
|
|
def _raise(_p: Any) -> Any:
|
|
raise FileNotFoundError("nope")
|
|
|
|
monkeypatch.setattr(mon.shutil, "disk_usage", _raise)
|
|
result = CliRunner().invoke(cli, ["monitor", "disk", "--drive-letter", "Z"])
|
|
assert result.exit_code == 2
|
|
assert "not found" in result.output
|
|
|
|
|
|
def test_disk_json_output_ok(monkeypatch: pytest.MonkeyPatch) -> None:
|
|
monkeypatch.setattr(mon.shutil, "disk_usage", lambda _p: _fake_usage(free_gb=200))
|
|
result = CliRunner().invoke(
|
|
cli, ["monitor", "disk", "--min-free-gb", "50", "--json"]
|
|
)
|
|
assert result.exit_code == 0
|
|
payload = json.loads(result.output.strip())
|
|
assert payload["status"] == "ok"
|
|
assert payload["free_gb"] == 200.0
|
|
|
|
|
|
def test_disk_webhook_invoked_on_alert(monkeypatch: pytest.MonkeyPatch) -> None:
|
|
calls: list[tuple[str, str]] = []
|
|
monkeypatch.setattr(mon, "_post_webhook", lambda url, content, **_kw: calls.append((url, content)) or True)
|
|
monkeypatch.setattr(mon.shutil, "disk_usage", lambda _p: _fake_usage(free_gb=10))
|
|
result = CliRunner().invoke(
|
|
cli,
|
|
[
|
|
"monitor",
|
|
"disk",
|
|
"--min-free-gb",
|
|
"50",
|
|
"--webhook-url",
|
|
"https://example.invalid/hook",
|
|
],
|
|
)
|
|
assert result.exit_code == 1
|
|
assert calls and calls[0][0] == "https://example.invalid/hook"
|
|
|
|
|
|
# ── monitor runner ─────────────────────────────────────────────────────────
|
|
|
|
|
|
def test_runner_running_exits_zero(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
|
|
monkeypatch.setattr(mon, "_service_status", lambda _s: "Running")
|
|
result = CliRunner().invoke(
|
|
cli,
|
|
["monitor", "runner", "--state-dir", str(tmp_path)],
|
|
)
|
|
assert result.exit_code == 0
|
|
assert "Running" in result.output
|
|
|
|
|
|
def test_runner_not_installed_exits_two(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
|
|
monkeypatch.setattr(mon, "_service_status", lambda _s: None)
|
|
result = CliRunner().invoke(
|
|
cli, ["monitor", "runner", "--state-dir", str(tmp_path)]
|
|
)
|
|
assert result.exit_code == 2
|
|
assert "not found" in result.output
|
|
|
|
|
|
def test_runner_maintenance_flag_skips(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
|
|
flag = tmp_path / "runner-maintenance.flag"
|
|
flag.write_text("")
|
|
result = CliRunner().invoke(
|
|
cli, ["monitor", "runner", "--state-dir", str(tmp_path)]
|
|
)
|
|
assert result.exit_code == 0
|
|
assert "maintenance mode" in result.output
|
|
|
|
|
|
def test_runner_restart_attempted_when_stopped(
|
|
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
|
) -> None:
|
|
statuses = iter(["Stopped", "Running"])
|
|
monkeypatch.setattr(mon, "_service_status", lambda _s: next(statuses))
|
|
monkeypatch.setattr(mon, "_restart_service", lambda _s: True)
|
|
result = CliRunner().invoke(
|
|
cli, ["monitor", "runner", "--state-dir", str(tmp_path)]
|
|
)
|
|
assert result.exit_code == 0
|
|
assert "restarted" in result.output
|
|
assert (tmp_path / "runner-restart-log.json").is_file()
|
|
|
|
|
|
def test_runner_rate_limit_exits_one(
|
|
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
|
) -> None:
|
|
# Pre-fill the cooldown log with recent restarts up to the limit.
|
|
recent = [datetime.now(tz=UTC).isoformat()] * 3
|
|
(tmp_path / "runner-restart-log.json").write_text(json.dumps(recent))
|
|
monkeypatch.setattr(mon, "_service_status", lambda _s: "Stopped")
|
|
monkeypatch.setattr(mon, "_restart_service", lambda _s: True)
|
|
result = CliRunner().invoke(
|
|
cli,
|
|
[
|
|
"monitor",
|
|
"runner",
|
|
"--state-dir",
|
|
str(tmp_path),
|
|
"--max-restarts",
|
|
"3",
|
|
],
|
|
)
|
|
assert result.exit_code == 1
|
|
assert "limit" in result.output
|
|
|
|
|
|
# ── helpers: _post_webhook ─────────────────────────────────────────────────
|
|
|
|
|
|
def test_post_webhook_empty_url_returns_false(monkeypatch: pytest.MonkeyPatch) -> None:
|
|
monkeypatch.undo()
|
|
assert mon._post_webhook("", "msg") is False
|
|
|
|
|
|
def test_post_webhook_success(monkeypatch: pytest.MonkeyPatch) -> None:
|
|
monkeypatch.undo()
|
|
class _Resp:
|
|
status = 204
|
|
|
|
def __enter__(self) -> _Resp:
|
|
return self
|
|
|
|
def __exit__(self, *_a: Any) -> None:
|
|
return None
|
|
|
|
monkeypatch.setattr(mon.urllib.request, "urlopen", lambda *_a, **_kw: _Resp())
|
|
assert mon._post_webhook("https://example.invalid/h", "hi") is True
|
|
|
|
|
|
def test_post_webhook_http_error_returns_false(monkeypatch: pytest.MonkeyPatch) -> None:
|
|
monkeypatch.undo()
|
|
def _raise(*_a: Any, **_kw: Any) -> Any:
|
|
raise mon.urllib.error.URLError("boom")
|
|
|
|
monkeypatch.setattr(mon.urllib.request, "urlopen", _raise)
|
|
assert mon._post_webhook("https://example.invalid/h", "hi") is False
|
|
|
|
|
|
def test_post_webhook_non_2xx_returns_false(monkeypatch: pytest.MonkeyPatch) -> None:
|
|
monkeypatch.undo()
|
|
class _Resp:
|
|
status = 500
|
|
|
|
def __enter__(self) -> _Resp:
|
|
return self
|
|
|
|
def __exit__(self, *_a: Any) -> None:
|
|
return None
|
|
|
|
monkeypatch.setattr(mon.urllib.request, "urlopen", lambda *_a, **_kw: _Resp())
|
|
assert mon._post_webhook("https://example.invalid/h", "hi") is False
|
|
|
|
|
|
# ── helpers: _write_windows_event ──────────────────────────────────────────
|
|
|
|
|
|
def test_write_windows_event_non_windows(monkeypatch: pytest.MonkeyPatch) -> None:
|
|
monkeypatch.setattr(mon.os, "name", "posix")
|
|
# Note: autouse fixture monkeypatches the symbol; call the real impl.
|
|
monkeypatch.undo()
|
|
monkeypatch.setattr(mon.os, "name", "posix")
|
|
assert mon._write_windows_event("src", 1, "Warning", "msg") is False
|
|
|
|
|
|
def test_write_windows_event_eventcreate_missing(monkeypatch: pytest.MonkeyPatch) -> None:
|
|
monkeypatch.undo()
|
|
monkeypatch.setattr(mon.os, "name", "nt")
|
|
monkeypatch.setattr(mon.shutil, "which", lambda _n: None)
|
|
assert mon._write_windows_event("src", 1, "Warning", "msg") is False
|
|
|
|
|
|
def test_write_windows_event_subprocess_ok(monkeypatch: pytest.MonkeyPatch) -> None:
|
|
monkeypatch.undo()
|
|
monkeypatch.setattr(mon.os, "name", "nt")
|
|
monkeypatch.setattr(mon.shutil, "which", lambda _n: r"C:\Windows\System32\eventcreate.exe")
|
|
|
|
class _R:
|
|
returncode = 0
|
|
stderr = ""
|
|
|
|
monkeypatch.setattr(mon.subprocess, "run", lambda *_a, **_kw: _R())
|
|
assert mon._write_windows_event("src", 1, "Information", "msg") is True
|
|
|
|
|
|
def test_write_windows_event_subprocess_nonzero(monkeypatch: pytest.MonkeyPatch) -> None:
|
|
monkeypatch.undo()
|
|
monkeypatch.setattr(mon.os, "name", "nt")
|
|
monkeypatch.setattr(mon.shutil, "which", lambda _n: r"C:\Windows\System32\eventcreate.exe")
|
|
|
|
class _R:
|
|
returncode = 5
|
|
stderr = "nope"
|
|
|
|
monkeypatch.setattr(mon.subprocess, "run", lambda *_a, **_kw: _R())
|
|
assert mon._write_windows_event("src", 1, "Error", "msg") is False
|
|
|
|
|
|
def test_write_windows_event_subprocess_oserror(monkeypatch: pytest.MonkeyPatch) -> None:
|
|
monkeypatch.undo()
|
|
monkeypatch.setattr(mon.os, "name", "nt")
|
|
monkeypatch.setattr(mon.shutil, "which", lambda _n: r"C:\Windows\System32\eventcreate.exe")
|
|
|
|
def _boom(*_a: Any, **_kw: Any) -> Any:
|
|
raise OSError("boom")
|
|
|
|
monkeypatch.setattr(mon.subprocess, "run", _boom)
|
|
assert mon._write_windows_event("src", 1, "Warning", "msg") is False
|
|
|
|
|
|
# ── helpers: _resolve_drive_path ───────────────────────────────────────────
|
|
|
|
|
|
def test_resolve_drive_path_windows(monkeypatch: pytest.MonkeyPatch) -> None:
|
|
monkeypatch.setattr(mon.os, "name", "nt")
|
|
assert mon._resolve_drive_path("F") == "F:\\"
|
|
|
|
|
|
def test_resolve_drive_path_non_windows(monkeypatch: pytest.MonkeyPatch) -> None:
|
|
monkeypatch.setattr(mon.os, "name", "posix")
|
|
assert mon._resolve_drive_path("/mnt/data") == "/mnt/data"
|
|
|
|
|
|
# ── helpers: _service_status ───────────────────────────────────────────────
|
|
|
|
|
|
class _RunResult:
|
|
def __init__(self, returncode: int = 0, stdout: str = "", stderr: str = "") -> None:
|
|
self.returncode = returncode
|
|
self.stdout = stdout
|
|
self.stderr = stderr
|
|
|
|
|
|
def test_service_status_windows_running(monkeypatch: pytest.MonkeyPatch) -> None:
|
|
monkeypatch.setattr(mon.os, "name", "nt")
|
|
monkeypatch.setattr(mon.shutil, "which", lambda _n: r"C:\Windows\System32\sc.exe")
|
|
monkeypatch.setattr(mon.subprocess, "run", lambda *_a, **_kw: _RunResult(0, "STATE : 4 RUNNING"))
|
|
assert mon._service_status("svc") == "Running"
|
|
|
|
|
|
def test_service_status_windows_stopped(monkeypatch: pytest.MonkeyPatch) -> None:
|
|
monkeypatch.setattr(mon.os, "name", "nt")
|
|
monkeypatch.setattr(mon.shutil, "which", lambda _n: r"C:\Windows\System32\sc.exe")
|
|
monkeypatch.setattr(mon.subprocess, "run", lambda *_a, **_kw: _RunResult(0, "STATE : 1 STOPPED"))
|
|
assert mon._service_status("svc") == "Stopped"
|
|
|
|
|
|
def test_service_status_windows_unknown_state(monkeypatch: pytest.MonkeyPatch) -> None:
|
|
monkeypatch.setattr(mon.os, "name", "nt")
|
|
monkeypatch.setattr(mon.shutil, "which", lambda _n: r"C:\Windows\System32\sc.exe")
|
|
monkeypatch.setattr(mon.subprocess, "run", lambda *_a, **_kw: _RunResult(0, "STATE : 7 PAUSED"))
|
|
assert mon._service_status("svc") == "Unknown"
|
|
|
|
|
|
def test_service_status_windows_sc_failed(monkeypatch: pytest.MonkeyPatch) -> None:
|
|
monkeypatch.setattr(mon.os, "name", "nt")
|
|
monkeypatch.setattr(mon.shutil, "which", lambda _n: r"C:\Windows\System32\sc.exe")
|
|
monkeypatch.setattr(mon.subprocess, "run", lambda *_a, **_kw: _RunResult(1, ""))
|
|
assert mon._service_status("svc") is None
|
|
|
|
|
|
def test_service_status_windows_subprocess_oserror(monkeypatch: pytest.MonkeyPatch) -> None:
|
|
monkeypatch.setattr(mon.os, "name", "nt")
|
|
monkeypatch.setattr(mon.shutil, "which", lambda _n: r"C:\Windows\System32\sc.exe")
|
|
|
|
def _boom(*_a: Any, **_kw: Any) -> Any:
|
|
raise OSError("boom")
|
|
|
|
monkeypatch.setattr(mon.subprocess, "run", _boom)
|
|
assert mon._service_status("svc") is None
|
|
|
|
|
|
def test_service_status_linux_active(monkeypatch: pytest.MonkeyPatch) -> None:
|
|
monkeypatch.setattr(mon.os, "name", "posix")
|
|
monkeypatch.setattr(mon.shutil, "which", lambda _n: "/usr/bin/systemctl")
|
|
monkeypatch.setattr(mon.subprocess, "run", lambda *_a, **_kw: _RunResult(0, "active\n"))
|
|
assert mon._service_status("svc") == "Running"
|
|
|
|
|
|
def test_service_status_linux_inactive(monkeypatch: pytest.MonkeyPatch) -> None:
|
|
monkeypatch.setattr(mon.os, "name", "posix")
|
|
monkeypatch.setattr(mon.shutil, "which", lambda _n: "/usr/bin/systemctl")
|
|
monkeypatch.setattr(mon.subprocess, "run", lambda *_a, **_kw: _RunResult(3, "inactive\n"))
|
|
assert mon._service_status("svc") == "Stopped"
|
|
|
|
|
|
def test_service_status_linux_unknown(monkeypatch: pytest.MonkeyPatch) -> None:
|
|
monkeypatch.setattr(mon.os, "name", "posix")
|
|
monkeypatch.setattr(mon.shutil, "which", lambda _n: "/usr/bin/systemctl")
|
|
monkeypatch.setattr(mon.subprocess, "run", lambda *_a, **_kw: _RunResult(4, "activating\n"))
|
|
assert mon._service_status("svc") is None
|
|
|
|
|
|
def test_service_status_linux_no_systemctl(monkeypatch: pytest.MonkeyPatch) -> None:
|
|
monkeypatch.setattr(mon.os, "name", "posix")
|
|
monkeypatch.setattr(mon.shutil, "which", lambda _n: None)
|
|
assert mon._service_status("svc") is None
|
|
|
|
|
|
def test_service_status_linux_subprocess_oserror(monkeypatch: pytest.MonkeyPatch) -> None:
|
|
monkeypatch.setattr(mon.os, "name", "posix")
|
|
monkeypatch.setattr(mon.shutil, "which", lambda _n: "/usr/bin/systemctl")
|
|
|
|
def _boom(*_a: Any, **_kw: Any) -> Any:
|
|
raise OSError("boom")
|
|
|
|
monkeypatch.setattr(mon.subprocess, "run", _boom)
|
|
assert mon._service_status("svc") is None
|
|
|
|
|
|
# ── helpers: _restart_service ──────────────────────────────────────────────
|
|
|
|
|
|
def test_restart_service_windows_success(monkeypatch: pytest.MonkeyPatch) -> None:
|
|
monkeypatch.setattr(mon.os, "name", "nt")
|
|
monkeypatch.setattr(mon.shutil, "which", lambda _n: r"C:\Windows\System32\sc.exe")
|
|
monkeypatch.setattr(mon.subprocess, "run", lambda *_a, **_kw: _RunResult(0, ""))
|
|
monkeypatch.setattr(mon.time, "sleep", lambda _s: None)
|
|
monkeypatch.setattr(mon, "_service_status", lambda _s: "Running")
|
|
assert mon._restart_service("svc") is True
|
|
|
|
|
|
def test_restart_service_windows_oserror(monkeypatch: pytest.MonkeyPatch) -> None:
|
|
monkeypatch.setattr(mon.os, "name", "nt")
|
|
monkeypatch.setattr(mon.shutil, "which", lambda _n: r"C:\Windows\System32\sc.exe")
|
|
|
|
def _boom(*_a: Any, **_kw: Any) -> Any:
|
|
raise OSError("boom")
|
|
|
|
monkeypatch.setattr(mon.subprocess, "run", _boom)
|
|
assert mon._restart_service("svc") is False
|
|
|
|
|
|
def test_restart_service_linux_success(monkeypatch: pytest.MonkeyPatch) -> None:
|
|
monkeypatch.setattr(mon.os, "name", "posix")
|
|
monkeypatch.setattr(mon.shutil, "which", lambda _n: "/usr/bin/systemctl")
|
|
monkeypatch.setattr(mon.subprocess, "run", lambda *_a, **_kw: _RunResult(0, ""))
|
|
monkeypatch.setattr(mon.time, "sleep", lambda _s: None)
|
|
monkeypatch.setattr(mon, "_service_status", lambda _s: "Running")
|
|
assert mon._restart_service("svc") is True
|
|
|
|
|
|
def test_restart_service_linux_no_systemctl(monkeypatch: pytest.MonkeyPatch) -> None:
|
|
monkeypatch.setattr(mon.os, "name", "posix")
|
|
monkeypatch.setattr(mon.shutil, "which", lambda _n: None)
|
|
assert mon._restart_service("svc") is False
|
|
|
|
|
|
def test_restart_service_linux_oserror(monkeypatch: pytest.MonkeyPatch) -> None:
|
|
monkeypatch.setattr(mon.os, "name", "posix")
|
|
monkeypatch.setattr(mon.shutil, "which", lambda _n: "/usr/bin/systemctl")
|
|
|
|
def _boom(*_a: Any, **_kw: Any) -> Any:
|
|
raise subprocess.TimeoutExpired(cmd="systemctl", timeout=30.0)
|
|
|
|
import subprocess
|
|
monkeypatch.setattr(mon.subprocess, "run", _boom)
|
|
assert mon._restart_service("svc") is False
|
|
|
|
|
|
# ── helpers: _load_restart_log / _save_restart_log ─────────────────────────
|
|
|
|
|
|
def test_load_restart_log_missing(tmp_path: Path) -> None:
|
|
assert mon._load_restart_log(tmp_path / "no.json") == []
|
|
|
|
|
|
def test_load_restart_log_empty(tmp_path: Path) -> None:
|
|
p = tmp_path / "empty.json"
|
|
p.write_text(" ")
|
|
assert mon._load_restart_log(p) == []
|
|
|
|
|
|
def test_load_restart_log_invalid_json(tmp_path: Path) -> None:
|
|
p = tmp_path / "bad.json"
|
|
p.write_text("{not json")
|
|
assert mon._load_restart_log(p) == []
|
|
|
|
|
|
def test_load_restart_log_non_list(tmp_path: Path) -> None:
|
|
p = tmp_path / "obj.json"
|
|
p.write_text(json.dumps({"a": 1}))
|
|
assert mon._load_restart_log(p) == []
|
|
|
|
|
|
def test_load_restart_log_valid(tmp_path: Path) -> None:
|
|
p = tmp_path / "ok.json"
|
|
p.write_text(json.dumps(["2026-05-14T10:00:00+00:00"]))
|
|
assert mon._load_restart_log(p) == ["2026-05-14T10:00:00+00:00"]
|
|
|
|
|
|
def test_load_restart_log_oserror(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
|
|
p = tmp_path / "bad.json"
|
|
p.write_text("[]")
|
|
|
|
def _boom(*_a: Any, **_kw: Any) -> Any:
|
|
raise OSError("boom")
|
|
|
|
monkeypatch.setattr(Path, "read_text", _boom)
|
|
assert mon._load_restart_log(p) == []
|
|
|
|
|
|
def test_save_restart_log_writes_file(tmp_path: Path) -> None:
|
|
p = tmp_path / "sub" / "log.json"
|
|
mon._save_restart_log(p, ["a", "b"])
|
|
assert json.loads(p.read_text()) == ["a", "b"]
|
|
|
|
|
|
def test_save_restart_log_oserror_swallowed(
|
|
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
|
) -> None:
|
|
def _boom(*_a: Any, **_kw: Any) -> Any:
|
|
raise OSError("readonly")
|
|
|
|
monkeypatch.setattr(Path, "mkdir", _boom)
|
|
# Should not raise.
|
|
mon._save_restart_log(tmp_path / "x" / "log.json", ["a"])
|
|
|
|
|
|
# ── helpers: _gitea_runners_online ─────────────────────────────────────────
|
|
|
|
|
|
def _make_urlopen(payload: Any) -> Any:
|
|
class _Resp:
|
|
def __enter__(self) -> _Resp:
|
|
return self
|
|
|
|
def __exit__(self, *_a: Any) -> None:
|
|
return None
|
|
|
|
def read(self) -> bytes:
|
|
return json.dumps(payload).encode("utf-8")
|
|
|
|
return lambda *_a, **_kw: _Resp()
|
|
|
|
|
|
def test_gitea_runners_online_counts(monkeypatch: pytest.MonkeyPatch) -> None:
|
|
monkeypatch.setattr(
|
|
mon.urllib.request,
|
|
"urlopen",
|
|
_make_urlopen([{"online": True}, {"online": False}, {"online": True}]),
|
|
)
|
|
assert mon._gitea_runners_online("https://g/", "tok") == 2
|
|
|
|
|
|
def test_gitea_runners_online_non_list(monkeypatch: pytest.MonkeyPatch) -> None:
|
|
monkeypatch.setattr(mon.urllib.request, "urlopen", _make_urlopen({"x": 1}))
|
|
assert mon._gitea_runners_online("https://g/", "tok") is None
|
|
|
|
|
|
def test_gitea_runners_online_url_error(monkeypatch: pytest.MonkeyPatch) -> None:
|
|
def _raise(*_a: Any, **_kw: Any) -> Any:
|
|
raise mon.urllib.error.URLError("nope")
|
|
|
|
monkeypatch.setattr(mon.urllib.request, "urlopen", _raise)
|
|
assert mon._gitea_runners_online("https://g/", "tok") is None
|
|
|
|
|
|
# ── monitor_runner: gitea-online branches & restart-fail path ──────────────
|
|
|
|
|
|
def _stub_keyring(monkeypatch: pytest.MonkeyPatch, password: str = "tok") -> None:
|
|
class _Cred:
|
|
username = "u"
|
|
|
|
def __init__(self, pw: str) -> None:
|
|
self.password = pw
|
|
|
|
class _Store:
|
|
def get(self, _target: str) -> Any:
|
|
return _Cred(password)
|
|
|
|
import ci_orchestrator.credentials as creds
|
|
|
|
monkeypatch.setattr(creds, "KeyringCredentialStore", _Store)
|
|
|
|
|
|
def test_runner_running_with_gitea_online(
|
|
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
|
) -> None:
|
|
monkeypatch.setattr(mon, "_service_status", lambda _s: "Running")
|
|
monkeypatch.setattr(mon, "_gitea_runners_online", lambda *_a, **_kw: 2)
|
|
_stub_keyring(monkeypatch)
|
|
result = CliRunner().invoke(
|
|
cli,
|
|
[
|
|
"monitor", "runner",
|
|
"--state-dir", str(tmp_path),
|
|
"--gitea-url", "https://g/",
|
|
"--gitea-credential-target", "tgt",
|
|
],
|
|
)
|
|
assert result.exit_code == 0
|
|
assert "2 runner(s) online" in result.output
|
|
|
|
|
|
def test_runner_running_with_gitea_zero_online(
|
|
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
|
) -> None:
|
|
monkeypatch.setattr(mon, "_service_status", lambda _s: "Running")
|
|
monkeypatch.setattr(mon, "_gitea_runners_online", lambda *_a, **_kw: 0)
|
|
_stub_keyring(monkeypatch)
|
|
result = CliRunner().invoke(
|
|
cli,
|
|
[
|
|
"monitor", "runner",
|
|
"--state-dir", str(tmp_path),
|
|
"--gitea-url", "https://g/",
|
|
"--gitea-credential-target", "tgt",
|
|
"--webhook-url", "https://hook/",
|
|
],
|
|
)
|
|
assert result.exit_code == 0
|
|
assert "0 online runners" in result.output
|
|
|
|
|
|
def test_runner_running_with_gitea_api_failure(
|
|
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
|
) -> None:
|
|
monkeypatch.setattr(mon, "_service_status", lambda _s: "Running")
|
|
monkeypatch.setattr(mon, "_gitea_runners_online", lambda *_a, **_kw: None)
|
|
_stub_keyring(monkeypatch)
|
|
result = CliRunner().invoke(
|
|
cli,
|
|
[
|
|
"monitor", "runner",
|
|
"--state-dir", str(tmp_path),
|
|
"--gitea-url", "https://g/",
|
|
"--gitea-credential-target", "tgt",
|
|
],
|
|
)
|
|
assert result.exit_code == 0
|
|
assert "skipped/failed" in result.output
|
|
|
|
|
|
def test_runner_running_with_gitea_keyring_keyerror(
|
|
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
|
) -> None:
|
|
monkeypatch.setattr(mon, "_service_status", lambda _s: "Running")
|
|
|
|
class _Store:
|
|
def get(self, _target: str) -> Any:
|
|
raise KeyError("missing")
|
|
|
|
import ci_orchestrator.credentials as creds
|
|
|
|
monkeypatch.setattr(creds, "KeyringCredentialStore", _Store)
|
|
result = CliRunner().invoke(
|
|
cli,
|
|
[
|
|
"monitor", "runner",
|
|
"--state-dir", str(tmp_path),
|
|
"--gitea-url", "https://g/",
|
|
"--gitea-credential-target", "tgt",
|
|
],
|
|
)
|
|
assert result.exit_code == 0
|
|
assert "could not load Gitea PAT" in result.output
|
|
|
|
|
|
def test_runner_restart_fails_exits_one(
|
|
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
|
) -> None:
|
|
statuses = iter(["Stopped", "Stopped"])
|
|
monkeypatch.setattr(mon, "_service_status", lambda _s: next(statuses))
|
|
monkeypatch.setattr(mon, "_restart_service", lambda _s: False)
|
|
result = CliRunner().invoke(
|
|
cli,
|
|
[
|
|
"monitor", "runner",
|
|
"--state-dir", str(tmp_path),
|
|
"--webhook-url", "https://hook/",
|
|
],
|
|
)
|
|
assert result.exit_code == 1
|
|
assert "restarted" in result.output
|
|
|
|
|
|
def test_runner_rate_limit_with_webhook(
|
|
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
|
) -> None:
|
|
recent = [datetime.now(tz=UTC).isoformat()] * 3
|
|
(tmp_path / "runner-restart-log.json").write_text(json.dumps(recent))
|
|
monkeypatch.setattr(mon, "_service_status", lambda _s: "Stopped")
|
|
result = CliRunner().invoke(
|
|
cli,
|
|
[
|
|
"monitor", "runner",
|
|
"--state-dir", str(tmp_path),
|
|
"--max-restarts", "3",
|
|
"--webhook-url", "https://hook/",
|
|
],
|
|
)
|
|
assert result.exit_code == 1
|
|
assert "limit" in result.output
|
|
|
|
|
|
def test_runner_pruned_old_entries(
|
|
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
|
) -> None:
|
|
"""Old timestamps and invalid entries must be pruned."""
|
|
from datetime import timedelta
|
|
|
|
old = (datetime.now(tz=UTC) - timedelta(hours=2)).isoformat()
|
|
naive = datetime.now(tz=UTC).replace(tzinfo=None).isoformat()
|
|
bad = "not-a-timestamp"
|
|
(tmp_path / "runner-restart-log.json").write_text(json.dumps([old, naive, bad]))
|
|
statuses = iter(["Stopped", "Running"])
|
|
monkeypatch.setattr(mon, "_service_status", lambda _s: next(statuses))
|
|
monkeypatch.setattr(mon, "_restart_service", lambda _s: True)
|
|
result = CliRunner().invoke(
|
|
cli, ["monitor", "runner", "--state-dir", str(tmp_path)]
|
|
)
|
|
assert result.exit_code == 0
|
|
|
|
|
|
# ── NEW: coverage gap tests ──────────────────────────────────────────────────
|
|
|
|
|
|
def test_disk_alert_json_includes_message_field(monkeypatch: pytest.MonkeyPatch) -> None:
|
|
"""Lines 186-187: disk below threshold + --json → JSON has 'message' key, exit 1."""
|
|
monkeypatch.setattr(mon.shutil, "disk_usage", lambda _p: _fake_usage(free_gb=5))
|
|
result = CliRunner().invoke(
|
|
cli, ["monitor", "disk", "--min-free-gb", "50", "--json"]
|
|
)
|
|
assert result.exit_code == 1
|
|
payload = json.loads(result.output.strip())
|
|
assert payload["status"] == "alert"
|
|
assert "message" in payload
|
|
assert "below" in payload["message"]
|
|
|
|
|
|
def test_runner_running_gitea_online_no_webhook(
|
|
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
|
) -> None:
|
|
"""Lines 398->402: online==0 but no webhook_url → branch skips _post_webhook,
|
|
continues to return (line 402). Covers the 398->402 branch (webhook_url falsy)."""
|
|
monkeypatch.setattr(mon, "_service_status", lambda _s: "Running")
|
|
monkeypatch.setattr(mon, "_gitea_runners_online", lambda *_a, **_kw: 0)
|
|
|
|
class _Store:
|
|
def get(self, _target: str) -> Any:
|
|
from ci_orchestrator.credentials import Credential
|
|
return Credential("u", "tok")
|
|
|
|
import ci_orchestrator.credentials as creds
|
|
monkeypatch.setattr(creds, "KeyringCredentialStore", _Store)
|
|
|
|
result = CliRunner().invoke(
|
|
cli,
|
|
[
|
|
"monitor", "runner",
|
|
"--state-dir", str(tmp_path),
|
|
"--gitea-url", "https://g/",
|
|
"--gitea-credential-target", "tgt",
|
|
# no --webhook-url → webhook_url is ""
|
|
],
|
|
)
|
|
assert result.exit_code == 0
|
|
assert "0 online runners" in result.output
|