Simone
80f6661ad5
Implements Phase A2 of plans/implementation-plan-A-B.md: - commands/wait.py -> wait-ready (replaces Wait-VMReady.ps1) - commands/vm.py -> vm remove + vm cleanup (replaces Remove-BuildVM.ps1, Cleanup-OrphanedBuildVMs.ps1); cleanup accepts injected VmBackend (Phase C ESXi hook preserved) - commands/monitor.py -> monitor disk + monitor runner (replaces Watch-DiskSpace.ps1, Watch-RunnerHealth.ps1) - commands/report.py -> report job (replaces Get-CIJobSummary.ps1) Each PS script reduced to a 3-line shim that delegates to the Python CLI and preserves \0. Tests: 69 pytest cases across new test_commands_*.py modules; ruff clean, mypy --strict clean, coverage 74.5% (>=70 gate).
153 lines
5.5 KiB
Python
153 lines
5.5 KiB
Python
"""Tests for ``ci_orchestrator monitor disk`` and ``monitor runner``."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import shutil
|
|
from datetime import UTC, datetime
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
import pytest
|
|
from click.testing import CliRunner
|
|
|
|
import ci_orchestrator.commands.monitor as mon
|
|
from ci_orchestrator.__main__ import cli
|
|
|
|
|
|
@pytest.fixture(autouse=True)
|
|
def _no_event_log_or_webhook(monkeypatch: pytest.MonkeyPatch) -> None:
|
|
monkeypatch.setattr(mon, "_write_windows_event", lambda *_a, **_kw: True)
|
|
monkeypatch.setattr(mon, "_post_webhook", lambda *_a, **_kw: True)
|
|
|
|
|
|
# ── monitor disk ───────────────────────────────────────────────────────────
|
|
|
|
|
|
def _fake_usage(free_gb: float, total_gb: float = 1000.0) -> shutil._ntuple_diskusage:
|
|
gb = 1024 ** 3
|
|
return shutil._ntuple_diskusage(int(total_gb * gb), int((total_gb - free_gb) * gb), int(free_gb * gb))
|
|
|
|
|
|
def test_disk_ok_exits_zero(monkeypatch: pytest.MonkeyPatch) -> None:
|
|
monkeypatch.setattr(mon.shutil, "disk_usage", lambda _p: _fake_usage(free_gb=200))
|
|
result = CliRunner().invoke(cli, ["monitor", "disk", "--min-free-gb", "50"])
|
|
assert result.exit_code == 0, result.output
|
|
assert "OK" in result.output
|
|
|
|
|
|
def test_disk_alert_exits_one(monkeypatch: pytest.MonkeyPatch) -> None:
|
|
monkeypatch.setattr(mon.shutil, "disk_usage", lambda _p: _fake_usage(free_gb=10))
|
|
result = CliRunner().invoke(cli, ["monitor", "disk", "--min-free-gb", "50"])
|
|
assert result.exit_code == 1
|
|
assert "below" in result.output or "WARNING" in result.output
|
|
|
|
|
|
def test_disk_drive_not_found(monkeypatch: pytest.MonkeyPatch) -> None:
|
|
def _raise(_p: Any) -> Any:
|
|
raise FileNotFoundError("nope")
|
|
|
|
monkeypatch.setattr(mon.shutil, "disk_usage", _raise)
|
|
result = CliRunner().invoke(cli, ["monitor", "disk", "--drive-letter", "Z"])
|
|
assert result.exit_code == 2
|
|
assert "not found" in result.output
|
|
|
|
|
|
def test_disk_json_output_ok(monkeypatch: pytest.MonkeyPatch) -> None:
|
|
monkeypatch.setattr(mon.shutil, "disk_usage", lambda _p: _fake_usage(free_gb=200))
|
|
result = CliRunner().invoke(
|
|
cli, ["monitor", "disk", "--min-free-gb", "50", "--json"]
|
|
)
|
|
assert result.exit_code == 0
|
|
payload = json.loads(result.output.strip())
|
|
assert payload["status"] == "ok"
|
|
assert payload["free_gb"] == 200.0
|
|
|
|
|
|
def test_disk_webhook_invoked_on_alert(monkeypatch: pytest.MonkeyPatch) -> None:
|
|
calls: list[tuple[str, str]] = []
|
|
monkeypatch.setattr(mon, "_post_webhook", lambda url, content, **_kw: calls.append((url, content)) or True)
|
|
monkeypatch.setattr(mon.shutil, "disk_usage", lambda _p: _fake_usage(free_gb=10))
|
|
result = CliRunner().invoke(
|
|
cli,
|
|
[
|
|
"monitor",
|
|
"disk",
|
|
"--min-free-gb",
|
|
"50",
|
|
"--webhook-url",
|
|
"https://example.invalid/hook",
|
|
],
|
|
)
|
|
assert result.exit_code == 1
|
|
assert calls and calls[0][0] == "https://example.invalid/hook"
|
|
|
|
|
|
# ── monitor runner ─────────────────────────────────────────────────────────
|
|
|
|
|
|
def test_runner_running_exits_zero(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
|
|
monkeypatch.setattr(mon, "_service_status", lambda _s: "Running")
|
|
result = CliRunner().invoke(
|
|
cli,
|
|
["monitor", "runner", "--state-dir", str(tmp_path)],
|
|
)
|
|
assert result.exit_code == 0
|
|
assert "Running" in result.output
|
|
|
|
|
|
def test_runner_not_installed_exits_two(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
|
|
monkeypatch.setattr(mon, "_service_status", lambda _s: None)
|
|
result = CliRunner().invoke(
|
|
cli, ["monitor", "runner", "--state-dir", str(tmp_path)]
|
|
)
|
|
assert result.exit_code == 2
|
|
assert "not found" in result.output
|
|
|
|
|
|
def test_runner_maintenance_flag_skips(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
|
|
flag = tmp_path / "runner-maintenance.flag"
|
|
flag.write_text("")
|
|
result = CliRunner().invoke(
|
|
cli, ["monitor", "runner", "--state-dir", str(tmp_path)]
|
|
)
|
|
assert result.exit_code == 0
|
|
assert "maintenance mode" in result.output
|
|
|
|
|
|
def test_runner_restart_attempted_when_stopped(
|
|
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
|
) -> None:
|
|
statuses = iter(["Stopped", "Running"])
|
|
monkeypatch.setattr(mon, "_service_status", lambda _s: next(statuses))
|
|
monkeypatch.setattr(mon, "_restart_service", lambda _s: True)
|
|
result = CliRunner().invoke(
|
|
cli, ["monitor", "runner", "--state-dir", str(tmp_path)]
|
|
)
|
|
assert result.exit_code == 0
|
|
assert "restarted" in result.output
|
|
assert (tmp_path / "runner-restart-log.json").is_file()
|
|
|
|
|
|
def test_runner_rate_limit_exits_one(
|
|
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
|
) -> None:
|
|
# Pre-fill the cooldown log with recent restarts up to the limit.
|
|
recent = [datetime.now(tz=UTC).isoformat()] * 3
|
|
(tmp_path / "runner-restart-log.json").write_text(json.dumps(recent))
|
|
monkeypatch.setattr(mon, "_service_status", lambda _s: "Stopped")
|
|
monkeypatch.setattr(mon, "_restart_service", lambda _s: True)
|
|
result = CliRunner().invoke(
|
|
cli,
|
|
[
|
|
"monitor",
|
|
"runner",
|
|
"--state-dir",
|
|
str(tmp_path),
|
|
"--max-restarts",
|
|
"3",
|
|
],
|
|
)
|
|
assert result.exit_code == 1
|
|
assert "limit" in result.output
|