Simone
d18321d4a0
A large, cohesive redesign of the web dashboard (all pieces depend on each other, so this lands as one commit): Layout / display: - Host/Runners/VMware/Cache split into a Host-VMware-Runners-Cache info row; Host card keeps uptime/load/cores/mem-total now that CPU%/RAM are promoted to the status-header tiles (new gauge tiles, red/amber/ green thresholds shared with the existing disk tile). - VMware modules rendered as a table; Live VMs/Templates/Artifacts/ Services/Maintenance span the full grid width (card--wide). - Backups moved from a standalone Files-panel table into an expandable per-template row on the Templates panel (backups_by_template), with a per-backup Delete action and a per-row Backup button replacing the old host-card template dropdown. - New Maintenance card: cleanup-orphans and cache-purge forms consolidated into one danger-zone card (red rule + row separators), each op's max-age control on its own row so it reads as the op's parameter, not decoration; cache purge gets an explicit "Empty all" button alongside the age-based "Purge". - Artifacts gain a per-row Delete action. Action/feedback layer: - Every mutating route now sets HX-Trigger: ci:refresh on its result, so polled panels reload immediately instead of waiting for their next poll interval; _result_partial takes an explicit level (ok/info/error) independent of the ok bool, so e.g. an age-based purge that finds nothing to remove reads as neutral "info" instead of a misleading green "ok" or a red "error". - Results render as toasts appended to a fixed #toast-stack (base.html) instead of swapping a single #action-slot -- concurrent actions (two backups running at once) now each keep their own toast instead of the later one clobbering the earlier. Per-state colour is a left stripe + tint (result-ok/info/error), fixing a cascade bug where the plain .toast background silently beat the state class. - The confirm dialog is a real modal <dialog> (app.js: showModal(), Cancel/ESC clear the slot); confirming closes it immediately and shows a busy spinner toast in the stack (since a real backup can run for minutes) instead of leaving the dialog open with no feedback. - host_backup now runs backup_template via asyncio.to_thread so a multi-GB 7z run no longer blocks the event loop (and every other dashboard poll) for its whole duration. - Static assets are served with a mtime-based ?v= query param so a redeploy is picked up without a hard refresh.
411 lines
13 KiB
Python
411 lines
13 KiB
Python
"""Tests for the read-only (W4) web dashboard app.
|
|
|
|
In-process only: routes call ``operations/`` + a backend injected via
|
|
FastAPI dependency override -- no real ``vmrun`` subprocess is ever
|
|
spawned. ``shutil.disk_usage``/``shutil.which`` are monkeypatched so no
|
|
real host state (disk, systemd) leaks into the assertions either.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import shutil
|
|
from collections import namedtuple
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
import pytest
|
|
from fastapi.testclient import TestClient
|
|
|
|
import ci_orchestrator.web.app as app_module
|
|
from ci_orchestrator.backends.errors import BackendNotAvailable
|
|
from ci_orchestrator.backends.protocol import VmHandle
|
|
from ci_orchestrator.config import BackendConfig, CargoCacheConfig, Config, Paths
|
|
from ci_orchestrator.operations.cache import CacheStatus
|
|
from ci_orchestrator.operations.host import DiskStatus, RunnerStatus
|
|
from ci_orchestrator.operations.vms import LiveVm
|
|
from ci_orchestrator.web.app import create_app, get_backend
|
|
|
|
_DiskUsage = namedtuple("_DiskUsage", ["total", "used", "free"])
|
|
|
|
|
|
def _fake_disk_usage(_path: Any) -> _DiskUsage:
|
|
gb = 1024**3
|
|
return _DiskUsage(total=100 * gb, used=40 * gb, free=60 * gb)
|
|
|
|
|
|
def _make_config(root: Path) -> Config:
|
|
return Config(
|
|
paths=Paths(
|
|
root=root,
|
|
templates=root / "templates",
|
|
build_vms=root / "build-vms",
|
|
artifacts=root / "artifacts",
|
|
keys=root / "keys",
|
|
),
|
|
backend=BackendConfig(),
|
|
cargo_cache=CargoCacheConfig(
|
|
enabled=True, root=root / "cache" / "cargo", retention_days=30
|
|
),
|
|
)
|
|
|
|
|
|
class _FakeBackend:
|
|
"""Stub ``VmBackend`` -- proves no real ``vmrun`` is ever invoked."""
|
|
|
|
def __init__(self, handles: list[VmHandle], ip: str | None = "192.168.79.201") -> None:
|
|
self._handles = handles
|
|
self._ip = ip
|
|
self.list_running_calls = 0
|
|
self.get_ip_calls = 0
|
|
|
|
def list_running(self) -> list[VmHandle]:
|
|
self.list_running_calls += 1
|
|
return self._handles
|
|
|
|
def get_ip(self, handle: VmHandle, timeout: float = 0.0) -> str | None:
|
|
self.get_ip_calls += 1
|
|
return self._ip
|
|
|
|
|
|
class _BoomBackend:
|
|
"""Backend whose ``list_running`` raises something other than ``BackendError``."""
|
|
|
|
def list_running(self) -> list[VmHandle]:
|
|
raise RuntimeError("boom")
|
|
|
|
def get_ip(self, handle: VmHandle, timeout: float = 0.0) -> str | None:
|
|
return None
|
|
|
|
|
|
def _quiet_host(monkeypatch: pytest.MonkeyPatch) -> None:
|
|
"""No real disk/systemctl calls from ``operations.host`` during a test."""
|
|
monkeypatch.setattr(shutil, "disk_usage", _fake_disk_usage)
|
|
monkeypatch.setattr(shutil, "which", lambda _name: None)
|
|
|
|
|
|
# ── /healthz ──────────────────────────────────────────────────────────────
|
|
|
|
|
|
def test_healthz_returns_ok(tmp_path: Path) -> None:
|
|
app = create_app(_make_config(tmp_path))
|
|
client = TestClient(app)
|
|
|
|
resp = client.get("/healthz")
|
|
|
|
assert resp.status_code == 200
|
|
assert resp.json() == {"status": "ok"}
|
|
|
|
|
|
# ── / (dashboard) ────────────────────────────────────────────────────────
|
|
|
|
|
|
def test_dashboard_renders_all_four_cards(
|
|
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
|
) -> None:
|
|
_quiet_host(monkeypatch)
|
|
app = create_app(_make_config(tmp_path))
|
|
app.dependency_overrides[get_backend] = lambda: _FakeBackend([])
|
|
client = TestClient(app)
|
|
|
|
resp = client.get("/")
|
|
|
|
assert resp.status_code == 200
|
|
for needle in ("Live VMs", "Cargo cache", "Disk", "Runners"):
|
|
assert needle in resp.text
|
|
|
|
|
|
def test_dashboard_shows_backend_unavailable_notice_when_backend_is_none(
|
|
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
|
) -> None:
|
|
_quiet_host(monkeypatch)
|
|
app = create_app(_make_config(tmp_path))
|
|
app.dependency_overrides[get_backend] = lambda: None
|
|
client = TestClient(app)
|
|
|
|
resp = client.get("/")
|
|
|
|
assert resp.status_code == 200
|
|
assert "unavailable" in resp.text.lower()
|
|
|
|
|
|
# ── /vms ──────────────────────────────────────────────────────────────────
|
|
|
|
|
|
def test_vms_route_renders_injected_vm_and_never_touches_real_vmrun(tmp_path: Path) -> None:
|
|
vmx = tmp_path / "build-vms" / "job1" / "job1.vmx"
|
|
fake = _FakeBackend([VmHandle(identifier=str(vmx), name="job1")])
|
|
app = create_app(_make_config(tmp_path))
|
|
app.dependency_overrides[get_backend] = lambda: fake
|
|
client = TestClient(app)
|
|
|
|
resp = client.get("/vms")
|
|
|
|
assert resp.status_code == 200
|
|
assert "job1" in resp.text
|
|
assert fake.list_running_calls == 1
|
|
|
|
|
|
def test_vms_route_degrades_when_backend_is_none(tmp_path: Path) -> None:
|
|
app = create_app(_make_config(tmp_path))
|
|
app.dependency_overrides[get_backend] = lambda: None
|
|
client = TestClient(app)
|
|
|
|
resp = client.get("/vms")
|
|
|
|
assert resp.status_code == 200
|
|
assert "unavailable" in resp.text.lower()
|
|
|
|
|
|
def test_vms_route_degrades_on_unexpected_backend_error(tmp_path: Path) -> None:
|
|
app = create_app(_make_config(tmp_path))
|
|
app.dependency_overrides[get_backend] = lambda: _BoomBackend()
|
|
client = TestClient(app)
|
|
|
|
resp = client.get("/vms")
|
|
|
|
assert resp.status_code == 200
|
|
|
|
|
|
def test_get_backend_degrades_to_none_when_vmrun_missing(
|
|
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
|
) -> None:
|
|
def _raise(_config: Config) -> Any:
|
|
raise BackendNotAvailable("vmrun not found")
|
|
|
|
monkeypatch.setattr(app_module, "load_backend", _raise)
|
|
app = create_app(_make_config(tmp_path))
|
|
client = TestClient(app)
|
|
|
|
resp = client.get("/vms")
|
|
|
|
assert resp.status_code == 200
|
|
assert "unavailable" in resp.text.lower()
|
|
|
|
|
|
# ── /cache ────────────────────────────────────────────────────────────────
|
|
|
|
|
|
def test_cache_route_renders_status(tmp_path: Path) -> None:
|
|
app = create_app(_make_config(tmp_path))
|
|
client = TestClient(app)
|
|
|
|
resp = client.get("/cache")
|
|
|
|
assert resp.status_code == 200
|
|
assert "Cargo cache" in resp.text
|
|
|
|
|
|
def test_cache_route_degrades_when_operations_raises(
|
|
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
|
) -> None:
|
|
def _boom(_config: Config) -> Any:
|
|
raise RuntimeError("boom")
|
|
|
|
monkeypatch.setattr(app_module, "cargo_cache_status", _boom)
|
|
app = create_app(_make_config(tmp_path))
|
|
client = TestClient(app)
|
|
|
|
resp = client.get("/cache")
|
|
|
|
assert resp.status_code == 200
|
|
|
|
|
|
# ── /host ─────────────────────────────────────────────────────────────────
|
|
|
|
|
|
def test_runners_route_renders_runner_status(
|
|
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
|
) -> None:
|
|
# Stage E: the /runners partial carries only the runner statuses -- the
|
|
# disk card was folded into the status-header Disk tile.
|
|
_quiet_host(monkeypatch)
|
|
app = create_app(_make_config(tmp_path))
|
|
client = TestClient(app)
|
|
|
|
resp = client.get("/runners")
|
|
|
|
assert resp.status_code == 200
|
|
assert "Runners" in resp.text
|
|
assert "act-runner.service" in resp.text
|
|
assert "act-runner-rust.service" in resp.text
|
|
|
|
|
|
def test_runners_route_degrades_when_disk_status_raises(
|
|
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
|
) -> None:
|
|
def _boom(_path: Path) -> Any:
|
|
raise RuntimeError("boom")
|
|
|
|
monkeypatch.setattr(app_module, "disk_status", _boom)
|
|
monkeypatch.setattr(shutil, "which", lambda _name: None)
|
|
app = create_app(_make_config(tmp_path))
|
|
client = TestClient(app)
|
|
|
|
resp = client.get("/runners")
|
|
|
|
assert resp.status_code == 200
|
|
|
|
|
|
def test_runners_route_degrades_when_runner_status_raises(
|
|
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
|
) -> None:
|
|
def _boom(_names: list[str]) -> Any:
|
|
raise RuntimeError("boom")
|
|
|
|
_quiet_host(monkeypatch)
|
|
monkeypatch.setattr(app_module, "runner_status", _boom)
|
|
app = create_app(_make_config(tmp_path))
|
|
client = TestClient(app)
|
|
|
|
resp = client.get("/runners")
|
|
|
|
assert resp.status_code == 200
|
|
|
|
|
|
# ── static mount ─────────────────────────────────────────────────────────
|
|
|
|
|
|
def test_static_mount_serves_app_css(tmp_path: Path) -> None:
|
|
app = create_app(_make_config(tmp_path))
|
|
client = TestClient(app)
|
|
|
|
resp = client.get("/static/app.css")
|
|
|
|
assert resp.status_code == 200
|
|
|
|
|
|
# ── _health_summary (A3) ─────────────────────────────────────────────────
|
|
|
|
_ACTIVE_RUNNER = RunnerStatus(name="act-runner.service", active=True, sub_state="running")
|
|
_DEAD_RUNNER = RunnerStatus(name="act-runner-rust.service", active=False, sub_state="dead")
|
|
_LIVE_VM = LiveVm(
|
|
name="job1", vmx="/x/job1.vmx", ip="10.0.0.5", guest_os="ubuntu-64",
|
|
uptime_sec=120, is_orphan=False,
|
|
)
|
|
_ORPHAN_VM = LiveVm(
|
|
name="job2", vmx="/x/job2.vmx", ip=None, guest_os="ubuntu-64",
|
|
uptime_sec=99999, is_orphan=True,
|
|
)
|
|
_CACHE = CacheStatus(
|
|
root=Path("/cache"), total_bytes=1024, entry_count=1, oldest_age_days=1, retention_days=30
|
|
)
|
|
|
|
|
|
def test_health_summary_backend_unavailable_is_vm_unknown() -> None:
|
|
health = app_module._health_summary(
|
|
[], backend_available=False, cache=None, disk=None, runners=[]
|
|
)
|
|
assert health.vm_class == "tile-unknown"
|
|
assert health.vm_label == "unknown"
|
|
|
|
|
|
def test_health_summary_orphan_present_is_vm_warn() -> None:
|
|
health = app_module._health_summary(
|
|
[_LIVE_VM, _ORPHAN_VM], backend_available=True, cache=None, disk=None, runners=[]
|
|
)
|
|
assert health.vm_count == 2
|
|
assert health.orphan_count == 1
|
|
assert health.vm_class == "tile-warn"
|
|
assert health.vm_label == "orphaned"
|
|
|
|
|
|
def test_health_summary_no_orphans_is_vm_ok() -> None:
|
|
health = app_module._health_summary(
|
|
[_LIVE_VM], backend_available=True, cache=None, disk=None, runners=[]
|
|
)
|
|
assert health.vm_class == "tile-ok"
|
|
assert health.vm_label == "ok"
|
|
|
|
|
|
def test_health_summary_no_runners_is_unknown() -> None:
|
|
health = app_module._health_summary(
|
|
[], backend_available=True, cache=None, disk=None, runners=[]
|
|
)
|
|
assert health.runners_total == 0
|
|
assert health.runner_class == "tile-unknown"
|
|
|
|
|
|
def test_health_summary_all_runners_active_is_ok() -> None:
|
|
health = app_module._health_summary(
|
|
[], backend_available=True, cache=None, disk=None, runners=[_ACTIVE_RUNNER]
|
|
)
|
|
assert health.runners_active == 1
|
|
assert health.runners_total == 1
|
|
assert health.runner_class == "tile-ok"
|
|
|
|
|
|
def test_health_summary_a_down_runner_is_bad() -> None:
|
|
health = app_module._health_summary(
|
|
[], backend_available=True, cache=None, disk=None,
|
|
runners=[_ACTIVE_RUNNER, _DEAD_RUNNER],
|
|
)
|
|
assert health.runners_active == 1
|
|
assert health.runners_total == 2
|
|
assert health.runner_class == "tile-bad"
|
|
assert health.runner_label == "down"
|
|
|
|
|
|
def test_health_summary_disk_none_is_unknown() -> None:
|
|
health = app_module._health_summary(
|
|
[], backend_available=True, cache=None, disk=None, runners=[]
|
|
)
|
|
assert health.disk_used_pct is None
|
|
assert health.disk_class == "tile-unknown"
|
|
|
|
|
|
def test_health_summary_disk_above_critical_is_bad() -> None:
|
|
disk = DiskStatus(path=Path("/"), total_gb=100.0, free_gb=5.0, used_pct=90.0)
|
|
health = app_module._health_summary(
|
|
[], backend_available=True, cache=None, disk=disk, runners=[]
|
|
)
|
|
assert health.disk_class == "tile-bad"
|
|
assert health.disk_label == "critical"
|
|
|
|
|
|
def test_health_summary_disk_above_elevated_is_warn() -> None:
|
|
disk = DiskStatus(path=Path("/"), total_gb=100.0, free_gb=25.0, used_pct=75.0)
|
|
health = app_module._health_summary(
|
|
[], backend_available=True, cache=None, disk=disk, runners=[]
|
|
)
|
|
assert health.disk_class == "tile-warn"
|
|
assert health.disk_label == "elevated"
|
|
|
|
|
|
def test_health_summary_disk_healthy_is_ok() -> None:
|
|
disk = DiskStatus(path=Path("/"), total_gb=100.0, free_gb=60.0, used_pct=40.0)
|
|
health = app_module._health_summary(
|
|
[], backend_available=True, cache=None, disk=disk, runners=[]
|
|
)
|
|
assert health.disk_class == "tile-ok"
|
|
assert health.disk_label == "ok"
|
|
|
|
|
|
def test_health_summary_cache_none_is_unknown() -> None:
|
|
health = app_module._health_summary(
|
|
[], backend_available=True, cache=None, disk=None, runners=[]
|
|
)
|
|
assert health.cache_bytes is None
|
|
assert health.cache_class == "tile-unknown"
|
|
|
|
|
|
def test_health_summary_cache_present_is_ok() -> None:
|
|
health = app_module._health_summary(
|
|
[], backend_available=True, cache=_CACHE, disk=None, runners=[]
|
|
)
|
|
assert health.cache_bytes == 1024
|
|
assert health.cache_class == "tile-ok"
|
|
|
|
|
|
def test_dashboard_includes_status_header_tiles(
|
|
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
|
) -> None:
|
|
_quiet_host(monkeypatch)
|
|
app = create_app(_make_config(tmp_path))
|
|
app.dependency_overrides[get_backend] = lambda: _FakeBackend([])
|
|
client = TestClient(app)
|
|
|
|
resp = client.get("/")
|
|
|
|
assert resp.status_code == 200
|
|
assert 'id="status-header"' in resp.text
|