"""FastAPI application factory for the web dashboard (W4 read-only + W5 auth + W6 actions). **In-process only**: routes call ``operations/`` + ``backends.load_backend`` directly and render Jinja2 templates. They never shell out to the CLI (see ``CLAUDE.md`` / the web-UI plan's architecture rules). **Auth (W5)**: the four data routes below are protected by :func:`~ci_orchestrator.web.auth.require_user`. Auth is ACTIVE iff ``config.web.oidc is not None``; when inactive (the W4 default -- bare localhost dev), ``require_user`` is a no-op and these routes behave exactly as in W4. ``create_app`` itself never chooses the bind address, that is ``commands/web.py``'s job. **Sessions are ALWAYS on (W6)**: ``SessionMiddleware`` is installed unconditionally, not just when OIDC is active -- CSRF (``web/security.py``) needs ``request.session`` in both auth modes. When OIDC is active the session is signed with the keyring-sourced, persistent ``config.web.session_secret_cred`` secret (stable across restarts, so a login survives a redeploy). When inactive, the secret is a fresh ``secrets.token_urlsafe(32)`` generated per process -- fine for an open-localhost dev session (it only needs to survive one process's lifetime; no keyring lookup, so open mode still needs no keyring at all). **Backend may be unavailable**: :func:`get_backend` is a FastAPI dependency that swallows :class:`~ci_orchestrator.backends.errors.BackendError` (raised when ``vmrun`` is missing on this host) and degrades to ``None`` so a route never 500s. Tests override it via ``app.dependency_overrides[get_backend]``. """ from __future__ import annotations import secrets from dataclasses import dataclass from pathlib import Path from typing import TYPE_CHECKING from fastapi import Depends, FastAPI, Request from fastapi.responses import HTMLResponse from fastapi.staticfiles import StaticFiles from fastapi.templating import Jinja2Templates from starlette.middleware.sessions import SessionMiddleware from ci_orchestrator.backends import load_backend from ci_orchestrator.backends.errors import BackendError from ci_orchestrator.config import gitea_base_url from ci_orchestrator.operations.cache import CacheStatus, cargo_cache_status from ci_orchestrator.operations.executions import ExecutionsView, list_executions from ci_orchestrator.operations.host import ( DiskStatus, HostResources, RunnerStatus, VmwareModuleHealth, disk_status, host_resources, runner_status, vmware_module_health, ) from ci_orchestrator.operations.inventory import ( FileEntry, TemplateInfo, backups_by_template, list_artifacts, list_templates, ) from ci_orchestrator.operations.services import ServiceState, service_states from ci_orchestrator.operations.vms import LiveVm, list_live_vms from ci_orchestrator.web import filters from ci_orchestrator.web.auth import ( _read_secret, configure_auth, gitea_token_for, refresh_gitea_token, require_user, ) from ci_orchestrator.web.security import issue_csrf if TYPE_CHECKING: # pragma: no cover from ci_orchestrator.backends.protocol import VmBackend from ci_orchestrator.config import Config _WEB_DIR = Path(__file__).resolve().parent #: The two real ``act_runner`` systemd units on the Linux host (see #: CLAUDE.md's "Phase C"). A module constant per the W4 plan section. RUNNER_SERVICE_NAMES: list[str] = ["act-runner.service", "act-runner-rust.service"] #: Disk-usage health thresholds for the status header (A3): above the #: "critical" percentage -> red; above "elevated" -> amber; else green. _DISK_PCT_CRITICAL = 85.0 _DISK_PCT_ELEVATED = 70.0 #: CPU / memory usage thresholds for the two Stage-E resource tiles, same #: red/amber/green shape as the disk thresholds above. _CPU_PCT_CRITICAL = 90.0 _CPU_PCT_ELEVATED = 75.0 _MEM_PCT_CRITICAL = 90.0 _MEM_PCT_ELEVATED = 75.0 #: CSS class used by ``partials/status_header.html`` for each health level. #: Health must never be color-only -- the template always pairs this class #: with a short text label (see :class:`HealthSummary`). _TILE_OK = "tile-ok" _TILE_WARN = "tile-warn" _TILE_BAD = "tile-bad" _TILE_UNKNOWN = "tile-unknown" # ─────────────────────────────────────────────────────────────── dependency def get_backend(request: Request) -> VmBackend | None: """FastAPI dependency: the VM backend for this request, or ``None``. Catches :class:`BackendError` (e.g. ``vmrun`` missing on this host -- ``BackendNotAvailable``) so routes degrade to an empty/"unavailable" state instead of raising. Overridable in tests via ``app.dependency_overrides[get_backend]`` with a fake backend. """ config: Config = request.app.state.config try: return load_backend(config) except BackendError: return None # ──────────────────────────────────────────────────────────── data fetchers def _fetch_live_vms(backend: VmBackend | None, config: Config) -> list[LiveVm]: """Render-ready VM list. Degrades to ``[]`` on any backend hiccup.""" if backend is None: return [] try: return list_live_vms(backend, config) except Exception: # a route must never 500 on a backend hiccup return [] def _fetch_cache_status(config: Config) -> CacheStatus | None: """Cargo cache snapshot, or ``None`` if it could not be read.""" try: return cargo_cache_status(config) except Exception: # degrade, never 500 return None def _pat_fallback(config: Config) -> str | None: """The optional keyring PAT for Gitea, or ``None`` when not configured. Covers the cases the login token cannot: a process restart (which empties the server-side token map while sessions stay valid) and a non-admin operator. A missing/unreadable keyring entry is not an error -- the card degrades to listing VMs. """ cred = config.web.gitea_token_cred if not cred: return None try: return _read_secret(cred) except Exception: # degrade, never 500 over an optional secret return None async def _fetch_executions( request: Request, config: Config, vms: list[LiveVm] ) -> ExecutionsView: """Recent Gitea runs with ``vms`` attached. Never raises. Token precedence: the access token from this operator's own OIDC login (held server-side, never in the cookie -- see ``web/auth.py``), then the optional keyring PAT. A ``401`` triggers exactly one refresh attempt before the card falls back to its degraded state. """ username = request.session.get("user") stored = gitea_token_for(request.app, username) if stored is not None and stored.expired and username: stored = await refresh_gitea_token(request, str(username)) or stored token = stored.access_token if stored is not None else _pat_fallback(config) try: view = list_executions(config, vms, token=token) except Exception: # the operations layer already degrades; belt and braces return ExecutionsView( runs=[], unmatched_vms=vms, source="none", error="Executions unavailable." ) if view.auth_failed and stored is not None and username: refreshed = await refresh_gitea_token(request, str(username)) if refreshed is not None: try: return list_executions(config, vms, token=refreshed.access_token) except Exception: return view return view def _fetch_disk_status(config: Config) -> DiskStatus | None: """Disk usage for ``config.paths.root``, or ``None`` if unreadable.""" try: return disk_status(config.paths.root) except Exception: # degrade, never 500 return None def _fetch_runner_status() -> list[RunnerStatus]: """Runner service statuses, or ``[]`` if the query could not run at all.""" try: return runner_status(RUNNER_SERVICE_NAMES) except Exception: # degrade, never 500 return [] def _fetch_resources() -> HostResources | None: """Host CPU/memory/uptime/load snapshot (B1), or ``None`` if unreadable. ``host_resources`` itself already degrades to an all-zero snapshot on ``OSError`` (never raises) -- this ``except Exception`` is a defensive second layer so a route can never 500 even on a genuinely unexpected failure. """ try: return host_resources() except Exception: # degrade, never 500 return None def _fetch_vmware_health() -> VmwareModuleHealth | None: """``vmmon``/``vmnet``/``vmware-host.service`` health (B2), or ``None`` if unreadable.""" try: return vmware_module_health() except Exception: # degrade, never 500 return None def _fetch_templates(config: Config) -> list[TemplateInfo] | None: """Template inventory rows (B3), or ``None`` if the scan itself failed. Distinct from an empty list (``[]``, a templates dir with nothing in it) -- ``None`` triggers the "unavailable" notice in ``templates.html``, ``[]`` triggers its plain "no templates found" state. """ try: return list_templates(config) except Exception: # degrade, never 500 return None def _fetch_artifacts(config: Config) -> list[FileEntry] | None: """Artifact listing (B4), or ``None`` if the scan itself failed.""" try: return list_artifacts(config) except Exception: # degrade, never 500 return None def _fetch_service_states(config: Config) -> list[ServiceState]: """Managed systemd unit states (Stage D), or ``[]`` if the query could not run at all. ``service_states`` itself already degrades a single unit to ``active=False, sub_state="unknown"`` on a ``systemctl`` hiccup (see ``operations/host.py::runner_status``) -- this ``except Exception`` is a defensive second layer, matching every other ``_fetch_*`` wrapper here. """ try: return service_states(config.web.managed_services) except Exception: # degrade, never 500 return [] # ────────────────────────────────────────────────────────── status header @dataclass(frozen=True) class HealthSummary: """Render-ready values for the status-header tiles (A3). Every field is derived purely from data the dashboard route already fetched -- :func:`_health_summary` makes no new call of its own. Each tile carries both a CSS class (``*_class``) and a short text ``*_label`` so health never reads from color alone. """ vm_count: int orphan_count: int vm_class: str vm_label: str runners_active: int runners_total: int runner_class: str runner_label: str cpu_pct: float | None ncpu: int cpu_class: str cpu_label: str mem_used_gb: float | None mem_total_gb: float | None mem_pct: float | None mem_class: str mem_label: str disk_used_pct: float | None disk_free_gb: float | None disk_total_gb: float | None disk_class: str disk_label: str cache_bytes: int | None cache_entries: int | None cache_class: str cache_label: str def _threshold_level(pct: float, *, elevated: float, critical: float) -> tuple[str, str]: """Map a usage percentage to a ``(tile-class, label)`` on the shared red/amber/green scale.""" if pct > critical: return _TILE_BAD, "critical" if pct > elevated: return _TILE_WARN, "elevated" return _TILE_OK, "ok" def _health_summary( vms: list[LiveVm], *, backend_available: bool, cache: CacheStatus | None, disk: DiskStatus | None, runners: list[RunnerStatus], resources: HostResources | None = None, ) -> HealthSummary: """Derive the A3 status-header tiles from already-fetched dashboard context. Thresholds (see the A3 brief): disk ``used_pct`` > 85 -> red, > 70 -> amber, else green; any runner not active -> red; an orphaned VM present -> amber. A missing data source (backend down, disk/cache unreadable) renders as a neutral "unknown" tile rather than false-green. """ orphan_count = sum(1 for vm in vms if vm.is_orphan) if not backend_available: vm_class, vm_label = _TILE_UNKNOWN, "unknown" elif orphan_count > 0: vm_class, vm_label = _TILE_WARN, "orphaned" else: vm_class, vm_label = _TILE_OK, "ok" runners_total = len(runners) runners_active = sum(1 for r in runners if r.active) if runners_total == 0: runner_class, runner_label = _TILE_UNKNOWN, "unknown" elif runners_active < runners_total: runner_class, runner_label = _TILE_BAD, "down" else: runner_class, runner_label = _TILE_OK, "ok" if resources is None: cpu_pct, ncpu = None, 0 cpu_class, cpu_label = _TILE_UNKNOWN, "unknown" mem_used_gb = mem_total_gb = mem_pct = None mem_class, mem_label = _TILE_UNKNOWN, "unknown" else: cpu_pct, ncpu = resources.cpu_pct, resources.ncpu cpu_class, cpu_label = _threshold_level( cpu_pct, elevated=_CPU_PCT_ELEVATED, critical=_CPU_PCT_CRITICAL ) mem_used_gb, mem_total_gb = resources.mem_used_gb, resources.mem_total_gb mem_pct = (mem_used_gb / mem_total_gb * 100.0) if mem_total_gb > 0 else 0.0 mem_class, mem_label = _threshold_level( mem_pct, elevated=_MEM_PCT_ELEVATED, critical=_MEM_PCT_CRITICAL ) disk_used_pct = disk.used_pct if disk is not None else None disk_free_gb = disk.free_gb if disk is not None else None disk_total_gb = disk.total_gb if disk is not None else None if disk_used_pct is None: disk_class, disk_label = _TILE_UNKNOWN, "unknown" else: disk_class, disk_label = _threshold_level( disk_used_pct, elevated=_DISK_PCT_ELEVATED, critical=_DISK_PCT_CRITICAL ) cache_bytes = cache.total_bytes if cache is not None else None cache_entries = cache.entry_count if cache is not None else None cache_class, cache_label = ( (_TILE_OK, "ok") if cache is not None else (_TILE_UNKNOWN, "unknown") ) return HealthSummary( vm_count=len(vms), orphan_count=orphan_count, vm_class=vm_class, vm_label=vm_label, runners_active=runners_active, runners_total=runners_total, runner_class=runner_class, runner_label=runner_label, cpu_pct=cpu_pct, ncpu=ncpu, cpu_class=cpu_class, cpu_label=cpu_label, mem_used_gb=mem_used_gb, mem_total_gb=mem_total_gb, mem_pct=mem_pct, mem_class=mem_class, mem_label=mem_label, disk_used_pct=disk_used_pct, disk_free_gb=disk_free_gb, disk_total_gb=disk_total_gb, disk_class=disk_class, disk_label=disk_label, cache_bytes=cache_bytes, cache_entries=cache_entries, cache_class=cache_class, cache_label=cache_label, ) # ────────────────────────────────────────────────────────────────── routes def healthz() -> dict[str, str]: """Liveness probe for systemd/monitor. Always ``200``.""" return {"status": "ok"} async def dashboard( request: Request, backend: VmBackend | None = Depends(get_backend) ) -> HTMLResponse: """Full dashboard: executions, cache, disk, runner, and Stage-B info cards.""" config: Config = request.app.state.config templates: Jinja2Templates = request.app.state.templates vms = _fetch_live_vms(backend, config) backend_available = backend is not None cache = _fetch_cache_status(config) disk = _fetch_disk_status(config) runners = _fetch_runner_status() resources = _fetch_resources() template_rows = _fetch_templates(config) context = { "vms": vms, "executions": await _fetch_executions(request, config, vms), "backend_available": backend_available, "cache": cache, "disk": disk, "runners": runners, "resources": resources, "vmware": _fetch_vmware_health(), "templates": template_rows, "template_backups": backups_by_template(config), "artifacts": _fetch_artifacts(config), "services": _fetch_service_states(config), "health": _health_summary( vms, backend_available=backend_available, cache=cache, disk=disk, runners=runners, resources=resources, ), "csrf_token": issue_csrf(request), } return templates.TemplateResponse(request, "dashboard.html", context) async def executions_partial( request: Request, backend: VmBackend | None = Depends(get_backend) ) -> HTMLResponse: """``GET /executions`` -- polled every 5s by the dashboard via htmx. Replaces the old ``GET /vms`` card: run-centric, with the live VMs grouped under the run that spawned them. The runs come from a 10s-TTL cache (``operations.executions``) while the VM data is read fresh on every poll, so this cadence costs Gitea at most one API call per TTL window. """ config: Config = request.app.state.config templates: Jinja2Templates = request.app.state.templates vms = _fetch_live_vms(backend, config) context = { "vms": vms, "executions": await _fetch_executions(request, config, vms), "backend_available": backend is not None, "csrf_token": issue_csrf(request), } return templates.TemplateResponse(request, "partials/executions.html", context) def cache_partial(request: Request) -> HTMLResponse: """``GET /cache`` -- cargo cache status partial.""" config: Config = request.app.state.config templates: Jinja2Templates = request.app.state.templates context = {"cache": _fetch_cache_status(config), "csrf_token": issue_csrf(request)} return templates.TemplateResponse(request, "partials/cache.html", context) def runners_partial(request: Request) -> HTMLResponse: """``GET /runners`` -- runner status partial (Stage E: disk/backup moved out). The disk card was folded into the status-header Disk tile and the standalone template-backup card into a per-row Templates action, so this read-only poll now carries only the runner statuses. """ templates: Jinja2Templates = request.app.state.templates context = {"runners": _fetch_runner_status()} return templates.TemplateResponse(request, "partials/runners.html", context) def host_partial(request: Request) -> HTMLResponse: """``GET /host`` (B5) -- host resources (CPU/RAM/load) partial, polled every 5s.""" templates: Jinja2Templates = request.app.state.templates context = {"resources": _fetch_resources()} return templates.TemplateResponse(request, "partials/host.html", context) def vmware_partial(request: Request) -> HTMLResponse: """``GET /vmware`` (B5) -- polled every 15s by the dashboard via htmx.""" templates: Jinja2Templates = request.app.state.templates context = {"vmware": _fetch_vmware_health()} return templates.TemplateResponse(request, "partials/vmware.html", context) def templates_partial(request: Request) -> HTMLResponse: """``GET /templates`` (B5) -- polled every 30s by the dashboard via htmx. Stage E: now carries the per-template backups map (for each row's expandable backup list) and a fresh ``csrf_token`` (the per-row Backup button is a mutating form, so this poll is no longer purely read-only). """ config: Config = request.app.state.config templates: Jinja2Templates = request.app.state.templates context = { "templates": _fetch_templates(config), "template_backups": backups_by_template(config), "csrf_token": issue_csrf(request), } return templates.TemplateResponse(request, "partials/templates.html", context) def files_partial(request: Request) -> HTMLResponse: """``GET /files`` (B5) -- polled every 30s by the dashboard via htmx. Stage E: the backups listing moved under the Templates panel (per-row expandable), so this poll carries only the artifacts list plus a fresh ``csrf_token`` for the per-row Delete button (now a mutating form). """ config: Config = request.app.state.config templates: Jinja2Templates = request.app.state.templates context = { "artifacts": _fetch_artifacts(config), "csrf_token": issue_csrf(request), } return templates.TemplateResponse(request, "partials/files.html", context) def services_partial(request: Request) -> HTMLResponse: """``GET /services`` (Stage D) -- polled every 15s by the dashboard via htmx. Needs a fresh ``csrf_token`` on every render (unlike the read-only ``host``/``vmware``/``runners`` cards) -- the panel's start/stop/restart forms are mutating, like the per-row forms in ``executions_partial``/``templates_partial``/``files_partial``. """ config: Config = request.app.state.config templates: Jinja2Templates = request.app.state.templates context = { "services": _fetch_service_states(config), "csrf_token": issue_csrf(request), } return templates.TemplateResponse(request, "partials/services.html", context) # ─────────────────────────────────────────────────────────────────── factory def create_app(config: Config) -> FastAPI: """Build the dashboard app: W4 read-only routes + W5 auth + W6 mutating actions. Binding to a concrete address (always ``127.0.0.1`` by default -- never ``0.0.0.0``) is ``commands/web.py``'s job, not this factory's. All data routes are guarded by ``require_user`` (W5): when ``config.web.oidc is None`` that dependency is a no-op (open mode, W4 parity); when set, unauthenticated requests redirect to ``/login``. ``/healthz`` and ``/static`` stay unprotected either way. The mutating routes (W6, ``web/routes.py``) are additionally behind CSRF -- see that module's docstring for the full safety contract. """ # Deferred import: avoids a module-level import cycle (web.routes imports # get_backend from this module). from ci_orchestrator.web.routes import ( encode_artifact_id, encode_vmx_id, register_action_routes, ) app = FastAPI(title="ci-orchestrator web") app.state.config = config app.state.templates = Jinja2Templates(directory=str(_WEB_DIR / "templates")) filters.register(app.state.templates) app.state.templates.env.filters["vmx_id"] = encode_vmx_id app.state.templates.env.filters["artifact_id"] = encode_artifact_id # Stage C2: a Jinja *global* (not per-route context) so every template # that extends base.html -- dashboard, audit, the action-page wrapper -- # picks up the nav's "Gitea ↗" link without each route builder having to # remember to thread it through. app.state.templates.env.globals["gitea_url"] = gitea_base_url(config.web) # Cache-busting for the stylesheet: StaticFiles sends only ETag/ # Last-Modified (no Cache-Control), so a plain browser reload can serve a # stale app.css after a redeploy. Stamping the with the file's # mtime -- re-read per render -- means every deploy changes the URL, so a # normal navigation refetches the CSS with no hard-refresh needed. static_css = _WEB_DIR / "static" / "app.css" def _asset_version() -> str: try: return str(int(static_css.stat().st_mtime)) except OSError: return "0" app.state.templates.env.globals["asset_version"] = _asset_version app.mount("/static", StaticFiles(directory=str(_WEB_DIR / "static")), name="static") app.get("/healthz")(healthz) app.get("/", response_class=HTMLResponse, dependencies=[Depends(require_user)])(dashboard) app.get( "/executions", response_class=HTMLResponse, dependencies=[Depends(require_user)] )(executions_partial) app.get("/cache", response_class=HTMLResponse, dependencies=[Depends(require_user)])( cache_partial ) app.get("/host", response_class=HTMLResponse, dependencies=[Depends(require_user)])( host_partial ) app.get("/runners", response_class=HTMLResponse, dependencies=[Depends(require_user)])( runners_partial ) app.get("/vmware", response_class=HTMLResponse, dependencies=[Depends(require_user)])( vmware_partial ) app.get("/templates", response_class=HTMLResponse, dependencies=[Depends(require_user)])( templates_partial ) app.get("/files", response_class=HTMLResponse, dependencies=[Depends(require_user)])( files_partial ) app.get("/services", response_class=HTMLResponse, dependencies=[Depends(require_user)])( services_partial ) register_action_routes(app) # SessionMiddleware is ALWAYS installed (W6) -- CSRF needs # ``request.session`` in both auth modes, not just when OIDC is active. # Added here, before the app is ever handed to a server -- Starlette # builds the middleware stack lazily on first request, so this is safe # regardless of where the route registrations above sit. # # Active OIDC: persistent, keyring-sourced secret so a login survives a # process restart. Inactive: a fresh per-process secret -- open mode # still needs no keyring at all, and the session only needs to outlive # this one process (there is nothing sensitive in it besides a CSRF # token when auth is off). session_secret = ( _read_secret(config.web.session_secret_cred) if config.web.oidc is not None else secrets.token_urlsafe(32) ) app.add_middleware( SessionMiddleware, secret_key=session_secret, same_site="lax", https_only=False, ) configure_auth(app, config) return app __all__ = [ "RUNNER_SERVICE_NAMES", "HealthSummary", "create_app", "dashboard", "executions_partial", "files_partial", "get_backend", "healthz", "host_partial", "runners_partial", "services_partial", "templates_partial", "vmware_partial", ]