local-ci-cd-system/src/ci_orchestrator/web/app.py

"""FastAPI application factory for the web dashboard (W4 read-only + W5 auth + W6 actions).

**In-process only**: routes call ``operations/`` + ``backends.load_backend``
directly and render Jinja2 templates. They never shell out to the CLI (see
``CLAUDE.md`` / the web-UI plan's architecture rules).

**Auth (W5)**: the four data routes below are protected by
:func:`~ci_orchestrator.web.auth.require_user`. Auth is ACTIVE iff
``config.web.oidc is not None``; when inactive (the W4 default -- bare
localhost dev), ``require_user`` is a no-op and these routes behave exactly
as in W4. ``create_app`` itself never chooses the bind address, that is
``commands/web.py``'s job.

**Sessions are ALWAYS on (W6)**: ``SessionMiddleware`` is installed
unconditionally, not just when OIDC is active -- CSRF (``web/security.py``)
needs ``request.session`` in both auth modes. When OIDC is active the
session is signed with the keyring-sourced, persistent
``config.web.session_secret_cred`` secret (stable across restarts, so a
login survives a redeploy). When inactive, the secret is a fresh
``secrets.token_urlsafe(32)`` generated per process -- fine for an
open-localhost dev session (it only needs to survive one process's
lifetime; no keyring lookup, so open mode still needs no keyring at all).

**Backend may be unavailable**: :func:`get_backend` is a FastAPI dependency
that swallows :class:`~ci_orchestrator.backends.errors.BackendError` (raised
when ``vmrun`` is missing on this host) and degrades to ``None`` so a route
never 500s. Tests override it via ``app.dependency_overrides[get_backend]``.
"""

from __future__ import annotations

import secrets
from dataclasses import dataclass
from pathlib import Path
from typing import TYPE_CHECKING

from fastapi import Depends, FastAPI, Request
from fastapi.responses import HTMLResponse
from fastapi.staticfiles import StaticFiles
from fastapi.templating import Jinja2Templates
from starlette.middleware.sessions import SessionMiddleware

from ci_orchestrator.backends import load_backend
from ci_orchestrator.backends.errors import BackendError
from ci_orchestrator.config import gitea_base_url
from ci_orchestrator.operations.cache import CacheStatus, cargo_cache_status
from ci_orchestrator.operations.executions import ExecutionsView, list_executions
from ci_orchestrator.operations.host import (
    DiskStatus,
    HostResources,
    RunnerStatus,
    VmwareModuleHealth,
    disk_status,
    host_resources,
    runner_status,
    vmware_module_health,
)
from ci_orchestrator.operations.inventory import (
    FileEntry,
    TemplateInfo,
    backups_by_template,
    list_artifacts,
    list_templates,
)
from ci_orchestrator.operations.services import ServiceState, service_states
from ci_orchestrator.operations.vms import LiveVm, list_live_vms
from ci_orchestrator.web import filters
from ci_orchestrator.web.auth import (
    _read_secret,
    configure_auth,
    gitea_token_for,
    refresh_gitea_token,
    require_user,
)
from ci_orchestrator.web.security import issue_csrf

if TYPE_CHECKING:  # pragma: no cover
    from ci_orchestrator.backends.protocol import VmBackend
    from ci_orchestrator.config import Config

_WEB_DIR = Path(__file__).resolve().parent

#: The two real ``act_runner`` systemd units on the Linux host (see
#: CLAUDE.md's "Phase C"). A module constant per the W4 plan section.
RUNNER_SERVICE_NAMES: list[str] = ["act-runner.service", "act-runner-rust.service"]

#: Disk-usage health thresholds for the status header (A3): above the
#: "critical" percentage -> red; above "elevated" -> amber; else green.
_DISK_PCT_CRITICAL = 85.0
_DISK_PCT_ELEVATED = 70.0

#: CPU / memory usage thresholds for the two Stage-E resource tiles, same
#: red/amber/green shape as the disk thresholds above.
_CPU_PCT_CRITICAL = 90.0
_CPU_PCT_ELEVATED = 75.0
_MEM_PCT_CRITICAL = 90.0
_MEM_PCT_ELEVATED = 75.0

#: CSS class used by ``partials/status_header.html`` for each health level.
#: Health must never be color-only -- the template always pairs this class
#: with a short text label (see :class:`HealthSummary`).
_TILE_OK = "tile-ok"
_TILE_WARN = "tile-warn"
_TILE_BAD = "tile-bad"
_TILE_UNKNOWN = "tile-unknown"


# ─────────────────────────────────────────────────────────────── dependency


def get_backend(request: Request) -> VmBackend | None:
    """FastAPI dependency: the VM backend for this request, or ``None``.

    Catches :class:`BackendError` (e.g. ``vmrun`` missing on this host --
    ``BackendNotAvailable``) so routes degrade to an empty/"unavailable"
    state instead of raising. Overridable in tests via
    ``app.dependency_overrides[get_backend]`` with a fake backend.
    """
    config: Config = request.app.state.config
    try:
        return load_backend(config)
    except BackendError:
        return None


# ──────────────────────────────────────────────────────────── data fetchers


def _fetch_live_vms(backend: VmBackend | None, config: Config) -> list[LiveVm]:
    """Render-ready VM list. Degrades to ``[]`` on any backend hiccup."""
    if backend is None:
        return []
    try:
        return list_live_vms(backend, config)
    except Exception:  # a route must never 500 on a backend hiccup
        return []


def _fetch_cache_status(config: Config) -> CacheStatus | None:
    """Cargo cache snapshot, or ``None`` if it could not be read."""
    try:
        return cargo_cache_status(config)
    except Exception:  # degrade, never 500
        return None


def _pat_fallback(config: Config) -> str | None:
    """The optional keyring PAT for Gitea, or ``None`` when not configured.

    Covers the cases the login token cannot: a process restart (which empties
    the server-side token map while sessions stay valid) and a non-admin
    operator. A missing/unreadable keyring entry is not an error -- the card
    degrades to listing VMs.
    """
    cred = config.web.gitea_token_cred
    if not cred:
        return None
    try:
        return _read_secret(cred)
    except Exception:  # degrade, never 500 over an optional secret
        return None


async def _fetch_executions(
    request: Request, config: Config, vms: list[LiveVm]
) -> ExecutionsView:
    """Recent Gitea runs with ``vms`` attached. Never raises.

    Token precedence: the access token from this operator's own OIDC login
    (held server-side, never in the cookie -- see ``web/auth.py``), then the
    optional keyring PAT. A ``401`` triggers exactly one refresh attempt
    before the card falls back to its degraded state.
    """
    username = request.session.get("user")
    stored = gitea_token_for(request.app, username)
    if stored is not None and stored.expired and username:
        stored = await refresh_gitea_token(request, str(username)) or stored
    token = stored.access_token if stored is not None else _pat_fallback(config)

    try:
        view = list_executions(config, vms, token=token)
    except Exception:  # the operations layer already degrades; belt and braces
        return ExecutionsView(
            runs=[], unmatched_vms=vms, source="none", error="Executions unavailable."
        )

    if view.auth_failed and stored is not None and username:
        refreshed = await refresh_gitea_token(request, str(username))
        if refreshed is not None:
            try:
                return list_executions(config, vms, token=refreshed.access_token)
            except Exception:
                return view
    return view


def _fetch_disk_status(config: Config) -> DiskStatus | None:
    """Disk usage for ``config.paths.root``, or ``None`` if unreadable."""
    try:
        return disk_status(config.paths.root)
    except Exception:  # degrade, never 500
        return None


def _fetch_runner_status() -> list[RunnerStatus]:
    """Runner service statuses, or ``[]`` if the query could not run at all."""
    try:
        return runner_status(RUNNER_SERVICE_NAMES)
    except Exception:  # degrade, never 500
        return []


def _fetch_resources() -> HostResources | None:
    """Host CPU/memory/uptime/load snapshot (B1), or ``None`` if unreadable.

    ``host_resources`` itself already degrades to an all-zero snapshot on
    ``OSError`` (never raises) -- this ``except Exception`` is a defensive
    second layer so a route can never 500 even on a genuinely unexpected
    failure.
    """
    try:
        return host_resources()
    except Exception:  # degrade, never 500
        return None


def _fetch_vmware_health() -> VmwareModuleHealth | None:
    """``vmmon``/``vmnet``/``vmware-host.service`` health (B2), or ``None`` if unreadable."""
    try:
        return vmware_module_health()
    except Exception:  # degrade, never 500
        return None


def _fetch_templates(config: Config) -> list[TemplateInfo] | None:
    """Template inventory rows (B3), or ``None`` if the scan itself failed.

    Distinct from an empty list (``[]``, a templates dir with nothing in
    it) -- ``None`` triggers the "unavailable" notice in ``templates.html``,
    ``[]`` triggers its plain "no templates found" state.
    """
    try:
        return list_templates(config)
    except Exception:  # degrade, never 500
        return None


def _fetch_artifacts(config: Config) -> list[FileEntry] | None:
    """Artifact listing (B4), or ``None`` if the scan itself failed."""
    try:
        return list_artifacts(config)
    except Exception:  # degrade, never 500
        return None


def _fetch_service_states(config: Config) -> list[ServiceState]:
    """Managed systemd unit states (Stage D), or ``[]`` if the query could not run at all.

    ``service_states`` itself already degrades a single unit to
    ``active=False, sub_state="unknown"`` on a ``systemctl`` hiccup (see
    ``operations/host.py::runner_status``) -- this ``except Exception`` is a
    defensive second layer, matching every other ``_fetch_*`` wrapper here.
    """
    try:
        return service_states(config.web.managed_services)
    except Exception:  # degrade, never 500
        return []


# ────────────────────────────────────────────────────────── status header


@dataclass(frozen=True)
class HealthSummary:
    """Render-ready values for the status-header tiles (A3).

    Every field is derived purely from data the dashboard route already
    fetched -- :func:`_health_summary` makes no new call of its own. Each
    tile carries both a CSS class (``*_class``) and a short text
    ``*_label`` so health never reads from color alone.
    """

    vm_count: int
    orphan_count: int
    vm_class: str
    vm_label: str

    runners_active: int
    runners_total: int
    runner_class: str
    runner_label: str

    cpu_pct: float | None
    ncpu: int
    cpu_class: str
    cpu_label: str

    mem_used_gb: float | None
    mem_total_gb: float | None
    mem_pct: float | None
    mem_class: str
    mem_label: str

    disk_used_pct: float | None
    disk_free_gb: float | None
    disk_total_gb: float | None
    disk_class: str
    disk_label: str

    cache_bytes: int | None
    cache_entries: int | None
    cache_class: str
    cache_label: str


def _threshold_level(pct: float, *, elevated: float, critical: float) -> tuple[str, str]:
    """Map a usage percentage to a ``(tile-class, label)`` on the shared red/amber/green scale."""
    if pct > critical:
        return _TILE_BAD, "critical"
    if pct > elevated:
        return _TILE_WARN, "elevated"
    return _TILE_OK, "ok"


def _health_summary(
    vms: list[LiveVm],
    *,
    backend_available: bool,
    cache: CacheStatus | None,
    disk: DiskStatus | None,
    runners: list[RunnerStatus],
    resources: HostResources | None = None,
) -> HealthSummary:
    """Derive the A3 status-header tiles from already-fetched dashboard context.

    Thresholds (see the A3 brief): disk ``used_pct`` > 85 -> red, > 70 ->
    amber, else green; any runner not active -> red; an orphaned VM present
    -> amber. A missing data source (backend down, disk/cache unreadable)
    renders as a neutral "unknown" tile rather than false-green.
    """
    orphan_count = sum(1 for vm in vms if vm.is_orphan)
    if not backend_available:
        vm_class, vm_label = _TILE_UNKNOWN, "unknown"
    elif orphan_count > 0:
        vm_class, vm_label = _TILE_WARN, "orphaned"
    else:
        vm_class, vm_label = _TILE_OK, "ok"

    runners_total = len(runners)
    runners_active = sum(1 for r in runners if r.active)
    if runners_total == 0:
        runner_class, runner_label = _TILE_UNKNOWN, "unknown"
    elif runners_active < runners_total:
        runner_class, runner_label = _TILE_BAD, "down"
    else:
        runner_class, runner_label = _TILE_OK, "ok"

    if resources is None:
        cpu_pct, ncpu = None, 0
        cpu_class, cpu_label = _TILE_UNKNOWN, "unknown"
        mem_used_gb = mem_total_gb = mem_pct = None
        mem_class, mem_label = _TILE_UNKNOWN, "unknown"
    else:
        cpu_pct, ncpu = resources.cpu_pct, resources.ncpu
        cpu_class, cpu_label = _threshold_level(
            cpu_pct, elevated=_CPU_PCT_ELEVATED, critical=_CPU_PCT_CRITICAL
        )
        mem_used_gb, mem_total_gb = resources.mem_used_gb, resources.mem_total_gb
        mem_pct = (mem_used_gb / mem_total_gb * 100.0) if mem_total_gb > 0 else 0.0
        mem_class, mem_label = _threshold_level(
            mem_pct, elevated=_MEM_PCT_ELEVATED, critical=_MEM_PCT_CRITICAL
        )

    disk_used_pct = disk.used_pct if disk is not None else None
    disk_free_gb = disk.free_gb if disk is not None else None
    disk_total_gb = disk.total_gb if disk is not None else None
    if disk_used_pct is None:
        disk_class, disk_label = _TILE_UNKNOWN, "unknown"
    else:
        disk_class, disk_label = _threshold_level(
            disk_used_pct, elevated=_DISK_PCT_ELEVATED, critical=_DISK_PCT_CRITICAL
        )

    cache_bytes = cache.total_bytes if cache is not None else None
    cache_entries = cache.entry_count if cache is not None else None
    cache_class, cache_label = (
        (_TILE_OK, "ok") if cache is not None else (_TILE_UNKNOWN, "unknown")
    )

    return HealthSummary(
        vm_count=len(vms),
        orphan_count=orphan_count,
        vm_class=vm_class,
        vm_label=vm_label,
        runners_active=runners_active,
        runners_total=runners_total,
        runner_class=runner_class,
        runner_label=runner_label,
        cpu_pct=cpu_pct,
        ncpu=ncpu,
        cpu_class=cpu_class,
        cpu_label=cpu_label,
        mem_used_gb=mem_used_gb,
        mem_total_gb=mem_total_gb,
        mem_pct=mem_pct,
        mem_class=mem_class,
        mem_label=mem_label,
        disk_used_pct=disk_used_pct,
        disk_free_gb=disk_free_gb,
        disk_total_gb=disk_total_gb,
        disk_class=disk_class,
        disk_label=disk_label,
        cache_bytes=cache_bytes,
        cache_entries=cache_entries,
        cache_class=cache_class,
        cache_label=cache_label,
    )


# ────────────────────────────────────────────────────────────────── routes


def healthz() -> dict[str, str]:
    """Liveness probe for systemd/monitor. Always ``200``."""
    return {"status": "ok"}


async def dashboard(
    request: Request, backend: VmBackend | None = Depends(get_backend)
) -> HTMLResponse:
    """Full dashboard: executions, cache, disk, runner, and Stage-B info cards."""
    config: Config = request.app.state.config
    templates: Jinja2Templates = request.app.state.templates
    vms = _fetch_live_vms(backend, config)
    backend_available = backend is not None
    cache = _fetch_cache_status(config)
    disk = _fetch_disk_status(config)
    runners = _fetch_runner_status()
    resources = _fetch_resources()
    template_rows = _fetch_templates(config)
    context = {
        "vms": vms,
        "executions": await _fetch_executions(request, config, vms),
        "backend_available": backend_available,
        "cache": cache,
        "disk": disk,
        "runners": runners,
        "resources": resources,
        "vmware": _fetch_vmware_health(),
        "templates": template_rows,
        "template_backups": backups_by_template(config),
        "artifacts": _fetch_artifacts(config),
        "services": _fetch_service_states(config),
        "health": _health_summary(
            vms,
            backend_available=backend_available,
            cache=cache,
            disk=disk,
            runners=runners,
            resources=resources,
        ),
        "csrf_token": issue_csrf(request),
    }
    return templates.TemplateResponse(request, "dashboard.html", context)


async def executions_partial(
    request: Request, backend: VmBackend | None = Depends(get_backend)
) -> HTMLResponse:
    """``GET /executions`` -- polled every 5s by the dashboard via htmx.

    Replaces the old ``GET /vms`` card: run-centric, with the live VMs grouped
    under the run that spawned them. The runs come from a 10s-TTL cache
    (``operations.executions``) while the VM data is read fresh on every poll,
    so this cadence costs Gitea at most one API call per TTL window.
    """
    config: Config = request.app.state.config
    templates: Jinja2Templates = request.app.state.templates
    vms = _fetch_live_vms(backend, config)
    context = {
        "vms": vms,
        "executions": await _fetch_executions(request, config, vms),
        "backend_available": backend is not None,
        "csrf_token": issue_csrf(request),
    }
    return templates.TemplateResponse(request, "partials/executions.html", context)


def cache_partial(request: Request) -> HTMLResponse:
    """``GET /cache`` -- cargo cache status partial."""
    config: Config = request.app.state.config
    templates: Jinja2Templates = request.app.state.templates
    context = {"cache": _fetch_cache_status(config), "csrf_token": issue_csrf(request)}
    return templates.TemplateResponse(request, "partials/cache.html", context)


def runners_partial(request: Request) -> HTMLResponse:
    """``GET /runners`` -- runner status partial (Stage E: disk/backup moved out).

    The disk card was folded into the status-header Disk tile and the
    standalone template-backup card into a per-row Templates action, so this
    read-only poll now carries only the runner statuses.
    """
    templates: Jinja2Templates = request.app.state.templates
    context = {"runners": _fetch_runner_status()}
    return templates.TemplateResponse(request, "partials/runners.html", context)


def host_partial(request: Request) -> HTMLResponse:
    """``GET /host`` (B5) -- host resources (CPU/RAM/load) partial, polled every 5s."""
    templates: Jinja2Templates = request.app.state.templates
    context = {"resources": _fetch_resources()}
    return templates.TemplateResponse(request, "partials/host.html", context)


def vmware_partial(request: Request) -> HTMLResponse:
    """``GET /vmware`` (B5) -- polled every 15s by the dashboard via htmx."""
    templates: Jinja2Templates = request.app.state.templates
    context = {"vmware": _fetch_vmware_health()}
    return templates.TemplateResponse(request, "partials/vmware.html", context)


def templates_partial(request: Request) -> HTMLResponse:
    """``GET /templates`` (B5) -- polled every 30s by the dashboard via htmx.

    Stage E: now carries the per-template backups map (for each row's
    expandable backup list) and a fresh ``csrf_token`` (the per-row Backup
    button is a mutating form, so this poll is no longer purely read-only).
    """
    config: Config = request.app.state.config
    templates: Jinja2Templates = request.app.state.templates
    context = {
        "templates": _fetch_templates(config),
        "template_backups": backups_by_template(config),
        "csrf_token": issue_csrf(request),
    }
    return templates.TemplateResponse(request, "partials/templates.html", context)


def files_partial(request: Request) -> HTMLResponse:
    """``GET /files`` (B5) -- polled every 30s by the dashboard via htmx.

    Stage E: the backups listing moved under the Templates panel (per-row
    expandable), so this poll carries only the artifacts list plus a fresh
    ``csrf_token`` for the per-row Delete button (now a mutating form).
    """
    config: Config = request.app.state.config
    templates: Jinja2Templates = request.app.state.templates
    context = {
        "artifacts": _fetch_artifacts(config),
        "csrf_token": issue_csrf(request),
    }
    return templates.TemplateResponse(request, "partials/files.html", context)


def services_partial(request: Request) -> HTMLResponse:
    """``GET /services`` (Stage D) -- polled every 15s by the dashboard via htmx.

    Needs a fresh ``csrf_token`` on every render (unlike the read-only
    ``host``/``vmware``/``runners`` cards) -- the panel's start/stop/restart
    forms are mutating, like the per-row forms in
    ``executions_partial``/``templates_partial``/``files_partial``.
    """
    config: Config = request.app.state.config
    templates: Jinja2Templates = request.app.state.templates
    context = {
        "services": _fetch_service_states(config),
        "csrf_token": issue_csrf(request),
    }
    return templates.TemplateResponse(request, "partials/services.html", context)


# ─────────────────────────────────────────────────────────────────── factory


def create_app(config: Config) -> FastAPI:
    """Build the dashboard app: W4 read-only routes + W5 auth + W6 mutating actions.

    Binding to a concrete address (always ``127.0.0.1`` by default -- never
    ``0.0.0.0``) is ``commands/web.py``'s job, not this factory's.

    All data routes are guarded by ``require_user`` (W5): when
    ``config.web.oidc is None`` that dependency is a no-op (open mode, W4
    parity); when set, unauthenticated requests redirect to ``/login``.
    ``/healthz`` and ``/static`` stay unprotected either way. The mutating
    routes (W6, ``web/routes.py``) are additionally behind CSRF -- see that
    module's docstring for the full safety contract.
    """
    # Deferred import: avoids a module-level import cycle (web.routes imports
    # get_backend from this module).
    from ci_orchestrator.web.routes import (
        encode_artifact_id,
        encode_vmx_id,
        register_action_routes,
    )

    app = FastAPI(title="ci-orchestrator web")
    app.state.config = config
    app.state.templates = Jinja2Templates(directory=str(_WEB_DIR / "templates"))
    filters.register(app.state.templates)
    app.state.templates.env.filters["vmx_id"] = encode_vmx_id
    app.state.templates.env.filters["artifact_id"] = encode_artifact_id
    # Stage C2: a Jinja *global* (not per-route context) so every template
    # that extends base.html -- dashboard, audit, the action-page wrapper --
    # picks up the nav's "Gitea ↗" link without each route builder having to
    # remember to thread it through.
    app.state.templates.env.globals["gitea_url"] = gitea_base_url(config.web)

    # Cache-busting for the stylesheet: StaticFiles sends only ETag/
    # Last-Modified (no Cache-Control), so a plain browser reload can serve a
    # stale app.css after a redeploy. Stamping the <link> with the file's
    # mtime -- re-read per render -- means every deploy changes the URL, so a
    # normal navigation refetches the CSS with no hard-refresh needed.
    static_css = _WEB_DIR / "static" / "app.css"

    def _asset_version() -> str:
        try:
            return str(int(static_css.stat().st_mtime))
        except OSError:
            return "0"

    app.state.templates.env.globals["asset_version"] = _asset_version
    app.mount("/static", StaticFiles(directory=str(_WEB_DIR / "static")), name="static")

    app.get("/healthz")(healthz)
    app.get("/", response_class=HTMLResponse, dependencies=[Depends(require_user)])(dashboard)
    app.get(
        "/executions", response_class=HTMLResponse, dependencies=[Depends(require_user)]
    )(executions_partial)
    app.get("/cache", response_class=HTMLResponse, dependencies=[Depends(require_user)])(
        cache_partial
    )
    app.get("/host", response_class=HTMLResponse, dependencies=[Depends(require_user)])(
        host_partial
    )
    app.get("/runners", response_class=HTMLResponse, dependencies=[Depends(require_user)])(
        runners_partial
    )
    app.get("/vmware", response_class=HTMLResponse, dependencies=[Depends(require_user)])(
        vmware_partial
    )
    app.get("/templates", response_class=HTMLResponse, dependencies=[Depends(require_user)])(
        templates_partial
    )
    app.get("/files", response_class=HTMLResponse, dependencies=[Depends(require_user)])(
        files_partial
    )
    app.get("/services", response_class=HTMLResponse, dependencies=[Depends(require_user)])(
        services_partial
    )
    register_action_routes(app)

    # SessionMiddleware is ALWAYS installed (W6) -- CSRF needs
    # ``request.session`` in both auth modes, not just when OIDC is active.
    # Added here, before the app is ever handed to a server -- Starlette
    # builds the middleware stack lazily on first request, so this is safe
    # regardless of where the route registrations above sit.
    #
    # Active OIDC: persistent, keyring-sourced secret so a login survives a
    # process restart. Inactive: a fresh per-process secret -- open mode
    # still needs no keyring at all, and the session only needs to outlive
    # this one process (there is nothing sensitive in it besides a CSRF
    # token when auth is off).
    session_secret = (
        _read_secret(config.web.session_secret_cred)
        if config.web.oidc is not None
        else secrets.token_urlsafe(32)
    )
    app.add_middleware(
        SessionMiddleware,
        secret_key=session_secret,
        same_site="lax",
        https_only=False,
    )

    configure_auth(app, config)

    return app


__all__ = [
    "RUNNER_SERVICE_NAMES",
    "HealthSummary",
    "create_app",
    "dashboard",
    "executions_partial",
    "files_partial",
    "get_backend",
    "healthz",
    "host_partial",
    "runners_partial",
    "services_partial",
    "templates_partial",
    "vmware_partial",
]