feat(web): mutating actions with confirm gate, CSRF, audit (W6)
Adds the four destructive routes (vm.remove, vm.cleanup, cache.purge, host.backup) plus a read-only /audit view, all behind require_user and a synchronizer-token CSRF dependency (web/security.py). Every mutating POST is a two-step confirm gate: a first POST always dry-runs the underlying operations/ call and renders a confirm partial; only a stage=execute POST performs the real action. Both stages write one audit entry via operations/audit.py. VM remove additionally validates that the target VMX resolves inside config.paths.build_vms before touching anything (400 otherwise), and vm.remove/vm.cleanup refuse to run (409) when a fresh vm-start.lock is present, guarding against racing a live job. SessionMiddleware is now installed unconditionally in create_app (was previously only added when OIDC auth was active) since CSRF needs request.session in both auth modes; the session secret stays keyring-sourced when OIDC is active and falls back to a per-process ephemeral secret in open mode. Adds python-multipart to the [web] extra (required by FastAPI/Starlette form parsing, including plain url-encoded forms, in the installed starlette version).
This commit is contained in:
@@ -35,6 +35,7 @@ web = [
|
|||||||
"authlib>=1.3",
|
"authlib>=1.3",
|
||||||
"itsdangerous>=2.2",
|
"itsdangerous>=2.2",
|
||||||
"httpx>=0.27",
|
"httpx>=0.27",
|
||||||
|
"python-multipart>=0.0.9",
|
||||||
]
|
]
|
||||||
|
|
||||||
[project.scripts]
|
[project.scripts]
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
"""FastAPI application factory for the web dashboard (W4 read-only + W5 auth).
|
"""FastAPI application factory for the web dashboard (W4 read-only + W5 auth + W6 actions).
|
||||||
|
|
||||||
**In-process only**: routes call ``operations/`` + ``backends.load_backend``
|
**In-process only**: routes call ``operations/`` + ``backends.load_backend``
|
||||||
directly and render Jinja2 templates. They never shell out to the CLI (see
|
directly and render Jinja2 templates. They never shell out to the CLI (see
|
||||||
@@ -11,6 +11,16 @@ localhost dev), ``require_user`` is a no-op and these routes behave exactly
|
|||||||
as in W4. ``create_app`` itself never chooses the bind address, that is
|
as in W4. ``create_app`` itself never chooses the bind address, that is
|
||||||
``commands/web.py``'s job.
|
``commands/web.py``'s job.
|
||||||
|
|
||||||
|
**Sessions are ALWAYS on (W6)**: ``SessionMiddleware`` is installed
|
||||||
|
unconditionally, not just when OIDC is active -- CSRF (``web/security.py``)
|
||||||
|
needs ``request.session`` in both auth modes. When OIDC is active the
|
||||||
|
session is signed with the keyring-sourced, persistent
|
||||||
|
``config.web.session_secret_cred`` secret (stable across restarts, so a
|
||||||
|
login survives a redeploy). When inactive, the secret is a fresh
|
||||||
|
``secrets.token_urlsafe(32)`` generated per process -- fine for an
|
||||||
|
open-localhost dev session (it only needs to survive one process's
|
||||||
|
lifetime; no keyring lookup, so open mode still needs no keyring at all).
|
||||||
|
|
||||||
**Backend may be unavailable**: :func:`get_backend` is a FastAPI dependency
|
**Backend may be unavailable**: :func:`get_backend` is a FastAPI dependency
|
||||||
that swallows :class:`~ci_orchestrator.backends.errors.BackendError` (raised
|
that swallows :class:`~ci_orchestrator.backends.errors.BackendError` (raised
|
||||||
when ``vmrun`` is missing on this host) and degrades to ``None`` so a route
|
when ``vmrun`` is missing on this host) and degrades to ``None`` so a route
|
||||||
@@ -19,6 +29,7 @@ never 500s. Tests override it via ``app.dependency_overrides[get_backend]``.
|
|||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import secrets
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import TYPE_CHECKING
|
from typing import TYPE_CHECKING
|
||||||
|
|
||||||
@@ -26,13 +37,15 @@ from fastapi import Depends, FastAPI, Request
|
|||||||
from fastapi.responses import HTMLResponse
|
from fastapi.responses import HTMLResponse
|
||||||
from fastapi.staticfiles import StaticFiles
|
from fastapi.staticfiles import StaticFiles
|
||||||
from fastapi.templating import Jinja2Templates
|
from fastapi.templating import Jinja2Templates
|
||||||
|
from starlette.middleware.sessions import SessionMiddleware
|
||||||
|
|
||||||
from ci_orchestrator.backends import load_backend
|
from ci_orchestrator.backends import load_backend
|
||||||
from ci_orchestrator.backends.errors import BackendError
|
from ci_orchestrator.backends.errors import BackendError
|
||||||
from ci_orchestrator.operations.cache import CacheStatus, cargo_cache_status
|
from ci_orchestrator.operations.cache import CacheStatus, cargo_cache_status
|
||||||
from ci_orchestrator.operations.host import DiskStatus, RunnerStatus, disk_status, runner_status
|
from ci_orchestrator.operations.host import DiskStatus, RunnerStatus, disk_status, runner_status
|
||||||
from ci_orchestrator.operations.vms import LiveVm, list_live_vms
|
from ci_orchestrator.operations.vms import LiveVm, list_live_vms
|
||||||
from ci_orchestrator.web.auth import configure_auth, require_user
|
from ci_orchestrator.web.auth import _read_secret, configure_auth, require_user
|
||||||
|
from ci_orchestrator.web.security import issue_csrf
|
||||||
|
|
||||||
if TYPE_CHECKING: # pragma: no cover
|
if TYPE_CHECKING: # pragma: no cover
|
||||||
from ci_orchestrator.backends.protocol import VmBackend
|
from ci_orchestrator.backends.protocol import VmBackend
|
||||||
@@ -120,6 +133,7 @@ def dashboard(
|
|||||||
"cache": _fetch_cache_status(config),
|
"cache": _fetch_cache_status(config),
|
||||||
"disk": _fetch_disk_status(config),
|
"disk": _fetch_disk_status(config),
|
||||||
"runners": _fetch_runner_status(),
|
"runners": _fetch_runner_status(),
|
||||||
|
"csrf_token": issue_csrf(request),
|
||||||
}
|
}
|
||||||
return templates.TemplateResponse(request, "dashboard.html", context)
|
return templates.TemplateResponse(request, "dashboard.html", context)
|
||||||
|
|
||||||
@@ -133,6 +147,7 @@ def vms_partial(
|
|||||||
context = {
|
context = {
|
||||||
"vms": _fetch_live_vms(backend, config),
|
"vms": _fetch_live_vms(backend, config),
|
||||||
"backend_available": backend is not None,
|
"backend_available": backend is not None,
|
||||||
|
"csrf_token": issue_csrf(request),
|
||||||
}
|
}
|
||||||
return templates.TemplateResponse(request, "partials/vms.html", context)
|
return templates.TemplateResponse(request, "partials/vms.html", context)
|
||||||
|
|
||||||
@@ -141,7 +156,7 @@ def cache_partial(request: Request) -> HTMLResponse:
|
|||||||
"""``GET /cache`` -- cargo cache status partial."""
|
"""``GET /cache`` -- cargo cache status partial."""
|
||||||
config: Config = request.app.state.config
|
config: Config = request.app.state.config
|
||||||
templates: Jinja2Templates = request.app.state.templates
|
templates: Jinja2Templates = request.app.state.templates
|
||||||
context = {"cache": _fetch_cache_status(config)}
|
context = {"cache": _fetch_cache_status(config), "csrf_token": issue_csrf(request)}
|
||||||
return templates.TemplateResponse(request, "partials/cache.html", context)
|
return templates.TemplateResponse(request, "partials/cache.html", context)
|
||||||
|
|
||||||
|
|
||||||
@@ -152,6 +167,7 @@ def host_partial(request: Request) -> HTMLResponse:
|
|||||||
context = {
|
context = {
|
||||||
"disk": _fetch_disk_status(config),
|
"disk": _fetch_disk_status(config),
|
||||||
"runners": _fetch_runner_status(),
|
"runners": _fetch_runner_status(),
|
||||||
|
"csrf_token": issue_csrf(request),
|
||||||
}
|
}
|
||||||
return templates.TemplateResponse(request, "partials/host.html", context)
|
return templates.TemplateResponse(request, "partials/host.html", context)
|
||||||
|
|
||||||
@@ -160,20 +176,26 @@ def host_partial(request: Request) -> HTMLResponse:
|
|||||||
|
|
||||||
|
|
||||||
def create_app(config: Config) -> FastAPI:
|
def create_app(config: Config) -> FastAPI:
|
||||||
"""Build the dashboard app.
|
"""Build the dashboard app: W4 read-only routes + W5 auth + W6 mutating actions.
|
||||||
|
|
||||||
Read-only routes only (mutating actions are W6). Binding to a concrete
|
Binding to a concrete address (always ``127.0.0.1`` by default -- never
|
||||||
address (always ``127.0.0.1`` by default -- never ``0.0.0.0``) is
|
``0.0.0.0``) is ``commands/web.py``'s job, not this factory's.
|
||||||
``commands/web.py``'s job, not this factory's.
|
|
||||||
|
|
||||||
All data routes are guarded by ``require_user`` (W5): when
|
All data routes are guarded by ``require_user`` (W5): when
|
||||||
``config.web.oidc is None`` that dependency is a no-op (open mode, W4
|
``config.web.oidc is None`` that dependency is a no-op (open mode, W4
|
||||||
parity); when set, unauthenticated requests redirect to ``/login``.
|
parity); when set, unauthenticated requests redirect to ``/login``.
|
||||||
``/healthz`` and ``/static`` stay unprotected either way.
|
``/healthz`` and ``/static`` stay unprotected either way. The mutating
|
||||||
|
routes (W6, ``web/routes.py``) are additionally behind CSRF -- see that
|
||||||
|
module's docstring for the full safety contract.
|
||||||
"""
|
"""
|
||||||
|
# Deferred import: avoids a module-level import cycle (web.routes imports
|
||||||
|
# get_backend from this module).
|
||||||
|
from ci_orchestrator.web.routes import encode_vmx_id, register_action_routes
|
||||||
|
|
||||||
app = FastAPI(title="ci-orchestrator web")
|
app = FastAPI(title="ci-orchestrator web")
|
||||||
app.state.config = config
|
app.state.config = config
|
||||||
app.state.templates = Jinja2Templates(directory=str(_WEB_DIR / "templates"))
|
app.state.templates = Jinja2Templates(directory=str(_WEB_DIR / "templates"))
|
||||||
|
app.state.templates.env.filters["vmx_id"] = encode_vmx_id
|
||||||
app.mount("/static", StaticFiles(directory=str(_WEB_DIR / "static")), name="static")
|
app.mount("/static", StaticFiles(directory=str(_WEB_DIR / "static")), name="static")
|
||||||
|
|
||||||
app.get("/healthz")(healthz)
|
app.get("/healthz")(healthz)
|
||||||
@@ -187,11 +209,31 @@ def create_app(config: Config) -> FastAPI:
|
|||||||
app.get("/host", response_class=HTMLResponse, dependencies=[Depends(require_user)])(
|
app.get("/host", response_class=HTMLResponse, dependencies=[Depends(require_user)])(
|
||||||
host_partial
|
host_partial
|
||||||
)
|
)
|
||||||
|
register_action_routes(app)
|
||||||
|
|
||||||
# SessionMiddleware (needed by the OAuth flow's state/nonce) is added
|
# SessionMiddleware is ALWAYS installed (W6) -- CSRF needs
|
||||||
# here, before the app is ever handed to a server -- Starlette builds
|
# ``request.session`` in both auth modes, not just when OIDC is active.
|
||||||
# the middleware stack lazily on first request, so this is safe
|
# Added here, before the app is ever handed to a server -- Starlette
|
||||||
|
# builds the middleware stack lazily on first request, so this is safe
|
||||||
# regardless of where the route registrations above sit.
|
# regardless of where the route registrations above sit.
|
||||||
|
#
|
||||||
|
# Active OIDC: persistent, keyring-sourced secret so a login survives a
|
||||||
|
# process restart. Inactive: a fresh per-process secret -- open mode
|
||||||
|
# still needs no keyring at all, and the session only needs to outlive
|
||||||
|
# this one process (there is nothing sensitive in it besides a CSRF
|
||||||
|
# token when auth is off).
|
||||||
|
session_secret = (
|
||||||
|
_read_secret(config.web.session_secret_cred)
|
||||||
|
if config.web.oidc is not None
|
||||||
|
else secrets.token_urlsafe(32)
|
||||||
|
)
|
||||||
|
app.add_middleware(
|
||||||
|
SessionMiddleware,
|
||||||
|
secret_key=session_secret,
|
||||||
|
same_site="lax",
|
||||||
|
https_only=False,
|
||||||
|
)
|
||||||
|
|
||||||
configure_auth(app, config)
|
configure_auth(app, config)
|
||||||
|
|
||||||
return app
|
return app
|
||||||
|
|||||||
@@ -3,11 +3,17 @@
|
|||||||
**Auth is ACTIVE iff ``config.web.oidc is not None``.** When inactive (the
|
**Auth is ACTIVE iff ``config.web.oidc is not None``.** When inactive (the
|
||||||
W4 default -- bare localhost dev), :func:`configure_auth` is a no-op and
|
W4 default -- bare localhost dev), :func:`configure_auth` is a no-op and
|
||||||
:func:`require_user` degrades to a no-op that returns ``"anonymous"``; this
|
:func:`require_user` degrades to a no-op that returns ``"anonymous"``; this
|
||||||
preserves the W4 open-localhost dashboard exactly. When active,
|
preserves the W4 open-localhost dashboard exactly. When active, an Authlib
|
||||||
``SessionMiddleware`` (itsdangerous-signed cookies) is installed, an Authlib
|
|
||||||
OAuth client is registered against Gitea's OIDC discovery document, and
|
OAuth client is registered against Gitea's OIDC discovery document, and
|
||||||
``/login`` / ``/auth/callback`` / ``/logout`` are wired in.
|
``/login`` / ``/auth/callback`` / ``/logout`` are wired in.
|
||||||
|
|
||||||
|
**``SessionMiddleware`` is always installed (W6)** -- by
|
||||||
|
:func:`~ci_orchestrator.web.app.create_app`, not here. CSRF protection
|
||||||
|
(``web/security.py``) needs ``request.session`` in *both* auth modes, so the
|
||||||
|
middleware can no longer be conditional on ``config.web.oidc``. This module
|
||||||
|
only ever registers the OAuth client + login routes, gated on
|
||||||
|
``config.web.oidc is not None`` as before.
|
||||||
|
|
||||||
**Secrets come from the OS keyring only** -- the session signing key
|
**Secrets come from the OS keyring only** -- the session signing key
|
||||||
(``config.web.session_secret_cred``) and the OIDC client secret
|
(``config.web.session_secret_cred``) and the OIDC client secret
|
||||||
(``config.web.oidc.client_secret_cred``) are keyring key *names* in
|
(``config.web.oidc.client_secret_cred``) are keyring key *names* in
|
||||||
@@ -29,7 +35,6 @@ from authlib.integrations.starlette_client import OAuth
|
|||||||
from fastapi import FastAPI, HTTPException, Request
|
from fastapi import FastAPI, HTTPException, Request
|
||||||
from fastapi.responses import RedirectResponse
|
from fastapi.responses import RedirectResponse
|
||||||
from starlette import status
|
from starlette import status
|
||||||
from starlette.middleware.sessions import SessionMiddleware
|
|
||||||
|
|
||||||
from ci_orchestrator.credentials import KeyringCredentialStore
|
from ci_orchestrator.credentials import KeyringCredentialStore
|
||||||
|
|
||||||
@@ -171,30 +176,18 @@ def configure_auth(app: FastAPI, config: Config) -> None:
|
|||||||
|
|
||||||
Inactive (``config.web.oidc is None``): no-op, open mode (W4 parity) --
|
Inactive (``config.web.oidc is None``): no-op, open mode (W4 parity) --
|
||||||
``require_user`` alone handles that case and the login routes are never
|
``require_user`` alone handles that case and the login routes are never
|
||||||
registered.
|
registered. ``SessionMiddleware`` is still present (added unconditionally
|
||||||
|
by ``create_app``), but nothing here needs it.
|
||||||
|
|
||||||
Active: adds ``SessionMiddleware`` *before* anything OAuth-related runs
|
Active: registers the OAuth client on ``app.state.oauth`` and registers
|
||||||
(Authlib needs the session for state/nonce), registers the OAuth client
|
``/login``, ``/auth/callback``, ``/logout``. ``SessionMiddleware`` is
|
||||||
on ``app.state.oauth``, and registers ``/login``, ``/auth/callback``,
|
guaranteed to already be installed by the time this runs (Authlib needs
|
||||||
``/logout``.
|
the session for state/nonce) -- ``create_app`` adds it before calling
|
||||||
|
this function.
|
||||||
"""
|
"""
|
||||||
if config.web.oidc is None:
|
if config.web.oidc is None:
|
||||||
return
|
return
|
||||||
|
|
||||||
# https_only=False: this host serves plain HTTP on the LAN (no TLS
|
|
||||||
# termination in front of it, see the web-UI plan's gotchas) -- forcing
|
|
||||||
# https_only=True here would make the browser silently drop the session
|
|
||||||
# cookie and break every login. same_site="lax" still blocks the cookie
|
|
||||||
# being sent on cross-site requests; the cookie is httponly by
|
|
||||||
# SessionMiddleware's own default (unconditional, not toggled by
|
|
||||||
# https_only).
|
|
||||||
app.add_middleware(
|
|
||||||
SessionMiddleware,
|
|
||||||
secret_key=_read_secret(config.web.session_secret_cred),
|
|
||||||
same_site="lax",
|
|
||||||
https_only=False,
|
|
||||||
)
|
|
||||||
|
|
||||||
app.state.oauth = register_oauth(config)
|
app.state.oauth = register_oauth(config)
|
||||||
|
|
||||||
app.add_api_route("/login", login, methods=["GET"])
|
app.add_api_route("/login", login, methods=["GET"])
|
||||||
|
|||||||
@@ -0,0 +1,439 @@
|
|||||||
|
"""Mutating web routes: VM remove/cleanup, cache purge, host backup, audit (W6).
|
||||||
|
|
||||||
|
**SECURITY-CRITICAL**: these routes destroy VMs, purge caches, and back up
|
||||||
|
templates. Every route in this module is registered behind
|
||||||
|
:func:`~ci_orchestrator.web.auth.require_user`. Every ``POST`` additionally
|
||||||
|
depends on :func:`~ci_orchestrator.web.security.require_csrf` and follows a
|
||||||
|
mandatory two-step confirm gate:
|
||||||
|
|
||||||
|
- A ``POST`` **without** ``stage=execute`` always runs the underlying
|
||||||
|
``operations/`` call with ``dry_run=True`` and renders a *confirm* partial
|
||||||
|
(``partials/confirm.html``) containing a fresh CSRF token and
|
||||||
|
``stage=execute`` -- there is no one-click destroy.
|
||||||
|
- A ``POST`` **with** ``stage=execute`` performs the real action.
|
||||||
|
|
||||||
|
Both stages append exactly one :class:`~ci_orchestrator.operations.audit.
|
||||||
|
AuditEntry`. Pure validation/security refusals (missing CSRF -> 403, VMX
|
||||||
|
outside ``config.paths.build_vms`` -> 400, a fresh ``vm-start.lock`` on
|
||||||
|
execute -> 409) short-circuit *before* any ``operations/`` call and are
|
||||||
|
never audited -- they are refusals to act, not actions.
|
||||||
|
|
||||||
|
In-process only: this module imports ``operations/`` directly and never
|
||||||
|
shells out to the CLI (see ``CLAUDE.md``).
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import base64
|
||||||
|
import binascii
|
||||||
|
from datetime import UTC, datetime, timedelta
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import TYPE_CHECKING
|
||||||
|
|
||||||
|
from fastapi import Depends, FastAPI, HTTPException, Request
|
||||||
|
from fastapi.responses import HTMLResponse
|
||||||
|
from starlette import status
|
||||||
|
|
||||||
|
from ci_orchestrator.operations.audit import AuditEntry, append_audit, read_audit
|
||||||
|
from ci_orchestrator.operations.cache import purge_cargo_cache
|
||||||
|
from ci_orchestrator.operations.host import backup_template
|
||||||
|
from ci_orchestrator.operations.vms import RemovalResult, cleanup_orphans, remove_vm
|
||||||
|
from ci_orchestrator.web.app import get_backend
|
||||||
|
from ci_orchestrator.web.auth import require_user
|
||||||
|
from ci_orchestrator.web.security import issue_csrf, require_csrf
|
||||||
|
|
||||||
|
if TYPE_CHECKING: # pragma: no cover
|
||||||
|
from fastapi.templating import Jinja2Templates
|
||||||
|
from starlette.datastructures import FormData
|
||||||
|
|
||||||
|
from ci_orchestrator.backends.protocol import VmBackend
|
||||||
|
from ci_orchestrator.config import Config
|
||||||
|
|
||||||
|
#: A ``vm-start.lock`` younger than this is treated as "a job may be
|
||||||
|
#: running" -- see the W6 plan's "best-effort job-race guard".
|
||||||
|
_LOCK_MAX_AGE_MINUTES = 30
|
||||||
|
_DEFAULT_CLEANUP_MAX_AGE_HOURS = 4
|
||||||
|
_STAGE_FIELD = "stage"
|
||||||
|
_STAGE_EXECUTE = "execute"
|
||||||
|
|
||||||
|
|
||||||
|
# ───────────────────────────────────────────────────────────── vmx_id codec
|
||||||
|
|
||||||
|
|
||||||
|
def encode_vmx_id(vmx: str | Path) -> str:
|
||||||
|
"""Urlsafe-base64 a VMX path for use as a ``/vms/{vmx_id}/remove`` path segment."""
|
||||||
|
return base64.urlsafe_b64encode(str(vmx).encode("utf-8")).decode("ascii").rstrip("=")
|
||||||
|
|
||||||
|
|
||||||
|
def _decode_vmx_id(vmx_id: str) -> Path | None:
|
||||||
|
"""Inverse of :func:`encode_vmx_id`. Returns ``None`` on any malformed input."""
|
||||||
|
padded = vmx_id + "=" * (-len(vmx_id) % 4)
|
||||||
|
try:
|
||||||
|
raw = base64.urlsafe_b64decode(padded)
|
||||||
|
except (binascii.Error, ValueError):
|
||||||
|
return None
|
||||||
|
try:
|
||||||
|
return Path(raw.decode("utf-8"))
|
||||||
|
except UnicodeDecodeError:
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _is_contained(path: Path, base: Path) -> bool:
|
||||||
|
"""True iff ``base`` is an ancestor of (resolved) ``path`` -- never equal.
|
||||||
|
|
||||||
|
Both sides are ``.resolve()``d before comparison so a symlink or ``..``
|
||||||
|
segment cannot smuggle a path outside ``base`` past this check. This is
|
||||||
|
the sole gate standing between an authenticated web user and deleting an
|
||||||
|
arbitrary host path -- see the W6 plan's "path containment" invariant.
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
resolved_base = base.resolve()
|
||||||
|
resolved_path = path.resolve()
|
||||||
|
except OSError:
|
||||||
|
return False
|
||||||
|
return resolved_base in resolved_path.parents
|
||||||
|
|
||||||
|
|
||||||
|
# ───────────────────────────────────────────────────────────────── helpers
|
||||||
|
|
||||||
|
|
||||||
|
def _form_str(form: FormData, key: str, default: str = "") -> str:
|
||||||
|
value = form.get(key, default)
|
||||||
|
return value if isinstance(value, str) else default
|
||||||
|
|
||||||
|
|
||||||
|
def _form_int(form: FormData, key: str, default: int) -> int:
|
||||||
|
try:
|
||||||
|
return int(_form_str(form, key, str(default)))
|
||||||
|
except ValueError:
|
||||||
|
return default
|
||||||
|
|
||||||
|
|
||||||
|
def _lock_path(config: Config) -> Path:
|
||||||
|
"""The job-lifecycle lock path this route treats as "a job may be running".
|
||||||
|
|
||||||
|
Mirrors the ``vm-start.lock`` convention used by the job orchestrator
|
||||||
|
(``config.paths.build_vms.parent / "vm-start.lock"``). Advisory only:
|
||||||
|
as of W6 the Python job path does not yet create this lock file, so this
|
||||||
|
guard is forward-looking (see the W6 plan's gotcha note).
|
||||||
|
"""
|
||||||
|
return config.paths.build_vms.parent / "vm-start.lock"
|
||||||
|
|
||||||
|
|
||||||
|
def _lock_is_fresh(lock_file: Path) -> bool:
|
||||||
|
try:
|
||||||
|
mtime = datetime.fromtimestamp(lock_file.stat().st_mtime, tz=UTC)
|
||||||
|
except OSError:
|
||||||
|
return False
|
||||||
|
return mtime >= datetime.now(tz=UTC) - timedelta(minutes=_LOCK_MAX_AGE_MINUTES)
|
||||||
|
|
||||||
|
|
||||||
|
def _audit(
|
||||||
|
config: Config, *, user: str, action: str, target: str, result: str, detail: str
|
||||||
|
) -> None:
|
||||||
|
entry = AuditEntry(
|
||||||
|
ts=datetime.now(tz=UTC).isoformat(),
|
||||||
|
user=user,
|
||||||
|
action=action,
|
||||||
|
target=target,
|
||||||
|
result=result,
|
||||||
|
detail=detail,
|
||||||
|
)
|
||||||
|
append_audit(config.web.audit_log, entry)
|
||||||
|
|
||||||
|
|
||||||
|
def _confirm_partial(
|
||||||
|
request: Request,
|
||||||
|
*,
|
||||||
|
action_label: str,
|
||||||
|
target: str,
|
||||||
|
summary: str,
|
||||||
|
action_url: str,
|
||||||
|
extra_fields: dict[str, str],
|
||||||
|
) -> HTMLResponse:
|
||||||
|
templates: Jinja2Templates = request.app.state.templates
|
||||||
|
context = {
|
||||||
|
"action_label": action_label,
|
||||||
|
"target": target,
|
||||||
|
"summary": summary,
|
||||||
|
"action_url": action_url,
|
||||||
|
"extra_fields": extra_fields,
|
||||||
|
"csrf_token": issue_csrf(request),
|
||||||
|
}
|
||||||
|
return templates.TemplateResponse(request, "partials/confirm.html", context)
|
||||||
|
|
||||||
|
|
||||||
|
def _result_partial(request: Request, *, action_label: str, ok: bool, detail: str) -> HTMLResponse:
|
||||||
|
templates: Jinja2Templates = request.app.state.templates
|
||||||
|
context = {"action_label": action_label, "ok": ok, "detail": detail}
|
||||||
|
return templates.TemplateResponse(request, "partials/result.html", context)
|
||||||
|
|
||||||
|
|
||||||
|
def _lock_conflict_partial(request: Request, *, action_label: str, target: str) -> HTMLResponse:
|
||||||
|
templates: Jinja2Templates = request.app.state.templates
|
||||||
|
context = {"action_label": action_label, "target": target}
|
||||||
|
return templates.TemplateResponse(
|
||||||
|
request, "partials/lock_conflict.html", context, status_code=status.HTTP_409_CONFLICT
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _removal_detail(result: RemovalResult) -> str:
|
||||||
|
if result.error is not None:
|
||||||
|
return f"error removing {result.target}: {result.error}"
|
||||||
|
return f"removed {result.target} (dir removed: {result.dir_removed})"
|
||||||
|
|
||||||
|
|
||||||
|
# ─────────────────────────────────────────────────────────── POST /vms/{id}/remove
|
||||||
|
|
||||||
|
|
||||||
|
async def vms_remove(
|
||||||
|
vmx_id: str,
|
||||||
|
request: Request,
|
||||||
|
user: str = Depends(require_user),
|
||||||
|
_csrf: None = Depends(require_csrf),
|
||||||
|
backend: VmBackend | None = Depends(get_backend),
|
||||||
|
) -> HTMLResponse:
|
||||||
|
"""Stop + permanently delete a single VM, behind the two-step confirm gate."""
|
||||||
|
config: Config = request.app.state.config
|
||||||
|
|
||||||
|
vmx = _decode_vmx_id(vmx_id)
|
||||||
|
if vmx is None or not _is_contained(vmx, config.paths.build_vms):
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=status.HTTP_400_BAD_REQUEST,
|
||||||
|
detail="vmx path is not contained in config.paths.build_vms",
|
||||||
|
)
|
||||||
|
target = str(vmx)
|
||||||
|
action_label = "Remove VM"
|
||||||
|
|
||||||
|
if backend is None:
|
||||||
|
detail = "Backend unavailable (vmrun not found on this host)."
|
||||||
|
_audit(config, user=user, action="vm.remove", target=target, result="error", detail=detail)
|
||||||
|
return _result_partial(request, action_label=action_label, ok=False, detail=detail)
|
||||||
|
|
||||||
|
form = await request.form()
|
||||||
|
execute = _form_str(form, _STAGE_FIELD) == _STAGE_EXECUTE
|
||||||
|
|
||||||
|
if execute:
|
||||||
|
conflict = _lock_path(config)
|
||||||
|
if conflict.is_file() and _lock_is_fresh(conflict):
|
||||||
|
return _lock_conflict_partial(request, action_label=action_label, target=target)
|
||||||
|
|
||||||
|
result = remove_vm(backend, vmx, dry_run=False)
|
||||||
|
ok = result.error is None
|
||||||
|
detail = _removal_detail(result)
|
||||||
|
_audit(
|
||||||
|
config,
|
||||||
|
user=user,
|
||||||
|
action="vm.remove",
|
||||||
|
target=target,
|
||||||
|
result="ok" if ok else "error",
|
||||||
|
detail=detail,
|
||||||
|
)
|
||||||
|
return _result_partial(request, action_label=action_label, ok=ok, detail=detail)
|
||||||
|
|
||||||
|
result = remove_vm(backend, vmx, dry_run=True)
|
||||||
|
detail = f"would remove VM at {target}"
|
||||||
|
_audit(config, user=user, action="vm.remove", target=target, result="dry-run", detail=detail)
|
||||||
|
return _confirm_partial(
|
||||||
|
request,
|
||||||
|
action_label=action_label,
|
||||||
|
target=target,
|
||||||
|
summary=detail,
|
||||||
|
action_url=f"/vms/{vmx_id}/remove",
|
||||||
|
extra_fields={},
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
# ───────────────────────────────────────────────────────────── POST /vms/cleanup
|
||||||
|
|
||||||
|
|
||||||
|
async def vms_cleanup(
|
||||||
|
request: Request,
|
||||||
|
user: str = Depends(require_user),
|
||||||
|
_csrf: None = Depends(require_csrf),
|
||||||
|
backend: VmBackend | None = Depends(get_backend),
|
||||||
|
) -> HTMLResponse:
|
||||||
|
"""Remove orphaned clone directories older than a threshold, confirm-gated."""
|
||||||
|
config: Config = request.app.state.config
|
||||||
|
action_label = "Cleanup orphaned VMs"
|
||||||
|
target = str(config.paths.build_vms)
|
||||||
|
lock_file = _lock_path(config)
|
||||||
|
|
||||||
|
form = await request.form()
|
||||||
|
execute = _form_str(form, _STAGE_FIELD) == _STAGE_EXECUTE
|
||||||
|
max_age_hours = _form_int(form, "max_age_hours", _DEFAULT_CLEANUP_MAX_AGE_HOURS)
|
||||||
|
|
||||||
|
if execute:
|
||||||
|
if lock_file.is_file() and _lock_is_fresh(lock_file):
|
||||||
|
return _lock_conflict_partial(request, action_label=action_label, target=target)
|
||||||
|
|
||||||
|
results = cleanup_orphans(
|
||||||
|
config.paths.build_vms,
|
||||||
|
max_age_hours,
|
||||||
|
backend,
|
||||||
|
dry_run=False,
|
||||||
|
lock_file=lock_file,
|
||||||
|
)
|
||||||
|
ok = all(r.error is None for r in results)
|
||||||
|
detail = f"removed {len(results)} orphan(s) older than {max_age_hours}h"
|
||||||
|
_audit(
|
||||||
|
config,
|
||||||
|
user=user,
|
||||||
|
action="vm.cleanup",
|
||||||
|
target=target,
|
||||||
|
result="ok" if ok else "error",
|
||||||
|
detail=detail,
|
||||||
|
)
|
||||||
|
return _result_partial(request, action_label=action_label, ok=ok, detail=detail)
|
||||||
|
|
||||||
|
results = cleanup_orphans(
|
||||||
|
config.paths.build_vms,
|
||||||
|
max_age_hours,
|
||||||
|
backend,
|
||||||
|
dry_run=True,
|
||||||
|
lock_file=lock_file,
|
||||||
|
)
|
||||||
|
detail = f"would remove {len(results)} orphan(s) older than {max_age_hours}h"
|
||||||
|
_audit(config, user=user, action="vm.cleanup", target=target, result="dry-run", detail=detail)
|
||||||
|
return _confirm_partial(
|
||||||
|
request,
|
||||||
|
action_label=action_label,
|
||||||
|
target=target,
|
||||||
|
summary=detail,
|
||||||
|
action_url="/vms/cleanup",
|
||||||
|
extra_fields={"max_age_hours": str(max_age_hours)},
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
# ───────────────────────────────────────────────────────────── POST /cache/purge
|
||||||
|
|
||||||
|
|
||||||
|
async def cache_purge(
|
||||||
|
request: Request,
|
||||||
|
user: str = Depends(require_user),
|
||||||
|
_csrf: None = Depends(require_csrf),
|
||||||
|
) -> HTMLResponse:
|
||||||
|
"""Purge cargo cache entries past retention, confirm-gated."""
|
||||||
|
config: Config = request.app.state.config
|
||||||
|
action_label = "Purge cargo cache"
|
||||||
|
target = str(config.cargo_cache.root)
|
||||||
|
|
||||||
|
form = await request.form()
|
||||||
|
execute = _form_str(form, _STAGE_FIELD) == _STAGE_EXECUTE
|
||||||
|
|
||||||
|
try:
|
||||||
|
result = purge_cargo_cache(config, dry_run=not execute)
|
||||||
|
except Exception as exc: # never 500 on an operational failure
|
||||||
|
detail = f"error: {exc}"
|
||||||
|
_audit(config, user=user, action="cache.purge", target=target, result="error", detail=detail)
|
||||||
|
return _result_partial(request, action_label=action_label, ok=False, detail=detail)
|
||||||
|
|
||||||
|
if execute:
|
||||||
|
detail = f"freed {result.freed_bytes} bytes across {result.removed_entries} entrie(s)"
|
||||||
|
_audit(config, user=user, action="cache.purge", target=target, result="ok", detail=detail)
|
||||||
|
return _result_partial(request, action_label=action_label, ok=True, detail=detail)
|
||||||
|
|
||||||
|
detail = f"would remove {result.removed_entries} entrie(s), freeing {result.freed_bytes} bytes"
|
||||||
|
_audit(config, user=user, action="cache.purge", target=target, result="dry-run", detail=detail)
|
||||||
|
return _confirm_partial(
|
||||||
|
request,
|
||||||
|
action_label=action_label,
|
||||||
|
target=target,
|
||||||
|
summary=detail,
|
||||||
|
action_url="/cache/purge",
|
||||||
|
extra_fields={},
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
# ───────────────────────────────────────────────────────────── POST /host/backup
|
||||||
|
|
||||||
|
|
||||||
|
async def host_backup(
|
||||||
|
request: Request,
|
||||||
|
user: str = Depends(require_user),
|
||||||
|
_csrf: None = Depends(require_csrf),
|
||||||
|
) -> HTMLResponse:
|
||||||
|
"""Archive a template directory to a timestamped 7z file, confirm-gated."""
|
||||||
|
config: Config = request.app.state.config
|
||||||
|
action_label = "Backup template"
|
||||||
|
|
||||||
|
form = await request.form()
|
||||||
|
template = _form_str(form, "template")
|
||||||
|
if not template:
|
||||||
|
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="template is required")
|
||||||
|
execute = _form_str(form, _STAGE_FIELD) == _STAGE_EXECUTE
|
||||||
|
|
||||||
|
try:
|
||||||
|
result = backup_template(config, template, dry_run=not execute)
|
||||||
|
except Exception as exc: # never 500 on an operational failure
|
||||||
|
detail = f"error: {exc}"
|
||||||
|
_audit(config, user=user, action="host.backup", target=template, result="error", detail=detail)
|
||||||
|
return _result_partial(request, action_label=action_label, ok=False, detail=detail)
|
||||||
|
|
||||||
|
if execute:
|
||||||
|
detail = (
|
||||||
|
f"archived to {result.archive_path}" if result.ok else (result.error or "unknown error")
|
||||||
|
)
|
||||||
|
_audit(
|
||||||
|
config,
|
||||||
|
user=user,
|
||||||
|
action="host.backup",
|
||||||
|
target=template,
|
||||||
|
result="ok" if result.ok else "error",
|
||||||
|
detail=detail,
|
||||||
|
)
|
||||||
|
return _result_partial(request, action_label=action_label, ok=result.ok, detail=detail)
|
||||||
|
|
||||||
|
if not result.ok:
|
||||||
|
# Nothing to confirm -- e.g. the template directory does not exist.
|
||||||
|
# This is a validation failure, not something a real run could fix.
|
||||||
|
detail = result.error or "unknown error"
|
||||||
|
_audit(config, user=user, action="host.backup", target=template, result="error", detail=detail)
|
||||||
|
return _result_partial(request, action_label=action_label, ok=False, detail=detail)
|
||||||
|
|
||||||
|
detail = f"would archive template '{template}' to {result.archive_path}"
|
||||||
|
_audit(config, user=user, action="host.backup", target=template, result="dry-run", detail=detail)
|
||||||
|
return _confirm_partial(
|
||||||
|
request,
|
||||||
|
action_label=action_label,
|
||||||
|
target=template,
|
||||||
|
summary=detail,
|
||||||
|
action_url="/host/backup",
|
||||||
|
extra_fields={"template": template},
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
# ──────────────────────────────────────────────────────────────── GET /audit
|
||||||
|
|
||||||
|
|
||||||
|
def audit_view(request: Request, user: str = Depends(require_user)) -> HTMLResponse:
|
||||||
|
"""Read-only audit log table. No CSRF -- this route mutates nothing."""
|
||||||
|
config: Config = request.app.state.config
|
||||||
|
templates: Jinja2Templates = request.app.state.templates
|
||||||
|
entries = read_audit(config.web.audit_log, last=100)
|
||||||
|
return templates.TemplateResponse(request, "partials/audit.html", {"entries": entries})
|
||||||
|
|
||||||
|
|
||||||
|
# ─────────────────────────────────────────────────────────────────── wiring
|
||||||
|
|
||||||
|
|
||||||
|
def register_action_routes(app: FastAPI) -> None:
|
||||||
|
"""Register every W6 mutating route (plus the read-only ``/audit``) on ``app``."""
|
||||||
|
app.add_api_route(
|
||||||
|
"/vms/{vmx_id}/remove", vms_remove, methods=["POST"], response_class=HTMLResponse
|
||||||
|
)
|
||||||
|
app.add_api_route("/vms/cleanup", vms_cleanup, methods=["POST"], response_class=HTMLResponse)
|
||||||
|
app.add_api_route("/cache/purge", cache_purge, methods=["POST"], response_class=HTMLResponse)
|
||||||
|
app.add_api_route("/host/backup", host_backup, methods=["POST"], response_class=HTMLResponse)
|
||||||
|
app.add_api_route("/audit", audit_view, methods=["GET"], response_class=HTMLResponse)
|
||||||
|
|
||||||
|
|
||||||
|
__all__ = [
|
||||||
|
"audit_view",
|
||||||
|
"cache_purge",
|
||||||
|
"encode_vmx_id",
|
||||||
|
"host_backup",
|
||||||
|
"register_action_routes",
|
||||||
|
"vms_cleanup",
|
||||||
|
"vms_remove",
|
||||||
|
]
|
||||||
@@ -0,0 +1,69 @@
|
|||||||
|
"""Synchronizer-token CSRF protection for mutating web routes (W6, SECURITY-CRITICAL).
|
||||||
|
|
||||||
|
Every mutating (``POST``) route in :mod:`ci_orchestrator.web.routes` depends
|
||||||
|
on :func:`require_csrf`. The token is a per-session ``secrets.token_urlsafe(32)``
|
||||||
|
value stored under the ``"csrf"`` key in the (always-on, see
|
||||||
|
``web/app.py::create_app``) signed session cookie -- never derived from a
|
||||||
|
secret, never logged, never put in a URL (form field only, so it never lands
|
||||||
|
in an access log or a browser history entry).
|
||||||
|
|
||||||
|
Requires ``SessionMiddleware`` to already be installed on the app (it is,
|
||||||
|
unconditionally, since W6 -- see ``web/app.py``); without it
|
||||||
|
``request.session`` raises ``AssertionError`` from Starlette itself.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import hmac
|
||||||
|
import secrets
|
||||||
|
|
||||||
|
from fastapi import HTTPException, Request
|
||||||
|
from starlette import status
|
||||||
|
|
||||||
|
_CSRF_KEY = "csrf"
|
||||||
|
|
||||||
|
|
||||||
|
def issue_csrf(request: Request) -> str:
|
||||||
|
"""Return this session's CSRF token, minting one on first use.
|
||||||
|
|
||||||
|
Idempotent within a session: repeated calls (e.g. rendering several
|
||||||
|
action forms on one page) return the same token until the session is
|
||||||
|
cleared (``/logout``) or expires.
|
||||||
|
"""
|
||||||
|
token = request.session.get(_CSRF_KEY)
|
||||||
|
if not isinstance(token, str) or not token:
|
||||||
|
token = secrets.token_urlsafe(32)
|
||||||
|
request.session[_CSRF_KEY] = token
|
||||||
|
return token
|
||||||
|
|
||||||
|
|
||||||
|
def validate_csrf(request: Request, submitted: str | None) -> bool:
|
||||||
|
"""Constant-time comparison of ``submitted`` against the session's token.
|
||||||
|
|
||||||
|
Returns ``False`` (never raises) if the session has no token yet or
|
||||||
|
nothing was submitted -- :func:`require_csrf` turns that into a 403.
|
||||||
|
"""
|
||||||
|
expected = request.session.get(_CSRF_KEY)
|
||||||
|
if not isinstance(expected, str) or not expected or not submitted:
|
||||||
|
return False
|
||||||
|
return hmac.compare_digest(expected, submitted)
|
||||||
|
|
||||||
|
|
||||||
|
async def require_csrf(request: Request) -> None:
|
||||||
|
"""FastAPI dependency: 403s a mutating request with a missing/invalid CSRF token.
|
||||||
|
|
||||||
|
Reads the ``csrf`` form field -- the synchronizer-token pattern, a
|
||||||
|
hidden input re-embedded in every confirm form (see
|
||||||
|
``web/templates/partials/confirm.html``). Raises before any
|
||||||
|
``operations/`` call runs, so an invalid token never triggers so much as
|
||||||
|
a dry-run.
|
||||||
|
"""
|
||||||
|
form = await request.form()
|
||||||
|
submitted = form.get(_CSRF_KEY)
|
||||||
|
if not isinstance(submitted, str) or not validate_csrf(request, submitted):
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=status.HTTP_403_FORBIDDEN, detail="invalid or missing CSRF token"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
__all__ = ["issue_csrf", "require_csrf", "validate_csrf"]
|
||||||
@@ -10,6 +10,7 @@
|
|||||||
<body>
|
<body>
|
||||||
<nav class="navbar">
|
<nav class="navbar">
|
||||||
<span class="navbar-brand">ci-orchestrator</span>
|
<span class="navbar-brand">ci-orchestrator</span>
|
||||||
|
<a href="/audit">Audit log</a>
|
||||||
</nav>
|
</nav>
|
||||||
<main class="content">
|
<main class="content">
|
||||||
{% block content %}{% endblock %}
|
{% block content %}{% endblock %}
|
||||||
|
|||||||
@@ -0,0 +1,37 @@
|
|||||||
|
{% extends "base.html" %}
|
||||||
|
|
||||||
|
{% block title %}Audit log - ci-orchestrator{% endblock %}
|
||||||
|
|
||||||
|
{% block content %}
|
||||||
|
<section class="card" id="audit-card">
|
||||||
|
<h2>Audit log</h2>
|
||||||
|
{% if entries %}
|
||||||
|
<table>
|
||||||
|
<thead>
|
||||||
|
<tr>
|
||||||
|
<th>Time</th>
|
||||||
|
<th>User</th>
|
||||||
|
<th>Action</th>
|
||||||
|
<th>Target</th>
|
||||||
|
<th>Result</th>
|
||||||
|
<th>Detail</th>
|
||||||
|
</tr>
|
||||||
|
</thead>
|
||||||
|
<tbody>
|
||||||
|
{% for e in entries %}
|
||||||
|
<tr class="{{ 'row-alert' if e.result == 'error' else '' }}">
|
||||||
|
<td>{{ e.ts }}</td>
|
||||||
|
<td>{{ e.user }}</td>
|
||||||
|
<td>{{ e.action }}</td>
|
||||||
|
<td><code>{{ e.target }}</code></td>
|
||||||
|
<td>{{ e.result }}</td>
|
||||||
|
<td>{{ e.detail }}</td>
|
||||||
|
</tr>
|
||||||
|
{% endfor %}
|
||||||
|
</tbody>
|
||||||
|
</table>
|
||||||
|
{% else %}
|
||||||
|
<p>No audit entries yet.</p>
|
||||||
|
{% endif %}
|
||||||
|
</section>
|
||||||
|
{% endblock %}
|
||||||
@@ -12,4 +12,8 @@
|
|||||||
{% else %}
|
{% else %}
|
||||||
<p class="notice notice-warn">Cargo cache status unavailable.</p>
|
<p class="notice notice-warn">Cargo cache status unavailable.</p>
|
||||||
{% endif %}
|
{% endif %}
|
||||||
|
<form method="post" action="/cache/purge" id="cache-purge-form">
|
||||||
|
<input type="hidden" name="csrf" value="{{ csrf_token }}">
|
||||||
|
<button type="submit" class="btn btn-danger">Purge cache</button>
|
||||||
|
</form>
|
||||||
</div>
|
</div>
|
||||||
|
|||||||
@@ -0,0 +1,13 @@
|
|||||||
|
<div class="confirm-box notice notice-warn" id="action-confirm">
|
||||||
|
<h3>Confirm: {{ action_label }}</h3>
|
||||||
|
<p>Target: <code>{{ target }}</code></p>
|
||||||
|
<p>{{ summary }}</p>
|
||||||
|
<form method="post" action="{{ action_url }}">
|
||||||
|
<input type="hidden" name="csrf" value="{{ csrf_token }}">
|
||||||
|
<input type="hidden" name="stage" value="execute">
|
||||||
|
{% for key, value in extra_fields.items() %}
|
||||||
|
<input type="hidden" name="{{ key }}" value="{{ value }}">
|
||||||
|
{% endfor %}
|
||||||
|
<button type="submit" class="btn btn-danger">Confirm</button>
|
||||||
|
</form>
|
||||||
|
</div>
|
||||||
@@ -37,3 +37,12 @@
|
|||||||
<p class="notice notice-warn">Runner status unavailable.</p>
|
<p class="notice notice-warn">Runner status unavailable.</p>
|
||||||
{% endif %}
|
{% endif %}
|
||||||
</section>
|
</section>
|
||||||
|
|
||||||
|
<section class="card" id="backup-card">
|
||||||
|
<h2>Template backup</h2>
|
||||||
|
<form method="post" action="/host/backup" id="host-backup-form">
|
||||||
|
<input type="hidden" name="csrf" value="{{ csrf_token }}">
|
||||||
|
<label>Template: <input type="text" name="template" required></label>
|
||||||
|
<button type="submit" class="btn btn-danger">Backup template</button>
|
||||||
|
</form>
|
||||||
|
</section>
|
||||||
|
|||||||
@@ -0,0 +1,5 @@
|
|||||||
|
<div class="notice notice-warn" id="action-lock-conflict">
|
||||||
|
<h3>{{ action_label }} refused</h3>
|
||||||
|
<p>A job may be running against <code>{{ target }}</code> (a fresh
|
||||||
|
<code>vm-start.lock</code> is present). Try again once it clears.</p>
|
||||||
|
</div>
|
||||||
@@ -0,0 +1,4 @@
|
|||||||
|
<div class="action-result {{ 'result-ok' if ok else 'result-error' }}" id="action-result">
|
||||||
|
<h3>{{ action_label }} -- {{ "ok" if ok else "error" }}</h3>
|
||||||
|
<p>{{ detail }}</p>
|
||||||
|
</div>
|
||||||
@@ -13,6 +13,7 @@
|
|||||||
<th>Guest OS</th>
|
<th>Guest OS</th>
|
||||||
<th>Uptime (s)</th>
|
<th>Uptime (s)</th>
|
||||||
<th>Orphan?</th>
|
<th>Orphan?</th>
|
||||||
|
<th>Actions</th>
|
||||||
</tr>
|
</tr>
|
||||||
</thead>
|
</thead>
|
||||||
<tbody>
|
<tbody>
|
||||||
@@ -24,6 +25,12 @@
|
|||||||
<td>{{ vm.guest_os or "-" }}</td>
|
<td>{{ vm.guest_os or "-" }}</td>
|
||||||
<td>{{ vm.uptime_sec if vm.uptime_sec is not none else "-" }}</td>
|
<td>{{ vm.uptime_sec if vm.uptime_sec is not none else "-" }}</td>
|
||||||
<td>{{ "yes" if vm.is_orphan else "no" }}</td>
|
<td>{{ "yes" if vm.is_orphan else "no" }}</td>
|
||||||
|
<td>
|
||||||
|
<form method="post" action="/vms/{{ vm.vmx | vmx_id }}/remove">
|
||||||
|
<input type="hidden" name="csrf" value="{{ csrf_token }}">
|
||||||
|
<button type="submit" class="btn btn-danger">Remove</button>
|
||||||
|
</form>
|
||||||
|
</td>
|
||||||
</tr>
|
</tr>
|
||||||
{% endfor %}
|
{% endfor %}
|
||||||
</tbody>
|
</tbody>
|
||||||
@@ -31,4 +38,9 @@
|
|||||||
{% else %}
|
{% else %}
|
||||||
<p>No VMs currently running.</p>
|
<p>No VMs currently running.</p>
|
||||||
{% endif %}
|
{% endif %}
|
||||||
|
<form method="post" action="/vms/cleanup" id="vms-cleanup-form">
|
||||||
|
<input type="hidden" name="csrf" value="{{ csrf_token }}">
|
||||||
|
<label>Max age (h): <input type="number" name="max_age_hours" value="4" min="0"></label>
|
||||||
|
<button type="submit" class="btn btn-danger">Cleanup orphans</button>
|
||||||
|
</form>
|
||||||
</div>
|
</div>
|
||||||
|
|||||||
@@ -0,0 +1,834 @@
|
|||||||
|
"""Tests for the W6 mutating web routes (VM remove/cleanup, cache purge, host
|
||||||
|
backup, audit) -- SECURITY-CRITICAL: these routes destroy VMs, purge caches,
|
||||||
|
and back up templates.
|
||||||
|
|
||||||
|
No real ``vmrun``/``7z``/``systemctl``/disk writes outside ``tmp_path``: VM
|
||||||
|
backends are fakes/mocks, and ``operations/`` functions are spied on (not
|
||||||
|
replaced) so real side effects still land only inside ``tmp_path``.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import re
|
||||||
|
from datetime import UTC, datetime, timedelta
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
from fastapi.testclient import TestClient
|
||||||
|
|
||||||
|
import ci_orchestrator.operations.host as host_module
|
||||||
|
import ci_orchestrator.web.routes as routes_module
|
||||||
|
from ci_orchestrator.backends.errors import BackendError
|
||||||
|
from ci_orchestrator.backends.protocol import VmHandle
|
||||||
|
from ci_orchestrator.config import (
|
||||||
|
BackendConfig,
|
||||||
|
CargoCacheConfig,
|
||||||
|
Config,
|
||||||
|
Paths,
|
||||||
|
WebConfig,
|
||||||
|
WebOidcConfig,
|
||||||
|
)
|
||||||
|
from ci_orchestrator.credentials import Credential, KeyringCredentialStore
|
||||||
|
from ci_orchestrator.operations.audit import AuditEntry, append_audit, read_audit
|
||||||
|
from ci_orchestrator.operations.vms import RemovalResult
|
||||||
|
from ci_orchestrator.web.app import create_app, get_backend
|
||||||
|
|
||||||
|
_SESSION_CRED_NAME = "TestWebSessionSecret"
|
||||||
|
_OIDC_CRED_NAME = "TestWebOidcClientSecret"
|
||||||
|
_FAKE_SESSION_SECRET = "sess-secret-do-not-leak-abc123"
|
||||||
|
_FAKE_OIDC_CLIENT_SECRET = "oidc-client-secret-do-not-leak-xyz987"
|
||||||
|
|
||||||
|
|
||||||
|
# ── shared fixtures / fakes ─────────────────────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
def _install_fake_keyring(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
|
secrets = {
|
||||||
|
_SESSION_CRED_NAME: _FAKE_SESSION_SECRET,
|
||||||
|
_OIDC_CRED_NAME: _FAKE_OIDC_CLIENT_SECRET,
|
||||||
|
}
|
||||||
|
|
||||||
|
def fake_get(self: KeyringCredentialStore, target: str) -> Credential:
|
||||||
|
if target not in secrets:
|
||||||
|
raise KeyError(target)
|
||||||
|
return Credential(username=target, password=secrets[target])
|
||||||
|
|
||||||
|
monkeypatch.setattr(KeyringCredentialStore, "get", fake_get)
|
||||||
|
|
||||||
|
|
||||||
|
def _make_config(root: Path, *, active: bool = False, allowed_users: list[str] | None = None) -> Config:
|
||||||
|
oidc = None
|
||||||
|
if active:
|
||||||
|
oidc = WebOidcConfig(
|
||||||
|
issuer="https://gitea.example.lan",
|
||||||
|
client_id="ci-web",
|
||||||
|
redirect_url="http://127.0.0.1:8800/auth/callback",
|
||||||
|
client_secret_cred=_OIDC_CRED_NAME,
|
||||||
|
scopes=["openid", "profile"],
|
||||||
|
)
|
||||||
|
return Config(
|
||||||
|
paths=Paths(
|
||||||
|
root=root,
|
||||||
|
templates=root / "templates",
|
||||||
|
build_vms=root / "build-vms",
|
||||||
|
artifacts=root / "artifacts",
|
||||||
|
keys=root / "keys",
|
||||||
|
),
|
||||||
|
backend=BackendConfig(),
|
||||||
|
cargo_cache=CargoCacheConfig(
|
||||||
|
enabled=True, root=root / "cache" / "cargo", retention_days=30
|
||||||
|
),
|
||||||
|
web=WebConfig(
|
||||||
|
enabled=active,
|
||||||
|
allowed_users=allowed_users if allowed_users is not None else [],
|
||||||
|
session_secret_cred=_SESSION_CRED_NAME,
|
||||||
|
audit_log=root / "web-audit.jsonl",
|
||||||
|
oidc=oidc,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class _FakeBackend:
|
||||||
|
"""Stub ``VmBackend`` -- proves no real ``vmrun`` is ever invoked."""
|
||||||
|
|
||||||
|
def __init__(self, handles: list[VmHandle], ip: str | None = "192.168.79.201") -> None:
|
||||||
|
self._handles = handles
|
||||||
|
self._ip = ip
|
||||||
|
self.stop_calls: list[VmHandle] = []
|
||||||
|
self.delete_calls: list[VmHandle] = []
|
||||||
|
|
||||||
|
def list_running(self) -> list[VmHandle]:
|
||||||
|
return self._handles
|
||||||
|
|
||||||
|
def get_ip(self, handle: VmHandle, timeout: float = 0.0) -> str | None:
|
||||||
|
return self._ip
|
||||||
|
|
||||||
|
def stop(self, handle: VmHandle, hard: bool = False) -> None:
|
||||||
|
self.stop_calls.append(handle)
|
||||||
|
|
||||||
|
def delete(self, handle: VmHandle) -> None:
|
||||||
|
self.delete_calls.append(handle)
|
||||||
|
|
||||||
|
|
||||||
|
def _make_vmx(root: Path, name: str = "job1") -> Path:
|
||||||
|
vm_dir = root / "build-vms" / name
|
||||||
|
vm_dir.mkdir(parents=True, exist_ok=True)
|
||||||
|
vmx = vm_dir / f"{name}.vmx"
|
||||||
|
vmx.write_text('guestOS = "ubuntu-64"\n', encoding="utf-8")
|
||||||
|
return vmx
|
||||||
|
|
||||||
|
|
||||||
|
def _spy(real: Any) -> Any:
|
||||||
|
"""Wrap ``real`` so ``.calls`` records every ``(args, kwargs)`` call, still delegating."""
|
||||||
|
|
||||||
|
def wrapper(*args: Any, **kwargs: Any) -> Any:
|
||||||
|
wrapper.calls.append((args, kwargs)) # type: ignore[attr-defined]
|
||||||
|
return real(*args, **kwargs)
|
||||||
|
|
||||||
|
wrapper.calls = [] # type: ignore[attr-defined]
|
||||||
|
return wrapper
|
||||||
|
|
||||||
|
|
||||||
|
def _csrf_from(text: str) -> str:
|
||||||
|
match = re.search(r'name="csrf" value="([^"]+)"', text)
|
||||||
|
assert match is not None, f"no csrf field found in: {text[:300]!r}"
|
||||||
|
return match.group(1)
|
||||||
|
|
||||||
|
|
||||||
|
def _get_csrf(client: TestClient, route: str = "/cache") -> str:
|
||||||
|
resp = client.get(route)
|
||||||
|
assert resp.status_code == 200
|
||||||
|
return _csrf_from(resp.text)
|
||||||
|
|
||||||
|
|
||||||
|
def _last_audit_entry(audit_log: Path) -> AuditEntry:
|
||||||
|
entries = read_audit(audit_log, last=100)
|
||||||
|
assert entries, "expected at least one audit entry"
|
||||||
|
return entries[-1]
|
||||||
|
|
||||||
|
|
||||||
|
# ── 1. CSRF enforcement on every POST ───────────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
def test_post_without_csrf_field_is_403(tmp_path: Path) -> None:
|
||||||
|
app = create_app(_make_config(tmp_path))
|
||||||
|
client = TestClient(app)
|
||||||
|
client.get("/cache") # establish a session
|
||||||
|
|
||||||
|
resp = client.post("/cache/purge", data={})
|
||||||
|
|
||||||
|
assert resp.status_code == 403
|
||||||
|
|
||||||
|
|
||||||
|
def test_post_with_wrong_csrf_is_403(tmp_path: Path) -> None:
|
||||||
|
app = create_app(_make_config(tmp_path))
|
||||||
|
client = TestClient(app)
|
||||||
|
client.get("/cache") # establish a session with a real token
|
||||||
|
|
||||||
|
resp = client.post("/cache/purge", data={"csrf": "totally-not-the-real-token"})
|
||||||
|
|
||||||
|
assert resp.status_code == 403
|
||||||
|
|
||||||
|
|
||||||
|
def test_wrong_csrf_never_runs_the_operation(
|
||||||
|
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||||
|
) -> None:
|
||||||
|
from ci_orchestrator.operations.cache import purge_cargo_cache as real_purge
|
||||||
|
|
||||||
|
spy = _spy(real_purge)
|
||||||
|
monkeypatch.setattr(routes_module, "purge_cargo_cache", spy)
|
||||||
|
app = create_app(_make_config(tmp_path))
|
||||||
|
client = TestClient(app)
|
||||||
|
client.get("/cache")
|
||||||
|
|
||||||
|
resp = client.post("/cache/purge", data={"csrf": "bogus", "stage": "execute"})
|
||||||
|
|
||||||
|
assert resp.status_code == 403
|
||||||
|
assert spy.calls == [] # type: ignore[attr-defined]
|
||||||
|
|
||||||
|
|
||||||
|
# ── 2. two-step confirm gate: preview always runs dry_run=True ─────────────
|
||||||
|
|
||||||
|
|
||||||
|
def test_cache_purge_preview_returns_confirm_partial_and_runs_dry_run(
|
||||||
|
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||||
|
) -> None:
|
||||||
|
from ci_orchestrator.operations.cache import purge_cargo_cache as real_purge
|
||||||
|
|
||||||
|
spy = _spy(real_purge)
|
||||||
|
monkeypatch.setattr(routes_module, "purge_cargo_cache", spy)
|
||||||
|
config = _make_config(tmp_path)
|
||||||
|
app = create_app(config)
|
||||||
|
client = TestClient(app)
|
||||||
|
csrf = _get_csrf(client)
|
||||||
|
|
||||||
|
resp = client.post("/cache/purge", data={"csrf": csrf})
|
||||||
|
|
||||||
|
assert resp.status_code == 200
|
||||||
|
assert "Confirm" in resp.text
|
||||||
|
assert "would remove" in resp.text
|
||||||
|
assert len(spy.calls) == 1 # type: ignore[attr-defined]
|
||||||
|
assert spy.calls[0][1]["dry_run"] is True # type: ignore[attr-defined]
|
||||||
|
|
||||||
|
entry = _last_audit_entry(config.web.audit_log)
|
||||||
|
assert entry.result == "dry-run"
|
||||||
|
assert entry.action == "cache.purge"
|
||||||
|
|
||||||
|
|
||||||
|
def test_cache_purge_preview_does_not_delete_anything(
|
||||||
|
tmp_path: Path,
|
||||||
|
) -> None:
|
||||||
|
cache_root = tmp_path / "cache" / "cargo"
|
||||||
|
cache_root.mkdir(parents=True)
|
||||||
|
entry_path = cache_root / "v1-deadbeef.tar.zst"
|
||||||
|
entry_path.write_bytes(b"x" * 10)
|
||||||
|
# Backdate it so it would be a purge candidate on a real run.
|
||||||
|
old = datetime.now(tz=UTC) - timedelta(days=999)
|
||||||
|
import os
|
||||||
|
|
||||||
|
os.utime(entry_path, (old.timestamp(), old.timestamp()))
|
||||||
|
|
||||||
|
config = _make_config(tmp_path)
|
||||||
|
app = create_app(config)
|
||||||
|
client = TestClient(app)
|
||||||
|
csrf = _get_csrf(client)
|
||||||
|
|
||||||
|
resp = client.post("/cache/purge", data={"csrf": csrf})
|
||||||
|
|
||||||
|
assert resp.status_code == 200
|
||||||
|
assert entry_path.exists()
|
||||||
|
|
||||||
|
|
||||||
|
# ── 3. execute stage: real dry_run=False, audit "ok", result partial ───────
|
||||||
|
|
||||||
|
|
||||||
|
def test_cache_purge_execute_runs_dry_run_false_and_audits_ok(
|
||||||
|
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||||
|
) -> None:
|
||||||
|
from ci_orchestrator.operations.cache import purge_cargo_cache as real_purge
|
||||||
|
|
||||||
|
spy = _spy(real_purge)
|
||||||
|
monkeypatch.setattr(routes_module, "purge_cargo_cache", spy)
|
||||||
|
config = _make_config(tmp_path)
|
||||||
|
app = create_app(config)
|
||||||
|
client = TestClient(app)
|
||||||
|
csrf = _get_csrf(client)
|
||||||
|
|
||||||
|
resp = client.post("/cache/purge", data={"csrf": csrf, "stage": "execute"})
|
||||||
|
|
||||||
|
assert resp.status_code == 200
|
||||||
|
assert len(spy.calls) == 1 # type: ignore[attr-defined]
|
||||||
|
assert spy.calls[0][1]["dry_run"] is False # type: ignore[attr-defined]
|
||||||
|
|
||||||
|
entry = _last_audit_entry(config.web.audit_log)
|
||||||
|
assert entry.result == "ok"
|
||||||
|
assert entry.action == "cache.purge"
|
||||||
|
|
||||||
|
|
||||||
|
def test_cache_purge_execute_actually_removes_expired_entries(tmp_path: Path) -> None:
|
||||||
|
cache_root = tmp_path / "cache" / "cargo"
|
||||||
|
cache_root.mkdir(parents=True)
|
||||||
|
entry_path = cache_root / "v1-deadbeef.tar.zst"
|
||||||
|
entry_path.write_bytes(b"x" * 10)
|
||||||
|
old = datetime.now(tz=UTC) - timedelta(days=999)
|
||||||
|
import os
|
||||||
|
|
||||||
|
os.utime(entry_path, (old.timestamp(), old.timestamp()))
|
||||||
|
|
||||||
|
config = _make_config(tmp_path)
|
||||||
|
app = create_app(config)
|
||||||
|
client = TestClient(app)
|
||||||
|
csrf = _get_csrf(client)
|
||||||
|
|
||||||
|
resp = client.post("/cache/purge", data={"csrf": csrf, "stage": "execute"})
|
||||||
|
|
||||||
|
assert resp.status_code == 200
|
||||||
|
assert not entry_path.exists()
|
||||||
|
|
||||||
|
|
||||||
|
def test_vm_remove_preview_then_execute_full_cycle(
|
||||||
|
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||||
|
) -> None:
|
||||||
|
from ci_orchestrator.operations.vms import remove_vm as real_remove_vm
|
||||||
|
|
||||||
|
spy = _spy(real_remove_vm)
|
||||||
|
monkeypatch.setattr(routes_module, "remove_vm", spy)
|
||||||
|
|
||||||
|
vmx = _make_vmx(tmp_path)
|
||||||
|
config = _make_config(tmp_path)
|
||||||
|
fake_backend = _FakeBackend([VmHandle(identifier=str(vmx), name="job1")])
|
||||||
|
app = create_app(config)
|
||||||
|
app.dependency_overrides[get_backend] = lambda: fake_backend
|
||||||
|
client = TestClient(app)
|
||||||
|
|
||||||
|
resp = client.get("/vms")
|
||||||
|
assert resp.status_code == 200
|
||||||
|
csrf = _csrf_from(resp.text)
|
||||||
|
vmx_id = routes_module.encode_vmx_id(vmx)
|
||||||
|
|
||||||
|
preview = client.post(f"/vms/{vmx_id}/remove", data={"csrf": csrf})
|
||||||
|
assert preview.status_code == 200
|
||||||
|
assert "Confirm" in preview.text
|
||||||
|
assert "would remove" in preview.text
|
||||||
|
assert spy.calls[-1][1]["dry_run"] is True # type: ignore[attr-defined]
|
||||||
|
assert fake_backend.stop_calls == []
|
||||||
|
assert fake_backend.delete_calls == []
|
||||||
|
assert _last_audit_entry(config.web.audit_log).result == "dry-run"
|
||||||
|
|
||||||
|
execute = client.post(f"/vms/{vmx_id}/remove", data={"csrf": csrf, "stage": "execute"})
|
||||||
|
assert execute.status_code == 200
|
||||||
|
assert spy.calls[-1][1]["dry_run"] is False # type: ignore[attr-defined]
|
||||||
|
assert len(fake_backend.stop_calls) == 1
|
||||||
|
assert len(fake_backend.delete_calls) == 1
|
||||||
|
assert not vmx.parent.exists()
|
||||||
|
|
||||||
|
entry = _last_audit_entry(config.web.audit_log)
|
||||||
|
assert entry.result == "ok"
|
||||||
|
assert entry.action == "vm.remove"
|
||||||
|
|
||||||
|
|
||||||
|
# ── 4. path containment ─────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
def test_vm_remove_outside_build_vms_is_400_and_does_not_delete(tmp_path: Path) -> None:
|
||||||
|
outside_dir = tmp_path / "not-build-vms" / "evil"
|
||||||
|
outside_dir.mkdir(parents=True)
|
||||||
|
outside_vmx = outside_dir / "evil.vmx"
|
||||||
|
outside_vmx.write_text("keep me", encoding="utf-8")
|
||||||
|
|
||||||
|
config = _make_config(tmp_path)
|
||||||
|
fake_backend = _FakeBackend([])
|
||||||
|
app = create_app(config)
|
||||||
|
app.dependency_overrides[get_backend] = lambda: fake_backend
|
||||||
|
client = TestClient(app)
|
||||||
|
csrf = _get_csrf(client)
|
||||||
|
vmx_id = routes_module.encode_vmx_id(outside_vmx)
|
||||||
|
|
||||||
|
resp = client.post(f"/vms/{vmx_id}/remove", data={"csrf": csrf, "stage": "execute"})
|
||||||
|
|
||||||
|
assert resp.status_code == 400
|
||||||
|
assert outside_vmx.exists()
|
||||||
|
assert fake_backend.stop_calls == []
|
||||||
|
assert fake_backend.delete_calls == []
|
||||||
|
|
||||||
|
entries = read_audit(config.web.audit_log, last=100)
|
||||||
|
assert not any(e.result == "ok" for e in entries)
|
||||||
|
|
||||||
|
|
||||||
|
def test_vm_remove_malformed_vmx_id_is_400(tmp_path: Path) -> None:
|
||||||
|
config = _make_config(tmp_path)
|
||||||
|
app = create_app(config)
|
||||||
|
app.dependency_overrides[get_backend] = lambda: _FakeBackend([])
|
||||||
|
client = TestClient(app)
|
||||||
|
csrf = _get_csrf(client)
|
||||||
|
|
||||||
|
resp = client.post("/vms/not-valid-base64!!!/remove", data={"csrf": csrf, "stage": "execute"})
|
||||||
|
|
||||||
|
assert resp.status_code == 400
|
||||||
|
|
||||||
|
|
||||||
|
# ── 5. unauthenticated POST -> redirect to /login, no execution ────────────
|
||||||
|
|
||||||
|
|
||||||
|
def test_unauthenticated_post_redirects_to_login_and_never_runs(
|
||||||
|
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||||
|
) -> None:
|
||||||
|
_install_fake_keyring(monkeypatch)
|
||||||
|
from ci_orchestrator.operations.cache import purge_cargo_cache as real_purge
|
||||||
|
|
||||||
|
spy = _spy(real_purge)
|
||||||
|
monkeypatch.setattr(routes_module, "purge_cargo_cache", spy)
|
||||||
|
config = _make_config(tmp_path, active=True, allowed_users=["simone"])
|
||||||
|
app = create_app(config)
|
||||||
|
client = TestClient(app, follow_redirects=False)
|
||||||
|
|
||||||
|
resp = client.post("/cache/purge", data={"csrf": "whatever", "stage": "execute"})
|
||||||
|
|
||||||
|
assert resp.status_code in (302, 303)
|
||||||
|
assert "/login" in resp.headers["location"]
|
||||||
|
assert spy.calls == [] # type: ignore[attr-defined]
|
||||||
|
assert not config.web.audit_log.exists()
|
||||||
|
|
||||||
|
|
||||||
|
# ── 6. job-race guard ───────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
def _write_fresh_lock(config: Config) -> Path:
|
||||||
|
lock_path = config.paths.build_vms.parent / "vm-start.lock"
|
||||||
|
lock_path.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
lock_path.write_text("job in progress", encoding="utf-8")
|
||||||
|
return lock_path
|
||||||
|
|
||||||
|
|
||||||
|
def test_vm_cleanup_execute_refused_409_when_lock_is_fresh(
|
||||||
|
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||||
|
) -> None:
|
||||||
|
from ci_orchestrator.operations.vms import cleanup_orphans as real_cleanup
|
||||||
|
|
||||||
|
spy = _spy(real_cleanup)
|
||||||
|
monkeypatch.setattr(routes_module, "cleanup_orphans", spy)
|
||||||
|
config = _make_config(tmp_path)
|
||||||
|
config.paths.build_vms.mkdir(parents=True, exist_ok=True)
|
||||||
|
_write_fresh_lock(config)
|
||||||
|
app = create_app(config)
|
||||||
|
app.dependency_overrides[get_backend] = lambda: _FakeBackend([])
|
||||||
|
client = TestClient(app)
|
||||||
|
csrf = _get_csrf(client)
|
||||||
|
|
||||||
|
resp = client.post("/vms/cleanup", data={"csrf": csrf, "stage": "execute"})
|
||||||
|
|
||||||
|
assert resp.status_code == 409
|
||||||
|
assert "job may be running" in resp.text.lower()
|
||||||
|
assert spy.calls == [] # type: ignore[attr-defined]
|
||||||
|
|
||||||
|
|
||||||
|
def test_vm_remove_execute_refused_409_when_lock_is_fresh(
|
||||||
|
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||||
|
) -> None:
|
||||||
|
from ci_orchestrator.operations.vms import remove_vm as real_remove_vm
|
||||||
|
|
||||||
|
spy = _spy(real_remove_vm)
|
||||||
|
monkeypatch.setattr(routes_module, "remove_vm", spy)
|
||||||
|
vmx = _make_vmx(tmp_path)
|
||||||
|
config = _make_config(tmp_path)
|
||||||
|
_write_fresh_lock(config)
|
||||||
|
fake_backend = _FakeBackend([VmHandle(identifier=str(vmx), name="job1")])
|
||||||
|
app = create_app(config)
|
||||||
|
app.dependency_overrides[get_backend] = lambda: fake_backend
|
||||||
|
client = TestClient(app)
|
||||||
|
csrf = _get_csrf(client)
|
||||||
|
vmx_id = routes_module.encode_vmx_id(vmx)
|
||||||
|
|
||||||
|
resp = client.post(f"/vms/{vmx_id}/remove", data={"csrf": csrf, "stage": "execute"})
|
||||||
|
|
||||||
|
assert resp.status_code == 409
|
||||||
|
assert spy.calls == [] # type: ignore[attr-defined]
|
||||||
|
assert fake_backend.stop_calls == []
|
||||||
|
assert vmx.exists()
|
||||||
|
|
||||||
|
|
||||||
|
def test_vm_cleanup_preview_ignores_fresh_lock(tmp_path: Path) -> None:
|
||||||
|
"""A preview (dry-run) is always safe -- the lock guard only gates execute."""
|
||||||
|
config = _make_config(tmp_path)
|
||||||
|
config.paths.build_vms.mkdir(parents=True, exist_ok=True)
|
||||||
|
_write_fresh_lock(config)
|
||||||
|
app = create_app(config)
|
||||||
|
app.dependency_overrides[get_backend] = lambda: _FakeBackend([])
|
||||||
|
client = TestClient(app)
|
||||||
|
csrf = _get_csrf(client)
|
||||||
|
|
||||||
|
resp = client.post("/vms/cleanup", data={"csrf": csrf})
|
||||||
|
|
||||||
|
assert resp.status_code == 200
|
||||||
|
assert "Confirm" in resp.text
|
||||||
|
|
||||||
|
|
||||||
|
def test_vm_cleanup_execute_proceeds_when_lock_is_stale(
|
||||||
|
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||||
|
) -> None:
|
||||||
|
from ci_orchestrator.operations.vms import cleanup_orphans as real_cleanup
|
||||||
|
|
||||||
|
spy = _spy(real_cleanup)
|
||||||
|
monkeypatch.setattr(routes_module, "cleanup_orphans", spy)
|
||||||
|
config = _make_config(tmp_path)
|
||||||
|
config.paths.build_vms.mkdir(parents=True, exist_ok=True)
|
||||||
|
lock_path = _write_fresh_lock(config)
|
||||||
|
stale = datetime.now(tz=UTC) - timedelta(minutes=90)
|
||||||
|
import os
|
||||||
|
|
||||||
|
os.utime(lock_path, (stale.timestamp(), stale.timestamp()))
|
||||||
|
app = create_app(config)
|
||||||
|
app.dependency_overrides[get_backend] = lambda: _FakeBackend([])
|
||||||
|
client = TestClient(app)
|
||||||
|
csrf = _get_csrf(client)
|
||||||
|
|
||||||
|
resp = client.post("/vms/cleanup", data={"csrf": csrf, "stage": "execute"})
|
||||||
|
|
||||||
|
assert resp.status_code == 200
|
||||||
|
assert len(spy.calls) == 1 # type: ignore[attr-defined]
|
||||||
|
assert _last_audit_entry(config.web.audit_log).result == "ok"
|
||||||
|
|
||||||
|
|
||||||
|
# ── 7. GET /audit renders written entries ───────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
def test_audit_view_renders_written_entries(tmp_path: Path) -> None:
|
||||||
|
config = _make_config(tmp_path)
|
||||||
|
append_audit(
|
||||||
|
config.web.audit_log,
|
||||||
|
AuditEntry(
|
||||||
|
ts="2026-07-23T10:00:00+00:00",
|
||||||
|
user="anonymous",
|
||||||
|
action="cache.purge",
|
||||||
|
target=str(config.cargo_cache.root),
|
||||||
|
result="ok",
|
||||||
|
detail="freed 123 bytes across 4 entrie(s)",
|
||||||
|
),
|
||||||
|
)
|
||||||
|
app = create_app(config)
|
||||||
|
client = TestClient(app)
|
||||||
|
|
||||||
|
resp = client.get("/audit")
|
||||||
|
|
||||||
|
assert resp.status_code == 200
|
||||||
|
assert "Audit log" in resp.text
|
||||||
|
assert "cache.purge" in resp.text
|
||||||
|
assert "freed 123 bytes across 4 entrie(s)" in resp.text
|
||||||
|
|
||||||
|
|
||||||
|
def test_audit_view_empty_log_still_200(tmp_path: Path) -> None:
|
||||||
|
config = _make_config(tmp_path)
|
||||||
|
app = create_app(config)
|
||||||
|
client = TestClient(app)
|
||||||
|
|
||||||
|
resp = client.get("/audit")
|
||||||
|
|
||||||
|
assert resp.status_code == 200
|
||||||
|
assert "Audit log" in resp.text
|
||||||
|
|
||||||
|
|
||||||
|
def test_audit_view_requires_auth_when_active(
|
||||||
|
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||||
|
) -> None:
|
||||||
|
_install_fake_keyring(monkeypatch)
|
||||||
|
config = _make_config(tmp_path, active=True, allowed_users=["simone"])
|
||||||
|
app = create_app(config)
|
||||||
|
client = TestClient(app, follow_redirects=False)
|
||||||
|
|
||||||
|
resp = client.get("/audit")
|
||||||
|
|
||||||
|
assert resp.status_code in (302, 303)
|
||||||
|
assert "/login" in resp.headers["location"]
|
||||||
|
|
||||||
|
|
||||||
|
# ── 8. no secret value leaks into any response ──────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
def test_no_secret_leaks_into_any_action_response(
|
||||||
|
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||||
|
) -> None:
|
||||||
|
_install_fake_keyring(monkeypatch)
|
||||||
|
config = _make_config(tmp_path, active=True, allowed_users=["simone"])
|
||||||
|
app = create_app(config)
|
||||||
|
app.dependency_overrides[get_backend] = lambda: _FakeBackend([])
|
||||||
|
|
||||||
|
from fastapi.responses import RedirectResponse
|
||||||
|
|
||||||
|
class _StubOAuthClient:
|
||||||
|
async def authorize_redirect(self, request: Any, redirect_uri: str | None = None, **kw: Any) -> Any:
|
||||||
|
return RedirectResponse(f"https://gitea.example.lan/authorize?r={redirect_uri}", 302)
|
||||||
|
|
||||||
|
async def authorize_access_token(self, request: Any, **kw: Any) -> dict[str, Any]:
|
||||||
|
return {"userinfo": {"preferred_username": "simone"}}
|
||||||
|
|
||||||
|
async def userinfo(self, **kw: Any) -> dict[str, Any]:
|
||||||
|
return {"preferred_username": "simone"}
|
||||||
|
|
||||||
|
app.state.oauth.gitea = _StubOAuthClient()
|
||||||
|
client = TestClient(app, follow_redirects=False)
|
||||||
|
client.get("/auth/callback") # logs in
|
||||||
|
|
||||||
|
bodies: list[str] = []
|
||||||
|
bodies.append(client.get("/cache").text)
|
||||||
|
csrf = _csrf_from(bodies[-1])
|
||||||
|
bodies.append(client.post("/cache/purge", data={"csrf": csrf}).text)
|
||||||
|
bodies.append(client.post("/cache/purge", data={"csrf": csrf, "stage": "execute"}).text)
|
||||||
|
bodies.append(client.get("/audit").text)
|
||||||
|
|
||||||
|
for body in bodies:
|
||||||
|
assert _FAKE_SESSION_SECRET not in body
|
||||||
|
assert _FAKE_OIDC_CLIENT_SECRET not in body
|
||||||
|
|
||||||
|
|
||||||
|
# ── extra coverage: helper edge cases + host.backup + degraded paths ───────
|
||||||
|
|
||||||
|
|
||||||
|
def test_is_contained_returns_false_on_resolve_oserror(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
|
def _boom(self: Path, strict: bool = False) -> Path:
|
||||||
|
raise OSError("boom")
|
||||||
|
|
||||||
|
monkeypatch.setattr(Path, "resolve", _boom)
|
||||||
|
|
||||||
|
assert routes_module._is_contained(Path("/a/b"), Path("/a")) is False
|
||||||
|
|
||||||
|
|
||||||
|
def test_lock_is_fresh_missing_file_returns_false(tmp_path: Path) -> None:
|
||||||
|
assert routes_module._lock_is_fresh(tmp_path / "missing.lock") is False
|
||||||
|
|
||||||
|
|
||||||
|
def test_vm_remove_vmx_id_with_bad_base64_padding_is_400(tmp_path: Path) -> None:
|
||||||
|
config = _make_config(tmp_path)
|
||||||
|
app = create_app(config)
|
||||||
|
app.dependency_overrides[get_backend] = lambda: _FakeBackend([])
|
||||||
|
client = TestClient(app)
|
||||||
|
csrf = _get_csrf(client)
|
||||||
|
|
||||||
|
resp = client.post("/vms/a/remove", data={"csrf": csrf, "stage": "execute"})
|
||||||
|
|
||||||
|
assert resp.status_code == 400
|
||||||
|
|
||||||
|
|
||||||
|
def test_vm_cleanup_preview_with_non_numeric_max_age_hours_defaults(tmp_path: Path) -> None:
|
||||||
|
config = _make_config(tmp_path)
|
||||||
|
config.paths.build_vms.mkdir(parents=True, exist_ok=True)
|
||||||
|
app = create_app(config)
|
||||||
|
app.dependency_overrides[get_backend] = lambda: _FakeBackend([])
|
||||||
|
client = TestClient(app)
|
||||||
|
csrf = _get_csrf(client)
|
||||||
|
|
||||||
|
resp = client.post("/vms/cleanup", data={"csrf": csrf, "max_age_hours": "not-a-number"})
|
||||||
|
|
||||||
|
assert resp.status_code == 200
|
||||||
|
assert "older than 4h" in resp.text
|
||||||
|
|
||||||
|
|
||||||
|
def test_vm_remove_backend_unavailable_renders_error_partial_and_audits_error(
|
||||||
|
tmp_path: Path,
|
||||||
|
) -> None:
|
||||||
|
vmx = _make_vmx(tmp_path)
|
||||||
|
config = _make_config(tmp_path)
|
||||||
|
app = create_app(config)
|
||||||
|
app.dependency_overrides[get_backend] = lambda: None
|
||||||
|
client = TestClient(app)
|
||||||
|
csrf = _get_csrf(client)
|
||||||
|
vmx_id = routes_module.encode_vmx_id(vmx)
|
||||||
|
|
||||||
|
resp = client.post(f"/vms/{vmx_id}/remove", data={"csrf": csrf})
|
||||||
|
|
||||||
|
assert resp.status_code == 200
|
||||||
|
assert "unavailable" in resp.text.lower()
|
||||||
|
entry = _last_audit_entry(config.web.audit_log)
|
||||||
|
assert entry.result == "error"
|
||||||
|
|
||||||
|
|
||||||
|
def test_vm_remove_execute_error_result_audits_error(
|
||||||
|
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||||
|
) -> None:
|
||||||
|
vmx = _make_vmx(tmp_path)
|
||||||
|
config = _make_config(tmp_path)
|
||||||
|
|
||||||
|
def _failing_remove(backend: Any, target_vmx: Path, dry_run: bool = False) -> RemovalResult:
|
||||||
|
return RemovalResult(
|
||||||
|
target=str(target_vmx), stopped=True, deleted=False, dir_removed=False, error="delete failed"
|
||||||
|
)
|
||||||
|
|
||||||
|
monkeypatch.setattr(routes_module, "remove_vm", _failing_remove)
|
||||||
|
app = create_app(config)
|
||||||
|
app.dependency_overrides[get_backend] = lambda: _FakeBackend([])
|
||||||
|
client = TestClient(app)
|
||||||
|
csrf = _get_csrf(client)
|
||||||
|
vmx_id = routes_module.encode_vmx_id(vmx)
|
||||||
|
|
||||||
|
resp = client.post(f"/vms/{vmx_id}/remove", data={"csrf": csrf, "stage": "execute"})
|
||||||
|
|
||||||
|
assert resp.status_code == 200
|
||||||
|
assert "error" in resp.text.lower()
|
||||||
|
entry = _last_audit_entry(config.web.audit_log)
|
||||||
|
assert entry.result == "error"
|
||||||
|
assert "error removing" in entry.detail
|
||||||
|
|
||||||
|
|
||||||
|
def test_vm_remove_swallows_real_backend_error_never_500s(tmp_path: Path) -> None:
|
||||||
|
"""End-to-end (not mocked) proof that a real BackendError never 500s."""
|
||||||
|
|
||||||
|
class _FailingBackend:
|
||||||
|
def list_running(self) -> list[VmHandle]:
|
||||||
|
return []
|
||||||
|
|
||||||
|
def get_ip(self, handle: VmHandle, timeout: float = 0.0) -> str | None:
|
||||||
|
return None
|
||||||
|
|
||||||
|
def stop(self, handle: VmHandle, hard: bool = False) -> None:
|
||||||
|
raise BackendError("stop failed")
|
||||||
|
|
||||||
|
def delete(self, handle: VmHandle) -> None:
|
||||||
|
raise BackendError("delete failed")
|
||||||
|
|
||||||
|
vmx = _make_vmx(tmp_path)
|
||||||
|
config = _make_config(tmp_path)
|
||||||
|
app = create_app(config)
|
||||||
|
app.dependency_overrides[get_backend] = lambda: _FailingBackend()
|
||||||
|
client = TestClient(app)
|
||||||
|
csrf = _get_csrf(client)
|
||||||
|
vmx_id = routes_module.encode_vmx_id(vmx)
|
||||||
|
|
||||||
|
resp = client.post(f"/vms/{vmx_id}/remove", data={"csrf": csrf}) # preview only, no retries
|
||||||
|
|
||||||
|
assert resp.status_code == 200
|
||||||
|
|
||||||
|
|
||||||
|
def test_cache_purge_exception_never_500s(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
|
def _boom(*args: Any, **kwargs: Any) -> Any:
|
||||||
|
raise RuntimeError("disk exploded")
|
||||||
|
|
||||||
|
monkeypatch.setattr(routes_module, "purge_cargo_cache", _boom)
|
||||||
|
config = _make_config(tmp_path)
|
||||||
|
app = create_app(config)
|
||||||
|
client = TestClient(app)
|
||||||
|
csrf = _get_csrf(client)
|
||||||
|
|
||||||
|
resp = client.post("/cache/purge", data={"csrf": csrf})
|
||||||
|
|
||||||
|
assert resp.status_code == 200
|
||||||
|
assert "error" in resp.text.lower()
|
||||||
|
entry = _last_audit_entry(config.web.audit_log)
|
||||||
|
assert entry.result == "error"
|
||||||
|
|
||||||
|
|
||||||
|
def test_host_backup_missing_template_field_is_400(tmp_path: Path) -> None:
|
||||||
|
config = _make_config(tmp_path)
|
||||||
|
app = create_app(config)
|
||||||
|
client = TestClient(app)
|
||||||
|
csrf = _get_csrf(client)
|
||||||
|
|
||||||
|
resp = client.post("/host/backup", data={"csrf": csrf})
|
||||||
|
|
||||||
|
assert resp.status_code == 400
|
||||||
|
|
||||||
|
|
||||||
|
def test_host_backup_preview_missing_template_dir_is_error_partial(tmp_path: Path) -> None:
|
||||||
|
config = _make_config(tmp_path)
|
||||||
|
app = create_app(config)
|
||||||
|
client = TestClient(app)
|
||||||
|
csrf = _get_csrf(client)
|
||||||
|
|
||||||
|
resp = client.post("/host/backup", data={"csrf": csrf, "template": "NoSuchTemplate"})
|
||||||
|
|
||||||
|
assert resp.status_code == 200
|
||||||
|
assert "error" in resp.text.lower()
|
||||||
|
entry = _last_audit_entry(config.web.audit_log)
|
||||||
|
assert entry.result == "error"
|
||||||
|
|
||||||
|
|
||||||
|
def test_host_backup_preview_then_execute_full_cycle(
|
||||||
|
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||||
|
) -> None:
|
||||||
|
monkeypatch.setattr(
|
||||||
|
host_module, "_compress_7z", lambda source, archive: archive.write_bytes(b"fake-7z")
|
||||||
|
)
|
||||||
|
template_dir = tmp_path / "templates" / "LinuxBuild2404"
|
||||||
|
template_dir.mkdir(parents=True)
|
||||||
|
(template_dir / "LinuxBuild2404.vmx").write_text("x", encoding="utf-8")
|
||||||
|
|
||||||
|
config = _make_config(tmp_path)
|
||||||
|
app = create_app(config)
|
||||||
|
client = TestClient(app)
|
||||||
|
csrf = _get_csrf(client)
|
||||||
|
|
||||||
|
preview = client.post(
|
||||||
|
"/host/backup", data={"csrf": csrf, "template": "LinuxBuild2404"}
|
||||||
|
)
|
||||||
|
assert preview.status_code == 200
|
||||||
|
assert "Confirm" in preview.text
|
||||||
|
assert "would archive" in preview.text
|
||||||
|
assert _last_audit_entry(config.web.audit_log).result == "dry-run"
|
||||||
|
|
||||||
|
execute = client.post(
|
||||||
|
"/host/backup", data={"csrf": csrf, "stage": "execute", "template": "LinuxBuild2404"}
|
||||||
|
)
|
||||||
|
assert execute.status_code == 200
|
||||||
|
entry = _last_audit_entry(config.web.audit_log)
|
||||||
|
assert entry.result == "ok"
|
||||||
|
assert entry.action == "host.backup"
|
||||||
|
|
||||||
|
|
||||||
|
def test_host_backup_execute_exception_never_500s(
|
||||||
|
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||||
|
) -> None:
|
||||||
|
def _boom(*args: Any, **kwargs: Any) -> Any:
|
||||||
|
raise RuntimeError("disk exploded")
|
||||||
|
|
||||||
|
monkeypatch.setattr(routes_module, "backup_template", _boom)
|
||||||
|
config = _make_config(tmp_path)
|
||||||
|
app = create_app(config)
|
||||||
|
client = TestClient(app)
|
||||||
|
csrf = _get_csrf(client)
|
||||||
|
|
||||||
|
resp = client.post(
|
||||||
|
"/host/backup", data={"csrf": csrf, "stage": "execute", "template": "X"}
|
||||||
|
)
|
||||||
|
|
||||||
|
assert resp.status_code == 200
|
||||||
|
assert "error" in resp.text.lower()
|
||||||
|
entry = _last_audit_entry(config.web.audit_log)
|
||||||
|
assert entry.result == "error"
|
||||||
|
|
||||||
|
|
||||||
|
# ── web/security.py: direct unit coverage of the CSRF primitives ──────────
|
||||||
|
|
||||||
|
|
||||||
|
def test_issue_csrf_is_idempotent_within_a_session() -> None:
|
||||||
|
from ci_orchestrator.web.security import issue_csrf
|
||||||
|
|
||||||
|
fake_request = type("_Req", (), {"session": {}})()
|
||||||
|
|
||||||
|
first = issue_csrf(fake_request) # type: ignore[arg-type]
|
||||||
|
second = issue_csrf(fake_request) # type: ignore[arg-type]
|
||||||
|
|
||||||
|
assert first == second
|
||||||
|
assert len(first) > 20
|
||||||
|
|
||||||
|
|
||||||
|
def test_validate_csrf_false_when_no_token_issued_yet() -> None:
|
||||||
|
from ci_orchestrator.web.security import validate_csrf
|
||||||
|
|
||||||
|
fake_request = type("_Req", (), {"session": {}})()
|
||||||
|
|
||||||
|
assert validate_csrf(fake_request, "anything") is False # type: ignore[arg-type]
|
||||||
|
|
||||||
|
|
||||||
|
def test_validate_csrf_false_when_nothing_submitted() -> None:
|
||||||
|
from ci_orchestrator.web.security import validate_csrf
|
||||||
|
|
||||||
|
fake_request = type("_Req", (), {"session": {"csrf": "real-token"}})()
|
||||||
|
|
||||||
|
assert validate_csrf(fake_request, None) is False # type: ignore[arg-type]
|
||||||
|
|
||||||
|
|
||||||
|
def test_validate_csrf_true_for_matching_token() -> None:
|
||||||
|
from ci_orchestrator.web.security import validate_csrf
|
||||||
|
|
||||||
|
fake_request = type("_Req", (), {"session": {"csrf": "real-token"}})()
|
||||||
|
|
||||||
|
assert validate_csrf(fake_request, "real-token") is True # type: ignore[arg-type]
|
||||||
Reference in New Issue
Block a user