feat(web): service management panel over a sudo allow-list (stage D)

Adds operations/services.py: a fixed ManagedService allow-list (act-runner, act-runner-rust, vmware-host, ci-web) with find_service/service_states/control_service. control_service builds an argument list only (never shell=True), validates the action against ALLOWED_ACTIONS (ValueError->400), refuses stop on restart-only services (ci-web, allow_stop=False) without building a command, and defers ci-web's own restart via 'sudo -n systemd-run --on-active=2 systemctl restart ci-web.service' so the HTTP response returns first.

Route POST /services/{service_id}/{action} in web/routes.py mirrors vms_remove: require_user + require_csrf + two-step confirm gate + one audit entry per stage; unknown id / bad action / disallowed-stop short-circuit to 400 before any subprocess or audit; the vmware-host confirm step warns with the active-VM count. New /services read partial with state-aware Start/Stop/Restart buttons, polled every 15s.

deploy/sudoers.d/ci-web grants ci-runner exact-command NOPASSWD for exactly these nine systemctl lines plus the systemd-run deferral — no wildcards, no bare systemctl, no ci-web stop; setup-web-linux.sh installs it mode 0440 and validates with 'visudo -cf', rolling back on failure. Config gains [web].managed_services override; docs/WEB-UI.md documents the panel + security model.

Gates: 1044 tests pass (46 new security/unit tests), coverage 94.20%, ruff + mypy --strict clean. Not yet deployed: sudoers install + prod-venv reinstall + ci-web restart are a human privilege-boundary step.

src/ci_orchestrator/web/app.py

@@ -61,6 +61,7 @@ from ci_orchestrator.operations.inventory import (
     list_backups,
     list_templates,
 )
+from ci_orchestrator.operations.services import ServiceState, service_states
 from ci_orchestrator.operations.vms import LiveVm, list_live_vms
 from ci_orchestrator.web import filters
 from ci_orchestrator.web.auth import _read_secret, configure_auth, require_user
@@ -202,6 +203,20 @@ def _fetch_artifacts(config: Config) -> list[FileEntry] | None:
         return None
 
 
+def _fetch_service_states(config: Config) -> list[ServiceState]:
+    """Managed systemd unit states (Stage D), or ``[]`` if the query could not run at all.
+
+    ``service_states`` itself already degrades a single unit to
+    ``active=False, sub_state="unknown"`` on a ``systemctl`` hiccup (see
+    ``operations/host.py::runner_status``) -- this ``except Exception`` is a
+    defensive second layer, matching every other ``_fetch_*`` wrapper here.
+    """
+    try:
+        return service_states(config.web.managed_services)
+    except Exception:  # degrade, never 500
+        return []
+
+
 # ────────────────────────────────────────────────────────── status header
 
 
@@ -335,6 +350,7 @@ def dashboard(
         "template_names": template_names,
         "backups": _fetch_backups(config),
         "artifacts": _fetch_artifacts(config),
+        "services": _fetch_service_states(config),
         "health": _health_summary(
             vms,
             backend_available=backend_available,
@@ -415,6 +431,23 @@ def files_partial(request: Request) -> HTMLResponse:
     return templates.TemplateResponse(request, "partials/files.html", context)
 
 
+def services_partial(request: Request) -> HTMLResponse:
+    """``GET /services`` (Stage D) -- polled every 15s by the dashboard via htmx.
+
+    Needs a fresh ``csrf_token`` on every render (unlike ``resources``/
+    ``vmware``/``templates``/``files``, which are pure read-only cards) --
+    the panel's start/stop/restart forms are mutating, same as
+    ``vms_partial``/``host_partial``.
+    """
+    config: Config = request.app.state.config
+    templates: Jinja2Templates = request.app.state.templates
+    context = {
+        "services": _fetch_service_states(config),
+        "csrf_token": issue_csrf(request),
+    }
+    return templates.TemplateResponse(request, "partials/services.html", context)
+
+
 # ─────────────────────────────────────────────────────────────────── factory
 
 
@@ -470,6 +503,9 @@ def create_app(config: Config) -> FastAPI:
     app.get("/files", response_class=HTMLResponse, dependencies=[Depends(require_user)])(
         files_partial
     )
+    app.get("/services", response_class=HTMLResponse, dependencies=[Depends(require_user)])(
+        services_partial
+    )
     register_action_routes(app)
 
     # SessionMiddleware is ALWAYS installed (W6) -- CSRF needs
@@ -509,6 +545,7 @@ __all__ = [
     "get_backend",
     "healthz",
     "resources_partial",
+    "services_partial",
     "templates_partial",
     "vmware_partial",
 ]

src/ci_orchestrator/web/routes.py

@@ -49,6 +49,7 @@ from ci_orchestrator.operations.audit import AuditEntry, append_audit, read_audi
 from ci_orchestrator.operations.cache import purge_cargo_cache
 from ci_orchestrator.operations.host import backup_template
 from ci_orchestrator.operations.inventory import list_templates
+from ci_orchestrator.operations.services import ALLOWED_ACTIONS, control_service, find_service
 from ci_orchestrator.operations.vms import RemovalResult, cleanup_orphans, remove_vm
 from ci_orchestrator.web.app import get_backend
 from ci_orchestrator.web.auth import require_user
@@ -200,13 +201,23 @@ def _confirm_partial(
     summary: str,
     action_url: str,
     extra_fields: dict[str, str],
+    warning: str | None = None,
 ) -> HTMLResponse:
+    """Render the shared confirm partial.
+
+    ``warning`` (Stage D) is an optional extra callout rendered above the
+    Confirm button -- used by the service-control route to surface "N
+    active VM(s) -- this interrupts running jobs" when confirming a
+    stop/restart of ``vmware-host.service``. ``None`` (every other caller)
+    renders nothing extra, unchanged from before Stage D.
+    """
     context = {
         "action_label": action_label,
         "target": target,
         "summary": summary,
         "action_url": action_url,
         "extra_fields": extra_fields,
+        "warning": warning,
         "csrf_token": issue_csrf(request),
     }
     return _render_action(request, "partials/confirm.html", context)
@@ -470,6 +481,100 @@ async def host_backup(
     )
 
 
+# ───────────────────────────────────────────────────── POST /services/{id}/{action}
+
+#: The ``ManagedService.id`` whose stop/restart confirm step warns about
+#: active VMs -- stopping the VMware host stack kills every running build.
+_VMWARE_HOST_SERVICE_ID = "vmware-host"
+
+
+def _active_vm_count(backend: VmBackend | None) -> int:
+    """Best-effort running-VM count for the ``vmware-host`` confirm warning.
+
+    Degrades to ``0`` on a missing backend or any hiccup listing VMs --
+    this is advisory copy in a confirm screen, never a value a route should
+    500 over.
+    """
+    if backend is None:
+        return 0
+    try:
+        return len(backend.list_running())
+    except Exception:  # a route must never 500 on a backend hiccup
+        return 0
+
+
+async def services_control(
+    service_id: str,
+    action: str,
+    request: Request,
+    user: str = Depends(require_user),
+    _csrf: None = Depends(require_csrf),
+    backend: VmBackend | None = Depends(get_backend),
+) -> HTMLResponse:
+    """Start/stop/restart one managed systemd unit, behind the two-step confirm gate.
+
+    Mirrors :func:`vms_remove`'s shape exactly. Three refusals short-circuit
+    **before** ``control_service`` (and therefore any subprocess) or the
+    audit log ever run -- an unknown ``service_id``, an unsupported
+    ``action``, and a ``stop`` request against a restart-only service (see
+    ``operations/services.py``'s module docstring on why ``ci-web`` is
+    restart-only). Each is a plain validation refusal, not an action, so
+    none of them is audited -- same rule as the path-containment check in
+    :func:`vms_remove`.
+    """
+    config: Config = request.app.state.config
+    services = config.web.managed_services
+
+    service = find_service(service_id, services)
+    if service is None:
+        raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="unknown service id")
+    if action not in ALLOWED_ACTIONS:
+        raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="unsupported action")
+    if action == "stop" and not service.allow_stop:
+        raise HTTPException(
+            status_code=status.HTTP_400_BAD_REQUEST,
+            detail=f"stop is disabled for '{service.label}' (restart-only)",
+        )
+
+    action_label = f"{service.label}: {action}"
+    target = service.unit
+    audit_action = f"service.{action}"
+
+    form = await request.form()
+    execute = _form_str(form, _STAGE_FIELD) == _STAGE_EXECUTE
+
+    if execute:
+        result = control_service(service, action, dry_run=False)
+        _audit(
+            config,
+            user=user,
+            action=audit_action,
+            target=target,
+            result="ok" if result.ok else "error",
+            detail=result.detail,
+        )
+        return _result_partial(request, action_label=action_label, ok=result.ok, detail=result.detail)
+
+    result = control_service(service, action, dry_run=True)
+    _audit(config, user=user, action=audit_action, target=target, result="dry-run", detail=result.detail)
+
+    warning: str | None = None
+    if service.id == _VMWARE_HOST_SERVICE_ID and action != "start":
+        vm_count = _active_vm_count(backend)
+        if vm_count > 0:
+            warning = f"{vm_count} active VM(s) -- this interrupts running jobs."
+
+    return _confirm_partial(
+        request,
+        action_label=action_label,
+        target=target,
+        summary=result.detail,
+        action_url=f"/services/{service_id}/{action}",
+        extra_fields={},
+        warning=warning,
+    )
+
+
 # ──────────────────────────────────────────────────────────────── GET /audit
 
 
@@ -492,6 +597,12 @@ def register_action_routes(app: FastAPI) -> None:
     app.add_api_route("/vms/cleanup", vms_cleanup, methods=["POST"], response_class=HTMLResponse)
     app.add_api_route("/cache/purge", cache_purge, methods=["POST"], response_class=HTMLResponse)
     app.add_api_route("/host/backup", host_backup, methods=["POST"], response_class=HTMLResponse)
+    app.add_api_route(
+        "/services/{service_id}/{action}",
+        services_control,
+        methods=["POST"],
+        response_class=HTMLResponse,
+    )
     app.add_api_route("/audit", audit_view, methods=["GET"], response_class=HTMLResponse)
 
 
@@ -501,6 +612,7 @@ __all__ = [
     "encode_vmx_id",
     "host_backup",
     "register_action_routes",
+    "services_control",
     "vms_cleanup",
     "vms_remove",
 ]

src/ci_orchestrator/web/static/app.css

@@ -305,6 +305,12 @@ input[type="text"], input[type="number"], select {
 /* One-line field description above a form control (B6 backup dropdown). */
 .field-hint { font-size: 0.8rem; color: var(--text-dim); margin: 0 0 0.5rem; }
 
+/* Multiple one-button action forms side-by-side in one table cell (Stage D
+ * services panel: Start/Stop/Restart). Overrides the default form's
+ * margin-top so the row stays compact inside a <td>. */
+.service-actions { display: flex; flex-wrap: wrap; align-items: center; gap: 0.5rem; }
+.service-actions form { margin-top: 0; }
+
 /* Muted inline text (e.g. a relative-time suffix next to a file name). */
 .text-dim { color: var(--text-dim); }
 

src/ci_orchestrator/web/templates/dashboard.html

@@ -13,5 +13,6 @@
   {% include "partials/vmware.html" %}
   {% include "partials/templates.html" %}
   {% include "partials/files.html" %}
+  {% include "partials/services.html" %}
 </div>
 {% endblock %}

src/ci_orchestrator/web/templates/partials/confirm.html

@@ -2,6 +2,9 @@
   <h3>Confirm: {{ action_label }}</h3>
   <p>Target: <code>{{ target }}</code></p>
   <p>{{ summary }}</p>
+  {% if warning %}
+  <p class="notice notice-warn" role="alert">{{ warning }}</p>
+  {% endif %}
   <form method="post" action="{{ action_url }}" hx-post="{{ action_url }}" hx-target="#action-slot" hx-swap="innerHTML">
     <input type="hidden" name="csrf" value="{{ csrf_token }}">
     <input type="hidden" name="stage" value="execute">

src/ci_orchestrator/web/templates/partials/services.html

@@ -0,0 +1,57 @@
+<section class="card" id="services-card" hx-get="/services" hx-trigger="every 15s" hx-swap="outerHTML">
+  <h2><svg class="icon" aria-hidden="true"><use href="#i-gear"></use></svg>Services</h2>
+  {% if services %}
+  <div class="table-wrap">
+    <table>
+      <thead>
+        <tr>
+          <th>Service</th>
+          <th>Unit</th>
+          <th>Status</th>
+          <th>Sub-state</th>
+          <th>Actions</th>
+        </tr>
+      </thead>
+      <tbody>
+        {% for state in services %}
+        <tr class="{{ 'row-alert' if not state.active else '' }}">
+          <td>{{ state.service.label }}</td>
+          <td><code>{{ state.service.unit }}</code></td>
+          <td>
+            {% if state.active %}
+            <span class="badge badge-success"><svg class="icon" aria-hidden="true"><use href="#i-check"></use></svg> active</span>
+            {% else %}
+            <span class="badge badge-danger"><svg class="icon" aria-hidden="true"><use href="#i-alert"></use></svg> down</span>
+            {% endif %}
+          </td>
+          <td>{{ state.sub_state }}</td>
+          <td class="service-actions">
+            {% if not state.active %}
+            <form method="post" action="/services/{{ state.service.id }}/start"
+                  hx-post="/services/{{ state.service.id }}/start" hx-target="#action-slot" hx-swap="innerHTML">
+              <input type="hidden" name="csrf" value="{{ csrf_token }}">
+              <button type="submit" class="btn btn-primary" aria-label="Start {{ state.service.label }}">Start</button>
+            </form>
+            {% endif %}
+            {% if state.active and state.service.allow_stop %}
+            <form method="post" action="/services/{{ state.service.id }}/stop"
+                  hx-post="/services/{{ state.service.id }}/stop" hx-target="#action-slot" hx-swap="innerHTML">
+              <input type="hidden" name="csrf" value="{{ csrf_token }}">
+              <button type="submit" class="btn btn-danger" aria-label="Stop {{ state.service.label }}">Stop</button>
+            </form>
+            {% endif %}
+            <form method="post" action="/services/{{ state.service.id }}/restart"
+                  hx-post="/services/{{ state.service.id }}/restart" hx-target="#action-slot" hx-swap="innerHTML">
+              <input type="hidden" name="csrf" value="{{ csrf_token }}">
+              <button type="submit" class="btn" aria-label="Restart {{ state.service.label }}">Restart</button>
+            </form>
+          </td>
+        </tr>
+        {% endfor %}
+      </tbody>
+    </table>
+  </div>
+  {% else %}
+  <p class="notice notice-warn" role="alert">Service status unavailable.</p>
+  {% endif %}
+</section>