feat(web): service management panel over a sudo allow-list (stage D)

Adds operations/services.py: a fixed ManagedService allow-list (act-runner, act-runner-rust, vmware-host, ci-web) with find_service/service_states/control_service. control_service builds an argument list only (never shell=True), validates the action against ALLOWED_ACTIONS (ValueError->400), refuses stop on restart-only services (ci-web, allow_stop=False) without building a command, and defers ci-web's own restart via 'sudo -n systemd-run --on-active=2 systemctl restart ci-web.service' so the HTTP response returns first.

Route POST /services/{service_id}/{action} in web/routes.py mirrors vms_remove: require_user + require_csrf + two-step confirm gate + one audit entry per stage; unknown id / bad action / disallowed-stop short-circuit to 400 before any subprocess or audit; the vmware-host confirm step warns with the active-VM count. New /services read partial with state-aware Start/Stop/Restart buttons, polled every 15s.

deploy/sudoers.d/ci-web grants ci-runner exact-command NOPASSWD for exactly these nine systemctl lines plus the systemd-run deferral — no wildcards, no bare systemctl, no ci-web stop; setup-web-linux.sh installs it mode 0440 and validates with 'visudo -cf', rolling back on failure. Config gains [web].managed_services override; docs/WEB-UI.md documents the panel + security model.

Gates: 1044 tests pass (46 new security/unit tests), coverage 94.20%, ruff + mypy --strict clean. Not yet deployed: sudoers install + prod-venv reinstall + ci-web restart are a human privilege-boundary step.

tests/python/test_config_web.py

@@ -10,6 +10,7 @@ from __future__ import annotations
 from pathlib import Path
 
 from ci_orchestrator.config import WebConfig, WebOidcConfig, gitea_base_url, load_config
+from ci_orchestrator.operations.services import MANAGED_SERVICES, ManagedService
 
 
 def _paths_section() -> str:
@@ -251,3 +252,88 @@ def test_gitea_base_url_none_when_issuer_unparseable() -> None:
     )
 
     assert gitea_base_url(web) is None
+
+
+# ── [web].managed_services (Stage D) ────────────────────────────────────────
+
+
+def test_managed_services_absent_defaults_to_builtin_allowlist() -> None:
+    assert WebConfig().managed_services == MANAGED_SERVICES
+
+
+def test_managed_services_absent_from_toml_defaults_to_builtin_allowlist(tmp_path: Path) -> None:
+    cfg = load_config(config_path=_write_toml(tmp_path), env={})
+
+    assert cfg.web.managed_services == MANAGED_SERVICES
+
+
+def test_managed_services_override_parses_rows(tmp_path: Path) -> None:
+    extra = (
+        "\n[[web.managed_services]]\n"
+        'id = "act-runner"\n'
+        'unit = "act-runner.service"\n'
+        'label = "Runner"\n'
+        "allow_stop = true\n"
+        "\n[[web.managed_services]]\n"
+        'id = "ci-web"\n'
+        'unit = "ci-web.service"\n'
+        'label = "Web UI"\n'
+        "allow_stop = false\n"
+    )
+    cfg = load_config(config_path=_write_toml(tmp_path, extra), env={})
+
+    assert cfg.web.managed_services == (
+        ManagedService("act-runner", "act-runner.service", "Runner", True),
+        ManagedService("ci-web", "ci-web.service", "Web UI", False),
+    )
+
+
+def test_managed_services_malformed_row_is_skipped_not_raised(tmp_path: Path) -> None:
+    extra = (
+        "\n[[web.managed_services]]\n"
+        'id = "act-runner"\n'
+        'unit = "act-runner.service"\n'
+        'label = "Runner"\n'
+        "allow_stop = true\n"
+        "\n[[web.managed_services]]\n"
+        # Missing 'unit' -- this row must be skipped, not crash config loading.
+        'id = "broken"\n'
+        'label = "Broken"\n'
+        "allow_stop = true\n"
+    )
+    cfg = load_config(config_path=_write_toml(tmp_path, extra), env={})
+
+    assert cfg.web.managed_services == (
+        ManagedService("act-runner", "act-runner.service", "Runner", True),
+    )
+
+
+def test_managed_services_all_rows_malformed_falls_back_to_default(tmp_path: Path) -> None:
+    extra = (
+        "\n[[web.managed_services]]\n"
+        'id = "broken"\n'
+        "allow_stop = true\n"
+    )
+    cfg = load_config(config_path=_write_toml(tmp_path, extra), env={})
+
+    assert cfg.web.managed_services == MANAGED_SERVICES
+
+
+def test_managed_services_wrong_type_for_allow_stop_is_skipped(tmp_path: Path) -> None:
+    extra = (
+        "\n[[web.managed_services]]\n"
+        'id = "act-runner"\n'
+        'unit = "act-runner.service"\n'
+        'label = "Runner"\n'
+        'allow_stop = "yes"\n'  # string, not bool -- must be rejected
+    )
+    cfg = load_config(config_path=_write_toml(tmp_path, extra), env={})
+
+    assert cfg.web.managed_services == MANAGED_SERVICES
+
+
+def test_managed_services_empty_list_falls_back_to_default(tmp_path: Path) -> None:
+    extra = "\n[web]\nmanaged_services = []\n"
+    cfg = load_config(config_path=_write_toml(tmp_path, extra), env={})
+
+    assert cfg.web.managed_services == MANAGED_SERVICES

tests/python/test_operations_services.py

@@ -0,0 +1,369 @@
+"""Tests for the pure ``operations.services`` layer (Stage D of the web-UI
+redesign) -- SECURITY-CRITICAL: this module builds the exact argv that ends
+up behind ``sudo`` on the live host.
+
+No real ``systemctl``/``sudo``/``systemd-run`` subprocess is ever invoked --
+every ``subprocess.run`` call is monkeypatched at the name this module
+imported it under (mirrors ``test_operations_host.py``'s convention).
+"""
+
+from __future__ import annotations
+
+import subprocess
+from typing import Any
+
+import pytest
+
+import ci_orchestrator.operations.host as host_module
+import ci_orchestrator.operations.services as services_module
+from ci_orchestrator.operations.services import (
+    ALLOWED_ACTIONS,
+    MANAGED_SERVICES,
+    ManagedService,
+    ServiceState,
+    control_service,
+    find_service,
+    service_states,
+)
+
+
+class _RunResult:
+    def __init__(self, stdout: str = "", stderr: str = "", returncode: int = 0) -> None:
+        self.stdout = stdout
+        self.stderr = stderr
+        self.returncode = returncode
+
+
+def _spy(real: Any) -> Any:
+    """Wrap ``real`` so ``.calls`` records every ``(args, kwargs)`` call, still delegating."""
+
+    def wrapper(*args: Any, **kwargs: Any) -> Any:
+        wrapper.calls.append((args, kwargs))  # type: ignore[attr-defined]
+        return real(*args, **kwargs)
+
+    wrapper.calls = []  # type: ignore[attr-defined]
+    return wrapper
+
+
+# ── MANAGED_SERVICES / find_service ─────────────────────────────────────────
+
+
+def test_managed_services_shape() -> None:
+    ids = {s.id for s in MANAGED_SERVICES}
+    assert ids == {"act-runner", "act-runner-rust", "vmware-host", "ci-web"}
+    by_id = {s.id: s for s in MANAGED_SERVICES}
+    assert by_id["ci-web"].allow_stop is False
+    assert by_id["ci-web"].unit == "ci-web.service"
+    assert by_id["act-runner"].allow_stop is True
+    assert by_id["vmware-host"].allow_stop is True
+
+
+def test_find_service_returns_allowlist_member() -> None:
+    svc = find_service("act-runner")
+    assert svc is not None
+    assert svc.unit == "act-runner.service"
+
+
+def test_find_service_unknown_id_returns_none() -> None:
+    assert find_service("does-not-exist") is None
+
+
+def test_find_service_uses_custom_services_tuple() -> None:
+    custom = (ManagedService("x", "x.service", "X", True),)
+    assert find_service("x", custom) is not None
+    # Not present in the default list, only in the custom one.
+    assert find_service("x", MANAGED_SERVICES) is None
+
+
+# ── service_states ───────────────────────────────────────────────────────────
+
+
+def test_service_states_reflects_active_and_inactive(monkeypatch: pytest.MonkeyPatch) -> None:
+    monkeypatch.setattr(host_module.shutil, "which", lambda _n: "/usr/bin/systemctl")
+
+    def _run(cmd: list[str], **_kw: Any) -> _RunResult:
+        unit = cmd[-1]
+        if cmd[1] == "is-active":
+            return _RunResult(stdout="active\n" if unit == "act-runner.service" else "inactive\n")
+        return _RunResult(stdout="running\n" if unit == "act-runner.service" else "dead\n")
+
+    monkeypatch.setattr(host_module.subprocess, "run", _run)
+
+    services = (
+        ManagedService("act-runner", "act-runner.service", "Runner", True),
+        ManagedService("ci-web", "ci-web.service", "Web UI", False),
+    )
+    states = service_states(services)
+
+    assert states == [
+        ServiceState(service=services[0], active=True, sub_state="running"),
+        ServiceState(service=services[1], active=False, sub_state="dead"),
+    ]
+
+
+def test_service_states_degrades_to_unknown_when_systemctl_missing(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    monkeypatch.setattr(host_module.shutil, "which", lambda _n: None)
+
+    states = service_states(MANAGED_SERVICES)
+
+    assert len(states) == len(MANAGED_SERVICES)
+    assert all(not s.active and s.sub_state == "unknown" for s in states)
+
+
+def test_service_states_degrades_to_unknown_on_subprocess_timeout(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    monkeypatch.setattr(host_module.shutil, "which", lambda _n: "/usr/bin/systemctl")
+
+    def _boom(*_a: Any, **_kw: Any) -> Any:
+        raise subprocess.TimeoutExpired(cmd="systemctl", timeout=10.0)
+
+    monkeypatch.setattr(host_module.subprocess, "run", _boom)
+
+    states = service_states(MANAGED_SERVICES)
+
+    assert all(not s.active and s.sub_state == "unknown" for s in states)
+
+
+def test_service_states_never_uses_shell_true(monkeypatch: pytest.MonkeyPatch) -> None:
+    monkeypatch.setattr(host_module.shutil, "which", lambda _n: "/usr/bin/systemctl")
+    seen_kwargs: list[dict[str, Any]] = []
+
+    def _run(cmd: list[str], **kw: Any) -> _RunResult:
+        seen_kwargs.append(kw)
+        return _RunResult(stdout="active\n")
+
+    monkeypatch.setattr(host_module.subprocess, "run", _run)
+    service_states(MANAGED_SERVICES)
+
+    assert seen_kwargs  # sanity: the fake was actually invoked
+    assert all(not kw.get("shell") for kw in seen_kwargs)
+
+
+# ── control_service: validation ──────────────────────────────────────────────
+
+
+def test_control_service_unsupported_action_raises_value_error(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    spy = _spy(subprocess.run)
+    monkeypatch.setattr(services_module.subprocess, "run", spy)
+    service = find_service("act-runner")
+    assert service is not None
+
+    with pytest.raises(ValueError):
+        control_service(service, "reboot")
+
+    assert spy.calls == []  # type: ignore[attr-defined]
+
+
+def test_control_service_stop_refused_for_disallowed_service_builds_no_command(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    spy = _spy(subprocess.run)
+    monkeypatch.setattr(services_module.subprocess, "run", spy)
+    ci_web = find_service("ci-web")
+    assert ci_web is not None
+
+    result = control_service(ci_web, "stop")
+
+    assert result.service_id == "ci-web"
+    assert result.action == "stop"
+    assert result.ok is False
+    assert result.command == []
+    assert "restart-only" in result.detail or "disabled" in result.detail
+    assert spy.calls == []  # type: ignore[attr-defined]
+
+
+def test_control_service_stop_refused_even_in_dry_run(monkeypatch: pytest.MonkeyPatch) -> None:
+    """The stop-refusal check must win over dry_run -- no command is built either way."""
+    spy = _spy(subprocess.run)
+    monkeypatch.setattr(services_module.subprocess, "run", spy)
+    ci_web = find_service("ci-web")
+    assert ci_web is not None
+
+    result = control_service(ci_web, "stop", dry_run=True)
+
+    assert result.ok is False
+    assert result.command == []
+    assert spy.calls == []  # type: ignore[attr-defined]
+
+
+# ── control_service: dry_run argv, incl. the ci-web restart special case ───
+
+
+@pytest.mark.parametrize("action", ALLOWED_ACTIONS)
+def test_control_service_dry_run_argv_for_every_normal_service(action: str) -> None:
+    for service in MANAGED_SERVICES:
+        if service.id == "ci-web":
+            continue  # covered separately below (special-cased argv)
+        if action == "stop" and not service.allow_stop:
+            continue  # covered by the refusal tests above
+        result = control_service(service, action, dry_run=True)
+
+        assert result.ok is True
+        assert result.command == ["sudo", "-n", "systemctl", action, service.unit]
+        assert result.detail.startswith("would run:")
+
+
+def test_control_service_dry_run_ci_web_start_and_stop_use_plain_systemctl() -> None:
+    ci_web = find_service("ci-web")
+    assert ci_web is not None
+
+    start_result = control_service(ci_web, "start", dry_run=True)
+    assert start_result.command == ["sudo", "-n", "systemctl", "start", "ci-web.service"]
+
+
+def test_control_service_dry_run_ci_web_restart_uses_systemd_run_deferred() -> None:
+    ci_web = find_service("ci-web")
+    assert ci_web is not None
+
+    result = control_service(ci_web, "restart", dry_run=True)
+
+    assert result.ok is True
+    assert result.command == [
+        "sudo",
+        "-n",
+        "systemd-run",
+        "--on-active=2",
+        "systemctl",
+        "restart",
+        "ci-web.service",
+    ]
+    assert "systemd-run" in result.detail
+    assert "--on-active=2" in result.detail
+
+
+def test_control_service_dry_run_never_executes_anything(monkeypatch: pytest.MonkeyPatch) -> None:
+    spy = _spy(subprocess.run)
+    monkeypatch.setattr(services_module.subprocess, "run", spy)
+    service = find_service("act-runner")
+    assert service is not None
+
+    control_service(service, "restart", dry_run=True)
+
+    assert spy.calls == []  # type: ignore[attr-defined]
+
+
+# ── control_service: real execution ──────────────────────────────────────────
+
+
+def test_control_service_real_run_success_builds_arg_list(monkeypatch: pytest.MonkeyPatch) -> None:
+    seen: list[tuple[Any, ...]] = []
+
+    def _run(cmd: list[str], **kw: Any) -> _RunResult:
+        seen.append(tuple(cmd))
+        assert not kw.get("shell")
+        return _RunResult(returncode=0)
+
+    monkeypatch.setattr(services_module.subprocess, "run", _run)
+    service = find_service("act-runner")
+    assert service is not None
+
+    result = control_service(service, "restart", dry_run=False)
+
+    assert result.ok is True
+    assert seen == [("sudo", "-n", "systemctl", "restart", "act-runner.service")]
+    assert seen[0][0:3] == ("sudo", "-n", "systemctl")
+    assert result.command == ["sudo", "-n", "systemctl", "restart", "act-runner.service"]
+
+
+def test_control_service_ci_web_restart_real_run_uses_systemd_run(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    seen: list[list[str]] = []
+
+    def _run(cmd: list[str], **_kw: Any) -> _RunResult:
+        seen.append(cmd)
+        return _RunResult(returncode=0)
+
+    monkeypatch.setattr(services_module.subprocess, "run", _run)
+    ci_web = find_service("ci-web")
+    assert ci_web is not None
+
+    result = control_service(ci_web, "restart", dry_run=False)
+
+    assert result.ok is True
+    assert "systemd-run" in seen[0]
+    assert "--on-active=2" in seen[0]
+    assert "scheduled" in result.detail
+
+
+def test_control_service_real_run_nonzero_exit_is_not_ok(monkeypatch: pytest.MonkeyPatch) -> None:
+    def _run(cmd: list[str], **_kw: Any) -> _RunResult:
+        return _RunResult(returncode=1, stderr="Unit act-runner.service not found.\n")
+
+    monkeypatch.setattr(services_module.subprocess, "run", _run)
+    service = find_service("act-runner")
+    assert service is not None
+
+    result = control_service(service, "start", dry_run=False)
+
+    assert result.ok is False
+    assert "not found" in result.detail
+    assert "1" in result.detail
+
+
+def test_control_service_real_run_nonzero_exit_no_stderr_still_reports_exit_code(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    def _run(cmd: list[str], **_kw: Any) -> _RunResult:
+        return _RunResult(returncode=1, stderr="")
+
+    monkeypatch.setattr(services_module.subprocess, "run", _run)
+    service = find_service("act-runner")
+    assert service is not None
+
+    result = control_service(service, "start", dry_run=False)
+
+    assert result.ok is False
+    assert "exit 1" in result.detail
+
+
+def test_control_service_timeout_degrades_to_error_never_raises(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    def _boom(*_a: Any, **_kw: Any) -> Any:
+        raise subprocess.TimeoutExpired(cmd="sudo", timeout=15.0)
+
+    monkeypatch.setattr(services_module.subprocess, "run", _boom)
+    service = find_service("act-runner")
+    assert service is not None
+
+    result = control_service(service, "restart", dry_run=False)
+
+    assert result.ok is False
+    assert "sudo" in result.detail or "restart" in result.detail.lower()
+
+
+def test_control_service_oserror_degrades_to_error_never_raises(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    def _boom(*_a: Any, **_kw: Any) -> Any:
+        raise OSError("sudo: command not found")
+
+    monkeypatch.setattr(services_module.subprocess, "run", _boom)
+    service = find_service("act-runner")
+    assert service is not None
+
+    result = control_service(service, "restart", dry_run=False)
+
+    assert result.ok is False
+    assert "sudo: command not found" in result.detail
+
+
+def test_control_service_stderr_tail_is_truncated(monkeypatch: pytest.MonkeyPatch) -> None:
+    huge_stderr = "x" * 5000
+
+    def _run(cmd: list[str], **_kw: Any) -> _RunResult:
+        return _RunResult(returncode=1, stderr=huge_stderr)
+
+    monkeypatch.setattr(services_module.subprocess, "run", _run)
+    service = find_service("act-runner")
+    assert service is not None
+
+    result = control_service(service, "start", dry_run=False)
+
+    assert len(result.detail) < len(huge_stderr) + 100

tests/python/test_web_stage_d.py

@@ -0,0 +1,648 @@
+"""Tests for the Stage D service-management web route -- SECURITY-CRITICAL:
+this route lets an authenticated web user start/stop/restart systemd
+services on the host via ``sudo``.
+
+Same conventions as ``test_web_actions.py``: no real ``vmrun``/``systemctl``/
+``sudo``/``systemd-run`` subprocess is ever invoked. ``operations.services.
+control_service`` is either spied on (still delegates to the real logic, so
+the confirm-gate/CSRF/audit plumbing is exercised for real) or has its
+underlying ``subprocess.run`` monkeypatched out from under it.
+"""
+
+from __future__ import annotations
+
+import re
+from pathlib import Path
+from typing import Any
+
+import pytest
+from fastapi.testclient import TestClient
+
+import ci_orchestrator.operations.services as services_module
+import ci_orchestrator.web.routes as routes_module
+from ci_orchestrator.config import (
+    BackendConfig,
+    CargoCacheConfig,
+    Config,
+    Paths,
+    WebConfig,
+    WebOidcConfig,
+)
+from ci_orchestrator.credentials import Credential, KeyringCredentialStore
+from ci_orchestrator.operations.audit import read_audit
+from ci_orchestrator.operations.services import (
+    MANAGED_SERVICES,
+    ManagedService,
+    control_service,
+)
+from ci_orchestrator.web.app import create_app, get_backend
+
+_SESSION_CRED_NAME = "TestWebSessionSecretD"
+_OIDC_CRED_NAME = "TestWebOidcClientSecretD"
+_FAKE_SESSION_SECRET = "sess-secret-do-not-leak-stage-d"
+_FAKE_OIDC_CLIENT_SECRET = "oidc-secret-do-not-leak-stage-d"
+
+
+@pytest.fixture(autouse=True)
+def _no_real_systemctl(monkeypatch: pytest.MonkeyPatch) -> None:
+    """Safety net: no test in this file may invoke a real ``systemctl``.
+
+    Every ``GET /services`` (including the ones ``_get_csrf`` issues just to
+    fetch a token) calls ``service_states()`` -> ``runner_status()``, which
+    otherwise would shell out for real. Defaulting ``shutil.which`` to
+    "systemctl not found" makes that read-only status query a guaranteed
+    no-subprocess no-op everywhere in this file; the handful of tests that
+    need a specific active/inactive shape re-patch both ``shutil.which`` and
+    ``subprocess.run`` locally, which simply overrides this default for the
+    duration of that one test.
+    """
+    import ci_orchestrator.operations.host as host_module
+
+    monkeypatch.setattr(host_module.shutil, "which", lambda _n: None)
+
+
+# ── shared fixtures / fakes ─────────────────────────────────────────────────
+
+
+def _make_config(
+    root: Path,
+    *,
+    managed_services: tuple[ManagedService, ...] | None = None,
+    active: bool = False,
+    allowed_users: list[str] | None = None,
+) -> Config:
+    oidc = None
+    if active:
+        oidc = WebOidcConfig(
+            issuer="https://gitea.example.lan",
+            client_id="ci-web",
+            redirect_url="http://127.0.0.1:8800/auth/callback",
+            client_secret_cred=_OIDC_CRED_NAME,
+        )
+    web_kwargs: dict[str, Any] = {
+        "audit_log": root / "web-audit.jsonl",
+        "session_secret_cred": _SESSION_CRED_NAME,
+        "enabled": active,
+        "allowed_users": allowed_users if allowed_users is not None else [],
+        "oidc": oidc,
+    }
+    if managed_services is not None:
+        web_kwargs["managed_services"] = managed_services
+
+    return Config(
+        paths=Paths(
+            root=root,
+            templates=root / "templates",
+            build_vms=root / "build-vms",
+            artifacts=root / "artifacts",
+            keys=root / "keys",
+        ),
+        backend=BackendConfig(),
+        cargo_cache=CargoCacheConfig(enabled=True, root=root / "cache" / "cargo", retention_days=30),
+        web=WebConfig(**web_kwargs),
+    )
+
+
+def _install_fake_keyring(monkeypatch: pytest.MonkeyPatch) -> None:
+    secrets = {
+        _SESSION_CRED_NAME: _FAKE_SESSION_SECRET,
+        _OIDC_CRED_NAME: _FAKE_OIDC_CLIENT_SECRET,
+    }
+
+    def fake_get(self: KeyringCredentialStore, target: str) -> Credential:
+        if target not in secrets:
+            raise KeyError(target)
+        return Credential(username=target, password=secrets[target])
+
+    monkeypatch.setattr(KeyringCredentialStore, "get", fake_get)
+
+
+class _FakeBackend:
+    """Stub ``VmBackend`` reporting a fixed number of running VMs."""
+
+    def __init__(self, running_count: int = 0) -> None:
+        self._running_count = running_count
+
+    def list_running(self) -> list[Any]:
+        return [object()] * self._running_count
+
+
+class _BoomBackend:
+    """A backend whose ``list_running`` always raises -- must degrade to 0, never 500."""
+
+    def list_running(self) -> list[Any]:
+        raise RuntimeError("backend hiccup")
+
+
+def _spy(real: Any) -> Any:
+    def wrapper(*args: Any, **kwargs: Any) -> Any:
+        wrapper.calls.append((args, kwargs))  # type: ignore[attr-defined]
+        return real(*args, **kwargs)
+
+    wrapper.calls = []  # type: ignore[attr-defined]
+    return wrapper
+
+
+def _csrf_from(text: str) -> str:
+    match = re.search(r'name="csrf" value="([^"]+)"', text)
+    assert match is not None, f"no csrf field found in: {text[:300]!r}"
+    return match.group(1)
+
+
+def _get_csrf(client: TestClient, route: str = "/services") -> str:
+    resp = client.get(route)
+    assert resp.status_code == 200
+    return _csrf_from(resp.text)
+
+
+def _last_audit_entry(audit_log: Path) -> Any:
+    entries = read_audit(audit_log, last=100)
+    assert entries, "expected at least one audit entry"
+    return entries[-1]
+
+
+def _quiet_subprocess_ok(monkeypatch: pytest.MonkeyPatch) -> list[list[str]]:
+    """Fake every real ``systemctl``/``sudo``/``systemd-run`` call as succeeding.
+
+    Returns the list argv's seen, in call order, for assertion.
+    """
+    seen: list[list[str]] = []
+
+    class _R:
+        returncode = 0
+        stdout = ""
+        stderr = ""
+
+    def _run(cmd: list[str], **_kw: Any) -> _R:
+        seen.append(list(cmd))
+        return _R()
+
+    monkeypatch.setattr(services_module.subprocess, "run", _run)
+    return seen
+
+
+# ── 1. unknown service_id -> 400, never calls control_service ──────────────
+
+
+def test_unknown_service_id_is_400_and_control_service_never_called(
+    tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    spy = _spy(control_service)
+    monkeypatch.setattr(routes_module, "control_service", spy)
+    config = _make_config(tmp_path)
+    app = create_app(config)
+    client = TestClient(app)
+    csrf = _get_csrf(client)
+
+    resp = client.post("/services/does-not-exist/restart", data={"csrf": csrf, "stage": "execute"})
+
+    assert resp.status_code == 400
+    assert spy.calls == []  # type: ignore[attr-defined]
+    assert not config.web.audit_log.exists()
+
+
+# ── 2. unsupported action -> 400, never calls control_service ──────────────
+
+
+def test_unsupported_action_is_400_and_control_service_never_called(
+    tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    spy = _spy(control_service)
+    monkeypatch.setattr(routes_module, "control_service", spy)
+    config = _make_config(tmp_path)
+    app = create_app(config)
+    client = TestClient(app)
+    csrf = _get_csrf(client)
+
+    resp = client.post("/services/act-runner/reboot", data={"csrf": csrf, "stage": "execute"})
+
+    assert resp.status_code == 400
+    assert spy.calls == []  # type: ignore[attr-defined]
+    assert not config.web.audit_log.exists()
+
+
+# ── 3. stop on a restart-only service (ci-web) -> 400, never called ────────
+
+
+def test_stop_on_ci_web_is_400_and_control_service_never_called(
+    tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    spy = _spy(control_service)
+    monkeypatch.setattr(routes_module, "control_service", spy)
+    config = _make_config(tmp_path)
+    app = create_app(config)
+    client = TestClient(app)
+    csrf = _get_csrf(client)
+
+    resp = client.post("/services/ci-web/stop", data={"csrf": csrf, "stage": "execute"})
+
+    assert resp.status_code == 400
+    assert spy.calls == []  # type: ignore[attr-defined]
+    assert not config.web.audit_log.exists()
+
+
+def test_stop_on_ci_web_preview_is_also_400_not_a_dry_run(
+    tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    """The refusal must win even on a preview (no ``stage=execute``) POST."""
+    spy = _spy(control_service)
+    monkeypatch.setattr(routes_module, "control_service", spy)
+    config = _make_config(tmp_path)
+    app = create_app(config)
+    client = TestClient(app)
+    csrf = _get_csrf(client)
+
+    resp = client.post("/services/ci-web/stop", data={"csrf": csrf})
+
+    assert resp.status_code == 400
+    assert spy.calls == []  # type: ignore[attr-defined]
+
+
+# ── 4. CSRF enforcement ──────────────────────────────────────────────────────
+
+
+def test_post_without_csrf_field_is_403(tmp_path: Path) -> None:
+    config = _make_config(tmp_path)
+    app = create_app(config)
+    client = TestClient(app)
+    client.get("/services")  # establish a session
+
+    resp = client.post("/services/act-runner/restart", data={})
+
+    assert resp.status_code == 403
+
+
+def test_post_with_wrong_csrf_is_403(tmp_path: Path) -> None:
+    config = _make_config(tmp_path)
+    app = create_app(config)
+    client = TestClient(app)
+    client.get("/services")
+
+    resp = client.post("/services/act-runner/restart", data={"csrf": "totally-not-the-token"})
+
+    assert resp.status_code == 403
+
+
+def test_wrong_csrf_never_calls_control_service(
+    tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    spy = _spy(control_service)
+    monkeypatch.setattr(routes_module, "control_service", spy)
+    config = _make_config(tmp_path)
+    app = create_app(config)
+    client = TestClient(app)
+    client.get("/services")
+
+    resp = client.post(
+        "/services/act-runner/restart", data={"csrf": "bogus", "stage": "execute"}
+    )
+
+    assert resp.status_code == 403
+    assert spy.calls == []  # type: ignore[attr-defined]
+
+
+# ── 5. two-step confirm gate ─────────────────────────────────────────────────
+
+
+def test_preview_returns_confirm_partial_dry_run_and_does_not_execute(
+    tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    spy = _spy(control_service)
+    monkeypatch.setattr(routes_module, "control_service", spy)
+    seen = _quiet_subprocess_ok(monkeypatch)
+    config = _make_config(tmp_path)
+    app = create_app(config)
+    client = TestClient(app)
+    csrf = _get_csrf(client)
+
+    resp = client.post("/services/act-runner/restart", data={"csrf": csrf})
+
+    assert resp.status_code == 200
+    assert "Confirm" in resp.text
+    assert len(spy.calls) == 1  # type: ignore[attr-defined]
+    assert spy.calls[0][1]["dry_run"] is True  # type: ignore[attr-defined]
+    assert seen == []  # dry_run never reaches subprocess.run
+
+    entry = _last_audit_entry(config.web.audit_log)
+    assert entry.result == "dry-run"
+    assert entry.action == "service.restart"
+    assert entry.target == "act-runner.service"
+
+
+def test_execute_runs_real_control_service_and_audits_ok(
+    tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    spy = _spy(control_service)
+    monkeypatch.setattr(routes_module, "control_service", spy)
+    seen = _quiet_subprocess_ok(monkeypatch)
+    config = _make_config(tmp_path)
+    app = create_app(config)
+    client = TestClient(app)
+    csrf = _get_csrf(client)
+
+    resp = client.post(
+        "/services/act-runner/restart", data={"csrf": csrf, "stage": "execute"}
+    )
+
+    assert resp.status_code == 200
+    assert len(spy.calls) == 1  # type: ignore[attr-defined]
+    assert spy.calls[0][1]["dry_run"] is False  # type: ignore[attr-defined]
+    assert seen == [["sudo", "-n", "systemctl", "restart", "act-runner.service"]]
+
+    entry = _last_audit_entry(config.web.audit_log)
+    assert entry.result == "ok"
+    assert entry.action == "service.restart"
+    assert entry.target == "act-runner.service"
+
+
+def test_execute_failure_audits_error_and_never_500s(
+    tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    class _R:
+        returncode = 1
+        stdout = ""
+        stderr = "Unit act-runner.service not found."
+
+    monkeypatch.setattr(services_module.subprocess, "run", lambda *_a, **_kw: _R())
+    config = _make_config(tmp_path)
+    app = create_app(config)
+    client = TestClient(app)
+    csrf = _get_csrf(client)
+
+    resp = client.post(
+        "/services/act-runner/restart", data={"csrf": csrf, "stage": "execute"}
+    )
+
+    assert resp.status_code == 200
+    assert "error" in resp.text.lower()
+    entry = _last_audit_entry(config.web.audit_log)
+    assert entry.result == "error"
+
+
+# ── 6. ci-web restart -> deferred systemd-run, never a direct restart ──────
+
+
+def test_ci_web_restart_execute_uses_systemd_run_not_direct_systemctl(
+    tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    seen = _quiet_subprocess_ok(monkeypatch)
+    config = _make_config(tmp_path)
+    app = create_app(config)
+    client = TestClient(app)
+    csrf = _get_csrf(client)
+
+    resp = client.post("/services/ci-web/restart", data={"csrf": csrf, "stage": "execute"})
+
+    assert resp.status_code == 200
+    assert len(seen) == 1
+    argv = seen[0]
+    assert "systemd-run" in argv
+    assert "--on-active=2" in argv
+    assert argv[0:3] == ["sudo", "-n", "systemd-run"]
+
+    entry = _last_audit_entry(config.web.audit_log)
+    assert entry.result == "ok"
+    assert entry.action == "service.restart"
+    assert entry.target == "ci-web.service"
+
+
+def test_ci_web_restart_preview_never_touches_subprocess(
+    tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    seen = _quiet_subprocess_ok(monkeypatch)
+    config = _make_config(tmp_path)
+    app = create_app(config)
+    client = TestClient(app)
+    csrf = _get_csrf(client)
+
+    resp = client.post("/services/ci-web/restart", data={"csrf": csrf})
+
+    assert resp.status_code == 200
+    assert "Confirm" in resp.text
+    assert seen == []
+
+
+# ── 7. vmware-host confirm step: active-VM count warning ───────────────────
+
+
+def test_vmware_host_confirm_shows_active_vm_warning(
+    tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    _quiet_subprocess_ok(monkeypatch)
+    config = _make_config(tmp_path)
+    app = create_app(config)
+    app.dependency_overrides[get_backend] = lambda: _FakeBackend(running_count=3)
+    client = TestClient(app)
+    csrf = _get_csrf(client)
+
+    resp = client.post("/services/vmware-host/stop", data={"csrf": csrf})
+
+    assert resp.status_code == 200
+    assert "3 active VM" in resp.text
+    assert "interrupts running jobs" in resp.text
+
+
+def test_vmware_host_confirm_no_warning_when_no_active_vms(
+    tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    _quiet_subprocess_ok(monkeypatch)
+    config = _make_config(tmp_path)
+    app = create_app(config)
+    app.dependency_overrides[get_backend] = lambda: _FakeBackend(running_count=0)
+    client = TestClient(app)
+    csrf = _get_csrf(client)
+
+    resp = client.post("/services/vmware-host/restart", data={"csrf": csrf})
+
+    assert resp.status_code == 200
+    assert "interrupts running jobs" not in resp.text
+
+
+def test_vmware_host_confirm_start_never_shows_warning_even_with_active_vms(
+    tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    _quiet_subprocess_ok(monkeypatch)
+    config = _make_config(tmp_path)
+    app = create_app(config)
+    app.dependency_overrides[get_backend] = lambda: _FakeBackend(running_count=5)
+    client = TestClient(app)
+    csrf = _get_csrf(client)
+
+    resp = client.post("/services/vmware-host/start", data={"csrf": csrf})
+
+    assert resp.status_code == 200
+    assert "interrupts running jobs" not in resp.text
+
+
+def test_vmware_host_confirm_backend_unavailable_degrades_to_no_warning(
+    tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    """Backend missing (``vmrun`` not found) -> 0 active VMs, no warning, no 500."""
+    _quiet_subprocess_ok(monkeypatch)
+    config = _make_config(tmp_path)
+    app = create_app(config)
+    app.dependency_overrides[get_backend] = lambda: None
+    client = TestClient(app)
+    csrf = _get_csrf(client)
+
+    resp = client.post("/services/vmware-host/stop", data={"csrf": csrf})
+
+    assert resp.status_code == 200
+    assert "interrupts running jobs" not in resp.text
+
+
+def test_vmware_host_confirm_backend_exception_degrades_to_no_warning(
+    tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    _quiet_subprocess_ok(monkeypatch)
+    config = _make_config(tmp_path)
+    app = create_app(config)
+    app.dependency_overrides[get_backend] = lambda: _BoomBackend()
+    client = TestClient(app)
+    csrf = _get_csrf(client)
+
+    resp = client.post("/services/vmware-host/restart", data={"csrf": csrf})
+
+    assert resp.status_code == 200
+    assert "interrupts running jobs" not in resp.text
+
+
+def test_non_vmware_host_service_confirm_never_shows_vm_warning(
+    tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    _quiet_subprocess_ok(monkeypatch)
+    config = _make_config(tmp_path)
+    app = create_app(config)
+    app.dependency_overrides[get_backend] = lambda: _FakeBackend(running_count=7)
+    client = TestClient(app)
+    csrf = _get_csrf(client)
+
+    resp = client.post("/services/act-runner/restart", data={"csrf": csrf})
+
+    assert resp.status_code == 200
+    assert "interrupts running jobs" not in resp.text
+
+
+# ── 8. custom config.web.managed_services override is actually wired in ────
+
+
+def test_route_uses_configured_managed_services_override(
+    tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    seen = _quiet_subprocess_ok(monkeypatch)
+    custom = (ManagedService("custom-svc", "custom.service", "Custom", True),)
+    config = _make_config(tmp_path, managed_services=custom)
+    app = create_app(config)
+    client = TestClient(app)
+    csrf = _get_csrf(client)
+
+    # The default allow-list's "act-runner" must NOT be reachable once overridden.
+    resp_default = client.post(
+        "/services/act-runner/restart", data={"csrf": csrf, "stage": "execute"}
+    )
+    assert resp_default.status_code == 400
+
+    resp_custom = client.post(
+        "/services/custom-svc/restart", data={"csrf": csrf, "stage": "execute"}
+    )
+    assert resp_custom.status_code == 200
+    assert seen == [["sudo", "-n", "systemctl", "restart", "custom.service"]]
+
+
+# ── 9. unauthenticated POST -> redirect to /login, never executes ──────────
+
+
+def test_unauthenticated_post_redirects_to_login_and_never_runs(
+    tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    _install_fake_keyring(monkeypatch)
+    seen = _quiet_subprocess_ok(monkeypatch)
+    config = _make_config(tmp_path, active=True, allowed_users=["simone"])
+    app = create_app(config)
+    client = TestClient(app, follow_redirects=False)
+
+    resp = client.post(
+        "/services/act-runner/restart", data={"csrf": "whatever", "stage": "execute"}
+    )
+
+    assert resp.status_code in (302, 303)
+    assert "/login" in resp.headers["location"]
+    assert seen == []
+    assert not config.web.audit_log.exists()
+
+
+# ── 10. GET /services partial: state-aware buttons ──────────────────────────
+
+
+def test_services_partial_all_inactive_shows_start_never_stop(
+    tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    # No real systemctl call: force runner_status() to degrade every unit to
+    # inactive/unknown, deterministically, regardless of the test host.
+    import ci_orchestrator.operations.host as host_module
+
+    monkeypatch.setattr(host_module.shutil, "which", lambda _n: None)
+    config = _make_config(tmp_path)
+    app = create_app(config)
+    client = TestClient(app)
+
+    resp = client.get("/services")
+
+    assert resp.status_code == 200
+    assert "Start" in resp.text
+    assert ">Stop<" not in resp.text
+    assert "Restart" in resp.text
+
+
+def test_services_partial_active_shows_stop_except_for_ci_web(
+    tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    import ci_orchestrator.operations.host as host_module
+
+    monkeypatch.setattr(host_module.shutil, "which", lambda _n: "/usr/bin/systemctl")
+
+    class _R:
+        def __init__(self, stdout: str) -> None:
+            self.stdout = stdout
+            self.returncode = 0
+
+    def _run(cmd: list[str], **_kw: Any) -> _R:
+        if cmd[1] == "is-active":
+            return _R("active\n")
+        return _R("running\n")
+
+    monkeypatch.setattr(host_module.subprocess, "run", _run)
+    config = _make_config(tmp_path)
+    app = create_app(config)
+    client = TestClient(app)
+
+    resp = client.get("/services")
+
+    assert resp.status_code == 200
+    assert 'aria-label="Stop Runner"' in resp.text
+    assert 'aria-label="Stop VMware host"' in resp.text
+    assert 'aria-label="Stop Web UI"' not in resp.text  # ci-web: allow_stop=False
+
+
+def test_dashboard_includes_services_panel(tmp_path: Path) -> None:
+    config = _make_config(tmp_path)
+    app = create_app(config)
+    client = TestClient(app)
+
+    resp = client.get("/")
+
+    assert resp.status_code == 200
+    assert "Services" in resp.text
+    assert 'id="services-card"' in resp.text
+
+
+# ── extra: find_service / MANAGED_SERVICES sanity from the route's angle ───
+
+
+def test_find_service_via_default_allowlist_matches_module_constant() -> None:
+    from ci_orchestrator.operations.services import find_service
+
+    assert find_service("act-runner") is not None
+    assert find_service("nonexistent-slug") is None
+    for service in MANAGED_SERVICES:
+        assert find_service(service.id) == service