diff --git a/docs/WEB-UI.md b/docs/WEB-UI.md index e37a09d..3e38225 100644 --- a/docs/WEB-UI.md +++ b/docs/WEB-UI.md @@ -318,7 +318,16 @@ sudo -u ci-runner /opt/ci/venv/bin/python -m ci_orchestrator creds set \ **Degraded states** never take the card down: no token, an expired token, or an unreachable Gitea each render a warning notice *and still list every running VM* below it, so the dashboard is never less useful than the VM table -it replaced. +it replaced. The notice includes whatever the server said (Gitea names the +missing scope in its own body), which is the fastest way to tell a token +problem from something in front of Gitea. + +**Gotcha — Cloudflare in front of Gitea.** `gitea.emulab.it` is proxied by +Cloudflare, which rejects urllib's default `Python-urllib/3.x` user agent with +`403` + `error code: 1010` *before the request reaches Gitea* — indistinguishable +from a token refusal if you only look at the status code. `operations/executions.py` +therefore always sends an explicit `User-Agent`; any ordinary agent string +passes. Do not remove that header. **Orphaned VMs are not handled in this card.** A powered-on VM whose run is no longer in the window (or that no run ever claimed) keeps a row of its own diff --git a/src/ci_orchestrator/operations/executions.py b/src/ci_orchestrator/operations/executions.py index 03806ed..3250c6a 100644 --- a/src/ci_orchestrator/operations/executions.py +++ b/src/ci_orchestrator/operations/executions.py @@ -47,6 +47,13 @@ logger = logging.getLogger(__name__) # request open long enough to pile polls up on top of each other. _TIMEOUT_SEC = 6.0 +# MUST be set explicitly. The Gitea instance this talks to sits behind +# Cloudflare, which answers urllib's default ``Python-urllib/3.x`` agent with a +# 403 ("error code: 1010") before the request ever reaches Gitea -- an opaque +# failure that looks exactly like a token problem. Any ordinary agent string +# passes. +_USER_AGENT = "ci-orchestrator (+local CI dashboard)" + # One API call per window at most, however many pollers ask. _TTL_SEC = 10.0 @@ -178,6 +185,16 @@ def _timestamp(payload: dict[str, Any], key: str) -> datetime | None: return parsed +def _workflow_name(path: str) -> str: + """Workflow file name out of Gitea's ``path``. + + Gitea sends either a plain path (``.gitea/workflows/lint.yml``) or the + ``@`` form (``lint.yml@refs/heads/main``). Splitting on ``/`` + alone would turn the latter into ``"main"``, so the ref is dropped first. + """ + return path.split("@", 1)[0].rsplit("/", 1)[-1] + + def _map_status(status: str, conclusion: str) -> str: """Collapse Gitea's ``status`` + ``conclusion`` pair into one card state.""" mapped = _STATUS_MAP.get(status) @@ -211,8 +228,11 @@ def _to_execution(payload: dict[str, Any], now: datetime) -> Execution: return Execution( run_id=_int(payload, "id"), number=_int(payload, "run_number"), + # Gitea reports ``run_attempt = 0`` for a first (never-retried) run, + # while the job id the workflow builds uses github.run_attempt, which + # starts at 1 -- so 0 must read as 1 or no VM would ever match. attempt=_int(payload, "run_attempt", 1) or 1, - workflow=_get(payload, "path").rsplit("/", 1)[-1], + workflow=_workflow_name(_get(payload, "path")), repo=repo, branch=_get(payload, "head_branch"), commit=_get(payload, "head_sha"), @@ -230,7 +250,12 @@ def _fetch(url: str, token: str) -> list[dict[str, Any]]: """GET one runs endpoint. Raises on transport/HTTP/payload trouble.""" # URL is built from the configured Gitea base URL, never user input. request = urllib.request.Request( - url, headers={"Authorization": f"Bearer {token}", "Accept": "application/json"} + url, + headers={ + "Authorization": f"Bearer {token}", + "Accept": "application/json", + "User-Agent": _USER_AGENT, + }, ) with urllib.request.urlopen(request, timeout=_TIMEOUT_SEC) as response: body = json.loads(response.read().decode("utf-8")) @@ -240,6 +265,31 @@ def _fetch(url: str, token: str) -> list[dict[str, Any]]: return [item for item in runs if isinstance(item, dict)] +def _http_detail(exc: urllib.error.HTTPError, *, limit: int = 160) -> str: + """Whatever the server said about the refusal, trimmed for a card notice. + + Worth surfacing: Gitea's own body names the missing scope + (``required=[read:user], token scope=...``), and a Cloudflare interception + in front of it says something else entirely. Guessing from the status code + alone sent this exact investigation down the wrong path once already. + Unreadable bodies degrade to ``""`` -- this must never mask the real error. + """ + try: + raw = exc.read().decode("utf-8", errors="replace").strip() + except Exception: + return "" + if not raw: + return "" + try: + parsed = json.loads(raw) + except json.JSONDecodeError: + detail = " ".join(raw.split()) + else: + message = parsed.get("message") if isinstance(parsed, dict) else None + detail = str(message) if message else " ".join(raw.split()) + return detail[:limit] + + def _fetch_runs( base_url: str, token: str, limit: int ) -> tuple[list[dict[str, Any]], str, str | None, bool, bool]: @@ -258,7 +308,7 @@ def _fetch_runs( try: return _fetch(url, token), source, None, False, False except urllib.error.HTTPError as exc: - last_error = f"Gitea API returned HTTP {exc.code}." + last_error = f"Gitea API returned HTTP {exc.code}. {_http_detail(exc)}".strip() if exc.code == 401: auth_failed = True refused = True diff --git a/tests/python/test_operations_executions.py b/tests/python/test_operations_executions.py index d90852e..971833e 100644 --- a/tests/python/test_operations_executions.py +++ b/tests/python/test_operations_executions.py @@ -100,9 +100,13 @@ class _FakeResponse: return None -def _http_error(code: int) -> urllib.error.HTTPError: +def _http_error(code: int, body: str = "") -> urllib.error.HTTPError: return urllib.error.HTTPError( - url="https://gitea.example", code=code, msg="nope", hdrs=None, fp=None + url="https://gitea.example", + code=code, + msg="nope", + hdrs=None, # type: ignore[arg-type] + fp=BytesIO(body.encode("utf-8")) if body else None, ) @@ -192,6 +196,95 @@ def test_admin_payload_parsed_into_executions( ] +def test_request_sends_an_explicit_user_agent( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """Cloudflare in front of Gitea 403s urllib's default agent (error 1010).""" + agents: list[str | None] = [] + + def _fake_urlopen(request: Any, timeout: float | None = None) -> Any: + agents.append(request.get_header("User-agent")) + return _FakeResponse({"workflow_runs": []}) + + monkeypatch.setattr(ex.urllib.request, "urlopen", _fake_urlopen) + + list_executions(_make_config(tmp_path), [], token="tok", now=_NOW) + + assert agents and agents[0] + assert "python-urllib" not in str(agents[0]).lower() + + +@pytest.mark.parametrize( + ("path", "expected"), + [ + (".gitea/workflows/build-ns7zip.yml", "build-ns7zip.yml"), + # Gitea's @ form: splitting on "/" alone yields "main". + ("lint.yml@refs/heads/main", "lint.yml"), + (".gitea/workflows/ci.yml@refs/heads/feat/x", "ci.yml"), + ("", ""), + ], +) +def test_workflow_name_drops_the_ref( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, path: str, expected: str +) -> None: + _install_urlopen(monkeypatch, _admin_ok([_run(path=path)])) + + view = list_executions(_make_config(tmp_path), [], token="tok", now=_NOW) + + assert view.runs[0].workflow == expected + + +def test_zero_attempt_reads_as_one( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """Gitea sends ``run_attempt: 0`` for a first run; job ids start at 1.""" + _install_urlopen(monkeypatch, _admin_ok([_run(run_id=384, attempt=0)])) + vm = _vm("Clone_384-1_20260725_115000_linux") + + view = list_executions(_make_config(tmp_path), [vm], token="tok", now=_NOW) + + assert view.runs[0].attempt == 1 + assert view.runs[0].vms == [vm] + + +def test_http_error_carries_the_server_message( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """Gitea names the missing scope in its body -- surface it, don't guess.""" + body = json.dumps( + {"message": "token does not have at least one of required scope(s)"} + ) + _install_urlopen(monkeypatch, lambda _url: _http_error(500, body)) + + view = list_executions(_make_config(tmp_path), [], token="tok", now=_NOW) + + assert view.error is not None + assert "HTTP 500" in view.error + assert "required scope(s)" in view.error + + +def test_non_json_error_body_is_included_verbatim( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """A Cloudflare interception is HTML, not Gitea JSON -- still show it.""" + _install_urlopen(monkeypatch, lambda _url: _http_error(500, "error code: 1010")) + + view = list_executions(_make_config(tmp_path), [], token="tok", now=_NOW) + + assert view.error is not None + assert "1010" in view.error + + +def test_empty_error_body_leaves_a_bare_status( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + _install_urlopen(monkeypatch, lambda _url: _http_error(500)) + + view = list_executions(_make_config(tmp_path), [], token="tok", now=_NOW) + + assert view.error == "Gitea API returned HTTP 500." + + def test_limit_argument_overrides_config( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: