fix(web): Cloudflare UA block, Gitea path@ref, attempt 0

Three bugs the real API surfaced, none visible against a stubbed payload:

- Gitea here is proxied by Cloudflare, which answers urllib's default
  `Python-urllib/3.x` agent with 403 + "error code: 1010" before the request
  reaches Gitea — every endpoint 403s, including unauthenticated `/version`,
  so it reads exactly like a token problem. Send an explicit User-Agent.
- Gitea reports `path` as either `.gitea/workflows/lint.yml` or the
  `lint.yml@refs/heads/main` form; splitting on "/" alone rendered the latter
  as the workflow name "main". Drop the ref first.
- Gitea reports `run_attempt: 0` for a never-retried run while the job id the
  workflow builds uses github.run_attempt, which starts at 1 — so 0 must read
  as 1 or no VM would ever correlate to its run. (The existing `or 1` already
  did this; it now has a test naming why, against the real payload shape.)

HTTP failures now carry the server's own message (Gitea names the missing
scope; a Cloudflare interception says something else entirely) instead of a
bare status code that sent this investigation the wrong way once already.
This commit is contained in:
2026-07-25 15:27:01 +02:00
parent d0ea7f661d
commit 91ea90348c
3 changed files with 158 additions and 6 deletions
+10 -1
View File
@@ -318,7 +318,16 @@ sudo -u ci-runner /opt/ci/venv/bin/python -m ci_orchestrator creds set \
**Degraded states** never take the card down: no token, an expired token, or
an unreachable Gitea each render a warning notice *and still list every
running VM* below it, so the dashboard is never less useful than the VM table
it replaced.
it replaced. The notice includes whatever the server said (Gitea names the
missing scope in its own body), which is the fastest way to tell a token
problem from something in front of Gitea.
**Gotcha — Cloudflare in front of Gitea.** `gitea.emulab.it` is proxied by
Cloudflare, which rejects urllib's default `Python-urllib/3.x` user agent with
`403` + `error code: 1010` *before the request reaches Gitea* — indistinguishable
from a token refusal if you only look at the status code. `operations/executions.py`
therefore always sends an explicit `User-Agent`; any ordinary agent string
passes. Do not remove that header.
**Orphaned VMs are not handled in this card.** A powered-on VM whose run is no
longer in the window (or that no run ever claimed) keeps a row of its own
+53 -3
View File
@@ -47,6 +47,13 @@ logger = logging.getLogger(__name__)
# request open long enough to pile polls up on top of each other.
_TIMEOUT_SEC = 6.0
# MUST be set explicitly. The Gitea instance this talks to sits behind
# Cloudflare, which answers urllib's default ``Python-urllib/3.x`` agent with a
# 403 ("error code: 1010") before the request ever reaches Gitea -- an opaque
# failure that looks exactly like a token problem. Any ordinary agent string
# passes.
_USER_AGENT = "ci-orchestrator (+local CI dashboard)"
# One API call per window at most, however many pollers ask.
_TTL_SEC = 10.0
@@ -178,6 +185,16 @@ def _timestamp(payload: dict[str, Any], key: str) -> datetime | None:
return parsed
def _workflow_name(path: str) -> str:
"""Workflow file name out of Gitea's ``path``.
Gitea sends either a plain path (``.gitea/workflows/lint.yml``) or the
``<file>@<ref>`` form (``lint.yml@refs/heads/main``). Splitting on ``/``
alone would turn the latter into ``"main"``, so the ref is dropped first.
"""
return path.split("@", 1)[0].rsplit("/", 1)[-1]
def _map_status(status: str, conclusion: str) -> str:
"""Collapse Gitea's ``status`` + ``conclusion`` pair into one card state."""
mapped = _STATUS_MAP.get(status)
@@ -211,8 +228,11 @@ def _to_execution(payload: dict[str, Any], now: datetime) -> Execution:
return Execution(
run_id=_int(payload, "id"),
number=_int(payload, "run_number"),
# Gitea reports ``run_attempt = 0`` for a first (never-retried) run,
# while the job id the workflow builds uses github.run_attempt, which
# starts at 1 -- so 0 must read as 1 or no VM would ever match.
attempt=_int(payload, "run_attempt", 1) or 1,
workflow=_get(payload, "path").rsplit("/", 1)[-1],
workflow=_workflow_name(_get(payload, "path")),
repo=repo,
branch=_get(payload, "head_branch"),
commit=_get(payload, "head_sha"),
@@ -230,7 +250,12 @@ def _fetch(url: str, token: str) -> list[dict[str, Any]]:
"""GET one runs endpoint. Raises on transport/HTTP/payload trouble."""
# URL is built from the configured Gitea base URL, never user input.
request = urllib.request.Request(
url, headers={"Authorization": f"Bearer {token}", "Accept": "application/json"}
url,
headers={
"Authorization": f"Bearer {token}",
"Accept": "application/json",
"User-Agent": _USER_AGENT,
},
)
with urllib.request.urlopen(request, timeout=_TIMEOUT_SEC) as response:
body = json.loads(response.read().decode("utf-8"))
@@ -240,6 +265,31 @@ def _fetch(url: str, token: str) -> list[dict[str, Any]]:
return [item for item in runs if isinstance(item, dict)]
def _http_detail(exc: urllib.error.HTTPError, *, limit: int = 160) -> str:
"""Whatever the server said about the refusal, trimmed for a card notice.
Worth surfacing: Gitea's own body names the missing scope
(``required=[read:user], token scope=...``), and a Cloudflare interception
in front of it says something else entirely. Guessing from the status code
alone sent this exact investigation down the wrong path once already.
Unreadable bodies degrade to ``""`` -- this must never mask the real error.
"""
try:
raw = exc.read().decode("utf-8", errors="replace").strip()
except Exception:
return ""
if not raw:
return ""
try:
parsed = json.loads(raw)
except json.JSONDecodeError:
detail = " ".join(raw.split())
else:
message = parsed.get("message") if isinstance(parsed, dict) else None
detail = str(message) if message else " ".join(raw.split())
return detail[:limit]
def _fetch_runs(
base_url: str, token: str, limit: int
) -> tuple[list[dict[str, Any]], str, str | None, bool, bool]:
@@ -258,7 +308,7 @@ def _fetch_runs(
try:
return _fetch(url, token), source, None, False, False
except urllib.error.HTTPError as exc:
last_error = f"Gitea API returned HTTP {exc.code}."
last_error = f"Gitea API returned HTTP {exc.code}. {_http_detail(exc)}".strip()
if exc.code == 401:
auth_failed = True
refused = True
+95 -2
View File
@@ -100,9 +100,13 @@ class _FakeResponse:
return None
def _http_error(code: int) -> urllib.error.HTTPError:
def _http_error(code: int, body: str = "") -> urllib.error.HTTPError:
return urllib.error.HTTPError(
url="https://gitea.example", code=code, msg="nope", hdrs=None, fp=None
url="https://gitea.example",
code=code,
msg="nope",
hdrs=None, # type: ignore[arg-type]
fp=BytesIO(body.encode("utf-8")) if body else None,
)
@@ -192,6 +196,95 @@ def test_admin_payload_parsed_into_executions(
]
def test_request_sends_an_explicit_user_agent(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
"""Cloudflare in front of Gitea 403s urllib's default agent (error 1010)."""
agents: list[str | None] = []
def _fake_urlopen(request: Any, timeout: float | None = None) -> Any:
agents.append(request.get_header("User-agent"))
return _FakeResponse({"workflow_runs": []})
monkeypatch.setattr(ex.urllib.request, "urlopen", _fake_urlopen)
list_executions(_make_config(tmp_path), [], token="tok", now=_NOW)
assert agents and agents[0]
assert "python-urllib" not in str(agents[0]).lower()
@pytest.mark.parametrize(
("path", "expected"),
[
(".gitea/workflows/build-ns7zip.yml", "build-ns7zip.yml"),
# Gitea's <file>@<ref> form: splitting on "/" alone yields "main".
("lint.yml@refs/heads/main", "lint.yml"),
(".gitea/workflows/ci.yml@refs/heads/feat/x", "ci.yml"),
("", ""),
],
)
def test_workflow_name_drops_the_ref(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch, path: str, expected: str
) -> None:
_install_urlopen(monkeypatch, _admin_ok([_run(path=path)]))
view = list_executions(_make_config(tmp_path), [], token="tok", now=_NOW)
assert view.runs[0].workflow == expected
def test_zero_attempt_reads_as_one(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
"""Gitea sends ``run_attempt: 0`` for a first run; job ids start at 1."""
_install_urlopen(monkeypatch, _admin_ok([_run(run_id=384, attempt=0)]))
vm = _vm("Clone_384-1_20260725_115000_linux")
view = list_executions(_make_config(tmp_path), [vm], token="tok", now=_NOW)
assert view.runs[0].attempt == 1
assert view.runs[0].vms == [vm]
def test_http_error_carries_the_server_message(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
"""Gitea names the missing scope in its body -- surface it, don't guess."""
body = json.dumps(
{"message": "token does not have at least one of required scope(s)"}
)
_install_urlopen(monkeypatch, lambda _url: _http_error(500, body))
view = list_executions(_make_config(tmp_path), [], token="tok", now=_NOW)
assert view.error is not None
assert "HTTP 500" in view.error
assert "required scope(s)" in view.error
def test_non_json_error_body_is_included_verbatim(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
"""A Cloudflare interception is HTML, not Gitea JSON -- still show it."""
_install_urlopen(monkeypatch, lambda _url: _http_error(500, "<html>error code: 1010</html>"))
view = list_executions(_make_config(tmp_path), [], token="tok", now=_NOW)
assert view.error is not None
assert "1010" in view.error
def test_empty_error_body_leaves_a_bare_status(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
_install_urlopen(monkeypatch, lambda _url: _http_error(500))
view = list_executions(_make_config(tmp_path), [], token="tok", now=_NOW)
assert view.error == "Gitea API returned HTTP 500."
def test_limit_argument_overrides_config(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None: