fix(web): Cloudflare UA block, Gitea path@ref, attempt 0
Three bugs the real API surfaced, none visible against a stubbed payload: - Gitea here is proxied by Cloudflare, which answers urllib's default `Python-urllib/3.x` agent with 403 + "error code: 1010" before the request reaches Gitea — every endpoint 403s, including unauthenticated `/version`, so it reads exactly like a token problem. Send an explicit User-Agent. - Gitea reports `path` as either `.gitea/workflows/lint.yml` or the `lint.yml@refs/heads/main` form; splitting on "/" alone rendered the latter as the workflow name "main". Drop the ref first. - Gitea reports `run_attempt: 0` for a never-retried run while the job id the workflow builds uses github.run_attempt, which starts at 1 — so 0 must read as 1 or no VM would ever correlate to its run. (The existing `or 1` already did this; it now has a test naming why, against the real payload shape.) HTTP failures now carry the server's own message (Gitea names the missing scope; a Cloudflare interception says something else entirely) instead of a bare status code that sent this investigation the wrong way once already.
This commit is contained in:
+10
-1
@@ -318,7 +318,16 @@ sudo -u ci-runner /opt/ci/venv/bin/python -m ci_orchestrator creds set \
|
||||
**Degraded states** never take the card down: no token, an expired token, or
|
||||
an unreachable Gitea each render a warning notice *and still list every
|
||||
running VM* below it, so the dashboard is never less useful than the VM table
|
||||
it replaced.
|
||||
it replaced. The notice includes whatever the server said (Gitea names the
|
||||
missing scope in its own body), which is the fastest way to tell a token
|
||||
problem from something in front of Gitea.
|
||||
|
||||
**Gotcha — Cloudflare in front of Gitea.** `gitea.emulab.it` is proxied by
|
||||
Cloudflare, which rejects urllib's default `Python-urllib/3.x` user agent with
|
||||
`403` + `error code: 1010` *before the request reaches Gitea* — indistinguishable
|
||||
from a token refusal if you only look at the status code. `operations/executions.py`
|
||||
therefore always sends an explicit `User-Agent`; any ordinary agent string
|
||||
passes. Do not remove that header.
|
||||
|
||||
**Orphaned VMs are not handled in this card.** A powered-on VM whose run is no
|
||||
longer in the window (or that no run ever claimed) keeps a row of its own
|
||||
|
||||
@@ -47,6 +47,13 @@ logger = logging.getLogger(__name__)
|
||||
# request open long enough to pile polls up on top of each other.
|
||||
_TIMEOUT_SEC = 6.0
|
||||
|
||||
# MUST be set explicitly. The Gitea instance this talks to sits behind
|
||||
# Cloudflare, which answers urllib's default ``Python-urllib/3.x`` agent with a
|
||||
# 403 ("error code: 1010") before the request ever reaches Gitea -- an opaque
|
||||
# failure that looks exactly like a token problem. Any ordinary agent string
|
||||
# passes.
|
||||
_USER_AGENT = "ci-orchestrator (+local CI dashboard)"
|
||||
|
||||
# One API call per window at most, however many pollers ask.
|
||||
_TTL_SEC = 10.0
|
||||
|
||||
@@ -178,6 +185,16 @@ def _timestamp(payload: dict[str, Any], key: str) -> datetime | None:
|
||||
return parsed
|
||||
|
||||
|
||||
def _workflow_name(path: str) -> str:
|
||||
"""Workflow file name out of Gitea's ``path``.
|
||||
|
||||
Gitea sends either a plain path (``.gitea/workflows/lint.yml``) or the
|
||||
``<file>@<ref>`` form (``lint.yml@refs/heads/main``). Splitting on ``/``
|
||||
alone would turn the latter into ``"main"``, so the ref is dropped first.
|
||||
"""
|
||||
return path.split("@", 1)[0].rsplit("/", 1)[-1]
|
||||
|
||||
|
||||
def _map_status(status: str, conclusion: str) -> str:
|
||||
"""Collapse Gitea's ``status`` + ``conclusion`` pair into one card state."""
|
||||
mapped = _STATUS_MAP.get(status)
|
||||
@@ -211,8 +228,11 @@ def _to_execution(payload: dict[str, Any], now: datetime) -> Execution:
|
||||
return Execution(
|
||||
run_id=_int(payload, "id"),
|
||||
number=_int(payload, "run_number"),
|
||||
# Gitea reports ``run_attempt = 0`` for a first (never-retried) run,
|
||||
# while the job id the workflow builds uses github.run_attempt, which
|
||||
# starts at 1 -- so 0 must read as 1 or no VM would ever match.
|
||||
attempt=_int(payload, "run_attempt", 1) or 1,
|
||||
workflow=_get(payload, "path").rsplit("/", 1)[-1],
|
||||
workflow=_workflow_name(_get(payload, "path")),
|
||||
repo=repo,
|
||||
branch=_get(payload, "head_branch"),
|
||||
commit=_get(payload, "head_sha"),
|
||||
@@ -230,7 +250,12 @@ def _fetch(url: str, token: str) -> list[dict[str, Any]]:
|
||||
"""GET one runs endpoint. Raises on transport/HTTP/payload trouble."""
|
||||
# URL is built from the configured Gitea base URL, never user input.
|
||||
request = urllib.request.Request(
|
||||
url, headers={"Authorization": f"Bearer {token}", "Accept": "application/json"}
|
||||
url,
|
||||
headers={
|
||||
"Authorization": f"Bearer {token}",
|
||||
"Accept": "application/json",
|
||||
"User-Agent": _USER_AGENT,
|
||||
},
|
||||
)
|
||||
with urllib.request.urlopen(request, timeout=_TIMEOUT_SEC) as response:
|
||||
body = json.loads(response.read().decode("utf-8"))
|
||||
@@ -240,6 +265,31 @@ def _fetch(url: str, token: str) -> list[dict[str, Any]]:
|
||||
return [item for item in runs if isinstance(item, dict)]
|
||||
|
||||
|
||||
def _http_detail(exc: urllib.error.HTTPError, *, limit: int = 160) -> str:
|
||||
"""Whatever the server said about the refusal, trimmed for a card notice.
|
||||
|
||||
Worth surfacing: Gitea's own body names the missing scope
|
||||
(``required=[read:user], token scope=...``), and a Cloudflare interception
|
||||
in front of it says something else entirely. Guessing from the status code
|
||||
alone sent this exact investigation down the wrong path once already.
|
||||
Unreadable bodies degrade to ``""`` -- this must never mask the real error.
|
||||
"""
|
||||
try:
|
||||
raw = exc.read().decode("utf-8", errors="replace").strip()
|
||||
except Exception:
|
||||
return ""
|
||||
if not raw:
|
||||
return ""
|
||||
try:
|
||||
parsed = json.loads(raw)
|
||||
except json.JSONDecodeError:
|
||||
detail = " ".join(raw.split())
|
||||
else:
|
||||
message = parsed.get("message") if isinstance(parsed, dict) else None
|
||||
detail = str(message) if message else " ".join(raw.split())
|
||||
return detail[:limit]
|
||||
|
||||
|
||||
def _fetch_runs(
|
||||
base_url: str, token: str, limit: int
|
||||
) -> tuple[list[dict[str, Any]], str, str | None, bool, bool]:
|
||||
@@ -258,7 +308,7 @@ def _fetch_runs(
|
||||
try:
|
||||
return _fetch(url, token), source, None, False, False
|
||||
except urllib.error.HTTPError as exc:
|
||||
last_error = f"Gitea API returned HTTP {exc.code}."
|
||||
last_error = f"Gitea API returned HTTP {exc.code}. {_http_detail(exc)}".strip()
|
||||
if exc.code == 401:
|
||||
auth_failed = True
|
||||
refused = True
|
||||
|
||||
@@ -100,9 +100,13 @@ class _FakeResponse:
|
||||
return None
|
||||
|
||||
|
||||
def _http_error(code: int) -> urllib.error.HTTPError:
|
||||
def _http_error(code: int, body: str = "") -> urllib.error.HTTPError:
|
||||
return urllib.error.HTTPError(
|
||||
url="https://gitea.example", code=code, msg="nope", hdrs=None, fp=None
|
||||
url="https://gitea.example",
|
||||
code=code,
|
||||
msg="nope",
|
||||
hdrs=None, # type: ignore[arg-type]
|
||||
fp=BytesIO(body.encode("utf-8")) if body else None,
|
||||
)
|
||||
|
||||
|
||||
@@ -192,6 +196,95 @@ def test_admin_payload_parsed_into_executions(
|
||||
]
|
||||
|
||||
|
||||
def test_request_sends_an_explicit_user_agent(
|
||||
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
"""Cloudflare in front of Gitea 403s urllib's default agent (error 1010)."""
|
||||
agents: list[str | None] = []
|
||||
|
||||
def _fake_urlopen(request: Any, timeout: float | None = None) -> Any:
|
||||
agents.append(request.get_header("User-agent"))
|
||||
return _FakeResponse({"workflow_runs": []})
|
||||
|
||||
monkeypatch.setattr(ex.urllib.request, "urlopen", _fake_urlopen)
|
||||
|
||||
list_executions(_make_config(tmp_path), [], token="tok", now=_NOW)
|
||||
|
||||
assert agents and agents[0]
|
||||
assert "python-urllib" not in str(agents[0]).lower()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("path", "expected"),
|
||||
[
|
||||
(".gitea/workflows/build-ns7zip.yml", "build-ns7zip.yml"),
|
||||
# Gitea's <file>@<ref> form: splitting on "/" alone yields "main".
|
||||
("lint.yml@refs/heads/main", "lint.yml"),
|
||||
(".gitea/workflows/ci.yml@refs/heads/feat/x", "ci.yml"),
|
||||
("", ""),
|
||||
],
|
||||
)
|
||||
def test_workflow_name_drops_the_ref(
|
||||
tmp_path: Path, monkeypatch: pytest.MonkeyPatch, path: str, expected: str
|
||||
) -> None:
|
||||
_install_urlopen(monkeypatch, _admin_ok([_run(path=path)]))
|
||||
|
||||
view = list_executions(_make_config(tmp_path), [], token="tok", now=_NOW)
|
||||
|
||||
assert view.runs[0].workflow == expected
|
||||
|
||||
|
||||
def test_zero_attempt_reads_as_one(
|
||||
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
"""Gitea sends ``run_attempt: 0`` for a first run; job ids start at 1."""
|
||||
_install_urlopen(monkeypatch, _admin_ok([_run(run_id=384, attempt=0)]))
|
||||
vm = _vm("Clone_384-1_20260725_115000_linux")
|
||||
|
||||
view = list_executions(_make_config(tmp_path), [vm], token="tok", now=_NOW)
|
||||
|
||||
assert view.runs[0].attempt == 1
|
||||
assert view.runs[0].vms == [vm]
|
||||
|
||||
|
||||
def test_http_error_carries_the_server_message(
|
||||
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
"""Gitea names the missing scope in its body -- surface it, don't guess."""
|
||||
body = json.dumps(
|
||||
{"message": "token does not have at least one of required scope(s)"}
|
||||
)
|
||||
_install_urlopen(monkeypatch, lambda _url: _http_error(500, body))
|
||||
|
||||
view = list_executions(_make_config(tmp_path), [], token="tok", now=_NOW)
|
||||
|
||||
assert view.error is not None
|
||||
assert "HTTP 500" in view.error
|
||||
assert "required scope(s)" in view.error
|
||||
|
||||
|
||||
def test_non_json_error_body_is_included_verbatim(
|
||||
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
"""A Cloudflare interception is HTML, not Gitea JSON -- still show it."""
|
||||
_install_urlopen(monkeypatch, lambda _url: _http_error(500, "<html>error code: 1010</html>"))
|
||||
|
||||
view = list_executions(_make_config(tmp_path), [], token="tok", now=_NOW)
|
||||
|
||||
assert view.error is not None
|
||||
assert "1010" in view.error
|
||||
|
||||
|
||||
def test_empty_error_body_leaves_a_bare_status(
|
||||
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
_install_urlopen(monkeypatch, lambda _url: _http_error(500))
|
||||
|
||||
view = list_executions(_make_config(tmp_path), [], token="tok", now=_NOW)
|
||||
|
||||
assert view.error == "Gitea API returned HTTP 500."
|
||||
|
||||
|
||||
def test_limit_argument_overrides_config(
|
||||
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
|
||||
Reference in New Issue
Block a user