feat(build): live-stream Linux build output; ns7zip via in-guest clone
Lint / pssa (push) Failing after 26s
Lint / python (push) Successful in 52s

- SshTransport.run_streaming: incremental channel recv, forwards
  stdout/stderr live (no more one-block-at-end). _linux_build uses it
  with PYTHONUNBUFFERED so the guest python flushes promptly.
- _tail_lines split out for failure messages (no double print).
- Windows live streaming deferred (TODO 8.2) — pypsrp execute_ps
  returns only at completion.
- build-ns7zip.yml: use-git-clone:'true' (in-guest), repo-url ->
  https://gitea.emulab.it/... (guest-reachable; host SSH alias was
  host-only). Requires GiteaPAT in the LocalSystem keyring or a public
  nsis-plugin-ns7zip.

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
Simone
2026-05-17 20:51:48 +02:00
parent 29c4ef09bc
commit 4f22d11a84
5 changed files with 110 additions and 10 deletions
+8 -6
View File
@@ -35,12 +35,14 @@ jobs:
artifact-source: 'dist' artifact-source: 'dist'
submodules: 'true' submodules: 'true'
guest-os: ${{ matrix.target == 'linux' && 'Linux' || 'Windows' }} guest-os: ${{ matrix.target == 'linux' && 'Linux' || 'Windows' }}
# repo-url uses the host SSH alias `gitea-ci` (~/.ssh/config), # In-guest clone (default). repo-url must be reachable FROM the
# which exists only on the host — so the clone MUST run on the # guest (VMnet8 NAT -> gitea.emulab.it), not the host-only SSH
# host, not in the guest. use-git-clone:'false' selects the # alias. ci_orchestrator injects the GiteaPAT credential into
# host-side clone + zip transfer transport. # the HTTPS URL, so GiteaPAT must exist in the LocalSystem
use-git-clone: 'false' # keyring (Set-CIGuestCredential.ps1 -Target GiteaPAT) — or
repo-url: 'ssh://gitea-ci/Simone/nsis-plugin-ns7zip.git' # nsis-plugin-ns7zip must be public.
use-git-clone: 'true'
repo-url: 'https://gitea.emulab.it/Simone/nsis-plugin-ns7zip.git'
# Cross-repo build: github.ref_name is this repo's branch, not # Cross-repo build: github.ref_name is this repo's branch, not
# nsis-plugin-ns7zip's. Pin the target repo's branch. # nsis-plugin-ns7zip's. Pin the target repo's branch.
repo-branch: 'main' repo-branch: 'main'
+19
View File
@@ -838,6 +838,25 @@ dopo il primo fetch, quindi outage GitHub impatta solo cold cache.
**Rifiori quando**: Host air-gapped, GitHub non raggiungibile dall'host CI, **Rifiori quando**: Host air-gapped, GitHub non raggiungibile dall'host CI,
o si vuole indipendenza totale da github.com (anche per `checkout`). o si vuole indipendenza totale da github.com (anche per `checkout`).
### 8.2 [P3] [ ] Streaming live output build su Windows (WinRM)
File: [src/ci_orchestrator/transport/winrm.py](src/ci_orchestrator/transport/winrm.py),
[src/ci_orchestrator/commands/build.py](src/ci_orchestrator/commands/build.py).
Linux/SSH ora streamma l'output del build in tempo reale
(`SshTransport.run_streaming` — recv incrementale + `PYTHONUNBUFFERED`).
Windows resta **log-a-fine-job**: `pypsrp.Client.execute_ps` ritorna
solo a comando concluso; `_windows_build` usa `_report_build_output`
(blocco unico) + coda nel messaggio d'errore.
**Stato attuale**: Deferred. Streaming WinRM richiede API basso livello
RunspacePool/PowerShell con poll di `ps.output`/stream invece di
`execute_ps`, semantica PSRP più complessa; output unbuffered dei tool
nativi (MSBuild) scomodo. Diagnostica già adeguata (output completo a
fine job + ultime righe nell'errore).
**Rifiori quando**: Build Windows lunghi rendono l'assenza di progress
live un problema operativo reale, o si standardizza lo streaming
cross-OS.
--- ---
## Suggerimento di sequencing ## Suggerimento di sequencing
+16 -4
View File
@@ -112,6 +112,11 @@ def _report_build_output(stdout: str, stderr: str) -> str:
sys.stderr.write(stderr if stderr.endswith("\n") else stderr + "\n") sys.stderr.write(stderr if stderr.endswith("\n") else stderr + "\n")
sys.stderr.flush() sys.stderr.flush()
click.echo("[build run] ----- end build output -----") click.echo("[build run] ----- end build output -----")
return _tail_lines(stdout, stderr)
def _tail_lines(stdout: str, stderr: str) -> str:
"""Last few output lines for a failure message (no printing)."""
combined = (stderr or "").strip() or (stdout or "").strip() combined = (stderr or "").strip() or (stdout or "").strip()
lines = combined.splitlines() lines = combined.splitlines()
return " | ".join(lines[-5:]) if lines else "(no build output)" return " | ".join(lines[-5:]) if lines else "(no build output)"
@@ -189,13 +194,20 @@ def _linux_build(
f"export {k}={_sh_quote(v)}; " for k, v in extra_env.items() f"export {k}={_sh_quote(v)}; " for k, v in extra_env.items()
) )
cmd = build_command or "make" cmd = build_command or "make"
full_cmd = f"{env_prefix}cd {_sh_quote(workdir)} && {cmd}" # PYTHONUNBUFFERED so the guest python flushes promptly and the
# streamed output is timely rather than emitted in one block.
full_cmd = (
f"{env_prefix}export PYTHONUNBUFFERED=1; "
f"cd {_sh_quote(workdir)} && {cmd}"
)
click.echo(f"[build run] running build (env vars redacted): cd {workdir} && {cmd}") click.echo(f"[build run] running build (env vars redacted): cd {workdir} && {cmd}")
result = t.run(full_cmd, check=False) click.echo("[build run] ----- build output (live) -----")
tail = _report_build_output(result.stdout, result.stderr) result = t.run_streaming(full_cmd, check=False)
click.echo("[build run] ----- end build output -----")
if result.returncode != 0: if result.returncode != 0:
raise click.ClickException( raise click.ClickException(
f"build command failed (exit {result.returncode}): {tail}" "build command failed "
f"(exit {result.returncode}): {_tail_lines(result.stdout, result.stderr)}"
) )
if skip_artifact: if skip_artifact:
+60
View File
@@ -14,6 +14,8 @@ on stderr output. With paramiko there is no equivalent issue.
from __future__ import annotations from __future__ import annotations
import contextlib import contextlib
import sys
import time
from dataclasses import dataclass from dataclasses import dataclass
from pathlib import Path from pathlib import Path
from typing import TYPE_CHECKING from typing import TYPE_CHECKING
@@ -123,6 +125,64 @@ class SshTransport:
raise TransportCommandError(result.returncode, result.stdout, result.stderr) raise TransportCommandError(result.returncode, result.stdout, result.stderr)
return result return result
def run_streaming(
self, command: str, *, check: bool = True, timeout: float | None = None
) -> SshResult:
"""Run ``command`` streaming stdout/stderr live to this process.
Unlike :meth:`run` (which blocks on a full channel read), output
is forwarded chunk-by-chunk as it arrives so long builds show
progress instead of one block at the end. Captured output is
still returned in :class:`SshResult`.
"""
client = self._connect()
try:
chan = client.get_transport().open_session() # type: ignore[union-attr]
if timeout:
chan.settimeout(timeout)
chan.exec_command(command)
except Exception as exc:
raise TransportConnectError(f"SSH exec_command failed: {exc}") from exc
out_parts: list[str] = []
err_parts: list[str] = []
try:
while True:
progressed = False
while chan.recv_ready():
d = chan.recv(65536).decode("utf-8", errors="replace")
out_parts.append(d)
sys.stdout.write(d)
sys.stdout.flush()
progressed = True
while chan.recv_stderr_ready():
d = chan.recv_stderr(65536).decode("utf-8", errors="replace")
err_parts.append(d)
sys.stderr.write(d)
sys.stderr.flush()
progressed = True
if (
chan.exit_status_ready()
and not chan.recv_ready()
and not chan.recv_stderr_ready()
):
break
if not progressed:
time.sleep(0.05)
rc = chan.recv_exit_status()
except Exception as exc:
raise TransportConnectError(f"SSH stream failed: {exc}") from exc
finally:
with contextlib.suppress(Exception):
chan.close()
result = SshResult(
stdout="".join(out_parts), stderr="".join(err_parts), returncode=rc
)
if check and not result.ok:
raise TransportCommandError(result.returncode, result.stdout, result.stderr)
return result
def copy(self, local_path: str | Path, remote_path: str) -> None: def copy(self, local_path: str | Path, remote_path: str) -> None:
"""Upload via SFTP.""" """Upload via SFTP."""
client = self._connect() client = self._connect()
+7
View File
@@ -53,6 +53,13 @@ class _FakeTransport:
raise TransportCommandError(result.returncode, result.stdout, result.stderr) raise TransportCommandError(result.returncode, result.stdout, result.stderr)
return result return result
def run_streaming(
self, script: str, *, check: bool = True, timeout: float | None = None
) -> _FakeResult:
# Tests don't exercise live streaming; delegate to run() so
# capture + check semantics (and _BuildFails overrides) hold.
return self.run(script, check=check)
def copy(self, local_path: str, remote_path: str) -> None: def copy(self, local_path: str, remote_path: str) -> None:
self.copies.append((str(local_path), remote_path)) self.copies.append((str(local_path), remote_path))