feat(build): live-stream Linux build output; ns7zip via in-guest clone
- SshTransport.run_streaming: incremental channel recv, forwards stdout/stderr live (no more one-block-at-end). _linux_build uses it with PYTHONUNBUFFERED so the guest python flushes promptly. - _tail_lines split out for failure messages (no double print). - Windows live streaming deferred (TODO 8.2) — pypsrp execute_ps returns only at completion. - build-ns7zip.yml: use-git-clone:'true' (in-guest), repo-url -> https://gitea.emulab.it/... (guest-reachable; host SSH alias was host-only). Requires GiteaPAT in the LocalSystem keyring or a public nsis-plugin-ns7zip. Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
@@ -35,12 +35,14 @@ jobs:
|
||||
artifact-source: 'dist'
|
||||
submodules: 'true'
|
||||
guest-os: ${{ matrix.target == 'linux' && 'Linux' || 'Windows' }}
|
||||
# repo-url uses the host SSH alias `gitea-ci` (~/.ssh/config),
|
||||
# which exists only on the host — so the clone MUST run on the
|
||||
# host, not in the guest. use-git-clone:'false' selects the
|
||||
# host-side clone + zip transfer transport.
|
||||
use-git-clone: 'false'
|
||||
repo-url: 'ssh://gitea-ci/Simone/nsis-plugin-ns7zip.git'
|
||||
# In-guest clone (default). repo-url must be reachable FROM the
|
||||
# guest (VMnet8 NAT -> gitea.emulab.it), not the host-only SSH
|
||||
# alias. ci_orchestrator injects the GiteaPAT credential into
|
||||
# the HTTPS URL, so GiteaPAT must exist in the LocalSystem
|
||||
# keyring (Set-CIGuestCredential.ps1 -Target GiteaPAT) — or
|
||||
# nsis-plugin-ns7zip must be public.
|
||||
use-git-clone: 'true'
|
||||
repo-url: 'https://gitea.emulab.it/Simone/nsis-plugin-ns7zip.git'
|
||||
# Cross-repo build: github.ref_name is this repo's branch, not
|
||||
# nsis-plugin-ns7zip's. Pin the target repo's branch.
|
||||
repo-branch: 'main'
|
||||
|
||||
@@ -838,6 +838,25 @@ dopo il primo fetch, quindi outage GitHub impatta solo cold cache.
|
||||
**Rifiori quando**: Host air-gapped, GitHub non raggiungibile dall'host CI,
|
||||
o si vuole indipendenza totale da github.com (anche per `checkout`).
|
||||
|
||||
### 8.2 [P3] [ ] Streaming live output build su Windows (WinRM)
|
||||
File: [src/ci_orchestrator/transport/winrm.py](src/ci_orchestrator/transport/winrm.py),
|
||||
[src/ci_orchestrator/commands/build.py](src/ci_orchestrator/commands/build.py).
|
||||
|
||||
Linux/SSH ora streamma l'output del build in tempo reale
|
||||
(`SshTransport.run_streaming` — recv incrementale + `PYTHONUNBUFFERED`).
|
||||
Windows resta **log-a-fine-job**: `pypsrp.Client.execute_ps` ritorna
|
||||
solo a comando concluso; `_windows_build` usa `_report_build_output`
|
||||
(blocco unico) + coda nel messaggio d'errore.
|
||||
|
||||
**Stato attuale**: Deferred. Streaming WinRM richiede API basso livello
|
||||
RunspacePool/PowerShell con poll di `ps.output`/stream invece di
|
||||
`execute_ps`, semantica PSRP più complessa; output unbuffered dei tool
|
||||
nativi (MSBuild) scomodo. Diagnostica già adeguata (output completo a
|
||||
fine job + ultime righe nell'errore).
|
||||
**Rifiori quando**: Build Windows lunghi rendono l'assenza di progress
|
||||
live un problema operativo reale, o si standardizza lo streaming
|
||||
cross-OS.
|
||||
|
||||
---
|
||||
|
||||
## Suggerimento di sequencing
|
||||
|
||||
@@ -112,6 +112,11 @@ def _report_build_output(stdout: str, stderr: str) -> str:
|
||||
sys.stderr.write(stderr if stderr.endswith("\n") else stderr + "\n")
|
||||
sys.stderr.flush()
|
||||
click.echo("[build run] ----- end build output -----")
|
||||
return _tail_lines(stdout, stderr)
|
||||
|
||||
|
||||
def _tail_lines(stdout: str, stderr: str) -> str:
|
||||
"""Last few output lines for a failure message (no printing)."""
|
||||
combined = (stderr or "").strip() or (stdout or "").strip()
|
||||
lines = combined.splitlines()
|
||||
return " | ".join(lines[-5:]) if lines else "(no build output)"
|
||||
@@ -189,13 +194,20 @@ def _linux_build(
|
||||
f"export {k}={_sh_quote(v)}; " for k, v in extra_env.items()
|
||||
)
|
||||
cmd = build_command or "make"
|
||||
full_cmd = f"{env_prefix}cd {_sh_quote(workdir)} && {cmd}"
|
||||
# PYTHONUNBUFFERED so the guest python flushes promptly and the
|
||||
# streamed output is timely rather than emitted in one block.
|
||||
full_cmd = (
|
||||
f"{env_prefix}export PYTHONUNBUFFERED=1; "
|
||||
f"cd {_sh_quote(workdir)} && {cmd}"
|
||||
)
|
||||
click.echo(f"[build run] running build (env vars redacted): cd {workdir} && {cmd}")
|
||||
result = t.run(full_cmd, check=False)
|
||||
tail = _report_build_output(result.stdout, result.stderr)
|
||||
click.echo("[build run] ----- build output (live) -----")
|
||||
result = t.run_streaming(full_cmd, check=False)
|
||||
click.echo("[build run] ----- end build output -----")
|
||||
if result.returncode != 0:
|
||||
raise click.ClickException(
|
||||
f"build command failed (exit {result.returncode}): {tail}"
|
||||
"build command failed "
|
||||
f"(exit {result.returncode}): {_tail_lines(result.stdout, result.stderr)}"
|
||||
)
|
||||
|
||||
if skip_artifact:
|
||||
|
||||
@@ -14,6 +14,8 @@ on stderr output. With paramiko there is no equivalent issue.
|
||||
from __future__ import annotations
|
||||
|
||||
import contextlib
|
||||
import sys
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING
|
||||
@@ -123,6 +125,64 @@ class SshTransport:
|
||||
raise TransportCommandError(result.returncode, result.stdout, result.stderr)
|
||||
return result
|
||||
|
||||
def run_streaming(
|
||||
self, command: str, *, check: bool = True, timeout: float | None = None
|
||||
) -> SshResult:
|
||||
"""Run ``command`` streaming stdout/stderr live to this process.
|
||||
|
||||
Unlike :meth:`run` (which blocks on a full channel read), output
|
||||
is forwarded chunk-by-chunk as it arrives so long builds show
|
||||
progress instead of one block at the end. Captured output is
|
||||
still returned in :class:`SshResult`.
|
||||
"""
|
||||
client = self._connect()
|
||||
try:
|
||||
chan = client.get_transport().open_session() # type: ignore[union-attr]
|
||||
if timeout:
|
||||
chan.settimeout(timeout)
|
||||
chan.exec_command(command)
|
||||
except Exception as exc:
|
||||
raise TransportConnectError(f"SSH exec_command failed: {exc}") from exc
|
||||
|
||||
out_parts: list[str] = []
|
||||
err_parts: list[str] = []
|
||||
try:
|
||||
while True:
|
||||
progressed = False
|
||||
while chan.recv_ready():
|
||||
d = chan.recv(65536).decode("utf-8", errors="replace")
|
||||
out_parts.append(d)
|
||||
sys.stdout.write(d)
|
||||
sys.stdout.flush()
|
||||
progressed = True
|
||||
while chan.recv_stderr_ready():
|
||||
d = chan.recv_stderr(65536).decode("utf-8", errors="replace")
|
||||
err_parts.append(d)
|
||||
sys.stderr.write(d)
|
||||
sys.stderr.flush()
|
||||
progressed = True
|
||||
if (
|
||||
chan.exit_status_ready()
|
||||
and not chan.recv_ready()
|
||||
and not chan.recv_stderr_ready()
|
||||
):
|
||||
break
|
||||
if not progressed:
|
||||
time.sleep(0.05)
|
||||
rc = chan.recv_exit_status()
|
||||
except Exception as exc:
|
||||
raise TransportConnectError(f"SSH stream failed: {exc}") from exc
|
||||
finally:
|
||||
with contextlib.suppress(Exception):
|
||||
chan.close()
|
||||
|
||||
result = SshResult(
|
||||
stdout="".join(out_parts), stderr="".join(err_parts), returncode=rc
|
||||
)
|
||||
if check and not result.ok:
|
||||
raise TransportCommandError(result.returncode, result.stdout, result.stderr)
|
||||
return result
|
||||
|
||||
def copy(self, local_path: str | Path, remote_path: str) -> None:
|
||||
"""Upload via SFTP."""
|
||||
client = self._connect()
|
||||
|
||||
@@ -53,6 +53,13 @@ class _FakeTransport:
|
||||
raise TransportCommandError(result.returncode, result.stdout, result.stderr)
|
||||
return result
|
||||
|
||||
def run_streaming(
|
||||
self, script: str, *, check: bool = True, timeout: float | None = None
|
||||
) -> _FakeResult:
|
||||
# Tests don't exercise live streaming; delegate to run() so
|
||||
# capture + check semantics (and _BuildFails overrides) hold.
|
||||
return self.run(script, check=check)
|
||||
|
||||
def copy(self, local_path: str, remote_path: str) -> None:
|
||||
self.copies.append((str(local_path), remote_path))
|
||||
|
||||
|
||||
Reference in New Issue
Block a user