Simone
456db0a3e2
Windows produced an intermediate C:\CI\output\artifacts.zip that upload-artifact then re-zipped (zip-in-zip), while Linux deposited raw files. Make both paths consistent: the build stages raw outputs into a collect dir; collect zips it only as a transport archive, fetches, and extracts into the host artifact dir; actions/upload-artifact produces the single final archive. - _windows_build: param artifact_zip -> output_dir; stage raw files (in-place when source IS the collect dir, else wipe+copy), no Compress-Archive; absolute artifact-source resolved. - _windows_collect: zip guest dir as transport, fetch, extract (mirror _linux_collect); guest_path is now the collect directory. - job.py / build_run CLI: pass output_dir; rename --guest-artifact-zip -> --guest-output-dir. - Tests updated for the raw-file contract. Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
295 lines
9.7 KiB
Python
295 lines
9.7 KiB
Python
"""``artifacts`` sub-commands.
|
|
|
|
Phase A3 ports the artifact-collection script:
|
|
|
|
* ``artifacts collect`` — replaces ``scripts/Get-BuildArtifacts.ps1``
|
|
|
|
Uses the abstract :meth:`~ci_orchestrator.transport.winrm.WinRmTransport.fetch`
|
|
and the SSH transport (via a tar+fetch helper) so no platform-specific
|
|
``Copy-Item`` / ``scp.exe`` invocations leak into command code.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import json
|
|
import shlex
|
|
import tarfile
|
|
import tempfile
|
|
import zipfile
|
|
from datetime import UTC, datetime
|
|
from pathlib import Path
|
|
|
|
import click
|
|
|
|
from ci_orchestrator.config import load_config
|
|
from ci_orchestrator.credentials import KeyringCredentialStore
|
|
from ci_orchestrator.transport.errors import TransportCommandError, TransportError
|
|
from ci_orchestrator.transport.ssh import SshTransport
|
|
from ci_orchestrator.transport.winrm import WinRmTransport
|
|
|
|
# ---------------------------------------------------------------- group
|
|
|
|
|
|
@click.group("artifacts")
|
|
def artifacts() -> None:
|
|
"""Artifact transfer commands."""
|
|
|
|
|
|
# ---------------------------------------------------------------- helpers
|
|
|
|
|
|
def _sh_quote(value: str) -> str:
|
|
return shlex.quote(value)
|
|
|
|
|
|
def _ps_quote(value: str) -> str:
|
|
return "'" + value.replace("'", "''") + "'"
|
|
|
|
|
|
def _sha256_hex(path: Path) -> str:
|
|
h = hashlib.sha256()
|
|
with path.open("rb") as fh:
|
|
for chunk in iter(lambda: fh.read(64 * 1024), b""):
|
|
h.update(chunk)
|
|
return h.hexdigest()
|
|
|
|
|
|
def _write_manifest(host_dir: Path, job_id: str, commit: str) -> int:
|
|
"""Write ``manifest.json`` summarising every collected file. Returns count."""
|
|
files = [
|
|
p
|
|
for p in host_dir.rglob("*")
|
|
if p.is_file() and p.name != "manifest.json"
|
|
]
|
|
entries = []
|
|
for f in files:
|
|
rel = f.relative_to(host_dir).as_posix()
|
|
entries.append(
|
|
{
|
|
"name": rel,
|
|
"size": f.stat().st_size,
|
|
"sha256": _sha256_hex(f),
|
|
}
|
|
)
|
|
payload = {
|
|
"ts": datetime.now(tz=UTC).isoformat(),
|
|
"jobId": job_id,
|
|
"commit": commit,
|
|
"files": entries,
|
|
}
|
|
out = host_dir / "manifest.json"
|
|
out.write_text(json.dumps(payload, indent=2), encoding="utf-8")
|
|
return len(entries)
|
|
|
|
|
|
# ---------------------------------------------------------------- Linux flow
|
|
|
|
|
|
def _linux_collect(
|
|
*,
|
|
ip_address: str,
|
|
ssh_user: str,
|
|
key_path: str | None,
|
|
known_hosts: str | None,
|
|
guest_path: str,
|
|
host_dir: Path,
|
|
) -> None:
|
|
"""Tar the guest directory, fetch the single archive, extract locally."""
|
|
click.echo(f"[artifacts collect] tarring guest:{guest_path}")
|
|
archive_remote = f"/tmp/ci-artifacts-{ip_address.replace('.', '-')}.tar.gz"
|
|
with SshTransport(
|
|
ip_address,
|
|
username=ssh_user,
|
|
key_path=key_path,
|
|
known_hosts=known_hosts,
|
|
) as t:
|
|
# Build a tarball whose contents are *inside* guest_path. Strip the
|
|
# parent component so files land directly under host_dir.
|
|
t.run(
|
|
f"tar -czf {_sh_quote(archive_remote)} "
|
|
f"-C {_sh_quote(guest_path)} ."
|
|
)
|
|
try:
|
|
with tempfile.TemporaryDirectory() as td:
|
|
local_tar = Path(td) / "artifacts.tar.gz"
|
|
t.fetch(archive_remote, str(local_tar))
|
|
with tarfile.open(local_tar, "r:gz") as tf:
|
|
# ``data`` filter (Python 3.12+) protects against path
|
|
# traversal in malicious tarballs.
|
|
tf.extractall(host_dir, filter="data")
|
|
finally:
|
|
t.run(f"rm -f {_sh_quote(archive_remote)}", check=False)
|
|
|
|
|
|
# ---------------------------------------------------------------- Windows flow
|
|
|
|
|
|
def _windows_collect(
|
|
*,
|
|
ip_address: str,
|
|
credential_target: str,
|
|
guest_path: str,
|
|
host_dir: Path,
|
|
include_logs: bool,
|
|
) -> None:
|
|
cred = KeyringCredentialStore().get(credential_target)
|
|
remote_zip = "C:\\CI\\ci-artifacts-transfer.zip"
|
|
with WinRmTransport(ip_address, cred.username, cred.password) as t:
|
|
# guest_path is the collect directory (raw build outputs).
|
|
# Verify it exists and is non-empty.
|
|
probe = t.run(
|
|
f"if ((Test-Path {_ps_quote(guest_path)}) -and "
|
|
f"(Get-ChildItem -Force -- {_ps_quote(guest_path)})) "
|
|
f"{{ 'OK' }} else {{ 'MISSING' }}",
|
|
check=False,
|
|
)
|
|
if "MISSING" in probe.stdout:
|
|
raise click.ClickException(
|
|
f"artifact not found or empty in guest: {guest_path}"
|
|
)
|
|
|
|
# Zip the directory contents as a single transport file, fetch,
|
|
# extract into host_dir. Mirrors the Linux tar flow so both OSes
|
|
# deposit raw files; the final single archive is produced by
|
|
# actions/upload-artifact (no zip-in-zip).
|
|
t.run(
|
|
f"if (Test-Path {_ps_quote(remote_zip)}) "
|
|
f"{{ Remove-Item {_ps_quote(remote_zip)} -Force }}; "
|
|
f"Compress-Archive -Path (Join-Path {_ps_quote(guest_path)} '*') "
|
|
f"-DestinationPath {_ps_quote(remote_zip)} -Force"
|
|
)
|
|
click.echo(f"[artifacts collect] fetching artifacts from {guest_path}")
|
|
try:
|
|
with tempfile.TemporaryDirectory() as td:
|
|
local_zip = Path(td) / "artifacts.zip"
|
|
t.fetch(remote_zip, str(local_zip))
|
|
with zipfile.ZipFile(local_zip) as zf:
|
|
zf.extractall(host_dir)
|
|
finally:
|
|
t.run(
|
|
f"Remove-Item {_ps_quote(remote_zip)} -Force "
|
|
f"-ErrorAction SilentlyContinue",
|
|
check=False,
|
|
)
|
|
|
|
if include_logs:
|
|
# List log files in the build dir and fetch each one.
|
|
list_ps = (
|
|
"Get-ChildItem -Path 'C:\\CI\\build' -Filter '*.log' -Recurse "
|
|
"-ErrorAction SilentlyContinue | "
|
|
"ForEach-Object { $_.FullName }"
|
|
)
|
|
listing = t.run(list_ps, check=False)
|
|
for line in (listing.stdout or "").splitlines():
|
|
line = line.strip()
|
|
if not line:
|
|
continue
|
|
dest = host_dir / Path(line).name
|
|
click.echo(f"[artifacts collect] fetching log: {Path(line).name}")
|
|
try:
|
|
t.fetch(line, str(dest))
|
|
except TransportError as exc:
|
|
click.echo(
|
|
f"[artifacts collect] log fetch failed ({line}): {exc}",
|
|
err=True,
|
|
)
|
|
|
|
if not any(host_dir.iterdir()):
|
|
raise click.ClickException(
|
|
f"no artifacts after collect from guest: {guest_path}"
|
|
)
|
|
|
|
|
|
# ---------------------------------------------------------------- CLI
|
|
|
|
|
|
@artifacts.command("collect")
|
|
@click.option("--ip-address", "ip_address", required=True)
|
|
@click.option(
|
|
"--guest-os",
|
|
type=click.Choice(["windows", "linux"], case_sensitive=False),
|
|
default="windows",
|
|
show_default=True,
|
|
)
|
|
@click.option(
|
|
"--guest-artifact-path",
|
|
"guest_artifact_path",
|
|
required=True,
|
|
help="Guest collect directory (Windows and Linux).",
|
|
)
|
|
@click.option(
|
|
"--host-artifact-dir",
|
|
"host_artifact_dir",
|
|
required=True,
|
|
help="Local directory to deposit artifacts (created if missing).",
|
|
)
|
|
@click.option("--include-logs", "include_logs", is_flag=True, default=False)
|
|
@click.option("--credential-target", "credential_target", default=None)
|
|
@click.option("--ssh-user", "ssh_user", default="ci_build", show_default=True)
|
|
@click.option("--ssh-key-path", "ssh_key_path", default=None)
|
|
@click.option("--ssh-known-hosts-file", "ssh_known_hosts_file", default=None)
|
|
@click.option("--job-id", "job_id", default="")
|
|
@click.option("--commit", "commit", default="")
|
|
def artifacts_collect(
|
|
ip_address: str,
|
|
guest_os: str,
|
|
guest_artifact_path: str,
|
|
host_artifact_dir: str,
|
|
include_logs: bool,
|
|
credential_target: str | None,
|
|
ssh_user: str,
|
|
ssh_key_path: str | None,
|
|
ssh_known_hosts_file: str | None,
|
|
job_id: str,
|
|
commit: str,
|
|
) -> None:
|
|
"""Copy build artifacts from a guest VM to a local directory.
|
|
|
|
Mirrors ``scripts/Get-BuildArtifacts.ps1``. Writes ``manifest.json``
|
|
when ``--job-id`` or ``--commit`` is provided.
|
|
"""
|
|
guest_os = guest_os.lower()
|
|
host_dir = Path(host_artifact_dir)
|
|
host_dir.mkdir(parents=True, exist_ok=True)
|
|
config = load_config()
|
|
|
|
try:
|
|
if guest_os == "linux":
|
|
key_path: str | None = ssh_key_path
|
|
if key_path is None and config.ssh_key_path is not None:
|
|
key_path = str(config.ssh_key_path)
|
|
_linux_collect(
|
|
ip_address=ip_address,
|
|
ssh_user=ssh_user,
|
|
key_path=key_path,
|
|
known_hosts=ssh_known_hosts_file,
|
|
guest_path=guest_artifact_path,
|
|
host_dir=host_dir,
|
|
)
|
|
else:
|
|
target = credential_target or config.guest_cred_target
|
|
_windows_collect(
|
|
ip_address=ip_address,
|
|
credential_target=target,
|
|
guest_path=guest_artifact_path,
|
|
host_dir=host_dir,
|
|
include_logs=include_logs,
|
|
)
|
|
except (TransportError, TransportCommandError) as exc:
|
|
raise click.ClickException(str(exc)) from exc
|
|
|
|
transferred = [p for p in host_dir.rglob("*") if p.is_file()]
|
|
if not transferred:
|
|
raise click.ClickException(
|
|
f"no files were transferred into {host_dir}"
|
|
)
|
|
click.echo(f"[artifacts collect] {len(transferred)} file(s) collected.")
|
|
|
|
if job_id or commit:
|
|
count = _write_manifest(host_dir, job_id, commit)
|
|
click.echo(f"[artifacts collect] manifest.json written ({count} file(s)).")
|
|
|
|
|
|
__all__ = ["artifacts", "artifacts_collect"]
|