diff --git a/plans/A3-closeout.md b/plans/A3-closeout.md new file mode 100644 index 0000000..7d528c4 --- /dev/null +++ b/plans/A3-closeout.md @@ -0,0 +1,31 @@ +# A3 — Closeout + +Fase A3 di `plans/implementation-plan-A-B.md`: pipeline core (clone → build → collect) in Python. +Branch: `feature/python-rewrite-phase-a`. + +## Stato attuale + +- [x] `commands/vm.py` esteso con `vm new` (sostituisce `New-BuildVM.ps1`) +- [x] `commands/build.py` con `build run` (sostituisce `Invoke-RemoteBuild.ps1`) +- [x] `commands/artifacts.py` con `artifacts collect` (sostituisce `Get-BuildArtifacts.ps1`) +- [x] Tutti i 3 `.ps1` ridotti a shim verso la CLI Python +- [x] Pester `tests/{New-BuildVM,Wait-VMReady,Remove-BuildVM}.Tests.ps1` rimossi (casi negativi coperti dai pytest equivalenti) +- [x] 91 pytest, ruff/mypy --strict clean, coverage 78.27% +- [x] Hook Fase C: `build run` + `artifacts collect` accettano `VmHandle`/`vmx` opaco; nessuna assunzione path Windows +- [ ] Smoke end-to-end manuale: clone WinBuild2025 → build script trivial → collect artifact ZIP +- [ ] Smoke end-to-end Linux equivalente + +## Definizione di "fatto" A3 + +| Criterio | Stato | +| --- | --- | +| Pipeline build completa via Python CLI | ✅ | +| Shim PS preservano API per caller esistenti | ✅ | +| Smoke build PASS contro VM reale Windows e Linux | ⏳ validazione hardware | +| Pester `New-BuildVM.Tests.ps1` rimosso e sostituito | ✅ | + +## Cosa resta a carico utente + +1. Smoke manuale end-to-end Windows: `python -m ci_orchestrator vm new ... && build run ... && artifacts collect ...` +2. Stesso smoke su Linux template +3. Verificare che `Test-NsinnounpBuild.ps1` continui a passare invocando gli shim diff --git a/plans/implementation-plan-A-B.md b/plans/implementation-plan-A-B.md index cb385e3..e4a0378 100644 --- a/plans/implementation-plan-A-B.md +++ b/plans/implementation-plan-A-B.md @@ -60,10 +60,10 @@ A4; cambiano solo path/env vars in Fase B). - [x] [A2] Portare `Watch-RunnerHealth.ps1` → `monitor runner` - [x] [A2] Portare `Get-CIJobSummary.ps1` → `report job` - [x] [A2] Sostituire ognuno dei `.ps1` portati con shim a 3 righe verso la CLI Python -- [ ] [A3] Portare `New-BuildVM.ps1` → `vm new` -- [ ] [A3] Portare `Invoke-RemoteBuild.ps1` → `build run` -- [ ] [A3] Portare `Get-BuildArtifacts.ps1` → `artifacts collect` -- [ ] [A3] Convertire test Pester `New-BuildVM.Tests.ps1`, `Wait-VMReady.Tests.ps1`, `Remove-BuildVM.Tests.ps1` in pytest +- [x] [A3] Portare `New-BuildVM.ps1` → `vm new` +- [x] [A3] Portare `Invoke-RemoteBuild.ps1` → `build run` +- [x] [A3] Portare `Get-BuildArtifacts.ps1` → `artifacts collect` +- [x] [A3] Convertire test Pester `New-BuildVM.Tests.ps1`, `Wait-VMReady.Tests.ps1`, `Remove-BuildVM.Tests.ps1` in pytest - [ ] [A4] Portare `Invoke-CIJob.ps1` → `python -m ci_orchestrator job` - [ ] [A4] Aggiornare `gitea/actions/local-ci-build/action.yml` per invocare la CLI Python direttamente - [ ] [A4] Forzare `PYTHONIOENCODING=utf-8` in `runner/config.yaml` @@ -229,15 +229,15 @@ Python preservando il comportamento dei `.ps1` esistenti. **Attività**: -- [ ] Implementare `vm new` in `commands/vm.py` (parametri `--template`, `--snapshot`, `--name`, `--guest-os`) -- [ ] Integrare `WorkstationVmrunBackend.clone_linked` + `start` + attesa IP via `get_ip` -- [ ] Implementare `commands/build.py` con `build run` (parametri `--vmx`, `--script`, `--workdir`, `--guest-os`); usa `transport.winrm` per Windows, `transport.ssh` per Linux -- [ ] Gestione streaming output build verso stdout (act_runner cattura stdout — preservare comportamento `Write-Host`) -- [ ] Implementare `commands/artifacts.py` con `artifacts collect` (parametri `--vmx`, `--remote-path`, `--local-dir`) -- [ ] Sostituire `scripts/New-BuildVM.ps1`, `Invoke-RemoteBuild.ps1`, `Get-BuildArtifacts.ps1` con shim -- [ ] Convertire `tests/New-BuildVM.Tests.ps1` in `tests/test_commands_vm_new.py` -- [ ] Aggiungere test pytest per `build run` (mock transport + capture stdout) -- [ ] Aggiungere test pytest per `artifacts collect` (mock SFTP/WinRM file copy + tmp_path) +- [x] Implementare `vm new` in `commands/vm.py` (parametri `--template`, `--snapshot`, `--name`, `--guest-os`) +- [x] Integrare `WorkstationVmrunBackend.clone_linked` + `start` + attesa IP via `get_ip` +- [x] Implementare `commands/build.py` con `build run` (parametri `--vmx`, `--script`, `--workdir`, `--guest-os`); usa `transport.winrm` per Windows, `transport.ssh` per Linux +- [x] Gestione streaming output build verso stdout (act_runner cattura stdout — preservare comportamento `Write-Host`) +- [x] Implementare `commands/artifacts.py` con `artifacts collect` (parametri `--vmx`, `--remote-path`, `--local-dir`) +- [x] Sostituire `scripts/New-BuildVM.ps1`, `Invoke-RemoteBuild.ps1`, `Get-BuildArtifacts.ps1` con shim +- [x] Convertire `tests/New-BuildVM.Tests.ps1` in `tests/test_commands_vm_new.py` +- [x] Aggiungere test pytest per `build run` (mock transport + capture stdout) +- [x] Aggiungere test pytest per `artifacts collect` (mock SFTP/WinRM file copy + tmp_path) - [ ] Validare end-to-end: clone WinBuild2025 → build script PowerShell trivial → collect artifact ZIP **Hook futuri Fase C**: `build run` deve ricevere un `VmHandle` opaco, @@ -257,10 +257,10 @@ Python; nessuna modifica ai workflow YAML in A3 (solo shim). **Definizione di fatto step A3**: -- [ ] Pipeline build completa accessibile via Python CLI -- [ ] Shim PS preservano l'API per i caller esistenti +- [x] Pipeline build completa accessibile via Python CLI +- [x] Shim PS preservano l'API per i caller esistenti - [ ] Smoke build PASS contro VM reale Windows e Linux -- [ ] Pester `New-BuildVM.Tests.ps1` rimosso e sostituito +- [x] Pester `New-BuildVM.Tests.ps1` rimosso e sostituito ### A4 — Orchestratore + switch workflow diff --git a/scripts/Get-BuildArtifacts.ps1 b/scripts/Get-BuildArtifacts.ps1 index 120d7d6..93a11a8 100644 --- a/scripts/Get-BuildArtifacts.ps1 +++ b/scripts/Get-BuildArtifacts.ps1 @@ -1,45 +1,16 @@ #Requires -Version 5.1 -<# -.SYNOPSIS - Copies build artifacts from an ephemeral VM to the host artifact directory. +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' -.DESCRIPTION - Opens a WinRM session to the build VM and uses Copy-Item -FromSession - to transfer the artifact archive and optional log files. - Validates that at least one file was successfully transferred. - Throws on failure so the caller's try/finally can clean up the VM. +# Shim: delegates to the Python ci_orchestrator CLI (`artifacts collect`). +# Original PS implementation moved to git history; see plans/A3-closeout.md. +# +# The bound `-Credential` is intentionally discarded — Python uses keyring +# via `--credential-target` (defaulting to `BuildVMGuest`). -.PARAMETER IPAddress - IP of the build VM. - -.PARAMETER Credential - PSCredential for the guest build user. - -.PARAMETER GuestArtifactPath - Full path to the artifact file or directory inside the VM. - Example: C:\CI\output\artifacts.zip - -.PARAMETER HostArtifactDir - Directory on the host where artifacts will be stored. - The directory is created if it does not exist. - Example: F:\CI\Artifacts\run-42 - -.PARAMETER IncludeLogs - If set, also copies *.log files from the guest work directory - (C:\CI\build\*.log) alongside the artifact archive. - -.EXAMPLE - $cred = Get-Credential - .\Get-BuildArtifacts.ps1 ` - -IPAddress "192.168.11.101" ` - -Credential $cred ` - -GuestArtifactPath "C:\CI\output\artifacts.zip" ` - -HostArtifactDir "F:\CI\Artifacts\run-42" -#> [CmdletBinding()] param( [Parameter(Mandatory)] - [ValidatePattern('^((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)\.){3}(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)$')] [string] $IPAddress, [Parameter()] @@ -53,175 +24,39 @@ param( [switch] $IncludeLogs, - # Guest OS type — Linux uses SCP instead of WinRM. [ValidateSet('Windows', 'Linux')] [string] $GuestOS = 'Windows', - # SSH private key path (used when GuestOS=Linux). [string] $SshKeyPath = 'F:\CI\keys\ci_linux', - - # SSH username (used when GuestOS=Linux). [string] $SshUser = 'ci_build', - - # Persistent known_hosts file for SSH calls (CI jobs). - # Empty string (default) = permissive mode (StrictHostKeyChecking=no, UserKnownHostsFile=NUL). [string] $SshKnownHostsFile = '', - # Optional job ID and commit SHA written into manifest.json. - # When either is non-empty a manifest.json is written to $HostArtifactDir - # listing all collected files with name, size, and SHA256. - [string] $JobId = '', - [string] $Commit = '' + [string] $JobId = '', + [string] $Commit = '', + + [string] $CredentialTarget = '' ) -Set-StrictMode -Version Latest -$ErrorActionPreference = 'Stop' +$pyArgs = @( + '--ip-address', $IPAddress, + '--guest-os', $GuestOS.ToLower(), + '--guest-artifact-path', $GuestArtifactPath, + '--host-artifact-dir', $HostArtifactDir, + '--ssh-user', $SshUser, + '--ssh-key-path', $SshKeyPath, + '--job-id', $JobId, + '--commit', $Commit +) -Import-Module (Join-Path $PSScriptRoot '_Common.psm1') -Force +if ($SshKnownHostsFile) { $pyArgs += @('--ssh-known-hosts-file', $SshKnownHostsFile) } +if ($CredentialTarget) { $pyArgs += @('--credential-target', $CredentialTarget) } +if ($IncludeLogs.IsPresent) { $pyArgs += '--include-logs' } -# ── Helper: write artifact manifest ───────────────────────────────────────── -function Write-ArtifactManifest { - param([string] $Dir, [string] $JobId, [string] $Commit) - $files = @(Get-ChildItem -Path $Dir -Recurse -File -ErrorAction SilentlyContinue | - Where-Object { $_.Name -ne 'manifest.json' }) - $fileEntries = foreach ($f in $files) { - [pscustomobject]@{ - name = $f.FullName.Substring($Dir.TrimEnd('\', '/').Length).TrimStart('\', '/') - size = $f.Length - sha256 = (Get-FileHash $f.FullName -Algorithm SHA256).Hash.ToLower() - } - } - $manifest = [ordered]@{ - ts = (Get-Date -Format 'o') - jobId = $JobId - commit = $Commit - files = @($fileEntries) - } - $manifestPath = Join-Path $Dir 'manifest.json' - try { - $manifest | ConvertTo-Json -Depth 4 | - Set-Content -Path $manifestPath -Encoding UTF8 -Force -ErrorAction Stop - Write-Host "[Get-BuildArtifacts] Manifest written: $manifestPath ($($files.Count) file(s))" - } catch { - Write-Warning "[Get-BuildArtifacts] Could not write manifest.json: $_" - } +if ($Credential) { + Write-Host '[Get-BuildArtifacts] -Credential ignored; Python CLI uses keyring (set -CredentialTarget to override target name).' } -if ($GuestOS -eq 'Windows' -and $null -eq $Credential) { - throw "Credential is required when GuestOS is Windows." -} - -# ── Linux branch: use scp to transfer artifacts ───────────────────────────────────────────── -if ($GuestOS -eq 'Linux') { - Import-Module (Join-Path $PSScriptRoot '_Transport.psm1') -Force - - if (-not (Test-Path $HostArtifactDir)) { - New-Item -ItemType Directory -Path $HostArtifactDir -Force | Out-Null - Write-Host "[Get-BuildArtifacts] Created host artifact directory: $HostArtifactDir" - } - - Write-Host "[Get-BuildArtifacts] Transferring artifacts via SCP from $GuestArtifactPath ..." - - Copy-SshItem ` - -Source "$GuestArtifactPath/." ` - -Destination $HostArtifactDir ` - -IP $IPAddress ` - -User $SshUser ` - -KeyPath $SshKeyPath ` - -KnownHostsFile $SshKnownHostsFile ` - -Direction 'FromGuest' ` - -Recurse - - $transferred = @(Get-ChildItem -Path $HostArtifactDir -Recurse -File -ErrorAction SilentlyContinue) - if ($transferred.Count -eq 0) { - throw "[Get-BuildArtifacts] No files found in $HostArtifactDir after SCP transfer." - } - Write-Host "[Get-BuildArtifacts] $($transferred.Count) file(s) collected from Linux guest." - if ($JobId -ne '' -or $Commit -ne '') { - Write-ArtifactManifest -Dir $HostArtifactDir -JobId $JobId -Commit $Commit - } - return -} - -# Ensure destination exists on host -if (-not (Test-Path $HostArtifactDir)) { - New-Item -ItemType Directory -Path $HostArtifactDir -Force | Out-Null - Write-Host "[Get-BuildArtifacts] Created host artifact directory: $HostArtifactDir" -} - -$sessionOptions = New-CISessionOption - -Write-Host "[Get-BuildArtifacts] Connecting to VM at $IPAddress..." -$session = $null -$attempts = 3 -$delay = 10 # seconds between attempts -for ($i = 1; $i -le $attempts; $i++) { - try { - $session = New-PSSession ` - -ComputerName $IPAddress ` - -Credential $Credential ` - -SessionOption $sessionOptions ` - -UseSSL ` - -Port 5986 ` - -Authentication Basic ` - -ErrorAction Stop - break - } catch { - if ($i -eq $attempts) { throw } - Write-Warning "[Get-BuildArtifacts] WinRM connect attempt $i/$attempts failed — retrying in ${delay}s... ($_)" - Start-Sleep -Seconds $delay - } -} - -try { - # ── Verify artifact exists in guest ────────────────────────────────── - $guestExists = Invoke-Command -Session $session -ScriptBlock { - param($path) - Test-Path $path - } -ArgumentList $GuestArtifactPath - - if (-not $guestExists) { - throw "Artifact not found in guest at: $GuestArtifactPath" - } - - # ── Copy main artifact ──────────────────────────────────────────────── - $artifactFileName = Split-Path $GuestArtifactPath -Leaf - $hostDestPath = Join-Path $HostArtifactDir $artifactFileName - - Write-Host "[Get-BuildArtifacts] Copying $artifactFileName from guest..." - Copy-Item -Path $GuestArtifactPath -Destination $hostDestPath -FromSession $session -Force - - # ── Copy logs if requested ──────────────────────────────────────────── - if ($IncludeLogs) { - $logFiles = Invoke-Command -Session $session -ScriptBlock { - Get-ChildItem -Path 'C:\CI\build' -Filter '*.log' -Recurse -ErrorAction SilentlyContinue | - Select-Object -ExpandProperty FullName - } - - foreach ($log in $logFiles) { - $logDest = Join-Path $HostArtifactDir (Split-Path $log -Leaf) - Write-Host "[Get-BuildArtifacts] Copying log: $(Split-Path $log -Leaf)" - Copy-Item -Path $log -Destination $logDest -FromSession $session -Force - } - } - - # ── Validate at least the main artifact landed ──────────────────────── - if (-not (Test-Path $hostDestPath)) { - throw "Artifact was not found at expected host path after copy: $hostDestPath" - } - - $fileItem = Get-Item $hostDestPath - if ($fileItem.Length -eq 0) { - throw "Artifact file exists but is empty: $hostDestPath" - } - $sizeKB = [math]::Round($fileItem.Length / 1KB, 1) - - Write-Host "[Get-BuildArtifacts] Artifact collected: $hostDestPath ($sizeKB KB)" - if ($JobId -ne '' -or $Commit -ne '') { - Write-ArtifactManifest -Dir $HostArtifactDir -JobId $JobId -Commit $Commit - } - return $hostDestPath -} -finally { - Remove-PSSession $session -ErrorAction SilentlyContinue -} +$venvPython = $env:CI_VENV_PYTHON +if (-not $venvPython) { $venvPython = 'F:\CI\python\venv\Scripts\python.exe' } +& $venvPython -m ci_orchestrator artifacts collect @pyArgs +exit $LASTEXITCODE diff --git a/scripts/Invoke-RemoteBuild.ps1 b/scripts/Invoke-RemoteBuild.ps1 index fd31f50..ed230dc 100644 --- a/scripts/Invoke-RemoteBuild.ps1 +++ b/scripts/Invoke-RemoteBuild.ps1 @@ -1,543 +1,91 @@ #Requires -Version 5.1 -<# -.SYNOPSIS - Executes a build inside a guest VM (via WinRM). Supports two clone modes: - (1) Host-side clone: copy pre-cloned source via WinRM zip, or - (2) Guest-side clone: clone repo directly in VM via git (§3.3, requires Git) +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' -.DESCRIPTION - Called by Invoke-CIJob.ps1 to orchestrate the build inside the VM. +# Shim: delegates to the Python ci_orchestrator CLI (`build run`). +# Original PS implementation moved to git history; see plans/A3-closeout.md. +# +# Typed parameters (`[PSCredential]`, `[hashtable]`, `[switch]`) cannot cross +# a process boundary as-is; this shim translates them. Credentials are pulled +# inside Python via keyring (`--credential-target`), so the bound `-Credential` +# is intentionally discarded — callers should set `-CredentialTarget` instead +# (or rely on the default `BuildVMGuest` resolved by the Python CLI). - **Mode 1: Host-side (default)** — Caller provides pre-cloned source: - 1. Compresses the source tree on host, copies archive into VM via WinRM - 2. Runs dotnet restore + dotnet build inside VM - 3. Packages output as C:\CI\output\artifacts.zip - - **Mode 2: Guest-side (§3.3, -UseGitClone in Invoke-CIJob)** — Repo cloned in VM: - 1. Clone repo directly into VM via git (eliminates host-zip-transfer overhead) - 2. Runs dotnet restore + dotnet build inside VM - 3. Packages output same as Mode 1 - - Supply either -HostSourceDir (Mode 1) or -CloneUrl (Mode 2), not both. - - In Mode 2, PAT for private repos is read from Credential Manager inside - this script's WinRM session (§1.5: no argv, no log, cleanup at end). - -.PARAMETER IPAddress - IP of the build VM (e.g. 192.168.11.101). - -.PARAMETER Credential - PSCredential for the build user inside the VM. - Retrieve from Windows Credential Manager in production: - $cred = Get-StoredCredential -Target 'BuildVMGuest' - -.PARAMETER HostSourceDir - [Mode 1] Full path on HOST to already-cloned repository (host-side clone mode). - Mutually exclusive with -CloneUrl. When omitted, -CloneUrl must be provided. - Example: C:\Users\user\AppData\Local\Temp\ci-src-run-42 - -.PARAMETER CloneUrl - [Mode 2, §3.3] Git URL to clone directly in VM (guest-side clone mode). - Mutually exclusive with -HostSourceDir. When provided, repo cloned via git inside VM. - PAT (if needed) read from Credential Manager at runtime. - Example: https://gitea.emulab.it/Simone/myrepo.git - -.PARAMETER CloneBranch - [Mode 2] Branch to clone. Ignored in Mode 1. Default: main - -.PARAMETER CloneCommit - [Mode 2] Specific commit SHA to checkout after clone. Ignored in Mode 1. - -.PARAMETER CloneSubmodules - [Mode 2] Pass --recurse-submodules to git clone. Ignored in Mode 1. - -.PARAMETER Configuration - MSBuild/dotnet build configuration. Default: Release - -.PARAMETER GuestWorkDir - Working directory inside the VM where source will be placed. - Default: C:\CI\build - -.PARAMETER GuestArtifactZip - Full path inside the VM where the artifact archive will be written. - Default: C:\CI\output\artifacts.zip - -.EXAMPLE - $cred = Get-Credential - .\Invoke-RemoteBuild.ps1 ` - -IPAddress "192.168.11.101" ` - -Credential $cred ` - -HostSourceDir "C:\Temp\ci-src-run-42" -#> [CmdletBinding()] param( [Parameter(Mandatory)] - [ValidatePattern('^((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)\.){3}(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)$')] [string] $IPAddress, [Parameter()] [System.Management.Automation.PSCredential] $Credential, - # [Mode 1] Host-side clone — pre-cloned source directory on HOST - [ValidateScript({ - if ($_ -and -not (Test-Path $_ -PathType Container)) { - throw "HostSourceDir path does not exist: $_" - } - return $true - })] [string] $HostSourceDir = '', - - # [Mode 2, §3.3] Guest-side clone — Git URL for in-VM clone [string] $CloneUrl = '', - - # [Mode 2] Branch to clone (default: main). Ignored if HostSourceDir provided. [string] $CloneBranch = 'main', - - # [Mode 2] Specific commit SHA to checkout. Ignored if HostSourceDir provided. [string] $CloneCommit = '', - - # [Mode 2] Clone with --recurse-submodules. Ignored if HostSourceDir provided. [switch] $CloneSubmodules, - - # [Mode 2] Credential Manager target for Gitea PAT (private repos). - # Store with: New-StoredCredential -Target 'GiteaPAT' -UserName '' -Password '' -Persist LocalMachine - # Leave empty for public repos (no auth needed). [string] $GiteaCredentialTarget = 'GiteaPAT', - [string] $Configuration = 'Release', - - # Custom build command to run inside the VM (working dir = GuestWorkDir). - # When empty, defaults to: dotnet restore + dotnet build --configuration $Configuration - # Example: 'python build_plugin.py --final' [string] $BuildCommand = '', - - # Glob pattern (relative to GuestWorkDir) of files to include in the artifact zip. - # Used only when BuildCommand is set. Default: collect everything under 'dist'. - # Example: 'dist\**' or 'Contrib\nsInnoUnp\Build\**' [string] $GuestArtifactSource = 'dist', - [string] $GuestWorkDir = 'C:\CI\build', - [string] $GuestArtifactZip = 'C:\CI\output\artifacts.zip', - - # Redirect dotnet NuGet package store and pip download cache to VMware shared - # folders on the host (F:\CI\Cache\NuGet and F:\CI\Cache\pip). - # Requires the template VMX to have been configured by Set-TemplateSharedFolders.ps1 - # and VMware Tools HGFS driver present in the guest. [switch] $UseSharedCache, - - # Skip artifact packaging entirely. Use for jobs that produce no output. - # When absent, missing GuestArtifactSource after a successful build is an error. [switch] $SkipArtifact, - # Guest OS type — Linux uses SSH instead of WinRM. [ValidateSet('Windows', 'Linux')] [string] $GuestOS = 'Windows', - # SSH private key path (used when GuestOS=Linux). [string] $SshKeyPath = 'F:\CI\keys\ci_linux', - - # SSH username (used when GuestOS=Linux). [string] $SshUser = 'ci_build', - - # Persistent known_hosts file for SSH calls (CI jobs). - # Empty string (default) = permissive mode (StrictHostKeyChecking=no, UserKnownHostsFile=NUL). [string] $SshKnownHostsFile = '', - - # Work directory inside the Linux guest. [string] $GuestLinuxWorkDir = '/opt/ci/build', - - # Output directory inside the Linux guest. [string] $GuestLinuxOutputDir = '/opt/ci/output', - # Extra environment variables injected into the guest before the build command (§6.5). - # Keys must be valid env var names. Values are plain strings (never logged). - [hashtable] $ExtraGuestEnv = @{} + [hashtable] $ExtraGuestEnv = @{}, + + # Optional override for the Credential Manager target (overrides any + # fallback resolved by Python). Discards $Credential. + [string] $CredentialTarget = '' ) -Set-StrictMode -Version Latest -$ErrorActionPreference = 'Stop' +$pyArgs = @( + '--ip-address', $IPAddress, + '--guest-os', $GuestOS.ToLower(), + '--ssh-user', $SshUser, + '--ssh-key-path', $SshKeyPath, + '--clone-branch', $CloneBranch, + '--gitea-credential-target', $GiteaCredentialTarget, + '--configuration', $Configuration, + '--guest-work-dir', $GuestWorkDir, + '--guest-artifact-zip', $GuestArtifactZip, + '--guest-linux-work-dir', $GuestLinuxWorkDir, + '--guest-linux-output-dir', $GuestLinuxOutputDir, + '--guest-artifact-source', $GuestArtifactSource +) -Import-Module (Join-Path $PSScriptRoot '_Common.psm1') -Force +if ($HostSourceDir) { $pyArgs += @('--host-source-dir', $HostSourceDir) } +if ($CloneUrl) { $pyArgs += @('--clone-url', $CloneUrl) } +if ($CloneCommit) { $pyArgs += @('--clone-commit', $CloneCommit) } +if ($BuildCommand) { $pyArgs += @('--build-command', $BuildCommand) } +if ($SshKnownHostsFile) { $pyArgs += @('--ssh-known-hosts-file', $SshKnownHostsFile) } +if ($CredentialTarget) { $pyArgs += @('--credential-target', $CredentialTarget) } -# ── Validate mutually exclusive modes ───────────────────────────────────── -$hostCloneMode = -not [string]::IsNullOrWhiteSpace($HostSourceDir) -$guestCloneMode = -not [string]::IsNullOrWhiteSpace($CloneUrl) -if ($hostCloneMode -and $guestCloneMode) { - throw "Provide either -HostSourceDir (host-side clone) or -CloneUrl (guest-side clone), not both." -} -if (-not $hostCloneMode -and -not $guestCloneMode) { - throw "Provide either -HostSourceDir (host-side clone) or -CloneUrl (guest-side clone)." -} -if ($guestCloneMode -and -not $CloneBranch) { - throw "CloneBranch is required when using -CloneUrl mode." -} +if ($CloneSubmodules.IsPresent) { $pyArgs += '--clone-submodules' } +if ($UseSharedCache.IsPresent) { $pyArgs += '--use-shared-cache' } +if ($SkipArtifact.IsPresent) { $pyArgs += '--skip-artifact' } -if ($GuestOS -eq 'Windows' -and $null -eq $Credential) { - throw "Credential is required when GuestOS is Windows." -} - -if ($GuestOS -eq 'Linux') { - Import-Module (Join-Path $PSScriptRoot '_Transport.psm1') -Force - - Write-Host "[Invoke-RemoteBuild] Linux build mode — SSH transport" - - # Step 1: Clean work dir (parent permissions handled by template setup) - Invoke-SshCommand -IP $IPAddress -User $SshUser -KeyPath $SshKeyPath ` - -KnownHostsFile $SshKnownHostsFile ` - -Command "rm -rf '$GuestLinuxWorkDir' && mkdir -p '$GuestLinuxWorkDir'" - - if ($hostCloneMode) { - # ── Mode 1 (Linux): tar host source -> scp -> untar in guest ── - Write-Host "[Invoke-RemoteBuild] Packaging host source ($HostSourceDir) ..." - $tarName = "ci-src-$([Guid]::NewGuid().ToString('N').Substring(0,8)).tar.gz" - $hostTar = Join-Path $env:TEMP $tarName - $guestTar = "/tmp/$tarName" - if (Test-Path $hostTar) { Remove-Item $hostTar -Force } - # bsdtar on Windows 10/11 supports -C and gz natively. - & tar -czf $hostTar -C $HostSourceDir . - if ($LASTEXITCODE -ne 0) { - throw "[Invoke-RemoteBuild] tar failed packaging $HostSourceDir (exit $LASTEXITCODE)" - } - try { - Write-Host "[Invoke-RemoteBuild] Copying source archive to guest:$guestTar ..." - Copy-SshItem -Source $hostTar -Destination $guestTar ` - -IP $IPAddress -User $SshUser -KeyPath $SshKeyPath ` - -KnownHostsFile $SshKnownHostsFile -Direction ToGuest - - Write-Host "[Invoke-RemoteBuild] Extracting source in guest ($GuestLinuxWorkDir) ..." - Invoke-SshCommand -IP $IPAddress -User $SshUser -KeyPath $SshKeyPath ` - -KnownHostsFile $SshKnownHostsFile ` - -Command "tar -xzf '$guestTar' -C '$GuestLinuxWorkDir' && rm -f '$guestTar'" - } - finally { - if (Test-Path $hostTar) { Remove-Item $hostTar -Force -ErrorAction SilentlyContinue } - } - } - else { - # ── Mode 2 (Linux): in-guest git clone ── - $cloneCmd = "git clone --depth 1 --branch '$CloneBranch'" - if ($CloneSubmodules) { $cloneCmd += ' --recurse-submodules' } - - $cloneCmd += " '$CloneUrl' '$GuestLinuxWorkDir'" # non-authed URL for logging - Write-Host "[Invoke-RemoteBuild] Cloning $CloneUrl into guest $GuestLinuxWorkDir ..." - - # PAT injection via git http.extraHeader — PAT never appears in argv or /proc/environ - $authHeader = '' - if ($GiteaCredentialTarget -ne '') { - try { - Import-Module CredentialManager -ErrorAction Stop - $credObj = Get-StoredCredential -Target $GiteaCredentialTarget -ErrorAction Stop - if ($credObj) { - $b64 = [Convert]::ToBase64String( - [System.Text.Encoding]::UTF8.GetBytes( - "$($credObj.UserName):$($credObj.GetNetworkCredential().Password)")) - $authHeader = "Authorization: Basic $b64" - } - } catch { - Write-Warning "[Invoke-RemoteBuild] Could not load PAT from Credential Manager ($GiteaCredentialTarget) — cloning without auth (public repo?)." - } - } - - if ($authHeader -ne '') { - $authCloneCmd = "git -c credential.helper= -c 'http.extraHeader=$authHeader' clone --depth 1 --branch '$CloneBranch'" - if ($CloneSubmodules) { $authCloneCmd += ' --recurse-submodules' } - $authCloneCmd += " '$CloneUrl' '$GuestLinuxWorkDir'" - Invoke-SshCommand -IP $IPAddress -User $SshUser -KeyPath $SshKeyPath ` - -KnownHostsFile $SshKnownHostsFile -Command $authCloneCmd - } else { - Invoke-SshCommand -IP $IPAddress -User $SshUser -KeyPath $SshKeyPath ` - -KnownHostsFile $SshKnownHostsFile -Command $cloneCmd - } - - # Checkout specific commit if requested - if ($CloneCommit -ne '') { - Invoke-SshCommand -IP $IPAddress -User $SshUser -KeyPath $SshKeyPath ` - -KnownHostsFile $SshKnownHostsFile ` - -Command "git -C '$GuestLinuxWorkDir' checkout '$CloneCommit'" - } - } - - # Step 3: Run build command (always cd into workdir first) - # Prepend extra env exports (§6.5) — scoped to this SSH command only - $envPrefix = '' - foreach ($envKey in $ExtraGuestEnv.Keys) { - $envVal = $ExtraGuestEnv[$envKey] -replace "'", "'\''" # escape ' for POSIX single-quoted string - $envPrefix += "export $envKey='$envVal'; " - } - if ($BuildCommand -ne '') { - $buildCmd = "${envPrefix}cd '$GuestLinuxWorkDir' && $BuildCommand" - } else { - $buildCmd = "${envPrefix}cd '$GuestLinuxWorkDir' && make" - } - $displayCmd = if ($BuildCommand -ne '') { $BuildCommand } else { 'make' } - Write-Host "[Invoke-RemoteBuild] Running build (env vars redacted): cd '$GuestLinuxWorkDir' && $displayCmd" - Invoke-SshCommand -IP $IPAddress -User $SshUser -KeyPath $SshKeyPath ` - -KnownHostsFile $SshKnownHostsFile -Command $buildCmd - - # Step 4: Stage artifact tree under $GuestLinuxOutputDir for SCP collection - $outDir = $GuestLinuxOutputDir - Invoke-SshCommand -IP $IPAddress -User $SshUser -KeyPath $SshKeyPath ` - -KnownHostsFile $SshKnownHostsFile ` - -Command "rm -rf '$outDir' && mkdir -p '$outDir'" - - $artSrc = if ($GuestArtifactSource -ne '') { $GuestArtifactSource } else { 'dist' } - Invoke-SshCommand -IP $IPAddress -User $SshUser -KeyPath $SshKeyPath ` - -KnownHostsFile $SshKnownHostsFile ` - -Command "if [ -d '$GuestLinuxWorkDir/$artSrc' ]; then cp -r '$GuestLinuxWorkDir/$artSrc/.' '$outDir/'; elif [ -e '$GuestLinuxWorkDir/$artSrc' ]; then cp '$GuestLinuxWorkDir/$artSrc' '$outDir/'; else echo '[Invoke-RemoteBuild] WARNING: artifact source not found: $artSrc' >&2; fi" - - Write-Host "[Invoke-RemoteBuild] Linux build complete." - return -} - -function Compress-BuildArtifact { - param( - [Parameter(Mandatory)] - [string] $Path, - [Parameter(Mandatory)] - [string] $DestinationPath - ) - $7zip = 'C:\Program Files\7-Zip\7z.exe' - if (Test-Path $7zip) { - Write-Host "[Compress-BuildArtifact] Using 7-Zip (multi-threaded)..." - & $7zip a -mmt=on -mx1 $DestinationPath $Path 2>&1 | Where-Object { $_ -notmatch '^7-Zip' } | ForEach-Object { Write-Host $_ } - if ($LASTEXITCODE -ne 0) { - throw "7-Zip compression failed (exit $LASTEXITCODE)" - } - } - else { - Write-Host "[Compress-BuildArtifact] 7-Zip not found, using Compress-Archive (fallback)..." - Compress-Archive -Path $Path -DestinationPath $DestinationPath -CompressionLevel Fastest -Force +if ($ExtraGuestEnv -and $ExtraGuestEnv.Count -gt 0) { + foreach ($key in $ExtraGuestEnv.Keys) { + $pyArgs += @('--extra-env', "$key=$($ExtraGuestEnv[$key])") } } -$sessionOptions = New-CISessionOption - -Write-Host "[Invoke-RemoteBuild] Connecting to VM at $IPAddress..." - -$session = New-PSSession ` - -ComputerName $IPAddress ` - -Credential $Credential ` - -SessionOption $sessionOptions ` - -UseSSL ` - -Port 5986 ` - -Authentication Basic ` - -ErrorAction Stop - -try { - # ── Ensure working directories exist on guest ──────────────────────── - Invoke-Command -Session $session -ScriptBlock { - param($workDir, $outputDir) - # Ensure artifact output parent exists - $outParent = Split-Path $outputDir -Parent - if (-not (Test-Path $outParent)) { - New-Item -ItemType Directory -Path $outParent -Force | Out-Null - } - # Clean and recreate build dir (ephemeral VM, but guard against partial prior run) - if (Test-Path $workDir) { Remove-Item $workDir -Recurse -Force } - New-Item -ItemType Directory -Path $workDir -Force | Out-Null - } -ArgumentList $GuestWorkDir, $GuestArtifactZip - - # ── Source preparation: Host clone (compress+transfer) or Guest clone (git) ── - if ($hostCloneMode) { - # Mode 1: Copy pre-cloned source from host to guest via WinRM zip - # Single-file transfer over WinRM is far faster than recursive Copy-Item. - $hostZip = [System.IO.Path]::GetTempFileName() -replace '\.tmp$', '.zip' - try { - Write-Host "[Invoke-RemoteBuild] Compressing source on host..." - Compress-BuildArtifact -Path "$HostSourceDir\*" -DestinationPath $hostZip - $zipSize = [math]::Round((Get-Item $hostZip).Length / 1MB, 1) - Write-Host "[Invoke-RemoteBuild] Transferring archive to guest ($zipSize MB)..." - $guestZip = 'C:\CI\src-transfer.zip' - Copy-Item -Path $hostZip -Destination $guestZip -ToSession $session -Force - Write-Host "[Invoke-RemoteBuild] Expanding archive in guest..." - Invoke-Command -Session $session -ScriptBlock { - param($zip, $dest) - Expand-Archive -Path $zip -DestinationPath $dest -Force - Remove-Item $zip -Force - } -ArgumentList $guestZip, $GuestWorkDir - Write-Host "[Invoke-RemoteBuild] Source transfer complete." - } - finally { - Remove-Item $hostZip -Force -ErrorAction SilentlyContinue - } - } - else { - # Mode 2 (§3.3): Clone repo directly in guest via git - # PAT (if needed) read from Credential Manager inside session, never logged - Write-Host "[Invoke-RemoteBuild] Cloning repository in guest (§3.3)..." - - # Read PAT from host Credential Manager (§1.5: never in argv or logs) - $gitPatUser = $null - $gitPatPass = $null - if (-not [string]::IsNullOrWhiteSpace($GiteaCredentialTarget)) { - try { - $stored = Get-StoredCredential -Target $GiteaCredentialTarget -ErrorAction Stop - $gitPatUser = $stored.UserName - $gitPatPass = $stored.GetNetworkCredential().Password - Write-Host "[Invoke-RemoteBuild] PAT loaded from Credential Manager ($GiteaCredentialTarget)." - } catch { - Write-Warning "[Invoke-RemoteBuild] Could not read PAT from Credential Manager '$GiteaCredentialTarget' — proceeding without auth (public repo only)." - } - } - - # Clone in guest via WinRM - Invoke-Command -Session $session -ScriptBlock { - param($cloneUrl, $branch, $commit, $workDir, $patUser, $patPass, $submodules) - - Set-Location (Split-Path $workDir -Parent) - $gitArgs = @('clone', '--depth', '1', '--branch', $branch) - if ($submodules) { $gitArgs += '--recurse-submodules' } - $gitArgs += @($cloneUrl, (Split-Path $workDir -Leaf)) - - # Disable interactive credential prompts — WinRM has no TTY - $env:GIT_TERMINAL_PROMPT = '0' - - # Build git config args: - # - credential.helper= (empty) clears GCM and any system credential helper - # - http.extraHeader injects Basic auth without credential helpers, temp files, - # or modifying the clone URL (§1.5: token never in URL, argv display, or logs) - $gitConfigArgs = @('-c', 'credential.helper=') - if ($patUser -and $patPass) { - $b64 = [Convert]::ToBase64String( - [Text.Encoding]::UTF8.GetBytes("${patUser}:${patPass}")) - $gitConfigArgs += @('-c', "http.extraHeader=Authorization: Basic $b64") - } - - $gitArgs = $gitConfigArgs + $gitArgs - - Write-Host "[Guest] Cloning $cloneUrl (branch: $branch)..." - & git @gitArgs 2>&1 | ForEach-Object { Write-Host "[Guest Clone] $_" } - if ($LASTEXITCODE -ne 0) { - throw "git clone failed in guest (exit $LASTEXITCODE)" - } - - # Checkout specific commit if provided - if ($commit) { - Write-Host "[Guest] Checking out commit: $commit" - Set-Location $workDir - & git checkout $commit 2>&1 | ForEach-Object { Write-Host "[Guest Checkout] $_" } - if ($LASTEXITCODE -ne 0) { - throw "git checkout $commit failed in guest (exit $LASTEXITCODE)" - } - } - } -ArgumentList $CloneUrl, $CloneBranch, $CloneCommit, $GuestWorkDir, $gitPatUser, $gitPatPass, $CloneSubmodules.IsPresent - - Write-Host "[Invoke-RemoteBuild] Guest clone complete." - } - - # ── Build ────────────────────────────────────────────────────────── - if ($BuildCommand -ne '') { - # ── Custom build command (e.g. python, msbuild, cmake…) ────────── - Write-Host "[Invoke-RemoteBuild] Running build command: $BuildCommand" - $buildExit = Invoke-Command -Session $session -ScriptBlock { - param($workDir, $cmd, $artifactZip, $artifactSrc, $useCache, $extraEnv, $skipArt) - # Unbuffered Python output — lines appear in real-time instead of being held in buffer - $env:PYTHONUNBUFFERED = '1' - if ($useCache) { - $env:PIP_CACHE_DIR = '\\vmware-host\Shared Folders\pip-cache' - } - # Inject extra env vars (§6.5 secret injection) - foreach ($pair in $extraEnv.GetEnumerator()) { - [System.Environment]::SetEnvironmentVariable($pair.Key, $pair.Value, 'Process') - } - Set-Location $workDir - # Stream each output line via Write-Host — WinRM forwards Write-Host in real-time - & cmd /c "$cmd 2>&1" | ForEach-Object { Write-Host $_ } - $exit = $LASTEXITCODE - - if ($exit -eq 0 -and -not $skipArt) { - $archiveDir = Split-Path $artifactZip -Parent - if (-not (Test-Path $archiveDir)) { - New-Item -ItemType Directory -Path $archiveDir -Force | Out-Null - } - $srcPath = Join-Path $workDir $artifactSrc - if (-not (Test-Path $srcPath)) { - throw "Artifact source directory not found: $srcPath. Build succeeded (exit 0) but produced no output. Check GuestArtifactSource or set skip-artifact: 'true' in the workflow." - } - else { - $7zip = 'C:\Program Files\7-Zip\7z.exe' - if (Test-Path $7zip) { - Write-Host "[Build] Compressing artifacts with 7-Zip..." - & $7zip a -mmt=on -mx1 $artifactZip $srcPath 2>&1 | Where-Object { $_ -notmatch '^7-Zip' } | ForEach-Object { Write-Host $_ } - if ($LASTEXITCODE -ne 0) { throw "7-Zip failed" } - } - else { - Compress-Archive -Path $srcPath -DestinationPath $artifactZip -Force - } - } - } - - return $exit - } -ArgumentList $GuestWorkDir, $BuildCommand, $GuestArtifactZip, $GuestArtifactSource, $UseSharedCache.IsPresent, $ExtraGuestEnv, $SkipArtifact.IsPresent - - if ($buildExit -ne 0) { - throw "Build command failed (exit $buildExit)" - } - } - else { - # ── Default: dotnet restore + dotnet build ──────────────────────── - Write-Host "[Invoke-RemoteBuild] Running dotnet restore..." - $restoreExit = Invoke-Command -Session $session -ScriptBlock { - param($workDir, $useCache) - if ($useCache) { - $env:NUGET_PACKAGES = '\\vmware-host\Shared Folders\nuget-cache' - } - Set-Location $workDir - dotnet restore 2>&1 | ForEach-Object { Write-Host $_ } - return $LASTEXITCODE - } -ArgumentList $GuestWorkDir, $UseSharedCache.IsPresent - - if ($restoreExit -ne 0) { - throw "dotnet restore failed (exit $restoreExit)" - } - - Write-Host "[Invoke-RemoteBuild] Running dotnet build (configuration: $Configuration)..." - $buildExit = Invoke-Command -Session $session -ScriptBlock { - param($workDir, $config, $artifactZip, $useCache) - if ($useCache) { - $env:NUGET_PACKAGES = '\\vmware-host\Shared Folders\nuget-cache' - } - Set-Location $workDir - $outputDir = Join-Path $workDir 'bin\CIOutput' - dotnet build --configuration $config --output $outputDir 2>&1 | ForEach-Object { Write-Host $_ } - $exit = $LASTEXITCODE - - if ($exit -eq 0) { - $archiveDir = Split-Path $artifactZip -Parent - if (-not (Test-Path $archiveDir)) { - New-Item -ItemType Directory -Path $archiveDir -Force | Out-Null - } - if (-not (Test-Path $outputDir)) { - Write-Host "[Build] Output dir '$outputDir' not found — skipping packaging." - } - else { - $7zip = 'C:\Program Files\7-Zip\7z.exe' - if (Test-Path $7zip) { - Write-Host "[Build] Compressing output with 7-Zip..." - & $7zip a -mmt=on -mx1 $artifactZip "$outputDir\*" 2>&1 | Where-Object { $_ -notmatch '^7-Zip' } | ForEach-Object { Write-Host $_ } - if ($LASTEXITCODE -ne 0) { throw "7-Zip failed" } - } - else { - Compress-Archive -Path "$outputDir\*" -DestinationPath $artifactZip -Force - } - } - } - - return $exit - } -ArgumentList $GuestWorkDir, $Configuration, $GuestArtifactZip, $UseSharedCache.IsPresent - - if ($buildExit -ne 0) { - throw "dotnet build failed (exit $buildExit)" - } - } - - if ($SkipArtifact) { - Write-Host '[Invoke-RemoteBuild] Build succeeded. Artifact packaging skipped (-SkipArtifact).' - } else { - Write-Host "[Invoke-RemoteBuild] Build succeeded. Artifact at $GuestArtifactZip" - } -} -finally { - Remove-PSSession $session -ErrorAction SilentlyContinue +if ($Credential) { + Write-Host '[Invoke-RemoteBuild] -Credential ignored; Python CLI uses keyring (set -CredentialTarget to override target name).' } + +$venvPython = $env:CI_VENV_PYTHON +if (-not $venvPython) { $venvPython = 'F:\CI\python\venv\Scripts\python.exe' } +& $venvPython -m ci_orchestrator build run @pyArgs +exit $LASTEXITCODE diff --git a/scripts/New-BuildVM.ps1 b/scripts/New-BuildVM.ps1 index c1964b5..5470f77 100644 --- a/scripts/New-BuildVM.ps1 +++ b/scripts/New-BuildVM.ps1 @@ -1,43 +1,10 @@ #Requires -Version 5.1 -<# -.SYNOPSIS - Creates a linked clone of the template VM for an ephemeral CI build. +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' -.DESCRIPTION - Uses vmrun.exe to create a linked clone from a frozen template snapshot. - Returns the full path to the new clone's .vmx file on success. - The template VM and its snapshot are never modified. +# Shim: delegates to the Python ci_orchestrator CLI (`vm new`). +# Original PS implementation moved to git history; see plans/A3-closeout.md. -.PARAMETER TemplatePath - Full path to the template VM's .vmx file. - Example: F:\CI\Templates\WinBuild\WinBuild.vmx - -.PARAMETER SnapshotName - Name of the snapshot on the template VM to clone from. - Default: BaseClean - -.PARAMETER CloneBaseDir - Directory under which the new clone folder will be created. - The clone folder is named after the job ID and timestamp. - Example: F:\CI\BuildVMs - -.PARAMETER JobId - Unique identifier for the CI job (e.g. run ID from Gitea Actions). - Used to name the clone: Clone_{JobId}_{yyyyMMdd_HHmmss} - -.PARAMETER VmrunPath - Full path to vmrun.exe. - Default: C:\Program Files (x86)\VMware\VMware Workstation\vmrun.exe - -.OUTPUTS - [string] Full path to the new clone's .vmx file. - -.EXAMPLE - $vmxPath = .\New-BuildVM.ps1 ` - -TemplatePath "F:\CI\Templates\WinBuild\WinBuild.vmx" ` - -CloneBaseDir "F:\CI\BuildVMs" ` - -JobId "run-42" -#> [CmdletBinding()] param( [Parameter(Mandatory)] @@ -55,49 +22,15 @@ param( [string] $VmrunPath = 'C:\Program Files (x86)\VMware\VMware Workstation\vmrun.exe' ) -Set-StrictMode -Version Latest -$ErrorActionPreference = 'Stop' +$pyArgs = @( + '--template-path', $TemplatePath, + '--snapshot-name', $SnapshotName, + '--clone-base-dir', $CloneBaseDir, + '--job-id', $JobId, + '--vmrun-path', $VmrunPath +) -Import-Module (Join-Path $PSScriptRoot '_Common.psm1') -Force - -Resolve-VmrunPath -VmrunPath $VmrunPath | Out-Null - -# --- Build clone path --- -$timestamp = Get-Date -Format 'yyyyMMdd_HHmmss' -$cloneName = "Clone_${JobId}_${timestamp}" -$cloneDir = Join-Path $CloneBaseDir $cloneName -$cloneVmx = Join-Path $cloneDir "$cloneName.vmx" - -# Ensure clone base dir exists -if (-not (Test-Path $CloneBaseDir)) { - New-Item -ItemType Directory -Path $CloneBaseDir -Force | Out-Null -} - -Write-Host "[New-BuildVM] Creating linked clone..." -Write-Host " Template : $TemplatePath" -Write-Host " Snapshot : $SnapshotName" -Write-Host " Clone VMX : $cloneVmx" - -$sw = [System.Diagnostics.Stopwatch]::StartNew() - -$cloneResult = Invoke-Vmrun -VmrunPath $VmrunPath -Operation clone ` - -Arguments @($TemplatePath, $cloneVmx, 'linked', '-snapshot', $SnapshotName) - -$sw.Stop() - -if ($cloneResult.ExitCode -ne 0) { - # Clean up partial clone directory if it was created - if (Test-Path $cloneDir) { - Remove-Item $cloneDir -Recurse -Force -ErrorAction SilentlyContinue - } - throw "vmrun clone failed (exit $($cloneResult.ExitCode)).`nOutput: $($cloneResult.Output)" -} - -if (-not (Test-Path $cloneVmx -PathType Leaf)) { - throw "vmrun reported success but clone VMX not found at: $cloneVmx" -} - -Write-Host "[New-BuildVM] Clone created in $($sw.Elapsed.TotalSeconds.ToString('F1'))s : $cloneVmx" - -# Return the VMX path as output -return $cloneVmx +$venvPython = $env:CI_VENV_PYTHON +if (-not $venvPython) { $venvPython = 'F:\CI\python\venv\Scripts\python.exe' } +& $venvPython -m ci_orchestrator vm new @pyArgs +exit $LASTEXITCODE diff --git a/src/ci_orchestrator/__main__.py b/src/ci_orchestrator/__main__.py index c00e2e2..6999518 100644 --- a/src/ci_orchestrator/__main__.py +++ b/src/ci_orchestrator/__main__.py @@ -1,8 +1,8 @@ """CLI entry point. Phase A2 ships ``wait-ready``, ``vm`` (remove/cleanup), ``monitor`` -(disk/runner) and ``report`` (job). Subsequent phases add ``vm new``, -``build``, ``artifacts``, ``job``. +(disk/runner) and ``report`` (job). Phase A3 adds ``vm new``, ``build``, +``artifacts``. Subsequent phases add ``job``. """ from __future__ import annotations @@ -12,6 +12,8 @@ import time as time import click from ci_orchestrator import __version__ +from ci_orchestrator.commands.artifacts import artifacts +from ci_orchestrator.commands.build import build from ci_orchestrator.commands.monitor import monitor from ci_orchestrator.commands.report import report from ci_orchestrator.commands.vm import vm @@ -34,6 +36,8 @@ def cli() -> None: cli.add_command(wait_ready) cli.add_command(vm) +cli.add_command(build) +cli.add_command(artifacts) cli.add_command(monitor) cli.add_command(report) diff --git a/src/ci_orchestrator/commands/artifacts.py b/src/ci_orchestrator/commands/artifacts.py new file mode 100644 index 0000000..f6b669e --- /dev/null +++ b/src/ci_orchestrator/commands/artifacts.py @@ -0,0 +1,268 @@ +"""``artifacts`` sub-commands. + +Phase A3 ports the artifact-collection script: + +* ``artifacts collect`` — replaces ``scripts/Get-BuildArtifacts.ps1`` + +Uses the abstract :meth:`~ci_orchestrator.transport.winrm.WinRmTransport.fetch` +and the SSH transport (via a tar+fetch helper) so no platform-specific +``Copy-Item`` / ``scp.exe`` invocations leak into command code. +""" + +from __future__ import annotations + +import hashlib +import json +import shlex +import tarfile +import tempfile +from datetime import UTC, datetime +from pathlib import Path + +import click + +from ci_orchestrator.config import load_config +from ci_orchestrator.credentials import KeyringCredentialStore +from ci_orchestrator.transport.errors import TransportCommandError, TransportError +from ci_orchestrator.transport.ssh import SshTransport +from ci_orchestrator.transport.winrm import WinRmTransport + +# ---------------------------------------------------------------- group + + +@click.group("artifacts") +def artifacts() -> None: + """Artifact transfer commands.""" + + +# ---------------------------------------------------------------- helpers + + +def _sh_quote(value: str) -> str: + return shlex.quote(value) + + +def _ps_quote(value: str) -> str: + return "'" + value.replace("'", "''") + "'" + + +def _sha256_hex(path: Path) -> str: + h = hashlib.sha256() + with path.open("rb") as fh: + for chunk in iter(lambda: fh.read(64 * 1024), b""): + h.update(chunk) + return h.hexdigest() + + +def _write_manifest(host_dir: Path, job_id: str, commit: str) -> int: + """Write ``manifest.json`` summarising every collected file. Returns count.""" + files = [ + p + for p in host_dir.rglob("*") + if p.is_file() and p.name != "manifest.json" + ] + entries = [] + for f in files: + rel = f.relative_to(host_dir).as_posix() + entries.append( + { + "name": rel, + "size": f.stat().st_size, + "sha256": _sha256_hex(f), + } + ) + payload = { + "ts": datetime.now(tz=UTC).isoformat(), + "jobId": job_id, + "commit": commit, + "files": entries, + } + out = host_dir / "manifest.json" + out.write_text(json.dumps(payload, indent=2), encoding="utf-8") + return len(entries) + + +# ---------------------------------------------------------------- Linux flow + + +def _linux_collect( + *, + ip_address: str, + ssh_user: str, + key_path: str | None, + known_hosts: str | None, + guest_path: str, + host_dir: Path, +) -> None: + """Tar the guest directory, fetch the single archive, extract locally.""" + click.echo(f"[artifacts collect] tarring guest:{guest_path}") + archive_remote = f"/tmp/ci-artifacts-{ip_address.replace('.', '-')}.tar.gz" + with SshTransport( + ip_address, + username=ssh_user, + key_path=key_path, + known_hosts=known_hosts, + ) as t: + # Build a tarball whose contents are *inside* guest_path. Strip the + # parent component so files land directly under host_dir. + t.run( + f"tar -czf {_sh_quote(archive_remote)} " + f"-C {_sh_quote(guest_path)} ." + ) + try: + with tempfile.TemporaryDirectory() as td: + local_tar = Path(td) / "artifacts.tar.gz" + t.fetch(archive_remote, str(local_tar)) + with tarfile.open(local_tar, "r:gz") as tf: + # ``data`` filter (Python 3.12+) protects against path + # traversal in malicious tarballs. + tf.extractall(host_dir, filter="data") + finally: + t.run(f"rm -f {_sh_quote(archive_remote)}", check=False) + + +# ---------------------------------------------------------------- Windows flow + + +def _windows_collect( + *, + ip_address: str, + credential_target: str, + guest_path: str, + host_dir: Path, + include_logs: bool, +) -> None: + cred = KeyringCredentialStore().get(credential_target) + with WinRmTransport(ip_address, cred.username, cred.password) as t: + # Verify artifact exists. + probe = t.run( + f"if (Test-Path {_ps_quote(guest_path)}) {{ 'OK' }} else {{ 'MISSING' }}", + check=False, + ) + if "MISSING" in probe.stdout: + raise click.ClickException(f"artifact not found in guest: {guest_path}") + + artifact_name = Path(guest_path).name + host_dest = host_dir / artifact_name + click.echo(f"[artifacts collect] fetching {artifact_name}") + t.fetch(guest_path, str(host_dest)) + + if include_logs: + # List log files in the build dir and fetch each one. + list_ps = ( + "Get-ChildItem -Path 'C:\\CI\\build' -Filter '*.log' -Recurse " + "-ErrorAction SilentlyContinue | " + "ForEach-Object { $_.FullName }" + ) + listing = t.run(list_ps, check=False) + for line in (listing.stdout or "").splitlines(): + line = line.strip() + if not line: + continue + dest = host_dir / Path(line).name + click.echo(f"[artifacts collect] fetching log: {Path(line).name}") + try: + t.fetch(line, str(dest)) + except TransportError as exc: + click.echo( + f"[artifacts collect] log fetch failed ({line}): {exc}", + err=True, + ) + + if not host_dest.is_file() or host_dest.stat().st_size == 0: + raise click.ClickException( + f"artifact missing or empty after fetch: {host_dest}" + ) + + +# ---------------------------------------------------------------- CLI + + +@artifacts.command("collect") +@click.option("--ip-address", "ip_address", required=True) +@click.option( + "--guest-os", + type=click.Choice(["windows", "linux"], case_sensitive=False), + default="windows", + show_default=True, +) +@click.option( + "--guest-artifact-path", + "guest_artifact_path", + required=True, + help="Guest path: file (Windows) or directory (Linux).", +) +@click.option( + "--host-artifact-dir", + "host_artifact_dir", + required=True, + help="Local directory to deposit artifacts (created if missing).", +) +@click.option("--include-logs", "include_logs", is_flag=True, default=False) +@click.option("--credential-target", "credential_target", default=None) +@click.option("--ssh-user", "ssh_user", default="ci_build", show_default=True) +@click.option("--ssh-key-path", "ssh_key_path", default=None) +@click.option("--ssh-known-hosts-file", "ssh_known_hosts_file", default=None) +@click.option("--job-id", "job_id", default="") +@click.option("--commit", "commit", default="") +def artifacts_collect( + ip_address: str, + guest_os: str, + guest_artifact_path: str, + host_artifact_dir: str, + include_logs: bool, + credential_target: str | None, + ssh_user: str, + ssh_key_path: str | None, + ssh_known_hosts_file: str | None, + job_id: str, + commit: str, +) -> None: + """Copy build artifacts from a guest VM to a local directory. + + Mirrors ``scripts/Get-BuildArtifacts.ps1``. Writes ``manifest.json`` + when ``--job-id`` or ``--commit`` is provided. + """ + guest_os = guest_os.lower() + host_dir = Path(host_artifact_dir) + host_dir.mkdir(parents=True, exist_ok=True) + config = load_config() + + try: + if guest_os == "linux": + key_path: str | None = ssh_key_path + if key_path is None and config.ssh_key_path is not None: + key_path = str(config.ssh_key_path) + _linux_collect( + ip_address=ip_address, + ssh_user=ssh_user, + key_path=key_path, + known_hosts=ssh_known_hosts_file, + guest_path=guest_artifact_path, + host_dir=host_dir, + ) + else: + target = credential_target or config.guest_cred_target + _windows_collect( + ip_address=ip_address, + credential_target=target, + guest_path=guest_artifact_path, + host_dir=host_dir, + include_logs=include_logs, + ) + except (TransportError, TransportCommandError) as exc: + raise click.ClickException(str(exc)) from exc + + transferred = [p for p in host_dir.rglob("*") if p.is_file()] + if not transferred: + raise click.ClickException( + f"no files were transferred into {host_dir}" + ) + click.echo(f"[artifacts collect] {len(transferred)} file(s) collected.") + + if job_id or commit: + count = _write_manifest(host_dir, job_id, commit) + click.echo(f"[artifacts collect] manifest.json written ({count} file(s)).") + + +__all__ = ["artifacts", "artifacts_collect"] diff --git a/src/ci_orchestrator/commands/build.py b/src/ci_orchestrator/commands/build.py new file mode 100644 index 0000000..08c7962 --- /dev/null +++ b/src/ci_orchestrator/commands/build.py @@ -0,0 +1,509 @@ +"""``build`` sub-commands. + +Phase A3 ports the remote build pipeline: + +* ``build run`` — replaces ``scripts/Invoke-RemoteBuild.ps1`` + +The original PS script supports two source-provisioning modes (host-side +clone uploaded via WinRM/SCP, and in-guest ``git clone``) plus a custom +build command and optional artifact packaging. This Python port preserves +the mechanics while keeping the guest workdir and artifact paths fully +parameterised — see Phase C hooks below. + +Phase C hook: the guest workdir / artifact paths are passed as opaque +strings; this module makes no assumption about Windows vs POSIX path +syntax beyond a sensible default per ``--guest-os``. ``build run`` does +not import ``WorkstationVmrunBackend`` directly; it relies on the +transport layer to talk to whatever guest the orchestrator points it at. +""" + +from __future__ import annotations + +import shlex +import sys +import tarfile +import tempfile +import zipfile +from collections.abc import Iterable +from pathlib import Path +from typing import TYPE_CHECKING + +import click + +from ci_orchestrator.config import load_config +from ci_orchestrator.credentials import KeyringCredentialStore +from ci_orchestrator.transport.errors import TransportCommandError, TransportError +from ci_orchestrator.transport.ssh import SshTransport +from ci_orchestrator.transport.winrm import WinRmTransport + +if TYPE_CHECKING: # pragma: no cover + pass + + +# ---------------------------------------------------------------- group + + +@click.group("build") +def build() -> None: + """Remote build commands.""" + + +# ---------------------------------------------------------------- helpers + + +def _parse_extra_env(items: Iterable[str]) -> dict[str, str]: + """Parse ``KEY=VALUE`` pairs into a dict. + + Empty entries are ignored. A pair without ``=`` raises ``BadParameter``. + """ + out: dict[str, str] = {} + for raw in items: + if not raw: + continue + if "=" not in raw: + raise click.BadParameter( + f"--extra-env value must be KEY=VALUE, got: {raw!r}" + ) + key, value = raw.split("=", 1) + if not key: + raise click.BadParameter(f"--extra-env key cannot be empty: {raw!r}") + out[key] = value + return out + + +def _zip_dir(source_dir: Path, archive_path: Path) -> None: + """Create a deterministic ZIP archive containing the contents of ``source_dir``.""" + with zipfile.ZipFile( + archive_path, "w", compression=zipfile.ZIP_DEFLATED, compresslevel=1 + ) as zf: + for item in source_dir.rglob("*"): + if item.is_file(): + zf.write(item, arcname=item.relative_to(source_dir)) + + +def _tar_dir(source_dir: Path, archive_path: Path) -> None: + """Create a gzip tar archive containing the contents of ``source_dir``.""" + with tarfile.open(archive_path, "w:gz") as tf: + for item in source_dir.iterdir(): + tf.add(item, arcname=item.name) + + +def _ps_quote(value: str) -> str: + """Quote a value for embedding in a PowerShell single-quoted string.""" + return "'" + value.replace("'", "''") + "'" + + +def _sh_quote(value: str) -> str: + """POSIX shell single-quote escaping.""" + return shlex.quote(value) + + +# ---------------------------------------------------------------- Linux flow + + +def _linux_build( + *, + ip_address: str, + ssh_user: str, + key_path: str | None, + known_hosts: str | None, + workdir: str, + output_dir: str, + host_source_dir: str | None, + clone_url: str | None, + clone_branch: str, + clone_commit: str, + clone_submodules: bool, + build_command: str, + artifact_source: str, + extra_env: dict[str, str], + skip_artifact: bool, +) -> None: + click.echo("[build run] Linux build mode (SSH transport)") + + with SshTransport( + ip_address, + username=ssh_user, + key_path=key_path, + known_hosts=known_hosts, + ) as t: + t.run(f"rm -rf {_sh_quote(workdir)} && mkdir -p {_sh_quote(workdir)}") + + if host_source_dir: + click.echo(f"[build run] packaging host source: {host_source_dir}") + with tempfile.TemporaryDirectory() as td: + tar_path = Path(td) / "ci-src.tar.gz" + _tar_dir(Path(host_source_dir), tar_path) + guest_tar = f"/tmp/{tar_path.name}" + t.copy(str(tar_path), guest_tar) + click.echo(f"[build run] extracting source in guest: {workdir}") + t.run( + f"tar -xzf {_sh_quote(guest_tar)} -C {_sh_quote(workdir)} " + f"&& rm -f {_sh_quote(guest_tar)}" + ) + elif clone_url: + click.echo(f"[build run] cloning {clone_url} into guest:{workdir}") + cmd = ( + f"git clone --depth 1 --branch {_sh_quote(clone_branch)} " + f"{'--recurse-submodules ' if clone_submodules else ''}" + f"{_sh_quote(clone_url)} {_sh_quote(workdir)}" + ) + t.run(cmd) + if clone_commit: + t.run( + f"git -C {_sh_quote(workdir)} checkout {_sh_quote(clone_commit)}" + ) + else: + raise click.ClickException( + "Linux build requires either --host-source-dir or --clone-url." + ) + + env_prefix = "".join( + f"export {k}={_sh_quote(v)}; " for k, v in extra_env.items() + ) + cmd = build_command or "make" + full_cmd = f"{env_prefix}cd {_sh_quote(workdir)} && {cmd}" + click.echo(f"[build run] running build (env vars redacted): cd {workdir} && {cmd}") + result = t.run(full_cmd, check=False) + # Stream guest stdout/stderr after the fact (paramiko buffers the channel). + if result.stdout: + sys.stdout.write(result.stdout) + if result.stderr: + sys.stderr.write(result.stderr) + if result.returncode != 0: + raise click.ClickException( + f"build command failed (exit {result.returncode})" + ) + + if skip_artifact: + click.echo("[build run] artifact packaging skipped (--skip-artifact).") + return + + t.run(f"rm -rf {_sh_quote(output_dir)} && mkdir -p {_sh_quote(output_dir)}") + src = artifact_source or "dist" + cp_cmd = ( + f"if [ -d {_sh_quote(workdir + '/' + src)} ]; then " + f"cp -r {_sh_quote(workdir + '/' + src + '/.')} {_sh_quote(output_dir + '/')}; " + f"elif [ -e {_sh_quote(workdir + '/' + src)} ]; then " + f"cp {_sh_quote(workdir + '/' + src)} {_sh_quote(output_dir + '/')}; " + f"else echo '[build run] WARNING: artifact source not found: {src}' >&2; fi" + ) + t.run(cp_cmd) + click.echo(f"[build run] artifact staged at {output_dir}") + + +# ---------------------------------------------------------------- Windows flow + + +def _windows_build( + *, + ip_address: str, + credential_target: str, + workdir: str, + artifact_zip: str, + host_source_dir: str | None, + clone_url: str | None, + clone_branch: str, + clone_commit: str, + clone_submodules: bool, + build_command: str, + artifact_source: str, + configuration: str, + extra_env: dict[str, str], + skip_artifact: bool, + use_shared_cache: bool, +) -> None: + click.echo("[build run] Windows build mode (WinRM transport)") + cred = KeyringCredentialStore().get(credential_target) + + with WinRmTransport(ip_address, cred.username, cred.password) as t: + # Prepare workdir + artifact output directory. + prep_ps = ( + f"$work = {_ps_quote(workdir)}; " + f"$out = Split-Path {_ps_quote(artifact_zip)} -Parent; " + "if (-not (Test-Path $out)) { New-Item -ItemType Directory -Path $out -Force | Out-Null }; " + "if (Test-Path $work) { Remove-Item $work -Recurse -Force }; " + "New-Item -ItemType Directory -Path $work -Force | Out-Null" + ) + t.run(prep_ps) + + if host_source_dir: + click.echo(f"[build run] zipping host source: {host_source_dir}") + with tempfile.TemporaryDirectory() as td: + zip_path = Path(td) / "ci-src.zip" + _zip_dir(Path(host_source_dir), zip_path) + size_mb = round(zip_path.stat().st_size / (1024 * 1024), 1) + click.echo(f"[build run] uploading source archive ({size_mb} MB)") + guest_zip = "C:\\CI\\src-transfer.zip" + t.copy(str(zip_path), guest_zip) + t.run( + f"Expand-Archive -Path {_ps_quote(guest_zip)} " + f"-DestinationPath {_ps_quote(workdir)} -Force; " + f"Remove-Item {_ps_quote(guest_zip)} -Force" + ) + elif clone_url: + click.echo(f"[build run] cloning {clone_url} into guest:{workdir}") + sub_flag = " --recurse-submodules" if clone_submodules else "" + t.run( + f"$env:GIT_TERMINAL_PROMPT = '0'; " + f"Set-Location (Split-Path {_ps_quote(workdir)} -Parent); " + f"git clone --depth 1 --branch {_ps_quote(clone_branch)}{sub_flag} " + f"{_ps_quote(clone_url)} (Split-Path {_ps_quote(workdir)} -Leaf)" + ) + if clone_commit: + t.run( + f"git -C {_ps_quote(workdir)} checkout {_ps_quote(clone_commit)}" + ) + else: + raise click.ClickException( + "Windows build requires either --host-source-dir or --clone-url." + ) + + # Inject env vars + run the build. + env_setup = "; ".join( + f"$env:{k} = {_ps_quote(v)}" for k, v in extra_env.items() + ) + if env_setup: + env_setup += "; " + if use_shared_cache: + env_setup += ( + "$env:NUGET_PACKAGES = '\\\\vmware-host\\Shared Folders\\nuget-cache'; " + "$env:PIP_CACHE_DIR = '\\\\vmware-host\\Shared Folders\\pip-cache'; " + ) + if build_command: + run_cmd = build_command + click.echo(f"[build run] custom build command: {run_cmd}") + else: + click.echo( + f"[build run] default build: dotnet restore + build (configuration={configuration})" + ) + run_cmd = ( + "dotnet restore; if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }; " + f"dotnet build --configuration {_ps_quote(configuration)} " + "--output (Join-Path (Get-Location) 'bin\\CIOutput')" + ) + + full_ps = ( + f"{env_setup}Set-Location {_ps_quote(workdir)}; " + f"$ErrorActionPreference='Continue'; " + f"& cmd /c {_ps_quote(run_cmd + ' 2>&1')}; " + "exit $LASTEXITCODE" + ) + result = t.run(full_ps, check=False) + if result.stdout: + sys.stdout.write(result.stdout) + if result.stderr: + sys.stderr.write(result.stderr) + if result.returncode != 0: + raise click.ClickException( + f"build command failed (exit {result.returncode})" + ) + + if skip_artifact: + click.echo("[build run] artifact packaging skipped (--skip-artifact).") + return + + # Package the artifact source into the requested zip on the guest. + if build_command: + src_rel = artifact_source or "dist" + src_expr = f"Join-Path {_ps_quote(workdir)} {_ps_quote(src_rel)}" + else: + src_expr = f"Join-Path {_ps_quote(workdir)} 'bin\\CIOutput'" + + pkg_ps = ( + f"$src = {src_expr}; " + f"$zip = {_ps_quote(artifact_zip)}; " + "if (-not (Test-Path $src)) { " + " Write-Host \"[build run] artifact source not found: $src\"; " + " exit 2 } ; " + "$dir = Split-Path $zip -Parent; " + "if (-not (Test-Path $dir)) { New-Item -ItemType Directory -Path $dir -Force | Out-Null }; " + "if (Test-Path $zip) { Remove-Item $zip -Force }; " + "Compress-Archive -Path (Join-Path $src '*') -DestinationPath $zip -Force" + ) + t.run(pkg_ps) + click.echo(f"[build run] artifact written to {artifact_zip}") + + +# ---------------------------------------------------------------- CLI + + +@build.command("run") +@click.option("--ip-address", "ip_address", required=True, help="Guest IP address.") +@click.option( + "--guest-os", + type=click.Choice(["windows", "linux"], case_sensitive=False), + default="windows", + show_default=True, +) +@click.option( + "--credential-target", + "credential_target", + default=None, + help="Keyring target for the guest user (Windows). Defaults to config.guest_cred_target.", +) +@click.option("--ssh-user", "ssh_user", default="ci_build", show_default=True) +@click.option( + "--ssh-key-path", + "ssh_key_path", + default=None, + help="SSH private key path (Linux). Defaults to config.ssh_key_path.", +) +@click.option( + "--ssh-known-hosts-file", + "ssh_known_hosts_file", + default=None, + help="Persistent known_hosts file (Linux). Default: permissive (None).", +) +@click.option("--host-source-dir", "host_source_dir", default=None) +@click.option("--clone-url", "clone_url", default=None) +@click.option("--clone-branch", "clone_branch", default="main", show_default=True) +@click.option("--clone-commit", "clone_commit", default="") +@click.option("--clone-submodules", "clone_submodules", is_flag=True, default=False) +@click.option( + "--gitea-credential-target", + "gitea_credential_target", + default="GiteaPAT", + show_default=True, + help="(Reserved) Keyring target for Gitea PAT — currently unused by the Python port.", +) +@click.option("--build-command", "build_command", default="") +@click.option( + "--guest-work-dir", + "guest_work_dir", + default=None, + help="Workdir inside the guest (defaults: Win=C:\\CI\\build, Linux=/opt/ci/build).", +) +@click.option( + "--guest-artifact-zip", + "guest_artifact_zip", + default=None, + help="Artifact zip path inside the Windows guest (default: C:\\CI\\output\\artifacts.zip).", +) +@click.option( + "--guest-linux-work-dir", + "guest_linux_work_dir", + default=None, + help="Alias for --guest-work-dir, kept for shim compatibility (Linux).", +) +@click.option( + "--guest-linux-output-dir", + "guest_linux_output_dir", + default="/opt/ci/output", + show_default=True, +) +@click.option("--guest-artifact-source", "guest_artifact_source", default="dist", show_default=True) +@click.option("--configuration", default="Release", show_default=True) +@click.option("--skip-artifact", "skip_artifact", is_flag=True, default=False) +@click.option("--use-shared-cache", "use_shared_cache", is_flag=True, default=False) +@click.option( + "--extra-env", + "extra_env", + multiple=True, + help="Repeatable KEY=VALUE pairs injected into the guest environment.", +) +def build_run( + ip_address: str, + guest_os: str, + credential_target: str | None, + ssh_user: str, + ssh_key_path: str | None, + ssh_known_hosts_file: str | None, + host_source_dir: str | None, + clone_url: str | None, + clone_branch: str, + clone_commit: str, + clone_submodules: bool, + gitea_credential_target: str, + build_command: str, + guest_work_dir: str | None, + guest_artifact_zip: str | None, + guest_linux_work_dir: str | None, + guest_linux_output_dir: str, + guest_artifact_source: str, + configuration: str, + skip_artifact: bool, + use_shared_cache: bool, + extra_env: tuple[str, ...], +) -> None: + """Run a build inside a guest VM, optionally packaging the result. + + Mirrors ``scripts/Invoke-RemoteBuild.ps1``. Source is provisioned via + one of: + + * ``--host-source-dir`` — local directory zipped/tarred and uploaded; + * ``--clone-url`` — ``git clone`` invoked inside the guest. + + Phase C hook: ``--guest-work-dir`` and ``--guest-artifact-zip`` are + parameterised; defaults are applied here, never inside the transport + layer. + """ + del gitea_credential_target # not yet honoured by Python port + guest_os = guest_os.lower() + + if host_source_dir and clone_url: + raise click.BadParameter( + "Provide either --host-source-dir or --clone-url, not both." + ) + if not host_source_dir and not clone_url: + raise click.BadParameter( + "Provide either --host-source-dir or --clone-url." + ) + if host_source_dir and not Path(host_source_dir).is_dir(): + raise click.BadParameter(f"--host-source-dir does not exist: {host_source_dir}") + + extra = _parse_extra_env(extra_env) + config = load_config() + + if guest_os == "linux": + workdir = guest_linux_work_dir or guest_work_dir or "/opt/ci/build" + key_path: str | None = ssh_key_path + if key_path is None and config.ssh_key_path is not None: + key_path = str(config.ssh_key_path) + try: + _linux_build( + ip_address=ip_address, + ssh_user=ssh_user, + key_path=key_path, + known_hosts=ssh_known_hosts_file, + workdir=workdir, + output_dir=guest_linux_output_dir, + host_source_dir=host_source_dir, + clone_url=clone_url, + clone_branch=clone_branch, + clone_commit=clone_commit, + clone_submodules=clone_submodules, + build_command=build_command, + artifact_source=guest_artifact_source, + extra_env=extra, + skip_artifact=skip_artifact, + ) + except (TransportError, TransportCommandError) as exc: + raise click.ClickException(str(exc)) from exc + else: + workdir = guest_work_dir or "C:\\CI\\build" + artifact_zip = guest_artifact_zip or "C:\\CI\\output\\artifacts.zip" + target = credential_target or config.guest_cred_target + try: + _windows_build( + ip_address=ip_address, + credential_target=target, + workdir=workdir, + artifact_zip=artifact_zip, + host_source_dir=host_source_dir, + clone_url=clone_url, + clone_branch=clone_branch, + clone_commit=clone_commit, + clone_submodules=clone_submodules, + build_command=build_command, + artifact_source=guest_artifact_source, + configuration=configuration, + extra_env=extra, + skip_artifact=skip_artifact, + use_shared_cache=use_shared_cache, + ) + except (TransportError, TransportCommandError) as exc: + raise click.ClickException(str(exc)) from exc + + +__all__ = ["build", "build_run"] diff --git a/src/ci_orchestrator/commands/vm.py b/src/ci_orchestrator/commands/vm.py index 5573fb3..97e3011 100644 --- a/src/ci_orchestrator/commands/vm.py +++ b/src/ci_orchestrator/commands/vm.py @@ -354,4 +354,116 @@ def vm_cleanup( ) +# ─────────────────────────────────────────────────────────────────────── new + + +@vm.command("new") +@click.option( + "--template", + "--template-path", + "template", + required=True, + help="Identifier of the template VM (Workstation: VMX path; ESXi: managed-object ref).", +) +@click.option( + "--snapshot", + "--snapshot-name", + "snapshot", + default="BaseClean", + show_default=True, + help="Snapshot name on the template to clone from.", +) +@click.option( + "--clone-base-dir", + "clone_base_dir", + required=True, + help="Directory under which the new clone folder will be created.", +) +@click.option( + "--job-id", + "job_id", + required=True, + help="Unique job identifier (used to name the clone folder).", +) +@click.option( + "--vmrun-path", + "vmrun_path", + default=None, + help="Override vmrun binary path.", +) +@click.option( + "--guest-os", + type=click.Choice(["windows", "linux"], case_sensitive=False), + default="windows", + show_default=True, + help="Informational; reserved for backend hints in Phase C.", +) +def vm_new( + template: str, + snapshot: str, + clone_base_dir: str, + job_id: str, + vmrun_path: str | None, + guest_os: str, +) -> None: + """Create a linked clone of the template VM for an ephemeral CI build. + + Mirrors ``scripts/New-BuildVM.ps1``. Prints the clone identifier + (Workstation: VMX path) on stdout on success. + + Phase C hook: ``--template`` is treated as an opaque string; the backend + is responsible for interpretation. Workstation builds the destination + path under ``--clone-base-dir`` from ``--job-id`` + timestamp. + """ + del guest_os # currently informational only + # Verify the template identifier looks plausible: for Workstation it is + # a VMX path. For other backends in Phase C this validation will move + # into the backend itself. + template_path = Path(template) + if not template_path.is_file(): + raise click.ClickException(f"Template VMX not found: {template}") + + base = Path(clone_base_dir) + base.mkdir(parents=True, exist_ok=True) + + timestamp = datetime.now(tz=UTC).strftime("%Y%m%d_%H%M%S") + clone_name = f"Clone_{job_id}_{timestamp}" + clone_dir = base / clone_name + clone_vmx = clone_dir / f"{clone_name}.vmx" + + click.echo("[vm new] creating linked clone...") + click.echo(f" template : {template}") + click.echo(f" snapshot : {snapshot}") + click.echo(f" clone vmx: {clone_vmx}") + + try: + backend = _make_backend(vmrun_path) + except BackendNotAvailable as exc: + raise click.ClickException(f"vmrun unavailable: {exc}") from exc + + start = time.monotonic() + try: + handle = backend.clone_linked( + template=str(template_path), + snapshot=snapshot, + name=clone_name, + destination=str(clone_vmx), + ) + except BackendError as exc: + # Clean up partial clone directory if vmrun left one behind. + if clone_dir.exists(): + _try_remove_dir(clone_dir) + raise click.ClickException(f"clone failed: {exc}") from exc + + if not clone_vmx.is_file(): + raise click.ClickException( + f"backend reported success but clone VMX is missing: {clone_vmx}" + ) + + elapsed = time.monotonic() - start + click.echo(f"[vm new] clone created in {elapsed:.1f}s") + # Final stdout line: the identifier itself, so PS callers can capture it. + click.echo(handle.identifier) + + __all__ = ["cleanup_orphans", "vm"] diff --git a/tests/New-BuildVM.Tests.ps1 b/tests/New-BuildVM.Tests.ps1 deleted file mode 100644 index 19ba063..0000000 --- a/tests/New-BuildVM.Tests.ps1 +++ /dev/null @@ -1,106 +0,0 @@ -#Requires -Version 5.1 -<# -.SYNOPSIS - Pester v5 tests for scripts/New-BuildVM.ps1 -#> - -BeforeAll { - $script:ScriptPath = Join-Path $PSScriptRoot '..\scripts\New-BuildVM.ps1' - $script:CloneBaseDir = Join-Path $env:TEMP "CI-Tests-CloneBase-$PID" - New-Item -ItemType Directory -Path $script:CloneBaseDir -Force | Out-Null - - # Fake vmrun.ps1 — exits with $env:FAKE_VMRUN_EXIT (default 0) - # On success it also creates the destination VMX so the post-clone Test-Path passes. - # Using .ps1 instead of .cmd avoids Windows cmd redirect limitations inside if blocks. - $script:FakeVmrun = Join-Path $env:TEMP "fake-vmrun-$PID.ps1" - Set-Content $script:FakeVmrun -Value @' -# Args: -T ws clone