From f5091d09039d3f3c4d7121b3476e83a65f413e96 Mon Sep 17 00:00:00 2001 From: Simone Date: Thu, 14 May 2026 17:29:56 +0200 Subject: [PATCH] feat(a4): port Invoke-CIJob.ps1 to ci_orchestrator job Phase A4 of plans/implementation-plan-A-B.md. Implements the full job orchestrator (clone -> start -> wait -> probe -> build -> collect -> guaranteed cleanup) as a new commands/job.py click command, registered under python -m ci_orchestrator job. Backend selection goes through backends.load_backend(config) so Phase C can swap in remote drivers without touching the command. The legacy scripts/Invoke-CIJob.ps1 is replaced by a thin PS 5.1 shim that delegates to the Python CLI; tests/python/test_commands_job.py adds 13 cases covering Linux/Windows happy paths, override application, skip-artifact, and cleanup on every failure mode. --- scripts/Invoke-CIJob.ps1 | 696 +--------------------------- src/ci_orchestrator/__main__.py | 2 + src/ci_orchestrator/commands/job.py | 550 ++++++++++++++++++++++ tests/python/test_commands_job.py | 394 ++++++++++++++++ 4 files changed, 960 insertions(+), 682 deletions(-) create mode 100644 src/ci_orchestrator/commands/job.py create mode 100644 tests/python/test_commands_job.py diff --git a/scripts/Invoke-CIJob.ps1 b/scripts/Invoke-CIJob.ps1 index eb63930..fb53043 100644 --- a/scripts/Invoke-CIJob.ps1 +++ b/scripts/Invoke-CIJob.ps1 @@ -1,691 +1,23 @@ #Requires -Version 5.1 -<# -.SYNOPSIS - Main CI orchestrator: creates an ephemeral VM, runs a build, collects - artifacts, and destroys the VM — in a guaranteed try/finally pattern. - -.DESCRIPTION - This script is called by act_runner as the primary build step. - It coordinates all sub-scripts: - 1. New-BuildVM.ps1 — linked clone from template - 2. vmrun start — start the clone - 3. Wait-VMReady.ps1 — poll until WinRM is ready - 4. Invoke-RemoteBuild.ps1 — execute build inside VM - 5. Get-BuildArtifacts.ps1 — copy artifacts to host - 6. Remove-BuildVM.ps1 — destroy VM (always runs via finally) - - Exit codes: - 0 = success - 1 = build or orchestration failure (VM destroyed, artifacts may be partial) - -.PARAMETER JobId - Unique job identifier (use ${{ github.run_id }} from Gitea Actions). - -.PARAMETER RepoUrl - Git URL of the repository to build (reachable from the build VM). - -.PARAMETER Branch - Branch to build. - -.PARAMETER Commit - Optional specific commit SHA to check out. - -.PARAMETER Configuration - Build configuration. Default: Release - -.PARAMETER TemplatePath - Full path to the template VM .vmx file. - Can also be set via env var GITEA_CI_TEMPLATE_PATH. - -.PARAMETER SnapshotName - Template snapshot to clone from. Default: BaseClean - -.PARAMETER CloneBaseDir - Base directory for clone VMs. - Can also be set via env var GITEA_CI_CLONE_BASE_DIR. - Default: F:\CI\BuildVMs - -.PARAMETER ArtifactBaseDir - Host directory where artifacts are stored per job. - Default: F:\CI\Artifacts - -.PARAMETER VMIPAddress - IP address to assign/expect for the build VM. - Must be unique per concurrent build. - Can be passed explicitly or derived from a registry/file-based IP allocator. - -.PARAMETER GuestCredentialTarget - Target name in Windows Credential Manager for guest credentials. - Default: BuildVMGuest - -.PARAMETER VmrunPath - Path to vmrun.exe. - Default: C:\Program Files (x86)\VMware\VMware Workstation\vmrun.exe - -.EXAMPLE - # Called from Gitea Actions workflow: - .\scripts\Invoke-CIJob.ps1 ` - -JobId "${{ github.run_id }}" ` - -RepoUrl "http://gitea.local/org/myrepo.git" ` - -Branch "${{ github.ref_name }}" ` - -Commit "${{ github.sha }}" ` - -VMIPAddress "192.168.11.101" -#> -[CmdletBinding()] -param( - [Parameter(Mandatory)] - [ValidatePattern('^[A-Za-z0-9._-]+$')] - [string] $JobId, - - [Parameter(Mandatory)] - [string] $RepoUrl, - - [Parameter(Mandatory)] - [string] $Branch, - - [string] $Commit = '', - - [string] $Configuration = 'Release', - - # Pass -Submodules to clone with --recurse-submodules - [switch] $Submodules, - - # Custom build command to run inside the VM (empty = dotnet build) - [string] $BuildCommand = '', - - # Artifact source pattern relative to GuestWorkDir (used with BuildCommand) - [string] $GuestArtifactSource = 'dist', - - # Clone repo IN the VM instead of on host (requires Git in template §6.6) - # Skips Phase 1 host clone, injects URL/branch/commit into VM for in-VM clone. - # Prerequisite: Git for Windows installed in template. - [switch] $UseGitClone, - - # Use VMware shared folders for NuGet/pip cache (§SharedCache). - # Requires Set-TemplateSharedFolders.ps1 to have been run on the template. - [switch] $UseSharedCache, - - # Skip artifact packaging (Invoke-RemoteBuild) and collection (Phase 6). - # Use for jobs that intentionally produce no output (lint, test-only, etc.). - # When absent, the build must produce GuestArtifactSource or the job fails. - [switch] $SkipArtifact, - - # Credential Manager target for Gitea PAT used by -UseGitClone (private repos). - # Store with: New-StoredCredential -Target 'GiteaPAT' -UserName '' -Password '' -Persist LocalMachine - # Ignored when -UseGitClone is not set. - [string] $GiteaCredentialTarget = 'GiteaPAT', - - [string] $TemplatePath = $env:GITEA_CI_TEMPLATE_PATH, - - [string] $SnapshotName = $(if ($env:GITEA_CI_SNAPSHOT_NAME) { $env:GITEA_CI_SNAPSHOT_NAME } else { 'BaseClean' }), - - [string] $CloneBaseDir = $(if ($env:GITEA_CI_CLONE_BASE_DIR) { $env:GITEA_CI_CLONE_BASE_DIR } else { 'F:\CI\BuildVMs' }), - - [string] $ArtifactBaseDir = 'F:\CI\Artifacts', - - [ValidatePattern('^((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)\.){3}(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)$')] - [string] $VMIPAddress = '', # if empty, auto-detected via vmrun getGuestIPAddress - - [string] $GuestCredentialTarget = 'BuildVMGuest', - - [string] $VmrunPath = 'C:\Program Files (x86)\VMware\VMware Workstation\vmrun.exe', - - # Directory for job execution logs. Each job gets its own sub-directory: - # $LogDir\$JobId\invoke-ci.log - [string] $LogDir = 'F:\CI\Logs', - - # Days to retain job logs. Directories older than this are purged at job end. - # Set to 0 to disable retention/purge. - [int] $LogRetentionDays = 30, - - # Guest OS type — determines transport (WinRM for Windows, SSH for Linux). - # When 'Auto', detected from the VMX guestOS field after clone. - [ValidateSet('Windows', 'Linux', 'Auto')] - [string] $GuestOS = 'Auto', - - [string] $SshKeyPath = 'F:\CI\keys\ci_linux', - [string] $SshUser = 'ci_build', - - # Persistent known_hosts file used for CI-job SSH calls. - # Empty string (default) = permissive mode (StrictHostKeyChecking=no, UserKnownHostsFile=NUL). - # Ephemeral CI VMs on a local NAT network do not need strict host-key verification. - # Pass a file path only if you need accept-new mode for a specific deployment. - [string] $SshKnownHostsFile = '', - - # Extra environment variables injected into the guest VM before the build command (§6.5). - # Keys must be valid environment variable names. Values must be plain strings. - # Example: @{ SIGN_PASS = $env:SIGN_PASS_FROM_SECRET; GPG_KEY_ID = '0xABCD1234' } - [hashtable] $ExtraGuestEnv = @{}, - - # Optional webhook URL (Discord/Gitea). When set, a background job fires a - # [WARNING] once the job has been running for 90 minutes. - [string] $WebhookUrl = '', - - # Optional per-job VMX CPU and RAM override (default 0 = use template value). - # Applied to the linked clone after New-BuildVM.ps1, before vmrun start. - # The template VMX is never modified. - # Example: -GuestCPU 4 -GuestMemoryMB 8192 - [ValidateRange(0, 32)] - [int] $GuestCPU = 0, - - [ValidateRange(0, 131072)] - [int] $GuestMemoryMB = 0 -) - Set-StrictMode -Version Latest $ErrorActionPreference = 'Stop' -# ── Resolve script directory for dot-sourcing sibling scripts ───────────────── -$scriptDir = $PSScriptRoot - -# ── Load shared helpers (Invoke-VmrunBounded, Get-GuestIPAddress, …) ────────── -Import-Module (Join-Path $scriptDir '_Common.psm1') -Force - -# ── Validate required config ────────────────────────────────────────────────── -if ([string]::IsNullOrWhiteSpace($TemplatePath)) { - Write-Error "TemplatePath is required. Set -TemplatePath or env:GITEA_CI_TEMPLATE_PATH." - exit 1 -} -if (-not (Test-Path $TemplatePath -PathType Leaf)) { - Write-Error "Template VMX not found: $TemplatePath" - exit 1 -} -# Sanity check: warn if template is outside the expected CI tree -if ($TemplatePath -notlike 'F:\CI\Templates\*') { - Write-Warning "[Invoke-CIJob] TemplatePath '$TemplatePath' is not under F:\CI\Templates\. Ensure this is intentional." -} -if (-not (Test-Path $VmrunPath -PathType Leaf)) { - Write-Error "vmrun.exe not found: $VmrunPath" - exit 1 -} - -# ── Validate ExtraGuestEnv keys ────────────────────────────────────────────── -foreach ($key in $ExtraGuestEnv.Keys) { - if ($key -notmatch '^[A-Za-z_][A-Za-z0-9_]*$') { - Write-Error "ExtraGuestEnv key '$key' is not a valid environment variable name. Keys must match ^[A-Za-z_][A-Za-z0-9_]*$." - exit 1 - } -} - -# ── Load guest credentials from Windows Credential Manager ─────────────────── -# Requires the CredentialManager module: Install-Module CredentialManager -# In a lab setup you can fall back to prompting, but never hardcode credentials. -$credential = $null -try { - $credential = Get-StoredCredential -Target $GuestCredentialTarget -ErrorAction Stop - Write-Host "[Invoke-CIJob] Loaded guest credentials from Credential Manager ($GuestCredentialTarget)." -} -catch { - Write-Warning "[Invoke-CIJob] Could not load credentials from Credential Manager ($GuestCredentialTarget)." - Write-Warning " Using interactive prompt as fallback. Store credentials with:" - Write-Warning " cmdkey /generic:$GuestCredentialTarget /user:DOMAIN\user /pass:password" - $credential = Get-Credential -Message "Enter guest VM credentials for job $JobId" -} - -# ── Setup paths ─────────────────────────────────────────────────────────────── -$jobArtifactDir = Join-Path $ArtifactBaseDir $JobId -$hostCloneDir = Join-Path $env:TEMP "ci-src-$JobId" -$cloneVmxPath = $null -$leaseFile = $null # §2.1 IP lease file path — released in finally -$jsonLog = $null # §4.1 JSONL structured log path -$startTime = Get-Date - -# ── Phase-duration tracking (Task 5.5) ─────────────────────────────────────── -$phaseTimings = [System.Collections.Generic.List[PSCustomObject]]::new() -$phaseStart = $startTime - -# ── Start transcript + JSONL log ────────────────────────────────────────────── -$transcriptStarted = $false -try { - $jobLogDir = Join-Path $LogDir $JobId - if (-not (Test-Path $jobLogDir)) { New-Item -ItemType Directory -Path $jobLogDir -Force | Out-Null } - $transcriptPath = Join-Path $jobLogDir 'invoke-ci.log' - $jsonLog = Join-Path $jobLogDir 'invoke-ci.jsonl' - # -Append allows starting a new transcript even when VS Code (or another host) - # has already opened a transcript on this PowerShell session. - Start-Transcript -Path $transcriptPath -Append -Force -ErrorAction Stop - $transcriptStarted = $true -} -catch { - Write-Warning "[Invoke-CIJob] Could not start transcript: $_" -} - -# ── §4.1 Structured event emitter ──────────────────────────────────────────── -# Appends a single JSON line to invoke-ci.jsonl per phase transition. -# Silent no-op if $jsonLog is null (transcript setup failed). -function Write-JobEvent { - param( - [Parameter(Mandatory)] [string] $Phase, - [Parameter(Mandatory)] [string] $Status, # start | success | failure | info - [hashtable] $Data = @{} - ) - if (-not $script:jsonLog) { return } - $logEntry = [ordered]@{ - ts = (Get-Date -Format 'o') - jobId = $script:JobId - phase = $Phase - status = $Status - data = $Data - } - try { - $logEntry | ConvertTo-Json -Compress -Depth 5 | - Add-Content -Path $script:jsonLog -Encoding UTF8 -ErrorAction Stop - } catch { Write-Debug "[Write-JobEvent] Logging skipped: $_" } # never let logging fail the build -} - -Write-Host "============================================================" -Write-Host "[Invoke-CIJob] Job : $JobId" -Write-Host "[Invoke-CIJob] Repository : $RepoUrl" -Write-Host "[Invoke-CIJob] Branch : $Branch" -if ($Commit) { Write-Host "[Invoke-CIJob] Commit : $Commit" } -Write-Host "[Invoke-CIJob] Build VM IP : $VMIPAddress" -Write-Host "[Invoke-CIJob] Template : $TemplatePath" -Write-Host "[Invoke-CIJob] Artifacts : $jobArtifactDir" -Write-Host "[Invoke-CIJob] Started : $startTime" -Write-Host "============================================================" - -Write-JobEvent -Phase 'job' -Status 'start' -Data @{ - repo = $RepoUrl - branch = $Branch - commit = $Commit - template = $TemplatePath - snapshot = $SnapshotName -} - -$exitCode = 0 -$durationWarnJob = $null - -try { - # ── 90-minute duration warning (fires once via background job) ─────────────── - if ($WebhookUrl -ne '') { - $warnMsg = "[WARNING] CI Job duration -- Job $JobId has been running for 90 minutes. Check $RepoUrl or kill the job if stuck." - $warnBody = (@{ content = $warnMsg } | ConvertTo-Json -Compress) - $durationWarnJob = Start-Job -ScriptBlock { - param($url, $body) - Start-Sleep -Seconds 5400 # 90 minutes - try { - Invoke-RestMethod -Uri $url -Method Post -Body $body ` - -ContentType 'application/json' -TimeoutSec 10 - } catch { Write-Verbose "Webhook POST failed (non-critical): $_" } - } -ArgumentList $WebhookUrl, $warnBody - } - - # ── Phase 1: Clone repo (HOST or GUEST) ─────────────────────────────── - if ($UseGitClone) { - # §3.3: Clone in guest VM instead of host. Skip host clone phase. - # PAT will be injected at runtime (Phase 5) from Credential Manager. - Write-Host "`n[Phase 1/6] Skipping host clone — will clone in guest VM (-UseGitClone)..." - Write-JobEvent -Phase 'phase1.clone-repo' -Status 'skipped' -Data @{ method = 'in-vm-clone' } +# Shim: delegates to the Python ci_orchestrator CLI. +# Original PS implementation moved to git history; see plans/A4-closeout.md. +$pyArgs = @() +foreach ($a in $args) { + if ($a -is [string] -and $a.Length -gt 1 -and $a.StartsWith('-') -and -not $a.StartsWith('--')) { + $name = $a.Substring(1) + $kebab = [Regex]::Replace($name, '(?<=[a-z0-9])([A-Z])', '-$1') + $kebab = [Regex]::Replace($kebab, '([A-Z]+)([A-Z][a-z])', '$1-$2') + $pyArgs += '--' + $kebab.ToLower() } else { - # Default: Clone on host, zip, transfer to guest via WinRM. - Write-Host "`n[Phase 1/6] Cloning repository on host..." - Write-JobEvent -Phase 'phase1.clone-repo' -Status 'start' - if (Test-Path $hostCloneDir) { Remove-Item $hostCloneDir -Recurse -Force } - $gitArgs = @('clone', '--depth', '1', '--progress', '--branch', $Branch) - if ($Submodules) { $gitArgs += @('--recurse-submodules', '--shallow-submodules') } - $gitArgs += @($RepoUrl, $hostCloneDir) - Write-Host "[Invoke-CIJob] git $($gitArgs -join ' ')" - $ErrorActionPreference = 'Continue' - # Stream git stdout/stderr live (otherwise heavy clones with submodules - # look frozen for a long time). Tee for diagnostic in case of failure. - $gitOutput = & git @gitArgs 2>&1 | Tee-Object -Variable streamed | ForEach-Object { Write-Host " [git] $_"; "$_" } - $gitExit = $LASTEXITCODE - $ErrorActionPreference = 'Stop' - if ($gitExit -ne 0) { - throw "git clone failed (exit $gitExit):`n$($gitOutput -join "`n")" - } - if ($Commit -ne '') { - $ErrorActionPreference = 'Continue' - $gitOutput = & git -C $hostCloneDir checkout $Commit 2>&1 | ForEach-Object { "$_" } - $gitExit = $LASTEXITCODE - if ($gitExit -ne 0) { - # Commit not in shallow history (e.g. manual run on older SHA). - # Fetch full history and retry. - Write-Host "[Invoke-CIJob] Shallow checkout failed — fetching full history..." - & git -C $hostCloneDir fetch --unshallow 2>&1 | Out-Null - $gitOutput = & git -C $hostCloneDir checkout $Commit 2>&1 | ForEach-Object { "$_" } - $gitExit = $LASTEXITCODE - } - $ErrorActionPreference = 'Stop' - if ($gitExit -ne 0) { - throw "git checkout $Commit failed (exit $gitExit):`n$($gitOutput -join "`n")" - } - } - Write-Host "[Invoke-CIJob] Source cloned to: $hostCloneDir" - Write-JobEvent -Phase 'phase1.clone-repo' -Status 'success' -Data @{ dir = $hostCloneDir } - } - $phaseTimings.Add([PSCustomObject]@{ Phase = 'Phase 1 - Clone repo'; Sec = [int]((Get-Date) - $phaseStart).TotalSeconds }) - $phaseStart = Get-Date - - # ── Phase 2-3b: Clone + Start + IP (serialized via file-based mutex) ──── - # §2.1: With capacity:4 and DHCP on VMnet8, concurrent vmrun starts can race - # and produce two VMs with the same DHCP lease. A file-based mutex ensures - # only one job is in the clone→start→IP phase at a time. After the IP is - # acquired and written to a lease file the lock is released, so build phases - # (the slow part) run fully in parallel. Lease is released in finally. - $stateDir = 'F:\CI\State' - $leasesDir = Join-Path $stateDir 'ip-leases' - $lockPath = Join-Path $stateDir 'vm-start.lock' - if (-not (Test-Path $leasesDir)) { New-Item -ItemType Directory -Path $leasesDir -Force | Out-Null } - - # ── Pre-clone disk space gate ───────────────────────────────────────── - $cloneDriveLetter = (Split-Path -Qualifier $CloneBaseDir).TrimEnd(':') - $cloneVol = Get-Volume -DriveLetter $cloneDriveLetter -ErrorAction SilentlyContinue - if ($null -ne $cloneVol) { - $freeGB = [math]::Round($cloneVol.SizeRemaining / 1GB, 1) - if ($freeGB -lt 20) { - throw "Insufficient disk space on ${cloneDriveLetter}: ($freeGB GB free). 20 GB minimum required before VM clone." - } - Write-Host "[Invoke-CIJob] Disk space on ${cloneDriveLetter}: ${freeGB} GB free (OK)." - } - - Write-Host "`n[Phase 2/6] Acquiring VM-start lock (prevents DHCP collision with capacity>1)..." - Write-JobEvent -Phase 'phase2-3b.vm-start' -Status 'start' - $lockStream = $null - $lockDeadline = (Get-Date).AddMinutes(10) - while ($true) { - try { - $lockStream = [System.IO.File]::Open( - $lockPath, - [System.IO.FileMode]::OpenOrCreate, - [System.IO.FileAccess]::ReadWrite, - [System.IO.FileShare]::None) - break - } catch [System.IO.IOException] { - if ((Get-Date) -ge $lockDeadline) { - throw "[Phase 2] VM-start lock timeout after 10 min — a concurrent job may be stuck." - } - Write-Host "[Invoke-CIJob] VM-start lock busy, retrying in 5s..." - Start-Sleep -Seconds 5 - } - } - Write-Host "[Invoke-CIJob] VM-start lock acquired." - - try { - # ── Phase 2: Clone VM ───────────────────────────────────────────── - Write-Host "[Phase 2/6] Cloning VM..." - $cloneVmxPath = & "$scriptDir\New-BuildVM.ps1" ` - -TemplatePath $TemplatePath ` - -SnapshotName $SnapshotName ` - -CloneBaseDir $CloneBaseDir ` - -JobId $JobId ` - -VmrunPath $VmrunPath - - # ── Optional VMX CPU/RAM override (post-clone, pre-start) ───────────── - if ($GuestCPU -gt 0 -or $GuestMemoryMB -gt 0) { - Write-Host "[Invoke-CIJob] Applying VMX override: CPU=$GuestCPU, RAM=${GuestMemoryMB}MB" - $vmxLines = Get-Content -LiteralPath $cloneVmxPath - if ($GuestCPU -gt 0) { - if ($vmxLines -imatch '^numvcpus\s*=') { - $vmxLines = $vmxLines -replace '(?i)^numvcpus\s*=.*', "numvcpus = `"$GuestCPU`"" - } else { - $vmxLines = $vmxLines + "numvcpus = `"$GuestCPU`"" - } - } - if ($GuestMemoryMB -gt 0) { - if ($vmxLines -imatch '^memsize\s*=') { - $vmxLines = $vmxLines -replace '(?i)^memsize\s*=.*', "memsize = `"$GuestMemoryMB`"" - } else { - $vmxLines = $vmxLines + "memsize = `"$GuestMemoryMB`"" - } - } - Set-Content -LiteralPath $cloneVmxPath -Value $vmxLines -Encoding UTF8 - Write-Host "[Invoke-CIJob] VMX updated: $cloneVmxPath" - } - - # ── Phase 3: Start VM ───────────────────────────────────────────── - Write-Host "`n[Phase 3/6] Starting VM..." - $startResult = Invoke-VmrunBounded -VmrunPath $VmrunPath ` - -Operation start ` - -Arguments @($cloneVmxPath, 'nogui') ` - -TimeoutSeconds 180 - if ($startResult.TimedOut) { - throw "vmrun start timed out after 180s for: $cloneVmxPath" - } - if ($startResult.ExitCode -ne 0) { - throw "vmrun start failed (exit $($startResult.ExitCode)): $($startResult.Output)" - } - Write-Host "[Invoke-CIJob] VM started (took $($startResult.ElapsedSeconds)s)." - - # ── Phase 3b: Resolve VM IP ─────────────────────────────────────── - if ([string]::IsNullOrWhiteSpace($VMIPAddress)) { - $detectedIP = Get-GuestIPAddress -VmrunPath $VmrunPath ` - -VmxPath $cloneVmxPath -TimeoutSeconds 120 - if ($detectedIP -eq '') { - throw "Could not detect VM IP address after 120s. Ensure ci-report-ip.service is enabled in the guest template (Linux) or open-vm-tools is running (Windows)." - } - $VMIPAddress = $detectedIP - Write-Host "[Invoke-CIJob] Detected VM IP: $VMIPAddress" - } - - # ── Lease IP ────────────────────────────────────────────────────── - # If two VMs somehow got the same IP (DHCP edge case), fail fast here - # rather than letting both jobs write to the same WinRM target. - $leaseFile = Join-Path $leasesDir "$($VMIPAddress.Replace('.', '-')).lease" - if (Test-Path $leaseFile) { - $holder = (Get-Content $leaseFile -Raw -ErrorAction SilentlyContinue).Trim() - throw "IP collision: $VMIPAddress already leased by job '$holder'. Retry or reduce runner capacity." - } - [System.IO.File]::WriteAllText($leaseFile, $JobId) - Write-Host "[Invoke-CIJob] IP $VMIPAddress leased for job $JobId." - Write-JobEvent -Phase 'phase2-3b.vm-start' -Status 'success' -Data @{ ip = $VMIPAddress; vmx = $cloneVmxPath } - - } finally { - # Release lock — next waiting job can now enter clone→start→IP phase - if ($lockStream) { - $lockStream.Close() - $lockStream.Dispose() - $lockStream = $null - } - Remove-Item $lockPath -Force -ErrorAction SilentlyContinue - Write-Host "[Invoke-CIJob] VM-start lock released." - } - $phaseTimings.Add([PSCustomObject]@{ Phase = 'Phase 2-3 - Clone+Start VM'; Sec = [int]((Get-Date) - $phaseStart).TotalSeconds }) - $phaseStart = Get-Date - # ── Resolve GuestOS from VMX if Auto ──────────────────────────────────── - if ($GuestOS -eq 'Auto' -and $cloneVmxPath) { - $vmxContent = Get-Content -LiteralPath $cloneVmxPath -Raw -ErrorAction SilentlyContinue - if ($vmxContent -match 'guestOS\s*=\s*"([^"]+)"') { - $detectedOS = $Matches[1] - $GuestOS = if ($detectedOS -like '*ubuntu*' -or $detectedOS -like '*linux*') { 'Linux' } else { 'Windows' } - Write-Host "[Invoke-CIJob] Auto-detected GuestOS: $GuestOS (VMX guestOS=$detectedOS)" - } else { - $GuestOS = 'Windows' - Write-Host "[Invoke-CIJob] Could not detect GuestOS from VMX — defaulting to Windows" - } - } - # ── Phase 4: Wait for readiness ─────────────────────────────────────── - Write-Host "`n[Phase 4/6] Waiting for VM readiness..." - Write-JobEvent -Phase 'phase4.wait-ready' -Status 'start' -Data @{ ip = $VMIPAddress } - $waitParams = @{ - VMPath = $cloneVmxPath - IPAddress = $VMIPAddress - VmrunPath = $VmrunPath - } - if ($GuestOS -eq 'Linux') { - $waitParams['Transport'] = 'SSH' - $waitParams['SshKeyPath'] = $SshKeyPath - $waitParams['SshUser'] = $SshUser - } - & "$scriptDir\Wait-VMReady.ps1" @waitParams - Write-JobEvent -Phase 'phase4.wait-ready' -Status 'success' - $phaseTimings.Add([PSCustomObject]@{ Phase = 'Phase 4 - Wait ready'; Sec = [int]((Get-Date) - $phaseStart).TotalSeconds }) - $phaseStart = Get-Date - - # ── Phase 5: Remote build (copy source + build inside VM) ───────────── - Write-Host "`n[Phase 5/6] Invoking remote build..." - Write-JobEvent -Phase 'phase5.build' -Status 'start' - $remoteBuildParams = @{ - IPAddress = $VMIPAddress - Credential = $credential - Configuration = $Configuration - } - if ($GuestOS -eq 'Linux') { - $remoteBuildParams['GuestOS'] = 'Linux' - $remoteBuildParams['SshKeyPath'] = $SshKeyPath - $remoteBuildParams['SshUser'] = $SshUser - $remoteBuildParams['SshKnownHostsFile'] = $SshKnownHostsFile - $remoteBuildParams.Remove('Credential') - if ($UseGitClone) { - # In-guest git clone (requires Gitea reachable + PAT/key configured in guest) - $remoteBuildParams['CloneUrl'] = $RepoUrl - $remoteBuildParams['CloneBranch'] = $Branch - $remoteBuildParams['GiteaCredentialTarget'] = $GiteaCredentialTarget - if ($Commit -ne '') { $remoteBuildParams['CloneCommit'] = $Commit } - if ($Submodules) { $remoteBuildParams['CloneSubmodules'] = $true } - } else { - # Default: host-cloned source shipped to guest via tar+scp - $remoteBuildParams['HostSourceDir'] = $hostCloneDir - } - } else { - if ($UseGitClone) { - # §3.3: Clone in VM — pass URL/branch/commit instead of HostSourceDir - $remoteBuildParams['CloneUrl'] = $RepoUrl - $remoteBuildParams['CloneBranch'] = $Branch - $remoteBuildParams['GiteaCredentialTarget'] = $GiteaCredentialTarget - if ($Commit -ne '') { $remoteBuildParams['CloneCommit'] = $Commit } - if ($Submodules) { $remoteBuildParams['CloneSubmodules'] = $true } - } - else { - # Default: Use pre-cloned source from host - $remoteBuildParams['HostSourceDir'] = $hostCloneDir - } - } - if ($BuildCommand -ne '') { $remoteBuildParams['BuildCommand'] = $BuildCommand } - if ($GuestArtifactSource -ne '') { $remoteBuildParams['GuestArtifactSource'] = $GuestArtifactSource } - if ($ExtraGuestEnv.Count -gt 0) { $remoteBuildParams['ExtraGuestEnv'] = $ExtraGuestEnv } - if ($UseSharedCache) { $remoteBuildParams['UseSharedCache'] = $true } - if ($SkipArtifact) { $remoteBuildParams['SkipArtifact'] = $true } - & "$scriptDir\Invoke-RemoteBuild.ps1" @remoteBuildParams - Write-JobEvent -Phase 'phase5.build' -Status 'success' - $phaseTimings.Add([PSCustomObject]@{ Phase = 'Phase 5 - Remote build'; Sec = [int]((Get-Date) - $phaseStart).TotalSeconds }) - $phaseStart = Get-Date - - # ── Phase 6: Collect artifacts ──────────────────────────────────────── - if ($SkipArtifact) { - Write-Host "`n[Phase 6/6] Artifact collection skipped (-SkipArtifact)." - Write-JobEvent -Phase 'phase6.artifacts' -Status 'skipped' - } else { - Write-Host "`n[Phase 6/6] Collecting artifacts..." - Write-JobEvent -Phase 'phase6.artifacts' -Status 'start' - if ($GuestOS -eq 'Linux') { - & "$scriptDir\Get-BuildArtifacts.ps1" ` - -IPAddress $VMIPAddress ` - -GuestOS 'Linux' ` - -SshKeyPath $SshKeyPath ` - -SshUser $SshUser ` - -SshKnownHostsFile $SshKnownHostsFile ` - -GuestArtifactPath '/opt/ci/output' ` - -HostArtifactDir $jobArtifactDir ` - -JobId $JobId ` - -Commit $Commit ` - -IncludeLogs - } else { - & "$scriptDir\Get-BuildArtifacts.ps1" ` - -IPAddress $VMIPAddress ` - -Credential $credential ` - -GuestArtifactPath 'C:\CI\output\artifacts.zip' ` - -HostArtifactDir $jobArtifactDir ` - -JobId $JobId ` - -Commit $Commit ` - -IncludeLogs - } - Write-JobEvent -Phase 'phase6.artifacts' -Status 'success' -Data @{ dir = $jobArtifactDir } - } - $elapsed = (Get-Date) - $startTime - Write-JobEvent -Phase 'job' -Status 'success' -Data @{ elapsedSec = [int]$elapsed.TotalSeconds } - $phaseTimings.Add([PSCustomObject]@{ Phase = 'Phase 6 - Artifacts'; Sec = [int]((Get-Date) - $phaseStart).TotalSeconds }) - Write-Host "`n============================================================" - Write-Host "[Invoke-CIJob] Phase durations:" - foreach ($pt in $phaseTimings) { - Write-Host (" {0,-30} {1,5}s" -f $pt.Phase, $pt.Sec) - } - Write-Host (" {0,-30} {1,5}s" -f 'TOTAL', [int]$elapsed.TotalSeconds) - Write-Host "============================================================" - Write-Host "[Invoke-CIJob] SUCCESS -- Job $JobId completed in $($elapsed.ToString('mm\:ss'))" - Write-Host "[Invoke-CIJob] Artifacts: $jobArtifactDir" - Write-Host "============================================================" -} -catch { - $exitCode = 1 - $elapsed = (Get-Date) - $startTime - Write-JobEvent -Phase 'job' -Status 'failure' -Data @{ - elapsedSec = [int]$elapsed.TotalSeconds - error = "$_" - } - - # ── Best-effort guest diagnostics (skip if VM was never started) ───────── - # Only collect when a VM IP is known (Phase 3b completed). Never throws. - if (-not [string]::IsNullOrWhiteSpace($VMIPAddress)) { - try { - $diagDir = Join-Path $jobArtifactDir 'diagnostics' - $diagParams = @{ - IPAddress = $VMIPAddress - GuestOS = if ($GuestOS -eq 'Auto') { 'Windows' } else { $GuestOS } - DestinationDir = $diagDir - } - if ($diagParams.GuestOS -eq 'Windows' -and $credential) { - $diagParams['Credential'] = $credential - } - if ($diagParams.GuestOS -eq 'Linux') { - $diagParams['SshKeyPath'] = $SshKeyPath - $diagParams['SshUser'] = $SshUser - } - Get-GuestDiagnostics @diagParams - } catch { - Write-Warning "[Invoke-CIJob] Diagnostics collection skipped: $_" - } - } - - Write-Host "`n============================================================" - Write-Host "[Invoke-CIJob] FAILURE — Job $JobId failed after $($elapsed.ToString('mm\:ss'))" - Write-Host "Error: $_" - Write-Host "============================================================" -} -finally { - # ── Release IP lease (§2.1) ─────────────────────────────────────────── - if ($leaseFile -and (Test-Path $leaseFile)) { - Remove-Item $leaseFile -Force -ErrorAction SilentlyContinue - Write-Host "[Cleanup] IP lease released: $VMIPAddress" - } - - # ── Always destroy the VM ───────────────────────────────────────────── - if ($null -ne $cloneVmxPath -and (Test-Path $cloneVmxPath -PathType Leaf)) { - Write-Host "`n[Cleanup] Destroying ephemeral VM..." - & "$scriptDir\Remove-BuildVM.ps1" -VMPath $cloneVmxPath -VmrunPath $VmrunPath - } - else { - Write-Host "[Cleanup] No VM to destroy (clone was not created or already gone)." - } - # ── Clean up host-side git clone (unless in-VM clone was used) ──────── - # When -UseGitClone, $hostCloneDir is never created - if (-not $UseGitClone -and (Test-Path $hostCloneDir)) { - Write-Host "[Cleanup] Removing host source clone ($hostCloneDir)..." - Remove-Item $hostCloneDir -Recurse -Force -ErrorAction SilentlyContinue - } - - # ── Cancel 90-minute warning job (build finished before it fired) ────────── - if ($null -ne $durationWarnJob) { - Remove-Job -Job $durationWarnJob -Force -ErrorAction SilentlyContinue - $durationWarnJob = $null - } - - # ── Stop transcript ─────────────────────────────────────────────────── - if ($transcriptStarted) { - try { Stop-Transcript -ErrorAction SilentlyContinue } catch { Write-Debug "[Invoke-CIJob] Stop-Transcript ignored: $_" } - } - - # ── Log retention: purge directories older than $LogRetentionDays days ─ - if ($LogRetentionDays -gt 0 -and (Test-Path $LogDir)) { - $cutoff = (Get-Date).AddDays(-$LogRetentionDays) - Get-ChildItem -Path $LogDir -Directory -ErrorAction SilentlyContinue | - Where-Object { $_.LastWriteTime -lt $cutoff } | - ForEach-Object { - Remove-Item $_.FullName -Recurse -Force -ErrorAction SilentlyContinue - Write-Host "[Cleanup] Purged old log dir: $($_.FullName)" - } + $pyArgs += $a } } -exit $exitCode +$venvPython = $env:CI_VENV_PYTHON +if (-not $venvPython) { $venvPython = 'F:\CI\python\venv\Scripts\python.exe' } +& $venvPython -m ci_orchestrator job @pyArgs +exit $LASTEXITCODE diff --git a/src/ci_orchestrator/__main__.py b/src/ci_orchestrator/__main__.py index 6999518..d9ce664 100644 --- a/src/ci_orchestrator/__main__.py +++ b/src/ci_orchestrator/__main__.py @@ -14,6 +14,7 @@ import click from ci_orchestrator import __version__ from ci_orchestrator.commands.artifacts import artifacts from ci_orchestrator.commands.build import build +from ci_orchestrator.commands.job import job from ci_orchestrator.commands.monitor import monitor from ci_orchestrator.commands.report import report from ci_orchestrator.commands.vm import vm @@ -40,6 +41,7 @@ cli.add_command(build) cli.add_command(artifacts) cli.add_command(monitor) cli.add_command(report) +cli.add_command(job) if __name__ == "__main__": # pragma: no cover diff --git a/src/ci_orchestrator/commands/job.py b/src/ci_orchestrator/commands/job.py new file mode 100644 index 0000000..ca4e909 --- /dev/null +++ b/src/ci_orchestrator/commands/job.py @@ -0,0 +1,550 @@ +"""``job`` sub-command — full CI orchestration. + +Phase A4 ports ``scripts/Invoke-CIJob.ps1``: clone an ephemeral VM from a +template snapshot, wait for the guest to become reachable, run the build +inside the guest, optionally collect artifacts, and **always** destroy the +clone (even on failure or ``KeyboardInterrupt``). + +Composition over re-spawn: this module imports the in-process helpers +from ``commands.build``, ``commands.artifacts`` and ``commands.vm`` rather +than re-invoking ``python -m ci_orchestrator`` for each phase. + +Phase C hook: the backend is selected via :func:`load_backend` so that an +ESXi backend can be plugged in by changing ``[backend].type`` in +``config.toml``. ``WorkstationVmrunBackend`` is never imported directly. +""" + +from __future__ import annotations + +import json +import re +import sys +import time +import uuid +from datetime import UTC, datetime +from pathlib import Path +from typing import TYPE_CHECKING + +import click + +from ci_orchestrator.backends import load_backend +from ci_orchestrator.backends.errors import BackendError, BackendNotAvailable +from ci_orchestrator.backends.protocol import VmHandle +from ci_orchestrator.commands.artifacts import ( + _linux_collect, + _windows_collect, + _write_manifest, +) +from ci_orchestrator.commands.build import _linux_build, _windows_build +from ci_orchestrator.commands.vm import _stop_with_fallback, _try_remove_dir +from ci_orchestrator.config import Config, load_config +from ci_orchestrator.credentials import KeyringCredentialStore +from ci_orchestrator.transport.errors import TransportError +from ci_orchestrator.transport.ssh import SshTransport +from ci_orchestrator.transport.winrm import WinRmTransport + +if TYPE_CHECKING: # pragma: no cover + from ci_orchestrator.backends.protocol import VmBackend + + +_VALID_ENV_KEY = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$") +_GUEST_OS_LINUX_HINTS = ("ubuntu", "linux", "debian", "centos", "rhel", "suse") + + +# ────────────────────────────────────────────────────────── helpers + + +def _now() -> float: + return time.monotonic() + + +def _read_guest_os_from_vmx(vmx_path: Path) -> str: + """Best-effort detect ``Linux`` / ``Windows`` from the VMX ``guestOS`` line.""" + try: + text = vmx_path.read_text(encoding="utf-8", errors="replace") + except OSError: + return "windows" + m = re.search(r'guestOS\s*=\s*"([^"]+)"', text) + if not m: + return "windows" + detected = m.group(1).lower() + if any(h in detected for h in _GUEST_OS_LINUX_HINTS): + return "linux" + return "windows" + + +def _apply_vmx_overrides(vmx_path: Path, cpu: int, memory_mb: int) -> None: + """Edit ``numvcpus`` / ``memsize`` in-place. No-op when both are 0.""" + if cpu <= 0 and memory_mb <= 0: + return + lines = vmx_path.read_text(encoding="utf-8", errors="replace").splitlines() + out: list[str] = [] + seen_cpu = False + seen_mem = False + for line in lines: + low = line.strip().lower() + if cpu > 0 and low.startswith("numvcpus"): + out.append(f'numvcpus = "{cpu}"') + seen_cpu = True + elif memory_mb > 0 and low.startswith("memsize"): + out.append(f'memsize = "{memory_mb}"') + seen_mem = True + else: + out.append(line) + if cpu > 0 and not seen_cpu: + out.append(f'numvcpus = "{cpu}"') + if memory_mb > 0 and not seen_mem: + out.append(f'memsize = "{memory_mb}"') + vmx_path.write_text("\n".join(out) + "\n", encoding="utf-8") + + +def _parse_extra_env_json(raw: str) -> dict[str, str]: + """Parse the ``--extra-env-json`` payload. Empty/``{}`` ⇒ empty dict.""" + raw = (raw or "").strip() + if not raw or raw == "{}": + return {} + try: + loaded = json.loads(raw) + except json.JSONDecodeError as exc: + raise click.BadParameter(f"--extra-env-json is not valid JSON: {exc}") from exc + if not isinstance(loaded, dict): + raise click.BadParameter("--extra-env-json must be a JSON object.") + out: dict[str, str] = {} + for key, value in loaded.items(): + if not isinstance(key, str) or not _VALID_ENV_KEY.match(key): + raise click.BadParameter( + f"--extra-env-json key {key!r} is not a valid env var name." + ) + out[key] = "" if value is None else str(value) + return out + + +def _wait_running( + backend: VmBackend, handle: VmHandle, deadline: float, poll: float +) -> bool: + while _now() < deadline: + try: + if backend.is_running(handle): + return True + except BackendError as exc: + click.echo(f"[job] backend probe error: {exc}", err=True) + time.sleep(poll) + return False + + +def _wait_for_ip( + backend: VmBackend, handle: VmHandle, deadline: float, poll: float +) -> str | None: + while _now() < deadline: + try: + ip = backend.get_ip(handle) + except BackendError: + ip = None + if ip: + return ip + time.sleep(poll) + return None + + +def _probe_transport( + *, + guest_os: str, + ip_address: str, + deadline: float, + poll: float, + credential_target: str, + ssh_user: str, + ssh_key_path: str | None, + ssh_known_hosts: str | None, +) -> bool: + """Loop until the guest answers WinRM (Windows) or SSH (Linux).""" + if guest_os == "windows": + cred = KeyringCredentialStore().get(credential_target) + while _now() < deadline: + try: + with WinRmTransport(ip_address, cred.username, cred.password) as t: + if t.is_ready(): + return True + except TransportError as exc: + click.echo(f"[job] WinRM probe failed: {exc}", err=True) + time.sleep(poll) + return False + while _now() < deadline: + try: + with SshTransport( + ip_address, + username=ssh_user, + key_path=ssh_key_path, + known_hosts=ssh_known_hosts, + ) as t: + if t.is_ready(): + return True + except TransportError as exc: + click.echo(f"[job] SSH probe failed: {exc}", err=True) + time.sleep(poll) + return False + + +def _make_clone_name(job_id: str) -> str: + timestamp = datetime.now(tz=UTC).strftime("%Y%m%d_%H%M%S") + # uuid4 suffix avoids collisions when several clones share a JobId. + suffix = uuid.uuid4().hex[:6] + return f"Clone_{job_id}_{timestamp}_{suffix}" + + +def _destroy_clone( + backend: VmBackend | None, + handle: VmHandle | None, + clone_dir: Path | None, +) -> None: + """Final cleanup. Tolerates every error — must never raise.""" + if backend is not None and handle is not None: + try: + _stop_with_fallback(backend, handle) + except Exception as exc: # pragma: no cover - belt and braces + click.echo(f"[job] stop ignored: {exc}", err=True) + try: + backend.delete(handle) + except BackendError as exc: + click.echo( + f"[job] vmrun deleteVM failed: {exc}; will remove dir manually.", + err=True, + ) + except Exception as exc: # pragma: no cover + click.echo(f"[job] delete ignored: {exc}", err=True) + if clone_dir is not None and clone_dir.exists(): + if _try_remove_dir(clone_dir): + click.echo(f"[job] clone directory removed: {clone_dir}") + else: + click.echo( + f"[job] could not fully remove: {clone_dir}", err=True + ) + + +# ────────────────────────────────────────────────────────── command + + +@click.command("job") +@click.option("--job-id", "job_id", required=True) +@click.option("--repo-url", "repo_url", required=True) +@click.option("--branch", required=True) +@click.option("--commit", default="") +@click.option("--configuration", default="Release", show_default=True) +@click.option("--build-command", "build_command", default="") +@click.option("--guest-artifact-source", "guest_artifact_source", default="dist", show_default=True) +@click.option("--submodules", is_flag=True, default=False) +@click.option("--use-git-clone", "use_git_clone", is_flag=True, default=False) +@click.option("--use-shared-cache", "use_shared_cache", is_flag=True, default=False) +@click.option("--skip-artifact", "skip_artifact", is_flag=True, default=False) +@click.option( + "--gitea-credential-target", + "gitea_credential_target", + default="GiteaPAT", + show_default=True, +) +@click.option( + "--template-path", + "template_path", + default=None, + help="Template VMX path (or backend-specific identifier).", +) +@click.option("--snapshot-name", "snapshot_name", default="BaseClean", show_default=True) +@click.option("--clone-base-dir", "clone_base_dir", default=None) +@click.option("--artifact-base-dir", "artifact_base_dir", default=None) +@click.option( + "--guest-os", + type=click.Choice(["windows", "linux", "auto"], case_sensitive=False), + default="auto", + show_default=True, +) +@click.option( + "--guest-credential-target", + "guest_credential_target", + default=None, + help="Override config.guest_cred_target (Windows).", +) +@click.option("--ssh-key-path", "ssh_key_path", default=None) +@click.option("--ssh-user", "ssh_user", default="ci_build", show_default=True) +@click.option("--ssh-known-hosts-file", "ssh_known_hosts_file", default=None) +@click.option( + "--extra-env-json", + "extra_env_json", + default="", + help="JSON object of env vars to inject into the guest before the build.", +) +@click.option("--guest-cpu", "guest_cpu", type=click.IntRange(0, 32), default=0) +@click.option( + "--guest-memory-mb", "guest_memory_mb", type=click.IntRange(0, 131072), default=0 +) +@click.option( + "--ready-timeout", + "ready_timeout", + type=click.FloatRange(10.0, 3600.0), + default=600.0, + show_default=True, + help="Total seconds to wait for VM running + IP + transport ready.", +) +@click.option( + "--poll-interval", + "poll_interval", + type=click.FloatRange(1.0, 60.0), + default=5.0, + show_default=True, +) +@click.option("--vmrun-path", "vmrun_path", default=None) +def job( + job_id: str, + repo_url: str, + branch: str, + commit: str, + configuration: str, + build_command: str, + guest_artifact_source: str, + submodules: bool, + use_git_clone: bool, + use_shared_cache: bool, + skip_artifact: bool, + gitea_credential_target: str, + template_path: str | None, + snapshot_name: str, + clone_base_dir: str | None, + artifact_base_dir: str | None, + guest_os: str, + guest_credential_target: str | None, + ssh_key_path: str | None, + ssh_user: str, + ssh_known_hosts_file: str | None, + extra_env_json: str, + guest_cpu: int, + guest_memory_mb: int, + ready_timeout: float, + poll_interval: float, + vmrun_path: str | None, +) -> None: + """Run the full CI pipeline against an ephemeral build VM. + + Replaces ``scripts/Invoke-CIJob.ps1``. Cleanup of the ephemeral VM is + guaranteed via ``try/finally`` even on failure or ``KeyboardInterrupt``. + """ + # Reserved for future host-side clone variant; the Python port always + # provisions sources via in-guest ``git clone``. + del submodules, use_git_clone, gitea_credential_target + + config: Config = load_config() + if not template_path: + raise click.ClickException( + "--template-path is required." + ) + template_p = Path(template_path) + if not template_p.is_file(): + raise click.ClickException(f"Template VMX not found: {template_path}") + + base_clone_dir = Path(clone_base_dir) if clone_base_dir else config.paths.build_vms + base_artifact_dir = Path(artifact_base_dir) if artifact_base_dir else config.paths.artifacts + base_clone_dir.mkdir(parents=True, exist_ok=True) + job_artifact_dir = base_artifact_dir / job_id + job_artifact_dir.mkdir(parents=True, exist_ok=True) + + extra_env = _parse_extra_env_json(extra_env_json) + + # Build backend from config; allow CLI override for vmrun path. + if vmrun_path is not None: + from ci_orchestrator.backends.workstation import WorkstationVmrunBackend + + backend: VmBackend = WorkstationVmrunBackend(vmrun_path=vmrun_path) + else: + try: + backend = load_backend(config) + except BackendNotAvailable as exc: + raise click.ClickException(f"backend unavailable: {exc}") from exc + + clone_name = _make_clone_name(job_id) + clone_dir = base_clone_dir / clone_name + clone_vmx = clone_dir / f"{clone_name}.vmx" + + handle: VmHandle | None = None + started = datetime.now(tz=UTC) + phase_timings: list[tuple[str, float]] = [] + phase_start = _now() + + click.echo("=" * 60) + click.echo(f"[job] Job : {job_id}") + click.echo(f"[job] Repository : {repo_url}") + click.echo(f"[job] Branch : {branch}") + if commit: + click.echo(f"[job] Commit : {commit}") + click.echo(f"[job] Template : {template_path}") + click.echo(f"[job] Snapshot : {snapshot_name}") + click.echo(f"[job] Artifacts : {job_artifact_dir}") + click.echo(f"[job] Started : {started.isoformat()}") + click.echo("=" * 60) + + try: + # ── Phase 1/5: clone VM ───────────────────────────────────────── + click.echo(f"\n[job] Phase 1/5 — cloning VM: {clone_vmx}") + try: + handle = backend.clone_linked( + template=str(template_p), + snapshot=snapshot_name, + name=clone_name, + destination=str(clone_vmx), + ) + except BackendError as exc: + raise click.ClickException(f"clone failed: {exc}") from exc + if not clone_vmx.is_file(): + raise click.ClickException( + f"backend reported success but clone VMX is missing: {clone_vmx}" + ) + + if guest_cpu > 0 or guest_memory_mb > 0: + click.echo( + f"[job] applying VMX override: CPU={guest_cpu}, RAM={guest_memory_mb}MB" + ) + _apply_vmx_overrides(clone_vmx, guest_cpu, guest_memory_mb) + + phase_timings.append(("clone", _now() - phase_start)) + phase_start = _now() + + # ── Phase 2/5: start VM ───────────────────────────────────────── + click.echo("\n[job] Phase 2/5 — starting VM (headless)") + try: + backend.start(handle, headless=True) + except BackendError as exc: + raise click.ClickException(f"vmrun start failed: {exc}") from exc + + # Resolve guest OS now that the clone exists. + if guest_os.lower() == "auto": + guest_os = _read_guest_os_from_vmx(clone_vmx) + click.echo(f"[job] auto-detected guest OS: {guest_os}") + else: + guest_os = guest_os.lower() + + deadline = _now() + ready_timeout + if not _wait_running(backend, handle, deadline, poll_interval): + raise click.ClickException("timeout waiting for VM to be running.") + ip_address = _wait_for_ip(backend, handle, deadline, poll_interval) + if not ip_address: + raise click.ClickException("timeout waiting for guest IP.") + click.echo(f"[job] guest IP: {ip_address}") + phase_timings.append(("start", _now() - phase_start)) + phase_start = _now() + + # ── Phase 3/5: wait for transport readiness ──────────────────── + click.echo(f"\n[job] Phase 3/5 — waiting for {guest_os} transport readiness") + target = guest_credential_target or config.guest_cred_target + key_path: str | None = ssh_key_path + if key_path is None and config.ssh_key_path is not None: + key_path = str(config.ssh_key_path) + + if not _probe_transport( + guest_os=guest_os, + ip_address=ip_address, + deadline=deadline, + poll=poll_interval, + credential_target=target, + ssh_user=ssh_user, + ssh_key_path=key_path, + ssh_known_hosts=ssh_known_hosts_file, + ): + raise click.ClickException( + "timeout waiting for guest transport to be ready." + ) + phase_timings.append(("wait-ready", _now() - phase_start)) + phase_start = _now() + + # ── Phase 4/5: build inside guest ────────────────────────────── + click.echo("\n[job] Phase 4/5 — invoking remote build") + if guest_os == "linux": + _linux_build( + ip_address=ip_address, + ssh_user=ssh_user, + key_path=key_path, + known_hosts=ssh_known_hosts_file, + workdir="/opt/ci/build", + output_dir="/opt/ci/output", + host_source_dir=None, + clone_url=repo_url, + clone_branch=branch, + clone_commit=commit, + clone_submodules=False, + build_command=build_command, + artifact_source=guest_artifact_source, + extra_env=extra_env, + skip_artifact=skip_artifact, + ) + else: + _windows_build( + ip_address=ip_address, + credential_target=target, + workdir="C:\\CI\\build", + artifact_zip="C:\\CI\\output\\artifacts.zip", + host_source_dir=None, + clone_url=repo_url, + clone_branch=branch, + clone_commit=commit, + clone_submodules=False, + build_command=build_command, + artifact_source=guest_artifact_source, + configuration=configuration, + extra_env=extra_env, + skip_artifact=skip_artifact, + use_shared_cache=use_shared_cache, + ) + phase_timings.append(("build", _now() - phase_start)) + phase_start = _now() + + # ── Phase 5/5: collect artifacts ─────────────────────────────── + if skip_artifact: + click.echo("\n[job] Phase 5/5 — artifact collection skipped (--skip-artifact)") + else: + click.echo("\n[job] Phase 5/5 — collecting artifacts") + if guest_os == "linux": + _linux_collect( + ip_address=ip_address, + ssh_user=ssh_user, + key_path=key_path, + known_hosts=ssh_known_hosts_file, + guest_path="/opt/ci/output", + host_dir=job_artifact_dir, + ) + else: + _windows_collect( + ip_address=ip_address, + credential_target=target, + guest_path="C:\\CI\\output\\artifacts.zip", + host_dir=job_artifact_dir, + include_logs=True, + ) + try: + count = _write_manifest(job_artifact_dir, job_id, commit) + click.echo(f"[job] manifest.json written ({count} file(s))") + except OSError as exc: + click.echo(f"[job] manifest write failed: {exc}", err=True) + phase_timings.append(("artifacts", _now() - phase_start)) + + elapsed = (datetime.now(tz=UTC) - started).total_seconds() + click.echo("\n" + "=" * 60) + click.echo("[job] phase durations:") + for name, secs in phase_timings: + click.echo(f" {name:<14} {int(secs):>5}s") + click.echo(f" {'TOTAL':<14} {int(elapsed):>5}s") + click.echo("=" * 60) + click.echo(f"[job] SUCCESS — Job {job_id} completed in {int(elapsed)}s") + click.echo(f"[job] Artifacts: {job_artifact_dir}") + click.echo("=" * 60) + except KeyboardInterrupt: + click.echo("\n[job] interrupted by user; cleaning up...", err=True) + _destroy_clone(backend, handle, clone_dir) + sys.exit(130) + except click.ClickException: + _destroy_clone(backend, handle, clone_dir) + raise + except Exception as exc: + click.echo(f"\n[job] FAILURE: {exc}", err=True) + _destroy_clone(backend, handle, clone_dir) + raise click.ClickException(str(exc)) from exc + else: + _destroy_clone(backend, handle, clone_dir) + + +__all__ = ["job"] diff --git a/tests/python/test_commands_job.py b/tests/python/test_commands_job.py new file mode 100644 index 0000000..122565a --- /dev/null +++ b/tests/python/test_commands_job.py @@ -0,0 +1,394 @@ +"""Tests for ``ci_orchestrator job`` (Phase A4).""" + +from __future__ import annotations + +from pathlib import Path +from typing import Any + +import click +import pytest +from click.testing import CliRunner + +import ci_orchestrator.commands.job as job_module +from ci_orchestrator.__main__ import cli +from ci_orchestrator.backends.errors import BackendOperationFailed +from ci_orchestrator.backends.protocol import VmHandle + +# ──────────────────────────────────────────────────────────── fakes + + +class _FakeBackend: + """Records every backend call and creates the VMX file on clone.""" + + def __init__( + self, + *, + clone_fail: bool = False, + start_fail: bool = False, + delete_fail: bool = False, + ip: str | None = "10.99.0.42", + running_after: int = 0, + clone_guest_os: str = "ubuntu-64", + ) -> None: + self.clone_fail = clone_fail + self.start_fail = start_fail + self.delete_fail = delete_fail + self.ip = ip + self.running_after = running_after + self.clone_guest_os = clone_guest_os + self.calls: list[str] = [] + self._is_running_count = 0 + + def clone_linked( + self, + template: str, + snapshot: str, + name: str, + destination: str | None = None, + ) -> VmHandle: + self.calls.append("clone") + if self.clone_fail: + raise BackendOperationFailed("clone", 1, "boom") + assert destination is not None + dst = Path(destination) + dst.parent.mkdir(parents=True, exist_ok=True) + dst.write_text( + f'config.version = "8"\nguestOS = "{self.clone_guest_os}"\n', + encoding="utf-8", + ) + return VmHandle(identifier=str(dst), name=name) + + def start(self, handle: VmHandle, headless: bool = True) -> None: + self.calls.append("start") + if self.start_fail: + raise BackendOperationFailed("start", 1, "no") + + def stop(self, handle: VmHandle, hard: bool = False) -> None: + self.calls.append(f"stop:{'hard' if hard else 'soft'}") + + def delete(self, handle: VmHandle) -> None: + self.calls.append("delete") + if self.delete_fail: + raise BackendOperationFailed("deleteVM", 1, "locked") + + def is_running(self, handle: VmHandle) -> bool: + self._is_running_count += 1 + return self._is_running_count > self.running_after + + def get_ip(self, handle: VmHandle, timeout: float = 0.0) -> str | None: + return self.ip + + def list_snapshots(self, handle: VmHandle) -> list[str]: + return [] + + +def _patch_common( + monkeypatch: pytest.MonkeyPatch, + backend: _FakeBackend, + *, + transport_ready: bool = True, + linux_build_raises: bool = False, +) -> dict[str, list[Any]]: + """Patch all in-process collaborators of the ``job`` command.""" + calls: dict[str, list[Any]] = { + "linux_build": [], + "windows_build": [], + "linux_collect": [], + "windows_collect": [], + "manifest": [], + } + + monkeypatch.setattr(job_module, "load_backend", lambda _cfg: backend) + + class _ProbeOk: + def __enter__(self) -> _ProbeOk: + return self + + def __exit__(self, *_e: object) -> None: + return None + + def is_ready(self) -> bool: + return transport_ready + + class _Store: + def get(self, _t: str) -> Any: + from ci_orchestrator.credentials import Credential + + return Credential("ci", "pwd") + + monkeypatch.setattr(job_module, "WinRmTransport", lambda *a, **k: _ProbeOk()) + monkeypatch.setattr(job_module, "SshTransport", lambda *a, **k: _ProbeOk()) + monkeypatch.setattr(job_module, "KeyringCredentialStore", lambda: _Store()) + + def _linux_build(**kwargs: Any) -> None: + calls["linux_build"].append(kwargs) + if linux_build_raises: + raise RuntimeError("build exploded") + + def _windows_build(**kwargs: Any) -> None: + calls["windows_build"].append(kwargs) + + def _linux_collect(**kwargs: Any) -> None: + calls["linux_collect"].append(kwargs) + # Simulate a transferred file so manifest writing has something to do. + Path(kwargs["host_dir"], "out.txt").write_text("ok", encoding="utf-8") + + def _windows_collect(**kwargs: Any) -> None: + calls["windows_collect"].append(kwargs) + + def _manifest(host_dir: Path, job_id: str, commit: str) -> int: + calls["manifest"].append((str(host_dir), job_id, commit)) + return 1 + + monkeypatch.setattr(job_module, "_linux_build", _linux_build) + monkeypatch.setattr(job_module, "_windows_build", _windows_build) + monkeypatch.setattr(job_module, "_linux_collect", _linux_collect) + monkeypatch.setattr(job_module, "_windows_collect", _windows_collect) + monkeypatch.setattr(job_module, "_write_manifest", _manifest) + # Make poll loops instant. + monkeypatch.setattr(job_module.time, "sleep", lambda _s: None) + return calls + + +@pytest.fixture +def template_vmx(tmp_path: Path) -> Path: + """Linux-flavoured VMX so auto-detect picks the SSH transport.""" + vmx = tmp_path / "templates" / "LinuxBuild2404.vmx" + vmx.parent.mkdir(parents=True) + vmx.write_text('guestOS = "ubuntu-64"\n', encoding="utf-8") + return vmx + + +def _common_args( + template: Path, + *, + clone_base: Path, + artifact_base: Path, + extra: list[str] | None = None, +) -> list[str]: + return [ + "job", + "--job-id", + "ci-test-1", + "--repo-url", + "http://gitea.local/org/repo.git", + "--branch", + "main", + "--template-path", + str(template), + "--clone-base-dir", + str(clone_base), + "--artifact-base-dir", + str(artifact_base), + "--ready-timeout", + "30", + "--poll-interval", + "1", + *(extra or []), + ] + + +# ──────────────────────────────────────────────────────────── tests + + +def test_job_happy_path_linux( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path, template_vmx: Path +) -> None: + backend = _FakeBackend(running_after=1) + calls = _patch_common(monkeypatch, backend) + + base_clone = tmp_path / "build-vms" + base_artifact = tmp_path / "artifacts" + + result = CliRunner().invoke( + cli, _common_args(template_vmx, clone_base=base_clone, artifact_base=base_artifact) + ) + assert result.exit_code == 0, result.output + # Backend received the full lifecycle including delete in finally. + assert backend.calls[0] == "clone" + assert "start" in backend.calls + assert backend.calls[-1] == "delete" + # Linux build branch was taken (auto-detected from VMX guestOS). + assert len(calls["linux_build"]) == 1 + assert calls["linux_build"][0]["build_command"] == "" + # Artifacts were collected into the per-job directory and manifest written. + assert (base_artifact / "ci-test-1" / "out.txt").is_file() + assert calls["manifest"] and calls["manifest"][0][1] == "ci-test-1" + # Clone directory removed. + assert not any((base_clone).iterdir()) or all(not p.exists() for p in base_clone.iterdir()) + + +def test_job_skip_artifact_branch( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path, template_vmx: Path +) -> None: + backend = _FakeBackend(running_after=0) + calls = _patch_common(monkeypatch, backend) + result = CliRunner().invoke( + cli, + _common_args( + template_vmx, + clone_base=tmp_path / "vms", + artifact_base=tmp_path / "art", + extra=["--skip-artifact"], + ), + ) + assert result.exit_code == 0, result.output + assert calls["linux_collect"] == [] + assert calls["windows_collect"] == [] + assert calls["manifest"] == [] + assert backend.calls[-1] == "delete" + + +def test_job_cleanup_on_build_failure( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path, template_vmx: Path +) -> None: + """If the build raises, the VM must still be destroyed (try/finally).""" + backend = _FakeBackend(running_after=0) + _patch_common(monkeypatch, backend, linux_build_raises=True) + result = CliRunner().invoke( + cli, + _common_args(template_vmx, clone_base=tmp_path / "vms", artifact_base=tmp_path / "art"), + ) + assert result.exit_code != 0 + assert "build exploded" in result.output + assert "delete" in backend.calls + # Stop attempt happened before delete. + assert any(c.startswith("stop:") for c in backend.calls) + + +def test_job_cleanup_on_clone_failure( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path, template_vmx: Path +) -> None: + backend = _FakeBackend(clone_fail=True) + _patch_common(monkeypatch, backend) + result = CliRunner().invoke( + cli, + _common_args(template_vmx, clone_base=tmp_path / "vms", artifact_base=tmp_path / "art"), + ) + assert result.exit_code != 0 + assert "clone failed" in result.output + # Backend.start/delete must not have been called: handle is None. + assert backend.calls == ["clone"] + + +def test_job_rejects_missing_template( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + backend = _FakeBackend() + _patch_common(monkeypatch, backend) + result = CliRunner().invoke( + cli, + _common_args( + tmp_path / "ghost.vmx", + clone_base=tmp_path / "vms", + artifact_base=tmp_path / "art", + ), + ) + assert result.exit_code != 0 + assert "Template VMX not found" in result.output + assert backend.calls == [] + + +def test_job_windows_branch_with_overrides( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + """Windows-flavoured VMX selects the WinRM build path; CPU/RAM override applied.""" + vmx = tmp_path / "templates" / "WinBuild2025.vmx" + vmx.parent.mkdir(parents=True) + vmx.write_text('guestOS = "windows10srv-64"\n', encoding="utf-8") + + backend = _FakeBackend(running_after=0, clone_guest_os="windows10srv-64") + calls = _patch_common(monkeypatch, backend) + extra = [ + "--guest-cpu", + "4", + "--guest-memory-mb", + "8192", + "--extra-env-json", + '{"BUILD_TAG":"abc"}', + ] + result = CliRunner().invoke( + cli, + _common_args( + vmx, + clone_base=tmp_path / "vms", + artifact_base=tmp_path / "art", + extra=extra, + ), + ) + assert result.exit_code == 0, result.output + assert calls["windows_build"], "Windows build branch not taken" + assert calls["windows_build"][0]["extra_env"] == {"BUILD_TAG": "abc"} + # CPU/RAM override patched the clone VMX. + clone_vmx = next((tmp_path / "vms").rglob("*.vmx"), None) + # After successful job the clone dir is deleted, so the file is gone. + # Check that the Windows collect helper got the correct artifact path instead. + assert calls["windows_collect"][0]["guest_path"].endswith("artifacts.zip") + assert clone_vmx is None # cleaned up + assert backend.calls[-1] == "delete" + + +def test_job_invalid_extra_env_json( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path, template_vmx: Path +) -> None: + backend = _FakeBackend() + _patch_common(monkeypatch, backend) + result = CliRunner().invoke( + cli, + _common_args( + template_vmx, + clone_base=tmp_path / "vms", + artifact_base=tmp_path / "art", + extra=["--extra-env-json", '{"bad-key":"x"}'], + ), + ) + assert result.exit_code != 0 + assert "not a valid env var name" in result.output + # Failure is parameter-time; clone never started. + assert backend.calls == [] + + +# ──────────────────────────────────────────────── helper-level tests + + +def test_apply_vmx_overrides_replaces_existing_lines(tmp_path: Path) -> None: + vmx = tmp_path / "x.vmx" + vmx.write_text( + 'config.version = "8"\nnumvcpus = "1"\nmemsize = "2048"\n', + encoding="utf-8", + ) + job_module._apply_vmx_overrides(vmx, cpu=8, memory_mb=16384) + text = vmx.read_text(encoding="utf-8") + assert 'numvcpus = "8"' in text + assert 'memsize = "16384"' in text + + +def test_apply_vmx_overrides_appends_when_missing(tmp_path: Path) -> None: + vmx = tmp_path / "x.vmx" + vmx.write_text('config.version = "8"\n', encoding="utf-8") + job_module._apply_vmx_overrides(vmx, cpu=2, memory_mb=4096) + text = vmx.read_text(encoding="utf-8") + assert 'numvcpus = "2"' in text + assert 'memsize = "4096"' in text + + +def test_read_guest_os_detects_linux(tmp_path: Path) -> None: + vmx = tmp_path / "x.vmx" + vmx.write_text('guestOS = "ubuntu-64"\n', encoding="utf-8") + assert job_module._read_guest_os_from_vmx(vmx) == "linux" + + +def test_read_guest_os_defaults_to_windows(tmp_path: Path) -> None: + vmx = tmp_path / "x.vmx" + vmx.write_text("# nothing useful\n", encoding="utf-8") + assert job_module._read_guest_os_from_vmx(vmx) == "windows" + + +def test_parse_extra_env_json_empty_inputs() -> None: + assert job_module._parse_extra_env_json("") == {} + assert job_module._parse_extra_env_json("{}") == {} + + +def test_parse_extra_env_json_rejects_non_object() -> None: + with pytest.raises(click.exceptions.UsageError): + job_module._parse_extra_env_json("[]")