Simone
509d1fc284
H4: Invoke-VmrunBounded added to _Common.psm1 (Start-Process + Wait-Process + taskkill on
timeout, returns {ExitCode;Output;TimedOut;ElapsedSeconds}). Wired in Invoke-CIJob.ps1
for vmrun start (180s) and Remove-BuildVM.ps1 for stop/deleteVM (60s/90s).
H5: Cleanup-OrphanedBuildVMs.ps1 removes stale vm-start.lock (>30 min). ValidateRange
lowered to 0 to allow emergency -MaxAgeHours 0. Invoke-RetentionPolicy.ps1 same.
H8: _Transport.psm1 SSH hardening: StrictHostKeyChecking=accept-new, F:\CI\State\known_hosts.
H9: Validate-DeployState.ps1 Linux branch added: checks ci-report-ip.service enabled,
ci-report-ip.sh executable via SSH. New params: -GuestOS, -SshKeyPath, -SshUser.
H12: Get-GuestIPAddress extracted into _Common.psm1 (guestinfo.ci-ip primary, getGuestIPAddress
fallback). Invoke-CIJob.ps1 Phase 3b replaced with single call. Measure-CIBenchmark.ps1
updated to match.
Also: _Common.psm1 adds Write-CIRedactedCommand and ConvertTo-SafeShellSingleQuotedString.
Pester suite updated: 30/30 pass.
615 lines
29 KiB
PowerShell
615 lines
29 KiB
PowerShell
#Requires -Version 5.1
|
|
<#
|
|
.SYNOPSIS
|
|
Main CI orchestrator: creates an ephemeral VM, runs a build, collects
|
|
artifacts, and destroys the VM — in a guaranteed try/finally pattern.
|
|
|
|
.DESCRIPTION
|
|
This script is called by act_runner as the primary build step.
|
|
It coordinates all sub-scripts:
|
|
1. New-BuildVM.ps1 — linked clone from template
|
|
2. vmrun start — start the clone
|
|
3. Wait-VMReady.ps1 — poll until WinRM is ready
|
|
4. Invoke-RemoteBuild.ps1 — execute build inside VM
|
|
5. Get-BuildArtifacts.ps1 — copy artifacts to host
|
|
6. Remove-BuildVM.ps1 — destroy VM (always runs via finally)
|
|
|
|
Exit codes:
|
|
0 = success
|
|
1 = build or orchestration failure (VM destroyed, artifacts may be partial)
|
|
|
|
.PARAMETER JobId
|
|
Unique job identifier (use ${{ github.run_id }} from Gitea Actions).
|
|
|
|
.PARAMETER RepoUrl
|
|
Git URL of the repository to build (reachable from the build VM).
|
|
|
|
.PARAMETER Branch
|
|
Branch to build.
|
|
|
|
.PARAMETER Commit
|
|
Optional specific commit SHA to check out.
|
|
|
|
.PARAMETER Configuration
|
|
Build configuration. Default: Release
|
|
|
|
.PARAMETER TemplatePath
|
|
Full path to the template VM .vmx file.
|
|
Can also be set via env var GITEA_CI_TEMPLATE_PATH.
|
|
|
|
.PARAMETER SnapshotName
|
|
Template snapshot to clone from. Default: BaseClean
|
|
|
|
.PARAMETER CloneBaseDir
|
|
Base directory for clone VMs.
|
|
Can also be set via env var GITEA_CI_CLONE_BASE_DIR.
|
|
Default: F:\CI\BuildVMs
|
|
|
|
.PARAMETER ArtifactBaseDir
|
|
Host directory where artifacts are stored per job.
|
|
Default: F:\CI\Artifacts
|
|
|
|
.PARAMETER VMIPAddress
|
|
IP address to assign/expect for the build VM.
|
|
Must be unique per concurrent build.
|
|
Can be passed explicitly or derived from a registry/file-based IP allocator.
|
|
|
|
.PARAMETER GuestCredentialTarget
|
|
Target name in Windows Credential Manager for guest credentials.
|
|
Default: BuildVMGuest
|
|
|
|
.PARAMETER VmrunPath
|
|
Path to vmrun.exe.
|
|
Default: C:\Program Files (x86)\VMware\VMware Workstation\vmrun.exe
|
|
|
|
.EXAMPLE
|
|
# Called from Gitea Actions workflow:
|
|
.\scripts\Invoke-CIJob.ps1 `
|
|
-JobId "${{ github.run_id }}" `
|
|
-RepoUrl "http://gitea.local/org/myrepo.git" `
|
|
-Branch "${{ github.ref_name }}" `
|
|
-Commit "${{ github.sha }}" `
|
|
-VMIPAddress "192.168.11.101"
|
|
#>
|
|
[CmdletBinding()]
|
|
param(
|
|
[Parameter(Mandatory)]
|
|
[ValidatePattern('^[A-Za-z0-9._-]+$')]
|
|
[string] $JobId,
|
|
|
|
[Parameter(Mandatory)]
|
|
[string] $RepoUrl,
|
|
|
|
[Parameter(Mandatory)]
|
|
[string] $Branch,
|
|
|
|
[string] $Commit = '',
|
|
|
|
[string] $Configuration = 'Release',
|
|
|
|
# Pass -Submodules to clone with --recurse-submodules
|
|
[switch] $Submodules,
|
|
|
|
# Custom build command to run inside the VM (empty = dotnet build)
|
|
[string] $BuildCommand = '',
|
|
|
|
# Artifact source pattern relative to GuestWorkDir (used with BuildCommand)
|
|
[string] $GuestArtifactSource = 'dist',
|
|
|
|
# Clone repo IN the VM instead of on host (requires Git in template §6.6)
|
|
# Skips Phase 1 host clone, injects URL/branch/commit into VM for in-VM clone.
|
|
# Prerequisite: Git for Windows installed in template.
|
|
[switch] $UseGitClone,
|
|
|
|
# Use VMware shared folders for NuGet/pip cache (§SharedCache).
|
|
# Requires Set-TemplateSharedFolders.ps1 to have been run on the template.
|
|
[switch] $UseSharedCache,
|
|
|
|
# Credential Manager target for Gitea PAT used by -UseGitClone (private repos).
|
|
# Store with: New-StoredCredential -Target 'GiteaPAT' -UserName '<user>' -Password '<pat>' -Persist LocalMachine
|
|
# Ignored when -UseGitClone is not set.
|
|
[string] $GiteaCredentialTarget = 'GiteaPAT',
|
|
|
|
[string] $TemplatePath = $env:GITEA_CI_TEMPLATE_PATH,
|
|
|
|
[string] $SnapshotName = $(if ($env:GITEA_CI_SNAPSHOT_NAME) { $env:GITEA_CI_SNAPSHOT_NAME } else { 'BaseClean' }),
|
|
|
|
[string] $CloneBaseDir = $(if ($env:GITEA_CI_CLONE_BASE_DIR) { $env:GITEA_CI_CLONE_BASE_DIR } else { 'F:\CI\BuildVMs' }),
|
|
|
|
[string] $ArtifactBaseDir = 'F:\CI\Artifacts',
|
|
|
|
[ValidatePattern('^((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)\.){3}(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)$')]
|
|
[string] $VMIPAddress = '', # if empty, auto-detected via vmrun getGuestIPAddress
|
|
|
|
[string] $GuestCredentialTarget = 'BuildVMGuest',
|
|
|
|
[string] $VmrunPath = 'C:\Program Files (x86)\VMware\VMware Workstation\vmrun.exe',
|
|
|
|
# Directory for job execution logs. Each job gets its own sub-directory:
|
|
# $LogDir\$JobId\invoke-ci.log
|
|
[string] $LogDir = 'F:\CI\Logs',
|
|
|
|
# Days to retain job logs. Directories older than this are purged at job end.
|
|
# Set to 0 to disable retention/purge.
|
|
[int] $LogRetentionDays = 30,
|
|
|
|
# Guest OS type — determines transport (WinRM for Windows, SSH for Linux).
|
|
# When 'Auto', detected from the VMX guestOS field after clone.
|
|
[ValidateSet('Windows', 'Linux', 'Auto')]
|
|
[string] $GuestOS = 'Auto',
|
|
|
|
[string] $SshKeyPath = 'F:\CI\keys\ci_linux',
|
|
[string] $SshUser = 'ci_build',
|
|
|
|
# Persistent known_hosts file used for CI-job SSH calls.
|
|
# Set to empty string to disable host-key checking (template provisioning only).
|
|
[string] $SshKnownHostsFile = 'F:\CI\State\known_hosts',
|
|
|
|
# Extra environment variables injected into the guest VM before the build command (§6.5).
|
|
# Keys must be valid environment variable names. Values must be plain strings.
|
|
# Example: @{ SIGN_PASS = $env:SIGN_PASS_FROM_SECRET; GPG_KEY_ID = '0xABCD1234' }
|
|
[hashtable] $ExtraGuestEnv = @{}
|
|
)
|
|
|
|
Set-StrictMode -Version Latest
|
|
$ErrorActionPreference = 'Stop'
|
|
|
|
# ── Resolve script directory for dot-sourcing sibling scripts ─────────────────
|
|
$scriptDir = $PSScriptRoot
|
|
|
|
# ── Load shared helpers (Invoke-VmrunBounded, Get-GuestIPAddress, …) ──────────
|
|
Import-Module (Join-Path $scriptDir '_Common.psm1') -Force
|
|
|
|
# ── Validate required config ──────────────────────────────────────────────────
|
|
if ([string]::IsNullOrWhiteSpace($TemplatePath)) {
|
|
Write-Error "TemplatePath is required. Set -TemplatePath or env:GITEA_CI_TEMPLATE_PATH."
|
|
exit 1
|
|
}
|
|
if (-not (Test-Path $TemplatePath -PathType Leaf)) {
|
|
Write-Error "Template VMX not found: $TemplatePath"
|
|
exit 1
|
|
}
|
|
if (-not (Test-Path $VmrunPath -PathType Leaf)) {
|
|
Write-Error "vmrun.exe not found: $VmrunPath"
|
|
exit 1
|
|
}
|
|
|
|
# ── Validate ExtraGuestEnv keys ──────────────────────────────────────────────
|
|
foreach ($key in $ExtraGuestEnv.Keys) {
|
|
if ($key -notmatch '^[A-Za-z_][A-Za-z0-9_]*$') {
|
|
Write-Error "ExtraGuestEnv key '$key' is not a valid environment variable name. Keys must match ^[A-Za-z_][A-Za-z0-9_]*$."
|
|
exit 1
|
|
}
|
|
}
|
|
|
|
# ── Load guest credentials from Windows Credential Manager ───────────────────
|
|
# Requires the CredentialManager module: Install-Module CredentialManager
|
|
# In a lab setup you can fall back to prompting, but never hardcode credentials.
|
|
$credential = $null
|
|
try {
|
|
$credential = Get-StoredCredential -Target $GuestCredentialTarget -ErrorAction Stop
|
|
Write-Host "[Invoke-CIJob] Loaded guest credentials from Credential Manager ($GuestCredentialTarget)."
|
|
}
|
|
catch {
|
|
Write-Warning "[Invoke-CIJob] Could not load credentials from Credential Manager ($GuestCredentialTarget)."
|
|
Write-Warning " Using interactive prompt as fallback. Store credentials with:"
|
|
Write-Warning " cmdkey /generic:$GuestCredentialTarget /user:DOMAIN\user /pass:password"
|
|
$credential = Get-Credential -Message "Enter guest VM credentials for job $JobId"
|
|
}
|
|
|
|
# ── Setup paths ───────────────────────────────────────────────────────────────
|
|
$jobArtifactDir = Join-Path $ArtifactBaseDir $JobId
|
|
$hostCloneDir = Join-Path $env:TEMP "ci-src-$JobId"
|
|
$cloneVmxPath = $null
|
|
$leaseFile = $null # §2.1 IP lease file path — released in finally
|
|
$jsonLog = $null # §4.1 JSONL structured log path
|
|
$startTime = Get-Date
|
|
|
|
# ── Phase-duration tracking (Task 5.5) ───────────────────────────────────────
|
|
$phaseTimings = [System.Collections.Generic.List[PSCustomObject]]::new()
|
|
$phaseStart = $startTime
|
|
|
|
# ── Start transcript + JSONL log ──────────────────────────────────────────────
|
|
$transcriptStarted = $false
|
|
try {
|
|
$jobLogDir = Join-Path $LogDir $JobId
|
|
if (-not (Test-Path $jobLogDir)) { New-Item -ItemType Directory -Path $jobLogDir -Force | Out-Null }
|
|
$transcriptPath = Join-Path $jobLogDir 'invoke-ci.log'
|
|
$jsonLog = Join-Path $jobLogDir 'invoke-ci.jsonl'
|
|
# -Append allows starting a new transcript even when VS Code (or another host)
|
|
# has already opened a transcript on this PowerShell session.
|
|
Start-Transcript -Path $transcriptPath -Append -Force -ErrorAction Stop
|
|
$transcriptStarted = $true
|
|
}
|
|
catch {
|
|
Write-Warning "[Invoke-CIJob] Could not start transcript: $_"
|
|
}
|
|
|
|
# ── §4.1 Structured event emitter ────────────────────────────────────────────
|
|
# Appends a single JSON line to invoke-ci.jsonl per phase transition.
|
|
# Silent no-op if $jsonLog is null (transcript setup failed).
|
|
function Write-JobEvent {
|
|
param(
|
|
[Parameter(Mandatory)] [string] $Phase,
|
|
[Parameter(Mandatory)] [string] $Status, # start | success | failure | info
|
|
[hashtable] $Data = @{}
|
|
)
|
|
if (-not $script:jsonLog) { return }
|
|
$logEntry = [ordered]@{
|
|
ts = (Get-Date -Format 'o')
|
|
jobId = $script:JobId
|
|
phase = $Phase
|
|
status = $Status
|
|
data = $Data
|
|
}
|
|
try {
|
|
$logEntry | ConvertTo-Json -Compress -Depth 5 |
|
|
Add-Content -Path $script:jsonLog -Encoding UTF8 -ErrorAction Stop
|
|
} catch { Write-Debug "[Write-JobEvent] Logging skipped: $_" } # never let logging fail the build
|
|
}
|
|
|
|
Write-Host "============================================================"
|
|
Write-Host "[Invoke-CIJob] Job : $JobId"
|
|
Write-Host "[Invoke-CIJob] Repository : $RepoUrl"
|
|
Write-Host "[Invoke-CIJob] Branch : $Branch"
|
|
if ($Commit) { Write-Host "[Invoke-CIJob] Commit : $Commit" }
|
|
Write-Host "[Invoke-CIJob] Build VM IP : $VMIPAddress"
|
|
Write-Host "[Invoke-CIJob] Template : $TemplatePath"
|
|
Write-Host "[Invoke-CIJob] Artifacts : $jobArtifactDir"
|
|
Write-Host "[Invoke-CIJob] Started : $startTime"
|
|
Write-Host "============================================================"
|
|
|
|
Write-JobEvent -Phase 'job' -Status 'start' -Data @{
|
|
repo = $RepoUrl
|
|
branch = $Branch
|
|
commit = $Commit
|
|
template = $TemplatePath
|
|
snapshot = $SnapshotName
|
|
}
|
|
|
|
$exitCode = 0
|
|
|
|
try {
|
|
# ── Phase 1: Clone repo (HOST or GUEST) ───────────────────────────────
|
|
if ($UseGitClone) {
|
|
# §3.3: Clone in guest VM instead of host. Skip host clone phase.
|
|
# PAT will be injected at runtime (Phase 5) from Credential Manager.
|
|
Write-Host "`n[Phase 1/6] Skipping host clone — will clone in guest VM (-UseGitClone)..."
|
|
Write-JobEvent -Phase 'phase1.clone-repo' -Status 'skipped' -Data @{ method = 'in-vm-clone' }
|
|
}
|
|
else {
|
|
# Default: Clone on host, zip, transfer to guest via WinRM.
|
|
Write-Host "`n[Phase 1/6] Cloning repository on host..."
|
|
Write-JobEvent -Phase 'phase1.clone-repo' -Status 'start'
|
|
if (Test-Path $hostCloneDir) { Remove-Item $hostCloneDir -Recurse -Force }
|
|
$gitArgs = @('clone', '--depth', '1', '--progress', '--branch', $Branch)
|
|
if ($Submodules) { $gitArgs += @('--recurse-submodules', '--shallow-submodules') }
|
|
$gitArgs += @($RepoUrl, $hostCloneDir)
|
|
Write-Host "[Invoke-CIJob] git $($gitArgs -join ' ')"
|
|
$ErrorActionPreference = 'Continue'
|
|
# Stream git stdout/stderr live (otherwise heavy clones with submodules
|
|
# look frozen for a long time). Tee for diagnostic in case of failure.
|
|
$gitOutput = & git @gitArgs 2>&1 | Tee-Object -Variable streamed | ForEach-Object { Write-Host " [git] $_"; "$_" }
|
|
$gitExit = $LASTEXITCODE
|
|
$ErrorActionPreference = 'Stop'
|
|
if ($gitExit -ne 0) {
|
|
throw "git clone failed (exit $gitExit):`n$($gitOutput -join "`n")"
|
|
}
|
|
if ($Commit -ne '') {
|
|
$ErrorActionPreference = 'Continue'
|
|
$gitOutput = & git -C $hostCloneDir checkout $Commit 2>&1 | ForEach-Object { "$_" }
|
|
$gitExit = $LASTEXITCODE
|
|
if ($gitExit -ne 0) {
|
|
# Commit not in shallow history (e.g. manual run on older SHA).
|
|
# Fetch full history and retry.
|
|
Write-Host "[Invoke-CIJob] Shallow checkout failed — fetching full history..."
|
|
& git -C $hostCloneDir fetch --unshallow 2>&1 | Out-Null
|
|
$gitOutput = & git -C $hostCloneDir checkout $Commit 2>&1 | ForEach-Object { "$_" }
|
|
$gitExit = $LASTEXITCODE
|
|
}
|
|
$ErrorActionPreference = 'Stop'
|
|
if ($gitExit -ne 0) {
|
|
throw "git checkout $Commit failed (exit $gitExit):`n$($gitOutput -join "`n")"
|
|
}
|
|
}
|
|
Write-Host "[Invoke-CIJob] Source cloned to: $hostCloneDir"
|
|
Write-JobEvent -Phase 'phase1.clone-repo' -Status 'success' -Data @{ dir = $hostCloneDir }
|
|
}
|
|
$phaseTimings.Add([PSCustomObject]@{ Phase = 'Phase 1 - Clone repo'; Sec = [int]((Get-Date) - $phaseStart).TotalSeconds })
|
|
$phaseStart = Get-Date
|
|
|
|
# ── Phase 2-3b: Clone + Start + IP (serialized via file-based mutex) ────
|
|
# §2.1: With capacity:4 and DHCP on VMnet8, concurrent vmrun starts can race
|
|
# and produce two VMs with the same DHCP lease. A file-based mutex ensures
|
|
# only one job is in the clone→start→IP phase at a time. After the IP is
|
|
# acquired and written to a lease file the lock is released, so build phases
|
|
# (the slow part) run fully in parallel. Lease is released in finally.
|
|
$stateDir = 'F:\CI\State'
|
|
$leasesDir = Join-Path $stateDir 'ip-leases'
|
|
$lockPath = Join-Path $stateDir 'vm-start.lock'
|
|
if (-not (Test-Path $leasesDir)) { New-Item -ItemType Directory -Path $leasesDir -Force | Out-Null }
|
|
|
|
# ── Pre-clone disk space gate ─────────────────────────────────────────
|
|
$cloneDriveLetter = (Split-Path -Qualifier $CloneBaseDir).TrimEnd(':')
|
|
$cloneVol = Get-Volume -DriveLetter $cloneDriveLetter -ErrorAction SilentlyContinue
|
|
if ($null -ne $cloneVol) {
|
|
$freeGB = [math]::Round($cloneVol.SizeRemaining / 1GB, 1)
|
|
if ($freeGB -lt 20) {
|
|
throw "Insufficient disk space on ${cloneDriveLetter}: ($freeGB GB free). 20 GB minimum required before VM clone."
|
|
}
|
|
Write-Host "[Invoke-CIJob] Disk space on ${cloneDriveLetter}: ${freeGB} GB free (OK)."
|
|
}
|
|
|
|
Write-Host "`n[Phase 2/6] Acquiring VM-start lock (prevents DHCP collision with capacity>1)..."
|
|
Write-JobEvent -Phase 'phase2-3b.vm-start' -Status 'start'
|
|
$lockStream = $null
|
|
$lockDeadline = (Get-Date).AddMinutes(10)
|
|
while ($true) {
|
|
try {
|
|
$lockStream = [System.IO.File]::Open(
|
|
$lockPath,
|
|
[System.IO.FileMode]::OpenOrCreate,
|
|
[System.IO.FileAccess]::ReadWrite,
|
|
[System.IO.FileShare]::None)
|
|
break
|
|
} catch [System.IO.IOException] {
|
|
if ((Get-Date) -ge $lockDeadline) {
|
|
throw "[Phase 2] VM-start lock timeout after 10 min — a concurrent job may be stuck."
|
|
}
|
|
Write-Host "[Invoke-CIJob] VM-start lock busy, retrying in 5s..."
|
|
Start-Sleep -Seconds 5
|
|
}
|
|
}
|
|
Write-Host "[Invoke-CIJob] VM-start lock acquired."
|
|
|
|
try {
|
|
# ── Phase 2: Clone VM ─────────────────────────────────────────────
|
|
Write-Host "[Phase 2/6] Cloning VM..."
|
|
$cloneVmxPath = & "$scriptDir\New-BuildVM.ps1" `
|
|
-TemplatePath $TemplatePath `
|
|
-SnapshotName $SnapshotName `
|
|
-CloneBaseDir $CloneBaseDir `
|
|
-JobId $JobId `
|
|
-VmrunPath $VmrunPath
|
|
|
|
# ── Phase 3: Start VM ─────────────────────────────────────────────
|
|
Write-Host "`n[Phase 3/6] Starting VM..."
|
|
$startResult = Invoke-VmrunBounded -VmrunPath $VmrunPath `
|
|
-Operation start `
|
|
-Arguments @($cloneVmxPath, 'nogui') `
|
|
-TimeoutSeconds 180
|
|
if ($startResult.TimedOut) {
|
|
throw "vmrun start timed out after 180s for: $cloneVmxPath"
|
|
}
|
|
if ($startResult.ExitCode -ne 0) {
|
|
throw "vmrun start failed (exit $($startResult.ExitCode)): $($startResult.Output)"
|
|
}
|
|
Write-Host "[Invoke-CIJob] VM started (took $($startResult.ElapsedSeconds)s)."
|
|
|
|
# ── Phase 3b: Resolve VM IP ───────────────────────────────────────
|
|
if ([string]::IsNullOrWhiteSpace($VMIPAddress)) {
|
|
$detectedIP = Get-GuestIPAddress -VmrunPath $VmrunPath `
|
|
-VmxPath $cloneVmxPath -TimeoutSeconds 120
|
|
if ($detectedIP -eq '') {
|
|
throw "Could not detect VM IP address after 120s. Ensure ci-report-ip.service is enabled in the guest template (Linux) or open-vm-tools is running (Windows)."
|
|
}
|
|
$VMIPAddress = $detectedIP
|
|
Write-Host "[Invoke-CIJob] Detected VM IP: $VMIPAddress"
|
|
}
|
|
|
|
# ── Lease IP ──────────────────────────────────────────────────────
|
|
# If two VMs somehow got the same IP (DHCP edge case), fail fast here
|
|
# rather than letting both jobs write to the same WinRM target.
|
|
$leaseFile = Join-Path $leasesDir "$($VMIPAddress.Replace('.', '-')).lease"
|
|
if (Test-Path $leaseFile) {
|
|
$holder = (Get-Content $leaseFile -Raw -ErrorAction SilentlyContinue).Trim()
|
|
throw "IP collision: $VMIPAddress already leased by job '$holder'. Retry or reduce runner capacity."
|
|
}
|
|
[System.IO.File]::WriteAllText($leaseFile, $JobId)
|
|
Write-Host "[Invoke-CIJob] IP $VMIPAddress leased for job $JobId."
|
|
Write-JobEvent -Phase 'phase2-3b.vm-start' -Status 'success' -Data @{ ip = $VMIPAddress; vmx = $cloneVmxPath }
|
|
|
|
} finally {
|
|
# Release lock — next waiting job can now enter clone→start→IP phase
|
|
if ($lockStream) {
|
|
$lockStream.Close()
|
|
$lockStream.Dispose()
|
|
$lockStream = $null
|
|
}
|
|
Remove-Item $lockPath -Force -ErrorAction SilentlyContinue
|
|
Write-Host "[Invoke-CIJob] VM-start lock released."
|
|
}
|
|
$phaseTimings.Add([PSCustomObject]@{ Phase = 'Phase 2-3 - Clone+Start VM'; Sec = [int]((Get-Date) - $phaseStart).TotalSeconds })
|
|
$phaseStart = Get-Date
|
|
# ── Resolve GuestOS from VMX if Auto ────────────────────────────────────
|
|
if ($GuestOS -eq 'Auto' -and $cloneVmxPath) {
|
|
$vmxContent = Get-Content -LiteralPath $cloneVmxPath -Raw -ErrorAction SilentlyContinue
|
|
if ($vmxContent -match 'guestOS\s*=\s*"([^"]+)"') {
|
|
$detectedOS = $Matches[1]
|
|
$GuestOS = if ($detectedOS -like '*ubuntu*' -or $detectedOS -like '*linux*') { 'Linux' } else { 'Windows' }
|
|
Write-Host "[Invoke-CIJob] Auto-detected GuestOS: $GuestOS (VMX guestOS=$detectedOS)"
|
|
} else {
|
|
$GuestOS = 'Windows'
|
|
Write-Host "[Invoke-CIJob] Could not detect GuestOS from VMX — defaulting to Windows"
|
|
}
|
|
}
|
|
# ── Phase 4: Wait for readiness ───────────────────────────────────────
|
|
Write-Host "`n[Phase 4/6] Waiting for VM readiness..."
|
|
Write-JobEvent -Phase 'phase4.wait-ready' -Status 'start' -Data @{ ip = $VMIPAddress }
|
|
$waitParams = @{
|
|
VMPath = $cloneVmxPath
|
|
IPAddress = $VMIPAddress
|
|
VmrunPath = $VmrunPath
|
|
}
|
|
if ($GuestOS -eq 'Linux') {
|
|
$waitParams['Transport'] = 'SSH'
|
|
$waitParams['SshKeyPath'] = $SshKeyPath
|
|
$waitParams['SshUser'] = $SshUser
|
|
}
|
|
& "$scriptDir\Wait-VMReady.ps1" @waitParams
|
|
Write-JobEvent -Phase 'phase4.wait-ready' -Status 'success'
|
|
$phaseTimings.Add([PSCustomObject]@{ Phase = 'Phase 4 - Wait ready'; Sec = [int]((Get-Date) - $phaseStart).TotalSeconds })
|
|
$phaseStart = Get-Date
|
|
|
|
# ── Phase 5: Remote build (copy source + build inside VM) ─────────────
|
|
Write-Host "`n[Phase 5/6] Invoking remote build..."
|
|
Write-JobEvent -Phase 'phase5.build' -Status 'start'
|
|
$remoteBuildParams = @{
|
|
IPAddress = $VMIPAddress
|
|
Credential = $credential
|
|
Configuration = $Configuration
|
|
}
|
|
if ($GuestOS -eq 'Linux') {
|
|
$remoteBuildParams['GuestOS'] = 'Linux'
|
|
$remoteBuildParams['SshKeyPath'] = $SshKeyPath
|
|
$remoteBuildParams['SshUser'] = $SshUser
|
|
$remoteBuildParams['SshKnownHostsFile'] = $SshKnownHostsFile
|
|
$remoteBuildParams.Remove('Credential')
|
|
if ($UseGitClone) {
|
|
# In-guest git clone (requires Gitea reachable + PAT/key configured in guest)
|
|
$remoteBuildParams['CloneUrl'] = $RepoUrl
|
|
$remoteBuildParams['CloneBranch'] = $Branch
|
|
$remoteBuildParams['GiteaCredentialTarget'] = $GiteaCredentialTarget
|
|
if ($Commit -ne '') { $remoteBuildParams['CloneCommit'] = $Commit }
|
|
if ($Submodules) { $remoteBuildParams['CloneSubmodules'] = $true }
|
|
} else {
|
|
# Default: host-cloned source shipped to guest via tar+scp
|
|
$remoteBuildParams['HostSourceDir'] = $hostCloneDir
|
|
}
|
|
} else {
|
|
if ($UseGitClone) {
|
|
# §3.3: Clone in VM — pass URL/branch/commit instead of HostSourceDir
|
|
$remoteBuildParams['CloneUrl'] = $RepoUrl
|
|
$remoteBuildParams['CloneBranch'] = $Branch
|
|
$remoteBuildParams['GiteaCredentialTarget'] = $GiteaCredentialTarget
|
|
if ($Commit -ne '') { $remoteBuildParams['CloneCommit'] = $Commit }
|
|
if ($Submodules) { $remoteBuildParams['CloneSubmodules'] = $true }
|
|
}
|
|
else {
|
|
# Default: Use pre-cloned source from host
|
|
$remoteBuildParams['HostSourceDir'] = $hostCloneDir
|
|
}
|
|
}
|
|
if ($BuildCommand -ne '') { $remoteBuildParams['BuildCommand'] = $BuildCommand }
|
|
if ($GuestArtifactSource -ne '') { $remoteBuildParams['GuestArtifactSource'] = $GuestArtifactSource }
|
|
if ($ExtraGuestEnv.Count -gt 0) { $remoteBuildParams['ExtraGuestEnv'] = $ExtraGuestEnv }
|
|
if ($UseSharedCache) { $remoteBuildParams['UseSharedCache'] = $true }
|
|
& "$scriptDir\Invoke-RemoteBuild.ps1" @remoteBuildParams
|
|
Write-JobEvent -Phase 'phase5.build' -Status 'success'
|
|
$phaseTimings.Add([PSCustomObject]@{ Phase = 'Phase 5 - Remote build'; Sec = [int]((Get-Date) - $phaseStart).TotalSeconds })
|
|
$phaseStart = Get-Date
|
|
|
|
# ── Phase 6: Collect artifacts ────────────────────────────────────────
|
|
Write-Host "`n[Phase 6/6] Collecting artifacts..."
|
|
Write-JobEvent -Phase 'phase6.artifacts' -Status 'start'
|
|
if ($GuestOS -eq 'Linux') {
|
|
& "$scriptDir\Get-BuildArtifacts.ps1" `
|
|
-IPAddress $VMIPAddress `
|
|
-GuestOS 'Linux' `
|
|
-SshKeyPath $SshKeyPath `
|
|
-SshUser $SshUser `
|
|
-SshKnownHostsFile $SshKnownHostsFile `
|
|
-GuestArtifactPath '/opt/ci/output' `
|
|
-HostArtifactDir $jobArtifactDir `
|
|
-IncludeLogs
|
|
} else {
|
|
& "$scriptDir\Get-BuildArtifacts.ps1" `
|
|
-IPAddress $VMIPAddress `
|
|
-Credential $credential `
|
|
-GuestArtifactPath 'C:\CI\output\artifacts.zip' `
|
|
-HostArtifactDir $jobArtifactDir `
|
|
-IncludeLogs
|
|
}
|
|
|
|
$elapsed = (Get-Date) - $startTime
|
|
Write-JobEvent -Phase 'phase6.artifacts' -Status 'success' -Data @{ dir = $jobArtifactDir }
|
|
Write-JobEvent -Phase 'job' -Status 'success' -Data @{ elapsedSec = [int]$elapsed.TotalSeconds }
|
|
$phaseTimings.Add([PSCustomObject]@{ Phase = 'Phase 6 - Artifacts'; Sec = [int]((Get-Date) - $phaseStart).TotalSeconds })
|
|
Write-Host "`n============================================================"
|
|
Write-Host "[Invoke-CIJob] Phase durations:"
|
|
foreach ($pt in $phaseTimings) {
|
|
Write-Host (" {0,-30} {1,5}s" -f $pt.Phase, $pt.Sec)
|
|
}
|
|
Write-Host " {0,-30} {1,5}s" -f 'TOTAL', [int]$elapsed.TotalSeconds
|
|
Write-Host "============================================================"
|
|
Write-Host "[Invoke-CIJob] SUCCESS -- Job $JobId completed in $($elapsed.ToString('mm\:ss'))"
|
|
Write-Host "[Invoke-CIJob] Artifacts: $jobArtifactDir"
|
|
Write-Host "============================================================"
|
|
}
|
|
catch {
|
|
$exitCode = 1
|
|
$elapsed = (Get-Date) - $startTime
|
|
Write-JobEvent -Phase 'job' -Status 'failure' -Data @{
|
|
elapsedSec = [int]$elapsed.TotalSeconds
|
|
error = "$_"
|
|
}
|
|
|
|
# ── Best-effort guest diagnostics (skip if VM was never started) ─────────
|
|
# Only collect when a VM IP is known (Phase 3b completed). Never throws.
|
|
if (-not [string]::IsNullOrWhiteSpace($VMIPAddress)) {
|
|
try {
|
|
$diagDir = Join-Path $jobArtifactDir 'diagnostics'
|
|
$diagParams = @{
|
|
IPAddress = $VMIPAddress
|
|
GuestOS = if ($GuestOS -eq 'Auto') { 'Windows' } else { $GuestOS }
|
|
DestinationDir = $diagDir
|
|
}
|
|
if ($diagParams.GuestOS -eq 'Windows' -and $credential) {
|
|
$diagParams['Credential'] = $credential
|
|
}
|
|
if ($diagParams.GuestOS -eq 'Linux') {
|
|
$diagParams['SshKeyPath'] = $SshKeyPath
|
|
$diagParams['SshUser'] = $SshUser
|
|
}
|
|
Get-GuestDiagnostics @diagParams
|
|
} catch {
|
|
Write-Warning "[Invoke-CIJob] Diagnostics collection skipped: $_"
|
|
}
|
|
}
|
|
|
|
Write-Host "`n============================================================"
|
|
Write-Host "[Invoke-CIJob] FAILURE — Job $JobId failed after $($elapsed.ToString('mm\:ss'))"
|
|
Write-Host "Error: $_"
|
|
Write-Host "============================================================"
|
|
}
|
|
finally {
|
|
# ── Release IP lease (§2.1) ───────────────────────────────────────────
|
|
if ($leaseFile -and (Test-Path $leaseFile)) {
|
|
Remove-Item $leaseFile -Force -ErrorAction SilentlyContinue
|
|
Write-Host "[Cleanup] IP lease released: $VMIPAddress"
|
|
}
|
|
|
|
# ── Always destroy the VM ─────────────────────────────────────────────
|
|
if ($null -ne $cloneVmxPath -and (Test-Path $cloneVmxPath -PathType Leaf)) {
|
|
Write-Host "`n[Cleanup] Destroying ephemeral VM..."
|
|
& "$scriptDir\Remove-BuildVM.ps1" -VMPath $cloneVmxPath -VmrunPath $VmrunPath
|
|
}
|
|
else {
|
|
Write-Host "[Cleanup] No VM to destroy (clone was not created or already gone)."
|
|
}
|
|
# ── Clean up host-side git clone (unless in-VM clone was used) ────────
|
|
# When -UseGitClone, $hostCloneDir is never created
|
|
if (-not $UseGitClone -and (Test-Path $hostCloneDir)) {
|
|
Write-Host "[Cleanup] Removing host source clone ($hostCloneDir)..."
|
|
Remove-Item $hostCloneDir -Recurse -Force -ErrorAction SilentlyContinue
|
|
}
|
|
|
|
# ── Stop transcript ───────────────────────────────────────────────────
|
|
if ($transcriptStarted) {
|
|
try { Stop-Transcript -ErrorAction SilentlyContinue } catch { Write-Debug "[Invoke-CIJob] Stop-Transcript ignored: $_" }
|
|
}
|
|
|
|
# ── Log retention: purge directories older than $LogRetentionDays days ─
|
|
if ($LogRetentionDays -gt 0 -and (Test-Path $LogDir)) {
|
|
$cutoff = (Get-Date).AddDays(-$LogRetentionDays)
|
|
Get-ChildItem -Path $LogDir -Directory -ErrorAction SilentlyContinue |
|
|
Where-Object { $_.LastWriteTime -lt $cutoff } |
|
|
ForEach-Object {
|
|
Remove-Item $_.FullName -Recurse -Force -ErrorAction SilentlyContinue
|
|
Write-Host "[Cleanup] Purged old log dir: $($_.FullName)"
|
|
}
|
|
}
|
|
}
|
|
|
|
exit $exitCode
|