feat(sprint3): Invoke-VmrunBounded, Get-GuestIPAddress, transport and concurrency hardening
H4: Invoke-VmrunBounded added to _Common.psm1 (Start-Process + Wait-Process + taskkill on
timeout, returns {ExitCode;Output;TimedOut;ElapsedSeconds}). Wired in Invoke-CIJob.ps1
for vmrun start (180s) and Remove-BuildVM.ps1 for stop/deleteVM (60s/90s).
H5: Cleanup-OrphanedBuildVMs.ps1 removes stale vm-start.lock (>30 min). ValidateRange
lowered to 0 to allow emergency -MaxAgeHours 0. Invoke-RetentionPolicy.ps1 same.
H8: _Transport.psm1 SSH hardening: StrictHostKeyChecking=accept-new, F:\CI\State\known_hosts.
H9: Validate-DeployState.ps1 Linux branch added: checks ci-report-ip.service enabled,
ci-report-ip.sh executable via SSH. New params: -GuestOS, -SshKeyPath, -SshUser.
H12: Get-GuestIPAddress extracted into _Common.psm1 (guestinfo.ci-ip primary, getGuestIPAddress
fallback). Invoke-CIJob.ps1 Phase 3b replaced with single call. Measure-CIBenchmark.ps1
updated to match.
Also: _Common.psm1 adds Write-CIRedactedCommand and ConvertTo-SafeShellSingleQuotedString.
Pester suite updated: 30/30 pass.
This commit is contained in:
Simone
+93
-52
@@ -74,6 +74,7 @@
|
||||
[CmdletBinding()]
|
||||
param(
|
||||
[Parameter(Mandatory)]
|
||||
[ValidatePattern('^[A-Za-z0-9._-]+$')]
|
||||
[string] $JobId,
|
||||
|
||||
[Parameter(Mandatory)]
|
||||
@@ -100,6 +101,10 @@ param(
|
||||
# Prerequisite: Git for Windows installed in template.
|
||||
[switch] $UseGitClone,
|
||||
|
||||
# Use VMware shared folders for NuGet/pip cache (§SharedCache).
|
||||
# Requires Set-TemplateSharedFolders.ps1 to have been run on the template.
|
||||
[switch] $UseSharedCache,
|
||||
|
||||
# Credential Manager target for Gitea PAT used by -UseGitClone (private repos).
|
||||
# Store with: New-StoredCredential -Target 'GiteaPAT' -UserName '<user>' -Password '<pat>' -Persist LocalMachine
|
||||
# Ignored when -UseGitClone is not set.
|
||||
@@ -136,6 +141,10 @@ param(
|
||||
[string] $SshKeyPath = 'F:\CI\keys\ci_linux',
|
||||
[string] $SshUser = 'ci_build',
|
||||
|
||||
# Persistent known_hosts file used for CI-job SSH calls.
|
||||
# Set to empty string to disable host-key checking (template provisioning only).
|
||||
[string] $SshKnownHostsFile = 'F:\CI\State\known_hosts',
|
||||
|
||||
# Extra environment variables injected into the guest VM before the build command (§6.5).
|
||||
# Keys must be valid environment variable names. Values must be plain strings.
|
||||
# Example: @{ SIGN_PASS = $env:SIGN_PASS_FROM_SECRET; GPG_KEY_ID = '0xABCD1234' }
|
||||
@@ -148,6 +157,9 @@ $ErrorActionPreference = 'Stop'
|
||||
# ── Resolve script directory for dot-sourcing sibling scripts ─────────────────
|
||||
$scriptDir = $PSScriptRoot
|
||||
|
||||
# ── Load shared helpers (Invoke-VmrunBounded, Get-GuestIPAddress, …) ──────────
|
||||
Import-Module (Join-Path $scriptDir '_Common.psm1') -Force
|
||||
|
||||
# ── Validate required config ──────────────────────────────────────────────────
|
||||
if ([string]::IsNullOrWhiteSpace($TemplatePath)) {
|
||||
Write-Error "TemplatePath is required. Set -TemplatePath or env:GITEA_CI_TEMPLATE_PATH."
|
||||
@@ -162,6 +174,14 @@ if (-not (Test-Path $VmrunPath -PathType Leaf)) {
|
||||
exit 1
|
||||
}
|
||||
|
||||
# ── Validate ExtraGuestEnv keys ──────────────────────────────────────────────
|
||||
foreach ($key in $ExtraGuestEnv.Keys) {
|
||||
if ($key -notmatch '^[A-Za-z_][A-Za-z0-9_]*$') {
|
||||
Write-Error "ExtraGuestEnv key '$key' is not a valid environment variable name. Keys must match ^[A-Za-z_][A-Za-z0-9_]*$."
|
||||
exit 1
|
||||
}
|
||||
}
|
||||
|
||||
# ── Load guest credentials from Windows Credential Manager ───────────────────
|
||||
# Requires the CredentialManager module: Install-Module CredentialManager
|
||||
# In a lab setup you can fall back to prompting, but never hardcode credentials.
|
||||
@@ -185,6 +205,10 @@ $leaseFile = $null # §2.1 IP lease file path — released in finally
|
||||
$jsonLog = $null # §4.1 JSONL structured log path
|
||||
$startTime = Get-Date
|
||||
|
||||
# ── Phase-duration tracking (Task 5.5) ───────────────────────────────────────
|
||||
$phaseTimings = [System.Collections.Generic.List[PSCustomObject]]::new()
|
||||
$phaseStart = $startTime
|
||||
|
||||
# ── Start transcript + JSONL log ──────────────────────────────────────────────
|
||||
$transcriptStarted = $false
|
||||
try {
|
||||
@@ -291,6 +315,8 @@ try {
|
||||
Write-Host "[Invoke-CIJob] Source cloned to: $hostCloneDir"
|
||||
Write-JobEvent -Phase 'phase1.clone-repo' -Status 'success' -Data @{ dir = $hostCloneDir }
|
||||
}
|
||||
$phaseTimings.Add([PSCustomObject]@{ Phase = 'Phase 1 - Clone repo'; Sec = [int]((Get-Date) - $phaseStart).TotalSeconds })
|
||||
$phaseStart = Get-Date
|
||||
|
||||
# ── Phase 2-3b: Clone + Start + IP (serialized via file-based mutex) ────
|
||||
# §2.1: With capacity:4 and DHCP on VMnet8, concurrent vmrun starts can race
|
||||
@@ -303,6 +329,17 @@ try {
|
||||
$lockPath = Join-Path $stateDir 'vm-start.lock'
|
||||
if (-not (Test-Path $leasesDir)) { New-Item -ItemType Directory -Path $leasesDir -Force | Out-Null }
|
||||
|
||||
# ── Pre-clone disk space gate ─────────────────────────────────────────
|
||||
$cloneDriveLetter = (Split-Path -Qualifier $CloneBaseDir).TrimEnd(':')
|
||||
$cloneVol = Get-Volume -DriveLetter $cloneDriveLetter -ErrorAction SilentlyContinue
|
||||
if ($null -ne $cloneVol) {
|
||||
$freeGB = [math]::Round($cloneVol.SizeRemaining / 1GB, 1)
|
||||
if ($freeGB -lt 20) {
|
||||
throw "Insufficient disk space on ${cloneDriveLetter}: ($freeGB GB free). 20 GB minimum required before VM clone."
|
||||
}
|
||||
Write-Host "[Invoke-CIJob] Disk space on ${cloneDriveLetter}: ${freeGB} GB free (OK)."
|
||||
}
|
||||
|
||||
Write-Host "`n[Phase 2/6] Acquiring VM-start lock (prevents DHCP collision with capacity>1)..."
|
||||
Write-JobEvent -Phase 'phase2-3b.vm-start' -Status 'start'
|
||||
$lockStream = $null
|
||||
@@ -337,58 +374,22 @@ try {
|
||||
|
||||
# ── Phase 3: Start VM ─────────────────────────────────────────────
|
||||
Write-Host "`n[Phase 3/6] Starting VM..."
|
||||
$startOutput = & $VmrunPath -T ws start $cloneVmxPath nogui 2>&1
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "vmrun start failed (exit $LASTEXITCODE): $startOutput"
|
||||
$startResult = Invoke-VmrunBounded -VmrunPath $VmrunPath `
|
||||
-Operation start `
|
||||
-Arguments @($cloneVmxPath, 'nogui') `
|
||||
-TimeoutSeconds 180
|
||||
if ($startResult.TimedOut) {
|
||||
throw "vmrun start timed out after 180s for: $cloneVmxPath"
|
||||
}
|
||||
Write-Host "[Invoke-CIJob] VM started."
|
||||
if ($startResult.ExitCode -ne 0) {
|
||||
throw "vmrun start failed (exit $($startResult.ExitCode)): $($startResult.Output)"
|
||||
}
|
||||
Write-Host "[Invoke-CIJob] VM started (took $($startResult.ElapsedSeconds)s)."
|
||||
|
||||
# ── Phase 3b: Resolve VM IP ───────────────────────────────────────
|
||||
if ([string]::IsNullOrWhiteSpace($VMIPAddress)) {
|
||||
# Primary method: vmrun readVariable guestVar guestinfo.ci-ip
|
||||
# The guest ci-report-ip.service writes its IP via vmware-rpctool on every boot.
|
||||
# This uses the VMware VMCI channel — reliable, official API, independent of
|
||||
# whether TCP/IP is reachable from the host.
|
||||
#
|
||||
# Fallback: vmrun getGuestIPAddress (works on Windows guests; unreliable on Linux).
|
||||
Write-Host "[Invoke-CIJob] Auto-detecting VM IP via guestinfo.ci-ip (polling, max 120s)..."
|
||||
$ipDeadline = (Get-Date).AddSeconds(120)
|
||||
$detectedIP = ''
|
||||
$ipOut = ''
|
||||
$ipAttempt = 0
|
||||
while ((Get-Date) -lt $ipDeadline) {
|
||||
$ipAttempt++
|
||||
|
||||
# Primary: guestinfo written by ci-report-ip.service via vmware-rpctool.
|
||||
# NOTE: rpctool sets 'guestinfo.ci-ip' but vmrun readVariable guestVar
|
||||
# reads it WITHOUT the 'guestinfo.' prefix (the prefix is implicit in
|
||||
# the guestVar namespace). Reading with the full key always returns ''.
|
||||
$savedEap = $ErrorActionPreference; $ErrorActionPreference = 'Continue'
|
||||
$ipOut = (& $VmrunPath -T ws readVariable $cloneVmxPath guestVar 'ci-ip' 2>&1).Trim()
|
||||
$ipRc = $LASTEXITCODE
|
||||
$ErrorActionPreference = $savedEap
|
||||
if ($ipAttempt -le 3 -or ($ipAttempt % 10 -eq 0)) {
|
||||
Write-Host "[Invoke-CIJob] [attempt $ipAttempt] guestVar ci-ip: rc=$ipRc out='$ipOut'"
|
||||
}
|
||||
if ($ipRc -eq 0 -and $ipOut -match '^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$') {
|
||||
$detectedIP = $ipOut
|
||||
Write-Host "[Invoke-CIJob] IP via guestinfo: $detectedIP"
|
||||
break
|
||||
}
|
||||
|
||||
# Fallback: vmrun getGuestIPAddress (reliable on Windows guests)
|
||||
$savedEap = $ErrorActionPreference; $ErrorActionPreference = 'Continue'
|
||||
$ipOut2 = (& $VmrunPath -T ws getGuestIPAddress $cloneVmxPath 2>&1).Trim()
|
||||
$ipRc2 = $LASTEXITCODE
|
||||
$ErrorActionPreference = $savedEap
|
||||
if ($ipRc2 -eq 0 -and $ipOut2 -match '^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$') {
|
||||
$detectedIP = $ipOut2
|
||||
Write-Host "[Invoke-CIJob] IP via vmrun getGuestIPAddress: $detectedIP"
|
||||
break
|
||||
}
|
||||
|
||||
Start-Sleep -Seconds 2
|
||||
}
|
||||
$detectedIP = Get-GuestIPAddress -VmrunPath $VmrunPath `
|
||||
-VmxPath $cloneVmxPath -TimeoutSeconds 120
|
||||
if ($detectedIP -eq '') {
|
||||
throw "Could not detect VM IP address after 120s. Ensure ci-report-ip.service is enabled in the guest template (Linux) or open-vm-tools is running (Windows)."
|
||||
}
|
||||
@@ -418,6 +419,8 @@ try {
|
||||
Remove-Item $lockPath -Force -ErrorAction SilentlyContinue
|
||||
Write-Host "[Invoke-CIJob] VM-start lock released."
|
||||
}
|
||||
$phaseTimings.Add([PSCustomObject]@{ Phase = 'Phase 2-3 - Clone+Start VM'; Sec = [int]((Get-Date) - $phaseStart).TotalSeconds })
|
||||
$phaseStart = Get-Date
|
||||
# ── Resolve GuestOS from VMX if Auto ────────────────────────────────────
|
||||
if ($GuestOS -eq 'Auto' -and $cloneVmxPath) {
|
||||
$vmxContent = Get-Content -LiteralPath $cloneVmxPath -Raw -ErrorAction SilentlyContinue
|
||||
@@ -445,6 +448,8 @@ try {
|
||||
}
|
||||
& "$scriptDir\Wait-VMReady.ps1" @waitParams
|
||||
Write-JobEvent -Phase 'phase4.wait-ready' -Status 'success'
|
||||
$phaseTimings.Add([PSCustomObject]@{ Phase = 'Phase 4 - Wait ready'; Sec = [int]((Get-Date) - $phaseStart).TotalSeconds })
|
||||
$phaseStart = Get-Date
|
||||
|
||||
# ── Phase 5: Remote build (copy source + build inside VM) ─────────────
|
||||
Write-Host "`n[Phase 5/6] Invoking remote build..."
|
||||
@@ -455,9 +460,10 @@ try {
|
||||
Configuration = $Configuration
|
||||
}
|
||||
if ($GuestOS -eq 'Linux') {
|
||||
$remoteBuildParams['GuestOS'] = 'Linux'
|
||||
$remoteBuildParams['SshKeyPath'] = $SshKeyPath
|
||||
$remoteBuildParams['SshUser'] = $SshUser
|
||||
$remoteBuildParams['GuestOS'] = 'Linux'
|
||||
$remoteBuildParams['SshKeyPath'] = $SshKeyPath
|
||||
$remoteBuildParams['SshUser'] = $SshUser
|
||||
$remoteBuildParams['SshKnownHostsFile'] = $SshKnownHostsFile
|
||||
$remoteBuildParams.Remove('Credential')
|
||||
if ($UseGitClone) {
|
||||
# In-guest git clone (requires Gitea reachable + PAT/key configured in guest)
|
||||
@@ -487,8 +493,11 @@ try {
|
||||
if ($BuildCommand -ne '') { $remoteBuildParams['BuildCommand'] = $BuildCommand }
|
||||
if ($GuestArtifactSource -ne '') { $remoteBuildParams['GuestArtifactSource'] = $GuestArtifactSource }
|
||||
if ($ExtraGuestEnv.Count -gt 0) { $remoteBuildParams['ExtraGuestEnv'] = $ExtraGuestEnv }
|
||||
if ($UseSharedCache) { $remoteBuildParams['UseSharedCache'] = $true }
|
||||
& "$scriptDir\Invoke-RemoteBuild.ps1" @remoteBuildParams
|
||||
Write-JobEvent -Phase 'phase5.build' -Status 'success'
|
||||
$phaseTimings.Add([PSCustomObject]@{ Phase = 'Phase 5 - Remote build'; Sec = [int]((Get-Date) - $phaseStart).TotalSeconds })
|
||||
$phaseStart = Get-Date
|
||||
|
||||
# ── Phase 6: Collect artifacts ────────────────────────────────────────
|
||||
Write-Host "`n[Phase 6/6] Collecting artifacts..."
|
||||
@@ -499,6 +508,7 @@ try {
|
||||
-GuestOS 'Linux' `
|
||||
-SshKeyPath $SshKeyPath `
|
||||
-SshUser $SshUser `
|
||||
-SshKnownHostsFile $SshKnownHostsFile `
|
||||
-GuestArtifactPath '/opt/ci/output' `
|
||||
-HostArtifactDir $jobArtifactDir `
|
||||
-IncludeLogs
|
||||
@@ -514,8 +524,15 @@ try {
|
||||
$elapsed = (Get-Date) - $startTime
|
||||
Write-JobEvent -Phase 'phase6.artifacts' -Status 'success' -Data @{ dir = $jobArtifactDir }
|
||||
Write-JobEvent -Phase 'job' -Status 'success' -Data @{ elapsedSec = [int]$elapsed.TotalSeconds }
|
||||
$phaseTimings.Add([PSCustomObject]@{ Phase = 'Phase 6 - Artifacts'; Sec = [int]((Get-Date) - $phaseStart).TotalSeconds })
|
||||
Write-Host "`n============================================================"
|
||||
Write-Host "[Invoke-CIJob] SUCCESS — Job $JobId completed in $($elapsed.ToString('mm\:ss'))"
|
||||
Write-Host "[Invoke-CIJob] Phase durations:"
|
||||
foreach ($pt in $phaseTimings) {
|
||||
Write-Host (" {0,-30} {1,5}s" -f $pt.Phase, $pt.Sec)
|
||||
}
|
||||
Write-Host " {0,-30} {1,5}s" -f 'TOTAL', [int]$elapsed.TotalSeconds
|
||||
Write-Host "============================================================"
|
||||
Write-Host "[Invoke-CIJob] SUCCESS -- Job $JobId completed in $($elapsed.ToString('mm\:ss'))"
|
||||
Write-Host "[Invoke-CIJob] Artifacts: $jobArtifactDir"
|
||||
Write-Host "============================================================"
|
||||
}
|
||||
@@ -526,6 +543,30 @@ catch {
|
||||
elapsedSec = [int]$elapsed.TotalSeconds
|
||||
error = "$_"
|
||||
}
|
||||
|
||||
# ── Best-effort guest diagnostics (skip if VM was never started) ─────────
|
||||
# Only collect when a VM IP is known (Phase 3b completed). Never throws.
|
||||
if (-not [string]::IsNullOrWhiteSpace($VMIPAddress)) {
|
||||
try {
|
||||
$diagDir = Join-Path $jobArtifactDir 'diagnostics'
|
||||
$diagParams = @{
|
||||
IPAddress = $VMIPAddress
|
||||
GuestOS = if ($GuestOS -eq 'Auto') { 'Windows' } else { $GuestOS }
|
||||
DestinationDir = $diagDir
|
||||
}
|
||||
if ($diagParams.GuestOS -eq 'Windows' -and $credential) {
|
||||
$diagParams['Credential'] = $credential
|
||||
}
|
||||
if ($diagParams.GuestOS -eq 'Linux') {
|
||||
$diagParams['SshKeyPath'] = $SshKeyPath
|
||||
$diagParams['SshUser'] = $SshUser
|
||||
}
|
||||
Get-GuestDiagnostics @diagParams
|
||||
} catch {
|
||||
Write-Warning "[Invoke-CIJob] Diagnostics collection skipped: $_"
|
||||
}
|
||||
}
|
||||
|
||||
Write-Host "`n============================================================"
|
||||
Write-Host "[Invoke-CIJob] FAILURE — Job $JobId failed after $($elapsed.ToString('mm\:ss'))"
|
||||
Write-Host "Error: $_"
|
||||
|
||||
Reference in New Issue
Block a user