feat(sprint3): Invoke-VmrunBounded, Get-GuestIPAddress, transport and concurrency hardening
H4: Invoke-VmrunBounded added to _Common.psm1 (Start-Process + Wait-Process + taskkill on
timeout, returns {ExitCode;Output;TimedOut;ElapsedSeconds}). Wired in Invoke-CIJob.ps1
for vmrun start (180s) and Remove-BuildVM.ps1 for stop/deleteVM (60s/90s).
H5: Cleanup-OrphanedBuildVMs.ps1 removes stale vm-start.lock (>30 min). ValidateRange
lowered to 0 to allow emergency -MaxAgeHours 0. Invoke-RetentionPolicy.ps1 same.
H8: _Transport.psm1 SSH hardening: StrictHostKeyChecking=accept-new, F:\CI\State\known_hosts.
H9: Validate-DeployState.ps1 Linux branch added: checks ci-report-ip.service enabled,
ci-report-ip.sh executable via SSH. New params: -GuestOS, -SshKeyPath, -SshUser.
H12: Get-GuestIPAddress extracted into _Common.psm1 (guestinfo.ci-ip primary, getGuestIPAddress
fallback). Invoke-CIJob.ps1 Phase 3b replaced with single call. Measure-CIBenchmark.ps1
updated to match.
Also: _Common.psm1 adds Write-CIRedactedCommand and ConvertTo-SafeShellSingleQuotedString.
Pester suite updated: 30/30 pass.
This commit is contained in:
Simone
@@ -37,7 +37,7 @@
|
||||
param(
|
||||
[string] $CloneBaseDir = 'F:\CI\BuildVMs',
|
||||
|
||||
[ValidateRange(1, 168)]
|
||||
[ValidateRange(0, 168)]
|
||||
[int] $MaxAgeHours = 4,
|
||||
|
||||
[string] $VmrunPath = 'C:\Program Files (x86)\VMware\VMware Workstation\vmrun.exe'
|
||||
@@ -102,3 +102,17 @@ foreach ($dir in $orphans) {
|
||||
}
|
||||
|
||||
Write-Host "[Cleanup] Done."
|
||||
|
||||
# ── Stale vm-start.lock cleanup ─────────────────────────────────────────────
|
||||
# The lock is released by Invoke-CIJob.ps1 normally. On host crash it persists
|
||||
# and causes a confusing 10-minute retry loop on the next job. Remove if older
|
||||
# than 30 minutes.
|
||||
$lockFile = 'F:\CI\State\vm-start.lock'
|
||||
$lockCutoff = (Get-Date).AddMinutes(-30)
|
||||
if ((Test-Path $lockFile) -and (Get-Item $lockFile).LastWriteTime -lt $lockCutoff) {
|
||||
if ($PSCmdlet.ShouldProcess($lockFile, 'Remove stale vm-start.lock')) {
|
||||
$ageMin = [int]((Get-Date) - (Get-Item $lockFile).LastWriteTime).TotalMinutes
|
||||
Remove-Item $lockFile -Force -ErrorAction SilentlyContinue
|
||||
Write-Host "[Cleanup] Removed stale vm-start.lock (age: ${ageMin} min)"
|
||||
}
|
||||
}
|
||||
|
||||
+90
-49
@@ -74,6 +74,7 @@
|
||||
[CmdletBinding()]
|
||||
param(
|
||||
[Parameter(Mandatory)]
|
||||
[ValidatePattern('^[A-Za-z0-9._-]+$')]
|
||||
[string] $JobId,
|
||||
|
||||
[Parameter(Mandatory)]
|
||||
@@ -100,6 +101,10 @@ param(
|
||||
# Prerequisite: Git for Windows installed in template.
|
||||
[switch] $UseGitClone,
|
||||
|
||||
# Use VMware shared folders for NuGet/pip cache (§SharedCache).
|
||||
# Requires Set-TemplateSharedFolders.ps1 to have been run on the template.
|
||||
[switch] $UseSharedCache,
|
||||
|
||||
# Credential Manager target for Gitea PAT used by -UseGitClone (private repos).
|
||||
# Store with: New-StoredCredential -Target 'GiteaPAT' -UserName '<user>' -Password '<pat>' -Persist LocalMachine
|
||||
# Ignored when -UseGitClone is not set.
|
||||
@@ -136,6 +141,10 @@ param(
|
||||
[string] $SshKeyPath = 'F:\CI\keys\ci_linux',
|
||||
[string] $SshUser = 'ci_build',
|
||||
|
||||
# Persistent known_hosts file used for CI-job SSH calls.
|
||||
# Set to empty string to disable host-key checking (template provisioning only).
|
||||
[string] $SshKnownHostsFile = 'F:\CI\State\known_hosts',
|
||||
|
||||
# Extra environment variables injected into the guest VM before the build command (§6.5).
|
||||
# Keys must be valid environment variable names. Values must be plain strings.
|
||||
# Example: @{ SIGN_PASS = $env:SIGN_PASS_FROM_SECRET; GPG_KEY_ID = '0xABCD1234' }
|
||||
@@ -148,6 +157,9 @@ $ErrorActionPreference = 'Stop'
|
||||
# ── Resolve script directory for dot-sourcing sibling scripts ─────────────────
|
||||
$scriptDir = $PSScriptRoot
|
||||
|
||||
# ── Load shared helpers (Invoke-VmrunBounded, Get-GuestIPAddress, …) ──────────
|
||||
Import-Module (Join-Path $scriptDir '_Common.psm1') -Force
|
||||
|
||||
# ── Validate required config ──────────────────────────────────────────────────
|
||||
if ([string]::IsNullOrWhiteSpace($TemplatePath)) {
|
||||
Write-Error "TemplatePath is required. Set -TemplatePath or env:GITEA_CI_TEMPLATE_PATH."
|
||||
@@ -162,6 +174,14 @@ if (-not (Test-Path $VmrunPath -PathType Leaf)) {
|
||||
exit 1
|
||||
}
|
||||
|
||||
# ── Validate ExtraGuestEnv keys ──────────────────────────────────────────────
|
||||
foreach ($key in $ExtraGuestEnv.Keys) {
|
||||
if ($key -notmatch '^[A-Za-z_][A-Za-z0-9_]*$') {
|
||||
Write-Error "ExtraGuestEnv key '$key' is not a valid environment variable name. Keys must match ^[A-Za-z_][A-Za-z0-9_]*$."
|
||||
exit 1
|
||||
}
|
||||
}
|
||||
|
||||
# ── Load guest credentials from Windows Credential Manager ───────────────────
|
||||
# Requires the CredentialManager module: Install-Module CredentialManager
|
||||
# In a lab setup you can fall back to prompting, but never hardcode credentials.
|
||||
@@ -185,6 +205,10 @@ $leaseFile = $null # §2.1 IP lease file path — released in finally
|
||||
$jsonLog = $null # §4.1 JSONL structured log path
|
||||
$startTime = Get-Date
|
||||
|
||||
# ── Phase-duration tracking (Task 5.5) ───────────────────────────────────────
|
||||
$phaseTimings = [System.Collections.Generic.List[PSCustomObject]]::new()
|
||||
$phaseStart = $startTime
|
||||
|
||||
# ── Start transcript + JSONL log ──────────────────────────────────────────────
|
||||
$transcriptStarted = $false
|
||||
try {
|
||||
@@ -291,6 +315,8 @@ try {
|
||||
Write-Host "[Invoke-CIJob] Source cloned to: $hostCloneDir"
|
||||
Write-JobEvent -Phase 'phase1.clone-repo' -Status 'success' -Data @{ dir = $hostCloneDir }
|
||||
}
|
||||
$phaseTimings.Add([PSCustomObject]@{ Phase = 'Phase 1 - Clone repo'; Sec = [int]((Get-Date) - $phaseStart).TotalSeconds })
|
||||
$phaseStart = Get-Date
|
||||
|
||||
# ── Phase 2-3b: Clone + Start + IP (serialized via file-based mutex) ────
|
||||
# §2.1: With capacity:4 and DHCP on VMnet8, concurrent vmrun starts can race
|
||||
@@ -303,6 +329,17 @@ try {
|
||||
$lockPath = Join-Path $stateDir 'vm-start.lock'
|
||||
if (-not (Test-Path $leasesDir)) { New-Item -ItemType Directory -Path $leasesDir -Force | Out-Null }
|
||||
|
||||
# ── Pre-clone disk space gate ─────────────────────────────────────────
|
||||
$cloneDriveLetter = (Split-Path -Qualifier $CloneBaseDir).TrimEnd(':')
|
||||
$cloneVol = Get-Volume -DriveLetter $cloneDriveLetter -ErrorAction SilentlyContinue
|
||||
if ($null -ne $cloneVol) {
|
||||
$freeGB = [math]::Round($cloneVol.SizeRemaining / 1GB, 1)
|
||||
if ($freeGB -lt 20) {
|
||||
throw "Insufficient disk space on ${cloneDriveLetter}: ($freeGB GB free). 20 GB minimum required before VM clone."
|
||||
}
|
||||
Write-Host "[Invoke-CIJob] Disk space on ${cloneDriveLetter}: ${freeGB} GB free (OK)."
|
||||
}
|
||||
|
||||
Write-Host "`n[Phase 2/6] Acquiring VM-start lock (prevents DHCP collision with capacity>1)..."
|
||||
Write-JobEvent -Phase 'phase2-3b.vm-start' -Status 'start'
|
||||
$lockStream = $null
|
||||
@@ -337,58 +374,22 @@ try {
|
||||
|
||||
# ── Phase 3: Start VM ─────────────────────────────────────────────
|
||||
Write-Host "`n[Phase 3/6] Starting VM..."
|
||||
$startOutput = & $VmrunPath -T ws start $cloneVmxPath nogui 2>&1
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "vmrun start failed (exit $LASTEXITCODE): $startOutput"
|
||||
$startResult = Invoke-VmrunBounded -VmrunPath $VmrunPath `
|
||||
-Operation start `
|
||||
-Arguments @($cloneVmxPath, 'nogui') `
|
||||
-TimeoutSeconds 180
|
||||
if ($startResult.TimedOut) {
|
||||
throw "vmrun start timed out after 180s for: $cloneVmxPath"
|
||||
}
|
||||
Write-Host "[Invoke-CIJob] VM started."
|
||||
if ($startResult.ExitCode -ne 0) {
|
||||
throw "vmrun start failed (exit $($startResult.ExitCode)): $($startResult.Output)"
|
||||
}
|
||||
Write-Host "[Invoke-CIJob] VM started (took $($startResult.ElapsedSeconds)s)."
|
||||
|
||||
# ── Phase 3b: Resolve VM IP ───────────────────────────────────────
|
||||
if ([string]::IsNullOrWhiteSpace($VMIPAddress)) {
|
||||
# Primary method: vmrun readVariable guestVar guestinfo.ci-ip
|
||||
# The guest ci-report-ip.service writes its IP via vmware-rpctool on every boot.
|
||||
# This uses the VMware VMCI channel — reliable, official API, independent of
|
||||
# whether TCP/IP is reachable from the host.
|
||||
#
|
||||
# Fallback: vmrun getGuestIPAddress (works on Windows guests; unreliable on Linux).
|
||||
Write-Host "[Invoke-CIJob] Auto-detecting VM IP via guestinfo.ci-ip (polling, max 120s)..."
|
||||
$ipDeadline = (Get-Date).AddSeconds(120)
|
||||
$detectedIP = ''
|
||||
$ipOut = ''
|
||||
$ipAttempt = 0
|
||||
while ((Get-Date) -lt $ipDeadline) {
|
||||
$ipAttempt++
|
||||
|
||||
# Primary: guestinfo written by ci-report-ip.service via vmware-rpctool.
|
||||
# NOTE: rpctool sets 'guestinfo.ci-ip' but vmrun readVariable guestVar
|
||||
# reads it WITHOUT the 'guestinfo.' prefix (the prefix is implicit in
|
||||
# the guestVar namespace). Reading with the full key always returns ''.
|
||||
$savedEap = $ErrorActionPreference; $ErrorActionPreference = 'Continue'
|
||||
$ipOut = (& $VmrunPath -T ws readVariable $cloneVmxPath guestVar 'ci-ip' 2>&1).Trim()
|
||||
$ipRc = $LASTEXITCODE
|
||||
$ErrorActionPreference = $savedEap
|
||||
if ($ipAttempt -le 3 -or ($ipAttempt % 10 -eq 0)) {
|
||||
Write-Host "[Invoke-CIJob] [attempt $ipAttempt] guestVar ci-ip: rc=$ipRc out='$ipOut'"
|
||||
}
|
||||
if ($ipRc -eq 0 -and $ipOut -match '^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$') {
|
||||
$detectedIP = $ipOut
|
||||
Write-Host "[Invoke-CIJob] IP via guestinfo: $detectedIP"
|
||||
break
|
||||
}
|
||||
|
||||
# Fallback: vmrun getGuestIPAddress (reliable on Windows guests)
|
||||
$savedEap = $ErrorActionPreference; $ErrorActionPreference = 'Continue'
|
||||
$ipOut2 = (& $VmrunPath -T ws getGuestIPAddress $cloneVmxPath 2>&1).Trim()
|
||||
$ipRc2 = $LASTEXITCODE
|
||||
$ErrorActionPreference = $savedEap
|
||||
if ($ipRc2 -eq 0 -and $ipOut2 -match '^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$') {
|
||||
$detectedIP = $ipOut2
|
||||
Write-Host "[Invoke-CIJob] IP via vmrun getGuestIPAddress: $detectedIP"
|
||||
break
|
||||
}
|
||||
|
||||
Start-Sleep -Seconds 2
|
||||
}
|
||||
$detectedIP = Get-GuestIPAddress -VmrunPath $VmrunPath `
|
||||
-VmxPath $cloneVmxPath -TimeoutSeconds 120
|
||||
if ($detectedIP -eq '') {
|
||||
throw "Could not detect VM IP address after 120s. Ensure ci-report-ip.service is enabled in the guest template (Linux) or open-vm-tools is running (Windows)."
|
||||
}
|
||||
@@ -418,6 +419,8 @@ try {
|
||||
Remove-Item $lockPath -Force -ErrorAction SilentlyContinue
|
||||
Write-Host "[Invoke-CIJob] VM-start lock released."
|
||||
}
|
||||
$phaseTimings.Add([PSCustomObject]@{ Phase = 'Phase 2-3 - Clone+Start VM'; Sec = [int]((Get-Date) - $phaseStart).TotalSeconds })
|
||||
$phaseStart = Get-Date
|
||||
# ── Resolve GuestOS from VMX if Auto ────────────────────────────────────
|
||||
if ($GuestOS -eq 'Auto' -and $cloneVmxPath) {
|
||||
$vmxContent = Get-Content -LiteralPath $cloneVmxPath -Raw -ErrorAction SilentlyContinue
|
||||
@@ -445,6 +448,8 @@ try {
|
||||
}
|
||||
& "$scriptDir\Wait-VMReady.ps1" @waitParams
|
||||
Write-JobEvent -Phase 'phase4.wait-ready' -Status 'success'
|
||||
$phaseTimings.Add([PSCustomObject]@{ Phase = 'Phase 4 - Wait ready'; Sec = [int]((Get-Date) - $phaseStart).TotalSeconds })
|
||||
$phaseStart = Get-Date
|
||||
|
||||
# ── Phase 5: Remote build (copy source + build inside VM) ─────────────
|
||||
Write-Host "`n[Phase 5/6] Invoking remote build..."
|
||||
@@ -458,6 +463,7 @@ try {
|
||||
$remoteBuildParams['GuestOS'] = 'Linux'
|
||||
$remoteBuildParams['SshKeyPath'] = $SshKeyPath
|
||||
$remoteBuildParams['SshUser'] = $SshUser
|
||||
$remoteBuildParams['SshKnownHostsFile'] = $SshKnownHostsFile
|
||||
$remoteBuildParams.Remove('Credential')
|
||||
if ($UseGitClone) {
|
||||
# In-guest git clone (requires Gitea reachable + PAT/key configured in guest)
|
||||
@@ -487,8 +493,11 @@ try {
|
||||
if ($BuildCommand -ne '') { $remoteBuildParams['BuildCommand'] = $BuildCommand }
|
||||
if ($GuestArtifactSource -ne '') { $remoteBuildParams['GuestArtifactSource'] = $GuestArtifactSource }
|
||||
if ($ExtraGuestEnv.Count -gt 0) { $remoteBuildParams['ExtraGuestEnv'] = $ExtraGuestEnv }
|
||||
if ($UseSharedCache) { $remoteBuildParams['UseSharedCache'] = $true }
|
||||
& "$scriptDir\Invoke-RemoteBuild.ps1" @remoteBuildParams
|
||||
Write-JobEvent -Phase 'phase5.build' -Status 'success'
|
||||
$phaseTimings.Add([PSCustomObject]@{ Phase = 'Phase 5 - Remote build'; Sec = [int]((Get-Date) - $phaseStart).TotalSeconds })
|
||||
$phaseStart = Get-Date
|
||||
|
||||
# ── Phase 6: Collect artifacts ────────────────────────────────────────
|
||||
Write-Host "`n[Phase 6/6] Collecting artifacts..."
|
||||
@@ -499,6 +508,7 @@ try {
|
||||
-GuestOS 'Linux' `
|
||||
-SshKeyPath $SshKeyPath `
|
||||
-SshUser $SshUser `
|
||||
-SshKnownHostsFile $SshKnownHostsFile `
|
||||
-GuestArtifactPath '/opt/ci/output' `
|
||||
-HostArtifactDir $jobArtifactDir `
|
||||
-IncludeLogs
|
||||
@@ -514,8 +524,15 @@ try {
|
||||
$elapsed = (Get-Date) - $startTime
|
||||
Write-JobEvent -Phase 'phase6.artifacts' -Status 'success' -Data @{ dir = $jobArtifactDir }
|
||||
Write-JobEvent -Phase 'job' -Status 'success' -Data @{ elapsedSec = [int]$elapsed.TotalSeconds }
|
||||
$phaseTimings.Add([PSCustomObject]@{ Phase = 'Phase 6 - Artifacts'; Sec = [int]((Get-Date) - $phaseStart).TotalSeconds })
|
||||
Write-Host "`n============================================================"
|
||||
Write-Host "[Invoke-CIJob] SUCCESS — Job $JobId completed in $($elapsed.ToString('mm\:ss'))"
|
||||
Write-Host "[Invoke-CIJob] Phase durations:"
|
||||
foreach ($pt in $phaseTimings) {
|
||||
Write-Host (" {0,-30} {1,5}s" -f $pt.Phase, $pt.Sec)
|
||||
}
|
||||
Write-Host " {0,-30} {1,5}s" -f 'TOTAL', [int]$elapsed.TotalSeconds
|
||||
Write-Host "============================================================"
|
||||
Write-Host "[Invoke-CIJob] SUCCESS -- Job $JobId completed in $($elapsed.ToString('mm\:ss'))"
|
||||
Write-Host "[Invoke-CIJob] Artifacts: $jobArtifactDir"
|
||||
Write-Host "============================================================"
|
||||
}
|
||||
@@ -526,6 +543,30 @@ catch {
|
||||
elapsedSec = [int]$elapsed.TotalSeconds
|
||||
error = "$_"
|
||||
}
|
||||
|
||||
# ── Best-effort guest diagnostics (skip if VM was never started) ─────────
|
||||
# Only collect when a VM IP is known (Phase 3b completed). Never throws.
|
||||
if (-not [string]::IsNullOrWhiteSpace($VMIPAddress)) {
|
||||
try {
|
||||
$diagDir = Join-Path $jobArtifactDir 'diagnostics'
|
||||
$diagParams = @{
|
||||
IPAddress = $VMIPAddress
|
||||
GuestOS = if ($GuestOS -eq 'Auto') { 'Windows' } else { $GuestOS }
|
||||
DestinationDir = $diagDir
|
||||
}
|
||||
if ($diagParams.GuestOS -eq 'Windows' -and $credential) {
|
||||
$diagParams['Credential'] = $credential
|
||||
}
|
||||
if ($diagParams.GuestOS -eq 'Linux') {
|
||||
$diagParams['SshKeyPath'] = $SshKeyPath
|
||||
$diagParams['SshUser'] = $SshUser
|
||||
}
|
||||
Get-GuestDiagnostics @diagParams
|
||||
} catch {
|
||||
Write-Warning "[Invoke-CIJob] Diagnostics collection skipped: $_"
|
||||
}
|
||||
}
|
||||
|
||||
Write-Host "`n============================================================"
|
||||
Write-Host "[Invoke-CIJob] FAILURE — Job $JobId failed after $($elapsed.ToString('mm\:ss'))"
|
||||
Write-Host "Error: $_"
|
||||
|
||||
@@ -130,23 +130,11 @@ for ($i = 1; $i -le $Iterations; $i++) {
|
||||
$t.start = [math]::Round($sw.Elapsed.TotalSeconds, 2)
|
||||
Write-Host "[bench] start: $($t.start)s"
|
||||
|
||||
# ── Phase: IP acquire (poll getGuestIPAddress) ────────────────────────
|
||||
# ── Phase: IP acquire (primary: guestinfo.ci-ip, fallback: getGuestIPAddress) ──
|
||||
Write-Host "[bench] Waiting for guest IP..."
|
||||
$sw.Restart()
|
||||
$deadline = (Get-Date).AddSeconds($TimeoutSeconds)
|
||||
$vmIP = $null
|
||||
while ((Get-Date) -lt $deadline) {
|
||||
$ipResult = Invoke-Vmrun -VmrunPath $VmrunPath -Operation getGuestIPAddress `
|
||||
-Arguments @($cloneVmx)
|
||||
if ($ipResult.ExitCode -eq 0) {
|
||||
$candidate = "$($ipResult.Output)".Trim()
|
||||
if ($candidate -match '^\d+\.\d+\.\d+\.\d+$') {
|
||||
$vmIP = $candidate
|
||||
break
|
||||
}
|
||||
}
|
||||
Start-Sleep -Seconds 3
|
||||
}
|
||||
$vmIP = Get-GuestIPAddress -VmrunPath $VmrunPath -VmxPath $cloneVmx `
|
||||
-TimeoutSeconds $TimeoutSeconds
|
||||
if (-not $vmIP) { throw "Timed out waiting for guest IP after $TimeoutSeconds s" }
|
||||
$t.ip = [math]::Round($sw.Elapsed.TotalSeconds, 2)
|
||||
$t.vmIP = $vmIP
|
||||
|
||||
+23
-16
@@ -39,6 +39,8 @@ param(
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = 'Continue' # Don't stop on errors — always try next step
|
||||
|
||||
Import-Module (Join-Path $PSScriptRoot '_Common.psm1') -Force
|
||||
|
||||
$cloneDir = Split-Path $VMPath -Parent
|
||||
|
||||
Write-Host "[Remove-BuildVM] Destroying VM: $VMPath"
|
||||
@@ -56,24 +58,23 @@ if (-not (Test-Path $VMPath -PathType Leaf)) {
|
||||
if (-not $PSCmdlet.ShouldProcess($VMPath, 'Stop and delete build VM')) { return }
|
||||
|
||||
# ── Step 2: Graceful stop ─────────────────────────────────────────────────────
|
||||
Write-Host "[Remove-BuildVM] Sending soft stop..."
|
||||
& $VmrunPath -T ws stop $VMPath soft 2>&1 | Out-Null
|
||||
|
||||
$deadline = (Get-Date).AddSeconds($GracefulTimeoutSeconds)
|
||||
while ((Get-Date) -lt $deadline) {
|
||||
Start-Sleep -Seconds 2
|
||||
$stateOutput = & $VmrunPath -T ws getState $VMPath 2>&1
|
||||
if ($stateOutput -notmatch 'running') {
|
||||
Write-Host "[Remove-BuildVM] Sending soft stop (timeout: ${GracefulTimeoutSeconds}s)..."
|
||||
$stopSoft = Invoke-VmrunBounded -VmrunPath $VmrunPath `
|
||||
-Operation stop -Arguments @($VMPath, 'soft') -TimeoutSeconds $GracefulTimeoutSeconds
|
||||
if ($stopSoft.TimedOut) {
|
||||
Write-Warning "[Remove-BuildVM] Soft stop timed out after ${GracefulTimeoutSeconds}s."
|
||||
} elseif ($stopSoft.ExitCode -eq 0) {
|
||||
Write-Host "[Remove-BuildVM] VM stopped gracefully."
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
# ── Step 3: Force stop if still running ──────────────────────────────────────
|
||||
$stateOutput = & $VmrunPath -T ws getState $VMPath 2>&1
|
||||
if ($stateOutput -match 'running') {
|
||||
Write-Host "[Remove-BuildVM] Graceful stop timed out — forcing hard stop..."
|
||||
& $VmrunPath -T ws stop $VMPath hard 2>&1 | Out-Null
|
||||
if ($stopSoft.TimedOut -or $stopSoft.ExitCode -ne 0) {
|
||||
Write-Host "[Remove-BuildVM] Graceful stop did not succeed — forcing hard stop..."
|
||||
$stopHard = Invoke-VmrunBounded -VmrunPath $VmrunPath `
|
||||
-Operation stop -Arguments @($VMPath, 'hard') -TimeoutSeconds 60
|
||||
if ($stopHard.TimedOut) {
|
||||
Write-Warning "[Remove-BuildVM] Hard stop timed out after 60s."
|
||||
}
|
||||
}
|
||||
|
||||
# Poll vmrun list until VMX is gone — vmware-vmx.exe may hold file locks briefly
|
||||
@@ -98,8 +99,14 @@ foreach ($delay in @(0, 3, 6)) {
|
||||
Write-Host "[Remove-BuildVM] Retrying deleteVM in ${delay}s..."
|
||||
Start-Sleep -Seconds $delay
|
||||
}
|
||||
$deleteOutput = & $VmrunPath -T ws deleteVM $VMPath 2>&1
|
||||
$deleteExit = $LASTEXITCODE
|
||||
$delResult = Invoke-VmrunBounded -VmrunPath $VmrunPath `
|
||||
-Operation deleteVM -Arguments @($VMPath) -TimeoutSeconds 90
|
||||
$deleteOutput = $delResult.Output
|
||||
$deleteExit = $delResult.ExitCode
|
||||
if ($delResult.TimedOut) {
|
||||
Write-Warning "[Remove-BuildVM] deleteVM timed out after 90s."
|
||||
$deleteExit = -1
|
||||
}
|
||||
if ($deleteExit -eq 0) { break }
|
||||
}
|
||||
|
||||
|
||||
+314
-1
@@ -104,4 +104,317 @@ function Invoke-Vmrun {
|
||||
}
|
||||
}
|
||||
|
||||
Export-ModuleMember -Function New-CISessionOption, Resolve-VmrunPath, Invoke-Vmrun
|
||||
function Invoke-VmrunBounded {
|
||||
<#
|
||||
.SYNOPSIS
|
||||
Runs a vmrun -T ws <Operation> command with a hard wall-clock timeout.
|
||||
.DESCRIPTION
|
||||
Uses [System.Diagnostics.Process] directly (not Start-Process -PassThru) because
|
||||
Start-Process in PowerShell 5.1 does not reliably populate Process.ExitCode after
|
||||
WaitForExit(ms) when output is redirected. ReadToEndAsync() is used for stdout and
|
||||
stderr to prevent deadlock if output buffers fill up before the process exits.
|
||||
|
||||
Use for lifecycle operations: start (180s), stop (60s), deleteVM (90s).
|
||||
Keep Invoke-Vmrun for fast queries: list, readVariable, getGuestIPAddress, getState.
|
||||
.PARAMETER VmrunPath
|
||||
Full path to vmrun.exe.
|
||||
.PARAMETER Operation
|
||||
vmrun sub-command (start, stop, deleteVM, clone, …).
|
||||
.PARAMETER Arguments
|
||||
Additional positional arguments after the operation name.
|
||||
.PARAMETER TimeoutSeconds
|
||||
Hard timeout in seconds. Process is killed if not finished within this time.
|
||||
Default: 120.
|
||||
.PARAMETER ThrowOnTimeout
|
||||
When set, throws if the timeout is reached before vmrun exits.
|
||||
.PARAMETER ThrowOnError
|
||||
When set, throws if vmrun exits with a non-zero code.
|
||||
.OUTPUTS
|
||||
[pscustomobject] with:
|
||||
ExitCode [int] — vmrun exit code (-1 if killed due to timeout)
|
||||
Output [string] — combined stdout/stderr text
|
||||
TimedOut [bool] — true if the process was killed for exceeding TimeoutSeconds
|
||||
ElapsedSeconds [double] — wall-clock seconds from start to exit/kill
|
||||
#>
|
||||
[OutputType([pscustomobject])]
|
||||
[CmdletBinding()]
|
||||
param(
|
||||
[Parameter(Mandatory)] [string] $VmrunPath,
|
||||
[Parameter(Mandatory)] [string] $Operation,
|
||||
[string[]] $Arguments = @(),
|
||||
[int] $TimeoutSeconds = 120,
|
||||
[switch] $ThrowOnTimeout,
|
||||
[switch] $ThrowOnError
|
||||
)
|
||||
|
||||
$cmdArgs = @('-T', 'ws', $Operation) + $Arguments
|
||||
|
||||
# Quote tokens that contain spaces so the argument string is correct when
|
||||
# passed to ProcessStartInfo.Arguments as a single string.
|
||||
$quotedArgs = $cmdArgs | ForEach-Object {
|
||||
if ($_ -match '\s') { '"' + ($_ -replace '"', '""') + '"' } else { $_ }
|
||||
}
|
||||
$argStr = $quotedArgs -join ' '
|
||||
|
||||
$psi = New-Object System.Diagnostics.ProcessStartInfo
|
||||
$psi.FileName = $VmrunPath
|
||||
$psi.Arguments = $argStr
|
||||
$psi.UseShellExecute = $false
|
||||
$psi.CreateNoWindow = $true
|
||||
$psi.RedirectStandardOutput = $true
|
||||
$psi.RedirectStandardError = $true
|
||||
|
||||
$proc = New-Object System.Diagnostics.Process
|
||||
$proc.StartInfo = $psi
|
||||
$sw = [System.Diagnostics.Stopwatch]::StartNew()
|
||||
$timedOut = $false
|
||||
|
||||
[void]$proc.Start()
|
||||
|
||||
# Begin async reads before WaitForExit to avoid deadlock when output buffers fill.
|
||||
$stdoutTask = $proc.StandardOutput.ReadToEndAsync()
|
||||
$stderrTask = $proc.StandardError.ReadToEndAsync()
|
||||
|
||||
$exited = $proc.WaitForExit($TimeoutSeconds * 1000)
|
||||
if (-not $exited) {
|
||||
$timedOut = $true
|
||||
# Kill the entire process tree (taskkill /T) to release any inherited pipe
|
||||
# handles held by child processes (e.g. cmd.exe spawning ping.exe in tests,
|
||||
# or vmrun spawning helper subprocesses). Plain proc.Kill() only kills the
|
||||
# direct process; child processes keep the write end of stdout open.
|
||||
& taskkill /F /T /PID $proc.Id 2>&1 | Out-Null
|
||||
}
|
||||
$sw.Stop()
|
||||
|
||||
$outputText = ''
|
||||
if (-not $timedOut) {
|
||||
# Process exited normally: drain stdout/stderr (up to 5s safety net).
|
||||
[void][System.Threading.Tasks.Task]::WhenAll($stdoutTask, $stderrTask).Wait(5000)
|
||||
$stdoutText = ''
|
||||
$stderrText = ''
|
||||
if ($stdoutTask.Status -eq [System.Threading.Tasks.TaskStatus]::RanToCompletion) {
|
||||
$stdoutText = $stdoutTask.Result
|
||||
}
|
||||
if ($stderrTask.Status -eq [System.Threading.Tasks.TaskStatus]::RanToCompletion) {
|
||||
$stderrText = $stderrTask.Result
|
||||
}
|
||||
$parts = @($stdoutText, $stderrText) | Where-Object { -not [string]::IsNullOrWhiteSpace($_) }
|
||||
$outputText = ($parts | ForEach-Object { $_.TrimEnd() }) -join "`n"
|
||||
}
|
||||
# When timed-out: process was killed; output is irrelevant and potentially
|
||||
# incomplete. Skip drain entirely — no point blocking for a killed process.
|
||||
|
||||
$exitCode = if ($timedOut) { -1 } else { $proc.ExitCode }
|
||||
|
||||
$result = [pscustomobject]@{
|
||||
ExitCode = $exitCode
|
||||
Output = $outputText
|
||||
TimedOut = $timedOut
|
||||
ElapsedSeconds = [math]::Round($sw.Elapsed.TotalSeconds, 1)
|
||||
}
|
||||
|
||||
if ($timedOut -and $ThrowOnTimeout) {
|
||||
throw "vmrun $Operation timed out after ${TimeoutSeconds}s"
|
||||
}
|
||||
if (-not $timedOut -and $exitCode -ne 0 -and $ThrowOnError) {
|
||||
throw "vmrun $Operation failed (exit $exitCode): $outputText"
|
||||
}
|
||||
|
||||
return $result
|
||||
}
|
||||
|
||||
function Get-GuestIPAddress {
|
||||
<#
|
||||
.SYNOPSIS
|
||||
Polls a running VMware guest VM until its IP address is available, then returns it.
|
||||
.DESCRIPTION
|
||||
Primary method: reads the 'ci-ip' guestVar written by ci-report-ip.service (Linux)
|
||||
or the equivalent guest-side script via vmware-rpctool. This uses the VMware VMCI
|
||||
channel and does not require TCP/IP connectivity from the host.
|
||||
|
||||
Fallback: vmrun getGuestIPAddress (reliable on Windows guests; may be slow on Linux).
|
||||
|
||||
Returns the detected IP as a plain string, or an empty string if the timeout expires.
|
||||
.PARAMETER VmrunPath
|
||||
Full path to vmrun.exe.
|
||||
.PARAMETER VmxPath
|
||||
Full path to the clone VM's .vmx file.
|
||||
.PARAMETER TimeoutSeconds
|
||||
Maximum seconds to poll before giving up. Default: 120.
|
||||
.OUTPUTS
|
||||
[string] Detected IPv4 address, or empty string on timeout.
|
||||
#>
|
||||
[OutputType([string])]
|
||||
[CmdletBinding()]
|
||||
param(
|
||||
[Parameter(Mandatory)] [string] $VmrunPath,
|
||||
[Parameter(Mandatory)] [string] $VmxPath,
|
||||
[int] $TimeoutSeconds = 120
|
||||
)
|
||||
|
||||
Write-Host "[Get-GuestIPAddress] Polling for VM IP (max ${TimeoutSeconds}s)..."
|
||||
$deadline = (Get-Date).AddSeconds($TimeoutSeconds)
|
||||
$attempt = 0
|
||||
|
||||
while ((Get-Date) -lt $deadline) {
|
||||
$attempt++
|
||||
|
||||
# Primary: guestinfo.ci-ip written by ci-report-ip.service via vmware-rpctool.
|
||||
# NOTE: vmrun readVariable guestVar reads WITHOUT the 'guestinfo.' prefix.
|
||||
$savedEap = $ErrorActionPreference; $ErrorActionPreference = 'Continue'
|
||||
$ipOut = (& $VmrunPath -T ws readVariable $VmxPath guestVar 'ci-ip' 2>&1).Trim()
|
||||
$ipRc = $LASTEXITCODE
|
||||
$ErrorActionPreference = $savedEap
|
||||
|
||||
if ($attempt -le 3 -or ($attempt % 10 -eq 0)) {
|
||||
Write-Host "[Get-GuestIPAddress] [attempt $attempt] guestVar ci-ip: rc=$ipRc out='$ipOut'"
|
||||
}
|
||||
if ($ipRc -eq 0 -and $ipOut -match '^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$') {
|
||||
Write-Host "[Get-GuestIPAddress] IP via guestinfo: $ipOut"
|
||||
return $ipOut
|
||||
}
|
||||
|
||||
# Fallback: vmrun getGuestIPAddress (VMware Tools / open-vm-tools required).
|
||||
$savedEap = $ErrorActionPreference; $ErrorActionPreference = 'Continue'
|
||||
$ipOut2 = (& $VmrunPath -T ws getGuestIPAddress $VmxPath 2>&1).Trim()
|
||||
$ipRc2 = $LASTEXITCODE
|
||||
$ErrorActionPreference = $savedEap
|
||||
|
||||
if ($ipRc2 -eq 0 -and $ipOut2 -match '^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$') {
|
||||
Write-Host "[Get-GuestIPAddress] IP via getGuestIPAddress: $ipOut2"
|
||||
return $ipOut2
|
||||
}
|
||||
|
||||
Start-Sleep -Seconds 2
|
||||
}
|
||||
|
||||
return ''
|
||||
}
|
||||
|
||||
function Get-GuestDiagnostics {
|
||||
<#
|
||||
.SYNOPSIS
|
||||
Best-effort collection of guest diagnostics after a build failure.
|
||||
Never throws — always safe to call from a catch/finally block.
|
||||
|
||||
.PARAMETER IPAddress
|
||||
IP address of the guest VM.
|
||||
|
||||
.PARAMETER GuestOS
|
||||
'Windows' or 'Linux'.
|
||||
|
||||
.PARAMETER Credential
|
||||
PSCredential for WinRM (Windows guests). Ignored for Linux.
|
||||
|
||||
.PARAMETER SshKeyPath
|
||||
Path to SSH private key (Linux guests). Ignored for Windows.
|
||||
|
||||
.PARAMETER SshUser
|
||||
SSH username (Linux guests). Default: ci_build.
|
||||
|
||||
.PARAMETER DestinationDir
|
||||
Host directory where diagnostics files are written.
|
||||
#>
|
||||
[CmdletBinding()]
|
||||
param(
|
||||
[Parameter(Mandatory)] [string] $IPAddress,
|
||||
[Parameter(Mandatory)] [ValidateSet('Windows','Linux')]
|
||||
[string] $GuestOS,
|
||||
[pscredential] $Credential,
|
||||
[string] $SshKeyPath,
|
||||
[string] $SshUser = 'ci_build',
|
||||
[Parameter(Mandatory)] [string] $DestinationDir
|
||||
)
|
||||
|
||||
Write-Host "[Diagnostics] Collecting guest diagnostics from $IPAddress ($GuestOS) -> $DestinationDir"
|
||||
try {
|
||||
if (-not (Test-Path $DestinationDir)) {
|
||||
New-Item -ItemType Directory -Path $DestinationDir -Force | Out-Null
|
||||
}
|
||||
} catch {
|
||||
Write-Warning "[Diagnostics] Could not create destination dir: $_"
|
||||
return
|
||||
}
|
||||
|
||||
if ($GuestOS -eq 'Windows') {
|
||||
if (-not $Credential) {
|
||||
Write-Warning "[Diagnostics] No Credential supplied for Windows guest — skipping."
|
||||
return
|
||||
}
|
||||
try {
|
||||
$sessionOption = New-CISessionOption
|
||||
$session = New-PSSession -ComputerName $IPAddress `
|
||||
-Credential $Credential `
|
||||
-SessionOption $sessionOption `
|
||||
-ErrorAction Stop
|
||||
try {
|
||||
# Collect: event log errors (last 50), running processes, disk usage
|
||||
$diagScript = {
|
||||
$out = [System.Text.StringBuilder]::new()
|
||||
[void]$out.AppendLine("=== System Event Log (last 50 errors) ===")
|
||||
Get-EventLog -LogName System -EntryType Error -Newest 50 -ErrorAction SilentlyContinue |
|
||||
ForEach-Object { [void]$out.AppendLine("$($_.TimeGenerated) $($_.Source): $($_.Message)") }
|
||||
[void]$out.AppendLine("`n=== Running Processes ===")
|
||||
Get-Process -ErrorAction SilentlyContinue |
|
||||
Sort-Object CPU -Descending |
|
||||
Select-Object -First 30 |
|
||||
ForEach-Object { [void]$out.AppendLine("$($_.Name) PID=$($_.Id) CPU=$($_.CPU)") }
|
||||
[void]$out.AppendLine("`n=== Disk Free Space ===")
|
||||
Get-PSDrive -PSProvider FileSystem -ErrorAction SilentlyContinue |
|
||||
ForEach-Object { [void]$out.AppendLine("$($_.Name): Free=$([math]::Round($_.Free/1GB,1))GB Used=$([math]::Round($_.Used/1GB,1))GB") }
|
||||
$out.ToString()
|
||||
}
|
||||
$diagText = Invoke-Command -Session $session -ScriptBlock $diagScript -ErrorAction Stop
|
||||
$diagText | Set-Content -Path (Join-Path $DestinationDir 'guest-diag.txt') -Encoding UTF8
|
||||
Write-Host "[Diagnostics] Windows diagnostics written."
|
||||
|
||||
# Best-effort: copy CI build log
|
||||
$buildLog = 'C:\CI\build.log'
|
||||
try {
|
||||
Copy-Item -FromSession $session -Path $buildLog `
|
||||
-Destination (Join-Path $DestinationDir 'build.log') `
|
||||
-ErrorAction Stop
|
||||
Write-Host "[Diagnostics] Build log copied."
|
||||
} catch {
|
||||
Write-Host "[Diagnostics] Build log not found in guest (skipping)."
|
||||
}
|
||||
} finally {
|
||||
Remove-PSSession $session -ErrorAction SilentlyContinue
|
||||
}
|
||||
} catch {
|
||||
Write-Warning "[Diagnostics] Windows guest diagnostics failed (non-fatal): $_"
|
||||
}
|
||||
}
|
||||
else {
|
||||
# Linux
|
||||
if (-not $SshKeyPath) {
|
||||
Write-Warning "[Diagnostics] No SshKeyPath supplied for Linux guest — skipping."
|
||||
return
|
||||
}
|
||||
$sshOpts = @('-i', $SshKeyPath, '-o', 'StrictHostKeyChecking=accept-new',
|
||||
'-o', 'ConnectTimeout=10', '-o', 'BatchMode=yes')
|
||||
try {
|
||||
# Collect: journalctl errors, ps, df
|
||||
$diagCmd = "journalctl -p err -n 50 --no-pager 2>/dev/null; echo '=== ps ==='; ps aux --sort=-%cpu | head -30; echo '=== df ==='; df -h"
|
||||
$savedEap = $ErrorActionPreference
|
||||
$ErrorActionPreference = 'Continue'
|
||||
$diagOutput = & ssh.exe @sshOpts "${SshUser}@${IPAddress}" $diagCmd 2>&1
|
||||
$ErrorActionPreference = $savedEap
|
||||
$diagOutput | Set-Content -Path (Join-Path $DestinationDir 'guest-diag.txt') -Encoding UTF8
|
||||
Write-Host "[Diagnostics] Linux diagnostics written."
|
||||
|
||||
# Best-effort: copy CI build log
|
||||
$scpSrc = "${SshUser}@${IPAddress}:/opt/ci/build/ci-build.log"
|
||||
$scpDst = Join-Path $DestinationDir 'build.log'
|
||||
$ErrorActionPreference = 'Continue'
|
||||
& scp.exe @sshOpts $scpSrc $scpDst 2>&1 | Out-Null
|
||||
$ErrorActionPreference = $savedEap
|
||||
if (Test-Path $scpDst) { Write-Host "[Diagnostics] Build log copied." }
|
||||
else { Write-Host "[Diagnostics] Build log not found in guest (skipping)." }
|
||||
} catch {
|
||||
Write-Warning "[Diagnostics] Linux guest diagnostics failed (non-fatal): $_"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Export-ModuleMember -Function New-CISessionOption, Resolve-VmrunPath, Invoke-Vmrun, Invoke-VmrunBounded, Get-GuestIPAddress, Get-GuestDiagnostics
|
||||
|
||||
+26
-7
@@ -40,6 +40,12 @@ function Invoke-SshCommand {
|
||||
|
||||
[int] $ConnectTimeout = 10,
|
||||
|
||||
# Persistent known_hosts file for CI jobs.
|
||||
# When non-empty: StrictHostKeyChecking=accept-new + this file.
|
||||
# When empty (default): permissive mode (StrictHostKeyChecking=no, UserKnownHostsFile=NUL)
|
||||
# — used during template provisioning when host key is not yet known.
|
||||
[string] $KnownHostsFile = '',
|
||||
|
||||
# Capture + return stdout/stderr instead of printing to console.
|
||||
[switch] $PassThru,
|
||||
|
||||
@@ -47,10 +53,14 @@ function Invoke-SshCommand {
|
||||
[switch] $AllowFail
|
||||
)
|
||||
|
||||
if ($KnownHostsFile -ne '') {
|
||||
$sshHostKeyOpts = @('-o', 'StrictHostKeyChecking=accept-new', '-o', "UserKnownHostsFile=$KnownHostsFile")
|
||||
} else {
|
||||
$sshHostKeyOpts = @('-o', 'StrictHostKeyChecking=no', '-o', 'UserKnownHostsFile=NUL')
|
||||
}
|
||||
$sshArgs = @(
|
||||
'-i', $KeyPath,
|
||||
'-o', 'StrictHostKeyChecking=no',
|
||||
'-o', 'UserKnownHostsFile=NUL',
|
||||
'-i', $KeyPath
|
||||
) + $sshHostKeyOpts + @(
|
||||
'-o', "ConnectTimeout=$ConnectTimeout",
|
||||
'-o', 'BatchMode=yes',
|
||||
"$User@$IP",
|
||||
@@ -104,7 +114,12 @@ function Copy-SshItem {
|
||||
# Pass -r to scp (recursive copy).
|
||||
[switch] $Recurse,
|
||||
|
||||
[int] $ConnectTimeout = 10
|
||||
[int] $ConnectTimeout = 10,
|
||||
|
||||
# Persistent known_hosts file for CI jobs.
|
||||
# When non-empty: StrictHostKeyChecking=accept-new + this file.
|
||||
# When empty (default): permissive mode — used during template provisioning.
|
||||
[string] $KnownHostsFile = ''
|
||||
)
|
||||
|
||||
if ($Direction -eq 'ToGuest') {
|
||||
@@ -115,10 +130,14 @@ function Copy-SshItem {
|
||||
}
|
||||
# Direction = 'Direct': use Source + Destination as-is
|
||||
|
||||
if ($KnownHostsFile -ne '') {
|
||||
$scpHostKeyOpts = @('-o', 'StrictHostKeyChecking=accept-new', '-o', "UserKnownHostsFile=$KnownHostsFile")
|
||||
} else {
|
||||
$scpHostKeyOpts = @('-o', 'StrictHostKeyChecking=no', '-o', 'UserKnownHostsFile=NUL')
|
||||
}
|
||||
$scpArgs = @(
|
||||
'-i', $KeyPath,
|
||||
'-o', 'StrictHostKeyChecking=no',
|
||||
'-o', 'UserKnownHostsFile=NUL',
|
||||
'-i', $KeyPath
|
||||
) + $scpHostKeyOpts + @(
|
||||
'-o', "ConnectTimeout=$ConnectTimeout"
|
||||
)
|
||||
if ($Recurse) { $scpArgs += '-r' }
|
||||
|
||||
@@ -1,35 +1,130 @@
|
||||
#Requires -Version 5.1
|
||||
<#
|
||||
.SYNOPSIS
|
||||
Validates that Deploy-WinBuild2025.ps1 has correctly configured a guest VM.
|
||||
Validates that a guest VM has been correctly configured.
|
||||
|
||||
.DESCRIPTION
|
||||
Connects to the VM via WinRM (HTTP/Basic) and runs all checks that
|
||||
Install-CIToolchain-WinBuild2025.ps1 Steps 1/2/3/4b/5b/5c assert as validation-only.
|
||||
Use this after Deploy completes and before running Prepare/Install-CIToolchain.
|
||||
For Windows guests: connects via WinRM (HTTPS/Basic) and runs all checks
|
||||
that Install-CIToolchain-WinBuild2025.ps1 asserts as validation-only.
|
||||
|
||||
For Linux guests: connects via SSH and verifies that ci-report-ip.service
|
||||
is enabled and that the ci-report-ip.sh script is executable.
|
||||
|
||||
Use this after Deploy completes and before taking the BaseClean snapshot.
|
||||
|
||||
.PARAMETER VMIPAddress
|
||||
IP address of the guest VM on VMnet8 (e.g. 192.168.79.129).
|
||||
|
||||
.PARAMETER GuestOS
|
||||
Target OS: Windows (default) or Linux.
|
||||
|
||||
.PARAMETER AdminUsername
|
||||
Admin account inside the VM (default: Administrator).
|
||||
Windows only — admin account inside the VM (default: Administrator).
|
||||
Prompted if -GuestOS Windows and -AdminPassword is not provided.
|
||||
|
||||
.PARAMETER AdminPassword
|
||||
Password for the admin account. Prompted if omitted.
|
||||
Windows only — password for the admin account.
|
||||
|
||||
.PARAMETER SshKeyPath
|
||||
Linux only — path to the SSH private key (no passphrase).
|
||||
Default: F:\CI\keys\ci_linux
|
||||
|
||||
.PARAMETER SshUser
|
||||
Linux only — SSH login user inside the VM.
|
||||
Default: ci_build
|
||||
|
||||
.EXAMPLE
|
||||
# Windows
|
||||
.\Validate-DeployState.ps1 -VMIPAddress 192.168.79.129
|
||||
|
||||
.EXAMPLE
|
||||
# Linux
|
||||
.\Validate-DeployState.ps1 -VMIPAddress 192.168.79.200 -GuestOS Linux
|
||||
#>
|
||||
[CmdletBinding()]
|
||||
param(
|
||||
[Parameter(Mandatory)] [string] $VMIPAddress,
|
||||
|
||||
[ValidateSet('Windows', 'Linux')]
|
||||
[string] $GuestOS = 'Windows',
|
||||
|
||||
# Windows params
|
||||
[string] $AdminUsername = 'Administrator',
|
||||
[string] $AdminPassword
|
||||
[string] $AdminPassword,
|
||||
|
||||
# Linux params
|
||||
[string] $SshKeyPath = 'F:\CI\keys\ci_linux',
|
||||
[string] $SshUser = 'ci_build'
|
||||
)
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = 'Stop'
|
||||
|
||||
# ── Linux branch ──────────────────────────────────────────────────────────────
|
||||
if ($GuestOS -eq 'Linux') {
|
||||
if (-not (Test-Path $SshKeyPath -PathType Leaf)) {
|
||||
Write-Error "SSH key not found: $SshKeyPath"
|
||||
exit 1
|
||||
}
|
||||
|
||||
$sshBase = @(
|
||||
'-i', $SshKeyPath,
|
||||
'-o', 'StrictHostKeyChecking=no',
|
||||
'-o', 'UserKnownHostsFile=NUL',
|
||||
'-o', 'ConnectTimeout=10',
|
||||
'-o', 'BatchMode=yes',
|
||||
"$SshUser@$VMIPAddress"
|
||||
)
|
||||
|
||||
Write-Host "`nValidating Linux Deploy state on $VMIPAddress..." -ForegroundColor Cyan
|
||||
|
||||
$checks = [System.Collections.Generic.List[pscustomobject]]::new()
|
||||
|
||||
function Add-LinuxCheck {
|
||||
param([string]$Name, [string]$SshCommand, [scriptblock]$Validator)
|
||||
$savedEap = $ErrorActionPreference; $ErrorActionPreference = 'Continue'
|
||||
$out = (& ssh.exe @sshBase $SshCommand 2>&1)
|
||||
$rc = $LASTEXITCODE
|
||||
$ErrorActionPreference = $savedEap
|
||||
$pass = [bool](& $Validator $out $rc)
|
||||
$checks.Add([pscustomobject]@{ Name = $Name; Pass = $pass; Err = if (-not $pass) { "rc=$rc out=$out" } else { '' } })
|
||||
}
|
||||
|
||||
# ci-report-ip.service must be enabled
|
||||
Add-LinuxCheck `
|
||||
-Name 'ci-report-ip.service enabled' `
|
||||
-SshCommand 'systemctl is-enabled ci-report-ip.service 2>/dev/null' `
|
||||
-Validator { param($out, $rc) $rc -eq 0 -and "$out".Trim() -eq 'enabled' }
|
||||
|
||||
# ci-report-ip.sh must be executable
|
||||
Add-LinuxCheck `
|
||||
-Name '/usr/local/bin/ci-report-ip.sh executable' `
|
||||
-SshCommand 'test -x /usr/local/bin/ci-report-ip.sh && echo ok' `
|
||||
-Validator { param($out, $rc) $rc -eq 0 }
|
||||
|
||||
Write-Host ''
|
||||
$failed = 0
|
||||
foreach ($r in $checks) {
|
||||
if ($r.Pass) {
|
||||
Write-Host " [OK] $($r.Name)" -ForegroundColor Green
|
||||
} else {
|
||||
$detail = if ($r.Err) { " ($($r.Err))" } else { '' }
|
||||
Write-Host " [FAIL] $($r.Name)$detail" -ForegroundColor Red
|
||||
$failed++
|
||||
}
|
||||
}
|
||||
|
||||
Write-Host ''
|
||||
if ($failed -eq 0) {
|
||||
Write-Host "All $($checks.Count) Linux Deploy checks passed." -ForegroundColor Green
|
||||
exit 0
|
||||
} else {
|
||||
Write-Host "$failed / $($checks.Count) Linux Deploy checks FAILED." -ForegroundColor Red
|
||||
exit 1
|
||||
}
|
||||
}
|
||||
|
||||
# ── Windows branch (WinRM HTTPS) ──────────────────────────────────────────────
|
||||
if (-not $AdminPassword) {
|
||||
$secure = Read-Host "Password for $AdminUsername@$VMIPAddress" -AsSecureString
|
||||
$AdminPassword = [Runtime.InteropServices.Marshal]::PtrToStringAuto(
|
||||
|
||||
@@ -88,3 +88,94 @@ Describe 'Invoke-Vmrun' {
|
||||
$env:FAKE_VMRUN_EXIT = $null
|
||||
}
|
||||
}
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
Describe 'Invoke-VmrunBounded' {
|
||||
BeforeAll {
|
||||
# Fake vmrun that sleeps for $FAKE_VMRUN_SLEEP seconds then exits with
|
||||
# $FAKE_VMRUN_EXIT. Both default to 0 if unset.
|
||||
$script:SlowFakeVmrun = Join-Path $env:TEMP "fake-vmrun-slow-$PID.cmd"
|
||||
Set-Content $script:SlowFakeVmrun -Value @'
|
||||
@echo off
|
||||
if "%FAKE_VMRUN_SLEEP%"=="" goto :nosleep
|
||||
ping -n %FAKE_VMRUN_SLEEP% 127.0.0.1 >nul
|
||||
:nosleep
|
||||
echo fake-vmrun-bounded-output
|
||||
if "%FAKE_VMRUN_EXIT%"=="" ( exit /b 0 ) else ( exit /b %FAKE_VMRUN_EXIT% )
|
||||
'@
|
||||
}
|
||||
|
||||
AfterAll {
|
||||
Remove-Item $script:SlowFakeVmrun -Force -ErrorAction SilentlyContinue
|
||||
}
|
||||
|
||||
AfterEach {
|
||||
$env:FAKE_VMRUN_EXIT = $null
|
||||
$env:FAKE_VMRUN_SLEEP = $null
|
||||
}
|
||||
|
||||
It 'returns ExitCode 0, TimedOut false and Output when command succeeds quickly' {
|
||||
$env:FAKE_VMRUN_EXIT = '0'
|
||||
$env:FAKE_VMRUN_SLEEP = '0'
|
||||
$r = Invoke-VmrunBounded -VmrunPath $script:SlowFakeVmrun -Operation 'start' -Arguments @('some.vmx', 'nogui') -TimeoutSeconds 30
|
||||
$r.ExitCode | Should -Be 0
|
||||
$r.TimedOut | Should -Be $false
|
||||
"$($r.Output)" | Should -Match 'fake-vmrun-bounded-output'
|
||||
$r.ElapsedSeconds | Should -BeGreaterOrEqual 0
|
||||
}
|
||||
|
||||
It 'returns non-zero ExitCode without throwing by default' {
|
||||
$env:FAKE_VMRUN_EXIT = '3'
|
||||
$env:FAKE_VMRUN_SLEEP = '0'
|
||||
$r = Invoke-VmrunBounded -VmrunPath $script:SlowFakeVmrun -Operation 'deleteVM' -TimeoutSeconds 30
|
||||
$r.ExitCode | Should -Be 3
|
||||
$r.TimedOut | Should -Be $false
|
||||
}
|
||||
|
||||
It 'throws when -ThrowOnError and ExitCode is non-zero' {
|
||||
$env:FAKE_VMRUN_EXIT = '2'
|
||||
$env:FAKE_VMRUN_SLEEP = '0'
|
||||
{ Invoke-VmrunBounded -VmrunPath $script:SlowFakeVmrun -Operation 'stop' -TimeoutSeconds 30 -ThrowOnError } |
|
||||
Should -Throw -ExpectedMessage '*stop failed*'
|
||||
}
|
||||
|
||||
It 'does not throw when -ThrowOnError and ExitCode is 0' {
|
||||
$env:FAKE_VMRUN_EXIT = '0'
|
||||
$env:FAKE_VMRUN_SLEEP = '0'
|
||||
{ Invoke-VmrunBounded -VmrunPath $script:SlowFakeVmrun -Operation 'start' -TimeoutSeconds 30 -ThrowOnError } |
|
||||
Should -Not -Throw
|
||||
}
|
||||
|
||||
It 'times out and returns TimedOut = true when process exceeds TimeoutSeconds' {
|
||||
# FAKE_VMRUN_SLEEP uses ping -n N which waits roughly (N-1) seconds.
|
||||
# Set sleep >> timeout so the process is guaranteed to still be running.
|
||||
$env:FAKE_VMRUN_SLEEP = '60'
|
||||
$env:FAKE_VMRUN_EXIT = '0'
|
||||
$r = Invoke-VmrunBounded -VmrunPath $script:SlowFakeVmrun -Operation 'start' -TimeoutSeconds 2
|
||||
$r.TimedOut | Should -Be $true
|
||||
$r.ExitCode | Should -Be -1
|
||||
$r.ElapsedSeconds | Should -BeLessOrEqual 10 # killed well before 60s
|
||||
}
|
||||
|
||||
It 'throws when -ThrowOnTimeout and process times out' {
|
||||
$env:FAKE_VMRUN_SLEEP = '60'
|
||||
$env:FAKE_VMRUN_EXIT = '0'
|
||||
{ Invoke-VmrunBounded -VmrunPath $script:SlowFakeVmrun -Operation 'deleteVM' -TimeoutSeconds 2 -ThrowOnTimeout } |
|
||||
Should -Throw -ExpectedMessage '*timed out*'
|
||||
}
|
||||
|
||||
It 'does not throw on timeout when neither flag is set' {
|
||||
$env:FAKE_VMRUN_SLEEP = '60'
|
||||
$env:FAKE_VMRUN_EXIT = '0'
|
||||
{ Invoke-VmrunBounded -VmrunPath $script:SlowFakeVmrun -Operation 'start' -TimeoutSeconds 2 } |
|
||||
Should -Not -Throw
|
||||
}
|
||||
|
||||
It 'ElapsedSeconds is populated and plausible' {
|
||||
$env:FAKE_VMRUN_EXIT = '0'
|
||||
$env:FAKE_VMRUN_SLEEP = '0'
|
||||
$r = Invoke-VmrunBounded -VmrunPath $script:SlowFakeVmrun -Operation 'list' -TimeoutSeconds 30
|
||||
$r.ElapsedSeconds | Should -BeGreaterOrEqual 0
|
||||
$r.ElapsedSeconds | Should -BeLessThan 30
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user