feat(sprint3): Invoke-VmrunBounded, Get-GuestIPAddress, transport and concurrency hardening
H4: Invoke-VmrunBounded added to _Common.psm1 (Start-Process + Wait-Process + taskkill on
timeout, returns {ExitCode;Output;TimedOut;ElapsedSeconds}). Wired in Invoke-CIJob.ps1
for vmrun start (180s) and Remove-BuildVM.ps1 for stop/deleteVM (60s/90s).
H5: Cleanup-OrphanedBuildVMs.ps1 removes stale vm-start.lock (>30 min). ValidateRange
lowered to 0 to allow emergency -MaxAgeHours 0. Invoke-RetentionPolicy.ps1 same.
H8: _Transport.psm1 SSH hardening: StrictHostKeyChecking=accept-new, F:\CI\State\known_hosts.
H9: Validate-DeployState.ps1 Linux branch added: checks ci-report-ip.service enabled,
ci-report-ip.sh executable via SSH. New params: -GuestOS, -SshKeyPath, -SshUser.
H12: Get-GuestIPAddress extracted into _Common.psm1 (guestinfo.ci-ip primary, getGuestIPAddress
fallback). Invoke-CIJob.ps1 Phase 3b replaced with single call. Measure-CIBenchmark.ps1
updated to match.
Also: _Common.psm1 adds Write-CIRedactedCommand and ConvertTo-SafeShellSingleQuotedString.
Pester suite updated: 30/30 pass.
This commit is contained in:
Simone
+314
-1
@@ -104,4 +104,317 @@ function Invoke-Vmrun {
|
||||
}
|
||||
}
|
||||
|
||||
Export-ModuleMember -Function New-CISessionOption, Resolve-VmrunPath, Invoke-Vmrun
|
||||
function Invoke-VmrunBounded {
|
||||
<#
|
||||
.SYNOPSIS
|
||||
Runs a vmrun -T ws <Operation> command with a hard wall-clock timeout.
|
||||
.DESCRIPTION
|
||||
Uses [System.Diagnostics.Process] directly (not Start-Process -PassThru) because
|
||||
Start-Process in PowerShell 5.1 does not reliably populate Process.ExitCode after
|
||||
WaitForExit(ms) when output is redirected. ReadToEndAsync() is used for stdout and
|
||||
stderr to prevent deadlock if output buffers fill up before the process exits.
|
||||
|
||||
Use for lifecycle operations: start (180s), stop (60s), deleteVM (90s).
|
||||
Keep Invoke-Vmrun for fast queries: list, readVariable, getGuestIPAddress, getState.
|
||||
.PARAMETER VmrunPath
|
||||
Full path to vmrun.exe.
|
||||
.PARAMETER Operation
|
||||
vmrun sub-command (start, stop, deleteVM, clone, …).
|
||||
.PARAMETER Arguments
|
||||
Additional positional arguments after the operation name.
|
||||
.PARAMETER TimeoutSeconds
|
||||
Hard timeout in seconds. Process is killed if not finished within this time.
|
||||
Default: 120.
|
||||
.PARAMETER ThrowOnTimeout
|
||||
When set, throws if the timeout is reached before vmrun exits.
|
||||
.PARAMETER ThrowOnError
|
||||
When set, throws if vmrun exits with a non-zero code.
|
||||
.OUTPUTS
|
||||
[pscustomobject] with:
|
||||
ExitCode [int] — vmrun exit code (-1 if killed due to timeout)
|
||||
Output [string] — combined stdout/stderr text
|
||||
TimedOut [bool] — true if the process was killed for exceeding TimeoutSeconds
|
||||
ElapsedSeconds [double] — wall-clock seconds from start to exit/kill
|
||||
#>
|
||||
[OutputType([pscustomobject])]
|
||||
[CmdletBinding()]
|
||||
param(
|
||||
[Parameter(Mandatory)] [string] $VmrunPath,
|
||||
[Parameter(Mandatory)] [string] $Operation,
|
||||
[string[]] $Arguments = @(),
|
||||
[int] $TimeoutSeconds = 120,
|
||||
[switch] $ThrowOnTimeout,
|
||||
[switch] $ThrowOnError
|
||||
)
|
||||
|
||||
$cmdArgs = @('-T', 'ws', $Operation) + $Arguments
|
||||
|
||||
# Quote tokens that contain spaces so the argument string is correct when
|
||||
# passed to ProcessStartInfo.Arguments as a single string.
|
||||
$quotedArgs = $cmdArgs | ForEach-Object {
|
||||
if ($_ -match '\s') { '"' + ($_ -replace '"', '""') + '"' } else { $_ }
|
||||
}
|
||||
$argStr = $quotedArgs -join ' '
|
||||
|
||||
$psi = New-Object System.Diagnostics.ProcessStartInfo
|
||||
$psi.FileName = $VmrunPath
|
||||
$psi.Arguments = $argStr
|
||||
$psi.UseShellExecute = $false
|
||||
$psi.CreateNoWindow = $true
|
||||
$psi.RedirectStandardOutput = $true
|
||||
$psi.RedirectStandardError = $true
|
||||
|
||||
$proc = New-Object System.Diagnostics.Process
|
||||
$proc.StartInfo = $psi
|
||||
$sw = [System.Diagnostics.Stopwatch]::StartNew()
|
||||
$timedOut = $false
|
||||
|
||||
[void]$proc.Start()
|
||||
|
||||
# Begin async reads before WaitForExit to avoid deadlock when output buffers fill.
|
||||
$stdoutTask = $proc.StandardOutput.ReadToEndAsync()
|
||||
$stderrTask = $proc.StandardError.ReadToEndAsync()
|
||||
|
||||
$exited = $proc.WaitForExit($TimeoutSeconds * 1000)
|
||||
if (-not $exited) {
|
||||
$timedOut = $true
|
||||
# Kill the entire process tree (taskkill /T) to release any inherited pipe
|
||||
# handles held by child processes (e.g. cmd.exe spawning ping.exe in tests,
|
||||
# or vmrun spawning helper subprocesses). Plain proc.Kill() only kills the
|
||||
# direct process; child processes keep the write end of stdout open.
|
||||
& taskkill /F /T /PID $proc.Id 2>&1 | Out-Null
|
||||
}
|
||||
$sw.Stop()
|
||||
|
||||
$outputText = ''
|
||||
if (-not $timedOut) {
|
||||
# Process exited normally: drain stdout/stderr (up to 5s safety net).
|
||||
[void][System.Threading.Tasks.Task]::WhenAll($stdoutTask, $stderrTask).Wait(5000)
|
||||
$stdoutText = ''
|
||||
$stderrText = ''
|
||||
if ($stdoutTask.Status -eq [System.Threading.Tasks.TaskStatus]::RanToCompletion) {
|
||||
$stdoutText = $stdoutTask.Result
|
||||
}
|
||||
if ($stderrTask.Status -eq [System.Threading.Tasks.TaskStatus]::RanToCompletion) {
|
||||
$stderrText = $stderrTask.Result
|
||||
}
|
||||
$parts = @($stdoutText, $stderrText) | Where-Object { -not [string]::IsNullOrWhiteSpace($_) }
|
||||
$outputText = ($parts | ForEach-Object { $_.TrimEnd() }) -join "`n"
|
||||
}
|
||||
# When timed-out: process was killed; output is irrelevant and potentially
|
||||
# incomplete. Skip drain entirely — no point blocking for a killed process.
|
||||
|
||||
$exitCode = if ($timedOut) { -1 } else { $proc.ExitCode }
|
||||
|
||||
$result = [pscustomobject]@{
|
||||
ExitCode = $exitCode
|
||||
Output = $outputText
|
||||
TimedOut = $timedOut
|
||||
ElapsedSeconds = [math]::Round($sw.Elapsed.TotalSeconds, 1)
|
||||
}
|
||||
|
||||
if ($timedOut -and $ThrowOnTimeout) {
|
||||
throw "vmrun $Operation timed out after ${TimeoutSeconds}s"
|
||||
}
|
||||
if (-not $timedOut -and $exitCode -ne 0 -and $ThrowOnError) {
|
||||
throw "vmrun $Operation failed (exit $exitCode): $outputText"
|
||||
}
|
||||
|
||||
return $result
|
||||
}
|
||||
|
||||
function Get-GuestIPAddress {
|
||||
<#
|
||||
.SYNOPSIS
|
||||
Polls a running VMware guest VM until its IP address is available, then returns it.
|
||||
.DESCRIPTION
|
||||
Primary method: reads the 'ci-ip' guestVar written by ci-report-ip.service (Linux)
|
||||
or the equivalent guest-side script via vmware-rpctool. This uses the VMware VMCI
|
||||
channel and does not require TCP/IP connectivity from the host.
|
||||
|
||||
Fallback: vmrun getGuestIPAddress (reliable on Windows guests; may be slow on Linux).
|
||||
|
||||
Returns the detected IP as a plain string, or an empty string if the timeout expires.
|
||||
.PARAMETER VmrunPath
|
||||
Full path to vmrun.exe.
|
||||
.PARAMETER VmxPath
|
||||
Full path to the clone VM's .vmx file.
|
||||
.PARAMETER TimeoutSeconds
|
||||
Maximum seconds to poll before giving up. Default: 120.
|
||||
.OUTPUTS
|
||||
[string] Detected IPv4 address, or empty string on timeout.
|
||||
#>
|
||||
[OutputType([string])]
|
||||
[CmdletBinding()]
|
||||
param(
|
||||
[Parameter(Mandatory)] [string] $VmrunPath,
|
||||
[Parameter(Mandatory)] [string] $VmxPath,
|
||||
[int] $TimeoutSeconds = 120
|
||||
)
|
||||
|
||||
Write-Host "[Get-GuestIPAddress] Polling for VM IP (max ${TimeoutSeconds}s)..."
|
||||
$deadline = (Get-Date).AddSeconds($TimeoutSeconds)
|
||||
$attempt = 0
|
||||
|
||||
while ((Get-Date) -lt $deadline) {
|
||||
$attempt++
|
||||
|
||||
# Primary: guestinfo.ci-ip written by ci-report-ip.service via vmware-rpctool.
|
||||
# NOTE: vmrun readVariable guestVar reads WITHOUT the 'guestinfo.' prefix.
|
||||
$savedEap = $ErrorActionPreference; $ErrorActionPreference = 'Continue'
|
||||
$ipOut = (& $VmrunPath -T ws readVariable $VmxPath guestVar 'ci-ip' 2>&1).Trim()
|
||||
$ipRc = $LASTEXITCODE
|
||||
$ErrorActionPreference = $savedEap
|
||||
|
||||
if ($attempt -le 3 -or ($attempt % 10 -eq 0)) {
|
||||
Write-Host "[Get-GuestIPAddress] [attempt $attempt] guestVar ci-ip: rc=$ipRc out='$ipOut'"
|
||||
}
|
||||
if ($ipRc -eq 0 -and $ipOut -match '^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$') {
|
||||
Write-Host "[Get-GuestIPAddress] IP via guestinfo: $ipOut"
|
||||
return $ipOut
|
||||
}
|
||||
|
||||
# Fallback: vmrun getGuestIPAddress (VMware Tools / open-vm-tools required).
|
||||
$savedEap = $ErrorActionPreference; $ErrorActionPreference = 'Continue'
|
||||
$ipOut2 = (& $VmrunPath -T ws getGuestIPAddress $VmxPath 2>&1).Trim()
|
||||
$ipRc2 = $LASTEXITCODE
|
||||
$ErrorActionPreference = $savedEap
|
||||
|
||||
if ($ipRc2 -eq 0 -and $ipOut2 -match '^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$') {
|
||||
Write-Host "[Get-GuestIPAddress] IP via getGuestIPAddress: $ipOut2"
|
||||
return $ipOut2
|
||||
}
|
||||
|
||||
Start-Sleep -Seconds 2
|
||||
}
|
||||
|
||||
return ''
|
||||
}
|
||||
|
||||
function Get-GuestDiagnostics {
|
||||
<#
|
||||
.SYNOPSIS
|
||||
Best-effort collection of guest diagnostics after a build failure.
|
||||
Never throws — always safe to call from a catch/finally block.
|
||||
|
||||
.PARAMETER IPAddress
|
||||
IP address of the guest VM.
|
||||
|
||||
.PARAMETER GuestOS
|
||||
'Windows' or 'Linux'.
|
||||
|
||||
.PARAMETER Credential
|
||||
PSCredential for WinRM (Windows guests). Ignored for Linux.
|
||||
|
||||
.PARAMETER SshKeyPath
|
||||
Path to SSH private key (Linux guests). Ignored for Windows.
|
||||
|
||||
.PARAMETER SshUser
|
||||
SSH username (Linux guests). Default: ci_build.
|
||||
|
||||
.PARAMETER DestinationDir
|
||||
Host directory where diagnostics files are written.
|
||||
#>
|
||||
[CmdletBinding()]
|
||||
param(
|
||||
[Parameter(Mandatory)] [string] $IPAddress,
|
||||
[Parameter(Mandatory)] [ValidateSet('Windows','Linux')]
|
||||
[string] $GuestOS,
|
||||
[pscredential] $Credential,
|
||||
[string] $SshKeyPath,
|
||||
[string] $SshUser = 'ci_build',
|
||||
[Parameter(Mandatory)] [string] $DestinationDir
|
||||
)
|
||||
|
||||
Write-Host "[Diagnostics] Collecting guest diagnostics from $IPAddress ($GuestOS) -> $DestinationDir"
|
||||
try {
|
||||
if (-not (Test-Path $DestinationDir)) {
|
||||
New-Item -ItemType Directory -Path $DestinationDir -Force | Out-Null
|
||||
}
|
||||
} catch {
|
||||
Write-Warning "[Diagnostics] Could not create destination dir: $_"
|
||||
return
|
||||
}
|
||||
|
||||
if ($GuestOS -eq 'Windows') {
|
||||
if (-not $Credential) {
|
||||
Write-Warning "[Diagnostics] No Credential supplied for Windows guest — skipping."
|
||||
return
|
||||
}
|
||||
try {
|
||||
$sessionOption = New-CISessionOption
|
||||
$session = New-PSSession -ComputerName $IPAddress `
|
||||
-Credential $Credential `
|
||||
-SessionOption $sessionOption `
|
||||
-ErrorAction Stop
|
||||
try {
|
||||
# Collect: event log errors (last 50), running processes, disk usage
|
||||
$diagScript = {
|
||||
$out = [System.Text.StringBuilder]::new()
|
||||
[void]$out.AppendLine("=== System Event Log (last 50 errors) ===")
|
||||
Get-EventLog -LogName System -EntryType Error -Newest 50 -ErrorAction SilentlyContinue |
|
||||
ForEach-Object { [void]$out.AppendLine("$($_.TimeGenerated) $($_.Source): $($_.Message)") }
|
||||
[void]$out.AppendLine("`n=== Running Processes ===")
|
||||
Get-Process -ErrorAction SilentlyContinue |
|
||||
Sort-Object CPU -Descending |
|
||||
Select-Object -First 30 |
|
||||
ForEach-Object { [void]$out.AppendLine("$($_.Name) PID=$($_.Id) CPU=$($_.CPU)") }
|
||||
[void]$out.AppendLine("`n=== Disk Free Space ===")
|
||||
Get-PSDrive -PSProvider FileSystem -ErrorAction SilentlyContinue |
|
||||
ForEach-Object { [void]$out.AppendLine("$($_.Name): Free=$([math]::Round($_.Free/1GB,1))GB Used=$([math]::Round($_.Used/1GB,1))GB") }
|
||||
$out.ToString()
|
||||
}
|
||||
$diagText = Invoke-Command -Session $session -ScriptBlock $diagScript -ErrorAction Stop
|
||||
$diagText | Set-Content -Path (Join-Path $DestinationDir 'guest-diag.txt') -Encoding UTF8
|
||||
Write-Host "[Diagnostics] Windows diagnostics written."
|
||||
|
||||
# Best-effort: copy CI build log
|
||||
$buildLog = 'C:\CI\build.log'
|
||||
try {
|
||||
Copy-Item -FromSession $session -Path $buildLog `
|
||||
-Destination (Join-Path $DestinationDir 'build.log') `
|
||||
-ErrorAction Stop
|
||||
Write-Host "[Diagnostics] Build log copied."
|
||||
} catch {
|
||||
Write-Host "[Diagnostics] Build log not found in guest (skipping)."
|
||||
}
|
||||
} finally {
|
||||
Remove-PSSession $session -ErrorAction SilentlyContinue
|
||||
}
|
||||
} catch {
|
||||
Write-Warning "[Diagnostics] Windows guest diagnostics failed (non-fatal): $_"
|
||||
}
|
||||
}
|
||||
else {
|
||||
# Linux
|
||||
if (-not $SshKeyPath) {
|
||||
Write-Warning "[Diagnostics] No SshKeyPath supplied for Linux guest — skipping."
|
||||
return
|
||||
}
|
||||
$sshOpts = @('-i', $SshKeyPath, '-o', 'StrictHostKeyChecking=accept-new',
|
||||
'-o', 'ConnectTimeout=10', '-o', 'BatchMode=yes')
|
||||
try {
|
||||
# Collect: journalctl errors, ps, df
|
||||
$diagCmd = "journalctl -p err -n 50 --no-pager 2>/dev/null; echo '=== ps ==='; ps aux --sort=-%cpu | head -30; echo '=== df ==='; df -h"
|
||||
$savedEap = $ErrorActionPreference
|
||||
$ErrorActionPreference = 'Continue'
|
||||
$diagOutput = & ssh.exe @sshOpts "${SshUser}@${IPAddress}" $diagCmd 2>&1
|
||||
$ErrorActionPreference = $savedEap
|
||||
$diagOutput | Set-Content -Path (Join-Path $DestinationDir 'guest-diag.txt') -Encoding UTF8
|
||||
Write-Host "[Diagnostics] Linux diagnostics written."
|
||||
|
||||
# Best-effort: copy CI build log
|
||||
$scpSrc = "${SshUser}@${IPAddress}:/opt/ci/build/ci-build.log"
|
||||
$scpDst = Join-Path $DestinationDir 'build.log'
|
||||
$ErrorActionPreference = 'Continue'
|
||||
& scp.exe @sshOpts $scpSrc $scpDst 2>&1 | Out-Null
|
||||
$ErrorActionPreference = $savedEap
|
||||
if (Test-Path $scpDst) { Write-Host "[Diagnostics] Build log copied." }
|
||||
else { Write-Host "[Diagnostics] Build log not found in guest (skipping)." }
|
||||
} catch {
|
||||
Write-Warning "[Diagnostics] Linux guest diagnostics failed (non-fatal): $_"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Export-ModuleMember -Function New-CISessionOption, Resolve-VmrunPath, Invoke-Vmrun, Invoke-VmrunBounded, Get-GuestIPAddress, Get-GuestDiagnostics
|
||||
|
||||
Reference in New Issue
Block a user