Simone
509d1fc284
H4: Invoke-VmrunBounded added to _Common.psm1 (Start-Process + Wait-Process + taskkill on
timeout, returns {ExitCode;Output;TimedOut;ElapsedSeconds}). Wired in Invoke-CIJob.ps1
for vmrun start (180s) and Remove-BuildVM.ps1 for stop/deleteVM (60s/90s).
H5: Cleanup-OrphanedBuildVMs.ps1 removes stale vm-start.lock (>30 min). ValidateRange
lowered to 0 to allow emergency -MaxAgeHours 0. Invoke-RetentionPolicy.ps1 same.
H8: _Transport.psm1 SSH hardening: StrictHostKeyChecking=accept-new, F:\CI\State\known_hosts.
H9: Validate-DeployState.ps1 Linux branch added: checks ci-report-ip.service enabled,
ci-report-ip.sh executable via SSH. New params: -GuestOS, -SshKeyPath, -SshUser.
H12: Get-GuestIPAddress extracted into _Common.psm1 (guestinfo.ci-ip primary, getGuestIPAddress
fallback). Invoke-CIJob.ps1 Phase 3b replaced with single call. Measure-CIBenchmark.ps1
updated to match.
Also: _Common.psm1 adds Write-CIRedactedCommand and ConvertTo-SafeShellSingleQuotedString.
Pester suite updated: 30/30 pass.
421 lines
17 KiB
PowerShell
421 lines
17 KiB
PowerShell
#Requires -Version 5.1
|
|
<#
|
|
.SYNOPSIS
|
|
Shared helper functions for the Local CI/CD System scripts.
|
|
|
|
.DESCRIPTION
|
|
Import this module at the top of any CI script that needs WinRM session
|
|
options or vmrun wrappers:
|
|
|
|
Import-Module (Join-Path $PSScriptRoot '_Common.psm1') -Force
|
|
|
|
Exported functions:
|
|
New-CISessionOption — PSSessionOption for WinRM over self-signed TLS
|
|
Resolve-VmrunPath — Validates vmrun.exe path, throws if missing
|
|
Invoke-Vmrun — Uniform vmrun -T ws wrapper; returns ExitCode + Output
|
|
#>
|
|
Set-StrictMode -Version Latest
|
|
$ErrorActionPreference = 'Stop'
|
|
|
|
function New-CISessionOption {
|
|
<#
|
|
.SYNOPSIS
|
|
Returns a PSSessionOption for WinRM HTTPS with self-signed certificate.
|
|
.DESCRIPTION
|
|
All CI build VMs use a self-signed TLS cert on WinRM port 5986.
|
|
This option set skips CA, CN, and revocation checks — appropriate for
|
|
an isolated lab network where PKI is not present.
|
|
.OUTPUTS
|
|
[System.Management.Automation.Remoting.PSSessionOption]
|
|
#>
|
|
[OutputType([System.Management.Automation.Remoting.PSSessionOption])]
|
|
[Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSUseShouldProcessForStateChangingFunctions', '',
|
|
Justification = 'New-CISessionOption creates an in-memory PSSessionOption object; no system state is changed.')]
|
|
[CmdletBinding()]
|
|
param()
|
|
New-PSSessionOption -SkipCACheck -SkipCNCheck -SkipRevocationCheck
|
|
}
|
|
|
|
function Resolve-VmrunPath {
|
|
<#
|
|
.SYNOPSIS
|
|
Validates that vmrun.exe exists at the specified path and returns it.
|
|
.DESCRIPTION
|
|
Throws a descriptive error if the file is not found, rather than letting
|
|
a later & call fail with a less useful message.
|
|
.PARAMETER VmrunPath
|
|
Full path to vmrun.exe.
|
|
Default: C:\Program Files (x86)\VMware\VMware Workstation\vmrun.exe
|
|
.OUTPUTS
|
|
[string] The validated VmrunPath.
|
|
#>
|
|
[OutputType([string])]
|
|
[CmdletBinding()]
|
|
param(
|
|
[string] $VmrunPath = 'C:\Program Files (x86)\VMware\VMware Workstation\vmrun.exe'
|
|
)
|
|
if (-not (Test-Path $VmrunPath -PathType Leaf)) {
|
|
throw "vmrun.exe not found at: $VmrunPath`nInstall VMware Workstation Pro and update -VmrunPath."
|
|
}
|
|
return $VmrunPath
|
|
}
|
|
|
|
function Invoke-Vmrun {
|
|
<#
|
|
.SYNOPSIS
|
|
Runs a vmrun -T ws <Operation> command and returns output and exit code.
|
|
.DESCRIPTION
|
|
Uniform wrapper so all vmrun calls share the same -T ws flag and output
|
|
capture pattern. Use -ThrowOnError for operations that must succeed
|
|
(clone, start); omit it for best-effort operations (stop, getState).
|
|
.PARAMETER VmrunPath
|
|
Full path to vmrun.exe.
|
|
.PARAMETER Operation
|
|
vmrun sub-command: clone, start, stop, deleteVM, list, getState,
|
|
listSnapshots, getGuestIPAddress, etc.
|
|
.PARAMETER Arguments
|
|
Additional positional arguments after the operation name.
|
|
.PARAMETER ThrowOnError
|
|
When set, throws if vmrun exits non-zero.
|
|
.OUTPUTS
|
|
[pscustomobject] with:
|
|
ExitCode [int] — vmrun exit code
|
|
Output [string[]] — combined stdout/stderr lines
|
|
#>
|
|
[OutputType([pscustomobject])]
|
|
[CmdletBinding()]
|
|
param(
|
|
[Parameter(Mandatory)] [string] $VmrunPath,
|
|
[Parameter(Mandatory)] [string] $Operation,
|
|
[string[]] $Arguments = @(),
|
|
[switch] $ThrowOnError
|
|
)
|
|
$cmdArgs = @('-T', 'ws', $Operation) + $Arguments
|
|
$output = & $VmrunPath @cmdArgs 2>&1
|
|
$exit = $LASTEXITCODE
|
|
|
|
if ($ThrowOnError -and $exit -ne 0) {
|
|
throw "vmrun $Operation failed (exit $exit): $($output -join '; ')"
|
|
}
|
|
|
|
return [pscustomobject]@{
|
|
ExitCode = $exit
|
|
Output = $output
|
|
}
|
|
}
|
|
|
|
function Invoke-VmrunBounded {
|
|
<#
|
|
.SYNOPSIS
|
|
Runs a vmrun -T ws <Operation> command with a hard wall-clock timeout.
|
|
.DESCRIPTION
|
|
Uses [System.Diagnostics.Process] directly (not Start-Process -PassThru) because
|
|
Start-Process in PowerShell 5.1 does not reliably populate Process.ExitCode after
|
|
WaitForExit(ms) when output is redirected. ReadToEndAsync() is used for stdout and
|
|
stderr to prevent deadlock if output buffers fill up before the process exits.
|
|
|
|
Use for lifecycle operations: start (180s), stop (60s), deleteVM (90s).
|
|
Keep Invoke-Vmrun for fast queries: list, readVariable, getGuestIPAddress, getState.
|
|
.PARAMETER VmrunPath
|
|
Full path to vmrun.exe.
|
|
.PARAMETER Operation
|
|
vmrun sub-command (start, stop, deleteVM, clone, …).
|
|
.PARAMETER Arguments
|
|
Additional positional arguments after the operation name.
|
|
.PARAMETER TimeoutSeconds
|
|
Hard timeout in seconds. Process is killed if not finished within this time.
|
|
Default: 120.
|
|
.PARAMETER ThrowOnTimeout
|
|
When set, throws if the timeout is reached before vmrun exits.
|
|
.PARAMETER ThrowOnError
|
|
When set, throws if vmrun exits with a non-zero code.
|
|
.OUTPUTS
|
|
[pscustomobject] with:
|
|
ExitCode [int] — vmrun exit code (-1 if killed due to timeout)
|
|
Output [string] — combined stdout/stderr text
|
|
TimedOut [bool] — true if the process was killed for exceeding TimeoutSeconds
|
|
ElapsedSeconds [double] — wall-clock seconds from start to exit/kill
|
|
#>
|
|
[OutputType([pscustomobject])]
|
|
[CmdletBinding()]
|
|
param(
|
|
[Parameter(Mandatory)] [string] $VmrunPath,
|
|
[Parameter(Mandatory)] [string] $Operation,
|
|
[string[]] $Arguments = @(),
|
|
[int] $TimeoutSeconds = 120,
|
|
[switch] $ThrowOnTimeout,
|
|
[switch] $ThrowOnError
|
|
)
|
|
|
|
$cmdArgs = @('-T', 'ws', $Operation) + $Arguments
|
|
|
|
# Quote tokens that contain spaces so the argument string is correct when
|
|
# passed to ProcessStartInfo.Arguments as a single string.
|
|
$quotedArgs = $cmdArgs | ForEach-Object {
|
|
if ($_ -match '\s') { '"' + ($_ -replace '"', '""') + '"' } else { $_ }
|
|
}
|
|
$argStr = $quotedArgs -join ' '
|
|
|
|
$psi = New-Object System.Diagnostics.ProcessStartInfo
|
|
$psi.FileName = $VmrunPath
|
|
$psi.Arguments = $argStr
|
|
$psi.UseShellExecute = $false
|
|
$psi.CreateNoWindow = $true
|
|
$psi.RedirectStandardOutput = $true
|
|
$psi.RedirectStandardError = $true
|
|
|
|
$proc = New-Object System.Diagnostics.Process
|
|
$proc.StartInfo = $psi
|
|
$sw = [System.Diagnostics.Stopwatch]::StartNew()
|
|
$timedOut = $false
|
|
|
|
[void]$proc.Start()
|
|
|
|
# Begin async reads before WaitForExit to avoid deadlock when output buffers fill.
|
|
$stdoutTask = $proc.StandardOutput.ReadToEndAsync()
|
|
$stderrTask = $proc.StandardError.ReadToEndAsync()
|
|
|
|
$exited = $proc.WaitForExit($TimeoutSeconds * 1000)
|
|
if (-not $exited) {
|
|
$timedOut = $true
|
|
# Kill the entire process tree (taskkill /T) to release any inherited pipe
|
|
# handles held by child processes (e.g. cmd.exe spawning ping.exe in tests,
|
|
# or vmrun spawning helper subprocesses). Plain proc.Kill() only kills the
|
|
# direct process; child processes keep the write end of stdout open.
|
|
& taskkill /F /T /PID $proc.Id 2>&1 | Out-Null
|
|
}
|
|
$sw.Stop()
|
|
|
|
$outputText = ''
|
|
if (-not $timedOut) {
|
|
# Process exited normally: drain stdout/stderr (up to 5s safety net).
|
|
[void][System.Threading.Tasks.Task]::WhenAll($stdoutTask, $stderrTask).Wait(5000)
|
|
$stdoutText = ''
|
|
$stderrText = ''
|
|
if ($stdoutTask.Status -eq [System.Threading.Tasks.TaskStatus]::RanToCompletion) {
|
|
$stdoutText = $stdoutTask.Result
|
|
}
|
|
if ($stderrTask.Status -eq [System.Threading.Tasks.TaskStatus]::RanToCompletion) {
|
|
$stderrText = $stderrTask.Result
|
|
}
|
|
$parts = @($stdoutText, $stderrText) | Where-Object { -not [string]::IsNullOrWhiteSpace($_) }
|
|
$outputText = ($parts | ForEach-Object { $_.TrimEnd() }) -join "`n"
|
|
}
|
|
# When timed-out: process was killed; output is irrelevant and potentially
|
|
# incomplete. Skip drain entirely — no point blocking for a killed process.
|
|
|
|
$exitCode = if ($timedOut) { -1 } else { $proc.ExitCode }
|
|
|
|
$result = [pscustomobject]@{
|
|
ExitCode = $exitCode
|
|
Output = $outputText
|
|
TimedOut = $timedOut
|
|
ElapsedSeconds = [math]::Round($sw.Elapsed.TotalSeconds, 1)
|
|
}
|
|
|
|
if ($timedOut -and $ThrowOnTimeout) {
|
|
throw "vmrun $Operation timed out after ${TimeoutSeconds}s"
|
|
}
|
|
if (-not $timedOut -and $exitCode -ne 0 -and $ThrowOnError) {
|
|
throw "vmrun $Operation failed (exit $exitCode): $outputText"
|
|
}
|
|
|
|
return $result
|
|
}
|
|
|
|
function Get-GuestIPAddress {
|
|
<#
|
|
.SYNOPSIS
|
|
Polls a running VMware guest VM until its IP address is available, then returns it.
|
|
.DESCRIPTION
|
|
Primary method: reads the 'ci-ip' guestVar written by ci-report-ip.service (Linux)
|
|
or the equivalent guest-side script via vmware-rpctool. This uses the VMware VMCI
|
|
channel and does not require TCP/IP connectivity from the host.
|
|
|
|
Fallback: vmrun getGuestIPAddress (reliable on Windows guests; may be slow on Linux).
|
|
|
|
Returns the detected IP as a plain string, or an empty string if the timeout expires.
|
|
.PARAMETER VmrunPath
|
|
Full path to vmrun.exe.
|
|
.PARAMETER VmxPath
|
|
Full path to the clone VM's .vmx file.
|
|
.PARAMETER TimeoutSeconds
|
|
Maximum seconds to poll before giving up. Default: 120.
|
|
.OUTPUTS
|
|
[string] Detected IPv4 address, or empty string on timeout.
|
|
#>
|
|
[OutputType([string])]
|
|
[CmdletBinding()]
|
|
param(
|
|
[Parameter(Mandatory)] [string] $VmrunPath,
|
|
[Parameter(Mandatory)] [string] $VmxPath,
|
|
[int] $TimeoutSeconds = 120
|
|
)
|
|
|
|
Write-Host "[Get-GuestIPAddress] Polling for VM IP (max ${TimeoutSeconds}s)..."
|
|
$deadline = (Get-Date).AddSeconds($TimeoutSeconds)
|
|
$attempt = 0
|
|
|
|
while ((Get-Date) -lt $deadline) {
|
|
$attempt++
|
|
|
|
# Primary: guestinfo.ci-ip written by ci-report-ip.service via vmware-rpctool.
|
|
# NOTE: vmrun readVariable guestVar reads WITHOUT the 'guestinfo.' prefix.
|
|
$savedEap = $ErrorActionPreference; $ErrorActionPreference = 'Continue'
|
|
$ipOut = (& $VmrunPath -T ws readVariable $VmxPath guestVar 'ci-ip' 2>&1).Trim()
|
|
$ipRc = $LASTEXITCODE
|
|
$ErrorActionPreference = $savedEap
|
|
|
|
if ($attempt -le 3 -or ($attempt % 10 -eq 0)) {
|
|
Write-Host "[Get-GuestIPAddress] [attempt $attempt] guestVar ci-ip: rc=$ipRc out='$ipOut'"
|
|
}
|
|
if ($ipRc -eq 0 -and $ipOut -match '^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$') {
|
|
Write-Host "[Get-GuestIPAddress] IP via guestinfo: $ipOut"
|
|
return $ipOut
|
|
}
|
|
|
|
# Fallback: vmrun getGuestIPAddress (VMware Tools / open-vm-tools required).
|
|
$savedEap = $ErrorActionPreference; $ErrorActionPreference = 'Continue'
|
|
$ipOut2 = (& $VmrunPath -T ws getGuestIPAddress $VmxPath 2>&1).Trim()
|
|
$ipRc2 = $LASTEXITCODE
|
|
$ErrorActionPreference = $savedEap
|
|
|
|
if ($ipRc2 -eq 0 -and $ipOut2 -match '^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$') {
|
|
Write-Host "[Get-GuestIPAddress] IP via getGuestIPAddress: $ipOut2"
|
|
return $ipOut2
|
|
}
|
|
|
|
Start-Sleep -Seconds 2
|
|
}
|
|
|
|
return ''
|
|
}
|
|
|
|
function Get-GuestDiagnostics {
|
|
<#
|
|
.SYNOPSIS
|
|
Best-effort collection of guest diagnostics after a build failure.
|
|
Never throws — always safe to call from a catch/finally block.
|
|
|
|
.PARAMETER IPAddress
|
|
IP address of the guest VM.
|
|
|
|
.PARAMETER GuestOS
|
|
'Windows' or 'Linux'.
|
|
|
|
.PARAMETER Credential
|
|
PSCredential for WinRM (Windows guests). Ignored for Linux.
|
|
|
|
.PARAMETER SshKeyPath
|
|
Path to SSH private key (Linux guests). Ignored for Windows.
|
|
|
|
.PARAMETER SshUser
|
|
SSH username (Linux guests). Default: ci_build.
|
|
|
|
.PARAMETER DestinationDir
|
|
Host directory where diagnostics files are written.
|
|
#>
|
|
[CmdletBinding()]
|
|
param(
|
|
[Parameter(Mandatory)] [string] $IPAddress,
|
|
[Parameter(Mandatory)] [ValidateSet('Windows','Linux')]
|
|
[string] $GuestOS,
|
|
[pscredential] $Credential,
|
|
[string] $SshKeyPath,
|
|
[string] $SshUser = 'ci_build',
|
|
[Parameter(Mandatory)] [string] $DestinationDir
|
|
)
|
|
|
|
Write-Host "[Diagnostics] Collecting guest diagnostics from $IPAddress ($GuestOS) -> $DestinationDir"
|
|
try {
|
|
if (-not (Test-Path $DestinationDir)) {
|
|
New-Item -ItemType Directory -Path $DestinationDir -Force | Out-Null
|
|
}
|
|
} catch {
|
|
Write-Warning "[Diagnostics] Could not create destination dir: $_"
|
|
return
|
|
}
|
|
|
|
if ($GuestOS -eq 'Windows') {
|
|
if (-not $Credential) {
|
|
Write-Warning "[Diagnostics] No Credential supplied for Windows guest — skipping."
|
|
return
|
|
}
|
|
try {
|
|
$sessionOption = New-CISessionOption
|
|
$session = New-PSSession -ComputerName $IPAddress `
|
|
-Credential $Credential `
|
|
-SessionOption $sessionOption `
|
|
-ErrorAction Stop
|
|
try {
|
|
# Collect: event log errors (last 50), running processes, disk usage
|
|
$diagScript = {
|
|
$out = [System.Text.StringBuilder]::new()
|
|
[void]$out.AppendLine("=== System Event Log (last 50 errors) ===")
|
|
Get-EventLog -LogName System -EntryType Error -Newest 50 -ErrorAction SilentlyContinue |
|
|
ForEach-Object { [void]$out.AppendLine("$($_.TimeGenerated) $($_.Source): $($_.Message)") }
|
|
[void]$out.AppendLine("`n=== Running Processes ===")
|
|
Get-Process -ErrorAction SilentlyContinue |
|
|
Sort-Object CPU -Descending |
|
|
Select-Object -First 30 |
|
|
ForEach-Object { [void]$out.AppendLine("$($_.Name) PID=$($_.Id) CPU=$($_.CPU)") }
|
|
[void]$out.AppendLine("`n=== Disk Free Space ===")
|
|
Get-PSDrive -PSProvider FileSystem -ErrorAction SilentlyContinue |
|
|
ForEach-Object { [void]$out.AppendLine("$($_.Name): Free=$([math]::Round($_.Free/1GB,1))GB Used=$([math]::Round($_.Used/1GB,1))GB") }
|
|
$out.ToString()
|
|
}
|
|
$diagText = Invoke-Command -Session $session -ScriptBlock $diagScript -ErrorAction Stop
|
|
$diagText | Set-Content -Path (Join-Path $DestinationDir 'guest-diag.txt') -Encoding UTF8
|
|
Write-Host "[Diagnostics] Windows diagnostics written."
|
|
|
|
# Best-effort: copy CI build log
|
|
$buildLog = 'C:\CI\build.log'
|
|
try {
|
|
Copy-Item -FromSession $session -Path $buildLog `
|
|
-Destination (Join-Path $DestinationDir 'build.log') `
|
|
-ErrorAction Stop
|
|
Write-Host "[Diagnostics] Build log copied."
|
|
} catch {
|
|
Write-Host "[Diagnostics] Build log not found in guest (skipping)."
|
|
}
|
|
} finally {
|
|
Remove-PSSession $session -ErrorAction SilentlyContinue
|
|
}
|
|
} catch {
|
|
Write-Warning "[Diagnostics] Windows guest diagnostics failed (non-fatal): $_"
|
|
}
|
|
}
|
|
else {
|
|
# Linux
|
|
if (-not $SshKeyPath) {
|
|
Write-Warning "[Diagnostics] No SshKeyPath supplied for Linux guest — skipping."
|
|
return
|
|
}
|
|
$sshOpts = @('-i', $SshKeyPath, '-o', 'StrictHostKeyChecking=accept-new',
|
|
'-o', 'ConnectTimeout=10', '-o', 'BatchMode=yes')
|
|
try {
|
|
# Collect: journalctl errors, ps, df
|
|
$diagCmd = "journalctl -p err -n 50 --no-pager 2>/dev/null; echo '=== ps ==='; ps aux --sort=-%cpu | head -30; echo '=== df ==='; df -h"
|
|
$savedEap = $ErrorActionPreference
|
|
$ErrorActionPreference = 'Continue'
|
|
$diagOutput = & ssh.exe @sshOpts "${SshUser}@${IPAddress}" $diagCmd 2>&1
|
|
$ErrorActionPreference = $savedEap
|
|
$diagOutput | Set-Content -Path (Join-Path $DestinationDir 'guest-diag.txt') -Encoding UTF8
|
|
Write-Host "[Diagnostics] Linux diagnostics written."
|
|
|
|
# Best-effort: copy CI build log
|
|
$scpSrc = "${SshUser}@${IPAddress}:/opt/ci/build/ci-build.log"
|
|
$scpDst = Join-Path $DestinationDir 'build.log'
|
|
$ErrorActionPreference = 'Continue'
|
|
& scp.exe @sshOpts $scpSrc $scpDst 2>&1 | Out-Null
|
|
$ErrorActionPreference = $savedEap
|
|
if (Test-Path $scpDst) { Write-Host "[Diagnostics] Build log copied." }
|
|
else { Write-Host "[Diagnostics] Build log not found in guest (skipping)." }
|
|
} catch {
|
|
Write-Warning "[Diagnostics] Linux guest diagnostics failed (non-fatal): $_"
|
|
}
|
|
}
|
|
}
|
|
|
|
Export-ModuleMember -Function New-CISessionOption, Resolve-VmrunPath, Invoke-Vmrun, Invoke-VmrunBounded, Get-GuestIPAddress, Get-GuestDiagnostics
|