#Requires -Version 5.1 <# .SYNOPSIS Shared helper functions for the Local CI/CD System scripts. .DESCRIPTION Import this module at the top of any CI script that needs WinRM session options or vmrun wrappers: Import-Module (Join-Path $PSScriptRoot '_Common.psm1') -Force Exported functions: New-CISessionOption — PSSessionOption for WinRM over self-signed TLS Resolve-VmrunPath — Validates vmrun.exe path, throws if missing Invoke-Vmrun — Uniform vmrun -T ws wrapper; returns ExitCode + Output #> Set-StrictMode -Version Latest $ErrorActionPreference = 'Stop' function New-CISessionOption { <# .SYNOPSIS Returns a PSSessionOption for WinRM HTTPS with self-signed certificate. .DESCRIPTION All CI build VMs use a self-signed TLS cert on WinRM port 5986. This option set skips CA, CN, and revocation checks — appropriate for an isolated lab network where PKI is not present. .OUTPUTS [System.Management.Automation.Remoting.PSSessionOption] #> [OutputType([System.Management.Automation.Remoting.PSSessionOption])] [Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSUseShouldProcessForStateChangingFunctions', '', Justification = 'New-CISessionOption creates an in-memory PSSessionOption object; no system state is changed.')] [CmdletBinding()] param() New-PSSessionOption -SkipCACheck -SkipCNCheck -SkipRevocationCheck } function Resolve-VmrunPath { <# .SYNOPSIS Validates that vmrun.exe exists at the specified path and returns it. .DESCRIPTION Throws a descriptive error if the file is not found, rather than letting a later & call fail with a less useful message. .PARAMETER VmrunPath Full path to vmrun.exe. Default: C:\Program Files (x86)\VMware\VMware Workstation\vmrun.exe .OUTPUTS [string] The validated VmrunPath. #> [OutputType([string])] [CmdletBinding()] param( [string] $VmrunPath = 'C:\Program Files (x86)\VMware\VMware Workstation\vmrun.exe' ) if (-not (Test-Path $VmrunPath -PathType Leaf)) { throw "vmrun.exe not found at: $VmrunPath`nInstall VMware Workstation Pro and update -VmrunPath." } return $VmrunPath } function Invoke-Vmrun { <# .SYNOPSIS Runs a vmrun -T ws command and returns output and exit code. .DESCRIPTION Uniform wrapper so all vmrun calls share the same -T ws flag and output capture pattern. Use -ThrowOnError for operations that must succeed (clone, start); omit it for best-effort operations (stop, getState). .PARAMETER VmrunPath Full path to vmrun.exe. .PARAMETER Operation vmrun sub-command: clone, start, stop, deleteVM, list, getState, listSnapshots, getGuestIPAddress, etc. .PARAMETER Arguments Additional positional arguments after the operation name. .PARAMETER ThrowOnError When set, throws if vmrun exits non-zero. .OUTPUTS [pscustomobject] with: ExitCode [int] — vmrun exit code Output [string[]] — combined stdout/stderr lines #> [OutputType([pscustomobject])] [CmdletBinding()] param( [Parameter(Mandatory)] [string] $VmrunPath, [Parameter(Mandatory)] [string] $Operation, [string[]] $Arguments = @(), [switch] $ThrowOnError ) $cmdArgs = @('-T', 'ws', $Operation) + $Arguments $output = & $VmrunPath @cmdArgs 2>&1 $exit = $LASTEXITCODE if ($ThrowOnError -and $exit -ne 0) { throw "vmrun $Operation failed (exit $exit): $($output -join '; ')" } return [pscustomobject]@{ ExitCode = $exit Output = $output } } function Invoke-VmrunBounded { <# .SYNOPSIS Runs a vmrun -T ws command with a hard wall-clock timeout. .DESCRIPTION Uses [System.Diagnostics.Process] directly (not Start-Process -PassThru) because Start-Process in PowerShell 5.1 does not reliably populate Process.ExitCode after WaitForExit(ms) when output is redirected. ReadToEndAsync() is used for stdout and stderr to prevent deadlock if output buffers fill up before the process exits. Use for lifecycle operations: start (180s), stop (60s), deleteVM (90s). Keep Invoke-Vmrun for fast queries: list, readVariable, getGuestIPAddress, getState. .PARAMETER VmrunPath Full path to vmrun.exe. .PARAMETER Operation vmrun sub-command (start, stop, deleteVM, clone, …). .PARAMETER Arguments Additional positional arguments after the operation name. .PARAMETER TimeoutSeconds Hard timeout in seconds. Process is killed if not finished within this time. Default: 120. .PARAMETER ThrowOnTimeout When set, throws if the timeout is reached before vmrun exits. .PARAMETER ThrowOnError When set, throws if vmrun exits with a non-zero code. .OUTPUTS [pscustomobject] with: ExitCode [int] — vmrun exit code (-1 if killed due to timeout) Output [string] — combined stdout/stderr text TimedOut [bool] — true if the process was killed for exceeding TimeoutSeconds ElapsedSeconds [double] — wall-clock seconds from start to exit/kill #> [OutputType([pscustomobject])] [CmdletBinding()] param( [Parameter(Mandatory)] [string] $VmrunPath, [Parameter(Mandatory)] [string] $Operation, [string[]] $Arguments = @(), [int] $TimeoutSeconds = 120, [switch] $ThrowOnTimeout, [switch] $ThrowOnError ) $cmdArgs = @('-T', 'ws', $Operation) + $Arguments # Quote tokens that contain spaces so the argument string is correct when # passed to ProcessStartInfo.Arguments as a single string. $quotedArgs = $cmdArgs | ForEach-Object { if ($_ -match '\s') { '"' + ($_ -replace '"', '""') + '"' } else { $_ } } $argStr = $quotedArgs -join ' ' $psi = New-Object System.Diagnostics.ProcessStartInfo $psi.FileName = $VmrunPath $psi.Arguments = $argStr $psi.UseShellExecute = $false $psi.CreateNoWindow = $true $psi.RedirectStandardOutput = $true $psi.RedirectStandardError = $true $proc = New-Object System.Diagnostics.Process $proc.StartInfo = $psi $sw = [System.Diagnostics.Stopwatch]::StartNew() $timedOut = $false [void]$proc.Start() # Begin async reads before WaitForExit to avoid deadlock when output buffers fill. $stdoutTask = $proc.StandardOutput.ReadToEndAsync() $stderrTask = $proc.StandardError.ReadToEndAsync() $exited = $proc.WaitForExit($TimeoutSeconds * 1000) if (-not $exited) { $timedOut = $true # Kill the entire process tree (taskkill /T) to release any inherited pipe # handles held by child processes (e.g. cmd.exe spawning ping.exe in tests, # or vmrun spawning helper subprocesses). Plain proc.Kill() only kills the # direct process; child processes keep the write end of stdout open. if ($null -ne $IsWindows -and $IsWindows -eq $false) { # .NET 5+ Kill(entireProcessTree) — works on Linux/macOS. $proc.Kill($true) } else { & taskkill /F /T /PID $proc.Id 2>&1 | Out-Null } } $sw.Stop() $outputText = '' if (-not $timedOut) { # Process exited normally: drain stdout/stderr (up to 5s safety net). [void][System.Threading.Tasks.Task]::WhenAll($stdoutTask, $stderrTask).Wait(5000) $stdoutText = '' $stderrText = '' if ($stdoutTask.Status -eq [System.Threading.Tasks.TaskStatus]::RanToCompletion) { $stdoutText = $stdoutTask.Result } if ($stderrTask.Status -eq [System.Threading.Tasks.TaskStatus]::RanToCompletion) { $stderrText = $stderrTask.Result } $parts = @($stdoutText, $stderrText) | Where-Object { -not [string]::IsNullOrWhiteSpace($_) } $outputText = ($parts | ForEach-Object { $_.TrimEnd() }) -join "`n" } # When timed-out: process was killed; output is irrelevant and potentially # incomplete. Skip drain entirely — no point blocking for a killed process. $exitCode = if ($timedOut) { -1 } else { $proc.ExitCode } $result = [pscustomobject]@{ ExitCode = $exitCode Output = $outputText TimedOut = $timedOut ElapsedSeconds = [math]::Round($sw.Elapsed.TotalSeconds, 1) } if ($timedOut -and $ThrowOnTimeout) { throw "vmrun $Operation timed out after ${TimeoutSeconds}s" } if (-not $timedOut -and $exitCode -ne 0 -and $ThrowOnError) { throw "vmrun $Operation failed (exit $exitCode): $outputText" } return $result } function Get-GuestIPAddress { <# .SYNOPSIS Polls a running VMware guest VM until its IP address is available, then returns it. .DESCRIPTION Primary method: reads the 'ci-ip' guestVar written by ci-report-ip.service (Linux) or the equivalent guest-side script via vmware-rpctool. This uses the VMware VMCI channel and does not require TCP/IP connectivity from the host. Fallback: vmrun getGuestIPAddress (reliable on Windows guests; may be slow on Linux). Returns the detected IP as a plain string, or an empty string if the timeout expires. .PARAMETER VmrunPath Full path to vmrun.exe. .PARAMETER VmxPath Full path to the clone VM's .vmx file. .PARAMETER TimeoutSeconds Maximum seconds to poll before giving up. Default: 120. .OUTPUTS [string] Detected IPv4 address, or empty string on timeout. #> [OutputType([string])] [CmdletBinding()] param( [Parameter(Mandatory)] [string] $VmrunPath, [Parameter(Mandatory)] [string] $VmxPath, [int] $TimeoutSeconds = 120 ) Write-Host "[Get-GuestIPAddress] Polling for VM IP (max ${TimeoutSeconds}s)..." $deadline = (Get-Date).AddSeconds($TimeoutSeconds) $attempt = 0 while ((Get-Date) -lt $deadline) { $attempt++ # Primary: guestinfo.ci-ip written by ci-report-ip.service via vmware-rpctool. # NOTE: vmrun readVariable guestVar reads WITHOUT the 'guestinfo.' prefix. $savedEap = $ErrorActionPreference; $ErrorActionPreference = 'Continue' $ipOut = (& $VmrunPath -T ws readVariable $VmxPath guestVar 'ci-ip' 2>&1).Trim() $ipRc = $LASTEXITCODE $ErrorActionPreference = $savedEap if ($attempt -le 3 -or ($attempt % 10 -eq 0)) { Write-Host "[Get-GuestIPAddress] [attempt $attempt] guestVar ci-ip: rc=$ipRc out='$ipOut'" } if ($ipRc -eq 0 -and $ipOut -match '^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$') { Write-Host "[Get-GuestIPAddress] IP via guestinfo: $ipOut" return $ipOut } # Fallback: vmrun getGuestIPAddress (VMware Tools / open-vm-tools required). $savedEap = $ErrorActionPreference; $ErrorActionPreference = 'Continue' $ipOut2 = (& $VmrunPath -T ws getGuestIPAddress $VmxPath 2>&1).Trim() $ipRc2 = $LASTEXITCODE $ErrorActionPreference = $savedEap if ($ipRc2 -eq 0 -and $ipOut2 -match '^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$') { Write-Host "[Get-GuestIPAddress] IP via getGuestIPAddress: $ipOut2" return $ipOut2 } Start-Sleep -Seconds 2 } return '' } function Get-GuestDiagnostics { <# .SYNOPSIS Best-effort collection of guest diagnostics after a build failure. Never throws — always safe to call from a catch/finally block. .PARAMETER IPAddress IP address of the guest VM. .PARAMETER GuestOS 'Windows' or 'Linux'. .PARAMETER Credential PSCredential for WinRM (Windows guests). Ignored for Linux. .PARAMETER SshKeyPath Path to SSH private key (Linux guests). Ignored for Windows. .PARAMETER SshUser SSH username (Linux guests). Default: ci_build. .PARAMETER DestinationDir Host directory where diagnostics files are written. #> [CmdletBinding()] param( [Parameter(Mandatory)] [string] $IPAddress, [Parameter(Mandatory)] [ValidateSet('Windows','Linux')] [string] $GuestOS, [pscredential] $Credential, [string] $SshKeyPath, [string] $SshUser = 'ci_build', [Parameter(Mandatory)] [string] $DestinationDir ) Write-Host "[Diagnostics] Collecting guest diagnostics from $IPAddress ($GuestOS) -> $DestinationDir" try { if (-not (Test-Path $DestinationDir)) { New-Item -ItemType Directory -Path $DestinationDir -Force | Out-Null } } catch { Write-Warning "[Diagnostics] Could not create destination dir: $_" return } if ($GuestOS -eq 'Windows') { if (-not $Credential) { Write-Warning "[Diagnostics] No Credential supplied for Windows guest — skipping." return } try { $sessionOption = New-CISessionOption $session = New-PSSession -ComputerName $IPAddress ` -Credential $Credential ` -SessionOption $sessionOption ` -ErrorAction Stop try { # Collect: event log errors (last 50), running processes, disk usage $diagScript = { $out = [System.Text.StringBuilder]::new() [void]$out.AppendLine("=== System Event Log (last 50 errors) ===") Get-EventLog -LogName System -EntryType Error -Newest 50 -ErrorAction SilentlyContinue | ForEach-Object { [void]$out.AppendLine("$($_.TimeGenerated) $($_.Source): $($_.Message)") } [void]$out.AppendLine("`n=== Running Processes ===") Get-Process -ErrorAction SilentlyContinue | Sort-Object CPU -Descending | Select-Object -First 30 | ForEach-Object { [void]$out.AppendLine("$($_.Name) PID=$($_.Id) CPU=$($_.CPU)") } [void]$out.AppendLine("`n=== Disk Free Space ===") Get-PSDrive -PSProvider FileSystem -ErrorAction SilentlyContinue | ForEach-Object { [void]$out.AppendLine("$($_.Name): Free=$([math]::Round($_.Free/1GB,1))GB Used=$([math]::Round($_.Used/1GB,1))GB") } $out.ToString() } $diagText = Invoke-Command -Session $session -ScriptBlock $diagScript -ErrorAction Stop $diagText | Set-Content -Path (Join-Path $DestinationDir 'guest-diag.txt') -Encoding UTF8 Write-Host "[Diagnostics] Windows diagnostics written." # Best-effort: copy CI build log $buildLog = 'C:\CI\build.log' try { Copy-Item -FromSession $session -Path $buildLog ` -Destination (Join-Path $DestinationDir 'build.log') ` -ErrorAction Stop Write-Host "[Diagnostics] Build log copied." } catch { Write-Host "[Diagnostics] Build log not found in guest (skipping)." } } finally { Remove-PSSession $session -ErrorAction SilentlyContinue } } catch { Write-Warning "[Diagnostics] Windows guest diagnostics failed (non-fatal): $_" } } else { # Linux if (-not $SshKeyPath) { Write-Warning "[Diagnostics] No SshKeyPath supplied for Linux guest — skipping." return } $sshOpts = @('-i', $SshKeyPath, '-o', 'StrictHostKeyChecking=no', '-o', 'UserKnownHostsFile=NUL', '-o', 'ConnectTimeout=10', '-o', 'BatchMode=yes') try { # Collect: journalctl errors, ps, df $diagCmd = "journalctl -p err -n 50 --no-pager 2>/dev/null; echo '=== ps ==='; ps aux --sort=-%cpu | head -30; echo '=== df ==='; df -h" $savedEap = $ErrorActionPreference $ErrorActionPreference = 'Continue' $diagOutput = & ssh.exe @sshOpts "${SshUser}@${IPAddress}" $diagCmd 2>&1 $ErrorActionPreference = $savedEap $diagOutput | Set-Content -Path (Join-Path $DestinationDir 'guest-diag.txt') -Encoding UTF8 Write-Host "[Diagnostics] Linux diagnostics written." # Best-effort: copy CI build log $scpSrc = "${SshUser}@${IPAddress}:/opt/ci/build/ci-build.log" $scpDst = Join-Path $DestinationDir 'build.log' $ErrorActionPreference = 'Continue' & scp.exe @sshOpts $scpSrc $scpDst 2>&1 | Out-Null $ErrorActionPreference = $savedEap if (Test-Path $scpDst) { Write-Host "[Diagnostics] Build log copied." } else { Write-Host "[Diagnostics] Build log not found in guest (skipping)." } } catch { Write-Warning "[Diagnostics] Linux guest diagnostics failed (non-fatal): $_" } } } Export-ModuleMember -Function New-CISessionOption, Resolve-VmrunPath, Invoke-Vmrun, Invoke-VmrunBounded, Get-GuestIPAddress, Get-GuestDiagnostics