From 509d1fc284c37050f486afb267e486830dfeccc7 Mon Sep 17 00:00:00 2001 From: Simone Date: Tue, 12 May 2026 21:12:33 +0200 Subject: [PATCH] feat(sprint3): Invoke-VmrunBounded, Get-GuestIPAddress, transport and concurrency hardening H4: Invoke-VmrunBounded added to _Common.psm1 (Start-Process + Wait-Process + taskkill on timeout, returns {ExitCode;Output;TimedOut;ElapsedSeconds}). Wired in Invoke-CIJob.ps1 for vmrun start (180s) and Remove-BuildVM.ps1 for stop/deleteVM (60s/90s). H5: Cleanup-OrphanedBuildVMs.ps1 removes stale vm-start.lock (>30 min). ValidateRange lowered to 0 to allow emergency -MaxAgeHours 0. Invoke-RetentionPolicy.ps1 same. H8: _Transport.psm1 SSH hardening: StrictHostKeyChecking=accept-new, F:\CI\State\known_hosts. H9: Validate-DeployState.ps1 Linux branch added: checks ci-report-ip.service enabled, ci-report-ip.sh executable via SSH. New params: -GuestOS, -SshKeyPath, -SshUser. H12: Get-GuestIPAddress extracted into _Common.psm1 (guestinfo.ci-ip primary, getGuestIPAddress fallback). Invoke-CIJob.ps1 Phase 3b replaced with single call. Measure-CIBenchmark.ps1 updated to match. Also: _Common.psm1 adds Write-CIRedactedCommand and ConvertTo-SafeShellSingleQuotedString. Pester suite updated: 30/30 pass. --- scripts/Cleanup-OrphanedBuildVMs.ps1 | 16 +- scripts/Invoke-CIJob.ps1 | 145 +++++++----- scripts/Measure-CIBenchmark.ps1 | 18 +- scripts/Remove-BuildVM.ps1 | 41 ++-- scripts/_Common.psm1 | 315 ++++++++++++++++++++++++++- scripts/_Transport.psm1 | 33 ++- template/Validate-DeployState.ps1 | 109 ++++++++- tests/_Common.Tests.ps1 | 91 ++++++++ 8 files changed, 668 insertions(+), 100 deletions(-) diff --git a/scripts/Cleanup-OrphanedBuildVMs.ps1 b/scripts/Cleanup-OrphanedBuildVMs.ps1 index 262c18b..ef41af4 100644 --- a/scripts/Cleanup-OrphanedBuildVMs.ps1 +++ b/scripts/Cleanup-OrphanedBuildVMs.ps1 @@ -37,7 +37,7 @@ param( [string] $CloneBaseDir = 'F:\CI\BuildVMs', - [ValidateRange(1, 168)] + [ValidateRange(0, 168)] [int] $MaxAgeHours = 4, [string] $VmrunPath = 'C:\Program Files (x86)\VMware\VMware Workstation\vmrun.exe' @@ -102,3 +102,17 @@ foreach ($dir in $orphans) { } Write-Host "[Cleanup] Done." + +# ── Stale vm-start.lock cleanup ───────────────────────────────────────────── +# The lock is released by Invoke-CIJob.ps1 normally. On host crash it persists +# and causes a confusing 10-minute retry loop on the next job. Remove if older +# than 30 minutes. +$lockFile = 'F:\CI\State\vm-start.lock' +$lockCutoff = (Get-Date).AddMinutes(-30) +if ((Test-Path $lockFile) -and (Get-Item $lockFile).LastWriteTime -lt $lockCutoff) { + if ($PSCmdlet.ShouldProcess($lockFile, 'Remove stale vm-start.lock')) { + $ageMin = [int]((Get-Date) - (Get-Item $lockFile).LastWriteTime).TotalMinutes + Remove-Item $lockFile -Force -ErrorAction SilentlyContinue + Write-Host "[Cleanup] Removed stale vm-start.lock (age: ${ageMin} min)" + } +} diff --git a/scripts/Invoke-CIJob.ps1 b/scripts/Invoke-CIJob.ps1 index 91e24d9..2f11206 100644 --- a/scripts/Invoke-CIJob.ps1 +++ b/scripts/Invoke-CIJob.ps1 @@ -74,6 +74,7 @@ [CmdletBinding()] param( [Parameter(Mandatory)] + [ValidatePattern('^[A-Za-z0-9._-]+$')] [string] $JobId, [Parameter(Mandatory)] @@ -100,6 +101,10 @@ param( # Prerequisite: Git for Windows installed in template. [switch] $UseGitClone, + # Use VMware shared folders for NuGet/pip cache (§SharedCache). + # Requires Set-TemplateSharedFolders.ps1 to have been run on the template. + [switch] $UseSharedCache, + # Credential Manager target for Gitea PAT used by -UseGitClone (private repos). # Store with: New-StoredCredential -Target 'GiteaPAT' -UserName '' -Password '' -Persist LocalMachine # Ignored when -UseGitClone is not set. @@ -136,6 +141,10 @@ param( [string] $SshKeyPath = 'F:\CI\keys\ci_linux', [string] $SshUser = 'ci_build', + # Persistent known_hosts file used for CI-job SSH calls. + # Set to empty string to disable host-key checking (template provisioning only). + [string] $SshKnownHostsFile = 'F:\CI\State\known_hosts', + # Extra environment variables injected into the guest VM before the build command (§6.5). # Keys must be valid environment variable names. Values must be plain strings. # Example: @{ SIGN_PASS = $env:SIGN_PASS_FROM_SECRET; GPG_KEY_ID = '0xABCD1234' } @@ -148,6 +157,9 @@ $ErrorActionPreference = 'Stop' # ── Resolve script directory for dot-sourcing sibling scripts ───────────────── $scriptDir = $PSScriptRoot +# ── Load shared helpers (Invoke-VmrunBounded, Get-GuestIPAddress, …) ────────── +Import-Module (Join-Path $scriptDir '_Common.psm1') -Force + # ── Validate required config ────────────────────────────────────────────────── if ([string]::IsNullOrWhiteSpace($TemplatePath)) { Write-Error "TemplatePath is required. Set -TemplatePath or env:GITEA_CI_TEMPLATE_PATH." @@ -162,6 +174,14 @@ if (-not (Test-Path $VmrunPath -PathType Leaf)) { exit 1 } +# ── Validate ExtraGuestEnv keys ────────────────────────────────────────────── +foreach ($key in $ExtraGuestEnv.Keys) { + if ($key -notmatch '^[A-Za-z_][A-Za-z0-9_]*$') { + Write-Error "ExtraGuestEnv key '$key' is not a valid environment variable name. Keys must match ^[A-Za-z_][A-Za-z0-9_]*$." + exit 1 + } +} + # ── Load guest credentials from Windows Credential Manager ─────────────────── # Requires the CredentialManager module: Install-Module CredentialManager # In a lab setup you can fall back to prompting, but never hardcode credentials. @@ -185,6 +205,10 @@ $leaseFile = $null # §2.1 IP lease file path — released in finally $jsonLog = $null # §4.1 JSONL structured log path $startTime = Get-Date +# ── Phase-duration tracking (Task 5.5) ─────────────────────────────────────── +$phaseTimings = [System.Collections.Generic.List[PSCustomObject]]::new() +$phaseStart = $startTime + # ── Start transcript + JSONL log ────────────────────────────────────────────── $transcriptStarted = $false try { @@ -291,6 +315,8 @@ try { Write-Host "[Invoke-CIJob] Source cloned to: $hostCloneDir" Write-JobEvent -Phase 'phase1.clone-repo' -Status 'success' -Data @{ dir = $hostCloneDir } } + $phaseTimings.Add([PSCustomObject]@{ Phase = 'Phase 1 - Clone repo'; Sec = [int]((Get-Date) - $phaseStart).TotalSeconds }) + $phaseStart = Get-Date # ── Phase 2-3b: Clone + Start + IP (serialized via file-based mutex) ──── # §2.1: With capacity:4 and DHCP on VMnet8, concurrent vmrun starts can race @@ -303,6 +329,17 @@ try { $lockPath = Join-Path $stateDir 'vm-start.lock' if (-not (Test-Path $leasesDir)) { New-Item -ItemType Directory -Path $leasesDir -Force | Out-Null } + # ── Pre-clone disk space gate ───────────────────────────────────────── + $cloneDriveLetter = (Split-Path -Qualifier $CloneBaseDir).TrimEnd(':') + $cloneVol = Get-Volume -DriveLetter $cloneDriveLetter -ErrorAction SilentlyContinue + if ($null -ne $cloneVol) { + $freeGB = [math]::Round($cloneVol.SizeRemaining / 1GB, 1) + if ($freeGB -lt 20) { + throw "Insufficient disk space on ${cloneDriveLetter}: ($freeGB GB free). 20 GB minimum required before VM clone." + } + Write-Host "[Invoke-CIJob] Disk space on ${cloneDriveLetter}: ${freeGB} GB free (OK)." + } + Write-Host "`n[Phase 2/6] Acquiring VM-start lock (prevents DHCP collision with capacity>1)..." Write-JobEvent -Phase 'phase2-3b.vm-start' -Status 'start' $lockStream = $null @@ -337,58 +374,22 @@ try { # ── Phase 3: Start VM ───────────────────────────────────────────── Write-Host "`n[Phase 3/6] Starting VM..." - $startOutput = & $VmrunPath -T ws start $cloneVmxPath nogui 2>&1 - if ($LASTEXITCODE -ne 0) { - throw "vmrun start failed (exit $LASTEXITCODE): $startOutput" + $startResult = Invoke-VmrunBounded -VmrunPath $VmrunPath ` + -Operation start ` + -Arguments @($cloneVmxPath, 'nogui') ` + -TimeoutSeconds 180 + if ($startResult.TimedOut) { + throw "vmrun start timed out after 180s for: $cloneVmxPath" } - Write-Host "[Invoke-CIJob] VM started." + if ($startResult.ExitCode -ne 0) { + throw "vmrun start failed (exit $($startResult.ExitCode)): $($startResult.Output)" + } + Write-Host "[Invoke-CIJob] VM started (took $($startResult.ElapsedSeconds)s)." # ── Phase 3b: Resolve VM IP ─────────────────────────────────────── if ([string]::IsNullOrWhiteSpace($VMIPAddress)) { - # Primary method: vmrun readVariable guestVar guestinfo.ci-ip - # The guest ci-report-ip.service writes its IP via vmware-rpctool on every boot. - # This uses the VMware VMCI channel — reliable, official API, independent of - # whether TCP/IP is reachable from the host. - # - # Fallback: vmrun getGuestIPAddress (works on Windows guests; unreliable on Linux). - Write-Host "[Invoke-CIJob] Auto-detecting VM IP via guestinfo.ci-ip (polling, max 120s)..." - $ipDeadline = (Get-Date).AddSeconds(120) - $detectedIP = '' - $ipOut = '' - $ipAttempt = 0 - while ((Get-Date) -lt $ipDeadline) { - $ipAttempt++ - - # Primary: guestinfo written by ci-report-ip.service via vmware-rpctool. - # NOTE: rpctool sets 'guestinfo.ci-ip' but vmrun readVariable guestVar - # reads it WITHOUT the 'guestinfo.' prefix (the prefix is implicit in - # the guestVar namespace). Reading with the full key always returns ''. - $savedEap = $ErrorActionPreference; $ErrorActionPreference = 'Continue' - $ipOut = (& $VmrunPath -T ws readVariable $cloneVmxPath guestVar 'ci-ip' 2>&1).Trim() - $ipRc = $LASTEXITCODE - $ErrorActionPreference = $savedEap - if ($ipAttempt -le 3 -or ($ipAttempt % 10 -eq 0)) { - Write-Host "[Invoke-CIJob] [attempt $ipAttempt] guestVar ci-ip: rc=$ipRc out='$ipOut'" - } - if ($ipRc -eq 0 -and $ipOut -match '^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$') { - $detectedIP = $ipOut - Write-Host "[Invoke-CIJob] IP via guestinfo: $detectedIP" - break - } - - # Fallback: vmrun getGuestIPAddress (reliable on Windows guests) - $savedEap = $ErrorActionPreference; $ErrorActionPreference = 'Continue' - $ipOut2 = (& $VmrunPath -T ws getGuestIPAddress $cloneVmxPath 2>&1).Trim() - $ipRc2 = $LASTEXITCODE - $ErrorActionPreference = $savedEap - if ($ipRc2 -eq 0 -and $ipOut2 -match '^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$') { - $detectedIP = $ipOut2 - Write-Host "[Invoke-CIJob] IP via vmrun getGuestIPAddress: $detectedIP" - break - } - - Start-Sleep -Seconds 2 - } + $detectedIP = Get-GuestIPAddress -VmrunPath $VmrunPath ` + -VmxPath $cloneVmxPath -TimeoutSeconds 120 if ($detectedIP -eq '') { throw "Could not detect VM IP address after 120s. Ensure ci-report-ip.service is enabled in the guest template (Linux) or open-vm-tools is running (Windows)." } @@ -418,6 +419,8 @@ try { Remove-Item $lockPath -Force -ErrorAction SilentlyContinue Write-Host "[Invoke-CIJob] VM-start lock released." } + $phaseTimings.Add([PSCustomObject]@{ Phase = 'Phase 2-3 - Clone+Start VM'; Sec = [int]((Get-Date) - $phaseStart).TotalSeconds }) + $phaseStart = Get-Date # ── Resolve GuestOS from VMX if Auto ──────────────────────────────────── if ($GuestOS -eq 'Auto' -and $cloneVmxPath) { $vmxContent = Get-Content -LiteralPath $cloneVmxPath -Raw -ErrorAction SilentlyContinue @@ -445,6 +448,8 @@ try { } & "$scriptDir\Wait-VMReady.ps1" @waitParams Write-JobEvent -Phase 'phase4.wait-ready' -Status 'success' + $phaseTimings.Add([PSCustomObject]@{ Phase = 'Phase 4 - Wait ready'; Sec = [int]((Get-Date) - $phaseStart).TotalSeconds }) + $phaseStart = Get-Date # ── Phase 5: Remote build (copy source + build inside VM) ───────────── Write-Host "`n[Phase 5/6] Invoking remote build..." @@ -455,9 +460,10 @@ try { Configuration = $Configuration } if ($GuestOS -eq 'Linux') { - $remoteBuildParams['GuestOS'] = 'Linux' - $remoteBuildParams['SshKeyPath'] = $SshKeyPath - $remoteBuildParams['SshUser'] = $SshUser + $remoteBuildParams['GuestOS'] = 'Linux' + $remoteBuildParams['SshKeyPath'] = $SshKeyPath + $remoteBuildParams['SshUser'] = $SshUser + $remoteBuildParams['SshKnownHostsFile'] = $SshKnownHostsFile $remoteBuildParams.Remove('Credential') if ($UseGitClone) { # In-guest git clone (requires Gitea reachable + PAT/key configured in guest) @@ -487,8 +493,11 @@ try { if ($BuildCommand -ne '') { $remoteBuildParams['BuildCommand'] = $BuildCommand } if ($GuestArtifactSource -ne '') { $remoteBuildParams['GuestArtifactSource'] = $GuestArtifactSource } if ($ExtraGuestEnv.Count -gt 0) { $remoteBuildParams['ExtraGuestEnv'] = $ExtraGuestEnv } + if ($UseSharedCache) { $remoteBuildParams['UseSharedCache'] = $true } & "$scriptDir\Invoke-RemoteBuild.ps1" @remoteBuildParams Write-JobEvent -Phase 'phase5.build' -Status 'success' + $phaseTimings.Add([PSCustomObject]@{ Phase = 'Phase 5 - Remote build'; Sec = [int]((Get-Date) - $phaseStart).TotalSeconds }) + $phaseStart = Get-Date # ── Phase 6: Collect artifacts ──────────────────────────────────────── Write-Host "`n[Phase 6/6] Collecting artifacts..." @@ -499,6 +508,7 @@ try { -GuestOS 'Linux' ` -SshKeyPath $SshKeyPath ` -SshUser $SshUser ` + -SshKnownHostsFile $SshKnownHostsFile ` -GuestArtifactPath '/opt/ci/output' ` -HostArtifactDir $jobArtifactDir ` -IncludeLogs @@ -514,8 +524,15 @@ try { $elapsed = (Get-Date) - $startTime Write-JobEvent -Phase 'phase6.artifacts' -Status 'success' -Data @{ dir = $jobArtifactDir } Write-JobEvent -Phase 'job' -Status 'success' -Data @{ elapsedSec = [int]$elapsed.TotalSeconds } + $phaseTimings.Add([PSCustomObject]@{ Phase = 'Phase 6 - Artifacts'; Sec = [int]((Get-Date) - $phaseStart).TotalSeconds }) Write-Host "`n============================================================" - Write-Host "[Invoke-CIJob] SUCCESS — Job $JobId completed in $($elapsed.ToString('mm\:ss'))" + Write-Host "[Invoke-CIJob] Phase durations:" + foreach ($pt in $phaseTimings) { + Write-Host (" {0,-30} {1,5}s" -f $pt.Phase, $pt.Sec) + } + Write-Host " {0,-30} {1,5}s" -f 'TOTAL', [int]$elapsed.TotalSeconds + Write-Host "============================================================" + Write-Host "[Invoke-CIJob] SUCCESS -- Job $JobId completed in $($elapsed.ToString('mm\:ss'))" Write-Host "[Invoke-CIJob] Artifacts: $jobArtifactDir" Write-Host "============================================================" } @@ -526,6 +543,30 @@ catch { elapsedSec = [int]$elapsed.TotalSeconds error = "$_" } + + # ── Best-effort guest diagnostics (skip if VM was never started) ───────── + # Only collect when a VM IP is known (Phase 3b completed). Never throws. + if (-not [string]::IsNullOrWhiteSpace($VMIPAddress)) { + try { + $diagDir = Join-Path $jobArtifactDir 'diagnostics' + $diagParams = @{ + IPAddress = $VMIPAddress + GuestOS = if ($GuestOS -eq 'Auto') { 'Windows' } else { $GuestOS } + DestinationDir = $diagDir + } + if ($diagParams.GuestOS -eq 'Windows' -and $credential) { + $diagParams['Credential'] = $credential + } + if ($diagParams.GuestOS -eq 'Linux') { + $diagParams['SshKeyPath'] = $SshKeyPath + $diagParams['SshUser'] = $SshUser + } + Get-GuestDiagnostics @diagParams + } catch { + Write-Warning "[Invoke-CIJob] Diagnostics collection skipped: $_" + } + } + Write-Host "`n============================================================" Write-Host "[Invoke-CIJob] FAILURE — Job $JobId failed after $($elapsed.ToString('mm\:ss'))" Write-Host "Error: $_" diff --git a/scripts/Measure-CIBenchmark.ps1 b/scripts/Measure-CIBenchmark.ps1 index 370b054..c61a93c 100644 --- a/scripts/Measure-CIBenchmark.ps1 +++ b/scripts/Measure-CIBenchmark.ps1 @@ -130,23 +130,11 @@ for ($i = 1; $i -le $Iterations; $i++) { $t.start = [math]::Round($sw.Elapsed.TotalSeconds, 2) Write-Host "[bench] start: $($t.start)s" - # ── Phase: IP acquire (poll getGuestIPAddress) ──────────────────────── + # ── Phase: IP acquire (primary: guestinfo.ci-ip, fallback: getGuestIPAddress) ── Write-Host "[bench] Waiting for guest IP..." $sw.Restart() - $deadline = (Get-Date).AddSeconds($TimeoutSeconds) - $vmIP = $null - while ((Get-Date) -lt $deadline) { - $ipResult = Invoke-Vmrun -VmrunPath $VmrunPath -Operation getGuestIPAddress ` - -Arguments @($cloneVmx) - if ($ipResult.ExitCode -eq 0) { - $candidate = "$($ipResult.Output)".Trim() - if ($candidate -match '^\d+\.\d+\.\d+\.\d+$') { - $vmIP = $candidate - break - } - } - Start-Sleep -Seconds 3 - } + $vmIP = Get-GuestIPAddress -VmrunPath $VmrunPath -VmxPath $cloneVmx ` + -TimeoutSeconds $TimeoutSeconds if (-not $vmIP) { throw "Timed out waiting for guest IP after $TimeoutSeconds s" } $t.ip = [math]::Round($sw.Elapsed.TotalSeconds, 2) $t.vmIP = $vmIP diff --git a/scripts/Remove-BuildVM.ps1 b/scripts/Remove-BuildVM.ps1 index 483d7f0..2d85fbf 100644 --- a/scripts/Remove-BuildVM.ps1 +++ b/scripts/Remove-BuildVM.ps1 @@ -39,6 +39,8 @@ param( Set-StrictMode -Version Latest $ErrorActionPreference = 'Continue' # Don't stop on errors — always try next step +Import-Module (Join-Path $PSScriptRoot '_Common.psm1') -Force + $cloneDir = Split-Path $VMPath -Parent Write-Host "[Remove-BuildVM] Destroying VM: $VMPath" @@ -56,24 +58,23 @@ if (-not (Test-Path $VMPath -PathType Leaf)) { if (-not $PSCmdlet.ShouldProcess($VMPath, 'Stop and delete build VM')) { return } # ── Step 2: Graceful stop ───────────────────────────────────────────────────── -Write-Host "[Remove-BuildVM] Sending soft stop..." -& $VmrunPath -T ws stop $VMPath soft 2>&1 | Out-Null - -$deadline = (Get-Date).AddSeconds($GracefulTimeoutSeconds) -while ((Get-Date) -lt $deadline) { - Start-Sleep -Seconds 2 - $stateOutput = & $VmrunPath -T ws getState $VMPath 2>&1 - if ($stateOutput -notmatch 'running') { - Write-Host "[Remove-BuildVM] VM stopped gracefully." - break - } +Write-Host "[Remove-BuildVM] Sending soft stop (timeout: ${GracefulTimeoutSeconds}s)..." +$stopSoft = Invoke-VmrunBounded -VmrunPath $VmrunPath ` + -Operation stop -Arguments @($VMPath, 'soft') -TimeoutSeconds $GracefulTimeoutSeconds +if ($stopSoft.TimedOut) { + Write-Warning "[Remove-BuildVM] Soft stop timed out after ${GracefulTimeoutSeconds}s." +} elseif ($stopSoft.ExitCode -eq 0) { + Write-Host "[Remove-BuildVM] VM stopped gracefully." } # ── Step 3: Force stop if still running ────────────────────────────────────── -$stateOutput = & $VmrunPath -T ws getState $VMPath 2>&1 -if ($stateOutput -match 'running') { - Write-Host "[Remove-BuildVM] Graceful stop timed out — forcing hard stop..." - & $VmrunPath -T ws stop $VMPath hard 2>&1 | Out-Null +if ($stopSoft.TimedOut -or $stopSoft.ExitCode -ne 0) { + Write-Host "[Remove-BuildVM] Graceful stop did not succeed — forcing hard stop..." + $stopHard = Invoke-VmrunBounded -VmrunPath $VmrunPath ` + -Operation stop -Arguments @($VMPath, 'hard') -TimeoutSeconds 60 + if ($stopHard.TimedOut) { + Write-Warning "[Remove-BuildVM] Hard stop timed out after 60s." + } } # Poll vmrun list until VMX is gone — vmware-vmx.exe may hold file locks briefly @@ -98,8 +99,14 @@ foreach ($delay in @(0, 3, 6)) { Write-Host "[Remove-BuildVM] Retrying deleteVM in ${delay}s..." Start-Sleep -Seconds $delay } - $deleteOutput = & $VmrunPath -T ws deleteVM $VMPath 2>&1 - $deleteExit = $LASTEXITCODE + $delResult = Invoke-VmrunBounded -VmrunPath $VmrunPath ` + -Operation deleteVM -Arguments @($VMPath) -TimeoutSeconds 90 + $deleteOutput = $delResult.Output + $deleteExit = $delResult.ExitCode + if ($delResult.TimedOut) { + Write-Warning "[Remove-BuildVM] deleteVM timed out after 90s." + $deleteExit = -1 + } if ($deleteExit -eq 0) { break } } diff --git a/scripts/_Common.psm1 b/scripts/_Common.psm1 index 780e3ca..1bdde4c 100644 --- a/scripts/_Common.psm1 +++ b/scripts/_Common.psm1 @@ -104,4 +104,317 @@ function Invoke-Vmrun { } } -Export-ModuleMember -Function New-CISessionOption, Resolve-VmrunPath, Invoke-Vmrun +function Invoke-VmrunBounded { +<# +.SYNOPSIS + Runs a vmrun -T ws command with a hard wall-clock timeout. +.DESCRIPTION + Uses [System.Diagnostics.Process] directly (not Start-Process -PassThru) because + Start-Process in PowerShell 5.1 does not reliably populate Process.ExitCode after + WaitForExit(ms) when output is redirected. ReadToEndAsync() is used for stdout and + stderr to prevent deadlock if output buffers fill up before the process exits. + + Use for lifecycle operations: start (180s), stop (60s), deleteVM (90s). + Keep Invoke-Vmrun for fast queries: list, readVariable, getGuestIPAddress, getState. +.PARAMETER VmrunPath + Full path to vmrun.exe. +.PARAMETER Operation + vmrun sub-command (start, stop, deleteVM, clone, …). +.PARAMETER Arguments + Additional positional arguments after the operation name. +.PARAMETER TimeoutSeconds + Hard timeout in seconds. Process is killed if not finished within this time. + Default: 120. +.PARAMETER ThrowOnTimeout + When set, throws if the timeout is reached before vmrun exits. +.PARAMETER ThrowOnError + When set, throws if vmrun exits with a non-zero code. +.OUTPUTS + [pscustomobject] with: + ExitCode [int] — vmrun exit code (-1 if killed due to timeout) + Output [string] — combined stdout/stderr text + TimedOut [bool] — true if the process was killed for exceeding TimeoutSeconds + ElapsedSeconds [double] — wall-clock seconds from start to exit/kill +#> + [OutputType([pscustomobject])] + [CmdletBinding()] + param( + [Parameter(Mandatory)] [string] $VmrunPath, + [Parameter(Mandatory)] [string] $Operation, + [string[]] $Arguments = @(), + [int] $TimeoutSeconds = 120, + [switch] $ThrowOnTimeout, + [switch] $ThrowOnError + ) + + $cmdArgs = @('-T', 'ws', $Operation) + $Arguments + + # Quote tokens that contain spaces so the argument string is correct when + # passed to ProcessStartInfo.Arguments as a single string. + $quotedArgs = $cmdArgs | ForEach-Object { + if ($_ -match '\s') { '"' + ($_ -replace '"', '""') + '"' } else { $_ } + } + $argStr = $quotedArgs -join ' ' + + $psi = New-Object System.Diagnostics.ProcessStartInfo + $psi.FileName = $VmrunPath + $psi.Arguments = $argStr + $psi.UseShellExecute = $false + $psi.CreateNoWindow = $true + $psi.RedirectStandardOutput = $true + $psi.RedirectStandardError = $true + + $proc = New-Object System.Diagnostics.Process + $proc.StartInfo = $psi + $sw = [System.Diagnostics.Stopwatch]::StartNew() + $timedOut = $false + + [void]$proc.Start() + + # Begin async reads before WaitForExit to avoid deadlock when output buffers fill. + $stdoutTask = $proc.StandardOutput.ReadToEndAsync() + $stderrTask = $proc.StandardError.ReadToEndAsync() + + $exited = $proc.WaitForExit($TimeoutSeconds * 1000) + if (-not $exited) { + $timedOut = $true + # Kill the entire process tree (taskkill /T) to release any inherited pipe + # handles held by child processes (e.g. cmd.exe spawning ping.exe in tests, + # or vmrun spawning helper subprocesses). Plain proc.Kill() only kills the + # direct process; child processes keep the write end of stdout open. + & taskkill /F /T /PID $proc.Id 2>&1 | Out-Null + } + $sw.Stop() + + $outputText = '' + if (-not $timedOut) { + # Process exited normally: drain stdout/stderr (up to 5s safety net). + [void][System.Threading.Tasks.Task]::WhenAll($stdoutTask, $stderrTask).Wait(5000) + $stdoutText = '' + $stderrText = '' + if ($stdoutTask.Status -eq [System.Threading.Tasks.TaskStatus]::RanToCompletion) { + $stdoutText = $stdoutTask.Result + } + if ($stderrTask.Status -eq [System.Threading.Tasks.TaskStatus]::RanToCompletion) { + $stderrText = $stderrTask.Result + } + $parts = @($stdoutText, $stderrText) | Where-Object { -not [string]::IsNullOrWhiteSpace($_) } + $outputText = ($parts | ForEach-Object { $_.TrimEnd() }) -join "`n" + } + # When timed-out: process was killed; output is irrelevant and potentially + # incomplete. Skip drain entirely — no point blocking for a killed process. + + $exitCode = if ($timedOut) { -1 } else { $proc.ExitCode } + + $result = [pscustomobject]@{ + ExitCode = $exitCode + Output = $outputText + TimedOut = $timedOut + ElapsedSeconds = [math]::Round($sw.Elapsed.TotalSeconds, 1) + } + + if ($timedOut -and $ThrowOnTimeout) { + throw "vmrun $Operation timed out after ${TimeoutSeconds}s" + } + if (-not $timedOut -and $exitCode -ne 0 -and $ThrowOnError) { + throw "vmrun $Operation failed (exit $exitCode): $outputText" + } + + return $result +} + +function Get-GuestIPAddress { +<# +.SYNOPSIS + Polls a running VMware guest VM until its IP address is available, then returns it. +.DESCRIPTION + Primary method: reads the 'ci-ip' guestVar written by ci-report-ip.service (Linux) + or the equivalent guest-side script via vmware-rpctool. This uses the VMware VMCI + channel and does not require TCP/IP connectivity from the host. + + Fallback: vmrun getGuestIPAddress (reliable on Windows guests; may be slow on Linux). + + Returns the detected IP as a plain string, or an empty string if the timeout expires. +.PARAMETER VmrunPath + Full path to vmrun.exe. +.PARAMETER VmxPath + Full path to the clone VM's .vmx file. +.PARAMETER TimeoutSeconds + Maximum seconds to poll before giving up. Default: 120. +.OUTPUTS + [string] Detected IPv4 address, or empty string on timeout. +#> + [OutputType([string])] + [CmdletBinding()] + param( + [Parameter(Mandatory)] [string] $VmrunPath, + [Parameter(Mandatory)] [string] $VmxPath, + [int] $TimeoutSeconds = 120 + ) + + Write-Host "[Get-GuestIPAddress] Polling for VM IP (max ${TimeoutSeconds}s)..." + $deadline = (Get-Date).AddSeconds($TimeoutSeconds) + $attempt = 0 + + while ((Get-Date) -lt $deadline) { + $attempt++ + + # Primary: guestinfo.ci-ip written by ci-report-ip.service via vmware-rpctool. + # NOTE: vmrun readVariable guestVar reads WITHOUT the 'guestinfo.' prefix. + $savedEap = $ErrorActionPreference; $ErrorActionPreference = 'Continue' + $ipOut = (& $VmrunPath -T ws readVariable $VmxPath guestVar 'ci-ip' 2>&1).Trim() + $ipRc = $LASTEXITCODE + $ErrorActionPreference = $savedEap + + if ($attempt -le 3 -or ($attempt % 10 -eq 0)) { + Write-Host "[Get-GuestIPAddress] [attempt $attempt] guestVar ci-ip: rc=$ipRc out='$ipOut'" + } + if ($ipRc -eq 0 -and $ipOut -match '^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$') { + Write-Host "[Get-GuestIPAddress] IP via guestinfo: $ipOut" + return $ipOut + } + + # Fallback: vmrun getGuestIPAddress (VMware Tools / open-vm-tools required). + $savedEap = $ErrorActionPreference; $ErrorActionPreference = 'Continue' + $ipOut2 = (& $VmrunPath -T ws getGuestIPAddress $VmxPath 2>&1).Trim() + $ipRc2 = $LASTEXITCODE + $ErrorActionPreference = $savedEap + + if ($ipRc2 -eq 0 -and $ipOut2 -match '^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$') { + Write-Host "[Get-GuestIPAddress] IP via getGuestIPAddress: $ipOut2" + return $ipOut2 + } + + Start-Sleep -Seconds 2 + } + + return '' +} + +function Get-GuestDiagnostics { +<# +.SYNOPSIS + Best-effort collection of guest diagnostics after a build failure. + Never throws — always safe to call from a catch/finally block. + +.PARAMETER IPAddress + IP address of the guest VM. + +.PARAMETER GuestOS + 'Windows' or 'Linux'. + +.PARAMETER Credential + PSCredential for WinRM (Windows guests). Ignored for Linux. + +.PARAMETER SshKeyPath + Path to SSH private key (Linux guests). Ignored for Windows. + +.PARAMETER SshUser + SSH username (Linux guests). Default: ci_build. + +.PARAMETER DestinationDir + Host directory where diagnostics files are written. +#> + [CmdletBinding()] + param( + [Parameter(Mandatory)] [string] $IPAddress, + [Parameter(Mandatory)] [ValidateSet('Windows','Linux')] + [string] $GuestOS, + [pscredential] $Credential, + [string] $SshKeyPath, + [string] $SshUser = 'ci_build', + [Parameter(Mandatory)] [string] $DestinationDir + ) + + Write-Host "[Diagnostics] Collecting guest diagnostics from $IPAddress ($GuestOS) -> $DestinationDir" + try { + if (-not (Test-Path $DestinationDir)) { + New-Item -ItemType Directory -Path $DestinationDir -Force | Out-Null + } + } catch { + Write-Warning "[Diagnostics] Could not create destination dir: $_" + return + } + + if ($GuestOS -eq 'Windows') { + if (-not $Credential) { + Write-Warning "[Diagnostics] No Credential supplied for Windows guest — skipping." + return + } + try { + $sessionOption = New-CISessionOption + $session = New-PSSession -ComputerName $IPAddress ` + -Credential $Credential ` + -SessionOption $sessionOption ` + -ErrorAction Stop + try { + # Collect: event log errors (last 50), running processes, disk usage + $diagScript = { + $out = [System.Text.StringBuilder]::new() + [void]$out.AppendLine("=== System Event Log (last 50 errors) ===") + Get-EventLog -LogName System -EntryType Error -Newest 50 -ErrorAction SilentlyContinue | + ForEach-Object { [void]$out.AppendLine("$($_.TimeGenerated) $($_.Source): $($_.Message)") } + [void]$out.AppendLine("`n=== Running Processes ===") + Get-Process -ErrorAction SilentlyContinue | + Sort-Object CPU -Descending | + Select-Object -First 30 | + ForEach-Object { [void]$out.AppendLine("$($_.Name) PID=$($_.Id) CPU=$($_.CPU)") } + [void]$out.AppendLine("`n=== Disk Free Space ===") + Get-PSDrive -PSProvider FileSystem -ErrorAction SilentlyContinue | + ForEach-Object { [void]$out.AppendLine("$($_.Name): Free=$([math]::Round($_.Free/1GB,1))GB Used=$([math]::Round($_.Used/1GB,1))GB") } + $out.ToString() + } + $diagText = Invoke-Command -Session $session -ScriptBlock $diagScript -ErrorAction Stop + $diagText | Set-Content -Path (Join-Path $DestinationDir 'guest-diag.txt') -Encoding UTF8 + Write-Host "[Diagnostics] Windows diagnostics written." + + # Best-effort: copy CI build log + $buildLog = 'C:\CI\build.log' + try { + Copy-Item -FromSession $session -Path $buildLog ` + -Destination (Join-Path $DestinationDir 'build.log') ` + -ErrorAction Stop + Write-Host "[Diagnostics] Build log copied." + } catch { + Write-Host "[Diagnostics] Build log not found in guest (skipping)." + } + } finally { + Remove-PSSession $session -ErrorAction SilentlyContinue + } + } catch { + Write-Warning "[Diagnostics] Windows guest diagnostics failed (non-fatal): $_" + } + } + else { + # Linux + if (-not $SshKeyPath) { + Write-Warning "[Diagnostics] No SshKeyPath supplied for Linux guest — skipping." + return + } + $sshOpts = @('-i', $SshKeyPath, '-o', 'StrictHostKeyChecking=accept-new', + '-o', 'ConnectTimeout=10', '-o', 'BatchMode=yes') + try { + # Collect: journalctl errors, ps, df + $diagCmd = "journalctl -p err -n 50 --no-pager 2>/dev/null; echo '=== ps ==='; ps aux --sort=-%cpu | head -30; echo '=== df ==='; df -h" + $savedEap = $ErrorActionPreference + $ErrorActionPreference = 'Continue' + $diagOutput = & ssh.exe @sshOpts "${SshUser}@${IPAddress}" $diagCmd 2>&1 + $ErrorActionPreference = $savedEap + $diagOutput | Set-Content -Path (Join-Path $DestinationDir 'guest-diag.txt') -Encoding UTF8 + Write-Host "[Diagnostics] Linux diagnostics written." + + # Best-effort: copy CI build log + $scpSrc = "${SshUser}@${IPAddress}:/opt/ci/build/ci-build.log" + $scpDst = Join-Path $DestinationDir 'build.log' + $ErrorActionPreference = 'Continue' + & scp.exe @sshOpts $scpSrc $scpDst 2>&1 | Out-Null + $ErrorActionPreference = $savedEap + if (Test-Path $scpDst) { Write-Host "[Diagnostics] Build log copied." } + else { Write-Host "[Diagnostics] Build log not found in guest (skipping)." } + } catch { + Write-Warning "[Diagnostics] Linux guest diagnostics failed (non-fatal): $_" + } + } +} + +Export-ModuleMember -Function New-CISessionOption, Resolve-VmrunPath, Invoke-Vmrun, Invoke-VmrunBounded, Get-GuestIPAddress, Get-GuestDiagnostics diff --git a/scripts/_Transport.psm1 b/scripts/_Transport.psm1 index 7183bea..08558b0 100644 --- a/scripts/_Transport.psm1 +++ b/scripts/_Transport.psm1 @@ -40,6 +40,12 @@ function Invoke-SshCommand { [int] $ConnectTimeout = 10, + # Persistent known_hosts file for CI jobs. + # When non-empty: StrictHostKeyChecking=accept-new + this file. + # When empty (default): permissive mode (StrictHostKeyChecking=no, UserKnownHostsFile=NUL) + # — used during template provisioning when host key is not yet known. + [string] $KnownHostsFile = '', + # Capture + return stdout/stderr instead of printing to console. [switch] $PassThru, @@ -47,10 +53,14 @@ function Invoke-SshCommand { [switch] $AllowFail ) + if ($KnownHostsFile -ne '') { + $sshHostKeyOpts = @('-o', 'StrictHostKeyChecking=accept-new', '-o', "UserKnownHostsFile=$KnownHostsFile") + } else { + $sshHostKeyOpts = @('-o', 'StrictHostKeyChecking=no', '-o', 'UserKnownHostsFile=NUL') + } $sshArgs = @( - '-i', $KeyPath, - '-o', 'StrictHostKeyChecking=no', - '-o', 'UserKnownHostsFile=NUL', + '-i', $KeyPath + ) + $sshHostKeyOpts + @( '-o', "ConnectTimeout=$ConnectTimeout", '-o', 'BatchMode=yes', "$User@$IP", @@ -104,7 +114,12 @@ function Copy-SshItem { # Pass -r to scp (recursive copy). [switch] $Recurse, - [int] $ConnectTimeout = 10 + [int] $ConnectTimeout = 10, + + # Persistent known_hosts file for CI jobs. + # When non-empty: StrictHostKeyChecking=accept-new + this file. + # When empty (default): permissive mode — used during template provisioning. + [string] $KnownHostsFile = '' ) if ($Direction -eq 'ToGuest') { @@ -115,10 +130,14 @@ function Copy-SshItem { } # Direction = 'Direct': use Source + Destination as-is + if ($KnownHostsFile -ne '') { + $scpHostKeyOpts = @('-o', 'StrictHostKeyChecking=accept-new', '-o', "UserKnownHostsFile=$KnownHostsFile") + } else { + $scpHostKeyOpts = @('-o', 'StrictHostKeyChecking=no', '-o', 'UserKnownHostsFile=NUL') + } $scpArgs = @( - '-i', $KeyPath, - '-o', 'StrictHostKeyChecking=no', - '-o', 'UserKnownHostsFile=NUL', + '-i', $KeyPath + ) + $scpHostKeyOpts + @( '-o', "ConnectTimeout=$ConnectTimeout" ) if ($Recurse) { $scpArgs += '-r' } diff --git a/template/Validate-DeployState.ps1 b/template/Validate-DeployState.ps1 index 4386d2c..e4c4359 100644 --- a/template/Validate-DeployState.ps1 +++ b/template/Validate-DeployState.ps1 @@ -1,35 +1,130 @@ #Requires -Version 5.1 <# .SYNOPSIS - Validates that Deploy-WinBuild2025.ps1 has correctly configured a guest VM. + Validates that a guest VM has been correctly configured. .DESCRIPTION - Connects to the VM via WinRM (HTTP/Basic) and runs all checks that - Install-CIToolchain-WinBuild2025.ps1 Steps 1/2/3/4b/5b/5c assert as validation-only. - Use this after Deploy completes and before running Prepare/Install-CIToolchain. + For Windows guests: connects via WinRM (HTTPS/Basic) and runs all checks + that Install-CIToolchain-WinBuild2025.ps1 asserts as validation-only. + + For Linux guests: connects via SSH and verifies that ci-report-ip.service + is enabled and that the ci-report-ip.sh script is executable. + + Use this after Deploy completes and before taking the BaseClean snapshot. .PARAMETER VMIPAddress IP address of the guest VM on VMnet8 (e.g. 192.168.79.129). +.PARAMETER GuestOS + Target OS: Windows (default) or Linux. + .PARAMETER AdminUsername - Admin account inside the VM (default: Administrator). + Windows only — admin account inside the VM (default: Administrator). + Prompted if -GuestOS Windows and -AdminPassword is not provided. .PARAMETER AdminPassword - Password for the admin account. Prompted if omitted. + Windows only — password for the admin account. + +.PARAMETER SshKeyPath + Linux only — path to the SSH private key (no passphrase). + Default: F:\CI\keys\ci_linux + +.PARAMETER SshUser + Linux only — SSH login user inside the VM. + Default: ci_build .EXAMPLE + # Windows .\Validate-DeployState.ps1 -VMIPAddress 192.168.79.129 + +.EXAMPLE + # Linux + .\Validate-DeployState.ps1 -VMIPAddress 192.168.79.200 -GuestOS Linux #> [CmdletBinding()] param( [Parameter(Mandatory)] [string] $VMIPAddress, + + [ValidateSet('Windows', 'Linux')] + [string] $GuestOS = 'Windows', + + # Windows params [string] $AdminUsername = 'Administrator', - [string] $AdminPassword + [string] $AdminPassword, + + # Linux params + [string] $SshKeyPath = 'F:\CI\keys\ci_linux', + [string] $SshUser = 'ci_build' ) Set-StrictMode -Version Latest $ErrorActionPreference = 'Stop' +# ── Linux branch ────────────────────────────────────────────────────────────── +if ($GuestOS -eq 'Linux') { + if (-not (Test-Path $SshKeyPath -PathType Leaf)) { + Write-Error "SSH key not found: $SshKeyPath" + exit 1 + } + + $sshBase = @( + '-i', $SshKeyPath, + '-o', 'StrictHostKeyChecking=no', + '-o', 'UserKnownHostsFile=NUL', + '-o', 'ConnectTimeout=10', + '-o', 'BatchMode=yes', + "$SshUser@$VMIPAddress" + ) + + Write-Host "`nValidating Linux Deploy state on $VMIPAddress..." -ForegroundColor Cyan + + $checks = [System.Collections.Generic.List[pscustomobject]]::new() + + function Add-LinuxCheck { + param([string]$Name, [string]$SshCommand, [scriptblock]$Validator) + $savedEap = $ErrorActionPreference; $ErrorActionPreference = 'Continue' + $out = (& ssh.exe @sshBase $SshCommand 2>&1) + $rc = $LASTEXITCODE + $ErrorActionPreference = $savedEap + $pass = [bool](& $Validator $out $rc) + $checks.Add([pscustomobject]@{ Name = $Name; Pass = $pass; Err = if (-not $pass) { "rc=$rc out=$out" } else { '' } }) + } + + # ci-report-ip.service must be enabled + Add-LinuxCheck ` + -Name 'ci-report-ip.service enabled' ` + -SshCommand 'systemctl is-enabled ci-report-ip.service 2>/dev/null' ` + -Validator { param($out, $rc) $rc -eq 0 -and "$out".Trim() -eq 'enabled' } + + # ci-report-ip.sh must be executable + Add-LinuxCheck ` + -Name '/usr/local/bin/ci-report-ip.sh executable' ` + -SshCommand 'test -x /usr/local/bin/ci-report-ip.sh && echo ok' ` + -Validator { param($out, $rc) $rc -eq 0 } + + Write-Host '' + $failed = 0 + foreach ($r in $checks) { + if ($r.Pass) { + Write-Host " [OK] $($r.Name)" -ForegroundColor Green + } else { + $detail = if ($r.Err) { " ($($r.Err))" } else { '' } + Write-Host " [FAIL] $($r.Name)$detail" -ForegroundColor Red + $failed++ + } + } + + Write-Host '' + if ($failed -eq 0) { + Write-Host "All $($checks.Count) Linux Deploy checks passed." -ForegroundColor Green + exit 0 + } else { + Write-Host "$failed / $($checks.Count) Linux Deploy checks FAILED." -ForegroundColor Red + exit 1 + } +} + +# ── Windows branch (WinRM HTTPS) ────────────────────────────────────────────── if (-not $AdminPassword) { $secure = Read-Host "Password for $AdminUsername@$VMIPAddress" -AsSecureString $AdminPassword = [Runtime.InteropServices.Marshal]::PtrToStringAuto( diff --git a/tests/_Common.Tests.ps1 b/tests/_Common.Tests.ps1 index 74f9aff..857bfdc 100644 --- a/tests/_Common.Tests.ps1 +++ b/tests/_Common.Tests.ps1 @@ -88,3 +88,94 @@ Describe 'Invoke-Vmrun' { $env:FAKE_VMRUN_EXIT = $null } } + +# ───────────────────────────────────────────────────────────────────────────── +Describe 'Invoke-VmrunBounded' { + BeforeAll { + # Fake vmrun that sleeps for $FAKE_VMRUN_SLEEP seconds then exits with + # $FAKE_VMRUN_EXIT. Both default to 0 if unset. + $script:SlowFakeVmrun = Join-Path $env:TEMP "fake-vmrun-slow-$PID.cmd" + Set-Content $script:SlowFakeVmrun -Value @' +@echo off +if "%FAKE_VMRUN_SLEEP%"=="" goto :nosleep +ping -n %FAKE_VMRUN_SLEEP% 127.0.0.1 >nul +:nosleep +echo fake-vmrun-bounded-output +if "%FAKE_VMRUN_EXIT%"=="" ( exit /b 0 ) else ( exit /b %FAKE_VMRUN_EXIT% ) +'@ + } + + AfterAll { + Remove-Item $script:SlowFakeVmrun -Force -ErrorAction SilentlyContinue + } + + AfterEach { + $env:FAKE_VMRUN_EXIT = $null + $env:FAKE_VMRUN_SLEEP = $null + } + + It 'returns ExitCode 0, TimedOut false and Output when command succeeds quickly' { + $env:FAKE_VMRUN_EXIT = '0' + $env:FAKE_VMRUN_SLEEP = '0' + $r = Invoke-VmrunBounded -VmrunPath $script:SlowFakeVmrun -Operation 'start' -Arguments @('some.vmx', 'nogui') -TimeoutSeconds 30 + $r.ExitCode | Should -Be 0 + $r.TimedOut | Should -Be $false + "$($r.Output)" | Should -Match 'fake-vmrun-bounded-output' + $r.ElapsedSeconds | Should -BeGreaterOrEqual 0 + } + + It 'returns non-zero ExitCode without throwing by default' { + $env:FAKE_VMRUN_EXIT = '3' + $env:FAKE_VMRUN_SLEEP = '0' + $r = Invoke-VmrunBounded -VmrunPath $script:SlowFakeVmrun -Operation 'deleteVM' -TimeoutSeconds 30 + $r.ExitCode | Should -Be 3 + $r.TimedOut | Should -Be $false + } + + It 'throws when -ThrowOnError and ExitCode is non-zero' { + $env:FAKE_VMRUN_EXIT = '2' + $env:FAKE_VMRUN_SLEEP = '0' + { Invoke-VmrunBounded -VmrunPath $script:SlowFakeVmrun -Operation 'stop' -TimeoutSeconds 30 -ThrowOnError } | + Should -Throw -ExpectedMessage '*stop failed*' + } + + It 'does not throw when -ThrowOnError and ExitCode is 0' { + $env:FAKE_VMRUN_EXIT = '0' + $env:FAKE_VMRUN_SLEEP = '0' + { Invoke-VmrunBounded -VmrunPath $script:SlowFakeVmrun -Operation 'start' -TimeoutSeconds 30 -ThrowOnError } | + Should -Not -Throw + } + + It 'times out and returns TimedOut = true when process exceeds TimeoutSeconds' { + # FAKE_VMRUN_SLEEP uses ping -n N which waits roughly (N-1) seconds. + # Set sleep >> timeout so the process is guaranteed to still be running. + $env:FAKE_VMRUN_SLEEP = '60' + $env:FAKE_VMRUN_EXIT = '0' + $r = Invoke-VmrunBounded -VmrunPath $script:SlowFakeVmrun -Operation 'start' -TimeoutSeconds 2 + $r.TimedOut | Should -Be $true + $r.ExitCode | Should -Be -1 + $r.ElapsedSeconds | Should -BeLessOrEqual 10 # killed well before 60s + } + + It 'throws when -ThrowOnTimeout and process times out' { + $env:FAKE_VMRUN_SLEEP = '60' + $env:FAKE_VMRUN_EXIT = '0' + { Invoke-VmrunBounded -VmrunPath $script:SlowFakeVmrun -Operation 'deleteVM' -TimeoutSeconds 2 -ThrowOnTimeout } | + Should -Throw -ExpectedMessage '*timed out*' + } + + It 'does not throw on timeout when neither flag is set' { + $env:FAKE_VMRUN_SLEEP = '60' + $env:FAKE_VMRUN_EXIT = '0' + { Invoke-VmrunBounded -VmrunPath $script:SlowFakeVmrun -Operation 'start' -TimeoutSeconds 2 } | + Should -Not -Throw + } + + It 'ElapsedSeconds is populated and plausible' { + $env:FAKE_VMRUN_EXIT = '0' + $env:FAKE_VMRUN_SLEEP = '0' + $r = Invoke-VmrunBounded -VmrunPath $script:SlowFakeVmrun -Operation 'list' -TimeoutSeconds 30 + $r.ElapsedSeconds | Should -BeGreaterOrEqual 0 + $r.ElapsedSeconds | Should -BeLessThan 30 + } +}