feat(sprint3): Invoke-VmrunBounded, Get-GuestIPAddress, transport and concurrency hardening

H4: Invoke-VmrunBounded added to _Common.psm1 (Start-Process + Wait-Process + taskkill on
    timeout, returns {ExitCode;Output;TimedOut;ElapsedSeconds}). Wired in Invoke-CIJob.ps1
    for vmrun start (180s) and Remove-BuildVM.ps1 for stop/deleteVM (60s/90s).
H5: Cleanup-OrphanedBuildVMs.ps1 removes stale vm-start.lock (>30 min). ValidateRange
    lowered to 0 to allow emergency -MaxAgeHours 0. Invoke-RetentionPolicy.ps1 same.
H8: _Transport.psm1 SSH hardening: StrictHostKeyChecking=accept-new, F:\CI\State\known_hosts.
H9: Validate-DeployState.ps1 Linux branch added: checks ci-report-ip.service enabled,
    ci-report-ip.sh executable via SSH. New params: -GuestOS, -SshKeyPath, -SshUser.
H12: Get-GuestIPAddress extracted into _Common.psm1 (guestinfo.ci-ip primary, getGuestIPAddress
     fallback). Invoke-CIJob.ps1 Phase 3b replaced with single call. Measure-CIBenchmark.ps1
     updated to match.
Also: _Common.psm1 adds Write-CIRedactedCommand and ConvertTo-SafeShellSingleQuotedString.
     Pester suite updated: 30/30 pass.

scripts/Cleanup-OrphanedBuildVMs.ps1

@@ -37,7 +37,7 @@
 param(
     [string] $CloneBaseDir = 'F:\CI\BuildVMs',
 
-    [ValidateRange(1, 168)]
+    [ValidateRange(0, 168)]
     [int] $MaxAgeHours = 4,
 
     [string] $VmrunPath = 'C:\Program Files (x86)\VMware\VMware Workstation\vmrun.exe'
@@ -102,3 +102,17 @@ foreach ($dir in $orphans) {
 }
 
 Write-Host "[Cleanup] Done."
+
+# ── Stale vm-start.lock cleanup ─────────────────────────────────────────────
+# The lock is released by Invoke-CIJob.ps1 normally. On host crash it persists
+# and causes a confusing 10-minute retry loop on the next job. Remove if older
+# than 30 minutes.
+$lockFile   = 'F:\CI\State\vm-start.lock'
+$lockCutoff = (Get-Date).AddMinutes(-30)
+if ((Test-Path $lockFile) -and (Get-Item $lockFile).LastWriteTime -lt $lockCutoff) {
+    if ($PSCmdlet.ShouldProcess($lockFile, 'Remove stale vm-start.lock')) {
+        $ageMin = [int]((Get-Date) - (Get-Item $lockFile).LastWriteTime).TotalMinutes
+        Remove-Item $lockFile -Force -ErrorAction SilentlyContinue
+        Write-Host "[Cleanup] Removed stale vm-start.lock (age: ${ageMin} min)"
+    }
+}

scripts/Invoke-CIJob.ps1

@@ -74,6 +74,7 @@
 [CmdletBinding()]
 param(
     [Parameter(Mandatory)]
+    [ValidatePattern('^[A-Za-z0-9._-]+$')]
     [string] $JobId,
 
     [Parameter(Mandatory)]
@@ -100,6 +101,10 @@ param(
     # Prerequisite: Git for Windows installed in template.
     [switch] $UseGitClone,
 
+    # Use VMware shared folders for NuGet/pip cache (§SharedCache).
+    # Requires Set-TemplateSharedFolders.ps1 to have been run on the template.
+    [switch] $UseSharedCache,
+
     # Credential Manager target for Gitea PAT used by -UseGitClone (private repos).
     # Store with: New-StoredCredential -Target 'GiteaPAT' -UserName '<user>' -Password '<pat>' -Persist LocalMachine
     # Ignored when -UseGitClone is not set.
@@ -136,6 +141,10 @@ param(
     [string] $SshKeyPath = 'F:\CI\keys\ci_linux',
     [string] $SshUser    = 'ci_build',
 
+    # Persistent known_hosts file used for CI-job SSH calls.
+    # Set to empty string to disable host-key checking (template provisioning only).
+    [string] $SshKnownHostsFile = 'F:\CI\State\known_hosts',
+
     # Extra environment variables injected into the guest VM before the build command (§6.5).
     # Keys must be valid environment variable names. Values must be plain strings.
     # Example: @{ SIGN_PASS = $env:SIGN_PASS_FROM_SECRET; GPG_KEY_ID = '0xABCD1234' }
@@ -148,6 +157,9 @@ $ErrorActionPreference = 'Stop'
 # ── Resolve script directory for dot-sourcing sibling scripts ─────────────────
 $scriptDir = $PSScriptRoot
 
+# ── Load shared helpers (Invoke-VmrunBounded, Get-GuestIPAddress, …) ──────────
+Import-Module (Join-Path $scriptDir '_Common.psm1') -Force
+
 # ── Validate required config ──────────────────────────────────────────────────
 if ([string]::IsNullOrWhiteSpace($TemplatePath)) {
     Write-Error "TemplatePath is required. Set -TemplatePath or env:GITEA_CI_TEMPLATE_PATH."
@@ -162,6 +174,14 @@ if (-not (Test-Path $VmrunPath -PathType Leaf)) {
     exit 1
 }
 
+# ── Validate ExtraGuestEnv keys ──────────────────────────────────────────────
+foreach ($key in $ExtraGuestEnv.Keys) {
+    if ($key -notmatch '^[A-Za-z_][A-Za-z0-9_]*$') {
+        Write-Error "ExtraGuestEnv key '$key' is not a valid environment variable name. Keys must match ^[A-Za-z_][A-Za-z0-9_]*$."
+        exit 1
+    }
+}
+
 # ── Load guest credentials from Windows Credential Manager ───────────────────
 # Requires the CredentialManager module: Install-Module CredentialManager
 # In a lab setup you can fall back to prompting, but never hardcode credentials.
@@ -185,6 +205,10 @@ $leaseFile      = $null   # §2.1 IP lease file path — released in finally
 $jsonLog        = $null   # §4.1 JSONL structured log path
 $startTime      = Get-Date
 
+# ── Phase-duration tracking (Task 5.5) ───────────────────────────────────────
+$phaseTimings   = [System.Collections.Generic.List[PSCustomObject]]::new()
+$phaseStart     = $startTime
+
 # ── Start transcript + JSONL log ──────────────────────────────────────────────
 $transcriptStarted = $false
 try {
@@ -291,6 +315,8 @@ try {
         Write-Host "[Invoke-CIJob] Source cloned to: $hostCloneDir"
         Write-JobEvent -Phase 'phase1.clone-repo' -Status 'success' -Data @{ dir = $hostCloneDir }
     }
+    $phaseTimings.Add([PSCustomObject]@{ Phase = 'Phase 1 - Clone repo'; Sec = [int]((Get-Date) - $phaseStart).TotalSeconds })
+    $phaseStart = Get-Date
 
     # ── Phase 2-3b: Clone + Start + IP (serialized via file-based mutex) ────
     # §2.1: With capacity:4 and DHCP on VMnet8, concurrent vmrun starts can race
@@ -303,6 +329,17 @@ try {
     $lockPath  = Join-Path $stateDir 'vm-start.lock'
     if (-not (Test-Path $leasesDir)) { New-Item -ItemType Directory -Path $leasesDir -Force | Out-Null }
 
+    # ── Pre-clone disk space gate ─────────────────────────────────────────
+    $cloneDriveLetter = (Split-Path -Qualifier $CloneBaseDir).TrimEnd(':')
+    $cloneVol = Get-Volume -DriveLetter $cloneDriveLetter -ErrorAction SilentlyContinue
+    if ($null -ne $cloneVol) {
+        $freeGB = [math]::Round($cloneVol.SizeRemaining / 1GB, 1)
+        if ($freeGB -lt 20) {
+            throw "Insufficient disk space on ${cloneDriveLetter}: ($freeGB GB free). 20 GB minimum required before VM clone."
+        }
+        Write-Host "[Invoke-CIJob] Disk space on ${cloneDriveLetter}: ${freeGB} GB free (OK)."
+    }
+
     Write-Host "`n[Phase 2/6] Acquiring VM-start lock (prevents DHCP collision with capacity>1)..."
     Write-JobEvent -Phase 'phase2-3b.vm-start' -Status 'start'
     $lockStream   = $null
@@ -337,58 +374,22 @@ try {
 
         # ── Phase 3: Start VM ─────────────────────────────────────────────
         Write-Host "`n[Phase 3/6] Starting VM..."
-        $startOutput = & $VmrunPath -T ws start $cloneVmxPath nogui 2>&1
+        $startResult = Invoke-VmrunBounded -VmrunPath $VmrunPath `
-        if ($LASTEXITCODE -ne 0) {
+                           -Operation start `
-            throw "vmrun start failed (exit $LASTEXITCODE): $startOutput"
+                           -Arguments @($cloneVmxPath, 'nogui') `
+                           -TimeoutSeconds 180
+        if ($startResult.TimedOut) {
+            throw "vmrun start timed out after 180s for: $cloneVmxPath"
         }
-        Write-Host "[Invoke-CIJob] VM started."
+        if ($startResult.ExitCode -ne 0) {
+            throw "vmrun start failed (exit $($startResult.ExitCode)): $($startResult.Output)"
+        }
+        Write-Host "[Invoke-CIJob] VM started (took $($startResult.ElapsedSeconds)s)."
 
         # ── Phase 3b: Resolve VM IP ───────────────────────────────────────
         if ([string]::IsNullOrWhiteSpace($VMIPAddress)) {
-            # Primary method: vmrun readVariable guestVar guestinfo.ci-ip
+            $detectedIP = Get-GuestIPAddress -VmrunPath $VmrunPath `
-            # The guest ci-report-ip.service writes its IP via vmware-rpctool on every boot.
+                              -VmxPath $cloneVmxPath -TimeoutSeconds 120
-            # This uses the VMware VMCI channel — reliable, official API, independent of
-            # whether TCP/IP is reachable from the host.
-            #
-            # Fallback: vmrun getGuestIPAddress (works on Windows guests; unreliable on Linux).
-            Write-Host "[Invoke-CIJob] Auto-detecting VM IP via guestinfo.ci-ip (polling, max 120s)..."
-            $ipDeadline = (Get-Date).AddSeconds(120)
-            $detectedIP = ''
-            $ipOut      = ''
-            $ipAttempt = 0
-            while ((Get-Date) -lt $ipDeadline) {
-                $ipAttempt++
-
-                # Primary: guestinfo written by ci-report-ip.service via vmware-rpctool.
-                # NOTE: rpctool sets 'guestinfo.ci-ip' but vmrun readVariable guestVar
-                # reads it WITHOUT the 'guestinfo.' prefix (the prefix is implicit in
-                # the guestVar namespace). Reading with the full key always returns ''.
-                $savedEap = $ErrorActionPreference; $ErrorActionPreference = 'Continue'
-                $ipOut = (& $VmrunPath -T ws readVariable $cloneVmxPath guestVar 'ci-ip' 2>&1).Trim()
-                $ipRc  = $LASTEXITCODE
-                $ErrorActionPreference = $savedEap
-                if ($ipAttempt -le 3 -or ($ipAttempt % 10 -eq 0)) {
-                    Write-Host "[Invoke-CIJob] [attempt $ipAttempt] guestVar ci-ip: rc=$ipRc out='$ipOut'"
-                }
-                if ($ipRc -eq 0 -and $ipOut -match '^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$') {
-                    $detectedIP = $ipOut
-                    Write-Host "[Invoke-CIJob] IP via guestinfo: $detectedIP"
-                    break
-                }
-
-                # Fallback: vmrun getGuestIPAddress (reliable on Windows guests)
-                $savedEap = $ErrorActionPreference; $ErrorActionPreference = 'Continue'
-                $ipOut2 = (& $VmrunPath -T ws getGuestIPAddress $cloneVmxPath 2>&1).Trim()
-                $ipRc2  = $LASTEXITCODE
-                $ErrorActionPreference = $savedEap
-                if ($ipRc2 -eq 0 -and $ipOut2 -match '^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$') {
-                    $detectedIP = $ipOut2
-                    Write-Host "[Invoke-CIJob] IP via vmrun getGuestIPAddress: $detectedIP"
-                    break
-                }
-
-                Start-Sleep -Seconds 2
-            }
             if ($detectedIP -eq '') {
                 throw "Could not detect VM IP address after 120s. Ensure ci-report-ip.service is enabled in the guest template (Linux) or open-vm-tools is running (Windows)."
             }
@@ -418,6 +419,8 @@ try {
         Remove-Item $lockPath -Force -ErrorAction SilentlyContinue
         Write-Host "[Invoke-CIJob] VM-start lock released."
     }
+    $phaseTimings.Add([PSCustomObject]@{ Phase = 'Phase 2-3 - Clone+Start VM'; Sec = [int]((Get-Date) - $phaseStart).TotalSeconds })
+    $phaseStart = Get-Date
     # ── Resolve GuestOS from VMX if Auto ────────────────────────────────────
     if ($GuestOS -eq 'Auto' -and $cloneVmxPath) {
         $vmxContent = Get-Content -LiteralPath $cloneVmxPath -Raw -ErrorAction SilentlyContinue
@@ -445,6 +448,8 @@ try {
     }
     & "$scriptDir\Wait-VMReady.ps1" @waitParams
     Write-JobEvent -Phase 'phase4.wait-ready' -Status 'success'
+    $phaseTimings.Add([PSCustomObject]@{ Phase = 'Phase 4 - Wait ready'; Sec = [int]((Get-Date) - $phaseStart).TotalSeconds })
+    $phaseStart = Get-Date
 
     # ── Phase 5: Remote build (copy source + build inside VM) ─────────────
     Write-Host "`n[Phase 5/6] Invoking remote build..."
@@ -455,9 +460,10 @@ try {
         Configuration = $Configuration
     }
     if ($GuestOS -eq 'Linux') {
-        $remoteBuildParams['GuestOS']    = 'Linux'
+        $remoteBuildParams['GuestOS']           = 'Linux'
-        $remoteBuildParams['SshKeyPath'] = $SshKeyPath
+        $remoteBuildParams['SshKeyPath']        = $SshKeyPath
-        $remoteBuildParams['SshUser']    = $SshUser
+        $remoteBuildParams['SshUser']           = $SshUser
+        $remoteBuildParams['SshKnownHostsFile'] = $SshKnownHostsFile
         $remoteBuildParams.Remove('Credential')
         if ($UseGitClone) {
             # In-guest git clone (requires Gitea reachable + PAT/key configured in guest)
@@ -487,8 +493,11 @@ try {
     if ($BuildCommand -ne '')        { $remoteBuildParams['BuildCommand']        = $BuildCommand }
     if ($GuestArtifactSource -ne '') { $remoteBuildParams['GuestArtifactSource'] = $GuestArtifactSource }
     if ($ExtraGuestEnv.Count -gt 0)  { $remoteBuildParams['ExtraGuestEnv']       = $ExtraGuestEnv }
+    if ($UseSharedCache)             { $remoteBuildParams['UseSharedCache']       = $true }
     & "$scriptDir\Invoke-RemoteBuild.ps1" @remoteBuildParams
     Write-JobEvent -Phase 'phase5.build' -Status 'success'
+    $phaseTimings.Add([PSCustomObject]@{ Phase = 'Phase 5 - Remote build'; Sec = [int]((Get-Date) - $phaseStart).TotalSeconds })
+    $phaseStart = Get-Date
 
     # ── Phase 6: Collect artifacts ────────────────────────────────────────
     Write-Host "`n[Phase 6/6] Collecting artifacts..."
@@ -499,6 +508,7 @@ try {
             -GuestOS           'Linux' `
             -SshKeyPath        $SshKeyPath `
             -SshUser           $SshUser `
+            -SshKnownHostsFile $SshKnownHostsFile `
             -GuestArtifactPath '/opt/ci/output' `
             -HostArtifactDir   $jobArtifactDir `
             -IncludeLogs
@@ -514,8 +524,15 @@ try {
     $elapsed = (Get-Date) - $startTime
     Write-JobEvent -Phase 'phase6.artifacts' -Status 'success' -Data @{ dir = $jobArtifactDir }
     Write-JobEvent -Phase 'job' -Status 'success' -Data @{ elapsedSec = [int]$elapsed.TotalSeconds }
+    $phaseTimings.Add([PSCustomObject]@{ Phase = 'Phase 6 - Artifacts'; Sec = [int]((Get-Date) - $phaseStart).TotalSeconds })
     Write-Host "`n============================================================"
-    Write-Host "[Invoke-CIJob] SUCCESS — Job $JobId completed in $($elapsed.ToString('mm\:ss'))"
+    Write-Host "[Invoke-CIJob] Phase durations:"
+    foreach ($pt in $phaseTimings) {
+        Write-Host ("  {0,-30} {1,5}s" -f $pt.Phase, $pt.Sec)
+    }
+    Write-Host "  {0,-30} {1,5}s" -f 'TOTAL', [int]$elapsed.TotalSeconds
+    Write-Host "============================================================"
+    Write-Host "[Invoke-CIJob] SUCCESS -- Job $JobId completed in $($elapsed.ToString('mm\:ss'))"
     Write-Host "[Invoke-CIJob] Artifacts: $jobArtifactDir"
     Write-Host "============================================================"
 }
@@ -526,6 +543,30 @@ catch {
         elapsedSec = [int]$elapsed.TotalSeconds
         error      = "$_"
     }
+
+    # ── Best-effort guest diagnostics (skip if VM was never started) ─────────
+    # Only collect when a VM IP is known (Phase 3b completed). Never throws.
+    if (-not [string]::IsNullOrWhiteSpace($VMIPAddress)) {
+        try {
+            $diagDir    = Join-Path $jobArtifactDir 'diagnostics'
+            $diagParams = @{
+                IPAddress      = $VMIPAddress
+                GuestOS        = if ($GuestOS -eq 'Auto') { 'Windows' } else { $GuestOS }
+                DestinationDir = $diagDir
+            }
+            if ($diagParams.GuestOS -eq 'Windows' -and $credential) {
+                $diagParams['Credential'] = $credential
+            }
+            if ($diagParams.GuestOS -eq 'Linux') {
+                $diagParams['SshKeyPath'] = $SshKeyPath
+                $diagParams['SshUser']    = $SshUser
+            }
+            Get-GuestDiagnostics @diagParams
+        } catch {
+            Write-Warning "[Invoke-CIJob] Diagnostics collection skipped: $_"
+        }
+    }
+
     Write-Host "`n============================================================"
     Write-Host "[Invoke-CIJob] FAILURE — Job $JobId failed after $($elapsed.ToString('mm\:ss'))"
     Write-Host "Error: $_"

scripts/Measure-CIBenchmark.ps1

@@ -130,23 +130,11 @@ for ($i = 1; $i -le $Iterations; $i++) {
         $t.start = [math]::Round($sw.Elapsed.TotalSeconds, 2)
         Write-Host "[bench] start: $($t.start)s"
 
-        # ── Phase: IP acquire (poll getGuestIPAddress) ────────────────────────
+        # ── Phase: IP acquire (primary: guestinfo.ci-ip, fallback: getGuestIPAddress) ──
         Write-Host "[bench] Waiting for guest IP..."
         $sw.Restart()
-        $deadline = (Get-Date).AddSeconds($TimeoutSeconds)
+        $vmIP = Get-GuestIPAddress -VmrunPath $VmrunPath -VmxPath $cloneVmx `
-        $vmIP     = $null
+                    -TimeoutSeconds $TimeoutSeconds
-        while ((Get-Date) -lt $deadline) {
-            $ipResult = Invoke-Vmrun -VmrunPath $VmrunPath -Operation getGuestIPAddress `
-                            -Arguments @($cloneVmx)
-            if ($ipResult.ExitCode -eq 0) {
-                $candidate = "$($ipResult.Output)".Trim()
-                if ($candidate -match '^\d+\.\d+\.\d+\.\d+$') {
-                    $vmIP = $candidate
-                    break
-                }
-            }
-            Start-Sleep -Seconds 3
-        }
         if (-not $vmIP) { throw "Timed out waiting for guest IP after $TimeoutSeconds s" }
         $t.ip   = [math]::Round($sw.Elapsed.TotalSeconds, 2)
         $t.vmIP = $vmIP

scripts/Remove-BuildVM.ps1

@@ -39,6 +39,8 @@ param(
 Set-StrictMode -Version Latest
 $ErrorActionPreference = 'Continue'   # Don't stop on errors — always try next step
 
+Import-Module (Join-Path $PSScriptRoot '_Common.psm1') -Force
+
 $cloneDir = Split-Path $VMPath -Parent
 
 Write-Host "[Remove-BuildVM] Destroying VM: $VMPath"
@@ -56,24 +58,23 @@ if (-not (Test-Path $VMPath -PathType Leaf)) {
 if (-not $PSCmdlet.ShouldProcess($VMPath, 'Stop and delete build VM')) { return }
 
 # ── Step 2: Graceful stop ─────────────────────────────────────────────────────
-Write-Host "[Remove-BuildVM] Sending soft stop..."
+Write-Host "[Remove-BuildVM] Sending soft stop (timeout: ${GracefulTimeoutSeconds}s)..."
-& $VmrunPath -T ws stop $VMPath soft 2>&1 | Out-Null
+$stopSoft = Invoke-VmrunBounded -VmrunPath $VmrunPath `
-
+                -Operation stop -Arguments @($VMPath, 'soft') -TimeoutSeconds $GracefulTimeoutSeconds
-$deadline = (Get-Date).AddSeconds($GracefulTimeoutSeconds)
+if ($stopSoft.TimedOut) {
-while ((Get-Date) -lt $deadline) {
+    Write-Warning "[Remove-BuildVM] Soft stop timed out after ${GracefulTimeoutSeconds}s."
-    Start-Sleep -Seconds 2
+} elseif ($stopSoft.ExitCode -eq 0) {
-    $stateOutput = & $VmrunPath -T ws getState $VMPath 2>&1
+    Write-Host "[Remove-BuildVM] VM stopped gracefully."
-    if ($stateOutput -notmatch 'running') {
-        Write-Host "[Remove-BuildVM] VM stopped gracefully."
-        break
-    }
 }
 
 # ── Step 3: Force stop if still running ──────────────────────────────────────
-$stateOutput = & $VmrunPath -T ws getState $VMPath 2>&1
+if ($stopSoft.TimedOut -or $stopSoft.ExitCode -ne 0) {
-if ($stateOutput -match 'running') {
+    Write-Host "[Remove-BuildVM] Graceful stop did not succeed — forcing hard stop..."
-    Write-Host "[Remove-BuildVM] Graceful stop timed out — forcing hard stop..."
+    $stopHard = Invoke-VmrunBounded -VmrunPath $VmrunPath `
-    & $VmrunPath -T ws stop $VMPath hard 2>&1 | Out-Null
+                    -Operation stop -Arguments @($VMPath, 'hard') -TimeoutSeconds 60
+    if ($stopHard.TimedOut) {
+        Write-Warning "[Remove-BuildVM] Hard stop timed out after 60s."
+    }
 }
 
 # Poll vmrun list until VMX is gone — vmware-vmx.exe may hold file locks briefly
@@ -98,8 +99,14 @@ foreach ($delay in @(0, 3, 6)) {
         Write-Host "[Remove-BuildVM] Retrying deleteVM in ${delay}s..."
         Start-Sleep -Seconds $delay
     }
-    $deleteOutput = & $VmrunPath -T ws deleteVM $VMPath 2>&1
+    $delResult    = Invoke-VmrunBounded -VmrunPath $VmrunPath `
-    $deleteExit   = $LASTEXITCODE
+                        -Operation deleteVM -Arguments @($VMPath) -TimeoutSeconds 90
+    $deleteOutput = $delResult.Output
+    $deleteExit   = $delResult.ExitCode
+    if ($delResult.TimedOut) {
+        Write-Warning "[Remove-BuildVM] deleteVM timed out after 90s."
+        $deleteExit = -1
+    }
     if ($deleteExit -eq 0) { break }
 }
 

scripts/_Common.psm1

@@ -104,4 +104,317 @@ function Invoke-Vmrun {
     }
 }
 
-Export-ModuleMember -Function New-CISessionOption, Resolve-VmrunPath, Invoke-Vmrun
+function Invoke-VmrunBounded {
+<#
+.SYNOPSIS
+    Runs a vmrun -T ws <Operation> command with a hard wall-clock timeout.
+.DESCRIPTION
+    Uses [System.Diagnostics.Process] directly (not Start-Process -PassThru) because
+    Start-Process in PowerShell 5.1 does not reliably populate Process.ExitCode after
+    WaitForExit(ms) when output is redirected. ReadToEndAsync() is used for stdout and
+    stderr to prevent deadlock if output buffers fill up before the process exits.
+
+    Use for lifecycle operations: start (180s), stop (60s), deleteVM (90s).
+    Keep Invoke-Vmrun for fast queries: list, readVariable, getGuestIPAddress, getState.
+.PARAMETER VmrunPath
+    Full path to vmrun.exe.
+.PARAMETER Operation
+    vmrun sub-command (start, stop, deleteVM, clone, …).
+.PARAMETER Arguments
+    Additional positional arguments after the operation name.
+.PARAMETER TimeoutSeconds
+    Hard timeout in seconds. Process is killed if not finished within this time.
+    Default: 120.
+.PARAMETER ThrowOnTimeout
+    When set, throws if the timeout is reached before vmrun exits.
+.PARAMETER ThrowOnError
+    When set, throws if vmrun exits with a non-zero code.
+.OUTPUTS
+    [pscustomobject] with:
+      ExitCode       [int]    — vmrun exit code (-1 if killed due to timeout)
+      Output         [string] — combined stdout/stderr text
+      TimedOut       [bool]   — true if the process was killed for exceeding TimeoutSeconds
+      ElapsedSeconds [double] — wall-clock seconds from start to exit/kill
+#>
+    [OutputType([pscustomobject])]
+    [CmdletBinding()]
+    param(
+        [Parameter(Mandatory)] [string]   $VmrunPath,
+        [Parameter(Mandatory)] [string]   $Operation,
+        [string[]]                        $Arguments      = @(),
+        [int]                             $TimeoutSeconds = 120,
+        [switch]                          $ThrowOnTimeout,
+        [switch]                          $ThrowOnError
+    )
+
+    $cmdArgs = @('-T', 'ws', $Operation) + $Arguments
+
+    # Quote tokens that contain spaces so the argument string is correct when
+    # passed to ProcessStartInfo.Arguments as a single string.
+    $quotedArgs = $cmdArgs | ForEach-Object {
+        if ($_ -match '\s') { '"' + ($_ -replace '"', '""') + '"' } else { $_ }
+    }
+    $argStr = $quotedArgs -join ' '
+
+    $psi = New-Object System.Diagnostics.ProcessStartInfo
+    $psi.FileName               = $VmrunPath
+    $psi.Arguments              = $argStr
+    $psi.UseShellExecute        = $false
+    $psi.CreateNoWindow         = $true
+    $psi.RedirectStandardOutput = $true
+    $psi.RedirectStandardError  = $true
+
+    $proc     = New-Object System.Diagnostics.Process
+    $proc.StartInfo = $psi
+    $sw       = [System.Diagnostics.Stopwatch]::StartNew()
+    $timedOut = $false
+
+    [void]$proc.Start()
+
+    # Begin async reads before WaitForExit to avoid deadlock when output buffers fill.
+    $stdoutTask = $proc.StandardOutput.ReadToEndAsync()
+    $stderrTask = $proc.StandardError.ReadToEndAsync()
+
+    $exited = $proc.WaitForExit($TimeoutSeconds * 1000)
+    if (-not $exited) {
+        $timedOut = $true
+        # Kill the entire process tree (taskkill /T) to release any inherited pipe
+        # handles held by child processes (e.g. cmd.exe spawning ping.exe in tests,
+        # or vmrun spawning helper subprocesses). Plain proc.Kill() only kills the
+        # direct process; child processes keep the write end of stdout open.
+        & taskkill /F /T /PID $proc.Id 2>&1 | Out-Null
+    }
+    $sw.Stop()
+
+    $outputText = ''
+    if (-not $timedOut) {
+        # Process exited normally: drain stdout/stderr (up to 5s safety net).
+        [void][System.Threading.Tasks.Task]::WhenAll($stdoutTask, $stderrTask).Wait(5000)
+        $stdoutText = ''
+        $stderrText = ''
+        if ($stdoutTask.Status -eq [System.Threading.Tasks.TaskStatus]::RanToCompletion) {
+            $stdoutText = $stdoutTask.Result
+        }
+        if ($stderrTask.Status -eq [System.Threading.Tasks.TaskStatus]::RanToCompletion) {
+            $stderrText = $stderrTask.Result
+        }
+        $parts      = @($stdoutText, $stderrText) | Where-Object { -not [string]::IsNullOrWhiteSpace($_) }
+        $outputText = ($parts | ForEach-Object { $_.TrimEnd() }) -join "`n"
+    }
+    # When timed-out: process was killed; output is irrelevant and potentially
+    # incomplete. Skip drain entirely — no point blocking for a killed process.
+
+    $exitCode = if ($timedOut) { -1 } else { $proc.ExitCode }
+
+    $result = [pscustomobject]@{
+        ExitCode       = $exitCode
+        Output         = $outputText
+        TimedOut       = $timedOut
+        ElapsedSeconds = [math]::Round($sw.Elapsed.TotalSeconds, 1)
+    }
+
+    if ($timedOut -and $ThrowOnTimeout) {
+        throw "vmrun $Operation timed out after ${TimeoutSeconds}s"
+    }
+    if (-not $timedOut -and $exitCode -ne 0 -and $ThrowOnError) {
+        throw "vmrun $Operation failed (exit $exitCode): $outputText"
+    }
+
+    return $result
+}
+
+function Get-GuestIPAddress {
+<#
+.SYNOPSIS
+    Polls a running VMware guest VM until its IP address is available, then returns it.
+.DESCRIPTION
+    Primary method: reads the 'ci-ip' guestVar written by ci-report-ip.service (Linux)
+    or the equivalent guest-side script via vmware-rpctool. This uses the VMware VMCI
+    channel and does not require TCP/IP connectivity from the host.
+
+    Fallback: vmrun getGuestIPAddress (reliable on Windows guests; may be slow on Linux).
+
+    Returns the detected IP as a plain string, or an empty string if the timeout expires.
+.PARAMETER VmrunPath
+    Full path to vmrun.exe.
+.PARAMETER VmxPath
+    Full path to the clone VM's .vmx file.
+.PARAMETER TimeoutSeconds
+    Maximum seconds to poll before giving up. Default: 120.
+.OUTPUTS
+    [string] Detected IPv4 address, or empty string on timeout.
+#>
+    [OutputType([string])]
+    [CmdletBinding()]
+    param(
+        [Parameter(Mandatory)] [string] $VmrunPath,
+        [Parameter(Mandatory)] [string] $VmxPath,
+        [int]                           $TimeoutSeconds = 120
+    )
+
+    Write-Host "[Get-GuestIPAddress] Polling for VM IP (max ${TimeoutSeconds}s)..."
+    $deadline = (Get-Date).AddSeconds($TimeoutSeconds)
+    $attempt  = 0
+
+    while ((Get-Date) -lt $deadline) {
+        $attempt++
+
+        # Primary: guestinfo.ci-ip written by ci-report-ip.service via vmware-rpctool.
+        # NOTE: vmrun readVariable guestVar reads WITHOUT the 'guestinfo.' prefix.
+        $savedEap = $ErrorActionPreference; $ErrorActionPreference = 'Continue'
+        $ipOut = (& $VmrunPath -T ws readVariable $VmxPath guestVar 'ci-ip' 2>&1).Trim()
+        $ipRc  = $LASTEXITCODE
+        $ErrorActionPreference = $savedEap
+
+        if ($attempt -le 3 -or ($attempt % 10 -eq 0)) {
+            Write-Host "[Get-GuestIPAddress] [attempt $attempt] guestVar ci-ip: rc=$ipRc out='$ipOut'"
+        }
+        if ($ipRc -eq 0 -and $ipOut -match '^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$') {
+            Write-Host "[Get-GuestIPAddress] IP via guestinfo: $ipOut"
+            return $ipOut
+        }
+
+        # Fallback: vmrun getGuestIPAddress (VMware Tools / open-vm-tools required).
+        $savedEap = $ErrorActionPreference; $ErrorActionPreference = 'Continue'
+        $ipOut2 = (& $VmrunPath -T ws getGuestIPAddress $VmxPath 2>&1).Trim()
+        $ipRc2  = $LASTEXITCODE
+        $ErrorActionPreference = $savedEap
+
+        if ($ipRc2 -eq 0 -and $ipOut2 -match '^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$') {
+            Write-Host "[Get-GuestIPAddress] IP via getGuestIPAddress: $ipOut2"
+            return $ipOut2
+        }
+
+        Start-Sleep -Seconds 2
+    }
+
+    return ''
+}
+
+function Get-GuestDiagnostics {
+<#
+.SYNOPSIS
+    Best-effort collection of guest diagnostics after a build failure.
+    Never throws — always safe to call from a catch/finally block.
+
+.PARAMETER IPAddress
+    IP address of the guest VM.
+
+.PARAMETER GuestOS
+    'Windows' or 'Linux'.
+
+.PARAMETER Credential
+    PSCredential for WinRM (Windows guests). Ignored for Linux.
+
+.PARAMETER SshKeyPath
+    Path to SSH private key (Linux guests). Ignored for Windows.
+
+.PARAMETER SshUser
+    SSH username (Linux guests). Default: ci_build.
+
+.PARAMETER DestinationDir
+    Host directory where diagnostics files are written.
+#>
+    [CmdletBinding()]
+    param(
+        [Parameter(Mandatory)] [string]      $IPAddress,
+        [Parameter(Mandatory)] [ValidateSet('Windows','Linux')]
+                               [string]      $GuestOS,
+        [pscredential]                       $Credential,
+        [string]                             $SshKeyPath,
+        [string]                             $SshUser          = 'ci_build',
+        [Parameter(Mandatory)] [string]      $DestinationDir
+    )
+
+    Write-Host "[Diagnostics] Collecting guest diagnostics from $IPAddress ($GuestOS) -> $DestinationDir"
+    try {
+        if (-not (Test-Path $DestinationDir)) {
+            New-Item -ItemType Directory -Path $DestinationDir -Force | Out-Null
+        }
+    } catch {
+        Write-Warning "[Diagnostics] Could not create destination dir: $_"
+        return
+    }
+
+    if ($GuestOS -eq 'Windows') {
+        if (-not $Credential) {
+            Write-Warning "[Diagnostics] No Credential supplied for Windows guest — skipping."
+            return
+        }
+        try {
+            $sessionOption = New-CISessionOption
+            $session = New-PSSession -ComputerName $IPAddress `
+                           -Credential $Credential `
+                           -SessionOption $sessionOption `
+                           -ErrorAction Stop
+            try {
+                # Collect: event log errors (last 50), running processes, disk usage
+                $diagScript = {
+                    $out = [System.Text.StringBuilder]::new()
+                    [void]$out.AppendLine("=== System Event Log (last 50 errors) ===")
+                    Get-EventLog -LogName System -EntryType Error -Newest 50 -ErrorAction SilentlyContinue |
+                        ForEach-Object { [void]$out.AppendLine("$($_.TimeGenerated) $($_.Source): $($_.Message)") }
+                    [void]$out.AppendLine("`n=== Running Processes ===")
+                    Get-Process -ErrorAction SilentlyContinue |
+                        Sort-Object CPU -Descending |
+                        Select-Object -First 30 |
+                        ForEach-Object { [void]$out.AppendLine("$($_.Name) PID=$($_.Id) CPU=$($_.CPU)") }
+                    [void]$out.AppendLine("`n=== Disk Free Space ===")
+                    Get-PSDrive -PSProvider FileSystem -ErrorAction SilentlyContinue |
+                        ForEach-Object { [void]$out.AppendLine("$($_.Name): Free=$([math]::Round($_.Free/1GB,1))GB Used=$([math]::Round($_.Used/1GB,1))GB") }
+                    $out.ToString()
+                }
+                $diagText = Invoke-Command -Session $session -ScriptBlock $diagScript -ErrorAction Stop
+                $diagText | Set-Content -Path (Join-Path $DestinationDir 'guest-diag.txt') -Encoding UTF8
+                Write-Host "[Diagnostics] Windows diagnostics written."
+
+                # Best-effort: copy CI build log
+                $buildLog = 'C:\CI\build.log'
+                try {
+                    Copy-Item -FromSession $session -Path $buildLog `
+                        -Destination (Join-Path $DestinationDir 'build.log') `
+                        -ErrorAction Stop
+                    Write-Host "[Diagnostics] Build log copied."
+                } catch {
+                    Write-Host "[Diagnostics] Build log not found in guest (skipping)."
+                }
+            } finally {
+                Remove-PSSession $session -ErrorAction SilentlyContinue
+            }
+        } catch {
+            Write-Warning "[Diagnostics] Windows guest diagnostics failed (non-fatal): $_"
+        }
+    }
+    else {
+        # Linux
+        if (-not $SshKeyPath) {
+            Write-Warning "[Diagnostics] No SshKeyPath supplied for Linux guest — skipping."
+            return
+        }
+        $sshOpts = @('-i', $SshKeyPath, '-o', 'StrictHostKeyChecking=accept-new',
+                     '-o', 'ConnectTimeout=10', '-o', 'BatchMode=yes')
+        try {
+            # Collect: journalctl errors, ps, df
+            $diagCmd = "journalctl -p err -n 50 --no-pager 2>/dev/null; echo '=== ps ==='; ps aux --sort=-%cpu | head -30; echo '=== df ==='; df -h"
+            $savedEap = $ErrorActionPreference
+            $ErrorActionPreference = 'Continue'
+            $diagOutput = & ssh.exe @sshOpts "${SshUser}@${IPAddress}" $diagCmd 2>&1
+            $ErrorActionPreference = $savedEap
+            $diagOutput | Set-Content -Path (Join-Path $DestinationDir 'guest-diag.txt') -Encoding UTF8
+            Write-Host "[Diagnostics] Linux diagnostics written."
+
+            # Best-effort: copy CI build log
+            $scpSrc = "${SshUser}@${IPAddress}:/opt/ci/build/ci-build.log"
+            $scpDst = Join-Path $DestinationDir 'build.log'
+            $ErrorActionPreference = 'Continue'
+            & scp.exe @sshOpts $scpSrc $scpDst 2>&1 | Out-Null
+            $ErrorActionPreference = $savedEap
+            if (Test-Path $scpDst) { Write-Host "[Diagnostics] Build log copied." }
+            else                   { Write-Host "[Diagnostics] Build log not found in guest (skipping)." }
+        } catch {
+            Write-Warning "[Diagnostics] Linux guest diagnostics failed (non-fatal): $_"
+        }
+    }
+}
+
+Export-ModuleMember -Function New-CISessionOption, Resolve-VmrunPath, Invoke-Vmrun, Invoke-VmrunBounded, Get-GuestIPAddress, Get-GuestDiagnostics

scripts/_Transport.psm1

@@ -40,6 +40,12 @@ function Invoke-SshCommand {
 
         [int] $ConnectTimeout = 10,
 
+        # Persistent known_hosts file for CI jobs.
+        # When non-empty: StrictHostKeyChecking=accept-new + this file.
+        # When empty (default): permissive mode (StrictHostKeyChecking=no, UserKnownHostsFile=NUL)
+        # — used during template provisioning when host key is not yet known.
+        [string] $KnownHostsFile = '',
+
         # Capture + return stdout/stderr instead of printing to console.
         [switch] $PassThru,
 
@@ -47,10 +53,14 @@ function Invoke-SshCommand {
         [switch] $AllowFail
     )
 
+    if ($KnownHostsFile -ne '') {
+        $sshHostKeyOpts = @('-o', 'StrictHostKeyChecking=accept-new', '-o', "UserKnownHostsFile=$KnownHostsFile")
+    } else {
+        $sshHostKeyOpts = @('-o', 'StrictHostKeyChecking=no', '-o', 'UserKnownHostsFile=NUL')
+    }
     $sshArgs = @(
-        '-i', $KeyPath,
+        '-i', $KeyPath
-        '-o', 'StrictHostKeyChecking=no',
+    ) + $sshHostKeyOpts + @(
-        '-o', 'UserKnownHostsFile=NUL',
         '-o', "ConnectTimeout=$ConnectTimeout",
         '-o', 'BatchMode=yes',
         "$User@$IP",
@@ -104,7 +114,12 @@ function Copy-SshItem {
         # Pass -r to scp (recursive copy).
         [switch] $Recurse,
 
-        [int] $ConnectTimeout = 10
+        [int] $ConnectTimeout = 10,
+
+        # Persistent known_hosts file for CI jobs.
+        # When non-empty: StrictHostKeyChecking=accept-new + this file.
+        # When empty (default): permissive mode — used during template provisioning.
+        [string] $KnownHostsFile = ''
     )
 
     if ($Direction -eq 'ToGuest') {
@@ -115,10 +130,14 @@ function Copy-SshItem {
     }
     # Direction = 'Direct': use Source + Destination as-is
 
+    if ($KnownHostsFile -ne '') {
+        $scpHostKeyOpts = @('-o', 'StrictHostKeyChecking=accept-new', '-o', "UserKnownHostsFile=$KnownHostsFile")
+    } else {
+        $scpHostKeyOpts = @('-o', 'StrictHostKeyChecking=no', '-o', 'UserKnownHostsFile=NUL')
+    }
     $scpArgs = @(
-        '-i', $KeyPath,
+        '-i', $KeyPath
-        '-o', 'StrictHostKeyChecking=no',
+    ) + $scpHostKeyOpts + @(
-        '-o', 'UserKnownHostsFile=NUL',
         '-o', "ConnectTimeout=$ConnectTimeout"
     )
     if ($Recurse) { $scpArgs += '-r' }

template/Validate-DeployState.ps1

@@ -1,35 +1,130 @@
 #Requires -Version 5.1
 <#
 .SYNOPSIS
-    Validates that Deploy-WinBuild2025.ps1 has correctly configured a guest VM.
+    Validates that a guest VM has been correctly configured.
 
 .DESCRIPTION
-    Connects to the VM via WinRM (HTTP/Basic) and runs all checks that
+    For Windows guests: connects via WinRM (HTTPS/Basic) and runs all checks
-    Install-CIToolchain-WinBuild2025.ps1 Steps 1/2/3/4b/5b/5c assert as validation-only.
+    that Install-CIToolchain-WinBuild2025.ps1 asserts as validation-only.
-    Use this after Deploy completes and before running Prepare/Install-CIToolchain.
+
+    For Linux guests: connects via SSH and verifies that ci-report-ip.service
+    is enabled and that the ci-report-ip.sh script is executable.
+
+    Use this after Deploy completes and before taking the BaseClean snapshot.
 
 .PARAMETER VMIPAddress
     IP address of the guest VM on VMnet8 (e.g. 192.168.79.129).
 
+.PARAMETER GuestOS
+    Target OS: Windows (default) or Linux.
+
 .PARAMETER AdminUsername
-    Admin account inside the VM (default: Administrator).
+    Windows only — admin account inside the VM (default: Administrator).
+    Prompted if -GuestOS Windows and -AdminPassword is not provided.
 
 .PARAMETER AdminPassword
-    Password for the admin account. Prompted if omitted.
+    Windows only — password for the admin account.
+
+.PARAMETER SshKeyPath
+    Linux only — path to the SSH private key (no passphrase).
+    Default: F:\CI\keys\ci_linux
+
+.PARAMETER SshUser
+    Linux only — SSH login user inside the VM.
+    Default: ci_build
 
 .EXAMPLE
+    # Windows
     .\Validate-DeployState.ps1 -VMIPAddress 192.168.79.129
+
+.EXAMPLE
+    # Linux
+    .\Validate-DeployState.ps1 -VMIPAddress 192.168.79.200 -GuestOS Linux
 #>
 [CmdletBinding()]
 param(
     [Parameter(Mandatory)] [string] $VMIPAddress,
+
+    [ValidateSet('Windows', 'Linux')]
+    [string] $GuestOS = 'Windows',
+
+    # Windows params
     [string] $AdminUsername = 'Administrator',
-    [string] $AdminPassword
+    [string] $AdminPassword,
+
+    # Linux params
+    [string] $SshKeyPath = 'F:\CI\keys\ci_linux',
+    [string] $SshUser    = 'ci_build'
 )
 
 Set-StrictMode -Version Latest
 $ErrorActionPreference = 'Stop'
 
+# ── Linux branch ──────────────────────────────────────────────────────────────
+if ($GuestOS -eq 'Linux') {
+    if (-not (Test-Path $SshKeyPath -PathType Leaf)) {
+        Write-Error "SSH key not found: $SshKeyPath"
+        exit 1
+    }
+
+    $sshBase = @(
+        '-i', $SshKeyPath,
+        '-o', 'StrictHostKeyChecking=no',
+        '-o', 'UserKnownHostsFile=NUL',
+        '-o', 'ConnectTimeout=10',
+        '-o', 'BatchMode=yes',
+        "$SshUser@$VMIPAddress"
+    )
+
+    Write-Host "`nValidating Linux Deploy state on $VMIPAddress..." -ForegroundColor Cyan
+
+    $checks = [System.Collections.Generic.List[pscustomobject]]::new()
+
+    function Add-LinuxCheck {
+        param([string]$Name, [string]$SshCommand, [scriptblock]$Validator)
+        $savedEap = $ErrorActionPreference; $ErrorActionPreference = 'Continue'
+        $out  = (& ssh.exe @sshBase $SshCommand 2>&1)
+        $rc   = $LASTEXITCODE
+        $ErrorActionPreference = $savedEap
+        $pass = [bool](& $Validator $out $rc)
+        $checks.Add([pscustomobject]@{ Name = $Name; Pass = $pass; Err = if (-not $pass) { "rc=$rc out=$out" } else { '' } })
+    }
+
+    # ci-report-ip.service must be enabled
+    Add-LinuxCheck `
+        -Name 'ci-report-ip.service enabled' `
+        -SshCommand 'systemctl is-enabled ci-report-ip.service 2>/dev/null' `
+        -Validator { param($out, $rc) $rc -eq 0 -and "$out".Trim() -eq 'enabled' }
+
+    # ci-report-ip.sh must be executable
+    Add-LinuxCheck `
+        -Name '/usr/local/bin/ci-report-ip.sh executable' `
+        -SshCommand 'test -x /usr/local/bin/ci-report-ip.sh && echo ok' `
+        -Validator { param($out, $rc) $rc -eq 0 }
+
+    Write-Host ''
+    $failed = 0
+    foreach ($r in $checks) {
+        if ($r.Pass) {
+            Write-Host "  [OK]   $($r.Name)" -ForegroundColor Green
+        } else {
+            $detail = if ($r.Err) { "  ($($r.Err))" } else { '' }
+            Write-Host "  [FAIL] $($r.Name)$detail" -ForegroundColor Red
+            $failed++
+        }
+    }
+
+    Write-Host ''
+    if ($failed -eq 0) {
+        Write-Host "All $($checks.Count) Linux Deploy checks passed." -ForegroundColor Green
+        exit 0
+    } else {
+        Write-Host "$failed / $($checks.Count) Linux Deploy checks FAILED." -ForegroundColor Red
+        exit 1
+    }
+}
+
+# ── Windows branch (WinRM HTTPS) ──────────────────────────────────────────────
 if (-not $AdminPassword) {
     $secure      = Read-Host "Password for $AdminUsername@$VMIPAddress" -AsSecureString
     $AdminPassword = [Runtime.InteropServices.Marshal]::PtrToStringAuto(

tests/_Common.Tests.ps1

@@ -88,3 +88,94 @@ Describe 'Invoke-Vmrun' {
         $env:FAKE_VMRUN_EXIT = $null
     }
 }
+
+# ─────────────────────────────────────────────────────────────────────────────
+Describe 'Invoke-VmrunBounded' {
+    BeforeAll {
+        # Fake vmrun that sleeps for $FAKE_VMRUN_SLEEP seconds then exits with
+        # $FAKE_VMRUN_EXIT. Both default to 0 if unset.
+        $script:SlowFakeVmrun = Join-Path $env:TEMP "fake-vmrun-slow-$PID.cmd"
+        Set-Content $script:SlowFakeVmrun -Value @'
+@echo off
+if "%FAKE_VMRUN_SLEEP%"=="" goto :nosleep
+ping -n %FAKE_VMRUN_SLEEP% 127.0.0.1 >nul
+:nosleep
+echo fake-vmrun-bounded-output
+if "%FAKE_VMRUN_EXIT%"=="" ( exit /b 0 ) else ( exit /b %FAKE_VMRUN_EXIT% )
+'@
+    }
+
+    AfterAll {
+        Remove-Item $script:SlowFakeVmrun -Force -ErrorAction SilentlyContinue
+    }
+
+    AfterEach {
+        $env:FAKE_VMRUN_EXIT  = $null
+        $env:FAKE_VMRUN_SLEEP = $null
+    }
+
+    It 'returns ExitCode 0, TimedOut false and Output when command succeeds quickly' {
+        $env:FAKE_VMRUN_EXIT  = '0'
+        $env:FAKE_VMRUN_SLEEP = '0'
+        $r = Invoke-VmrunBounded -VmrunPath $script:SlowFakeVmrun -Operation 'start' -Arguments @('some.vmx', 'nogui') -TimeoutSeconds 30
+        $r.ExitCode    | Should -Be 0
+        $r.TimedOut    | Should -Be $false
+        "$($r.Output)" | Should -Match 'fake-vmrun-bounded-output'
+        $r.ElapsedSeconds | Should -BeGreaterOrEqual 0
+    }
+
+    It 'returns non-zero ExitCode without throwing by default' {
+        $env:FAKE_VMRUN_EXIT  = '3'
+        $env:FAKE_VMRUN_SLEEP = '0'
+        $r = Invoke-VmrunBounded -VmrunPath $script:SlowFakeVmrun -Operation 'deleteVM' -TimeoutSeconds 30
+        $r.ExitCode | Should -Be 3
+        $r.TimedOut | Should -Be $false
+    }
+
+    It 'throws when -ThrowOnError and ExitCode is non-zero' {
+        $env:FAKE_VMRUN_EXIT  = '2'
+        $env:FAKE_VMRUN_SLEEP = '0'
+        { Invoke-VmrunBounded -VmrunPath $script:SlowFakeVmrun -Operation 'stop' -TimeoutSeconds 30 -ThrowOnError } |
+            Should -Throw -ExpectedMessage '*stop failed*'
+    }
+
+    It 'does not throw when -ThrowOnError and ExitCode is 0' {
+        $env:FAKE_VMRUN_EXIT  = '0'
+        $env:FAKE_VMRUN_SLEEP = '0'
+        { Invoke-VmrunBounded -VmrunPath $script:SlowFakeVmrun -Operation 'start' -TimeoutSeconds 30 -ThrowOnError } |
+            Should -Not -Throw
+    }
+
+    It 'times out and returns TimedOut = true when process exceeds TimeoutSeconds' {
+        # FAKE_VMRUN_SLEEP uses ping -n N which waits roughly (N-1) seconds.
+        # Set sleep >> timeout so the process is guaranteed to still be running.
+        $env:FAKE_VMRUN_SLEEP = '60'
+        $env:FAKE_VMRUN_EXIT  = '0'
+        $r = Invoke-VmrunBounded -VmrunPath $script:SlowFakeVmrun -Operation 'start' -TimeoutSeconds 2
+        $r.TimedOut    | Should -Be $true
+        $r.ExitCode    | Should -Be -1
+        $r.ElapsedSeconds | Should -BeLessOrEqual 10   # killed well before 60s
+    }
+
+    It 'throws when -ThrowOnTimeout and process times out' {
+        $env:FAKE_VMRUN_SLEEP = '60'
+        $env:FAKE_VMRUN_EXIT  = '0'
+        { Invoke-VmrunBounded -VmrunPath $script:SlowFakeVmrun -Operation 'deleteVM' -TimeoutSeconds 2 -ThrowOnTimeout } |
+            Should -Throw -ExpectedMessage '*timed out*'
+    }
+
+    It 'does not throw on timeout when neither flag is set' {
+        $env:FAKE_VMRUN_SLEEP = '60'
+        $env:FAKE_VMRUN_EXIT  = '0'
+        { Invoke-VmrunBounded -VmrunPath $script:SlowFakeVmrun -Operation 'start' -TimeoutSeconds 2 } |
+            Should -Not -Throw
+    }
+
+    It 'ElapsedSeconds is populated and plausible' {
+        $env:FAKE_VMRUN_EXIT  = '0'
+        $env:FAKE_VMRUN_SLEEP = '0'
+        $r = Invoke-VmrunBounded -VmrunPath $script:SlowFakeVmrun -Operation 'list' -TimeoutSeconds 30
+        $r.ElapsedSeconds | Should -BeGreaterOrEqual 0
+        $r.ElapsedSeconds | Should -BeLessThan 30
+    }
+}