diff --git a/scripts/Measure-CIBenchmark.ps1 b/scripts/Measure-CIBenchmark.ps1 deleted file mode 100644 index 50e95be..0000000 --- a/scripts/Measure-CIBenchmark.ps1 +++ /dev/null @@ -1,335 +0,0 @@ -#Requires -Version 5.1 -<# -.SYNOPSIS - Measures CI infrastructure phase timings: clone, start, IP acquire, transport ready, destroy. - -.DESCRIPTION - Creates one or more ephemeral VMs from the template, times each phase, destroys - them, then prints a summary table. Results are appended to benchmark.jsonl - for long-term trend tracking. - - Phases measured: - clone — vmrun clone (linked clone from snapshot) - start — vmrun start (until command returns) - ip — vmrun getGuestIPAddress (polling until IP appears) - ready — TCP port reachable: 5986 (WinRM, Windows) or 22 (SSH, Linux) - destroy — stop + deleteVM + dir removal - - No actual build is run. - -.PARAMETER TemplatePath - Full path to the template VM's .vmx file. - -.PARAMETER SnapshotName - Snapshot name to clone from. Default: BaseClean - -.PARAMETER CloneBaseDir - Directory for ephemeral clone folders. - -.PARAMETER VmrunPath - Path to vmrun executable. - -.PARAMETER GuestOS - 'Windows', 'Linux', or 'Auto' (default). Auto reads guestOS from the VMX: - ubuntu-* → Linux (probe SSH/22); anything else → Windows (probe WinRM/5986). - -.PARAMETER Iterations - Number of clone→ready→destroy cycles to run. Default: 1 - -.PARAMETER TimeoutSeconds - Maximum seconds to wait for IP and transport per iteration. Default: 300 - -.PARAMETER OutputDir - Directory where benchmark.jsonl is appended. - -.EXAMPLE - # Single baseline run (Linux host, Windows template) - .\Measure-CIBenchmark.ps1 - - # 4 iterations, Linux guest template - .\Measure-CIBenchmark.ps1 -TemplatePath /var/lib/ci/templates/LinuxBuild2404/LinuxBuild2404.vmx ` - -SnapshotName BaseClean-Linux -Iterations 4 - - # Windows host - .\Measure-CIBenchmark.ps1 -TemplatePath 'F:\CI\Templates\WinBuild2025\WinBuild2025.vmx' ` - -Iterations 3 -#> -[CmdletBinding()] -param( - [string] $TemplatePath = $(if ($null -ne $IsWindows -and $IsWindows -eq $false) { - '/var/lib/ci/templates/WinBuild2025/WinBuild2025.vmx' - } else { - 'F:\CI\Templates\WinBuild2025\WinBuild2025.vmx' - }), - - [string] $SnapshotName = 'BaseClean', - - [string] $CloneBaseDir = $(if ($null -ne $IsWindows -and $IsWindows -eq $false) { - '/var/lib/ci/build-vms/' - } else { - 'F:\CI\BuildVMs' - }), - - [string] $VmrunPath = $(if ($null -ne $IsWindows -and $IsWindows -eq $false) { - '/usr/bin/vmrun' - } else { - 'C:\Program Files (x86)\VMware\VMware Workstation\vmrun.exe' - }), - - [ValidateSet('Auto', 'Windows', 'Linux')] - [string] $GuestOS = 'Auto', - - [ValidateRange(1, 10)] - [int] $Iterations = 1, - - [ValidateRange(60, 600)] - [int] $TimeoutSeconds = 300, - - [string] $OutputDir = $(if ($null -ne $IsWindows -and $IsWindows -eq $false) { - '/var/lib/ci/logs/' - } else { - 'F:\CI\Logs' - }), - - # Optional static IP injection (mirrors ip_pool in job.py). - # If supplied, guestinfo.ip-assignment/netmask/gateway are written into the - # cloned VMX before start so ci-static-ip.ps1 applies the IP at boot. - [string] $StaticIP = '', - [string] $Netmask = '255.255.255.0', - [string] $Gateway = '' -) - -Set-StrictMode -Version Latest -$ErrorActionPreference = 'Stop' -$ProgressPreference = 'SilentlyContinue' - -Import-Module (Join-Path $PSScriptRoot '_Common.psm1') -Force -Resolve-VmrunPath -VmrunPath $VmrunPath | Out-Null - -# Cross-platform TCP port probe (replaces Test-NetConnection which is Windows-only). -function Test-TcpPort { - param([string]$ComputerName, [int]$Port, [int]$TimeoutMs = 3000) - try { - $tcp = [System.Net.Sockets.TcpClient]::new() - $ar = $tcp.BeginConnect($ComputerName, $Port, $null, $null) - $ok = $ar.AsyncWaitHandle.WaitOne($TimeoutMs, $false) - $connected = $ok -and $tcp.Connected - $tcp.Close() - return $connected - } catch { - return $false - } -} - -# Resolve effective GuestOS from VMX when Auto. -$resolvedGuestOS = $GuestOS -if ($resolvedGuestOS -eq 'Auto') { - $vmxRaw = Get-Content -LiteralPath $TemplatePath -Raw -ErrorAction SilentlyContinue - if ($vmxRaw -match 'guestOS\s*=\s*"([^"]+)"' -and $Matches[1] -like 'ubuntu*') { - $resolvedGuestOS = 'Linux' - } else { - $resolvedGuestOS = 'Windows' - } - Write-Host "[bench] GuestOS auto-detected: $resolvedGuestOS" -} -$transportPort = if ($resolvedGuestOS -eq 'Linux') { 22 } else { 5986 } -$transportLabel = if ($resolvedGuestOS -eq 'Linux') { 'SSH/22' } else { 'WinRM/5986' } - -if (-not (Test-Path $TemplatePath -PathType Leaf)) { - throw "Template VMX not found: $TemplatePath" -} -if (-not (Test-Path $CloneBaseDir)) { - New-Item -ItemType Directory -Path $CloneBaseDir -Force | Out-Null -} -if (-not (Test-Path $OutputDir)) { - New-Item -ItemType Directory -Path $OutputDir -Force | Out-Null -} - -function Set-GuestInfoIP { - [CmdletBinding(SupportsShouldProcess)] - param([string]$VmxPath, [string]$IP, [string]$Mask, [string]$Gw) - $keys = @('guestinfo.ip-assignment', 'guestinfo.netmask', 'guestinfo.gateway') - $lines = (Get-Content -LiteralPath $VmxPath -Encoding UTF8) | - Where-Object { ($_ -split '=')[0].Trim().ToLower() -notin $keys } - $lines += "guestinfo.ip-assignment = `"$IP`"" - if ($Mask) { $lines += "guestinfo.netmask = `"$Mask`"" } - if ($Gw) { $lines += "guestinfo.gateway = `"$Gw`"" } - Set-Content -LiteralPath $VmxPath -Value $lines -Encoding UTF8 -} - -$benchmarkLog = Join-Path $OutputDir 'benchmark.jsonl' -$results = [System.Collections.Generic.List[pscustomobject]]::new() - -$useStaticIP = $StaticIP -match '^\d{1,3}(\.\d{1,3}){3}$' -if ($useStaticIP) { - Write-Host "[bench] Static IP mode: $StaticIP / $Netmask gw=$Gateway" -} else { - Write-Host "[bench] DHCP mode (no -StaticIP supplied)" -} - -Write-Host "" -Write-Host "=== CI Benchmark template: $(Split-Path $TemplatePath -Leaf) snapshot: $SnapshotName iterations: $Iterations ===" -Write-Host "" - -for ($i = 1; $i -le $Iterations; $i++) { - Write-Host "--- Iteration $i / $Iterations ---" - $runId = "bench-$(Get-Date -Format 'yyyyMMdd-HHmmss')-$i" - $cloneVmx = $null - $cloneDir = $null - - $t = [ordered]@{ - clone = $null - start = $null - ip = $null - ready = $null - destroy = $null - deltaKB = $null - vmIP = $null - error = $null - } - - try { - # ── Phase: clone ────────────────────────────────────────────────────── - $cloneName = "Clone_${runId}" - $cloneDir = Join-Path $CloneBaseDir $cloneName - $cloneVmx = Join-Path $cloneDir "$cloneName.vmx" - - Write-Host "[bench] Cloning..." - $sw = [System.Diagnostics.Stopwatch]::StartNew() - $null = Invoke-Vmrun -VmrunPath $VmrunPath -Operation clone ` - -Arguments @($TemplatePath, $cloneVmx, 'linked', '-snapshot', $SnapshotName) ` - -ThrowOnError - $t.clone = [math]::Round($sw.Elapsed.TotalSeconds, 2) - Write-Host "[bench] clone: $($t.clone)s" - - # Measure linked-clone disk footprint - try { - $cloneSizeBytes = (Get-ChildItem -Path $cloneDir -Recurse -File -ErrorAction Stop | - Measure-Object -Property Length -Sum).Sum - $t.deltaKB = [math]::Round($cloneSizeBytes / 1KB, 0) - Write-Host "[bench] clone size: $($t.deltaKB) KB" - } catch { - Write-Warning "[bench] Could not measure clone size: $_" - } - - # ── Static IP injection (before start) ─────────────────────────────── - if ($useStaticIP) { - Set-GuestInfoIP -VmxPath $cloneVmx -IP $StaticIP -Mask $Netmask -Gw $Gateway - Write-Host "[bench] guestinfo injected: $StaticIP" - } - - # ── Phase: start ────────────────────────────────────────────────────── - Write-Host "[bench] Starting VM..." - $sw.Restart() - Invoke-Vmrun -VmrunPath $VmrunPath -Operation start ` - -Arguments @($cloneVmx, 'nogui') -ThrowOnError | Out-Null - $t.start = [math]::Round($sw.Elapsed.TotalSeconds, 2) - Write-Host "[bench] start: $($t.start)s" - - # ── Phase: IP acquire (primary: guestinfo.ci-ip, fallback: getGuestIPAddress) ── - Write-Host "[bench] Waiting for guest IP..." - $sw.Restart() - $vmIP = Get-GuestIPAddress -VmrunPath $VmrunPath -VmxPath $cloneVmx ` - -TimeoutSeconds $TimeoutSeconds -GuestInfoOnly:$useStaticIP - if (-not $vmIP) { throw "Timed out waiting for guest IP after $TimeoutSeconds s" } - $t.ip = [math]::Round($sw.Elapsed.TotalSeconds, 2) - $t.vmIP = $vmIP - Write-Host "[bench] ip ($vmIP): $($t.ip)s" - - # ── Phase: transport ready (SSH/22 for Linux, WinRM/5986 for Windows) ── - Write-Host "[bench] Waiting for $transportLabel..." - $sw.Restart() - $deadline = (Get-Date).AddSeconds($TimeoutSeconds) - $transportUp = $false - while ((Get-Date) -lt $deadline) { - $transportUp = Test-TcpPort -ComputerName $vmIP -Port $transportPort - if ($transportUp) { break } - Start-Sleep -Seconds 3 - } - if (-not $transportUp) { - throw "Timed out waiting for $transportLabel on $vmIP after $TimeoutSeconds s" - } - $t.ready = [math]::Round($sw.Elapsed.TotalSeconds, 2) - Write-Host "[bench] ready ($transportLabel): $($t.ready)s" - } - catch { - $t.error = "$_" - Write-Warning "[bench] Iteration $i failed: $_" - } - finally { - # ── Phase: destroy ──────────────────────────────────────────────────── - if ($cloneVmx -and (Test-Path $cloneVmx -PathType Leaf)) { - Write-Host "[bench] Destroying clone..." - $sw = [System.Diagnostics.Stopwatch]::StartNew() - & (Join-Path $PSScriptRoot 'Remove-BuildVM.ps1') ` - -VMPath $cloneVmx -VmrunPath $VmrunPath -GracefulTimeoutSeconds 10 ` - -ErrorAction SilentlyContinue - $t.destroy = [math]::Round($sw.Elapsed.TotalSeconds, 2) - Write-Host "[bench] destroy: $($t.destroy)s" - } elseif ($cloneDir -and (Test-Path $cloneDir)) { - Remove-Item $cloneDir -Recurse -Force -ErrorAction SilentlyContinue - } - } - - $totalBootSec = if ($null -ne $t.ip -and $null -ne $t.ready) { - [math]::Round($t.clone + $t.start + $t.ip + $t.ready, 2) - } else { $null } - - $result = [pscustomobject]@{ - ts = (Get-Date -Format 'o') - runId = $runId - iteration = $i - template = Split-Path $TemplatePath -Leaf - snapshot = $SnapshotName - guestOS = $resolvedGuestOS - staticIP = if ($useStaticIP) { $StaticIP } else { $null } - cloneSec = $t.clone - startSec = $t.start - ipSec = $t.ip - readySec = $t.ready - destroySec = $t.destroy - totalBootSec = $totalBootSec - deltaKB = $t.deltaKB - vmIP = $t.vmIP - error = $t.error - } - $results.Add($result) - - # Append to JSONL log - try { - $result | ConvertTo-Json -Compress -Depth 3 | - Add-Content -Path $benchmarkLog -Encoding UTF8 -ErrorAction Stop - } catch { - Write-Warning "[bench] Could not write benchmark.jsonl: $_" - } - - Write-Host "" -} - -# ── Summary table ───────────────────────────────────────────────────────────── -Write-Host "=== Results ===" -$results | Format-Table @( - @{ L = 'Iter'; E = { $_.iteration } } - @{ L = 'Clone(s)'; E = { $_.cloneSec } } - @{ L = 'Start(s)'; E = { $_.startSec } } - @{ L = 'IP(s)'; E = { $_.ipSec } } - @{ L = 'Ready(s)'; E = { $_.readySec } } - @{ L = 'Destroy(s)'; E = { $_.destroySec } } - @{ L = 'Boot total'; E = { $_.totalBootSec } } - @{ L = 'Delta(KB)'; E = { $_.deltaKB } } - @{ L = 'IP'; E = { $_.vmIP } } - @{ L = 'Error'; E = { $_.error } } -) -AutoSize - -if ($Iterations -gt 1) { - $ok = @($results | Where-Object { -not $_.error }) - if ($ok.Count -gt 0) { - $avgBoot = [math]::Round(($ok | ForEach-Object { $_.totalBootSec } | - Measure-Object -Average).Average, 2) - Write-Host "Average boot-to-ready ($($ok.Count) successful runs): $avgBoot s" - } -} - -Write-Host "" -Write-Host "Results appended to: $benchmarkLog" - diff --git a/scripts/_Common.psm1 b/scripts/_Common.psm1 deleted file mode 100644 index 20d9bdf..0000000 --- a/scripts/_Common.psm1 +++ /dev/null @@ -1,435 +0,0 @@ -#Requires -Version 5.1 -<# -.SYNOPSIS - Shared helper functions for the Local CI/CD System scripts. - -.DESCRIPTION - Import this module at the top of any CI script that needs WinRM session - options or vmrun wrappers: - - Import-Module (Join-Path $PSScriptRoot '_Common.psm1') -Force - - Exported functions: - New-CISessionOption — PSSessionOption for WinRM over self-signed TLS - Resolve-VmrunPath — Validates vmrun.exe path, throws if missing - Invoke-Vmrun — Uniform vmrun -T ws wrapper; returns ExitCode + Output -#> -Set-StrictMode -Version Latest -$ErrorActionPreference = 'Stop' - -function New-CISessionOption { -<# -.SYNOPSIS - Returns a PSSessionOption for WinRM HTTPS with self-signed certificate. -.DESCRIPTION - All CI build VMs use a self-signed TLS cert on WinRM port 5986. - This option set skips CA, CN, and revocation checks — appropriate for - an isolated lab network where PKI is not present. -.OUTPUTS - [System.Management.Automation.Remoting.PSSessionOption] -#> - [OutputType([System.Management.Automation.Remoting.PSSessionOption])] - [Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSUseShouldProcessForStateChangingFunctions', '', - Justification = 'New-CISessionOption creates an in-memory PSSessionOption object; no system state is changed.')] - [CmdletBinding()] - param() - New-PSSessionOption -SkipCACheck -SkipCNCheck -SkipRevocationCheck -} - -function Resolve-VmrunPath { -<# -.SYNOPSIS - Validates that vmrun.exe exists at the specified path and returns it. -.DESCRIPTION - Throws a descriptive error if the file is not found, rather than letting - a later & call fail with a less useful message. -.PARAMETER VmrunPath - Full path to vmrun.exe. - Default: C:\Program Files (x86)\VMware\VMware Workstation\vmrun.exe -.OUTPUTS - [string] The validated VmrunPath. -#> - [OutputType([string])] - [CmdletBinding()] - param( - [string] $VmrunPath = 'C:\Program Files (x86)\VMware\VMware Workstation\vmrun.exe' - ) - if (-not (Test-Path $VmrunPath -PathType Leaf)) { - throw "vmrun.exe not found at: $VmrunPath`nInstall VMware Workstation Pro and update -VmrunPath." - } - return $VmrunPath -} - -function Invoke-Vmrun { -<# -.SYNOPSIS - Runs a vmrun -T ws command and returns output and exit code. -.DESCRIPTION - Uniform wrapper so all vmrun calls share the same -T ws flag and output - capture pattern. Use -ThrowOnError for operations that must succeed - (clone, start); omit it for best-effort operations (stop, getState). -.PARAMETER VmrunPath - Full path to vmrun.exe. -.PARAMETER Operation - vmrun sub-command: clone, start, stop, deleteVM, list, getState, - listSnapshots, getGuestIPAddress, etc. -.PARAMETER Arguments - Additional positional arguments after the operation name. -.PARAMETER ThrowOnError - When set, throws if vmrun exits non-zero. -.OUTPUTS - [pscustomobject] with: - ExitCode [int] — vmrun exit code - Output [string[]] — combined stdout/stderr lines -#> - [OutputType([pscustomobject])] - [CmdletBinding()] - param( - [Parameter(Mandatory)] [string] $VmrunPath, - [Parameter(Mandatory)] [string] $Operation, - [string[]] $Arguments = @(), - [switch] $ThrowOnError - ) - $cmdArgs = @('-T', 'ws', $Operation) + $Arguments - $output = & $VmrunPath @cmdArgs 2>&1 - $exit = $LASTEXITCODE - - if ($ThrowOnError -and $exit -ne 0) { - throw "vmrun $Operation failed (exit $exit): $($output -join '; ')" - } - - return [pscustomobject]@{ - ExitCode = $exit - Output = $output - } -} - -function Invoke-VmrunBounded { -<# -.SYNOPSIS - Runs a vmrun -T ws command with a hard wall-clock timeout. -.DESCRIPTION - Uses [System.Diagnostics.Process] directly (not Start-Process -PassThru) because - Start-Process in PowerShell 5.1 does not reliably populate Process.ExitCode after - WaitForExit(ms) when output is redirected. ReadToEndAsync() is used for stdout and - stderr to prevent deadlock if output buffers fill up before the process exits. - - Use for lifecycle operations: start (180s), stop (60s), deleteVM (90s). - Keep Invoke-Vmrun for fast queries: list, readVariable, getGuestIPAddress, getState. -.PARAMETER VmrunPath - Full path to vmrun.exe. -.PARAMETER Operation - vmrun sub-command (start, stop, deleteVM, clone, …). -.PARAMETER Arguments - Additional positional arguments after the operation name. -.PARAMETER TimeoutSeconds - Hard timeout in seconds. Process is killed if not finished within this time. - Default: 120. -.PARAMETER ThrowOnTimeout - When set, throws if the timeout is reached before vmrun exits. -.PARAMETER ThrowOnError - When set, throws if vmrun exits with a non-zero code. -.OUTPUTS - [pscustomobject] with: - ExitCode [int] — vmrun exit code (-1 if killed due to timeout) - Output [string] — combined stdout/stderr text - TimedOut [bool] — true if the process was killed for exceeding TimeoutSeconds - ElapsedSeconds [double] — wall-clock seconds from start to exit/kill -#> - [OutputType([pscustomobject])] - [CmdletBinding()] - param( - [Parameter(Mandatory)] [string] $VmrunPath, - [Parameter(Mandatory)] [string] $Operation, - [string[]] $Arguments = @(), - [int] $TimeoutSeconds = 120, - [switch] $ThrowOnTimeout, - [switch] $ThrowOnError - ) - - $cmdArgs = @('-T', 'ws', $Operation) + $Arguments - - # Quote tokens that contain spaces so the argument string is correct when - # passed to ProcessStartInfo.Arguments as a single string. - $quotedArgs = $cmdArgs | ForEach-Object { - if ($_ -match '\s') { '"' + ($_ -replace '"', '""') + '"' } else { $_ } - } - $argStr = $quotedArgs -join ' ' - - $psi = New-Object System.Diagnostics.ProcessStartInfo - $psi.FileName = $VmrunPath - $psi.Arguments = $argStr - $psi.UseShellExecute = $false - $psi.CreateNoWindow = $true - $psi.RedirectStandardOutput = $true - $psi.RedirectStandardError = $true - - $proc = New-Object System.Diagnostics.Process - $proc.StartInfo = $psi - $sw = [System.Diagnostics.Stopwatch]::StartNew() - $timedOut = $false - - [void]$proc.Start() - - # Begin async reads before WaitForExit to avoid deadlock when output buffers fill. - $stdoutTask = $proc.StandardOutput.ReadToEndAsync() - $stderrTask = $proc.StandardError.ReadToEndAsync() - - $exited = $proc.WaitForExit($TimeoutSeconds * 1000) - if (-not $exited) { - $timedOut = $true - # Kill the entire process tree (taskkill /T) to release any inherited pipe - # handles held by child processes (e.g. cmd.exe spawning ping.exe in tests, - # or vmrun spawning helper subprocesses). Plain proc.Kill() only kills the - # direct process; child processes keep the write end of stdout open. - if ($null -ne $IsWindows -and $IsWindows -eq $false) { - # .NET 5+ Kill(entireProcessTree) — works on Linux/macOS. - $proc.Kill($true) - } else { - & taskkill /F /T /PID $proc.Id 2>&1 | Out-Null - } - } - $sw.Stop() - - $outputText = '' - if (-not $timedOut) { - # Process exited normally: drain stdout/stderr (up to 5s safety net). - [void][System.Threading.Tasks.Task]::WhenAll($stdoutTask, $stderrTask).Wait(5000) - $stdoutText = '' - $stderrText = '' - if ($stdoutTask.Status -eq [System.Threading.Tasks.TaskStatus]::RanToCompletion) { - $stdoutText = $stdoutTask.Result - } - if ($stderrTask.Status -eq [System.Threading.Tasks.TaskStatus]::RanToCompletion) { - $stderrText = $stderrTask.Result - } - $parts = @($stdoutText, $stderrText) | Where-Object { -not [string]::IsNullOrWhiteSpace($_) } - $outputText = ($parts | ForEach-Object { $_.TrimEnd() }) -join "`n" - } - # When timed-out: process was killed; output is irrelevant and potentially - # incomplete. Skip drain entirely — no point blocking for a killed process. - - $exitCode = if ($timedOut) { -1 } else { $proc.ExitCode } - - $result = [pscustomobject]@{ - ExitCode = $exitCode - Output = $outputText - TimedOut = $timedOut - ElapsedSeconds = [math]::Round($sw.Elapsed.TotalSeconds, 1) - } - - if ($timedOut -and $ThrowOnTimeout) { - throw "vmrun $Operation timed out after ${TimeoutSeconds}s" - } - if (-not $timedOut -and $exitCode -ne 0 -and $ThrowOnError) { - throw "vmrun $Operation failed (exit $exitCode): $outputText" - } - - return $result -} - -function Get-GuestIPAddress { -<# -.SYNOPSIS - Polls a running VMware guest VM until its IP address is available, then returns it. -.DESCRIPTION - Primary method: reads the 'ci-ip' guestVar written by ci-report-ip.service (Linux) - or the equivalent guest-side script via vmware-rpctool. This uses the VMware VMCI - channel and does not require TCP/IP connectivity from the host. - - Fallback: vmrun getGuestIPAddress (reliable on Windows guests; may be slow on Linux). - - Returns the detected IP as a plain string, or an empty string if the timeout expires. -.PARAMETER VmrunPath - Full path to vmrun.exe. -.PARAMETER VmxPath - Full path to the clone VM's .vmx file. -.PARAMETER TimeoutSeconds - Maximum seconds to poll before giving up. Default: 120. -.OUTPUTS - [string] Detected IPv4 address, or empty string on timeout. -#> - [OutputType([string])] - [CmdletBinding()] - param( - [Parameter(Mandatory)] [string] $VmrunPath, - [Parameter(Mandatory)] [string] $VmxPath, - [int] $TimeoutSeconds = 120, - # When set, skip vmrun getGuestIPAddress fallback and poll only - # guestinfo.ci-ip. Use when guestinfo.ip-assignment was injected into - # the VMX — avoids returning a transient DHCP address before - # ci-static-ip.ps1 has finished reconfiguring the NIC. - [switch] $GuestInfoOnly - ) - - $modeLabel = if ($GuestInfoOnly) { 'guestinfo only' } else { 'guestinfo + getGuestIPAddress' } - Write-Host "[Get-GuestIPAddress] Polling for VM IP (max ${TimeoutSeconds}s, mode: $modeLabel)..." - $deadline = (Get-Date).AddSeconds($TimeoutSeconds) - $attempt = 0 - - while ((Get-Date) -lt $deadline) { - $attempt++ - - # Primary: guestinfo.ci-ip written by ci-report-ip.service via vmware-rpctool. - # NOTE: vmrun readVariable guestVar reads WITHOUT the 'guestinfo.' prefix. - $savedEap = $ErrorActionPreference; $ErrorActionPreference = 'Continue' - $ipOut = (& $VmrunPath -T ws readVariable $VmxPath guestVar 'ci-ip' 2>&1).Trim() - $ipRc = $LASTEXITCODE - $ErrorActionPreference = $savedEap - - if ($attempt -le 3 -or ($attempt % 10 -eq 0)) { - Write-Host "[Get-GuestIPAddress] [attempt $attempt] guestVar ci-ip: rc=$ipRc out='$ipOut'" - } - if ($ipRc -eq 0 -and $ipOut -match '^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$') { - Write-Host "[Get-GuestIPAddress] IP via guestinfo: $ipOut" - return $ipOut - } - - # Fallback: vmrun getGuestIPAddress — skipped when GuestInfoOnly is set - # to avoid returning a transient DHCP address during static-IP reconfiguration. - if (-not $GuestInfoOnly) { - $savedEap = $ErrorActionPreference; $ErrorActionPreference = 'Continue' - $ipOut2 = (& $VmrunPath -T ws getGuestIPAddress $VmxPath 2>&1).Trim() - $ipRc2 = $LASTEXITCODE - $ErrorActionPreference = $savedEap - - if ($ipRc2 -eq 0 -and $ipOut2 -match '^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$') { - Write-Host "[Get-GuestIPAddress] IP via getGuestIPAddress: $ipOut2" - return $ipOut2 - } - } - - Start-Sleep -Seconds 2 - } - - return '' -} - -function Get-GuestDiagnostics { -<# -.SYNOPSIS - Best-effort collection of guest diagnostics after a build failure. - Never throws — always safe to call from a catch/finally block. - -.PARAMETER IPAddress - IP address of the guest VM. - -.PARAMETER GuestOS - 'Windows' or 'Linux'. - -.PARAMETER Credential - PSCredential for WinRM (Windows guests). Ignored for Linux. - -.PARAMETER SshKeyPath - Path to SSH private key (Linux guests). Ignored for Windows. - -.PARAMETER SshUser - SSH username (Linux guests). Default: ci_build. - -.PARAMETER DestinationDir - Host directory where diagnostics files are written. -#> - [CmdletBinding()] - param( - [Parameter(Mandatory)] [string] $IPAddress, - [Parameter(Mandatory)] [ValidateSet('Windows','Linux')] - [string] $GuestOS, - [pscredential] $Credential, - [string] $SshKeyPath, - [string] $SshUser = 'ci_build', - [Parameter(Mandatory)] [string] $DestinationDir - ) - - Write-Host "[Diagnostics] Collecting guest diagnostics from $IPAddress ($GuestOS) -> $DestinationDir" - try { - if (-not (Test-Path $DestinationDir)) { - New-Item -ItemType Directory -Path $DestinationDir -Force | Out-Null - } - } catch { - Write-Warning "[Diagnostics] Could not create destination dir: $_" - return - } - - if ($GuestOS -eq 'Windows') { - if (-not $Credential) { - Write-Warning "[Diagnostics] No Credential supplied for Windows guest — skipping." - return - } - try { - $sessionOption = New-CISessionOption - $session = New-PSSession -ComputerName $IPAddress ` - -Credential $Credential ` - -SessionOption $sessionOption ` - -ErrorAction Stop - try { - # Collect: event log errors (last 50), running processes, disk usage - $diagScript = { - $out = [System.Text.StringBuilder]::new() - [void]$out.AppendLine("=== System Event Log (last 50 errors) ===") - Get-EventLog -LogName System -EntryType Error -Newest 50 -ErrorAction SilentlyContinue | - ForEach-Object { [void]$out.AppendLine("$($_.TimeGenerated) $($_.Source): $($_.Message)") } - [void]$out.AppendLine("`n=== Running Processes ===") - Get-Process -ErrorAction SilentlyContinue | - Sort-Object CPU -Descending | - Select-Object -First 30 | - ForEach-Object { [void]$out.AppendLine("$($_.Name) PID=$($_.Id) CPU=$($_.CPU)") } - [void]$out.AppendLine("`n=== Disk Free Space ===") - Get-PSDrive -PSProvider FileSystem -ErrorAction SilentlyContinue | - ForEach-Object { [void]$out.AppendLine("$($_.Name): Free=$([math]::Round($_.Free/1GB,1))GB Used=$([math]::Round($_.Used/1GB,1))GB") } - $out.ToString() - } - $diagText = Invoke-Command -Session $session -ScriptBlock $diagScript -ErrorAction Stop - $diagText | Set-Content -Path (Join-Path $DestinationDir 'guest-diag.txt') -Encoding UTF8 - Write-Host "[Diagnostics] Windows diagnostics written." - - # Best-effort: copy CI build log - $buildLog = 'C:\CI\build.log' - try { - Copy-Item -FromSession $session -Path $buildLog ` - -Destination (Join-Path $DestinationDir 'build.log') ` - -ErrorAction Stop - Write-Host "[Diagnostics] Build log copied." - } catch { - Write-Host "[Diagnostics] Build log not found in guest (skipping)." - } - } finally { - Remove-PSSession $session -ErrorAction SilentlyContinue - } - } catch { - Write-Warning "[Diagnostics] Windows guest diagnostics failed (non-fatal): $_" - } - } - else { - # Linux - if (-not $SshKeyPath) { - Write-Warning "[Diagnostics] No SshKeyPath supplied for Linux guest — skipping." - return - } - $sshOpts = @('-i', $SshKeyPath, '-o', 'StrictHostKeyChecking=no', - '-o', 'UserKnownHostsFile=NUL', - '-o', 'ConnectTimeout=10', '-o', 'BatchMode=yes') - try { - # Collect: journalctl errors, ps, df - $diagCmd = "journalctl -p err -n 50 --no-pager 2>/dev/null; echo '=== ps ==='; ps aux --sort=-%cpu | head -30; echo '=== df ==='; df -h" - $savedEap = $ErrorActionPreference - $ErrorActionPreference = 'Continue' - $diagOutput = & ssh.exe @sshOpts "${SshUser}@${IPAddress}" $diagCmd 2>&1 - $ErrorActionPreference = $savedEap - $diagOutput | Set-Content -Path (Join-Path $DestinationDir 'guest-diag.txt') -Encoding UTF8 - Write-Host "[Diagnostics] Linux diagnostics written." - - # Best-effort: copy CI build log - $scpSrc = "${SshUser}@${IPAddress}:/opt/ci/build/ci-build.log" - $scpDst = Join-Path $DestinationDir 'build.log' - $ErrorActionPreference = 'Continue' - & scp.exe @sshOpts $scpSrc $scpDst 2>&1 | Out-Null - $ErrorActionPreference = $savedEap - if (Test-Path $scpDst) { Write-Host "[Diagnostics] Build log copied." } - else { Write-Host "[Diagnostics] Build log not found in guest (skipping)." } - } catch { - Write-Warning "[Diagnostics] Linux guest diagnostics failed (non-fatal): $_" - } - } -} - -Export-ModuleMember -Function New-CISessionOption, Resolve-VmrunPath, Invoke-Vmrun, Invoke-VmrunBounded, Get-GuestIPAddress, Get-GuestDiagnostics diff --git a/scripts/_Transport.psm1 b/scripts/_Transport.psm1 deleted file mode 100644 index 08558b0..0000000 --- a/scripts/_Transport.psm1 +++ /dev/null @@ -1,201 +0,0 @@ -#Requires -Version 5.1 -<# -.SYNOPSIS - SSH/SCP transport helpers for Linux build VMs. - -.DESCRIPTION - SSH-only transport layer for the Linux branch of the CI/CD system. - The existing WinRM transport (used by Windows VMs) is NOT modified. - - Exported functions: - Invoke-SshCommand — run a command in a Linux guest via ssh.exe - Copy-SshItem — copy files to/from a Linux guest via scp.exe - Test-SshReady — poll until SSH is ready (port 22 open + login succeeds) - - Requirements: - - ssh.exe and scp.exe in PATH (default on Windows 11 / Server 2022) - - SSH private key at the path provided - - Guest user must be set up for key-based auth (no passphrase) -#> -Set-StrictMode -Version Latest -$ErrorActionPreference = 'Stop' -$ProgressPreference = 'SilentlyContinue' # suppress Write-Progress (Test-NetConnection, IWR, etc.) - -function Invoke-SshCommand { - <# - .SYNOPSIS - Run a command in a Linux guest via ssh.exe - #> - [CmdletBinding()] - param( - [Parameter(Mandatory)] - [string] $IP, - - [string] $User = 'ci_build', - - [string] $KeyPath = 'F:\CI\keys\ci_linux', - - [Parameter(Mandatory)] - [string] $Command, - - [int] $ConnectTimeout = 10, - - # Persistent known_hosts file for CI jobs. - # When non-empty: StrictHostKeyChecking=accept-new + this file. - # When empty (default): permissive mode (StrictHostKeyChecking=no, UserKnownHostsFile=NUL) - # — used during template provisioning when host key is not yet known. - [string] $KnownHostsFile = '', - - # Capture + return stdout/stderr instead of printing to console. - [switch] $PassThru, - - # Don't throw on non-zero exit — just return the output. - [switch] $AllowFail - ) - - if ($KnownHostsFile -ne '') { - $sshHostKeyOpts = @('-o', 'StrictHostKeyChecking=accept-new', '-o', "UserKnownHostsFile=$KnownHostsFile") - } else { - $sshHostKeyOpts = @('-o', 'StrictHostKeyChecking=no', '-o', 'UserKnownHostsFile=NUL') - } - $sshArgs = @( - '-i', $KeyPath - ) + $sshHostKeyOpts + @( - '-o', "ConnectTimeout=$ConnectTimeout", - '-o', 'BatchMode=yes', - "$User@$IP", - $Command - ) - - if ($PassThru) { - $output = & ssh @sshArgs 2>&1 - $exit = $LASTEXITCODE - } - else { - & ssh @sshArgs - $exit = $LASTEXITCODE - $output = @() - } - - if ($exit -ne 0 -and -not $AllowFail) { - throw "[SSH] Command failed (exit $exit): $Command" - } - - return [pscustomobject]@{ - ExitCode = [int]$exit - Output = [string[]]$output - } -} - -function Copy-SshItem { - <# - .SYNOPSIS - Copy files to/from Linux guest via scp.exe - #> - [CmdletBinding()] - param( - [Parameter(Mandatory)] - [string] $Source, - - [Parameter(Mandatory)] - [string] $Destination, - - # Guest IP — used to build user@host prefix when Direction is set. - [string] $IP = '', - - [string] $User = 'ci_build', - - [Parameter(Mandatory)] - [string] $KeyPath, - - [ValidateSet('ToGuest', 'FromGuest', 'Direct')] - [string] $Direction = 'Direct', - - # Pass -r to scp (recursive copy). - [switch] $Recurse, - - [int] $ConnectTimeout = 10, - - # Persistent known_hosts file for CI jobs. - # When non-empty: StrictHostKeyChecking=accept-new + this file. - # When empty (default): permissive mode — used during template provisioning. - [string] $KnownHostsFile = '' - ) - - if ($Direction -eq 'ToGuest') { - $Destination = "$User@${IP}:$Destination" - } - elseif ($Direction -eq 'FromGuest') { - $Source = "$User@${IP}:$Source" - } - # Direction = 'Direct': use Source + Destination as-is - - if ($KnownHostsFile -ne '') { - $scpHostKeyOpts = @('-o', 'StrictHostKeyChecking=accept-new', '-o', "UserKnownHostsFile=$KnownHostsFile") - } else { - $scpHostKeyOpts = @('-o', 'StrictHostKeyChecking=no', '-o', 'UserKnownHostsFile=NUL') - } - $scpArgs = @( - '-i', $KeyPath - ) + $scpHostKeyOpts + @( - '-o', "ConnectTimeout=$ConnectTimeout" - ) - if ($Recurse) { $scpArgs += '-r' } - $scpArgs += @($Source, $Destination) - - & scp @scpArgs - if ($LASTEXITCODE -ne 0) { - throw "[SCP] Copy failed (exit $LASTEXITCODE): $Source -> $Destination" - } -} - -function Test-SshReady { - <# - .SYNOPSIS - Poll until SSH port 22 is open and a login command succeeds. - #> - [CmdletBinding()] - param( - [Parameter(Mandatory)] - [string] $IP, - - [string] $User = 'ci_build', - - [string] $KeyPath = 'F:\CI\keys\ci_linux', - - [int] $TimeoutSec = 300, - - [int] $PollSec = 10 - ) - - $deadline = (Get-Date).AddSeconds($TimeoutSec) - - # Phase 1: wait for port 22 to be open - $portOpen = $false - while (-not $portOpen -and (Get-Date) -lt $deadline) { - $portOpen = (Test-NetConnection -ComputerName $IP -Port 22 ` - -InformationLevel Quiet -WarningAction SilentlyContinue ` - -ErrorAction SilentlyContinue) - if (-not $portOpen) { - Start-Sleep -Seconds $PollSec - } - } - - if (-not $portOpen) { - throw "[Test-SshReady] Timed out after ${TimeoutSec}s waiting for SSH at $IP" - } - - # Phase 2: try ssh echo — success when output contains 'ready' - while ((Get-Date) -lt $deadline) { - $result = Invoke-SshCommand -IP $IP -User $User -KeyPath $KeyPath ` - -Command 'echo ready' -AllowFail -PassThru - if ($result.ExitCode -eq 0 -and ($result.Output -join '') -like '*ready*') { - return $true - } - Start-Sleep -Seconds $PollSec - } - - throw "[Test-SshReady] Timed out after ${TimeoutSec}s waiting for SSH at $IP" -} - -Export-ModuleMember -Function Invoke-SshCommand, Copy-SshItem, Test-SshReady diff --git a/tests/_Common.Tests.ps1 b/tests/_Common.Tests.ps1 deleted file mode 100644 index 857bfdc..0000000 --- a/tests/_Common.Tests.ps1 +++ /dev/null @@ -1,181 +0,0 @@ -#Requires -Version 5.1 -<# -.SYNOPSIS - Pester v5 tests for scripts/_Common.psm1 -#> - -BeforeAll { - Import-Module (Join-Path $PSScriptRoot '..\scripts\_Common.psm1') -Force - - # Fake vmrun: accepts any args, exits with %FAKE_VMRUN_EXIT% (default 0) - $script:FakeVmrun = Join-Path $env:TEMP "fake-vmrun-$PID.cmd" - Set-Content $script:FakeVmrun -Value @' -@echo off -echo fake-vmrun-output -if "%FAKE_VMRUN_EXIT%"=="" ( exit /b 0 ) else ( exit /b %FAKE_VMRUN_EXIT% ) -'@ -} - -AfterAll { - Remove-Item $script:FakeVmrun -Force -ErrorAction SilentlyContinue -} - -# ───────────────────────────────────────────────────────────────────────────── -Describe 'New-CISessionOption' { - It 'returns a PSSessionOption' { - $opt = New-CISessionOption - $opt | Should -BeOfType [System.Management.Automation.Remoting.PSSessionOption] - } - - It 'has SkipCACheck = true' { - (New-CISessionOption).SkipCACheck | Should -Be $true - } - - It 'has SkipCNCheck = true' { - (New-CISessionOption).SkipCNCheck | Should -Be $true - } - - It 'has SkipRevocationCheck = true' { - (New-CISessionOption).SkipRevocationCheck | Should -Be $true - } -} - -# ───────────────────────────────────────────────────────────────────────────── -Describe 'Resolve-VmrunPath' { - It 'returns the path when the file exists' { - $tmp = [System.IO.Path]::GetTempFileName() - try { - Resolve-VmrunPath -VmrunPath $tmp | Should -Be $tmp - } finally { - Remove-Item $tmp -Force -ErrorAction SilentlyContinue - } - } - - It 'throws with a descriptive message when file is missing' { - { Resolve-VmrunPath -VmrunPath 'C:\DoesNotExist\vmrun.exe' } | - Should -Throw -ExpectedMessage '*vmrun.exe not found*' - } -} - -# ───────────────────────────────────────────────────────────────────────────── -Describe 'Invoke-Vmrun' { - It 'returns ExitCode 0 and Output when command succeeds' { - $env:FAKE_VMRUN_EXIT = '0' - $r = Invoke-Vmrun -VmrunPath $script:FakeVmrun -Operation 'list' - $r.ExitCode | Should -Be 0 - "$($r.Output)" | Should -Match 'fake-vmrun-output' - } - - It 'returns non-zero ExitCode without throwing by default' { - $env:FAKE_VMRUN_EXIT = '1' - $r = Invoke-Vmrun -VmrunPath $script:FakeVmrun -Operation 'clone' - $r.ExitCode | Should -Be 1 - } - - It 'throws when -ThrowOnError and ExitCode is non-zero' { - $env:FAKE_VMRUN_EXIT = '1' - { Invoke-Vmrun -VmrunPath $script:FakeVmrun -Operation 'clone' -ThrowOnError } | - Should -Throw -ExpectedMessage '*clone failed*' - } - - It 'does not throw when -ThrowOnError and ExitCode is 0' { - $env:FAKE_VMRUN_EXIT = '0' - { Invoke-Vmrun -VmrunPath $script:FakeVmrun -Operation 'start' -ThrowOnError } | - Should -Not -Throw - } - - AfterEach { - $env:FAKE_VMRUN_EXIT = $null - } -} - -# ───────────────────────────────────────────────────────────────────────────── -Describe 'Invoke-VmrunBounded' { - BeforeAll { - # Fake vmrun that sleeps for $FAKE_VMRUN_SLEEP seconds then exits with - # $FAKE_VMRUN_EXIT. Both default to 0 if unset. - $script:SlowFakeVmrun = Join-Path $env:TEMP "fake-vmrun-slow-$PID.cmd" - Set-Content $script:SlowFakeVmrun -Value @' -@echo off -if "%FAKE_VMRUN_SLEEP%"=="" goto :nosleep -ping -n %FAKE_VMRUN_SLEEP% 127.0.0.1 >nul -:nosleep -echo fake-vmrun-bounded-output -if "%FAKE_VMRUN_EXIT%"=="" ( exit /b 0 ) else ( exit /b %FAKE_VMRUN_EXIT% ) -'@ - } - - AfterAll { - Remove-Item $script:SlowFakeVmrun -Force -ErrorAction SilentlyContinue - } - - AfterEach { - $env:FAKE_VMRUN_EXIT = $null - $env:FAKE_VMRUN_SLEEP = $null - } - - It 'returns ExitCode 0, TimedOut false and Output when command succeeds quickly' { - $env:FAKE_VMRUN_EXIT = '0' - $env:FAKE_VMRUN_SLEEP = '0' - $r = Invoke-VmrunBounded -VmrunPath $script:SlowFakeVmrun -Operation 'start' -Arguments @('some.vmx', 'nogui') -TimeoutSeconds 30 - $r.ExitCode | Should -Be 0 - $r.TimedOut | Should -Be $false - "$($r.Output)" | Should -Match 'fake-vmrun-bounded-output' - $r.ElapsedSeconds | Should -BeGreaterOrEqual 0 - } - - It 'returns non-zero ExitCode without throwing by default' { - $env:FAKE_VMRUN_EXIT = '3' - $env:FAKE_VMRUN_SLEEP = '0' - $r = Invoke-VmrunBounded -VmrunPath $script:SlowFakeVmrun -Operation 'deleteVM' -TimeoutSeconds 30 - $r.ExitCode | Should -Be 3 - $r.TimedOut | Should -Be $false - } - - It 'throws when -ThrowOnError and ExitCode is non-zero' { - $env:FAKE_VMRUN_EXIT = '2' - $env:FAKE_VMRUN_SLEEP = '0' - { Invoke-VmrunBounded -VmrunPath $script:SlowFakeVmrun -Operation 'stop' -TimeoutSeconds 30 -ThrowOnError } | - Should -Throw -ExpectedMessage '*stop failed*' - } - - It 'does not throw when -ThrowOnError and ExitCode is 0' { - $env:FAKE_VMRUN_EXIT = '0' - $env:FAKE_VMRUN_SLEEP = '0' - { Invoke-VmrunBounded -VmrunPath $script:SlowFakeVmrun -Operation 'start' -TimeoutSeconds 30 -ThrowOnError } | - Should -Not -Throw - } - - It 'times out and returns TimedOut = true when process exceeds TimeoutSeconds' { - # FAKE_VMRUN_SLEEP uses ping -n N which waits roughly (N-1) seconds. - # Set sleep >> timeout so the process is guaranteed to still be running. - $env:FAKE_VMRUN_SLEEP = '60' - $env:FAKE_VMRUN_EXIT = '0' - $r = Invoke-VmrunBounded -VmrunPath $script:SlowFakeVmrun -Operation 'start' -TimeoutSeconds 2 - $r.TimedOut | Should -Be $true - $r.ExitCode | Should -Be -1 - $r.ElapsedSeconds | Should -BeLessOrEqual 10 # killed well before 60s - } - - It 'throws when -ThrowOnTimeout and process times out' { - $env:FAKE_VMRUN_SLEEP = '60' - $env:FAKE_VMRUN_EXIT = '0' - { Invoke-VmrunBounded -VmrunPath $script:SlowFakeVmrun -Operation 'deleteVM' -TimeoutSeconds 2 -ThrowOnTimeout } | - Should -Throw -ExpectedMessage '*timed out*' - } - - It 'does not throw on timeout when neither flag is set' { - $env:FAKE_VMRUN_SLEEP = '60' - $env:FAKE_VMRUN_EXIT = '0' - { Invoke-VmrunBounded -VmrunPath $script:SlowFakeVmrun -Operation 'start' -TimeoutSeconds 2 } | - Should -Not -Throw - } - - It 'ElapsedSeconds is populated and plausible' { - $env:FAKE_VMRUN_EXIT = '0' - $env:FAKE_VMRUN_SLEEP = '0' - $r = Invoke-VmrunBounded -VmrunPath $script:SlowFakeVmrun -Operation 'list' -TimeoutSeconds 30 - $r.ElapsedSeconds | Should -BeGreaterOrEqual 0 - $r.ElapsedSeconds | Should -BeLessThan 30 - } -}