From fee5963a65bb024c87f51cfc0d0856f65f7c0009 Mon Sep 17 00:00:00 2001 From: Simone Date: Mon, 11 May 2026 18:34:16 +0200 Subject: [PATCH] feat(linux): update orchestration scripts for dual-OS (Windows/Linux) support Wait-VMReady.ps1: - Phase 1: vmrun list + path matching instead of getGuestIPAddress (avoids hang) - Phase 2/3: WinRM (Windows) or TCP:22 + ssh echo (Linux) based on -Transport param - -Transport SSH auto-selected by Invoke-CIJob when guestOS=ubuntu-64 Invoke-CIJob.ps1: - Auto-detect GuestOS from clone VMX guestOS field - Pass -GuestOS Linux to Wait-VMReady, Invoke-RemoteBuild, Get-BuildArtifacts - Git clone output streamed line-by-line (fix buffered stdout) Invoke-RemoteBuild.ps1: - -GuestOS Linux path: SSH+SCP via _Transport.psm1 - Always uses in-VM git clone for Linux builds Get-BuildArtifacts.ps1: - -GuestOS Linux path: scp -r ci_build@:/opt/ci/output/ to host artifact dir --- scripts/Get-BuildArtifacts.ps1 | 46 +++++++++- scripts/Invoke-CIJob.ps1 | 162 ++++++++++++++++++++++++++------- scripts/Invoke-RemoteBuild.ps1 | 119 +++++++++++++++++++++++- scripts/Wait-VMReady.ps1 | 137 +++++++++++++++++++++++----- 4 files changed, 405 insertions(+), 59 deletions(-) diff --git a/scripts/Get-BuildArtifacts.ps1 b/scripts/Get-BuildArtifacts.ps1 index a8cbbd7..86b475f 100644 --- a/scripts/Get-BuildArtifacts.ps1 +++ b/scripts/Get-BuildArtifacts.ps1 @@ -42,7 +42,7 @@ param( [ValidatePattern('^((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)\.){3}(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)$')] [string] $IPAddress, - [Parameter(Mandatory)] + [Parameter()] [System.Management.Automation.PSCredential] $Credential, [Parameter(Mandatory)] @@ -51,7 +51,17 @@ param( [Parameter(Mandatory)] [string] $HostArtifactDir, - [switch] $IncludeLogs + [switch] $IncludeLogs, + + # Guest OS type — Linux uses SCP instead of WinRM. + [ValidateSet('Windows', 'Linux')] + [string] $GuestOS = 'Windows', + + # SSH private key path (used when GuestOS=Linux). + [string] $SshKeyPath = 'F:\CI\keys\ci_linux', + + # SSH username (used when GuestOS=Linux). + [string] $SshUser = 'ci_build' ) Set-StrictMode -Version Latest @@ -59,6 +69,38 @@ $ErrorActionPreference = 'Stop' Import-Module (Join-Path $PSScriptRoot '_Common.psm1') -Force +if ($GuestOS -eq 'Windows' -and $null -eq $Credential) { + throw "Credential is required when GuestOS is Windows." +} + +# ── Linux branch: use scp to transfer artifacts ───────────────────────────────────────────── +if ($GuestOS -eq 'Linux') { + Import-Module (Join-Path $PSScriptRoot '_Transport.psm1') -Force + + if (-not (Test-Path $HostArtifactDir)) { + New-Item -ItemType Directory -Path $HostArtifactDir -Force | Out-Null + Write-Host "[Get-BuildArtifacts] Created host artifact directory: $HostArtifactDir" + } + + Write-Host "[Get-BuildArtifacts] Transferring artifacts via SCP from $GuestArtifactPath ..." + + Copy-SshItem ` + -Source "$GuestArtifactPath/." ` + -Destination $HostArtifactDir ` + -IP $IPAddress ` + -User $SshUser ` + -KeyPath $SshKeyPath ` + -Direction 'FromGuest' ` + -Recurse + + $transferred = @(Get-ChildItem -Path $HostArtifactDir -Recurse -File -ErrorAction SilentlyContinue) + if ($transferred.Count -eq 0) { + throw "[Get-BuildArtifacts] No files found in $HostArtifactDir after SCP transfer." + } + Write-Host "[Get-BuildArtifacts] $($transferred.Count) file(s) collected from Linux guest." + return +} + # Ensure destination exists on host if (-not (Test-Path $HostArtifactDir)) { New-Item -ItemType Directory -Path $HostArtifactDir -Force | Out-Null diff --git a/scripts/Invoke-CIJob.ps1 b/scripts/Invoke-CIJob.ps1 index f0e56a4..09ff549 100644 --- a/scripts/Invoke-CIJob.ps1 +++ b/scripts/Invoke-CIJob.ps1 @@ -126,7 +126,15 @@ param( # Days to retain job logs. Directories older than this are purged at job end. # Set to 0 to disable retention/purge. - [int] $LogRetentionDays = 30 + [int] $LogRetentionDays = 30, + + # Guest OS type — determines transport (WinRM for Windows, SSH for Linux). + # When 'Auto', detected from the VMX guestOS field after clone. + [ValidateSet('Windows', 'Linux', 'Auto')] + [string] $GuestOS = 'Auto', + + [string] $SshKeyPath = 'F:\CI\keys\ci_linux', + [string] $SshUser = 'ci_build' ) Set-StrictMode -Version Latest @@ -245,11 +253,14 @@ try { Write-Host "`n[Phase 1/6] Cloning repository on host..." Write-JobEvent -Phase 'phase1.clone-repo' -Status 'start' if (Test-Path $hostCloneDir) { Remove-Item $hostCloneDir -Recurse -Force } - $gitArgs = @('clone', '--depth', '1', '--branch', $Branch) - if ($Submodules) { $gitArgs += '--recurse-submodules' } + $gitArgs = @('clone', '--depth', '1', '--progress', '--branch', $Branch) + if ($Submodules) { $gitArgs += @('--recurse-submodules', '--shallow-submodules') } $gitArgs += @($RepoUrl, $hostCloneDir) + Write-Host "[Invoke-CIJob] git $($gitArgs -join ' ')" $ErrorActionPreference = 'Continue' - $gitOutput = & git @gitArgs 2>&1 | ForEach-Object { "$_" } + # Stream git stdout/stderr live (otherwise heavy clones with submodules + # look frozen for a long time). Tee for diagnostic in case of failure. + $gitOutput = & git @gitArgs 2>&1 | Tee-Object -Variable streamed | ForEach-Object { Write-Host " [git] $_"; "$_" } $gitExit = $LASTEXITCODE $ErrorActionPreference = 'Stop' if ($gitExit -ne 0) { @@ -329,12 +340,54 @@ try { # ── Phase 3b: Resolve VM IP ─────────────────────────────────────── if ([string]::IsNullOrWhiteSpace($VMIPAddress)) { - Write-Host "[Invoke-CIJob] Auto-detecting VM IP via vmrun getGuestIPAddress -wait..." - $detectedIP = & $VmrunPath getGuestIPAddress $cloneVmxPath -wait 2>&1 - if ($LASTEXITCODE -ne 0 -or $detectedIP -notmatch '^(\d{1,3}\.){3}\d{1,3}$') { - throw "Could not detect VM IP address: $detectedIP" + # Primary method: vmrun readVariable guestVar guestinfo.ci-ip + # The guest ci-report-ip.service writes its IP via vmware-rpctool on every boot. + # This uses the VMware VMCI channel — reliable, official API, independent of + # whether TCP/IP is reachable from the host. + # + # Fallback: vmrun getGuestIPAddress (works on Windows guests; unreliable on Linux). + Write-Host "[Invoke-CIJob] Auto-detecting VM IP via guestinfo.ci-ip (polling, max 120s)..." + $ipDeadline = (Get-Date).AddSeconds(120) + $detectedIP = '' + $ipOut = '' + $ipAttempt = 0 + while ((Get-Date) -lt $ipDeadline) { + $ipAttempt++ + + # Primary: guestinfo written by ci-report-ip.service via vmware-rpctool. + # NOTE: rpctool sets 'guestinfo.ci-ip' but vmrun readVariable guestVar + # reads it WITHOUT the 'guestinfo.' prefix (the prefix is implicit in + # the guestVar namespace). Reading with the full key always returns ''. + $savedEap = $ErrorActionPreference; $ErrorActionPreference = 'Continue' + $ipOut = (& $VmrunPath -T ws readVariable $cloneVmxPath guestVar 'ci-ip' 2>&1).Trim() + $ipRc = $LASTEXITCODE + $ErrorActionPreference = $savedEap + if ($ipAttempt -le 3 -or ($ipAttempt % 10 -eq 0)) { + Write-Host "[Invoke-CIJob] [attempt $ipAttempt] guestVar ci-ip: rc=$ipRc out='$ipOut'" + } + if ($ipRc -eq 0 -and $ipOut -match '^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$') { + $detectedIP = $ipOut + Write-Host "[Invoke-CIJob] IP via guestinfo: $detectedIP" + break + } + + # Fallback: vmrun getGuestIPAddress (reliable on Windows guests) + $savedEap = $ErrorActionPreference; $ErrorActionPreference = 'Continue' + $ipOut2 = (& $VmrunPath -T ws getGuestIPAddress $cloneVmxPath 2>&1).Trim() + $ipRc2 = $LASTEXITCODE + $ErrorActionPreference = $savedEap + if ($ipRc2 -eq 0 -and $ipOut2 -match '^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$') { + $detectedIP = $ipOut2 + Write-Host "[Invoke-CIJob] IP via vmrun getGuestIPAddress: $detectedIP" + break + } + + Start-Sleep -Seconds 2 } - $VMIPAddress = $detectedIP.Trim() + if ($detectedIP -eq '') { + throw "Could not detect VM IP address after 120s. Ensure ci-report-ip.service is enabled in the guest template (Linux) or open-vm-tools is running (Windows)." + } + $VMIPAddress = $detectedIP Write-Host "[Invoke-CIJob] Detected VM IP: $VMIPAddress" } @@ -360,14 +413,32 @@ try { Remove-Item $lockPath -Force -ErrorAction SilentlyContinue Write-Host "[Invoke-CIJob] VM-start lock released." } - + # ── Resolve GuestOS from VMX if Auto ──────────────────────────────────── + if ($GuestOS -eq 'Auto' -and $cloneVmxPath) { + $vmxContent = Get-Content -LiteralPath $cloneVmxPath -Raw -ErrorAction SilentlyContinue + if ($vmxContent -match 'guestOS\s*=\s*"([^"]+)"') { + $detectedOS = $Matches[1] + $GuestOS = if ($detectedOS -like '*ubuntu*' -or $detectedOS -like '*linux*') { 'Linux' } else { 'Windows' } + Write-Host "[Invoke-CIJob] Auto-detected GuestOS: $GuestOS (VMX guestOS=$detectedOS)" + } else { + $GuestOS = 'Windows' + Write-Host "[Invoke-CIJob] Could not detect GuestOS from VMX — defaulting to Windows" + } + } # ── Phase 4: Wait for readiness ─────────────────────────────────────── Write-Host "`n[Phase 4/6] Waiting for VM readiness..." Write-JobEvent -Phase 'phase4.wait-ready' -Status 'start' -Data @{ ip = $VMIPAddress } - & "$scriptDir\Wait-VMReady.ps1" ` - -VMPath $cloneVmxPath ` - -IPAddress $VMIPAddress ` - -VmrunPath $VmrunPath + $waitParams = @{ + VMPath = $cloneVmxPath + IPAddress = $VMIPAddress + VmrunPath = $VmrunPath + } + if ($GuestOS -eq 'Linux') { + $waitParams['Transport'] = 'SSH' + $waitParams['SshKeyPath'] = $SshKeyPath + $waitParams['SshUser'] = $SshUser + } + & "$scriptDir\Wait-VMReady.ps1" @waitParams Write-JobEvent -Phase 'phase4.wait-ready' -Status 'success' # ── Phase 5: Remote build (copy source + build inside VM) ───────────── @@ -378,17 +449,35 @@ try { Credential = $credential Configuration = $Configuration } - if ($UseGitClone) { - # §3.3: Clone in VM — pass URL/branch/commit instead of HostSourceDir - $remoteBuildParams['CloneUrl'] = $RepoUrl - $remoteBuildParams['CloneBranch'] = $Branch - $remoteBuildParams['GiteaCredentialTarget'] = $GiteaCredentialTarget - if ($Commit -ne '') { $remoteBuildParams['CloneCommit'] = $Commit } - if ($Submodules) { $remoteBuildParams['CloneSubmodules'] = $true } - } - else { - # Default: Use pre-cloned source from host - $remoteBuildParams['HostSourceDir'] = $hostCloneDir + if ($GuestOS -eq 'Linux') { + $remoteBuildParams['GuestOS'] = 'Linux' + $remoteBuildParams['SshKeyPath'] = $SshKeyPath + $remoteBuildParams['SshUser'] = $SshUser + $remoteBuildParams.Remove('Credential') + if ($UseGitClone) { + # In-guest git clone (requires Gitea reachable + PAT/key configured in guest) + $remoteBuildParams['CloneUrl'] = $RepoUrl + $remoteBuildParams['CloneBranch'] = $Branch + $remoteBuildParams['GiteaCredentialTarget'] = $GiteaCredentialTarget + if ($Commit -ne '') { $remoteBuildParams['CloneCommit'] = $Commit } + if ($Submodules) { $remoteBuildParams['CloneSubmodules'] = $true } + } else { + # Default: host-cloned source shipped to guest via tar+scp + $remoteBuildParams['HostSourceDir'] = $hostCloneDir + } + } else { + if ($UseGitClone) { + # §3.3: Clone in VM — pass URL/branch/commit instead of HostSourceDir + $remoteBuildParams['CloneUrl'] = $RepoUrl + $remoteBuildParams['CloneBranch'] = $Branch + $remoteBuildParams['GiteaCredentialTarget'] = $GiteaCredentialTarget + if ($Commit -ne '') { $remoteBuildParams['CloneCommit'] = $Commit } + if ($Submodules) { $remoteBuildParams['CloneSubmodules'] = $true } + } + else { + # Default: Use pre-cloned source from host + $remoteBuildParams['HostSourceDir'] = $hostCloneDir + } } if ($BuildCommand -ne '') { $remoteBuildParams['BuildCommand'] = $BuildCommand } if ($GuestArtifactSource -ne '') { $remoteBuildParams['GuestArtifactSource'] = $GuestArtifactSource } @@ -398,12 +487,23 @@ try { # ── Phase 6: Collect artifacts ──────────────────────────────────────── Write-Host "`n[Phase 6/6] Collecting artifacts..." Write-JobEvent -Phase 'phase6.artifacts' -Status 'start' - & "$scriptDir\Get-BuildArtifacts.ps1" ` - -IPAddress $VMIPAddress ` - -Credential $credential ` - -GuestArtifactPath 'C:\CI\output\artifacts.zip' ` - -HostArtifactDir $jobArtifactDir ` - -IncludeLogs + if ($GuestOS -eq 'Linux') { + & "$scriptDir\Get-BuildArtifacts.ps1" ` + -IPAddress $VMIPAddress ` + -GuestOS 'Linux' ` + -SshKeyPath $SshKeyPath ` + -SshUser $SshUser ` + -GuestArtifactPath '/opt/ci/output' ` + -HostArtifactDir $jobArtifactDir ` + -IncludeLogs + } else { + & "$scriptDir\Get-BuildArtifacts.ps1" ` + -IPAddress $VMIPAddress ` + -Credential $credential ` + -GuestArtifactPath 'C:\CI\output\artifacts.zip' ` + -HostArtifactDir $jobArtifactDir ` + -IncludeLogs + } $elapsed = (Get-Date) - $startTime Write-JobEvent -Phase 'phase6.artifacts' -Status 'success' -Data @{ dir = $jobArtifactDir } diff --git a/scripts/Invoke-RemoteBuild.ps1 b/scripts/Invoke-RemoteBuild.ps1 index dbedb7c..b7d95a8 100644 --- a/scripts/Invoke-RemoteBuild.ps1 +++ b/scripts/Invoke-RemoteBuild.ps1 @@ -75,7 +75,7 @@ param( [ValidatePattern('^((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)\.){3}(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)$')] [string] $IPAddress, - [Parameter(Mandatory)] + [Parameter()] [System.Management.Automation.PSCredential] $Credential, # [Mode 1] Host-side clone — pre-cloned source directory on HOST @@ -124,7 +124,23 @@ param( # folders on the host (F:\CI\Cache\NuGet and F:\CI\Cache\pip). # Requires the template VMX to have been configured by Set-TemplateSharedFolders.ps1 # and VMware Tools HGFS driver present in the guest. - [switch] $UseSharedCache + [switch] $UseSharedCache, + + # Guest OS type — Linux uses SSH instead of WinRM. + [ValidateSet('Windows', 'Linux')] + [string] $GuestOS = 'Windows', + + # SSH private key path (used when GuestOS=Linux). + [string] $SshKeyPath = 'F:\CI\keys\ci_linux', + + # SSH username (used when GuestOS=Linux). + [string] $SshUser = 'ci_build', + + # Work directory inside the Linux guest. + [string] $GuestLinuxWorkDir = '/opt/ci/build', + + # Output directory inside the Linux guest. + [string] $GuestLinuxOutputDir = '/opt/ci/output' ) Set-StrictMode -Version Latest @@ -145,6 +161,105 @@ if ($guestCloneMode -and -not $CloneBranch) { throw "CloneBranch is required when using -CloneUrl mode." } +if ($GuestOS -eq 'Windows' -and $null -eq $Credential) { + throw "Credential is required when GuestOS is Windows." +} + +if ($GuestOS -eq 'Linux') { + Import-Module (Join-Path $PSScriptRoot '_Transport.psm1') -Force + + Write-Host "[Invoke-RemoteBuild] Linux build mode — SSH transport" + + # Step 1: Clean work dir (parent permissions handled by template setup) + Invoke-SshCommand -IP $IPAddress -User $SshUser -KeyPath $SshKeyPath ` + -Command "rm -rf '$GuestLinuxWorkDir' && mkdir -p '$GuestLinuxWorkDir'" + + if ($hostCloneMode) { + # ── Mode 1 (Linux): tar host source -> scp -> untar in guest ── + Write-Host "[Invoke-RemoteBuild] Packaging host source ($HostSourceDir) ..." + $tarName = "ci-src-$([Guid]::NewGuid().ToString('N').Substring(0,8)).tar.gz" + $hostTar = Join-Path $env:TEMP $tarName + $guestTar = "/tmp/$tarName" + if (Test-Path $hostTar) { Remove-Item $hostTar -Force } + # bsdtar on Windows 10/11 supports -C and gz natively. + & tar -czf $hostTar -C $HostSourceDir . + if ($LASTEXITCODE -ne 0) { + throw "[Invoke-RemoteBuild] tar failed packaging $HostSourceDir (exit $LASTEXITCODE)" + } + try { + Write-Host "[Invoke-RemoteBuild] Copying source archive to guest:$guestTar ..." + Copy-SshItem -Source $hostTar -Destination $guestTar ` + -IP $IPAddress -User $SshUser -KeyPath $SshKeyPath -Direction ToGuest + + Write-Host "[Invoke-RemoteBuild] Extracting source in guest ($GuestLinuxWorkDir) ..." + Invoke-SshCommand -IP $IPAddress -User $SshUser -KeyPath $SshKeyPath ` + -Command "tar -xzf '$guestTar' -C '$GuestLinuxWorkDir' && rm -f '$guestTar'" + } + finally { + if (Test-Path $hostTar) { Remove-Item $hostTar -Force -ErrorAction SilentlyContinue } + } + } + else { + # ── Mode 2 (Linux): in-guest git clone ── + $cloneCmd = "git clone --depth 1 --branch '$CloneBranch'" + if ($CloneSubmodules) { $cloneCmd += ' --recurse-submodules' } + + # PAT injection for private repos + $cloneTargetUrl = $CloneUrl + if ($GiteaCredentialTarget -ne '') { + try { + Import-Module CredentialManager -ErrorAction Stop + $pat = Get-StoredCredential -Target $GiteaCredentialTarget -ErrorAction Stop + if ($pat) { + $cloneTargetUrl = $CloneUrl -replace '(https?://)', ('$1' + [uri]::EscapeDataString($pat.UserName) + ':' + [uri]::EscapeDataString($pat.GetNetworkCredential().Password) + '@') + } + } catch { + Write-Warning "[Invoke-RemoteBuild] Could not load PAT from Credential Manager ($GiteaCredentialTarget) — cloning without auth (public repo?)." + } + } + + $cloneCmd += " '$CloneUrl' '$GuestLinuxWorkDir'" # non-authed URL for logging + Write-Host "[Invoke-RemoteBuild] Cloning $CloneUrl into guest $GuestLinuxWorkDir ..." + + if ($cloneTargetUrl -ne $CloneUrl) { + # PAT injected via env var inside SSH session — never in args + $envCloneCmd = "export GIT_CLONE_URL='$cloneTargetUrl'; git clone --depth 1 --branch '$CloneBranch'" + if ($CloneSubmodules) { $envCloneCmd += ' --recurse-submodules' } + $envCloneCmd += " `"`\$GIT_CLONE_URL`" '$GuestLinuxWorkDir'; unset GIT_CLONE_URL" + Invoke-SshCommand -IP $IPAddress -User $SshUser -KeyPath $SshKeyPath -Command $envCloneCmd + } else { + Invoke-SshCommand -IP $IPAddress -User $SshUser -KeyPath $SshKeyPath -Command $cloneCmd + } + + # Checkout specific commit if requested + if ($CloneCommit -ne '') { + Invoke-SshCommand -IP $IPAddress -User $SshUser -KeyPath $SshKeyPath ` + -Command "git -C '$GuestLinuxWorkDir' checkout '$CloneCommit'" + } + } + + # Step 3: Run build command (always cd into workdir first) + if ($BuildCommand -ne '') { + $buildCmd = "cd '$GuestLinuxWorkDir' && $BuildCommand" + } else { + $buildCmd = "cd '$GuestLinuxWorkDir' && make" + } + Write-Host "[Invoke-RemoteBuild] Running build: $buildCmd" + Invoke-SshCommand -IP $IPAddress -User $SshUser -KeyPath $SshKeyPath -Command $buildCmd + + # Step 4: Stage artifact tree under $GuestLinuxOutputDir for SCP collection + $outDir = $GuestLinuxOutputDir + Invoke-SshCommand -IP $IPAddress -User $SshUser -KeyPath $SshKeyPath ` + -Command "rm -rf '$outDir' && mkdir -p '$outDir'" + + $artSrc = if ($GuestArtifactSource -ne '') { $GuestArtifactSource } else { 'dist' } + Invoke-SshCommand -IP $IPAddress -User $SshUser -KeyPath $SshKeyPath ` + -Command "if [ -d '$GuestLinuxWorkDir/$artSrc' ]; then cp -r '$GuestLinuxWorkDir/$artSrc/.' '$outDir/'; elif [ -e '$GuestLinuxWorkDir/$artSrc' ]; then cp '$GuestLinuxWorkDir/$artSrc' '$outDir/'; else echo '[Invoke-RemoteBuild] WARNING: artifact source not found: $artSrc' >&2; fi" + + Write-Host "[Invoke-RemoteBuild] Linux build complete." + return +} + function Compress-BuildArtifact { param( [Parameter(Mandatory)] diff --git a/scripts/Wait-VMReady.ps1 b/scripts/Wait-VMReady.ps1 index f71be8c..84839b7 100644 --- a/scripts/Wait-VMReady.ps1 +++ b/scripts/Wait-VMReady.ps1 @@ -6,11 +6,15 @@ .DESCRIPTION Performs a three-phase readiness check in a polling loop: 1. vmrun getState returns "running" - 2. ICMP ping succeeds - 3. Test-WSMan (WinRM) responds without error + 2. Network reachability (ICMP ping for WinRM; TCP port 22 check for SSH) + 3. Transport-specific login: WinRM port 5986 (Windows) or SSH echo (Linux) Throws if the VM does not become ready within the timeout period. + Transport modes: + WinRM (default) — existing Windows VM path: ping + WinRM port 5986. + SSH — Linux VM path: port 22 check + ssh echo test. + .PARAMETER VMPath Full path to the clone VM's .vmx file. @@ -29,6 +33,19 @@ Full path to vmrun.exe. Default: C:\Program Files (x86)\VMware\VMware Workstation\vmrun.exe +.PARAMETER Transport + Transport protocol for readiness checks. + WinRM (default): checks ICMP ping + TCP 5986 (Windows VMs). + SSH: checks TCP 22 + ssh echo (Linux VMs). + +.PARAMETER SshKeyPath + Path to the SSH private key for key-based auth when Transport=SSH. + Default: F:\CI\keys\ci_linux + +.PARAMETER SshUser + SSH username when Transport=SSH. + Default: ci_build + .EXAMPLE .\Wait-VMReady.ps1 ` -VMPath "F:\CI\BuildVMs\Clone_run-42_20260508_143022\Clone_run-42_20260508_143022.vmx" ` @@ -54,7 +71,18 @@ param( # Skip ICMP ping phase (phase 2). Useful when the guest firewall blocks ICMP # but WinRM (TCP 5986) is open. Phase 1 (vmrun state) and phase 3 (WinRM) # are still checked. - [switch] $SkipPing + [switch] $SkipPing, + + # Transport protocol for phase 2/3 checks. + # WinRM (default) = Windows VM; SSH = Linux VM. + [ValidateSet('WinRM', 'SSH')] + [string] $Transport = 'WinRM', + + # SSH private key path (used when Transport=SSH). + [string] $SshKeyPath = 'F:\CI\keys\ci_linux', + + # SSH username (used when Transport=SSH). + [string] $SshUser = 'ci_build' ) Set-StrictMode -Version Latest @@ -75,10 +103,22 @@ while ((Get-Date) -lt $deadline) { $attempt++ # ── Phase 1: VM must be running ───────────────────────────────────── - # vmrun has no getState; getGuestIPAddress exits non-zero when the VM - # is not powered on. Native commands don't throw — check exit code only. - & $VmrunPath getGuestIPAddress $VMPath 2>&1 | Out-Null - $isRunning = ($LASTEXITCODE -eq 0) + # vmrun has no getState. We previously used `getGuestIPAddress`, but that + # requires VMware Tools to be up and responding inside the guest. In + # headless boots Tools can take 30-60s to come online, during which the + # call exits non-zero even though the VM is fully powered on. That caused + # Phase 1 to appear "stuck" until a GUI was opened manually. + # `vmrun list` reports actual power state without needing Tools. + $vmxNorm = (Resolve-Path -LiteralPath $VMPath -ErrorAction SilentlyContinue) + if ($null -ne $vmxNorm) { $vmxNorm = $vmxNorm.Path } else { $vmxNorm = $VMPath } + $listOut = & $VmrunPath list 2>&1 + $isRunning = $false + foreach ($line in $listOut) { + if ([string]::Equals($line.Trim(), $vmxNorm, [System.StringComparison]::OrdinalIgnoreCase)) { + $isRunning = $true + break + } + } if (-not $isRunning) { if ($phase -ne 'vmrun-state') { @@ -92,8 +132,22 @@ while ((Get-Date) -lt $deadline) { continue } - # ── Phase 2: Network ping (optional) ─────────────────────────────── - if (-not $SkipPing) { + # ── Phase 2: Network check ────────────────────────────────────────── + if ($Transport -eq 'SSH') { + $port22Open = Test-NetConnection -ComputerName $IPAddress -Port 22 ` + -InformationLevel Quiet -WarningAction SilentlyContinue ` + -ErrorAction SilentlyContinue + if (-not $port22Open) { + $phase = 'ssh-port' + if ($lastPhase -ne $phase) { + Write-Host "[Wait-VMReady] Phase 2/3: VM running, waiting for SSH port 22 at $IPAddress..." + $lastPhase = $phase + } + Start-Sleep -Seconds $PollIntervalSeconds + continue + } + } + elseif (-not $SkipPing) { $pingOk = Test-Connection -ComputerName $IPAddress -Count 1 -Quiet -ErrorAction SilentlyContinue if (-not $pingOk) { @@ -107,24 +161,59 @@ while ((Get-Date) -lt $deadline) { } } - # ── Phase 3: WinRM port 5986 accepting TCP connections ─────────────── - # Test-WSMan has no -SessionOption in PS 5.1 and fails cert validation for - # self-signed certs. TCP open on 5986 = HTTPS listener up = VM ready. - $winrmUp = Test-NetConnection -ComputerName $IPAddress -Port 5986 ` - -InformationLevel Quiet -WarningAction SilentlyContinue ` - -ErrorAction SilentlyContinue - if ($winrmUp) { - Write-Host "[Wait-VMReady] VM ready after ~$([int]($attempt * $PollIntervalSeconds))s (attempt $attempt)." - return + # ── Phase 3: Transport-specific login check ──────────────────────── + if ($Transport -eq 'SSH') { + $sshArgs = @( + '-i', $SshKeyPath, + '-o', 'StrictHostKeyChecking=no', + '-o', 'UserKnownHostsFile=NUL', + '-o', 'ConnectTimeout=5', + '-o', 'BatchMode=yes', + "$SshUser@$IPAddress", + 'echo ready' + ) + # ssh writes diagnostic warnings (e.g. "Permanently added ... to known hosts") + # to stderr. Under $ErrorActionPreference='Stop' (set at script top), + # those stderr lines captured via 2>&1 are promoted to terminating errors + # and abort the script. Wrap the call to demote stderr to non-terminating. + $savedEap = $ErrorActionPreference; $ErrorActionPreference = 'Continue' + $sshOut = & ssh @sshArgs 2>&1 + $sshExit = $LASTEXITCODE + $ErrorActionPreference = $savedEap + if ($sshExit -eq 0 -and ($sshOut -join '') -like '*ready*') { + Write-Host "[Wait-VMReady] VM ready after ~$([int]($attempt * $PollIntervalSeconds))s (attempt $attempt)." + return + } + else { + $phase = 'ssh-echo' + if ($lastPhase -ne $phase) { + Write-Host "[Wait-VMReady] Phase 3/3: SSH port open, waiting for login ($SshUser@$IPAddress)..." + $lastPhase = $phase + } + Start-Sleep -Seconds $PollIntervalSeconds + continue + } } else { - $phase = 'winrm' - if ($lastPhase -ne $phase) { - Write-Host "[Wait-VMReady] Phase 3/3: ping OK, waiting for WinRM on port 5986 (HTTPS)..." - $lastPhase = $phase + # ── WinRM port 5986 accepting TCP connections ───────────────────── + # Test-WSMan has no -SessionOption in PS 5.1 and fails cert validation for + # self-signed certs. TCP open on 5986 = HTTPS listener up = VM ready. + $winrmUp = Test-NetConnection -ComputerName $IPAddress -Port 5986 ` + -InformationLevel Quiet -WarningAction SilentlyContinue ` + -ErrorAction SilentlyContinue + if ($winrmUp) { + Write-Host "[Wait-VMReady] VM ready after ~$([int]($attempt * $PollIntervalSeconds))s (attempt $attempt)." + return + } + else { + $phase = 'winrm' + if ($lastPhase -ne $phase) { + Write-Host "[Wait-VMReady] Phase 3/3: ping OK, waiting for WinRM on port 5986 (HTTPS)..." + $lastPhase = $phase + } + Start-Sleep -Seconds $PollIntervalSeconds + continue } - Start-Sleep -Seconds $PollIntervalSeconds - continue } }