Simone
10998247a0
All shims defaulted to F:\CI\python\venv\Scripts\python.exe when CI_VENV_PYTHON was not set. On Linux host this caused instant failure. Now uses $IsWindows (PS 6+ automatic var; absent on PS 5.1 = Windows) to select /opt/ci/venv/bin/python on Linux. Windows paths unchanged. Affects: Invoke-CIJob, Get-BuildArtifacts, Cleanup-OrphanedBuildVMs, New-BuildVM, Remove-BuildVM, Wait-VMReady, Get-CIJobSummary, Invoke-RemoteBuild, Watch-DiskSpace, Watch-RunnerHealth, Set-CIGuestCredential, Test-CIGuestWinRM. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
315 lines
13 KiB
PowerShell
315 lines
13 KiB
PowerShell
#Requires -Version 5.1
|
|
#Requires -RunAsAdministrator
|
|
<#
|
|
.SYNOPSIS
|
|
Diagnoses WinRM reachability to a CI build VM from the runner's context.
|
|
|
|
.DESCRIPTION
|
|
The orchestrator's readiness probe (WinRmTransport.is_ready) swallows
|
|
connection and auth errors and returns False, so a failing guest just
|
|
hangs "waiting for windows transport readiness" until timeout with no
|
|
cause in the log.
|
|
|
|
This script reproduces the exact probe the runner performs, but
|
|
surfaces the real error:
|
|
|
|
1. TCP test to the WinRM HTTPS port (network / listener / firewall).
|
|
2. A real pypsrp `execute_ps('$true')` against the guest, run as
|
|
SYSTEM (the act_runner account) so it reads the same SYSTEM
|
|
keyring vault credential the job uses. The full Python traceback
|
|
is captured and classified.
|
|
|
|
Classifications:
|
|
CONNECT — port closed / refused / unreachable (guest WinRM
|
|
listener or firewall; check template WinRM 5986 setup).
|
|
SSL — TLS handshake/cert problem.
|
|
AUTH — credential rejected (wrong password, or username form;
|
|
try 'HOSTNAME\ci_build' instead of bare 'ci_build').
|
|
KEYRING — credential missing from the SYSTEM vault (run
|
|
Set-CIGuestCredential.ps1).
|
|
OK — WinRM works; the hang is elsewhere.
|
|
|
|
.PARAMETER IpAddress
|
|
Guest IP shown in the job log ("[job] guest IP: ...").
|
|
|
|
.PARAMETER Target
|
|
Credential target in the SYSTEM keyring. Default: BuildVMGuest
|
|
|
|
.PARAMETER Port
|
|
WinRM HTTPS port. Default: 5986
|
|
|
|
.PARAMETER VenvPython
|
|
Production venv interpreter. Default: CI_VENV_PYTHON env, else
|
|
F:\CI\python\venv\Scripts\python.exe
|
|
|
|
.PARAMETER TimeoutSec
|
|
Per-probe connection timeout (also the task wait budget). Default: 30
|
|
|
|
.EXAMPLE
|
|
.\Test-CIGuestWinRM.ps1 -IpAddress 192.168.79.237
|
|
|
|
.EXAMPLE
|
|
.\Test-CIGuestWinRM.ps1 -IpAddress 192.168.79.237 -Target BuildVMGuest -Port 5986
|
|
#>
|
|
[CmdletBinding()]
|
|
param(
|
|
[Parameter(Mandatory)]
|
|
[ValidateNotNullOrEmpty()]
|
|
[string] $IpAddress,
|
|
|
|
[ValidateNotNullOrEmpty()]
|
|
[string] $Target = 'BuildVMGuest',
|
|
|
|
[ValidateRange(1, 65535)]
|
|
[int] $Port = 5986,
|
|
|
|
# Optional username override (password still read from the SYSTEM
|
|
# keyring). Use to test forms like 'HOSTNAME\ci_build' without
|
|
# re-storing the credential. Empty = use the keyring username.
|
|
[string] $UserName = '',
|
|
|
|
# Auth packages to try, in order. Local/workgroup accounts usually
|
|
# need 'ntlm' (Negotiate->Kerberos is meaningless without a domain).
|
|
[ValidateNotNullOrEmpty()]
|
|
[string[]] $AuthOrder = @('ntlm', 'negotiate'),
|
|
|
|
[string] $VenvPython = $(if ($env:CI_VENV_PYTHON) { $env:CI_VENV_PYTHON }
|
|
elseif ($null -ne $IsWindows -and $IsWindows -eq $false) { '/opt/ci/venv/bin/python' }
|
|
else { 'F:\CI\python\venv\Scripts\python.exe' }),
|
|
|
|
[ValidateRange(5, 300)]
|
|
[int] $TimeoutSec = 30
|
|
)
|
|
|
|
Set-StrictMode -Version Latest
|
|
$ErrorActionPreference = 'Stop'
|
|
|
|
if (-not (Test-Path -LiteralPath $VenvPython)) {
|
|
throw "venv interpreter not found: '$VenvPython'. Set -VenvPython or CI_VENV_PYTHON."
|
|
}
|
|
|
|
# ── 1. TCP reachability ───────────────────────────────────────────────────
|
|
Write-Host "[1/2] TCP $IpAddress`:$Port ..." -ForegroundColor Cyan
|
|
$tcp = Test-NetConnection -ComputerName $IpAddress -Port $Port `
|
|
-WarningAction SilentlyContinue
|
|
if (-not $tcp.TcpTestSucceeded) {
|
|
Write-Host " RESULT: CONNECT — port $Port closed/unreachable." `
|
|
-ForegroundColor Red
|
|
Write-Host " WinRM HTTPS listener or guest firewall. Check the" `
|
|
-ForegroundColor Yellow
|
|
Write-Host " Windows template WinRM 5986 setup (WINDOWS-TEMPLATE-SETUP.md)." `
|
|
-ForegroundColor Yellow
|
|
return
|
|
}
|
|
Write-Host " port open." -ForegroundColor Green
|
|
|
|
# ── 2a. Discover guest hostname from the WinRM TLS certificate CN ──────────
|
|
# WinRM's self-signed cert CN is normally the guest computer name. Used to
|
|
# build the COMPUTERNAME\user form NTLM needs for a local/workgroup account.
|
|
$guestCn = ''
|
|
try {
|
|
$tcpc = New-Object Net.Sockets.TcpClient
|
|
$tcpc.Connect($IpAddress, $Port)
|
|
$ssl = New-Object Net.Security.SslStream(
|
|
$tcpc.GetStream(), $false,
|
|
([Net.Security.RemoteCertificateValidationCallback] { $true }))
|
|
$ssl.AuthenticateAsClient($IpAddress)
|
|
$cn = $ssl.RemoteCertificate.Subject # e.g. "CN=WINBUILD"
|
|
if ($cn -match 'CN=([^,]+)') { $guestCn = $Matches[1].Trim() }
|
|
$ssl.Dispose(); $tcpc.Close()
|
|
} catch {
|
|
Write-Verbose "cert CN probe failed (hostname auto-detect skipped): $_"
|
|
}
|
|
if ($guestCn) {
|
|
Write-Host " guest cert CN (hostname): $guestCn" -ForegroundColor Cyan
|
|
}
|
|
|
|
# ── 2b. Build the username candidate matrix ───────────────────────────────
|
|
$authList = ($AuthOrder | ForEach-Object { "'" + $_ + "'" }) -join ', '
|
|
Write-Host "[2/2] pypsrp execute_ps as SYSTEM — auth order: $($AuthOrder -join ', ')" `
|
|
-ForegroundColor Cyan
|
|
if ($UserName) {
|
|
Write-Host " username override: $UserName" -ForegroundColor Cyan
|
|
}
|
|
|
|
$probePy = Join-Path $env:TEMP ("winrmprobe_{0}.py" -f ([guid]::NewGuid().ToString('N')))
|
|
$probeOut = Join-Path $env:TEMP ("winrmprobe_{0}.txt" -f ([guid]::NewGuid().ToString('N')))
|
|
|
|
$body = @"
|
|
import keyring, traceback
|
|
out = r'''$probeOut'''
|
|
auths = [$authList]
|
|
override = r'''$UserName'''
|
|
cn = r'''$guestCn'''
|
|
c = keyring.get_credential(r'''$Target''', None)
|
|
if c is None:
|
|
open(out, 'w', encoding='utf-8').write('KEYRING')
|
|
else:
|
|
base = override if override else c.username
|
|
leaf = base.split('\\')[-1] # strip any DOMAIN\ prefix
|
|
users = [base]
|
|
if cn:
|
|
users.append(cn + '\\' + leaf) # COMPUTERNAME\user
|
|
users.append('.\\' + leaf) # .\user
|
|
if cn:
|
|
users.append(leaf + '@' + cn) # user@COMPUTERNAME
|
|
# de-dup, keep order
|
|
seen, cand = set(), []
|
|
for u in users:
|
|
if u not in seen:
|
|
seen.add(u); cand.append(u)
|
|
from pypsrp.client import Client
|
|
lines, success = [], None
|
|
lines.append('cred user=%r pwlen=%d' % (c.username, len(c.password or '')))
|
|
for u in cand:
|
|
for a in auths:
|
|
try:
|
|
cl = Client(r'''$IpAddress''', username=u, password=c.password,
|
|
port=$Port, ssl=True, cert_validation=False,
|
|
auth=a, connection_timeout=$TimeoutSec)
|
|
so, se, rc = cl.execute_ps('`$true')
|
|
lines.append('user=%r auth=%s OK rc=%s' % (u, a, rc))
|
|
success = (u, a)
|
|
break
|
|
except Exception as e:
|
|
msg = (str(e).splitlines() or [''])[0]
|
|
lines.append('user=%r auth=%s FAIL %s: %s'
|
|
% (u, a, type(e).__name__, msg))
|
|
if success:
|
|
break
|
|
if success:
|
|
hdr = 'OK user=%r auth=%s' % success
|
|
tb = ''
|
|
else:
|
|
hdr = 'EXC (all combos failed)'
|
|
tb = '\n' + traceback.format_exc()
|
|
open(out, 'w', encoding='utf-8').write(
|
|
hdr + '\n' + '\n'.join(lines) + tb)
|
|
"@
|
|
Set-Content -LiteralPath $probePy -Value $body -Encoding UTF8
|
|
|
|
try {
|
|
$action = New-ScheduledTaskAction -Execute $VenvPython -Argument "`"$probePy`""
|
|
$principal = New-ScheduledTaskPrincipal -UserId 'SYSTEM' `
|
|
-LogonType ServiceAccount -RunLevel Highest
|
|
Register-ScheduledTask -TaskName 'CI-WinRmProbe' -Action $action `
|
|
-Principal $principal -Force | Out-Null
|
|
try {
|
|
Start-ScheduledTask -TaskName 'CI-WinRmProbe'
|
|
$deadline = (Get-Date).AddSeconds($TimeoutSec + 10)
|
|
do {
|
|
Start-Sleep -Milliseconds 500
|
|
$state = (Get-ScheduledTask -TaskName 'CI-WinRmProbe').State
|
|
} while ($state -eq 'Running' -and (Get-Date) -lt $deadline)
|
|
}
|
|
finally {
|
|
Unregister-ScheduledTask -TaskName 'CI-WinRmProbe' -Confirm:$false `
|
|
-ErrorAction SilentlyContinue
|
|
}
|
|
|
|
$raw = if (Test-Path -LiteralPath $probeOut) {
|
|
(Get-Content -LiteralPath $probeOut -Raw)
|
|
} else { '' }
|
|
|
|
if (-not $raw) {
|
|
Write-Host " RESULT: probe produced no output (task may have failed)." `
|
|
-ForegroundColor Red
|
|
return
|
|
}
|
|
|
|
if ($raw -eq 'KEYRING') {
|
|
Write-Host " RESULT: KEYRING — '$Target' not in the SYSTEM vault." `
|
|
-ForegroundColor Red
|
|
Write-Host " Run: .\Set-CIGuestCredential.ps1 -UserName '<guest user>'" `
|
|
-ForegroundColor Yellow
|
|
return
|
|
}
|
|
if ($raw -like 'OK *') {
|
|
$okUser = if ($raw -match "OK user='([^']+)'") { $Matches[1] } else { '?' }
|
|
$okAuth = if ($raw -match 'auth=(\w+)') { $Matches[1] } else { '?' }
|
|
Write-Host " RESULT: OK — WinRM works." -ForegroundColor Green
|
|
Write-Host " Winning combo: user='$okUser' auth='$okAuth'" `
|
|
-ForegroundColor Green
|
|
Write-Host $raw
|
|
Write-Host ""
|
|
$okLeaf = $okUser.Split('\')[-1].Split('@')[0]
|
|
$needsQualified = ($okUser -ne $okLeaf)
|
|
if ($needsQualified) {
|
|
Write-Host " -> FIX (credential): the stored bare username is rejected;" `
|
|
-ForegroundColor Yellow
|
|
Write-Host " re-store in the working form:" -ForegroundColor Yellow
|
|
Write-Host " .\scripts\Set-CIGuestCredential.ps1 -UserName '$okUser'" `
|
|
-ForegroundColor Yellow
|
|
}
|
|
if ($okAuth -ne 'negotiate') {
|
|
Write-Host " -> FIX (code): WinRmTransport (src/ci_orchestrator/transport/winrm.py)" `
|
|
-ForegroundColor Yellow
|
|
Write-Host " builds pypsrp Client() without auth= (default 'negotiate')." `
|
|
-ForegroundColor Yellow
|
|
Write-Host " Pass auth='$okAuth' for local-account guests." `
|
|
-ForegroundColor Yellow
|
|
}
|
|
if (-not $needsQualified -and $okAuth -eq 'negotiate') {
|
|
Write-Host " Transport auth/connect is fine; the hang is elsewhere." `
|
|
-ForegroundColor Yellow
|
|
}
|
|
return
|
|
}
|
|
|
|
# EXC ... : classify the traceback.
|
|
Write-Host " RESULT: probe raised an exception:" -ForegroundColor Red
|
|
Write-Host $raw
|
|
$allFailed = $raw -like 'EXC (all combos failed)*'
|
|
$cls = switch -Regex ($raw) {
|
|
'CertificateError|SSLError|SSL:|certificate' { 'SSL'; break }
|
|
'2146893043|UNKNOWN_CREDENTIALS|401|403|AuthenticationError|the user name or password|AccessDenied|access is denied' { 'AUTH'; break }
|
|
'ConnectionError|timed out|TimeoutError|refused|No route|Failed to establish|getaddrinfo|Max retries' { 'CONNECT'; break }
|
|
default { 'UNKNOWN' }
|
|
}
|
|
Write-Host ""
|
|
switch ($cls) {
|
|
'AUTH' {
|
|
if ($allFailed) {
|
|
Write-Host " -> AUTH: EVERY user-form x auth combo was rejected with" `
|
|
-ForegroundColor Yellow
|
|
Write-Host " SEC_E_UNKNOWN_CREDENTIALS. Username form is NOT the cause." `
|
|
-ForegroundColor Yellow
|
|
Write-Host " Most likely: the password stored in the SYSTEM keyring is" `
|
|
-ForegroundColor Yellow
|
|
Write-Host " wrong / does not match the guest 'ci_build' account." `
|
|
-ForegroundColor Yellow
|
|
Write-Host " 1. Confirm the real ci_build password in the template." `
|
|
-ForegroundColor Yellow
|
|
Write-Host " 2. Re-run: .\scripts\Set-CIGuestCredential.ps1 -UserName 'ci_build'" `
|
|
-ForegroundColor Yellow
|
|
Write-Host " If the password is definitely correct, suspect the" `
|
|
-ForegroundColor Yellow
|
|
Write-Host " pyspnego SSPI provider under the SYSTEM account." `
|
|
-ForegroundColor Yellow
|
|
}
|
|
else {
|
|
Write-Host " -> AUTH: credential rejected for the tested form(s)." `
|
|
-ForegroundColor Yellow
|
|
Write-Host " Try a hostname-qualified form via Set-CIGuestCredential.ps1." `
|
|
-ForegroundColor Yellow
|
|
}
|
|
}
|
|
'CONNECT' {
|
|
Write-Host " -> CONNECT: WinRM service/listener not answering on the guest." `
|
|
-ForegroundColor Yellow
|
|
}
|
|
'SSL' {
|
|
Write-Host " -> SSL: TLS/cert problem (probe uses cert_validation=False; investigate listener cert)." `
|
|
-ForegroundColor Yellow
|
|
}
|
|
default {
|
|
Write-Host " -> UNKNOWN: inspect the traceback above." `
|
|
-ForegroundColor Yellow
|
|
}
|
|
}
|
|
}
|
|
finally {
|
|
Remove-Item -LiteralPath $probePy -Force -ErrorAction SilentlyContinue
|
|
Remove-Item -LiteralPath $probeOut -Force -ErrorAction SilentlyContinue
|
|
}
|