Simone/local-ci-cd-system
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(scripts): add Test-CIGuestWinRM.ps1 diagnostic
Lint / python (push) Successful in 21s
Details
Lint / pssa (push) Failing after 24s
Details

Browse Source 
Reproduces the orchestrator WinRM readiness probe as SYSTEM (the
act_runner account) and surfaces the real error the silent
is_ready() swallows: TCP reachability, guest hostname from the TLS
cert CN, and an auth x username-form matrix, with a classified
fix recommendation.

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
Simone
2026-05-17 12:24:35 +02:00
parent ef6f36c702
commit 21681e7419
1 changed files with 311 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/Test-CIGuestWinRM.ps1
+311
View File
@@ -0,0 +1,311 @@
#Requires -Version 5.1
#Requires -RunAsAdministrator
<#
.SYNOPSIS
Diagnoses WinRM reachability to a CI build VM from the runner's context.
.DESCRIPTION
The orchestrator's readiness probe (WinRmTransport.is_ready) swallows
connection and auth errors and returns False, so a failing guest just
hangs "waiting for windows transport readiness" until timeout with no
cause in the log.
This script reproduces the exact probe the runner performs, but
surfaces the real error:
1. TCP test to the WinRM HTTPS port (network / listener / firewall).
2. A real pypsrp `execute_ps('$true')` against the guest, run as
SYSTEM (the act_runner account) so it reads the same SYSTEM
keyring vault credential the job uses. The full Python traceback
is captured and classified.
Classifications:
CONNECT — port closed / refused / unreachable (guest WinRM
listener or firewall; check template WinRM 5986 setup).
SSL — TLS handshake/cert problem.
AUTH — credential rejected (wrong password, or username form;
try 'HOSTNAME\ci_build' instead of bare 'ci_build').
KEYRING — credential missing from the SYSTEM vault (run
Set-CIGuestCredential.ps1).
OK — WinRM works; the hang is elsewhere.
.PARAMETER IpAddress
Guest IP shown in the job log ("[job] guest IP: ...").
.PARAMETER Target
Credential target in the SYSTEM keyring. Default: BuildVMGuest
.PARAMETER Port
WinRM HTTPS port. Default: 5986
.PARAMETER VenvPython
Production venv interpreter. Default: CI_VENV_PYTHON env, else
F:\CI\python\venv\Scripts\python.exe
.PARAMETER TimeoutSec
Per-probe connection timeout (also the task wait budget). Default: 30
.EXAMPLE
.\Test-CIGuestWinRM.ps1 -IpAddress 192.168.79.237
.EXAMPLE
.\Test-CIGuestWinRM.ps1 -IpAddress 192.168.79.237 -Target BuildVMGuest -Port 5986
#>
[CmdletBinding()]
param(
[Parameter(Mandatory)]
[ValidateNotNullOrEmpty()]
[string] $IpAddress,
[ValidateNotNullOrEmpty()]
[string] $Target = 'BuildVMGuest',
[ValidateRange(1, 65535)]
[int] $Port = 5986,
# Optional username override (password still read from the SYSTEM
# keyring). Use to test forms like 'HOSTNAME\ci_build' without
# re-storing the credential. Empty = use the keyring username.
[string] $UserName = '',
# Auth packages to try, in order. Local/workgroup accounts usually
# need 'ntlm' (Negotiate->Kerberos is meaningless without a domain).
[ValidateNotNullOrEmpty()]
[string[]] $AuthOrder = @('ntlm', 'negotiate'),
[string] $VenvPython = $(if ($env:CI_VENV_PYTHON) { $env:CI_VENV_PYTHON }
else { 'F:\CI\python\venv\Scripts\python.exe' }),
[ValidateRange(5, 300)]
[int] $TimeoutSec = 30
)
Set-StrictMode -Version Latest
$ErrorActionPreference = 'Stop'
if (-not (Test-Path -LiteralPath $VenvPython)) {
throw "venv interpreter not found: '$VenvPython'. Set -VenvPython or CI_VENV_PYTHON."
}
# ── 1. TCP reachability ───────────────────────────────────────────────────
Write-Host "[1/2] TCP $IpAddress`:$Port ..." -ForegroundColor Cyan
$tcp = Test-NetConnection -ComputerName $IpAddress -Port $Port `
-WarningAction SilentlyContinue
if (-not $tcp.TcpTestSucceeded) {
Write-Host " RESULT: CONNECT — port $Port closed/unreachable." `
-ForegroundColor Red
Write-Host " WinRM HTTPS listener or guest firewall. Check the" `
-ForegroundColor Yellow
Write-Host " Windows template WinRM 5986 setup (WINDOWS-TEMPLATE-SETUP.md)." `
-ForegroundColor Yellow
return
}
Write-Host " port open." -ForegroundColor Green
# ── 2a. Discover guest hostname from the WinRM TLS certificate CN ──────────
# WinRM's self-signed cert CN is normally the guest computer name. Used to
# build the COMPUTERNAME\user form NTLM needs for a local/workgroup account.
$guestCn = ''
try {
$tcpc = New-Object Net.Sockets.TcpClient
$tcpc.Connect($IpAddress, $Port)
$ssl = New-Object Net.Security.SslStream(
$tcpc.GetStream(), $false,
([Net.Security.RemoteCertificateValidationCallback] { $true }))
$ssl.AuthenticateAsClient($IpAddress)
$cn = $ssl.RemoteCertificate.Subject # e.g. "CN=WINBUILD"
if ($cn -match 'CN=([^,]+)') { $guestCn = $Matches[1].Trim() }
$ssl.Dispose(); $tcpc.Close()
} catch { }
if ($guestCn) {
Write-Host " guest cert CN (hostname): $guestCn" -ForegroundColor Cyan
}
# ── 2b. Build the username candidate matrix ───────────────────────────────
$authList = ($AuthOrder | ForEach-Object { "'" + $_ + "'" }) -join ', '
Write-Host "[2/2] pypsrp execute_ps as SYSTEM — auth order: $($AuthOrder -join ', ')" `
-ForegroundColor Cyan
if ($UserName) {
Write-Host " username override: $UserName" -ForegroundColor Cyan
}
$probePy = Join-Path $env:TEMP ("winrmprobe_{0}.py" -f ([guid]::NewGuid().ToString('N')))
$probeOut = Join-Path $env:TEMP ("winrmprobe_{0}.txt" -f ([guid]::NewGuid().ToString('N')))
$body = @"
import keyring, traceback
out = r'''$probeOut'''
auths = [$authList]
override = r'''$UserName'''
cn = r'''$guestCn'''
c = keyring.get_credential(r'''$Target''', None)
if c is None:
open(out, 'w', encoding='utf-8').write('KEYRING')
else:
base = override if override else c.username
leaf = base.split('\\')[-1] # strip any DOMAIN\ prefix
users = [base]
if cn:
users.append(cn + '\\' + leaf) # COMPUTERNAME\user
users.append('.\\' + leaf) # .\user
if cn:
users.append(leaf + '@' + cn) # user@COMPUTERNAME
# de-dup, keep order
seen, cand = set(), []
for u in users:
if u not in seen:
seen.add(u); cand.append(u)
from pypsrp.client import Client
lines, success = [], None
lines.append('cred user=%r pwlen=%d' % (c.username, len(c.password or '')))
for u in cand:
for a in auths:
try:
cl = Client(r'''$IpAddress''', username=u, password=c.password,
port=$Port, ssl=True, cert_validation=False,
auth=a, connection_timeout=$TimeoutSec)
so, se, rc = cl.execute_ps('`$true')
lines.append('user=%r auth=%s OK rc=%s' % (u, a, rc))
success = (u, a)
break
except Exception as e:
msg = (str(e).splitlines() or [''])[0]
lines.append('user=%r auth=%s FAIL %s: %s'
% (u, a, type(e).__name__, msg))
if success:
break
if success:
hdr = 'OK user=%r auth=%s' % success
tb = ''
else:
hdr = 'EXC (all combos failed)'
tb = '\n' + traceback.format_exc()
open(out, 'w', encoding='utf-8').write(
hdr + '\n' + '\n'.join(lines) + tb)
"@
Set-Content -LiteralPath $probePy -Value $body -Encoding UTF8
try {
$action = New-ScheduledTaskAction -Execute $VenvPython -Argument "`"$probePy`""
$principal = New-ScheduledTaskPrincipal -UserId 'SYSTEM' `
-LogonType ServiceAccount -RunLevel Highest
Register-ScheduledTask -TaskName 'CI-WinRmProbe' -Action $action `
-Principal $principal -Force | Out-Null
try {
Start-ScheduledTask -TaskName 'CI-WinRmProbe'
$deadline = (Get-Date).AddSeconds($TimeoutSec + 10)
do {
Start-Sleep -Milliseconds 500
$state = (Get-ScheduledTask -TaskName 'CI-WinRmProbe').State
} while ($state -eq 'Running' -and (Get-Date) -lt $deadline)
}
finally {
Unregister-ScheduledTask -TaskName 'CI-WinRmProbe' -Confirm:$false `
-ErrorAction SilentlyContinue
}
$raw = if (Test-Path -LiteralPath $probeOut) {
(Get-Content -LiteralPath $probeOut -Raw)
} else { '' }
if (-not $raw) {
Write-Host " RESULT: probe produced no output (task may have failed)." `
-ForegroundColor Red
return
}
if ($raw -eq 'KEYRING') {
Write-Host " RESULT: KEYRING — '$Target' not in the SYSTEM vault." `
-ForegroundColor Red
Write-Host " Run: .\Set-CIGuestCredential.ps1 -UserName '<guest user>'" `
-ForegroundColor Yellow
return
}
if ($raw -like 'OK *') {
$okUser = if ($raw -match "OK user='([^']+)'") { $Matches[1] } else { '?' }
$okAuth = if ($raw -match 'auth=(\w+)') { $Matches[1] } else { '?' }
Write-Host " RESULT: OK — WinRM works." -ForegroundColor Green
Write-Host " Winning combo: user='$okUser' auth='$okAuth'" `
-ForegroundColor Green
Write-Host $raw
Write-Host ""
$okLeaf = $okUser.Split('\')[-1].Split('@')[0]
$needsQualified = ($okUser -ne $okLeaf)
if ($needsQualified) {
Write-Host " -> FIX (credential): the stored bare username is rejected;" `
-ForegroundColor Yellow
Write-Host " re-store in the working form:" -ForegroundColor Yellow
Write-Host " .\scripts\Set-CIGuestCredential.ps1 -UserName '$okUser'" `
-ForegroundColor Yellow
}
if ($okAuth -ne 'negotiate') {
Write-Host " -> FIX (code): WinRmTransport (src/ci_orchestrator/transport/winrm.py)" `
-ForegroundColor Yellow
Write-Host " builds pypsrp Client() without auth= (default 'negotiate')." `
-ForegroundColor Yellow
Write-Host " Pass auth='$okAuth' for local-account guests." `
-ForegroundColor Yellow
}
if (-not $needsQualified -and $okAuth -eq 'negotiate') {
Write-Host " Transport auth/connect is fine; the hang is elsewhere." `
-ForegroundColor Yellow
}
return
}
# EXC ... : classify the traceback.
Write-Host " RESULT: probe raised an exception:" -ForegroundColor Red
Write-Host $raw
$allFailed = $raw -like 'EXC (all combos failed)*'
$cls = switch -Regex ($raw) {
'CertificateError|SSLError|SSL:|certificate' { 'SSL'; break }
'2146893043|UNKNOWN_CREDENTIALS|401|403|AuthenticationError|the user name or password|AccessDenied|access is denied' { 'AUTH'; break }
'ConnectionError|timed out|TimeoutError|refused|No route|Failed to establish|getaddrinfo|Max retries' { 'CONNECT'; break }
default { 'UNKNOWN' }
}
Write-Host ""
switch ($cls) {
'AUTH' {
if ($allFailed) {
Write-Host " -> AUTH: EVERY user-form x auth combo was rejected with" `
-ForegroundColor Yellow
Write-Host " SEC_E_UNKNOWN_CREDENTIALS. Username form is NOT the cause." `
-ForegroundColor Yellow
Write-Host " Most likely: the password stored in the SYSTEM keyring is" `
-ForegroundColor Yellow
Write-Host " wrong / does not match the guest 'ci_build' account." `
-ForegroundColor Yellow
Write-Host " 1. Confirm the real ci_build password in the template." `
-ForegroundColor Yellow
Write-Host " 2. Re-run: .\scripts\Set-CIGuestCredential.ps1 -UserName 'ci_build'" `
-ForegroundColor Yellow
Write-Host " If the password is definitely correct, suspect the" `
-ForegroundColor Yellow
Write-Host " pyspnego SSPI provider under the SYSTEM account." `
-ForegroundColor Yellow
}
else {
Write-Host " -> AUTH: credential rejected for the tested form(s)." `
-ForegroundColor Yellow
Write-Host " Try a hostname-qualified form via Set-CIGuestCredential.ps1." `
-ForegroundColor Yellow
}
}
'CONNECT' {
Write-Host " -> CONNECT: WinRM service/listener not answering on the guest." `
-ForegroundColor Yellow
}
'SSL' {
Write-Host " -> SSL: TLS/cert problem (probe uses cert_validation=False; investigate listener cert)." `
-ForegroundColor Yellow
}
default {
Write-Host " -> UNKNOWN: inspect the traceback above." `
-ForegroundColor Yellow
}
}
}
finally {
Remove-Item -LiteralPath $probePy -Force -ErrorAction SilentlyContinue
Remove-Item -LiteralPath $probeOut -Force -ErrorAction SilentlyContinue
}