diff --git a/scripts/Test-CIGuestWinRM.ps1 b/scripts/Test-CIGuestWinRM.ps1 new file mode 100644 index 0000000..00e6823 --- /dev/null +++ b/scripts/Test-CIGuestWinRM.ps1 @@ -0,0 +1,311 @@ +#Requires -Version 5.1 +#Requires -RunAsAdministrator +<# +.SYNOPSIS + Diagnoses WinRM reachability to a CI build VM from the runner's context. + +.DESCRIPTION + The orchestrator's readiness probe (WinRmTransport.is_ready) swallows + connection and auth errors and returns False, so a failing guest just + hangs "waiting for windows transport readiness" until timeout with no + cause in the log. + + This script reproduces the exact probe the runner performs, but + surfaces the real error: + + 1. TCP test to the WinRM HTTPS port (network / listener / firewall). + 2. A real pypsrp `execute_ps('$true')` against the guest, run as + SYSTEM (the act_runner account) so it reads the same SYSTEM + keyring vault credential the job uses. The full Python traceback + is captured and classified. + + Classifications: + CONNECT — port closed / refused / unreachable (guest WinRM + listener or firewall; check template WinRM 5986 setup). + SSL — TLS handshake/cert problem. + AUTH — credential rejected (wrong password, or username form; + try 'HOSTNAME\ci_build' instead of bare 'ci_build'). + KEYRING — credential missing from the SYSTEM vault (run + Set-CIGuestCredential.ps1). + OK — WinRM works; the hang is elsewhere. + +.PARAMETER IpAddress + Guest IP shown in the job log ("[job] guest IP: ..."). + +.PARAMETER Target + Credential target in the SYSTEM keyring. Default: BuildVMGuest + +.PARAMETER Port + WinRM HTTPS port. Default: 5986 + +.PARAMETER VenvPython + Production venv interpreter. Default: CI_VENV_PYTHON env, else + F:\CI\python\venv\Scripts\python.exe + +.PARAMETER TimeoutSec + Per-probe connection timeout (also the task wait budget). Default: 30 + +.EXAMPLE + .\Test-CIGuestWinRM.ps1 -IpAddress 192.168.79.237 + +.EXAMPLE + .\Test-CIGuestWinRM.ps1 -IpAddress 192.168.79.237 -Target BuildVMGuest -Port 5986 +#> +[CmdletBinding()] +param( + [Parameter(Mandatory)] + [ValidateNotNullOrEmpty()] + [string] $IpAddress, + + [ValidateNotNullOrEmpty()] + [string] $Target = 'BuildVMGuest', + + [ValidateRange(1, 65535)] + [int] $Port = 5986, + + # Optional username override (password still read from the SYSTEM + # keyring). Use to test forms like 'HOSTNAME\ci_build' without + # re-storing the credential. Empty = use the keyring username. + [string] $UserName = '', + + # Auth packages to try, in order. Local/workgroup accounts usually + # need 'ntlm' (Negotiate->Kerberos is meaningless without a domain). + [ValidateNotNullOrEmpty()] + [string[]] $AuthOrder = @('ntlm', 'negotiate'), + + [string] $VenvPython = $(if ($env:CI_VENV_PYTHON) { $env:CI_VENV_PYTHON } + else { 'F:\CI\python\venv\Scripts\python.exe' }), + + [ValidateRange(5, 300)] + [int] $TimeoutSec = 30 +) + +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' + +if (-not (Test-Path -LiteralPath $VenvPython)) { + throw "venv interpreter not found: '$VenvPython'. Set -VenvPython or CI_VENV_PYTHON." +} + +# ── 1. TCP reachability ─────────────────────────────────────────────────── +Write-Host "[1/2] TCP $IpAddress`:$Port ..." -ForegroundColor Cyan +$tcp = Test-NetConnection -ComputerName $IpAddress -Port $Port ` + -WarningAction SilentlyContinue +if (-not $tcp.TcpTestSucceeded) { + Write-Host " RESULT: CONNECT — port $Port closed/unreachable." ` + -ForegroundColor Red + Write-Host " WinRM HTTPS listener or guest firewall. Check the" ` + -ForegroundColor Yellow + Write-Host " Windows template WinRM 5986 setup (WINDOWS-TEMPLATE-SETUP.md)." ` + -ForegroundColor Yellow + return +} +Write-Host " port open." -ForegroundColor Green + +# ── 2a. Discover guest hostname from the WinRM TLS certificate CN ────────── +# WinRM's self-signed cert CN is normally the guest computer name. Used to +# build the COMPUTERNAME\user form NTLM needs for a local/workgroup account. +$guestCn = '' +try { + $tcpc = New-Object Net.Sockets.TcpClient + $tcpc.Connect($IpAddress, $Port) + $ssl = New-Object Net.Security.SslStream( + $tcpc.GetStream(), $false, + ([Net.Security.RemoteCertificateValidationCallback] { $true })) + $ssl.AuthenticateAsClient($IpAddress) + $cn = $ssl.RemoteCertificate.Subject # e.g. "CN=WINBUILD" + if ($cn -match 'CN=([^,]+)') { $guestCn = $Matches[1].Trim() } + $ssl.Dispose(); $tcpc.Close() +} catch { } +if ($guestCn) { + Write-Host " guest cert CN (hostname): $guestCn" -ForegroundColor Cyan +} + +# ── 2b. Build the username candidate matrix ─────────────────────────────── +$authList = ($AuthOrder | ForEach-Object { "'" + $_ + "'" }) -join ', ' +Write-Host "[2/2] pypsrp execute_ps as SYSTEM — auth order: $($AuthOrder -join ', ')" ` + -ForegroundColor Cyan +if ($UserName) { + Write-Host " username override: $UserName" -ForegroundColor Cyan +} + +$probePy = Join-Path $env:TEMP ("winrmprobe_{0}.py" -f ([guid]::NewGuid().ToString('N'))) +$probeOut = Join-Path $env:TEMP ("winrmprobe_{0}.txt" -f ([guid]::NewGuid().ToString('N'))) + +$body = @" +import keyring, traceback +out = r'''$probeOut''' +auths = [$authList] +override = r'''$UserName''' +cn = r'''$guestCn''' +c = keyring.get_credential(r'''$Target''', None) +if c is None: + open(out, 'w', encoding='utf-8').write('KEYRING') +else: + base = override if override else c.username + leaf = base.split('\\')[-1] # strip any DOMAIN\ prefix + users = [base] + if cn: + users.append(cn + '\\' + leaf) # COMPUTERNAME\user + users.append('.\\' + leaf) # .\user + if cn: + users.append(leaf + '@' + cn) # user@COMPUTERNAME + # de-dup, keep order + seen, cand = set(), [] + for u in users: + if u not in seen: + seen.add(u); cand.append(u) + from pypsrp.client import Client + lines, success = [], None + lines.append('cred user=%r pwlen=%d' % (c.username, len(c.password or ''))) + for u in cand: + for a in auths: + try: + cl = Client(r'''$IpAddress''', username=u, password=c.password, + port=$Port, ssl=True, cert_validation=False, + auth=a, connection_timeout=$TimeoutSec) + so, se, rc = cl.execute_ps('`$true') + lines.append('user=%r auth=%s OK rc=%s' % (u, a, rc)) + success = (u, a) + break + except Exception as e: + msg = (str(e).splitlines() or [''])[0] + lines.append('user=%r auth=%s FAIL %s: %s' + % (u, a, type(e).__name__, msg)) + if success: + break + if success: + hdr = 'OK user=%r auth=%s' % success + tb = '' + else: + hdr = 'EXC (all combos failed)' + tb = '\n' + traceback.format_exc() + open(out, 'w', encoding='utf-8').write( + hdr + '\n' + '\n'.join(lines) + tb) +"@ +Set-Content -LiteralPath $probePy -Value $body -Encoding UTF8 + +try { + $action = New-ScheduledTaskAction -Execute $VenvPython -Argument "`"$probePy`"" + $principal = New-ScheduledTaskPrincipal -UserId 'SYSTEM' ` + -LogonType ServiceAccount -RunLevel Highest + Register-ScheduledTask -TaskName 'CI-WinRmProbe' -Action $action ` + -Principal $principal -Force | Out-Null + try { + Start-ScheduledTask -TaskName 'CI-WinRmProbe' + $deadline = (Get-Date).AddSeconds($TimeoutSec + 10) + do { + Start-Sleep -Milliseconds 500 + $state = (Get-ScheduledTask -TaskName 'CI-WinRmProbe').State + } while ($state -eq 'Running' -and (Get-Date) -lt $deadline) + } + finally { + Unregister-ScheduledTask -TaskName 'CI-WinRmProbe' -Confirm:$false ` + -ErrorAction SilentlyContinue + } + + $raw = if (Test-Path -LiteralPath $probeOut) { + (Get-Content -LiteralPath $probeOut -Raw) + } else { '' } + + if (-not $raw) { + Write-Host " RESULT: probe produced no output (task may have failed)." ` + -ForegroundColor Red + return + } + + if ($raw -eq 'KEYRING') { + Write-Host " RESULT: KEYRING — '$Target' not in the SYSTEM vault." ` + -ForegroundColor Red + Write-Host " Run: .\Set-CIGuestCredential.ps1 -UserName ''" ` + -ForegroundColor Yellow + return + } + if ($raw -like 'OK *') { + $okUser = if ($raw -match "OK user='([^']+)'") { $Matches[1] } else { '?' } + $okAuth = if ($raw -match 'auth=(\w+)') { $Matches[1] } else { '?' } + Write-Host " RESULT: OK — WinRM works." -ForegroundColor Green + Write-Host " Winning combo: user='$okUser' auth='$okAuth'" ` + -ForegroundColor Green + Write-Host $raw + Write-Host "" + $okLeaf = $okUser.Split('\')[-1].Split('@')[0] + $needsQualified = ($okUser -ne $okLeaf) + if ($needsQualified) { + Write-Host " -> FIX (credential): the stored bare username is rejected;" ` + -ForegroundColor Yellow + Write-Host " re-store in the working form:" -ForegroundColor Yellow + Write-Host " .\scripts\Set-CIGuestCredential.ps1 -UserName '$okUser'" ` + -ForegroundColor Yellow + } + if ($okAuth -ne 'negotiate') { + Write-Host " -> FIX (code): WinRmTransport (src/ci_orchestrator/transport/winrm.py)" ` + -ForegroundColor Yellow + Write-Host " builds pypsrp Client() without auth= (default 'negotiate')." ` + -ForegroundColor Yellow + Write-Host " Pass auth='$okAuth' for local-account guests." ` + -ForegroundColor Yellow + } + if (-not $needsQualified -and $okAuth -eq 'negotiate') { + Write-Host " Transport auth/connect is fine; the hang is elsewhere." ` + -ForegroundColor Yellow + } + return + } + + # EXC ... : classify the traceback. + Write-Host " RESULT: probe raised an exception:" -ForegroundColor Red + Write-Host $raw + $allFailed = $raw -like 'EXC (all combos failed)*' + $cls = switch -Regex ($raw) { + 'CertificateError|SSLError|SSL:|certificate' { 'SSL'; break } + '2146893043|UNKNOWN_CREDENTIALS|401|403|AuthenticationError|the user name or password|AccessDenied|access is denied' { 'AUTH'; break } + 'ConnectionError|timed out|TimeoutError|refused|No route|Failed to establish|getaddrinfo|Max retries' { 'CONNECT'; break } + default { 'UNKNOWN' } + } + Write-Host "" + switch ($cls) { + 'AUTH' { + if ($allFailed) { + Write-Host " -> AUTH: EVERY user-form x auth combo was rejected with" ` + -ForegroundColor Yellow + Write-Host " SEC_E_UNKNOWN_CREDENTIALS. Username form is NOT the cause." ` + -ForegroundColor Yellow + Write-Host " Most likely: the password stored in the SYSTEM keyring is" ` + -ForegroundColor Yellow + Write-Host " wrong / does not match the guest 'ci_build' account." ` + -ForegroundColor Yellow + Write-Host " 1. Confirm the real ci_build password in the template." ` + -ForegroundColor Yellow + Write-Host " 2. Re-run: .\scripts\Set-CIGuestCredential.ps1 -UserName 'ci_build'" ` + -ForegroundColor Yellow + Write-Host " If the password is definitely correct, suspect the" ` + -ForegroundColor Yellow + Write-Host " pyspnego SSPI provider under the SYSTEM account." ` + -ForegroundColor Yellow + } + else { + Write-Host " -> AUTH: credential rejected for the tested form(s)." ` + -ForegroundColor Yellow + Write-Host " Try a hostname-qualified form via Set-CIGuestCredential.ps1." ` + -ForegroundColor Yellow + } + } + 'CONNECT' { + Write-Host " -> CONNECT: WinRM service/listener not answering on the guest." ` + -ForegroundColor Yellow + } + 'SSL' { + Write-Host " -> SSL: TLS/cert problem (probe uses cert_validation=False; investigate listener cert)." ` + -ForegroundColor Yellow + } + default { + Write-Host " -> UNKNOWN: inspect the traceback above." ` + -ForegroundColor Yellow + } + } +} +finally { + Remove-Item -LiteralPath $probePy -Force -ErrorAction SilentlyContinue + Remove-Item -LiteralPath $probeOut -Force -ErrorAction SilentlyContinue +}