fix(winrm): retry transient WSMan host-process faults (1018)
Lint / pssa (push) Failing after 25s
Lint / python (push) Successful in 57s

After a memory-heavy parallel build the guest can momentarily fail to
launch wsmprovhost (WSManFault 1018), which broke artifact collect
(now does Compress-Archive + fetch over WinRM). run() retries up to 4x
with exponential backoff on transient WSMan markers, dropping the
cached shell so a fresh one is opened once the guest recovers.

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
Simone
2026-05-17 21:30:56 +02:00
parent 3d3b00c71b
commit 9b42823984
+43 -6
View File
@@ -9,12 +9,24 @@ for an isolated lab network (see ``scripts/_Common.psm1``).
from __future__ import annotations from __future__ import annotations
import contextlib import contextlib
import time
from dataclasses import dataclass from dataclasses import dataclass
from pathlib import Path from pathlib import Path
from typing import TYPE_CHECKING, Any from typing import TYPE_CHECKING, Any
from ci_orchestrator.transport.errors import TransportCommandError, TransportConnectError from ci_orchestrator.transport.errors import TransportCommandError, TransportConnectError
# WSMan faults that are transient — the guest momentarily cannot launch
# the WSMan host process (wsmprovhost), typically right after a
# memory-heavy build. Retrying after a short settle usually succeeds.
_WSMAN_TRANSIENT_MARKERS = (
"1018", # could not launch a host process to process the given request
"could not launch a host process",
"the wsman provider host server",
"1726", # RPC server unavailable (service still coming back)
"connection was aborted",
)
if TYPE_CHECKING: # pragma: no cover if TYPE_CHECKING: # pragma: no cover
from pypsrp.client import Client from pypsrp.client import Client
@@ -118,12 +130,37 @@ class WinRmTransport:
# --------------------------------------------------------------- ops # --------------------------------------------------------------- ops
def run(self, script: str, *, check: bool = True) -> WinRmResult: def run(self, script: str, *, check: bool = True) -> WinRmResult:
"""Run a PowerShell script block in the guest.""" """Run a PowerShell script block in the guest.
client = self._connect()
try: Retries transient WSMan host-process-launch failures (e.g. fault
stdout, stderr, returncode = client.execute_ps(script) 1018 right after a memory-heavy build) with a short backoff.
except Exception as exc: """
raise TransportConnectError(f"WinRM execute_ps failed: {exc}") from exc attempts = 4
delay = 5.0
last_exc: Exception | None = None
for attempt in range(attempts):
client = self._connect()
try:
stdout, stderr, returncode = client.execute_ps(script)
break
except Exception as exc:
last_exc = exc
msg = str(exc).lower()
transient = any(m in msg for m in _WSMAN_TRANSIENT_MARKERS)
if transient and attempt < attempts - 1:
# Drop the cached client so the next try opens a fresh
# WSMan shell once the guest can spawn the host again.
self.close()
time.sleep(delay)
delay *= 2
continue
raise TransportConnectError(
f"WinRM execute_ps failed: {exc}"
) from exc
else: # pragma: no cover - loop always breaks or raises
raise TransportConnectError(
f"WinRM execute_ps failed: {last_exc}"
) from last_exc
# pypsrp returns stderr as a PSDataStreams object (newer versions) or a # pypsrp returns stderr as a PSDataStreams object (newer versions) or a
# list of error records (older versions); normalise to a plain string. # list of error records (older versions); normalise to a plain string.
stderr_any: Any = stderr stderr_any: Any = stderr