fix(winrm): retry transient WSMan host-process faults (1018)
After a memory-heavy parallel build the guest can momentarily fail to launch wsmprovhost (WSManFault 1018), which broke artifact collect (now does Compress-Archive + fetch over WinRM). run() retries up to 4x with exponential backoff on transient WSMan markers, dropping the cached shell so a fresh one is opened once the guest recovers. Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
@@ -9,12 +9,24 @@ for an isolated lab network (see ``scripts/_Common.psm1``).
|
||||
from __future__ import annotations
|
||||
|
||||
import contextlib
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
from ci_orchestrator.transport.errors import TransportCommandError, TransportConnectError
|
||||
|
||||
# WSMan faults that are transient — the guest momentarily cannot launch
|
||||
# the WSMan host process (wsmprovhost), typically right after a
|
||||
# memory-heavy build. Retrying after a short settle usually succeeds.
|
||||
_WSMAN_TRANSIENT_MARKERS = (
|
||||
"1018", # could not launch a host process to process the given request
|
||||
"could not launch a host process",
|
||||
"the wsman provider host server",
|
||||
"1726", # RPC server unavailable (service still coming back)
|
||||
"connection was aborted",
|
||||
)
|
||||
|
||||
if TYPE_CHECKING: # pragma: no cover
|
||||
from pypsrp.client import Client
|
||||
|
||||
@@ -118,12 +130,37 @@ class WinRmTransport:
|
||||
|
||||
# --------------------------------------------------------------- ops
|
||||
def run(self, script: str, *, check: bool = True) -> WinRmResult:
|
||||
"""Run a PowerShell script block in the guest."""
|
||||
"""Run a PowerShell script block in the guest.
|
||||
|
||||
Retries transient WSMan host-process-launch failures (e.g. fault
|
||||
1018 right after a memory-heavy build) with a short backoff.
|
||||
"""
|
||||
attempts = 4
|
||||
delay = 5.0
|
||||
last_exc: Exception | None = None
|
||||
for attempt in range(attempts):
|
||||
client = self._connect()
|
||||
try:
|
||||
stdout, stderr, returncode = client.execute_ps(script)
|
||||
break
|
||||
except Exception as exc:
|
||||
raise TransportConnectError(f"WinRM execute_ps failed: {exc}") from exc
|
||||
last_exc = exc
|
||||
msg = str(exc).lower()
|
||||
transient = any(m in msg for m in _WSMAN_TRANSIENT_MARKERS)
|
||||
if transient and attempt < attempts - 1:
|
||||
# Drop the cached client so the next try opens a fresh
|
||||
# WSMan shell once the guest can spawn the host again.
|
||||
self.close()
|
||||
time.sleep(delay)
|
||||
delay *= 2
|
||||
continue
|
||||
raise TransportConnectError(
|
||||
f"WinRM execute_ps failed: {exc}"
|
||||
) from exc
|
||||
else: # pragma: no cover - loop always breaks or raises
|
||||
raise TransportConnectError(
|
||||
f"WinRM execute_ps failed: {last_exc}"
|
||||
) from last_exc
|
||||
# pypsrp returns stderr as a PSDataStreams object (newer versions) or a
|
||||
# list of error records (older versions); normalise to a plain string.
|
||||
stderr_any: Any = stderr
|
||||
|
||||
Reference in New Issue
Block a user