fix(winrm): retry transient WSMan host-process faults (1018)
After a memory-heavy parallel build the guest can momentarily fail to launch wsmprovhost (WSManFault 1018), which broke artifact collect (now does Compress-Archive + fetch over WinRM). run() retries up to 4x with exponential backoff on transient WSMan markers, dropping the cached shell so a fresh one is opened once the guest recovers. Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
@@ -9,12 +9,24 @@ for an isolated lab network (see ``scripts/_Common.psm1``).
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import contextlib
|
import contextlib
|
||||||
|
import time
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import TYPE_CHECKING, Any
|
from typing import TYPE_CHECKING, Any
|
||||||
|
|
||||||
from ci_orchestrator.transport.errors import TransportCommandError, TransportConnectError
|
from ci_orchestrator.transport.errors import TransportCommandError, TransportConnectError
|
||||||
|
|
||||||
|
# WSMan faults that are transient — the guest momentarily cannot launch
|
||||||
|
# the WSMan host process (wsmprovhost), typically right after a
|
||||||
|
# memory-heavy build. Retrying after a short settle usually succeeds.
|
||||||
|
_WSMAN_TRANSIENT_MARKERS = (
|
||||||
|
"1018", # could not launch a host process to process the given request
|
||||||
|
"could not launch a host process",
|
||||||
|
"the wsman provider host server",
|
||||||
|
"1726", # RPC server unavailable (service still coming back)
|
||||||
|
"connection was aborted",
|
||||||
|
)
|
||||||
|
|
||||||
if TYPE_CHECKING: # pragma: no cover
|
if TYPE_CHECKING: # pragma: no cover
|
||||||
from pypsrp.client import Client
|
from pypsrp.client import Client
|
||||||
|
|
||||||
@@ -118,12 +130,37 @@ class WinRmTransport:
|
|||||||
|
|
||||||
# --------------------------------------------------------------- ops
|
# --------------------------------------------------------------- ops
|
||||||
def run(self, script: str, *, check: bool = True) -> WinRmResult:
|
def run(self, script: str, *, check: bool = True) -> WinRmResult:
|
||||||
"""Run a PowerShell script block in the guest."""
|
"""Run a PowerShell script block in the guest.
|
||||||
client = self._connect()
|
|
||||||
try:
|
Retries transient WSMan host-process-launch failures (e.g. fault
|
||||||
stdout, stderr, returncode = client.execute_ps(script)
|
1018 right after a memory-heavy build) with a short backoff.
|
||||||
except Exception as exc:
|
"""
|
||||||
raise TransportConnectError(f"WinRM execute_ps failed: {exc}") from exc
|
attempts = 4
|
||||||
|
delay = 5.0
|
||||||
|
last_exc: Exception | None = None
|
||||||
|
for attempt in range(attempts):
|
||||||
|
client = self._connect()
|
||||||
|
try:
|
||||||
|
stdout, stderr, returncode = client.execute_ps(script)
|
||||||
|
break
|
||||||
|
except Exception as exc:
|
||||||
|
last_exc = exc
|
||||||
|
msg = str(exc).lower()
|
||||||
|
transient = any(m in msg for m in _WSMAN_TRANSIENT_MARKERS)
|
||||||
|
if transient and attempt < attempts - 1:
|
||||||
|
# Drop the cached client so the next try opens a fresh
|
||||||
|
# WSMan shell once the guest can spawn the host again.
|
||||||
|
self.close()
|
||||||
|
time.sleep(delay)
|
||||||
|
delay *= 2
|
||||||
|
continue
|
||||||
|
raise TransportConnectError(
|
||||||
|
f"WinRM execute_ps failed: {exc}"
|
||||||
|
) from exc
|
||||||
|
else: # pragma: no cover - loop always breaks or raises
|
||||||
|
raise TransportConnectError(
|
||||||
|
f"WinRM execute_ps failed: {last_exc}"
|
||||||
|
) from last_exc
|
||||||
# pypsrp returns stderr as a PSDataStreams object (newer versions) or a
|
# pypsrp returns stderr as a PSDataStreams object (newer versions) or a
|
||||||
# list of error records (older versions); normalise to a plain string.
|
# list of error records (older versions); normalise to a plain string.
|
||||||
stderr_any: Any = stderr
|
stderr_any: Any = stderr
|
||||||
|
|||||||
Reference in New Issue
Block a user