diff --git a/src/ci_orchestrator/transport/winrm.py b/src/ci_orchestrator/transport/winrm.py index 1a77a2f..09e361a 100644 --- a/src/ci_orchestrator/transport/winrm.py +++ b/src/ci_orchestrator/transport/winrm.py @@ -9,12 +9,24 @@ for an isolated lab network (see ``scripts/_Common.psm1``). from __future__ import annotations import contextlib +import time from dataclasses import dataclass from pathlib import Path from typing import TYPE_CHECKING, Any from ci_orchestrator.transport.errors import TransportCommandError, TransportConnectError +# WSMan faults that are transient — the guest momentarily cannot launch +# the WSMan host process (wsmprovhost), typically right after a +# memory-heavy build. Retrying after a short settle usually succeeds. +_WSMAN_TRANSIENT_MARKERS = ( + "1018", # could not launch a host process to process the given request + "could not launch a host process", + "the wsman provider host server", + "1726", # RPC server unavailable (service still coming back) + "connection was aborted", +) + if TYPE_CHECKING: # pragma: no cover from pypsrp.client import Client @@ -118,12 +130,37 @@ class WinRmTransport: # --------------------------------------------------------------- ops def run(self, script: str, *, check: bool = True) -> WinRmResult: - """Run a PowerShell script block in the guest.""" - client = self._connect() - try: - stdout, stderr, returncode = client.execute_ps(script) - except Exception as exc: - raise TransportConnectError(f"WinRM execute_ps failed: {exc}") from exc + """Run a PowerShell script block in the guest. + + Retries transient WSMan host-process-launch failures (e.g. fault + 1018 right after a memory-heavy build) with a short backoff. + """ + attempts = 4 + delay = 5.0 + last_exc: Exception | None = None + for attempt in range(attempts): + client = self._connect() + try: + stdout, stderr, returncode = client.execute_ps(script) + break + except Exception as exc: + last_exc = exc + msg = str(exc).lower() + transient = any(m in msg for m in _WSMAN_TRANSIENT_MARKERS) + if transient and attempt < attempts - 1: + # Drop the cached client so the next try opens a fresh + # WSMan shell once the guest can spawn the host again. + self.close() + time.sleep(delay) + delay *= 2 + continue + raise TransportConnectError( + f"WinRM execute_ps failed: {exc}" + ) from exc + else: # pragma: no cover - loop always breaks or raises + raise TransportConnectError( + f"WinRM execute_ps failed: {last_exc}" + ) from last_exc # pypsrp returns stderr as a PSDataStreams object (newer versions) or a # list of error records (older versions); normalise to a plain string. stderr_any: Any = stderr