feat(template): heartbeat thread for long-running toolchain install
execute_ps is fully blocking with no streaming. Add _run_with_heartbeat() which runs the WinRM call in a daemon thread and prints " ... still running (Ns elapsed) ..." every 30 s so the operator can distinguish a hung session from a slow Windows Update pass. Used only for the toolchain script invocation (the slow step); all other transport.run() calls keep the direct path. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -60,6 +60,7 @@ import json
|
|||||||
import shutil
|
import shutil
|
||||||
import socket
|
import socket
|
||||||
import subprocess
|
import subprocess
|
||||||
|
import threading
|
||||||
import time
|
import time
|
||||||
from datetime import UTC, datetime
|
from datetime import UTC, datetime
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
@@ -70,7 +71,7 @@ from ci_orchestrator.backends.workstation import WorkstationVmrunBackend
|
|||||||
from ci_orchestrator.backends.protocol import VmHandle
|
from ci_orchestrator.backends.protocol import VmHandle
|
||||||
from ci_orchestrator.config import load_config
|
from ci_orchestrator.config import load_config
|
||||||
from ci_orchestrator.transport.errors import TransportError
|
from ci_orchestrator.transport.errors import TransportError
|
||||||
from ci_orchestrator.transport.winrm import WinRmTransport
|
from ci_orchestrator.transport.winrm import WinRmResult, WinRmTransport
|
||||||
|
|
||||||
|
|
||||||
@click.group()
|
@click.group()
|
||||||
@@ -302,6 +303,43 @@ def _wait_winrm(transport: WinRmTransport, timeout: int) -> None:
|
|||||||
raise click.ClickException(f"WinRM not ready within {timeout}s.")
|
raise click.ClickException(f"WinRM not ready within {timeout}s.")
|
||||||
|
|
||||||
|
|
||||||
|
def _run_with_heartbeat(
|
||||||
|
transport: WinRmTransport,
|
||||||
|
script: str,
|
||||||
|
*,
|
||||||
|
check: bool = True,
|
||||||
|
heartbeat: int = 30,
|
||||||
|
) -> WinRmResult:
|
||||||
|
"""Run *script* via *transport* and print a heartbeat line every *heartbeat* s.
|
||||||
|
|
||||||
|
pypsrp's execute_ps is fully blocking — no streaming. The heartbeat
|
||||||
|
thread reassures the operator that the command is still running during
|
||||||
|
long-running steps (e.g. Windows Update, which can take 20-40 min).
|
||||||
|
"""
|
||||||
|
result_holder: list[WinRmResult] = []
|
||||||
|
exc_holder: list[BaseException] = []
|
||||||
|
stop_event = threading.Event()
|
||||||
|
|
||||||
|
def _worker() -> None:
|
||||||
|
try:
|
||||||
|
result_holder.append(transport.run(script, check=check))
|
||||||
|
except BaseException as exc: # noqa: BLE001
|
||||||
|
exc_holder.append(exc)
|
||||||
|
finally:
|
||||||
|
stop_event.set()
|
||||||
|
|
||||||
|
worker = threading.Thread(target=_worker, daemon=True)
|
||||||
|
worker.start()
|
||||||
|
elapsed = 0
|
||||||
|
while not stop_event.wait(timeout=float(heartbeat)):
|
||||||
|
elapsed += heartbeat
|
||||||
|
click.echo(f" ... still running ({elapsed}s elapsed) ...")
|
||||||
|
worker.join()
|
||||||
|
if exc_holder:
|
||||||
|
raise exc_holder[0]
|
||||||
|
return result_holder[0]
|
||||||
|
|
||||||
|
|
||||||
# ═══════════════════════════════════════════════════════════════════════════
|
# ═══════════════════════════════════════════════════════════════════════════
|
||||||
# template prepare-win command
|
# template prepare-win command
|
||||||
# ═══════════════════════════════════════════════════════════════════════════
|
# ═══════════════════════════════════════════════════════════════════════════
|
||||||
@@ -397,7 +435,7 @@ def template_prepare_win(
|
|||||||
|
|
||||||
if not backend.is_running(handle):
|
if not backend.is_running(handle):
|
||||||
click.echo(f"[prepare-win] Starting VM: {vmx_path.name} ...")
|
click.echo(f"[prepare-win] Starting VM: {vmx_path.name} ...")
|
||||||
backend.start(handle, headless=False)
|
backend.start(handle, headless=True)
|
||||||
else:
|
else:
|
||||||
click.echo(f"[prepare-win] VM already running: {vmx_path.name}")
|
click.echo(f"[prepare-win] VM already running: {vmx_path.name}")
|
||||||
|
|
||||||
@@ -479,7 +517,7 @@ def template_prepare_win(
|
|||||||
'"CI_EXITCODE:$ec"\n'
|
'"CI_EXITCODE:$ec"\n'
|
||||||
)
|
)
|
||||||
try:
|
try:
|
||||||
result = transport.run(invoke, check=False)
|
result = _run_with_heartbeat(transport, invoke, check=False)
|
||||||
exit_code = _parse_exit_marker(result.stdout)
|
exit_code = _parse_exit_marker(result.stdout)
|
||||||
if result.stdout.strip():
|
if result.stdout.strip():
|
||||||
click.echo(result.stdout.strip())
|
click.echo(result.stdout.strip())
|
||||||
|
|||||||
Reference in New Issue
Block a user