Simone/local-ci-cd-system
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(ip-pool): pre-assign static IPs to build VMs via VMX guestinfo

Browse Source 
Eliminates VMware-Tools IP-detect polling (observed: 39–177 s variance on
Linux host B7 benchmark).  When [ip_pool] is configured the orchestrator:

  1. Allocates a slot IP from a JSON-backed pool (fcntl-locked on Linux)
  2. Injects guestinfo.ip-assignment / netmask / gateway into the cloned VMX
     before vmrun start
  3. Skips _wait_for_ip entirely — uses the pre-assigned IP immediately
  4. Releases the slot in _destroy_clone (all paths: success/failure/interrupt)

New modules / changes:
  src/ci_orchestrator/ip_pool.py      IpSlotPool + _file_lock context manager
  src/ci_orchestrator/config.py       IpPoolConfig dataclass; [ip_pool] TOML
  src/ci_orchestrator/commands/job.py _inject_guestinfo_ip helper; pool wiring
  tests/python/test_ip_pool.py        12 unit tests (acquire/release/persist)
  tests/python/test_config.py         ip_pool config parsing tests
  tests/python/test_commands_job.py   pool path + _inject_guestinfo_ip tests
  config.example.toml                 [ip_pool] section (commented example)

Coverage 90.02%; mypy --strict clean.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Simone
2026-05-24 00:26:59 +02:00
parent 203515f5d7
commit 927d6927fd
7 changed files with 524 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/ci_orchestrator/commands/job.py
+60 -8
View File
@@ -42,6 +42,7 @@ from ci_orchestrator.commands.build import _linux_build, _windows_build
from ci_orchestrator.commands.vm import _stop_with_fallback, _try_remove_dir
from ci_orchestrator.config import Config, load_config
from ci_orchestrator.credentials import KeyringCredentialStore
from ci_orchestrator.ip_pool import IpSlotPool
from ci_orchestrator.transport.errors import TransportError
from ci_orchestrator.transport.ssh import SshTransport
from ci_orchestrator.transport.winrm import WinRmTransport
@@ -188,6 +189,28 @@ def _probe_transport(
return False
def _inject_guestinfo_ip(
vmx_path: Path, ip: str, netmask: str, gateway: str
) -> None:
"""Write guestinfo.ip-assignment/netmask/gateway into the cloned VMX.
Removes any pre-existing lines for these keys before appending fresh
values, so re-runs are idempotent. Called after clone and before start.
"""
_KEYS = {"guestinfo.ip-assignment", "guestinfo.netmask", "guestinfo.gateway"}
lines = vmx_path.read_text(encoding="utf-8", errors="replace").splitlines()
lines = [
ln for ln in lines
if ln.split("=")[0].strip().lower() not in _KEYS
]
lines.append(f'guestinfo.ip-assignment = "{ip}"')
if netmask:
lines.append(f'guestinfo.netmask = "{netmask}"')
if gateway:
lines.append(f'guestinfo.gateway = "{gateway}"')
vmx_path.write_text("\n".join(lines) + "\n", encoding="utf-8")
def _make_clone_name(job_id: str) -> str:
timestamp = datetime.now(tz=UTC).strftime("%Y%m%d_%H%M%S")
# uuid4 suffix avoids collisions when several clones share a JobId.
@@ -242,6 +265,9 @@ def _destroy_clone(
backend: VmBackend | None,
handle: VmHandle | None,
clone_dir: Path | None,
*,
pool: IpSlotPool | None = None,
pool_ip: str | None = None,
) -> None:
"""Final cleanup. Tolerates every error — must never raise."""
if backend is not None and handle is not None:
@@ -265,6 +291,12 @@ def _destroy_clone(
click.echo(
f"[job] could not fully remove: {clone_dir}", err=True
)
if pool is not None and pool_ip is not None:
try:
pool.release(pool_ip)
click.echo(f"[job] IP slot released: {pool_ip}")
except Exception as exc:
click.echo(f"[job] IP slot release failed (ignored): {exc}", err=True)
# ────────────────────────────────────────────────────────── command
@@ -435,6 +467,8 @@ def job(
clone_vmx = clone_dir / f"{clone_name}.vmx"
handle: VmHandle | None = None
slot_pool: IpSlotPool | None = None
pool_ip: str | None = None
started = datetime.now(tz=UTC)
phase_timings: list[tuple[str, float]] = []
phase_start = _now()
@@ -474,6 +508,19 @@ def job(
)
_apply_vmx_overrides(clone_vmx, guest_cpu, guest_memory_mb)
if config.ip_pool is not None:
slot_pool = IpSlotPool(
addresses=config.ip_pool.addresses,
netmask=config.ip_pool.netmask,
gateway=config.ip_pool.gateway,
lease_file=config.ip_pool.lease_file,
)
pool_ip = slot_pool.acquire(clone_name)
_inject_guestinfo_ip(
clone_vmx, pool_ip, config.ip_pool.netmask, config.ip_pool.gateway
)
click.echo(f"[job] pre-assigned guest IP: {pool_ip}")
phase_timings.append(("clone", _now() - phase_start))
phase_start = _now()
@@ -494,10 +541,15 @@ def job(
deadline = _now() + ready_timeout
if not _wait_running(backend, handle, deadline, poll_interval):
raise click.ClickException("timeout waiting for VM to be running.")
ip_address = _wait_for_ip(backend, handle, deadline, poll_interval)
if not ip_address:
raise click.ClickException("timeout waiting for guest IP.")
click.echo(f"[job] guest IP: {ip_address}")
if pool_ip is not None:
ip_address: str = pool_ip
click.echo(f"[job] guest IP: {ip_address} (pre-assigned, skipping poll)")
else:
_ip = _wait_for_ip(backend, handle, deadline, poll_interval)
if not _ip:
raise click.ClickException("timeout waiting for guest IP.")
ip_address = _ip
click.echo(f"[job] guest IP: {ip_address}")
phase_timings.append(("start", _now() - phase_start))
phase_start = _now()
@@ -622,17 +674,17 @@ def job(
click.echo("=" * 60)
except KeyboardInterrupt:
click.echo("\n[job] interrupted by user; cleaning up...", err=True)
_destroy_clone(backend, handle, clone_dir)
_destroy_clone(backend, handle, clone_dir, pool=slot_pool, pool_ip=pool_ip)
sys.exit(130)
except click.ClickException:
_destroy_clone(backend, handle, clone_dir)
_destroy_clone(backend, handle, clone_dir, pool=slot_pool, pool_ip=pool_ip)
raise
except Exception as exc:
click.echo(f"\n[job] FAILURE: {exc}", err=True)
_destroy_clone(backend, handle, clone_dir)
_destroy_clone(backend, handle, clone_dir, pool=slot_pool, pool_ip=pool_ip)
raise click.ClickException(str(exc)) from exc
else:
_destroy_clone(backend, handle, clone_dir)
_destroy_clone(backend, handle, clone_dir, pool=slot_pool, pool_ip=pool_ip)
__all__ = ["job"]
src/ci_orchestrator/config.py
+41 -1
View File
@@ -22,6 +22,9 @@ from typing import Any
_ENV_KEYS = ("CI_ROOT", "CI_TEMPLATES", "CI_BUILD_VMS", "CI_ARTIFACTS", "CI_KEYS")
_IP_POOL_LEASE_DEFAULT_LINUX = "/var/lib/ci/ip-pool.json"
_IP_POOL_LEASE_DEFAULT_WIN = r"F:\CI\ip-pool.json"
@dataclass(frozen=True)
class Paths:
@@ -34,6 +37,26 @@ class Paths:
keys: Path
@dataclass(frozen=True)
class IpPoolConfig:
"""Pre-assigned IP pool for zero-polling VM ready detection.
When present, the orchestrator injects ``guestinfo.ip-assignment`` into
the cloned VMX before starting the VM so the guest can apply a static IP
at boot (see ``template/guest-setup/``).
"""
addresses: list[str]
netmask: str = "255.255.255.0"
gateway: str = ""
lease_file: Path = field(
default_factory=lambda: Path(
_IP_POOL_LEASE_DEFAULT_WIN if os.name == "nt"
else _IP_POOL_LEASE_DEFAULT_LINUX
)
)
@dataclass(frozen=True)
class BackendConfig:
"""Backend selector. ``type`` is currently always 'workstation'."""
@@ -51,6 +74,7 @@ class Config:
vmrun_path: str | None = None
ssh_key_path: Path | None = None
guest_cred_target: str = "BuildVMGuest"
ip_pool: IpPoolConfig | None = None
def _platform_defaults() -> Paths:
@@ -133,6 +157,7 @@ def load_config(
vmrun_path: str | None = env.get("CI_VMRUN_PATH")
ssh_key_path = Path(env["CI_SSH_KEY_PATH"]) if env.get("CI_SSH_KEY_PATH") else None
guest_cred_target = env.get("CI_GUEST_CRED_TARGET", "BuildVMGuest")
ip_pool: IpPoolConfig | None = None
if toml_path and toml_path.is_file():
data = _load_toml(toml_path)
@@ -148,6 +173,20 @@ def load_config(
if ssh_key_path is None and data.get("ssh_key_path"):
ssh_key_path = Path(data["ssh_key_path"])
guest_cred_target = data.get("guest_cred_target", guest_cred_target)
pool_section = data.get("ip_pool", {})
if pool_section and pool_section.get("addresses"):
addrs = pool_section["addresses"]
if isinstance(addrs, list) and addrs:
_lease_default = (
_IP_POOL_LEASE_DEFAULT_WIN if os.name == "nt"
else _IP_POOL_LEASE_DEFAULT_LINUX
)
ip_pool = IpPoolConfig(
addresses=[str(a) for a in addrs],
netmask=str(pool_section.get("netmask", "255.255.255.0")),
gateway=str(pool_section.get("gateway", "")),
lease_file=Path(pool_section.get("lease_file") or _lease_default),
)
paths = _apply_env(paths, env)
@@ -157,7 +196,8 @@ def load_config(
vmrun_path=vmrun_path,
ssh_key_path=ssh_key_path,
guest_cred_target=guest_cred_target,
ip_pool=ip_pool,
)
__all__ = ["BackendConfig", "Config", "Paths", "load_config"]
__all__ = ["BackendConfig", "Config", "IpPoolConfig", "Paths", "load_config"]
src/ci_orchestrator/ip_pool.py
+140
View File
@@ -0,0 +1,140 @@
"""IP slot pool — pre-assigns static IPs to build VMs via VMX guestinfo.
When ``[ip_pool]`` is configured, the orchestrator allocates one IP from the
pool before starting each VM, injects it into the cloned VMX as::
guestinfo.ip-assignment = "192.168.79.201"
guestinfo.netmask = "255.255.255.0"
guestinfo.gateway = "192.168.79.2"
and releases it on VM destroy. The guest applies the static IP at boot:
* **Linux**: ``/usr/local/bin/ci-report-ip.sh`` (updated version in
``template/guest-setup/linux/``) reads ``guestinfo.ip-assignment`` and
configures a static NIC address before ``network-online.target`` is reached.
* **Windows**: the ``CI-StaticIp`` Task Scheduler task
(``template/guest-setup/windows/ci-static-ip.ps1``) runs at system startup
as SYSTEM and applies the static IP before WinRM starts.
This eliminates the DHCP / VMware-Tools IP-detect polling loop (20–180 s
variance observed in B7 benchmarks). After the VM reaches *running* state the
host knows the IP immediately and goes straight to the transport probe.
Lease file — JSON dict mapping each pool IP to its current owner string, or
``null`` if free::
{
"192.168.79.201": "Clone_job123_20260524_abc123",
"192.168.79.202": null,
"192.168.79.203": null,
"192.168.79.204": null
}
File locking: ``fcntl.flock`` (POSIX/Linux); no-op on Windows (low risk —
pool size matches concurrency, failures are detectable via transport probe).
"""
from __future__ import annotations
import json
import time
from contextlib import contextmanager
from pathlib import Path
from typing import Generator
class IpSlotPool:
"""Process-safe IP slot allocator backed by a JSON lease file."""
def __init__(
self,
addresses: list[str],
netmask: str,
gateway: str,
lease_file: Path,
) -> None:
if not addresses:
raise ValueError("IpSlotPool requires at least one address.")
self._addresses = list(addresses)
self.netmask = netmask
self.gateway = gateway
self._lease_file = lease_file
self._lock_file = lease_file.with_suffix(".lock")
# ── public API ─────────────────────────────────────────────────────────
def acquire(self, owner: str, timeout: float = 60.0) -> str:
"""Allocate a free IP slot. Blocks up to *timeout* seconds.
Raises :exc:`RuntimeError` if the pool stays exhausted past *timeout*.
"""
deadline = time.monotonic() + timeout
while True:
with _file_lock(self._lock_file):
leases = self._read_leases()
for ip in self._addresses:
if leases.get(ip) is None:
leases[ip] = owner
self._write_leases(leases)
return ip
if time.monotonic() >= deadline:
raise RuntimeError(
f"IP pool exhausted after {timeout:.0f}s; "
f"addresses={self._addresses}"
)
time.sleep(2.0)
def release(self, ip: str) -> None:
"""Return *ip* to the pool (no-op if not in pool)."""
with _file_lock(self._lock_file):
leases = self._read_leases()
if ip in leases:
leases[ip] = None
self._write_leases(leases)
# ── internals ──────────────────────────────────────────────────────────
def _read_leases(self) -> dict[str, str | None]:
result: dict[str, str | None] = {ip: None for ip in self._addresses}
if not self._lease_file.exists():
return result
try:
raw: object = json.loads(self._lease_file.read_text(encoding="utf-8"))
except (json.JSONDecodeError, OSError):
return result
if not isinstance(raw, dict):
return result
for ip in self._addresses:
val = raw.get(ip)
result[ip] = val if isinstance(val, str) else None
return result
def _write_leases(self, leases: dict[str, str | None]) -> None:
self._lease_file.parent.mkdir(parents=True, exist_ok=True)
tmp = self._lease_file.with_suffix(".tmp")
tmp.write_text(json.dumps(leases, indent=2) + "\n", encoding="utf-8")
tmp.replace(self._lease_file)
@contextmanager
def _file_lock(path: Path) -> Generator[None, None, None]:
"""Exclusive advisory lock on *path* via ``fcntl.flock`` (POSIX only)."""
path.parent.mkdir(parents=True, exist_ok=True)
with open(path, "w") as fd:
try:
import fcntl
fcntl.flock(fd, fcntl.LOCK_EX)
except ImportError:
pass
try:
yield
finally:
try:
import fcntl
fcntl.flock(fd, fcntl.LOCK_UN)
except ImportError:
pass
__all__ = ["IpSlotPool"]