fix(ip-pool): reconcile stale leases against live clones (§2.8, W1)

A job killed mid-flight never released its IP lease; with only 4 pool addresses a single leak exhausted the pool ("IP pool exhausted after 60s"). IpSlotPool.reconcile() releases leases whose owner clone-dir is gone; vm cleanup runs it only against the real build_vms base. Closes TODO §2.8.
This commit is contained in:
2026-07-23 17:15:23 +02:00
parent b0415f36e4
commit f0d856b644
8 changed files with 263 additions and 3 deletions
+19
View File
@@ -27,6 +27,7 @@ from ci_orchestrator.backends import load_backend
from ci_orchestrator.backends.errors import BackendError, BackendNotAvailable
from ci_orchestrator.backends.protocol import VmHandle
from ci_orchestrator.config import load_config
from ci_orchestrator.ip_pool import IpSlotPool
from ci_orchestrator.operations import vms as ops_vms
if TYPE_CHECKING: # pragma: no cover
@@ -307,6 +308,24 @@ def vm_cleanup(
f"[vm cleanup] removed stale lock file (age: {lock_age_min} min): {lock_path}"
)
# Reconcile the IP pool against surviving clones (TODO §2.8). Only when
# cleaning the *real* clone base — the lease owners are clone names created
# under config.paths.build_vms, so reconciling a custom --clone-base-dir
# would wrongly release every live lease.
if config.ip_pool is not None and base_path.resolve() == config.paths.build_vms.resolve():
pool = IpSlotPool(
addresses=config.ip_pool.addresses,
netmask=config.ip_pool.netmask,
gateway=config.ip_pool.gateway,
lease_file=config.ip_pool.lease_file,
)
released = ops_vms.reconcile_ip_leases(base_path, pool, dry_run=what_if)
if released:
verb = "would release" if what_if else "released"
click.echo(
f"[vm cleanup] {verb} {len(released)} stale IP lease(s): {', '.join(released)}"
)
click.echo("[vm cleanup] done.")
+26
View File
@@ -93,6 +93,32 @@ class IpSlotPool:
leases[ip] = None
self._write_leases(leases)
def reconcile(self, live_owners: set[str], *, dry_run: bool = False) -> list[str]:
"""Release leases whose owner is not among *live_owners*.
Owners are clone names — identical to the clone directory names under
``CloneBaseDir`` (``acquire`` is called with the clone name). A lease
whose owner clone no longer exists on disk is a permanent leak: a job
killed mid-flight (SIGKILL/crash) never calls :meth:`release`, and with
only four pool addresses a single leak exhausts the pool (TODO §2.8).
Returns the IPs released — or, when *dry_run* is set, the IPs that
*would* be released (no write performed).
"""
with _file_lock(self._lock_file):
leases = self._read_leases()
stale = [
ip
for ip, owner in leases.items()
if owner is not None and owner not in live_owners
]
if not stale or dry_run:
return stale
for ip in stale:
leases[ip] = None
self._write_leases(leases)
return stale
# ── internals ──────────────────────────────────────────────────────────
def _read_leases(self) -> dict[str, str | None]:
+23
View File
@@ -21,6 +21,7 @@ from ci_orchestrator.backends.protocol import VmHandle
if TYPE_CHECKING: # pragma: no cover
from ci_orchestrator.backends.protocol import VmBackend
from ci_orchestrator.config import Config
from ci_orchestrator.ip_pool import IpSlotPool
# Matches the default ``vm cleanup --max-age-hours`` threshold; used to flag
# a live VM as an orphan candidate in ``list_live_vms``.
@@ -302,6 +303,27 @@ def cleanup_orphans(
return results
def reconcile_ip_leases(
clone_base: Path, pool: IpSlotPool, *, dry_run: bool = False
) -> list[str]:
"""Release IP-pool leases whose owner clone directory no longer exists (TODO §2.8).
``live_owners`` is the set of sub-directory names directly under
``clone_base`` (each clone dir name equals the lease owner recorded at
``acquire`` time). Any lease pointing at a clone that is gone is released,
closing the permanent-leak hole left by a job killed mid-flight.
Returns the IPs released (or, with *dry_run*, those that would be). If
``clone_base`` cannot be enumerated, returns ``[]`` without touching the
lease file — we must not release leases we cannot prove are stale.
"""
try:
live_owners = {entry.name for entry in clone_base.iterdir() if entry.is_dir()}
except OSError:
return []
return pool.reconcile(live_owners, dry_run=dry_run)
__all__ = [
"LiveVm",
"Orphan",
@@ -309,5 +331,6 @@ __all__ = [
"cleanup_orphans",
"list_live_vms",
"list_orphans",
"reconcile_ip_leases",
"remove_vm",
]