fix(ip-pool): reconcile stale leases against live clones (§2.8, W1)
A job killed mid-flight never released its IP lease; with only 4 pool addresses a single leak exhausted the pool ("IP pool exhausted after 60s"). IpSlotPool.reconcile() releases leases whose owner clone-dir is gone; vm cleanup runs it only against the real build_vms base. Closes TODO §2.8.
This commit is contained in:
@@ -27,6 +27,7 @@ from ci_orchestrator.backends import load_backend
|
||||
from ci_orchestrator.backends.errors import BackendError, BackendNotAvailable
|
||||
from ci_orchestrator.backends.protocol import VmHandle
|
||||
from ci_orchestrator.config import load_config
|
||||
from ci_orchestrator.ip_pool import IpSlotPool
|
||||
from ci_orchestrator.operations import vms as ops_vms
|
||||
|
||||
if TYPE_CHECKING: # pragma: no cover
|
||||
@@ -307,6 +308,24 @@ def vm_cleanup(
|
||||
f"[vm cleanup] removed stale lock file (age: {lock_age_min} min): {lock_path}"
|
||||
)
|
||||
|
||||
# Reconcile the IP pool against surviving clones (TODO §2.8). Only when
|
||||
# cleaning the *real* clone base — the lease owners are clone names created
|
||||
# under config.paths.build_vms, so reconciling a custom --clone-base-dir
|
||||
# would wrongly release every live lease.
|
||||
if config.ip_pool is not None and base_path.resolve() == config.paths.build_vms.resolve():
|
||||
pool = IpSlotPool(
|
||||
addresses=config.ip_pool.addresses,
|
||||
netmask=config.ip_pool.netmask,
|
||||
gateway=config.ip_pool.gateway,
|
||||
lease_file=config.ip_pool.lease_file,
|
||||
)
|
||||
released = ops_vms.reconcile_ip_leases(base_path, pool, dry_run=what_if)
|
||||
if released:
|
||||
verb = "would release" if what_if else "released"
|
||||
click.echo(
|
||||
f"[vm cleanup] {verb} {len(released)} stale IP lease(s): {', '.join(released)}"
|
||||
)
|
||||
|
||||
click.echo("[vm cleanup] done.")
|
||||
|
||||
|
||||
|
||||
@@ -93,6 +93,32 @@ class IpSlotPool:
|
||||
leases[ip] = None
|
||||
self._write_leases(leases)
|
||||
|
||||
def reconcile(self, live_owners: set[str], *, dry_run: bool = False) -> list[str]:
|
||||
"""Release leases whose owner is not among *live_owners*.
|
||||
|
||||
Owners are clone names — identical to the clone directory names under
|
||||
``CloneBaseDir`` (``acquire`` is called with the clone name). A lease
|
||||
whose owner clone no longer exists on disk is a permanent leak: a job
|
||||
killed mid-flight (SIGKILL/crash) never calls :meth:`release`, and with
|
||||
only four pool addresses a single leak exhausts the pool (TODO §2.8).
|
||||
|
||||
Returns the IPs released — or, when *dry_run* is set, the IPs that
|
||||
*would* be released (no write performed).
|
||||
"""
|
||||
with _file_lock(self._lock_file):
|
||||
leases = self._read_leases()
|
||||
stale = [
|
||||
ip
|
||||
for ip, owner in leases.items()
|
||||
if owner is not None and owner not in live_owners
|
||||
]
|
||||
if not stale or dry_run:
|
||||
return stale
|
||||
for ip in stale:
|
||||
leases[ip] = None
|
||||
self._write_leases(leases)
|
||||
return stale
|
||||
|
||||
# ── internals ──────────────────────────────────────────────────────────
|
||||
|
||||
def _read_leases(self) -> dict[str, str | None]:
|
||||
|
||||
@@ -21,6 +21,7 @@ from ci_orchestrator.backends.protocol import VmHandle
|
||||
if TYPE_CHECKING: # pragma: no cover
|
||||
from ci_orchestrator.backends.protocol import VmBackend
|
||||
from ci_orchestrator.config import Config
|
||||
from ci_orchestrator.ip_pool import IpSlotPool
|
||||
|
||||
# Matches the default ``vm cleanup --max-age-hours`` threshold; used to flag
|
||||
# a live VM as an orphan candidate in ``list_live_vms``.
|
||||
@@ -302,6 +303,27 @@ def cleanup_orphans(
|
||||
return results
|
||||
|
||||
|
||||
def reconcile_ip_leases(
|
||||
clone_base: Path, pool: IpSlotPool, *, dry_run: bool = False
|
||||
) -> list[str]:
|
||||
"""Release IP-pool leases whose owner clone directory no longer exists (TODO §2.8).
|
||||
|
||||
``live_owners`` is the set of sub-directory names directly under
|
||||
``clone_base`` (each clone dir name equals the lease owner recorded at
|
||||
``acquire`` time). Any lease pointing at a clone that is gone is released,
|
||||
closing the permanent-leak hole left by a job killed mid-flight.
|
||||
|
||||
Returns the IPs released (or, with *dry_run*, those that would be). If
|
||||
``clone_base`` cannot be enumerated, returns ``[]`` without touching the
|
||||
lease file — we must not release leases we cannot prove are stale.
|
||||
"""
|
||||
try:
|
||||
live_owners = {entry.name for entry in clone_base.iterdir() if entry.is_dir()}
|
||||
except OSError:
|
||||
return []
|
||||
return pool.reconcile(live_owners, dry_run=dry_run)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"LiveVm",
|
||||
"Orphan",
|
||||
@@ -309,5 +331,6 @@ __all__ = [
|
||||
"cleanup_orphans",
|
||||
"list_live_vms",
|
||||
"list_orphans",
|
||||
"reconcile_ip_leases",
|
||||
"remove_vm",
|
||||
]
|
||||
|
||||
Reference in New Issue
Block a user