fix(ip-pool): reconcile stale leases against live clones (§2.8, W1)

A job killed mid-flight never released its IP lease; with only 4 pool addresses a single leak exhausted the pool ("IP pool exhausted after 60s"). IpSlotPool.reconcile() releases leases whose owner clone-dir is gone; vm cleanup runs it only against the real build_vms base. Closes TODO §2.8.
2026-07-23 17:15:23 +02:00
parent b0415f36e4
commit f0d856b644
8 changed files with 263 additions and 3 deletions
@@ -395,6 +395,85 @@ def test_vm_cleanup_backend_unavailable(monkeypatch: pytest.MonkeyPatch, tmp_pat
    assert "vmrun not available" in result.output


+def _config_with_pool(base: Path, lease_file: Path) -> Any:
+    from ci_orchestrator.config import BackendConfig, Config, IpPoolConfig, Paths
+
+    return Config(
+        paths=Paths(
+            root=base.parent,
+            templates=base.parent / "templates",
+            build_vms=base,
+            artifacts=base.parent / "artifacts",
+            keys=base.parent / "keys",
+        ),
+        backend=BackendConfig(),
+        ip_pool=IpPoolConfig(
+            addresses=["10.0.0.1", "10.0.0.2"],
+            netmask="255.255.255.0",
+            gateway="10.0.0.254",
+            lease_file=lease_file,
+        ),
+    )
+
+
+def test_vm_cleanup_reconciles_ip_pool_on_real_base(
+    monkeypatch: pytest.MonkeyPatch, tmp_path: Path
+) -> None:
+    """§2.8: cleaning the real clone base releases leases of vanished clones."""
+    from ci_orchestrator.ip_pool import IpSlotPool
+
+    base = tmp_path / "build-vms"
+    base.mkdir()
+    (base / "Clone_live").mkdir()  # still on disk
+
+    lease = tmp_path / "ip-pool.json"
+    pool = IpSlotPool(["10.0.0.1", "10.0.0.2"], "255.255.255.0", "10.0.0.254", lease)
+    pool.acquire("Clone_live")   # 10.0.0.1 — kept
+    pool.acquire("Clone_dead")   # 10.0.0.2 — dir gone, must be released
+
+    monkeypatch.setattr(vm_module, "_make_backend", lambda _v: None)
+    monkeypatch.setattr(vm_module, "load_config", lambda: _config_with_pool(base, lease))
+
+    result = CliRunner().invoke(cli, ["vm", "cleanup", "--clone-base-dir", str(base)])
+    assert result.exit_code == 0, result.output
+    assert "released 1 stale IP lease" in result.output
+    assert "10.0.0.2" in result.output
+
+    import json
+
+    leases = json.loads(lease.read_text())
+    assert leases["10.0.0.1"] == "Clone_live"
+    assert leases["10.0.0.2"] is None
+
+
+def test_vm_cleanup_skips_reconcile_on_custom_base(
+    monkeypatch: pytest.MonkeyPatch, tmp_path: Path
+) -> None:
+    """A custom --clone-base-dir must NOT reconcile the real pool (leases stay)."""
+    from ci_orchestrator.ip_pool import IpSlotPool
+
+    real_base = tmp_path / "build-vms"
+    real_base.mkdir()
+    custom = tmp_path / "elsewhere"
+    custom.mkdir()
+
+    lease = tmp_path / "ip-pool.json"
+    pool = IpSlotPool(["10.0.0.1"], "255.255.255.0", "10.0.0.254", lease)
+    pool.acquire("Clone_dead")  # would be released if the guard misfired
+
+    monkeypatch.setattr(vm_module, "_make_backend", lambda _v: None)
+    monkeypatch.setattr(vm_module, "load_config", lambda: _config_with_pool(real_base, lease))
+
+    result = CliRunner().invoke(cli, ["vm", "cleanup", "--clone-base-dir", str(custom)])
+    assert result.exit_code == 0, result.output
+    assert "stale IP lease" not in result.output
+
+    import json
+
+    leases = json.loads(lease.read_text())
+    assert leases["10.0.0.1"] == "Clone_dead"  # untouched
+
+
 # ── vmware ───────────────────────────────────────────────────────────────────


@@ -115,3 +115,45 @@ def test_non_dict_lease_json_treated_as_empty(tmp_path: Path) -> None:
    pool = make_pool(tmp_path)
    ip = pool.acquire("owner-1")
    assert ip == "10.0.0.1"
+
+
+# ── reconcile (TODO §2.8) ─────────────────────────────────────────────────────
+
+
+def test_reconcile_releases_leases_with_dead_owners(tmp_path: Path) -> None:
+    pool = make_pool(tmp_path)
+    pool.acquire("clone-alive")   # 10.0.0.1
+    pool.acquire("clone-dead")    # 10.0.0.2
+
+    released = pool.reconcile({"clone-alive"})
+
+    assert released == ["10.0.0.2"]
+    leases = json.loads((tmp_path / "ip-pool.json").read_text())
+    assert leases["10.0.0.1"] == "clone-alive"  # kept
+    assert leases["10.0.0.2"] is None           # released
+
+
+def test_reconcile_keeps_all_when_all_owners_live(tmp_path: Path) -> None:
+    pool = make_pool(tmp_path)
+    pool.acquire("clone-a")
+    pool.acquire("clone-b")
+    assert pool.reconcile({"clone-a", "clone-b"}) == []
+    leases = json.loads((tmp_path / "ip-pool.json").read_text())
+    assert leases["10.0.0.1"] == "clone-a"
+    assert leases["10.0.0.2"] == "clone-b"
+
+
+def test_reconcile_dry_run_reports_without_writing(tmp_path: Path) -> None:
+    pool = make_pool(tmp_path)
+    pool.acquire("clone-dead")  # 10.0.0.1
+
+    released = pool.reconcile(set(), dry_run=True)
+
+    assert released == ["10.0.0.1"]
+    leases = json.loads((tmp_path / "ip-pool.json").read_text())
+    assert leases["10.0.0.1"] == "clone-dead"  # untouched
+
+
+def test_reconcile_empty_pool_returns_empty(tmp_path: Path) -> None:
+    pool = make_pool(tmp_path)  # no acquires → all leases None
+    assert pool.reconcile(set()) == []
@@ -445,3 +445,64 @@ def test_remove_vm_dir_removal_failure(monkeypatch: pytest.MonkeyPatch, tmp_path
    monkeypatch.setattr(vms_module, "_try_remove_dir", lambda _p: False)
    result = remove_vm(backend, vmx)  # type: ignore[arg-type]
    assert result.dir_removed is False
+
+
+# ── reconcile_ip_leases (TODO §2.8) ───────────────────────────────────────────
+
+
+def _make_pool(tmp_path: Path) -> Any:
+    from ci_orchestrator.ip_pool import IpSlotPool
+
+    return IpSlotPool(
+        addresses=["10.0.0.1", "10.0.0.2"],
+        netmask="255.255.255.0",
+        gateway="10.0.0.254",
+        lease_file=tmp_path / "ip-pool.json",
+    )
+
+
+def test_reconcile_ip_leases_releases_dead_clone(tmp_path: Path) -> None:
+    from ci_orchestrator.operations.vms import reconcile_ip_leases
+
+    base = tmp_path / "build-vms"
+    base.mkdir()
+    (base / "clone-alive").mkdir()          # survives on disk
+    pool = _make_pool(tmp_path)
+    pool.acquire("clone-alive")             # 10.0.0.1
+    pool.acquire("clone-dead")              # 10.0.0.2 — no dir on disk
+
+    released = reconcile_ip_leases(base, pool)
+
+    assert released == ["10.0.0.2"]
+
+
+def test_reconcile_ip_leases_dry_run_no_write(tmp_path: Path) -> None:
+    from ci_orchestrator.operations.vms import reconcile_ip_leases
+
+    base = tmp_path / "build-vms"
+    base.mkdir()
+    pool = _make_pool(tmp_path)
+    pool.acquire("clone-dead")              # 10.0.0.1, dir absent
+
+    released = reconcile_ip_leases(base, pool, dry_run=True)
+
+    assert released == ["10.0.0.1"]
+    import json
+
+    leases = json.loads((tmp_path / "ip-pool.json").read_text())
+    assert leases["10.0.0.1"] == "clone-dead"  # not written
+
+
+def test_reconcile_ip_leases_unreadable_base_is_noop(tmp_path: Path) -> None:
+    from ci_orchestrator.operations.vms import reconcile_ip_leases
+
+    missing = tmp_path / "does-not-exist"
+    pool = _make_pool(tmp_path)
+    pool.acquire("clone-dead")
+
+    # clone_base cannot be enumerated → must NOT release anything.
+    assert reconcile_ip_leases(missing, pool) == []
+    import json
+
+    leases = json.loads((tmp_path / "ip-pool.json").read_text())
+    assert leases["10.0.0.1"] == "clone-dead"