From 3da992119b62f7732376367e3e4b6bd6613f2326 Mon Sep 17 00:00:00 2001 From: Simone Date: Sun, 26 Jul 2026 02:38:40 +0200 Subject: [PATCH] fix(web): VM uptime pinned near 0s while a build was running uptime_sec came from the clone directory's own mtime, but a running VM's hypervisor can rewrite lock files under that same directory at any point, resetting the mtime and pinning "up" near 0s for the VM's entire life -- worst exactly while a build is actively running, since that's when the VM is busiest. The clone name already carries its own creation timestamp (_make_clone_name, parsed back via parse_clone_name), set once and never disturbed after -- prefer it, falling back to the old mtime read only for a name that doesn't parse. --- src/ci_orchestrator/operations/vms.py | 19 ++++++++++++++----- tests/python/test_operations_vms.py | 25 +++++++++++++++++++++++++ 2 files changed, 39 insertions(+), 5 deletions(-) diff --git a/src/ci_orchestrator/operations/vms.py b/src/ci_orchestrator/operations/vms.py index a539e67..6d7449e 100644 --- a/src/ci_orchestrator/operations/vms.py +++ b/src/ci_orchestrator/operations/vms.py @@ -204,11 +204,20 @@ def list_live_vms(backend: VmBackend, config: Config) -> list[LiveVm]: uptime_sec: int | None = None if under_base: - try: - mtime = datetime.fromtimestamp(vmx.parent.stat().st_mtime, tz=UTC) - uptime_sec = int((now - mtime).total_seconds()) - except OSError: - uptime_sec = None + created = parse_clone_name(name).started + if created is not None: + # The clone name's own timestamp, set once at creation and + # never touched again -- unlike the directory's mtime, which + # a running VM's hypervisor can reset at any point (lock + # files rewritten under it), silently pinning uptime near 0 + # for the VM's entire life. + uptime_sec = int((now - created.replace(tzinfo=UTC)).total_seconds()) + else: + try: + mtime = datetime.fromtimestamp(vmx.parent.stat().st_mtime, tz=UTC) + uptime_sec = int((now - mtime).total_seconds()) + except OSError: + uptime_sec = None is_orphan = False if under_base and uptime_sec is not None: diff --git a/tests/python/test_operations_vms.py b/tests/python/test_operations_vms.py index 145af06..39408dc 100644 --- a/tests/python/test_operations_vms.py +++ b/tests/python/test_operations_vms.py @@ -321,6 +321,31 @@ def test_list_live_vms_outside_build_vms_not_orphan(tmp_path: Path) -> None: assert vm.uptime_sec is None +def test_list_live_vms_uptime_uses_clone_name_timestamp_not_directory_mtime( + tmp_path: Path, +) -> None: + """The clone directory's own mtime can be disturbed by the hypervisor + while the VM is running (VMware rewrites lock files under it) -- the + timestamp already baked into the clone's own name at creation time can't + be, so it must win over the filesystem mtime.""" + build_vms = tmp_path / "build-vms" + build_vms.mkdir() + two_hours_ago = datetime.now(UTC) - timedelta(hours=2) + name = f"Clone_412-1_{two_hours_ago:%Y%m%d_%H%M%S}_linux" + clone_dir, vmx = _make_clone_dir(build_vms, name) + # Something touched the directory moments ago -- its mtime says "just + # now" even though the clone itself is 2h old. + _age_dir(clone_dir, hours=0) + + backend = _FakeLiveBackend([VmHandle(identifier=str(vmx), name=name)]) + config = _make_config(build_vms) + + result = list_live_vms(backend, config) # type: ignore[arg-type] + assert len(result) == 1 + vm = result[0] + assert vm.uptime_sec is not None and vm.uptime_sec >= 7000 + + def test_list_live_vms_get_ip_none(tmp_path: Path) -> None: build_vms = tmp_path / "build-vms" build_vms.mkdir()