test_coord_heartbeat_release.py
python
sha256:609941412be74ff9d83ca3c6921d119706e988e16408184660ed94aca17ab51c
feat(coord): the hub honours a forced release ("force": tru…
Opus 5.5
2 days ago
| 1 | """Hub side of muse#202 (pushed heartbeats extend reservations) and muse#201 (release ownership). |
| 2 | |
| 3 | The muse client pushes one ``heartbeat`` record per beat, ``record_id = <reservation_id>:<last_beat_at epoch µs>``, |
| 4 | payload ``{reservation_id, run_id, last_beat_at, extended_expires_at, ...}`` with ``extended_expires_at`` also as the |
| 5 | record's ``expires_at``; a ``release`` carries the releasing run's ``run_id`` (``Release.to_dict``). |
| 6 | """ |
| 7 | |
| 8 | from __future__ import annotations |
| 9 | |
| 10 | from datetime import datetime, timedelta, timezone |
| 11 | |
| 12 | import pytest |
| 13 | from muse.core.types import fake_id |
| 14 | from sqlalchemy import select |
| 15 | from sqlalchemy.ext.asyncio import AsyncSession |
| 16 | |
| 17 | from musehub.db import coord_models as _cm |
| 18 | from musehub.models.coord import CoordPushRequest, CoordRecordIn |
| 19 | from musehub.services.musehub_coord import coord_push |
| 20 | from musehub.services.musehub_coord_server import list_reservations |
| 21 | from tests.factories import create_repo |
| 22 | |
| 23 | _EPOCH = datetime(1970, 1, 1, tzinfo=timezone.utc) |
| 24 | _NOW = datetime.now(tz=timezone.utc).replace(microsecond=0) |
| 25 | |
| 26 | |
| 27 | def _reservation(res_id: str, run_id: str = "agent-a", expires_at: datetime | None = None) -> CoordRecordIn: |
| 28 | exp = expires_at or _NOW + timedelta(seconds=300) |
| 29 | return CoordRecordIn( |
| 30 | kind="reservation", record_id=res_id, run_id=run_id, expires_at=exp, |
| 31 | payload={"reservation_id": res_id, "run_id": run_id, "addresses": ["a.py::f", "a.py::g"], "ttl_s": 300, |
| 32 | "expires_at": exp.isoformat()}, |
| 33 | ) |
| 34 | |
| 35 | |
| 36 | def _heartbeat(res_id: str, beat: datetime, extended: datetime, run_id: str = "agent-a") -> CoordRecordIn: |
| 37 | us = (beat - _EPOCH) // timedelta(microseconds=1) |
| 38 | return CoordRecordIn( |
| 39 | kind="heartbeat", record_id=f"{res_id}:{us}", run_id=run_id, expires_at=extended, |
| 40 | payload={"schema_version": "1", "reservation_id": res_id, "run_id": run_id, "agent_id": run_id, |
| 41 | "last_beat_at": beat.isoformat(), "extended_expires_at": extended.isoformat()}, |
| 42 | ) |
| 43 | |
| 44 | |
| 45 | def _release(res_id: str, run_id: str, force: object = None) -> CoordRecordIn: |
| 46 | payload = {"schema_version": "1", "reservation_id": res_id, "run_id": run_id, "released_at": _NOW.isoformat(), "reason": "completed"} |
| 47 | if force is not None: |
| 48 | payload["force"] = force |
| 49 | return CoordRecordIn(kind="release", record_id=fake_id(f"rel-{res_id}-{run_id}"), run_id=run_id, payload=payload) |
| 50 | |
| 51 | |
| 52 | async def _push(session: AsyncSession, repo_id: str, *records: CoordRecordIn) -> None: |
| 53 | await coord_push(session, repo_id, CoordPushRequest(records=list(records))) |
| 54 | |
| 55 | |
| 56 | async def _expiries(session: AsyncSession, repo_id: str, res_id: str) -> set[datetime]: |
| 57 | rows = (await session.execute( |
| 58 | select(_cm.MusehubCoordReservation).where(_cm.MusehubCoordReservation.reservation_id == res_id, |
| 59 | _cm.MusehubCoordReservation.repo_id == repo_id) |
| 60 | )).scalars().all() |
| 61 | for r in rows: |
| 62 | await session.refresh(r) |
| 63 | return {r.expires_at for r in rows} |
| 64 | |
| 65 | |
| 66 | async def _heartbeat_rows(session: AsyncSession, repo_id: str) -> list[_cm.MusehubCoordRecord]: |
| 67 | return list((await session.execute( |
| 68 | select(_cm.MusehubCoordRecord).where(_cm.MusehubCoordRecord.repo_id == repo_id, |
| 69 | _cm.MusehubCoordRecord.kind == "heartbeat") |
| 70 | )).scalars().all()) |
| 71 | |
| 72 | |
| 73 | class TestPushedHeartbeats: |
| 74 | @pytest.mark.asyncio |
| 75 | async def test_a_heartbeat_extends_every_row_of_the_reservation(self, db_session: AsyncSession) -> None: |
| 76 | repo = await create_repo(db_session, slug="hb-extend") |
| 77 | res = fake_id("hb-extend") |
| 78 | await _push(db_session, repo.repo_id, _reservation(res)) |
| 79 | later = _NOW + timedelta(seconds=3600) |
| 80 | await _push(db_session, repo.repo_id, _heartbeat(res, _NOW + timedelta(seconds=10), later)) |
| 81 | assert await _expiries(db_session, repo.repo_id, res) == {later} |
| 82 | |
| 83 | @pytest.mark.asyncio |
| 84 | async def test_a_stale_heartbeat_never_shortens_the_expiry(self, db_session: AsyncSession) -> None: |
| 85 | repo = await create_repo(db_session, slug="hb-stale") |
| 86 | res = fake_id("hb-stale") |
| 87 | await _push(db_session, repo.repo_id, _reservation(res)) |
| 88 | later = _NOW + timedelta(seconds=3600) |
| 89 | await _push(db_session, repo.repo_id, _heartbeat(res, _NOW + timedelta(seconds=20), later)) |
| 90 | await _push(db_session, repo.repo_id, _heartbeat(res, _NOW + timedelta(seconds=5), _NOW + timedelta(seconds=60))) |
| 91 | assert await _expiries(db_session, repo.repo_id, res) == {later} |
| 92 | |
| 93 | @pytest.mark.asyncio |
| 94 | async def test_a_released_reservation_is_not_extended(self, db_session: AsyncSession) -> None: |
| 95 | repo = await create_repo(db_session, slug="hb-released") |
| 96 | res = fake_id("hb-released") |
| 97 | original = _NOW + timedelta(seconds=300) |
| 98 | await _push(db_session, repo.repo_id, _reservation(res, expires_at=original)) |
| 99 | await _push(db_session, repo.repo_id, _release(res, "agent-a")) |
| 100 | await _push(db_session, repo.repo_id, _heartbeat(res, _NOW + timedelta(seconds=10), _NOW + timedelta(seconds=3600))) |
| 101 | assert await _expiries(db_session, repo.repo_id, res) == {original} |
| 102 | |
| 103 | @pytest.mark.asyncio |
| 104 | async def test_a_heartbeat_for_an_unknown_reservation_is_a_no_op(self, db_session: AsyncSession) -> None: |
| 105 | repo = await create_repo(db_session, slug="hb-unknown") |
| 106 | other = fake_id("hb-unknown-other") |
| 107 | await _push(db_session, repo.repo_id, _reservation(other)) |
| 108 | await _push(db_session, repo.repo_id, _heartbeat(fake_id("nobody"), _NOW, _NOW + timedelta(seconds=3600))) |
| 109 | assert await _expiries(db_session, repo.repo_id, other) == {_NOW + timedelta(seconds=300)} |
| 110 | assert len(await _heartbeat_rows(db_session, repo.repo_id)) == 1 # the record itself is stored |
| 111 | |
| 112 | @pytest.mark.asyncio |
| 113 | async def test_a_forced_heartbeat_from_another_run_still_extends(self, db_session: AsyncSession) -> None: |
| 114 | """No owner check: an orchestrator's ``--force`` heartbeat carries its own run id.""" |
| 115 | repo = await create_repo(db_session, slug="hb-forced") |
| 116 | res = fake_id("hb-forced") |
| 117 | await _push(db_session, repo.repo_id, _reservation(res, run_id="agent-a")) |
| 118 | later = _NOW + timedelta(seconds=900) |
| 119 | await _push(db_session, repo.repo_id, _heartbeat(res, _NOW + timedelta(seconds=1), later, run_id="orchestrator")) |
| 120 | assert await _expiries(db_session, repo.repo_id, res) == {later} |
| 121 | |
| 122 | |
| 123 | class TestHeartbeatRetention: |
| 124 | @pytest.mark.asyncio |
| 125 | async def test_only_the_latest_heartbeat_per_reservation_is_kept(self, db_session: AsyncSession) -> None: |
| 126 | repo = await create_repo(db_session, slug="hb-retain") |
| 127 | res_a, res_b = fake_id("hb-retain-a"), fake_id("hb-retain-b") |
| 128 | await _push(db_session, repo.repo_id, _reservation(res_a), _reservation(res_b)) |
| 129 | for i in (1, 2, 3): |
| 130 | await _push(db_session, repo.repo_id, _heartbeat(res_a, _NOW + timedelta(seconds=i), _NOW + timedelta(seconds=600 + i))) |
| 131 | await _push(db_session, repo.repo_id, _heartbeat(res_b, _NOW, _NOW + timedelta(seconds=600))) |
| 132 | # a stale beat pushed last does not displace the newest one |
| 133 | await _push(db_session, repo.repo_id, _heartbeat(res_a, _NOW, _NOW + timedelta(seconds=601))) |
| 134 | rows = (await db_session.execute( |
| 135 | select(_cm.MusehubCoordRecord.record_id, _cm.MusehubCoordRecord.payload["reservation_id"].astext).where( |
| 136 | _cm.MusehubCoordRecord.repo_id == repo.repo_id, _cm.MusehubCoordRecord.kind == "heartbeat") |
| 137 | )).all() |
| 138 | by_res = {rid: record_id for record_id, rid in rows} |
| 139 | assert len(rows) == 2 |
| 140 | assert by_res[res_a] == _heartbeat(res_a, _NOW + timedelta(seconds=3), _NOW).record_id |
| 141 | assert by_res[res_b] == _heartbeat(res_b, _NOW, _NOW).record_id |
| 142 | |
| 143 | |
| 144 | class TestReleaseOwnership: |
| 145 | @pytest.mark.asyncio |
| 146 | async def test_the_owners_release_applies(self, db_session: AsyncSession) -> None: |
| 147 | repo = await create_repo(db_session, slug="rel-owner") |
| 148 | res = fake_id("rel-owner") |
| 149 | await _push(db_session, repo.repo_id, _reservation(res, run_id="agent-a")) |
| 150 | await _push(db_session, repo.repo_id, _release(res, "agent-a")) |
| 151 | assert await list_reservations(db_session, repo.repo_id) == [] |
| 152 | |
| 153 | @pytest.mark.asyncio |
| 154 | async def test_another_runs_release_is_ignored_and_logged(self, db_session: AsyncSession, caplog: pytest.LogCaptureFixture) -> None: |
| 155 | repo = await create_repo(db_session, slug="rel-other") |
| 156 | res = fake_id("rel-other") |
| 157 | await _push(db_session, repo.repo_id, _reservation(res, run_id="agent-a")) |
| 158 | with caplog.at_level("WARNING", logger="musehub.services.musehub_coord_server"): |
| 159 | await _push(db_session, repo.repo_id, _release(res, "agent-b")) |
| 160 | active = await list_reservations(db_session, repo.repo_id) |
| 161 | assert {r.reservation_id for r in active} == {res} |
| 162 | assert any("release ignored" in r.getMessage() and "agent-b" in r.getMessage() for r in caplog.records) |
| 163 | # the owner can still release it afterwards |
| 164 | await _push(db_session, repo.repo_id, _release(res, "agent-a")) |
| 165 | assert await list_reservations(db_session, repo.repo_id) == [] |
| 166 | |
| 167 | @pytest.mark.asyncio |
| 168 | async def test_a_release_without_run_id_is_ignored(self, db_session: AsyncSession) -> None: |
| 169 | repo = await create_repo(db_session, slug="rel-norun") |
| 170 | res = fake_id("rel-norun") |
| 171 | await _push(db_session, repo.repo_id, _reservation(res, run_id="agent-a")) |
| 172 | rec = CoordRecordIn(kind="release", record_id=fake_id("rel-norun-x"), run_id="", payload={"reservation_id": res}) |
| 173 | await _push(db_session, repo.repo_id, rec) |
| 174 | assert len(await list_reservations(db_session, repo.repo_id)) == 2 # two address rows, still active |
| 175 | |
| 176 | |
| 177 | class TestForcedRelease: |
| 178 | @pytest.mark.asyncio |
| 179 | async def test_a_forced_release_from_an_orchestrator_applies(self, db_session: AsyncSession) -> None: |
| 180 | """`muse coord release --force` (the client enforces who may) marks the payload "force": true; the hub honours it.""" |
| 181 | repo = await create_repo(db_session, slug="rel-forced") |
| 182 | res = fake_id("rel-forced") |
| 183 | await _push(db_session, repo.repo_id, _reservation(res, run_id="agent-a")) |
| 184 | await _push(db_session, repo.repo_id, _release(res, "orchestrator-1", force=True)) |
| 185 | assert await list_reservations(db_session, repo.repo_id) == [] |
| 186 | |
| 187 | @pytest.mark.asyncio |
| 188 | async def test_a_force_flag_that_is_not_true_does_not_bypass_ownership(self, db_session: AsyncSession) -> None: |
| 189 | repo = await create_repo(db_session, slug="rel-notforced") |
| 190 | res = fake_id("rel-notforced") |
| 191 | await _push(db_session, repo.repo_id, _reservation(res, run_id="agent-a")) |
| 192 | await _push(db_session, repo.repo_id, _release(res, "agent-b", force="yes")) |
| 193 | assert {r.reservation_id for r in await list_reservations(db_session, repo.repo_id)} == {res} |
File History
3 commits
sha256:609941412be74ff9d83ca3c6921d119706e988e16408184660ed94aca17ab51c
feat(coord): the hub honours a forced release ("force": tru…
Opus 5.5
2 days ago
sha256:41ed2ded57d17544c111c81960ee6a7a7febc9a0d81a778616319265ac93a654
merge: rc25 — Mists wave (#316 #317 #318 #322; #319 #320 #3…
Opus 5.5
minor
⚠
2 days ago
sha256:fbcefb2702e89b64aeab652f89821a80bd1f548a003d1ca0637698ab10eeb1b8
fix(coord): pushed heartbeats extend hub reservations, keep…
Opus 5.5
patch
2 days ago