gabriel / musehub public
test_coord_heartbeat_release.py python
193 lines 10.6 KB
Raw
sha256:609941412be74ff9d83ca3c6921d119706e988e16408184660ed94aca17ab51c feat(coord): the hub honours a forced release ("force": tru… Opus 5.5 2 days ago
1 """Hub side of muse#202 (pushed heartbeats extend reservations) and muse#201 (release ownership).
2
3 The muse client pushes one ``heartbeat`` record per beat, ``record_id = <reservation_id>:<last_beat_at epoch µs>``,
4 payload ``{reservation_id, run_id, last_beat_at, extended_expires_at, ...}`` with ``extended_expires_at`` also as the
5 record's ``expires_at``; a ``release`` carries the releasing run's ``run_id`` (``Release.to_dict``).
6 """
7
8 from __future__ import annotations
9
10 from datetime import datetime, timedelta, timezone
11
12 import pytest
13 from muse.core.types import fake_id
14 from sqlalchemy import select
15 from sqlalchemy.ext.asyncio import AsyncSession
16
17 from musehub.db import coord_models as _cm
18 from musehub.models.coord import CoordPushRequest, CoordRecordIn
19 from musehub.services.musehub_coord import coord_push
20 from musehub.services.musehub_coord_server import list_reservations
21 from tests.factories import create_repo
22
23 _EPOCH = datetime(1970, 1, 1, tzinfo=timezone.utc)
24 _NOW = datetime.now(tz=timezone.utc).replace(microsecond=0)
25
26
27 def _reservation(res_id: str, run_id: str = "agent-a", expires_at: datetime | None = None) -> CoordRecordIn:
28 exp = expires_at or _NOW + timedelta(seconds=300)
29 return CoordRecordIn(
30 kind="reservation", record_id=res_id, run_id=run_id, expires_at=exp,
31 payload={"reservation_id": res_id, "run_id": run_id, "addresses": ["a.py::f", "a.py::g"], "ttl_s": 300,
32 "expires_at": exp.isoformat()},
33 )
34
35
36 def _heartbeat(res_id: str, beat: datetime, extended: datetime, run_id: str = "agent-a") -> CoordRecordIn:
37 us = (beat - _EPOCH) // timedelta(microseconds=1)
38 return CoordRecordIn(
39 kind="heartbeat", record_id=f"{res_id}:{us}", run_id=run_id, expires_at=extended,
40 payload={"schema_version": "1", "reservation_id": res_id, "run_id": run_id, "agent_id": run_id,
41 "last_beat_at": beat.isoformat(), "extended_expires_at": extended.isoformat()},
42 )
43
44
45 def _release(res_id: str, run_id: str, force: object = None) -> CoordRecordIn:
46 payload = {"schema_version": "1", "reservation_id": res_id, "run_id": run_id, "released_at": _NOW.isoformat(), "reason": "completed"}
47 if force is not None:
48 payload["force"] = force
49 return CoordRecordIn(kind="release", record_id=fake_id(f"rel-{res_id}-{run_id}"), run_id=run_id, payload=payload)
50
51
52 async def _push(session: AsyncSession, repo_id: str, *records: CoordRecordIn) -> None:
53 await coord_push(session, repo_id, CoordPushRequest(records=list(records)))
54
55
56 async def _expiries(session: AsyncSession, repo_id: str, res_id: str) -> set[datetime]:
57 rows = (await session.execute(
58 select(_cm.MusehubCoordReservation).where(_cm.MusehubCoordReservation.reservation_id == res_id,
59 _cm.MusehubCoordReservation.repo_id == repo_id)
60 )).scalars().all()
61 for r in rows:
62 await session.refresh(r)
63 return {r.expires_at for r in rows}
64
65
66 async def _heartbeat_rows(session: AsyncSession, repo_id: str) -> list[_cm.MusehubCoordRecord]:
67 return list((await session.execute(
68 select(_cm.MusehubCoordRecord).where(_cm.MusehubCoordRecord.repo_id == repo_id,
69 _cm.MusehubCoordRecord.kind == "heartbeat")
70 )).scalars().all())
71
72
73 class TestPushedHeartbeats:
74 @pytest.mark.asyncio
75 async def test_a_heartbeat_extends_every_row_of_the_reservation(self, db_session: AsyncSession) -> None:
76 repo = await create_repo(db_session, slug="hb-extend")
77 res = fake_id("hb-extend")
78 await _push(db_session, repo.repo_id, _reservation(res))
79 later = _NOW + timedelta(seconds=3600)
80 await _push(db_session, repo.repo_id, _heartbeat(res, _NOW + timedelta(seconds=10), later))
81 assert await _expiries(db_session, repo.repo_id, res) == {later}
82
83 @pytest.mark.asyncio
84 async def test_a_stale_heartbeat_never_shortens_the_expiry(self, db_session: AsyncSession) -> None:
85 repo = await create_repo(db_session, slug="hb-stale")
86 res = fake_id("hb-stale")
87 await _push(db_session, repo.repo_id, _reservation(res))
88 later = _NOW + timedelta(seconds=3600)
89 await _push(db_session, repo.repo_id, _heartbeat(res, _NOW + timedelta(seconds=20), later))
90 await _push(db_session, repo.repo_id, _heartbeat(res, _NOW + timedelta(seconds=5), _NOW + timedelta(seconds=60)))
91 assert await _expiries(db_session, repo.repo_id, res) == {later}
92
93 @pytest.mark.asyncio
94 async def test_a_released_reservation_is_not_extended(self, db_session: AsyncSession) -> None:
95 repo = await create_repo(db_session, slug="hb-released")
96 res = fake_id("hb-released")
97 original = _NOW + timedelta(seconds=300)
98 await _push(db_session, repo.repo_id, _reservation(res, expires_at=original))
99 await _push(db_session, repo.repo_id, _release(res, "agent-a"))
100 await _push(db_session, repo.repo_id, _heartbeat(res, _NOW + timedelta(seconds=10), _NOW + timedelta(seconds=3600)))
101 assert await _expiries(db_session, repo.repo_id, res) == {original}
102
103 @pytest.mark.asyncio
104 async def test_a_heartbeat_for_an_unknown_reservation_is_a_no_op(self, db_session: AsyncSession) -> None:
105 repo = await create_repo(db_session, slug="hb-unknown")
106 other = fake_id("hb-unknown-other")
107 await _push(db_session, repo.repo_id, _reservation(other))
108 await _push(db_session, repo.repo_id, _heartbeat(fake_id("nobody"), _NOW, _NOW + timedelta(seconds=3600)))
109 assert await _expiries(db_session, repo.repo_id, other) == {_NOW + timedelta(seconds=300)}
110 assert len(await _heartbeat_rows(db_session, repo.repo_id)) == 1 # the record itself is stored
111
112 @pytest.mark.asyncio
113 async def test_a_forced_heartbeat_from_another_run_still_extends(self, db_session: AsyncSession) -> None:
114 """No owner check: an orchestrator's ``--force`` heartbeat carries its own run id."""
115 repo = await create_repo(db_session, slug="hb-forced")
116 res = fake_id("hb-forced")
117 await _push(db_session, repo.repo_id, _reservation(res, run_id="agent-a"))
118 later = _NOW + timedelta(seconds=900)
119 await _push(db_session, repo.repo_id, _heartbeat(res, _NOW + timedelta(seconds=1), later, run_id="orchestrator"))
120 assert await _expiries(db_session, repo.repo_id, res) == {later}
121
122
123 class TestHeartbeatRetention:
124 @pytest.mark.asyncio
125 async def test_only_the_latest_heartbeat_per_reservation_is_kept(self, db_session: AsyncSession) -> None:
126 repo = await create_repo(db_session, slug="hb-retain")
127 res_a, res_b = fake_id("hb-retain-a"), fake_id("hb-retain-b")
128 await _push(db_session, repo.repo_id, _reservation(res_a), _reservation(res_b))
129 for i in (1, 2, 3):
130 await _push(db_session, repo.repo_id, _heartbeat(res_a, _NOW + timedelta(seconds=i), _NOW + timedelta(seconds=600 + i)))
131 await _push(db_session, repo.repo_id, _heartbeat(res_b, _NOW, _NOW + timedelta(seconds=600)))
132 # a stale beat pushed last does not displace the newest one
133 await _push(db_session, repo.repo_id, _heartbeat(res_a, _NOW, _NOW + timedelta(seconds=601)))
134 rows = (await db_session.execute(
135 select(_cm.MusehubCoordRecord.record_id, _cm.MusehubCoordRecord.payload["reservation_id"].astext).where(
136 _cm.MusehubCoordRecord.repo_id == repo.repo_id, _cm.MusehubCoordRecord.kind == "heartbeat")
137 )).all()
138 by_res = {rid: record_id for record_id, rid in rows}
139 assert len(rows) == 2
140 assert by_res[res_a] == _heartbeat(res_a, _NOW + timedelta(seconds=3), _NOW).record_id
141 assert by_res[res_b] == _heartbeat(res_b, _NOW, _NOW).record_id
142
143
144 class TestReleaseOwnership:
145 @pytest.mark.asyncio
146 async def test_the_owners_release_applies(self, db_session: AsyncSession) -> None:
147 repo = await create_repo(db_session, slug="rel-owner")
148 res = fake_id("rel-owner")
149 await _push(db_session, repo.repo_id, _reservation(res, run_id="agent-a"))
150 await _push(db_session, repo.repo_id, _release(res, "agent-a"))
151 assert await list_reservations(db_session, repo.repo_id) == []
152
153 @pytest.mark.asyncio
154 async def test_another_runs_release_is_ignored_and_logged(self, db_session: AsyncSession, caplog: pytest.LogCaptureFixture) -> None:
155 repo = await create_repo(db_session, slug="rel-other")
156 res = fake_id("rel-other")
157 await _push(db_session, repo.repo_id, _reservation(res, run_id="agent-a"))
158 with caplog.at_level("WARNING", logger="musehub.services.musehub_coord_server"):
159 await _push(db_session, repo.repo_id, _release(res, "agent-b"))
160 active = await list_reservations(db_session, repo.repo_id)
161 assert {r.reservation_id for r in active} == {res}
162 assert any("release ignored" in r.getMessage() and "agent-b" in r.getMessage() for r in caplog.records)
163 # the owner can still release it afterwards
164 await _push(db_session, repo.repo_id, _release(res, "agent-a"))
165 assert await list_reservations(db_session, repo.repo_id) == []
166
167 @pytest.mark.asyncio
168 async def test_a_release_without_run_id_is_ignored(self, db_session: AsyncSession) -> None:
169 repo = await create_repo(db_session, slug="rel-norun")
170 res = fake_id("rel-norun")
171 await _push(db_session, repo.repo_id, _reservation(res, run_id="agent-a"))
172 rec = CoordRecordIn(kind="release", record_id=fake_id("rel-norun-x"), run_id="", payload={"reservation_id": res})
173 await _push(db_session, repo.repo_id, rec)
174 assert len(await list_reservations(db_session, repo.repo_id)) == 2 # two address rows, still active
175
176
177 class TestForcedRelease:
178 @pytest.mark.asyncio
179 async def test_a_forced_release_from_an_orchestrator_applies(self, db_session: AsyncSession) -> None:
180 """`muse coord release --force` (the client enforces who may) marks the payload "force": true; the hub honours it."""
181 repo = await create_repo(db_session, slug="rel-forced")
182 res = fake_id("rel-forced")
183 await _push(db_session, repo.repo_id, _reservation(res, run_id="agent-a"))
184 await _push(db_session, repo.repo_id, _release(res, "orchestrator-1", force=True))
185 assert await list_reservations(db_session, repo.repo_id) == []
186
187 @pytest.mark.asyncio
188 async def test_a_force_flag_that_is_not_true_does_not_bypass_ownership(self, db_session: AsyncSession) -> None:
189 repo = await create_repo(db_session, slug="rel-notforced")
190 res = fake_id("rel-notforced")
191 await _push(db_session, repo.repo_id, _reservation(res, run_id="agent-a"))
192 await _push(db_session, repo.repo_id, _release(res, "agent-b", force="yes"))
193 assert {r.reservation_id for r in await list_reservations(db_session, repo.repo_id)} == {res}
File History 3 commits
sha256:609941412be74ff9d83ca3c6921d119706e988e16408184660ed94aca17ab51c feat(coord): the hub honours a forced release ("force": tru… Opus 5.5 2 days ago
sha256:41ed2ded57d17544c111c81960ee6a7a7febc9a0d81a778616319265ac93a654 merge: rc25 — Mists wave (#316 #317 #318 #322; #319 #320 #3… Opus 5.5 minor ⚠ 2 days ago
sha256:fbcefb2702e89b64aeab652f89821a80bd1f548a003d1ca0637698ab10eeb1b8 fix(coord): pushed heartbeats extend hub reservations, keep… Opus 5.5 patch 2 days ago