gabriel / musehub public
test_coordination.py python
373 lines 19.8 KB
Raw
sha256:16d33f5988942b0ea6160933bdd2ac8e5c97aa4a0816c1d4be03abdfc73da2f4 docs: follow Muse 0.2.2rc8 — fixed defects leave the pages Opus 5.5 minor ⚠ breaking 1 day ago
1 """DOC_CRD — /muse/coordination contract (musehub#251, QA contract v2).
2
3 Every claim on the page was checked against muse 0.2.1rc20 (muse ``dev`` 5becda10) in a disposable
4 sandbox on 2026-11-07. These tests keep the corrected page honest in both directions: they run the real
5 ``muse coord`` commands in a throwaway repository (offline, a temporary HOME, no identity, no remotes)
6 and compare the engine's behaviour and JSON with what the page shows, so a change to either side fails
7 here. Known product defects stay documented as contract plus warning (#241 defect-preservation rule):
8 the deterministic ones are pinned as strict xfails against their product tickets, so a fix flips them to
9 XPASS and the marker must go in the same commit. The two race defects (re-claim, concurrent DAG writes)
10 are not pinned here, because a race test cannot fail deterministically; their tickets carry the repro.
11 Rendering is the shared in-process fixture (no database).
12 """
13
14 from __future__ import annotations
15
16 import html as html_lib
17 import json
18 import os
19 import pathlib
20 import re
21 import subprocess
22 import sys
23 import time
24 from collections.abc import Callable
25
26 import pytest
27
28 from tests.docs_contract import registry_check as rc
29
30 PAGE = rc.phase_path("coordination")
31 MUSE_ISSUES = "https://staging.musehub.ai/gabriel/muse/issues/"
32
33 # Product tickets the page's known-issue warnings link to (filed under #251, assigned to gabriel).
34 _T_RECLAIM = "198" # re-claim of an expired claim can double-claim
35 _T_DAG_RACE = "200" # concurrent --depends-on writers can create a cycle
36
37 _ENVELOPE = {"muse_version", "schema", "exit_code", "duration_ms", "timestamp", "warnings"}
38 _FILES = {
39 "src/models.py": "class Invoice:\n def __init__(self, lines):\n self.lines = lines\n",
40 "src/billing.py": (
41 "from src.models import Invoice\n\n\n"
42 "def compute_total(invoice: Invoice) -> int:\n return sum(invoice.lines)\n\n\n"
43 "def render(invoice: Invoice) -> str:\n return str(compute_total(invoice))\n"
44 ),
45 "src/tokens.py": "def make_token(user: str) -> str:\n return 't-' + user\n",
46 "src/auth.py": "from src.tokens import make_token\n\n\ndef login(user: str) -> str:\n return make_token(user)\n",
47 "src/middleware.py": "def wrap(handler):\n return handler\n",
48 "src/api.py": "from src.middleware import wrap\n\n\n@wrap\ndef handle(request):\n return request\n",
49 "tests/test_billing.py": "def test_total():\n assert True\n",
50 }
51 _SHARDS = [["src/billing.py", "src/models.py"], ["src/auth.py", "src/tokens.py"], ["src/api.py", "src/middleware.py"]]
52
53 Muse = Callable[..., dict]
54
55
56 @pytest.fixture(scope="module")
57 def html(rendered: dict[str, str]) -> str:
58 return rendered[PAGE].replace("'", "'").replace(""", '"').replace(""", '"')
59
60
61 def _text(fragment: str) -> str:
62 return html_lib.unescape(re.sub(r"<[^>]+>", "", fragment))
63
64
65 def _section(html: str, anchor: str) -> str:
66 rest = html.split(f'id="{anchor}"', 1)[1]
67 return " ".join(rest.split("</section>", 1)[0].split())
68
69
70 # ── Engine harness: the real CLI in a throwaway repository ──────────────────
71
72
73 def _engine(tmp: pathlib.Path) -> Muse:
74 home = tmp / "home"
75 home.mkdir()
76 env = {k: v for k, v in os.environ.items() if not k.startswith(("MUSE_", "GIT_"))}
77 env.update(HOME=str(home), XDG_CONFIG_HOME=str(home / ".config"), MUSE_KEYCHAIN_BACKEND="disabled",
78 PYTHON_KEYRING_BACKEND="keyring.backends.fail.Keyring", NO_COLOR="1",
79 PYTHONPATH=os.pathsep.join(p for p in sys.path if p))
80 repo = tmp / "repo"
81 repo.mkdir()
82
83 def muse(*args: str, ok: bool = True) -> dict:
84 p = subprocess.run([sys.executable, "-c", "import sys; from muse.cli.app import main; main(sys.argv[1:])", *args],
85 cwd=repo, env=env, capture_output=True, text=True, timeout=120)
86 if ok:
87 assert p.returncode == 0, (args, p.stdout, p.stderr)
88 try:
89 out = json.loads(p.stdout)
90 except ValueError:
91 out = {}
92 return {"rc": p.returncode, "json": out, "stdout": p.stdout, "stderr": p.stderr}
93
94 muse("init")
95 assert "No remotes" in muse("remote", "-v")["stdout"] + muse("remote", "-v")["stderr"]
96 for rel, text in _FILES.items():
97 (repo / rel).parent.mkdir(parents=True, exist_ok=True)
98 (repo / rel).write_text(text)
99 muse("code", "add", ".")
100 muse("commit", "-m", "base")
101 return muse
102
103
104 def _body(r: dict) -> dict:
105 return {k: v for k, v in r["json"].items() if k not in _ENVELOPE}
106
107
108 @pytest.fixture(scope="module")
109 def engine(tmp_path_factory: pytest.TempPathFactory) -> dict:
110 """The page's scenario and DAG chain, executed."""
111 muse = _engine(tmp_path_factory.mktemp("coord-engine"))
112 out: dict = {"shard": _body(muse("coord", "shard", "--agents", "3", "--json"))}
113 for i, files in enumerate(_SHARDS, 1):
114 muse("coord", "enqueue", f"Refactor shard {i}", "--queue", "refactor", "--priority", "10",
115 "--payload", json.dumps({"files": files}), "--run-id", "orchestrator", "--json")
116 claims = [muse("coord", "claim", "--queue", "refactor", "--run-id", a, "--json") for a in ("agent-1", "agent-2", "agent-3")]
117 out["claim"] = claims[0]
118 out["claim_body"] = _body(claims[0])
119 r1 = muse("coord", "reserve", "src/billing.py::compute_total", "--run-id", "agent-1", "--op", "modify", "--ttl", "7200", "--json")
120 r2 = muse("coord", "reserve", "src/billing.py::compute_total", "--run-id", "agent-2", "--op", "rename", "--ttl", "7200", "--json")
121 out["forecast"] = _body(muse("coord", "forecast", "--json"))
122 muse("coord", "release", r2["json"]["reservation_id"], "--run-id", "agent-2", "--json")
123 dep = muse("coord", "reserve", "src/billing.py::compute_total", "--run-id", "agent-2", "--op", "modify", "--ttl", "7200",
124 "--depends-on", r1["json"]["reservation_id"], "--json")
125 out["dep_node"] = _body(muse("coord", "dag", "--reservation-id", dep["json"]["reservation_id"], "--json"))
126 muse("coord", "release", "--all-for-run", "agent-1", "--run-id", "agent-1", "--json")
127 muse("coord", "release", "--all-for-run", "agent-2", "--run-id", "agent-2", "--json")
128 t1 = claims[0]["json"]["task_id"]
129 muse("coord", "complete", t1, "--run-id", "agent-1", "--result", '{"symbols_modified": 12, "tests_passing": true}', "--json")
130 out["tasks_completed"] = _body(muse("coord", "tasks", "--queue", "refactor", "--status", "completed", "--json"))
131 # DAG chain (the page's three-step example)
132 a = muse("coord", "reserve", "src/billing.py::compute_total", "--run-id", "agent-A", "--op", "rename", "--json")["json"]["reservation_id"]
133 b = muse("coord", "reserve", "src/billing.py::compute_invoice_total", "--run-id", "agent-B", "--op", "modify",
134 "--depends-on", a, "--json")["json"]["reservation_id"]
135 c = muse("coord", "reserve", "tests/test_billing.py::test_total", "--run-id", "agent-C", "--op", "modify",
136 "--depends-on", b, "--json")["json"]["reservation_id"]
137 out["chain"] = (a, b, c)
138 out["dag"] = _body(muse("coord", "dag", "--active-only", "--json"))
139 out["cycle_attempt"] = muse("coord", "reserve", "src/billing.py::compute_total", "--run-id", "agent-A", "--op", "rename",
140 "--depends-on", c, "--json", ok=False)
141 # Pass 3: intents, and reservation ops alone
142 for run, op in (("fc-4", "delete"), ("fc-5", "modify"), ("fc-6", "rename")):
143 muse("coord", "intent", "src/models.py::Invoice", "--op", op, "--run-id", run, "--json")
144 out["forecast_intents"] = _body(muse("coord", "forecast", "--json"))
145 out["muse"] = muse
146 return out
147
148
149 # ── The page's JSON matches the engine ───────────────────────────────────────
150
151
152 def _documented_json(html: str) -> dict[str, object]:
153 blocks = re.findall(r'data-crd-json="([^"]+)"(.*?)</pre>', html, re.S)
154 return {name: json.loads(_text(body.split("<pre>", 1)[1])) for name, body in blocks}
155
156
157 def test_every_json_output_on_the_page_is_checked(html: str) -> None:
158 assert set(_documented_json(html)) == {"claim", "forecast", "dag", "shard", "forecast-conflict"}
159
160
161 def test_documented_json_keys_are_the_engine_keys(html: str, engine: dict) -> None:
162 doc = _documented_json(html)
163 claim = engine["claim_body"]
164 assert set(doc["claim"]) == set(claim)
165 assert set(doc["claim"]["task"]) == set(claim["task"])
166 assert set(doc["forecast"]) == set(engine["forecast"])
167 assert set(doc["forecast"]["conflicts"][0]) == set(engine["forecast"]["conflicts"][0])
168 assert set(doc["dag"]) == set(engine["dag"])
169 assert set(doc["dag"]["nodes"][0]) == set(engine["dag"]["nodes"][0])
170 assert set(doc["shard"]) == set(engine["shard"])
171 assert set(doc["shard"]["shards"][0]) == set(engine["shard"]["shards"][0])
172 pass3 = [c for c in engine["forecast_intents"]["conflicts"] if c["conflict_type"] == "operation_conflict"]
173 assert pass3 and set(doc["forecast-conflict"]) == set(pass3[0])
174 assert doc["forecast-conflict"]["confidence"] == pass3[0]["confidence"] == 0.9
175 assert doc["forecast-conflict"]["description"] == pass3[0]["description"]
176
177
178 def test_scenario_forecast_matches_the_engine(html: str, engine: dict) -> None:
179 doc, real = _documented_json(html)["forecast"], engine["forecast"]
180 assert [c["conflict_type"] for c in real["conflicts"]] == [c["conflict_type"] for c in doc["conflicts"]] == ["address_overlap"]
181 assert real["conflicts"][0]["description"] == doc["conflicts"][0]["description"]
182 assert sorted(real["conflicts"][0]["agents"]) == sorted(doc["conflicts"][0]["agents"])
183 # modify vs rename on reservations does not reach pass 3; the page says so
184 assert "that pass reads" in _section(html, "scenario")
185
186
187 def test_scenario_extraction_scripts_work_on_the_engine_output(html: str, engine: dict) -> None:
188 claim = engine["claim"]["json"]
189 assert claim["task_id"].startswith("sha256:")
190 assert claim["task"]["payload"]["files"] in _SHARDS # the page's FILES= line
191 assert "['task']['payload']['files']" in html
192 items = engine["tasks_completed"]["items"]
193 assert [(t["title"], t["status"], t["claimer_run_id"]) for t in items] == [(claim["task"]["title"], "completed", "agent-1")]
194 assert "data['items']" in html and "data['tasks']" not in html
195 assert {s["shard"] for s in engine["shard"]["shards"]} == {1, 2, 3}
196
197
198 def test_depends_on_blocks_and_the_chain_matches_the_page(html: str, engine: dict) -> None:
199 assert engine["dep_node"]["blocked"] is True
200 a, b, c = engine["chain"]
201 dag = engine["dag"]
202 chain = [n for n in dag["nodes"] if n["reservation_id"] in (a, b, c)]
203 assert [n["reservation_id"] for n in sorted(chain, key=lambda n: n["topo_index"])] == [a, b, c]
204 assert [n["blocked"] for n in sorted(chain, key=lambda n: n["topo_index"])] == [False, True, True]
205 assert dag["cycle"] is None
206 doc = _documented_json(html)["dag"]
207 assert [n["blocked"] for n in doc["nodes"]] == [False, True, True]
208 # a single writer cannot create a cycle
209 attempt = engine["cycle_attempt"]
210 assert attempt["rc"] == 1 and "cycle" in (attempt["json"].get("dependency_error") or "")
211
212
213 # ── V1: the claim lifecycle figure, executed row by row ─────────────────────
214
215
216 def _figure(html: str) -> str:
217 return html.split('id="claim-lifecycle-figure"', 1)[1].split("</figure>", 1)[0]
218
219
220 def test_figure_renders_as_an_accessible_static_figure(html: str) -> None:
221 fig = html.split('<figure class="docfig" id="claim-lifecycle-figure"', 1)[1].split("</figure>", 1)[0]
222 assert 'aria-labelledby="claim-lifecycle-figure-title"' in fig
223 assert 'id="claim-lifecycle-figure-title"' in fig and "<figcaption" in fig
224 assert "<caption>" in fig and fig.count('scope="col"') == 4
225 assert "<script" not in fig and "<svg" not in fig
226 states = re.findall(r'data-state="([a-z_]+)"', fig)
227 assert states == ["pending", "claimed", "timed_out", "completed", "failed", "cancelled"]
228
229
230 def test_figure_transitions_are_the_engine_transitions(html: str, tmp_path: pathlib.Path) -> None:
231 rows = re.findall(r'data-from="([a-z_]+)" data-to="([a-z_]+)"', _figure(html))
232 assert rows == [("pending", "claimed"), ("pending", "cancelled"), ("claimed", "completed"), ("claimed", "failed"),
233 ("claimed", "cancelled"), ("claimed", "timed_out"), ("timed_out", "claimed")]
234 muse = _engine(tmp_path)
235
236 def task(title: str) -> str:
237 return muse("coord", "enqueue", title, "--queue", "fig", "--run-id", "orch", "--json")["json"]["task_id"]
238
239 def claim(run: str, *extra: str) -> dict:
240 return muse("coord", "claim", "--queue", "fig", "--run-id", run, *extra, "--json")["json"]
241
242 def status(tid: str) -> str:
243 items = muse("coord", "tasks", "--queue", "fig", "--json")["json"]["items"]
244 return next(i["status"] for i in items if i["task_id"] == tid)
245
246 # pending -> cancelled by any run
247 t = task("cancel me")
248 assert muse("coord", "cancel-task", t, "--run-id", "someone", "--json")["json"]["status"] == "cancelled"
249 # pending -> claimed -> completed (claimer only)
250 t = task("complete me")
251 assert claim("w1")["task_id"] == t
252 assert muse("coord", "complete", t, "--run-id", "w2", "--json", ok=False)["rc"] == 1
253 assert muse("coord", "complete", t, "--run-id", "w1", "--result", "{}", "--json")["json"]["status"] == "completed"
254 # claimed -> failed (claimer only)
255 t = task("fail me")
256 assert claim("w1")["task_id"] == t
257 assert muse("coord", "fail-task", t, "--run-id", "w2", "--json", ok=False)["rc"] == 1
258 assert muse("coord", "fail-task", t, "--run-id", "w1", "--error", "x", "--json")["json"]["status"] == "failed"
259 # claimed -> cancelled: refused without --force, accepted with it
260 t = task("force me")
261 assert claim("w1")["task_id"] == t
262 assert muse("coord", "cancel-task", t, "--run-id", "orch", "--json", ok=False)["rc"] == 1
263 assert muse("coord", "cancel-task", t, "--run-id", "orch", "--force", "--json")["json"]["status"] == "cancelled"
264 # claimed -> timed_out (derived) -> claimed by another run
265 t = task("expire me")
266 assert claim("w1", "--claim-ttl", "1")["task_id"] == t
267 time.sleep(1.3)
268 assert status(t) == "timed_out"
269 again = claim("w3")
270 assert again["task_id"] == t and again["claimer_run_id"] == "w3"
271 assert status(t) == "claimed"
272
273
274 # ── Fixed in Muse batch 1 (muse#199, #201, #202, #203, #204): the contracts, pinned ──
275
276
277 def test_reserving_a_function_and_its_caller_reports_blast_radius(engine: dict) -> None:
278 muse = engine["muse"]
279 muse("coord", "reserve", "src/billing.py::render", "--run-id", "caller-agent", "--op", "modify", "--json")
280 out = muse("coord", "forecast", "--run-id", "caller-agent", "--json")["json"]
281 assert any(c["conflict_type"] == "blast_radius_overlap" for c in out["conflicts"]), out
282
283
284 def test_a_foreign_run_cannot_heartbeat_a_reservation(engine: dict) -> None:
285 muse = engine["muse"]
286 rid = muse("coord", "reserve", "src/auth.py::login", "--run-id", "owner-1", "--json")["json"]["reservation_id"]
287 r = muse("coord", "heartbeat", rid, "--run-id", "intruder", "--json", ok=False)
288 assert r["rc"] == 1 and r["json"]["status"] == "not_owner" and r["json"]["owner_run_id"] == "owner-1"
289 assert muse("coord", "release", rid, "--run-id", "intruder", "--json", ok=False)["json"]["status"] == "not_owner"
290 # an orchestrator may act for the owner with --force
291 assert muse("coord", "heartbeat", rid, "--run-id", "orch", "--force", "--json")["json"]["kind"] == "reservation"
292 assert muse("coord", "release", "--all-for-run", "owner-1", "--run-id", "orch", "--json", ok=False)["rc"] == 1
293
294
295 def test_reserving_again_after_a_release_is_active(engine: dict) -> None:
296 muse = engine["muse"]
297 first = muse("coord", "reserve", "src/api.py::handle", "--run-id", "rr-1", "--op", "modify", "--json")["json"]
298 again = muse("coord", "reserve", "src/api.py::handle", "--run-id", "rr-1", "--op", "modify", "--json")["json"]
299 assert again["reservation_id"] == first["reservation_id"] and again["renewed"] is True # live: renewed, same ID
300 muse("coord", "release", first["reservation_id"], "--run-id", "rr-1", "--json")
301 after = muse("coord", "reserve", "src/api.py::handle", "--run-id", "rr-1", "--op", "modify", "--json")["json"]
302 assert after["reservation_id"] != first["reservation_id"] and after["renewed"] is False # after a release: a new ID
303 assert muse("coord", "list", "--run-id", "rr-1", "--json")["json"]["reservations"], "re-reservation is not active"
304
305
306 def test_a_glob_reservation_overlaps_a_literal_one(engine: dict) -> None:
307 muse = engine["muse"]
308 muse("coord", "reserve", "src/tokens.py::make_token", "--run-id", "lit-1", "--json")
309 muse("coord", "reserve", "src/tokens.py::*", "--run-id", "glob-1", "--json")
310 out = muse("coord", "forecast", "--run-id", "glob-1", "--json")["json"]
311 assert any(c["conflict_type"] == "address_overlap" for c in out["conflicts"]), out
312
313
314 def test_a_worker_can_heartbeat_its_claim(html: str, engine: dict) -> None:
315 muse = engine["muse"]
316 muse("coord", "enqueue", "long job", "--queue", "hb", "--run-id", "orch", "--json")
317 tid = muse("coord", "claim", "--queue", "hb", "--run-id", "w-hb", "--json")["json"]["task_id"]
318 r = muse("coord", "heartbeat", tid, "--run-id", "w-hb", "--json", ok=False)
319 assert r["rc"] == 0
320 body = _body(r)
321 assert body["kind"] == "task" and body["claimer_run_id"] == "w-hb"
322 documented = re.search(r'A task heartbeat answers <code>\{([^}]*)\}</code>', html).group(1)
323 assert set(re.findall(r'"(\w+)"', documented)) - {"ok", "task"} == set(body)
324 assert muse("coord", "heartbeat", tid, "--run-id", "w-other", "--json", ok=False)["json"]["status"] == "not_owner"
325
326
327 def test_tasks_lists_the_result_of_a_completed_task(engine: dict) -> None:
328 item = engine["tasks_completed"]["items"][0]
329 assert item["result"] == {"symbols_modified": 12, "tests_passing": True} and item["error"] is None
330
331
332 def test_an_unclaimed_task_past_its_ttl_is_listed_as_expired(html: str, tmp_path: pathlib.Path) -> None:
333 muse = _engine(tmp_path)
334 tid = muse("coord", "enqueue", "short-lived", "--queue", "ttl", "--ttl", "1", "--run-id", "orch", "--json")["json"]["task_id"]
335 time.sleep(1.3)
336 items = muse("coord", "tasks", "--queue", "ttl", "--json")["json"]["items"]
337 assert [(i["task_id"], i["status"]) for i in items] == [(tid, "expired")]
338 assert "listed by <code>coord tasks</code> as <code>expired</code>" in _figure(html)
339
340
341 # ── Corrections stay corrected; gaps stay linked ─────────────────────────────
342
343
344 @pytest.mark.parametrize("stale", [
345 "json.load(sys.stdin)['payload']", # payload is nested under task
346 "data['tasks']", # the list key is items
347 "--tags</span> billing python", # --tags takes one comma-separated value
348 "--kind reservation ", # kind values are the plural directory names
349 "full_commit_id",
350 "structurally impossible",
351 "eliminating merge conflicts by construction",
352 "exit after first event",
353 "If the nonce matches, the agent won",
354 "used by Tier 3 forecast",
355 "zero-interference",
356 '"operation_conflict"</span>,\n <span class="tok-key">"addresses"</span>: [<span class="tok-str">"src/billing.py::compute_total"',
357 ])
358 def test_corrected_claims_stay_corrected(html: str, stale: str) -> None:
359 assert stale not in html
360
361
362 def test_fixed_coordination_gaps_are_gone(html: str) -> None:
363 # muse#198 and muse#200 shipped in 0.2.2rc8: callouts and table rows dropped, tickets no longer linked.
364 for fixed in (_T_RECLAIM, _T_DAG_RACE, "199", "201", "202", "203", "204"):
365 assert f'{MUSE_ISSUES}{fixed}"' not in html, fixed
366 assert 'id="known-reclaim"' not in html and 'id="known-dag-race"' not in html
367 assert "None in muse 0.2.2rc8" in _section(html, "known-issues")
368
369
370 def test_every_in_page_link_has_its_target(html: str) -> None:
371 targets = set(re.findall(r'href="#([^"]+)"', html))
372 ids = set(re.findall(r'id="([^"]+)"', html))
373 assert targets - ids == set()
File History 4 commits
sha256:16d33f5988942b0ea6160933bdd2ac8e5c97aa4a0816c1d4be03abdfc73da2f4 docs: follow Muse 0.2.2rc8 — fixed defects leave the pages Opus 5.5 minor ⚠ 1 day ago
sha256:e1629e5f328982dbf6e535d3f253993c78ff225dfee4ebb0e0a15fc1db69046b merge: rc25 — docs and docs-contract tests match Muse batch… Opus 5.5 minor ⚠ 2 days ago
sha256:18af802c77ac12ce342231ec4148bc096f6441de0f76c58699c2ef688e5d1742 docs(coordination): blast radius, ownership, re-reserve, gl… Opus 5.5 minor ⚠ 2 days ago
sha256:395afcc81bdadd355e87f50a1eb01b857ffdcce78f54635a77fe893ca7352528 docs(coordination): #251 Agent Coordination QA contract v2 … Opus 5.5 minor ⚠ 9 days ago