gabriel / musehub public
test_harmony.py python
257 lines 12.3 KB
Raw
sha256:369f691c7aaec3ca84a864ad1d099d5409045ddc12d185a922b3c84b9e2a1bad merge: 100% passing, parallelized MuseHub test suite (task/… Sonnet 5.5 patch 3 days ago
1 """DOC_HAR — /muse/harmony contract (musehub#250, QA contract v2).
2
3 Claims were checked against muse 0.2.1rc20 in disposable sandboxes on 2026-11-07. The page's own
4 warning callouts carry the known-defect links; the asserts below keep the corrected page honest in
5 both directions — a change to either side fails here.
6
7 * DOC_HAR_01/02/03 — CLI parse, anchors, loopback URLs: the shared checkers (``test_doc_inf_*``).
8 * DOC_HAR_04 — the learning-loop scenario, executed: ``muse resolve`` earns exact replay;
9 ``muse checkout --ours/--theirs`` + ``code add`` alone (no ``muse resolve``) does not.
10 * DOC_HAR_06 — audit coverage of the automatic path (strict xfail, gabriel/muse#206).
11 * DOC_HAR_07 — conflict_fingerprint() opt-in and the sha256: format the plugin section states.
12 * DOC_HAR_05 — tier-count terminology: the page says four stages and names both surfaces; the CLI's
13 own help text still says "three-tier" (strict xfail, gabriel/muse#155).
14 """
15
16 from __future__ import annotations
17
18 import json
19 import os
20 import pathlib
21 import shutil
22 import subprocess
23 import sys
24 from dataclasses import dataclass, field
25
26 import pytest
27
28 from tests.docs_contract import registry_check as rc
29
30 PAGE = rc.phase_path("harmony")
31
32 XFAIL = dict(strict=True, raises=AssertionError)
33
34
35 @pytest.fixture(scope="module")
36 def html(rendered: dict[str, str]) -> str:
37 return rendered[PAGE].replace("'", "'").replace(""", '"').replace(""", '"')
38
39
40 @dataclass
41 class R:
42 rc: int
43 out: str
44 err: str
45 json: dict = field(default_factory=dict)
46
47
48 class Sandbox:
49 def __init__(self, root: pathlib.Path) -> None:
50 self.root = root
51 self.home = root / "home"
52 self.home.mkdir(parents=True)
53 env = {k: v for k, v in os.environ.items() if not k.startswith(("MUSE_", "GIT_"))}
54 env.update(HOME=str(self.home), XDG_CONFIG_HOME=str(self.home / ".config"), MUSE_KEYCHAIN_BACKEND="disabled",
55 PYTHON_KEYRING_BACKEND="keyring.backends.fail.Keyring", NO_COLOR="1",
56 PYTHONPATH=os.pathsep.join(p for p in sys.path if p))
57 self.env = env
58 self.n = 0
59
60 def run(self, repo: pathlib.Path, *args: str, ok: bool = True) -> R:
61 assert str(repo.resolve()).startswith(str(self.root.resolve())), repo
62 p = subprocess.run([sys.executable, "-c", "import sys; from muse.cli.app import main; main(sys.argv[1:])", *args],
63 cwd=repo, env=self.env, capture_output=True, text=True, timeout=120)
64 if ok:
65 assert p.returncode == 0, (args, p.stdout, p.stderr)
66 try:
67 out = json.loads(p.stdout)
68 except ValueError:
69 out = {}
70 return R(p.returncode, p.stdout, p.stderr, out if isinstance(out, dict) else {})
71
72 def write(self, repo: pathlib.Path, files: dict[str, str]) -> None:
73 for rel, text in files.items():
74 path = repo / rel
75 path.parent.mkdir(parents=True, exist_ok=True)
76 path.write_text(text)
77
78 def commit(self, repo: pathlib.Path, files: dict[str, str], msg: str) -> None:
79 self.write(repo, files)
80 self.run(repo, "code", "add", ".")
81 self.run(repo, "commit", "-m", msg)
82
83 def copy(self, repo: pathlib.Path) -> pathlib.Path:
84 self.n += 1
85 dest = self.root / "copies" / f"c{self.n}"
86 shutil.copytree(repo, dest, symlinks=True)
87 return dest
88
89 def build(self, base: dict, feat: dict, dev: dict) -> pathlib.Path:
90 self.n += 1
91 repo = self.root / "repos" / f"r{self.n}"
92 repo.mkdir(parents=True)
93 self.run(repo, "init", "--domain", "code", "--default-branch", "dev")
94 self.commit(repo, base, "base")
95 self.run(repo, "checkout", "-b", "feat")
96 self.commit(repo, feat, "feat")
97 self.run(repo, "checkout", "dev")
98 self.commit(repo, dev, "dev")
99 return repo
100
101
102 @pytest.fixture(scope="module")
103 def sb(tmp_path_factory: pytest.TempPathFactory) -> Sandbox:
104 return Sandbox(tmp_path_factory.mktemp("harmony"))
105
106
107 _BASE = {"config.py": "def a():\n return 10\n", "server.py": "def timeout():\n return 10\n"}
108 _FEAT = {"config.py": "def a():\n return 50\n", "server.py": "def timeout():\n return 50\n"}
109 _DEV = {"config.py": "def a():\n return 20\n", "server.py": "def timeout():\n return 20\n"}
110 _HAND = "def a():\n return 35\n"
111
112
113 def _one(files: dict[str, str], name: str) -> dict[str, str]:
114 return {name: files[name]}
115
116
117 def _best(sb: Sandbox, repo: pathlib.Path) -> dict[str, dict]:
118 pats = sb.run(repo, "harmony", "list", "--json").json["patterns"]
119 return {p["path"]: sb.run(repo, "harmony", "best", p["pattern_id"], "--json").json["resolution"] for p in pats}
120
121
122 def _replay(sb: Sandbox, learned: pathlib.Path, fresh: pathlib.Path) -> dict[str, dict]:
123 """Recreate the identical conflict: a pre-merge copy that carries the learned harmony store."""
124 shutil.copytree(learned / ".muse" / "harmony", fresh / ".muse" / "harmony")
125 # Resolutions point at outcome blobs; without the blob, auto_apply() reports "no_pattern".
126 for obj in (learned / ".muse" / "objects").rglob("*"):
127 dest = fresh / ".muse" / "objects" / obj.relative_to(learned / ".muse" / "objects")
128 if obj.is_file() and not dest.exists():
129 dest.parent.mkdir(parents=True, exist_ok=True)
130 shutil.copy2(obj, dest)
131 r = sb.run(fresh, "merge", "feat", "--explain", "--json", ok=False)
132 return {e["path"]: e for e in r.json["explain"]["per_path"]} | {"_status": {"status": r.json["status"]}}
133
134
135 def test_doc_har_04_single_side_pick_learns_at_1_0_and_replays(sb: Sandbox) -> None:
136 """The page's scenario: checkout --theirs, code add, commit. No path is in manually_resolved, so
137 record_resolutions() falls back to confidence=1.0 / human_verified=true, and the identical
138 conflict replays during the next muse merge."""
139 repo = sb.build(_one(_BASE, "config.py"), _one(_FEAT, "config.py"), _one(_DEV, "config.py"))
140 work = sb.copy(repo)
141 r1 = sb.run(work, "merge", "feat", "--json", ok=False)
142 assert r1.json["status"] == "conflict" and r1.json["conflicts"] == ["config.py::a"], r1.json
143 sb.run(work, "checkout", "--theirs", "config.py")
144 sb.run(work, "code", "add", "config.py")
145 sb.run(work, "commit", "-m", "merge: accept feat", "--agent-id", "qa", "--model-id", "qa")
146
147 best = _best(sb, work)["config.py::a"]
148 assert (best["strategy"], best["confidence"], best["human_verified"]) == ("manual", 1.0, True), best
149 assert best["resolved_by"]["type"] == "human", best # commit attribution not threaded (muse#193)
150
151 per_path = _replay(sb, work, sb.copy(repo))
152 assert per_path["_status"]["status"] == "merged", per_path
153 assert per_path["config.py"]["harmony_result"] == "auto_resolved", per_path
154
155
156 def test_doc_har_04b_side_pick_after_last_checkout_rule_gives_0_8_and_no_replay(sb: Sandbox) -> None:
157 """Side-pick server.py first, then hand-edit config.py and muse resolve it: config.py enters
158 manually_resolved, so the side-picked server.py is recorded at 0.8 / human_verified=false. 0.8 is
159 below the 0.85 gate of auto_apply(), so it never replays in muse merge (below_confidence_threshold)."""
160 repo = sb.build(_BASE, _FEAT, _DEV)
161 work = sb.copy(repo)
162 sb.run(work, "merge", "feat", "--json", ok=False)
163 sb.run(work, "checkout", "--theirs", "server.py")
164 sb.write(work, {"config.py": _HAND})
165 sb.run(work, "resolve", "config.py")
166 sb.run(work, "code", "add", "-u") # muse#192: the side-picked server.py is an unstaged tracked change the commit refuses to omit
167 sb.run(work, "commit", "-m", "merge: mixed", "--agent-id", "qa", "--model-id", "qa")
168
169 best = _best(sb, work)
170 assert (best["config.py::a"]["confidence"], best["config.py::a"]["human_verified"]) == (1.0, True), best
171 assert (best["server.py::timeout"]["confidence"], best["server.py::timeout"]["human_verified"]) == (0.8, False), best
172
173 per_path = _replay(sb, work, sb.copy(repo))
174 assert per_path["config.py"]["harmony_result"] == "auto_resolved", per_path
175 assert per_path["server.py"]["harmony_result"] == "below_confidence_threshold", per_path
176
177
178 def test_doc_har_04c_resolve_after_checkout_is_a_no_op(sb: Sandbox) -> None:
179 """muse checkout --theirs already removed the path from conflict_paths, so a later muse resolve
180 clears nothing (the page no longer tells readers to run it after a side-pick)."""
181 repo = sb.build(_one(_BASE, "config.py"), _one(_FEAT, "config.py"), _one(_DEV, "config.py"))
182 work = sb.copy(repo)
183 sb.run(work, "merge", "feat", "--json", ok=False)
184 sb.run(work, "checkout", "--theirs", "config.py")
185 r = sb.run(work, "resolve", "config.py")
186 assert "is not in the conflict list" in r.out, (r.out, r.err)
187
188
189 @pytest.mark.xfail(**XFAIL)
190 def test_doc_har_04d_side_pick_confidence_is_order_independent(sb: Sandbox) -> None:
191 """Contract: the same two resolutions record the same confidences whatever the command order.
192 rc20: checkout --ours/--theirs rewrites MERGE_STATE without manually_resolved, so doing the
193 hand-edit first and the side-pick last records both at 1.0 (gabriel/muse#196)."""
194 repo = sb.build(_BASE, _FEAT, _DEV)
195 work = sb.copy(repo)
196 sb.run(work, "merge", "feat", "--json", ok=False)
197 sb.write(work, {"config.py": _HAND})
198 sb.run(work, "resolve", "config.py")
199 sb.run(work, "checkout", "--theirs", "server.py")
200 sb.run(work, "commit", "-m", "merge: mixed, other order", "--agent-id", "qa", "--model-id", "qa")
201 best = _best(sb, work)
202 assert best["config.py::a"]["confidence"] == 1.0, best # setup sanity: holds in rc20
203 assert best["server.py::timeout"]["confidence"] == 0.8, best
204
205
206 @pytest.mark.xfail(**XFAIL)
207 def test_doc_har_06_merge_replay_writes_an_audit_entry(sb: Sandbox) -> None:
208 """Contract: an automatic replay is audited. rc20: auto_apply() writes no audit entry and
209 record_resolutions() writes none at commit, so the log stays empty (gabriel/muse#206)."""
210 repo = sb.build(_one(_BASE, "config.py"), _one(_FEAT, "config.py"), _one(_DEV, "config.py"))
211 work = sb.copy(repo)
212 sb.run(work, "merge", "feat", "--json", ok=False)
213 sb.run(work, "checkout", "--theirs", "config.py")
214 sb.run(work, "code", "add", "config.py")
215 sb.run(work, "commit", "-m", "merge: accept feat", "--agent-id", "qa", "--model-id", "qa")
216 fresh = sb.copy(repo)
217 per_path = _replay(sb, work, fresh)
218 assert per_path["config.py"]["harmony_result"] == "auto_resolved", per_path # setup sanity: holds in rc20
219 events = [e["event_type"] for e in sb.run(fresh, "harmony", "audit", "--json").json["entries"]]
220 assert "resolution_applied" in events, events
221
222
223 def test_doc_har_07_conflict_fingerprint_must_be_sha256_prefixed(tmp_path: pathlib.Path) -> None:
224 """The plugin section: a domain opts in by implementing conflict_fingerprint() on its own plugin
225 class; blob_id() output is used, a bare hexdigest is silently replaced by the blob fingerprint."""
226 import hashlib
227
228 from muse.core.harmony import blob_fingerprint
229 from muse.core.harmony.fingerprint import compute_semantic_fingerprint
230 from muse.core.types import blob_id
231 from muse.plugins.code.plugin import CodePlugin as CodeDomain
232
233 ours, theirs = blob_id(b"ours"), blob_id(b"theirs")
234 good = blob_id(b"shape")
235
236 class Prefixed(CodeDomain):
237 def conflict_fingerprint(self, path: str, ours_id: str, theirs_id: str, repo_root: pathlib.Path) -> str:
238 return good
239
240 class Bare(CodeDomain):
241 def conflict_fingerprint(self, path: str, ours_id: str, theirs_id: str, repo_root: pathlib.Path) -> str:
242 return hashlib.sha256(b"shape").hexdigest()
243
244 assert compute_semantic_fingerprint("a.json", ours, theirs, Prefixed(), tmp_path) == good
245 assert compute_semantic_fingerprint("a.json", ours, theirs, Bare(), tmp_path) == blob_fingerprint(ours, theirs)
246 assert compute_semantic_fingerprint("a.json", ours, theirs, CodeDomain(), tmp_path) == blob_fingerprint(ours, theirs)
247
248
249 # ── DOC_HAR_05: tier-count terminology (strict xfail, gabriel/muse#155) ────
250
251
252 @pytest.mark.xfail(**XFAIL)
253 def test_doc_har_05_cli_help_matches_page_tier_language() -> None:
254 out = subprocess.run([sys.executable, "-c",
255 "import sys; from muse.cli.app import main; main(['harmony', '--help'])"],
256 capture_output=True, text=True).stdout
257 assert "three-tier" not in out.lower(), "gabriel/muse#155: CLI help still says three-tier for a four-stage pipeline"
File History 3 commits
sha256:369f691c7aaec3ca84a864ad1d099d5409045ddc12d185a922b3c84b9e2a1bad merge: 100% passing, parallelized MuseHub test suite (task/… Sonnet 5.5 patch 3 days ago
sha256:8166b51ae53275831023a7ce334228b3ccc8237b55d68a35172389db9ff46ad5 test: 100% passing MuseHub suite, parallelized (xdist, one … Sonnet 5.5 patch 3 days ago
sha256:4c51ee348c9f4848b67c2553ad2375f244c210f999589abb2898c80f7b43a8ea docs(harmony): #250 Harmony QA contract v2 — correct /muse/… Opus 5.5 minor ⚠ 9 days ago