gabriel / musehub public
test_intel_cli_differential.py python
335 lines 16.7 KB
Raw
sha256:b4b999434d042d74edc7032f1f2cfd20fda5b3580fe85966351db5b8eef0abb3 feat(intel): every grammar on the hub, and every intel job … Opus 5.5 minor ⚠ breaking 1 day ago
1 """muse#251 — differential test: the hub's background jobs agree with the Muse CLI on the same repository.
2
3 A repository is built with the real ``muse`` CLI (Python and Java, several commits with co-changes, an import cycle, a
4 duplicated function), then loaded into the hub database the way a push lands it: objects, snapshots, and commits carrying
5 their structured deltas. The hub's own indexer (``intel.code``) and every intel job run over it, and each stored result
6 is compared with the CLI's ``--json`` output for the same revision.
7
8 DIFF_<job>: one test per job; the comparison names the fields both sides report.
9 """
10 from __future__ import annotations
11
12 import json
13 import pathlib
14 from datetime import timezone
15
16 import msgpack
17 import pytest
18 import pytest_asyncio
19 from sqlalchemy import select
20
21 from musehub.db.musehub_intel_models import (
22 MusehubIntelApiSurface,
23 MusehubIntelBlastRisk,
24 MusehubIntelBreakageIssue,
25 MusehubIntelClones,
26 MusehubIntelRefactorEvent,
27 MusehubIntelStable,
28 MusehubIntelType,
29 MusehubIntelVelocity,
30 MusehubIntelCodemapMeta,
31 MusehubIntelCodemapModule,
32 MusehubIntelCoupling,
33 MusehubIntelDead,
34 MusehubIntelEntangle,
35 MusehubIntelLanguages,
36 MusehubSymbolIntel,
37 )
38 from musehub.db.musehub_repo_models import MusehubCommit, MusehubCommitRef, MusehubObject, MusehubSnapshot
39 from musehub.services import musehub_intel_providers as P
40 from tests.docs_contract.test_intelligence import _cli, _env
41 from tests.factories import create_repo
42 from tests.test_intel_code_correctness import index_and_persist
43
44 V1 = {
45 "app/core.py": "def helper():\n return 1\n\n\ndef unused():\n return 2\n\n\ndef twin():\n x = 1\n return x + 2\n",
46 "app/api.py": "from app.core import helper\nfrom app.cycle import ping\n\n\ndef route():\n return helper() + ping()\n",
47 "app/cycle.py": "from app.api import route\n\n\ndef ping():\n return 0\n\n\ndef pong():\n return route()\n",
48 "app/copy.py": "def twin2():\n x = 1\n return x + 2\n",
49 "tests/test_core.py": "from app.core import helper\n\n\ndef test_helper():\n assert helper() == 1\n",
50 "src/bank/Ledger.java": "package bank;\npublic class Ledger {\n public void post(int a) { System.out.println(a); }\n}\n",
51 "src/bank/Account.java": "package bank;\nimport bank.Ledger;\npublic class Account {\n private final Ledger l = new Ledger();\n public int deposit(int x) { l.post(x); return x; }\n}\n",
52 "src/bank/App.java": "package bank;\npublic class App {\n public static void main(String[] a) { new Account().deposit(1); }\n}\n",
53 "app/stale.py": "from app.gone import vanished\n\n\ndef stale():\n return vanished()\n",
54 }
55 MASS = 60 # files in each of two "mass" commits: more than the CLI's 50-file coupling guard
56
57
58 def _edit(files: dict[str, str], i: int) -> dict[str, str]:
59 """A commit that changes helper, route and Account.deposit together (co-change across three files)."""
60 out = dict(files)
61 out["app/core.py"] = out["app/core.py"].replace(" return 1\n", f" return {i}\n", 1)
62 out["app/api.py"] = out["app/api.py"].replace("helper() + ping()", f"helper() + ping() + {i}")
63 out["src/bank/Account.java"] = out["src/bank/Account.java"].replace("return x;", f"return x + {i};")
64 return out
65
66
67 @pytest_asyncio.fixture
68 async def session_factory(db_session, session_factory): # noqa: F811
69 return session_factory
70
71
72 def _write(root: pathlib.Path, files: dict[str, str]) -> None:
73 for p, s in files.items():
74 (root / p).parent.mkdir(parents=True, exist_ok=True)
75 (root / p).write_text(s)
76
77
78 @pytest.fixture(scope="module")
79 def cli_repo(tmp_path_factory: pytest.TempPathFactory) -> tuple[pathlib.Path, dict[str, str]]:
80 tmp = tmp_path_factory.mktemp("diff")
81 root = tmp / "repo"
82 root.mkdir()
83 env = _env(tmp)
84 assert _cli(root, env, ("init",))["rc"] == 0
85 files = V1
86 for i in range(1, 5):
87 files = files if i == 1 else _edit(files, i)
88 _write(root, files)
89 assert _cli(root, env, ("code", "add", "."))["rc"] == 0
90 assert _cli(root, env, ("commit", "-m", f"c{i}"))["rc"] == 0, "commit failed"
91 for i in (5, 6): # two mass commits over the same 60 files: coupling noise both sides must ignore
92 _write(root, {f"gen/g{k}.py": f"def g{k}():\n return {i}\n" for k in range(MASS)})
93 assert _cli(root, env, ("code", "add", "."))["rc"] == 0
94 assert _cli(root, env, ("commit", "-m", f"mass{i}"))["rc"] == 0
95 return root, env
96
97
98 def _json(root: pathlib.Path, env: dict[str, str], *args: str) -> dict:
99 out = _cli(root, env, ("code", *args, "--json"))
100 assert out["rc"] == 0, out["out"]
101 return json.loads(out["stdout"].strip().splitlines()[-1])
102
103
104 async def _load(s, root: pathlib.Path, slug: str) -> tuple[str, str]:
105 """Load the CLI repository into the hub tables the way a push lands it; run the hub's indexer. ``(repo_id, head)``."""
106 from muse.core.commits import resolve_commit_ref
107 from muse.core.object_store import read_object
108 from muse.core.refs import read_current_branch
109 from muse.core.snapshots import get_commit_snapshot_manifest
110 from muse.plugins.code._query import walk_commits_bfs
111
112 repo = await create_repo(s, owner="gabriel", slug=slug)
113 head = resolve_commit_ref(root, read_current_branch(root), None)
114 assert head is not None
115 commits, _ = walk_commits_bfs(root, head.commit_id, 10_000)
116 for c in reversed(commits):
117 manifest = get_commit_snapshot_manifest(root, c.commit_id) or {}
118 for oid in manifest.values():
119 if await s.get(MusehubObject, oid) is None:
120 raw = read_object(root, oid)
121 assert raw is not None
122 s.add(MusehubObject(object_id=oid, path="x", size_bytes=len(raw), storage_uri="pending", content_cache=raw))
123 if await s.get(MusehubSnapshot, c.snapshot_id) is None:
124 s.add(MusehubSnapshot(snapshot_id=c.snapshot_id, manifest_blob=msgpack.packb(manifest), directories=[], entry_count=len(manifest)))
125 await s.flush()
126 parents = c.parent_commit_ids or ([c.parent_commit_id] if c.parent_commit_id else [])
127 s.add(MusehubCommit(commit_id=c.commit_id, message=c.message, author=c.author or "gabriel", branch="main",
128 parent_ids=parents, snapshot_id=c.snapshot_id, structured_delta=c.structured_delta,
129 timestamp=c.committed_at.astimezone(timezone.utc)))
130 s.add(MusehubCommitRef(repo_id=repo.repo_id, commit_id=c.commit_id))
131 await s.flush()
132 await index_and_persist(s, repo.repo_id, head.commit_id)
133 await s.commit()
134 return repo.repo_id, head.commit_id
135
136
137 async def _rows(s, model, repo_id: str) -> list:
138 await s.rollback()
139 return list((await s.execute(select(model).where(model.repo_id == repo_id))).scalars())
140
141
142 # ── call graph ──────────────────────────────────────────────────────────────
143
144
145 @pytest.mark.asyncio
146 async def test_DIFF_gravity(session_factory, cli_repo) -> None:
147 root, env = cli_repo
148 cli = {r["address"]: r for r in _json(root, env, "gravity", "--top", "0")["symbols"]}
149 async with session_factory() as s:
150 rid, head = await _load(s, root, "diff-grav")
151 await P.GravityProvider().compute(s, rid, head, {})
152 await s.commit()
153 hub = {r.address: r for r in await _rows(s, MusehubSymbolIntel, rid) if r.gravity_pct is not None}
154 assert set(hub) == set(cli)
155 for a, c in cli.items():
156 h = hub[a]
157 assert (h.gravity_pct, h.gravity_direct_dependents, h.gravity_transitive_dependents, h.gravity_max_depth,
158 h.gravity_depth_distribution) == (c["gravity_pct"], c["direct_dependents"], c["transitive_dependents"],
159 c["max_depth"], c["depth_distribution"]), a
160
161
162 @pytest.mark.asyncio
163 async def test_DIFF_dead(session_factory, cli_repo) -> None:
164 root, env = cli_repo
165 cli = {r["address"]: r for r in _json(root, env, "dead", "--commit", "HEAD")["results"]}
166 async with session_factory() as s:
167 rid, head = await _load(s, root, "diff-dead")
168 await P.DeadProvider().compute(s, rid, head, {})
169 await s.commit()
170 hub = {r.address: r for r in await _rows(s, MusehubIntelDead, rid)}
171 assert set(hub) == set(cli) and "app/core.py::unused" in hub
172 for a, c in cli.items():
173 assert (hub[a].confidence, hub[a].reason, hub[a].kind) == (c["confidence"], c["reason"], c["kind"]), a
174
175
176 @pytest.mark.asyncio
177 async def test_DIFF_blast_risk(session_factory, cli_repo) -> None:
178 root, env = cli_repo
179 cli = {r["address"]: r for r in _json(root, env, "blast-risk", "--top", "10000")["symbols"]}
180 async with session_factory() as s:
181 rid, head = await _load(s, root, "diff-risk")
182 await P.BlastRiskProvider().compute(s, rid, head, {})
183 await s.commit()
184 hub = {r.address: r for r in await _rows(s, MusehubIntelBlastRisk, rid)}
185 assert set(hub) == set(cli)
186 for a, c in cli.items():
187 h = hub[a]
188 assert (h.risk_score, h.risk) == (c["risk"], c["risk_label"]), a
189 assert (round(h.impact_score * 100), round(h.churn_score * 100), round(h.test_gap_score * 100), round(h.coupling_score * 100)) == (
190 c["impact_score"], c["churn_score"], c["test_gap_score"], c["coupling_score"]), a
191
192
193 @pytest.mark.asyncio
194 async def test_DIFF_entangle(session_factory, cli_repo) -> None:
195 root, env = cli_repo
196 cli = _json(root, env, "entangle", "--top", "1000")["pairs"]
197 async with session_factory() as s:
198 rid, head = await _load(s, root, "diff-ent")
199 await P.EntangleProvider().compute(s, rid, head, {})
200 await s.commit()
201 hub = await _rows(s, MusehubIntelEntangle, rid)
202 # The hub stores the best EntangleProvider._MAX_PAIRS pairs: the CLI's list, in its order, cut at that size.
203 top = cli[: P.EntangleProvider._MAX_PAIRS]
204 want = sorted(((p["symbol_a"], p["symbol_b"]), p["co_changes"], p["commits_both_active"], p["co_change_rate"], p["structurally_linked"]) for p in top)
205 got = sorted(((r.symbol_a, r.symbol_b), r.co_changes, r.commits_both_active, r.co_change_rate, r.structurally_linked) for r in hub)
206 assert len(cli) > P.EntangleProvider._MAX_PAIRS and got == want
207
208
209 @pytest.mark.asyncio
210 async def test_DIFF_codemap(session_factory, cli_repo) -> None:
211 root, env = cli_repo
212 cli = _json(root, env, "codemap")
213 async with session_factory() as s:
214 rid, head = await _load(s, root, "diff-map")
215 await P.CodemapProvider().compute(s, rid, head, {})
216 await s.commit()
217 mods = {r.file_path: (r.fan_in, r.fan_out) for r in await _rows(s, MusehubIntelCodemapModule, rid)}
218 meta = [(m.cycles_json, m.cycle_count) for m in await _rows(s, MusehubIntelCodemapMeta, rid)]
219 for m in cli["modules"]:
220 assert mods[m["file"]] == (m["importers"], m["imports"]), m["file"]
221 assert meta == [(cli["import_cycles"], len(cli["import_cycles"]))] and len(cli["import_cycles"]) == 1
222
223
224 # ── history ─────────────────────────────────────────────────────────────────
225
226
227 @pytest.mark.asyncio
228 async def test_DIFF_coupling(session_factory, cli_repo) -> None:
229 root, env = cli_repo
230 cli = _json(root, env, "coupling", "--top", "1000")["pairs"]
231 async with session_factory() as s:
232 rid, head = await _load(s, root, "diff-coup")
233 await P.CouplingProvider().compute(s, rid, head, {})
234 await s.commit()
235 hub = await _rows(s, MusehubIntelCoupling, rid)
236 assert cli and sorted((r.file_a, r.file_b, r.co_changes) for r in hub) == sorted((p["file_a"], p["file_b"], p["co_changes"]) for p in cli)
237
238
239 # ── snapshot ────────────────────────────────────────────────────────────────
240
241
242 @pytest.mark.asyncio
243 async def test_DIFF_api_surface_and_languages(session_factory, cli_repo) -> None:
244 root, env = cli_repo
245 api = {r["address"] for r in _json(root, env, "api-surface")["results"]}
246 langs = {r["language"]: r for r in _json(root, env, "languages")["languages"]}
247 async with session_factory() as s:
248 rid, head = await _load(s, root, "diff-api")
249 await P.ApiSurfaceProvider().compute(s, rid, head, {})
250 await P.LanguagesProvider().compute(s, rid, head, {})
251 await s.commit()
252 hub_api = {r.address for r in await _rows(s, MusehubIntelApiSurface, rid)}
253 hub_langs = {r.language: r for r in await _rows(s, MusehubIntelLanguages, rid)}
254 assert hub_api == api
255 for name in ("Python", "Java"):
256 assert (hub_langs[name].file_count, hub_langs[name].symbol_count) == (langs[name]["files"], langs[name]["symbols"]), name
257
258
259 @pytest.mark.asyncio
260 async def test_DIFF_velocity(session_factory, cli_repo) -> None:
261 root, env = cli_repo
262 cli = {m["module"]: m for m in _json(root, env, "velocity", "--top", "1000")["modules"]}
263 async with session_factory() as s:
264 rid, head = await _load(s, root, "diff-vel")
265 await P.VelocityProvider().compute(s, rid, head, {})
266 await s.commit()
267 hub = {r.module: (r.added, r.removed, r.net, r.modified, r.active_commits, r.prior_added, r.prior_net, r.prior_modified,
268 r.prior_active_commits, r.acceleration, r.stagnant_commits) for r in await _rows(s, MusehubIntelVelocity, rid)}
269 want = {m: (c["current"]["added"], c["current"]["removed"], c["current"]["net"], c["current"]["modified"], c["current"]["active_commits"],
270 c["prior"]["added"], c["prior"]["net"], c["prior"]["modified"], c["prior"]["active_commits"], c["acceleration"],
271 c["stagnant_commits"]) for m, c in cli.items()}
272 assert hub == want
273
274
275 @pytest.mark.asyncio
276 async def test_DIFF_clones(session_factory, cli_repo) -> None:
277 root, env = cli_repo
278 cli = _json(root, env, "clones")["clusters"]
279 async with session_factory() as s:
280 rid, head = await _load(s, root, "diff-clone")
281 await P.ClonesProvider().compute(s, rid, head, {})
282 await s.commit()
283 hub = [(r.tier, r.member_count, sorted(m["address"] for m in json.loads(r.members_json))) for r in await _rows(s, MusehubIntelClones, rid)]
284 want = [(c["tier"], c["count"], sorted(m["address"] for m in c["members"])) for c in cli if c["tier"] == "exact"]
285 assert want and sorted(hub) == sorted(want)
286
287
288 @pytest.mark.asyncio
289 async def test_DIFF_stable(session_factory, cli_repo) -> None:
290 root, env = cli_repo
291 cli = {r["address"]: r for r in _json(root, env, "stable", "--top", "10000")["stable"]}
292 async with session_factory() as s:
293 rid, head = await _load(s, root, "diff-stable")
294 await P.StableProvider().compute(s, rid, head, {})
295 await s.commit()
296 hub = {r.address: (r.unchanged_for, r.since_start) for r in await _rows(s, MusehubIntelStable, rid)}
297 assert hub == {a: (c["unchanged_for"], c["since_start_of_range"]) for a, c in cli.items()}
298
299
300 @pytest.mark.asyncio
301 async def test_DIFF_type(session_factory, cli_repo) -> None:
302 root, env = cli_repo
303 cli = {r["address"]: r for r in _json(root, env, "type")["symbols"]}
304 async with session_factory() as s:
305 rid, head = await _load(s, root, "diff-type")
306 await P.TypeProvider().compute(s, rid, head, {})
307 await s.commit()
308 hub = {r.address: (r.kind, r.params_total, r.params_annotated, r.params_with_any, r.return_is_any, r.type_score)
309 for r in await _rows(s, MusehubIntelType, rid)}
310 assert hub == {a: (c["kind"], c["params_total"], c["params_annotated"], c["params_with_any"], c["return_is_any"], c["type_score"])
311 for a, c in cli.items()}
312
313
314 @pytest.mark.asyncio
315 async def test_DIFF_breakage(session_factory, cli_repo) -> None:
316 root, env = cli_repo
317 cli = _json(root, env, "breakage")["issues"]
318 async with session_factory() as s:
319 rid, head = await _load(s, root, "diff-break")
320 await P.BreakageProvider().compute(s, rid, head, {})
321 await s.commit()
322 hub = sorted((r.file_path, r.issue_type) for r in await _rows(s, MusehubIntelBreakageIssue, rid))
323 assert cli and hub == sorted((i["path"], i["issue_type"]) for i in cli)
324
325
326 @pytest.mark.asyncio
327 async def test_DIFF_detect_refactor(session_factory, cli_repo) -> None:
328 root, env = cli_repo
329 cli = _json(root, env, "detect-refactor")["events"]
330 async with session_factory() as s:
331 rid, head = await _load(s, root, "diff-refactor")
332 await P.DetectRefactorProvider().compute(s, rid, head, {})
333 await s.commit()
334 hub = sorted((r.kind, r.address, r.commit_id) for r in await _rows(s, MusehubIntelRefactorEvent, rid))
335 assert cli and hub == sorted((e["kind"], e["address"], e["commit_id"]) for e in cli)
File History 1 commit
sha256:b4b999434d042d74edc7032f1f2cfd20fda5b3580fe85966351db5b8eef0abb3 feat(intel): every grammar on the hub, and every intel job … Opus 5.5 minor ⚠ 1 day ago