test_intel_cli_differential.py
python
sha256:b4b999434d042d74edc7032f1f2cfd20fda5b3580fe85966351db5b8eef0abb3
feat(intel): every grammar on the hub, and every intel job …
Opus 5.5
minor
⚠ breaking
1 day ago
| 1 | """muse#251 — differential test: the hub's background jobs agree with the Muse CLI on the same repository. |
| 2 | |
| 3 | A repository is built with the real ``muse`` CLI (Python and Java, several commits with co-changes, an import cycle, a |
| 4 | duplicated function), then loaded into the hub database the way a push lands it: objects, snapshots, and commits carrying |
| 5 | their structured deltas. The hub's own indexer (``intel.code``) and every intel job run over it, and each stored result |
| 6 | is compared with the CLI's ``--json`` output for the same revision. |
| 7 | |
| 8 | DIFF_<job>: one test per job; the comparison names the fields both sides report. |
| 9 | """ |
| 10 | from __future__ import annotations |
| 11 | |
| 12 | import json |
| 13 | import pathlib |
| 14 | from datetime import timezone |
| 15 | |
| 16 | import msgpack |
| 17 | import pytest |
| 18 | import pytest_asyncio |
| 19 | from sqlalchemy import select |
| 20 | |
| 21 | from musehub.db.musehub_intel_models import ( |
| 22 | MusehubIntelApiSurface, |
| 23 | MusehubIntelBlastRisk, |
| 24 | MusehubIntelBreakageIssue, |
| 25 | MusehubIntelClones, |
| 26 | MusehubIntelRefactorEvent, |
| 27 | MusehubIntelStable, |
| 28 | MusehubIntelType, |
| 29 | MusehubIntelVelocity, |
| 30 | MusehubIntelCodemapMeta, |
| 31 | MusehubIntelCodemapModule, |
| 32 | MusehubIntelCoupling, |
| 33 | MusehubIntelDead, |
| 34 | MusehubIntelEntangle, |
| 35 | MusehubIntelLanguages, |
| 36 | MusehubSymbolIntel, |
| 37 | ) |
| 38 | from musehub.db.musehub_repo_models import MusehubCommit, MusehubCommitRef, MusehubObject, MusehubSnapshot |
| 39 | from musehub.services import musehub_intel_providers as P |
| 40 | from tests.docs_contract.test_intelligence import _cli, _env |
| 41 | from tests.factories import create_repo |
| 42 | from tests.test_intel_code_correctness import index_and_persist |
| 43 | |
| 44 | V1 = { |
| 45 | "app/core.py": "def helper():\n return 1\n\n\ndef unused():\n return 2\n\n\ndef twin():\n x = 1\n return x + 2\n", |
| 46 | "app/api.py": "from app.core import helper\nfrom app.cycle import ping\n\n\ndef route():\n return helper() + ping()\n", |
| 47 | "app/cycle.py": "from app.api import route\n\n\ndef ping():\n return 0\n\n\ndef pong():\n return route()\n", |
| 48 | "app/copy.py": "def twin2():\n x = 1\n return x + 2\n", |
| 49 | "tests/test_core.py": "from app.core import helper\n\n\ndef test_helper():\n assert helper() == 1\n", |
| 50 | "src/bank/Ledger.java": "package bank;\npublic class Ledger {\n public void post(int a) { System.out.println(a); }\n}\n", |
| 51 | "src/bank/Account.java": "package bank;\nimport bank.Ledger;\npublic class Account {\n private final Ledger l = new Ledger();\n public int deposit(int x) { l.post(x); return x; }\n}\n", |
| 52 | "src/bank/App.java": "package bank;\npublic class App {\n public static void main(String[] a) { new Account().deposit(1); }\n}\n", |
| 53 | "app/stale.py": "from app.gone import vanished\n\n\ndef stale():\n return vanished()\n", |
| 54 | } |
| 55 | MASS = 60 # files in each of two "mass" commits: more than the CLI's 50-file coupling guard |
| 56 | |
| 57 | |
| 58 | def _edit(files: dict[str, str], i: int) -> dict[str, str]: |
| 59 | """A commit that changes helper, route and Account.deposit together (co-change across three files).""" |
| 60 | out = dict(files) |
| 61 | out["app/core.py"] = out["app/core.py"].replace(" return 1\n", f" return {i}\n", 1) |
| 62 | out["app/api.py"] = out["app/api.py"].replace("helper() + ping()", f"helper() + ping() + {i}") |
| 63 | out["src/bank/Account.java"] = out["src/bank/Account.java"].replace("return x;", f"return x + {i};") |
| 64 | return out |
| 65 | |
| 66 | |
| 67 | @pytest_asyncio.fixture |
| 68 | async def session_factory(db_session, session_factory): # noqa: F811 |
| 69 | return session_factory |
| 70 | |
| 71 | |
| 72 | def _write(root: pathlib.Path, files: dict[str, str]) -> None: |
| 73 | for p, s in files.items(): |
| 74 | (root / p).parent.mkdir(parents=True, exist_ok=True) |
| 75 | (root / p).write_text(s) |
| 76 | |
| 77 | |
| 78 | @pytest.fixture(scope="module") |
| 79 | def cli_repo(tmp_path_factory: pytest.TempPathFactory) -> tuple[pathlib.Path, dict[str, str]]: |
| 80 | tmp = tmp_path_factory.mktemp("diff") |
| 81 | root = tmp / "repo" |
| 82 | root.mkdir() |
| 83 | env = _env(tmp) |
| 84 | assert _cli(root, env, ("init",))["rc"] == 0 |
| 85 | files = V1 |
| 86 | for i in range(1, 5): |
| 87 | files = files if i == 1 else _edit(files, i) |
| 88 | _write(root, files) |
| 89 | assert _cli(root, env, ("code", "add", "."))["rc"] == 0 |
| 90 | assert _cli(root, env, ("commit", "-m", f"c{i}"))["rc"] == 0, "commit failed" |
| 91 | for i in (5, 6): # two mass commits over the same 60 files: coupling noise both sides must ignore |
| 92 | _write(root, {f"gen/g{k}.py": f"def g{k}():\n return {i}\n" for k in range(MASS)}) |
| 93 | assert _cli(root, env, ("code", "add", "."))["rc"] == 0 |
| 94 | assert _cli(root, env, ("commit", "-m", f"mass{i}"))["rc"] == 0 |
| 95 | return root, env |
| 96 | |
| 97 | |
| 98 | def _json(root: pathlib.Path, env: dict[str, str], *args: str) -> dict: |
| 99 | out = _cli(root, env, ("code", *args, "--json")) |
| 100 | assert out["rc"] == 0, out["out"] |
| 101 | return json.loads(out["stdout"].strip().splitlines()[-1]) |
| 102 | |
| 103 | |
| 104 | async def _load(s, root: pathlib.Path, slug: str) -> tuple[str, str]: |
| 105 | """Load the CLI repository into the hub tables the way a push lands it; run the hub's indexer. ``(repo_id, head)``.""" |
| 106 | from muse.core.commits import resolve_commit_ref |
| 107 | from muse.core.object_store import read_object |
| 108 | from muse.core.refs import read_current_branch |
| 109 | from muse.core.snapshots import get_commit_snapshot_manifest |
| 110 | from muse.plugins.code._query import walk_commits_bfs |
| 111 | |
| 112 | repo = await create_repo(s, owner="gabriel", slug=slug) |
| 113 | head = resolve_commit_ref(root, read_current_branch(root), None) |
| 114 | assert head is not None |
| 115 | commits, _ = walk_commits_bfs(root, head.commit_id, 10_000) |
| 116 | for c in reversed(commits): |
| 117 | manifest = get_commit_snapshot_manifest(root, c.commit_id) or {} |
| 118 | for oid in manifest.values(): |
| 119 | if await s.get(MusehubObject, oid) is None: |
| 120 | raw = read_object(root, oid) |
| 121 | assert raw is not None |
| 122 | s.add(MusehubObject(object_id=oid, path="x", size_bytes=len(raw), storage_uri="pending", content_cache=raw)) |
| 123 | if await s.get(MusehubSnapshot, c.snapshot_id) is None: |
| 124 | s.add(MusehubSnapshot(snapshot_id=c.snapshot_id, manifest_blob=msgpack.packb(manifest), directories=[], entry_count=len(manifest))) |
| 125 | await s.flush() |
| 126 | parents = c.parent_commit_ids or ([c.parent_commit_id] if c.parent_commit_id else []) |
| 127 | s.add(MusehubCommit(commit_id=c.commit_id, message=c.message, author=c.author or "gabriel", branch="main", |
| 128 | parent_ids=parents, snapshot_id=c.snapshot_id, structured_delta=c.structured_delta, |
| 129 | timestamp=c.committed_at.astimezone(timezone.utc))) |
| 130 | s.add(MusehubCommitRef(repo_id=repo.repo_id, commit_id=c.commit_id)) |
| 131 | await s.flush() |
| 132 | await index_and_persist(s, repo.repo_id, head.commit_id) |
| 133 | await s.commit() |
| 134 | return repo.repo_id, head.commit_id |
| 135 | |
| 136 | |
| 137 | async def _rows(s, model, repo_id: str) -> list: |
| 138 | await s.rollback() |
| 139 | return list((await s.execute(select(model).where(model.repo_id == repo_id))).scalars()) |
| 140 | |
| 141 | |
| 142 | # ── call graph ────────────────────────────────────────────────────────────── |
| 143 | |
| 144 | |
| 145 | @pytest.mark.asyncio |
| 146 | async def test_DIFF_gravity(session_factory, cli_repo) -> None: |
| 147 | root, env = cli_repo |
| 148 | cli = {r["address"]: r for r in _json(root, env, "gravity", "--top", "0")["symbols"]} |
| 149 | async with session_factory() as s: |
| 150 | rid, head = await _load(s, root, "diff-grav") |
| 151 | await P.GravityProvider().compute(s, rid, head, {}) |
| 152 | await s.commit() |
| 153 | hub = {r.address: r for r in await _rows(s, MusehubSymbolIntel, rid) if r.gravity_pct is not None} |
| 154 | assert set(hub) == set(cli) |
| 155 | for a, c in cli.items(): |
| 156 | h = hub[a] |
| 157 | assert (h.gravity_pct, h.gravity_direct_dependents, h.gravity_transitive_dependents, h.gravity_max_depth, |
| 158 | h.gravity_depth_distribution) == (c["gravity_pct"], c["direct_dependents"], c["transitive_dependents"], |
| 159 | c["max_depth"], c["depth_distribution"]), a |
| 160 | |
| 161 | |
| 162 | @pytest.mark.asyncio |
| 163 | async def test_DIFF_dead(session_factory, cli_repo) -> None: |
| 164 | root, env = cli_repo |
| 165 | cli = {r["address"]: r for r in _json(root, env, "dead", "--commit", "HEAD")["results"]} |
| 166 | async with session_factory() as s: |
| 167 | rid, head = await _load(s, root, "diff-dead") |
| 168 | await P.DeadProvider().compute(s, rid, head, {}) |
| 169 | await s.commit() |
| 170 | hub = {r.address: r for r in await _rows(s, MusehubIntelDead, rid)} |
| 171 | assert set(hub) == set(cli) and "app/core.py::unused" in hub |
| 172 | for a, c in cli.items(): |
| 173 | assert (hub[a].confidence, hub[a].reason, hub[a].kind) == (c["confidence"], c["reason"], c["kind"]), a |
| 174 | |
| 175 | |
| 176 | @pytest.mark.asyncio |
| 177 | async def test_DIFF_blast_risk(session_factory, cli_repo) -> None: |
| 178 | root, env = cli_repo |
| 179 | cli = {r["address"]: r for r in _json(root, env, "blast-risk", "--top", "10000")["symbols"]} |
| 180 | async with session_factory() as s: |
| 181 | rid, head = await _load(s, root, "diff-risk") |
| 182 | await P.BlastRiskProvider().compute(s, rid, head, {}) |
| 183 | await s.commit() |
| 184 | hub = {r.address: r for r in await _rows(s, MusehubIntelBlastRisk, rid)} |
| 185 | assert set(hub) == set(cli) |
| 186 | for a, c in cli.items(): |
| 187 | h = hub[a] |
| 188 | assert (h.risk_score, h.risk) == (c["risk"], c["risk_label"]), a |
| 189 | assert (round(h.impact_score * 100), round(h.churn_score * 100), round(h.test_gap_score * 100), round(h.coupling_score * 100)) == ( |
| 190 | c["impact_score"], c["churn_score"], c["test_gap_score"], c["coupling_score"]), a |
| 191 | |
| 192 | |
| 193 | @pytest.mark.asyncio |
| 194 | async def test_DIFF_entangle(session_factory, cli_repo) -> None: |
| 195 | root, env = cli_repo |
| 196 | cli = _json(root, env, "entangle", "--top", "1000")["pairs"] |
| 197 | async with session_factory() as s: |
| 198 | rid, head = await _load(s, root, "diff-ent") |
| 199 | await P.EntangleProvider().compute(s, rid, head, {}) |
| 200 | await s.commit() |
| 201 | hub = await _rows(s, MusehubIntelEntangle, rid) |
| 202 | # The hub stores the best EntangleProvider._MAX_PAIRS pairs: the CLI's list, in its order, cut at that size. |
| 203 | top = cli[: P.EntangleProvider._MAX_PAIRS] |
| 204 | want = sorted(((p["symbol_a"], p["symbol_b"]), p["co_changes"], p["commits_both_active"], p["co_change_rate"], p["structurally_linked"]) for p in top) |
| 205 | got = sorted(((r.symbol_a, r.symbol_b), r.co_changes, r.commits_both_active, r.co_change_rate, r.structurally_linked) for r in hub) |
| 206 | assert len(cli) > P.EntangleProvider._MAX_PAIRS and got == want |
| 207 | |
| 208 | |
| 209 | @pytest.mark.asyncio |
| 210 | async def test_DIFF_codemap(session_factory, cli_repo) -> None: |
| 211 | root, env = cli_repo |
| 212 | cli = _json(root, env, "codemap") |
| 213 | async with session_factory() as s: |
| 214 | rid, head = await _load(s, root, "diff-map") |
| 215 | await P.CodemapProvider().compute(s, rid, head, {}) |
| 216 | await s.commit() |
| 217 | mods = {r.file_path: (r.fan_in, r.fan_out) for r in await _rows(s, MusehubIntelCodemapModule, rid)} |
| 218 | meta = [(m.cycles_json, m.cycle_count) for m in await _rows(s, MusehubIntelCodemapMeta, rid)] |
| 219 | for m in cli["modules"]: |
| 220 | assert mods[m["file"]] == (m["importers"], m["imports"]), m["file"] |
| 221 | assert meta == [(cli["import_cycles"], len(cli["import_cycles"]))] and len(cli["import_cycles"]) == 1 |
| 222 | |
| 223 | |
| 224 | # ── history ───────────────────────────────────────────────────────────────── |
| 225 | |
| 226 | |
| 227 | @pytest.mark.asyncio |
| 228 | async def test_DIFF_coupling(session_factory, cli_repo) -> None: |
| 229 | root, env = cli_repo |
| 230 | cli = _json(root, env, "coupling", "--top", "1000")["pairs"] |
| 231 | async with session_factory() as s: |
| 232 | rid, head = await _load(s, root, "diff-coup") |
| 233 | await P.CouplingProvider().compute(s, rid, head, {}) |
| 234 | await s.commit() |
| 235 | hub = await _rows(s, MusehubIntelCoupling, rid) |
| 236 | assert cli and sorted((r.file_a, r.file_b, r.co_changes) for r in hub) == sorted((p["file_a"], p["file_b"], p["co_changes"]) for p in cli) |
| 237 | |
| 238 | |
| 239 | # ── snapshot ──────────────────────────────────────────────────────────────── |
| 240 | |
| 241 | |
| 242 | @pytest.mark.asyncio |
| 243 | async def test_DIFF_api_surface_and_languages(session_factory, cli_repo) -> None: |
| 244 | root, env = cli_repo |
| 245 | api = {r["address"] for r in _json(root, env, "api-surface")["results"]} |
| 246 | langs = {r["language"]: r for r in _json(root, env, "languages")["languages"]} |
| 247 | async with session_factory() as s: |
| 248 | rid, head = await _load(s, root, "diff-api") |
| 249 | await P.ApiSurfaceProvider().compute(s, rid, head, {}) |
| 250 | await P.LanguagesProvider().compute(s, rid, head, {}) |
| 251 | await s.commit() |
| 252 | hub_api = {r.address for r in await _rows(s, MusehubIntelApiSurface, rid)} |
| 253 | hub_langs = {r.language: r for r in await _rows(s, MusehubIntelLanguages, rid)} |
| 254 | assert hub_api == api |
| 255 | for name in ("Python", "Java"): |
| 256 | assert (hub_langs[name].file_count, hub_langs[name].symbol_count) == (langs[name]["files"], langs[name]["symbols"]), name |
| 257 | |
| 258 | |
| 259 | @pytest.mark.asyncio |
| 260 | async def test_DIFF_velocity(session_factory, cli_repo) -> None: |
| 261 | root, env = cli_repo |
| 262 | cli = {m["module"]: m for m in _json(root, env, "velocity", "--top", "1000")["modules"]} |
| 263 | async with session_factory() as s: |
| 264 | rid, head = await _load(s, root, "diff-vel") |
| 265 | await P.VelocityProvider().compute(s, rid, head, {}) |
| 266 | await s.commit() |
| 267 | hub = {r.module: (r.added, r.removed, r.net, r.modified, r.active_commits, r.prior_added, r.prior_net, r.prior_modified, |
| 268 | r.prior_active_commits, r.acceleration, r.stagnant_commits) for r in await _rows(s, MusehubIntelVelocity, rid)} |
| 269 | want = {m: (c["current"]["added"], c["current"]["removed"], c["current"]["net"], c["current"]["modified"], c["current"]["active_commits"], |
| 270 | c["prior"]["added"], c["prior"]["net"], c["prior"]["modified"], c["prior"]["active_commits"], c["acceleration"], |
| 271 | c["stagnant_commits"]) for m, c in cli.items()} |
| 272 | assert hub == want |
| 273 | |
| 274 | |
| 275 | @pytest.mark.asyncio |
| 276 | async def test_DIFF_clones(session_factory, cli_repo) -> None: |
| 277 | root, env = cli_repo |
| 278 | cli = _json(root, env, "clones")["clusters"] |
| 279 | async with session_factory() as s: |
| 280 | rid, head = await _load(s, root, "diff-clone") |
| 281 | await P.ClonesProvider().compute(s, rid, head, {}) |
| 282 | await s.commit() |
| 283 | hub = [(r.tier, r.member_count, sorted(m["address"] for m in json.loads(r.members_json))) for r in await _rows(s, MusehubIntelClones, rid)] |
| 284 | want = [(c["tier"], c["count"], sorted(m["address"] for m in c["members"])) for c in cli if c["tier"] == "exact"] |
| 285 | assert want and sorted(hub) == sorted(want) |
| 286 | |
| 287 | |
| 288 | @pytest.mark.asyncio |
| 289 | async def test_DIFF_stable(session_factory, cli_repo) -> None: |
| 290 | root, env = cli_repo |
| 291 | cli = {r["address"]: r for r in _json(root, env, "stable", "--top", "10000")["stable"]} |
| 292 | async with session_factory() as s: |
| 293 | rid, head = await _load(s, root, "diff-stable") |
| 294 | await P.StableProvider().compute(s, rid, head, {}) |
| 295 | await s.commit() |
| 296 | hub = {r.address: (r.unchanged_for, r.since_start) for r in await _rows(s, MusehubIntelStable, rid)} |
| 297 | assert hub == {a: (c["unchanged_for"], c["since_start_of_range"]) for a, c in cli.items()} |
| 298 | |
| 299 | |
| 300 | @pytest.mark.asyncio |
| 301 | async def test_DIFF_type(session_factory, cli_repo) -> None: |
| 302 | root, env = cli_repo |
| 303 | cli = {r["address"]: r for r in _json(root, env, "type")["symbols"]} |
| 304 | async with session_factory() as s: |
| 305 | rid, head = await _load(s, root, "diff-type") |
| 306 | await P.TypeProvider().compute(s, rid, head, {}) |
| 307 | await s.commit() |
| 308 | hub = {r.address: (r.kind, r.params_total, r.params_annotated, r.params_with_any, r.return_is_any, r.type_score) |
| 309 | for r in await _rows(s, MusehubIntelType, rid)} |
| 310 | assert hub == {a: (c["kind"], c["params_total"], c["params_annotated"], c["params_with_any"], c["return_is_any"], c["type_score"]) |
| 311 | for a, c in cli.items()} |
| 312 | |
| 313 | |
| 314 | @pytest.mark.asyncio |
| 315 | async def test_DIFF_breakage(session_factory, cli_repo) -> None: |
| 316 | root, env = cli_repo |
| 317 | cli = _json(root, env, "breakage")["issues"] |
| 318 | async with session_factory() as s: |
| 319 | rid, head = await _load(s, root, "diff-break") |
| 320 | await P.BreakageProvider().compute(s, rid, head, {}) |
| 321 | await s.commit() |
| 322 | hub = sorted((r.file_path, r.issue_type) for r in await _rows(s, MusehubIntelBreakageIssue, rid)) |
| 323 | assert cli and hub == sorted((i["path"], i["issue_type"]) for i in cli) |
| 324 | |
| 325 | |
| 326 | @pytest.mark.asyncio |
| 327 | async def test_DIFF_detect_refactor(session_factory, cli_repo) -> None: |
| 328 | root, env = cli_repo |
| 329 | cli = _json(root, env, "detect-refactor")["events"] |
| 330 | async with session_factory() as s: |
| 331 | rid, head = await _load(s, root, "diff-refactor") |
| 332 | await P.DetectRefactorProvider().compute(s, rid, head, {}) |
| 333 | await s.commit() |
| 334 | hub = sorted((r.kind, r.address, r.commit_id) for r in await _rows(s, MusehubIntelRefactorEvent, rid)) |
| 335 | assert cli and hub == sorted((e["kind"], e["address"], e["commit_id"]) for e in cli) |
File History
1 commit
sha256:b4b999434d042d74edc7032f1f2cfd20fda5b3580fe85966351db5b8eef0abb3
feat(intel): every grammar on the hub, and every intel job …
Opus 5.5
minor
⚠
1 day ago