gabriel / musehub public
test_intel_topk_determinism.py python
129 lines 6.6 KB
Raw
sha256:b4b999434d042d74edc7032f1f2cfd20fda5b3580fe85966351db5b8eef0abb3 feat(intel): every grammar on the hub, and every intel job … Opus 5.5 minor ⚠ breaking 1 day ago
1 """Top-K truncation is a pure function of the data — musehub#8, Tier A review D2.
2
3 Run ONLY this file, against a private Postgres (never the live hub database)::
4
5 TEST_DATABASE_URL=postgresql+asyncpg://musehub:musehub@<private-host>:<port>/musehub_test \
6 python -P -m pytest tests/test_intel_topk_determinism.py
7
8 The defect: coupling kept the best 200 pairs and velocity the best 20 modules, sorted by ONE key (``co_changes`` / ``active_commits``).
9 Among candidates that tie on it, which ones survived the cut followed dict / set iteration order (database row order, and string-hash
10 randomisation per process), so re-running the same revision could publish a different leaderboard. Both now sort by the metric and
11 then by the candidate's own identity, so the survivors are the lexicographically first of the tied ones.
12
13 Each test has MORE tied candidates than the output limit, and asserts (1) the exact expected survivors, (2) the same answer on a repeat
14 run, and (3) the same answer when the history rows were written in a different order.
15 """
16 from __future__ import annotations
17
18 import pytest
19 import pytest_asyncio
20 from sqlalchemy import select
21
22 from musehub.db.musehub_intel_models import MusehubIntelCoupling, MusehubIntelVelocity
23 from musehub.services import musehub_intel_providers as P
24 from tests.factories import create_repo
25 from tests.test_intel_code_correctness import Dag, index_and_persist, ins, rep
26
27
28 @pytest_asyncio.fixture
29 async def session_factory(db_session, session_factory): # noqa: F811
30 return session_factory
31
32
33 async def build(session_factory, name: str, ops_per_commit: list[list[dict]], *, reverse: bool) -> tuple[str, str]:
34 """A repository whose linear history applies ``ops_per_commit`` (each list in the given or reversed order)."""
35 dag = Dag()
36 parent: list[str] = []
37 for i, ops in enumerate(ops_per_commit):
38 dag.add(f"{name}-{i}", parent, list(reversed(ops)) if reverse else ops, i * 10)
39 parent = [f"{name}-{i}"]
40 head = f"{name}-{len(ops_per_commit) - 1}"
41 async with session_factory() as s:
42 repo = await create_repo(s, owner="gabriel", slug=f"topk-{name}")
43 rid = repo.repo_id
44 await dag.install(s, rid, [f"{name}-{i}" for i in range(len(ops_per_commit))])
45 await index_and_persist(s, rid, dag.commits_id(head))
46 await s.commit()
47 return rid, dag.commits_id(head)
48
49
50 # Two groups of 20 files; the files of a group change together in two commits. Every in-group pair (190 per group, 380 in all) has
51 # exactly 2 co-changes, which is more tied pairs than the 200 kept. Which group's pairs the old code kept depended on which group's
52 # commits the database returned first.
53 GROUP_A = [f"a{i:02d}.py" for i in range(20)]
54 GROUP_B = [f"b{i:02d}.py" for i in range(20)]
55 A_COMMITS = [[ins(f, "x", f"{f}-0") for f in GROUP_A], [rep(f, "x", f"{f}-0", f"{f}-1") for f in GROUP_A]]
56 B_COMMITS = [[ins(f, "x", f"{f}-0") for f in GROUP_B], [rep(f, "x", f"{f}-0", f"{f}-1") for f in GROUP_B]]
57 ALL_PAIRS = sorted((a, b) for g in (GROUP_A, GROUP_B) for i, a in enumerate(g) for b in g[i + 1:])
58
59
60 async def coupling_pairs(session_factory, rid: str, head: str) -> list[tuple[str, str, int]]:
61 async with session_factory() as s:
62 await P.CouplingProvider().compute(s, rid, head, {})
63 await s.commit()
64 rows = (await s.execute(select(MusehubIntelCoupling).where(MusehubIntelCoupling.repo_id == rid))).scalars().all()
65 return sorted((r.file_a, r.file_b, r.co_changes) for r in rows)
66
67
68 @pytest.mark.asyncio
69 async def test_coupling_keeps_the_lexicographically_first_of_the_tied_pairs_whatever_the_row_order(session_factory) -> None:
70 limit = P.CouplingProvider._MAX_PAIRS
71 assert len(ALL_PAIRS) > limit, "the fixture must have more tied candidates than the output limit"
72 expected = [(a, b, 2) for a, b in ALL_PAIRS[:limit]]
73
74 forward_id, forward_head = await build(session_factory, "cf", A_COMMITS + B_COMMITS, reverse=False)
75 reverse_id, reverse_head = await build(session_factory, "cr", B_COMMITS + A_COMMITS, reverse=True) # other row order, other op order
76
77 first = await coupling_pairs(session_factory, forward_id, forward_head)
78 assert first == expected
79 assert await coupling_pairs(session_factory, forward_id, forward_head) == first # repeat run
80 assert await coupling_pairs(session_factory, reverse_id, reverse_head) == first # different history-row insertion order
81
82
83 # 30 directories each touched by one commit: every module has active_commits == 1 (> the 20 kept).
84 DIRS = [f"d{i:02d}" for i in range(30)]
85 VELOCITY_COMMITS = [[ins(f"{d}/m.py", "x", f"{d}-0") for d in DIRS]]
86
87
88 async def velocity_modules(session_factory, rid: str, head: str) -> list[str]:
89 async with session_factory() as s:
90 await P.VelocityProvider().compute(s, rid, head, {})
91 await s.commit()
92 rows = (await s.execute(select(MusehubIntelVelocity).where(MusehubIntelVelocity.repo_id == rid))).scalars().all()
93 return sorted(r.module for r in rows)
94
95
96 @pytest.mark.asyncio
97 async def test_velocity_keeps_the_lexicographically_first_of_the_tied_modules_whatever_the_row_order(session_factory) -> None:
98 limit = P.VelocityProvider._TOP
99 assert len(DIRS) > limit
100 expected = [f"{d}/" for d in DIRS[:limit]]
101
102 forward_id, forward_head = await build(session_factory, "vf", VELOCITY_COMMITS, reverse=False)
103 reverse_id, reverse_head = await build(session_factory, "vr", VELOCITY_COMMITS, reverse=True)
104
105 first = await velocity_modules(session_factory, forward_id, forward_head)
106 assert first == expected
107 assert await velocity_modules(session_factory, forward_id, forward_head) == first
108 assert await velocity_modules(session_factory, reverse_id, reverse_head) == first
109
110
111 @pytest.mark.parametrize("seed", [0, 1, 2, 3])
112 def test_the_coupling_cut_does_not_depend_on_the_order_the_pairs_were_counted_in(seed: int) -> None:
113 """The ranking itself, with the commit order varied directly (muse.plugins.code.analysis.coupled_files, shared with the CLI)."""
114 import random
115
116 from muse.plugins.code.analysis import coupled_files
117
118 commits = [{a, b} for a, b in ALL_PAIRS] * 2 # every pair co-changes twice
119 random.Random(seed).shuffle(commits)
120 got = coupled_files(commits, min_co_changes=2, limit=201)
121 assert len(got) == 201 # more than the 200 the job stores: truncated
122 assert [(a, b) for a, b, _ in got[:200]] == ALL_PAIRS[:200]
123
124
125 def test_the_coupling_cut_still_ranks_by_co_changes_first() -> None:
126 from muse.plugins.code.analysis import coupled_files
127
128 commits = [{"a", "b"}] * 2 + [{"a", "c"}] * 9 + [{"b", "c"}] * 5 + [{"c", "d"}]
129 assert coupled_files(commits, min_co_changes=2, limit=2) == [("a", "c", 9), ("b", "c", 5)]
File History 2 commits
sha256:b4b999434d042d74edc7032f1f2cfd20fda5b3580fe85966351db5b8eef0abb3 feat(intel): every grammar on the hub, and every intel job … Opus 5.5 minor ⚠ 1 day ago
sha256:8b2fa4a268caa2ba0bee56c58011540f61e76492d83e5e496264660efba6cf25 fix(intel): coupling and velocity top-K are a pure function… Sonnet 5.5 minor ⚠ 4 days ago