test_intel_topk_determinism.py
python
sha256:b4b999434d042d74edc7032f1f2cfd20fda5b3580fe85966351db5b8eef0abb3
feat(intel): every grammar on the hub, and every intel job …
Opus 5.5
minor
⚠ breaking
1 day ago
| 1 | """Top-K truncation is a pure function of the data — musehub#8, Tier A review D2. |
| 2 | |
| 3 | Run ONLY this file, against a private Postgres (never the live hub database):: |
| 4 | |
| 5 | TEST_DATABASE_URL=postgresql+asyncpg://musehub:musehub@<private-host>:<port>/musehub_test \ |
| 6 | python -P -m pytest tests/test_intel_topk_determinism.py |
| 7 | |
| 8 | The defect: coupling kept the best 200 pairs and velocity the best 20 modules, sorted by ONE key (``co_changes`` / ``active_commits``). |
| 9 | Among candidates that tie on it, which ones survived the cut followed dict / set iteration order (database row order, and string-hash |
| 10 | randomisation per process), so re-running the same revision could publish a different leaderboard. Both now sort by the metric and |
| 11 | then by the candidate's own identity, so the survivors are the lexicographically first of the tied ones. |
| 12 | |
| 13 | Each test has MORE tied candidates than the output limit, and asserts (1) the exact expected survivors, (2) the same answer on a repeat |
| 14 | run, and (3) the same answer when the history rows were written in a different order. |
| 15 | """ |
| 16 | from __future__ import annotations |
| 17 | |
| 18 | import pytest |
| 19 | import pytest_asyncio |
| 20 | from sqlalchemy import select |
| 21 | |
| 22 | from musehub.db.musehub_intel_models import MusehubIntelCoupling, MusehubIntelVelocity |
| 23 | from musehub.services import musehub_intel_providers as P |
| 24 | from tests.factories import create_repo |
| 25 | from tests.test_intel_code_correctness import Dag, index_and_persist, ins, rep |
| 26 | |
| 27 | |
| 28 | @pytest_asyncio.fixture |
| 29 | async def session_factory(db_session, session_factory): # noqa: F811 |
| 30 | return session_factory |
| 31 | |
| 32 | |
| 33 | async def build(session_factory, name: str, ops_per_commit: list[list[dict]], *, reverse: bool) -> tuple[str, str]: |
| 34 | """A repository whose linear history applies ``ops_per_commit`` (each list in the given or reversed order).""" |
| 35 | dag = Dag() |
| 36 | parent: list[str] = [] |
| 37 | for i, ops in enumerate(ops_per_commit): |
| 38 | dag.add(f"{name}-{i}", parent, list(reversed(ops)) if reverse else ops, i * 10) |
| 39 | parent = [f"{name}-{i}"] |
| 40 | head = f"{name}-{len(ops_per_commit) - 1}" |
| 41 | async with session_factory() as s: |
| 42 | repo = await create_repo(s, owner="gabriel", slug=f"topk-{name}") |
| 43 | rid = repo.repo_id |
| 44 | await dag.install(s, rid, [f"{name}-{i}" for i in range(len(ops_per_commit))]) |
| 45 | await index_and_persist(s, rid, dag.commits_id(head)) |
| 46 | await s.commit() |
| 47 | return rid, dag.commits_id(head) |
| 48 | |
| 49 | |
| 50 | # Two groups of 20 files; the files of a group change together in two commits. Every in-group pair (190 per group, 380 in all) has |
| 51 | # exactly 2 co-changes, which is more tied pairs than the 200 kept. Which group's pairs the old code kept depended on which group's |
| 52 | # commits the database returned first. |
| 53 | GROUP_A = [f"a{i:02d}.py" for i in range(20)] |
| 54 | GROUP_B = [f"b{i:02d}.py" for i in range(20)] |
| 55 | A_COMMITS = [[ins(f, "x", f"{f}-0") for f in GROUP_A], [rep(f, "x", f"{f}-0", f"{f}-1") for f in GROUP_A]] |
| 56 | B_COMMITS = [[ins(f, "x", f"{f}-0") for f in GROUP_B], [rep(f, "x", f"{f}-0", f"{f}-1") for f in GROUP_B]] |
| 57 | ALL_PAIRS = sorted((a, b) for g in (GROUP_A, GROUP_B) for i, a in enumerate(g) for b in g[i + 1:]) |
| 58 | |
| 59 | |
| 60 | async def coupling_pairs(session_factory, rid: str, head: str) -> list[tuple[str, str, int]]: |
| 61 | async with session_factory() as s: |
| 62 | await P.CouplingProvider().compute(s, rid, head, {}) |
| 63 | await s.commit() |
| 64 | rows = (await s.execute(select(MusehubIntelCoupling).where(MusehubIntelCoupling.repo_id == rid))).scalars().all() |
| 65 | return sorted((r.file_a, r.file_b, r.co_changes) for r in rows) |
| 66 | |
| 67 | |
| 68 | @pytest.mark.asyncio |
| 69 | async def test_coupling_keeps_the_lexicographically_first_of_the_tied_pairs_whatever_the_row_order(session_factory) -> None: |
| 70 | limit = P.CouplingProvider._MAX_PAIRS |
| 71 | assert len(ALL_PAIRS) > limit, "the fixture must have more tied candidates than the output limit" |
| 72 | expected = [(a, b, 2) for a, b in ALL_PAIRS[:limit]] |
| 73 | |
| 74 | forward_id, forward_head = await build(session_factory, "cf", A_COMMITS + B_COMMITS, reverse=False) |
| 75 | reverse_id, reverse_head = await build(session_factory, "cr", B_COMMITS + A_COMMITS, reverse=True) # other row order, other op order |
| 76 | |
| 77 | first = await coupling_pairs(session_factory, forward_id, forward_head) |
| 78 | assert first == expected |
| 79 | assert await coupling_pairs(session_factory, forward_id, forward_head) == first # repeat run |
| 80 | assert await coupling_pairs(session_factory, reverse_id, reverse_head) == first # different history-row insertion order |
| 81 | |
| 82 | |
| 83 | # 30 directories each touched by one commit: every module has active_commits == 1 (> the 20 kept). |
| 84 | DIRS = [f"d{i:02d}" for i in range(30)] |
| 85 | VELOCITY_COMMITS = [[ins(f"{d}/m.py", "x", f"{d}-0") for d in DIRS]] |
| 86 | |
| 87 | |
| 88 | async def velocity_modules(session_factory, rid: str, head: str) -> list[str]: |
| 89 | async with session_factory() as s: |
| 90 | await P.VelocityProvider().compute(s, rid, head, {}) |
| 91 | await s.commit() |
| 92 | rows = (await s.execute(select(MusehubIntelVelocity).where(MusehubIntelVelocity.repo_id == rid))).scalars().all() |
| 93 | return sorted(r.module for r in rows) |
| 94 | |
| 95 | |
| 96 | @pytest.mark.asyncio |
| 97 | async def test_velocity_keeps_the_lexicographically_first_of_the_tied_modules_whatever_the_row_order(session_factory) -> None: |
| 98 | limit = P.VelocityProvider._TOP |
| 99 | assert len(DIRS) > limit |
| 100 | expected = [f"{d}/" for d in DIRS[:limit]] |
| 101 | |
| 102 | forward_id, forward_head = await build(session_factory, "vf", VELOCITY_COMMITS, reverse=False) |
| 103 | reverse_id, reverse_head = await build(session_factory, "vr", VELOCITY_COMMITS, reverse=True) |
| 104 | |
| 105 | first = await velocity_modules(session_factory, forward_id, forward_head) |
| 106 | assert first == expected |
| 107 | assert await velocity_modules(session_factory, forward_id, forward_head) == first |
| 108 | assert await velocity_modules(session_factory, reverse_id, reverse_head) == first |
| 109 | |
| 110 | |
| 111 | @pytest.mark.parametrize("seed", [0, 1, 2, 3]) |
| 112 | def test_the_coupling_cut_does_not_depend_on_the_order_the_pairs_were_counted_in(seed: int) -> None: |
| 113 | """The ranking itself, with the commit order varied directly (muse.plugins.code.analysis.coupled_files, shared with the CLI).""" |
| 114 | import random |
| 115 | |
| 116 | from muse.plugins.code.analysis import coupled_files |
| 117 | |
| 118 | commits = [{a, b} for a, b in ALL_PAIRS] * 2 # every pair co-changes twice |
| 119 | random.Random(seed).shuffle(commits) |
| 120 | got = coupled_files(commits, min_co_changes=2, limit=201) |
| 121 | assert len(got) == 201 # more than the 200 the job stores: truncated |
| 122 | assert [(a, b) for a, b, _ in got[:200]] == ALL_PAIRS[:200] |
| 123 | |
| 124 | |
| 125 | def test_the_coupling_cut_still_ranks_by_co_changes_first() -> None: |
| 126 | from muse.plugins.code.analysis import coupled_files |
| 127 | |
| 128 | commits = [{"a", "b"}] * 2 + [{"a", "c"}] * 9 + [{"b", "c"}] * 5 + [{"c", "d"}] |
| 129 | assert coupled_files(commits, min_co_changes=2, limit=2) == [("a", "c", 9), ("b", "c", 5)] |
File History
2 commits
sha256:b4b999434d042d74edc7032f1f2cfd20fda5b3580fe85966351db5b8eef0abb3
feat(intel): every grammar on the hub, and every intel job …
Opus 5.5
minor
⚠
1 day ago
sha256:8b2fa4a268caa2ba0bee56c58011540f61e76492d83e5e496264660efba6cf25
fix(intel): coupling and velocity top-K are a pure function…
Sonnet 5.5
minor
⚠
4 days ago