test_intel_parse_cache.py
python
sha256:cb45801289825e65d4aea6c6dc8be73fe32a49bb997b1da5f9d3ac9d54712c91
perf(intel): share symbol-parse results across snapshot pro…
Sonnet 5.5
minor
⚠ breaking
4 days ago
| 1 | """Contract for the shared symbol-parse results — musehub#8 release gate R2 (performance), second bottleneck. |
| 2 | |
| 3 | After the source-object cache, five snapshot providers still each ran the pinned engine's ``parse_symbols`` over the same bytes: ~4-5 s of a |
| 4 | 300-commit run per provider, 80% of it ``ast.unparse`` for content hashes. The result is a pure function of (object bytes, path), so it is shared — |
| 5 | as a read-only view, never as a mutable structure two providers could corrupt for each other. |
| 6 | |
| 7 | Run ONLY this file, against a private Postgres (never the live hub database):: |
| 8 | |
| 9 | TEST_DATABASE_URL=postgresql+asyncpg://musehub:musehub@<private-host>:<port>/musehub_test python -P -m pytest tests/test_intel_parse_cache.py |
| 10 | |
| 11 | T1 A second provider reading the same object does not parse it again. |
| 12 | T2 The shared tree and its records are read-only: a provider cannot mutate what another provider will see. |
| 13 | T3 The cache is bounded by record count (least recently used first). |
| 14 | T4 A file that fails to parse is counted per provider and never cached. |
| 15 | T5 The key includes the path: the same bytes at two paths are two parses (record keys embed the path). |
| 16 | T6 A cached tree equals a fresh parse, record for record. |
| 17 | """ |
| 18 | from __future__ import annotations |
| 19 | |
| 20 | import pytest |
| 21 | import pytest_asyncio |
| 22 | |
| 23 | from musehub.services import musehub_intel_providers as P |
| 24 | from tests.factories import create_repo |
| 25 | from tests.test_intel_snapshot_providers import A_V1, B_SRC, World |
| 26 | |
| 27 | |
| 28 | @pytest_asyncio.fixture |
| 29 | async def session_factory(db_session, session_factory): # noqa: F811 |
| 30 | return session_factory |
| 31 | |
| 32 | |
| 33 | @pytest.fixture(autouse=True) |
| 34 | def fresh_cache(): |
| 35 | P.clear_source_cache() |
| 36 | P.clear_parse_cache() |
| 37 | yield |
| 38 | P.clear_parse_cache() |
| 39 | |
| 40 | |
| 41 | class ParseSpy: |
| 42 | def __init__(self, monkeypatch) -> None: |
| 43 | self.calls: list[str] = [] |
| 44 | real = P.parse_symbols |
| 45 | |
| 46 | def spy(src, path): |
| 47 | self.calls.append(path) |
| 48 | return real(src, path) |
| 49 | monkeypatch.setattr(P, "parse_symbols", spy) |
| 50 | |
| 51 | |
| 52 | async def read_trees(s, items): |
| 53 | stats = P._ReadStats() |
| 54 | out = {path: tree async for path, tree in P._read_trees(s, items, stats)} |
| 55 | return out, stats |
| 56 | |
| 57 | |
| 58 | @pytest.mark.asyncio |
| 59 | async def test_t1_a_second_reader_does_not_parse_the_same_object_again(session_factory, monkeypatch) -> None: |
| 60 | spy = ParseSpy(monkeypatch) |
| 61 | async with session_factory() as s: |
| 62 | repo = await create_repo(s, owner="gabriel", slug="parse1") |
| 63 | w = World(s, repo.repo_id) |
| 64 | a = await w.obj(A_V1) |
| 65 | first, _ = await read_trees(s, [("pkg/a.py", a)]) |
| 66 | second, _ = await read_trees(s, [("pkg/a.py", a)]) |
| 67 | assert spy.calls == ["pkg/a.py"] |
| 68 | assert first["pkg/a.py"] == second["pkg/a.py"] |
| 69 | stats = P.parse_cache_stats() |
| 70 | assert (stats["hits"], stats["misses"]) == (1, 1) |
| 71 | |
| 72 | |
| 73 | @pytest.mark.asyncio |
| 74 | async def test_t2_the_shared_tree_and_its_records_cannot_be_mutated(session_factory) -> None: |
| 75 | async with session_factory() as s: |
| 76 | repo = await create_repo(s, owner="gabriel", slug="parse2") |
| 77 | w = World(s, repo.repo_id) |
| 78 | a = await w.obj(A_V1) |
| 79 | trees, _ = await read_trees(s, [("pkg/a.py", a)]) |
| 80 | tree = trees["pkg/a.py"] |
| 81 | assert tree # public_fn, Thing, Thing.method ... |
| 82 | with pytest.raises(TypeError): |
| 83 | tree["injected"] = {} # type: ignore[index] |
| 84 | record = next(iter(tree.values())) |
| 85 | with pytest.raises(TypeError): |
| 86 | record["kind"] = "tampered" # type: ignore[index] |
| 87 | assert record["kind"] != "tampered" |
| 88 | |
| 89 | |
| 90 | @pytest.mark.asyncio |
| 91 | async def test_t3_the_cache_is_bounded_by_record_count_and_evicts_least_recently_used(session_factory, monkeypatch) -> None: |
| 92 | async with session_factory() as s: |
| 93 | repo = await create_repo(s, owner="gabriel", slug="parse3") |
| 94 | w = World(s, repo.repo_id) |
| 95 | srcs = [f"def f{i}_a():\n return 1\n\n\ndef f{i}_b():\n return 2\n" for i in range(6)] # 2 records each |
| 96 | ids = [await w.obj(src) for src in srcs] |
| 97 | monkeypatch.setattr(P, "_PARSE_CACHE", P._ParseCache(max_records=10)) |
| 98 | for i in range(5): |
| 99 | await read_trees(s, [(f"m{i}.py", ids[i])]) # 10 records: full, nothing evicted |
| 100 | assert P.parse_cache_stats()["records"] == 10 and P.parse_cache_stats()["evictions"] == 0 |
| 101 | await read_trees(s, [("m0.py", ids[0])]) # m0 most recently used; m1 least |
| 102 | await read_trees(s, [("m5.py", ids[5])]) # 12 > 10: m1 goes |
| 103 | stats = P.parse_cache_stats() |
| 104 | assert stats["records"] <= 10 and stats["evictions"] == 1 |
| 105 | spy = ParseSpy(monkeypatch) |
| 106 | await read_trees(s, [("m0.py", ids[0]), ("m2.py", ids[2]), ("m5.py", ids[5])]) |
| 107 | assert spy.calls == [] |
| 108 | await read_trees(s, [("m1.py", ids[1])]) |
| 109 | assert spy.calls == ["m1.py"] |
| 110 | |
| 111 | |
| 112 | @pytest.mark.asyncio |
| 113 | async def test_t4_a_parse_failure_is_counted_for_every_reader_and_never_cached(session_factory, monkeypatch) -> None: |
| 114 | calls = {"n": 0} |
| 115 | |
| 116 | def boom(src, path): |
| 117 | calls["n"] += 1 |
| 118 | raise ValueError("unparseable") |
| 119 | monkeypatch.setattr(P, "parse_symbols", boom) |
| 120 | async with session_factory() as s: |
| 121 | repo = await create_repo(s, owner="gabriel", slug="parse4") |
| 122 | w = World(s, repo.repo_id) |
| 123 | a = await w.obj(A_V1) |
| 124 | out1, st1 = await read_trees(s, [("pkg/a.py", a)]) |
| 125 | out2, st2 = await read_trees(s, [("pkg/a.py", a)]) |
| 126 | assert out1 == out2 == {} |
| 127 | assert st1.parse_failures == st2.parse_failures == 1 |
| 128 | assert calls["n"] == 2 |
| 129 | assert P.parse_cache_stats()["records"] == 0 |
| 130 | |
| 131 | |
| 132 | @pytest.mark.asyncio |
| 133 | async def test_t5_the_same_bytes_at_two_paths_are_parsed_separately(session_factory, monkeypatch) -> None: |
| 134 | spy = ParseSpy(monkeypatch) |
| 135 | async with session_factory() as s: |
| 136 | repo = await create_repo(s, owner="gabriel", slug="parse5") |
| 137 | w = World(s, repo.repo_id) |
| 138 | a = await w.obj(A_V1) |
| 139 | out, _ = await read_trees(s, [("one/a.py", a), ("two/a.py", a)]) |
| 140 | assert spy.calls == ["one/a.py", "two/a.py"] |
| 141 | keys_one = {r["canonical_key"] for r in out["one/a.py"].values()} |
| 142 | keys_two = {r["canonical_key"] for r in out["two/a.py"].values()} |
| 143 | assert keys_one and keys_one.isdisjoint(keys_two) |
| 144 | |
| 145 | |
| 146 | @pytest.mark.asyncio |
| 147 | async def test_t6_a_cached_tree_equals_a_fresh_parse_record_for_record(session_factory) -> None: |
| 148 | async with session_factory() as s: |
| 149 | repo = await create_repo(s, owner="gabriel", slug="parse6") |
| 150 | w = World(s, repo.repo_id) |
| 151 | b = await w.obj(B_SRC) |
| 152 | await read_trees(s, [("pkg/b.py", b)]) |
| 153 | cached, _ = await read_trees(s, [("pkg/b.py", b)]) |
| 154 | fresh = P.parse_symbols(B_SRC.encode(), "pkg/b.py") |
| 155 | assert {a: dict(r) for a, r in cached["pkg/b.py"].items()} == {a: dict(r) for a, r in fresh.items()} |
File History
1 commit
sha256:cb45801289825e65d4aea6c6dc8be73fe32a49bb997b1da5f9d3ac9d54712c91
perf(intel): share symbol-parse results across snapshot pro…
Sonnet 5.5
minor
⚠
4 days ago