gabriel / musehub public
test_intel_source_cache.py python
274 lines 15.0 KB
Raw
sha256:bdc0664ea517f0343fee86e1998df5d9ee2d833bb76e4ce43f85d9f8fb8dd621 perf(intel): share bounded, immutable source-object reads a… Sonnet 5.5 patch 4 days ago
1 """Contract for the shared, bounded snapshot-object read layer — musehub#8 release gate R2 (performance + object-size protection).
2
3 Six snapshot providers read and parse the same repository objects; 100 commits took 49.8 s and 300 commits exceeded the 60 s limit.
4 Objects are content-addressed and immutable, so their decompressed bytes can be shared between providers — within a hard memory bound.
5
6 Run ONLY this file, against a private Postgres (never the live hub database)::
7
8 TEST_DATABASE_URL=postgresql+asyncpg://musehub:musehub@<private-host>:<port>/musehub_test python -P -m pytest tests/test_intel_source_cache.py
9
10 S1 A second read of an object is served from memory: the storage layer is touched once per object, across providers.
11 S2 The cache is bounded: least-recently-used entries are evicted and the byte budget is never exceeded.
12 S3 A failed read is never cached (the next attempt reads again) and an unreadable object still fails the job — fail closed.
13 S4 A zero-byte file is valid content in the cache (``b""`` is a hit, not a miss and not "unreadable").
14 S5 Cache hits never bypass the object row: a deleted row is unreadable even when its bytes are cached.
15 S6 The recorded size is an early rejection guard (no storage read for an oversize object), and NOT the only protection: a small
16 compressed object that expands past the limit is rejected by a bounded decompression, never fully materialised.
17 S7 Revision changes do not contaminate: a warm cache never shows a provider the wrong revision's content.
18 """
19 from __future__ import annotations
20
21 import hashlib
22 import zlib
23
24 import pytest
25 import pytest_asyncio
26 from sqlalchemy import select
27
28 from musehub.db.musehub_intel_models import MusehubIntelType
29 from musehub.db.musehub_repo_models import MusehubObject
30 from musehub.services import musehub_intel_providers as P
31 from musehub.types.compression import DecompressedTooLarge, decompress_bounded
32 from tests.factories import create_repo
33 from tests.test_intel_snapshot_providers import A_V1, B_SRC, World, world_with_side_branch
34
35
36 @pytest_asyncio.fixture
37 async def session_factory(db_session, session_factory): # noqa: F811
38 return session_factory
39
40
41 @pytest.fixture(autouse=True)
42 def fresh_cache():
43 P.clear_source_cache()
44 yield
45 P.clear_source_cache()
46
47
48 class StorageSpy:
49 """Counts calls into the storage layer — the object-store reads a cache hit must avoid."""
50
51 def __init__(self, monkeypatch) -> None:
52 from musehub.storage import backends
53 self.calls: list[str] = []
54 real = backends.read_object_bytes
55
56 async def read(obj, session=None):
57 self.calls.append(obj.object_id)
58 return await real(obj, session=session)
59 monkeypatch.setattr(backends, "read_object_bytes", read)
60
61
62 async def add_object(s, content: bytes, *, size_bytes: int | None = None, oid: str | None = None) -> str:
63 oid = oid or hashlib.sha256(content).hexdigest()
64 s.add(MusehubObject(object_id=oid, path="x", size_bytes=len(content) if size_bytes is None else size_bytes,
65 storage_uri="pending", content_cache=content))
66 await s.flush()
67 return oid
68
69
70 # ── S1 ──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
71
72 @pytest.mark.asyncio
73 async def test_s1_a_second_read_of_an_object_does_not_touch_storage(session_factory, monkeypatch) -> None:
74 spy = StorageSpy(monkeypatch)
75 async with session_factory() as s:
76 oid = await add_object(s, b"def f():\n return 1\n")
77 first = await P._read_source(s, oid)
78 second = await P._read_source(s, oid)
79 assert first == second == b"def f():\n return 1\n"
80 assert spy.calls == [oid]
81 stats = P.source_cache_stats()
82 assert (stats["hits"], stats["misses"]) == (1, 1)
83
84
85 @pytest.mark.asyncio
86 async def test_s1_snapshot_providers_share_one_read_per_object(session_factory, monkeypatch) -> None:
87 spy = StorageSpy(monkeypatch)
88 async with session_factory() as s:
89 w, c2, _side = await world_with_side_branch(s, "share1")
90 for provider in (P.TypeProvider(), P.ApiSurfaceProvider(), P.LanguagesProvider(), P.CodemapProvider(), P.BreakageProvider()):
91 await provider.compute(s, w.repo_id, c2, {})
92 assert len(spy.calls) == len(set(spy.calls)) == 2 # pkg/a.py and pkg/b.py, once each — not once per provider
93 assert P.source_cache_stats()["hits"] == 2 * 4
94
95
96 # ── S2 ──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
97
98 @pytest.mark.asyncio
99 async def test_s2_the_cache_evicts_least_recently_used_and_never_exceeds_its_budget(session_factory, monkeypatch) -> None:
100 monkeypatch.setattr(P, "_SOURCE_CACHE", P._SourceCache(max_bytes=1000))
101 spy = StorageSpy(monkeypatch)
102 async with session_factory() as s:
103 a, b, c, d, e = [await add_object(s, bytes([65 + i]) * 240) for i in range(5)]
104 for oid in (a, b, c, d):
105 await P._read_source(s, oid) # 960 B: all fit
106 await P._read_source(s, a) # a is now the most recently used; b is the least
107 await P._read_source(s, e) # 1200 B > 1000 B: b goes
108 assert P.source_cache_stats()["bytes"] <= 1000
109 assert P.source_cache_stats()["evictions"] == 1
110 before = len(spy.calls)
111 for oid in (a, c, d, e):
112 await P._read_source(s, oid)
113 assert len(spy.calls) == before # all four survived
114 await P._read_source(s, b)
115 assert len(spy.calls) == before + 1 # b was evicted and is re-read
116
117
118 @pytest.mark.asyncio
119 async def test_s2_an_object_larger_than_a_quarter_of_the_budget_is_not_cached(session_factory, monkeypatch) -> None:
120 monkeypatch.setattr(P, "_SOURCE_CACHE", P._SourceCache(max_bytes=400))
121 async with session_factory() as s:
122 small = await add_object(s, b"s" * 50)
123 big = await add_object(s, b"B" * 300) # would evict everything for one entry
124 await P._read_source(s, small)
125 await P._read_source(s, big)
126 await P._read_source(s, big)
127 assert P.source_cache_stats()["bytes"] == 50
128 assert P.source_cache_stats()["evictions"] == 0
129
130
131 # ── S3 ──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
132
133 @pytest.mark.asyncio
134 async def test_s3_a_failed_read_is_not_cached_and_the_next_attempt_reads_again(session_factory, monkeypatch) -> None:
135 from musehub.storage import backends
136 real = backends.read_object_bytes
137 attempts = {"n": 0}
138
139 async def flaky(obj, session=None):
140 attempts["n"] += 1
141 return None if attempts["n"] == 1 else await real(obj, session=session)
142 monkeypatch.setattr(backends, "read_object_bytes", flaky)
143 async with session_factory() as s:
144 oid = await add_object(s, b"x = 1\n")
145 assert await P._read_source(s, oid) is None
146 assert await P._read_source(s, oid) == b"x = 1\n"
147 assert attempts["n"] == 2
148
149
150 @pytest.mark.asyncio
151 async def test_s3_an_unreadable_object_still_fails_the_provider_with_a_warm_cache(session_factory, monkeypatch) -> None:
152 async with session_factory() as s:
153 w, c2, _ = await world_with_side_branch(s, "unreadable1")
154 await P.TypeProvider().compute(s, w.repo_id, c2, {}) # warm the cache
155 from musehub.storage import backends
156 real = backends.read_object_bytes
157 broken = B_SRC.encode()
158
159 async def read(obj, session=None):
160 return None if obj.object_id == hashlib.sha256(A_V1.encode()).hexdigest() else await real(obj, session=session)
161 monkeypatch.setattr(backends, "read_object_bytes", read)
162 P.clear_source_cache()
163 with pytest.raises(P.UnreadableObjects):
164 await P.TypeProvider().compute(s, w.repo_id, c2, {})
165 assert broken # (b is readable, a is not: the job fails rather than publishing an analysis of less)
166
167
168 # ── S4 ──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
169
170 @pytest.mark.asyncio
171 async def test_s4_a_zero_byte_file_is_a_cache_hit_not_unreadable(session_factory, monkeypatch) -> None:
172 spy = StorageSpy(monkeypatch)
173 async with session_factory() as s:
174 oid = await add_object(s, b"")
175 assert await P._read_source(s, oid) == b""
176 assert await P._read_source(s, oid) == b""
177 assert spy.calls == [oid]
178 assert P.source_cache_stats()["hits"] == 1
179
180
181 # ── S5 ──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
182
183 @pytest.mark.asyncio
184 async def test_s5_a_cached_object_whose_row_is_gone_is_unreadable(session_factory) -> None:
185 async with session_factory() as s:
186 oid = await add_object(s, b"y = 2\n")
187 assert await P._read_source(s, oid) == b"y = 2\n"
188 await s.delete(await s.get(MusehubObject, oid))
189 await s.flush()
190 assert await P._read_source(s, oid) is None
191
192
193 # ── S6 ──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
194
195 @pytest.mark.asyncio
196 async def test_s6_the_recorded_size_rejects_an_oversize_object_before_any_read(session_factory, monkeypatch) -> None:
197 spy = StorageSpy(monkeypatch)
198 async with session_factory() as s:
199 oid = await add_object(s, b"tiny", size_bytes=P._MAX_SOURCE_BYTES + 1)
200 with pytest.raises(P.OversizeObject):
201 await P._read_source(s, oid)
202 assert spy.calls == []
203
204
205 @pytest.mark.asyncio
206 async def test_s6_a_decompression_bomb_with_a_small_recorded_size_is_rejected_bounded(session_factory) -> None:
207 bomb = zlib.compress(b"a" * (P._MAX_SOURCE_BYTES * 40))
208 assert len(bomb) < 100_000 # tiny on the wire, 40 MiB expanded
209 async with session_factory() as s:
210 oid = await add_object(s, bomb, size_bytes=len(bomb))
211 with pytest.raises(P.OversizeObject):
212 await P._read_source(s, oid)
213 assert P.source_cache_stats()["bytes"] == 0
214
215
216 def test_s6_decompress_bounded_stops_at_the_limit_for_zlib_and_passes_small_and_plain_data() -> None:
217 assert decompress_bounded(zlib.compress(b"hello"), 100) == b"hello"
218 assert decompress_bounded(b"plain text, no magic", 100) == b"plain text, no magic"
219 assert decompress_bounded(b"", 100) == b""
220 with pytest.raises(DecompressedTooLarge):
221 decompress_bounded(zlib.compress(b"a" * 10_000), 100)
222 with pytest.raises(DecompressedTooLarge):
223 decompress_bounded(b"p" * 101, 100) # an uncompressed object over the limit is oversize too
224
225
226 def test_s6_decompress_bounded_handles_zstd_and_a_corrupt_stream_is_treated_as_plain_bytes() -> None:
227 import zstandard
228 assert decompress_bounded(zstandard.ZstdCompressor().compress(b"z" * 50), 100) == b"z" * 50
229 with pytest.raises(DecompressedTooLarge):
230 decompress_bounded(zstandard.ZstdCompressor().compress(b"z" * 10_000), 100)
231 corrupt = b"x^2 is a plain text file that merely starts like a zlib header"
232 assert decompress_bounded(corrupt, 1000) == corrupt
233
234
235 @pytest.mark.asyncio
236 async def test_s6_an_oversize_file_is_counted_not_silently_dropped_and_the_job_still_completes(session_factory) -> None:
237 async with session_factory() as s:
238 repo = await create_repo(s, owner="gabriel", slug="oversize-count")
239 w = World(s, repo.repo_id)
240 big = await add_object(s, b"#" * 10, size_bytes=P._MAX_SOURCE_BYTES + 5, oid=hashlib.sha256(b"big").hexdigest())
241 a = await w.obj(A_V1)
242 import msgpack
243 from musehub.db.musehub_repo_models import MusehubCommit, MusehubCommitRef, MusehubSnapshot
244 from tests.test_intel_snapshot_providers import BASE, h
245 s.add(MusehubSnapshot(snapshot_id=h("snap:o"), manifest_blob=msgpack.packb({"gen/big.py": big, "pkg/a.py": a}), directories=[], entry_count=2))
246 await s.flush()
247 s.add(MusehubCommit(commit_id=h("commit:o"), message="o", author="gabriel", branch="main", parent_ids=[], snapshot_id=h("snap:o"), timestamp=BASE))
248 s.add(MusehubCommitRef(repo_id=repo.repo_id, commit_id=h("commit:o")))
249 await s.commit()
250 out = await P.TypeProvider().compute(s, w.repo_id, h("commit:o"), {})
251 assert out[0][1]["files_oversize"] == 1 and out[0][1]["files_read"] == 1
252
253
254 # ── S7 ──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
255
256 @pytest.mark.asyncio
257 async def test_s7_a_warm_cache_never_shows_a_provider_another_revisions_content(session_factory) -> None:
258 v1 = "def only_in_v1(x: int) -> int:\n return x\n"
259 v2 = "def only_in_v2(x: int) -> int:\n return x\n"
260 async with session_factory() as s:
261 repo = await create_repo(s, owner="gabriel", slug="rev-cache")
262 w = World(s, repo.repo_id)
263 c1 = await w.commit("c1", {"pkg/m.py": v1}, minute=0)
264 c2 = await w.commit("c2", {"pkg/m.py": v2}, ["c1"], minute=5)
265 await s.commit()
266 await P.TypeProvider().compute(s, w.repo_id, c2, {})
267 await s.commit()
268 rows2 = {r.address for r in (await s.execute(select(MusehubIntelType).where(MusehubIntelType.repo_id == w.repo_id))).scalars()}
269 out1 = await P.TypeProvider().compute(s, w.repo_id, c1, {})
270 assert out1
271 await s.commit()
272 rows1 = {r.address for r in (await s.execute(select(MusehubIntelType).where(MusehubIntelType.repo_id == w.repo_id))).scalars()}
273 assert "pkg/m.py::only_in_v2" in rows2 and "pkg/m.py::only_in_v1" not in rows2
274 assert "pkg/m.py::only_in_v1" in rows1 and "pkg/m.py::only_in_v2" not in rows1
File History 1 commit
sha256:bdc0664ea517f0343fee86e1998df5d9ee2d833bb76e4ce43f85d9f8fb8dd621 perf(intel): share bounded, immutable source-object reads a… Sonnet 5.5 patch 4 days ago