test_intel_source_cache.py
python
sha256:bdc0664ea517f0343fee86e1998df5d9ee2d833bb76e4ce43f85d9f8fb8dd621
perf(intel): share bounded, immutable source-object reads a…
Sonnet 5.5
patch
4 days ago
| 1 | """Contract for the shared, bounded snapshot-object read layer — musehub#8 release gate R2 (performance + object-size protection). |
| 2 | |
| 3 | Six snapshot providers read and parse the same repository objects; 100 commits took 49.8 s and 300 commits exceeded the 60 s limit. |
| 4 | Objects are content-addressed and immutable, so their decompressed bytes can be shared between providers — within a hard memory bound. |
| 5 | |
| 6 | Run ONLY this file, against a private Postgres (never the live hub database):: |
| 7 | |
| 8 | TEST_DATABASE_URL=postgresql+asyncpg://musehub:musehub@<private-host>:<port>/musehub_test python -P -m pytest tests/test_intel_source_cache.py |
| 9 | |
| 10 | S1 A second read of an object is served from memory: the storage layer is touched once per object, across providers. |
| 11 | S2 The cache is bounded: least-recently-used entries are evicted and the byte budget is never exceeded. |
| 12 | S3 A failed read is never cached (the next attempt reads again) and an unreadable object still fails the job — fail closed. |
| 13 | S4 A zero-byte file is valid content in the cache (``b""`` is a hit, not a miss and not "unreadable"). |
| 14 | S5 Cache hits never bypass the object row: a deleted row is unreadable even when its bytes are cached. |
| 15 | S6 The recorded size is an early rejection guard (no storage read for an oversize object), and NOT the only protection: a small |
| 16 | compressed object that expands past the limit is rejected by a bounded decompression, never fully materialised. |
| 17 | S7 Revision changes do not contaminate: a warm cache never shows a provider the wrong revision's content. |
| 18 | """ |
| 19 | from __future__ import annotations |
| 20 | |
| 21 | import hashlib |
| 22 | import zlib |
| 23 | |
| 24 | import pytest |
| 25 | import pytest_asyncio |
| 26 | from sqlalchemy import select |
| 27 | |
| 28 | from musehub.db.musehub_intel_models import MusehubIntelType |
| 29 | from musehub.db.musehub_repo_models import MusehubObject |
| 30 | from musehub.services import musehub_intel_providers as P |
| 31 | from musehub.types.compression import DecompressedTooLarge, decompress_bounded |
| 32 | from tests.factories import create_repo |
| 33 | from tests.test_intel_snapshot_providers import A_V1, B_SRC, World, world_with_side_branch |
| 34 | |
| 35 | |
| 36 | @pytest_asyncio.fixture |
| 37 | async def session_factory(db_session, session_factory): # noqa: F811 |
| 38 | return session_factory |
| 39 | |
| 40 | |
| 41 | @pytest.fixture(autouse=True) |
| 42 | def fresh_cache(): |
| 43 | P.clear_source_cache() |
| 44 | yield |
| 45 | P.clear_source_cache() |
| 46 | |
| 47 | |
| 48 | class StorageSpy: |
| 49 | """Counts calls into the storage layer — the object-store reads a cache hit must avoid.""" |
| 50 | |
| 51 | def __init__(self, monkeypatch) -> None: |
| 52 | from musehub.storage import backends |
| 53 | self.calls: list[str] = [] |
| 54 | real = backends.read_object_bytes |
| 55 | |
| 56 | async def read(obj, session=None): |
| 57 | self.calls.append(obj.object_id) |
| 58 | return await real(obj, session=session) |
| 59 | monkeypatch.setattr(backends, "read_object_bytes", read) |
| 60 | |
| 61 | |
| 62 | async def add_object(s, content: bytes, *, size_bytes: int | None = None, oid: str | None = None) -> str: |
| 63 | oid = oid or hashlib.sha256(content).hexdigest() |
| 64 | s.add(MusehubObject(object_id=oid, path="x", size_bytes=len(content) if size_bytes is None else size_bytes, |
| 65 | storage_uri="pending", content_cache=content)) |
| 66 | await s.flush() |
| 67 | return oid |
| 68 | |
| 69 | |
| 70 | # ── S1 ────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────── |
| 71 | |
| 72 | @pytest.mark.asyncio |
| 73 | async def test_s1_a_second_read_of_an_object_does_not_touch_storage(session_factory, monkeypatch) -> None: |
| 74 | spy = StorageSpy(monkeypatch) |
| 75 | async with session_factory() as s: |
| 76 | oid = await add_object(s, b"def f():\n return 1\n") |
| 77 | first = await P._read_source(s, oid) |
| 78 | second = await P._read_source(s, oid) |
| 79 | assert first == second == b"def f():\n return 1\n" |
| 80 | assert spy.calls == [oid] |
| 81 | stats = P.source_cache_stats() |
| 82 | assert (stats["hits"], stats["misses"]) == (1, 1) |
| 83 | |
| 84 | |
| 85 | @pytest.mark.asyncio |
| 86 | async def test_s1_snapshot_providers_share_one_read_per_object(session_factory, monkeypatch) -> None: |
| 87 | spy = StorageSpy(monkeypatch) |
| 88 | async with session_factory() as s: |
| 89 | w, c2, _side = await world_with_side_branch(s, "share1") |
| 90 | for provider in (P.TypeProvider(), P.ApiSurfaceProvider(), P.LanguagesProvider(), P.CodemapProvider(), P.BreakageProvider()): |
| 91 | await provider.compute(s, w.repo_id, c2, {}) |
| 92 | assert len(spy.calls) == len(set(spy.calls)) == 2 # pkg/a.py and pkg/b.py, once each — not once per provider |
| 93 | assert P.source_cache_stats()["hits"] == 2 * 4 |
| 94 | |
| 95 | |
| 96 | # ── S2 ────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────── |
| 97 | |
| 98 | @pytest.mark.asyncio |
| 99 | async def test_s2_the_cache_evicts_least_recently_used_and_never_exceeds_its_budget(session_factory, monkeypatch) -> None: |
| 100 | monkeypatch.setattr(P, "_SOURCE_CACHE", P._SourceCache(max_bytes=1000)) |
| 101 | spy = StorageSpy(monkeypatch) |
| 102 | async with session_factory() as s: |
| 103 | a, b, c, d, e = [await add_object(s, bytes([65 + i]) * 240) for i in range(5)] |
| 104 | for oid in (a, b, c, d): |
| 105 | await P._read_source(s, oid) # 960 B: all fit |
| 106 | await P._read_source(s, a) # a is now the most recently used; b is the least |
| 107 | await P._read_source(s, e) # 1200 B > 1000 B: b goes |
| 108 | assert P.source_cache_stats()["bytes"] <= 1000 |
| 109 | assert P.source_cache_stats()["evictions"] == 1 |
| 110 | before = len(spy.calls) |
| 111 | for oid in (a, c, d, e): |
| 112 | await P._read_source(s, oid) |
| 113 | assert len(spy.calls) == before # all four survived |
| 114 | await P._read_source(s, b) |
| 115 | assert len(spy.calls) == before + 1 # b was evicted and is re-read |
| 116 | |
| 117 | |
| 118 | @pytest.mark.asyncio |
| 119 | async def test_s2_an_object_larger_than_a_quarter_of_the_budget_is_not_cached(session_factory, monkeypatch) -> None: |
| 120 | monkeypatch.setattr(P, "_SOURCE_CACHE", P._SourceCache(max_bytes=400)) |
| 121 | async with session_factory() as s: |
| 122 | small = await add_object(s, b"s" * 50) |
| 123 | big = await add_object(s, b"B" * 300) # would evict everything for one entry |
| 124 | await P._read_source(s, small) |
| 125 | await P._read_source(s, big) |
| 126 | await P._read_source(s, big) |
| 127 | assert P.source_cache_stats()["bytes"] == 50 |
| 128 | assert P.source_cache_stats()["evictions"] == 0 |
| 129 | |
| 130 | |
| 131 | # ── S3 ────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────── |
| 132 | |
| 133 | @pytest.mark.asyncio |
| 134 | async def test_s3_a_failed_read_is_not_cached_and_the_next_attempt_reads_again(session_factory, monkeypatch) -> None: |
| 135 | from musehub.storage import backends |
| 136 | real = backends.read_object_bytes |
| 137 | attempts = {"n": 0} |
| 138 | |
| 139 | async def flaky(obj, session=None): |
| 140 | attempts["n"] += 1 |
| 141 | return None if attempts["n"] == 1 else await real(obj, session=session) |
| 142 | monkeypatch.setattr(backends, "read_object_bytes", flaky) |
| 143 | async with session_factory() as s: |
| 144 | oid = await add_object(s, b"x = 1\n") |
| 145 | assert await P._read_source(s, oid) is None |
| 146 | assert await P._read_source(s, oid) == b"x = 1\n" |
| 147 | assert attempts["n"] == 2 |
| 148 | |
| 149 | |
| 150 | @pytest.mark.asyncio |
| 151 | async def test_s3_an_unreadable_object_still_fails_the_provider_with_a_warm_cache(session_factory, monkeypatch) -> None: |
| 152 | async with session_factory() as s: |
| 153 | w, c2, _ = await world_with_side_branch(s, "unreadable1") |
| 154 | await P.TypeProvider().compute(s, w.repo_id, c2, {}) # warm the cache |
| 155 | from musehub.storage import backends |
| 156 | real = backends.read_object_bytes |
| 157 | broken = B_SRC.encode() |
| 158 | |
| 159 | async def read(obj, session=None): |
| 160 | return None if obj.object_id == hashlib.sha256(A_V1.encode()).hexdigest() else await real(obj, session=session) |
| 161 | monkeypatch.setattr(backends, "read_object_bytes", read) |
| 162 | P.clear_source_cache() |
| 163 | with pytest.raises(P.UnreadableObjects): |
| 164 | await P.TypeProvider().compute(s, w.repo_id, c2, {}) |
| 165 | assert broken # (b is readable, a is not: the job fails rather than publishing an analysis of less) |
| 166 | |
| 167 | |
| 168 | # ── S4 ────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────── |
| 169 | |
| 170 | @pytest.mark.asyncio |
| 171 | async def test_s4_a_zero_byte_file_is_a_cache_hit_not_unreadable(session_factory, monkeypatch) -> None: |
| 172 | spy = StorageSpy(monkeypatch) |
| 173 | async with session_factory() as s: |
| 174 | oid = await add_object(s, b"") |
| 175 | assert await P._read_source(s, oid) == b"" |
| 176 | assert await P._read_source(s, oid) == b"" |
| 177 | assert spy.calls == [oid] |
| 178 | assert P.source_cache_stats()["hits"] == 1 |
| 179 | |
| 180 | |
| 181 | # ── S5 ────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────── |
| 182 | |
| 183 | @pytest.mark.asyncio |
| 184 | async def test_s5_a_cached_object_whose_row_is_gone_is_unreadable(session_factory) -> None: |
| 185 | async with session_factory() as s: |
| 186 | oid = await add_object(s, b"y = 2\n") |
| 187 | assert await P._read_source(s, oid) == b"y = 2\n" |
| 188 | await s.delete(await s.get(MusehubObject, oid)) |
| 189 | await s.flush() |
| 190 | assert await P._read_source(s, oid) is None |
| 191 | |
| 192 | |
| 193 | # ── S6 ────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────── |
| 194 | |
| 195 | @pytest.mark.asyncio |
| 196 | async def test_s6_the_recorded_size_rejects_an_oversize_object_before_any_read(session_factory, monkeypatch) -> None: |
| 197 | spy = StorageSpy(monkeypatch) |
| 198 | async with session_factory() as s: |
| 199 | oid = await add_object(s, b"tiny", size_bytes=P._MAX_SOURCE_BYTES + 1) |
| 200 | with pytest.raises(P.OversizeObject): |
| 201 | await P._read_source(s, oid) |
| 202 | assert spy.calls == [] |
| 203 | |
| 204 | |
| 205 | @pytest.mark.asyncio |
| 206 | async def test_s6_a_decompression_bomb_with_a_small_recorded_size_is_rejected_bounded(session_factory) -> None: |
| 207 | bomb = zlib.compress(b"a" * (P._MAX_SOURCE_BYTES * 40)) |
| 208 | assert len(bomb) < 100_000 # tiny on the wire, 40 MiB expanded |
| 209 | async with session_factory() as s: |
| 210 | oid = await add_object(s, bomb, size_bytes=len(bomb)) |
| 211 | with pytest.raises(P.OversizeObject): |
| 212 | await P._read_source(s, oid) |
| 213 | assert P.source_cache_stats()["bytes"] == 0 |
| 214 | |
| 215 | |
| 216 | def test_s6_decompress_bounded_stops_at_the_limit_for_zlib_and_passes_small_and_plain_data() -> None: |
| 217 | assert decompress_bounded(zlib.compress(b"hello"), 100) == b"hello" |
| 218 | assert decompress_bounded(b"plain text, no magic", 100) == b"plain text, no magic" |
| 219 | assert decompress_bounded(b"", 100) == b"" |
| 220 | with pytest.raises(DecompressedTooLarge): |
| 221 | decompress_bounded(zlib.compress(b"a" * 10_000), 100) |
| 222 | with pytest.raises(DecompressedTooLarge): |
| 223 | decompress_bounded(b"p" * 101, 100) # an uncompressed object over the limit is oversize too |
| 224 | |
| 225 | |
| 226 | def test_s6_decompress_bounded_handles_zstd_and_a_corrupt_stream_is_treated_as_plain_bytes() -> None: |
| 227 | import zstandard |
| 228 | assert decompress_bounded(zstandard.ZstdCompressor().compress(b"z" * 50), 100) == b"z" * 50 |
| 229 | with pytest.raises(DecompressedTooLarge): |
| 230 | decompress_bounded(zstandard.ZstdCompressor().compress(b"z" * 10_000), 100) |
| 231 | corrupt = b"x^2 is a plain text file that merely starts like a zlib header" |
| 232 | assert decompress_bounded(corrupt, 1000) == corrupt |
| 233 | |
| 234 | |
| 235 | @pytest.mark.asyncio |
| 236 | async def test_s6_an_oversize_file_is_counted_not_silently_dropped_and_the_job_still_completes(session_factory) -> None: |
| 237 | async with session_factory() as s: |
| 238 | repo = await create_repo(s, owner="gabriel", slug="oversize-count") |
| 239 | w = World(s, repo.repo_id) |
| 240 | big = await add_object(s, b"#" * 10, size_bytes=P._MAX_SOURCE_BYTES + 5, oid=hashlib.sha256(b"big").hexdigest()) |
| 241 | a = await w.obj(A_V1) |
| 242 | import msgpack |
| 243 | from musehub.db.musehub_repo_models import MusehubCommit, MusehubCommitRef, MusehubSnapshot |
| 244 | from tests.test_intel_snapshot_providers import BASE, h |
| 245 | s.add(MusehubSnapshot(snapshot_id=h("snap:o"), manifest_blob=msgpack.packb({"gen/big.py": big, "pkg/a.py": a}), directories=[], entry_count=2)) |
| 246 | await s.flush() |
| 247 | s.add(MusehubCommit(commit_id=h("commit:o"), message="o", author="gabriel", branch="main", parent_ids=[], snapshot_id=h("snap:o"), timestamp=BASE)) |
| 248 | s.add(MusehubCommitRef(repo_id=repo.repo_id, commit_id=h("commit:o"))) |
| 249 | await s.commit() |
| 250 | out = await P.TypeProvider().compute(s, w.repo_id, h("commit:o"), {}) |
| 251 | assert out[0][1]["files_oversize"] == 1 and out[0][1]["files_read"] == 1 |
| 252 | |
| 253 | |
| 254 | # ── S7 ────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────── |
| 255 | |
| 256 | @pytest.mark.asyncio |
| 257 | async def test_s7_a_warm_cache_never_shows_a_provider_another_revisions_content(session_factory) -> None: |
| 258 | v1 = "def only_in_v1(x: int) -> int:\n return x\n" |
| 259 | v2 = "def only_in_v2(x: int) -> int:\n return x\n" |
| 260 | async with session_factory() as s: |
| 261 | repo = await create_repo(s, owner="gabriel", slug="rev-cache") |
| 262 | w = World(s, repo.repo_id) |
| 263 | c1 = await w.commit("c1", {"pkg/m.py": v1}, minute=0) |
| 264 | c2 = await w.commit("c2", {"pkg/m.py": v2}, ["c1"], minute=5) |
| 265 | await s.commit() |
| 266 | await P.TypeProvider().compute(s, w.repo_id, c2, {}) |
| 267 | await s.commit() |
| 268 | rows2 = {r.address for r in (await s.execute(select(MusehubIntelType).where(MusehubIntelType.repo_id == w.repo_id))).scalars()} |
| 269 | out1 = await P.TypeProvider().compute(s, w.repo_id, c1, {}) |
| 270 | assert out1 |
| 271 | await s.commit() |
| 272 | rows1 = {r.address for r in (await s.execute(select(MusehubIntelType).where(MusehubIntelType.repo_id == w.repo_id))).scalars()} |
| 273 | assert "pkg/m.py::only_in_v2" in rows2 and "pkg/m.py::only_in_v1" not in rows2 |
| 274 | assert "pkg/m.py::only_in_v1" in rows1 and "pkg/m.py::only_in_v2" not in rows1 |
File History
1 commit
sha256:bdc0664ea517f0343fee86e1998df5d9ee2d833bb76e4ce43f85d9f8fb8dd621
perf(intel): share bounded, immutable source-object reads a…
Sonnet 5.5
patch
4 days ago