test_prebuild_snapshot_memory.py
python
sha256:4b96079123117ef5cb85534b46736f29ce37198ade580b968ebf68209c8282f0
perf(fetch): rebuild delta-only snapshots parent-first from…
Opus 5.5
patch
4 days ago
| 1 | """musehub#315 / #331 — fetch.mpack.prebuild must not hold every decoded snapshot manifest. |
| 2 | |
| 3 | A full-clone fetch bundle for gabriel/muse carries 1,682 full snapshot manifests (~215 MB). |
| 4 | ``wire_fetch_mpack`` loaded every snapshot row at once and kept each as a decoded manifest dict |
| 5 | until ``build_wire_mpack`` re-encoded them, which itself held ~3x its output. Offline on the real |
| 6 | history that peaked ~964 MB above baseline; production has ~1.6 GB available and no swap (the |
| 7 | 2026-11-11 OOM). Contract: |
| 8 | |
| 9 | * PSM_01 snapshots reach assembly as ``EncodedRecords`` (compact JSON written verbatim) through |
| 10 | ``build_wire_mpack_buffer`` — no list of decoded manifests, no extra bundle copy; |
| 11 | * PSM_02 the fetched bundle is unchanged: every snapshot record and manifest round-trips; |
| 12 | * PSM_03 snapshot rows are loaded in bounded chunks, not all at once. |
| 13 | """ |
| 14 | |
| 15 | from __future__ import annotations |
| 16 | |
| 17 | import pytest |
| 18 | from sqlalchemy import event |
| 19 | from sqlalchemy.ext.asyncio import AsyncSession |
| 20 | |
| 21 | import muse.core.mpack as engine_mpack |
| 22 | from muse.core.types import blob_id |
| 23 | from musehub.services import musehub_wire_fetch |
| 24 | from musehub.services.musehub_repository import create_repo |
| 25 | from tests.test_multi_tip_manifest_union import ( |
| 26 | _IDENTITY_ID, |
| 27 | _OWNER, |
| 28 | _InMemBackend, |
| 29 | _cid, |
| 30 | _fetch, |
| 31 | _push_full_manifest, |
| 32 | ) |
| 33 | |
| 34 | |
| 35 | async def _seed_chain(session: AsyncSession, name: str, n: int) -> tuple[str, _InMemBackend, list[str], list[dict[str, str]]]: |
| 36 | repo = await create_repo(session, name=name, owner=_OWNER, owner_user_id=_IDENTITY_ID, |
| 37 | visibility="public", initialize=False) |
| 38 | await session.commit() |
| 39 | backend = _InMemBackend() |
| 40 | cids: list[str] = [] |
| 41 | manifests: list[dict[str, str]] = [] |
| 42 | manifest: dict[str, str] = {} |
| 43 | parent = None |
| 44 | for i in range(n): |
| 45 | content = f"{name}-file-{i}".encode() |
| 46 | oid = blob_id(content) |
| 47 | manifest = {**manifest, f"src/f{i}.py": oid} |
| 48 | cid = _cid(f"{name}-{i}") |
| 49 | await _push_full_manifest(session, backend, repo.repo_id, "main", cid, parent, manifest, {oid: content}) |
| 50 | cids.append(cid); manifests.append(dict(manifest)); parent = cid |
| 51 | return repo.repo_id, backend, cids, manifests |
| 52 | |
| 53 | |
| 54 | @pytest.mark.asyncio |
| 55 | async def test_psm_01_snapshots_reach_assembly_as_encoded_records(db_session: AsyncSession, monkeypatch: pytest.MonkeyPatch) -> None: |
| 56 | repo_id, backend, cids, _ = await _seed_chain(db_session, "psm-01", 3) |
| 57 | seen: list[object] = [] |
| 58 | real = engine_mpack.build_wire_mpack_buffer |
| 59 | |
| 60 | def spy(mpack, *, meta=None): # type: ignore[no-untyped-def] |
| 61 | seen.append(mpack.get("snapshots")) |
| 62 | return real(mpack, meta=meta) |
| 63 | |
| 64 | monkeypatch.setattr(engine_mpack, "build_wire_mpack_buffer", spy) |
| 65 | await _fetch(db_session, backend, repo_id, want=[cids[-1]]) |
| 66 | assert len(seen) == 1, "prebuild/fetch assembly must go through build_wire_mpack_buffer" |
| 67 | assert isinstance(seen[0], engine_mpack.EncodedRecords) |
| 68 | assert len(seen[0]) == 3 |
| 69 | |
| 70 | |
| 71 | @pytest.mark.asyncio |
| 72 | async def test_psm_02_fetched_bundle_round_trips_every_snapshot(db_session: AsyncSession) -> None: |
| 73 | repo_id, backend, cids, manifests = await _seed_chain(db_session, "psm-02", 4) |
| 74 | parsed = await _fetch(db_session, backend, repo_id, want=[cids[-1]]) |
| 75 | got = sorted((tuple(sorted(s["manifest"].items())) for s in parsed["snapshots"])) |
| 76 | assert got == sorted(tuple(sorted(m.items())) for m in manifests) |
| 77 | assert {b["object_id"] for b in parsed["blobs"]} == set(manifests[-1].values()) |
| 78 | |
| 79 | |
| 80 | @pytest.mark.asyncio |
| 81 | async def test_psm_03_snapshot_rows_load_in_bounded_chunks(db_session: AsyncSession, monkeypatch: pytest.MonkeyPatch) -> None: |
| 82 | repo_id, backend, cids, _ = await _seed_chain(db_session, "psm-03", 5) |
| 83 | monkeypatch.setattr(musehub_wire_fetch, "_SNAPSHOT_LOAD_CHUNK", 2) |
| 84 | statements: list[str] = [] |
| 85 | sync_engine = db_session.bind.sync_engine if hasattr(db_session.bind, "sync_engine") else db_session.get_bind() |
| 86 | |
| 87 | def count(conn, cursor, statement, parameters, context, executemany): # type: ignore[no-untyped-def] |
| 88 | # full-row loads only (the parent-id pre-query selects two columns, no blobs) |
| 89 | if "FROM musehub_snapshots" in statement and "ANY(" in statement and "manifest_blob" in statement: |
| 90 | statements.append(statement) |
| 91 | |
| 92 | event.listen(sync_engine, "before_cursor_execute", count) |
| 93 | try: |
| 94 | parsed = await _fetch(db_session, backend, repo_id, want=[cids[-1]]) |
| 95 | finally: |
| 96 | event.remove(sync_engine, "before_cursor_execute", count) |
| 97 | assert len(parsed["snapshots"]) == 5 |
| 98 | assert len(statements) == 3, f"expected 3 chunked snapshot queries (5 rows, chunk 2), got {len(statements)}" |
File History
2 commits
sha256:4b96079123117ef5cb85534b46736f29ce37198ade580b968ebf68209c8282f0
perf(fetch): rebuild delta-only snapshots parent-first from…
Opus 5.5
patch
4 days ago
sha256:1479064e993435b009b3f402ec7a7c3e5b33edd3826c5fc785db3b589e4bf72a
perf(fetch): prebuild keeps snapshots encoded and loads the…
Opus 5.5
patch
4 days ago