gabriel / musehub public
test_prebuild_snapshot_memory.py python
98 lines 4.5 KB
Raw
sha256:4b96079123117ef5cb85534b46736f29ce37198ade580b968ebf68209c8282f0 perf(fetch): rebuild delta-only snapshots parent-first from… Opus 5.5 patch 4 days ago
1 """musehub#315 / #331 — fetch.mpack.prebuild must not hold every decoded snapshot manifest.
2
3 A full-clone fetch bundle for gabriel/muse carries 1,682 full snapshot manifests (~215 MB).
4 ``wire_fetch_mpack`` loaded every snapshot row at once and kept each as a decoded manifest dict
5 until ``build_wire_mpack`` re-encoded them, which itself held ~3x its output. Offline on the real
6 history that peaked ~964 MB above baseline; production has ~1.6 GB available and no swap (the
7 2026-11-11 OOM). Contract:
8
9 * PSM_01 snapshots reach assembly as ``EncodedRecords`` (compact JSON written verbatim) through
10 ``build_wire_mpack_buffer`` — no list of decoded manifests, no extra bundle copy;
11 * PSM_02 the fetched bundle is unchanged: every snapshot record and manifest round-trips;
12 * PSM_03 snapshot rows are loaded in bounded chunks, not all at once.
13 """
14
15 from __future__ import annotations
16
17 import pytest
18 from sqlalchemy import event
19 from sqlalchemy.ext.asyncio import AsyncSession
20
21 import muse.core.mpack as engine_mpack
22 from muse.core.types import blob_id
23 from musehub.services import musehub_wire_fetch
24 from musehub.services.musehub_repository import create_repo
25 from tests.test_multi_tip_manifest_union import (
26 _IDENTITY_ID,
27 _OWNER,
28 _InMemBackend,
29 _cid,
30 _fetch,
31 _push_full_manifest,
32 )
33
34
35 async def _seed_chain(session: AsyncSession, name: str, n: int) -> tuple[str, _InMemBackend, list[str], list[dict[str, str]]]:
36 repo = await create_repo(session, name=name, owner=_OWNER, owner_user_id=_IDENTITY_ID,
37 visibility="public", initialize=False)
38 await session.commit()
39 backend = _InMemBackend()
40 cids: list[str] = []
41 manifests: list[dict[str, str]] = []
42 manifest: dict[str, str] = {}
43 parent = None
44 for i in range(n):
45 content = f"{name}-file-{i}".encode()
46 oid = blob_id(content)
47 manifest = {**manifest, f"src/f{i}.py": oid}
48 cid = _cid(f"{name}-{i}")
49 await _push_full_manifest(session, backend, repo.repo_id, "main", cid, parent, manifest, {oid: content})
50 cids.append(cid); manifests.append(dict(manifest)); parent = cid
51 return repo.repo_id, backend, cids, manifests
52
53
54 @pytest.mark.asyncio
55 async def test_psm_01_snapshots_reach_assembly_as_encoded_records(db_session: AsyncSession, monkeypatch: pytest.MonkeyPatch) -> None:
56 repo_id, backend, cids, _ = await _seed_chain(db_session, "psm-01", 3)
57 seen: list[object] = []
58 real = engine_mpack.build_wire_mpack_buffer
59
60 def spy(mpack, *, meta=None): # type: ignore[no-untyped-def]
61 seen.append(mpack.get("snapshots"))
62 return real(mpack, meta=meta)
63
64 monkeypatch.setattr(engine_mpack, "build_wire_mpack_buffer", spy)
65 await _fetch(db_session, backend, repo_id, want=[cids[-1]])
66 assert len(seen) == 1, "prebuild/fetch assembly must go through build_wire_mpack_buffer"
67 assert isinstance(seen[0], engine_mpack.EncodedRecords)
68 assert len(seen[0]) == 3
69
70
71 @pytest.mark.asyncio
72 async def test_psm_02_fetched_bundle_round_trips_every_snapshot(db_session: AsyncSession) -> None:
73 repo_id, backend, cids, manifests = await _seed_chain(db_session, "psm-02", 4)
74 parsed = await _fetch(db_session, backend, repo_id, want=[cids[-1]])
75 got = sorted((tuple(sorted(s["manifest"].items())) for s in parsed["snapshots"]))
76 assert got == sorted(tuple(sorted(m.items())) for m in manifests)
77 assert {b["object_id"] for b in parsed["blobs"]} == set(manifests[-1].values())
78
79
80 @pytest.mark.asyncio
81 async def test_psm_03_snapshot_rows_load_in_bounded_chunks(db_session: AsyncSession, monkeypatch: pytest.MonkeyPatch) -> None:
82 repo_id, backend, cids, _ = await _seed_chain(db_session, "psm-03", 5)
83 monkeypatch.setattr(musehub_wire_fetch, "_SNAPSHOT_LOAD_CHUNK", 2)
84 statements: list[str] = []
85 sync_engine = db_session.bind.sync_engine if hasattr(db_session.bind, "sync_engine") else db_session.get_bind()
86
87 def count(conn, cursor, statement, parameters, context, executemany): # type: ignore[no-untyped-def]
88 # full-row loads only (the parent-id pre-query selects two columns, no blobs)
89 if "FROM musehub_snapshots" in statement and "ANY(" in statement and "manifest_blob" in statement:
90 statements.append(statement)
91
92 event.listen(sync_engine, "before_cursor_execute", count)
93 try:
94 parsed = await _fetch(db_session, backend, repo_id, want=[cids[-1]])
95 finally:
96 event.remove(sync_engine, "before_cursor_execute", count)
97 assert len(parsed["snapshots"]) == 5
98 assert len(statements) == 3, f"expected 3 chunked snapshot queries (5 rows, chunk 2), got {len(statements)}"
File History 2 commits
sha256:4b96079123117ef5cb85534b46736f29ce37198ade580b968ebf68209c8282f0 perf(fetch): rebuild delta-only snapshots parent-first from… Opus 5.5 patch 4 days ago
sha256:1479064e993435b009b3f402ec7a7c3e5b33edd3826c5fc785db3b589e4bf72a perf(fetch): prebuild keeps snapshots encoded and loads the… Opus 5.5 patch 4 days ago