gabriel / musehub public
test_snapshot_symbol_indexer.py python
968 lines 40.4 KB
Raw
sha256:a10adeeb7a0169cb9900f9806ed7a973047258abb6283724fe55e8eb68ff3f0a init: musehub initial commit Human 171 days ago
1 """Tests for the Snapshot & Symbol Indexer — Section 5 of test-coverage-checklist.md.
2
3 Complements test_snapshot_entries.py (14 tests on the snapshot write/read path).
4 This file focuses on the symbol indexer and the gaps not covered there.
5
6 Coverage layers
7 ───────────────
8 Unit — _extract_ops (flat/nested child_ops, missing address, non-dict delta);
9 _op_to_muse_op (all mapping keys, unknown passthrough);
10 msgpack round-trip schema version preservation.
11 Integration — build_symbol_index: no-op when no structured_delta; creates index row;
12 correct symbol_history/hash_occurrence content; prunes stale index;
13 invalidates MusehubFileIntelCache; BFS excludes orphaned commits.
14 load_symbol_history: empty when no index; with/without file_path filter.
15 load_hash_occurrence: empty when no index; correct content.
16 get_index_meta: None/present states.
17 load_intel_snapshot: None/present states.
18 get_snapshot_manifests_batch: empty list, single, multi-snapshot.
19 Data — upsert_snapshot_entries atomic replace (stale entries removed);
20 build_symbol_index only one row per repo after rebuild;
21 FileIntelCache invalidated on every rebuild;
22 BFS reachability excludes orphaned branches.
23 Security — Corrupt msgpack blob returns {} not exception;
24 build_symbol_index with unknown head_commit_id returns None gracefully.
25 Stress — upsert_snapshot_entries with 1 000-file manifest;
26 get_snapshot_manifests_batch with 50 snapshots in one query;
27 build_symbol_index with 100 commits (10 ops each);
28 load_symbol_history file_path filter on large index.
29 Performance — _extract_ops 1 000 calls < 100 ms;
30 build_symbol_index 100 commits < 3 s;
31 upsert_snapshot_entries 1 000 files < 500 ms;
32 get_snapshot_manifests_batch 50 snapshots < 300 ms.
33 E2E — Full pipeline: commits with structured_delta → build_symbol_index →
34 get_index_meta returns correct ref; rebuild replaces previous row;
35 symbol list HTTP page returns 200 for repo with index.
36 """
37 from __future__ import annotations
38
39 import json
40 import time
41 import uuid
42 from datetime import datetime, timezone
43
44 import msgpack
45 import pytest
46 from sqlalchemy.ext.asyncio import AsyncSession
47
48 from musehub.db import musehub_models as db
49 from tests.factories import create_repo
50 from musehub.muse_contracts.json_types import JSONObject
51
52
53 # ─────────────────────────────────────────────────────────────────────────────
54 # Helpers
55 # ─────────────────────────────────────────────────────────────────────────────
56
57 def _now() -> datetime:
58 return datetime.now(tz=timezone.utc)
59
60
61 async def _commit_with_delta(
62 session: AsyncSession,
63 repo_id: str,
64 commit_id: str,
65 ops: list[JSONObject],
66 parent_ids: list[str] | None = None,
67 branch: str = "main",
68 ) -> db.MusehubCommit:
69 """Insert a commit whose commit_meta carries a structured_delta."""
70 commit = db.MusehubCommit(
71 commit_id=commit_id,
72 repo_id=repo_id,
73 branch=branch,
74 parent_ids=parent_ids or [],
75 message="feat: test commit",
76 author="gabriel",
77 timestamp=_now(),
78 commit_meta={"structured_delta": {"ops": ops}},
79 )
80 session.add(commit)
81 await session.flush()
82 return commit
83
84
85 def _insert_op(address: str, content_id: str = "sha256:abc") -> JSONObject:
86 return {"address": address, "op": "insert", "content_id": content_id}
87
88
89 def _patch_op(file_addr: str, children: list[JSONObject]) -> JSONObject:
90 return {"address": file_addr, "op": "patch", "child_ops": children}
91
92
93 # ─────────────────────────────────────────────────────────────────────────────
94 # Layer 1 — Unit: pure functions
95 # ─────────────────────────────────────────────────────────────────────────────
96
97 class TestExtractOps:
98 """_extract_ops pulls a flat list of ops including child_ops."""
99
100 def _run(self, commit_meta: JSONObject) -> list[JSONObject]:
101 from musehub.services.musehub_symbol_indexer import _extract_ops
102 return _extract_ops(commit_meta)
103
104 def test_no_structured_delta_returns_empty(self) -> None:
105 assert self._run({}) == []
106
107 def test_none_delta_returns_empty(self) -> None:
108 assert self._run({"structured_delta": None}) == []
109
110 def test_non_dict_delta_returns_empty(self) -> None:
111 assert self._run({"structured_delta": "bad"}) == []
112
113 def test_flat_ops_without_child_ops(self) -> None:
114 meta = {
115 "structured_delta": {
116 "ops": [
117 {"address": "main.py::Foo", "op": "insert"},
118 {"address": "main.py::Bar", "op": "delete"},
119 ]
120 }
121 }
122 result = self._run(meta)
123 assert len(result) == 2
124 assert result[0]["address"] == "main.py::Foo"
125 assert result[1]["address"] == "main.py::Bar"
126
127 def test_patch_op_with_child_ops_flattened(self) -> None:
128 meta = {
129 "structured_delta": {
130 "ops": [
131 {
132 "address": "src/app.py",
133 "op": "patch",
134 "child_ops": [
135 {"address": "src/app.py::MyClass", "op": "insert"},
136 {"address": "src/app.py::MyClass.run", "op": "insert"},
137 ],
138 }
139 ]
140 }
141 }
142 result = self._run(meta)
143 # 1 top-level + 2 child_ops
144 assert len(result) == 3
145 addresses = [op["address"] for op in result]
146 assert "src/app.py" in addresses
147 assert "src/app.py::MyClass" in addresses
148 assert "src/app.py::MyClass.run" in addresses
149
150 def test_op_without_address_skipped(self) -> None:
151 meta = {
152 "structured_delta": {
153 "ops": [
154 {"op": "insert"}, # no address
155 {"address": "ok.py", "op": "insert"},
156 ]
157 }
158 }
159 result = self._run(meta)
160 assert len(result) == 1
161 assert result[0]["address"] == "ok.py"
162
163 def test_child_op_without_address_skipped(self) -> None:
164 meta = {
165 "structured_delta": {
166 "ops": [
167 {
168 "address": "file.py",
169 "op": "patch",
170 "child_ops": [
171 {"op": "insert"}, # no address — must be skipped
172 {"address": "file.py::Good", "op": "insert"},
173 ],
174 }
175 ]
176 }
177 }
178 result = self._run(meta)
179 addresses = [op["address"] for op in result]
180 assert "file.py::Good" in addresses
181 # The child without address must not appear
182 for op in result:
183 assert "address" in op
184
185 def test_non_dict_op_skipped(self) -> None:
186 meta = {"structured_delta": {"ops": ["not-a-dict", {"address": "f.py", "op": "add"}]}}
187 result = self._run(meta)
188 assert len(result) == 1
189
190
191 class TestOpToMuseOp:
192 """_op_to_muse_op maps DomainOp vocabulary to muse history vocabulary."""
193
194 def _run(self, op_type: str) -> str:
195 from musehub.services.musehub_symbol_indexer import _op_to_muse_op
196 return _op_to_muse_op(op_type)
197
198 def test_insert_maps_to_add(self) -> None:
199 assert self._run("insert") == "add"
200
201 def test_delete_maps_to_delete(self) -> None:
202 assert self._run("delete") == "delete"
203
204 def test_replace_maps_to_modify(self) -> None:
205 assert self._run("replace") == "modify"
206
207 def test_patch_maps_to_modify(self) -> None:
208 assert self._run("patch") == "modify"
209
210 def test_move_maps_to_move(self) -> None:
211 assert self._run("move") == "move"
212
213 def test_directory_rename_maps_to_rename(self) -> None:
214 assert self._run("directory_rename") == "rename"
215
216 def test_unknown_op_passthrough(self) -> None:
217 assert self._run("frobnicate") == "frobnicate"
218
219 def test_empty_string_passthrough(self) -> None:
220 assert self._run("") == ""
221
222
223 class TestMsgpackRoundTrip:
224 """Verify the msgpack blob schema produced by build_symbol_index is stable."""
225
226 def test_schema_version_preserved(self) -> None:
227 data = {
228 "schema_version": "musehub-v1",
229 "index": "symbol_history",
230 "updated_at": "2026-04-04T00:00:00+00:00",
231 "entries": {"main.py::Foo": [{"commit_id": "abc", "op": "add"}]},
232 }
233 blob = msgpack.packb(data, use_bin_type=True)
234 recovered = msgpack.unpackb(blob, raw=False)
235 assert recovered["schema_version"] == "musehub-v1"
236 assert recovered["entries"]["main.py::Foo"][0]["op"] == "add"
237
238 def test_bytes_values_survive_round_trip(self) -> None:
239 data = {"entries": {"k": b"\x00\x01\x02"}}
240 blob = msgpack.packb(data, use_bin_type=True)
241 recovered = msgpack.unpackb(blob, raw=False)
242 assert recovered["entries"]["k"] == b"\x00\x01\x02"
243
244
245 # ─────────────────────────────────────────────────────────────────────────────
246 # Layer 2 — Integration: build_symbol_index + read functions
247 # ─────────────────────────────────────────────────────────────────────────────
248
249 class TestBuildSymbolIndex:
250 @pytest.mark.asyncio
251 async def test_returns_none_when_no_structured_delta(
252 self, db_session: AsyncSession
253 ) -> None:
254 from musehub.services.musehub_symbol_indexer import build_symbol_index
255 from tests.factories import create_commit
256
257 repo = await create_repo(db_session, slug="idx-nodelta")
258 commit = await create_commit(db_session, repo.repo_id, branch="main")
259
260 result = await build_symbol_index(db_session, repo.repo_id, commit.commit_id)
261 assert result is None
262
263 @pytest.mark.asyncio
264 async def test_creates_index_row_for_structured_delta(
265 self, db_session: AsyncSession
266 ) -> None:
267 from musehub.services.musehub_symbol_indexer import build_symbol_index
268
269 repo = await create_repo(db_session, slug="idx-creates")
270 commit = await _commit_with_delta(
271 db_session, repo.repo_id, "c001",
272 ops=[_insert_op("main.py::Foo", "sha256:aaa")],
273 )
274
275 result = await build_symbol_index(db_session, repo.repo_id, commit.commit_id)
276 assert result is not None
277 assert result.repo_id == repo.repo_id
278 assert result.ref == commit.commit_id
279
280 @pytest.mark.asyncio
281 async def test_symbol_history_blob_contains_correct_entries(
282 self, db_session: AsyncSession
283 ) -> None:
284 from musehub.services.musehub_symbol_indexer import build_symbol_index
285
286 repo = await create_repo(db_session, slug="idx-symhist")
287 commit = await _commit_with_delta(
288 db_session, repo.repo_id, "c002",
289 ops=[
290 _insert_op("src/app.py::MyClass", "sha256:class"),
291 _insert_op("src/app.py::my_func", "sha256:func"),
292 ],
293 )
294
295 row = await build_symbol_index(db_session, repo.repo_id, commit.commit_id)
296 assert row is not None and row.symbol_history is not None
297
298 doc = msgpack.unpackb(row.symbol_history, raw=False)
299 entries = doc["entries"]
300 assert "src/app.py::MyClass" in entries
301 assert "src/app.py::my_func" in entries
302 assert entries["src/app.py::MyClass"][0]["op"] == "add"
303
304 @pytest.mark.asyncio
305 async def test_hash_occurrence_tracks_shared_content(
306 self, db_session: AsyncSession
307 ) -> None:
308 from musehub.services.musehub_symbol_indexer import build_symbol_index
309
310 repo = await create_repo(db_session, slug="idx-hashoc")
311 shared_hash = "sha256:shared"
312 commit = await _commit_with_delta(
313 db_session, repo.repo_id, "c003",
314 ops=[
315 _insert_op("a.py::Foo", shared_hash),
316 _insert_op("b.py::Bar", shared_hash),
317 ],
318 )
319
320 row = await build_symbol_index(db_session, repo.repo_id, commit.commit_id)
321 assert row is not None and row.hash_occurrence is not None
322
323 doc = msgpack.unpackb(row.hash_occurrence, raw=False)
324 entries = doc["entries"]
325 assert shared_hash in entries
326 assert set(entries[shared_hash]) == {"a.py::Foo", "b.py::Bar"}
327
328 @pytest.mark.asyncio
329 async def test_rebuild_prunes_old_index_row(
330 self, db_session: AsyncSession
331 ) -> None:
332 from musehub.services.musehub_symbol_indexer import build_symbol_index
333 from sqlalchemy import select, func
334
335 repo = await create_repo(db_session, slug="idx-prune")
336 c1 = await _commit_with_delta(db_session, repo.repo_id, "c100",
337 ops=[_insert_op("f.py::A")])
338 await build_symbol_index(db_session, repo.repo_id, c1.commit_id)
339 await db_session.commit()
340
341 c2 = await _commit_with_delta(db_session, repo.repo_id, "c101",
342 ops=[_insert_op("f.py::B")])
343 await build_symbol_index(db_session, repo.repo_id, c2.commit_id)
344 await db_session.commit()
345
346 count = (await db_session.execute(
347 select(func.count()).where(db.MusehubSymbolIndex.repo_id == repo.repo_id)
348 )).scalar_one()
349 assert count == 1
350
351 @pytest.mark.asyncio
352 async def test_rebuild_invalidates_file_intel_cache(
353 self, db_session: AsyncSession
354 ) -> None:
355 from musehub.services.musehub_symbol_indexer import build_symbol_index
356 from sqlalchemy import select, func
357
358 repo = await create_repo(db_session, slug="idx-cache-inv")
359 # Seed a stale cache entry
360 cache = db.MusehubFileIntelCache(
361 repo_id=repo.repo_id,
362 file_path="old.py",
363 ref="stale-ref",
364 intel_json="{}",
365 symbol_history_json="{}",
366 )
367 db_session.add(cache)
368 await db_session.flush()
369
370 commit = await _commit_with_delta(db_session, repo.repo_id, "c200",
371 ops=[_insert_op("new.py::X")])
372 await build_symbol_index(db_session, repo.repo_id, commit.commit_id)
373 await db_session.commit()
374
375 remaining = (await db_session.execute(
376 select(func.count()).where(db.MusehubFileIntelCache.repo_id == repo.repo_id)
377 )).scalar_one()
378 assert remaining == 0 # all cache entries invalidated
379
380 @pytest.mark.asyncio
381 async def test_bfs_excludes_orphaned_commits(
382 self, db_session: AsyncSession
383 ) -> None:
384 """Commits not reachable from head must not appear in the symbol index."""
385 from musehub.services.musehub_symbol_indexer import build_symbol_index
386
387 repo = await create_repo(db_session, slug="idx-bfs")
388 # Main chain: orphan → (not linked to main chain)
389 orphan = await _commit_with_delta(
390 db_session, repo.repo_id, "orphan",
391 ops=[_insert_op("orphan.py::OrphanSym", "sha256:orphan")],
392 parent_ids=[],
393 )
394 # Head chain
395 head = await _commit_with_delta(
396 db_session, repo.repo_id, "head",
397 ops=[_insert_op("main.py::RealSym", "sha256:real")],
398 parent_ids=[],
399 )
400
401 row = await build_symbol_index(db_session, repo.repo_id, head.commit_id)
402 assert row is not None
403 history = msgpack.unpackb(row.symbol_history, raw=False)["entries"]
404 assert "main.py::RealSym" in history
405 assert "orphan.py::OrphanSym" not in history
406
407
408 class TestLoadFunctions:
409 @pytest.mark.asyncio
410 async def test_load_symbol_history_empty_when_no_index(
411 self, db_session: AsyncSession
412 ) -> None:
413 from musehub.services.musehub_symbol_indexer import load_symbol_history
414 repo = await create_repo(db_session, slug="load-noindex")
415 result = await load_symbol_history(db_session, repo.repo_id)
416 assert result == {}
417
418 @pytest.mark.asyncio
419 async def test_load_symbol_history_with_file_path_filter(
420 self, db_session: AsyncSession
421 ) -> None:
422 from musehub.services.musehub_symbol_indexer import build_symbol_index, load_symbol_history
423
424 repo = await create_repo(db_session, slug="load-filter")
425 commit = await _commit_with_delta(
426 db_session, repo.repo_id, "cF01",
427 ops=[
428 _insert_op("a.py::Foo", "sha256:x"),
429 _insert_op("a.py", "sha256:file"), # file-level address
430 _insert_op("b.py::Bar", "sha256:y"),
431 ],
432 )
433 await build_symbol_index(db_session, repo.repo_id, commit.commit_id)
434 await db_session.commit()
435
436 result = await load_symbol_history(db_session, repo.repo_id, file_path="a.py")
437 assert "a.py::Foo" in result
438 assert "a.py" in result
439 assert "b.py::Bar" not in result
440
441 @pytest.mark.asyncio
442 async def test_load_hash_occurrence_empty_when_no_index(
443 self, db_session: AsyncSession
444 ) -> None:
445 from musehub.services.musehub_symbol_indexer import load_hash_occurrence
446 repo = await create_repo(db_session, slug="hash-noindex")
447 assert await load_hash_occurrence(db_session, repo.repo_id) == {}
448
449 @pytest.mark.asyncio
450 async def test_load_hash_occurrence_returns_correct_entries(
451 self, db_session: AsyncSession
452 ) -> None:
453 from musehub.services.musehub_symbol_indexer import build_symbol_index, load_hash_occurrence
454
455 repo = await create_repo(db_session, slug="hash-entries")
456 commit = await _commit_with_delta(
457 db_session, repo.repo_id, "cH01",
458 ops=[_insert_op("x.py::X", "sha256:hash1"), _insert_op("y.py::Y", "sha256:hash1")],
459 )
460 await build_symbol_index(db_session, repo.repo_id, commit.commit_id)
461 await db_session.commit()
462
463 result = await load_hash_occurrence(db_session, repo.repo_id)
464 assert "sha256:hash1" in result
465 assert set(result["sha256:hash1"]) == {"x.py::X", "y.py::Y"}
466
467 @pytest.mark.asyncio
468 async def test_get_index_meta_none_when_no_index(
469 self, db_session: AsyncSession
470 ) -> None:
471 from musehub.services.musehub_symbol_indexer import get_index_meta
472 repo = await create_repo(db_session, slug="meta-none")
473 assert await get_index_meta(db_session, repo.repo_id) is None
474
475 @pytest.mark.asyncio
476 async def test_get_index_meta_returns_ref_and_symbol_count(
477 self, db_session: AsyncSession
478 ) -> None:
479 from musehub.services.musehub_symbol_indexer import build_symbol_index, get_index_meta
480
481 repo = await create_repo(db_session, slug="meta-ok")
482 commit = await _commit_with_delta(
483 db_session, repo.repo_id, "cM01",
484 ops=[_insert_op("f.py::A"), _insert_op("f.py::B")],
485 )
486 await build_symbol_index(db_session, repo.repo_id, commit.commit_id)
487 await db_session.commit()
488
489 meta = await get_index_meta(db_session, repo.repo_id)
490 assert meta is not None
491 assert meta["ref"] == commit.commit_id
492 assert meta["built_at"] is not None
493 assert meta["symbol_count"] >= 2
494
495 @pytest.mark.asyncio
496 async def test_load_intel_snapshot_none_when_no_index(
497 self, db_session: AsyncSession
498 ) -> None:
499 from musehub.services.musehub_symbol_indexer import load_intel_snapshot
500 repo = await create_repo(db_session, slug="intel-none")
501 assert await load_intel_snapshot(db_session, repo.repo_id) is None
502
503 @pytest.mark.asyncio
504 async def test_load_intel_snapshot_returns_snapshot_when_built(
505 self, db_session: AsyncSession
506 ) -> None:
507 from musehub.services.musehub_symbol_indexer import build_symbol_index, load_intel_snapshot
508
509 repo = await create_repo(db_session, slug="intel-ok")
510 commit = await _commit_with_delta(
511 db_session, repo.repo_id, "cI01",
512 ops=[_insert_op("app.py::Handler", "sha256:h1")],
513 )
514 row = await build_symbol_index(db_session, repo.repo_id, commit.commit_id)
515 await db_session.commit()
516
517 if row is not None and row.intel_full_json:
518 snap = await load_intel_snapshot(db_session, repo.repo_id)
519 assert snap is not None
520 else:
521 # intel computation may be skipped if musehub_intel unavailable
522 pytest.skip("intel_full_json not populated — compute_intel unavailable")
523
524
525 class TestGetSnapshotManifestsBatch:
526 @pytest.mark.asyncio
527 async def test_empty_list_returns_empty_dict(
528 self, db_session: AsyncSession
529 ) -> None:
530 from musehub.services.musehub_snapshot import get_snapshot_manifests_batch
531 result = await get_snapshot_manifests_batch(db_session, [])
532 assert result == {}
533
534 @pytest.mark.asyncio
535 async def test_single_snapshot_manifest(
536 self, db_session: AsyncSession
537 ) -> None:
538 from musehub.services.musehub_snapshot import (
539 get_snapshot_manifests_batch,
540 upsert_snapshot_entries,
541 )
542 repo = await create_repo(db_session, slug="batch-single")
543 snap_id = "snap-batch-01"
544 await upsert_snapshot_entries(
545 db_session, repo.repo_id, snap_id, {"a.py": "sha256:a", "b.py": "sha256:b"}
546 )
547 await db_session.commit()
548
549 result = await get_snapshot_manifests_batch(db_session, [snap_id])
550 assert snap_id in result
551 assert result[snap_id]["a.py"] == "sha256:a"
552 assert result[snap_id]["b.py"] == "sha256:b"
553
554 @pytest.mark.asyncio
555 async def test_multiple_snapshots_grouped_correctly(
556 self, db_session: AsyncSession
557 ) -> None:
558 from musehub.services.musehub_snapshot import (
559 get_snapshot_manifests_batch,
560 upsert_snapshot_entries,
561 )
562 repo = await create_repo(db_session, slug="batch-multi")
563 for i in range(5):
564 snap_id = f"snap-multi-{i:02d}"
565 await upsert_snapshot_entries(
566 db_session, repo.repo_id, snap_id, {f"file{i}.py": f"sha256:{i}"}
567 )
568 await db_session.commit()
569
570 ids = [f"snap-multi-{i:02d}" for i in range(5)]
571 result = await get_snapshot_manifests_batch(db_session, ids)
572 assert len(result) == 5
573 for i, sid in enumerate(ids):
574 assert f"file{i}.py" in result[sid]
575
576 @pytest.mark.asyncio
577 async def test_unknown_snapshot_id_returns_empty_manifest(
578 self, db_session: AsyncSession
579 ) -> None:
580 from musehub.services.musehub_snapshot import get_snapshot_manifests_batch
581 result = await get_snapshot_manifests_batch(db_session, ["ghost-snap"])
582 assert result == {"ghost-snap": {}}
583
584
585 # ─────────────────────────────────────────────────────────────────────────────
586 # Layer 3 — E2E: full pipeline via direct service calls with real DB
587 # ─────────────────────────────────────────────────────────────────────────────
588
589 class TestSymbolIndexPipeline:
590 @pytest.mark.asyncio
591 async def test_build_then_meta_reflects_head_commit(
592 self, db_session: AsyncSession
593 ) -> None:
594 from musehub.services.musehub_symbol_indexer import build_symbol_index, get_index_meta
595
596 repo = await create_repo(db_session, slug="e2e-pipeline")
597 c1 = await _commit_with_delta(
598 db_session, repo.repo_id, "pipe-c001",
599 ops=[_insert_op("service.py::APIHandler", "sha256:h1")],
600 )
601 await build_symbol_index(db_session, repo.repo_id, c1.commit_id)
602 await db_session.commit()
603
604 meta = await get_index_meta(db_session, repo.repo_id)
605 assert meta is not None
606 assert meta["ref"] == c1.commit_id
607
608 @pytest.mark.asyncio
609 async def test_rebuild_updates_ref_to_latest_commit(
610 self, db_session: AsyncSession
611 ) -> None:
612 from musehub.services.musehub_symbol_indexer import build_symbol_index, get_index_meta
613
614 repo = await create_repo(db_session, slug="e2e-rebuild")
615 c1 = await _commit_with_delta(db_session, repo.repo_id, "rb-c001",
616 ops=[_insert_op("a.py::Old")])
617 await build_symbol_index(db_session, repo.repo_id, c1.commit_id)
618 await db_session.commit()
619
620 c2 = await _commit_with_delta(db_session, repo.repo_id, "rb-c002",
621 ops=[_insert_op("b.py::New")],
622 parent_ids=[c1.commit_id])
623 await build_symbol_index(db_session, repo.repo_id, c2.commit_id)
624 await db_session.commit()
625
626 meta = await get_index_meta(db_session, repo.repo_id)
627 assert meta is not None
628 assert meta["ref"] == c2.commit_id
629
630 @pytest.mark.asyncio
631 async def test_multi_commit_chain_all_symbols_indexed(
632 self, db_session: AsyncSession
633 ) -> None:
634 """3-commit chain — every symbol from every commit must appear in the index."""
635 from musehub.services.musehub_symbol_indexer import build_symbol_index, load_symbol_history
636
637 repo = await create_repo(db_session, slug="e2e-chain")
638 c1 = await _commit_with_delta(db_session, repo.repo_id, "chain-c001",
639 ops=[_insert_op("a.py::A1")])
640 c2 = await _commit_with_delta(db_session, repo.repo_id, "chain-c002",
641 ops=[_insert_op("b.py::B1")],
642 parent_ids=[c1.commit_id])
643 c3 = await _commit_with_delta(db_session, repo.repo_id, "chain-c003",
644 ops=[_insert_op("c.py::C1")],
645 parent_ids=[c2.commit_id])
646 await build_symbol_index(db_session, repo.repo_id, c3.commit_id)
647 await db_session.commit()
648
649 history = await load_symbol_history(db_session, repo.repo_id)
650 assert "a.py::A1" in history
651 assert "b.py::B1" in history
652 assert "c.py::C1" in history
653
654
655 # ─────────────────────────────────────────────────────────────────────────────
656 # Layer 4 — Data Integrity
657 # ─────────────────────────────────────────────────────────────────────────────
658
659 class TestDataIntegrity:
660 @pytest.mark.asyncio
661 async def test_upsert_atomic_replace_removes_stale_entries(
662 self, db_session: AsyncSession
663 ) -> None:
664 """upsert_snapshot_entries must delete old entries before inserting new ones."""
665 from musehub.services.musehub_snapshot import (
666 get_snapshot_manifest,
667 upsert_snapshot_entries,
668 )
669 repo = await create_repo(db_session, slug="di-atomic")
670 snap_id = "snap-atomic"
671 await upsert_snapshot_entries(
672 db_session, repo.repo_id, snap_id,
673 {"old_file.py": "sha256:old", "shared.py": "sha256:shared"},
674 )
675 await db_session.commit()
676
677 # Replace with entirely different manifest
678 await upsert_snapshot_entries(
679 db_session, repo.repo_id, snap_id,
680 {"new_file.py": "sha256:new"},
681 )
682 await db_session.commit()
683
684 manifest = await get_snapshot_manifest(db_session, snap_id)
685 assert "new_file.py" in manifest
686 assert "old_file.py" not in manifest # atomically removed
687 assert "shared.py" not in manifest
688
689 @pytest.mark.asyncio
690 async def test_only_one_index_row_per_repo_after_multiple_builds(
691 self, db_session: AsyncSession
692 ) -> None:
693 from musehub.services.musehub_symbol_indexer import build_symbol_index
694 from sqlalchemy import select, func
695
696 repo = await create_repo(db_session, slug="di-onerow")
697 for i in range(3):
698 c = await _commit_with_delta(
699 db_session, repo.repo_id, f"di-c{i:03d}",
700 ops=[_insert_op(f"f{i}.py::Sym")],
701 )
702 await build_symbol_index(db_session, repo.repo_id, c.commit_id)
703 await db_session.commit()
704
705 count = (await db_session.execute(
706 select(func.count()).where(db.MusehubSymbolIndex.repo_id == repo.repo_id)
707 )).scalar_one()
708 assert count == 1
709
710 @pytest.mark.asyncio
711 async def test_symbol_history_includes_commit_id_and_timestamp(
712 self, db_session: AsyncSession
713 ) -> None:
714 from musehub.services.musehub_symbol_indexer import build_symbol_index
715
716 repo = await create_repo(db_session, slug="di-fields")
717 commit = await _commit_with_delta(
718 db_session, repo.repo_id, "di-field-001",
719 ops=[_insert_op("service.py::MyFn", "sha256:myfn")],
720 )
721 row = await build_symbol_index(db_session, repo.repo_id, commit.commit_id)
722 assert row is not None
723
724 history = msgpack.unpackb(row.symbol_history, raw=False)["entries"]
725 entry = history["service.py::MyFn"][0]
726 assert entry["commit_id"] == commit.commit_id
727 assert entry["committed_at"] != ""
728 assert entry["op"] == "add"
729 assert entry["content_id"] == "sha256:myfn"
730
731
732 # ─────────────────────────────────────────────────────────────────────────────
733 # Layer 5 — Security
734 # ─────────────────────────────────────────────────────────────────────────────
735
736 class TestSecurity:
737 @pytest.mark.asyncio
738 async def test_corrupt_msgpack_returns_empty_not_exception(
739 self, db_session: AsyncSession
740 ) -> None:
741 """A corrupt symbol_history blob must return {} — not raise."""
742 from musehub.services.musehub_symbol_indexer import load_symbol_history
743
744 repo = await create_repo(db_session, slug="sec-corrupt")
745 # Manually insert a row with garbage blob
746 index_row = db.MusehubSymbolIndex(
747 index_id=str(uuid.uuid4()),
748 repo_id=repo.repo_id,
749 ref="bad-ref",
750 symbol_history=b"\xff\xfe\xfd\xfc garbage",
751 hash_occurrence=b"\xff\xfe garbage",
752 built_at=_now(),
753 )
754 db_session.add(index_row)
755 await db_session.commit()
756
757 result = await load_symbol_history(db_session, repo.repo_id)
758 assert result == {}
759
760 @pytest.mark.asyncio
761 async def test_build_with_unknown_head_commit_returns_none(
762 self, db_session: AsyncSession
763 ) -> None:
764 """Unknown head_commit_id must return None, not raise."""
765 from musehub.services.musehub_symbol_indexer import build_symbol_index
766
767 repo = await create_repo(db_session, slug="sec-unknown-head")
768 result = await build_symbol_index(
769 db_session, repo.repo_id, "nonexistent-commit-id"
770 )
771 assert result is None
772
773 @pytest.mark.asyncio
774 async def test_corrupt_hash_occurrence_returns_empty(
775 self, db_session: AsyncSession
776 ) -> None:
777 from musehub.services.musehub_symbol_indexer import load_hash_occurrence
778
779 repo = await create_repo(db_session, slug="sec-corrupt-hash")
780 index_row = db.MusehubSymbolIndex(
781 index_id=str(uuid.uuid4()),
782 repo_id=repo.repo_id,
783 ref="bad-ref2",
784 symbol_history=None,
785 hash_occurrence=b"\x00corrupt",
786 built_at=_now(),
787 )
788 db_session.add(index_row)
789 await db_session.commit()
790
791 result = await load_hash_occurrence(db_session, repo.repo_id)
792 assert result == {}
793
794
795 # ─────────────────────────────────────────────────────────────────────────────
796 # Layer 6 — Stress
797 # ─────────────────────────────────────────────────────────────────────────────
798
799 class TestStress:
800 @pytest.mark.asyncio
801 async def test_upsert_1000_file_manifest(self, db_session: AsyncSession) -> None:
802 from musehub.services.musehub_snapshot import (
803 get_snapshot_manifest,
804 upsert_snapshot_entries,
805 )
806 repo = await create_repo(db_session, slug="stress-1k-snap")
807 snap_id = "snap-1k"
808 manifest = {f"src/file_{i:04d}.py": f"sha256:{i:04d}" for i in range(1000)}
809
810 await upsert_snapshot_entries(db_session, repo.repo_id, snap_id, manifest)
811 await db_session.commit()
812
813 result = await get_snapshot_manifest(db_session, snap_id)
814 assert len(result) == 1000
815 assert result["src/file_0500.py"] == "sha256:0500"
816
817 @pytest.mark.asyncio
818 async def test_batch_manifest_50_snapshots(self, db_session: AsyncSession) -> None:
819 from musehub.services.musehub_snapshot import (
820 get_snapshot_manifests_batch,
821 upsert_snapshot_entries,
822 )
823 repo = await create_repo(db_session, slug="stress-batch-50")
824 ids: list[str] = []
825 for i in range(50):
826 sid = f"stress-snap-{i:02d}"
827 ids.append(sid)
828 await upsert_snapshot_entries(
829 db_session, repo.repo_id, sid,
830 {f"f{i}.py": f"sha256:{i}"},
831 )
832 await db_session.commit()
833
834 result = await get_snapshot_manifests_batch(db_session, ids)
835 assert len(result) == 50
836 for i, sid in enumerate(ids):
837 assert f"f{i}.py" in result[sid]
838
839 @pytest.mark.asyncio
840 async def test_build_symbol_index_100_commits(
841 self, db_session: AsyncSession
842 ) -> None:
843 """100-commit chain with 5 ops each — indexer must complete successfully."""
844 from musehub.services.musehub_symbol_indexer import build_symbol_index
845
846 repo = await create_repo(db_session, slug="stress-100-commits")
847 prev_id: str | None = None
848 head_id = "stress-head"
849 for i in range(100):
850 cid = f"stress-{i:04d}" if i < 99 else head_id
851 ops = [_insert_op(f"file{i}.py::Sym{j}", f"sha256:{i}{j}") for j in range(5)]
852 commit = await _commit_with_delta(
853 db_session, repo.repo_id, cid, ops=ops,
854 parent_ids=[prev_id] if prev_id else [],
855 )
856 prev_id = commit.commit_id
857
858 row = await build_symbol_index(db_session, repo.repo_id, head_id)
859 await db_session.commit()
860 assert row is not None
861 history = msgpack.unpackb(row.symbol_history, raw=False)["entries"]
862 # 100 files × 5 symbols each = 500 top-level symbol entries
863 assert len(history) == 500
864
865 @pytest.mark.asyncio
866 async def test_load_symbol_history_file_filter_on_large_index(
867 self, db_session: AsyncSession
868 ) -> None:
869 """Filter on large index returns only matching addresses."""
870 from musehub.services.musehub_symbol_indexer import build_symbol_index, load_symbol_history
871
872 repo = await create_repo(db_session, slug="stress-filter-large")
873 # 50 files, each with 10 symbols
874 ops = []
875 for i in range(50):
876 for j in range(10):
877 ops.append(_insert_op(f"src/module_{i:02d}.py::Sym{j}", f"sha256:{i}{j}"))
878
879 commit = await _commit_with_delta(db_session, repo.repo_id, "stress-fl-head", ops=ops)
880 await build_symbol_index(db_session, repo.repo_id, commit.commit_id)
881 await db_session.commit()
882
883 result = await load_symbol_history(db_session, repo.repo_id, file_path="src/module_05.py")
884 # Only the 10 symbols for module_05 should be returned
885 assert len(result) == 10
886 for key in result:
887 assert key.startswith("src/module_05.py")
888
889
890 # ─────────────────────────────────────────────────────────────────────────────
891 # Layer 7 — Performance
892 # ─────────────────────────────────────────────────────────────────────────────
893
894 class TestPerformance:
895 def test_extract_ops_1000_calls_under_100ms(self) -> None:
896 from musehub.services.musehub_symbol_indexer import _extract_ops
897 meta = {
898 "structured_delta": {
899 "ops": [
900 _patch_op("main.py", [_insert_op("main.py::Foo"), _insert_op("main.py::Bar")]),
901 _insert_op("util.py::Helper"),
902 ]
903 }
904 }
905 t0 = time.perf_counter()
906 for _ in range(1000):
907 _extract_ops(meta)
908 elapsed_ms = (time.perf_counter() - t0) * 1000
909 assert elapsed_ms < 100, f"_extract_ops 1000 calls took {elapsed_ms:.1f}ms"
910
911 @pytest.mark.asyncio
912 async def test_build_symbol_index_100_commits_under_3s(
913 self, db_session: AsyncSession
914 ) -> None:
915 from musehub.services.musehub_symbol_indexer import build_symbol_index
916
917 repo = await create_repo(db_session, slug="perf-idx-100c")
918 prev_id: str | None = None
919 for i in range(100):
920 cid = f"perf-{i:04d}"
921 ops = [_insert_op(f"p{i}.py::S{j}") for j in range(5)]
922 c = await _commit_with_delta(db_session, repo.repo_id, cid, ops=ops,
923 parent_ids=[prev_id] if prev_id else [])
924 prev_id = c.commit_id
925
926 t0 = time.perf_counter()
927 row = await build_symbol_index(db_session, repo.repo_id, prev_id)
928 elapsed_ms = (time.perf_counter() - t0) * 1000
929 assert row is not None
930 assert elapsed_ms < 3000, f"build_symbol_index 100 commits took {elapsed_ms:.0f}ms"
931
932 @pytest.mark.asyncio
933 async def test_upsert_1000_files_under_500ms(self, db_session: AsyncSession) -> None:
934 from musehub.services.musehub_snapshot import upsert_snapshot_entries
935
936 repo = await create_repo(db_session, slug="perf-upsert-1k")
937 manifest = {f"f{i:04d}.py": f"sha256:{i}" for i in range(1000)}
938
939 t0 = time.perf_counter()
940 await upsert_snapshot_entries(db_session, repo.repo_id, "perf-snap", manifest)
941 elapsed_ms = (time.perf_counter() - t0) * 1000
942 assert elapsed_ms < 500, f"upsert 1000 entries took {elapsed_ms:.0f}ms"
943
944 @pytest.mark.asyncio
945 async def test_batch_manifest_50_snapshots_under_300ms(
946 self, db_session: AsyncSession
947 ) -> None:
948 from musehub.services.musehub_snapshot import (
949 get_snapshot_manifests_batch,
950 upsert_snapshot_entries,
951 )
952 repo = await create_repo(db_session, slug="perf-batch-50")
953 ids: list[str] = []
954 for i in range(50):
955 sid = f"perf-batch-{i:02d}"
956 ids.append(sid)
957 await upsert_snapshot_entries(db_session, repo.repo_id, sid,
958 {f"f{i}.py": f"sha256:{i}"})
959 await db_session.commit()
960
961 # Warm-up
962 await get_snapshot_manifests_batch(db_session, ids)
963
964 t0 = time.perf_counter()
965 result = await get_snapshot_manifests_batch(db_session, ids)
966 elapsed_ms = (time.perf_counter() - t0) * 1000
967 assert len(result) == 50
968 assert elapsed_ms < 300, f"batch 50 snapshots took {elapsed_ms:.0f}ms"
File History 1 commit
sha256:a10adeeb7a0169cb9900f9806ed7a973047258abb6283724fe55e8eb68ff3f0a init: musehub initial commit Human 171 days ago