test_mist_phase3_snapshot_indexer.py
python
sha256:a34090cc4a394a78bd72cbbe34b08cc59525141e19135b6c0ab154f10611b9ef
debug(push/stream): instrument O-frame decode path with INF…
Sonnet 4.6
patch
121 days ago
| 1 | """Phase 3 TDD: Mist snapshot indexer — symbol anchor extraction on push. |
| 2 | |
| 3 | Tests are written RED first. Run before touching musehub_mist_indexer.py |
| 4 | and musehub_intel_providers.py to confirm they fail, then implement to green. |
| 5 | |
| 6 | The indexer reads a mist repo's HEAD commit snapshot manifest, loads each |
| 7 | artifact's bytes from the object store, extracts symbol anchors, and writes |
| 8 | normalized rows to: |
| 9 | musehub_symbol_history_entries — one row per (repo_id, address, commit_id) |
| 10 | musehub_symbol_intel — one row per (repo_id, address) |
| 11 | |
| 12 | This makes mist anchors discoverable via muse code grep / code impact across |
| 13 | the entire hub, using the same infrastructure as code-domain symbols. |
| 14 | |
| 15 | Idempotency: indexing the same commit twice must produce the same row count. |
| 16 | """ |
| 17 | from __future__ import annotations |
| 18 | |
| 19 | import secrets |
| 20 | from datetime import datetime, timezone |
| 21 | |
| 22 | import msgpack |
| 23 | import pytest |
| 24 | from muse.core.types import blob_id |
| 25 | from sqlalchemy import func, select |
| 26 | from sqlalchemy.ext.asyncio import AsyncSession |
| 27 | |
| 28 | from musehub.core.genesis import compute_identity_id, compute_repo_id |
| 29 | from musehub.db import musehub_models as db |
| 30 | from musehub.types.json_types import StrDict |
| 31 | |
| 32 | |
| 33 | # --------------------------------------------------------------------------- |
| 34 | # Helpers |
| 35 | # --------------------------------------------------------------------------- |
| 36 | |
| 37 | def _now() -> datetime: |
| 38 | return datetime.now(tz=timezone.utc) |
| 39 | |
| 40 | |
| 41 | def _oid(content: bytes) -> str: |
| 42 | return blob_id(content) |
| 43 | |
| 44 | |
| 45 | def _manifest_blob(manifest: StrDict) -> bytes: |
| 46 | return msgpack.packb(manifest, use_bin_type=True) |
| 47 | |
| 48 | |
| 49 | def _commit_id() -> str: |
| 50 | return blob_id(secrets.token_bytes(16)) |
| 51 | |
| 52 | |
| 53 | def _snap_id(manifest: StrDict) -> str: |
| 54 | return blob_id(msgpack.packb(sorted(manifest.items()), use_bin_type=True)) |
| 55 | |
| 56 | |
| 57 | async def _seed_mist_vcs_repo( |
| 58 | session: AsyncSession, |
| 59 | *, |
| 60 | owner: str = "testuser", |
| 61 | artifacts: dict[str, bytes], # filename → raw content bytes |
| 62 | ) -> tuple[db.MusehubRepo, db.MusehubCommit]: |
| 63 | """Create a mist repo with a commit pointing at a snapshot of the given artifacts. |
| 64 | |
| 65 | Each artifact becomes a MusehubObject with content_cache populated so |
| 66 | read_object_bytes() can serve it without hitting disk or S3. |
| 67 | """ |
| 68 | owner_id = compute_identity_id(owner.encode()) |
| 69 | slug = f"mist-{secrets.token_hex(4)}" |
| 70 | created_at = _now() |
| 71 | repo_id = compute_repo_id(owner_id, slug, "mist", created_at.isoformat()) |
| 72 | |
| 73 | repo = db.MusehubRepo( |
| 74 | repo_id=repo_id, |
| 75 | name=slug, |
| 76 | owner=owner, |
| 77 | slug=slug, |
| 78 | visibility="public", |
| 79 | owner_user_id=owner_id, |
| 80 | domain_id="mist", |
| 81 | description="", |
| 82 | tags=[], |
| 83 | created_at=created_at, |
| 84 | ) |
| 85 | session.add(repo) |
| 86 | await session.flush() |
| 87 | |
| 88 | # Create MusehubObject rows with content_cache for each artifact. |
| 89 | manifest: dict[str, str] = {} |
| 90 | for filename, raw in artifacts.items(): |
| 91 | oid = _oid(raw) |
| 92 | manifest[filename] = oid |
| 93 | obj = db.MusehubObject( |
| 94 | object_id=oid, |
| 95 | path=filename, |
| 96 | size_bytes=len(raw), |
| 97 | disk_path="", |
| 98 | content_cache=raw, |
| 99 | ) |
| 100 | # ON CONFLICT DO NOTHING — same bytes may appear in multiple artifacts. |
| 101 | existing = await session.get(db.MusehubObject, oid) |
| 102 | if existing is None: |
| 103 | session.add(obj) |
| 104 | await session.flush() |
| 105 | |
| 106 | # Create snapshot row. |
| 107 | snap_id = _snap_id(manifest) |
| 108 | snap = db.MusehubSnapshot( |
| 109 | snapshot_id=snap_id, |
| 110 | repo_id=repo_id, |
| 111 | entry_count=len(manifest), |
| 112 | manifest_blob=_manifest_blob(manifest), |
| 113 | ) |
| 114 | existing_snap = await session.get(db.MusehubSnapshot, snap_id) |
| 115 | if existing_snap is None: |
| 116 | session.add(snap) |
| 117 | await session.flush() |
| 118 | |
| 119 | # Create commit row pointing at the snapshot. |
| 120 | cid = _commit_id() |
| 121 | commit = db.MusehubCommit( |
| 122 | commit_id=cid, |
| 123 | repo_id=repo_id, |
| 124 | message="initial mist", |
| 125 | author=owner, |
| 126 | branch="main", |
| 127 | parent_ids=[], |
| 128 | snapshot_id=snap_id, |
| 129 | timestamp=_now(), |
| 130 | ) |
| 131 | session.add(commit) |
| 132 | await session.commit() |
| 133 | await session.refresh(repo) |
| 134 | await session.refresh(commit) |
| 135 | return repo, commit |
| 136 | |
| 137 | |
| 138 | # --------------------------------------------------------------------------- |
| 139 | # 1. build_mist_anchor_index exists and is importable |
| 140 | # --------------------------------------------------------------------------- |
| 141 | |
| 142 | class TestBuildMistAnchorIndexExists: |
| 143 | def test_function_is_importable(self) -> None: |
| 144 | from musehub.services.musehub_mist_indexer import build_mist_anchor_index |
| 145 | import inspect |
| 146 | assert inspect.iscoroutinefunction(build_mist_anchor_index) |
| 147 | |
| 148 | def test_function_signature(self) -> None: |
| 149 | from musehub.services.musehub_mist_indexer import build_mist_anchor_index |
| 150 | import inspect |
| 151 | sig = inspect.signature(build_mist_anchor_index) |
| 152 | assert "repo_id" in sig.parameters |
| 153 | assert "head_commit_id" in sig.parameters |
| 154 | |
| 155 | |
| 156 | # --------------------------------------------------------------------------- |
| 157 | # 2. Anchor extraction → musehub_symbol_history_entries |
| 158 | # --------------------------------------------------------------------------- |
| 159 | |
| 160 | class TestMistAnchorIndexerHistoryEntries: |
| 161 | @pytest.mark.asyncio |
| 162 | async def test_python_artifact_writes_history_entries( |
| 163 | self, db_session: AsyncSession, test_user: db.MusehubIdentity |
| 164 | ) -> None: |
| 165 | from musehub.services.musehub_mist_indexer import build_mist_anchor_index |
| 166 | |
| 167 | repo, commit = await _seed_mist_vcs_repo( |
| 168 | db_session, |
| 169 | owner=test_user.handle, |
| 170 | artifacts={ |
| 171 | "utils.py": b"def add(a, b):\n return a + b\n\ndef sub(a, b):\n return a - b\n", |
| 172 | }, |
| 173 | ) |
| 174 | |
| 175 | await build_mist_anchor_index(db_session, repo.repo_id, commit.commit_id) |
| 176 | await db_session.commit() |
| 177 | |
| 178 | rows = (await db_session.execute( |
| 179 | select(db.MusehubSymbolHistoryEntry).where( |
| 180 | db.MusehubSymbolHistoryEntry.repo_id == repo.repo_id, |
| 181 | ) |
| 182 | )).scalars().all() |
| 183 | |
| 184 | addresses = {r.address for r in rows} |
| 185 | assert any("add" in a for a in addresses), f"Expected 'add' anchor; got {addresses}" |
| 186 | assert any("sub" in a for a in addresses), f"Expected 'sub' anchor; got {addresses}" |
| 187 | |
| 188 | @pytest.mark.asyncio |
| 189 | async def test_history_entry_fields( |
| 190 | self, db_session: AsyncSession, test_user: db.MusehubIdentity |
| 191 | ) -> None: |
| 192 | from musehub.services.musehub_mist_indexer import build_mist_anchor_index |
| 193 | |
| 194 | content = b"def process(x):\n return x\n" |
| 195 | repo, commit = await _seed_mist_vcs_repo( |
| 196 | db_session, |
| 197 | owner=test_user.handle, |
| 198 | artifacts={"module.py": content}, |
| 199 | ) |
| 200 | |
| 201 | await build_mist_anchor_index(db_session, repo.repo_id, commit.commit_id) |
| 202 | await db_session.commit() |
| 203 | |
| 204 | row = (await db_session.execute( |
| 205 | select(db.MusehubSymbolHistoryEntry).where( |
| 206 | db.MusehubSymbolHistoryEntry.repo_id == repo.repo_id, |
| 207 | db.MusehubSymbolHistoryEntry.address.like("module.py::%"), |
| 208 | ) |
| 209 | )).scalars().first() |
| 210 | |
| 211 | assert row is not None |
| 212 | assert row.commit_id == commit.commit_id |
| 213 | assert row.author == test_user.handle |
| 214 | assert row.op in ("add", "modify") |
| 215 | assert row.committed_at is not None |
| 216 | |
| 217 | @pytest.mark.asyncio |
| 218 | async def test_multiple_artifacts_all_indexed( |
| 219 | self, db_session: AsyncSession, test_user: db.MusehubIdentity |
| 220 | ) -> None: |
| 221 | from musehub.services.musehub_mist_indexer import build_mist_anchor_index |
| 222 | |
| 223 | repo, commit = await _seed_mist_vcs_repo( |
| 224 | db_session, |
| 225 | owner=test_user.handle, |
| 226 | artifacts={ |
| 227 | "a.py": b"def alpha(): pass\n", |
| 228 | "b.py": b"def beta(): pass\n", |
| 229 | }, |
| 230 | ) |
| 231 | |
| 232 | await build_mist_anchor_index(db_session, repo.repo_id, commit.commit_id) |
| 233 | await db_session.commit() |
| 234 | |
| 235 | rows = (await db_session.execute( |
| 236 | select(db.MusehubSymbolHistoryEntry).where( |
| 237 | db.MusehubSymbolHistoryEntry.repo_id == repo.repo_id, |
| 238 | ) |
| 239 | )).scalars().all() |
| 240 | |
| 241 | addresses = {r.address for r in rows} |
| 242 | assert any("alpha" in a for a in addresses) |
| 243 | assert any("beta" in a for a in addresses) |
| 244 | |
| 245 | @pytest.mark.asyncio |
| 246 | async def test_binary_artifact_produces_no_history_entries( |
| 247 | self, db_session: AsyncSession, test_user: db.MusehubIdentity |
| 248 | ) -> None: |
| 249 | from musehub.services.musehub_mist_indexer import build_mist_anchor_index |
| 250 | |
| 251 | repo, commit = await _seed_mist_vcs_repo( |
| 252 | db_session, |
| 253 | owner=test_user.handle, |
| 254 | artifacts={"image.png": b"\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR\x00"}, |
| 255 | ) |
| 256 | |
| 257 | await build_mist_anchor_index(db_session, repo.repo_id, commit.commit_id) |
| 258 | await db_session.commit() |
| 259 | |
| 260 | count = (await db_session.execute( |
| 261 | select(func.count()).select_from(db.MusehubSymbolHistoryEntry).where( |
| 262 | db.MusehubSymbolHistoryEntry.repo_id == repo.repo_id, |
| 263 | ) |
| 264 | )).scalar_one() |
| 265 | |
| 266 | assert count == 0 |
| 267 | |
| 268 | |
| 269 | # --------------------------------------------------------------------------- |
| 270 | # 3. Anchor extraction → musehub_symbol_intel |
| 271 | # --------------------------------------------------------------------------- |
| 272 | |
| 273 | class TestMistAnchorIndexerSymbolIntel: |
| 274 | @pytest.mark.asyncio |
| 275 | async def test_python_artifact_writes_symbol_intel( |
| 276 | self, db_session: AsyncSession, test_user: db.MusehubIdentity |
| 277 | ) -> None: |
| 278 | from musehub.services.musehub_mist_indexer import build_mist_anchor_index |
| 279 | |
| 280 | repo, commit = await _seed_mist_vcs_repo( |
| 281 | db_session, |
| 282 | owner=test_user.handle, |
| 283 | artifacts={"calc.py": b"def mul(a, b):\n return a * b\n"}, |
| 284 | ) |
| 285 | |
| 286 | await build_mist_anchor_index(db_session, repo.repo_id, commit.commit_id) |
| 287 | await db_session.commit() |
| 288 | |
| 289 | rows = (await db_session.execute( |
| 290 | select(db.MusehubSymbolIntel).where( |
| 291 | db.MusehubSymbolIntel.repo_id == repo.repo_id, |
| 292 | ) |
| 293 | )).scalars().all() |
| 294 | |
| 295 | assert len(rows) >= 1 |
| 296 | addresses = {r.address for r in rows} |
| 297 | assert any("mul" in a for a in addresses) |
| 298 | |
| 299 | @pytest.mark.asyncio |
| 300 | async def test_symbol_intel_churn_is_at_least_one( |
| 301 | self, db_session: AsyncSession, test_user: db.MusehubIdentity |
| 302 | ) -> None: |
| 303 | from musehub.services.musehub_mist_indexer import build_mist_anchor_index |
| 304 | |
| 305 | repo, commit = await _seed_mist_vcs_repo( |
| 306 | db_session, |
| 307 | owner=test_user.handle, |
| 308 | artifacts={"api.py": b"async def fetch(url):\n pass\n"}, |
| 309 | ) |
| 310 | |
| 311 | await build_mist_anchor_index(db_session, repo.repo_id, commit.commit_id) |
| 312 | await db_session.commit() |
| 313 | |
| 314 | row = (await db_session.execute( |
| 315 | select(db.MusehubSymbolIntel).where( |
| 316 | db.MusehubSymbolIntel.repo_id == repo.repo_id, |
| 317 | db.MusehubSymbolIntel.address.like("api.py::%"), |
| 318 | ) |
| 319 | )).scalars().first() |
| 320 | |
| 321 | assert row is not None |
| 322 | assert row.churn >= 1 |
| 323 | |
| 324 | |
| 325 | # --------------------------------------------------------------------------- |
| 326 | # 4. Idempotency |
| 327 | # --------------------------------------------------------------------------- |
| 328 | |
| 329 | class TestMistAnchorIndexerIdempotency: |
| 330 | @pytest.mark.asyncio |
| 331 | async def test_indexing_same_commit_twice_is_idempotent( |
| 332 | self, db_session: AsyncSession, test_user: db.MusehubIdentity |
| 333 | ) -> None: |
| 334 | from musehub.services.musehub_mist_indexer import build_mist_anchor_index |
| 335 | |
| 336 | repo, commit = await _seed_mist_vcs_repo( |
| 337 | db_session, |
| 338 | owner=test_user.handle, |
| 339 | artifacts={"ops.py": b"def create(): pass\ndef delete(): pass\n"}, |
| 340 | ) |
| 341 | |
| 342 | for _ in range(2): |
| 343 | await build_mist_anchor_index(db_session, repo.repo_id, commit.commit_id) |
| 344 | await db_session.commit() |
| 345 | |
| 346 | history_count = (await db_session.execute( |
| 347 | select(func.count()).select_from(db.MusehubSymbolHistoryEntry).where( |
| 348 | db.MusehubSymbolHistoryEntry.repo_id == repo.repo_id, |
| 349 | ) |
| 350 | )).scalar_one() |
| 351 | |
| 352 | intel_count = (await db_session.execute( |
| 353 | select(func.count()).select_from(db.MusehubSymbolIntel).where( |
| 354 | db.MusehubSymbolIntel.repo_id == repo.repo_id, |
| 355 | ) |
| 356 | )).scalar_one() |
| 357 | |
| 358 | assert history_count == intel_count, ( |
| 359 | "Each anchor should produce exactly one history entry and one intel row" |
| 360 | ) |
| 361 | # Verify rows are present (not zero from double-delete or something) |
| 362 | assert history_count >= 2, f"Expected ≥2 anchors for create+delete; got {history_count}" |
| 363 | |
| 364 | |
| 365 | # --------------------------------------------------------------------------- |
| 366 | # 5. Edge cases |
| 367 | # --------------------------------------------------------------------------- |
| 368 | |
| 369 | class TestMistAnchorIndexerEdgeCases: |
| 370 | @pytest.mark.asyncio |
| 371 | async def test_commit_without_snapshot_returns_empty( |
| 372 | self, db_session: AsyncSession, test_user: db.MusehubIdentity |
| 373 | ) -> None: |
| 374 | from musehub.services.musehub_mist_indexer import build_mist_anchor_index |
| 375 | from musehub.core.genesis import compute_repo_id |
| 376 | |
| 377 | owner_id = compute_identity_id(test_user.handle.encode()) |
| 378 | created_at = _now() |
| 379 | repo_id = compute_repo_id(owner_id, "no-snap", "mist", created_at.isoformat()) |
| 380 | repo = db.MusehubRepo( |
| 381 | repo_id=repo_id, name="no-snap", owner=test_user.handle, |
| 382 | slug="no-snap", visibility="public", owner_user_id=owner_id, |
| 383 | domain_id="mist", description="", tags=[], created_at=created_at, |
| 384 | ) |
| 385 | db_session.add(repo) |
| 386 | |
| 387 | cid = _commit_id() |
| 388 | commit = db.MusehubCommit( |
| 389 | commit_id=cid, repo_id=repo_id, message="empty", |
| 390 | author=test_user.handle, branch="main", parent_ids=[], |
| 391 | snapshot_id=None, timestamp=_now(), |
| 392 | ) |
| 393 | db_session.add(commit) |
| 394 | await db_session.commit() |
| 395 | |
| 396 | result = await build_mist_anchor_index(db_session, repo_id, cid) |
| 397 | assert result == [] |
| 398 | |
| 399 | @pytest.mark.asyncio |
| 400 | async def test_object_missing_from_store_is_skipped( |
| 401 | self, db_session: AsyncSession, test_user: db.MusehubIdentity |
| 402 | ) -> None: |
| 403 | """Object_id in manifest but no MusehubObject row → skip gracefully.""" |
| 404 | from musehub.services.musehub_mist_indexer import build_mist_anchor_index |
| 405 | |
| 406 | owner_id = compute_identity_id(test_user.handle.encode()) |
| 407 | created_at = _now() |
| 408 | repo_id = compute_repo_id(owner_id, "ghost-obj", "mist", created_at.isoformat()) |
| 409 | repo = db.MusehubRepo( |
| 410 | repo_id=repo_id, name="ghost-obj", owner=test_user.handle, |
| 411 | slug="ghost-obj", visibility="public", owner_user_id=owner_id, |
| 412 | domain_id="mist", description="", tags=[], created_at=created_at, |
| 413 | ) |
| 414 | db_session.add(repo) |
| 415 | await db_session.flush() |
| 416 | |
| 417 | ghost_oid = blob_id(b"ghost content that has no DB row") |
| 418 | manifest = {"ghost.py": ghost_oid} |
| 419 | snap_id = _snap_id(manifest) |
| 420 | snap = db.MusehubSnapshot( |
| 421 | snapshot_id=snap_id, repo_id=repo_id, entry_count=1, |
| 422 | manifest_blob=_manifest_blob(manifest), |
| 423 | ) |
| 424 | db_session.add(snap) |
| 425 | await db_session.flush() |
| 426 | |
| 427 | cid = _commit_id() |
| 428 | commit = db.MusehubCommit( |
| 429 | commit_id=cid, repo_id=repo_id, message="ghost", |
| 430 | author=test_user.handle, branch="main", parent_ids=[], |
| 431 | snapshot_id=snap_id, timestamp=_now(), |
| 432 | ) |
| 433 | db_session.add(commit) |
| 434 | await db_session.commit() |
| 435 | |
| 436 | # Must not raise — silently skips the missing object. |
| 437 | result = await build_mist_anchor_index(db_session, repo_id, cid) |
| 438 | assert isinstance(result, list) |
| 439 | |
| 440 | @pytest.mark.asyncio |
| 441 | async def test_returns_intel_result_tuple( |
| 442 | self, db_session: AsyncSession, test_user: db.MusehubIdentity |
| 443 | ) -> None: |
| 444 | from musehub.services.musehub_mist_indexer import build_mist_anchor_index |
| 445 | |
| 446 | repo, commit = await _seed_mist_vcs_repo( |
| 447 | db_session, |
| 448 | owner=test_user.handle, |
| 449 | artifacts={"result.py": b"def answer(): return 42\n"}, |
| 450 | ) |
| 451 | |
| 452 | result = await build_mist_anchor_index(db_session, repo.repo_id, commit.commit_id) |
| 453 | |
| 454 | assert len(result) == 1 |
| 455 | intel_type, data = result[0] |
| 456 | assert intel_type == "mist.anchor_index" |
| 457 | assert "anchor_count" in data |
| 458 | assert data["anchor_count"] >= 1 |
| 459 | |
| 460 | |
| 461 | # --------------------------------------------------------------------------- |
| 462 | # 6. MistProvider delegates to build_mist_anchor_index |
| 463 | # --------------------------------------------------------------------------- |
| 464 | |
| 465 | class TestMistProviderDelegatesToIndexer: |
| 466 | @pytest.mark.asyncio |
| 467 | async def test_mist_provider_writes_normalized_rows( |
| 468 | self, db_session: AsyncSession, test_user: db.MusehubIdentity |
| 469 | ) -> None: |
| 470 | """MistProvider.compute triggers the normalized indexer for VCS-backed mists.""" |
| 471 | from musehub.services.musehub_intel_providers import _PROVIDER_REGISTRY |
| 472 | |
| 473 | provider = _PROVIDER_REGISTRY["intel.mist"] |
| 474 | repo, commit = await _seed_mist_vcs_repo( |
| 475 | db_session, |
| 476 | owner=test_user.handle, |
| 477 | artifacts={"svc.py": b"class Service:\n def run(self): pass\n"}, |
| 478 | ) |
| 479 | |
| 480 | await provider.compute(db_session, repo.repo_id, commit.commit_id, {}) |
| 481 | await db_session.commit() |
| 482 | |
| 483 | rows = (await db_session.execute( |
| 484 | select(db.MusehubSymbolHistoryEntry).where( |
| 485 | db.MusehubSymbolHistoryEntry.repo_id == repo.repo_id, |
| 486 | ) |
| 487 | )).scalars().all() |
| 488 | |
| 489 | assert len(rows) >= 1, "MistProvider must write normalized symbol history entries" |
File History
1 commit
sha256:a34090cc4a394a78bd72cbbe34b08cc59525141e19135b6c0ab154f10611b9ef
debug(push/stream): instrument O-frame decode path with INF…
Sonnet 4.6
patch
121 days ago