test_object_bytes_all_forms.py
python
sha256:b77a9ac02fb6c01305ab105f50d094b255d60015247f33e229cc81e4b694a013
fix(objects): a loose-blob row whose blob is absent reads f…
Opus 5.5
patch
2 days ago
| 1 | """Objects stored INSIDE an mpack must be served like any other object. |
| 2 | |
| 3 | Demonstrated defect (staging, 2026-11-14): the object-content API, the blob-metadata API, the sync pull, the incremental fetch, release analysis and |
| 4 | the integrity scan read only the loose blob key (``backend.get(object_id)``), which knows nothing of ``mpack://`` storage. Six objects whose bytes |
| 5 | were intact inside their push mpacks answered ``410 Gone`` ("removed from storage"); the incremental fetch would ship such an object as EMPTY content. |
| 6 | Every reader now goes through ``read_object_bytes`` (cache, s3, mpack byte range or whole pack); paths that must not serve a wrong answer use |
| 7 | ``read_object_bytes_required``, which raises instead of substituting ``b""``. |
| 8 | """ |
| 9 | from __future__ import annotations |
| 10 | |
| 11 | import pytest |
| 12 | from httpx import AsyncClient |
| 13 | from sqlalchemy.ext.asyncio import AsyncSession |
| 14 | |
| 15 | from muse.core.types import blob_id |
| 16 | from musehub.db.musehub_repo_models import MusehubMPackIndex, MusehubObject, MusehubObjectRef |
| 17 | from musehub.storage import backends as B |
| 18 | from tests.factories import create_repo |
| 19 | |
| 20 | DATA = b'"""A file whose bytes live only inside a push mpack."""\nX = 1\n' |
| 21 | |
| 22 | |
| 23 | async def _packed_object(db: AsyncSession, repo_id: str, data: bytes = DATA, *, indexed: bool = True) -> str: |
| 24 | oid = blob_id(data) |
| 25 | key = "sha256:" + "ab" * 32 |
| 26 | await B.get_backend().put_mpack(key, b"MUSE-HEADER-" + data + b"-TRAILER") |
| 27 | db.add(MusehubObject(object_id=oid, path="pkg/packed.py", size_bytes=len(data), storage_uri=f"mpack://{key}")) |
| 28 | db.add(MusehubObjectRef(repo_id=repo_id, object_id=oid)) |
| 29 | if indexed: |
| 30 | db.add(MusehubMPackIndex(entity_id=oid, mpack_id=key, entity_type="object", byte_offset=len(b"MUSE-HEADER-"), byte_length=len(data))) |
| 31 | await db.commit() |
| 32 | assert await B.get_backend().get(oid) is None # NOT a loose blob: only the mpack holds it |
| 33 | return oid |
| 34 | |
| 35 | |
| 36 | @pytest.mark.asyncio |
| 37 | async def test_content_api_serves_an_mpack_stored_object(client: AsyncClient, db_session: AsyncSession) -> None: |
| 38 | repo = await create_repo(db_session, visibility="public") |
| 39 | oid = await _packed_object(db_session, repo.repo_id) |
| 40 | r = await client.get(f"/api/repos/{repo.repo_id}/objects/{oid}/content") |
| 41 | assert r.status_code == 200, r.text |
| 42 | assert r.content == DATA |
| 43 | |
| 44 | |
| 45 | @pytest.mark.asyncio |
| 46 | async def test_bulk_reader_reads_every_form_concurrently_and_fails_closed(db_session: AsyncSession) -> None: |
| 47 | repo = await create_repo(db_session, visibility="public") |
| 48 | packed = await _packed_object(db_session, repo.repo_id) |
| 49 | unindexed_data = b"in a pack but not indexed: whole-pack fallback\n" |
| 50 | loose_data, cached_data = b"a loose blob\n", b"held in content_cache\n" |
| 51 | loose, cached, unindexed = blob_id(loose_data), blob_id(cached_data), blob_id(unindexed_data) |
| 52 | from muse.core.mpack import build_wire_mpack |
| 53 | pack_key = "sha256:" + "ef" * 32 |
| 54 | await B.get_backend().put_mpack(pack_key, build_wire_mpack({"commits": [], "snapshots": [], "blobs": [{"object_id": unindexed, "content": unindexed_data}], "tags": []})) |
| 55 | uri = await B.get_backend().put(loose, loose_data) |
| 56 | db_session.add_all([ |
| 57 | MusehubObject(object_id=loose, path="l.txt", size_bytes=len(loose_data), storage_uri=uri), |
| 58 | MusehubObject(object_id=cached, path="c.txt", size_bytes=len(cached_data), storage_uri="mpack://sha256:" + "00" * 32, content_cache=cached_data), |
| 59 | MusehubObject(object_id=unindexed, path="u.txt", size_bytes=len(unindexed_data), storage_uri=f"mpack://{pack_key}"), |
| 60 | ]) |
| 61 | await db_session.commit() |
| 62 | rows = [await db_session.get(MusehubObject, o) for o in (packed, loose, cached, unindexed)] |
| 63 | got = await B.read_objects_bytes_required(rows, db_session) |
| 64 | assert got == {packed: DATA, loose: loose_data, cached: cached_data, unindexed: unindexed_data} |
| 65 | |
| 66 | gone = MusehubObject(object_id=blob_id(b"nowhere"), path="g.txt", size_bytes=7, storage_uri="mpack://sha256:" + "11" * 32) |
| 67 | with pytest.raises(B.ObjectBytesUnavailable) as exc: |
| 68 | await B.read_objects_bytes_required([*rows, gone], db_session) |
| 69 | assert exc.value.object_ids == [gone.object_id] |
| 70 | |
| 71 | |
| 72 | @pytest.mark.asyncio |
| 73 | async def test_required_read_raises_instead_of_inventing_empty_content(db_session: AsyncSession) -> None: |
| 74 | missing = MusehubObject(object_id=blob_id(b"absent bytes"), path="x.py", size_bytes=12, storage_uri="mpack://sha256:" + "cd" * 32) |
| 75 | with pytest.raises(B.ObjectBytesUnavailable): |
| 76 | await B.read_object_bytes_required(missing, session=db_session) |
| 77 | empty = MusehubObject(object_id=blob_id(b""), path="empty.py", size_bytes=0, storage_uri="mpack://sha256:" + "cd" * 32) |
| 78 | assert await B.read_object_bytes_required(empty, session=db_session) == b"" |
| 79 | |
| 80 | |
| 81 | @pytest.mark.asyncio |
| 82 | async def test_integrity_scan_does_not_report_mpack_objects_missing(db_session: AsyncSession) -> None: |
| 83 | from musehub.maintenance.object_integrity import scan_object_integrity |
| 84 | repo = await create_repo(db_session, visibility="public") |
| 85 | oid = await _packed_object(db_session, repo.repo_id) |
| 86 | result = await scan_object_integrity(db_session, B.get_backend(), sample_size=10_000) |
| 87 | assert oid not in {m.object_id for m in result.mismatches} |
| 88 | |
| 89 | |
| 90 | @pytest.mark.asyncio |
| 91 | async def test_loose_row_whose_blob_is_absent_reads_from_an_indexed_mpack(db_session: AsyncSession) -> None: |
| 92 | """Staging: 669 gabriel/muse rows said s3:// but no loose blob existed; their bytes were indexed inside push mpacks.""" |
| 93 | data = b"recorded as a loose blob, present only inside an mpack\n" |
| 94 | oid = blob_id(data) |
| 95 | key = "sha256:" + "fa" * 32 |
| 96 | await B.get_backend().put_mpack(key, b"HDR" + data) |
| 97 | db_session.add(MusehubObject(object_id=oid, path="l.py", size_bytes=len(data), storage_uri=f"s3://bucket/objects/{oid}")) |
| 98 | db_session.add(MusehubMPackIndex(entity_id=oid, mpack_id=key, entity_type="object", byte_offset=3, byte_length=len(data))) |
| 99 | await db_session.commit() |
| 100 | row = await db_session.get(MusehubObject, oid) |
| 101 | assert await B.get_backend().get(oid) is None |
| 102 | assert await B.read_object_bytes(row, session=db_session) == data |
| 103 | assert (await B.read_objects_bytes_required([row], db_session))[oid] == data |
File History
3 commits
sha256:b77a9ac02fb6c01305ab105f50d094b255d60015247f33e229cc81e4b694a013
fix(objects): a loose-blob row whose blob is absent reads f…
Opus 5.5
patch
2 days ago
sha256:9e7fb25b45eb883646cdd15c4c6ed27f0ca88cf6259a90d3067cfaba744f0ea3
fix(objects): review of a0c73d55 — fetch/sync read concurre…
Opus 5.5
minor
⚠
2 days ago
sha256:69161439990eeb42a0502d2417d8492b076c88e5ca0fb842dbb1bfb1ef8fca7c
fix(objects): P0 object availability — serve mpack-stored o…
Opus 5.5
minor
⚠
2 days ago