gabriel / musehub public
test_object_bytes_all_forms.py python
103 lines 6.0 KB
Raw
sha256:b77a9ac02fb6c01305ab105f50d094b255d60015247f33e229cc81e4b694a013 fix(objects): a loose-blob row whose blob is absent reads f… Opus 5.5 patch 2 days ago
1 """Objects stored INSIDE an mpack must be served like any other object.
2
3 Demonstrated defect (staging, 2026-11-14): the object-content API, the blob-metadata API, the sync pull, the incremental fetch, release analysis and
4 the integrity scan read only the loose blob key (``backend.get(object_id)``), which knows nothing of ``mpack://`` storage. Six objects whose bytes
5 were intact inside their push mpacks answered ``410 Gone`` ("removed from storage"); the incremental fetch would ship such an object as EMPTY content.
6 Every reader now goes through ``read_object_bytes`` (cache, s3, mpack byte range or whole pack); paths that must not serve a wrong answer use
7 ``read_object_bytes_required``, which raises instead of substituting ``b""``.
8 """
9 from __future__ import annotations
10
11 import pytest
12 from httpx import AsyncClient
13 from sqlalchemy.ext.asyncio import AsyncSession
14
15 from muse.core.types import blob_id
16 from musehub.db.musehub_repo_models import MusehubMPackIndex, MusehubObject, MusehubObjectRef
17 from musehub.storage import backends as B
18 from tests.factories import create_repo
19
20 DATA = b'"""A file whose bytes live only inside a push mpack."""\nX = 1\n'
21
22
23 async def _packed_object(db: AsyncSession, repo_id: str, data: bytes = DATA, *, indexed: bool = True) -> str:
24 oid = blob_id(data)
25 key = "sha256:" + "ab" * 32
26 await B.get_backend().put_mpack(key, b"MUSE-HEADER-" + data + b"-TRAILER")
27 db.add(MusehubObject(object_id=oid, path="pkg/packed.py", size_bytes=len(data), storage_uri=f"mpack://{key}"))
28 db.add(MusehubObjectRef(repo_id=repo_id, object_id=oid))
29 if indexed:
30 db.add(MusehubMPackIndex(entity_id=oid, mpack_id=key, entity_type="object", byte_offset=len(b"MUSE-HEADER-"), byte_length=len(data)))
31 await db.commit()
32 assert await B.get_backend().get(oid) is None # NOT a loose blob: only the mpack holds it
33 return oid
34
35
36 @pytest.mark.asyncio
37 async def test_content_api_serves_an_mpack_stored_object(client: AsyncClient, db_session: AsyncSession) -> None:
38 repo = await create_repo(db_session, visibility="public")
39 oid = await _packed_object(db_session, repo.repo_id)
40 r = await client.get(f"/api/repos/{repo.repo_id}/objects/{oid}/content")
41 assert r.status_code == 200, r.text
42 assert r.content == DATA
43
44
45 @pytest.mark.asyncio
46 async def test_bulk_reader_reads_every_form_concurrently_and_fails_closed(db_session: AsyncSession) -> None:
47 repo = await create_repo(db_session, visibility="public")
48 packed = await _packed_object(db_session, repo.repo_id)
49 unindexed_data = b"in a pack but not indexed: whole-pack fallback\n"
50 loose_data, cached_data = b"a loose blob\n", b"held in content_cache\n"
51 loose, cached, unindexed = blob_id(loose_data), blob_id(cached_data), blob_id(unindexed_data)
52 from muse.core.mpack import build_wire_mpack
53 pack_key = "sha256:" + "ef" * 32
54 await B.get_backend().put_mpack(pack_key, build_wire_mpack({"commits": [], "snapshots": [], "blobs": [{"object_id": unindexed, "content": unindexed_data}], "tags": []}))
55 uri = await B.get_backend().put(loose, loose_data)
56 db_session.add_all([
57 MusehubObject(object_id=loose, path="l.txt", size_bytes=len(loose_data), storage_uri=uri),
58 MusehubObject(object_id=cached, path="c.txt", size_bytes=len(cached_data), storage_uri="mpack://sha256:" + "00" * 32, content_cache=cached_data),
59 MusehubObject(object_id=unindexed, path="u.txt", size_bytes=len(unindexed_data), storage_uri=f"mpack://{pack_key}"),
60 ])
61 await db_session.commit()
62 rows = [await db_session.get(MusehubObject, o) for o in (packed, loose, cached, unindexed)]
63 got = await B.read_objects_bytes_required(rows, db_session)
64 assert got == {packed: DATA, loose: loose_data, cached: cached_data, unindexed: unindexed_data}
65
66 gone = MusehubObject(object_id=blob_id(b"nowhere"), path="g.txt", size_bytes=7, storage_uri="mpack://sha256:" + "11" * 32)
67 with pytest.raises(B.ObjectBytesUnavailable) as exc:
68 await B.read_objects_bytes_required([*rows, gone], db_session)
69 assert exc.value.object_ids == [gone.object_id]
70
71
72 @pytest.mark.asyncio
73 async def test_required_read_raises_instead_of_inventing_empty_content(db_session: AsyncSession) -> None:
74 missing = MusehubObject(object_id=blob_id(b"absent bytes"), path="x.py", size_bytes=12, storage_uri="mpack://sha256:" + "cd" * 32)
75 with pytest.raises(B.ObjectBytesUnavailable):
76 await B.read_object_bytes_required(missing, session=db_session)
77 empty = MusehubObject(object_id=blob_id(b""), path="empty.py", size_bytes=0, storage_uri="mpack://sha256:" + "cd" * 32)
78 assert await B.read_object_bytes_required(empty, session=db_session) == b""
79
80
81 @pytest.mark.asyncio
82 async def test_integrity_scan_does_not_report_mpack_objects_missing(db_session: AsyncSession) -> None:
83 from musehub.maintenance.object_integrity import scan_object_integrity
84 repo = await create_repo(db_session, visibility="public")
85 oid = await _packed_object(db_session, repo.repo_id)
86 result = await scan_object_integrity(db_session, B.get_backend(), sample_size=10_000)
87 assert oid not in {m.object_id for m in result.mismatches}
88
89
90 @pytest.mark.asyncio
91 async def test_loose_row_whose_blob_is_absent_reads_from_an_indexed_mpack(db_session: AsyncSession) -> None:
92 """Staging: 669 gabriel/muse rows said s3:// but no loose blob existed; their bytes were indexed inside push mpacks."""
93 data = b"recorded as a loose blob, present only inside an mpack\n"
94 oid = blob_id(data)
95 key = "sha256:" + "fa" * 32
96 await B.get_backend().put_mpack(key, b"HDR" + data)
97 db_session.add(MusehubObject(object_id=oid, path="l.py", size_bytes=len(data), storage_uri=f"s3://bucket/objects/{oid}"))
98 db_session.add(MusehubMPackIndex(entity_id=oid, mpack_id=key, entity_type="object", byte_offset=3, byte_length=len(data)))
99 await db_session.commit()
100 row = await db_session.get(MusehubObject, oid)
101 assert await B.get_backend().get(oid) is None
102 assert await B.read_object_bytes(row, session=db_session) == data
103 assert (await B.read_objects_bytes_required([row], db_session))[oid] == data
File History 3 commits
sha256:b77a9ac02fb6c01305ab105f50d094b255d60015247f33e229cc81e4b694a013 fix(objects): a loose-blob row whose blob is absent reads f… Opus 5.5 patch 2 days ago
sha256:9e7fb25b45eb883646cdd15c4c6ed27f0ca88cf6259a90d3067cfaba744f0ea3 fix(objects): review of a0c73d55 — fetch/sync read concurre… Opus 5.5 minor ⚠ 2 days ago
sha256:69161439990eeb42a0502d2417d8492b076c88e5ca0fb842dbb1bfb1ef8fca7c fix(objects): P0 object availability — serve mpack-stored o… Opus 5.5 minor ⚠ 2 days ago