test_object_completeness.py
python
sha256:4a3b59d1812fc38ecfb6f736bba7018f73dfc578fb4fcea6d150a23aa4b85499
fix(objects): completeness repair repoints rows whose mpack…
Opus 5.5
patch
2 days ago
| 1 | """Object completeness scan and verified repair (musehub.maintenance.object_completeness).""" |
| 2 | from __future__ import annotations |
| 3 | |
| 4 | import datetime |
| 5 | import pathlib |
| 6 | |
| 7 | import msgpack |
| 8 | import pytest |
| 9 | from sqlalchemy import select |
| 10 | from sqlalchemy.ext.asyncio import AsyncSession |
| 11 | |
| 12 | from muse.core.types import blob_id |
| 13 | from musehub.db.musehub_repo_models import MusehubCommit, MusehubCommitRef, MusehubObject, MusehubObjectRef, MusehubSnapshot |
| 14 | from musehub.maintenance.object_completeness import blob_object_id, repair_repo, scan_repo |
| 15 | from musehub.storage import backends as B |
| 16 | from tests.factories import create_repo |
| 17 | |
| 18 | OK, LOST, UNLINKED = b"present and linked\n", b"lost by a partial-clone push\n", b"stored but not linked\n" |
| 19 | |
| 20 | |
| 21 | async def _repo_with_gaps(db: AsyncSession) -> tuple[str, dict[str, str]]: |
| 22 | repo = await create_repo(db, visibility="public") |
| 23 | ids = {"ok": blob_id(OK), "lost": blob_id(LOST), "unlinked": blob_id(UNLINKED)} |
| 24 | for key, data in (("ok", OK), ("unlinked", UNLINKED)): |
| 25 | uri = await B.get_backend().put(ids[key], data) |
| 26 | db.add(MusehubObject(object_id=ids[key], path=f"{key}.txt", size_bytes=len(data), storage_uri=uri)) |
| 27 | db.add(MusehubObjectRef(repo_id=repo.repo_id, object_id=ids["ok"])) |
| 28 | sid = "sha256:" + "5" * 64 |
| 29 | db.add(MusehubSnapshot(snapshot_id=sid, manifest_blob=msgpack.packb({"ok.txt": ids["ok"], "lost.txt": ids["lost"], "unlinked.txt": ids["unlinked"]}))) |
| 30 | cid = blob_id(b"completeness-commit") |
| 31 | db.add(MusehubCommit(commit_id=cid, branch="main", parent_ids=[], message="c", author="gabriel", |
| 32 | timestamp=datetime.datetime(2026, 11, 1, tzinfo=datetime.timezone.utc), snapshot_id=sid)) |
| 33 | db.add(MusehubCommitRef(repo_id=repo.repo_id, commit_id=cid)) |
| 34 | await db.commit() |
| 35 | return repo.repo_id, ids |
| 36 | |
| 37 | |
| 38 | @pytest.mark.asyncio |
| 39 | async def test_scan_reports_missing_rows_and_unlinked_objects(db_session: AsyncSession) -> None: |
| 40 | repo_id, ids = await _repo_with_gaps(db_session) |
| 41 | res = await scan_repo(db_session, repo_id, "t/r", check_bytes=True) |
| 42 | assert (res.snapshots, res.objects) == (1, 3) |
| 43 | assert res.missing_rows == [ids["lost"]] and res.unlinked == [ids["unlinked"]] and res.unreadable == [] |
| 44 | assert res.paths[ids["lost"]] == "lost.txt" and not res.complete |
| 45 | |
| 46 | |
| 47 | @pytest.mark.asyncio |
| 48 | async def test_repair_uses_only_verified_bytes_and_is_idempotent(db_session: AsyncSession, tmp_path: pathlib.Path) -> None: |
| 49 | repo_id, ids = await _repo_with_gaps(db_session) |
| 50 | (tmp_path / ids["lost"].split(":")[1]).write_bytes(b"tampered bytes\n") # wrong content under the right name: refused |
| 51 | res = await scan_repo(db_session, repo_id, "t/r") |
| 52 | await repair_repo(db_session, res, tmp_path) |
| 53 | assert res.repaired == [ids["unlinked"]] and res.missing_rows == [ids["lost"]] |
| 54 | assert await db_session.get(MusehubObject, ids["lost"]) is None |
| 55 | |
| 56 | (tmp_path / ids["lost"].split(":")[1]).write_bytes(LOST) |
| 57 | res = await scan_repo(db_session, repo_id, "t/r") |
| 58 | await repair_repo(db_session, res, tmp_path) |
| 59 | assert res.repaired == [ids["lost"]] and res.complete |
| 60 | row = await db_session.get(MusehubObject, ids["lost"]) |
| 61 | assert row is not None and row.path == "lost.txt" and await B.read_object_bytes(row, session=db_session) == LOST |
| 62 | |
| 63 | again = await scan_repo(db_session, repo_id, "t/r", check_bytes=True) |
| 64 | assert again.complete |
| 65 | await repair_repo(db_session, again, tmp_path) # nothing to do, nothing changes |
| 66 | assert again.repaired == [] |
| 67 | refs = (await db_session.execute(select(MusehubObjectRef).where(MusehubObjectRef.repo_id == repo_id))).scalars().all() |
| 68 | assert len(refs) == 3 |
| 69 | |
| 70 | |
| 71 | def test_blob_object_id_matches_muse() -> None: |
| 72 | assert blob_object_id(LOST) == blob_id(LOST) |
| 73 | |
| 74 | |
| 75 | |
| 76 | @pytest.mark.asyncio |
| 77 | async def test_repair_without_a_source_links_only_verified_stored_objects(db_session: AsyncSession) -> None: |
| 78 | repo_id, ids = await _repo_with_gaps(db_session) |
| 79 | res = await scan_repo(db_session, repo_id, "t/r") |
| 80 | await repair_repo(db_session, res, None) |
| 81 | assert res.repaired == [ids["unlinked"]] and res.unlinked == [] and res.missing_rows == [ids["lost"]] |
| 82 | |
| 83 | |
| 84 | @pytest.mark.asyncio |
| 85 | async def test_repair_never_links_an_object_known_only_to_another_owners_private_repo(db_session: AsyncSession) -> None: |
| 86 | repo_id, ids = await _repo_with_gaps(db_session) |
| 87 | other = await create_repo(db_session, owner="someone-else", visibility="private") |
| 88 | db_session.add(MusehubObjectRef(repo_id=other.repo_id, object_id=ids["unlinked"])) |
| 89 | await db_session.commit() |
| 90 | res = await scan_repo(db_session, repo_id, "t/r") |
| 91 | await repair_repo(db_session, res, None) |
| 92 | assert res.skipped_private == [ids["unlinked"]] and res.unlinked == [ids["unlinked"]] and not res.complete |
| 93 | refs = (await db_session.execute(select(MusehubObjectRef.repo_id).where(MusehubObjectRef.object_id == ids["unlinked"]))).scalars().all() |
| 94 | assert refs == [other.repo_id] |
| 95 | |
| 96 | |
| 97 | @pytest.mark.asyncio |
| 98 | async def test_repair_links_an_object_already_readable_publicly(db_session: AsyncSession) -> None: |
| 99 | repo_id, ids = await _repo_with_gaps(db_session) |
| 100 | public = await create_repo(db_session, owner="someone-else", visibility="public") |
| 101 | db_session.add(MusehubObjectRef(repo_id=public.repo_id, object_id=ids["unlinked"])) |
| 102 | await db_session.commit() |
| 103 | res = await scan_repo(db_session, repo_id, "t/r") |
| 104 | await repair_repo(db_session, res, None) |
| 105 | assert res.repaired == [ids["unlinked"]] and res.skipped_private == [] |
| 106 | |
| 107 | |
| 108 | @pytest.mark.asyncio |
| 109 | async def test_malformed_manifest_entries_are_reported_not_crashed_on(db_session: AsyncSession, tmp_path: pathlib.Path) -> None: |
| 110 | repo = await create_repo(db_session, visibility="public") |
| 111 | sid = "sha256:" + "6" * 64 |
| 112 | db_session.add(MusehubSnapshot(snapshot_id=sid, manifest_blob=msgpack.packb({"empty.txt": "", "null.txt": None}))) |
| 113 | cid = blob_id(b"malformed-commit") |
| 114 | db_session.add(MusehubCommit(commit_id=cid, branch="main", parent_ids=[], message="m", author="gabriel", |
| 115 | timestamp=datetime.datetime(2026, 11, 2, tzinfo=datetime.timezone.utc), snapshot_id=sid)) |
| 116 | db_session.add(MusehubCommitRef(repo_id=repo.repo_id, commit_id=cid)) |
| 117 | await db_session.commit() |
| 118 | res = await scan_repo(db_session, repo.repo_id, "t/m") |
| 119 | assert sorted(res.malformed_entries) == [f"{sid}:empty.txt", f"{sid}:null.txt"] and res.objects == 0 and res.complete |
| 120 | await repair_repo(db_session, res, tmp_path) # must not raise |
| 121 | |
| 122 | |
| 123 | @pytest.mark.asyncio |
| 124 | async def test_repair_repoints_a_row_whose_mpack_is_gone_using_verified_bytes(db_session: AsyncSession, tmp_path: pathlib.Path) -> None: |
| 125 | repo = await create_repo(db_session, visibility="public") |
| 126 | data = b"bytes whose push mpack no longer exists\n" |
| 127 | oid = blob_id(data) |
| 128 | db_session.add(MusehubObject(object_id=oid, path="gone.txt", size_bytes=len(data), storage_uri="mpack://sha256:" + "7" * 64)) |
| 129 | sid = "sha256:" + "8" * 64 |
| 130 | db_session.add(MusehubSnapshot(snapshot_id=sid, manifest_blob=msgpack.packb({"gone.txt": oid}))) |
| 131 | cid = blob_id(b"gone-commit") |
| 132 | db_session.add(MusehubCommit(commit_id=cid, branch="main", parent_ids=[], message="g", author="gabriel", |
| 133 | timestamp=datetime.datetime(2026, 11, 3, tzinfo=datetime.timezone.utc), snapshot_id=sid)) |
| 134 | db_session.add(MusehubCommitRef(repo_id=repo.repo_id, commit_id=cid)) |
| 135 | await db_session.commit() |
| 136 | |
| 137 | res = await scan_repo(db_session, repo.repo_id, "t/g", check_bytes=True) |
| 138 | assert res.unlinked == [oid] and res.unreadable == [oid] |
| 139 | await repair_repo(db_session, res, None) # no source: refused, nothing changes |
| 140 | assert res.repaired == [] and not res.complete |
| 141 | (tmp_path / oid.split(":")[1]).write_bytes(data) |
| 142 | res = await scan_repo(db_session, repo.repo_id, "t/g", check_bytes=True) |
| 143 | await repair_repo(db_session, res, tmp_path) |
| 144 | assert res.repaired == [oid] and res.complete |
| 145 | row = await db_session.get(MusehubObject, oid, populate_existing=True) |
| 146 | assert row.storage_uri.startswith("s3://") and await B.read_object_bytes(row, session=db_session) == data |
| 147 | assert (await scan_repo(db_session, repo.repo_id, "t/g", check_bytes=True)).complete |
File History
3 commits
sha256:4a3b59d1812fc38ecfb6f736bba7018f73dfc578fb4fcea6d150a23aa4b85499
fix(objects): completeness repair repoints rows whose mpack…
Opus 5.5
patch
2 days ago
sha256:9e7fb25b45eb883646cdd15c4c6ed27f0ca88cf6259a90d3067cfaba744f0ea3
fix(objects): review of a0c73d55 — fetch/sync read concurre…
Opus 5.5
minor
⚠
2 days ago
sha256:69161439990eeb42a0502d2417d8492b076c88e5ca0fb842dbb1bfb1ef8fca7c
fix(objects): P0 object availability — serve mpack-stored o…
Opus 5.5
minor
⚠
2 days ago