gabriel / musehub public
test_object_completeness.py python
147 lines 7.9 KB
Raw
sha256:4a3b59d1812fc38ecfb6f736bba7018f73dfc578fb4fcea6d150a23aa4b85499 fix(objects): completeness repair repoints rows whose mpack… Opus 5.5 patch 2 days ago
1 """Object completeness scan and verified repair (musehub.maintenance.object_completeness)."""
2 from __future__ import annotations
3
4 import datetime
5 import pathlib
6
7 import msgpack
8 import pytest
9 from sqlalchemy import select
10 from sqlalchemy.ext.asyncio import AsyncSession
11
12 from muse.core.types import blob_id
13 from musehub.db.musehub_repo_models import MusehubCommit, MusehubCommitRef, MusehubObject, MusehubObjectRef, MusehubSnapshot
14 from musehub.maintenance.object_completeness import blob_object_id, repair_repo, scan_repo
15 from musehub.storage import backends as B
16 from tests.factories import create_repo
17
18 OK, LOST, UNLINKED = b"present and linked\n", b"lost by a partial-clone push\n", b"stored but not linked\n"
19
20
21 async def _repo_with_gaps(db: AsyncSession) -> tuple[str, dict[str, str]]:
22 repo = await create_repo(db, visibility="public")
23 ids = {"ok": blob_id(OK), "lost": blob_id(LOST), "unlinked": blob_id(UNLINKED)}
24 for key, data in (("ok", OK), ("unlinked", UNLINKED)):
25 uri = await B.get_backend().put(ids[key], data)
26 db.add(MusehubObject(object_id=ids[key], path=f"{key}.txt", size_bytes=len(data), storage_uri=uri))
27 db.add(MusehubObjectRef(repo_id=repo.repo_id, object_id=ids["ok"]))
28 sid = "sha256:" + "5" * 64
29 db.add(MusehubSnapshot(snapshot_id=sid, manifest_blob=msgpack.packb({"ok.txt": ids["ok"], "lost.txt": ids["lost"], "unlinked.txt": ids["unlinked"]})))
30 cid = blob_id(b"completeness-commit")
31 db.add(MusehubCommit(commit_id=cid, branch="main", parent_ids=[], message="c", author="gabriel",
32 timestamp=datetime.datetime(2026, 11, 1, tzinfo=datetime.timezone.utc), snapshot_id=sid))
33 db.add(MusehubCommitRef(repo_id=repo.repo_id, commit_id=cid))
34 await db.commit()
35 return repo.repo_id, ids
36
37
38 @pytest.mark.asyncio
39 async def test_scan_reports_missing_rows_and_unlinked_objects(db_session: AsyncSession) -> None:
40 repo_id, ids = await _repo_with_gaps(db_session)
41 res = await scan_repo(db_session, repo_id, "t/r", check_bytes=True)
42 assert (res.snapshots, res.objects) == (1, 3)
43 assert res.missing_rows == [ids["lost"]] and res.unlinked == [ids["unlinked"]] and res.unreadable == []
44 assert res.paths[ids["lost"]] == "lost.txt" and not res.complete
45
46
47 @pytest.mark.asyncio
48 async def test_repair_uses_only_verified_bytes_and_is_idempotent(db_session: AsyncSession, tmp_path: pathlib.Path) -> None:
49 repo_id, ids = await _repo_with_gaps(db_session)
50 (tmp_path / ids["lost"].split(":")[1]).write_bytes(b"tampered bytes\n") # wrong content under the right name: refused
51 res = await scan_repo(db_session, repo_id, "t/r")
52 await repair_repo(db_session, res, tmp_path)
53 assert res.repaired == [ids["unlinked"]] and res.missing_rows == [ids["lost"]]
54 assert await db_session.get(MusehubObject, ids["lost"]) is None
55
56 (tmp_path / ids["lost"].split(":")[1]).write_bytes(LOST)
57 res = await scan_repo(db_session, repo_id, "t/r")
58 await repair_repo(db_session, res, tmp_path)
59 assert res.repaired == [ids["lost"]] and res.complete
60 row = await db_session.get(MusehubObject, ids["lost"])
61 assert row is not None and row.path == "lost.txt" and await B.read_object_bytes(row, session=db_session) == LOST
62
63 again = await scan_repo(db_session, repo_id, "t/r", check_bytes=True)
64 assert again.complete
65 await repair_repo(db_session, again, tmp_path) # nothing to do, nothing changes
66 assert again.repaired == []
67 refs = (await db_session.execute(select(MusehubObjectRef).where(MusehubObjectRef.repo_id == repo_id))).scalars().all()
68 assert len(refs) == 3
69
70
71 def test_blob_object_id_matches_muse() -> None:
72 assert blob_object_id(LOST) == blob_id(LOST)
73
74
75
76 @pytest.mark.asyncio
77 async def test_repair_without_a_source_links_only_verified_stored_objects(db_session: AsyncSession) -> None:
78 repo_id, ids = await _repo_with_gaps(db_session)
79 res = await scan_repo(db_session, repo_id, "t/r")
80 await repair_repo(db_session, res, None)
81 assert res.repaired == [ids["unlinked"]] and res.unlinked == [] and res.missing_rows == [ids["lost"]]
82
83
84 @pytest.mark.asyncio
85 async def test_repair_never_links_an_object_known_only_to_another_owners_private_repo(db_session: AsyncSession) -> None:
86 repo_id, ids = await _repo_with_gaps(db_session)
87 other = await create_repo(db_session, owner="someone-else", visibility="private")
88 db_session.add(MusehubObjectRef(repo_id=other.repo_id, object_id=ids["unlinked"]))
89 await db_session.commit()
90 res = await scan_repo(db_session, repo_id, "t/r")
91 await repair_repo(db_session, res, None)
92 assert res.skipped_private == [ids["unlinked"]] and res.unlinked == [ids["unlinked"]] and not res.complete
93 refs = (await db_session.execute(select(MusehubObjectRef.repo_id).where(MusehubObjectRef.object_id == ids["unlinked"]))).scalars().all()
94 assert refs == [other.repo_id]
95
96
97 @pytest.mark.asyncio
98 async def test_repair_links_an_object_already_readable_publicly(db_session: AsyncSession) -> None:
99 repo_id, ids = await _repo_with_gaps(db_session)
100 public = await create_repo(db_session, owner="someone-else", visibility="public")
101 db_session.add(MusehubObjectRef(repo_id=public.repo_id, object_id=ids["unlinked"]))
102 await db_session.commit()
103 res = await scan_repo(db_session, repo_id, "t/r")
104 await repair_repo(db_session, res, None)
105 assert res.repaired == [ids["unlinked"]] and res.skipped_private == []
106
107
108 @pytest.mark.asyncio
109 async def test_malformed_manifest_entries_are_reported_not_crashed_on(db_session: AsyncSession, tmp_path: pathlib.Path) -> None:
110 repo = await create_repo(db_session, visibility="public")
111 sid = "sha256:" + "6" * 64
112 db_session.add(MusehubSnapshot(snapshot_id=sid, manifest_blob=msgpack.packb({"empty.txt": "", "null.txt": None})))
113 cid = blob_id(b"malformed-commit")
114 db_session.add(MusehubCommit(commit_id=cid, branch="main", parent_ids=[], message="m", author="gabriel",
115 timestamp=datetime.datetime(2026, 11, 2, tzinfo=datetime.timezone.utc), snapshot_id=sid))
116 db_session.add(MusehubCommitRef(repo_id=repo.repo_id, commit_id=cid))
117 await db_session.commit()
118 res = await scan_repo(db_session, repo.repo_id, "t/m")
119 assert sorted(res.malformed_entries) == [f"{sid}:empty.txt", f"{sid}:null.txt"] and res.objects == 0 and res.complete
120 await repair_repo(db_session, res, tmp_path) # must not raise
121
122
123 @pytest.mark.asyncio
124 async def test_repair_repoints_a_row_whose_mpack_is_gone_using_verified_bytes(db_session: AsyncSession, tmp_path: pathlib.Path) -> None:
125 repo = await create_repo(db_session, visibility="public")
126 data = b"bytes whose push mpack no longer exists\n"
127 oid = blob_id(data)
128 db_session.add(MusehubObject(object_id=oid, path="gone.txt", size_bytes=len(data), storage_uri="mpack://sha256:" + "7" * 64))
129 sid = "sha256:" + "8" * 64
130 db_session.add(MusehubSnapshot(snapshot_id=sid, manifest_blob=msgpack.packb({"gone.txt": oid})))
131 cid = blob_id(b"gone-commit")
132 db_session.add(MusehubCommit(commit_id=cid, branch="main", parent_ids=[], message="g", author="gabriel",
133 timestamp=datetime.datetime(2026, 11, 3, tzinfo=datetime.timezone.utc), snapshot_id=sid))
134 db_session.add(MusehubCommitRef(repo_id=repo.repo_id, commit_id=cid))
135 await db_session.commit()
136
137 res = await scan_repo(db_session, repo.repo_id, "t/g", check_bytes=True)
138 assert res.unlinked == [oid] and res.unreadable == [oid]
139 await repair_repo(db_session, res, None) # no source: refused, nothing changes
140 assert res.repaired == [] and not res.complete
141 (tmp_path / oid.split(":")[1]).write_bytes(data)
142 res = await scan_repo(db_session, repo.repo_id, "t/g", check_bytes=True)
143 await repair_repo(db_session, res, tmp_path)
144 assert res.repaired == [oid] and res.complete
145 row = await db_session.get(MusehubObject, oid, populate_existing=True)
146 assert row.storage_uri.startswith("s3://") and await B.read_object_bytes(row, session=db_session) == data
147 assert (await scan_repo(db_session, repo.repo_id, "t/g", check_bytes=True)).complete
File History 3 commits
sha256:4a3b59d1812fc38ecfb6f736bba7018f73dfc578fb4fcea6d150a23aa4b85499 fix(objects): completeness repair repoints rows whose mpack… Opus 5.5 patch 2 days ago
sha256:9e7fb25b45eb883646cdd15c4c6ed27f0ca88cf6259a90d3067cfaba744f0ea3 fix(objects): review of a0c73d55 — fetch/sync read concurre… Opus 5.5 minor ⚠ 2 days ago
sha256:69161439990eeb42a0502d2417d8492b076c88e5ca0fb842dbb1bfb1ef8fca7c fix(objects): P0 object availability — serve mpack-stored o… Opus 5.5 minor ⚠ 2 days ago