test_gc_object_refs.py
python
sha256:9590cee1e0ccd6c76528f005b95d634d80f5019f0dcb7c371e149adc31d1fb65
refactor: enforce gRPC framing on all MWP wire traffic
Sonnet 4.6
minor
⚠ breaking
156 days ago
| 1 | """TDD: GC object-ref pruning and global object cleanup. |
| 2 | |
| 3 | After a force push or branch rewrite, old commits and their snapshots become |
| 4 | orphaned. Objects referenced exclusively by orphaned snapshots should have |
| 5 | their ref rows removed. Objects with zero remaining refs across all repos |
| 6 | should be deleted from musehub_objects and from storage. |
| 7 | |
| 8 | Coverage matrix: |
| 9 | 1. GC deletes ref row for object unreachable from all live snapshots. |
| 10 | 2. GC does NOT delete ref when object is still referenced by a live snapshot. |
| 11 | 3. GC deletes musehub_objects row when no refs remain globally. |
| 12 | 4. GC does NOT delete musehub_objects row when another repo still holds a ref. |
| 13 | 5. GC on a clean repo (no orphaned commits) is a no-op — no refs disturbed. |
| 14 | 6. GCResult fields are populated correctly. |
| 15 | """ |
| 16 | from __future__ import annotations |
| 17 | |
| 18 | import hashlib |
| 19 | import uuid |
| 20 | from datetime import datetime, timezone |
| 21 | |
| 22 | import msgpack |
| 23 | import pytest |
| 24 | import sqlalchemy as sa |
| 25 | from sqlalchemy.ext.asyncio import AsyncSession |
| 26 | |
| 27 | from musehub.db import musehub_models as db |
| 28 | from musehub.services.musehub_gc import run_gc |
| 29 | from musehub.types.json_types import StrDict |
| 30 | from tests.factories import create_repo |
| 31 | |
| 32 | |
| 33 | def _now() -> datetime: |
| 34 | return datetime.now(tz=timezone.utc) |
| 35 | |
| 36 | |
| 37 | def _oid(seed: str) -> str: |
| 38 | return hashlib.sha256(seed.encode()).hexdigest() |
| 39 | |
| 40 | |
| 41 | def _manifest(mapping: StrDict) -> bytes: |
| 42 | """Encode a {path: object_id} dict as msgpack.""" |
| 43 | return msgpack.packb(mapping, use_bin_type=True) |
| 44 | |
| 45 | |
| 46 | # --------------------------------------------------------------------------- |
| 47 | # Low-level DB helpers |
| 48 | # --------------------------------------------------------------------------- |
| 49 | |
| 50 | async def _insert_object(session: AsyncSession, oid: str, repo_id: str) -> None: |
| 51 | """Insert a minimal musehub_objects row and ref (skips if already present).""" |
| 52 | exists = (await session.execute( |
| 53 | sa.select(db.MusehubObject.object_id).where(db.MusehubObject.object_id == oid) |
| 54 | )).scalar_one_or_none() |
| 55 | if exists: |
| 56 | return |
| 57 | obj = db.MusehubObject( |
| 58 | object_id=oid, |
| 59 | size_bytes=10, |
| 60 | path="test.md", |
| 61 | disk_path="", |
| 62 | content_cache=b"test content", |
| 63 | ) |
| 64 | session.add(obj) |
| 65 | await session.flush() |
| 66 | |
| 67 | |
| 68 | async def _insert_ref(session: AsyncSession, repo_id: str, oid: str) -> None: |
| 69 | """Insert a musehub_object_refs row (idempotent).""" |
| 70 | existing = (await session.execute( |
| 71 | sa.select(db.MusehubObjectRef).where( |
| 72 | db.MusehubObjectRef.repo_id == repo_id, |
| 73 | db.MusehubObjectRef.object_id == oid, |
| 74 | ) |
| 75 | )).scalar_one_or_none() |
| 76 | if existing: |
| 77 | return |
| 78 | session.add(db.MusehubObjectRef(repo_id=repo_id, object_id=oid)) |
| 79 | await session.flush() |
| 80 | |
| 81 | |
| 82 | async def _insert_snapshot( |
| 83 | session: AsyncSession, |
| 84 | snapshot_id: str, |
| 85 | manifest: dict[str, str], |
| 86 | repo_id: str = "", |
| 87 | ) -> db.MusehubSnapshot: |
| 88 | snap = db.MusehubSnapshot( |
| 89 | snapshot_id=snapshot_id, |
| 90 | repo_id=repo_id, |
| 91 | manifest_blob=_manifest(manifest), |
| 92 | entry_count=len(manifest), |
| 93 | directories=[], |
| 94 | ) |
| 95 | session.add(snap) |
| 96 | await session.flush() |
| 97 | return snap |
| 98 | |
| 99 | |
| 100 | async def _insert_commit( |
| 101 | session: AsyncSession, |
| 102 | repo_id: str, |
| 103 | commit_id: str, |
| 104 | snapshot_id: str | None = None, |
| 105 | parent_ids: list[str] | None = None, |
| 106 | branch: str = "main", |
| 107 | ) -> db.MusehubCommit: |
| 108 | commit = db.MusehubCommit( |
| 109 | commit_id=commit_id, |
| 110 | repo_id=repo_id, |
| 111 | message="test commit", |
| 112 | author="test-user", |
| 113 | branch=branch, |
| 114 | parent_ids=parent_ids or [], |
| 115 | snapshot_id=snapshot_id, |
| 116 | timestamp=_now(), |
| 117 | ) |
| 118 | session.add(commit) |
| 119 | await session.flush() |
| 120 | return commit |
| 121 | |
| 122 | |
| 123 | async def _insert_branch( |
| 124 | session: AsyncSession, |
| 125 | repo_id: str, |
| 126 | head_commit_id: str, |
| 127 | name: str = "main", |
| 128 | ) -> db.MusehubBranch: |
| 129 | branch = db.MusehubBranch( |
| 130 | repo_id=repo_id, |
| 131 | name=name, |
| 132 | head_commit_id=head_commit_id, |
| 133 | ) |
| 134 | session.add(branch) |
| 135 | await session.flush() |
| 136 | return branch |
| 137 | |
| 138 | |
| 139 | async def _ref_exists(session: AsyncSession, repo_id: str, oid: str) -> bool: |
| 140 | row = (await session.execute( |
| 141 | sa.select(db.MusehubObjectRef).where( |
| 142 | db.MusehubObjectRef.repo_id == repo_id, |
| 143 | db.MusehubObjectRef.object_id == oid, |
| 144 | ) |
| 145 | )).scalar_one_or_none() |
| 146 | return row is not None |
| 147 | |
| 148 | |
| 149 | async def _object_exists(session: AsyncSession, oid: str) -> bool: |
| 150 | row = (await session.execute( |
| 151 | sa.select(db.MusehubObject.object_id).where(db.MusehubObject.object_id == oid) |
| 152 | )).scalar_one_or_none() |
| 153 | return row is not None |
| 154 | |
| 155 | |
| 156 | # --------------------------------------------------------------------------- |
| 157 | # Test 1: GC removes ref for object only in orphaned snapshot |
| 158 | # --------------------------------------------------------------------------- |
| 159 | |
| 160 | @pytest.mark.asyncio |
| 161 | async def test_gc_removes_stale_ref_for_orphaned_object( |
| 162 | db_session: AsyncSession, |
| 163 | ) -> None: |
| 164 | """A ref row for an object that only appears in an orphaned snapshot must be deleted.""" |
| 165 | repo = await create_repo(db_session, slug="gc-stale-ref", owner="test-user-wire") |
| 166 | oid = _oid("stale-object-only-in-orphaned-snapshot") |
| 167 | |
| 168 | # Orphaned commit chain: C1 -> C2 (orphaned after force-push to C3) |
| 169 | snap_orphan_id = f"snap_{uuid.uuid4().hex[:8]}" |
| 170 | c1_id = uuid.uuid4().hex |
| 171 | c2_id = uuid.uuid4().hex |
| 172 | c3_id = uuid.uuid4().hex |
| 173 | snap_live_id = f"snap_{uuid.uuid4().hex[:8]}" |
| 174 | |
| 175 | await _insert_object(db_session, oid, repo.repo_id) |
| 176 | await _insert_ref(db_session, repo.repo_id, oid) |
| 177 | # Orphaned snapshot references the object |
| 178 | await _insert_snapshot(db_session, snap_orphan_id, {"file.md": oid}, repo_id=repo.repo_id) |
| 179 | # Live snapshot is empty (object not referenced by any live snapshot) |
| 180 | await _insert_snapshot(db_session, snap_live_id, {}, repo_id=repo.repo_id) |
| 181 | await _insert_commit(db_session, repo.repo_id, c1_id) |
| 182 | await _insert_commit(db_session, repo.repo_id, c2_id, snapshot_id=snap_orphan_id, parent_ids=[c1_id]) |
| 183 | await _insert_commit(db_session, repo.repo_id, c3_id, snapshot_id=snap_live_id) # force-push resets branch |
| 184 | await _insert_branch(db_session, repo.repo_id, c3_id) |
| 185 | await db_session.commit() |
| 186 | |
| 187 | result = await run_gc(db_session, repo.repo_id) |
| 188 | |
| 189 | assert result.object_refs_deleted >= 1, "stale ref must be deleted" |
| 190 | assert not await _ref_exists(db_session, repo.repo_id, oid), \ |
| 191 | "ref row must be gone after GC" |
| 192 | |
| 193 | |
| 194 | # --------------------------------------------------------------------------- |
| 195 | # Test 2: GC keeps ref when object is still in a live snapshot |
| 196 | # --------------------------------------------------------------------------- |
| 197 | |
| 198 | @pytest.mark.asyncio |
| 199 | async def test_gc_keeps_ref_for_live_object( |
| 200 | db_session: AsyncSession, |
| 201 | ) -> None: |
| 202 | """A ref for an object that appears in both orphaned and live snapshots must survive.""" |
| 203 | repo = await create_repo(db_session, slug="gc-live-ref", owner="test-user-wire") |
| 204 | oid = _oid("object-in-both-orphaned-and-live-snapshot") |
| 205 | |
| 206 | snap_orphan_id = f"snap_{uuid.uuid4().hex[:8]}" |
| 207 | snap_live_id = f"snap_{uuid.uuid4().hex[:8]}" |
| 208 | c_orphan_id = uuid.uuid4().hex |
| 209 | c_live_id = uuid.uuid4().hex |
| 210 | |
| 211 | await _insert_object(db_session, oid, repo.repo_id) |
| 212 | await _insert_ref(db_session, repo.repo_id, oid) |
| 213 | # Both snapshots reference the same object |
| 214 | await _insert_snapshot(db_session, snap_orphan_id, {"a.md": oid}, repo_id=repo.repo_id) |
| 215 | await _insert_snapshot(db_session, snap_live_id, {"b.md": oid}, repo_id=repo.repo_id) |
| 216 | await _insert_commit(db_session, repo.repo_id, c_orphan_id, snapshot_id=snap_orphan_id) |
| 217 | await _insert_commit(db_session, repo.repo_id, c_live_id, snapshot_id=snap_live_id) |
| 218 | await _insert_branch(db_session, repo.repo_id, c_live_id) |
| 219 | await db_session.commit() |
| 220 | |
| 221 | result = await run_gc(db_session, repo.repo_id) |
| 222 | |
| 223 | assert result.object_refs_deleted == 0, "ref to live object must not be deleted" |
| 224 | assert await _ref_exists(db_session, repo.repo_id, oid), \ |
| 225 | "ref row must survive GC when object is live" |
| 226 | |
| 227 | |
| 228 | # --------------------------------------------------------------------------- |
| 229 | # Test 3: GC deletes musehub_objects row when globally orphaned |
| 230 | # --------------------------------------------------------------------------- |
| 231 | |
| 232 | @pytest.mark.asyncio |
| 233 | async def test_gc_deletes_globally_orphaned_object( |
| 234 | db_session: AsyncSession, |
| 235 | ) -> None: |
| 236 | """After the last ref is deleted, the musehub_objects row must be deleted too.""" |
| 237 | repo = await create_repo(db_session, slug="gc-global-orphan", owner="test-user-wire") |
| 238 | oid = _oid("globally-orphaned-no-other-repo-refs") |
| 239 | |
| 240 | snap_orphan_id = f"snap_{uuid.uuid4().hex[:8]}" |
| 241 | snap_live_id = f"snap_{uuid.uuid4().hex[:8]}" |
| 242 | c_orphan_id = uuid.uuid4().hex |
| 243 | c_live_id = uuid.uuid4().hex |
| 244 | |
| 245 | await _insert_object(db_session, oid, repo.repo_id) |
| 246 | await _insert_ref(db_session, repo.repo_id, oid) |
| 247 | await _insert_snapshot(db_session, snap_orphan_id, {"file.md": oid}, repo_id=repo.repo_id) |
| 248 | await _insert_snapshot(db_session, snap_live_id, {}, repo_id=repo.repo_id) # live snapshot has no objects |
| 249 | await _insert_commit(db_session, repo.repo_id, c_orphan_id, snapshot_id=snap_orphan_id) |
| 250 | await _insert_commit(db_session, repo.repo_id, c_live_id, snapshot_id=snap_live_id) |
| 251 | await _insert_branch(db_session, repo.repo_id, c_live_id) |
| 252 | await db_session.commit() |
| 253 | |
| 254 | result = await run_gc(db_session, repo.repo_id) |
| 255 | |
| 256 | assert result.objects_deleted >= 1, "globally orphaned object must be deleted from DB" |
| 257 | assert not await _object_exists(db_session, oid), \ |
| 258 | "musehub_objects row must be gone after GC" |
| 259 | |
| 260 | |
| 261 | # --------------------------------------------------------------------------- |
| 262 | # Test 4: GC does NOT delete musehub_objects when another repo still refs it |
| 263 | # --------------------------------------------------------------------------- |
| 264 | |
| 265 | @pytest.mark.asyncio |
| 266 | async def test_gc_keeps_object_when_other_repo_holds_ref( |
| 267 | db_session: AsyncSession, |
| 268 | ) -> None: |
| 269 | """An object shared with another repo must NOT be deleted from musehub_objects.""" |
| 270 | repo_a = await create_repo(db_session, slug="gc-shared-a", owner="test-user-wire") |
| 271 | repo_b = await create_repo(db_session, slug="gc-shared-b", owner="test-user-wire") |
| 272 | oid = _oid("shared-object-two-repos") |
| 273 | |
| 274 | # Set up repo_a with orphaned snapshot referencing the object |
| 275 | snap_orphan_id = f"snap_{uuid.uuid4().hex[:8]}" |
| 276 | snap_live_id = f"snap_{uuid.uuid4().hex[:8]}" |
| 277 | c_orphan_id = uuid.uuid4().hex |
| 278 | c_live_id = uuid.uuid4().hex |
| 279 | |
| 280 | await _insert_object(db_session, oid, repo_a.repo_id) |
| 281 | # Both repos hold a ref |
| 282 | await _insert_ref(db_session, repo_a.repo_id, oid) |
| 283 | await _insert_ref(db_session, repo_b.repo_id, oid) |
| 284 | |
| 285 | await _insert_snapshot(db_session, snap_orphan_id, {"file.md": oid}, repo_id=repo_a.repo_id) |
| 286 | await _insert_snapshot(db_session, snap_live_id, {}, repo_id=repo_a.repo_id) |
| 287 | await _insert_commit(db_session, repo_a.repo_id, c_orphan_id, snapshot_id=snap_orphan_id) |
| 288 | await _insert_commit(db_session, repo_a.repo_id, c_live_id, snapshot_id=snap_live_id) |
| 289 | await _insert_branch(db_session, repo_a.repo_id, c_live_id) |
| 290 | await db_session.commit() |
| 291 | |
| 292 | # GC repo_a — prunes repo_a's ref but repo_b's ref survives |
| 293 | result = await run_gc(db_session, repo_a.repo_id) |
| 294 | |
| 295 | assert result.objects_deleted == 0, \ |
| 296 | "must not delete object still referenced by repo_b" |
| 297 | assert await _object_exists(db_session, oid), \ |
| 298 | "musehub_objects row must survive because repo_b still holds a ref" |
| 299 | assert await _ref_exists(db_session, repo_b.repo_id, oid), \ |
| 300 | "repo_b ref must be untouched by repo_a GC" |
| 301 | |
| 302 | |
| 303 | # --------------------------------------------------------------------------- |
| 304 | # Test 5: Clean repo (no orphaned commits) — no refs disturbed |
| 305 | # --------------------------------------------------------------------------- |
| 306 | |
| 307 | @pytest.mark.asyncio |
| 308 | async def test_gc_clean_repo_does_not_touch_refs( |
| 309 | db_session: AsyncSession, |
| 310 | ) -> None: |
| 311 | """GC on a fully-reachable repo must be a no-op for object refs.""" |
| 312 | repo = await create_repo(db_session, slug="gc-clean-repo", owner="test-user-wire") |
| 313 | oid = _oid("clean-repo-live-object") |
| 314 | |
| 315 | snap_id = f"snap_{uuid.uuid4().hex[:8]}" |
| 316 | c_id = uuid.uuid4().hex |
| 317 | |
| 318 | await _insert_object(db_session, oid, repo.repo_id) |
| 319 | await _insert_ref(db_session, repo.repo_id, oid) |
| 320 | await _insert_snapshot(db_session, snap_id, {"readme.md": oid}, repo_id=repo.repo_id) |
| 321 | await _insert_commit(db_session, repo.repo_id, c_id, snapshot_id=snap_id) |
| 322 | await _insert_branch(db_session, repo.repo_id, c_id) |
| 323 | await db_session.commit() |
| 324 | |
| 325 | result = await run_gc(db_session, repo.repo_id) |
| 326 | |
| 327 | assert result.commits_deleted == 0 |
| 328 | assert result.snapshots_deleted == 0 |
| 329 | assert result.object_refs_deleted == 0 |
| 330 | assert result.objects_deleted == 0 |
| 331 | |
| 332 | assert await _ref_exists(db_session, repo.repo_id, oid), \ |
| 333 | "live object ref must survive a clean GC run" |
| 334 | |
| 335 | |
| 336 | # --------------------------------------------------------------------------- |
| 337 | # Test 6: GCResult fields are correctly populated |
| 338 | # --------------------------------------------------------------------------- |
| 339 | |
| 340 | @pytest.mark.asyncio |
| 341 | async def test_gc_result_fields( |
| 342 | db_session: AsyncSession, |
| 343 | ) -> None: |
| 344 | """GCResult must accurately reflect what was deleted.""" |
| 345 | repo = await create_repo(db_session, slug="gc-result-fields", owner="test-user-wire") |
| 346 | oid = _oid("result-fields-object") |
| 347 | |
| 348 | snap_orphan_id = f"snap_{uuid.uuid4().hex[:8]}" |
| 349 | snap_live_id = f"snap_{uuid.uuid4().hex[:8]}" |
| 350 | c_orphan_id = uuid.uuid4().hex |
| 351 | c_live_id = uuid.uuid4().hex |
| 352 | |
| 353 | await _insert_object(db_session, oid, repo.repo_id) |
| 354 | await _insert_ref(db_session, repo.repo_id, oid) |
| 355 | await _insert_snapshot(db_session, snap_orphan_id, {"file.md": oid}, repo_id=repo.repo_id) |
| 356 | await _insert_snapshot(db_session, snap_live_id, {}, repo_id=repo.repo_id) |
| 357 | await _insert_commit(db_session, repo.repo_id, c_orphan_id, snapshot_id=snap_orphan_id) |
| 358 | await _insert_commit(db_session, repo.repo_id, c_live_id, snapshot_id=snap_live_id) |
| 359 | await _insert_branch(db_session, repo.repo_id, c_live_id) |
| 360 | await db_session.commit() |
| 361 | |
| 362 | result = await run_gc(db_session, repo.repo_id) |
| 363 | |
| 364 | assert result.repo_id == repo.repo_id |
| 365 | assert result.commits_deleted == 1, "one orphaned commit" |
| 366 | assert result.snapshots_deleted == 1, "one orphaned snapshot" |
| 367 | assert result.object_refs_deleted == 1, "one stale ref" |
| 368 | assert result.objects_deleted == 1, "one globally orphaned object" |
| 369 | assert result.reachable_commit_count == 1, "one live commit" |
| 370 | # errors list exists even when empty |
| 371 | assert isinstance(result.errors, list) |
File History
1 commit
sha256:9590cee1e0ccd6c76528f005b95d634d80f5019f0dcb7c371e149adc31d1fb65
refactor: enforce gRPC framing on all MWP wire traffic
Sonnet 4.6
minor
⚠
156 days ago