test_gc_corrupt_commit_object_retention.py
python
sha256:51ce277f663e01a43eaffbe77509b1de7ac2d4251b55d23306304bcdeb92c90d
feat(pack): delta-encode snapshots in MPackBundle wire format
Sonnet 4.6
minor
⚠ breaking
121 days ago
| 1 | """Tests for Bug 10: GC deletes objects reachable from a corrupt commit. |
| 2 | |
| 3 | When a commit file has a corrupt snapshot_id field (bit-flip, tampered, or |
| 4 | previously written by the now-fixed from_dict timestamp substitution bug), |
| 5 | _collect_reachable_objects returns an empty set for that commit's snapshot. |
| 6 | muse gc then deletes those objects, permanently destroying file content. |
| 7 | |
| 8 | The sequence of doom: |
| 9 | 1. Commit A on disk: snapshot_id = "f"*64 (corrupt — should be "S1") |
| 10 | 2. `get_all_commits` returns this commit (no hash verification) |
| 11 | 3. `read_snapshot(root, "f"*64)` → None (file for "f"*64 doesn't exist) |
| 12 | 4. Objects from the REAL snapshot S1 are NOT added to reachable |
| 13 | 5. `muse gc` deletes those objects (no other commit references them) |
| 14 | 6. The working tree cannot be reconstructed from commit A |
| 15 | |
| 16 | Fix: `_collect_reachable_objects` must also scan snapshot files directly |
| 17 | (without hash verification) when read_snapshot returns None for a commit's |
| 18 | snapshot_id. This conservatively retains any object referenced in any |
| 19 | snapshot manifest on disk, regardless of whether the commit's snapshot_id |
| 20 | field is correct. |
| 21 | |
| 22 | Scope of tests |
| 23 | -------------- |
| 24 | Unit (_collect_reachable_objects): |
| 25 | - Objects reachable from a valid commit are retained |
| 26 | - Objects reachable from a commit with corrupt snapshot_id (field points |
| 27 | to non-existent snapshot) are still retained via raw snapshot scan |
| 28 | - Objects reachable from a commit with corrupt snapshot_id (field points |
| 29 | to a WRONG existing snapshot) are still retained via raw snapshot scan |
| 30 | - GC does NOT delete objects that are in ANY snapshot on disk, even orphaned |
| 31 | - Empty store returns empty reachable set (no crash) |
| 32 | - Multiple commits, one corrupt: all objects from all snapshots retained |
| 33 | |
| 34 | Integration (run_gc with corrupt commit): |
| 35 | - run_gc dry_run=True does not delete anything from a store with corrupt commit |
| 36 | - run_gc does not delete objects from corrupt commit's snapshot (default mode) |
| 37 | - run_gc --full does not delete objects from corrupt commit's snapshot |
| 38 | - Objects from a legitimately unreachable commit ARE deleted (GC still works) |
| 39 | |
| 40 | Stress: |
| 41 | - 50 commits, 5 with corrupt snapshot_ids: all objects from all 50 snapshots retained |
| 42 | """ |
| 43 | from __future__ import annotations |
| 44 | |
| 45 | type _FileStore = dict[str, bytes] |
| 46 | |
| 47 | import datetime |
| 48 | import hashlib |
| 49 | import pathlib |
| 50 | |
| 51 | import msgpack |
| 52 | import pytest |
| 53 | |
| 54 | from muse.core.gc import _collect_reachable_objects, run_gc |
| 55 | from muse.core.paths import commits_dir, heads_dir, muse_dir, snapshots_dir |
| 56 | from muse.core.types import NULL_COMMIT_ID |
| 57 | from muse.core.object_store import write_object |
| 58 | from muse.core.snapshot import compute_commit_id, compute_snapshot_id |
| 59 | from muse.core.store import ( |
| 60 | CommitRecord, |
| 61 | SnapshotRecord, |
| 62 | read_snapshot, |
| 63 | write_commit, |
| 64 | write_snapshot, |
| 65 | ) |
| 66 | |
| 67 | _TS = datetime.datetime(2024, 6, 15, 10, 0, 0, tzinfo=datetime.timezone.utc) |
| 68 | |
| 69 | |
| 70 | def _make_repo(tmp_path: pathlib.Path) -> pathlib.Path: |
| 71 | repo = tmp_path / "repo" |
| 72 | repo.mkdir() |
| 73 | muse_dir(repo).mkdir() |
| 74 | return repo |
| 75 | |
| 76 | |
| 77 | def _write_object(repo: pathlib.Path, content: bytes) -> str: |
| 78 | from muse.core.types import blob_id |
| 79 | oid = blob_id(content) |
| 80 | write_object(repo, oid, content) |
| 81 | return oid |
| 82 | |
| 83 | |
| 84 | def _make_snapshot(repo: pathlib.Path, files: _FileStore) -> SnapshotRecord: |
| 85 | manifest = {} |
| 86 | for path, content in files.items(): |
| 87 | oid = _write_object(repo, content) |
| 88 | manifest[path] = oid |
| 89 | snap_id = compute_snapshot_id(manifest) |
| 90 | snap = SnapshotRecord( |
| 91 | snapshot_id=snap_id, |
| 92 | manifest=manifest, |
| 93 | directories=[], |
| 94 | created_at=_TS, |
| 95 | note="", |
| 96 | ) |
| 97 | write_snapshot(repo, snap) |
| 98 | return snap |
| 99 | |
| 100 | |
| 101 | def _make_commit( |
| 102 | repo: pathlib.Path, |
| 103 | snap_id: str, |
| 104 | *, |
| 105 | message: str = "test", |
| 106 | ts: datetime.datetime = _TS, |
| 107 | parent: str | None = None, |
| 108 | ) -> CommitRecord: |
| 109 | parent_ids = [parent] if parent else [] |
| 110 | commit_id = compute_commit_id( |
| 111 | parent_ids=parent_ids, |
| 112 | snapshot_id=snap_id, |
| 113 | message=message, |
| 114 | committed_at_iso=ts.isoformat(), |
| 115 | author="gabriel", |
| 116 | ) |
| 117 | record = CommitRecord( |
| 118 | repo_id="gc-test", |
| 119 | commit_id=commit_id, |
| 120 | branch="main", |
| 121 | snapshot_id=snap_id, |
| 122 | message=message, |
| 123 | committed_at=ts, |
| 124 | parent_commit_id=parent, |
| 125 | parent2_commit_id=None, |
| 126 | author="gabriel", |
| 127 | metadata={}, |
| 128 | structured_delta=None, |
| 129 | sem_ver_bump="none", |
| 130 | breaking_changes=[], |
| 131 | agent_id="", |
| 132 | model_id="", |
| 133 | toolchain_id="", |
| 134 | prompt_hash="", |
| 135 | signature="", |
| 136 | signer_key_id="", |
| 137 | reviewed_by=[], |
| 138 | test_runs=0, |
| 139 | ) |
| 140 | write_commit(repo, record) |
| 141 | return record |
| 142 | |
| 143 | |
| 144 | def _corrupt_commit_snapshot_id( |
| 145 | repo: pathlib.Path, commit_id: str, bad_snapshot_id: str = "f" * 64 |
| 146 | ) -> None: |
| 147 | """Directly corrupt the snapshot_id field in a commit file on disk.""" |
| 148 | from muse.core.types import split_id |
| 149 | algo, hex_id = split_id(commit_id) |
| 150 | path = commits_dir(repo) / algo / f"{hex_id}.msgpack" |
| 151 | data = msgpack.unpackb(path.read_bytes(), raw=False) |
| 152 | data["snapshot_id"] = bad_snapshot_id |
| 153 | path.write_bytes(msgpack.packb(data, use_bin_type=True)) |
| 154 | |
| 155 | |
| 156 | # ────────────────────────────────────────────────────────────────────────────── |
| 157 | # Unit: _collect_reachable_objects |
| 158 | # ────────────────────────────────────────────────────────────────────────────── |
| 159 | |
| 160 | class TestCollectReachableObjects: |
| 161 | |
| 162 | def test_valid_commit_objects_retained(self, tmp_path: pathlib.Path) -> None: |
| 163 | repo = _make_repo(tmp_path) |
| 164 | snap = _make_snapshot(repo, {"src/a.py": b"content_a"}) |
| 165 | _make_commit(repo, snap.snapshot_id) |
| 166 | reachable = _collect_reachable_objects(repo) |
| 167 | for oid in snap.manifest.values(): |
| 168 | assert oid in reachable, f"Object {oid[:8]} should be reachable" |
| 169 | |
| 170 | def test_corrupt_snapshot_id_objects_still_retained(self, tmp_path: pathlib.Path) -> None: |
| 171 | """BUG: When commit's snapshot_id is corrupt, objects are not retained.""" |
| 172 | repo = _make_repo(tmp_path) |
| 173 | snap = _make_snapshot(repo, {"src/main.py": b"important data"}) |
| 174 | commit = _make_commit(repo, snap.snapshot_id) |
| 175 | |
| 176 | # Corrupt the snapshot_id to a non-existent value |
| 177 | _corrupt_commit_snapshot_id(repo, commit.commit_id, "f" * 64) |
| 178 | |
| 179 | # Verify that read_snapshot now fails for the commit |
| 180 | stored = _collect_reachable_objects.__module__ # just to check we're testing the right thing |
| 181 | |
| 182 | reachable = _collect_reachable_objects(repo) |
| 183 | for oid in snap.manifest.values(): |
| 184 | assert oid in reachable, ( |
| 185 | f"DATA LOSS: Object {oid[:8]} from snapshot {snap.snapshot_id[:8]} " |
| 186 | f"was NOT retained by GC after the commit's snapshot_id was corrupted. " |
| 187 | f"Running muse gc would delete this object permanently." |
| 188 | ) |
| 189 | |
| 190 | def test_corrupt_snapshot_id_points_to_wrong_existing_snapshot(self, tmp_path: pathlib.Path) -> None: |
| 191 | """Even worse: corrupt snapshot_id points to a DIFFERENT existing snapshot. |
| 192 | The objects from the ORIGINAL snapshot are still not retained. |
| 193 | """ |
| 194 | repo = _make_repo(tmp_path) |
| 195 | snap1 = _make_snapshot(repo, {"src/file1.py": b"content 1"}) |
| 196 | snap2 = _make_snapshot(repo, {"src/file2.py": b"content 2"}) |
| 197 | commit = _make_commit(repo, snap1.snapshot_id) |
| 198 | |
| 199 | # Corrupt: now points to snap2's ID instead of snap1's |
| 200 | _corrupt_commit_snapshot_id(repo, commit.commit_id, snap2.snapshot_id) |
| 201 | |
| 202 | reachable = _collect_reachable_objects(repo) |
| 203 | for oid in snap1.manifest.values(): |
| 204 | assert oid in reachable, ( |
| 205 | f"DATA LOSS: Object from snap1 ({oid[:8]}) not retained — " |
| 206 | f"corrupt snapshot_id pointed to snap2 instead, and snap1's " |
| 207 | f"objects were not retained. GC would delete them." |
| 208 | ) |
| 209 | |
| 210 | def test_two_commits_one_corrupt_all_objects_retained(self, tmp_path: pathlib.Path) -> None: |
| 211 | """One corrupt commit must not prevent the other commit's objects from being retained.""" |
| 212 | repo = _make_repo(tmp_path) |
| 213 | snap1 = _make_snapshot(repo, {"a.py": b"aaa"}) |
| 214 | snap2 = _make_snapshot(repo, {"b.py": b"bbb"}) |
| 215 | commit1 = _make_commit(repo, snap1.snapshot_id, message="c1") |
| 216 | _make_commit(repo, snap2.snapshot_id, message="c2") |
| 217 | |
| 218 | _corrupt_commit_snapshot_id(repo, commit1.commit_id, NULL_COMMIT_ID) |
| 219 | |
| 220 | reachable = _collect_reachable_objects(repo) |
| 221 | # Both snapshots' objects must be retained |
| 222 | for oid in snap1.manifest.values(): |
| 223 | assert oid in reachable, f"snap1 object {oid[:8]} not retained after corruption" |
| 224 | for oid in snap2.manifest.values(): |
| 225 | assert oid in reachable, f"snap2 object {oid[:8]} not retained" |
| 226 | |
| 227 | def test_empty_store_no_crash(self, tmp_path: pathlib.Path) -> None: |
| 228 | repo = _make_repo(tmp_path) |
| 229 | reachable = _collect_reachable_objects(repo) |
| 230 | assert reachable == set() |
| 231 | |
| 232 | def test_valid_commit_chain_all_retained(self, tmp_path: pathlib.Path) -> None: |
| 233 | repo = _make_repo(tmp_path) |
| 234 | snap1 = _make_snapshot(repo, {"a.py": b"v1"}) |
| 235 | snap2 = _make_snapshot(repo, {"a.py": b"v2"}) |
| 236 | c1 = _make_commit(repo, snap1.snapshot_id, message="v1") |
| 237 | _make_commit(repo, snap2.snapshot_id, message="v2", parent=c1.commit_id) |
| 238 | |
| 239 | reachable = _collect_reachable_objects(repo) |
| 240 | for oid in snap1.manifest.values(): |
| 241 | assert oid in reachable |
| 242 | for oid in snap2.manifest.values(): |
| 243 | assert oid in reachable |
| 244 | |
| 245 | |
| 246 | # ────────────────────────────────────────────────────────────────────────────── |
| 247 | # Integration: run_gc with corrupt commit |
| 248 | # ────────────────────────────────────────────────────────────────────────────── |
| 249 | |
| 250 | class TestRunGcCorruptCommit: |
| 251 | |
| 252 | def test_gc_dry_run_reports_no_collected_objects_for_corrupt_commit(self, tmp_path: pathlib.Path) -> None: |
| 253 | """Dry run must show 0 objects to collect when all objects are reachable.""" |
| 254 | repo = _make_repo(tmp_path) |
| 255 | snap = _make_snapshot(repo, {"main.py": b"code"}) |
| 256 | commit = _make_commit(repo, snap.snapshot_id) |
| 257 | _corrupt_commit_snapshot_id(repo, commit.commit_id) |
| 258 | |
| 259 | result = run_gc(repo, dry_run=True, grace_period_seconds=0) |
| 260 | assert result.collected_count == 0, ( |
| 261 | f"BUG: dry_run GC reports {result.collected_count} objects to collect, " |
| 262 | f"but all objects are reachable (just via a corrupt commit). " |
| 263 | f"Running without dry_run would permanently delete these objects." |
| 264 | ) |
| 265 | |
| 266 | def test_gc_does_not_delete_objects_from_corrupt_commit(self, tmp_path: pathlib.Path) -> None: |
| 267 | """Objects from a corrupt commit's snapshot must survive GC.""" |
| 268 | repo = _make_repo(tmp_path) |
| 269 | snap = _make_snapshot(repo, {"main.py": b"valuable content"}) |
| 270 | commit = _make_commit(repo, snap.snapshot_id) |
| 271 | _corrupt_commit_snapshot_id(repo, commit.commit_id) |
| 272 | |
| 273 | result = run_gc(repo, dry_run=False, grace_period_seconds=0) |
| 274 | assert result.collected_count == 0, ( |
| 275 | f"DATA LOSS: GC deleted {result.collected_count} object(s) that were " |
| 276 | f"reachable from a commit with a corrupt snapshot_id. Those objects " |
| 277 | f"are now permanently gone." |
| 278 | ) |
| 279 | # Verify the object is still on disk |
| 280 | oid = list(snap.manifest.values())[0] |
| 281 | from muse.core.object_store import read_object |
| 282 | assert read_object(repo, oid) is not None, ( |
| 283 | f"CONFIRMED DATA LOSS: Object {oid[:8]} was deleted by GC." |
| 284 | ) |
| 285 | |
| 286 | def test_gc_full_retains_objects_when_corrupt_snapshot_file_exists(self, tmp_path: pathlib.Path) -> None: |
| 287 | """GC --full must retain objects when a snapshot FILE exists at the correct |
| 288 | path but its stored snapshot_id field doesn't match the computed hash. |
| 289 | The commit references snap_id S1; the file at S1 has a corrupt snapshot_id |
| 290 | field (not the manifest) so _verify_snapshot_id fails. Our raw-fallback |
| 291 | path must read the manifest directly and retain its object IDs. |
| 292 | """ |
| 293 | repo = _make_repo(tmp_path) |
| 294 | snap = _make_snapshot(repo, {"main.py": b"important"}) |
| 295 | commit = _make_commit(repo, snap.snapshot_id) |
| 296 | |
| 297 | # Point the branch ref at the commit (making it reachable) |
| 298 | h_dir = heads_dir(repo) |
| 299 | h_dir.mkdir(parents=True, exist_ok=True) |
| 300 | (h_dir / "main").write_text(commit.commit_id) |
| 301 | |
| 302 | # Corrupt the SNAPSHOT FILE's stored snapshot_id field (NOT the manifest). |
| 303 | # The manifest still has the correct object IDs. |
| 304 | # _verify_snapshot_id will fail (recomputed hash != stored snapshot_id field), |
| 305 | # causing read_snapshot to return None — but the object IDs in the manifest |
| 306 | # are still valid and should be retained by our raw-fallback path. |
| 307 | from muse.core.types import split_id |
| 308 | _algo, _hex = split_id(snap.snapshot_id) |
| 309 | snap_path = snapshots_dir(repo) / _algo / f"{_hex}.msgpack" |
| 310 | snap_data = msgpack.unpackb(snap_path.read_bytes(), raw=False) |
| 311 | oid = list(snap_data["manifest"].values())[0] # save the real object ID |
| 312 | snap_data["snapshot_id"] = f"corrupt_id_{'0' * 53}" # corrupt the stored ID field |
| 313 | snap_path.write_bytes(msgpack.packb(snap_data, use_bin_type=True)) |
| 314 | |
| 315 | result = run_gc(repo, dry_run=False, grace_period_seconds=0, full=True) |
| 316 | from muse.core.object_store import read_object |
| 317 | obj = read_object(repo, oid) |
| 318 | assert obj is not None, ( |
| 319 | f"DATA LOSS: GC --full deleted object {oid[:8]} that was referenced " |
| 320 | f"in a corrupt snapshot file (corrupt stored snapshot_id field, " |
| 321 | f"valid manifest). The raw-fallback path must have retained it." |
| 322 | ) |
| 323 | |
| 324 | def test_gc_full_corrupt_commit_snapshot_id_no_file_no_crash(self, tmp_path: pathlib.Path) -> None: |
| 325 | """Document the known edge case: if a commit's snapshot_id is corrupt AND |
| 326 | the referenced file doesn't exist, GC --full cannot retain those objects. |
| 327 | Users must run `muse verify-pack` before `muse gc --full` in this scenario. |
| 328 | This test verifies no crash occurs (the behavior is a known limitation). |
| 329 | """ |
| 330 | repo = _make_repo(tmp_path) |
| 331 | snap = _make_snapshot(repo, {"main.py": b"important"}) |
| 332 | commit = _make_commit(repo, snap.snapshot_id) |
| 333 | |
| 334 | h_dir = heads_dir(repo) |
| 335 | h_dir.mkdir(parents=True, exist_ok=True) |
| 336 | (h_dir / "main").write_text(commit.commit_id) |
| 337 | |
| 338 | # Corrupt the commit so its snapshot_id points to a non-existent file |
| 339 | _corrupt_commit_snapshot_id(repo, commit.commit_id, "f" * 64) |
| 340 | |
| 341 | # Known limitation: when commit.snapshot_id points to a non-existent file, |
| 342 | # GC --full cannot determine which objects to retain. No crash must occur. |
| 343 | result = run_gc(repo, dry_run=False, grace_period_seconds=0, full=True) |
| 344 | # No assertion about objects — this is the documented limitation. |
| 345 | |
| 346 | def test_gc_still_collects_truly_orphaned_objects(self, tmp_path: pathlib.Path) -> None: |
| 347 | """Regression: GC must still delete truly unreachable objects.""" |
| 348 | repo = _make_repo(tmp_path) |
| 349 | # Write an object that is NOT in any snapshot |
| 350 | orphan_content = b"orphaned content - no snapshot references this" |
| 351 | orphan_oid = _write_object(repo, orphan_content) |
| 352 | |
| 353 | # Write a valid commit with a snapshot that does NOT reference the orphan |
| 354 | snap = _make_snapshot(repo, {"other.py": b"other content"}) |
| 355 | _make_commit(repo, snap.snapshot_id) |
| 356 | |
| 357 | result = run_gc(repo, dry_run=False, grace_period_seconds=0) |
| 358 | assert orphan_oid in result.collected_ids, ( |
| 359 | f"Orphaned object {orphan_oid[:8]} was not collected by GC. " |
| 360 | f"GC is too conservative." |
| 361 | ) |
| 362 | |
| 363 | |
| 364 | # ────────────────────────────────────────────────────────────────────────────── |
| 365 | # Stress |
| 366 | # ────────────────────────────────────────────────────────────────────────────── |
| 367 | |
| 368 | class TestGcCorruptStress: |
| 369 | |
| 370 | def test_50_commits_5_corrupt_all_objects_retained(self, tmp_path: pathlib.Path) -> None: |
| 371 | """50 commits, 5 with corrupt snapshot_ids: all objects retained, no crash.""" |
| 372 | repo = _make_repo(tmp_path) |
| 373 | commit_records = [] |
| 374 | all_oids: set[str] = set() |
| 375 | |
| 376 | for i in range(50): |
| 377 | content = f"content_{i}".encode() |
| 378 | snap = _make_snapshot(repo, {f"f{i}.py": content}) |
| 379 | ts = _TS + datetime.timedelta(seconds=i) |
| 380 | commit = _make_commit(repo, snap.snapshot_id, message=f"commit {i}", ts=ts) |
| 381 | commit_records.append(commit) |
| 382 | all_oids.update(snap.manifest.values()) |
| 383 | |
| 384 | # Corrupt 5 commits (indices 10, 20, 30, 40, 49) |
| 385 | corrupt_indices = {10, 20, 30, 40, 49} |
| 386 | for idx in corrupt_indices: |
| 387 | _corrupt_commit_snapshot_id( |
| 388 | repo, commit_records[idx].commit_id, "9" * 64 |
| 389 | ) |
| 390 | |
| 391 | reachable = _collect_reachable_objects(repo) |
| 392 | |
| 393 | missing = [oid for oid in all_oids if oid not in reachable] |
| 394 | assert not missing, ( |
| 395 | f"DATA LOSS: {len(missing)} object(s) not retained by GC despite " |
| 396 | f"being reachable from snapshots on disk. " |
| 397 | f"Missing: {[o[:8] for o in missing[:5]]}" |
| 398 | ) |
File History
1 commit
sha256:51ce277f663e01a43eaffbe77509b1de7ac2d4251b55d23306304bcdeb92c90d
feat(pack): delta-encode snapshots in MPackBundle wire format
Sonnet 4.6
minor
⚠
121 days ago