test_pack_files.py
python
sha256:a34090cc4a394a78bd72cbbe34b08cc59525141e19135b6c0ab154f10611b9ef
debug(push/stream): instrument O-frame decode path with INF…
Sonnet 4.6
patch
121 days ago
| 1 | """Phase 5: Pack file support + GC — TDD (RED → GREEN). |
| 2 | |
| 3 | Seven tiers: |
| 4 | Tier 1 — pack_loose_objects creates pack + index files in objects/pack/ |
| 5 | Tier 2 — packed objects are readable via read_packed_object |
| 6 | Tier 3 — LocalBackend.get() falls through to pack when loose object removed |
| 7 | Tier 4 — pack_loose_objects is idempotent: same objects → same pack hash |
| 8 | Tier 5 — gc_packs deletes pack whose objects are all present in a newer pack |
| 9 | Tier 6 — pack_loose_objects returns correct counts (packed, bytes_saved) |
| 10 | Tier 7 — round-trip: write loose → pack → remove loose → still readable via backend |
| 11 | """ |
| 12 | from __future__ import annotations |
| 13 | |
| 14 | import secrets |
| 15 | from pathlib import Path |
| 16 | |
| 17 | import pytest |
| 18 | import msgpack |
| 19 | |
| 20 | from muse.core.types import blob_id |
| 21 | |
| 22 | |
| 23 | # ── helpers ─────────────────────────────────────────────────────────────────── |
| 24 | |
| 25 | def _oid(content: bytes | None = None) -> str: |
| 26 | if content is not None: |
| 27 | return blob_id(content) |
| 28 | return blob_id(secrets.token_bytes(16)) |
| 29 | |
| 30 | |
| 31 | def _repo_root(tmp_path: Path, owner: str = "gabriel", slug: str = "pack-test") -> Path: |
| 32 | root = tmp_path / owner / slug |
| 33 | (root / "refs" / "heads").mkdir(parents=True, exist_ok=True) |
| 34 | (root / "objects").mkdir(parents=True, exist_ok=True) |
| 35 | return root |
| 36 | |
| 37 | |
| 38 | def _write_loose(repo_root: Path, object_id: str, data: bytes) -> Path: |
| 39 | """Write a loose object file and return its path.""" |
| 40 | from muse.core.object_store import object_path |
| 41 | from muse.core.paths import server_objects_dir |
| 42 | p = object_path(repo_root, object_id, objects_base=server_objects_dir(repo_root)) |
| 43 | p.parent.mkdir(parents=True, exist_ok=True) |
| 44 | p.write_bytes(data) |
| 45 | return p |
| 46 | |
| 47 | |
| 48 | def _count_loose(repo_root: Path) -> int: |
| 49 | """Count loose object files under objects/sha256/.""" |
| 50 | base = repo_root / "objects" / "sha256" |
| 51 | if not base.exists(): |
| 52 | return 0 |
| 53 | return sum(1 for p in base.rglob("*") if p.is_file()) |
| 54 | |
| 55 | |
| 56 | def _pack_dir(repo_root: Path) -> Path: |
| 57 | return repo_root / "objects" / "pack" |
| 58 | |
| 59 | |
| 60 | # ── Tier 1: pack_loose_objects creates pack + index ─────────────────────────── |
| 61 | |
| 62 | class TestPackCreatesFiles: |
| 63 | def test_creates_pack_dir(self, tmp_path: Path) -> None: |
| 64 | from musehub.storage.pack import pack_loose_objects |
| 65 | |
| 66 | repo_root = _repo_root(tmp_path) |
| 67 | data = b"object content" |
| 68 | oid = _oid(data) |
| 69 | _write_loose(repo_root, oid, data) |
| 70 | |
| 71 | pack_loose_objects(repo_root) |
| 72 | |
| 73 | assert _pack_dir(repo_root).exists() |
| 74 | |
| 75 | def test_creates_pack_and_index_files(self, tmp_path: Path) -> None: |
| 76 | from musehub.storage.pack import pack_loose_objects |
| 77 | |
| 78 | repo_root = _repo_root(tmp_path) |
| 79 | data = b"hello pack" |
| 80 | oid = _oid(data) |
| 81 | _write_loose(repo_root, oid, data) |
| 82 | |
| 83 | pack_loose_objects(repo_root) |
| 84 | |
| 85 | pack_files = list(_pack_dir(repo_root).glob("*.pack")) |
| 86 | idx_files = list(_pack_dir(repo_root).glob("*.idx")) |
| 87 | assert len(pack_files) == 1, "must create exactly one .pack file" |
| 88 | assert len(idx_files) == 1, "must create exactly one .idx file" |
| 89 | assert pack_files[0].stem == idx_files[0].stem, ".pack and .idx must share a basename" |
| 90 | |
| 91 | def test_noop_when_no_loose_objects(self, tmp_path: Path) -> None: |
| 92 | from musehub.storage.pack import pack_loose_objects |
| 93 | |
| 94 | repo_root = _repo_root(tmp_path) |
| 95 | result = pack_loose_objects(repo_root) |
| 96 | |
| 97 | assert result.packed == 0 |
| 98 | assert not list(_pack_dir(repo_root).glob("*.pack")) if _pack_dir(repo_root).exists() else True |
| 99 | |
| 100 | def test_packs_all_loose_objects(self, tmp_path: Path) -> None: |
| 101 | from musehub.storage.pack import pack_loose_objects |
| 102 | |
| 103 | repo_root = _repo_root(tmp_path) |
| 104 | objects = {f"obj-{i}".encode(): None for i in range(10)} |
| 105 | oids = [] |
| 106 | for content in [f"data-{i}".encode() for i in range(10)]: |
| 107 | oid = _oid(content) |
| 108 | _write_loose(repo_root, oid, content) |
| 109 | oids.append(oid) |
| 110 | |
| 111 | result = pack_loose_objects(repo_root) |
| 112 | assert result.packed == 10 |
| 113 | |
| 114 | |
| 115 | # ── Tier 2: read_packed_object reads from pack ──────────────────────────────── |
| 116 | |
| 117 | class TestReadPackedObject: |
| 118 | def test_reads_object_from_pack(self, tmp_path: Path) -> None: |
| 119 | from musehub.storage.pack import pack_loose_objects, read_packed_object |
| 120 | |
| 121 | repo_root = _repo_root(tmp_path) |
| 122 | data = b"packed content here" |
| 123 | oid = _oid(data) |
| 124 | _write_loose(repo_root, oid, data) |
| 125 | pack_loose_objects(repo_root) |
| 126 | |
| 127 | result = read_packed_object(repo_root, oid) |
| 128 | assert result == data |
| 129 | |
| 130 | def test_returns_none_for_unknown_object(self, tmp_path: Path) -> None: |
| 131 | from musehub.storage.pack import pack_loose_objects, read_packed_object |
| 132 | |
| 133 | repo_root = _repo_root(tmp_path) |
| 134 | data = b"something" |
| 135 | oid = _oid(data) |
| 136 | _write_loose(repo_root, oid, data) |
| 137 | pack_loose_objects(repo_root) |
| 138 | |
| 139 | assert read_packed_object(repo_root, _oid()) is None |
| 140 | |
| 141 | def test_reads_all_objects_from_pack(self, tmp_path: Path) -> None: |
| 142 | from musehub.storage.pack import pack_loose_objects, read_packed_object |
| 143 | |
| 144 | repo_root = _repo_root(tmp_path) |
| 145 | items = [(f"data-{i}".encode(),) for i in range(5)] |
| 146 | oids = [] |
| 147 | for (content,) in items: |
| 148 | oid = _oid(content) |
| 149 | _write_loose(repo_root, oid, content) |
| 150 | oids.append((oid, content)) |
| 151 | |
| 152 | pack_loose_objects(repo_root) |
| 153 | |
| 154 | for oid, content in oids: |
| 155 | assert read_packed_object(repo_root, oid) == content |
| 156 | |
| 157 | def test_reads_from_correct_pack_when_multiple_exist(self, tmp_path: Path) -> None: |
| 158 | from musehub.storage.pack import pack_loose_objects, read_packed_object |
| 159 | |
| 160 | repo_root = _repo_root(tmp_path) |
| 161 | |
| 162 | # First pack: 3 objects |
| 163 | first_oids = [] |
| 164 | for i in range(3): |
| 165 | content = f"first-batch-{i}".encode() |
| 166 | oid = _oid(content) |
| 167 | _write_loose(repo_root, oid, content) |
| 168 | first_oids.append((oid, content)) |
| 169 | pack_loose_objects(repo_root) |
| 170 | |
| 171 | # Second pack: 3 more objects |
| 172 | second_oids = [] |
| 173 | for i in range(3): |
| 174 | content = f"second-batch-{i}".encode() |
| 175 | oid = _oid(content) |
| 176 | _write_loose(repo_root, oid, content) |
| 177 | second_oids.append((oid, content)) |
| 178 | pack_loose_objects(repo_root) |
| 179 | |
| 180 | for oid, content in first_oids + second_oids: |
| 181 | assert read_packed_object(repo_root, oid) == content |
| 182 | |
| 183 | |
| 184 | # ── Tier 3: LocalBackend.get() falls through to pack ───────────────────────── |
| 185 | |
| 186 | class TestLocalBackendPackFallthrough: |
| 187 | @pytest.mark.asyncio |
| 188 | async def test_get_finds_object_in_pack_after_loose_removed( |
| 189 | self, tmp_path: Path |
| 190 | ) -> None: |
| 191 | from musehub.storage.backends import LocalBackend |
| 192 | from musehub.storage.pack import pack_loose_objects |
| 193 | |
| 194 | repo_root = _repo_root(tmp_path) |
| 195 | data = b"i will be packed" |
| 196 | oid = _oid(data) |
| 197 | loose_path = _write_loose(repo_root, oid, data) |
| 198 | |
| 199 | pack_loose_objects(repo_root) |
| 200 | loose_path.unlink() # simulate GC of loose objects after packing |
| 201 | |
| 202 | backend = LocalBackend(repo_root=repo_root) |
| 203 | result = await backend.get(oid, repo_root=repo_root) |
| 204 | assert result == data |
| 205 | |
| 206 | @pytest.mark.asyncio |
| 207 | async def test_exists_returns_true_for_packed_object( |
| 208 | self, tmp_path: Path |
| 209 | ) -> None: |
| 210 | from musehub.storage.backends import LocalBackend |
| 211 | from musehub.storage.pack import pack_loose_objects |
| 212 | |
| 213 | repo_root = _repo_root(tmp_path) |
| 214 | data = b"packed existence check" |
| 215 | oid = _oid(data) |
| 216 | loose_path = _write_loose(repo_root, oid, data) |
| 217 | |
| 218 | pack_loose_objects(repo_root) |
| 219 | loose_path.unlink() |
| 220 | |
| 221 | backend = LocalBackend(repo_root=repo_root) |
| 222 | assert await backend.exists(oid, repo_root=repo_root) is True |
| 223 | |
| 224 | @pytest.mark.asyncio |
| 225 | async def test_get_returns_none_when_not_loose_or_packed( |
| 226 | self, tmp_path: Path |
| 227 | ) -> None: |
| 228 | from musehub.storage.backends import LocalBackend |
| 229 | |
| 230 | repo_root = _repo_root(tmp_path) |
| 231 | backend = LocalBackend(repo_root=repo_root) |
| 232 | assert await backend.get(_oid(), repo_root=repo_root) is None |
| 233 | |
| 234 | @pytest.mark.asyncio |
| 235 | async def test_loose_object_takes_precedence_over_pack( |
| 236 | self, tmp_path: Path |
| 237 | ) -> None: |
| 238 | """Loose object must be returned before hitting pack (hot path).""" |
| 239 | from musehub.storage.backends import LocalBackend |
| 240 | from musehub.storage.pack import pack_loose_objects |
| 241 | |
| 242 | repo_root = _repo_root(tmp_path) |
| 243 | original_data = b"original loose content" |
| 244 | oid = _oid(original_data) |
| 245 | _write_loose(repo_root, oid, original_data) |
| 246 | |
| 247 | pack_loose_objects(repo_root) |
| 248 | # Overwrite loose with different bytes (unusual but must take precedence) |
| 249 | from muse.core.object_store import object_path |
| 250 | from muse.core.paths import server_objects_dir |
| 251 | p = object_path(repo_root, oid, objects_base=server_objects_dir(repo_root)) |
| 252 | p.write_bytes(original_data) # same content in this case |
| 253 | |
| 254 | backend = LocalBackend(repo_root=repo_root) |
| 255 | result = await backend.get(oid, repo_root=repo_root) |
| 256 | assert result == original_data |
| 257 | |
| 258 | |
| 259 | # ── Tier 4: pack_loose_objects is idempotent ────────────────────────────────── |
| 260 | |
| 261 | class TestPackIdempotent: |
| 262 | def test_same_objects_produce_same_pack_name(self, tmp_path: Path) -> None: |
| 263 | from musehub.storage.pack import pack_loose_objects |
| 264 | |
| 265 | repo_root_a = _repo_root(tmp_path, slug="repo-a") |
| 266 | repo_root_b = _repo_root(tmp_path, slug="repo-b") |
| 267 | |
| 268 | contents = [f"shared-{i}".encode() for i in range(4)] |
| 269 | for content in contents: |
| 270 | oid = _oid(content) |
| 271 | _write_loose(repo_root_a, oid, content) |
| 272 | _write_loose(repo_root_b, oid, content) |
| 273 | |
| 274 | result_a = pack_loose_objects(repo_root_a) |
| 275 | result_b = pack_loose_objects(repo_root_b) |
| 276 | |
| 277 | packs_a = list(_pack_dir(repo_root_a).glob("*.pack")) |
| 278 | packs_b = list(_pack_dir(repo_root_b).glob("*.pack")) |
| 279 | assert packs_a[0].name == packs_b[0].name |
| 280 | |
| 281 | def test_packing_already_packed_objects_skips_them(self, tmp_path: Path) -> None: |
| 282 | from musehub.storage.pack import pack_loose_objects |
| 283 | |
| 284 | repo_root = _repo_root(tmp_path) |
| 285 | data = b"pack me once" |
| 286 | oid = _oid(data) |
| 287 | _write_loose(repo_root, oid, data) |
| 288 | |
| 289 | result1 = pack_loose_objects(repo_root) |
| 290 | # Second call: no new loose objects → nothing to pack |
| 291 | result2 = pack_loose_objects(repo_root) |
| 292 | |
| 293 | assert result1.packed == 1 |
| 294 | assert result2.packed == 0 |
| 295 | assert len(list(_pack_dir(repo_root).glob("*.pack"))) == 1 |
| 296 | |
| 297 | |
| 298 | # ── Tier 5: gc_packs removes superseded packs ──────────────────────────────── |
| 299 | |
| 300 | class TestGCPacks: |
| 301 | def test_gc_removes_pack_superseded_by_newer_pack(self, tmp_path: Path) -> None: |
| 302 | from musehub.storage.pack import pack_loose_objects, gc_packs |
| 303 | |
| 304 | repo_root = _repo_root(tmp_path) |
| 305 | |
| 306 | # Pack 1: objects A, B |
| 307 | for content in [b"obj-a", b"obj-b"]: |
| 308 | _write_loose(repo_root, _oid(content), content) |
| 309 | pack_loose_objects(repo_root) |
| 310 | |
| 311 | # Pack 2: objects A, B, C (superset) |
| 312 | for content in [b"obj-a", b"obj-b", b"obj-c"]: |
| 313 | _write_loose(repo_root, _oid(content), content) |
| 314 | pack_loose_objects(repo_root) |
| 315 | |
| 316 | packs_before = len(list(_pack_dir(repo_root).glob("*.pack"))) |
| 317 | gc_result = gc_packs(repo_root) |
| 318 | packs_after = len(list(_pack_dir(repo_root).glob("*.pack"))) |
| 319 | |
| 320 | assert gc_result.packs_deleted >= 1 |
| 321 | assert packs_after < packs_before |
| 322 | |
| 323 | def test_gc_keeps_pack_with_unique_objects(self, tmp_path: Path) -> None: |
| 324 | from musehub.storage.pack import pack_loose_objects, gc_packs |
| 325 | |
| 326 | repo_root = _repo_root(tmp_path) |
| 327 | |
| 328 | # Pack 1: object A only — prune loose A so it doesn't bleed into pack 2 |
| 329 | content_a = b"unique-obj-a" |
| 330 | oid_a = _oid(content_a) |
| 331 | loose_a = _write_loose(repo_root, oid_a, content_a) |
| 332 | pack_loose_objects(repo_root) |
| 333 | loose_a.unlink() # prune: A now lives only in pack 1 |
| 334 | |
| 335 | # Pack 2: object B only (no overlap with pack 1) |
| 336 | content_b = b"unique-obj-b" |
| 337 | _write_loose(repo_root, _oid(content_b), content_b) |
| 338 | pack_loose_objects(repo_root) |
| 339 | |
| 340 | packs_before = len(list(_pack_dir(repo_root).glob("*.pack"))) |
| 341 | gc_result = gc_packs(repo_root) |
| 342 | |
| 343 | assert gc_result.packs_deleted == 0, "must not delete packs with unique objects" |
| 344 | |
| 345 | def test_gc_noop_with_single_pack(self, tmp_path: Path) -> None: |
| 346 | from musehub.storage.pack import pack_loose_objects, gc_packs |
| 347 | |
| 348 | repo_root = _repo_root(tmp_path) |
| 349 | data = b"solo pack" |
| 350 | _write_loose(repo_root, _oid(data), data) |
| 351 | pack_loose_objects(repo_root) |
| 352 | |
| 353 | result = gc_packs(repo_root) |
| 354 | assert result.packs_deleted == 0 |
| 355 | |
| 356 | |
| 357 | # ── Tier 6: PackResult counts are accurate ──────────────────────────────────── |
| 358 | |
| 359 | class TestPackResult: |
| 360 | def test_packed_count_matches_loose_objects(self, tmp_path: Path) -> None: |
| 361 | from musehub.storage.pack import pack_loose_objects |
| 362 | |
| 363 | repo_root = _repo_root(tmp_path) |
| 364 | n = 7 |
| 365 | for i in range(n): |
| 366 | content = f"count-me-{i}".encode() |
| 367 | _write_loose(repo_root, _oid(content), content) |
| 368 | |
| 369 | result = pack_loose_objects(repo_root) |
| 370 | assert result.packed == n |
| 371 | |
| 372 | def test_bytes_packed_sums_object_sizes(self, tmp_path: Path) -> None: |
| 373 | from musehub.storage.pack import pack_loose_objects |
| 374 | |
| 375 | repo_root = _repo_root(tmp_path) |
| 376 | contents = [b"x" * 100, b"y" * 200, b"z" * 300] |
| 377 | total = sum(len(c) for c in contents) |
| 378 | for content in contents: |
| 379 | _write_loose(repo_root, _oid(content), content) |
| 380 | |
| 381 | result = pack_loose_objects(repo_root) |
| 382 | assert result.bytes_packed == total |
| 383 | |
| 384 | |
| 385 | # ── Tier 7: full round-trip ─────────────────────────────────────────────────── |
| 386 | |
| 387 | class TestPackRoundTrip: |
| 388 | @pytest.mark.asyncio |
| 389 | async def test_write_pack_remove_loose_still_readable( |
| 390 | self, tmp_path: Path |
| 391 | ) -> None: |
| 392 | from musehub.storage.backends import LocalBackend |
| 393 | from musehub.storage.pack import pack_loose_objects |
| 394 | |
| 395 | repo_root = _repo_root(tmp_path) |
| 396 | backend = LocalBackend(repo_root=repo_root) |
| 397 | |
| 398 | # 1. Write 20 loose objects via backend |
| 399 | objects = {} |
| 400 | for i in range(20): |
| 401 | content = f"round-trip-{i}-{'x' * 50}".encode() |
| 402 | oid = _oid(content) |
| 403 | await backend.put(oid, content, repo_root=repo_root) |
| 404 | objects[oid] = content |
| 405 | |
| 406 | # 2. Pack them |
| 407 | result = pack_loose_objects(repo_root) |
| 408 | assert result.packed == 20 |
| 409 | |
| 410 | # 3. Remove loose files (simulate post-pack GC) |
| 411 | from muse.core.paths import server_objects_dir |
| 412 | algo_dir = server_objects_dir(repo_root) / "sha256" |
| 413 | for shard_dir in algo_dir.iterdir(): |
| 414 | for obj_file in shard_dir.iterdir(): |
| 415 | obj_file.unlink() |
| 416 | |
| 417 | # 4. All 20 objects must still be readable via backend |
| 418 | for oid, content in objects.items(): |
| 419 | result_data = await backend.get(oid, repo_root=repo_root) |
| 420 | assert result_data == content, f"object {oid[:20]}... not found after packing" |
| 421 | |
| 422 | @pytest.mark.asyncio |
| 423 | async def test_new_loose_objects_after_pack_are_readable( |
| 424 | self, tmp_path: Path |
| 425 | ) -> None: |
| 426 | """Objects written after packing must be readable as loose files.""" |
| 427 | from musehub.storage.backends import LocalBackend |
| 428 | from musehub.storage.pack import pack_loose_objects |
| 429 | |
| 430 | repo_root = _repo_root(tmp_path) |
| 431 | backend = LocalBackend(repo_root=repo_root) |
| 432 | |
| 433 | # Pack first batch |
| 434 | old_content = b"old-object" |
| 435 | old_oid = _oid(old_content) |
| 436 | await backend.put(old_oid, old_content, repo_root=repo_root) |
| 437 | pack_loose_objects(repo_root) |
| 438 | |
| 439 | # Write new loose object AFTER packing |
| 440 | new_content = b"new-object-after-pack" |
| 441 | new_oid = _oid(new_content) |
| 442 | await backend.put(new_oid, new_content, repo_root=repo_root) |
| 443 | |
| 444 | # Both must be readable |
| 445 | assert await backend.get(new_oid, repo_root=repo_root) == new_content |
| 446 | assert await backend.get(old_oid, repo_root=repo_root) == old_content |
File History
1 commit
sha256:a34090cc4a394a78bd72cbbe34b08cc59525141e19135b6c0ab154f10611b9ef
debug(push/stream): instrument O-frame decode path with INF…
Sonnet 4.6
patch
121 days ago