test_storage_tiers.py
python
sha256:a34090cc4a394a78bd72cbbe34b08cc59525141e19135b6c0ab154f10611b9ef
debug(push/stream): instrument O-frame decode path with INF…
Sonnet 4.6
patch
121 days ago
| 1 | """Phase 6: Storage tier formalisation — TDD (RED → GREEN). |
| 2 | |
| 3 | Seven tiers: |
| 4 | Tier 1 — prune_packed removes loose objects that are covered by packs |
| 5 | Tier 2 — prune_packed preserves loose objects NOT in any pack |
| 6 | Tier 3 — classify_object returns correct StorageTier (HOT / WARM) |
| 7 | Tier 4 — classify_object returns None for unknown objects |
| 8 | Tier 5 — storage_stats returns accurate counts per tier |
| 9 | Tier 6 — auto_pack packs + prunes when loose count meets threshold |
| 10 | Tier 7 — full promotion workflow: write → pack → prune → all still readable |
| 11 | """ |
| 12 | from __future__ import annotations |
| 13 | |
| 14 | import secrets |
| 15 | from pathlib import Path |
| 16 | |
| 17 | import pytest |
| 18 | |
| 19 | from muse.core.types import blob_id |
| 20 | |
| 21 | |
| 22 | # ── helpers ─────────────────────────────────────────────────────────────────── |
| 23 | |
| 24 | def _oid(content: bytes | None = None) -> str: |
| 25 | if content is not None: |
| 26 | return blob_id(content) |
| 27 | return blob_id(secrets.token_bytes(16)) |
| 28 | |
| 29 | |
| 30 | def _repo_root(tmp_path: Path, owner: str = "gabriel", slug: str = "tier-test") -> Path: |
| 31 | root = tmp_path / owner / slug |
| 32 | (root / "refs" / "heads").mkdir(parents=True, exist_ok=True) |
| 33 | (root / "objects").mkdir(parents=True, exist_ok=True) |
| 34 | return root |
| 35 | |
| 36 | |
| 37 | def _write_loose(repo_root: Path, object_id: str, data: bytes) -> Path: |
| 38 | from muse.core.object_store import object_path |
| 39 | from muse.core.paths import server_objects_dir |
| 40 | p = object_path(repo_root, object_id, objects_base=server_objects_dir(repo_root)) |
| 41 | p.parent.mkdir(parents=True, exist_ok=True) |
| 42 | p.write_bytes(data) |
| 43 | return p |
| 44 | |
| 45 | |
| 46 | def _loose_path(repo_root: Path, object_id: str) -> Path: |
| 47 | from muse.core.object_store import object_path |
| 48 | from muse.core.paths import server_objects_dir |
| 49 | return object_path(repo_root, object_id, objects_base=server_objects_dir(repo_root)) |
| 50 | |
| 51 | |
| 52 | def _count_loose(repo_root: Path) -> int: |
| 53 | from muse.core.paths import server_objects_dir |
| 54 | base = server_objects_dir(repo_root) / "sha256" |
| 55 | if not base.exists(): |
| 56 | return 0 |
| 57 | return sum(1 for p in base.rglob("*") if p.is_file()) |
| 58 | |
| 59 | |
| 60 | def _pack_dir(repo_root: Path) -> Path: |
| 61 | return repo_root / "objects" / "pack" |
| 62 | |
| 63 | |
| 64 | # ── Tier 1: prune_packed removes covered loose objects ──────────────────────── |
| 65 | |
| 66 | class TestPrunePacked: |
| 67 | def test_removes_loose_after_packing(self, tmp_path: Path) -> None: |
| 68 | from musehub.storage.tiers import prune_packed |
| 69 | from musehub.storage.pack import pack_loose_objects |
| 70 | |
| 71 | repo_root = _repo_root(tmp_path) |
| 72 | data = b"i will be pruned" |
| 73 | oid = _oid(data) |
| 74 | _write_loose(repo_root, oid, data) |
| 75 | |
| 76 | pack_loose_objects(repo_root) |
| 77 | assert _loose_path(repo_root, oid).exists(), "loose must exist before prune" |
| 78 | |
| 79 | result = prune_packed(repo_root) |
| 80 | |
| 81 | assert not _loose_path(repo_root, oid).exists(), "loose must be removed after prune" |
| 82 | assert result.pruned == 1 |
| 83 | |
| 84 | def test_removes_all_covered_loose_objects(self, tmp_path: Path) -> None: |
| 85 | from musehub.storage.tiers import prune_packed |
| 86 | from musehub.storage.pack import pack_loose_objects |
| 87 | |
| 88 | repo_root = _repo_root(tmp_path) |
| 89 | oids = [] |
| 90 | for i in range(8): |
| 91 | data = f"object-{i}".encode() |
| 92 | oid = _oid(data) |
| 93 | _write_loose(repo_root, oid, data) |
| 94 | oids.append(oid) |
| 95 | |
| 96 | pack_loose_objects(repo_root) |
| 97 | result = prune_packed(repo_root) |
| 98 | |
| 99 | assert result.pruned == 8 |
| 100 | assert _count_loose(repo_root) == 0 |
| 101 | |
| 102 | def test_noop_when_no_packs_exist(self, tmp_path: Path) -> None: |
| 103 | from musehub.storage.tiers import prune_packed |
| 104 | |
| 105 | repo_root = _repo_root(tmp_path) |
| 106 | data = b"unpacked" |
| 107 | oid = _oid(data) |
| 108 | _write_loose(repo_root, oid, data) |
| 109 | |
| 110 | result = prune_packed(repo_root) |
| 111 | |
| 112 | assert result.pruned == 0 |
| 113 | assert _loose_path(repo_root, oid).exists(), "loose must survive when no packs exist" |
| 114 | |
| 115 | def test_noop_when_no_loose_exists(self, tmp_path: Path) -> None: |
| 116 | from musehub.storage.tiers import prune_packed |
| 117 | from musehub.storage.pack import pack_loose_objects |
| 118 | |
| 119 | repo_root = _repo_root(tmp_path) |
| 120 | data = b"pack then prune" |
| 121 | oid = _oid(data) |
| 122 | _write_loose(repo_root, oid, data) |
| 123 | pack_loose_objects(repo_root) |
| 124 | _loose_path(repo_root, oid).unlink() # already gone |
| 125 | |
| 126 | result = prune_packed(repo_root) |
| 127 | assert result.pruned == 0 |
| 128 | |
| 129 | |
| 130 | # ── Tier 2: prune_packed preserves unpackaged loose objects ─────────────────── |
| 131 | |
| 132 | class TestPrunePackedPreservesUnpacked: |
| 133 | def test_preserves_loose_objects_not_in_any_pack(self, tmp_path: Path) -> None: |
| 134 | from musehub.storage.tiers import prune_packed |
| 135 | from musehub.storage.pack import pack_loose_objects |
| 136 | |
| 137 | repo_root = _repo_root(tmp_path) |
| 138 | |
| 139 | # Pack object A |
| 140 | data_a = b"packed-object-a" |
| 141 | oid_a = _oid(data_a) |
| 142 | _write_loose(repo_root, oid_a, data_a) |
| 143 | pack_loose_objects(repo_root) |
| 144 | |
| 145 | # Write object B after packing — it is NOT in any pack |
| 146 | data_b = b"unpacked-object-b" |
| 147 | oid_b = _oid(data_b) |
| 148 | _write_loose(repo_root, oid_b, data_b) |
| 149 | |
| 150 | result = prune_packed(repo_root) |
| 151 | |
| 152 | assert result.pruned == 1 |
| 153 | assert not _loose_path(repo_root, oid_a).exists(), "A must be pruned" |
| 154 | assert _loose_path(repo_root, oid_b).exists(), "B must survive" |
| 155 | |
| 156 | def test_mixed_prune_removes_only_covered(self, tmp_path: Path) -> None: |
| 157 | from musehub.storage.tiers import prune_packed |
| 158 | from musehub.storage.pack import pack_loose_objects |
| 159 | |
| 160 | repo_root = _repo_root(tmp_path) |
| 161 | |
| 162 | # Pack first batch |
| 163 | packed_oids = [] |
| 164 | for i in range(5): |
| 165 | data = f"packed-{i}".encode() |
| 166 | oid = _oid(data) |
| 167 | _write_loose(repo_root, oid, data) |
| 168 | packed_oids.append(oid) |
| 169 | pack_loose_objects(repo_root) |
| 170 | |
| 171 | # New loose objects written after packing |
| 172 | new_oids = [] |
| 173 | for i in range(3): |
| 174 | data = f"new-{i}".encode() |
| 175 | oid = _oid(data) |
| 176 | _write_loose(repo_root, oid, data) |
| 177 | new_oids.append(oid) |
| 178 | |
| 179 | result = prune_packed(repo_root) |
| 180 | |
| 181 | assert result.pruned == 5 |
| 182 | for oid in packed_oids: |
| 183 | assert not _loose_path(repo_root, oid).exists() |
| 184 | for oid in new_oids: |
| 185 | assert _loose_path(repo_root, oid).exists() |
| 186 | |
| 187 | |
| 188 | # ── Tier 3: classify_object returns correct tier ────────────────────────────── |
| 189 | |
| 190 | class TestClassifyObject: |
| 191 | def test_hot_for_loose_object(self, tmp_path: Path) -> None: |
| 192 | from musehub.storage.tiers import classify_object, StorageTier |
| 193 | |
| 194 | repo_root = _repo_root(tmp_path) |
| 195 | data = b"hot object" |
| 196 | oid = _oid(data) |
| 197 | _write_loose(repo_root, oid, data) |
| 198 | |
| 199 | assert classify_object(repo_root, oid) is StorageTier.HOT |
| 200 | |
| 201 | def test_warm_for_packed_object_after_prune(self, tmp_path: Path) -> None: |
| 202 | from musehub.storage.tiers import classify_object, StorageTier, prune_packed |
| 203 | from musehub.storage.pack import pack_loose_objects |
| 204 | |
| 205 | repo_root = _repo_root(tmp_path) |
| 206 | data = b"warm object" |
| 207 | oid = _oid(data) |
| 208 | _write_loose(repo_root, oid, data) |
| 209 | pack_loose_objects(repo_root) |
| 210 | prune_packed(repo_root) |
| 211 | |
| 212 | assert classify_object(repo_root, oid) is StorageTier.WARM |
| 213 | |
| 214 | def test_hot_before_prune_loose_and_pack_both_present(self, tmp_path: Path) -> None: |
| 215 | """Loose takes precedence — object is HOT even if also in a pack.""" |
| 216 | from musehub.storage.tiers import classify_object, StorageTier |
| 217 | from musehub.storage.pack import pack_loose_objects |
| 218 | |
| 219 | repo_root = _repo_root(tmp_path) |
| 220 | data = b"still hot" |
| 221 | oid = _oid(data) |
| 222 | _write_loose(repo_root, oid, data) |
| 223 | pack_loose_objects(repo_root) # loose still exists after packing |
| 224 | |
| 225 | assert classify_object(repo_root, oid) is StorageTier.HOT |
| 226 | |
| 227 | def test_warm_for_object_in_pack_loose_removed(self, tmp_path: Path) -> None: |
| 228 | from musehub.storage.tiers import classify_object, StorageTier |
| 229 | from musehub.storage.pack import pack_loose_objects |
| 230 | |
| 231 | repo_root = _repo_root(tmp_path) |
| 232 | data = b"pack me" |
| 233 | oid = _oid(data) |
| 234 | loose = _write_loose(repo_root, oid, data) |
| 235 | pack_loose_objects(repo_root) |
| 236 | loose.unlink() |
| 237 | |
| 238 | assert classify_object(repo_root, oid) is StorageTier.WARM |
| 239 | |
| 240 | |
| 241 | # ── Tier 4: classify_object returns None for unknown object ─────────────────── |
| 242 | |
| 243 | class TestClassifyObjectUnknown: |
| 244 | def test_returns_none_for_missing_object(self, tmp_path: Path) -> None: |
| 245 | from musehub.storage.tiers import classify_object |
| 246 | |
| 247 | repo_root = _repo_root(tmp_path) |
| 248 | assert classify_object(repo_root, _oid()) is None |
| 249 | |
| 250 | def test_returns_none_in_empty_repo(self, tmp_path: Path) -> None: |
| 251 | from musehub.storage.tiers import classify_object |
| 252 | |
| 253 | repo_root = _repo_root(tmp_path) |
| 254 | for _ in range(3): |
| 255 | assert classify_object(repo_root, _oid()) is None |
| 256 | |
| 257 | def test_returns_none_after_object_deleted_from_all_tiers(self, tmp_path: Path) -> None: |
| 258 | from musehub.storage.tiers import classify_object, prune_packed |
| 259 | from musehub.storage.pack import pack_loose_objects, gc_packs |
| 260 | |
| 261 | repo_root = _repo_root(tmp_path) |
| 262 | data = b"temporary" |
| 263 | oid = _oid(data) |
| 264 | _write_loose(repo_root, oid, data) |
| 265 | pack_loose_objects(repo_root) |
| 266 | prune_packed(repo_root) |
| 267 | |
| 268 | # Nuke the pack |
| 269 | for f in _pack_dir(repo_root).glob("*"): |
| 270 | f.unlink() |
| 271 | |
| 272 | assert classify_object(repo_root, oid) is None |
| 273 | |
| 274 | |
| 275 | # ── Tier 5: storage_stats returns accurate counts per tier ──────────────────── |
| 276 | |
| 277 | class TestStorageStats: |
| 278 | def test_all_loose_before_packing(self, tmp_path: Path) -> None: |
| 279 | from musehub.storage.tiers import storage_stats |
| 280 | |
| 281 | repo_root = _repo_root(tmp_path) |
| 282 | for i in range(4): |
| 283 | data = f"loose-{i}".encode() |
| 284 | _write_loose(repo_root, _oid(data), data) |
| 285 | |
| 286 | stats = storage_stats(repo_root) |
| 287 | assert stats.hot == 4 |
| 288 | assert stats.warm == 0 |
| 289 | |
| 290 | def test_warm_count_after_pack_and_prune(self, tmp_path: Path) -> None: |
| 291 | from musehub.storage.tiers import storage_stats, prune_packed |
| 292 | from musehub.storage.pack import pack_loose_objects |
| 293 | |
| 294 | repo_root = _repo_root(tmp_path) |
| 295 | for i in range(6): |
| 296 | data = f"to-warm-{i}".encode() |
| 297 | _write_loose(repo_root, _oid(data), data) |
| 298 | |
| 299 | pack_loose_objects(repo_root) |
| 300 | prune_packed(repo_root) |
| 301 | |
| 302 | stats = storage_stats(repo_root) |
| 303 | assert stats.hot == 0 |
| 304 | assert stats.warm == 6 |
| 305 | |
| 306 | def test_mixed_hot_and_warm(self, tmp_path: Path) -> None: |
| 307 | from musehub.storage.tiers import storage_stats, prune_packed |
| 308 | from musehub.storage.pack import pack_loose_objects |
| 309 | |
| 310 | repo_root = _repo_root(tmp_path) |
| 311 | |
| 312 | # 3 objects packed + pruned → warm |
| 313 | for i in range(3): |
| 314 | data = f"warm-obj-{i}".encode() |
| 315 | oid = _oid(data) |
| 316 | loose = _write_loose(repo_root, oid, data) |
| 317 | pack_loose_objects(repo_root) |
| 318 | prune_packed(repo_root) |
| 319 | |
| 320 | # 2 new objects loose → hot |
| 321 | for i in range(2): |
| 322 | data = f"hot-obj-{i}".encode() |
| 323 | _write_loose(repo_root, _oid(data), data) |
| 324 | |
| 325 | stats = storage_stats(repo_root) |
| 326 | assert stats.hot == 2 |
| 327 | assert stats.warm == 3 |
| 328 | |
| 329 | def test_empty_repo_all_zeros(self, tmp_path: Path) -> None: |
| 330 | from musehub.storage.tiers import storage_stats |
| 331 | |
| 332 | repo_root = _repo_root(tmp_path) |
| 333 | stats = storage_stats(repo_root) |
| 334 | assert stats.hot == 0 |
| 335 | assert stats.warm == 0 |
| 336 | |
| 337 | def test_total_equals_hot_plus_warm(self, tmp_path: Path) -> None: |
| 338 | from musehub.storage.tiers import storage_stats, prune_packed |
| 339 | from musehub.storage.pack import pack_loose_objects |
| 340 | |
| 341 | repo_root = _repo_root(tmp_path) |
| 342 | for i in range(5): |
| 343 | data = f"obj-{i}".encode() |
| 344 | oid = _oid(data) |
| 345 | loose = _write_loose(repo_root, oid, data) |
| 346 | pack_loose_objects(repo_root) |
| 347 | prune_packed(repo_root) |
| 348 | |
| 349 | for i in range(3): |
| 350 | data = f"new-obj-{i}".encode() |
| 351 | _write_loose(repo_root, _oid(data), data) |
| 352 | |
| 353 | stats = storage_stats(repo_root) |
| 354 | assert stats.total == stats.hot + stats.warm |
| 355 | |
| 356 | |
| 357 | # ── Tier 6: auto_pack packs + prunes at threshold ──────────────────────────── |
| 358 | |
| 359 | class TestAutoPack: |
| 360 | def test_packs_and_prunes_when_loose_meets_threshold(self, tmp_path: Path) -> None: |
| 361 | from musehub.storage.tiers import auto_pack, storage_stats |
| 362 | |
| 363 | repo_root = _repo_root(tmp_path) |
| 364 | n = 10 |
| 365 | for i in range(n): |
| 366 | data = f"auto-pack-{i}".encode() |
| 367 | _write_loose(repo_root, _oid(data), data) |
| 368 | |
| 369 | result = auto_pack(repo_root, min_loose=5) |
| 370 | |
| 371 | assert result.triggered is True |
| 372 | assert result.packed == n |
| 373 | stats = storage_stats(repo_root) |
| 374 | assert stats.hot == 0 |
| 375 | assert stats.warm == n |
| 376 | |
| 377 | def test_skips_when_loose_below_threshold(self, tmp_path: Path) -> None: |
| 378 | from musehub.storage.tiers import auto_pack |
| 379 | |
| 380 | repo_root = _repo_root(tmp_path) |
| 381 | for i in range(3): |
| 382 | data = f"few-{i}".encode() |
| 383 | _write_loose(repo_root, _oid(data), data) |
| 384 | |
| 385 | result = auto_pack(repo_root, min_loose=10) |
| 386 | |
| 387 | assert result.triggered is False |
| 388 | assert result.packed == 0 |
| 389 | assert _count_loose(repo_root) == 3 # unchanged |
| 390 | |
| 391 | def test_skips_when_no_loose_objects(self, tmp_path: Path) -> None: |
| 392 | from musehub.storage.tiers import auto_pack |
| 393 | |
| 394 | repo_root = _repo_root(tmp_path) |
| 395 | result = auto_pack(repo_root, min_loose=1) |
| 396 | |
| 397 | assert result.triggered is False |
| 398 | assert result.packed == 0 |
| 399 | |
| 400 | def test_exact_threshold_triggers(self, tmp_path: Path) -> None: |
| 401 | from musehub.storage.tiers import auto_pack |
| 402 | |
| 403 | repo_root = _repo_root(tmp_path) |
| 404 | n = 5 |
| 405 | for i in range(n): |
| 406 | data = f"exact-{i}".encode() |
| 407 | _write_loose(repo_root, _oid(data), data) |
| 408 | |
| 409 | result = auto_pack(repo_root, min_loose=n) |
| 410 | assert result.triggered is True |
| 411 | |
| 412 | |
| 413 | # ── Tier 7: full promotion workflow ────────────────────────────────────────── |
| 414 | |
| 415 | class TestFullPromotion: |
| 416 | @pytest.mark.asyncio |
| 417 | async def test_objects_readable_through_all_lifecycle_stages( |
| 418 | self, tmp_path: Path |
| 419 | ) -> None: |
| 420 | """Write → pack → prune → all objects still readable via backend.""" |
| 421 | from musehub.storage.backends import LocalBackend |
| 422 | from musehub.storage.pack import pack_loose_objects |
| 423 | from musehub.storage.tiers import prune_packed, classify_object, StorageTier |
| 424 | |
| 425 | repo_root = _repo_root(tmp_path) |
| 426 | backend = LocalBackend() |
| 427 | |
| 428 | # Stage 1: write 15 objects as loose (HOT) |
| 429 | objects: dict[str, bytes] = {} |
| 430 | for i in range(15): |
| 431 | content = f"lifecycle-{i}-{'x' * 40}".encode() |
| 432 | oid = _oid(content) |
| 433 | await backend.put(oid, content, repo_root=repo_root) |
| 434 | objects[oid] = content |
| 435 | assert classify_object(repo_root, oid) is StorageTier.HOT |
| 436 | |
| 437 | # Stage 2: pack — objects still loose (HOT), also in pack (WARM) |
| 438 | pack_loose_objects(repo_root) |
| 439 | for oid in objects: |
| 440 | assert classify_object(repo_root, oid) is StorageTier.HOT # loose wins |
| 441 | |
| 442 | # Stage 3: prune — objects move to WARM |
| 443 | prune_packed(repo_root) |
| 444 | for oid in objects: |
| 445 | assert classify_object(repo_root, oid) is StorageTier.WARM |
| 446 | |
| 447 | # Stage 4: all still readable through backend after prune |
| 448 | for oid, content in objects.items(): |
| 449 | result = await backend.get(oid, repo_root=repo_root) |
| 450 | assert result == content, f"object {oid[:20]}... lost after promotion" |
| 451 | |
| 452 | @pytest.mark.asyncio |
| 453 | async def test_new_objects_written_after_prune_are_hot( |
| 454 | self, tmp_path: Path |
| 455 | ) -> None: |
| 456 | """Objects written after a prune cycle start in the HOT tier.""" |
| 457 | from musehub.storage.backends import LocalBackend |
| 458 | from musehub.storage.pack import pack_loose_objects |
| 459 | from musehub.storage.tiers import prune_packed, classify_object, StorageTier |
| 460 | |
| 461 | repo_root = _repo_root(tmp_path) |
| 462 | backend = LocalBackend() |
| 463 | |
| 464 | # Full cycle on first batch |
| 465 | old_content = b"old-batch" |
| 466 | old_oid = _oid(old_content) |
| 467 | await backend.put(old_oid, old_content, repo_root=repo_root) |
| 468 | pack_loose_objects(repo_root) |
| 469 | prune_packed(repo_root) |
| 470 | |
| 471 | assert classify_object(repo_root, old_oid) is StorageTier.WARM |
| 472 | |
| 473 | # New object — starts HOT |
| 474 | new_content = b"fresh-after-prune" |
| 475 | new_oid = _oid(new_content) |
| 476 | await backend.put(new_oid, new_content, repo_root=repo_root) |
| 477 | |
| 478 | assert classify_object(repo_root, new_oid) is StorageTier.HOT |
| 479 | assert await backend.get(new_oid, repo_root=repo_root) == new_content |
| 480 | assert await backend.get(old_oid, repo_root=repo_root) == old_content |
File History
1 commit
sha256:a34090cc4a394a78bd72cbbe34b08cc59525141e19135b6c0ab154f10611b9ef
debug(push/stream): instrument O-frame decode path with INF…
Sonnet 4.6
patch
121 days ago