test_mpack_blob_extraction.py
python
sha256:4c1d427f8ff7e2e3557a355edc72231e039930d280d65d299999a1cc898649d2
test(prebuild): RED — bounded ranged-read blob extraction w…
Sonnet 5.5
4 days ago
| 1 | """TDD — prebuild blob extraction by bounded byte-range reads (RED/GREEN, this ticket). |
| 2 | |
| 3 | ``wire_fetch_mpack`` used to download and parse every stored mpack that held a wanted |
| 4 | blob, however few blobs it wanted (peak 1754 MB on staging for 33.5 MB of tip blobs). |
| 5 | ``fetch_mpack_blobs`` reads each indexed blob with one bounded ``get_range`` and verifies |
| 6 | its content-addressed ID, and falls back to the whole mpack only for blobs whose range is |
| 7 | absent or unusable. |
| 8 | |
| 9 | Database-free (see ``_NO_DB_PREFIXES`` in conftest): a pure function over a storage backend. |
| 10 | |
| 11 | Verified format facts these tests pin |
| 12 | ------------------------------------- |
| 13 | * ``compute_object_byte_offsets`` addresses the *raw, uncompressed* content of each blob |
| 14 | inside the stored MUSE wire mpack; the stored mpack has no outer compression, so a ranged |
| 15 | read returns the content itself and needs no decoding. |
| 16 | * Object IDs are ``blob_id(content)`` (git-style ``sha256("blob <n>\\0" + data)``). |
| 17 | |
| 18 | Coverage matrix |
| 19 | --------------- |
| 20 | RR-1 indexed blobs come from get_range only — never get_mpack — and match the originals. |
| 21 | RR-2 ranged extraction and whole-mpack extraction return identical id → content sets. |
| 22 | RR-3 several objects sharing one stored mpack: only the requested ranges are transferred. |
| 23 | RR-4 legacy compression: msgpack/zstd-encoded entries; zstd bytes in a range; a raw blob |
| 24 | that merely starts with the zstd magic is returned as stored. |
| 25 | RR-5 NULL / invalid offsets fall back to the whole mpack (once per mpack), never omit. |
| 26 | RR-6 missing, truncated, corrupted or mis-addressed ranges never yield wrong content. |
| 27 | RR-7 concurrency is bounded (range and whole-mpack) and nothing is left running. |
| 28 | RR-8 the empty blob needs no I/O. |
| 29 | """ |
| 30 | from __future__ import annotations |
| 31 | |
| 32 | import asyncio |
| 33 | |
| 34 | import msgpack |
| 35 | import pytest |
| 36 | import zstandard |
| 37 | |
| 38 | from muse.core.mpack import build_wire_mpack |
| 39 | from muse.core.types import blob_id |
| 40 | from musehub.services.musehub_mpack_blobs import ( |
| 41 | RANGE_CONCURRENCY, |
| 42 | WHOLE_MPACK_CONCURRENCY, |
| 43 | BlobLocation, |
| 44 | fetch_mpack_blobs, |
| 45 | ) |
| 46 | from musehub.services.musehub_wire_push import compute_object_byte_offsets |
| 47 | from musehub.storage.backends import MemoryBackend |
| 48 | |
| 49 | _ZSTD_MAGIC = b"\x28\xb5\x2f\xfd" |
| 50 | _EMPTY_OID = blob_id(b"") |
| 51 | |
| 52 | |
| 53 | class _SpyBackend(MemoryBackend): |
| 54 | """MemoryBackend that records every read.""" |
| 55 | |
| 56 | def __init__(self) -> None: |
| 57 | super().__init__() |
| 58 | self.range_calls: list[tuple[str, int, int]] = [] |
| 59 | self.whole_calls: list[str] = [] |
| 60 | self.range_bytes = 0 |
| 61 | self.range_delay = 0.0 |
| 62 | self.whole_delay = 0.0 |
| 63 | self.in_flight_ranges = 0 |
| 64 | self.max_in_flight_ranges = 0 |
| 65 | self.in_flight_wholes = 0 |
| 66 | self.max_in_flight_wholes = 0 |
| 67 | |
| 68 | async def get_range(self, mpack_key: str, byte_offset: int, byte_length: int) -> bytes | None: |
| 69 | self.range_calls.append((mpack_key, byte_offset, byte_length)) |
| 70 | self.in_flight_ranges += 1 |
| 71 | self.max_in_flight_ranges = max(self.max_in_flight_ranges, self.in_flight_ranges) |
| 72 | try: |
| 73 | if self.range_delay: |
| 74 | await asyncio.sleep(self.range_delay) |
| 75 | data = await super().get_range(mpack_key, byte_offset, byte_length) |
| 76 | self.range_bytes += len(data or b"") |
| 77 | return data |
| 78 | finally: |
| 79 | self.in_flight_ranges -= 1 |
| 80 | |
| 81 | async def get_mpack(self, mpack_key: str) -> bytes | None: |
| 82 | self.whole_calls.append(mpack_key) |
| 83 | self.in_flight_wholes += 1 |
| 84 | self.max_in_flight_wholes = max(self.max_in_flight_wholes, self.in_flight_wholes) |
| 85 | try: |
| 86 | if self.whole_delay: |
| 87 | await asyncio.sleep(self.whole_delay) |
| 88 | return await super().get_mpack(mpack_key) |
| 89 | finally: |
| 90 | self.in_flight_wholes -= 1 |
| 91 | |
| 92 | |
| 93 | def _blob(content: bytes) -> tuple[str, bytes]: |
| 94 | return blob_id(content), content |
| 95 | |
| 96 | |
| 97 | async def _store(backend: MemoryBackend, blobs: list[tuple[str, bytes]]) -> tuple[str, dict[str, BlobLocation]]: |
| 98 | """Store a wire mpack of *blobs*; return (mpack_id, {oid: indexed location}).""" |
| 99 | wire = build_wire_mpack({ |
| 100 | "commits": [], "snapshots": [], "tags": [], |
| 101 | "blobs": [{"object_id": oid, "content": data} for oid, data in blobs], |
| 102 | }) |
| 103 | mpack_id = blob_id(wire) |
| 104 | await backend.put_mpack(mpack_id, wire) |
| 105 | offsets = compute_object_byte_offsets(wire) |
| 106 | return mpack_id, {oid: BlobLocation(oid, *offsets[oid]) for oid, _ in blobs} |
| 107 | |
| 108 | |
| 109 | def _sizeable(n: int, tag: str) -> bytes: |
| 110 | return (f"{tag}:".encode() + bytes(range(256))) * n |
| 111 | |
| 112 | |
| 113 | # ── RR-1 ───────────────────────────────────────────────────────────────────── |
| 114 | |
| 115 | |
| 116 | async def test_indexed_blobs_are_read_by_range_only() -> None: |
| 117 | backend = _SpyBackend() |
| 118 | blobs = [_blob(_sizeable(40, f"f{i}")) for i in range(5)] |
| 119 | mpack_id, locs = await _store(backend, blobs) |
| 120 | |
| 121 | hits, misses = await fetch_mpack_blobs(backend, {mpack_id: list(locs.values())}) |
| 122 | |
| 123 | assert misses == [] |
| 124 | assert hits == dict(blobs) |
| 125 | assert backend.whole_calls == [], "an indexed blob must never trigger a whole-mpack download" |
| 126 | assert len(backend.range_calls) == len(blobs) |
| 127 | |
| 128 | |
| 129 | # ── RR-2 ───────────────────────────────────────────────────────────────────── |
| 130 | |
| 131 | |
| 132 | async def test_ranged_and_whole_mpack_extraction_are_identical() -> None: |
| 133 | backend = _SpyBackend() |
| 134 | blobs = [_blob(_sizeable(7 + i, f"g{i}")) for i in range(8)] + [_blob(b"tiny"), _blob(b"\x00\x01\x02")] |
| 135 | mpack_id, locs = await _store(backend, blobs) |
| 136 | |
| 137 | ranged, ranged_miss = await fetch_mpack_blobs(backend, {mpack_id: list(locs.values())}) |
| 138 | unindexed = [BlobLocation(oid, None, None) for oid in locs] |
| 139 | whole, whole_miss = await fetch_mpack_blobs(backend, {mpack_id: unindexed}) |
| 140 | |
| 141 | assert ranged_miss == whole_miss == [] |
| 142 | assert ranged == whole == dict(blobs) |
| 143 | assert all(blob_id(c) == oid for oid, c in ranged.items()) |
| 144 | |
| 145 | |
| 146 | # ── RR-3 ───────────────────────────────────────────────────────────────────── |
| 147 | |
| 148 | |
| 149 | async def test_shared_mpack_transfers_only_the_requested_ranges() -> None: |
| 150 | backend = _SpyBackend() |
| 151 | big = [_blob(_sizeable(400, f"big{i}")) for i in range(6)] |
| 152 | wanted = _blob(_sizeable(3, "wanted")) |
| 153 | mpack_a, locs_a = await _store(backend, big + [wanted]) |
| 154 | other = _blob(_sizeable(3, "other")) |
| 155 | mpack_b, locs_b = await _store(backend, [other]) |
| 156 | |
| 157 | hits, misses = await fetch_mpack_blobs( |
| 158 | backend, |
| 159 | {mpack_a: [locs_a[wanted[0]]], mpack_b: [locs_b[other[0]]]}, |
| 160 | ) |
| 161 | |
| 162 | assert misses == [] |
| 163 | assert hits == dict([wanted, other]) |
| 164 | assert backend.range_bytes == len(wanted[1]) + len(other[1]) |
| 165 | assert backend.range_bytes < len(backend._mpacks[mpack_a]) // 10 |
| 166 | assert backend.whole_calls == [] |
| 167 | |
| 168 | |
| 169 | # ── RR-4 ───────────────────────────────────────────────────────────────────── |
| 170 | |
| 171 | |
| 172 | async def test_legacy_msgpack_mpack_with_zstd_encoded_entries() -> None: |
| 173 | backend = _SpyBackend() |
| 174 | content = _sizeable(50, "legacy") |
| 175 | oid = blob_id(content) |
| 176 | legacy = msgpack.packb( |
| 177 | {"blobs": [{"object_id": oid, "content": zstandard.ZstdCompressor().compress(content), "encoding": "zstd"}]}, |
| 178 | use_bin_type=True, |
| 179 | ) |
| 180 | await backend.put_mpack("legacy-mpack", legacy) |
| 181 | |
| 182 | hits, misses = await fetch_mpack_blobs(backend, {"legacy-mpack": [BlobLocation(oid, None, None)]}) |
| 183 | |
| 184 | assert misses == [] |
| 185 | assert hits == {oid: content} |
| 186 | |
| 187 | |
| 188 | async def test_zstd_bytes_in_a_stored_range_are_decompressed_and_verified() -> None: |
| 189 | backend = _SpyBackend() |
| 190 | content = _sizeable(60, "zst") |
| 191 | oid = blob_id(content) |
| 192 | compressed = zstandard.ZstdCompressor().compress(content) |
| 193 | # An older writer stored compressed bytes in the pack without an encoding marker. |
| 194 | wire = build_wire_mpack({ |
| 195 | "commits": [], "snapshots": [], "tags": [], |
| 196 | "blobs": [{"object_id": oid, "content": compressed}], |
| 197 | }) |
| 198 | mpack_id = blob_id(wire) |
| 199 | await backend.put_mpack(mpack_id, wire) |
| 200 | off, length = compute_object_byte_offsets(wire)[oid] |
| 201 | |
| 202 | hits, misses = await fetch_mpack_blobs(backend, {mpack_id: [BlobLocation(oid, off, length)]}) |
| 203 | |
| 204 | assert misses == [] |
| 205 | assert hits == {oid: content} |
| 206 | assert backend.whole_calls == [] |
| 207 | assert backend.range_bytes == len(compressed) |
| 208 | |
| 209 | |
| 210 | async def test_raw_blob_that_starts_with_the_zstd_magic_is_returned_as_stored() -> None: |
| 211 | backend = _SpyBackend() |
| 212 | content = _ZSTD_MAGIC + b"this is a .zst-looking file that is not valid zstd" * 5 |
| 213 | oid = blob_id(content) |
| 214 | mpack_id, locs = await _store(backend, [(oid, content)]) |
| 215 | |
| 216 | ranged, ranged_miss = await fetch_mpack_blobs(backend, {mpack_id: [locs[oid]]}) |
| 217 | whole, whole_miss = await fetch_mpack_blobs(backend, {mpack_id: [BlobLocation(oid, None, None)]}) |
| 218 | |
| 219 | assert ranged_miss == whole_miss == [] |
| 220 | assert ranged == whole == {oid: content} |
| 221 | |
| 222 | |
| 223 | # ── RR-5 ───────────────────────────────────────────────────────────────────── |
| 224 | |
| 225 | |
| 226 | async def test_null_offsets_fall_back_to_one_whole_mpack_read_per_mpack() -> None: |
| 227 | backend = _SpyBackend() |
| 228 | blobs = [_blob(_sizeable(5, f"n{i}")) for i in range(6)] |
| 229 | mpack_id, locs = await _store(backend, blobs) |
| 230 | |
| 231 | hits, misses = await fetch_mpack_blobs( |
| 232 | backend, {mpack_id: [BlobLocation(oid, None, None) for oid in locs]}, |
| 233 | ) |
| 234 | |
| 235 | assert misses == [] |
| 236 | assert hits == dict(blobs) |
| 237 | assert backend.range_calls == [] |
| 238 | assert backend.whole_calls == [mpack_id] |
| 239 | |
| 240 | |
| 241 | @pytest.mark.parametrize("make", [ |
| 242 | pytest.param(lambda loc, size: BlobLocation(loc.object_id, -1, loc.byte_length), id="negative-offset"), |
| 243 | pytest.param(lambda loc, size: BlobLocation(loc.object_id, loc.byte_offset, -5), id="negative-length"), |
| 244 | pytest.param(lambda loc, size: BlobLocation(loc.object_id, loc.byte_offset, 0), id="zero-length-for-nonempty"), |
| 245 | pytest.param(lambda loc, size: BlobLocation(loc.object_id, size + 10, loc.byte_length), id="offset-past-end"), |
| 246 | pytest.param(lambda loc, size: BlobLocation(loc.object_id, loc.byte_offset, None), id="half-null"), |
| 247 | ]) |
| 248 | async def test_invalid_offsets_fall_back_and_never_omit(make) -> None: # type: ignore[no-untyped-def] |
| 249 | backend = _SpyBackend() |
| 250 | blobs = [_blob(_sizeable(5, f"v{i}")) for i in range(3)] |
| 251 | mpack_id, locs = await _store(backend, blobs) |
| 252 | size = len(backend._mpacks[mpack_id]) |
| 253 | |
| 254 | hits, misses = await fetch_mpack_blobs( |
| 255 | backend, {mpack_id: [make(loc, size) for loc in locs.values()]}, |
| 256 | ) |
| 257 | |
| 258 | assert misses == [] |
| 259 | assert hits == dict(blobs) |
| 260 | |
| 261 | |
| 262 | # ── RR-6 ───────────────────────────────────────────────────────────────────── |
| 263 | |
| 264 | |
| 265 | async def test_missing_mpack_reports_a_miss_not_an_exception() -> None: |
| 266 | backend = _SpyBackend() |
| 267 | oid = blob_id(b"gone") |
| 268 | |
| 269 | hits, misses = await fetch_mpack_blobs(backend, {"no-such-mpack": [BlobLocation(oid, 100, 4)]}) |
| 270 | |
| 271 | assert hits == {} |
| 272 | assert misses == [oid] |
| 273 | |
| 274 | |
| 275 | async def test_truncated_range_falls_back_to_the_whole_mpack() -> None: |
| 276 | class Truncating(_SpyBackend): |
| 277 | async def get_range(self, mpack_key: str, byte_offset: int, byte_length: int) -> bytes | None: |
| 278 | data = await super().get_range(mpack_key, byte_offset, byte_length) |
| 279 | return None if data is None else data[: len(data) // 2] |
| 280 | |
| 281 | backend = Truncating() |
| 282 | blobs = [_blob(_sizeable(9, f"t{i}")) for i in range(3)] |
| 283 | mpack_id, locs = await _store(backend, blobs) |
| 284 | |
| 285 | hits, misses = await fetch_mpack_blobs(backend, {mpack_id: list(locs.values())}) |
| 286 | |
| 287 | assert misses == [] |
| 288 | assert hits == dict(blobs) |
| 289 | assert backend.whole_calls == [mpack_id] |
| 290 | |
| 291 | |
| 292 | async def test_corrupted_range_is_never_returned() -> None: |
| 293 | class Flipping(_SpyBackend): |
| 294 | async def get_range(self, mpack_key: str, byte_offset: int, byte_length: int) -> bytes | None: |
| 295 | data = await super().get_range(mpack_key, byte_offset, byte_length) |
| 296 | if data is None: |
| 297 | return None |
| 298 | return bytes([data[0] ^ 0xFF]) + data[1:] |
| 299 | |
| 300 | backend = Flipping() |
| 301 | blobs = [_blob(_sizeable(9, f"c{i}")) for i in range(3)] |
| 302 | mpack_id, locs = await _store(backend, blobs) |
| 303 | |
| 304 | hits, misses = await fetch_mpack_blobs(backend, {mpack_id: list(locs.values())}) |
| 305 | |
| 306 | assert misses == [] |
| 307 | assert hits == dict(blobs), "ranges are corrupt, the stored mpack is intact: the fallback must serve the truth" |
| 308 | |
| 309 | |
| 310 | async def test_corrupted_stored_mpack_yields_misses_never_wrong_bytes() -> None: |
| 311 | backend = _SpyBackend() |
| 312 | blobs = [_blob(_sizeable(9, f"d{i}")) for i in range(3)] |
| 313 | mpack_id, locs = await _store(backend, blobs) |
| 314 | stored = bytearray(backend._mpacks[mpack_id]) |
| 315 | first = locs[blobs[0][0]] |
| 316 | stored[first.byte_offset + 3] ^= 0xFF # flip one byte inside the first blob |
| 317 | backend._mpacks[mpack_id] = bytes(stored) |
| 318 | |
| 319 | hits, misses = await fetch_mpack_blobs(backend, {mpack_id: list(locs.values())}) |
| 320 | |
| 321 | assert blobs[0][0] not in hits, "corrupt content must not be returned" |
| 322 | assert blobs[0][0] in misses |
| 323 | assert all(blob_id(c) == oid for oid, c in hits.items()) |
| 324 | assert sorted(set(hits) | set(misses)) == sorted(oid for oid, _ in blobs) |
| 325 | |
| 326 | |
| 327 | async def test_range_addressing_another_object_is_rejected() -> None: |
| 328 | backend = _SpyBackend() |
| 329 | a, b = _blob(_sizeable(9, "a")), _blob(_sizeable(9, "b")) |
| 330 | mpack_id, locs = await _store(backend, [a, b]) |
| 331 | # b's location filed under a's id: same length, wrong bytes. |
| 332 | swapped = BlobLocation(a[0], locs[b[0]].byte_offset, locs[b[0]].byte_length) |
| 333 | |
| 334 | hits, misses = await fetch_mpack_blobs(backend, {mpack_id: [swapped]}) |
| 335 | |
| 336 | assert hits == {a[0]: a[1]} |
| 337 | assert misses == [] |
| 338 | assert backend.whole_calls == [mpack_id] |
| 339 | |
| 340 | |
| 341 | async def test_a_backend_error_on_one_range_does_not_lose_other_blobs() -> None: |
| 342 | class Exploding(_SpyBackend): |
| 343 | async def get_range(self, mpack_key: str, byte_offset: int, byte_length: int) -> bytes | None: |
| 344 | if byte_length % 2 == 0: |
| 345 | raise OSError("boom") |
| 346 | return await super().get_range(mpack_key, byte_offset, byte_length) |
| 347 | |
| 348 | backend = Exploding() |
| 349 | blobs = [_blob(b"x" * n) for n in (11, 12, 13, 14, 15)] |
| 350 | mpack_id, locs = await _store(backend, blobs) |
| 351 | |
| 352 | hits, misses = await fetch_mpack_blobs(backend, {mpack_id: list(locs.values())}) |
| 353 | |
| 354 | assert misses == [] |
| 355 | assert hits == dict(blobs) |
| 356 | |
| 357 | |
| 358 | # ── RR-7 ───────────────────────────────────────────────────────────────────── |
| 359 | |
| 360 | |
| 361 | async def test_range_reads_are_bounded() -> None: |
| 362 | backend = _SpyBackend() |
| 363 | backend.range_delay = 0.005 |
| 364 | blobs = [_blob(f"obj-{i}".encode() * 20) for i in range(RANGE_CONCURRENCY * 6)] |
| 365 | mpack_id, locs = await _store(backend, blobs) |
| 366 | |
| 367 | hits, misses = await fetch_mpack_blobs(backend, {mpack_id: list(locs.values())}) |
| 368 | |
| 369 | assert misses == [] |
| 370 | assert hits == dict(blobs) |
| 371 | assert 1 < backend.max_in_flight_ranges <= RANGE_CONCURRENCY |
| 372 | |
| 373 | |
| 374 | async def test_whole_mpack_fallback_is_bounded() -> None: |
| 375 | backend = _SpyBackend() |
| 376 | backend.whole_delay = 0.01 |
| 377 | wanted: dict[str, list[BlobLocation]] = {} |
| 378 | expected: dict[str, bytes] = {} |
| 379 | for i in range(WHOLE_MPACK_CONCURRENCY * 4): |
| 380 | blob = _blob(_sizeable(3, f"m{i}")) |
| 381 | mpack_id, _ = await _store(backend, [blob]) |
| 382 | wanted[mpack_id] = [BlobLocation(blob[0], None, None)] |
| 383 | expected[blob[0]] = blob[1] |
| 384 | |
| 385 | hits, misses = await fetch_mpack_blobs(backend, wanted) |
| 386 | |
| 387 | assert misses == [] |
| 388 | assert hits == expected |
| 389 | assert backend.max_in_flight_wholes <= WHOLE_MPACK_CONCURRENCY |
| 390 | |
| 391 | |
| 392 | async def test_cancellation_leaves_no_tasks_and_no_stuck_slots() -> None: |
| 393 | backend = _SpyBackend() |
| 394 | backend.range_delay = 0.2 |
| 395 | blobs = [_blob(f"cancel-{i}".encode() * 10) for i in range(RANGE_CONCURRENCY * 3)] |
| 396 | mpack_id, locs = await _store(backend, blobs) |
| 397 | before = asyncio.all_tasks() |
| 398 | |
| 399 | task = asyncio.create_task(fetch_mpack_blobs(backend, {mpack_id: list(locs.values())})) |
| 400 | await asyncio.sleep(0.05) |
| 401 | task.cancel() |
| 402 | with pytest.raises(asyncio.CancelledError): |
| 403 | await task |
| 404 | await asyncio.sleep(0) |
| 405 | |
| 406 | assert asyncio.all_tasks() - before - {asyncio.current_task()} == set() |
| 407 | assert backend.in_flight_ranges == 0 |
| 408 | |
| 409 | backend.range_delay = 0.0 |
| 410 | hits, misses = await fetch_mpack_blobs(backend, {mpack_id: list(locs.values())}) |
| 411 | assert misses == [] and hits == dict(blobs) |
| 412 | |
| 413 | |
| 414 | # ── RR-8 ───────────────────────────────────────────────────────────────────── |
| 415 | |
| 416 | |
| 417 | async def test_empty_blob_needs_no_io() -> None: |
| 418 | backend = _SpyBackend() |
| 419 | mpack_id, locs = await _store(backend, [_blob(b"keep")]) |
| 420 | empty_off = locs[blob_id(b"keep")].byte_offset |
| 421 | |
| 422 | hits, misses = await fetch_mpack_blobs(backend, {mpack_id: [BlobLocation(_EMPTY_OID, empty_off, 0)]}) |
| 423 | |
| 424 | assert misses == [] |
| 425 | assert hits == {_EMPTY_OID: b""} |
| 426 | assert backend.range_calls == [] and backend.whole_calls == [] |
File History
1 commit
sha256:4c1d427f8ff7e2e3557a355edc72231e039930d280d65d299999a1cc898649d2
test(prebuild): RED — bounded ranged-read blob extraction w…
Sonnet 5.5
4 days ago