gabriel / musehub public
test_mpack_blob_extraction.py python
426 lines 17.3 KB
Raw
sha256:4c1d427f8ff7e2e3557a355edc72231e039930d280d65d299999a1cc898649d2 test(prebuild): RED — bounded ranged-read blob extraction w… Sonnet 5.5 4 days ago
1 """TDD — prebuild blob extraction by bounded byte-range reads (RED/GREEN, this ticket).
2
3 ``wire_fetch_mpack`` used to download and parse every stored mpack that held a wanted
4 blob, however few blobs it wanted (peak 1754 MB on staging for 33.5 MB of tip blobs).
5 ``fetch_mpack_blobs`` reads each indexed blob with one bounded ``get_range`` and verifies
6 its content-addressed ID, and falls back to the whole mpack only for blobs whose range is
7 absent or unusable.
8
9 Database-free (see ``_NO_DB_PREFIXES`` in conftest): a pure function over a storage backend.
10
11 Verified format facts these tests pin
12 -------------------------------------
13 * ``compute_object_byte_offsets`` addresses the *raw, uncompressed* content of each blob
14 inside the stored MUSE wire mpack; the stored mpack has no outer compression, so a ranged
15 read returns the content itself and needs no decoding.
16 * Object IDs are ``blob_id(content)`` (git-style ``sha256("blob <n>\\0" + data)``).
17
18 Coverage matrix
19 ---------------
20 RR-1 indexed blobs come from get_range only — never get_mpack — and match the originals.
21 RR-2 ranged extraction and whole-mpack extraction return identical id → content sets.
22 RR-3 several objects sharing one stored mpack: only the requested ranges are transferred.
23 RR-4 legacy compression: msgpack/zstd-encoded entries; zstd bytes in a range; a raw blob
24 that merely starts with the zstd magic is returned as stored.
25 RR-5 NULL / invalid offsets fall back to the whole mpack (once per mpack), never omit.
26 RR-6 missing, truncated, corrupted or mis-addressed ranges never yield wrong content.
27 RR-7 concurrency is bounded (range and whole-mpack) and nothing is left running.
28 RR-8 the empty blob needs no I/O.
29 """
30 from __future__ import annotations
31
32 import asyncio
33
34 import msgpack
35 import pytest
36 import zstandard
37
38 from muse.core.mpack import build_wire_mpack
39 from muse.core.types import blob_id
40 from musehub.services.musehub_mpack_blobs import (
41 RANGE_CONCURRENCY,
42 WHOLE_MPACK_CONCURRENCY,
43 BlobLocation,
44 fetch_mpack_blobs,
45 )
46 from musehub.services.musehub_wire_push import compute_object_byte_offsets
47 from musehub.storage.backends import MemoryBackend
48
49 _ZSTD_MAGIC = b"\x28\xb5\x2f\xfd"
50 _EMPTY_OID = blob_id(b"")
51
52
53 class _SpyBackend(MemoryBackend):
54 """MemoryBackend that records every read."""
55
56 def __init__(self) -> None:
57 super().__init__()
58 self.range_calls: list[tuple[str, int, int]] = []
59 self.whole_calls: list[str] = []
60 self.range_bytes = 0
61 self.range_delay = 0.0
62 self.whole_delay = 0.0
63 self.in_flight_ranges = 0
64 self.max_in_flight_ranges = 0
65 self.in_flight_wholes = 0
66 self.max_in_flight_wholes = 0
67
68 async def get_range(self, mpack_key: str, byte_offset: int, byte_length: int) -> bytes | None:
69 self.range_calls.append((mpack_key, byte_offset, byte_length))
70 self.in_flight_ranges += 1
71 self.max_in_flight_ranges = max(self.max_in_flight_ranges, self.in_flight_ranges)
72 try:
73 if self.range_delay:
74 await asyncio.sleep(self.range_delay)
75 data = await super().get_range(mpack_key, byte_offset, byte_length)
76 self.range_bytes += len(data or b"")
77 return data
78 finally:
79 self.in_flight_ranges -= 1
80
81 async def get_mpack(self, mpack_key: str) -> bytes | None:
82 self.whole_calls.append(mpack_key)
83 self.in_flight_wholes += 1
84 self.max_in_flight_wholes = max(self.max_in_flight_wholes, self.in_flight_wholes)
85 try:
86 if self.whole_delay:
87 await asyncio.sleep(self.whole_delay)
88 return await super().get_mpack(mpack_key)
89 finally:
90 self.in_flight_wholes -= 1
91
92
93 def _blob(content: bytes) -> tuple[str, bytes]:
94 return blob_id(content), content
95
96
97 async def _store(backend: MemoryBackend, blobs: list[tuple[str, bytes]]) -> tuple[str, dict[str, BlobLocation]]:
98 """Store a wire mpack of *blobs*; return (mpack_id, {oid: indexed location})."""
99 wire = build_wire_mpack({
100 "commits": [], "snapshots": [], "tags": [],
101 "blobs": [{"object_id": oid, "content": data} for oid, data in blobs],
102 })
103 mpack_id = blob_id(wire)
104 await backend.put_mpack(mpack_id, wire)
105 offsets = compute_object_byte_offsets(wire)
106 return mpack_id, {oid: BlobLocation(oid, *offsets[oid]) for oid, _ in blobs}
107
108
109 def _sizeable(n: int, tag: str) -> bytes:
110 return (f"{tag}:".encode() + bytes(range(256))) * n
111
112
113 # ── RR-1 ─────────────────────────────────────────────────────────────────────
114
115
116 async def test_indexed_blobs_are_read_by_range_only() -> None:
117 backend = _SpyBackend()
118 blobs = [_blob(_sizeable(40, f"f{i}")) for i in range(5)]
119 mpack_id, locs = await _store(backend, blobs)
120
121 hits, misses = await fetch_mpack_blobs(backend, {mpack_id: list(locs.values())})
122
123 assert misses == []
124 assert hits == dict(blobs)
125 assert backend.whole_calls == [], "an indexed blob must never trigger a whole-mpack download"
126 assert len(backend.range_calls) == len(blobs)
127
128
129 # ── RR-2 ─────────────────────────────────────────────────────────────────────
130
131
132 async def test_ranged_and_whole_mpack_extraction_are_identical() -> None:
133 backend = _SpyBackend()
134 blobs = [_blob(_sizeable(7 + i, f"g{i}")) for i in range(8)] + [_blob(b"tiny"), _blob(b"\x00\x01\x02")]
135 mpack_id, locs = await _store(backend, blobs)
136
137 ranged, ranged_miss = await fetch_mpack_blobs(backend, {mpack_id: list(locs.values())})
138 unindexed = [BlobLocation(oid, None, None) for oid in locs]
139 whole, whole_miss = await fetch_mpack_blobs(backend, {mpack_id: unindexed})
140
141 assert ranged_miss == whole_miss == []
142 assert ranged == whole == dict(blobs)
143 assert all(blob_id(c) == oid for oid, c in ranged.items())
144
145
146 # ── RR-3 ─────────────────────────────────────────────────────────────────────
147
148
149 async def test_shared_mpack_transfers_only_the_requested_ranges() -> None:
150 backend = _SpyBackend()
151 big = [_blob(_sizeable(400, f"big{i}")) for i in range(6)]
152 wanted = _blob(_sizeable(3, "wanted"))
153 mpack_a, locs_a = await _store(backend, big + [wanted])
154 other = _blob(_sizeable(3, "other"))
155 mpack_b, locs_b = await _store(backend, [other])
156
157 hits, misses = await fetch_mpack_blobs(
158 backend,
159 {mpack_a: [locs_a[wanted[0]]], mpack_b: [locs_b[other[0]]]},
160 )
161
162 assert misses == []
163 assert hits == dict([wanted, other])
164 assert backend.range_bytes == len(wanted[1]) + len(other[1])
165 assert backend.range_bytes < len(backend._mpacks[mpack_a]) // 10
166 assert backend.whole_calls == []
167
168
169 # ── RR-4 ─────────────────────────────────────────────────────────────────────
170
171
172 async def test_legacy_msgpack_mpack_with_zstd_encoded_entries() -> None:
173 backend = _SpyBackend()
174 content = _sizeable(50, "legacy")
175 oid = blob_id(content)
176 legacy = msgpack.packb(
177 {"blobs": [{"object_id": oid, "content": zstandard.ZstdCompressor().compress(content), "encoding": "zstd"}]},
178 use_bin_type=True,
179 )
180 await backend.put_mpack("legacy-mpack", legacy)
181
182 hits, misses = await fetch_mpack_blobs(backend, {"legacy-mpack": [BlobLocation(oid, None, None)]})
183
184 assert misses == []
185 assert hits == {oid: content}
186
187
188 async def test_zstd_bytes_in_a_stored_range_are_decompressed_and_verified() -> None:
189 backend = _SpyBackend()
190 content = _sizeable(60, "zst")
191 oid = blob_id(content)
192 compressed = zstandard.ZstdCompressor().compress(content)
193 # An older writer stored compressed bytes in the pack without an encoding marker.
194 wire = build_wire_mpack({
195 "commits": [], "snapshots": [], "tags": [],
196 "blobs": [{"object_id": oid, "content": compressed}],
197 })
198 mpack_id = blob_id(wire)
199 await backend.put_mpack(mpack_id, wire)
200 off, length = compute_object_byte_offsets(wire)[oid]
201
202 hits, misses = await fetch_mpack_blobs(backend, {mpack_id: [BlobLocation(oid, off, length)]})
203
204 assert misses == []
205 assert hits == {oid: content}
206 assert backend.whole_calls == []
207 assert backend.range_bytes == len(compressed)
208
209
210 async def test_raw_blob_that_starts_with_the_zstd_magic_is_returned_as_stored() -> None:
211 backend = _SpyBackend()
212 content = _ZSTD_MAGIC + b"this is a .zst-looking file that is not valid zstd" * 5
213 oid = blob_id(content)
214 mpack_id, locs = await _store(backend, [(oid, content)])
215
216 ranged, ranged_miss = await fetch_mpack_blobs(backend, {mpack_id: [locs[oid]]})
217 whole, whole_miss = await fetch_mpack_blobs(backend, {mpack_id: [BlobLocation(oid, None, None)]})
218
219 assert ranged_miss == whole_miss == []
220 assert ranged == whole == {oid: content}
221
222
223 # ── RR-5 ─────────────────────────────────────────────────────────────────────
224
225
226 async def test_null_offsets_fall_back_to_one_whole_mpack_read_per_mpack() -> None:
227 backend = _SpyBackend()
228 blobs = [_blob(_sizeable(5, f"n{i}")) for i in range(6)]
229 mpack_id, locs = await _store(backend, blobs)
230
231 hits, misses = await fetch_mpack_blobs(
232 backend, {mpack_id: [BlobLocation(oid, None, None) for oid in locs]},
233 )
234
235 assert misses == []
236 assert hits == dict(blobs)
237 assert backend.range_calls == []
238 assert backend.whole_calls == [mpack_id]
239
240
241 @pytest.mark.parametrize("make", [
242 pytest.param(lambda loc, size: BlobLocation(loc.object_id, -1, loc.byte_length), id="negative-offset"),
243 pytest.param(lambda loc, size: BlobLocation(loc.object_id, loc.byte_offset, -5), id="negative-length"),
244 pytest.param(lambda loc, size: BlobLocation(loc.object_id, loc.byte_offset, 0), id="zero-length-for-nonempty"),
245 pytest.param(lambda loc, size: BlobLocation(loc.object_id, size + 10, loc.byte_length), id="offset-past-end"),
246 pytest.param(lambda loc, size: BlobLocation(loc.object_id, loc.byte_offset, None), id="half-null"),
247 ])
248 async def test_invalid_offsets_fall_back_and_never_omit(make) -> None: # type: ignore[no-untyped-def]
249 backend = _SpyBackend()
250 blobs = [_blob(_sizeable(5, f"v{i}")) for i in range(3)]
251 mpack_id, locs = await _store(backend, blobs)
252 size = len(backend._mpacks[mpack_id])
253
254 hits, misses = await fetch_mpack_blobs(
255 backend, {mpack_id: [make(loc, size) for loc in locs.values()]},
256 )
257
258 assert misses == []
259 assert hits == dict(blobs)
260
261
262 # ── RR-6 ─────────────────────────────────────────────────────────────────────
263
264
265 async def test_missing_mpack_reports_a_miss_not_an_exception() -> None:
266 backend = _SpyBackend()
267 oid = blob_id(b"gone")
268
269 hits, misses = await fetch_mpack_blobs(backend, {"no-such-mpack": [BlobLocation(oid, 100, 4)]})
270
271 assert hits == {}
272 assert misses == [oid]
273
274
275 async def test_truncated_range_falls_back_to_the_whole_mpack() -> None:
276 class Truncating(_SpyBackend):
277 async def get_range(self, mpack_key: str, byte_offset: int, byte_length: int) -> bytes | None:
278 data = await super().get_range(mpack_key, byte_offset, byte_length)
279 return None if data is None else data[: len(data) // 2]
280
281 backend = Truncating()
282 blobs = [_blob(_sizeable(9, f"t{i}")) for i in range(3)]
283 mpack_id, locs = await _store(backend, blobs)
284
285 hits, misses = await fetch_mpack_blobs(backend, {mpack_id: list(locs.values())})
286
287 assert misses == []
288 assert hits == dict(blobs)
289 assert backend.whole_calls == [mpack_id]
290
291
292 async def test_corrupted_range_is_never_returned() -> None:
293 class Flipping(_SpyBackend):
294 async def get_range(self, mpack_key: str, byte_offset: int, byte_length: int) -> bytes | None:
295 data = await super().get_range(mpack_key, byte_offset, byte_length)
296 if data is None:
297 return None
298 return bytes([data[0] ^ 0xFF]) + data[1:]
299
300 backend = Flipping()
301 blobs = [_blob(_sizeable(9, f"c{i}")) for i in range(3)]
302 mpack_id, locs = await _store(backend, blobs)
303
304 hits, misses = await fetch_mpack_blobs(backend, {mpack_id: list(locs.values())})
305
306 assert misses == []
307 assert hits == dict(blobs), "ranges are corrupt, the stored mpack is intact: the fallback must serve the truth"
308
309
310 async def test_corrupted_stored_mpack_yields_misses_never_wrong_bytes() -> None:
311 backend = _SpyBackend()
312 blobs = [_blob(_sizeable(9, f"d{i}")) for i in range(3)]
313 mpack_id, locs = await _store(backend, blobs)
314 stored = bytearray(backend._mpacks[mpack_id])
315 first = locs[blobs[0][0]]
316 stored[first.byte_offset + 3] ^= 0xFF # flip one byte inside the first blob
317 backend._mpacks[mpack_id] = bytes(stored)
318
319 hits, misses = await fetch_mpack_blobs(backend, {mpack_id: list(locs.values())})
320
321 assert blobs[0][0] not in hits, "corrupt content must not be returned"
322 assert blobs[0][0] in misses
323 assert all(blob_id(c) == oid for oid, c in hits.items())
324 assert sorted(set(hits) | set(misses)) == sorted(oid for oid, _ in blobs)
325
326
327 async def test_range_addressing_another_object_is_rejected() -> None:
328 backend = _SpyBackend()
329 a, b = _blob(_sizeable(9, "a")), _blob(_sizeable(9, "b"))
330 mpack_id, locs = await _store(backend, [a, b])
331 # b's location filed under a's id: same length, wrong bytes.
332 swapped = BlobLocation(a[0], locs[b[0]].byte_offset, locs[b[0]].byte_length)
333
334 hits, misses = await fetch_mpack_blobs(backend, {mpack_id: [swapped]})
335
336 assert hits == {a[0]: a[1]}
337 assert misses == []
338 assert backend.whole_calls == [mpack_id]
339
340
341 async def test_a_backend_error_on_one_range_does_not_lose_other_blobs() -> None:
342 class Exploding(_SpyBackend):
343 async def get_range(self, mpack_key: str, byte_offset: int, byte_length: int) -> bytes | None:
344 if byte_length % 2 == 0:
345 raise OSError("boom")
346 return await super().get_range(mpack_key, byte_offset, byte_length)
347
348 backend = Exploding()
349 blobs = [_blob(b"x" * n) for n in (11, 12, 13, 14, 15)]
350 mpack_id, locs = await _store(backend, blobs)
351
352 hits, misses = await fetch_mpack_blobs(backend, {mpack_id: list(locs.values())})
353
354 assert misses == []
355 assert hits == dict(blobs)
356
357
358 # ── RR-7 ─────────────────────────────────────────────────────────────────────
359
360
361 async def test_range_reads_are_bounded() -> None:
362 backend = _SpyBackend()
363 backend.range_delay = 0.005
364 blobs = [_blob(f"obj-{i}".encode() * 20) for i in range(RANGE_CONCURRENCY * 6)]
365 mpack_id, locs = await _store(backend, blobs)
366
367 hits, misses = await fetch_mpack_blobs(backend, {mpack_id: list(locs.values())})
368
369 assert misses == []
370 assert hits == dict(blobs)
371 assert 1 < backend.max_in_flight_ranges <= RANGE_CONCURRENCY
372
373
374 async def test_whole_mpack_fallback_is_bounded() -> None:
375 backend = _SpyBackend()
376 backend.whole_delay = 0.01
377 wanted: dict[str, list[BlobLocation]] = {}
378 expected: dict[str, bytes] = {}
379 for i in range(WHOLE_MPACK_CONCURRENCY * 4):
380 blob = _blob(_sizeable(3, f"m{i}"))
381 mpack_id, _ = await _store(backend, [blob])
382 wanted[mpack_id] = [BlobLocation(blob[0], None, None)]
383 expected[blob[0]] = blob[1]
384
385 hits, misses = await fetch_mpack_blobs(backend, wanted)
386
387 assert misses == []
388 assert hits == expected
389 assert backend.max_in_flight_wholes <= WHOLE_MPACK_CONCURRENCY
390
391
392 async def test_cancellation_leaves_no_tasks_and_no_stuck_slots() -> None:
393 backend = _SpyBackend()
394 backend.range_delay = 0.2
395 blobs = [_blob(f"cancel-{i}".encode() * 10) for i in range(RANGE_CONCURRENCY * 3)]
396 mpack_id, locs = await _store(backend, blobs)
397 before = asyncio.all_tasks()
398
399 task = asyncio.create_task(fetch_mpack_blobs(backend, {mpack_id: list(locs.values())}))
400 await asyncio.sleep(0.05)
401 task.cancel()
402 with pytest.raises(asyncio.CancelledError):
403 await task
404 await asyncio.sleep(0)
405
406 assert asyncio.all_tasks() - before - {asyncio.current_task()} == set()
407 assert backend.in_flight_ranges == 0
408
409 backend.range_delay = 0.0
410 hits, misses = await fetch_mpack_blobs(backend, {mpack_id: list(locs.values())})
411 assert misses == [] and hits == dict(blobs)
412
413
414 # ── RR-8 ─────────────────────────────────────────────────────────────────────
415
416
417 async def test_empty_blob_needs_no_io() -> None:
418 backend = _SpyBackend()
419 mpack_id, locs = await _store(backend, [_blob(b"keep")])
420 empty_off = locs[blob_id(b"keep")].byte_offset
421
422 hits, misses = await fetch_mpack_blobs(backend, {mpack_id: [BlobLocation(_EMPTY_OID, empty_off, 0)]})
423
424 assert misses == []
425 assert hits == {_EMPTY_OID: b""}
426 assert backend.range_calls == [] and backend.whole_calls == []
File History 1 commit
sha256:4c1d427f8ff7e2e3557a355edc72231e039930d280d65d299999a1cc898649d2 test(prebuild): RED — bounded ranged-read blob extraction w… Sonnet 5.5 4 days ago