gabriel / muse public
test_integrity_I4_msgpack_size.py python
785 lines 31.4 KB
Raw
sha256:51ce277f663e01a43eaffbe77509b1de7ac2d4251b55d23306304bcdeb92c90d feat(pack): delta-encode snapshots in MPackBundle wire format Sonnet 4.6 minor ⚠ breaking 121 days ago
1 """I-4: Msgpack read size limit — prevent OOM from oversized store files.
2
3 Problem (pre-fix): ``_read_msgpack`` called ``path.read_bytes()`` with no
4 size guard. A 10 GiB corrupt or adversarially crafted ``.msgpack`` file
5 would allocate 10 GiB of RAM, crashing the process or triggering the OOM
6 killer — a critical data-integrity and availability failure.
7
8 ``read_object`` in the object store already had a 256 MiB cap. The commit,
9 snapshot, tag, release, and index stores did not.
10
11 Fix: added to both ``muse/core/store.py`` and ``muse/core/indices.py``:
12
13 1. ``MAX_MSGPACK_BYTES = 64 MiB`` — ``stat().st_size`` is checked *before*
14 ``read_bytes()`` so no allocation ever occurs.
15 2. Per-value limits on ``msgpack.unpackb`` — ``max_str_len``,
16 ``max_bin_len``, ``max_array_len``, ``max_map_len`` — prevent deeply
17 nested or pathologically large single-value documents from consuming
18 unbounded memory even within the size cap.
19
20 This file proves every aspect of the fix:
21
22 Tier 0 — constant export
23 Low-level — stat check before read (OOM prevention)
24 High-level — per-value unpack limits
25 Tier 3 — all high-level read functions (read_commit, read_snapshot, …)
26 Tier 4 — index file protection
27 Tier 5 — CLI command (clean JSON error, no traceback)
28 Tier 6 — boundary / exact-limit behaviour
29 Tier 7 — performance (size check adds < 1 ms overhead)
30 Tier 8 — warning log on oversized file
31 """
32 from __future__ import annotations
33
34 import datetime
35 import logging
36 import pathlib
37 import time
38 from unittest.mock import patch, MagicMock
39
40 import msgpack
41 import pytest
42
43 from muse.core.snapshot import compute_commit_id, compute_snapshot_id
44 from muse.core.store import (
45 MAX_MSGPACK_BYTES,
46 MsgpackValue,
47 CommitRecord,
48 TagRecord,
49 SnapshotRecord,
50 commit_path,
51 snapshot_path,
52 read_commit,
53 read_snapshot,
54 write_commit,
55 write_snapshot,
56 write_tag,
57 get_all_tags,
58 list_releases,
59 )
60
61 from muse.core.types import Manifest, MsgpackDict, fake_id
62 from muse.core.indices import (
63 load_symbol_history,
64 load_hash_occurrence,
65 )
66 from muse.core.paths import commits_dir, indices_dir, muse_dir, releases_dir, snapshots_dir
67
68
69 # ---------------------------------------------------------------------------
70 # Helpers
71 # ---------------------------------------------------------------------------
72
73 _REPO_ID = fake_id("test-repo")
74
75
76 def _repo(tmp_path: pathlib.Path) -> pathlib.Path:
77 muse = muse_dir(tmp_path)
78 (muse / "commits").mkdir(parents=True)
79 (muse / "snapshots").mkdir()
80 (muse / "tags").mkdir()
81 (muse / "releases").mkdir()
82 (muse / "indices").mkdir()
83 (muse / "refs" / "heads").mkdir(parents=True)
84 (muse / "HEAD").write_text("ref: refs/heads/main\n")
85 (muse / "repo.json").write_text(f'{{"repo_id": "{_REPO_ID}"}}\n')
86 return tmp_path
87
88
89 def _commit(idx: int = 0) -> CommitRecord:
90 snapshot_id = compute_snapshot_id({})
91 committed_at = datetime.datetime(2026, 1, 1, tzinfo=datetime.timezone.utc)
92 message = f"commit {idx}"
93 commit_id = compute_commit_id(
94 parent_ids=[],
95 snapshot_id=snapshot_id,
96 message=message,
97 committed_at_iso=committed_at.isoformat(),
98 author="tester",
99 )
100 return CommitRecord(
101 repo_id=_REPO_ID,
102 commit_id=commit_id,
103 branch="main",
104 snapshot_id=snapshot_id,
105 message=message,
106 committed_at=committed_at,
107 author="tester",
108 parent_commit_id=None,
109 parent2_commit_id=None,
110 )
111
112
113 def _snapshot(idx: int = 0) -> SnapshotRecord:
114 manifest: Manifest = {f"__idx__": fake_id(f"snap-{idx}")}
115 sid = compute_snapshot_id(manifest)
116 return SnapshotRecord(
117 snapshot_id=sid,
118 manifest=manifest,
119 )
120
121
122 def _tag(idx: int = 0) -> TagRecord:
123 return TagRecord(
124 repo_id=_REPO_ID,
125 tag_id=fake_id(f"tag-id-{idx}"),
126 commit_id=fake_id(f"tag-commit-{idx}"),
127 tag=f"v{idx}.0.0",
128 )
129
130
131 # ---------------------------------------------------------------------------
132 # Tier 0 — constant export
133 # ---------------------------------------------------------------------------
134
135 class TestConstantExport:
136 """MAX_MSGPACK_BYTES must be importable and have the correct value."""
137
138 def test_max_msgpack_bytes_is_exported(self) -> None:
139 from muse.core.store import MAX_MSGPACK_BYTES as cap
140 assert cap == 64 * 1024 * 1024, (
141 f"Expected 64 MiB (67108864), got {cap}"
142 )
143
144 def test_max_msgpack_bytes_is_int(self) -> None:
145 assert isinstance(MAX_MSGPACK_BYTES, int)
146
147 def test_max_msgpack_bytes_less_than_256mib(self) -> None:
148 """Commit/snapshot records should be capped well below 256 MiB."""
149 assert MAX_MSGPACK_BYTES < 256 * 1024 * 1024, (
150 "Msgpack records should be capped below the object store's 256 MiB limit"
151 )
152
153
154 # ---------------------------------------------------------------------------
155 # Low-level — stat check fires BEFORE read_bytes (the OOM prevention)
156 # ---------------------------------------------------------------------------
157
158 class TestStatCheckBeforeRead:
159 """The size guard must fire before any read_bytes() call.
160
161 We prove this by mocking stat to report an oversized file while keeping
162 the actual file tiny — if read_bytes() were called first, we would NOT
163 trigger the OSError from the stat check.
164 """
165
166 def _oversized_stat(self, real_path: pathlib.Path) -> MagicMock:
167 """Return a MagicMock that reports st_size = MAX_MSGPACK_BYTES + 1."""
168 stat_result = MagicMock()
169 stat_result.st_size = MAX_MSGPACK_BYTES + 1
170 return stat_result
171
172 def test_read_commit_checks_stat_before_read_bytes(
173 self, tmp_path: pathlib.Path
174 ) -> None:
175 root = _repo(tmp_path)
176 c = _commit(0)
177 write_commit(root, c)
178
179 commit_path = commits_dir(root) / f"{c.commit_id}.msgpack"
180 real_stat = commit_path.stat # preserve reference
181
182 with patch.object(type(commit_path), "stat") as mock_stat:
183 mock_stat.return_value = self._oversized_stat(commit_path)
184 read_bytes_called = [False]
185 real_read_bytes = commit_path.read_bytes
186
187 def tracking_read_bytes() -> bytes:
188 read_bytes_called[0] = True
189 return real_read_bytes()
190
191 with patch.object(type(commit_path), "read_bytes", tracking_read_bytes):
192 result = read_commit(root, c.commit_id)
193
194 assert result is None, "read_commit should return None for oversized file"
195 assert not read_bytes_called[0], (
196 "read_bytes() was called BEFORE the stat size check — OOM risk!"
197 )
198
199 def test_read_snapshot_checks_stat_before_read_bytes(
200 self, tmp_path: pathlib.Path
201 ) -> None:
202 root = _repo(tmp_path)
203 s = _snapshot(0)
204 write_snapshot(root, s)
205
206 snap_path = snapshots_dir(root) / f"{s.snapshot_id}.msgpack"
207 read_bytes_called = [False]
208 real_read_bytes = snap_path.read_bytes
209
210 def tracking_read_bytes() -> bytes:
211 read_bytes_called[0] = True
212 return real_read_bytes()
213
214 with patch.object(type(snap_path), "stat") as mock_stat:
215 mock_stat.return_value = self._oversized_stat(snap_path)
216 with patch.object(type(snap_path), "read_bytes", tracking_read_bytes):
217 result = read_snapshot(root, s.snapshot_id)
218
219 assert result is None
220 assert not read_bytes_called[0], (
221 "read_bytes() was called before the stat size check in read_snapshot"
222 )
223
224
225 # ---------------------------------------------------------------------------
226 # High-level — high-level read functions return None for oversized files
227 # ---------------------------------------------------------------------------
228
229 class TestReadFunctionsReturnNoneOnOversize:
230 """All public read functions must gracefully handle oversized files.
231
232 We patch MAX_MSGPACK_BYTES to a small value so we can create real files
233 that exceed it without writing gigabytes to disk.
234 """
235
236 def _write_oversized_commit(
237 self, root: pathlib.Path, c: CommitRecord, limit: int
238 ) -> None:
239 """Write a commit, then inflate the file beyond *limit* bytes."""
240 write_commit(root, c)
241 path = commits_dir(root) / f"{c.commit_id}.msgpack"
242 # Overwrite with limit+1 bytes of valid-looking (but unparseable) data.
243 path.write_bytes(b"\x00" * (limit + 1))
244
245 def test_read_commit_returns_none_for_oversized_file(
246 self, tmp_path: pathlib.Path
247 ) -> None:
248 root = _repo(tmp_path)
249 c = _commit(1)
250 with patch("muse.core.store.MAX_MSGPACK_BYTES", 100):
251 self._write_oversized_commit(root, c, 100)
252 result = read_commit(root, c.commit_id)
253 assert result is None, "read_commit must return None, not raise, for oversized file"
254
255 def test_read_snapshot_returns_none_for_oversized_file(
256 self, tmp_path: pathlib.Path
257 ) -> None:
258 root = _repo(tmp_path)
259 s = _snapshot(1)
260 write_snapshot(root, s)
261 snap_path = snapshots_dir(root) / f"{s.snapshot_id}.msgpack"
262 snap_path.write_bytes(b"\x00" * 101)
263 with patch("muse.core.store.MAX_MSGPACK_BYTES", 100):
264 result = read_snapshot(root, s.snapshot_id)
265 assert result is None
266
267 def test_get_all_tags_skips_oversized_files(
268 self, tmp_path: pathlib.Path
269 ) -> None:
270 """get_all_tags iterates all tag files — oversized ones are skipped."""
271 root = _repo(tmp_path)
272 good = _tag(0)
273 bad = _tag(1)
274 write_tag(root, good)
275 write_tag(root, bad)
276
277 # A real tag record is ~200 bytes packed (64-char IDs + timestamp).
278 # Choose a limit above a real tag but below our inflated bad file.
279 from muse.core.store import tag_path
280 good_path = tag_path(root, _REPO_ID, good.tag_id)
281 real_size = good_path.stat().st_size
282 test_limit = real_size * 2 # real tag fits; we'll inflate the bad tag to 3×
283
284 bad_path = tag_path(root, _REPO_ID, bad.tag_id)
285 bad_path.write_bytes(b"\x00" * (real_size * 3)) # definitely exceeds limit
286
287 with patch("muse.core.store.MAX_MSGPACK_BYTES", test_limit):
288 tags = get_all_tags(root, _REPO_ID)
289 tag_ids = {t.tag_id for t in tags}
290 assert good.tag_id in tag_ids, "Good tag was incorrectly dropped"
291 assert bad.tag_id not in tag_ids, "Oversized tag was not skipped"
292
293 def test_list_releases_skips_oversized_files(
294 self, tmp_path: pathlib.Path
295 ) -> None:
296 """list_releases must skip oversized release files."""
297 root = _repo(tmp_path)
298 from muse.core.types import split_id
299 r_algo, r_hex = split_id(_REPO_ID)
300 rel_dir = releases_dir(root) / r_algo / r_hex
301 rel_dir.mkdir(parents=True)
302 # Write a fake oversized release file.
303 fake_release = rel_dir / f"{'a' * 64}.msgpack"
304 fake_release.write_bytes(b"\x00" * 101)
305 with patch("muse.core.store.MAX_MSGPACK_BYTES", 100):
306 results = list_releases(root, _REPO_ID)
307 assert results == [], "Oversized release should be skipped, not crash"
308
309
310 # ---------------------------------------------------------------------------
311 # Tier 3 — exact boundary behaviour
312 # ---------------------------------------------------------------------------
313
314 class TestExactBoundary:
315 """At the boundary: MAX_MSGPACK_BYTES is the last allowed size."""
316
317 def test_file_exactly_at_limit_is_read(self, tmp_path: pathlib.Path) -> None:
318 """A file of exactly MAX_MSGPACK_BYTES bytes passes the size check.
319
320 The content may be unparseable (zeros are not valid msgpack), but the
321 OSError raised is a parse error, not a size-limit error.
322 """
323 test_limit = 256 # small limit for test speed
324 path = tmp_path / "exactly_at_limit.msgpack"
325 path.write_bytes(b"\x00" * test_limit)
326 with patch("muse.core.store.MAX_MSGPACK_BYTES", test_limit):
327 # Should raise a parse error (invalid msgpack), NOT an OSError about size.
328 from muse.core.store import _read_msgpack
329 try:
330 _read_msgpack(path)
331 pytest.fail("Expected an error for invalid msgpack content")
332 except OSError as exc:
333 assert "MiB read limit" not in str(exc), (
334 f"Got size-limit OSError at the boundary — should be parse error: {exc}"
335 )
336 except Exception:
337 pass # Any non-size-limit error is acceptable here
338
339 def test_file_one_byte_over_limit_raises_oslimit_error(
340 self, tmp_path: pathlib.Path
341 ) -> None:
342 """A file of MAX_MSGPACK_BYTES + 1 bytes raises OSError before reading."""
343 test_limit = 256
344 path = tmp_path / "one_over.msgpack"
345 path.write_bytes(b"\x00" * (test_limit + 1))
346 with patch("muse.core.store.MAX_MSGPACK_BYTES", test_limit):
347 from muse.core.store import _read_msgpack
348 with pytest.raises(OSError, match="read limit"):
349 _read_msgpack(path)
350
351 def test_zero_byte_file_does_not_trigger_size_limit(
352 self, tmp_path: pathlib.Path
353 ) -> None:
354 """An empty file passes the size check but fails msgpack parse."""
355 path = tmp_path / "empty.msgpack"
356 path.write_bytes(b"")
357 from muse.core.store import _read_msgpack
358 with pytest.raises(Exception): # parse error, not size error
359 _read_msgpack(path)
360
361 def test_size_limit_error_message_includes_filename_and_limit(
362 self, tmp_path: pathlib.Path
363 ) -> None:
364 """The OSError message must include the file name and limit in MiB."""
365 test_limit = 1024 # 1 KiB for test speed
366 path = tmp_path / "big.msgpack"
367 path.write_bytes(b"\x00" * (test_limit + 1))
368 with patch("muse.core.store.MAX_MSGPACK_BYTES", test_limit):
369 from muse.core.store import _read_msgpack
370 with pytest.raises(OSError) as exc_info:
371 _read_msgpack(path)
372 msg = str(exc_info.value)
373 assert "big.msgpack" in msg, f"Filename missing from error: {msg}"
374 assert "KiB" in msg or "MiB" in msg or "bytes" in msg, (
375 f"Size info missing from error: {msg}"
376 )
377
378
379 # ---------------------------------------------------------------------------
380 # Tier 4 — per-value unpack limits
381 # ---------------------------------------------------------------------------
382
383 class TestPerValueUnpackLimits:
384 """Verify that per-value limits from msgpack.unpackb are enforced."""
385
386 def _pack_to_path(self, tmp_path: pathlib.Path, data: MsgpackValue) -> pathlib.Path:
387 path = tmp_path / "test.msgpack"
388 path.write_bytes(msgpack.packb(data, use_bin_type=True))
389 return path
390
391 def test_string_exceeding_max_str_len_rejected(self, tmp_path: pathlib.Path) -> None:
392 """A string longer than _MSGPACK_MAX_STR_LEN must raise an exception."""
393 huge_str = "x" * 200
394 path = self._pack_to_path(tmp_path, {"key": huge_str})
395 from muse.core.store import _read_msgpack
396 with patch("muse.core.store._MSGPACK_MAX_STR_LEN", 100):
397 with pytest.raises(Exception):
398 _read_msgpack(path)
399
400 def test_string_within_max_str_len_accepted(self, tmp_path: pathlib.Path) -> None:
401 """A string within the limit unpacks normally."""
402 path = self._pack_to_path(tmp_path, {"key": "short"})
403 from muse.core.store import _read_msgpack
404 result = _read_msgpack(path)
405 assert isinstance(result, dict)
406
407 def test_binary_blob_rejected_in_store_records(self, tmp_path: pathlib.Path) -> None:
408 """Binary data (msgpack bin type) must be rejected for store records.
409
410 Commit/snapshot/tag records contain no binary fields. A file with
411 binary data is either corrupt or tampered. max_bin_len=0 ensures
412 this is caught immediately during unpack rather than producing a
413 ``bytes`` value that callers are not prepared to handle.
414 """
415 path = self._pack_to_path(tmp_path, {"body": b"some binary blob"})
416 from muse.core.store import _read_msgpack
417 # max_bin_len=0 means any bin-type value raises an error.
418 with pytest.raises(Exception):
419 _read_msgpack(path)
420
421 def test_map_exceeding_max_map_len_rejected(self, tmp_path: pathlib.Path) -> None:
422 """A map with more than _MSGPACK_MAX_MAP_LEN entries must raise."""
423 big_map: MsgpackDict = {str(i): i for i in range(200)}
424 path = self._pack_to_path(tmp_path, big_map)
425 from muse.core.store import _read_msgpack
426 with patch("muse.core.store._MSGPACK_MAX_MAP_LEN", 100):
427 with pytest.raises(Exception):
428 _read_msgpack(path)
429
430 def test_array_exceeding_max_array_len_rejected(self, tmp_path: pathlib.Path) -> None:
431 """An array with more than _MSGPACK_MAX_ARRAY_LEN entries must raise."""
432 big_list: list[MsgpackValue] = list(range(200))
433 path = self._pack_to_path(tmp_path, big_list)
434 from muse.core.store import _read_msgpack
435 with patch("muse.core.store._MSGPACK_MAX_ARRAY_LEN", 100):
436 with pytest.raises(Exception):
437 _read_msgpack(path)
438
439 def _make_deep_nested_msgpack(self, depth: int) -> bytes:
440 """Build msgpack bytes for a *depth*-deep nested dict without Python recursion.
441
442 ``msgpack.packb`` uses Python-level recursion so packing a 600-deep
443 dict hits the default recursion limit. We build the bytes directly:
444
445 fixmap(1) fixstr("x") fixmap(1) fixstr("x") ... fixmap(0)
446
447 Each level is 3 bytes: ``0x81`` (fixmap 1 entry) + ``0xa1 0x78``
448 (fixstr "x"). The leaf is ``0x80`` (fixmap 0 entries).
449
450 This produces a valid msgpack binary that ``unpackb`` will parse up
451 to its stack limit and then raise ``StackError``.
452 """
453 # 0x81 = fixmap with 1 item; 0xa1 0x78 = fixstr "x"
454 frame = b"\x81\xa1x"
455 leaf = b"\x80" # fixmap with 0 items
456 return frame * depth + leaf
457
458 def test_deeply_nested_map_raises_stack_error(self, tmp_path: pathlib.Path) -> None:
459 """A pathologically nested document hits msgpack's StackError.
460
461 At extreme depth (10 000 levels), msgpack's C-extension stack limit is
462 exceeded and an exception is raised. The file is only ~30 KiB so the
463 size check passes; the protection comes from msgpack's internal stack
464 guard, not the 64 MiB cap.
465 """
466 packed = self._make_deep_nested_msgpack(10_000)
467 path = tmp_path / "deep_nest.msgpack"
468 path.write_bytes(packed)
469 from muse.core.store import _read_msgpack
470 with pytest.raises(Exception): # msgpack.exceptions.StackError
471 _read_msgpack(path)
472
473 def test_deeply_nested_terminates_quickly(self, tmp_path: pathlib.Path) -> None:
474 """The StackError for deeply nested documents is raised in < 1 second."""
475 packed = self._make_deep_nested_msgpack(10_000)
476 path = tmp_path / "deep_nest_perf.msgpack"
477 path.write_bytes(packed)
478 from muse.core.store import _read_msgpack
479 start = time.perf_counter()
480 try:
481 _read_msgpack(path)
482 except Exception:
483 pass
484 elapsed = time.perf_counter() - start
485 assert elapsed < 1.0, (
486 f"Deeply nested document took {elapsed:.3f}s to fail — not fast enough"
487 )
488
489 def test_valid_large_map_within_limits_is_accepted(self, tmp_path: pathlib.Path) -> None:
490 """A large but within-limit map (simulating a 1k-file snapshot) unpacks cleanly."""
491 # Simulate a 1000-file snapshot manifest: {path: object_id}
492 manifest = {f"src/file_{i:04d}.py": fake_id(f"obj-{i}") for i in range(1000)}
493 path = tmp_path / "big_valid.msgpack"
494 path.write_bytes(msgpack.packb(manifest, use_bin_type=True))
495 from muse.core.store import _read_msgpack
496 result = _read_msgpack(path)
497 assert isinstance(result, dict)
498 assert len(result) == 1000
499
500
501 # ---------------------------------------------------------------------------
502 # Tier 5 — index file protection
503 # ---------------------------------------------------------------------------
504
505 class TestIndexReadProtection:
506 """muse/core/indices.py has its own _read_msgpack — must also be protected."""
507
508 def test_load_symbol_history_skips_oversized_index(
509 self, tmp_path: pathlib.Path
510 ) -> None:
511 """An oversized symbol history index returns an empty dict, not OOM."""
512 (indices_dir(tmp_path)).mkdir(parents=True)
513 index_path = indices_dir(tmp_path) / "symbol_history.msgpack"
514 index_path.write_bytes(b"\x00" * 101)
515 with patch("muse.core.indices._MAX_INDEX_BYTES", 100):
516 result = load_symbol_history(tmp_path)
517 assert result == {}, "Oversized index must return empty dict, not crash"
518
519 def test_load_hash_occurrence_skips_oversized_index(
520 self, tmp_path: pathlib.Path
521 ) -> None:
522 """An oversized hash_occurrence index returns an empty dict."""
523 (indices_dir(tmp_path)).mkdir(parents=True)
524 index_path = indices_dir(tmp_path) / "hash_occurrence.msgpack"
525 index_path.write_bytes(b"\x00" * 101)
526 with patch("muse.core.indices._MAX_INDEX_BYTES", 100):
527 result = load_hash_occurrence(tmp_path)
528 assert result == {}
529
530 def test_index_size_limit_is_more_generous_than_store(self) -> None:
531 """Index files are allowed to be larger than store records."""
532 from muse.core.indices import _MAX_INDEX_BYTES
533 assert _MAX_INDEX_BYTES > MAX_MSGPACK_BYTES, (
534 "Index limit should be larger than store limit — indices grow with repo size"
535 )
536
537 def test_index_read_checks_stat_before_read_bytes(
538 self, tmp_path: pathlib.Path
539 ) -> None:
540 """The index stat check must fire before read_bytes (no allocation)."""
541 (indices_dir(tmp_path)).mkdir(parents=True)
542 index_path = indices_dir(tmp_path) / "symbol_history.msgpack"
543 index_path.write_bytes(b"\x85") # 1 byte — well within any size limit
544 read_bytes_called = [False]
545 real_rb = index_path.read_bytes
546
547 def tracking_rb() -> bytes:
548 read_bytes_called[0] = True
549 return real_rb()
550
551 stat_result = MagicMock()
552 stat_result.st_size = 1024 * 1024 * 1024 # 1 GiB — way over limit
553
554 with patch.object(type(index_path), "stat", return_value=stat_result):
555 with patch.object(type(index_path), "read_bytes", tracking_rb):
556 result = load_symbol_history(tmp_path)
557
558 assert result == {}
559 assert not read_bytes_called[0], "read_bytes was called before the stat check!"
560
561
562 # ---------------------------------------------------------------------------
563 # Tier 6 — warning log on oversized file
564 # ---------------------------------------------------------------------------
565
566 class TestWarningLogOnOversizedFile:
567 """Operators need to know when oversized files are detected.
568
569 read_commit / read_snapshot log a WARNING when they catch the OSError
570 from _read_msgpack — this surfaces corruption or tampering in monitoring.
571 """
572
573 def test_warning_logged_for_oversized_commit(
574 self, tmp_path: pathlib.Path, caplog: pytest.LogCaptureFixture
575 ) -> None:
576 root = _repo(tmp_path)
577 c = _commit(10)
578 with patch("muse.core.store.MAX_MSGPACK_BYTES", 50):
579 path = commit_path(root, c.commit_id)
580 path.parent.mkdir(parents=True, exist_ok=True)
581 path.write_bytes(b"\x00" * 51)
582 with caplog.at_level(logging.WARNING, logger="muse.core.store"):
583 result = read_commit(root, c.commit_id)
584 assert result is None
585 # A warning must have been emitted for the corrupt/oversized file.
586 assert any(
587 "Corrupt" in rec.message or "corrupt" in rec.message
588 or "oversized" in rec.message or "limit" in rec.message.lower()
589 for rec in caplog.records
590 ), f"No warning logged for oversized commit. Records: {[r.message for r in caplog.records]}"
591
592 def test_warning_logged_for_oversized_snapshot(
593 self, tmp_path: pathlib.Path, caplog: pytest.LogCaptureFixture
594 ) -> None:
595 root = _repo(tmp_path)
596 s = _snapshot(10)
597 with patch("muse.core.store.MAX_MSGPACK_BYTES", 50):
598 path = snapshot_path(root, s.snapshot_id)
599 path.parent.mkdir(parents=True, exist_ok=True)
600 path.write_bytes(b"\x00" * 51)
601 with caplog.at_level(logging.WARNING, logger="muse.core.store"):
602 result = read_snapshot(root, s.snapshot_id)
603 assert result is None
604 assert any(
605 "Corrupt" in rec.message or "corrupt" in rec.message
606 for rec in caplog.records
607 ), f"No warning logged. Records: {[r.message for r in caplog.records]}"
608
609
610 # ---------------------------------------------------------------------------
611 # Tier 7 — CLI: clean JSON error, no traceback
612 # ---------------------------------------------------------------------------
613
614 class TestPlumbingReadCommitOversized:
615 """muse read-commit with an oversized commit file must produce
616 a clean, machine-readable JSON error — no Python traceback, no process crash.
617 """
618
619 def test_oversized_commit_produces_json_error_not_traceback(
620 self, tmp_path: pathlib.Path
621 ) -> None:
622 """write a commit, corrupt its file, run read-commit — must get JSON error."""
623 import json
624 import sys
625 from tests.cli_test_helper import CliRunner
626
627 root = _repo(tmp_path)
628 c = _commit(99)
629 write_commit(root, c)
630
631 # Corrupt the commit file to exceed the limit.
632 commit_path_file = commit_path(root, c.commit_id)
633 commit_path_file.write_bytes(b"\x00" * 101)
634
635 runner = CliRunner()
636 with patch("muse.core.store.MAX_MSGPACK_BYTES", 100):
637 result = runner.invoke(None, ["read-commit", c.commit_id],
638 env={"MUSE_REPO_ROOT": str(root)})
639
640 # Must not crash (exit code may be non-zero, but not a Python traceback).
641 assert "Traceback" not in (result.output or ""), (
642 f"CLI produced a Python traceback for oversized commit:\n{result.output}"
643 )
644 assert "Traceback" not in (result.stderr or ""), (
645 f"CLI stderr has a Python traceback:\n{result.stderr}"
646 )
647 # The error output must be valid JSON (or include a meaningful error).
648 combined = (result.output or "") + (result.stderr or "")
649 try:
650 # Check if any JSON blob exists in the output.
651 for line in combined.splitlines():
652 line = line.strip()
653 if line.startswith("{"):
654 parsed = json.loads(line)
655 assert "error" in parsed, f"JSON lacks 'error' key: {parsed}"
656 break
657 else:
658 # If no JSON line found, at minimum confirm no traceback and
659 # that "not found" or "error" appears in the output.
660 assert (
661 "not found" in combined.lower()
662 or "error" in combined.lower()
663 ), f"No useful error in CLI output:\n{combined}"
664 except json.JSONDecodeError as exc:
665 pytest.fail(f"Output is not valid JSON: {exc}\nOutput:\n{combined}")
666
667
668 # ---------------------------------------------------------------------------
669 # Tier 8 — round-trip: valid files still read correctly
670 # ---------------------------------------------------------------------------
671
672 class TestValidFilesUnaffected:
673 """The size guard must not regress normal reads."""
674
675 def test_read_commit_roundtrip_unaffected(self, tmp_path: pathlib.Path) -> None:
676 root = _repo(tmp_path)
677 c = _commit(42)
678 write_commit(root, c)
679 got = read_commit(root, c.commit_id)
680 assert got is not None
681 assert got.commit_id == c.commit_id
682 assert got.message == c.message
683
684 def test_read_snapshot_roundtrip_unaffected(self, tmp_path: pathlib.Path) -> None:
685 root = _repo(tmp_path)
686 s = _snapshot(42)
687 write_snapshot(root, s)
688 got = read_snapshot(root, s.snapshot_id)
689 assert got is not None
690 assert got.snapshot_id == s.snapshot_id
691
692 def test_snapshot_with_large_manifest_reads_correctly(
693 self, tmp_path: pathlib.Path
694 ) -> None:
695 """A 1000-file snapshot manifest (realistic scale) reads without issue."""
696 root = _repo(tmp_path)
697 manifest = {f"src/file_{i:05d}.py": fake_id(f"obj-{i}") for i in range(1000)}
698 sid = compute_snapshot_id(manifest)
699 s = SnapshotRecord(
700 snapshot_id=sid,
701 manifest=manifest,
702 )
703 write_snapshot(root, s)
704 got = read_snapshot(root, sid)
705 assert got is not None
706 assert len(got.manifest) == 1000
707
708 def test_commit_with_long_message_reads_correctly(
709 self, tmp_path: pathlib.Path
710 ) -> None:
711 """A commit with a 64 KiB message reads correctly (well within 1 MiB str limit)."""
712 root = _repo(tmp_path)
713 long_msg = "a" * 65536
714 committed_at = datetime.datetime(2026, 1, 1, tzinfo=datetime.timezone.utc)
715 snapshot_id = compute_snapshot_id({})
716 cid = compute_commit_id(
717 parent_ids=[],
718 snapshot_id=snapshot_id,
719 message=long_msg,
720 committed_at_iso=committed_at.isoformat(),
721 author="tester",
722 )
723 c = CommitRecord(
724 repo_id=_REPO_ID,
725 commit_id=cid,
726 branch="main",
727 snapshot_id=snapshot_id,
728 message=long_msg,
729 committed_at=committed_at,
730 author="tester",
731 parent_commit_id=None,
732 parent2_commit_id=None,
733 )
734 write_commit(root, c)
735 got = read_commit(root, cid)
736 assert got is not None
737 assert len(got.message) == 65536
738
739
740 # ---------------------------------------------------------------------------
741 # Tier 9 — performance: size check adds < 1 ms per read
742 # ---------------------------------------------------------------------------
743
744 class TestSizeCheckPerformance:
745 """The stat() check should add negligible overhead to normal reads."""
746
747 @pytest.mark.perf
748 def test_stat_check_overhead_under_1ms_per_read(
749 self, tmp_path: pathlib.Path
750 ) -> None:
751 """100 sequential read_commit calls with the size guard active < 100ms total."""
752 root = _repo(tmp_path)
753 commits = [_commit(i) for i in range(100)]
754 for c in commits:
755 write_commit(root, c)
756
757 start = time.perf_counter()
758 for c in commits:
759 result = read_commit(root, c.commit_id)
760 assert result is not None
761 elapsed = time.perf_counter() - start
762
763 assert elapsed < 0.1, (
764 f"100 read_commit calls took {elapsed:.3f}s — "
765 "size check is adding too much overhead (< 100ms expected)"
766 )
767
768 @pytest.mark.perf
769 def test_oversized_rejection_under_1ms(self, tmp_path: pathlib.Path) -> None:
770 """Rejecting an oversized file (via stat) takes < 1ms — no disk I/O."""
771 root = _repo(tmp_path)
772 c = _commit(200)
773 write_commit(root, c)
774 path = commits_dir(root) / f"{c.commit_id}.msgpack"
775 path.write_bytes(b"\x00" * 101)
776
777 start = time.perf_counter()
778 with patch("muse.core.store.MAX_MSGPACK_BYTES", 100):
779 for _ in range(1000):
780 read_commit(root, c.commit_id)
781 elapsed = time.perf_counter() - start
782
783 assert elapsed < 1.0, (
784 f"1000 oversized-rejection calls took {elapsed:.3f}s (> 1ms each)"
785 )
File History 1 commit
sha256:51ce277f663e01a43eaffbe77509b1de7ac2d4251b55d23306304bcdeb92c90d feat(pack): delta-encode snapshots in MPackBundle wire format Sonnet 4.6 minor ⚠ 121 days ago