gabriel / muse public
test_integrity_I4_msgpack_size.py python
788 lines 31.5 KB
Raw
sha256:b636f72dcba9e190afb980bece906fa5b717fbde014b76ef023df8cb96e01eb9 docs: expand cache plan with all seven testing tiers and do… Sonnet 4.6 131 days ago
1 """I-4: Msgpack read size limit — prevent OOM from oversized store files.
2
3 Problem (pre-fix): ``_read_msgpack`` called ``path.read_bytes()`` with no
4 size guard. A 10 GiB corrupt or adversarially crafted ``.msgpack`` file
5 would allocate 10 GiB of RAM, crashing the process or triggering the OOM
6 killer — a critical data-integrity and availability failure.
7
8 ``read_object`` in the object store already had a 256 MiB cap. The commit,
9 snapshot, tag, release, and index stores did not.
10
11 Fix: added to both ``muse/core/store.py`` and ``muse/core/indices.py``:
12
13 1. ``MAX_MSGPACK_BYTES = 64 MiB`` — ``stat().st_size`` is checked *before*
14 ``read_bytes()`` so no allocation ever occurs.
15 2. Per-value limits on ``msgpack.unpackb`` — ``max_str_len``,
16 ``max_bin_len``, ``max_array_len``, ``max_map_len`` — prevent deeply
17 nested or pathologically large single-value documents from consuming
18 unbounded memory even within the size cap.
19
20 This file proves every aspect of the fix:
21
22 Tier 0 — constant export
23 Low-level — stat check before read (OOM prevention)
24 High-level — per-value unpack limits
25 Tier 3 — all high-level read functions (read_commit, read_snapshot, …)
26 Tier 4 — index file protection
27 Tier 5 — CLI command (clean JSON error, no traceback)
28 Tier 6 — boundary / exact-limit behaviour
29 Tier 7 — performance (size check adds < 1 ms overhead)
30 Tier 8 — warning log on oversized file
31 """
32 from __future__ import annotations
33
34 import datetime
35 import logging
36 import pathlib
37 import time
38 from unittest.mock import patch, MagicMock
39
40 import msgpack
41 import pytest
42
43 from muse.core.snapshot import compute_commit_id, compute_snapshot_id
44 from muse.core.store import (
45 MAX_MSGPACK_BYTES,
46 MsgpackValue,
47 CommitRecord,
48 TagRecord,
49 SnapshotRecord,
50 commit_path,
51 snapshot_path,
52 read_commit,
53 read_snapshot,
54 write_commit,
55 write_snapshot,
56 write_tag,
57 get_all_tags,
58 list_releases,
59 )
60
61 from muse.core._types import Manifest, MsgpackDict, fake_id
62 from muse.core.indices import (
63 load_symbol_history,
64 load_hash_occurrence,
65 )
66
67
68 # ---------------------------------------------------------------------------
69 # Helpers
70 # ---------------------------------------------------------------------------
71
72 _REPO_ID = fake_id("test-repo")
73
74
75 def _repo(tmp_path: pathlib.Path) -> pathlib.Path:
76 muse = tmp_path / ".muse"
77 (muse / "commits").mkdir(parents=True)
78 (muse / "snapshots").mkdir()
79 (muse / "tags").mkdir()
80 (muse / "releases").mkdir()
81 (muse / "indices").mkdir()
82 (muse / "refs" / "heads").mkdir(parents=True)
83 (muse / "HEAD").write_text("ref: refs/heads/main\n")
84 (muse / "repo.json").write_text(f'{{"repo_id": "{_REPO_ID}"}}\n')
85 return tmp_path
86
87
88 def _sha(seed: str) -> str:
89 return fake_id(seed)
90
91
92 def _commit(idx: int = 0) -> CommitRecord:
93 snapshot_id = compute_snapshot_id({})
94 committed_at = datetime.datetime(2026, 1, 1, tzinfo=datetime.timezone.utc)
95 message = f"commit {idx}"
96 commit_id = compute_commit_id(
97 repo_id=_REPO_ID,
98 parent_ids=[],
99 snapshot_id=snapshot_id,
100 message=message,
101 committed_at_iso=committed_at.isoformat(),
102 author="tester",)
103 return CommitRecord(
104 commit_id=commit_id,
105 repo_id=_REPO_ID,
106 created_on_branch="main",
107 snapshot_id=snapshot_id,
108 message=message,
109 committed_at=committed_at,
110 author="tester",
111 parent_commit_id=None,
112 parent2_commit_id=None,
113 )
114
115
116 def _snapshot(idx: int = 0) -> SnapshotRecord:
117 manifest: Manifest = {f"__idx__": _sha(f"snap-{idx}")}
118 sid = compute_snapshot_id(manifest)
119 return SnapshotRecord(
120 snapshot_id=sid,
121 manifest=manifest,
122 )
123
124
125 def _tag(idx: int = 0) -> TagRecord:
126 return TagRecord(
127 tag_id=_sha(f"tag-id-{idx}"),
128 repo_id=_REPO_ID,
129 commit_id=_sha(f"tag-commit-{idx}"),
130 tag=f"v{idx}.0.0",
131 )
132
133
134 # ---------------------------------------------------------------------------
135 # Tier 0 — constant export
136 # ---------------------------------------------------------------------------
137
138 class TestConstantExport:
139 """MAX_MSGPACK_BYTES must be importable and have the correct value."""
140
141 def test_max_msgpack_bytes_is_exported(self) -> None:
142 from muse.core.store import MAX_MSGPACK_BYTES as cap
143 assert cap == 64 * 1024 * 1024, (
144 f"Expected 64 MiB (67108864), got {cap}"
145 )
146
147 def test_max_msgpack_bytes_is_int(self) -> None:
148 assert isinstance(MAX_MSGPACK_BYTES, int)
149
150 def test_max_msgpack_bytes_less_than_256mib(self) -> None:
151 """Commit/snapshot records should be capped well below 256 MiB."""
152 assert MAX_MSGPACK_BYTES < 256 * 1024 * 1024, (
153 "Msgpack records should be capped below the object store's 256 MiB limit"
154 )
155
156
157 # ---------------------------------------------------------------------------
158 # Low-level — stat check fires BEFORE read_bytes (the OOM prevention)
159 # ---------------------------------------------------------------------------
160
161 class TestStatCheckBeforeRead:
162 """The size guard must fire before any read_bytes() call.
163
164 We prove this by mocking stat to report an oversized file while keeping
165 the actual file tiny — if read_bytes() were called first, we would NOT
166 trigger the OSError from the stat check.
167 """
168
169 def _oversized_stat(self, real_path: pathlib.Path) -> MagicMock:
170 """Return a MagicMock that reports st_size = MAX_MSGPACK_BYTES + 1."""
171 stat_result = MagicMock()
172 stat_result.st_size = MAX_MSGPACK_BYTES + 1
173 return stat_result
174
175 def test_read_commit_checks_stat_before_read_bytes(
176 self, tmp_path: pathlib.Path
177 ) -> None:
178 root = _repo(tmp_path)
179 c = _commit(0)
180 write_commit(root, c)
181
182 commit_path = root / ".muse" / "commits" / f"{c.commit_id}.msgpack"
183 real_stat = commit_path.stat # preserve reference
184
185 with patch.object(type(commit_path), "stat") as mock_stat:
186 mock_stat.return_value = self._oversized_stat(commit_path)
187 read_bytes_called = [False]
188 real_read_bytes = commit_path.read_bytes
189
190 def tracking_read_bytes() -> bytes:
191 read_bytes_called[0] = True
192 return real_read_bytes()
193
194 with patch.object(type(commit_path), "read_bytes", tracking_read_bytes):
195 result = read_commit(root, c.commit_id)
196
197 assert result is None, "read_commit should return None for oversized file"
198 assert not read_bytes_called[0], (
199 "read_bytes() was called BEFORE the stat size check — OOM risk!"
200 )
201
202 def test_read_snapshot_checks_stat_before_read_bytes(
203 self, tmp_path: pathlib.Path
204 ) -> None:
205 root = _repo(tmp_path)
206 s = _snapshot(0)
207 write_snapshot(root, s)
208
209 snap_path = root / ".muse" / "snapshots" / f"{s.snapshot_id}.msgpack"
210 read_bytes_called = [False]
211 real_read_bytes = snap_path.read_bytes
212
213 def tracking_read_bytes() -> bytes:
214 read_bytes_called[0] = True
215 return real_read_bytes()
216
217 with patch.object(type(snap_path), "stat") as mock_stat:
218 mock_stat.return_value = self._oversized_stat(snap_path)
219 with patch.object(type(snap_path), "read_bytes", tracking_read_bytes):
220 result = read_snapshot(root, s.snapshot_id)
221
222 assert result is None
223 assert not read_bytes_called[0], (
224 "read_bytes() was called before the stat size check in read_snapshot"
225 )
226
227
228 # ---------------------------------------------------------------------------
229 # High-level — high-level read functions return None for oversized files
230 # ---------------------------------------------------------------------------
231
232 class TestReadFunctionsReturnNoneOnOversize:
233 """All public read functions must gracefully handle oversized files.
234
235 We patch MAX_MSGPACK_BYTES to a small value so we can create real files
236 that exceed it without writing gigabytes to disk.
237 """
238
239 def _write_oversized_commit(
240 self, root: pathlib.Path, c: CommitRecord, limit: int
241 ) -> None:
242 """Write a commit, then inflate the file beyond *limit* bytes."""
243 write_commit(root, c)
244 path = root / ".muse" / "commits" / f"{c.commit_id}.msgpack"
245 # Overwrite with limit+1 bytes of valid-looking (but unparseable) data.
246 path.write_bytes(b"\x00" * (limit + 1))
247
248 def test_read_commit_returns_none_for_oversized_file(
249 self, tmp_path: pathlib.Path
250 ) -> None:
251 root = _repo(tmp_path)
252 c = _commit(1)
253 with patch("muse.core.store.MAX_MSGPACK_BYTES", 100):
254 self._write_oversized_commit(root, c, 100)
255 result = read_commit(root, c.commit_id)
256 assert result is None, "read_commit must return None, not raise, for oversized file"
257
258 def test_read_snapshot_returns_none_for_oversized_file(
259 self, tmp_path: pathlib.Path
260 ) -> None:
261 root = _repo(tmp_path)
262 s = _snapshot(1)
263 write_snapshot(root, s)
264 snap_path = root / ".muse" / "snapshots" / f"{s.snapshot_id}.msgpack"
265 snap_path.write_bytes(b"\x00" * 101)
266 with patch("muse.core.store.MAX_MSGPACK_BYTES", 100):
267 result = read_snapshot(root, s.snapshot_id)
268 assert result is None
269
270 def test_get_all_tags_skips_oversized_files(
271 self, tmp_path: pathlib.Path
272 ) -> None:
273 """get_all_tags iterates all tag files — oversized ones are skipped."""
274 root = _repo(tmp_path)
275 good = _tag(0)
276 bad = _tag(1)
277 write_tag(root, good)
278 write_tag(root, bad)
279
280 # A real tag record is ~200 bytes packed (64-char IDs + timestamp).
281 # Choose a limit above a real tag but below our inflated bad file.
282 from muse.core.store import tag_path
283 good_path = tag_path(root, _REPO_ID, good.tag_id)
284 real_size = good_path.stat().st_size
285 test_limit = real_size * 2 # real tag fits; we'll inflate the bad tag to 3×
286
287 bad_path = tag_path(root, _REPO_ID, bad.tag_id)
288 bad_path.write_bytes(b"\x00" * (real_size * 3)) # definitely exceeds limit
289
290 with patch("muse.core.store.MAX_MSGPACK_BYTES", test_limit):
291 tags = get_all_tags(root, _REPO_ID)
292 tag_ids = {t.tag_id for t in tags}
293 assert good.tag_id in tag_ids, "Good tag was incorrectly dropped"
294 assert bad.tag_id not in tag_ids, "Oversized tag was not skipped"
295
296 def test_list_releases_skips_oversized_files(
297 self, tmp_path: pathlib.Path
298 ) -> None:
299 """list_releases must skip oversized release files."""
300 root = _repo(tmp_path)
301 from muse.core._types import split_id
302 r_algo, r_hex = split_id(_REPO_ID)
303 releases_dir = root / ".muse" / "releases" / r_algo / r_hex
304 releases_dir.mkdir(parents=True)
305 # Write a fake oversized release file.
306 fake_release = releases_dir / f"{'a' * 64}.msgpack"
307 fake_release.write_bytes(b"\x00" * 101)
308 with patch("muse.core.store.MAX_MSGPACK_BYTES", 100):
309 results = list_releases(root, _REPO_ID)
310 assert results == [], "Oversized release should be skipped, not crash"
311
312
313 # ---------------------------------------------------------------------------
314 # Tier 3 — exact boundary behaviour
315 # ---------------------------------------------------------------------------
316
317 class TestExactBoundary:
318 """At the boundary: MAX_MSGPACK_BYTES is the last allowed size."""
319
320 def test_file_exactly_at_limit_is_read(self, tmp_path: pathlib.Path) -> None:
321 """A file of exactly MAX_MSGPACK_BYTES bytes passes the size check.
322
323 The content may be unparseable (zeros are not valid msgpack), but the
324 OSError raised is a parse error, not a size-limit error.
325 """
326 test_limit = 256 # small limit for test speed
327 path = tmp_path / "exactly_at_limit.msgpack"
328 path.write_bytes(b"\x00" * test_limit)
329 with patch("muse.core.store.MAX_MSGPACK_BYTES", test_limit):
330 # Should raise a parse error (invalid msgpack), NOT an OSError about size.
331 from muse.core.store import _read_msgpack
332 try:
333 _read_msgpack(path)
334 pytest.fail("Expected an error for invalid msgpack content")
335 except OSError as exc:
336 assert "MiB read limit" not in str(exc), (
337 f"Got size-limit OSError at the boundary — should be parse error: {exc}"
338 )
339 except Exception:
340 pass # Any non-size-limit error is acceptable here
341
342 def test_file_one_byte_over_limit_raises_oslimit_error(
343 self, tmp_path: pathlib.Path
344 ) -> None:
345 """A file of MAX_MSGPACK_BYTES + 1 bytes raises OSError before reading."""
346 test_limit = 256
347 path = tmp_path / "one_over.msgpack"
348 path.write_bytes(b"\x00" * (test_limit + 1))
349 with patch("muse.core.store.MAX_MSGPACK_BYTES", test_limit):
350 from muse.core.store import _read_msgpack
351 with pytest.raises(OSError, match="read limit"):
352 _read_msgpack(path)
353
354 def test_zero_byte_file_does_not_trigger_size_limit(
355 self, tmp_path: pathlib.Path
356 ) -> None:
357 """An empty file passes the size check but fails msgpack parse."""
358 path = tmp_path / "empty.msgpack"
359 path.write_bytes(b"")
360 from muse.core.store import _read_msgpack
361 with pytest.raises(Exception): # parse error, not size error
362 _read_msgpack(path)
363
364 def test_size_limit_error_message_includes_filename_and_limit(
365 self, tmp_path: pathlib.Path
366 ) -> None:
367 """The OSError message must include the file name and limit in MiB."""
368 test_limit = 1024 # 1 KiB for test speed
369 path = tmp_path / "big.msgpack"
370 path.write_bytes(b"\x00" * (test_limit + 1))
371 with patch("muse.core.store.MAX_MSGPACK_BYTES", test_limit):
372 from muse.core.store import _read_msgpack
373 with pytest.raises(OSError) as exc_info:
374 _read_msgpack(path)
375 msg = str(exc_info.value)
376 assert "big.msgpack" in msg, f"Filename missing from error: {msg}"
377 assert "KiB" in msg or "MiB" in msg or "bytes" in msg, (
378 f"Size info missing from error: {msg}"
379 )
380
381
382 # ---------------------------------------------------------------------------
383 # Tier 4 — per-value unpack limits
384 # ---------------------------------------------------------------------------
385
386 class TestPerValueUnpackLimits:
387 """Verify that per-value limits from msgpack.unpackb are enforced."""
388
389 def _pack_to_path(self, tmp_path: pathlib.Path, data: MsgpackValue) -> pathlib.Path:
390 path = tmp_path / "test.msgpack"
391 path.write_bytes(msgpack.packb(data, use_bin_type=True))
392 return path
393
394 def test_string_exceeding_max_str_len_rejected(self, tmp_path: pathlib.Path) -> None:
395 """A string longer than _MSGPACK_MAX_STR_LEN must raise an exception."""
396 huge_str = "x" * 200
397 path = self._pack_to_path(tmp_path, {"key": huge_str})
398 from muse.core.store import _read_msgpack
399 with patch("muse.core.store._MSGPACK_MAX_STR_LEN", 100):
400 with pytest.raises(Exception):
401 _read_msgpack(path)
402
403 def test_string_within_max_str_len_accepted(self, tmp_path: pathlib.Path) -> None:
404 """A string within the limit unpacks normally."""
405 path = self._pack_to_path(tmp_path, {"key": "short"})
406 from muse.core.store import _read_msgpack
407 result = _read_msgpack(path)
408 assert isinstance(result, dict)
409
410 def test_binary_blob_rejected_in_store_records(self, tmp_path: pathlib.Path) -> None:
411 """Binary data (msgpack bin type) must be rejected for store records.
412
413 Commit/snapshot/tag records contain no binary fields. A file with
414 binary data is either corrupt or tampered. max_bin_len=0 ensures
415 this is caught immediately during unpack rather than producing a
416 ``bytes`` value that callers are not prepared to handle.
417 """
418 path = self._pack_to_path(tmp_path, {"body": b"some binary blob"})
419 from muse.core.store import _read_msgpack
420 # max_bin_len=0 means any bin-type value raises an error.
421 with pytest.raises(Exception):
422 _read_msgpack(path)
423
424 def test_map_exceeding_max_map_len_rejected(self, tmp_path: pathlib.Path) -> None:
425 """A map with more than _MSGPACK_MAX_MAP_LEN entries must raise."""
426 big_map: MsgpackDict = {str(i): i for i in range(200)}
427 path = self._pack_to_path(tmp_path, big_map)
428 from muse.core.store import _read_msgpack
429 with patch("muse.core.store._MSGPACK_MAX_MAP_LEN", 100):
430 with pytest.raises(Exception):
431 _read_msgpack(path)
432
433 def test_array_exceeding_max_array_len_rejected(self, tmp_path: pathlib.Path) -> None:
434 """An array with more than _MSGPACK_MAX_ARRAY_LEN entries must raise."""
435 big_list: list[MsgpackValue] = list(range(200))
436 path = self._pack_to_path(tmp_path, big_list)
437 from muse.core.store import _read_msgpack
438 with patch("muse.core.store._MSGPACK_MAX_ARRAY_LEN", 100):
439 with pytest.raises(Exception):
440 _read_msgpack(path)
441
442 def _make_deep_nested_msgpack(self, depth: int) -> bytes:
443 """Build msgpack bytes for a *depth*-deep nested dict without Python recursion.
444
445 ``msgpack.packb`` uses Python-level recursion so packing a 600-deep
446 dict hits the default recursion limit. We build the bytes directly:
447
448 fixmap(1) fixstr("x") fixmap(1) fixstr("x") ... fixmap(0)
449
450 Each level is 3 bytes: ``0x81`` (fixmap 1 entry) + ``0xa1 0x78``
451 (fixstr "x"). The leaf is ``0x80`` (fixmap 0 entries).
452
453 This produces a valid msgpack binary that ``unpackb`` will parse up
454 to its stack limit and then raise ``StackError``.
455 """
456 # 0x81 = fixmap with 1 item; 0xa1 0x78 = fixstr "x"
457 frame = b"\x81\xa1x"
458 leaf = b"\x80" # fixmap with 0 items
459 return frame * depth + leaf
460
461 def test_deeply_nested_map_raises_stack_error(self, tmp_path: pathlib.Path) -> None:
462 """A pathologically nested document hits msgpack's StackError.
463
464 At extreme depth (10 000 levels), msgpack's C-extension stack limit is
465 exceeded and an exception is raised. The file is only ~30 KiB so the
466 size check passes; the protection comes from msgpack's internal stack
467 guard, not the 64 MiB cap.
468 """
469 packed = self._make_deep_nested_msgpack(10_000)
470 path = tmp_path / "deep_nest.msgpack"
471 path.write_bytes(packed)
472 from muse.core.store import _read_msgpack
473 with pytest.raises(Exception): # msgpack.exceptions.StackError
474 _read_msgpack(path)
475
476 def test_deeply_nested_terminates_quickly(self, tmp_path: pathlib.Path) -> None:
477 """The StackError for deeply nested documents is raised in < 1 second."""
478 packed = self._make_deep_nested_msgpack(10_000)
479 path = tmp_path / "deep_nest_perf.msgpack"
480 path.write_bytes(packed)
481 from muse.core.store import _read_msgpack
482 start = time.perf_counter()
483 try:
484 _read_msgpack(path)
485 except Exception:
486 pass
487 elapsed = time.perf_counter() - start
488 assert elapsed < 1.0, (
489 f"Deeply nested document took {elapsed:.3f}s to fail — not fast enough"
490 )
491
492 def test_valid_large_map_within_limits_is_accepted(self, tmp_path: pathlib.Path) -> None:
493 """A large but within-limit map (simulating a 1k-file snapshot) unpacks cleanly."""
494 # Simulate a 1000-file snapshot manifest: {path: object_id}
495 manifest = {f"src/file_{i:04d}.py": _sha(f"obj-{i}") for i in range(1000)}
496 path = tmp_path / "big_valid.msgpack"
497 path.write_bytes(msgpack.packb(manifest, use_bin_type=True))
498 from muse.core.store import _read_msgpack
499 result = _read_msgpack(path)
500 assert isinstance(result, dict)
501 assert len(result) == 1000
502
503
504 # ---------------------------------------------------------------------------
505 # Tier 5 — index file protection
506 # ---------------------------------------------------------------------------
507
508 class TestIndexReadProtection:
509 """muse/core/indices.py has its own _read_msgpack — must also be protected."""
510
511 def test_load_symbol_history_skips_oversized_index(
512 self, tmp_path: pathlib.Path
513 ) -> None:
514 """An oversized symbol history index returns an empty dict, not OOM."""
515 (tmp_path / ".muse" / "indices").mkdir(parents=True)
516 index_path = tmp_path / ".muse" / "indices" / "symbol_history.msgpack"
517 index_path.write_bytes(b"\x00" * 101)
518 with patch("muse.core.indices._MAX_INDEX_BYTES", 100):
519 result = load_symbol_history(tmp_path)
520 assert result == {}, "Oversized index must return empty dict, not crash"
521
522 def test_load_hash_occurrence_skips_oversized_index(
523 self, tmp_path: pathlib.Path
524 ) -> None:
525 """An oversized hash_occurrence index returns an empty dict."""
526 (tmp_path / ".muse" / "indices").mkdir(parents=True)
527 index_path = tmp_path / ".muse" / "indices" / "hash_occurrence.msgpack"
528 index_path.write_bytes(b"\x00" * 101)
529 with patch("muse.core.indices._MAX_INDEX_BYTES", 100):
530 result = load_hash_occurrence(tmp_path)
531 assert result == {}
532
533 def test_index_size_limit_is_more_generous_than_store(self) -> None:
534 """Index files are allowed to be larger than store records."""
535 from muse.core.indices import _MAX_INDEX_BYTES
536 assert _MAX_INDEX_BYTES > MAX_MSGPACK_BYTES, (
537 "Index limit should be larger than store limit — indices grow with repo size"
538 )
539
540 def test_index_read_checks_stat_before_read_bytes(
541 self, tmp_path: pathlib.Path
542 ) -> None:
543 """The index stat check must fire before read_bytes (no allocation)."""
544 (tmp_path / ".muse" / "indices").mkdir(parents=True)
545 index_path = tmp_path / ".muse" / "indices" / "symbol_history.msgpack"
546 index_path.write_bytes(b"\x85") # 1 byte — well within any size limit
547 read_bytes_called = [False]
548 real_rb = index_path.read_bytes
549
550 def tracking_rb() -> bytes:
551 read_bytes_called[0] = True
552 return real_rb()
553
554 stat_result = MagicMock()
555 stat_result.st_size = 1024 * 1024 * 1024 # 1 GiB — way over limit
556
557 with patch.object(type(index_path), "stat", return_value=stat_result):
558 with patch.object(type(index_path), "read_bytes", tracking_rb):
559 result = load_symbol_history(tmp_path)
560
561 assert result == {}
562 assert not read_bytes_called[0], "read_bytes was called before the stat check!"
563
564
565 # ---------------------------------------------------------------------------
566 # Tier 6 — warning log on oversized file
567 # ---------------------------------------------------------------------------
568
569 class TestWarningLogOnOversizedFile:
570 """Operators need to know when oversized files are detected.
571
572 read_commit / read_snapshot log a WARNING when they catch the OSError
573 from _read_msgpack — this surfaces corruption or tampering in monitoring.
574 """
575
576 def test_warning_logged_for_oversized_commit(
577 self, tmp_path: pathlib.Path, caplog: pytest.LogCaptureFixture
578 ) -> None:
579 root = _repo(tmp_path)
580 c = _commit(10)
581 with patch("muse.core.store.MAX_MSGPACK_BYTES", 50):
582 path = commit_path(root, c.commit_id)
583 path.parent.mkdir(parents=True, exist_ok=True)
584 path.write_bytes(b"\x00" * 51)
585 with caplog.at_level(logging.WARNING, logger="muse.core.store"):
586 result = read_commit(root, c.commit_id)
587 assert result is None
588 # A warning must have been emitted for the corrupt/oversized file.
589 assert any(
590 "Corrupt" in rec.message or "corrupt" in rec.message
591 or "oversized" in rec.message or "limit" in rec.message.lower()
592 for rec in caplog.records
593 ), f"No warning logged for oversized commit. Records: {[r.message for r in caplog.records]}"
594
595 def test_warning_logged_for_oversized_snapshot(
596 self, tmp_path: pathlib.Path, caplog: pytest.LogCaptureFixture
597 ) -> None:
598 root = _repo(tmp_path)
599 s = _snapshot(10)
600 with patch("muse.core.store.MAX_MSGPACK_BYTES", 50):
601 path = snapshot_path(root, s.snapshot_id)
602 path.parent.mkdir(parents=True, exist_ok=True)
603 path.write_bytes(b"\x00" * 51)
604 with caplog.at_level(logging.WARNING, logger="muse.core.store"):
605 result = read_snapshot(root, s.snapshot_id)
606 assert result is None
607 assert any(
608 "Corrupt" in rec.message or "corrupt" in rec.message
609 for rec in caplog.records
610 ), f"No warning logged. Records: {[r.message for r in caplog.records]}"
611
612
613 # ---------------------------------------------------------------------------
614 # Tier 7 — CLI: clean JSON error, no traceback
615 # ---------------------------------------------------------------------------
616
617 class TestPlumbingReadCommitOversized:
618 """muse read-commit with an oversized commit file must produce
619 a clean, machine-readable JSON error — no Python traceback, no process crash.
620 """
621
622 def test_oversized_commit_produces_json_error_not_traceback(
623 self, tmp_path: pathlib.Path
624 ) -> None:
625 """write a commit, corrupt its file, run read-commit — must get JSON error."""
626 import json
627 import sys
628 from tests.cli_test_helper import CliRunner
629
630 root = _repo(tmp_path)
631 c = _commit(99)
632 write_commit(root, c)
633
634 # Corrupt the commit file to exceed the limit.
635 commit_path_file = commit_path(root, c.commit_id)
636 commit_path_file.write_bytes(b"\x00" * 101)
637
638 runner = CliRunner()
639 with patch("muse.core.store.MAX_MSGPACK_BYTES", 100):
640 result = runner.invoke(None, ["read-commit", c.commit_id],
641 env={"MUSE_REPO_ROOT": str(root)})
642
643 # Must not crash (exit code may be non-zero, but not a Python traceback).
644 assert "Traceback" not in (result.output or ""), (
645 f"CLI produced a Python traceback for oversized commit:\n{result.output}"
646 )
647 assert "Traceback" not in (result.stderr or ""), (
648 f"CLI stderr has a Python traceback:\n{result.stderr}"
649 )
650 # The error output must be valid JSON (or include a meaningful error).
651 combined = (result.output or "") + (result.stderr or "")
652 try:
653 # Check if any JSON blob exists in the output.
654 for line in combined.splitlines():
655 line = line.strip()
656 if line.startswith("{"):
657 parsed = json.loads(line)
658 assert "error" in parsed, f"JSON lacks 'error' key: {parsed}"
659 break
660 else:
661 # If no JSON line found, at minimum confirm no traceback and
662 # that "not found" or "error" appears in the output.
663 assert (
664 "not found" in combined.lower()
665 or "error" in combined.lower()
666 ), f"No useful error in CLI output:\n{combined}"
667 except json.JSONDecodeError as exc:
668 pytest.fail(f"Output is not valid JSON: {exc}\nOutput:\n{combined}")
669
670
671 # ---------------------------------------------------------------------------
672 # Tier 8 — round-trip: valid files still read correctly
673 # ---------------------------------------------------------------------------
674
675 class TestValidFilesUnaffected:
676 """The size guard must not regress normal reads."""
677
678 def test_read_commit_roundtrip_unaffected(self, tmp_path: pathlib.Path) -> None:
679 root = _repo(tmp_path)
680 c = _commit(42)
681 write_commit(root, c)
682 got = read_commit(root, c.commit_id)
683 assert got is not None
684 assert got.commit_id == c.commit_id
685 assert got.message == c.message
686
687 def test_read_snapshot_roundtrip_unaffected(self, tmp_path: pathlib.Path) -> None:
688 root = _repo(tmp_path)
689 s = _snapshot(42)
690 write_snapshot(root, s)
691 got = read_snapshot(root, s.snapshot_id)
692 assert got is not None
693 assert got.snapshot_id == s.snapshot_id
694
695 def test_snapshot_with_large_manifest_reads_correctly(
696 self, tmp_path: pathlib.Path
697 ) -> None:
698 """A 1000-file snapshot manifest (realistic scale) reads without issue."""
699 root = _repo(tmp_path)
700 manifest = {f"src/file_{i:05d}.py": _sha(f"obj-{i}") for i in range(1000)}
701 sid = compute_snapshot_id(manifest)
702 s = SnapshotRecord(
703 snapshot_id=sid,
704 manifest=manifest,
705 )
706 write_snapshot(root, s)
707 got = read_snapshot(root, sid)
708 assert got is not None
709 assert len(got.manifest) == 1000
710
711 def test_commit_with_long_message_reads_correctly(
712 self, tmp_path: pathlib.Path
713 ) -> None:
714 """A commit with a 64 KiB message reads correctly (well within 1 MiB str limit)."""
715 root = _repo(tmp_path)
716 long_msg = "a" * 65536
717 committed_at = datetime.datetime(2026, 1, 1, tzinfo=datetime.timezone.utc)
718 snapshot_id = compute_snapshot_id({})
719 cid = compute_commit_id(
720 repo_id=_REPO_ID,
721 parent_ids=[],
722 snapshot_id=snapshot_id,
723 message=long_msg,
724 committed_at_iso=committed_at.isoformat(),
725 author="tester",)
726 c = CommitRecord(
727 commit_id=cid,
728 repo_id=_REPO_ID,
729 created_on_branch="main",
730 snapshot_id=snapshot_id,
731 message=long_msg,
732 committed_at=committed_at,
733 author="tester",
734 parent_commit_id=None,
735 parent2_commit_id=None,
736 )
737 write_commit(root, c)
738 got = read_commit(root, cid)
739 assert got is not None
740 assert len(got.message) == 65536
741
742
743 # ---------------------------------------------------------------------------
744 # Tier 9 — performance: size check adds < 1 ms per read
745 # ---------------------------------------------------------------------------
746
747 class TestSizeCheckPerformance:
748 """The stat() check should add negligible overhead to normal reads."""
749
750 @pytest.mark.perf
751 def test_stat_check_overhead_under_1ms_per_read(
752 self, tmp_path: pathlib.Path
753 ) -> None:
754 """100 sequential read_commit calls with the size guard active < 100ms total."""
755 root = _repo(tmp_path)
756 commits = [_commit(i) for i in range(100)]
757 for c in commits:
758 write_commit(root, c)
759
760 start = time.perf_counter()
761 for c in commits:
762 result = read_commit(root, c.commit_id)
763 assert result is not None
764 elapsed = time.perf_counter() - start
765
766 assert elapsed < 0.1, (
767 f"100 read_commit calls took {elapsed:.3f}s — "
768 "size check is adding too much overhead (< 100ms expected)"
769 )
770
771 @pytest.mark.perf
772 def test_oversized_rejection_under_1ms(self, tmp_path: pathlib.Path) -> None:
773 """Rejecting an oversized file (via stat) takes < 1ms — no disk I/O."""
774 root = _repo(tmp_path)
775 c = _commit(200)
776 write_commit(root, c)
777 path = root / ".muse" / "commits" / f"{c.commit_id}.msgpack"
778 path.write_bytes(b"\x00" * 101)
779
780 start = time.perf_counter()
781 with patch("muse.core.store.MAX_MSGPACK_BYTES", 100):
782 for _ in range(1000):
783 read_commit(root, c.commit_id)
784 elapsed = time.perf_counter() - start
785
786 assert elapsed < 1.0, (
787 f"1000 oversized-rejection calls took {elapsed:.3f}s (> 1ms each)"
788 )
File History 3 commits
sha256:b636f72dcba9e190afb980bece906fa5b717fbde014b76ef023df8cb96e01eb9 docs: expand cache plan with all seven testing tiers and do… Sonnet 4.6 131 days ago
sha256:7f9e2ef5286aedad9c1e6011b4c46ca27f39dbdad6e3409357e36b26e46b3b7c docs: docstring sprint for-each-ref→hotspots — idiomatic ru… Sonnet 4.6 patch 138 days ago
sha256:a09b1b4f6838754495547f200aa0ce88e2f56ffc5b20b900f6f0cff2c3cdede9 fix(cursorignore): remove git-ism (.git/worktrees) Human minor ⚠ 140 days ago