gabriel / muse public
test_perf_phase3.py python
1,219 lines 42.0 KB
Raw
sha256:b636f72dcba9e190afb980bece906fa5b717fbde014b76ef023df8cb96e01eb9 docs: expand cache plan with all seven testing tiers and do… Sonnet 4.6 131 days ago
1 """Phase 3 — Performance regression tests.
2
3 Target metrics (measured on a 2024 MacBook Pro M4, macOS 15):
4
5 Phase 3.1 — Linux-kernel commit throughput
6 write_commit: ≥ 1 000 commits/sec
7 write_object: ≥ 2 000 objects/sec
8 build_snapshot_manifest: ≥ 10 000 files/sec
9 muse commit (e2e, 1 000-file workdir): < 5 000 ms
10
11 Phase 3.2 — Concurrent agent write storm
12 200 threads × write_object: all objects readable, no corruption
13 200 threads × write_commit: all commits readable, no corruption
14 100 threads × write_head_commit: last-write-wins, valid ID written
15
16 Phase 3.3 — Memory ceiling
17 write_commit (10 000 commits): peak RSS < 512 MiB
18 build_snapshot_manifest (5 000 files): peak RSS < 128 MiB
19
20 Phase 3.3 extended — Linux-scale memory ceiling (100k / 75k)
21 get_all_commits (100 000 commits): peak RSS < 2 GiB [@slow]
22 get_commits_for_branch walk-cap: max_walk_commits bounds RSS
23 muse log --json pseudo-streaming: no double-buffer
24 build_snapshot_manifest (75 000 files): peak RSS < 512 MiB [@slow]
25 find_merge_base (deep chain): cap fires, no OOM
26 walk_commits_between: silent truncation at cap, not OOM
27
28 All slow tests are decorated with ``@pytest.mark.slow`` and are skipped by
29 default. Run the full suite with ``pytest tests/test_perf_phase3.py -v``.
30 """
31
32 from __future__ import annotations
33
34 type _FileStore = dict[str, bytes]
35
36 import datetime
37 import os
38 import pathlib
39 import resource
40 import sys
41 import threading
42 import time
43 import tracemalloc
44
45 import pytest
46 from unittest.mock import patch
47
48 from muse.core.object_store import (
49 _created_object_shards,
50 has_object,
51 object_path,
52 read_object,
53 write_object,
54 )
55 from muse.core.merge_engine import find_merge_base
56 from muse.core.snapshot import build_snapshot_manifest, compute_commit_id
57 from muse.core.store import (
58 CommitRecord,
59 get_all_commits,
60 get_commits_for_branch,
61 read_commit,
62 walk_commits_between_result,
63 write_branch_ref,
64 write_commit,
65 write_head_commit,
66 )
67
68 # ---------------------------------------------------------------------------
69 # Helpers
70 # ---------------------------------------------------------------------------
71
72
73 from muse.core._types import blob_id, split_id
74
75
76 def _sha256(data: bytes) -> str:
77 return blob_id(data)
78
79
80 def _repo(tmp_path: pathlib.Path) -> pathlib.Path:
81 muse_dir = tmp_path / ".muse"
82 muse_dir.mkdir()
83 (muse_dir / "repo.json").write_text('{"repo_id": "bench", "owner": "bench"}')
84 (muse_dir / "commits").mkdir()
85 (muse_dir / "snapshots").mkdir()
86 (muse_dir / "refs" / "heads").mkdir(parents=True)
87 (muse_dir / "HEAD").write_text("ref: refs/heads/main\n")
88 return tmp_path
89
90
91 def _write_chain(
92 repo: pathlib.Path,
93 branch: str,
94 n: int,
95 snap_id: str = "f" * 64,
96 start: int = 0,
97 ) -> str:
98 """Write a linear commit chain of length *n* and return the tip commit ID.
99
100 Sets the branch ref to the tip so ``get_commits_for_branch`` can walk it.
101 """
102 parent: str | None = None
103 tip = ""
104 for i in range(start, start + n):
105 msg = f"chain-{i:07d}"
106 ts = datetime.datetime(2026, 1, 1, i % 3600 // 3600, i % 3600 % 60, tzinfo=datetime.timezone.utc)
107 cid = compute_commit_id(
108 repo_id="bench",
109 parent_ids=[parent] if parent else [],
110 snapshot_id=snap_id,
111 message=msg,
112 committed_at_iso=ts.isoformat(),
113 author="chain-agent",)
114 rec = CommitRecord(
115 commit_id=cid,
116 repo_id="bench",
117 created_on_branch=branch,
118 snapshot_id=snap_id,
119 message=msg,
120 committed_at=ts,
121 parent_commit_id=parent,
122 parent2_commit_id=None,
123 author="chain-agent",
124 metadata={},
125 structured_delta=None,
126 sem_ver_bump="none",
127 breaking_changes=[],
128 agent_id="",
129 model_id="",
130 toolchain_id="",
131 prompt_hash="",
132 signature="",
133 signer_key_id="",
134 )
135 write_commit(repo, rec)
136 parent = cid
137 tip = cid
138 write_branch_ref(repo, branch, tip)
139 return tip
140
141
142 def _make_commit(index: int, snap_id: str, parent: str | None = None) -> CommitRecord:
143 """Build a CommitRecord whose ``commit_id`` passes content-hash verification."""
144 ts = datetime.datetime(2026, 1, 1, tzinfo=datetime.timezone.utc)
145 msg = f"commit-{index:07d}"
146 cid = compute_commit_id(
147 repo_id="bench",
148 parent_ids=[parent] if parent else [],
149 snapshot_id=snap_id,
150 message=msg,
151 committed_at_iso=ts.isoformat(),
152 author="perf-agent",)
153 return CommitRecord(
154 commit_id=cid,
155 repo_id="bench",
156 created_on_branch="main",
157 snapshot_id=snap_id,
158 message=msg,
159 committed_at=ts,
160 parent_commit_id=parent,
161 parent2_commit_id=None,
162 author="perf-agent",
163 metadata={},
164 structured_delta=None,
165 sem_ver_bump="none",
166 breaking_changes=[],
167 agent_id="",
168 model_id="",
169 toolchain_id="",
170 prompt_hash="",
171 signature="",
172 signer_key_id="",
173 )
174
175
176 # ---------------------------------------------------------------------------
177 # Phase 3.1 — Linux-kernel commit throughput
178 # ---------------------------------------------------------------------------
179
180
181 class TestWriteCommitThroughput:
182 """write_commit must sustain ≥ 1 000 commits/sec in isolation.
183
184 The Linux-kernel migration target is 472 commits/sec over 850 000 commits
185 (< 30 min). A 1 000 commits/sec floor gives comfortable headroom for
186 real workload overhead (snapshot building, object writes, disk pressure).
187
188 fsync is mocked: these tests measure msgpack serialisation + filesystem
189 metadata throughput, not OS I/O durability. Durability ordering is
190 verified by test_integrity_I2_fsync.py.
191 """
192
193 _MIN_COMMITS_PER_SEC = 1_000
194
195 @pytest.fixture(autouse=True)
196 def no_fsync(self) -> None:
197 """Mock out all fsync calls so the test measures algorithmic throughput."""
198 with patch("muse.core.store.os.fsync", return_value=None), \
199 patch("muse.core.store.fcntl.fcntl", return_value=0):
200 yield
201
202 @pytest.mark.slow
203 def test_write_commit_throughput_10k(self, tmp_path: pathlib.Path) -> None:
204 """Write 10 000 commits and assert throughput ≥ 1 000 commits/sec."""
205 repo = _repo(tmp_path)
206 snap_id = "a" * 64
207 N = 10_000
208 commits = [_make_commit(i, snap_id) for i in range(N)]
209
210 t0 = time.perf_counter()
211 for rec in commits:
212 write_commit(repo, rec)
213 elapsed = time.perf_counter() - t0
214 rate = N / elapsed
215
216 assert rate >= self._MIN_COMMITS_PER_SEC, (
217 f"write_commit throughput {rate:.0f} commits/sec is below the "
218 f"minimum {self._MIN_COMMITS_PER_SEC} commits/sec. "
219 f"(10k commits took {elapsed:.2f}s. "
220 f"Linux-kernel migration target: 472 commits/sec.)"
221 )
222
223 def test_write_commit_throughput_1k_fast(self, tmp_path: pathlib.Path) -> None:
224 """Smoke-speed: 1 000 commits must complete within 5 seconds."""
225 repo = _repo(tmp_path)
226 snap_id = "b" * 64
227 N = 1_000
228 commits = [_make_commit(i, snap_id) for i in range(N)]
229
230 t0 = time.perf_counter()
231 for rec in commits:
232 write_commit(repo, rec)
233 elapsed = time.perf_counter() - t0
234
235 assert elapsed <= 15.0, (
236 f"1 000 write_commit calls took {elapsed:.2f}s — expected ≤ 15.0s."
237 )
238
239
240 class TestWriteObjectThroughput:
241 """write_object must sustain ≥ 1 500 objects/sec in isolation.
242
243 fsync is mocked: the test measures mkstemp + hash-verify + fchmod +
244 os.replace throughput without OS I/O latency. Durability ordering is
245 verified by test_integrity_I2_fsync.py.
246 """
247
248 _MIN_OBJECTS_PER_SEC = 1_500
249
250 @pytest.fixture(autouse=True)
251 def no_fsync(self) -> None:
252 """Mock out all fsync calls so the test measures algorithmic throughput."""
253 with patch("muse.core.object_store._fsync_fd", return_value=None):
254 yield
255
256 @pytest.mark.slow
257 def test_write_object_throughput_10k(self, tmp_path: pathlib.Path) -> None:
258 """Write 10 000 4-KiB objects and assert throughput ≥ 2 000 objects/sec."""
259 repo = _repo(tmp_path)
260 N = 10_000
261 items = [
262 (
263 _sha256(f"perf-obj-{i:08d}".encode() * 16),
264 f"perf-obj-{i:08d}".encode() * 16,
265 )
266 for i in range(N)
267 ]
268
269 t0 = time.perf_counter()
270 for oid, content in items:
271 write_object(repo, oid, content)
272 elapsed = time.perf_counter() - t0
273 rate = N / elapsed
274
275 assert rate >= self._MIN_OBJECTS_PER_SEC, (
276 f"write_object throughput {rate:.0f} objects/sec is below the "
277 f"minimum {self._MIN_OBJECTS_PER_SEC} objects/sec. "
278 f"(10k objects took {elapsed:.2f}s.)"
279 )
280
281 @pytest.mark.perf
282 def test_write_object_throughput_1k_fast(self, tmp_path: pathlib.Path) -> None:
283 """Smoke-speed: 1 000 objects must complete within 2 seconds."""
284 repo = _repo(tmp_path)
285 N = 1_000
286 items = [
287 (
288 _sha256(f"fast-obj-{i:08d}".encode() * 8),
289 f"fast-obj-{i:08d}".encode() * 8,
290 )
291 for i in range(N)
292 ]
293
294 t0 = time.perf_counter()
295 for oid, content in items:
296 write_object(repo, oid, content)
297 elapsed = time.perf_counter() - t0
298
299 assert elapsed <= 2.0, (
300 f"1 000 write_object calls took {elapsed:.2f}s — expected ≤ 2.0s."
301 )
302
303
304 class TestSnapshotManifestThroughput:
305 """build_snapshot_manifest must sustain ≥ 10 000 files/sec."""
306
307 _MIN_FILES_PER_SEC = 10_000
308
309 @pytest.mark.slow
310 def test_snapshot_5k_files(self, tmp_path: pathlib.Path) -> None:
311 """Build manifest of 5 000 files; assert ≥ 10 000 files/sec."""
312 root = tmp_path / "workdir"
313 root.mkdir()
314 (root / ".muse").mkdir()
315 (root / ".muse" / "repo.json").write_text('{"repo_id": "bench"}')
316
317 # 50 dirs × 100 files = 5 000 files
318 for d in range(50):
319 dp = root / f"pkg_{d:03d}"
320 dp.mkdir()
321 for f in range(100):
322 (dp / f"file_{f:03d}.py").write_bytes(
323 f"# content-{d}-{f}\n".encode() * 50
324 )
325
326 t0 = time.perf_counter()
327 manifest = build_snapshot_manifest(root)
328 elapsed = time.perf_counter() - t0
329 rate = len(manifest) / elapsed
330
331 assert len(manifest) == 5_000
332 assert rate >= self._MIN_FILES_PER_SEC, (
333 f"build_snapshot_manifest throughput {rate:.0f} files/sec is below "
334 f"the minimum {self._MIN_FILES_PER_SEC} files/sec. "
335 f"(5k files took {elapsed:.3f}s.)"
336 )
337
338 def test_snapshot_500_files_fast(self, tmp_path: pathlib.Path) -> None:
339 """Smoke-speed: 500-file manifest must complete within 500 ms."""
340 root = tmp_path / "workdir"
341 root.mkdir()
342 (root / ".muse").mkdir()
343 (root / ".muse" / "repo.json").write_text('{"repo_id": "bench"}')
344
345 for d in range(25):
346 dp = root / f"pkg_{d:02d}"
347 dp.mkdir()
348 for f in range(20):
349 (dp / f"file_{f:02d}.py").write_bytes(b"x" * 200)
350
351 t0 = time.perf_counter()
352 manifest = build_snapshot_manifest(root)
353 elapsed = time.perf_counter() - t0
354
355 assert len(manifest) == 500
356 assert elapsed <= 0.5, (
357 f"500-file manifest took {elapsed*1000:.1f} ms — expected ≤ 500 ms."
358 )
359
360
361 # ---------------------------------------------------------------------------
362 # Phase 3.2 — Concurrent agent write storm
363 # ---------------------------------------------------------------------------
364
365
366 class TestConcurrentWriteObjectStorm:
367 """200 threads writing distinct objects — no corruption, no data loss."""
368
369 def test_200_threads_write_distinct_objects(self, tmp_path: pathlib.Path) -> None:
370 """200 threads each write 50 distinct objects; all must be readable after join."""
371 repo = _repo(tmp_path)
372 N_THREADS = 200
373 N_PER_THREAD = 50
374 written: _FileStore = {}
375 lock = threading.Lock()
376 errors: list[str] = []
377
378 # Pre-compute all objects to avoid per-thread hashing noise.
379 all_items: list[list[tuple[str, bytes]]] = []
380 for t in range(N_THREADS):
381 thread_items: list[tuple[str, bytes]] = []
382 for i in range(N_PER_THREAD):
383 content = f"thread-{t:03d}-obj-{i:03d}".encode() * 4
384 oid = _sha256(content)
385 thread_items.append((oid, content))
386 with lock:
387 written[oid] = content
388 all_items.append(thread_items)
389
390 def writer(items: list[tuple[str, bytes]]) -> None:
391 try:
392 for oid, content in items:
393 write_object(repo, oid, content)
394 except Exception as exc:
395 with lock:
396 errors.append(str(exc))
397
398 threads = [
399 threading.Thread(target=writer, args=(all_items[t],))
400 for t in range(N_THREADS)
401 ]
402 for th in threads:
403 th.start()
404 for th in threads:
405 th.join(timeout=30.0)
406
407 assert not errors, f"Write errors during concurrent storm: {errors[:3]}"
408
409 # Verify every object is readable and byte-identical.
410 missing: list[str] = []
411 corrupt: list[str] = []
412 for oid, expected in written.items():
413 if not has_object(repo, oid):
414 missing.append(oid[:8])
415 else:
416 actual = read_object(repo, oid)
417 if actual != expected:
418 corrupt.append(oid[:8])
419
420 assert not missing, f"{len(missing)} objects missing after concurrent write storm"
421 assert not corrupt, f"{len(corrupt)} objects corrupted after concurrent write storm"
422
423
424 class TestConcurrentWriteCommitStorm:
425 """200 threads writing distinct commits — all must be readable after join."""
426
427 def test_200_threads_write_distinct_commits(self, tmp_path: pathlib.Path) -> None:
428 """200 threads each write 25 distinct commits; all must be readable."""
429 repo = _repo(tmp_path)
430 N_THREADS = 200
431 N_PER_THREAD = 25
432 snap_id = "c" * 64
433 all_records: list[list[CommitRecord]] = []
434 errors: list[str] = []
435 lock = threading.Lock()
436
437 for t in range(N_THREADS):
438 thread_recs: list[CommitRecord] = []
439 for i in range(N_PER_THREAD):
440 rec = _make_commit(t * N_PER_THREAD + i, snap_id)
441 thread_recs.append(rec)
442 all_records.append(thread_recs)
443
444 def writer(recs: list[CommitRecord]) -> None:
445 try:
446 for rec in recs:
447 write_commit(repo, rec)
448 except Exception as exc:
449 with lock:
450 errors.append(str(exc))
451
452 threads = [
453 threading.Thread(target=writer, args=(all_records[t],))
454 for t in range(N_THREADS)
455 ]
456 for th in threads:
457 th.start()
458 for th in threads:
459 th.join(timeout=30.0)
460
461 assert not errors, f"Write errors during concurrent commit storm: {errors[:3]}"
462
463 # All commits must be readable.
464 total = N_THREADS * N_PER_THREAD
465 missing: list[str] = []
466 for recs in all_records:
467 for rec in recs:
468 result = read_commit(repo, rec.commit_id)
469 if result is None:
470 missing.append(rec.commit_id[:8])
471
472 assert not missing, (
473 f"{len(missing)}/{total} commits missing after concurrent write storm"
474 )
475
476
477 class TestConcurrentWriteBranchRef:
478 """100 threads racing on write_branch_ref — last write wins, no corruption."""
479
480 def test_100_threads_write_branch_ref_last_write_wins(
481 self, tmp_path: pathlib.Path
482 ) -> None:
483 """100 threads each writing write_branch_ref; result must be a valid 64-char hex ID."""
484 repo = _repo(tmp_path)
485 refs_dir = repo / ".muse" / "refs" / "heads" # already created by _repo()
486
487 N = 100
488 snap_id = "d" * 64
489 commit_ids: list[str] = []
490 errors: list[str] = []
491 lock = threading.Lock()
492
493 for i in range(N):
494 cid = compute_commit_id(repo_id="bench", parent_ids=[], snapshot_id=snap_id, message=f"head-{i}", committed_at_iso="2026-01-01T00:00:00+00:00")
495 commit_ids.append(cid)
496
497 def writer(cid: str) -> None:
498 try:
499 write_branch_ref(repo, "main", cid)
500 except Exception as exc:
501 with lock:
502 errors.append(str(exc))
503
504 threads = [threading.Thread(target=writer, args=(commit_ids[i],)) for i in range(N)]
505 for th in threads:
506 th.start()
507 for th in threads:
508 th.join(timeout=10.0)
509
510 assert not errors, f"Errors during concurrent write_branch_ref: {errors[:3]}"
511
512 # The branch ref must contain exactly one of the written IDs.
513 ref_file = refs_dir / "main"
514 assert ref_file.exists(), "Branch ref file was lost after concurrent writes"
515 final_cid = ref_file.read_text(encoding="utf-8").strip()
516 # Commit IDs are now "sha256:<64hex>" = 71 chars
517 assert final_cid.startswith("sha256:"), f"Branch ref has unexpected format: {final_cid!r}"
518 _, hex_part = split_id(final_cid)
519 assert len(hex_part) == 64, f"Branch ref hex part has unexpected length: {final_cid!r}"
520 assert all(c in "0123456789abcdef" for c in hex_part), (
521 f"Branch ref is not a valid hex ID: {final_cid!r}"
522 )
523 assert final_cid in commit_ids, (
524 f"Branch ref {final_cid[:19]} is not one of the written IDs"
525 )
526
527
528 # ---------------------------------------------------------------------------
529 # Phase 3.3 — Memory ceiling under load
530 # ---------------------------------------------------------------------------
531
532
533 class TestMemoryCeiling:
534 """Peak memory must not grow proportionally to the number of objects or commits."""
535
536 _MAX_WRITE_COMMIT_MIB = 512
537 _MAX_SNAPSHOT_MIB = 128
538
539 @pytest.mark.slow
540 def test_write_10k_commits_peak_rss_under_512_mib(
541 self, tmp_path: pathlib.Path
542 ) -> None:
543 """Writing 10 000 commits must not exceed 512 MiB peak RSS.
544
545 write_commit buffers one commit at a time; it must not accumulate
546 a list of all commits in memory.
547 """
548 repo = _repo(tmp_path)
549 snap_id = "e" * 64
550 N = 10_000
551
552 tracemalloc.start()
553 tracemalloc.clear_traces()
554
555 for i in range(N):
556 rec = _make_commit(i, snap_id)
557 write_commit(repo, rec)
558
559 _, peak_bytes = tracemalloc.get_traced_memory()
560 tracemalloc.stop()
561
562 peak_mib = peak_bytes / (1024 * 1024)
563 assert peak_mib <= self._MAX_WRITE_COMMIT_MIB, (
564 f"write_commit peak allocation {peak_mib:.1f} MiB exceeds "
565 f"{self._MAX_WRITE_COMMIT_MIB} MiB for 10k commits. "
566 "write_commit must stream one record at a time."
567 )
568
569 def test_snapshot_5k_files_peak_rss_under_128_mib(
570 self, tmp_path: pathlib.Path
571 ) -> None:
572 """build_snapshot_manifest on 5 000 small files stays under 128 MiB."""
573 root = tmp_path / "workdir"
574 root.mkdir()
575 (root / ".muse").mkdir()
576 (root / ".muse" / "repo.json").write_text('{"repo_id": "bench"}')
577
578 for d in range(50):
579 dp = root / f"d_{d:02d}"
580 dp.mkdir()
581 for f in range(100):
582 (dp / f"f_{f:02d}.txt").write_bytes(b"x" * 512)
583
584 tracemalloc.start()
585 tracemalloc.clear_traces()
586 build_snapshot_manifest(root)
587 _, peak_bytes = tracemalloc.get_traced_memory()
588 tracemalloc.stop()
589
590 peak_mib = peak_bytes / (1024 * 1024)
591 assert peak_mib <= self._MAX_SNAPSHOT_MIB, (
592 f"build_snapshot_manifest peak allocation {peak_mib:.1f} MiB "
593 f"exceeds {self._MAX_SNAPSHOT_MIB} MiB for 5k files. "
594 "The manifest dict must not accumulate large intermediate buffers."
595 )
596
597
598 # ---------------------------------------------------------------------------
599 # Phase 3.1 — Shard-cache amortisation (regression guard)
600 # ---------------------------------------------------------------------------
601
602
603 class TestShardCacheAmortisation:
604 """The shard-validation cache eliminates O(objects) resolve() calls.
605
606 This test is not a timing test — it verifies the structural invariant:
607 after N writes to the same shard, _created_object_shards contains exactly
608 that shard entry and subsequent writes to the same shard do not re-trigger
609 path-resolution (proved by exercising the idempotent write path).
610 """
611
612 def test_shard_cache_populated_after_first_write(
613 self, tmp_path: pathlib.Path
614 ) -> None:
615 """After the first write to a shard, _created_object_shards contains its path."""
616 repo = _repo(tmp_path)
617 content = b"shard-cache-test"
618 oid = _sha256(content)
619 shard_str = str(object_path(repo, oid).parent)
620
621 # Ensure cache starts without this shard.
622 _created_object_shards.discard(shard_str)
623
624 write_object(repo, oid, content)
625
626 assert shard_str in _created_object_shards, (
627 f"Shard {oid[:2]} not recorded in _created_object_shards after first write. "
628 "The amortisation optimisation is not active."
629 )
630
631 def test_repeated_writes_to_same_shard_succeed(
632 self, tmp_path: pathlib.Path
633 ) -> None:
634 """50 distinct writes to the same shard all land correctly."""
635 repo = _repo(tmp_path)
636 # Force a fixed shard prefix by constructing objects with the same prefix.
637 # We use a known prefix and craft content that happens to hash to it.
638 # Easier: just write 50 objects and verify they all land.
639 N = 50
640 items = [
641 (
642 _sha256(f"repeat-shard-{i:04d}".encode()),
643 f"repeat-shard-{i:04d}".encode(),
644 )
645 for i in range(N)
646 ]
647
648 for oid, content in items:
649 write_object(repo, oid, content)
650
651 missing = [oid[:8] for oid, _ in items if not has_object(repo, oid)]
652 assert not missing, (
653 f"{len(missing)} objects missing after repeated writes: {missing[:5]}"
654 )
655
656 def test_idempotent_write_bypasses_shard_write(
657 self, tmp_path: pathlib.Path
658 ) -> None:
659 """write_object returns False (idempotent) on the second call for the same OID."""
660 repo = _repo(tmp_path)
661 content = b"idempotent-check"
662 oid = _sha256(content)
663
664 first = write_object(repo, oid, content)
665 second = write_object(repo, oid, content)
666
667 assert first is True, "First write should return True"
668 assert second is False, "Second write should return False (already exists)"
669
670
671 # ---------------------------------------------------------------------------
672 # Phase 3.3 extended — Linux-scale memory ceiling
673 # ---------------------------------------------------------------------------
674
675
676 class TestGetAllCommitsMemory:
677 """get_all_commits must not OOM under Linux-scale commit counts.
678
679 This is the highest-risk accumulator: it loads *every* CommitRecord in the
680 store into a list simultaneously with no cap. At 100k commits × ~2 KB per
681 serialised record the baseline is ~200 MiB. Commits with large
682 ``structured_delta`` payloads can be 100 KB each (100k × 100 KB = 10 GiB).
683 The 64 MiB per-record msgpack cap (MAX_MSGPACK_BYTES) provides the guard.
684
685 The @slow variants build real on-disk commit chains; the fast variant uses
686 a small chain to confirm the structural property with tracemalloc.
687 """
688
689 def test_get_all_commits_1k_peak_rss_under_128_mib(
690 self, tmp_path: pathlib.Path
691 ) -> None:
692 """get_all_commits on 1 000 commits stays under 128 MiB (fast smoke)."""
693 repo = _repo(tmp_path)
694 N = 1_000
695 snap_id = "aa" * 32
696 for i in range(N):
697 write_commit(repo, _make_commit(i, snap_id))
698
699 tracemalloc.start()
700 tracemalloc.clear_traces()
701 results = get_all_commits(repo)
702 _, peak_bytes = tracemalloc.get_traced_memory()
703 tracemalloc.stop()
704
705 assert len(results) == N, f"Expected {N} commits, got {len(results)}"
706 peak_mib = peak_bytes / (1024 * 1024)
707 assert peak_mib <= 128, (
708 f"get_all_commits({N}) peak {peak_mib:.1f} MiB — expected ≤ 128 MiB. "
709 "CommitRecord size has grown; re-audit the dataclass fields."
710 )
711
712 @pytest.mark.slow
713 def test_get_all_commits_100k_under_2_gib(
714 self, tmp_path: pathlib.Path
715 ) -> None:
716 """get_all_commits on 100 000 commits stays under 2 GiB.
717
718 100k × minimal CommitRecord ≈ 200 MiB. The 2 GiB ceiling allows a 10×
719 margin for realistic payloads (metadata, structured_delta) while still
720 catching runaway accumulation.
721 """
722 repo = _repo(tmp_path)
723 N = 100_000
724 snap_id = "bb" * 32
725
726 for i in range(N):
727 write_commit(repo, _make_commit(i, snap_id))
728
729 _MAX_MIB = 2_048 # 2 GiB
730
731 tracemalloc.start()
732 tracemalloc.clear_traces()
733 results = get_all_commits(repo)
734 _, peak_bytes = tracemalloc.get_traced_memory()
735 tracemalloc.stop()
736
737 assert len(results) == N
738 peak_mib = peak_bytes / (1024 * 1024)
739 assert peak_mib <= _MAX_MIB, (
740 f"get_all_commits(100k) peak {peak_mib:.1f} MiB exceeds {_MAX_MIB} MiB. "
741 "The function loads all CommitRecords into a list simultaneously — "
742 "consider streaming or paginating for very large repos."
743 )
744
745
746 class TestGetCommitsForBranchWalkCap:
747 """get_commits_for_branch must honour its walk cap.
748
749 The cap is the primary memory guard for ``muse log --json``. A branch with
750 N commits deeper than the cap must return exactly cap records, not N — even
751 when filters are active and the caller passes max_count=0.
752 """
753
754 def test_walk_cap_bounds_returned_records(
755 self, tmp_path: pathlib.Path
756 ) -> None:
757 """Chain of 500 commits with cap=100 returns exactly 100 records."""
758 repo = _repo(tmp_path)
759 N = 500
760 CAP = 100
761 _write_chain(repo, "main", N)
762
763 results = get_commits_for_branch(repo, "bench", "main", max_count=CAP)
764
765 assert len(results) == CAP, (
766 f"Expected cap={CAP} records, got {len(results)} — "
767 "walk cap is not being respected."
768 )
769
770 def test_walk_cap_memory_bounded_deep_chain(
771 self, tmp_path: pathlib.Path
772 ) -> None:
773 """2 000-commit chain with default cap stays under 64 MiB."""
774 repo = _repo(tmp_path)
775 N = 2_000
776 CAP = 500
777 _write_chain(repo, "main", N)
778
779 tracemalloc.start()
780 tracemalloc.clear_traces()
781 results = get_commits_for_branch(repo, "bench", "main", max_count=CAP)
782 _, peak_bytes = tracemalloc.get_traced_memory()
783 tracemalloc.stop()
784
785 assert len(results) == CAP
786 peak_mib = peak_bytes / (1024 * 1024)
787 assert peak_mib <= 64, (
788 f"get_commits_for_branch(cap={CAP}) peak {peak_mib:.1f} MiB — "
789 f"expected ≤ 64 MiB for {CAP} records."
790 )
791
792
793 class TestLogJsonStreaming:
794 """``muse log --json`` must not double-buffer commit JSON strings.
795
796 The previous implementation accumulated ``commit_jsons: list[str]`` before
797 writing to stdout — doubling peak memory vs. the CommitRecord list alone.
798 The fix uses a first-item flag to write each record inline immediately.
799 This test verifies the fix: tracemalloc peak for a 500-commit log should
800 not contain the signature of a large string buffer.
801 """
802
803 def test_log_json_output_is_valid_json(
804 self, tmp_path: pathlib.Path
805 ) -> None:
806 """muse log --json on a 100-commit branch emits valid JSON with all commits."""
807 import json
808
809 from tests.cli_test_helper import CliRunner
810
811 repo = _repo(tmp_path)
812 N = 100
813 _write_chain(repo, "main", N)
814 (repo / ".muse" / "config.toml").write_text("")
815
816 runner = CliRunner()
817 result = runner.invoke(
818 None,
819 ["log", "--json", "--max-count", str(N)],
820 env={"MUSE_REPO_ROOT": str(repo)},
821 )
822 assert result.exit_code == 0, f"muse log --json failed:\n{result.output}"
823
824 payload = json.loads(result.output)
825 assert "commits" in payload, f"Missing 'commits' key: {payload}"
826 assert len(payload["commits"]) == N, (
827 f"Expected {N} commits in JSON output, got {len(payload['commits'])}"
828 )
829
830 def test_log_json_empty_repo_returns_empty_array(
831 self, tmp_path: pathlib.Path
832 ) -> None:
833 """muse log --json on a branch with no commits returns empty commits array."""
834 import json
835
836 from tests.cli_test_helper import CliRunner
837
838 repo = _repo(tmp_path)
839 (repo / ".muse" / "config.toml").write_text("")
840
841 runner = CliRunner()
842 result = runner.invoke(
843 None,
844 ["log", "--json"],
845 env={"MUSE_REPO_ROOT": str(repo)},
846 )
847 # No commits → either exit 0 with empty commits array or a graceful
848 # "(no commits)" message; both are acceptable.
849 if result.exit_code == 0 and result.output.startswith("{"):
850 payload = json.loads(result.output)
851 assert payload.get("commits") == []
852
853
854 class TestFindMergeBaseMemory:
855 """find_merge_base must fire its cap cleanly, not OOM.
856
857 The default ``max_ancestors`` cap is 50 000. A chain deeper than the cap
858 must raise ``MuseCLIError`` with an actionable message rather than
859 exhausting all available memory. This test confirms the error path, not
860 the OOM path.
861 """
862
863 def test_merge_base_cap_raises_gracefully_on_deep_chain(
864 self, tmp_path: pathlib.Path
865 ) -> None:
866 """Chain deeper than max_ancestors raises MuseCLIError, not MemoryError."""
867 from muse.core.errors import MuseCLIError
868
869 repo = _repo(tmp_path)
870 (repo / ".muse" / "config.toml").write_text(
871 "[limits]\nmax_ancestors = 50\n"
872 )
873
874 tip_a = _write_chain(repo, "branchA", 60, snap_id="cc" * 32)
875 # branchB is entirely separate — no common ancestor with branchA.
876 tip_b = _write_chain(repo, "branchB", 60, snap_id="dd" * 32, start=1000)
877
878 with pytest.raises(MuseCLIError, match="max_ancestors"):
879 find_merge_base(repo, tip_a, tip_b)
880
881 def test_merge_base_found_within_cap(
882 self, tmp_path: pathlib.Path
883 ) -> None:
884 """find_merge_base finds the base when both branches are within cap."""
885 repo = _repo(tmp_path)
886 (repo / ".muse" / "config.toml").write_text(
887 "[limits]\nmax_ancestors = 500\n"
888 )
889
890 # Build a common root commit.
891 snap_id = "ee" * 32
892 root_cid = compute_commit_id(
893 repo_id="bench",
894 parent_ids=[],
895 snapshot_id=snap_id,
896 message="root",
897 committed_at_iso="2026-01-01T00:00:00+00:00",
898 author="test",)
899 root_rec = CommitRecord(
900 commit_id=root_cid,
901 repo_id="bench",
902 created_on_branch="main",
903 snapshot_id=snap_id,
904 message="root",
905 committed_at=datetime.datetime(2026, 1, 1, tzinfo=datetime.timezone.utc),
906 parent_commit_id=None,
907 parent2_commit_id=None,
908 author="test",
909 metadata={},
910 structured_delta=None,
911 sem_ver_bump="none",
912 breaking_changes=[],
913 agent_id="",
914 model_id="",
915 toolchain_id="",
916 prompt_hash="",
917 signature="",
918 signer_key_id="",
919 )
920 write_commit(repo, root_rec)
921
922 # Extend branchA and branchB from the same root.
923 parent_a: str | None = root_cid
924 tip_a = root_cid
925 for i in range(20):
926 msg = f"a-{i:04d}"
927 ts = datetime.datetime(2026, 1, 2, tzinfo=datetime.timezone.utc)
928 cid = compute_commit_id(
929 repo_id="bench",
930 parent_ids=[parent_a] if parent_a else [],
931 snapshot_id=snap_id,
932 message=msg,
933 committed_at_iso=ts.isoformat(),
934 author="test",)
935 rec = CommitRecord(
936 commit_id=cid,
937 repo_id="bench",
938 created_on_branch="branchA",
939 snapshot_id=snap_id,
940 message=msg,
941 committed_at=ts,
942 parent_commit_id=parent_a,
943 parent2_commit_id=None,
944 author="test",
945 metadata={},
946 structured_delta=None,
947 sem_ver_bump="none",
948 breaking_changes=[],
949 agent_id="",
950 model_id="",
951 toolchain_id="",
952 prompt_hash="",
953 signature="",
954 signer_key_id="",
955 )
956 write_commit(repo, rec)
957 parent_a = cid
958 tip_a = cid
959
960 parent_b: str | None = root_cid
961 tip_b = root_cid
962 for i in range(15):
963 msg = f"b-{i:04d}"
964 ts = datetime.datetime(2026, 1, 3, tzinfo=datetime.timezone.utc)
965 cid = compute_commit_id(
966 repo_id="bench",
967 parent_ids=[parent_b] if parent_b else [],
968 snapshot_id=snap_id,
969 message=msg,
970 committed_at_iso=ts.isoformat(),
971 author="test",)
972 rec = CommitRecord(
973 commit_id=cid,
974 repo_id="bench",
975 created_on_branch="branchB",
976 snapshot_id=snap_id,
977 message=msg,
978 committed_at=ts,
979 parent_commit_id=parent_b,
980 parent2_commit_id=None,
981 author="test",
982 metadata={},
983 structured_delta=None,
984 sem_ver_bump="none",
985 breaking_changes=[],
986 agent_id="",
987 model_id="",
988 toolchain_id="",
989 prompt_hash="",
990 signature="",
991 signer_key_id="",
992 )
993 write_commit(repo, rec)
994 parent_b = cid
995 tip_b = cid
996
997 base = find_merge_base(repo, tip_a, tip_b)
998 assert base == root_cid, (
999 f"Expected merge base {root_cid[:8]}, got {base[:8] if base else None}"
1000 )
1001
1002 def test_merge_base_memory_bounded_within_cap(
1003 self, tmp_path: pathlib.Path
1004 ) -> None:
1005 """find_merge_base BFS uses bounded memory proportional to max_ancestors."""
1006 from muse.core.errors import MuseCLIError
1007
1008 repo = _repo(tmp_path)
1009 CAP = 200
1010 (repo / ".muse" / "config.toml").write_text(
1011 f"[limits]\nmax_ancestors = {CAP}\n"
1012 )
1013
1014 # Build a shallow divergence: 50 commits each, well inside the cap.
1015 snap_id = "ff" * 32
1016 root_cid = compute_commit_id(
1017 repo_id="bench",
1018 parent_ids=[],
1019 snapshot_id=snap_id,
1020 message="base",
1021 committed_at_iso="2026-01-01T00:00:00+00:00",
1022 author="t",)
1023 root_rec = CommitRecord(
1024 commit_id=root_cid,
1025 repo_id="bench",
1026 created_on_branch="main",
1027 snapshot_id=snap_id,
1028 message="base",
1029 committed_at=datetime.datetime(2026, 1, 1, tzinfo=datetime.timezone.utc),
1030 parent_commit_id=None,
1031 parent2_commit_id=None,
1032 author="t",
1033 metadata={},
1034 structured_delta=None,
1035 sem_ver_bump="none",
1036 breaking_changes=[],
1037 agent_id="",
1038 model_id="",
1039 toolchain_id="",
1040 prompt_hash="",
1041 signature="",
1042 signer_key_id="",
1043 )
1044 write_commit(repo, root_rec)
1045
1046 def _extend(parent: str, prefix: str, n: int) -> str:
1047 tip = parent
1048 for i in range(n):
1049 msg = f"{prefix}-{i:04d}"
1050 ts = datetime.datetime(2026, 2, 1, tzinfo=datetime.timezone.utc)
1051 cid = compute_commit_id(
1052 repo_id="bench",
1053 parent_ids=[tip],
1054 snapshot_id=snap_id,
1055 message=msg,
1056 committed_at_iso=ts.isoformat(),
1057 author="t",)
1058 rec = CommitRecord(
1059 commit_id=cid,
1060 repo_id="bench",
1061 created_on_branch=prefix,
1062 snapshot_id=snap_id,
1063 message=msg,
1064 committed_at=ts,
1065 parent_commit_id=tip,
1066 parent2_commit_id=None,
1067 author="t",
1068 metadata={},
1069 structured_delta=None,
1070 sem_ver_bump="none",
1071 breaking_changes=[],
1072 agent_id="",
1073 model_id="",
1074 toolchain_id="",
1075 prompt_hash="",
1076 signature="",
1077 signer_key_id="",
1078 )
1079 write_commit(repo, rec)
1080 tip = cid
1081 return tip
1082
1083 tip_a = _extend(root_cid, "xa", 50)
1084 tip_b = _extend(root_cid, "xb", 50)
1085
1086 tracemalloc.start()
1087 tracemalloc.clear_traces()
1088 try:
1089 base = find_merge_base(repo, tip_a, tip_b)
1090 except MuseCLIError:
1091 base = None # Cap triggered — that's also a valid outcome.
1092 _, peak_bytes = tracemalloc.get_traced_memory()
1093 tracemalloc.stop()
1094
1095 peak_mib = peak_bytes / (1024 * 1024)
1096 assert peak_mib <= 64, (
1097 f"find_merge_base peak {peak_mib:.1f} MiB — expected ≤ 64 MiB "
1098 f"for two 50-commit branches (cap={CAP})."
1099 )
1100
1101
1102 class TestSnapshotManifest75kFiles:
1103 """build_snapshot_manifest on 75 000 files must stay under 512 MiB.
1104
1105 The manifest dict holds only ``{rel_path: sha256_hex}`` — strings only.
1106 75k × (60-char path + 64-char hash) ≈ 9.3 MiB for the dict alone.
1107 The peak should be dominated by the stat-cache msgpack load, not by
1108 the manifest itself. 512 MiB is a generous ceiling to catch any
1109 accidental full-file-content buffering.
1110 """
1111
1112 @pytest.mark.slow
1113 def test_75k_files_peak_rss_under_512_mib(
1114 self, tmp_path: pathlib.Path
1115 ) -> None:
1116 """build_snapshot_manifest on 75 000 small files stays under 512 MiB."""
1117 root = tmp_path / "workdir"
1118 root.mkdir()
1119 (root / ".muse").mkdir()
1120 (root / ".muse" / "repo.json").write_text('{"repo_id": "bench"}')
1121
1122 # 750 dirs × 100 files = 75 000 files, each 128 bytes.
1123 for d in range(750):
1124 dp = root / f"pkg_{d:04d}"
1125 dp.mkdir()
1126 for f in range(100):
1127 (dp / f"f_{f:03d}.py").write_bytes(b"x" * 128)
1128
1129 tracemalloc.start()
1130 tracemalloc.clear_traces()
1131 manifest = build_snapshot_manifest(root)
1132 _, peak_bytes = tracemalloc.get_traced_memory()
1133 tracemalloc.stop()
1134
1135 assert len(manifest) == 75_000, (
1136 f"Expected 75 000 files in manifest, got {len(manifest)}"
1137 )
1138 peak_mib = peak_bytes / (1024 * 1024)
1139 assert peak_mib <= 512, (
1140 f"build_snapshot_manifest(75k) peak {peak_mib:.1f} MiB exceeds 512 MiB. "
1141 "File content must never be loaded into memory — only stat + SHA-256."
1142 )
1143
1144 def test_10k_files_peak_rss_under_64_mib(
1145 self, tmp_path: pathlib.Path
1146 ) -> None:
1147 """10k-file manifest stays under 64 MiB (fast smoke for the ceiling property)."""
1148 root = tmp_path / "workdir"
1149 root.mkdir()
1150 (root / ".muse").mkdir()
1151 (root / ".muse" / "repo.json").write_text('{"repo_id": "bench"}')
1152
1153 for d in range(100):
1154 dp = root / f"pkg_{d:03d}"
1155 dp.mkdir()
1156 for f in range(100):
1157 (dp / f"f_{f:03d}.py").write_bytes(b"y" * 64)
1158
1159 tracemalloc.start()
1160 tracemalloc.clear_traces()
1161 manifest = build_snapshot_manifest(root)
1162 _, peak_bytes = tracemalloc.get_traced_memory()
1163 tracemalloc.stop()
1164
1165 assert len(manifest) == 10_000
1166 peak_mib = peak_bytes / (1024 * 1024)
1167 assert peak_mib <= 64, (
1168 f"build_snapshot_manifest(10k) peak {peak_mib:.1f} MiB — expected ≤ 64 MiB."
1169 )
1170
1171
1172 class TestWalkCommitsBetweenCap:
1173 """walk_commits_between must truncate at its cap, never OOM.
1174
1175 This function is used by ``muse status --json`` for ahead/behind counts.
1176 A branch 100k commits ahead of remote must return at most ``max_commits``
1177 records and set ``truncated=True`` — it must never allocate memory
1178 proportional to the full chain depth.
1179 """
1180
1181 def test_truncates_at_cap_not_oom(
1182 self, tmp_path: pathlib.Path
1183 ) -> None:
1184 """Chain of 1 000 commits with cap=100 truncates, doesn't exhaust memory."""
1185 repo = _repo(tmp_path)
1186 N = 1_000
1187 CAP = 100
1188 tip = _write_chain(repo, "main", N)
1189
1190 result = walk_commits_between_result(repo, tip, max_commits=CAP)
1191
1192 assert result["truncated"] is True, (
1193 "Expected truncated=True for chain longer than cap"
1194 )
1195 assert len(result["commits"]) == CAP, (
1196 f"Expected exactly {CAP} commits, got {len(result['commits'])}"
1197 )
1198
1199 def test_truncation_memory_bounded(
1200 self, tmp_path: pathlib.Path
1201 ) -> None:
1202 """walk_commits_between_result peak memory is bounded by cap, not chain depth."""
1203 repo = _repo(tmp_path)
1204 N = 2_000
1205 CAP = 200
1206 tip = _write_chain(repo, "main", N)
1207
1208 tracemalloc.start()
1209 tracemalloc.clear_traces()
1210 result = walk_commits_between_result(repo, tip, max_commits=CAP)
1211 _, peak_bytes = tracemalloc.get_traced_memory()
1212 tracemalloc.stop()
1213
1214 assert result["count"] == CAP
1215 peak_mib = peak_bytes / (1024 * 1024)
1216 assert peak_mib <= 32, (
1217 f"walk_commits_between_result(cap={CAP}) peak {peak_mib:.1f} MiB — "
1218 "memory must be proportional to cap, not chain depth."
1219 )
File History 3 commits
sha256:b636f72dcba9e190afb980bece906fa5b717fbde014b76ef023df8cb96e01eb9 docs: expand cache plan with all seven testing tiers and do… Sonnet 4.6 131 days ago
sha256:7f9e2ef5286aedad9c1e6011b4c46ca27f39dbdad6e3409357e36b26e46b3b7c docs: docstring sprint for-each-ref→hotspots — idiomatic ru… Sonnet 4.6 patch 137 days ago
sha256:a09b1b4f6838754495547f200aa0ce88e2f56ffc5b20b900f6f0cff2c3cdede9 fix(cursorignore): remove git-ism (.git/worktrees) Human minor 140 days ago