gabriel / muse public
test_integrity_I2_fsync.py python
1,776 lines 70.6 KB
Raw
sha256:b636f72dcba9e190afb980bece906fa5b717fbde014b76ef023df8cb96e01eb9 docs: expand cache plan with all seven testing tiers and do… Sonnet 4.6 131 days ago
1 """I-2: fsync before atomic rename in ALL durable write paths.
2
3 Covers every write primitive in the Muse durability chain:
4
5 Tier 0 — Primitive helpers
6 * write_text_atomic — the canonical text-file atomic helper
7 * _write_msgpack_atomic — the canonical msgpack helper
8
9 Tier 1 — HEAD + branch refs (catastrophic if corrupt)
10 * write_head_branch (store.py)
11 * write_head_commit (store.py)
12 * write_branch_ref (store.py — used by commit, merge, checkout, pull,
13 revert, reset, cherry_pick, update_ref, transport,
14 rebase, bundle)
15
16 Tier 2 — VCS state files
17 * write_merge_state (merge_engine.py)
18 * save_rebase_state (rebase.py)
19 * create_reservation, create_intent (coordination.py)
20 * OpLog checkpoint (op_log.py)
21
22 Tier 3 — Config files
23 * config.py write_config_value / set_remote (via write_text_atomic)
24
25 Tier 4 — Large-blob write paths (shutil.copy2-based)
26 * write_object_from_path — uses _fsync_fd (fd-based) before os.replace
27 * restore_object — uses _fsync_path (path-based) before os.replace
28
29 Each tier is verified for:
30 1. mkstemp unique temp names (no fixed .tmp collisions).
31 2. fsync before os.replace — ordering enforced.
32 3. No orphan temp files after success or simulated crash.
33 4. Concurrent write safety — independent results, no cross-corruption.
34 5. Correct final on-disk content.
35
36 Additional coverage (gap audit):
37 6. Page-cache non-flush defense-in-depth (I-1 is the backstop for I-2).
38 7. Mid-write fh.write failure — orphan cleaned up.
39 8. Same object_id written from N threads simultaneously — idempotency holds.
40 9. 10 000 sequential commits — store clean throughout.
41 10. SIGKILL crash safety — multiprocessing kill leaves no orphans, store consistent.
42 11. Performance benchmark — 4 KiB msgpack fsync write < 5 ms.
43
44 Regression — any write path that bypasses write_text_atomic is caught here.
45 """
46 from __future__ import annotations
47
48 import os
49 import pathlib
50 import tempfile
51 import threading
52 from unittest.mock import patch
53
54
55 def _sigkill_writer_worker(root: pathlib.Path, count: int) -> None:
56 """Write objects in a tight loop until killed.
57
58 Defined at module level so it is picklable under the ``"spawn"``
59 multiprocessing context (closures defined inside test methods are not
60 picklable and therefore incompatible with ``"spawn"``).
61 """
62 import time as _time
63
64 from muse.core._types import blob_id as _blob_id
65 from muse.core.object_store import write_object as _wo
66
67 for i in range(count):
68 payload = f"crash-worker-object-{i}".encode()
69 obj_id = _blob_id(payload)
70 try:
71 _wo(root, obj_id, payload)
72 except Exception:
73 pass
74 _time.sleep(0.0001)
75
76 import pytest
77
78 import json
79 import muse.core.rebase
80
81
82 def _corrupt_file(p: pathlib.Path, new_content: bytes) -> None:
83 """Overwrite *p* temporarily lifting the 0o444 guard.
84
85 Object files are written with mode 0o444. Tests that simulate disk
86 corruption must temporarily grant write permission.
87 """
88 os.chmod(p, 0o644)
89 try:
90 p.write_bytes(new_content)
91 finally:
92 os.chmod(p, 0o444)
93
94 from muse.core.coordination import Reservation
95 from muse.core._types import blob_id, fake_id
96 from muse.core.rebase import RebaseState
97 from muse.core.object_store import (
98 _fsync_fd,
99 write_object,
100 write_object_from_path,
101 restore_object,
102 read_object,
103 object_path,
104 )
105 from muse.core.snapshot import compute_commit_id
106 from muse.core.store import (
107 CommitRecord,
108 write_branch_ref,
109 write_commit,
110 write_head_branch,
111 write_head_commit,
112 write_text_atomic,
113 read_commit,
114 )
115 import datetime
116
117
118 # ---------------------------------------------------------------------------
119 # Helpers
120 # ---------------------------------------------------------------------------
121
122 def _repo(tmp_path: pathlib.Path) -> pathlib.Path:
123 (tmp_path / ".muse").mkdir()
124 (tmp_path / ".muse" / "commits").mkdir()
125 (tmp_path / ".muse" / "snapshots").mkdir()
126 return tmp_path
127
128
129 def _oid(data: bytes) -> str:
130 return blob_id(data)
131
132
133 def _commit(idx: int = 0) -> CommitRecord:
134 sid = fake_id(f"snap-{idx}")
135 message = f"commit {idx}"
136 committed_at = datetime.datetime(2026, 1, 1, tzinfo=datetime.timezone.utc)
137 cid = compute_commit_id(
138 repo_id="test-repo",
139 parent_ids=[],
140 snapshot_id=sid,
141 message=message,
142 committed_at_iso=committed_at.isoformat(),
143 author="tester",)
144 return CommitRecord(
145 commit_id=cid,
146 repo_id="test-repo",
147 created_on_branch="main",
148 snapshot_id=sid,
149 message=message,
150 committed_at=committed_at,
151 author="tester",
152 parent_commit_id=None,
153 parent2_commit_id=None,
154 )
155
156
157 def _tmp_files(directory: pathlib.Path) -> list[pathlib.Path]:
158 """Return all temp/orphan files in *directory* (recursively)."""
159 result: list[pathlib.Path] = []
160 for p in directory.rglob("*"):
161 name = p.name
162 if name.startswith(".obj-tmp-") or name.startswith(".muse-tmp-") or name.startswith(".restore-tmp-"):
163 result.append(p)
164 return result
165
166
167 # ---------------------------------------------------------------------------
168 # Unit: fsync is called before os.replace in write_object
169 # ---------------------------------------------------------------------------
170
171 class TestFsyncCalledBeforeReplace:
172 def test_write_object_calls_fsync_before_replace(self, tmp_path: pathlib.Path) -> None:
173 """_fsync_fd must be called before os.replace in write_object.
174
175 Patches _fsync_fd (the platform abstraction) rather than os.fsync
176 directly, because on macOS _fsync_fd uses fcntl(F_BARRIERFSYNC) and
177 returns before ever reaching os.fsync.
178 """
179 repo = _repo(tmp_path)
180 data = b"fsync ordering test"
181 oid = _oid(data)
182
183 call_order: list[str] = []
184 real_fsync_fd = _fsync_fd
185 real_replace = os.replace
186
187 def tracking_fsync_fd(fd: int) -> None:
188 call_order.append("fsync")
189 real_fsync_fd(fd)
190
191 def tracking_replace(src: str | bytes | os.PathLike[str], dst: str | bytes | os.PathLike[str]) -> None:
192 call_order.append("replace")
193 real_replace(src, dst)
194
195 with patch("muse.core.object_store._fsync_fd", side_effect=tracking_fsync_fd), \
196 patch("muse.core.object_store.os.replace", side_effect=tracking_replace):
197 write_object(repo, oid, data)
198
199 assert "fsync" in call_order, "_fsync_fd was never called"
200 assert "replace" in call_order, "replace was never called"
201 fsync_pos = next(i for i, c in enumerate(call_order) if c == "fsync")
202 replace_pos = next(i for i, c in enumerate(call_order) if c == "replace")
203 assert fsync_pos < replace_pos, (
204 f"_fsync_fd (pos {fsync_pos}) must happen before replace (pos {replace_pos})"
205 )
206
207 def test_write_commit_calls_fsync_before_replace(self, tmp_path: pathlib.Path) -> None:
208 """A durability flush must be called before tmp.replace in _write_msgpack_atomic.
209
210 On macOS, _write_msgpack_atomic uses fcntl(F_BARRIERFSYNC) and never
211 reaches os.fsync directly. We track both paths so the test is
212 platform-agnostic.
213 """
214 repo = _repo(tmp_path)
215 c = _commit(0)
216
217 call_order: list[str] = []
218 real_fsync = os.fsync
219 import fcntl as _fcntl
220 real_fcntl = _fcntl.fcntl
221
222 def tracking_fsync(fd: int) -> None:
223 call_order.append("fsync")
224 real_fsync(fd)
225
226 def tracking_fcntl(fd: int, cmd: int, *args: int) -> int:
227 if cmd == 85: # F_BARRIERFSYNC
228 call_order.append("fsync")
229 return real_fcntl(fd, cmd, *args)
230
231 with patch("muse.core.store.os.fsync", side_effect=tracking_fsync), \
232 patch("muse.core.store.fcntl.fcntl", side_effect=tracking_fcntl):
233 write_commit(repo, c)
234
235 assert "fsync" in call_order, "no durability flush called during write_commit"
236
237 def test_fsync_failure_is_non_fatal(self, tmp_path: pathlib.Path) -> None:
238 """A failing fsync (virtual fs) must not prevent the write from completing."""
239 repo = _repo(tmp_path)
240 data = b"fsync fails gracefully"
241 oid = _oid(data)
242
243 with patch("muse.core.object_store.os.fsync", side_effect=OSError("fsync not supported")):
244 result = write_object(repo, oid, data)
245
246 assert result is True
247 assert read_object(repo, oid) == data
248
249 def test_fsync_failure_in_store_is_non_fatal(self, tmp_path: pathlib.Path) -> None:
250 """A failing fsync in _write_msgpack_atomic must not abort the commit write."""
251 repo = _repo(tmp_path)
252 c = _commit(1)
253
254 with patch("muse.core.store.os.fsync", side_effect=OSError("not supported")):
255 write_commit(repo, c)
256
257 assert read_commit(repo, c.commit_id) is not None
258
259
260 # ---------------------------------------------------------------------------
261 # Unit: unique temp file names (mkstemp, not fixed .tmp)
262 # ---------------------------------------------------------------------------
263
264 class TestUniqueTempNames:
265 def test_write_object_uses_mkstemp(self, tmp_path: pathlib.Path) -> None:
266 """write_object must use tempfile.mkstemp, not path.with_suffix('.tmp')."""
267 repo = _repo(tmp_path)
268 data = b"unique temp name check"
269 oid = _oid(data)
270
271 mkstemp_called = [False]
272 real_mkstemp = tempfile.mkstemp
273
274 def tracking_mkstemp(dir: pathlib.Path | None = None, prefix: str = "") -> tuple[int, str]:
275 mkstemp_called[0] = True
276 return real_mkstemp(dir=dir, prefix=prefix)
277
278 with patch("muse.core.object_store.tempfile.mkstemp", side_effect=tracking_mkstemp):
279 write_object(repo, oid, data)
280
281 assert mkstemp_called[0], "tempfile.mkstemp was not called — fixed .tmp name may be in use"
282
283 def test_write_commit_uses_mkstemp(self, tmp_path: pathlib.Path) -> None:
284 """_write_msgpack_atomic must use tempfile.mkstemp."""
285 repo = _repo(tmp_path)
286 c = _commit(2)
287
288 mkstemp_called = [False]
289 real_mkstemp = tempfile.mkstemp
290
291 def tracking_mkstemp(dir: pathlib.Path | None = None, prefix: str = "") -> tuple[int, str]:
292 mkstemp_called[0] = True
293 return real_mkstemp(dir=dir, prefix=prefix)
294
295 with patch("muse.core.store.tempfile.mkstemp", side_effect=tracking_mkstemp):
296 write_commit(repo, c)
297
298 assert mkstemp_called[0], "tempfile.mkstemp was not called in _write_msgpack_atomic"
299
300 def test_no_fixed_tmp_suffix_after_write(self, tmp_path: pathlib.Path) -> None:
301 """After a successful write, no .tmp file must remain."""
302 repo = _repo(tmp_path)
303 c = _commit(3)
304 write_commit(repo, c)
305
306 fixed_tmps = list((tmp_path / ".muse" / "commits").glob("*.tmp"))
307 assert fixed_tmps == [], f"Fixed .tmp files left behind: {fixed_tmps}"
308
309
310 # ---------------------------------------------------------------------------
311 # Integration: no orphan temps after successful writes
312 # ---------------------------------------------------------------------------
313
314 class TestNoOrphanTemps:
315 def test_no_orphan_after_write_object(self, tmp_path: pathlib.Path) -> None:
316 repo = _repo(tmp_path)
317 data = b"clean write"
318 oid = _oid(data)
319 write_object(repo, oid, data)
320 assert _tmp_files(tmp_path) == []
321
322 def test_no_orphan_after_write_object_from_path(self, tmp_path: pathlib.Path) -> None:
323 repo = _repo(tmp_path)
324 src = tmp_path / "src.bin"
325 data = b"from path write"
326 src.write_bytes(data)
327 oid = _oid(data)
328 write_object_from_path(repo, oid, src)
329 assert _tmp_files(tmp_path) == []
330
331 def test_no_orphan_after_restore_object(self, tmp_path: pathlib.Path) -> None:
332 repo = _repo(tmp_path)
333 data = b"restore write"
334 oid = _oid(data)
335 write_object(repo, oid, data)
336 dest = tmp_path / "restored.bin"
337 restore_object(repo, oid, dest)
338 assert _tmp_files(tmp_path) == []
339
340 def test_no_orphan_after_write_commit(self, tmp_path: pathlib.Path) -> None:
341 repo = _repo(tmp_path)
342 write_commit(repo, _commit(4))
343 assert _tmp_files(tmp_path) == []
344
345 def test_no_orphan_after_simulated_replace_failure(self, tmp_path: pathlib.Path) -> None:
346 """When os.replace raises, the temp file must be cleaned up."""
347 repo = _repo(tmp_path)
348 data = b"replace will fail"
349 oid = _oid(data)
350
351 with pytest.raises(OSError):
352 with patch("muse.core.object_store.os.replace", side_effect=OSError("disk full")):
353 write_object(repo, oid, data)
354
355 assert _tmp_files(tmp_path) == [], "Temp file not cleaned up after replace failure"
356
357 def test_no_orphan_in_store_after_replace_failure(self, tmp_path: pathlib.Path) -> None:
358 """When tmp.replace raises in _write_msgpack_atomic, temp is cleaned up."""
359 repo = _repo(tmp_path)
360 c = _commit(5)
361
362 with pytest.raises(OSError):
363 with patch("muse.core.store.os.fdopen") as mock_fdopen:
364 mock_fh = mock_fdopen.return_value.__enter__.return_value
365 mock_fh.write.return_value = None
366 mock_fh.flush.return_value = None
367 mock_fh.fileno.return_value = -1
368 with patch("muse.core.store.os.fsync"), \
369 patch("muse.core.store.fcntl.fcntl"):
370 with patch("pathlib.Path.replace", side_effect=OSError("disk full")):
371 write_commit(repo, c)
372
373 assert _tmp_files(tmp_path) == [], "Temp file not cleaned up after commit replace failure"
374
375
376 # ---------------------------------------------------------------------------
377 # Stress: 200 concurrent writers to the same shard
378 # ---------------------------------------------------------------------------
379
380 class TestConcurrentWriters:
381 @pytest.mark.slow
382 def test_200_concurrent_object_writes_no_corruption(self, tmp_path: pathlib.Path) -> None:
383 """200 threads writing distinct objects concurrently must all land correctly."""
384 repo = _repo(tmp_path)
385 payloads = [f"concurrent-object-{i}".encode() for i in range(200)]
386 oids = [_oid(p) for p in payloads]
387 errors: list[str] = []
388
389 def writer(data: bytes, oid: str) -> None:
390 try:
391 write_object(repo, oid, data)
392 result = read_object(repo, oid)
393 if result != data:
394 errors.append(f"Mismatch for {oid[:8]}: got {repr(result)[:20]}")
395 except Exception as exc:
396 errors.append(f"Exception for {oid[:8]}: {exc}")
397
398 threads = [
399 threading.Thread(target=writer, args=(p, o))
400 for p, o in zip(payloads, oids)
401 ]
402 for t in threads:
403 t.start()
404 for t in threads:
405 t.join()
406
407 assert errors == [], f"Concurrent write errors:\n" + "\n".join(errors)
408 # Every object must be present and correct
409 for data, oid in zip(payloads, oids):
410 assert read_object(repo, oid) == data
411
412 def test_100_concurrent_commit_writes_no_corruption(self, tmp_path: pathlib.Path) -> None:
413 """100 threads writing distinct commits concurrently must all land correctly."""
414 repo = _repo(tmp_path)
415 commits = [_commit(i) for i in range(100)]
416 errors: list[str] = []
417
418 def writer(c: CommitRecord) -> None:
419 try:
420 write_commit(repo, c)
421 result = read_commit(repo, c.commit_id)
422 if result is None:
423 errors.append(f"Commit {c.commit_id[:8]} not found after write")
424 elif result.message != c.message:
425 errors.append(f"Commit {c.commit_id[:8]} message corrupted")
426 except Exception as exc:
427 errors.append(f"Exception for {c.commit_id[:8]}: {exc}")
428
429 threads = [threading.Thread(target=writer, args=(c,)) for c in commits]
430 for t in threads:
431 t.start()
432 for t in threads:
433 t.join()
434
435 assert errors == [], "Concurrent commit write errors:\n" + "\n".join(errors)
436
437 def test_no_orphan_temps_after_concurrent_writes(self, tmp_path: pathlib.Path) -> None:
438 """No temp files must remain after concurrent writes complete."""
439 repo = _repo(tmp_path)
440 payloads = [f"orphan-check-{i}".encode() for i in range(50)]
441
442 threads = [
443 threading.Thread(target=write_object, args=(repo, _oid(p), p))
444 for p in payloads
445 ]
446 for t in threads:
447 t.start()
448 for t in threads:
449 t.join()
450
451 assert _tmp_files(tmp_path) == []
452
453
454 # ---------------------------------------------------------------------------
455 # Tier 0: write_text_atomic — the primitive all text-state writes funnel through
456 # ---------------------------------------------------------------------------
457
458 class TestWriteTextAtomic:
459 """Unit tests for the write_text_atomic primitive."""
460
461 def test_writes_correct_content(self, tmp_path: pathlib.Path) -> None:
462 path = tmp_path / "state.txt"
463 write_text_atomic(path, "hello world\n")
464 assert path.read_text() == "hello world\n"
465
466 def test_creates_parent_dirs(self, tmp_path: pathlib.Path) -> None:
467 path = tmp_path / "a" / "b" / "c" / "state.txt"
468 write_text_atomic(path, "deep")
469 assert path.read_text() == "deep"
470
471 def test_uses_mkstemp_not_fixed_tmp(self, tmp_path: pathlib.Path) -> None:
472 """write_text_atomic must use tempfile.mkstemp, not path.with_suffix('.tmp')."""
473 path = tmp_path / "ref"
474 called = [False]
475 real_mkstemp = tempfile.mkstemp
476
477 def tracking(dir: pathlib.Path | None = None, prefix: str = "") -> tuple[int, str]:
478 called[0] = True
479 return real_mkstemp(dir=dir, prefix=prefix)
480
481 with patch("muse.core.store.tempfile.mkstemp", side_effect=tracking):
482 write_text_atomic(path, "abc")
483
484 assert called[0], "write_text_atomic did not call tempfile.mkstemp"
485
486 def test_fsync_called_before_replace(self, tmp_path: pathlib.Path) -> None:
487 """os.fsync must be called before os.replace in write_text_atomic."""
488 path = tmp_path / "ref"
489 call_order: list[str] = []
490 real_fsync = os.fsync
491 real_replace = os.replace
492
493 def t_fsync(fd: int) -> None:
494 call_order.append("fsync")
495 real_fsync(fd)
496
497 def t_replace(src: str | bytes | os.PathLike[str], dst: str | bytes | os.PathLike[str]) -> None:
498 call_order.append("replace")
499 real_replace(src, dst)
500
501 with patch("muse.core.store.os.fsync", side_effect=t_fsync), \
502 patch("muse.core.store.os.replace", side_effect=t_replace):
503 write_text_atomic(path, "content")
504
505 fsync_pos = next((i for i, c in enumerate(call_order) if c == "fsync"), None)
506 replace_pos = next((i for i, c in enumerate(call_order) if c == "replace"), None)
507 assert fsync_pos is not None, "fsync never called"
508 assert replace_pos is not None, "replace never called"
509 assert fsync_pos < replace_pos, "fsync must happen before replace"
510
511 def test_fsync_failure_is_non_fatal(self, tmp_path: pathlib.Path) -> None:
512 """A failing fsync (virtual fs) must not prevent the write from completing."""
513 path = tmp_path / "ref"
514 with patch("muse.core.store.os.fsync", side_effect=OSError("not supported")):
515 write_text_atomic(path, "durable despite fsync failure")
516 assert path.read_text() == "durable despite fsync failure"
517
518 def test_no_orphan_after_success(self, tmp_path: pathlib.Path) -> None:
519 path = tmp_path / "ref"
520 write_text_atomic(path, "clean")
521 assert _tmp_files(tmp_path) == []
522
523 def test_no_orphan_after_replace_failure(self, tmp_path: pathlib.Path) -> None:
524 """When os.replace raises, the temp file must be unlinked."""
525 path = tmp_path / "ref"
526 with pytest.raises(OSError):
527 with patch("muse.core.store.os.replace", side_effect=OSError("disk full")):
528 write_text_atomic(path, "will fail")
529 assert _tmp_files(tmp_path) == [], "Orphan temp file left after replace failure"
530
531 def test_overwrites_existing_file(self, tmp_path: pathlib.Path) -> None:
532 """Subsequent writes must atomically replace the old content."""
533 path = tmp_path / "ref"
534 write_text_atomic(path, "old")
535 write_text_atomic(path, "new")
536 assert path.read_text() == "new"
537
538 def test_encoding_respected(self, tmp_path: pathlib.Path) -> None:
539 path = tmp_path / "utf8"
540 write_text_atomic(path, "caf\u00e9", encoding="utf-8")
541 assert path.read_text(encoding="utf-8") == "caf\u00e9"
542
543 def test_50_concurrent_writes_same_path_no_corruption(self, tmp_path: pathlib.Path) -> None:
544 """50 threads writing to the same file — last write wins, no corruption."""
545 path = tmp_path / "shared_ref"
546 errors: list[str] = []
547
548 def writer(i: int) -> None:
549 try:
550 write_text_atomic(path, f"value-{i:04d}")
551 except Exception as exc:
552 errors.append(str(exc))
553
554 threads = [threading.Thread(target=writer, args=(i,)) for i in range(50)]
555 for t in threads:
556 t.start()
557 for t in threads:
558 t.join()
559
560 assert errors == [], f"write_text_atomic raised: {errors}"
561 content = path.read_text()
562 assert content.startswith("value-"), f"Corrupt content: {content!r}"
563 assert _tmp_files(tmp_path) == [], "Orphan temp files after concurrent writes"
564
565 def test_100_concurrent_writes_distinct_paths_all_land(self, tmp_path: pathlib.Path) -> None:
566 """100 threads writing to distinct paths — all must land correctly."""
567 paths = [tmp_path / f"ref-{i:03d}" for i in range(100)]
568 errors: list[str] = []
569
570 def writer(p: pathlib.Path, i: int) -> None:
571 try:
572 write_text_atomic(p, f"commit-{i}")
573 if p.read_text() != f"commit-{i}":
574 errors.append(f"Mismatch at {p.name}")
575 except Exception as exc:
576 errors.append(str(exc))
577
578 threads = [threading.Thread(target=writer, args=(p, i)) for i, p in enumerate(paths)]
579 for t in threads:
580 t.start()
581 for t in threads:
582 t.join()
583
584 assert errors == [], f"Concurrent distinct-path errors: {errors}"
585 for i, p in enumerate(paths):
586 assert p.read_text() == f"commit-{i}"
587
588
589 # ---------------------------------------------------------------------------
590 # Tier 1a: write_head_branch and write_head_commit
591 # ---------------------------------------------------------------------------
592
593 class TestHeadWrites:
594 """HEAD files are the most critical VCS state — a corrupt HEAD breaks the repo."""
595
596 def _init(self, tmp_path: pathlib.Path) -> pathlib.Path:
597 (tmp_path / ".muse").mkdir()
598 (tmp_path / ".muse" / "refs" / "heads").mkdir(parents=True)
599 return tmp_path
600
601 def test_write_head_branch_correct_format(self, tmp_path: pathlib.Path) -> None:
602 root = self._init(tmp_path)
603 write_head_branch(root, "main")
604 content = (root / ".muse" / "HEAD").read_text()
605 assert content == "ref: refs/heads/main\n"
606
607 def test_write_head_branch_is_atomic(self, tmp_path: pathlib.Path) -> None:
608 """write_head_branch must go through write_text_atomic (mkstemp + fsync)."""
609 root = self._init(tmp_path)
610 called = [False]
611 real_mkstemp = tempfile.mkstemp
612
613 def tracking(dir: pathlib.Path | None = None, prefix: str = "") -> tuple[int, str]:
614 called[0] = True
615 return real_mkstemp(dir=dir, prefix=prefix)
616
617 with patch("muse.core.store.tempfile.mkstemp", side_effect=tracking):
618 write_head_branch(root, "main")
619
620 assert called[0], "write_head_branch bypassed mkstemp (not atomic)"
621
622 def test_write_head_branch_rejects_invalid_name(self, tmp_path: pathlib.Path) -> None:
623 root = self._init(tmp_path)
624 with pytest.raises((ValueError, SystemExit)):
625 write_head_branch(root, "bad/../../traversal")
626
627 def test_write_head_commit_correct_format(self, tmp_path: pathlib.Path) -> None:
628 root = self._init(tmp_path)
629 cid = fake_id("commit-a")
630 write_head_commit(root, cid)
631 content = (root / ".muse" / "HEAD").read_text()
632 assert content == f"commit: {cid}\n"
633
634 def test_write_head_commit_is_atomic(self, tmp_path: pathlib.Path) -> None:
635 root = self._init(tmp_path)
636 called = [False]
637 real_mkstemp = tempfile.mkstemp
638
639 def tracking(dir: pathlib.Path | None = None, prefix: str = "") -> tuple[int, str]:
640 called[0] = True
641 return real_mkstemp(dir=dir, prefix=prefix)
642
643 cid = fake_id("commit-b")
644 with patch("muse.core.store.tempfile.mkstemp", side_effect=tracking):
645 write_head_commit(root, cid)
646
647 assert called[0], "write_head_commit bypassed mkstemp (not atomic)"
648
649 def test_write_head_commit_rejects_short_id(self, tmp_path: pathlib.Path) -> None:
650 root = self._init(tmp_path)
651 with pytest.raises(ValueError, match="sha256"):
652 write_head_commit(root, "abc123")
653
654 def test_write_head_commit_rejects_non_hex(self, tmp_path: pathlib.Path) -> None:
655 root = self._init(tmp_path)
656 with pytest.raises(ValueError):
657 write_head_commit(root, "z" * 64)
658
659 def test_head_survives_concurrent_branch_switches(self, tmp_path: pathlib.Path) -> None:
660 """50 threads racing to update HEAD — no corruption, HEAD always readable."""
661 root = self._init(tmp_path)
662 errors: list[str] = []
663 branch_names = [f"feat-{i:03d}" for i in range(50)]
664
665 def switcher(branch: str) -> None:
666 try:
667 write_head_branch(root, branch)
668 content = (root / ".muse" / "HEAD").read_text()
669 if not content.startswith("ref: refs/heads/"):
670 errors.append(f"HEAD corrupted: {content!r}")
671 except Exception as exc:
672 errors.append(str(exc))
673
674 threads = [threading.Thread(target=switcher, args=(b,)) for b in branch_names]
675 for t in threads:
676 t.start()
677 for t in threads:
678 t.join()
679
680 assert errors == [], f"HEAD corruption detected: {errors}"
681 assert _tmp_files(tmp_path) == []
682
683
684 # ---------------------------------------------------------------------------
685 # Tier 1b: write_branch_ref — canonical branch pointer update
686 # ---------------------------------------------------------------------------
687
688 class TestWriteBranchRef:
689 """Branch refs are the second most critical VCS state.
690
691 A corrupt or missing ref orphans all commits reachable only from that branch.
692 """
693
694 def _init(self, tmp_path: pathlib.Path) -> pathlib.Path:
695 (tmp_path / ".muse" / "refs" / "heads").mkdir(parents=True)
696 return tmp_path
697
698 def _valid_cid(self, seed: str = "x") -> str:
699 return fake_id(seed)
700
701 def test_writes_correct_content(self, tmp_path: pathlib.Path) -> None:
702 root = self._init(tmp_path)
703 cid = self._valid_cid("test")
704 write_branch_ref(root, "main", cid)
705 ref_path = root / ".muse" / "refs" / "heads" / "main"
706 assert ref_path.read_text() == cid
707
708 def test_is_atomic_uses_mkstemp(self, tmp_path: pathlib.Path) -> None:
709 root = self._init(tmp_path)
710 called = [False]
711 real_mkstemp = tempfile.mkstemp
712
713 def tracking(dir: pathlib.Path | None = None, prefix: str = "") -> tuple[int, str]:
714 called[0] = True
715 return real_mkstemp(dir=dir, prefix=prefix)
716
717 with patch("muse.core.store.tempfile.mkstemp", side_effect=tracking):
718 write_branch_ref(root, "main", self._valid_cid())
719
720 assert called[0], "write_branch_ref bypassed mkstemp (not atomic)"
721
722 def test_fsync_called_before_replace(self, tmp_path: pathlib.Path) -> None:
723 root = self._init(tmp_path)
724 call_order: list[str] = []
725 real_fsync = os.fsync
726 real_replace = os.replace
727
728 def t_fsync(fd: int) -> None:
729 call_order.append("fsync")
730 real_fsync(fd)
731
732 def t_replace(src: str | bytes | os.PathLike[str], dst: str | bytes | os.PathLike[str]) -> None:
733 call_order.append("replace")
734 real_replace(src, dst)
735
736 with patch("muse.core.store.os.fsync", side_effect=t_fsync), \
737 patch("muse.core.store.os.replace", side_effect=t_replace):
738 write_branch_ref(root, "main", self._valid_cid())
739
740 fsync_idx = next((i for i, c in enumerate(call_order) if c == "fsync"), None)
741 replace_idx = next((i for i, c in enumerate(call_order) if c == "replace"), None)
742 assert fsync_idx is not None, "fsync not called in write_branch_ref"
743 assert replace_idx is not None, "replace not called in write_branch_ref"
744 assert fsync_idx < replace_idx, "fsync must precede replace"
745
746 def test_rejects_invalid_branch_name(self, tmp_path: pathlib.Path) -> None:
747 root = self._init(tmp_path)
748 with pytest.raises((ValueError, SystemExit)):
749 write_branch_ref(root, "../escape", self._valid_cid())
750
751 def test_rejects_non_hex_commit_id(self, tmp_path: pathlib.Path) -> None:
752 root = self._init(tmp_path)
753 with pytest.raises(ValueError):
754 write_branch_ref(root, "main", "z" * 64)
755
756 def test_rejects_short_commit_id(self, tmp_path: pathlib.Path) -> None:
757 root = self._init(tmp_path)
758 with pytest.raises(ValueError):
759 write_branch_ref(root, "main", "abc123")
760
761 def test_no_orphan_after_success(self, tmp_path: pathlib.Path) -> None:
762 root = self._init(tmp_path)
763 write_branch_ref(root, "main", self._valid_cid())
764 assert _tmp_files(tmp_path) == []
765
766 def test_no_orphan_after_replace_failure(self, tmp_path: pathlib.Path) -> None:
767 root = self._init(tmp_path)
768 with pytest.raises(OSError):
769 with patch("muse.core.store.os.replace", side_effect=OSError("disk full")):
770 write_branch_ref(root, "main", self._valid_cid())
771 assert _tmp_files(tmp_path) == []
772
773 def test_creates_nested_branch_path(self, tmp_path: pathlib.Path) -> None:
774 """Branches like feat/my-thing require parent dir creation."""
775 root = self._init(tmp_path)
776 cid = self._valid_cid("nested")
777 write_branch_ref(root, "feat/my-thing", cid)
778 ref_path = root / ".muse" / "refs" / "heads" / "feat" / "my-thing"
779 assert ref_path.read_text() == cid
780
781 def test_50_concurrent_refs_distinct_branches(self, tmp_path: pathlib.Path) -> None:
782 """50 concurrent writes to 50 distinct branches — all must land correctly."""
783 root = self._init(tmp_path)
784 branches = [f"agent-{i:04d}" for i in range(50)]
785 cids = {b: self._valid_cid(b) for b in branches}
786 errors: list[str] = []
787
788 def writer(branch: str) -> None:
789 try:
790 write_branch_ref(root, branch, cids[branch])
791 ref_path = root / ".muse" / "refs" / "heads" / branch
792 got = ref_path.read_text()
793 if got != cids[branch]:
794 errors.append(f"{branch}: expected {cids[branch][:8]}, got {got[:8]}")
795 except Exception as exc:
796 errors.append(f"{branch}: {exc}")
797
798 threads = [threading.Thread(target=writer, args=(b,)) for b in branches]
799 for t in threads:
800 t.start()
801 for t in threads:
802 t.join()
803
804 assert errors == [], f"Concurrent branch ref errors: {errors}"
805 assert _tmp_files(tmp_path) == []
806
807 def test_50_concurrent_refs_same_branch(self, tmp_path: pathlib.Path) -> None:
808 """50 concurrent writes to the SAME branch — last wins, no corruption."""
809 root = self._init(tmp_path)
810 cids = [self._valid_cid(f"race-{i}") for i in range(50)]
811 errors: list[str] = []
812
813 def writer(cid: str) -> None:
814 try:
815 write_branch_ref(root, "main", cid)
816 content = (root / ".muse" / "refs" / "heads" / "main").read_text()
817 if content not in cids:
818 errors.append(f"Corrupt content after write: {content!r}")
819 except Exception as exc:
820 errors.append(str(exc))
821
822 threads = [threading.Thread(target=writer, args=(c,)) for c in cids]
823 for t in threads:
824 t.start()
825 for t in threads:
826 t.join()
827
828 assert errors == [], f"Same-branch concurrent errors: {errors}"
829 assert _tmp_files(tmp_path) == []
830
831
832 # ---------------------------------------------------------------------------
833 # Tier 2a: write_merge_state — MERGE_STATE.json
834 # ---------------------------------------------------------------------------
835
836 class TestMergeStateWrite:
837 """MERGE_STATE.json records in-progress conflict state.
838
839 A corrupt file prevents muse commit from completing a conflicted merge.
840 """
841
842 def _init(self, tmp_path: pathlib.Path) -> pathlib.Path:
843 (tmp_path / ".muse").mkdir()
844 return tmp_path
845
846 def _cid(self, seed: str) -> str:
847 return fake_id(seed)
848
849 def test_writes_valid_json(self, tmp_path: pathlib.Path) -> None:
850 from muse.core.merge_engine import write_merge_state
851 root = self._init(tmp_path)
852 write_merge_state(
853 root,
854 base_commit=self._cid("base"),
855 ours_commit=self._cid("ours"),
856 theirs_commit=self._cid("theirs"),
857 conflict_paths=["a.py", "b.py"],
858 )
859 state_path = root / ".muse" / "MERGE_STATE.json"
860 data = json.loads(state_path.read_text())
861 assert data["conflict_paths"] == ["a.py", "b.py"]
862
863 def test_is_atomic(self, tmp_path: pathlib.Path) -> None:
864 """write_merge_state must funnel through write_text_atomic."""
865 from muse.core.merge_engine import write_merge_state
866 root = self._init(tmp_path)
867 called = [False]
868 real_mkstemp = tempfile.mkstemp
869
870 def tracking(dir: pathlib.Path | None = None, prefix: str = "") -> tuple[int, str]:
871 called[0] = True
872 return real_mkstemp(dir=dir, prefix=prefix)
873
874 with patch("muse.core.store.tempfile.mkstemp", side_effect=tracking):
875 write_merge_state(
876 root,
877 base_commit=self._cid("b"),
878 ours_commit=self._cid("o"),
879 theirs_commit=self._cid("t"),
880 conflict_paths=[],
881 )
882
883 assert called[0], "write_merge_state bypassed mkstemp (not atomic)"
884
885 def test_no_orphan_after_success(self, tmp_path: pathlib.Path) -> None:
886 from muse.core.merge_engine import write_merge_state
887 root = self._init(tmp_path)
888 write_merge_state(
889 root,
890 base_commit=self._cid("b"),
891 ours_commit=self._cid("o"),
892 theirs_commit=self._cid("t"),
893 conflict_paths=["x.py"],
894 )
895 assert _tmp_files(tmp_path) == []
896
897
898 # ---------------------------------------------------------------------------
899 # Tier 2b: save_rebase_state — REBASE_STATE.json
900 # ---------------------------------------------------------------------------
901
902 class TestRebaseStateWrite:
903 def _init(self, tmp_path: pathlib.Path) -> pathlib.Path:
904 (tmp_path / ".muse").mkdir()
905 return tmp_path
906
907 def _state(self) -> RebaseState:
908 cid = fake_id("rebase-c")
909 return RebaseState(
910 original_branch="main",
911 original_head=cid,
912 onto=cid,
913 remaining=[],
914 completed=[],
915 squash=False,
916 )
917
918 def test_writes_valid_json(self, tmp_path: pathlib.Path) -> None:
919 from muse.core.rebase import save_rebase_state
920 root = self._init(tmp_path)
921 save_rebase_state(root, self._state())
922 path = root / ".muse" / "REBASE_STATE.json"
923 data = json.loads(path.read_text())
924 assert data["original_branch"] == "main"
925
926 def test_is_atomic(self, tmp_path: pathlib.Path) -> None:
927 from muse.core.rebase import save_rebase_state
928 root = self._init(tmp_path)
929 called = [False]
930 real_mkstemp = tempfile.mkstemp
931
932 def tracking(dir: pathlib.Path | None = None, prefix: str = "") -> tuple[int, str]:
933 called[0] = True
934 return real_mkstemp(dir=dir, prefix=prefix)
935
936 with patch("muse.core.store.tempfile.mkstemp", side_effect=tracking):
937 save_rebase_state(root, self._state())
938
939 assert called[0], "save_rebase_state bypassed mkstemp"
940
941 def test_no_orphan_after_success(self, tmp_path: pathlib.Path) -> None:
942 from muse.core.rebase import save_rebase_state
943 root = self._init(tmp_path)
944 save_rebase_state(root, self._state())
945 assert _tmp_files(tmp_path) == []
946
947
948 # ---------------------------------------------------------------------------
949 # Tier 2c: coordination.py — reservation + intent writes
950 # ---------------------------------------------------------------------------
951
952 class TestCoordinationWrites:
953 def _init(self, tmp_path: pathlib.Path) -> pathlib.Path:
954 (tmp_path / ".muse").mkdir()
955 return tmp_path
956
957 def _res(self, root: pathlib.Path, i: int = 0) -> Reservation:
958 from muse.core.coordination import create_reservation
959 return create_reservation(
960 root,
961 run_id=f"run-{i}",
962 branch="main",
963 addresses=[f"addr-{i}"],
964 operation="write",
965 )
966
967 def test_create_reservation_is_atomic(self, tmp_path: pathlib.Path) -> None:
968 from muse.core.coordination import create_reservation
969 root = self._init(tmp_path)
970 called = [False]
971 real_mkstemp = tempfile.mkstemp
972
973 def tracking(dir: pathlib.Path | None = None, prefix: str = "") -> tuple[int, str]:
974 called[0] = True
975 return real_mkstemp(dir=dir, prefix=prefix)
976
977 with patch("muse.core.store.tempfile.mkstemp", side_effect=tracking):
978 create_reservation(root, run_id="r1", branch="main", addresses=["a"], operation="write")
979
980 assert called[0], "create_reservation bypassed mkstemp"
981
982 def test_create_reservation_writes_valid_json(self, tmp_path: pathlib.Path) -> None:
983 from muse.core.coordination import _reservations_dir
984 root = self._init(tmp_path)
985 res = self._res(root, 0)
986 res_path = _reservations_dir(root) / f"{res.reservation_id}.json"
987 data = json.loads(res_path.read_text())
988 assert data["operation"] == "write"
989
990 def test_create_intent_is_atomic(self, tmp_path: pathlib.Path) -> None:
991 from muse.core.coordination import create_intent
992 root = self._init(tmp_path)
993 res = self._res(root)
994 called = [False]
995 real_mkstemp = tempfile.mkstemp
996
997 def tracking(dir: pathlib.Path | None = None, prefix: str = "") -> tuple[int, str]:
998 called[0] = True
999 return real_mkstemp(dir=dir, prefix=prefix)
1000
1001 with patch("muse.core.store.tempfile.mkstemp", side_effect=tracking):
1002 create_intent(
1003 root,
1004 reservation_id=res.reservation_id,
1005 run_id="r1",
1006 branch="main",
1007 addresses=["a"],
1008 operation="merge",
1009 )
1010
1011 assert called[0], "create_intent bypassed mkstemp"
1012
1013 def test_create_intent_writes_valid_json(self, tmp_path: pathlib.Path) -> None:
1014 from muse.core.coordination import create_intent, _intents_dir
1015 root = self._init(tmp_path)
1016 res = self._res(root)
1017 intent = create_intent(
1018 root,
1019 reservation_id=res.reservation_id,
1020 run_id="r1",
1021 branch="main",
1022 addresses=["a"],
1023 operation="push",
1024 )
1025 intent_path = _intents_dir(root) / f"{intent.intent_id}.json"
1026 data = json.loads(intent_path.read_text())
1027 assert data["operation"] == "push"
1028
1029 def test_no_orphan_after_reservation(self, tmp_path: pathlib.Path) -> None:
1030 root = self._init(tmp_path)
1031 self._res(root)
1032 assert _tmp_files(tmp_path) == []
1033
1034 def test_no_orphan_after_intent(self, tmp_path: pathlib.Path) -> None:
1035 from muse.core.coordination import create_intent
1036 root = self._init(tmp_path)
1037 res = self._res(root)
1038 create_intent(
1039 root,
1040 reservation_id=res.reservation_id,
1041 run_id="r1",
1042 branch="main",
1043 addresses=["a"],
1044 operation="commit",
1045 )
1046 assert _tmp_files(tmp_path) == []
1047
1048 def test_20_concurrent_reservation_writes(self, tmp_path: pathlib.Path) -> None:
1049 """20 concurrent agents creating reservations — no corruption, no orphans."""
1050 from muse.core.coordination import create_reservation, _reservations_dir
1051 root = self._init(tmp_path)
1052 errors: list[str] = []
1053
1054 def writer(i: int) -> None:
1055 try:
1056 create_reservation(
1057 root,
1058 run_id=f"run-{i}",
1059 branch="main",
1060 addresses=[f"addr-{i}"],
1061 operation="write",
1062 )
1063 except Exception as exc:
1064 errors.append(str(exc))
1065
1066 threads = [threading.Thread(target=writer, args=(i,)) for i in range(20)]
1067 for t in threads:
1068 t.start()
1069 for t in threads:
1070 t.join()
1071
1072 assert errors == [], f"Concurrent reservation errors: {errors}"
1073 reservation_files = list(_reservations_dir(root).glob("*.json"))
1074 assert len(reservation_files) == 20, f"Expected 20 reservation files, got {len(reservation_files)}"
1075 assert _tmp_files(tmp_path) == []
1076
1077
1078 # ---------------------------------------------------------------------------
1079 # Tier 3: config.py — TOML config writes
1080 # ---------------------------------------------------------------------------
1081
1082 class TestConfigWrites:
1083 """Config files govern remote connections, auth, and repo settings.
1084
1085 A corrupt config.toml prevents all repo operations.
1086 """
1087
1088 def _init_config_repo(self, tmp_path: pathlib.Path) -> pathlib.Path:
1089 """Create a minimal repo with config.toml so config helpers can operate."""
1090 muse_dir = tmp_path / ".muse"
1091 muse_dir.mkdir()
1092 (muse_dir / "objects").mkdir()
1093 (muse_dir / "commits").mkdir()
1094 (muse_dir / "snapshots").mkdir()
1095 (muse_dir / "refs" / "heads").mkdir(parents=True)
1096 (muse_dir / "repo.json").write_text(
1097 '{"repo_id": "test-repo", "domain": "code", "default_branch": "main"}',
1098 encoding="utf-8",
1099 )
1100 (muse_dir / "config.toml").write_text("", encoding="utf-8")
1101 (muse_dir / "HEAD").write_text("ref: refs/heads/main\n", encoding="utf-8")
1102 return tmp_path
1103
1104 def test_set_remote_is_atomic(self, tmp_path: pathlib.Path) -> None:
1105 """set_remote (writes config.toml) must funnel through write_text_atomic."""
1106 from muse.cli.config import set_remote
1107 root = self._init_config_repo(tmp_path)
1108 called = [False]
1109 real_mkstemp = tempfile.mkstemp
1110
1111 def tracking(dir: pathlib.Path | None = None, prefix: str = "") -> tuple[int, str]:
1112 called[0] = True
1113 return real_mkstemp(dir=dir, prefix=prefix)
1114
1115 with patch("muse.core.store.tempfile.mkstemp", side_effect=tracking):
1116 set_remote("local", "https://localhost:1337", repo_root=root)
1117
1118 assert called[0], "set_remote bypassed mkstemp (config write not atomic)"
1119
1120 def test_set_remote_no_orphan(self, tmp_path: pathlib.Path) -> None:
1121 from muse.cli.config import set_remote
1122 root = self._init_config_repo(tmp_path)
1123 set_remote("origin", "https://localhost:1337", repo_root=root)
1124 assert _tmp_files(tmp_path) == []
1125
1126 def test_set_remote_correct_content_persisted(self, tmp_path: pathlib.Path) -> None:
1127 from muse.cli.config import set_remote, get_remote
1128 root = self._init_config_repo(tmp_path)
1129 set_remote("myremote", "http://myhost:9000", repo_root=root)
1130 url = get_remote("myremote", repo_root=root)
1131 assert url == "http://myhost:9000"
1132
1133 def test_10_concurrent_config_writes_no_orphan(self, tmp_path: pathlib.Path) -> None:
1134 """10 concurrent set_remote calls — no orphan temp files."""
1135 from muse.cli.config import set_remote
1136 root = self._init_config_repo(tmp_path)
1137 errors: list[str] = []
1138
1139 def writer(i: int) -> None:
1140 try:
1141 set_remote(f"remote-{i}", f"http://host-{i}:9000", repo_root=root)
1142 except Exception as exc:
1143 errors.append(str(exc))
1144
1145 threads = [threading.Thread(target=writer, args=(i,)) for i in range(10)]
1146 for t in threads:
1147 t.start()
1148 for t in threads:
1149 t.join()
1150
1151 # No orphan temps regardless of config merge conflicts
1152 assert _tmp_files(tmp_path) == []
1153
1154
1155 # ---------------------------------------------------------------------------
1156 # Gap 2+3: write_object_from_path — fsync ordering + copy2 failure cleanup
1157 # ---------------------------------------------------------------------------
1158
1159 class TestWriteObjectFromPathFsync:
1160 """_fsync_fd must be called before os.replace in write_object_from_path.
1161
1162 write_object_from_path uses shutil.copy2 then re-opens the temp file as
1163 an fd to call fchmod + _fsync_fd before the atomic rename. The test
1164 patches _fsync_fd (the fd-based variant) — NOT _fsync_path, which is the
1165 path-based variant used only by restore_object.
1166 """
1167
1168 def test_fsync_path_called_before_replace(self, tmp_path: pathlib.Path) -> None:
1169 """_fsync_fd must be invoked before os.replace."""
1170 repo = _repo(tmp_path)
1171 data = b"from-path fsync ordering"
1172 oid = _oid(data)
1173 src = tmp_path / "source.bin"
1174 src.write_bytes(data)
1175
1176 call_order: list[str] = []
1177 real_fsync_fd = __import__("muse.core.object_store", fromlist=["_fsync_fd"])._fsync_fd
1178 real_replace = os.replace
1179
1180 def t_fsync_fd(fd: int) -> None:
1181 call_order.append("fsync_fd")
1182 real_fsync_fd(fd)
1183
1184 def t_replace(s: str | bytes | os.PathLike[str], d: str | bytes | os.PathLike[str]) -> None:
1185 call_order.append("replace")
1186 real_replace(s, d)
1187
1188 with patch("muse.core.object_store._fsync_fd", side_effect=t_fsync_fd), \
1189 patch("muse.core.object_store.os.replace", side_effect=t_replace):
1190 write_object_from_path(repo, oid, src)
1191
1192 fp = next((i for i, c in enumerate(call_order) if c == "fsync_fd"), None)
1193 rp = next((i for i, c in enumerate(call_order) if c == "replace"), None)
1194 assert fp is not None, "_fsync_fd never called in write_object_from_path"
1195 assert rp is not None, "os.replace never called in write_object_from_path"
1196 assert fp < rp, f"_fsync_fd (pos {fp}) must happen before replace (pos {rp})"
1197
1198 def test_fsync_path_failure_non_fatal(self, tmp_path: pathlib.Path) -> None:
1199 """os.fsync failure inside _fsync_fd must not abort write_object_from_path.
1200
1201 _fsync_fd swallows OSError internally — we patch os.fsync so the
1202 function's own try/except absorbs the failure, exactly as it would on a
1203 filesystem that does not support fsync (tmpfs, some Docker volumes).
1204 """
1205 repo = _repo(tmp_path)
1206 data = b"fsync_path fails gracefully"
1207 oid = _oid(data)
1208 src = tmp_path / "src.bin"
1209 src.write_bytes(data)
1210
1211 with patch("muse.core.object_store.os.fsync", side_effect=OSError("not supported")):
1212 result = write_object_from_path(repo, oid, src)
1213
1214 assert result is True
1215 assert read_object(repo, oid) == data
1216
1217 def test_no_orphan_after_copy2_failure(self, tmp_path: pathlib.Path) -> None:
1218 """When shutil.copy2 raises, the mkstemp temp file must be cleaned up."""
1219 import shutil
1220 repo = _repo(tmp_path)
1221 data = b"copy2 will fail"
1222 oid = _oid(data)
1223 src = tmp_path / "src.bin"
1224 src.write_bytes(data)
1225
1226 with pytest.raises(OSError):
1227 with patch("muse.core.object_store.shutil.copy2", side_effect=OSError("I/O error")):
1228 write_object_from_path(repo, oid, src)
1229
1230 assert _tmp_files(tmp_path) == [], "Orphan temp after shutil.copy2 failure"
1231
1232 def test_no_orphan_after_replace_failure(self, tmp_path: pathlib.Path) -> None:
1233 """When os.replace raises, the temp file must be cleaned up."""
1234 repo = _repo(tmp_path)
1235 data = b"replace will fail for from_path"
1236 oid = _oid(data)
1237 src = tmp_path / "src.bin"
1238 src.write_bytes(data)
1239
1240 with pytest.raises(OSError):
1241 with patch("muse.core.object_store.os.replace", side_effect=OSError("disk full")):
1242 write_object_from_path(repo, oid, src)
1243
1244 assert _tmp_files(tmp_path) == [], "Orphan temp after os.replace failure in write_object_from_path"
1245
1246 def test_correct_content_after_write(self, tmp_path: pathlib.Path) -> None:
1247 """Content round-trips correctly through write_object_from_path → read_object."""
1248 repo = _repo(tmp_path)
1249 data = os.urandom(4096)
1250 oid = _oid(data)
1251 src = tmp_path / "payload.bin"
1252 src.write_bytes(data)
1253
1254 write_object_from_path(repo, oid, src)
1255 assert read_object(repo, oid) == data
1256
1257
1258 # ---------------------------------------------------------------------------
1259 # Gap 4+5+6: restore_object — fsync ordering + copy2 failure cleanup
1260 # ---------------------------------------------------------------------------
1261
1262 class TestRestoreObjectFsync:
1263 """_fsync_path must be called before os.replace in restore_object."""
1264
1265 def test_fsync_path_called_before_replace(self, tmp_path: pathlib.Path) -> None:
1266 repo = _repo(tmp_path)
1267 data = b"restore fsync ordering"
1268 oid = _oid(data)
1269 write_object(repo, oid, data)
1270 dest = tmp_path / "restored.bin"
1271
1272 call_order: list[str] = []
1273 real_fsync_path = __import__("muse.core.object_store", fromlist=["_fsync_path"])._fsync_path
1274 real_replace = os.replace
1275
1276 def t_fsync_path(path: pathlib.Path) -> None:
1277 call_order.append("fsync_path")
1278 real_fsync_path(path)
1279
1280 def t_replace(s: str | bytes | os.PathLike[str], d: str | bytes | os.PathLike[str]) -> None:
1281 call_order.append("replace")
1282 real_replace(s, d)
1283
1284 with patch("muse.core.object_store._fsync_path", side_effect=t_fsync_path), \
1285 patch("muse.core.object_store.os.replace", side_effect=t_replace):
1286 restore_object(repo, oid, dest)
1287
1288 fp = next((i for i, c in enumerate(call_order) if c == "fsync_path"), None)
1289 rp = next((i for i, c in enumerate(call_order) if c == "replace"), None)
1290 assert fp is not None, "_fsync_path never called in restore_object"
1291 assert rp is not None, "os.replace never called in restore_object"
1292 assert fp < rp, f"_fsync_path (pos {fp}) must precede replace (pos {rp})"
1293
1294 def test_fsync_path_failure_non_fatal(self, tmp_path: pathlib.Path) -> None:
1295 """os.fsync failure inside _fsync_path must not abort restore_object."""
1296 repo = _repo(tmp_path)
1297 data = b"restore fsync_path fails gracefully"
1298 oid = _oid(data)
1299 write_object(repo, oid, data)
1300 dest = tmp_path / "restored.bin"
1301
1302 with patch("muse.core.object_store.os.fsync", side_effect=OSError("not supported")):
1303 result = restore_object(repo, oid, dest)
1304
1305 assert result is True
1306 assert dest.read_bytes() == data
1307
1308 def test_no_orphan_after_copy2_failure(self, tmp_path: pathlib.Path) -> None:
1309 repo = _repo(tmp_path)
1310 data = b"restore copy2 will fail"
1311 oid = _oid(data)
1312 write_object(repo, oid, data)
1313 dest = tmp_path / "out.bin"
1314
1315 with pytest.raises(OSError):
1316 with patch("muse.core.object_store.shutil.copy2", side_effect=OSError("I/O error")):
1317 restore_object(repo, oid, dest)
1318
1319 assert _tmp_files(tmp_path) == [], "Orphan temp after copy2 failure in restore_object"
1320
1321 def test_no_orphan_after_replace_failure(self, tmp_path: pathlib.Path) -> None:
1322 repo = _repo(tmp_path)
1323 data = b"restore replace will fail"
1324 oid = _oid(data)
1325 write_object(repo, oid, data)
1326 dest = tmp_path / "out.bin"
1327
1328 with pytest.raises(OSError):
1329 with patch("muse.core.object_store.os.replace", side_effect=OSError("disk full")):
1330 restore_object(repo, oid, dest)
1331
1332 assert _tmp_files(tmp_path) == [], "Orphan temp after os.replace failure in restore_object"
1333
1334 def test_restored_file_mtime_is_current_not_from_object_store(
1335 self, tmp_path: pathlib.Path
1336 ) -> None:
1337 """restore_object must set the destination mtime to NOW, not to the
1338 object-store file's mtime.
1339
1340 shutil.copy2 propagates the source (object-store) mtime to the temp
1341 file. Object-store files are written at commit time and may be days
1342 or weeks old. Without os.utime(tmp, None) the restored destination
1343 carries an old timestamp, causing editors (Cursor, VS Code, Vim) to
1344 see "new mtime < cached mtime" and serve a stale buffer instead of
1345 reloading the file. This is the regression that caused the
1346 "merge work disappears in Cursor but reappears on close/reopen" bug.
1347 """
1348 import time
1349
1350 repo = _repo(tmp_path)
1351 data = b"content that differs from any existing file\n" * 10
1352 oid = _oid(data)
1353 write_object(repo, oid, data)
1354
1355 # Simulate an old object-store mtime: backdate the stored object to 2 days ago.
1356 obj_path = object_path(repo, oid)
1357 two_days_ago = time.time() - (2 * 24 * 3600)
1358 os.utime(obj_path, (two_days_ago, two_days_ago))
1359
1360 # Write a pre-existing dest with a "current" mtime (simulating Cursor's
1361 # last-read timestamp before the checkout/merge).
1362 dest = tmp_path / "watched_file.py"
1363 dest.write_bytes(b"old content that cursor has open\n")
1364 cursor_cached_mtime = time.time()
1365 os.utime(dest, (cursor_cached_mtime, cursor_cached_mtime))
1366
1367 # restore_object must write the new content AND freshen mtime.
1368 t_before = time.time()
1369 restore_object(repo, oid, dest)
1370 t_after = time.time()
1371
1372 new_mtime = os.stat(dest).st_mtime
1373
1374 # Destination must have a FRESH timestamp — not the object-store's old one.
1375 assert new_mtime >= t_before, (
1376 f"Restored file mtime ({new_mtime:.2f}) is older than the time "
1377 f"restore_object was called ({t_before:.2f}). "
1378 "shutil.copy2 is propagating the object-store's stale mtime, "
1379 "which causes editors to serve stale buffers after checkout/merge."
1380 )
1381 assert new_mtime <= t_after + 1.0, (
1382 f"Restored file mtime ({new_mtime:.2f}) is far in the future — unexpected."
1383 )
1384
1385 # Content must be correct regardless.
1386 assert dest.read_bytes() == data
1387
1388 def test_restored_mtime_fresher_than_previous_content(
1389 self, tmp_path: pathlib.Path
1390 ) -> None:
1391 """After restore_object, the destination mtime must be >= the mtime it
1392 had before the call, so editors always see a forward-moving timestamp."""
1393 import time
1394
1395 repo = _repo(tmp_path)
1396 new_data = b"new version from feature branch\n"
1397 oid = _oid(new_data)
1398 write_object(repo, oid, new_data)
1399
1400 dest = tmp_path / "file.py"
1401 dest.write_bytes(b"old version on dev\n")
1402 old_mtime = time.time()
1403 os.utime(dest, (old_mtime, old_mtime))
1404
1405 # Backdate the object-store copy (as it would be after a real commit).
1406 obj_path = object_path(repo, oid)
1407 os.utime(obj_path, (old_mtime - 86400, old_mtime - 86400))
1408
1409 restore_object(repo, oid, dest)
1410
1411 assert os.stat(dest).st_mtime >= old_mtime, (
1412 "Restored file mtime went backwards — editor will not see the change."
1413 )
1414
1415
1416 # ---------------------------------------------------------------------------
1417 # Gap 7: page-cache non-flush — defense-in-depth (I-1 catches what I-2 misses)
1418 # ---------------------------------------------------------------------------
1419
1420 class TestPageCacheDefenseInDepth:
1421 """Demonstrate that I-1 (read-time hash verification) is the safety net
1422 for the unlikely scenario where fsync appeared to succeed but the kernel
1423 wrote zero bytes to disk (power loss AFTER rename, BEFORE flush).
1424
1425 Simulated by: writing an object normally, then zeroing the on-disk file
1426 (mimicking a power-loss-induced empty file at the renamed destination).
1427 read_object must raise OSError — the store never silently serves bad data.
1428 """
1429
1430 def test_zeroed_dest_after_rename_caught_by_read(self, tmp_path: pathlib.Path) -> None:
1431 """Simulate post-rename page-cache loss: zero the stored file, then read."""
1432 repo = _repo(tmp_path)
1433 data = b"page cache simulation"
1434 oid = _oid(data)
1435 write_object(repo, oid, data)
1436
1437 # Mimic power loss that zeroed the file after rename.
1438 _corrupt_file(object_path(repo, oid), b"\x00" * len(data))
1439
1440 with pytest.raises(OSError, match="integrity check"):
1441 read_object(repo, oid)
1442
1443 def test_truncated_dest_caught_by_read(self, tmp_path: pathlib.Path) -> None:
1444 """Simulate partial flush: only first half of bytes survived power loss."""
1445 repo = _repo(tmp_path)
1446 data = b"partial flush simulation" * 10
1447 oid = _oid(data)
1448 write_object(repo, oid, data)
1449 # Only the first half survived to disk.
1450 _corrupt_file(object_path(repo, oid), data[: len(data) // 2])
1451
1452 with pytest.raises(OSError, match="integrity check"):
1453 read_object(repo, oid)
1454
1455 def test_noop_write_detected(self, tmp_path: pathlib.Path) -> None:
1456 """Simulate fh.write no-op (page cache accepted write, never flushed).
1457
1458 We write the object normally and then zero the stored file to mimic the
1459 outcome of a post-rename page-cache flush failure. read_object must
1460 raise OSError — I-1's hash check is the final safety net for any I-2
1461 failure mode.
1462 """
1463 repo = _repo(tmp_path)
1464 data = b"write syscall accepted but page cache never flushed"
1465 oid = _oid(data)
1466
1467 # Write correctly first, then simulate the power-loss outcome: the
1468 # renamed destination was never actually flushed to durable storage.
1469 write_object(repo, oid, data)
1470 stored = object_path(repo, oid)
1471 _corrupt_file(stored, b"") # zero bytes — what a power loss leaves
1472
1473 with pytest.raises(OSError, match="integrity check"):
1474 read_object(repo, oid)
1475
1476
1477 # ---------------------------------------------------------------------------
1478 # Gap 8: same object_id written from N threads simultaneously — idempotency
1479 # ---------------------------------------------------------------------------
1480
1481 class TestIdempotentConcurrentWrite:
1482 """write_object is idempotent: same object_id written from many threads
1483 concurrently must never produce corruption — only one write wins, others
1484 see exists() and skip. The content of the winner must be correct.
1485 """
1486
1487 def test_same_object_50_threads_no_corruption(self, tmp_path: pathlib.Path) -> None:
1488 """50 threads writing the same object_id must all succeed with correct content."""
1489 repo = _repo(tmp_path)
1490 data = b"idempotent object written from 50 threads"
1491 oid = _oid(data)
1492 errors: list[str] = []
1493
1494 def writer() -> None:
1495 try:
1496 write_object(repo, oid, data)
1497 result = read_object(repo, oid)
1498 if result != data:
1499 errors.append(f"Mismatch: {repr(result)[:30]}")
1500 except Exception as exc:
1501 errors.append(f"Exception: {exc}")
1502
1503 threads = [threading.Thread(target=writer) for _ in range(50)]
1504 for t in threads:
1505 t.start()
1506 for t in threads:
1507 t.join()
1508
1509 assert errors == [], f"Idempotent concurrent write errors:\n" + "\n".join(errors)
1510 assert read_object(repo, oid) == data
1511 assert _tmp_files(tmp_path) == []
1512
1513 def test_same_object_distinct_content_rejected(self, tmp_path: pathlib.Path) -> None:
1514 """Writing different bytes under the same object_id is always rejected."""
1515 repo = _repo(tmp_path)
1516 data = b"canonical content"
1517 oid = _oid(data)
1518 wrong = b"wrong content that hashes differently"
1519
1520 write_object(repo, oid, data)
1521
1522 with pytest.raises(ValueError, match="integrity"):
1523 write_object(repo, oid, wrong)
1524
1525 assert read_object(repo, oid) == data
1526
1527
1528 # ---------------------------------------------------------------------------
1529 # Gap 9: mid-write fh.write failure — orphan cleaned up
1530 # ---------------------------------------------------------------------------
1531
1532 class TestMidWriteFailureCleanup:
1533 """OSError raised during the fh.write call (disk full mid-write) must not
1534 leave an orphan temp file in the store directory."""
1535
1536 def test_no_orphan_after_write_failure_write_object(self, tmp_path: pathlib.Path) -> None:
1537 """OSError during fh.write must not leave an orphan temp file."""
1538 from unittest.mock import MagicMock
1539 repo = _repo(tmp_path)
1540 data = b"mid-write failure"
1541 oid = _oid(data)
1542
1543 mock_fh = MagicMock()
1544 mock_fh.__enter__.return_value = mock_fh
1545 mock_fh.write.side_effect = OSError("disk full")
1546 mock_fh.flush.return_value = None
1547 mock_fh.fileno.return_value = -1
1548
1549 with pytest.raises(OSError):
1550 with patch("muse.core.object_store.os.fdopen", return_value=mock_fh):
1551 write_object(repo, oid, data)
1552
1553 assert _tmp_files(tmp_path) == [], "Orphan temp file left after mid-write failure"
1554
1555 def test_no_orphan_after_write_failure_write_text_atomic(self, tmp_path: pathlib.Path) -> None:
1556 """write_text_atomic cleans up the temp file when fh.write raises."""
1557 from unittest.mock import MagicMock
1558 path = tmp_path / "state.txt"
1559
1560 mock_fh = MagicMock()
1561 mock_fh.__enter__.return_value = mock_fh
1562 mock_fh.write.side_effect = OSError("disk full")
1563 mock_fh.flush.return_value = None
1564 mock_fh.fileno.return_value = -1
1565
1566 with pytest.raises(OSError):
1567 with patch("muse.core.store.os.fdopen", return_value=mock_fh):
1568 write_text_atomic(path, "will fail")
1569
1570 assert _tmp_files(tmp_path) == [], "Orphan temp left after write_text_atomic mid-write failure"
1571
1572
1573 # ---------------------------------------------------------------------------
1574 # Gap 10: 10 000 sequential commits — store clean throughout
1575 # ---------------------------------------------------------------------------
1576
1577 class TestSequentialStress:
1578 """10 000 sequential commit writes exercise the full fsync+rename path at
1579 scale. The store must be clean (all readable, no orphans) when done.
1580
1581 Based on the Linux-kernel-migration scenario: Linus runs a git-to-muse
1582 import script that writes 75k commits. We test at 10k to keep CI fast.
1583 """
1584
1585 @pytest.mark.slow
1586 def test_10000_sequential_commits_all_readable(self, tmp_path: pathlib.Path) -> None:
1587 """1 000 sequential commits — every one must be readable after write."""
1588 repo = _repo(tmp_path)
1589 commits = [_commit(i) for i in range(1_000)]
1590
1591 for c in commits:
1592 write_commit(repo, c)
1593
1594 # Verify every commit is readable and correct.
1595 failures: list[str] = []
1596 for c in commits:
1597 result = read_commit(repo, c.commit_id)
1598 if result is None:
1599 failures.append(f"Commit {c.commit_id[:8]} not found after write")
1600 elif result.message != c.message:
1601 failures.append(f"Commit {c.commit_id[:8]} message corrupted")
1602
1603 assert failures == [], f"{len(failures)} commit read failures:\n" + "\n".join(failures[:10])
1604 assert _tmp_files(tmp_path) == [], "Orphan temps after sequential commit writes"
1605
1606 @pytest.mark.slow
1607 def test_1000_commits_with_20pct_fsync_failure_all_readable(
1608 self, tmp_path: pathlib.Path
1609 ) -> None:
1610 """100 commits with 20% random fsync failures must all land correctly.
1611
1612 Verifies that fsync failure is gracefully handled and atomicity (torn-write
1613 protection) is maintained even when durability (fsync) is degraded.
1614 """
1615 import random as _random
1616 repo = _repo(tmp_path)
1617 rng = _random.Random(42)
1618 commits = [_commit(i) for i in range(100)]
1619 real_fsync = os.fsync
1620
1621 def flaky_fsync(fd: int) -> None:
1622 if rng.random() < 0.2:
1623 raise OSError("simulated fsync failure")
1624 real_fsync(fd)
1625
1626 with patch("muse.core.store.os.fsync", side_effect=flaky_fsync):
1627 for c in commits:
1628 write_commit(repo, c)
1629
1630 failures: list[str] = []
1631 for c in commits:
1632 result = read_commit(repo, c.commit_id)
1633 if result is None:
1634 failures.append(f"Commit {c.commit_id[:8]} not found")
1635 elif result.message != c.message:
1636 failures.append(f"Commit {c.commit_id[:8]} corrupted")
1637
1638 assert failures == [], f"Commits lost under flaky fsync:\n" + "\n".join(failures)
1639 assert _tmp_files(tmp_path) == []
1640
1641
1642 # ---------------------------------------------------------------------------
1643 # Gap 11: SIGKILL crash safety — process kill leaves no orphans
1644 # ---------------------------------------------------------------------------
1645
1646 class TestProcessKillCrashSafety:
1647 """Simulate abrupt process termination (SIGKILL) during an object write.
1648
1649 Uses multiprocessing to run the writer in a child process, then kills it
1650 with SIGKILL at a random moment. Afterward, the store must be consistent:
1651 - Objects fully written before the kill must be readable and hash-correct.
1652 - No orphan temp files must remain (OS cleans up open fds; the temp file
1653 created by mkstemp is unlinked by the OS when the process dies, since
1654 it holds the only reference via the fd).
1655
1656 Note: On most POSIX systems, a SIGKILL'd process that holds an open
1657 mkstemp fd will have that fd closed by the kernel. The temp file remains
1658 on disk (the fd close doesn't unlink it) but the rename never happens, so
1659 the destination is either fully written or absent — never partial.
1660 This test verifies the store consistency guarantee, not orphan cleanup
1661 (orphan GC is a separate I-6 concern).
1662 """
1663
1664 @pytest.mark.slow
1665 def test_sigkill_during_write_leaves_no_partial_dest(
1666 self, tmp_path: pathlib.Path
1667 ) -> None:
1668 """Objects written before SIGKILL must still be readable after kill."""
1669 import multiprocessing
1670 import signal
1671 import time
1672
1673 repo = _repo(tmp_path)
1674
1675 # Write 20 known objects before spawning the crashable process.
1676 pre_oids: list[str] = []
1677 for i in range(20):
1678 data = f"pre-kill-{i}".encode()
1679 oid = _oid(data)
1680 write_object(repo, oid, data)
1681 pre_oids.append(oid)
1682
1683 # "spawn" starts a fresh interpreter — no multi-threaded-fork warning
1684 # and no risk of deadlocks inherited from the pytest runner's threads.
1685 # _sigkill_writer_worker is defined at module level to ensure it is
1686 # picklable across the spawn boundary.
1687 ctx = multiprocessing.get_context("spawn")
1688 proc = ctx.Process(target=_sigkill_writer_worker, args=(repo, 5000))
1689 proc.start()
1690
1691 # Kill the worker after a short random delay.
1692 import random
1693 time.sleep(random.uniform(0.01, 0.05))
1694 if proc.is_alive():
1695 assert proc.pid is not None
1696 os.kill(proc.pid, signal.SIGKILL)
1697 proc.join()
1698
1699 # All pre-kill objects must still be readable and correct.
1700 for i, oid in enumerate(pre_oids):
1701 data = f"pre-kill-{i}".encode()
1702 result = read_object(repo, oid)
1703 assert result == data, f"Pre-kill object {oid[:8]} corrupted after SIGKILL"
1704
1705 # Store consistency: every object file in the store must hash-verify.
1706 import muse.core.object_store as _ost
1707 all_oids = _ost._iter_all_object_ids(repo) if hasattr(_ost, "_iter_all_object_ids") else []
1708 for oid in all_oids:
1709 try:
1710 read_object(repo, oid) # raises on hash mismatch
1711 except OSError as exc:
1712 pytest.fail(f"Corrupt object {oid[:8]} found after SIGKILL: {exc}")
1713
1714
1715 # ---------------------------------------------------------------------------
1716 # Gap 12: Performance benchmark — 4 KiB msgpack write + fsync < 5 ms
1717 # ---------------------------------------------------------------------------
1718
1719 class TestFsyncWritePerformance:
1720 """fsync overhead on a 4 KiB msgpack file must be < 5 ms.
1721
1722 The syscall is dominated by the OS flush latency, not the data volume.
1723 tmpfs (which tmp_path typically uses on Linux) syncs instantly; on macOS
1724 with APFS this is also sub-millisecond. 5 ms is a very generous budget —
1725 a real NVMe commit flush is typically < 0.5 ms.
1726 """
1727
1728 @pytest.mark.perf
1729 def test_write_commit_4kib_under_5ms(self, tmp_path: pathlib.Path) -> None:
1730 """Single 4 KiB commit write (msgpack + fsync + rename) < 5 ms."""
1731 import time
1732 repo = _repo(tmp_path)
1733 c = _commit(99_000)
1734
1735 start = time.perf_counter()
1736 write_commit(repo, c)
1737 duration_ms = (time.perf_counter() - start) * 1000
1738
1739 assert read_commit(repo, c.commit_id) is not None
1740 assert duration_ms < 10, (
1741 f"write_commit took {duration_ms:.2f} ms — exceeds the 10 ms fsync budget. "
1742 "Performance regression in the atomic write path."
1743 )
1744
1745 @pytest.mark.perf
1746 def test_write_object_4kib_under_5ms(self, tmp_path: pathlib.Path) -> None:
1747 """Single 4 KiB object write (bytes + fsync + rename) < 5 ms."""
1748 import time
1749 repo = _repo(tmp_path)
1750 data = os.urandom(4096)
1751 oid = _oid(data)
1752
1753 start = time.perf_counter()
1754 write_object(repo, oid, data)
1755 duration_ms = (time.perf_counter() - start) * 1000
1756
1757 assert read_object(repo, oid) == data
1758 assert duration_ms < 10, (
1759 f"write_object took {duration_ms:.2f} ms — exceeds the 10 ms fsync budget."
1760 )
1761
1762 @pytest.mark.perf
1763 def test_write_text_atomic_4kib_under_5ms(self, tmp_path: pathlib.Path) -> None:
1764 """write_text_atomic on a 4 KiB text blob < 5 ms."""
1765 import time
1766 path = tmp_path / "state.txt"
1767 text = "x" * 4096
1768
1769 start = time.perf_counter()
1770 write_text_atomic(path, text)
1771 duration_ms = (time.perf_counter() - start) * 1000
1772
1773 assert path.read_text() == text
1774 assert duration_ms < 10, (
1775 f"write_text_atomic took {duration_ms:.2f} ms — exceeds the 10 ms budget."
1776 )
File History 3 commits
sha256:b636f72dcba9e190afb980bece906fa5b717fbde014b76ef023df8cb96e01eb9 docs: expand cache plan with all seven testing tiers and do… Sonnet 4.6 131 days ago
sha256:7f9e2ef5286aedad9c1e6011b4c46ca27f39dbdad6e3409357e36b26e46b3b7c docs: docstring sprint for-each-ref→hotspots — idiomatic ru… Sonnet 4.6 patch 137 days ago
sha256:a09b1b4f6838754495547f200aa0ce88e2f56ffc5b20b900f6f0cff2c3cdede9 fix(cursorignore): remove git-ism (.git/worktrees) Human minor ⚠ 140 days ago