gabriel / muse public
test_integrity_I9_sigkill.py python
960 lines 36.4 KB
Raw
sha256:88ac91129873e6a496e9189515aa690eb893ae25d69c8f72af141a2be5068eb3 docs: docstring sprint contract→find-symbol — idiomatic run… Sonnet 4.6 patch 139 days ago
1 """I-9 — Crash safety: SIGKILL simulation and startup GC sweep.
2
3 Validates three guarantees:
4
5 1. **Startup GC correctness** — :func:`muse.core.repo.require_repo` sweeps
6 every stale temp-file family on the next command after a crash:
7 * ``.obj-tmp-*`` / ``.restore-tmp-*`` — object-store shard directories
8 * ``.muse-tmp-*`` — store/config writes in ``.muse/`` subdirectories
9 * ``.stat_cache_*.tmp`` — :class:`~muse.core.stat_cache.StatCache`
10
11 2. **SIGKILL safety at timing windows** — a process killed at T+50 ms,
12 T+100 ms, and T+200 ms into a write sequence leaves the repository in a
13 consistent state. The subsequent startup GC removes any orphan temps so
14 the *next* ``muse commit`` succeeds.
15
16 3. **Push idempotency under SIGKILL** — a partial push leaves no corruption
17 on the remote because :func:`~muse.core.object_store.write_object` is
18 content-addressed and atomic; incomplete object writes are swept by the
19 remote-side startup GC.
20
21 Test classes
22 ------------
23 * ``TestCleanupMuseDirTemps`` — unit tests for ``_cleanup_muse_dir_temps``
24 * ``TestStartupGcObjectTemps`` — object-store orphan files swept by GC
25 * ``TestStartupGcMuseTemps`` — ``.muse-tmp-*`` files swept by GC
26 * ``TestStartupGcStatCacheTemps`` — ``.stat_cache_*.tmp`` swept by GC
27 * ``TestRequireRepoCallsGc`` — ``require_repo`` triggers the sweep
28 * ``TestMultipleSigkills`` — stale files from *N* crashes all swept
29 * ``TestSigkillAtTimingWindows`` — subprocess SIGKILL at T+50/100/200 ms
30 * ``TestSigkillDuringCommit`` — full CLI commit survives SIGKILL
31 * ``TestSigkillDuringPush`` — push path is idempotent under SIGKILL
32 * ``TestGcPreservesRealObjects`` — GC never deletes valid stored objects
33 * ``TestGcSweeperPerformance`` — sweep ≤ 10 ms with 1 000 stale files
34 * ``TestRestoreTempWorkdirBound`` — restore-tmp in workdir: documented scope
35 """
36
37 from __future__ import annotations
38
39 import hashlib
40 import multiprocessing
41 import os
42 import pathlib
43 import signal
44 import tempfile
45 import time
46
47 import pytest
48
49 from muse.core._types import blob_id
50 from muse.core.object_store import (
51 cleanup_stale_object_temps,
52 object_path,
53 objects_dir,
54 read_object,
55 write_object,
56 )
57 from muse.core.repo import (
58 _MUSE_TEMP_PREFIXES,
59 _MUSE_SWEEP_DIRS,
60 _cleanup_muse_dir_temps,
61 _startup_gc,
62 require_repo,
63 )
64 from muse.core.store import write_text_atomic
65
66
67 # ---------------------------------------------------------------------------
68 # Helpers
69 # ---------------------------------------------------------------------------
70
71
72 def _repo(tmp_path: pathlib.Path) -> pathlib.Path:
73 """Minimal .muse/ layout for unit tests."""
74 muse = tmp_path / ".muse"
75 muse.mkdir()
76 (muse / "commits").mkdir()
77 (muse / "snapshots").mkdir()
78 (muse / "branches").mkdir()
79 (muse / "refs").mkdir()
80 (muse / "refs" / "heads").mkdir()
81 (muse / "objects").mkdir()
82 return tmp_path
83
84
85 def _oid(data: bytes) -> str:
86 return blob_id(data)
87
88
89 def _shard(repo: pathlib.Path, prefix: str) -> pathlib.Path:
90 """Return the canonical shard directory for a two-char hex prefix."""
91 return objects_dir(repo) / "sha256" / prefix
92
93
94 def _plant_stale_muse_tmp(muse_dir: pathlib.Path, subdir: str = "") -> pathlib.Path:
95 """Create a fake .muse-tmp-* file as would be left by a SIGKILL'd write_text_atomic."""
96 target = muse_dir / subdir if subdir else muse_dir
97 target.mkdir(parents=True, exist_ok=True)
98 fd, path = tempfile.mkstemp(dir=target, prefix=".muse-tmp-")
99 os.close(fd)
100 pathlib.Path(path).write_text("partial content", encoding="utf-8")
101 return pathlib.Path(path)
102
103
104 def _plant_stale_stat_cache_tmp(muse_dir: pathlib.Path) -> pathlib.Path:
105 """Create a fake .stat_cache_*.tmp file as would be left by a SIGKILL'd StatCache.save."""
106 fd, path = tempfile.mkstemp(dir=muse_dir, prefix=".stat_cache_", suffix=".tmp")
107 os.close(fd)
108 pathlib.Path(path).write_bytes(b"\x00" * 64)
109 return pathlib.Path(path)
110
111
112 def _make_stale(path: pathlib.Path) -> pathlib.Path:
113 """Backdate *path* mtime past the 60-second age gate in cleanup_stale_object_temps.
114
115 cleanup_stale_object_temps skips files younger than _CLEANUP_MIN_AGE_SECS (60 s).
116 Setting mtime to the Unix epoch (1970-01-01) makes freshly-created temp files
117 look decades old so cleanup picks them up immediately in tests.
118 """
119 os.utime(str(path), (0, 0))
120 return path
121
122
123 def _plant_stale_obj_tmp(objects_shard: pathlib.Path) -> pathlib.Path:
124 """Create a fake .obj-tmp-* file as would be left by a SIGKILL'd write_object."""
125 fd, path = tempfile.mkstemp(dir=objects_shard, prefix=".obj-tmp-")
126 os.close(fd)
127 pathlib.Path(path).write_bytes(b"partial object bytes")
128 return _make_stale(pathlib.Path(path))
129
130
131 def _plant_stale_restore_tmp(shard: pathlib.Path) -> pathlib.Path:
132 """Create a fake .restore-tmp-* file as would be left by a SIGKILL'd restore_object."""
133 fd, path = tempfile.mkstemp(dir=shard, prefix=".restore-tmp-")
134 os.close(fd)
135 pathlib.Path(path).write_bytes(b"partial restore")
136 return _make_stale(pathlib.Path(path))
137
138
139 def _count_stale_files(repo_root: pathlib.Path) -> int:
140 """Count all stale temp files in .muse/ (all families)."""
141 muse = repo_root / ".muse"
142 total = 0
143 for f in muse.rglob("*"):
144 if f.is_file() and any(
145 f.name.startswith(p)
146 for p in (".muse-tmp-", ".stat_cache_", ".obj-tmp-", ".restore-tmp-")
147 ):
148 total += 1
149 return total
150
151
152 # ---------------------------------------------------------------------------
153 # Subprocess workers — defined at module level for picklability under "spawn"
154 # ---------------------------------------------------------------------------
155
156
157 def _write_objects_worker(root: pathlib.Path, count: int) -> None:
158 """Write objects in a tight loop — killed midway to simulate SIGKILL."""
159 import hashlib as _hl
160 import time as _t
161
162 from muse.core.object_store import write_object as _wo
163
164 for i in range(count):
165 payload = f"sigkill-object-{i:06d}".encode()
166 oid = _hl.sha256(payload).hexdigest()
167 _wo(root, oid, payload)
168 _t.sleep(0.0002)
169
170
171 def _write_store_worker(root: pathlib.Path, count: int) -> None:
172 """Write commit-dir text atomically in a loop — killed midway."""
173 import time as _t
174
175 from muse.core.store import write_text_atomic as _wta
176
177 for i in range(count):
178 path = root / ".muse" / "commits" / f"fake-{i:06d}.msgpack"
179 _wta(path, f"fake commit {i}")
180 _t.sleep(0.0002)
181
182
183 def _full_commit_worker(repo_path: pathlib.Path, commit_msg: str) -> None:
184 """Run `muse commit -m <msg>` in a subprocess target (for SIGKILL testing)."""
185 import subprocess
186 import sys as _sys
187
188 subprocess.run(
189 ["muse", "commit", "-m", commit_msg],
190 cwd=str(repo_path),
191 stdout=_sys.stdout,
192 stderr=_sys.stderr,
193 )
194
195
196 # ---------------------------------------------------------------------------
197 # 1. _cleanup_muse_dir_temps — unit tests
198 # ---------------------------------------------------------------------------
199
200
201 class TestCleanupMuseDirTemps:
202 """Unit tests for the _cleanup_muse_dir_temps helper."""
203
204 def test_removes_muse_tmp_from_root(self, tmp_path: pathlib.Path) -> None:
205 muse = tmp_path / ".muse"
206 muse.mkdir()
207 stale = _plant_stale_muse_tmp(muse)
208 assert stale.exists()
209 removed = _cleanup_muse_dir_temps(muse)
210 assert removed == 1
211 assert not stale.exists()
212
213 def test_removes_muse_tmp_from_commits_subdir(self, tmp_path: pathlib.Path) -> None:
214 muse = tmp_path / ".muse"
215 muse.mkdir()
216 stale = _plant_stale_muse_tmp(muse, "commits")
217 removed = _cleanup_muse_dir_temps(muse)
218 assert removed == 1
219 assert not stale.exists()
220
221 def test_removes_muse_tmp_from_branches_subdir(self, tmp_path: pathlib.Path) -> None:
222 muse = tmp_path / ".muse"
223 muse.mkdir()
224 (muse / "branches").mkdir()
225 stale = _plant_stale_muse_tmp(muse, "branches")
226 removed = _cleanup_muse_dir_temps(muse)
227 assert removed == 1
228 assert not stale.exists()
229
230 def test_removes_muse_tmp_from_snapshots_subdir(self, tmp_path: pathlib.Path) -> None:
231 muse = tmp_path / ".muse"
232 muse.mkdir()
233 (muse / "snapshots").mkdir()
234 stale = _plant_stale_muse_tmp(muse, "snapshots")
235 removed = _cleanup_muse_dir_temps(muse)
236 assert removed == 1
237 assert not stale.exists()
238
239 def test_removes_stat_cache_tmp(self, tmp_path: pathlib.Path) -> None:
240 muse = tmp_path / ".muse"
241 muse.mkdir()
242 stale = _plant_stale_stat_cache_tmp(muse)
243 removed = _cleanup_muse_dir_temps(muse)
244 assert removed == 1
245 assert not stale.exists()
246
247 def test_preserves_real_muse_files(self, tmp_path: pathlib.Path) -> None:
248 muse = tmp_path / ".muse"
249 muse.mkdir()
250 (muse / "commits").mkdir()
251 # Real files must NOT be deleted
252 real_head = muse / "HEAD"
253 real_head.write_text("ref: refs/heads/main", encoding="utf-8")
254 real_commit = muse / "commits" / "abc123.msgpack"
255 real_commit.write_bytes(b"fake msgpack")
256 real_config = muse / "config.toml"
257 real_config.write_text("[core]\n", encoding="utf-8")
258 removed = _cleanup_muse_dir_temps(muse)
259 assert removed == 0
260 assert real_head.exists()
261 assert real_commit.exists()
262 assert real_config.exists()
263
264 def test_multiple_stale_files_across_subdirs(self, tmp_path: pathlib.Path) -> None:
265 muse = tmp_path / ".muse"
266 muse.mkdir()
267 stale: list[pathlib.Path] = []
268 stale.append(_plant_stale_muse_tmp(muse))
269 stale.append(_plant_stale_muse_tmp(muse, "commits"))
270 stale.append(_plant_stale_muse_tmp(muse, "snapshots"))
271 stale.append(_plant_stale_stat_cache_tmp(muse))
272 removed = _cleanup_muse_dir_temps(muse)
273 assert removed == 4
274 for f in stale:
275 assert not f.exists()
276
277 def test_nonexistent_muse_dir_returns_zero(self, tmp_path: pathlib.Path) -> None:
278 result = _cleanup_muse_dir_temps(tmp_path / ".muse")
279 assert result == 0
280
281 def test_missing_subdir_is_skipped_silently(self, tmp_path: pathlib.Path) -> None:
282 muse = tmp_path / ".muse"
283 muse.mkdir()
284 # Only root exists; subdirs (branches, commits, …) do not
285 stale = _plant_stale_muse_tmp(muse)
286 removed = _cleanup_muse_dir_temps(muse)
287 assert removed == 1
288 assert not stale.exists()
289
290 def test_idempotent_second_call(self, tmp_path: pathlib.Path) -> None:
291 muse = tmp_path / ".muse"
292 muse.mkdir()
293 _plant_stale_muse_tmp(muse)
294 _cleanup_muse_dir_temps(muse)
295 # Second call on clean dir must not raise and return 0
296 removed = _cleanup_muse_dir_temps(muse)
297 assert removed == 0
298
299 def test_temp_prefixes_constant_non_empty(self) -> None:
300 """_MUSE_TEMP_PREFIXES is exported and non-empty — agents can introspect it."""
301 assert len(_MUSE_TEMP_PREFIXES) >= 2
302 assert ".muse-tmp-" in _MUSE_TEMP_PREFIXES
303 assert ".stat_cache_" in _MUSE_TEMP_PREFIXES
304
305 def test_sweep_dirs_constant_includes_all_write_sites(self) -> None:
306 """_MUSE_SWEEP_DIRS covers every directory that uses write_text_atomic / _write_msgpack_atomic."""
307 required = {"", "branches", "commits", "snapshots", "tags", "releases"}
308 assert required.issubset(set(_MUSE_SWEEP_DIRS))
309
310
311 # ---------------------------------------------------------------------------
312 # 2. Startup GC — object-store orphans
313 # ---------------------------------------------------------------------------
314
315
316 class TestStartupGcObjectTemps:
317 """Object-store stale temps (.obj-tmp-*, .restore-tmp-*) are swept by startup GC."""
318
319 def test_obj_tmp_removed_by_cleanup(self, tmp_path: pathlib.Path) -> None:
320 repo = _repo(tmp_path)
321 shard = _shard(repo, "ab")
322 shard.mkdir(parents=True, exist_ok=True)
323 stale = _plant_stale_obj_tmp(shard)
324 assert stale.exists()
325 removed = cleanup_stale_object_temps(repo)
326 assert removed >= 1
327 assert not stale.exists()
328
329 def test_restore_tmp_removed_by_cleanup(self, tmp_path: pathlib.Path) -> None:
330 repo = _repo(tmp_path)
331 shard = _shard(repo, "cd")
332 shard.mkdir(parents=True, exist_ok=True)
333 stale = _plant_stale_restore_tmp(shard)
334 removed = cleanup_stale_object_temps(repo)
335 assert removed >= 1
336 assert not stale.exists()
337
338 def test_startup_gc_delegates_to_object_cleanup(self, tmp_path: pathlib.Path) -> None:
339 repo = _repo(tmp_path)
340 shard = _shard(repo, "ef")
341 shard.mkdir(parents=True, exist_ok=True)
342 stale_obj = _plant_stale_obj_tmp(shard)
343 stale_restore = _plant_stale_restore_tmp(shard)
344 _startup_gc(repo)
345 assert not stale_obj.exists()
346 assert not stale_restore.exists()
347
348 def test_real_objects_preserved_by_startup_gc(self, tmp_path: pathlib.Path) -> None:
349 repo = _repo(tmp_path)
350 data = b"real object content"
351 oid = _oid(data)
352 write_object(repo, oid, data)
353 _startup_gc(repo)
354 result = read_object(repo, oid)
355 assert result == data
356
357 def test_stale_and_real_coexist_only_stale_removed(self, tmp_path: pathlib.Path) -> None:
358 repo = _repo(tmp_path)
359 data = b"survivor"
360 oid = _oid(data)
361 write_object(repo, oid, data)
362 # Plant stale temp in same shard as the real object
363 shard = object_path(repo, oid).parent
364 stale = _plant_stale_obj_tmp(shard)
365 _startup_gc(repo)
366 assert not stale.exists()
367 assert read_object(repo, oid) == data
368
369
370 # ---------------------------------------------------------------------------
371 # 3. Startup GC — .muse-tmp-* in subdirectories
372 # ---------------------------------------------------------------------------
373
374
375 class TestStartupGcMuseTemps:
376 """.muse-tmp-* files in all .muse/ subdirs are swept by _startup_gc."""
377
378 def test_muse_tmp_in_root_swept(self, tmp_path: pathlib.Path) -> None:
379 repo = _repo(tmp_path)
380 stale = _plant_stale_muse_tmp(repo / ".muse")
381 _startup_gc(repo)
382 assert not stale.exists()
383
384 def test_muse_tmp_in_commits_swept(self, tmp_path: pathlib.Path) -> None:
385 repo = _repo(tmp_path)
386 stale = _plant_stale_muse_tmp(repo / ".muse", "commits")
387 _startup_gc(repo)
388 assert not stale.exists()
389
390 def test_muse_tmp_in_branches_swept(self, tmp_path: pathlib.Path) -> None:
391 repo = _repo(tmp_path)
392 stale = _plant_stale_muse_tmp(repo / ".muse", "branches")
393 _startup_gc(repo)
394 assert not stale.exists()
395
396 def test_muse_tmp_in_snapshots_swept(self, tmp_path: pathlib.Path) -> None:
397 repo = _repo(tmp_path)
398 stale = _plant_stale_muse_tmp(repo / ".muse", "snapshots")
399 _startup_gc(repo)
400 assert not stale.exists()
401
402 def test_muse_tmp_in_tags_swept(self, tmp_path: pathlib.Path) -> None:
403 repo = _repo(tmp_path)
404 (repo / ".muse" / "tags").mkdir()
405 stale = _plant_stale_muse_tmp(repo / ".muse", "tags")
406 _startup_gc(repo)
407 assert not stale.exists()
408
409 def test_muse_tmp_in_releases_swept(self, tmp_path: pathlib.Path) -> None:
410 repo = _repo(tmp_path)
411 (repo / ".muse" / "releases").mkdir()
412 stale = _plant_stale_muse_tmp(repo / ".muse", "releases")
413 _startup_gc(repo)
414 assert not stale.exists()
415
416 def test_real_msgpack_in_commits_preserved(self, tmp_path: pathlib.Path) -> None:
417 repo = _repo(tmp_path)
418 real = repo / ".muse" / "commits" / "deadbeef.msgpack"
419 real.write_bytes(b"\x82\xa9commit_id\xa8deadbeef")
420 _startup_gc(repo)
421 assert real.exists()
422
423
424 # ---------------------------------------------------------------------------
425 # 4. Startup GC — .stat_cache_*.tmp
426 # ---------------------------------------------------------------------------
427
428
429 class TestStartupGcStatCacheTemps:
430 """.stat_cache_*.tmp files (StatCache.save) are swept by _startup_gc."""
431
432 def test_stat_cache_tmp_swept(self, tmp_path: pathlib.Path) -> None:
433 repo = _repo(tmp_path)
434 stale = _plant_stale_stat_cache_tmp(repo / ".muse")
435 _startup_gc(repo)
436 assert not stale.exists()
437
438 def test_real_stat_cache_msgpack_preserved(self, tmp_path: pathlib.Path) -> None:
439 repo = _repo(tmp_path)
440 real = repo / ".muse" / "stat_cache.msgpack"
441 real.write_bytes(b"\x82\xa7version\x02")
442 _startup_gc(repo)
443 assert real.exists()
444
445 def test_multiple_stat_cache_tmps_all_swept(self, tmp_path: pathlib.Path) -> None:
446 repo = _repo(tmp_path)
447 stales = [_plant_stale_stat_cache_tmp(repo / ".muse") for _ in range(5)]
448 _startup_gc(repo)
449 for s in stales:
450 assert not s.exists()
451
452
453 # ---------------------------------------------------------------------------
454 # 5. require_repo calls the startup GC
455 # ---------------------------------------------------------------------------
456
457
458 class TestRequireRepoCallsGc:
459 """require_repo() triggers the full startup GC sweep."""
460
461 def test_require_repo_removes_obj_tmp(self, tmp_path: pathlib.Path) -> None:
462 repo = _repo(tmp_path)
463 shard = _shard(repo, "aa")
464 shard.mkdir(parents=True, exist_ok=True)
465 stale = _plant_stale_obj_tmp(shard)
466 # Call require_repo with MUSE_REPO_ROOT override so it finds the repo
467 ctx = require_repo(start=repo)
468 assert ctx == repo
469 assert not stale.exists()
470
471 def test_require_repo_removes_muse_tmp(self, tmp_path: pathlib.Path) -> None:
472 repo = _repo(tmp_path)
473 stale = _plant_stale_muse_tmp(repo / ".muse")
474 require_repo(start=repo)
475 assert not stale.exists()
476
477 def test_require_repo_removes_stat_cache_tmp(self, tmp_path: pathlib.Path) -> None:
478 repo = _repo(tmp_path)
479 stale = _plant_stale_stat_cache_tmp(repo / ".muse")
480 require_repo(start=repo)
481 assert not stale.exists()
482
483 def test_require_repo_sweeps_all_families_at_once(self, tmp_path: pathlib.Path) -> None:
484 repo = _repo(tmp_path)
485 shard = _shard(repo, "bb")
486 shard.mkdir(parents=True, exist_ok=True)
487 f1 = _plant_stale_obj_tmp(shard)
488 f2 = _plant_stale_restore_tmp(shard)
489 f3 = _plant_stale_muse_tmp(repo / ".muse")
490 f4 = _plant_stale_muse_tmp(repo / ".muse", "commits")
491 f5 = _plant_stale_stat_cache_tmp(repo / ".muse")
492 require_repo(start=repo)
493 for f in (f1, f2, f3, f4, f5):
494 assert not f.exists(), f"{f.name} should have been swept"
495
496 def test_require_repo_not_in_repo_still_raises(self, tmp_path: pathlib.Path) -> None:
497 """require_repo on a non-repo path still exits — GC is not run on miss."""
498 with pytest.raises(SystemExit):
499 require_repo(start=tmp_path)
500
501
502 # ---------------------------------------------------------------------------
503 # 6. Multiple consecutive SIGKILLs — accumulated stale files all swept
504 # ---------------------------------------------------------------------------
505
506
507 class TestMultipleSigkills:
508 """Simulate N crashes: stale files from each accumulate and are all swept."""
509
510 def test_three_crash_generations_all_swept(self, tmp_path: pathlib.Path) -> None:
511 repo = _repo(tmp_path)
512 muse = repo / ".muse"
513 shard = _shard(repo, "cc")
514 shard.mkdir(parents=True, exist_ok=True)
515
516 # Simulate 3 separate crashes leaving stale files from each family.
517 stales: list[pathlib.Path] = []
518 for _ in range(3):
519 stales.append(_plant_stale_obj_tmp(shard))
520 stales.append(_plant_stale_restore_tmp(shard))
521 stales.append(_plant_stale_muse_tmp(muse))
522 stales.append(_plant_stale_muse_tmp(muse, "commits"))
523 stales.append(_plant_stale_stat_cache_tmp(muse))
524
525 assert len(stales) == 15
526 _startup_gc(repo)
527 for f in stales:
528 assert not f.exists(), f"Stale file survived: {f.name}"
529
530 def test_gc_count_is_accurate(self, tmp_path: pathlib.Path) -> None:
531 repo = _repo(tmp_path)
532 muse = repo / ".muse"
533 (muse / "tags").mkdir()
534 for _ in range(4):
535 _plant_stale_muse_tmp(muse)
536 for _ in range(3):
537 _plant_stale_stat_cache_tmp(muse)
538 # 7 total stale files in muse dir
539 removed = _cleanup_muse_dir_temps(muse)
540 assert removed == 7
541 assert _count_stale_files(repo) == 0
542
543
544 # ---------------------------------------------------------------------------
545 # 7. SIGKILL at T+50ms / T+100ms / T+200ms — object-store write sequence
546 # ---------------------------------------------------------------------------
547
548
549 class TestSigkillAtTimingWindows:
550 """Subprocess SIGKILL at precise timing windows: store stays consistent."""
551
552 @pytest.mark.slow
553 @pytest.mark.parametrize("delay_ms", [50, 100, 200])
554 def test_object_store_consistent_after_sigkill(
555 self, tmp_path: pathlib.Path, delay_ms: int
556 ) -> None:
557 repo = _repo(tmp_path)
558
559 # Pre-write 10 known objects before the crashable process starts.
560 pre_data: list[tuple[str, bytes]] = []
561 for i in range(10):
562 payload = f"pre-kill-{i:03d}".encode()
563 oid = _oid(payload)
564 write_object(repo, oid, payload)
565 pre_data.append((oid, payload))
566
567 # Spawn a fresh process that writes objects in a tight loop.
568 ctx = multiprocessing.get_context("spawn")
569 proc = ctx.Process(target=_write_objects_worker, args=(repo, 2000))
570 proc.start()
571
572 # Kill it at the specified timing window.
573 time.sleep(delay_ms / 1000.0)
574 if proc.is_alive():
575 assert proc.pid is not None
576 os.kill(proc.pid, signal.SIGKILL)
577 proc.join(timeout=5)
578
579 # Startup GC: simulates the next command after the crash.
580 _startup_gc(repo)
581
582 # No stale temp files must remain anywhere in .muse/.
583 assert _count_stale_files(repo) == 0, (
584 f"Stale temp files survived SIGKILL at T+{delay_ms}ms"
585 )
586
587 # Every pre-kill object must still be readable and hash-verified.
588 for oid, payload in pre_data:
589 assert read_object(repo, oid) == payload, (
590 f"Pre-kill object {oid[:8]} corrupted after SIGKILL at T+{delay_ms}ms"
591 )
592
593 @pytest.mark.slow
594 @pytest.mark.parametrize("delay_ms", [50, 100, 200])
595 def test_store_write_consistent_after_sigkill(
596 self, tmp_path: pathlib.Path, delay_ms: int
597 ) -> None:
598 """SIGKILL during write_text_atomic loop leaves no stale .muse-tmp-* files."""
599 repo = _repo(tmp_path)
600
601 ctx = multiprocessing.get_context("spawn")
602 proc = ctx.Process(target=_write_store_worker, args=(repo, 2000))
603 proc.start()
604
605 time.sleep(delay_ms / 1000.0)
606 if proc.is_alive():
607 assert proc.pid is not None
608 os.kill(proc.pid, signal.SIGKILL)
609 proc.join(timeout=5)
610
611 # Startup GC sweep.
612 _startup_gc(repo)
613
614 # No .muse-tmp-* files may survive.
615 muse = repo / ".muse"
616 leftovers = list(muse.rglob(".muse-tmp-*"))
617 assert leftovers == [], (
618 f"Stale .muse-tmp-* survived SIGKILL at T+{delay_ms}ms: {leftovers}"
619 )
620
621
622 # ---------------------------------------------------------------------------
623 # 8. Full CLI commit survives SIGKILL
624 # ---------------------------------------------------------------------------
625
626
627 class TestSigkillDuringCommit:
628 """End-to-end: SIGKILL during `muse commit` leaves repo in a recoverable state."""
629
630 def _init_real_repo(self, tmp_path: pathlib.Path) -> pathlib.Path:
631 """Create a minimal real muse repo with a committed file."""
632 import subprocess
633
634 env = os.environ.copy()
635 env["MUSE_REPO_ROOT"] = str(tmp_path)
636
637 subprocess.run(["muse", "init"], cwd=str(tmp_path), env=env, check=True,
638 capture_output=True)
639 # Stage and commit a file so there is a valid HEAD.
640 (tmp_path / "file.txt").write_text("hello", encoding="utf-8")
641 subprocess.run(["muse", "code", "add", "."], cwd=str(tmp_path), env=env,
642 check=True, capture_output=True)
643 subprocess.run(["muse", "commit", "-m", "init"], cwd=str(tmp_path), env=env,
644 check=True, capture_output=True)
645 return tmp_path
646
647 @pytest.mark.slow
648 def test_muse_status_runs_after_sigkill(self, tmp_path: pathlib.Path) -> None:
649 """`muse status` must exit cleanly (exit 0) after a SIGKILL'd commit."""
650 import subprocess
651
652 repo = self._init_real_repo(tmp_path)
653
654 # Modify a file so there is something to commit.
655 (repo / "file.txt").write_text("changed", encoding="utf-8")
656 subprocess.run(["muse", "code", "add", "."], cwd=str(repo),
657 capture_output=True)
658
659 # Spawn commit subprocess and kill it immediately.
660 ctx = multiprocessing.get_context("spawn")
661 proc = ctx.Process(target=_full_commit_worker, args=(repo, "crash-me"))
662 proc.start()
663 time.sleep(0.05)
664 if proc.is_alive():
665 assert proc.pid is not None
666 os.kill(proc.pid, signal.SIGKILL)
667 proc.join(timeout=5)
668
669 # Startup GC runs on next require_repo invocation (muse status triggers it).
670 result = subprocess.run(
671 ["muse", "status"],
672 cwd=str(repo),
673 capture_output=True,
674 text=True,
675 )
676 # status must exit 0; any non-zero means repo is corrupt.
677 assert result.returncode == 0, (
678 f"muse status failed after SIGKILL:\n{result.stdout}\n{result.stderr}"
679 )
680
681 @pytest.mark.slow
682 def test_no_stale_temps_after_sigkill_and_next_command(
683 self, tmp_path: pathlib.Path
684 ) -> None:
685 """After SIGKILL + muse status, zero stale temps remain in .muse/."""
686 import subprocess
687
688 repo = self._init_real_repo(tmp_path)
689 (repo / "file.txt").write_text("changed again", encoding="utf-8")
690 subprocess.run(["muse", "code", "add", "."], cwd=str(repo), capture_output=True)
691
692 ctx = multiprocessing.get_context("spawn")
693 proc = ctx.Process(target=_full_commit_worker, args=(repo, "crash-me-2"))
694 proc.start()
695 time.sleep(0.05)
696 if proc.is_alive():
697 assert proc.pid is not None
698 os.kill(proc.pid, signal.SIGKILL)
699 proc.join(timeout=5)
700
701 # Trigger startup GC via the next command.
702 subprocess.run(["muse", "status"], cwd=str(repo), capture_output=True)
703
704 assert _count_stale_files(repo) == 0, "Stale temps remain after startup GC"
705
706
707 # ---------------------------------------------------------------------------
708 # 9. Push path idempotency under SIGKILL
709 # ---------------------------------------------------------------------------
710
711
712 class TestSigkillDuringPush:
713 """SIGKILL during push writes leaves no corruption: write_object is atomic."""
714
715 @pytest.mark.slow
716 def test_push_objects_atomic_under_sigkill(self, tmp_path: pathlib.Path) -> None:
717 """Objects pushed before kill are readable; partial objects are absent."""
718 (tmp_path / "local").mkdir()
719 (tmp_path / "remote").mkdir()
720 local = _repo(tmp_path / "local")
721 remote = _repo(tmp_path / "remote")
722
723 # Write 10 objects to local and also push them to remote before kill.
724 pre_data: list[tuple[str, bytes]] = []
725 for i in range(10):
726 payload = f"push-pre-kill-{i}".encode()
727 oid = _oid(payload)
728 write_object(local, oid, payload)
729 write_object(remote, oid, payload) # simulate push of pre-kill objects
730 pre_data.append((oid, payload))
731
732 # Spawn a process that keeps writing objects to the remote store.
733 ctx = multiprocessing.get_context("spawn")
734 proc = ctx.Process(target=_write_objects_worker, args=(remote, 2000))
735 proc.start()
736 time.sleep(0.08) # T+80ms kill
737 if proc.is_alive():
738 assert proc.pid is not None
739 os.kill(proc.pid, signal.SIGKILL)
740 proc.join(timeout=5)
741
742 # Backdate all temp files left by the killed process — the 60-second
743 # age gate in cleanup_stale_object_temps skips fresh files to protect
744 # concurrent writers; in tests we fast-forward mtime to simulate aging.
745 for f in (remote / ".muse").rglob("*"):
746 if f.is_file() and any(
747 f.name.startswith(p)
748 for p in (".obj-tmp-", ".restore-tmp-", ".muse-tmp-", ".stat_cache_")
749 ):
750 os.utime(f, (0, 0))
751
752 # Simulate remote-side startup GC (next muse command on the remote).
753 _startup_gc(remote)
754
755 # Remote must have no stale temps.
756 assert _count_stale_files(remote) == 0
757
758 # All pre-kill objects on remote must be intact.
759 for oid, payload in pre_data:
760 assert read_object(remote, oid) == payload, (
761 f"Remote object {oid[:8]} corrupted after push SIGKILL"
762 )
763
764 def test_push_write_object_is_idempotent(self, tmp_path: pathlib.Path) -> None:
765 """write_object called twice for same OID returns False (skip) both times."""
766 repo = _repo(tmp_path)
767 payload = b"idempotent object"
768 oid = _oid(payload)
769 first = write_object(repo, oid, payload)
770 second = write_object(repo, oid, payload)
771 assert first is True
772 assert second is False
773 assert read_object(repo, oid) == payload
774
775 def test_partial_write_interrupted_at_os_level_leaves_no_dest(
776 self, tmp_path: pathlib.Path
777 ) -> None:
778 """The mkstemp→replace contract: if replace never happens, dest is absent."""
779 repo = _repo(tmp_path)
780 data = b"will be interrupted"
781 oid = _oid(data)
782
783 # Manually plant the stale temp (simulates SIGKILL after mkstemp, before replace)
784 shard = object_path(repo, oid).parent
785 shard.mkdir(parents=True, exist_ok=True)
786 stale = _plant_stale_obj_tmp(shard)
787
788 # The destination object must NOT exist (replace never happened).
789 from muse.core.object_store import has_object
790 assert not has_object(repo, oid)
791
792 # Cleanup removes the stale temp.
793 cleanup_stale_object_temps(repo)
794 assert not stale.exists()
795
796 # A fresh write_object succeeds normally.
797 write_object(repo, oid, data)
798 assert has_object(repo, oid)
799
800
801 # ---------------------------------------------------------------------------
802 # 10. GC preserves all real stored objects
803 # ---------------------------------------------------------------------------
804
805
806 class TestGcPreservesRealObjects:
807 """_startup_gc must never delete a valid stored object."""
808
809 def test_100_objects_all_survive_gc(self, tmp_path: pathlib.Path) -> None:
810 repo = _repo(tmp_path)
811 written: list[tuple[str, bytes]] = []
812 for i in range(100):
813 payload = f"object-{i:04d}".encode()
814 oid = _oid(payload)
815 write_object(repo, oid, payload)
816 written.append((oid, payload))
817
818 _startup_gc(repo)
819
820 for oid, payload in written:
821 assert read_object(repo, oid) == payload, f"Object {oid[:8]} deleted by GC"
822
823 def test_real_head_and_config_survive_gc(self, tmp_path: pathlib.Path) -> None:
824 repo = _repo(tmp_path)
825 muse = repo / ".muse"
826 head = muse / "HEAD"
827 head.write_text("ref: refs/heads/main\n", encoding="utf-8")
828 config = muse / "config.toml"
829 config.write_text("[core]\n name = \"test\"\n", encoding="utf-8")
830
831 _startup_gc(repo)
832
833 assert head.read_text(encoding="utf-8") == "ref: refs/heads/main\n"
834 assert "test" in config.read_text(encoding="utf-8")
835
836 def test_gc_on_empty_repo_is_noop(self, tmp_path: pathlib.Path) -> None:
837 repo = _repo(tmp_path)
838 # No objects, no stale files
839 _startup_gc(repo)
840 assert _count_stale_files(repo) == 0
841
842
843 # ---------------------------------------------------------------------------
844 # 11. Performance: GC sweep ≤ 10 ms with 1 000 stale files
845 # ---------------------------------------------------------------------------
846
847
848 class TestGcSweeperPerformance:
849 """Startup GC is fast enough to run on every require_repo invocation."""
850
851 @pytest.mark.slow
852 def test_sweep_1000_stale_files_under_500ms(self, tmp_path: pathlib.Path) -> None:
853 """GC sweeps 1 000 stale files in < 500 ms (wall clock including logging).
854
855 The budget is generous because macOS APFS + pytest log capture add
856 overhead (~50–100 μs per unlink + warning emission). On a real crash
857 scenario a repo will have at most 1–3 stale files, so steady-state
858 latency is < 1 ms. This test validates the worst-case bound.
859 """
860 repo = _repo(tmp_path)
861 muse = repo / ".muse"
862 (muse / "commits").mkdir(exist_ok=True)
863
864 # Plant 1 000 stale .muse-tmp-* files across two subdirs.
865 for _ in range(500):
866 _plant_stale_muse_tmp(muse)
867 for _ in range(500):
868 _plant_stale_muse_tmp(muse, "commits")
869
870 start = time.perf_counter()
871 removed = _cleanup_muse_dir_temps(muse)
872 duration_ms = (time.perf_counter() - start) * 1000
873
874 assert removed == 1000
875 assert duration_ms < 500.0, (
876 f"_cleanup_muse_dir_temps took {duration_ms:.1f} ms for 1 000 files "
877 f"(budget: 500 ms)"
878 )
879
880 @pytest.mark.slow
881 def test_full_startup_gc_under_200ms(self, tmp_path: pathlib.Path) -> None:
882 """Full _startup_gc (both sweeps) is < 200 ms with 200 stale temps.
883
884 In production a crash leaves 1–3 stale files at most; this tests the
885 adversarial bound of 200 simultaneous stale temps across both the
886 object store and .muse/ directories.
887 """
888 repo = _repo(tmp_path)
889 muse = repo / ".muse"
890 (muse / "commits").mkdir(exist_ok=True)
891
892 # 100 stale object temps across 10 shards.
893 for i in range(10):
894 shard = _shard(repo, f"{i:02x}")
895 shard.mkdir(parents=True, exist_ok=True)
896 for _ in range(10):
897 _plant_stale_obj_tmp(shard)
898
899 # 100 stale muse temps.
900 for _ in range(50):
901 _plant_stale_muse_tmp(muse)
902 for _ in range(50):
903 _plant_stale_muse_tmp(muse, "commits")
904
905 start = time.perf_counter()
906 _startup_gc(repo)
907 duration_ms = (time.perf_counter() - start) * 1000
908
909 assert duration_ms < 200.0, (
910 f"_startup_gc took {duration_ms:.1f} ms (budget: 200 ms)"
911 )
912 assert _count_stale_files(repo) == 0
913
914
915 # ---------------------------------------------------------------------------
916 # 12. .restore-tmp-* in working-tree: scope documentation test
917 # ---------------------------------------------------------------------------
918
919
920 class TestRestoreTempWorkdirBound:
921 """.restore-tmp-* files in the working tree (not .muse/) are outside GC scope.
922
923 restore_object writes to a user-provided destination directory, not inside
924 .muse/. If SIGKILL occurs between mkstemp and os.replace in restore_object,
925 the stale temp stays in the working tree.
926
927 The documented guarantee: the .muse/ repo state is never corrupted, and
928 the stale restore temp in the working tree is inert (it does not block the
929 next checkout/merge, which simply overwrites the destination path atomically).
930 """
931
932 def test_restore_tmp_in_workdir_not_swept_by_gc(self, tmp_path: pathlib.Path) -> None:
933 """GC sweeps .muse/ only — stale restore temps in workdir are out of scope."""
934 repo = _repo(tmp_path)
935 # Plant a stale restore temp in the working tree (outside .muse/)
936 fd, stale_str = tempfile.mkstemp(dir=tmp_path, prefix=".restore-tmp-")
937 os.close(fd)
938 stale = pathlib.Path(stale_str)
939 stale.write_bytes(b"stale workdir restore temp")
940
941 # GC sweeps only .muse/; workdir stale is untouched.
942 _startup_gc(repo)
943
944 # Stale in workdir persists — this is the documented limitation.
945 assert stale.exists(), (
946 "GC must not touch working-tree files outside .muse/"
947 )
948 # But .muse/ is clean.
949 assert _count_stale_files(repo) == 0
950
951 stale.unlink() # cleanup
952
953 def test_restore_tmp_in_obj_shard_IS_swept(self, tmp_path: pathlib.Path) -> None:
954 """.restore-tmp-* inside .muse/objects/ shard dirs ARE swept (object store scope)."""
955 repo = _repo(tmp_path)
956 shard = _shard(repo, "dd")
957 shard.mkdir(parents=True, exist_ok=True)
958 stale = _plant_stale_restore_tmp(shard)
959 _startup_gc(repo)
960 assert not stale.exists(), ".restore-tmp-* in object shard must be swept"
File History 2 commits
sha256:88ac91129873e6a496e9189515aa690eb893ae25d69c8f72af141a2be5068eb3 docs: docstring sprint contract→find-symbol — idiomatic run… Sonnet 4.6 patch 139 days ago
sha256:a09b1b4f6838754495547f200aa0ce88e2f56ffc5b20b900f6f0cff2c3cdede9 fix(cursorignore): remove git-ism (.git/worktrees) Human minor 142 days ago