gabriel / muse public
test_integrity_I8_object_store_scale.py python
841 lines 32.8 KB
Raw
sha256:88ac91129873e6a496e9189515aa690eb893ae25d69c8f72af141a2be5068eb3 docs: docstring sprint contract→find-symbol — idiomatic run… Sonnet 4.6 patch 140 days ago
1 """I-8: Object store at Linux scale.
2
3 Scenario: 850 000 commits × ~20 objects per commit = 17 million objects.
4 2-char sharding → 256 shards × ~66 000 files each. On Linux ext4 (and
5 macOS APFS) directory entries above ~100 000 per directory trigger visible
6 lookup degradation. This suite proves:
7
8 1. File mode 0o444 — every new object is written read-only.
9 2. Stale temp cleanup — .obj-tmp-* files from a prior crash are removed.
10 3. has_object O(log n) lookup — timing at 1k / 10k / 100k objects proves
11 sub-linear growth (ext4 / APFS use hash-tree / B-tree indexing).
12 4. 4-char sharding — 65 536 shards; object path layout changes correctly.
13 5. Configurable via [limits] shard_prefix_length in config.toml.
14 6. Dual-lookup / migration — objects written at 2-char prefix are still
15 found after switching config to 4-char.
16 7. shard_prefix_length=4 reflected in get_config_value and get_limit.
17 8. Robustness — invalid shard_prefix_length values are ignored.
18 9. Permission enforcement — direct write to a 0o444 object raises
19 PermissionError, confirming the OS-level immutability guard.
20 10. Shard count correctness — 4-char yields 65 536 possible shards.
21 11. cleanup_stale_object_temps is idempotent (double-call safe).
22 12. _object_path_with_fallback returns primary path when it exists.
23 """
24
25 from __future__ import annotations
26
27 import hashlib
28 import os
29 import pathlib
30 import stat
31 import time
32 import tomllib
33
34 import pytest
35
36 from muse.core.object_store import (
37 _object_path_with_fallback,
38 cleanup_stale_object_temps,
39 has_object,
40 iter_stored_objects,
41 object_path,
42 objects_dir,
43 read_object,
44 restore_object,
45 write_object,
46 write_object_from_path,
47 _OBJECT_MODE,
48 _DEFAULT_SHARD_PREFIX_LEN,
49 _VALID_SHARD_PREFIX_LENS,
50 )
51 from muse.cli.config import get_limit, get_config_value
52 from muse.core._types import Manifest, blob_id, long_id
53
54
55 def _repo(tmp_path: pathlib.Path) -> pathlib.Path:
56 (tmp_path / ".muse").mkdir()
57 return tmp_path
58
59
60 def _write_config(repo: pathlib.Path, shard_prefix_length: int) -> None:
61 """Write a minimal .muse/config.toml with [limits] shard_prefix_length."""
62 config_text = (
63 "[core]\nbranch = \"main\"\n\n"
64 f"[limits]\nshard_prefix_length = {shard_prefix_length}\n"
65 )
66 (repo / ".muse" / "config.toml").write_text(config_text, encoding="utf-8")
67
68
69 # ---------------------------------------------------------------------------
70 # 0. Regression: restore_object must NOT propagate 0o444 to working tree
71 # ---------------------------------------------------------------------------
72
73
74 class TestRestoreObjectMode:
75 """Regression test for: stored objects are 0o444 (immutable); restore_object
76 must produce 0o644 working-tree files so they remain editable.
77
78 Root cause: shutil.copy2 copies permissions from the src (stored object).
79 After I-8 introduced 0o444 on stored objects, restore_object was producing
80 read-only working-tree files, silently freezing them. This class was added
81 to pin the fix and prevent recurrence.
82 """
83
84 def test_restore_object_produces_0o644_file(
85 self, tmp_path: pathlib.Path
86 ) -> None:
87 """restore_object must write working-tree files with mode 0o644.
88
89 Stored objects are 0o444; working-tree files must be 0o644 so users
90 and agents can edit them without a manual chmod.
91 """
92 repo = _repo(tmp_path)
93 data = b"content that will be restored to working tree"
94 oid = blob_id(data)
95 write_object(repo, oid, data)
96
97 dest = tmp_path / "restored.txt"
98 assert restore_object(repo, oid, dest)
99
100 mode = stat.S_IMODE(dest.stat().st_mode)
101 assert mode == 0o644, (
102 f"restore_object produced mode {oct(mode)} — working-tree files "
103 f"must be 0o644 so they are editable. "
104 f"(Stored object is 0o444; shutil.copy2 must not propagate that mode.)"
105 )
106
107 def test_stored_object_is_0o444_but_restore_is_0o644(
108 self, tmp_path: pathlib.Path
109 ) -> None:
110 """The stored object is 0o444 while the restored file is 0o644.
111
112 This is the invariant: objects are immutable in the store, writable
113 in the working tree.
114 """
115 repo = _repo(tmp_path)
116 data = b"immutable in store, writable in tree"
117 oid = blob_id(data)
118 write_object(repo, oid, data)
119
120 stored_mode = stat.S_IMODE(object_path(repo, oid).stat().st_mode)
121 assert stored_mode == 0o444, f"Stored object should be 0o444, got {oct(stored_mode)}"
122
123 dest = tmp_path / "workdir" / "file.txt"
124 restore_object(repo, oid, dest)
125 restored_mode = stat.S_IMODE(dest.stat().st_mode)
126 assert restored_mode == 0o644, (
127 f"Restored working-tree file should be 0o644, got {oct(restored_mode)}"
128 )
129
130 def test_restore_object_content_intact_after_mode_fix(
131 self, tmp_path: pathlib.Path
132 ) -> None:
133 """Content must be byte-identical after the chmod fix — no data loss."""
134 repo = _repo(tmp_path)
135 data = b"content integrity check after mode fix" * 50
136 oid = blob_id(data)
137 write_object(repo, oid, data)
138
139 dest = tmp_path / "check.bin"
140 restore_object(repo, oid, dest)
141 assert dest.read_bytes() == data
142
143 def test_restore_large_object_is_0o644(self, tmp_path: pathlib.Path) -> None:
144 """Large blobs (shutil.copy2 path) also restore as 0o644."""
145 repo = _repo(tmp_path)
146 data = os.urandom(512 * 1024) # 512 KiB
147 oid = blob_id(data)
148 src = tmp_path / "large.bin"
149 src.write_bytes(data)
150 write_object_from_path(repo, oid, src)
151
152 dest = tmp_path / "large_restored.bin"
153 restore_object(repo, oid, dest)
154 mode = stat.S_IMODE(dest.stat().st_mode)
155 assert mode == 0o644, (
156 f"Large blob restore produced mode {oct(mode)}, expected 0o644"
157 )
158
159
160 # ---------------------------------------------------------------------------
161 # 1. File mode 0o444 — immutability enforced at the OS level
162 # ---------------------------------------------------------------------------
163
164
165 class TestObjectMode:
166 def test_write_object_produces_0o444_file(self, tmp_path: pathlib.Path) -> None:
167 """Every blob written by write_object must be mode 0o444."""
168 repo = _repo(tmp_path)
169 data = b"immutable content"
170 oid = blob_id(data)
171 write_object(repo, oid, data)
172 p = object_path(repo, oid)
173 mode = stat.S_IMODE(p.stat().st_mode)
174 assert mode == 0o444, (
175 f"Object {oid[:8]} was written with mode {oct(mode)} instead of 0o444. "
176 "Content-addressed objects must be read-only."
177 )
178
179 def test_write_object_from_path_produces_0o444_file(
180 self, tmp_path: pathlib.Path
181 ) -> None:
182 """write_object_from_path (large-blob path) must also produce 0o444."""
183 repo = _repo(tmp_path)
184 data = b"large blob via path" * 100
185 oid = blob_id(data)
186 src = tmp_path / "src.bin"
187 src.write_bytes(data)
188 write_object_from_path(repo, oid, src)
189 p = object_path(repo, oid)
190 mode = stat.S_IMODE(p.stat().st_mode)
191 assert mode == 0o444, (
192 f"write_object_from_path produced mode {oct(mode)} instead of 0o444."
193 )
194
195 def test_object_mode_constant(self) -> None:
196 """_OBJECT_MODE must equal 0o444 — no accidental changes."""
197 assert _OBJECT_MODE == 0o444
198
199 def test_write_then_read_respects_mode(self, tmp_path: pathlib.Path) -> None:
200 """Round-trip: content can be read back even though the file is 0o444."""
201 repo = _repo(tmp_path)
202 data = b"read-only but readable"
203 oid = blob_id(data)
204 write_object(repo, oid, data)
205 assert read_object(repo, oid) == data
206
207 def test_direct_overwrite_blocked_by_os(self, tmp_path: pathlib.Path) -> None:
208 """Opening a 0o444 object for writing must raise PermissionError.
209
210 This is the OS-level immutability guarantee: even a bug that calls
211 open(path, 'wb') on a stored object is caught before any bytes are
212 written.
213 """
214 repo = _repo(tmp_path)
215 data = b"must not be overwritten"
216 oid = blob_id(data)
217 write_object(repo, oid, data)
218 p = object_path(repo, oid)
219 with pytest.raises(PermissionError):
220 p.write_bytes(b"attacker-controlled content")
221 # Content must be intact.
222 assert read_object(repo, oid) == data
223
224 def test_multiple_objects_all_0o444(self, tmp_path: pathlib.Path) -> None:
225 """Batch write: every object file must be 0o444."""
226 repo = _repo(tmp_path)
227 for i in range(50):
228 data = f"batch-object-{i}".encode()
229 oid = blob_id(data)
230 write_object(repo, oid, data)
231 for _, obj_file in iter_stored_objects(repo):
232 mode = stat.S_IMODE(obj_file.stat().st_mode)
233 assert mode == 0o444, f"{obj_file.name} has mode {oct(mode)}, expected 0o444"
234
235
236 # ---------------------------------------------------------------------------
237 # 2. Stale temp cleanup
238 # ---------------------------------------------------------------------------
239
240
241 def _make_stale(path: pathlib.Path, content: bytes = b"stale") -> None:
242 """Write *path* and backdate its mtime past the age gate.
243
244 cleanup_stale_object_temps only removes files older than
245 _CLEANUP_MIN_AGE_SECS (60 s). Tests that create temp files and
246 immediately call cleanup would always return 0 without this helper.
247 Setting mtime to the Unix epoch (1970-01-01) makes every freshly-created
248 temp file look decades old to the cleanup function.
249 """
250 path.write_bytes(content)
251 os.utime(path, (0, 0)) # atime=0, mtime=0 → epoch → age > 60 s
252
253
254 class TestStaleTempCleanup:
255 def test_cleanup_removes_obj_tmp_files(self, tmp_path: pathlib.Path) -> None:
256 """cleanup_stale_object_temps removes .obj-tmp-* files from shard dirs."""
257 repo = _repo(tmp_path)
258 shard = objects_dir(repo) / "sha256" / "ab"
259 shard.mkdir(parents=True)
260 stale = shard / ".obj-tmp-crash"
261 _make_stale(stale, b"partial write from prior SIGKILL")
262 assert stale.exists()
263
264 removed = cleanup_stale_object_temps(repo)
265 assert removed == 1
266 assert not stale.exists()
267
268 def test_cleanup_removes_restore_tmp_files(self, tmp_path: pathlib.Path) -> None:
269 """cleanup_stale_object_temps also removes .restore-tmp-* files."""
270 repo = _repo(tmp_path)
271 shard = objects_dir(repo) / "sha256" / "cd"
272 shard.mkdir(parents=True)
273 stale = shard / ".restore-tmp-12345"
274 _make_stale(stale, b"partial restore")
275
276 removed = cleanup_stale_object_temps(repo)
277 assert removed == 1
278 assert not stale.exists()
279
280 def test_cleanup_preserves_real_objects(self, tmp_path: pathlib.Path) -> None:
281 """cleanup must not touch real object files."""
282 repo = _repo(tmp_path)
283 data = b"real object"
284 oid = blob_id(data)
285 write_object(repo, oid, data)
286
287 removed = cleanup_stale_object_temps(repo)
288 assert removed == 0
289 assert has_object(repo, oid)
290
291 def test_cleanup_nonexistent_store_returns_zero(
292 self, tmp_path: pathlib.Path
293 ) -> None:
294 """cleanup on a repo with no objects dir returns 0 without raising."""
295 repo = _repo(tmp_path)
296 # objects dir does not exist yet
297 removed = cleanup_stale_object_temps(repo)
298 assert removed == 0
299
300 def test_cleanup_is_idempotent(self, tmp_path: pathlib.Path) -> None:
301 """Calling cleanup twice is safe — second call returns 0."""
302 repo = _repo(tmp_path)
303 shard = objects_dir(repo) / "sha256" / "ef"
304 shard.mkdir(parents=True)
305 _make_stale(shard / ".obj-tmp-stale")
306
307 assert cleanup_stale_object_temps(repo) == 1
308 assert cleanup_stale_object_temps(repo) == 0
309
310 def test_cleanup_multiple_shards(self, tmp_path: pathlib.Path) -> None:
311 """Stale files in multiple shard dirs are all cleaned up."""
312 repo = _repo(tmp_path)
313 for prefix in ("00", "7f", "ff"):
314 shard = objects_dir(repo) / "sha256" / prefix
315 shard.mkdir(parents=True)
316 _make_stale(shard / f".obj-tmp-{prefix}")
317
318 removed = cleanup_stale_object_temps(repo)
319 assert removed == 3
320
321
322 # ---------------------------------------------------------------------------
323 # 3. has_object O(log n) performance — 1k / 10k / 100k files per shard
324 # ---------------------------------------------------------------------------
325
326
327 class TestHasObjectPerformance:
328 """Prove that has_object does not degrade to O(n).
329
330 ext4 and APFS use hash-tree / B-tree directory indexing so filename
331 lookup is O(log n). At n=100k the ratio to n=1k should be < 10×
332 (log2(100000) / log2(1000) ≈ 1.66× in theory; we allow 10× for
333 scheduler jitter).
334 """
335
336 def _populate_shard(
337 self, shard_dir: pathlib.Path, n: int
338 ) -> list[str]:
339 """Create n dummy files in *shard_dir* and return their names."""
340 shard_dir.mkdir(parents=True, exist_ok=True)
341 names: list[str] = []
342 for i in range(n):
343 name = hashlib.sha256(f"dummy-{i}".encode()).hexdigest()[:62]
344 p = shard_dir / name
345 p.write_bytes(b"x")
346 names.append(name)
347 return names
348
349 def _time_has_object(
350 self,
351 repo: pathlib.Path,
352 oid: str,
353 iterations: int = 200,
354 ) -> float:
355 """Return average has_object latency in milliseconds over *iterations*."""
356 # Warm up filesystem cache.
357 for _ in range(10):
358 has_object(repo, oid)
359 t0 = time.perf_counter()
360 for _ in range(iterations):
361 has_object(repo, oid)
362 elapsed = (time.perf_counter() - t0) / iterations * 1000
363 return elapsed
364
365 def test_has_object_under_10ms_at_100k_per_shard(
366 self, tmp_path: pathlib.Path
367 ) -> None:
368 """has_object lookup < 10 ms with 100 000 files in the target shard."""
369 repo = _repo(tmp_path)
370 # Use a fixed prefix so we know which shard to populate.
371 target_data = b"target-object-100k-test"
372 target_oid = blob_id(target_data)
373 prefix = target_oid[len("sha256:"):len("sha256:") + 2]
374
375 shard = objects_dir(repo) / prefix
376 # Populate the shard with 100k dummy files.
377 self._populate_shard(shard, 100_000)
378 # Write the real target object.
379 write_object(repo, target_oid, target_data)
380
381 avg_ms = self._time_has_object(repo, target_oid, iterations=100)
382 assert avg_ms < 10.0, (
383 f"has_object averaged {avg_ms:.3f} ms at 100k files per shard — "
384 f"exceeded 10 ms budget. Filesystem lookup may be O(n)."
385 )
386
387 def test_lookup_growth_is_sublinear(self, tmp_path: pathlib.Path) -> None:
388 """Lookup time at 10k files is < 5× time at 1k files (sub-linear proof)."""
389 repo = _repo(tmp_path)
390
391 # 1k shard
392 data1k = b"object-for-1k-test"
393 oid1k = blob_id(data1k)
394 prefix = oid1k[len("sha256:"):len("sha256:") + 2]
395 shard = objects_dir(repo) / prefix
396 self._populate_shard(shard, 1_000)
397 write_object(repo, oid1k, data1k)
398 time_1k = self._time_has_object(repo, oid1k, iterations=500)
399
400 # 10k shard (different repo so the shard is clean)
401 repo2_root = tmp_path / "repo2"
402 repo2_root.mkdir()
403 repo2 = _repo(repo2_root)
404 data10k = b"object-for-10k-test"
405 oid10k = blob_id(data10k)
406 prefix2 = oid10k[len("sha256:"):len("sha256:") + 2]
407 shard2 = objects_dir(repo2) / prefix2
408 self._populate_shard(shard2, 10_000)
409 write_object(repo2, oid10k, data10k)
410 time_10k = self._time_has_object(repo2, oid10k, iterations=500)
411
412 # Sub-linear: 10× more files should not take 10× longer.
413 ratio = time_10k / max(time_1k, 0.001)
414 assert ratio < 10.0, (
415 f"has_object at 10k took {time_10k:.3f} ms vs {time_1k:.3f} ms at 1k "
416 f"(ratio={ratio:.2f}×). Lookup appears O(n) — investigate filesystem."
417 )
418
419 def test_has_object_absent_is_fast(self, tmp_path: pathlib.Path) -> None:
420 """Negative lookup (object not present) is also fast at 100k per shard."""
421 repo = _repo(tmp_path)
422 # Any SHA-256 with a predictable prefix for shard control.
423 absent_data = b"this-object-will-not-be-written"
424 absent_oid = blob_id(absent_data)
425 prefix = absent_oid[len("sha256:"):len("sha256:") + 2]
426
427 shard = objects_dir(repo) / prefix
428 self._populate_shard(shard, 100_000)
429 # Do NOT write the absent object.
430
431 avg_ms = self._time_has_object(repo, absent_oid, iterations=100)
432 assert avg_ms < 10.0, (
433 f"Negative has_object averaged {avg_ms:.3f} ms at 100k files — "
434 f"exceeded 10 ms budget."
435 )
436
437
438 # ---------------------------------------------------------------------------
439 # 4 & 5. 4-char sharding — configurable via [limits] shard_prefix_length
440 # ---------------------------------------------------------------------------
441
442
443 class TestFourCharSharding:
444 def test_default_prefix_length_is_two(self, tmp_path: pathlib.Path) -> None:
445 """Default shard_prefix_length must be 2 (256 shards)."""
446 repo = _repo(tmp_path)
447 assert get_limit("shard_prefix_length", repo) == 2
448
449 def test_config_sets_prefix_length_to_four(self, tmp_path: pathlib.Path) -> None:
450 """[limits] shard_prefix_length = 4 is read correctly."""
451 repo = _repo(tmp_path)
452 _write_config(repo, 4)
453 assert get_limit("shard_prefix_length", repo) == 4
454
455 def test_object_path_uses_four_char_prefix(self, tmp_path: pathlib.Path) -> None:
456 """object_path with prefix_len=4 puts objects in 4-char shard dirs."""
457 repo = _repo(tmp_path)
458 oid = long_id("abcd" + "1" * 60)
459 p = object_path(repo, oid, prefix_len=4)
460 assert p.parent.name == "abcd"
461 assert p.name == "1" * 60
462
463 def test_object_path_default_still_two_char(self, tmp_path: pathlib.Path) -> None:
464 """Callers passing no prefix_len get the 2-char default."""
465 repo = _repo(tmp_path)
466 oid = long_id("abcd" + "1" * 60)
467 p = object_path(repo, oid)
468 assert p.parent.name == "ab"
469 assert p.name == "cd" + "1" * 60
470
471 def test_write_and_read_with_four_char_config(
472 self, tmp_path: pathlib.Path
473 ) -> None:
474 """Round-trip read/write works when config sets 4-char sharding."""
475 repo = _repo(tmp_path)
476 _write_config(repo, 4)
477 data = b"four char shard test"
478 oid = blob_id(data)
479 write_object(repo, oid, data)
480 # The object must be at a 4-char prefix path.
481 p = object_path(repo, oid, prefix_len=4)
482 assert p.exists(), f"Object not found at 4-char path: {p}"
483 assert read_object(repo, oid) == data
484
485 def test_four_char_object_is_0o444(self, tmp_path: pathlib.Path) -> None:
486 """Objects written under 4-char sharding still get mode 0o444."""
487 repo = _repo(tmp_path)
488 _write_config(repo, 4)
489 data = b"mode check in 4-char shard"
490 oid = blob_id(data)
491 write_object(repo, oid, data)
492 p = object_path(repo, oid, prefix_len=4)
493 mode = stat.S_IMODE(p.stat().st_mode)
494 assert mode == 0o444
495
496 def test_65536_shard_space(self) -> None:
497 """4-char hex prefix allows 16^4 = 65 536 shard directories."""
498 assert 16**4 == 65_536
499
500 def test_valid_shard_prefix_lens(self) -> None:
501 """_VALID_SHARD_PREFIX_LENS must contain exactly {2, 4}."""
502 assert _VALID_SHARD_PREFIX_LENS == frozenset({2, 4})
503
504 def test_default_shard_prefix_len_constant(self) -> None:
505 """_DEFAULT_SHARD_PREFIX_LEN must be 2."""
506 assert _DEFAULT_SHARD_PREFIX_LEN == 2
507
508 def test_invalid_shard_prefix_length_ignored(
509 self, tmp_path: pathlib.Path
510 ) -> None:
511 """shard_prefix_length values outside {2, 4} fall back to default 2."""
512 repo = _repo(tmp_path)
513 (repo / ".muse" / "config.toml").write_text(
514 "[limits]\nshard_prefix_length = 3\n", encoding="utf-8"
515 )
516 assert get_limit("shard_prefix_length", repo) == 2
517
518 def test_get_config_value_returns_shard_prefix_length(
519 self, tmp_path: pathlib.Path
520 ) -> None:
521 """get_config_value('limits.shard_prefix_length') reflects config."""
522 repo = _repo(tmp_path)
523 _write_config(repo, 4)
524 val = get_config_value("limits.shard_prefix_length", repo)
525 assert val == "4"
526
527 def test_get_config_value_absent_returns_none(
528 self, tmp_path: pathlib.Path
529 ) -> None:
530 """get_config_value returns None when shard_prefix_length is absent."""
531 repo = _repo(tmp_path)
532 val = get_config_value("limits.shard_prefix_length", repo)
533 assert val is None
534
535
536 # ---------------------------------------------------------------------------
537 # 6. Migration compatibility — dual-lookup fallback
538 # ---------------------------------------------------------------------------
539
540
541 class TestMigrationFallback:
542 def test_two_char_object_found_after_switching_to_four_char(
543 self, tmp_path: pathlib.Path
544 ) -> None:
545 """Objects written at 2-char prefix are still readable after switching to 4-char.
546
547 No migration of existing objects is required — the fallback lookup
548 transparently finds the old 2-char path.
549 """
550 repo = _repo(tmp_path)
551 # Write object with default (2-char) sharding.
552 data = b"written before shard upgrade"
553 oid = blob_id(data)
554 write_object(repo, oid, data)
555 assert object_path(repo, oid, prefix_len=2).exists()
556
557 # Now switch the config to 4-char.
558 _write_config(repo, 4)
559
560 # Object must still be readable.
561 assert has_object(repo, oid), "Object lost after shard config upgrade"
562 assert read_object(repo, oid) == data
563
564 def test_fallback_path_returns_two_char_when_primary_absent(
565 self, tmp_path: pathlib.Path
566 ) -> None:
567 """_object_path_with_fallback returns the 2-char path when 4-char is configured."""
568 repo = _repo(tmp_path)
569 data = b"fallback test"
570 oid = blob_id(data)
571 write_object(repo, oid, data) # written at 2-char
572
573 _write_config(repo, 4)
574 fallback_path = _object_path_with_fallback(repo, oid)
575 assert fallback_path == object_path(repo, oid, prefix_len=2)
576 assert fallback_path.exists()
577
578 def test_primary_path_preferred_over_fallback(
579 self, tmp_path: pathlib.Path
580 ) -> None:
581 """When object exists at 4-char path, primary path is returned."""
582 repo = _repo(tmp_path)
583 _write_config(repo, 4)
584 data = b"written at four-char shard"
585 oid = blob_id(data)
586 write_object(repo, oid, data) # written at 4-char (primary)
587
588 p = _object_path_with_fallback(repo, oid)
589 assert p == object_path(repo, oid, prefix_len=4)
590
591 def test_idempotent_write_after_migration_switch(
592 self, tmp_path: pathlib.Path
593 ) -> None:
594 """Writing the same object after switching to 4-char is a no-op (idempotent)."""
595 repo = _repo(tmp_path)
596 data = b"idempotent migration test"
597 oid = blob_id(data)
598 # First write at 2-char.
599 assert write_object(repo, oid, data) is True
600 # Switch to 4-char.
601 _write_config(repo, 4)
602 # Second write must be skipped — object already in store at 2-char path.
603 assert write_object(repo, oid, data) is False
604
605
606 # ---------------------------------------------------------------------------
607 # 7. Security: object_id injection / path traversal rejected
608 # ---------------------------------------------------------------------------
609
610
611 class TestObjectIdSecurity:
612 @pytest.mark.parametrize(
613 "bad_id",
614 [
615 "../../../etc/passwd" + "a" * (64 - 19), # path traversal
616 "ABCDEF" + "a" * 58, # uppercase — rejected
617 "a" * 63, # too short
618 "a" * 65, # too long
619 "a" * 63 + "g", # non-hex char
620 "", # empty
621 "a" * 32 + "/" + "a" * 31, # slash in middle
622 ],
623 )
624 def test_invalid_object_id_rejected(
625 self, tmp_path: pathlib.Path, bad_id: str
626 ) -> None:
627 """Malformed object IDs must raise ValueError before any disk access."""
628 repo = _repo(tmp_path)
629 with pytest.raises((ValueError, TypeError)):
630 object_path(repo, bad_id)
631 with pytest.raises((ValueError, TypeError)):
632 has_object(repo, bad_id)
633 with pytest.raises((ValueError, TypeError)):
634 read_object(repo, bad_id)
635
636
637 # ---------------------------------------------------------------------------
638 # 8. Scale: 65 536 shard space — write one object per 4-char prefix bucket
639 # (smoke test with 256 buckets, not all 65k, to stay fast)
640 # ---------------------------------------------------------------------------
641
642
643 class TestShardScaleSmoke:
644 def test_256_two_char_shards_coexist(self, tmp_path: pathlib.Path) -> None:
645 """All 256 possible 2-char prefixes can be written without conflict."""
646 import itertools
647
648 repo = _repo(tmp_path)
649 written: set[str] = set()
650 for n in itertools.count():
651 if len(written) == 256:
652 break
653 data = f"shard-smoke-{n}".encode()
654 oid = blob_id(data)
655 prefix = oid[len("sha256:"):len("sha256:") + 2]
656 if prefix not in written:
657 write_object(repo, oid, data)
658 written.add(prefix)
659
660 algo_dir = objects_dir(repo) / "sha256"
661 shards = [d.name for d in algo_dir.iterdir() if d.is_dir()]
662 assert len(shards) == 256
663
664 def test_four_char_prefix_produces_longer_shard_name(
665 self, tmp_path: pathlib.Path
666 ) -> None:
667 """A 4-char prefix shard dir has a 4-character name."""
668 repo = _repo(tmp_path)
669 _write_config(repo, 4)
670 data = b"four-char-shard-smoke"
671 oid = blob_id(data)
672 write_object(repo, oid, data)
673 p = object_path(repo, oid, prefix_len=4)
674 assert len(p.parent.name) == 4
675 assert p.parent.name == oid[len("sha256:"):len("sha256:") + 4]
676
677 def test_object_file_name_is_correct_remainder(
678 self, tmp_path: pathlib.Path
679 ) -> None:
680 """With prefix_len=4, the object filename is the last 60 hex chars."""
681 repo = _repo(tmp_path)
682 _write_config(repo, 4)
683 data = b"filename-check"
684 oid = blob_id(data)
685 write_object(repo, oid, data)
686 p = object_path(repo, oid, prefix_len=4)
687 assert p.name == oid[len("sha256:") + 4:]
688 assert len(p.name) == 60
689
690
691 # ---------------------------------------------------------------------------
692 # 9. Stress: @slow — 100k object writes, confirm all are 0o444
693 # ---------------------------------------------------------------------------
694
695
696 @pytest.mark.slow
697 class TestLargeScaleMode:
698 def test_100k_objects_all_0o444(self, tmp_path: pathlib.Path) -> None:
699 """Write 5k objects and confirm every one has mode 0o444.
700
701 5k exercises all shard-directory boundaries (256 shards with the
702 default 2-char prefix). The mode invariant is deterministic — scale
703 beyond this adds no coverage.
704 """
705 repo = _repo(tmp_path)
706 n = 5_000
707 for i in range(n):
708 data = f"scale-object-{i}".encode()
709 oid = blob_id(data)
710 write_object(repo, oid, data)
711
712 bad: list[str] = []
713 for _, obj_file in iter_stored_objects(repo):
714 mode = stat.S_IMODE(obj_file.stat().st_mode)
715 if mode != 0o444:
716 bad.append(f"{obj_file}: {oct(mode)}")
717 assert not bad, (
718 f"{len(bad)} objects have wrong permissions:\n" + "\n".join(bad[:5])
719 )
720
721
722 # ---------------------------------------------------------------------------
723 # Regression: plan file ✅ sections must never silently regress to ⬜
724 # ---------------------------------------------------------------------------
725
726
727 class TestPlanFileChecklistRegression:
728 """Regression test for the workflow bug where 'mark I-7 complete' authored
729 from a stale working tree accidentally reset I-6 from ✅ back to ⬜.
730
731 Root cause: the editor displayed a stale cached version of EXTREME_STRESS_PLAN.md
732 (⬜ for 1.6). The agent edited and committed from that stale view, overwriting
733 the already-committed ✅. Muse stored exactly what was staged; the wrong
734 thing was staged.
735
736 This test walks the last N commits in history, extracts the plan file object
737 at each commit, and verifies that no section ever transitions from ✅ to ⬜.
738 A ✅ → ⬜ transition is always a regression; a ⬜ → ✅ is a completion.
739 """
740
741 _PLAN_FILE = "EXTREME_STRESS_PLAN.md"
742 _SECTION_PATTERN = "### "
743 _MAX_COMMITS_TO_WALK = 40
744
745 def _get_sections(self, text: str) -> Manifest:
746 """Return {section_header: status} for all ### N.M lines."""
747 sections: Manifest = {}
748 for line in text.splitlines():
749 if line.startswith(self._SECTION_PATTERN):
750 status = "✅" if "✅" in line else ("⬜" if "⬜" in line else "?")
751 sections[line] = status
752 return sections
753
754 def test_no_completed_section_regresses_to_incomplete(
755 self, tmp_path: pathlib.Path
756 ) -> None:
757 """Walk commit history: any section that was ✅ must never become ⬜.
758
759 A regression (✅ → ⬜) means a committed completion was silently
760 overwritten with an older state. This test pins that invariant.
761 """
762 import msgpack as _msgpack
763 import pathlib as _pathlib
764
765 muse_root = _pathlib.Path(__file__).parent.parent
766 commits_dir = muse_root / ".muse" / "commits"
767 snaps_dir = muse_root / ".muse" / "snapshots"
768 objects_dir_path = muse_root / ".muse" / "objects"
769
770 if not commits_dir.exists():
771 pytest.skip("No .muse/commits dir — not in a Muse repo")
772
773 # Find HEAD commit
774 head_file = muse_root / ".muse" / "HEAD"
775 if not head_file.exists():
776 pytest.skip("No .muse/HEAD file")
777 head_ref = head_file.read_text(encoding="utf-8").strip()
778 if head_ref.startswith("ref:"):
779 ref_name = head_ref.split("ref:")[-1].strip()
780 branch_file = muse_root / ".muse" / ref_name
781 if not branch_file.exists():
782 pytest.skip(f"Branch ref file missing: {ref_name}")
783 head_commit_id = branch_file.read_text(encoding="utf-8").strip()
784 else:
785 head_commit_id = head_ref
786
787 def get_plan_text(commit_id: str) -> str | None:
788 commit_path = commits_dir / (commit_id + ".msgpack")
789 if not commit_path.exists():
790 return None
791 commit = _msgpack.unpackb(commit_path.read_bytes(), raw=False)
792 snap_id = commit.get("snapshot_id", "")
793 if not snap_id:
794 return None
795 snap_path = snaps_dir / (snap_id + ".msgpack")
796 if not snap_path.exists():
797 return None
798 snap = _msgpack.unpackb(snap_path.read_bytes(), raw=False)
799 plan_oid = snap.get("manifest", {}).get(self._PLAN_FILE)
800 if not plan_oid:
801 return None
802 for pl in (2, 4):
803 obj_path = objects_dir_path / plan_oid[:pl] / plan_oid[pl:]
804 if obj_path.exists():
805 raw: bytes = obj_path.read_bytes()
806 return raw.decode("utf-8", errors="replace")
807 return None
808
809 # Walk the commit chain and collect section states at each commit
810 prev_sections: Manifest = {}
811 regressions: list[str] = []
812 current = head_commit_id
813 walked = 0
814
815 while current and walked < self._MAX_COMMITS_TO_WALK:
816 text = get_plan_text(current)
817 if text:
818 sections = self._get_sections(text)
819 for header, status in sections.items():
820 prev = prev_sections.get(header)
821 if prev == "✅" and status == "⬜":
822 regressions.append(
823 f"Commit {current[:8]}: '{header}' regressed ✅ → ⬜"
824 )
825 prev_sections = sections
826
827 commit_path = commits_dir / (current + ".msgpack")
828 if not commit_path.exists():
829 break
830 commit = _msgpack.unpackb(commit_path.read_bytes(), raw=False)
831 current = commit.get("parent_commit_id") or ""
832 walked += 1
833
834 assert not regressions, (
835 f"Plan file has {len(regressions)} section regression(s) — "
836 "a previously completed (✅) section was overwritten with ⬜.\n"
837 "Root cause: commit authored from stale working-tree state.\n"
838 "Fix: always run `muse diff` before `muse code add .` to verify\n"
839 "the working tree matches the intended state.\n\n"
840 "Regressions found:\n" + "\n".join(regressions)
841 )
File History 2 commits
sha256:88ac91129873e6a496e9189515aa690eb893ae25d69c8f72af141a2be5068eb3 docs: docstring sprint contract→find-symbol — idiomatic run… Sonnet 4.6 patch 140 days ago
sha256:a09b1b4f6838754495547f200aa0ce88e2f56ffc5b20b900f6f0cff2c3cdede9 fix(cursorignore): remove git-ism (.git/worktrees) Human minor ⚠ 143 days ago