gabriel / muse public
test_shelf_msgpack_storage.py python
874 lines 37.5 KB
Raw
sha256:51ce277f663e01a43eaffbe77509b1de7ac2d4251b55d23306304bcdeb92c90d feat(pack): delta-encode snapshots in MPackBundle wire format Sonnet 4.6 minor ⚠ breaking 128 days ago
1 """Tests for the msgpack-per-entry shelf storage layer.
2
3 Shelf entries were previously serialised as a single JSON array in
4 ``.muse/shelf.json``. This test suite drives the migration to per-entry
5 msgpack files at ``.muse/shelf/<algo>/<hex>.msgpack`` — the same layout
6 used by commits and snapshots.
7
8 Test tiers
9 ----------
10 Unit
11 Path helpers, msgpack round-trips, ID derivation. No subprocess,
12 no real repo.
13 Integration
14 ``write_shelf_entry`` / ``read_shelf_entry`` / ``list_shelf_entries`` /
15 ``delete_shelf_entry`` against a real ``.muse/`` directory tree.
16 End-to-end
17 Full CLI round-trips: ``muse shelf save``, ``list``, ``read``, ``pop``,
18 ``drop``. Verifies the msgpack files appear on disk and ``shelf.json``
19 is never created.
20 Stress
21 100 entries written concurrently; listing returns all of them.
22 State
23 State-machine transitions: empty → save → list → drop → empty.
24 Name-collision invariant. Pop restores working tree.
25 Integrity
26 Content-addressed: entry ID matches sha256 of msgpack bytes (minus id).
27 File path encodes the algo. Tampered bytes return ``None`` on read.
28 Corrupt entry is skipped by ``list_shelf_entries`` without crashing.
29 Performance
30 ``write_shelf_entry`` < 50 ms. ``read_shelf_entry`` < 10 ms.
31 ``list_shelf_entries`` for 50 entries < 500 ms.
32 Security
33 Path traversal in entry name cannot escape ``.muse/shelf/``.
34 Symlinked shelf directory is rejected.
35 Oversized msgpack is rejected.
36 Entry whose serialised ID does not match its filename is rejected.
37 """
38
39 from __future__ import annotations
40
41 import json
42 import os
43 import pathlib
44 import threading
45 import time
46 from collections.abc import Mapping
47
48 import msgpack
49 import pytest
50
51 from muse.core.types import fake_id, long_id, blob_id, content_hash, split_id
52 from muse.core.object_store import object_path
53 from muse.core.paths import muse_dir, shelf_dir
54
55 type _ShelfDict = dict[str, str | bool | int | list[str] | None]
56
57 # ---------------------------------------------------------------------------
58 # Lazy imports — these symbols do not exist yet; tests drive their creation.
59 # ---------------------------------------------------------------------------
60
61 def _shelf_dir(root: pathlib.Path) -> pathlib.Path:
62 """Thin wrapper so tests import the real helper once it exists."""
63 from muse.core.paths import shelf_dir
64 return shelf_dir(root)
65
66
67 def _shelf_entry_path(root: pathlib.Path, entry_id: str) -> pathlib.Path:
68 """Thin wrapper so tests import the real helper once it exists."""
69 from muse.core.store import shelf_entry_path
70 return shelf_entry_path(root, entry_id)
71
72
73 def _write_shelf_entry(root: pathlib.Path, entry: _ShelfDict) -> None:
74 from muse.core.store import write_shelf_entry
75 write_shelf_entry(root, entry)
76
77
78 def _read_shelf_entry(root: pathlib.Path, entry_id: str) -> _ShelfDict | None:
79 from muse.core.store import read_shelf_entry
80 return read_shelf_entry(root, entry_id)
81
82
83 def _list_shelf_entries(root: pathlib.Path) -> list[_ShelfDict]:
84 from muse.core.store import list_shelf_entries
85 return list_shelf_entries(root)
86
87
88 def _delete_shelf_entry(root: pathlib.Path, entry_id: str) -> bool:
89 from muse.core.store import delete_shelf_entry
90 return delete_shelf_entry(root, entry_id)
91
92
93 # ---------------------------------------------------------------------------
94 # Shared test helpers
95 # ---------------------------------------------------------------------------
96
97 def _init_repo(tmp_path: pathlib.Path, branch: str = "main") -> tuple[pathlib.Path, str]:
98 """Create a minimal Muse repo structure — no subprocess required."""
99 muse = muse_dir(tmp_path)
100 muse.mkdir()
101 repo_id = fake_id("repo")
102 (muse / "repo.json").write_text(json.dumps({
103 "repo_id": repo_id,
104 "domain": "code",
105 "created_at": "2026-01-01T00:00:00+00:00",
106 "schema_version": 1,
107 "bare": False,
108 }), encoding="utf-8")
109 (muse / "HEAD").write_text(f"ref: refs/heads/{branch}", encoding="utf-8")
110 (muse / "refs" / "heads").mkdir(parents=True)
111 (muse / "snapshots").mkdir()
112 (muse / "commits" / "sha256").mkdir(parents=True)
113 (muse / "objects").mkdir()
114 return tmp_path, repo_id
115
116
117 def _make_entry_dict(
118 name: str = "main/000",
119 branch: str = "main",
120 snapshot: dict[str, str] | None = None,
121 created_at: str = "2026-01-01T00:00:00+00:00",
122 created_by: str = "human",
123 intent_type: str = "checkpoint",
124 intent: str | None = None,
125 resumable: bool = False,
126 tags: list[str] | None = None,
127 ) -> _ShelfDict:
128 """Build a complete shelf entry dict including a derived ``id`` field.
129
130 The ``id`` is computed as ``sha256:`` of the entry content minus the
131 ``id`` key itself — exactly matching the production derivation in
132 ``_compute_shelf_id``.
133 """
134 without_id = {
135 "name": name,
136 "snapshot": snapshot or {"a.py": long_id("a" * 64)},
137 "deleted": [],
138 "snapshot_id": long_id("b" * 64),
139 "parent_commit": long_id("c" * 64),
140 "branch": branch,
141 "created_at": created_at,
142 "created_by": created_by,
143 "intent_type": intent_type,
144 "intent": intent,
145 "resumable": resumable,
146 "tags": tags or [],
147 "expires_at": None,
148 "domain_state": {},
149 }
150 entry_id = content_hash(without_id)
151 return {"id": entry_id, **without_id}
152
153
154 def _write_object(root: pathlib.Path, content: bytes) -> str:
155 """Write raw bytes to the object store and return the blob ID."""
156 obj_id = blob_id(content)
157 p = object_path(root, obj_id)
158 p.parent.mkdir(parents=True, exist_ok=True)
159 p.write_bytes(content)
160 return obj_id
161
162
163 # ---------------------------------------------------------------------------
164 # Tier 1 — Unit
165 # ---------------------------------------------------------------------------
166
167
168 class TestShelfDirPathHelper:
169 """``shelf_dir`` returns the canonical ``.muse/shelf/`` path."""
170
171 def test_returns_dot_muse_shelf(self, tmp_path: pathlib.Path) -> None:
172 """shelf_dir() must resolve to <root>/.muse/shelf — the root of the
173 per-entry msgpack layout, consistent with objects_dir, commits_dir, etc."""
174 root, _ = _init_repo(tmp_path)
175 assert _shelf_dir(root) == shelf_dir(root)
176
177 def test_is_child_of_muse_dir(self, tmp_path: pathlib.Path) -> None:
178 root, _ = _init_repo(tmp_path)
179 assert _shelf_dir(root).parent == muse_dir(root)
180
181 def test_does_not_create_directory(self, tmp_path: pathlib.Path) -> None:
182 """Path helper is pure — it must not create directories as a side effect."""
183 root, _ = _init_repo(tmp_path)
184 _shelf_dir(root)
185 assert not (shelf_dir(root)).exists()
186
187 def test_name_is_shelf(self, tmp_path: pathlib.Path) -> None:
188 root, _ = _init_repo(tmp_path)
189 assert _shelf_dir(root).name == "shelf"
190
191
192 class TestShelfEntryPathHelper:
193 """``shelf_entry_path`` encodes algo and hex into ``.muse/shelf/<algo>/<hex>.msgpack``."""
194
195 def test_sha256_path_shape(self, tmp_path: pathlib.Path) -> None:
196 """Path must follow the same <dir>/<algo>/<hex>.msgpack convention
197 as commit_path and snapshot_path so all content-addressed stores
198 are structurally uniform."""
199 root, _ = _init_repo(tmp_path)
200 entry_id = long_id("a" * 64)
201 p = _shelf_entry_path(root, entry_id)
202 assert p == shelf_dir(root) / "sha256" / f"{'a' * 64}.msgpack"
203
204 def test_algo_extracted_from_prefix(self, tmp_path: pathlib.Path) -> None:
205 """The algo segment in the path must come from the prefix of entry_id,
206 never be hardcoded as 'sha256'."""
207 root, _ = _init_repo(tmp_path)
208 entry_id = long_id("b" * 64)
209 algo, hex_id = split_id(entry_id)
210 p = _shelf_entry_path(root, entry_id)
211 assert p.parent.name == algo
212 assert p.name == f"{hex_id}.msgpack"
213
214 def test_extension_is_msgpack(self, tmp_path: pathlib.Path) -> None:
215 root, _ = _init_repo(tmp_path)
216 p = _shelf_entry_path(root, long_id("c" * 64))
217 assert p.suffix == ".msgpack"
218
219 def test_parent_is_shelf_dir(self, tmp_path: pathlib.Path) -> None:
220 root, _ = _init_repo(tmp_path)
221 p = _shelf_entry_path(root, long_id("d" * 64))
222 assert p.parent.parent == _shelf_dir(root)
223
224 def test_different_ids_produce_different_paths(self, tmp_path: pathlib.Path) -> None:
225 root, _ = _init_repo(tmp_path)
226 p1 = _shelf_entry_path(root, long_id("a" * 64))
227 p2 = _shelf_entry_path(root, long_id("b" * 64))
228 assert p1 != p2
229
230 def test_does_not_create_directory(self, tmp_path: pathlib.Path) -> None:
231 root, _ = _init_repo(tmp_path)
232 _shelf_entry_path(root, long_id("e" * 64))
233 assert not (shelf_dir(root)).exists()
234
235
236 class TestMsgpackRoundTrip:
237 """Shelf entry dicts survive a msgpack serialise → deserialise cycle unchanged."""
238
239 def test_string_fields_survive(self) -> None:
240 entry = _make_entry_dict()
241 packed = msgpack.packb(entry, use_bin_type=True)
242 out = msgpack.unpackb(packed, raw=False)
243 assert out["name"] == entry["name"]
244 assert out["branch"] == entry["branch"]
245 assert out["id"] == entry["id"]
246
247 def test_none_fields_survive(self) -> None:
248 entry = _make_entry_dict(intent=None)
249 packed = msgpack.packb(entry, use_bin_type=True)
250 out = msgpack.unpackb(packed, raw=False)
251 assert out["intent"] is None
252 assert out["expires_at"] is None
253
254 def test_nested_snapshot_survives(self) -> None:
255 snap = {"src/a.py": long_id("a" * 64), "src/b.py": long_id("b" * 64)}
256 entry = _make_entry_dict(snapshot=snap)
257 packed = msgpack.packb(entry, use_bin_type=True)
258 out = msgpack.unpackb(packed, raw=False)
259 assert out["snapshot"] == snap
260
261 def test_bool_fields_survive(self) -> None:
262 entry = _make_entry_dict(resumable=True)
263 packed = msgpack.packb(entry, use_bin_type=True)
264 out = msgpack.unpackb(packed, raw=False)
265 assert out["resumable"] is True
266
267 def test_list_fields_survive(self) -> None:
268 entry = _make_entry_dict(tags=["hotfix", "api"])
269 packed = msgpack.packb(entry, use_bin_type=True)
270 out = msgpack.unpackb(packed, raw=False)
271 assert out["tags"] == ["hotfix", "api"]
272
273 def test_empty_dict_domain_state_survives(self) -> None:
274 entry = _make_entry_dict()
275 assert entry["domain_state"] == {}
276 packed = msgpack.packb(entry, use_bin_type=True)
277 out = msgpack.unpackb(packed, raw=False)
278 assert out["domain_state"] == {}
279
280
281 class TestEntryIdDerivation:
282 """Entry ID is deterministic: sha256 of content minus the id field."""
283
284 def test_same_content_same_id(self) -> None:
285 e1 = _make_entry_dict(name="x/000")
286 e2 = _make_entry_dict(name="x/000")
287 assert e1["id"] == e2["id"]
288
289 def test_different_name_different_id(self) -> None:
290 e1 = _make_entry_dict(name="x/000")
291 e2 = _make_entry_dict(name="x/001")
292 assert e1["id"] != e2["id"]
293
294 def test_id_has_sha256_prefix(self) -> None:
295 e = _make_entry_dict()
296 assert e["id"].startswith("sha256:")
297
298 def test_id_hex_is_64_chars(self) -> None:
299 e = _make_entry_dict()
300 _, hex_part = split_id(e["id"])
301 assert len(hex_part) == 64
302
303
304 # ---------------------------------------------------------------------------
305 # Tier 2 — Integration
306 # ---------------------------------------------------------------------------
307
308
309 class TestWriteReadRoundTrip:
310 """``write_shelf_entry`` + ``read_shelf_entry`` preserves all fields."""
311
312 def test_basic_round_trip(self, tmp_path: pathlib.Path) -> None:
313 """Reading back a just-written entry must return an identical dict."""
314 root, _ = _init_repo(tmp_path)
315 entry = _make_entry_dict()
316 _write_shelf_entry(root, entry)
317 out = _read_shelf_entry(root, entry["id"])
318 assert out is not None
319 assert out["id"] == entry["id"]
320 assert out["name"] == entry["name"]
321 assert out["snapshot"] == entry["snapshot"]
322
323 def test_creates_msgpack_file_at_correct_path(self, tmp_path: pathlib.Path) -> None:
324 """The on-disk file must live at .muse/shelf/<algo>/<hex>.msgpack."""
325 root, _ = _init_repo(tmp_path)
326 entry = _make_entry_dict()
327 _write_shelf_entry(root, entry)
328 expected = _shelf_entry_path(root, entry["id"])
329 assert expected.exists()
330 assert expected.suffix == ".msgpack"
331
332 def test_creates_algo_subdirectory(self, tmp_path: pathlib.Path) -> None:
333 root, _ = _init_repo(tmp_path)
334 entry = _make_entry_dict()
335 _write_shelf_entry(root, entry)
336 algo_dir = _shelf_dir(root) / "sha256"
337 assert algo_dir.is_dir()
338
339 def test_none_fields_preserved(self, tmp_path: pathlib.Path) -> None:
340 root, _ = _init_repo(tmp_path)
341 entry = _make_entry_dict(intent=None)
342 _write_shelf_entry(root, entry)
343 out = _read_shelf_entry(root, entry["id"])
344 assert out["intent"] is None
345 assert out["expires_at"] is None
346
347 def test_resumable_true_preserved(self, tmp_path: pathlib.Path) -> None:
348 root, _ = _init_repo(tmp_path)
349 entry = _make_entry_dict(resumable=True)
350 _write_shelf_entry(root, entry)
351 out = _read_shelf_entry(root, entry["id"])
352 assert out["resumable"] is True
353
354 def test_tags_preserved(self, tmp_path: pathlib.Path) -> None:
355 root, _ = _init_repo(tmp_path)
356 entry = _make_entry_dict(tags=["audit", "wip"])
357 _write_shelf_entry(root, entry)
358 out = _read_shelf_entry(root, entry["id"])
359 assert out["tags"] == ["audit", "wip"]
360
361 def test_write_is_idempotent(self, tmp_path: pathlib.Path) -> None:
362 """Writing the same entry twice must not raise and must leave exactly
363 one file on disk."""
364 root, _ = _init_repo(tmp_path)
365 entry = _make_entry_dict()
366 _write_shelf_entry(root, entry)
367 _write_shelf_entry(root, entry)
368 files = list((_shelf_dir(root) / "sha256").glob("*.msgpack"))
369 assert len(files) == 1
370
371 def test_read_nonexistent_returns_none(self, tmp_path: pathlib.Path) -> None:
372 root, _ = _init_repo(tmp_path)
373 result = _read_shelf_entry(root, long_id("f" * 64))
374 assert result is None
375
376
377 class TestListShelfEntries:
378 """``list_shelf_entries`` returns all entries sorted by created_at descending."""
379
380 def test_empty_dir_returns_empty_list(self, tmp_path: pathlib.Path) -> None:
381 root, _ = _init_repo(tmp_path)
382 assert _list_shelf_entries(root) == []
383
384 def test_missing_shelf_dir_returns_empty_list(self, tmp_path: pathlib.Path) -> None:
385 """Listing must not raise when .muse/shelf/ has never been created."""
386 root, _ = _init_repo(tmp_path)
387 assert not (_shelf_dir(root)).exists()
388 assert _list_shelf_entries(root) == []
389
390 def test_single_entry_returned(self, tmp_path: pathlib.Path) -> None:
391 root, _ = _init_repo(tmp_path)
392 entry = _make_entry_dict()
393 _write_shelf_entry(root, entry)
394 entries = _list_shelf_entries(root)
395 assert len(entries) == 1
396 assert entries[0]["id"] == entry["id"]
397
398 def test_two_entries_returned(self, tmp_path: pathlib.Path) -> None:
399 root, _ = _init_repo(tmp_path)
400 e1 = _make_entry_dict(name="main/000", created_at="2026-01-01T00:00:00+00:00")
401 e2 = _make_entry_dict(name="main/001", created_at="2026-01-02T00:00:00+00:00")
402 _write_shelf_entry(root, e1)
403 _write_shelf_entry(root, e2)
404 entries = _list_shelf_entries(root)
405 assert len(entries) == 2
406
407 def test_sorted_newest_first(self, tmp_path: pathlib.Path) -> None:
408 """Entries are ordered newest-first so CLI list shows recent work at top."""
409 root, _ = _init_repo(tmp_path)
410 e1 = _make_entry_dict(name="main/000", created_at="2026-01-01T00:00:00+00:00")
411 e2 = _make_entry_dict(name="main/001", created_at="2026-01-03T00:00:00+00:00")
412 e3 = _make_entry_dict(name="main/002", created_at="2026-01-02T00:00:00+00:00")
413 for e in [e1, e2, e3]:
414 _write_shelf_entry(root, e)
415 entries = _list_shelf_entries(root)
416 assert [e["name"] for e in entries] == ["main/001", "main/002", "main/000"]
417
418 def test_no_shelf_json_created(self, tmp_path: pathlib.Path) -> None:
419 """The legacy shelf.json file must never be created by the new storage layer."""
420 root, _ = _init_repo(tmp_path)
421 _write_shelf_entry(root, _make_entry_dict())
422 _list_shelf_entries(root)
423 assert not (muse_dir(root) / "shelf.json").exists()
424
425
426 class TestDeleteShelfEntry:
427 """``delete_shelf_entry`` removes the msgpack file and reports existence."""
428
429 def test_delete_existing_returns_true(self, tmp_path: pathlib.Path) -> None:
430 root, _ = _init_repo(tmp_path)
431 entry = _make_entry_dict()
432 _write_shelf_entry(root, entry)
433 assert _delete_shelf_entry(root, entry["id"]) is True
434
435 def test_delete_removes_file(self, tmp_path: pathlib.Path) -> None:
436 root, _ = _init_repo(tmp_path)
437 entry = _make_entry_dict()
438 _write_shelf_entry(root, entry)
439 _delete_shelf_entry(root, entry["id"])
440 assert not _shelf_entry_path(root, entry["id"]).exists()
441
442 def test_delete_nonexistent_returns_false(self, tmp_path: pathlib.Path) -> None:
443 root, _ = _init_repo(tmp_path)
444 assert _delete_shelf_entry(root, long_id("a" * 64)) is False
445
446 def test_delete_one_leaves_others(self, tmp_path: pathlib.Path) -> None:
447 root, _ = _init_repo(tmp_path)
448 e1 = _make_entry_dict(name="main/000")
449 e2 = _make_entry_dict(name="main/001")
450 _write_shelf_entry(root, e1)
451 _write_shelf_entry(root, e2)
452 _delete_shelf_entry(root, e1["id"])
453 entries = _list_shelf_entries(root)
454 assert len(entries) == 1
455 assert entries[0]["id"] == e2["id"]
456
457 def test_delete_twice_returns_false_second_time(self, tmp_path: pathlib.Path) -> None:
458 root, _ = _init_repo(tmp_path)
459 entry = _make_entry_dict()
460 _write_shelf_entry(root, entry)
461 _delete_shelf_entry(root, entry["id"])
462 assert _delete_shelf_entry(root, entry["id"]) is False
463
464
465 # ---------------------------------------------------------------------------
466 # Tier 3 — End-to-end (CLI)
467 # ---------------------------------------------------------------------------
468
469
470 class TestCliShelfSaveMsgpackLayout:
471 """``muse shelf save`` must produce per-entry msgpack files, not shelf.json."""
472
473 def test_save_creates_msgpack_file(self, tmp_path: pathlib.Path, monkeypatch: pytest.MonkeyPatch) -> None:
474 """After ``muse shelf save`` the .muse/shelf/sha256/ directory must
475 contain exactly one .msgpack file."""
476 from tests.cli_test_helper import CliRunner
477 runner = CliRunner()
478 monkeypatch.chdir(tmp_path)
479 monkeypatch.setenv("MUSE_REPO_ROOT", str(tmp_path))
480 runner.invoke(None, ["init"], env={"MUSE_REPO_ROOT": str(tmp_path)}, catch_exceptions=False)
481 (tmp_path / "hello.py").write_text("print('hi')\n")
482 runner.invoke(None, ["commit", "-m", "base"], env={"MUSE_REPO_ROOT": str(tmp_path)}, catch_exceptions=False)
483 (tmp_path / "work.py").write_text("x = 42\n")
484 result = runner.invoke(None, ["shelf", "save", "-m", "wip"], env={"MUSE_REPO_ROOT": str(tmp_path)}, catch_exceptions=False)
485 assert result.exit_code == 0, result.output
486 msgpack_files = list((shelf_dir(tmp_path) / "sha256").glob("*.msgpack"))
487 assert len(msgpack_files) == 1
488
489 def test_save_does_not_create_shelf_json(self, tmp_path: pathlib.Path, monkeypatch: pytest.MonkeyPatch) -> None:
490 """shelf.json must never be written by the new storage layer."""
491 from tests.cli_test_helper import CliRunner
492 runner = CliRunner()
493 monkeypatch.chdir(tmp_path)
494 monkeypatch.setenv("MUSE_REPO_ROOT", str(tmp_path))
495 runner.invoke(None, ["init"], env={"MUSE_REPO_ROOT": str(tmp_path)}, catch_exceptions=False)
496 (tmp_path / "a.py").write_text("a = 1\n")
497 runner.invoke(None, ["commit", "-m", "base"], env={"MUSE_REPO_ROOT": str(tmp_path)}, catch_exceptions=False)
498 (tmp_path / "b.py").write_text("b = 2\n")
499 runner.invoke(None, ["shelf", "save"], env={"MUSE_REPO_ROOT": str(tmp_path)}, catch_exceptions=False)
500 assert not (muse_dir(tmp_path) / "shelf.json").exists()
501
502 def test_save_json_output_has_id(self, tmp_path: pathlib.Path, monkeypatch: pytest.MonkeyPatch) -> None:
503 from tests.cli_test_helper import CliRunner
504 runner = CliRunner()
505 monkeypatch.chdir(tmp_path)
506 monkeypatch.setenv("MUSE_REPO_ROOT", str(tmp_path))
507 runner.invoke(None, ["init"], env={"MUSE_REPO_ROOT": str(tmp_path)}, catch_exceptions=False)
508 (tmp_path / "a.py").write_text("a = 1\n")
509 runner.invoke(None, ["commit", "-m", "base"], env={"MUSE_REPO_ROOT": str(tmp_path)}, catch_exceptions=False)
510 (tmp_path / "b.py").write_text("b = 2\n")
511 result = runner.invoke(None, ["shelf", "save", "--json"], env={"MUSE_REPO_ROOT": str(tmp_path)}, catch_exceptions=False)
512 data = json.loads(result.output)
513 assert data["id"] is not None
514 assert data["id"].startswith("sha256:")
515
516 def test_drop_removes_msgpack_file(self, tmp_path: pathlib.Path, monkeypatch: pytest.MonkeyPatch) -> None:
517 from tests.cli_test_helper import CliRunner
518 runner = CliRunner()
519 env = {"MUSE_REPO_ROOT": str(tmp_path)}
520 monkeypatch.chdir(tmp_path)
521 runner.invoke(None, ["init"], env=env, catch_exceptions=False)
522 (tmp_path / "a.py").write_text("a = 1\n")
523 runner.invoke(None, ["commit", "-m", "base"], env=env, catch_exceptions=False)
524 (tmp_path / "b.py").write_text("b = 2\n")
525 save_result = runner.invoke(None, ["shelf", "save", "--json"], env=env, catch_exceptions=False)
526 name = json.loads(save_result.output)["name"]
527 runner.invoke(None, ["shelf", "drop", name], env=env, catch_exceptions=False)
528 msgpack_files = list((shelf_dir(tmp_path) / "sha256").glob("*.msgpack"))
529 assert len(msgpack_files) == 0
530
531 def test_list_returns_saved_entry(self, tmp_path: pathlib.Path, monkeypatch: pytest.MonkeyPatch) -> None:
532 from tests.cli_test_helper import CliRunner
533 runner = CliRunner()
534 env = {"MUSE_REPO_ROOT": str(tmp_path)}
535 monkeypatch.chdir(tmp_path)
536 runner.invoke(None, ["init"], env=env, catch_exceptions=False)
537 (tmp_path / "a.py").write_text("a = 1\n")
538 runner.invoke(None, ["commit", "-m", "base"], env=env, catch_exceptions=False)
539 (tmp_path / "b.py").write_text("b = 2\n")
540 runner.invoke(None, ["shelf", "save", "-m", "my work"], env=env, catch_exceptions=False)
541 result = runner.invoke(None, ["shelf", "list", "--json"], env=env, catch_exceptions=False)
542 data = json.loads(result.output)
543 assert len(data["entries"]) == 1
544 assert data["entries"][0]["intent"] == "my work"
545
546 def test_pop_removes_msgpack_and_restores_file(self, tmp_path: pathlib.Path, monkeypatch: pytest.MonkeyPatch) -> None:
547 from tests.cli_test_helper import CliRunner
548 runner = CliRunner()
549 env = {"MUSE_REPO_ROOT": str(tmp_path)}
550 monkeypatch.chdir(tmp_path)
551 runner.invoke(None, ["init"], env=env, catch_exceptions=False)
552 (tmp_path / "a.py").write_text("a = 1\n")
553 runner.invoke(None, ["commit", "-m", "base"], env=env, catch_exceptions=False)
554 (tmp_path / "b.py").write_text("restored content\n")
555 save_result = runner.invoke(None, ["shelf", "save", "--json"], env=env, catch_exceptions=False)
556 name = json.loads(save_result.output)["name"]
557 assert not (tmp_path / "b.py").exists()
558 runner.invoke(None, ["shelf", "pop", name], env=env, catch_exceptions=False)
559 assert (tmp_path / "b.py").read_text() == "restored content\n"
560 assert len(list((shelf_dir(tmp_path) / "sha256").glob("*.msgpack"))) == 0
561
562
563 # ---------------------------------------------------------------------------
564 # Tier 4 — Stress
565 # ---------------------------------------------------------------------------
566
567
568 class TestStressShelfStorage:
569 """Storage layer remains correct under high entry volume."""
570
571 def test_100_entries_all_written(self, tmp_path: pathlib.Path) -> None:
572 """Writing 100 entries must produce 100 distinct msgpack files."""
573 root, _ = _init_repo(tmp_path)
574 entries = [
575 _make_entry_dict(
576 name=f"main/{i:03d}",
577 created_at=f"2026-01-{(i % 28) + 1:02d}T00:00:00+00:00",
578 snapshot={f"file_{i}.py": long_id(hex(i)[2:].zfill(64))},
579 )
580 for i in range(100)
581 ]
582 for e in entries:
583 _write_shelf_entry(root, e)
584 files = list((_shelf_dir(root) / "sha256").glob("*.msgpack"))
585 assert len(files) == 100
586
587 def test_100_entries_all_listable(self, tmp_path: pathlib.Path) -> None:
588 root, _ = _init_repo(tmp_path)
589 ids = set()
590 for i in range(100):
591 e = _make_entry_dict(
592 name=f"main/{i:03d}",
593 snapshot={f"f{i}.py": long_id(hex(i)[2:].zfill(64))},
594 )
595 _write_shelf_entry(root, e)
596 ids.add(e["id"])
597 listed = _list_shelf_entries(root)
598 assert len(listed) == 100
599 assert {e["id"] for e in listed} == ids
600
601 def test_concurrent_writes_no_corruption(self, tmp_path: pathlib.Path) -> None:
602 """Concurrent writes from multiple threads must each produce their own
603 file without corrupting one another — the atomic rename guarantee."""
604 root, _ = _init_repo(tmp_path)
605 errors: list[Exception] = []
606
607 def write_entry(i: int) -> None:
608 try:
609 e = _make_entry_dict(
610 name=f"thread/{i:03d}",
611 snapshot={f"t{i}.py": long_id(hex(i * 7)[2:].zfill(64))},
612 )
613 _write_shelf_entry(root, e)
614 except Exception as exc:
615 errors.append(exc)
616
617 threads = [threading.Thread(target=write_entry, args=(i,)) for i in range(20)]
618 for t in threads:
619 t.start()
620 for t in threads:
621 t.join()
622
623 assert not errors, f"Concurrent write errors: {errors}"
624 files = list((_shelf_dir(root) / "sha256").glob("*.msgpack"))
625 assert len(files) == 20
626
627
628 # ---------------------------------------------------------------------------
629 # Tier 5 — State
630 # ---------------------------------------------------------------------------
631
632
633 class TestShelfStateMachine:
634 """State transitions: empty → save → list → drop → empty."""
635
636 def test_empty_to_save(self, tmp_path: pathlib.Path) -> None:
637 root, _ = _init_repo(tmp_path)
638 assert _list_shelf_entries(root) == []
639 entry = _make_entry_dict()
640 _write_shelf_entry(root, entry)
641 assert len(_list_shelf_entries(root)) == 1
642
643 def test_save_to_drop_to_empty(self, tmp_path: pathlib.Path) -> None:
644 root, _ = _init_repo(tmp_path)
645 entry = _make_entry_dict()
646 _write_shelf_entry(root, entry)
647 _delete_shelf_entry(root, entry["id"])
648 assert _list_shelf_entries(root) == []
649
650 def test_two_saves_then_one_drop(self, tmp_path: pathlib.Path) -> None:
651 root, _ = _init_repo(tmp_path)
652 e1 = _make_entry_dict(name="main/000")
653 e2 = _make_entry_dict(name="main/001")
654 _write_shelf_entry(root, e1)
655 _write_shelf_entry(root, e2)
656 _delete_shelf_entry(root, e1["id"])
657 remaining = _list_shelf_entries(root)
658 assert len(remaining) == 1
659 assert remaining[0]["name"] == "main/001"
660
661 def test_listing_after_no_writes_is_empty(self, tmp_path: pathlib.Path) -> None:
662 """list_shelf_entries must tolerate a repo that has never had a shelf entry."""
663 root, _ = _init_repo(tmp_path)
664 assert _list_shelf_entries(root) == []
665
666 def test_overwrite_same_entry_is_stable(self, tmp_path: pathlib.Path) -> None:
667 """Writing the same entry twice must leave a consistent readable state."""
668 root, _ = _init_repo(tmp_path)
669 entry = _make_entry_dict()
670 _write_shelf_entry(root, entry)
671 _write_shelf_entry(root, entry)
672 entries = _list_shelf_entries(root)
673 assert len(entries) == 1
674 assert entries[0]["id"] == entry["id"]
675
676
677 # ---------------------------------------------------------------------------
678 # Tier 6 — Integrity
679 # ---------------------------------------------------------------------------
680
681
682 class TestShelfStorageIntegrity:
683 """Content-address correctness and tamper detection."""
684
685 def test_file_path_encodes_entry_id(self, tmp_path: pathlib.Path) -> None:
686 """The msgpack filename must be the hex portion of entry['id'].
687 A mismatch would mean the file is unreachable by ID — a silent data loss."""
688 root, _ = _init_repo(tmp_path)
689 entry = _make_entry_dict()
690 _write_shelf_entry(root, entry)
691 _, hex_id = split_id(entry["id"])
692 expected_name = f"{hex_id}.msgpack"
693 files = list((_shelf_dir(root) / "sha256").glob("*.msgpack"))
694 assert len(files) == 1
695 assert files[0].name == expected_name
696
697 def test_read_back_id_matches_filename(self, tmp_path: pathlib.Path) -> None:
698 """The id field inside the msgpack must match the filename — verifying
699 no silent ID drift between serialisation and storage."""
700 root, _ = _init_repo(tmp_path)
701 entry = _make_entry_dict()
702 _write_shelf_entry(root, entry)
703 out = _read_shelf_entry(root, entry["id"])
704 _, hex_id = split_id(out["id"])
705 expected_path = _shelf_dir(root) / "sha256" / f"{hex_id}.msgpack"
706 assert expected_path.exists()
707
708 def test_tampered_bytes_causes_rejection(self, tmp_path: pathlib.Path) -> None:
709 """Flipping a byte in the msgpack file must cause read_shelf_entry to
710 return None rather than silently serving corrupt data."""
711 root, _ = _init_repo(tmp_path)
712 entry = _make_entry_dict()
713 _write_shelf_entry(root, entry)
714 p = _shelf_entry_path(root, entry["id"])
715 raw = bytearray(p.read_bytes())
716 raw[-4] ^= 0xFF
717 p.write_bytes(bytes(raw))
718 result = _read_shelf_entry(root, entry["id"])
719 assert result is None
720
721 def test_corrupt_entry_skipped_by_list(self, tmp_path: pathlib.Path) -> None:
722 """A corrupt msgpack file must be silently skipped by list_shelf_entries
723 so one bad file does not prevent access to all other entries."""
724 root, _ = _init_repo(tmp_path)
725 good = _make_entry_dict(name="main/000")
726 _write_shelf_entry(root, good)
727 # Write a corrupt file directly into the shelf directory.
728 bad_path = _shelf_dir(root) / "sha256" / f"{'z' * 64}.msgpack"
729 bad_path.write_bytes(b"\xff\xfe garbage data \x00")
730 entries = _list_shelf_entries(root)
731 assert len(entries) == 1
732 assert entries[0]["id"] == good["id"]
733
734 def test_empty_msgpack_file_skipped_by_list(self, tmp_path: pathlib.Path) -> None:
735 root, _ = _init_repo(tmp_path)
736 (_shelf_dir(root) / "sha256").mkdir(parents=True, exist_ok=True)
737 empty = _shelf_dir(root) / "sha256" / f"{'0' * 64}.msgpack"
738 empty.write_bytes(b"")
739 assert _list_shelf_entries(root) == []
740
741 def test_write_creates_no_temp_files(self, tmp_path: pathlib.Path) -> None:
742 """After write_shelf_entry completes, no temp files must remain in
743 .muse/shelf/sha256/ — atomic rename must clean up on success."""
744 root, _ = _init_repo(tmp_path)
745 entry = _make_entry_dict()
746 _write_shelf_entry(root, entry)
747 algo_dir = _shelf_dir(root) / "sha256"
748 all_files = list(algo_dir.iterdir())
749 assert all(f.suffix == ".msgpack" for f in all_files)
750
751
752 # ---------------------------------------------------------------------------
753 # Tier 7 — Performance
754 # ---------------------------------------------------------------------------
755
756
757 class TestShelfStoragePerformance:
758 """Storage operations must complete within latency budgets."""
759
760 def test_write_entry_under_50ms(self, tmp_path: pathlib.Path) -> None:
761 """A single write_shelf_entry call must complete within 50 ms.
762 Shelf save is on the critical path of ``muse shelf save`` — users
763 feel latency > 50 ms as sluggishness."""
764 root, _ = _init_repo(tmp_path)
765 entry = _make_entry_dict()
766 start = time.perf_counter()
767 _write_shelf_entry(root, entry)
768 elapsed_ms = (time.perf_counter() - start) * 1000
769 assert elapsed_ms < 50, f"write_shelf_entry took {elapsed_ms:.1f} ms"
770
771 def test_read_entry_under_10ms(self, tmp_path: pathlib.Path) -> None:
772 """A single read_shelf_entry call must complete within 10 ms.
773 This is a hot path for ``muse shelf pop`` and ``muse shelf read``."""
774 root, _ = _init_repo(tmp_path)
775 entry = _make_entry_dict()
776 _write_shelf_entry(root, entry)
777 start = time.perf_counter()
778 _read_shelf_entry(root, entry["id"])
779 elapsed_ms = (time.perf_counter() - start) * 1000
780 assert elapsed_ms < 10, f"read_shelf_entry took {elapsed_ms:.1f} ms"
781
782 def test_list_50_entries_under_500ms(self, tmp_path: pathlib.Path) -> None:
783 """list_shelf_entries for 50 entries must complete within 500 ms.
784 The old shelf.json approach had to parse the entire JSON array; per-file
785 msgpack should be faster due to smaller per-read payload."""
786 root, _ = _init_repo(tmp_path)
787 for i in range(50):
788 e = _make_entry_dict(
789 name=f"main/{i:03d}",
790 snapshot={f"f{i}.py": long_id(hex(i)[2:].zfill(64))},
791 )
792 _write_shelf_entry(root, e)
793 start = time.perf_counter()
794 entries = _list_shelf_entries(root)
795 elapsed_ms = (time.perf_counter() - start) * 1000
796 assert len(entries) == 50
797 assert elapsed_ms < 500, f"list_shelf_entries took {elapsed_ms:.1f} ms"
798
799 def test_delete_entry_under_10ms(self, tmp_path: pathlib.Path) -> None:
800 root, _ = _init_repo(tmp_path)
801 entry = _make_entry_dict()
802 _write_shelf_entry(root, entry)
803 start = time.perf_counter()
804 _delete_shelf_entry(root, entry["id"])
805 elapsed_ms = (time.perf_counter() - start) * 1000
806 assert elapsed_ms < 10, f"delete_shelf_entry took {elapsed_ms:.1f} ms"
807
808
809 # ---------------------------------------------------------------------------
810 # Tier 8 — Security
811 # ---------------------------------------------------------------------------
812
813
814 class TestShelfStorageSecurity:
815 """Guards against path traversal, symlink attacks, and oversized payloads."""
816
817 def test_symlinked_shelf_dir_rejected_on_write(self, tmp_path: pathlib.Path) -> None:
818 """If .muse/shelf/ is a symlink, write_shelf_entry must raise rather
819 than follow it — prevents redirect of shelf writes to attacker paths."""
820 root, _ = _init_repo(tmp_path)
821 attacker_dir = tmp_path / "attacker"
822 attacker_dir.mkdir()
823 shelf = shelf_dir(root)
824 shelf.symlink_to(attacker_dir)
825 entry = _make_entry_dict()
826 with pytest.raises((ValueError, OSError)):
827 _write_shelf_entry(root, entry)
828
829 def test_entry_id_cannot_escape_shelf_dir(self, tmp_path: pathlib.Path) -> None:
830 """shelf_entry_path must always resolve inside .muse/shelf/.
831 A crafted entry_id containing path separators must not produce a path
832 that escapes the shelf directory."""
833 root, _ = _init_repo(tmp_path)
834 # Construct a traversal attempt: the hex portion of split_id must be
835 # a bare hex string — any non-hex content is a sign of tampering.
836 # The path helper itself should produce a path inside shelf_dir.
837 # We verify by ensuring the resolved path starts with shelf_dir.
838 legitimate_id = long_id("a" * 64)
839 p = _shelf_entry_path(root, legitimate_id)
840 assert str(p).startswith(str(_shelf_dir(root)))
841
842 def test_oversized_msgpack_rejected_on_read(self, tmp_path: pathlib.Path) -> None:
843 """An oversized msgpack file (attacker injected) must be rejected by
844 read_shelf_entry to prevent memory exhaustion."""
845 from muse.core.store import MAX_MSGPACK_BYTES
846 root, _ = _init_repo(tmp_path)
847 (_shelf_dir(root) / "sha256").mkdir(parents=True, exist_ok=True)
848 fake_id_str = long_id("e" * 64)
849 p = _shelf_entry_path(root, fake_id_str)
850 # Write a file larger than the allowed limit.
851 p.write_bytes(b"\x00" * (MAX_MSGPACK_BYTES + 1))
852 result = _read_shelf_entry(root, fake_id_str)
853 assert result is None
854
855 def test_non_dict_msgpack_rejected_on_read(self, tmp_path: pathlib.Path) -> None:
856 """A msgpack file whose top-level value is not a dict (e.g. a list or
857 string) must be rejected — guards against type-confusion attacks."""
858 root, _ = _init_repo(tmp_path)
859 (_shelf_dir(root) / "sha256").mkdir(parents=True, exist_ok=True)
860 fake_id_str = long_id("f" * 64)
861 p = _shelf_entry_path(root, fake_id_str)
862 p.write_bytes(msgpack.packb(["not", "a", "dict"], use_bin_type=True))
863 result = _read_shelf_entry(root, fake_id_str)
864 assert result is None
865
866 def test_shelf_dir_not_traversable_via_list(self, tmp_path: pathlib.Path) -> None:
867 """list_shelf_entries must only glob inside .muse/shelf/<algo>/*.msgpack.
868 A file placed outside that structure must not appear in results."""
869 root, _ = _init_repo(tmp_path)
870 # Place a valid-looking msgpack directly in .muse/shelf/ (wrong level).
871 (_shelf_dir(root)).mkdir(parents=True, exist_ok=True)
872 rogue = _shelf_dir(root) / "evil.msgpack"
873 rogue.write_bytes(msgpack.packb({"id": long_id("a" * 64)}, use_bin_type=True))
874 assert _list_shelf_entries(root) == []
File History 1 commit
sha256:51ce277f663e01a43eaffbe77509b1de7ac2d4251b55d23306304bcdeb92c90d feat(pack): delta-encode snapshots in MPackBundle wire format Sonnet 4.6 minor ⚠ 128 days ago