gabriel / muse public
test_cmd_verify_shallow.py python
536 lines 20.8 KB
Raw
sha256:51ce277f663e01a43eaffbe77509b1de7ac2d4251b55d23306304bcdeb92c90d feat(pack): delta-encode snapshots in MPackBundle wire format Sonnet 4.6 minor ⚠ breaking 125 days ago
1 """Integration tests for shallow-object-store + promisor-remote architecture.
2
3 Core semantics
4 --------------
5 A Muse repo's local object store is legitimately *shallow*: it may not hold
6 every historical object blob. Missing objects are not automatically failures.
7 Their status depends on what is known about the remote:
8
9 PRESENT → object file exists locally → verified (checked)
10 PROMISED → absent but a promisor remote exists → not a failure; counted
11 MISSING → absent AND no promisor remote at all → kind="object" failure
12
13 Shallow graft semantics
14 -----------------------
15 `.muse/shallow` lists the deepest commits included in local history. The BFS
16 walk in run_verify stops at these commits — it does NOT enqueue their parents.
17 Objects beyond the graft boundary are not expected locally.
18
19 strict mode
20 -----------
21 `run_verify(strict=True)` treats every absent object as a failure, regardless
22 of promisor remotes. Use this when you need to prove complete local integrity.
23
24 Coverage
25 --------
26 U — unit: VerifyResult has promised_objects, shallow_commits, is_shallow,
27 promisor_remotes fields
28 E — promisor: missing objects with promisor → not failures
29 missing objects without promisor → failures
30 F — strict: strict=True fails on promised objects
31 S — shallow: BFS stops at graft boundary; parents beyond not checked
32 C — CLI: --strict flag; JSON output includes new fields
33 I — integration: real repo layout, multi-branch, orphan sweep
34 """
35
36 from __future__ import annotations
37 from collections.abc import Mapping
38
39 import datetime
40 import json
41 import pathlib
42 import threading
43
44 import pytest
45 from tests.cli_test_helper import CliRunner, InvokeResult
46
47 from muse.core.types import blob_id, long_id, fake_id
48 from muse.core.object_store import object_path, write_object
49 from muse.core.shallow import add_shallow, write_shallow
50 from muse.core.snapshot import compute_commit_id, compute_snapshot_id
51 from muse.core.store import CommitRecord, SnapshotRecord, write_commit, write_snapshot
52 from muse.core.verify import run_verify
53 from muse.core.paths import muse_dir, ref_path
54
55 runner = CliRunner()
56 _REPO_ID = "shallow-verify-test"
57
58
59 # ---------------------------------------------------------------------------
60 # Helpers
61 # ---------------------------------------------------------------------------
62
63
64
65 def _init_repo(
66 path: pathlib.Path,
67 remotes: Mapping[str, object] | None = None,
68 ) -> pathlib.Path:
69 dot_muse = muse_dir(path)
70 for d in ("commits", "snapshots", "objects", "refs/heads"):
71 (dot_muse / d).mkdir(parents=True, exist_ok=True)
72 (dot_muse / "HEAD").write_text("ref: refs/heads/main")
73 (dot_muse / "repo.json").write_text(
74 json.dumps({"repo_id": _REPO_ID, "domain": "code"})
75 )
76 if remotes:
77 lines = []
78 for name, cfg in remotes.items():
79 lines.append(f"[remotes.{name}]")
80 lines.append(f'url = "{cfg["url"]}"')
81 if "promisor" in cfg:
82 val = "true" if cfg["promisor"] else "false"
83 lines.append(f"promisor = {val}")
84 (dot_muse / "config.toml").write_text("\n".join(lines) + "\n")
85 return path
86
87
88 def _make_commit(
89 root: pathlib.Path,
90 parent_id: str | None = None,
91 content: bytes = b"data",
92 branch: str = "main",
93 idx: int = 0,
94 write_objects: bool = True,
95 ) -> tuple[str, str]:
96 """Create a commit and return (commit_id, obj_id).
97
98 When write_objects=False, the object is NOT written to the store —
99 simulating a shallow gap.
100 """
101 raw = content + str(idx).encode()
102 obj_id = blob_id(raw)
103 if write_objects:
104 write_object(root, obj_id, raw)
105 manifest = {f"file_{idx}.txt": obj_id}
106 snap_id = compute_snapshot_id(manifest)
107 if write_objects:
108 write_snapshot(root, SnapshotRecord(snapshot_id=snap_id, manifest=manifest))
109 else:
110 # Write the snapshot record even for shallow commits so the commit
111 # can be read back, but omit the object file.
112 write_snapshot(root, SnapshotRecord(snapshot_id=snap_id, manifest=manifest))
113 committed_at = (
114 datetime.datetime(2026, 1, 1, tzinfo=datetime.timezone.utc)
115 + datetime.timedelta(hours=idx)
116 )
117 parent_ids = [parent_id] if parent_id else []
118 commit_id = compute_commit_id(
119 parent_ids, snap_id, f"commit {idx}", committed_at.isoformat(),
120 )
121 write_commit(
122 root,
123 CommitRecord(
124 repo_id=_REPO_ID,
125 commit_id=commit_id,
126 branch=branch,
127 snapshot_id=snap_id,
128 message=f"commit {idx}",
129 committed_at=committed_at,
130 parent_commit_id=parent_id,
131 ),
132 )
133 (ref_path(root, branch)).write_text(commit_id)
134 return commit_id, obj_id
135
136
137 def _env(repo: pathlib.Path) -> Mapping[str, str]:
138 return {"MUSE_REPO_ROOT": str(repo)}
139
140
141 def _invoke(repo: pathlib.Path, *args: str) -> InvokeResult:
142 from muse.cli.app import main as cli_main
143 return runner.invoke(cli_main, ["verify", *args], env=_env(repo))
144
145
146 # ---------------------------------------------------------------------------
147 # U — VerifyResult shape: new fields present
148 # ---------------------------------------------------------------------------
149
150 class TestVerifyResultShape:
151 def test_promised_objects_field_present(self, tmp_path: pathlib.Path) -> None:
152 repo = _init_repo(tmp_path)
153 result = run_verify(repo)
154 assert "promised_objects" in result
155
156 def test_shallow_commits_field_present(self, tmp_path: pathlib.Path) -> None:
157 repo = _init_repo(tmp_path)
158 result = run_verify(repo)
159 assert "shallow_commits" in result
160
161 def test_is_shallow_field_present(self, tmp_path: pathlib.Path) -> None:
162 repo = _init_repo(tmp_path)
163 result = run_verify(repo)
164 assert "is_shallow" in result
165
166 def test_promisor_remotes_field_present(self, tmp_path: pathlib.Path) -> None:
167 repo = _init_repo(tmp_path)
168 result = run_verify(repo)
169 assert "promisor_remotes" in result
170
171 def test_promised_objects_zero_for_clean_repo(self, tmp_path: pathlib.Path) -> None:
172 repo = _init_repo(tmp_path)
173 _make_commit(repo, idx=0)
174 result = run_verify(repo)
175 assert result["promised_objects"] == 0
176
177 def test_is_shallow_false_without_shallow_file(self, tmp_path: pathlib.Path) -> None:
178 repo = _init_repo(tmp_path)
179 result = run_verify(repo)
180 assert result["is_shallow"] is False
181
182 def test_promisor_remotes_empty_without_config(self, tmp_path: pathlib.Path) -> None:
183 repo = _init_repo(tmp_path)
184 result = run_verify(repo)
185 assert result["promisor_remotes"] == []
186
187
188 # ---------------------------------------------------------------------------
189 # E — Promisor semantics: PROMISED ≠ failure
190 # ---------------------------------------------------------------------------
191
192 class TestPromisorSemantics:
193 def test_missing_object_with_promisor_not_a_failure(
194 self, tmp_path: pathlib.Path
195 ) -> None:
196 repo = _init_repo(tmp_path, remotes={
197 "local": {"url": "https://localhost:1337/gabriel/muse"},
198 })
199 # Write commit + snapshot but NOT the object — shallow gap
200 _make_commit(repo, idx=0, write_objects=False)
201 result = run_verify(repo)
202 assert result["all_ok"] is True
203 assert result["promised_objects"] >= 1
204 assert result["failures"] == []
205
206 def test_missing_object_without_promisor_is_failure(
207 self, tmp_path: pathlib.Path
208 ) -> None:
209 repo = _init_repo(tmp_path) # no remotes
210 _make_commit(repo, idx=0, write_objects=False)
211 result = run_verify(repo)
212 assert result["all_ok"] is False
213 assert any(f["kind"] == "object" for f in result["failures"])
214
215 def test_promised_objects_counted_correctly(
216 self, tmp_path: pathlib.Path
217 ) -> None:
218 repo = _init_repo(tmp_path, remotes={
219 "local": {"url": "https://localhost:1337/gabriel/muse"},
220 })
221 # 3 commits, each with a missing object
222 prev: str | None = None
223 for i in range(3):
224 prev, _ = _make_commit(repo, parent_id=prev, idx=i, write_objects=False)
225 result = run_verify(repo)
226 assert result["promised_objects"] == 3
227 assert result["all_ok"] is True
228
229 def test_present_objects_not_counted_as_promised(
230 self, tmp_path: pathlib.Path
231 ) -> None:
232 repo = _init_repo(tmp_path, remotes={
233 "local": {"url": "https://localhost:1337/gabriel/muse"},
234 })
235 _make_commit(repo, idx=0, write_objects=True) # object IS present
236 result = run_verify(repo)
237 assert result["promised_objects"] == 0
238
239 def test_promisor_false_opt_out_causes_failure(
240 self, tmp_path: pathlib.Path
241 ) -> None:
242 repo = _init_repo(tmp_path, remotes={
243 "mirror": {"url": "http://mirror.example.com/muse", "promisor": False},
244 })
245 _make_commit(repo, idx=0, write_objects=False)
246 result = run_verify(repo)
247 assert result["all_ok"] is False
248 assert result["promised_objects"] == 0
249
250 def test_promisor_remotes_listed_in_result(
251 self, tmp_path: pathlib.Path
252 ) -> None:
253 repo = _init_repo(tmp_path, remotes={
254 "local": {"url": "https://localhost:1337/gabriel/muse"},
255 "staging": {"url": "https://staging.musehub.ai/gabriel/muse"},
256 })
257 result = run_verify(repo)
258 assert "local" in result["promisor_remotes"]
259 assert "staging" in result["promisor_remotes"]
260
261 def test_mixed_present_and_promised(self, tmp_path: pathlib.Path) -> None:
262 repo = _init_repo(tmp_path, remotes={
263 "local": {"url": "https://localhost:1337/gabriel/muse"},
264 })
265 _make_commit(repo, idx=0, write_objects=True) # PRESENT
266 prev, _ = _make_commit(repo, parent_id=None, idx=1, write_objects=False) # PROMISED
267 # update ref to idx=1
268 result = run_verify(repo)
269 assert result["all_ok"] is True
270 assert result["objects_checked"] >= 1 # idx=0 present and checked
271 assert result["promised_objects"] >= 1 # idx=1 promised
272
273
274 # ---------------------------------------------------------------------------
275 # F — strict mode: promised objects become failures
276 # ---------------------------------------------------------------------------
277
278 class TestStrictMode:
279 def test_strict_fails_on_promised_object(self, tmp_path: pathlib.Path) -> None:
280 repo = _init_repo(tmp_path, remotes={
281 "local": {"url": "https://localhost:1337/gabriel/muse"},
282 })
283 _make_commit(repo, idx=0, write_objects=False)
284 result = run_verify(repo, strict=True)
285 assert result["all_ok"] is False
286 assert any(f["kind"] == "object" for f in result["failures"])
287
288 def test_strict_does_not_change_result_for_present_objects(
289 self, tmp_path: pathlib.Path
290 ) -> None:
291 repo = _init_repo(tmp_path, remotes={
292 "local": {"url": "https://localhost:1337/gabriel/muse"},
293 })
294 _make_commit(repo, idx=0, write_objects=True)
295 result = run_verify(repo, strict=True)
296 assert result["all_ok"] is True
297 assert result["promised_objects"] == 0
298
299 def test_strict_promised_objects_still_zero_in_strict(
300 self, tmp_path: pathlib.Path
301 ) -> None:
302 repo = _init_repo(tmp_path, remotes={
303 "local": {"url": "https://localhost:1337/gabriel/muse"},
304 })
305 _make_commit(repo, idx=0, write_objects=False)
306 result = run_verify(repo, strict=True)
307 # In strict mode, absent objects go to failures, not promised_objects
308 assert result["promised_objects"] == 0
309
310 def test_strict_fail_fast(self, tmp_path: pathlib.Path) -> None:
311 repo = _init_repo(tmp_path, remotes={
312 "local": {"url": "https://localhost:1337/gabriel/muse"},
313 })
314 prev: str | None = None
315 for i in range(5):
316 prev, _ = _make_commit(repo, parent_id=prev, idx=i, write_objects=False)
317 result = run_verify(repo, strict=True, fail_fast=True)
318 assert result["all_ok"] is False
319 assert len(result["failures"]) == 1
320
321
322 # ---------------------------------------------------------------------------
323 # S — shallow graft: BFS stops at boundary
324 # ---------------------------------------------------------------------------
325
326 class TestShallowGraft:
327 def test_is_shallow_true_when_shallow_file_exists(
328 self, tmp_path: pathlib.Path
329 ) -> None:
330 repo = _init_repo(tmp_path)
331 cid, _ = _make_commit(repo, idx=0)
332 add_shallow(repo, cid)
333 result = run_verify(repo)
334 assert result["is_shallow"] is True
335
336 def test_shallow_commits_counted(self, tmp_path: pathlib.Path) -> None:
337 repo = _init_repo(tmp_path)
338 cid, _ = _make_commit(repo, idx=0)
339 add_shallow(repo, cid)
340 result = run_verify(repo)
341 assert result["shallow_commits"] >= 1
342
343 def test_parents_beyond_graft_not_checked(self, tmp_path: pathlib.Path) -> None:
344 """Commit chain: old → graft → new.
345 The graft is in .muse/shallow. The old commit's objects are not in
346 the local store. Verify must NOT report the old commit's objects as
347 missing — they're beyond the graft boundary.
348 """
349 repo = _init_repo(tmp_path) # no remotes — would fail if walked past graft
350
351 # old commit: object NOT in store
352 old_cid, old_obj_id = _make_commit(repo, idx=0, write_objects=False)
353
354 # graft commit: parents=old, object IS in store
355 graft_cid, _ = _make_commit(repo, parent_id=old_cid, idx=1, write_objects=True)
356 add_shallow(repo, graft_cid)
357
358 # current tip: parent=graft, object IS in store
359 tip_cid, _ = _make_commit(repo, parent_id=graft_cid, idx=2, write_objects=True)
360
361 result = run_verify(repo)
362 # No failures: old commit's objects are beyond the graft, not checked
363 assert result["all_ok"] is True, f"Unexpected failures: {result['failures']}"
364
365 def test_graft_objects_themselves_are_checked(self, tmp_path: pathlib.Path) -> None:
366 """The graft commit's own objects ARE expected locally."""
367 repo = _init_repo(tmp_path)
368 cid, obj_id = _make_commit(repo, idx=0, write_objects=True)
369 add_shallow(repo, cid)
370 result = run_verify(repo)
371 assert result["all_ok"] is True
372 assert result["objects_checked"] >= 1
373
374 def test_multiple_grafts(self, tmp_path: pathlib.Path) -> None:
375 # Two grafts on separate branches so both are reachable from branch refs.
376 repo = _init_repo(tmp_path)
377 cid_a, _ = _make_commit(repo, idx=0, branch="main", write_objects=True)
378 cid_b, _ = _make_commit(repo, idx=1, branch="dev", write_objects=True)
379 write_shallow(repo, {cid_a, cid_b})
380 result = run_verify(repo)
381 assert result["shallow_commits"] >= 2
382 assert result["is_shallow"] is True
383
384
385 # ---------------------------------------------------------------------------
386 # C — CLI surface
387 # ---------------------------------------------------------------------------
388
389 class TestCLIShallow:
390 def test_json_has_promised_objects(self, tmp_path: pathlib.Path) -> None:
391 repo = _init_repo(tmp_path)
392 _make_commit(repo, idx=0)
393 d = json.loads(_invoke(repo, "--json").output)
394 assert "promised_objects" in d
395
396 def test_json_has_shallow_commits(self, tmp_path: pathlib.Path) -> None:
397 repo = _init_repo(tmp_path)
398 _make_commit(repo, idx=0)
399 d = json.loads(_invoke(repo, "--json").output)
400 assert "shallow_commits" in d
401
402 def test_json_has_is_shallow(self, tmp_path: pathlib.Path) -> None:
403 repo = _init_repo(tmp_path)
404 _make_commit(repo, idx=0)
405 d = json.loads(_invoke(repo, "--json").output)
406 assert "is_shallow" in d
407
408 def test_json_has_promisor_remotes(self, tmp_path: pathlib.Path) -> None:
409 repo = _init_repo(tmp_path)
410 _make_commit(repo, idx=0)
411 d = json.loads(_invoke(repo, "--json").output)
412 assert "promisor_remotes" in d
413
414 def test_strict_flag_exists(self, tmp_path: pathlib.Path) -> None:
415 repo = _init_repo(tmp_path)
416 _make_commit(repo, idx=0)
417 r = _invoke(repo, "--strict", "--json")
418 # Just check it doesn't error on unknown flag
419 assert r.exit_code in (0, 1) # 0=ok 1=failures
420
421 def test_strict_fails_on_promised_via_cli(self, tmp_path: pathlib.Path) -> None:
422 repo = _init_repo(tmp_path, remotes={
423 "local": {"url": "https://localhost:1337/gabriel/muse"},
424 })
425 _make_commit(repo, idx=0, write_objects=False)
426 # Without --strict: ok
427 r_default = _invoke(repo, "--json")
428 d_default = json.loads(r_default.output)
429 assert d_default["all_ok"] is True
430 # With --strict: failure
431 r_strict = _invoke(repo, "--strict", "--json")
432 assert r_strict.exit_code == 1
433 d_strict = json.loads(r_strict.output)
434 assert d_strict["all_ok"] is False
435
436 def test_is_shallow_true_in_json_when_shallow_file(
437 self, tmp_path: pathlib.Path
438 ) -> None:
439 repo = _init_repo(tmp_path)
440 cid, _ = _make_commit(repo, idx=0)
441 add_shallow(repo, cid)
442 d = json.loads(_invoke(repo, "--json").output)
443 assert d["is_shallow"] is True
444
445 def test_promisor_remotes_listed_in_json(self, tmp_path: pathlib.Path) -> None:
446 repo = _init_repo(tmp_path, remotes={
447 "local": {"url": "https://localhost:1337/gabriel/muse"},
448 })
449 _make_commit(repo, idx=0)
450 d = json.loads(_invoke(repo, "--json").output)
451 assert "local" in d["promisor_remotes"]
452
453
454 # ---------------------------------------------------------------------------
455 # I — Integration: realistic scenario
456 # ---------------------------------------------------------------------------
457
458 class TestIntegration:
459 def test_clean_repo_no_remotes_all_ok(self, tmp_path: pathlib.Path) -> None:
460 repo = _init_repo(tmp_path)
461 prev: str | None = None
462 for i in range(5):
463 prev, _ = _make_commit(repo, parent_id=prev, idx=i)
464 result = run_verify(repo)
465 assert result["all_ok"] is True
466 assert result["promised_objects"] == 0
467
468 def test_shallow_repo_with_promisor_all_ok(self, tmp_path: pathlib.Path) -> None:
469 """Simulate a normal agent repo: recent objects present, history shallow."""
470 repo = _init_repo(tmp_path, remotes={
471 "local": {"url": "https://localhost:1337/gabriel/muse"},
472 })
473 # "old" history: objects not local (shallow gap)
474 prev: str | None = None
475 for i in range(10):
476 prev, _ = _make_commit(repo, parent_id=prev, idx=i, write_objects=False)
477 graft = prev
478 add_shallow(repo, graft)
479 # "recent" history: objects local
480 for i in range(10, 15):
481 prev, _ = _make_commit(repo, parent_id=prev, idx=i, write_objects=True)
482 result = run_verify(repo)
483 assert result["all_ok"] is True
484 assert result["is_shallow"] is True
485 # The graft commit's own objects are verified (they may be absent/promised).
486 # Its ancestors' snapshots are collected during the graft walk and skipped
487 # by the orphan sweep — so only the graft's own missing object counts.
488 assert result["promised_objects"] <= 1 # at most the graft's own object
489 assert result["objects_checked"] >= 5 # recent objects verified
490
491 def test_orphan_snapshot_with_missing_object_and_promisor(
492 self, tmp_path: pathlib.Path
493 ) -> None:
494 repo = _init_repo(tmp_path, remotes={
495 "local": {"url": "https://localhost:1337/gabriel/muse"},
496 })
497 # Orphan snapshot (no branch ref) with missing object
498 obj_id = fake_id("orphan-obj-f")
499 manifest = {"orphan.py": obj_id}
500 snap_id = compute_snapshot_id(manifest)
501 write_snapshot(repo, SnapshotRecord(snapshot_id=snap_id, manifest=manifest))
502 result = run_verify(repo)
503 assert result["all_ok"] is True
504 assert result["promised_objects"] >= 1
505
506 def test_concurrent_reads_stable(self, tmp_path: pathlib.Path) -> None:
507 repo = _init_repo(tmp_path, remotes={
508 "local": {"url": "https://localhost:1337/gabriel/muse"},
509 })
510 prev: str | None = None
511 for i in range(5):
512 prev, _ = _make_commit(repo, parent_id=prev, idx=i)
513
514 results: list[dict] = []
515 errors: list[Exception] = []
516 lock = threading.Lock()
517
518 def _read() -> None:
519 try:
520 r = _invoke(repo, "--json")
521 d = json.loads(r.output)
522 with lock:
523 results.append(d)
524 except Exception as exc:
525 with lock:
526 errors.append(exc)
527
528 threads = [threading.Thread(target=_read) for _ in range(8)]
529 for t in threads:
530 t.start()
531 for t in threads:
532 t.join()
533
534 assert errors == []
535 assert len(results) == 8
536 assert all(d["all_ok"] is True for d in results)
File History 1 commit
sha256:51ce277f663e01a43eaffbe77509b1de7ac2d4251b55d23306304bcdeb92c90d feat(pack): delta-encode snapshots in MPackBundle wire format Sonnet 4.6 minor ⚠ 125 days ago