gabriel / muse public
test_cmd_verify_shallow.py python
538 lines 20.9 KB
Raw
sha256:b636f72dcba9e190afb980bece906fa5b717fbde014b76ef023df8cb96e01eb9 docs: expand cache plan with all seven testing tiers and do… Sonnet 4.6 131 days ago
1 """Integration tests for shallow-object-store + promisor-remote architecture.
2
3 Core semantics
4 --------------
5 A Muse repo's local object store is legitimately *shallow*: it may not hold
6 every historical object blob. Missing objects are not automatically failures.
7 Their status depends on what is known about the remote:
8
9 PRESENT → object file exists locally → verified (checked)
10 PROMISED → absent but a promisor remote exists → not a failure; counted
11 MISSING → absent AND no promisor remote at all → kind="object" failure
12
13 Shallow graft semantics
14 -----------------------
15 `.muse/shallow` lists the deepest commits included in local history. The BFS
16 walk in run_verify stops at these commits — it does NOT enqueue their parents.
17 Objects beyond the graft boundary are not expected locally.
18
19 strict mode
20 -----------
21 `run_verify(strict=True)` treats every absent object as a failure, regardless
22 of promisor remotes. Use this when you need to prove complete local integrity.
23
24 Coverage
25 --------
26 U — unit: VerifyResult has promised_objects, shallow_commits, is_shallow,
27 promisor_remotes fields
28 E — promisor: missing objects with promisor → not failures
29 missing objects without promisor → failures
30 F — strict: strict=True fails on promised objects
31 S — shallow: BFS stops at graft boundary; parents beyond not checked
32 C — CLI: --strict flag; JSON output includes new fields
33 I — integration: real repo layout, multi-branch, orphan sweep
34 """
35
36 from __future__ import annotations
37 from collections.abc import Mapping
38
39 import datetime
40 import json
41 import pathlib
42 import threading
43
44 import pytest
45 from tests.cli_test_helper import CliRunner, InvokeResult
46
47 from muse.core._types import blob_id, long_id, fake_id
48 from muse.core.object_store import object_path, write_object
49 from muse.core.shallow import add_shallow, write_shallow
50 from muse.core.snapshot import compute_commit_id, compute_snapshot_id
51 from muse.core.store import CommitRecord, SnapshotRecord, write_commit, write_snapshot
52 from muse.core.verify import run_verify
53
54 runner = CliRunner()
55 _REPO_ID = "shallow-verify-test"
56
57
58 # ---------------------------------------------------------------------------
59 # Helpers
60 # ---------------------------------------------------------------------------
61
62 def _sha(data: bytes) -> str:
63 return blob_id(data)
64
65
66 def _init_repo(
67 path: pathlib.Path,
68 remotes: Mapping[str, object] | None = None,
69 ) -> pathlib.Path:
70 muse = path / ".muse"
71 for d in ("commits", "snapshots", "objects", "refs/heads"):
72 (muse / d).mkdir(parents=True, exist_ok=True)
73 (muse / "HEAD").write_text("ref: refs/heads/main")
74 (muse / "repo.json").write_text(
75 json.dumps({"repo_id": _REPO_ID, "domain": "code"})
76 )
77 if remotes:
78 lines = []
79 for name, cfg in remotes.items():
80 lines.append(f"[remotes.{name}]")
81 lines.append(f'url = "{cfg["url"]}"')
82 if "promisor" in cfg:
83 val = "true" if cfg["promisor"] else "false"
84 lines.append(f"promisor = {val}")
85 (muse / "config.toml").write_text("\n".join(lines) + "\n")
86 return path
87
88
89 def _make_commit(
90 root: pathlib.Path,
91 parent_id: str | None = None,
92 content: bytes = b"data",
93 branch: str = "main",
94 idx: int = 0,
95 write_objects: bool = True,
96 ) -> tuple[str, str]:
97 """Create a commit and return (commit_id, obj_id).
98
99 When write_objects=False, the object is NOT written to the store —
100 simulating a shallow gap.
101 """
102 raw = content + str(idx).encode()
103 obj_id = _sha(raw)
104 if write_objects:
105 write_object(root, obj_id, raw)
106 manifest = {f"file_{idx}.txt": obj_id}
107 snap_id = compute_snapshot_id(manifest)
108 if write_objects:
109 write_snapshot(root, SnapshotRecord(snapshot_id=snap_id, manifest=manifest))
110 else:
111 # Write the snapshot record even for shallow commits so the commit
112 # can be read back, but omit the object file.
113 write_snapshot(root, SnapshotRecord(snapshot_id=snap_id, manifest=manifest))
114 committed_at = (
115 datetime.datetime(2026, 1, 1, tzinfo=datetime.timezone.utc)
116 + datetime.timedelta(hours=idx)
117 )
118 parent_ids = [parent_id] if parent_id else []
119 commit_id = compute_commit_id(
120 parent_ids, snap_id, f"commit {idx}", committed_at.isoformat(),
121 repo_id=_REPO_ID,
122 )
123 write_commit(
124 root,
125 CommitRecord(
126 commit_id=commit_id,
127 repo_id=_REPO_ID,
128 created_on_branch=branch,
129 snapshot_id=snap_id,
130 message=f"commit {idx}",
131 committed_at=committed_at,
132 parent_commit_id=parent_id,
133 ),
134 )
135 (root / ".muse" / "refs" / "heads" / branch).write_text(commit_id)
136 return commit_id, obj_id
137
138
139 def _env(repo: pathlib.Path) -> Mapping[str, str]:
140 return {"MUSE_REPO_ROOT": str(repo)}
141
142
143 def _invoke(repo: pathlib.Path, *args: str) -> InvokeResult:
144 from muse.cli.app import main as cli_main
145 return runner.invoke(cli_main, ["verify", *args], env=_env(repo))
146
147
148 # ---------------------------------------------------------------------------
149 # U — VerifyResult shape: new fields present
150 # ---------------------------------------------------------------------------
151
152 class TestVerifyResultShape:
153 def test_promised_objects_field_present(self, tmp_path: pathlib.Path) -> None:
154 repo = _init_repo(tmp_path)
155 result = run_verify(repo)
156 assert "promised_objects" in result
157
158 def test_shallow_commits_field_present(self, tmp_path: pathlib.Path) -> None:
159 repo = _init_repo(tmp_path)
160 result = run_verify(repo)
161 assert "shallow_commits" in result
162
163 def test_is_shallow_field_present(self, tmp_path: pathlib.Path) -> None:
164 repo = _init_repo(tmp_path)
165 result = run_verify(repo)
166 assert "is_shallow" in result
167
168 def test_promisor_remotes_field_present(self, tmp_path: pathlib.Path) -> None:
169 repo = _init_repo(tmp_path)
170 result = run_verify(repo)
171 assert "promisor_remotes" in result
172
173 def test_promised_objects_zero_for_clean_repo(self, tmp_path: pathlib.Path) -> None:
174 repo = _init_repo(tmp_path)
175 _make_commit(repo, idx=0)
176 result = run_verify(repo)
177 assert result["promised_objects"] == 0
178
179 def test_is_shallow_false_without_shallow_file(self, tmp_path: pathlib.Path) -> None:
180 repo = _init_repo(tmp_path)
181 result = run_verify(repo)
182 assert result["is_shallow"] is False
183
184 def test_promisor_remotes_empty_without_config(self, tmp_path: pathlib.Path) -> None:
185 repo = _init_repo(tmp_path)
186 result = run_verify(repo)
187 assert result["promisor_remotes"] == []
188
189
190 # ---------------------------------------------------------------------------
191 # E — Promisor semantics: PROMISED ≠ failure
192 # ---------------------------------------------------------------------------
193
194 class TestPromisorSemantics:
195 def test_missing_object_with_promisor_not_a_failure(
196 self, tmp_path: pathlib.Path
197 ) -> None:
198 repo = _init_repo(tmp_path, remotes={
199 "local": {"url": "https://localhost:1337/gabriel/muse"},
200 })
201 # Write commit + snapshot but NOT the object — shallow gap
202 _make_commit(repo, idx=0, write_objects=False)
203 result = run_verify(repo)
204 assert result["all_ok"] is True
205 assert result["promised_objects"] >= 1
206 assert result["failures"] == []
207
208 def test_missing_object_without_promisor_is_failure(
209 self, tmp_path: pathlib.Path
210 ) -> None:
211 repo = _init_repo(tmp_path) # no remotes
212 _make_commit(repo, idx=0, write_objects=False)
213 result = run_verify(repo)
214 assert result["all_ok"] is False
215 assert any(f["kind"] == "object" for f in result["failures"])
216
217 def test_promised_objects_counted_correctly(
218 self, tmp_path: pathlib.Path
219 ) -> None:
220 repo = _init_repo(tmp_path, remotes={
221 "local": {"url": "https://localhost:1337/gabriel/muse"},
222 })
223 # 3 commits, each with a missing object
224 prev: str | None = None
225 for i in range(3):
226 prev, _ = _make_commit(repo, parent_id=prev, idx=i, write_objects=False)
227 result = run_verify(repo)
228 assert result["promised_objects"] == 3
229 assert result["all_ok"] is True
230
231 def test_present_objects_not_counted_as_promised(
232 self, tmp_path: pathlib.Path
233 ) -> None:
234 repo = _init_repo(tmp_path, remotes={
235 "local": {"url": "https://localhost:1337/gabriel/muse"},
236 })
237 _make_commit(repo, idx=0, write_objects=True) # object IS present
238 result = run_verify(repo)
239 assert result["promised_objects"] == 0
240
241 def test_promisor_false_opt_out_causes_failure(
242 self, tmp_path: pathlib.Path
243 ) -> None:
244 repo = _init_repo(tmp_path, remotes={
245 "mirror": {"url": "http://mirror.example.com/muse", "promisor": False},
246 })
247 _make_commit(repo, idx=0, write_objects=False)
248 result = run_verify(repo)
249 assert result["all_ok"] is False
250 assert result["promised_objects"] == 0
251
252 def test_promisor_remotes_listed_in_result(
253 self, tmp_path: pathlib.Path
254 ) -> None:
255 repo = _init_repo(tmp_path, remotes={
256 "local": {"url": "https://localhost:1337/gabriel/muse"},
257 "staging": {"url": "https://staging.musehub.ai/gabriel/muse"},
258 })
259 result = run_verify(repo)
260 assert "local" in result["promisor_remotes"]
261 assert "staging" in result["promisor_remotes"]
262
263 def test_mixed_present_and_promised(self, tmp_path: pathlib.Path) -> None:
264 repo = _init_repo(tmp_path, remotes={
265 "local": {"url": "https://localhost:1337/gabriel/muse"},
266 })
267 _make_commit(repo, idx=0, write_objects=True) # PRESENT
268 prev, _ = _make_commit(repo, parent_id=None, idx=1, write_objects=False) # PROMISED
269 # update ref to idx=1
270 result = run_verify(repo)
271 assert result["all_ok"] is True
272 assert result["objects_checked"] >= 1 # idx=0 present and checked
273 assert result["promised_objects"] >= 1 # idx=1 promised
274
275
276 # ---------------------------------------------------------------------------
277 # F — strict mode: promised objects become failures
278 # ---------------------------------------------------------------------------
279
280 class TestStrictMode:
281 def test_strict_fails_on_promised_object(self, tmp_path: pathlib.Path) -> None:
282 repo = _init_repo(tmp_path, remotes={
283 "local": {"url": "https://localhost:1337/gabriel/muse"},
284 })
285 _make_commit(repo, idx=0, write_objects=False)
286 result = run_verify(repo, strict=True)
287 assert result["all_ok"] is False
288 assert any(f["kind"] == "object" for f in result["failures"])
289
290 def test_strict_does_not_change_result_for_present_objects(
291 self, tmp_path: pathlib.Path
292 ) -> None:
293 repo = _init_repo(tmp_path, remotes={
294 "local": {"url": "https://localhost:1337/gabriel/muse"},
295 })
296 _make_commit(repo, idx=0, write_objects=True)
297 result = run_verify(repo, strict=True)
298 assert result["all_ok"] is True
299 assert result["promised_objects"] == 0
300
301 def test_strict_promised_objects_still_zero_in_strict(
302 self, tmp_path: pathlib.Path
303 ) -> None:
304 repo = _init_repo(tmp_path, remotes={
305 "local": {"url": "https://localhost:1337/gabriel/muse"},
306 })
307 _make_commit(repo, idx=0, write_objects=False)
308 result = run_verify(repo, strict=True)
309 # In strict mode, absent objects go to failures, not promised_objects
310 assert result["promised_objects"] == 0
311
312 def test_strict_fail_fast(self, tmp_path: pathlib.Path) -> None:
313 repo = _init_repo(tmp_path, remotes={
314 "local": {"url": "https://localhost:1337/gabriel/muse"},
315 })
316 prev: str | None = None
317 for i in range(5):
318 prev, _ = _make_commit(repo, parent_id=prev, idx=i, write_objects=False)
319 result = run_verify(repo, strict=True, fail_fast=True)
320 assert result["all_ok"] is False
321 assert len(result["failures"]) == 1
322
323
324 # ---------------------------------------------------------------------------
325 # S — shallow graft: BFS stops at boundary
326 # ---------------------------------------------------------------------------
327
328 class TestShallowGraft:
329 def test_is_shallow_true_when_shallow_file_exists(
330 self, tmp_path: pathlib.Path
331 ) -> None:
332 repo = _init_repo(tmp_path)
333 cid, _ = _make_commit(repo, idx=0)
334 add_shallow(repo, cid)
335 result = run_verify(repo)
336 assert result["is_shallow"] is True
337
338 def test_shallow_commits_counted(self, tmp_path: pathlib.Path) -> None:
339 repo = _init_repo(tmp_path)
340 cid, _ = _make_commit(repo, idx=0)
341 add_shallow(repo, cid)
342 result = run_verify(repo)
343 assert result["shallow_commits"] >= 1
344
345 def test_parents_beyond_graft_not_checked(self, tmp_path: pathlib.Path) -> None:
346 """Commit chain: old → graft → new.
347 The graft is in .muse/shallow. The old commit's objects are not in
348 the local store. Verify must NOT report the old commit's objects as
349 missing — they're beyond the graft boundary.
350 """
351 repo = _init_repo(tmp_path) # no remotes — would fail if walked past graft
352
353 # old commit: object NOT in store
354 old_cid, old_obj_id = _make_commit(repo, idx=0, write_objects=False)
355
356 # graft commit: parents=old, object IS in store
357 graft_cid, _ = _make_commit(repo, parent_id=old_cid, idx=1, write_objects=True)
358 add_shallow(repo, graft_cid)
359
360 # current tip: parent=graft, object IS in store
361 tip_cid, _ = _make_commit(repo, parent_id=graft_cid, idx=2, write_objects=True)
362
363 result = run_verify(repo)
364 # No failures: old commit's objects are beyond the graft, not checked
365 assert result["all_ok"] is True, f"Unexpected failures: {result['failures']}"
366
367 def test_graft_objects_themselves_are_checked(self, tmp_path: pathlib.Path) -> None:
368 """The graft commit's own objects ARE expected locally."""
369 repo = _init_repo(tmp_path)
370 cid, obj_id = _make_commit(repo, idx=0, write_objects=True)
371 add_shallow(repo, cid)
372 result = run_verify(repo)
373 assert result["all_ok"] is True
374 assert result["objects_checked"] >= 1
375
376 def test_multiple_grafts(self, tmp_path: pathlib.Path) -> None:
377 # Two grafts on separate branches so both are reachable from branch refs.
378 repo = _init_repo(tmp_path)
379 cid_a, _ = _make_commit(repo, idx=0, branch="main", write_objects=True)
380 cid_b, _ = _make_commit(repo, idx=1, branch="dev", write_objects=True)
381 write_shallow(repo, {cid_a, cid_b})
382 result = run_verify(repo)
383 assert result["shallow_commits"] >= 2
384 assert result["is_shallow"] is True
385
386
387 # ---------------------------------------------------------------------------
388 # C — CLI surface
389 # ---------------------------------------------------------------------------
390
391 class TestCLIShallow:
392 def test_json_has_promised_objects(self, tmp_path: pathlib.Path) -> None:
393 repo = _init_repo(tmp_path)
394 _make_commit(repo, idx=0)
395 d = json.loads(_invoke(repo, "--json").output)
396 assert "promised_objects" in d
397
398 def test_json_has_shallow_commits(self, tmp_path: pathlib.Path) -> None:
399 repo = _init_repo(tmp_path)
400 _make_commit(repo, idx=0)
401 d = json.loads(_invoke(repo, "--json").output)
402 assert "shallow_commits" in d
403
404 def test_json_has_is_shallow(self, tmp_path: pathlib.Path) -> None:
405 repo = _init_repo(tmp_path)
406 _make_commit(repo, idx=0)
407 d = json.loads(_invoke(repo, "--json").output)
408 assert "is_shallow" in d
409
410 def test_json_has_promisor_remotes(self, tmp_path: pathlib.Path) -> None:
411 repo = _init_repo(tmp_path)
412 _make_commit(repo, idx=0)
413 d = json.loads(_invoke(repo, "--json").output)
414 assert "promisor_remotes" in d
415
416 def test_strict_flag_exists(self, tmp_path: pathlib.Path) -> None:
417 repo = _init_repo(tmp_path)
418 _make_commit(repo, idx=0)
419 r = _invoke(repo, "--strict", "--json")
420 # Just check it doesn't error on unknown flag
421 assert r.exit_code in (0, 1) # 0=ok 1=failures
422
423 def test_strict_fails_on_promised_via_cli(self, tmp_path: pathlib.Path) -> None:
424 repo = _init_repo(tmp_path, remotes={
425 "local": {"url": "https://localhost:1337/gabriel/muse"},
426 })
427 _make_commit(repo, idx=0, write_objects=False)
428 # Without --strict: ok
429 r_default = _invoke(repo, "--json")
430 d_default = json.loads(r_default.output)
431 assert d_default["all_ok"] is True
432 # With --strict: failure
433 r_strict = _invoke(repo, "--strict", "--json")
434 assert r_strict.exit_code == 1
435 d_strict = json.loads(r_strict.output)
436 assert d_strict["all_ok"] is False
437
438 def test_is_shallow_true_in_json_when_shallow_file(
439 self, tmp_path: pathlib.Path
440 ) -> None:
441 repo = _init_repo(tmp_path)
442 cid, _ = _make_commit(repo, idx=0)
443 add_shallow(repo, cid)
444 d = json.loads(_invoke(repo, "--json").output)
445 assert d["is_shallow"] is True
446
447 def test_promisor_remotes_listed_in_json(self, tmp_path: pathlib.Path) -> None:
448 repo = _init_repo(tmp_path, remotes={
449 "local": {"url": "https://localhost:1337/gabriel/muse"},
450 })
451 _make_commit(repo, idx=0)
452 d = json.loads(_invoke(repo, "--json").output)
453 assert "local" in d["promisor_remotes"]
454
455
456 # ---------------------------------------------------------------------------
457 # I — Integration: realistic scenario
458 # ---------------------------------------------------------------------------
459
460 class TestIntegration:
461 def test_clean_repo_no_remotes_all_ok(self, tmp_path: pathlib.Path) -> None:
462 repo = _init_repo(tmp_path)
463 prev: str | None = None
464 for i in range(5):
465 prev, _ = _make_commit(repo, parent_id=prev, idx=i)
466 result = run_verify(repo)
467 assert result["all_ok"] is True
468 assert result["promised_objects"] == 0
469
470 def test_shallow_repo_with_promisor_all_ok(self, tmp_path: pathlib.Path) -> None:
471 """Simulate a normal agent repo: recent objects present, history shallow."""
472 repo = _init_repo(tmp_path, remotes={
473 "local": {"url": "https://localhost:1337/gabriel/muse"},
474 })
475 # "old" history: objects not local (shallow gap)
476 prev: str | None = None
477 for i in range(10):
478 prev, _ = _make_commit(repo, parent_id=prev, idx=i, write_objects=False)
479 graft = prev
480 add_shallow(repo, graft)
481 # "recent" history: objects local
482 for i in range(10, 15):
483 prev, _ = _make_commit(repo, parent_id=prev, idx=i, write_objects=True)
484 result = run_verify(repo)
485 assert result["all_ok"] is True
486 assert result["is_shallow"] is True
487 # The graft commit's own objects are verified (they may be absent/promised).
488 # Its ancestors' snapshots are collected during the graft walk and skipped
489 # by the orphan sweep — so only the graft's own missing object counts.
490 assert result["promised_objects"] <= 1 # at most the graft's own object
491 assert result["objects_checked"] >= 5 # recent objects verified
492
493 def test_orphan_snapshot_with_missing_object_and_promisor(
494 self, tmp_path: pathlib.Path
495 ) -> None:
496 repo = _init_repo(tmp_path, remotes={
497 "local": {"url": "https://localhost:1337/gabriel/muse"},
498 })
499 # Orphan snapshot (no branch ref) with missing object
500 obj_id = fake_id("orphan-obj-f")
501 manifest = {"orphan.py": obj_id}
502 snap_id = compute_snapshot_id(manifest)
503 write_snapshot(repo, SnapshotRecord(snapshot_id=snap_id, manifest=manifest))
504 result = run_verify(repo)
505 assert result["all_ok"] is True
506 assert result["promised_objects"] >= 1
507
508 def test_concurrent_reads_stable(self, tmp_path: pathlib.Path) -> None:
509 repo = _init_repo(tmp_path, remotes={
510 "local": {"url": "https://localhost:1337/gabriel/muse"},
511 })
512 prev: str | None = None
513 for i in range(5):
514 prev, _ = _make_commit(repo, parent_id=prev, idx=i)
515
516 results: list[dict] = []
517 errors: list[Exception] = []
518 lock = threading.Lock()
519
520 def _read() -> None:
521 try:
522 r = _invoke(repo, "--json")
523 d = json.loads(r.output)
524 with lock:
525 results.append(d)
526 except Exception as exc:
527 with lock:
528 errors.append(exc)
529
530 threads = [threading.Thread(target=_read) for _ in range(8)]
531 for t in threads:
532 t.start()
533 for t in threads:
534 t.join()
535
536 assert errors == []
537 assert len(results) == 8
538 assert all(d["all_ok"] is True for d in results)
File History 2 commits
sha256:b636f72dcba9e190afb980bece906fa5b717fbde014b76ef023df8cb96e01eb9 docs: expand cache plan with all seven testing tiers and do… Sonnet 4.6 131 days ago
sha256:7f9e2ef5286aedad9c1e6011b4c46ca27f39dbdad6e3409357e36b26e46b3b7c docs: docstring sprint for-each-ref→hotspots — idiomatic ru… Sonnet 4.6 patch 137 days ago