gabriel / muse public
test_cmd_verify_shallow.py python
537 lines 20.8 KB
Raw
sha256:88ac91129873e6a496e9189515aa690eb893ae25d69c8f72af141a2be5068eb3 docs: docstring sprint contract→find-symbol — idiomatic run… Sonnet 4.6 patch 140 days ago
1 """Integration tests for shallow-object-store + promisor-remote architecture.
2
3 Core semantics
4 --------------
5 A Muse repo's local object store is legitimately *shallow*: it may not hold
6 every historical object blob. Missing objects are not automatically failures.
7 Their status depends on what is known about the remote:
8
9 PRESENT → object file exists locally → verified (checked)
10 PROMISED → absent but a promisor remote exists → not a failure; counted
11 MISSING → absent AND no promisor remote at all → kind="object" failure
12
13 Shallow graft semantics
14 -----------------------
15 `.muse/shallow` lists the deepest commits included in local history. The BFS
16 walk in run_verify stops at these commits — it does NOT enqueue their parents.
17 Objects beyond the graft boundary are not expected locally.
18
19 strict mode
20 -----------
21 `run_verify(strict=True)` treats every absent object as a failure, regardless
22 of promisor remotes. Use this when you need to prove complete local integrity.
23
24 Coverage
25 --------
26 U — unit: VerifyResult has promised_objects, shallow_commits, is_shallow,
27 promisor_remotes fields
28 E — promisor: missing objects with promisor → not failures
29 missing objects without promisor → failures
30 F — strict: strict=True fails on promised objects
31 S — shallow: BFS stops at graft boundary; parents beyond not checked
32 C — CLI: --strict flag; JSON output includes new fields
33 I — integration: real repo layout, multi-branch, orphan sweep
34 """
35
36 from __future__ import annotations
37
38 import datetime
39 import hashlib
40 import json
41 import pathlib
42 import threading
43
44 import pytest
45 from tests.cli_test_helper import CliRunner, InvokeResult
46
47 from muse.core._types import long_id
48 from muse.core.object_store import object_path, write_object
49 from muse.core.shallow import add_shallow, write_shallow
50 from muse.core.snapshot import compute_commit_id, compute_snapshot_id
51 from muse.core.store import CommitRecord, SnapshotRecord, write_commit, write_snapshot
52 from muse.core.verify import run_verify
53
54 runner = CliRunner()
55 _REPO_ID = "shallow-verify-test"
56
57
58 # ---------------------------------------------------------------------------
59 # Helpers
60 # ---------------------------------------------------------------------------
61
62 def _sha(data: bytes) -> str:
63 return long_id(hashlib.sha256(data).hexdigest())
64
65
66 def _init_repo(
67 path: pathlib.Path,
68 remotes: dict | None = None,
69 ) -> pathlib.Path:
70 muse = path / ".muse"
71 for d in ("commits", "snapshots", "objects", "refs/heads"):
72 (muse / d).mkdir(parents=True, exist_ok=True)
73 (muse / "HEAD").write_text("ref: refs/heads/main")
74 (muse / "repo.json").write_text(
75 json.dumps({"repo_id": _REPO_ID, "domain": "code"})
76 )
77 if remotes:
78 lines = []
79 for name, cfg in remotes.items():
80 lines.append(f"[remotes.{name}]")
81 lines.append(f'url = "{cfg["url"]}"')
82 if "promisor" in cfg:
83 val = "true" if cfg["promisor"] else "false"
84 lines.append(f"promisor = {val}")
85 (muse / "config.toml").write_text("\n".join(lines) + "\n")
86 return path
87
88
89 def _make_commit(
90 root: pathlib.Path,
91 parent_id: str | None = None,
92 content: bytes = b"data",
93 branch: str = "main",
94 idx: int = 0,
95 write_objects: bool = True,
96 ) -> tuple[str, str]:
97 """Create a commit and return (commit_id, obj_id).
98
99 When write_objects=False, the object is NOT written to the store —
100 simulating a shallow gap.
101 """
102 raw = content + str(idx).encode()
103 obj_id = _sha(raw)
104 if write_objects:
105 write_object(root, obj_id, raw)
106 manifest = {f"file_{idx}.txt": obj_id}
107 snap_id = compute_snapshot_id(manifest)
108 if write_objects:
109 write_snapshot(root, SnapshotRecord(snapshot_id=snap_id, manifest=manifest))
110 else:
111 # Write the snapshot record even for shallow commits so the commit
112 # can be read back, but omit the object file.
113 write_snapshot(root, SnapshotRecord(snapshot_id=snap_id, manifest=manifest))
114 committed_at = (
115 datetime.datetime(2026, 1, 1, tzinfo=datetime.timezone.utc)
116 + datetime.timedelta(hours=idx)
117 )
118 parent_ids = [parent_id] if parent_id else []
119 commit_id = compute_commit_id(
120 parent_ids, snap_id, f"commit {idx}", committed_at.isoformat()
121 )
122 write_commit(
123 root,
124 CommitRecord(
125 commit_id=commit_id,
126 repo_id=_REPO_ID,
127 branch=branch,
128 snapshot_id=snap_id,
129 message=f"commit {idx}",
130 committed_at=committed_at,
131 parent_commit_id=parent_id,
132 ),
133 )
134 (root / ".muse" / "refs" / "heads" / branch).write_text(commit_id)
135 return commit_id, obj_id
136
137
138 def _env(repo: pathlib.Path) -> dict[str, str]:
139 return {"MUSE_REPO_ROOT": str(repo)}
140
141
142 def _invoke(repo: pathlib.Path, *args: str) -> InvokeResult:
143 from muse.cli.app import main as cli_main
144 return runner.invoke(cli_main, ["verify", *args], env=_env(repo))
145
146
147 # ---------------------------------------------------------------------------
148 # U — VerifyResult shape: new fields present
149 # ---------------------------------------------------------------------------
150
151 class TestVerifyResultShape:
152 def test_promised_objects_field_present(self, tmp_path: pathlib.Path) -> None:
153 repo = _init_repo(tmp_path)
154 result = run_verify(repo)
155 assert "promised_objects" in result
156
157 def test_shallow_commits_field_present(self, tmp_path: pathlib.Path) -> None:
158 repo = _init_repo(tmp_path)
159 result = run_verify(repo)
160 assert "shallow_commits" in result
161
162 def test_is_shallow_field_present(self, tmp_path: pathlib.Path) -> None:
163 repo = _init_repo(tmp_path)
164 result = run_verify(repo)
165 assert "is_shallow" in result
166
167 def test_promisor_remotes_field_present(self, tmp_path: pathlib.Path) -> None:
168 repo = _init_repo(tmp_path)
169 result = run_verify(repo)
170 assert "promisor_remotes" in result
171
172 def test_promised_objects_zero_for_clean_repo(self, tmp_path: pathlib.Path) -> None:
173 repo = _init_repo(tmp_path)
174 _make_commit(repo, idx=0)
175 result = run_verify(repo)
176 assert result["promised_objects"] == 0
177
178 def test_is_shallow_false_without_shallow_file(self, tmp_path: pathlib.Path) -> None:
179 repo = _init_repo(tmp_path)
180 result = run_verify(repo)
181 assert result["is_shallow"] is False
182
183 def test_promisor_remotes_empty_without_config(self, tmp_path: pathlib.Path) -> None:
184 repo = _init_repo(tmp_path)
185 result = run_verify(repo)
186 assert result["promisor_remotes"] == []
187
188
189 # ---------------------------------------------------------------------------
190 # E — Promisor semantics: PROMISED ≠ failure
191 # ---------------------------------------------------------------------------
192
193 class TestPromisorSemantics:
194 def test_missing_object_with_promisor_not_a_failure(
195 self, tmp_path: pathlib.Path
196 ) -> None:
197 repo = _init_repo(tmp_path, remotes={
198 "local": {"url": "http://localhost:10003/gabriel/muse"},
199 })
200 # Write commit + snapshot but NOT the object — shallow gap
201 _make_commit(repo, idx=0, write_objects=False)
202 result = run_verify(repo)
203 assert result["all_ok"] is True
204 assert result["promised_objects"] >= 1
205 assert result["failures"] == []
206
207 def test_missing_object_without_promisor_is_failure(
208 self, tmp_path: pathlib.Path
209 ) -> None:
210 repo = _init_repo(tmp_path) # no remotes
211 _make_commit(repo, idx=0, write_objects=False)
212 result = run_verify(repo)
213 assert result["all_ok"] is False
214 assert any(f["kind"] == "object" for f in result["failures"])
215
216 def test_promised_objects_counted_correctly(
217 self, tmp_path: pathlib.Path
218 ) -> None:
219 repo = _init_repo(tmp_path, remotes={
220 "local": {"url": "http://localhost:10003/gabriel/muse"},
221 })
222 # 3 commits, each with a missing object
223 prev: str | None = None
224 for i in range(3):
225 prev, _ = _make_commit(repo, parent_id=prev, idx=i, write_objects=False)
226 result = run_verify(repo)
227 assert result["promised_objects"] == 3
228 assert result["all_ok"] is True
229
230 def test_present_objects_not_counted_as_promised(
231 self, tmp_path: pathlib.Path
232 ) -> None:
233 repo = _init_repo(tmp_path, remotes={
234 "local": {"url": "http://localhost:10003/gabriel/muse"},
235 })
236 _make_commit(repo, idx=0, write_objects=True) # object IS present
237 result = run_verify(repo)
238 assert result["promised_objects"] == 0
239
240 def test_promisor_false_opt_out_causes_failure(
241 self, tmp_path: pathlib.Path
242 ) -> None:
243 repo = _init_repo(tmp_path, remotes={
244 "mirror": {"url": "http://mirror.example.com/muse", "promisor": False},
245 })
246 _make_commit(repo, idx=0, write_objects=False)
247 result = run_verify(repo)
248 assert result["all_ok"] is False
249 assert result["promised_objects"] == 0
250
251 def test_promisor_remotes_listed_in_result(
252 self, tmp_path: pathlib.Path
253 ) -> None:
254 repo = _init_repo(tmp_path, remotes={
255 "local": {"url": "http://localhost:10003/gabriel/muse"},
256 "staging": {"url": "https://staging.musehub.ai/gabriel/muse"},
257 })
258 result = run_verify(repo)
259 assert "local" in result["promisor_remotes"]
260 assert "staging" in result["promisor_remotes"]
261
262 def test_mixed_present_and_promised(self, tmp_path: pathlib.Path) -> None:
263 repo = _init_repo(tmp_path, remotes={
264 "local": {"url": "http://localhost:10003/gabriel/muse"},
265 })
266 _make_commit(repo, idx=0, write_objects=True) # PRESENT
267 prev, _ = _make_commit(repo, parent_id=None, idx=1, write_objects=False) # PROMISED
268 # update ref to idx=1
269 result = run_verify(repo)
270 assert result["all_ok"] is True
271 assert result["objects_checked"] >= 1 # idx=0 present and checked
272 assert result["promised_objects"] >= 1 # idx=1 promised
273
274
275 # ---------------------------------------------------------------------------
276 # F — strict mode: promised objects become failures
277 # ---------------------------------------------------------------------------
278
279 class TestStrictMode:
280 def test_strict_fails_on_promised_object(self, tmp_path: pathlib.Path) -> None:
281 repo = _init_repo(tmp_path, remotes={
282 "local": {"url": "http://localhost:10003/gabriel/muse"},
283 })
284 _make_commit(repo, idx=0, write_objects=False)
285 result = run_verify(repo, strict=True)
286 assert result["all_ok"] is False
287 assert any(f["kind"] == "object" for f in result["failures"])
288
289 def test_strict_does_not_change_result_for_present_objects(
290 self, tmp_path: pathlib.Path
291 ) -> None:
292 repo = _init_repo(tmp_path, remotes={
293 "local": {"url": "http://localhost:10003/gabriel/muse"},
294 })
295 _make_commit(repo, idx=0, write_objects=True)
296 result = run_verify(repo, strict=True)
297 assert result["all_ok"] is True
298 assert result["promised_objects"] == 0
299
300 def test_strict_promised_objects_still_zero_in_strict(
301 self, tmp_path: pathlib.Path
302 ) -> None:
303 repo = _init_repo(tmp_path, remotes={
304 "local": {"url": "http://localhost:10003/gabriel/muse"},
305 })
306 _make_commit(repo, idx=0, write_objects=False)
307 result = run_verify(repo, strict=True)
308 # In strict mode, absent objects go to failures, not promised_objects
309 assert result["promised_objects"] == 0
310
311 def test_strict_fail_fast(self, tmp_path: pathlib.Path) -> None:
312 repo = _init_repo(tmp_path, remotes={
313 "local": {"url": "http://localhost:10003/gabriel/muse"},
314 })
315 prev: str | None = None
316 for i in range(5):
317 prev, _ = _make_commit(repo, parent_id=prev, idx=i, write_objects=False)
318 result = run_verify(repo, strict=True, fail_fast=True)
319 assert result["all_ok"] is False
320 assert len(result["failures"]) == 1
321
322
323 # ---------------------------------------------------------------------------
324 # S — shallow graft: BFS stops at boundary
325 # ---------------------------------------------------------------------------
326
327 class TestShallowGraft:
328 def test_is_shallow_true_when_shallow_file_exists(
329 self, tmp_path: pathlib.Path
330 ) -> None:
331 repo = _init_repo(tmp_path)
332 cid, _ = _make_commit(repo, idx=0)
333 add_shallow(repo, cid)
334 result = run_verify(repo)
335 assert result["is_shallow"] is True
336
337 def test_shallow_commits_counted(self, tmp_path: pathlib.Path) -> None:
338 repo = _init_repo(tmp_path)
339 cid, _ = _make_commit(repo, idx=0)
340 add_shallow(repo, cid)
341 result = run_verify(repo)
342 assert result["shallow_commits"] >= 1
343
344 def test_parents_beyond_graft_not_checked(self, tmp_path: pathlib.Path) -> None:
345 """Commit chain: old → graft → new.
346 The graft is in .muse/shallow. The old commit's objects are not in
347 the local store. Verify must NOT report the old commit's objects as
348 missing — they're beyond the graft boundary.
349 """
350 repo = _init_repo(tmp_path) # no remotes — would fail if walked past graft
351
352 # old commit: object NOT in store
353 old_cid, old_obj_id = _make_commit(repo, idx=0, write_objects=False)
354
355 # graft commit: parents=old, object IS in store
356 graft_cid, _ = _make_commit(repo, parent_id=old_cid, idx=1, write_objects=True)
357 add_shallow(repo, graft_cid)
358
359 # current tip: parent=graft, object IS in store
360 tip_cid, _ = _make_commit(repo, parent_id=graft_cid, idx=2, write_objects=True)
361
362 result = run_verify(repo)
363 # No failures: old commit's objects are beyond the graft, not checked
364 assert result["all_ok"] is True, f"Unexpected failures: {result['failures']}"
365
366 def test_graft_objects_themselves_are_checked(self, tmp_path: pathlib.Path) -> None:
367 """The graft commit's own objects ARE expected locally."""
368 repo = _init_repo(tmp_path)
369 cid, obj_id = _make_commit(repo, idx=0, write_objects=True)
370 add_shallow(repo, cid)
371 result = run_verify(repo)
372 assert result["all_ok"] is True
373 assert result["objects_checked"] >= 1
374
375 def test_multiple_grafts(self, tmp_path: pathlib.Path) -> None:
376 # Two grafts on separate branches so both are reachable from branch refs.
377 repo = _init_repo(tmp_path)
378 cid_a, _ = _make_commit(repo, idx=0, branch="main", write_objects=True)
379 cid_b, _ = _make_commit(repo, idx=1, branch="dev", write_objects=True)
380 write_shallow(repo, {cid_a, cid_b})
381 result = run_verify(repo)
382 assert result["shallow_commits"] >= 2
383 assert result["is_shallow"] is True
384
385
386 # ---------------------------------------------------------------------------
387 # C — CLI surface
388 # ---------------------------------------------------------------------------
389
390 class TestCLIShallow:
391 def test_json_has_promised_objects(self, tmp_path: pathlib.Path) -> None:
392 repo = _init_repo(tmp_path)
393 _make_commit(repo, idx=0)
394 d = json.loads(_invoke(repo, "--json").output)
395 assert "promised_objects" in d
396
397 def test_json_has_shallow_commits(self, tmp_path: pathlib.Path) -> None:
398 repo = _init_repo(tmp_path)
399 _make_commit(repo, idx=0)
400 d = json.loads(_invoke(repo, "--json").output)
401 assert "shallow_commits" in d
402
403 def test_json_has_is_shallow(self, tmp_path: pathlib.Path) -> None:
404 repo = _init_repo(tmp_path)
405 _make_commit(repo, idx=0)
406 d = json.loads(_invoke(repo, "--json").output)
407 assert "is_shallow" in d
408
409 def test_json_has_promisor_remotes(self, tmp_path: pathlib.Path) -> None:
410 repo = _init_repo(tmp_path)
411 _make_commit(repo, idx=0)
412 d = json.loads(_invoke(repo, "--json").output)
413 assert "promisor_remotes" in d
414
415 def test_strict_flag_exists(self, tmp_path: pathlib.Path) -> None:
416 repo = _init_repo(tmp_path)
417 _make_commit(repo, idx=0)
418 r = _invoke(repo, "--strict", "--json")
419 # Just check it doesn't error on unknown flag
420 assert r.exit_code in (0, 1) # 0=ok 1=failures
421
422 def test_strict_fails_on_promised_via_cli(self, tmp_path: pathlib.Path) -> None:
423 repo = _init_repo(tmp_path, remotes={
424 "local": {"url": "http://localhost:10003/gabriel/muse"},
425 })
426 _make_commit(repo, idx=0, write_objects=False)
427 # Without --strict: ok
428 r_default = _invoke(repo, "--json")
429 d_default = json.loads(r_default.output)
430 assert d_default["all_ok"] is True
431 # With --strict: failure
432 r_strict = _invoke(repo, "--strict", "--json")
433 assert r_strict.exit_code == 1
434 d_strict = json.loads(r_strict.output)
435 assert d_strict["all_ok"] is False
436
437 def test_is_shallow_true_in_json_when_shallow_file(
438 self, tmp_path: pathlib.Path
439 ) -> None:
440 repo = _init_repo(tmp_path)
441 cid, _ = _make_commit(repo, idx=0)
442 add_shallow(repo, cid)
443 d = json.loads(_invoke(repo, "--json").output)
444 assert d["is_shallow"] is True
445
446 def test_promisor_remotes_listed_in_json(self, tmp_path: pathlib.Path) -> None:
447 repo = _init_repo(tmp_path, remotes={
448 "local": {"url": "http://localhost:10003/gabriel/muse"},
449 })
450 _make_commit(repo, idx=0)
451 d = json.loads(_invoke(repo, "--json").output)
452 assert "local" in d["promisor_remotes"]
453
454
455 # ---------------------------------------------------------------------------
456 # I — Integration: realistic scenario
457 # ---------------------------------------------------------------------------
458
459 class TestIntegration:
460 def test_clean_repo_no_remotes_all_ok(self, tmp_path: pathlib.Path) -> None:
461 repo = _init_repo(tmp_path)
462 prev: str | None = None
463 for i in range(5):
464 prev, _ = _make_commit(repo, parent_id=prev, idx=i)
465 result = run_verify(repo)
466 assert result["all_ok"] is True
467 assert result["promised_objects"] == 0
468
469 def test_shallow_repo_with_promisor_all_ok(self, tmp_path: pathlib.Path) -> None:
470 """Simulate a normal agent repo: recent objects present, history shallow."""
471 repo = _init_repo(tmp_path, remotes={
472 "local": {"url": "http://localhost:10003/gabriel/muse"},
473 })
474 # "old" history: objects not local (shallow gap)
475 prev: str | None = None
476 for i in range(10):
477 prev, _ = _make_commit(repo, parent_id=prev, idx=i, write_objects=False)
478 graft = prev
479 add_shallow(repo, graft)
480 # "recent" history: objects local
481 for i in range(10, 15):
482 prev, _ = _make_commit(repo, parent_id=prev, idx=i, write_objects=True)
483 result = run_verify(repo)
484 assert result["all_ok"] is True
485 assert result["is_shallow"] is True
486 # The graft commit's own objects are verified (they may be absent/promised).
487 # Its ancestors' snapshots are collected during the graft walk and skipped
488 # by the orphan sweep — so only the graft's own missing object counts.
489 assert result["promised_objects"] <= 1 # at most the graft's own object
490 assert result["objects_checked"] >= 5 # recent objects verified
491
492 def test_orphan_snapshot_with_missing_object_and_promisor(
493 self, tmp_path: pathlib.Path
494 ) -> None:
495 repo = _init_repo(tmp_path, remotes={
496 "local": {"url": "http://localhost:10003/gabriel/muse"},
497 })
498 # Orphan snapshot (no branch ref) with missing object
499 obj_id = long_id("f" * 64)
500 manifest = {"orphan.py": obj_id}
501 snap_id = compute_snapshot_id(manifest)
502 write_snapshot(repo, SnapshotRecord(snapshot_id=snap_id, manifest=manifest))
503 result = run_verify(repo)
504 assert result["all_ok"] is True
505 assert result["promised_objects"] >= 1
506
507 def test_concurrent_reads_stable(self, tmp_path: pathlib.Path) -> None:
508 repo = _init_repo(tmp_path, remotes={
509 "local": {"url": "http://localhost:10003/gabriel/muse"},
510 })
511 prev: str | None = None
512 for i in range(5):
513 prev, _ = _make_commit(repo, parent_id=prev, idx=i)
514
515 results: list[dict] = []
516 errors: list[Exception] = []
517 lock = threading.Lock()
518
519 def _read() -> None:
520 try:
521 r = _invoke(repo, "--json")
522 d = json.loads(r.output)
523 with lock:
524 results.append(d)
525 except Exception as exc:
526 with lock:
527 errors.append(exc)
528
529 threads = [threading.Thread(target=_read) for _ in range(8)]
530 for t in threads:
531 t.start()
532 for t in threads:
533 t.join()
534
535 assert errors == []
536 assert len(results) == 8
537 assert all(d["all_ok"] is True for d in results)
File History 1 commit
sha256:88ac91129873e6a496e9189515aa690eb893ae25d69c8f72af141a2be5068eb3 docs: docstring sprint contract→find-symbol — idiomatic run… Sonnet 4.6 patch 140 days ago