gabriel / muse public
test_verify_supercharge.py python
562 lines 21.3 KB
Raw
sha256:51ce277f663e01a43eaffbe77509b1de7ac2d4251b55d23306304bcdeb92c90d feat(pack): delta-encode snapshots in MPackBundle wire format Sonnet 4.6 minor ⚠ breaking 121 days ago
1 """Supercharge tests for ``muse verify``.
2
3 Every JSON success response must carry ``duration_ms`` (float, ms) and
4 ``exit_code`` (int). Every JSON error response routes to *stdout*, not stderr,
5 so agent pipelines never receive mixed-mode output.
6
7 Coverage tiers
8 --------------
9 U — duration_ms / exit_code on all code paths (clean, failures, no-objects,
10 branch-scoped, fail-fast)
11 E — JSON error routing: _emit_error() writes to stdout in JSON mode, stderr in
12 text mode; no traceback on any error path
13 S — Schema completeness: all _VerifyJson fields present in every success
14 response; all _VerifyErrorJson fields present in every error response
15 D — Data integrity: exit_code=0 ↔ all_ok=True; exit_code=1 ↔ all_ok=False;
16 duration_ms > 0; duration_ms is float; counters are non-negative
17 IO — OSError during run_verify → exit_code=3 in JSON, stderr in text mode
18 P — Performance: duration_ms < 5 000 ms for a 50-commit chain; monotone
19 (two runs on same repo differ only by noise)
20 Sec — No traceback on any error; no raw Python exception in stdout
21 C — Concurrent readers produce valid JSON (10 threads, same repo)
22 """
23
24 from __future__ import annotations
25 from collections.abc import Mapping
26
27 import datetime
28 import json
29 import pathlib
30 import threading
31 import unittest.mock as mock
32
33 import pytest
34 from tests.cli_test_helper import CliRunner, InvokeResult
35
36 from muse.core.types import blob_id, long_id, short_id
37 from muse.core.object_store import object_path, write_object
38 from muse.core.snapshot import compute_commit_id, compute_snapshot_id
39 from muse.core.store import CommitRecord, SnapshotRecord, write_commit, write_snapshot
40 from muse.core.verify import run_verify
41 from muse.core.paths import muse_dir, heads_dir, ref_path
42
43 runner = CliRunner()
44 cli = None # argparse migration — CliRunner ignores this arg
45
46 _REPO_ID = "verify-supercharge-test"
47
48
49 # ---------------------------------------------------------------------------
50 # Helpers
51 # ---------------------------------------------------------------------------
52
53
54
55
56 def _init_repo(path: pathlib.Path) -> pathlib.Path:
57 muse = muse_dir(path)
58 for d in ("commits", "snapshots", "objects", "refs/heads"):
59 (muse / d).mkdir(parents=True, exist_ok=True)
60 (muse / "HEAD").write_text("ref: refs/heads/main", encoding="utf-8")
61 (muse / "repo.json").write_text(
62 json.dumps({"repo_id": _REPO_ID, "domain": "midi"}), encoding="utf-8"
63 )
64 return path
65
66
67 def _env(repo: pathlib.Path) -> Mapping[str, str]:
68 return {"MUSE_REPO_ROOT": str(repo)}
69
70
71 def _make_commit(
72 root: pathlib.Path,
73 parent_id: str | None = None,
74 content: bytes = b"data",
75 branch: str = "main",
76 idx: int = 0,
77 ) -> str:
78 raw = content + str(idx).encode()
79 obj_id = blob_id(raw)
80 write_object(root, obj_id, raw)
81 manifest = {f"file_{idx}.txt": obj_id}
82 snap_id = compute_snapshot_id(manifest)
83 write_snapshot(root, SnapshotRecord(snapshot_id=snap_id, manifest=manifest))
84 committed_at = (
85 datetime.datetime(2026, 1, 1, tzinfo=datetime.timezone.utc)
86 + datetime.timedelta(hours=idx)
87 )
88 parent_ids = [parent_id] if parent_id else []
89 commit_id = compute_commit_id(
90 parent_ids=parent_ids,
91 snapshot_id=snap_id,
92 message=f"commit {idx}",
93 committed_at_iso=committed_at.isoformat(),
94 )
95 write_commit(
96 root,
97 CommitRecord(
98 commit_id=commit_id,
99 repo_id="test-repo",
100 branch=branch,
101 snapshot_id=snap_id,
102 message=f"commit {idx}",
103 committed_at=committed_at,
104 parent_commit_id=parent_id,
105 ),
106 )
107 (ref_path(root, branch)).write_text(commit_id, encoding="utf-8")
108 return commit_id
109
110
111 def _invoke(repo: pathlib.Path, *args: str) -> InvokeResult:
112 from muse.cli.app import main as cli_main
113 return runner.invoke(cli_main, ["verify", *args], env=_env(repo))
114
115
116 # ---------------------------------------------------------------------------
117 # U — duration_ms and exit_code on all code paths
118 # ---------------------------------------------------------------------------
119
120
121 class TestElapsedAndExitCode:
122 def test_clean_repo_exit_code_zero(self, tmp_path: pathlib.Path) -> None:
123 repo = _init_repo(tmp_path)
124 _make_commit(repo, idx=0)
125 r = _invoke(repo, "--json")
126 assert r.exit_code == 0
127 d = json.loads(r.output)
128 assert d["exit_code"] == 0
129
130 def test_clean_repo_duration_ms_present(self, tmp_path: pathlib.Path) -> None:
131 repo = _init_repo(tmp_path)
132 _make_commit(repo, idx=0)
133 r = _invoke(repo, "--json")
134 d = json.loads(r.output)
135 assert "duration_ms" in d
136 assert isinstance(d["duration_ms"], float)
137 assert d["duration_ms"] > 0
138
139 def test_failures_exit_code_one(self, tmp_path: pathlib.Path) -> None:
140 repo = _init_repo(tmp_path)
141 # Write a ref pointing at a non-existent commit (bare hex — invalid ref format)
142 (heads_dir(repo) / "main").write_text("b" * 64)
143 r = _invoke(repo, "--json")
144 assert r.exit_code == 1
145 d = json.loads(r.output)
146 assert d["exit_code"] == 1
147 assert d["all_ok"] is False
148
149 def test_failures_duration_ms_present(self, tmp_path: pathlib.Path) -> None:
150 repo = _init_repo(tmp_path)
151 (heads_dir(repo) / "main").write_text("c" * 64)
152 r = _invoke(repo, "--json")
153 d = json.loads(r.output)
154 assert isinstance(d["duration_ms"], float)
155 assert d["duration_ms"] > 0
156
157 def test_no_objects_exit_code_zero(self, tmp_path: pathlib.Path) -> None:
158 repo = _init_repo(tmp_path)
159 _make_commit(repo, idx=1)
160 r = _invoke(repo, "--json", "--no-objects")
161 assert r.exit_code == 0
162 d = json.loads(r.output)
163 assert d["exit_code"] == 0
164 assert d["duration_ms"] > 0
165
166 def test_branch_scoped_exit_code_zero(self, tmp_path: pathlib.Path) -> None:
167 repo = _init_repo(tmp_path)
168 _make_commit(repo, idx=2)
169 r = _invoke(repo, "--json", "--branch", "main")
170 assert r.exit_code == 0
171 d = json.loads(r.output)
172 assert d["exit_code"] == 0
173 assert d["duration_ms"] > 0
174
175 def test_fail_fast_exit_code_one(self, tmp_path: pathlib.Path) -> None:
176 repo = _init_repo(tmp_path)
177 (heads_dir(repo) / "main").write_text("d" * 64)
178 r = _invoke(repo, "--json", "--fail-fast")
179 assert r.exit_code == 1
180 d = json.loads(r.output)
181 assert d["exit_code"] == 1
182 assert d["duration_ms"] > 0
183
184
185 # ---------------------------------------------------------------------------
186 # E — Error routing: JSON mode → stdout; text mode → stderr
187 # ---------------------------------------------------------------------------
188
189
190 class TestErrorRouting:
191 def test_io_error_json_to_stdout(self, tmp_path: pathlib.Path) -> None:
192 """OSError during run_verify → JSON error on stdout in JSON mode."""
193 repo = _init_repo(tmp_path)
194 _make_commit(repo, idx=0)
195 with mock.patch(
196 "muse.cli.commands.verify.run_verify",
197 side_effect=OSError("disk full"),
198 ):
199 r = _invoke(repo, "--json")
200 assert r.stderr.strip() == "", f"Expected empty stderr, got: {r.stderr!r}"
201 d = json.loads(r.output)
202 assert d["error"] == "io_error"
203 assert "disk full" in d["message"]
204 assert d["exit_code"] == 3
205 assert isinstance(d["duration_ms"], float)
206
207 def test_io_error_text_to_stderr(self, tmp_path: pathlib.Path) -> None:
208 """OSError during run_verify → error on stderr in text mode."""
209 repo = _init_repo(tmp_path)
210 _make_commit(repo, idx=0)
211 with mock.patch(
212 "muse.cli.commands.verify.run_verify",
213 side_effect=OSError("disk full"),
214 ):
215 r = _invoke(repo)
216 assert r.exit_code == 3
217 assert r.stderr.strip() != ""
218 assert "disk full" in r.stderr
219
220 def test_io_error_quiet_no_output(self, tmp_path: pathlib.Path) -> None:
221 """OSError in quiet mode → no output at all, exit 3."""
222 repo = _init_repo(tmp_path)
223 _make_commit(repo, idx=0)
224 with mock.patch(
225 "muse.cli.commands.verify.run_verify",
226 side_effect=OSError("disk full"),
227 ):
228 r = _invoke(repo, "--quiet")
229 assert r.exit_code == 3
230 assert r.output.strip() == ""
231 assert r.stderr.strip() == ""
232
233 def test_io_error_json_no_traceback(self, tmp_path: pathlib.Path) -> None:
234 """No Python traceback lands on stdout in JSON mode."""
235 repo = _init_repo(tmp_path)
236 _make_commit(repo, idx=0)
237 with mock.patch(
238 "muse.cli.commands.verify.run_verify",
239 side_effect=OSError("broken pipe"),
240 ):
241 r = _invoke(repo, "--json")
242 assert "Traceback" not in r.output
243 assert "Traceback" not in r.stderr
244
245 def test_io_error_json_schema(self, tmp_path: pathlib.Path) -> None:
246 """JSON error payload has exactly the documented keys."""
247 repo = _init_repo(tmp_path)
248 _make_commit(repo, idx=0)
249 with mock.patch(
250 "muse.cli.commands.verify.run_verify",
251 side_effect=OSError("nfs timeout"),
252 ):
253 r = _invoke(repo, "--json")
254 d = json.loads(r.output)
255 assert set(d) >= {"error", "message", "duration_ms", "exit_code"}
256
257
258 # ---------------------------------------------------------------------------
259 # S — Schema completeness
260 # ---------------------------------------------------------------------------
261
262 _SUCCESS_KEYS = {
263 "repo_id", "refs_checked", "commits_checked", "snapshots_checked",
264 "objects_checked", "signatures_checked", "all_ok", "nothing_checked",
265 "check_objects", "branch", "fail_fast", "duration_ms", "exit_code",
266 "failures",
267 }
268
269 _ERROR_KEYS = {"error", "message", "duration_ms", "exit_code"}
270
271
272 class TestSchemaCompleteness:
273 def test_all_success_keys_present_clean(self, tmp_path: pathlib.Path) -> None:
274 repo = _init_repo(tmp_path)
275 _make_commit(repo, idx=0)
276 d = json.loads(_invoke(repo, "--json").output)
277 assert _SUCCESS_KEYS <= set(d), f"Missing keys: {_SUCCESS_KEYS - set(d)}"
278
279 def test_all_success_keys_present_with_failures(self, tmp_path: pathlib.Path) -> None:
280 repo = _init_repo(tmp_path)
281 (heads_dir(repo) / "main").write_text("e" * 64)
282 d = json.loads(_invoke(repo, "--json").output)
283 assert _SUCCESS_KEYS <= set(d), f"Missing keys: {_SUCCESS_KEYS - set(d)}"
284
285 def test_all_error_keys_present(self, tmp_path: pathlib.Path) -> None:
286 repo = _init_repo(tmp_path)
287 with mock.patch(
288 "muse.cli.commands.verify.run_verify", side_effect=OSError("fail")
289 ):
290 d = json.loads(_invoke(repo, "--json").output)
291 assert _ERROR_KEYS <= set(d), f"Missing keys: {_ERROR_KEYS - set(d)}"
292
293 def test_failures_list_schema(self, tmp_path: pathlib.Path) -> None:
294 repo = _init_repo(tmp_path)
295 # Missing commit: write a ref pointing to a valid-format but missing commit.
296 cid = long_id("f" * 64)
297 (heads_dir(repo) / "main").write_text(cid)
298 d = json.loads(_invoke(repo, "--json").output)
299 assert len(d["failures"]) >= 1
300 for f in d["failures"]:
301 assert {"kind", "id", "error"} <= set(f)
302
303 def test_failures_kind_is_documented_literal(self, tmp_path: pathlib.Path) -> None:
304 repo = _init_repo(tmp_path)
305 cid = long_id("a" * 64)
306 (heads_dir(repo) / "main").write_text(cid)
307 d = json.loads(_invoke(repo, "--json").output)
308 valid_kinds = {"ref", "commit", "snapshot", "object", "signature", "key_missing"}
309 for f in d["failures"]:
310 assert f["kind"] in valid_kinds, f"Unexpected kind: {f['kind']!r}"
311
312
313 # ---------------------------------------------------------------------------
314 # D — Data integrity
315 # ---------------------------------------------------------------------------
316
317
318 class TestDataIntegrity:
319 def test_exit_code_zero_iff_all_ok_true(self, tmp_path: pathlib.Path) -> None:
320 repo = _init_repo(tmp_path)
321 _make_commit(repo, idx=0)
322 d = json.loads(_invoke(repo, "--json").output)
323 assert (d["exit_code"] == 0) == (d["all_ok"] is True)
324
325 def test_exit_code_one_iff_all_ok_false(self, tmp_path: pathlib.Path) -> None:
326 repo = _init_repo(tmp_path)
327 cid = long_id("b" * 64)
328 (heads_dir(repo) / "main").write_text(cid)
329 d = json.loads(_invoke(repo, "--json").output)
330 assert d["exit_code"] == 1
331 assert d["all_ok"] is False
332
333 def test_counters_non_negative(self, tmp_path: pathlib.Path) -> None:
334 repo = _init_repo(tmp_path)
335 _make_commit(repo, idx=0)
336 d = json.loads(_invoke(repo, "--json").output)
337 for key in ("refs_checked", "commits_checked", "snapshots_checked",
338 "objects_checked", "signatures_checked"):
339 assert d[key] >= 0, f"{key} is negative: {d[key]}"
340
341 def test_check_objects_reflected_true(self, tmp_path: pathlib.Path) -> None:
342 repo = _init_repo(tmp_path)
343 _make_commit(repo, idx=0)
344 d = json.loads(_invoke(repo, "--json").output)
345 assert d["check_objects"] is True
346
347 def test_check_objects_reflected_false(self, tmp_path: pathlib.Path) -> None:
348 repo = _init_repo(tmp_path)
349 _make_commit(repo, idx=0)
350 d = json.loads(_invoke(repo, "--json", "--no-objects").output)
351 assert d["check_objects"] is False
352
353 def test_branch_reflected_in_json(self, tmp_path: pathlib.Path) -> None:
354 repo = _init_repo(tmp_path)
355 _make_commit(repo, idx=0)
356 d = json.loads(_invoke(repo, "--json", "--branch", "main").output)
357 assert d["branch"] == "main"
358
359 def test_branch_none_when_not_specified(self, tmp_path: pathlib.Path) -> None:
360 repo = _init_repo(tmp_path)
361 _make_commit(repo, idx=0)
362 d = json.loads(_invoke(repo, "--json").output)
363 assert d["branch"] is None
364
365 def test_fail_fast_reflected_true(self, tmp_path: pathlib.Path) -> None:
366 repo = _init_repo(tmp_path)
367 _make_commit(repo, idx=0)
368 d = json.loads(_invoke(repo, "--json", "--fail-fast").output)
369 assert d["fail_fast"] is True
370
371 def test_fail_fast_reflected_false(self, tmp_path: pathlib.Path) -> None:
372 repo = _init_repo(tmp_path)
373 _make_commit(repo, idx=0)
374 d = json.loads(_invoke(repo, "--json").output)
375 assert d["fail_fast"] is False
376
377 def test_nothing_checked_false_when_commits_exist(self, tmp_path: pathlib.Path) -> None:
378 repo = _init_repo(tmp_path)
379 _make_commit(repo, idx=0)
380 d = json.loads(_invoke(repo, "--json").output)
381 assert d["nothing_checked"] is False
382
383 def test_nothing_checked_true_empty_repo(self, tmp_path: pathlib.Path) -> None:
384 repo = _init_repo(tmp_path)
385 d = json.loads(_invoke(repo, "--json").output)
386 assert d["nothing_checked"] is True
387
388 def test_duration_ms_is_float(self, tmp_path: pathlib.Path) -> None:
389 repo = _init_repo(tmp_path)
390 _make_commit(repo, idx=0)
391 d = json.loads(_invoke(repo, "--json").output)
392 assert isinstance(d["duration_ms"], float)
393
394 def test_blob_id_unique_per_content(self, tmp_path: pathlib.Path) -> None:
395 """blob_id produces distinct IDs for distinct byte sequences."""
396 ids = {blob_id(b"content-a"), blob_id(b"content-b"), blob_id(b"content-c")}
397 assert len(ids) == 3
398
399 def test_long_id_round_trips(self) -> None:
400 """long_id strips sha256: prefix correctly for comparison."""
401 hex_val = "a" * 64
402 full = long_id(hex_val)
403 assert full == "sha256:" + hex_val
404 assert full == f"sha256:{hex_val}"
405
406 def test_short_id_abbreviates(self) -> None:
407 full = long_id("f" * 64)
408 s = short_id(full)
409 assert s.startswith("sha256:")
410 assert len(s) < len(full)
411
412
413 # ---------------------------------------------------------------------------
414 # IO — OSError handling
415 # ---------------------------------------------------------------------------
416
417
418 class TestIOErrorHandling:
419 def test_exit_code_3_on_io_error_json(self, tmp_path: pathlib.Path) -> None:
420 repo = _init_repo(tmp_path)
421 with mock.patch(
422 "muse.cli.commands.verify.run_verify", side_effect=OSError("io fail")
423 ):
424 r = _invoke(repo, "--json")
425 assert r.exit_code == 3
426 d = json.loads(r.output)
427 assert d["exit_code"] == 3
428
429 def test_exit_code_3_on_io_error_text(self, tmp_path: pathlib.Path) -> None:
430 repo = _init_repo(tmp_path)
431 with mock.patch(
432 "muse.cli.commands.verify.run_verify", side_effect=OSError("io fail")
433 ):
434 r = _invoke(repo)
435 assert r.exit_code == 3
436
437 def test_io_error_json_no_stderr(self, tmp_path: pathlib.Path) -> None:
438 repo = _init_repo(tmp_path)
439 with mock.patch(
440 "muse.cli.commands.verify.run_verify", side_effect=OSError("io fail")
441 ):
442 r = _invoke(repo, "--json")
443 assert r.stderr.strip() == ""
444
445 def test_io_error_text_has_stderr(self, tmp_path: pathlib.Path) -> None:
446 repo = _init_repo(tmp_path)
447 with mock.patch(
448 "muse.cli.commands.verify.run_verify", side_effect=OSError("io fail")
449 ):
450 r = _invoke(repo)
451 assert r.stderr.strip() != ""
452
453
454 # ---------------------------------------------------------------------------
455 # P — Performance
456 # ---------------------------------------------------------------------------
457
458
459 class TestPerformance:
460 def test_duration_ms_positive(self, tmp_path: pathlib.Path) -> None:
461 repo = _init_repo(tmp_path)
462 _make_commit(repo, idx=0)
463 d = json.loads(_invoke(repo, "--json").output)
464 assert d["duration_ms"] > 0
465
466 def test_50_commit_chain_under_5000ms(self, tmp_path: pathlib.Path) -> None:
467 repo = _init_repo(tmp_path)
468 prev: str | None = None
469 for i in range(50):
470 prev = _make_commit(repo, parent_id=prev, idx=i)
471 d = json.loads(_invoke(repo, "--json").output)
472 assert d["duration_ms"] < 5_000, f"Too slow: {d['duration_ms']} ms"
473 assert d["all_ok"] is True
474
475 def test_50_commit_chain_no_objects_faster(self, tmp_path: pathlib.Path) -> None:
476 repo = _init_repo(tmp_path)
477 prev: str | None = None
478 for i in range(50):
479 prev = _make_commit(repo, parent_id=prev, idx=i)
480 full = json.loads(_invoke(repo, "--json").output)["duration_ms"]
481 fast = json.loads(_invoke(repo, "--json", "--no-objects").output)["duration_ms"]
482 # --no-objects should generally be faster; we allow some timing noise
483 # but cap both under 10 s to prevent runaway
484 assert fast < 10_000
485 assert full < 10_000
486
487
488 # ---------------------------------------------------------------------------
489 # Sec — Security
490 # ---------------------------------------------------------------------------
491
492
493 class TestSecurity:
494 def test_no_traceback_on_json_io_error(self, tmp_path: pathlib.Path) -> None:
495 repo = _init_repo(tmp_path)
496 with mock.patch(
497 "muse.cli.commands.verify.run_verify", side_effect=OSError("fail")
498 ):
499 r = _invoke(repo, "--json")
500 assert "Traceback" not in r.output
501 assert "Traceback" not in r.stderr
502
503 def test_no_traceback_on_text_io_error(self, tmp_path: pathlib.Path) -> None:
504 repo = _init_repo(tmp_path)
505 with mock.patch(
506 "muse.cli.commands.verify.run_verify", side_effect=OSError("fail")
507 ):
508 r = _invoke(repo)
509 assert "Traceback" not in r.output
510 assert "Traceback" not in r.stderr
511
512 def test_no_raw_exception_in_stdout(self, tmp_path: pathlib.Path) -> None:
513 repo = _init_repo(tmp_path)
514 with mock.patch(
515 "muse.cli.commands.verify.run_verify", side_effect=OSError("secret path")
516 ):
517 r = _invoke(repo, "--json")
518 # The exception message may appear in the JSON "message" field — that's
519 # intentional. What we check is that no raw Python exception string
520 # (e.g. "OSError:") leaks outside the JSON structure.
521 assert "OSError:" not in r.output
522 assert "OSError:" not in r.stderr
523
524
525 # ---------------------------------------------------------------------------
526 # C — Concurrent readers
527 # ---------------------------------------------------------------------------
528
529
530 class TestConcurrent:
531 def test_10_concurrent_reads_all_valid_json(self, tmp_path: pathlib.Path) -> None:
532 repo = _init_repo(tmp_path)
533 prev: str | None = None
534 for i in range(10):
535 prev = _make_commit(repo, parent_id=prev, idx=i)
536
537 results: list[dict] = []
538 errors: list[Exception] = []
539 lock = threading.Lock()
540
541 def _read() -> None:
542 try:
543 r = _invoke(repo, "--json")
544 d = json.loads(r.output)
545 with lock:
546 results.append(d)
547 except Exception as exc:
548 with lock:
549 errors.append(exc)
550
551 threads = [threading.Thread(target=_read) for _ in range(10)]
552 for t in threads:
553 t.start()
554 for t in threads:
555 t.join()
556
557 assert errors == [], f"Thread errors: {errors}"
558 assert len(results) == 10
559 for d in results:
560 assert d["all_ok"] is True
561 assert d["exit_code"] == 0
562 assert isinstance(d["duration_ms"], float)
File History 1 commit
sha256:51ce277f663e01a43eaffbe77509b1de7ac2d4251b55d23306304bcdeb92c90d feat(pack): delta-encode snapshots in MPackBundle wire format Sonnet 4.6 minor ⚠ 121 days ago