gabriel / muse public
test_verify_supercharge.py python
560 lines 21.4 KB
Raw
sha256:88ac91129873e6a496e9189515aa690eb893ae25d69c8f72af141a2be5068eb3 docs: docstring sprint contract→find-symbol — idiomatic run… Sonnet 4.6 patch 142 days ago
1 """Supercharge tests for ``muse verify``.
2
3 Every JSON success response must carry ``duration_ms`` (float, ms) and
4 ``exit_code`` (int). Every JSON error response routes to *stdout*, not stderr,
5 so agent pipelines never receive mixed-mode output.
6
7 Coverage tiers
8 --------------
9 U — duration_ms / exit_code on all code paths (clean, failures, no-objects,
10 branch-scoped, fail-fast)
11 E — JSON error routing: _emit_error() writes to stdout in JSON mode, stderr in
12 text mode; no traceback on any error path
13 S — Schema completeness: all _VerifyJson fields present in every success
14 response; all _VerifyErrorJson fields present in every error response
15 D — Data integrity: exit_code=0 ↔ all_ok=True; exit_code=1 ↔ all_ok=False;
16 duration_ms > 0; duration_ms is float; counters are non-negative
17 IO — OSError during run_verify → exit_code=3 in JSON, stderr in text mode
18 P — Performance: duration_ms < 5 000 ms for a 50-commit chain; monotone
19 (two runs on same repo differ only by noise)
20 Sec — No traceback on any error; no raw Python exception in stdout
21 C — Concurrent readers produce valid JSON (10 threads, same repo)
22 """
23
24 from __future__ import annotations
25
26 import datetime
27 import hashlib
28 import json
29 import pathlib
30 import threading
31 import unittest.mock as mock
32
33 import pytest
34 from tests.cli_test_helper import CliRunner, InvokeResult
35
36 from muse.core._types import blob_id, long_id, short_id
37 from muse.core.object_store import object_path, write_object
38 from muse.core.snapshot import compute_commit_id, compute_snapshot_id
39 from muse.core.store import CommitRecord, SnapshotRecord, write_commit, write_snapshot
40 from muse.core.verify import run_verify
41
42 runner = CliRunner()
43 cli = None # argparse migration — CliRunner ignores this arg
44
45 _REPO_ID = "verify-supercharge-test"
46
47
48 # ---------------------------------------------------------------------------
49 # Helpers
50 # ---------------------------------------------------------------------------
51
52
53 def _sha(data: bytes) -> str:
54 return long_id(hashlib.sha256(data).hexdigest())
55
56
57 def _init_repo(path: pathlib.Path) -> pathlib.Path:
58 muse = path / ".muse"
59 for d in ("commits", "snapshots", "objects", "refs/heads"):
60 (muse / d).mkdir(parents=True, exist_ok=True)
61 (muse / "HEAD").write_text("ref: refs/heads/main", encoding="utf-8")
62 (muse / "repo.json").write_text(
63 json.dumps({"repo_id": _REPO_ID, "domain": "midi"}), encoding="utf-8"
64 )
65 return path
66
67
68 def _env(repo: pathlib.Path) -> dict[str, str]:
69 return {"MUSE_REPO_ROOT": str(repo)}
70
71
72 def _make_commit(
73 root: pathlib.Path,
74 parent_id: str | None = None,
75 content: bytes = b"data",
76 branch: str = "main",
77 idx: int = 0,
78 ) -> str:
79 raw = content + str(idx).encode()
80 obj_id = _sha(raw)
81 write_object(root, obj_id, raw)
82 manifest = {f"file_{idx}.txt": obj_id}
83 snap_id = compute_snapshot_id(manifest)
84 write_snapshot(root, SnapshotRecord(snapshot_id=snap_id, manifest=manifest))
85 committed_at = (
86 datetime.datetime(2026, 1, 1, tzinfo=datetime.timezone.utc)
87 + datetime.timedelta(hours=idx)
88 )
89 parent_ids = [parent_id] if parent_id else []
90 commit_id = compute_commit_id(
91 parent_ids, snap_id, f"commit {idx}", committed_at.isoformat()
92 )
93 write_commit(
94 root,
95 CommitRecord(
96 commit_id=commit_id,
97 repo_id=_REPO_ID,
98 branch=branch,
99 snapshot_id=snap_id,
100 message=f"commit {idx}",
101 committed_at=committed_at,
102 parent_commit_id=parent_id,
103 ),
104 )
105 (root / ".muse" / "refs" / "heads" / branch).write_text(commit_id, encoding="utf-8")
106 return commit_id
107
108
109 def _invoke(repo: pathlib.Path, *args: str) -> InvokeResult:
110 from muse.cli.app import main as cli_main
111 return runner.invoke(cli_main, ["verify", *args], env=_env(repo))
112
113
114 # ---------------------------------------------------------------------------
115 # U — duration_ms and exit_code on all code paths
116 # ---------------------------------------------------------------------------
117
118
119 class TestElapsedAndExitCode:
120 def test_clean_repo_exit_code_zero(self, tmp_path: pathlib.Path) -> None:
121 repo = _init_repo(tmp_path)
122 _make_commit(repo, idx=0)
123 r = _invoke(repo, "--json")
124 assert r.exit_code == 0
125 d = json.loads(r.output)
126 assert d["exit_code"] == 0
127
128 def test_clean_repo_duration_ms_present(self, tmp_path: pathlib.Path) -> None:
129 repo = _init_repo(tmp_path)
130 _make_commit(repo, idx=0)
131 r = _invoke(repo, "--json")
132 d = json.loads(r.output)
133 assert "duration_ms" in d
134 assert isinstance(d["duration_ms"], float)
135 assert d["duration_ms"] > 0
136
137 def test_failures_exit_code_one(self, tmp_path: pathlib.Path) -> None:
138 repo = _init_repo(tmp_path)
139 # Write a ref pointing at a non-existent commit (bare hex — invalid ref format)
140 (repo / ".muse" / "refs" / "heads" / "main").write_text("b" * 64)
141 r = _invoke(repo, "--json")
142 assert r.exit_code == 1
143 d = json.loads(r.output)
144 assert d["exit_code"] == 1
145 assert d["all_ok"] is False
146
147 def test_failures_duration_ms_present(self, tmp_path: pathlib.Path) -> None:
148 repo = _init_repo(tmp_path)
149 (repo / ".muse" / "refs" / "heads" / "main").write_text("c" * 64)
150 r = _invoke(repo, "--json")
151 d = json.loads(r.output)
152 assert isinstance(d["duration_ms"], float)
153 assert d["duration_ms"] > 0
154
155 def test_no_objects_exit_code_zero(self, tmp_path: pathlib.Path) -> None:
156 repo = _init_repo(tmp_path)
157 _make_commit(repo, idx=1)
158 r = _invoke(repo, "--json", "--no-objects")
159 assert r.exit_code == 0
160 d = json.loads(r.output)
161 assert d["exit_code"] == 0
162 assert d["duration_ms"] > 0
163
164 def test_branch_scoped_exit_code_zero(self, tmp_path: pathlib.Path) -> None:
165 repo = _init_repo(tmp_path)
166 _make_commit(repo, idx=2)
167 r = _invoke(repo, "--json", "--branch", "main")
168 assert r.exit_code == 0
169 d = json.loads(r.output)
170 assert d["exit_code"] == 0
171 assert d["duration_ms"] > 0
172
173 def test_fail_fast_exit_code_one(self, tmp_path: pathlib.Path) -> None:
174 repo = _init_repo(tmp_path)
175 (repo / ".muse" / "refs" / "heads" / "main").write_text("d" * 64)
176 r = _invoke(repo, "--json", "--fail-fast")
177 assert r.exit_code == 1
178 d = json.loads(r.output)
179 assert d["exit_code"] == 1
180 assert d["duration_ms"] > 0
181
182
183 # ---------------------------------------------------------------------------
184 # E — Error routing: JSON mode → stdout; text mode → stderr
185 # ---------------------------------------------------------------------------
186
187
188 class TestErrorRouting:
189 def test_io_error_json_to_stdout(self, tmp_path: pathlib.Path) -> None:
190 """OSError during run_verify → JSON error on stdout in JSON mode."""
191 repo = _init_repo(tmp_path)
192 _make_commit(repo, idx=0)
193 with mock.patch(
194 "muse.cli.commands.verify.run_verify",
195 side_effect=OSError("disk full"),
196 ):
197 r = _invoke(repo, "--json")
198 assert r.stderr.strip() == "", f"Expected empty stderr, got: {r.stderr!r}"
199 d = json.loads(r.output)
200 assert d["error"] == "io_error"
201 assert "disk full" in d["message"]
202 assert d["exit_code"] == 3
203 assert isinstance(d["duration_ms"], float)
204
205 def test_io_error_text_to_stderr(self, tmp_path: pathlib.Path) -> None:
206 """OSError during run_verify → error on stderr in text mode."""
207 repo = _init_repo(tmp_path)
208 _make_commit(repo, idx=0)
209 with mock.patch(
210 "muse.cli.commands.verify.run_verify",
211 side_effect=OSError("disk full"),
212 ):
213 r = _invoke(repo)
214 assert r.exit_code == 3
215 assert r.stderr.strip() != ""
216 assert "disk full" in r.stderr
217
218 def test_io_error_quiet_no_output(self, tmp_path: pathlib.Path) -> None:
219 """OSError in quiet mode → no output at all, exit 3."""
220 repo = _init_repo(tmp_path)
221 _make_commit(repo, idx=0)
222 with mock.patch(
223 "muse.cli.commands.verify.run_verify",
224 side_effect=OSError("disk full"),
225 ):
226 r = _invoke(repo, "--quiet")
227 assert r.exit_code == 3
228 assert r.output.strip() == ""
229 assert r.stderr.strip() == ""
230
231 def test_io_error_json_no_traceback(self, tmp_path: pathlib.Path) -> None:
232 """No Python traceback lands on stdout in JSON mode."""
233 repo = _init_repo(tmp_path)
234 _make_commit(repo, idx=0)
235 with mock.patch(
236 "muse.cli.commands.verify.run_verify",
237 side_effect=OSError("broken pipe"),
238 ):
239 r = _invoke(repo, "--json")
240 assert "Traceback" not in r.output
241 assert "Traceback" not in r.stderr
242
243 def test_io_error_json_schema(self, tmp_path: pathlib.Path) -> None:
244 """JSON error payload has exactly the documented keys."""
245 repo = _init_repo(tmp_path)
246 _make_commit(repo, idx=0)
247 with mock.patch(
248 "muse.cli.commands.verify.run_verify",
249 side_effect=OSError("nfs timeout"),
250 ):
251 r = _invoke(repo, "--json")
252 d = json.loads(r.output)
253 assert set(d) >= {"error", "message", "duration_ms", "exit_code"}
254
255
256 # ---------------------------------------------------------------------------
257 # S — Schema completeness
258 # ---------------------------------------------------------------------------
259
260 _SUCCESS_KEYS = {
261 "repo_id", "refs_checked", "commits_checked", "snapshots_checked",
262 "objects_checked", "signatures_checked", "all_ok", "nothing_checked",
263 "check_objects", "branch", "fail_fast", "duration_ms", "exit_code",
264 "failures",
265 }
266
267 _ERROR_KEYS = {"error", "message", "duration_ms", "exit_code"}
268
269
270 class TestSchemaCompleteness:
271 def test_all_success_keys_present_clean(self, tmp_path: pathlib.Path) -> None:
272 repo = _init_repo(tmp_path)
273 _make_commit(repo, idx=0)
274 d = json.loads(_invoke(repo, "--json").output)
275 assert _SUCCESS_KEYS <= set(d), f"Missing keys: {_SUCCESS_KEYS - set(d)}"
276
277 def test_all_success_keys_present_with_failures(self, tmp_path: pathlib.Path) -> None:
278 repo = _init_repo(tmp_path)
279 (repo / ".muse" / "refs" / "heads" / "main").write_text("e" * 64)
280 d = json.loads(_invoke(repo, "--json").output)
281 assert _SUCCESS_KEYS <= set(d), f"Missing keys: {_SUCCESS_KEYS - set(d)}"
282
283 def test_all_error_keys_present(self, tmp_path: pathlib.Path) -> None:
284 repo = _init_repo(tmp_path)
285 with mock.patch(
286 "muse.cli.commands.verify.run_verify", side_effect=OSError("fail")
287 ):
288 d = json.loads(_invoke(repo, "--json").output)
289 assert _ERROR_KEYS <= set(d), f"Missing keys: {_ERROR_KEYS - set(d)}"
290
291 def test_failures_list_schema(self, tmp_path: pathlib.Path) -> None:
292 repo = _init_repo(tmp_path)
293 # Missing commit: write a ref pointing to a valid-format but missing commit.
294 cid = long_id("f" * 64)
295 (repo / ".muse" / "refs" / "heads" / "main").write_text(cid)
296 d = json.loads(_invoke(repo, "--json").output)
297 assert len(d["failures"]) >= 1
298 for f in d["failures"]:
299 assert {"kind", "id", "error"} <= set(f)
300
301 def test_failures_kind_is_documented_literal(self, tmp_path: pathlib.Path) -> None:
302 repo = _init_repo(tmp_path)
303 cid = long_id("a" * 64)
304 (repo / ".muse" / "refs" / "heads" / "main").write_text(cid)
305 d = json.loads(_invoke(repo, "--json").output)
306 valid_kinds = {"ref", "commit", "snapshot", "object", "signature", "key_missing"}
307 for f in d["failures"]:
308 assert f["kind"] in valid_kinds, f"Unexpected kind: {f['kind']!r}"
309
310
311 # ---------------------------------------------------------------------------
312 # D — Data integrity
313 # ---------------------------------------------------------------------------
314
315
316 class TestDataIntegrity:
317 def test_exit_code_zero_iff_all_ok_true(self, tmp_path: pathlib.Path) -> None:
318 repo = _init_repo(tmp_path)
319 _make_commit(repo, idx=0)
320 d = json.loads(_invoke(repo, "--json").output)
321 assert (d["exit_code"] == 0) == (d["all_ok"] is True)
322
323 def test_exit_code_one_iff_all_ok_false(self, tmp_path: pathlib.Path) -> None:
324 repo = _init_repo(tmp_path)
325 cid = long_id("b" * 64)
326 (repo / ".muse" / "refs" / "heads" / "main").write_text(cid)
327 d = json.loads(_invoke(repo, "--json").output)
328 assert d["exit_code"] == 1
329 assert d["all_ok"] is False
330
331 def test_counters_non_negative(self, tmp_path: pathlib.Path) -> None:
332 repo = _init_repo(tmp_path)
333 _make_commit(repo, idx=0)
334 d = json.loads(_invoke(repo, "--json").output)
335 for key in ("refs_checked", "commits_checked", "snapshots_checked",
336 "objects_checked", "signatures_checked"):
337 assert d[key] >= 0, f"{key} is negative: {d[key]}"
338
339 def test_check_objects_reflected_true(self, tmp_path: pathlib.Path) -> None:
340 repo = _init_repo(tmp_path)
341 _make_commit(repo, idx=0)
342 d = json.loads(_invoke(repo, "--json").output)
343 assert d["check_objects"] is True
344
345 def test_check_objects_reflected_false(self, tmp_path: pathlib.Path) -> None:
346 repo = _init_repo(tmp_path)
347 _make_commit(repo, idx=0)
348 d = json.loads(_invoke(repo, "--json", "--no-objects").output)
349 assert d["check_objects"] is False
350
351 def test_branch_reflected_in_json(self, tmp_path: pathlib.Path) -> None:
352 repo = _init_repo(tmp_path)
353 _make_commit(repo, idx=0)
354 d = json.loads(_invoke(repo, "--json", "--branch", "main").output)
355 assert d["branch"] == "main"
356
357 def test_branch_none_when_not_specified(self, tmp_path: pathlib.Path) -> None:
358 repo = _init_repo(tmp_path)
359 _make_commit(repo, idx=0)
360 d = json.loads(_invoke(repo, "--json").output)
361 assert d["branch"] is None
362
363 def test_fail_fast_reflected_true(self, tmp_path: pathlib.Path) -> None:
364 repo = _init_repo(tmp_path)
365 _make_commit(repo, idx=0)
366 d = json.loads(_invoke(repo, "--json", "--fail-fast").output)
367 assert d["fail_fast"] is True
368
369 def test_fail_fast_reflected_false(self, tmp_path: pathlib.Path) -> None:
370 repo = _init_repo(tmp_path)
371 _make_commit(repo, idx=0)
372 d = json.loads(_invoke(repo, "--json").output)
373 assert d["fail_fast"] is False
374
375 def test_nothing_checked_false_when_commits_exist(self, tmp_path: pathlib.Path) -> None:
376 repo = _init_repo(tmp_path)
377 _make_commit(repo, idx=0)
378 d = json.loads(_invoke(repo, "--json").output)
379 assert d["nothing_checked"] is False
380
381 def test_nothing_checked_true_empty_repo(self, tmp_path: pathlib.Path) -> None:
382 repo = _init_repo(tmp_path)
383 d = json.loads(_invoke(repo, "--json").output)
384 assert d["nothing_checked"] is True
385
386 def test_duration_ms_is_float(self, tmp_path: pathlib.Path) -> None:
387 repo = _init_repo(tmp_path)
388 _make_commit(repo, idx=0)
389 d = json.loads(_invoke(repo, "--json").output)
390 assert isinstance(d["duration_ms"], float)
391
392 def test_blob_id_unique_per_content(self, tmp_path: pathlib.Path) -> None:
393 """blob_id produces distinct IDs for distinct byte sequences."""
394 ids = {blob_id(b"content-a"), blob_id(b"content-b"), blob_id(b"content-c")}
395 assert len(ids) == 3
396
397 def test_long_id_round_trips(self) -> None:
398 """long_id strips sha256: prefix correctly for comparison."""
399 hex_val = "a" * 64
400 full = long_id(hex_val)
401 assert full == "sha256:" + hex_val
402 assert full == f"sha256:{hex_val}"
403
404 def test_short_id_abbreviates(self) -> None:
405 full = long_id("f" * 64)
406 s = short_id(full)
407 assert s.startswith("sha256:")
408 assert len(s) < len(full)
409
410
411 # ---------------------------------------------------------------------------
412 # IO — OSError handling
413 # ---------------------------------------------------------------------------
414
415
416 class TestIOErrorHandling:
417 def test_exit_code_3_on_io_error_json(self, tmp_path: pathlib.Path) -> None:
418 repo = _init_repo(tmp_path)
419 with mock.patch(
420 "muse.cli.commands.verify.run_verify", side_effect=OSError("io fail")
421 ):
422 r = _invoke(repo, "--json")
423 assert r.exit_code == 3
424 d = json.loads(r.output)
425 assert d["exit_code"] == 3
426
427 def test_exit_code_3_on_io_error_text(self, tmp_path: pathlib.Path) -> None:
428 repo = _init_repo(tmp_path)
429 with mock.patch(
430 "muse.cli.commands.verify.run_verify", side_effect=OSError("io fail")
431 ):
432 r = _invoke(repo)
433 assert r.exit_code == 3
434
435 def test_io_error_json_no_stderr(self, tmp_path: pathlib.Path) -> None:
436 repo = _init_repo(tmp_path)
437 with mock.patch(
438 "muse.cli.commands.verify.run_verify", side_effect=OSError("io fail")
439 ):
440 r = _invoke(repo, "--json")
441 assert r.stderr.strip() == ""
442
443 def test_io_error_text_has_stderr(self, tmp_path: pathlib.Path) -> None:
444 repo = _init_repo(tmp_path)
445 with mock.patch(
446 "muse.cli.commands.verify.run_verify", side_effect=OSError("io fail")
447 ):
448 r = _invoke(repo)
449 assert r.stderr.strip() != ""
450
451
452 # ---------------------------------------------------------------------------
453 # P — Performance
454 # ---------------------------------------------------------------------------
455
456
457 class TestPerformance:
458 def test_duration_ms_positive(self, tmp_path: pathlib.Path) -> None:
459 repo = _init_repo(tmp_path)
460 _make_commit(repo, idx=0)
461 d = json.loads(_invoke(repo, "--json").output)
462 assert d["duration_ms"] > 0
463
464 def test_50_commit_chain_under_5000ms(self, tmp_path: pathlib.Path) -> None:
465 repo = _init_repo(tmp_path)
466 prev: str | None = None
467 for i in range(50):
468 prev = _make_commit(repo, parent_id=prev, idx=i)
469 d = json.loads(_invoke(repo, "--json").output)
470 assert d["duration_ms"] < 5_000, f"Too slow: {d['duration_ms']} ms"
471 assert d["all_ok"] is True
472
473 def test_50_commit_chain_no_objects_faster(self, tmp_path: pathlib.Path) -> None:
474 repo = _init_repo(tmp_path)
475 prev: str | None = None
476 for i in range(50):
477 prev = _make_commit(repo, parent_id=prev, idx=i)
478 full = json.loads(_invoke(repo, "--json").output)["duration_ms"]
479 fast = json.loads(_invoke(repo, "--json", "--no-objects").output)["duration_ms"]
480 # --no-objects should generally be faster; we allow some timing noise
481 # but cap both under 10 s to prevent runaway
482 assert fast < 10_000
483 assert full < 10_000
484
485
486 # ---------------------------------------------------------------------------
487 # Sec — Security
488 # ---------------------------------------------------------------------------
489
490
491 class TestSecurity:
492 def test_no_traceback_on_json_io_error(self, tmp_path: pathlib.Path) -> None:
493 repo = _init_repo(tmp_path)
494 with mock.patch(
495 "muse.cli.commands.verify.run_verify", side_effect=OSError("fail")
496 ):
497 r = _invoke(repo, "--json")
498 assert "Traceback" not in r.output
499 assert "Traceback" not in r.stderr
500
501 def test_no_traceback_on_text_io_error(self, tmp_path: pathlib.Path) -> None:
502 repo = _init_repo(tmp_path)
503 with mock.patch(
504 "muse.cli.commands.verify.run_verify", side_effect=OSError("fail")
505 ):
506 r = _invoke(repo)
507 assert "Traceback" not in r.output
508 assert "Traceback" not in r.stderr
509
510 def test_no_raw_exception_in_stdout(self, tmp_path: pathlib.Path) -> None:
511 repo = _init_repo(tmp_path)
512 with mock.patch(
513 "muse.cli.commands.verify.run_verify", side_effect=OSError("secret path")
514 ):
515 r = _invoke(repo, "--json")
516 # The exception message may appear in the JSON "message" field — that's
517 # intentional. What we check is that no raw Python exception string
518 # (e.g. "OSError:") leaks outside the JSON structure.
519 assert "OSError:" not in r.output
520 assert "OSError:" not in r.stderr
521
522
523 # ---------------------------------------------------------------------------
524 # C — Concurrent readers
525 # ---------------------------------------------------------------------------
526
527
528 class TestConcurrent:
529 def test_10_concurrent_reads_all_valid_json(self, tmp_path: pathlib.Path) -> None:
530 repo = _init_repo(tmp_path)
531 prev: str | None = None
532 for i in range(10):
533 prev = _make_commit(repo, parent_id=prev, idx=i)
534
535 results: list[dict] = []
536 errors: list[Exception] = []
537 lock = threading.Lock()
538
539 def _read() -> None:
540 try:
541 r = _invoke(repo, "--json")
542 d = json.loads(r.output)
543 with lock:
544 results.append(d)
545 except Exception as exc:
546 with lock:
547 errors.append(exc)
548
549 threads = [threading.Thread(target=_read) for _ in range(10)]
550 for t in threads:
551 t.start()
552 for t in threads:
553 t.join()
554
555 assert errors == [], f"Thread errors: {errors}"
556 assert len(results) == 10
557 for d in results:
558 assert d["all_ok"] is True
559 assert d["exit_code"] == 0
560 assert isinstance(d["duration_ms"], float)
File History 1 commit
sha256:88ac91129873e6a496e9189515aa690eb893ae25d69c8f72af141a2be5068eb3 docs: docstring sprint contract→find-symbol — idiomatic run… Sonnet 4.6 patch 142 days ago