gabriel / muse public
test_release_supercharge.py python
499 lines 20.0 KB
Raw
sha256:b636f72dcba9e190afb980bece906fa5b717fbde014b76ef023df8cb96e01eb9 docs: expand cache plan with all seven testing tiers and do… Sonnet 4.6 132 days ago
1 """Supercharge tests for ``muse release``.
2
3 Coverage tiers
4 --------------
5 - Unit: _short_id helper — sha256:-prefixed input
6 - Integration: duration_ms + exit_code on every JSON path
7 (add, list, read, push --dry-run, delete --dry-run, suggest)
8 - Data: short commit IDs in text output (not bare truncated hex)
9 - Data: list JSON wraps in {total, releases} envelope
10 - Security: no tracebacks; ANSI stripped in text
11 - Performance: suggest on 50 commits under 500ms
12 """
13 from __future__ import annotations
14
15 import datetime
16 import json
17 import pathlib
18 import time
19
20 from muse.core.errors import ExitCode
21 from tests.cli_test_helper import CliRunner, InvokeResult
22 from muse.core._types import content_hash, long_id, short_id as _short_id
23
24 runner = CliRunner()
25
26 _SHA256_SHORT_19 = __import__("re").compile(r"^sha256:[0-9a-f]{12}$")
27
28
29 # ---------------------------------------------------------------------------
30 # Repo + commit helpers (mirrors test_release.py)
31 # ---------------------------------------------------------------------------
32
33
34 def _make_repo(tmp_path: pathlib.Path) -> tuple[pathlib.Path, str]:
35 muse_dir = tmp_path / ".muse"
36 muse_dir.mkdir()
37 repo_id = content_hash({"path": str(tmp_path), "domain": "code", "created_at": "2025-01-01T00:00:00+00:00"})
38 (muse_dir / "repo.json").write_text(
39 json.dumps({"repo_id": repo_id, "domain": "code",
40 "default_branch": "main",
41 "created_at": "2025-01-01T00:00:00+00:00"}),
42 encoding="utf-8",
43 )
44 (muse_dir / "HEAD").write_text("ref: refs/heads/main\n", encoding="utf-8")
45 (muse_dir / "refs" / "heads").mkdir(parents=True)
46 (muse_dir / "snapshots").mkdir()
47 (muse_dir / "commits").mkdir()
48 (muse_dir / "objects").mkdir()
49 return tmp_path, repo_id
50
51
52 def _commit(
53 root: pathlib.Path,
54 repo_id: str,
55 *,
56 message: str = "feat: add something",
57 sem_ver_bump: str = "minor",
58 agent_id: str = "claude-code",
59 model_id: str = "claude-sonnet-4-6",
60 ) -> str:
61 from muse.core.snapshot import compute_commit_id, compute_snapshot_id
62 from muse.core.store import CommitRecord, SnapshotRecord, write_commit, write_snapshot
63 from muse.domain import SemVerBump
64
65 branch = "main"
66 ref_file = root / ".muse" / "refs" / "heads" / branch
67 raw_parent = ref_file.read_text().strip() if ref_file.exists() else ""
68 parent_id: str | None = raw_parent if raw_parent else None
69 snap_id = compute_snapshot_id({})
70 write_snapshot(root, SnapshotRecord(snapshot_id=snap_id, manifest={}))
71 now = datetime.datetime.now(datetime.timezone.utc)
72 parent_ids = [parent_id] if parent_id else []
73 commit_id = compute_commit_id(
74 repo_id=repo_id,
75 parent_ids=parent_ids,
76 snapshot_id=snap_id,
77 message=message,
78 committed_at_iso=now.isoformat(),
79 )
80 bump: SemVerBump = sem_ver_bump # type: ignore[assignment]
81 write_commit(root, CommitRecord(
82 commit_id=commit_id, repo_id=repo_id, created_on_branch=branch,
83 snapshot_id=snap_id, message=message, committed_at=now,
84 parent_commit_id=parent_id, sem_ver_bump=bump,
85 breaking_changes=[], agent_id=agent_id, model_id=model_id,
86 ))
87 ref_file.write_text(commit_id, encoding="utf-8")
88 return commit_id
89
90
91 def _invoke(root: pathlib.Path, *args: str) -> InvokeResult:
92 from muse.cli.app import main as cli
93 return runner.invoke(cli, ["release", *args],
94 env={"MUSE_REPO_ROOT": str(root)})
95
96
97 def _add(root: pathlib.Path, tag: str = "v0.1.0", **kwargs: str) -> InvokeResult:
98 extra = []
99 for k, v in kwargs.items():
100 extra.extend([f"--{k}", v])
101 return _invoke(root, "add", tag, "--json", *extra)
102
103
104 # ---------------------------------------------------------------------------
105 # Unit — _short_id
106 # ---------------------------------------------------------------------------
107
108
109 class TestShortId:
110 """_short_id normalises sha256:-prefixed commit IDs to sha256:<12-hex>."""
111
112 def test_sha256_prefixed_returns_sha256_short(self) -> None:
113
114 cid = long_id("a" * 64)
115 assert _short_id(cid) == long_id("a" * 12)
116
117 def test_result_is_19_chars(self) -> None:
118
119 assert len(_short_id(long_id("b" * 64))) == 19
120
121 def test_bare_hex_also_handled(self) -> None:
122 # bare hex in → first 12 chars out, no prefix added
123 result = _short_id("c" * 64)
124 assert result == "c" * 12
125
126 def test_matches_short_regex(self) -> None:
127
128 assert _SHA256_SHORT_19.match(_short_id(long_id("d" * 64)))
129
130 def test_idempotent_on_short_input(self) -> None:
131 """Already-short sha256:<12-hex> passes through unchanged."""
132
133 short = long_id("e" * 12)
134 assert _short_id(short) == short
135
136
137 # ---------------------------------------------------------------------------
138 # Integration — duration_ms and exit_code on every JSON path
139 # ---------------------------------------------------------------------------
140
141
142 class TestDurationAndExitCode:
143 """Every subcommand's JSON output must carry duration_ms and exit_code."""
144
145 def test_add_json_has_duration_ms(self, tmp_path: pathlib.Path) -> None:
146 root, repo_id = _make_repo(tmp_path)
147 _commit(root, repo_id)
148 data = json.loads(_add(root).output)
149 assert "duration_ms" in data
150
151 def test_add_json_exit_code_zero(self, tmp_path: pathlib.Path) -> None:
152 root, repo_id = _make_repo(tmp_path)
153 _commit(root, repo_id)
154 data = json.loads(_add(root).output)
155 assert data["exit_code"] == 0
156
157 def test_add_duration_ms_is_float(self, tmp_path: pathlib.Path) -> None:
158 root, repo_id = _make_repo(tmp_path)
159 _commit(root, repo_id)
160 assert isinstance(json.loads(_add(root).output)["duration_ms"], float)
161
162 def test_list_json_has_duration_ms(self, tmp_path: pathlib.Path) -> None:
163 root, repo_id = _make_repo(tmp_path)
164 _commit(root, repo_id)
165 _add(root)
166 data = json.loads(_invoke(root, "list", "--json").output)
167 assert "duration_ms" in data
168
169 def test_list_json_exit_code_zero(self, tmp_path: pathlib.Path) -> None:
170 root, repo_id = _make_repo(tmp_path)
171 data = json.loads(_invoke(root, "list", "--json").output)
172 assert data["exit_code"] == 0
173
174 def test_list_empty_json_has_duration_ms(self, tmp_path: pathlib.Path) -> None:
175 """Even with zero releases, metadata fields must be present."""
176 root, repo_id = _make_repo(tmp_path)
177 data = json.loads(_invoke(root, "list", "--json").output)
178 assert "duration_ms" in data
179
180 def test_read_json_has_duration_ms(self, tmp_path: pathlib.Path) -> None:
181 root, repo_id = _make_repo(tmp_path)
182 _commit(root, repo_id)
183 _add(root)
184 data = json.loads(_invoke(root, "read", "v0.1.0", "--json").output)
185 assert "duration_ms" in data
186
187 def test_read_json_exit_code_zero(self, tmp_path: pathlib.Path) -> None:
188 root, repo_id = _make_repo(tmp_path)
189 _commit(root, repo_id)
190 _add(root)
191 data = json.loads(_invoke(root, "read", "v0.1.0", "--json").output)
192 assert data["exit_code"] == 0
193
194 def test_push_dry_run_json_has_duration_ms(self, tmp_path: pathlib.Path) -> None:
195 root, repo_id = _make_repo(tmp_path)
196 _commit(root, repo_id)
197 _add(root)
198 data = json.loads(
199 _invoke(root, "push", "v0.1.0", "--dry-run", "--json").output
200 )
201 assert "duration_ms" in data
202
203 def test_push_dry_run_json_exit_code_zero(self, tmp_path: pathlib.Path) -> None:
204 root, repo_id = _make_repo(tmp_path)
205 _commit(root, repo_id)
206 _add(root)
207 data = json.loads(
208 _invoke(root, "push", "v0.1.0", "--dry-run", "--json").output
209 )
210 assert data["exit_code"] == 0
211
212 def test_delete_dry_run_json_has_duration_ms(self, tmp_path: pathlib.Path) -> None:
213 root, repo_id = _make_repo(tmp_path)
214 _commit(root, repo_id)
215 _add(root)
216 data = json.loads(
217 _invoke(root, "delete", "v0.1.0", "--dry-run", "--json").output
218 )
219 assert "duration_ms" in data
220
221 def test_delete_dry_run_json_exit_code_zero(self, tmp_path: pathlib.Path) -> None:
222 root, repo_id = _make_repo(tmp_path)
223 _commit(root, repo_id)
224 _add(root)
225 data = json.loads(
226 _invoke(root, "delete", "v0.1.0", "--dry-run", "--json").output
227 )
228 assert data["exit_code"] == 0
229
230 def test_suggest_json_has_duration_ms(self, tmp_path: pathlib.Path) -> None:
231 root, repo_id = _make_repo(tmp_path)
232 _commit(root, repo_id, sem_ver_bump="minor")
233 data = json.loads(_invoke(root, "suggest", "--json").output)
234 assert "duration_ms" in data
235
236 def test_suggest_json_exit_code_zero(self, tmp_path: pathlib.Path) -> None:
237 root, repo_id = _make_repo(tmp_path)
238 _commit(root, repo_id, sem_ver_bump="minor")
239 data = json.loads(_invoke(root, "suggest", "--json").output)
240 assert data["exit_code"] == 0
241
242 def test_duration_ms_non_negative(self, tmp_path: pathlib.Path) -> None:
243 root, repo_id = _make_repo(tmp_path)
244 _commit(root, repo_id)
245 _add(root)
246 assert json.loads(_add(root, "v0.2.0").output)["duration_ms"] >= 0.0
247
248 def test_duration_ms_3dp_precision(self, tmp_path: pathlib.Path) -> None:
249 root, repo_id = _make_repo(tmp_path)
250 _commit(root, repo_id)
251 ms = json.loads(_add(root).output)["duration_ms"]
252 assert round(ms, 3) == ms
253
254
255 # ---------------------------------------------------------------------------
256 # Data integrity — text format short IDs
257 # ---------------------------------------------------------------------------
258
259
260 class TestTextFormatShortId:
261 """Text output must show sha256:<12-hex> commit IDs, not bare [:8] slices."""
262
263 def _short_tokens(self, text: str) -> list[str]:
264 return [tok for tok in text.split() if _SHA256_SHORT_19.match(tok)]
265
266 def test_read_text_shows_sha256_short_commit(self, tmp_path: pathlib.Path) -> None:
267 root, repo_id = _make_repo(tmp_path)
268 cid = _commit(root, repo_id)
269 _invoke(root, "add", "v0.1.0")
270 result = _invoke(root, "read", "v0.1.0")
271 assert result.exit_code == 0
272 tokens = self._short_tokens(result.output)
273 assert tokens, f"no sha256:<12-hex> token in read text:\n{result.output}"
274
275 def test_list_text_shows_sha256_short_commit(self, tmp_path: pathlib.Path) -> None:
276 root, repo_id = _make_repo(tmp_path)
277 _commit(root, repo_id)
278 _invoke(root, "add", "v0.1.0")
279 result = _invoke(root, "list")
280 assert result.exit_code == 0
281 tokens = self._short_tokens(result.output)
282 assert tokens, f"no sha256:<12-hex> token in list text:\n{result.output}"
283
284 def test_suggest_text_shows_sha256_short_in_driver(self, tmp_path: pathlib.Path) -> None:
285 root, repo_id = _make_repo(tmp_path)
286 _commit(root, repo_id, sem_ver_bump="minor")
287 result = _invoke(root, "suggest")
288 assert result.exit_code == 0
289 tokens = self._short_tokens(result.output)
290 assert tokens, f"no sha256:<12-hex> token in suggest text:\n{result.output}"
291
292 def test_read_text_commit_not_bare_hex_prefix_only(self, tmp_path: pathlib.Path) -> None:
293 """Regression: commit_id[:8] on sha256:-prefixed ID yields 'sha256:a' — wrong."""
294 root, repo_id = _make_repo(tmp_path)
295 _commit(root, repo_id)
296 _invoke(root, "add", "v0.1.0")
297 result = _invoke(root, "read", "v0.1.0")
298 # "sha256:a" with no further hex after colon would indicate the bug
299 assert "sha256:a\n" not in result.output
300 assert "sha256:b\n" not in result.output
301
302 def test_changelog_entries_show_sha256_short(self, tmp_path: pathlib.Path) -> None:
303 root, repo_id = _make_repo(tmp_path)
304 _commit(root, repo_id, message="feat: first", sem_ver_bump="minor")
305 _commit(root, repo_id, message="feat: second", sem_ver_bump="minor")
306 _invoke(root, "add", "v0.1.0")
307 result = _invoke(root, "read", "v0.1.0")
308 tokens = self._short_tokens(result.output)
309 # changelog has 2 entries, each with a short commit ID
310 assert len(tokens) >= 2, f"expected ≥2 sha256:<12-hex> tokens:\n{result.output}"
311
312
313 # ---------------------------------------------------------------------------
314 # Data integrity — list JSON envelope
315 # ---------------------------------------------------------------------------
316
317
318 class TestListJsonEnvelope:
319 """list --json emits {total, releases, duration_ms, exit_code} — not a bare array."""
320
321 def test_list_json_top_level_is_object(self, tmp_path: pathlib.Path) -> None:
322 root, repo_id = _make_repo(tmp_path)
323 data = json.loads(_invoke(root, "list", "--json").output)
324 assert isinstance(data, dict)
325
326 def test_list_json_has_total_key(self, tmp_path: pathlib.Path) -> None:
327 root, repo_id = _make_repo(tmp_path)
328 data = json.loads(_invoke(root, "list", "--json").output)
329 assert "total" in data
330
331 def test_list_json_has_releases_key(self, tmp_path: pathlib.Path) -> None:
332 root, repo_id = _make_repo(tmp_path)
333 data = json.loads(_invoke(root, "list", "--json").output)
334 assert "releases" in data
335
336 def test_list_json_releases_is_array(self, tmp_path: pathlib.Path) -> None:
337 root, repo_id = _make_repo(tmp_path)
338 data = json.loads(_invoke(root, "list", "--json").output)
339 assert isinstance(data["releases"], list)
340
341 def test_list_json_total_matches_releases_length(self, tmp_path: pathlib.Path) -> None:
342 root, repo_id = _make_repo(tmp_path)
343 _commit(root, repo_id)
344 _add(root, "v0.1.0")
345 _commit(root, repo_id)
346 _add(root, "v0.2.0")
347 data = json.loads(_invoke(root, "list", "--json").output)
348 assert data["total"] == len(data["releases"]) == 2
349
350 def test_list_json_empty_total_is_zero(self, tmp_path: pathlib.Path) -> None:
351 root, repo_id = _make_repo(tmp_path)
352 data = json.loads(_invoke(root, "list", "--json").output)
353 assert data["total"] == 0
354 assert data["releases"] == []
355
356 def test_bare_release_json_flag_uses_envelope(self, tmp_path: pathlib.Path) -> None:
357 """`muse release --json` (no subcommand) also uses the envelope."""
358 root, repo_id = _make_repo(tmp_path)
359 data = json.loads(_invoke(root, "--json").output)
360 assert "releases" in data
361 assert "total" in data
362
363
364 # ---------------------------------------------------------------------------
365 # Security
366 # ---------------------------------------------------------------------------
367
368
369 class TestSecuritySupercharge:
370 def test_no_traceback_on_bad_semver(self, tmp_path: pathlib.Path) -> None:
371 root, _ = _make_repo(tmp_path)
372 result = _invoke(root, "add", "not-semver", "--json")
373 assert result.exit_code == ExitCode.USER_ERROR
374 assert "Traceback" not in result.output
375
376 def test_no_traceback_on_missing_read(self, tmp_path: pathlib.Path) -> None:
377 root, _ = _make_repo(tmp_path)
378 result = _invoke(root, "read", "v9.9.9", "--json")
379 assert result.exit_code == ExitCode.NOT_FOUND
380 assert "Traceback" not in result.output
381
382 def test_no_traceback_on_suggest_no_commits(self, tmp_path: pathlib.Path) -> None:
383 root, _ = _make_repo(tmp_path)
384 result = _invoke(root, "suggest", "--json")
385 # Either no commits (exit 1) or 0 unreleased — must not traceback
386 assert "Traceback" not in result.output
387
388 def test_add_json_error_on_duplicate(self, tmp_path: pathlib.Path) -> None:
389 root, repo_id = _make_repo(tmp_path)
390 _commit(root, repo_id)
391 _add(root)
392 result = _invoke(root, "add", "v0.1.0", "--json")
393 assert result.exit_code == ExitCode.USER_ERROR
394 # JSON error is on stdout (first line); ❌ message goes to stderr
395 first_line = result.output.splitlines()[0]
396 data = json.loads(first_line)
397 assert data["error"] == "already_exists"
398
399
400 # ---------------------------------------------------------------------------
401 # Performance
402 # ---------------------------------------------------------------------------
403
404
405 class TestPerformanceSupercharge:
406 def test_list_50_releases_under_500ms(self, tmp_path: pathlib.Path) -> None:
407 root, repo_id = _make_repo(tmp_path)
408 for i in range(50):
409 _commit(root, repo_id, message=f"feat: thing {i}")
410 _invoke(root, "add", f"v0.{i}.0")
411 t0 = time.monotonic()
412 result = _invoke(root, "list", "--json")
413 duration_ms = (time.monotonic() - t0) * 1000
414 assert result.exit_code == 0
415 assert duration_ms < 500
416
417 def test_suggest_50_commits_under_500ms(self, tmp_path: pathlib.Path) -> None:
418 root, repo_id = _make_repo(tmp_path)
419 for i in range(50):
420 _commit(root, repo_id, sem_ver_bump="patch")
421 t0 = time.monotonic()
422 result = _invoke(root, "suggest", "--json")
423 duration_ms = (time.monotonic() - t0) * 1000
424 assert result.exit_code == 0
425 assert duration_ms < 500
426
427 def test_duration_ms_plausible(self, tmp_path: pathlib.Path) -> None:
428 root, repo_id = _make_repo(tmp_path)
429 _commit(root, repo_id)
430 ms = json.loads(_add(root).output)["duration_ms"]
431 assert 0.0 <= ms < 500
432
433
434 # ---------------------------------------------------------------------------
435 # Content-addressed release_id
436 # ---------------------------------------------------------------------------
437
438
439 class TestReleaseIdContentAddressed:
440 """release_id must be sha256: of genesis content, not a UUID."""
441
442 def test_release_id_is_sha256_prefixed(self, tmp_path: pathlib.Path) -> None:
443 root, repo_id = _make_repo(tmp_path)
444 _commit(root, repo_id)
445 data = json.loads(_add(root, "v1.0.0").output)
446 assert data["release_id"].startswith("sha256:"), f"Expected sha256: prefix, got {data['release_id']!r}"
447 assert len(data["release_id"]) == 71
448
449 def test_release_id_not_uuid(self, tmp_path: pathlib.Path) -> None:
450 import re
451 root, repo_id = _make_repo(tmp_path)
452 _commit(root, repo_id)
453 data = json.loads(_add(root, "v1.0.0").output)
454 uuid_re = re.compile(r"^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$")
455 assert not uuid_re.match(data["release_id"])
456
457 def test_release_id_is_deterministic(self, tmp_path: pathlib.Path) -> None:
458 """Same repo + tag + commit → same release_id."""
459 from muse.core.store import compute_release_id
460 release_id = compute_release_id(repo_id="test-repo", tag="v1.0.0", commit_id=long_id("a" * 64))
461 assert release_id == compute_release_id(repo_id="test-repo", tag="v1.0.0", commit_id=long_id("a" * 64))
462
463 def test_release_id_differs_by_tag(self, tmp_path: pathlib.Path) -> None:
464 from muse.core.store import compute_release_id
465 cid = long_id("a" * 64)
466 assert compute_release_id("repo", "v1.0.0", cid) != compute_release_id("repo", "v2.0.0", cid)
467
468 def test_release_id_differs_by_commit(self, tmp_path: pathlib.Path) -> None:
469 from muse.core.store import compute_release_id
470 assert compute_release_id("repo", "v1.0.0", long_id("a" * 64)) != compute_release_id("repo", "v1.0.0", long_id("b" * 64))
471
472
473 class TestRegisterFlags:
474 def test_default_json_out_is_false(self):
475 import argparse
476 from muse.cli.commands.release import register
477 p = argparse.ArgumentParser()
478 subs = p.add_subparsers()
479 register(subs)
480 args = p.parse_args(["release"])
481 assert args.json_out is False
482
483 def test_json_flag_sets_json_out(self):
484 import argparse
485 from muse.cli.commands.release import register
486 p = argparse.ArgumentParser()
487 subs = p.add_subparsers()
488 register(subs)
489 args = p.parse_args(["release", "--json"])
490 assert args.json_out is True
491
492 def test_j_shorthand_sets_json_out(self):
493 import argparse
494 from muse.cli.commands.release import register
495 p = argparse.ArgumentParser()
496 subs = p.add_subparsers()
497 register(subs)
498 args = p.parse_args(["release", "-j"])
499 assert args.json_out is True
File History 2 commits
sha256:b636f72dcba9e190afb980bece906fa5b717fbde014b76ef023df8cb96e01eb9 docs: expand cache plan with all seven testing tiers and do… Sonnet 4.6 132 days ago
sha256:7f9e2ef5286aedad9c1e6011b4c46ca27f39dbdad6e3409357e36b26e46b3b7c docs: docstring sprint for-each-ref→hotspots — idiomatic ru… Sonnet 4.6 patch 138 days ago