gabriel / muse public
test_languages_supercharge.py python
476 lines 19.7 KB
Raw
sha256:b636f72dcba9e190afb980bece906fa5b717fbde014b76ef023df8cb96e01eb9 docs: expand cache plan with all seven testing tiers and do… Sonnet 4.6 131 days ago
1 """Supercharge tests for ``muse code languages`` — agent-usability gaps.
2
3 The existing test_code_language_config.py covers the language detection
4 machinery (AST parser, CodeConfig, adapter routing) but has zero CLI tests.
5
6 This file covers the CLI command end-to-end:
7
8 Coverage matrix
9 ---------------
10 - --json / -j: -j alias (was missing — caused argparse error)
11 - exit_code: both JSON paths (snapshot + diff) include exit_code = 0
12 - duration_ms: both JSON paths include non-negative float duration_ms
13 - TypedDicts: _SnapshotOutputJson and _DiffOutputJson carry all envelope fields
14 - Docstrings: run() docstring mentions exit_code and duration_ms
15 - Snapshot JSON: shape, required keys, language entry structure
16 - Diff JSON: shape, required keys, diff entry structure
17 - --sort: name / files / symbols all accepted; output ordering correct
18 - --include-imports: import pseudo-symbols added to counts
19 - --commit: historical snapshot accepted; bad ref exits cleanly
20 - --diff: diff mode activates; bad ref exits cleanly
21 - ANSI: no escape codes in JSON output
22 - Performance: duration_ms < 5000 ms on a small repo
23 """
24
25 from __future__ import annotations
26 from collections.abc import Mapping
27
28 import json
29 import pathlib
30 import textwrap
31
32 import pytest
33
34 from tests.cli_test_helper import CliRunner
35
36 runner = CliRunner()
37
38
39 # ---------------------------------------------------------------------------
40 # Helpers
41 # ---------------------------------------------------------------------------
42
43
44 def _env(root: pathlib.Path) -> Mapping[str, str]:
45 return {"MUSE_REPO_ROOT": str(root)}
46
47
48 def _run(root: pathlib.Path, *args: str):
49 return runner.invoke(None, list(args), env=_env(root))
50
51
52 # ---------------------------------------------------------------------------
53 # Fixture — two-commit repo with Python + Markdown files
54 # ---------------------------------------------------------------------------
55
56
57 @pytest.fixture()
58 def lang_repo(
59 tmp_path: pathlib.Path, monkeypatch: pytest.MonkeyPatch
60 ) -> pathlib.Path:
61 """Repo with Python and Markdown files across two commits.
62
63 Commit 1: billing.py (Invoice class + validate_amount fn)
64 Commit 2: auth.py (AuthError class + verify_token fn) + README.md
65 """
66 monkeypatch.chdir(tmp_path)
67 r = _run(tmp_path, "init", "--domain", "code")
68 assert r.exit_code == 0, r.output
69
70 (tmp_path / "billing.py").write_text(textwrap.dedent("""\
71 import decimal
72
73 class Invoice:
74 def compute_total(self, items):
75 return sum(items)
76
77 def validate_amount(amount):
78 return amount > 0
79 """))
80 r = _run(tmp_path, "code", "add", ".")
81 assert r.exit_code == 0, r.output
82 r = _run(tmp_path, "commit", "-m", "first commit")
83 assert r.exit_code == 0, r.output
84
85 (tmp_path / "auth.py").write_text(textwrap.dedent("""\
86 class AuthError(Exception):
87 pass
88
89 def verify_token(token):
90 return bool(token)
91 """))
92 (tmp_path / "README.md").write_text("# Test Repo\n\nA test repo.\n")
93 r = _run(tmp_path, "code", "add", ".")
94 assert r.exit_code == 0, r.output
95 r = _run(tmp_path, "commit", "-m", "second commit")
96 assert r.exit_code == 0, r.output
97
98 return tmp_path
99
100
101 # ---------------------------------------------------------------------------
102 # TestJsonAlias — -j alias must work
103 # ---------------------------------------------------------------------------
104
105
106 class TestJsonAlias:
107 """-j must be accepted and produce identical output to --json."""
108
109 def test_j_alias_exits_zero(self, lang_repo: pathlib.Path) -> None:
110 r = _run(lang_repo, "code", "languages", "-j")
111 assert r.exit_code == 0, r.output
112
113 def test_j_alias_valid_json(self, lang_repo: pathlib.Path) -> None:
114 r = _run(lang_repo, "code", "languages", "-j")
115 json.loads(r.output)
116
117 def test_j_alias_same_keys_as_json_flag(self, lang_repo: pathlib.Path) -> None:
118 r1 = _run(lang_repo, "code", "languages", "--json")
119 r2 = _run(lang_repo, "code", "languages", "-j")
120 d1, d2 = json.loads(r1.output), json.loads(r2.output)
121 d1.pop("duration_ms", None)
122 d2.pop("duration_ms", None)
123 assert set(d1.keys()) == set(d2.keys())
124
125 def test_j_alias_diff_mode(self, lang_repo: pathlib.Path) -> None:
126 r = _run(lang_repo, "code", "languages", "--diff", "HEAD~1", "-j")
127 assert r.exit_code == 0, r.output
128 json.loads(r.output)
129
130 def test_j_alias_no_ansi(self, lang_repo: pathlib.Path) -> None:
131 r = _run(lang_repo, "code", "languages", "-j")
132 assert "\x1b" not in r.output
133
134
135 # ---------------------------------------------------------------------------
136 # TestSnapshotJson — snapshot mode JSON envelope
137 # ---------------------------------------------------------------------------
138
139
140 class TestSnapshotJson:
141 """Snapshot mode JSON must include exit_code, duration_ms, and correct shape."""
142
143 def test_has_exit_code(self, lang_repo: pathlib.Path) -> None:
144 r = _run(lang_repo, "code", "languages", "--json")
145 assert "exit_code" in json.loads(r.output)
146
147 def test_exit_code_zero(self, lang_repo: pathlib.Path) -> None:
148 r = _run(lang_repo, "code", "languages", "--json")
149 assert r.exit_code == 0
150 assert json.loads(r.output)["exit_code"] == 0
151
152 def test_exit_code_mirrors_process_exit(self, lang_repo: pathlib.Path) -> None:
153 r = _run(lang_repo, "code", "languages", "--json")
154 assert json.loads(r.output)["exit_code"] == r.exit_code
155
156 def test_has_duration_ms(self, lang_repo: pathlib.Path) -> None:
157 r = _run(lang_repo, "code", "languages", "--json")
158 assert "duration_ms" in json.loads(r.output)
159
160 def test_duration_ms_is_float(self, lang_repo: pathlib.Path) -> None:
161 r = _run(lang_repo, "code", "languages", "--json")
162 assert isinstance(json.loads(r.output)["duration_ms"], float)
163
164 def test_duration_ms_nonnegative(self, lang_repo: pathlib.Path) -> None:
165 r = _run(lang_repo, "code", "languages", "--json")
166 assert json.loads(r.output)["duration_ms"] >= 0
167
168 def test_has_languages_key(self, lang_repo: pathlib.Path) -> None:
169 r = _run(lang_repo, "code", "languages", "--json")
170 assert "languages" in json.loads(r.output)
171
172 def test_languages_is_list(self, lang_repo: pathlib.Path) -> None:
173 r = _run(lang_repo, "code", "languages", "--json")
174 assert isinstance(json.loads(r.output)["languages"], list)
175
176 def test_has_commit_key(self, lang_repo: pathlib.Path) -> None:
177 r = _run(lang_repo, "code", "languages", "--json")
178 assert "commit" in json.loads(r.output)
179
180 def test_has_include_imports_key(self, lang_repo: pathlib.Path) -> None:
181 r = _run(lang_repo, "code", "languages", "--json")
182 assert "include_imports" in json.loads(r.output)
183
184 def test_include_imports_false_by_default(self, lang_repo: pathlib.Path) -> None:
185 r = _run(lang_repo, "code", "languages", "--json")
186 assert json.loads(r.output)["include_imports"] is False
187
188 def test_python_present_in_languages(self, lang_repo: pathlib.Path) -> None:
189 r = _run(lang_repo, "code", "languages", "--json")
190 langs = {e["language"] for e in json.loads(r.output)["languages"]}
191 assert "Python" in langs
192
193 def test_language_entry_has_required_keys(self, lang_repo: pathlib.Path) -> None:
194 r = _run(lang_repo, "code", "languages", "--json")
195 for entry in json.loads(r.output)["languages"]:
196 assert "language" in entry
197 assert "files" in entry
198 assert "symbols" in entry
199 assert "kinds" in entry
200
201 def test_files_and_symbols_are_ints(self, lang_repo: pathlib.Path) -> None:
202 r = _run(lang_repo, "code", "languages", "--json")
203 for entry in json.loads(r.output)["languages"]:
204 assert isinstance(entry["files"], int)
205 assert isinstance(entry["symbols"], int)
206
207 def test_no_ansi_in_json(self, lang_repo: pathlib.Path) -> None:
208 r = _run(lang_repo, "code", "languages", "--json")
209 assert "\x1b" not in r.output
210
211
212 # ---------------------------------------------------------------------------
213 # TestDiffJson — diff mode JSON envelope
214 # ---------------------------------------------------------------------------
215
216
217 class TestDiffJson:
218 """Diff mode JSON must include exit_code, duration_ms, and correct shape."""
219
220 def test_diff_has_exit_code(self, lang_repo: pathlib.Path) -> None:
221 r = _run(lang_repo, "code", "languages", "--diff", "HEAD~1", "--json")
222 assert r.exit_code == 0, r.output
223 assert "exit_code" in json.loads(r.output)
224
225 def test_diff_exit_code_zero(self, lang_repo: pathlib.Path) -> None:
226 r = _run(lang_repo, "code", "languages", "--diff", "HEAD~1", "--json")
227 assert json.loads(r.output)["exit_code"] == 0
228
229 def test_diff_has_duration_ms(self, lang_repo: pathlib.Path) -> None:
230 r = _run(lang_repo, "code", "languages", "--diff", "HEAD~1", "--json")
231 assert "duration_ms" in json.loads(r.output)
232
233 def test_diff_duration_ms_is_float(self, lang_repo: pathlib.Path) -> None:
234 r = _run(lang_repo, "code", "languages", "--diff", "HEAD~1", "--json")
235 assert isinstance(json.loads(r.output)["duration_ms"], float)
236
237 def test_diff_has_from_and_to(self, lang_repo: pathlib.Path) -> None:
238 r = _run(lang_repo, "code", "languages", "--diff", "HEAD~1", "--json")
239 d = json.loads(r.output)
240 assert "from_commit" in d
241 assert "to_commit" in d
242
243 def test_diff_has_diff_key(self, lang_repo: pathlib.Path) -> None:
244 r = _run(lang_repo, "code", "languages", "--diff", "HEAD~1", "--json")
245 assert "diff" in json.loads(r.output)
246
247 def test_diff_entries_have_required_keys(self, lang_repo: pathlib.Path) -> None:
248 r = _run(lang_repo, "code", "languages", "--diff", "HEAD~1", "--json")
249 for entry in json.loads(r.output)["diff"]:
250 assert "language" in entry
251 assert "delta_files" in entry
252 assert "delta_symbols" in entry
253 assert "status" in entry
254
255 def test_diff_python_added_symbols(self, lang_repo: pathlib.Path) -> None:
256 """Second commit added auth.py — Python should show positive delta."""
257 r = _run(lang_repo, "code", "languages", "--diff", "HEAD~1", "--json")
258 entries = {e["language"]: e for e in json.loads(r.output)["diff"]}
259 assert "Python" in entries
260 assert entries["Python"]["delta_files"] >= 0
261 assert entries["Python"]["delta_symbols"] >= 0
262
263 def test_diff_status_values_valid(self, lang_repo: pathlib.Path) -> None:
264 r = _run(lang_repo, "code", "languages", "--diff", "HEAD~1", "--json")
265 valid = {"added", "removed", "changed", "unchanged"}
266 for entry in json.loads(r.output)["diff"]:
267 assert entry["status"] in valid
268
269 def test_diff_no_ansi(self, lang_repo: pathlib.Path) -> None:
270 r = _run(lang_repo, "code", "languages", "--diff", "HEAD~1", "--json")
271 assert "\x1b" not in r.output
272
273
274 # ---------------------------------------------------------------------------
275 # TestSortFlag — --sort ordering
276 # ---------------------------------------------------------------------------
277
278
279 class TestSortFlag:
280 """--sort name / files / symbols must produce correctly ordered output."""
281
282 def test_sort_name_alphabetical(self, lang_repo: pathlib.Path) -> None:
283 r = _run(lang_repo, "code", "languages", "--sort", "name", "--json")
284 langs = [e["language"] for e in json.loads(r.output)["languages"]]
285 assert langs == sorted(langs)
286
287 def test_sort_files_descending(self, lang_repo: pathlib.Path) -> None:
288 r = _run(lang_repo, "code", "languages", "--sort", "files", "--json")
289 counts = [e["files"] for e in json.loads(r.output)["languages"]]
290 assert counts == sorted(counts, reverse=True)
291
292 def test_sort_symbols_descending(self, lang_repo: pathlib.Path) -> None:
293 r = _run(lang_repo, "code", "languages", "--sort", "symbols", "--json")
294 counts = [e["symbols"] for e in json.loads(r.output)["languages"]]
295 assert counts == sorted(counts, reverse=True)
296
297 def test_invalid_sort_exits_nonzero(self, lang_repo: pathlib.Path) -> None:
298 r = _run(lang_repo, "code", "languages", "--sort", "bogus", "--json")
299 assert r.exit_code != 0
300
301
302 # ---------------------------------------------------------------------------
303 # TestIncludeImports — import pseudo-symbols
304 # ---------------------------------------------------------------------------
305
306
307 class TestIncludeImports:
308 """--include-imports must add import pseudo-symbols to counts."""
309
310 def test_include_imports_flag_in_json(self, lang_repo: pathlib.Path) -> None:
311 r = _run(lang_repo, "code", "languages", "--include-imports", "--json")
312 assert json.loads(r.output)["include_imports"] is True
313
314 def test_symbols_higher_with_imports(self, lang_repo: pathlib.Path) -> None:
315 r_no = _run(lang_repo, "code", "languages", "--json")
316 r_yes = _run(lang_repo, "code", "languages", "--include-imports", "--json")
317 py_no = next(e for e in json.loads(r_no.output)["languages"] if e["language"] == "Python")
318 py_yes = next(e for e in json.loads(r_yes.output)["languages"] if e["language"] == "Python")
319 assert py_yes["symbols"] >= py_no["symbols"]
320
321 def test_import_kind_present_with_flag(self, lang_repo: pathlib.Path) -> None:
322 r = _run(lang_repo, "code", "languages", "--include-imports", "--json")
323 py = next(e for e in json.loads(r.output)["languages"] if e["language"] == "Python")
324 assert "import" in py["kinds"]
325
326 def test_import_kind_absent_without_flag(self, lang_repo: pathlib.Path) -> None:
327 r = _run(lang_repo, "code", "languages", "--json")
328 py = next(e for e in json.loads(r.output)["languages"] if e["language"] == "Python")
329 assert "import" not in py["kinds"]
330
331
332 # ---------------------------------------------------------------------------
333 # TestHistoricalCommit — --commit flag
334 # ---------------------------------------------------------------------------
335
336
337 class TestHistoricalCommit:
338 """--commit must accept branch names and commit IDs; bad refs exit cleanly."""
339
340 def test_commit_head_tilde_1_accepted(self, lang_repo: pathlib.Path) -> None:
341 r = _run(lang_repo, "code", "languages", "--commit", "HEAD~1", "--json")
342 assert r.exit_code == 0, r.output
343
344 def test_commit_head_tilde_1_has_envelope(self, lang_repo: pathlib.Path) -> None:
345 r = _run(lang_repo, "code", "languages", "--commit", "HEAD~1", "--json")
346 d = json.loads(r.output)
347 assert "exit_code" in d
348 assert "duration_ms" in d
349
350 def test_commit_head_tilde_1_fewer_languages(self, lang_repo: pathlib.Path) -> None:
351 """First commit has no README.md, so Markdown absent or zero."""
352 r_old = _run(lang_repo, "code", "languages", "--commit", "HEAD~1", "--json")
353 r_new = _run(lang_repo, "code", "languages", "--json")
354 old_langs = {e["language"] for e in json.loads(r_old.output)["languages"] if e["files"] > 0}
355 new_langs = {e["language"] for e in json.loads(r_new.output)["languages"] if e["files"] > 0}
356 # HEAD~1 should have fewer or equal active languages
357 assert len(old_langs) <= len(new_langs)
358
359 def test_bad_commit_ref_exits_nonzero(self, lang_repo: pathlib.Path) -> None:
360 r = _run(lang_repo, "code", "languages", "--commit", "nonexistent_branch", "--json")
361 assert r.exit_code != 0
362
363 def test_bad_commit_no_json_on_error(self, lang_repo: pathlib.Path) -> None:
364 r = _run(lang_repo, "code", "languages", "--commit", "nonexistent_branch", "--json")
365 assert r.exit_code != 0
366 # Should not emit JSON on error
367 with pytest.raises(Exception):
368 json.loads(r.output)
369
370
371 # ---------------------------------------------------------------------------
372 # TestTypedDicts — TypedDicts carry envelope fields
373 # ---------------------------------------------------------------------------
374
375
376 class TestTypedDicts:
377 """_SnapshotOutputJson and _DiffOutputJson must carry exit_code and duration_ms."""
378
379 def test_snapshot_typeddict_exists(self) -> None:
380 from muse.cli.commands.languages import _SnapshotOutputJson # noqa: F401
381
382 def test_snapshot_has_exit_code_annotation(self) -> None:
383 from muse.cli.commands.languages import _SnapshotOutputJson
384 assert "exit_code" in _SnapshotOutputJson.__annotations__
385
386 def test_snapshot_has_duration_ms_annotation(self) -> None:
387 from muse.cli.commands.languages import _SnapshotOutputJson
388 assert "duration_ms" in _SnapshotOutputJson.__annotations__
389
390 def test_snapshot_has_languages_annotation(self) -> None:
391 from muse.cli.commands.languages import _SnapshotOutputJson
392 assert "languages" in _SnapshotOutputJson.__annotations__
393
394 def test_diff_typeddict_exists(self) -> None:
395 from muse.cli.commands.languages import _DiffOutputJson # noqa: F401
396
397 def test_diff_has_exit_code_annotation(self) -> None:
398 from muse.cli.commands.languages import _DiffOutputJson
399 assert "exit_code" in _DiffOutputJson.__annotations__
400
401 def test_diff_has_duration_ms_annotation(self) -> None:
402 from muse.cli.commands.languages import _DiffOutputJson
403 assert "duration_ms" in _DiffOutputJson.__annotations__
404
405 def test_diff_has_diff_annotation(self) -> None:
406 from muse.cli.commands.languages import _DiffOutputJson
407 assert "diff" in _DiffOutputJson.__annotations__
408
409
410 # ---------------------------------------------------------------------------
411 # TestDocstrings
412 # ---------------------------------------------------------------------------
413
414
415 class TestDocstrings:
416 """run() must document exit_code."""
417
418 def test_run_mentions_exit_code(self) -> None:
419 from muse.cli.commands.languages import run
420 assert run.__doc__ is not None
421 assert "exit_code" in run.__doc__
422
423
424 # ---------------------------------------------------------------------------
425 # TestPerformance
426 # ---------------------------------------------------------------------------
427
428
429 class TestPerformance:
430 """duration_ms must be present and reasonable on a small repo."""
431
432 def test_snapshot_duration_under_5000ms(self, lang_repo: pathlib.Path) -> None:
433 r = _run(lang_repo, "code", "languages", "--json")
434 assert json.loads(r.output)["duration_ms"] < 5000
435
436 def test_diff_duration_under_5000ms(self, lang_repo: pathlib.Path) -> None:
437 r = _run(lang_repo, "code", "languages", "--diff", "HEAD~1", "--json")
438 assert json.loads(r.output)["duration_ms"] < 5000
439
440 def test_duration_ms_is_float_not_int(self, lang_repo: pathlib.Path) -> None:
441 r = _run(lang_repo, "code", "languages", "--json")
442 assert isinstance(json.loads(r.output)["duration_ms"], float)
443
444
445 # ---------------------------------------------------------------------------
446 # TestRegisterFlags — argparse-level verification
447 # ---------------------------------------------------------------------------
448
449
450 class TestRegisterFlags:
451 """Verify that register() wires --json / -j correctly."""
452
453 def _make_parser(self):
454 import argparse
455 from muse.cli.commands.languages import register
456 ap = argparse.ArgumentParser()
457 subs = ap.add_subparsers()
458 register(subs)
459 return ap
460
461 def test_json_flag_long(self):
462 ns = self._make_parser().parse_args(["languages", "--json"])
463 assert ns.json_out is True
464
465 def test_j_alias(self):
466 ns = self._make_parser().parse_args(["languages", "-j"])
467 assert ns.json_out is True
468
469 def test_default_is_text(self):
470 ns = self._make_parser().parse_args(["languages"])
471 assert ns.json_out is False
472
473 def test_dest_is_json_out(self):
474 ns = self._make_parser().parse_args(["languages", "-j"])
475 assert hasattr(ns, "json_out")
476 assert not hasattr(ns, "fmt")
File History 2 commits
sha256:b636f72dcba9e190afb980bece906fa5b717fbde014b76ef023df8cb96e01eb9 docs: expand cache plan with all seven testing tiers and do… Sonnet 4.6 131 days ago
sha256:7f9e2ef5286aedad9c1e6011b4c46ca27f39dbdad6e3409357e36b26e46b3b7c docs: docstring sprint for-each-ref→hotspots — idiomatic ru… Sonnet 4.6 patch 138 days ago