gabriel / muse public
test_gravity_supercharge.py python
486 lines 19.7 KB
Raw
sha256:b636f72dcba9e190afb980bece906fa5b717fbde014b76ef023df8cb96e01eb9 docs: expand cache plan with all seven testing tiers and do… Sonnet 4.6 134 days ago
1 """Supercharge tests for ``muse code gravity`` — agent-usability gaps.
2
3 There are NO existing gravity tests (confirmed: test_cmd_gravity.py is empty,
4 no other gravity test files exist).
5
6 This file targets both correctness gaps and agent-usability gaps:
7
8 Coverage matrix
9 ---------------
10 - --json / -j: -j alias works identically to --json (both modes)
11 - exit_code: JSON output includes exit_code = 0 on success (both modes)
12 - duration_ms: JSON output includes non-negative float duration_ms (both)
13 - TypedDicts: _JsonOut and _GravityExplainJson carry exit_code/duration_ms
14 - Docstrings: run() docstring mentions exit_code and duration_ms
15 - ANSI: JSON output never contains terminal escape sequences
16 - Performance: duration_ms stays under 5000 ms for a small repo
17 - Schema: leaderboard JSON has required top-level keys
18 - Explain schema: --explain JSON has required fields
19 - args.as_json: --json flag uses dest="as_json" (idiomatic)
20
21 Two JSON modes exercised
22 ------------------------
23 1. Leaderboard mode: --json / -j → _JsonOut envelope
24 2. Explain mode: --explain ADDR --json → _GravityExplainJson envelope
25 """
26
27 from __future__ import annotations
28 from collections.abc import Mapping
29
30 import json
31 import pathlib
32 import textwrap
33
34 import pytest
35
36 from tests.cli_test_helper import CliRunner
37
38 runner = CliRunner()
39
40
41 # ---------------------------------------------------------------------------
42 # Helpers
43 # ---------------------------------------------------------------------------
44
45
46 def _env(root: pathlib.Path) -> Mapping[str, str]:
47 return {"MUSE_REPO_ROOT": str(root)}
48
49
50 def _run(root: pathlib.Path, *args: str):
51 return runner.invoke(None, list(args), env=_env(root))
52
53
54 # ---------------------------------------------------------------------------
55 # Fixture — small Python repo with call-graph structure
56 # ---------------------------------------------------------------------------
57
58
59 @pytest.fixture()
60 def gravity_repo(
61 tmp_path: pathlib.Path, monkeypatch: pytest.MonkeyPatch
62 ) -> pathlib.Path:
63 """Repo with a simple Python call graph for gravity analysis.
64
65 core.py:
66 def read_object() ← foundation, called by everything
67 def validate(x) ← helper, called by process
68
69 service.py:
70 def process(x) ← calls validate, read_object
71 def publish(x) ← calls process
72
73 api.py:
74 def handle(req) ← calls publish, read_object
75
76 This gives:
77 read_object: high gravity (called by process, publish, handle)
78 validate: medium gravity (called by process → publish → handle)
79 process: medium gravity (called by publish → handle)
80 publish: lower gravity (called by handle)
81 handle: zero gravity (nobody calls it)
82 """
83 monkeypatch.chdir(tmp_path)
84 r = _run(tmp_path, "init", "--domain", "code")
85 assert r.exit_code == 0, r.output
86
87 (tmp_path / "core.py").write_text(textwrap.dedent("""\
88 def read_object(obj_id):
89 \"\"\"Load an object by ID.\"\"\"
90 return {"id": obj_id}
91
92 def validate(x):
93 \"\"\"Validate input.\"\"\"
94 if x is None:
95 raise ValueError("x must not be None")
96 return x
97 """))
98 (tmp_path / "service.py").write_text(textwrap.dedent("""\
99 from core import read_object, validate
100
101 def process(x):
102 \"\"\"Process with validation.\"\"\"
103 v = validate(x)
104 obj = read_object(v)
105 return obj
106
107 def publish(x):
108 \"\"\"Publish a processed result.\"\"\"
109 return process(x)
110 """))
111 (tmp_path / "api.py").write_text(textwrap.dedent("""\
112 from service import publish
113 from core import read_object
114
115 def handle(req):
116 \"\"\"Handle an incoming request.\"\"\"
117 read_object(req)
118 return publish(req)
119 """))
120 r = _run(tmp_path, "code", "add", ".")
121 assert r.exit_code == 0, r.output
122 r = _run(tmp_path, "commit", "-m", "seed gravity repo")
123 assert r.exit_code == 0, r.output
124
125 return tmp_path
126
127
128 # ---------------------------------------------------------------------------
129 # TestJsonAlias — -j works identically to --json (leaderboard mode)
130 # ---------------------------------------------------------------------------
131
132
133 class TestJsonAlias:
134 """-j shorthand must behave identically to --json in leaderboard mode."""
135
136 def test_j_alias_exits_zero(self, gravity_repo: pathlib.Path) -> None:
137 r = _run(gravity_repo, "code", "gravity", "-j")
138 assert r.exit_code == 0, r.output
139
140 def test_j_alias_valid_json(self, gravity_repo: pathlib.Path) -> None:
141 r = _run(gravity_repo, "code", "gravity", "-j")
142 json.loads(r.output) # must not raise
143
144 def test_j_alias_has_symbols_key(self, gravity_repo: pathlib.Path) -> None:
145 r = _run(gravity_repo, "code", "gravity", "-j")
146 assert "symbols" in json.loads(r.output)
147
148 def test_j_alias_has_ref_key(self, gravity_repo: pathlib.Path) -> None:
149 r = _run(gravity_repo, "code", "gravity", "-j")
150 assert "ref" in json.loads(r.output)
151
152 def test_j_alias_has_filters_key(self, gravity_repo: pathlib.Path) -> None:
153 r = _run(gravity_repo, "code", "gravity", "-j")
154 assert "filters" in json.loads(r.output)
155
156 def test_j_alias_same_top_level_keys_as_json_flag(
157 self, gravity_repo: pathlib.Path
158 ) -> None:
159 r1 = _run(gravity_repo, "code", "gravity", "--json")
160 r2 = _run(gravity_repo, "code", "gravity", "-j")
161 d1 = json.loads(r1.output)
162 d2 = json.loads(r2.output)
163 d1.pop("duration_ms", None)
164 d2.pop("duration_ms", None)
165 assert set(d1.keys()) == set(d2.keys())
166
167 def test_j_alias_symbol_count_matches_json_flag(
168 self, gravity_repo: pathlib.Path
169 ) -> None:
170 r1 = _run(gravity_repo, "code", "gravity", "--json")
171 r2 = _run(gravity_repo, "code", "gravity", "-j")
172 assert len(json.loads(r1.output)["symbols"]) == len(
173 json.loads(r2.output)["symbols"]
174 )
175
176 def test_j_alias_with_top_filter(self, gravity_repo: pathlib.Path) -> None:
177 r = _run(gravity_repo, "code", "gravity", "-j", "--top", "2")
178 assert r.exit_code == 0, r.output
179 assert len(json.loads(r.output)["symbols"]) <= 2
180
181 def test_j_alias_with_min_gravity(self, gravity_repo: pathlib.Path) -> None:
182 r = _run(gravity_repo, "code", "gravity", "-j", "--min-gravity", "0")
183 assert r.exit_code == 0, r.output
184 data = json.loads(r.output)
185 assert "symbols" in data
186
187
188 # ---------------------------------------------------------------------------
189 # TestDurationMs — JSON output must include duration_ms in both modes
190 # ---------------------------------------------------------------------------
191
192
193 class TestDurationMs:
194 """Every JSON path must include a non-negative float duration_ms."""
195
196 def test_json_has_duration_ms_leaderboard(self, gravity_repo: pathlib.Path) -> None:
197 r = _run(gravity_repo, "code", "gravity", "--json")
198 assert "duration_ms" in json.loads(r.output)
199
200 def test_json_duration_ms_nonnegative(self, gravity_repo: pathlib.Path) -> None:
201 r = _run(gravity_repo, "code", "gravity", "--json")
202 assert json.loads(r.output)["duration_ms"] >= 0
203
204 def test_json_duration_ms_is_float(self, gravity_repo: pathlib.Path) -> None:
205 r = _run(gravity_repo, "code", "gravity", "--json")
206 assert isinstance(json.loads(r.output)["duration_ms"], float)
207
208 def test_j_alias_duration_ms_present(self, gravity_repo: pathlib.Path) -> None:
209 r = _run(gravity_repo, "code", "gravity", "-j")
210 assert "duration_ms" in json.loads(r.output)
211
212 def test_duration_ms_with_top_filter(self, gravity_repo: pathlib.Path) -> None:
213 r = _run(gravity_repo, "code", "gravity", "--json", "--top", "2")
214 data = json.loads(r.output)
215 assert "duration_ms" in data
216 assert data["duration_ms"] >= 0
217
218 def test_duration_ms_explain_mode(self, gravity_repo: pathlib.Path) -> None:
219 r = _run(gravity_repo, "code", "gravity", "--json", "--explain", "core.py::read_object")
220 assert r.exit_code == 0, r.output
221 data = json.loads(r.output)
222 assert "duration_ms" in data
223 assert isinstance(data["duration_ms"], float)
224 assert data["duration_ms"] >= 0
225
226 def test_j_alias_duration_ms_explain(self, gravity_repo: pathlib.Path) -> None:
227 r = _run(gravity_repo, "code", "gravity", "-j", "--explain", "core.py::read_object")
228 assert r.exit_code == 0, r.output
229 assert "duration_ms" in json.loads(r.output)
230
231
232 # ---------------------------------------------------------------------------
233 # TestExitCode — JSON includes exit_code = 0 on success (both modes)
234 # ---------------------------------------------------------------------------
235
236
237 class TestExitCode:
238 """JSON exit_code must be 0 on success in both leaderboard and explain modes."""
239
240 def test_json_has_exit_code_leaderboard(self, gravity_repo: pathlib.Path) -> None:
241 r = _run(gravity_repo, "code", "gravity", "--json")
242 assert "exit_code" in json.loads(r.output)
243
244 def test_json_exit_code_zero_leaderboard(self, gravity_repo: pathlib.Path) -> None:
245 r = _run(gravity_repo, "code", "gravity", "--json")
246 assert r.exit_code == 0
247 assert json.loads(r.output)["exit_code"] == 0
248
249 def test_json_exit_code_is_int_leaderboard(self, gravity_repo: pathlib.Path) -> None:
250 r = _run(gravity_repo, "code", "gravity", "--json")
251 assert isinstance(json.loads(r.output)["exit_code"], int)
252
253 def test_j_alias_exit_code_present(self, gravity_repo: pathlib.Path) -> None:
254 r = _run(gravity_repo, "code", "gravity", "-j")
255 assert "exit_code" in json.loads(r.output)
256
257 def test_exit_code_mirrors_process_exit(self, gravity_repo: pathlib.Path) -> None:
258 r = _run(gravity_repo, "code", "gravity", "--json")
259 assert json.loads(r.output)["exit_code"] == r.exit_code
260
261 def test_json_has_exit_code_explain(self, gravity_repo: pathlib.Path) -> None:
262 r = _run(gravity_repo, "code", "gravity", "--json", "--explain", "core.py::read_object")
263 assert r.exit_code == 0, r.output
264 assert "exit_code" in json.loads(r.output)
265
266 def test_json_exit_code_zero_explain(self, gravity_repo: pathlib.Path) -> None:
267 r = _run(gravity_repo, "code", "gravity", "--json", "--explain", "core.py::read_object")
268 assert r.exit_code == 0
269 assert json.loads(r.output)["exit_code"] == 0
270
271 def test_exit_code_is_int_explain(self, gravity_repo: pathlib.Path) -> None:
272 r = _run(gravity_repo, "code", "gravity", "--json", "--explain", "core.py::read_object")
273 assert isinstance(json.loads(r.output)["exit_code"], int)
274
275 def test_exit_code_mirrors_process_exit_explain(
276 self, gravity_repo: pathlib.Path
277 ) -> None:
278 r = _run(gravity_repo, "code", "gravity", "--json", "--explain", "core.py::read_object")
279 assert json.loads(r.output)["exit_code"] == r.exit_code
280
281
282 # ---------------------------------------------------------------------------
283 # TestTypedDicts — TypedDicts carry exit_code and duration_ms
284 # ---------------------------------------------------------------------------
285
286
287 class TestTypedDicts:
288 """_JsonOut and _GravityExplainJson must carry exit_code and duration_ms."""
289
290 def test_json_out_typeddict_exists(self) -> None:
291 from muse.cli.commands.gravity import _JsonOut # noqa: F401
292
293 def test_json_out_has_exit_code_annotation(self) -> None:
294 from muse.cli.commands.gravity import _JsonOut
295 assert "exit_code" in _JsonOut.__annotations__
296
297 def test_json_out_has_duration_ms_annotation(self) -> None:
298 from muse.cli.commands.gravity import _JsonOut
299 assert "duration_ms" in _JsonOut.__annotations__
300
301 def test_json_out_retains_symbols_annotation(self) -> None:
302 from muse.cli.commands.gravity import _JsonOut
303 assert "symbols" in _JsonOut.__annotations__
304
305 def test_json_out_retains_filters_annotation(self) -> None:
306 from muse.cli.commands.gravity import _JsonOut
307 assert "filters" in _JsonOut.__annotations__
308
309 def test_gravity_explain_json_exists(self) -> None:
310 from muse.cli.commands.gravity import _GravityExplainJson # noqa: F401
311
312 def test_gravity_explain_json_has_exit_code(self) -> None:
313 from muse.cli.commands.gravity import _GravityExplainJson
314 assert "exit_code" in _GravityExplainJson.__annotations__
315
316 def test_gravity_explain_json_has_duration_ms(self) -> None:
317 from muse.cli.commands.gravity import _GravityExplainJson
318 assert "duration_ms" in _GravityExplainJson.__annotations__
319
320 def test_gravity_explain_json_has_address(self) -> None:
321 from muse.cli.commands.gravity import _GravityExplainJson
322 assert "address" in _GravityExplainJson.__annotations__
323
324 def test_gravity_explain_json_has_gravity_pct(self) -> None:
325 from muse.cli.commands.gravity import _GravityExplainJson
326 assert "gravity_pct" in _GravityExplainJson.__annotations__
327
328
329 # ---------------------------------------------------------------------------
330 # TestAnsiSanitization — no escape codes in JSON output
331 # ---------------------------------------------------------------------------
332
333
334 class TestAnsiSanitization:
335 """No ANSI escape sequences anywhere in the JSON output."""
336
337 def test_json_output_no_ansi_leaderboard(self, gravity_repo: pathlib.Path) -> None:
338 r = _run(gravity_repo, "code", "gravity", "--json")
339 assert "\x1b" not in r.output
340
341 def test_j_alias_output_no_ansi(self, gravity_repo: pathlib.Path) -> None:
342 r = _run(gravity_repo, "code", "gravity", "-j")
343 assert "\x1b" not in r.output
344
345 def test_json_output_no_ansi_explain(self, gravity_repo: pathlib.Path) -> None:
346 r = _run(gravity_repo, "code", "gravity", "--json", "--explain", "core.py::read_object")
347 assert "\x1b" not in r.output
348
349
350 # ---------------------------------------------------------------------------
351 # TestLeaderboardSchema — JSON shape for leaderboard mode
352 # ---------------------------------------------------------------------------
353
354
355 class TestLeaderboardSchema:
356 """Leaderboard JSON must carry the documented top-level keys."""
357
358 def test_has_ref_key(self, gravity_repo: pathlib.Path) -> None:
359 r = _run(gravity_repo, "code", "gravity", "--json")
360 assert "ref" in json.loads(r.output)
361
362 def test_has_snapshot_id_key(self, gravity_repo: pathlib.Path) -> None:
363 r = _run(gravity_repo, "code", "gravity", "--json")
364 assert "snapshot_id" in json.loads(r.output)
365
366 def test_has_total_production_symbols_key(self, gravity_repo: pathlib.Path) -> None:
367 r = _run(gravity_repo, "code", "gravity", "--json")
368 assert "total_production_symbols" in json.loads(r.output)
369
370 def test_has_include_tests_key(self, gravity_repo: pathlib.Path) -> None:
371 r = _run(gravity_repo, "code", "gravity", "--json")
372 assert "include_tests" in json.loads(r.output)
373
374 def test_include_tests_is_false_by_default(self, gravity_repo: pathlib.Path) -> None:
375 r = _run(gravity_repo, "code", "gravity", "--json")
376 assert json.loads(r.output)["include_tests"] is False
377
378 def test_symbols_is_list(self, gravity_repo: pathlib.Path) -> None:
379 r = _run(gravity_repo, "code", "gravity", "--json")
380 assert isinstance(json.loads(r.output)["symbols"], list)
381
382 def test_symbol_entries_have_gravity_pct(self, gravity_repo: pathlib.Path) -> None:
383 r = _run(gravity_repo, "code", "gravity", "--json")
384 data = json.loads(r.output)
385 for sym in data["symbols"]:
386 assert "gravity_pct" in sym
387
388 def test_symbol_entries_have_address(self, gravity_repo: pathlib.Path) -> None:
389 r = _run(gravity_repo, "code", "gravity", "--json")
390 data = json.loads(r.output)
391 for sym in data["symbols"]:
392 assert "address" in sym
393
394 def test_top_filter_bounds_symbols(self, gravity_repo: pathlib.Path) -> None:
395 r = _run(gravity_repo, "code", "gravity", "--json", "--top", "2")
396 data = json.loads(r.output)
397 assert len(data["symbols"]) <= 2
398
399
400 # ---------------------------------------------------------------------------
401 # TestExplainSchema — JSON shape for --explain mode
402 # ---------------------------------------------------------------------------
403
404
405 class TestExplainSchema:
406 """Explain JSON must carry the documented fields."""
407
408 def test_explain_has_address(self, gravity_repo: pathlib.Path) -> None:
409 r = _run(gravity_repo, "code", "gravity", "--json", "--explain", "core.py::read_object")
410 assert r.exit_code == 0, r.output
411 assert "address" in json.loads(r.output)
412
413 def test_explain_has_gravity_pct(self, gravity_repo: pathlib.Path) -> None:
414 r = _run(gravity_repo, "code", "gravity", "--json", "--explain", "core.py::read_object")
415 data = json.loads(r.output)
416 assert "gravity_pct" in data
417 assert isinstance(data["gravity_pct"], float)
418
419 def test_explain_has_direct_dependents(self, gravity_repo: pathlib.Path) -> None:
420 r = _run(gravity_repo, "code", "gravity", "--json", "--explain", "core.py::read_object")
421 assert "direct_dependents" in json.loads(r.output)
422
423 def test_explain_has_depth_distribution(self, gravity_repo: pathlib.Path) -> None:
424 r = _run(gravity_repo, "code", "gravity", "--json", "--explain", "core.py::read_object")
425 assert "depth_distribution" in json.loads(r.output)
426
427 def test_explain_address_matches_flag(self, gravity_repo: pathlib.Path) -> None:
428 r = _run(gravity_repo, "code", "gravity", "--json", "--explain", "core.py::validate")
429 assert r.exit_code == 0, r.output
430 data = json.loads(r.output)
431 assert data["address"] == "core.py::validate"
432
433
434 # ---------------------------------------------------------------------------
435 # TestPerformance — duration_ms under 5000 ms for a small repo
436 # ---------------------------------------------------------------------------
437
438
439 class TestPerformance:
440 """duration_ms must stay under 5000 ms for small repos (AST parse overhead)."""
441
442 def test_leaderboard_duration_under_5000ms(self, gravity_repo: pathlib.Path) -> None:
443 r = _run(gravity_repo, "code", "gravity", "--json")
444 assert json.loads(r.output)["duration_ms"] < 5000
445
446 def test_explain_duration_under_5000ms(self, gravity_repo: pathlib.Path) -> None:
447 r = _run(gravity_repo, "code", "gravity", "--json", "--explain", "core.py::read_object")
448 assert json.loads(r.output)["duration_ms"] < 5000
449
450 def test_duration_ms_is_float_not_int(self, gravity_repo: pathlib.Path) -> None:
451 r = _run(gravity_repo, "code", "gravity", "--json")
452 assert isinstance(json.loads(r.output)["duration_ms"], float)
453
454
455 # ---------------------------------------------------------------------------
456 # TestRegisterFlags — argparse-level verification
457 # ---------------------------------------------------------------------------
458
459
460 class TestRegisterFlags:
461 """Verify that register() wires --json / -j correctly."""
462
463 def _make_parser(self):
464 import argparse
465 from muse.cli.commands.gravity import register
466 ap = argparse.ArgumentParser()
467 subs = ap.add_subparsers()
468 register(subs)
469 return ap
470
471 def test_json_flag_long(self):
472 ns = self._make_parser().parse_args(["gravity", "--json"])
473 assert ns.json_out is True
474
475 def test_j_alias(self):
476 ns = self._make_parser().parse_args(["gravity", "-j"])
477 assert ns.json_out is True
478
479 def test_default_is_text(self):
480 ns = self._make_parser().parse_args(["gravity"])
481 assert ns.json_out is False
482
483 def test_dest_is_json_out(self):
484 ns = self._make_parser().parse_args(["gravity", "-j"])
485 assert hasattr(ns, "json_out")
486 assert not hasattr(ns, "fmt")
File History 2 commits
sha256:b636f72dcba9e190afb980bece906fa5b717fbde014b76ef023df8cb96e01eb9 docs: expand cache plan with all seven testing tiers and do… Sonnet 4.6 134 days ago
sha256:7f9e2ef5286aedad9c1e6011b4c46ca27f39dbdad6e3409357e36b26e46b3b7c docs: docstring sprint for-each-ref→hotspots — idiomatic ru… Sonnet 4.6 patch 140 days ago