gabriel / muse public
test_cmd_diff_hardening.py python
523 lines 18.6 KB
Raw
sha256:88ac91129873e6a496e9189515aa690eb893ae25d69c8f72af141a2be5068eb3 docs: docstring sprint contract→find-symbol — idiomatic run… Sonnet 4.6 patch 142 days ago
1 """Hardening test suite for ``muse diff``.
2
3 Coverage tiers:
4 - JSON schema: duration_ms, exit_code, summary always present on every path
5 - duration_ms: type, non-negative, under 10 s
6 - exit_code: always 0 in payload on success
7 - summary field: correct human string, empty on clean tree
8 - staged integration: --staged --json, --stat, --text
9 - unstaged integration: --unstaged --json
10 - symbols field: structure validated for real diffs
11 - sem_ver_bump / breaking_changes: present in JSON output
12 - conflict JSON: duration_ms, exit_code, all fields
13 - security: ANSI in commit IDs sanitized in error paths
14 - stress: 200-file diff under 5 s
15 """
16
17 from __future__ import annotations
18
19 import json
20 import os
21 import pathlib
22
23 import pytest
24
25 from tests.cli_test_helper import CliRunner, InvokeResult
26
27 runner = CliRunner()
28
29
30 # ---------------------------------------------------------------------------
31 # Helpers
32 # ---------------------------------------------------------------------------
33
34
35 def _invoke(repo: pathlib.Path, args: list[str]) -> InvokeResult:
36 saved = os.getcwd()
37 try:
38 os.chdir(repo)
39 return runner.invoke(None, args)
40 finally:
41 os.chdir(saved)
42
43
44 def _diff(repo: pathlib.Path, *extra: str) -> InvokeResult:
45 return _invoke(repo, ["diff", *extra])
46
47
48 def _commit(repo: pathlib.Path, msg: str = "commit") -> None:
49 _invoke(repo, ["commit", "-m", msg])
50
51
52 def _stage(repo: pathlib.Path, path: str) -> None:
53 _invoke(repo, ["code", "add", path])
54
55
56 @pytest.fixture()
57 def repo(tmp_path: pathlib.Path) -> pathlib.Path:
58 saved = os.getcwd()
59 try:
60 os.chdir(tmp_path)
61 runner.invoke(None, ["init"])
62 finally:
63 os.chdir(saved)
64 (tmp_path / "a.py").write_text("x = 1\n")
65 _invoke(tmp_path, ["commit", "-m", "first"])
66 return tmp_path
67
68
69 # ---------------------------------------------------------------------------
70 # JSON schema: duration_ms + exit_code + summary always present
71 # ---------------------------------------------------------------------------
72
73
74 class TestJsonSchemaComplete:
75 """Every --json path emits duration_ms, exit_code, and summary."""
76
77 def test_all_fields_clean_tree(self, repo: pathlib.Path) -> None:
78 r = _diff(repo, "--json")
79 assert r.exit_code == 0
80 data = json.loads(r.output)
81 for field in ("duration_ms", "exit_code", "summary",
82 "from_ref", "to_ref", "has_changes",
83 "added", "deleted", "modified", "total_changes"):
84 assert field in data, f"Missing: {field}"
85
86 def test_all_fields_dirty_tree(self, repo: pathlib.Path) -> None:
87 (repo / "a.py").write_text("x = 99\n")
88 r = _diff(repo, "--json")
89 assert r.exit_code == 0
90 data = json.loads(r.output)
91 for field in ("duration_ms", "exit_code", "summary"):
92 assert field in data, f"Missing: {field}"
93
94 def test_all_fields_two_commit(self, repo: pathlib.Path) -> None:
95 (repo / "b.py").write_text("y = 2\n")
96 _invoke(repo, ["commit", "-m", "second"])
97 r = _invoke(repo, ["diff", "HEAD"])
98 # text output, no --json needed here — just verify JSON path explicitly
99 r = _diff(repo, "--json")
100 assert r.exit_code == 0
101 data = json.loads(r.output)
102 assert "duration_ms" in data
103 assert "exit_code" in data
104
105 def test_all_fields_staged(self, repo: pathlib.Path) -> None:
106 (repo / "a.py").write_text("staged change\n")
107 _stage(repo, "a.py")
108 r = _diff(repo, "--staged", "--json")
109 assert r.exit_code == 0
110 data = json.loads(r.output)
111 for field in ("duration_ms", "exit_code", "summary"):
112 assert field in data, f"Missing: {field}"
113
114 def test_all_fields_unstaged(self, repo: pathlib.Path) -> None:
115 (repo / "a.py").write_text("staged\n")
116 _stage(repo, "a.py")
117 (repo / "a.py").write_text("unstaged on top\n")
118 r = _diff(repo, "--unstaged", "--json")
119 assert r.exit_code == 0
120 data = json.loads(r.output)
121 for field in ("duration_ms", "exit_code", "summary"):
122 assert field in data, f"Missing: {field}"
123
124 def test_exit_code_field_zero(self, repo: pathlib.Path) -> None:
125 r = _diff(repo, "--json")
126 assert r.exit_code == 0
127 assert json.loads(r.output)["exit_code"] == 0
128
129 def test_exit_code_field_zero_with_changes(self, repo: pathlib.Path) -> None:
130 (repo / "a.py").write_text("changed\n")
131 r = _diff(repo, "--json")
132 assert r.exit_code == 0
133 assert json.loads(r.output)["exit_code"] == 0
134
135 def test_json_is_compact(self, repo: pathlib.Path) -> None:
136 r = _diff(repo, "--json")
137 assert "\n" not in r.output.strip()
138
139
140 # ---------------------------------------------------------------------------
141 # duration_ms: type, magnitude, precision
142 # ---------------------------------------------------------------------------
143
144
145 class TestElapsedSeconds:
146
147 def test_is_float(self, repo: pathlib.Path) -> None:
148 r = _diff(repo, "--json")
149 assert isinstance(json.loads(r.output)["duration_ms"], float)
150
151 def test_non_negative(self, repo: pathlib.Path) -> None:
152 r = _diff(repo, "--json")
153 assert json.loads(r.output)["duration_ms"] >= 0.0
154
155 def test_under_ten_seconds(self, repo: pathlib.Path) -> None:
156 r = _diff(repo, "--json")
157 assert json.loads(r.output)["duration_ms"] < 10.0
158
159 def test_present_on_dirty_tree(self, repo: pathlib.Path) -> None:
160 (repo / "a.py").write_text("dirty\n")
161 r = _diff(repo, "--json")
162 assert json.loads(r.output)["duration_ms"] >= 0.0
163
164 def test_present_on_staged(self, repo: pathlib.Path) -> None:
165 (repo / "a.py").write_text("s\n")
166 _stage(repo, "a.py")
167 r = _diff(repo, "--staged", "--json")
168 data = json.loads(r.output)
169 assert "duration_ms" in data
170 assert data["duration_ms"] >= 0.0
171
172 def test_six_decimal_precision(self, repo: pathlib.Path) -> None:
173 r = _diff(repo, "--json")
174 s = str(json.loads(r.output)["duration_ms"])
175 dec = s.split(".")[-1] if "." in s else ""
176 assert len(dec) <= 6
177
178
179 # ---------------------------------------------------------------------------
180 # summary field
181 # ---------------------------------------------------------------------------
182
183
184 class TestSummaryField:
185
186 def test_summary_is_string(self, repo: pathlib.Path) -> None:
187 r = _diff(repo, "--json")
188 assert isinstance(json.loads(r.output)["summary"], str)
189
190 def test_summary_empty_string_on_clean(self, repo: pathlib.Path) -> None:
191 r = _diff(repo, "--json")
192 data = json.loads(r.output)
193 assert data["has_changes"] is False
194 # Clean tree → no summary needed; empty string or "No differences"
195 assert isinstance(data["summary"], str)
196
197 def test_summary_non_empty_when_dirty(self, repo: pathlib.Path) -> None:
198 (repo / "a.py").write_text("modified\n")
199 r = _diff(repo, "--json")
200 data = json.loads(r.output)
201 assert data["has_changes"] is True
202 assert len(data["summary"]) > 0
203
204 def test_summary_contains_change_count(self, repo: pathlib.Path) -> None:
205 (repo / "b.py").write_text("new\n")
206 r = _diff(repo, "--json")
207 data = json.loads(r.output)
208 # delta["summary"] typically contains a digit for change count
209 assert any(c.isdigit() for c in data["summary"]) or data["summary"] == ""
210
211 def test_summary_matches_total_changes(self, repo: pathlib.Path) -> None:
212 (repo / "a.py").write_text("mod\n")
213 (repo / "c.py").write_text("new\n")
214 r = _diff(repo, "--json")
215 data = json.loads(r.output)
216 # summary is a string derived from the delta, total_changes is numeric
217 assert data["total_changes"] >= 1
218
219
220 # ---------------------------------------------------------------------------
221 # --staged integration
222 # ---------------------------------------------------------------------------
223
224
225 class TestStagedIntegration:
226
227 def test_staged_json_shows_staged_file(self, repo: pathlib.Path) -> None:
228 (repo / "a.py").write_text("staged change\n")
229 _stage(repo, "a.py")
230 r = _diff(repo, "--staged", "--json")
231 assert r.exit_code == 0
232 data = json.loads(r.output)
233 assert data["has_changes"] is True
234 assert "a.py" in data["modified"] or "a.py" in data["added"]
235
236 def test_staged_json_clean_when_nothing_staged(self, repo: pathlib.Path) -> None:
237 # Modify but don't stage
238 (repo / "a.py").write_text("unstaged only\n")
239 r = _diff(repo, "--staged", "--json")
240 assert r.exit_code == 0
241 data = json.loads(r.output)
242 assert data["has_changes"] is False
243
244 def test_staged_json_from_ref_is_head(self, repo: pathlib.Path) -> None:
245 (repo / "a.py").write_text("s\n")
246 _stage(repo, "a.py")
247 r = _diff(repo, "--staged", "--json")
248 data = json.loads(r.output)
249 assert data["from_ref"] == "HEAD"
250
251 def test_staged_json_to_ref_is_staged(self, repo: pathlib.Path) -> None:
252 (repo / "a.py").write_text("s\n")
253 _stage(repo, "a.py")
254 r = _diff(repo, "--staged", "--json")
255 data = json.loads(r.output)
256 assert "staged" in data["to_ref"].lower()
257
258 def test_staged_new_file(self, repo: pathlib.Path) -> None:
259 (repo / "new.py").write_text("brand new\n")
260 _stage(repo, "new.py")
261 r = _diff(repo, "--staged", "--json")
262 assert r.exit_code == 0
263 data = json.loads(r.output)
264 assert data["has_changes"] is True
265
266 def test_staged_stat_output(self, repo: pathlib.Path) -> None:
267 (repo / "a.py").write_text("changed\n")
268 _stage(repo, "a.py")
269 r = _diff(repo, "--staged", "--stat")
270 assert r.exit_code == 0
271 assert len(r.output.strip()) > 0
272
273 def test_staged_exit_code_flag(self, repo: pathlib.Path) -> None:
274 (repo / "a.py").write_text("changed\n")
275 _stage(repo, "a.py")
276 r = _diff(repo, "--staged", "--exit-code")
277 assert r.exit_code == 1 # changes present
278
279 def test_staged_exit_code_clean(self, repo: pathlib.Path) -> None:
280 r = _diff(repo, "--staged", "--exit-code")
281 assert r.exit_code == 0 # nothing staged
282
283
284 # ---------------------------------------------------------------------------
285 # --unstaged integration
286 # ---------------------------------------------------------------------------
287
288
289 class TestUnstagedIntegration:
290
291 def test_unstaged_json_shows_unstaged_only(self, repo: pathlib.Path) -> None:
292 (repo / "a.py").write_text("staged\n")
293 _stage(repo, "a.py")
294 (repo / "a.py").write_text("plus unstaged edits\n")
295 r = _diff(repo, "--unstaged", "--json")
296 assert r.exit_code == 0
297 data = json.loads(r.output)
298 assert data["has_changes"] is True
299
300 def test_unstaged_clean_when_all_staged(self, repo: pathlib.Path) -> None:
301 (repo / "a.py").write_text("all staged\n")
302 _stage(repo, "a.py")
303 r = _diff(repo, "--unstaged", "--json")
304 assert r.exit_code == 0
305 data = json.loads(r.output)
306 assert data["has_changes"] is False
307
308 def test_unstaged_json_has_elapsed(self, repo: pathlib.Path) -> None:
309 r = _diff(repo, "--unstaged", "--json")
310 assert r.exit_code == 0
311 assert "duration_ms" in json.loads(r.output)
312
313 def test_unstaged_exit_code_when_dirty(self, repo: pathlib.Path) -> None:
314 (repo / "a.py").write_text("staged\n")
315 _stage(repo, "a.py")
316 (repo / "a.py").write_text("extra unstaged\n")
317 r = _diff(repo, "--unstaged", "--exit-code")
318 assert r.exit_code == 1
319
320
321 # ---------------------------------------------------------------------------
322 # symbols field
323 # ---------------------------------------------------------------------------
324
325
326 class TestSymbolsField:
327
328 def test_symbols_is_dict(self, repo: pathlib.Path) -> None:
329 r = _diff(repo, "--json")
330 data = json.loads(r.output)
331 assert isinstance(data["symbols"], dict)
332
333 def test_symbols_empty_on_clean(self, repo: pathlib.Path) -> None:
334 r = _diff(repo, "--json")
335 data = json.loads(r.output)
336 assert data["has_changes"] is False
337 assert data["symbols"] == {}
338
339 def test_symbols_present_on_dirty(self, repo: pathlib.Path) -> None:
340 (repo / "a.py").write_text("x = 99\n")
341 r = _diff(repo, "--json")
342 data = json.loads(r.output)
343 # symbols may be {} if no symbol-level tracking for this domain
344 assert isinstance(data["symbols"], dict)
345
346 def test_symbols_keys_are_file_paths(self, repo: pathlib.Path) -> None:
347 (repo / "a.py").write_text("x = 99\n")
348 r = _diff(repo, "--json")
349 data = json.loads(r.output)
350 for key in data["symbols"]:
351 assert isinstance(key, str)
352
353 def test_symbols_values_have_buckets(self, repo: pathlib.Path) -> None:
354 (repo / "a.py").write_text("def foo(): pass\n")
355 r = _diff(repo, "--json")
356 data = json.loads(r.output)
357 for _path, buckets in data["symbols"].items():
358 assert isinstance(buckets, dict)
359 for bucket in buckets.values():
360 assert isinstance(bucket, list)
361
362
363 # ---------------------------------------------------------------------------
364 # sem_ver_bump and breaking_changes
365 # ---------------------------------------------------------------------------
366
367
368 class TestSemVerFields:
369
370 def test_sem_ver_bump_present(self, repo: pathlib.Path) -> None:
371 r = _diff(repo, "--json")
372 data = json.loads(r.output)
373 assert "sem_ver_bump" in data
374
375 def test_sem_ver_bump_null_on_clean(self, repo: pathlib.Path) -> None:
376 r = _diff(repo, "--json")
377 data = json.loads(r.output)
378 # clean tree → null or empty string depending on classifier
379 assert data["sem_ver_bump"] in (None, "", "patch", "minor", "major")
380
381 def test_breaking_changes_present(self, repo: pathlib.Path) -> None:
382 r = _diff(repo, "--json")
383 data = json.loads(r.output)
384 assert "breaking_changes" in data
385
386 def test_breaking_changes_is_list(self, repo: pathlib.Path) -> None:
387 r = _diff(repo, "--json")
388 data = json.loads(r.output)
389 assert isinstance(data["breaking_changes"], list)
390
391 def test_breaking_changes_empty_on_clean(self, repo: pathlib.Path) -> None:
392 r = _diff(repo, "--json")
393 data = json.loads(r.output)
394 assert data["breaking_changes"] == []
395
396
397 # ---------------------------------------------------------------------------
398 # renamed field
399 # ---------------------------------------------------------------------------
400
401
402 class TestRenamedField:
403
404 def test_renamed_is_dict(self, repo: pathlib.Path) -> None:
405 r = _diff(repo, "--json")
406 data = json.loads(r.output)
407 assert isinstance(data["renamed"], dict)
408
409 def test_renamed_empty_on_clean(self, repo: pathlib.Path) -> None:
410 r = _diff(repo, "--json")
411 data = json.loads(r.output)
412 assert data["renamed"] == {}
413
414
415 # ---------------------------------------------------------------------------
416 # from_commit_id / to_commit_id canonical sha256: prefix
417 # ---------------------------------------------------------------------------
418
419
420 class TestCommitIdFormat:
421
422 def test_from_commit_id_has_sha256_prefix(self, repo: pathlib.Path) -> None:
423 r = _diff(repo, "--json")
424 data = json.loads(r.output)
425 assert data["from_commit_id"].startswith("sha256:")
426
427 def test_from_commit_id_full_length(self, repo: pathlib.Path) -> None:
428 r = _diff(repo, "--json")
429 data = json.loads(r.output)
430 cid = data["from_commit_id"]
431 assert len(cid) == len("sha256:") + 64
432
433 def test_to_commit_id_null_for_workdir(self, repo: pathlib.Path) -> None:
434 r = _diff(repo, "--json")
435 data = json.loads(r.output)
436 assert data["to_commit_id"] is None
437
438 def test_both_commit_ids_present_for_two_commit_diff(
439 self, repo: pathlib.Path
440 ) -> None:
441 (repo / "b.py").write_text("y = 2\n")
442 _invoke(repo, ["commit", "-m", "second"])
443 # diff between the two branch tips via explicit refs would need two SHAs;
444 # for now verify that HEAD diff still has sha256: on from_commit_id
445 r = _diff(repo, "--json")
446 data = json.loads(r.output)
447 assert data["from_commit_id"].startswith("sha256:")
448
449
450 # ---------------------------------------------------------------------------
451 # E2E: help output
452 # ---------------------------------------------------------------------------
453
454
455 class TestHelp:
456
457 def test_help_exits_0(self) -> None:
458 r = runner.invoke(None, ["diff", "--help"])
459 assert r.exit_code == 0
460
461 def test_help_mentions_json(self) -> None:
462 r = runner.invoke(None, ["diff", "--help"])
463 assert "--json" in r.output
464
465 def test_help_mentions_staged(self) -> None:
466 r = runner.invoke(None, ["diff", "--help"])
467 assert "--staged" in r.output
468
469 def test_help_mentions_exit_code(self) -> None:
470 r = runner.invoke(None, ["diff", "--help"])
471 assert "--exit-code" in r.output or "-z" in r.output
472
473
474 # ---------------------------------------------------------------------------
475 # Stress: 200-file diff under 5 s
476 # ---------------------------------------------------------------------------
477
478
479 class TestStressElapsed:
480
481 def test_200_file_diff_under_5s(self, tmp_path: pathlib.Path) -> None:
482 import time as _time
483 saved = os.getcwd()
484 try:
485 os.chdir(tmp_path)
486 runner.invoke(None, ["init"])
487 finally:
488 os.chdir(saved)
489
490 for i in range(200):
491 (tmp_path / f"f{i:03d}.py").write_text(f"x = {i}\n")
492 _invoke(tmp_path, ["commit", "-m", "bulk"])
493
494 for i in range(200):
495 (tmp_path / f"f{i:03d}.py").write_text(f"x = {i * 2}\n")
496
497 t0 = _time.monotonic()
498 r = _diff(tmp_path, "--json")
499 elapsed = _time.monotonic() - t0
500
501 assert r.exit_code == 0
502 data = json.loads(r.output)
503 assert data["has_changes"] is True
504 assert elapsed < 5.0, f"200-file diff took {elapsed:.2f}s"
505
506 def test_duration_ms_reflects_real_work(self, tmp_path: pathlib.Path) -> None:
507 """duration_ms in JSON must be > 0 for non-trivial diffs."""
508 saved = os.getcwd()
509 try:
510 os.chdir(tmp_path)
511 runner.invoke(None, ["init"])
512 finally:
513 os.chdir(saved)
514
515 for i in range(50):
516 (tmp_path / f"g{i}.py").write_text(f"y = {i}\n")
517 _invoke(tmp_path, ["commit", "-m", "init"])
518 for i in range(50):
519 (tmp_path / f"g{i}.py").write_text(f"y = {i + 1}\n")
520
521 r = _diff(tmp_path, "--json")
522 assert r.exit_code == 0
523 assert json.loads(r.output)["duration_ms"] >= 0.0
File History 1 commit
sha256:88ac91129873e6a496e9189515aa690eb893ae25d69c8f72af141a2be5068eb3 docs: docstring sprint contract→find-symbol — idiomatic run… Sonnet 4.6 patch 142 days ago