gabriel / musehub public
test_typing_ratchet.py python
72 lines 3.4 KB
Raw
sha256:8028d4db940dfae1ad1467013b1609aa2c2f9da245ce6af5bcab6d4d090eec34 test: per-file strict-mypy ratchet (baseline 1,284 errors, … Opus 5.5 14 hours ago
1 """Type-check ratchet: no file may gain strict-mypy errors, and the baseline only goes down.
2
3 MuseHub runs ``mypy --strict`` (pyproject ``[tool.mypy]``), but nothing enforced it (1,284 errors on 2026-11-16).
4 ``tests/typing/mypy-baseline.json`` records per-file error counts and the environment they were measured in: the exact
5 mypy version, and the Muse engine version, because MuseHub is checked against Muse's own annotations (PEP 561
6 ``py.typed``, muse 0.2.2rc9+), and a different engine yields different counts. Where the environment differs, the test
7 skips and says why, instead of comparing numbers that mean something else.
8
9 TYP_01 no file has more errors than its baseline, and no file absent from the baseline has any
10 TYP_02 the baseline is current: an improvement must be locked in (``python tests/test_typing_ratchet.py --update``)
11
12 The goal is an empty baseline: zero errors.
13 """
14 from __future__ import annotations
15
16 import collections
17 import json
18 import pathlib
19 import re
20 import subprocess
21 import sys
22
23 import pytest
24
25 ROOT = pathlib.Path(__file__).resolve().parents[1]
26 BASELINE = ROOT / "tests" / "typing" / "mypy-baseline.json"
27 PACKAGE = "musehub"
28 _ERROR = re.compile(r"^(?P<path>[^:\n]+\.py):\d+: error:", re.M)
29
30
31 def environment() -> dict[str, str]:
32 import muse
33 from muse._version import _read_version
34
35 mypy = subprocess.run([sys.executable, "-m", "mypy", "--version"], capture_output=True, text=True, check=True)
36 typed = (pathlib.Path(muse.__file__).parent / "py.typed").is_file()
37 return {"mypy": mypy.stdout.split()[1], "muse": _read_version(), "muse_py_typed": str(typed).lower()}
38
39
40 def measure() -> dict[str, int]:
41 proc = subprocess.run([sys.executable, "-m", "mypy", PACKAGE], cwd=ROOT, capture_output=True, text=True, timeout=1800)
42 if proc.returncode not in (0, 1) or ("error:" not in proc.stdout and "Success" not in proc.stdout):
43 raise RuntimeError(f"mypy did not run cleanly (exit {proc.returncode}):\n{proc.stdout[-2000:]}{proc.stderr[-2000:]}")
44 return dict(sorted(collections.Counter(m.group("path") for m in _ERROR.finditer(proc.stdout)).items()))
45
46
47 def _baseline() -> dict[str, object]:
48 base: dict[str, object] = json.loads(BASELINE.read_text())
49 env = environment()
50 if env != base["environment"]:
51 pytest.skip(f"type-check environment {env} is not the baseline's {base['environment']}")
52 return base
53
54
55 def test_TYP_01_no_file_gains_type_errors() -> None:
56 allowed: dict[str, int] = _baseline()["files"] # type: ignore[assignment]
57 counts = measure()
58 worse = {p: f"{allowed.get(p, 0)} -> {n}" for p, n in counts.items() if n > allowed.get(p, 0)}
59 assert not worse, f"new strict-mypy errors (fix them; never raise the baseline): {worse}"
60
61
62 def test_TYP_02_the_baseline_is_lowered_when_a_file_improves() -> None:
63 allowed: dict[str, int] = _baseline()["files"] # type: ignore[assignment]
64 counts = measure()
65 better = {p: f"{n} -> {counts.get(p, 0)}" for p, n in allowed.items() if counts.get(p, 0) < n}
66 assert not better, f"files improved; lock it in with `python tests/test_typing_ratchet.py --update`: {better}"
67
68
69 if __name__ == "__main__" and sys.argv[1:] == ["--update"]:
70 counts = measure()
71 BASELINE.write_text(json.dumps({"environment": environment(), "total": sum(counts.values()), "files": counts}, indent=1) + "\n")
72 print(f"baseline: {sum(counts.values())} errors in {len(counts)} files ({environment()})")
File History 1 commit
sha256:8028d4db940dfae1ad1467013b1609aa2c2f9da245ce6af5bcab6d4d090eec34 test: per-file strict-mypy ratchet (baseline 1,284 errors, … Opus 5.5 14 hours ago