test_typing_ratchet.py
python
sha256:8028d4db940dfae1ad1467013b1609aa2c2f9da245ce6af5bcab6d4d090eec34
test: per-file strict-mypy ratchet (baseline 1,284 errors, …
Opus 5.5
14 hours ago
| 1 | """Type-check ratchet: no file may gain strict-mypy errors, and the baseline only goes down. |
| 2 | |
| 3 | MuseHub runs ``mypy --strict`` (pyproject ``[tool.mypy]``), but nothing enforced it (1,284 errors on 2026-11-16). |
| 4 | ``tests/typing/mypy-baseline.json`` records per-file error counts and the environment they were measured in: the exact |
| 5 | mypy version, and the Muse engine version, because MuseHub is checked against Muse's own annotations (PEP 561 |
| 6 | ``py.typed``, muse 0.2.2rc9+), and a different engine yields different counts. Where the environment differs, the test |
| 7 | skips and says why, instead of comparing numbers that mean something else. |
| 8 | |
| 9 | TYP_01 no file has more errors than its baseline, and no file absent from the baseline has any |
| 10 | TYP_02 the baseline is current: an improvement must be locked in (``python tests/test_typing_ratchet.py --update``) |
| 11 | |
| 12 | The goal is an empty baseline: zero errors. |
| 13 | """ |
| 14 | from __future__ import annotations |
| 15 | |
| 16 | import collections |
| 17 | import json |
| 18 | import pathlib |
| 19 | import re |
| 20 | import subprocess |
| 21 | import sys |
| 22 | |
| 23 | import pytest |
| 24 | |
| 25 | ROOT = pathlib.Path(__file__).resolve().parents[1] |
| 26 | BASELINE = ROOT / "tests" / "typing" / "mypy-baseline.json" |
| 27 | PACKAGE = "musehub" |
| 28 | _ERROR = re.compile(r"^(?P<path>[^:\n]+\.py):\d+: error:", re.M) |
| 29 | |
| 30 | |
| 31 | def environment() -> dict[str, str]: |
| 32 | import muse |
| 33 | from muse._version import _read_version |
| 34 | |
| 35 | mypy = subprocess.run([sys.executable, "-m", "mypy", "--version"], capture_output=True, text=True, check=True) |
| 36 | typed = (pathlib.Path(muse.__file__).parent / "py.typed").is_file() |
| 37 | return {"mypy": mypy.stdout.split()[1], "muse": _read_version(), "muse_py_typed": str(typed).lower()} |
| 38 | |
| 39 | |
| 40 | def measure() -> dict[str, int]: |
| 41 | proc = subprocess.run([sys.executable, "-m", "mypy", PACKAGE], cwd=ROOT, capture_output=True, text=True, timeout=1800) |
| 42 | if proc.returncode not in (0, 1) or ("error:" not in proc.stdout and "Success" not in proc.stdout): |
| 43 | raise RuntimeError(f"mypy did not run cleanly (exit {proc.returncode}):\n{proc.stdout[-2000:]}{proc.stderr[-2000:]}") |
| 44 | return dict(sorted(collections.Counter(m.group("path") for m in _ERROR.finditer(proc.stdout)).items())) |
| 45 | |
| 46 | |
| 47 | def _baseline() -> dict[str, object]: |
| 48 | base: dict[str, object] = json.loads(BASELINE.read_text()) |
| 49 | env = environment() |
| 50 | if env != base["environment"]: |
| 51 | pytest.skip(f"type-check environment {env} is not the baseline's {base['environment']}") |
| 52 | return base |
| 53 | |
| 54 | |
| 55 | def test_TYP_01_no_file_gains_type_errors() -> None: |
| 56 | allowed: dict[str, int] = _baseline()["files"] # type: ignore[assignment] |
| 57 | counts = measure() |
| 58 | worse = {p: f"{allowed.get(p, 0)} -> {n}" for p, n in counts.items() if n > allowed.get(p, 0)} |
| 59 | assert not worse, f"new strict-mypy errors (fix them; never raise the baseline): {worse}" |
| 60 | |
| 61 | |
| 62 | def test_TYP_02_the_baseline_is_lowered_when_a_file_improves() -> None: |
| 63 | allowed: dict[str, int] = _baseline()["files"] # type: ignore[assignment] |
| 64 | counts = measure() |
| 65 | better = {p: f"{n} -> {counts.get(p, 0)}" for p, n in allowed.items() if counts.get(p, 0) < n} |
| 66 | assert not better, f"files improved; lock it in with `python tests/test_typing_ratchet.py --update`: {better}" |
| 67 | |
| 68 | |
| 69 | if __name__ == "__main__" and sys.argv[1:] == ["--update"]: |
| 70 | counts = measure() |
| 71 | BASELINE.write_text(json.dumps({"environment": environment(), "total": sum(counts.values()), "files": counts}, indent=1) + "\n") |
| 72 | print(f"baseline: {sum(counts.values())} errors in {len(counts)} files ({environment()})") |
File History
1 commit
sha256:8028d4db940dfae1ad1467013b1609aa2c2f9da245ce6af5bcab6d4d090eec34
test: per-file strict-mypy ratchet (baseline 1,284 errors, …
Opus 5.5
14 hours ago