test_semantic_test_coverage_supercharge.py
python
sha256:b636f72dcba9e190afb980bece906fa5b717fbde014b76ef023df8cb96e01eb9
docs: expand cache plan with all seven testing tiers and do…
Sonnet 4.6
131 days ago
| 1 | """Supercharge tests for ``muse code semantic-test-coverage``. |
| 2 | |
| 3 | Coverage tiers |
| 4 | -------------- |
| 5 | Unit — TypedDict shape (_JsonOut gains exit_code/duration_ms/schema_version), |
| 6 | alias registration (-j), docstring completeness. |
| 7 | Integration — -j alias produces same output as --json; exit_code/duration_ms/ |
| 8 | schema_version present in JSON; --min-coverage exit code mirrors |
| 9 | process exit; uncovered-only JSON still carries envelope fields. |
| 10 | Security — ANSI in --file/--kind args not echoed raw. |
| 11 | Performance — duration_ms < 5000 ms on a small repo. |
| 12 | """ |
| 13 | |
| 14 | from __future__ import annotations |
| 15 | |
| 16 | import json |
| 17 | import os |
| 18 | import pathlib |
| 19 | import textwrap |
| 20 | |
| 21 | import pytest |
| 22 | |
| 23 | from tests.cli_test_helper import CliRunner |
| 24 | |
| 25 | runner = CliRunner() |
| 26 | |
| 27 | |
| 28 | # ────────────────────────────────────────────────────────────────────────────── |
| 29 | # Fixtures |
| 30 | # ────────────────────────────────────────────────────────────────────────────── |
| 31 | |
| 32 | |
| 33 | def _invoke(repo: pathlib.Path, args: list[str]): |
| 34 | saved = os.getcwd() |
| 35 | try: |
| 36 | os.chdir(repo) |
| 37 | return runner.invoke(None, args) |
| 38 | finally: |
| 39 | os.chdir(saved) |
| 40 | |
| 41 | |
| 42 | def _stc(repo: pathlib.Path, *args: str): |
| 43 | return _invoke(repo, ["code", "semantic-test-coverage", *args]) |
| 44 | |
| 45 | |
| 46 | @pytest.fixture() |
| 47 | def cov_repo(tmp_path: pathlib.Path) -> pathlib.Path: |
| 48 | """Minimal repo with one production file and one test file.""" |
| 49 | saved = os.getcwd() |
| 50 | try: |
| 51 | os.chdir(tmp_path) |
| 52 | runner.invoke(None, ["init"]) |
| 53 | finally: |
| 54 | os.chdir(saved) |
| 55 | |
| 56 | (tmp_path / "billing.py").write_text( |
| 57 | textwrap.dedent("""\ |
| 58 | class Invoice: |
| 59 | def compute_total(self, items): |
| 60 | return sum(item["price"] for item in items) |
| 61 | |
| 62 | def apply_discount(self, total, pct): |
| 63 | return total * (1 - pct / 100) |
| 64 | |
| 65 | def generate_pdf(self): |
| 66 | pass |
| 67 | """), |
| 68 | encoding="utf-8", |
| 69 | ) |
| 70 | (tmp_path / "tests").mkdir() |
| 71 | (tmp_path / "tests" / "test_billing.py").write_text( |
| 72 | textwrap.dedent("""\ |
| 73 | from billing import Invoice |
| 74 | |
| 75 | def test_compute_total(): |
| 76 | inv = Invoice() |
| 77 | assert inv.compute_total([{"price": 10}]) == 10 |
| 78 | |
| 79 | def test_apply_discount(): |
| 80 | inv = Invoice() |
| 81 | assert inv.apply_discount(100, 10) == 90 |
| 82 | """), |
| 83 | encoding="utf-8", |
| 84 | ) |
| 85 | |
| 86 | saved = os.getcwd() |
| 87 | try: |
| 88 | os.chdir(tmp_path) |
| 89 | runner.invoke(None, ["code", "add", "."]) |
| 90 | runner.invoke(None, ["commit", "-m", "init"]) |
| 91 | finally: |
| 92 | os.chdir(saved) |
| 93 | |
| 94 | return tmp_path |
| 95 | |
| 96 | |
| 97 | # ────────────────────────────────────────────────────────────────────────────── |
| 98 | # Unit — TypedDict |
| 99 | # ────────────────────────────────────────────────────────────────────────────── |
| 100 | |
| 101 | |
| 102 | class TestTypedDict: |
| 103 | def test_json_out_has_exit_code(self) -> None: |
| 104 | import typing |
| 105 | from muse.cli.commands.semantic_test_coverage import _JsonOut |
| 106 | |
| 107 | hints = typing.get_type_hints(_JsonOut) |
| 108 | assert "exit_code" in hints, "exit_code missing from _JsonOut" |
| 109 | |
| 110 | def test_json_out_has_duration_ms(self) -> None: |
| 111 | import typing |
| 112 | from muse.cli.commands.semantic_test_coverage import _JsonOut |
| 113 | |
| 114 | hints = typing.get_type_hints(_JsonOut) |
| 115 | assert "duration_ms" in hints, "duration_ms missing from _JsonOut" |
| 116 | |
| 117 | def test_json_out_has_schema_version(self) -> None: |
| 118 | import typing |
| 119 | from muse.cli.commands.semantic_test_coverage import _JsonOut |
| 120 | |
| 121 | hints = typing.get_type_hints(_JsonOut) |
| 122 | assert "schema" in hints, "schema_version missing from _JsonOut" |
| 123 | |
| 124 | def test_json_out_retains_core_fields(self) -> None: |
| 125 | import typing |
| 126 | from muse.cli.commands.semantic_test_coverage import _JsonOut |
| 127 | |
| 128 | hints = typing.get_type_hints(_JsonOut) |
| 129 | required = {"ref", "snapshot_id", "depth", "transitive", "filters", "summary", "files"} |
| 130 | missing = required - set(hints) |
| 131 | assert not missing, f"Core fields missing from _JsonOut: {missing}" |
| 132 | |
| 133 | |
| 134 | # ────────────────────────────────────────────────────────────────────────────── |
| 135 | # Unit — -j alias registration |
| 136 | # ────────────────────────────────────────────────────────────────────────────── |
| 137 | |
| 138 | |
| 139 | class TestAliasRegistration: |
| 140 | def _make_parser(self): |
| 141 | import argparse |
| 142 | from muse.cli.commands.semantic_test_coverage import register |
| 143 | |
| 144 | p = argparse.ArgumentParser() |
| 145 | sub = p.add_subparsers() |
| 146 | register(sub) |
| 147 | return p |
| 148 | |
| 149 | def test_j_alias_sets_json_true(self) -> None: |
| 150 | p = self._make_parser() |
| 151 | ns = p.parse_args(["semantic-test-coverage", "-j"]) |
| 152 | assert ns.json_out is True |
| 153 | |
| 154 | def test_j_alias_and_other_flags_coexist(self) -> None: |
| 155 | p = self._make_parser() |
| 156 | ns = p.parse_args(["semantic-test-coverage", "-j", "--uncovered-only"]) |
| 157 | assert ns.json_out is True |
| 158 | assert ns.uncovered_only is True |
| 159 | |
| 160 | |
| 161 | # ────────────────────────────────────────────────────────────────────────────── |
| 162 | # Integration — -j alias |
| 163 | # ────────────────────────────────────────────────────────────────────────────── |
| 164 | |
| 165 | |
| 166 | class TestJsonAlias: |
| 167 | def test_j_alias_exit_code_zero(self, cov_repo: pathlib.Path) -> None: |
| 168 | result = _stc(cov_repo, "-j") |
| 169 | assert result.exit_code == 0 |
| 170 | |
| 171 | def test_j_alias_valid_json(self, cov_repo: pathlib.Path) -> None: |
| 172 | result = _stc(cov_repo, "-j") |
| 173 | data = json.loads(result.output) |
| 174 | assert isinstance(data, dict) |
| 175 | |
| 176 | def test_j_alias_same_top_level_keys_as_json_flag(self, cov_repo: pathlib.Path) -> None: |
| 177 | r_json = _stc(cov_repo, "--json") |
| 178 | r_j = _stc(cov_repo, "-j") |
| 179 | assert set(json.loads(r_json.output)) == set(json.loads(r_j.output)) |
| 180 | |
| 181 | def test_j_alias_summary_matches_json_flag(self, cov_repo: pathlib.Path) -> None: |
| 182 | r_json = _stc(cov_repo, "--json") |
| 183 | r_j = _stc(cov_repo, "-j") |
| 184 | assert json.loads(r_json.output)["summary"] == json.loads(r_j.output)["summary"] |
| 185 | |
| 186 | def test_j_alias_with_uncovered_only(self, cov_repo: pathlib.Path) -> None: |
| 187 | result = _stc(cov_repo, "-j", "--uncovered-only") |
| 188 | data = json.loads(result.output) |
| 189 | # All symbols in output should be uncovered. |
| 190 | for fc in data["files"]: |
| 191 | for sym in fc["symbols"]: |
| 192 | assert not sym["covered"] |
| 193 | |
| 194 | |
| 195 | # ────────────────────────────────────────────────────────────────────────────── |
| 196 | # Integration — JSON envelope: exit_code, duration_ms, schema_version |
| 197 | # ────────────────────────────────────────────────────────────────────────────── |
| 198 | |
| 199 | |
| 200 | class TestJsonEnvelope: |
| 201 | def test_has_exit_code(self, cov_repo: pathlib.Path) -> None: |
| 202 | result = _stc(cov_repo, "--json") |
| 203 | data = json.loads(result.output) |
| 204 | assert "exit_code" in data |
| 205 | |
| 206 | def test_exit_code_is_zero_on_success(self, cov_repo: pathlib.Path) -> None: |
| 207 | result = _stc(cov_repo, "--json") |
| 208 | data = json.loads(result.output) |
| 209 | assert data["exit_code"] == 0 |
| 210 | |
| 211 | def test_exit_code_is_int(self, cov_repo: pathlib.Path) -> None: |
| 212 | result = _stc(cov_repo, "--json") |
| 213 | data = json.loads(result.output) |
| 214 | assert isinstance(data["exit_code"], int) |
| 215 | |
| 216 | def test_has_duration_ms(self, cov_repo: pathlib.Path) -> None: |
| 217 | result = _stc(cov_repo, "--json") |
| 218 | data = json.loads(result.output) |
| 219 | assert "duration_ms" in data |
| 220 | |
| 221 | def test_duration_ms_is_nonnegative_float(self, cov_repo: pathlib.Path) -> None: |
| 222 | result = _stc(cov_repo, "--json") |
| 223 | data = json.loads(result.output) |
| 224 | assert isinstance(data["duration_ms"], float) |
| 225 | assert data["duration_ms"] >= 0.0 |
| 226 | |
| 227 | def test_has_schema_version(self, cov_repo: pathlib.Path) -> None: |
| 228 | result = _stc(cov_repo, "--json") |
| 229 | data = json.loads(result.output) |
| 230 | assert "schema" in data |
| 231 | |
| 232 | def test_schema_version_is_nonempty_string(self, cov_repo: pathlib.Path) -> None: |
| 233 | result = _stc(cov_repo, "--json") |
| 234 | data = json.loads(result.output) |
| 235 | assert isinstance(data["schema"], int) |
| 236 | assert data["schema"] > 0 |
| 237 | |
| 238 | def test_uncovered_only_json_still_has_envelope(self, cov_repo: pathlib.Path) -> None: |
| 239 | result = _stc(cov_repo, "--json", "--uncovered-only") |
| 240 | data = json.loads(result.output) |
| 241 | assert "exit_code" in data |
| 242 | assert "duration_ms" in data |
| 243 | assert "schema" in data |
| 244 | |
| 245 | def test_j_alias_has_exit_code(self, cov_repo: pathlib.Path) -> None: |
| 246 | result = _stc(cov_repo, "-j") |
| 247 | data = json.loads(result.output) |
| 248 | assert "exit_code" in data |
| 249 | |
| 250 | def test_j_alias_has_duration_ms(self, cov_repo: pathlib.Path) -> None: |
| 251 | result = _stc(cov_repo, "-j") |
| 252 | data = json.loads(result.output) |
| 253 | assert "duration_ms" in data |
| 254 | |
| 255 | def test_j_alias_has_schema_version(self, cov_repo: pathlib.Path) -> None: |
| 256 | result = _stc(cov_repo, "-j") |
| 257 | data = json.loads(result.output) |
| 258 | assert "schema" in data |
| 259 | |
| 260 | def test_min_coverage_violation_exit_code_one(self, cov_repo: pathlib.Path) -> None: |
| 261 | # generate_pdf is uncovered, so 100% threshold must fail. |
| 262 | result = _stc(cov_repo, "--min-coverage", "100") |
| 263 | assert result.exit_code == 1 |
| 264 | |
| 265 | def test_min_coverage_zero_exit_code_zero(self, cov_repo: pathlib.Path) -> None: |
| 266 | result = _stc(cov_repo, "--min-coverage", "0") |
| 267 | assert result.exit_code == 0 |
| 268 | |
| 269 | |
| 270 | # ────────────────────────────────────────────────────────────────────────────── |
| 271 | # Security — ANSI injection |
| 272 | # ────────────────────────────────────────────────────────────────────────────── |
| 273 | |
| 274 | |
| 275 | class TestSecurity: |
| 276 | def test_ansi_in_file_filter_not_echoed(self, cov_repo: pathlib.Path) -> None: |
| 277 | result = _stc(cov_repo, "--file", "\x1b[31mbilling\x1b[0m") |
| 278 | combined = result.output + (result.stderr or "") |
| 279 | assert "\x1b[31m" not in combined |
| 280 | |
| 281 | def test_null_byte_in_file_filter_no_crash(self, cov_repo: pathlib.Path) -> None: |
| 282 | # Should not crash — may return empty or error gracefully. |
| 283 | result = _stc(cov_repo, "--file", "billing\x00evil") |
| 284 | assert result.exit_code in (0, 1) |
| 285 | |
| 286 | |
| 287 | # ────────────────────────────────────────────────────────────────────────────── |
| 288 | # Performance |
| 289 | # ────────────────────────────────────────────────────────────────────────────── |
| 290 | |
| 291 | |
| 292 | class TestPerformance: |
| 293 | def test_duration_ms_under_5000(self, cov_repo: pathlib.Path) -> None: |
| 294 | result = _stc(cov_repo, "--json") |
| 295 | data = json.loads(result.output) |
| 296 | assert data["duration_ms"] < 5000.0 |
| 297 | |
| 298 | |
| 299 | # ────────────────────────────────────────────────────────────────────────────── |
| 300 | # Unit — docstrings |
| 301 | # ────────────────────────────────────────────────────────────────────────────── |
| 302 | |
| 303 | |
| 304 | class TestDocstrings: |
| 305 | def test_run_has_docstring(self) -> None: |
| 306 | from muse.cli.commands.semantic_test_coverage import run |
| 307 | |
| 308 | assert run.__doc__ and len(run.__doc__.strip()) > 30 |
| 309 | |
| 310 | def test_register_has_docstring(self) -> None: |
| 311 | from muse.cli.commands.semantic_test_coverage import register |
| 312 | |
| 313 | assert register.__doc__ and len(register.__doc__.strip()) > 20 |
| 314 | |
| 315 | def test_run_docstring_mentions_schema_version(self) -> None: |
| 316 | from muse.cli.commands.semantic_test_coverage import run |
| 317 | |
| 318 | doc = run.__doc__ or "" |
| 319 | assert "exit_code" in doc or "json" in doc.lower() |
| 320 | |
| 321 | def test_register_docstring_mentions_j_alias(self) -> None: |
| 322 | from muse.cli.commands.semantic_test_coverage import register |
| 323 | |
| 324 | doc = register.__doc__ or "" |
| 325 | assert "-j" in doc |
| 326 | |
| 327 | |
| 328 | class TestRegisterFlags: |
| 329 | def test_default_json_out_is_false(self): |
| 330 | import argparse |
| 331 | from muse.cli.commands.semantic_test_coverage import register |
| 332 | p = argparse.ArgumentParser() |
| 333 | subs = p.add_subparsers() |
| 334 | register(subs) |
| 335 | args = p.parse_args(["semantic-test-coverage"]) |
| 336 | assert args.json_out is False |
| 337 | |
| 338 | def test_json_flag_sets_json_out(self): |
| 339 | import argparse |
| 340 | from muse.cli.commands.semantic_test_coverage import register |
| 341 | p = argparse.ArgumentParser() |
| 342 | subs = p.add_subparsers() |
| 343 | register(subs) |
| 344 | args = p.parse_args(["semantic-test-coverage", "--json"]) |
| 345 | assert args.json_out is True |
| 346 | |
| 347 | def test_j_shorthand_sets_json_out(self): |
| 348 | import argparse |
| 349 | from muse.cli.commands.semantic_test_coverage import register |
| 350 | p = argparse.ArgumentParser() |
| 351 | subs = p.add_subparsers() |
| 352 | register(subs) |
| 353 | args = p.parse_args(["semantic-test-coverage", "-j"]) |
| 354 | assert args.json_out is True |
File History
2 commits
sha256:b636f72dcba9e190afb980bece906fa5b717fbde014b76ef023df8cb96e01eb9
docs: expand cache plan with all seven testing tiers and do…
Sonnet 4.6
131 days ago
sha256:7f9e2ef5286aedad9c1e6011b4c46ca27f39dbdad6e3409357e36b26e46b3b7c
docs: docstring sprint for-each-ref→hotspots — idiomatic ru…
Sonnet 4.6
patch
138 days ago