test_doc_inf_cli.py
python
sha256:27d91ce6191e753cb9f0dbd6b0b5d59dde18bc69fe10acc63bda269a0a0b238e
docs(mists): #257 Mist Domain QA contract v2 (replay of 510…
Opus 5.5
minor
⚠ breaking
8 days ago
| 1 | """DOC_INF_01 — every documented ``muse …`` command parses against the real muse CLI (musehub#242 W2.1). |
| 2 | |
| 3 | Each page's ``<pre>`` examples must parse completely; inline ``<code>`` references may omit |
| 4 | required arguments. Known findings are strict-xfailed against the page ticket that owns the |
| 5 | fix. The assertion message lists every offending command and the argparse reason. |
| 6 | ``TestCheckerDetects`` proves the checker reports each kind of defect, and |
| 7 | ``TestNeverExecutes`` proves that checking, destructive examples included, has no side effects. |
| 8 | """ |
| 9 | |
| 10 | from __future__ import annotations |
| 11 | |
| 12 | import argparse |
| 13 | import os |
| 14 | import shutil |
| 15 | import subprocess |
| 16 | from collections.abc import Iterator |
| 17 | from unittest.mock import patch |
| 18 | |
| 19 | import pytest |
| 20 | |
| 21 | from tests.docs_contract import cli_check as cc |
| 22 | from tests.docs_contract.pages import PAGES, xfail_known |
| 23 | |
| 24 | # Known findings, each fixed under its page ticket. None are open: every documented command parses. |
| 25 | _CLI_DEFECTS: set[str] = set() |
| 26 | |
| 27 | |
| 28 | @pytest.fixture(scope="module") |
| 29 | def parser() -> argparse.ArgumentParser: |
| 30 | return cc.capture_parser() |
| 31 | |
| 32 | |
| 33 | def _pre(line: str) -> cc.DocCommand: |
| 34 | return cc.DocCommand(line, in_pre=True) |
| 35 | |
| 36 | |
| 37 | # ── Live pages ────────────────────────────────────────────────────────────── |
| 38 | |
| 39 | |
| 40 | def test_extractor_finds_the_documented_commands(rendered: dict[str, str]) -> None: |
| 41 | counts = {p: len(cc.extract_commands(rendered[p])) for p in PAGES} |
| 42 | assert sum(counts.values()) > 400, counts # guards against the extractor silently matching nothing |
| 43 | |
| 44 | |
| 45 | @pytest.mark.parametrize("path", [xfail_known(p, _CLI_DEFECTS, "documented muse commands that do not parse") for p in PAGES]) |
| 46 | def test_documented_commands_parse(path: str, parser: argparse.ArgumentParser, rendered: dict[str, str]) -> None: |
| 47 | _, problems = cc.check_page(parser, path, rendered[path]) |
| 48 | assert problems == [], f"muse {cc.cli_version()}:\n" + "\n".join(problems) |
| 49 | |
| 50 | |
| 51 | # ── The checker detects real inconsistencies ──────────────────────────────── |
| 52 | |
| 53 | |
| 54 | class TestCheckerDetects: |
| 55 | @pytest.mark.parametrize("line", [ |
| 56 | 'muse commit -m "msg" --agent-id claude-code --model-id m --sign', |
| 57 | "muse -C <repo> status --json | jq '.branch'", |
| 58 | "muse log --oneline 2>&1 > out.txt # trailing comment", |
| 59 | "muse init [--domain code|midi|identity]", |
| 60 | "muse coord enqueue <title> --queue q --priority <n> --json", |
| 61 | "muse hub issue read $ISSUE --json", |
| 62 | "muse read $(muse rev-parse HEAD) --json", |
| 63 | "muse --version", |
| 64 | "muse hub issue --help", |
| 65 | ]) |
| 66 | def test_valid_commands_pass(self, parser: argparse.ArgumentParser, line: str) -> None: |
| 67 | assert cc.check_line(parser, _pre(line)).kind == "ok" |
| 68 | |
| 69 | def test_unknown_command_and_subcommand(self, parser: argparse.ArgumentParser) -> None: |
| 70 | assert cc.check_line(parser, _pre("muse api-surface --json")).kind == "unknown-command" |
| 71 | r = cc.check_line(parser, _pre("muse harmony policies --json")) |
| 72 | assert r.kind == "unknown-command" |
| 73 | assert "policy-list" in r.detail # muse's own "did you mean" hint reaches the diagnostic |
| 74 | |
| 75 | def test_unsupported_flag(self, parser: argparse.ArgumentParser) -> None: |
| 76 | r = cc.check_line(parser, _pre("muse shelf list --tags billing --json")) |
| 77 | assert (r.kind, r.detail) == ("unsupported-flag", "unrecognized arguments: --tags billing") |
| 78 | |
| 79 | def test_positional_arity(self, parser: argparse.ArgumentParser) -> None: |
| 80 | assert cc.check_line(parser, _pre('muse code find-symbol "Name" --json')).kind == "arity" |
| 81 | |
| 82 | def test_inline_reference_may_omit_arguments_but_pre_may_not(self, parser: argparse.ArgumentParser) -> None: |
| 83 | assert cc.check_line(parser, cc.DocCommand("muse merge --strategy", in_pre=False)).kind == "ok" |
| 84 | assert cc.check_line(parser, _pre("muse merge --strategy")).kind == "arity" |
| 85 | |
| 86 | def test_invalid_choice_value(self, parser: argparse.ArgumentParser) -> None: |
| 87 | r = cc.check_line(parser, _pre("muse coord watch --kind reservation --json")) |
| 88 | assert r.kind == "invalid-value" |
| 89 | assert "reservations" in r.detail |
| 90 | |
| 91 | @pytest.mark.parametrize(("line", "spelling"), [ |
| 92 | ("muse add src/app.py", "muse code add"), ("muse stash", "muse shelf save"), ("muse show HEAD", "muse read"), |
| 93 | ]) |
| 94 | def test_git_muscle_memory_is_named(self, parser: argparse.ArgumentParser, line: str, spelling: str) -> None: |
| 95 | r = cc.check_line(parser, _pre(line)) |
| 96 | assert (r.kind, r.gitism) == ("unknown-command", spelling) |
| 97 | |
| 98 | def test_diagnostic_names_page_and_command(self, parser: argparse.ArgumentParser) -> None: |
| 99 | html = "<pre>$ muse stash\n$ muse status</pre>" |
| 100 | _, problems = cc.check_page(parser, "/muse/x", html) |
| 101 | assert len(problems) == 1 |
| 102 | assert problems[0].startswith("/muse/x: `muse stash` → unknown-command: ") |
| 103 | assert problems[0].endswith("[git-ism → muse shelf save]") |
| 104 | |
| 105 | def test_extraction(self) -> None: |
| 106 | html = ( |
| 107 | "<pre><span>$</span> muse status \\\n --json\n# output\nmuse 0.2.1\n</pre>" |
| 108 | "<p>Run <code>muse merge --strategy</code>; muse commit, muse push all sign.</p>" |
| 109 | "<code>muse commit, muse push</code><pre>> muse log</pre>" |
| 110 | ) |
| 111 | assert cc.extract_commands(html) == [ |
| 112 | cc.DocCommand("muse status --json", in_pre=True), |
| 113 | cc.DocCommand("muse merge --strategy", in_pre=False), |
| 114 | cc.DocCommand("muse log", in_pre=True), |
| 115 | ] |
| 116 | |
| 117 | |
| 118 | # ── Checking never executes anything ──────────────────────────────────────── |
| 119 | |
| 120 | _DESTRUCTIVE = [ |
| 121 | "muse reset --hard HEAD~3", |
| 122 | "muse branch -D feat/x", |
| 123 | "muse -C / rm src/app.py", |
| 124 | "muse push origin feat/x --delete", |
| 125 | "muse checkout --theirs config.py", |
| 126 | "muse auth keygen --force --destroy-mnemonic", |
| 127 | ] |
| 128 | |
| 129 | |
| 130 | @pytest.fixture |
| 131 | def no_side_effects() -> Iterator[None]: |
| 132 | def _forbidden(*_: object, **__: object) -> None: |
| 133 | raise AssertionError("the docs CLI checker must never execute, chdir or delete") |
| 134 | |
| 135 | with ( |
| 136 | patch.object(os, "chdir", _forbidden), patch.object(os, "remove", _forbidden), |
| 137 | patch.object(os, "unlink", _forbidden), patch.object(shutil, "rmtree", _forbidden), |
| 138 | patch.object(subprocess, "Popen", _forbidden), |
| 139 | ): |
| 140 | yield |
| 141 | |
| 142 | |
| 143 | class TestNeverExecutes: |
| 144 | def test_parser_capture_dispatches_nothing(self, no_side_effects: None) -> None: |
| 145 | captured = cc.capture_parser() |
| 146 | assert {"commit", "reset", "rm"} <= set(next( |
| 147 | a for a in captured._actions if isinstance(a, argparse._SubParsersAction) |
| 148 | ).choices) |
| 149 | |
| 150 | @pytest.mark.parametrize("line", _DESTRUCTIVE) |
| 151 | def test_destructive_examples_are_classified_not_run( |
| 152 | self, parser: argparse.ArgumentParser, no_side_effects: None, line: str |
| 153 | ) -> None: |
| 154 | r = cc.check_line(parser, _pre(line)) |
| 155 | assert r.destructive, line |
| 156 | |
| 157 | @pytest.mark.parametrize("line", ["muse status", "muse branch feat/x", "muse push origin feat/x", "muse checkout -b feat/x"]) |
| 158 | def test_safe_examples_are_not_flagged_destructive(self, parser: argparse.ArgumentParser, line: str) -> None: |
| 159 | assert not cc.check_line(parser, _pre(line)).destructive |
File History
14 commits
sha256:27d91ce6191e753cb9f0dbd6b0b5d59dde18bc69fe10acc63bda269a0a0b238e
docs(mists): #257 Mist Domain QA contract v2 (replay of 510…
Opus 5.5
minor
⚠
8 days ago
sha256:af8b93688057d89c35730de054ab1ce0648f4c405e3ebb884729a1a8b23fcd76
docs(wire): #254 Wire Protocol QA contract v2 (replay of c0…
Opus 5.5
minor
⚠
9 days ago
sha256:417ffbc1d6761e3a4cd26ca5475d5a0b786ae8af936912b5ce31121e2e55c19a
docs(mcp): #252 MCP Tooling QA contract v2 — correct /muse/…
Opus 5.5
minor
⚠
9 days ago
sha256:395afcc81bdadd355e87f50a1eb01b857ffdcce78f54635a77fe893ca7352528
docs(coordination): #251 Agent Coordination QA contract v2 …
Opus 5.5
minor
⚠
9 days ago
sha256:4c51ee348c9f4848b67c2553ad2375f244c210f999589abb2898c80f7b43a8ea
docs(harmony): #250 Harmony QA contract v2 — correct /muse/…
Opus 5.5
minor
⚠
9 days ago
sha256:0914c802f6e5260fd54b28ecfb013b7815a76983780d5d2f854ef46c3ed36ab8
docs(merge): #249 Merge Engine QA contract v2 — correct /mu…
Sonnet 5.5
minor
⚠
9 days ago
sha256:e16f407d787df2320fb726a5ea8a5b5eb1d53e68135d8992373f0fd5b5319fa9
docs(intelligence): QA contract v2 — 148-claim audit, 20 co…
Sonnet 5
minor
⚠
9 days ago
sha256:7bef01440bfabe2977a6efa1a492091c4717e67d9ac79f5ebeb120d3fb87c3e0
docs(domains): QA contract v2 — executable minimal plugin, …
Opus 5.5
minor
⚠
9 days ago
sha256:634da27c5b4ea6e3cb5aa9cfb4e58dbf922b627caef67ffb0cb64bcaf70b7b9f
Merge task/246-identity-qa-v2 (#246 Cryptographic Identity …
Opus 5.5
minor
⚠
9 days ago
sha256:2de1e96b7f547494958a8edace1860efc2c0de0933c4f00f777f2cc18abaf324
docs(identity): QA contract v2 for /muse/identity against m…
Opus 5.5
minor
⚠
9 days ago
sha256:56adfc888ae758960c3b8cf64dccb458d15a59a20505c0a97dc372867cd91404
Merge task/253-shelves-qa-v2 (#253 Shelves QA contract v2) …
Opus 5.5
minor
⚠
9 days ago
sha256:aba9127573acf011a1da39099eb32f7b3eeeeb98719df179f5ca39bf70b51e4e
docs(shelves): QA contract v2 — engine-verified shelf contr…
Opus 5.5
minor
⚠
9 days ago
sha256:f7413c1abec6d6a8e5bdabcd4cc7ad5fcd059ce6a6f50f566e2b0118a0a8b578
tests: make tests/docs_contract/ database-free by directory…
Opus 5.5
10 days ago
sha256:0e4f9453dfdeb2c10fb31581a5b195aa7af7d349f90e5904bd38e1c1f7d19cc0
test(docs): check every documented muse command against the…
Opus 5.5
10 days ago