gabriel / musehub public
test_doc_db_isolation.py python
227 lines 10.7 KB
Raw
sha256:1dd276b0d6531bd2d11e9a083cfe57657caa58eff512d0e1bd6ac74d3d49cd79 test(docs): the two whole-docs-session isolation probes get… Opus 5.5 1 day ago
1 """Docs-contract database isolation (musehub#273): ``tests/docs_contract/`` is database-free by directory.
2
3 In-process tests drive the collection hook in ``tests/conftest.py`` with stand-in items. Subprocess
4 tests run real pytest sessions under ``tests.docs_contract.db_guard``, which refuses (and counts)
5 every ``create_engine`` and every ``.invalid`` address lookup, with a sandbox ``HOME`` and an
6 unresolvable ``TEST_DATABASE_URL``: no probe can reach a database, let alone reset a schema.
7 Every subprocess session ignores this file, so the probes never recurse.
8 """
9
10 from __future__ import annotations
11
12 import json
13 import os
14 import subprocess
15 import sys
16 from collections import Counter
17 from collections.abc import Iterable
18 from pathlib import Path
19 from typing import cast
20
21 import pytest
22
23 from tests import conftest as root_conftest
24 from tests.docs_contract import pages
25
26 ROOT = Path(__file__).resolve().parents[2]
27 _THIS = "tests/docs_contract/test_doc_db_isolation.py"
28 _DOCS = "tests/docs_contract/"
29 _LAND = "tests/docs_contract/test_doc_land.py::test_doc_land_06_reading_paths_use_registered_sections"
30 _REGISTRY = "tests/docs_contract/test_doc_inf_registry.py"
31 # Offline modules that still carry strict xfails for open, deferred tickets (muse#215, the Mists follow-ups); any one suffices.
32 _XFAIL_SOURCES = ("tests/docs_contract/test_security.py", "tests/docs_contract/test_mists.py")
33 _DB_HOST = "db.docdb-273.invalid" # RFC 6761: never resolves
34 _TEST_DB = f"postgresql+asyncpg://musehub:musehub@{_DB_HOST}:5432/musehub_test"
35 _NOT_A_TEST_DB = "postgresql+asyncpg://musehub:[email protected]:5432/musehub"
36 _AUTOUSE = ("_db_schema", "_tmp_objects_dir", "reset_rate_limiter", "_reset_variation_store")
37
38
39 # ── In-process: the collection hook ─────────────────────────────────────────
40
41
42 class _Item:
43 """The two attributes the hook reads, plus ``add_marker``."""
44
45 def __init__(self, nodeid: str, fixturenames: Iterable[str]) -> None:
46 self.nodeid = nodeid
47 self.fixturenames = [*_AUTOUSE, *fixturenames]
48 self.marks: list[str] = []
49
50 def add_marker(self, marker: pytest.MarkDecorator) -> None:
51 self.marks.append(marker.name)
52
53
54 def _classified_no_db(nodeid: str, fixturenames: Iterable[str]) -> bool:
55 item = _Item(nodeid, fixturenames)
56 root_conftest.pytest_collection_modifyitems(cast(pytest.Config, None), cast(list[pytest.Item], [item]))
57 return "no_db" in item.marks
58
59
60 def test_docdb_01_unmarked_docs_module_is_no_db_by_directory() -> None:
61 assert _classified_no_db("tests/docs_contract/test_new.py::test_x", ["rendered"])
62 assert _classified_no_db("tests/docs_contract/test_new.py::TestX::test_y[param]", [])
63
64
65 @pytest.mark.parametrize(
66 "nodeid",
67 ["tests/docs_contract/test_new.py::test_x", "tests/test_release_gate.py::test_x", "tests/install_e2e/test_new.py::test_x"],
68 )
69 @pytest.mark.parametrize(
70 "closure",
71 [["db_session"], ["session_factory"], ["client", "db_session"], ["test_user", "db_session", "session_factory"]],
72 ids=["db_session", "session_factory", "indirect_client", "both"],
73 )
74 def test_docdb_02_explicit_db_fixtures_are_never_no_db(nodeid: str, closure: list[str]) -> None:
75 # pytest's fixture closure is transitive, so an indirect request (client → db_session) is visible here.
76 assert not _classified_no_db(nodeid, closure)
77
78
79 @pytest.mark.parametrize(
80 "nodeid",
81 [
82 "tests/install_e2e/test_x.py::test_x",
83 "tests/test_release_gate.py::test_x",
84 "tests/test_prune_releases.py::test_x",
85 "tests/test_build_context.py::test_x",
86 ],
87 )
88 def test_docdb_05_existing_no_db_prefixes_keep_their_classification(nodeid: str) -> None:
89 assert _classified_no_db(nodeid, ["tmp_path"])
90
91
92 def test_docdb_05_paths_outside_the_prefixes_are_not_auto_marked() -> None:
93 assert not _classified_no_db("tests/test_api_repos.py::test_x", ["tmp_path"])
94 assert not _classified_no_db("tests/docs_contract_extra/test_x.py::test_x", [])
95 assert not _classified_no_db("tests/unit/test_docs_contract.py::test_x", [])
96
97
98 def test_docdb_04_docs_xfails_only_absorb_assertion_failures() -> None:
99 param = cast(pytest.ParameterSet, pages.xfail_known(pages.PAGES[0], {pages.PAGES[0]}, "probe"))
100 (mark,) = param.marks
101 assert mark.kwargs.get("raises") is AssertionError
102 assert mark.kwargs.get("strict") is True
103
104
105 # ── Subprocess: real sessions under the fail-closed guard ───────────────────
106
107
108 def _session(
109 tmp_path: Path, *args: str, db_url: str = _TEST_DB, strip: bool = False, inject: str = "", collect_only: bool = False
110 ) -> dict[str, object]:
111 """Run one guarded pytest session; return the guard's report plus the exit code and output tail."""
112 report = tmp_path / f"report-{len(list(tmp_path.glob('report-*')))}.json"
113 home = tmp_path / "home"
114 home.mkdir(exist_ok=True)
115 env = {
116 "PATH": os.environ.get("PATH", "/usr/bin:/bin"),
117 "HOME": str(home),
118 "TEST_DATABASE_URL": db_url,
119 "DATABASE_URL": db_url,
120 "PYTHONDONTWRITEBYTECODE": "1",
121 "DOCDB_REPORT": str(report),
122 }
123 if os.environ.get("PYTHONPATH"):
124 env["PYTHONPATH"] = os.environ["PYTHONPATH"]
125 if strip:
126 env["DOCDB_STRIP_NO_DB"] = "1"
127 if inject:
128 env["DOCDB_INJECT"] = inject
129 cmd = [sys.executable, "-m", "pytest", "-p", "tests.docs_contract.db_guard", "-p", "no:cacheprovider", "-p", "no:randomly"]
130 cmd += ["-q", "-o", "addopts=", "--ignore", _THIS, *(["--collect-only"] if collect_only else []), *args]
131 # Each nested session reruns the whole docs suite (minus this file). The suite outgrew 300 s when #250/#251
132 # added real-CLI sandbox tests (348 s alone on 2026-11-07); 900 s leaves headroom without hiding a hang.
133 proc = subprocess.run(cmd, cwd=ROOT, env=env, capture_output=True, text=True, timeout=900)
134 assert report.exists(), f"guard wrote no report (exit {proc.returncode}):\n{proc.stdout[-3000:]}{proc.stderr[-3000:]}"
135 result: dict[str, object] = json.loads(report.read_text())
136 result["returncode"] = proc.returncode
137 result["output"] = proc.stdout[-3000:]
138 return result
139
140
141 def _outcomes(result: dict[str, object]) -> dict[str, str]:
142 return cast(dict[str, str], result["outcomes"])
143
144
145 def _no_db(result: dict[str, object]) -> dict[str, bool]:
146 return cast(dict[str, bool], result["no_db"])
147
148
149 def _assert_clean_docs_run(result: dict[str, object]) -> None:
150 counts = Counter(_outcomes(result).values())
151 assert result["engine_attempts"] == [] and result["socket_attempts"] == [], result["output"]
152 assert set(counts) <= {"passed", "xfailed"} and counts["passed"] > 0, (counts, result["output"])
153 assert counts["xfailed"] > 0, "known findings must stay strict-xfailed"
154 assert result["returncode"] == 0, result["output"]
155
156
157 def test_docdb_01_docs_session_without_module_markers_never_creates_an_engine(tmp_path: Path) -> None:
158 result = _session(tmp_path, _DOCS, strip=True)
159 assert _no_db(result) and all(_no_db(result).values())
160 _assert_clean_docs_run(result)
161
162
163 @pytest.mark.parametrize("fixture", ["db_session", "session_factory"])
164 def test_docdb_02_docs_test_requesting_a_db_fixture_reaches_db_schema(tmp_path: Path, fixture: str) -> None:
165 result = _session(tmp_path, _LAND, inject=f"{fixture}={_LAND}")
166 assert _no_db(result) == {_LAND: False}
167 assert result["engine_attempts"] == [_DB_HOST], "exactly one blocked _db_schema engine attempt"
168 assert result["socket_attempts"] == [], "no connection attempted"
169 assert _outcomes(result) == {_LAND: "error"}
170
171
172 def test_docdb_03_docs_session_ignores_a_target_that_is_not_a_test_database(tmp_path: Path) -> None:
173 # Rejecting such a target belongs to #263 (TISO_02/TISO_09); a docs-only session must just never reach it.
174 result = _session(tmp_path, _DOCS, db_url=_NOT_A_TEST_DB)
175 assert all(_no_db(result).values())
176 _assert_clean_docs_run(result)
177
178
179 def test_docdb_04_failed_db_setup_is_an_error_not_an_expected_failure(tmp_path: Path) -> None:
180 baseline = _session(tmp_path, _REGISTRY, *_XFAIL_SOURCES)
181 xfails = {n for n, o in _outcomes(baseline).items() if o == "xfailed"}
182 # The registry module has no open findings left (#260, and the scheme module's went with #276), so the expected failures come
183 # from the sources; the helper itself is pinned by test_docdb_04_docs_xfails_only_absorb_assertion_failures.
184 assert xfails and {n.split("::")[0] for n in xfails} <= set(_XFAIL_SOURCES), "strict xfails are exercised"
185 db_item = next(n for n, o in _outcomes(baseline).items() if o == "passed" and n.startswith(_REGISTRY))
186 # The ticket's scenario: no module markers, and one DB-backed docs test whose session setup fails.
187 broken = _session(tmp_path, _REGISTRY, *_XFAIL_SOURCES, strip=True, inject=f"db_session={db_item}")
188 assert broken["engine_attempts"] == [_DB_HOST] and broken["socket_attempts"] == []
189 outcomes = _outcomes(broken)
190 assert {n: outcomes[n] for n in xfails} == dict.fromkeys(xfails, "error"), "a setup failure must not read as XFAIL"
191 assert set(outcomes.values()) == {"error"}
192
193
194 @pytest.mark.parametrize(
195 "selection",
196 [
197 [_DOCS],
198 ["tests/docs_contract/test_doc_gs.py"],
199 [_LAND],
200 [_DOCS, "-k", "https or landing"],
201 ],
202 ids=["directory", "module", "node_id", "keyword"],
203 )
204 def test_docdb_05_every_selection_classifies_docs_items_no_db(tmp_path: Path, selection: list[str]) -> None:
205 result = _session(tmp_path, *selection, strip=True, collect_only=True)
206 assert result["returncode"] == 0, result["output"]
207 assert _no_db(result) and all(_no_db(result).values()), {n for n, v in _no_db(result).items() if not v}
208 assert result["engine_attempts"] == []
209
210
211 def test_docdb_05_existing_prefix_suites_keep_their_classification(tmp_path: Path) -> None:
212 suites = ["tests/test_prune_releases.py", "tests/test_build_context.py", "tests/test_release_gate.py"]
213 result = _session(tmp_path, *suites, "tests/test_release_analysis_no_commit_meta.py", collect_only=True)
214 assert result["returncode"] == 0, result["output"]
215 closure = cast(dict[str, list[str]], result["db_closure"])
216 assert {n: v for n, v in _no_db(result).items()} == {n: not closure[n] for n in closure}
217 assert any(closure.values()) and not all(closure.values()), "the sample holds both DB-backed and pure tests"
218
219
220 def test_docdb_05_mixed_session_with_a_db_backed_test_still_runs_db_schema(tmp_path: Path) -> None:
221 db_test = "tests/test_users_no_commit_meta.py"
222 result = _session(tmp_path, _LAND, db_test)
223 no_db = _no_db(result)
224 assert no_db[_LAND] is True
225 assert [v for n, v in no_db.items() if n.startswith(db_test)] == [False]
226 assert result["engine_attempts"] == [_DB_HOST] and result["socket_attempts"] == []
227 assert set(_outcomes(result).values()) == {"error"}
File History 6 commits
sha256:1dd276b0d6531bd2d11e9a083cfe57657caa58eff512d0e1bd6ac74d3d49cd79 test(docs): the two whole-docs-session isolation probes get… Opus 5.5 1 day ago
sha256:c2ee54efa85fcbdeea4ffc84a2dd1b8fcd4623463b22e35765df3013a26834aa merge: #276 public URLs + muse#181 docs onto dev (#313 present) Opus 5.5 minor ⚠ 2 days ago
sha256:c777acd5605d4a83bbf9d2903bff0b0d419325b7bc49317700e4037b56594e8e fix(web): absolute public URLs come from the canonical publ… Opus 5.5 minor ⚠ 2 days ago
sha256:7513f245748d6667481e0ecef54e261ad11d4a664ba7128527060b240f4707bc docs(security): #260 Security Reference QA contract v2 — co… Sonnet 5.5 minor ⚠ 8 days ago
sha256:2978eb11903d1a9d08902c41c8bb6cb874b470c19e8df579ebb66d033dbb8bdd test(docs_contract): docdb nested-session timeout 300 -> 900 s Opus 5.5 9 days ago
sha256:f7413c1abec6d6a8e5bdabcd4cc7ad5fcd059ce6a6f50f566e2b0118a0a8b578 tests: make tests/docs_contract/ database-free by directory… Opus 5.5 10 days ago