gabriel / musehub public
test_doc_db_isolation.py python
227 lines 10.5 KB
Raw
sha256:1dd276b0d6531bd2d11e9a083cfe57657caa58eff512d0e1bd6ac74d3d49cd79 test(docs): the two whole-docs-session isolation probes get… Opus 5.5 1 day ago
1 """Docs-contract database isolation (musehub#273): ``tests/docs_contract/`` is database-free by directory.
2
3 In-process tests drive the collection hook in ``tests/conftest.py`` with stand-in items. Subprocess
4 tests run real pytest sessions under ``tests.docs_contract.db_guard``, which refuses (and counts)
5 every ``create_engine`` and every ``.invalid`` address lookup, with a sandbox ``HOME`` and an
6 unresolvable ``TEST_DATABASE_URL``: no probe can reach a database, let alone reset a schema.
7 Every subprocess session ignores this file, so the probes never recurse.
8 """
9
10 from __future__ import annotations
11
12 import json
13 import os
14 import subprocess
15 import sys
16 from collections import Counter
17 from collections.abc import Iterable
18 from pathlib import Path
19 from typing import cast
20
21 import pytest
22
23 from tests import conftest as root_conftest
24 from tests.docs_contract import pages
25 from tests.docs_contract import test_doc_inf_scheme as scheme
26
27 ROOT = Path(__file__).resolve().parents[2]
28 _THIS = "tests/docs_contract/test_doc_db_isolation.py"
29 _DOCS = "tests/docs_contract/"
30 _LAND = "tests/docs_contract/test_doc_land.py::test_doc_land_06_reading_paths_use_registered_sections"
31 _REGISTRY = "tests/docs_contract/test_doc_inf_registry.py"
32 _SCHEME = "tests/docs_contract/test_doc_inf_scheme.py"
33 _DB_HOST = "db.docdb-273.invalid" # RFC 6761: never resolves
34 _TEST_DB = f"postgresql+asyncpg://musehub:musehub@{_DB_HOST}:5432/musehub_test"
35 _NOT_A_TEST_DB = "postgresql+asyncpg://musehub:[email protected]:5432/musehub"
36 _AUTOUSE = ("_db_schema", "_tmp_objects_dir", "reset_rate_limiter", "_reset_variation_store")
37
38
39 # ── In-process: the collection hook ─────────────────────────────────────────
40
41
42 class _Item:
43 """The two attributes the hook reads, plus ``add_marker``."""
44
45 def __init__(self, nodeid: str, fixturenames: Iterable[str]) -> None:
46 self.nodeid = nodeid
47 self.fixturenames = [*_AUTOUSE, *fixturenames]
48 self.marks: list[str] = []
49
50 def add_marker(self, marker: pytest.MarkDecorator) -> None:
51 self.marks.append(marker.name)
52
53
54 def _classified_no_db(nodeid: str, fixturenames: Iterable[str]) -> bool:
55 item = _Item(nodeid, fixturenames)
56 root_conftest.pytest_collection_modifyitems(cast(pytest.Config, None), cast(list[pytest.Item], [item]))
57 return "no_db" in item.marks
58
59
60 def test_docdb_01_unmarked_docs_module_is_no_db_by_directory() -> None:
61 assert _classified_no_db("tests/docs_contract/test_new.py::test_x", ["rendered"])
62 assert _classified_no_db("tests/docs_contract/test_new.py::TestX::test_y[param]", [])
63
64
65 @pytest.mark.parametrize(
66 "nodeid",
67 ["tests/docs_contract/test_new.py::test_x", "tests/test_release_gate.py::test_x", "tests/install_e2e/test_new.py::test_x"],
68 )
69 @pytest.mark.parametrize(
70 "closure",
71 [["db_session"], ["session_factory"], ["client", "db_session"], ["test_user", "db_session", "session_factory"]],
72 ids=["db_session", "session_factory", "indirect_client", "both"],
73 )
74 def test_docdb_02_explicit_db_fixtures_are_never_no_db(nodeid: str, closure: list[str]) -> None:
75 # pytest's fixture closure is transitive, so an indirect request (client → db_session) is visible here.
76 assert not _classified_no_db(nodeid, closure)
77
78
79 @pytest.mark.parametrize(
80 "nodeid",
81 [
82 "tests/install_e2e/test_x.py::test_x",
83 "tests/test_release_gate.py::test_x",
84 "tests/test_prune_releases.py::test_x",
85 "tests/test_build_context.py::test_x",
86 ],
87 )
88 def test_docdb_05_existing_no_db_prefixes_keep_their_classification(nodeid: str) -> None:
89 assert _classified_no_db(nodeid, ["tmp_path"])
90
91
92 def test_docdb_05_paths_outside_the_prefixes_are_not_auto_marked() -> None:
93 assert not _classified_no_db("tests/test_api_repos.py::test_x", ["tmp_path"])
94 assert not _classified_no_db("tests/docs_contract_extra/test_x.py::test_x", [])
95 assert not _classified_no_db("tests/unit/test_docs_contract.py::test_x", [])
96
97
98 def test_docdb_04_docs_xfails_only_absorb_assertion_failures() -> None:
99 param = cast(pytest.ParameterSet, pages.xfail_known(pages.PAGES[0], {pages.PAGES[0]}, "probe"))
100 (mark,) = param.marks
101 assert mark.kwargs.get("raises") is AssertionError
102 assert mark.kwargs.get("strict") is True
103 assert scheme._HTTP_ON_HTTPS.kwargs.get("raises") is AssertionError
104 assert scheme._HTTP_ON_HTTPS.kwargs.get("strict") is True
105
106
107 # ── Subprocess: real sessions under the fail-closed guard ───────────────────
108
109
110 def _session(
111 tmp_path: Path, *args: str, db_url: str = _TEST_DB, strip: bool = False, inject: str = "", collect_only: bool = False
112 ) -> dict[str, object]:
113 """Run one guarded pytest session; return the guard's report plus the exit code and output tail."""
114 report = tmp_path / f"report-{len(list(tmp_path.glob('report-*')))}.json"
115 home = tmp_path / "home"
116 home.mkdir(exist_ok=True)
117 env = {
118 "PATH": os.environ.get("PATH", "/usr/bin:/bin"),
119 "HOME": str(home),
120 "TEST_DATABASE_URL": db_url,
121 "DATABASE_URL": db_url,
122 "PYTHONDONTWRITEBYTECODE": "1",
123 "DOCDB_REPORT": str(report),
124 }
125 if os.environ.get("PYTHONPATH"):
126 env["PYTHONPATH"] = os.environ["PYTHONPATH"]
127 if strip:
128 env["DOCDB_STRIP_NO_DB"] = "1"
129 if inject:
130 env["DOCDB_INJECT"] = inject
131 cmd = [sys.executable, "-m", "pytest", "-p", "tests.docs_contract.db_guard", "-p", "no:cacheprovider", "-p", "no:randomly"]
132 cmd += ["-q", "-o", "addopts=", "--ignore", _THIS, *(["--collect-only"] if collect_only else []), *args]
133 # Each nested session reruns the whole docs suite (minus this file). The suite outgrew 300 s when #250/#251
134 # added real-CLI sandbox tests (348 s alone on 2026-11-07); 900 s leaves headroom without hiding a hang.
135 proc = subprocess.run(cmd, cwd=ROOT, env=env, capture_output=True, text=True, timeout=900)
136 assert report.exists(), f"guard wrote no report (exit {proc.returncode}):\n{proc.stdout[-3000:]}{proc.stderr[-3000:]}"
137 result: dict[str, object] = json.loads(report.read_text())
138 result["returncode"] = proc.returncode
139 result["output"] = proc.stdout[-3000:]
140 return result
141
142
143 def _outcomes(result: dict[str, object]) -> dict[str, str]:
144 return cast(dict[str, str], result["outcomes"])
145
146
147 def _no_db(result: dict[str, object]) -> dict[str, bool]:
148 return cast(dict[str, bool], result["no_db"])
149
150
151 def _assert_clean_docs_run(result: dict[str, object]) -> None:
152 counts = Counter(_outcomes(result).values())
153 assert result["engine_attempts"] == [] and result["socket_attempts"] == [], result["output"]
154 assert set(counts) <= {"passed", "xfailed"} and counts["passed"] > 0, (counts, result["output"])
155 assert counts["xfailed"] > 0, "known findings must stay strict-xfailed"
156 assert result["returncode"] == 0, result["output"]
157
158
159 def test_docdb_01_docs_session_without_module_markers_never_creates_an_engine(tmp_path: Path) -> None:
160 result = _session(tmp_path, _DOCS, strip=True)
161 assert _no_db(result) and all(_no_db(result).values())
162 _assert_clean_docs_run(result)
163
164
165 @pytest.mark.parametrize("fixture", ["db_session", "session_factory"])
166 def test_docdb_02_docs_test_requesting_a_db_fixture_reaches_db_schema(tmp_path: Path, fixture: str) -> None:
167 result = _session(tmp_path, _LAND, inject=f"{fixture}={_LAND}")
168 assert _no_db(result) == {_LAND: False}
169 assert result["engine_attempts"] == [_DB_HOST], "exactly one blocked _db_schema engine attempt"
170 assert result["socket_attempts"] == [], "no connection attempted"
171 assert _outcomes(result) == {_LAND: "error"}
172
173
174 def test_docdb_03_docs_session_ignores_a_target_that_is_not_a_test_database(tmp_path: Path) -> None:
175 # Rejecting such a target belongs to #263 (TISO_02/TISO_09); a docs-only session must just never reach it.
176 result = _session(tmp_path, _DOCS, db_url=_NOT_A_TEST_DB)
177 assert all(_no_db(result).values())
178 _assert_clean_docs_run(result)
179
180
181 def test_docdb_04_failed_db_setup_is_an_error_not_an_expected_failure(tmp_path: Path) -> None:
182 baseline = _session(tmp_path, _REGISTRY, _SCHEME)
183 xfails = {n for n, o in _outcomes(baseline).items() if o == "xfailed"}
184 assert {n.split("::")[0] for n in xfails} == {_REGISTRY, _SCHEME}, "both docs xfail helpers are exercised"
185 db_item = next(n for n, o in _outcomes(baseline).items() if o == "passed" and n.startswith(_REGISTRY))
186 # The ticket's scenario: no module markers, and one DB-backed docs test whose session setup fails.
187 broken = _session(tmp_path, _REGISTRY, _SCHEME, strip=True, inject=f"db_session={db_item}")
188 assert broken["engine_attempts"] == [_DB_HOST] and broken["socket_attempts"] == []
189 outcomes = _outcomes(broken)
190 assert {n: outcomes[n] for n in xfails} == dict.fromkeys(xfails, "error"), "a setup failure must not read as XFAIL"
191 assert set(outcomes.values()) == {"error"}
192
193
194 @pytest.mark.parametrize(
195 "selection",
196 [
197 [_DOCS],
198 ["tests/docs_contract/test_doc_gs.py"],
199 [_LAND],
200 [_DOCS, "-k", "https or landing"],
201 ],
202 ids=["directory", "module", "node_id", "keyword"],
203 )
204 def test_docdb_05_every_selection_classifies_docs_items_no_db(tmp_path: Path, selection: list[str]) -> None:
205 result = _session(tmp_path, *selection, strip=True, collect_only=True)
206 assert result["returncode"] == 0, result["output"]
207 assert _no_db(result) and all(_no_db(result).values()), {n for n, v in _no_db(result).items() if not v}
208 assert result["engine_attempts"] == []
209
210
211 def test_docdb_05_existing_prefix_suites_keep_their_classification(tmp_path: Path) -> None:
212 suites = ["tests/test_prune_releases.py", "tests/test_build_context.py", "tests/test_release_gate.py"]
213 result = _session(tmp_path, *suites, "tests/test_release_analysis_no_commit_meta.py", collect_only=True)
214 assert result["returncode"] == 0, result["output"]
215 closure = cast(dict[str, list[str]], result["db_closure"])
216 assert {n: v for n, v in _no_db(result).items()} == {n: not closure[n] for n in closure}
217 assert any(closure.values()) and not all(closure.values()), "the sample holds both DB-backed and pure tests"
218
219
220 def test_docdb_05_mixed_session_with_a_db_backed_test_still_runs_db_schema(tmp_path: Path) -> None:
221 db_test = "tests/test_users_no_commit_meta.py"
222 result = _session(tmp_path, _LAND, db_test)
223 no_db = _no_db(result)
224 assert no_db[_LAND] is True
225 assert [v for n, v in no_db.items() if n.startswith(db_test)] == [False]
226 assert result["engine_attempts"] == [_DB_HOST] and result["socket_attempts"] == []
227 assert set(_outcomes(result).values()) == {"error"}
File History 6 commits
sha256:1dd276b0d6531bd2d11e9a083cfe57657caa58eff512d0e1bd6ac74d3d49cd79 test(docs): the two whole-docs-session isolation probes get… Opus 5.5 1 day ago
sha256:c2ee54efa85fcbdeea4ffc84a2dd1b8fcd4623463b22e35765df3013a26834aa merge: #276 public URLs + muse#181 docs onto dev (#313 present) Opus 5.5 minor ⚠ 1 day ago
sha256:c777acd5605d4a83bbf9d2903bff0b0d419325b7bc49317700e4037b56594e8e fix(web): absolute public URLs come from the canonical publ… Opus 5.5 minor ⚠ 1 day ago
sha256:7513f245748d6667481e0ecef54e261ad11d4a664ba7128527060b240f4707bc docs(security): #260 Security Reference QA contract v2 — co… Sonnet 5.5 minor ⚠ 8 days ago
sha256:2978eb11903d1a9d08902c41c8bb6cb874b470c19e8df579ebb66d033dbb8bdd test(docs_contract): docdb nested-session timeout 300 -> 900 s Opus 5.5 9 days ago
sha256:f7413c1abec6d6a8e5bdabcd4cc7ad5fcd059ce6a6f50f566e2b0118a0a8b578 tests: make tests/docs_contract/ database-free by directory… Opus 5.5 10 days ago