test_doc_db_isolation.py
python
sha256:1dd276b0d6531bd2d11e9a083cfe57657caa58eff512d0e1bd6ac74d3d49cd79
test(docs): the two whole-docs-session isolation probes get…
Opus 5.5
1 day ago
| 1 | """Docs-contract database isolation (musehub#273): ``tests/docs_contract/`` is database-free by directory. |
| 2 | |
| 3 | In-process tests drive the collection hook in ``tests/conftest.py`` with stand-in items. Subprocess |
| 4 | tests run real pytest sessions under ``tests.docs_contract.db_guard``, which refuses (and counts) |
| 5 | every ``create_engine`` and every ``.invalid`` address lookup, with a sandbox ``HOME`` and an |
| 6 | unresolvable ``TEST_DATABASE_URL``: no probe can reach a database, let alone reset a schema. |
| 7 | Every subprocess session ignores this file, so the probes never recurse. |
| 8 | """ |
| 9 | |
| 10 | from __future__ import annotations |
| 11 | |
| 12 | import json |
| 13 | import os |
| 14 | import subprocess |
| 15 | import sys |
| 16 | from collections import Counter |
| 17 | from collections.abc import Iterable |
| 18 | from pathlib import Path |
| 19 | from typing import cast |
| 20 | |
| 21 | import pytest |
| 22 | |
| 23 | from tests import conftest as root_conftest |
| 24 | from tests.docs_contract import pages |
| 25 | from tests.docs_contract import test_doc_inf_scheme as scheme |
| 26 | |
| 27 | ROOT = Path(__file__).resolve().parents[2] |
| 28 | _THIS = "tests/docs_contract/test_doc_db_isolation.py" |
| 29 | _DOCS = "tests/docs_contract/" |
| 30 | _LAND = "tests/docs_contract/test_doc_land.py::test_doc_land_06_reading_paths_use_registered_sections" |
| 31 | _REGISTRY = "tests/docs_contract/test_doc_inf_registry.py" |
| 32 | _SCHEME = "tests/docs_contract/test_doc_inf_scheme.py" |
| 33 | _DB_HOST = "db.docdb-273.invalid" # RFC 6761: never resolves |
| 34 | _TEST_DB = f"postgresql+asyncpg://musehub:musehub@{_DB_HOST}:5432/musehub_test" |
| 35 | _NOT_A_TEST_DB = "postgresql+asyncpg://musehub:[email protected]:5432/musehub" |
| 36 | _AUTOUSE = ("_db_schema", "_tmp_objects_dir", "reset_rate_limiter", "_reset_variation_store") |
| 37 | |
| 38 | |
| 39 | # ── In-process: the collection hook ───────────────────────────────────────── |
| 40 | |
| 41 | |
| 42 | class _Item: |
| 43 | """The two attributes the hook reads, plus ``add_marker``.""" |
| 44 | |
| 45 | def __init__(self, nodeid: str, fixturenames: Iterable[str]) -> None: |
| 46 | self.nodeid = nodeid |
| 47 | self.fixturenames = [*_AUTOUSE, *fixturenames] |
| 48 | self.marks: list[str] = [] |
| 49 | |
| 50 | def add_marker(self, marker: pytest.MarkDecorator) -> None: |
| 51 | self.marks.append(marker.name) |
| 52 | |
| 53 | |
| 54 | def _classified_no_db(nodeid: str, fixturenames: Iterable[str]) -> bool: |
| 55 | item = _Item(nodeid, fixturenames) |
| 56 | root_conftest.pytest_collection_modifyitems(cast(pytest.Config, None), cast(list[pytest.Item], [item])) |
| 57 | return "no_db" in item.marks |
| 58 | |
| 59 | |
| 60 | def test_docdb_01_unmarked_docs_module_is_no_db_by_directory() -> None: |
| 61 | assert _classified_no_db("tests/docs_contract/test_new.py::test_x", ["rendered"]) |
| 62 | assert _classified_no_db("tests/docs_contract/test_new.py::TestX::test_y[param]", []) |
| 63 | |
| 64 | |
| 65 | @pytest.mark.parametrize( |
| 66 | "nodeid", |
| 67 | ["tests/docs_contract/test_new.py::test_x", "tests/test_release_gate.py::test_x", "tests/install_e2e/test_new.py::test_x"], |
| 68 | ) |
| 69 | @pytest.mark.parametrize( |
| 70 | "closure", |
| 71 | [["db_session"], ["session_factory"], ["client", "db_session"], ["test_user", "db_session", "session_factory"]], |
| 72 | ids=["db_session", "session_factory", "indirect_client", "both"], |
| 73 | ) |
| 74 | def test_docdb_02_explicit_db_fixtures_are_never_no_db(nodeid: str, closure: list[str]) -> None: |
| 75 | # pytest's fixture closure is transitive, so an indirect request (client → db_session) is visible here. |
| 76 | assert not _classified_no_db(nodeid, closure) |
| 77 | |
| 78 | |
| 79 | @pytest.mark.parametrize( |
| 80 | "nodeid", |
| 81 | [ |
| 82 | "tests/install_e2e/test_x.py::test_x", |
| 83 | "tests/test_release_gate.py::test_x", |
| 84 | "tests/test_prune_releases.py::test_x", |
| 85 | "tests/test_build_context.py::test_x", |
| 86 | ], |
| 87 | ) |
| 88 | def test_docdb_05_existing_no_db_prefixes_keep_their_classification(nodeid: str) -> None: |
| 89 | assert _classified_no_db(nodeid, ["tmp_path"]) |
| 90 | |
| 91 | |
| 92 | def test_docdb_05_paths_outside_the_prefixes_are_not_auto_marked() -> None: |
| 93 | assert not _classified_no_db("tests/test_api_repos.py::test_x", ["tmp_path"]) |
| 94 | assert not _classified_no_db("tests/docs_contract_extra/test_x.py::test_x", []) |
| 95 | assert not _classified_no_db("tests/unit/test_docs_contract.py::test_x", []) |
| 96 | |
| 97 | |
| 98 | def test_docdb_04_docs_xfails_only_absorb_assertion_failures() -> None: |
| 99 | param = cast(pytest.ParameterSet, pages.xfail_known(pages.PAGES[0], {pages.PAGES[0]}, "probe")) |
| 100 | (mark,) = param.marks |
| 101 | assert mark.kwargs.get("raises") is AssertionError |
| 102 | assert mark.kwargs.get("strict") is True |
| 103 | assert scheme._HTTP_ON_HTTPS.kwargs.get("raises") is AssertionError |
| 104 | assert scheme._HTTP_ON_HTTPS.kwargs.get("strict") is True |
| 105 | |
| 106 | |
| 107 | # ── Subprocess: real sessions under the fail-closed guard ─────────────────── |
| 108 | |
| 109 | |
| 110 | def _session( |
| 111 | tmp_path: Path, *args: str, db_url: str = _TEST_DB, strip: bool = False, inject: str = "", collect_only: bool = False |
| 112 | ) -> dict[str, object]: |
| 113 | """Run one guarded pytest session; return the guard's report plus the exit code and output tail.""" |
| 114 | report = tmp_path / f"report-{len(list(tmp_path.glob('report-*')))}.json" |
| 115 | home = tmp_path / "home" |
| 116 | home.mkdir(exist_ok=True) |
| 117 | env = { |
| 118 | "PATH": os.environ.get("PATH", "/usr/bin:/bin"), |
| 119 | "HOME": str(home), |
| 120 | "TEST_DATABASE_URL": db_url, |
| 121 | "DATABASE_URL": db_url, |
| 122 | "PYTHONDONTWRITEBYTECODE": "1", |
| 123 | "DOCDB_REPORT": str(report), |
| 124 | } |
| 125 | if os.environ.get("PYTHONPATH"): |
| 126 | env["PYTHONPATH"] = os.environ["PYTHONPATH"] |
| 127 | if strip: |
| 128 | env["DOCDB_STRIP_NO_DB"] = "1" |
| 129 | if inject: |
| 130 | env["DOCDB_INJECT"] = inject |
| 131 | cmd = [sys.executable, "-m", "pytest", "-p", "tests.docs_contract.db_guard", "-p", "no:cacheprovider", "-p", "no:randomly"] |
| 132 | cmd += ["-q", "-o", "addopts=", "--ignore", _THIS, *(["--collect-only"] if collect_only else []), *args] |
| 133 | # Each nested session reruns the whole docs suite (minus this file). The suite outgrew 300 s when #250/#251 |
| 134 | # added real-CLI sandbox tests (348 s alone on 2026-11-07); 900 s leaves headroom without hiding a hang. |
| 135 | proc = subprocess.run(cmd, cwd=ROOT, env=env, capture_output=True, text=True, timeout=900) |
| 136 | assert report.exists(), f"guard wrote no report (exit {proc.returncode}):\n{proc.stdout[-3000:]}{proc.stderr[-3000:]}" |
| 137 | result: dict[str, object] = json.loads(report.read_text()) |
| 138 | result["returncode"] = proc.returncode |
| 139 | result["output"] = proc.stdout[-3000:] |
| 140 | return result |
| 141 | |
| 142 | |
| 143 | def _outcomes(result: dict[str, object]) -> dict[str, str]: |
| 144 | return cast(dict[str, str], result["outcomes"]) |
| 145 | |
| 146 | |
| 147 | def _no_db(result: dict[str, object]) -> dict[str, bool]: |
| 148 | return cast(dict[str, bool], result["no_db"]) |
| 149 | |
| 150 | |
| 151 | def _assert_clean_docs_run(result: dict[str, object]) -> None: |
| 152 | counts = Counter(_outcomes(result).values()) |
| 153 | assert result["engine_attempts"] == [] and result["socket_attempts"] == [], result["output"] |
| 154 | assert set(counts) <= {"passed", "xfailed"} and counts["passed"] > 0, (counts, result["output"]) |
| 155 | assert counts["xfailed"] > 0, "known findings must stay strict-xfailed" |
| 156 | assert result["returncode"] == 0, result["output"] |
| 157 | |
| 158 | |
| 159 | def test_docdb_01_docs_session_without_module_markers_never_creates_an_engine(tmp_path: Path) -> None: |
| 160 | result = _session(tmp_path, _DOCS, strip=True) |
| 161 | assert _no_db(result) and all(_no_db(result).values()) |
| 162 | _assert_clean_docs_run(result) |
| 163 | |
| 164 | |
| 165 | @pytest.mark.parametrize("fixture", ["db_session", "session_factory"]) |
| 166 | def test_docdb_02_docs_test_requesting_a_db_fixture_reaches_db_schema(tmp_path: Path, fixture: str) -> None: |
| 167 | result = _session(tmp_path, _LAND, inject=f"{fixture}={_LAND}") |
| 168 | assert _no_db(result) == {_LAND: False} |
| 169 | assert result["engine_attempts"] == [_DB_HOST], "exactly one blocked _db_schema engine attempt" |
| 170 | assert result["socket_attempts"] == [], "no connection attempted" |
| 171 | assert _outcomes(result) == {_LAND: "error"} |
| 172 | |
| 173 | |
| 174 | def test_docdb_03_docs_session_ignores_a_target_that_is_not_a_test_database(tmp_path: Path) -> None: |
| 175 | # Rejecting such a target belongs to #263 (TISO_02/TISO_09); a docs-only session must just never reach it. |
| 176 | result = _session(tmp_path, _DOCS, db_url=_NOT_A_TEST_DB) |
| 177 | assert all(_no_db(result).values()) |
| 178 | _assert_clean_docs_run(result) |
| 179 | |
| 180 | |
| 181 | def test_docdb_04_failed_db_setup_is_an_error_not_an_expected_failure(tmp_path: Path) -> None: |
| 182 | baseline = _session(tmp_path, _REGISTRY, _SCHEME) |
| 183 | xfails = {n for n, o in _outcomes(baseline).items() if o == "xfailed"} |
| 184 | assert {n.split("::")[0] for n in xfails} == {_REGISTRY, _SCHEME}, "both docs xfail helpers are exercised" |
| 185 | db_item = next(n for n, o in _outcomes(baseline).items() if o == "passed" and n.startswith(_REGISTRY)) |
| 186 | # The ticket's scenario: no module markers, and one DB-backed docs test whose session setup fails. |
| 187 | broken = _session(tmp_path, _REGISTRY, _SCHEME, strip=True, inject=f"db_session={db_item}") |
| 188 | assert broken["engine_attempts"] == [_DB_HOST] and broken["socket_attempts"] == [] |
| 189 | outcomes = _outcomes(broken) |
| 190 | assert {n: outcomes[n] for n in xfails} == dict.fromkeys(xfails, "error"), "a setup failure must not read as XFAIL" |
| 191 | assert set(outcomes.values()) == {"error"} |
| 192 | |
| 193 | |
| 194 | @pytest.mark.parametrize( |
| 195 | "selection", |
| 196 | [ |
| 197 | [_DOCS], |
| 198 | ["tests/docs_contract/test_doc_gs.py"], |
| 199 | [_LAND], |
| 200 | [_DOCS, "-k", "https or landing"], |
| 201 | ], |
| 202 | ids=["directory", "module", "node_id", "keyword"], |
| 203 | ) |
| 204 | def test_docdb_05_every_selection_classifies_docs_items_no_db(tmp_path: Path, selection: list[str]) -> None: |
| 205 | result = _session(tmp_path, *selection, strip=True, collect_only=True) |
| 206 | assert result["returncode"] == 0, result["output"] |
| 207 | assert _no_db(result) and all(_no_db(result).values()), {n for n, v in _no_db(result).items() if not v} |
| 208 | assert result["engine_attempts"] == [] |
| 209 | |
| 210 | |
| 211 | def test_docdb_05_existing_prefix_suites_keep_their_classification(tmp_path: Path) -> None: |
| 212 | suites = ["tests/test_prune_releases.py", "tests/test_build_context.py", "tests/test_release_gate.py"] |
| 213 | result = _session(tmp_path, *suites, "tests/test_release_analysis_no_commit_meta.py", collect_only=True) |
| 214 | assert result["returncode"] == 0, result["output"] |
| 215 | closure = cast(dict[str, list[str]], result["db_closure"]) |
| 216 | assert {n: v for n, v in _no_db(result).items()} == {n: not closure[n] for n in closure} |
| 217 | assert any(closure.values()) and not all(closure.values()), "the sample holds both DB-backed and pure tests" |
| 218 | |
| 219 | |
| 220 | def test_docdb_05_mixed_session_with_a_db_backed_test_still_runs_db_schema(tmp_path: Path) -> None: |
| 221 | db_test = "tests/test_users_no_commit_meta.py" |
| 222 | result = _session(tmp_path, _LAND, db_test) |
| 223 | no_db = _no_db(result) |
| 224 | assert no_db[_LAND] is True |
| 225 | assert [v for n, v in no_db.items() if n.startswith(db_test)] == [False] |
| 226 | assert result["engine_attempts"] == [_DB_HOST] and result["socket_attempts"] == [] |
| 227 | assert set(_outcomes(result).values()) == {"error"} |
File History
6 commits
sha256:1dd276b0d6531bd2d11e9a083cfe57657caa58eff512d0e1bd6ac74d3d49cd79
test(docs): the two whole-docs-session isolation probes get…
Opus 5.5
1 day ago
sha256:c2ee54efa85fcbdeea4ffc84a2dd1b8fcd4623463b22e35765df3013a26834aa
merge: #276 public URLs + muse#181 docs onto dev (#313 present)
Opus 5.5
minor
⚠
1 day ago
sha256:c777acd5605d4a83bbf9d2903bff0b0d419325b7bc49317700e4037b56594e8e
fix(web): absolute public URLs come from the canonical publ…
Opus 5.5
minor
⚠
1 day ago
sha256:7513f245748d6667481e0ecef54e261ad11d4a664ba7128527060b240f4707bc
docs(security): #260 Security Reference QA contract v2 — co…
Sonnet 5.5
minor
⚠
8 days ago
sha256:2978eb11903d1a9d08902c41c8bb6cb874b470c19e8df579ebb66d033dbb8bdd
test(docs_contract): docdb nested-session timeout 300 -> 900 s
Opus 5.5
9 days ago
sha256:f7413c1abec6d6a8e5bdabcd4cc7ad5fcd059ce6a6f50f566e2b0118a0a8b578
tests: make tests/docs_contract/ database-free by directory…
Opus 5.5
10 days ago