test_release_gate_env.py
python
sha256:a977b39d08e15683210a33608e25712fe3ed9c8f05f0388fb69ef38acf835426
feat(release): hermetic release gates in a provisioned, pin…
Opus 5.5
minor
⚠ breaking
12 days ago
| 1 | """Hermetic release-gate environment (musehub#114 Phase 6D, D6). |
| 2 | |
| 3 | The gates must not depend on the invoking Python. These tests drive ``deploy/gate_env.py`` with a |
| 4 | recording fake runner (no network, no real pip) to prove every failure blocks the gate and leaves |
| 5 | nothing behind, plus real probes of the interpreter-selection rules. The real end-to-end run is the |
| 6 | 12-gate pipeline itself, recorded in the issue. |
| 7 | """ |
| 8 | |
| 9 | from __future__ import annotations |
| 10 | |
| 11 | import io |
| 12 | import json |
| 13 | import pathlib |
| 14 | import subprocess |
| 15 | import sys |
| 16 | import tarfile |
| 17 | from collections.abc import Mapping |
| 18 | from typing import Any |
| 19 | |
| 20 | import pytest |
| 21 | |
| 22 | from deploy import gate_env |
| 23 | from deploy.gate_env import ( |
| 24 | GateEnv, |
| 25 | GateEnvError, |
| 26 | Interpreter, |
| 27 | build_env, |
| 28 | gate_environment, |
| 29 | gate_process_env, |
| 30 | parse_requirements, |
| 31 | requires_python_of, |
| 32 | select_interpreter, |
| 33 | verify_complete, |
| 34 | ) |
| 35 | from deploy.release_gate import GateError, GateResult, default_gate_commands, run_release_gates, verify_report |
| 36 | from tests.release_fixtures import CHECKOUT, NOW, SOURCE_REVISION, good_environment, make_artifact, make_report, mutate_report, passing_results, sha256_of |
| 37 | |
| 38 | PY = "/opt/python/bin/python3.14" |
| 39 | |
| 40 | |
| 41 | def sdist(tmp: pathlib.Path, version: str = "0.2.1rc19", requires: str | None = ">=3.14", payload: bytes = b"") -> pathlib.Path: |
| 42 | path = tmp / f"muse-{version}.tar.gz" |
| 43 | info = f"Metadata-Version: 2.1\nName: muse\nVersion: {version}\n" + (f"Requires-Python: {requires}\n" if requires else "") |
| 44 | with tarfile.open(path, "w:gz") as tf: |
| 45 | data = info.encode() |
| 46 | member = tarfile.TarInfo(f"muse-{version}/PKG-INFO") |
| 47 | member.size = len(data) |
| 48 | tf.addfile(member, io.BytesIO(data)) |
| 49 | if payload: |
| 50 | extra = tarfile.TarInfo(f"muse-{version}/payload") |
| 51 | extra.size = len(payload) |
| 52 | tf.addfile(extra, io.BytesIO(payload)) |
| 53 | return path |
| 54 | |
| 55 | |
| 56 | class FakeRunner: |
| 57 | """Pretends to be the interpreter: records argv/env, creates the venv python, answers probes and verification.""" |
| 58 | |
| 59 | def __init__(self, *, version: str = "3.14.4", impl: str = "cpython", fail: str | None = None, problems: list[str] | None = None, |
| 60 | installed: Mapping[str, str] | None = None, archive_sha: str | None = None, make_venv: bool = True) -> None: |
| 61 | self.calls: list[tuple[list[str], dict[str, str] | None]] = [] |
| 62 | self.version, self.impl, self.fail, self.problems = version, impl, fail, problems or [] |
| 63 | self.installed = dict(installed) if installed is not None else None |
| 64 | self.archive_sha, self.make_venv = archive_sha, make_venv |
| 65 | self.pins: dict[str, str] = {} |
| 66 | |
| 67 | def __call__(self, argv: list[str], **kw: Any) -> subprocess.CompletedProcess[str]: |
| 68 | env = kw.get("env") |
| 69 | self.calls.append((list(argv), env)) |
| 70 | text = " ".join(argv) |
| 71 | if self.fail and self.fail in text: |
| 72 | return subprocess.CompletedProcess(argv, 1, "", f"simulated failure of {self.fail}") |
| 73 | if "sys.implementation.name" in text: |
| 74 | return subprocess.CompletedProcess(argv, 0, json.dumps({"v": self.version, "impl": self.impl, "exe": argv[0]}), "") |
| 75 | if "-m venv" in text: |
| 76 | if self.make_venv: |
| 77 | target = pathlib.Path(argv[-1]) / "bin" |
| 78 | target.mkdir(parents=True) |
| 79 | (target / "python").write_text("#!/bin/sh\n") |
| 80 | return subprocess.CompletedProcess(argv, 0, "", "") |
| 81 | if "importlib.metadata" in text: |
| 82 | version = argv[argv.index("-c") + 3] |
| 83 | pins = json.loads(argv[argv.index("-c") + 2]) |
| 84 | installed = self.installed if self.installed is not None else {**pins, "muse": version} |
| 85 | return subprocess.CompletedProcess(argv, 0, json.dumps({"problems": self.problems, "installed": installed, "archive_sha256": self.archive_sha, "grammar_modules": []}), "") |
| 86 | return subprocess.CompletedProcess(argv, 0, "", "") |
| 87 | |
| 88 | def pip_calls(self) -> list[list[str]]: |
| 89 | return [a for a, _ in self.calls if "pip" in a and "install" in a] |
| 90 | |
| 91 | |
| 92 | @pytest.fixture |
| 93 | def reqs(tmp_path: pathlib.Path) -> pathlib.Path: |
| 94 | path = tmp_path / "reqs.txt" |
| 95 | path.write_text("pytest==9.0.3\nbuild==1.2.2\npackaging==26.0\n") |
| 96 | return path |
| 97 | |
| 98 | |
| 99 | @pytest.fixture |
| 100 | def art(tmp_path: pathlib.Path) -> pathlib.Path: |
| 101 | (tmp_path / "a").mkdir() |
| 102 | return sdist(tmp_path / "a") |
| 103 | |
| 104 | |
| 105 | @pytest.fixture |
| 106 | def interpreter(tmp_path: pathlib.Path) -> Interpreter: |
| 107 | return Interpreter(pathlib.Path(PY), "3.14.4", "cpython") |
| 108 | |
| 109 | |
| 110 | def fake_exe(tmp: pathlib.Path) -> pathlib.Path: |
| 111 | exe = tmp / "python3.14" |
| 112 | exe.write_text("#!/bin/sh\n") |
| 113 | exe.chmod(0o755) |
| 114 | return exe |
| 115 | |
| 116 | |
| 117 | # ── interpreter selection ───────────────────────────────────────────────────── |
| 118 | |
| 119 | |
| 120 | class TestInterpreterSelection: |
| 121 | def test_the_candidates_own_requires_python_is_the_contract(self, art: pathlib.Path) -> None: |
| 122 | assert requires_python_of(art) == ">=3.14" |
| 123 | |
| 124 | def test_artifact_without_requires_python_or_pkg_info_is_refused(self, tmp_path: pathlib.Path) -> None: |
| 125 | with pytest.raises(GateEnvError, match="Requires-Python"): |
| 126 | requires_python_of(sdist(tmp_path, requires=None)) |
| 127 | bogus = tmp_path / "muse-1.0.tar.gz" |
| 128 | bogus.write_bytes(b"not a tarball") |
| 129 | with pytest.raises(GateEnvError, match="PKG-INFO"): |
| 130 | requires_python_of(bogus) |
| 131 | |
| 132 | @pytest.mark.parametrize("python", [None, "", "python3", "relative/python", "/nonexistent/python3.14"]) |
| 133 | def test_missing_or_non_explicit_interpreter_is_refused_and_nothing_runs(self, python: str | None) -> None: |
| 134 | runner = FakeRunner() |
| 135 | with pytest.raises(GateEnvError, match="interpreter|absolute|explicit"): |
| 136 | select_interpreter(python, ">=3.14", runner=runner) |
| 137 | assert runner.calls == [] |
| 138 | |
| 139 | def test_non_executable_file_is_refused(self, tmp_path: pathlib.Path) -> None: |
| 140 | exe = tmp_path / "python3.14" |
| 141 | exe.write_text("") |
| 142 | with pytest.raises(GateEnvError, match="executable"): |
| 143 | select_interpreter(exe, ">=3.14", runner=FakeRunner()) |
| 144 | |
| 145 | def test_unsupported_python_version_is_refused(self, tmp_path: pathlib.Path) -> None: |
| 146 | with pytest.raises(GateEnvError, match="requires >=3.14"): |
| 147 | select_interpreter(fake_exe(tmp_path), ">=3.14", runner=FakeRunner(version="3.13.1")) |
| 148 | |
| 149 | def test_other_implementations_are_refused(self, tmp_path: pathlib.Path) -> None: |
| 150 | with pytest.raises(GateEnvError, match="CPython"): |
| 151 | select_interpreter(fake_exe(tmp_path), ">=3.14", runner=FakeRunner(impl="pypy")) |
| 152 | |
| 153 | def test_probe_failure_is_refused(self, tmp_path: pathlib.Path) -> None: |
| 154 | with pytest.raises(GateEnvError, match="probing"): |
| 155 | select_interpreter(fake_exe(tmp_path), ">=3.14", runner=FakeRunner(fail="sys.implementation.name")) |
| 156 | |
| 157 | def test_the_probe_runs_isolated_so_the_callers_environment_cannot_change_the_answer(self, tmp_path: pathlib.Path) -> None: |
| 158 | runner = FakeRunner() |
| 159 | select_interpreter(fake_exe(tmp_path), ">=3.14", runner=runner) |
| 160 | assert "-I" in runner.calls[0][0] |
| 161 | |
| 162 | def test_the_real_interpreter_probe_accepts_this_python(self) -> None: |
| 163 | real = pathlib.Path(sys.base_prefix) / "bin" / "python3" |
| 164 | if not real.is_file(): |
| 165 | pytest.skip("no base interpreter at the expected path") |
| 166 | info = select_interpreter(real, f">={sys.version_info.major}.{sys.version_info.minor}") |
| 167 | assert info.implementation == "cpython" and info.version.startswith(f"{sys.version_info.major}.{sys.version_info.minor}") |
| 168 | with pytest.raises(GateEnvError, match="requires"): |
| 169 | select_interpreter(real, ">=99") |
| 170 | |
| 171 | |
| 172 | # ── pins ────────────────────────────────────────────────────────────────────── |
| 173 | |
| 174 | |
| 175 | class TestPins: |
| 176 | def test_exact_pins_parse_with_canonical_names(self) -> None: |
| 177 | assert parse_requirements("PyTest==9.0.3 # tool\n\nTree_Sitter==0.25.2\n") == {"pytest": "9.0.3", "tree-sitter": "0.25.2"} |
| 178 | |
| 179 | @pytest.mark.parametrize("text", ["pytest>=9", "pytest", "pytest~=9.0", "pytest==9.*", "pytest==9,<10", "", "# only a comment\n", "pytest==1\npytest==2", "not a requirement !!"]) |
| 180 | def test_anything_but_an_exact_pin_is_refused(self, text: str) -> None: |
| 181 | with pytest.raises(GateEnvError): |
| 182 | parse_requirements(text) |
| 183 | |
| 184 | def test_the_shipped_requirements_are_exact_and_cover_the_gate_tooling(self) -> None: |
| 185 | pins = parse_requirements(gate_env.REQUIREMENTS.read_text()) |
| 186 | for needed in ("pytest", "build", "packaging", "fastapi", "pydantic", "boto3", "pytest-json-report"): |
| 187 | assert needed in pins, needed |
| 188 | assert "muse" not in pins, "the candidate comes from the artifact, never from a pin" |
| 189 | |
| 190 | |
| 191 | # ── building ────────────────────────────────────────────────────────────────── |
| 192 | |
| 193 | |
| 194 | class TestBuild: |
| 195 | def build(self, tmp: pathlib.Path, art: pathlib.Path, interp: Interpreter, reqs: pathlib.Path, runner: FakeRunner) -> GateEnv: |
| 196 | return build_env(tmp / "work" / "env", art, interp, requirements=reqs, runner=runner) |
| 197 | |
| 198 | def test_happy_path_installs_the_pins_then_the_exact_artifact_with_all_extra(self, tmp_path: pathlib.Path, art: pathlib.Path, interpreter: Interpreter, reqs: pathlib.Path) -> None: |
| 199 | runner = FakeRunner() |
| 200 | env = self.build(tmp_path, art, interpreter, reqs, runner) |
| 201 | venv_py = str(env.root / "venv" / "bin" / "python") |
| 202 | pips = runner.pip_calls() |
| 203 | assert [p[0] for p in pips] == [venv_py, venv_py], "pip is only ever run by the venv's own interpreter" |
| 204 | assert pips[0][-2:] == ["-r", str(reqs)] |
| 205 | assert pips[1][-3:-1] == ["-c", str(reqs)] and pips[1][-1].endswith(f"candidate/{art.name}[all]") |
| 206 | assert (env.root / "candidate" / art.name).read_bytes() == art.read_bytes(), "installs a hash-checked copy of the artifact" |
| 207 | assert env.packages["muse"] == "0.2.1rc19" and env.candidate_sha256 == sha256_of(art) |
| 208 | verify_complete(env, art) |
| 209 | |
| 210 | def test_global_python_is_only_asked_to_make_a_venv(self, tmp_path: pathlib.Path, art: pathlib.Path, interpreter: Interpreter, reqs: pathlib.Path) -> None: |
| 211 | runner = FakeRunner() |
| 212 | self.build(tmp_path, art, interpreter, reqs, runner) |
| 213 | by_global = [a for a, _ in runner.calls if a[0] == PY] |
| 214 | assert len(by_global) == 1 and by_global[0][1:4] == ["-I", "-m", "venv"] |
| 215 | assert not any(a[0] == PY and "pip" in a for a, _ in runner.calls) |
| 216 | |
| 217 | def test_every_subprocess_is_scrubbed_of_the_callers_python_settings(self, tmp_path: pathlib.Path, art: pathlib.Path, interpreter: Interpreter, reqs: pathlib.Path, monkeypatch: pytest.MonkeyPatch) -> None: |
| 218 | for name, value in (("PYTHONPATH", "/dev/checkout"), ("PIP_INDEX_URL", "http://evil"), ("PYTHONHOME", "/x"), ("VIRTUAL_ENV", "/dev/venv"), ("PIP_USER", "1")): |
| 219 | monkeypatch.setenv(name, value) |
| 220 | runner = FakeRunner() |
| 221 | self.build(tmp_path, art, interpreter, reqs, runner) |
| 222 | for argv, env in runner.calls: |
| 223 | if argv[0] == PY and "sys.implementation.name" in " ".join(argv): |
| 224 | continue |
| 225 | assert env is not None |
| 226 | assert "PYTHONPATH" not in env and "PYTHONHOME" not in env and "PIP_INDEX_URL" not in env and "PIP_USER" not in env |
| 227 | assert env["PYTHONNOUSERSITE"] == "1" and env["PIP_REQUIRE_VIRTUALENV"] == "1" |
| 228 | |
| 229 | def test_gate_processes_get_the_venv_first_on_path_and_none_of_the_callers_python_env(self, tmp_path: pathlib.Path, art: pathlib.Path, interpreter: Interpreter, reqs: pathlib.Path, monkeypatch: pytest.MonkeyPatch) -> None: |
| 230 | monkeypatch.setenv("PYTHONPATH", "/dev/checkout") |
| 231 | env = self.build(tmp_path, art, interpreter, reqs, FakeRunner()) |
| 232 | proc_env = gate_process_env(env) |
| 233 | assert proc_env["PATH"].startswith(str(env.root / "venv" / "bin")) and "PYTHONPATH" not in proc_env |
| 234 | |
| 235 | def test_dependency_installation_failure_blocks(self, tmp_path: pathlib.Path, art: pathlib.Path, interpreter: Interpreter, reqs: pathlib.Path) -> None: |
| 236 | runner = FakeRunner(fail=f"-r {reqs}") |
| 237 | with pytest.raises(GateEnvError, match="pinned gate tooling"): |
| 238 | self.build(tmp_path, art, interpreter, reqs, runner) |
| 239 | assert len(runner.pip_calls()) == 0 or "candidate" not in " ".join(runner.pip_calls()[-1]) |
| 240 | |
| 241 | def test_grammar_installation_failure_blocks(self, tmp_path: pathlib.Path, art: pathlib.Path, interpreter: Interpreter, reqs: pathlib.Path) -> None: |
| 242 | with pytest.raises(GateEnvError, match=r"\[all\] grammars"): |
| 243 | self.build(tmp_path, art, interpreter, reqs, FakeRunner(fail="[all]")) |
| 244 | |
| 245 | def test_a_grammar_that_installed_but_does_not_import_blocks(self, tmp_path: pathlib.Path, art: pathlib.Path, interpreter: Interpreter, reqs: pathlib.Path) -> None: |
| 246 | with pytest.raises(GateEnvError, match="grammar tree_sitter_go"): |
| 247 | self.build(tmp_path, art, interpreter, reqs, FakeRunner(problems=["grammar tree_sitter_go: ImportError: boom"])) |
| 248 | |
| 249 | def test_a_missing_test_dependency_blocks(self, tmp_path: pathlib.Path, art: pathlib.Path, interpreter: Interpreter, reqs: pathlib.Path) -> None: |
| 250 | with pytest.raises(GateEnvError, match="missing pytest==9.0.3"): |
| 251 | self.build(tmp_path, art, interpreter, reqs, FakeRunner(problems=["missing pytest==9.0.3"])) |
| 252 | |
| 253 | def test_a_wrong_candidate_in_the_environment_blocks(self, tmp_path: pathlib.Path, art: pathlib.Path, interpreter: Interpreter, reqs: pathlib.Path) -> None: |
| 254 | with pytest.raises(GateEnvError, match="installed muse is 0.2.1rc18"): |
| 255 | self.build(tmp_path, art, interpreter, reqs, FakeRunner(problems=["installed muse is 0.2.1rc18, candidate is 0.2.1rc19"])) |
| 256 | |
| 257 | def test_muse_imported_from_outside_the_venv_blocks(self, tmp_path: pathlib.Path, art: pathlib.Path, interpreter: Interpreter, reqs: pathlib.Path) -> None: |
| 258 | with pytest.raises(GateEnvError, match="outside the gate venv"): |
| 259 | self.build(tmp_path, art, interpreter, reqs, FakeRunner(problems=["muse imported from /dev/muse/__init__.py, outside the gate venv"])) |
| 260 | |
| 261 | def test_pip_recording_a_different_archive_hash_blocks(self, tmp_path: pathlib.Path, art: pathlib.Path, interpreter: Interpreter, reqs: pathlib.Path) -> None: |
| 262 | with pytest.raises(GateEnvError, match="not the candidate"): |
| 263 | self.build(tmp_path, art, interpreter, reqs, FakeRunner(archive_sha="0" * 64)) |
| 264 | |
| 265 | def test_unreadable_verification_output_blocks(self, tmp_path: pathlib.Path, art: pathlib.Path, interpreter: Interpreter, reqs: pathlib.Path) -> None: |
| 266 | class Garbled(FakeRunner): |
| 267 | def __call__(self, argv: list[str], **kw: Any) -> subprocess.CompletedProcess[str]: |
| 268 | if "importlib.metadata" in " ".join(argv): |
| 269 | return subprocess.CompletedProcess(argv, 0, "no json here", "") |
| 270 | return super().__call__(argv, **kw) |
| 271 | |
| 272 | with pytest.raises(GateEnvError, match="unreadable"): |
| 273 | self.build(tmp_path, art, interpreter, reqs, Garbled()) |
| 274 | |
| 275 | def test_a_venv_without_an_interpreter_is_a_partial_environment(self, tmp_path: pathlib.Path, art: pathlib.Path, interpreter: Interpreter, reqs: pathlib.Path) -> None: |
| 276 | with pytest.raises(GateEnvError, match="no interpreter"): |
| 277 | self.build(tmp_path, art, interpreter, reqs, FakeRunner(make_venv=False)) |
| 278 | |
| 279 | def test_an_unpinned_requirements_file_is_refused_before_anything_is_created(self, tmp_path: pathlib.Path, art: pathlib.Path, interpreter: Interpreter) -> None: |
| 280 | loose = tmp_path / "loose.txt" |
| 281 | loose.write_text("pytest>=9\n") |
| 282 | runner = FakeRunner() |
| 283 | with pytest.raises(GateEnvError, match="exact"): |
| 284 | self.build(tmp_path, art, interpreter, loose, runner) |
| 285 | assert runner.calls == [] and not (tmp_path / "work").exists() |
| 286 | |
| 287 | |
| 288 | class TestVerificationScript: |
| 289 | """Run the real in-venv verification script against a stub ``muse`` so its decisions are exercised, not mocked.""" |
| 290 | |
| 291 | def stub(self, tmp: pathlib.Path, specs: str, version: str = "0.2.1rc19", present: tuple[str, ...] = ("fake_full_grammar", "tree_sitter_markdown", "tree_sitter_html")) -> pathlib.Path: |
| 292 | site = tmp / "site" |
| 293 | for rel in ("muse/plugins/code",): |
| 294 | (site / rel).mkdir(parents=True) |
| 295 | for pkg in ("muse", "muse/plugins", "muse/plugins/code"): |
| 296 | (site / pkg / "__init__.py").write_text("") |
| 297 | (site / "muse/plugins/code/ast_parser.py").write_text(f"_TS_LANG_SPECS = {specs}\n") |
| 298 | for mod in present: |
| 299 | (site / f"{mod}.py").write_text("") |
| 300 | info = site / f"muse-{version}.dist-info" |
| 301 | info.mkdir() |
| 302 | (info / "METADATA").write_text(f"Metadata-Version: 2.1\nName: muse\nVersion: {version}\n") |
| 303 | return site |
| 304 | |
| 305 | def run(self, site: pathlib.Path, pins: dict[str, str], version: str = "0.2.1rc19") -> dict[str, Any]: |
| 306 | import os |
| 307 | |
| 308 | proc = subprocess.run( |
| 309 | [sys.executable, "-c", gate_env._VERIFY, json.dumps(pins), version, str(site)], |
| 310 | capture_output=True, text=True, env={**os.environ, "PYTHONPATH": str(site)}, timeout=60, |
| 311 | ) |
| 312 | assert proc.returncode == 0, proc.stderr |
| 313 | return json.loads(proc.stdout.strip().splitlines()[-1]) # type: ignore[no-any-return] |
| 314 | |
| 315 | SPECS = '[{"module_name": "fake_full_grammar", "profile": "full"}, {"module_name": "py_tree_sitter_swift", "profile": "opt_in"}]' |
| 316 | |
| 317 | def test_opt_in_grammars_are_not_required_but_guaranteed_ones_are(self, tmp_path: pathlib.Path) -> None: |
| 318 | result = self.run(self.stub(tmp_path, self.SPECS), {}) |
| 319 | assert result["problems"] == [], "Swift is opt-in and must not fail the gate environment" |
| 320 | assert "py_tree_sitter_swift" not in result["grammar_modules"] and "fake_full_grammar" in result["grammar_modules"] |
| 321 | |
| 322 | def test_a_missing_guaranteed_grammar_is_reported(self, tmp_path: pathlib.Path) -> None: |
| 323 | result = self.run(self.stub(tmp_path, self.SPECS, present=("tree_sitter_markdown", "tree_sitter_html")), {}) |
| 324 | assert any("fake_full_grammar" in p for p in result["problems"]) |
| 325 | |
| 326 | def test_a_missing_or_wrong_version_pin_is_reported(self, tmp_path: pathlib.Path) -> None: |
| 327 | site = self.stub(tmp_path, self.SPECS) |
| 328 | result = self.run(site, {"nonexistent-package": "1.0", "packaging": "0.0.1"}) |
| 329 | assert any("missing nonexistent-package==1.0" in p for p in result["problems"]) |
| 330 | assert any(p.startswith("packaging is ") and "pinned 0.0.1" in p for p in result["problems"]) |
| 331 | |
| 332 | def test_a_different_installed_muse_version_is_reported(self, tmp_path: pathlib.Path) -> None: |
| 333 | result = self.run(self.stub(tmp_path, self.SPECS, version="0.2.1rc18"), {}, version="0.2.1rc19") |
| 334 | assert any("installed muse is 0.2.1rc18" in p for p in result["problems"]) |
| 335 | |
| 336 | def test_muse_imported_from_outside_the_environment_root_is_reported(self, tmp_path: pathlib.Path) -> None: |
| 337 | site = self.stub(tmp_path, self.SPECS) |
| 338 | import os |
| 339 | |
| 340 | proc = subprocess.run( |
| 341 | [sys.executable, "-c", gate_env._VERIFY, "{}", "0.2.1rc19", str(tmp_path / "somewhere-else")], |
| 342 | capture_output=True, text=True, env={**os.environ, "PYTHONPATH": str(site)}, timeout=60, |
| 343 | ) |
| 344 | assert any("outside the gate venv" in p for p in json.loads(proc.stdout.strip().splitlines()[-1])["problems"]) |
| 345 | |
| 346 | |
| 347 | class TestStaleAndPartialEnvironments: |
| 348 | def test_an_existing_environment_directory_is_never_reused(self, tmp_path: pathlib.Path, art: pathlib.Path, interpreter: Interpreter, reqs: pathlib.Path) -> None: |
| 349 | stale = tmp_path / "old-env" |
| 350 | (stale / "venv" / "bin").mkdir(parents=True) |
| 351 | runner = FakeRunner() |
| 352 | with pytest.raises(GateEnvError, match="never reused"): |
| 353 | build_env(stale, art, interpreter, requirements=reqs, runner=runner) |
| 354 | assert runner.calls == [] |
| 355 | |
| 356 | def test_an_environment_without_its_completion_marker_is_refused(self, tmp_path: pathlib.Path, art: pathlib.Path, interpreter: Interpreter, reqs: pathlib.Path) -> None: |
| 357 | env = build_env(tmp_path / "e", art, interpreter, requirements=reqs, runner=FakeRunner()) |
| 358 | (env.root / gate_env.MARKER).unlink() |
| 359 | with pytest.raises(GateEnvError, match="not a complete gate environment"): |
| 360 | verify_complete(env, art) |
| 361 | |
| 362 | def test_an_environment_built_for_another_artifact_is_refused(self, tmp_path: pathlib.Path, art: pathlib.Path, interpreter: Interpreter, reqs: pathlib.Path) -> None: |
| 363 | env = build_env(tmp_path / "e", art, interpreter, requirements=reqs, runner=FakeRunner()) |
| 364 | (tmp_path / "o").mkdir() |
| 365 | other = sdist(tmp_path / "o", payload=b"different bytes, same version") |
| 366 | with pytest.raises(GateEnvError, match="different artifact"): |
| 367 | verify_complete(env, other) |
| 368 | |
| 369 | def test_the_marker_is_written_last_so_a_failed_build_never_has_one(self, tmp_path: pathlib.Path, art: pathlib.Path, interpreter: Interpreter, reqs: pathlib.Path) -> None: |
| 370 | root = tmp_path / "e" |
| 371 | with pytest.raises(GateEnvError): |
| 372 | build_env(root, art, interpreter, requirements=reqs, runner=FakeRunner(fail="[all]")) |
| 373 | assert not (root / gate_env.MARKER).exists() |
| 374 | |
| 375 | |
| 376 | class TestLifecycle: |
| 377 | def make(self, tmp: pathlib.Path, art: pathlib.Path, reqs: pathlib.Path, runner: FakeRunner, **kw: Any) -> Any: |
| 378 | return gate_environment(art, fake_exe(tmp), parent=tmp / "scratch", requirements=reqs, runner=runner, **kw) |
| 379 | |
| 380 | @pytest.fixture(autouse=True) |
| 381 | def scratch(self, tmp_path: pathlib.Path) -> None: |
| 382 | (tmp_path / "scratch").mkdir() |
| 383 | |
| 384 | def leftovers(self, tmp: pathlib.Path) -> list[str]: |
| 385 | return [p.name for p in (tmp / "scratch").iterdir()] |
| 386 | |
| 387 | def test_the_environment_exists_inside_and_is_gone_after(self, tmp_path: pathlib.Path, art: pathlib.Path, reqs: pathlib.Path) -> None: |
| 388 | with self.make(tmp_path, art, reqs, FakeRunner()) as env: |
| 389 | assert env.python.is_file() and self.leftovers(tmp_path) |
| 390 | assert self.leftovers(tmp_path) == [] |
| 391 | |
| 392 | def test_a_failure_while_building_leaves_nothing_behind(self, tmp_path: pathlib.Path, art: pathlib.Path, reqs: pathlib.Path) -> None: |
| 393 | with pytest.raises(GateEnvError): |
| 394 | with self.make(tmp_path, art, reqs, FakeRunner(fail="[all]")): |
| 395 | pytest.fail("must not run gates in a broken environment") |
| 396 | assert self.leftovers(tmp_path) == [] |
| 397 | |
| 398 | def test_an_interrupted_run_is_cleaned_up_and_the_interrupt_propagates(self, tmp_path: pathlib.Path, art: pathlib.Path, reqs: pathlib.Path) -> None: |
| 399 | with pytest.raises(KeyboardInterrupt): |
| 400 | with self.make(tmp_path, art, reqs, FakeRunner()): |
| 401 | raise KeyboardInterrupt |
| 402 | assert self.leftovers(tmp_path) == [] |
| 403 | |
| 404 | def test_failure_to_remove_the_environment_is_an_error_not_a_pass(self, tmp_path: pathlib.Path, art: pathlib.Path, reqs: pathlib.Path, monkeypatch: pytest.MonkeyPatch) -> None: |
| 405 | monkeypatch.setattr(gate_env.shutil, "rmtree", lambda *a, **k: None) |
| 406 | with pytest.raises(GateEnvError, match="could not remove"): |
| 407 | with self.make(tmp_path, art, reqs, FakeRunner()): |
| 408 | pass |
| 409 | |
| 410 | def test_unsupported_interpreter_never_creates_anything(self, tmp_path: pathlib.Path, art: pathlib.Path, reqs: pathlib.Path) -> None: |
| 411 | with pytest.raises(GateEnvError, match="requires"): |
| 412 | with self.make(tmp_path, art, reqs, FakeRunner(version="3.12.0")): |
| 413 | pass |
| 414 | assert self.leftovers(tmp_path) == [] |
| 415 | |
| 416 | |
| 417 | # ── the gate run uses it, and reports it ────────────────────────────────────── |
| 418 | |
| 419 | |
| 420 | class TestGateRunUsesTheEnvironment: |
| 421 | def run(self, tmp: pathlib.Path, art: pathlib.Path, reqs: pathlib.Path, runner: FakeRunner, gate_runner: Any, python: Any = "ok") -> dict[str, Any]: |
| 422 | (tmp / "scratch").mkdir(exist_ok=True) |
| 423 | from deploy.release_gate import run_release_gates as rrg |
| 424 | from tests.test_release_provenance import Observer |
| 425 | |
| 426 | return rrg( |
| 427 | art, tmp / "muse", tmp / "hub", SOURCE_REVISION, 2, python=fake_exe(tmp) if python == "ok" else python, now=NOW, |
| 428 | observe=Observer(CHECKOUT), run=gate_runner, |
| 429 | environment=lambda a, p: gate_environment(a, p, parent=tmp / "scratch", requirements=reqs, runner=runner), |
| 430 | ) |
| 431 | |
| 432 | def artifact(self, tmp: pathlib.Path) -> pathlib.Path: |
| 433 | (tmp / "cand").mkdir(exist_ok=True) |
| 434 | return sdist(tmp / "cand") |
| 435 | |
| 436 | def test_every_gate_command_is_run_by_the_environments_interpreter_never_the_callers(self, tmp_path: pathlib.Path) -> None: |
| 437 | cmds = default_gate_commands(tmp_path / "muse-0.2.1rc19.tar.gz", tmp_path / "muse", tmp_path / "hub", "/gate/venv/bin/python") |
| 438 | for gate, argv in cmds.items(): |
| 439 | assert "/gate/venv/bin/python" in argv, gate |
| 440 | assert sys.executable not in argv, gate |
| 441 | assert "python3" not in argv and "python" not in argv, gate |
| 442 | |
| 443 | def test_gates_receive_the_scrubbed_env_and_the_gate_interpreter(self, tmp_path: pathlib.Path, reqs: pathlib.Path, monkeypatch: pytest.MonkeyPatch) -> None: |
| 444 | monkeypatch.setenv("PYTHONPATH", "/dev/checkout") |
| 445 | seen: dict[str, Any] = {} |
| 446 | |
| 447 | def gates(commands: Any, *, env: Mapping[str, str], cwds: Any) -> dict[str, GateResult]: |
| 448 | seen.update(env=dict(env), commands=commands) |
| 449 | return passing_results() |
| 450 | |
| 451 | report = self.run(tmp_path, self.artifact(tmp_path), reqs, FakeRunner(), gates) |
| 452 | env = seen["env"] |
| 453 | assert env["TEST_DATABASE_URL"].endswith(".invalid:1/none"), "gates must not be able to reach a real database" |
| 454 | assert "PYTHONPATH" not in env and env["MUSE_GATE_PYTHON"].endswith("/venv/bin/python") and env["PIP_REQUIRE_VIRTUALENV"] == "1" |
| 455 | assert all(cmd.count(env["MUSE_GATE_PYTHON"]) >= 1 for cmd in seen["commands"].values()) |
| 456 | assert report["decision"] == "pass" |
| 457 | |
| 458 | def test_the_report_records_interpreter_packages_and_cleanup(self, tmp_path: pathlib.Path, reqs: pathlib.Path) -> None: |
| 459 | art = self.artifact(tmp_path) |
| 460 | report = self.run(tmp_path, art, reqs, FakeRunner(), lambda c, **k: passing_results()) |
| 461 | e = report["environment"] |
| 462 | assert e["interpreter"] == {"path": str(tmp_path / "python3.14"), "version": "3.14.4", "implementation": "cpython"} |
| 463 | assert e["isolated"] is True and e["cleanup"] == "removed" and e["candidate_sha256"] == sha256_of(art) |
| 464 | assert e["packages"]["muse"] == "0.2.1rc19" and e["packages"]["pytest"] == "9.0.3" |
| 465 | assert len(e["packages_sha256"]) == 64 and len(e["requirements_sha256"]) == 64 |
| 466 | verify_report(json.dumps(report), art, now=NOW, checkout=CHECKOUT) |
| 467 | |
| 468 | def test_a_failing_gate_still_cleans_up_and_cannot_authorize(self, tmp_path: pathlib.Path, reqs: pathlib.Path) -> None: |
| 469 | art = self.artifact(tmp_path) |
| 470 | failing = {**passing_results(), "runtime.doctor": GateResult("fail", 1, "x")} |
| 471 | report = self.run(tmp_path, art, reqs, FakeRunner(), lambda c, **k: failing) |
| 472 | assert report["decision"] == "fail" and not list((tmp_path / "scratch").iterdir()) |
| 473 | with pytest.raises(GateError, match="runtime.doctor"): |
| 474 | verify_report(json.dumps(report), art, now=NOW, checkout=CHECKOUT) |
| 475 | |
| 476 | @pytest.mark.parametrize("fail", ["[all]", f"-r ", "-m venv"]) |
| 477 | def test_any_environment_failure_yields_no_report_and_no_gate_runs(self, tmp_path: pathlib.Path, reqs: pathlib.Path, fail: str) -> None: |
| 478 | ran: list[int] = [] |
| 479 | with pytest.raises(GateError, match="hermetic gate environment"): |
| 480 | self.run(tmp_path, self.artifact(tmp_path), reqs, FakeRunner(fail=fail), lambda c, **k: ran.append(1) or passing_results()) |
| 481 | assert ran == [] and not list((tmp_path / "scratch").iterdir()) |
| 482 | |
| 483 | def test_missing_interpreter_yields_no_report(self, tmp_path: pathlib.Path, reqs: pathlib.Path) -> None: |
| 484 | with pytest.raises(GateError, match="explicit"): |
| 485 | self.run(tmp_path, self.artifact(tmp_path), reqs, FakeRunner(), lambda c, **k: passing_results(), python=None) |
| 486 | |
| 487 | def test_interruption_during_the_gates_yields_no_report_and_no_environment(self, tmp_path: pathlib.Path, reqs: pathlib.Path) -> None: |
| 488 | def interrupted(*_a: object, **_k: object) -> dict[str, GateResult]: |
| 489 | raise KeyboardInterrupt |
| 490 | |
| 491 | with pytest.raises(KeyboardInterrupt): |
| 492 | self.run(tmp_path, self.artifact(tmp_path), reqs, FakeRunner(), interrupted) |
| 493 | assert not list((tmp_path / "scratch").iterdir()) |
| 494 | |
| 495 | |
| 496 | class TestVerifyReportEnvironment: |
| 497 | @pytest.fixture |
| 498 | def art(self, tmp_path: pathlib.Path) -> pathlib.Path: |
| 499 | return make_artifact(tmp_path, "0.2.1rc19") |
| 500 | |
| 501 | def verify(self, art: pathlib.Path, text: str) -> None: |
| 502 | verify_report(text, art, now=NOW, checkout=CHECKOUT) |
| 503 | |
| 504 | def test_a_report_without_environment_evidence_is_rejected(self, art: pathlib.Path) -> None: |
| 505 | data = json.loads(make_report(art)) |
| 506 | del data["environment"] |
| 507 | with pytest.raises(GateError, match="environment is required"): |
| 508 | self.verify(art, json.dumps(data)) |
| 509 | |
| 510 | @pytest.mark.parametrize( |
| 511 | "change, match", |
| 512 | [ |
| 513 | ({"environment__isolated": False}, "isolated"), |
| 514 | ({"environment__cleanup": "pending"}, "cleanup"), |
| 515 | ({"environment__cleanup": "unknown"}, "cleanup"), |
| 516 | ({"environment__candidate_sha256": "0" * 64}, "different artifact"), |
| 517 | ({"environment__packages": {}}, "packages"), |
| 518 | ({"environment__packages": {"muse": "0.2.1rc18"}}, "not the candidate"), |
| 519 | ({"environment__packages": {"pytest": "9"}}, "installed muse None"), |
| 520 | ({"environment__interpreter": {"path": "", "version": "3", "implementation": "cpython"}}, "interpreter"), |
| 521 | ({"environment__interpreter": "python"}, "interpreter"), |
| 522 | ({"environment__packages_sha256": "xyz"}, "packages_sha256"), |
| 523 | ({"environment__requirements_sha256": None}, "requirements_sha256"), |
| 524 | ({"environment": "isolated"}, "environment is required"), |
| 525 | ], |
| 526 | ) |
| 527 | def test_unverifiable_environment_evidence_is_rejected(self, art: pathlib.Path, change: dict[str, object], match: str) -> None: |
| 528 | with pytest.raises(GateError, match=match): |
| 529 | self.verify(art, mutate_report(make_report(art), **change)) |
| 530 | |
| 531 | def test_good_evidence_is_accepted(self, art: pathlib.Path) -> None: |
| 532 | self.verify(art, make_report(art, environment=good_environment(sha256_of(art), "0.2.1rc19"))) |
File History
1 commit
sha256:a977b39d08e15683210a33608e25712fe3ed9c8f05f0388fb69ef38acf835426
feat(release): hermetic release gates in a provisioned, pin…
Opus 5.5
minor
⚠
12 days ago