gabriel / musehub public
test_release_gate_env.py python
532 lines 30.1 KB
Raw
sha256:a977b39d08e15683210a33608e25712fe3ed9c8f05f0388fb69ef38acf835426 feat(release): hermetic release gates in a provisioned, pin… Opus 5.5 minor ⚠ breaking 12 days ago
1 """Hermetic release-gate environment (musehub#114 Phase 6D, D6).
2
3 The gates must not depend on the invoking Python. These tests drive ``deploy/gate_env.py`` with a
4 recording fake runner (no network, no real pip) to prove every failure blocks the gate and leaves
5 nothing behind, plus real probes of the interpreter-selection rules. The real end-to-end run is the
6 12-gate pipeline itself, recorded in the issue.
7 """
8
9 from __future__ import annotations
10
11 import io
12 import json
13 import pathlib
14 import subprocess
15 import sys
16 import tarfile
17 from collections.abc import Mapping
18 from typing import Any
19
20 import pytest
21
22 from deploy import gate_env
23 from deploy.gate_env import (
24 GateEnv,
25 GateEnvError,
26 Interpreter,
27 build_env,
28 gate_environment,
29 gate_process_env,
30 parse_requirements,
31 requires_python_of,
32 select_interpreter,
33 verify_complete,
34 )
35 from deploy.release_gate import GateError, GateResult, default_gate_commands, run_release_gates, verify_report
36 from tests.release_fixtures import CHECKOUT, NOW, SOURCE_REVISION, good_environment, make_artifact, make_report, mutate_report, passing_results, sha256_of
37
38 PY = "/opt/python/bin/python3.14"
39
40
41 def sdist(tmp: pathlib.Path, version: str = "0.2.1rc19", requires: str | None = ">=3.14", payload: bytes = b"") -> pathlib.Path:
42 path = tmp / f"muse-{version}.tar.gz"
43 info = f"Metadata-Version: 2.1\nName: muse\nVersion: {version}\n" + (f"Requires-Python: {requires}\n" if requires else "")
44 with tarfile.open(path, "w:gz") as tf:
45 data = info.encode()
46 member = tarfile.TarInfo(f"muse-{version}/PKG-INFO")
47 member.size = len(data)
48 tf.addfile(member, io.BytesIO(data))
49 if payload:
50 extra = tarfile.TarInfo(f"muse-{version}/payload")
51 extra.size = len(payload)
52 tf.addfile(extra, io.BytesIO(payload))
53 return path
54
55
56 class FakeRunner:
57 """Pretends to be the interpreter: records argv/env, creates the venv python, answers probes and verification."""
58
59 def __init__(self, *, version: str = "3.14.4", impl: str = "cpython", fail: str | None = None, problems: list[str] | None = None,
60 installed: Mapping[str, str] | None = None, archive_sha: str | None = None, make_venv: bool = True) -> None:
61 self.calls: list[tuple[list[str], dict[str, str] | None]] = []
62 self.version, self.impl, self.fail, self.problems = version, impl, fail, problems or []
63 self.installed = dict(installed) if installed is not None else None
64 self.archive_sha, self.make_venv = archive_sha, make_venv
65 self.pins: dict[str, str] = {}
66
67 def __call__(self, argv: list[str], **kw: Any) -> subprocess.CompletedProcess[str]:
68 env = kw.get("env")
69 self.calls.append((list(argv), env))
70 text = " ".join(argv)
71 if self.fail and self.fail in text:
72 return subprocess.CompletedProcess(argv, 1, "", f"simulated failure of {self.fail}")
73 if "sys.implementation.name" in text:
74 return subprocess.CompletedProcess(argv, 0, json.dumps({"v": self.version, "impl": self.impl, "exe": argv[0]}), "")
75 if "-m venv" in text:
76 if self.make_venv:
77 target = pathlib.Path(argv[-1]) / "bin"
78 target.mkdir(parents=True)
79 (target / "python").write_text("#!/bin/sh\n")
80 return subprocess.CompletedProcess(argv, 0, "", "")
81 if "importlib.metadata" in text:
82 version = argv[argv.index("-c") + 3]
83 pins = json.loads(argv[argv.index("-c") + 2])
84 installed = self.installed if self.installed is not None else {**pins, "muse": version}
85 return subprocess.CompletedProcess(argv, 0, json.dumps({"problems": self.problems, "installed": installed, "archive_sha256": self.archive_sha, "grammar_modules": []}), "")
86 return subprocess.CompletedProcess(argv, 0, "", "")
87
88 def pip_calls(self) -> list[list[str]]:
89 return [a for a, _ in self.calls if "pip" in a and "install" in a]
90
91
92 @pytest.fixture
93 def reqs(tmp_path: pathlib.Path) -> pathlib.Path:
94 path = tmp_path / "reqs.txt"
95 path.write_text("pytest==9.0.3\nbuild==1.2.2\npackaging==26.0\n")
96 return path
97
98
99 @pytest.fixture
100 def art(tmp_path: pathlib.Path) -> pathlib.Path:
101 (tmp_path / "a").mkdir()
102 return sdist(tmp_path / "a")
103
104
105 @pytest.fixture
106 def interpreter(tmp_path: pathlib.Path) -> Interpreter:
107 return Interpreter(pathlib.Path(PY), "3.14.4", "cpython")
108
109
110 def fake_exe(tmp: pathlib.Path) -> pathlib.Path:
111 exe = tmp / "python3.14"
112 exe.write_text("#!/bin/sh\n")
113 exe.chmod(0o755)
114 return exe
115
116
117 # ── interpreter selection ─────────────────────────────────────────────────────
118
119
120 class TestInterpreterSelection:
121 def test_the_candidates_own_requires_python_is_the_contract(self, art: pathlib.Path) -> None:
122 assert requires_python_of(art) == ">=3.14"
123
124 def test_artifact_without_requires_python_or_pkg_info_is_refused(self, tmp_path: pathlib.Path) -> None:
125 with pytest.raises(GateEnvError, match="Requires-Python"):
126 requires_python_of(sdist(tmp_path, requires=None))
127 bogus = tmp_path / "muse-1.0.tar.gz"
128 bogus.write_bytes(b"not a tarball")
129 with pytest.raises(GateEnvError, match="PKG-INFO"):
130 requires_python_of(bogus)
131
132 @pytest.mark.parametrize("python", [None, "", "python3", "relative/python", "/nonexistent/python3.14"])
133 def test_missing_or_non_explicit_interpreter_is_refused_and_nothing_runs(self, python: str | None) -> None:
134 runner = FakeRunner()
135 with pytest.raises(GateEnvError, match="interpreter|absolute|explicit"):
136 select_interpreter(python, ">=3.14", runner=runner)
137 assert runner.calls == []
138
139 def test_non_executable_file_is_refused(self, tmp_path: pathlib.Path) -> None:
140 exe = tmp_path / "python3.14"
141 exe.write_text("")
142 with pytest.raises(GateEnvError, match="executable"):
143 select_interpreter(exe, ">=3.14", runner=FakeRunner())
144
145 def test_unsupported_python_version_is_refused(self, tmp_path: pathlib.Path) -> None:
146 with pytest.raises(GateEnvError, match="requires >=3.14"):
147 select_interpreter(fake_exe(tmp_path), ">=3.14", runner=FakeRunner(version="3.13.1"))
148
149 def test_other_implementations_are_refused(self, tmp_path: pathlib.Path) -> None:
150 with pytest.raises(GateEnvError, match="CPython"):
151 select_interpreter(fake_exe(tmp_path), ">=3.14", runner=FakeRunner(impl="pypy"))
152
153 def test_probe_failure_is_refused(self, tmp_path: pathlib.Path) -> None:
154 with pytest.raises(GateEnvError, match="probing"):
155 select_interpreter(fake_exe(tmp_path), ">=3.14", runner=FakeRunner(fail="sys.implementation.name"))
156
157 def test_the_probe_runs_isolated_so_the_callers_environment_cannot_change_the_answer(self, tmp_path: pathlib.Path) -> None:
158 runner = FakeRunner()
159 select_interpreter(fake_exe(tmp_path), ">=3.14", runner=runner)
160 assert "-I" in runner.calls[0][0]
161
162 def test_the_real_interpreter_probe_accepts_this_python(self) -> None:
163 real = pathlib.Path(sys.base_prefix) / "bin" / "python3"
164 if not real.is_file():
165 pytest.skip("no base interpreter at the expected path")
166 info = select_interpreter(real, f">={sys.version_info.major}.{sys.version_info.minor}")
167 assert info.implementation == "cpython" and info.version.startswith(f"{sys.version_info.major}.{sys.version_info.minor}")
168 with pytest.raises(GateEnvError, match="requires"):
169 select_interpreter(real, ">=99")
170
171
172 # ── pins ──────────────────────────────────────────────────────────────────────
173
174
175 class TestPins:
176 def test_exact_pins_parse_with_canonical_names(self) -> None:
177 assert parse_requirements("PyTest==9.0.3 # tool\n\nTree_Sitter==0.25.2\n") == {"pytest": "9.0.3", "tree-sitter": "0.25.2"}
178
179 @pytest.mark.parametrize("text", ["pytest>=9", "pytest", "pytest~=9.0", "pytest==9.*", "pytest==9,<10", "", "# only a comment\n", "pytest==1\npytest==2", "not a requirement !!"])
180 def test_anything_but_an_exact_pin_is_refused(self, text: str) -> None:
181 with pytest.raises(GateEnvError):
182 parse_requirements(text)
183
184 def test_the_shipped_requirements_are_exact_and_cover_the_gate_tooling(self) -> None:
185 pins = parse_requirements(gate_env.REQUIREMENTS.read_text())
186 for needed in ("pytest", "build", "packaging", "fastapi", "pydantic", "boto3", "pytest-json-report"):
187 assert needed in pins, needed
188 assert "muse" not in pins, "the candidate comes from the artifact, never from a pin"
189
190
191 # ── building ──────────────────────────────────────────────────────────────────
192
193
194 class TestBuild:
195 def build(self, tmp: pathlib.Path, art: pathlib.Path, interp: Interpreter, reqs: pathlib.Path, runner: FakeRunner) -> GateEnv:
196 return build_env(tmp / "work" / "env", art, interp, requirements=reqs, runner=runner)
197
198 def test_happy_path_installs_the_pins_then_the_exact_artifact_with_all_extra(self, tmp_path: pathlib.Path, art: pathlib.Path, interpreter: Interpreter, reqs: pathlib.Path) -> None:
199 runner = FakeRunner()
200 env = self.build(tmp_path, art, interpreter, reqs, runner)
201 venv_py = str(env.root / "venv" / "bin" / "python")
202 pips = runner.pip_calls()
203 assert [p[0] for p in pips] == [venv_py, venv_py], "pip is only ever run by the venv's own interpreter"
204 assert pips[0][-2:] == ["-r", str(reqs)]
205 assert pips[1][-3:-1] == ["-c", str(reqs)] and pips[1][-1].endswith(f"candidate/{art.name}[all]")
206 assert (env.root / "candidate" / art.name).read_bytes() == art.read_bytes(), "installs a hash-checked copy of the artifact"
207 assert env.packages["muse"] == "0.2.1rc19" and env.candidate_sha256 == sha256_of(art)
208 verify_complete(env, art)
209
210 def test_global_python_is_only_asked_to_make_a_venv(self, tmp_path: pathlib.Path, art: pathlib.Path, interpreter: Interpreter, reqs: pathlib.Path) -> None:
211 runner = FakeRunner()
212 self.build(tmp_path, art, interpreter, reqs, runner)
213 by_global = [a for a, _ in runner.calls if a[0] == PY]
214 assert len(by_global) == 1 and by_global[0][1:4] == ["-I", "-m", "venv"]
215 assert not any(a[0] == PY and "pip" in a for a, _ in runner.calls)
216
217 def test_every_subprocess_is_scrubbed_of_the_callers_python_settings(self, tmp_path: pathlib.Path, art: pathlib.Path, interpreter: Interpreter, reqs: pathlib.Path, monkeypatch: pytest.MonkeyPatch) -> None:
218 for name, value in (("PYTHONPATH", "/dev/checkout"), ("PIP_INDEX_URL", "http://evil"), ("PYTHONHOME", "/x"), ("VIRTUAL_ENV", "/dev/venv"), ("PIP_USER", "1")):
219 monkeypatch.setenv(name, value)
220 runner = FakeRunner()
221 self.build(tmp_path, art, interpreter, reqs, runner)
222 for argv, env in runner.calls:
223 if argv[0] == PY and "sys.implementation.name" in " ".join(argv):
224 continue
225 assert env is not None
226 assert "PYTHONPATH" not in env and "PYTHONHOME" not in env and "PIP_INDEX_URL" not in env and "PIP_USER" not in env
227 assert env["PYTHONNOUSERSITE"] == "1" and env["PIP_REQUIRE_VIRTUALENV"] == "1"
228
229 def test_gate_processes_get_the_venv_first_on_path_and_none_of_the_callers_python_env(self, tmp_path: pathlib.Path, art: pathlib.Path, interpreter: Interpreter, reqs: pathlib.Path, monkeypatch: pytest.MonkeyPatch) -> None:
230 monkeypatch.setenv("PYTHONPATH", "/dev/checkout")
231 env = self.build(tmp_path, art, interpreter, reqs, FakeRunner())
232 proc_env = gate_process_env(env)
233 assert proc_env["PATH"].startswith(str(env.root / "venv" / "bin")) and "PYTHONPATH" not in proc_env
234
235 def test_dependency_installation_failure_blocks(self, tmp_path: pathlib.Path, art: pathlib.Path, interpreter: Interpreter, reqs: pathlib.Path) -> None:
236 runner = FakeRunner(fail=f"-r {reqs}")
237 with pytest.raises(GateEnvError, match="pinned gate tooling"):
238 self.build(tmp_path, art, interpreter, reqs, runner)
239 assert len(runner.pip_calls()) == 0 or "candidate" not in " ".join(runner.pip_calls()[-1])
240
241 def test_grammar_installation_failure_blocks(self, tmp_path: pathlib.Path, art: pathlib.Path, interpreter: Interpreter, reqs: pathlib.Path) -> None:
242 with pytest.raises(GateEnvError, match=r"\[all\] grammars"):
243 self.build(tmp_path, art, interpreter, reqs, FakeRunner(fail="[all]"))
244
245 def test_a_grammar_that_installed_but_does_not_import_blocks(self, tmp_path: pathlib.Path, art: pathlib.Path, interpreter: Interpreter, reqs: pathlib.Path) -> None:
246 with pytest.raises(GateEnvError, match="grammar tree_sitter_go"):
247 self.build(tmp_path, art, interpreter, reqs, FakeRunner(problems=["grammar tree_sitter_go: ImportError: boom"]))
248
249 def test_a_missing_test_dependency_blocks(self, tmp_path: pathlib.Path, art: pathlib.Path, interpreter: Interpreter, reqs: pathlib.Path) -> None:
250 with pytest.raises(GateEnvError, match="missing pytest==9.0.3"):
251 self.build(tmp_path, art, interpreter, reqs, FakeRunner(problems=["missing pytest==9.0.3"]))
252
253 def test_a_wrong_candidate_in_the_environment_blocks(self, tmp_path: pathlib.Path, art: pathlib.Path, interpreter: Interpreter, reqs: pathlib.Path) -> None:
254 with pytest.raises(GateEnvError, match="installed muse is 0.2.1rc18"):
255 self.build(tmp_path, art, interpreter, reqs, FakeRunner(problems=["installed muse is 0.2.1rc18, candidate is 0.2.1rc19"]))
256
257 def test_muse_imported_from_outside_the_venv_blocks(self, tmp_path: pathlib.Path, art: pathlib.Path, interpreter: Interpreter, reqs: pathlib.Path) -> None:
258 with pytest.raises(GateEnvError, match="outside the gate venv"):
259 self.build(tmp_path, art, interpreter, reqs, FakeRunner(problems=["muse imported from /dev/muse/__init__.py, outside the gate venv"]))
260
261 def test_pip_recording_a_different_archive_hash_blocks(self, tmp_path: pathlib.Path, art: pathlib.Path, interpreter: Interpreter, reqs: pathlib.Path) -> None:
262 with pytest.raises(GateEnvError, match="not the candidate"):
263 self.build(tmp_path, art, interpreter, reqs, FakeRunner(archive_sha="0" * 64))
264
265 def test_unreadable_verification_output_blocks(self, tmp_path: pathlib.Path, art: pathlib.Path, interpreter: Interpreter, reqs: pathlib.Path) -> None:
266 class Garbled(FakeRunner):
267 def __call__(self, argv: list[str], **kw: Any) -> subprocess.CompletedProcess[str]:
268 if "importlib.metadata" in " ".join(argv):
269 return subprocess.CompletedProcess(argv, 0, "no json here", "")
270 return super().__call__(argv, **kw)
271
272 with pytest.raises(GateEnvError, match="unreadable"):
273 self.build(tmp_path, art, interpreter, reqs, Garbled())
274
275 def test_a_venv_without_an_interpreter_is_a_partial_environment(self, tmp_path: pathlib.Path, art: pathlib.Path, interpreter: Interpreter, reqs: pathlib.Path) -> None:
276 with pytest.raises(GateEnvError, match="no interpreter"):
277 self.build(tmp_path, art, interpreter, reqs, FakeRunner(make_venv=False))
278
279 def test_an_unpinned_requirements_file_is_refused_before_anything_is_created(self, tmp_path: pathlib.Path, art: pathlib.Path, interpreter: Interpreter) -> None:
280 loose = tmp_path / "loose.txt"
281 loose.write_text("pytest>=9\n")
282 runner = FakeRunner()
283 with pytest.raises(GateEnvError, match="exact"):
284 self.build(tmp_path, art, interpreter, loose, runner)
285 assert runner.calls == [] and not (tmp_path / "work").exists()
286
287
288 class TestVerificationScript:
289 """Run the real in-venv verification script against a stub ``muse`` so its decisions are exercised, not mocked."""
290
291 def stub(self, tmp: pathlib.Path, specs: str, version: str = "0.2.1rc19", present: tuple[str, ...] = ("fake_full_grammar", "tree_sitter_markdown", "tree_sitter_html")) -> pathlib.Path:
292 site = tmp / "site"
293 for rel in ("muse/plugins/code",):
294 (site / rel).mkdir(parents=True)
295 for pkg in ("muse", "muse/plugins", "muse/plugins/code"):
296 (site / pkg / "__init__.py").write_text("")
297 (site / "muse/plugins/code/ast_parser.py").write_text(f"_TS_LANG_SPECS = {specs}\n")
298 for mod in present:
299 (site / f"{mod}.py").write_text("")
300 info = site / f"muse-{version}.dist-info"
301 info.mkdir()
302 (info / "METADATA").write_text(f"Metadata-Version: 2.1\nName: muse\nVersion: {version}\n")
303 return site
304
305 def run(self, site: pathlib.Path, pins: dict[str, str], version: str = "0.2.1rc19") -> dict[str, Any]:
306 import os
307
308 proc = subprocess.run(
309 [sys.executable, "-c", gate_env._VERIFY, json.dumps(pins), version, str(site)],
310 capture_output=True, text=True, env={**os.environ, "PYTHONPATH": str(site)}, timeout=60,
311 )
312 assert proc.returncode == 0, proc.stderr
313 return json.loads(proc.stdout.strip().splitlines()[-1]) # type: ignore[no-any-return]
314
315 SPECS = '[{"module_name": "fake_full_grammar", "profile": "full"}, {"module_name": "py_tree_sitter_swift", "profile": "opt_in"}]'
316
317 def test_opt_in_grammars_are_not_required_but_guaranteed_ones_are(self, tmp_path: pathlib.Path) -> None:
318 result = self.run(self.stub(tmp_path, self.SPECS), {})
319 assert result["problems"] == [], "Swift is opt-in and must not fail the gate environment"
320 assert "py_tree_sitter_swift" not in result["grammar_modules"] and "fake_full_grammar" in result["grammar_modules"]
321
322 def test_a_missing_guaranteed_grammar_is_reported(self, tmp_path: pathlib.Path) -> None:
323 result = self.run(self.stub(tmp_path, self.SPECS, present=("tree_sitter_markdown", "tree_sitter_html")), {})
324 assert any("fake_full_grammar" in p for p in result["problems"])
325
326 def test_a_missing_or_wrong_version_pin_is_reported(self, tmp_path: pathlib.Path) -> None:
327 site = self.stub(tmp_path, self.SPECS)
328 result = self.run(site, {"nonexistent-package": "1.0", "packaging": "0.0.1"})
329 assert any("missing nonexistent-package==1.0" in p for p in result["problems"])
330 assert any(p.startswith("packaging is ") and "pinned 0.0.1" in p for p in result["problems"])
331
332 def test_a_different_installed_muse_version_is_reported(self, tmp_path: pathlib.Path) -> None:
333 result = self.run(self.stub(tmp_path, self.SPECS, version="0.2.1rc18"), {}, version="0.2.1rc19")
334 assert any("installed muse is 0.2.1rc18" in p for p in result["problems"])
335
336 def test_muse_imported_from_outside_the_environment_root_is_reported(self, tmp_path: pathlib.Path) -> None:
337 site = self.stub(tmp_path, self.SPECS)
338 import os
339
340 proc = subprocess.run(
341 [sys.executable, "-c", gate_env._VERIFY, "{}", "0.2.1rc19", str(tmp_path / "somewhere-else")],
342 capture_output=True, text=True, env={**os.environ, "PYTHONPATH": str(site)}, timeout=60,
343 )
344 assert any("outside the gate venv" in p for p in json.loads(proc.stdout.strip().splitlines()[-1])["problems"])
345
346
347 class TestStaleAndPartialEnvironments:
348 def test_an_existing_environment_directory_is_never_reused(self, tmp_path: pathlib.Path, art: pathlib.Path, interpreter: Interpreter, reqs: pathlib.Path) -> None:
349 stale = tmp_path / "old-env"
350 (stale / "venv" / "bin").mkdir(parents=True)
351 runner = FakeRunner()
352 with pytest.raises(GateEnvError, match="never reused"):
353 build_env(stale, art, interpreter, requirements=reqs, runner=runner)
354 assert runner.calls == []
355
356 def test_an_environment_without_its_completion_marker_is_refused(self, tmp_path: pathlib.Path, art: pathlib.Path, interpreter: Interpreter, reqs: pathlib.Path) -> None:
357 env = build_env(tmp_path / "e", art, interpreter, requirements=reqs, runner=FakeRunner())
358 (env.root / gate_env.MARKER).unlink()
359 with pytest.raises(GateEnvError, match="not a complete gate environment"):
360 verify_complete(env, art)
361
362 def test_an_environment_built_for_another_artifact_is_refused(self, tmp_path: pathlib.Path, art: pathlib.Path, interpreter: Interpreter, reqs: pathlib.Path) -> None:
363 env = build_env(tmp_path / "e", art, interpreter, requirements=reqs, runner=FakeRunner())
364 (tmp_path / "o").mkdir()
365 other = sdist(tmp_path / "o", payload=b"different bytes, same version")
366 with pytest.raises(GateEnvError, match="different artifact"):
367 verify_complete(env, other)
368
369 def test_the_marker_is_written_last_so_a_failed_build_never_has_one(self, tmp_path: pathlib.Path, art: pathlib.Path, interpreter: Interpreter, reqs: pathlib.Path) -> None:
370 root = tmp_path / "e"
371 with pytest.raises(GateEnvError):
372 build_env(root, art, interpreter, requirements=reqs, runner=FakeRunner(fail="[all]"))
373 assert not (root / gate_env.MARKER).exists()
374
375
376 class TestLifecycle:
377 def make(self, tmp: pathlib.Path, art: pathlib.Path, reqs: pathlib.Path, runner: FakeRunner, **kw: Any) -> Any:
378 return gate_environment(art, fake_exe(tmp), parent=tmp / "scratch", requirements=reqs, runner=runner, **kw)
379
380 @pytest.fixture(autouse=True)
381 def scratch(self, tmp_path: pathlib.Path) -> None:
382 (tmp_path / "scratch").mkdir()
383
384 def leftovers(self, tmp: pathlib.Path) -> list[str]:
385 return [p.name for p in (tmp / "scratch").iterdir()]
386
387 def test_the_environment_exists_inside_and_is_gone_after(self, tmp_path: pathlib.Path, art: pathlib.Path, reqs: pathlib.Path) -> None:
388 with self.make(tmp_path, art, reqs, FakeRunner()) as env:
389 assert env.python.is_file() and self.leftovers(tmp_path)
390 assert self.leftovers(tmp_path) == []
391
392 def test_a_failure_while_building_leaves_nothing_behind(self, tmp_path: pathlib.Path, art: pathlib.Path, reqs: pathlib.Path) -> None:
393 with pytest.raises(GateEnvError):
394 with self.make(tmp_path, art, reqs, FakeRunner(fail="[all]")):
395 pytest.fail("must not run gates in a broken environment")
396 assert self.leftovers(tmp_path) == []
397
398 def test_an_interrupted_run_is_cleaned_up_and_the_interrupt_propagates(self, tmp_path: pathlib.Path, art: pathlib.Path, reqs: pathlib.Path) -> None:
399 with pytest.raises(KeyboardInterrupt):
400 with self.make(tmp_path, art, reqs, FakeRunner()):
401 raise KeyboardInterrupt
402 assert self.leftovers(tmp_path) == []
403
404 def test_failure_to_remove_the_environment_is_an_error_not_a_pass(self, tmp_path: pathlib.Path, art: pathlib.Path, reqs: pathlib.Path, monkeypatch: pytest.MonkeyPatch) -> None:
405 monkeypatch.setattr(gate_env.shutil, "rmtree", lambda *a, **k: None)
406 with pytest.raises(GateEnvError, match="could not remove"):
407 with self.make(tmp_path, art, reqs, FakeRunner()):
408 pass
409
410 def test_unsupported_interpreter_never_creates_anything(self, tmp_path: pathlib.Path, art: pathlib.Path, reqs: pathlib.Path) -> None:
411 with pytest.raises(GateEnvError, match="requires"):
412 with self.make(tmp_path, art, reqs, FakeRunner(version="3.12.0")):
413 pass
414 assert self.leftovers(tmp_path) == []
415
416
417 # ── the gate run uses it, and reports it ──────────────────────────────────────
418
419
420 class TestGateRunUsesTheEnvironment:
421 def run(self, tmp: pathlib.Path, art: pathlib.Path, reqs: pathlib.Path, runner: FakeRunner, gate_runner: Any, python: Any = "ok") -> dict[str, Any]:
422 (tmp / "scratch").mkdir(exist_ok=True)
423 from deploy.release_gate import run_release_gates as rrg
424 from tests.test_release_provenance import Observer
425
426 return rrg(
427 art, tmp / "muse", tmp / "hub", SOURCE_REVISION, 2, python=fake_exe(tmp) if python == "ok" else python, now=NOW,
428 observe=Observer(CHECKOUT), run=gate_runner,
429 environment=lambda a, p: gate_environment(a, p, parent=tmp / "scratch", requirements=reqs, runner=runner),
430 )
431
432 def artifact(self, tmp: pathlib.Path) -> pathlib.Path:
433 (tmp / "cand").mkdir(exist_ok=True)
434 return sdist(tmp / "cand")
435
436 def test_every_gate_command_is_run_by_the_environments_interpreter_never_the_callers(self, tmp_path: pathlib.Path) -> None:
437 cmds = default_gate_commands(tmp_path / "muse-0.2.1rc19.tar.gz", tmp_path / "muse", tmp_path / "hub", "/gate/venv/bin/python")
438 for gate, argv in cmds.items():
439 assert "/gate/venv/bin/python" in argv, gate
440 assert sys.executable not in argv, gate
441 assert "python3" not in argv and "python" not in argv, gate
442
443 def test_gates_receive_the_scrubbed_env_and_the_gate_interpreter(self, tmp_path: pathlib.Path, reqs: pathlib.Path, monkeypatch: pytest.MonkeyPatch) -> None:
444 monkeypatch.setenv("PYTHONPATH", "/dev/checkout")
445 seen: dict[str, Any] = {}
446
447 def gates(commands: Any, *, env: Mapping[str, str], cwds: Any) -> dict[str, GateResult]:
448 seen.update(env=dict(env), commands=commands)
449 return passing_results()
450
451 report = self.run(tmp_path, self.artifact(tmp_path), reqs, FakeRunner(), gates)
452 env = seen["env"]
453 assert env["TEST_DATABASE_URL"].endswith(".invalid:1/none"), "gates must not be able to reach a real database"
454 assert "PYTHONPATH" not in env and env["MUSE_GATE_PYTHON"].endswith("/venv/bin/python") and env["PIP_REQUIRE_VIRTUALENV"] == "1"
455 assert all(cmd.count(env["MUSE_GATE_PYTHON"]) >= 1 for cmd in seen["commands"].values())
456 assert report["decision"] == "pass"
457
458 def test_the_report_records_interpreter_packages_and_cleanup(self, tmp_path: pathlib.Path, reqs: pathlib.Path) -> None:
459 art = self.artifact(tmp_path)
460 report = self.run(tmp_path, art, reqs, FakeRunner(), lambda c, **k: passing_results())
461 e = report["environment"]
462 assert e["interpreter"] == {"path": str(tmp_path / "python3.14"), "version": "3.14.4", "implementation": "cpython"}
463 assert e["isolated"] is True and e["cleanup"] == "removed" and e["candidate_sha256"] == sha256_of(art)
464 assert e["packages"]["muse"] == "0.2.1rc19" and e["packages"]["pytest"] == "9.0.3"
465 assert len(e["packages_sha256"]) == 64 and len(e["requirements_sha256"]) == 64
466 verify_report(json.dumps(report), art, now=NOW, checkout=CHECKOUT)
467
468 def test_a_failing_gate_still_cleans_up_and_cannot_authorize(self, tmp_path: pathlib.Path, reqs: pathlib.Path) -> None:
469 art = self.artifact(tmp_path)
470 failing = {**passing_results(), "runtime.doctor": GateResult("fail", 1, "x")}
471 report = self.run(tmp_path, art, reqs, FakeRunner(), lambda c, **k: failing)
472 assert report["decision"] == "fail" and not list((tmp_path / "scratch").iterdir())
473 with pytest.raises(GateError, match="runtime.doctor"):
474 verify_report(json.dumps(report), art, now=NOW, checkout=CHECKOUT)
475
476 @pytest.mark.parametrize("fail", ["[all]", f"-r ", "-m venv"])
477 def test_any_environment_failure_yields_no_report_and_no_gate_runs(self, tmp_path: pathlib.Path, reqs: pathlib.Path, fail: str) -> None:
478 ran: list[int] = []
479 with pytest.raises(GateError, match="hermetic gate environment"):
480 self.run(tmp_path, self.artifact(tmp_path), reqs, FakeRunner(fail=fail), lambda c, **k: ran.append(1) or passing_results())
481 assert ran == [] and not list((tmp_path / "scratch").iterdir())
482
483 def test_missing_interpreter_yields_no_report(self, tmp_path: pathlib.Path, reqs: pathlib.Path) -> None:
484 with pytest.raises(GateError, match="explicit"):
485 self.run(tmp_path, self.artifact(tmp_path), reqs, FakeRunner(), lambda c, **k: passing_results(), python=None)
486
487 def test_interruption_during_the_gates_yields_no_report_and_no_environment(self, tmp_path: pathlib.Path, reqs: pathlib.Path) -> None:
488 def interrupted(*_a: object, **_k: object) -> dict[str, GateResult]:
489 raise KeyboardInterrupt
490
491 with pytest.raises(KeyboardInterrupt):
492 self.run(tmp_path, self.artifact(tmp_path), reqs, FakeRunner(), interrupted)
493 assert not list((tmp_path / "scratch").iterdir())
494
495
496 class TestVerifyReportEnvironment:
497 @pytest.fixture
498 def art(self, tmp_path: pathlib.Path) -> pathlib.Path:
499 return make_artifact(tmp_path, "0.2.1rc19")
500
501 def verify(self, art: pathlib.Path, text: str) -> None:
502 verify_report(text, art, now=NOW, checkout=CHECKOUT)
503
504 def test_a_report_without_environment_evidence_is_rejected(self, art: pathlib.Path) -> None:
505 data = json.loads(make_report(art))
506 del data["environment"]
507 with pytest.raises(GateError, match="environment is required"):
508 self.verify(art, json.dumps(data))
509
510 @pytest.mark.parametrize(
511 "change, match",
512 [
513 ({"environment__isolated": False}, "isolated"),
514 ({"environment__cleanup": "pending"}, "cleanup"),
515 ({"environment__cleanup": "unknown"}, "cleanup"),
516 ({"environment__candidate_sha256": "0" * 64}, "different artifact"),
517 ({"environment__packages": {}}, "packages"),
518 ({"environment__packages": {"muse": "0.2.1rc18"}}, "not the candidate"),
519 ({"environment__packages": {"pytest": "9"}}, "installed muse None"),
520 ({"environment__interpreter": {"path": "", "version": "3", "implementation": "cpython"}}, "interpreter"),
521 ({"environment__interpreter": "python"}, "interpreter"),
522 ({"environment__packages_sha256": "xyz"}, "packages_sha256"),
523 ({"environment__requirements_sha256": None}, "requirements_sha256"),
524 ({"environment": "isolated"}, "environment is required"),
525 ],
526 )
527 def test_unverifiable_environment_evidence_is_rejected(self, art: pathlib.Path, change: dict[str, object], match: str) -> None:
528 with pytest.raises(GateError, match=match):
529 self.verify(art, mutate_report(make_report(art), **change))
530
531 def test_good_evidence_is_accepted(self, art: pathlib.Path) -> None:
532 self.verify(art, make_report(art, environment=good_environment(sha256_of(art), "0.2.1rc19")))
File History 1 commit
sha256:a977b39d08e15683210a33608e25712fe3ed9c8f05f0388fb69ef38acf835426 feat(release): hermetic release gates in a provisioned, pin… Opus 5.5 minor ⚠ 12 days ago