gabriel / musehub public
test_release_gate.py python
203 lines 10.1 KB
Raw
sha256:982b4ce08b140ea3fa9057705ff1806a702c9203689c22b797a19f9db678fe5f fix(release-gate): install gates select the opt-in install_… Opus 5.5 patch 2 days ago
1 """Release gate report: construction, binding to the artifact, and the runner (musehub#114 Phase 6).
2
3 A gate report is evidence about *one exact artifact*. Everything here proves the
4 verifier rejects anything that is not complete, passing, fresh and bound.
5 """
6
7 from __future__ import annotations
8
9 import json
10 import pathlib
11 import subprocess
12 import sys
13 from datetime import timedelta
14
15 import pytest
16
17 from deploy.release_gate import REQUIRED_GATES, GateError, GateResult, run_gates, verify_report
18 from tests.release_fixtures import CHECKOUT, NOW, STALE, make_artifact, make_report, mutate_report, passing_results
19
20
21 @pytest.fixture
22 def artifact(tmp_path: pathlib.Path) -> pathlib.Path:
23 return make_artifact(tmp_path, "0.2.1rc19")
24
25
26 class TestVerifyReport:
27 def test_complete_report_bound_to_the_artifact_is_accepted(self, artifact: pathlib.Path) -> None:
28 candidate = verify_report(make_report(artifact), artifact, now=NOW, checkout=CHECKOUT)
29 assert candidate.version == "0.2.1rc19"
30 assert candidate.sha256 and candidate.source_revision and candidate.installer_revision == 2
31
32 def test_required_gates_cover_packaging_install_and_runtime(self) -> None:
33 names = set(REQUIRED_GATES)
34 for needed in (
35 "packaging.sdist_reproducible", "packaging.version", "packaging.dependency_metadata",
36 "packaging.manifest_contract", "install.full", "install.slim", "install.legacy_upgrade",
37 "install.profile_preservation", "install.explicit_grammar", "runtime.doctor",
38 "runtime.grammar_availability", "runtime.installed_parser_parity",
39 ):
40 assert needed in names
41
42 @pytest.mark.parametrize("gate", REQUIRED_GATES)
43 def test_every_required_gate_failing_blocks(self, artifact: pathlib.Path, gate: str) -> None:
44 results = passing_results()
45 results[gate] = GateResult(status="fail", exit_code=1, detail="boom")
46 with pytest.raises(GateError, match=gate):
47 verify_report(make_report(artifact, results=results), artifact, now=NOW, checkout=CHECKOUT)
48
49 @pytest.mark.parametrize("gate", REQUIRED_GATES)
50 def test_every_required_gate_missing_blocks(self, artifact: pathlib.Path, gate: str) -> None:
51 results = passing_results()
52 del results[gate]
53 with pytest.raises(GateError, match=gate):
54 verify_report(make_report(artifact, results=results), artifact, now=NOW, checkout=CHECKOUT)
55
56 def test_pass_status_with_nonzero_exit_code_is_inconsistent_and_blocks(self, artifact: pathlib.Path) -> None:
57 results = passing_results()
58 results["install.slim"] = GateResult(status="pass", exit_code=2, detail="")
59 with pytest.raises(GateError, match="install.slim"):
60 verify_report(make_report(artifact, results=results), artifact, now=NOW, checkout=CHECKOUT)
61
62 def test_decision_field_is_never_trusted(self, artifact: pathlib.Path) -> None:
63 results = passing_results()
64 results["runtime.doctor"] = GateResult(status="fail", exit_code=1, detail="x")
65 forged = mutate_report(make_report(artifact, results=results), decision="pass")
66 with pytest.raises(GateError):
67 verify_report(forged, artifact, now=NOW, checkout=CHECKOUT)
68
69 def test_report_for_a_different_version_does_not_authorize_this_one(self, artifact: pathlib.Path, tmp_path: pathlib.Path) -> None:
70 (tmp_path / "old").mkdir()
71 rc18 = make_artifact(tmp_path / "old", "0.2.1rc18")
72 with pytest.raises(GateError, match="version"):
73 verify_report(make_report(rc18), artifact, now=NOW, checkout=CHECKOUT)
74
75 def test_report_for_a_different_hash_of_the_same_version_is_rejected(self, artifact: pathlib.Path) -> None:
76 report = make_report(artifact)
77 artifact.write_bytes(b"same version, different bytes")
78 with pytest.raises(GateError, match="sha256"):
79 verify_report(report, artifact, now=NOW, checkout=CHECKOUT)
80
81 def test_stale_report_is_rejected(self, artifact: pathlib.Path) -> None:
82 with pytest.raises(GateError, match="stale"):
83 verify_report(make_report(artifact, created_at=STALE), artifact, now=NOW, checkout=CHECKOUT)
84
85 def test_report_from_the_future_is_rejected(self, artifact: pathlib.Path) -> None:
86 with pytest.raises(GateError, match="future"):
87 verify_report(make_report(artifact, created_at=NOW + timedelta(hours=2)), artifact, now=NOW, checkout=CHECKOUT)
88
89 @pytest.mark.parametrize("text", ["", "not json", "[]", "null", '{"schema": 1}', '"pass"'])
90 def test_malformed_report_is_rejected(self, artifact: pathlib.Path, text: str) -> None:
91 with pytest.raises(GateError):
92 verify_report(text, artifact, now=NOW, checkout=CHECKOUT)
93
94 @pytest.mark.parametrize("change", [
95 {"schema": 1},
96 {"schema": 3},
97 {"candidate__source_revision": ""},
98 {"candidate__source_revision": "abc123"},
99 {"candidate__installer_revision": "2"},
100 {"candidate__sha256": "XYZ"},
101 {"created_at": "yesterday"},
102 {"gates": "all of them"},
103 ])
104 def test_schema_violations_are_rejected(self, artifact: pathlib.Path, change: dict[str, object]) -> None:
105 with pytest.raises(GateError):
106 verify_report(mutate_report(make_report(artifact), **change), artifact, now=NOW, checkout=CHECKOUT)
107
108 def test_duplicate_gate_entries_are_rejected(self, artifact: pathlib.Path) -> None:
109 data = json.loads(make_report(artifact))
110 data["gates"].append(dict(data["gates"][0]))
111 with pytest.raises(GateError, match="duplicate"):
112 verify_report(json.dumps(data), artifact, now=NOW, checkout=CHECKOUT)
113
114 def test_an_extra_failing_gate_also_blocks(self, artifact: pathlib.Path) -> None:
115 data = json.loads(make_report(artifact))
116 data["gates"].append({"name": "extra.lint", "status": "fail", "exit_code": 1, "detail": ""})
117 with pytest.raises(GateError, match="extra.lint"):
118 verify_report(json.dumps(data), artifact, now=NOW, checkout=CHECKOUT)
119
120 def test_artifact_filename_must_carry_the_candidate_version(self, tmp_path: pathlib.Path) -> None:
121 odd = tmp_path / "candidate.tar.gz"
122 odd.write_bytes(b"x")
123 with pytest.raises(GateError, match="filename"):
124 verify_report(make_report(odd, version="0.2.1rc19"), odd, now=NOW, checkout=CHECKOUT)
125
126 def test_missing_artifact_is_rejected(self, artifact: pathlib.Path) -> None:
127 report = make_report(artifact)
128 artifact.unlink()
129 with pytest.raises(GateError, match="artifact"):
130 verify_report(report, artifact, now=NOW, checkout=CHECKOUT)
131
132
133 class TestRunGates:
134 def _cmds(self, exit_code: int = 0) -> dict[str, list[str]]:
135 return {g: [sys.executable, "-c", f"raise SystemExit({exit_code})"] for g in REQUIRED_GATES}
136
137 def test_all_zero_exits_pass(self) -> None:
138 results = run_gates(self._cmds())
139 assert set(results) == set(REQUIRED_GATES)
140 assert all(r.status == "pass" and r.exit_code == 0 for r in results.values())
141
142 def test_nonzero_exit_fails_that_gate_and_records_output(self) -> None:
143 cmds = self._cmds()
144 cmds["install.full"] = [sys.executable, "-c", "import sys; print('grammar broken'); sys.exit(3)"]
145 results = run_gates(cmds)
146 assert results["install.full"].status == "fail"
147 assert results["install.full"].exit_code == 3
148 assert "grammar broken" in results["install.full"].detail
149 assert results["install.slim"].status == "pass"
150
151 def test_gate_without_a_command_is_a_failure_not_a_skip(self) -> None:
152 cmds = self._cmds()
153 del cmds["runtime.doctor"]
154 assert run_gates(cmds)["runtime.doctor"].status == "fail"
155
156 def test_unlaunchable_command_fails(self) -> None:
157 cmds = self._cmds()
158 cmds["runtime.doctor"] = ["/nonexistent/binary"]
159 assert run_gates(cmds)["runtime.doctor"].status == "fail"
160
161 def test_timeout_fails_the_gate(self) -> None:
162 cmds = self._cmds()
163 cmds["runtime.doctor"] = [sys.executable, "-c", "import time; time.sleep(30)"]
164 result = run_gates(cmds, timeout=0.5)["runtime.doctor"]
165 assert result.status == "fail" and "timed out" in result.detail
166
167 def test_interruption_propagates_so_no_report_is_produced(self) -> None:
168 def interrupted(*_a: object, **_k: object) -> subprocess.CompletedProcess[str]:
169 raise KeyboardInterrupt
170
171 with pytest.raises(KeyboardInterrupt):
172 run_gates(self._cmds(), runner=interrupted)
173
174
175 class TestDefaultGateCommands:
176 def test_every_required_gate_has_a_command_and_none_extra(self, tmp_path: pathlib.Path) -> None:
177 from deploy.release_gate import default_gate_commands
178
179 cmds = default_gate_commands(tmp_path / "muse-1.tar.gz", tmp_path / "muse", tmp_path / "hub", "/gate/venv/bin/python")
180 assert set(cmds) == set(REQUIRED_GATES)
181
182 def test_parity_gate_opts_in_to_the_installed_marker(self, tmp_path: pathlib.Path) -> None:
183 """tests/installed is deselected by default (addopts -m 'not installed'); without -m installed the
184 gate would collect nothing and could never pass."""
185 from deploy.release_gate import default_gate_commands
186
187 argv = default_gate_commands(tmp_path / "muse-1.tar.gz", tmp_path / "muse", tmp_path / "hub", "/gate/venv/bin/python")["runtime.installed_parser_parity"]
188 assert any(a == "-m" and b == "installed" for a, b in zip(argv, argv[1:])) and "tests/installed" in argv
189
190
191
192 def test_install_gates_select_the_opt_in_install_e2e_marker() -> None:
193 """pyproject's addopts deselects install_e2e for the dev suite; every install/runtime gate built on the e2e suite must re-select it,
194 or it collects nothing (pytest exit 5) and the release gate can never pass."""
195 import pathlib as _pl
196
197 from deploy.release_gate import default_gate_commands
198
199 cmds = default_gate_commands(_pl.Path("/a.tar.gz"), _pl.Path("/muse"), _pl.Path("/musehub"), "py")
200 e2e = {n: c for n, c in cmds.items() if any("tests/install_e2e/" in a for a in c)}
201 assert e2e, "expected the install/runtime gates"
202 for name, cmd in e2e.items():
203 assert cmd[cmd.index("-m", 2): cmd.index("-m", 2) + 2] == ["-m", "install_e2e"], (name, cmd)
File History 5 commits
sha256:982b4ce08b140ea3fa9057705ff1806a702c9203689c22b797a19f9db678fe5f fix(release-gate): install gates select the opt-in install_… Opus 5.5 patch 2 days ago
sha256:a977b39d08e15683210a33608e25712fe3ed9c8f05f0388fb69ef38acf835426 feat(release): hermetic release gates in a provisioned, pin… Opus 5.5 minor ⚠ 12 days ago
sha256:04db68e43d9007348dbd726b2a3e52f73e83c5646a4511c07e19cb9a66e28b5f fix(release): gate provenance is observed from the checkout… Opus 5.5 minor ⚠ 12 days ago
sha256:28b9e16197029996ce556a27058a9faa4eb27e088440726c5d8aaa8a811a4a64 docs(release): runbook separating build/validate/publish/ac… Opus 5.5 patch 12 days ago
sha256:c1e420ecafdcfabfd74f76e07d538ec0c884be32b14f6062d32cb19690a3d4d3 test(release): red tests for artifact immutability, retenti… Opus 5.5 12 days ago