test_release_gate.py
python
sha256:982b4ce08b140ea3fa9057705ff1806a702c9203689c22b797a19f9db678fe5f
fix(release-gate): install gates select the opt-in install_…
Opus 5.5
patch
2 days ago
| 1 | """Release gate report: construction, binding to the artifact, and the runner (musehub#114 Phase 6). |
| 2 | |
| 3 | A gate report is evidence about *one exact artifact*. Everything here proves the |
| 4 | verifier rejects anything that is not complete, passing, fresh and bound. |
| 5 | """ |
| 6 | |
| 7 | from __future__ import annotations |
| 8 | |
| 9 | import json |
| 10 | import pathlib |
| 11 | import subprocess |
| 12 | import sys |
| 13 | from datetime import timedelta |
| 14 | |
| 15 | import pytest |
| 16 | |
| 17 | from deploy.release_gate import REQUIRED_GATES, GateError, GateResult, run_gates, verify_report |
| 18 | from tests.release_fixtures import CHECKOUT, NOW, STALE, make_artifact, make_report, mutate_report, passing_results |
| 19 | |
| 20 | |
| 21 | @pytest.fixture |
| 22 | def artifact(tmp_path: pathlib.Path) -> pathlib.Path: |
| 23 | return make_artifact(tmp_path, "0.2.1rc19") |
| 24 | |
| 25 | |
| 26 | class TestVerifyReport: |
| 27 | def test_complete_report_bound_to_the_artifact_is_accepted(self, artifact: pathlib.Path) -> None: |
| 28 | candidate = verify_report(make_report(artifact), artifact, now=NOW, checkout=CHECKOUT) |
| 29 | assert candidate.version == "0.2.1rc19" |
| 30 | assert candidate.sha256 and candidate.source_revision and candidate.installer_revision == 2 |
| 31 | |
| 32 | def test_required_gates_cover_packaging_install_and_runtime(self) -> None: |
| 33 | names = set(REQUIRED_GATES) |
| 34 | for needed in ( |
| 35 | "packaging.sdist_reproducible", "packaging.version", "packaging.dependency_metadata", |
| 36 | "packaging.manifest_contract", "install.full", "install.slim", "install.legacy_upgrade", |
| 37 | "install.profile_preservation", "install.explicit_grammar", "runtime.doctor", |
| 38 | "runtime.grammar_availability", "runtime.installed_parser_parity", |
| 39 | ): |
| 40 | assert needed in names |
| 41 | |
| 42 | @pytest.mark.parametrize("gate", REQUIRED_GATES) |
| 43 | def test_every_required_gate_failing_blocks(self, artifact: pathlib.Path, gate: str) -> None: |
| 44 | results = passing_results() |
| 45 | results[gate] = GateResult(status="fail", exit_code=1, detail="boom") |
| 46 | with pytest.raises(GateError, match=gate): |
| 47 | verify_report(make_report(artifact, results=results), artifact, now=NOW, checkout=CHECKOUT) |
| 48 | |
| 49 | @pytest.mark.parametrize("gate", REQUIRED_GATES) |
| 50 | def test_every_required_gate_missing_blocks(self, artifact: pathlib.Path, gate: str) -> None: |
| 51 | results = passing_results() |
| 52 | del results[gate] |
| 53 | with pytest.raises(GateError, match=gate): |
| 54 | verify_report(make_report(artifact, results=results), artifact, now=NOW, checkout=CHECKOUT) |
| 55 | |
| 56 | def test_pass_status_with_nonzero_exit_code_is_inconsistent_and_blocks(self, artifact: pathlib.Path) -> None: |
| 57 | results = passing_results() |
| 58 | results["install.slim"] = GateResult(status="pass", exit_code=2, detail="") |
| 59 | with pytest.raises(GateError, match="install.slim"): |
| 60 | verify_report(make_report(artifact, results=results), artifact, now=NOW, checkout=CHECKOUT) |
| 61 | |
| 62 | def test_decision_field_is_never_trusted(self, artifact: pathlib.Path) -> None: |
| 63 | results = passing_results() |
| 64 | results["runtime.doctor"] = GateResult(status="fail", exit_code=1, detail="x") |
| 65 | forged = mutate_report(make_report(artifact, results=results), decision="pass") |
| 66 | with pytest.raises(GateError): |
| 67 | verify_report(forged, artifact, now=NOW, checkout=CHECKOUT) |
| 68 | |
| 69 | def test_report_for_a_different_version_does_not_authorize_this_one(self, artifact: pathlib.Path, tmp_path: pathlib.Path) -> None: |
| 70 | (tmp_path / "old").mkdir() |
| 71 | rc18 = make_artifact(tmp_path / "old", "0.2.1rc18") |
| 72 | with pytest.raises(GateError, match="version"): |
| 73 | verify_report(make_report(rc18), artifact, now=NOW, checkout=CHECKOUT) |
| 74 | |
| 75 | def test_report_for_a_different_hash_of_the_same_version_is_rejected(self, artifact: pathlib.Path) -> None: |
| 76 | report = make_report(artifact) |
| 77 | artifact.write_bytes(b"same version, different bytes") |
| 78 | with pytest.raises(GateError, match="sha256"): |
| 79 | verify_report(report, artifact, now=NOW, checkout=CHECKOUT) |
| 80 | |
| 81 | def test_stale_report_is_rejected(self, artifact: pathlib.Path) -> None: |
| 82 | with pytest.raises(GateError, match="stale"): |
| 83 | verify_report(make_report(artifact, created_at=STALE), artifact, now=NOW, checkout=CHECKOUT) |
| 84 | |
| 85 | def test_report_from_the_future_is_rejected(self, artifact: pathlib.Path) -> None: |
| 86 | with pytest.raises(GateError, match="future"): |
| 87 | verify_report(make_report(artifact, created_at=NOW + timedelta(hours=2)), artifact, now=NOW, checkout=CHECKOUT) |
| 88 | |
| 89 | @pytest.mark.parametrize("text", ["", "not json", "[]", "null", '{"schema": 1}', '"pass"']) |
| 90 | def test_malformed_report_is_rejected(self, artifact: pathlib.Path, text: str) -> None: |
| 91 | with pytest.raises(GateError): |
| 92 | verify_report(text, artifact, now=NOW, checkout=CHECKOUT) |
| 93 | |
| 94 | @pytest.mark.parametrize("change", [ |
| 95 | {"schema": 1}, |
| 96 | {"schema": 3}, |
| 97 | {"candidate__source_revision": ""}, |
| 98 | {"candidate__source_revision": "abc123"}, |
| 99 | {"candidate__installer_revision": "2"}, |
| 100 | {"candidate__sha256": "XYZ"}, |
| 101 | {"created_at": "yesterday"}, |
| 102 | {"gates": "all of them"}, |
| 103 | ]) |
| 104 | def test_schema_violations_are_rejected(self, artifact: pathlib.Path, change: dict[str, object]) -> None: |
| 105 | with pytest.raises(GateError): |
| 106 | verify_report(mutate_report(make_report(artifact), **change), artifact, now=NOW, checkout=CHECKOUT) |
| 107 | |
| 108 | def test_duplicate_gate_entries_are_rejected(self, artifact: pathlib.Path) -> None: |
| 109 | data = json.loads(make_report(artifact)) |
| 110 | data["gates"].append(dict(data["gates"][0])) |
| 111 | with pytest.raises(GateError, match="duplicate"): |
| 112 | verify_report(json.dumps(data), artifact, now=NOW, checkout=CHECKOUT) |
| 113 | |
| 114 | def test_an_extra_failing_gate_also_blocks(self, artifact: pathlib.Path) -> None: |
| 115 | data = json.loads(make_report(artifact)) |
| 116 | data["gates"].append({"name": "extra.lint", "status": "fail", "exit_code": 1, "detail": ""}) |
| 117 | with pytest.raises(GateError, match="extra.lint"): |
| 118 | verify_report(json.dumps(data), artifact, now=NOW, checkout=CHECKOUT) |
| 119 | |
| 120 | def test_artifact_filename_must_carry_the_candidate_version(self, tmp_path: pathlib.Path) -> None: |
| 121 | odd = tmp_path / "candidate.tar.gz" |
| 122 | odd.write_bytes(b"x") |
| 123 | with pytest.raises(GateError, match="filename"): |
| 124 | verify_report(make_report(odd, version="0.2.1rc19"), odd, now=NOW, checkout=CHECKOUT) |
| 125 | |
| 126 | def test_missing_artifact_is_rejected(self, artifact: pathlib.Path) -> None: |
| 127 | report = make_report(artifact) |
| 128 | artifact.unlink() |
| 129 | with pytest.raises(GateError, match="artifact"): |
| 130 | verify_report(report, artifact, now=NOW, checkout=CHECKOUT) |
| 131 | |
| 132 | |
| 133 | class TestRunGates: |
| 134 | def _cmds(self, exit_code: int = 0) -> dict[str, list[str]]: |
| 135 | return {g: [sys.executable, "-c", f"raise SystemExit({exit_code})"] for g in REQUIRED_GATES} |
| 136 | |
| 137 | def test_all_zero_exits_pass(self) -> None: |
| 138 | results = run_gates(self._cmds()) |
| 139 | assert set(results) == set(REQUIRED_GATES) |
| 140 | assert all(r.status == "pass" and r.exit_code == 0 for r in results.values()) |
| 141 | |
| 142 | def test_nonzero_exit_fails_that_gate_and_records_output(self) -> None: |
| 143 | cmds = self._cmds() |
| 144 | cmds["install.full"] = [sys.executable, "-c", "import sys; print('grammar broken'); sys.exit(3)"] |
| 145 | results = run_gates(cmds) |
| 146 | assert results["install.full"].status == "fail" |
| 147 | assert results["install.full"].exit_code == 3 |
| 148 | assert "grammar broken" in results["install.full"].detail |
| 149 | assert results["install.slim"].status == "pass" |
| 150 | |
| 151 | def test_gate_without_a_command_is_a_failure_not_a_skip(self) -> None: |
| 152 | cmds = self._cmds() |
| 153 | del cmds["runtime.doctor"] |
| 154 | assert run_gates(cmds)["runtime.doctor"].status == "fail" |
| 155 | |
| 156 | def test_unlaunchable_command_fails(self) -> None: |
| 157 | cmds = self._cmds() |
| 158 | cmds["runtime.doctor"] = ["/nonexistent/binary"] |
| 159 | assert run_gates(cmds)["runtime.doctor"].status == "fail" |
| 160 | |
| 161 | def test_timeout_fails_the_gate(self) -> None: |
| 162 | cmds = self._cmds() |
| 163 | cmds["runtime.doctor"] = [sys.executable, "-c", "import time; time.sleep(30)"] |
| 164 | result = run_gates(cmds, timeout=0.5)["runtime.doctor"] |
| 165 | assert result.status == "fail" and "timed out" in result.detail |
| 166 | |
| 167 | def test_interruption_propagates_so_no_report_is_produced(self) -> None: |
| 168 | def interrupted(*_a: object, **_k: object) -> subprocess.CompletedProcess[str]: |
| 169 | raise KeyboardInterrupt |
| 170 | |
| 171 | with pytest.raises(KeyboardInterrupt): |
| 172 | run_gates(self._cmds(), runner=interrupted) |
| 173 | |
| 174 | |
| 175 | class TestDefaultGateCommands: |
| 176 | def test_every_required_gate_has_a_command_and_none_extra(self, tmp_path: pathlib.Path) -> None: |
| 177 | from deploy.release_gate import default_gate_commands |
| 178 | |
| 179 | cmds = default_gate_commands(tmp_path / "muse-1.tar.gz", tmp_path / "muse", tmp_path / "hub", "/gate/venv/bin/python") |
| 180 | assert set(cmds) == set(REQUIRED_GATES) |
| 181 | |
| 182 | def test_parity_gate_opts_in_to_the_installed_marker(self, tmp_path: pathlib.Path) -> None: |
| 183 | """tests/installed is deselected by default (addopts -m 'not installed'); without -m installed the |
| 184 | gate would collect nothing and could never pass.""" |
| 185 | from deploy.release_gate import default_gate_commands |
| 186 | |
| 187 | argv = default_gate_commands(tmp_path / "muse-1.tar.gz", tmp_path / "muse", tmp_path / "hub", "/gate/venv/bin/python")["runtime.installed_parser_parity"] |
| 188 | assert any(a == "-m" and b == "installed" for a, b in zip(argv, argv[1:])) and "tests/installed" in argv |
| 189 | |
| 190 | |
| 191 | |
| 192 | def test_install_gates_select_the_opt_in_install_e2e_marker() -> None: |
| 193 | """pyproject's addopts deselects install_e2e for the dev suite; every install/runtime gate built on the e2e suite must re-select it, |
| 194 | or it collects nothing (pytest exit 5) and the release gate can never pass.""" |
| 195 | import pathlib as _pl |
| 196 | |
| 197 | from deploy.release_gate import default_gate_commands |
| 198 | |
| 199 | cmds = default_gate_commands(_pl.Path("/a.tar.gz"), _pl.Path("/muse"), _pl.Path("/musehub"), "py") |
| 200 | e2e = {n: c for n, c in cmds.items() if any("tests/install_e2e/" in a for a in c)} |
| 201 | assert e2e, "expected the install/runtime gates" |
| 202 | for name, cmd in e2e.items(): |
| 203 | assert cmd[cmd.index("-m", 2): cmd.index("-m", 2) + 2] == ["-m", "install_e2e"], (name, cmd) |
File History
5 commits
sha256:982b4ce08b140ea3fa9057705ff1806a702c9203689c22b797a19f9db678fe5f
fix(release-gate): install gates select the opt-in install_…
Opus 5.5
patch
2 days ago
sha256:a977b39d08e15683210a33608e25712fe3ed9c8f05f0388fb69ef38acf835426
feat(release): hermetic release gates in a provisioned, pin…
Opus 5.5
minor
⚠
12 days ago
sha256:04db68e43d9007348dbd726b2a3e52f73e83c5646a4511c07e19cb9a66e28b5f
fix(release): gate provenance is observed from the checkout…
Opus 5.5
minor
⚠
12 days ago
sha256:28b9e16197029996ce556a27058a9faa4eb27e088440726c5d8aaa8a811a4a64
docs(release): runbook separating build/validate/publish/ac…
Opus 5.5
patch
12 days ago
sha256:c1e420ecafdcfabfd74f76e07d538ec0c884be32b14f6062d32cb19690a3d4d3
test(release): red tests for artifact immutability, retenti…
Opus 5.5
12 days ago