test_release_fail_closed.py
python
sha256:675003834b19b109f8cb126e543a064a7bd6256930dd4a80ffcfe87c6eff3f17
fix(release): idempotent concurrent publish (D3) and compar…
Opus 5.5
minor
⚠ breaking
12 days ago
| 1 | """Negative-path proof that release gates fail closed (musehub#114 Phase 6). |
| 2 | |
| 3 | Every scenario below drives the *real* publish + activate path against a fake |
| 4 | bucket that already holds a healthy release (rc18, active on staging and |
| 5 | production, with rc17 as production's rollback target). For every failure: |
| 6 | |
| 7 | no artifact is published, no pointer moves, nothing is deleted -- |
| 8 | the bucket is byte-for-byte what it was, and the op log has no writes. |
| 9 | |
| 10 | Nothing here touches S3, ECR, ECS or any staging/production service. |
| 11 | """ |
| 12 | |
| 13 | from __future__ import annotations |
| 14 | |
| 15 | import json |
| 16 | import pathlib |
| 17 | import sys |
| 18 | from collections.abc import Callable |
| 19 | |
| 20 | import pytest |
| 21 | |
| 22 | from deploy.release_gate import REQUIRED_GATES, GateError, GateResult, run_gates |
| 23 | from deploy.release_safety import ( |
| 24 | ReleaseSafetyError, |
| 25 | StorageUnavailable, |
| 26 | activate, |
| 27 | execute_cleanup, |
| 28 | plan_cleanup, |
| 29 | publish, |
| 30 | ) |
| 31 | from tests.release_fixtures import ( |
| 32 | observed, |
| 33 | CHECKOUT, |
| 34 | NOW, |
| 35 | STALE, |
| 36 | FakeStorage, |
| 37 | make_artifact, |
| 38 | make_report, |
| 39 | mutate_report, |
| 40 | passing_results, |
| 41 | sha256_of, |
| 42 | ) |
| 43 | |
| 44 | Scenario = Callable[[pathlib.Path], tuple[pathlib.Path, str | None]] |
| 45 | |
| 46 | |
| 47 | def failing(gate: str, detail: str) -> dict[str, GateResult]: |
| 48 | results = passing_results() |
| 49 | results[gate] = GateResult(status="fail", exit_code=1, detail=detail) |
| 50 | return results |
| 51 | |
| 52 | |
| 53 | def candidate(tmp: pathlib.Path, **report_kwargs: object) -> tuple[pathlib.Path, str]: |
| 54 | (tmp / "cand").mkdir(exist_ok=True) |
| 55 | art = make_artifact(tmp / "cand", "0.2.1rc19") |
| 56 | return art, make_report(art, **report_kwargs) # type: ignore[arg-type] |
| 57 | |
| 58 | |
| 59 | def s_broken_grammar(tmp: pathlib.Path) -> tuple[pathlib.Path, str | None]: |
| 60 | return candidate(tmp, results=failing("runtime.doctor", "tree-sitter-go: BROKEN")) |
| 61 | |
| 62 | |
| 63 | def s_missing_grammar(tmp: pathlib.Path) -> tuple[pathlib.Path, str | None]: |
| 64 | return candidate(tmp, results=failing("runtime.grammar_availability", "tree-sitter-rust: missing")) |
| 65 | |
| 66 | |
| 67 | def s_failed_parity(tmp: pathlib.Path) -> tuple[pathlib.Path, str | None]: |
| 68 | return candidate(tmp, results=failing("runtime.installed_parser_parity", "37 checks, 3 failed")) |
| 69 | |
| 70 | |
| 71 | def s_failed_upgrade(tmp: pathlib.Path) -> tuple[pathlib.Path, str | None]: |
| 72 | return candidate(tmp, results=failing("install.legacy_upgrade", "rc18 -> rc19 left no manifest")) |
| 73 | |
| 74 | |
| 75 | def s_failed_slim(tmp: pathlib.Path) -> tuple[pathlib.Path, str | None]: |
| 76 | return candidate(tmp, results=failing("install.slim", "slim pulled grammars")) |
| 77 | |
| 78 | |
| 79 | def s_missing_report(tmp: pathlib.Path) -> tuple[pathlib.Path, str | None]: |
| 80 | art, _ = candidate(tmp) |
| 81 | return art, None |
| 82 | |
| 83 | |
| 84 | def s_malformed_report(tmp: pathlib.Path) -> tuple[pathlib.Path, str | None]: |
| 85 | art, report = candidate(tmp) |
| 86 | return art, report[: len(report) // 2] |
| 87 | |
| 88 | |
| 89 | def s_stale_report(tmp: pathlib.Path) -> tuple[pathlib.Path, str | None]: |
| 90 | return candidate(tmp, created_at=STALE) |
| 91 | |
| 92 | |
| 93 | def s_hash_mismatch(tmp: pathlib.Path) -> tuple[pathlib.Path, str | None]: |
| 94 | art, report = candidate(tmp) |
| 95 | art.write_bytes(b"rebuilt after validation") |
| 96 | return art, report |
| 97 | |
| 98 | |
| 99 | def s_rc18_report_for_rc19(tmp: pathlib.Path) -> tuple[pathlib.Path, str | None]: |
| 100 | art, _ = candidate(tmp) |
| 101 | (tmp / "rc18").mkdir() |
| 102 | return art, make_report(make_artifact(tmp / "rc18", "0.2.1rc18")) |
| 103 | |
| 104 | |
| 105 | def s_forged_decision(tmp: pathlib.Path) -> tuple[pathlib.Path, str | None]: |
| 106 | art, report = candidate(tmp, results=failing("install.full", "x")) |
| 107 | return art, mutate_report(report, decision="pass") |
| 108 | |
| 109 | |
| 110 | def s_gate_removed_from_report(tmp: pathlib.Path) -> tuple[pathlib.Path, str | None]: |
| 111 | art, report = candidate(tmp) |
| 112 | data = json.loads(report) |
| 113 | data["gates"] = [g for g in data["gates"] if g["name"] != "install.slim"] |
| 114 | return art, json.dumps(data) |
| 115 | |
| 116 | |
| 117 | def s_version_collision(tmp: pathlib.Path) -> tuple[pathlib.Path, str | None]: |
| 118 | # rc18 already exists in the bucket; this is a *different* rc18. |
| 119 | (tmp / "cand").mkdir(exist_ok=True) |
| 120 | art = make_artifact(tmp / "cand", "0.2.1rc18", b"not the published rc18") |
| 121 | return art, make_report(art) |
| 122 | |
| 123 | |
| 124 | SCENARIOS: dict[str, Scenario] = { |
| 125 | "broken_grammar": s_broken_grammar, |
| 126 | "missing_grammar": s_missing_grammar, |
| 127 | "failed_installed_parity": s_failed_parity, |
| 128 | "failed_upgrade": s_failed_upgrade, |
| 129 | "failed_slim_install": s_failed_slim, |
| 130 | "missing_report": s_missing_report, |
| 131 | "malformed_report": s_malformed_report, |
| 132 | "stale_report": s_stale_report, |
| 133 | "candidate_hash_mismatch": s_hash_mismatch, |
| 134 | "rc18_report_for_rc19": s_rc18_report_for_rc19, |
| 135 | "forged_decision": s_forged_decision, |
| 136 | "gate_missing_from_report": s_gate_removed_from_report, |
| 137 | "version_collision": s_version_collision, |
| 138 | } |
| 139 | |
| 140 | |
| 141 | @pytest.fixture |
| 142 | def healthy(tmp_path: pathlib.Path) -> FakeStorage: |
| 143 | """rc17 + rc18 published; rc18 active in both environments; rc17 = production rollback.""" |
| 144 | storage = FakeStorage() |
| 145 | shas = {} |
| 146 | (tmp_path / "seed").mkdir() |
| 147 | for v in ("0.2.1rc17", "0.2.1rc18"): |
| 148 | art = make_artifact(tmp_path / "seed", v) |
| 149 | publish(storage, art, make_report(art), now=NOW, checkout=CHECKOUT) |
| 150 | shas[v] = sha256_of(art) |
| 151 | activate(storage, "staging", "0.2.1rc18", shas["0.2.1rc18"], expect=observed(storage, "staging"), now=NOW) |
| 152 | activate(storage, "production", "0.2.1rc18", shas["0.2.1rc18"], expect=observed(storage, "production"), now=NOW, approve_production=True) |
| 153 | pointer = json.loads(storage.objects["pointers/production.json"]) |
| 154 | pointer["previous"] = {"version": "0.2.1rc17", "sha256": shas["0.2.1rc17"]} |
| 155 | storage.objects["pointers/production.json"] = json.dumps(pointer).encode() |
| 156 | storage.ops.clear() |
| 157 | return storage |
| 158 | |
| 159 | |
| 160 | def run_release(storage: FakeStorage, artifact: pathlib.Path, report: str | None) -> None: |
| 161 | """Publish then activate on staging -- the whole mutation surface of a release.""" |
| 162 | result = publish(storage, artifact, report, now=NOW, checkout=CHECKOUT) # type: ignore[arg-type] |
| 163 | activate(storage, "staging", result.version, result.sha256, expect=observed(storage, "staging"), now=NOW) |
| 164 | |
| 165 | |
| 166 | @pytest.mark.parametrize("name", sorted(SCENARIOS)) |
| 167 | def test_failed_gate_changes_nothing(name: str, healthy: FakeStorage, tmp_path: pathlib.Path) -> None: |
| 168 | artifact, report = SCENARIOS[name](tmp_path) |
| 169 | before = healthy.snapshot() |
| 170 | |
| 171 | with pytest.raises((GateError, ReleaseSafetyError)): |
| 172 | run_release(healthy, artifact, report) |
| 173 | |
| 174 | assert healthy.snapshot() == before, "bucket contents changed" |
| 175 | assert healthy.writes() == [], f"unexpected writes: {healthy.writes()}" |
| 176 | assert healthy.deletes() == [], f"unexpected deletes: {healthy.deletes()}" |
| 177 | assert "muse-0.2.1rc19.tar.gz" not in healthy.objects |
| 178 | |
| 179 | |
| 180 | def test_every_required_gate_individually_blocks_publication_and_activation(healthy: FakeStorage, tmp_path: pathlib.Path) -> None: |
| 181 | before = healthy.snapshot() |
| 182 | for i, gate in enumerate(REQUIRED_GATES): |
| 183 | d = tmp_path / f"g{i}" |
| 184 | d.mkdir() |
| 185 | art = make_artifact(d, "0.2.1rc19") |
| 186 | with pytest.raises((GateError, ReleaseSafetyError)): |
| 187 | run_release(healthy, art, make_report(art, results=failing(gate, "x"))) |
| 188 | assert healthy.snapshot() == before and healthy.ops == [] |
| 189 | |
| 190 | |
| 191 | def test_interrupted_validation_produces_no_report_and_so_no_release(healthy: FakeStorage, tmp_path: pathlib.Path) -> None: |
| 192 | def interrupted(*_a: object, **_k: object) -> None: |
| 193 | raise KeyboardInterrupt |
| 194 | |
| 195 | cmds = {g: [sys.executable, "-c", "pass"] for g in REQUIRED_GATES} |
| 196 | with pytest.raises(KeyboardInterrupt): |
| 197 | run_gates(cmds, runner=interrupted) # type: ignore[arg-type] |
| 198 | art, _ = candidate(tmp_path) |
| 199 | before = healthy.snapshot() |
| 200 | with pytest.raises((GateError, ReleaseSafetyError)): |
| 201 | run_release(healthy, art, None) |
| 202 | assert healthy.snapshot() == before and healthy.ops == [] |
| 203 | |
| 204 | |
| 205 | def test_storage_outage_during_a_passing_release_publishes_nothing_and_moves_no_pointer(healthy: FakeStorage, tmp_path: pathlib.Path) -> None: |
| 206 | art, report = candidate(tmp_path) |
| 207 | before = healthy.snapshot() |
| 208 | healthy.unavailable = True |
| 209 | with pytest.raises(StorageUnavailable): |
| 210 | run_release(healthy, art, report) |
| 211 | healthy.unavailable = False |
| 212 | assert healthy.snapshot() == before and healthy.ops == [] |
| 213 | |
| 214 | |
| 215 | def test_interruption_between_artifact_and_record_never_moves_a_pointer(healthy: FakeStorage, tmp_path: pathlib.Path) -> None: |
| 216 | art, report = candidate(tmp_path) |
| 217 | pointers = {k: v for k, v in healthy.objects.items() if k.startswith("pointers/")} |
| 218 | healthy.fail_put_new_on = "muse-0.2.1rc19.release.json" |
| 219 | with pytest.raises(StorageUnavailable): |
| 220 | run_release(healthy, art, report) |
| 221 | assert {k: v for k, v in healthy.objects.items() if k.startswith("pointers/")} == pointers |
| 222 | assert healthy.deletes() == [] |
| 223 | |
| 224 | |
| 225 | @pytest.mark.parametrize("keep", [0, 1, 3]) |
| 226 | def test_unauthorized_deletion_attempts_cannot_remove_active_or_rollback_targets(healthy: FakeStorage, keep: int) -> None: |
| 227 | plan = plan_cleanup(healthy, keep=keep) |
| 228 | assert not {"muse-0.2.1rc18.tar.gz", "muse-0.2.1rc17.tar.gz"} & set(plan.deletions) |
| 229 | # Even a hand-forged plan naming them, with a self-consistent digest, is refused. |
| 230 | forged = type(plan)(deletions=["muse-0.2.1rc17.tar.gz"], preserved={}, blocked=None) |
| 231 | before = healthy.snapshot() |
| 232 | with pytest.raises(ReleaseSafetyError): |
| 233 | execute_cleanup(healthy, forged, confirm=forged.digest) |
| 234 | assert healthy.snapshot() == before and healthy.deletes() == [] |
| 235 | |
| 236 | |
| 237 | def test_publication_path_contains_no_delete_call() -> None: |
| 238 | """Structural guard: publish/activate/rollback may not reference deletion at all.""" |
| 239 | import inspect |
| 240 | |
| 241 | from deploy import release_safety |
| 242 | |
| 243 | for fn in (release_safety.publish, release_safety.activate, release_safety.rollback): |
| 244 | assert ".delete(" not in inspect.getsource(fn), fn.__name__ |
File History
3 commits
sha256:675003834b19b109f8cb126e543a064a7bd6256930dd4a80ffcfe87c6eff3f17
fix(release): idempotent concurrent publish (D3) and compar…
Opus 5.5
minor
⚠
12 days ago
sha256:04db68e43d9007348dbd726b2a3e52f73e83c5646a4511c07e19cb9a66e28b5f
fix(release): gate provenance is observed from the checkout…
Opus 5.5
minor
⚠
12 days ago
sha256:c1e420ecafdcfabfd74f76e07d538ec0c884be32b14f6062d32cb19690a3d4d3
test(release): red tests for artifact immutability, retenti…
Opus 5.5
13 days ago