gabriel / musehub public
test_release_fail_closed.py python
244 lines 9.5 KB
Raw
sha256:675003834b19b109f8cb126e543a064a7bd6256930dd4a80ffcfe87c6eff3f17 fix(release): idempotent concurrent publish (D3) and compar… Opus 5.5 minor ⚠ breaking 12 days ago
1 """Negative-path proof that release gates fail closed (musehub#114 Phase 6).
2
3 Every scenario below drives the *real* publish + activate path against a fake
4 bucket that already holds a healthy release (rc18, active on staging and
5 production, with rc17 as production's rollback target). For every failure:
6
7 no artifact is published, no pointer moves, nothing is deleted --
8 the bucket is byte-for-byte what it was, and the op log has no writes.
9
10 Nothing here touches S3, ECR, ECS or any staging/production service.
11 """
12
13 from __future__ import annotations
14
15 import json
16 import pathlib
17 import sys
18 from collections.abc import Callable
19
20 import pytest
21
22 from deploy.release_gate import REQUIRED_GATES, GateError, GateResult, run_gates
23 from deploy.release_safety import (
24 ReleaseSafetyError,
25 StorageUnavailable,
26 activate,
27 execute_cleanup,
28 plan_cleanup,
29 publish,
30 )
31 from tests.release_fixtures import (
32 observed,
33 CHECKOUT,
34 NOW,
35 STALE,
36 FakeStorage,
37 make_artifact,
38 make_report,
39 mutate_report,
40 passing_results,
41 sha256_of,
42 )
43
44 Scenario = Callable[[pathlib.Path], tuple[pathlib.Path, str | None]]
45
46
47 def failing(gate: str, detail: str) -> dict[str, GateResult]:
48 results = passing_results()
49 results[gate] = GateResult(status="fail", exit_code=1, detail=detail)
50 return results
51
52
53 def candidate(tmp: pathlib.Path, **report_kwargs: object) -> tuple[pathlib.Path, str]:
54 (tmp / "cand").mkdir(exist_ok=True)
55 art = make_artifact(tmp / "cand", "0.2.1rc19")
56 return art, make_report(art, **report_kwargs) # type: ignore[arg-type]
57
58
59 def s_broken_grammar(tmp: pathlib.Path) -> tuple[pathlib.Path, str | None]:
60 return candidate(tmp, results=failing("runtime.doctor", "tree-sitter-go: BROKEN"))
61
62
63 def s_missing_grammar(tmp: pathlib.Path) -> tuple[pathlib.Path, str | None]:
64 return candidate(tmp, results=failing("runtime.grammar_availability", "tree-sitter-rust: missing"))
65
66
67 def s_failed_parity(tmp: pathlib.Path) -> tuple[pathlib.Path, str | None]:
68 return candidate(tmp, results=failing("runtime.installed_parser_parity", "37 checks, 3 failed"))
69
70
71 def s_failed_upgrade(tmp: pathlib.Path) -> tuple[pathlib.Path, str | None]:
72 return candidate(tmp, results=failing("install.legacy_upgrade", "rc18 -> rc19 left no manifest"))
73
74
75 def s_failed_slim(tmp: pathlib.Path) -> tuple[pathlib.Path, str | None]:
76 return candidate(tmp, results=failing("install.slim", "slim pulled grammars"))
77
78
79 def s_missing_report(tmp: pathlib.Path) -> tuple[pathlib.Path, str | None]:
80 art, _ = candidate(tmp)
81 return art, None
82
83
84 def s_malformed_report(tmp: pathlib.Path) -> tuple[pathlib.Path, str | None]:
85 art, report = candidate(tmp)
86 return art, report[: len(report) // 2]
87
88
89 def s_stale_report(tmp: pathlib.Path) -> tuple[pathlib.Path, str | None]:
90 return candidate(tmp, created_at=STALE)
91
92
93 def s_hash_mismatch(tmp: pathlib.Path) -> tuple[pathlib.Path, str | None]:
94 art, report = candidate(tmp)
95 art.write_bytes(b"rebuilt after validation")
96 return art, report
97
98
99 def s_rc18_report_for_rc19(tmp: pathlib.Path) -> tuple[pathlib.Path, str | None]:
100 art, _ = candidate(tmp)
101 (tmp / "rc18").mkdir()
102 return art, make_report(make_artifact(tmp / "rc18", "0.2.1rc18"))
103
104
105 def s_forged_decision(tmp: pathlib.Path) -> tuple[pathlib.Path, str | None]:
106 art, report = candidate(tmp, results=failing("install.full", "x"))
107 return art, mutate_report(report, decision="pass")
108
109
110 def s_gate_removed_from_report(tmp: pathlib.Path) -> tuple[pathlib.Path, str | None]:
111 art, report = candidate(tmp)
112 data = json.loads(report)
113 data["gates"] = [g for g in data["gates"] if g["name"] != "install.slim"]
114 return art, json.dumps(data)
115
116
117 def s_version_collision(tmp: pathlib.Path) -> tuple[pathlib.Path, str | None]:
118 # rc18 already exists in the bucket; this is a *different* rc18.
119 (tmp / "cand").mkdir(exist_ok=True)
120 art = make_artifact(tmp / "cand", "0.2.1rc18", b"not the published rc18")
121 return art, make_report(art)
122
123
124 SCENARIOS: dict[str, Scenario] = {
125 "broken_grammar": s_broken_grammar,
126 "missing_grammar": s_missing_grammar,
127 "failed_installed_parity": s_failed_parity,
128 "failed_upgrade": s_failed_upgrade,
129 "failed_slim_install": s_failed_slim,
130 "missing_report": s_missing_report,
131 "malformed_report": s_malformed_report,
132 "stale_report": s_stale_report,
133 "candidate_hash_mismatch": s_hash_mismatch,
134 "rc18_report_for_rc19": s_rc18_report_for_rc19,
135 "forged_decision": s_forged_decision,
136 "gate_missing_from_report": s_gate_removed_from_report,
137 "version_collision": s_version_collision,
138 }
139
140
141 @pytest.fixture
142 def healthy(tmp_path: pathlib.Path) -> FakeStorage:
143 """rc17 + rc18 published; rc18 active in both environments; rc17 = production rollback."""
144 storage = FakeStorage()
145 shas = {}
146 (tmp_path / "seed").mkdir()
147 for v in ("0.2.1rc17", "0.2.1rc18"):
148 art = make_artifact(tmp_path / "seed", v)
149 publish(storage, art, make_report(art), now=NOW, checkout=CHECKOUT)
150 shas[v] = sha256_of(art)
151 activate(storage, "staging", "0.2.1rc18", shas["0.2.1rc18"], expect=observed(storage, "staging"), now=NOW)
152 activate(storage, "production", "0.2.1rc18", shas["0.2.1rc18"], expect=observed(storage, "production"), now=NOW, approve_production=True)
153 pointer = json.loads(storage.objects["pointers/production.json"])
154 pointer["previous"] = {"version": "0.2.1rc17", "sha256": shas["0.2.1rc17"]}
155 storage.objects["pointers/production.json"] = json.dumps(pointer).encode()
156 storage.ops.clear()
157 return storage
158
159
160 def run_release(storage: FakeStorage, artifact: pathlib.Path, report: str | None) -> None:
161 """Publish then activate on staging -- the whole mutation surface of a release."""
162 result = publish(storage, artifact, report, now=NOW, checkout=CHECKOUT) # type: ignore[arg-type]
163 activate(storage, "staging", result.version, result.sha256, expect=observed(storage, "staging"), now=NOW)
164
165
166 @pytest.mark.parametrize("name", sorted(SCENARIOS))
167 def test_failed_gate_changes_nothing(name: str, healthy: FakeStorage, tmp_path: pathlib.Path) -> None:
168 artifact, report = SCENARIOS[name](tmp_path)
169 before = healthy.snapshot()
170
171 with pytest.raises((GateError, ReleaseSafetyError)):
172 run_release(healthy, artifact, report)
173
174 assert healthy.snapshot() == before, "bucket contents changed"
175 assert healthy.writes() == [], f"unexpected writes: {healthy.writes()}"
176 assert healthy.deletes() == [], f"unexpected deletes: {healthy.deletes()}"
177 assert "muse-0.2.1rc19.tar.gz" not in healthy.objects
178
179
180 def test_every_required_gate_individually_blocks_publication_and_activation(healthy: FakeStorage, tmp_path: pathlib.Path) -> None:
181 before = healthy.snapshot()
182 for i, gate in enumerate(REQUIRED_GATES):
183 d = tmp_path / f"g{i}"
184 d.mkdir()
185 art = make_artifact(d, "0.2.1rc19")
186 with pytest.raises((GateError, ReleaseSafetyError)):
187 run_release(healthy, art, make_report(art, results=failing(gate, "x")))
188 assert healthy.snapshot() == before and healthy.ops == []
189
190
191 def test_interrupted_validation_produces_no_report_and_so_no_release(healthy: FakeStorage, tmp_path: pathlib.Path) -> None:
192 def interrupted(*_a: object, **_k: object) -> None:
193 raise KeyboardInterrupt
194
195 cmds = {g: [sys.executable, "-c", "pass"] for g in REQUIRED_GATES}
196 with pytest.raises(KeyboardInterrupt):
197 run_gates(cmds, runner=interrupted) # type: ignore[arg-type]
198 art, _ = candidate(tmp_path)
199 before = healthy.snapshot()
200 with pytest.raises((GateError, ReleaseSafetyError)):
201 run_release(healthy, art, None)
202 assert healthy.snapshot() == before and healthy.ops == []
203
204
205 def test_storage_outage_during_a_passing_release_publishes_nothing_and_moves_no_pointer(healthy: FakeStorage, tmp_path: pathlib.Path) -> None:
206 art, report = candidate(tmp_path)
207 before = healthy.snapshot()
208 healthy.unavailable = True
209 with pytest.raises(StorageUnavailable):
210 run_release(healthy, art, report)
211 healthy.unavailable = False
212 assert healthy.snapshot() == before and healthy.ops == []
213
214
215 def test_interruption_between_artifact_and_record_never_moves_a_pointer(healthy: FakeStorage, tmp_path: pathlib.Path) -> None:
216 art, report = candidate(tmp_path)
217 pointers = {k: v for k, v in healthy.objects.items() if k.startswith("pointers/")}
218 healthy.fail_put_new_on = "muse-0.2.1rc19.release.json"
219 with pytest.raises(StorageUnavailable):
220 run_release(healthy, art, report)
221 assert {k: v for k, v in healthy.objects.items() if k.startswith("pointers/")} == pointers
222 assert healthy.deletes() == []
223
224
225 @pytest.mark.parametrize("keep", [0, 1, 3])
226 def test_unauthorized_deletion_attempts_cannot_remove_active_or_rollback_targets(healthy: FakeStorage, keep: int) -> None:
227 plan = plan_cleanup(healthy, keep=keep)
228 assert not {"muse-0.2.1rc18.tar.gz", "muse-0.2.1rc17.tar.gz"} & set(plan.deletions)
229 # Even a hand-forged plan naming them, with a self-consistent digest, is refused.
230 forged = type(plan)(deletions=["muse-0.2.1rc17.tar.gz"], preserved={}, blocked=None)
231 before = healthy.snapshot()
232 with pytest.raises(ReleaseSafetyError):
233 execute_cleanup(healthy, forged, confirm=forged.digest)
234 assert healthy.snapshot() == before and healthy.deletes() == []
235
236
237 def test_publication_path_contains_no_delete_call() -> None:
238 """Structural guard: publish/activate/rollback may not reference deletion at all."""
239 import inspect
240
241 from deploy import release_safety
242
243 for fn in (release_safety.publish, release_safety.activate, release_safety.rollback):
244 assert ".delete(" not in inspect.getsource(fn), fn.__name__
File History 3 commits
sha256:675003834b19b109f8cb126e543a064a7bd6256930dd4a80ffcfe87c6eff3f17 fix(release): idempotent concurrent publish (D3) and compar… Opus 5.5 minor ⚠ 12 days ago
sha256:04db68e43d9007348dbd726b2a3e52f73e83c5646a4511c07e19cb9a66e28b5f fix(release): gate provenance is observed from the checkout… Opus 5.5 minor ⚠ 12 days ago
sha256:c1e420ecafdcfabfd74f76e07d538ec0c884be32b14f6062d32cb19690a3d4d3 test(release): red tests for artifact immutability, retenti… Opus 5.5 13 days ago