gabriel / musehub public
test_release_safety.py python
310 lines 16.9 KB
Raw
sha256:675003834b19b109f8cb126e543a064a7bd6256930dd4a80ffcfe87c6eff3f17 fix(release): idempotent concurrent publish (D3) and compar… Opus 5.5 minor ⚠ breaking 12 days ago
1 """Release artifact immutability, activation, rollback and retention (musehub#114 Phase 6).
2
3 Incident this answers: publishing rc19 to *staging* deleted rc16 from the bucket
4 shared with *production*. The invariants proven here:
5
6 * publication never deletes and never overwrites;
7 * a version maps to one hash for ever;
8 * build/validate/publish/activate/promote/rollback/delete are separate operations;
9 * cleanup is dry-run by default, names the exact deletion set, and preserves
10 anything it cannot positively prove safe.
11 """
12
13 from __future__ import annotations
14
15 import json
16 import pathlib
17
18 import pytest
19
20 from deploy.release_gate import GateError
21 from deploy.release_safety import (
22 ObjectExists,
23 ReleaseSafetyError,
24 StorageUnavailable,
25 activate,
26 execute_cleanup,
27 plan_cleanup,
28 publish,
29 )
30 from tests.release_fixtures import observed, CHECKOUT, NOW, FakeStorage, make_artifact, make_report, sha256_of
31
32 STAGING = "pointers/staging.json"
33 PRODUCTION = "pointers/production.json"
34
35
36 def release(storage: FakeStorage, tmp: pathlib.Path, version: str, payload: bytes | None = None) -> str:
37 """Publish *version* through the real path; returns its sha256."""
38 art = make_artifact(tmp, version, payload)
39 publish(storage, art, make_report(art), now=NOW, checkout=CHECKOUT)
40 return sha256_of(art)
41
42
43 @pytest.fixture
44 def storage() -> FakeStorage:
45 return FakeStorage()
46
47
48 class TestPublishImmutability:
49 def test_publish_writes_artifact_and_record_only(self, storage: FakeStorage, tmp_path: pathlib.Path) -> None:
50 sha = release(storage, tmp_path, "0.2.1rc19")
51 assert sorted(storage.objects) == ["muse-0.2.1rc19.release.json", "muse-0.2.1rc19.tar.gz"]
52 record = json.loads(storage.objects["muse-0.2.1rc19.release.json"])
53 assert record["version"] == "0.2.1rc19" and record["sha256"] == sha
54 assert record["source_revision"] and record["gate_report_sha256"]
55 assert storage.deletes() == []
56
57 def test_same_version_same_hash_is_idempotent(self, storage: FakeStorage, tmp_path: pathlib.Path) -> None:
58 art = make_artifact(tmp_path, "0.2.1rc19")
59 publish(storage, art, make_report(art), now=NOW, checkout=CHECKOUT)
60 before = storage.snapshot()
61 storage.ops.clear()
62 result = publish(storage, art, make_report(art), now=NOW, checkout=CHECKOUT)
63 assert result.status == "already_published"
64 assert storage.snapshot() == before and storage.writes() == []
65
66 def test_same_version_different_hash_fails_and_preserves_the_original(self, storage: FakeStorage, tmp_path: pathlib.Path) -> None:
67 release(storage, tmp_path, "0.2.1rc19", b"original bytes")
68 before = storage.snapshot()
69 storage.ops.clear()
70 other = tmp_path / "other"
71 other.mkdir()
72 art = make_artifact(other, "0.2.1rc19", b"DIFFERENT bytes")
73 with pytest.raises(ReleaseSafetyError, match="different"):
74 publish(storage, art, make_report(art), now=NOW, checkout=CHECKOUT)
75 assert storage.snapshot() == before and storage.ops == []
76
77 def test_storage_refuses_to_overwrite_an_existing_object(self, storage: FakeStorage) -> None:
78 storage.put_new("muse-1.tar.gz", b"a")
79 with pytest.raises(ObjectExists):
80 storage.put_new("muse-1.tar.gz", b"b")
81 assert storage.objects["muse-1.tar.gz"] == b"a"
82
83 def test_missing_artifact_is_rejected_without_writes(self, storage: FakeStorage, tmp_path: pathlib.Path) -> None:
84 art = make_artifact(tmp_path, "0.2.1rc19")
85 report = make_report(art)
86 art.unlink()
87 with pytest.raises((GateError, ReleaseSafetyError)):
88 publish(storage, art, report, now=NOW, checkout=CHECKOUT)
89 assert storage.ops == [] and storage.objects == {}
90
91 def test_publishing_a_new_release_preserves_every_existing_one(self, storage: FakeStorage, tmp_path: pathlib.Path) -> None:
92 for v in ("0.2.1rc16", "0.2.1rc17", "0.2.1rc18"):
93 release(storage, tmp_path, v)
94 before = set(storage.objects)
95 release(storage, tmp_path, "0.2.1rc19")
96 assert before <= set(storage.objects)
97 assert storage.deletes() == []
98
99 def test_unavailable_storage_fails_closed(self, storage: FakeStorage, tmp_path: pathlib.Path) -> None:
100 art = make_artifact(tmp_path, "0.2.1rc19")
101 storage.unavailable = True
102 with pytest.raises(StorageUnavailable):
103 publish(storage, art, make_report(art), now=NOW, checkout=CHECKOUT)
104 assert storage.objects == {} and storage.ops == []
105
106 def test_interrupted_publication_is_not_a_release_and_is_completed_by_rerun(self, storage: FakeStorage, tmp_path: pathlib.Path) -> None:
107 art = make_artifact(tmp_path, "0.2.1rc19")
108 storage.fail_put_new_on = "muse-0.2.1rc19.release.json"
109 with pytest.raises(StorageUnavailable):
110 publish(storage, art, make_report(art), now=NOW, checkout=CHECKOUT)
111 # Artifact landed, record did not: the release is not activatable and no pointer exists.
112 assert "muse-0.2.1rc19.tar.gz" in storage.objects
113 with pytest.raises(ReleaseSafetyError, match="record"):
114 activate(storage, "staging", "0.2.1rc19", sha256_of(art), expect=observed(storage, "staging"), now=NOW)
115 assert STAGING not in storage.objects
116 # Re-running with the same bytes completes it.
117 storage.fail_put_new_on = None
118 assert publish(storage, art, make_report(art), now=NOW, checkout=CHECKOUT).status == "completed"
119 assert "muse-0.2.1rc19.release.json" in storage.objects
120
121 def test_orphan_artifact_with_different_bytes_is_never_adopted_by_rerun(self, storage: FakeStorage, tmp_path: pathlib.Path) -> None:
122 storage.objects["muse-0.2.1rc19.tar.gz"] = b"orphan of unknown provenance"
123 art = make_artifact(tmp_path, "0.2.1rc19")
124 with pytest.raises(ReleaseSafetyError, match="different"):
125 publish(storage, art, make_report(art), now=NOW, checkout=CHECKOUT)
126 assert storage.ops == []
127
128
129 class TestActivationAndPromotion:
130 def test_staging_activation_writes_only_the_staging_pointer(self, storage: FakeStorage, tmp_path: pathlib.Path) -> None:
131 sha = release(storage, tmp_path, "0.2.1rc19")
132 storage.ops.clear()
133 activate(storage, "staging", "0.2.1rc19", sha, expect=observed(storage, "staging"), now=NOW)
134 assert storage.ops == [("put_new", STAGING)]
135 assert PRODUCTION not in storage.objects
136
137 def test_activation_with_the_wrong_hash_is_refused(self, storage: FakeStorage, tmp_path: pathlib.Path) -> None:
138 release(storage, tmp_path, "0.2.1rc19")
139 storage.ops.clear()
140 with pytest.raises(ReleaseSafetyError, match="sha256"):
141 activate(storage, "staging", "0.2.1rc19", "0" * 64, expect=observed(storage, "staging"), now=NOW)
142 assert storage.ops == []
143
144 def test_activating_an_unpublished_version_is_refused(self, storage: FakeStorage) -> None:
145 with pytest.raises(ReleaseSafetyError):
146 activate(storage, "staging", "9.9.9", "0" * 64, expect=observed(storage, "staging"), now=NOW)
147 assert storage.ops == []
148
149 def test_unknown_environment_is_refused(self, storage: FakeStorage, tmp_path: pathlib.Path) -> None:
150 sha = release(storage, tmp_path, "0.2.1rc19")
151 with pytest.raises(ReleaseSafetyError, match="environment"):
152 activate(storage, "prod", "0.2.1rc19", sha, expect=observed(storage, "prod"), now=NOW)
153
154 def test_production_requires_explicit_approval(self, storage: FakeStorage, tmp_path: pathlib.Path) -> None:
155 sha = release(storage, tmp_path, "0.2.1rc19")
156 activate(storage, "staging", "0.2.1rc19", sha, expect=observed(storage, "staging"), now=NOW)
157 storage.ops.clear()
158 with pytest.raises(ReleaseSafetyError, match="approv"):
159 activate(storage, "production", "0.2.1rc19", sha, expect=observed(storage, "production"), now=NOW)
160 assert storage.ops == []
161
162 def test_production_only_takes_what_staging_has_active(self, storage: FakeStorage, tmp_path: pathlib.Path) -> None:
163 sha18 = release(storage, tmp_path, "0.2.1rc18")
164 sha19 = release(storage, tmp_path, "0.2.1rc19")
165 activate(storage, "staging", "0.2.1rc19", sha19, expect=observed(storage, "staging"), now=NOW)
166 storage.ops.clear()
167 with pytest.raises(ReleaseSafetyError, match="staging"):
168 activate(storage, "production", "0.2.1rc18", sha18, expect=observed(storage, "production"), now=NOW, approve_production=True)
169 assert storage.ops == []
170
171 def test_promotion_with_approval_moves_only_the_production_pointer(self, storage: FakeStorage, tmp_path: pathlib.Path) -> None:
172 sha = release(storage, tmp_path, "0.2.1rc19")
173 activate(storage, "staging", "0.2.1rc19", sha, expect=observed(storage, "staging"), now=NOW)
174 staging_before = storage.objects[STAGING]
175 storage.ops.clear()
176 activate(storage, "production", "0.2.1rc19", sha, expect=observed(storage, "production"), now=NOW, approve_production=True)
177 assert storage.ops == [("put_new", PRODUCTION)]
178 assert storage.objects[STAGING] == staging_before
179
180 def test_pointer_remembers_the_previous_release(self, storage: FakeStorage, tmp_path: pathlib.Path) -> None:
181 sha18 = release(storage, tmp_path, "0.2.1rc18")
182 sha19 = release(storage, tmp_path, "0.2.1rc19")
183 activate(storage, "staging", "0.2.1rc18", sha18, expect=observed(storage, "staging"), now=NOW)
184 activate(storage, "staging", "0.2.1rc19", sha19, expect=observed(storage, "staging"), now=NOW)
185 pointer = json.loads(storage.objects[STAGING])
186 assert pointer["version"] == "0.2.1rc19"
187 assert pointer["previous"] == {"version": "0.2.1rc18", "sha256": sha18}
188
189
190 class TestCleanup:
191 def _fleet(self, storage: FakeStorage, tmp: pathlib.Path) -> dict[str, str]:
192 shas = {v: release(storage, tmp, v) for v in ("0.2.1rc16", "0.2.1rc17", "0.2.1rc18", "0.2.1rc19")}
193 activate(storage, "staging", "0.2.1rc19", shas["0.2.1rc19"], expect=observed(storage, "staging"), now=NOW)
194 activate(storage, "production", "0.2.1rc19", shas["0.2.1rc19"], expect=observed(storage, "production"), now=NOW, approve_production=True)
195 return shas
196
197 def test_plan_is_pure_and_names_the_exact_deletion_set(self, storage: FakeStorage, tmp_path: pathlib.Path) -> None:
198 self._fleet(storage, tmp_path)
199 storage.ops.clear()
200 plan = plan_cleanup(storage, keep=2)
201 assert storage.ops == []
202 assert plan.blocked is None
203 assert plan.deletions == ["muse-0.2.1rc16.release.json", "muse-0.2.1rc16.tar.gz",
204 "muse-0.2.1rc17.release.json", "muse-0.2.1rc17.tar.gz"]
205 assert plan.digest
206
207 def test_newest_by_pep440_not_by_time_are_kept(self, storage: FakeStorage, tmp_path: pathlib.Path) -> None:
208 self._fleet(storage, tmp_path)
209 assert not {"muse-0.2.1rc18.tar.gz", "muse-0.2.1rc19.tar.gz"} & set(plan_cleanup(storage, keep=2).deletions)
210
211 def test_execute_requires_the_matching_digest(self, storage: FakeStorage, tmp_path: pathlib.Path) -> None:
212 self._fleet(storage, tmp_path)
213 plan = plan_cleanup(storage, keep=2)
214 before = storage.snapshot()
215 for bad in ("", "sha256:nope", "0" * 64):
216 with pytest.raises(ReleaseSafetyError, match="confirm"):
217 execute_cleanup(storage, plan, confirm=bad)
218 assert storage.snapshot() == before and storage.deletes() == []
219
220 def test_execute_with_the_digest_deletes_exactly_the_plan(self, storage: FakeStorage, tmp_path: pathlib.Path) -> None:
221 self._fleet(storage, tmp_path)
222 plan = plan_cleanup(storage, keep=2)
223 deleted = execute_cleanup(storage, plan, confirm=plan.digest)
224 assert deleted == plan.deletions
225 assert sorted(storage.deletes()) == sorted(plan.deletions)
226 assert "muse-0.2.1rc18.tar.gz" in storage.objects
227
228 def test_a_plan_gone_stale_since_it_was_shown_is_refused(self, storage: FakeStorage, tmp_path: pathlib.Path) -> None:
229 shas = self._fleet(storage, tmp_path)
230 plan = plan_cleanup(storage, keep=2)
231 # Between dry-run and execute, production is rolled onto rc17.
232 pointer = json.loads(storage.objects[PRODUCTION])
233 pointer["previous"] = {"version": "0.2.1rc17", "sha256": shas["0.2.1rc17"]}
234 storage.objects[PRODUCTION] = json.dumps(pointer).encode()
235 with pytest.raises(ReleaseSafetyError, match="stale"):
236 execute_cleanup(storage, plan, confirm=plan.digest)
237 assert storage.deletes() == []
238
239 def test_hand_built_plan_with_an_unverified_target_is_refused(self, storage: FakeStorage, tmp_path: pathlib.Path) -> None:
240 self._fleet(storage, tmp_path)
241 plan = plan_cleanup(storage, keep=2)
242 forged = type(plan)(deletions=[*plan.deletions, "muse-0.2.1rc19.tar.gz"], preserved=plan.preserved, blocked=None)
243 with pytest.raises(ReleaseSafetyError):
244 execute_cleanup(storage, forged, confirm=forged.digest)
245 assert storage.deletes() == []
246
247 def test_artifacts_either_environment_points_at_are_never_deletable(self, storage: FakeStorage, tmp_path: pathlib.Path) -> None:
248 shas = self._fleet(storage, tmp_path)
249 # staging moves forward to rc18 only; production stays on rc19 with rc17 as rollback target
250 activate(storage, "staging", "0.2.1rc18", shas["0.2.1rc18"], expect=observed(storage, "staging"), now=NOW)
251 pointer = json.loads(storage.objects[PRODUCTION])
252 pointer["previous"] = {"version": "0.2.1rc16", "sha256": shas["0.2.1rc16"]}
253 storage.objects[PRODUCTION] = json.dumps(pointer).encode()
254 plan = plan_cleanup(storage, keep=0)
255 assert plan.deletions == ["muse-0.2.1rc17.release.json", "muse-0.2.1rc17.tar.gz"]
256 for protected in ("0.2.1rc16", "0.2.1rc18", "0.2.1rc19"): # prod-rollback, staging active+previous, both active
257 assert f"muse-{protected}.tar.gz" not in plan.deletions
258 assert f"muse-{protected}.tar.gz" in plan.preserved
259
260 def test_the_incident_staging_publish_cannot_remove_the_production_rollback_target(self, storage: FakeStorage, tmp_path: pathlib.Path) -> None:
261 shas = {v: release(storage, tmp_path, v) for v in ("0.2.1rc16", "0.2.1rc17", "0.2.1rc18")}
262 activate(storage, "staging", "0.2.1rc18", shas["0.2.1rc18"], expect=observed(storage, "staging"), now=NOW)
263 activate(storage, "production", "0.2.1rc18", shas["0.2.1rc18"], expect=observed(storage, "production"), now=NOW, approve_production=True)
264 pointer = json.loads(storage.objects[PRODUCTION])
265 pointer["previous"] = {"version": "0.2.1rc16", "sha256": shas["0.2.1rc16"]}
266 storage.objects[PRODUCTION] = json.dumps(pointer).encode()
267 release(storage, tmp_path, "0.2.1rc19") # the publish that used to delete rc16
268 assert "muse-0.2.1rc16.tar.gz" in storage.objects
269 assert "muse-0.2.1rc16.tar.gz" not in plan_cleanup(storage, keep=3).deletions
270
271 @pytest.mark.parametrize("damage", ["missing_production", "missing_staging", "malformed", "unavailable"])
272 def test_ambiguous_ownership_blocks_all_deletion(self, storage: FakeStorage, tmp_path: pathlib.Path, damage: str) -> None:
273 self._fleet(storage, tmp_path)
274 if damage == "missing_production":
275 del storage.objects[PRODUCTION]
276 elif damage == "missing_staging":
277 del storage.objects[STAGING]
278 elif damage == "malformed":
279 storage.objects[PRODUCTION] = b"{not json"
280 else:
281 storage.unavailable = True
282 if damage == "unavailable":
283 with pytest.raises(StorageUnavailable):
284 plan_cleanup(storage, keep=0)
285 return
286 plan = plan_cleanup(storage, keep=0)
287 assert plan.deletions == [] and plan.blocked
288 with pytest.raises(ReleaseSafetyError):
289 execute_cleanup(storage, plan, confirm=plan.digest)
290 assert storage.deletes() == []
291
292 @pytest.mark.parametrize("damage", ["no_record", "bad_record", "hash_drift"])
293 def test_a_release_that_cannot_be_verified_is_preserved(self, storage: FakeStorage, tmp_path: pathlib.Path, damage: str) -> None:
294 self._fleet(storage, tmp_path)
295 if damage == "no_record":
296 del storage.objects["muse-0.2.1rc16.release.json"]
297 elif damage == "bad_record":
298 storage.objects["muse-0.2.1rc16.release.json"] = b"[]"
299 else:
300 storage.objects["muse-0.2.1rc16.tar.gz"] = b"bytes no longer matching the record"
301 plan = plan_cleanup(storage, keep=2)
302 assert "muse-0.2.1rc16.tar.gz" not in plan.deletions
303 assert "muse-0.2.1rc16.tar.gz" in plan.preserved
304
305 def test_unrelated_objects_in_the_shared_bucket_are_never_candidates(self, storage: FakeStorage, tmp_path: pathlib.Path) -> None:
306 self._fleet(storage, tmp_path)
307 for key in ("db-backups/dump.sql", "tmp/x", "staging-db-backups/y", "muse-weird.tar.gz"):
308 storage.objects[key] = b"x"
309 plan = plan_cleanup(storage, keep=0)
310 assert not [k for k in plan.deletions if not k.startswith("muse-0.2.1rc")]
File History 4 commits
sha256:675003834b19b109f8cb126e543a064a7bd6256930dd4a80ffcfe87c6eff3f17 fix(release): idempotent concurrent publish (D3) and compar… Opus 5.5 minor ⚠ 12 days ago
sha256:04db68e43d9007348dbd726b2a3e52f73e83c5646a4511c07e19cb9a66e28b5f fix(release): gate provenance is observed from the checkout… Opus 5.5 minor ⚠ 12 days ago
sha256:9f814406166bc157b00dfd4078fb4516ad46e9844628da6bb3823e8448eca1e9 fix(release): rollback is an explicit, idempotent, stale-pr… Opus 5.5 minor ⚠ 12 days ago
sha256:c1e420ecafdcfabfd74f76e07d538ec0c884be32b14f6062d32cb19690a3d4d3 test(release): red tests for artifact immutability, retenti… Opus 5.5 12 days ago