test_release_safety.py
python
sha256:675003834b19b109f8cb126e543a064a7bd6256930dd4a80ffcfe87c6eff3f17
fix(release): idempotent concurrent publish (D3) and compar…
Opus 5.5
minor
⚠ breaking
12 days ago
| 1 | """Release artifact immutability, activation, rollback and retention (musehub#114 Phase 6). |
| 2 | |
| 3 | Incident this answers: publishing rc19 to *staging* deleted rc16 from the bucket |
| 4 | shared with *production*. The invariants proven here: |
| 5 | |
| 6 | * publication never deletes and never overwrites; |
| 7 | * a version maps to one hash for ever; |
| 8 | * build/validate/publish/activate/promote/rollback/delete are separate operations; |
| 9 | * cleanup is dry-run by default, names the exact deletion set, and preserves |
| 10 | anything it cannot positively prove safe. |
| 11 | """ |
| 12 | |
| 13 | from __future__ import annotations |
| 14 | |
| 15 | import json |
| 16 | import pathlib |
| 17 | |
| 18 | import pytest |
| 19 | |
| 20 | from deploy.release_gate import GateError |
| 21 | from deploy.release_safety import ( |
| 22 | ObjectExists, |
| 23 | ReleaseSafetyError, |
| 24 | StorageUnavailable, |
| 25 | activate, |
| 26 | execute_cleanup, |
| 27 | plan_cleanup, |
| 28 | publish, |
| 29 | ) |
| 30 | from tests.release_fixtures import observed, CHECKOUT, NOW, FakeStorage, make_artifact, make_report, sha256_of |
| 31 | |
| 32 | STAGING = "pointers/staging.json" |
| 33 | PRODUCTION = "pointers/production.json" |
| 34 | |
| 35 | |
| 36 | def release(storage: FakeStorage, tmp: pathlib.Path, version: str, payload: bytes | None = None) -> str: |
| 37 | """Publish *version* through the real path; returns its sha256.""" |
| 38 | art = make_artifact(tmp, version, payload) |
| 39 | publish(storage, art, make_report(art), now=NOW, checkout=CHECKOUT) |
| 40 | return sha256_of(art) |
| 41 | |
| 42 | |
| 43 | @pytest.fixture |
| 44 | def storage() -> FakeStorage: |
| 45 | return FakeStorage() |
| 46 | |
| 47 | |
| 48 | class TestPublishImmutability: |
| 49 | def test_publish_writes_artifact_and_record_only(self, storage: FakeStorage, tmp_path: pathlib.Path) -> None: |
| 50 | sha = release(storage, tmp_path, "0.2.1rc19") |
| 51 | assert sorted(storage.objects) == ["muse-0.2.1rc19.release.json", "muse-0.2.1rc19.tar.gz"] |
| 52 | record = json.loads(storage.objects["muse-0.2.1rc19.release.json"]) |
| 53 | assert record["version"] == "0.2.1rc19" and record["sha256"] == sha |
| 54 | assert record["source_revision"] and record["gate_report_sha256"] |
| 55 | assert storage.deletes() == [] |
| 56 | |
| 57 | def test_same_version_same_hash_is_idempotent(self, storage: FakeStorage, tmp_path: pathlib.Path) -> None: |
| 58 | art = make_artifact(tmp_path, "0.2.1rc19") |
| 59 | publish(storage, art, make_report(art), now=NOW, checkout=CHECKOUT) |
| 60 | before = storage.snapshot() |
| 61 | storage.ops.clear() |
| 62 | result = publish(storage, art, make_report(art), now=NOW, checkout=CHECKOUT) |
| 63 | assert result.status == "already_published" |
| 64 | assert storage.snapshot() == before and storage.writes() == [] |
| 65 | |
| 66 | def test_same_version_different_hash_fails_and_preserves_the_original(self, storage: FakeStorage, tmp_path: pathlib.Path) -> None: |
| 67 | release(storage, tmp_path, "0.2.1rc19", b"original bytes") |
| 68 | before = storage.snapshot() |
| 69 | storage.ops.clear() |
| 70 | other = tmp_path / "other" |
| 71 | other.mkdir() |
| 72 | art = make_artifact(other, "0.2.1rc19", b"DIFFERENT bytes") |
| 73 | with pytest.raises(ReleaseSafetyError, match="different"): |
| 74 | publish(storage, art, make_report(art), now=NOW, checkout=CHECKOUT) |
| 75 | assert storage.snapshot() == before and storage.ops == [] |
| 76 | |
| 77 | def test_storage_refuses_to_overwrite_an_existing_object(self, storage: FakeStorage) -> None: |
| 78 | storage.put_new("muse-1.tar.gz", b"a") |
| 79 | with pytest.raises(ObjectExists): |
| 80 | storage.put_new("muse-1.tar.gz", b"b") |
| 81 | assert storage.objects["muse-1.tar.gz"] == b"a" |
| 82 | |
| 83 | def test_missing_artifact_is_rejected_without_writes(self, storage: FakeStorage, tmp_path: pathlib.Path) -> None: |
| 84 | art = make_artifact(tmp_path, "0.2.1rc19") |
| 85 | report = make_report(art) |
| 86 | art.unlink() |
| 87 | with pytest.raises((GateError, ReleaseSafetyError)): |
| 88 | publish(storage, art, report, now=NOW, checkout=CHECKOUT) |
| 89 | assert storage.ops == [] and storage.objects == {} |
| 90 | |
| 91 | def test_publishing_a_new_release_preserves_every_existing_one(self, storage: FakeStorage, tmp_path: pathlib.Path) -> None: |
| 92 | for v in ("0.2.1rc16", "0.2.1rc17", "0.2.1rc18"): |
| 93 | release(storage, tmp_path, v) |
| 94 | before = set(storage.objects) |
| 95 | release(storage, tmp_path, "0.2.1rc19") |
| 96 | assert before <= set(storage.objects) |
| 97 | assert storage.deletes() == [] |
| 98 | |
| 99 | def test_unavailable_storage_fails_closed(self, storage: FakeStorage, tmp_path: pathlib.Path) -> None: |
| 100 | art = make_artifact(tmp_path, "0.2.1rc19") |
| 101 | storage.unavailable = True |
| 102 | with pytest.raises(StorageUnavailable): |
| 103 | publish(storage, art, make_report(art), now=NOW, checkout=CHECKOUT) |
| 104 | assert storage.objects == {} and storage.ops == [] |
| 105 | |
| 106 | def test_interrupted_publication_is_not_a_release_and_is_completed_by_rerun(self, storage: FakeStorage, tmp_path: pathlib.Path) -> None: |
| 107 | art = make_artifact(tmp_path, "0.2.1rc19") |
| 108 | storage.fail_put_new_on = "muse-0.2.1rc19.release.json" |
| 109 | with pytest.raises(StorageUnavailable): |
| 110 | publish(storage, art, make_report(art), now=NOW, checkout=CHECKOUT) |
| 111 | # Artifact landed, record did not: the release is not activatable and no pointer exists. |
| 112 | assert "muse-0.2.1rc19.tar.gz" in storage.objects |
| 113 | with pytest.raises(ReleaseSafetyError, match="record"): |
| 114 | activate(storage, "staging", "0.2.1rc19", sha256_of(art), expect=observed(storage, "staging"), now=NOW) |
| 115 | assert STAGING not in storage.objects |
| 116 | # Re-running with the same bytes completes it. |
| 117 | storage.fail_put_new_on = None |
| 118 | assert publish(storage, art, make_report(art), now=NOW, checkout=CHECKOUT).status == "completed" |
| 119 | assert "muse-0.2.1rc19.release.json" in storage.objects |
| 120 | |
| 121 | def test_orphan_artifact_with_different_bytes_is_never_adopted_by_rerun(self, storage: FakeStorage, tmp_path: pathlib.Path) -> None: |
| 122 | storage.objects["muse-0.2.1rc19.tar.gz"] = b"orphan of unknown provenance" |
| 123 | art = make_artifact(tmp_path, "0.2.1rc19") |
| 124 | with pytest.raises(ReleaseSafetyError, match="different"): |
| 125 | publish(storage, art, make_report(art), now=NOW, checkout=CHECKOUT) |
| 126 | assert storage.ops == [] |
| 127 | |
| 128 | |
| 129 | class TestActivationAndPromotion: |
| 130 | def test_staging_activation_writes_only_the_staging_pointer(self, storage: FakeStorage, tmp_path: pathlib.Path) -> None: |
| 131 | sha = release(storage, tmp_path, "0.2.1rc19") |
| 132 | storage.ops.clear() |
| 133 | activate(storage, "staging", "0.2.1rc19", sha, expect=observed(storage, "staging"), now=NOW) |
| 134 | assert storage.ops == [("put_new", STAGING)] |
| 135 | assert PRODUCTION not in storage.objects |
| 136 | |
| 137 | def test_activation_with_the_wrong_hash_is_refused(self, storage: FakeStorage, tmp_path: pathlib.Path) -> None: |
| 138 | release(storage, tmp_path, "0.2.1rc19") |
| 139 | storage.ops.clear() |
| 140 | with pytest.raises(ReleaseSafetyError, match="sha256"): |
| 141 | activate(storage, "staging", "0.2.1rc19", "0" * 64, expect=observed(storage, "staging"), now=NOW) |
| 142 | assert storage.ops == [] |
| 143 | |
| 144 | def test_activating_an_unpublished_version_is_refused(self, storage: FakeStorage) -> None: |
| 145 | with pytest.raises(ReleaseSafetyError): |
| 146 | activate(storage, "staging", "9.9.9", "0" * 64, expect=observed(storage, "staging"), now=NOW) |
| 147 | assert storage.ops == [] |
| 148 | |
| 149 | def test_unknown_environment_is_refused(self, storage: FakeStorage, tmp_path: pathlib.Path) -> None: |
| 150 | sha = release(storage, tmp_path, "0.2.1rc19") |
| 151 | with pytest.raises(ReleaseSafetyError, match="environment"): |
| 152 | activate(storage, "prod", "0.2.1rc19", sha, expect=observed(storage, "prod"), now=NOW) |
| 153 | |
| 154 | def test_production_requires_explicit_approval(self, storage: FakeStorage, tmp_path: pathlib.Path) -> None: |
| 155 | sha = release(storage, tmp_path, "0.2.1rc19") |
| 156 | activate(storage, "staging", "0.2.1rc19", sha, expect=observed(storage, "staging"), now=NOW) |
| 157 | storage.ops.clear() |
| 158 | with pytest.raises(ReleaseSafetyError, match="approv"): |
| 159 | activate(storage, "production", "0.2.1rc19", sha, expect=observed(storage, "production"), now=NOW) |
| 160 | assert storage.ops == [] |
| 161 | |
| 162 | def test_production_only_takes_what_staging_has_active(self, storage: FakeStorage, tmp_path: pathlib.Path) -> None: |
| 163 | sha18 = release(storage, tmp_path, "0.2.1rc18") |
| 164 | sha19 = release(storage, tmp_path, "0.2.1rc19") |
| 165 | activate(storage, "staging", "0.2.1rc19", sha19, expect=observed(storage, "staging"), now=NOW) |
| 166 | storage.ops.clear() |
| 167 | with pytest.raises(ReleaseSafetyError, match="staging"): |
| 168 | activate(storage, "production", "0.2.1rc18", sha18, expect=observed(storage, "production"), now=NOW, approve_production=True) |
| 169 | assert storage.ops == [] |
| 170 | |
| 171 | def test_promotion_with_approval_moves_only_the_production_pointer(self, storage: FakeStorage, tmp_path: pathlib.Path) -> None: |
| 172 | sha = release(storage, tmp_path, "0.2.1rc19") |
| 173 | activate(storage, "staging", "0.2.1rc19", sha, expect=observed(storage, "staging"), now=NOW) |
| 174 | staging_before = storage.objects[STAGING] |
| 175 | storage.ops.clear() |
| 176 | activate(storage, "production", "0.2.1rc19", sha, expect=observed(storage, "production"), now=NOW, approve_production=True) |
| 177 | assert storage.ops == [("put_new", PRODUCTION)] |
| 178 | assert storage.objects[STAGING] == staging_before |
| 179 | |
| 180 | def test_pointer_remembers_the_previous_release(self, storage: FakeStorage, tmp_path: pathlib.Path) -> None: |
| 181 | sha18 = release(storage, tmp_path, "0.2.1rc18") |
| 182 | sha19 = release(storage, tmp_path, "0.2.1rc19") |
| 183 | activate(storage, "staging", "0.2.1rc18", sha18, expect=observed(storage, "staging"), now=NOW) |
| 184 | activate(storage, "staging", "0.2.1rc19", sha19, expect=observed(storage, "staging"), now=NOW) |
| 185 | pointer = json.loads(storage.objects[STAGING]) |
| 186 | assert pointer["version"] == "0.2.1rc19" |
| 187 | assert pointer["previous"] == {"version": "0.2.1rc18", "sha256": sha18} |
| 188 | |
| 189 | |
| 190 | class TestCleanup: |
| 191 | def _fleet(self, storage: FakeStorage, tmp: pathlib.Path) -> dict[str, str]: |
| 192 | shas = {v: release(storage, tmp, v) for v in ("0.2.1rc16", "0.2.1rc17", "0.2.1rc18", "0.2.1rc19")} |
| 193 | activate(storage, "staging", "0.2.1rc19", shas["0.2.1rc19"], expect=observed(storage, "staging"), now=NOW) |
| 194 | activate(storage, "production", "0.2.1rc19", shas["0.2.1rc19"], expect=observed(storage, "production"), now=NOW, approve_production=True) |
| 195 | return shas |
| 196 | |
| 197 | def test_plan_is_pure_and_names_the_exact_deletion_set(self, storage: FakeStorage, tmp_path: pathlib.Path) -> None: |
| 198 | self._fleet(storage, tmp_path) |
| 199 | storage.ops.clear() |
| 200 | plan = plan_cleanup(storage, keep=2) |
| 201 | assert storage.ops == [] |
| 202 | assert plan.blocked is None |
| 203 | assert plan.deletions == ["muse-0.2.1rc16.release.json", "muse-0.2.1rc16.tar.gz", |
| 204 | "muse-0.2.1rc17.release.json", "muse-0.2.1rc17.tar.gz"] |
| 205 | assert plan.digest |
| 206 | |
| 207 | def test_newest_by_pep440_not_by_time_are_kept(self, storage: FakeStorage, tmp_path: pathlib.Path) -> None: |
| 208 | self._fleet(storage, tmp_path) |
| 209 | assert not {"muse-0.2.1rc18.tar.gz", "muse-0.2.1rc19.tar.gz"} & set(plan_cleanup(storage, keep=2).deletions) |
| 210 | |
| 211 | def test_execute_requires_the_matching_digest(self, storage: FakeStorage, tmp_path: pathlib.Path) -> None: |
| 212 | self._fleet(storage, tmp_path) |
| 213 | plan = plan_cleanup(storage, keep=2) |
| 214 | before = storage.snapshot() |
| 215 | for bad in ("", "sha256:nope", "0" * 64): |
| 216 | with pytest.raises(ReleaseSafetyError, match="confirm"): |
| 217 | execute_cleanup(storage, plan, confirm=bad) |
| 218 | assert storage.snapshot() == before and storage.deletes() == [] |
| 219 | |
| 220 | def test_execute_with_the_digest_deletes_exactly_the_plan(self, storage: FakeStorage, tmp_path: pathlib.Path) -> None: |
| 221 | self._fleet(storage, tmp_path) |
| 222 | plan = plan_cleanup(storage, keep=2) |
| 223 | deleted = execute_cleanup(storage, plan, confirm=plan.digest) |
| 224 | assert deleted == plan.deletions |
| 225 | assert sorted(storage.deletes()) == sorted(plan.deletions) |
| 226 | assert "muse-0.2.1rc18.tar.gz" in storage.objects |
| 227 | |
| 228 | def test_a_plan_gone_stale_since_it_was_shown_is_refused(self, storage: FakeStorage, tmp_path: pathlib.Path) -> None: |
| 229 | shas = self._fleet(storage, tmp_path) |
| 230 | plan = plan_cleanup(storage, keep=2) |
| 231 | # Between dry-run and execute, production is rolled onto rc17. |
| 232 | pointer = json.loads(storage.objects[PRODUCTION]) |
| 233 | pointer["previous"] = {"version": "0.2.1rc17", "sha256": shas["0.2.1rc17"]} |
| 234 | storage.objects[PRODUCTION] = json.dumps(pointer).encode() |
| 235 | with pytest.raises(ReleaseSafetyError, match="stale"): |
| 236 | execute_cleanup(storage, plan, confirm=plan.digest) |
| 237 | assert storage.deletes() == [] |
| 238 | |
| 239 | def test_hand_built_plan_with_an_unverified_target_is_refused(self, storage: FakeStorage, tmp_path: pathlib.Path) -> None: |
| 240 | self._fleet(storage, tmp_path) |
| 241 | plan = plan_cleanup(storage, keep=2) |
| 242 | forged = type(plan)(deletions=[*plan.deletions, "muse-0.2.1rc19.tar.gz"], preserved=plan.preserved, blocked=None) |
| 243 | with pytest.raises(ReleaseSafetyError): |
| 244 | execute_cleanup(storage, forged, confirm=forged.digest) |
| 245 | assert storage.deletes() == [] |
| 246 | |
| 247 | def test_artifacts_either_environment_points_at_are_never_deletable(self, storage: FakeStorage, tmp_path: pathlib.Path) -> None: |
| 248 | shas = self._fleet(storage, tmp_path) |
| 249 | # staging moves forward to rc18 only; production stays on rc19 with rc17 as rollback target |
| 250 | activate(storage, "staging", "0.2.1rc18", shas["0.2.1rc18"], expect=observed(storage, "staging"), now=NOW) |
| 251 | pointer = json.loads(storage.objects[PRODUCTION]) |
| 252 | pointer["previous"] = {"version": "0.2.1rc16", "sha256": shas["0.2.1rc16"]} |
| 253 | storage.objects[PRODUCTION] = json.dumps(pointer).encode() |
| 254 | plan = plan_cleanup(storage, keep=0) |
| 255 | assert plan.deletions == ["muse-0.2.1rc17.release.json", "muse-0.2.1rc17.tar.gz"] |
| 256 | for protected in ("0.2.1rc16", "0.2.1rc18", "0.2.1rc19"): # prod-rollback, staging active+previous, both active |
| 257 | assert f"muse-{protected}.tar.gz" not in plan.deletions |
| 258 | assert f"muse-{protected}.tar.gz" in plan.preserved |
| 259 | |
| 260 | def test_the_incident_staging_publish_cannot_remove_the_production_rollback_target(self, storage: FakeStorage, tmp_path: pathlib.Path) -> None: |
| 261 | shas = {v: release(storage, tmp_path, v) for v in ("0.2.1rc16", "0.2.1rc17", "0.2.1rc18")} |
| 262 | activate(storage, "staging", "0.2.1rc18", shas["0.2.1rc18"], expect=observed(storage, "staging"), now=NOW) |
| 263 | activate(storage, "production", "0.2.1rc18", shas["0.2.1rc18"], expect=observed(storage, "production"), now=NOW, approve_production=True) |
| 264 | pointer = json.loads(storage.objects[PRODUCTION]) |
| 265 | pointer["previous"] = {"version": "0.2.1rc16", "sha256": shas["0.2.1rc16"]} |
| 266 | storage.objects[PRODUCTION] = json.dumps(pointer).encode() |
| 267 | release(storage, tmp_path, "0.2.1rc19") # the publish that used to delete rc16 |
| 268 | assert "muse-0.2.1rc16.tar.gz" in storage.objects |
| 269 | assert "muse-0.2.1rc16.tar.gz" not in plan_cleanup(storage, keep=3).deletions |
| 270 | |
| 271 | @pytest.mark.parametrize("damage", ["missing_production", "missing_staging", "malformed", "unavailable"]) |
| 272 | def test_ambiguous_ownership_blocks_all_deletion(self, storage: FakeStorage, tmp_path: pathlib.Path, damage: str) -> None: |
| 273 | self._fleet(storage, tmp_path) |
| 274 | if damage == "missing_production": |
| 275 | del storage.objects[PRODUCTION] |
| 276 | elif damage == "missing_staging": |
| 277 | del storage.objects[STAGING] |
| 278 | elif damage == "malformed": |
| 279 | storage.objects[PRODUCTION] = b"{not json" |
| 280 | else: |
| 281 | storage.unavailable = True |
| 282 | if damage == "unavailable": |
| 283 | with pytest.raises(StorageUnavailable): |
| 284 | plan_cleanup(storage, keep=0) |
| 285 | return |
| 286 | plan = plan_cleanup(storage, keep=0) |
| 287 | assert plan.deletions == [] and plan.blocked |
| 288 | with pytest.raises(ReleaseSafetyError): |
| 289 | execute_cleanup(storage, plan, confirm=plan.digest) |
| 290 | assert storage.deletes() == [] |
| 291 | |
| 292 | @pytest.mark.parametrize("damage", ["no_record", "bad_record", "hash_drift"]) |
| 293 | def test_a_release_that_cannot_be_verified_is_preserved(self, storage: FakeStorage, tmp_path: pathlib.Path, damage: str) -> None: |
| 294 | self._fleet(storage, tmp_path) |
| 295 | if damage == "no_record": |
| 296 | del storage.objects["muse-0.2.1rc16.release.json"] |
| 297 | elif damage == "bad_record": |
| 298 | storage.objects["muse-0.2.1rc16.release.json"] = b"[]" |
| 299 | else: |
| 300 | storage.objects["muse-0.2.1rc16.tar.gz"] = b"bytes no longer matching the record" |
| 301 | plan = plan_cleanup(storage, keep=2) |
| 302 | assert "muse-0.2.1rc16.tar.gz" not in plan.deletions |
| 303 | assert "muse-0.2.1rc16.tar.gz" in plan.preserved |
| 304 | |
| 305 | def test_unrelated_objects_in_the_shared_bucket_are_never_candidates(self, storage: FakeStorage, tmp_path: pathlib.Path) -> None: |
| 306 | self._fleet(storage, tmp_path) |
| 307 | for key in ("db-backups/dump.sql", "tmp/x", "staging-db-backups/y", "muse-weird.tar.gz"): |
| 308 | storage.objects[key] = b"x" |
| 309 | plan = plan_cleanup(storage, keep=0) |
| 310 | assert not [k for k in plan.deletions if not k.startswith("muse-0.2.1rc")] |
File History
4 commits
sha256:675003834b19b109f8cb126e543a064a7bd6256930dd4a80ffcfe87c6eff3f17
fix(release): idempotent concurrent publish (D3) and compar…
Opus 5.5
minor
⚠
12 days ago
sha256:04db68e43d9007348dbd726b2a3e52f73e83c5646a4511c07e19cb9a66e28b5f
fix(release): gate provenance is observed from the checkout…
Opus 5.5
minor
⚠
12 days ago
sha256:9f814406166bc157b00dfd4078fb4516ad46e9844628da6bb3823e8448eca1e9
fix(release): rollback is an explicit, idempotent, stale-pr…
Opus 5.5
minor
⚠
12 days ago
sha256:c1e420ecafdcfabfd74f76e07d538ec0c884be32b14f6062d32cb19690a3d4d3
test(release): red tests for artifact immutability, retenti…
Opus 5.5
12 days ago