test_release_rollback.py
python
sha256:675003834b19b109f8cb126e543a064a7bd6256930dd4a80ffcfe87c6eff3f17
fix(release): idempotent concurrent publish (D3) and compar…
Opus 5.5
minor
⚠ breaking
12 days ago
| 1 | """Rollback is an explicit, idempotent, stale-proof target operation (musehub#114 Phase 6c, defect D1). |
| 2 | |
| 3 | The old ``rollback`` swapped ``active`` and ``previous``, so repeating it re-activated the |
| 4 | release being abandoned. A recovery request now names the release to restore AND the |
| 5 | pointer state the operator observed (version, sha256, activated_at); it is refused if the |
| 6 | pointer is anywhere else, and a request whose outcome is already in place is a no-op. |
| 7 | """ |
| 8 | |
| 9 | from __future__ import annotations |
| 10 | |
| 11 | import json |
| 12 | import pathlib |
| 13 | from datetime import timedelta |
| 14 | |
| 15 | import pytest |
| 16 | |
| 17 | from deploy.release_safety import ReleaseSafetyError, activate, rollback |
| 18 | from tests.release_fixtures import observed, CHECKOUT, NOW, FakeStorage, make_artifact, make_report, sha256_of |
| 19 | from deploy.release_safety import publish |
| 20 | |
| 21 | STAGING, PRODUCTION = "pointers/staging.json", "pointers/production.json" |
| 22 | |
| 23 | |
| 24 | def release(storage: FakeStorage, tmp: pathlib.Path, version: str) -> str: |
| 25 | art = make_artifact(tmp, version) |
| 26 | publish(storage, art, make_report(art), now=NOW, checkout=CHECKOUT) |
| 27 | return sha256_of(art) |
| 28 | |
| 29 | |
| 30 | class World: |
| 31 | """rc18 then rc19 active on staging (rc19 is the bad release); production on rc18.""" |
| 32 | |
| 33 | def __init__(self, tmp: pathlib.Path) -> None: |
| 34 | self.storage = FakeStorage() |
| 35 | self.sha18 = release(self.storage, tmp, "0.2.1rc18") |
| 36 | self.sha19 = release(self.storage, tmp, "0.2.1rc19") |
| 37 | activate(self.storage, "staging", "0.2.1rc18", self.sha18, expect=observed(self.storage, "staging"), now=NOW) |
| 38 | activate(self.storage, "production", "0.2.1rc18", self.sha18, expect=observed(self.storage, "production"), now=NOW, approve_production=True) |
| 39 | self.bad_at = (NOW + timedelta(minutes=5)) |
| 40 | activate(self.storage, "staging", "0.2.1rc19", self.sha19, expect=observed(self.storage, "staging"), now=self.bad_at) |
| 41 | |
| 42 | def request(self, **over: object) -> dict[str, object]: |
| 43 | req: dict[str, object] = dict( |
| 44 | to_version="0.2.1rc18", to_sha256=self.sha18, |
| 45 | expect_version="0.2.1rc19", expect_sha256=self.sha19, expect_activated_at=self.bad_at.isoformat(), |
| 46 | now=NOW + timedelta(minutes=10), |
| 47 | ) |
| 48 | req.update(over) |
| 49 | return req |
| 50 | |
| 51 | def pointer(self, env: str = "staging") -> dict[str, object]: |
| 52 | return json.loads(self.storage.objects[f"pointers/{env}.json"]) |
| 53 | |
| 54 | |
| 55 | @pytest.fixture() |
| 56 | def world(tmp_path: pathlib.Path) -> World: |
| 57 | return World(tmp_path) |
| 58 | |
| 59 | |
| 60 | def test_rollback_restores_the_named_release(world: World) -> None: |
| 61 | pointer = rollback(world.storage, "staging", **world.request()) # type: ignore[arg-type] |
| 62 | assert (pointer.version, pointer.sha256) == ("0.2.1rc18", world.sha18) |
| 63 | assert world.pointer()["version"] == "0.2.1rc18" |
| 64 | assert world.pointer()["previous"] == {"version": "0.2.1rc19", "sha256": world.sha19} |
| 65 | |
| 66 | |
| 67 | def test_repeating_the_same_request_never_reactivates_the_failed_release(world: World) -> None: |
| 68 | rollback(world.storage, "staging", **world.request()) # type: ignore[arg-type] |
| 69 | after_first = world.storage.snapshot() |
| 70 | world.storage.ops.clear() |
| 71 | for _ in range(3): |
| 72 | pointer = rollback(world.storage, "staging", **world.request(now=NOW + timedelta(hours=1))) # type: ignore[arg-type] |
| 73 | assert (pointer.version, pointer.sha256) == ("0.2.1rc18", world.sha18) |
| 74 | assert world.storage.snapshot() == after_first |
| 75 | assert world.storage.ops == [] # an already-satisfied request writes nothing |
| 76 | |
| 77 | |
| 78 | def test_interrupted_rollback_is_retried_with_the_same_request(world: World) -> None: |
| 79 | world.storage.fail_put_replace_on = STAGING # type: ignore[attr-defined] |
| 80 | with pytest.raises(Exception): |
| 81 | rollback(world.storage, "staging", **world.request()) # type: ignore[arg-type] |
| 82 | assert world.pointer()["version"] == "0.2.1rc19" # nothing half-written |
| 83 | world.storage.fail_put_replace_on = None # type: ignore[attr-defined] |
| 84 | assert rollback(world.storage, "staging", **world.request()).version == "0.2.1rc18" # type: ignore[arg-type] |
| 85 | assert rollback(world.storage, "staging", **world.request()).version == "0.2.1rc18" # type: ignore[arg-type] |
| 86 | |
| 87 | |
| 88 | def test_stale_request_cannot_reverse_a_newer_decision(world: World) -> None: |
| 89 | rollback(world.storage, "staging", **world.request()) # type: ignore[arg-type] |
| 90 | # an operator then deliberately re-activates rc19 (a newer decision) |
| 91 | activate(world.storage, "staging", "0.2.1rc19", world.sha19, expect=observed(world.storage, "staging"), now=NOW + timedelta(hours=2)) |
| 92 | before = world.storage.snapshot() |
| 93 | world.storage.ops.clear() |
| 94 | with pytest.raises(ReleaseSafetyError, match="stale|changed"): |
| 95 | rollback(world.storage, "staging", **world.request(now=NOW + timedelta(hours=3))) # type: ignore[arg-type] |
| 96 | assert world.storage.snapshot() == before and world.storage.ops == [] |
| 97 | |
| 98 | |
| 99 | @pytest.mark.parametrize("field,value", [ |
| 100 | ("expect_version", "0.2.1rc18"), ("expect_sha256", "0" * 64), ("expect_activated_at", "2000-01-01T00:00:00+00:00"), |
| 101 | ]) |
| 102 | def test_any_mismatch_with_the_observed_state_is_refused(world: World, field: str, value: str) -> None: |
| 103 | before = world.storage.snapshot() |
| 104 | with pytest.raises(ReleaseSafetyError): |
| 105 | rollback(world.storage, "staging", **world.request(**{field: value})) # type: ignore[arg-type] |
| 106 | assert world.storage.snapshot() == before |
| 107 | |
| 108 | |
| 109 | @pytest.mark.parametrize("damage", ["missing", "garbage", "wrong-schema", "wrong-env", "bad-hash", "not-object"]) |
| 110 | def test_missing_or_malformed_pointer_fails_closed(world: World, damage: str) -> None: |
| 111 | good = world.pointer() |
| 112 | if damage == "missing": |
| 113 | del world.storage.objects[STAGING] |
| 114 | else: |
| 115 | bad: object = { |
| 116 | "garbage": None, "wrong-schema": {**good, "schema": 99}, "wrong-env": {**good, "environment": "production"}, |
| 117 | "bad-hash": {**good, "sha256": "xyz"}, "not-object": [1], |
| 118 | }[damage] |
| 119 | world.storage.objects[STAGING] = b"{not json" if damage == "garbage" else json.dumps(bad).encode() |
| 120 | before = world.storage.snapshot() |
| 121 | world.storage.ops.clear() |
| 122 | with pytest.raises(ReleaseSafetyError): |
| 123 | rollback(world.storage, "staging", **world.request()) # type: ignore[arg-type] |
| 124 | assert world.storage.snapshot() == before and world.storage.ops == [] |
| 125 | |
| 126 | |
| 127 | def test_target_must_be_a_verified_published_release(world: World) -> None: |
| 128 | del world.storage.objects["muse-0.2.1rc18.tar.gz"] |
| 129 | before = world.storage.snapshot() |
| 130 | with pytest.raises(ReleaseSafetyError): |
| 131 | rollback(world.storage, "staging", **world.request()) # type: ignore[arg-type] |
| 132 | assert world.storage.snapshot() == before |
| 133 | with pytest.raises(ReleaseSafetyError): |
| 134 | rollback(world.storage, "staging", **world.request(to_version="0.2.1rc99", to_sha256="a" * 64)) # type: ignore[arg-type] |
| 135 | |
| 136 | |
| 137 | def test_target_hash_must_match_the_record(world: World) -> None: |
| 138 | with pytest.raises(ReleaseSafetyError): |
| 139 | rollback(world.storage, "staging", **world.request(to_sha256=world.sha19)) # type: ignore[arg-type] |
| 140 | |
| 141 | |
| 142 | def test_rollback_preserves_every_immutable_artifact_and_record(world: World) -> None: |
| 143 | immutable = {k: v for k, v in world.storage.objects.items() if not k.startswith("pointers/")} |
| 144 | world.storage.ops.clear() |
| 145 | rollback(world.storage, "staging", **world.request()) # type: ignore[arg-type] |
| 146 | rollback(world.storage, "staging", **world.request()) # type: ignore[arg-type] |
| 147 | assert {k: v for k, v in world.storage.objects.items() if not k.startswith("pointers/")} == immutable |
| 148 | assert world.storage.deletes() == [] |
| 149 | assert [k for _, k in world.storage.ops] == [STAGING] |
| 150 | |
| 151 | |
| 152 | def test_staging_rollback_never_touches_the_production_pointer(world: World) -> None: |
| 153 | prod = world.storage.objects[PRODUCTION] |
| 154 | world.storage.ops.clear() |
| 155 | rollback(world.storage, "staging", **world.request()) # type: ignore[arg-type] |
| 156 | assert world.storage.objects[PRODUCTION] == prod |
| 157 | assert all(k != PRODUCTION for _, k in world.storage.ops) |
| 158 | |
| 159 | |
| 160 | def test_production_rollback_still_needs_approval(world: World) -> None: |
| 161 | activate(world.storage, "staging", "0.2.1rc19", world.sha19, expect=observed(world.storage, "staging"), now=NOW + timedelta(minutes=6)) |
| 162 | activate(world.storage, "production", "0.2.1rc19", world.sha19, expect=observed(world.storage, "production"), now=world.bad_at, approve_production=True) |
| 163 | req = world.request(expect_activated_at=world.bad_at.isoformat()) |
| 164 | with pytest.raises(ReleaseSafetyError, match="approv"): |
| 165 | rollback(world.storage, "production", **req) # type: ignore[arg-type] |
| 166 | assert rollback(world.storage, "production", **req, approve_production=True).version == "0.2.1rc18" # type: ignore[arg-type] |
| 167 | |
| 168 | |
| 169 | def test_there_is_no_blind_swap_entry_point() -> None: |
| 170 | import inspect |
| 171 | |
| 172 | params = inspect.signature(rollback).parameters |
| 173 | for required in ("to_version", "to_sha256", "expect_version", "expect_sha256", "expect_activated_at"): |
| 174 | assert params[required].default is inspect.Parameter.empty |
File History
4 commits
sha256:675003834b19b109f8cb126e543a064a7bd6256930dd4a80ffcfe87c6eff3f17
fix(release): idempotent concurrent publish (D3) and compar…
Opus 5.5
minor
⚠
12 days ago
sha256:04db68e43d9007348dbd726b2a3e52f73e83c5646a4511c07e19cb9a66e28b5f
fix(release): gate provenance is observed from the checkout…
Opus 5.5
minor
⚠
12 days ago
sha256:9f814406166bc157b00dfd4078fb4516ad46e9844628da6bb3823e8448eca1e9
fix(release): rollback is an explicit, idempotent, stale-pr…
Opus 5.5
minor
⚠
12 days ago
sha256:be248fd80e5c5381dba3672e40683905135a665b8582a3c4e0db432c4162a441
test(release): red tests for explicit, idempotent, stale-pr…
Opus 5.5
12 days ago