gabriel / musehub public
test_release_activation.py python
383 lines 18.5 KB
Raw
sha256:675003834b19b109f8cb126e543a064a7bd6256930dd4a80ffcfe87c6eff3f17 fix(release): idempotent concurrent publish (D3) and compar… Opus 5.5 minor ⚠ breaking 12 days ago
1 """Activation is a compare-and-swap (musehub#114 Phase 6D, D4).
2
3 An activation names the pointer state the operator observed. The write is storage-conditional, so
4 two racing operators cannot both win; a stale or malformed pointer is refused; an already
5 satisfied request writes nothing. Rollback (D1) shares the same swap and must not regress.
6 """
7
8 from __future__ import annotations
9
10 import json
11 import pathlib
12 import threading
13 from collections.abc import Callable
14 from datetime import timedelta
15 from typing import Any
16
17 import pytest
18
19 from deploy.release_safety import (
20 DirStorage,
21 ExpectedPointer,
22 ReleaseSafetyError,
23 StaleState,
24 StorageUnavailable,
25 activate,
26 publish,
27 rollback,
28 )
29 from tests.release_fixtures import CHECKOUT, NOW, FakeStorage, make_artifact, make_report, observed, sha256_of
30
31 STAGING, PRODUCTION = "pointers/staging.json", "pointers/production.json"
32
33
34 class World:
35 """rc18, rc19, rc20 published; nothing active."""
36
37 def __init__(self, tmp: pathlib.Path) -> None:
38 self.storage = FakeStorage()
39 self.sha: dict[str, str] = {}
40 for v in ("0.2.1rc18", "0.2.1rc19", "0.2.1rc20"):
41 (tmp / v).mkdir()
42 art = make_artifact(tmp / v, v)
43 publish(self.storage, art, make_report(art), now=NOW, checkout=CHECKOUT)
44 self.sha[v] = sha256_of(art)
45 self.storage.ops.clear()
46
47 def act(self, version: str, *, env: str = "staging", expect: ExpectedPointer | None = None, at: int = 0, **kw: Any) -> Any:
48 return activate(
49 self.storage, env, version, self.sha[version],
50 expect=expect if expect is not None else observed(self.storage, env), now=NOW + timedelta(minutes=at), **kw,
51 )
52
53 def pointer(self, env: str = "staging") -> dict[str, Any]:
54 return json.loads(self.storage.objects[f"pointers/{env}.json"])
55
56
57 @pytest.fixture
58 def w(tmp_path: pathlib.Path) -> World:
59 return World(tmp_path)
60
61
62 def run_threads(*fns: Callable[[], Any]) -> list[Any]:
63 out: list[Any] = [None] * len(fns)
64
65 def wrap(i: int, fn: Callable[[], Any]) -> None:
66 try:
67 out[i] = fn()
68 except BaseException as exc: # noqa: BLE001
69 out[i] = exc
70
71 ts = [threading.Thread(target=wrap, args=(i, fn)) for i, fn in enumerate(fns)]
72 for t in ts:
73 t.start()
74 for t in ts:
75 t.join(timeout=10)
76 assert not t.is_alive()
77 return out
78
79
80 def barrier_on_pointer_write(storage: FakeStorage, parties: int) -> None:
81 barrier = threading.Barrier(parties, timeout=5)
82
83 def hook(op: str, key: str) -> None:
84 if op in ("put_new", "put_if_match") and key.startswith("pointers/"):
85 barrier.wait()
86
87 storage.hook = hook
88
89
90 class TestCompareAndSwap:
91 def test_first_activation_requires_the_pointer_to_be_absent_and_is_create_only(self, w: World) -> None:
92 pointer = w.act("0.2.1rc18", expect=ExpectedPointer.absent())
93 assert pointer.version == "0.2.1rc18" and pointer.previous is None
94 assert w.storage.ops == [("put_new", STAGING)]
95
96 def test_matching_expectation_swaps_atomically_and_keeps_the_old_release_as_previous(self, w: World) -> None:
97 w.act("0.2.1rc18", expect=ExpectedPointer.absent())
98 w.storage.ops.clear()
99 pointer = w.act("0.2.1rc19", at=1)
100 assert (pointer.version, pointer.previous.version) == ("0.2.1rc19", "0.2.1rc18") # type: ignore[union-attr]
101 assert w.storage.ops == [("put_if_match", STAGING)]
102
103 def test_expecting_absent_when_a_pointer_exists_is_refused(self, w: World) -> None:
104 w.act("0.2.1rc18", expect=ExpectedPointer.absent())
105 before = w.storage.snapshot()
106 with pytest.raises(ReleaseSafetyError, match="stale activation"):
107 w.act("0.2.1rc19", expect=ExpectedPointer.absent(), at=1)
108 assert w.storage.snapshot() == before
109
110 def test_expecting_a_pointer_when_none_exists_is_refused(self, w: World) -> None:
111 with pytest.raises(ReleaseSafetyError, match="stale activation"):
112 w.act("0.2.1rc19", expect=ExpectedPointer("0.2.1rc18", w.sha["0.2.1rc18"], NOW.isoformat()))
113 assert w.storage.ops == []
114
115 def test_stale_expectation_is_refused_without_writing(self, w: World) -> None:
116 w.act("0.2.1rc18", expect=ExpectedPointer.absent())
117 stale = observed(w.storage, "staging")
118 w.act("0.2.1rc19", at=1) # another operator moved it on
119 w.storage.ops.clear()
120 before = w.storage.snapshot()
121 with pytest.raises(ReleaseSafetyError, match="stale activation"):
122 w.act("0.2.1rc20", expect=stale, at=2)
123 assert w.storage.snapshot() == before and w.storage.ops == []
124
125 def test_same_version_but_different_generation_is_stale(self, w: World) -> None:
126 """Version+sha alone does not identify the observed state: activated_at does."""
127 w.act("0.2.1rc18", expect=ExpectedPointer.absent())
128 seen = observed(w.storage, "staging")
129 w.act("0.2.1rc19", at=1)
130 w.act("0.2.1rc18", at=2) # back on rc18, but a different generation
131 with pytest.raises(ReleaseSafetyError, match="stale"):
132 w.act("0.2.1rc20", expect=seen, at=3)
133
134 @pytest.mark.parametrize("partial", [("0.2.1rc18", None, None), (None, "a" * 64, None), ("0.2.1rc18", "a" * 64, None), (None, None, "2026-01-01T00:00:00+00:00")])
135 def test_incompletely_specified_expectation_is_refused(self, w: World, partial: tuple[Any, Any, Any]) -> None:
136 with pytest.raises(ReleaseSafetyError, match="fully specified"):
137 w.act("0.2.1rc18", expect=ExpectedPointer(*partial))
138 assert w.storage.ops == []
139
140 def test_malformed_expectation_values_are_refused(self, w: World) -> None:
141 with pytest.raises(ReleaseSafetyError):
142 w.act("0.2.1rc18", expect=ExpectedPointer("../../etc", "a" * 64, "t"))
143 with pytest.raises(ReleaseSafetyError):
144 w.act("0.2.1rc18", expect=ExpectedPointer("0.2.1rc18", "nothex", "t"))
145
146
147 class TestIdempotentRetry:
148 def test_duplicate_activation_writes_nothing_and_returns_the_current_pointer(self, w: World) -> None:
149 first = w.act("0.2.1rc18", expect=ExpectedPointer.absent())
150 w.storage.ops.clear()
151 before = w.storage.snapshot()
152 again = w.act("0.2.1rc18", expect=ExpectedPointer.absent(), at=5) # same request, replayed
153 assert again == first and w.storage.snapshot() == before and w.storage.ops == []
154
155 def test_retry_after_the_pointer_already_reached_the_target_is_a_noop_even_with_a_stale_expectation(self, w: World) -> None:
156 w.act("0.2.1rc18", expect=ExpectedPointer.absent())
157 seen = observed(w.storage, "staging")
158 done = w.act("0.2.1rc19", expect=seen, at=1)
159 w.storage.ops.clear()
160 assert w.act("0.2.1rc19", expect=seen, at=2) == done # operator re-ran after a lost response
161 assert w.storage.ops == [] and done.previous.version == "0.2.1rc18" # type: ignore[union-attr]
162
163 def test_retry_keeps_the_rollback_target(self, w: World) -> None:
164 w.act("0.2.1rc18", expect=ExpectedPointer.absent())
165 w.act("0.2.1rc19", at=1)
166 w.act("0.2.1rc19", expect=ExpectedPointer.absent(), at=2)
167 assert w.pointer()["previous"]["version"] == "0.2.1rc18"
168
169
170 class TestTargetAndPointerValidity:
171 def test_unpublished_version_is_refused(self, w: World) -> None:
172 with pytest.raises(ReleaseSafetyError, match="not a published release"):
173 activate(w.storage, "staging", "9.9.9", "a" * 64, expect=ExpectedPointer.absent(), now=NOW)
174 assert w.storage.ops == []
175
176 def test_hash_mismatch_is_refused(self, w: World) -> None:
177 with pytest.raises(ReleaseSafetyError, match="not the requested"):
178 activate(w.storage, "staging", "0.2.1rc19", w.sha["0.2.1rc18"], expect=ExpectedPointer.absent(), now=NOW)
179 assert w.storage.ops == []
180
181 def test_missing_target_artifact_is_refused(self, w: World) -> None:
182 del w.storage.objects["muse-0.2.1rc19.tar.gz"]
183 with pytest.raises(ReleaseSafetyError, match="missing from storage"):
184 w.act("0.2.1rc19", expect=ExpectedPointer.absent())
185 assert w.storage.ops == []
186
187 def test_corrupted_target_artifact_is_refused(self, w: World) -> None:
188 w.storage.objects["muse-0.2.1rc19.tar.gz"] = b"bit rot"
189 with pytest.raises(ReleaseSafetyError, match="does not hash"):
190 w.act("0.2.1rc19", expect=ExpectedPointer.absent())
191 assert w.storage.ops == []
192
193 def test_invalid_release_record_is_refused(self, w: World) -> None:
194 w.storage.objects["muse-0.2.1rc19.release.json"] = b"{"
195 with pytest.raises(ReleaseSafetyError, match="no valid release record"):
196 w.act("0.2.1rc19", expect=ExpectedPointer.absent())
197
198 @pytest.mark.parametrize(
199 "bad",
200 [b"", b"{", b"[]", b'{"schema": 9}', b"\xff\xfe", json.dumps({"schema": 1, "environment": "production", "version": "0.2.1rc18", "sha256": "a" * 64, "previous": None, "activated_at": "x"}).encode()],
201 )
202 def test_malformed_pointer_is_refused_and_left_untouched(self, w: World, bad: bytes) -> None:
203 w.storage.objects[STAGING] = bad
204 with pytest.raises(ReleaseSafetyError):
205 w.act("0.2.1rc19", expect=ExpectedPointer("0.2.1rc18", w.sha["0.2.1rc18"], "x"))
206 assert w.storage.objects[STAGING] == bad and w.storage.ops == []
207
208 def test_production_still_needs_approval_and_must_equal_staging(self, w: World) -> None:
209 w.act("0.2.1rc19", expect=ExpectedPointer.absent())
210 with pytest.raises(ReleaseSafetyError, match="approval"):
211 w.act("0.2.1rc19", env="production", expect=ExpectedPointer.absent())
212 with pytest.raises(ReleaseSafetyError, match="only take what staging"):
213 w.act("0.2.1rc18", env="production", expect=ExpectedPointer.absent(), approve_production=True)
214 assert PRODUCTION not in w.storage.objects
215
216
217 class TestInterruptedWrites:
218 def test_interrupted_first_write_leaves_no_pointer_and_retries_cleanly(self, w: World) -> None:
219 w.storage.fail_put_new_on = STAGING
220 with pytest.raises(StorageUnavailable):
221 w.act("0.2.1rc18", expect=ExpectedPointer.absent())
222 assert STAGING not in w.storage.objects
223 w.storage.fail_put_new_on = None
224 assert w.act("0.2.1rc18", expect=ExpectedPointer.absent()).version == "0.2.1rc18"
225
226 def test_interrupted_swap_leaves_the_old_pointer_and_the_same_request_retries(self, w: World) -> None:
227 w.act("0.2.1rc18", expect=ExpectedPointer.absent())
228 seen = observed(w.storage, "staging")
229 w.storage.fail_put_replace_on = STAGING
230 with pytest.raises(StorageUnavailable):
231 w.act("0.2.1rc19", expect=seen, at=1)
232 assert w.pointer()["version"] == "0.2.1rc18"
233 w.storage.fail_put_replace_on = None
234 assert w.act("0.2.1rc19", expect=seen, at=2).version == "0.2.1rc19"
235
236 def test_unavailable_storage_changes_nothing(self, w: World) -> None:
237 w.storage.unavailable = True
238 with pytest.raises(StorageUnavailable):
239 w.act("0.2.1rc18", expect=ExpectedPointer.absent())
240 w.storage.unavailable = False
241 assert w.storage.ops == [] and STAGING not in w.storage.objects
242
243
244 class TestConcurrentActivations:
245 def test_two_competing_first_activations_exactly_one_wins(self, w: World) -> None:
246 barrier_on_pointer_write(w.storage, 2)
247 a, b = run_threads(
248 lambda: w.act("0.2.1rc18", expect=ExpectedPointer.absent(), at=1),
249 lambda: w.act("0.2.1rc19", expect=ExpectedPointer.absent(), at=2),
250 )
251 outcomes = [a, b]
252 assert sum(isinstance(o, BaseException) for o in outcomes) == 1
253 loser = next(o for o in outcomes if isinstance(o, BaseException))
254 assert isinstance(loser, ReleaseSafetyError) and "changed while" in str(loser)
255 winner = next(o for o in outcomes if not isinstance(o, BaseException))
256 assert w.pointer()["version"] == winner.version
257 assert w.storage.deletes() == [] and [op for op, _ in w.storage.ops] == ["put_new"]
258
259 def test_two_competing_swaps_from_the_same_observation_exactly_one_wins(self, w: World) -> None:
260 w.act("0.2.1rc18", expect=ExpectedPointer.absent())
261 seen = observed(w.storage, "staging")
262 w.storage.ops.clear()
263 barrier_on_pointer_write(w.storage, 2)
264 a, b = run_threads(
265 lambda: w.act("0.2.1rc19", expect=seen, at=1),
266 lambda: w.act("0.2.1rc20", expect=seen, at=2),
267 )
268 errors = [o for o in (a, b) if isinstance(o, BaseException)]
269 wins = [o for o in (a, b) if not isinstance(o, BaseException)]
270 assert len(errors) == 1 and len(wins) == 1 and isinstance(errors[0], ReleaseSafetyError)
271 assert w.pointer()["version"] == wins[0].version
272 assert w.pointer()["previous"]["version"] == "0.2.1rc18", "the abandoned release is still protected"
273 assert w.storage.ops == [("put_if_match", STAGING)]
274
275 def test_two_identical_concurrent_activations_both_succeed_with_one_write(self, w: World) -> None:
276 w.act("0.2.1rc18", expect=ExpectedPointer.absent())
277 seen = observed(w.storage, "staging")
278 w.storage.ops.clear()
279 barrier_on_pointer_write(w.storage, 2)
280 a, b = run_threads(lambda: w.act("0.2.1rc19", expect=seen, at=1), lambda: w.act("0.2.1rc19", expect=seen, at=2))
281 assert not isinstance(a, BaseException) and not isinstance(b, BaseException), (a, b)
282 assert a.version == b.version == "0.2.1rc19" == w.pointer()["version"]
283 assert len(w.storage.ops) == 1
284
285 def test_a_competitor_landing_between_our_read_and_our_write_is_not_overwritten(self, w: World) -> None:
286 w.act("0.2.1rc18", expect=ExpectedPointer.absent())
287 seen = observed(w.storage, "staging")
288 fired: list[int] = []
289
290 def competitor(op: str, key: str) -> None:
291 if op == "put_if_match" and not fired:
292 fired.append(1)
293 w.storage.hook = None
294 w.act("0.2.1rc20", expect=seen, at=9) # wins first
295 w.storage.hook = competitor
296
297 w.storage.hook = competitor
298 with pytest.raises(ReleaseSafetyError, match="changed while"):
299 w.act("0.2.1rc19", expect=seen, at=1)
300 assert w.pointer()["version"] == "0.2.1rc20"
301
302 def test_dir_storage_swap_is_atomic_under_real_threads(self, tmp_path: pathlib.Path) -> None:
303 store = DirStorage(tmp_path)
304 store.put_new("p", b"0")
305 _, token = store.read_versioned("p")
306 results = run_threads(*[lambda i=i: store.put_if_match("p", str(i).encode(), token) for i in range(8)])
307 assert sum(r is None for r in results) == 1
308 assert all(isinstance(r, StaleState) for r in results if r is not None)
309
310
311 class TestCrossEnvironmentIsolation:
312 def test_activating_staging_never_touches_production(self, w: World) -> None:
313 w.act("0.2.1rc19", expect=ExpectedPointer.absent())
314 w.act("0.2.1rc19", env="production", expect=ExpectedPointer.absent(), approve_production=True)
315 prod_before = w.storage.objects[PRODUCTION]
316 w.act("0.2.1rc20", at=1)
317 assert w.storage.objects[PRODUCTION] == prod_before
318 assert [k for _, k in w.storage.ops if k == PRODUCTION] == [PRODUCTION] # only the setup write
319
320 def test_a_staging_observation_cannot_authorize_a_production_swap(self, w: World) -> None:
321 w.act("0.2.1rc19", expect=ExpectedPointer.absent())
322 staging_seen = observed(w.storage, "staging")
323 w.act("0.2.1rc19", env="production", expect=ExpectedPointer.absent(), approve_production=True, at=5)
324 w.act("0.2.1rc20", at=1)
325 with pytest.raises(ReleaseSafetyError): # staging's generation is not production's
326 w.act("0.2.1rc20", env="production", expect=staging_seen, approve_production=True)
327
328 def test_a_pointer_file_claiming_the_other_environment_is_refused(self, w: World) -> None:
329 w.act("0.2.1rc19", expect=ExpectedPointer.absent())
330 w.storage.objects[PRODUCTION] = w.storage.objects[STAGING]
331 with pytest.raises(ReleaseSafetyError, match="claims environment"):
332 w.act("0.2.1rc19", env="production", expect=ExpectedPointer.absent(), approve_production=True)
333
334 def test_unknown_environment_is_refused(self, w: World) -> None:
335 with pytest.raises(ReleaseSafetyError, match="unknown environment"):
336 w.act("0.2.1rc19", env="prod", expect=ExpectedPointer.absent())
337
338
339 class TestActivationThenExplicitRollback:
340 def _rollback(self, w: World, to: str, seen: dict[str, Any], at: int) -> Any:
341 return rollback(
342 w.storage, "staging", to_version=to, to_sha256=w.sha[to],
343 expect_version=seen["version"], expect_sha256=seen["sha256"], expect_activated_at=seen["activated_at"],
344 now=NOW + timedelta(minutes=at),
345 )
346
347 def test_activate_then_rollback_then_repeat_rollback_is_stable(self, w: World) -> None:
348 w.act("0.2.1rc18", expect=ExpectedPointer.absent())
349 w.act("0.2.1rc19", at=1)
350 bad = w.pointer()
351 restored = self._rollback(w, "0.2.1rc18", bad, 2)
352 assert restored.version == "0.2.1rc18" and restored.previous.version == "0.2.1rc19" # type: ignore[union-attr]
353 w.storage.ops.clear()
354 assert self._rollback(w, "0.2.1rc18", bad, 3) == restored and w.storage.ops == []
355
356 def test_rollback_uses_the_conditional_write(self, w: World) -> None:
357 w.act("0.2.1rc18", expect=ExpectedPointer.absent())
358 w.act("0.2.1rc19", at=1)
359 w.storage.ops.clear()
360 self._rollback(w, "0.2.1rc18", w.pointer(), 2)
361 assert w.storage.ops == [("put_if_match", STAGING)]
362
363 def test_replaying_the_old_activation_after_a_rollback_is_refused_not_reapplied(self, w: World) -> None:
364 w.act("0.2.1rc18", expect=ExpectedPointer.absent())
365 old_view = observed(w.storage, "staging")
366 w.act("0.2.1rc19", expect=old_view, at=1)
367 self._rollback(w, "0.2.1rc18", w.pointer(), 2)
368 with pytest.raises(ReleaseSafetyError, match="stale activation"):
369 w.act("0.2.1rc19", expect=old_view, at=3)
370 assert w.pointer()["version"] == "0.2.1rc18"
371
372 def test_competing_rollback_and_activation_exactly_one_wins(self, w: World) -> None:
373 w.act("0.2.1rc18", expect=ExpectedPointer.absent())
374 w.act("0.2.1rc19", at=1)
375 seen = w.pointer()
376 w.storage.ops.clear()
377 barrier_on_pointer_write(w.storage, 2)
378 a, b = run_threads(
379 lambda: self._rollback(w, "0.2.1rc18", seen, 2),
380 lambda: w.act("0.2.1rc20", expect=ExpectedPointer(seen["version"], seen["sha256"], seen["activated_at"]), at=2),
381 )
382 assert sum(isinstance(o, BaseException) for o in (a, b)) == 1
383 assert w.pointer()["version"] in ("0.2.1rc18", "0.2.1rc20") and len(w.storage.ops) == 1
File History 1 commit
sha256:675003834b19b109f8cb126e543a064a7bd6256930dd4a80ffcfe87c6eff3f17 fix(release): idempotent concurrent publish (D3) and compar… Opus 5.5 minor ⚠ 12 days ago