gabriel / musehub public
test_release_concurrency.py python
246 lines 11.8 KB
Raw
sha256:675003834b19b109f8cb126e543a064a7bd6256930dd4a80ffcfe87c6eff3f17 fix(release): idempotent concurrent publish (D3) and compar… Opus 5.5 minor ⚠ breaking 12 days ago
1 """Concurrent and interrupted publication (musehub#114 Phase 6D, D3).
2
3 Safety comes from storage-level create-only writes, not from read-then-write. These tests drive
4 the interleavings deterministically (a storage hook runs a competing operation at the exact
5 moment a read-then-write implementation would be racing; barriers line real threads up on the
6 write itself). Nothing here touches live storage.
7 """
8
9 from __future__ import annotations
10
11 import json
12 import pathlib
13 import threading
14 from collections.abc import Callable
15 from typing import Any
16
17 import pytest
18
19 from deploy.release_gate import GateError
20 from deploy.release_safety import ReleaseSafetyError, StorageUnavailable, publish
21 from tests.release_fixtures import CHECKOUT, NOW, FakeStorage, make_artifact, make_report, sha256_of
22
23 VERSION = "0.2.1rc19"
24 AKEY, RKEY = f"muse-{VERSION}.tar.gz", f"muse-{VERSION}.release.json"
25
26
27 def candidate(tmp: pathlib.Path, name: str, payload: bytes | None = None) -> tuple[pathlib.Path, str]:
28 d = tmp / name
29 d.mkdir()
30 art = make_artifact(d, VERSION, payload=payload)
31 return art, make_report(art)
32
33
34 def do_publish(storage: FakeStorage, cand: tuple[pathlib.Path, str]) -> Any:
35 return publish(storage, cand[0], cand[1], now=NOW, checkout=CHECKOUT)
36
37
38 def run_threads(*fns: Callable[[], Any]) -> list[Any]:
39 """Run each fn in its own thread; return value or the raised exception, in order."""
40 out: list[Any] = [None] * len(fns)
41
42 def wrap(i: int, fn: Callable[[], Any]) -> None:
43 try:
44 out[i] = fn()
45 except BaseException as exc: # noqa: BLE001 - the test inspects it
46 out[i] = exc
47
48 threads = [threading.Thread(target=wrap, args=(i, fn)) for i, fn in enumerate(fns)]
49 for t in threads:
50 t.start()
51 for t in threads:
52 t.join(timeout=10)
53 assert not t.is_alive(), "publisher deadlocked"
54 return out
55
56
57 def barrier_on_artifact_write(storage: FakeStorage, parties: int) -> None:
58 """Hold every publisher at its artifact create until all have arrived: any check made earlier is now stale."""
59 barrier = threading.Barrier(parties, timeout=5)
60
61 def hook(op: str, key: str) -> None:
62 if op == "put_new" and key == AKEY:
63 barrier.wait()
64
65 storage.hook = hook
66
67
68 def inject_once(storage: FakeStorage, on: tuple[str, str], action: Callable[[], None]) -> None:
69 """Run *action* the first time (op, key) is about to happen -- i.e. between a publisher's check and its write."""
70 fired = []
71
72 def hook(op: str, key: str) -> None:
73 if (op, key) == on and not fired:
74 fired.append(1)
75 storage.hook = None
76 action()
77 storage.hook = hook
78
79 storage.hook = hook
80
81
82 class TestIdenticalConcurrentPublishers:
83 @pytest.mark.parametrize("n", [2, 5])
84 def test_identical_publishers_all_succeed_and_converge(self, tmp_path: pathlib.Path, n: int) -> None:
85 storage = FakeStorage()
86 cands = [candidate(tmp_path, f"p{i}") for i in range(n)]
87 barrier_on_artifact_write(storage, n)
88 results = run_threads(*[lambda c=c: do_publish(storage, c) for c in cands])
89 assert all(not isinstance(r, BaseException) for r in results), results
90 assert {r.sha256 for r in results} == {sha256_of(cands[0][0])}
91 assert {r.status for r in results} <= {"published", "completed", "already_published"}
92 assert sorted(storage.objects) == [RKEY, AKEY] or sorted(storage.objects) == sorted([AKEY, RKEY])
93 assert storage.objects[AKEY] == cands[0][0].read_bytes()
94 assert storage.deletes() == []
95 assert [k for _, k in storage.ops if k.startswith("pointers/")] == []
96 assert [k for op, k in storage.ops if k == AKEY] == [AKEY], "exactly one artifact create may succeed"
97
98 def test_loser_of_the_artifact_race_is_not_told_it_collided(self, tmp_path: pathlib.Path) -> None:
99 """The race a read-then-write implementation loses: the winner appears between check and create."""
100 storage = FakeStorage()
101 winner, loser = candidate(tmp_path, "w"), candidate(tmp_path, "l")
102 inject_once(storage, ("put_new", AKEY), lambda: do_publish(storage, winner))
103 result = do_publish(storage, loser)
104 assert result.status == "already_published"
105 assert storage.objects[AKEY] == winner[0].read_bytes()
106 assert storage.deletes() == []
107
108 def test_loser_of_the_record_race_is_not_told_it_collided(self, tmp_path: pathlib.Path) -> None:
109 storage = FakeStorage()
110 winner, loser = candidate(tmp_path, "w"), candidate(tmp_path, "l")
111 inject_once(storage, ("put_new", RKEY), lambda: do_publish(storage, winner))
112 result = do_publish(storage, loser)
113 assert result.status == "completed"
114 assert json.loads(storage.objects[RKEY])["sha256"] == sha256_of(loser[0])
115 assert storage.deletes() == []
116
117 def test_republishing_is_idempotent_and_writes_nothing(self, tmp_path: pathlib.Path) -> None:
118 storage = FakeStorage()
119 c = candidate(tmp_path, "a")
120 do_publish(storage, c)
121 before, ops = storage.snapshot(), len(storage.ops)
122 assert do_publish(storage, c).status == "already_published"
123 assert storage.snapshot() == before and len(storage.ops) == ops
124
125
126 class TestConflictingConcurrentPublishers:
127 def test_exactly_one_identity_is_accepted_when_two_publish_different_bytes(self, tmp_path: pathlib.Path) -> None:
128 storage = FakeStorage()
129 a, b = candidate(tmp_path, "a", b"artifact A"), candidate(tmp_path, "b", b"artifact B")
130 barrier_on_artifact_write(storage, 2)
131 ra, rb = run_threads(lambda: do_publish(storage, a), lambda: do_publish(storage, b))
132 outcomes = [ra, rb]
133 winners = [r for r in outcomes if not isinstance(r, BaseException)]
134 losers = [r for r in outcomes if isinstance(r, BaseException)]
135 assert len(winners) == 1 and len(losers) == 1
136 assert isinstance(losers[0], ReleaseSafetyError) and "immutable" in str(losers[0])
137 stored = storage.objects[AKEY]
138 assert stored in (b"artifact A", b"artifact B")
139 assert json.loads(storage.objects[RKEY])["sha256"] == winners[0].sha256 == sha256_of(a[0] if stored == b"artifact A" else b[0])
140 assert storage.deletes() == []
141
142 def test_conflicting_publisher_cannot_overwrite_a_winner_that_appears_mid_flight(self, tmp_path: pathlib.Path) -> None:
143 storage = FakeStorage()
144 winner, loser = candidate(tmp_path, "w", b"winner"), candidate(tmp_path, "l", b"loser")
145 inject_once(storage, ("put_new", AKEY), lambda: do_publish(storage, winner))
146 with pytest.raises(ReleaseSafetyError, match="different bytes"):
147 do_publish(storage, loser)
148 assert storage.objects[AKEY] == b"winner"
149 assert json.loads(storage.objects[RKEY])["sha256"] == sha256_of(winner[0])
150 assert storage.deletes() == []
151
152 def test_conflicting_record_appearing_between_artifact_and_record_is_refused(self, tmp_path: pathlib.Path) -> None:
153 """A conflicting publisher's record lands after we created the artifact: we must not report success."""
154 storage = FakeStorage()
155 ours = candidate(tmp_path, "o", b"ours")
156 foreign = json.dumps({"schema": 1, "version": VERSION, "sha256": "f" * 64}).encode()
157 inject_once(storage, ("put_new", RKEY), lambda: storage.objects.__setitem__(RKEY, foreign))
158 with pytest.raises(ReleaseSafetyError, match="does not describe"):
159 do_publish(storage, ours)
160 assert storage.objects[RKEY] == foreign
161 assert storage.deletes() == []
162
163 def test_no_pointer_changes_in_any_race(self, tmp_path: pathlib.Path) -> None:
164 storage = FakeStorage({"pointers/staging.json": b"{}", "pointers/production.json": b"{}"})
165 before = {k: v for k, v in storage.objects.items() if k.startswith("pointers/")}
166 a, b = candidate(tmp_path, "a", b"A"), candidate(tmp_path, "b", b"B")
167 barrier_on_artifact_write(storage, 2)
168 run_threads(lambda: do_publish(storage, a), lambda: do_publish(storage, b))
169 assert {k: v for k, v in storage.objects.items() if k.startswith("pointers/")} == before
170
171
172 class TestFailClosedOnExistingState:
173 def test_malformed_existing_record_is_refused_before_any_write(self, tmp_path: pathlib.Path) -> None:
174 for bad in (b"not json", b"[]", json.dumps({"schema": 1, "version": "9.9.9", "sha256": "a" * 64}).encode()):
175 storage = FakeStorage({RKEY: bad})
176 with pytest.raises(ReleaseSafetyError, match="malformed"):
177 do_publish(storage, candidate(tmp_path, f"m{len(bad)}"))
178 assert storage.ops == [] and AKEY not in storage.objects
179
180 def test_record_for_other_bytes_without_an_artifact_is_refused_before_any_write(self, tmp_path: pathlib.Path) -> None:
181 record = json.dumps({"schema": 1, "version": VERSION, "sha256": "e" * 64}).encode()
182 storage = FakeStorage({RKEY: record})
183 with pytest.raises(ReleaseSafetyError, match="immutable"):
184 do_publish(storage, candidate(tmp_path, "x"))
185 assert storage.ops == [] and AKEY not in storage.objects
186
187 def test_malformed_record_appearing_mid_flight_is_refused(self, tmp_path: pathlib.Path) -> None:
188 storage = FakeStorage()
189 inject_once(storage, ("put_new", RKEY), lambda: storage.objects.__setitem__(RKEY, b"garbage"))
190 with pytest.raises(ReleaseSafetyError, match="malformed"):
191 do_publish(storage, candidate(tmp_path, "g"))
192 assert storage.objects[RKEY] == b"garbage"
193
194 def test_missing_artifact_file_fails_without_writes(self, tmp_path: pathlib.Path) -> None:
195 storage = FakeStorage()
196 c = candidate(tmp_path, "gone")
197 c[0].unlink()
198 with pytest.raises(GateError, match="artifact"):
199 do_publish(storage, c)
200 assert storage.ops == []
201
202 def test_unavailable_storage_during_the_loser_verification_fails_closed(self, tmp_path: pathlib.Path) -> None:
203 storage = FakeStorage()
204 winner, loser = candidate(tmp_path, "w"), candidate(tmp_path, "l")
205
206 def winner_then_outage() -> None:
207 do_publish(storage, winner)
208 storage.unavailable = True
209
210 inject_once(storage, ("put_new", AKEY), winner_then_outage)
211 with pytest.raises(StorageUnavailable):
212 do_publish(storage, loser)
213
214
215 class TestPartialFailuresAreRecoverable:
216 def test_interrupted_between_artifact_and_record_is_completed_by_a_rerun(self, tmp_path: pathlib.Path) -> None:
217 storage = FakeStorage()
218 c = candidate(tmp_path, "i")
219 storage.fail_put_new_on = RKEY
220 with pytest.raises(StorageUnavailable):
221 do_publish(storage, c)
222 assert AKEY in storage.objects and RKEY not in storage.objects
223 storage.fail_put_new_on = None
224 assert do_publish(storage, c).status == "completed"
225 assert storage.deletes() == []
226
227 def test_a_crashed_publisher_does_not_let_a_different_artifact_take_the_version(self, tmp_path: pathlib.Path) -> None:
228 storage = FakeStorage()
229 crashed = candidate(tmp_path, "c", b"first")
230 storage.fail_put_new_on = RKEY
231 with pytest.raises(StorageUnavailable):
232 do_publish(storage, crashed)
233 storage.fail_put_new_on = None
234 with pytest.raises(ReleaseSafetyError, match="different bytes"):
235 do_publish(storage, candidate(tmp_path, "d", b"second"))
236 assert storage.objects[AKEY] == b"first"
237
238 def test_record_without_artifact_for_the_same_bytes_is_completed(self, tmp_path: pathlib.Path) -> None:
239 storage = FakeStorage()
240 c = candidate(tmp_path, "r")
241 do_publish(storage, c)
242 del storage.objects[AKEY] # e.g. a half-restored bucket; the record is still ours
243 storage.ops.clear()
244 assert do_publish(storage, c).status == "completed"
245 assert storage.objects[AKEY] == c[0].read_bytes()
246 assert [k for _, k in storage.ops] == [AKEY]
File History 1 commit
sha256:675003834b19b109f8cb126e543a064a7bd6256930dd4a80ffcfe87c6eff3f17 fix(release): idempotent concurrent publish (D3) and compar… Opus 5.5 minor ⚠ 12 days ago