test_release_concurrency.py
python
sha256:675003834b19b109f8cb126e543a064a7bd6256930dd4a80ffcfe87c6eff3f17
fix(release): idempotent concurrent publish (D3) and compar…
Opus 5.5
minor
⚠ breaking
12 days ago
| 1 | """Concurrent and interrupted publication (musehub#114 Phase 6D, D3). |
| 2 | |
| 3 | Safety comes from storage-level create-only writes, not from read-then-write. These tests drive |
| 4 | the interleavings deterministically (a storage hook runs a competing operation at the exact |
| 5 | moment a read-then-write implementation would be racing; barriers line real threads up on the |
| 6 | write itself). Nothing here touches live storage. |
| 7 | """ |
| 8 | |
| 9 | from __future__ import annotations |
| 10 | |
| 11 | import json |
| 12 | import pathlib |
| 13 | import threading |
| 14 | from collections.abc import Callable |
| 15 | from typing import Any |
| 16 | |
| 17 | import pytest |
| 18 | |
| 19 | from deploy.release_gate import GateError |
| 20 | from deploy.release_safety import ReleaseSafetyError, StorageUnavailable, publish |
| 21 | from tests.release_fixtures import CHECKOUT, NOW, FakeStorage, make_artifact, make_report, sha256_of |
| 22 | |
| 23 | VERSION = "0.2.1rc19" |
| 24 | AKEY, RKEY = f"muse-{VERSION}.tar.gz", f"muse-{VERSION}.release.json" |
| 25 | |
| 26 | |
| 27 | def candidate(tmp: pathlib.Path, name: str, payload: bytes | None = None) -> tuple[pathlib.Path, str]: |
| 28 | d = tmp / name |
| 29 | d.mkdir() |
| 30 | art = make_artifact(d, VERSION, payload=payload) |
| 31 | return art, make_report(art) |
| 32 | |
| 33 | |
| 34 | def do_publish(storage: FakeStorage, cand: tuple[pathlib.Path, str]) -> Any: |
| 35 | return publish(storage, cand[0], cand[1], now=NOW, checkout=CHECKOUT) |
| 36 | |
| 37 | |
| 38 | def run_threads(*fns: Callable[[], Any]) -> list[Any]: |
| 39 | """Run each fn in its own thread; return value or the raised exception, in order.""" |
| 40 | out: list[Any] = [None] * len(fns) |
| 41 | |
| 42 | def wrap(i: int, fn: Callable[[], Any]) -> None: |
| 43 | try: |
| 44 | out[i] = fn() |
| 45 | except BaseException as exc: # noqa: BLE001 - the test inspects it |
| 46 | out[i] = exc |
| 47 | |
| 48 | threads = [threading.Thread(target=wrap, args=(i, fn)) for i, fn in enumerate(fns)] |
| 49 | for t in threads: |
| 50 | t.start() |
| 51 | for t in threads: |
| 52 | t.join(timeout=10) |
| 53 | assert not t.is_alive(), "publisher deadlocked" |
| 54 | return out |
| 55 | |
| 56 | |
| 57 | def barrier_on_artifact_write(storage: FakeStorage, parties: int) -> None: |
| 58 | """Hold every publisher at its artifact create until all have arrived: any check made earlier is now stale.""" |
| 59 | barrier = threading.Barrier(parties, timeout=5) |
| 60 | |
| 61 | def hook(op: str, key: str) -> None: |
| 62 | if op == "put_new" and key == AKEY: |
| 63 | barrier.wait() |
| 64 | |
| 65 | storage.hook = hook |
| 66 | |
| 67 | |
| 68 | def inject_once(storage: FakeStorage, on: tuple[str, str], action: Callable[[], None]) -> None: |
| 69 | """Run *action* the first time (op, key) is about to happen -- i.e. between a publisher's check and its write.""" |
| 70 | fired = [] |
| 71 | |
| 72 | def hook(op: str, key: str) -> None: |
| 73 | if (op, key) == on and not fired: |
| 74 | fired.append(1) |
| 75 | storage.hook = None |
| 76 | action() |
| 77 | storage.hook = hook |
| 78 | |
| 79 | storage.hook = hook |
| 80 | |
| 81 | |
| 82 | class TestIdenticalConcurrentPublishers: |
| 83 | @pytest.mark.parametrize("n", [2, 5]) |
| 84 | def test_identical_publishers_all_succeed_and_converge(self, tmp_path: pathlib.Path, n: int) -> None: |
| 85 | storage = FakeStorage() |
| 86 | cands = [candidate(tmp_path, f"p{i}") for i in range(n)] |
| 87 | barrier_on_artifact_write(storage, n) |
| 88 | results = run_threads(*[lambda c=c: do_publish(storage, c) for c in cands]) |
| 89 | assert all(not isinstance(r, BaseException) for r in results), results |
| 90 | assert {r.sha256 for r in results} == {sha256_of(cands[0][0])} |
| 91 | assert {r.status for r in results} <= {"published", "completed", "already_published"} |
| 92 | assert sorted(storage.objects) == [RKEY, AKEY] or sorted(storage.objects) == sorted([AKEY, RKEY]) |
| 93 | assert storage.objects[AKEY] == cands[0][0].read_bytes() |
| 94 | assert storage.deletes() == [] |
| 95 | assert [k for _, k in storage.ops if k.startswith("pointers/")] == [] |
| 96 | assert [k for op, k in storage.ops if k == AKEY] == [AKEY], "exactly one artifact create may succeed" |
| 97 | |
| 98 | def test_loser_of_the_artifact_race_is_not_told_it_collided(self, tmp_path: pathlib.Path) -> None: |
| 99 | """The race a read-then-write implementation loses: the winner appears between check and create.""" |
| 100 | storage = FakeStorage() |
| 101 | winner, loser = candidate(tmp_path, "w"), candidate(tmp_path, "l") |
| 102 | inject_once(storage, ("put_new", AKEY), lambda: do_publish(storage, winner)) |
| 103 | result = do_publish(storage, loser) |
| 104 | assert result.status == "already_published" |
| 105 | assert storage.objects[AKEY] == winner[0].read_bytes() |
| 106 | assert storage.deletes() == [] |
| 107 | |
| 108 | def test_loser_of_the_record_race_is_not_told_it_collided(self, tmp_path: pathlib.Path) -> None: |
| 109 | storage = FakeStorage() |
| 110 | winner, loser = candidate(tmp_path, "w"), candidate(tmp_path, "l") |
| 111 | inject_once(storage, ("put_new", RKEY), lambda: do_publish(storage, winner)) |
| 112 | result = do_publish(storage, loser) |
| 113 | assert result.status == "completed" |
| 114 | assert json.loads(storage.objects[RKEY])["sha256"] == sha256_of(loser[0]) |
| 115 | assert storage.deletes() == [] |
| 116 | |
| 117 | def test_republishing_is_idempotent_and_writes_nothing(self, tmp_path: pathlib.Path) -> None: |
| 118 | storage = FakeStorage() |
| 119 | c = candidate(tmp_path, "a") |
| 120 | do_publish(storage, c) |
| 121 | before, ops = storage.snapshot(), len(storage.ops) |
| 122 | assert do_publish(storage, c).status == "already_published" |
| 123 | assert storage.snapshot() == before and len(storage.ops) == ops |
| 124 | |
| 125 | |
| 126 | class TestConflictingConcurrentPublishers: |
| 127 | def test_exactly_one_identity_is_accepted_when_two_publish_different_bytes(self, tmp_path: pathlib.Path) -> None: |
| 128 | storage = FakeStorage() |
| 129 | a, b = candidate(tmp_path, "a", b"artifact A"), candidate(tmp_path, "b", b"artifact B") |
| 130 | barrier_on_artifact_write(storage, 2) |
| 131 | ra, rb = run_threads(lambda: do_publish(storage, a), lambda: do_publish(storage, b)) |
| 132 | outcomes = [ra, rb] |
| 133 | winners = [r for r in outcomes if not isinstance(r, BaseException)] |
| 134 | losers = [r for r in outcomes if isinstance(r, BaseException)] |
| 135 | assert len(winners) == 1 and len(losers) == 1 |
| 136 | assert isinstance(losers[0], ReleaseSafetyError) and "immutable" in str(losers[0]) |
| 137 | stored = storage.objects[AKEY] |
| 138 | assert stored in (b"artifact A", b"artifact B") |
| 139 | assert json.loads(storage.objects[RKEY])["sha256"] == winners[0].sha256 == sha256_of(a[0] if stored == b"artifact A" else b[0]) |
| 140 | assert storage.deletes() == [] |
| 141 | |
| 142 | def test_conflicting_publisher_cannot_overwrite_a_winner_that_appears_mid_flight(self, tmp_path: pathlib.Path) -> None: |
| 143 | storage = FakeStorage() |
| 144 | winner, loser = candidate(tmp_path, "w", b"winner"), candidate(tmp_path, "l", b"loser") |
| 145 | inject_once(storage, ("put_new", AKEY), lambda: do_publish(storage, winner)) |
| 146 | with pytest.raises(ReleaseSafetyError, match="different bytes"): |
| 147 | do_publish(storage, loser) |
| 148 | assert storage.objects[AKEY] == b"winner" |
| 149 | assert json.loads(storage.objects[RKEY])["sha256"] == sha256_of(winner[0]) |
| 150 | assert storage.deletes() == [] |
| 151 | |
| 152 | def test_conflicting_record_appearing_between_artifact_and_record_is_refused(self, tmp_path: pathlib.Path) -> None: |
| 153 | """A conflicting publisher's record lands after we created the artifact: we must not report success.""" |
| 154 | storage = FakeStorage() |
| 155 | ours = candidate(tmp_path, "o", b"ours") |
| 156 | foreign = json.dumps({"schema": 1, "version": VERSION, "sha256": "f" * 64}).encode() |
| 157 | inject_once(storage, ("put_new", RKEY), lambda: storage.objects.__setitem__(RKEY, foreign)) |
| 158 | with pytest.raises(ReleaseSafetyError, match="does not describe"): |
| 159 | do_publish(storage, ours) |
| 160 | assert storage.objects[RKEY] == foreign |
| 161 | assert storage.deletes() == [] |
| 162 | |
| 163 | def test_no_pointer_changes_in_any_race(self, tmp_path: pathlib.Path) -> None: |
| 164 | storage = FakeStorage({"pointers/staging.json": b"{}", "pointers/production.json": b"{}"}) |
| 165 | before = {k: v for k, v in storage.objects.items() if k.startswith("pointers/")} |
| 166 | a, b = candidate(tmp_path, "a", b"A"), candidate(tmp_path, "b", b"B") |
| 167 | barrier_on_artifact_write(storage, 2) |
| 168 | run_threads(lambda: do_publish(storage, a), lambda: do_publish(storage, b)) |
| 169 | assert {k: v for k, v in storage.objects.items() if k.startswith("pointers/")} == before |
| 170 | |
| 171 | |
| 172 | class TestFailClosedOnExistingState: |
| 173 | def test_malformed_existing_record_is_refused_before_any_write(self, tmp_path: pathlib.Path) -> None: |
| 174 | for bad in (b"not json", b"[]", json.dumps({"schema": 1, "version": "9.9.9", "sha256": "a" * 64}).encode()): |
| 175 | storage = FakeStorage({RKEY: bad}) |
| 176 | with pytest.raises(ReleaseSafetyError, match="malformed"): |
| 177 | do_publish(storage, candidate(tmp_path, f"m{len(bad)}")) |
| 178 | assert storage.ops == [] and AKEY not in storage.objects |
| 179 | |
| 180 | def test_record_for_other_bytes_without_an_artifact_is_refused_before_any_write(self, tmp_path: pathlib.Path) -> None: |
| 181 | record = json.dumps({"schema": 1, "version": VERSION, "sha256": "e" * 64}).encode() |
| 182 | storage = FakeStorage({RKEY: record}) |
| 183 | with pytest.raises(ReleaseSafetyError, match="immutable"): |
| 184 | do_publish(storage, candidate(tmp_path, "x")) |
| 185 | assert storage.ops == [] and AKEY not in storage.objects |
| 186 | |
| 187 | def test_malformed_record_appearing_mid_flight_is_refused(self, tmp_path: pathlib.Path) -> None: |
| 188 | storage = FakeStorage() |
| 189 | inject_once(storage, ("put_new", RKEY), lambda: storage.objects.__setitem__(RKEY, b"garbage")) |
| 190 | with pytest.raises(ReleaseSafetyError, match="malformed"): |
| 191 | do_publish(storage, candidate(tmp_path, "g")) |
| 192 | assert storage.objects[RKEY] == b"garbage" |
| 193 | |
| 194 | def test_missing_artifact_file_fails_without_writes(self, tmp_path: pathlib.Path) -> None: |
| 195 | storage = FakeStorage() |
| 196 | c = candidate(tmp_path, "gone") |
| 197 | c[0].unlink() |
| 198 | with pytest.raises(GateError, match="artifact"): |
| 199 | do_publish(storage, c) |
| 200 | assert storage.ops == [] |
| 201 | |
| 202 | def test_unavailable_storage_during_the_loser_verification_fails_closed(self, tmp_path: pathlib.Path) -> None: |
| 203 | storage = FakeStorage() |
| 204 | winner, loser = candidate(tmp_path, "w"), candidate(tmp_path, "l") |
| 205 | |
| 206 | def winner_then_outage() -> None: |
| 207 | do_publish(storage, winner) |
| 208 | storage.unavailable = True |
| 209 | |
| 210 | inject_once(storage, ("put_new", AKEY), winner_then_outage) |
| 211 | with pytest.raises(StorageUnavailable): |
| 212 | do_publish(storage, loser) |
| 213 | |
| 214 | |
| 215 | class TestPartialFailuresAreRecoverable: |
| 216 | def test_interrupted_between_artifact_and_record_is_completed_by_a_rerun(self, tmp_path: pathlib.Path) -> None: |
| 217 | storage = FakeStorage() |
| 218 | c = candidate(tmp_path, "i") |
| 219 | storage.fail_put_new_on = RKEY |
| 220 | with pytest.raises(StorageUnavailable): |
| 221 | do_publish(storage, c) |
| 222 | assert AKEY in storage.objects and RKEY not in storage.objects |
| 223 | storage.fail_put_new_on = None |
| 224 | assert do_publish(storage, c).status == "completed" |
| 225 | assert storage.deletes() == [] |
| 226 | |
| 227 | def test_a_crashed_publisher_does_not_let_a_different_artifact_take_the_version(self, tmp_path: pathlib.Path) -> None: |
| 228 | storage = FakeStorage() |
| 229 | crashed = candidate(tmp_path, "c", b"first") |
| 230 | storage.fail_put_new_on = RKEY |
| 231 | with pytest.raises(StorageUnavailable): |
| 232 | do_publish(storage, crashed) |
| 233 | storage.fail_put_new_on = None |
| 234 | with pytest.raises(ReleaseSafetyError, match="different bytes"): |
| 235 | do_publish(storage, candidate(tmp_path, "d", b"second")) |
| 236 | assert storage.objects[AKEY] == b"first" |
| 237 | |
| 238 | def test_record_without_artifact_for_the_same_bytes_is_completed(self, tmp_path: pathlib.Path) -> None: |
| 239 | storage = FakeStorage() |
| 240 | c = candidate(tmp_path, "r") |
| 241 | do_publish(storage, c) |
| 242 | del storage.objects[AKEY] # e.g. a half-restored bucket; the record is still ours |
| 243 | storage.ops.clear() |
| 244 | assert do_publish(storage, c).status == "completed" |
| 245 | assert storage.objects[AKEY] == c[0].read_bytes() |
| 246 | assert [k for _, k in storage.ops] == [AKEY] |
File History
1 commit
sha256:675003834b19b109f8cb126e543a064a7bd6256930dd4a80ffcfe87c6eff3f17
fix(release): idempotent concurrent publish (D3) and compar…
Opus 5.5
minor
⚠
12 days ago