release_safety.py
python
sha256:0ddcb29475cfe996169320691ffb2c5705e510191a4a944487df867959d81b36
fix(release): #275 tag the verified source revision, signed…
Sonnet 5.5
minor
⚠ breaking
7 days ago
| 1 | #!/usr/bin/env python3 |
| 2 | """Safe muse release handling: publish, activate, roll back, clean up (musehub#114 Phase 6). |
| 3 | |
| 4 | Each of these is a *separate* operation with its own authorization; none implies another. |
| 5 | |
| 6 | publish gate-verified artifact -> storage, create-only. Never deletes, never overwrites. |
| 7 | activate move ONE environment's pointer to an already-published (version, sha256), guarded |
| 8 | by the pointer state the operator observed (compare-and-swap): satisfied requests |
| 9 | write nothing, a pointer that moved since is refused. Production additionally needs |
| 10 | explicit approval and must equal what staging runs. |
| 11 | rollback restore a pointer to an explicitly named published release, guarded by the pointer |
| 12 | state the operator observed; idempotent, and refused if the pointer has moved on. |
| 13 | cleanup dry-run by default; prints the exact deletion set. Deleting requires |
| 14 | ``--execute --confirm <digest-of-that-set>``. |
| 15 | |
| 16 | Bucket layout (the bucket is shared by staging and production -- state is separated by key):: |
| 17 | |
| 18 | muse-<version>.tar.gz immutable artifact (one version == one sha256, for ever) |
| 19 | muse-<version>.release.json immutable record: version, sha256, source revision, gate evidence |
| 20 | pointers/staging.json what staging should serve (only ``activate staging`` writes it) |
| 21 | pointers/production.json what production should serve (only approved ``activate production``) |
| 22 | |
| 23 | Cleanup deletes only releases that (a) have a record whose sha256 matches the stored |
| 24 | artifact, (b) are not the active or rollback (``previous``) release of ANY environment, |
| 25 | and (c) fall outside the newest ``--keep`` by PEP 440 order. If either pointer is |
| 26 | missing, malformed or unreadable, or storage is unavailable, nothing is deletable. |
| 27 | Anything that cannot be positively proven safe is preserved. |
| 28 | |
| 29 | Atomicity. Every write that must not race uses a storage-level conditional write, never a |
| 30 | read-then-write: artifacts/records are ``put_new`` (S3 ``If-None-Match: *``), and a pointer |
| 31 | that already exists is replaced with ``put_if_match`` (S3 ``If-Match: <etag read with it>``). |
| 32 | Backends that cannot provide these cannot be used for live releases. |
| 33 | |
| 34 | Run from the musehub repo root:: |
| 35 | |
| 36 | python -m deploy.release_safety --bucket musehub-releases publish --artifact A --report R |
| 37 | """ |
| 38 | from __future__ import annotations |
| 39 | |
| 40 | import argparse |
| 41 | import hashlib |
| 42 | import json |
| 43 | import sys |
| 44 | from dataclasses import dataclass, field |
| 45 | from datetime import datetime, timedelta, timezone |
| 46 | from pathlib import Path |
| 47 | from typing import Any, Protocol |
| 48 | |
| 49 | from deploy.prune_releases import stale_releases |
| 50 | from deploy.release_gate import DEFAULT_MAX_AGE, GateError, verify_report |
| 51 | from deploy.release_provenance import CheckoutState, ProvenanceError, observe_checkout, validate_revision |
| 52 | from musehub.release_pointer import ( |
| 53 | ENVIRONMENTS, |
| 54 | POINTER_SCHEMA, |
| 55 | ReleasePointer, |
| 56 | ReleaseRef, |
| 57 | ReleaseSelectionError, |
| 58 | parse_pointer, |
| 59 | validate_sha256, |
| 60 | validate_version, |
| 61 | ) |
| 62 | |
| 63 | RECORD_SCHEMA = 1 |
| 64 | |
| 65 | |
| 66 | class StorageError(Exception): |
| 67 | pass |
| 68 | |
| 69 | |
| 70 | class StorageUnavailable(StorageError): |
| 71 | """The backend could not be reached or answered unexpectedly. Always fail closed.""" |
| 72 | |
| 73 | |
| 74 | class ObjectExists(StorageError): |
| 75 | """A create-only write found the key already present.""" |
| 76 | |
| 77 | |
| 78 | class StaleState(StorageError): |
| 79 | """A conditional replace found the object changed (or gone) since it was read.""" |
| 80 | |
| 81 | |
| 82 | class ReleaseSafetyError(Exception): |
| 83 | """An operation was refused to protect an existing release or environment.""" |
| 84 | |
| 85 | |
| 86 | class Storage(Protocol): |
| 87 | def exists(self, key: str) -> bool: ... |
| 88 | def read(self, key: str) -> bytes: ... |
| 89 | def list_keys(self, prefix: str = "") -> list[str]: ... |
| 90 | def put_new(self, key: str, data: bytes) -> None: ... # create-only; ObjectExists if present |
| 91 | def read_versioned(self, key: str) -> tuple[bytes, str]: ... # bytes + opaque version token (S3 ETag) |
| 92 | def put_if_match(self, key: str, data: bytes, token: str) -> None: ... # replace iff unchanged; StaleState if not |
| 93 | def delete(self, key: str) -> None: ... # cleanup only |
| 94 | |
| 95 | |
| 96 | def _sha(data: bytes) -> str: |
| 97 | return hashlib.sha256(data).hexdigest() |
| 98 | |
| 99 | |
| 100 | def _artifact_key(version: str) -> str: |
| 101 | return f"muse-{version}.tar.gz" |
| 102 | |
| 103 | |
| 104 | def _record_key(version: str) -> str: |
| 105 | return f"muse-{version}.release.json" |
| 106 | |
| 107 | |
| 108 | def _pointer_key(environment: str) -> str: |
| 109 | return f"pointers/{environment}.json" |
| 110 | |
| 111 | |
| 112 | # ── publish ─────────────────────────────────────────────────────────────────── |
| 113 | |
| 114 | |
| 115 | @dataclass(frozen=True) |
| 116 | class PublishResult: |
| 117 | status: str # "published" | "already_published" | "completed" |
| 118 | version: str |
| 119 | sha256: str |
| 120 | |
| 121 | |
| 122 | def publish( |
| 123 | storage: Storage, |
| 124 | artifact: Path, |
| 125 | report_text: str | None, |
| 126 | *, |
| 127 | now: datetime, |
| 128 | checkout: CheckoutState, |
| 129 | max_age: timedelta = DEFAULT_MAX_AGE, |
| 130 | ) -> PublishResult: |
| 131 | """Publish *artifact* iff the gate report authorizes exactly its bytes. |
| 132 | |
| 133 | *checkout* is the Muse-observed state of the source checkout now; the report must have been |
| 134 | gated at exactly that revision on a clean tree. |
| 135 | |
| 136 | Create-only: the same (version, bytes) is idempotent; the same version with |
| 137 | different bytes is refused. No pointer is touched and nothing is deleted. |
| 138 | """ |
| 139 | candidate = verify_report(report_text, artifact, now=now, checkout=checkout, max_age=max_age) # raises GateError |
| 140 | data = artifact.read_bytes() |
| 141 | if _sha(data) != candidate.sha256: # re-check the bytes we are about to upload |
| 142 | raise GateError("artifact changed between verification and upload") |
| 143 | version, sha = candidate.version, candidate.sha256 |
| 144 | akey, rkey = _artifact_key(version), _record_key(version) |
| 145 | |
| 146 | # Advisory pre-check only (refuses early, before any write, on a record that already |
| 147 | # contradicts us). It is NOT what makes publication safe -- the create-only writes below are. |
| 148 | if storage.exists(rkey): |
| 149 | record = _read_record(storage, version) |
| 150 | if record is None: |
| 151 | raise ReleaseSafetyError(f"{rkey} exists but is malformed; refusing to publish over an unverifiable record") |
| 152 | if record["sha256"] != sha: |
| 153 | raise ReleaseSafetyError( |
| 154 | f"{_artifact_key(version)} is already published with different bytes (recorded sha256 {record['sha256']}, " |
| 155 | f"candidate {sha}); versions are immutable" |
| 156 | ) |
| 157 | |
| 158 | # Create-only, first: whoever loses a race finds the winner's object and compares identity. |
| 159 | artifact_created = _create_or_match(storage, akey, data, sha) |
| 160 | record_created = _create_record(storage, version, sha, _record_bytes(candidate, report_text or "", now)) |
| 161 | if artifact_created and record_created: |
| 162 | return PublishResult("published", version, sha) |
| 163 | if not artifact_created and not record_created: |
| 164 | return PublishResult("already_published", version, sha) |
| 165 | return PublishResult("completed", version, sha) |
| 166 | |
| 167 | |
| 168 | def _create_or_match(storage: Storage, key: str, data: bytes, sha: str) -> bool: |
| 169 | """Create *key* iff absent. True if we created it; False if identical bytes were already there. |
| 170 | |
| 171 | Different bytes under the same key are a hard refusal: a version is one artifact, for ever. |
| 172 | """ |
| 173 | try: |
| 174 | storage.put_new(key, data) |
| 175 | return True |
| 176 | except ObjectExists: |
| 177 | pass |
| 178 | try: |
| 179 | stored = _sha(storage.read(key)) |
| 180 | except KeyError as exc: # deleted between the refusal and the read: not something to guess about |
| 181 | raise StorageUnavailable(f"{key} vanished while verifying an existing object") from exc |
| 182 | if stored != sha: |
| 183 | raise ReleaseSafetyError( |
| 184 | f"{key} already exists with different bytes (stored sha256 {stored}, candidate {sha}); versions are immutable" |
| 185 | ) |
| 186 | return False |
| 187 | |
| 188 | |
| 189 | def _create_record(storage: Storage, version: str, sha: str, record: bytes) -> bool: |
| 190 | """Create the release record iff absent; an existing record must be valid and describe *sha*.""" |
| 191 | try: |
| 192 | storage.put_new(_record_key(version), record) |
| 193 | return True |
| 194 | except ObjectExists: |
| 195 | pass |
| 196 | existing = _read_record(storage, version) |
| 197 | if existing is None or existing["sha256"] != sha: |
| 198 | raise ReleaseSafetyError(f"{_record_key(version)} exists but is malformed or does not describe sha256 {sha}") |
| 199 | return False |
| 200 | |
| 201 | |
| 202 | def _record_bytes(candidate: Any, report_text: str, now: datetime) -> bytes: |
| 203 | return json.dumps( |
| 204 | { |
| 205 | "schema": RECORD_SCHEMA, |
| 206 | "version": candidate.version, |
| 207 | "sha256": candidate.sha256, |
| 208 | "source_revision": candidate.source_revision, |
| 209 | "installer_revision": candidate.installer_revision, |
| 210 | "gate_report_sha256": _sha(report_text.encode()), |
| 211 | "published_at": now.astimezone(timezone.utc).isoformat(), |
| 212 | }, |
| 213 | indent=2, |
| 214 | sort_keys=True, |
| 215 | ).encode() |
| 216 | |
| 217 | |
| 218 | def _read_record(storage: Storage, version: str) -> dict[str, Any] | None: |
| 219 | """The parsed release record, or None if absent/invalid (callers treat None as unverified).""" |
| 220 | key = _record_key(version) |
| 221 | if not storage.exists(key): |
| 222 | return None |
| 223 | try: |
| 224 | record = json.loads(storage.read(key)) |
| 225 | if ( |
| 226 | isinstance(record, dict) |
| 227 | and record.get("schema") == RECORD_SCHEMA |
| 228 | and record.get("version") == version |
| 229 | and validate_sha256(record.get("sha256")) |
| 230 | ): |
| 231 | return record # type: ignore[no-any-return] |
| 232 | except (ValueError, ReleaseSelectionError): |
| 233 | pass |
| 234 | return None |
| 235 | |
| 236 | |
| 237 | def _verified_release(storage: Storage, version: str, sha256: str) -> None: |
| 238 | """Raise unless *version* has a valid record and a stored artifact, both matching *sha256*.""" |
| 239 | record = _read_record(storage, version) |
| 240 | if record is None: |
| 241 | raise ReleaseSafetyError(f"no valid release record for {version}; it is not a published release") |
| 242 | if record["sha256"] != sha256: |
| 243 | raise ReleaseSafetyError(f"release {version} has sha256 {record['sha256']}, not the requested {sha256}") |
| 244 | akey = _artifact_key(version) |
| 245 | if not storage.exists(akey): |
| 246 | raise ReleaseSafetyError(f"artifact {akey} is missing from storage") |
| 247 | if _sha(storage.read(akey)) != sha256: |
| 248 | raise ReleaseSafetyError(f"stored {akey} does not hash to the recorded sha256 {sha256}") |
| 249 | |
| 250 | |
| 251 | def release_source_revision(storage: Storage, version: str, sha256: str) -> str: |
| 252 | """The Muse commit the release record for *version* names as its source. |
| 253 | |
| 254 | Raises ``ReleaseSafetyError`` unless a valid record exists and describes *sha256*, and ``ProvenanceError`` if the |
| 255 | recorded ``source_revision`` is not a ``sha256:<64 hex>`` commit id. Read-only: the one place the tag step learns |
| 256 | which commit a release was built from (musehub#275). |
| 257 | """ |
| 258 | record = _read_record(storage, version) |
| 259 | if record is None: |
| 260 | raise ReleaseSafetyError(f"no valid release record for {version}; it is not a published release") |
| 261 | if record["sha256"] != sha256: |
| 262 | raise ReleaseSafetyError(f"release {version} has sha256 {record['sha256']}, not the requested {sha256}") |
| 263 | return validate_revision(record.get("source_revision")) |
| 264 | |
| 265 | |
| 266 | # ── activate / rollback ─────────────────────────────────────────────────────── |
| 267 | |
| 268 | |
| 269 | def _check_environment(environment: str) -> None: |
| 270 | if environment not in ENVIRONMENTS: |
| 271 | raise ReleaseSafetyError(f"unknown environment {environment!r}; expected one of {ENVIRONMENTS}") |
| 272 | |
| 273 | |
| 274 | def _read_pointer(storage: Storage, environment: str) -> ReleasePointer | None: |
| 275 | return _read_pointer_versioned(storage, environment)[0] |
| 276 | |
| 277 | |
| 278 | def _read_pointer_versioned(storage: Storage, environment: str) -> tuple[ReleasePointer | None, str | None]: |
| 279 | """(pointer, token). Both None when absent. The token is what a conditional replace must match.""" |
| 280 | key = _pointer_key(environment) |
| 281 | try: |
| 282 | raw, token = storage.read_versioned(key) |
| 283 | except KeyError: |
| 284 | return None, None |
| 285 | try: |
| 286 | pointer = parse_pointer(raw.decode()) |
| 287 | except (ReleaseSelectionError, UnicodeDecodeError) as exc: |
| 288 | raise ReleaseSafetyError(f"{key} is malformed: {exc}") from exc |
| 289 | if pointer.environment != environment: |
| 290 | raise ReleaseSafetyError(f"{key} claims environment {pointer.environment!r}") |
| 291 | return pointer, token |
| 292 | |
| 293 | |
| 294 | def _swap_pointer(storage: Storage, new: ReleasePointer, *, expected_token: str | None) -> ReleasePointer: |
| 295 | """Atomically install *new*: create-only if no pointer existed, else replace-iff-unchanged. |
| 296 | |
| 297 | Raises ``ReleaseSafetyError`` if another writer got there first; the caller re-reads to decide |
| 298 | whether that writer already did what was asked. |
| 299 | """ |
| 300 | key = _pointer_key(new.environment) |
| 301 | try: |
| 302 | if expected_token is None: |
| 303 | storage.put_new(key, new.to_json().encode()) |
| 304 | else: |
| 305 | storage.put_if_match(key, new.to_json().encode(), expected_token) |
| 306 | except (ObjectExists, StaleState) as exc: |
| 307 | raise ReleaseSafetyError(f"{key} changed while this operation was in flight; nothing was overwritten") from exc |
| 308 | return new |
| 309 | |
| 310 | |
| 311 | def _require_production_approval(environment: str, approved: bool) -> None: |
| 312 | if environment == "production" and not approved: |
| 313 | raise ReleaseSafetyError("changing production requires explicit approval (--approve-production)") |
| 314 | |
| 315 | |
| 316 | @dataclass(frozen=True) |
| 317 | class ExpectedPointer: |
| 318 | """The pointer state the operator observed when they decided -- the "expected current" of the swap. |
| 319 | |
| 320 | ``activated_at`` has microsecond resolution and every write sets it, so |
| 321 | (version, sha256, activated_at) identifies one exact pointer generation. |
| 322 | """ |
| 323 | |
| 324 | version: str | None = None |
| 325 | sha256: str | None = None |
| 326 | activated_at: str | None = None |
| 327 | |
| 328 | @classmethod |
| 329 | def absent(cls) -> "ExpectedPointer": |
| 330 | return cls() |
| 331 | |
| 332 | @property |
| 333 | def is_absent(self) -> bool: |
| 334 | return self.version is None and self.sha256 is None and self.activated_at is None |
| 335 | |
| 336 | def matches(self, pointer: ReleasePointer | None) -> bool: |
| 337 | if pointer is None: |
| 338 | return self.is_absent |
| 339 | return (pointer.version, pointer.sha256, pointer.activated_at) == (self.version, self.sha256, self.activated_at) |
| 340 | |
| 341 | |
| 342 | def _validated_expectation(expect: ExpectedPointer) -> ExpectedPointer: |
| 343 | if expect.is_absent: |
| 344 | return expect |
| 345 | if expect.version is None or expect.sha256 is None or expect.activated_at is None: |
| 346 | raise ReleaseSafetyError("expected pointer state must be fully specified (version, sha256, activated_at) or explicitly absent") |
| 347 | try: |
| 348 | return ExpectedPointer(validate_version(expect.version), validate_sha256(expect.sha256), expect.activated_at) |
| 349 | except ReleaseSelectionError as exc: |
| 350 | raise ReleaseSafetyError(str(exc)) from exc |
| 351 | |
| 352 | |
| 353 | def _describe(pointer: ReleasePointer | None) -> str: |
| 354 | return "absent" if pointer is None else f"{pointer.version} ({pointer.sha256}) activated {pointer.activated_at}" |
| 355 | |
| 356 | |
| 357 | def _move_pointer( |
| 358 | storage: Storage, |
| 359 | environment: str, |
| 360 | target: ReleaseRef, |
| 361 | expect: ExpectedPointer, |
| 362 | *, |
| 363 | now: datetime, |
| 364 | verb: str, |
| 365 | ) -> ReleasePointer: |
| 366 | """The one compare-and-swap shared by activate and rollback. |
| 367 | |
| 368 | * pointer already at *target* -> returned as is, nothing written (safe retry); |
| 369 | * pointer is exactly the *expect*ed one -> atomically replaced; the abandoned release becomes ``previous``; |
| 370 | * anything else (moved, stale, absent when one was expected, or vice versa) -> refused. |
| 371 | |
| 372 | The final write is a storage-level conditional write against the token read together with the |
| 373 | pointer, so two racing operators cannot both succeed. If we lose that race but the winner |
| 374 | installed exactly our target, that is success (nothing for us to write). |
| 375 | """ |
| 376 | current, token = _read_pointer_versioned(storage, environment) |
| 377 | if current is not None and (current.version, current.sha256) == (target.version, target.sha256): |
| 378 | return current |
| 379 | if not expect.matches(current): |
| 380 | raise ReleaseSafetyError( |
| 381 | f"stale {verb} request: {environment} is now {_describe(current)}, not the observed " |
| 382 | f"{'absent' if expect.is_absent else f'{expect.version} activated {expect.activated_at}'}; " |
| 383 | "the pointer changed since you looked -- re-inspect before deciding" |
| 384 | ) |
| 385 | previous = None if current is None else ReleaseRef(current.version, current.sha256) |
| 386 | new = ReleasePointer(environment, target.version, target.sha256, previous, now.astimezone(timezone.utc).isoformat()) |
| 387 | try: |
| 388 | return _swap_pointer(storage, new, expected_token=token) |
| 389 | except ReleaseSafetyError: |
| 390 | winner, _ = _read_pointer_versioned(storage, environment) |
| 391 | if winner is not None and (winner.version, winner.sha256) == (target.version, target.sha256): |
| 392 | return winner |
| 393 | raise |
| 394 | |
| 395 | |
| 396 | def activate( |
| 397 | storage: Storage, |
| 398 | environment: str, |
| 399 | version: str, |
| 400 | sha256: str, |
| 401 | *, |
| 402 | expect: ExpectedPointer, |
| 403 | now: datetime, |
| 404 | approve_production: bool = False, |
| 405 | ) -> ReleasePointer: |
| 406 | """Point *environment* at an already-published (version, sha256), given the pointer state the operator observed. |
| 407 | |
| 408 | *expect* is required: ``ExpectedPointer.absent()`` for a first activation, otherwise the exact |
| 409 | (version, sha256, activated_at) read from ``pointers/<env>.json``. Writes only that pointer. |
| 410 | """ |
| 411 | _check_environment(environment) |
| 412 | try: |
| 413 | version, sha256 = validate_version(version), validate_sha256(sha256) |
| 414 | except ReleaseSelectionError as exc: |
| 415 | raise ReleaseSafetyError(str(exc)) from exc |
| 416 | expect = _validated_expectation(expect) |
| 417 | _require_production_approval(environment, approve_production) |
| 418 | _verified_release(storage, version, sha256) |
| 419 | if environment == "production": |
| 420 | staging = _read_pointer(storage, "staging") |
| 421 | if staging is None or (staging.version, staging.sha256) != (version, sha256): |
| 422 | raise ReleaseSafetyError( |
| 423 | f"production may only take what staging currently runs; staging is " |
| 424 | f"{'unset' if staging is None else staging.version}, requested {version}" |
| 425 | ) |
| 426 | return _move_pointer(storage, environment, ReleaseRef(version, sha256), expect, now=now, verb="activation") |
| 427 | |
| 428 | |
| 429 | def rollback( |
| 430 | storage: Storage, |
| 431 | environment: str, |
| 432 | *, |
| 433 | to_version: str, |
| 434 | to_sha256: str, |
| 435 | expect_version: str, |
| 436 | expect_sha256: str, |
| 437 | expect_activated_at: str, |
| 438 | now: datetime, |
| 439 | approve_production: bool = False, |
| 440 | ) -> ReleasePointer: |
| 441 | """Restore *environment* to the named published release, given the pointer state the operator observed. |
| 442 | |
| 443 | Never a blind swap: see ``_move_pointer``. A missing or malformed pointer is refused (there is |
| 444 | nothing to roll back from), as is an unverified target. |
| 445 | """ |
| 446 | _check_environment(environment) |
| 447 | try: |
| 448 | to_version, to_sha256 = validate_version(to_version), validate_sha256(to_sha256) |
| 449 | expect_version, expect_sha256 = validate_version(expect_version), validate_sha256(expect_sha256) |
| 450 | except ReleaseSelectionError as exc: |
| 451 | raise ReleaseSafetyError(str(exc)) from exc |
| 452 | _require_production_approval(environment, approve_production) |
| 453 | if _read_pointer(storage, environment) is None: |
| 454 | raise ReleaseSafetyError(f"{environment} has no release pointer; refusing to roll back") |
| 455 | _verified_release(storage, to_version, to_sha256) |
| 456 | return _move_pointer( |
| 457 | storage, environment, ReleaseRef(to_version, to_sha256), |
| 458 | ExpectedPointer(expect_version, expect_sha256, expect_activated_at), now=now, verb="rollback", |
| 459 | ) |
| 460 | |
| 461 | |
| 462 | # ── cleanup ─────────────────────────────────────────────────────────────────── |
| 463 | |
| 464 | |
| 465 | @dataclass(frozen=True) |
| 466 | class CleanupPlan: |
| 467 | deletions: list[str] |
| 468 | preserved: dict[str, str] = field(default_factory=dict) |
| 469 | blocked: str | None = None |
| 470 | |
| 471 | @property |
| 472 | def digest(self) -> str: |
| 473 | return _sha("\n".join(self.deletions).encode()) |
| 474 | |
| 475 | |
| 476 | def _inventory(storage: Storage) -> tuple[dict[str, str] | str, dict[str, str]]: |
| 477 | """(protected version -> reason | blocked-reason, verified version -> ''). |
| 478 | |
| 479 | Returns a ``str`` as first element when ownership is ambiguous, in which case |
| 480 | nothing may be deleted. |
| 481 | """ |
| 482 | protected: dict[str, str] = {} |
| 483 | for env in ENVIRONMENTS: |
| 484 | try: |
| 485 | pointer = _read_pointer(storage, env) |
| 486 | except ReleaseSafetyError as exc: |
| 487 | return str(exc), {} |
| 488 | if pointer is None: |
| 489 | return f"{_pointer_key(env)} is missing: cannot establish what {env} depends on", {} |
| 490 | protected.setdefault(pointer.version, f"active in {env}") |
| 491 | if pointer.previous is not None: |
| 492 | protected.setdefault(pointer.previous.version, f"rollback target of {env}") |
| 493 | verified: dict[str, str] = {} |
| 494 | for key in storage.list_keys("muse-"): |
| 495 | if not key.endswith(".tar.gz"): |
| 496 | continue |
| 497 | raw = key.removeprefix("muse-").removesuffix(".tar.gz") |
| 498 | try: |
| 499 | version = validate_version(raw) |
| 500 | except ReleaseSelectionError: |
| 501 | continue # not a release artifact name: never a candidate |
| 502 | record = _read_record(storage, version) |
| 503 | if record is not None and _sha(storage.read(key)) == record["sha256"]: |
| 504 | verified[version] = record["sha256"] |
| 505 | return protected, verified |
| 506 | |
| 507 | |
| 508 | def plan_cleanup(storage: Storage, *, keep: int) -> CleanupPlan: |
| 509 | """Pure: compute (never perform) the deletion set. ``StorageUnavailable`` propagates.""" |
| 510 | protected, verified = _inventory(storage) |
| 511 | if isinstance(protected, str): |
| 512 | return CleanupPlan(deletions=[], preserved={}, blocked=protected) |
| 513 | preserved: dict[str, str] = {} |
| 514 | for key in storage.list_keys("muse-"): |
| 515 | if key.endswith(".tar.gz"): |
| 516 | version = key.removeprefix("muse-").removesuffix(".tar.gz") |
| 517 | if version not in verified: |
| 518 | preserved[key] = "unverified: no valid record matching the stored bytes" |
| 519 | for version, reason in protected.items(): |
| 520 | preserved[_artifact_key(version)] = reason |
| 521 | candidates = stale_releases([_artifact_key(v) for v in verified], keep) # oldest first, PEP 440 |
| 522 | deletions: list[str] = [] |
| 523 | for name in candidates: |
| 524 | version = name.removeprefix("muse-").removesuffix(".tar.gz") |
| 525 | if version in protected: |
| 526 | continue |
| 527 | deletions += [_record_key(version), name] |
| 528 | for version in verified: |
| 529 | if _artifact_key(version) not in deletions and _artifact_key(version) not in preserved: |
| 530 | preserved[_artifact_key(version)] = f"within the newest {keep}" |
| 531 | return CleanupPlan(deletions=deletions, preserved=preserved, blocked=None) |
| 532 | |
| 533 | |
| 534 | @dataclass(frozen=True) |
| 535 | class CleanupGuidance: |
| 536 | """What the operator may do next. ``confirm_args`` is set only when execution would actually be allowed.""" |
| 537 | |
| 538 | state: str # "executable" | "blocked" | "empty" |
| 539 | message: str |
| 540 | confirm_args: str | None = None |
| 541 | |
| 542 | |
| 543 | def cleanup_guidance(plan: CleanupPlan) -> CleanupGuidance: |
| 544 | """Pure. Never suggests ``--execute`` for a plan that ``execute_cleanup`` would refuse or that has nothing to delete.""" |
| 545 | unverified = sorted(k for k, why in plan.preserved.items() if why.startswith("unverified")) |
| 546 | protected = sorted(k for k, why in plan.preserved.items() if why.startswith(("active in", "rollback target"))) |
| 547 | notes = [] |
| 548 | if protected: |
| 549 | notes.append(f"{len(protected)} release(s) protected as active/rollback targets") |
| 550 | if unverified: |
| 551 | notes.append(f"{len(unverified)} unverified artifact(s) preserved (no valid record matching their bytes): {', '.join(unverified)}") |
| 552 | suffix = ("; " + "; ".join(notes)) if notes else "" |
| 553 | if plan.blocked: |
| 554 | return CleanupGuidance("blocked", f"BLOCKED: nothing can be deleted until this is resolved: {plan.blocked}{suffix}") |
| 555 | if not plan.deletions: |
| 556 | return CleanupGuidance("empty", f"NOTHING TO DELETE: every release is protected, unverified or within the retention window{suffix}") |
| 557 | return CleanupGuidance( |
| 558 | "executable", |
| 559 | f"DRY RUN: nothing deleted. {len(plan.deletions)} object(s) would be deleted{suffix}. To delete exactly this set: --execute --confirm {plan.digest}", |
| 560 | f"--execute --confirm {plan.digest}", |
| 561 | ) |
| 562 | |
| 563 | |
| 564 | def execute_cleanup(storage: Storage, plan: CleanupPlan, *, confirm: str) -> list[str]: |
| 565 | """Delete exactly *plan*, only if *confirm* is its digest and every target is still provably safe.""" |
| 566 | if plan.blocked: |
| 567 | raise ReleaseSafetyError(f"cleanup is blocked: {plan.blocked}") |
| 568 | if confirm != plan.digest: |
| 569 | raise ReleaseSafetyError("confirm value does not match the digest of this deletion set; nothing deleted") |
| 570 | protected, verified = _inventory(storage) |
| 571 | if isinstance(protected, str): |
| 572 | raise ReleaseSafetyError(f"cleanup is blocked: {protected}") |
| 573 | for key in plan.deletions: |
| 574 | version = key.removeprefix("muse-").removesuffix(".tar.gz").removesuffix(".release.json") |
| 575 | if key not in (_artifact_key(version), _record_key(version)) or version not in verified or version in protected: |
| 576 | raise ReleaseSafetyError(f"stale or unauthorized deletion target {key}: not a verified, unprotected release; nothing deleted") |
| 577 | for key in plan.deletions: |
| 578 | storage.delete(key) |
| 579 | return list(plan.deletions) |
| 580 | |
| 581 | |
| 582 | # ── backends ────────────────────────────────────────────────────────────────── |
| 583 | |
| 584 | |
| 585 | class DirStorage: |
| 586 | """Directory-backed storage (rehearsals and tests).""" |
| 587 | |
| 588 | def __init__(self, root: Path) -> None: |
| 589 | self.root = root |
| 590 | |
| 591 | def _p(self, key: str) -> Path: |
| 592 | return self.root / key |
| 593 | |
| 594 | def exists(self, key: str) -> bool: |
| 595 | return self._p(key).is_file() |
| 596 | |
| 597 | def read(self, key: str) -> bytes: |
| 598 | return self._p(key).read_bytes() |
| 599 | |
| 600 | def list_keys(self, prefix: str = "") -> list[str]: |
| 601 | return sorted(str(p.relative_to(self.root)) for p in self.root.rglob("*") if p.is_file() and str(p.relative_to(self.root)).startswith(prefix)) |
| 602 | |
| 603 | def put_new(self, key: str, data: bytes) -> None: |
| 604 | path = self._p(key) |
| 605 | path.parent.mkdir(parents=True, exist_ok=True) |
| 606 | try: |
| 607 | with path.open("xb") as fh: |
| 608 | fh.write(data) |
| 609 | except FileExistsError as exc: |
| 610 | raise ObjectExists(key) from exc |
| 611 | |
| 612 | def read_versioned(self, key: str) -> tuple[bytes, str]: |
| 613 | try: |
| 614 | return self._p(key).read_bytes(), self._token(key) |
| 615 | except FileNotFoundError as exc: |
| 616 | raise KeyError(key) from exc |
| 617 | |
| 618 | def _token(self, key: str) -> str: |
| 619 | """Identity of this exact write: content AND inode/mtime, so rewriting identical bytes (ABA) still invalidates it.""" |
| 620 | path = self._p(key) |
| 621 | st = path.stat() |
| 622 | return f"{_sha(path.read_bytes())}:{st.st_ino}:{st.st_mtime_ns}" |
| 623 | |
| 624 | def put_if_match(self, key: str, data: bytes, token: str) -> None: |
| 625 | """Replace iff the content still hashes to *token*; serialised by an exclusive lock on the root.""" |
| 626 | import fcntl |
| 627 | |
| 628 | path = self._p(key) |
| 629 | self.root.mkdir(parents=True, exist_ok=True) |
| 630 | with (self.root / ".lock").open("a") as lock: |
| 631 | fcntl.flock(lock, fcntl.LOCK_EX) |
| 632 | try: |
| 633 | current = self._token(key) |
| 634 | except FileNotFoundError as exc: |
| 635 | raise StaleState(key) from exc |
| 636 | if current != token: |
| 637 | raise StaleState(key) |
| 638 | tmp = path.with_name(path.name + ".tmp") |
| 639 | tmp.write_bytes(data) |
| 640 | tmp.replace(path) |
| 641 | |
| 642 | def delete(self, key: str) -> None: |
| 643 | self._p(key).unlink() |
| 644 | |
| 645 | |
| 646 | class S3Storage: |
| 647 | """S3 backend. Create-only writes use ``If-None-Match: *`` so S3 itself refuses overwrites.""" |
| 648 | |
| 649 | def __init__(self, bucket: str, client: Any | None = None, region: str = "us-east-1") -> None: |
| 650 | if client is None: |
| 651 | import boto3 |
| 652 | |
| 653 | client = boto3.client("s3", region_name=region) |
| 654 | self.bucket, self.client = bucket, client |
| 655 | |
| 656 | def _call(self, fn: Any, *, on_conflict: type[StorageError] = ObjectExists, **kwargs: Any) -> Any: |
| 657 | from botocore.exceptions import BotoCoreError, ClientError |
| 658 | |
| 659 | try: |
| 660 | return fn(Bucket=self.bucket, **kwargs) |
| 661 | except ClientError as exc: |
| 662 | code = str(exc.response.get("Error", {}).get("Code", "")) |
| 663 | if code in ("PreconditionFailed", "ConditionalRequestConflict", "412", "409"): |
| 664 | raise on_conflict(kwargs.get("Key", "")) from exc |
| 665 | if code in ("404", "NoSuchKey", "NotFound"): |
| 666 | raise KeyError(kwargs.get("Key", "")) from exc |
| 667 | raise StorageUnavailable(f"S3 error {code or exc}") from exc |
| 668 | except BotoCoreError as exc: |
| 669 | raise StorageUnavailable(str(exc)) from exc |
| 670 | |
| 671 | def exists(self, key: str) -> bool: |
| 672 | try: |
| 673 | self._call(self.client.head_object, Key=key) |
| 674 | except KeyError: |
| 675 | return False |
| 676 | return True |
| 677 | |
| 678 | def read(self, key: str) -> bytes: |
| 679 | return bytes(self._call(self.client.get_object, Key=key)["Body"].read()) |
| 680 | |
| 681 | def list_keys(self, prefix: str = "") -> list[str]: |
| 682 | keys: list[str] = [] |
| 683 | token: str | None = None |
| 684 | while True: |
| 685 | kwargs: dict[str, Any] = {"Prefix": prefix} |
| 686 | if token: |
| 687 | kwargs["ContinuationToken"] = token |
| 688 | page = self._call(self.client.list_objects_v2, **kwargs) |
| 689 | keys += [o["Key"] for o in page.get("Contents", [])] |
| 690 | if not page.get("IsTruncated"): |
| 691 | return sorted(keys) |
| 692 | token = page["NextContinuationToken"] |
| 693 | |
| 694 | def put_new(self, key: str, data: bytes) -> None: |
| 695 | self._call(self.client.put_object, Key=key, Body=data, IfNoneMatch="*") |
| 696 | |
| 697 | def read_versioned(self, key: str) -> tuple[bytes, str]: |
| 698 | obj = self._call(self.client.get_object, Key=key) |
| 699 | etag = obj.get("ETag") |
| 700 | if not isinstance(etag, str) or not etag: |
| 701 | raise StorageUnavailable(f"S3 returned no ETag for {key}; cannot make a conditional write") |
| 702 | return bytes(obj["Body"].read()), etag |
| 703 | |
| 704 | def put_if_match(self, key: str, data: bytes, token: str) -> None: |
| 705 | """Conditional replace (``If-Match``). A key that vanished (404) is also stale, never recreated.""" |
| 706 | try: |
| 707 | self._call(self.client.put_object, on_conflict=StaleState, Key=key, Body=data, IfMatch=token) |
| 708 | except KeyError as exc: |
| 709 | raise StaleState(key) from exc |
| 710 | |
| 711 | def delete(self, key: str) -> None: |
| 712 | self._call(self.client.delete_object, Key=key) |
| 713 | |
| 714 | |
| 715 | # ── CLI ─────────────────────────────────────────────────────────────────────── |
| 716 | |
| 717 | |
| 718 | def main(argv: list[str] | None = None) -> int: |
| 719 | parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) |
| 720 | where = parser.add_mutually_exclusive_group(required=True) |
| 721 | where.add_argument("--bucket", help="S3 release bucket") |
| 722 | where.add_argument("--dir", type=Path, help="local directory acting as the bucket (rehearsal)") |
| 723 | sub = parser.add_subparsers(dest="command", required=True) |
| 724 | |
| 725 | pub = sub.add_parser("publish") |
| 726 | pub.add_argument("--artifact", type=Path, required=True) |
| 727 | pub.add_argument("--report", type=Path, required=True) |
| 728 | pub.add_argument("--muse-repo", type=Path, required=True, help="source checkout the gates ran against; observed again now") |
| 729 | act = sub.add_parser("activate") |
| 730 | act.add_argument("--env", required=True) |
| 731 | act.add_argument("--version", required=True) |
| 732 | act.add_argument("--sha256", required=True) |
| 733 | act.add_argument("--approve-production", action="store_true") |
| 734 | seen = act.add_mutually_exclusive_group(required=True) |
| 735 | seen.add_argument("--expect-absent", action="store_true", help="the pointer must not exist yet (first activation)") |
| 736 | seen.add_argument("--expect-version", help="version the pointer showed when you decided (with --expect-sha256, --expect-activated-at)") |
| 737 | act.add_argument("--expect-sha256") |
| 738 | act.add_argument("--expect-activated-at") |
| 739 | rb = sub.add_parser("rollback") |
| 740 | rb.add_argument("--env", required=True) |
| 741 | rb.add_argument("--to-version", required=True) |
| 742 | rb.add_argument("--to-sha256", required=True) |
| 743 | rb.add_argument("--expect-version", required=True, help="version the pointer showed when you decided") |
| 744 | rb.add_argument("--expect-sha256", required=True) |
| 745 | rb.add_argument("--expect-activated-at", required=True) |
| 746 | rb.add_argument("--approve-production", action="store_true") |
| 747 | cl = sub.add_parser("cleanup", help="dry-run unless --execute AND --confirm <digest>") |
| 748 | cl.add_argument("--keep", type=int, required=True) |
| 749 | cl.add_argument("--execute", action="store_true") |
| 750 | cl.add_argument("--confirm", default="") |
| 751 | args = parser.parse_args(argv) |
| 752 | |
| 753 | storage: Storage = S3Storage(args.bucket) if args.bucket else DirStorage(args.dir) |
| 754 | now = datetime.now(timezone.utc) |
| 755 | try: |
| 756 | if args.command == "publish": |
| 757 | report = args.report.read_text() if args.report.is_file() else None |
| 758 | result = publish(storage, args.artifact, report, now=now, checkout=observe_checkout(args.muse_repo)) |
| 759 | print(json.dumps({"status": result.status, "version": result.version, "sha256": result.sha256})) |
| 760 | elif args.command == "activate": |
| 761 | if args.expect_absent: |
| 762 | expect = ExpectedPointer.absent() |
| 763 | elif args.expect_sha256 and args.expect_activated_at: |
| 764 | expect = ExpectedPointer(args.expect_version, args.expect_sha256, args.expect_activated_at) |
| 765 | else: |
| 766 | raise ReleaseSafetyError("--expect-version needs --expect-sha256 and --expect-activated-at") |
| 767 | print(activate(storage, args.env, args.version, args.sha256, expect=expect, now=now, approve_production=args.approve_production).to_json()) |
| 768 | elif args.command == "rollback": |
| 769 | print(rollback( |
| 770 | storage, args.env, to_version=args.to_version, to_sha256=args.to_sha256, |
| 771 | expect_version=args.expect_version, expect_sha256=args.expect_sha256, |
| 772 | expect_activated_at=args.expect_activated_at, now=now, approve_production=args.approve_production, |
| 773 | ).to_json()) |
| 774 | else: |
| 775 | plan = plan_cleanup(storage, keep=args.keep) |
| 776 | print(json.dumps({"blocked": plan.blocked, "deletions": plan.deletions, "preserved": plan.preserved, "digest": plan.digest}, indent=2)) |
| 777 | guidance = cleanup_guidance(plan) |
| 778 | if guidance.state == "blocked": |
| 779 | print(guidance.message, file=sys.stderr) |
| 780 | return 1 # a blocked cleanup is a refusal, not a dry run to be confirmed |
| 781 | if guidance.state == "empty": |
| 782 | print(guidance.message, file=sys.stderr) |
| 783 | elif not args.execute: |
| 784 | print(guidance.message, file=sys.stderr) |
| 785 | else: |
| 786 | execute_cleanup(storage, plan, confirm=args.confirm) |
| 787 | print(f"deleted {len(plan.deletions)} objects", file=sys.stderr) |
| 788 | except (GateError, ProvenanceError, ReleaseSafetyError, StorageError) as exc: |
| 789 | print(f"REFUSED: {exc}", file=sys.stderr) |
| 790 | return 1 |
| 791 | return 0 |
| 792 | |
| 793 | |
| 794 | if __name__ == "__main__": |
| 795 | raise SystemExit(main()) |
File History
6 commits
sha256:0ddcb29475cfe996169320691ffb2c5705e510191a4a944487df867959d81b36
fix(release): #275 tag the verified source revision, signed…
Sonnet 5.5
minor
⚠
7 days ago
sha256:22ea76418f13ef089b10088cb0c77fe3cd3f3da50b325cb412073c7f2f46f7f5
fix(release): cleanup shows an execution command only when …
Opus 5.5
patch
12 days ago
sha256:675003834b19b109f8cb126e543a064a7bd6256930dd4a80ffcfe87c6eff3f17
fix(release): idempotent concurrent publish (D3) and compar…
Opus 5.5
minor
⚠
12 days ago
sha256:04db68e43d9007348dbd726b2a3e52f73e83c5646a4511c07e19cb9a66e28b5f
fix(release): gate provenance is observed from the checkout…
Opus 5.5
minor
⚠
12 days ago
sha256:9f814406166bc157b00dfd4078fb4516ad46e9844628da6bb3823e8448eca1e9
fix(release): rollback is an explicit, idempotent, stale-pr…
Opus 5.5
minor
⚠
12 days ago
sha256:6bdfa0ed7d0cfe13d40be53b23ab86de47653bc9a0f796b820fd3486b9abee8c
feat(release): create-only publish, separate activate/rollb…
Opus 5.5
patch
12 days ago