test_release_scripts.py
python
sha256:369f691c7aaec3ca84a864ad1d099d5409045ddc12d185a922b3c84b9e2a1bad
merge: 100% passing, parallelized MuseHub test suite (task/…
Sonnet 5.5
patch
3 days ago
| 1 | """publish/activate shell scripts: gate-before-mutation and no-deletion properties (musehub#114 Phase 6). |
| 2 | |
| 3 | The scripts run for real, but with stub ``aws`` / ``python3`` / ``curl`` binaries first on PATH that |
| 4 | only *log* their invocations. Nothing reaches AWS, ECR, ECS or any instance. |
| 5 | """ |
| 6 | |
| 7 | from __future__ import annotations |
| 8 | |
| 9 | import os |
| 10 | import pathlib |
| 11 | import re |
| 12 | import shutil |
| 13 | import subprocess |
| 14 | import sys |
| 15 | |
| 16 | import pytest |
| 17 | |
| 18 | from tests.release_fixtures import SOURCE_REVISION, clean_status_json, make_artifact, make_report |
| 19 | |
| 20 | ROOT = pathlib.Path(__file__).resolve().parents[1] |
| 21 | PUBLISH = ROOT / "deploy" / "publish_muse_release.sh" |
| 22 | ACTIVATE = ROOT / "deploy" / "activate_muse_release.sh" |
| 23 | STATUS_JSON = clean_status_json(SOURCE_REVISION) |
| 24 | BARE_RM = re.compile(r"(?<![-\w])rm(?![-\w])") |
| 25 | |
| 26 | |
| 27 | def code(path: pathlib.Path) -> str: |
| 28 | return "\n".join(line for line in path.read_text().splitlines() if not line.strip().startswith("#")) |
| 29 | |
| 30 | |
| 31 | @pytest.fixture |
| 32 | def stubs(tmp_path: pathlib.Path) -> tuple[dict[str, str], pathlib.Path]: |
| 33 | """PATH with logging stubs. ``python3 -m deploy.release_safety`` and ``deploy.release_tag`` are stubbed (they would hit S3); |
| 34 | ``python3 -m deploy.release_gate`` and plain python run for real.""" |
| 35 | bin_dir = tmp_path / "bin" |
| 36 | bin_dir.mkdir() |
| 37 | log = tmp_path / "calls.log" |
| 38 | log.write_text("") |
| 39 | for name in ("aws", "curl"): |
| 40 | (bin_dir / name).write_text(f'#!/bin/sh\necho "{name} $*" >> "{log}"\nexit 0\n') |
| 41 | # muse reports a clean checkout at SOURCE_REVISION for the provenance check (rev-parse / status) |
| 42 | (bin_dir / "muse").write_text( |
| 43 | f'#!/bin/sh\necho "muse $*" >> "{log}"\n' |
| 44 | f'case "$*" in\n' |
| 45 | f' *rev-parse*) echo \'{{"commit_id": "{SOURCE_REVISION}"}}\' ;;\n' |
| 46 | f' *status*) echo \'{STATUS_JSON}\' ;;\nesac\nexit 0\n' |
| 47 | ) |
| 48 | (bin_dir / "python3").write_text( |
| 49 | f'#!/bin/sh\ncase "$*" in\n *deploy.release_safety*) echo "python3 $*" >> "{log}"; exit "${{SAFETY_EXIT:-1}}" ;;\n' |
| 50 | f' *deploy.release_tag*) echo "python3 $*" >> "{log}"; case "$*" in *" apply "*) exit "${{TAG_APPLY_EXIT:-0}}" ;; *) exit "${{TAG_CHECK_EXIT:-0}}" ;; esac ;;\n' |
| 51 | f' *) exec {sys.executable} "$@" ;;\nesac\n' |
| 52 | ) |
| 53 | if shutil.which("shasum") is None: |
| 54 | # The scripts use macOS's ``shasum -a 256``; Linux images ship ``sha256sum`` (same output format). |
| 55 | (bin_dir / "shasum").write_text('#!/bin/sh\nif [ "$1" = "-a" ]; then shift 2; fi\nexec sha256sum "$@"\n') |
| 56 | for f in bin_dir.iterdir(): |
| 57 | f.chmod(0o755) |
| 58 | env = {**os.environ, "PATH": f"{bin_dir}:{os.environ['PATH']}", "PYTHONPATH": str(ROOT)} |
| 59 | return env, log |
| 60 | |
| 61 | |
| 62 | def run(script: pathlib.Path, args: list[str], env: dict[str, str], **extra: str) -> subprocess.CompletedProcess[str]: |
| 63 | return subprocess.run(["bash", str(script), *args], env={**env, **extra}, capture_output=True, text=True, cwd=ROOT, timeout=60) |
| 64 | |
| 65 | |
| 66 | class TestPublishScript: |
| 67 | @pytest.mark.parametrize("args", [[], ["prod"], ["Staging"], [""]]) |
| 68 | def test_environment_is_required_and_exact(self, stubs, args: list[str]) -> None: # type: ignore[no-untyped-def] |
| 69 | env, log = stubs |
| 70 | r = run(PUBLISH, args, env) |
| 71 | assert r.returncode != 0 and "Usage" in r.stderr |
| 72 | assert log.read_text() == "" |
| 73 | |
| 74 | def test_missing_report_stops_before_any_aws_call(self, stubs, tmp_path: pathlib.Path) -> None: # type: ignore[no-untyped-def] |
| 75 | env, log = stubs |
| 76 | art = make_artifact(tmp_path, "0.2.1rc19") |
| 77 | r = run(PUBLISH, ["staging"], env, MUSE_ARTIFACT=str(art), MUSE_GATE_REPORT=str(tmp_path / "nope.json")) |
| 78 | assert r.returncode != 0 and "GATE FAILED" in r.stderr |
| 79 | calls = log.read_text() |
| 80 | assert "aws " not in calls and "release_safety" not in calls # read-only muse observation is allowed |
| 81 | |
| 82 | def test_report_for_other_bytes_stops_before_any_aws_call(self, stubs, tmp_path: pathlib.Path) -> None: # type: ignore[no-untyped-def] |
| 83 | env, log = stubs |
| 84 | art = make_artifact(tmp_path, "0.2.1rc19") |
| 85 | report = tmp_path / "r.json" |
| 86 | report.write_text(make_report(art)) |
| 87 | art.write_bytes(b"rebuilt after validation") |
| 88 | r = run(PUBLISH, ["staging"], env, MUSE_ARTIFACT=str(art), MUSE_GATE_REPORT=str(report)) |
| 89 | assert r.returncode != 0 and "GATE FAILED" in r.stderr and "nothing was published" in r.stderr |
| 90 | calls = log.read_text() |
| 91 | assert "aws " not in calls and "release_safety" not in calls # read-only muse observation is allowed |
| 92 | |
| 93 | def test_missing_artifact_or_report_variables_fail_early(self, stubs, tmp_path: pathlib.Path) -> None: # type: ignore[no-untyped-def] |
| 94 | env, log = stubs |
| 95 | assert run(PUBLISH, ["staging"], env).returncode != 0 |
| 96 | assert run(PUBLISH, ["staging"], env, MUSE_ARTIFACT=str(tmp_path / "x.tar.gz")).returncode != 0 |
| 97 | assert log.read_text() == "" |
| 98 | |
| 99 | def test_a_valid_report_still_stops_if_the_safe_publish_refuses(self, stubs, tmp_path: pathlib.Path) -> None: # type: ignore[no-untyped-def] |
| 100 | env, log = stubs |
| 101 | art = make_artifact(tmp_path, "0.2.1rc19") |
| 102 | report = tmp_path / "r.json" |
| 103 | report.write_text(make_report(art)) |
| 104 | from datetime import datetime, timezone |
| 105 | |
| 106 | # make_report stamps a fixed date; refresh it so the real verifier accepts it |
| 107 | import json |
| 108 | |
| 109 | data = json.loads(report.read_text()) |
| 110 | data["created_at"] = datetime.now(timezone.utc).isoformat() |
| 111 | report.write_text(json.dumps(data)) |
| 112 | r = run(PUBLISH, ["staging"], env, MUSE_ARTIFACT=str(art), MUSE_GATE_REPORT=str(report)) |
| 113 | calls = log.read_text() |
| 114 | assert r.returncode != 0 and "publish refused" in r.stderr |
| 115 | assert "release_safety" in calls and "publish" in calls |
| 116 | assert "aws " not in calls, "no instance/S3 command may run after a refused publish" |
| 117 | |
| 118 | def test_source_has_no_deletion_pruning_or_activation(self) -> None: |
| 119 | src = code(PUBLISH) |
| 120 | for forbidden in ("s3 rm", "delete", "prune", "KEEP_RELEASES", "active.json", "set-active", "--force"): |
| 121 | assert forbidden not in src, forbidden |
| 122 | assert not BARE_RM.search(src), "no rm command (docker --rm flags are fine)" |
| 123 | |
| 124 | def test_gate_verification_precedes_every_mutation(self) -> None: |
| 125 | src = code(PUBLISH) |
| 126 | gate = src.index("release_gate verify") |
| 127 | for mutation in ("release_safety", "ssm_run", "s3 cp"): |
| 128 | assert gate < src.index(mutation), mutation |
| 129 | |
| 130 | def test_instance_copy_is_additive_and_hash_checked(self) -> None: |
| 131 | src = code(PUBLISH) |
| 132 | assert "sha256sum -c" in src and "[ ! -e /data/releases/$NAME ]" in src and "cmp " in src |
| 133 | |
| 134 | |
| 135 | class TestActivateScript: |
| 136 | def test_production_activation_without_approval_is_refused_by_the_safe_layer(self, stubs) -> None: # type: ignore[no-untyped-def] |
| 137 | env, log = stubs |
| 138 | r = run(ACTIVATE, ["production", "0.2.1rc19", "0" * 64, "--expect-absent"], env) |
| 139 | calls = log.read_text() |
| 140 | assert r.returncode != 0 and "nothing changed" in r.stderr |
| 141 | assert "--approve-production" not in calls |
| 142 | assert "aws " not in calls |
| 143 | |
| 144 | def test_approval_flag_is_passed_only_when_explicitly_granted(self, stubs) -> None: # type: ignore[no-untyped-def] |
| 145 | env, log = stubs |
| 146 | run(ACTIVATE, ["production", "0.2.1rc19", "0" * 64, "--expect-absent"], env, APPROVE_PRODUCTION="1") |
| 147 | assert "--approve-production" in log.read_text() |
| 148 | |
| 149 | @pytest.mark.parametrize("args", [ |
| 150 | [], ["staging"], ["staging", "0.2.1rc19"], ["dev", "1", "2"], |
| 151 | ["staging", "0.2.1rc19", "a" * 64], # no observed state: activation is a compare-and-swap, never blind |
| 152 | ["staging", "0.2.1rc19", "a" * 64, "--expect", "0.2.1rc18", "b" * 64], # activated_at missing |
| 153 | ["staging", "0.2.1rc19", "a" * 64, "--expect-absent", "extra"], |
| 154 | ]) |
| 155 | def test_bad_usage_changes_nothing(self, stubs, args: list[str]) -> None: # type: ignore[no-untyped-def] |
| 156 | env, log = stubs |
| 157 | assert run(ACTIVATE, args, env).returncode != 0 |
| 158 | assert "aws " not in log.read_text() |
| 159 | |
| 160 | def test_activation_forwards_the_observed_state_to_the_compare_and_swap(self, stubs) -> None: # type: ignore[no-untyped-def] |
| 161 | env, log = stubs |
| 162 | run(ACTIVATE, ["staging", "0.2.1rc19", "a" * 64, "--expect", "0.2.1rc18", "b" * 64, "2026-11-03T00:00:00+00:00"], env) |
| 163 | call = log.read_text() |
| 164 | assert "--expect-version 0.2.1rc18" in call and "--expect-sha256 " + "b" * 64 in call and "--expect-activated-at 2026-11-03T00:00:00+00:00" in call |
| 165 | log.write_text("") |
| 166 | run(ACTIVATE, ["staging", "0.2.1rc19", "a" * 64, "--expect-absent"], env) |
| 167 | assert "--expect-absent" in log.read_text() |
| 168 | |
| 169 | def test_pointer_moves_before_the_instance_and_the_instance_is_hash_checked(self) -> None: |
| 170 | src = code(ACTIVATE) |
| 171 | assert src.index("release_safety") < src.index("ssm_run") |
| 172 | assert "sha256sum -c" in src and "mv -f active.json.new active.json" in src |
| 173 | |
| 174 | def test_source_never_uploads_or_deletes(self) -> None: |
| 175 | src = code(ACTIVATE) |
| 176 | for forbidden in ("s3 cp", "s3 rm", "delete", "prune", "--force"): |
| 177 | assert forbidden not in src, forbidden |
| 178 | assert not BARE_RM.search(src) |
| 179 | |
| 180 | def test_only_production_forward_activations_are_tagged(self, stubs) -> None: # type: ignore[no-untyped-def] |
| 181 | env, log = stubs |
| 182 | staging = ["staging", "0.2.1rc19", "a" * 64, "--expect-absent"] |
| 183 | run(ACTIVATE, staging, env, SAFETY_EXIT="1") |
| 184 | run(ACTIVATE, ["production", "--rollback-to", "0.2.1rc18", "b" * 64, "--expect", "0.2.1rc19", "a" * 64, "2026-11-03T00:00:00+00:00"], env, APPROVE_PRODUCTION="1") |
| 185 | assert "release_tag" not in log.read_text(), "no tag for staging, and none for a rollback (an already-released version)" |
| 186 | |
| 187 | def test_production_preflight_checks_the_named_release_before_the_pointer_moves(self, stubs) -> None: # type: ignore[no-untyped-def] |
| 188 | env, log = stubs |
| 189 | run(ACTIVATE, ["production", "0.2.1rc19", "0" * 64, "--expect-absent"], env, APPROVE_PRODUCTION="1") |
| 190 | calls = log.read_text().splitlines() |
| 191 | check = next(i for i, c in enumerate(calls) if "deploy.release_tag preflight" in c) |
| 192 | activate = next(i for i, c in enumerate(calls) if "deploy.release_safety" in c) |
| 193 | assert check < activate |
| 194 | assert "--version 0.2.1rc19 --sha256 " + "0" * 64 in calls[check] and "--muse muse" in calls[check] |
| 195 | assert not any("deploy.release_tag check" in c for c in calls), "the pre-flight is the full one: signing capability included" |
| 196 | |
| 197 | def test_a_failed_preflight_changes_nothing(self, stubs) -> None: # type: ignore[no-untyped-def] |
| 198 | env, log = stubs |
| 199 | r = run(ACTIVATE, ["production", "0.2.1rc19", "0" * 64, "--expect-absent"], env, APPROVE_PRODUCTION="1", TAG_CHECK_EXIT="1", SAFETY_EXIT="0") |
| 200 | calls = log.read_text() |
| 201 | assert r.returncode != 0 and "pre-flight refused; nothing changed" in r.stderr |
| 202 | assert "release_safety" not in calls and "aws " not in calls, "the pointer must not move for a release that cannot be tagged" |
| 203 | |
| 204 | def test_the_muse_executable_can_be_named(self, stubs) -> None: # type: ignore[no-untyped-def] |
| 205 | env, log = stubs |
| 206 | run(ACTIVATE, ["production", "0.2.1rc19", "0" * 64, "--expect-absent"], env, APPROVE_PRODUCTION="1", MUSE_BIN="/opt/muse/bin/muse", MUSE_REPO="/repo/muse") |
| 207 | assert "--muse-repo /repo/muse --muse /opt/muse/bin/muse" in log.read_text() |
File History
7 commits
sha256:369f691c7aaec3ca84a864ad1d099d5409045ddc12d185a922b3c84b9e2a1bad
merge: 100% passing, parallelized MuseHub test suite (task/…
Sonnet 5.5
patch
3 days ago
sha256:8166b51ae53275831023a7ce334228b3ccc8237b55d68a35172389db9ff46ad5
test: 100% passing MuseHub suite, parallelized (xdist, one …
Sonnet 5.5
patch
3 days ago
sha256:ec55e4f4980249bb2c95c8c0615a40a9ded3e1eb1c181fe3de7cefbd8298ff0d
feat(release): #275 safeguard — prove a signed tag can be m…
Sonnet 5.5
patch
7 days ago
sha256:0ddcb29475cfe996169320691ffb2c5705e510191a4a944487df867959d81b36
fix(release): #275 tag the verified source revision, signed…
Sonnet 5.5
minor
⚠
7 days ago
sha256:675003834b19b109f8cb126e543a064a7bd6256930dd4a80ffcfe87c6eff3f17
fix(release): idempotent concurrent publish (D3) and compar…
Opus 5.5
minor
⚠
12 days ago
sha256:04db68e43d9007348dbd726b2a3e52f73e83c5646a4511c07e19cb9a66e28b5f
fix(release): gate provenance is observed from the checkout…
Opus 5.5
minor
⚠
12 days ago
sha256:86464e3e1451e0c4629967e7d2812f4d6db4fe412f2931dd5dc28b5e20b4adbe
feat(deploy): publish is gated and additive; activation is …
Opus 5.5
minor
⚠
12 days ago