gabriel / musehub public
test_release_scripts.py python
207 lines 11.2 KB
Raw
sha256:369f691c7aaec3ca84a864ad1d099d5409045ddc12d185a922b3c84b9e2a1bad merge: 100% passing, parallelized MuseHub test suite (task/… Sonnet 5.5 patch 3 days ago
1 """publish/activate shell scripts: gate-before-mutation and no-deletion properties (musehub#114 Phase 6).
2
3 The scripts run for real, but with stub ``aws`` / ``python3`` / ``curl`` binaries first on PATH that
4 only *log* their invocations. Nothing reaches AWS, ECR, ECS or any instance.
5 """
6
7 from __future__ import annotations
8
9 import os
10 import pathlib
11 import re
12 import shutil
13 import subprocess
14 import sys
15
16 import pytest
17
18 from tests.release_fixtures import SOURCE_REVISION, clean_status_json, make_artifact, make_report
19
20 ROOT = pathlib.Path(__file__).resolve().parents[1]
21 PUBLISH = ROOT / "deploy" / "publish_muse_release.sh"
22 ACTIVATE = ROOT / "deploy" / "activate_muse_release.sh"
23 STATUS_JSON = clean_status_json(SOURCE_REVISION)
24 BARE_RM = re.compile(r"(?<![-\w])rm(?![-\w])")
25
26
27 def code(path: pathlib.Path) -> str:
28 return "\n".join(line for line in path.read_text().splitlines() if not line.strip().startswith("#"))
29
30
31 @pytest.fixture
32 def stubs(tmp_path: pathlib.Path) -> tuple[dict[str, str], pathlib.Path]:
33 """PATH with logging stubs. ``python3 -m deploy.release_safety`` and ``deploy.release_tag`` are stubbed (they would hit S3);
34 ``python3 -m deploy.release_gate`` and plain python run for real."""
35 bin_dir = tmp_path / "bin"
36 bin_dir.mkdir()
37 log = tmp_path / "calls.log"
38 log.write_text("")
39 for name in ("aws", "curl"):
40 (bin_dir / name).write_text(f'#!/bin/sh\necho "{name} $*" >> "{log}"\nexit 0\n')
41 # muse reports a clean checkout at SOURCE_REVISION for the provenance check (rev-parse / status)
42 (bin_dir / "muse").write_text(
43 f'#!/bin/sh\necho "muse $*" >> "{log}"\n'
44 f'case "$*" in\n'
45 f' *rev-parse*) echo \'{{"commit_id": "{SOURCE_REVISION}"}}\' ;;\n'
46 f' *status*) echo \'{STATUS_JSON}\' ;;\nesac\nexit 0\n'
47 )
48 (bin_dir / "python3").write_text(
49 f'#!/bin/sh\ncase "$*" in\n *deploy.release_safety*) echo "python3 $*" >> "{log}"; exit "${{SAFETY_EXIT:-1}}" ;;\n'
50 f' *deploy.release_tag*) echo "python3 $*" >> "{log}"; case "$*" in *" apply "*) exit "${{TAG_APPLY_EXIT:-0}}" ;; *) exit "${{TAG_CHECK_EXIT:-0}}" ;; esac ;;\n'
51 f' *) exec {sys.executable} "$@" ;;\nesac\n'
52 )
53 if shutil.which("shasum") is None:
54 # The scripts use macOS's ``shasum -a 256``; Linux images ship ``sha256sum`` (same output format).
55 (bin_dir / "shasum").write_text('#!/bin/sh\nif [ "$1" = "-a" ]; then shift 2; fi\nexec sha256sum "$@"\n')
56 for f in bin_dir.iterdir():
57 f.chmod(0o755)
58 env = {**os.environ, "PATH": f"{bin_dir}:{os.environ['PATH']}", "PYTHONPATH": str(ROOT)}
59 return env, log
60
61
62 def run(script: pathlib.Path, args: list[str], env: dict[str, str], **extra: str) -> subprocess.CompletedProcess[str]:
63 return subprocess.run(["bash", str(script), *args], env={**env, **extra}, capture_output=True, text=True, cwd=ROOT, timeout=60)
64
65
66 class TestPublishScript:
67 @pytest.mark.parametrize("args", [[], ["prod"], ["Staging"], [""]])
68 def test_environment_is_required_and_exact(self, stubs, args: list[str]) -> None: # type: ignore[no-untyped-def]
69 env, log = stubs
70 r = run(PUBLISH, args, env)
71 assert r.returncode != 0 and "Usage" in r.stderr
72 assert log.read_text() == ""
73
74 def test_missing_report_stops_before_any_aws_call(self, stubs, tmp_path: pathlib.Path) -> None: # type: ignore[no-untyped-def]
75 env, log = stubs
76 art = make_artifact(tmp_path, "0.2.1rc19")
77 r = run(PUBLISH, ["staging"], env, MUSE_ARTIFACT=str(art), MUSE_GATE_REPORT=str(tmp_path / "nope.json"))
78 assert r.returncode != 0 and "GATE FAILED" in r.stderr
79 calls = log.read_text()
80 assert "aws " not in calls and "release_safety" not in calls # read-only muse observation is allowed
81
82 def test_report_for_other_bytes_stops_before_any_aws_call(self, stubs, tmp_path: pathlib.Path) -> None: # type: ignore[no-untyped-def]
83 env, log = stubs
84 art = make_artifact(tmp_path, "0.2.1rc19")
85 report = tmp_path / "r.json"
86 report.write_text(make_report(art))
87 art.write_bytes(b"rebuilt after validation")
88 r = run(PUBLISH, ["staging"], env, MUSE_ARTIFACT=str(art), MUSE_GATE_REPORT=str(report))
89 assert r.returncode != 0 and "GATE FAILED" in r.stderr and "nothing was published" in r.stderr
90 calls = log.read_text()
91 assert "aws " not in calls and "release_safety" not in calls # read-only muse observation is allowed
92
93 def test_missing_artifact_or_report_variables_fail_early(self, stubs, tmp_path: pathlib.Path) -> None: # type: ignore[no-untyped-def]
94 env, log = stubs
95 assert run(PUBLISH, ["staging"], env).returncode != 0
96 assert run(PUBLISH, ["staging"], env, MUSE_ARTIFACT=str(tmp_path / "x.tar.gz")).returncode != 0
97 assert log.read_text() == ""
98
99 def test_a_valid_report_still_stops_if_the_safe_publish_refuses(self, stubs, tmp_path: pathlib.Path) -> None: # type: ignore[no-untyped-def]
100 env, log = stubs
101 art = make_artifact(tmp_path, "0.2.1rc19")
102 report = tmp_path / "r.json"
103 report.write_text(make_report(art))
104 from datetime import datetime, timezone
105
106 # make_report stamps a fixed date; refresh it so the real verifier accepts it
107 import json
108
109 data = json.loads(report.read_text())
110 data["created_at"] = datetime.now(timezone.utc).isoformat()
111 report.write_text(json.dumps(data))
112 r = run(PUBLISH, ["staging"], env, MUSE_ARTIFACT=str(art), MUSE_GATE_REPORT=str(report))
113 calls = log.read_text()
114 assert r.returncode != 0 and "publish refused" in r.stderr
115 assert "release_safety" in calls and "publish" in calls
116 assert "aws " not in calls, "no instance/S3 command may run after a refused publish"
117
118 def test_source_has_no_deletion_pruning_or_activation(self) -> None:
119 src = code(PUBLISH)
120 for forbidden in ("s3 rm", "delete", "prune", "KEEP_RELEASES", "active.json", "set-active", "--force"):
121 assert forbidden not in src, forbidden
122 assert not BARE_RM.search(src), "no rm command (docker --rm flags are fine)"
123
124 def test_gate_verification_precedes_every_mutation(self) -> None:
125 src = code(PUBLISH)
126 gate = src.index("release_gate verify")
127 for mutation in ("release_safety", "ssm_run", "s3 cp"):
128 assert gate < src.index(mutation), mutation
129
130 def test_instance_copy_is_additive_and_hash_checked(self) -> None:
131 src = code(PUBLISH)
132 assert "sha256sum -c" in src and "[ ! -e /data/releases/$NAME ]" in src and "cmp " in src
133
134
135 class TestActivateScript:
136 def test_production_activation_without_approval_is_refused_by_the_safe_layer(self, stubs) -> None: # type: ignore[no-untyped-def]
137 env, log = stubs
138 r = run(ACTIVATE, ["production", "0.2.1rc19", "0" * 64, "--expect-absent"], env)
139 calls = log.read_text()
140 assert r.returncode != 0 and "nothing changed" in r.stderr
141 assert "--approve-production" not in calls
142 assert "aws " not in calls
143
144 def test_approval_flag_is_passed_only_when_explicitly_granted(self, stubs) -> None: # type: ignore[no-untyped-def]
145 env, log = stubs
146 run(ACTIVATE, ["production", "0.2.1rc19", "0" * 64, "--expect-absent"], env, APPROVE_PRODUCTION="1")
147 assert "--approve-production" in log.read_text()
148
149 @pytest.mark.parametrize("args", [
150 [], ["staging"], ["staging", "0.2.1rc19"], ["dev", "1", "2"],
151 ["staging", "0.2.1rc19", "a" * 64], # no observed state: activation is a compare-and-swap, never blind
152 ["staging", "0.2.1rc19", "a" * 64, "--expect", "0.2.1rc18", "b" * 64], # activated_at missing
153 ["staging", "0.2.1rc19", "a" * 64, "--expect-absent", "extra"],
154 ])
155 def test_bad_usage_changes_nothing(self, stubs, args: list[str]) -> None: # type: ignore[no-untyped-def]
156 env, log = stubs
157 assert run(ACTIVATE, args, env).returncode != 0
158 assert "aws " not in log.read_text()
159
160 def test_activation_forwards_the_observed_state_to_the_compare_and_swap(self, stubs) -> None: # type: ignore[no-untyped-def]
161 env, log = stubs
162 run(ACTIVATE, ["staging", "0.2.1rc19", "a" * 64, "--expect", "0.2.1rc18", "b" * 64, "2026-11-03T00:00:00+00:00"], env)
163 call = log.read_text()
164 assert "--expect-version 0.2.1rc18" in call and "--expect-sha256 " + "b" * 64 in call and "--expect-activated-at 2026-11-03T00:00:00+00:00" in call
165 log.write_text("")
166 run(ACTIVATE, ["staging", "0.2.1rc19", "a" * 64, "--expect-absent"], env)
167 assert "--expect-absent" in log.read_text()
168
169 def test_pointer_moves_before_the_instance_and_the_instance_is_hash_checked(self) -> None:
170 src = code(ACTIVATE)
171 assert src.index("release_safety") < src.index("ssm_run")
172 assert "sha256sum -c" in src and "mv -f active.json.new active.json" in src
173
174 def test_source_never_uploads_or_deletes(self) -> None:
175 src = code(ACTIVATE)
176 for forbidden in ("s3 cp", "s3 rm", "delete", "prune", "--force"):
177 assert forbidden not in src, forbidden
178 assert not BARE_RM.search(src)
179
180 def test_only_production_forward_activations_are_tagged(self, stubs) -> None: # type: ignore[no-untyped-def]
181 env, log = stubs
182 staging = ["staging", "0.2.1rc19", "a" * 64, "--expect-absent"]
183 run(ACTIVATE, staging, env, SAFETY_EXIT="1")
184 run(ACTIVATE, ["production", "--rollback-to", "0.2.1rc18", "b" * 64, "--expect", "0.2.1rc19", "a" * 64, "2026-11-03T00:00:00+00:00"], env, APPROVE_PRODUCTION="1")
185 assert "release_tag" not in log.read_text(), "no tag for staging, and none for a rollback (an already-released version)"
186
187 def test_production_preflight_checks_the_named_release_before_the_pointer_moves(self, stubs) -> None: # type: ignore[no-untyped-def]
188 env, log = stubs
189 run(ACTIVATE, ["production", "0.2.1rc19", "0" * 64, "--expect-absent"], env, APPROVE_PRODUCTION="1")
190 calls = log.read_text().splitlines()
191 check = next(i for i, c in enumerate(calls) if "deploy.release_tag preflight" in c)
192 activate = next(i for i, c in enumerate(calls) if "deploy.release_safety" in c)
193 assert check < activate
194 assert "--version 0.2.1rc19 --sha256 " + "0" * 64 in calls[check] and "--muse muse" in calls[check]
195 assert not any("deploy.release_tag check" in c for c in calls), "the pre-flight is the full one: signing capability included"
196
197 def test_a_failed_preflight_changes_nothing(self, stubs) -> None: # type: ignore[no-untyped-def]
198 env, log = stubs
199 r = run(ACTIVATE, ["production", "0.2.1rc19", "0" * 64, "--expect-absent"], env, APPROVE_PRODUCTION="1", TAG_CHECK_EXIT="1", SAFETY_EXIT="0")
200 calls = log.read_text()
201 assert r.returncode != 0 and "pre-flight refused; nothing changed" in r.stderr
202 assert "release_safety" not in calls and "aws " not in calls, "the pointer must not move for a release that cannot be tagged"
203
204 def test_the_muse_executable_can_be_named(self, stubs) -> None: # type: ignore[no-untyped-def]
205 env, log = stubs
206 run(ACTIVATE, ["production", "0.2.1rc19", "0" * 64, "--expect-absent"], env, APPROVE_PRODUCTION="1", MUSE_BIN="/opt/muse/bin/muse", MUSE_REPO="/repo/muse")
207 assert "--muse-repo /repo/muse --muse /opt/muse/bin/muse" in log.read_text()
File History 7 commits
sha256:369f691c7aaec3ca84a864ad1d099d5409045ddc12d185a922b3c84b9e2a1bad merge: 100% passing, parallelized MuseHub test suite (task/… Sonnet 5.5 patch 3 days ago
sha256:8166b51ae53275831023a7ce334228b3ccc8237b55d68a35172389db9ff46ad5 test: 100% passing MuseHub suite, parallelized (xdist, one … Sonnet 5.5 patch 3 days ago
sha256:ec55e4f4980249bb2c95c8c0615a40a9ded3e1eb1c181fe3de7cefbd8298ff0d feat(release): #275 safeguard — prove a signed tag can be m… Sonnet 5.5 patch 7 days ago
sha256:0ddcb29475cfe996169320691ffb2c5705e510191a4a944487df867959d81b36 fix(release): #275 tag the verified source revision, signed… Sonnet 5.5 minor ⚠ 7 days ago
sha256:675003834b19b109f8cb126e543a064a7bd6256930dd4a80ffcfe87c6eff3f17 fix(release): idempotent concurrent publish (D3) and compar… Opus 5.5 minor ⚠ 12 days ago
sha256:04db68e43d9007348dbd726b2a3e52f73e83c5646a4511c07e19cb9a66e28b5f fix(release): gate provenance is observed from the checkout… Opus 5.5 minor ⚠ 12 days ago
sha256:86464e3e1451e0c4629967e7d2812f4d6db4fe412f2931dd5dc28b5e20b4adbe feat(deploy): publish is gated and additive; activation is … Opus 5.5 minor ⚠ 12 days ago