gabriel / musehub public
test_release_scripts.py python
177 lines 8.7 KB
Raw
sha256:369f691c7aaec3ca84a864ad1d099d5409045ddc12d185a922b3c84b9e2a1bad merge: 100% passing, parallelized MuseHub test suite (task/… Sonnet 5.5 patch 3 days ago
1 """publish/activate shell scripts: gate-before-mutation and no-deletion properties (musehub#114 Phase 6).
2
3 The scripts run for real, but with stub ``aws`` / ``python3`` / ``curl`` binaries first on PATH that
4 only *log* their invocations. Nothing reaches AWS, ECR, ECS or any instance.
5 """
6
7 from __future__ import annotations
8
9 import os
10 import pathlib
11 import re
12 import subprocess
13 import sys
14
15 import pytest
16
17 from tests.release_fixtures import SOURCE_REVISION, clean_status_json, make_artifact, make_report
18
19 ROOT = pathlib.Path(__file__).resolve().parents[1]
20 PUBLISH = ROOT / "deploy" / "publish_muse_release.sh"
21 ACTIVATE = ROOT / "deploy" / "activate_muse_release.sh"
22 STATUS_JSON = clean_status_json(SOURCE_REVISION)
23 BARE_RM = re.compile(r"(?<![-\w])rm(?![-\w])")
24
25
26 def code(path: pathlib.Path) -> str:
27 return "\n".join(line for line in path.read_text().splitlines() if not line.strip().startswith("#"))
28
29
30 @pytest.fixture
31 def stubs(tmp_path: pathlib.Path) -> tuple[dict[str, str], pathlib.Path]:
32 """PATH with logging stubs. ``python3 -m deploy.release_safety`` is stubbed (it would hit S3);
33 ``python3 -m deploy.release_gate`` and plain python run for real."""
34 bin_dir = tmp_path / "bin"
35 bin_dir.mkdir()
36 log = tmp_path / "calls.log"
37 log.write_text("")
38 for name in ("aws", "curl"):
39 (bin_dir / name).write_text(f'#!/bin/sh\necho "{name} $*" >> "{log}"\nexit 0\n')
40 # muse reports a clean checkout at SOURCE_REVISION for the provenance check (rev-parse / status)
41 (bin_dir / "muse").write_text(
42 f'#!/bin/sh\necho "muse $*" >> "{log}"\n'
43 f'case "$*" in\n'
44 f' *rev-parse*) echo \'{{"commit_id": "{SOURCE_REVISION}"}}\' ;;\n'
45 f' *status*) echo \'{STATUS_JSON}\' ;;\nesac\nexit 0\n'
46 )
47 (bin_dir / "python3").write_text(
48 f'#!/bin/sh\ncase "$*" in\n *deploy.release_safety*) echo "python3 $*" >> "{log}"; exit "${{SAFETY_EXIT:-1}}" ;;\n'
49 f' *) exec {sys.executable} "$@" ;;\nesac\n'
50 )
51 for f in bin_dir.iterdir():
52 f.chmod(0o755)
53 env = {**os.environ, "PATH": f"{bin_dir}:{os.environ['PATH']}", "PYTHONPATH": str(ROOT)}
54 return env, log
55
56
57 def run(script: pathlib.Path, args: list[str], env: dict[str, str], **extra: str) -> subprocess.CompletedProcess[str]:
58 return subprocess.run(["bash", str(script), *args], env={**env, **extra}, capture_output=True, text=True, cwd=ROOT, timeout=60)
59
60
61 class TestPublishScript:
62 @pytest.mark.parametrize("args", [[], ["prod"], ["Staging"], [""]])
63 def test_environment_is_required_and_exact(self, stubs, args: list[str]) -> None: # type: ignore[no-untyped-def]
64 env, log = stubs
65 r = run(PUBLISH, args, env)
66 assert r.returncode != 0 and "Usage" in r.stderr
67 assert log.read_text() == ""
68
69 def test_missing_report_stops_before_any_aws_call(self, stubs, tmp_path: pathlib.Path) -> None: # type: ignore[no-untyped-def]
70 env, log = stubs
71 art = make_artifact(tmp_path, "0.2.1rc19")
72 r = run(PUBLISH, ["staging"], env, MUSE_ARTIFACT=str(art), MUSE_GATE_REPORT=str(tmp_path / "nope.json"))
73 assert r.returncode != 0 and "GATE FAILED" in r.stderr
74 calls = log.read_text()
75 assert "aws " not in calls and "release_safety" not in calls # read-only muse observation is allowed
76
77 def test_report_for_other_bytes_stops_before_any_aws_call(self, stubs, tmp_path: pathlib.Path) -> None: # type: ignore[no-untyped-def]
78 env, log = stubs
79 art = make_artifact(tmp_path, "0.2.1rc19")
80 report = tmp_path / "r.json"
81 report.write_text(make_report(art))
82 art.write_bytes(b"rebuilt after validation")
83 r = run(PUBLISH, ["staging"], env, MUSE_ARTIFACT=str(art), MUSE_GATE_REPORT=str(report))
84 assert r.returncode != 0 and "GATE FAILED" in r.stderr and "nothing was published" in r.stderr
85 calls = log.read_text()
86 assert "aws " not in calls and "release_safety" not in calls # read-only muse observation is allowed
87
88 def test_missing_artifact_or_report_variables_fail_early(self, stubs, tmp_path: pathlib.Path) -> None: # type: ignore[no-untyped-def]
89 env, log = stubs
90 assert run(PUBLISH, ["staging"], env).returncode != 0
91 assert run(PUBLISH, ["staging"], env, MUSE_ARTIFACT=str(tmp_path / "x.tar.gz")).returncode != 0
92 assert log.read_text() == ""
93
94 def test_a_valid_report_still_stops_if_the_safe_publish_refuses(self, stubs, tmp_path: pathlib.Path) -> None: # type: ignore[no-untyped-def]
95 env, log = stubs
96 art = make_artifact(tmp_path, "0.2.1rc19")
97 report = tmp_path / "r.json"
98 report.write_text(make_report(art))
99 from datetime import datetime, timezone
100
101 # make_report stamps a fixed date; refresh it so the real verifier accepts it
102 import json
103
104 data = json.loads(report.read_text())
105 data["created_at"] = datetime.now(timezone.utc).isoformat()
106 report.write_text(json.dumps(data))
107 r = run(PUBLISH, ["staging"], env, MUSE_ARTIFACT=str(art), MUSE_GATE_REPORT=str(report))
108 calls = log.read_text()
109 assert r.returncode != 0 and "publish refused" in r.stderr
110 assert "release_safety" in calls and "publish" in calls
111 assert "aws " not in calls, "no instance/S3 command may run after a refused publish"
112
113 def test_source_has_no_deletion_pruning_or_activation(self) -> None:
114 src = code(PUBLISH)
115 for forbidden in ("s3 rm", "delete", "prune", "KEEP_RELEASES", "active.json", "set-active", "--force"):
116 assert forbidden not in src, forbidden
117 assert not BARE_RM.search(src), "no rm command (docker --rm flags are fine)"
118
119 def test_gate_verification_precedes_every_mutation(self) -> None:
120 src = code(PUBLISH)
121 gate = src.index("release_gate verify")
122 for mutation in ("release_safety", "ssm_run", "s3 cp"):
123 assert gate < src.index(mutation), mutation
124
125 def test_instance_copy_is_additive_and_hash_checked(self) -> None:
126 src = code(PUBLISH)
127 assert "sha256sum -c" in src and "[ ! -e /data/releases/$NAME ]" in src and "cmp " in src
128
129
130 class TestActivateScript:
131 def test_production_activation_without_approval_is_refused_by_the_safe_layer(self, stubs) -> None: # type: ignore[no-untyped-def]
132 env, log = stubs
133 r = run(ACTIVATE, ["production", "0.2.1rc19", "0" * 64, "--expect-absent"], env)
134 calls = log.read_text()
135 assert r.returncode != 0 and "nothing changed" in r.stderr
136 assert "--approve-production" not in calls
137 assert "aws " not in calls
138
139 def test_approval_flag_is_passed_only_when_explicitly_granted(self, stubs) -> None: # type: ignore[no-untyped-def]
140 env, log = stubs
141 run(ACTIVATE, ["production", "0.2.1rc19", "0" * 64, "--expect-absent"], env, APPROVE_PRODUCTION="1")
142 assert "--approve-production" in log.read_text()
143
144 @pytest.mark.parametrize("args", [
145 [], ["staging"], ["staging", "0.2.1rc19"], ["dev", "1", "2"],
146 ["staging", "0.2.1rc19", "a" * 64], # no observed state: activation is a compare-and-swap, never blind
147 ["staging", "0.2.1rc19", "a" * 64, "--expect", "0.2.1rc18", "b" * 64], # activated_at missing
148 ["staging", "0.2.1rc19", "a" * 64, "--expect-absent", "extra"],
149 ])
150 def test_bad_usage_changes_nothing(self, stubs, args: list[str]) -> None: # type: ignore[no-untyped-def]
151 env, log = stubs
152 assert run(ACTIVATE, args, env).returncode != 0
153 assert "aws " not in log.read_text()
154
155 def test_activation_forwards_the_observed_state_to_the_compare_and_swap(self, stubs) -> None: # type: ignore[no-untyped-def]
156 env, log = stubs
157 run(ACTIVATE, ["staging", "0.2.1rc19", "a" * 64, "--expect", "0.2.1rc18", "b" * 64, "2026-11-03T00:00:00+00:00"], env)
158 call = log.read_text()
159 assert "--expect-version 0.2.1rc18" in call and "--expect-sha256 " + "b" * 64 in call and "--expect-activated-at 2026-11-03T00:00:00+00:00" in call
160 log.write_text("")
161 run(ACTIVATE, ["staging", "0.2.1rc19", "a" * 64, "--expect-absent"], env)
162 assert "--expect-absent" in log.read_text()
163
164 def test_pointer_moves_before_the_instance_and_the_instance_is_hash_checked(self) -> None:
165 src = code(ACTIVATE)
166 assert src.index("release_safety") < src.index("ssm_run")
167 assert "sha256sum -c" in src and "mv -f active.json.new active.json" in src
168
169 def test_source_never_uploads_or_deletes(self) -> None:
170 src = code(ACTIVATE)
171 for forbidden in ("s3 cp", "s3 rm", "delete", "prune", "--force"):
172 assert forbidden not in src, forbidden
173 assert not BARE_RM.search(src)
174
175 def test_only_production_is_tagged(self) -> None:
176 src = code(ACTIVATE)
177 assert src.index('if [ "$ENV" = "production" ]') < src.index("tag add")
File History 7 commits
sha256:369f691c7aaec3ca84a864ad1d099d5409045ddc12d185a922b3c84b9e2a1bad merge: 100% passing, parallelized MuseHub test suite (task/… Sonnet 5.5 patch 3 days ago
sha256:8166b51ae53275831023a7ce334228b3ccc8237b55d68a35172389db9ff46ad5 test: 100% passing MuseHub suite, parallelized (xdist, one … Sonnet 5.5 patch 3 days ago
sha256:ec55e4f4980249bb2c95c8c0615a40a9ded3e1eb1c181fe3de7cefbd8298ff0d feat(release): #275 safeguard — prove a signed tag can be m… Sonnet 5.5 patch 7 days ago
sha256:0ddcb29475cfe996169320691ffb2c5705e510191a4a944487df867959d81b36 fix(release): #275 tag the verified source revision, signed… Sonnet 5.5 minor ⚠ 7 days ago
sha256:675003834b19b109f8cb126e543a064a7bd6256930dd4a80ffcfe87c6eff3f17 fix(release): idempotent concurrent publish (D3) and compar… Opus 5.5 minor ⚠ 12 days ago
sha256:04db68e43d9007348dbd726b2a3e52f73e83c5646a4511c07e19cb9a66e28b5f fix(release): gate provenance is observed from the checkout… Opus 5.5 minor ⚠ 12 days ago
sha256:86464e3e1451e0c4629967e7d2812f4d6db4fe412f2931dd5dc28b5e20b4adbe feat(deploy): publish is gated and additive; activation is … Opus 5.5 minor ⚠ 12 days ago