gabriel / musehub public
test_release_provenance.py python
328 lines 16.1 KB
Raw
sha256:a977b39d08e15683210a33608e25712fe3ed9c8f05f0388fb69ef38acf835426 feat(release): hermetic release gates in a provisioned, pin… Opus 5.5 minor ⚠ breaking 12 days ago
1 """Source provenance of a gate report (musehub#114 Phase 6D, D2).
2
3 The revision in a report must be *observed* from the checkout with Muse, never trusted from the
4 operator, and the report must be unusable for any other revision, tree state or artifact.
5 """
6
7 from __future__ import annotations
8
9 import json
10 import pathlib
11 import subprocess
12 import sys
13 from collections.abc import Callable, Iterator
14 from contextlib import contextmanager
15 from typing import Any
16
17 import pytest
18
19 from deploy.gate_env import GateEnv, Interpreter
20 from deploy.release_gate import REQUIRED_GATES, GateError, GateResult, run_release_gates, verify_report
21 from deploy.release_provenance import CheckoutState, ProvenanceError, observe_checkout
22 from deploy.release_safety import publish
23 from tests.release_fixtures import (
24 CHECKOUT,
25 NOW,
26 OTHER_REVISION,
27 SOURCE_REVISION,
28 FakeStorage,
29 clean_status_json,
30 make_artifact,
31 make_report,
32 mutate_report,
33 passing_results,
34 sha256_of,
35 )
36
37
38 @pytest.fixture
39 def artifact(tmp_path: pathlib.Path) -> pathlib.Path:
40 return make_artifact(tmp_path, "0.2.1rc19")
41
42
43 # ── observing the checkout with Muse ──────────────────────────────────────────
44
45
46 def fake_muse(rev: object = SOURCE_REVISION, status: str | None = None, code: int = 0, raises: Exception | None = None) -> Callable[..., Any]:
47 def runner(argv: list[str], **_kw: object) -> subprocess.CompletedProcess[str]:
48 if raises:
49 raise raises
50 if "rev-parse" in argv:
51 out = json.dumps({"commit_id": rev})
52 else:
53 out = status if status is not None else clean_status_json(SOURCE_REVISION)
54 return subprocess.CompletedProcess(argv, code, out, "boom" if code else "")
55
56 return runner
57
58
59 class TestObserveCheckout:
60 def test_clean_checkout_reports_its_revision(self, tmp_path: pathlib.Path) -> None:
61 assert observe_checkout(tmp_path, runner=fake_muse()) == CheckoutState(SOURCE_REVISION, True, ())
62
63 def test_uses_muse_not_git(self, tmp_path: pathlib.Path) -> None:
64 seen: list[list[str]] = []
65
66 def spy(argv: list[str], **kw: object) -> subprocess.CompletedProcess[str]:
67 seen.append(argv)
68 return fake_muse()(argv, **kw)
69
70 observe_checkout(tmp_path, runner=spy)
71 assert seen and all(a[0] == "muse" and a[1] == "-C" for a in seen)
72 assert not any("git" in part for argv in seen for part in argv[:1])
73
74 @pytest.mark.parametrize(
75 "override",
76 [
77 {"modified": ["muse/cli.py"], "clean": False, "dirty": True},
78 {"added": ["new.py"], "clean": False, "dirty": True},
79 {"deleted": ["gone.py"], "clean": False, "dirty": True},
80 {"untracked": ["stray.py"]},
81 {"renamed": {"a.py": "b.py"}},
82 {"conflict_paths": ["c.py"]},
83 {"staged": {"added": [], "modified": ["x.py"], "deleted": [], "renamed": {}}},
84 {"merge_in_progress": True},
85 {"checkout_interrupted": True},
86 {"clean": False, "dirty": True},
87 ],
88 )
89 def test_any_drift_marks_the_tree_dirty(self, tmp_path: pathlib.Path, override: dict[str, object]) -> None:
90 state = observe_checkout(tmp_path, runner=fake_muse(status=clean_status_json(SOURCE_REVISION, **override)))
91 assert state.revision == SOURCE_REVISION and state.clean is False and state.dirty
92
93 def test_muse_missing_is_an_error_not_an_unknown(self, tmp_path: pathlib.Path) -> None:
94 with pytest.raises(ProvenanceError, match="could not run"):
95 observe_checkout(tmp_path, runner=fake_muse(raises=FileNotFoundError("muse")))
96
97 def test_muse_failure_is_an_error(self, tmp_path: pathlib.Path) -> None:
98 with pytest.raises(ProvenanceError, match="exited 2"):
99 observe_checkout(tmp_path, runner=fake_muse(code=2))
100
101 def test_timeout_is_an_error(self, tmp_path: pathlib.Path) -> None:
102 with pytest.raises(ProvenanceError):
103 observe_checkout(tmp_path, runner=fake_muse(raises=subprocess.TimeoutExpired("muse", 60)))
104
105 @pytest.mark.parametrize("rev", ["", None, "abc", "sha256:xyz", "sha256:" + "A" * 64, "g" * 64, 5])
106 def test_malformed_revision_is_an_error(self, tmp_path: pathlib.Path, rev: object) -> None:
107 with pytest.raises(ProvenanceError):
108 observe_checkout(tmp_path, runner=fake_muse(rev=rev))
109
110 @pytest.mark.parametrize("status", ["not json", "[]", "{}", json.dumps({"head_commit": SOURCE_REVISION})])
111 def test_unparseable_or_incomplete_status_is_an_error(self, tmp_path: pathlib.Path, status: str) -> None:
112 with pytest.raises(ProvenanceError):
113 observe_checkout(tmp_path, runner=fake_muse(status=status))
114
115 def test_rev_parse_and_status_disagreeing_is_an_error(self, tmp_path: pathlib.Path) -> None:
116 with pytest.raises(ProvenanceError, match="disagree"):
117 observe_checkout(tmp_path, runner=fake_muse(rev=OTHER_REVISION))
118
119
120 # ── the gate run observes, it does not trust ──────────────────────────────────
121
122
123 class Observer:
124 """Returns the given states in order (the last repeats)."""
125
126 def __init__(self, *states: CheckoutState | Exception) -> None:
127 self.states, self.calls = list(states), 0
128
129 def __call__(self, _repo: pathlib.Path) -> CheckoutState:
130 state = self.states[min(self.calls, len(self.states) - 1)]
131 self.calls += 1
132 if isinstance(state, Exception):
133 raise state
134 return state
135
136
137 @contextmanager
138 def fake_environment(artifact: pathlib.Path, python: object) -> Iterator[GateEnv]:
139 """Stands in for deploy.gate_env.gate_environment; its own behaviour is tested in test_release_gate_env.py."""
140 sha = sha256_of(artifact)
141 version = artifact.name.removeprefix("muse-").removesuffix(".tar.gz")
142 yield GateEnv(
143 pathlib.Path("/nonexistent/env"), pathlib.Path("/nonexistent/env/venv/bin/python"),
144 Interpreter(pathlib.Path("/opt/python/bin/python3.14"), "3.14.4", "cpython"),
145 {"muse": version, "pytest": "9.0.3"}, "1" * 64, sha,
146 )
147
148
149 class Spy:
150 def __init__(self, on_run: Callable[[], None] | None = None) -> None:
151 self.ran, self.on_run = False, on_run
152
153 def __call__(self, commands: object, **_kw: object) -> dict[str, GateResult]:
154 self.ran = True
155 if self.on_run:
156 self.on_run()
157 return passing_results()
158
159
160 def gate(artifact: pathlib.Path, observer: Observer, spy: Spy, claimed: str = SOURCE_REVISION) -> dict[str, Any]:
161 return run_release_gates(
162 artifact, artifact.parent, artifact.parent, claimed, 2, python="/py", now=NOW, observe=observer, run=spy,
163 environment=fake_environment,
164 )
165
166
167 class TestGateRunObservesTheCheckout:
168 def test_report_records_claimed_and_observed_revisions_and_binds_the_artifact(self, artifact: pathlib.Path) -> None:
169 spy = Spy()
170 report = gate(artifact, Observer(CHECKOUT), spy)
171 prov = report["provenance"]
172 assert spy.ran and report["decision"] == "pass"
173 assert prov["claimed_revision"] == prov["observed_revision_before"] == prov["observed_revision_after"] == SOURCE_REVISION
174 assert prov["tree_clean"] is True and prov["artifact_sha256"] == report["candidate"]["sha256"]
175 assert "unattested" in prov["build_relation"], "the build link must not be overstated"
176 verify_report(json.dumps(report), artifact, now=NOW, checkout=CHECKOUT)
177
178 def test_operator_supplied_wrong_revision_is_refused_before_any_gate_runs(self, artifact: pathlib.Path) -> None:
179 spy = Spy()
180 with pytest.raises(GateError, match="claimed revision"):
181 gate(artifact, Observer(CHECKOUT), spy, claimed=OTHER_REVISION)
182 assert not spy.ran
183
184 def test_malformed_claimed_revision_is_refused(self, artifact: pathlib.Path) -> None:
185 spy = Spy()
186 with pytest.raises(GateError, match="source provenance"):
187 gate(artifact, Observer(CHECKOUT), spy, claimed="HEAD")
188 assert not spy.ran
189
190 def test_undeterminable_revision_is_refused(self, artifact: pathlib.Path) -> None:
191 spy = Spy()
192 with pytest.raises(GateError, match="source provenance"):
193 gate(artifact, Observer(ProvenanceError("muse not found")), spy)
194 assert not spy.ran
195
196 def test_uncommitted_changes_refuse_the_run(self, artifact: pathlib.Path) -> None:
197 spy = Spy()
198 with pytest.raises(GateError, match="uncommitted"):
199 gate(artifact, Observer(CheckoutState(SOURCE_REVISION, False, ("muse/cli.py",))), spy)
200 assert not spy.ran
201
202 def test_checkout_moving_during_the_run_voids_the_evidence(self, artifact: pathlib.Path) -> None:
203 with pytest.raises(GateError, match="changed while the gates ran"):
204 gate(artifact, Observer(CHECKOUT, CheckoutState(OTHER_REVISION, True, ())), Spy())
205
206 def test_checkout_dirtied_during_the_run_voids_the_evidence(self, artifact: pathlib.Path) -> None:
207 with pytest.raises(GateError, match="changed while the gates ran"):
208 gate(artifact, Observer(CHECKOUT, CheckoutState(SOURCE_REVISION, False, ("x.py",))), Spy())
209
210 def test_unobservable_after_the_run_voids_the_evidence(self, artifact: pathlib.Path) -> None:
211 with pytest.raises(GateError, match="after the run"):
212 gate(artifact, Observer(CHECKOUT, ProvenanceError("gone")), Spy())
213
214 def test_artifact_swapped_during_the_run_voids_the_evidence(self, artifact: pathlib.Path) -> None:
215 spy = Spy(on_run=lambda: artifact.write_bytes(b"rebuilt mid-run"))
216 with pytest.raises(GateError, match="artifact changed"):
217 gate(artifact, Observer(CHECKOUT), spy)
218
219 def test_a_candidate_not_built_from_the_tested_checkout_fails_the_reproducibility_gate(self, artifact: pathlib.Path) -> None:
220 """The only build evidence is content-equivalence of a fresh sdist of the observed checkout.
221 When that gate fails, the report is a fail and can never authorize publication."""
222
223 class FailingBuild(Spy):
224 def __call__(self, commands: object, **_kw: object) -> dict[str, GateResult]:
225 results = passing_results()
226 results["packaging.sdist_reproducible"] = GateResult("fail", 1, "differing members: ['muse/cli.py']")
227 return results
228
229 report = gate(artifact, Observer(CHECKOUT), FailingBuild())
230 assert report["decision"] == "fail"
231 with pytest.raises(GateError, match="packaging.sdist_reproducible"):
232 verify_report(json.dumps(report), artifact, now=NOW, checkout=CHECKOUT)
233 assert "packaging.sdist_reproducible" in REQUIRED_GATES
234
235
236 # ── verification re-checks provenance ─────────────────────────────────────────
237
238
239 class TestVerifyReportProvenance:
240 def test_checkout_changed_after_the_gate_ran_is_rejected(self, artifact: pathlib.Path) -> None:
241 with pytest.raises(GateError, match="checkout is now at"):
242 verify_report(make_report(artifact), artifact, now=NOW, checkout=CheckoutState(OTHER_REVISION, True, ()))
243
244 def test_report_replayed_against_another_revision_is_rejected(self, artifact: pathlib.Path) -> None:
245 other = make_report(artifact, source_revision=OTHER_REVISION)
246 with pytest.raises(GateError, match="checkout is now at"):
247 verify_report(other, artifact, now=NOW, checkout=CHECKOUT)
248
249 def test_checkout_with_uncommitted_changes_at_verification_is_rejected(self, artifact: pathlib.Path) -> None:
250 with pytest.raises(GateError, match="uncommitted"):
251 verify_report(make_report(artifact), artifact, now=NOW, checkout=CheckoutState(SOURCE_REVISION, False, ("muse/cli.py",)))
252
253 def test_candidate_hash_changed_is_rejected(self, artifact: pathlib.Path) -> None:
254 report = make_report(artifact)
255 artifact.write_bytes(b"different bytes")
256 with pytest.raises(GateError, match="sha256"):
257 verify_report(report, artifact, now=NOW, checkout=CHECKOUT)
258
259 def test_report_for_another_artifact_is_rejected_even_if_the_candidate_hash_is_edited(self, artifact: pathlib.Path, tmp_path: pathlib.Path) -> None:
260 (tmp_path / "o").mkdir()
261 other = make_artifact(tmp_path / "o", "0.2.1rc19", payload=b"another artifact, same version")
262 forged = mutate_report(make_report(other), candidate__sha256=json.loads(make_report(artifact))["candidate"]["sha256"])
263 with pytest.raises(GateError, match="different artifact"):
264 verify_report(forged, artifact, now=NOW, checkout=CHECKOUT)
265
266 def test_report_without_provenance_is_rejected(self, artifact: pathlib.Path) -> None:
267 data = json.loads(make_report(artifact))
268 del data["provenance"]
269 with pytest.raises(GateError, match="provenance is required"):
270 verify_report(json.dumps(data), artifact, now=NOW, checkout=CHECKOUT)
271
272 def test_schema_1_report_with_only_an_operator_typed_revision_is_refused(self, artifact: pathlib.Path) -> None:
273 data = json.loads(make_report(artifact))
274 data["schema"] = 1
275 del data["provenance"]
276 with pytest.raises(GateError, match="schema"):
277 verify_report(json.dumps(data), artifact, now=NOW, checkout=CHECKOUT)
278
279 @pytest.mark.parametrize(
280 "change, match",
281 [
282 ({"provenance__claimed_revision": OTHER_REVISION}, "claimed"),
283 ({"provenance__observed_revision_before": OTHER_REVISION}, "claimed"),
284 ({"provenance__observed_revision_after": OTHER_REVISION}, "moved during"),
285 ({"provenance__tree_clean": False}, "uncommitted"),
286 ({"provenance__tree_clean": "yes"}, "boolean"),
287 ({"provenance__claimed_revision": "main"}, "invalid"),
288 ({"provenance__artifact_sha256": "0" * 64}, "different artifact"),
289 ({"provenance__artifact_sha256": "zz"}, "invalid"),
290 ({"candidate__source_revision": OTHER_REVISION}, "not the observed"),
291 ],
292 )
293 def test_inconsistent_or_forged_provenance_is_rejected(self, artifact: pathlib.Path, change: dict[str, object], match: str) -> None:
294 with pytest.raises(GateError, match=match):
295 verify_report(mutate_report(make_report(artifact), **change), artifact, now=NOW, checkout=CHECKOUT)
296
297 @pytest.mark.parametrize("raw", [None, [], "clean", 1])
298 def test_non_object_provenance_is_rejected(self, artifact: pathlib.Path, raw: object) -> None:
299 with pytest.raises(GateError, match="provenance"):
300 verify_report(mutate_report(make_report(artifact), provenance=raw), artifact, now=NOW, checkout=CHECKOUT)
301
302
303 # ── publication is bound to the live checkout ─────────────────────────────────
304
305
306 class TestPublishRechecksTheCheckout:
307 def test_publish_refuses_when_the_checkout_moved_since_the_gate(self, artifact: pathlib.Path) -> None:
308 storage = FakeStorage()
309 with pytest.raises(GateError, match="checkout is now at"):
310 publish(storage, artifact, make_report(artifact), now=NOW, checkout=CheckoutState(OTHER_REVISION, True, ()))
311 assert storage.ops == []
312
313 def test_publish_refuses_when_the_checkout_is_dirty(self, artifact: pathlib.Path) -> None:
314 storage = FakeStorage()
315 with pytest.raises(GateError, match="uncommitted"):
316 publish(storage, artifact, make_report(artifact), now=NOW, checkout=CheckoutState(SOURCE_REVISION, False, ("a.py",)))
317 assert storage.ops == []
318
319 def test_the_stored_record_carries_the_observed_revision(self, artifact: pathlib.Path) -> None:
320 storage = FakeStorage()
321 publish(storage, artifact, make_report(artifact), now=NOW, checkout=CHECKOUT)
322 assert json.loads(storage.objects["muse-0.2.1rc19.release.json"])["source_revision"] == SOURCE_REVISION
323
324
325 def test_module_is_stdlib_only_and_never_shells_out_to_git() -> None:
326 src = (pathlib.Path(__file__).resolve().parents[1] / "deploy" / "release_provenance.py").read_text()
327 code = "\n".join(line for line in src.splitlines() if not line.strip().startswith("#"))
328 assert '"git"' not in code and "'git'" not in code and sys.version_info >= (3, 11)
File History 2 commits
sha256:a977b39d08e15683210a33608e25712fe3ed9c8f05f0388fb69ef38acf835426 feat(release): hermetic release gates in a provisioned, pin… Opus 5.5 minor ⚠ 12 days ago
sha256:04db68e43d9007348dbd726b2a3e52f73e83c5646a4511c07e19cb9a66e28b5f fix(release): gate provenance is observed from the checkout… Opus 5.5 minor ⚠ 12 days ago