test_release_provenance.py
python
sha256:a977b39d08e15683210a33608e25712fe3ed9c8f05f0388fb69ef38acf835426
feat(release): hermetic release gates in a provisioned, pin…
Opus 5.5
minor
⚠ breaking
12 days ago
| 1 | """Source provenance of a gate report (musehub#114 Phase 6D, D2). |
| 2 | |
| 3 | The revision in a report must be *observed* from the checkout with Muse, never trusted from the |
| 4 | operator, and the report must be unusable for any other revision, tree state or artifact. |
| 5 | """ |
| 6 | |
| 7 | from __future__ import annotations |
| 8 | |
| 9 | import json |
| 10 | import pathlib |
| 11 | import subprocess |
| 12 | import sys |
| 13 | from collections.abc import Callable, Iterator |
| 14 | from contextlib import contextmanager |
| 15 | from typing import Any |
| 16 | |
| 17 | import pytest |
| 18 | |
| 19 | from deploy.gate_env import GateEnv, Interpreter |
| 20 | from deploy.release_gate import REQUIRED_GATES, GateError, GateResult, run_release_gates, verify_report |
| 21 | from deploy.release_provenance import CheckoutState, ProvenanceError, observe_checkout |
| 22 | from deploy.release_safety import publish |
| 23 | from tests.release_fixtures import ( |
| 24 | CHECKOUT, |
| 25 | NOW, |
| 26 | OTHER_REVISION, |
| 27 | SOURCE_REVISION, |
| 28 | FakeStorage, |
| 29 | clean_status_json, |
| 30 | make_artifact, |
| 31 | make_report, |
| 32 | mutate_report, |
| 33 | passing_results, |
| 34 | sha256_of, |
| 35 | ) |
| 36 | |
| 37 | |
| 38 | @pytest.fixture |
| 39 | def artifact(tmp_path: pathlib.Path) -> pathlib.Path: |
| 40 | return make_artifact(tmp_path, "0.2.1rc19") |
| 41 | |
| 42 | |
| 43 | # ── observing the checkout with Muse ────────────────────────────────────────── |
| 44 | |
| 45 | |
| 46 | def fake_muse(rev: object = SOURCE_REVISION, status: str | None = None, code: int = 0, raises: Exception | None = None) -> Callable[..., Any]: |
| 47 | def runner(argv: list[str], **_kw: object) -> subprocess.CompletedProcess[str]: |
| 48 | if raises: |
| 49 | raise raises |
| 50 | if "rev-parse" in argv: |
| 51 | out = json.dumps({"commit_id": rev}) |
| 52 | else: |
| 53 | out = status if status is not None else clean_status_json(SOURCE_REVISION) |
| 54 | return subprocess.CompletedProcess(argv, code, out, "boom" if code else "") |
| 55 | |
| 56 | return runner |
| 57 | |
| 58 | |
| 59 | class TestObserveCheckout: |
| 60 | def test_clean_checkout_reports_its_revision(self, tmp_path: pathlib.Path) -> None: |
| 61 | assert observe_checkout(tmp_path, runner=fake_muse()) == CheckoutState(SOURCE_REVISION, True, ()) |
| 62 | |
| 63 | def test_uses_muse_not_git(self, tmp_path: pathlib.Path) -> None: |
| 64 | seen: list[list[str]] = [] |
| 65 | |
| 66 | def spy(argv: list[str], **kw: object) -> subprocess.CompletedProcess[str]: |
| 67 | seen.append(argv) |
| 68 | return fake_muse()(argv, **kw) |
| 69 | |
| 70 | observe_checkout(tmp_path, runner=spy) |
| 71 | assert seen and all(a[0] == "muse" and a[1] == "-C" for a in seen) |
| 72 | assert not any("git" in part for argv in seen for part in argv[:1]) |
| 73 | |
| 74 | @pytest.mark.parametrize( |
| 75 | "override", |
| 76 | [ |
| 77 | {"modified": ["muse/cli.py"], "clean": False, "dirty": True}, |
| 78 | {"added": ["new.py"], "clean": False, "dirty": True}, |
| 79 | {"deleted": ["gone.py"], "clean": False, "dirty": True}, |
| 80 | {"untracked": ["stray.py"]}, |
| 81 | {"renamed": {"a.py": "b.py"}}, |
| 82 | {"conflict_paths": ["c.py"]}, |
| 83 | {"staged": {"added": [], "modified": ["x.py"], "deleted": [], "renamed": {}}}, |
| 84 | {"merge_in_progress": True}, |
| 85 | {"checkout_interrupted": True}, |
| 86 | {"clean": False, "dirty": True}, |
| 87 | ], |
| 88 | ) |
| 89 | def test_any_drift_marks_the_tree_dirty(self, tmp_path: pathlib.Path, override: dict[str, object]) -> None: |
| 90 | state = observe_checkout(tmp_path, runner=fake_muse(status=clean_status_json(SOURCE_REVISION, **override))) |
| 91 | assert state.revision == SOURCE_REVISION and state.clean is False and state.dirty |
| 92 | |
| 93 | def test_muse_missing_is_an_error_not_an_unknown(self, tmp_path: pathlib.Path) -> None: |
| 94 | with pytest.raises(ProvenanceError, match="could not run"): |
| 95 | observe_checkout(tmp_path, runner=fake_muse(raises=FileNotFoundError("muse"))) |
| 96 | |
| 97 | def test_muse_failure_is_an_error(self, tmp_path: pathlib.Path) -> None: |
| 98 | with pytest.raises(ProvenanceError, match="exited 2"): |
| 99 | observe_checkout(tmp_path, runner=fake_muse(code=2)) |
| 100 | |
| 101 | def test_timeout_is_an_error(self, tmp_path: pathlib.Path) -> None: |
| 102 | with pytest.raises(ProvenanceError): |
| 103 | observe_checkout(tmp_path, runner=fake_muse(raises=subprocess.TimeoutExpired("muse", 60))) |
| 104 | |
| 105 | @pytest.mark.parametrize("rev", ["", None, "abc", "sha256:xyz", "sha256:" + "A" * 64, "g" * 64, 5]) |
| 106 | def test_malformed_revision_is_an_error(self, tmp_path: pathlib.Path, rev: object) -> None: |
| 107 | with pytest.raises(ProvenanceError): |
| 108 | observe_checkout(tmp_path, runner=fake_muse(rev=rev)) |
| 109 | |
| 110 | @pytest.mark.parametrize("status", ["not json", "[]", "{}", json.dumps({"head_commit": SOURCE_REVISION})]) |
| 111 | def test_unparseable_or_incomplete_status_is_an_error(self, tmp_path: pathlib.Path, status: str) -> None: |
| 112 | with pytest.raises(ProvenanceError): |
| 113 | observe_checkout(tmp_path, runner=fake_muse(status=status)) |
| 114 | |
| 115 | def test_rev_parse_and_status_disagreeing_is_an_error(self, tmp_path: pathlib.Path) -> None: |
| 116 | with pytest.raises(ProvenanceError, match="disagree"): |
| 117 | observe_checkout(tmp_path, runner=fake_muse(rev=OTHER_REVISION)) |
| 118 | |
| 119 | |
| 120 | # ── the gate run observes, it does not trust ────────────────────────────────── |
| 121 | |
| 122 | |
| 123 | class Observer: |
| 124 | """Returns the given states in order (the last repeats).""" |
| 125 | |
| 126 | def __init__(self, *states: CheckoutState | Exception) -> None: |
| 127 | self.states, self.calls = list(states), 0 |
| 128 | |
| 129 | def __call__(self, _repo: pathlib.Path) -> CheckoutState: |
| 130 | state = self.states[min(self.calls, len(self.states) - 1)] |
| 131 | self.calls += 1 |
| 132 | if isinstance(state, Exception): |
| 133 | raise state |
| 134 | return state |
| 135 | |
| 136 | |
| 137 | @contextmanager |
| 138 | def fake_environment(artifact: pathlib.Path, python: object) -> Iterator[GateEnv]: |
| 139 | """Stands in for deploy.gate_env.gate_environment; its own behaviour is tested in test_release_gate_env.py.""" |
| 140 | sha = sha256_of(artifact) |
| 141 | version = artifact.name.removeprefix("muse-").removesuffix(".tar.gz") |
| 142 | yield GateEnv( |
| 143 | pathlib.Path("/nonexistent/env"), pathlib.Path("/nonexistent/env/venv/bin/python"), |
| 144 | Interpreter(pathlib.Path("/opt/python/bin/python3.14"), "3.14.4", "cpython"), |
| 145 | {"muse": version, "pytest": "9.0.3"}, "1" * 64, sha, |
| 146 | ) |
| 147 | |
| 148 | |
| 149 | class Spy: |
| 150 | def __init__(self, on_run: Callable[[], None] | None = None) -> None: |
| 151 | self.ran, self.on_run = False, on_run |
| 152 | |
| 153 | def __call__(self, commands: object, **_kw: object) -> dict[str, GateResult]: |
| 154 | self.ran = True |
| 155 | if self.on_run: |
| 156 | self.on_run() |
| 157 | return passing_results() |
| 158 | |
| 159 | |
| 160 | def gate(artifact: pathlib.Path, observer: Observer, spy: Spy, claimed: str = SOURCE_REVISION) -> dict[str, Any]: |
| 161 | return run_release_gates( |
| 162 | artifact, artifact.parent, artifact.parent, claimed, 2, python="/py", now=NOW, observe=observer, run=spy, |
| 163 | environment=fake_environment, |
| 164 | ) |
| 165 | |
| 166 | |
| 167 | class TestGateRunObservesTheCheckout: |
| 168 | def test_report_records_claimed_and_observed_revisions_and_binds_the_artifact(self, artifact: pathlib.Path) -> None: |
| 169 | spy = Spy() |
| 170 | report = gate(artifact, Observer(CHECKOUT), spy) |
| 171 | prov = report["provenance"] |
| 172 | assert spy.ran and report["decision"] == "pass" |
| 173 | assert prov["claimed_revision"] == prov["observed_revision_before"] == prov["observed_revision_after"] == SOURCE_REVISION |
| 174 | assert prov["tree_clean"] is True and prov["artifact_sha256"] == report["candidate"]["sha256"] |
| 175 | assert "unattested" in prov["build_relation"], "the build link must not be overstated" |
| 176 | verify_report(json.dumps(report), artifact, now=NOW, checkout=CHECKOUT) |
| 177 | |
| 178 | def test_operator_supplied_wrong_revision_is_refused_before_any_gate_runs(self, artifact: pathlib.Path) -> None: |
| 179 | spy = Spy() |
| 180 | with pytest.raises(GateError, match="claimed revision"): |
| 181 | gate(artifact, Observer(CHECKOUT), spy, claimed=OTHER_REVISION) |
| 182 | assert not spy.ran |
| 183 | |
| 184 | def test_malformed_claimed_revision_is_refused(self, artifact: pathlib.Path) -> None: |
| 185 | spy = Spy() |
| 186 | with pytest.raises(GateError, match="source provenance"): |
| 187 | gate(artifact, Observer(CHECKOUT), spy, claimed="HEAD") |
| 188 | assert not spy.ran |
| 189 | |
| 190 | def test_undeterminable_revision_is_refused(self, artifact: pathlib.Path) -> None: |
| 191 | spy = Spy() |
| 192 | with pytest.raises(GateError, match="source provenance"): |
| 193 | gate(artifact, Observer(ProvenanceError("muse not found")), spy) |
| 194 | assert not spy.ran |
| 195 | |
| 196 | def test_uncommitted_changes_refuse_the_run(self, artifact: pathlib.Path) -> None: |
| 197 | spy = Spy() |
| 198 | with pytest.raises(GateError, match="uncommitted"): |
| 199 | gate(artifact, Observer(CheckoutState(SOURCE_REVISION, False, ("muse/cli.py",))), spy) |
| 200 | assert not spy.ran |
| 201 | |
| 202 | def test_checkout_moving_during_the_run_voids_the_evidence(self, artifact: pathlib.Path) -> None: |
| 203 | with pytest.raises(GateError, match="changed while the gates ran"): |
| 204 | gate(artifact, Observer(CHECKOUT, CheckoutState(OTHER_REVISION, True, ())), Spy()) |
| 205 | |
| 206 | def test_checkout_dirtied_during_the_run_voids_the_evidence(self, artifact: pathlib.Path) -> None: |
| 207 | with pytest.raises(GateError, match="changed while the gates ran"): |
| 208 | gate(artifact, Observer(CHECKOUT, CheckoutState(SOURCE_REVISION, False, ("x.py",))), Spy()) |
| 209 | |
| 210 | def test_unobservable_after_the_run_voids_the_evidence(self, artifact: pathlib.Path) -> None: |
| 211 | with pytest.raises(GateError, match="after the run"): |
| 212 | gate(artifact, Observer(CHECKOUT, ProvenanceError("gone")), Spy()) |
| 213 | |
| 214 | def test_artifact_swapped_during_the_run_voids_the_evidence(self, artifact: pathlib.Path) -> None: |
| 215 | spy = Spy(on_run=lambda: artifact.write_bytes(b"rebuilt mid-run")) |
| 216 | with pytest.raises(GateError, match="artifact changed"): |
| 217 | gate(artifact, Observer(CHECKOUT), spy) |
| 218 | |
| 219 | def test_a_candidate_not_built_from_the_tested_checkout_fails_the_reproducibility_gate(self, artifact: pathlib.Path) -> None: |
| 220 | """The only build evidence is content-equivalence of a fresh sdist of the observed checkout. |
| 221 | When that gate fails, the report is a fail and can never authorize publication.""" |
| 222 | |
| 223 | class FailingBuild(Spy): |
| 224 | def __call__(self, commands: object, **_kw: object) -> dict[str, GateResult]: |
| 225 | results = passing_results() |
| 226 | results["packaging.sdist_reproducible"] = GateResult("fail", 1, "differing members: ['muse/cli.py']") |
| 227 | return results |
| 228 | |
| 229 | report = gate(artifact, Observer(CHECKOUT), FailingBuild()) |
| 230 | assert report["decision"] == "fail" |
| 231 | with pytest.raises(GateError, match="packaging.sdist_reproducible"): |
| 232 | verify_report(json.dumps(report), artifact, now=NOW, checkout=CHECKOUT) |
| 233 | assert "packaging.sdist_reproducible" in REQUIRED_GATES |
| 234 | |
| 235 | |
| 236 | # ── verification re-checks provenance ───────────────────────────────────────── |
| 237 | |
| 238 | |
| 239 | class TestVerifyReportProvenance: |
| 240 | def test_checkout_changed_after_the_gate_ran_is_rejected(self, artifact: pathlib.Path) -> None: |
| 241 | with pytest.raises(GateError, match="checkout is now at"): |
| 242 | verify_report(make_report(artifact), artifact, now=NOW, checkout=CheckoutState(OTHER_REVISION, True, ())) |
| 243 | |
| 244 | def test_report_replayed_against_another_revision_is_rejected(self, artifact: pathlib.Path) -> None: |
| 245 | other = make_report(artifact, source_revision=OTHER_REVISION) |
| 246 | with pytest.raises(GateError, match="checkout is now at"): |
| 247 | verify_report(other, artifact, now=NOW, checkout=CHECKOUT) |
| 248 | |
| 249 | def test_checkout_with_uncommitted_changes_at_verification_is_rejected(self, artifact: pathlib.Path) -> None: |
| 250 | with pytest.raises(GateError, match="uncommitted"): |
| 251 | verify_report(make_report(artifact), artifact, now=NOW, checkout=CheckoutState(SOURCE_REVISION, False, ("muse/cli.py",))) |
| 252 | |
| 253 | def test_candidate_hash_changed_is_rejected(self, artifact: pathlib.Path) -> None: |
| 254 | report = make_report(artifact) |
| 255 | artifact.write_bytes(b"different bytes") |
| 256 | with pytest.raises(GateError, match="sha256"): |
| 257 | verify_report(report, artifact, now=NOW, checkout=CHECKOUT) |
| 258 | |
| 259 | def test_report_for_another_artifact_is_rejected_even_if_the_candidate_hash_is_edited(self, artifact: pathlib.Path, tmp_path: pathlib.Path) -> None: |
| 260 | (tmp_path / "o").mkdir() |
| 261 | other = make_artifact(tmp_path / "o", "0.2.1rc19", payload=b"another artifact, same version") |
| 262 | forged = mutate_report(make_report(other), candidate__sha256=json.loads(make_report(artifact))["candidate"]["sha256"]) |
| 263 | with pytest.raises(GateError, match="different artifact"): |
| 264 | verify_report(forged, artifact, now=NOW, checkout=CHECKOUT) |
| 265 | |
| 266 | def test_report_without_provenance_is_rejected(self, artifact: pathlib.Path) -> None: |
| 267 | data = json.loads(make_report(artifact)) |
| 268 | del data["provenance"] |
| 269 | with pytest.raises(GateError, match="provenance is required"): |
| 270 | verify_report(json.dumps(data), artifact, now=NOW, checkout=CHECKOUT) |
| 271 | |
| 272 | def test_schema_1_report_with_only_an_operator_typed_revision_is_refused(self, artifact: pathlib.Path) -> None: |
| 273 | data = json.loads(make_report(artifact)) |
| 274 | data["schema"] = 1 |
| 275 | del data["provenance"] |
| 276 | with pytest.raises(GateError, match="schema"): |
| 277 | verify_report(json.dumps(data), artifact, now=NOW, checkout=CHECKOUT) |
| 278 | |
| 279 | @pytest.mark.parametrize( |
| 280 | "change, match", |
| 281 | [ |
| 282 | ({"provenance__claimed_revision": OTHER_REVISION}, "claimed"), |
| 283 | ({"provenance__observed_revision_before": OTHER_REVISION}, "claimed"), |
| 284 | ({"provenance__observed_revision_after": OTHER_REVISION}, "moved during"), |
| 285 | ({"provenance__tree_clean": False}, "uncommitted"), |
| 286 | ({"provenance__tree_clean": "yes"}, "boolean"), |
| 287 | ({"provenance__claimed_revision": "main"}, "invalid"), |
| 288 | ({"provenance__artifact_sha256": "0" * 64}, "different artifact"), |
| 289 | ({"provenance__artifact_sha256": "zz"}, "invalid"), |
| 290 | ({"candidate__source_revision": OTHER_REVISION}, "not the observed"), |
| 291 | ], |
| 292 | ) |
| 293 | def test_inconsistent_or_forged_provenance_is_rejected(self, artifact: pathlib.Path, change: dict[str, object], match: str) -> None: |
| 294 | with pytest.raises(GateError, match=match): |
| 295 | verify_report(mutate_report(make_report(artifact), **change), artifact, now=NOW, checkout=CHECKOUT) |
| 296 | |
| 297 | @pytest.mark.parametrize("raw", [None, [], "clean", 1]) |
| 298 | def test_non_object_provenance_is_rejected(self, artifact: pathlib.Path, raw: object) -> None: |
| 299 | with pytest.raises(GateError, match="provenance"): |
| 300 | verify_report(mutate_report(make_report(artifact), provenance=raw), artifact, now=NOW, checkout=CHECKOUT) |
| 301 | |
| 302 | |
| 303 | # ── publication is bound to the live checkout ───────────────────────────────── |
| 304 | |
| 305 | |
| 306 | class TestPublishRechecksTheCheckout: |
| 307 | def test_publish_refuses_when_the_checkout_moved_since_the_gate(self, artifact: pathlib.Path) -> None: |
| 308 | storage = FakeStorage() |
| 309 | with pytest.raises(GateError, match="checkout is now at"): |
| 310 | publish(storage, artifact, make_report(artifact), now=NOW, checkout=CheckoutState(OTHER_REVISION, True, ())) |
| 311 | assert storage.ops == [] |
| 312 | |
| 313 | def test_publish_refuses_when_the_checkout_is_dirty(self, artifact: pathlib.Path) -> None: |
| 314 | storage = FakeStorage() |
| 315 | with pytest.raises(GateError, match="uncommitted"): |
| 316 | publish(storage, artifact, make_report(artifact), now=NOW, checkout=CheckoutState(SOURCE_REVISION, False, ("a.py",))) |
| 317 | assert storage.ops == [] |
| 318 | |
| 319 | def test_the_stored_record_carries_the_observed_revision(self, artifact: pathlib.Path) -> None: |
| 320 | storage = FakeStorage() |
| 321 | publish(storage, artifact, make_report(artifact), now=NOW, checkout=CHECKOUT) |
| 322 | assert json.loads(storage.objects["muse-0.2.1rc19.release.json"])["source_revision"] == SOURCE_REVISION |
| 323 | |
| 324 | |
| 325 | def test_module_is_stdlib_only_and_never_shells_out_to_git() -> None: |
| 326 | src = (pathlib.Path(__file__).resolve().parents[1] / "deploy" / "release_provenance.py").read_text() |
| 327 | code = "\n".join(line for line in src.splitlines() if not line.strip().startswith("#")) |
| 328 | assert '"git"' not in code and "'git'" not in code and sys.version_info >= (3, 11) |
File History
2 commits
sha256:a977b39d08e15683210a33608e25712fe3ed9c8f05f0388fb69ef38acf835426
feat(release): hermetic release gates in a provisioned, pin…
Opus 5.5
minor
⚠
12 days ago
sha256:04db68e43d9007348dbd726b2a3e52f73e83c5646a4511c07e19cb9a66e28b5f
fix(release): gate provenance is observed from the checkout…
Opus 5.5
minor
⚠
12 days ago