gabriel / musehub public
gate_env.py python
332 lines 15.3 KB
Raw
sha256:a977b39d08e15683210a33608e25712fe3ed9c8f05f0388fb69ef38acf835426 feat(release): hermetic release gates in a provisioned, pin… Opus 5.5 minor ⚠ breaking 12 days ago
1 """Hermetic environment for the release gates (musehub#114 Phase 6D, D6).
2
3 The gates used to run under whatever Python invoked them: a global interpreter without the tree-sitter
4 grammars failed the same rc19 artifact that passed in a hand-built venv. A gate result must not depend on
5 the operator's environment, so ``run`` now provisions its own:
6
7 1. an **explicit** interpreter (``--python``) that is CPython and satisfies the artifact's own
8 ``Requires-Python``;
9 2. a **fresh** venv in a new temporary directory (never reused; a half-built one carries no marker and is
10 refused), created with the user site and ``PYTHONPATH`` ignored;
11 3. the pinned tooling in ``deploy/gate_requirements.txt`` (exact ``==`` pins: pytest, build, the MuseHub
12 runtime the installer harness imports, ...);
13 4. **the exact candidate artifact** with its ``[all]`` extra, copied into the environment first and
14 hash-checked, so the grammar packages are those the artifact's own metadata resolves;
15 5. a verification pass before any gate runs: every pin installed at its pinned version, the installed
16 ``muse`` is the candidate (version, location inside the venv, archive hash when pip recorded one) and
17 every grammar module the installed ``muse`` declares imports.
18
19 The environment is removed afterwards even on failure or interruption. The global interpreter is only ever
20 asked to run ``-m venv``; every ``pip`` call is made by the venv's own interpreter with
21 ``PIP_REQUIRE_VIRTUALENV=1``.
22
23 Two kinds of Python then meet in the gates, deliberately kept apart:
24
25 * the **gate venv** (this module) holds test tooling plus the installed candidate and its grammars;
26 * the **muse checkout** is added to ``PYTHONPATH`` for exactly one gate, ``runtime.installed_parser_parity``,
27 because that gate compares the *development* parser (the checkout) with the *installed* candidate. Every
28 other gate must not see the checkout on its import path.
29
30 The installer E2E gates build their own throw-away venvs from the candidate sdist through the real
31 installer; they use this venv only as the interpreter that runs pytest.
32 """
33 from __future__ import annotations
34
35 import hashlib
36 import json
37 import os
38 import re
39 import shutil
40 import subprocess
41 import tarfile
42 import tempfile
43 from collections.abc import Callable, Iterator, Mapping
44 from contextlib import contextmanager
45 from dataclasses import dataclass, field
46 from pathlib import Path
47 from typing import Any
48
49 from packaging.requirements import InvalidRequirement, Requirement
50 from packaging.specifiers import InvalidSpecifier, SpecifierSet
51 from packaging.utils import canonicalize_name
52 from packaging.version import Version
53
54 MARKER = "gate-env.json"
55 REQUIREMENTS = Path(__file__).resolve().parent / "gate_requirements.txt"
56
57 Runner = Callable[..., "subprocess.CompletedProcess[str]"]
58
59 # Inherited variables that would let the caller's environment leak into the gates.
60 _SCRUBBED_PREFIXES = ("PIP_", "PYTHON", "VIRTUAL_ENV", "CONDA", "UV_")
61
62
63 class GateEnvError(Exception):
64 """The hermetic environment could not be built or verified. Always blocks the gate run."""
65
66
67 @dataclass(frozen=True)
68 class Interpreter:
69 path: Path
70 version: str
71 implementation: str
72
73
74 @dataclass(frozen=True)
75 class GateEnv:
76 root: Path
77 python: Path
78 interpreter: Interpreter
79 packages: Mapping[str, str]
80 requirements_sha256: str
81 candidate_sha256: str
82 cleanup: str = "pending"
83 extra: Mapping[str, Any] = field(default_factory=dict)
84
85 def evidence(self) -> dict[str, Any]:
86 listing = "\n".join(f"{n}=={v}" for n, v in sorted(self.packages.items()))
87 return {
88 "interpreter": {"path": str(self.interpreter.path), "version": self.interpreter.version, "implementation": self.interpreter.implementation},
89 "isolated": True,
90 "requirements_sha256": self.requirements_sha256,
91 "candidate_sha256": self.candidate_sha256,
92 "packages": dict(sorted(self.packages.items())),
93 "packages_sha256": hashlib.sha256(listing.encode()).hexdigest(),
94 "cleanup": self.cleanup,
95 }
96
97
98 # ── interpreter selection ─────────────────────────────────────────────────────
99
100 _PROBE = "import json,sys;print(json.dumps({'v':'.'.join(map(str,sys.version_info[:3])),'impl':sys.implementation.name,'exe':sys.executable}))"
101
102
103 def _run(runner: Runner, argv: list[str], what: str, *, env: Mapping[str, str] | None = None, timeout: float = 1800) -> str:
104 try:
105 proc = runner(argv, capture_output=True, text=True, timeout=timeout, check=False, env=dict(env) if env is not None else None)
106 except subprocess.TimeoutExpired as exc:
107 raise GateEnvError(f"{what}: timed out after {timeout}s") from exc
108 except OSError as exc:
109 raise GateEnvError(f"{what}: could not run {argv[0]}: {exc}") from exc
110 if proc.returncode != 0:
111 tail = ((proc.stderr or "") + (proc.stdout or "")).strip()[-600:]
112 raise GateEnvError(f"{what}: exit {proc.returncode}: {tail}")
113 return proc.stdout or ""
114
115
116 def requires_python_of(artifact: Path) -> str:
117 """The artifact's own ``Requires-Python`` -- the supported-interpreter contract is the candidate's, not ours."""
118 version = artifact.name.removeprefix("muse-").removesuffix(".tar.gz")
119 try:
120 with tarfile.open(artifact) as tf:
121 member = tf.extractfile(f"muse-{version}/PKG-INFO")
122 if member is None:
123 raise GateEnvError(f"{artifact.name} has no PKG-INFO")
124 text = member.read().decode()
125 except (OSError, tarfile.TarError, KeyError) as exc:
126 raise GateEnvError(f"cannot read PKG-INFO from {artifact}: {exc}") from exc
127 m = re.search(r"^Requires-Python:\s*(.+)$", text, re.M)
128 if not m:
129 raise GateEnvError(f"{artifact.name} declares no Requires-Python")
130 try:
131 SpecifierSet(m.group(1).strip())
132 except InvalidSpecifier as exc:
133 raise GateEnvError(f"{artifact.name} has an invalid Requires-Python {m.group(1)!r}") from exc
134 return m.group(1).strip()
135
136
137 def select_interpreter(python: Path | str | None, requires_python: str, *, runner: Runner = subprocess.run) -> Interpreter:
138 """Validate the explicit interpreter. No PATH search, no ``sys.executable`` fallback."""
139 if not python:
140 raise GateEnvError("an explicit Python interpreter is required (--python); the gate never uses the invoking environment")
141 path = Path(python)
142 if not path.is_absolute() or not path.is_file() or not os.access(path, os.X_OK):
143 raise GateEnvError(f"interpreter {python!s} is not an absolute path to an executable file")
144 out = _run(runner, [str(path), "-I", "-c", _PROBE], f"probing interpreter {path}", timeout=60)
145 try:
146 info = json.loads(out)
147 version, impl = str(info["v"]), str(info["impl"])
148 Version(version)
149 except (ValueError, KeyError, TypeError) as exc:
150 raise GateEnvError(f"interpreter {path} returned an unreadable version probe: {out[:200]!r}") from exc
151 if impl != "cpython":
152 raise GateEnvError(f"interpreter {path} is {impl}; only CPython is supported")
153 if not SpecifierSet(requires_python).contains(Version(version), prereleases=True):
154 raise GateEnvError(f"interpreter {path} is Python {version}, but the candidate requires {requires_python}")
155 return Interpreter(path, version, impl)
156
157
158 # ── pins ──────────────────────────────────────────────────────────────────────
159
160
161 def parse_requirements(text: str) -> dict[str, str]:
162 """name -> exact version. Anything not ``name==version`` is rejected: the gate tooling is pinned or it is not used."""
163 pins: dict[str, str] = {}
164 for raw in text.splitlines():
165 line = raw.split("#", 1)[0].strip()
166 if not line:
167 continue
168 try:
169 req = Requirement(line)
170 except InvalidRequirement as exc:
171 raise GateEnvError(f"gate requirements: invalid line {line!r}: {exc}") from exc
172 specs = list(req.specifier)
173 if len(specs) != 1 or specs[0].operator != "==" or specs[0].version.endswith(".*"):
174 raise GateEnvError(f"gate requirements: {line!r} is not an exact == pin")
175 name = canonicalize_name(req.name)
176 if name in pins:
177 raise GateEnvError(f"gate requirements: duplicate pin for {name}")
178 pins[name] = specs[0].version
179 if not pins:
180 raise GateEnvError("gate requirements are empty")
181 return pins
182
183
184 # ── building ──────────────────────────────────────────────────────────────────
185
186
187 def _clean_env(venv: Path | None) -> dict[str, str]:
188 env = {k: v for k, v in os.environ.items() if not k.startswith(_SCRUBBED_PREFIXES)}
189 env.update(PYTHONNOUSERSITE="1", PIP_REQUIRE_VIRTUALENV="1", PIP_DISABLE_PIP_VERSION_CHECK="1", PIP_NO_INPUT="1")
190 if venv is not None:
191 env["VIRTUAL_ENV"] = str(venv)
192 env["PATH"] = f"{venv / 'bin'}{os.pathsep}{env.get('PATH', '')}"
193 return env
194
195
196 def gate_process_env(env: GateEnv) -> dict[str, str]:
197 """The environment every gate subprocess gets: scrubbed of the caller's Python settings, venv first on PATH."""
198 return _clean_env(env.root / "venv")
199
200
201 _VERIFY = r'''
202 import importlib, importlib.metadata as md, json, pathlib, sys
203 from packaging.utils import canonicalize_name
204 pins, version, root = json.loads(sys.argv[1]), sys.argv[2], pathlib.Path(sys.argv[3]).resolve()
205 installed = {canonicalize_name(d.metadata["Name"]): d.version for d in md.distributions() if d.metadata["Name"]}
206 problems = []
207 for name, want in pins.items():
208 have = installed.get(name)
209 if have is None:
210 problems.append(f"missing {name}=={want}")
211 elif have != want:
212 problems.append(f"{name} is {have}, pinned {want}")
213 dist = md.distribution("muse")
214 if dist.version != version:
215 problems.append(f"installed muse is {dist.version}, candidate is {version}")
216 import muse
217 loc = pathlib.Path(muse.__file__).resolve()
218 if root not in loc.parents:
219 problems.append(f"muse imported from {loc}, outside the gate venv")
220 direct = dist.read_text("direct_url.json")
221 archive_hash = None
222 if direct:
223 archive_hash = json.loads(direct).get("archive_info", {}).get("hashes", {}).get("sha256")
224 from muse.plugins.code import ast_parser as a
225 # Only what the full profile guarantees; opt-in grammars (Swift) are deliberately not part of [all].
226 mods = sorted({s["module_name"] for s in a._TS_LANG_SPECS if s.get("profile") == "full"} | {"tree_sitter_markdown", "tree_sitter_html"})
227 for m in mods:
228 try:
229 importlib.import_module(m)
230 except Exception as exc: # noqa: BLE001
231 problems.append(f"grammar {m}: {type(exc).__name__}: {exc}")
232 print(json.dumps({"problems": problems, "installed": installed, "archive_sha256": archive_hash, "grammar_modules": mods}))
233 '''
234
235
236 def _sha(path: Path) -> str:
237 h = hashlib.sha256()
238 with path.open("rb") as fh:
239 for chunk in iter(lambda: fh.read(1 << 20), b""):
240 h.update(chunk)
241 return h.hexdigest()
242
243
244 def build_env(
245 root: Path,
246 artifact: Path,
247 interpreter: Interpreter,
248 *,
249 requirements: Path = REQUIREMENTS,
250 runner: Runner = subprocess.run,
251 ) -> GateEnv:
252 """Create the environment under *root*, which must not exist. Raises ``GateEnvError``; the caller removes *root*."""
253 if root.exists():
254 raise GateEnvError(f"{root} already exists; gate environments are never reused")
255 try:
256 want_sha = _sha(artifact)
257 req_text = requirements.read_text()
258 except OSError as exc:
259 raise GateEnvError(f"cannot read gate inputs: {exc}") from exc
260 pins = parse_requirements(req_text)
261 version = artifact.name.removeprefix("muse-").removesuffix(".tar.gz")
262
263 root.mkdir(parents=True)
264 venv = root / "venv"
265 py = venv / "bin" / "python"
266 base_env = _clean_env(None)
267 _run(runner, [str(interpreter.path), "-I", "-m", "venv", str(venv)], "creating the venv", env=base_env, timeout=300)
268 if not py.is_file():
269 raise GateEnvError(f"venv creation produced no interpreter at {py}")
270 env = _clean_env(venv)
271
272 copy = root / "candidate" / artifact.name
273 copy.parent.mkdir()
274 shutil.copyfile(artifact, copy)
275 if _sha(copy) != want_sha or _sha(artifact) != want_sha:
276 raise GateEnvError("the candidate artifact changed while the environment was being built")
277
278 pip = [str(py), "-m", "pip", "install", "--no-input", "--disable-pip-version-check"]
279 _run(runner, [*pip, "-r", str(requirements)], "installing the pinned gate tooling", env=env)
280 _run(runner, [*pip, "-c", str(requirements), f"{copy}[all]"], "installing the candidate artifact and its [all] grammars", env=env)
281
282 out = _run(runner, [str(py), "-I", "-c", _VERIFY, json.dumps(pins), version, str(venv)], "verifying the installed environment", env=env, timeout=300)
283 try:
284 report = json.loads(out.strip().splitlines()[-1])
285 problems: list[str] = list(report["problems"])
286 installed: dict[str, str] = dict(report["installed"])
287 archive = report.get("archive_sha256")
288 except (ValueError, KeyError, IndexError, TypeError) as exc:
289 raise GateEnvError(f"environment verification was unreadable: {out[-300:]!r}") from exc
290 if archive is not None and archive != want_sha:
291 problems.append(f"pip installed an archive with sha256 {archive}, not the candidate {want_sha}")
292 if problems:
293 raise GateEnvError("environment verification failed: " + "; ".join(problems))
294
295 result = GateEnv(root, py, interpreter, installed, hashlib.sha256(req_text.encode()).hexdigest(), want_sha)
296 (root / MARKER).write_text(json.dumps({"candidate_sha256": want_sha, "version": version, "python": interpreter.version})) # last: absence == partial
297 return result
298
299
300 def verify_complete(env: GateEnv, artifact: Path) -> None:
301 """Refuse an environment that is partial, for another artifact, or whose artifact has changed since it was built."""
302 try:
303 marker = json.loads((env.root / MARKER).read_text())
304 except (OSError, ValueError) as exc:
305 raise GateEnvError(f"{env.root} is not a complete gate environment (no valid {MARKER})") from exc
306 if marker.get("candidate_sha256") != _sha(artifact):
307 raise GateEnvError("gate environment was built for a different artifact than the one being gated")
308 if not env.python.is_file():
309 raise GateEnvError("gate environment has no interpreter")
310
311
312 @contextmanager
313 def gate_environment(
314 artifact: Path,
315 python: Path | str | None,
316 *,
317 parent: Path | None = None,
318 requirements: Path = REQUIREMENTS,
319 runner: Runner = subprocess.run,
320 build: Callable[..., GateEnv] = build_env,
321 ) -> Iterator[GateEnv]:
322 """A verified, fresh environment for *artifact*, removed on exit -- success, failure or interruption."""
323 interpreter = select_interpreter(python, requires_python_of(artifact), runner=runner)
324 workdir = Path(tempfile.mkdtemp(prefix="muse-gate-env-", dir=parent))
325 try:
326 env = build(workdir / "env", artifact, interpreter, requirements=requirements, runner=runner)
327 verify_complete(env, artifact)
328 yield env
329 finally:
330 shutil.rmtree(workdir, ignore_errors=True)
331 if workdir.exists():
332 raise GateEnvError(f"could not remove the gate environment {workdir}")
File History 1 commit
sha256:a977b39d08e15683210a33608e25712fe3ed9c8f05f0388fb69ef38acf835426 feat(release): hermetic release gates in a provisioned, pin… Opus 5.5 minor ⚠ 12 days ago