gabriel / musehub public
build_context.py python
369 lines 17.3 KB
Raw
sha256:b4b999434d042d74edc7032f1f2cfd20fda5b3580fe85966351db5b8eef0abb3 feat(intel): every grammar on the hub, and every intel job … Opus 5.5 minor ⚠ breaking 1 day ago
1 #!/usr/bin/env python3
2 """Assemble the Docker build context from *committed* files only (musehub#114 Phase 6D).
3
4 ``push.sh`` used to hand Docker the whole ``~/ecosystem`` working tree, so any untracked local file under
5 a directory the Dockerfile copies (for example ``docs/work/session-handoff.md``) ended up in the deployed
6 image, and the image said nothing about which revision it was built from.
7
8 This builds the context instead from ``muse archive`` of each repository the image needs, so untracked
9 and ignored files are not merely filtered out -- they are never present::
10
11 python -m deploy.build_context --ecosystem ~/ecosystem --out /tmp/ctx
12
13 The compiled UI assets the app serves from ``/static`` (``FRONTEND_ASSETS``) are build outputs, ignored by
14 ``.museignore``, so no export contains them. They are built here from the exported sources -- ``npm ci``
15 from the committed lockfile, then one build script per asset, in a scratch directory -- and only the
16 outputs are copied into the context. The working tree, its ``node_modules`` and any locally built assets
17 are never used.
18
19 It refuses, before producing anything, when
20
21 * a repository has tracked changes (the image would then differ from the revision it claims; untracked
22 files are reported as excluded, never copied),
23 * the revision cannot be determined by Muse (no Git metadata is consulted),
24 * a required input is missing from the exported tree (``REQUIRED``), or the exported tree lacks a
25 migration the repository tracks,
26 * npm is unavailable, or a compiled asset cannot be built or comes out empty.
27
28 It writes ``build-info.json`` (the revision of every repository, the excluded untracked paths, the
29 SHA-256 and size of every compiled asset) in the context root; ``push.sh`` passes the revisions to
30 ``docker build`` as ``--build-arg`` and verifies the resulting image's labels. The output directory must
31 not already exist: contexts are never reused.
32
33 The context reaches the builder as a tar *stream* (``--stream``), never as a directory. BuildKit transfers
34 a directory context incrementally and treats a file as unchanged when its size, mtime and mode match an
35 earlier transfer; every file here has mtime 0, so an edit that keeps the byte length (a version bump, a
36 pin) was silently not transferred and the build used the previous content (musehub c17, 2026-11-11; and
37 the 2026-10-21 ``0.2.1rc4`` incident). A stream carries no such state::
38
39 python -m deploy.build_context --stream /tmp/ctx | docker build -f musehub/Dockerfile -
40
41 BuildKit does not apply ``Dockerfile.dockerignore`` to a streamed context, so the stream applies the same
42 allowlist itself (``musehub/Dockerfile.dockerignore``, the one the Dockerfile's COPY lines are checked
43 against): the image receives exactly what a directory build with that file would.
44
45 The stream is written by Python, so macOS ``tar`` cannot add AppleDouble ``._*`` files; ``._*`` and
46 ``.DS_Store`` files are refused outright, in the context and in the stream.
47 """
48 from __future__ import annotations
49
50 import argparse
51 import hashlib
52 import json
53 import re
54 import shutil
55 import subprocess
56 import sys
57 import tarfile
58 import tempfile
59 from collections.abc import Callable
60 from pathlib import Path
61 from typing import IO, Any
62
63 from deploy.release_provenance import ProvenanceError, muse_json, validate_revision
64
65 Runner = Callable[..., "subprocess.CompletedProcess[str]"]
66
67 #: repositories the image is built from, in Dockerfile order
68 REPOS: tuple[str, ...] = ("musehub", "muse")
69
70 #: files and directories that must exist in the exported tree, relative to the context root
71 REQUIRED: tuple[str, ...] = (
72 "musehub/Dockerfile",
73 "musehub/Dockerfile.dockerignore",
74 "musehub/requirements.txt",
75 "musehub/constraints.txt",
76 "musehub/grammar-requirements.txt",
77 "musehub/pyproject.toml",
78 "musehub/alembic.ini",
79 "musehub/entrypoint.sh",
80 "musehub/musehub/__init__.py",
81 "musehub/alembic/env.py",
82 "musehub/deploy/deploy.sh",
83 "musehub/docs",
84 "musehub/package.json",
85 "musehub/package-lock.json",
86 "musehub/src",
87 "muse/pyproject.toml",
88 "muse/README.md",
89 "muse/muse/__init__.py",
90 )
91 MIGRATIONS = "musehub/alembic/versions"
92
93 #: where the app serves /static from, relative to the context root (the Dockerfile copies musehub/musehub/)
94 STATIC_DIR = "musehub/musehub/templates/musehub/static"
95 #: compiled UI asset -> the package.json script that builds it (into musehub/templates/musehub/static/)
96 FRONTEND_ASSETS: dict[str, str] = {"app.js": "build:js", "app.css": "build:css", "embed.css": "build:embed"}
97 #: committed files the frontend build reads, relative to the musehub repository root
98 FRONTEND_INPUTS: tuple[str, ...] = ("package.json", "package-lock.json", "tsconfig.json", "src")
99
100
101 class BuildContextError(Exception):
102 """The build context cannot be assembled faithfully. Nothing is built."""
103
104
105 def _tracked_changes(status: dict[str, Any]) -> list[str]:
106 changes: list[str] = []
107 for key in ("added", "modified", "deleted", "conflict_paths"):
108 value = status.get(key)
109 if not isinstance(value, list):
110 raise BuildContextError(f"muse status --json: {key!r} missing or not a list")
111 changes += [str(p) for p in value]
112 changes += [f"{a} -> {b}" for a, b in (status.get("renamed") or {}).items()]
113 for stage in ("staged", "unstaged"):
114 block = status.get(stage)
115 if not isinstance(block, dict):
116 raise BuildContextError(f"muse status --json: {stage!r} missing or not an object")
117 for key in ("added", "modified", "deleted"):
118 changes += [str(p) for p in block.get(key, [])]
119 changes += [f"{a} -> {b}" for a, b in (block.get("renamed") or {}).items()]
120 for flag in ("merge_in_progress", "checkout_interrupted"):
121 if status.get(flag) is not False:
122 changes.append(f"<{flag}>")
123 return list(dict.fromkeys(changes))
124
125
126 def _inspect(repo: Path, runner: Runner) -> tuple[str, list[str]]:
127 """(revision, untracked paths). Raises if the repo is unobservable or has tracked changes."""
128 try:
129 revision = validate_revision(muse_json(repo, ["rev-parse", "HEAD"], runner).get("commit_id"))
130 status = muse_json(repo, ["status"], runner)
131 except ProvenanceError as exc:
132 raise BuildContextError(f"{repo.name}: cannot determine the revision with Muse: {exc}") from exc
133 if status.get("head_commit") != revision:
134 raise BuildContextError(f"{repo.name}: rev-parse and status disagree about HEAD")
135 changes = _tracked_changes(status)
136 if changes:
137 raise BuildContextError(
138 f"{repo.name} has tracked changes that are not in {revision}; the image would not be that revision. "
139 f"Commit or discard them first: {', '.join(changes[:8])}"
140 )
141 untracked = status.get("untracked")
142 if not isinstance(untracked, list):
143 raise BuildContextError("muse status --json: 'untracked' missing or not a list")
144 return revision, sorted(str(p) for p in untracked)
145
146
147 def _export(repo: Path, revision: str, out: Path, scratch: Path, runner: Runner) -> list[str]:
148 archive = scratch / f"{repo.name}.tar.gz"
149 argv = ["muse", "-C", str(repo), "archive", "--ref", revision, "--format", "tar.gz", "--output", str(archive), "--prefix", repo.name]
150 try:
151 proc = runner(argv, capture_output=True, text=True, timeout=600, check=False)
152 except (OSError, subprocess.SubprocessError) as exc:
153 raise BuildContextError(f"{repo.name}: muse archive could not run: {exc}") from exc
154 if proc.returncode != 0 or not archive.is_file():
155 raise BuildContextError(f"{repo.name}: muse archive failed ({proc.returncode}): {(proc.stderr or proc.stdout or '').strip()[:300]}")
156 try:
157 with tarfile.open(archive) as tf:
158 names = [m.name for m in tf.getmembers() if m.isfile()]
159 tf.extractall(out, filter="data") # rejects absolute paths, '..', links outside the target
160 except (tarfile.TarError, OSError) as exc:
161 raise BuildContextError(f"{repo.name}: cannot extract the archive: {exc}") from exc
162 return names
163
164
165 def _check_required(out: Path) -> None:
166 missing = [p for p in REQUIRED if not (out / p).exists()]
167 if missing:
168 raise BuildContextError(f"required build inputs are missing from the exported tree: {', '.join(missing)}")
169 if not any((out / MIGRATIONS).glob("*.py")):
170 raise BuildContextError(f"{MIGRATIONS} contains no migrations")
171
172
173 def _check_migrations(repo: Path, out: Path, runner: Runner) -> None:
174 """Every tracked migration is in the context (a truncated export would deploy a schema the code outruns)."""
175 try:
176 listed = muse_json(repo, ["ls-files"], runner)
177 except ProvenanceError as exc:
178 raise BuildContextError(f"musehub: cannot list tracked files: {exc}") from exc
179 files = listed.get("files") or listed.get("paths") or []
180 paths = [f if isinstance(f, str) else str(f.get("path", "")) for f in files]
181 expected = sorted(Path(p).name for p in paths if p.startswith("alembic/versions/") and p.endswith(".py"))
182 got = sorted(p.name for p in (out / MIGRATIONS).glob("*.py"))
183 if not expected:
184 raise BuildContextError("could not read the tracked migration list from muse ls-files")
185 if expected != got:
186 raise BuildContextError(f"exported migrations differ from the tracked ones: missing {sorted(set(expected) - set(got))[:5]}")
187
188
189 def _check_no_macos_metadata(root: Path) -> None:
190 """Refuse AppleDouble (``._*``) and ``.DS_Store`` files: they are never intended image content."""
191 found = sorted(str(p.relative_to(root)) for p in root.rglob("*") if p.name == ".DS_Store" or p.name.startswith("._"))
192 if found:
193 raise BuildContextError(f"macOS metadata files are in the build context: {', '.join(found[:8])}")
194
195
196 #: the allowlist, relative to the context root; also the one BuildKit applies to a *directory* context
197 DOCKERIGNORE = "musehub/Dockerfile.dockerignore"
198 #: files the builder always receives, as it does with a directory context
199 ALWAYS_SENT: frozenset[str] = frozenset({"musehub/Dockerfile", DOCKERIGNORE})
200
201
202 def _ignore_regex(pattern: str) -> re.Pattern[str]:
203 """One ``.dockerignore`` pattern as a regex over a ``/``-separated path (``**``, ``*``, ``?`` and literals only)."""
204 out: list[str] = []
205 i = 0
206 while i < len(pattern):
207 if pattern.startswith("**/", i):
208 out.append("(?:.*/)?")
209 i += 3
210 elif pattern.startswith("**", i):
211 out.append(".*")
212 i += 2
213 elif pattern[i] == "*":
214 out.append("[^/]*")
215 i += 1
216 elif pattern[i] == "?":
217 out.append("[^/]")
218 i += 1
219 elif pattern[i] in "[]\\{}()^$+|":
220 raise BuildContextError(f"{DOCKERIGNORE}: unsupported pattern {pattern!r}")
221 else:
222 out.append(re.escape(pattern[i]))
223 i += 1
224 return re.compile("".join(out))
225
226
227 def _allowlist(ctx: Path) -> Callable[[str], bool]:
228 """Whether a file at *path* (relative, ``/``-separated) is sent: the last matching rule wins, a match on a parent directory
229 counts for everything below it, and a context without the allowlist file is sent whole."""
230 file = ctx / DOCKERIGNORE
231 if not file.is_file():
232 return lambda path: True
233 rules: list[tuple[bool, re.Pattern[str]]] = []
234 for raw in file.read_text().splitlines():
235 line = raw.strip()
236 if line and not line.startswith("#"):
237 negated = line.startswith("!")
238 rules.append((negated, _ignore_regex(line.removeprefix("!").strip("/"))))
239
240 def sent(path: str) -> bool:
241 if path in ALWAYS_SENT:
242 return True
243 keep = True
244 parts = path.split("/")
245 for negated, rx in rules:
246 if any(rx.fullmatch("/".join(parts[: n + 1])) for n in range(len(parts))):
247 keep = negated
248 return keep
249
250 return sent
251
252
253 def context_tar(ctx: Path, out: IO[bytes]) -> None:
254 """Write *ctx* to *out* as a deterministic tar: sorted, root-owned, mtime 0, modes kept, no leading ``./``.
255
256 Only files the allowlist sends are written, plus their parent directories. The same context always gives the same
257 bytes, and the builder receives it fresh every time (module docstring).
258 """
259 _check_no_macos_metadata(ctx)
260 sent = _allowlist(ctx)
261 with tarfile.open(fileobj=out, mode="w|", format=tarfile.PAX_FORMAT) as tf:
262 paths = sorted(p for p in ctx.rglob("*") if not p.is_dir() and sent(p.relative_to(ctx).as_posix()))
263 directories = sorted({parent for p in paths for parent in p.relative_to(ctx).parents if parent != Path(".")})
264 for path in sorted([ctx / d for d in directories] + paths):
265 info = tf.gettarinfo(str(path), arcname=path.relative_to(ctx).as_posix())
266 info.uid = info.gid = 0
267 info.uname = info.gname = ""
268 info.mtime = 0
269 if info.isreg():
270 with path.open("rb") as fh:
271 tf.addfile(info, fh)
272 else:
273 tf.addfile(info)
274
275
276 def _npm() -> str | None:
277 return shutil.which("npm")
278
279
280 def _build_frontend(out: Path, scratch: Path, runner: Runner) -> dict[str, dict[str, Any]]:
281 """Build every ``FRONTEND_ASSETS`` entry from the exported sources in *scratch*; copy only the outputs into *out*."""
282 npm = _npm()
283 if npm is None:
284 raise BuildContextError("npm is not available; the compiled frontend assets cannot be built")
285 work = scratch / "frontend"
286 work.mkdir()
287 for name in FRONTEND_INPUTS:
288 src = out / "musehub" / name
289 if src.is_dir():
290 shutil.copytree(src, work / name)
291 elif src.is_file():
292 shutil.copy2(src, work / name)
293 steps = [[npm, "ci", "--ignore-scripts", "--no-audit", "--no-fund"]] + [[npm, "run", "--silent", script] for script in FRONTEND_ASSETS.values()]
294 for argv in steps:
295 try:
296 proc = runner(argv, capture_output=True, text=True, timeout=900, check=False, cwd=work)
297 except (OSError, subprocess.SubprocessError) as exc:
298 raise BuildContextError(f"frontend build could not run {' '.join(argv[1:])}: {exc}") from exc
299 if proc.returncode != 0:
300 raise BuildContextError(f"frontend build failed ({' '.join(argv[1:])}, exit {proc.returncode}): {(proc.stderr or proc.stdout or '').strip()[-300:]}")
301 built = work / Path(STATIC_DIR).relative_to("musehub")
302 assets: dict[str, dict[str, Any]] = {}
303 for name in FRONTEND_ASSETS:
304 data = (built / name).read_bytes() if (built / name).is_file() else b""
305 if not data:
306 raise BuildContextError(f"frontend build produced no {name} (missing or empty)")
307 (out / STATIC_DIR).mkdir(parents=True, exist_ok=True)
308 (out / STATIC_DIR / name).write_bytes(data)
309 assets[name] = {"sha256": hashlib.sha256(data).hexdigest(), "bytes": len(data)}
310 return assets
311
312
313 def assemble(ecosystem: Path, out: Path, *, runner: Runner = subprocess.run) -> dict[str, Any]:
314 """Create *out* holding exactly the committed files of every repository in ``REPOS``. Returns build-info."""
315 if out.exists():
316 raise BuildContextError(f"{out} already exists; build contexts are never reused")
317 repos = {name: ecosystem / name for name in REPOS}
318 for name, path in repos.items():
319 if not path.is_dir():
320 raise BuildContextError(f"repository {name} not found at {path}")
321 inspected = {name: _inspect(path, runner) for name, path in repos.items()} # all checks before anything is written
322
323 out.mkdir(parents=True)
324 try:
325 with tempfile.TemporaryDirectory(prefix="muse-ctx-") as scratch:
326 for name, path in repos.items():
327 _export(path, inspected[name][0], out, Path(scratch), runner)
328 _check_required(out)
329 _check_migrations(repos["musehub"], out, runner)
330 _check_no_macos_metadata(out)
331 assets = _build_frontend(out, Path(scratch), runner)
332 info = {
333 "schema": 1,
334 "repositories": {name: {"revision": rev, "untracked_excluded": untracked} for name, (rev, untracked) in sorted(inspected.items())},
335 "frontend_assets": assets,
336 }
337 (out / "build-info.json").write_text(json.dumps(info, indent=2, sort_keys=True) + "\n")
338 except BaseException:
339 shutil.rmtree(out, ignore_errors=True) # never leave a half-assembled context to be mistaken for a good one
340 raise
341 return info
342
343
344 def main(argv: list[str] | None = None) -> int:
345 parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
346 parser.add_argument("--ecosystem", type=Path, default=Path.home() / "ecosystem")
347 mode = parser.add_mutually_exclusive_group(required=True)
348 mode.add_argument("--out", type=Path, help="assemble a new context directory here")
349 mode.add_argument("--stream", type=Path, metavar="CONTEXT", help="write an assembled context to stdout as a tar, for `docker build -`")
350 args = parser.parse_args(argv)
351 if args.stream is not None:
352 try:
353 context_tar(args.stream, sys.stdout.buffer)
354 except (BuildContextError, OSError) as exc:
355 print(f"BUILD CONTEXT REFUSED: {exc}", file=sys.stderr)
356 return 1
357 sys.stdout.buffer.flush()
358 return 0
359 try:
360 info = assemble(args.ecosystem, args.out)
361 except BuildContextError as exc:
362 print(f"BUILD CONTEXT REFUSED: {exc}", file=sys.stderr)
363 return 1
364 print(json.dumps(info, indent=2, sort_keys=True))
365 return 0
366
367
368 if __name__ == "__main__":
369 raise SystemExit(main())
File History 5 commits
sha256:b4b999434d042d74edc7032f1f2cfd20fda5b3580fe85966351db5b8eef0abb3 feat(intel): every grammar on the hub, and every intel job … Opus 5.5 minor ⚠ 1 day ago
sha256:414893924229cfb6dd5a5154d9526c1a9b57a2721289daba6ad89a6c98c64c09 fix(deploy): stream the Docker build context so same-size e… Sonnet 5.5 patch 5 days ago
sha256:70e2deebbf676f002c5122f381222eb21fe65b51c04380538ea9acb3fe62dfc5 build: install the verified dependency versions instead of … Opus 5.5 minor ⚠ 11 days ago
sha256:6b75088fadf04c065f8c92b3b71f8c7ffe315d0e41ab71e4dc7b0b9e99f29426 fix(deploy): build the compiled UI assets into the pinned b… Opus 5.5 minor ⚠ 12 days ago
sha256:3c6d2e2751e9222a0872ba699f16baa457d9990c1998c20956badb9186ee69ee fix(deploy): image built from committed files only, with re… Opus 5.5 minor ⚠ 12 days ago