build_context.py
python
sha256:b4b999434d042d74edc7032f1f2cfd20fda5b3580fe85966351db5b8eef0abb3
feat(intel): every grammar on the hub, and every intel job …
Opus 5.5
minor
⚠ breaking
1 day ago
| 1 | #!/usr/bin/env python3 |
| 2 | """Assemble the Docker build context from *committed* files only (musehub#114 Phase 6D). |
| 3 | |
| 4 | ``push.sh`` used to hand Docker the whole ``~/ecosystem`` working tree, so any untracked local file under |
| 5 | a directory the Dockerfile copies (for example ``docs/work/session-handoff.md``) ended up in the deployed |
| 6 | image, and the image said nothing about which revision it was built from. |
| 7 | |
| 8 | This builds the context instead from ``muse archive`` of each repository the image needs, so untracked |
| 9 | and ignored files are not merely filtered out -- they are never present:: |
| 10 | |
| 11 | python -m deploy.build_context --ecosystem ~/ecosystem --out /tmp/ctx |
| 12 | |
| 13 | The compiled UI assets the app serves from ``/static`` (``FRONTEND_ASSETS``) are build outputs, ignored by |
| 14 | ``.museignore``, so no export contains them. They are built here from the exported sources -- ``npm ci`` |
| 15 | from the committed lockfile, then one build script per asset, in a scratch directory -- and only the |
| 16 | outputs are copied into the context. The working tree, its ``node_modules`` and any locally built assets |
| 17 | are never used. |
| 18 | |
| 19 | It refuses, before producing anything, when |
| 20 | |
| 21 | * a repository has tracked changes (the image would then differ from the revision it claims; untracked |
| 22 | files are reported as excluded, never copied), |
| 23 | * the revision cannot be determined by Muse (no Git metadata is consulted), |
| 24 | * a required input is missing from the exported tree (``REQUIRED``), or the exported tree lacks a |
| 25 | migration the repository tracks, |
| 26 | * npm is unavailable, or a compiled asset cannot be built or comes out empty. |
| 27 | |
| 28 | It writes ``build-info.json`` (the revision of every repository, the excluded untracked paths, the |
| 29 | SHA-256 and size of every compiled asset) in the context root; ``push.sh`` passes the revisions to |
| 30 | ``docker build`` as ``--build-arg`` and verifies the resulting image's labels. The output directory must |
| 31 | not already exist: contexts are never reused. |
| 32 | |
| 33 | The context reaches the builder as a tar *stream* (``--stream``), never as a directory. BuildKit transfers |
| 34 | a directory context incrementally and treats a file as unchanged when its size, mtime and mode match an |
| 35 | earlier transfer; every file here has mtime 0, so an edit that keeps the byte length (a version bump, a |
| 36 | pin) was silently not transferred and the build used the previous content (musehub c17, 2026-11-11; and |
| 37 | the 2026-10-21 ``0.2.1rc4`` incident). A stream carries no such state:: |
| 38 | |
| 39 | python -m deploy.build_context --stream /tmp/ctx | docker build -f musehub/Dockerfile - |
| 40 | |
| 41 | BuildKit does not apply ``Dockerfile.dockerignore`` to a streamed context, so the stream applies the same |
| 42 | allowlist itself (``musehub/Dockerfile.dockerignore``, the one the Dockerfile's COPY lines are checked |
| 43 | against): the image receives exactly what a directory build with that file would. |
| 44 | |
| 45 | The stream is written by Python, so macOS ``tar`` cannot add AppleDouble ``._*`` files; ``._*`` and |
| 46 | ``.DS_Store`` files are refused outright, in the context and in the stream. |
| 47 | """ |
| 48 | from __future__ import annotations |
| 49 | |
| 50 | import argparse |
| 51 | import hashlib |
| 52 | import json |
| 53 | import re |
| 54 | import shutil |
| 55 | import subprocess |
| 56 | import sys |
| 57 | import tarfile |
| 58 | import tempfile |
| 59 | from collections.abc import Callable |
| 60 | from pathlib import Path |
| 61 | from typing import IO, Any |
| 62 | |
| 63 | from deploy.release_provenance import ProvenanceError, muse_json, validate_revision |
| 64 | |
| 65 | Runner = Callable[..., "subprocess.CompletedProcess[str]"] |
| 66 | |
| 67 | #: repositories the image is built from, in Dockerfile order |
| 68 | REPOS: tuple[str, ...] = ("musehub", "muse") |
| 69 | |
| 70 | #: files and directories that must exist in the exported tree, relative to the context root |
| 71 | REQUIRED: tuple[str, ...] = ( |
| 72 | "musehub/Dockerfile", |
| 73 | "musehub/Dockerfile.dockerignore", |
| 74 | "musehub/requirements.txt", |
| 75 | "musehub/constraints.txt", |
| 76 | "musehub/grammar-requirements.txt", |
| 77 | "musehub/pyproject.toml", |
| 78 | "musehub/alembic.ini", |
| 79 | "musehub/entrypoint.sh", |
| 80 | "musehub/musehub/__init__.py", |
| 81 | "musehub/alembic/env.py", |
| 82 | "musehub/deploy/deploy.sh", |
| 83 | "musehub/docs", |
| 84 | "musehub/package.json", |
| 85 | "musehub/package-lock.json", |
| 86 | "musehub/src", |
| 87 | "muse/pyproject.toml", |
| 88 | "muse/README.md", |
| 89 | "muse/muse/__init__.py", |
| 90 | ) |
| 91 | MIGRATIONS = "musehub/alembic/versions" |
| 92 | |
| 93 | #: where the app serves /static from, relative to the context root (the Dockerfile copies musehub/musehub/) |
| 94 | STATIC_DIR = "musehub/musehub/templates/musehub/static" |
| 95 | #: compiled UI asset -> the package.json script that builds it (into musehub/templates/musehub/static/) |
| 96 | FRONTEND_ASSETS: dict[str, str] = {"app.js": "build:js", "app.css": "build:css", "embed.css": "build:embed"} |
| 97 | #: committed files the frontend build reads, relative to the musehub repository root |
| 98 | FRONTEND_INPUTS: tuple[str, ...] = ("package.json", "package-lock.json", "tsconfig.json", "src") |
| 99 | |
| 100 | |
| 101 | class BuildContextError(Exception): |
| 102 | """The build context cannot be assembled faithfully. Nothing is built.""" |
| 103 | |
| 104 | |
| 105 | def _tracked_changes(status: dict[str, Any]) -> list[str]: |
| 106 | changes: list[str] = [] |
| 107 | for key in ("added", "modified", "deleted", "conflict_paths"): |
| 108 | value = status.get(key) |
| 109 | if not isinstance(value, list): |
| 110 | raise BuildContextError(f"muse status --json: {key!r} missing or not a list") |
| 111 | changes += [str(p) for p in value] |
| 112 | changes += [f"{a} -> {b}" for a, b in (status.get("renamed") or {}).items()] |
| 113 | for stage in ("staged", "unstaged"): |
| 114 | block = status.get(stage) |
| 115 | if not isinstance(block, dict): |
| 116 | raise BuildContextError(f"muse status --json: {stage!r} missing or not an object") |
| 117 | for key in ("added", "modified", "deleted"): |
| 118 | changes += [str(p) for p in block.get(key, [])] |
| 119 | changes += [f"{a} -> {b}" for a, b in (block.get("renamed") or {}).items()] |
| 120 | for flag in ("merge_in_progress", "checkout_interrupted"): |
| 121 | if status.get(flag) is not False: |
| 122 | changes.append(f"<{flag}>") |
| 123 | return list(dict.fromkeys(changes)) |
| 124 | |
| 125 | |
| 126 | def _inspect(repo: Path, runner: Runner) -> tuple[str, list[str]]: |
| 127 | """(revision, untracked paths). Raises if the repo is unobservable or has tracked changes.""" |
| 128 | try: |
| 129 | revision = validate_revision(muse_json(repo, ["rev-parse", "HEAD"], runner).get("commit_id")) |
| 130 | status = muse_json(repo, ["status"], runner) |
| 131 | except ProvenanceError as exc: |
| 132 | raise BuildContextError(f"{repo.name}: cannot determine the revision with Muse: {exc}") from exc |
| 133 | if status.get("head_commit") != revision: |
| 134 | raise BuildContextError(f"{repo.name}: rev-parse and status disagree about HEAD") |
| 135 | changes = _tracked_changes(status) |
| 136 | if changes: |
| 137 | raise BuildContextError( |
| 138 | f"{repo.name} has tracked changes that are not in {revision}; the image would not be that revision. " |
| 139 | f"Commit or discard them first: {', '.join(changes[:8])}" |
| 140 | ) |
| 141 | untracked = status.get("untracked") |
| 142 | if not isinstance(untracked, list): |
| 143 | raise BuildContextError("muse status --json: 'untracked' missing or not a list") |
| 144 | return revision, sorted(str(p) for p in untracked) |
| 145 | |
| 146 | |
| 147 | def _export(repo: Path, revision: str, out: Path, scratch: Path, runner: Runner) -> list[str]: |
| 148 | archive = scratch / f"{repo.name}.tar.gz" |
| 149 | argv = ["muse", "-C", str(repo), "archive", "--ref", revision, "--format", "tar.gz", "--output", str(archive), "--prefix", repo.name] |
| 150 | try: |
| 151 | proc = runner(argv, capture_output=True, text=True, timeout=600, check=False) |
| 152 | except (OSError, subprocess.SubprocessError) as exc: |
| 153 | raise BuildContextError(f"{repo.name}: muse archive could not run: {exc}") from exc |
| 154 | if proc.returncode != 0 or not archive.is_file(): |
| 155 | raise BuildContextError(f"{repo.name}: muse archive failed ({proc.returncode}): {(proc.stderr or proc.stdout or '').strip()[:300]}") |
| 156 | try: |
| 157 | with tarfile.open(archive) as tf: |
| 158 | names = [m.name for m in tf.getmembers() if m.isfile()] |
| 159 | tf.extractall(out, filter="data") # rejects absolute paths, '..', links outside the target |
| 160 | except (tarfile.TarError, OSError) as exc: |
| 161 | raise BuildContextError(f"{repo.name}: cannot extract the archive: {exc}") from exc |
| 162 | return names |
| 163 | |
| 164 | |
| 165 | def _check_required(out: Path) -> None: |
| 166 | missing = [p for p in REQUIRED if not (out / p).exists()] |
| 167 | if missing: |
| 168 | raise BuildContextError(f"required build inputs are missing from the exported tree: {', '.join(missing)}") |
| 169 | if not any((out / MIGRATIONS).glob("*.py")): |
| 170 | raise BuildContextError(f"{MIGRATIONS} contains no migrations") |
| 171 | |
| 172 | |
| 173 | def _check_migrations(repo: Path, out: Path, runner: Runner) -> None: |
| 174 | """Every tracked migration is in the context (a truncated export would deploy a schema the code outruns).""" |
| 175 | try: |
| 176 | listed = muse_json(repo, ["ls-files"], runner) |
| 177 | except ProvenanceError as exc: |
| 178 | raise BuildContextError(f"musehub: cannot list tracked files: {exc}") from exc |
| 179 | files = listed.get("files") or listed.get("paths") or [] |
| 180 | paths = [f if isinstance(f, str) else str(f.get("path", "")) for f in files] |
| 181 | expected = sorted(Path(p).name for p in paths if p.startswith("alembic/versions/") and p.endswith(".py")) |
| 182 | got = sorted(p.name for p in (out / MIGRATIONS).glob("*.py")) |
| 183 | if not expected: |
| 184 | raise BuildContextError("could not read the tracked migration list from muse ls-files") |
| 185 | if expected != got: |
| 186 | raise BuildContextError(f"exported migrations differ from the tracked ones: missing {sorted(set(expected) - set(got))[:5]}") |
| 187 | |
| 188 | |
| 189 | def _check_no_macos_metadata(root: Path) -> None: |
| 190 | """Refuse AppleDouble (``._*``) and ``.DS_Store`` files: they are never intended image content.""" |
| 191 | found = sorted(str(p.relative_to(root)) for p in root.rglob("*") if p.name == ".DS_Store" or p.name.startswith("._")) |
| 192 | if found: |
| 193 | raise BuildContextError(f"macOS metadata files are in the build context: {', '.join(found[:8])}") |
| 194 | |
| 195 | |
| 196 | #: the allowlist, relative to the context root; also the one BuildKit applies to a *directory* context |
| 197 | DOCKERIGNORE = "musehub/Dockerfile.dockerignore" |
| 198 | #: files the builder always receives, as it does with a directory context |
| 199 | ALWAYS_SENT: frozenset[str] = frozenset({"musehub/Dockerfile", DOCKERIGNORE}) |
| 200 | |
| 201 | |
| 202 | def _ignore_regex(pattern: str) -> re.Pattern[str]: |
| 203 | """One ``.dockerignore`` pattern as a regex over a ``/``-separated path (``**``, ``*``, ``?`` and literals only).""" |
| 204 | out: list[str] = [] |
| 205 | i = 0 |
| 206 | while i < len(pattern): |
| 207 | if pattern.startswith("**/", i): |
| 208 | out.append("(?:.*/)?") |
| 209 | i += 3 |
| 210 | elif pattern.startswith("**", i): |
| 211 | out.append(".*") |
| 212 | i += 2 |
| 213 | elif pattern[i] == "*": |
| 214 | out.append("[^/]*") |
| 215 | i += 1 |
| 216 | elif pattern[i] == "?": |
| 217 | out.append("[^/]") |
| 218 | i += 1 |
| 219 | elif pattern[i] in "[]\\{}()^$+|": |
| 220 | raise BuildContextError(f"{DOCKERIGNORE}: unsupported pattern {pattern!r}") |
| 221 | else: |
| 222 | out.append(re.escape(pattern[i])) |
| 223 | i += 1 |
| 224 | return re.compile("".join(out)) |
| 225 | |
| 226 | |
| 227 | def _allowlist(ctx: Path) -> Callable[[str], bool]: |
| 228 | """Whether a file at *path* (relative, ``/``-separated) is sent: the last matching rule wins, a match on a parent directory |
| 229 | counts for everything below it, and a context without the allowlist file is sent whole.""" |
| 230 | file = ctx / DOCKERIGNORE |
| 231 | if not file.is_file(): |
| 232 | return lambda path: True |
| 233 | rules: list[tuple[bool, re.Pattern[str]]] = [] |
| 234 | for raw in file.read_text().splitlines(): |
| 235 | line = raw.strip() |
| 236 | if line and not line.startswith("#"): |
| 237 | negated = line.startswith("!") |
| 238 | rules.append((negated, _ignore_regex(line.removeprefix("!").strip("/")))) |
| 239 | |
| 240 | def sent(path: str) -> bool: |
| 241 | if path in ALWAYS_SENT: |
| 242 | return True |
| 243 | keep = True |
| 244 | parts = path.split("/") |
| 245 | for negated, rx in rules: |
| 246 | if any(rx.fullmatch("/".join(parts[: n + 1])) for n in range(len(parts))): |
| 247 | keep = negated |
| 248 | return keep |
| 249 | |
| 250 | return sent |
| 251 | |
| 252 | |
| 253 | def context_tar(ctx: Path, out: IO[bytes]) -> None: |
| 254 | """Write *ctx* to *out* as a deterministic tar: sorted, root-owned, mtime 0, modes kept, no leading ``./``. |
| 255 | |
| 256 | Only files the allowlist sends are written, plus their parent directories. The same context always gives the same |
| 257 | bytes, and the builder receives it fresh every time (module docstring). |
| 258 | """ |
| 259 | _check_no_macos_metadata(ctx) |
| 260 | sent = _allowlist(ctx) |
| 261 | with tarfile.open(fileobj=out, mode="w|", format=tarfile.PAX_FORMAT) as tf: |
| 262 | paths = sorted(p for p in ctx.rglob("*") if not p.is_dir() and sent(p.relative_to(ctx).as_posix())) |
| 263 | directories = sorted({parent for p in paths for parent in p.relative_to(ctx).parents if parent != Path(".")}) |
| 264 | for path in sorted([ctx / d for d in directories] + paths): |
| 265 | info = tf.gettarinfo(str(path), arcname=path.relative_to(ctx).as_posix()) |
| 266 | info.uid = info.gid = 0 |
| 267 | info.uname = info.gname = "" |
| 268 | info.mtime = 0 |
| 269 | if info.isreg(): |
| 270 | with path.open("rb") as fh: |
| 271 | tf.addfile(info, fh) |
| 272 | else: |
| 273 | tf.addfile(info) |
| 274 | |
| 275 | |
| 276 | def _npm() -> str | None: |
| 277 | return shutil.which("npm") |
| 278 | |
| 279 | |
| 280 | def _build_frontend(out: Path, scratch: Path, runner: Runner) -> dict[str, dict[str, Any]]: |
| 281 | """Build every ``FRONTEND_ASSETS`` entry from the exported sources in *scratch*; copy only the outputs into *out*.""" |
| 282 | npm = _npm() |
| 283 | if npm is None: |
| 284 | raise BuildContextError("npm is not available; the compiled frontend assets cannot be built") |
| 285 | work = scratch / "frontend" |
| 286 | work.mkdir() |
| 287 | for name in FRONTEND_INPUTS: |
| 288 | src = out / "musehub" / name |
| 289 | if src.is_dir(): |
| 290 | shutil.copytree(src, work / name) |
| 291 | elif src.is_file(): |
| 292 | shutil.copy2(src, work / name) |
| 293 | steps = [[npm, "ci", "--ignore-scripts", "--no-audit", "--no-fund"]] + [[npm, "run", "--silent", script] for script in FRONTEND_ASSETS.values()] |
| 294 | for argv in steps: |
| 295 | try: |
| 296 | proc = runner(argv, capture_output=True, text=True, timeout=900, check=False, cwd=work) |
| 297 | except (OSError, subprocess.SubprocessError) as exc: |
| 298 | raise BuildContextError(f"frontend build could not run {' '.join(argv[1:])}: {exc}") from exc |
| 299 | if proc.returncode != 0: |
| 300 | raise BuildContextError(f"frontend build failed ({' '.join(argv[1:])}, exit {proc.returncode}): {(proc.stderr or proc.stdout or '').strip()[-300:]}") |
| 301 | built = work / Path(STATIC_DIR).relative_to("musehub") |
| 302 | assets: dict[str, dict[str, Any]] = {} |
| 303 | for name in FRONTEND_ASSETS: |
| 304 | data = (built / name).read_bytes() if (built / name).is_file() else b"" |
| 305 | if not data: |
| 306 | raise BuildContextError(f"frontend build produced no {name} (missing or empty)") |
| 307 | (out / STATIC_DIR).mkdir(parents=True, exist_ok=True) |
| 308 | (out / STATIC_DIR / name).write_bytes(data) |
| 309 | assets[name] = {"sha256": hashlib.sha256(data).hexdigest(), "bytes": len(data)} |
| 310 | return assets |
| 311 | |
| 312 | |
| 313 | def assemble(ecosystem: Path, out: Path, *, runner: Runner = subprocess.run) -> dict[str, Any]: |
| 314 | """Create *out* holding exactly the committed files of every repository in ``REPOS``. Returns build-info.""" |
| 315 | if out.exists(): |
| 316 | raise BuildContextError(f"{out} already exists; build contexts are never reused") |
| 317 | repos = {name: ecosystem / name for name in REPOS} |
| 318 | for name, path in repos.items(): |
| 319 | if not path.is_dir(): |
| 320 | raise BuildContextError(f"repository {name} not found at {path}") |
| 321 | inspected = {name: _inspect(path, runner) for name, path in repos.items()} # all checks before anything is written |
| 322 | |
| 323 | out.mkdir(parents=True) |
| 324 | try: |
| 325 | with tempfile.TemporaryDirectory(prefix="muse-ctx-") as scratch: |
| 326 | for name, path in repos.items(): |
| 327 | _export(path, inspected[name][0], out, Path(scratch), runner) |
| 328 | _check_required(out) |
| 329 | _check_migrations(repos["musehub"], out, runner) |
| 330 | _check_no_macos_metadata(out) |
| 331 | assets = _build_frontend(out, Path(scratch), runner) |
| 332 | info = { |
| 333 | "schema": 1, |
| 334 | "repositories": {name: {"revision": rev, "untracked_excluded": untracked} for name, (rev, untracked) in sorted(inspected.items())}, |
| 335 | "frontend_assets": assets, |
| 336 | } |
| 337 | (out / "build-info.json").write_text(json.dumps(info, indent=2, sort_keys=True) + "\n") |
| 338 | except BaseException: |
| 339 | shutil.rmtree(out, ignore_errors=True) # never leave a half-assembled context to be mistaken for a good one |
| 340 | raise |
| 341 | return info |
| 342 | |
| 343 | |
| 344 | def main(argv: list[str] | None = None) -> int: |
| 345 | parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) |
| 346 | parser.add_argument("--ecosystem", type=Path, default=Path.home() / "ecosystem") |
| 347 | mode = parser.add_mutually_exclusive_group(required=True) |
| 348 | mode.add_argument("--out", type=Path, help="assemble a new context directory here") |
| 349 | mode.add_argument("--stream", type=Path, metavar="CONTEXT", help="write an assembled context to stdout as a tar, for `docker build -`") |
| 350 | args = parser.parse_args(argv) |
| 351 | if args.stream is not None: |
| 352 | try: |
| 353 | context_tar(args.stream, sys.stdout.buffer) |
| 354 | except (BuildContextError, OSError) as exc: |
| 355 | print(f"BUILD CONTEXT REFUSED: {exc}", file=sys.stderr) |
| 356 | return 1 |
| 357 | sys.stdout.buffer.flush() |
| 358 | return 0 |
| 359 | try: |
| 360 | info = assemble(args.ecosystem, args.out) |
| 361 | except BuildContextError as exc: |
| 362 | print(f"BUILD CONTEXT REFUSED: {exc}", file=sys.stderr) |
| 363 | return 1 |
| 364 | print(json.dumps(info, indent=2, sort_keys=True)) |
| 365 | return 0 |
| 366 | |
| 367 | |
| 368 | if __name__ == "__main__": |
| 369 | raise SystemExit(main()) |
File History
5 commits
sha256:b4b999434d042d74edc7032f1f2cfd20fda5b3580fe85966351db5b8eef0abb3
feat(intel): every grammar on the hub, and every intel job …
Opus 5.5
minor
⚠
1 day ago
sha256:414893924229cfb6dd5a5154d9526c1a9b57a2721289daba6ad89a6c98c64c09
fix(deploy): stream the Docker build context so same-size e…
Sonnet 5.5
patch
5 days ago
sha256:70e2deebbf676f002c5122f381222eb21fe65b51c04380538ea9acb3fe62dfc5
build: install the verified dependency versions instead of …
Opus 5.5
minor
⚠
11 days ago
sha256:6b75088fadf04c065f8c92b3b71f8c7ffe315d0e41ab71e4dc7b0b9e99f29426
fix(deploy): build the compiled UI assets into the pinned b…
Opus 5.5
minor
⚠
12 days ago
sha256:3c6d2e2751e9222a0872ba699f16baa457d9990c1998c20956badb9186ee69ee
fix(deploy): image built from committed files only, with re…
Opus 5.5
minor
⚠
12 days ago