build_context.py
python
sha256:b4b999434d042d74edc7032f1f2cfd20fda5b3580fe85966351db5b8eef0abb3
feat(intel): every grammar on the hub, and every intel job …
Opus 5.5
minor
⚠ breaking
1 day ago
| 1 | #!/usr/bin/env python3 |
| 2 | """Assemble the Docker build context from *committed* files only (musehub#114 Phase 6D). |
| 3 | |
| 4 | ``push.sh`` used to hand Docker the whole ``~/ecosystem`` working tree, so any untracked local file under |
| 5 | a directory the Dockerfile copies (for example ``docs/work/session-handoff.md``) ended up in the deployed |
| 6 | image, and the image said nothing about which revision it was built from. |
| 7 | |
| 8 | This builds the context instead from ``muse archive`` of each repository the image needs, so untracked |
| 9 | and ignored files are not merely filtered out -- they are never present:: |
| 10 | |
| 11 | python -m deploy.build_context --ecosystem ~/ecosystem --out /tmp/ctx |
| 12 | |
| 13 | The compiled UI assets the app serves from ``/static`` (``FRONTEND_ASSETS``) are build outputs, ignored by |
| 14 | ``.museignore``, so no export contains them. They are built here from the exported sources -- ``npm ci`` |
| 15 | from the committed lockfile, then one build script per asset, in a scratch directory -- and only the |
| 16 | outputs are copied into the context. The working tree, its ``node_modules`` and any locally built assets |
| 17 | are never used. |
| 18 | |
| 19 | It refuses, before producing anything, when |
| 20 | |
| 21 | * a repository has tracked changes (the image would then differ from the revision it claims; untracked |
| 22 | files are reported as excluded, never copied), |
| 23 | * the revision cannot be determined by Muse (no Git metadata is consulted), |
| 24 | * a required input is missing from the exported tree (``REQUIRED``), or the exported tree lacks a |
| 25 | migration the repository tracks, |
| 26 | * npm is unavailable, or a compiled asset cannot be built or comes out empty. |
| 27 | |
| 28 | It writes ``build-info.json`` (the revision of every repository, the excluded untracked paths, the |
| 29 | SHA-256 and size of every compiled asset) in the context root; ``push.sh`` passes the revisions to |
| 30 | ``docker build`` as ``--build-arg`` and verifies the resulting image's labels. The output directory must |
| 31 | not already exist: contexts are never reused. |
| 32 | """ |
| 33 | from __future__ import annotations |
| 34 | |
| 35 | import argparse |
| 36 | import hashlib |
| 37 | import json |
| 38 | import shutil |
| 39 | import subprocess |
| 40 | import sys |
| 41 | import tarfile |
| 42 | import tempfile |
| 43 | from collections.abc import Callable |
| 44 | from pathlib import Path |
| 45 | from typing import Any |
| 46 | |
| 47 | from deploy.release_provenance import ProvenanceError, muse_json, validate_revision |
| 48 | |
| 49 | Runner = Callable[..., "subprocess.CompletedProcess[str]"] |
| 50 | |
| 51 | #: repositories the image is built from, in Dockerfile order |
| 52 | REPOS: tuple[str, ...] = ("musehub", "muse") |
| 53 | |
| 54 | #: files and directories that must exist in the exported tree, relative to the context root |
| 55 | REQUIRED: tuple[str, ...] = ( |
| 56 | "musehub/Dockerfile", |
| 57 | "musehub/Dockerfile.dockerignore", |
| 58 | "musehub/requirements.txt", |
| 59 | "musehub/constraints.txt", |
| 60 | "musehub/pyproject.toml", |
| 61 | "musehub/alembic.ini", |
| 62 | "musehub/entrypoint.sh", |
| 63 | "musehub/musehub/__init__.py", |
| 64 | "musehub/alembic/env.py", |
| 65 | "musehub/deploy/deploy.sh", |
| 66 | "musehub/docs", |
| 67 | "musehub/package.json", |
| 68 | "musehub/package-lock.json", |
| 69 | "musehub/src", |
| 70 | "muse/pyproject.toml", |
| 71 | "muse/README.md", |
| 72 | "muse/muse/__init__.py", |
| 73 | ) |
| 74 | MIGRATIONS = "musehub/alembic/versions" |
| 75 | |
| 76 | #: where the app serves /static from, relative to the context root (the Dockerfile copies musehub/musehub/) |
| 77 | STATIC_DIR = "musehub/musehub/templates/musehub/static" |
| 78 | #: compiled UI asset -> the package.json script that builds it (into musehub/templates/musehub/static/) |
| 79 | FRONTEND_ASSETS: dict[str, str] = {"app.js": "build:js", "app.css": "build:css", "embed.css": "build:embed"} |
| 80 | #: committed files the frontend build reads, relative to the musehub repository root |
| 81 | FRONTEND_INPUTS: tuple[str, ...] = ("package.json", "package-lock.json", "tsconfig.json", "src") |
| 82 | |
| 83 | |
| 84 | class BuildContextError(Exception): |
| 85 | """The build context cannot be assembled faithfully. Nothing is built.""" |
| 86 | |
| 87 | |
| 88 | def _tracked_changes(status: dict[str, Any]) -> list[str]: |
| 89 | changes: list[str] = [] |
| 90 | for key in ("added", "modified", "deleted", "conflict_paths"): |
| 91 | value = status.get(key) |
| 92 | if not isinstance(value, list): |
| 93 | raise BuildContextError(f"muse status --json: {key!r} missing or not a list") |
| 94 | changes += [str(p) for p in value] |
| 95 | changes += [f"{a} -> {b}" for a, b in (status.get("renamed") or {}).items()] |
| 96 | for stage in ("staged", "unstaged"): |
| 97 | block = status.get(stage) |
| 98 | if not isinstance(block, dict): |
| 99 | raise BuildContextError(f"muse status --json: {stage!r} missing or not an object") |
| 100 | for key in ("added", "modified", "deleted"): |
| 101 | changes += [str(p) for p in block.get(key, [])] |
| 102 | changes += [f"{a} -> {b}" for a, b in (block.get("renamed") or {}).items()] |
| 103 | for flag in ("merge_in_progress", "checkout_interrupted"): |
| 104 | if status.get(flag) is not False: |
| 105 | changes.append(f"<{flag}>") |
| 106 | return list(dict.fromkeys(changes)) |
| 107 | |
| 108 | |
| 109 | def _inspect(repo: Path, runner: Runner) -> tuple[str, list[str]]: |
| 110 | """(revision, untracked paths). Raises if the repo is unobservable or has tracked changes.""" |
| 111 | try: |
| 112 | revision = validate_revision(muse_json(repo, ["rev-parse", "HEAD"], runner).get("commit_id")) |
| 113 | status = muse_json(repo, ["status"], runner) |
| 114 | except ProvenanceError as exc: |
| 115 | raise BuildContextError(f"{repo.name}: cannot determine the revision with Muse: {exc}") from exc |
| 116 | if status.get("head_commit") != revision: |
| 117 | raise BuildContextError(f"{repo.name}: rev-parse and status disagree about HEAD") |
| 118 | changes = _tracked_changes(status) |
| 119 | if changes: |
| 120 | raise BuildContextError( |
| 121 | f"{repo.name} has tracked changes that are not in {revision}; the image would not be that revision. " |
| 122 | f"Commit or discard them first: {', '.join(changes[:8])}" |
| 123 | ) |
| 124 | untracked = status.get("untracked") |
| 125 | if not isinstance(untracked, list): |
| 126 | raise BuildContextError("muse status --json: 'untracked' missing or not a list") |
| 127 | return revision, sorted(str(p) for p in untracked) |
| 128 | |
| 129 | |
| 130 | def _export(repo: Path, revision: str, out: Path, scratch: Path, runner: Runner) -> list[str]: |
| 131 | archive = scratch / f"{repo.name}.tar.gz" |
| 132 | argv = ["muse", "-C", str(repo), "archive", "--ref", revision, "--format", "tar.gz", "--output", str(archive), "--prefix", repo.name] |
| 133 | try: |
| 134 | proc = runner(argv, capture_output=True, text=True, timeout=600, check=False) |
| 135 | except (OSError, subprocess.SubprocessError) as exc: |
| 136 | raise BuildContextError(f"{repo.name}: muse archive could not run: {exc}") from exc |
| 137 | if proc.returncode != 0 or not archive.is_file(): |
| 138 | raise BuildContextError(f"{repo.name}: muse archive failed ({proc.returncode}): {(proc.stderr or proc.stdout or '').strip()[:300]}") |
| 139 | try: |
| 140 | with tarfile.open(archive) as tf: |
| 141 | names = [m.name for m in tf.getmembers() if m.isfile()] |
| 142 | tf.extractall(out, filter="data") # rejects absolute paths, '..', links outside the target |
| 143 | except (tarfile.TarError, OSError) as exc: |
| 144 | raise BuildContextError(f"{repo.name}: cannot extract the archive: {exc}") from exc |
| 145 | return names |
| 146 | |
| 147 | |
| 148 | def _check_required(out: Path) -> None: |
| 149 | missing = [p for p in REQUIRED if not (out / p).exists()] |
| 150 | if missing: |
| 151 | raise BuildContextError(f"required build inputs are missing from the exported tree: {', '.join(missing)}") |
| 152 | if not any((out / MIGRATIONS).glob("*.py")): |
| 153 | raise BuildContextError(f"{MIGRATIONS} contains no migrations") |
| 154 | |
| 155 | |
| 156 | def _check_migrations(repo: Path, out: Path, runner: Runner) -> None: |
| 157 | """Every tracked migration is in the context (a truncated export would deploy a schema the code outruns).""" |
| 158 | try: |
| 159 | listed = muse_json(repo, ["ls-files"], runner) |
| 160 | except ProvenanceError as exc: |
| 161 | raise BuildContextError(f"musehub: cannot list tracked files: {exc}") from exc |
| 162 | files = listed.get("files") or listed.get("paths") or [] |
| 163 | paths = [f if isinstance(f, str) else str(f.get("path", "")) for f in files] |
| 164 | expected = sorted(Path(p).name for p in paths if p.startswith("alembic/versions/") and p.endswith(".py")) |
| 165 | got = sorted(p.name for p in (out / MIGRATIONS).glob("*.py")) |
| 166 | if not expected: |
| 167 | raise BuildContextError("could not read the tracked migration list from muse ls-files") |
| 168 | if expected != got: |
| 169 | raise BuildContextError(f"exported migrations differ from the tracked ones: missing {sorted(set(expected) - set(got))[:5]}") |
| 170 | |
| 171 | |
| 172 | def _npm() -> str | None: |
| 173 | return shutil.which("npm") |
| 174 | |
| 175 | |
| 176 | def _build_frontend(out: Path, scratch: Path, runner: Runner) -> dict[str, dict[str, Any]]: |
| 177 | """Build every ``FRONTEND_ASSETS`` entry from the exported sources in *scratch*; copy only the outputs into *out*.""" |
| 178 | npm = _npm() |
| 179 | if npm is None: |
| 180 | raise BuildContextError("npm is not available; the compiled frontend assets cannot be built") |
| 181 | work = scratch / "frontend" |
| 182 | work.mkdir() |
| 183 | for name in FRONTEND_INPUTS: |
| 184 | src = out / "musehub" / name |
| 185 | if src.is_dir(): |
| 186 | shutil.copytree(src, work / name) |
| 187 | elif src.is_file(): |
| 188 | shutil.copy2(src, work / name) |
| 189 | steps = [[npm, "ci", "--ignore-scripts", "--no-audit", "--no-fund"]] + [[npm, "run", "--silent", script] for script in FRONTEND_ASSETS.values()] |
| 190 | for argv in steps: |
| 191 | try: |
| 192 | proc = runner(argv, capture_output=True, text=True, timeout=900, check=False, cwd=work) |
| 193 | except (OSError, subprocess.SubprocessError) as exc: |
| 194 | raise BuildContextError(f"frontend build could not run {' '.join(argv[1:])}: {exc}") from exc |
| 195 | if proc.returncode != 0: |
| 196 | raise BuildContextError(f"frontend build failed ({' '.join(argv[1:])}, exit {proc.returncode}): {(proc.stderr or proc.stdout or '').strip()[-300:]}") |
| 197 | built = work / Path(STATIC_DIR).relative_to("musehub") |
| 198 | assets: dict[str, dict[str, Any]] = {} |
| 199 | for name in FRONTEND_ASSETS: |
| 200 | data = (built / name).read_bytes() if (built / name).is_file() else b"" |
| 201 | if not data: |
| 202 | raise BuildContextError(f"frontend build produced no {name} (missing or empty)") |
| 203 | (out / STATIC_DIR).mkdir(parents=True, exist_ok=True) |
| 204 | (out / STATIC_DIR / name).write_bytes(data) |
| 205 | assets[name] = {"sha256": hashlib.sha256(data).hexdigest(), "bytes": len(data)} |
| 206 | return assets |
| 207 | |
| 208 | |
| 209 | def assemble(ecosystem: Path, out: Path, *, runner: Runner = subprocess.run) -> dict[str, Any]: |
| 210 | """Create *out* holding exactly the committed files of every repository in ``REPOS``. Returns build-info.""" |
| 211 | if out.exists(): |
| 212 | raise BuildContextError(f"{out} already exists; build contexts are never reused") |
| 213 | repos = {name: ecosystem / name for name in REPOS} |
| 214 | for name, path in repos.items(): |
| 215 | if not path.is_dir(): |
| 216 | raise BuildContextError(f"repository {name} not found at {path}") |
| 217 | inspected = {name: _inspect(path, runner) for name, path in repos.items()} # all checks before anything is written |
| 218 | |
| 219 | out.mkdir(parents=True) |
| 220 | try: |
| 221 | with tempfile.TemporaryDirectory(prefix="muse-ctx-") as scratch: |
| 222 | for name, path in repos.items(): |
| 223 | _export(path, inspected[name][0], out, Path(scratch), runner) |
| 224 | _check_required(out) |
| 225 | _check_migrations(repos["musehub"], out, runner) |
| 226 | assets = _build_frontend(out, Path(scratch), runner) |
| 227 | info = { |
| 228 | "schema": 1, |
| 229 | "repositories": {name: {"revision": rev, "untracked_excluded": untracked} for name, (rev, untracked) in sorted(inspected.items())}, |
| 230 | "frontend_assets": assets, |
| 231 | } |
| 232 | (out / "build-info.json").write_text(json.dumps(info, indent=2, sort_keys=True) + "\n") |
| 233 | except BaseException: |
| 234 | shutil.rmtree(out, ignore_errors=True) # never leave a half-assembled context to be mistaken for a good one |
| 235 | raise |
| 236 | return info |
| 237 | |
| 238 | |
| 239 | def main(argv: list[str] | None = None) -> int: |
| 240 | parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) |
| 241 | parser.add_argument("--ecosystem", type=Path, default=Path.home() / "ecosystem") |
| 242 | parser.add_argument("--out", type=Path, required=True) |
| 243 | args = parser.parse_args(argv) |
| 244 | try: |
| 245 | info = assemble(args.ecosystem, args.out) |
| 246 | except BuildContextError as exc: |
| 247 | print(f"BUILD CONTEXT REFUSED: {exc}", file=sys.stderr) |
| 248 | return 1 |
| 249 | print(json.dumps(info, indent=2, sort_keys=True)) |
| 250 | return 0 |
| 251 | |
| 252 | |
| 253 | if __name__ == "__main__": |
| 254 | raise SystemExit(main()) |
File History
5 commits
sha256:b4b999434d042d74edc7032f1f2cfd20fda5b3580fe85966351db5b8eef0abb3
feat(intel): every grammar on the hub, and every intel job …
Opus 5.5
minor
⚠
1 day ago
sha256:414893924229cfb6dd5a5154d9526c1a9b57a2721289daba6ad89a6c98c64c09
fix(deploy): stream the Docker build context so same-size e…
Sonnet 5.5
patch
6 days ago
sha256:70e2deebbf676f002c5122f381222eb21fe65b51c04380538ea9acb3fe62dfc5
build: install the verified dependency versions instead of …
Opus 5.5
minor
⚠
11 days ago
sha256:6b75088fadf04c065f8c92b3b71f8c7ffe315d0e41ab71e4dc7b0b9e99f29426
fix(deploy): build the compiled UI assets into the pinned b…
Opus 5.5
minor
⚠
12 days ago
sha256:3c6d2e2751e9222a0872ba699f16baa457d9990c1998c20956badb9186ee69ee
fix(deploy): image built from committed files only, with re…
Opus 5.5
minor
⚠
12 days ago