gabriel / musehub public
build_context.py python
254 lines 12.1 KB
Raw
sha256:b4b999434d042d74edc7032f1f2cfd20fda5b3580fe85966351db5b8eef0abb3 feat(intel): every grammar on the hub, and every intel job … Opus 5.5 minor ⚠ breaking 1 day ago
1 #!/usr/bin/env python3
2 """Assemble the Docker build context from *committed* files only (musehub#114 Phase 6D).
3
4 ``push.sh`` used to hand Docker the whole ``~/ecosystem`` working tree, so any untracked local file under
5 a directory the Dockerfile copies (for example ``docs/work/session-handoff.md``) ended up in the deployed
6 image, and the image said nothing about which revision it was built from.
7
8 This builds the context instead from ``muse archive`` of each repository the image needs, so untracked
9 and ignored files are not merely filtered out -- they are never present::
10
11 python -m deploy.build_context --ecosystem ~/ecosystem --out /tmp/ctx
12
13 The compiled UI assets the app serves from ``/static`` (``FRONTEND_ASSETS``) are build outputs, ignored by
14 ``.museignore``, so no export contains them. They are built here from the exported sources -- ``npm ci``
15 from the committed lockfile, then one build script per asset, in a scratch directory -- and only the
16 outputs are copied into the context. The working tree, its ``node_modules`` and any locally built assets
17 are never used.
18
19 It refuses, before producing anything, when
20
21 * a repository has tracked changes (the image would then differ from the revision it claims; untracked
22 files are reported as excluded, never copied),
23 * the revision cannot be determined by Muse (no Git metadata is consulted),
24 * a required input is missing from the exported tree (``REQUIRED``), or the exported tree lacks a
25 migration the repository tracks,
26 * npm is unavailable, or a compiled asset cannot be built or comes out empty.
27
28 It writes ``build-info.json`` (the revision of every repository, the excluded untracked paths, the
29 SHA-256 and size of every compiled asset) in the context root; ``push.sh`` passes the revisions to
30 ``docker build`` as ``--build-arg`` and verifies the resulting image's labels. The output directory must
31 not already exist: contexts are never reused.
32 """
33 from __future__ import annotations
34
35 import argparse
36 import hashlib
37 import json
38 import shutil
39 import subprocess
40 import sys
41 import tarfile
42 import tempfile
43 from collections.abc import Callable
44 from pathlib import Path
45 from typing import Any
46
47 from deploy.release_provenance import ProvenanceError, muse_json, validate_revision
48
49 Runner = Callable[..., "subprocess.CompletedProcess[str]"]
50
51 #: repositories the image is built from, in Dockerfile order
52 REPOS: tuple[str, ...] = ("musehub", "muse")
53
54 #: files and directories that must exist in the exported tree, relative to the context root
55 REQUIRED: tuple[str, ...] = (
56 "musehub/Dockerfile",
57 "musehub/Dockerfile.dockerignore",
58 "musehub/requirements.txt",
59 "musehub/constraints.txt",
60 "musehub/pyproject.toml",
61 "musehub/alembic.ini",
62 "musehub/entrypoint.sh",
63 "musehub/musehub/__init__.py",
64 "musehub/alembic/env.py",
65 "musehub/deploy/deploy.sh",
66 "musehub/docs",
67 "musehub/package.json",
68 "musehub/package-lock.json",
69 "musehub/src",
70 "muse/pyproject.toml",
71 "muse/README.md",
72 "muse/muse/__init__.py",
73 )
74 MIGRATIONS = "musehub/alembic/versions"
75
76 #: where the app serves /static from, relative to the context root (the Dockerfile copies musehub/musehub/)
77 STATIC_DIR = "musehub/musehub/templates/musehub/static"
78 #: compiled UI asset -> the package.json script that builds it (into musehub/templates/musehub/static/)
79 FRONTEND_ASSETS: dict[str, str] = {"app.js": "build:js", "app.css": "build:css", "embed.css": "build:embed"}
80 #: committed files the frontend build reads, relative to the musehub repository root
81 FRONTEND_INPUTS: tuple[str, ...] = ("package.json", "package-lock.json", "tsconfig.json", "src")
82
83
84 class BuildContextError(Exception):
85 """The build context cannot be assembled faithfully. Nothing is built."""
86
87
88 def _tracked_changes(status: dict[str, Any]) -> list[str]:
89 changes: list[str] = []
90 for key in ("added", "modified", "deleted", "conflict_paths"):
91 value = status.get(key)
92 if not isinstance(value, list):
93 raise BuildContextError(f"muse status --json: {key!r} missing or not a list")
94 changes += [str(p) for p in value]
95 changes += [f"{a} -> {b}" for a, b in (status.get("renamed") or {}).items()]
96 for stage in ("staged", "unstaged"):
97 block = status.get(stage)
98 if not isinstance(block, dict):
99 raise BuildContextError(f"muse status --json: {stage!r} missing or not an object")
100 for key in ("added", "modified", "deleted"):
101 changes += [str(p) for p in block.get(key, [])]
102 changes += [f"{a} -> {b}" for a, b in (block.get("renamed") or {}).items()]
103 for flag in ("merge_in_progress", "checkout_interrupted"):
104 if status.get(flag) is not False:
105 changes.append(f"<{flag}>")
106 return list(dict.fromkeys(changes))
107
108
109 def _inspect(repo: Path, runner: Runner) -> tuple[str, list[str]]:
110 """(revision, untracked paths). Raises if the repo is unobservable or has tracked changes."""
111 try:
112 revision = validate_revision(muse_json(repo, ["rev-parse", "HEAD"], runner).get("commit_id"))
113 status = muse_json(repo, ["status"], runner)
114 except ProvenanceError as exc:
115 raise BuildContextError(f"{repo.name}: cannot determine the revision with Muse: {exc}") from exc
116 if status.get("head_commit") != revision:
117 raise BuildContextError(f"{repo.name}: rev-parse and status disagree about HEAD")
118 changes = _tracked_changes(status)
119 if changes:
120 raise BuildContextError(
121 f"{repo.name} has tracked changes that are not in {revision}; the image would not be that revision. "
122 f"Commit or discard them first: {', '.join(changes[:8])}"
123 )
124 untracked = status.get("untracked")
125 if not isinstance(untracked, list):
126 raise BuildContextError("muse status --json: 'untracked' missing or not a list")
127 return revision, sorted(str(p) for p in untracked)
128
129
130 def _export(repo: Path, revision: str, out: Path, scratch: Path, runner: Runner) -> list[str]:
131 archive = scratch / f"{repo.name}.tar.gz"
132 argv = ["muse", "-C", str(repo), "archive", "--ref", revision, "--format", "tar.gz", "--output", str(archive), "--prefix", repo.name]
133 try:
134 proc = runner(argv, capture_output=True, text=True, timeout=600, check=False)
135 except (OSError, subprocess.SubprocessError) as exc:
136 raise BuildContextError(f"{repo.name}: muse archive could not run: {exc}") from exc
137 if proc.returncode != 0 or not archive.is_file():
138 raise BuildContextError(f"{repo.name}: muse archive failed ({proc.returncode}): {(proc.stderr or proc.stdout or '').strip()[:300]}")
139 try:
140 with tarfile.open(archive) as tf:
141 names = [m.name for m in tf.getmembers() if m.isfile()]
142 tf.extractall(out, filter="data") # rejects absolute paths, '..', links outside the target
143 except (tarfile.TarError, OSError) as exc:
144 raise BuildContextError(f"{repo.name}: cannot extract the archive: {exc}") from exc
145 return names
146
147
148 def _check_required(out: Path) -> None:
149 missing = [p for p in REQUIRED if not (out / p).exists()]
150 if missing:
151 raise BuildContextError(f"required build inputs are missing from the exported tree: {', '.join(missing)}")
152 if not any((out / MIGRATIONS).glob("*.py")):
153 raise BuildContextError(f"{MIGRATIONS} contains no migrations")
154
155
156 def _check_migrations(repo: Path, out: Path, runner: Runner) -> None:
157 """Every tracked migration is in the context (a truncated export would deploy a schema the code outruns)."""
158 try:
159 listed = muse_json(repo, ["ls-files"], runner)
160 except ProvenanceError as exc:
161 raise BuildContextError(f"musehub: cannot list tracked files: {exc}") from exc
162 files = listed.get("files") or listed.get("paths") or []
163 paths = [f if isinstance(f, str) else str(f.get("path", "")) for f in files]
164 expected = sorted(Path(p).name for p in paths if p.startswith("alembic/versions/") and p.endswith(".py"))
165 got = sorted(p.name for p in (out / MIGRATIONS).glob("*.py"))
166 if not expected:
167 raise BuildContextError("could not read the tracked migration list from muse ls-files")
168 if expected != got:
169 raise BuildContextError(f"exported migrations differ from the tracked ones: missing {sorted(set(expected) - set(got))[:5]}")
170
171
172 def _npm() -> str | None:
173 return shutil.which("npm")
174
175
176 def _build_frontend(out: Path, scratch: Path, runner: Runner) -> dict[str, dict[str, Any]]:
177 """Build every ``FRONTEND_ASSETS`` entry from the exported sources in *scratch*; copy only the outputs into *out*."""
178 npm = _npm()
179 if npm is None:
180 raise BuildContextError("npm is not available; the compiled frontend assets cannot be built")
181 work = scratch / "frontend"
182 work.mkdir()
183 for name in FRONTEND_INPUTS:
184 src = out / "musehub" / name
185 if src.is_dir():
186 shutil.copytree(src, work / name)
187 elif src.is_file():
188 shutil.copy2(src, work / name)
189 steps = [[npm, "ci", "--ignore-scripts", "--no-audit", "--no-fund"]] + [[npm, "run", "--silent", script] for script in FRONTEND_ASSETS.values()]
190 for argv in steps:
191 try:
192 proc = runner(argv, capture_output=True, text=True, timeout=900, check=False, cwd=work)
193 except (OSError, subprocess.SubprocessError) as exc:
194 raise BuildContextError(f"frontend build could not run {' '.join(argv[1:])}: {exc}") from exc
195 if proc.returncode != 0:
196 raise BuildContextError(f"frontend build failed ({' '.join(argv[1:])}, exit {proc.returncode}): {(proc.stderr or proc.stdout or '').strip()[-300:]}")
197 built = work / Path(STATIC_DIR).relative_to("musehub")
198 assets: dict[str, dict[str, Any]] = {}
199 for name in FRONTEND_ASSETS:
200 data = (built / name).read_bytes() if (built / name).is_file() else b""
201 if not data:
202 raise BuildContextError(f"frontend build produced no {name} (missing or empty)")
203 (out / STATIC_DIR).mkdir(parents=True, exist_ok=True)
204 (out / STATIC_DIR / name).write_bytes(data)
205 assets[name] = {"sha256": hashlib.sha256(data).hexdigest(), "bytes": len(data)}
206 return assets
207
208
209 def assemble(ecosystem: Path, out: Path, *, runner: Runner = subprocess.run) -> dict[str, Any]:
210 """Create *out* holding exactly the committed files of every repository in ``REPOS``. Returns build-info."""
211 if out.exists():
212 raise BuildContextError(f"{out} already exists; build contexts are never reused")
213 repos = {name: ecosystem / name for name in REPOS}
214 for name, path in repos.items():
215 if not path.is_dir():
216 raise BuildContextError(f"repository {name} not found at {path}")
217 inspected = {name: _inspect(path, runner) for name, path in repos.items()} # all checks before anything is written
218
219 out.mkdir(parents=True)
220 try:
221 with tempfile.TemporaryDirectory(prefix="muse-ctx-") as scratch:
222 for name, path in repos.items():
223 _export(path, inspected[name][0], out, Path(scratch), runner)
224 _check_required(out)
225 _check_migrations(repos["musehub"], out, runner)
226 assets = _build_frontend(out, Path(scratch), runner)
227 info = {
228 "schema": 1,
229 "repositories": {name: {"revision": rev, "untracked_excluded": untracked} for name, (rev, untracked) in sorted(inspected.items())},
230 "frontend_assets": assets,
231 }
232 (out / "build-info.json").write_text(json.dumps(info, indent=2, sort_keys=True) + "\n")
233 except BaseException:
234 shutil.rmtree(out, ignore_errors=True) # never leave a half-assembled context to be mistaken for a good one
235 raise
236 return info
237
238
239 def main(argv: list[str] | None = None) -> int:
240 parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
241 parser.add_argument("--ecosystem", type=Path, default=Path.home() / "ecosystem")
242 parser.add_argument("--out", type=Path, required=True)
243 args = parser.parse_args(argv)
244 try:
245 info = assemble(args.ecosystem, args.out)
246 except BuildContextError as exc:
247 print(f"BUILD CONTEXT REFUSED: {exc}", file=sys.stderr)
248 return 1
249 print(json.dumps(info, indent=2, sort_keys=True))
250 return 0
251
252
253 if __name__ == "__main__":
254 raise SystemExit(main())
File History 5 commits
sha256:b4b999434d042d74edc7032f1f2cfd20fda5b3580fe85966351db5b8eef0abb3 feat(intel): every grammar on the hub, and every intel job … Opus 5.5 minor ⚠ 1 day ago
sha256:414893924229cfb6dd5a5154d9526c1a9b57a2721289daba6ad89a6c98c64c09 fix(deploy): stream the Docker build context so same-size e… Sonnet 5.5 patch 6 days ago
sha256:70e2deebbf676f002c5122f381222eb21fe65b51c04380538ea9acb3fe62dfc5 build: install the verified dependency versions instead of … Opus 5.5 minor ⚠ 11 days ago
sha256:6b75088fadf04c065f8c92b3b71f8c7ffe315d0e41ab71e4dc7b0b9e99f29426 fix(deploy): build the compiled UI assets into the pinned b… Opus 5.5 minor ⚠ 12 days ago
sha256:3c6d2e2751e9222a0872ba699f16baa457d9990c1998c20956badb9186ee69ee fix(deploy): image built from committed files only, with re… Opus 5.5 minor ⚠ 12 days ago