intel_experiment.sh
bash
sha256:64d53876b5299082b6b4bfbe92721976ec7bb5f1d9a88fdf6eafc1490148d27c
fix(intel): explicit, fail-closed deployment identity repla…
Sonnet 5.5
minor
⚠ breaking
4 days ago
| 1 | #!/usr/bin/env bash |
| 2 | # Localhost-only intel.code experiment runner. See musehub/intel_experiment.py for the guards it enforces. |
| 3 | # |
| 4 | # MUSEHUB_INTEL_EXPERIMENT=1 scripts/intel_experiment.sh make-fixture <intel-fixture-slug> [commits] |
| 5 | # MUSEHUB_INTEL_EXPERIMENT=1 scripts/intel_experiment.sh run <repo_id> # intel.code only |
| 6 | # MUSEHUB_INTEL_EXPERIMENT=1 scripts/intel_experiment.sh run-all <repo_id> # every provider, one at a time, per-provider time/memory |
| 7 | # |
| 8 | # Runs ONE short-lived container from the worker's own image with this checkout's `musehub/` mounted read-only, so |
| 9 | # neither the live API nor the live worker is touched, restarted or rebuilt. The limits are real, enforced by the |
| 10 | # container runtime and not by the Python process being limited: |
| 11 | # INTEL_EXP_MEMORY hard memory cgroup limit, swap disabled (default 1g) |
| 12 | # INTEL_EXP_CPUS CPU quota (default 1) |
| 13 | # INTEL_EXP_TIMEOUT seconds; the child arms SIGALRM, and this script kills the container 10 s later (default 300) |
| 14 | # The container name is fixed, so Docker itself refuses a second concurrent run. |
| 15 | set -euo pipefail |
| 16 | |
| 17 | if [ "${MUSEHUB_INTEL_EXPERIMENT:-}" != "1" ]; then |
| 18 | echo "refused: disabled by default; set MUSEHUB_INTEL_EXPERIMENT=1 (localhost experiment only)" >&2 |
| 19 | exit 2 |
| 20 | fi |
| 21 | cmd="${1:-}"; arg="${2:-}" |
| 22 | case "$cmd" in |
| 23 | run|run-all) case "$arg" in sha256:*) ;; *) echo "refused: $cmd needs an explicit repo_id (sha256:...)" >&2; exit 2;; esac |
| 24 | child_args=("$cmd" "$arg" --timeout "${INTEL_EXP_TIMEOUT:-300}") ;; |
| 25 | make-fixture) [ -n "$arg" ] || { echo "refused: make-fixture needs a slug" >&2; exit 2; } |
| 26 | child_args=(make-fixture "$arg" --commits "${3:-40}") ;; |
| 27 | *) echo "usage: $0 run <repo_id> | run-all <repo_id> | make-fixture <slug> [commits]" >&2; exit 2 ;; |
| 28 | esac |
| 29 | |
| 30 | MEM="${INTEL_EXP_MEMORY:-1g}"; CPUS="${INTEL_EXP_CPUS:-1}"; TIMEOUT="${INTEL_EXP_TIMEOUT:-300}" |
| 31 | ROOT="$(cd "$(dirname "$0")/.." && pwd)" |
| 32 | NAME="musehub_intel_exp" |
| 33 | OUT="${INTEL_EXP_OUT:-$ROOT/.intel_experiment_out}" |
| 34 | mkdir -p "$OUT" |
| 35 | stamp="$(date -u +%Y%m%dT%H%M%SZ)" |
| 36 | |
| 37 | IMAGE="$(docker inspect musehub_worker --format '{{.Image}}')" |
| 38 | DATABASE_URL="$(docker exec musehub_worker printenv DATABASE_URL)" # the secret never appears on a command line or in output |
| 39 | export DATABASE_URL |
| 40 | # run-all reads snapshot objects (mpack byte ranges / blobs) from the LOCAL object store the worker already uses. Same rule as the database |
| 41 | # URL: the values never appear on a command line or in output, and a non-local endpoint is refused. |
| 42 | for v in BLOB_STORAGE_ENDPOINT BLOB_STORAGE_BUCKET BLOB_STORAGE_REGION BLOB_STORAGE_ACCESS_KEY_ID BLOB_STORAGE_SECRET_ACCESS_KEY; do |
| 43 | export "$v=$(docker exec musehub_worker printenv "$v" 2>/dev/null || true)" |
| 44 | done |
| 45 | case "${BLOB_STORAGE_ENDPOINT:-}" in ""|http://minio:*|http://localhost:*|http://127.0.0.1:*|http://host.docker.internal:*) ;; *) echo "refused: object store endpoint is not local" >&2; exit 2;; esac |
| 46 | case "$DATABASE_URL" in *@postgres:5432/*) ;; *) echo "refused: worker database is not the compose 'postgres' service" >&2; exit 2;; esac |
| 47 | |
| 48 | echo "image=$IMAGE memory=$MEM cpus=$CPUS timeout=${TIMEOUT}s source=$ROOT/musehub" >&2 |
| 49 | # --init is load-bearing: as PID 1 the Python child would silently discard SIGALRM (Linux drops unhandled signals sent |
| 50 | # to a namespace's init), so its hard stop would never fire. Found by a live 1-second-timeout test, 2026-11-13. |
| 51 | docker run -d --init --name "$NAME" --network musehub_musehub-internal \ |
| 52 | --memory "$MEM" --memory-swap "$MEM" --cpus "$CPUS" --pids-limit 128 \ |
| 53 | --read-only --tmpfs /tmp --cap-drop ALL --security-opt no-new-privileges \ |
| 54 | -v "$ROOT/musehub:/app/musehub:ro" \ |
| 55 | -e DATABASE_URL -e DEBUG=false -e MUSEHUB_INTEL_EXPERIMENT=1 -e MUSEHUB_DEPLOYMENT=local \ |
| 56 | -e BLOB_STORAGE_ENDPOINT -e BLOB_STORAGE_BUCKET -e BLOB_STORAGE_REGION -e BLOB_STORAGE_ACCESS_KEY_ID -e BLOB_STORAGE_SECRET_ACCESS_KEY \ |
| 57 | -e MUSEHUB_INTEL_EXPERIMENT_MAX_COMMITS \ |
| 58 | "$IMAGE" python -m musehub.intel_experiment "${child_args[@]}" >/dev/null |
| 59 | |
| 60 | peak_mib=0; start=$SECONDS; killed_by="" |
| 61 | while [ "$(docker inspect "$NAME" --format '{{.State.Running}}')" = "true" ]; do |
| 62 | usage="$(docker stats --no-stream --format '{{.MemUsage}}' "$NAME" 2>/dev/null | awk '{print $1}')" || usage="" |
| 63 | mib="$(printf '%s' "$usage" | awk '/GiB/{sub("GiB","");printf "%d",$0*1024;next} /MiB/{sub("MiB","");printf "%d",$0;next} /KiB/{sub("KiB","");printf "%d",$0/1024;next} {print 0}')" |
| 64 | [ "${mib:-0}" -gt "$peak_mib" ] && peak_mib="$mib" |
| 65 | if [ $((SECONDS - start)) -ge $((TIMEOUT + 10)) ]; then killed_by="host-timeout"; docker kill "$NAME" >/dev/null; break; fi |
| 66 | sleep 1 |
| 67 | done |
| 68 | elapsed=$((SECONDS - start)) |
| 69 | |
| 70 | state="$(docker inspect "$NAME" --format '{{.State.ExitCode}} {{.State.OOMKilled}}')" |
| 71 | exit_code="${state% *}"; oom="${state#* }" |
| 72 | docker logs "$NAME" >"$OUT/$stamp.stdout" 2>"$OUT/$stamp.stderr" || true |
| 73 | docker rm "$NAME" >/dev/null |
| 74 | |
| 75 | job_id="$(grep -o '"job_id": "[^"]*"' "$OUT/$stamp.stderr" | head -1 | sed -E 's/.*: "([^"]*)"/\1/' || true)" |
| 76 | reason="" |
| 77 | [ "$oom" = "true" ] && reason="oom-killed (memory limit $MEM)" |
| 78 | [ "$killed_by" = "host-timeout" ] && reason="killed by host after ${TIMEOUT}+10 s" |
| 79 | [ -z "$reason" ] && [ "$exit_code" = "142" ] && reason="SIGALRM timeout (${TIMEOUT}s)" |
| 80 | [ -z "$reason" ] && [ "$exit_code" != "0" ] && reason="exit code $exit_code" |
| 81 | |
| 82 | # A killed child cannot record its own death. Mark ITS job (and only a still-running one) failed; never 'done'. |
| 83 | if [ -n "$reason" ] && [ -n "$job_id" ]; then |
| 84 | docker exec musehub_postgres psql -U musehub -d musehub -v ON_ERROR_STOP=1 -q -c \ |
| 85 | "UPDATE musehub_background_jobs SET status='failed', done_at=now(), error='$reason', |
| 86 | result=jsonb_build_object('executed', false, 'failed', true, 'reason', '$reason') |
| 87 | WHERE job_id='$job_id' AND status='running' AND payload->>'experiment'='intel-code-localhost-v1';" >&2 |
| 88 | fi |
| 89 | |
| 90 | echo "--- result (exit=$exit_code oom=$oom wall=${elapsed}s sampled_container_peak=${peak_mib}MiB reason='${reason:-none}') ---" |
| 91 | tail -n 1 "$OUT/$stamp.stdout" |
| 92 | echo "logs: $OUT/$stamp.stdout $OUT/$stamp.stderr" >&2 |
| 93 | exit "$exit_code" |
File History
3 commits
sha256:64d53876b5299082b6b4bfbe92721976ec7bb5f1d9a88fdf6eafc1490148d27c
fix(intel): explicit, fail-closed deployment identity repla…
Sonnet 5.5
minor
⚠
4 days ago
sha256:78744d379faf9e46bfc8a45b2998d385b92f232df4fd33a1d06bfe62efa0b1d2
feat(intel): run-all measures every provider through the pr…
Sonnet 5.5
patch
4 days ago
sha256:0a96f5e99fe9a92a356e7503f94752135aa45cd0fe8c657fcb403422cb30ff43
feat(intel): localhost-only bounded intel.code experiment (…
Sonnet 5.5
patch
4 days ago