gabriel / musehub public
intel_experiment.sh bash
93 lines 6.0 KB
Raw
sha256:64d53876b5299082b6b4bfbe92721976ec7bb5f1d9a88fdf6eafc1490148d27c fix(intel): explicit, fail-closed deployment identity repla… Sonnet 5.5 minor ⚠ breaking 4 days ago
1 #!/usr/bin/env bash
2 # Localhost-only intel.code experiment runner. See musehub/intel_experiment.py for the guards it enforces.
3 #
4 # MUSEHUB_INTEL_EXPERIMENT=1 scripts/intel_experiment.sh make-fixture <intel-fixture-slug> [commits]
5 # MUSEHUB_INTEL_EXPERIMENT=1 scripts/intel_experiment.sh run <repo_id> # intel.code only
6 # MUSEHUB_INTEL_EXPERIMENT=1 scripts/intel_experiment.sh run-all <repo_id> # every provider, one at a time, per-provider time/memory
7 #
8 # Runs ONE short-lived container from the worker's own image with this checkout's `musehub/` mounted read-only, so
9 # neither the live API nor the live worker is touched, restarted or rebuilt. The limits are real, enforced by the
10 # container runtime and not by the Python process being limited:
11 # INTEL_EXP_MEMORY hard memory cgroup limit, swap disabled (default 1g)
12 # INTEL_EXP_CPUS CPU quota (default 1)
13 # INTEL_EXP_TIMEOUT seconds; the child arms SIGALRM, and this script kills the container 10 s later (default 300)
14 # The container name is fixed, so Docker itself refuses a second concurrent run.
15 set -euo pipefail
16
17 if [ "${MUSEHUB_INTEL_EXPERIMENT:-}" != "1" ]; then
18 echo "refused: disabled by default; set MUSEHUB_INTEL_EXPERIMENT=1 (localhost experiment only)" >&2
19 exit 2
20 fi
21 cmd="${1:-}"; arg="${2:-}"
22 case "$cmd" in
23 run|run-all) case "$arg" in sha256:*) ;; *) echo "refused: $cmd needs an explicit repo_id (sha256:...)" >&2; exit 2;; esac
24 child_args=("$cmd" "$arg" --timeout "${INTEL_EXP_TIMEOUT:-300}") ;;
25 make-fixture) [ -n "$arg" ] || { echo "refused: make-fixture needs a slug" >&2; exit 2; }
26 child_args=(make-fixture "$arg" --commits "${3:-40}") ;;
27 *) echo "usage: $0 run <repo_id> | run-all <repo_id> | make-fixture <slug> [commits]" >&2; exit 2 ;;
28 esac
29
30 MEM="${INTEL_EXP_MEMORY:-1g}"; CPUS="${INTEL_EXP_CPUS:-1}"; TIMEOUT="${INTEL_EXP_TIMEOUT:-300}"
31 ROOT="$(cd "$(dirname "$0")/.." && pwd)"
32 NAME="musehub_intel_exp"
33 OUT="${INTEL_EXP_OUT:-$ROOT/.intel_experiment_out}"
34 mkdir -p "$OUT"
35 stamp="$(date -u +%Y%m%dT%H%M%SZ)"
36
37 IMAGE="$(docker inspect musehub_worker --format '{{.Image}}')"
38 DATABASE_URL="$(docker exec musehub_worker printenv DATABASE_URL)" # the secret never appears on a command line or in output
39 export DATABASE_URL
40 # run-all reads snapshot objects (mpack byte ranges / blobs) from the LOCAL object store the worker already uses. Same rule as the database
41 # URL: the values never appear on a command line or in output, and a non-local endpoint is refused.
42 for v in BLOB_STORAGE_ENDPOINT BLOB_STORAGE_BUCKET BLOB_STORAGE_REGION BLOB_STORAGE_ACCESS_KEY_ID BLOB_STORAGE_SECRET_ACCESS_KEY; do
43 export "$v=$(docker exec musehub_worker printenv "$v" 2>/dev/null || true)"
44 done
45 case "${BLOB_STORAGE_ENDPOINT:-}" in ""|http://minio:*|http://localhost:*|http://127.0.0.1:*|http://host.docker.internal:*) ;; *) echo "refused: object store endpoint is not local" >&2; exit 2;; esac
46 case "$DATABASE_URL" in *@postgres:5432/*) ;; *) echo "refused: worker database is not the compose 'postgres' service" >&2; exit 2;; esac
47
48 echo "image=$IMAGE memory=$MEM cpus=$CPUS timeout=${TIMEOUT}s source=$ROOT/musehub" >&2
49 # --init is load-bearing: as PID 1 the Python child would silently discard SIGALRM (Linux drops unhandled signals sent
50 # to a namespace's init), so its hard stop would never fire. Found by a live 1-second-timeout test, 2026-11-13.
51 docker run -d --init --name "$NAME" --network musehub_musehub-internal \
52 --memory "$MEM" --memory-swap "$MEM" --cpus "$CPUS" --pids-limit 128 \
53 --read-only --tmpfs /tmp --cap-drop ALL --security-opt no-new-privileges \
54 -v "$ROOT/musehub:/app/musehub:ro" \
55 -e DATABASE_URL -e DEBUG=false -e MUSEHUB_INTEL_EXPERIMENT=1 -e MUSEHUB_DEPLOYMENT=local \
56 -e BLOB_STORAGE_ENDPOINT -e BLOB_STORAGE_BUCKET -e BLOB_STORAGE_REGION -e BLOB_STORAGE_ACCESS_KEY_ID -e BLOB_STORAGE_SECRET_ACCESS_KEY \
57 -e MUSEHUB_INTEL_EXPERIMENT_MAX_COMMITS \
58 "$IMAGE" python -m musehub.intel_experiment "${child_args[@]}" >/dev/null
59
60 peak_mib=0; start=$SECONDS; killed_by=""
61 while [ "$(docker inspect "$NAME" --format '{{.State.Running}}')" = "true" ]; do
62 usage="$(docker stats --no-stream --format '{{.MemUsage}}' "$NAME" 2>/dev/null | awk '{print $1}')" || usage=""
63 mib="$(printf '%s' "$usage" | awk '/GiB/{sub("GiB","");printf "%d",$0*1024;next} /MiB/{sub("MiB","");printf "%d",$0;next} /KiB/{sub("KiB","");printf "%d",$0/1024;next} {print 0}')"
64 [ "${mib:-0}" -gt "$peak_mib" ] && peak_mib="$mib"
65 if [ $((SECONDS - start)) -ge $((TIMEOUT + 10)) ]; then killed_by="host-timeout"; docker kill "$NAME" >/dev/null; break; fi
66 sleep 1
67 done
68 elapsed=$((SECONDS - start))
69
70 state="$(docker inspect "$NAME" --format '{{.State.ExitCode}} {{.State.OOMKilled}}')"
71 exit_code="${state% *}"; oom="${state#* }"
72 docker logs "$NAME" >"$OUT/$stamp.stdout" 2>"$OUT/$stamp.stderr" || true
73 docker rm "$NAME" >/dev/null
74
75 job_id="$(grep -o '"job_id": "[^"]*"' "$OUT/$stamp.stderr" | head -1 | sed -E 's/.*: "([^"]*)"/\1/' || true)"
76 reason=""
77 [ "$oom" = "true" ] && reason="oom-killed (memory limit $MEM)"
78 [ "$killed_by" = "host-timeout" ] && reason="killed by host after ${TIMEOUT}+10 s"
79 [ -z "$reason" ] && [ "$exit_code" = "142" ] && reason="SIGALRM timeout (${TIMEOUT}s)"
80 [ -z "$reason" ] && [ "$exit_code" != "0" ] && reason="exit code $exit_code"
81
82 # A killed child cannot record its own death. Mark ITS job (and only a still-running one) failed; never 'done'.
83 if [ -n "$reason" ] && [ -n "$job_id" ]; then
84 docker exec musehub_postgres psql -U musehub -d musehub -v ON_ERROR_STOP=1 -q -c \
85 "UPDATE musehub_background_jobs SET status='failed', done_at=now(), error='$reason',
86 result=jsonb_build_object('executed', false, 'failed', true, 'reason', '$reason')
87 WHERE job_id='$job_id' AND status='running' AND payload->>'experiment'='intel-code-localhost-v1';" >&2
88 fi
89
90 echo "--- result (exit=$exit_code oom=$oom wall=${elapsed}s sampled_container_peak=${peak_mib}MiB reason='${reason:-none}') ---"
91 tail -n 1 "$OUT/$stamp.stdout"
92 echo "logs: $OUT/$stamp.stdout $OUT/$stamp.stderr" >&2
93 exit "$exit_code"
File History 3 commits
sha256:64d53876b5299082b6b4bfbe92721976ec7bb5f1d9a88fdf6eafc1490148d27c fix(intel): explicit, fail-closed deployment identity repla… Sonnet 5.5 minor ⚠ 4 days ago
sha256:78744d379faf9e46bfc8a45b2998d385b92f232df4fd33a1d06bfe62efa0b1d2 feat(intel): run-all measures every provider through the pr… Sonnet 5.5 patch 4 days ago
sha256:0a96f5e99fe9a92a356e7503f94752135aa45cd0fe8c657fcb403422cb30ff43 feat(intel): localhost-only bounded intel.code experiment (… Sonnet 5.5 patch 4 days ago