gabriel / musehub public
push.sh bash
350 lines 16.2 KB Hotspot Cold
Raw
sha256:c8e165f9c0e0c8c2f888cea908ff5b6b51548acd1b6dbdf7ed11a06211cbe0eb feat(release): MuseHub builds carry a SemVer release identi… Opus 5.5 minor ⚠ breaking 5 days ago
1 #!/usr/bin/env bash
2 # MuseHub deploy orchestrator — build, push to ECR, trigger blue-green via SSM.
3 #
4 # Usage:
5 # bash deploy/push.sh staging # deploy to staging only
6 # bash deploy/push.sh prod # deploy to prod only
7 # bash deploy/push.sh staging prod # staging first, then prod
8 #
9 # What it does:
10 # 1. Assembles the build context from COMMITTED files only (deploy/build_context.py:
11 # `muse archive` of musehub and muse). Untracked local files cannot enter the image;
12 # tracked-but-uncommitted changes refuse the build. Then builds a linux/amd64 image
13 # from it, recording both repositories' Muse revisions as image labels.
14 # 2. Tags it with <musehub-revision-8>-<timestamp>. That tag is the image's identity.
15 # There is no :latest: a mutable tag is not a rollback or audit identity.
16 # 3. Pushes the image to each target's own ECR (staging and prod are
17 # separate AWS accounts with separate, non-shared ECR repos) and prints the
18 # pushed digest, which is what to record for rollback.
19 # 4. Sends an SSM command to each target instance to run deploy.sh,
20 # which pulls the image and performs a zero-downtime blue-green swap.
21 # 5. Polls SSM until the deploy completes or fails, streaming the output.
22 #
23 # Staging and production are different AWS accounts, each authenticated via
24 # the operator's own IAM Identity Center SSO session — run `aws sso login
25 # --profile musehub-nonproduction` (staging) or `--profile musehub-production`
26 # (prod) first. No shared IAM user is used by either target; the legacy
27 # `musehub-infra` static credential is retired from this path (kept around
28 # for now for any scripts that still reference it directly, but no longer
29 # used by push.sh).
30 #
31 # Prerequisites (one-time, already done):
32 # - `musehub-nonproduction` and `musehub-production` SSO profiles configured
33 # in ~/.aws/config
34 # - Docker Desktop running
35 # - musehub-ec2-ssm / musehub-production-ec2-ssm roles have ecr pull permissions
36 # - AWS CLI installed on instances (run deploy/bootstrap-instance.sh once)
37 #
38 # Rollback to a previous image:
39 # IMAGE_TAG=<previous-tag> bash deploy/push.sh staging
40 # (skips build+push, triggers SSM with the specified tag directly)
41
42 set -euo pipefail
43
44 SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
45 REPO_DIR="$(dirname "$SCRIPT_DIR")"
46 ECOSYSTEM_DIR="$(dirname "$REPO_DIR")"
47
48 ECR_REPO="musehub/musehub"
49 REGION="us-east-1"
50
51 # Per-environment config — staging and prod are separate AWS accounts.
52 #
53 # Deliberately `case` dispatch rather than `declare -A`: associative arrays
54 # require bash 4+, but macOS ships bash 3.2 as /bin/bash with no version
55 # guard elsewhere in this script, so `declare -A` fails outright for anyone
56 # running under the system bash rather than a Homebrew-installed bash 4/5.
57 instance_id_for() {
58 case "$1" in
59 staging) echo "i-07547cd20bee2dea5" ;;
60 prod) echo "i-043aaed71bef11903" ;;
61 esac
62 }
63 ecr_registry_for() {
64 case "$1" in
65 staging) echo "992382692655.dkr.ecr.us-east-1.amazonaws.com" ;;
66 prod) echo "672469410277.dkr.ecr.us-east-1.amazonaws.com" ;;
67 esac
68 }
69 aws_profile_for() {
70 case "$1" in
71 staging) echo "musehub-nonproduction" ;; # operator's own SSO session
72 prod) echo "musehub-production" ;; # operator's own SSO session
73 esac
74 }
75
76 # ── Parse targets ─────────────────────────────────────────────────────────────
77
78 if [ $# -eq 0 ]; then
79 echo "Usage: bash deploy/push.sh [staging] [prod]"
80 echo " staging deploy to staging.musehub.ai"
81 echo " prod deploy to musehub.ai"
82 echo " staging prod staging first, then prod"
83 echo ""
84 echo "Rollback (skips build+push, redeploys a previous tag):"
85 echo " IMAGE_TAG=<tag> bash deploy/push.sh staging"
86 exit 1
87 fi
88
89 TARGETS=()
90 for arg in "$@"; do
91 case "$arg" in
92 staging|prod) TARGETS+=("$arg") ;;
93 *) echo "Unknown target: $arg (must be staging or prod)" >&2; exit 1 ;;
94 esac
95 done
96
97 # ── Helpers ───────────────────────────────────────────────────────────────────
98
99 log() { echo "[push] $*"; }
100 die() { echo "[push] ERROR: $*" >&2; exit 1; }
101
102 profile_args() {
103 local env="$1"
104 local profile
105 profile="$(aws_profile_for "$env")"
106 if [ -n "$profile" ]; then
107 echo "--profile $profile"
108 fi
109 }
110
111 check_auth() {
112 local env="$1"
113 local profile
114 profile="$(aws_profile_for "$env")"
115 if [ -n "$profile" ]; then
116 if ! aws sts get-caller-identity --profile "$profile" > /dev/null 2>&1; then
117 die "No valid SSO session for profile '$profile'. Run: aws sso login --profile $profile"
118 fi
119 fi
120 }
121
122 # ── Image tag ─────────────────────────────────────────────────────────────────
123
124 # If IMAGE_TAG is already set (rollback mode), skip build+push.
125 if [ -n "${IMAGE_TAG:-}" ]; then
126 log "Rollback mode — using existing tag: $IMAGE_TAG"
127 SKIP_BUILD=true
128 else
129 # The revision is mandatory: an image that cannot say what it was built from is never deployed.
130 COMMIT_HASH=$(muse -C "$REPO_DIR" rev-parse HEAD --json \
131 | python3 -c "import sys,json; cid=json.load(sys.stdin)['commit_id']; print(cid.removeprefix('sha256:')[:8])") \
132 || die "cannot determine the musehub revision with Muse; refusing to build an unidentifiable image"
133 case "$COMMIT_HASH" in
134 [0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f]) ;;
135 *) die "unexpected musehub revision '$COMMIT_HASH'" ;;
136 esac
137 IMAGE_TAG="${COMMIT_HASH}-$(date +%Y%m%d%H%M%S)"
138 SKIP_BUILD=false
139 fi
140
141 log "Image tag: $IMAGE_TAG"
142
143 for target in "${TARGETS[@]}"; do
144 check_auth "$target"
145 done
146
147 # ── Build ─────────────────────────────────────────────────────────────────────
148
149 LOCAL_TAG="musehub-build:${IMAGE_TAG}"
150 CRANE_TAR="/tmp/musehub-${IMAGE_TAG}.tar"
151
152 if [ "$SKIP_BUILD" = false ]; then
153 log "[1/3] Assembling build context from committed files..."
154 CONTEXT_DIR="$(mktemp -d "${TMPDIR:-/tmp}/musehub-context.XXXXXX")/ctx"
155 trap 'rm -rf "$(dirname "$CONTEXT_DIR")"' EXIT
156 (cd "$REPO_DIR" && python3 -m deploy.build_context --ecosystem "$ECOSYSTEM_DIR" --out "$CONTEXT_DIR" > "$CONTEXT_DIR.info.json") \
157 || die "build context refused (see above); nothing was built"
158 MUSEHUB_REVISION="$(python3 -c "import json,sys; print(json.load(open(sys.argv[1]))['repositories']['musehub']['revision'])" "$CONTEXT_DIR.info.json")"
159 MUSE_REVISION="$(python3 -c "import json,sys; print(json.load(open(sys.argv[1]))['repositories']['muse']['revision'])" "$CONTEXT_DIR.info.json")"
160 [ -n "$MUSEHUB_REVISION" ] && [ -n "$MUSE_REVISION" ] || die "context did not record both revisions"
161 case "$MUSEHUB_REVISION" in sha256:"${COMMIT_HASH}"*) ;; *) die "context revision $MUSEHUB_REVISION is not the tagged $COMMIT_HASH" ;; esac
162 log " musehub $MUSEHUB_REVISION"
163 log " muse $MUSE_REVISION"
164
165 log "[1/3] Building image for linux/amd64..."
166 # --no-cache-filter runtime: a stale BuildKit cache hit was observed live
167 # (2026-10-21) on the runtime stage's COPY of musehub/pyproject.toml --
168 # the deployed container kept serving an old version string (0.2.1rc4)
169 # three commits after a version bump to 0.2.1rc5 that touched nothing
170 # but that one file, even though every .py file change in between had
171 # correctly busted the cache. A clean no-cache rebuild immediately
172 # produced the correct content, confirming a cache-key bug rather than a
173 # build-context problem. The expensive builder stage (wheel builds,
174 # cryptography compiled from source) keeps its cache; only the cheap
175 # runtime stage (plain file COPYs) is forced fresh every deploy.
176 docker build \
177 --platform linux/amd64 \
178 --no-cache-filter runtime \
179 --build-arg "MUSEHUB_REVISION=${MUSEHUB_REVISION}" \
180 --build-arg "MUSE_REVISION=${MUSE_REVISION}" \
181 --tag "$LOCAL_TAG" \
182 -f "$CONTEXT_DIR/musehub/Dockerfile" \
183 "$CONTEXT_DIR"
184 log "Build complete."
185 # The image must say what it was built from, or it is not pushed.
186 built_rev="$(docker image inspect --format '{{ index .Config.Labels "org.opencontainers.image.revision" }}' "$LOCAL_TAG")"
187 built_muse="$(docker image inspect --format '{{ index .Config.Labels "ai.musehub.muse-engine-revision" }}' "$LOCAL_TAG")"
188 [ "$built_rev" = "$MUSEHUB_REVISION" ] && [ "$built_muse" = "$MUSE_REVISION" ] \
189 || die "image labels (${built_rev:-none} / ${built_muse:-none}) do not match the context revisions; not pushing"
190 docker save "$LOCAL_TAG" -o "$CRANE_TAR"
191
192 # ── Push to each target's ECR via crane ───────────────────────────────────
193 # crane bypasses Docker Desktop's VPNKit proxy, which drops connections
194 # mid-upload on large layer pushes. Never use `docker push` to ECR.
195 # Staging and prod are separate AWS accounts, so the same tarball is
196 # pushed to each target's own registry under its own credentials.
197
198 log "[2/3] Pushing to ECR for: ${TARGETS[*]}"
199 for target in "${TARGETS[@]}"; do
200 registry="$(ecr_registry_for "$target")"
201 image="${registry}/${ECR_REPO}"
202 eval "aws ecr get-login-password --region \"$REGION\" $(profile_args "$target")" | \
203 crane auth login "$registry" --username AWS --password-stdin
204 # Create-only by convention: the tag embeds a revision and a timestamp and is never moved.
205 crane push "$CRANE_TAR" "${image}:${IMAGE_TAG}"
206 digest="$(crane digest "${image}:${IMAGE_TAG}")" || die "could not read the pushed digest from $target"
207 log " Pushed to $target ($registry): ${image}:${IMAGE_TAG}@${digest}"
208 done
209 rm -f "$CRANE_TAR"
210 log "Push complete. Tag: $IMAGE_TAG"
211 else
212 log "[1/3] Skipping build (rollback mode)."
213 log "[2/3] Skipping push (rollback mode) — assumes the tag already exists in each target registry."
214 fi
215
216 # ── Trigger deploy via SSM ────────────────────────────────────────────────────
217
218 log "[3/3] Triggering deploy on: ${TARGETS[*]}"
219
220 deploy_to() {
221 local env="$1"
222 local instance_id
223 instance_id="$(instance_id_for "$env")"
224 local ecr_image
225 ecr_image="$(ecr_registry_for "$env")/${ECR_REPO}"
226 local pargs
227 pargs=$(profile_args "$env")
228
229 # deploy.sh's MUSEHUB_ENV drives its CloudWatch log group — "prod" (the
230 # CLI target name) must map to "production" (the actual env/log-group name).
231 local musehub_env
232 if [ "$env" = "prod" ]; then musehub_env="production"; else musehub_env="staging"; fi
233
234 log ""
235 log "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
236 log "→ Deploying to $env ($instance_id)"
237 log "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
238
239 local deploy_sh_b64
240 deploy_sh_b64=$(base64 -i "$SCRIPT_DIR/deploy.sh" | tr -d '\n')
241
242 local nginx_conf_b64
243 nginx_conf_b64=$(base64 -i "$SCRIPT_DIR/nginx-cf.conf" | tr -d '\n')
244
245 local set_slot_b64
246 set_slot_b64=$(base64 -i "$SCRIPT_DIR/set-active-slot.sh" | tr -d '\n')
247
248 # ── Fire the deploy command ───────────────────────────────────────────────
249 local deploy_cmd_id
250 deploy_cmd_id=$(eval "aws ssm send-command \
251 --region \"$REGION\" \
252 $pargs \
253 --instance-ids \"$instance_id\" \
254 --document-name \"AWS-RunShellScript\" \
255 --parameters \"commands=[
256 \\\"echo '${deploy_sh_b64}' | base64 -d > /opt/musehub/deploy/deploy.sh && chmod +x /opt/musehub/deploy/deploy.sh\\\",
257 \\\"echo '${nginx_conf_b64}' | base64 -d > /opt/musehub/deploy/nginx-cf.conf\\\",
258 \\\"echo '${set_slot_b64}' | base64 -d > /usr/local/bin/musehub-set-slot && chmod +x /usr/local/bin/musehub-set-slot\\\",
259 \\\"export ECR_IMAGE=${ecr_image}\\\",
260 \\\"export IMAGE_TAG=${IMAGE_TAG}\\\",
261 \\\"export MUSEHUB_ENV=${musehub_env}\\\",
262 \\\"bash /opt/musehub/deploy/deploy.sh\\\"
263 ]\" \
264 --comment \"musehub ${IMAGE_TAG} → ${env}\" \
265 --timeout-seconds 600 \
266 --query \"Command.CommandId\" \
267 --output text")
268
269 log " SSM command: $deploy_cmd_id"
270
271 # ── Stream deploy output live ─────────────────────────────────────────────
272 # SSM updates StandardOutputContent as the command runs. We poll every 5s,
273 # diff against what we've already printed, and show new lines immediately.
274 log " Live output:"
275 log ""
276
277 local lines_seen=0
278 local elapsed=0
279 local final_status=""
280
281 while true; do
282 sleep 5
283 elapsed=$((elapsed + 5))
284
285 local invocation
286 invocation=$(eval "aws ssm get-command-invocation \
287 --region \"$REGION\" \
288 $pargs \
289 --command-id \"$deploy_cmd_id\" \
290 --instance-id \"$instance_id\" \
291 --output json" 2>/dev/null || echo '{"Status":"Pending","StandardOutputContent":"","StandardErrorContent":""}')
292
293 local cmd_status all_stdout all_stderr
294 cmd_status=$(echo "$invocation" | python3 -c "import sys,json; d=json.load(sys.stdin); print(d.get('Status','Pending'))" 2>/dev/null || echo "Pending")
295 all_stdout=$(echo "$invocation" | python3 -c "import sys,json; d=json.load(sys.stdin); print(d.get('StandardOutputContent',''),end='')" 2>/dev/null || echo "")
296 all_stderr=$(echo "$invocation" | python3 -c "import sys,json; d=json.load(sys.stdin); print(d.get('StandardErrorContent',''),end='')" 2>/dev/null || echo "")
297
298 # Print any new stdout lines
299 local total_lines
300 total_lines=$(echo "$all_stdout" | wc -l)
301 if [ "$total_lines" -gt "$lines_seen" ]; then
302 echo "$all_stdout" | tail -n +"$((lines_seen + 1))"
303 lines_seen=$total_lines
304 fi
305
306 case "$cmd_status" in
307 Success)
308 final_status="success"
309 break
310 ;;
311 Failed|Cancelled|TimedOut|Cancelling)
312 final_status="failed"
313 [ -n "$all_stderr" ] && echo "STDERR: $all_stderr"
314 break
315 ;;
316 esac
317
318 if [ "$elapsed" -ge 600 ]; then
319 die "Deploy timed out after 10 min. SSM command: $deploy_cmd_id"
320 fi
321 done
322
323 echo ""
324 if [ "$final_status" = "success" ]; then
325 log "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
326 log "✅ $env deploy succeeded."
327 log "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
328 return 0
329 else
330 log "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
331 log "❌ $env deploy FAILED. Full output:"
332 log "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
333 eval "aws ssm get-command-invocation \
334 --region \"$REGION\" \
335 $pargs \
336 --command-id \"$deploy_cmd_id\" \
337 --instance-id \"$instance_id\" \
338 --query \"[StandardOutputContent,StandardErrorContent]\" \
339 --output text" 2>/dev/null || true
340 return 1
341 fi
342 }
343
344 for target in "${TARGETS[@]}"; do
345 deploy_to "$target"
346 done
347
348 log ""
349 log "All done."
350 log " Tag: $IMAGE_TAG"
File History 17 commits
sha256:c8e165f9c0e0c8c2f888cea908ff5b6b51548acd1b6dbdf7ed11a06211cbe0eb feat(release): MuseHub builds carry a SemVer release identi… Opus 5.5 minor ⚠ 5 days ago
sha256:414893924229cfb6dd5a5154d9526c1a9b57a2721289daba6ad89a6c98c64c09 fix(deploy): stream the Docker build context so same-size e… Sonnet 5.5 patch 6 days ago
sha256:3c6d2e2751e9222a0872ba699f16baa457d9990c1998c20956badb9186ee69ee fix(deploy): image built from committed files only, with re… Opus 5.5 minor ⚠ 12 days ago
sha256:47c92210e23600ae975e8943d129f6c752fb4d5835fd19b8d74fbc8489ec83f8 fix: deploy build silently served a stale Docker layer for … Sonnet 5 minor ⚠ 26 days ago
sha256:915b1f581ce1d3bca63153d03c07a5ccbbfbe77de3bf561e75e85f9427dd32a3 fix(musehub#199): remove declare -A from push.sh for bash 3… Sonnet 5 minor ⚠ 66 days ago
sha256:7ca89a509c5c90deba3aee7664f0fdaa8f58f7c5d5b7b9893262f04ac955c408 fix: push.sh staging deploys now use individual SSO identit… Sonnet 5 patch 70 days ago
sha256:b3d08ea378601022e598a698496dcbec8b699d379bacb956a9b640fb99bcc35c fix: raise app/worker container memory limits, fix hardcode… Sonnet 5 patch 80 days ago
sha256:7093b91d5684806df3a8d0f4b520e53716bb19f540822aace6e30c5a05cdf6e4 fix: push.sh now targets the real Production AWS account, n… Sonnet 5 minor ⚠ 80 days ago
sha256:864d59986b43001d6ccc58c432a4c7d104e722e11f334072f6ed6e9fe67edc39 fix(deploy): use JSON parsing for SSM output, fixes macOS h… Sonnet 4.6 patch 203 days ago
sha256:0ae04fc8fc7903e2f21f9260ed87d40a60a93cb88da713d87e85ae246d7a5439 ops: live-stream deploy output to terminal via SSM partial reads Sonnet 4.6 minor ⚠ 203 days ago
sha256:ef5614c0cfb6a312635cbf4ca2e4439d1ad7ce211e63761b3b883e527bcab84e fix(deploy): replace direct nginx port-file writes with mus… Sonnet 4.6 minor ⚠ 203 days ago
sha256:a325283bfb91be72f1c53346da7102a2928e48312293a78cdfd26ed151ef4156 chore: fix Docker image name — use musehub/musehub explicitly Sonnet 4.6 minor ⚠ 205 days ago
sha256:86593681e1bc1391ab4c14f4cf8c0fc85b9bf29cc79f28493c22c07f47abae6f chore: rename Docker image from musehub/musehub to musehub/app Sonnet 4.6 minor ⚠ 205 days ago
sha256:edb21864b55adab9d6903aa1f5c5cb74c0e989ff8b62feb00c92299314b0e58c fix(nginx): enable HTTP/2, fix push/stream location, disabl… Sonnet 4.6 minor ⚠ 208 days ago
sha256:0d72b21399b308cc808bc337907eae581da93aa764ab6848069e0e3421b5b8a0 build: bundle muse package into production Docker image Sonnet 4.6 minor ⚠ 209 days ago
sha256:9509b5628f0f94668fbab4a7a31302342abc34d52447a653757a47edafe12d42 fix(deploy): strip sha256: prefix from commit_id before usi… Human patch 215 days ago
sha256:96ccab63b6360641ea8a3f1cab1af5f7c25017453057cd2e3bac74a09eccf5fc fix(deploy): use crane for ECR push, sync deploy.sh via SSM… Human major ⚠ 222 days ago