deploy.sh
bash
sha256:b50eb009cfc837b9898722d09b046f5aeb54d6e6b1b281d97317eeeeb7d54497
Merge 'infra/ecr-pull-retry' into 'dev' — proposal: fix(dep…
Human
4 hours ago
| 1 | #!/usr/bin/env bash |
| 2 | # Zero-downtime blue-green deploy for MuseHub. |
| 3 | # |
| 4 | # Strategy: |
| 5 | # Two slots — blue (port 1337) and green (port 1338). |
| 6 | # The active slot serves traffic via nginx. The inactive slot is stopped. |
| 7 | # Deploy: |
| 8 | # 1. Pull the new image from ECR (old slot keeps serving). |
| 9 | # 2. Run migrations against the live DB (before swap — forward-compatible). |
| 10 | # 3. Start the inactive slot with the new image. |
| 11 | # 4. Health-check the new slot. |
| 12 | # 5. Flip nginx to the new slot (nginx -s reload — instant, zero downtime). |
| 13 | # 6. Stop the old slot. |
| 14 | # |
| 15 | # Called by deploy/push.sh via SSM — do not run directly in production. |
| 16 | # For manual use on the instance (ECR_IMAGE must match the account this |
| 17 | # instance lives in — Nonproduction for staging, Production for prod): |
| 18 | # ECR_IMAGE=<account-id>.dkr.ecr.us-east-1.amazonaws.com/musehub/musehub \ |
| 19 | # IMAGE_TAG=<tag> bash deploy/deploy.sh |
| 20 | # |
| 21 | # First-time setup: |
| 22 | # bash deploy/deploy.sh --init |
| 23 | # (Initialises .active-slot and /etc/nginx/musehub-active-port if missing) |
| 24 | |
| 25 | set -euo pipefail |
| 26 | |
| 27 | APP_DIR="/opt/musehub" |
| 28 | DEPLOY_LOG="/tmp/musehub-deploy.log" |
| 29 | |
| 30 | # Tee all output to a log file so push.sh can stream it live via a second SSM call. |
| 31 | exec > >(tee -a "$DEPLOY_LOG") 2>&1 |
| 32 | echo "" >> "$DEPLOY_LOG" |
| 33 | echo "=== deploy started at $(date -u '+%Y-%m-%dT%H:%M:%SZ') ===" >> "$DEPLOY_LOG" |
| 34 | SLOT_FILE="$APP_DIR/.active-slot" |
| 35 | NGINX_PORT_FILE="/etc/nginx/musehub-active-port" |
| 36 | ECR_REGISTRY="992382692655.dkr.ecr.us-east-1.amazonaws.com" |
| 37 | ECR_IMAGE="${ECR_IMAGE:-${ECR_REGISTRY}/musehub/musehub}" |
| 38 | IMAGE_TAG="${IMAGE_TAG:-latest}" |
| 39 | MUSEHUB_ENV="${MUSEHUB_ENV:-staging}" |
| 40 | # Ceiling for the app/worker containers' memory cgroup. Unpacking a pushed |
| 41 | # mpack currently loads the whole payload + all decoded objects into memory |
| 42 | # at once (no streaming) — repos with large mpacks (~300MB+) can OOM at the |
| 43 | # default 2g. 3g leaves real headroom on this instance's 3.7G total without |
| 44 | # starving postgres. This is a stopgap, not a fix for the underlying |
| 45 | # non-streaming unpack path (see production-readiness follow-up). |
| 46 | APP_MEMORY_LIMIT="${APP_MEMORY_LIMIT:-3g}" |
| 47 | FULL_IMAGE="${ECR_IMAGE}:${IMAGE_TAG}" |
| 48 | REGION="us-east-1" |
| 49 | HEALTH_URL_BLUE="http://127.0.0.1:1337/healthz" |
| 50 | HEALTH_URL_GREEN="http://127.0.0.1:1338/healthz" |
| 51 | HEALTH_RETRIES=30 # × 2s = 60s max wait |
| 52 | |
| 53 | cd "$APP_DIR" |
| 54 | |
| 55 | # ── Helpers ─────────────────────────────────────────────────────────────────── |
| 56 | |
| 57 | log() { echo "[deploy] $*"; } |
| 58 | die() { echo "[deploy] ERROR: $*" >&2; exit 1; } |
| 59 | |
| 60 | health_check() { |
| 61 | local url="$1" |
| 62 | local slot="$2" |
| 63 | log "Health-checking $slot at $url ..." |
| 64 | for i in $(seq 1 "$HEALTH_RETRIES"); do |
| 65 | if curl -sf --max-time 3 "$url" > /dev/null 2>&1; then |
| 66 | log "$slot is healthy (attempt $i)" |
| 67 | return 0 |
| 68 | fi |
| 69 | sleep 2 |
| 70 | done |
| 71 | die "$slot failed health check after $((HEALTH_RETRIES * 2))s" |
| 72 | } |
| 73 | |
| 74 | nginx_point_to() { |
| 75 | local slot="$1" |
| 76 | sudo musehub-set-slot "$slot" |
| 77 | log "nginx now pointing to $slot" |
| 78 | } |
| 79 | |
| 80 | # Repair the active-port file if it contains a bare port number instead of |
| 81 | # a full nginx upstream directive. Called once at startup so a botched |
| 82 | # manual intervention cannot be the root cause of a new deploy failing. |
| 83 | sanitize_nginx_port_file() { |
| 84 | [ -f "$NGINX_PORT_FILE" ] || return 0 |
| 85 | local content |
| 86 | content=$(cat "$NGINX_PORT_FILE") |
| 87 | # Already correct — nothing to do |
| 88 | if echo "$content" | grep -qE '^server 127\.0\.0\.1:[0-9]+;$'; then |
| 89 | return 0 |
| 90 | fi |
| 91 | # Derive correct slot from .active-slot file, or fall back to blue |
| 92 | local slot |
| 93 | slot=$(cat "$SLOT_FILE" 2>/dev/null || echo "blue") |
| 94 | if [ "$slot" != "blue" ] && [ "$slot" != "green" ]; then |
| 95 | slot="blue" |
| 96 | fi |
| 97 | log "WARNING: $NGINX_PORT_FILE has unexpected content — correcting via musehub-set-slot $slot" |
| 98 | sudo musehub-set-slot "$slot" |
| 99 | log "Sanitized active-port file; nginx reloaded." |
| 100 | } |
| 101 | |
| 102 | # ── Init mode ───────────────────────────────────────────────────────────────── |
| 103 | |
| 104 | if [ "${1:-}" = "--init" ]; then |
| 105 | log "Init: installing musehub-set-slot and pointing nginx to blue" |
| 106 | sudo cp "$APP_DIR/deploy/set-active-slot.sh" /usr/local/bin/musehub-set-slot |
| 107 | sudo chmod +x /usr/local/bin/musehub-set-slot |
| 108 | sudo musehub-set-slot blue |
| 109 | log "Done. Run 'bash deploy/deploy.sh' (with ECR_IMAGE and IMAGE_TAG set) to deploy." |
| 110 | exit 0 |
| 111 | fi |
| 112 | |
| 113 | # ── Validate required env vars ──────────────────────────────────────────────── |
| 114 | |
| 115 | [ -n "${ECR_IMAGE:-}" ] || die "ECR_IMAGE is not set." |
| 116 | [ -n "${IMAGE_TAG:-}" ] || die "IMAGE_TAG is not set." |
| 117 | |
| 118 | # ── Read active slot ────────────────────────────────────────────────────────── |
| 119 | |
| 120 | if [ ! -f "$SLOT_FILE" ]; then |
| 121 | die ".active-slot not found. Run: bash deploy/deploy.sh --init" |
| 122 | fi |
| 123 | |
| 124 | ACTIVE_SLOT=$(cat "$SLOT_FILE") |
| 125 | if [ "$ACTIVE_SLOT" = "blue" ]; then |
| 126 | NEW_SLOT="green" |
| 127 | NEW_PORT=1338 |
| 128 | OLD_CONTAINER="musehub-blue" |
| 129 | NEW_CONTAINER="musehub-green" |
| 130 | HEALTH_URL="$HEALTH_URL_GREEN" |
| 131 | else |
| 132 | NEW_SLOT="blue" |
| 133 | NEW_PORT=1337 |
| 134 | OLD_CONTAINER="musehub-green" |
| 135 | NEW_CONTAINER="musehub-blue" |
| 136 | HEALTH_URL="$HEALTH_URL_BLUE" |
| 137 | fi |
| 138 | |
| 139 | log "Image: $FULL_IMAGE" |
| 140 | log "Active slot: $ACTIVE_SLOT → deploying to: $NEW_SLOT (port $NEW_PORT)" |
| 141 | |
| 142 | # Guard: ensure the nginx upstream file is well-formed before we touch anything. |
| 143 | sanitize_nginx_port_file |
| 144 | |
| 145 | # ── Step 0: Apply nginx config if updated ──────────────────────────────────── |
| 146 | # Determine the domain from the current installed config, re-substitute, and |
| 147 | # reload nginx if the content changed. Safe to run on every deploy. |
| 148 | |
| 149 | NGINX_CONF_SRC="$APP_DIR/deploy/nginx-cf.conf" |
| 150 | NGINX_CONF_DEST="/etc/nginx/sites-available/musehub-staging" |
| 151 | NGINX_CONF_DEST_PROD="/etc/nginx/sites-available/musehub" |
| 152 | |
| 153 | if [ -f "$NGINX_CONF_SRC" ]; then |
| 154 | # Detect which installed config exists (staging vs prod) |
| 155 | if [ -f "$NGINX_CONF_DEST" ]; then |
| 156 | NGINX_CONF_INSTALLED="$NGINX_CONF_DEST" |
| 157 | elif [ -f "$NGINX_CONF_DEST_PROD" ]; then |
| 158 | NGINX_CONF_INSTALLED="$NGINX_CONF_DEST_PROD" |
| 159 | else |
| 160 | NGINX_CONF_INSTALLED="" |
| 161 | fi |
| 162 | |
| 163 | if [ -n "$NGINX_CONF_INSTALLED" ]; then |
| 164 | # Extract domain from the installed config (first server_name line) |
| 165 | DOMAIN=$(grep -m1 'server_name' "$NGINX_CONF_INSTALLED" | awk '{print $2}' | tr -d ';') |
| 166 | if [ -n "$DOMAIN" ]; then |
| 167 | NEW_CONF=$(sed "s/DOMAIN_PLACEHOLDER/$DOMAIN/g" "$NGINX_CONF_SRC") |
| 168 | CURRENT_CONF=$(cat "$NGINX_CONF_INSTALLED") |
| 169 | if [ "$NEW_CONF" != "$CURRENT_CONF" ]; then |
| 170 | log "[0/6] nginx config changed — applying update for $DOMAIN..." |
| 171 | echo "$NEW_CONF" | sudo tee "$NGINX_CONF_INSTALLED" > /dev/null |
| 172 | if sudo nginx -t 2>&1; then |
| 173 | sudo nginx -s reload |
| 174 | log "nginx config updated and reloaded." |
| 175 | else |
| 176 | log "WARNING: new nginx config failed validation — reverting." |
| 177 | echo "$CURRENT_CONF" | sudo tee "$NGINX_CONF_INSTALLED" > /dev/null |
| 178 | fi |
| 179 | else |
| 180 | log "[0/6] nginx config unchanged — skipping reload." |
| 181 | fi |
| 182 | fi |
| 183 | fi |
| 184 | fi |
| 185 | |
| 186 | # ── Step 1: Login to ECR and pull new image ─────────────────────────────────── |
| 187 | # Retries the full login+pull cycle (not just the pull) since a stale/expired |
| 188 | # token is the failure mode seen in practice ("Your authorization token has |
| 189 | # expired" immediately after a successful `docker login`) -- re-fetching a |
| 190 | # fresh token from scratch on each attempt is the fix, not just retrying the |
| 191 | # pull with the same (possibly bad) token. |
| 192 | |
| 193 | log "[1/6] Pulling image from ECR..." |
| 194 | PULL_ATTEMPTS=3 |
| 195 | for attempt in $(seq 1 "$PULL_ATTEMPTS"); do |
| 196 | if aws ecr get-login-password --region "$REGION" | \ |
| 197 | sudo docker login --username AWS --password-stdin "$ECR_REGISTRY" \ |
| 198 | && sudo docker pull "$FULL_IMAGE"; then |
| 199 | break |
| 200 | fi |
| 201 | if [ "$attempt" -eq "$PULL_ATTEMPTS" ]; then |
| 202 | die "ECR login/pull failed after $PULL_ATTEMPTS attempts." |
| 203 | fi |
| 204 | log "ECR login/pull failed (attempt $attempt/$PULL_ATTEMPTS) — retrying in 5s..." |
| 205 | sleep 5 |
| 206 | done |
| 207 | log "Pull complete." |
| 208 | |
| 209 | # ── Step 2: Run migrations against the live DB ──────────────────────────────── |
| 210 | |
| 211 | log "[2/6] Running migrations..." |
| 212 | |
| 213 | _alembic() { |
| 214 | sudo docker run --rm \ |
| 215 | --network musehub_musehub-internal \ |
| 216 | --env-file "$APP_DIR/.env" \ |
| 217 | -e SKIP_MIGRATIONS=0 \ |
| 218 | "$FULL_IMAGE" "$@" |
| 219 | } |
| 220 | |
| 221 | # If upgrade head fails (e.g. stale revision ID from a migration history reset), |
| 222 | # stamp to the current head to re-anchor Alembic's tracking, then retry. |
| 223 | # The retry is a no-op when the schema already matches head. |
| 224 | if ! _alembic alembic upgrade head; then |
| 225 | log "upgrade head failed — re-anchoring Alembic revision to head and retrying..." |
| 226 | _alembic alembic stamp --purge head |
| 227 | _alembic alembic upgrade head |
| 228 | fi |
| 229 | log "Migrations complete." |
| 230 | |
| 231 | # Schema parity gate — hard fail. Uses the same benign-diff filter as the S2 |
| 232 | # test (alembic_version table, semantically-equivalent server_default variants, |
| 233 | # column comments) so spurious false positives never block a deploy. |
| 234 | _alembic python -m musehub.db.schema_gate \ |
| 235 | || die "Schema gate failed — ORM drift detected. Write a migration (alembic revision --autogenerate) before deploying." |
| 236 | |
| 237 | # ── Step 3: Start the new slot ──────────────────────────────────────────────── |
| 238 | |
| 239 | log "[3/6] Starting $NEW_SLOT on port $NEW_PORT..." |
| 240 | |
| 241 | # Remove if a failed previous deploy left it around |
| 242 | sudo docker rm -f "$NEW_CONTAINER" 2>/dev/null || true |
| 243 | |
| 244 | sudo docker run -d \ |
| 245 | --name "$NEW_CONTAINER" \ |
| 246 | --network musehub_musehub-internal \ |
| 247 | --env-file "$APP_DIR/.env" \ |
| 248 | -e SKIP_MIGRATIONS=1 \ |
| 249 | -e RELEASE_VERSION="${IMAGE_TAG}" \ |
| 250 | -v musehub_data:/data \ |
| 251 | -p "127.0.0.1:${NEW_PORT}:1337" \ |
| 252 | --restart unless-stopped \ |
| 253 | --memory "$APP_MEMORY_LIMIT" \ |
| 254 | --log-driver awslogs \ |
| 255 | --log-opt awslogs-region=us-east-1 \ |
| 256 | --log-opt awslogs-group=/musehub/${MUSEHUB_ENV} \ |
| 257 | --log-opt awslogs-stream="$NEW_CONTAINER" \ |
| 258 | --log-opt awslogs-create-group=true \ |
| 259 | "$FULL_IMAGE" |
| 260 | |
| 261 | # ── Step 4: Health-check the new slot ──────────────────────────────────────── |
| 262 | |
| 263 | health_check "$HEALTH_URL" "$NEW_SLOT" |
| 264 | |
| 265 | # ── Step 5: Flip nginx to the new slot (instant, zero downtime) ─────────────── |
| 266 | |
| 267 | log "[5/6] Switching nginx to $NEW_SLOT (port $NEW_PORT)..." |
| 268 | nginx_point_to "$NEW_SLOT" |
| 269 | |
| 270 | # ── Step 6: Stop the old slot ──────────────────────────────────────────────── |
| 271 | |
| 272 | log "[6/6] Stopping old slot ($ACTIVE_SLOT)..." |
| 273 | # `docker stop` sends SIGTERM and waits (--time) before SIGKILL, giving the |
| 274 | # app's lifespan shutdown handler (closes the DB pool, stops the Playwright |
| 275 | # browser) a chance to actually run, and letting any in-flight requests that |
| 276 | # were accepted just before the nginx flip finish rather than being dropped. |
| 277 | # `docker rm -f` (the previous behavior) sends SIGKILL immediately and skips |
| 278 | # all of that — the graceful-shutdown code existed but was never triggered. |
| 279 | sudo docker stop --time 15 "$OLD_CONTAINER" 2>/dev/null || true |
| 280 | sudo docker rm -f "$OLD_CONTAINER" 2>/dev/null || true |
| 281 | |
| 282 | # ── Step 7: Restart the background worker ──────────────────────────────────── |
| 283 | |
| 284 | log "[7/7] Restarting background worker..." |
| 285 | sudo docker stop --time 15 musehub-worker 2>/dev/null || true |
| 286 | sudo docker rm -f musehub-worker 2>/dev/null || true |
| 287 | sudo docker run -d \ |
| 288 | --name musehub-worker \ |
| 289 | --network musehub_musehub-internal \ |
| 290 | --env-file "$APP_DIR/.env" \ |
| 291 | -e SKIP_MIGRATIONS=1 \ |
| 292 | -e RELEASE_VERSION="${IMAGE_TAG}" \ |
| 293 | -v musehub_data:/data \ |
| 294 | --restart unless-stopped \ |
| 295 | --no-healthcheck \ |
| 296 | --memory "$APP_MEMORY_LIMIT" \ |
| 297 | --log-driver awslogs \ |
| 298 | --log-opt awslogs-region=us-east-1 \ |
| 299 | --log-opt awslogs-group=/musehub/${MUSEHUB_ENV} \ |
| 300 | --log-opt awslogs-stream=musehub-worker \ |
| 301 | --log-opt awslogs-create-group=true \ |
| 302 | "$FULL_IMAGE" python -m musehub.worker |
| 303 | log "Worker started." |
| 304 | |
| 305 | # ── Step 8: Prune old images (keep last 3) ─────────────────────────────────── |
| 306 | |
| 307 | log "[8/8] Pruning old images (keeping last 3)..." |
| 308 | KEEP_IMAGES=3 |
| 309 | OLD_IDS=$(sudo docker images "$ECR_IMAGE" --format "{{.ID}}" \ |
| 310 | | awk '!seen[$0]++' \ |
| 311 | | tail -n +$((KEEP_IMAGES + 1))) |
| 312 | if [ -n "$OLD_IDS" ]; then |
| 313 | echo "$OLD_IDS" | xargs sudo docker rmi -f 2>/dev/null || true |
| 314 | log "Image prune complete." |
| 315 | else |
| 316 | log "No old images to prune." |
| 317 | fi |
| 318 | |
| 319 | log "" |
| 320 | log "Deploy complete. Active slot: $NEW_SLOT (port $NEW_PORT)" |
| 321 | log "Image: $FULL_IMAGE" |
File History
6 commits
sha256:b50eb009cfc837b9898722d09b046f5aeb54d6e6b1b281d97317eeeeb7d54497
Merge 'infra/ecr-pull-retry' into 'dev' — proposal: fix(dep…
Human
4 hours ago
sha256:bee12c5cbde2334f98421c6c209d768fa6b8004d6705c9ea798ce6c1651bc11f
Merge 'infra/database-phase3-4-cleanup' into 'dev' — propos…
Human
4 hours ago
sha256:23efc08a3fcec5132abb7a4827626dd99f59bf69f64eeeaefad3c1d72a08fe36
Merge 'fix/cloudwatch-alerts-and-log-fields' into 'dev' — p…
Human
10 hours ago
sha256:9c64dbfd65ef4e8a85f500e5909c06c2e6c65255a69e84188ee73b63f111cecd
Merge 'docs/status-banners-closed-tickets' into 'dev' — pro…
Human
11 hours ago
sha256:3fadb0439bba9451b89229676971c0d4a40900dec7810e9d5f8791b8d950d505
fix: install.sh version from latest published tarball, not …
Sonnet 4.6
minor
⚠
102 days ago
sha256:763eb2cb8675073b84c19345b27586d2ed939a9aee97c5479b69f502f1a70eff
fix(tests): update test suite to match current implementation
Sonnet 4.6
patch
124 days ago