deploy/deploy.sh · gabriel/musehub

1

#!/usr/bin/env bash

2

# Zero-downtime blue-green deploy for MuseHub.

3

#

4

# Strategy:

5

# Two slots — blue (port 1337) and green (port 1338).

6

# The active slot serves traffic via nginx. The inactive slot is stopped.

7

# Deploy:

8

# 1. Pull the new image from ECR (old slot keeps serving).

9

# 2. Run migrations against the live DB (before swap — forward-compatible).

10

# 3. Start the inactive slot with the new image.

11

# 4. Health-check the new slot.

12

# 5. Flip nginx to the new slot (nginx -s reload — instant, zero downtime).

13

# 6. Stop the old slot.

14

#

15

# Called by deploy/push.sh via SSM — do not run directly in production.

16

# For manual use on the instance:

17

# ECR_IMAGE=992382692655.dkr.ecr.us-east-1.amazonaws.com/musehub/musehub \

18

# IMAGE_TAG=<tag> bash deploy/deploy.sh

19

#

20

# First-time setup:

21

# bash deploy/deploy.sh --init

22

# (Initialises .active-slot and /etc/nginx/musehub-active-port if missing)

set -euo pipefail

APP_DIR="/opt/musehub"

27

SLOT_FILE="$APP_DIR/.active-slot"

28

NGINX_PORT_FILE="/etc/nginx/musehub-active-port"

29

ECR_REGISTRY="992382692655.dkr.ecr.us-east-1.amazonaws.com"

30

ECR_IMAGE="${ECR_IMAGE:-${ECR_REGISTRY}/musehub/musehub}"

31

IMAGE_TAG="${IMAGE_TAG:-latest}"

32

FULL_IMAGE="${ECR_IMAGE}:${IMAGE_TAG}"

33

REGION="us-east-1"

34

HEALTH_URL_BLUE="http://127.0.0.1:1337/healthz"

35

HEALTH_URL_GREEN="http://127.0.0.1:1338/healthz"

36

HEALTH_RETRIES=30 # × 2s = 60s max wait

cd "$APP_DIR"

# ── Helpers ───────────────────────────────────────────────────────────────────

41

42

log() { echo "[deploy] $*"; }

43

die() { echo "[deploy] ERROR: $*" >&2; exit 1; }

health_check() {

local url="$1"

local slot="$2"

log "Health-checking $slot at $url ..."

49

for i in $(seq 1 "$HEALTH_RETRIES"); do

50

if curl -sf --max-time 3 "$url" > /dev/null 2>&1; then

51

log "$slot is healthy (attempt $i)"

return 0

fi

sleep 2

done

die "$slot failed health check after $((HEALTH_RETRIES * 2))s"

}

nginx_point_to() {

local port="$1"

echo "server 127.0.0.1:${port};" | sudo tee "$NGINX_PORT_FILE" > /dev/null

62

sudo nginx -s reload

63

log "nginx now pointing to port $port"

64

}

65

66

# ── Init mode ─────────────────────────────────────────────────────────────────

67

68

if [ "${1:-}" = "--init" ]; then

69

log "Init: writing .active-slot=blue and nginx upstream file"

70

echo "blue" > "$SLOT_FILE"

71

echo "server 127.0.0.1:1337;" | sudo tee "$NGINX_PORT_FILE" > /dev/null

72

sudo nginx -s reload

73

log "Done. Run 'bash deploy/deploy.sh' (with ECR_IMAGE and IMAGE_TAG set) to deploy."

exit 0

fi

# ── Validate required env vars ────────────────────────────────────────────────

78

79

[ -n "${ECR_IMAGE:-}" ] || die "ECR_IMAGE is not set."

80

[ -n "${IMAGE_TAG:-}" ] || die "IMAGE_TAG is not set."

81

82

# ── Read active slot ──────────────────────────────────────────────────────────

83

84

if [ ! -f "$SLOT_FILE" ]; then

85

die ".active-slot not found. Run: bash deploy/deploy.sh --init"

86

fi

87

88

ACTIVE_SLOT=$(cat "$SLOT_FILE")

89

if [ "$ACTIVE_SLOT" = "blue" ]; then

90

NEW_SLOT="green"

91

NEW_PORT=1338

92

OLD_CONTAINER="musehub-blue"

93

NEW_CONTAINER="musehub-green"

94

HEALTH_URL="$HEALTH_URL_GREEN"

else

NEW_SLOT="blue"

NEW_PORT=1337

OLD_CONTAINER="musehub-green"

99

NEW_CONTAINER="musehub-blue"

100

HEALTH_URL="$HEALTH_URL_BLUE"

101

fi

102

103

log "Image: $FULL_IMAGE"

104

log "Active slot: $ACTIVE_SLOT → deploying to: $NEW_SLOT (port $NEW_PORT)"

105

106

# ── Step 0: Apply nginx config if updated ────────────────────────────────────

107

# Determine the domain from the current installed config, re-substitute, and

108

# reload nginx if the content changed. Safe to run on every deploy.

109

110

NGINX_CONF_SRC="$APP_DIR/deploy/nginx-cf.conf"

111

NGINX_CONF_DEST="/etc/nginx/sites-available/musehub-staging"

112

NGINX_CONF_DEST_PROD="/etc/nginx/sites-available/musehub"

113

114

if [ -f "$NGINX_CONF_SRC" ]; then

115

# Detect which installed config exists (staging vs prod)

116

if [ -f "$NGINX_CONF_DEST" ]; then

117

NGINX_CONF_INSTALLED="$NGINX_CONF_DEST"

118

elif [ -f "$NGINX_CONF_DEST_PROD" ]; then

119

NGINX_CONF_INSTALLED="$NGINX_CONF_DEST_PROD"

120

else

121

NGINX_CONF_INSTALLED=""

122

fi

123

124

if [ -n "$NGINX_CONF_INSTALLED" ]; then

125

# Extract domain from the installed config (first server_name line)

126

DOMAIN=$(grep -m1 'server_name' "$NGINX_CONF_INSTALLED" | awk '{print $2}' | tr -d ';')

127

if [ -n "$DOMAIN" ]; then

128

NEW_CONF=$(sed "s/DOMAIN_PLACEHOLDER/$DOMAIN/g" "$NGINX_CONF_SRC")

129

CURRENT_CONF=$(cat "$NGINX_CONF_INSTALLED")

130

if [ "$NEW_CONF" != "$CURRENT_CONF" ]; then

131

log "[0/6] nginx config changed — applying update for $DOMAIN..."

132

echo "$NEW_CONF" | sudo tee "$NGINX_CONF_INSTALLED" > /dev/null

133

if sudo nginx -t 2>&1; then

134

sudo nginx -s reload

135

log "nginx config updated and reloaded."

136

else

137

log "WARNING: new nginx config failed validation — reverting."

138

echo "$CURRENT_CONF" | sudo tee "$NGINX_CONF_INSTALLED" > /dev/null

139

fi

140

else

141

log "[0/6] nginx config unchanged — skipping reload."

fi

fi

fi

fi

# ── Step 1: Login to ECR and pull new image ───────────────────────────────────

148

149

log "[1/6] Pulling image from ECR..."

150

aws ecr get-login-password --region "$REGION" | \

151

sudo docker login --username AWS --password-stdin "$ECR_REGISTRY"

152

sudo docker pull "$FULL_IMAGE"

153

log "Pull complete."

154

155

# ── Step 2: Run migrations against the live DB ────────────────────────────────

156

157

log "[2/6] Running migrations..."

158

DB_PASSWORD=$(grep ^DB_PASSWORD "$APP_DIR/.env" | cut -d= -f2)

159

160

_alembic() {

161

sudo docker run --rm \

162

--network musehub_musehub-internal \

163

--env-file "$APP_DIR/.env" \

164

-e "DATABASE_URL=postgresql+asyncpg://musehub:${DB_PASSWORD}@postgres:5432/musehub" \

165

-e SKIP_MIGRATIONS=0 \

"$FULL_IMAGE" "$@"

}

# If upgrade head fails (e.g. stale revision ID from a migration history reset),

170

# stamp to the current head to re-anchor Alembic's tracking, then retry.

171

# The retry is a no-op when the schema already matches head.

172

if ! _alembic alembic upgrade head; then

173

log "upgrade head failed — re-anchoring Alembic revision to head and retrying..."

174

_alembic alembic stamp --purge head

175

_alembic alembic upgrade head

176

fi

177

log "Migrations complete."

178

179

# ── Step 3: Start the new slot ────────────────────────────────────────────────

180

181

log "[3/6] Starting $NEW_SLOT on port $NEW_PORT..."

182

183

# Remove if a failed previous deploy left it around

184

sudo docker rm -f "$NEW_CONTAINER" 2>/dev/null || true

185

186

sudo docker run -d \

187

--name "$NEW_CONTAINER" \

188

--network musehub_musehub-internal \

189

--env-file "$APP_DIR/.env" \

190

-e "DATABASE_URL=postgresql+asyncpg://musehub:${DB_PASSWORD}@postgres:5432/musehub" \

191

-e SKIP_MIGRATIONS=1 \

192

-v musehub_data:/data \

193

-p "127.0.0.1:${NEW_PORT}:1337" \

194

--restart unless-stopped \

195

"$FULL_IMAGE"

196

197

# ── Step 4: Health-check the new slot ────────────────────────────────────────

198

199

health_check "$HEALTH_URL" "$NEW_SLOT"

200

201

# ── Step 5: Flip nginx to the new slot (instant, zero downtime) ───────────────

202

203

log "[5/6] Switching nginx to $NEW_SLOT (port $NEW_PORT)..."

204

nginx_point_to "$NEW_PORT"

205

echo "$NEW_SLOT" > "$SLOT_FILE"

206

207

# ── Step 6: Stop the old slot ────────────────────────────────────────────────

208

209

log "[6/6] Stopping old slot ($ACTIVE_SLOT)..."

210

sudo docker rm -f "$OLD_CONTAINER" 2>/dev/null || true

211

212

# ── Step 7: Restart the background worker ────────────────────────────────────

213

214

log "[7/7] Restarting background worker..."

215

sudo docker rm -f musehub-worker 2>/dev/null || true

216

sudo docker run -d \

217

--name musehub-worker \

218

--network musehub_musehub-internal \

219

--env-file "$APP_DIR/.env" \

220

-e "DATABASE_URL=postgresql+asyncpg://musehub:${DB_PASSWORD}@postgres:5432/musehub" \

221

-e SKIP_MIGRATIONS=1 \

222

-v musehub_data:/data \

223

--restart unless-stopped \

224

"$FULL_IMAGE" python -m musehub.worker

225

log "Worker started."

226

227

log ""

228

log "Deploy complete. Active slot: $NEW_SLOT (port $NEW_PORT)"

229

log "Image: $FULL_IMAGE"