cloudwatch-alerts.sh
bash
sha256:bee12c5cbde2334f98421c6c209d768fa6b8004d6705c9ea798ce6c1651bc11f
Merge 'infra/database-phase3-4-cleanup' into 'dev' — propos…
Human
5 hours ago
| 1 | #!/usr/bin/env bash |
| 2 | # MuseHub — CloudWatch log group, metric filters, alarms, and SNS alerting. |
| 3 | # |
| 4 | # Run once per environment; safe to re-run (put-metric-alarm is idempotent, |
| 5 | # log groups/topics/subscriptions are created only if absent). |
| 6 | # |
| 7 | # Staging and production are separate AWS accounts with separate log groups |
| 8 | # (/musehub/staging, /musehub/production) — this script targets one |
| 9 | # environment per run, matching deploy/push.sh's per-environment AWS_PROFILE |
| 10 | # pattern. Run it once per environment, under that environment's own SSO |
| 11 | # session (aws sso login --profile musehub-nonproduction|musehub-production). |
| 12 | # |
| 13 | # Prerequisites: |
| 14 | # * AWS CLI configured with permissions for logs:*, cloudwatch:*, sns:* |
| 15 | # (the operator's own SSO session already has this via AdministratorAccess) |
| 16 | # * ALERT_EMAILS set in the environment, or accept the default (Gabriel + Aaron) |
| 17 | # |
| 18 | # Usage: |
| 19 | # bash deploy/cloudwatch-alerts.sh --env staging --profile musehub-nonproduction |
| 20 | # bash deploy/cloudwatch-alerts.sh --env production --profile musehub-production |
| 21 | |
| 22 | set -euo pipefail |
| 23 | |
| 24 | # ── Config ──────────────────────────────────────────────────────────────────── |
| 25 | |
| 26 | AWS_REGION="${AWS_REGION:-us-east-1}" |
| 27 | AWS_PROFILE_ARG="${AWS_PROFILE_ARG:-}" |
| 28 | ENV="" |
| 29 | LOG_RETENTION_DAYS=30 # Hot retention: 30 days in CloudWatch |
| 30 | # Route alerts to both maintainers by default — override with ALERT_EMAILS |
| 31 | # (comma-separated) if that ever needs to change. |
| 32 | ALERT_EMAILS="${ALERT_EMAILS:[email protected],[email protected]}" |
| 33 | |
| 34 | # Alarm thresholds — aligned with the pre-launch checklist and #149's RPO/RTO decisions. |
| 35 | THRESHOLD_5XX_RATE="1" # percent: alarm when 5xx / total > 1% |
| 36 | THRESHOLD_P99_LATENCY_MS="2000" # milliseconds |
| 37 | THRESHOLD_DB_CONNECTIONS="40" # db.t3.micro's default max_connections is ~87; alarm well before exhaustion |
| 38 | |
| 39 | # ── Helpers ─────────────────────────────────────────────────────────────────── |
| 40 | |
| 41 | log() { echo "[cloudwatch] $*"; } |
| 42 | die() { echo "[cloudwatch] ERROR: $*" >&2; exit 1; } |
| 43 | |
| 44 | aws_cmd() { |
| 45 | if [[ -n "$AWS_PROFILE_ARG" ]]; then |
| 46 | aws --region "$AWS_REGION" --profile "$AWS_PROFILE_ARG" "$@" |
| 47 | else |
| 48 | aws --region "$AWS_REGION" "$@" |
| 49 | fi |
| 50 | } |
| 51 | |
| 52 | # ── Parse args ──────────────────────────────────────────────────────────────── |
| 53 | |
| 54 | while [[ $# -gt 0 ]]; do |
| 55 | case "$1" in |
| 56 | --env) ENV="$2"; shift 2 ;; |
| 57 | --profile) AWS_PROFILE_ARG="$2"; shift 2 ;; |
| 58 | --region) AWS_REGION="$2"; shift 2 ;; |
| 59 | *) die "Unknown argument: $1" ;; |
| 60 | esac |
| 61 | done |
| 62 | |
| 63 | case "$ENV" in |
| 64 | staging|production) ;; |
| 65 | *) die "Usage: bash deploy/cloudwatch-alerts.sh --env staging|production --profile <sso-profile>" ;; |
| 66 | esac |
| 67 | |
| 68 | LOG_GROUP="/musehub/${ENV}" |
| 69 | SNS_TOPIC_NAME="musehub-${ENV}-alerts" |
| 70 | |
| 71 | # Both environments run on managed AWS RDS as of 2026-09-08 — see |
| 72 | # docs/database-architecture.md. One instance per environment, same naming |
| 73 | # convention. |
| 74 | case "$ENV" in |
| 75 | staging) RDS_INSTANCE_ID="musehub-staging-db" ;; |
| 76 | production) RDS_INSTANCE_ID="musehub-production-db" ;; |
| 77 | esac |
| 78 | |
| 79 | # ── Step 1: Log group + explicit hot retention ─────────────────────────────── |
| 80 | # The log group should already exist (created by the app's awslogs driver on |
| 81 | # first deploy) — this just ensures retention is set, since CloudWatch Logs |
| 82 | # default to unlimited retention otherwise (a cost/compliance gap, not an |
| 83 | # observability one). |
| 84 | |
| 85 | log "[1/5] Setting ${LOG_RETENTION_DAYS}-day retention on $LOG_GROUP..." |
| 86 | aws_cmd logs create-log-group --log-group-name "$LOG_GROUP" 2>/dev/null || true # idempotent |
| 87 | aws_cmd logs put-retention-policy \ |
| 88 | --log-group-name "$LOG_GROUP" \ |
| 89 | --retention-in-days "$LOG_RETENTION_DAYS" |
| 90 | |
| 91 | # ── Step 2: SNS topic + subscriptions ──────────────────────────────────────── |
| 92 | |
| 93 | log "[2/5] Creating SNS topic $SNS_TOPIC_NAME..." |
| 94 | |
| 95 | SNS_ARN=$(aws_cmd sns create-topic \ |
| 96 | --name "$SNS_TOPIC_NAME" \ |
| 97 | --query TopicArn --output text) |
| 98 | |
| 99 | log "SNS topic ARN: $SNS_ARN" |
| 100 | |
| 101 | IFS=',' read -ra EMAILS <<< "$ALERT_EMAILS" |
| 102 | for email in "${EMAILS[@]}"; do |
| 103 | email="$(echo "$email" | xargs)" # trim whitespace |
| 104 | [[ -z "$email" ]] && continue |
| 105 | aws_cmd sns subscribe \ |
| 106 | --topic-arn "$SNS_ARN" \ |
| 107 | --protocol email \ |
| 108 | --notification-endpoint "$email" \ |
| 109 | --query SubscriptionArn --output text > /dev/null |
| 110 | log " Email subscription created for $email (must confirm via the email AWS sends)" |
| 111 | done |
| 112 | |
| 113 | # ── Step 3: Metric filters (extract from structured JSON logs) ───────────────── |
| 114 | |
| 115 | log "[3/5] Creating metric filters on $LOG_GROUP..." |
| 116 | |
| 117 | aws_cmd logs put-metric-filter \ |
| 118 | --log-group-name "$LOG_GROUP" \ |
| 119 | --filter-name "musehub-5xx-count" \ |
| 120 | --filter-pattern '{ $.status >= 500 }' \ |
| 121 | --metric-transformations \ |
| 122 | metricName="5xxCount-${ENV}",metricNamespace="MuseHub",metricValue="1",defaultValue="0" |
| 123 | |
| 124 | aws_cmd logs put-metric-filter \ |
| 125 | --log-group-name "$LOG_GROUP" \ |
| 126 | --filter-name "musehub-request-count" \ |
| 127 | --filter-pattern '{ $.status >= 100 }' \ |
| 128 | --metric-transformations \ |
| 129 | metricName="RequestCount-${ENV}",metricNamespace="MuseHub",metricValue="1",defaultValue="0" |
| 130 | |
| 131 | aws_cmd logs put-metric-filter \ |
| 132 | --log-group-name "$LOG_GROUP" \ |
| 133 | --filter-name "musehub-duration-ms" \ |
| 134 | --filter-pattern '{ $.duration_ms > 0 }' \ |
| 135 | --metric-transformations \ |
| 136 | metricName="RequestDurationMs-${ENV}",metricNamespace="MuseHub",metricValue='$.duration_ms',defaultValue="0" |
| 137 | |
| 138 | log "Metric filters created: 5xxCount-${ENV}, RequestCount-${ENV}, RequestDurationMs-${ENV}" |
| 139 | |
| 140 | # ── Step 4: CloudWatch Alarms ───────────────────────────────────────────────── |
| 141 | |
| 142 | log "[4/5] Creating CloudWatch alarms (5xx>${THRESHOLD_5XX_RATE}%, p99>${THRESHOLD_P99_LATENCY_MS}ms)..." |
| 143 | |
| 144 | # ─ 4a: 5xx error rate ──────────────────────────────────────────────────────── |
| 145 | aws_cmd cloudwatch put-metric-alarm \ |
| 146 | --alarm-name "musehub-${ENV}-5xx-rate-high" \ |
| 147 | --alarm-description "[$ENV] 5xx error rate exceeded ${THRESHOLD_5XX_RATE}% — investigate immediately" \ |
| 148 | --alarm-actions "$SNS_ARN" \ |
| 149 | --ok-actions "$SNS_ARN" \ |
| 150 | --metrics \ |
| 151 | "[{\"Id\":\"e1\",\"Expression\":\"(m1/m2)*100\",\"Label\":\"5xxRate\",\"ReturnData\":true}, |
| 152 | {\"Id\":\"m1\",\"MetricStat\":{\"Metric\":{\"Namespace\":\"MuseHub\",\"MetricName\":\"5xxCount-${ENV}\"},\"Period\":60,\"Stat\":\"Sum\"},\"ReturnData\":false}, |
| 153 | {\"Id\":\"m2\",\"MetricStat\":{\"Metric\":{\"Namespace\":\"MuseHub\",\"MetricName\":\"RequestCount-${ENV}\"},\"Period\":60,\"Stat\":\"Sum\"},\"ReturnData\":false}]" \ |
| 154 | --comparison-operator GreaterThanThreshold \ |
| 155 | --threshold "$THRESHOLD_5XX_RATE" \ |
| 156 | --evaluation-periods 2 \ |
| 157 | --datapoints-to-alarm 2 \ |
| 158 | --treat-missing-data notBreaching |
| 159 | |
| 160 | # ─ 4b: p99 request latency ─────────────────────────────────────────────────── |
| 161 | aws_cmd cloudwatch put-metric-alarm \ |
| 162 | --alarm-name "musehub-${ENV}-p99-latency-high" \ |
| 163 | --alarm-description "[$ENV] p99 request latency exceeded ${THRESHOLD_P99_LATENCY_MS}ms" \ |
| 164 | --alarm-actions "$SNS_ARN" \ |
| 165 | --ok-actions "$SNS_ARN" \ |
| 166 | --namespace MuseHub \ |
| 167 | --metric-name "RequestDurationMs-${ENV}" \ |
| 168 | --period 60 \ |
| 169 | --evaluation-periods 3 \ |
| 170 | --datapoints-to-alarm 2 \ |
| 171 | --threshold "$THRESHOLD_P99_LATENCY_MS" \ |
| 172 | --comparison-operator GreaterThanThreshold \ |
| 173 | --treat-missing-data notBreaching \ |
| 174 | --extended-statistic "p99" |
| 175 | |
| 176 | log "Alarms created: musehub-${ENV}-5xx-rate-high, musehub-${ENV}-p99-latency-high" |
| 177 | |
| 178 | # ─ 4c: RDS database connections ────────────────────────────────────────────── |
| 179 | # Both environments are on managed AWS RDS as of 2026-09-08 (see |
| 180 | # docs/database-architecture.md) — this alarm was previously removed |
| 181 | # (2026-09-08, during #160) on the mistaken belief that this project only |
| 182 | # ever ran self-hosted Postgres. It's real and correct now for both |
| 183 | # environments: AWS/RDS DatabaseConnections is a native RDS metric, no |
| 184 | # CloudWatch Agent install required. |
| 185 | aws_cmd cloudwatch put-metric-alarm \ |
| 186 | --alarm-name "musehub-${ENV}-db-connections-high" \ |
| 187 | --alarm-description "[$ENV] RDS ($RDS_INSTANCE_ID) connection count exceeded ${THRESHOLD_DB_CONNECTIONS}" \ |
| 188 | --alarm-actions "$SNS_ARN" \ |
| 189 | --ok-actions "$SNS_ARN" \ |
| 190 | --namespace AWS/RDS \ |
| 191 | --metric-name DatabaseConnections \ |
| 192 | --dimensions "Name=DBInstanceIdentifier,Value=${RDS_INSTANCE_ID}" \ |
| 193 | --period 300 \ |
| 194 | --statistic Average \ |
| 195 | --evaluation-periods 2 \ |
| 196 | --datapoints-to-alarm 2 \ |
| 197 | --threshold "$THRESHOLD_DB_CONNECTIONS" \ |
| 198 | --comparison-operator GreaterThanThreshold \ |
| 199 | --treat-missing-data missing |
| 200 | |
| 201 | log "Alarms created: musehub-${ENV}-db-connections-high" |
| 202 | |
| 203 | # ── Not done here — needs its own follow-up, not a "quick win" ────────────── |
| 204 | # |
| 205 | # Disk/memory alarms for the EC2 app instance itself (not the database) need |
| 206 | # the CloudWatch Agent installed and configured on the instance (it is not |
| 207 | # installed today), publishing to the CWAgent namespace. |
| 208 | # |
| 209 | # CloudWatch Agent install (Ubuntu — the original version of this script |
| 210 | # incorrectly said `yum`, which doesn't exist on these instances): |
| 211 | # curl -O https://s3.amazonaws.com/amazoncloudwatch-agent/ubuntu/amd64/latest/amazon-cloudwatch-agent.deb |
| 212 | # sudo dpkg -i amazon-cloudwatch-agent.deb |
| 213 | # sudo /opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl \ |
| 214 | # -a fetch-config -m ec2 -s -c file:/opt/aws/amazon-cloudwatch-agent/etc/amazon-cloudwatch-agent.json |
| 215 | |
| 216 | # ── Step 5: Summary ─────────────────────────────────────────────────────────── |
| 217 | |
| 218 | log "[5/5] Done." |
| 219 | log "" |
| 220 | log " Environment: $ENV" |
| 221 | log " Log group : $LOG_GROUP (${LOG_RETENTION_DAYS}d retention)" |
| 222 | log " SNS topic : $SNS_ARN (subscribers: $ALERT_EMAILS)" |
| 223 | log " Alarms : 5xx rate > ${THRESHOLD_5XX_RATE}% | p99 > ${THRESHOLD_P99_LATENCY_MS}ms | DB connections > ${THRESHOLD_DB_CONNECTIONS}" |
| 224 | log "" |
| 225 | log " Verify with: aws cloudwatch describe-alarms --alarm-name-prefix musehub-${ENV} --region $AWS_REGION" |
File History
5 commits
sha256:bee12c5cbde2334f98421c6c209d768fa6b8004d6705c9ea798ce6c1651bc11f
Merge 'infra/database-phase3-4-cleanup' into 'dev' — propos…
Human
5 hours ago
sha256:316e70bc7bfc59633679c76f96aee8b77de3e8b04f70f3bb38ba83df3cb1a5ed
Merge 'infra/database-phase2-production-rds' into 'dev' — p…
Human
7 hours ago
sha256:9c64dbfd65ef4e8a85f500e5909c06c2e6c65255a69e84188ee73b63f111cecd
Merge 'docs/status-banners-closed-tickets' into 'dev' — pro…
Human
12 hours ago
sha256:3fadb0439bba9451b89229676971c0d4a40900dec7810e9d5f8791b8d950d505
fix: install.sh version from latest published tarball, not …
Sonnet 4.6
minor
⚠
102 days ago
sha256:763eb2cb8675073b84c19345b27586d2ed939a9aee97c5479b69f502f1a70eff
fix(tests): update test suite to match current implementation
Sonnet 4.6
patch
124 days ago