#!/usr/bin/env bash
# MuseHub — CloudWatch log group, metric filters, alarms, and SNS alerting.
#
# Run once per environment; safe to re-run (put-metric-alarm is idempotent,
# log groups/topics/subscriptions are created only if absent).
#
# Staging and production are separate AWS accounts with separate log groups
# (/musehub/staging, /musehub/production) — this script targets one
# environment per run, matching deploy/push.sh's per-environment AWS_PROFILE
# pattern. Run it once per environment, under that environment's own SSO
# session (aws sso login --profile musehub-nonproduction|musehub-production).
#
# Prerequisites:
#   * AWS CLI configured with permissions for logs:*, cloudwatch:*, sns:*
#     (the operator's own SSO session already has this via AdministratorAccess)
#   * ALERT_EMAILS set in the environment, or accept the default (Gabriel + Aaron)
#
# Usage:
#   bash deploy/cloudwatch-alerts.sh --env staging --profile musehub-nonproduction
#   bash deploy/cloudwatch-alerts.sh --env production --profile musehub-production

set -euo pipefail

# ── Config ────────────────────────────────────────────────────────────────────

AWS_REGION="${AWS_REGION:-us-east-1}"
AWS_PROFILE_ARG="${AWS_PROFILE_ARG:-}"
ENV=""
LOG_RETENTION_DAYS=30                   # Hot retention: 30 days in CloudWatch
# Route alerts to both maintainers by default — override with ALERT_EMAILS
# (comma-separated) if that ever needs to change.
ALERT_EMAILS="${ALERT_EMAILS:-gabriel@musehub.ai,aaronrene@musehub.ai}"

# Alarm thresholds — aligned with the pre-launch checklist and #149's RPO/RTO decisions.
THRESHOLD_5XX_RATE="1"          # percent: alarm when 5xx / total > 1%
THRESHOLD_P99_LATENCY_MS="2000" # milliseconds
THRESHOLD_DB_CONNECTIONS="40"   # db.t3.micro's default max_connections is ~87; alarm well before exhaustion

# ── Helpers ───────────────────────────────────────────────────────────────────

log()  { echo "[cloudwatch] $*"; }
die()  { echo "[cloudwatch] ERROR: $*" >&2; exit 1; }

aws_cmd() {
    if [[ -n "$AWS_PROFILE_ARG" ]]; then
        aws --region "$AWS_REGION" --profile "$AWS_PROFILE_ARG" "$@"
    else
        aws --region "$AWS_REGION" "$@"
    fi
}

# ── Parse args ────────────────────────────────────────────────────────────────

while [[ $# -gt 0 ]]; do
    case "$1" in
        --env)     ENV="$2";             shift 2 ;;
        --profile) AWS_PROFILE_ARG="$2"; shift 2 ;;
        --region)  AWS_REGION="$2";      shift 2 ;;
        *) die "Unknown argument: $1" ;;
    esac
done

case "$ENV" in
    staging|production) ;;
    *) die "Usage: bash deploy/cloudwatch-alerts.sh --env staging|production --profile <sso-profile>" ;;
esac

LOG_GROUP="/musehub/${ENV}"
SNS_TOPIC_NAME="musehub-${ENV}-alerts"

# Both environments run on managed AWS RDS as of 2026-09-08 — see
# docs/database-architecture.md. One instance per environment, same naming
# convention.
case "$ENV" in
    staging)    RDS_INSTANCE_ID="musehub-staging-db" ;;
    production) RDS_INSTANCE_ID="musehub-production-db" ;;
esac

# ── Step 1: Log group + explicit hot retention ───────────────────────────────
# The log group should already exist (created by the app's awslogs driver on
# first deploy) — this just ensures retention is set, since CloudWatch Logs
# default to unlimited retention otherwise (a cost/compliance gap, not an
# observability one).

log "[1/5] Setting ${LOG_RETENTION_DAYS}-day retention on $LOG_GROUP..."
aws_cmd logs create-log-group --log-group-name "$LOG_GROUP" 2>/dev/null || true   # idempotent
aws_cmd logs put-retention-policy \
    --log-group-name "$LOG_GROUP" \
    --retention-in-days "$LOG_RETENTION_DAYS"

# ── Step 2: SNS topic + subscriptions ────────────────────────────────────────

log "[2/5] Creating SNS topic $SNS_TOPIC_NAME..."

SNS_ARN=$(aws_cmd sns create-topic \
    --name "$SNS_TOPIC_NAME" \
    --query TopicArn --output text)

log "SNS topic ARN: $SNS_ARN"

IFS=',' read -ra EMAILS <<< "$ALERT_EMAILS"
for email in "${EMAILS[@]}"; do
    email="$(echo "$email" | xargs)"  # trim whitespace
    [[ -z "$email" ]] && continue
    aws_cmd sns subscribe \
        --topic-arn "$SNS_ARN" \
        --protocol email \
        --notification-endpoint "$email" \
        --query SubscriptionArn --output text > /dev/null
    log "  Email subscription created for $email (must confirm via the email AWS sends)"
done

# ── Step 3: Metric filters (extract from structured JSON logs) ─────────────────

log "[3/5] Creating metric filters on $LOG_GROUP..."

aws_cmd logs put-metric-filter \
    --log-group-name "$LOG_GROUP" \
    --filter-name "musehub-5xx-count" \
    --filter-pattern '{ $.status >= 500 }' \
    --metric-transformations \
        metricName="5xxCount-${ENV}",metricNamespace="MuseHub",metricValue="1",defaultValue="0"

aws_cmd logs put-metric-filter \
    --log-group-name "$LOG_GROUP" \
    --filter-name "musehub-request-count" \
    --filter-pattern '{ $.status >= 100 }' \
    --metric-transformations \
        metricName="RequestCount-${ENV}",metricNamespace="MuseHub",metricValue="1",defaultValue="0"

aws_cmd logs put-metric-filter \
    --log-group-name "$LOG_GROUP" \
    --filter-name "musehub-duration-ms" \
    --filter-pattern '{ $.duration_ms > 0 }' \
    --metric-transformations \
        metricName="RequestDurationMs-${ENV}",metricNamespace="MuseHub",metricValue='$.duration_ms',defaultValue="0"

log "Metric filters created: 5xxCount-${ENV}, RequestCount-${ENV}, RequestDurationMs-${ENV}"

# ── Step 4: CloudWatch Alarms ─────────────────────────────────────────────────

log "[4/5] Creating CloudWatch alarms (5xx>${THRESHOLD_5XX_RATE}%, p99>${THRESHOLD_P99_LATENCY_MS}ms)..."

# ─ 4a: 5xx error rate ────────────────────────────────────────────────────────
aws_cmd cloudwatch put-metric-alarm \
    --alarm-name "musehub-${ENV}-5xx-rate-high" \
    --alarm-description "[$ENV] 5xx error rate exceeded ${THRESHOLD_5XX_RATE}% — investigate immediately" \
    --alarm-actions "$SNS_ARN" \
    --ok-actions "$SNS_ARN" \
    --metrics \
        "[{\"Id\":\"e1\",\"Expression\":\"(m1/m2)*100\",\"Label\":\"5xxRate\",\"ReturnData\":true},
          {\"Id\":\"m1\",\"MetricStat\":{\"Metric\":{\"Namespace\":\"MuseHub\",\"MetricName\":\"5xxCount-${ENV}\"},\"Period\":60,\"Stat\":\"Sum\"},\"ReturnData\":false},
          {\"Id\":\"m2\",\"MetricStat\":{\"Metric\":{\"Namespace\":\"MuseHub\",\"MetricName\":\"RequestCount-${ENV}\"},\"Period\":60,\"Stat\":\"Sum\"},\"ReturnData\":false}]" \
    --comparison-operator GreaterThanThreshold \
    --threshold "$THRESHOLD_5XX_RATE" \
    --evaluation-periods 2 \
    --datapoints-to-alarm 2 \
    --treat-missing-data notBreaching

# ─ 4b: p99 request latency ───────────────────────────────────────────────────
aws_cmd cloudwatch put-metric-alarm \
    --alarm-name "musehub-${ENV}-p99-latency-high" \
    --alarm-description "[$ENV] p99 request latency exceeded ${THRESHOLD_P99_LATENCY_MS}ms" \
    --alarm-actions "$SNS_ARN" \
    --ok-actions "$SNS_ARN" \
    --namespace MuseHub \
    --metric-name "RequestDurationMs-${ENV}" \
    --period 60 \
    --evaluation-periods 3 \
    --datapoints-to-alarm 2 \
    --threshold "$THRESHOLD_P99_LATENCY_MS" \
    --comparison-operator GreaterThanThreshold \
    --treat-missing-data notBreaching \
    --extended-statistic "p99"

log "Alarms created: musehub-${ENV}-5xx-rate-high, musehub-${ENV}-p99-latency-high"

# ─ 4c: RDS database connections ──────────────────────────────────────────────
# Both environments are on managed AWS RDS as of 2026-09-08 (see
# docs/database-architecture.md) — this alarm was previously removed
# (2026-09-08, during #160) on the mistaken belief that this project only
# ever ran self-hosted Postgres. It's real and correct now for both
# environments: AWS/RDS DatabaseConnections is a native RDS metric, no
# CloudWatch Agent install required.
aws_cmd cloudwatch put-metric-alarm \
    --alarm-name "musehub-${ENV}-db-connections-high" \
    --alarm-description "[$ENV] RDS ($RDS_INSTANCE_ID) connection count exceeded ${THRESHOLD_DB_CONNECTIONS}" \
    --alarm-actions "$SNS_ARN" \
    --ok-actions "$SNS_ARN" \
    --namespace AWS/RDS \
    --metric-name DatabaseConnections \
    --dimensions "Name=DBInstanceIdentifier,Value=${RDS_INSTANCE_ID}" \
    --period 300 \
    --statistic Average \
    --evaluation-periods 2 \
    --datapoints-to-alarm 2 \
    --threshold "$THRESHOLD_DB_CONNECTIONS" \
    --comparison-operator GreaterThanThreshold \
    --treat-missing-data missing

log "Alarms created: musehub-${ENV}-db-connections-high"

# ── Not done here — needs its own follow-up, not a "quick win" ──────────────
#
# Disk/memory alarms for the EC2 app instance itself (not the database) need
# the CloudWatch Agent installed and configured on the instance (it is not
# installed today), publishing to the CWAgent namespace.
#
# CloudWatch Agent install (Ubuntu — the original version of this script
# incorrectly said `yum`, which doesn't exist on these instances):
#   curl -O https://s3.amazonaws.com/amazoncloudwatch-agent/ubuntu/amd64/latest/amazon-cloudwatch-agent.deb
#   sudo dpkg -i amazon-cloudwatch-agent.deb
#   sudo /opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl \
#       -a fetch-config -m ec2 -s -c file:/opt/aws/amazon-cloudwatch-agent/etc/amazon-cloudwatch-agent.json

# ── Step 5: Summary ───────────────────────────────────────────────────────────

log "[5/5] Done."
log ""
log "  Environment: $ENV"
log "  Log group  : $LOG_GROUP  (${LOG_RETENTION_DAYS}d retention)"
log "  SNS topic  : $SNS_ARN  (subscribers: $ALERT_EMAILS)"
log "  Alarms     : 5xx rate > ${THRESHOLD_5XX_RATE}%  |  p99 > ${THRESHOLD_P99_LATENCY_MS}ms  |  DB connections > ${THRESHOLD_DB_CONNECTIONS}"
log ""
log "  Verify with: aws cloudwatch describe-alarms --alarm-name-prefix musehub-${ENV} --region $AWS_REGION"