# Alarms, notifications, and the budget.
#
# The alarm set is deliberately small. An alarm nobody acts on trains people to ignore the channel,
# and this is a two-task service watched by one person; four alarms that always mean something beat
# twenty that mostly do not.
#
# That principle was stated here from the start and then undercut twice in the first hour of the
# stack existing. Both mistakes are corrected below, and both are worth naming because they are the
# ordinary way an alert channel dies.
#
#   1. Every alarm also notified on RECOVERY (`ok_actions`), so a single flap sent two mails.
#   2. The alarms were live before the service had an image to run, so they flapped continuously
#      against data that did not exist yet.
#
# The result was a mailbox full of alarms about a service that had never started. Nobody reads the
# fifth one of those, which means nobody reads the first real one either.

# Belongs in variables.tf and is here only because that file was not editable when this was written.
# Move it when you next touch variables.tf.
variable "alarm_notifications" {
  description = <<-EOT
    Whether alarms are allowed to notify.

    Defaults to TRUE, deliberately. The safe state is the default, and silence is the thing that has
    to be asked for, in a file a person reads. A default of false would mean a stack that never
    pages and never tells anyone it decided not to.

    Set it to false in terraform.tfvars while the services have no image to run, because until the
    first CI deploy the alarms are describing an empty cluster rather than a fault. REMOVE that line
    once the first deploy is green. If you find this still set to false weeks later, the alarms have
    been off the whole time and nobody noticed, which is the failure this comment exists to prevent.
  EOT
  type        = bool
  default     = true
}

resource "aws_sns_topic" "alerts" {
  count = var.alert_email != "" ? 1 : 0
  name  = "mcp-alerts"
}

resource "aws_sns_topic_subscription" "email" {
  count = var.alert_email != "" ? 1 : 0

  topic_arn = aws_sns_topic.alerts[0].arn
  protocol  = "email"
  endpoint  = var.alert_email

  # The subscription is created PENDING and stays that way until the address confirms it by email.
  # Terraform reports success either way, so an unconfirmed subscription is a silent no-op: alarms
  # fire, the topic publishes, and nobody is told. Check the subscription state once after the
  # first apply.
}

locals {
  alarm_actions = var.alert_email != "" ? [aws_sns_topic.alerts[0].arn] : []
}

# NO `ok_actions` on anything in this file.
#
# Recovery mail sounds harmless and is the reason an alert channel becomes noise: it doubles the
# volume, and half of what it doubles is "the thing that was broken is no longer broken", which
# nobody needs at 3am. If a recovery genuinely matters, the alarm state is in the console and the
# next real notification will say so.

# Something is answering. The single most useful alarm here: it covers the app crashing, the image
# failing to start, and the task role being wrong, all of which look identical from outside.
resource "aws_cloudwatch_metric_alarm" "unhealthy_targets" {
  for_each = {
    official = aws_lb_target_group.official_prod.arn_suffix
    poc      = aws_lb_target_group.poc.arn_suffix
  }

  alarm_name          = "mcp-${each.key}-unhealthy-targets"
  namespace           = "AWS/ApplicationELB"
  metric_name         = "UnHealthyHostCount"
  statistic           = "Maximum"
  period              = 60
  evaluation_periods  = 3
  threshold           = 0
  comparison_operator = "GreaterThanThreshold"

  dimensions = {
    TargetGroup  = each.value
    LoadBalancer = each.key == "official" ? aws_lb.official.arn_suffix : aws_lb.poc.arn_suffix
  }

  # A target that is missing entirely reports no datapoints rather than a zero, so treating missing
  # data as breaching is what makes "the service is gone" alarm at all. It is also why this alarm
  # sits in ALARM from the moment it is created until the first image is deployed, and why
  # `alarm_notifications` exists.
  treat_missing_data = "breaching"

  actions_enabled = var.alarm_notifications
  alarm_actions   = local.alarm_actions
}

# Our own 5xx, not the target's. Separated because they mean different things: the ALB returning
# 5xx means it could not reach a healthy task, which is an infrastructure problem.
resource "aws_cloudwatch_metric_alarm" "elb_5xx" {
  for_each = {
    official = aws_lb.official.arn_suffix
    poc      = aws_lb.poc.arn_suffix
  }

  alarm_name          = "mcp-${each.key}-elb-5xx"
  namespace           = "AWS/ApplicationELB"
  metric_name         = "HTTPCode_ELB_5XX_Count"
  statistic           = "Sum"
  period              = 300
  evaluation_periods  = 1
  threshold           = 5
  comparison_operator = "GreaterThanThreshold"

  dimensions = { LoadBalancer = each.value }

  # Absence of 5xx is absence of traffic here, not a problem.
  treat_missing_data = "notBreaching"

  actions_enabled = var.alarm_notifications
  alarm_actions   = local.alarm_actions
}

# Running fewer tasks than asked for. Catches a task that keeps crash-looping, which the health
# alarm above may miss if a replacement briefly registers healthy each time.
resource "aws_cloudwatch_metric_alarm" "running_below_desired" {
  alarm_name          = "mcp-official-running-below-desired"
  namespace           = "ECS/ContainerInsights"
  metric_name         = "RunningTaskCount"
  statistic           = "Minimum"
  period              = 300
  evaluation_periods  = 2
  threshold           = var.official_task_count
  comparison_operator = "LessThanThreshold"

  dimensions = {
    ClusterName = aws_ecs_cluster.main.name
    ServiceName = aws_ecs_service.official_prod.name
  }

  # Container Insights is disabled on the cluster to avoid per-metric charges, so this alarm has no
  # data source unless it is turned on, and it sits in INSUFFICIENT_DATA. It is kept as the switch
  # to flip if the service ever justifies the cost. Missing data is not treated as breaching
  # precisely because that would make it page immediately and forever.
  treat_missing_data = "missing"

  actions_enabled = var.alarm_notifications
  alarm_actions   = local.alarm_actions
}

# ---------------------------------------------------------------------------------------------
# Budget
# ---------------------------------------------------------------------------------------------

# Alerts at 50/80/100 percent of the ceiling, not of the estimate.
#
# The estimate is 85-95 and the ceiling is 150. Alerting on percentages of the estimate would page
# on ordinary variance; alerting only at the ceiling would page when the month is already spent.
# The 50 percent notice is the one that catches a mistake early, and it lands around the point
# where normal usage sits.
#
# These are NOT gated by `alarm_notifications`. A cost surprise is worth hearing about whether or
# not the service is running, and an empty stack that starts costing money is exactly the case you
# would want told about.
locals {
  # A Budgets notification with no subscriber is rejected by the API, so the notifications are
  # driven off a list that is EMPTY when no address is configured, rather than off a list of
  # thresholds with an empty subscriber field. The budget itself still exists and still tracks
  # spend; it just has nobody to tell.
  budget_notifications = var.alert_email == "" ? [] : [
    { threshold = 50, type = "ACTUAL" },
    { threshold = 80, type = "ACTUAL" },
    { threshold = 100, type = "ACTUAL" },
    { threshold = 100, type = "FORECASTED" },
  ]
}

resource "aws_budgets_budget" "monthly" {
  name         = "mcp-monthly"
  budget_type  = "COST"
  limit_amount = tostring(var.monthly_budget_usd)
  limit_unit   = "USD"
  time_unit    = "MONTHLY"

  dynamic "notification" {
    for_each = local.budget_notifications
    content {
      comparison_operator        = "GREATER_THAN"
      threshold                  = notification.value.threshold
      threshold_type             = "PERCENTAGE"
      notification_type          = notification.value.type
      subscriber_email_addresses = [var.alert_email]
    }
  }
}