groups:
  - name: auditlog.rules
    rules:
      - alert: AuditLogSchedulerIsDown
        annotations:
          message: |
            {{ $labels.region }} audit-log-scheduler is down.
          region: '{{ $labels.region }}'
        expr: |
          up{pod=~"audit-log-scheduler.*"} == 0
        for: 5m
        labels:
          tier: serverless
          severity: critical
          response_class: page
          provider_type: aws-free-tier
          component: ecosystem-server
          oncall_team: serverless
      - alert: AuditLogSchedulerFrequentlyRestart
        expr:  sum(changes(process_start_time_seconds{pod=~"audit-log-scheduler.*"}[30m])) by (instance, region) >= 4
        for: 5m
        labels:
          tier: serverless
          provider_type: aws-free-tier
          severity: critical
          response_class: page
          component: ecosystem-server
          oncall_team: serverless
        annotations:
          message: 'Audit Log Scheduler has restarted at least 4 times in the last 30 minutes. region: {{ $labels.region }}, instance: {{ $labels.instance }}'
          value: '{{ $value }}'
      - alert: AuditLogSchedulerMergeFailed
        annotations:
          message: |
            Audit log scheduler merge {{ $labels.cluster_id }} failed.
        expr: |
          sum(increase(audit_log_scheduler_merge_failed_number[5m])) by (cluster_id) > 0
        labels:
          tier: serverless
          severity: warning
          provider_type: aws-free-tier
          component: ecosystem-server
          oncall_team: serverless
      - alert: AuditLogSchedulerDispatchFailed
        annotations:
          message: |
            Audit log scheduler dispatch failed. region: {{ $labels.region }}, type : {{ $labels.type }}.
        expr: |
          sum(increase(audit_log_scheduler_dispatch_failed_number[5m])) by (region,type) > 0
        labels:
          tier: serverless
          severity: warning
          provider_type: aws-free-tier
          component: ecosystem-server
          oncall_team: serverless
      - alert: AuditLogTaskCorrupted
        annotations:
          message: |
            Audit log task corrupted, please check the logs. region: {{ $labels.region }}
        expr: |
          sum(increase(audit_log_task_corrupted_number[5m])) by (region) > 0
        labels:
          tier: serverless
          severity: warning
          provider_type: aws-free-tier
          component: ecosystem-server
          oncall_team: serverless
      - alert: AuditLogTaskRunningTooLong
        annotations:
          message: |
            Audit log task running more than an hour. cluster_id: {{ $labels.cluster_id }}, task_id: {{ $labels.task_id }}, duration: {{ $value }}s
        expr: |
          sum(increase(audit_log_task_duration_seconds[5m])) by (cluster_id, task_id) > 0
        labels:
          tier: serverless
          severity: warning
          provider_type: aws-free-tier
          component: ecosystem-server
          oncall_team: serverless