groups:
- name: manager.rules
  rules:
  - alert: ServerlessMeteringIsDown
    expr: up{component="serverless-metering"} == 0
    for: 10m
    labels:
      tier: serverless
      provider_type: aws-free-tier
      severity: critical
      response_class: page
      component: serverless-metering
      oncall_team: serverless
    annotations:
      message: 'region: {{ $labels.region }}, pod: {{ $labels.pod }}, values: {{ $value }}'
      summary: Serverless metering component is down.
  - alert: ServerelssMeteringPodRestarts
    annotations:
      summary: Serverless metering pod restarted.
      message: 'region:{{ $labels.region}}, container {{ $labels.namespace }}/{{ $labels.container }} restarted {{ printf "%.2f" $value }} times in 30 minutes.'
    expr: |
      increase(kube_pod_container_status_restarts_total{container=~"serverless-metering(.*)|serverless-meta(.*)"}[30m]) >= 1
    for: 1m
    labels:
      tier: serverless
      provider_type: aws-free-tier
      severity: warning
      component: serverless-metering
      oncall_team: serverless
  - alert: ServerlessMeteringLocalLoggerRotateLagTooHigh
    annotations:
      summary: Serverless metering local logger rotate lag too high.
      message: 'region:{{ $labels.region}}, pod: {{ $labels.pod }}, lag value: {{ $value }}'
    expr: |
      sum(timestamp(serverless_metering_local_logger_rotate_progress_timestmap) - serverless_metering_local_logger_rotate_progress_timestmap) by (region, pod) > 3900
    for: 5m
    labels:
      tier: serverless
      provider_type: aws-free-tier
      severity: critical
      response_class: page
      component: serverless-metering
      oncall_team: serverless
  - alert: ServerlessMeteringStoreUploadLagTooHigh
    annotations:
      summary: Serverless metering store upload lag too high.
      message: 'region:{{ $labels.region}}, pod: {{ $labels.pod }}, lag value: {{ $value }}'
    expr: |
      sum(timestamp(serverless_metering_store_upload_progress_timestmap) - (serverless_metering_store_upload_progress_timestmap > 0)) by (region, pod) > 3900
    for: 5m
    labels:
      tier: serverless
      provider_type: aws-free-tier
      severity: critical
      response_class: page
      component: serverless-metering
      oncall_team: serverless
  - alert: ServerlessMeteringWorkerDefaultLagTooHigh
    annotations:
      summary: Serverless metering worker default lag too high.
      message: 'region:{{ $labels.region}}, worker_name: {{ $labels.worker_name }}, lag value: {{ $value }}'
    expr: |
      sum(timestamp(serverless_metering_report_progress_timestmap{worker_name="worker-default"}) - serverless_metering_report_progress_timestmap{worker_name="worker-default"}) by (region, worker_name) > 5400
    for: 5m
    labels:
      tier: serverless
      provider_type: aws-free-tier
      severity: critical
      response_class: page
      component: serverless-metering
      oncall_team: serverless
  - alert: ServerlessMeteringWorkerStorageMediumLagTooHigh
    annotations:
      summary: Serverless metering worker storage medium lag too high.
      message: 'region:{{ $labels.region}}, worker_name: {{ $labels.worker_name }}, lag value: {{ $value }}'
    expr: |
      sum(timestamp(serverless_metering_report_progress_timestmap{worker_name="worker-storage-medium"}) - serverless_metering_report_progress_timestmap{worker_name="worker-storage-medium"}) by (region, worker_name) > 9000
    for: 5m
    labels:
      tier: serverless
      provider_type: aws-free-tier
      severity: critical
      response_class: page
      component: serverless-metering
      oncall_team: serverless
  - alert: ServerlessMeteringWorkerStorageMediumOneDayLagTooHigh
    annotations:
      summary: Serverless metering worker storage medium one day lag too high.
      message: 'region:{{ $labels.region}}, worker_name: {{ $labels.worker_name }}, lag value: {{ $value }}'
    expr: |
      sum(timestamp(serverless_metering_report_progress_timestmap{worker_name="worker-storage-medium-one-day"}) - serverless_metering_report_progress_timestmap{worker_name="worker-storage-medium-one-day"}) by (region, worker_name) > 88200
    for: 5m
    labels:
      tier: serverless
      provider_type: aws-free-tier
      severity: critical
      response_class: page
      component: serverless-metering
      oncall_team: serverless
  - alert: ServerlessMeteringPVCUsageTooHigh
    annotations:
      summary: Serverless metering pvc usage too high.
      message: 'region:{{ $labels.region}}, pvc_name: {{ $labels.persistentvolumeclaim }}, value: {{ $value }}'
    expr: |
      kubelet_volume_stats_available_bytes{persistentvolumeclaim=~"data-serverless-metering(.*)"}/kubelet_volume_stats_capacity_bytes{persistentvolumeclaim=~"data-serverless-metering(.*)"} < 0.2
    for: 10m
    labels:
      tier: serverless
      provider_type: aws-free-tier
      severity: critical
      response_class: page
      component: serverless-metering
      oncall_team: serverless