groups:
  - name: manager.rules
    rules:
      - alert: ManagerOutOfMemory
        annotations:
          description: 'At least one Manager node in cluster TiDBClusterName in project TiDBProjectName ran out of memory.'
          summary: 'Serverless manager node ran out of memory'
          region: '{{ $labels.region }}'
        expr: |
          sum(label_replace(increase(kube_pod_container_status_restarts_total{cluster=~"freetier.*",job="manager"}[5m])>=1
          AND ignoring(reason) kube_pod_container_status_last_terminated_reason{cluster=~"freetier.*",reason='OOMKilled',job="manager"}==1,"cluster_id","$1","pod","db(.*)-(.*)-(.*)"))by(cluster_id)
        labels:
          tier: serverless
          severity: critical
          response_class: page
          provider_type: aws-free-tier
          component: manager
          oncall_team: serverless
      - alert: ManagerIsDown
        annotations:
          message: |
            region {{ $labels.region }}, manager of serverless is down.
        expr: |
          up{cluster=~"freetier.*", job="manager"} == 0
        for: 3m
        labels:
          tier: serverless
          severity: critical
          response_class: page
          provider_type: aws-free-tier
          component: manager
          oncall_team: serverless
      - alert: ManagerHighCPUUsage
        annotations:
          message: |
            cluster {{ $labels.cluster }}, Manager CPU usage is high (>{{ $value | humanizePercentage }}) for pod {{ $labels.pod }}.
        expr: |
          rate(container_cpu_usage_seconds_total{namespace="tidb-admin",pod=~"manager-.*",image=""}[5m]) / on(pod) kube_pod_container_resource_limits{resource="cpu",namespace="tidb-admin",pod=~"manager-.*"} > 0.8
        for: 5m
        labels:
          tier: serverless
          severity: critical
          response_class: page
          provider_type: aws-free-tier
          component: manager
          oncall_team: serverless
      - alert: ManagerHighMemoryUsage
        annotations:
          message: |
            Manager memory usage is high (>{{ $value | humanizePercentage }}) for pod {{ $labels.pod }}.
        expr: |
          container_memory_working_set_bytes{pod=~"manager-.*",container!="POD",container!="",image!="",namespace="tidb-admin"}/ on(pod) kube_pod_container_resource_limits{resource="memory",namespace="tidb-admin",pod=~"manager-.*"} > 0.8
        for: 5m
        labels:
          tier: serverless
          severity: critical
          response_class: page
          provider_type: aws-free-tier
          component: manager
          oncall_team: serverless
      - alert: ManagerAssignTidbFailedTooMuch
        annotations:
          message: |
            manager assign tidb failed too much, affects {{ $value }} users.
        expr: |
          count(sum(increase(tidbmanager_assign_tidb_seconds_count{stage="total_fail"}[10m])) by (token) > 0) > 10
        for: 10m
        labels:
          tier: serverless
          provider_type: aws-free-tier
          severity: warning
          component: manager
          oncall_team: serverless
      - alert: ManagerAssignTidbFailedByToken
        annotations:
          message: |
            manager assign tidb failed for user token: {{ $labels.token }}, count: {{ $value }}.
        expr: |
          sum(increase(tidbmanager_assign_tidb_seconds_count{stage="total_fail"}[1m])) by (token) > 2
        for: 2m
        labels:
          tier: serverless
          provider_type: aws-free-tier
          severity: warning
          component: manager
          oncall_team: serverless
      - alert: ManagerFixTidbPool
        annotations:
          message: |
            cluster {{ $labels.cluster }}, manager fixed empty tidb pool {{ $labels.pool_ns }}.
        expr: |
          increase(tidbmanager_fix_tidbpool_total[1m]) > 0
        labels:
          tier: serverless
          provider_type: aws-free-tier
          severity: warning
          component: manager
          oncall_team: serverless
      - alert: ManagerResourceReconcileFailedTooManyTimes
        annotations:
          description: 'manager controller {{ $labels.controller }} keeps reporting reconcile errors for 30 minutes. Serverless manager reconcile errors keep occurring for 30 minutes.'
          summary: 'Serverless manager controller keeps failing to reconcile for 30 minutes'
        expr: |
          sum(increase(controller_runtime_reconcile_errors_total{job="manager"}[1m])) by (controller) > 0
        for: 30m
        labels:
          tier: serverless
          provider_type: aws-free-tier
          severity: major
          response_class: ticket
          component: manager
          oncall_team: serverless
      - alert: SuperVipManagerAssignTidbFailedByToken
        annotations:
          message: |
            manager assign tidb failed for user token: {{ $labels.token }}, count: {{ $value }}.
        expr: |
          sum(increase(tidbmanager_assign_tidb_seconds_count{stage="total_fail", pool_ns="super-vip-tidb-pool"}[1m])) by (token) > 2
        for: 5m
        labels:
          tier: serverless
          provider_type: aws-free-tier
          severity: critical
          response_class: page
          component: manager
          oncall_team: serverless
      - alert: ManagerZRUnevictablePVC
        annotations:
          message: |
            Manager ZR found unevictable PVC, region {{ $labels.region }}, pod {{ $labels.podname }}.
          region: '{{ $labels.region }}'
        expr: |
          increase(zr_evicter_unevictable_pvc_total{job="manager"}[1m]) > 0
        labels:
          tier: serverless
          provider_type: aws-free-tier
          severity: warning
          component: '{{ $labels.component }}'
          oncall_team: serverless
      - alert: ManagerTidbFreePodMoreThan200
        annotations:
          message: |
            Manager tidb free pod more than 200, current number {{ $value }}, pool {{ $labels.pool_ns }}.
        expr: |
          sum(tidbmanager_cache_size{type="free",region!="us-east-1"}) by (pool_ns) > 200
        for: 10m
        labels:
          tier: serverless
          provider_type: aws-free-tier
          severity: warning
          component: manager
          oncall_team: serverless
      - alert: ManagerTidbFreePodMoreThan300
        annotations:
          message: |
            Manager tidb free pod more than 300, current number {{ $value }}, pool {{ $labels.pool_ns }}.
        expr: |
          sum(tidbmanager_cache_size{type="free",region="us-east-1"}) by (pool_ns) > 300
        for: 10m
        labels:
          tier: serverless
          provider_type: aws-free-tier
          severity: warning
          component: manager
          oncall_team: serverless
      - alert: ManagerNoFreeTidbPod
        annotations:
          message: |
            Tidb pool {{ $labels.pool_ns }} has no free tidb pods for 1min.
        expr: |
          sum(tidbmanager_cache_size{type="free", pool_ns="tidb-pool", cluster=~".*us-east-1-f01|.*us-west-2-f01*|.*ap-northeast-1-f01|.*eu-central-1-f01|.*ap-southeast-1.*|.*sa-east-1-f01"}) by (pool_ns) == 0
        for: 1m
        labels:
          tier: serverless
          provider_type: aws-free-tier
          severity: warning
          component: manager
          oncall_team: serverless
      - alert: ManagerNoFreeTidbPod
        annotations:
          message: |
            Tidb pool {{ $labels.pool_ns }} has no free tidb pods for 10min.
        expr: |
          sum(tidbmanager_cache_size{type="free", pool_ns="tidb-pool", cluster=~".*us-east-1-f01|.*us-west-2-f01*|.*ap-northeast-1-f01|.*eu-central-1-f01|.*ap-southeast-1.*|.*sa-east-1-f01"}) by (pool_ns) == 0
        for: 10m
        labels:
          tier: serverless
          provider_type: aws-free-tier
          severity: critical
          response_class: page
          component: manager
          oncall_team: serverless
      - alert: ManagerNoFreeTidbPod
        annotations:
          message: |
            Tidb pool {{ $labels.pool_ns }} has no free tidb pods for 2m.
        expr: |
          sum(
            tidbmanager_cache_size{type="free",pool_ns=~".*vip.*",pool_ns!~".*export.*|.*canary.*"}
            unless
            tidbmanager_cache_size{pool_ns="super-vip-tidb-pool", cluster=~"freetier/eks/prod-(us-(east-1-f0[2-9]|west-2-f03)|sa-east-1-f01)"}
            unless
            tidbmanager_cache_size{pool_ns="vip-tidb-pool", cluster!~".*us-(east-1-f0[1-9])|.*us-west-2-f01*|.*ap-northeast-1-f01|.*eu-central-1-f01|.*ap-southeast-1.*|.*sa-east-1-f01"}
          ) by (pool_ns) == 0
        for: 2m
        labels:
          tier: serverless
          provider_type: aws-free-tier
          severity: warning
          component: manager
          oncall_team: serverless
      - alert: ManagerNoFreeTidbPod
        annotations:
          message: |
            Tidb pool {{ $labels.pool_ns }} has no free tidb pods for 5min.
        expr: |
          sum(
            tidbmanager_cache_size{type="free",pool_ns=~".*vip.*",pool_ns!~".*export.*|.*canary.*"}
            unless
            tidbmanager_cache_size{pool_ns="super-vip-tidb-pool", cluster=~"freetier/eks/prod-(us-(east-1-f0[2-9]|west-2-f03)|sa-east-1-f01)"}
            unless
            tidbmanager_cache_size{pool_ns="vip-tidb-pool", cluster!~".*us-(east-1-f0[1-9])|.*us-west-2-f01*|.*ap-northeast-1-f01|.*eu-central-1-f01|.*ap-southeast-1.*|.*sa-east-1-f01"}
          ) by (pool_ns) == 0
        for: 5m
        labels:
          tier: serverless
          provider_type: aws-free-tier
          severity: major
          response_class: ticket
          component: manager
          oncall_team: serverless
      - alert: ManagerNoFreeTidbPod
        annotations:
          message: |
            Tidb pool {{ $labels.pool_ns }} has no free tidb pods for 8min.
        expr: |
          sum(
            tidbmanager_cache_size{type="free",pool_ns=~".*vip.*",pool_ns!~".*export.*|.*canary.*"}
            unless
            tidbmanager_cache_size{pool_ns="super-vip-tidb-pool", cluster=~"freetier/eks/prod-(us-(east-1-f0[2-9]|west-2-f03)|sa-east-1-f01)"}
            unless
            tidbmanager_cache_size{pool_ns="vip-tidb-pool", cluster!~".*us-(east-1-f0[1-9])|.*us-west-2-f01*|.*ap-northeast-1-f01|.*eu-central-1-f01|.*ap-southeast-1.*|.*sa-east-1-f01"}
          ) by (pool_ns) == 0
        for: 8m
        labels:
          tier: serverless
          provider_type: aws-free-tier
          severity: critical
          response_class: page
          component: manager
          oncall_team: serverless
      - alert: ManagerClusterCreationTakesTooLong
        annotations:
          message: |
            Cluster {{ $labels.cluster_id }} creation takes {{ $value }} seconds longer than 300, region {{ $labels.region }}, cluster_type {{ $labels.cluster_type}}, is_restore {{ $labels.is_restore }}.
        expr: |
          (sum(cluster_creation_seconds_sum{status="Running"}) by (cluster_id,region,cluster_type,is_restore) / sum(cluster_creation_seconds_count{status="Running"}) by (cluster_id,region,cluster_type,is_restore)) > 300
        labels:
          tier: serverless
          provider_type: aws-free-tier
          severity: warning
          component: manager
          oncall_team: serverless
      - alert: ManagerRemovePodQueueTooLong
        annotations:
          message: |
            The tidb pod remove queue too long, current length {{ $value }}, region {{ $labels.region }}.
        expr: |
          count(tidbmanager_remove_pod_queue_len > 0)
        for: 10m
        labels:
          tier: serverless
          provider_type: aws-free-tier
          severity: warning
          component: manager
          oncall_team: serverless
      - alert: ManagerAvailableKeyspaceNotEnough
        annotations:
          message: |
            The available keyspace not enough in keyspace-pool, current value {{ $value }}, region {{ $labels.region }}.
        expr: |
          keyspace_available{pool_ns="keyspace-pool"} < 8
        for: 10m
        labels:
          tier: serverless
          provider_type: aws-free-tier
          severity: warning
          component: manager
          oncall_team: serverless
      - alert: ManagerEncryptAvailableKeyspaceNotEnough
        annotations:
          message: |
            The available keyspace not enough in encrypt-keyspace-pool, current value {{ $value }}, region {{ $labels.region }}.
        expr: |
          keyspace_available{pool_ns="encrypt-keyspace-pool"} < 2
        for: 10m
        labels:
          tier: serverless
          provider_type: aws-free-tier
          severity: warning
          component: manager
          oncall_team: serverless
      - alert: ManagerKeyspaceAvailableLinearPredictionExhausted
        annotations:
          message: |
            The keyspace_available 1h linear trend predicts depletion within the next hour (predict_linear < 0) for prod us freetier EKS, pool {{ $labels.pool_ns }}, value {{ $value }}.
        expr: |
          predict_linear((
            sum(keyspace_available{cluster=~"freetier/eks/prod-us-(east-1-f0[2-9]|west-2-f03)"})
            by (pool_ns)
          )[1h:1m], 3600) < 0
        for: 5m
        labels:
          tier: serverless
          provider_type: aws-free-tier
          severity: major
          response_class: ticket
          component: manager
          oncall_team: serverless
      - alert: ManagerKeyspaceAvailableDrop50PctVs1h
        annotations:
          message: |
            The keyspace_available is below 50% of the value 1h ago (ratio) for prod us freetier EKS, pool {{ $labels.pool_ns }}, value {{ printf "%.2f" $value }}.
        expr: |
          (
            sum(keyspace_available{cluster=~"freetier/eks/prod-us-(east-1-f0[2-9]|west-2-f03)"}) by (pool_ns)
            /
            sum(keyspace_available{cluster=~"freetier/eks/prod-us-(east-1-f0[2-9]|west-2-f03)"} offset 1h) by (pool_ns)
          ) < 0.5
        for: 5m
        labels:
          tier: serverless
          provider_type: aws-free-tier
          severity: major
          response_class: ticket
          component: manager
          oncall_team: serverless
      - alert: UnmanagedTiDBPodsTooMany
        annotations:
          message: |
            The ns {{ $labels.pool_ns }} has {{ $value }} unmanaged pods for 1h. [Clinic Dashboard](https://clinic.pingcap.com/grafana/d/KmTtYdBNk/serverless-resource-usage?orgId=1&var-control_plane_info=ap-northeast-1-a01%2Falicloud-ap-northeast-1%2Fdevtier-infra)
        expr: |
          sum(label_replace(sum(kube_pod_status_phase{phase="Running",namespace=~".*tidb-pool.*"}) by (namespace), "pool_ns", "$1", "namespace", "(.*)")) by (pool_ns)
          -
          sum(max(tidbmanager_cache_size{type!="assign_token"}) by (pool_ns, type)) by (pool_ns) > 5
        for: 1h
        labels:
          tier: serverless
          provider_type: aws-free-tier
          severity: warning
          component: manager
          oncall_team: serverless
      - alert: TiDBPodReplicasTooMany
        annotations:
          message: |
            The cluster {{ $labels.cluster_id }}/{{ $labels.token }}/{{ $labels.keyspace_id }} has {{ $value }} tidb pod replicas for 24h. [Clinic Dashboard](https://clinic.pingcap.com/grafana/d/KmTtYdBNk/serverless-resource-usage?orgId=1&var-control_plane_info=ap-northeast-1-a01%2Falicloud-ap-northeast-1%2Fdevtier-infra)
        expr: |
          count(sum(rate(tidb_executor_statement_total[1m])) by (instance,keyspace_id)) by (keyspace_id) > 5
          or
          max(gateway_tidb_replicas) by (cluster_id, token) > 5
        for: 24h
        labels:
          tier: serverless
          provider_type: aws-free-tier
          severity: warning
          component: manager
          oncall_team: serverless
      - alert: TiDBPodPendingLong
        annotations:
          message: Pending > 15m ns={{ $labels.namespace }} pod={{ $labels.pod }}
        expr: |
          max by (namespace, pod) (kube_pod_status_phase{namespace=~".*tidb-pool",phase="Pending"}) > 0
        for: 15m
        labels:
          severity: critical
          response_class: page
          provider_type: aws-free-tier
          tier: serverless
          component: manager
          oncall_team: serverless
      - alert: UpgraderErrorDurationTooLong
        annotations:
          message: Upgrader error duration > 5m, ns={{ $labels.pool_ns }} count={{ $value }}
        expr: |
          sum(tidbupgrader_upgrader_error_duration) by (pool_ns) > 300
        labels:
          severity: warning
          provider_type: aws-free-tier
          tier: serverless
          component: manager
          oncall_team: serverless
      - alert: UpgraderRollingTooSlow
        annotations:
          message: Upgrader old revision pod hasn't be rolled more than 1 hour, ns={{ $labels.pool_ns }}
        expr: |
          sum(tidbupgrader_old_pod_assign_cache_size+tidbupgrader_old_pod_free_cache_size) by (pool_ns) > 0
        for: 1h
        labels:
          severity: warning
          provider_type: aws-free-tier
          tier: serverless
          component: manager
          oncall_team: serverless
      - alert: O11yPromQueryError
        annotations:
          message: cluster {{ $labels.cluster }}, manager PromQuery error {{ $labels.reason }}.
        expr: |
          increase(utilsquery_prom_query_seconds_count{reason!="ok"}[1m]) > 0
        for: 10m
        labels:
          tier: serverless
          provider_type: aws-free-tier
          severity: major
          response_class: ticket
          component: manager
          oncall_team: serverless