groups:
  - name: regional-infra-provider.rules
    rules:
      - alert: InfraProviderGuardCheckFail
        expr: infra_cluster_guard_check_fail{} == 1
        for: 1m
        labels:
          component: infra-provider
          oncall_team: dedicated_service
          service: infra-provider
          severity: critical
          response_class: page
          type: infra
        annotations:
          description: The Guard {{ $labels.guard }} of Cluster {{ $labels.cluster_id }} check fails.
          diagnosis: https://pingcap.feishu.cn/docx/YwoZdIbndoYptTxFHMoczK8EnPh#doxcnSFuYUZAIq15nSMIB0MAREf

      # - alert: UnexpectedConfigChange
      #   expr: infra_cluster_unexpected_config_change{} == 1
      #   for: 1m
      #   labels:
      #     component: infra-provider
      #     service: infra-provider
      #     severity: major
      #     type: infra
      #   annotations:
      #     description: There is an unexpected config change happen on {{ $labels.cluster_id }} / {{ $labels.component }}, cluster phase is {{ $labels.phase }}.
      #     diagnosis: See code https://github.com/tidbcloud/infra-provider/blob/opt/dim-no-unexpected-configchange/pkg/gitops-sdk/manager.go#L309
      - alert: ConfigChangeFailed
        expr: infra_exporter_configchange{phase="Failed"} == 1
        for: 1m
        labels:
          component: infra-provider
          oncall_team: dedicated_service
          service: infra-provider
          severity: critical
          response_class: page
          type: infra
        annotations:
          description: The ConfigChange {{ $labels.exported_namespace }}/{{ $labels.name }} failed.
          diagnosis: https://pingcap.feishu.cn/wiki/wikcnRe6t2VlsB14p9drxXexdRQ#doxcnqhILMwGQ1WaXANJA0d6Kme
      - alert: ConfigChangeDurationTooLong
        expr: |
          label_replace(infra_exporter_cluster{status!~"Suspend|Deleting|Creating"}, "cluster_name", "$1", "name", "(.*)")
          * on (cluster_name) group_left() infra_exporter_configchange{phase!~"Completed|Failed"} == 1
        for: 2d
        labels:
          component: infra-provider
          oncall_team: dedicated_service
          service: infra-provider
          severity: critical
          response_class: page
          type: infra
        annotations:
          description: The ConfigChange {{ $labels.exported_namespace }}/{{ $labels.name }} runs too long than 2 days.
          diagnosis: https://pingcap.feishu.cn/wiki/wikcnRe6t2VlsB14p9drxXexdRQ#doxcn8xnrJwhbjniWVDeuXOZFjh
      - alert: ConfigChangeDurationLong
        expr: |
          label_replace(infra_exporter_cluster{status!~"Suspend|Deleting|Creating"}, "cluster_name", "$1", "name", "(.*)")
          * on (cluster_name) group_left() infra_exporter_configchange{phase!~"Completed|Failed"} == 1
        for: 2h
        labels:
          component: infra-provider
          oncall_team: dedicated_service
          service: infra-provider
          severity: major
          response_class: ticket
          type: infra
        annotations:
          description: The ConfigChange {{ $labels.exported_namespace }}/{{ $labels.name }} runs too long than 1 hours.
          diagnosis: https://pingcap.feishu.cn/wiki/wikcnRe6t2VlsB14p9drxXexdRQ#doxcn8xnrJwhbjniWVDeuXOZFjh
      - alert: ClusterInfraProviderPanicManyTimesInOneHour
        expr: increase(infra_provider_controller_panic{controller=~"cluster"}[1h])> 5
        for: 1m
        labels:
          component: infra-provider
          oncall_team: dedicated_service
          service: infra-provider
          severity: critical
          response_class: page
          type: infra
        annotations:
          description: The Controller {{ $labels.controller }} raises {{ $value }} panics in 1 hour(s).
      - alert: OtherInfraProviderPanicManyTimesInOneHour
        expr: increase(infra_provider_controller_panic{controller!~"cluster|br|cdc|dumpling|dwworker|import"}[1h])> 5
        for: 1m
        labels:
          component: infra-provider
          oncall_team: dedicated_service
          service: infra-provider
          severity: critical
          response_class: page
          type: infra
        annotations:
          description: The Controller {{ $labels.controller }} raises {{ $value }} panics in 1 hour(s).
      - alert: ClusterInfraProviderPanicFound
        expr: increase(infra_provider_controller_panic{controller=~"cluster"}[10m])>= 1
        for: 1m
        labels:
          component: infra-provider
          oncall_team: dedicated_service
          service: infra-provider
          severity: major
          response_class: ticket
          type: infra
        annotations:
          description: The Controller {{ $labels.controller }} raises {{ $value }} panics in 10 minute(s).

      - alert: ClusterServiceKubePodCrashLooping
        annotations:
          message: Pod {{ $labels.namespace }}/{{ $labels.pod }} in the {{ $labels.cluster }} is restarting {{ printf "%.2f" $value }} times in 1 hours.
          runbook_url: https://github.com/tidbcloud/runbooks/wiki/KubePodCrashLooping
        expr: |
          increase(kube_pod_container_status_restarts_total{namespace=~"infra|cluster-service"}[1h]) > 5
        for: 5m
        labels:
          service: infra-provider
          component: infra-provider
          oncall_team: dedicated_service
          type: infra
          severity: critical
          response_class: page