groups:
  - name: core-svc-basic
    rules:
      - alert: GoroutineContinuouslyIncrease
        annotations:
          message: Goroutine of pod {{ $labels.pod }} is continuous increasing
          diagnosis: 'click here to see detail: https://clinic.pingcap.com/grafana/explore?orgId=1&left={"datasource":"Ak6VpZD4z","queries":[{"refId":"A","editorMode":"code","expr":"go_goroutines{pod=\"{{ $labels.pod }}\"}","legendFormat":"__auto","range":true,"instant":true}],"range":{"from":"now-1h","to":"now"}}'
        expr: |
          avg(delta(go_goroutines{service=~"(dbaas-core-portal-server-metrics)"}[5m])) by (pod)
          > 10
        for: 10m
        labels:
          severity: major
          response_class: ticket
          component: core-svc
          oncall_team: dedicated_service

      - alert: GoroutineContinuouslyIncrease
        annotations:
          message: Goroutine of pod {{ $labels.pod }} is continuous increasing
          diagnosis: 'click here to see detail: https://clinic.pingcap.com/grafana/explore?orgId=1&left={"datasource":"Ak6VpZD4z","queries":[{"refId":"A","editorMode":"code","expr":"go_goroutines{pod=\"{{ $labels.pod }}\"}","legendFormat":"__auto","range":true,"instant":true}],"range":{"from":"now-1h","to":"now"}}'
        expr: |
          avg(delta(go_goroutines{service=~"(dbaas-core-portal-server-metrics)"}[5m])) by (pod)
          > 100
        for: 5m
        labels:
          severity: critical
          response_class: page
          component: core-svc
          oncall_team: dedicated_service

      - alert: GoroutineCountIsTooHigh
        annotations:
          message: Goroutine of pod {{ $labels.pod }} is more than 1000
          diagnosis: 'click here to see detail: https://clinic.pingcap.com/grafana/explore?orgId=1&left={"datasource":"Ak6VpZD4z","queries":[{"refId":"A","editorMode":"code","expr":"go_goroutines{pod=\"{{ $labels.pod }}\"}","legendFormat":"__auto","range":true,"instant":true}],"range":{"from":"now-1h","to":"now"}}'
        expr: |
          go_goroutines{service=~"(dbaas-core-portal-server-metrics)"}
          > 1000
        for: 5m
        labels:
          severity: critical
          response_class: page
          component: core-svc
          oncall_team: dedicated_service

      - alert: CPUUsageIsTooHigh
        annotations:
          message: CPU utilization of pod {{ $labels.pod }} is more than 40% for 5m.
          diagnosis: 'click here to see detail: https://clinic.pingcap.com/grafana/explore?orgId=1&left={"datasource":"Ak6VpZD4z","queries":[{"refId":"A","editorMode":"code","expr":"(sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{ pod=\"{{ $labels.pod }}\"}) by (pod)/sum(kube_pod_container_resource_limits{resource=\"cpu\", pod=\"{{ $labels.pod }}\"}) by (pod)) *100","legendFormat":"__auto","range":true,"instant":true,"hide":false}],"range":{"from":"now-1h","to":"now"}}'
        expr: |
          (sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{pod=~"(dbaas-core-portal-server)(.*)"}) by (namespace,pod)/
            sum(kube_pod_container_resource_limits{resource="cpu",pod=~"(dbaas-core-portal-server)(.*)"}) by (namespace,pod)
          ) *100 > 40
        for: 5m
        labels:
          severity: major
          response_class: ticket
          component: core-svc
          oncall_team: dedicated_service

      - alert: CPUUsageIsTooHigh
        annotations:
          message: CPU utilization of pod {{ $labels.pod }} is more than 50% for 10m.
          diagnosis: 'click here to see detail: https://clinic.pingcap.com/grafana/explore?orgId=1&left={"datasource":"Ak6VpZD4z","queries":[{"refId":"A","editorMode":"code","expr":"(sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{ pod=\"{{ $labels.pod }}\"}) by (pod)/sum(kube_pod_container_resource_limits{resource=\"cpu\", pod=\"{{ $labels.pod }}\"}) by (pod)) *100","legendFormat":"__auto","range":true,"instant":true,"hide":false}],"range":{"from":"now-1h","to":"now"}}'
        expr: |
          (sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{pod=~"(dbaas-core-portal-server)(.*)"}) by (namespace,pod)/
            sum(kube_pod_container_resource_limits{resource="cpu",pod=~"(dbaas-core-portal-server)(.*)"}) by (namespace,pod)
          ) *100 > 50
        for: 10m
        labels:
          severity: critical
          response_class: page
          component: core-svc
          oncall_team: dedicated_service

      - alert: MemUsageIsTooHigh
        annotations:
          message: Memory utilization of pod {{ $labels.pod }} is more than 60% for 5m.
          diagnosis: 'click here to see detail: https://clinic.pingcap.com/grafana/explore?orgId=1&left={"datasource":"Ak6VpZD4z","queries":[{"refId":"A","editorMode":"code","expr":"(sum(container_memory_working_set_bytes{container!=\"POD\", container!=\"\", image!=\"\", pod=\"{{ $labels.pod }}\"})by(pod)/sum(kube_pod_container_resource_limits{resource=\"memory\", pod=\"{{ $labels.pod }}\"}) by (pod) * 100)","legendFormat":"__auto","range":true,"instant":true,"hide":false}],"range":{"from":"now-1h","to":"now"}}'
        expr: |
          (sum(container_memory_working_set_bytes{pod=~"(dbaas-core-portal-server)(.*)",container!="POD", container!="", image!=""})by(pod)/
          sum(kube_pod_container_resource_limits{resource="memory",pod=~"(dbaas-core-portal-server)(.*)"}) by (pod)
          * 100) > 60
        for: 5m
        labels:
          severity: major
          response_class: ticket
          component: core-svc
          oncall_team: dedicated_service

      - alert: PodRestartsFrequently
        annotations:
          summary: The pod {{ $labels.pod }} restart frequently
          message: Container {{ $labels.namespace }}/{{ $labels.container }} is restarting {{ printf "%.2f" $value }} times in 5 minutes.
        expr: |
          increase(kube_pod_container_status_restarts_total{pod=~"dbaas-core-(.*)"}[5m]) > 2
        for: 5m
        labels:
          severity: critical
          response_class: page
          component: core-svc
          oncall_team: dedicated_service

      - alert: UnhealthyPod
        annotations:
          summary: The pod {{ $labels.pod }} is not ready
          message: Pod {{ $labels.namespace }}/{{ $labels.pod }} in {{ $labels.cluster }} has been in a non-ready state for longer than 2 minutes.
        expr: |
          kube_pod_status_phase{phase=~"Pending|Unknown|Failed",namespace=~"nightly-ms|staging-ms|prod-ms",pod=~"dbaas-core-.*"} == 1
        for: 2m
        labels:
          severity: critical
          response_class: page
          component: core-svc
          oncall_team: dedicated_service