groups: 
  - name: clinic.pod
    rules:
      - alert: ClinicPodRestartsFrequently
        expr: increase(kube_pod_container_status_restarts_total{namespace=~"clinic.*|kong.*"}[5m]) > 0
        for: 10m
        annotations:
          message: |-
            Pod {{ $labels.namespace }}/{{ $labels.pod }} has been consistently restarting for the last 10 minutes.
            (Value: {{ printf "%.0f" $value }} restarts in the last 5m window)
            Check the grafana dashboard for more details:
            https://clinic.pingcap.com/grafana/d/M7l00woSk/kubernetes-compute-resources-pod?orgId=1&var-pod={{ $labels.pod }}&var-namespace={{ $labels.namespace }}
        labels:
          severity: critical
          response_class: page
          component: clinic
          oncall_team: clinic
      - alert: KubePodRestartsFrequently
        expr: increase(kube_pod_container_status_restarts_total{namespace!~"clinic.*|kong.*"}[1h]) > 3
        for: 1m
        annotations:
          message: |-
            Pod {{ $labels.namespace }}/{{ $labels.pod }} is restarting {{ printf "%.2f" $value }} times in 1 hours.
            Check the grafana dashboard for more details:
            https://clinic.pingcap.com/grafana/d/M7l00woSk/kubernetes-compute-resources-pod?orgId=1&var-pod={{ $labels.pod }}&var-namespace={{ $labels.namespace }}
        labels:
          severity: major
          response_class: ticket
          component: clinic
          oncall_team: clinic
      - alert: ClinicPodCPUUsageTooHigh(90%)
        annotations:
          message: |
            CPU utilization of pod {{ $labels.pod }} is more than 90%.
        expr: |
          (
            sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{namespace=~"clinic.*|kong.*", pod!~"importer.*"}) by (namespace,pod)
            /
            sum(kube_pod_container_resource_limits{resource="cpu",namespace=~"clinic.*|kong.*", pod!~"importer.*"}) by (namespace,pod)
          ) > 0.9
        for: 10m
        labels:
          severity: major
          response_class: ticket
          component: clinic
          oncall_team: clinic
      # - alert: ClinicPodCPUUsageTooHigh(80%)
      #   annotations:
      #     message: |
      #       CPU utilization of pod {{ $labels.pod }} is more than 80%.
      #       Check the grafana dashboard for more details:
      #       https://clinic.pingcap.com/grafana/d/M7l00woSk/kubernetes-compute-resources-pod?orgId=1&var-pod={{ $labels.pod }}&var-namespace={{ $labels.namespace }}
      #   expr: |
      #     (
      #       sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{namespace=~"clinic.*|kong.*"}) by (namespace,pod)
      #       /
      #       sum(kube_pod_container_resource_limits{resource="cpu",namespace=~"clinic.*|kong.*"}) by (namespace,pod)
      #     ) > 0.8
      #   for: 3m
      #   labels:
      #     severity: major
      #     component: clinic
      - alert: ClinicPodMemoryUsageTooHigh(90%)
        annotations:
          message: |
            Memory utilization of pod {{ $labels.pod }} is more than 90%.
            Check the grafana dashboard for more details:
            https://clinic.pingcap.com/grafana/d/M7l00woSk/kubernetes-compute-resources-pod?orgId=1&var-pod={{ $labels.pod }}&var-namespace={{ $labels.namespace }}
        expr: |
          (
            sum(container_memory_working_set_bytes{namespace=~"clinic.*|kong.*",container!="", image!=""}) by (pod)
            /
            sum(kube_pod_container_resource_limits{resource="memory",namespace=~"clinic.*|kong.*"}) by (pod)
          ) > 0.9
        for: 3m
        labels:
          severity: major
          response_class: ticket
          component: clinic
          oncall_team: clinic
      # - alert: ClinicPodMemoryUsageTooHigh(80%)
      #   annotations:
      #     message: |
      #       Memory utilization of pod {{ $labels.pod }} is more than 80%.
      #       Check the grafana dashboard for more details:
      #       https://clinic.pingcap.com/grafana/d/M7l00woSk/kubernetes-compute-resources-pod?orgId=1&var-pod={{ $labels.pod }}&var-namespace={{ $labels.namespace }}
      #   expr: |
      #     (
      #       sum(container_memory_working_set_bytes{namespace=~"clinic.*|kong.*",container!="", image!=""}) by (pod)
      #       /
      #       sum(kube_pod_container_resource_limits{resource="memory",namespace=~"clinic.*|kong.*"}) by (pod)
      #     ) > 0.8
      #   for: 3m
      #   labels:
      #     severity: major
      #     component: clinic
      - alert: ClinicVMStoragePersistentVolumeFillingUp
        annotations:
          message: |
            The PersistentVolume claimed by {{ $labels.persistentvolumeclaim
            }} in Namespace {{ $labels.namespace }} is only {{ $value | humanizePercentage
            }} free.
            Check the grafana dashboard for more details:
            https://clinic.pingcap.com/grafana/d/peR80gTGk/kubernetes-persistent-volumes?from=now-3h&to=now
          runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubepersistentvolumefillingup
        expr: |
          kubelet_volume_stats_available_bytes{job="kubelet",persistentvolumeclaim=~"vmstorage-volume-victoria-metrics-cluster-vmstorage.*"}
            /
          kubelet_volume_stats_capacity_bytes{job="kubelet",persistentvolumeclaim=~"vmstorage-volume-victoria-metrics-cluster-vmstorage.*"}
            < 0.10
        for: 10m
        labels:
          component: clinic
          oncall_team: clinic
          severity: critical
          response_class: page