groups:
- name: kubernetes-apps
  rules:
  - alert: KubePodCrashLooping
    annotations:
      message: Pod {{ $labels.namespace }}/{{ $labels.pod }} ({{ $labels.container
        }}) is restarting {{ printf "%.2f" $value }} times in 1 hours.
      runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubepodcrashlooping
    expr: |
      increase(kube_pod_container_status_restarts_total{namespace=~"(.*)-ms|kong|jaeger"}[1h]) > 5
    for: 10m
    labels:
      component: api-gateway
      oncall_team: global_control
      severity: warning
  - alert: KubePodNotReady
    annotations:
      message: Pod {{ $labels.namespace }}/{{ $labels.pod }} has been in a non-ready
        state for longer than 15 minutes.
      runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubepodnotready
    expr: |
      sum by (namespace, pod) (max by(namespace, pod) (kube_pod_status_phase{job="kube-state-metrics", phase=~"Pending|Unknown", namespace=~"(.*)-ms|kong|jaeger"}) * on(namespace, pod) group_left(owner_kind) max by(namespace, pod, owner_kind) (kube_pod_owner{owner_kind!="Job"})) > 0
    for: 15m
    labels:
      component: api-gateway
      oncall_team: global_control
      severity: warning
  - alert: PodCpuUsageTooHigh
    expr: |
      sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{namespace=~"nightly-ms|staging-ms|prod-ms|kong"}) by (pod)/sum(kube_pod_container_resource_limits{namespace=~"nightly-ms|staging-ms|prod-ms|kong",resource="cpu"}) by (pod) > 0.5
    for: 5m
    annotations:
      message: Pod {{ $labels.namespace }}/{{ $labels.pod }} cpu usage has been greater than 50% for longer than 5 minutes.
    labels:
      component: api-gateway
      oncall_team: global_control
      severity: warning
  - alert: PodMemUsageTooHigh
    expr: |
      sum(container_memory_working_set_bytes{ namespace=~"nightly-ms|staging-ms|prod-ms|kong",container!="POD", container!="", image!=""})by(pod)/sum(kube_pod_container_resource_limits{namespace=~"nightly-ms|staging-ms|prod-ms|kong",resource="memory"}) by (pod) > 0.7
    for: 5m
    annotations:
      message: Pod {{ $labels.namespace }}/{{ $labels.pod }} memory usage has been greater than 70% for longer than 5 minutes.
    labels:
      component: api-gateway
      oncall_team: global_control
      severity: warning