groups:
- name: kube-dns
  rules:
  - alert: KubeDnsDown
    annotations:
      description: kube dns is down
    expr: up{job="kube-dns"} == 0
    for: 5m
    labels:
      component: kubernetes
      oncall_team: dedicated_service
      service: kube-dns
      severity: critical
      response_class: page
  - alert: CoreDNSLatencyHigh
    annotations:
      message: CoreDNS has 99th percentile latency of {{ $value }} seconds for server
        {{ $labels.server }} zone {{ $labels.zone }} .
      runbook_url: https://github.com/povilasv/coredns-mixin/tree/master/runbook.md#alert-name-corednslatencyhigh
    expr: |
      histogram_quantile(0.99, sum(rate(coredns_dns_request_duration_seconds_bucket{job="kube-dns"}[5m])) by(server, zone, le)) > 4
    for: 10m
    labels:
      component: kubernetes
      oncall_team: dedicated_service
      severity: critical
      response_class: page
  - alert: CoreDNSErrorsHigh
    annotations:
      message: CoreDNS is returning SERVFAIL for {{ $value | humanizePercentage }}
        of requests.
      runbook_url: https://github.com/povilasv/coredns-mixin/tree/master/runbook.md#alert-name-corednserrorshigh
    expr: |
      sum(rate(coredns_dns_responses_total{job="kube-dns",rcode="SERVFAIL"}[5m]))
        /
      sum(rate(coredns_dns_responses_total{job="kube-dns"}[5m])) > 0.03
    for: 10m
    labels:
      component: kubernetes
      oncall_team: dedicated_service
      severity: critical
      response_class: page
  - alert: CoreDNSErrorsHigh
    annotations:
      message: CoreDNS is returning SERVFAIL for {{ $value | humanizePercentage }}
        of requests.
      runbook_url: https://github.com/povilasv/coredns-mixin/tree/master/runbook.md#alert-name-corednserrorshigh
    expr: |
      sum(rate(coredns_dns_responses_total{job="kube-dns",rcode="SERVFAIL"}[5m]))
        /
      sum(rate(coredns_dns_responses_total{job="kube-dns"}[5m])) > 0.01
    for: 10m
    labels:
      component: kubernetes
      oncall_team: dedicated_service
      severity: warning
- name: coredns_forward
  rules:
  - alert: CoreDNSForwardLatencyHigh
    annotations:
      message: CoreDNS has 99th percentile latency of {{ $value }} seconds forwarding
        requests to {{ $labels.to }}.
      runbook_url: https://github.com/povilasv/coredns-mixin/tree/master/runbook.md#alert-name-corednsforwardlatencyhigh
    expr: |
      histogram_quantile(0.99, sum(rate(coredns_forward_request_duration_seconds_bucket{job="kube-dns"}[5m])) by(to, le)) > 4
    for: 10m
    labels:
      component: kubernetes
      oncall_team: dedicated_service
      severity: critical
      response_class: page
  - alert: CoreDNSForwardErrorsHigh
    annotations:
      message: CoreDNS is returning SERVFAIL for {{ $value | humanizePercentage }}
        of forward requests to {{ $labels.to }}.
      runbook_url: https://github.com/povilasv/coredns-mixin/tree/master/runbook.md#alert-name-corednsforwarderrorshigh
    expr: |
      sum(rate(coredns_forward_responses_total{job="kube-dns",rcode="SERVFAIL"}[5m]))
        /
      sum(rate(coredns_forward_responses_total{job="kube-dns"}[5m])) > 0.03
    for: 10m
    labels:
      component: kubernetes
      oncall_team: dedicated_service
      severity: critical
      response_class: page
  - alert: CoreDNSForwardErrorsHigh
    annotations:
      message: CoreDNS is returning SERVFAIL for {{ $value | humanizePercentage }}
        of forward requests to {{ $labels.to }}.
      runbook_url: https://github.com/povilasv/coredns-mixin/tree/master/runbook.md#alert-name-corednsforwarderrorshigh
    expr: |
      sum(rate(coredns_forward_responses_total{job="kube-dns",rcode="SERVFAIL"}[5m]))
        /
      sum(rate(coredns_forward_responses_total{job="kube-dns"}[5m])) > 0.01
    for: 10m
    labels:
      component: kubernetes
      oncall_team: dedicated_service
      severity: warning
  - alert: CoreDNSForwardHealthcheckFailureCount
    annotations:
      message: CoreDNS health checks have failed to upstream server {{ $labels.to
        }}.
      runbook_url: https://github.com/povilasv/coredns-mixin/tree/master/runbook.md#alert-name-corednsforwardhealthcheckfailurecount
    expr: |
      sum(rate(coredns_forward_healthcheck_failures_total{job="kube-dns"}[5m])) by (to) > 0
    for: 10m
    labels:
      component: kubernetes
      oncall_team: dedicated_service
      severity: warning
  - alert: CoreDNSForwardHealthcheckBrokenCount
    annotations:
      message: CoreDNS health checks have failed for all upstream servers.
      runbook_url: https://github.com/povilasv/coredns-mixin/tree/master/runbook.md#alert-name-corednsforwardhealthcheckbrokencount
    expr: |
      sum(rate(coredns_forward_healthcheck_broken_total{job="kube-dns"}[5m])) > 0
    for: 10m
    labels:
      component: kubernetes
      oncall_team: dedicated_service
      severity: warning