groups:
- name: kube-apiserver-burnrate.rules
  rules:
  - expr: |
      (
        (
          # too slow
          sum by (cluster) (rate(apiserver_request_duration_seconds_count{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward"}[1d]))
          -
          (
            (
              sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope=~"resource|",le="1"}[1d]))
              or
              vector(0)
            )
            +
            sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="namespace",le="5"}[1d]))
            +
            sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="cluster",le="30"}[1d]))
          )
        )
        +
        # errors
        sum by (cluster) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET",code=~"5.."}[1d]))
      )
      /
      sum by (cluster) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET"}[1d]))
    labels:
      verb: read
    record: apiserver_request:burnrate1d
  - expr: |
      (
        (
          # too slow
          sum by (cluster) (rate(apiserver_request_duration_seconds_count{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward"}[1h]))
          -
          (
            (
              sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope=~"resource|",le="1"}[1h]))
              or
              vector(0)
            )
            +
            sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="namespace",le="5"}[1h]))
            +
            sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="cluster",le="30"}[1h]))
          )
        )
        +
        # errors
        sum by (cluster) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET",code=~"5.."}[1h]))
      )
      /
      sum by (cluster) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET"}[1h]))
    labels:
      verb: read
    record: apiserver_request:burnrate1h
  - expr: |
      (
        (
          # too slow
          sum by (cluster) (rate(apiserver_request_duration_seconds_count{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward"}[2h]))
          -
          (
            (
              sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope=~"resource|",le="1"}[2h]))
              or
              vector(0)
            )
            +
            sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="namespace",le="5"}[2h]))
            +
            sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="cluster",le="30"}[2h]))
          )
        )
        +
        # errors
        sum by (cluster) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET",code=~"5.."}[2h]))
      )
      /
      sum by (cluster) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET"}[2h]))
    labels:
      verb: read
    record: apiserver_request:burnrate2h
  - expr: |
      (
        (
          # too slow
          sum by (cluster) (rate(apiserver_request_duration_seconds_count{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward"}[30m]))
          -
          (
            (
              sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope=~"resource|",le="1"}[30m]))
              or
              vector(0)
            )
            +
            sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="namespace",le="5"}[30m]))
            +
            sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="cluster",le="30"}[30m]))
          )
        )
        +
        # errors
        sum by (cluster) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET",code=~"5.."}[30m]))
      )
      /
      sum by (cluster) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET"}[30m]))
    labels:
      verb: read
    record: apiserver_request:burnrate30m
  - expr: |
      (
        (
          # too slow
          sum by (cluster) (rate(apiserver_request_duration_seconds_count{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward"}[3d]))
          -
          (
            (
              sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope=~"resource|",le="1"}[3d]))
              or
              vector(0)
            )
            +
            sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="namespace",le="5"}[3d]))
            +
            sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="cluster",le="30"}[3d]))
          )
        )
        +
        # errors
        sum by (cluster) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET",code=~"5.."}[3d]))
      )
      /
      sum by (cluster) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET"}[3d]))
    labels:
      verb: read
    record: apiserver_request:burnrate3d
  - expr: |
      (
        (
          # too slow
          sum by (cluster) (rate(apiserver_request_duration_seconds_count{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward"}[5m]))
          -
          (
            (
              sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope=~"resource|",le="1"}[5m]))
              or
              vector(0)
            )
            +
            sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="namespace",le="5"}[5m]))
            +
            sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="cluster",le="30"}[5m]))
          )
        )
        +
        # errors
        sum by (cluster) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET",code=~"5.."}[5m]))
      )
      /
      sum by (cluster) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET"}[5m]))
    labels:
      verb: read
    record: apiserver_request:burnrate5m
  - expr: |
      (
        (
          # too slow
          sum by (cluster) (rate(apiserver_request_duration_seconds_count{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward"}[6h]))
          -
          (
            (
              sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope=~"resource|",le="1"}[6h]))
              or
              vector(0)
            )
            +
            sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="namespace",le="5"}[6h]))
            +
            sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="cluster",le="30"}[6h]))
          )
        )
        +
        # errors
        sum by (cluster) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET",code=~"5.."}[6h]))
      )
      /
      sum by (cluster) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET"}[6h]))
    labels:
      verb: read
    record: apiserver_request:burnrate6h
  - expr: |
      (
        (
          # too slow
          sum by (cluster) (rate(apiserver_request_duration_seconds_count{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward"}[1d]))
          -
          sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward",le="1"}[1d]))
        )
        +
        sum by (cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",code=~"5.."}[1d]))
      )
      /
      sum by (cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[1d]))
    labels:
      verb: write
    record: apiserver_request:burnrate1d
  - expr: |
      (
        (
          # too slow
          sum by (cluster) (rate(apiserver_request_duration_seconds_count{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward"}[1h]))
          -
          sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward",le="1"}[1h]))
        )
        +
        sum by (cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",code=~"5.."}[1h]))
      )
      /
      sum by (cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[1h]))
    labels:
      verb: write
    record: apiserver_request:burnrate1h
  - expr: |
      (
        (
          # too slow
          sum by (cluster) (rate(apiserver_request_duration_seconds_count{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward"}[2h]))
          -
          sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward",le="1"}[2h]))
        )
        +
        sum by (cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",code=~"5.."}[2h]))
      )
      /
      sum by (cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[2h]))
    labels:
      verb: write
    record: apiserver_request:burnrate2h
  - expr: |
      (
        (
          # too slow
          sum by (cluster) (rate(apiserver_request_duration_seconds_count{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward"}[30m]))
          -
          sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward",le="1"}[30m]))
        )
        +
        sum by (cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",code=~"5.."}[30m]))
      )
      /
      sum by (cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[30m]))
    labels:
      verb: write
    record: apiserver_request:burnrate30m
  - expr: |
      (
        (
          # too slow
          sum by (cluster) (rate(apiserver_request_duration_seconds_count{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward"}[3d]))
          -
          sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward",le="1"}[3d]))
        )
        +
        sum by (cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",code=~"5.."}[3d]))
      )
      /
      sum by (cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[3d]))
    labels:
      verb: write
    record: apiserver_request:burnrate3d
  - expr: |
      (
        (
          # too slow
          sum by (cluster) (rate(apiserver_request_duration_seconds_count{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward"}[5m]))
          -
          sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward",le="1"}[5m]))
        )
        +
        sum by (cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",code=~"5.."}[5m]))
      )
      /
      sum by (cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[5m]))
    labels:
      verb: write
    record: apiserver_request:burnrate5m
  - expr: |
      (
        (
          # too slow
          sum by (cluster) (rate(apiserver_request_duration_seconds_count{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward"}[6h]))
          -
          sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward",le="1"}[6h]))
        )
        +
        sum by (cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",code=~"5.."}[6h]))
      )
      /
      sum by (cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[6h]))
    labels:
      verb: write
    record: apiserver_request:burnrate6h
  # REF: https://github.com/kubernetes-monitoring/kubernetes-mixin/issues/411
  - expr: |
      sum by (code, verb) (increase(apiserver_request_total{job="apiserver"}[1h]))
    record: code_verb:apiserver_request_total:increase1h
  - expr: |
      sum by (code,resource) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET"}[5m]))
    labels:
      verb: read
    record: code_resource:apiserver_request_total:rate5m
  - expr: |
      sum by (code,resource) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[5m]))
    labels:
      verb: write
    record: code_resource:apiserver_request_total:rate5m
  - expr: |
      histogram_quantile(0.99, sum by (le, resource) (rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET"}[5m]))) > 0
    labels:
      quantile: "0.99"
      verb: read
    record: cluster_quantile:apiserver_request_duration_seconds:histogram_quantile
  - expr: |
      histogram_quantile(0.99, sum by (le, resource) (rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[5m]))) > 0
    labels:
      quantile: "0.99"
      verb: write
    record: cluster_quantile:apiserver_request_duration_seconds:histogram_quantile
  - expr: |
      sum(rate(apiserver_request_duration_seconds_sum{subresource!="log",verb!~"LIST|WATCH|WATCHLIST|DELETECOLLECTION|PROXY|CONNECT"}[5m])) without(instance, pod)
      /
      sum(rate(apiserver_request_duration_seconds_count{subresource!="log",verb!~"LIST|WATCH|WATCHLIST|DELETECOLLECTION|PROXY|CONNECT"}[5m])) without(instance, pod)
    record: cluster:apiserver_request_duration_seconds:mean5m
  - expr: |
      histogram_quantile(0.99, sum(rate(apiserver_request_duration_seconds_bucket{job="apiserver",subresource!="log",verb!~"LIST|WATCH|WATCHLIST|DELETECOLLECTION|PROXY|CONNECT"}[5m])) without(instance, pod))
    labels:
      quantile: "0.99"
    record: cluster_quantile:apiserver_request_duration_seconds:histogram_quantile
  - expr: |
      histogram_quantile(0.9, sum(rate(apiserver_request_duration_seconds_bucket{job="apiserver",subresource!="log",verb!~"LIST|WATCH|WATCHLIST|DELETECOLLECTION|PROXY|CONNECT"}[5m])) without(instance, pod))
    labels:
      quantile: "0.9"
    record: cluster_quantile:apiserver_request_duration_seconds:histogram_quantile
  - expr: |
      histogram_quantile(0.5, sum(rate(apiserver_request_duration_seconds_bucket{job="apiserver",subresource!="log",verb!~"LIST|WATCH|WATCHLIST|DELETECOLLECTION|PROXY|CONNECT"}[5m])) without(instance, pod))
    labels:
      quantile: "0.5"
    record: cluster_quantile:apiserver_request_duration_seconds:histogram_quantile
- name: kube-apiserver-slos
  rules:
  - alert: KubeAPIErrorBudgetBurn
    annotations:
      message: The API server is burning too much error budget
      runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeapierrorbudgetburn
    expr: |
      sum(apiserver_request:burnrate1h) by (cluster) > (14.40 * 0.01000)
      and
      sum(apiserver_request:burnrate5m) by (cluster) > (14.40 * 0.01000)
    for: 2m
    labels:
      component: kubernetes
      oncall_team: dedicated_service
      severity: critical
      response_class: page
      long: 1h
      short: 5m
  - alert: KubeAPIErrorBudgetBurn
    annotations:
      message: The API server is burning too much error budget
      runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeapierrorbudgetburn
    expr: |
      sum(apiserver_request:burnrate6h) by (cluster) > (6.00 * 0.01000)
      and
      sum(apiserver_request:burnrate30m) by (cluster) > (6.00 * 0.01000)
    for: 15m
    labels:
      component: kubernetes
      oncall_team: dedicated_service
      severity: critical
      response_class: page
      long: 6h
      short: 30m
  # - alert: KubeAPIErrorBudgetBurn
  #   annotations:
  #     message: The API server is burning too much error budget
  #     runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeapierrorbudgetburn
  #   expr: |
  #     sum(apiserver_request:burnrate1d) > (3.00 * 0.01000)
  #     and
  #     sum(apiserver_request:burnrate2h) > (3.00 * 0.01000)
  #   for: 1h
  #   labels:
  #     severity: warning
  # - alert: KubeAPIErrorBudgetBurn
  #   annotations:
  #     message: The API server is burning too much error budget
  #     runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeapierrorbudgetburn
  #   expr: |
  #     sum(apiserver_request:burnrate3d) > (1.00 * 0.01000)
  #     and
  #     sum(apiserver_request:burnrate6h) > (1.00 * 0.01000)
  #   for: 3h
  #   labels:
  #     severity: warning
- name: kubernetes-system-apiserver
  rules:
  - alert: KubeAPILatencyHigh
    annotations:
      message: The API server has an abnormal latency of {{ $value }} seconds for
        {{ $labels.verb }} {{ $labels.resource }}.
      runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeapilatencyhigh
    expr: |
      (
        cluster:apiserver_request_duration_seconds:mean5m{job="apiserver"}
        >
        on (verb) group_left()
        (
          avg by (verb) (cluster:apiserver_request_duration_seconds:mean5m{job="apiserver"} >= 0)
          +
          2*stddev by (verb) (cluster:apiserver_request_duration_seconds:mean5m{job="apiserver"} >= 0)
        )
      ) > on (verb) group_left()
      1.2 * avg by (verb) (cluster:apiserver_request_duration_seconds:mean5m{job="apiserver"} >= 0)
      and on (verb,resource)
      cluster_quantile:apiserver_request_duration_seconds:histogram_quantile{job="apiserver",quantile="0.99"}
      >
      1
    for: 15m
    labels:
      component: kubernetes
      oncall_team: dedicated_service
      severity: warning
  - alert: KubeAPIErrorsHigh
    annotations:
      message: API server is returning errors for {{ $value | humanizePercentage
        }} of requests for {{ $labels.verb }} {{ $labels.resource }} {{ $labels.subresource
        }}.
      runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeapierrorshigh
    expr: |
      sum(rate(apiserver_request_total{job="apiserver",code=~"5.."}[5m])) by (resource,subresource,verb)
        /
      sum(rate(apiserver_request_total{job="apiserver"}[5m])) by (resource,subresource,verb) > 0.1
    for: 10m
    labels:
      component: kubernetes
      oncall_team: dedicated_service
      severity: warning
  - alert: KubeClientCertificateExpirationIn3Days
    annotations:
      message: A client certificate used to authenticate to the apiserver is expiring
        in less than 3 days.
      runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeclientcertificateexpiration
    expr: |
      histogram_quantile(0.01, sum by (job, le, name, cluster) (rate(apiserver_client_certificate_expiration_seconds_bucket{job="apiserver"}[5m]))) < 259200
    labels:
      component: kubernetes
      oncall_team: dedicated_service
      severity: critical
      response_class: page
  - alert: KubeClientCertificateExpirationIn24Hours
    annotations:
      message: A client certificate used to authenticate to the apiserver is expiring
        in less than 24.0 hours.
      runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeclientcertificateexpiration
    expr: |
      histogram_quantile(0.01, sum by (job, le, name, cluster) (rate(apiserver_client_certificate_expiration_seconds_bucket{job="apiserver"}[15m]))) < 86400
    labels:
      component: kubernetes
      oncall_team: dedicated_service
      severity: critical
      response_class: page
  - alert: AggregatedAPIErrors
    annotations:
      message: An aggregated API {{ $labels.name }}/{{ $labels.namespace }} has
        reported errors. The number of errors have increased for it in the past
        five minutes. High values indicate that the availability of the service
        changes too often.
      runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-aggregatedapierrors
    expr: |
      sum by(name, namespace)(increase(aggregator_unavailable_apiservice_count[5m])) > 2
    labels:
      component: kubernetes
      oncall_team: dedicated_service
      severity: warning
  # open this when we use aggregated apiserver
  # - alert: AggregatedAPIDown
  #   annotations:
  #     message: An aggregated API {{ $labels.name }}/{{ $labels.namespace }} is down.
  #       It has not been available at least for the past five minutes.
  #     runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-aggregatedapidown
  #   expr: |
  #     sum by(name, namespace)(sum_over_time(aggregator_unavailable_apiservice[5m])) > 0
  #   for: 5m
  #   labels:
  #     severity: warning
  - alert: KubeAPIDown
    annotations:
      message: KubeAPI has disappeared from Prometheus target discovery.
      runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeapidown
    expr: |
      up{job="apiserver"} == 0
    for: 15m
    labels:
      component: kubernetes
      oncall_team: dedicated_service
      severity: major
      response_class: ticket