groups:
- name: loki
  rules:
  - alert: LokiProcessTooManyRestarts
    annotations:
      description: |-
        A loki process had too many restarts (target {{ $labels.instance }})
          VALUE = {{ $value }}
          LABELS = {{ $labels }}
      summary: Loki process too many restarts (instance {{ $labels.instance }})
    expr: changes(process_start_time_seconds{job=~"loki"}[15m]) > 2
    for: 0m
    labels:
      oncall_team: o11y
      severity: warning
  - alert: LokiRequestErrors
    annotations:
      description: |-
        The {{ $labels.job }} and {{ $labels.route }} are experiencing errors
          VALUE = {{ $value }}
          LABELS = {{ $labels }}
      summary: Loki request errors (instance {{ $labels.instance }})
      runbook_url: https://pingcap.feishu.cn/wiki/wikcnJ3TYvhozrRtWCMBh5z92Wd#doxcnVfVFx49L34oFHEdR4lgjGd
    expr: 100 * sum(rate(loki_request_duration_seconds_count{status_code=~"5.."}[1m]))
      by (namespace, job, route) / sum(rate(loki_request_duration_seconds_count[1m]))
      by (namespace, job, route) > 10
    for: 15m
    labels:
      oncall_team: o11y
      severity: critical
  - alert: LokiRequestPanic
    annotations:
      description: |-
        The {{ $labels.job }} is experiencing {{ printf "%.2f" $value }}% increase of panics
          VALUE = {{ $value }}
          LABELS = {{ $labels }}
      summary: Loki request panic (instance {{ $labels.instance }})
      runbook_url: https://pingcap.feishu.cn/wiki/wikcnJ3TYvhozrRtWCMBh5z92Wd#doxcnavgNvLkRNGc8aNBwDYTyQd
    expr: sum(increase(loki_panic_total[10m])) by (namespace, job) > 0
    for: 5m
    labels:
      oncall_team: o11y
      severity: critical
  # - alert: LokiP99RequestLatencyHigh
  #   annotations:
  #     description: |-
  #       The {{ $labels.job }} {{ $labels.route }} is experiencing {{ printf "%.2f" $value }}s 99th percentile latency
  #     summary: Loki request latency (instance {{ $labels.instance }})
  #   expr: (histogram_quantile(0.99, sum(rate(loki_request_duration_seconds_bucket{method!="POST", route!~"(?i).*tail.*"}[5m]))
  #     by (job,route,le)))  > 1
  #   for: 15m
  #   labels:
  #     severity: warning
  # - alert: LokiP90ReadRequestLatency
  #   annotations:
  #     description: |-
  #       The {{ $labels.job }} {{ $labels.route }} is experiencing {{ printf "%.2f" $value }}s 99th percentile latency
  #     summary: Loki request latency (instance {{ $labels.instance }})
  #     runbook_url: https://pingcap.feishu.cn/wiki/wikcnJ3TYvhozrRtWCMBh5z92Wd#doxcnmr5iZ8ABl8HXJMWeuZZENb
  #   expr: (histogram_quantile(0.9, sum(rate(loki_request_duration_seconds_bucket{method=~"GET",route!~"(?i).*tail.*"}[5m]))
  #     by (route, le)))  > 1
  #   for: 30m
  #   labels:
  #     severity: warning
  - alert: LokiP99WriteRequestLatencyHigh
    annotations:
      description: |-
        The {{ $labels.job }} {{ $labels.route }} is experiencing {{ printf "%.2f" $value }}s 99th percentile latency
      summary: Loki write request latency (instance {{ $labels.instance }})
      runbook_url: https://pingcap.feishu.cn/wiki/wikcnJ3TYvhozrRtWCMBh5z92Wd#doxcnShmMVZBfeghgDFmwpr3S0f
    expr: (histogram_quantile(0.99, sum(rate(loki_request_duration_seconds_bucket{method="POST",route!~"(?i).*tail.*"}[5m]))
      by (route, le)))  > 1
    for: 30m
    labels:
      oncall_team: o11y
      severity: warning
  - alert: LokiWriteS3Failed
    annotations:
      description: |-
        The {{ $labels.job }}  in {{ $labels.namespace  }} put object to S3 failed
          VALUE = {{ $value }}
          LABELS = {{ $labels }}
      summary: Loki put object to S3 failed (instance {{ $labels.instance }})
      runbook_url: https://pingcap.feishu.cn/wiki/wikcnJ3TYvhozrRtWCMBh5z92Wd#doxcnQdTlQ3hEHexBFQMcElOp0g
    expr: rate(loki_s3_request_duration_seconds_count{status_code="5.*",operation="S3.PutObject"}[2m]) >10
    for: 5m
    labels:
      oncall_team: o11y
      severity: major
  - alert: LokiWriteGCSFailed
    annotations:
      description: |-
        The {{ $labels.job }}  in {{ $labels.namespace  }} put object to GCS failed
          VALUE = {{ $value }}
          LABELS = {{ $labels }}
      summary: Loki put object to GCS failed (instance {{ $labels.instance }})
      runbook_url: https://pingcap.feishu.cn/wiki/wikcnJ3TYvhozrRtWCMBh5z92Wd#doxcn56i9JBboCZOx9BVoqVN8Jf
    expr: rate(loki_gcs_request_duration_seconds_count{operation="POST",status_code=~"5.*"}[2m]) >10
    for: 5m
    labels:
      oncall_team: o11y
      severity: major
  - alert: LokiReadAutoscalerMaxedOut
    expr: kube_horizontalpodautoscaler_status_current_replicas{namespace="observability",horizontalpodautoscaler="loki-ha-read"} == kube_horizontalpodautoscaler_spec_max_replicas{namespace="observability",horizontalpodautoscaler="loki-ha-read"}
    for: 12h
    labels:
      oncall_team: o11y
      severity: major
    annotations:
      description: HPA {{ $labels.namespace }}/{{ $labels.horizontalpodautoscaler }} has been running at max replicas for longer than 1h; this can indicate underprovisioning.
      summary: HPA has been running at max replicas for an extended time
      runbook_url: https://pingcap.feishu.cn/docx/doxcnIP9B4qQyNO92A02TEMNmKg#doxcncslqU56UQylv3c7gPXcCxb
  - alert: LokiWriteAutoscalerMaxedOut
    expr: kube_horizontalpodautoscaler_status_current_replicas{namespace="observability",horizontalpodautoscaler="loki-ha-write"} == kube_horizontalpodautoscaler_spec_max_replicas{namespace="observability",horizontalpodautoscaler="loki-ha-write"}
    for: 12h
    labels:
      oncall_team: o11y
      severity: major
    annotations:
      description: HPA {{ $labels.namespace }}/{{ $labels.horizontalpodautoscaler }} has been running at max replicas for longer than 3h; this can indicate underprovisioning.
      summary: HPA has been running at max replicas for an extended time
      runbook_url: https://pingcap.feishu.cn/docx/doxcnIP9B4qQyNO92A02TEMNmKg#doxcncslqU56UQylv3c7gPXcCxb