apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
  labels:
    release: self-monitoring
  name: loki
  namespace: {{ .Values.namespace.selfMonitoring }}
spec:
  groups:
  - name: loki
    rules:
    - alert: LokiProcessTooManyRestarts
      annotations:
        description: |-
          A loki process had too many restarts (target {{`{{ $labels.instance }}`}})
            VALUE = {{`{{ $value }}`}}
            LABELS = {{`{{ $labels }}`}}
        summary: Loki process too many restarts (instance {{`{{ $labels.instance }}`}})
      expr: changes(process_start_time_seconds{job=~"loki"}[15m]) > 2
      for: 0m
      labels:
        severity: warning
    - alert: LokiRequestErrors
      annotations:
        description: |-
          The {{`{{ $labels.job }}`}} and {{`{{ $labels.route }}`}} are experiencing errors
            VALUE = {{`{{ $value }}`}}
            LABELS = {{`{{ $labels }}`}}
        summary: Loki request errors (instance {{`{{ $labels.instance }}`}})
        runbook_url: https://pingcap.feishu.cn/wiki/wikcnJ3TYvhozrRtWCMBh5z92Wd#doxcnVfVFx49L34oFHEdR4lgjGd
      expr: 100 * sum(rate(loki_request_duration_seconds_count{status_code=~"5.."}[1m]))
        by (namespace, job, route) / sum(rate(loki_request_duration_seconds_count[1m]))
        by (namespace, job, route) > 10
      for: 15m
      labels:
        severity: critical
    - alert: LokiRequestPanic
      annotations:
        description: |-
          The {{`{{ $labels.job }}`}} is experiencing {{`{{ printf "%.2f" $value }}`}}% increase of panics
            VALUE = {{`{{ $value }}`}}
            LABELS = {{`{{ $labels }}`}}
        summary: Loki request panic (instance {{`{{ $labels.instance }}`}})
        runbook_url: https://pingcap.feishu.cn/wiki/wikcnJ3TYvhozrRtWCMBh5z92Wd#doxcnavgNvLkRNGc8aNBwDYTyQd
      expr: sum(increase(loki_panic_total[10m])) by (namespace, job) > 0
      for: 5m
      labels:
        severity: critical
    - alert: LokiP99RequestLatencyHigh
      annotations:
        description: |-
          The {{`{{ $labels.job }}`}} {{`{{ $labels.route }}`}} is experiencing {{`{{ printf "%.2f" $value }}`}}s 99th percentile latency
        summary: Loki request latency (instance {{`{{ $labels.instance }}`}})
      expr: (histogram_quantile(0.99, sum(rate(loki_request_duration_seconds_bucket{method!="POST", route!~"(?i).*tail.*"}[5m]))
        by (job,route,le)))  > 1
      for: 15m
      labels:
        severity: warning
    - alert: LokiP90ReadRequestLatency
      annotations:
        description: |-
          The {{`{{ $labels.job }}`}} {{`{{ $labels.route }}`}} is experiencing {{`{{ printf "%.2f" $value }}`}}s 99th percentile latency
        summary: Loki request latency (instance {{`{{ $labels.instance }}`}})
        runbook_url: https://pingcap.feishu.cn/wiki/wikcnJ3TYvhozrRtWCMBh5z92Wd#doxcnmr5iZ8ABl8HXJMWeuZZENb
      expr: (histogram_quantile(0.9, sum(rate(loki_request_duration_seconds_bucket{method=~"GET",route!~"(?i).*tail.*"}[5m]))
        by (route, le)))  > 1
      for: 30m
      labels:
        severity: warning
    - alert: LokiP99WriteRequestLatencyHigh
      annotations:
        description: |-
          The {{`{{ $labels.job }}`}} {{`{{ $labels.route }}`}} is experiencing {{`{{ printf "%.2f" $value }}`}}s 99th percentile latency
        summary: Loki write request latency (instance {{`{{ $labels.instance }}`}})
        runbook_url: https://pingcap.feishu.cn/wiki/wikcnJ3TYvhozrRtWCMBh5z92Wd#doxcnShmMVZBfeghgDFmwpr3S0f
      expr: (histogram_quantile(0.99, sum(rate(loki_request_duration_seconds_bucket{method="POST",route!~"(?i).*tail.*"}[5m]))
        by (route, le)))  > 1
      for: 30m
      labels:
        severity: warning
    - alert: LokiWriteS3Failed
      annotations:
        description: |-
          The {{`{{ $labels.job }}`}}  in {{`{{ $labels.namespace  }}`}} put object to S3 failed
            VALUE = {{`{{ $value }}`}}
            LABELS = {{`{{ $labels }}`}}
        summary: Loki put object to S3 failed (instance {{`{{ $labels.instance }}`}})
        runbook_url: https://pingcap.feishu.cn/wiki/wikcnJ3TYvhozrRtWCMBh5z92Wd#doxcnQdTlQ3hEHexBFQMcElOp0g
      expr: rate(loki_s3_request_duration_seconds_count{status_code="500",operation="S3.PutObject"}[1m]) >10
      for: 5m
      labels:
        severity: major
    - alert: LokiWriteGCSFailed
      annotations:
        description: |-
          The {{`{{ $labels.job }}`}}  in {{`{{ $labels.namespace  }}`}} put object to GCS failed
            VALUE = {{`{{ $value }}`}}
            LABELS = {{`{{ $labels }}`}}
        summary: Loki put object to GCS failed (instance {{`{{ $labels.instance }}`}})
        runbook_url: https://pingcap.feishu.cn/wiki/wikcnJ3TYvhozrRtWCMBh5z92Wd#doxcn56i9JBboCZOx9BVoqVN8Jf
      expr: rate(loki_gcs_request_duration_seconds_count{operation="POST",status_code="500"}[1m]) >10
      for: 5m
      labels:
        severity: major
    - alert: LokiReadAutoscalerMaxedOut
      expr: kube_horizontalpodautoscaler_status_current_replicas{namespace="observability",horizontalpodautoscaler="loki-ha-read"} == kube_horizontalpodautoscaler_spec_max_replicas{namespace="observability",horizontalpodautoscaler="loki-ha-read"}
      for: 2h
      labels:
        severity: major
      annotations:
        description: HPA {{`{{ $labels.namespace }}`}}/{{`{{ $labels.horizontalpodautoscaler }}`}} has been running at max replicas for longer than 1h; this can indicate underprovisioning.
        summary: HPA has been running at max replicas for an extended time
        runbook_url: https://pingcap.feishu.cn/docx/doxcnIP9B4qQyNO92A02TEMNmKg#doxcncslqU56UQylv3c7gPXcCxb
    - alert: LokiWriteAutoscalerMaxedOut
      expr: kube_horizontalpodautoscaler_status_current_replicas{namespace="observability",horizontalpodautoscaler="loki-ha-write"} == kube_horizontalpodautoscaler_spec_max_replicas{namespace="observability",horizontalpodautoscaler="loki-ha-write"}
      for: 6h
      labels:
        severity: major
      annotations:
        description: HPA {{`{{ $labels.namespace }}`}}/{{`{{ $labels.horizontalpodautoscaler }}`}} has been running at max replicas for longer than 3h; this can indicate underprovisioning.
        summary: HPA has been running at max replicas for an extended time
        runbook_url: https://pingcap.feishu.cn/docx/doxcnIP9B4qQyNO92A02TEMNmKg#doxcncslqU56UQylv3c7gPXcCxb