groups:
- name: loki
  rules:
  - alert: LokiProcessTooManyRestarts
    annotations:
      description: |-
        A loki process had too many restarts (target {{ $labels.instance }})
          VALUE = {{ $value }}
          LABELS = {{ $labels }}
      summary: Loki process too many restarts (instance {{ $labels.instance }})
    expr: changes(process_start_time_seconds{job=~"loki"}[15m]) > 2
    for: 0m
    labels:
      oncall_team: o11y
      severity: warning
  - alert: LokiRequestErrors
    annotations:
      description: |-
        The {{ $labels.job }} and {{ $labels.route }} are experiencing errors
          VALUE = {{ $value }}
          LABELS = {{ $labels }}
      summary: Loki request errors (instance {{ $labels.instance }})
    expr: 100 * sum(rate(loki_request_duration_seconds_count{status_code=~"5.."}[1m]))
      by (namespace, job, route) / sum(rate(loki_request_duration_seconds_count[1m]))
      by (namespace, job, route) > 10
    for: 15m
    labels:
      oncall_team: o11y
      severity: critical
  - alert: LokiRequestPanic
    annotations:
      description: |-
        The {{ $labels.job }} is experiencing {{ printf "%.2f" $value }}% increase of panics
          VALUE = {{ $value }}
          LABELS = {{ $labels }}
      summary: Loki request panic (instance {{ $labels.instance }})
    expr: sum(increase(loki_panic_total[10m])) by (namespace, job) > 0
    for: 5m
    labels:
      oncall_team: o11y
      severity: critical
  # - alert: LokiRequestLatency
  #   annotations:
  #     description: |-
  #       The {{ $labels.job }} {{ $labels.route }} is experiencing {{ printf "%.2f" $value }}s 99th percentile latency
  #         VALUE = {{ $value }}
  #         LABELS = {{ $labels }}
  #     summary: Loki request latency (instance {{ $labels.instance }})
  #   expr: (histogram_quantile(0.99, sum(rate(loki_request_duration_seconds_bucket{route!~"(?i).*tail.*"}[5m]))
  #     by (le)))  > 1
  #   for: 5m
  #   labels:
  #     severity: warning