groups:
- name: tiproxy
  rules:
  - alert: TiProxyServerIsDown
    expr: up{component="tiproxy"} == 0
    for: 5m
    labels:
      tier: dedicated
      severity: major
      response_class: ticket
      expr: up{component="tiproxy"} == 0
      component: tiproxy
      oncall_team: tiproxy
    annotations:
      description: 'cluster: {{ $labels.cluster_id }}, instance: {{ $labels.instance }} is down for 10 mins'
      runbook_url: https://docs.google.com/document/d/17dQQS3ONxhqdlqKth9kZHTtpANp49UgUBF_dMk5ewCE/edit#heading=h.mvt8hlb2k7p
      summary: TiProxy server is down
  - alert: TiProxyServerFrequentlyRestart
    expr: sum(changes(floor(process_start_time_seconds{job=~".*tiproxy"})[1h:15s])) by (cluster_id, instance, tenant,provider_type) >= 3
    for: 1m
    labels:
      tier: dedicated
      severity: critical
      response_class: page
      expr: sum(changes(floor(process_start_time_seconds{job=~".*tiproxy"})[1h:15s])) by (cluster_id, instance, tenant,provider_type) >= 3
      component: tiproxy
      oncall_team: tiproxy
    annotations:
      description: 'cluster: {{ $labels.cluster_id }}, instance: {{ $labels.instance }}, value: {{ $value }}'
      runbook_url: https://docs.google.com/document/d/17dQQS3ONxhqdlqKth9kZHTtpANp49UgUBF_dMk5ewCE/edit#heading=h.utnvsvbf6hy8
      summary: TiProxy server restarted {{ $value }} times in past an hour
  - alert: TiProxyClusterFrequentlyRestart
    expr: |-
      sum by (cluster_id, tenant, provider_type) (
          resets((-floor(process_start_time_seconds{job=~".*tiproxy"}))[1h:15s])
        and
          max_over_time(process_start_time_seconds{job=~".*tiproxy"}[30m] offset 1h) > 0
      ) >= 3
    for: 1m
    labels:
      tier: dedicated
      severity: critical
      response_class: page
      component: tiproxy
      oncall_team: tiproxy
    annotations:
      description: 'cluster: {{ $labels.cluster_id }}, instance: {{ $labels.instance }}, value: {{ $value }}'
      runbook_url: https://docs.google.com/document/d/17dQQS3ONxhqdlqKth9kZHTtpANp49UgUBF_dMk5ewCE/edit#heading=h.utnvsvbf6hy8
      summary: TiProxy servers in a cluster restarted {{ $value }} times in past an hour
  - alert: TiProxyDiscoveredTimeJumpBack
    expr: increase(tiproxy_monitor_time_jump_back_total[10m])  > 0
    for: 5m
    labels:
      tier: dedicated
      severity: major
      response_class: ticket
      expr: increase(tiproxy_monitor_time_jump_back_total[10m])  > 0
      component: tiproxy
      oncall_team: tiproxy
    annotations:
      description: 'cluster: {{ $labels.cluster_id }}, instance: {{ $labels.instance }}, values:{{ $value }}'
      runbook_url: https://docs.google.com/document/d/17dQQS3ONxhqdlqKth9kZHTtpANp49UgUBF_dMk5ewCE/edit#heading=h.ttcqlika9pqq
      summary: TiProxy monitor found time jump back error
  - alert: TiProxyReportErr
    expr: increase(tiproxy_server_err[10m]) > 5
    for: 5m
    labels:
      tier: dedicated
      severity: major
      response_class: ticket
      expr: increase(tiproxy_server_err[10m]) > 5
      component: tiproxy
      oncall_team: tiproxy
    annotations:
      description: 'cluster: {{ $labels.cluster_id }}, instance: {{ $labels.instance }}, value: {{ $value }}'
      runbook_url: https://docs.google.com/document/d/17dQQS3ONxhqdlqKth9kZHTtpANp49UgUBF_dMk5ewCE/edit#heading=h.bb5bh8qqc12b
      summary: TiProxy reports critical error
  - alert: TiProxyGetBackendFail
    expr: sum(rate(tiproxy_backend_get_backend{res="fail"}[1m])) by (cluster_id, instance, tenant, provider_type) > 10
    for: 5m
    labels:
      tier: dedicated
      severity: major
      response_class: ticket
      expr: sum(rate(tiproxy_backend_get_backend{res="fail"}[1m])) by (cluster_id, instance, tenant, provider_type) > 10
      component: tiproxy
      oncall_team: tiproxy
    annotations:
      description: 'cluster: {{ $labels.cluster_id }}, instance: {{ $labels.instance }}, value: {{ $value }}'
      runbook_url: https://docs.google.com/document/d/17dQQS3ONxhqdlqKth9kZHTtpANp49UgUBF_dMk5ewCE/edit#heading=h.k8bm4rl7zzt3
      summary: TiProxy fails to get backends for 5 minutes