groups:
- name: tiproxy-data-plane
  rules:
  - alert: TiProxyServerIsDown
    expr: up{component="tiproxy"} == 0
    for: 5m
    labels:
      tier: next-gen
      severity: major
      response_class: ticket
      component: tiproxy
      oncall_team: tiproxy
    annotations:
      description: 'TiProxy server {{ $labels.instance }} in cluster {{ $labels.tidb_cluster_id }} has been down for 5 minutes. TiProxy is critical for connection routing and load balancing. Please immediately check and restart the TiProxy server to restore connection routing functionality. cluster: {{ $labels.tidb_cluster_id }}, instance: {{ $labels.instance }} is down for 10 mins'
      runbook_url: https://docs.google.com/document/d/17dQQS3ONxhqdlqKth9kZHTtpANp49UgUBF_dMk5ewCE/edit#heading=h.mvt8hlb2k7p
      summary: TiProxy server is down
  - alert: TiProxyServerFrequentlyRestart
    expr: |-
      sum by (tidb_cluster_id, instance, tenant, provider_type) (
          resets((-floor(process_start_time_seconds{job=~".*tiproxy"}))[1h:15s])
        and
          max_over_time(process_start_time_seconds{job=~".*tiproxy"}[30m] offset 1h) > 0
      ) >= 3
    for: 1m
    labels:
      tier: next-gen
      severity: critical
      response_class: page
      component: tiproxy
      oncall_team: tiproxy
    annotations:
      description: 'TiProxy server {{ $labels.instance }} in cluster {{ $labels.tidb_cluster_id }} has restarted {{ $value }} times in the past hour. Frequent TiProxy restarts can affect connection routing and application connectivity. Please investigate the root cause and ensure TiProxy server stability. cluster: {{ $labels.tidb_cluster_id }}, instance: {{ $labels.instance }}, value: {{ $value }}'
      runbook_url: https://docs.google.com/document/d/17dQQS3ONxhqdlqKth9kZHTtpANp49UgUBF_dMk5ewCE/edit#heading=h.utnvsvbf6hy8
      summary: TiProxy server restarted {{ $value }} times in past an hour
  - alert: TiProxyClusterFrequentlyRestart
    expr: |-
      (
        sum by (tidb_cluster_id, tenant, provider_type) (
            resets((-floor(process_start_time_seconds{job=~".*tiproxy"}))[1h:15s])
          and
            max_over_time(process_start_time_seconds{job=~".*tiproxy"}[30m] offset 1h) > 0
        ) >= 3
      )
      and
      (
        sum(
          (
            sum(
              resets((-floor(process_start_time_seconds{job=~".*tiproxy"}))[1h:15s])
            ) by (tidb_cluster_id, instance, tenant, provider_type) >= 2
          )
        ) by (tidb_cluster_id, tenant, provider_type) > 0
      )
    for: 1m
    labels:
      tier: next-gen
      severity: critical
      response_class: page
      component: tiproxy
      oncall_team: tiproxy
    annotations:
      description: 'TiProxy servers in cluster {{ $labels.tidb_cluster_id }} have restarted {{ $value }} times in the past hour. Multiple TiProxy server restarts across the cluster can severely impact connection routing and application connectivity. Please investigate the root cause and ensure cluster stability. cluster: {{ $labels.tidb_cluster_id }}, instance: {{ $labels.instance }}, value: {{ $value }}'
      runbook_url: https://docs.google.com/document/d/17dQQS3ONxhqdlqKth9kZHTtpANp49UgUBF_dMk5ewCE/edit#heading=h.utnvsvbf6hy8
      summary: TiProxy servers in a cluster restarted {{ $value }} times in past an hour
  - alert: TiProxyDiscoveredTimeJumpBack
    expr: increase(tiproxy_monitor_time_jump_back_total[10m])  > 0
    for: 5m
    labels:
      tier: next-gen
      severity: major
      response_class: ticket
      component: tiproxy
      oncall_team: tiproxy
    annotations:
      description: 'TiProxy monitor on instance {{ $labels.instance }} in cluster {{ $labels.tidb_cluster_id }} has detected time jump back errors ({{ $value }}). Time inconsistencies can affect connection routing and proxy operations. Please check system time synchronization and NTP configuration.'
      runbook_url: https://docs.google.com/document/d/17dQQS3ONxhqdlqKth9kZHTtpANp49UgUBF_dMk5ewCE/edit#heading=h.ttcqlika9pqq
      summary: TiProxy monitor found time jump back error
  - alert: TiProxyReportErr
    expr: increase(tiproxy_server_err[10m]) > 5
    for: 5m
    labels:
      tier: next-gen
      severity: major
      response_class: ticket
      component: tiproxy
      oncall_team: tiproxy
    annotations:
      description: 'TiProxy instance {{ $labels.instance }} in cluster {{ $labels.tidb_cluster_id }} is reporting critical errors ({{ $value }}). Critical errors may affect connection routing and proxy functionality. Please investigate the error logs and resolve the underlying issues immediately.'
      runbook_url: https://docs.google.com/document/d/17dQQS3ONxhqdlqKth9kZHTtpANp49UgUBF_dMk5ewCE/edit#heading=h.bb5bh8qqc12b
      summary: TiProxy reports critical error
  - alert: TiProxyGetBackendFail
    expr: sum(rate(tiproxy_backend_get_backend{res="fail"}[1m])) by (tidb_cluster_id, instance, tenant, provider_type) > 10
    for: 5m
    labels:
      tier: next-gen
      severity: major
      response_class: ticket
      component: tiproxy
      oncall_team: tiproxy
    annotations:
      description: 'TiProxy instance {{ $labels.instance }} in cluster {{ $labels.tidb_cluster_id }} is failing to get backend connections ({{ $value }} failures per minute). Backend connection failures can prevent proper load balancing and affect client connectivity. Please check backend TiDB server availability and network connectivity.'
      runbook_url: https://docs.google.com/document/d/17dQQS3ONxhqdlqKth9kZHTtpANp49UgUBF_dMk5ewCE/edit#heading=h.k8bm4rl7zzt3
      summary: TiProxy fails to get backends for 5 minutes
  - alert: TiProxyBackendPingDurationTooHigh
    expr: tiproxy_backend_ping_duration_seconds > 0.5
    for: 2m
    labels:
      tier: next-gen
      severity: major
      response_class: ticket
      component: tiproxy
      oncall_team: tiproxy
    annotations:
      description: 'TiProxy instance {{ $labels.instance }} in cluster {{ $labels.tidb_cluster_id }} has backend ping duration above 0.5 seconds for 2 minutes (current value: {{ $value }}). This may indicate elevated network latency or unstable connectivity between TiProxy and backend TiDB nodes. Please check backend node availability and the network path between TiProxy and TiDB.'
      runbook_url: https://github.com/tidbcloud/runbooks/wiki/TiProxyBackendPingDurationTooHigh
      summary: TiProxy backend ping duration is higher than 0.5s for 2 minutes