groups:
- name: dedicated-tiproxy-kernel-governance.rules
  rules:
  - alert: TiProxyToTiDBLatencyGapTooHigh
    expr: |-
      (
        label_replace(
          histogram_quantile(
            0.99,
            sum(rate(tiproxy_session_query_duration_seconds_bucket{component="tiproxy"}[1m])) by (le, tidb_cluster_id, backend)
          ),
          "tidb_pod", "$1", "backend", "^([^.]+).*"
        )
        -
        on (tidb_cluster_id, tidb_pod) group_left(backend)
        label_replace(
          label_replace(
            histogram_quantile(
              0.99,
              sum by (cluster_id, instance, le) (
                rate(tidb_server_handle_query_duration_seconds_bucket{component="tidb", sql_type!="internal"}[1m])
              )
            ),
            "tidb_cluster_id", "$1", "cluster_id", "(.*)"
          ),
          "tidb_pod", "$1", "instance", "(.+)"
        )
      )
      > 0.01
      and on (tidb_cluster_id, backend)
      sum by (tidb_cluster_id, backend) (rate(tiproxy_session_query_total{component="tiproxy"}[2h])) > 100
    for: 10m
    labels:
      tier: dedicated
      severity: major
      response_class: ticket
      component: tiproxy
      stability_governance: dedicated-kernel-governance
      visibility: internal
      source_component: tiproxy
      oncall_team: tiproxy
    annotations:
      description: 'Cluster {{ $labels.tidb_cluster_id }} TiProxy backend {{ $labels.backend }} and matched TiDB instance {{ $labels.tidb_pod }} have a P99 latency gap above 10ms for 10 minutes while backend 2-hour average QPS remains above 100. This suggests extra latency on the TiProxy side or on the path between TiProxy and TiDB. value: {{ $value }}'
      runbook_url: https://github.com/tidbcloud/runbooks/wiki/TiProxyToTiDBLatencyGapTooHigh
      summary: TiProxy and TiDB latency gap is too high
  - alert: TiProxyToTiDBLatencyGapTooHighP95
    expr: |-
      (
        label_replace(
          histogram_quantile(
            0.95,
            sum(rate(tiproxy_session_query_duration_seconds_bucket{component="tiproxy"}[1m])) by (le, tidb_cluster_id, backend)
          ),
          "tidb_pod", "$1", "backend", "^([^.]+).*"
        )
        -
        on (tidb_cluster_id, tidb_pod) group_left(backend)
        label_replace(
          label_replace(
            histogram_quantile(
              0.95,
              sum by (cluster_id, instance, le) (
                rate(tidb_server_handle_query_duration_seconds_bucket{component="tidb", sql_type!="internal"}[1m])
              )
            ),
            "tidb_cluster_id", "$1", "cluster_id", "(.*)"
          ),
          "tidb_pod", "$1", "instance", "(.+)"
        )
      )
      > 0.01
      and on (tidb_cluster_id, backend)
      sum by (tidb_cluster_id, backend) (rate(tiproxy_session_query_total{component="tiproxy"}[2h])) > 100
    for: 5m
    labels:
      tier: dedicated
      severity: major
      response_class: ticket
      component: tiproxy
      stability_governance: dedicated-kernel-governance
      visibility: internal
      source_component: tiproxy
      oncall_team: tiproxy
    annotations:
      description: 'Cluster {{ $labels.tidb_cluster_id }} TiProxy backend {{ $labels.backend }} and matched TiDB instance {{ $labels.tidb_pod }} have a P95 latency gap above 10ms for 5 minutes while backend 2-hour average QPS remains above 100. This suggests extra latency on the TiProxy side or on the path between TiProxy and TiDB. value: {{ $value }}'
      runbook_url: https://github.com/tidbcloud/runbooks/wiki/TiProxyToTiDBLatencyGapTooHighP95
      summary: TiProxy and TiDB P95 latency gap is too high
  # QPS drop: per-tiproxy-instance rate[2m] < 10% of 1h/2h rolling means; floor rate[2h] > 1000; unless cluster sum < 30% of cluster 2h mean; for 10m.
  - alert: TiProxyQPSDroppedBy90Percent
    expr: |-
      (
        (
          sum by (tidb_cluster_id, instance) (rate(tiproxy_session_query_total{component="tiproxy"}[2m]))
          < 0.1 * (
            avg_over_time(
              sum by (tidb_cluster_id, instance) (rate(tiproxy_session_query_total{component="tiproxy"}[2m]))[2h:]
            )
          )
        )
        and
        (
          sum by (tidb_cluster_id, instance) (rate(tiproxy_session_query_total{component="tiproxy"}[2m]))
          < 0.1 * (
            avg_over_time(
              sum by (tidb_cluster_id, instance) (rate(tiproxy_session_query_total{component="tiproxy"}[2m]))[1h:]
            )
          )
        )
        and
        sum by (tidb_cluster_id, instance) (rate(tiproxy_session_query_total{component="tiproxy"}[2h])) > 1000
        unless on (tidb_cluster_id) (
          sum by (tidb_cluster_id) (rate(tiproxy_session_query_total{component="tiproxy"}[2m]))
          < 0.3 * (
            avg_over_time(
              sum by (tidb_cluster_id) (rate(tiproxy_session_query_total{component="tiproxy"}[2m]))[2h:]
            )
          )
        )
      )
    for: 10m
    labels:
      tier: dedicated
      severity: critical
      response_class: page
      component: tiproxy
      stability_governance: dedicated-kernel-governance
      visibility: internal
      source_component: tiproxy
      oncall_team: tiproxy
    annotations:
      description: 'Cluster {{ $labels.tidb_cluster_id }} TiProxy instance {{ $labels.instance }} tiproxy_session_query_total QPS (2m rate) is below 10% of both the 1h and 2h rolling averages for 10 minutes, while that instance 2-hour average QPS remains above 1000, and cluster-wide traffic is not already below 30% of the cluster 2-hour average. Check TiProxy instance health, traffic routing, and upstream connectivity. value: {{ $value }}'
      runbook_url: https://docs.pingcap.com/tidb/stable/tiproxy-grafana/
      summary: TiProxy per-instance QPS below 10% of 1h/2h rolling averages (≈90% drop; 2h-average QPS > 1000; suppressed when cluster-wide QPS < 30% of 2h cluster average; for 10m)
  # QPS drop: per-backend rate[2m] < 30% of 1h/2h rolling means; floor rate[2h] > 1000; unless cluster sum < 50% of cluster 2h mean; TiDB uptime > 12h; for 10m.
  - alert: TiProxyQPSDroppedBy70Percent
    expr: |-
      max by (tidb_cluster_id) (
        label_replace(
          (
            (
              sum by (tidb_cluster_id, backend) (rate(tiproxy_session_query_total{component="tiproxy"}[2m]))
              < 0.3 * (
                avg_over_time(
                  sum by (tidb_cluster_id, backend) (rate(tiproxy_session_query_total{component="tiproxy"}[2m]))[2h:]
                )
              )
            )
            and
            (
              sum by (tidb_cluster_id, backend) (rate(tiproxy_session_query_total{component="tiproxy"}[2m]))
              < 0.3 * (
                avg_over_time(
                  sum by (tidb_cluster_id, backend) (rate(tiproxy_session_query_total{component="tiproxy"}[2m]))[1h:]
                )
              )
            )
            and
            sum by (tidb_cluster_id, backend) (rate(tiproxy_session_query_total{component="tiproxy"}[2h])) > 1000
            unless on (tidb_cluster_id) (
              sum by (tidb_cluster_id) (rate(tiproxy_session_query_total{component="tiproxy"}[2m]))
              < 0.5 * (
                avg_over_time(
                  sum by (tidb_cluster_id) (rate(tiproxy_session_query_total{component="tiproxy"}[2m]))[2h:]
                )
              )
            )
          ),
          "tidb_pod", "$1", "backend", "^([^.]+).*"
        )
        and on (tidb_cluster_id, tidb_pod) (
          label_replace(
            (time() - process_start_time_seconds{component="tidb",instance=~"^db-[0-9]+-tidb-.+"}) > 43200,
            "tidb_pod", "$1", "instance", "(.+)"
          )
        )
      )
    for: 10m
    labels:
      tier: dedicated
      severity: major
      response_class: ticket
      component: tiproxy
      stability_governance: dedicated-kernel-governance
      visibility: internal
      source_component: tiproxy
      oncall_team: tiproxy
    annotations:
      description: 'Cluster {{ $labels.tidb_cluster_id }} tiproxy_session_query_total per-backend QPS (2m rate) is below 30% of both the 1h and 2h rolling averages on at least one backend for 10 minutes, while that backend 2-hour average QPS remains above 1000, the matching TiDB process has been up more than 12 hours, and cluster-wide traffic is not already below 50% of the cluster 2-hour average. Check traffic routing, backend health, and TiProxy. cluster: {{ $labels.tidb_cluster_id }}, per-backend QPS < 30% of 1h/2h rolling (unless cluster sum < 50% of cluster 2h); 2h QPS floor > 1000; TiDB uptime > 12h: {{ $value }}'
      runbook_url: https://docs.pingcap.com/tidb/stable/tiproxy-grafana/
      summary: TiProxy per-backend QPS below 30% of 1h/2h rolling averages (≈70% drop; max across backends; 2h-average QPS > 1000; suppressed when cluster-wide QPS < 50% of 2h cluster average; TiDB backend uptime > 12h; for 10m)