groups:
- name: cluster-next-gen-tiproxy-kernel-governance.rules
  rules:
  # P99: 10× 24h rolling baseline + per-instance rate(...[2h]) QPS > 1000 + instance QPS < 5× 24h rolling QPS baseline; for 20m (major).
  - alert: TiProxyQueryP99LatencySurge10X
    expr: |-
      max by (tidb_cluster_id) (
        (
          histogram_quantile(
            0.99,
            sum(rate(tiproxy_session_query_duration_seconds_bucket{component="tiproxy",sharedpool_id!="2048291787704893440",tidb_cluster_id!~"^(10833157969408330931|10542225016081428407|10378381880046679304)$"}[1m])) by (le, tidb_cluster_id, instance)
          )
          >
          10 * histogram_quantile(
            0.99,
            avg_over_time(
              sum by (le, tidb_cluster_id, instance) (
                rate(tiproxy_session_query_duration_seconds_bucket{component="tiproxy",sharedpool_id!="2048291787704893440",tidb_cluster_id!~"^(10833157969408330931|10542225016081428407|10378381880046679304)$"}[5m])
              )[24h:5m]
            )
          )
        )
        and
        sum by (tidb_cluster_id, instance) (rate(tiproxy_session_query_total{component="tiproxy",sharedpool_id!="2048291787704893440",tidb_cluster_id!~"^(10833157969408330931|10542225016081428407|10378381880046679304)$"}[2h])) > 1000
        and
        sum by (tidb_cluster_id, instance) (rate(tiproxy_session_query_total{component="tiproxy",sharedpool_id!="2048291787704893440",tidb_cluster_id!~"^(10833157969408330931|10542225016081428407|10378381880046679304)$"}[2m]))
        <
        5 * avg_over_time(
          sum by (tidb_cluster_id, instance) (rate(tiproxy_session_query_total{component="tiproxy",sharedpool_id!="2048291787704893440",tidb_cluster_id!~"^(10833157969408330931|10542225016081428407|10378381880046679304)$"}[5m]))[24h:5m]
        )
      )
    for: 20m
    labels:
      tier: next-gen
      severity: major
      response_class: ticket
      component: tiproxy
      stability_governance: next-gen-kernel-governance
      visibility: internal
      source_component: tiproxy
      oncall_team: tiproxy
    annotations:
      description: 'Cluster {{ $labels.tidb_cluster_id }} tiproxy_session_query_duration_seconds P99 (1m) exceeds 10 times the 24-hour rolling P99 baseline on at least one TiProxy instance while 2-hour average QPS remains above 1000 and instance QPS (2m rate) is below 5 times the 24-hour rolling QPS baseline for 20 minutes. P99 surges driven by QPS growth (>=5x) are suppressed. Check TiDB load, network, and slow queries. cluster: {{ $labels.tidb_cluster_id }}, P99 vs 24h rolling baseline (10× threshold) with 2h-rate QPS > 1000 and QPS < 5× 24h baseline (max across instances): {{ $value }}'
      runbook_url: https://docs.pingcap.com/tidb/stable/tiproxy-grafana/
      summary: TiProxy query P99 exceeds 10x the 24h rolling P99 baseline with instance 2h-average QPS > 1000 and instance QPS < 5x 24h rolling QPS baseline (per instance sum, max across cluster; for 20m; major)
  # QPS drop: per-tiproxy-instance rate[2m] < 10% of 1h/2h rolling means; floor rate[2h] > 1000; unless cluster sum < 30% of cluster 2h mean; for 10m.
  - alert: TiProxyQPSDroppedBy90Percent
    expr: |-
      (
        (
          sum by (tidb_cluster_id, instance) (rate(tiproxy_session_query_total{component="tiproxy",sharedpool_id!="2048291787704893440",tidb_cluster_id!~"^(10833157969408330931|10542225016081428407|10378381880046679304)$"}[2m]))
          < 0.1 * (
            avg_over_time(
              sum by (tidb_cluster_id, instance) (rate(tiproxy_session_query_total{component="tiproxy",sharedpool_id!="2048291787704893440",tidb_cluster_id!~"^(10833157969408330931|10542225016081428407|10378381880046679304)$"}[2m]))[2h:]
            )
          )
        )
        and
        (
          sum by (tidb_cluster_id, instance) (rate(tiproxy_session_query_total{component="tiproxy",sharedpool_id!="2048291787704893440",tidb_cluster_id!~"^(10833157969408330931|10542225016081428407|10378381880046679304)$"}[2m]))
          < 0.1 * (
            avg_over_time(
              sum by (tidb_cluster_id, instance) (rate(tiproxy_session_query_total{component="tiproxy",sharedpool_id!="2048291787704893440",tidb_cluster_id!~"^(10833157969408330931|10542225016081428407|10378381880046679304)$"}[2m]))[1h:]
            )
          )
        )
        and
        sum by (tidb_cluster_id, instance) (rate(tiproxy_session_query_total{component="tiproxy",sharedpool_id!="2048291787704893440",tidb_cluster_id!~"^(10833157969408330931|10542225016081428407|10378381880046679304)$"}[2h])) > 1000
        unless on (tidb_cluster_id) (
          sum by (tidb_cluster_id) (rate(tiproxy_session_query_total{component="tiproxy",sharedpool_id!="2048291787704893440",tidb_cluster_id!~"^(10833157969408330931|10542225016081428407|10378381880046679304)$"}[2m]))
          < 0.3 * (
            avg_over_time(
              sum by (tidb_cluster_id) (rate(tiproxy_session_query_total{component="tiproxy",sharedpool_id!="2048291787704893440",tidb_cluster_id!~"^(10833157969408330931|10542225016081428407|10378381880046679304)$"}[2m]))[2h:]
            )
          )
        )
      )
    for: 10m
    labels:
      tier: next-gen
      severity: critical
      response_class: page
      component: tiproxy
      stability_governance: next-gen-kernel-governance
      visibility: internal
      source_component: tiproxy
      oncall_team: tiproxy
    annotations:
      description: 'Cluster {{ $labels.tidb_cluster_id }} TiProxy instance {{ $labels.instance }} tiproxy_session_query_total QPS (2m rate) is below 10% of both the 1h and 2h rolling averages for 10 minutes, while that instance 2-hour average QPS remains above 1000, and cluster-wide traffic is not already below 30% of the cluster 2-hour average. Check TiProxy instance health, traffic routing, and upstream connectivity. value: {{ $value }}'
      runbook_url: https://docs.pingcap.com/tidb/stable/tiproxy-grafana/
      summary: TiProxy per-instance QPS below 10% of 1h/2h rolling averages (≈90% drop; 2h-average QPS > 1000; suppressed when cluster-wide QPS < 30% of 2h cluster average; for 10m)
  - alert: TiProxyToTiDBLatencyGapTooHigh
    expr: |-
      (
        label_replace(
          histogram_quantile(
            0.99,
            sum(rate(tiproxy_session_query_duration_seconds_bucket{component="tiproxy"}[1m])) by (le, tidb_cluster_id, backend)
          ),
          "tidb_pod", "$1", "backend", "^([^.]+).*"
        )
        -
        on (tidb_cluster_id, tidb_pod) group_left(backend)
        label_replace(
          label_replace(
            histogram_quantile(
              0.99,
              sum by (cluster_id, instance, le) (
                rate(tidb_server_handle_query_duration_seconds_bucket{component="tidb", sql_type!="internal"}[1m])
              )
            ),
            "tidb_cluster_id", "$1", "cluster_id", "(.*)"
          ),
          "tidb_pod", "$1", "instance", "(.+)"
        )
      )
      > 0.01
      and on (tidb_cluster_id, backend)
      sum by (tidb_cluster_id, backend) (rate(tiproxy_session_query_total{component="tiproxy"}[2h])) > 100
    for: 10m
    labels:
      tier: next-gen
      severity: major
      response_class: ticket
      component: tiproxy
      stability_governance: next-gen-kernel-governance
      visibility: internal
      source_component: tiproxy
      oncall_team: tiproxy
    annotations:
      description: 'Cluster {{ $labels.tidb_cluster_id }} TiProxy backend {{ $labels.backend }} and matched TiDB instance {{ $labels.tidb_pod }} have a P99 latency gap above 10ms for 10 minutes while backend 2-hour average QPS remains above 100. This suggests extra latency on the TiProxy side or on the path between TiProxy and TiDB. value: {{ $value }}'
      runbook_url: https://github.com/tidbcloud/runbooks/wiki/TiProxyToTiDBLatencyGapTooHigh
      summary: TiProxy and TiDB latency gap is too high
  - alert: TiProxyToTiDBLatencyGapTooHighP95
    expr: |-
      (
        label_replace(
          histogram_quantile(
            0.95,
            sum(rate(tiproxy_session_query_duration_seconds_bucket{component="tiproxy"}[1m])) by (le, tidb_cluster_id, backend)
          ),
          "tidb_pod", "$1", "backend", "^([^.]+).*"
        )
        -
        on (tidb_cluster_id, tidb_pod) group_left(backend)
        label_replace(
          label_replace(
            histogram_quantile(
              0.95,
              sum by (cluster_id, instance, le) (
                rate(tidb_server_handle_query_duration_seconds_bucket{component="tidb", sql_type!="internal"}[1m])
              )
            ),
            "tidb_cluster_id", "$1", "cluster_id", "(.*)"
          ),
          "tidb_pod", "$1", "instance", "(.+)"
        )
      )
      > 0.01
      and on (tidb_cluster_id, backend)
      sum by (tidb_cluster_id, backend) (rate(tiproxy_session_query_total{component="tiproxy"}[2h])) > 100
    for: 5m
    labels:
      tier: next-gen
      severity: major
      response_class: ticket
      component: tiproxy
      stability_governance: next-gen-kernel-governance
      visibility: internal
      source_component: tiproxy
      oncall_team: tiproxy
    annotations:
      description: 'Cluster {{ $labels.tidb_cluster_id }} TiProxy backend {{ $labels.backend }} and matched TiDB instance {{ $labels.tidb_pod }} have a P95 latency gap above 10ms for 5 minutes while backend 2-hour average QPS remains above 100. This suggests extra latency on the TiProxy side or on the path between TiProxy and TiDB. value: {{ $value }}'
      runbook_url: https://github.com/tidbcloud/runbooks/wiki/TiProxyToTiDBLatencyGapTooHighP95
      summary: TiProxy and TiDB P95 latency gap is too high