groups:
- name: self-monitoring-o11y-metrics-server
  rules:
  - alert: MetricsServerRequestsErrorRateHigh
    annotations:
      message: "The error rate of requesting o11y server API is over 5%, VALUE = {{ $value }}"
      runbook_url: "https://pingcap.feishu.cn/wiki/wikcnUxwI6jMhjlqchchdtlw1Id"
    expr: |-
      sum(rate(dbaas_o11y_server_request_error_count{namespace=~"(nightly|staging|prod)-ms",service=~"metrics-.*"}[2m])) by(url,path,service) / sum(rate(dbaas_o11y_server_request_duration_ms_count{namespace=~"(nightly|staging|prod)-ms",service=~"metrics-.*"}[2m])) by(url,path,service) * 100 >= 5
    for: 5m
    labels:
      oncall_team: resilience
      component: metrics-server
      severity: major
  - alert: MetricsServerRequestsErrorRateHighFor15Min
    annotations:
      message: "The error rate of requesting o11y server API is over 5% for 15 minutes, VALUE = {{ $value }}"
      runbook_url: "https://pingcap.feishu.cn/wiki/wikcnUxwI6jMhjlqchchdtlw1Id"
    expr: |-
      sum(rate(dbaas_o11y_server_request_error_count{namespace=~"(nightly|staging|prod)-ms",service=~"metrics-.*"}[2m])) by(url,path,service) / sum(rate(dbaas_o11y_server_request_duration_ms_count{namespace=~"(nightly|staging|prod)-ms",service=~"metrics-.*"}[2m])) by(url,path,service) * 100 >= 5
    for: 15m
    labels:
      oncall_team: resilience
      component: metrics-server
      severity: critical
  - alert: MetricsServerRequestsDurationP99HighFor5Min
    annotations:
      message: "The P99 duration of requesting o11y server API is over 10s for 5 minutes, VALUE = {{ $value }}"
      runbook_url: "https://pingcap.feishu.cn/wiki/wikcnUxwI6jMhjlqchchdtlw1Id"
    expr: |-
      histogram_quantile(0.99, sum(rate(dbaas_o11y_server_request_duration_ms_bucket{namespace=~"(nightly|staging|prod)-ms", service=~"metrics-.*"}[2m])) by (url,path,service,le)) >= 10000
    for: 5m
    labels:
      oncall_team: resilience
      component: metrics-server
      severity: major
  - alert: DbaasGrpcServerRequestsErrorRateHighFor5Min
    annotations:
      message: "The error rate of requesting dbaas grpc server API is over 5% for 5 minutes, VALUE = {{ $value }}"
      runbook_url: "https://pingcap.feishu.cn/wiki/wikcnUxwI6jMhjlqchchdtlw1Id"
    expr: |-
      sum(rate(grpc_client_handled_total{namespace=~"(nightly|staging|prod)-ms",grpc_code!="OK",service=~"metrics-.*"}[2m])) by(grpc_service,grpc_method,service) / sum(rate(grpc_client_handled_total{namespace=~"(nightly|staging|prod)-ms",service=~"metrics-.*"}[2m])) by(grpc_service,grpc_method,service) * 100 >= 5
    for: 5m
    labels:
      oncall_team: resilience
      component: metrics-server
      severity: critical
  - alert: DbaasGrpcServerRequestsDurationP99HighFor5Min
    annotations:
      message: "The P99 duration of requesting dbaas grpc server API is over 10s for 5 minutes, VALUE = {{ $value }}"
      runbook_url: "https://pingcap.feishu.cn/wiki/wikcnUxwI6jMhjlqchchdtlw1Id"
    expr: |-
      histogram_quantile(0.99, sum(rate(grpc_client_handling_seconds_bucket{namespace=~"(nightly|staging|prod)-ms", service=~"metrics-.*"}[2m])) by (grpc_service,grpc_method,service,le)) >= 10
    for: 5m
    labels:
      oncall_team: resilience
      component: metrics-server
      severity: major
  - alert: MetricsServerQueryRequestsError
    expr: sum(increase(dbaas_o11y_server_query_request_total{status!="success",namespace=~"(nightly|staging|prod)-ms",service=~"metrics-.*", status!="empty", metric_name=~"tidb_cloud.db_(query|queries).*"}[2m])) by (tidb_cluster_id, url, status) > 0
    for: 5m
    annotations:
      message: "query o11y server API failed with retries for cluster {{ $labels.tidb_cluster_id }} due to {{ $labels.status }}"
      description: "query o11y server failed, this may cause data loss in metrics integration, please check the o11y server {{ $labels.url }} status"
      runbook_url: "https://pingcap.feishu.cn/wiki/wikcnUxwI6jMhjlqchchdtlw1Id#doxcnqIUAGXlqEPuwKzQETiBn2f"
    labels:
      oncall_team: resilience
      severity: warning
      component: metrics-server