groups:
- name: self-monitoring-o11y-business
  rules:
  - alert: O11yClusterPendingCreatingOver10m
    annotations:
      description: The clusters name {{ $labels.cluster_id }}  status in creating more than 10m
      summary: o11y cluster creating too long.
      runbook_url: https://pingcap.feishu.cn/wiki/wikcnJ3TYvhozrRtWCMBh5z92Wd#doxcnim8uEMQ4wm2mkd7SZKpCvf
    expr: |-
      sum(observability_cluster_info{bizType!~".*(dev|staging).*", status="CREATING"}) by(bizType, tenantID, cluster_id) and on (cluster_id) label_replace(agent_control_server_agent_info{status="Normal"},"cluster_id","$1","name","(.*)")  >=1
    for: 10m
    labels:
      oncall_team: o11y
      severity: critical
  - alert: O11yClusterNonProdPendingCreatingOver30m
    annotations:
      description: The clusters name {{ $labels.cluster_id }}  status in creating more than 30m
      summary: o11y cluster creating too long.
      runbook_url: https://pingcap.feishu.cn/wiki/wikcnJ3TYvhozrRtWCMBh5z92Wd#doxcnim8uEMQ4wm2mkd7SZKpCvf
    expr: |-
      sum(observability_cluster_info{bizType=~".*(dev|staging).*", status="CREATING"}) by(bizType, tenantID, cluster_id) and on (cluster_id) label_replace(agent_control_server_agent_info{status="Normal"},"cluster_id","$1","name","(.*)")  >=1
    for: 30m
    labels:
      oncall_team: o11y
      severity: major
  - alert: O11yAgentPendingCreatingOver30m
    annotations:
      description: The agent of tidb_cluster {{ $labels.cluster_id }} under tenant {{ $labels.tenant_id }} project {{ $labels.project_id }} status in creating more than 30m
      summary: o11y agent of tidb_cluster {{ $labels.cluster_id }} creating too long.
      runbook_url: https://pingcap-cn.feishu.cn/wiki/N4Fmw3BLqiL83OkNPAIcTbTmnQf#WvP2dVD0ho8h2rxnPSXcrrPcn5g
    expr: |-
      sum(agent_control_server_agent_info{biz_type!~".*(dev|staging).*",status="Creating"}) by(biz_type, tenant_id, project_id, cluster_id, name) > 0
    for: 30m
    labels:
      oncall_team: o11y
      severity: major
  - alert: O11yClusterPendingDeletionTooLong
    annotations:
      message: The cluster named {{ $labels.cluster_id }} pending in "TO_BE_DELETED" status for more than 1h.
      runbook_url: https://pingcap.feishu.cn/wiki/wikcnJ3TYvhozrRtWCMBh5z92Wd#doxcnFCSD4vcyEY3NQLoHcrHHAg
    expr: |-
      sum(observability_cluster_info{status="TO_BE_DELETED"}) by(cluster_id) >=1
    for: 1h
    labels:
      oncall_team: o11y
      severity: major
  - alert: O11yServiceHttpRequestCode5xx
    annotations:
      description: The service name {{ $labels.service }}  handler {{ $labels.handler }} 5xx code is {{ $labels.code }}
      summary: o11y service 5xx code.
      runbook_url: https://pingcap.feishu.cn/wiki/wikcnJ3TYvhozrRtWCMBh5z92Wd#doxcnVqvp1omXOTH2ob6g8x9eoN
    expr: |-
      sum(increase(observability_http_requests_total{code=~"5[0-9]{2}"}[1m])) by(handler,service,code) > 10
    for: 5m
    labels:
      oncall_team: o11y
      severity: major
  - alert: O11yRDSClientRequestError
    annotations:
      description: mysql client operation {{ $labels.operation }} is experiencing error
      message: mysql client operation {{ $labels.operation }} is experiencing error
      runbook_url: https://pingcap.feishu.cn/wiki/C2beweHGsinehIkmKqkcT9BUn0b#share-XQefdxOSToOsYoxhrmLcE45inub
    expr: |-
      sum(increase(observability_mysql_client_operation_duration_seconds_count{status="error"}[5m])) by (operation) > 1
    for: 5m
    labels:
      oncall_team: o11y
      severity: major
  - alert: PrometheusTsdbWalCorruptions
    expr: increase(prometheus_tsdb_wal_corruptions_total[1m]) > 0
    for: 0m
    labels:
      oncall_team: o11y
      severity: critical
      runbook_url: https://pingcap.feishu.cn/wiki/wikcnJ3TYvhozrRtWCMBh5z92Wd#doxcn87vNFAWoNXxuVjILE2GM0O
    annotations:
      summary: Prometheus TSDB WAL corruptions (instance {{ $labels.instance }})
      description: "Prometheus encountered {{ $value }} TSDB WAL corruptions\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
  - alert: PrometheusTsdbWalTruncationsFailed
    expr: increase(prometheus_tsdb_wal_truncations_failed_total[1m]) > 0
    for: 0m
    labels:
      oncall_team: o11y
      severity: critical
    annotations:
      summary: Prometheus TSDB WAL truncations failed (instance {{ $labels.instance }})
      description: "Prometheus encountered {{ $value }} TSDB WAL truncation failures\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
      runbook_url: https://pingcap.feishu.cn/wiki/wikcnJ3TYvhozrRtWCMBh5z92Wd#doxcn87vNFAWoNXxuVjILE2GM0O
  - alert: O11yAWSAPIRequestError
    expr: increase(observability_aws_api_request_total[1m]) > 5
    for: 0m
    labels:
      oncall_team: o11y
      severity: warning
    annotations:
      summary: O11y AWS API request failed (error {{ $labels.error_msg }})
      description: "O11y AWS API request failed, client = {{ $labels.aws_client }}, api = {{ $labels.aws_api }}, resource_name = {{ $labels.resource_name }}, error_msg = {{ $labels.error_msg }}"
      runbook_url: https://pingcap.feishu.cn/wiki/wikcnJ3TYvhozrRtWCMBh5z92Wd#doxcne2284Kq46IasVRspO0dMi8
  - alert: O11yGCPAPIRequestError
    expr: increase(observability_gcp_api_request_total[1m]) > 5
    for: 0m
    labels:
      oncall_team: o11y
      severity: warning
    annotations:
      summary: O11y GCP API request failed (error {{ $labels.error_msg }})
      description: "O11y GCP API request failed, client = {{ $labels.gcp_client }}, api = {{ $labels.gcp_api }}, resource_name = {{ $labels.resource_name }}, error_msg = {{ $labels.error_msg }}"
      runbook_url: https://pingcap.feishu.cn/wiki/wikcnJ3TYvhozrRtWCMBh5z92Wd#doxcnMy22UioMSAOGA5K7V82l9f
  - alert: DrhouseSlowlogsConverterLatencyHigh
    expr: (time() - slowlogconverter_task_last_run_timestamp) > 1800
    for: 2m
    labels:
      oncall_team: o11y
      severity: warning
    annotations:
      summary:  The clusters {{ $labels.cluster }}  slowlogs converter latency exceeds 30min
      description: The cluster {{ $labels.cluster }} slowlogs converter latency {{ $value }} seconds exceeds 30min
      runbook_url: https://pingcap.feishu.cn/wiki/wikcnJ3TYvhozrRtWCMBh5z92Wd#doxcnMy22UioMSAOGA5K7V82l9f
  - alert: SlowlogsQueryAPIRequestError
    expr: sum by (path) (rate(slowquery_handle_duration_seconds_count{status_code!="200"}[5m])) > 0
    for: 2m
    labels:
      oncall_team: o11y
      severity: warning
    annotations:
      summary:  slowlogs query api request error
      description: slowlogs query api failed
      runbook_url: https://pingcap.feishu.cn/wiki/wikcnJ3TYvhozrRtWCMBh5z92Wd#share-KfmMdfFZ6oJdjwxn5e3cUgUZnad
  - alert: SlowlogsQueryListLatencyHigh
    expr: sum(rate(slowquery_handle_duration_seconds_sum{status_code="200", path="/list"}[5m])) by (path) / sum(rate(slowquery_handle_duration_seconds_count{status_code="200", path="/list"}[5m])) by (path) > 15
    for: 3m
    labels:
      oncall_team: o11y
      severity: warning
    annotations:
      summary:  slowlogs list API latency high
      description: slowlogs list API latency high
      runbook_url: https://pingcap.feishu.cn/wiki/wikcnJ3TYvhozrRtWCMBh5z92Wd#share-NFl7dP5BPoytl9x2nClc9jpKnrd
  - alert: O11yIntegrationTaskFailed
    expr: observability_integration_task_info{status="FAILED"} > 0
    for: 5m
    labels:
      oncall_team: o11y
      severity: major
    annotations:
      message: O11y integration task created by github user {{ $labels.user_id }} for cluster {{ $labels.cluster_id }} of tenant {{ $labels.tenant_id }} failed (task {{ $labels.task_name }})
      runbook_url: https://pingcap.feishu.cn/wiki/C2beweHGsinehIkmKqkcT9BUn0b#share-ACEUd4RvvoVWEgxs0vVcaHFJnFh