apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
  labels:
    release: self-monitoring
  name: self-monitoring-o11y-business
  namespace: {{ .Values.namespace.selfMonitoring }}
spec:
  groups:
  - name: self-monitoring-o11y-business
    rules:
    - alert: O11yClusterCreatingTooLong
      annotations:
        description: The clusters name {{`{{ $labels.cluster_id }}`}}  status in creating more than 5m
        summary: o11y cluster creating too long.
        runbook_url: https://pingcap.feishu.cn/wiki/wikcnJ3TYvhozrRtWCMBh5z92Wd#doxcnim8uEMQ4wm2mkd7SZKpCvf
      expr: |-
        sum(observability_cluster_info{status="CREATING"}) by(cluster_id) >=1
      for: 5m
      labels:
        severity: critical
    - alert: O11yClusterPendingDeletionTooLong
      annotations:
        message: The cluster named {{`{{ $labels.cluster_id }}`}} pending in "TO_BE_DELETED" status for more than 1h.
        runbook_url: https://pingcap.feishu.cn/wiki/wikcnJ3TYvhozrRtWCMBh5z92Wd#doxcnFCSD4vcyEY3NQLoHcrHHAg
      expr: |-
        sum(observability_cluster_info{status="TO_BE_DELETED"}) by(cluster_id) >=1
      for: 1h
      labels:
        severity: major
    - alert: O11yServiceHttpRequestCode5xx
      annotations:
        description: The service name {{`{{ $labels.service }}`}}  handler {{`{{ $labels.handler }}`}} 5xx code is {{`{{ $labels.code }}`}}
        summary: o11y service 5xx code.
        runbook_url: https://pingcap.feishu.cn/wiki/wikcnJ3TYvhozrRtWCMBh5z92Wd#doxcnVqvp1omXOTH2ob6g8x9eoN
      expr: |-
        sum(increase(observability_http_requests_total{code=~"5[0-9]{2}"}[1m])) by(handler,service,code) > 10
      for: 5m
      labels:
        severity: major
    - alert: PrometheusTsdbWalCorruptions
      expr: increase(prometheus_tsdb_wal_corruptions_total[1m]) > 0
      for: 0m
      labels:
        severity: critical
        runbook_url: https://pingcap.feishu.cn/wiki/wikcnJ3TYvhozrRtWCMBh5z92Wd#doxcn87vNFAWoNXxuVjILE2GM0O
      annotations:
        summary: Prometheus TSDB WAL corruptions (instance {{`{{ $labels.instance }}`}})
        description: "Prometheus encountered {{`{{ $value }}`}} TSDB WAL corruptions\n  VALUE = {{`{{ $value }}`}}\n  LABELS = {{`{{ $labels }}`}}"
    - alert: PrometheusTsdbWalTruncationsFailed
      expr: increase(prometheus_tsdb_wal_truncations_failed_total[1m]) > 0
      for: 0m
      labels:
        severity: critical
      annotations:
        summary: Prometheus TSDB WAL truncations failed (instance {{`{{ $labels.instance }}`}})
        description: "Prometheus encountered {{`{{ $value }}`}} TSDB WAL truncation failures\n  VALUE = {{`{{ $value }}`}}\n  LABELS = {{`{{ $labels }}`}}"
        runbook_url: https://pingcap.feishu.cn/wiki/wikcnJ3TYvhozrRtWCMBh5z92Wd#doxcn87vNFAWoNXxuVjILE2GM0O
    - alert: O11yAWSAPIRequestError
      expr: increase(observability_aws_api_request_total[1m]) > 5
      for: 0m
      labels:
        severity: warning
      annotations:
        summary: O11y AWS API request failed (error {{`{{ $labels.error_msg }}`}})
        description: "O11y AWS API request failed, client = {{`{{ $labels.aws_client }}`}}, api = {{`{{ $labels.aws_api }}`}}, resource_name = {{`{{ $labels.resource_name }}`}}, error_msg = {{`{{ $labels.error_msg }}`}}"
        runbook_url: https://pingcap.feishu.cn/wiki/wikcnJ3TYvhozrRtWCMBh5z92Wd#doxcne2284Kq46IasVRspO0dMi8
    - alert: O11yGCPAPIRequestError
      expr: increase(observability_gcp_api_request_total[1m]) > 5
      for: 0m
      labels:
        severity: warning
      annotations:
        summary: O11y GCP API request failed (error {{`{{ $labels.error_msg }}`}})
        description: "O11y GCP API request failed, client = {{`{{ $labels.gcp_client }}`}}, api = {{`{{ $labels.gcp_api }}`}}, resource_name = {{`{{ $labels.resource_name }}`}}, error_msg = {{`{{ $labels.error_msg }}`}}"
        runbook_url: https://pingcap.feishu.cn/wiki/wikcnJ3TYvhozrRtWCMBh5z92Wd#doxcnMy22UioMSAOGA5K7V82l9f