groups:
- name: o11y-agent.rules
  rules:
  - alert: KubePodCrashLooping
    annotations:
      message: Pod {{ $labels.namespace }}/{{ $labels.pod }} ({{ $labels.container
        }}) is restarting {{ printf "%.2f" $value }} times in 30 mins.
      runbook_url: https://pingcap.feishu.cn/wiki/wikcnJ3TYvhozrRtWCMBh5z92Wd#doxcn0CqykEm486QiIfOTbvsQQh
    expr: |
      increase(kube_pod_container_status_restarts_total{pod=~".*victoria-metrics-agent.*|.*-vector-.*|.*vmagent.*"}[30m]) > 4
    for: 15m
    labels:
      component: o11y
      oncall_team: o11y
      severity: critical
      response_class: page

  - alert: VmagentVolumeStackingUp
    annotations:
      message: The PersistentVolume claimed by {{ $labels.persistentvolumeclaim
        }} in Namespace {{ $labels.namespace }} usage > 20%, it mean vmagent write to o11y is stacking up. check o11y status.
      runbook_url: https://pingcap.feishu.cn/wiki/wikcnJ3TYvhozrRtWCMBh5z92Wd#doxcnaliXrQV47CVK9Oeu5JNp7f
    expr: |
      kubelet_volume_stats_available_bytes{persistentvolumeclaim=~".*victoria-metrics-agent.*|.*vmagent.*"}
        /
      kubelet_volume_stats_capacity_bytes{persistentvolumeclaim=~".*victoria-metrics-agent.*|.*vmagent.*"}
        < 0.8
    for: 1m
    labels:
      component: vmagent
      oncall_team: o11y
      severity: critical
      response_class: page
  - alert: VmagentVolumeStackingUp
    annotations:
      message: The PersistentVolume claimed by {{ $labels.persistentvolumeclaim
        }} in Namespace {{ $labels.namespace }} usage > 10%, it mean vmagent write to o11y is stacking up. check o11y status.
      runbook_url: https://pingcap.feishu.cn/wiki/wikcnJ3TYvhozrRtWCMBh5z92Wd#doxcnaliXrQV47CVK9Oeu5JNp7f
    expr: |
      kubelet_volume_stats_available_bytes{persistentvolumeclaim=~".*victoria-metrics-agent.*|.*vmagent.*"}
        /
      kubelet_volume_stats_capacity_bytes{persistentvolumeclaim=~".*victoria-metrics-agent.*|.*vmagent.*"}
        < 0.9
    for: 1m
    labels:
      component: vmagent
      oncall_team: o11y
      severity: major
      response_class: ticket

  - alert: VMagentPersistentQueueIsDroppingData
    expr: sum(increase(vm_persistentqueue_bytes_dropped_total[5m])) by (job,cluster,namespace,instance) > 0
    for: 5m
    labels:
      severity: critical
      response_class: page
      component: vmagent
      oncall_team: o11y
    annotations:
      dashboard: "http://localhost:3000/d/G7Z9GzMGz?viewPanel=49&var-instance={{ $labels.instance }}"
      runbook_url: https://pingcap.feishu.cn/wiki/wikcnJ3TYvhozrRtWCMBh5z92Wd#doxcnSjpbzFMfnZd4t8M1BT1Ihe
      summary: "Instance {{ $labels.instance }} in namespace {{ $labels.namespace }} in cluster {{ $labels.cluster }} is dropping data from persistent queue"
      description: "Vmagent dropped {{ $value | humanize1024 }} from persistent queue
          on instance {{ $labels.instance }} for the last 10m."

  - alert: VMagentRejectedRemoteWriteDataBlocksAreDropped
    expr: sum(increase(vmagent_remotewrite_packets_dropped_total[5m])) by (job, instance, cluster) > 0
    for: 15m
    labels:
      severity: warning
      component: vmagent
      oncall_team: o11y
    annotations:
      dashboard: "http://localhost:3000/d/G7Z9GzMGz?viewPanel=79&var-instance={{ $labels.instance }}"
      summary: "Job \"{{ $labels.job }}\" on instance {{ $labels.instance }} in cluster {{ $labels.cluster }} drops the rejected by remote-write server data blocks. Check the logs to find the reason for rejects."

  - alert: VMagentTooManyWriteErrors
    expr: |
      (sum(increase(vm_ingestserver_request_errors_total[5m])) by (job, instance, cluster)
      +
      sum(increase(vmagent_http_request_errors_total[5m])) by (job, instance, cluster)) > 0
    for: 15m
    labels:
      severity: critical
      response_class: page
      component: vmagent
      oncall_team: o11y
    annotations:
      dashboard: "http://localhost:3000/d/G7Z9GzMGz?viewPanel=77&var-instance={{ $labels.instance }}"
      summary: "Job \"{{ $labels.job }}\" on instance {{ $labels.instance }} in cluster {{ $labels.cluster }}  namespace {{ $labels.namespace }} responds with errors to write requests for last 15m."
      runbook_url: https://pingcap.feishu.cn/wiki/wikcnJ3TYvhozrRtWCMBh5z92Wd#doxcnxEi1iru6F9ANoQ68Ojysfc

  - alert: VMagentTooManyRemoteWriteErrors
    expr: sum(rate(vmagent_remotewrite_retries_count_total[5m])) by(job, instance, url, cluster) > 5
    for: 15m
    labels:
      severity: critical
      response_class: page
      component: vmagent
      oncall_team: o11y
    annotations:
      dashboard: "http://localhost:3000/d/G7Z9GzMGz?viewPanel=61&var-instance={{ $labels.instance }}"
      summary: "Job \"{{ $labels.job }}\" on instance {{ $labels.instance }} in cluster {{ $labels.cluster }}  namespace {{ $labels.namespace }} fails to push to remote storage"
      description: "Vmagent fails to push data via remote write protocol to destination \"{{ $labels.url }}\"\n
        Ensure that destination is up and reachable."
      runbook_url: https://pingcap.feishu.cn/wiki/wikcnJ3TYvhozrRtWCMBh5z92Wd#doxcnxEi1iru6F9ANoQ68Ojysfc

  - alert: VMagentRemoteWriteConnectionIsSaturated
    expr: |
      sum(rate(vmagent_remotewrite_send_duration_seconds_total[5m])) by(job, instance, url, cluster)
      > 0.9 * max(vmagent_remotewrite_queues) by(job, instance, url, cluster)
    for: 15m
    labels:
      severity: warning
      component: vmagent
      oncall_team: o11y
    annotations:
      dashboard: "http://localhost:3000/d/G7Z9GzMGz?viewPanel=84&var-instance={{ $labels.instance }}"
      summary: "Remote write connection from \"{{ $labels.job }}\" (instance {{ $labels.instance }}) to {{ $labels.url }} is saturated"
      description: "The remote write connection between vmagent \"{{ $labels.job }}\" (instance {{ $labels.instance }}) and destination \"{{ $labels.url }}\"
        in cluster {{ $labels.cluster }} is saturated by more than 90% and vmagent won't be able to keep up.\n
        This usually means that `-remoteWrite.queues` command-line flag must be increased in order to increase
        the number of connections per each remote storage."

  - alert: VMagentPersistentQueueForWritesIsSaturated
    expr: rate(vm_persistentqueue_write_duration_seconds_total[5m]) > 0.9
    for: 15m
    labels:
      severity: warning
      component: vmagent
      oncall_team: o11y
    annotations:
      dashboard: "http://localhost:3000/d/G7Z9GzMGz?viewPanel=98&var-instance={{ $labels.instance }}"
      summary: "Persistent queue writes for instance {{ $labels.instance }} are saturated"
      description: "Persistent queue writes for vmagent \"{{ $labels.job }}\" (instance {{ $labels.instance }})
        in cluster {{ $labels.cluster }} are saturated by more than 90% and vmagent won't be able to keep up with flushing data on disk.
        In this case, consider to decrease load on the vmagent or improve the disk throughput."

  - alert: VMagentPersistentQueueForReadsIsSaturated
    expr: rate(vm_persistentqueue_read_duration_seconds_total[5m]) > 0.9
    for: 15m
    labels:
      severity: warning
      component: vmagent
      oncall_team: o11y
    annotations:
      dashboard: "http://localhost:3000/d/G7Z9GzMGz?viewPanel=99&var-instance={{ $labels.instance }}"
      summary: "Persistent queue reads for instance {{ $labels.instance }} are saturated"
      description: "Persistent queue reads for vmagent \"{{ $labels.job }}\" (instance {{ $labels.instance }})
        in cluster {{ $labels.cluster }} are saturated by more than 90% and vmagent won't be able to keep up with reading data from the disk.
        In this case, consider to decrease load on the vmagent or improve the disk throughput."

  - alert: VMagentTooManyScrapeTimeoutErrors
    expr: sum(increase(vm_promscrape_scrapes_timed_out_total{provider_type!~"aws-free-tier|alicloud-serverless"}[2m])) by (cluster, namespace, pod, instance) > 0
    for: 10m
    labels:
      severity: warning
      component: vmagent
      oncall_team: o11y
    annotations:
      summary: "vmagent {{ $labels.cluster }} {{ $labels.namespace }} {{ $labels.pod }} {{ $labels.instance }} fails to scrape targets for last 10m"
      runbook_url: https://github.com/tidbcloud/runbooks/wiki/VMagentTooManyScrapeErrors

  - alert: VMagentTooManyScrapeTimeoutErrors
    expr: sum(increase(vm_promscrape_scrapes_timed_out_total{provider_type=~"aws-free-tier|alicloud-serverless"}[1m])) by (cluster, namespace, pod, instance) > 1
    for: 30m
    labels:
      severity: warning
      component: vmagent
      oncall_team: o11y
    annotations:
      message: "vmagent {{ $labels.cluster }} {{ $labels.namespace }} {{ $labels.pod }} {{ $labels.instance }} fails to scrape targets for last 10m"
      runbook_url: https://github.com/tidbcloud/runbooks/wiki/VMagentTooManyScrapeErrors

  - alert: VMagentSeriesLimitHourReached
    expr: (vmagent_hourly_series_limit_current_series / vmagent_hourly_series_limit_max_series) > 0.9
    labels:
      severity: critical
      response_class: page
      component: vmagent
      oncall_team: o11y
    annotations:
      dashboard: "http://localhost:3000/d/G7Z9GzMGz?viewPanel=88&var-instance={{ $labels.instance }}"
      summary: "Instance {{ $labels.instance }} in cluster {{ $labels.cluster }} reached 90% of the limit"
      description: "Max series limit set via -remoteWrite.maxHourlySeries flag is close to reaching the max value.
        Then samples for new time series will be dropped instead of sending them to remote storage systems."
      runbook_url: https://github.com/tidbcloud/runbooks/wiki/VMagentSeriesLimitHourReached

  - alert: VMagentSeriesLimitDayReached
    expr: (vmagent_daily_series_limit_current_series / vmagent_daily_series_limit_max_series) > 0.9
    labels:
      severity: critical
      response_class: page
      component: vmagent
      oncall_team: o11y
    annotations:
      dashboard: "http://localhost:3000/d/G7Z9GzMGz?viewPanel=90&var-instance={{ $labels.instance }}"
      summary: "Instance {{ $labels.instance }} in cluster {{ $labels.cluster }} reached 90% of the limit"
      description: "Max series limit set via -remoteWrite.maxDailySeries flag is close to reaching the max value.
        Then samples for new time series will be dropped instead of sending them to remote storage systems."
      runbook_url: https://github.com/tidbcloud/runbooks/wiki/VMagentSeriesLimitHourReached

  - alert: VectorTooManySinkError
    expr: sum(increase(vector_component_errors_total{component_kind="sink"}[1m])) by (job, instance, cluster,component_type)   > 5
    for: 5m
    labels:
      severity: major
      response_class: ticket
      component: o11y
      oncall_team: o11y
    annotations:
      summary: "Vector Instance {{ $labels.instance }} in cluster {{ $labels.cluster }} have too many http errors"
      description: "Vector have too many http errors"
      runbook_url: https://pingcap.feishu.cn/wiki/wikcnJ3TYvhozrRtWCMBh5z92Wd#doxcnmDDgJXmr6UpB1V62hwIhVf

  - alert: VectorNoEventsSentToLoki
    expr: sum(increase(vector_component_sent_events_total{component_type="loki"}[30m])) by (job, instance, cluster, component_type) == 0 and sum(increase(vector_component_received_events_total{component_type="kubernetes_logs"}[30m])) by (job, instance, cluster, component_type) > 0
    for: 5m
    labels:
      severity: major
      response_class: ticket
      component: o11y
      oncall_team: o11y
    annotations:
      summary: "Vector Instance {{ $labels.instance }} in cluster {{ $labels.cluster }} has not sent any events to Loki in the past 30 minutes"
      description: "Vector has not sent any events to Loki in the past 30 minutes, which may indicate issues with log collection or transmission"
      runbook_url: https://pingcap.feishu.cn/wiki/wikcnJ3TYvhozrRtWCMBh5z92Wd#doxcnurCiJt3lEWCFoIfDe2K6Rf

  - alert: VectorNoEventsSentToCloudStorage
    expr: sum(increase(vector_component_sent_events_total{component_type=~"aws_s3|gcp_cloud_storage|azure_blob"}[30m])) by (job, instance, cluster, component_type) == 0 and sum(increase(vector_component_received_events_total{component_type="kubernetes_logs"}[30m])) by (job, instance, cluster, component_type) > 0
    for: 5m
    labels:
      severity: major
      response_class: ticket
      component: o11y
      oncall_team: o11y
    annotations:
      summary: "Vector Instance {{ $labels.instance }} in cluster {{ $labels.cluster }} has not sent any events to cloud storage in the past 30 minutes"
      description: "Vector has not sent any events to cloud storage in the past 30 minutes, which may indicate issues with log collection or transmission"
      runbook_url: https://pingcap.feishu.cn/wiki/wikcnJ3TYvhozrRtWCMBh5z92Wd#doxcnurCiJt3lEWCFoIfDe2K6Rf

  - alert: VectorEventsMoreThan8000
    expr: sum(rate(vector_component_received_events_count_sum{component_type="loki"}[1m])) by (job, instance, cluster,component_type) > 8000
    for: 5m
    labels:
      severity: warning
      component: o11y
      oncall_team: o11y
    annotations:
      description: 'Vector Instance {{ $labels.instance }} in cluster {{ $labels.cluster }} have too many events'
      summary: 'Vector Loki component receives more than 8000 events per second'
      runbook_url: https://pingcap.feishu.cn/wiki/wikcnJ3TYvhozrRtWCMBh5z92Wd#doxcnurCiJt3lEWCFoIfDe2K6Rf

  - alert: VectorTooManyEventsDiscarded
    expr: sum(increase(vector_component_discarded_events_total{component_type="loki"}[1m])) by (job, instance, cluster,component_type) > 1000
    for: 5m
    labels:
      severity: warning
      component: o11y
      oncall_team: o11y
    annotations:
      summary: "Vector Instance {{ $labels.instance }} in cluster {{ $labels.cluster }} have too many events"
      description: "Vector events have been discarded"
      runbook_url: https://pingcap.feishu.cn/wiki/wikcnJ3TYvhozrRtWCMBh5z92Wd#doxcnurCiJt3lEWCFoIfDe2K6Rf

  - alert: VectorTooManyEventOnBufferFor1h
    expr: vector_buffer_byte_size/vector_buffer_max_byte_size > 0.5 unless on (host) label_replace(node_namespace_pod:kube_pod_info:{pod=~"db-tiflash-.*"},"host","$1","node","(.*)")
    for: 60m
    labels:
      severity: warning
      component: o11y
      oncall_team: o11y
    annotations:
      summary: "Vector Instance {{ $labels.instance }} in cluster {{ $labels.cluster }} have too many event can't be sent"
      description: "Vector have too many event accumulation on buffer"
      runbook_url: https://pingcap.feishu.cn/wiki/wikcnJ3TYvhozrRtWCMBh5z92Wd#doxcnurCiJt3lEWCFoIfDe2K6Rf

  - alert: VectorBackpressureHangs
    expr: increase(vector_buffer_received_events_total{component_type="loki"}[1m]) == 0 and vector_buffer_byte_size{component_type="loki"}/vector_buffer_max_byte_size{component_type="loki"} > 0.8
    for: 15m
    labels:
      severity: major
      response_class: ticket
      component: o11y
      oncall_team: o11y
    annotations:
      summary: "Vector Instance {{ $labels.instance }} in cluster {{ $labels.cluster }} Backpressure high and hangs"
      description: "Vector may hangs at this time"
      runbook_url: https://pingcap.feishu.cn/wiki/wikcnJ3TYvhozrRtWCMBh5z92Wd#doxcnBwaIvod7VYajmAwhU9idYe

  - alert: VectorCPUThrottlingHigh
    expr: |-
      (
        sum(increase(container_cpu_cfs_throttled_periods_total{namespace=~"tidb.*", pod=~".*vector.*", provider_type!="aws-free-tier"}[5m])) by (cluster, pod, namespace)
          /
        sum(increase(container_cpu_cfs_periods_total{namespace=~"tidb.*", pod=~".*vector.*", provider_type!="aws-free-tier"}[5m])) by (cluster, pod, namespace)
          > 0.6
      )
      AND
      (
        sum(rate(container_cpu_usage_seconds_total{namespace=~"tidb.*", pod=~".*vector.*", container!="", provider_type!="aws-free-tier"}[5m])) by (cluster, pod, namespace)
          > 0.85
      )
    for: 30m
    labels:
      severity: major
      response_class: ticket
      component: o11y
      oncall_team: o11y
    annotations:
      message: "Vector Pod {{ $labels.namespace }}/{{ $labels.pod }} in cluster {{ $labels.cluster }} experience elevated CPU throttling, this may result in data loss"
      runbook_url: https://pingcap.feishu.cn/docx/doxcnIP9B4qQyNO92A02TEMNmKg#doxcnYx945y9mkRNUBHClwjCxNf

  - alert: VMagentPodNotReadyServerless
    expr: |
      (
        (kube_pod_status_phase{phase=~"Pending|Unknown",pod=~".*vmagent.*",provider_type="aws-free-tier"} == 1)
          * on(namespace, pod) group_left(node) kube_pod_info{pod=~".*vmagent.*"}
        and on(node) kube_node_status_condition{condition="Ready", status="true"} == 1
      )
      or
      (
        (kube_pod_status_phase{phase=~"Pending|Unknown",pod=~".*vmagent.*",provider_type="aws-free-tier"} == 1)
          * on(namespace, pod) group_left(node) kube_pod_info{pod=~".*vmagent.*",node=""}
      )
    for: 10m
    labels:
      severity: major
      response_class: ticket
      component: vmagent
      oncall_team: o11y
    annotations:
      summary: "vmagent pod {{ $labels.pod }} in cluster {{ $labels.cluster }} namespace {{ $labels.namespace }} pending too long"
      description: "vmagent pod pending too long"
      runbook_url: https://pingcap.feishu.cn/wiki/wikcnJ3TYvhozrRtWCMBh5z92Wd#doxcn8WdfEAoMuTAJJWso9KF9zg

  - alert: VectorPodNotReadyServerless
    expr: |
      (
        (kube_pod_status_phase{phase=~"Pending|Unknown",pod=~".*-vector-.*",provider_type="aws-free-tier"} == 1)
          * on(namespace, pod) group_left(node) kube_pod_info{pod=~".*-vector-.*"}
        and on(node) kube_node_status_condition{condition="Ready", status="true"} == 1
      )
      or
      (
        (kube_pod_status_phase{phase=~"Pending|Unknown",pod=~".*-vector-.*",provider_type="aws-free-tier"} == 1)
          * on(namespace, pod) group_left(node) kube_pod_info{pod=~".*-vector-.*",node=""}
      )
    for: 10m
    labels:
      severity: major
      response_class: ticket
      component: o11y
      oncall_team: o11y
    annotations:
      summary: "vector pod {{ $labels.pod }} in cluster {{ $labels.cluster }} namespace {{ $labels.namespace }} pending too long"
      description: "vector pod pending too long"
      runbook_url: https://pingcap.feishu.cn/wiki/wikcnJ3TYvhozrRtWCMBh5z92Wd#doxcnC0u03MpYL2KHMltIwfovah

  - alert: VMagentPodNotReady
    expr: |
      (
        (kube_pod_status_phase{phase=~"Pending|Unknown",pod=~".*vmagent.*",provider_type!="aws-free-tier"} == 1)
          * on(namespace, pod) group_left(node) kube_pod_info{pod=~".*vmagent.*"}
        and on(node) kube_node_status_condition{condition="Ready", status="true"} == 1
      )
      or
      (
        (kube_pod_status_phase{phase=~"Pending|Unknown",pod=~".*vmagent.*",provider_type!="aws-free-tier"} == 1)
          * on(namespace, pod) group_left(node) kube_pod_info{pod=~".*vmagent.*",node=""}
      )
    for: 5m
    labels:
      severity: major
      response_class: ticket
      component: vmagent
      oncall_team: o11y
    annotations:
      summary: "vmagent pod {{ $labels.pod }} in cluster {{ $labels.cluster }} namespace {{ $labels.namespace }} pending too long"
      description: "vmagent pod pending too long"
      runbook_url: https://pingcap.feishu.cn/wiki/wikcnJ3TYvhozrRtWCMBh5z92Wd#doxcn8WdfEAoMuTAJJWso9KF9zg

  - alert: VMagentPodNotReadyCritical
    expr: |
      (
        (kube_pod_status_phase{phase=~"Pending|Unknown",pod=~".*vmagent.*",provider_type!="aws-free-tier"} == 1)
          * on(namespace, pod) group_left(node) kube_pod_info{pod=~".*vmagent.*"}
        and on(node) kube_node_status_condition{condition="Ready", status="true"} == 1
      )
      or
      (
        (kube_pod_status_phase{phase=~"Pending|Unknown",pod=~".*vmagent.*",provider_type!="aws-free-tier"} == 1)
          * on(namespace, pod) group_left(node) kube_pod_info{pod=~".*vmagent.*",node=""}
      )
    for: 10m
    labels:
      severity: critical
      response_class: page
      component: vmagent
      oncall_team: o11y
    annotations:
      summary: "vmagent pod {{ $labels.pod }} in cluster {{ $labels.cluster }} namespace {{ $labels.namespace }} pending too long"
      description: "vmagent pod pending too long"
      runbook_url: https://pingcap.feishu.cn/wiki/wikcnJ3TYvhozrRtWCMBh5z92Wd#doxcn8WdfEAoMuTAJJWso9KF9zg

  - alert: VectorPodNotReady
    expr: |
      (
        (kube_pod_status_phase{phase=~"Pending|Unknown",pod=~".*-vector-.*",provider_type!="aws-free-tier"} == 1)
          * on(namespace, pod) group_left(node) kube_pod_info{pod=~".*-vector-.*"}
        and on(node) kube_node_status_condition{condition="Ready", status="true"} == 1
      )
      or
      (
        (kube_pod_status_phase{phase=~"Pending|Unknown",pod=~".*-vector-.*",provider_type!="aws-free-tier"} == 1)
          * on(namespace, pod) group_left(node) kube_pod_info{pod=~".*-vector-.*",node=""}
      )
    for: 5m
    labels:
      severity: major
      response_class: ticket
      component: o11y
      oncall_team: o11y
    annotations:
      summary: "vector pod {{ $labels.pod }} in cluster {{ $labels.cluster }} namespace {{ $labels.namespace }} pending too long"
      description: "vector pod pending too long"
      runbook_url: https://pingcap.feishu.cn/wiki/wikcnJ3TYvhozrRtWCMBh5z92Wd#doxcnC0u03MpYL2KHMltIwfovah

  - alert: VectorPodNotReadyCritical
    expr: |
      (
        (kube_pod_status_phase{phase=~"Pending|Unknown",pod=~".*-vector-.*",provider_type!="aws-free-tier"} == 1)
          * on(namespace, pod) group_left(node) kube_pod_info{pod=~".*-vector-.*"}
        and on(node) kube_node_status_condition{condition="Ready", status="true"} == 1
      )
      or
      (
        (kube_pod_status_phase{phase=~"Pending|Unknown",pod=~".*-vector-.*",provider_type!="aws-free-tier"} == 1)
          * on(namespace, pod) group_left(node) kube_pod_info{pod=~".*-vector-.*",node=""}
      )
    for: 10m
    labels:
      severity: critical
      response_class: page
      component: o11y
      oncall_team: o11y
    annotations:
      summary: "vector pod {{ $labels.pod }} in cluster {{ $labels.cluster }} namespace {{ $labels.namespace }} pending too long"
      description: "vector pod pending too long"
      runbook_url: https://pingcap.feishu.cn/wiki/wikcnJ3TYvhozrRtWCMBh5z92Wd#doxcnC0u03MpYL2KHMltIwfovah

  - alert: O11yAgentControllerNoAvailableReplicas
    expr: kube_deployment_status_replicas_available{deployment="o11y-agent-control-server"}==0
    for: 10m
    labels:
      component: o11y
      oncall_team: o11y
      severity: critical
      response_class: page
    annotations:
      summary: "o11y agent control server in cluster {{ $labels.cluster }} namespace {{ $labels.namespace }} status is not available"
      description: There are no running o11y agent  controller pods.
      runbook_url: https://pingcap.feishu.cn/wiki/wikcnJ3TYvhozrRtWCMBh5z92Wd#doxcnh7b7KVKRBilZhceOoCatxe

  - alert: O11yAgentControllerAgentNotNormal
    expr: increase(acs_agent_status{namespace=~"observability|regional-server-system",status!~"Normal|Hibernated"}[1m]) > 0
    for: 30m
    labels:
      component: o11y
      oncall_team: o11y
      severity: major
      response_class: ticket
    annotations:
      summary: "agents managed by o11y agent controller in cluster {{ $labels.cluster }} namespace {{ $labels.namespace }} report non-Normal status"
      description: "{{ $labels.agent }} agent of application {{ $labels.application_id }} in cluster {{ $labels.k8s_cluster_name }} reports status '{{ $labels.status }}' (not Normal). Check agent health, configuration, and controller connectivity/logs."

  - alert: O11yAgentControllerReachMaxSpec
    expr: increase(acs_scaled_resource_total{action="maxspec",provider_type!="aws-free-tier"}[1m]) > 0
    labels:
      component: o11y
      oncall_team: o11y
      severity: major
      response_class: ticket
    annotations:
      summary: "auto-scaled resource in cluster {{ $labels.cluster }} reached max spec, cannot scale further"
      description: "The controller requested a larger spec for {{ $labels.namespace }}/{{ $labels.pod }} ({{ $labels.agent }}) in cluster {{ $labels.k8s_cluster_name }}, but it has already reached the configured maxspec limit. Consider increasing the maxspec threshold if this scale-up is expected, or investigate the workload driving the request."

  - alert: O11yAgentControllerVPCEndpointNotReady
    expr: increase(acs_vpc_endpoint_status{namespace=~"observability|regional-server-system",status!="Ready",vpc_endpoint_type!="Gateway"}[1m]) > 0
    for: 10m
    labels:
      component: o11y
      oncall_team: o11y
      severity: major
      response_class: ticket
    annotations:
      summary: "vpcendpoint managed by o11y agent controller in cluster {{ $labels.cluster }} namespace {{ $labels.namespace }} report non-Ready status"
      description: "vpcendpoints report status '{{ $labels.status }}' (not Ready). Check vpcendpoint health, configuration, and controller connectivity/logs."

  - alert: RapidSeedClusterExpansion
    expr: >
      (
        (
          sum by (o11y_cluster_info) (acs_nodegroup_desired_node_count) > 400
        )
        and
        (
          sum by (o11y_cluster_info) (acs_nodegroup_desired_node_count)
          /
          sum by (o11y_cluster_info) (acs_nodegroup_desired_node_count offset 15m) > 1.25
        )
      ) or (
        (
          sum by (o11y_cluster_info) (acs_nodegroup_desired_node_count) > 100
          and
          sum by (o11y_cluster_info) (acs_nodegroup_desired_node_count) <= 400
        )
        and
        (
          sum by (o11y_cluster_info) (acs_nodegroup_desired_node_count)
          /
          sum by (o11y_cluster_info) (acs_nodegroup_desired_node_count offset 15m) > 1.50
        )
      )
    labels:
      severity: major
      response_class: ticket
      component: o11y
      oncall_team: o11y
    annotations:
      summary: "Rapid scaling detected for seed cluster {{ $labels.o11y_cluster_info }}"
      description: >
        Seed cluster {{ $labels.o11y_cluster_info }} is scaling rapidly.
        It currently desires {{ $value }} nodes. This triggered an alert because it either
        exceeded 400 nodes with >25% growth, or is between 100-400 nodes with >50% growth
        over the last 15 minutes.
  - alert: VmagentOOMKilled
    expr: increase(kube_pod_container_status_restarts_total{pod=~".*victoria-metrics-agent.*|.*vmagent.*"}[5m]) >= 1 AND ignoring(reason) kube_pod_container_status_last_terminated_reason{reason="OOMKilled",pod=~".*victoria-metrics-agent.*|.*vmagent.*"} == 1
    for: 3m
    labels:
     component: o11y
     oncall_team: o11y
     severity: major
     response_class: ticket
    annotations:
      description: "Vmagent  in {{ $labels.cluster }} has been OOMKilled {{ $value }} times in the last 5 minutes.\n LABELS = {{ $labels }}"
      runbook_url: https://pingcap.feishu.cn/wiki/wikcnJ3TYvhozrRtWCMBh5z92Wd#doxcnPwy7lxnTHBfCojbDAMftVd

  - alert: VmagentFrequentOOMKilled
    expr: increase(kube_pod_container_status_restarts_total{pod=~".*victoria-metrics-agent.*|.*vmagent.*"}[30m]) >= 3 AND ignoring(reason) kube_pod_container_status_last_terminated_reason{reason="OOMKilled",pod=~".*victoria-metrics-agent.*|.*vmagent.*"} == 1
    for: 3m
    labels:
     component: o11y
     oncall_team: o11y
     severity: critical
     response_class: page
    annotations:
      description: "Vmagent  in {{ $labels.cluster }} has been OOMKilled {{ $value }} times in the last 30 minutes.\n LABELS = {{ $labels }}"
      runbook_url: https://pingcap.feishu.cn/wiki/wikcnJ3TYvhozrRtWCMBh5z92Wd#doxcnPwy7lxnTHBfCojbDAMftVd

  - alert: VectorOOMKilled
    expr: increase(kube_pod_container_status_restarts_total{pod=~".*-vector-.*",namespace=~"monitoring|logging"}[5m]) >= 2 AND ignoring(reason) kube_pod_container_status_last_terminated_reason{reason="OOMKilled",pod=~".*-vector-.*",namespace="monitoring"} == 1
    for: 3m
    labels:
     component: o11y
     oncall_team: o11y
     severity: major
     response_class: ticket
    annotations:
      description: "Vector in {{ $labels.cluster }} has been OOMKilled {{ $value }} times in the last 5 minutes.\n LABELS = {{ $labels }}"
      runbook_url: https://pingcap.feishu.cn/wiki/wikcnJ3TYvhozrRtWCMBh5z92Wd#doxcnPwy7lxnTHBfCojbDAMftVd

  - alert: ClusterVectorOOMKilled
    expr: increase(kube_pod_container_status_restarts_total{pod=~".*-vector-.*",namespace!~"monitoring|logging"}[5m]) >= 1 AND ignoring(reason) kube_pod_container_status_last_terminated_reason{reason="OOMKilled",pod=~".*-vector-.*",namespace!="monitoring"} == 1
    for: 3m
    labels:
     component: o11y
     oncall_team: o11y
     severity: major
     response_class: ticket
    annotations:
      description: "CLuster Vector in {{ $labels.cluster }} has been OOMKilled {{ $value }} times in the last 5 minutes.\n LABELS = {{ $labels }}"
      runbook_url: https://pingcap.feishu.cn/wiki/wikcnJ3TYvhozrRtWCMBh5z92Wd#doxcnPwy7lxnTHBfCojbDAMftVd

  - alert: ClusterVectorFrequentOOMKilled
    expr: increase(kube_pod_container_status_restarts_total{pod=~".*-vector-.*",namespace!~"monitoring|logging"}[30m]) >= 3 AND ignoring(reason) kube_pod_container_status_last_terminated_reason{reason="OOMKilled",pod=~".*-vector-.*",namespace!="monitoring"} == 1
    for: 3m
    labels:
     component: o11y
     oncall_team: o11y
     severity: critical
     response_class: page
    annotations:
      description: "CLuster Vector in {{ $labels.cluster }} has been OOMKilled {{ $value }} times in the last 30 minutes.\n LABELS = {{ $labels }}"
      runbook_url: https://pingcap.feishu.cn/wiki/wikcnJ3TYvhozrRtWCMBh5z92Wd#doxcnPwy7lxnTHBfCojbDAMftVd