groups:
- name: node-exporter.rules
  rules:
  - alert: NodeHighCPUUsageThan85For15m
    expr: |
      # for default nodes
      label_replace((100 - (avg by(instance) (irate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 85), "node","$1","instance", "(.+)") * on (node) group_right() kube_node_labels{label_serverless_tidbcloud_com_node!~"burstable|critical-services|tidb|tikv|tiflash-wn|tmp-worker|tiflash-cn"}
    for: 15m
    labels:
      component: node
      oncall_team: serverless
      severity: warning
    annotations:
      runbook_url: https://docs.google.com/document/d/17dQQS3ONxhqdlqKth9kZHTtpANp49UgUBF_dMk5ewCE/edit#heading=h.5l5xttfiy56o
      message: nodegroup {{ $labels.label_serverless_tidbcloud_com_nodegroup }}, node {{ $labels.node }} CPU load is {{ printf "%.2f" $value }}%, larger than 85% for 15m.
  - alert: CriticalServiceNodeHighCPUUsageThan70For10m
    expr: |
      # for CPU sensitive nodes
      label_replace((100 - (avg by(instance) (irate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 70), "node","$1","instance", "(.+)") * on (node) group_right() kube_node_labels{label_serverless_tidbcloud_com_node=~"critical-services|tidb|tikv"}
    for: 10m
    labels:
      component: node
      oncall_team: serverless
      severity: warning
  - alert: CriticalServiceNodeHighCPUUsageThan80For10m
    expr: |
      # for CPU sensitive nodes
      label_replace((100 - (avg by(instance) (irate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 80), "node","$1","instance", "(.+)") * on (node) group_right() kube_node_labels{label_serverless_tidbcloud_com_node=~"critical-services|tidb" }
    for: 15m
    labels:
      component: node
      oncall_team: serverless
      severity: critical
      response_class: page
    annotations:
      runbook_url: https://docs.google.com/document/d/17dQQS3ONxhqdlqKth9kZHTtpANp49UgUBF_dMk5ewCE/edit#heading=h.5l5xttfiy56o
      message: "nodegroup {{ $labels.label_serverless_tidbcloud_com_nodegroup }}, node {{ $labels.node }} CPU load is {{ $value }}%, larger than 80% for 15m."
  - alert: TikvStandardNodeHighCPUUsageThan80For10m
    expr: |
      # for CPU sensitive nodes
      label_replace((100 - (avg by(instance) (irate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 80), "node","$1","instance", "(.+)") * on (node) group_right() kube_node_labels{label_serverless_tidbcloud_com_node="tikv", label_serverless_tidbcloud_com_tier="standard"}
    for: 15m
    labels:
      component: tikv
      oncall_team: tikv_unified
      severity: critical
      response_class: page
    annotations:
      runbook_url: https://docs.google.com/document/d/17dQQS3ONxhqdlqKth9kZHTtpANp49UgUBF_dMk5ewCE/edit#heading=h.5l5xttfiy56o
      message: "nodegroup {{ $labels.label_serverless_tidbcloud_com_nodegroup }}, node {{ $labels.node }} CPU load is {{ $value }}%, larger than 80% for 15m."
  - alert: TikvNonStandardNodeHighCPUUsageThan90For10m
    expr: |
      # for CPU sensitive nodes
      label_replace((100 - (avg by(instance) (irate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 90), "node","$1","instance", "(.+)") * on (node) group_right() kube_node_labels{label_serverless_tidbcloud_com_node="tikv", label_serverless_tidbcloud_com_tier!="standard"}
    for: 15m
    labels:
      component: tikv
      oncall_team: tikv_unified
      severity: critical
      response_class: page
    annotations:
      runbook_url: https://docs.google.com/document/d/17dQQS3ONxhqdlqKth9kZHTtpANp49UgUBF_dMk5ewCE/edit#heading=h.5l5xttfiy56o
      message: "nodegroup {{ $labels.label_serverless_tidbcloud_com_nodegroup }}, node {{ $labels.node }} CPU load is {{ $value }}%, larger than 90% for 15m."
  - alert: NodeHighCPUUsageThan90For15m
    expr: |
      # for CPU low-sensitive nodes
      label_replace((100 - (avg by(instance) (irate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 90), "node","$1","instance", "(.+)") * on (node) group_right() kube_node_labels{label_serverless_tidbcloud_com_node=~"burstable|tmp-worker|tiflash-cn"}
    for: 15m
    labels:
      component: node
      oncall_team: serverless
      severity: warning
    annotations:
      runbook_url: https://docs.google.com/document/d/17dQQS3ONxhqdlqKth9kZHTtpANp49UgUBF_dMk5ewCE/edit#heading=h.5l5xttfiy56o
      message: "nodegroup {{ $labels.label_serverless_tidbcloud_com_nodegroup }}, node {{ $labels.node }} CPU load is {{ $value }}%, larger than 90% for 15m."
  - alert: NodeHighCPUUsageThan90For30m
    expr: |
      # for TiFlash write nodes
      label_replace((100 - (avg by(instance) (irate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 90), "node","$1","instance", "(.+)") * on (node) group_right() kube_node_labels{label_serverless_tidbcloud_com_node=~"tiflash-wn"}
    for: 30m
    labels:
      component: node
      oncall_team: serverless
      severity: critical
      response_class: page
    annotations:
      runbook_url: https://docs.google.com/document/d/17dQQS3ONxhqdlqKth9kZHTtpANp49UgUBF_dMk5ewCE/edit#heading=h.5l5xttfiy56o
      message: "nodegroup {{ $labels.label_serverless_tidbcloud_com_nodegroup }}, node {{ $labels.node }} CPU load is {{ $value }}%, larger than 90% for 30m."
  - alert: NodesIncreaseTooQuick
    expr: |
      (count(:node_cpu_idle:kube_node_labels) by (label_serverless_tidbcloud_com_node)-count(:node_cpu_idle:kube_node_labels offset 1h) by (label_serverless_tidbcloud_com_node)) > 5
      or
      (count(:node_cpu_idle:kube_node_labels) by (label_serverless_tidbcloud_com_node)-count(:node_cpu_idle:kube_node_labels offset 1d) by (label_serverless_tidbcloud_com_node)) > 5
    for: 1h
    labels:
      severity: warning
      provider_type: aws-free-tier
      tier: serverless
      component: node
      oncall_team: serverless
    annotations:
      message: "The component {{ $labels.label_serverless_tidbcloud_com_node }} has {{ $value }} increased nodes. [Clinic Dashboard](https://clinic.pingcap.com/grafana/d/KmTtYdBNk/serverless-resource-usage?orgId=1&var-control_plane_info=ap-northeast-1-a01%2Falicloud-ap-northeast-1%2Fdevtier-infra)"