groups:
- name: tikv-worker
  rules:
  - alert: TiKVWorkerDowntimeIsTooLong
    expr: up{component=~"tikv-worker|coprocessor-worker", job=~".*tikv-worker.*|.*coprocessor-worker.*"} == 0
    for: 10m
    labels:
      tier: next-gen
      severity: critical
      response_class: page
      component: tikv
      oncall_team: tikv_unified
    annotations:
      description: 'TiKV worker {{ $labels.instance }} in cluster {{ $labels.cluster_id }} has been down for more than 10 minutes. This affects coprocessor operations and analytical query processing. Please immediately check and restart the TiKV worker to restore analytical processing functionality. cluster: {{ $labels.cluster_id }}, instance: {{ $labels.instance }}, values: {{ $value }}'
      runbook_url: https://docs.google.com/document/d/17dQQS3ONxhqdlqKth9kZHTtpANp49UgUBF_dMk5ewCE/edit#heading=h.t0ws8abzblbu
      value: '{{ $value }}'
      summary: TiKV worker downtime is more than 10 min
  - alert: TiKVWorkerFrequentlyRestart
    expr: |-
      sum by (cluster_id, instance, tenant, provider_type) (
          resets((-floor(process_start_time_seconds{job=~".*tikv-worker.*|.*coprocessor-worker.*"}))[1h:15s])
        and
          max_over_time(process_start_time_seconds{job=~".*tikv-worker.*|.*coprocessor-worker.*"}[30m] offset 1h) > 0
      ) >= 3
    for: 1m
    labels:
      tier: next-gen
      severity: critical
      response_class: page
      component: tikv
      oncall_team: tikv_unified
    annotations:
      description: 'Multiple TiKV worker instances in cluster {{ $labels.cluster_id }} have restarted {{ $value }} times in the past hour. This indicates potential cluster-wide issues affecting TiKV worker stability. Please investigate the cluster infrastructure and TiKV worker configuration to prevent further restarts. instance: {{ $labels.instance }}'
      runbook_url: https://docs.google.com/document/d/17dQQS3ONxhqdlqKth9kZHTtpANp49UgUBF_dMk5ewCE/edit#heading=h.t0ws8abzblbu
      value: '{{ $value }}'
      summary: TiKV worker is down for at least 3 times in a hour
  - alert: TiKVWorkerFrequentlyRestart
    expr: |-
      (
        sum by (cluster_id, tenant, provider_type) (
            resets((-floor(process_start_time_seconds{job=~".*tikv-worker.*|.*coprocessor-worker.*"}))[1h:15s])
          and
            max_over_time(process_start_time_seconds{job=~".*tikv-worker.*|.*coprocessor-worker.*"}[30m] offset 1h) > 0
        ) >= 3
      )
      and
      (
        sum(
          (
            sum(
              resets((-floor(process_start_time_seconds{job=~".*tikv-worker.*|.*coprocessor-worker.*"}))[1h:15s])
            ) by (cluster_id, instance, tenant, provider_type) >= 2
          )
        ) by (cluster_id, tenant, provider_type) > 0
      )
    for: 1m
    labels:
      tier: next-gen
      severity: critical
      response_class: page
      component: tikv
      oncall_team: tikv_unified
    annotations:
      description: 'Multiple TiKV worker instances in cluster {{ $labels.cluster_id }} have restarted {{ $value }} times in the past hour. This indicates potential cluster-wide issues affecting TiKV worker stability. Please investigate the cluster infrastructure and TiKV worker configuration to prevent further restarts. instance: {{ $labels.instance }}'
      runbook_url: https://docs.google.com/document/d/17dQQS3ONxhqdlqKth9kZHTtpANp49UgUBF_dMk5ewCE/edit#heading=h.t0ws8abzblbu
      value: '{{ $value }}'
      summary: TiKV worker is down for at least 3 times in a hour
  - alert: TiKVWorkerRestarted
    expr: sum(resets((-floor(process_start_time_seconds{job=~".*tikv-worker.*|.*coprocessor-worker.*"}))[1h:15s])) by (cluster_id, instance, tenant, provider_type) >= 1 and sum(max_over_time(up{job=~".*tikv-worker.*|.*coprocessor-worker.*"}[100m] offset 60m)) by (cluster_id, instance, tenant, provider_type) > 0
    for: 10m
    labels:
      tier: next-gen
      severity: warning
      component: tikv
      oncall_team: tikv_unified
    annotations:
      description: 'TiKV worker {{ $labels.instance }} in cluster {{ $labels.cluster_id }} has restarted {{ $value }} time(s) in the past hour. Please monitor the TiKV worker for stability and investigate if restarts become frequent. cluster: {{ $labels.cluster_id }}, instance: {{ $labels.instance }}, values: {{ $value }}'
      runbook_url: https://docs.google.com/document/d/17dQQS3ONxhqdlqKth9kZHTtpANp49UgUBF_dMk5ewCE/edit#heading=h.t0ws8abzblbu
      value: '{{ $value }}'
      summary: TiKV worker is down for at least 1 times in a hour