groups:
  - name: mgmt.go-runtime
    rules:
      - alert: GoroutineGrowth
        annotations:
          message: Goroutines growing {{ printf "%.2f" $value }} /s pod={{ $labels.pod }}
        expr: |
          clamp_min(
            deriv(go_goroutines{namespace="tidb-management-service"}[15m]),
            0
          ) > 0.5
        for: 10m
        labels:
          severity: major
          response_class: ticket
          component: tidb-management-service
          oncall_team: global_control

      - alert: GoroutinesHigh
        annotations:
          message: Goroutines avg 5m {{ printf "%.0f" $value }} pod={{ $labels.pod }}
        expr: |
          avg_over_time(go_goroutines{namespace="tidb-management-service"}[5m])
            > 1500
        for: 10m
        labels:
          severity: major
          response_class: ticket
          component: tidb-management-service
          oncall_team: global_control

      - alert: GoroutinesVeryHigh
        annotations:
          message: Goroutines avg 5m {{ printf "%.0f" $value }} pod={{ $labels.pod }}
        expr: |
          avg_over_time(go_goroutines{namespace="tidb-management-service"}[5m])
            > 3000
        for: 5m
        labels:
          severity: critical
          response_class: page
          component: tidb-management-service
          oncall_team: global_control

      - alert: GCPauseP99High
        annotations:
          message: GC p99 pause {{ printf "%.0f" $value }} ms pod={{ $labels.pod }}
        expr: |
          max by (pod) (
            1000 * go_gc_duration_seconds{namespace="tidb-management-service", quantile="0.99"}
          ) > 200
        for: 5m
        labels:
          severity: major
          response_class: ticket
          component: tidb-management-service
          oncall_team: global_control

      - alert: GCCPUHigh
        annotations:
          message: GC CPU fraction {{ printf "%.2f" $value }} pod={{ $labels.pod }}
        expr: |
          avg by (pod) (
            go_memstats_gc_cpu_fraction{namespace="tidb-management-service"}
          ) > 0.5
        for: 10m
        labels:
          severity: major
          response_class: ticket
          component: tidb-management-service
          oncall_team: global_control

      - alert: HeapGrowth
        annotations:
          message: Heap growth {{ printf "%.2f" $value }} MiB/s pod={{ $labels.pod }}
        expr: |
          (
            clamp_min(
              deriv(go_memstats_heap_inuse_bytes{namespace="tidb-management-service"}[30m]),
              0
            ) / 1024 / 1024
          ) > 0.095
        for: 15m
        labels:
          severity: major
          response_class: ticket
          component: tidb-management-service
          oncall_team: global_control

      - alert: GoThreadsHigh
        annotations:
          message: go_threads avg 10m {{ printf "%.0f" $value }} pod={{ $labels.pod }}
        expr: |
          avg_over_time(go_threads{namespace="tidb-management-service"}[10m]) > 512
        for: 10m
        labels:
          severity: major
          response_class: ticket
          component: tidb-management-service
          oncall_team: global_control

      - alert: FDUtilHigh
        annotations:
          message: FD utilization {{ printf "%.0f" $value }}% pod={{ $labels.pod }}
        expr: |
          max by (pod) (
            100 * process_open_fds{namespace="tidb-management-service"}
            /
            process_max_fds{namespace="tidb-management-service"}
          ) > 85
        for: 10m
        labels:
          severity: major
          response_class: ticket
          component: tidb-management-service
          oncall_team: global_control

      - alert: ProcessCPUHigh
        annotations:
          message: CPU {{ printf "%.2f" $value }} cores pod={{ $labels.pod }}
        expr: |
          avg by (pod) (
            rate(process_cpu_seconds_total{namespace="tidb-management-service"}[5m])
          ) > 0.9
        for: 10m
        labels:
          severity: major
          response_class: ticket
          component: tidb-management-service
          oncall_team: global_control