groups:
  - name: mgmt.gorm
    rules:
      - alert: GormPoolHigh
        annotations:
          message: In-use/max {{ printf "%.0f" $value }}% job={{ $labels.job }} pod={{ $labels.pod }}
        expr: |
          max by (job,pod) (
            100 * gorm_dbstats_in_use{namespace="tidb-management-service"}
            /
            gorm_dbstats_max_open_connections{namespace="tidb-management-service"}
          ) > 80
        for: 10m
        labels:
          severity: major
          response_class: ticket
          component: tidb-management-service
          oncall_team: global_control

      - alert: GormPoolVeryHigh
        annotations:
          message: In-use/max {{ printf "%.0f" $value }}% job={{ $labels.job }} pod={{ $labels.pod }}
        expr: |
          max by (job,pod) (
            100 * gorm_dbstats_in_use{namespace="tidb-management-service"}
            /
            gorm_dbstats_max_open_connections{namespace="tidb-management-service"}
          ) > 95
        for: 5m
        labels:
          severity: critical
          response_class: page
          component: tidb-management-service
          oncall_team: global_control

      - alert: GormAvgWaitHigh
        annotations:
          message: Avg wait 5m {{ printf "%.0f" $value }} ms job={{ $labels.job }} pod={{ $labels.pod }}
        expr: |
          (
            1e-6 *
            (
              sum by (job,pod) (increase(gorm_dbstats_wait_duration{namespace="tidb-management-service"}[5m]))
              /
              sum by (job,pod) (increase(gorm_dbstats_wait_count{namespace="tidb-management-service"}[5m]))
            )
          ) > 100
          and on (job,pod)
          sum by (job,pod) (increase(gorm_dbstats_wait_count{namespace="tidb-management-service"}[5m])) > 50
        for: 10m
        labels:
          severity: major
          response_class: ticket
          component: tidb-management-service
          oncall_team: global_control

      - alert: GormWaitAccumHigh
        annotations:
          message: Wait time accrual {{ printf "%.2f" $value }} s/s job={{ $labels.job }} pod={{ $labels.pod }}
        expr: |
          sum by (job,pod) (
            rate(gorm_dbstats_wait_duration{namespace="tidb-management-service"}[5m])
          ) / 1e9 > 1
        for: 5m
        labels:
          severity: critical
          response_class: page
          component: tidb-management-service
          oncall_team: global_control

      - alert: GormIdleStarvation
        annotations:
          message: Idle=0 while pool hot job={{ $labels.job }} pod={{ $labels.pod }}
        expr: |
          max by (job,pod) (gorm_dbstats_idle{namespace="tidb-management-service"}) == 0
          and on (job,pod)
          max by (job,pod) (
            gorm_dbstats_in_use{namespace="tidb-management-service"}
            /
            gorm_dbstats_max_open_connections{namespace="tidb-management-service"}
          ) > 0.70
        for: 5m
        labels:
          severity: major
          response_class: ticket
          component: tidb-management-service
          oncall_team: global_control

      - alert: GormOpenConnsLeak
        annotations:
          message: Open connections slope {{ printf "%.2f" $value }} /s job={{ $labels.job }} pod={{ $labels.pod }}
        expr: |
          clamp_min(
            deriv(gorm_dbstats_open_connections{namespace="tidb-management-service"}[30m]),
            0
          ) > 0.10
        for: 20m
        labels:
          severity: major
          response_class: ticket
          component: tidb-management-service
          oncall_team: global_control

      - alert: GormCloseChurnHigh
        annotations:
          message: Close rate {{ printf "%.2f" $value }} /s job={{ $labels.job }} pod={{ $labels.pod }}
        expr: |
          sum by (job,pod) (
            rate(gorm_dbstats_max_idle_closed{namespace="tidb-management-service"}[5m])
            +
            rate(gorm_dbstats_max_lifetime_closed{namespace="tidb-management-service"}[5m])
          ) > 5
        for: 10m
        labels:
          severity: major
          response_class: ticket
          component: tidb-management-service
          oncall_team: global_control