## Drools rules
# rule "Raise alarm"
# when
#     $s : Sensor()
#     accumulate( Reading( sensor == $s, $temp : temperature );
#                 $min : min( $temp ),
#                 $max : max( $temp ),
#                 $avg : average( $temp );
#                 $min < 20, $avg > 70 )
# then
#     // raise the alarm
# end

# metrics 中如果包含 expr，则为不在当前 promql.yaml 里的，这种可以额外查询，额外计算需要的算子，然后写规则

# PD:
# cpu usage
# instance down
# cpu unbalance
# memory unbalance
# TiKV:
# TiFlash:
# io imbalance
# cpu inbalance
# cpu high
# memory imbalance
# down



- id: 208
  name: "Database time is unbalanced"
  qualified: false
  level: info
  end2end: true
  switch: on
  category: 
    - database_time
  component: "TiDB"
  metrics:
    - name: Database_time:by_instance
      variables:
        imbalance_dbtime: imbalance
    - name: TiDB_cpu_usage_norm:by_instance
      variables:
        mean_tidb_cpu: mean
  condition: "imbalance_dbtime > 0.15 and max(list(mean_tidb_cpu.values())) > 70"
  advice:
    key: "metrics_diag_208"
    variables:

- id: 231
  name: "Get token in Database time takes too much time"
  qualified: false
  level: warning
  end2end: false
  switch: on
  category: 
    - database_time
  component: "TiDB"
  metrics:
    - name: Database_time:by_phase
      variables:
        get_token_proportion: proportion['Get_token_time:total']
  condition: "get_token_proportion > 2"
  advice:
    key: "metrics_diag_231"
    variables:
    
- id: 232
  name: "Parse in Database time takes too much time"
  qualified: false
  level: warning
  end2end: false
  switch: on
  category: 
    - database_time
  component: "TiDB"
  metrics:
    - name: Database_time:by_phase
      variables:
        parse_proportion: proportion['Parse_time:total']
  condition: "parse_proportion > 5"
  advice:
    key: "metrics_diag_232"
    variables:
    
- id: 233
  name: "Compile in Database time takes too much time"
  qualified: false
  level: warning
  end2end: false
  switch: on
  category: 
    - database_time
  component: "TiDB"
  metrics:
    - name: Database_time:by_phase
      variables:
        compile_proportion: proportion['Compile_time:total']
  condition: "compile_proportion > 15"
  advice:
    key: "metrics_diag_233"
    variables:

- id: 51
  name: "Pessimistic lock wait time is too long"
  qualified: false
  level: warning
  end2end: true
  switch: on
  category: 
    - database_time
  component: "TiKV"
  metrics:
    - name: TiKV_pesslock_wait_time:by_instance
      variables:
        max_avg: max_kv_avg
  condition: "max_avg[1] > 0.05"
  advice:
    key: "tikv_pessmistic_txn_conflict_51"
    variables:
    - var_1: "max_avg[0]"
    - var_2: "max_avg[1] * 1000"

- id: 147
  name: "QPS is not balanced from applications"
  qualified: true
  level: info
  end2end: true
  switch: on
  category: 
    - qps
  component: "TiDB"
  metrics:
    - name: QPS:by_instance
      variables:
        imbalance_qps: imbalance
    - name: TiDB_cpu_usage_norm:by_instance
      variables:
        mean_tidb_cpu: mean
  condition: "imbalance_qps > 0.15 and max(list(mean_tidb_cpu.values())) > 70"
  advice:
    key: "metrics_diag_147"
    variables:

- id: 160
  name: "Too many failed QPS"
  qualified: true
  level: info
  end2end: false
  switch: on
  category: 
    - qps
  component: "TiDB"
  metrics:
    - name: QPS_failed:total
      variables:
        failed_qps: mean
  condition: "failed_qps > 10"
  advice:
    key: "metrics_diag_160"
    variables:

- id: 145
  name: "Plan cache hit rate is low"
  qualified: true
  level: info
  end2end: true
  switch: on
  category: 
    - qps
  component: "TiDB"
  metrics:
    - name: Plan_cache_OPS:total
      variables:
        plan_cache_mean: mean
    - name: CPS:by_type
      variables:
        stmtexecute: mean.StmtExecute
  # condition: "(plan_cache_men > 0.1) and (plan_cache_mean / (stmtexecute + 0.01) < 0.2)" # mock with error
  condition: "(plan_cache_mean > 0.1) and (plan_cache_mean / (stmtexecute + 0.01) < 0.2)" # true condition
  advice:
    key: "metrics_diag_145"
    variables:

- id: 157
  name: "StmtPrepare usage is low"
  qualified: true
  level: warning
  end2end: false
  switch: on
  category: 
    - qps
  component: "TiDB"
  metrics:
    - name: CPS:by_type
      variables:
        stmtexecute: mean.StmtExecute
    - name: QPS:total
      variables:
        cps_total: mean
  condition: "stmtexecute / (cps_total + 0.01) < 0.2"
  advice:
    key: "metrics_diag_157"
    variables:

- id: 161
  name: "The latency of TiDB nodes is not balanced"
  qualified: false
  level: info
  end2end: false
  switch: on
  category: 
    - duration
  component: "TiDB"
  metrics:
    - name: Avg_query_duration:by_instance
      variables:
        dur_imbalance: imbalance
    - name: TiDB_cpu_usage_norm:by_instance
      variables:
        mean_tidb_cpu: mean
  condition: "dur_imbalance > 0.15 and max(list(mean_tidb_cpu.values())) > 70"
  advice:
    key: "metrics_diag_161"
    variables:

- id: 38
  name: "Get-token is slow in TiDB"
  qualified: false
  level: warning
  end2end: true
  switch: on
  category: 
    - duration
  component: "TiDB"
  metrics:
    - name: Avg_get_token_duration:by_instance
      variables:
        max_avg: max_kv_avg
  condition: "max_avg[1] > 0.001"
  advice:
    key: "tidb_get_token_duration_38"
    variables:
    - var_1: "max_avg[0]"
    - var_2: "max_avg[1] * 1000"

- id: 39
  name: "Parsing query string is slow in TiDB"
  qualified: false
  level: warning
  end2end: true
  switch: on
  category: 
    - duration
  component: "TiDB"
  metrics:
    - name: Avg_parse_duration:by_instance
      variables:
        max_avg: max_kv_avg
  condition: "max_avg[1] > 0.003"
  advice:
    key: "tidb_diag_parser_duration_39"
    variables:
    - var_1: "max_avg[0]"
    - var_2: "max_avg[1] * 1000"

- id: 40
  name: "Compiling query statement is slow in TiDB"
  qualified: false
  level: warning
  end2end: true
  switch: on
  category: 
    - duration
  component: "TiDB"
  metrics:
    - name: Avg_compile_duration:by_instance
      variables:
        max_avg: max_kv_avg
  condition: "max_avg[1] > 0.003"
  advice:
    key: "tidb_diag_compile_duration_40"
    variables:
    - var_1: "max_avg[0]"
    - var_2: "max_avg[1] * 1000"

- id: 43
  name: "TiDB takes a long time to wait for tso"
  qualified: false
  level: warning
  end2end: false
  switch: on
  category: 
    - duration
  component: "TiDB"
  metrics:
    - name: Avg_PD_wait_TSO_duration:by_instance
      variables:
        max_avg: max_kv_avg
  condition: "max_avg[1] > 0.008"
  advice:
    key: "tidb_wait_tso_duration_43"
    variables:
    - var_1: "max_avg[0]"
    - var_2: "max_avg[1] * 1000"

- id: 209
  name: "Parse duration is unbalanced"
  qualified: false
  level: info
  end2end: false
  switch: on
  category: 
    - duration
  component: "TiDB"
  metrics:
    - name: Avg_parse_duration:by_instance
      variables:
        dur_imbalance: imbalance
    - name: TiDB_cpu_usage_norm:by_instance
      variables:
        mean_tidb_cpu: mean
  condition: "dur_imbalance > 0.15 and max(list(mean_tidb_cpu.values())) > 70"
  advice:
    key: "metrics_diag_209"
    variables:

- id: 210
  name: "Compile duration is unbalanced"
  qualified: false
  level: info
  end2end: false
  switch: on
  category: 
    - duration
  component: "TiDB"
  metrics:
    - name: Avg_compile_duration:by_instance
      variables:
        dur_imbalance: imbalance
    - name: TiDB_cpu_usage_norm:by_instance
      variables:
        mean_tidb_cpu: mean
  condition: "dur_imbalance > 0.15 and max(list(mean_tidb_cpu.values())) > 70"
  advice:
    key: "metrics_diag_210"
    variables:

- id: 211
  name: "Execute duration is unbalanced"
  qualified: false
  level: info
  end2end: false
  switch: on
  category: 
    - duration
  component: "TiDB"
  metrics:
    - name: Avg_execute_duration:by_instance
      variables:
        dur_imbalance: imbalance
    - name: TiDB_cpu_usage_norm:by_instance
      variables:
        mean_tidb_cpu: mean
  condition: "dur_imbalance > 0.15 and max(list(mean_tidb_cpu.values())) > 70"
  advice:
    key: "metrics_diag_211"
    variables:

- id: 212
  name: "Get token duration is unbalanced"
  qualified: false
  level: info
  end2end: false
  switch: on
  category: 
    - duration
  component: "TiDB"
  metrics:
    - name: Avg_get_token_duration:by_instance
      variables:
        dur_imbalance: imbalance
    - name: TiDB_cpu_usage_norm:by_instance
      variables:
        mean_tidb_cpu: mean
  condition: "dur_imbalance > 0.15 and max(list(mean_tidb_cpu.values())) > 70"
  advice:
    key: "metrics_diag_212"
    variables:

- id: 148
  name: "Active connection counts are imbalance"
  qualified: false
  level: info
  end2end: true
  switch: on
  category: 
    - active_connection
  component: "TiDB"
  metrics:
    - name: Active_connection_count:by_instance
      variables:
        act_conn_imbalance: imbalance
    - name: TiDB_cpu_usage_norm:by_instance
      variables:
        mean_tidb_cpu: mean
  condition: "dur_imbalance > 0.15 and max(list(mean_tidb_cpu.values())) > 70"
  advice:
    key: "metrics_diag_148"
    variables:

- id: 146
  name: "QPS downward trend follows active connection counts"
  qualified: true
  level: info
  end2end: true
  switch: on
  category: 
    - active_connection
  component: "TiDB"
  metrics:
    - name: QPS:total
      variables:
        qps_trend: over_alltrend
        corr_qps_actconn: pcc_Active_connection_count
    - name: Avg_query_duration:total
      variables:
        tidb_duration_trend: over_alltrend
  # condition: "(qps_trend < 10000) and (corr_qps_actconn > 0) and (tidb_duration_trend < 10000)" # mock with fail
  condition: "(qps_trend < -0.1) and (corr_qps_actconn > 0.85) and (tidb_duration_trend < 0.1)" # true condition
  advice:
    key: "metrics_diag_146"
    variables:

- id: 159
  name: "Active connection increased while QPS not"
  qualified: false
  level: info
  end2end: false
  switch: on
  category: 
    - active_connection
  component: "TiDB"
  metrics:
    - name: QPS:total
      variables:
        qps_trend: over_alltrend
    - name: Avg_query_duration:total
      variables:
        tidb_duration_trend: over_alltrend
    - name: Active_connection_count:total
      variables:
        act_conn_trend: over_alltrend
  condition: "(act_conn_trend > 0) and (qps_trend <= 0) and (tidb_duration_trend > 0)"
  advice:
    key: "metrics_diag_159"
    variables:

- id: 144
  name: "Latency bottleneck outside TiDB"
  qualified: true
  level: warning
  end2end: true
  switch: on
  category: 
    - idle_duration
  component: "Application"
  metrics:
    - name: Avg_idle_connection_duration_in_txn:total
      variables:
        idle_mean: mean
    - name: Avg_query_duration:total
      variables:
        cluster_mean: mean
  condition: "idle_mean > 3 * cluster_mean and idle_mean > 0.01 and cluster_mean < 0.05"
  advice:
    key: "metrics_diag_144"
    variables:
    - var_1: "idle_mean * 1000"
    - var_2: "cluster_mean * 1000"
    
- id: 234
  name: "Idle connection duration not in txn is high"
  qualified: false
  level: warning
  end2end: true
  switch: on
  category: 
    - idle_duration
  component: "Application"
  metrics:
    - name: Avg_idle_connection_duration_not_in_txn:total
      variables:
        idle_mean: mean
    - name: Avg_query_duration:total
      variables:
        cluster_mean: mean
  condition: "idle_mean > 0.5 * cluster_mean"
  advice:
    key: "metrics_diag_234"
    variables:
    - var_1: "idle_mean * 1000"
    - var_2: "cluster_mean * 1000"

- id: 165
  name: "GC process is abnormal"
  qualified: true
  level: info
  end2end: true
  switch: on
  category: 
    #- gc
    - other
  component: "TiKV"
  metrics:
    - name: TiKV_safepoint:total
      variables:
        gc_interval: interval
  condition: "max(gc_interval) / np.median(gc_interval) > 5"
  advice:
    key: "metrics_diag_165"
    variables:

- id: 85
  name: "Too many MVCC versions"
  qualified: false
  level: warning
  end2end: true
  switch: on
  category: 
    - other
  component: "TiKV"
  metrics:
    - name: Scan_total_versions:total
      variables:
        scan_total: mean
    - name: Scan_processed_keys:total
      variables:
        scan_processed: mean
  condition: "scan_total > 5 * scan_processed"
  advice:
    key: "tikv_diag_mvcc_versions_too_many_85"
    variables:

- id: 167
  name: "PD schedulers are abnormal"
  qualified: false
  level: info
  end2end: false
  switch: on
  category: 
    #- scheduler
    - other
  component: "PD"
  metrics:
    - name: PD_schedulers
      variables:
        pd_schedulers: mean
  #condition: "all(x in list(pd_schedulers.keys()) for x in ['balance-hot-region-scheduler', 'balance-leader-scheduler', 'balance-region-scheduler']) == False"
  condition: "not('balance-hot-region-scheduler' in list(pd_schedulers.keys()) and 'balance-leader-scheduler' in list(pd_schedulers.keys()) and 'balance-region-scheduler' in list(pd_schedulers.keys()))"
  advice:
    key: "metrics_diag_167"
    variables:

- id: 213
  name: "Some TiDB instances are rebooted"
  qualified: false
  level: warning
  end2end: false
  switch: on
  category: 
    - TiDB_resource
  component: "TiDB"
  metrics:
    - name: TiDB_uptime:by_instance
      variables:
        uptime_falldown: falldown
  condition: "any([len(x) > 0 for x in uptime_falldown.values()])"
  advice:
    key: "metrics_diag_213"
    variables:

- id: 215
  name: "Some TiDB instances' CPU is over 80% for 5 minutes"
  qualified: false
  level: info
  end2end: false
  switch: on
  category: 
    - TiDB_resource
  component: "TiDB"
  metrics:
    - name: TiDB_cpu_usage_norm:by_instance
      unit: "%"
      expr: 'label_replace(rate(process_cpu_seconds_total{{label_cluster_id},{label_component}="tidb"}[2m])/tidb_server_maxprocs{{label_cluster_id},{label_component}="tidb"}*100,"name","$1","instance","(.*)")'
      rawdata: false
      operators:
        - "check_continuous_high(0.8,5)"
      variables:
        cpu_80: 'range_gt_80%'
  condition: "any([len(x) > 0 for x in cpu_80.values()])"
  advice:
    key: "metrics_diag_215"
    variables:

# should be changed to over 80% for 5 minutes
# this is not a basic value in snapshot but how to re-calculated on business layer
- id: 151
  name: "TiDB CPU is high"
  qualified: true
  level: warning
  end2end: true
  switch: on
  category: 
    - TiDB_resource
  component: "TiDB"
  metrics:
    - name: TiDB_cpu_usage_norm:by_instance
      variables:
        tidb_cpu_by_instance: mean
  condition: "min(list(tidb_cpu_by_instance.values())) > 60"
  advice:
    key: "metrics_diag_151"
    variables:
    - var_1: "min(list(tidb_cpu_by_instance.values()))"

- id: 163
  name: "The CPU of TiDB nodes is not balanced"
  qualified: true
  level: info
  end2end: false
  switch: on
  category: 
    - TiDB_resource
  component: "TiDB"
  metrics:
    - name: TiDB_CPU_usage:by_instance
      variables:
        tidb_cpu_imbalance: imbalance
  condition: "tidb_cpu_imbalance > 0.15"
  advice:
    key: "metrics_diag_163"
    variables:

- id: 164
  name: "The memory of TiDB nodes is not balanced"
  qualified: false
  level: info
  end2end: true
  switch: on
  category: 
    - TiDB_resource
  component: "TiDB"
  metrics:
    - name: TiDB_memory_usage:by_instance
      variables:
        tidb_mem_imbalance: imbalance
  condition: "tidb_mem_imbalance > 0.15"
  advice:
    key: "metrics_diag_164"
    variables:

- id: 149
  name: "TiKV CPU is high"
  qualified: false
  level: warning
  end2end: true
  switch: on
  category: 
    - TiKV_resource
  component: "TiKV"
  metrics:
    - name: TiKV_cpu_usage_norm:by_instance
      variables:
        mean_tikv_cpu: mean
  condition: "min(list(mean_tikv_cpu.values())) > 60"
  advice:
    key: "metrics_diag_149"
    variables:
    - var_1: "min(list(mean_tikv_cpu.values()))"

- id: 150
  name: "TiKV disk usage is high"
  qualified: true
  level: warning
  end2end: true
  switch: on
  category: 
    - TiKV_resource
  component: "TiKV"
  metrics:
    - name: TiKV_disk_usage:by_instance
      variables:
        mean_tikv_disk: mean
  condition: "min(list(mean_tikv_disk.values())) > 70"
  advice:
    key: "metrics_diag_150"
    variables:
    - var_1: "min(list(mean_tikv_disk.values()))"

- id: 216
  name: "Some TiKV instances are rebooted"
  qualified: false
  level: warning
  end2end: false
  switch: on
  category: 
    - TiKV_resource
  component: "TiKV"
  metrics:
    - name: TiKV_uptime:by_instance
      variables:
        uptime_falldown: falldown
  condition: "any([len(x) > 0 for x in uptime_falldown.values()])"
  advice:
    key: "metrics_diag_216"
    variables:

- id: 217
  name: "Some TiKV instances' CPU is over 80% for 5 minutes"
  qualified: false
  level: info
  end2end: false
  switch: on
  category: 
    - TiKV_resource
  component: "TiKV"
  metrics:
    - name: TiKV_cpu_usage_norm:by_instance
      unit: "%"
      expr: 'label_replace(sum(rate(tikv_thread_cpu_seconds_total{{label_cluster_id}}[2m]))by(instance)/sum(tikv_server_cpu_cores_quota{{label_cluster_id},{label_component}="tikv"})by(instance)*100,"name","$1","instance","(.*)")'
      rawdata: false
      operators:
        - "check_continuous_high(0.8,5)"
      variables:
        cpu_80: 'range_gt_80%'
  condition: "any([len(x) > 0 for x in cpu_80.values()])"
  advice:
    key: "metrics_diag_217"
    variables:

- id: 158
  name: "The CPU usage of TiKV nodes is not balanced"
  qualified: false
  level: info
  end2end: false
  switch: on
  category: 
    - TiKV_resource
  component: "TiKV"
  metrics:
    - name: TiKV_CPU_usage:by_instance
      variables:
        tikv_cpu_imbalance: imbalance
  condition: "tikv_cpu_imbalance > 0.15"
  advice:
    key: "metrics_diag_158"
    variables:

- id: 162
  name: "The memory usage of TiKV nodes is not balanced"
  qualified: true
  level: info
  end2end: false
  switch: on
  category: 
    - TiKV_resource
  component: "TiKV"
  metrics:
    - name: TiKV_memory_usage:by_instance
      variables:
        tikv_mem_imbalance: imbalance
  condition: "tikv_mem_imbalance > 0.15"
  advice:
    key: "metrics_diag_162"
    variables:

- id: 166
  name: "Region size is too large"
  qualified: true
  level: info
  end2end: true
  switch: on
  category: 
    - TiKV_resource
  component: "TiKV"
  metrics:
    - name: TiKV_region_size:total
      variables:
        avg_P99_region: mean
  condition: "avg_P99_region > 1024 * 1024 * 1024"
  advice:
    key: "metrics_diag_166"
    variables:
      - var_1: "avg_P99_region / 1024 / 1024"

- id: 220
  name: "TiKV storage is umbalanced"
  qualified: false
  level: info
  end2end: false
  switch: on
  category: 
    - TiKV_resource
  component: "TiKV"
  metrics:
    - name: TiKV_disk_usage:by_instance
      variables:
        imbalance_tikv_disk: imbalance
  condition: "imbalance_tikv_disk > 0.15"
  advice:
    key: "metrics_diag_220"
    variables:        

- id: 221
  name: "The disk of some TiKV instances might be full in 8 hours"
  qualified: false
  level: warning
  end2end: false
  switch: on
  category: 
    - TiKV_resource
  component: "TiKV"
  metrics:
    - name: TiKV_disk_usage:by_instance
      variables:
        remain_seconds: remain_seconds
    #- name: TiKV_full_remain_time:by_instance
      #variables:
        #last_remain: 
  condition: "any([x < 8 * 3600 for x in remain_seconds.values()])"
  advice:
    key: "metrics_diag_221"
    variables:    

- id: 223
  name: "TiKV read IO is unbalanced"
  qualified: false
  level: info
  end2end: false
  switch: on
  category: 
    - TiKV_resource
  component: "TiKV"
  metrics:
    - name: TiKV_IO_read_MBps:by_instance
      variables:
        imbalance_tikv_read: imbalance
  condition: "imbalance_tikv_read > 0.15"
  advice:
    key: "metrics_diag_223"
    variables:    

- id: 224
  name: "TiKV write IO is unbalanced"
  qualified: false
  level: info
  end2end: false
  switch: on
  category: 
    - TiKV_resource
  component: "TiKV"
  metrics:
    - name: TiKV_IO_write_MBps:by_instance
      variables:
        imbalance_tikv_read: imbalance
  condition: "imbalance_tikv_read > 0.15"
  advice:
    key: "metrics_diag_224"
    variables: 

- id: 218
  name: "Some PD instances are rebooted"
  qualified: false
  level: warning
  end2end: false
  switch: on
  category: 
    - PD_resource
  component: "PD"
  metrics:
    - name: PD_uptime:by_instance
      variables:
        uptime_falldown: falldown
  condition: "any([len(x) > 0 for x in uptime_falldown.values()])"
  advice:
    key: "metrics_diag_218"
    variables:

- id: 219
  name: "Some TiFlash instances are rebooted"
  qualified: false
  level: warning
  end2end: false
  switch: on
  category: 
    - TiFlash_resource
  component: "TiFlash"
  metrics:
    - name: TiFlash_uptime:by_instance
      variables:
        uptime_falldown: falldown
  condition: "any([len(x) > 0 for x in uptime_falldown.values()])"
  advice:
    key: "metrics_diag_219"
    variables:

- id: 227
  name: "TiFlash CPU is unbalanced"
  qualified: false
  level: info
  end2end: false
  switch: on
  category: 
    - TiFlash_resource
  component: "TiFlash"
  metrics:
    - name: TiFlash_CPU_usage:by_instance
      variables:
        tiflash_cpu_imbalance: imbalance
  condition: "tiflash_cpu_imbalance > 0.15"
  advice:
    key: "metrics_diag_227"
    variables:

- id: 228
  name: "TiFlash memory is unbalanced"
  qualified: false
  level: info
  end2end: false
  switch: on
  category: 
    - TiFlash_resource
  component: "TiFlash"
  metrics:
    - name: TiFlash_memory_usage:by_instance
      variables:
        tiflash_mem_imbalance: imbalance
  condition: "tiflash_mem_imbalance > 0.15"
  advice:
    key: "metrics_diag_228"
    variables:

- id: 229
  name: "TiFlash storage is umbalanced"
  qualified: false
  level: info
  end2end: false
  switch: on
  category: 
    - TiFlash_resource
  component: "TiFlash"
  metrics:
    - name: TiFlash_disk_usage:by_instance
      variables:
        imbalance_tiflash_disk: imbalance
  condition: "imbalance_tiflash_disk > 0.15"
  advice:
    key: "metrics_diag_229"
    variables:

- id: 222
  name: "The disk of some TiFlash instances might be full in 8 hours"
  qualified: false
  level: warning
  end2end: false
  switch: on
  category: 
    - TiFlash_resource
  component: "TiFlash"
  metrics:
    - name: TiFlash_disk_usage:by_instance
      variables:
        remain_seconds: remain_seconds
    #- name: TiKV_full_remain_time:by_instance
      #variables:
        #last_remain: 
  condition: "any([x < 8 * 3600 for x in remain_seconds.values()])"
  advice:
    key: "metrics_diag_222"
    variables:  

- id: 226
  name: "TiFlash read IO is unbalanced"
  qualified: false
  level: info
  end2end: false
  switch: on
  category: 
    - TiFlash_resource
  component: "TiFlash"
  metrics:
    - name: TiFlash_IO_read_MBps:by_instance
      variables:
        imbalance_tiflash_read: imbalance
  condition: "imbalance_tiflash_read > 0.15"
  advice:
    key: "metrics_diag_226"
    variables:    

- id: 225
  name: "TiFlash write IO is unbalanced"
  qualified: false
  level: info
  end2end: false
  switch: on
  category: 
    - TiFlash_resource
  component: "TiFlash"
  metrics:
    - name: TiFlash_IO_write_MBps:by_instance
      variables:
        imbalance_tiflash_write: imbalance
  condition: "imbalance_tiflash_write > 0.15"
  advice:
    key: "metrics_diag_225"
    variables: 

- id: 143
  name: "JDBC set recommended"
  qualified: true
  level: warning
  end2end: true
  switch: on
  category: 
    - other
  component: "Application"
  metrics:
    - name: CPS:by_type
      #customized_operator: xxx_operator()
      variables:
        stmtclose: mean.StmtClose
        stmtexecute: mean.StmtExecute
        stmtprepare: mean.StmtPrepare
  condition: "(abs(stmtclose - stmtexecute) < stmtexecute * 0.2) and ((stmtprepare - stmtexecute) < stmtexecute * 0.2) and (stmtexecute > 10)"
  advice:
    key: "metrics_diag_143"
    variables:

- id: 152
  name: "Plan cache is not enabled"
  qualified: true
  level: info
  end2end: true
  switch: off
  category: 
    - qps
  component: "TiDB"
  metrics:
    - name: Plan_cache_OPS:total
      variables:
        max_plancache: max
  condition: "max_plancache < 0.0001"
  advice:
    key: "metrics_diag_152"
    variables:

- id: 153
  name: "Scatter scheduler should be disabled"
  qualified: true
  level: info
  end2end: false
  switch: on
  category: 
    #- scheduler
    - other
  component: "PD"
  metrics:
    - name: PD_schedulers
      variables:
        pd_schedulers: mean
  condition: "any('scatter-range-' in x for x in list(pd_schedulers.keys()))"
  advice:
    key: "metrics_diag_153"
    variables:

- id: 154
  name: "Raft-engine is not enable"
  qualified: true
  level: info
  end2end: false
  switch: on
  category: 
    - other
  component: "TiKV"
  metrics:
    - name: Raft-engine:total
      variables:
        max_raft_engine: max
  condition: "max_raft_engine < 0.001"
  advice:
    key: "metrics_diag_154"
    variables:

- id: 155
  name: "TiKV AsyncIO is not enabled"
  qualified: false
  level: info
  end2end: false
  switch: on
  category: 
    - other
  component: "TiKV"
  metrics:
    - name: Asyncio_cpu:total
      variables:
        mean_asyncio_cpu: mean
    - name: TiKV_cpu_usage_norm:total
      variables:
        mean_tikv_cpu: mean
  # condition: "np.isnan(avg_stats['asyncio_cpu:total']) and (avg_stats['tikv_cpu_usage:total'] < 0.75)"
  condition: "(mean_asyncio_cpu < 1e-5 and mean_tikv_cpu < 75)"
  advice:
    key: "metrics_diag_155"
    variables:

- id: 156
  name: "Network latency high between TiDB and TiKV"
  qualified: false
  level: warning
  end2end: false
  switch: on
  category: 
    - duration
  component: "Network"
  metrics:
    - name: Avg_KV_request_duration:total
      variables:
        mean_kv_req_du: mean
    - name: Avg_TiKV_gRPC_duration:total
      variables:
        mean_rpc_du: mean
  condition: "mean_kv_req_du - mean_rpc_du >= 0.03"
  advice:
    key: "metrics_diag_156"
    variables:
      - var_1: "(mean_kv_req_du - mean_rpc_du) * 1000"

- id: 168
  name: "Too many heartbeat messages"
  qualified: false
  level: info
  end2end: false
  switch: off
  category: 
    - other
  component: "TiKV"
  metrics:
    - name: Raft_messages:by_type
      variables:
        mean_raft_msg_top: mean
  condition: "'heartbeat' in max(mean_raft_msg_top, key=mean_raft_msg_top.get)"
  advice:
    key: "metrics_diag_168"
    variables:

- id:
  name: ""
  qualified: false
  level: warning
  end2end: true
  switch: off
  category: 
    - other
  component: "TiDB"
  metrics:
    - name: Plan_cache_OPS:total
      variables:
        mean_plancache: mean
    - name: CPS:by_type
      variables:
        mean_cps: mean
  condition: "(mean_plancache > 0.1) and (abs(mean_cps['StmtReset'] - mean_cps['StmtExecute']) < mean_cps['StmtExecute'] * 0.1) and (mean_cps['StmtExecute'] > 10)"
  advice:
    key: ""
    variables:
    - var_1: "mean_cps['StmtReset']"
    - var_2: "mean_cps['StmtExecute']"

- id:
  name: ""
  qualified: false
  level: warning
  end2end: false
  switch: off
  category: 
    - other
  component: "TiDB"
  metrics:
    - name: Connnection_count:total
      variables:
        conn_trend: over_alltrend
    - name: Active_connection_count:total
      variables:
        act_conn_trend: over_alltrend
        mean_act_conn: mean
  condition: "(conn_trend > 0) and (act_conn_trend <= 0) and (mean_act_conn < 1000)"
  advice:
    key: ""
    variables:

- id: 44
  name: "Tso rpc duration is high"
  qualified: false
  level: warning
  end2end: true
  switch: on
  category: 
    - duration
  component: "TiDB"
  metrics:
    - name: TSO_rpc_duration:by_instance
      variables:
        max_avg: max_kv_avg
  condition: "max_avg[1] > 0.007"
  advice:
    key: "tidb_wait_tso_duration_44"
    variables:
    - var_1: "max_avg[0]"
    - var_2: "max_avg[1] * 1000"


- id: 51
  name: "Pessimistic lock wait count is too many"
  qualified: false
  level: warning
  end2end: true
  switch: off
  category: 
    - other
  component: "TiKV"
  metrics:
    - name: TiKV_pesslock_wait_count:total
      variables:
        mean_pesslock: mean
  condition: "mean_pesslock > 100"
  advice:
    key: "tikv_pessmistic_txn_conflict_5111"
    variables:
    - var_1: "mean_pesslock"  

- id: 78
  name: "The wait duration of raft propose is high"
  qualified: false
  level: warning
  end2end: true
  switch: on
  category: 
    - duration
  component: "TiKV"
  metrics:
    - name: TiKV_raft_propose_wait:by_instance
      variables:
        max_avg: max_kv_avg
  condition: "max_avg[1] > 0.015"
  advice:
    key: "tikv_raft_propose_wait_duration_78"
    variables:
    - var_1: "max_avg[0]"
    - var_2: "max_avg[1] * 1000"

- id: 171
  name: "TiDB coprocessor duration is high"
  qualified: false
  level: warning
  end2end: true
  switch: on
  category: 
    - duration
  component: "TiDB"
  metrics:
    - name: Coprocessor_duration:by_instance
      variables:
        max_avg: max_kv_avg
  condition: "max_avg[1] > 0.02"
  advice:
    key: "tikv_diag_cop_task_duration_171"
    variables:
    - var_1: "max_avg[0]"
    - var_2: "max_avg[1] * 1000"

- id: 259
  name: "Internal activity is high"
  qualified: false
  level: info
  end2end: false
  switch: on
  category:
    - other
  component: "TiDB"
  metrics:
    - name: KV_Request_OPS:total
      variables:
        mean_kvops: mean
    - name: KV_Request_OPS_Internal:total
      variables:
        mean_kvops_internal: mean
  condition: "mean_kvops * 0.1 < mean_kvops_internal"
  advice:
    key: "internal_activity_is_high_259"
    variables:

- id: 260
  name: "Frequent database connections"
  qualified: false
  level: warning
  end2end: true
  switch: on
  category:
    - qps
  component: "TiDB"
  metrics:
    - name: Disconnection_rate_OK:total
      variables:
        mean_disconn_rate: mean
  condition: "mean_disconn_rate > 50"
  advice:
    key: "short_connection_260"
    variables: