{{- if .Values.monitor.create }}
apiVersion: v1
kind: ConfigMap
metadata:
  namespace: {{ .Release.Namespace }}
  name: {{ .Values.clusterName }}-monitor-configmap
data:
  prometheus-config: |-
    global:
      scrape_interval: 15s
      evaluation_interval: 15s
    {{- if .Values.alertmanagerURL }}
    alerting:
      alertmanagers:
      - static_configs:
        - targets:
          - {{ .Values.alertmanagerURL }}
    {{- end }}
    scrape_configs:
      - job_name: 'tidb-cluster'
        scrape_interval: 15s
        honor_labels: true
        kubernetes_sd_configs:
        - role: pod
          namespaces:
            names:
            - {{ .Release.Namespace }}
        tls_config:
          insecure_skip_verify: true
        relabel_configs:
        - source_labels: [__meta_kubernetes_pod_label_cluster_pingcap_com_tidbCluster]
          action: keep
          regex: {{ .Values.clusterName }}
        - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape]
          action: keep
          regex: true
        - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path]
          action: replace
          target_label: __metrics_path__
          regex: (.+)
        - source_labels: [__meta_kubernetes_pod_container_port_name]
          action: keep
          regex: metrics        # only scrape container port named metrics
        - source_labels: [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port]
          action: replace
          regex: ([^:]+)(?::\d+)?;(\d+)
          replacement: $1:$2
          target_label: __address__
        - source_labels: [__meta_kubernetes_namespace]
          action: replace
          target_label: kubernetes_namespace
        - source_labels: [__meta_kubernetes_pod_node_name]
          action: replace
          target_label: kubernetes_node
        - source_labels: [__meta_kubernetes_pod_ip]
          action: replace
          target_label: kubernetes_pod_ip
        - source_labels: [__meta_kubernetes_pod_name]
          action: replace
          target_label: instance
        - source_labels: [__meta_kubernetes_pod_label_cluster_pingcap_com_tidbCluster]
          action: replace
          target_label: cluster
    rule_files:
      - 'alert.rules'

  alert-rules-config: |-
    groups:
    - name: tidb-alert-rules
      rules:
      - alert: PD_cluster_offline_tikv_nums
        expr: sum ( pd_cluster_status{type="store_down_count"} ) > 0
        for: 1m
        labels:
          env: '{{ .Values.clusterName }}'
          level: emergency
          expr:  sum ( pd_cluster_status{type="store_down_count"} ) > 0
        annotations:
          description: 'alert: instance: {{ .Values.metaInstance }}   values:{{ .Values.metaValue }}'
          value: '{{ .Values.metaValue }}'
          summary: PD_cluster_offline_tikv_nums

      - alert: PD_etcd_write_disk_latency
        expr: histogram_quantile(0.99, sum(rate(etcd_disk_wal_fsync_duration_seconds_bucket[1m])) by (instance,job,le) ) > 1
        for: 1m
        labels:
          env: '{{ .Values.clusterName }}'
          level: critical
          expr:  histogram_quantile(0.99, sum(rate(etcd_disk_wal_fsync_duration_seconds_bucket[1m])) by (instance,job,le) ) > 1
        annotations:
          description: 'alert: instance: {{ .Values.metaInstance }}   values:{{ .Values.metaValue }}'
          value: '{{ .Values.metaValue }}'
          summary: PD_etcd_write_disk_latency

      - alert: PD_miss_peer_region_count
        expr: sum( pd_regions_status{type="miss_peer_region_count"} )  > 100
        for: 1m
        labels:
          env: '{{ .Values.clusterName }}'
          level: critical
          expr:  sum( pd_regions_status{type="miss_peer_region_count"} )  > 100
        annotations:
          description: 'alert: instance: {{ .Values.metaInstance }}   values:{{ .Values.metaValue }}'
          value: '{{ .Values.metaValue }}'
          summary: PD_miss_peer_region_count

      - alert: PD_cluster_lost_connect_tikv_nums
        expr: sum ( pd_cluster_status{type="store_disconnected_count"} )   > 0
        for: 1m
        labels:
          env: '{{ .Values.clusterName }}'
          level: warning
          expr:  sum ( pd_cluster_status{type="store_disconnected_count"} )   > 0
        annotations:
          description: 'alert: instance: {{ .Values.metaInstance }}   values:{{ .Values.metaValue }}'
          value: '{{ .Values.metaValue }}'
          summary: PD_cluster_lost_connect_tikv_nums

      - alert: PD_cluster_low_space
        expr: sum ( pd_cluster_status{type="store_low_space_count"} )  > 0
        for: 1m
        labels:
          env: '{{ .Values.clusterName }}'
          level: warning
          expr:  sum ( pd_cluster_status{type="store_low_space_count"} )  > 0
        annotations:
          description: 'alert: instance: {{ .Values.metaInstance }}   values:{{ .Values.metaValue }}'
          value: '{{ .Values.metaValue }}'
          summary: PD_cluster_low_space

      - alert: PD_etcd_network_peer_latency
        expr: histogram_quantile(0.99, sum(rate(etcd_network_peer_round_trip_time_seconds_bucket[1m])) by (To,instance,job,le) ) > 1
        for: 1m
        labels:
          env: '{{ .Values.clusterName }}'
          level: warning
          expr:  histogram_quantile(0.99, sum(rate(etcd_network_peer_round_trip_time_seconds_bucket[1m])) by (To,instance,job,le) ) > 1
        annotations:
          description: 'alert: instance: {{ .Values.metaInstance }}   values:{{ .Values.metaValue }}'
          value: '{{ .Values.metaValue }}'
          summary: PD_etcd_network_peer_latency

      - alert: PD_tidb_handle_requests_duration
        expr: histogram_quantile(0.99, sum(rate(pd_client_request_handle_requests_duration_seconds_bucket{type="tso"}[1m])) by (instance,job,le) ) > 0.1
        for: 1m
        labels:
          env: '{{ .Values.clusterName }}'
          level: warning
          expr:  histogram_quantile(0.99, sum(rate(pd_client_request_handle_requests_duration_seconds_bucket{type="tso"}[1m])) by (instance,job,le) ) > 0.1
        annotations:
          description: 'alert: instance: {{ .Values.metaInstance }}   values:{{ .Values.metaValue }}'
          value: '{{ .Values.metaValue }}'
          summary: PD_tidb_handle_requests_duration

      - alert: PD_down_peer_region_nums
        expr: sum ( pd_regions_status{type="down_peer_region_count"} ) > 0
        for: 1m
        labels:
          env: '{{ .Values.clusterName }}'
          level: warning
          expr:  sum ( pd_regions_status{type="down_peer_region_count"} ) > 0
        annotations:
          description: 'alert: instance: {{ .Values.metaInstance }}   values:{{ .Values.metaValue }}'
          value: '{{ .Values.metaValue }}'
          summary: PD_down_peer_region_nums

      - alert: PD_incorrect_namespace_region_count
        expr: sum ( pd_regions_status{type="incorrect_namespace_region_count"} ) > 100
        for: 1m
        labels:
          env: '{{ .Values.clusterName }}'
          level: warning
          expr: sum ( pd_regions_status{type="incorrect_namespace_region_count"} ) > 0
        annotations:
          description: 'alert: instance: {{ .Values.metaInstance }}   values:{{ .Values.metaValue }}'
          value: '{{ .Values.metaValue }}'
          summary: PD_incorrect_namespace_region_count

      - alert: PD_pending_peer_region_count
        expr: sum( pd_regions_status{type="pending_peer_region_count"} )  > 100
        for: 1m
        labels:
          env: '{{ .Values.clusterName }}'
          level: warning
          expr:  sum( pd_regions_status{type="pending_peer_region_count"} )  > 100
        annotations:
          description: 'alert: instance: {{ .Values.metaInstance }}   values:{{ .Values.metaValue }}'
          value: '{{ .Values.metaValue }}'
          summary: PD_pending_peer_region_count

      - alert: PD_leader_change
        expr: count( changes(pd_server_tso{type="save"}[10m]) > 0 )   >= 2
        for: 1m
        labels:
          env: '{{ .Values.clusterName }}'
          level: warning
          expr:  count( changes(pd_server_tso{type="save"}[10m]) > 0 )   >= 2
        annotations:
          description: 'alert: instance: {{ .Values.metaInstance }}   values:{{ .Values.metaValue }}'
          value: '{{ .Values.metaValue }}'
          summary: PD_leader_change

      - alert: TiKV_space_used_more_than_80%
        expr: sum(pd_cluster_status{type="storage_size"}) / sum(pd_cluster_status{type="storage_capacity"}) * 100  > 80
        for: 1m
        labels:
          env: '{{ .Values.clusterName }}'
          level: warning
          expr:  sum(pd_cluster_status{type="storage_size"}) / sum(pd_cluster_status{type="storage_capacity"}) * 100  > 80
        annotations:
          description: 'alert: type: {{ .Values.metaType }} instance: {{ .Values.metaInstance }}  values: {{ .Values.metaValue }}'
          value: '{{ .Values.metaValue }}'
          summary: TiKV_space_used_more_than_80%
      - alert: TiDB_schema_error
        expr: increase(tidb_session_schema_lease_error_total{type="outdated"}[15m]) > 0
        for: 1m
        labels:
          env: '{{ .Values.clusterName }}'
          level: emergency
          expr:  increase(tidb_session_schema_lease_error_total{type="outdated"}[15m]) > 0
        annotations:
          description: 'alert: instance: {{ .Values.metaInstance }} values: {{ .Values.metaValue }}'
          value: '{{ .Values.metaValue }}'
          summary: TiDB schema error

      - alert: TiDB_tikvclient_region_err_total
        expr: increase( tidb_tikvclient_region_err_total[10m] )  > 6000
        for: 1m
        labels:
          env: '{{ .Values.clusterName }}'
          level: emergency
          expr:  increase( tidb_tikvclient_region_err_total[10m] )  > 6000
        annotations:
          description: 'alert: instance: {{ .Values.metaInstance }} values: {{ .Values.metaValue }}'
          value: '{{ .Values.metaValue }}'
          summary: TiDB tikvclient_backoff_count error

      - alert: TiDB_domain_load_schema_total
        expr: increase( tidb_domain_load_schema_total{type="failed"}[10m] )  > 10
        for: 1m
        labels:
          env: '{{ .Values.clusterName }}'
          level: emergency
          expr:  increase( tidb_domain_load_schema_total{type="failed"}[10m] )  > 10
        annotations:
          description: 'alert: instance: {{ .Values.metaInstance }} values: {{ .Values.metaValue }}'
          value: '{{ .Values.metaValue }}'
          summary: TiDB domain_load_schema_total error

      - alert: TiDB_monitor_keep_alive
        expr: increase(tidb_monitor_keep_alive_total[10m]) < 100
        for: 1m
        labels:
          env: '{{ .Values.clusterName }}'
          level: emergency
          expr:  increase(tidb_monitor_keep_alive_total[10m]) < 100
        annotations:
          description: 'alert: instance: {{ .Values.metaInstance }} values: {{ .Values.metaValue }}'
          value: '{{ .Values.metaValue }}'
          summary: TiDB monitor_keep_alive error

      - alert: TiDB_server_panic_total
        expr: increase(tidb_server_panic_total[10m]) > 0
        for: 1m
        labels:
          env: '{{ .Values.clusterName }}'
          level: critical
          expr:  increase(tidb_server_panic_total[10m]) > 0
        annotations:
          description: 'alert: instance: {{ .Values.metaInstance }} values: {{ .Values.metaValue }}'
          value: '{{ .Values.metaValue }}'
          summary: TiDB server panic total

      - alert: TiDB_memery_abnormal
        expr: go_memstats_heap_inuse_bytes{job="tidb"} > 1e+10
        for: 1m
        labels:
          env: '{{ .Values.clusterName }}'
          level: warning
          expr: go_memstats_heap_inuse_bytes{job="tidb"} > 1e+10
        annotations:
          description: 'alert: values: {{ .Values.metaValue }}'
          value: '{{ .Values.metaValue }}'
          summary: TiDB mem heap is over 1GiB

      - alert: TiDB_query_duration
        expr: histogram_quantile(0.99, sum(rate(tidb_server_handle_query_duration_seconds_bucket[1m])) BY (le, instance)) > 1
        for: 1m
        labels:
          env: '{{ .Values.clusterName }}'
          level: warning
          expr:  histogram_quantile(0.99, sum(rate(tidb_server_handle_query_duration_seconds_bucket[1m])) BY (le, instance)) > 1
        annotations:
          description: 'alert: instance: {{ .Values.metaInstance }} values: {{ .Values.metaValue }}'
          value: '{{ .Values.metaValue }}'
          summary: TiDB query duration 99th percentile is above 1s

      - alert: TiDB_server_event_error
        expr: increase(tidb_server_server_event{type=~"server_start|server_hang"}[15m])  > 0
        for: 1m
        labels:
          env: '{{ .Values.clusterName }}'
          level: warning
          expr:  increase(tidb_server_server_event{type=~"server_start|server_hang"}[15m])  > 0
        annotations:
          description: 'alert: instance: {{ .Values.metaInstance }} values: {{ .Values.metaValue }}'
          value: '{{ .Values.metaValue }}'
          summary: TiDB server event error

      - alert: TiDB_tikvclient_backoff_count
        expr: increase( tidb_tikvclient_backoff_count[10m] )  > 10
        for: 1m
        labels:
          env: '{{ .Values.clusterName }}'
          level: warning
          expr:  increase( tidb_tikvclient_backoff_count[10m] )  > 10
        annotations:
          description: 'alert: instance: {{ .Values.metaInstance }} values: {{ .Values.metaValue }}'
          value: '{{ .Values.metaValue }}'
          summary: TiDB tikvclient_backoff_count error

      - alert: TiDB_monitor_time_jump_back_error
        expr: increase(tidb_monitor_time_jump_back_total[10m])  > 0
        for: 1m
        labels:
          env: '{{ .Values.clusterName }}'
          level: warning
          expr:  increase(tidb_monitor_time_jump_back_total[10m])  > 0
        annotations:
          description: 'alert: instance: {{ .Values.metaInstance }} values: {{ .Values.metaValue }}'
          value: '{{ .Values.metaValue }}'
          summary: TiDB monitor time_jump_back error

      - alert: TiDB_ddl_waiting_jobs
        expr: sum(tidb_ddl_waiting_jobs) > 5
        for: 1m
        labels:
          env: '{{ .Values.clusterName }}'
          level: warning
          expr:  sum(tidb_ddl_waiting_jobs) > 5
        annotations:
          description: 'alert: instance: {{ .Values.metaInstance }} values: {{ .Values.metaValue }}'
          value: '{{ .Values.metaValue }}'
          summary: TiDB ddl waiting_jobs too much

      - alert: TiKV_memory_used_too_fast
        expr: (node_memory_MemAvailable offset 5m) - node_memory_MemAvailable > 5*1024*1024*1024
        for: 5m
        labels:
          env: '{{ .Values.clusterName }}'
          level: emergency
          expr: (node_memory_MemAvailable offset 5m) - node_memory_MemAvailable > 5*1024*1024*1024
        annotations:
          description: 'alert: instance: {{ .Values.metaInstance }}  values: {{ .Values.metaValue }}'
          value: '{{ .Values.metaValue }}'
          summary: TiKV memory used too fast

      - alert: TiKV_GC_can_not_work
        expr: sum(increase(tidb_tikvclient_gc_action_result{type="success"}[6h])) < 1
        for: 1m
        labels:
          env: '{{ .Values.clusterName }}'
          level: emergency
          expr: sum(increase(tidb_tikvclient_gc_action_result{type="success"}[6h])) < 1
        annotations:
          description: 'alert: instance: {{ .Values.metaInstance }}  values: {{ .Values.metaValue }}'
          value: '{{ .Values.metaValue }}'
          summary: TiKV GC can not work

      - alert: TiKV_server_report_failure_msg_total
        expr:  sum(rate(tikv_server_report_failure_msg_total{type="unreachable"}[10m])) BY (store_id) > 10
        for: 1m
        labels:
          env: '{{ .Values.clusterName }}'
          level: critical
          expr:  sum(rate(tikv_server_report_failure_msg_total{type="unreachable"}[10m])) BY (store_id) > 10
        annotations:
          description: 'alert: instance: {{ .Values.metaInstance }}  values: {{ .Values.metaValue }}'
          value: '{{ .Values.metaValue }}'
          summary: TiKV server_report_failure_msg_total error

      - alert: TiKV_channel_full_total
        expr: sum(rate(tikv_channel_full_total[10m])) BY (type, instance) > 0
        for: 1m
        labels:
          env: '{{ .Values.clusterName }}'
          level: critical
          expr:  sum(rate(tikv_channel_full_total[10m])) BY (type, instance) > 0
        annotations:
          description: 'alert: instance: {{ .Values.metaInstance }}  values: {{ .Values.metaValue }}'
          value: '{{ .Values.metaValue }}'
          summary: TiKV channel full

      - alert: TiKV_write_stall
        expr: delta( tikv_engine_write_stall[10m])  > 0
        for: 1m
        labels:
          env: '{{ .Values.clusterName }}'
          level: critical
          expr:  delta( tikv_engine_write_stall[10m])  > 0
        annotations:
          description: 'alert: type: {{ .Values.metaType }} instance: {{ .Values.metaInstance }}  values: {{ .Values.metaValue }}'
          value: '{{ .Values.metaValue }}'
          summary: TiKV write stall

      - alert: TiKV_raft_log_lag
        expr: histogram_quantile(0.99, sum(rate(tikv_raftstore_log_lag_bucket[1m])) by (le, instance, job))  > 5000
        for: 1m
        labels:
          env: '{{ .Values.clusterName }}'
          level: critical
          expr:  histogram_quantile(0.99, sum(rate(tikv_raftstore_log_lag_bucket[1m])) by (le, instance, job))  > 5000
        annotations:
          description: 'alert: instance {{ .Values.metaInstance }}  values: {{ .Values.metaValue }}'
          value: '{{ .Values.metaValue }}'
          summary: TiKV raftstore log lag more than 5000

      - alert: TiKV_async_request_snapshot_duration_seconds
        expr: histogram_quantile(0.99, sum(rate(tikv_storage_engine_async_request_duration_seconds_bucket{type="snapshot"}[1m])) by (le, instance, job,type)) > 1
        for: 1m
        labels:
          env: '{{ .Values.clusterName }}'
          level: critical
          expr:  histogram_quantile(0.99, sum(rate(tikv_storage_engine_async_request_duration_seconds_bucket{type="snapshot"}[1m])) by (le, instance, job,type)) > 1
        annotations:
          description: 'alert: instance: {{ .Values.metaInstance }}  values: {{ .Values.metaValue }}'
          value: '{{ .Values.metaValue }}'
          summary: TiKV async request snapshot duration seconds more than 1s

      - alert: TiKV_async_request_write_duration_seconds
        expr: histogram_quantile(0.99, sum(rate(tikv_storage_engine_async_request_duration_seconds_bucket{type="write"}[1m])) by (le, instance, job,type)) > 1
        for: 1m
        labels:
          env: '{{ .Values.clusterName }}'
          level: critical
          expr:  histogram_quantile(0.99, sum(rate(tikv_storage_engine_async_request_duration_seconds_bucket{type="write"}[1m])) by (le, instance, job,type)) > 1
        annotations:
          description: 'alert: instance: {{ .Values.metaInstance }}  values: {{ .Values.metaValue }}'
          value: '{{ .Values.metaValue }}'
          summary: TiKV async request write duration seconds more than 1s

      - alert: TiKV_coprocessor_request_wait_seconds
        expr: histogram_quantile(0.9999, sum(rate(tikv_coprocessor_request_wait_seconds_bucket[1m])) by (le, instance, job,req)) > 10
        for: 1m
        labels:
          env: '{{ .Values.clusterName }}'
          level: critical
          expr:  histogram_quantile(0.9999, sum(rate(tikv_coprocessor_request_wait_seconds_bucket[1m])) by (le, instance, job,req)) > 10
        annotations:
          description: 'alert: instance: {{ .Values.metaInstance }}  values: {{ .Values.metaValue }}'
          value: '{{ .Values.metaValue }}'
          summary: TiKV coprocessor request wait seconds more than 10s

      - alert: TiKV_raftstore_thread_cpu_seconds_total
        expr: sum(rate(tikv_thread_cpu_seconds_total{name=~"raftstore_.*"}[1m])) by (job, name)  > 0.8
        for: 1m
        labels:
          env: '{{ .Values.clusterName }}'
          level: critical
          expr: sum(rate(tikv_thread_cpu_seconds_total{name=~"raftstore_.*"}[1m])) by (job, name)  > 0.8
        annotations:
          description: 'alert: instance: {{ .Values.metaInstance }}  values: {{ .Values.metaValue }}'
          value: '{{ .Values.metaValue }}'
          summary: TiKV raftstore thread CPU seconds is high

      - alert: TiKV_raft_append_log_duration_secs
        expr: histogram_quantile(0.99, sum(rate(tikv_raftstore_append_log_duration_seconds_bucket[1m])) by (le, instance, job)) > 1
        for: 1m
        labels:
          env: '{{ .Values.clusterName }}'
          level: critical
          expr: histogram_quantile(0.99, sum(rate(tikv_raftstore_append_log_duration_seconds_bucket[1m])) by (le, instance, job)) > 1
        annotations:
          description: 'alert: instance: {{ .Values.metaInstance }}  values: {{ .Values.metaValue }}'
          value: '{{ .Values.metaValue }}'
          summary: TiKV_raft_append_log_duration_secs

      - alert: TiKV_raft_apply_log_duration_secs
        expr: histogram_quantile(0.99, sum(rate(tikv_raftstore_apply_log_duration_seconds_bucket[1m])) by (le, instance, job)) > 1
        for: 1m
        labels:
          env: '{{ .Values.clusterName }}'
          level: critical
          expr: histogram_quantile(0.99, sum(rate(tikv_raftstore_apply_log_duration_seconds_bucket[1m])) by (le, instance, job)) > 1
        annotations:
          description: 'alert: instance: {{ .Values.metaInstance }}  values: {{ .Values.metaValue }}'
          value: '{{ .Values.metaValue }}'
          summary: TiKV_raft_apply_log_duration_secs

      - alert: TiKV_scheduler_latch_wait_duration_seconds
        expr: histogram_quantile(0.99, sum(rate(tikv_scheduler_latch_wait_duration_seconds_bucket[1m])) by (le, instance, job,type))  > 1
        for: 1m
        labels:
          env: '{{ .Values.clusterName }}'
          level: critical
          expr:  histogram_quantile(0.99, sum(rate(tikv_scheduler_latch_wait_duration_seconds_bucket[1m])) by (le, instance, job,type))  > 1
        annotations:
          description: 'alert: instance:
            {{ .Values.metaInstance }}  values: {{ .Values.metaValue }}'
          value: '{{ .Values.metaValue }}'
          summary: TiKV scheduler latch wait duration seconds more than 1s

      - alert: TiKV_thread_apply_worker_cpu_seconds
        expr: sum(rate(tikv_thread_cpu_seconds_total{name="apply_worker"}[1m])) by (job) > 0.9
        for: 1m
        labels:
          env: '{{ .Values.clusterName }}'
          level: critical
          expr:  sum(rate(tikv_thread_cpu_seconds_total{name="apply_worker"}[1m])) by (job) > 0.9
        annotations:
          description: 'alert: type: {{ .Values.metaType }} instance: {{ .Values.metaInstance }}  values: {{ .Values.metaValue }}'
          value: '{{ .Values.metaValue }}'
          summary: TiKV thread apply worker cpu seconds is high

      - alert: TiDB_tikvclient_gc_action_fail
        expr: sum(increase(tidb_tikvclient_gc_action_result{type="fail"}[1m])) > 10
        for: 1m
        labels:
          env: '{{ .Values.clusterName }}'
          level: critical
          expr: sum(increase(tidb_tikvclient_gc_action_result{type="fail"}[1m])) > 10
        annotations:
          description: 'alert: type: {{ .Values.metaType }} instance: {{ .Values.metaInstance }}  values: {{ .Values.metaValue }}'
          value: '{{ .Values.metaValue }}'
          summary: TiDB_tikvclient_gc_action_fail

      - alert: TiKV_leader_drops
        expr: delta(tikv_pd_heartbeat_tick_total{type="leader"}[30s]) < -10
        for: 1m
        labels:
          env: '{{ .Values.clusterName }}'
          level: warning
          expr: delta(tikv_pd_heartbeat_tick_total{type="leader"}[30s]) < -10
        annotations:
          description: 'alert: instance: {{ .Values.metaInstance }}  values:{{ .Values.metaValue }}'
          value: '{{ .Values.metaValue }}'
          summary: TiKV leader drops

      - alert: TiKV_raft_process_ready_duration_secs
        expr: histogram_quantile(0.999, sum(rate(tikv_raftstore_raft_process_duration_secs_bucket{type='ready'}[1m])) by (le, instance, job,type)) > 2
        for: 1m
        labels:
          env: '{{ .Values.clusterName }}'
          level: warning
          expr: histogram_quantile(0.999, sum(rate(tikv_raftstore_raft_process_duration_secs_bucket{type='ready'}[1m])) by (le, instance, job,type)) > 2
        annotations:
          description: 'alert: instance: {{ .Values.metaInstance }}  values:{{ .Values.metaValue }}'
          value: '{{ .Values.metaValue }}'
          summary: TiKV_raft_process_ready_duration_secs

      - alert: TiKV_raft_process_tick_duration_secs
        expr: histogram_quantile(0.999, sum(rate(tikv_raftstore_raft_process_duration_secs_bucket{type='tick'}[1m])) by (le, instance, job,type)) > 2
        for: 1m
        labels:
          env: '{{ .Values.clusterName }}'
          level: warning
          expr: histogram_quantile(0.999, sum(rate(tikv_raftstore_raft_process_duration_secs_bucket{type='tick'}[1m])) by (le, instance, job,type)) > 2
        annotations:
          description: 'alert: instance: {{ .Values.metaInstance }}  values:{{ .Values.metaValue }}'
          value: '{{ .Values.metaValue }}'
          summary: TiKV_raft_process_tick_duration_secs

      - alert: TiKV_scheduler_context_total
        expr: abs(delta( tikv_scheduler_contex_total[5m])) > 1000
        for: 1m
        labels:
          env: '{{ .Values.clusterName }}'
          level: warning
          expr:  abs(delta( tikv_scheduler_contex_total[5m])) > 1000
        annotations:
          description: 'alert: instance: {{ .Values.metaInstance }}  values: {{ .Values.metaValue }}'
          value: '{{ .Values.metaValue }}'
          summary: TiKV scheduler context total

      - alert: TiKV_scheduler_command_duration_seconds
        expr: histogram_quantile(0.99, sum(rate(tikv_scheduler_command_duration_seconds_bucket[1m])) by (le, instance, job,type)  / 1000)  > 1
        for: 1m
        labels:
          env: '{{ .Values.clusterName }}'
          level: warning
          expr:  histogram_quantile(0.99, sum(rate(tikv_scheduler_command_duration_seconds_bucket[1m])) by (le, instance, job,type)  / 1000)  > 1
        annotations:
          description: 'alert: instance: {{ .Values.metaInstance }}  values: {{ .Values.metaValue }}'
          value: '{{ .Values.metaValue }}'
          summary: TiKV scheduler command duration seconds more than 1s

      - alert: TiKV_thread_storage_scheduler_cpu_seconds
        expr: sum(rate(tikv_thread_cpu_seconds_total{name=~"storage_schedul.*"}[1m])) by (job) > 0.8
        for: 1m
        labels:
          env: '{{ .Values.clusterName }}'
          level: warning
          expr:  sum(rate(tikv_thread_cpu_seconds_total{name=~"storage_schedul.*"}[1m])) by (job) > 0.8
        annotations:
          description: 'alert: instance: {{ .Values.metaInstance }}  values: {{ .Values.metaValue }}'
          value: '{{ .Values.metaValue }}'
          summary: TiKV storage scheduler cpu seconds more than 80%

      - alert: TiKV_coprocessor_outdated_request_wait_seconds
        expr: delta( tikv_coprocessor_outdated_request_wait_seconds_count[10m] )  > 0
        for: 1m
        labels:
          env: '{{ .Values.clusterName }}'
          level: warning
          expr:  delta( tikv_coprocessor_outdated_request_wait_seconds_count[10m] )  > 0
        annotations:
          description: 'alert: instance: {{ .Values.metaInstance }}  values: {{ .Values.metaValue }}'
          value: '{{ .Values.metaValue }}'
          summary: TiKV coprocessor outdated request wait seconds

      - alert: TiKV_coprocessor_request_error
        expr: increase(tikv_coprocessor_request_error[10m]) > 100
        for: 1m
        labels:
          env: '{{ .Values.clusterName }}'
          level: warning
          expr:  increase(tikv_coprocessor_request_error[10m]) > 100
        annotations:
          description: 'alert: type: {{ .Values.metaType }} instance: {{ .Values.metaInstance }}  values: {{ .Values.metaValue }}'
          value: '{{ .Values.metaValue }}'
          summary: TiKV coprocessor_request_error

      - alert: TiKV_coprocessor_pending_request
        expr: delta( tikv_coprocessor_pending_request[10m]) > 5000
        for: 1m
        labels:
          env: '{{ .Values.clusterName }}'
          level: warning
          expr:  delta( tikv_coprocessor_pending_request[10m]) > 5000
        annotations:
          description: 'alert: type: {{ .Values.metaType }} instance: {{ .Values.metaInstance }}  values: {{ .Values.metaValue }}'
          value: '{{ .Values.metaValue }}'
          summary: TiKV pending {{ .Values.metaType }} request is high

      - alert: TiKV_batch_request_snapshot_nums
        expr: sum(rate(tikv_thread_cpu_seconds_total{name=~"cop_.*"}[1m])) by (job) / ( count(tikv_thread_cpu_seconds_total{name=~"cop_.*"}) *  0.9 ) / count(count(tikv_thread_cpu_seconds_total) by (instance)) > 0
        for: 1m
        labels:
          env: '{{ .Values.clusterName }}'
          level: warning
          expr:  sum(rate(tikv_thread_cpu_seconds_total{name=~"cop_.*"}[1m])) by (job) / ( count(tikv_thread_cpu_seconds_total{name=~"cop_.*"}) *  0.9 ) / count(count(tikv_thread_cpu_seconds_total) by (instance)) > 0
        annotations:
          description: 'alert: type: {{ .Values.metaType }} instance: {{ .Values.metaInstance }}  values: {{ .Values.metaValue }}'
          value: '{{ .Values.metaValue }}'
          summary: TiKV batch request snapshot nums is high

      - alert: TiKV_pending_task
        expr: sum(tikv_worker_pending_task_total) BY (job,instance,name)  > 1000
        for: 1m
        labels:
          env: '{{ .Values.clusterName }}'
          level: warning
          expr:  sum(tikv_worker_pending_task_total) BY (job,instance,name)  > 1000
        annotations:
          description: 'alert: type: {{ .Values.metaType }} instance: {{ .Values.metaInstance }}  values: {{ .Values.metaValue }}'
          value: '{{ .Values.metaValue }}'
          summary: TiKV pending task too much

      - alert: TiKV_low_space_and_add_region
        expr: count( (sum(tikv_store_size_bytes{type="available"}) by (job) / sum(tikv_store_size_bytes{type="capacity"}) by (job) < 0.2) and (sum(tikv_raftstore_snapshot_traffic_total{type="applying"}) by (job) > 0 ) ) > 0
        for: 1m
        labels:
          env: '{{ .Values.clusterName }}'
          level: warning
          expr:  count( (sum(tikv_store_size_bytes{type="available"}) by (job) / sum(tikv_store_size_bytes{type="capacity"}) by (job) < 0.2) and (sum(tikv_raftstore_snapshot_traffic_total{type="applying"}) by (job) > 0 ) ) > 0
        annotations:
          description: 'alert: type: {{ .Values.metaType }} instance: {{ .Values.metaInstance }}  values: {{ .Values.metaValue }}'
          value: '{{ .Values.metaValue }}'
          summary: TiKV low_space and add_region

  grafana-config: |-
    ##################### Grafana Configuration Example #####################
    #
    # Everything has defaults so you only need to uncomment things you want to
    # change

    # possible values : production, development
    ; app_mode = production

    # instance name, defaults to HOSTNAME environment variable value or hostname if HOSTNAME var is empty
    ; instance_name = ${HOSTNAME}

    #################################### Paths ####################################
    [paths]
    # Path to where grafana can store temp files, sessions, and the sqlite3 db (if that is used)
    #
    ;data = /var/lib/grafana
    #
    # Directory where grafana can store logs
    #
    ;logs = /var/log/grafana
    #
    # Directory where grafana will automatically scan and look for plugins
    #
    ;plugins = /var/lib/grafana/plugins

    #
    #################################### Server ####################################
    [server]
    # Protocol (http or https)
    ;protocol = http

    # The ip address to bind to, empty will bind to all interfaces
    ;http_addr =

    # The http port  to use
    ;http_port = 3000

    # The public facing domain name used to access grafana from a browser
    ;domain = localhost

    # Redirect to correct domain if host header does not match domain
    # Prevents DNS rebinding attacks
    ;enforce_domain = false

    # The full public facing url you use in browser, used for redirects and emails
    # If you use reverse proxy and sub path specify full url (with sub path)
    root_url = {{ .Values.grafanaUrl }}

    # Log web requests
    ;router_logging = false

    # the path relative working path
    ;static_root_path = public

    # enable gzip
    ;enable_gzip = false

    # https certs & key file
    ;cert_file =
    ;cert_key =

    #################################### Database ####################################
    [database]
    # You can configure the database connection by specifying type, host, name, user and password
    # as seperate properties or as on string using the url propertie.

    # Either "mysql", "postgres" or "sqlite3", it's your choice
    ;type = sqlite3
    ;host = 127.0.0.1:3306
    ;name = grafana
    ;user = root
    # If the password contains # or ; you have to wrap it with trippel quotes. Ex """#password;"""
    ;password =

    # Use either URL or the previous fields to configure the database
    # Example: mysql://user:secret@host:port/database
    ;url =

    # For "postgres" only, either "disable", "require" or "verify-full"
    ;ssl_mode = disable

    # For "sqlite3" only, path relative to data_path setting
    ;path = grafana.db

    # Max conn setting default is 0 (mean not set)
    ;max_conn =
    ;max_idle_conn =
    ;max_open_conn =


    #################################### Session ####################################
    [session]
    # Either "memory", "file", "redis", "mysql", "postgres", default is "file"
    ;provider = file

    # Provider config options
    # memory: not have any config yet
    # file: session dir path, is relative to grafana data_path
    # redis: config like redis server e.g. `addr=127.0.0.1:6379,pool_size=100,db=grafana`
    # mysql: go-sql-driver/mysql dsn config string, e.g. `user:password@tcp(127.0.0.1:3306)/database_name`
    # postgres: user=a password=b host=localhost port=5432 dbname=c sslmode=disable
    ;provider_config = sessions

    # Session cookie name
    ;cookie_name = grafana_sess

    # If you use session in https only, default is false
    ;cookie_secure = false

    # Session life time, default is 86400
    ;session_life_time = 86400

    #################################### Data proxy ###########################
    [dataproxy]

    # This enables data proxy logging, default is false
    ;logging = false


    #################################### Analytics ####################################
    [analytics]
    # Server reporting, sends usage counters to stats.grafana.org every 24 hours.
    # No ip addresses are being tracked, only simple counters to track
    # running instances, dashboard and error counts. It is very helpful to us.
    # Change this option to false to disable reporting.
    ;reporting_enabled = true

    # Set to false to disable all checks to https://grafana.net
    # for new vesions (grafana itself and plugins), check is used
    # in some UI views to notify that grafana or plugin update exists
    # This option does not cause any auto updates, nor send any information
    # only a GET request to http://grafana.net to get latest versions
    ;check_for_updates = true

    # Google Analytics universal tracking code, only enabled if you specify an id here
    ;google_analytics_ua_id =

    #################################### Security ####################################
    [security]
    # default admin user, created on startup
    # admin_user = admin

    # default admin password, can be changed before first start of grafana,  or in profile settings
    # admin_password = admin

    # used for signing
    ;secret_key = SW2YcwTIb9zpOOhoPsMm

    # Auto-login remember days
    ;login_remember_days = 7
    ;cookie_username = grafana_user
    ;cookie_remember_name = grafana_remember

    # disable gravatar profile images
    ;disable_gravatar = false

    # data source proxy whitelist (ip_or_domain:port separated by spaces)
    ;data_source_proxy_whitelist =

    [snapshots]
    # snapshot sharing options
    ;external_enabled = true
    ;external_snapshot_url = https://snapshots-origin.raintank.io
    ;external_snapshot_name = Publish to snapshot.raintank.io

    # remove expired snapshot
    ;snapshot_remove_expired = true

    # remove snapshots after 90 days
    ;snapshot_TTL_days = 90

    #################################### Users ####################################
    [users]
    # disable user signup / registration
    ;allow_sign_up = true

    # Allow non admin users to create organizations
    ;allow_org_create = true

    # Set to true to automatically assign new users to the default organization (id 1)
    ;auto_assign_org = true

    # Default role new users will be automatically assigned (if disabled above is set to true)
    ;auto_assign_org_role = Viewer

    # Background text for the user field on the login page
    ;login_hint = email or username

    # Default UI theme ("dark" or "light")
    ;default_theme = dark

    [auth]
    # Set to true to disable (hide) the login form, useful if you use OAuth, defaults to false
    ;disable_login_form = false

    #################################### Anonymous Auth ##########################
    [auth.anonymous]
    # enable anonymous access
    ;enabled = false

    # specify organization name that should be used for unauthenticated users
    ;org_name = Main Org.

    # specify role for unauthenticated users
    ;org_role = Viewer

    #################################### Github Auth ##########################
    [auth.github]
    ;enabled = false
    ;allow_sign_up = true
    ;client_id = some_id
    ;client_secret = some_secret
    ;scopes = user:email,read:org
    ;auth_url = https://github.com/login/oauth/authorize
    ;token_url = https://github.com/login/oauth/access_token
    ;api_url = https://api.github.com/user
    ;team_ids =
    ;allowed_organizations =

    #################################### Google Auth ##########################
    [auth.google]
    ;enabled = false
    ;allow_sign_up = true
    ;client_id = some_client_id
    ;client_secret = some_client_secret
    ;scopes = https://www.googleapis.com/auth/userinfo.profile https://www.googleapis.com/auth/userinfo.email
    ;auth_url = https://accounts.google.com/o/oauth2/auth
    ;token_url = https://accounts.google.com/o/oauth2/token
    ;api_url = https://www.googleapis.com/oauth2/v1/userinfo
    ;allowed_domains =

    #################################### Generic OAuth ##########################
    [auth.generic_oauth]
    ;enabled = false
    ;name = OAuth
    ;allow_sign_up = true
    ;client_id = some_id
    ;client_secret = some_secret
    ;scopes = user:email,read:org
    ;auth_url = https://foo.bar/login/oauth/authorize
    ;token_url = https://foo.bar/login/oauth/access_token
    ;api_url = https://foo.bar/user
    ;team_ids =
    ;allowed_organizations =

    #################################### Grafana.net Auth ####################
    [auth.grafananet]
    ;enabled = false
    ;allow_sign_up = true
    ;client_id = some_id
    ;client_secret = some_secret
    ;scopes = user:email
    ;allowed_organizations =

    #################################### Auth Proxy ##########################
    [auth.proxy]
    ;enabled = false
    ;header_name = X-WEBAUTH-USER
    ;header_property = username
    ;auto_sign_up = true
    ;ldap_sync_ttl = 60
    ;whitelist = 192.168.1.1, 192.168.2.1

    #################################### Basic Auth ##########################
    [auth.basic]
    ;enabled = true

    #################################### Auth LDAP ##########################
    [auth.ldap]
    ;enabled = false
    ;config_file = /etc/grafana/ldap.toml
    ;allow_sign_up = true

    #################################### SMTP / Emailing ##########################
    [smtp]
    ;enabled = false
    ;host = localhost:25
    ;user =
    # If the password contains # or ; you have to wrap it with trippel quotes. Ex """#password;"""
    ;password =
    ;cert_file =
    ;key_file =
    ;skip_verify = false
    ;from_address = admin@grafana.localhost
    ;from_name = Grafana

    [emails]
    ;welcome_email_on_sign_up = false

    #################################### Logging ##########################
    [log]
    # Either "console", "file", "syslog". Default is console and  file
    # Use space to separate multiple modes, e.g. "console file"
    ;mode = console file

    # Either "trace", "debug", "info", "warn", "error", "critical", default is "info"
    ;level = info

    # optional settings to set different levels for specific loggers. Ex filters = sqlstore:debug
    ;filters =


    # For "console" mode only
    [log.console]
    ;level =

    # log line format, valid options are text, console and json
    ;format = console

    # For "file" mode only
    [log.file]
    ;level =

    # log line format, valid options are text, console and json
    ;format = text

    # This enables automated log rotate(switch of following options), default is true
    ;log_rotate = true

    # Max line number of single file, default is 1000000
    ;max_lines = 1000000

    # Max size shift of single file, default is 28 means 1 << 28, 256MB
    ;max_size_shift = 28

    # Segment log daily, default is true
    ;daily_rotate = true

    # Expired days of log file(delete after max days), default is 7
    ;max_days = 7

    [log.syslog]
    ;level =

    # log line format, valid options are text, console and json
    ;format = text

    # Syslog network type and address. This can be udp, tcp, or unix. If left blank, the default unix endpoints will be used.
    ;network =
    ;address =

    # Syslog facility. user, daemon and local0 through local7 are valid.
    ;facility =

    # Syslog tag. By default, the process' argv[0] is used.
    ;tag =


    #################################### AMQP Event Publisher ##########################
    [event_publisher]
    ;enabled = false
    ;rabbitmq_url = amqp://localhost/
    ;exchange = grafana_events

    ;#################################### Dashboard JSON files ##########################
    [dashboards.json]
    ;enabled = false
    ;path = /var/lib/grafana/dashboards

    #################################### Alerting ############################
    [alerting]
    # Disable alerting engine & UI features
    ;enabled = true
    # Makes it possible to turn off alert rule execution but alerting UI is visible
    ;execute_alerts = true

    #################################### Internal Grafana Metrics ##########################
    # Metrics available at HTTP API Url /api/metrics
    [metrics]
    # Disable / Enable internal metrics
    ;enabled           = true

    # Publish interval
    ;interval_seconds  = 10

    # Send internal metrics to Graphite
    [metrics.graphite]
    # Enable by setting the address setting (ex localhost:2003)
    ;address =
    ;prefix = prod.grafana.%(instance_name)s.

    #################################### Internal Grafana Metrics ##########################
    # Url used to to import dashboards directly from Grafana.net
    [grafana_net]
    ;url = https://grafana.net

    #################################### External image storage ##########################
    [external_image_storage]
    # Used for uploading images to public servers so they can be included in slack/email messages.
    # you can choose between (s3, webdav)
    ;provider =

    [external_image_storage.s3]
    ;bucket_url =
    ;access_key =
    ;secret_key =

    [external_image_storage.webdav]
    ;url =
    ;username =
    ;password =
{{- end }}