groups:
  - name: nextgen-tiflash-errorlog-governance
    interval: 30s
    rules:
    - alert: TiFlashMemoryExceededLimitNextGen
      expr: |
        max by (tidb_cluster_id) (sum by (tidb_cluster_id, instance_id) (
          count_over_time(
            {cluster_env="prod", project_id = "", container="tiflash"}
              |= `SegmentReader.cpp`
              |= `ErrMsg: Memory limit (total) exceeded caused by 'RSS(Resident Set Size) much larger than limit'`
            [1m]
          ))
        ) > 4
      for: 0s
      labels:
        severity: major
        response_class: ticket
        component: tiflash
        oncall_team: tiflash
        tier: next-gen
        stability_governance: nextgen-errorlog-governance
      annotations:
        summary: "Query failure caused by high memory pressure on TiFlash"
        message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.tidb_cluster_id }} reported "ErrMsg: Memory limit (total) exceeded caused by RSS(Resident Set Size) much larger than limit" It may be that the query plan is wrong, or a new business query is added, or other problems cause the tiflash memory to be high. It is recommended to contact R&D for processing'
    - alert: TiFlashTLSHandshakeErrorNextGen
      expr: |
        max by (tidb_cluster_id) (sum by (tidb_cluster_id, instance_id) (
          count_over_time(
            {cluster_env="prod", project_id = "", container="tiflash"}
              |= `status_server.rs`
              |= `Status server error: TLS handshake error`
            [1m]
          ))
        ) > 1
      for: 4m
      labels:
        severity: critical
        response_class: page
        component: tiflash
        oncall_team: tiflash
        tier: next-gen
        stability_governance: nextgen-errorlog-governance
      annotations:
        summary: ""
        message: ''
    - alert: TiFlashRowKeyMetUnexpectedEncodingFormatNextGen
      expr: |
        max by (tidb_cluster_id) (sum by (tidb_cluster_id, instance_id) (
          count_over_time(
            {cluster_env="prod", project_id = "", container="tiflash"}
              |= `RowKeyRange.h`
              |= `unexpected encoding format`
            [1m]
          ))
        ) > 0
      for: 0s
      labels:
        severity: critical
        response_class: page
        component: tiflash
        oncall_team: tiflash
        tier: next-gen
        stability_governance: nextgen-errorlog-governance
      annotations:
        summary: "TiFlash RowKey Met Unexpected Encoding Format"
        message: 'TiFlash instance {{ $labels.instance }} in cluster {{ $labels.tidb_cluster_id }} detected "unexpected encoding format" in RowKey. This indicates **data inconsistency** caused by an irregular key. This alert ensures that even if `profiles.default.dt_enable_ingest_check` is set to `false`, data inconsistency issues due to irregular keys are detected via logs instead of causing a TiFlash panic. An SOP can then be followed to **retrieve logs and identify affected regions for manual data inconsistency repair**. **Contact R&D directly** to handle this alert.'
    - alert: TiFlashMonotonicTimeJumpedBackNextGen
      expr: |
        max by (tidb_cluster_id) (sum by (tidb_cluster_id, instance_id) (
          count_over_time(
            {cluster_env="prod", project_id = "", container="tiflash"}
              |= `time.rs`
              |= `monotonic time jumped back`
            [1m]
          ))
        ) > 1
      for: 1m
      labels:
        severity: major
        response_class: ticket
        component: tiflash
        oncall_team: tiflash
        tier: next-gen
        stability_governance: nextgen-errorlog-governance
      annotations:
        summary: "TiFlash Monotonic Time Jumped Back"
        message: 'TiFlash instance {{ $labels.instance }} in cluster {{ $labels.tidb_cluster_id }} reported "monotonic time jumped back" errors more than 1 time in the last 5 minutes. This signifies a **clock jump (time callback)**, which is **severe** and could potentially lead to **cluster crashes**. **Contact R&D directly** upon this alert.'
    - alert: TiFlashResolveStoreAddressFailedNextGen
      expr: |
        max by (tidb_cluster_id) (sum by (tidb_cluster_id, instance_id) (
          count_over_time(
            {cluster_env="prod", project_id = "", container="tiflash"}
              |= `raft_client.rs`
              |= `resolve store address failed`
              !~ "has been removed"
            [1m]
          ))
        ) > 4
      for: 1m
      labels:
        severity: major
        response_class: ticket
        component: tiflash
        oncall_team: tiflash
        tier: next-gen
        stability_governance: nextgen-errorlog-governance
      annotations:
        summary: "TiFlash Resolve Store Address Failed"
        message: 'TiFlash instance {{ $labels.instance }} in cluster {{ $labels.tidb_cluster_id }} reported "resolve store address failed" errors more than 4 times in the last 1 minute. The inability to resolve store addresses, if persistent, will **lead to KV node failures**. This could be due to **disk offline events** or **old metadata not being properly cleaned up** after a disk was decommissioned. First, check if the disk is offline. If the disk is mounted and the error log shows a `store_id` mismatch, try using `tikv_ctl remove-fail-stores` or `pd_ctl store delete` to remove the old `store_id` information. If unresolved, **contact R&D**.'
    - alert: TiFlashRSSMuchLargerThanLimitNextGen
      expr: |
        max by (tidb_cluster_id) (sum by (tidb_cluster_id, instance_id) (
          count_over_time(
            {cluster_env="prod", project_id = "", container="tiflash"}
              |= `DAGDriver.cpp`
              |= `[FLASH:Coprocessor:MemoryLimitExceeded] Memory limit (total) exceeded caused by 'RSS(Resident Set Size) much larger than limit'`
            [1m]
          ))
        ) > 4
      for: 0s
      labels:
        severity: major
        response_class: ticket
        component: tiflash
        oncall_team: tiflash
        tier: next-gen
        stability_governance: nextgen-errorlog-governance
      annotations:
        summary: "TiFlash RSS Much Larger Than Limit"
        message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.tidb_cluster_id }} reported "[FLASH:Coprocessor:MemoryLimitExceeded] Memory limit (total) exceeded caused by "RSS(Resident Set Size) much larger than limit" The query failed due to high TiFlash memory pressure. This could be due to an incorrect query plan, a newly added business query, or other issues causing high TiFlash memory usage. We recommend contacting our oncall team for assistance.'
    - alert: TiFlashRSSMuchLargerThanLimitExceptionNextGen
      expr: |
        max by (tidb_cluster_id) (sum by (tidb_cluster_id, instance_id) (
          count_over_time(
            {cluster_env="prod", project_id = "", container="tiflash"}
              |= `Exception.cpp`
              |= `RSS(Resident Set Size) much larger than limit`
            [1m]
          ))
        ) > 4
      for: 0s
      labels:
        severity: major
        response_class: ticket
        component: tiflash
        oncall_team: tiflash
        tier: next-gen
        stability_governance: nextgen-errorlog-governance
      annotations:
        summary: "TiFlash RSS Much Larger Than Limit (Exception)"
        message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.tidb_cluster_id }} reported "RSS(Resident Set Size) much larger than limit" The query failed due to high TiFlash memory pressure. This could be due to an incorrect query plan, a newly added business query, or other issues causing high TiFlash memory usage. We recommend contacting our oncall team for assistance.'
    - alert: TiFlashSyncSchemaFailedNextGen
      expr: |
        max by (tidb_cluster_id) (sum by (tidb_cluster_id, instance_id) (
          count_over_time(
            {cluster_env="prod", project_id = "", container="tiflash"}
              |= `SchemaSyncService.cpp`
              |= `RSS(Resident Set Size) much larger than limit`
            [1m]
          ))
        ) > 9
      for: 0s
      labels:
        severity: major
        response_class: ticket
        component: tiflash
        oncall_team: tiflash
        tier: next-gen
        stability_governance: nextgen-errorlog-governance
      annotations:
        summary: "TiFlash Sync Schema Failed"
        message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.tidb_cluster_id }} reported "RSS(Resident Set Size) much larger than limit" Broadly speaking, this is an error caused by the inability to connect to the PD.'