groups:
- name: import
  rules:
  - alert: ImportStilSyncingAfterLongTime
    annotations:
      message: "import {{ $labels.import_id }} for cluster {{ $labels.cluster_id }} still syncing after long time."
    expr: |
      dataflow_import_syncing_import_since_created_seconds > 3600 * 24 * 2 # 2 days
    labels:
      severity: critical
      response_class: page
      component: lightning
      oncall_team: tidb_x_tools
  - alert: ImportFail
    annotations:
      message: "import {{ $labels.import_id }} for cluster {{ $labels.cluster_id }} failed: {{ $labels.msg }}."
    expr: |
      dataflow_import_fail
    for: 1m
    labels:
      severity: major
      response_class: ticket
      alerttype: event
      component: lightning
      oncall_team: tidb_x_tools
  - alert: ImportCriticalFail
    annotations:
      message: "import {{ $labels.import_id }} for an external customer's cluster {{ $labels.cluster_id }} failed: {{ $labels.msg }}."
    expr: |
      dataflow_import_fail{tenant_kind=~"external", err_type!~"SyntaxError|PreCheckFailed|SchemaNotExists|InvalidSchemaFile|ChecksumMismatch|TooManyConflict"}
    for: 1m
    labels:
      severity: critical
      response_class: page
      alerttype: event
      component: lightning
      oncall_team: tidb_x_tools
  - alert: ImportFail
    annotations:
      message: "import {{ $labels.import_id }} for an external customer's cluster {{ $labels.cluster_id }} failed: {{ $labels.msg }}."
    expr: |
      dataflow_import_fail{tenant_kind=~"external", err_type=~"SyntaxError|PreCheckFailed|SchemaNotExists|InvalidSchemaFile|ChecksumMismatch|TooManyConflict"}
    for: 1m
    labels:
      severity: major
      response_class: ticket
      alerttype: event
      component: lightning
      oncall_team: tidb_x_tools

  # - alert: SomeDownstreamErrors
  #   expr: |
  #     sum(increase(dataflow_downstream_request_duration_ms_count{ok="false"}[5m])) by (service,action) >= 3
  #   for: 5m
  #   labels:
  #     severity: major
  #     # component: tcidm-regional
  #     component: lightning
  #   annotations:
  #     message: The Downstream Request {{ $labels.service }} {{ $labels.action }} failed more than 3 times in 5 minutes

  # - alert: TooManyDownstreamErrors
  #   expr: |
  #     sum(increase(dataflow_downstream_request_duration_ms_count{ok="false"}[5m])) by (service,action) >= 10
  #   for: 5m
  #   labels:
  #     severity: critical
  #     # component: tcidm-regional
  #     component: lightning
  #   annotations:
  #     message: The Downstream Request {{ $labels.service }} {{ $labels.action }} failed more than 10 times in 5 minutes

  - alert: TooSlowDownstreamRequest
    expr: |
      histogram_quantile(0.99,
        sum(rate(dataflow_downstream_request_duration_ms{}[5m])) by (service,action,le)
      ) >= 10000
    for: 5m
    labels:
      severity: major
      response_class: ticket
      # component: tcidm-regional
      component: lightning
      oncall_team: tidb_x_tools
    annotations:
      message: The Downstream Request {{ $labels.service }} {{ $labels.action }} T99 cost time exceeds 10 seconds in 5 minutes

  - alert: ImportOverallSlow
    annotations:
      message: "import {{ $labels.import_id }} for an external customer's cluster {{ $labels.cluster_id }} overall import speed less than 200G/h."
    expr: |
      dataflow_import_slow_task{slow_type="overall_duration_long"}
    for: 10m
    labels:
      severity: major
      response_class: ticket
      alerttype: event
      component: lightning
      oncall_team: tidb_x_tools
  - alert: ImportOverallTooSlow
    annotations:
      message: "import {{ $labels.import_id }} for an external customer's cluster {{ $labels.cluster_id }} overall import speed less than 100G/h."
    expr: |
      dataflow_import_slow_task{slow_type="overall_duration_too_long"}
    for: 30m
    labels:
      severity: major
      response_class: ticket
      alerttype: event
      component: lightning
      oncall_team: tidb_x_tools

  - alert: ImportPrepareSlow
    annotations:
      message: "import {{ $labels.import_id }} for an external customer's cluster {{ $labels.cluster_id }} in preparation phase exceeds 10 minutes."
    expr: |
      dataflow_import_slow_task{slow_type="prepare_duration_gt_10m0s"}
    for: 10m
    labels:
      severity: major
      response_class: ticket
      alerttype: event
      component: lightning
      oncall_team: tidb_x_tools

  - alert: ImportPrepareTooSlow
    annotations:
      message: "import {{ $labels.import_id }} for an external customer's cluster {{ $labels.cluster_id }} in preparation phase exceeds 20 minutes."
    expr: |
      dataflow_import_slow_task{slow_type="prepare_duration_gt_20m0s"}
    for: 10m
    labels:
      severity: critical
      response_class: page
      alerttype: event
      component: lightning
      oncall_team: tidb_x_tools

  - alert: ImportEncodeSlow
    annotations:
      message: "import {{ $labels.import_id }} for an external customer's cluster {{ $labels.cluster_id }} encode speed less than 300M/s."
    expr: |
      dataflow_import_slow_task{slow_type="encode_speed_lt_300Ms"}
    for: 10m
    labels:
      severity: major
      response_class: ticket
      alerttype: event
      component: lightning
      oncall_team: tidb_x_tools
  - alert: ImportEncodeTooSlow
    annotations:
      message: "import {{ $labels.import_id }} for an external customer's cluster {{ $labels.cluster_id }} encode speed less than 150M/s."
    for: 30m
    expr: |
      dataflow_import_slow_task{slow_type="encode_speed_lt_150Ms"}
    labels:
      severity: critical
      response_class: page
      alerttype: event
      component: lightning
      oncall_team: tidb_x_tools

  - alert: ImportTiKVImportSlow
    annotations:
      message: "import {{ $labels.import_id }} for an external customer's cluster {{ $labels.cluster_id }} tikv-import speed less than 300M/s."
    expr: |
      dataflow_import_slow_task{slow_type="import_duration_long"}
    for: 10m
    labels:
      severity: major
      response_class: ticket
      alerttype: event
      component: lightning
      oncall_team: tidb_x_tools
  - alert: ImportTiKVImportTooSlow
    annotations:
      message: "import {{ $labels.import_id }} for an external customer's cluster {{ $labels.cluster_id }} tikv-import speed less than 150M/s."
    expr: |
      dataflow_import_slow_task{slow_type="import_duration_too_long"}
    for: 30m
    labels:
      severity: major
      response_class: ticket
      alerttype: event
      component: lightning
      oncall_team: tidb_x_tools

  - alert: ImportPostProcessingSlow
    annotations:
      message: "import {{ $labels.import_id }} for an external customer's cluster {{ $labels.cluster_id }} in post-processing phase long."
    expr: |
      dataflow_import_slow_task{slow_type="postproce_duration_long"}
    for: 10m
    labels:
      severity: major
      response_class: ticket
      alerttype: event
      component: lightning
      oncall_team: tidb_x_tools
  - alert: ImportPostProcessingTooSlow
    annotations:
      message: "import {{ $labels.import_id }} for an external customer's cluster {{ $labels.cluster_id }} in post-processing phase too lang."
    expr: |
      dataflow_import_slow_task{slow_type="postproce_duration_too_long"}
    for: 30m
    labels:
      severity: major
      response_class: ticket
      alerttype: event
      component: lightning
      oncall_team: tidb_x_tools