# use infra-layer metrics to alert such case
groups:
- name: import-controller-resource-operations
  rules:

  - alert: ImportControllerReconcileSlow
    annotations:
      message: The Import Controller Reconcile in {{ $labels.provider }} of {{ $labels.region }} T99 cost time exceeds 10 seconds in 5 minutes
    expr: |
      histogram_quantile(0.99,
        sum(rate(infra_controller_provider_reconcile_duration_ms{controller="import-3layer"}[5m])) by (provider,region,le)
      ) >= 10000
    for: 5m
    labels:
      severity: major
      response_class: ticket
      component: import-controller
      oncall_team: tidb_x_tools

  - alert: ImportControllerReconcileFailed
    annotations:
      message: The Import Controller Reconcile {{ $labels.sence }} failed in {{ $labels.provider }} of {{ $labels.region }} in 5 minutes
    expr: |
      sum(increase(infra_import_import_fail[5m])) by (provider, region, sence) >= 20
    for: 5m
    labels:
      severity: major
      response_class: ticket
      component: import-controller
      oncall_team: tidb_x_tools

  - alert: TooManyImportControllerReconcileFailed
    annotations:
      message: The Import Controller Reconcile {{ $labels.sence }} failed in {{ $labels.provider }} of {{ $labels.region }} exceeds 20 times in 5 minutes
    expr: |
      sum(increase(infra_import_import_fail[5m])) by (provider, region, sence) >= 100
    for: 5m
    labels:
      severity: critical
      response_class: page
      component: import-controller
      oncall_team: tidb_x_tools

  - alert: ImportControllerTaskSlow
    annotations:
      message: The Import Controller Reconcile task {{ $labels.task }}/{{ $labels.step }} in {{ $labels.provider }} of {{ $labels.region }} T99 cost time exceeds 10 seconds in 5 minutes
    expr: |
      histogram_quantile(0.99,
        sum(rate(infra_controller_controller_task_duration_ms{controller="import-3layer"}[5m])) by (provider,region,task,step,le)
      ) >= 10000
    for: 5m
    labels:
      severity: major
      response_class: ticket
      component: import-controller
      oncall_team: tidb_x_tools

  - alert: ImportControllerTaskFailed
    annotations:
      message: The Import Controller Reconcile task {{ $labels.task }}/{{ $labels.step }} failed in {{ $labels.provider }} of {{ $labels.region }} in 5 minutes
    expr: |
      sum(increase(infra_controller_controller_task_duration_ms_count{controller="import-3layer",ok="false"}[5m])) by (provider,region,task,step) >= 3
    labels:
      severity: major
      response_class: ticket
      component: import-controller
      oncall_team: tidb_x_tools

  - alert: TooManyImportControllerTaskFailed
    annotations:
      message: The Import Controller Reconcile task {{ $labels.task }}/{{ $labels.step }} failed in {{ $labels.provider }} of {{ $labels.region }} exceeds 20 times in 5 minutes
    expr: |
      sum(increase(infra_controller_controller_task_duration_ms_count{controller="import-3layer",ok="false"}[5m])) by (provider,region,task,step) >= 20
    labels:
      severity: critical
      response_class: page
      component: import-controller
      oncall_team: tidb_x_tools