groups:
  - name: next-gen.br
    rules:
      # api related alerts
      - alert: NextgenBrOpenAPI5XXError
        expr: |
          100 *
          (
            sum by (grpc_service, grpc_method) (
              rate(
                grpc_server_handled_total{
                  namespace="tidb-management-service",
                  job="tidb-mgmt-service",
                  grpc_code=~"Code.5.+",
                  grpc_method=~".*(Backup|Restore|backup|restore|ListDeletedTidbs).*"
                }[5m]
              )
            )
            /
            clamp_min(
              sum by (grpc_service, grpc_method) (
                rate(
                  grpc_server_handled_total{
                    namespace="tidb-management-service",
                    job="tidb-mgmt-service",
                    grpc_method=~".*(Backup|Restore|backup|restore|ListDeletedTidbs).*"
                  }[5m]
                )
              ),
              1
            )
          ) > 5
        for: 3m
        labels:
          severity: critical
          response_class: page
          tier: next-gen
          component: next-gen-br
          oncall_team: tidb_x_tools
        annotations:
          message: Internal error (bizcode 5xxxxxxx) rate {{ printf "%.2f" $value }}% grpc_service={{ $labels.grpc_service }} grpc_method={{ $labels.grpc_method }}
      - alert: NextgenBrInternalAPI5XXError
        expr: |
          100 *
          (
            sum by (grpc_service, grpc_method) (
              rate(
                grpc_server_handled_total{
                  namespace="tidb-management-service",
                  job="nextgen-global-service",
                  grpc_code=~"Code.5.+",
                  grpc_method=~".*(Backup|Restore|backup|restore).*"
                }[5m]
              )
            )
            /
            clamp_min(
              sum by (grpc_service, grpc_method) (
                rate(
                  grpc_server_handled_total{
                    namespace="tidb-management-service",
                    job="nextgen-global-service",
                    grpc_method=~".*(Backup|Restore|backup|restore).*"
                  }[5m]
                )
              ),
              1
            )
          ) > 5
        for: 3m
        labels:
          severity: major
          response_class: ticket
          tier: next-gen
          component: next-gen-br
          oncall_team: tidb_x_tools
        annotations:
          message: Internal error (bizcode 5xxxxxxx) rate {{ printf "%.2f" $value }}% grpc_service={{ $labels.grpc_service }} grpc_method={{ $labels.grpc_method }}
      # restore workflow related alerts
      - alert: NextGenGlobalRestoreWorkflowRunning3h
        annotations:
          message: Nextgen-global restore workflow > 3h. tenant_id={{ $labels.tenant_id }} biz_id={{ $labels.biz_id }} job={{ $labels.job }} dag={{ $labels.workflow_dag }} dag_ins_id={{ $labels.dag_ins_id }}
        expr: |
          max by (tenant_id, biz_id, workflow_dag, job, dag_ins_id) (
            workflow_info{
              namespace=~"tidb-management-service",
              job="nextgen-global-service",
              workflow_dag="restore_cluster"
            }
          ) > 10800
        for: 3m
        labels:
          severity: major
          response_class: ticket
          component: next-gen-br
          oncall_team: tidb_x_tools
      - alert: NextGenGlobalRestoreTerminalKernelFailure
        annotations:
          message: Nextgen-global restore workflow terminal kernel failure. cluster_id={{ $labels.cluster_id }} restore_mode={{ $labels.restore_mode }} action={{ $labels.action }} failure_code={{ $labels.failure_code }}
        # Terminal failures increment the counter exactly once. increase() covers
        # repeated failures on an existing series; the unless-offset term covers
        # the first failure of a fresh series, where increase() would stay 0 on
        # engines without counter created-timestamp correction (Prometheus 2.x, VM).
        expr: |
          (
            max by (cluster_id, restore_mode, action, failure_code) (increase(restore_terminal_failures_total{failure_source="kernel"}[5m])) > 0
          )
          or
          (
            max by (cluster_id, restore_mode, action, failure_code) (restore_terminal_failures_total{failure_source="kernel"})
            unless
            max by (cluster_id, restore_mode, action, failure_code) (restore_terminal_failures_total{failure_source="kernel"} offset 10m)
          )
        labels:
          severity: critical
          response_class: page
          component: next-gen-br-kernel
          oncall_team: br
      - alert: NextGenGlobalRestoreTerminalFailure
        annotations:
          message: Nextgen-global restore workflow terminal failure. cluster_id={{ $labels.cluster_id }} restore_mode={{ $labels.restore_mode }} action={{ $labels.action }} failure_source={{ $labels.failure_source }} failure_code={{ $labels.failure_code }}
        # See NextGenGlobalRestoreTerminalKernelFailure for why the or/unless-offset form is used.
        expr: |
          (
            max by (cluster_id, restore_mode, action, failure_source, failure_code) (increase(restore_terminal_failures_total{failure_source!="kernel"}[5m])) > 0
          )
          or
          (
            max by (cluster_id, restore_mode, action, failure_source, failure_code) (restore_terminal_failures_total{failure_source!="kernel"})
            unless
            max by (cluster_id, restore_mode, action, failure_source, failure_code) (restore_terminal_failures_total{failure_source!="kernel"} offset 10m)
          )
        labels:
          severity: critical
          response_class: page
          component: next-gen-br
          oncall_team: tidb_x_tools
      # backup related alerts
      - alert: NextGenGlobalBackupSettingInitFailed
        annotations:
          message: Nextgen-global backup setting init failed. cluster_id={{ $labels.cluster_id }}
        expr: increase(create_backup_setting_action_failures_total[5m]) > 0
        for: 5m
        labels:
          severity: critical
          response_class: page
          component: next-gen-br
          oncall_team: tidb_x_tools
      - alert: NextGenGlobalBackupSyncError
        annotations:
          message: Nextgen-global backup sync error. cluster_id={{ $labels.cluster_id }}, shared_pool_id={{ $labels.shared_pool_id }}, error_type={{ $labels.error_type }}
        expr: increase(backup_normal_sync_error_total{error_type!="backup_missing"}[1h]) > 0
        for: 90m # backup is schedule hourly, make it bigger than 1h to cover 2 times schedule, if two times backup sync failed, it will trigger this alert
        labels:
          severity: critical
          response_class: page
          component: next-gen-br
          oncall_team: tidb_x_tools
      - alert: NextGenGlobalStillMissingAfter2TimesResync
        annotations:
          message: Nextgen-global backup still missing after 2 times resync. cluster_id={{ $labels.cluster_id }}, shared_pool_id={{ $labels.shared_pool_id }}
        expr: increase(backup_sync_missing_total[2h]) > 0
        for: 5h # sync missing per 2 hours, make it bigger than 4h to cover 3 times schedule, if 3 times sync missing failed, it will trigger this alert
        labels:
          severity: critical
          response_class: page
          component: next-gen-br
          oncall_team: tidb_x_tools
      - alert: NextGenGlobalBackupSyncMissingError
        annotations:
          message: Nextgen-global backup sync missing error. cluster_id={{ $labels.cluster_id }}, shared_pool_id={{ $labels.shared_pool_id }}, error_type={{ $labels.error_type }}
        expr: increase(backup_sync_missing_error_total{error_type!="backup_missing"}[2h]) > 0
        for: 150m # sync missing per 2 hours, make it bigger than 120m to cover 2 times schedule
        labels:
          severity: major
          response_class: ticket
          component: next-gen-br
          oncall_team: tidb_x_tools
      - alert: NextGenGlobalExpireBackupError
        annotations:
          message: Nextgen-global expire backup error. cluster_id={{ $labels.cluster_id }}, shared_pool_id={{ $labels.shared_pool_id }}, error_type={{ $labels.error_type }}
        expr: increase(expire_backups_error_total[2h]) > 0
        for: 150m # expire backup per 2 hours, make it bigger than 120m to cover 2 times schedule
        labels:
          severity: major
          response_class: ticket
          component: next-gen-br
          oncall_team: tidb_x_tools