groups:
  - name: subscribe.mgmt.workflow
    rules:
      - alert: WorkflowRunningTooLong
        annotations:
          message: workflow {{ $labels.workflow_dag }} is taking more then {{ $value }}s for 5m. [Diagnosis Board](https://clinic.pingcap.com/grafana/d/0hqVrAVvz/serverless-global-service-monitor?orgId=1)
        expr: |
          max(workflow_info{namespace="tidb-management-service",job="serverless-global-service"}) by (workflow_dag) > 120
        for: 5m
        labels:
          severity: major
          response_class: ticket
          component: serverless-svc
          oncall_team: serverless
          server: serverless-global-service
          provider_type: aws-free-tier
      - alert: TooManyCreatingServerlesWorkflow
        annotations:
          message: there are {{ $value }} pending {{ $labels.workflow_dag }} events in workflow dag. [Diagnosis Board](https://clinic.pingcap.com/grafana/d/0hqVrAVvz/serverless-global-service-monitor?orgId=1)
        expr: |
          count(workflow_info{namespace="tidb-management-service",job="serverless-global-service"}) by (workflow_dag) > 10
        for: 5m
        labels:
          severity: major
          response_class: ticket
          component: serverless-svc
          oncall_team: serverless
          server: serverless-global-service
          provider_type: aws-free-tier
      - alert: ImportantCustomerCreatingServerlessWorkflowRunningTooLong
        annotations:
          message: important customer tenant {{ $labels.tenant_id }} has create_cluster workflow running more than 10 minutes. cluster={{ $labels.biz_id }} project={{ $labels.project_id }}. [Diagnosis Board](https://clinic.pingcap.com/grafana/d/0hqVrAVvz/serverless-global-service-monitor?orgId=1)
        # Kimi/Manus tenant IDs.
        expr: |
          max(workflow_info{namespace="tidb-management-service",job="serverless-global-service",workflow_dag="create_cluster",tenant_id=~"1372813089209301246|1372813089209245725|1372813089209260919"}) by (tenant_id, project_id, biz_id) > 10 * 60
        for: 1m
        labels:
          severity: critical
          response_class: page
          component: serverless-svc
          oncall_team: serverless
          server: serverless-global-service
          provider_type: aws-free-tier
      - alert: RetryWorkflowInstanceFailed
        annotations:
          message: Workflow dag instance {{ $labels.dag_ins_id }} of {{ $labels.job }}/{{ $labels.workflow_dag }} for project {{ $labels.project_id }} on region {{ $labels.target_region }} failed to retry. [Diagnosis Board](https://clinic.pingcap.com/grafana/d/0hqVrAVvz/serverless-global-service-monitor?orgId=1)
        expr: |
          increase(workflow_watchdog_retried_dag_instance_total{namespace=~"tidb-management-service",job="serverless-global-service",retry_command_error="true"}[1m]) > 10
        for: 1m
        labels:
          severity: major
          response_class: ticket
          component: serverless-svc
          oncall_team: serverless
          provider_type: aws-free-tier
          server: serverless-global-service