groups:
  - name: cluster-service-workflow-controller.rules
    rules:
      - alert: ClusterServiceWorkflowFailed
        expr: |
          (infra_exporter_workflow{phase="Failed"} >0)
          and on(exported_namespace,name) 
          (min_over_time(infra_exporter_workflow{phase!="Failed"}[5m]) > 0)
        for: 0m
        labels:
          component: infra-provider
          oncall_team: dedicated_service
          service: infra-provider
          controller: workflow
          severity: critical
          response_class: page
          type: infra
        annotations:
          description: The Workflow {{ $labels.exported_namespace }}/{{ $labels.name }} failed.
      - alert: ClusterServiceWorkflowDurationTooLong
        expr: |
          infra_exporter_workflow{phase!~"Completed|Failed|Cancelled", paused!="true"} >= 0
        for: 2h
        labels:
          component: infra-provider
          oncall_team: dedicated_service
          service: infra-provider
          controller: workflow
          severity: major
          response_class: ticket
          type: infra
        annotations:
          description: The Workflow {{ $labels.exported_namespace }}/{{ $labels.name }} runs too long than 2 hours.
      - alert: ClusterServiceWorkflowDurationTooLongThan24Hours
        expr: |
          infra_exporter_workflow{phase!~"Completed|Failed|Cancelled", paused!="true"} >= 0
        for: 24h
        labels:
          component: infra-provider
          oncall_team: dedicated_service
          service: infra-provider
          controller: workflow
          severity: major
          response_class: ticket
          type: infra
        annotations:
          description: The Workflow {{ $labels.exported_namespace }}/{{ $labels.name }} runs too long than 24 hours.
      - alert: ClusterServiceWorkflowDurationTooLongThan48Hours
        expr: |
          infra_exporter_workflow{phase!~"Completed|Failed|Cancelled", paused!="true"} >= 0
        for: 48h
        labels:
          component: infra-provider
          oncall_team: dedicated_service
          service: infra-provider
          controller: workflow
          severity: major
          response_class: ticket
          type: infra
        annotations:
          description: The Workflow {{ $labels.exported_namespace }}/{{ $labels.name }} runs too long than 48 hours.
      - alert: ClusterServiceWorkflowGetStuckOnOneStepFor24Hours
        expr: |
          infra_exporter_workflow{phase="Running", paused!="true"} -  infra_exporter_workflow{phase="Running", paused!="true"} offset 24h
        for: 1m
        labels:
          component: infra-provider
          oncall_team: dedicated_service
          service: infra-provider
          controller: workflow
          severity: major
          response_class: ticket
          type: infra
        annotations:
          description: The Workflow {{ $labels.exported_namespace }}/{{ $labels.name }} gets stuck at progress {{ printf "%.2f" $value }}.
      - alert: ClusterServiceWorkflowCompleted
        expr: |
          (infra_exporter_workflow{phase="Completed"} >0)
          and on(exported_namespace,name) 
          (min_over_time(infra_exporter_workflow{phase!="Completed"}[5m]) > 0)
        for: 0m
        labels:
          component: infra-provider
          oncall_team: dedicated_service
          service: infra-provider
          controller: workflow
          severity: warning  # just for notify
          intent: cluster-service-remind-only
          type: infra
        annotations:
          description: The Workflow {{ $labels.exported_namespace }}/{{ $labels.name }} is Completed.
      - alert: ClusterServiceWorkflowCancelled
        expr: |
          (infra_exporter_workflow{phase="Cancelled"} >0)
          and on(exported_namespace,name) 
          (min_over_time(infra_exporter_workflow{phase!="Cancelled"}[5m]) > 0)
        for: 0m
        labels:
          component: infra-provider
          oncall_team: dedicated_service
          service: infra-provider
          controller: workflow
          severity: warning  # just for notify
          intent: cluster-service-remind-only
          type: infra
        annotations:
          description: The Workflow {{ $labels.exported_namespace }}/{{ $labels.name }} is Cancelled.