groups:
- name: SLOsForDataflow
  rules:
  - alert: SLODataflowHttpApiAvailabilityBreakRisk
    annotations:
      message: |
        SLO 'Dataflow HTTP API Availability' Break Risk, Runbook: https://clinic.pingcap.com/grafana/d/QIw-n_l4k/slo-global-control-plane?orgId=1
    expr: |
      sum(rate(kong_custom_http_status{kube_service=~"prod-ms.dataflow-http.*", code!~"5.*", path!~".*/imports.*|.*/import-target-tables.*|.*/data-ingestion-profile.*|.*/dataflow.*"}[10m]))
      /
      sum(rate(kong_custom_http_status{kube_service=~"prod-ms.dataflow-http.*", path!~".*/imports.*|.*/import-target-tables.*|.*/data-ingestion-profile.*|.*/dataflow.*"}[10m]))
      < 0.9995
    for: 5m
    labels:
      severity: major
      response_class: ticket
      component: dataflow-service
      oncall_team: tidb_x_tools
  - alert: SLODataflowHttpApiLatencyP99BreakRisk
    annotations:
      message: |
        SLO 'Dataflow HTTP API Latency P99' Break Risk, P99 should < 2s, Runbook: https://clinic.pingcap.com/grafana/d/QIw-n_l4k/slo-global-control-plane?orgId=1
    expr: |
      (
        1-(sum(rate(kong_custom_request_latency_ms_bucket{kube_service=~"prod-ms.dataflow-http.*", path!~".*/imports.*|.*/import-target-tables.*|.*/data-ingestion-profile.*|.*/dataflow.*|.*/backup.*", le="2000"}[1h]))
        /
        sum(rate(kong_custom_request_latency_ms_count{kube_service=~"prod-ms.dataflow-http.*", path!~".*/imports.*|.*/import-target-tables.*|.*/data-ingestion-profile.*|.*/dataflow.*|.*/backup.*"}[1h]))) 
        > 16.8*0.01
      ) and (
        1-(sum(rate(kong_custom_request_latency_ms_bucket{kube_service=~"prod-ms.dataflow-http.*", path!~".*/imports.*|.*/import-target-tables.*|.*/data-ingestion-profile.*|.*/dataflow.*|.*/backup.*", le="2000"}[5m]))
        /
        sum(rate(kong_custom_request_latency_ms_count{kube_service=~"prod-ms.dataflow-http.*", path!~".*/imports.*|.*/import-target-tables.*|.*/data-ingestion-profile.*|.*/dataflow.*|.*/backup.*"}[5m])))
        > 16.8*0.01
      )
    labels:
      severity: major
      response_class: ticket
      component: dataflow-service
      oncall_team: tidb_x_tools
  - alert: SLODataflowHttpApiLatencyP90BreakRisk
    annotations:
      message: |
        SLO 'Dataflow HTTP API Latency P90' Break Risk, P90 should < 1s, Runbook: https://clinic.pingcap.com/grafana/d/QIw-n_l4k/slo-global-control-plane?orgId=1
    expr: |
      (
        1-(sum(rate(kong_custom_request_latency_ms_bucket{kube_service=~"prod-ms.dataflow-http.*", path!~".*/imports.*|.*/import-target-tables.*|.*/data-ingestion-profile.*|.*/dataflow.*|.*/backup.*", le="1000"}[1h]))
        /
        sum(rate(kong_custom_request_latency_ms_count{kube_service=~"prod-ms.dataflow-http.*", path!~".*/imports.*|.*/import-target-tables.*|.*/data-ingestion-profile.*|.*/dataflow.*|.*/backup.*"}[1h]))) 
        > 2.8*0.1
      ) and (
        1-(sum(rate(kong_custom_request_latency_ms_bucket{kube_service=~"prod-ms.dataflow-http.*", path!~".*/imports.*|.*/import-target-tables.*|.*/data-ingestion-profile.*|.*/dataflow.*|.*/backup.*", le="1000"}[5m]))
        /
        sum(rate(kong_custom_request_latency_ms_count{kube_service=~"prod-ms.dataflow-http.*", path!~".*/imports.*|.*/import-target-tables.*|.*/data-ingestion-profile.*|.*/dataflow.*|.*/backup.*"}[5m])))
        > 2.8*0.1
      )
    labels:
      severity: major
      response_class: ticket
      component: dataflow-service
      oncall_team: tidb_x_tools
  - alert: SLODataflowRestoreHttpApiLatencyP90BreakRisk
    annotations:
      message: |
        SLO 'Dataflow Restore HTTP API Latency P90' Break Risk, P90 should < 5s, Runbook: https://clinic.pingcap.com/grafana/d/QIw-n_l4k/slo-global-control-plane?orgId=1
    expr: |
      (
        1-(sum(rate(kong_custom_request_latency_ms_bucket{kube_service=~"prod-ms.dataflow-http.*", path=~".*/backup.*", le="10000"}[1h]))
        /
        sum(rate(kong_custom_request_latency_ms_count{kube_service=~"prod-ms.dataflow-http.*", path=~".*/backup.*"}[1h]))) 
        > 2.8*0.1
      ) and (
        1-(sum(rate(kong_custom_request_latency_ms_bucket{kube_service=~"prod-ms.dataflow-http.*", path=~".*/backup.*", le="10000"}[5m]))
        /
        sum(rate(kong_custom_request_latency_ms_count{kube_service=~"prod-ms.dataflow-http.*", path=~".*/backup.*"}[5m])))
        > 2.8*0.1
      )
    labels:
      severity: major
      response_class: ticket
      component: dataflow-service
      oncall_team: tidb_x_tools
  - alert: SLODataflowGrpcApiAvailabilityBreakRisk
    annotations:
      message: SLO 'Dataflow GRPC API Availability' Break Risk
    expr: |
      sum(rate(grpc_server_handled_total{namespace="prod-ms", pod=~"dataflow-grpc.*|dataflow-http.*", grpc_type="unary", grpc_code!="UNKNOWN|DEADLINE_EXCEEDED|INTERNAL|UNAVAILABLE|DATA_LOSS", grpc_service!~".*Import.*|^dm.Service$"}[10m]))
      /
      sum(rate(grpc_server_handled_total{namespace="prod-ms", pod=~"dataflow-grpc.*|dataflow-http.*", grpc_type="unary", grpc_service!~".*Import.*|^dm.Service$"}[10m]))
      < 0.9995
    for: 5m
    labels:
      severity: major
      response_class: ticket
      component: dataflow-service
      oncall_team: tidb_x_tools
  - alert: SLODataflowGrpcApiLatencyP99BreakRisk
    annotations:
      message: |
        SLO 'Dataflow GRPC API Latency P99' Break Risk, P99 should < 2.5s, Runbook: https://clinic.pingcap.com/grafana/d/QIw-n_l4k/slo-global-control-plane?orgId=1
    expr: |
      (
        1-(sum(rate(grpc_server_handling_seconds_bucket{namespace=~"nightly-ms|staging-ms|prod-ms", pod=~"dataflow-grpc.*|dataflow-http.*", grpc_type="unary", grpc_service!~".*Import.*|^dm.Service$", grpc_method!~".*Restore.*", le="2.5"}[1h]))
        /
        sum(rate(grpc_server_handling_seconds_count{namespace=~"nightly-ms|staging-ms|prod-ms", pod=~"dataflow-grpc.*|dataflow-http.*", grpc_type="unary", grpc_service!~".*Import.*|^dm.Service$", grpc_method!~".*Restore.*"}[1h])))
        > 16.8*0.01
      ) and (
        1-(sum(rate(grpc_server_handling_seconds_bucket{namespace=~"nightly-ms|staging-ms|prod-ms", pod=~"dataflow-grpc.*|dataflow-http.*", grpc_type="unary", grpc_service!~".*Import.*|^dm.Service$", grpc_method!~".*Restore.*", le="2.5"}[5m]))
        /
        sum(rate(grpc_server_handling_seconds_count{namespace=~"nightly-ms|staging-ms|prod-ms", pod=~"dataflow-grpc.*|dataflow-http.*", grpc_type="unary", grpc_service!~".*Import.*|^dm.Service$", grpc_method!~".*Restore.*"}[5m])))
        > 16.8*0.01
      )
    labels:
      severity: major
      response_class: ticket
      component: dataflow-service
      oncall_team: tidb_x_tools
  - alert: SLODataflowGrpcApiLatencyP90BreakRisk
    annotations:
      message: |
        SLO 'Dataflow GRPC API Latency P90' Break Risk, P90 should < 1s, Runbook: https://clinic.pingcap.com/grafana/d/QIw-n_l4k/slo-global-control-plane?orgId=1
    expr: |
      (
        1-(sum(rate(grpc_server_handling_seconds_bucket{namespace=~"nightly-ms|staging-ms|prod-ms", pod=~"dataflow-grpc.*|dataflow-http.*", grpc_type="unary", grpc_service!~".*Import.*|^dm.Service$", grpc_method!~".*Restore.*", le="1"}[1h]))
        /
        sum(rate(grpc_server_handling_seconds_count{namespace=~"nightly-ms|staging-ms|prod-ms", pod=~"dataflow-grpc.*|dataflow-http.*", grpc_type="unary", grpc_service!~".*Import.*|^dm.Service$", grpc_method!~".*Restore.*"}[1h])))
        > 2.8*0.1
      ) and (
        1-(sum(rate(grpc_server_handling_seconds_bucket{namespace=~"nightly-ms|staging-ms|prod-ms", pod=~"dataflow-grpc.*|dataflow-http.*", grpc_type="unary", grpc_service!~".*Import.*|^dm.Service$", grpc_method!~".*Restore.*", le="1"}[5m]))
        /
        sum(rate(grpc_server_handling_seconds_count{namespace=~"nightly-ms|staging-ms|prod-ms", pod=~"dataflow-grpc.*|dataflow-http.*", grpc_type="unary", grpc_service!~".*Import.*|^dm.Service$", grpc_method!~".*Restore.*"}[5m])))
        > 2.8*0.1
      )
    labels:
      severity: major
      response_class: ticket
      component: dataflow-service
      oncall_team: tidb_x_tools
  - alert: SLODataflowRestoreGrpcApiLatencyP90BreakRisk
    annotations:
      message: |
        SLO 'Dataflow Restore GRPC API Latency P90' Break Risk, P90 should < 5s, Runbook: https://clinic.pingcap.com/grafana/d/QIw-n_l4k/slo-global-control-plane?orgId=1
    expr: |
      (
        1-(sum(rate(grpc_server_handling_seconds_bucket{namespace=~"nightly-ms|staging-ms|prod-ms", pod=~"dataflow-grpc.*|dataflow-http.*", grpc_type="unary", grpc_service!~".*Import.*|^dm.Service$", grpc_method=~".*Restore.*", le="10"}[1h]))
        /
        sum(rate(grpc_server_handling_seconds_count{namespace=~"nightly-ms|staging-ms|prod-ms", pod=~"dataflow-grpc.*|dataflow-http.*", grpc_type="unary", grpc_service!~".*Import.*|^dm.Service$", grpc_method=~".*Restore.*"}[1h])))
        > 2.8*0.1
      ) and (
        1-(sum(rate(grpc_server_handling_seconds_bucket{namespace=~"nightly-ms|staging-ms|prod-ms", pod=~"dataflow-grpc.*|dataflow-http.*", grpc_type="unary", grpc_service!~".*Import.*|^dm.Service$", grpc_method=~".*Restore.*", le="10"}[5m]))
        /
        sum(rate(grpc_server_handling_seconds_count{namespace=~"nightly-ms|staging-ms|prod-ms", pod=~"dataflow-grpc.*|dataflow-http.*", grpc_type="unary", grpc_service!~".*Import.*|^dm.Service$", grpc_method=~".*Restore.*"}[5m])))
        > 2.8*0.1
      )
    labels:
      severity: major
      response_class: ticket
      component: dataflow-service
      oncall_team: tidb_x_tools

- name: SLOsForImport
  rules:
    - alert: SLOImportHttpApiAvailabilityBreakRisk
      annotations:
        message: SLO 'Import HTTP API Availability' Break Risk
      expr: |
        sum(rate(kong_custom_http_status{kube_service=~"prod-ms.dataflow-http.*", code!~"5.*", path=~".*/imports.*|.*/import-target-tables.*|.*/data-ingestion-profile.*|.*/dataflow.*"}[10m]))
        /
        sum(rate(kong_custom_http_status{kube_service=~"prod-ms.dataflow-http.*", path=~".*/imports.*|.*/import-target-tables.*|.*/data-ingestion-profile.*|.*/dataflow.*"}[10m]))
        < 0.9995
      for: 5m
      labels:
        severity: major
        response_class: ticket
        component: data-migration
        oncall_team: tidb_x_tools
    - alert: SLOImportHttpApiLatencyP99BreakRisk
      annotations:
        message: |
          SLO 'Import HTTP API Latency P99' Break Risk, Runbook: https://clinic.pingcap.com/grafana/d/QIw-n_l4k/slo-global-control-plane?orgId=1
      expr: |
        (
          1-(sum(rate(kong_custom_request_latency_ms_bucket{kube_service=~"prod-ms.dataflow-http.*", path=~".*/imports.*|.*/import-target-tables.*|.*/data-ingestion-profile.*|.*/dataflow.*", le="2000"}[1h]))
          /
          sum(rate(kong_custom_request_latency_ms_count{kube_service=~"prod-ms.dataflow-http.*", path=~".*/imports.*|.*/import-target-tables.*|.*/data-ingestion-profile.*|.*/dataflow.*"}[1h]))) 
          > 16.8*0.01
        ) and (
          1-(sum(rate(kong_custom_request_latency_ms_bucket{kube_service=~"prod-ms.dataflow-http.*", path=~".*/imports.*|.*/import-target-tables.*|.*/data-ingestion-profile.*|.*/dataflow.*", le="2000"}[5m]))
          /
          sum(rate(kong_custom_request_latency_ms_count{kube_service=~"prod-ms.dataflow-http.*", path=~".*/imports.*|.*/import-target-tables.*|.*/data-ingestion-profile.*|.*/dataflow.*"}[5m])))
          > 16.8*0.01
        )
      labels:
        severity: major
        response_class: ticket
        component: data-migration
        oncall_team: tidb_x_tools
    - alert: SLOImportHttpApiLatencyP90BreakRisk
      annotations:
        message: |
          SLO 'Import HTTP API Latency P90' Break Risk, Runbook: https://clinic.pingcap.com/grafana/d/QIw-n_l4k/slo-global-control-plane?orgId=1
      expr: |
        (
          1-(sum(rate(kong_custom_request_latency_ms_bucket{kube_service=~"prod-ms.dataflow-http.*", path=~".*/imports.*|.*/import-target-tables.*|.*/data-ingestion-profile.*|.*/dataflow.*", le="1000"}[1h]))
          /
          sum(rate(kong_custom_request_latency_ms_count{kube_service=~"prod-ms.dataflow-http.*", path=~".*/imports.*|.*/import-target-tables.*|.*/data-ingestion-profile.*|.*/dataflow.*"}[1h]))) 
          > 2.8*0.1
        ) and (
          1-(sum(rate(kong_custom_request_latency_ms_bucket{kube_service=~"prod-ms.dataflow-http.*", path=~".*/imports.*|.*/import-target-tables.*|.*/data-ingestion-profile.*|.*/dataflow.*", le="1000"}[5m]))
          /
          sum(rate(kong_custom_request_latency_ms_count{kube_service=~"prod-ms.dataflow-http.*", path=~".*/imports.*|.*/import-target-tables.*|.*/data-ingestion-profile.*|.*/dataflow.*"}[5m])))
          > 2.8*0.1
        )
      labels:
        severity: major
        response_class: ticket
        component: data-migration
        oncall_team: tidb_x_tools
    - alert: SLOImportGrpcApiAvailabilityBreakRisk
      annotations:
        message: |
          SLO 'Import GRPC API Availability' Break Risk, Runbook: https://clinic.pingcap.com/grafana/d/QIw-n_l4k/slo-global-control-plane?orgId=1
      expr: |
        sum(rate(grpc_server_handled_total{namespace="prod-ms", pod=~"dataflow-grpc.*", grpc_type="unary", grpc_code!="UNKNOWN|DEADLINE_EXCEEDED|INTERNAL|UNAVAILABLE|DATA_LOSS", grpc_service=~".*Import.*|^dm.Service$"}[10m]))
        /
        sum(rate(grpc_server_handled_total{namespace="prod-ms", pod=~"dataflow-grpc.*", grpc_type="unary", grpc_service=~".*Import.*|^dm.Service$"}[10m]))
        < 0.9995
      for: 5m
      labels:
        severity: major
        response_class: ticket
        component: data-migration
        oncall_team: tidb_x_tools
    - alert: SLOImportGrpcApiLatencyP99BreakRisk
      annotations:
        message: |
          SLO 'Import GRPC API Latency P99' Break Risk, Runbook: https://clinic.pingcap.com/grafana/d/QIw-n_l4k/slo-global-control-plane?orgId=1
      expr: |
        (
          1-(sum(rate(grpc_server_handling_seconds_bucket{namespace=~"nightly-ms|staging-ms|prod-ms", pod=~"dataflow-grpc.*", grpc_type="unary", grpc_service=~".*Import.*|^dm.Service$", le="1"}[1h]))
          /
          sum(rate(grpc_server_handling_seconds_count{namespace=~"nightly-ms|staging-ms|prod-ms", pod=~"dataflow-grpc.*", grpc_type="unary", grpc_service=~".*Import.*|^dm.Service$"}[1h])))
          > 16.8*0.01
        ) and (
          1-(sum(rate(grpc_server_handling_seconds_bucket{namespace=~"nightly-ms|staging-ms|prod-ms", pod=~"dataflow-grpc.*", grpc_type="unary", grpc_service=~".*Import.*|^dm.Service$", le="1"}[5m]))
          /
          sum(rate(grpc_server_handling_seconds_count{namespace=~"nightly-ms|staging-ms|prod-ms", pod=~"dataflow-grpc.*", grpc_type="unary", grpc_service=~".*Import.*|^dm.Service$"}[5m])))
          > 16.8*0.01
        )
      labels:
        severity: major
        response_class: ticket
        component: data-migration
        oncall_team: tidb_x_tools
    - alert: SLOImportGrpcApiLatencyP90BreakRisk
      annotations:
        message: |
          SLO 'Import GRPC API Latency P90' Break Risk, Runbook: https://clinic.pingcap.com/grafana/d/QIw-n_l4k/slo-global-control-plane?orgId=1
      expr: |
        (
          1-(sum(rate(grpc_server_handling_seconds_bucket{namespace=~"nightly-ms|staging-ms|prod-ms", pod=~"dataflow-grpc.*", grpc_type="unary", grpc_service=~".*Import.*|^dm.Service$", le="2.5"}[1h]))
          /
          sum(rate(grpc_server_handling_seconds_count{namespace=~"nightly-ms|staging-ms|prod-ms", pod=~"dataflow-grpc.*", grpc_type="unary", grpc_service=~".*Import.*|^dm.Service$"}[1h])))
          > 2.8*0.1
        ) and (
          1-(sum(rate(grpc_server_handling_seconds_bucket{namespace=~"nightly-ms|staging-ms|prod-ms", pod=~"dataflow-grpc.*", grpc_type="unary", grpc_service=~".*Import.*|^dm.Service$", le="2.5"}[5m]))
          /
          sum(rate(grpc_server_handling_seconds_count{namespace=~"nightly-ms|staging-ms|prod-ms", pod=~"dataflow-grpc.*", grpc_type="unary", grpc_service=~".*Import.*|^dm.Service$"}[5m])))
          > 2.8*0.1
        )
      labels:
        severity: major
        response_class: ticket
        component: data-migration
        oncall_team: tidb_x_tools