groups:
- name: SLOsForAIPower
  rules:
  - alert: SLOAIPowerApiAvailabilityBreakRisk
    annotations:
      message: |
        SLO 'AIPower API Availability' Break Risk, Runbook: https://clinic.pingcap.com/grafana/d/QIw-n_l4k/slo-global-control-plane?orgId=1
    expr: |
      sum(rate(request_latency_seconds_count{job="edaservice", status_code!~"5.*"}[10m]))
      /
      sum(rate(request_latency_seconds_count{job="edaservice"}[10m]))
      < 0.999
    labels:
      severity: major
      response_class: ticket
      component: aipower
      oncall_team: clinic
  - alert: SLOAIPowerApiLatencyP99BreakRisk
    annotations:
      message: |
        SLO 'AIPower API Latency P99' Break Risk, Runbook: https://clinic.pingcap.com/grafana/d/QIw-n_l4k/slo-global-control-plane?orgId=1
    expr: |
      (
        1-(sum(rate(request_latency_seconds_bucket{job="edaservice", le="2.5"}[1h]))
        /
        sum(rate(request_latency_seconds_count{job="edaservice"}[1h]))) 
        > 16.8*0.01
      ) and (
        1-(sum(rate(request_latency_seconds_bucket{job="edaservice", le="2.5"}[5m]))
        /
        sum(rate(request_latency_seconds_count{job="edaservice"}[5m])))
        > 16.8*0.01
      )
    labels:
      severity: major
      response_class: ticket
      component: aipower
      oncall_team: clinic
  - alert: SLOAIPowerApiLatencyP90BreakRisk
    annotations:
      message: |
        SLO 'AIPower API Latency P90' Break Risk, Runbook: https://clinic.pingcap.com/grafana/d/QIw-n_l4k/slo-global-control-plane?orgId=1
    expr: |
      (
        1-(sum(rate(request_latency_seconds_bucket{job="edaservice", le="5.0"}[1h]))
        /
        sum(rate(request_latency_seconds_count{job="edaservice"}[1h]))) 
        > 2.8*0.1
      ) and (
        1-(sum(rate(request_latency_seconds_bucket{job="edaservice", le="5.0"}[5m]))
        /
        sum(rate(request_latency_seconds_count{job="edaservice"}[5m])))
        > 2.8*0.1
      )
    labels:
      severity: major
      response_class: ticket
      component: aipower
      oncall_team: clinic
  - alert: SLOAIPowerWorkerAvailabilityBreakRisk
    annotations:
      message: |
        SLO 'AIPower Worker Availability' Break Risk, Runbook: https://clinic.pingcap.com/grafana/d/QIw-n_l4k/slo-global-control-plane?orgId=1
    expr: |
      sum(rate(biz_latency_seconds_count{job="edaservice-worker", status_code!~"5.*"}[10m]))
      /
      sum(rate(biz_latency_seconds_count{job="edaservice-worker"}[10m]))
      < 0.99
    labels:
      severity: major
      response_class: ticket
      component: aipower
      oncall_team: clinic
  - alert: SLOAIPowerWorkerLatencyP99BreakRiskChat2DataGPT35Turbo
    annotations:
      message: |
        SLO 'AIPower Worker Latency P99' Break Risk, Runbook: https://clinic.pingcap.com/grafana/d/QIw-n_l4k/slo-global-control-plane?orgId=1
    expr: |
      (
        1-(sum(rate(biz_latency_seconds_bucket{job="edaservice-worker",name="task_chat2data|task_chat2sql",llm_model="gpt-3.5-turbo",le="60.0"}[1h]))
        /
        sum(rate(biz_latency_seconds_count{job="edaservice-worker",name="task_chat2data|task_chat2sql",llm_model="gpt-3.5-turbo"}[1h]))) 
        > 16.8*0.01
      ) and (
        1-(sum(rate(biz_latency_seconds_bucket{job="edaservice-worker",name="task_chat2data|task_chat2sql",llm_model="gpt-3.5-turbo",le="60.0"}[5m]))
        /
        sum(rate(biz_latency_seconds_count{job="edaservice-worker",name="task_chat2data|task_chat2sql",llm_model="gpt-3.5-turbo"}[5m])))
        > 16.8*0.01
      )
    labels:
      severity: major
      response_class: ticket
      component: aipower
      oncall_team: clinic
  - alert: SLOAIPowerWorkerLatencyP90BreakRiskChat2DataGPT35Turbo
    annotations:
      message: |
        SLO 'AIPower Worker Latency P90' Break Risk, Runbook: https://clinic.pingcap.com/grafana/d/QIw-n_l4k/slo-global-control-plane?orgId=1
    expr: |
      (
        1-(sum(rate(biz_latency_seconds_bucket{job="edaservice-worker",name="task_chat2data|task_chat2sql",llm_model="gpt-3.5-turbo",le="20.0"}[1h]))
        /
        sum(rate(biz_latency_seconds_count{job="edaservice-worker",name="task_chat2data|task_chat2sql",llm_model="gpt-3.5-turbo"}[1h]))) 
        > 2.8*0.1
      ) and (
        1-(sum(rate(biz_latency_seconds_bucket{job="edaservice-worker",name="task_chat2data|task_chat2sql",llm_model="gpt-3.5-turbo",le="20.0"}[5m]))
        /
        sum(rate(biz_latency_seconds_count{job="edaservice-worker",name="task_chat2data|task_chat2sql",llm_model="gpt-3.5-turbo"}[5m])))
        > 2.8*0.1
      )
    labels:
      severity: major
      response_class: ticket
      component: aipower
      oncall_team: clinic
  - alert: SLOAIPowerWorkerLatencyP99BreakRiskChat2DataGPT4
    annotations:
      message: |
        SLO 'AIPower Worker Latency P99' Break Risk, Runbook: https://clinic.pingcap.com/grafana/d/QIw-n_l4k/slo-global-control-plane?orgId=1
    expr: |
      (
        1-(sum(rate(biz_latency_seconds_bucket{job="edaservice-worker",name="task_chat2data|task_chat2sql",llm_model="gpt-4",le="300.0"}[1h]))
        /
        sum(rate(biz_latency_seconds_count{job="edaservice-worker",name="task_chat2data|task_chat2sql",llm_model="gpt-4"}[1h]))) 
        > 16.8*0.01
      ) and (
        1-(sum(rate(biz_latency_seconds_bucket{job="edaservice-worker",name="task_chat2data|task_chat2sql",llm_model="gpt-4",le="300.0"}[5m]))
        /
        sum(rate(biz_latency_seconds_count{job="edaservice-worker",name="task_chat2data|task_chat2sql",llm_model="gpt-4"}[5m])))
        > 16.8*0.01
      )
    labels:
      severity: major
      response_class: ticket
      component: aipower
      oncall_team: clinic
  - alert: SLOAIPowerWorkerLatencyP90BreakRiskChat2DataGPT4
    annotations:
      message: |
        SLO 'AIPower Worker Latency P90' Break Risk, Runbook: https://clinic.pingcap.com/grafana/d/QIw-n_l4k/slo-global-control-plane?orgId=1
    expr: |
      (
        1-(sum(rate(biz_latency_seconds_bucket{job="edaservice-worker",name="task_chat2data|task_chat2sql",llm_model="gpt-4",le="100.0"}[1h]))
        /
        sum(rate(biz_latency_seconds_count{job="edaservice-worker",name="task_chat2data|task_chat2sql",llm_model="gpt-4"}[1h]))) 
        > 2.8*0.1
      ) and (
        1-(sum(rate(biz_latency_seconds_bucket{job="edaservice-worker",name="task_chat2data|task_chat2sql",llm_model="gpt-4",le="100.0"}[5m]))
        /
        sum(rate(biz_latency_seconds_count{job="edaservice-worker",name="task_chat2data|task_chat2sql",llm_model="gpt-4"}[5m])))
        > 2.8*0.1
      )
    labels:
      severity: major
      response_class: ticket
      component: aipower
      oncall_team: clinic
  - alert: SLOAIPowerWorkerLatencyP99BreakRiskUnderstandDBGPT35Turbo
    annotations:
      message: |
        SLO 'AIPower Worker Latency P99' Break Risk, Runbook: https://clinic.pingcap.com/grafana/d/QIw-n_l4k/slo-global-control-plane?orgId=1
    expr: |
      (
        1-(sum(rate(biz_latency_seconds_bucket{job="edaservice-worker",name="task_understand_db",llm_model="gpt-3.5-turbo",le="300.0"}[1h]))
        /
        sum(rate(biz_latency_seconds_count{job="edaservice-worker",name="task_understand_db",llm_model="gpt-3.5-turbo"}[1h])))
        > 16.8*0.01
      ) and (
        1-(sum(rate(biz_latency_seconds_bucket{job="edaservice-worker",name="task_understand_db",llm_model="gpt-3.5-turbo",le="300.0"}[5m]))
        /
        sum(rate(biz_latency_seconds_count{job="edaservice-worker",name="task_understand_db",llm_model="gpt-3.5-turbo"}[5m])))
        > 16.8*0.01
      )
    labels:
      severity: major
      response_class: ticket
      component: aipower
      oncall_team: clinic
  - alert: SLOAIPowerWorkerLatencyP90BreakRiskUnderstandDBGPT35Turbo
    annotations:
      message: |
        SLO 'AIPower Worker Latency P90' Break Risk, Runbook: https://clinic.pingcap.com/grafana/d/QIw-n_l4k/slo-global-control-plane?orgId=1
    expr: |
      (
        1-(sum(rate(biz_latency_seconds_bucket{job="edaservice-worker",name="task_understand_db",llm_model="gpt-3.5-turbo",le="60.0"}[1h]))
        /
        sum(rate(biz_latency_seconds_count{job="edaservice-worker",name="task_understand_db",llm_model="gpt-3.5-turbo"}[1h]))) 
        > 2.8*0.1
      ) and (
        1-(sum(rate(biz_latency_seconds_bucket{job="edaservice-worker",name="task_understand_db",llm_model="gpt-3.5-turbo",le="60.0"}[5m]))
        /
        sum(rate(biz_latency_seconds_count{job="edaservice-worker",name="task_understand_db",llm_model="gpt-3.5-turbo"}[5m])))
        > 2.8*0.1
      )
    labels:
      severity: major
      response_class: ticket
      component: aipower
      oncall_team: clinic
  - alert: SLOAIPowerWorkerLatencyP99BreakRiskUnderstandDBGPT4
    annotations:
      message: |
        SLO 'AIPower Worker Latency P99' Break Risk, Runbook: https://clinic.pingcap.com/grafana/d/QIw-n_l4k/slo-global-control-plane?orgId=1
    expr: |
      (
        1-(sum(rate(biz_latency_seconds_bucket{job="edaservice-worker",name="task_understand_db",llm_model="gpt-4",le="900.0"}[1h]))
        /
        sum(rate(biz_latency_seconds_count{job="edaservice-worker",name="task_understand_db",llm_model="gpt-4"}[1h]))) 
        > 16.8*0.01
      ) and (
        1-(sum(rate(biz_latency_seconds_bucket{job="edaservice-worker",name="task_understand_db",llm_model="gpt-4",le="900.0"}[5m]))
        /
        sum(rate(biz_latency_seconds_count{job="edaservice-worker",name="task_understand_db",llm_model="gpt-4"}[5m])))
        > 16.8*0.01
      )
    labels:
      severity: major
      response_class: ticket
      component: aipower
      oncall_team: clinic
  - alert: SLOAIPowerWorkerLatencyP90BreakRiskUnderstandDBGPT4
    annotations:
      message: |
        SLO 'AIPower Worker Latency P90' Break Risk, Runbook: https://clinic.pingcap.com/grafana/d/QIw-n_l4k/slo-global-control-plane?orgId=1
    expr: |
      (
        1-(sum(rate(biz_latency_seconds_bucket{job="edaservice-worker",name="task_understand_db",llm_model="gpt-4",le="300.0"}[1h]))
        /
        sum(rate(biz_latency_seconds_count{job="edaservice-worker",name="task_understand_db",llm_model="gpt-4"}[1h]))) 
        > 2.8*0.1
      ) and (
        1-(sum(rate(biz_latency_seconds_bucket{job="edaservice-worker",name="task_understand_db",llm_model="gpt-4",le="300.0"}[5m]))
        /
        sum(rate(biz_latency_seconds_count{job="edaservice-worker",name="task_understand_db",llm_model="gpt-4"}[5m])))
        > 2.8*0.1
      )
    labels:
      severity: major
      response_class: ticket
      component: aipower
      oncall_team: clinic