groups:
- name: aipower
  rules:
  - alert: AIPowerDown
    annotations:
      message: AIPower has disappeared from Prometheus target discovery.
    expr: |
      up{job=~"aipower.*",namespace=~"(.*)-ms"} == 0
    for: 5m
    labels:
      severity: critical
      response_class: page
      component: aipower
      oncall_team: clinic
  - alert: AIPowerAPIAverage(Chat2SQL)
    annotations:
      message: AIPower API(Chat2SQL) Average > 30s in  [5m].
    expr: |
       ((sum(rate(aipower_openai_request_duration_ms_sum{model="chat2sql"}[5m]))/sum(rate(aipower_openai_request_duration_ms_count{model="chat2sql"}[5m]))) / 1000) > 30
    for: 5m
    labels:
      severity: major
      response_class: ticket
      component: aipower
      oncall_team: clinic
  - alert: AIPowerAPIAverage(Chat2Chart)
    annotations:
      message: AIPower API(Chat2Chart) Average > 20s in  [5m].
    expr: |
       ((sum(rate(aipower_openai_request_duration_ms_sum{model="chat2chart"}[5m]))/sum(rate(aipower_openai_request_duration_ms_count{model="chat2chart"}[5m]))) / 1000) > 20
    for: 5m
    labels:
      severity: major
      response_class: ticket
      component: aipower
      oncall_team: clinic
  - alert: AIPowerAPIAverage(Chat2Question)
    annotations:
      message: AIPower API(Chat2Question) Average > 55s in  [5m].
    expr: |
       ((sum(rate(aipower_openai_request_duration_ms_sum{model="chat2question"}[5m]))/sum(rate(aipower_openai_request_duration_ms_count{model="chat2question"}[5m]))) / 1000) > 55
    for: 5m
    labels:
      severity: major
      response_class: ticket
      component: aipower
      oncall_team: clinic
  - alert: AIPowerAPIAverage(ColumnMatching)
    annotations:
      message: AIPower API(ColumnMatching) Average > 30s in  [5m].
    expr: |
       ((sum(rate(aipower_openai_request_duration_ms_sum{model="data2columnMatching"}[5m]))/sum(rate(aipower_openai_request_duration_ms_count{model="data2columnMatching"}[5m]))) / 1000) > 55
    for: 5m
    labels:
      severity: major
      response_class: ticket
      component: aipower
      oncall_team: clinic
  - alert: AIPowerLatencyP90
    annotations:
      message: AIPower LatencyP90  > 20s in  [2m].
    expr: |
        histogram_quantile(0.90, sum(rate(grpc_server_handling_seconds_bucket{grpc_service="api.ChatBotService",namespace=~"(.*)-ms"}[1m])) by (le)) > 20
    for: 2m
    labels:
      severity: major
      response_class: ticket
      component: aipower
      oncall_team: clinic
  - alert: AIPowerPodRestartCount
    annotations:
      message: AIPower Pod Restart Count > 1 .
    expr: |
      (sum(increase(kube_pod_container_status_restarts_total{  pod=~"aipower(.*)"}[1h]) )) > 1
    for: 1m
    labels:
      severity: major
      response_class: ticket
      component: aipower
      oncall_team: clinic
  - alert: status_failed_percentage_greater_than_15_percent
    expr: |
      (sum by(model) (aipower_openai_request_counter{status="failed"} / sum(aipower_openai_request_counter)) * 100) > 15
    for: 10m
    labels:
      severity: major
      response_class: ticket
      component: aipower
      oncall_team: clinic
    annotations:
      summary: High percentage of failed requests for aipower
      description: The percentage of failed requests for aipower has been consistently high over the last 10 minutes.
  - alert: EdaServicePodRestartCount
    annotations:
      message: EdaService Pod Restart Count > 1 .
    expr: |
      (sum(increase(kube_pod_container_status_restarts_total{  pod=~"edaservice(.*)"}[1h]) )) > 1
    for: 1m
    labels:
      severity: major
      response_class: ticket
      component: edaservice
      oncall_team: clinic
  - alert: "CeleryTaskHighFailRate"
    expr: |
      sum(
        increase(
          celery_task_failed_total{
            job=~"edaservice-worker-monitor",
            queue_name!~"None",
            name!~"None"
          }[10m]
        )
      )  by (job, namespace, queue_name, name)
      /
      (
        sum(
          increase(
            celery_task_failed_total{
              job=~"edaservice-worker-monitor",
              queue_name!~"None",
              name!~"None"
            }[10m]
          )
        )  by (job, namespace, queue_name, name)
        +
        sum(
          increase(
            celery_task_succeeded_total{
              job=~"edaservice-worker-monitor",
              queue_name!~"None",
              name!~"None"
            }[10m]
          )
        )  by (job, namespace, queue_name, name)
      )
      * 100 > 5
    for: "1m"
    labels:
      severity: warning
      component: aipower
      oncall_team: clinic
  - alert: "CeleryHighQueueLength"
    expr: |
      sum(
        celery_queue_length{
          job=~"edaservice-worker-monitor",
          queue_name!~"None"
        }
      )  by (job, namespace, queue_name)
      > 100
    for: "20m"
    labels:
      severity: warning
      component: aipower
      oncall_team: clinic
  - alert: "CeleryWorkerDown"
    expr: |
      celery_worker_up{job=~"edaservice-worker-monitor"} == 0
    for: "15m"
    labels:
      severity: warning
      component: aipower
      oncall_team: clinic
  - alert: "Chat2QueryAbnormalRequests"
    expr: |
      sum(increase(dataservices_dataapi_gateway_http_requests_total{path="/api/v1beta/app/:external_app_id/endpoint/*path",status!="200"}[10m])) by (status, method, path) > 10
    for: "10m"
    labels:
      severity: warning
      component: aipower
      oncall_team: clinic
  - alert: "Chat2QueryDurationTooLong"
    expr: |
      (histogram_quantile(0.95, sum(rate(dataservices_dataapi_gateway_http_request_duration_ms_bucket{path="/api/v1beta/app/:external_app_id/endpoint/*path"}[10m])) by (method, path, le)) / 1000) > 5
    for: "10m"
    labels:
      severity: warning
      component: aipower
      oncall_team: clinic
  - alert: "OpenAI Request success rate too low"
    expr: |
      (sum(increase(eda_openai_histogram_count{status="success"}[10m])) / sum(increase(eda_openai_histogram_count[10m]))) < 0.9
    for: "10m"
    labels:
      severity: warning
      component: aipower
      oncall_team: clinic
  - alert: "OpenAI retries too many times"
    expr: |
      (sum(increase(eda_openai_histogram_count{attempt!="0"}[10m])) / sum(increase(eda_openai_histogram_count[10m]))) > 0.1
    for: "10m"
    labels:
      severity: warning
      component: aipower
      oncall_team: clinic
  - alert: "OpenAI UnderstandDB takes too much time"
    expr: |
      histogram_quantile(0.95, sum by(le) (rate(eda_openai_histogram_bucket{exported_endpoint="POST - /v2/dataSummaries"}[10m]))) > 300
    for: "10m"
    labels:
      severity: warning
      component: aipower
      oncall_team: clinic
  - alert: "OpenAI Text2SQL takes too much time"
    expr: |
      histogram_quantile(0.95, sum by(le) (rate(eda_openai_histogram_bucket{exported_endpoint="POST - /v2/chat2data"}[10m]))) > 120
    for: "10m"
    labels:
      severity: warning
      component: aipower
      oncall_team: clinic