groups:
- name: billing2
  rules:
  - alert: BillingDown
    annotations:
      message: Billing has disappeared from Prometheus target discovery.
    expr: |
      up{job=~"billing2.*",namespace=~"(.*)-ms"} == 0
    for: 5m
    labels:
      severity: critical
      response_class: page
      component: billing-server
      oncall_team: billing
  - alert: BillingRPCSLO
    annotations:
      message: Billing Inner-RPC-SLO (credits_summary || measures) < 95%.
    expr: |
      ( sum(increase (billing2_requests_total{service="billing2",code="200", url=~"(.*)credits_summary||(.*)measures" ,namespace="prod-ms"  }[2m]))
        / sum(increase (billing2_requests_total{service="billing2",url=~"(.*)credits_summary||(.*)measures" ,namespace="prod-ms" }[2m]))
      ) < 0.950
    for: 2m
    labels:
      severity: major
      response_class: ticket
      component: billing-server
      oncall_team: billing
  - alert: BillingAPIAverageProd
    annotations:
      message: Billing API Average > 0.5s in  [5m].
    expr: |
       ( sum(rate(billing2_request_duration_seconds_sum{namespace="prod-ms"}[1m]))/sum(rate(billing2_request_duration_seconds_count{namespace="prod-ms"}[1m])) ) > 0.5
    for: 5m
    labels:
      severity: major
      response_class: ticket
      component: billing-server
      oncall_team: billing
  - alert: BillingAPIMeasure
    annotations:
      message: Billing API -Measure  Average > 0.30s in  [5m].
    expr: |
       ( sum(rate(billing2_request_duration_seconds_sum{namespace="prod-ms",url="/api/v1/billing/tenants/:tenant_id/measures"}[1m]))/sum(rate(billing2_request_duration_seconds_count{namespace="prod-ms",url="/api/v1/billing/tenants/:tenant_id/measures"}[1m])) ) >0.30
    for: 10m
    labels:
      severity: major
      response_class: ticket
      component: billing-server
      oncall_team: billing
  - alert: BillingLatencyP90Prod
    annotations:
      message: Billing LatencyP90  > 2s in  [5m].
    expr: |
        histogram_quantile(0.90, sum(rate(billing2_request_duration_seconds_bucket{namespace="prod-ms"}[1m])) by (le)) > 2
    for: 2m
    labels:
      severity: major
      response_class: ticket
      component: billing-server
      oncall_team: billing
  - alert: BillingAPIAverageDeV
    annotations:
      message: Billing API Average > 2s in  [5m].
    expr: |
      ( sum(rate(billing2_request_duration_seconds_sum{namespace!="prod-ms"}[1m]))/sum(rate(billing2_request_duration_seconds_count{namespace!="prod-ms"}[1m])) ) > 2.0
    for: 5m
    labels:
      severity: major
      response_class: ticket
      component: billing-server
      oncall_team: billing
  - alert: BillingLatencyP90DeV
    annotations:
      message: Billing LatencyP90  > 4s in  [10m].
    expr: |
      histogram_quantile(0.90, sum(rate(billing2_request_duration_seconds_bucket{namespace!="prod-ms"}[1m])) by (le)) > 4
    for: 10m
    labels:
      severity: major
      response_class: ticket
      component: billing-server
      oncall_team: billing
  - alert: BillingOfficialWebsiteUserExperienceSLO
    annotations:
      message: Billing OfficialWebsiteUserExperience-SLO.
    expr: |
        ( sum(increase (billing2_requests_total{service="billing2",code="200",namespace="prod-ms", url=~"(.*)month||(.*)export||(.*)dryrun||(.*)payment_methods||(.*)invoices||(.*)setup_intent"   }[60m])) 
        / sum(increase (billing2_requests_total{service="billing2",namespace="prod-ms", url=~"(.*)month||(.*)export||(.*)dryrun||(.*)payment_methods||(.*)invoices||(.*)setup_intent" }[60m])) 
        ) < 0.990
    for: 10m
    labels:
      severity: major
      response_class: ticket
      component: billing-server
      oncall_team: billing
  - alert: BillingInnerUserExperienceSLO
    annotations:
        message: Billing InnerUserExperience-SLO.
    expr: |
      ( sum(increase (billing2_requests_total{service="billing2",code="200", namespace="prod-ms", url=~"(.*)tenant_ids||(.*)support_plan||(.*)billing_info||(.*)binded_packages||(.*)binded_package_id||(.*)binded_support_plan||(.*)credits||(.*)invoices||(.*)payments||(.*)customer"  }[60m])) 
      / sum(increase (billing2_requests_total{service="billing2",namespace="prod-ms", url=~"(.*)tenant_ids||(.*)support_plan||(.*)billing_info||(.*)binded_packages||(.*)binded_package_id||(.*)binded_support_plan||(.*)credits||(.*)invoices||(.*)payments||(.*)customer"  }[60m]))
      ) < 0.960
    for: 10m
    labels:
      severity: major
      response_class: ticket
      component: billing-server
      oncall_team: billing
  - alert: BillingGenerateInvoiceSLO
    annotations:
        message: Billing GenerateInvoice-SLO.
    expr: |
      ( sum( increase(dbaas_billing_invoice{status="true" }[60m])) /sum(increase(dbaas_billing_invoice{status="init"}[60m]) )
      ) < 0.800
    for: 10m
    labels:
      severity: major
      response_class: ticket
      component: billing-server
      oncall_team: billing
  - alert: BillingPaymentError
    annotations:
        message: Billing PaymentError.
    expr: |
      sum(delta(dbaas_billing_invoice_pay{status="false",namespace="prod-ms", pod=~"billing2(.*)"}[60m])) > 1
    for: 2m
    labels:
      severity: major
      response_class: ticket
      component: billing-server
      oncall_team: billing
  - alert: BillingPodRestartCount
    annotations:
      message: Billing Pod Restart Count >= 1 .
    expr: |
      ( sum(increase(kube_pod_container_status_restarts_total{  pod=~"billing2(.*)"}[5m]) )
      ) > 0
    for: 1m
    labels:
      severity: major
      response_class: ticket
      component: billing-server
      oncall_team: billing
  - alert: BillingMeasureRetryError
    annotations:
      message: Billing Measure Retry Error cnt >10  /1h.
    expr: |
      (sum(delta(dbaas_billing_measures_failed_retry{ namespace="prod-ms", pod=~"billing2(.*)"}[1h]) ) ) > 10
    for: 2m
    labels:
      severity: major
      response_class: ticket
      component: billing-server
      oncall_team: billing
  - alert: BillingMeasureNotMatchPackage
    annotations:
      message: Billing Measure Not Match Package  cnt >1000 /1h .
    expr: |
      (sum(delta(dbaas_billing_measures_not_match_package{ namespace="prod-ms", pod=~"billing2(.*)"}[1h]) ) ) > 1000
    for: 2m
    labels:
      severity: major
      response_class: ticket
      component: billing-server
      oncall_team: billing
  - alert: BillingMeasuresDelay
    annotations:
      message: Billing Measure Delay Uint {{ $labels.measuresDelayUnitKey }} .
    expr: |
      (sum(delta(dbaas_billing_measures_delay{ pod=~"billing2(.*)"}[1h]) ) ) > 1
    for: 2m
    labels:
      severity: major
      response_class: ticket
      component: billing-server
      oncall_team: billing
  - alert: BillingEventbusDLQ
    annotations:
      message: Event enter DLQ at Billing Eventbus.
    expr: |
      sum(increase(billing_eventbus_events_published_total{target="dlq" }[1m])) > 0
    for: 0m
    labels:
      severity: major
      response_class: ticket
      component: billing-server
      oncall_team: billing
  - alert: BillingEventbusPublishFailed
    annotations:
      message: Eventbus Publishing failed.
    expr: |
      sum(increase(billing_eventbus_events_published_total{success="false"}[1m])) > 0
    for: 0m
    labels:
      severity: major
      response_class: ticket
      component: billing-server
      oncall_team: billing
  - alert: BillingCpuAlert1
    annotations:
      message: Billing Cpu Alert > 70% for 15min.
    expr: |
      (sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{ namespace=~"nightly-ms|staging-ms|prod-ms",pod=~"billing(.*)"}) by (pod)/
          sum(kube_pod_container_resource_limits{ namespace=~"nightly-ms|staging-ms|prod-ms",resource="cpu",pod=~"billing(.*)"}) by (pod) > 0.7)
    for: 15m
    labels:
      severity: major
      response_class: ticket
      component: billing-server
      oncall_team: billing
  - alert: BillingCpuAlert2
    annotations:
      message: Billing Cpu Alert > 90% for 3min.
    expr: |
      (sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{ namespace=~"nightly-ms|staging-ms|prod-ms",pod=~"^billing2-.*",pod!~"^billing2-worker.*"}) by (pod)/
          sum(kube_pod_container_resource_limits{ namespace=~"nightly-ms|staging-ms|prod-ms",resource="cpu",pod=~"^billing2-.*",pod!~"^billing2-worker.*"}) by (pod) > 0.9)
    for: 3m
    labels:
      severity: major
      response_class: ticket
      component: billing-server
      oncall_team: billing
  - alert: BillingCpuAlert3
    annotations:
      message: Billing worker Cpu Alert > 90% for 10min.
    expr: |
      (sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{ namespace=~"nightly-ms|staging-ms|prod-ms",pod=~"billing2-worker(.*)"}) by (pod)/
          sum(kube_pod_container_resource_limits{ namespace=~"nightly-ms|staging-ms|prod-ms",resource="cpu",pod=~"billing2-worker(.*)"}) by (pod) > 0.9)
    for: 10m
    labels:
      severity: major
      response_class: ticket
      component: billing-server
      oncall_team: billing
  - alert: BillingMemoryAlert
    annotations:
      message: Billing Memory Alert > 80%.
    expr: |
      (sum(container_memory_working_set_bytes{ namespace=~"nightly-ms|staging-ms|prod-ms",pod=~"billing(.*)",container!=""})by(pod)/
      sum(kube_pod_container_resource_limits{ namespace=~"nightly-ms|staging-ms|prod-ms",resource="memory",pod=~"billing(.*)"}) by (pod) ) > 0.8
    for: 5m
    labels:
      severity: major
      response_class: ticket
      component: billing-server
      oncall_team: billing