groups:
- name: o11y-svc
  rules:
  - alert: O11yWebconsoleApi5xxErrors
    annotations:
      message: "O11y Webconsole API is returning 5xx errors on host {{ $labels.host }} path {{ $labels.path }} to service {{ $labels.kube_service }} with status code {{ $labels.code }}, VALUE = {{ $value }}"
    expr: |
      sum(rate(kong_custom_http_status{code=~"5(.*)",kube_service=~".*o11y.*"}[5m])) by (path, code, host, kube_service) > 0
    for: 10m
    labels:
      component: o11y
      oncall_team: o11y
      severity: major
      response_class: ticket
  - alert: O11yWebconsoleApiHigh403Rate
    annotations:
      summary: "High rate of 403 Forbidden errors on O11y Webconsole API"
      message: "O11y Webconsole API is returning 403 (Forbidden) errors on host {{ $labels.host }}, path {{ $labels.path }} (Service: {{ $labels.kube_service }}). Current percentage: {{ $value }}, indicating a potential RBAC misconfiguration."
    expr: |
      sum by (path, host, kube_service) (
        rate(kong_custom_http_status{code="403", kube_service=~".*o11y.*"}[5m])
      ) /
      sum by (path, host, kube_service) (
        rate(kong_custom_http_status{kube_service=~".*o11y.*"}[5m])
      )
      > 0.6
    for: 10m
    labels:
      component: o11y
      oncall_team: o11y
      severity: major
      response_class: ticket