groups:
  - name: mnemos
    rules:
      # High error rate alert - 5xx error rate exceeds 5%.
      # Suppress this slower warning once the burst threshold is reached.
      - alert: MnemosHighErrorRate
        expr: |
          (
            sum(rate(mnemo_http_requests_total{route=~"/v1.*",status=~"5.."}[5m]))
            /
            sum(rate(mnemo_http_requests_total{route=~"/v1.*"}[5m]))
          ) > 0.05
          unless on ()
          sum(increase(mnemo_http_requests_total{route=~"/v1.*",status=~"5.."}[5m])) >= 3
        for: 2m
        labels:
          severity: warning
          component: mnemos
          oncall_team: mem9
        annotations:
          summary: "Mnemos high error rate detected"
          description: "Mnemos service 5xx error rate exceeded 5% over the past 5 minutes, current value: {{ $value | humanizePercentage }}"

      # Major overall burst alert. This fires only when the three errors are
      # spread across routes; a route-local burst is handled by the next rule.
      - alert: MnemosCriticalErrorRate
        expr: |
          (
            sum(increase(mnemo_http_requests_total{route=~"/v1.*",status=~"5.."}[5m])) >= 3
          )
          unless on ()
          (
            max(
              sum by (method, route) (
                increase(mnemo_http_requests_total{route=~"/v1.*",status=~"5.."}[5m])
              )
            ) >= 3
          )
        for: 0m
        labels:
          severity: major
          response_class: ticket
          component: mnemos
          oncall_team: mem9
        annotations:
          summary: "Mnemos overall 5xx burst detected"
          description: "Mnemos external APIs recorded at least three 5xx responses across routes over the past 5 minutes, current count: {{ $value | humanize }}"

      # Major route-local burst alert.
      - alert: MnemosRoute5xxBurst
        expr: |
          sum by (method, route) (
            increase(mnemo_http_requests_total{route=~"/v1.*",status=~"5.."}[5m])
          ) >= 3
        for: 0m
        labels:
          severity: major
          response_class: ticket
          component: mnemos
          oncall_team: mem9
        annotations:
          summary: "Mnemos route 5xx burst detected"
          description: "{{ $labels.method }} {{ $labels.route }} recorded at least three 5xx responses over the past 5 minutes, current count: {{ $value | humanize }}"

      # P95 external API latency alert - exceeds 5 seconds.
      - alert: MnemosHighLatency
        expr: |
          (
            histogram_quantile(
              0.95,
              sum by (le, method, route) (
                rate(mnemo_http_request_duration_seconds_bucket{route=~"/v1.*"}[5m])
              )
            ) > 5
          )
          and on (method, route)
          sum by (method, route) (
            increase(mnemo_http_request_duration_seconds_count{route=~"/v1.*"}[5m])
          ) >= 10
        for: 0m
        labels:
          severity: warning
          component: mnemos
          oncall_team: mem9
        annotations:
          summary: "Mnemos P95 latency is high"
          description: "P95 latency for {{ $labels.method }} {{ $labels.route }} exceeded 5 seconds over the past 5 minutes with at least 10 requests, current value: {{ $value | humanizeDuration }}"

      # P99 latency critical alert - exceeds 40 seconds
      - alert: MnemosCriticalLatency
        expr: |
          histogram_quantile(0.99,
            sum(rate(mnemo_http_request_duration_seconds_bucket[5m])) by (le, route)
          ) > 40
        for: 2m
        labels:
          severity: critical
          response_class: page
          component: mnemos
          oncall_team: mem9
        annotations:
          summary: "Mnemos P99 latency is critically high"
          description: "P99 latency for route {{ $labels.route }} exceeded 40 seconds over the past 5 minutes, current value: {{ $value | humanizeDuration }}"

      # Memory recall P99 latency alert - exceeds 10 seconds
      - alert: MnemosMemoryRecallHighLatency
        expr: |
          (
            histogram_quantile(0.99,
              sum(rate(mnemo_memory_recall_duration_seconds_bucket[5m])) by (le, mode)
            ) > 10
          )
          and on (mode)
          (
            sum by (mode) (increase(mnemo_memory_recall_duration_seconds_count[5m])) > 50
          )
        for: 3m
        labels:
          severity: warning
          component: mnemos
          oncall_team: mem9
        annotations:
          summary: "Mnemos memory recall P99 latency is high"
          description: "Memory recall mode {{ $labels.mode }} P99 latency exceeded 10 seconds over the past 5 minutes, current value: {{ $value | humanizeDuration }}"

      # Memory recall timeout alert
      - alert: MnemosMemoryRecallTimeouts
        expr: |
          (
            sum by (mode) (increase(mnemo_memory_recall_timeouts_total[5m])) > 3
          )
          and on (mode)
          (
            (
              sum by (mode) (increase(mnemo_memory_recall_timeouts_total[5m]))
              /
              sum by (mode) (increase(mnemo_memory_recall_duration_seconds_count[5m]))
            ) > 0.01
          )
        for: 0m
        labels:
          severity: warning
          component: mnemos
          oncall_team: mem9
        annotations:
          summary: "Mnemos memory recall timed out"
          description: "Memory recall mode {{ $labels.mode }} had {{ $value }} timeout(s) in the past 5 minutes"

      # Traffic drop alert - request volume dropped by more than 50% compared to 1 hour ago
      - alert: MnemosTrafficDrop
        expr: |
          (
            sum(rate(mnemo_http_requests_total[5m]))
            /
            sum(rate(mnemo_http_requests_total[1h] offset 1h))
          ) < 0.5
        for: 5m
        labels:
          severity: warning
          component: mnemos
          oncall_team: mem9
        annotations:
          summary: "Mnemos traffic drop detected"
          description: "Mnemos service request volume dropped by more than 50% compared to 1 hour ago, possible service degradation"

      # 4xx error rate alert - exceeds 10%
      - alert: MnemosHighClientErrorRate
        expr: |
          (
            sum(rate(mnemo_http_requests_total{status=~"4.."}[5m]))
            /
            sum(rate(mnemo_http_requests_total[5m]))
          ) > 0.10
        for: 5m
        labels:
          severity: warning
          component: mnemos
          oncall_team: mem9
        annotations:
          summary: "Mnemos client error rate is high"
          description: "Mnemos service 4xx error rate exceeded 10% over the past 5 minutes, current value: {{ $value | humanizePercentage }}"

      # Provision error rate alert - provision failures exceed 10%
      - alert: MnemosProvisionHighErrorRate
        expr: |
          (
            (
              sum(rate(mnemo_provision_total{result="error"}[5m]))
              /
              sum(rate(mnemo_provision_total[5m]))
            ) > 0.10
          )
          and
          (
            sum(increase(mnemo_provision_total[5m])) > 10
          )
        for: 2m
        labels:
          severity: warning
          component: mnemos
          oncall_team: mem9
        annotations:
          summary: "Mnemos provision error rate is high"
          description: "Provision failure rate exceeded 10% over the past 5 minutes, current value: {{ $value | humanizePercentage }}"

      # Provision critical error rate alert - provision failures exceed 30%
      - alert: MnemosProvisionCriticalErrorRate
        expr: |
          (
            (
              sum(rate(mnemo_provision_total{result="error"}[5m]))
              /
              sum(rate(mnemo_provision_total[5m]))
            ) > 0.30
          )
          and
          (
            sum(increase(mnemo_provision_total[5m])) > 10
          )
        for: 1m
        labels:
          severity: critical
          response_class: page
          component: mnemos
          oncall_team: mem9
        annotations:
          summary: "Mnemos provision critical error rate"
          description: "Provision failure rate exceeded 30% over the past 5 minutes, current value: {{ $value | humanizePercentage }}"

      # Provision total duration alert - P99 exceeds 30 seconds
      - alert: MnemosProvisionHighLatency
        expr: |
          histogram_quantile(0.99,
            sum(rate(mnemo_provision_step_duration_seconds_bucket{step="total"}[5m])) by (le)
          ) > 30
        for: 2m
        labels:
          severity: warning
          component: mnemos
          oncall_team: mem9
        annotations:
          summary: "Mnemos provision latency is high"
          description: "P99 provision total duration exceeded 30 seconds, current value: {{ $value | humanizeDuration }}"

      # TiDB Zero instance creation step slow alert
      - alert: MnemosProvisionTiDBZeroSlow
        expr: |
          histogram_quantile(0.99,
            sum(rate(mnemo_provision_step_duration_seconds_bucket{step="tidb_zero_create_instance"}[5m])) by (le)
          ) > 20
        for: 3m
        labels:
          severity: warning
          component: mnemos
          oncall_team: mem9
        annotations:
          summary: "Mnemos TiDB Zero instance creation is slow"
          description: "P99 TiDB Zero create instance step exceeded 20 seconds, current value: {{ $value | humanizeDuration }}"

      # K8s Pod restarting too frequently
      - alert: MnemosPodRestarting
        expr: |
          increase(kube_pod_container_status_restarts_total{namespace="mnemos",container="mnemos-server"}[30m]) > 3
        for: 5m
        labels:
          severity: warning
          component: mnemos
          oncall_team: mem9
        annotations:
          summary: "Mnemos pod is restarting frequently"
          description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has restarted {{ $value }} times in 30 minutes"

      # K8s Pod CrashLoopBackOff
      - alert: MnemosPodCrashLooping
        expr: |
          rate(kube_pod_container_status_restarts_total{namespace="mnemos",container="mnemos-server"}[5m]) > 0
          and
          kube_pod_container_status_waiting{namespace="mnemos",container="mnemos-server",reason="CrashLoopBackOff"} == 1
        for: 5m
        labels:
          severity: critical
          response_class: page
          component: mnemos
          oncall_team: mem9
        annotations:
          summary: "Mnemos pod is crash looping"
          description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} is in CrashLoopBackOff state"

      # K8s Pod Not Ready
      - alert: MnemosPodNotReady
        expr: |
          kube_deployment_status_replicas{namespace="mnemos",deployment="mnemos-server"}
          !=
          kube_deployment_status_replicas_available{namespace="mnemos",deployment="mnemos-server"}
        for: 5m
        labels:
          severity: critical
          response_class: page
          component: mnemos
          oncall_team: mem9
        annotations:
          summary: "Mnemos pod not ready"
          description: "Mnemos deployment has unavailable pods"

      # K8s Pod OOMKilled
      - alert: MnemosPodOOMKilled
        expr: |
          kube_pod_container_status_terminated_reason{namespace="mnemos",container="mnemos-server",reason="OOMKilled"} == 1
        for: 0m
        labels:
          severity: critical
          response_class: page
          component: mnemos
          oncall_team: mem9
        annotations:
          summary: "Mnemos pod OOMKilled"
          description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} was OOMKilled"

      # K8s Deployment replicas mismatch
      - alert: MnemosDeploymentReplicasMismatch
        expr: |
          (
            kube_deployment_spec_replicas{namespace="mnemos",deployment="mnemos-server"}
            !=
            kube_deployment_status_replicas_available{namespace="mnemos",deployment="mnemos-server"}
          ) and (
            changes(kube_deployment_status_replicas_updated{namespace="mnemos",deployment="mnemos-server"}[5m]) == 0
          )
        for: 10m
        labels:
          severity: warning
          component: mnemos
          oncall_team: mem9
        annotations:
          summary: "Mnemos deployment replicas mismatch"
          description: "Deployment mnemos/mnemos-server has not matched expected replicas for 10 minutes"

      # K8s High CPU usage
      - alert: MnemosHighCPUUsage
        expr: |
          (
            sum(rate(container_cpu_usage_seconds_total{namespace="mnemos",container="mnemos-server"}[5m]))
            /
            sum(kube_pod_container_resource_requests{namespace="mnemos",container="mnemos-server",resource="cpu"})
          ) > 0.8
        for: 10m
        labels:
          severity: warning
          component: mnemos
          oncall_team: mem9
        annotations:
          summary: "Mnemos high CPU usage"
          description: "Mnemos CPU usage is above 80% of requested for more than 10 minutes"

      # K8s High memory usage
      - alert: MnemosHighMemoryUsage
        expr: |
          (
            sum(container_memory_working_set_bytes{namespace="mnemos",container="mnemos-server"})
            /
            sum(kube_pod_container_resource_requests{namespace="mnemos",container="mnemos-server",resource="memory"})
          ) > 0.85
        for: 10m
        labels:
          severity: warning
          component: mnemos
          oncall_team: mem9
        annotations:
          summary: "Mnemos high memory usage"
          description: "Mnemos memory usage is above 85% of requested for more than 10 minutes"