rule_files: - ../rules/mem9/mnemos/mnemos-alerts.yaml evaluation_interval: 1m tests: - name: route burst fires on the third external 5xx and later resolves interval: 1m input_series: - series: 'mnemo_http_requests_total{method="GET",route="/v1alpha2/mem9s/{tenantID}/memories",status="500"}' values: '0 0 1 2 3 3 3 3 3 3 3' - series: 'mnemo_http_requests_total{method="GET",route="/v1alpha2/mem9s/{tenantID}/memories",status="200"}' values: '0 100 200 300 400 500 600 700 800 900 1000' alert_rule_test: - eval_time: 4m alertname: MnemosRoute5xxBurst exp_alerts: - exp_labels: method: GET route: /v1alpha2/mem9s/{tenantID}/memories severity: major response_class: ticket component: mnemos oncall_team: mem9 exp_annotations: summary: "Mnemos route 5xx burst detected" description: "GET /v1alpha2/mem9s/{tenantID}/memories recorded at least three 5xx responses over the past 5 minutes, current count: 3" - eval_time: 4m alertname: MnemosCriticalErrorRate exp_alerts: [] - eval_time: 9m alertname: MnemosRoute5xxBurst exp_alerts: [] - name: two external 5xx responses do not fire interval: 1m input_series: - series: 'mnemo_http_requests_total{method="POST",route="/v1alpha2/mem9s/{tenantID}/memories",status="500"}' values: '0 0 1 2 2 2' - series: 'mnemo_http_requests_total{method="POST",route="/v1alpha2/mem9s/{tenantID}/memories",status="200"}' values: '0 10 20 30 40 50' alert_rule_test: - eval_time: 5m alertname: MnemosRoute5xxBurst exp_alerts: [] - eval_time: 5m alertname: MnemosCriticalErrorRate exp_alerts: [] - name: burst critical suppresses the slower high error rate warning interval: 1m input_series: - series: 'mnemo_http_requests_total{method="GET",route="/v1alpha2/status",status="500"}' values: '0 1 2 3 4 5' - series: 'mnemo_http_requests_total{method="GET",route="/v1alpha2/status",status="200"}' values: '0 0 0 0 0 0' alert_rule_test: - eval_time: 5m alertname: MnemosHighErrorRate exp_alerts: [] - name: errors spread across routes fire only the overall alert interval: 1m input_series: - series: 'mnemo_http_requests_total{method="GET",route="/v1alpha2/status",status="500"}' values: '0 0 1 2 2 2' - series: 'mnemo_http_requests_total{method="POST",route="/v1alpha1/mem9s",status="500"}' values: '0 0 0 0 1 1' alert_rule_test: - eval_time: 5m alertname: MnemosCriticalErrorRate exp_alerts: - exp_labels: severity: major response_class: ticket component: mnemos oncall_team: mem9 exp_annotations: summary: "Mnemos overall 5xx burst detected" description: "Mnemos external APIs recorded at least three 5xx responses across routes over the past 5 minutes, current count: 3" - eval_time: 5m alertname: MnemosRoute5xxBurst exp_alerts: [] - name: internal endpoint errors are excluded interval: 1m input_series: - series: 'mnemo_http_requests_total{method="GET",route="/healthz",status="500"}' values: '0 0 1 2 3 3' alert_rule_test: - eval_time: 5m alertname: MnemosCriticalErrorRate exp_alerts: [] - eval_time: 5m alertname: MnemosRoute5xxBurst exp_alerts: [] - name: external API P95 latency above five seconds fires with enough traffic interval: 1m input_series: - series: 'mnemo_http_request_duration_seconds_bucket{method="GET",route="/v1alpha2/status",le="5"}' values: '0+0x6' - series: 'mnemo_http_request_duration_seconds_bucket{method="GET",route="/v1alpha2/status",le="10"}' values: '0+10x6' - series: 'mnemo_http_request_duration_seconds_bucket{method="GET",route="/v1alpha2/status",le="+Inf"}' values: '0+10x6' - series: 'mnemo_http_request_duration_seconds_count{method="GET",route="/v1alpha2/status"}' values: '0+10x6' alert_rule_test: - eval_time: 5m alertname: MnemosHighLatency exp_alerts: - exp_labels: method: GET route: /v1alpha2/status severity: warning component: mnemos oncall_team: mem9 exp_annotations: summary: "Mnemos P95 latency is high" description: "P95 latency for GET /v1alpha2/status exceeded 5 seconds over the past 5 minutes with at least 10 requests, current value: 9.75s" - name: latency below threshold does not fire interval: 1m input_series: - series: 'mnemo_http_request_duration_seconds_bucket{method="GET",route="/v1alpha2/status",le="5"}' values: '0+10x6' - series: 'mnemo_http_request_duration_seconds_bucket{method="GET",route="/v1alpha2/status",le="10"}' values: '0+10x6' - series: 'mnemo_http_request_duration_seconds_bucket{method="GET",route="/v1alpha2/status",le="+Inf"}' values: '0+10x6' - series: 'mnemo_http_request_duration_seconds_count{method="GET",route="/v1alpha2/status"}' values: '0+10x6' alert_rule_test: - eval_time: 5m alertname: MnemosHighLatency exp_alerts: [] - name: latency requires at least ten requests and excludes internal endpoints interval: 1m input_series: - series: 'mnemo_http_request_duration_seconds_bucket{method="GET",route="/v1alpha2/status",le="5"}' values: '0+0x6' - series: 'mnemo_http_request_duration_seconds_bucket{method="GET",route="/v1alpha2/status",le="10"}' values: '0+1x6' - series: 'mnemo_http_request_duration_seconds_bucket{method="GET",route="/v1alpha2/status",le="+Inf"}' values: '0+1x6' - series: 'mnemo_http_request_duration_seconds_count{method="GET",route="/v1alpha2/status"}' values: '0+1x6' - series: 'mnemo_http_request_duration_seconds_bucket{method="GET",route="/metrics",le="5"}' values: '0+0x6' - series: 'mnemo_http_request_duration_seconds_bucket{method="GET",route="/metrics",le="10"}' values: '0+10x6' - series: 'mnemo_http_request_duration_seconds_bucket{method="GET",route="/metrics",le="+Inf"}' values: '0+10x6' - series: 'mnemo_http_request_duration_seconds_count{method="GET",route="/metrics"}' values: '0+10x6' alert_rule_test: - eval_time: 5m alertname: MnemosHighLatency exp_alerts: []