apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
  labels:
    release: self-monitoring
  name: self-monitoring-dataservice-dataapp-gateway
  namespace: self-monitoring
spec:
  groups:
    - name: self-monitoring-dataservice-dataapp-gateway
      rules:
        - alert: multi-region-alert-test
          annotations:
            message: multi-region-alert-test
            description: multi-region-alert-test
            summary: multi-region-alert-test
          expr: |
            (sum(increase(tiinf_gin_request_total{url="/api/v1/dataapps/test"}[2m])))>=1
          labels:
            severity: critical
            component: dataservice

        - alert: DataappGateway_PodCpuUsage_GT50
          expr: |
            sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{namespace="dataservice",container="dataservice-dataapp-gateway"}) by (pod) / sum(kube_pod_container_resource_limits{namespace="dataservice",container="dataservice-dataapp-gateway",resource="cpu"}) by (pod) > 0.5
          for: 5m
          annotations:
            message: Pod {{ $labels.namespace }} / {{$labels.container}} / {{ $labels.pod }} cpu usage has been greater than 50% for longer than 5 minutes.
          labels:
            severity: critical
            component: dataservice

        - alert: DataappGateway_PodMemUsage_GT70
          expr: |
            sum(container_memory_working_set_bytes{namespace="dataservice",container="dataservice-dataapp-gateway"}) by (pod) / sum(kube_pod_container_resource_limits{namespace="dataservice",container="dataservice-dataapp-gateway",resource="memory"}) by (pod) > 0.7
          for: 5m
          annotations:
            message: Pod {{ $labels.namespace }} / {{$labels.container}} / {{ $labels.pod }} memory usage has been greater than 70% for longer than 5 minutes.
          labels:
            severity: critical
            component: dataservice

        - alert: DataappGatewayRegion_QPS_GT50
          expr: |
            sum(increase(dataservices_dataapi_gateway_http_requests_total{}[10m])) / 600 > 50
          for: 5m
          annotations:
            message: dataservice-dataapp-gateway region QPS has been greater than 50 for longer than 5 minutes.
          labels:
            severity: major
            component: dataservice

        - alert: DataappGatewayRegionEndpoint_QPS_GT50
          expr: |
            sum(increase(dataservices_dataapi_gateway_http_requests_total{path=~"/api/v1beta/app/.*"}[10m])) / 600 > 50
          for: 5m
          annotations:
            message: dataservice-dataapp-gateway region endpoint QPS has been greater than 50 for longer than 5 minutes.
          labels:
            severity: major
            component: dataservice

        - alert: DataappGatewayRegionAPIRequest500
          expr: |
            sum(increase(dataservices_dataapi_gateway_http_requests_total{status=~"5.*"}[2m])) by(path, method) > 0
          annotations:
            message: dataservice-dataapp-gateway region API Request 500.
          labels:
            severity: major
            component: dataservice

        - alert: DataappGatewayRegionAPIRequest500_Many
          expr: |
            sum(increase(dataservices_dataapi_gateway_http_requests_total{status=~"5.*"}[2m])) by(path, method) > 5
          annotations:
            message: dataservice-dataapp-gateway region API Request 500 count greater than 5 in 2 minutes.
          labels:
            severity: critical
            component: dataservice

        - alert: DataappGatewayRegionAPIRequest500_Last10min
          expr: |
            sum(increase(dataservices_dataapi_gateway_http_requests_total{status=~"5.*"}[2m])) by(path, method) > 0
          for: 10m
          annotations:
            message: dataservice-dataapp-gateway region API Request 500 last 10 minutes.
          labels:
            severity: critical
            component: dataservice