apiVersion: v1
kind: ConfigMap
metadata:
  name: community-prometheus-v2
  namespace: community-monitor
data:
  prometheus.yml: |
    global:
      scrape_interval:     10s
      evaluation_interval: 10s

    alerting:
      alertmanagers:
        - scheme: http
          static_configs:
            - targets: [ 'community-alertmanager' ]

    rule_files:
      - "sre_rules.yml"
      - "pingcap_account_rules.yml"
      - "asktug_pg_rules.yml"

    scrape_configs:
      - job_name: prometheus
        static_configs:
          - targets: ['localhost:9090']
      - job_name: node_exporter
        static_configs:
          - targets: ['10.9.138.134:9100', '10.9.168.59:9100', '10.9.47.164:9100', '10.9.15.246:9100']
      - job_name: mysqld_exporter
        static_configs:
          - targets: ['10.9.47.164:9104']
      - job_name: discourse
        scheme: https
        static_configs:
          - targets: ['asktug.com']
      - job_name: pingcap_account
        metrics_path: /monitoring/metrics
        scheme: https
        static_configs:
          - targets: ['accounts.pingcap.cn']
      - job_name: asktug_pg
        static_configs:
          - targets: ['10.9.138.134:9187']

---

apiVersion: v1
kind: ConfigMap
metadata:
  name: community-prometheus-rules
  namespace: community-monitor
data:
  sre_rules.yml: |
    groups:
    - name: sre_rules
      rules:
      - alert: PrometheusJobMissing
        expr: absent(up{job="prometheus"})
        for: 20s
        labels:
          severity: warning
        annotations:
          summary: Prometheus job missing (instance {{ $labels.instance }})
          description: "A Prometheus job has disappeared\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
      - alert: PrometheusTargetMissing
        expr: up == 0
        for: 20s
        labels:
          severity: critical
        annotations:
          summary: Prometheus target missing (instance {{ $labels.instance }})
          description: "A Prometheus target has disappeared. An exporter might be crashed.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
      - alert: HostOutOfMemory
        expr: node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes * 100 < 10
        for: 2m
        labels:
          severity: warning
        annotations:
          summary: Host out of memory (instance {{ $labels.instance }})
          description: "Node memory is filling up (< 10% left)\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
      - alert: HostMemoryUnderMemoryPressure
        expr: rate(node_vmstat_pgmajfault[1m]) > 1000
        for: 2m
        labels:
          severity: warning
        annotations:
          summary: Host memory under memory pressure (instance {{ $labels.instance }})
          description: "The node is under heavy memory pressure. High rate of major page faults\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
      - alert: HostUnusualNetworkThroughputIn
        expr: sum by (instance) (rate(node_network_receive_bytes_total[2m])) / 1024 / 1024 > 80
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: Host unusual network throughput in (instance {{ $labels.instance }})
          description: "Host network interfaces are probably receiving too much data (> 80 MB/s)\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
      - alert: HostUnusualNetworkThroughputOut
        expr: sum by (instance) (rate(node_network_transmit_bytes_total[2m])) / 1024 / 1024 > 80
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: Host unusual network throughput out (instance {{ $labels.instance }})
          description: "Host network interfaces are probably sending too much data (> 80 MB/s)\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
      - alert: HostUnusualDiskReadRate
        expr: sum by (instance) (rate(node_disk_read_bytes_total[2m])) / 1024 / 1024 > 100
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: Host unusual disk read rate (instance {{ $labels.instance }})
          description: "Disk is probably reading too much data (> 100 MB/s)\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
      - alert: HostUnusualDiskWriteRate
        expr: sum by (instance) (rate(node_disk_written_bytes_total[2m])) / 1024 / 1024 > 100
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: Host unusual disk write rate (instance {{ $labels.instance }})
          description: "Disk is probably writing too much data (> 100 MB/s)\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
      - alert: HostOutOfDiskSpace
        expr: (node_filesystem_avail_bytes * 100) / node_filesystem_size_bytes < 10 and ON (instance, device, mountpoint) node_filesystem_readonly == 0
        for: 2m
        labels:
          severity: warning
        annotations:
          summary: Host out of disk space (instance {{ $labels.instance }})
          description: "Disk is almost full (< 10% left)\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
      # Please add ignored mountpoints in node_exporter parameters like
      # "--collector.filesystem.ignored-mount-points=^/(sys|proc|dev|run)($|/)".
      # Same rule using "node_filesystem_free_bytes" will fire when disk fills for non-root users.
      - alert: HostOutOfDiskSpace
        expr: (node_filesystem_avail_bytes * 100) / node_filesystem_size_bytes < 10 and ON (instance, device, mountpoint) node_filesystem_readonly == 0
        for: 2m
        labels:
          severity: warning
        annotations:
          summary: Host out of disk space (instance {{ $labels.instance }})
          description: "Disk is almost full (< 10% left)\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
      # Please add ignored mountpoints in node_exporter parameters like
      # "--collector.filesystem.ignored-mount-points=^/(sys|proc|dev|run)($|/)".
      # Same rule using "node_filesystem_free_bytes" will fire when disk fills for non-root users.
      - alert: HostDiskWillFillIn24Hours
        expr: (node_filesystem_avail_bytes * 100) / node_filesystem_size_bytes < 10 and ON (instance, device, mountpoint) predict_linear(node_filesystem_avail_bytes{fstype!~"tmpfs"}[1h], 24 * 3600) < 0 and ON (instance, device, mountpoint) node_filesystem_readonly == 0
        for: 2m
        labels:
          severity: warning
        annotations:
          summary: Host disk will fill in 24 hours (instance {{ $labels.instance }})
          description: "Filesystem is predicted to run out of space within the next 24 hours at current write rate\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
      - alert: HostOutOfInodes
        expr: node_filesystem_files_free{mountpoint ="/rootfs"} / node_filesystem_files{mountpoint="/rootfs"} * 100 < 10 and ON (instance, device, mountpoint) node_filesystem_readonly{mountpoint="/rootfs"} == 0
        for: 2m
        labels:
          severity: warning
        annotations:
          summary: Host out of inodes (instance {{ $labels.instance }})
          description: "Disk is almost running out of available inodes (< 10% left)\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
      - alert: HostUnusualDiskReadLatency
        expr: rate(node_disk_read_time_seconds_total[1m]) / rate(node_disk_reads_completed_total[1m]) > 0.1 and rate(node_disk_reads_completed_total[1m]) > 0
        for: 2m
        labels:
          severity: warning
        annotations:
          summary: Host unusual disk read latency (instance {{ $labels.instance }})
          description: "Disk latency is growing (read operations > 100ms)\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
      - alert: HostUnusualDiskWriteLatency
        expr: rate(node_disk_write_time_seconds_total[1m]) / rate(node_disk_writes_completed_total[1m]) > 0.1 and rate(node_disk_writes_completed_total[1m]) > 0
        for: 2m
        labels:
          severity: warning
        annotations:
          summary: Host unusual disk write latency (instance {{ $labels.instance }})
          description: "Disk latency is growing (write operations > 100ms)\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
      - alert: HostHighCpuLoad
        expr: 100 - (avg by(instance) (rate(node_cpu_seconds_total{mode="idle"}[2m])) * 100) > 80
        for: 1m
        labels:
          severity: warning
        annotations:
          summary: Host high CPU load (instance {{ $labels.instance }})
          description: "CPU load is > 80%\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }} for 1 minute"
      - alert: HostSystemdServiceCrashed
        expr: node_systemd_unit_state{state="failed"} == 1
        for: 0m
        labels:
          severity: warning
        annotations:
          summary: Host systemd service crashed (instance {{ $labels.instance }})
          description: "systemd service crashed\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
      - alert: HostOomKillDetected
        expr: increase(node_vmstat_oom_kill[1m]) > 0
        for: 0m
        labels:
          severity: warning
        annotations:
          summary: Host OOM kill detected (instance {{ $labels.instance }})
          description: "OOM kill detected\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
  asktug_pg_rules.yml: |
    groups:
    - name: sre_rules
      rules:
      - alert: PostgresqlDown
        expr: pg_up == 0
        for: 0m
        labels:
          severity: critical
        annotations:
          summary: Postgresql down (instance {{ $labels.instance }})
          description: "Postgresql instance is down\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
      - alert: PostgresqlExporterError
        expr: pg_exporter_last_scrape_error > 0
        for: 0m
        labels:
          severity: critical
        annotations:
          summary: Postgresql exporter error (instance {{ $labels.instance }})
          description: "Postgresql exporter is showing errors. A query may be buggy in query.yaml\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
      - alert: PostgresqlTooManyConnections
        expr: sum by (datname) (pg_stat_activity_count{datname!~"template.*|postgres"}) > pg_settings_max_connections * 0.8
        for: 2m
        labels:
          severity: warning
        annotations:
          summary: Postgresql too many connections (instance {{ $labels.instance }})
          description: "PostgreSQL instance has too many connections (> 80%).\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
      # - alert: PostgresqlNotEnoughConnections
      #   expr: sum by (datname) (pg_stat_activity_count{datname='discourse'}) < 5
      #   for: 2m
      #   labels:
      #     severity: warning
      #   annotations:
      #     summary: Postgresql not enough connections (instance {{ $labels.instance }})
      #     description: "PostgreSQL instance should have more connections (> 5)\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
      - alert: PostgresqlDeadLocks
        expr: increase(pg_stat_database_deadlocks{datname!~"template.*|postgres"}[1m]) > 5
        for: 0m
        labels:
          severity: warning
        annotations:
          summary: Postgresql dead locks (instance {{ $labels.instance }})
          description: "PostgreSQL has dead-locks\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
      - alert: PostgresqlHighRollbackRate
        expr: rate(pg_stat_database_xact_rollback{datname!~"template.*"}[3m]) / rate(pg_stat_database_xact_commit{datname!~"template.*"}[3m]) > 0.02
        for: 0m
        labels:
          severity: warning
        annotations:
          summary: Postgresql high rollback rate (instance {{ $labels.instance }})
          description: "Ratio of transactions being aborted compared to committed is > 2 %\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
      # - alert: PostgresqlCommitRateLow
      #   expr: rate(pg_stat_database_xact_commit[1m]) < 10
      #   for: 2m
      #   labels:
      #     severity: critical
      #   annotations:
      #     summary: Postgresql commit rate low (instance {{ $labels.instance }})
      #     description: "Postgresql seems to be processing very few transactions\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
      - alert: PostgresqlTooManyLocksAcquired
        expr: ((sum (pg_locks_count)) / (pg_settings_max_locks_per_transaction * pg_settings_max_connections)) > 0.20
        for: 2m
        labels:
          severity: critical
        annotations:
          summary: Postgresql too many locks acquired (instance {{ $labels.instance }})
          description: "Too many locks acquired on the database. If this alert happens frequently, we may need to increase the postgres setting max_locks_per_transaction.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
      - alert: PostgreSQLSlowQueries
        expr: avg(rate(pg_stat_activity_max_tx_duration{datname!~"template.*"}[2m])) by (datname) > 2 * 60
        for: 2m
        labels:
          severity: warning
        annotations:
          summary: PostgreSQL high number of slow queries per second {{ $labels.cluster }} for database {{ $labels.datname }}
          description: PostgreSQL high number of slow queries per second on {{ $labels.cluster }} for database {{ $labels.datname }} with a value of {{ $value }}
      - alert: PostgreSQLQPS
        expr: avg(irate(pg_stat_database_xact_commit{datname!~"template.*"}[5m]) + irate(pg_stat_database_xact_rollback{datname!~"template.*"}[5m])) by (datname) > 10000
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: PostgreSQL high number of queries per second {{ $labels.cluster }} for database {{ $labels.datname }}
          description: PostgreSQL high number of queries per second on {{ $labels.cluster }} for database {{ $labels.datname }} with a value of {{ $value }}
      - alert: PostgreSQLCacheHitRatio
        expr: avg(rate(pg_stat_database_blks_hit{datname!~"template.*"}[5m]) / (rate(pg_stat_database_blks_hit{datname!~"template.*"}[5m]) + rate(pg_stat_database_blks_read{datname!~"template.*"}[5m]))) by (datname) < 0.98
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: PostgreSQL low cache hit rate on {{ $labels.cluster }} for database {{ $labels.datname }}
          description: PostgreSQL low on cache hit rate on {{ $labels.cluster }} for database {{ $labels.datname }} with a value of {{ $value }}
  pingcap_account_rules.yml: |
    groups:
    - name: pingcap_account_rules
      rules:
      - record: job:django_http_requests_latency_including_middlewares_seconds:quantile_rate30s
        expr: histogram_quantile(0.5, sum(rate(django_http_requests_latency_including_middlewares_seconds_bucket[30s]))
          BY (job, le))
        labels:
          quantile: "50"
      - record: job:django_http_requests_latency_including_middlewares_seconds:quantile_rate30s
        expr: histogram_quantile(0.95, sum(rate(django_http_requests_latency_including_middlewares_seconds_bucket[30s]))
          BY (job, le))
        labels:
          quantile: "95"
      - record: job:django_http_requests_latency_including_middlewares_seconds:quantile_rate30s
        expr: histogram_quantile(0.99, sum(rate(django_http_requests_latency_including_middlewares_seconds_bucket[30s]))
          BY (job, le))
        labels:
          quantile: "99"
      - record: job:django_http_requests_latency_including_middlewares_seconds:quantile_rate30s
        expr: histogram_quantile(0.999, sum(rate(django_http_requests_latency_including_middlewares_seconds_bucket[30s]))
          BY (job, le))
        labels:
          quantile: "99.9"
      - alert: TooManyUnusedSecurityCodes
        expr: app_last_1hr_security_code{status='unused',job='pingcap_account'} > 20
        for: 30s
        labels:
          severity: warning
        annotations:
          summary: There are too many unused security code, it may indicate an attack
      - alert: LarkMessageUndelivered
        expr: app_undelivered_lark_message{job='pingcap_account'} > 0
        for: 1m
        labels:
          severity: critical
        annotations:
          summary: '{{ $value }} lark messages not delivered, check your message queue'
      - alert: CRMLeadsCreateFailed
        expr: app_undelivered_crm_leads{job='pingcap_account'} > 0
        for: 1m
        labels:
          severity: critical
        annotations:
          summary: '{{ $value }} leads sync to crm failed, check your message queue'

---

apiVersion: apps/v1
kind: Deployment
metadata:
  name: prometheus
  namespace: community-monitor
  labels:
    app: community-prometheus
spec:
  selector:
    matchLabels:
      app: community-prometheus
  replicas: 1
  template:
    metadata:
      labels:
        app: community-prometheus
    spec:
      containers:
      - name: prometheus
        image: uhub.service.ucloud.cn/pingcap-community/prometheus:v2.39.1
        args:
        - --config.file=/etc/prometheus/prometheus.yml
        - --storage.tsdb.path=/prometheus
        - --storage.tsdb.retention.time=240h
        - --storage.tsdb.retention.size=90GB
        - --storage.tsdb.no-lockfile
        - --web.console.libraries=/usr/share/prometheus/console_libraries
        - --web.console.templates=/usr/share/prometheus/consoles
        - --web.enable-lifecycle
        volumeMounts:
        - name: data-volume
          mountPath: /prometheus
        - name: prometheus-conf
          mountPath: /etc/prometheus/prometheus.yml
          subPath: prometheus.yml
        - name: prometheus-rules-conf
          mountPath: /etc/prometheus/sre_rules.yml
          subPath: sre_rules.yml
        - name: prometheus-rules-conf
          mountPath: /etc/prometheus/pingcap_account_rules.yml
          subPath: pingcap_account_rules.yml
        - name: prometheus-rules-conf
          mountPath: /etc/prometheus/asktug_pg_rules.yml
          subPath: asktug_pg_rules.yml
        livenessProbe:
          httpGet:
            path: /-/healthy
            port: http
          periodSeconds: 10
          initialDelaySeconds: 10
          timeoutSeconds: 4
        readinessProbe:
          httpGet:
            path: /-/ready
            port: http
          periodSeconds: 5
          successThreshold: 1
          timeoutSeconds: 4
        ports:
        - containerPort: 9090
          name: http
        securityContext:
          runAsNonRoot: true
          runAsUser: 1000
        resources:
          limits:
            cpu: "2"
            memory: 512Mi
          requests:
            cpu: "0.25"
            memory: 128Mi
      - name: config-reloader
        image: uhub.service.ucloud.cn/uk8s/configmap-reload:v0.3.0
        imagePullPolicy: IfNotPresent
        args:
        - -webhook-url=http://127.0.0.1:9090/-/reload
        - -volume-dir=/etc/prometheus_config
        - -volume-dir=/etc/prometheus_rules
        resources:
          limits:
            cpu: 100m
            memory: 25Mi
        terminationMessagePath: /dev/termination-log
        terminationMessagePolicy: FallbackToLogsOnError
        volumeMounts:
        - name: prometheus-conf
          mountPath: /etc/prometheus_config
        - name: prometheus-rules-conf
          mountPath: /etc/prometheus_rules
      volumes:
      - name: data-volume
        persistentVolumeClaim:
          claimName: community-prometheus-ufs-01
      - name: prometheus-conf
        configMap:
          name: community-prometheus-v2
          defaultMode: 420
      - name: prometheus-rules-conf
        configMap:
          name: community-prometheus-rules
          defaultMode: 420

---

apiVersion: v1
kind: Service
metadata:
  labels:
    app: community-prometheus
  name: community-prometheus
  namespace: community-monitor
spec:
  ports:
  - name: http
    port: 80
    protocol: TCP
    targetPort: http
  selector:
    app: community-prometheus

---

apiVersion: v1
kind: PersistentVolume
metadata:
  name: community-prometheus-ufs-01
spec:
  capacity:
    storage: 50Gi
  accessModes:
    - ReadWriteMany
  persistentVolumeReclaimPolicy: Retain
  nfs:
    path: /community-monitor/prometheus/
    server: 10.19.173.98
  mountOptions:
    - nolock
    - nfsvers=4.0

---

apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: community-prometheus-ufs-01
  namespace: community-monitor
spec:
  accessModes:
  - ReadWriteMany
  resources:
    requests:
      storage: 50Gi