# helm upgrade --kubeconfig ~/.kube/o11y-eks --kube-context arn:aws:eks:us-west-2:552185537300:cluster/centralized-storage-dev-regional-us-west-2 -n self-monitoring -i -f ./kube-prom-stack-regional.yaml self-monitoring ../../../charts/kube-prometheus-stack
# helm upgrade --kubeconfig ~/.kube/o11y-eks --kube-context arn:aws:eks:us-west-2:552185537300:cluster/o11y-staging-regional-us-west-2 -n self-monitoring -i -f ./kube-prom-stack-regional.yaml self-monitoring ../../../charts/kube-prometheus-stack

# helm upgrade --kubeconfig ~/.kube/o11y-eks --kube-context arn:aws:eks:us-west-2:557537366020:cluster/o11y-prod-regional-us-west-2 --set prometheus.prometheusSpec.externalLabels.o11y_k8s_id=o11y-prod-regional-us-west-2  -n self-monitoring -i -f ./kube-prom-stack-regional.yaml self-monitoring ../../../charts/kube-prometheus-stack
# helm upgrade --kubeconfig ~/.kube/o11y-eks --kube-context arn:aws:eks:us-east-1:557537366020:cluster/o11y-prod-regional-us-east-1 --set prometheus.prometheusSpec.externalLabels.o11y_k8s_id=o11y-prod-regional-us-east-1 -n self-monitoring -i -f ./kube-prom-stack-regional.yaml self-monitoring ../../../charts/kube-prometheus-stack
# helm upgrade --kubeconfig ~/.kube/o11y-eks --kube-context arn:aws:eks:ap-southeast-1:557537366020:cluster/o11y-prod-regional-ap-southeast-1 --set prometheus.prometheusSpec.externalLabels.o11y_k8s_id=o11y-prod-regional-ap-southeast-1 -n self-monitoring -i -f ./kube-prom-stack-regional.yaml self-monitoring ../../../charts/kube-prometheus-stack
# helm upgrade --kubeconfig ~/.kube/o11y-eks --kube-context arn:aws:eks:ap-northeast-1:557537366020:cluster/o11y-prod-regional-ap-northeast-1 --set prometheus.prometheusSpec.externalLabels.o11y_k8s_id=o11y-prod-regional-ap-northeast-1 -n self-monitoring -i -f ./kube-prom-stack-regional.yaml self-monitoring ../../../charts/kube-prometheus-stack
# helm upgrade --kubeconfig ~/.kube/o11y-eks --kube-context arn:aws:eks:ap-northeast-2:557537366020:cluster/o11y-prod-regional-ap-northeast-2 --set prometheus.prometheusSpec.externalLabels.o11y_k8s_id=o11y-prod-regional-ap-northeast-2 -n self-monitoring -i -f ./kube-prom-stack-regional.yaml self-monitoring ../../../charts/kube-prometheus-stack
# helm upgrade --kubeconfig ~/.kube/o11y-eks --kube-context arn:aws:eks:ap-south-1:557537366020:cluster/o11y-prod-regional-ap-south-1 --set prometheus.prometheusSpec.externalLabels.o11y_k8s_id=o11y-prod-regional-ap-south-1 -n self-monitoring -i -f ./kube-prom-stack-regional.yaml self-monitoring ../../../charts/kube-prometheus-stack
# helm upgrade --kubeconfig ~/.kube/o11y-eks --kube-context arn:aws:eks:eu-central-1:557537366020:cluster/o11y-prod-regional-eu-central-1 --set prometheus.prometheusSpec.externalLabels.o11y_k8s_id=o11y-prod-regional-eu-central-1 -n self-monitoring -i -f ./kube-prom-stack-regional.yaml self-monitoring ../../../charts/kube-prometheus-stack

alertmanager:
  enabled: true
  ingress:
    enabled: true
    ingressClassName: nginx
    annotations:
      nginx.ingress.kubernetes.io/rewrite-target: /$2
      nginx.ingress.kubernetes.io/use-regex: "true"
    paths:
    - "/internal/alerts/self-monitoring(/|$)(.*)"
  serviceAccount:
    create: true
    annotations:
      eks.amazonaws.com/role-arn: arn:aws:iam::557537366020:role/autonomous-o11y-self-monitoring-eks-prod
  config:
    global:
      resolve_timeout: 5m
    inhibit_rules:
      - source_matchers:
          - 'severity = critical'
        target_matchers:
          - 'severity =~ warning|info'
        equal:
          - 'namespace'
          - 'alertname'
      - source_matchers:
          - 'severity = warning'
        target_matchers:
          - 'severity = info'
        equal:
          - 'namespace'
          - 'alertname'
      - source_matchers:
          - 'alertname = InfoInhibitor'
        target_matchers:
          - 'severity = info'
        equal:
          - 'namespace'
    route:
      group_by: ['...']
      group_wait: 30s
      group_interval: 5m
      repeat_interval: 12h
      receiver: pager_test
      routes:
      - receiver: 'null'
        matchers:
          - alertname =~ "InfoInhibitor"
      - receiver: dms
        continue: true
        group_wait: 10s
        group_interval: 30s
        repeat_interval: 1m
        group_by: ['alertname','severity']
        matchers:
          - alertname = "Watchdog"
      - receiver: dms
        group_wait: 10s
        group_interval: 30s
        repeat_interval: 1m
        matchers:
        - alertname =~ "AlertmanagerClusterFailedToSendAlerts"
        - integration = "webhook" #jiralert
      - receiver: dead-mans-switch
        group_wait: 10s
        group_interval: 30s
        repeat_interval: 1m
        matchers:
          - alertname =~ "Watchdog"
      - receiver: auto-scaler-api-prod
        continue: true
        matchers:
          - alertname =~ "PrometheusCPUThrottlingHigh|O11yPodMemoryFillingUp|KubePersistentVolumeFillingUp"
      - receiver: default/observability-alertmanager-config/jiralert-prod-warning
        continue: false
        matchers:
          - 'severity =~ warning|info'
      - receiver: default/observability-alertmanager-config/jiralert-prod-major
        continue: false
        matchers:
          - 'severity =~ major'
      - receiver: default/observability-alertmanager-config/jiralert-prod
        continue: false
    receivers:
    - name: 'null'
    - name: pager_test
      pagerduty_configs:
      - send_resolved: true
        http_config:
          follow_redirects: true
        service_key: 66f7402e9daf4609d09b948ec3e966b2
        url: https://events.pagerduty.com/generic/2010-04-15/create_event.json
        client: '{{ template "pagerduty.default.client" . }}'
        client_url: '{{ template "pagerduty.default.clientURL" . }}'
        description: '{{ template "pagerduty.default.description" .}}'
        details:
          firing: '{{ template "pagerduty.default.instances" .Alerts.Firing }}'
          num_firing: '{{ .Alerts.Firing | len }}'
          num_resolved: '{{ .Alerts.Resolved | len }}'
          resolved: '{{ template "pagerduty.default.instances" .Alerts.Resolved }}'
    - name: 'default/observability-alertmanager-config/jiralert-prod'
      webhook_configs:
      - url: 'http://eelb-1570947222.us-west-1.elb.amazonaws.com:9097/alert'
        send_resolved: true
    - name: 'default/observability-alertmanager-config/jiralert-prod-warning'
      webhook_configs:
      - url: 'http://eelb-1570947222.us-west-1.elb.amazonaws.com:9097/alert'
        send_resolved: true
    - name: 'default/observability-alertmanager-config/jiralert-prod-major'
      webhook_configs:
      - url: 'http://eelb-1570947222.us-west-1.elb.amazonaws.com:9097/alert'
        send_resolved: true
    - name: dead-mans-switch
      webhook_configs:
      - url: http://dead-mans-switch:8080/webhook
    - name: 'auto-scaler-api-prod'
      webhook_configs:
        - url: 'https://www.gs.us-west-2.aws.observability.tidbcloud.com/api/v1/ops/events/alerts'
          send_resolved: true
    - name: dms
      webhook_configs:
      - url: http://localhost:8088/alert
  alertmanagerSpec:
    containers:
    - name: deadmanswatch
      image: 557537366020.dkr.ecr.us-west-2.amazonaws.com/deadmanswatch:sha-891e26c
      imagePullPolicy: IfNotPresent
      args:
        - watch
        - --port=8088
        - --listen-address=0.0.0.0
        - --region=us-west-2
        - --alert-source-label=o11y_k8s_id
        - --metric-dimensions="scope=self-monitoring"
        - --metric-dimensions="environment=prod"
      ports:
        - name: http
          containerPort: 8088
          protocol: TCP
      livenessProbe:
        httpGet:
          path: /ping
          port: http
      resources:
        requests:
          cpu: 100m
          memory: 50Mi
        limits:
          cpu: 100m
          memory: 50Mi
grafana:
  enabled: false

prometheusOperator:
  enabled: true

kubeControllerManager:
  enabled: false

kubeScheduler:
  enabled: false

kubeProxy:
  enabled: false

kubeApiServer:
  enabled: false

defaultRules:
  disabled:
    KubeAggregatedAPIDown: true
    KubeCPUOvercommit: true #we have Autoscaler so disable this alert
    KubeMemoryOvercommit: true #we have Autoscaler so disable this alert
    CPUThrottlingHigh: true #cfs limits are not accurate

prometheus:
  enabled: true
  prometheusSpec:
    externalLabels:
      o11y_k8s_id: o11y-prod-regional-ap-south-1
      component: observability-regional-server
    retention: 120d
    ## EXPERIMENTAL: Number of shards to distribute targets onto.
    ## Number of replicas multiplied by shards is the total number of Pods created.
    ## Note that scaling down shards will not reshard data onto remaining instances, it must be manually moved.
    ## Increasing shards will not reshard data either but it will continue to be available from the same instances.
    ## To query globally use Thanos sidecar and Thanos querier or remote write data to a central location.
    ## Sharding is done on the content of the `__address__` target meta-label.
    ##
    shards: 1
    resources:
      requests:
        memory: 2Gi

    ## Prometheus StorageSpec for persistent data
    ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/user-guides/storage.md
    ##
    storageSpec:
      volumeClaimTemplate:
        spec:
          storageClassName: prometheus.ssd
          accessModes: ["ReadWriteOnce"]
          resources:
            requests:
              storage: 100Gi
    serviceMonitorSelectorNilUsesHelmValues: false
    ## AdditionalScrapeConfigs allows specifying additional Prometheus scrape configurations. Scrape configurations
    ## are appended to the configurations generated by the Prometheus Operator. Job configurations must have the form
    ## as specified in the official Prometheus documentation:
    ## https://prometheus.io/docs/prometheus/latest/configuration/configuration/#scrape_config. As scrape configs are
    ## appended, the user is responsible to make sure it is valid. Note that using this feature may expose the possibility
    ## to break upgrades of Prometheus. It is advised to review Prometheus release notes to ensure that no incompatible
    ## scrape configs are going to break Prometheus after the upgrade.
    ## AdditionalScrapeConfigs can be defined as a list or as a templated string.
    ##
    ## The scrape configuration example below will find master nodes, provided they have the name .*mst.*, relabel the
    ## port to 2379 and allow etcd scraping provided it is running on all Kubernetes master nodes
    ##
    additionalScrapeConfigs: []
    # - job_name: kube-etcd
    #   kubernetes_sd_configs:
    #     - role: node
    #   scheme: https
    #   tls_config:
    #     ca_file:   /etc/prometheus/secrets/etcd-client-cert/etcd-ca
    #     cert_file: /etc/prometheus/secrets/etcd-client-cert/etcd-client
    #     key_file:  /etc/prometheus/secrets/etcd-client-cert/etcd-client-key
    #   relabel_configs:
    #   - action: labelmap
    #     regex: __meta_kubernetes_node_label_(.+)
    #   - source_labels: [__address__]
    #     action: replace
    #     targetLabel: __address__
    #     regex: ([^:;]+):(\d+)
    #     replacement: ${1}:2379
    #   - source_labels: [__meta_kubernetes_node_name]
    #     action: keep
    #     regex: .*mst.*
    #   - source_labels: [__meta_kubernetes_node_name]
    #     action: replace
    #     targetLabel: node
    #     regex: (.*)
    #     replacement: ${1}
    #   metric_relabel_configs:
    #   - regex: (kubernetes_io_hostname|failure_domain_beta_kubernetes_io_region|beta_kubernetes_io_os|beta_kubernetes_io_arch|beta_kubernetes_io_instance_type|failure_domain_beta_kubernetes_io_zone)
    #     action: labeldrop
    #
    ## If scrape config contains a repetitive section, you may want to use a template.
    ## In the following example, you can see how to define `gce_sd_configs` for multiple zones
    # additionalScrapeConfigs: |
    #  - job_name: "node-exporter"
    #    gce_sd_configs:
    #    {{range $zone := .Values.gcp_zones}}
    #    - project: "project1"
    #      zone: "{{$zone}}"
    #      port: 9100
    #    {{end}}
    #    relabel_configs:
    #    ...


    ## If additional scrape configurations are already deployed in a single secret file you can use this section.
    ## Expected values are the secret name and key
    ## Cannot be used with additionalScrapeConfigs
    additionalScrapeConfigsSecret: {}
      # enabled: false
      # name:
      # key:


    ## AdditionalAlertManagerConfigs allows for manual configuration of alertmanager jobs in the form as specified
    ## in the official Prometheus documentation https://prometheus.io/docs/prometheus/latest/configuration/configuration/#<alertmanager_config>.
    ## AlertManager configurations specified are appended to the configurations generated by the Prometheus Operator.
    ## As AlertManager configs are appended, the user is responsible to make sure it is valid. Note that using this
    ## feature may expose the possibility to break upgrades of Prometheus. It is advised to review Prometheus release
    ## notes to ensure that no incompatible AlertManager configs are going to break Prometheus after the upgrade.
    ##
    additionalAlertManagerConfigs: []
    # - consul_sd_configs:
    #   - server: consul.dev.test:8500
    #     scheme: http
    #     datacenter: dev
    #     tag_separator: ','
    #     services:
    #       - metrics-prometheus-alertmanager

    ## If additional alertmanager configurations are already deployed in a single secret, or you want to manage
    ## them separately from the helm deployment, you can use this section.
    ## Expected values are the secret name and key
    ## Cannot be used with additionalAlertManagerConfigs
    additionalAlertManagerConfigsSecret: {}
      # name:
      # key:

    ## AdditionalAlertRelabelConfigs allows specifying Prometheus alert relabel configurations. Alert relabel configurations specified are appended
    ## to the configurations generated by the Prometheus Operator. Alert relabel configurations specified must have the form as specified in the
    ## official Prometheus documentation: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#alert_relabel_configs.
    ## As alert relabel configs are appended, the user is responsible to make sure it is valid. Note that using this feature may expose the
    ## possibility to break upgrades of Prometheus. It is advised to review Prometheus release notes to ensure that no incompatible alert relabel
    ## configs are going to break Prometheus after the upgrade.
    ##
    additionalAlertRelabelConfigs: []
    # - separator: ;
    #   regex: prometheus_replica
    #   replacement: $1
    #   action: labeldrop

    ## If additional alert relabel configurations are already deployed in a single secret, or you want to manage
    ## them separately from the helm deployment, you can use this section.
    ## Expected values are the secret name and key
    ## Cannot be used with additionalAlertRelabelConfigs
    additionalAlertRelabelConfigsSecret: {}
      # name:
      # key: