# helm upgrade --kubeconfig ~/.kube/o11y-eks --kube-context arn:aws:eks:us-west-2:552185537300:cluster/centralized-storage-dev-regional-us-west-2 -n self-monitoring -i -f ./kube-prom-stack-regional.yaml self-monitoring ../../../charts/kube-prometheus-stack
# helm upgrade --kubeconfig ~/.kube/o11y-eks --kube-context arn:aws:eks:us-west-2:552185537300:cluster/o11y-staging-regional-us-west-2 -n self-monitoring -i -f ./kube-prom-stack-regional.yaml self-monitoring ../../../charts/kube-prometheus-stack

# helm upgrade --kubeconfig ~/.kube/o11y-eks --kube-context arn:aws:eks:us-west-2:557537366020:cluster/o11y-prod-regional-us-west-2 -n self-monitoring -i -f ./kube-prom-stack-regional.yaml self-monitoring ../../../charts/kube-prometheus-stack
# helm upgrade --kubeconfig ~/.kube/o11y-eks --kube-context arn:aws:eks:us-east-1:557537366020:cluster/o11y-prod-regional-us-east-1 -n self-monitoring -i -f ./kube-prom-stack-regional.yaml self-monitoring ../../../charts/kube-prometheus-stack
# helm upgrade --kubeconfig ~/.kube/o11y-eks --kube-context arn:aws:eks:ap-southeast-1:557537366020:cluster/o11y-prod-regional-ap-southeast-1 -n self-monitoring -i -f ./kube-prom-stack-regional.yaml self-monitoring ../../../charts/kube-prometheus-stack
# helm upgrade --kubeconfig ~/.kube/o11y-eks --kube-context arn:aws:eks:ap-northeast-1:557537366020:cluster/o11y-prod-regional-ap-northeast-1 -n self-monitoring -i -f ./kube-prom-stack-regional.yaml self-monitoring ../../../charts/kube-prometheus-stack
# helm upgrade --kubeconfig ~/.kube/o11y-eks --kube-context arn:aws:eks:ap-northeast-2:557537366020:cluster/o11y-prod-regional-ap-northeast-2 -n self-monitoring -i -f ./kube-prom-stack-regional.yaml self-monitoring ../../../charts/kube-prometheus-stack
# helm upgrade --kubeconfig ~/.kube/o11y-eks --kube-context arn:aws:eks:ap-south-1:557537366020:cluster/o11y-prod-regional-ap-south-1 -n self-monitoring -i -f ./kube-prom-stack-regional.yaml self-monitoring ../../../charts/kube-prometheus-stack
# helm upgrade --kubeconfig ~/.kube/o11y-eks --kube-context arn:aws:eks:eu-central-1:557537366020:cluster/o11y-prod-regional-eu-central-1 -n self-monitoring -i -f ./kube-prom-stack-regional.yaml self-monitoring ../../../charts/kube-prometheus-stack
alertmanager:
  enabled: true
  ingress:
    enabled: true
    ingressClassName: nginx
    annotations:
      nginx.ingress.kubernetes.io/rewrite-target: /$2
      nginx.ingress.kubernetes.io/use-regex: "true"
    paths:
      - "/internal/alerts/self-monitoring(/|$)(.*)"
  config:
    global:
      resolve_timeout: 5m
    inhibit_rules:
      - source_matchers:
          - "severity = critical"
        target_matchers:
          - "severity =~ warning|info"
        equal:
          - "namespace"
          - "alertname"
      - source_matchers:
          - "severity = warning"
        target_matchers:
          - "severity = info"
        equal:
          - "namespace"
          - "alertname"
      - source_matchers:
          - "alertname = InfoInhibitor"
        target_matchers:
          - "severity = info"
        equal:
          - "namespace"
    route:
      group_by: ["..."]
      group_wait: 30s
      group_interval: 5m
      repeat_interval: 12h
      receiver: pager_test
      routes:
        - receiver: "null"
          matchers:
            - alertname =~ "InfoInhibitor"
        - receiver: dms
          group_wait: 10s
          group_interval: 30s
          repeat_interval: 1m
          matchers:
            - alertname =~ "Watchdog"
        - receiver: dms
          group_wait: 10s
          group_interval: 30s
          repeat_interval: 1m
          matchers:
            - alertname =~ "AlertmanagerClusterFailedToSendAlerts"
            - integration = "webhook" #jiralert
        - receiver: auto-scaler-api
          continue: true
          matchers:
            - alertname =~ "PrometheusCPUThrottlingHigh|O11yPodMemoryFillingUp|KubePersistentVolumeFillingUp"
        - receiver: default/observability-alertmanager-config/jiralert-dev-warning
          continue: false
          matchers:
            - "severity =~ warning|info"
        - receiver: default/observability-alertmanager-config/jiralert-dev
          continue: false
    receivers:
      - name: "null"
      - name: pager_test
        pagerduty_configs:
          - send_resolved: true
            http_config:
              follow_redirects: true
            service_key: 66f7402e9daf4609d09b948ec3e966b2
            url: https://events.pagerduty.com/generic/2010-04-15/create_event.json
            client: '{{ template "pagerduty.default.client" . }}'
            client_url: '{{ template "pagerduty.default.clientURL" . }}'
            description: '{{ template "pagerduty.default.description" .}}'
            details:
              firing: '{{ template "pagerduty.default.instances" .Alerts.Firing }}'
              num_firing: "{{ .Alerts.Firing | len }}"
              num_resolved: "{{ .Alerts.Resolved | len }}"
              resolved: '{{ template "pagerduty.default.instances" .Alerts.Resolved }}'
      - name: "default/observability-alertmanager-config/jiralert-dev"
        webhook_configs:
          - url: "http://eelb-1570947222.us-west-1.elb.amazonaws.com:9097/alert"
            send_resolved: true
      - name: "default/observability-alertmanager-config/jiralert-dev-warning"
        webhook_configs:
          - url: "http://eelb-1570947222.us-west-1.elb.amazonaws.com:9097/alert"
            send_resolved: true
      - name: "auto-scaler-api"
        webhook_configs:
          - url: "http://k8s-observab-globalin-22824fa614-584301555.us-west-2.elb.amazonaws.com/api/v1/ops/events/alerts"
            send_resolved: true
      - name: dms
        webhook_configs:
          - url: http://localhost:8088/alert
  alertmanagerSpec:
    volumes:
      - name: aws-iam-token
        projected:
          sources:
            - serviceAccountToken:
                audience: sts.amazonaws.com
                expirationSeconds: 7200
                path: aws-iam-token
    containers:
      - name: deadmanswatch
        image: us-west1-docker.pkg.dev/gcp-autonomous-dev/observability-regional-server/deadmanswatch:sha-891e26c
        imagePullPolicy: IfNotPresent
        args:
          - watch
          - --port=8088
          - --listen-address=0.0.0.0
          - --region=us-west-2
          - --alert-source-label=o11y_k8s_id
          - --metric-dimensions="scope=self-monitoring"
          - --metric-dimensions="environment=dev"
        ports:
          - name: http
            containerPort: 8088
            protocol: TCP
        env:
          - name: AWS_REGION
            value: us-west-2
          - name: AWS_ROLE_ARN
            value: arn:aws:iam::552185537300:role/regional-server-role-edf68ba
          - name: AWS_STS_REGIONAL_ENDPOINTS
            value: regional
          - name: AWS_WEB_IDENTITY_TOKEN_FILE
            value: /var/run/secrets/token/aws-iam-token
        volumeMounts:
          - mountPath: /var/run/secrets/token
            name: aws-iam-token
        livenessProbe:
          httpGet:
            path: /ping
            port: http
        resources:
          requests:
            cpu: 100m
            memory: 50Mi
          limits:
            cpu: 100m
            memory: 50Mi

grafana:
  enabled: false

prometheusOperator:
  enabled: true

kubeControllerManager:
  enabled: false

kubeScheduler:
  enabled: false

coreDns:
  enabled: false

kubeDns:
  enabled: true

kubeProxy:
  enabled: false

kubeApiServer:
  enabled: false

defaultRules:
  rules:
    kubeProxy: false
  disabled:
    KubeAggregatedAPIDown: true
    KubeCPUOvercommit: true #we will overshell so disable this alert
    KubeMemoryOvercommit: true #we have Autoscaler so disable this alert
    CPUThrottlingHigh: true #cfs limits are not accurate

prometheus:
  enabled: true
  prometheusSpec:
    externalLabels:
      o11y_k8s_id: o11y-dev-regional-us-west1
      component: observability-regional-server
    retention: 120d
    ## EXPERIMENTAL: Number of shards to distribute targets onto.
    ## Number of replicas multiplied by shards is the total number of Pods created.
    ## Note that scaling down shards will not reshard data onto remaining instances, it must be manually moved.
    ## Increasing shards will not reshard data either but it will continue to be available from the same instances.
    ## To query globally use Thanos sidecar and Thanos querier or remote write data to a central location.
    ## Sharding is done on the content of the `__address__` target meta-label.
    ##
    shards: 1
    resources:
      requests:
        memory: 2Gi

    ## Prometheus StorageSpec for persistent data
    ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/user-guides/storage.md
    ##
    storageSpec:
      volumeClaimTemplate:
        spec:
          storageClassName: prometheus.ssd
          accessModes: ["ReadWriteOnce"]
          resources:
            requests:
              storage: 100Gi
    serviceMonitorSelectorNilUsesHelmValues: false
    ## AdditionalScrapeConfigs allows specifying additional Prometheus scrape configurations. Scrape configurations
    ## are appended to the configurations generated by the Prometheus Operator. Job configurations must have the form
    ## as specified in the official Prometheus documentation:
    ## https://prometheus.io/docs/prometheus/latest/configuration/configuration/#scrape_config. As scrape configs are
    ## appended, the user is responsible to make sure it is valid. Note that using this feature may expose the possibility
    ## to break upgrades of Prometheus. It is advised to review Prometheus release notes to ensure that no incompatible
    ## scrape configs are going to break Prometheus after the upgrade.
    ## AdditionalScrapeConfigs can be defined as a list or as a templated string.
    ##
    ## The scrape configuration example below will find master nodes, provided they have the name .*mst.*, relabel the
    ## port to 2379 and allow etcd scraping provided it is running on all Kubernetes master nodes
    ##
    additionalScrapeConfigs: []
    # - job_name: kube-etcd
    #   kubernetes_sd_configs:
    #     - role: node
    #   scheme: https
    #   tls_config:
    #     ca_file:   /etc/prometheus/secrets/etcd-client-cert/etcd-ca
    #     cert_file: /etc/prometheus/secrets/etcd-client-cert/etcd-client
    #     key_file:  /etc/prometheus/secrets/etcd-client-cert/etcd-client-key
    #   relabel_configs:
    #   - action: labelmap
    #     regex: __meta_kubernetes_node_label_(.+)
    #   - source_labels: [__address__]
    #     action: replace
    #     targetLabel: __address__
    #     regex: ([^:;]+):(\d+)
    #     replacement: ${1}:2379
    #   - source_labels: [__meta_kubernetes_node_name]
    #     action: keep
    #     regex: .*mst.*
    #   - source_labels: [__meta_kubernetes_node_name]
    #     action: replace
    #     targetLabel: node
    #     regex: (.*)
    #     replacement: ${1}
    #   metric_relabel_configs:
    #   - regex: (kubernetes_io_hostname|failure_domain_beta_kubernetes_io_region|beta_kubernetes_io_os|beta_kubernetes_io_arch|beta_kubernetes_io_instance_type|failure_domain_beta_kubernetes_io_zone)
    #     action: labeldrop
    #
    ## If scrape config contains a repetitive section, you may want to use a template.
    ## In the following example, you can see how to define `gce_sd_configs` for multiple zones
    # additionalScrapeConfigs: |
    #  - job_name: "node-exporter"
    #    gce_sd_configs:
    #    {{range $zone := .Values.gcp_zones}}
    #    - project: "project1"
    #      zone: "{{$zone}}"
    #      port: 9100
    #    {{end}}
    #    relabel_configs:
    #    ...

    ## If additional scrape configurations are already deployed in a single secret file you can use this section.
    ## Expected values are the secret name and key
    ## Cannot be used with additionalScrapeConfigs
    additionalScrapeConfigsSecret:
      {}
      # enabled: false
      # name:
      # key:

    ## AdditionalAlertManagerConfigs allows for manual configuration of alertmanager jobs in the form as specified
    ## in the official Prometheus documentation https://prometheus.io/docs/prometheus/latest/configuration/configuration/#<alertmanager_config>.
    ## AlertManager configurations specified are appended to the configurations generated by the Prometheus Operator.
    ## As AlertManager configs are appended, the user is responsible to make sure it is valid. Note that using this
    ## feature may expose the possibility to break upgrades of Prometheus. It is advised to review Prometheus release
    ## notes to ensure that no incompatible AlertManager configs are going to break Prometheus after the upgrade.
    ##
    additionalAlertManagerConfigs: []
    # - consul_sd_configs:
    #   - server: consul.dev.test:8500
    #     scheme: http
    #     datacenter: dev
    #     tag_separator: ','
    #     services:
    #       - metrics-prometheus-alertmanager

    ## If additional alertmanager configurations are already deployed in a single secret, or you want to manage
    ## them separately from the helm deployment, you can use this section.
    ## Expected values are the secret name and key
    ## Cannot be used with additionalAlertManagerConfigs
    additionalAlertManagerConfigsSecret:
      {}
      # name:
      # key:

    ## AdditionalAlertRelabelConfigs allows specifying Prometheus alert relabel configurations. Alert relabel configurations specified are appended
    ## to the configurations generated by the Prometheus Operator. Alert relabel configurations specified must have the form as specified in the
    ## official Prometheus documentation: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#alert_relabel_configs.
    ## As alert relabel configs are appended, the user is responsible to make sure it is valid. Note that using this feature may expose the
    ## possibility to break upgrades of Prometheus. It is advised to review Prometheus release notes to ensure that no incompatible alert relabel
    ## configs are going to break Prometheus after the upgrade.
    ##
    additionalAlertRelabelConfigs: []
    # - separator: ;
    #   regex: prometheus_replica
    #   replacement: $1
    #   action: labeldrop

    ## If additional alert relabel configurations are already deployed in a single secret, or you want to manage
    ## them separately from the helm deployment, you can use this section.
    ## Expected values are the secret name and key
    ## Cannot be used with additionalAlertRelabelConfigs
    additionalAlertRelabelConfigsSecret:
      {}
      # name:
      # key: