alertmanager: enabled: true ingress: enabled: true ingressClassName: nginx-read annotations: nginx.ingress.kubernetes.io/rewrite-target: /$2 nginx.ingress.kubernetes.io/use-regex: "true" paths: - "/internal/alerts/self-monitoring(/|$)(.*)" serviceAccount: create: true annotations: eks.amazonaws.com/role-arn: arn:aws:iam::729581434105:role/tidbcloud-o11y-control-plane-role config: global: resolve_timeout: 5m externalAlertmanagerConfig: enabled: true spec: inhibitRules: - sourceMatch: - name: severity value: critical matchType: '=' targetMatch: - name: severity value: warning|major|info matchType: '=~' equal: - namespace - alertname - sourceMatch: - name: severity value: major matchType: '=' targetMatch: - name: severity value: warning|info matchType: '=~' equal: - namespace - alertname - sourceMatch: - name: alertname value: VMClusterTooHighQueryLoad matchType: '=' targetMatch: - name: alertname value: AlertingRulesError|RecordingRulesError matchType: '=~' equal: - o11y_k8s_id route: groupBy: ['...'] groupWait: 30s groupInterval: 5m repeatInterval: 12h receiver: 'null' routes: - receiver: 'null' matchers: - name: alertname value: InfoInhibitor matchType: '=~' - receiver: dms groupWait: 10s groupInterval: 30s repeatInterval: 1m matchers: - name: alertname value: Watchdog matchType: '=~' - receiver: dms groupWait: 10s groupInterval: 30s repeatInterval: 1m matchers: - name: alertname value: AlertmanagerClusterFailedToSendAlerts matchType: '=~' - name: integration value: webhook matchType: '=' - receiver: pingcap continue: false routes: - receiver: pingcap-warning matchers: - name: severity value: warning|info matchType: '=~' - receiver: pingcap-major matchers: - name: severity value: major matchType: '=~' receivers: - name: 'null' - name: dms webhookConfigs: - url: http://localhost:8088/alert - name: pingcap webhookConfigs: - httpConfig: basicAuth: username: name: clinic-alert-proxy-basic-auth key: username password: name: clinic-alert-proxy-basic-auth key: password sendResolved: true url: http://vpce-0b58b7e08f7e85349-sb0zc0xk.vpce-svc-02085f4a5969e0430.us-west-2.vpce.amazonaws.com/api/byoc/v1/alert - name: pingcap-warning webhookConfigs: - httpConfig: basicAuth: username: name: clinic-alert-proxy-basic-auth key: username password: name: clinic-alert-proxy-basic-auth key: password sendResolved: true url: http://vpce-0b58b7e08f7e85349-sb0zc0xk.vpce-svc-02085f4a5969e0430.us-west-2.vpce.amazonaws.com/api/byoc/v1/alert - name: pingcap-major webhookConfigs: - httpConfig: basicAuth: username: name: clinic-alert-proxy-basic-auth key: username password: name: clinic-alert-proxy-basic-auth key: password sendResolved: true url: http://vpce-0b58b7e08f7e85349-sb0zc0xk.vpce-svc-02085f4a5969e0430.us-west-2.vpce.amazonaws.com/api/byoc/v1/alert alertmanagerSpec: alertmanagerConfiguration: name: observability-alertmanager-config image: repository: 729581434105.dkr.ecr.us-east-1.amazonaws.com/third-party/prometheus/alertmanager tag: "v0.28.1" nodeSelector: node_type: worker containers: - name: deadmanswatch image: "729581434105.dkr.ecr.us-east-1.amazonaws.com/deadmanswatch:v1.0.1-e7bfbe5" imagePullPolicy: IfNotPresent args: - watch - --port=8088 - --listen-address=0.0.0.0 - --region=us-west-2 - --alert-source-label=o11y_k8s_id - --metric-dimensions="scope=self-monitoring" - --metric-dimensions="environment=prod" ports: - name: http containerPort: 8088 protocol: TCP livenessProbe: httpGet: path: /ping port: http resources: requests: cpu: 100m memory: 50Mi limits: cpu: 100m memory: 50Mi grafana: enabled: false prometheusOperator: enabled: true image: repository: 729581434105.dkr.ecr.us-east-1.amazonaws.com/third-party/prometheus-operator/prometheus-operator tag: v0.57.0 tls: enabled: false admissionWebhooks: enabled: false nodeSelector: node_type: worker prometheusConfigReloader: # image to use for config and rule reloading image: repository: 729581434105.dkr.ecr.us-east-1.amazonaws.com/third-party/prometheus-operator/prometheus-config-reloader tag: v0.84.1 kubeControllerManager: enabled: false kubeScheduler: enabled: false kubeProxy: enabled: false kubeApiServer: enabled: false defaultRules: disabled: KubeAggregatedAPIDown: true KubeCPUOvercommit: true #we have Autoscaler so disable this alert KubeMemoryOvercommit: true #we have Autoscaler so disable this alert CPUThrottlingHigh: true #cfs limits are not accurate prometheus: enabled: true prometheusSpec: nodeSelector: node_type: worker image: repository: 729581434105.dkr.ecr.us-east-1.amazonaws.com/third-party/prometheus/prometheus tag: v2.53.5 externalLabels: o11y_k8s_id: o11y-prod-use1-byoc-jpg-test cloudProvider: aws component: observability-regional-server tenant: "1372813089194211283" retention: 1y externalUrl: https://www.ds.use1-byoc-jpg-test.aws.tidbkit.com/prom/self-monitoring/ ## EXPERIMENTAL: Number of shards to distribute targets onto. ## Number of replicas multiplied by shards is the total number of Pods created. ## Note that scaling down shards will not reshard data onto remaining instances, it must be manually moved. ## Increasing shards will not reshard data either but it will continue to be available from the same instances. ## To query globally use Thanos sidecar and Thanos querier or remote write data to a central location. ## Sharding is done on the content of the `__address__` target meta-label. ## shards: 1 resources: requests: memory: 2Gi ## Prometheus StorageSpec for persistent data ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/user-guides/storage.md ## storageSpec: volumeClaimTemplate: spec: storageClassName: prometheus.gp3 accessModes: ["ReadWriteOnce"] resources: requests: storage: 100Gi serviceMonitorSelectorNilUsesHelmValues: false ## AdditionalScrapeConfigs allows specifying additional Prometheus scrape configurations. Scrape configurations ## are appended to the configurations generated by the Prometheus Operator. Job configurations must have the form ## as specified in the official Prometheus documentation: ## https://prometheus.io/docs/prometheus/latest/configuration/configuration/#scrape_config. As scrape configs are ## appended, the user is responsible to make sure it is valid. Note that using this feature may expose the possibility ## to break upgrades of Prometheus. It is advised to review Prometheus release notes to ensure that no incompatible ## scrape configs are going to break Prometheus after the upgrade. ## AdditionalScrapeConfigs can be defined as a list or as a templated string. ## ## The scrape configuration example below will find master nodes, provided they have the name .*mst.*, relabel the ## port to 2379 and allow etcd scraping provided it is running on all Kubernetes master nodes ## additionalScrapeConfigs: [] # - job_name: kube-etcd # kubernetes_sd_configs: # - role: node # scheme: https # tls_config: # ca_file: /etc/prometheus/secrets/etcd-client-cert/etcd-ca # cert_file: /etc/prometheus/secrets/etcd-client-cert/etcd-client # key_file: /etc/prometheus/secrets/etcd-client-cert/etcd-client-key # relabel_configs: # - action: labelmap # regex: __meta_kubernetes_node_label_(.+) # - source_labels: [__address__] # action: replace # targetLabel: __address__ # regex: ([^:;]+):(\d+) # replacement: ${1}:2379 # - source_labels: [__meta_kubernetes_node_name] # action: keep # regex: .*mst.* # - source_labels: [__meta_kubernetes_node_name] # action: replace # targetLabel: node # regex: (.*) # replacement: ${1} # metric_relabel_configs: # - regex: (kubernetes_io_hostname|failure_domain_beta_kubernetes_io_region|beta_kubernetes_io_os|beta_kubernetes_io_arch|beta_kubernetes_io_instance_type|failure_domain_beta_kubernetes_io_zone) # action: labeldrop # ## If scrape config contains a repetitive section, you may want to use a template. ## In the following example, you can see how to define `gce_sd_configs` for multiple zones # additionalScrapeConfigs: | # - job_name: "node-exporter" # gce_sd_configs: # {{range $zone := .Values.gcp_zones}} # - project: "project1" # zone: "{{$zone}}" # port: 9100 # {{end}} # relabel_configs: # ... ## If additional scrape configurations are already deployed in a single secret file you can use this section. ## Expected values are the secret name and key ## Cannot be used with additionalScrapeConfigs additionalScrapeConfigsSecret: {} # enabled: false # name: # key: ## AdditionalAlertManagerConfigs allows for manual configuration of alertmanager jobs in the form as specified ## in the official Prometheus documentation https://prometheus.io/docs/prometheus/latest/configuration/configuration/#. ## AlertManager configurations specified are appended to the configurations generated by the Prometheus Operator. ## As AlertManager configs are appended, the user is responsible to make sure it is valid. Note that using this ## feature may expose the possibility to break upgrades of Prometheus. It is advised to review Prometheus release ## notes to ensure that no incompatible AlertManager configs are going to break Prometheus after the upgrade. ## additionalAlertManagerConfigs: [] # - consul_sd_configs: # - server: consul.dev.test:8500 # scheme: http # datacenter: dev # tag_separator: ',' # services: # - metrics-prometheus-alertmanager ## If additional alertmanager configurations are already deployed in a single secret, or you want to manage ## them separately from the helm deployment, you can use this section. ## Expected values are the secret name and key ## Cannot be used with additionalAlertManagerConfigs additionalAlertManagerConfigsSecret: {} # name: # key: ## AdditionalAlertRelabelConfigs allows specifying Prometheus alert relabel configurations. Alert relabel configurations specified are appended ## to the configurations generated by the Prometheus Operator. Alert relabel configurations specified must have the form as specified in the ## official Prometheus documentation: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#alert_relabel_configs. ## As alert relabel configs are appended, the user is responsible to make sure it is valid. Note that using this feature may expose the ## possibility to break upgrades of Prometheus. It is advised to review Prometheus release notes to ensure that no incompatible alert relabel ## configs are going to break Prometheus after the upgrade. ## additionalAlertRelabelConfigs: [] # - separator: ; # regex: prometheus_replica # replacement: $1 # action: labeldrop ## If additional alert relabel configurations are already deployed in a single secret, or you want to manage ## them separately from the helm deployment, you can use this section. ## Expected values are the secret name and key ## Cannot be used with additionalAlertRelabelConfigs additionalAlertRelabelConfigsSecret: {} # name: # key: ingress: enabled: true ingressClassName: nginx-read annotations: nginx.ingress.kubernetes.io/rewrite-target: /$2 nginx.ingress.kubernetes.io/use-regex: "true" paths: - /prom/self-monitoring(/|$)(.*) prometheus-node-exporter: image: repository: 729581434105.dkr.ecr.us-east-1.amazonaws.com/third-party/prometheus/node-exporter tag: v1.9.1 extraArgs: - --collector.filesystem.mount-points-exclude=^/(dev|proc|sys|var/lib/docker/.+|var/lib/kubelet/.+)($|/) - --collector.filesystem.fs-types-exclude=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs|erofs)$ kube-state-metrics: image: repository: 729581434105.dkr.ecr.us-east-1.amazonaws.com/third-party/kube-state-metrics/kube-state-metrics tag: v2.16.0 metricLabelsAllowlist: - nodes=[*]