# helm upgrade --kubeconfig ~/.kube/o11y-eks --kube-context arn:aws:eks:us-west-2:557537366020:cluster/o11y-prod-global-us-west-2 -n self-monitoring -i -f ./kube-prom-stack-global-prod.yaml self-monitoring ../../../charts/kube-prometheus-stack alertmanager: enabled: true ingress: enabled: true ingressClassName: nginx annotations: nginx.ingress.kubernetes.io/rewrite-target: /$2 nginx.ingress.kubernetes.io/use-regex: "true" paths: - "/self-monitoring/alertmanager(/|$)(.*)" serviceAccount: create: true annotations: eks.amazonaws.com/role-arn: arn:aws:iam::557537366020:role/autonomous-o11y-self-monitoring-eks-prod config: global: resolve_timeout: 5m inhibit_rules: - source_matchers: - 'severity = critical' target_matchers: - 'severity =~ warning|major|info' equal: - 'namespace' - 'alertname' - source_matchers: - 'severity = major' target_matchers: - 'severity = warning|info' equal: - 'namespace' - 'alertname' - source_matchers: - 'alertname = VMClusterTooHighQueryLoad' target_matchers: - 'alertname =~ AlertingRulesError|RecordingRulesError' # avoids alert Storm from vmalerts when vmcluster is on high load equal: - 'o11y_k8s_id' route: group_by: ['...'] group_wait: 30s group_interval: 5m repeat_interval: 12h receiver: default/observability-alertmanager-config/jiralert-prod routes: - receiver: dms group_wait: 10s group_interval: 30s repeat_interval: 1m group_by: ['alertname', 'severity'] matchers: - alertname = "Watchdog" - receiver: dms group_wait: 10s group_interval: 30s repeat_interval: 1m matchers: - alertname =~ "AlertmanagerClusterFailedToSendAlerts" - integration = "webhook" #jiralert - receiver: 'null' matchers: - alertname =~ "InfoInhibitor" - receiver: default/observability-alertmanager-config/jiralert-prod-warning continue: false matchers: - 'severity =~ warning|info' - receiver: default/observability-alertmanager-config/jiralert-prod-major continue: false matchers: - 'severity =~ major' - receiver: default/observability-alertmanager-config/jiralert-prod continue: false receivers: - name: 'null' - name: 'default/observability-alertmanager-config/jiralert-prod' webhook_configs: - url: 'https://jiralert-eks-prod.pingcap.net/alert' send_resolved: true - name: 'default/observability-alertmanager-config/jiralert-prod-warning' webhook_configs: - url: 'https://jiralert-eks-prod.pingcap.net/alert' send_resolved: true - name: 'default/observability-alertmanager-config/jiralert-prod-major' webhook_configs: - url: 'https://jiralert-eks-prod.pingcap.net/alert' send_resolved: true - name: dms webhook_configs: - url: http://localhost:8088/alert alertmanagerSpec: containers: - name: deadmanswatch image: 557537366020.dkr.ecr.us-west-2.amazonaws.com/deadmanswatch:sha-891e26c imagePullPolicy: IfNotPresent args: - watch - --port=8088 - --listen-address=0.0.0.0 - --region=us-west-2 - --alert-source-label=o11y_k8s_id - --metric-dimensions="scope=self-monitoring" - --metric-dimensions="environment=prod" ports: - name: http containerPort: 8088 protocol: TCP livenessProbe: httpGet: path: /ping port: http resources: requests: cpu: 100m memory: 50Mi limits: cpu: 100m memory: 50Mi grafana: enabled: true ## Timezone for the default dashboards ## Other options are: browser or a specific timezone, i.e. Europe/Luxembourg ## defaultDashboardsTimezone: browser admin: existingSecret: self-monitoring-grafana-admin-credentials userKey: admin-user passwordKey: admin-password extraObjects: - apiVersion: external-secrets.io/v1 kind: ExternalSecret metadata: name: self-monitoring-grafana-admin-credentials namespace: self-monitoring spec: refreshInterval: 1h secretStoreRef: name: aws-secret-store kind: ClusterSecretStore target: name: self-monitoring-grafana-admin-credentials creationPolicy: Owner data: - secretKey: admin-user remoteRef: key: prod/self-monitoring-grafana property: self-monitoring-grafana-admin-user - secretKey: admin-password remoteRef: key: prod/self-monitoring-grafana property: self-monitoring-grafana-admin-password useStatefulSet: true service: type: NodePort persistence: enabled: true storageClassName: gp2 grafana.ini: server: domain: self-monitoring-grafana.observability.tidbcloud.com root_url: https://self-monitoring-grafana.observability.tidbcloud.com/ auth.generic_oauth: enabled: true allow_sign_up: true name: PingCAP Auth client_id: _ client_secret: _ scopes: openid profile email auth_url: https://tidb-soc2.us.auth0.com/authorize token_url: https://tidb-soc2.us.auth0.com/oauth/token api_url: https://tidb-soc2.us.auth0.com/userinfo use_pkce: true users: viewers_can_edit: true auto_assign_org_role: Editor editors_can_admin: true deleteDatasources: - name: Loki-regional-us-east-2 orgId: 1 - name: Prometheus-regional-us-east-2 orgId: 1 - name: Loki-regional-ap-southeast-5 orgId: 1 - name: Prometheus-regional-ap-southeast-5 orgId: 1 - name: Loki-regional-ap-southeast-5 orgId: 1 - name: Prometheus-regional-ap-southeast-5 orgId: 1 - name: Loki-regional-ap-northeast-3 orgId: 1 - name: Prometheus-regional-ap-northeast-3 orgId: 1 - name: Loki-regional-europe-west3 orgId: 1 - name: Prometheus-regional-europe-west3 orgId: 1 - name: Loki-regional-af-south-1 orgId: 1 - name: Prometheus-regional-af-south-1 orgId: 1 - name: Loki-regional-ap-southeast-2 orgId: 1 - name: Prometheus-regional-ap-southeast-2 orgId: 1 - name: Loki-regional-asia-east2 orgId: 1 - name: Prometheus-regional-asia-east2 orgId: 1 - name: Loki-regional-us-west-1-premium orgId: 1 - name: Prometheus-regional-us-west-1-premium orgId: 1 additionalDataSources: - name: Loki access: proxy basicAuth: false orgId: 1 type: loki url: http://loki.self-monitoring.svc.cluster.local:3100 version: 1 withCredentials: false - name: Prometheus-regional-ap-northeast-1 access: proxy basicAuth: false jsonData: httpMethod: POST timeInterval: 30s orgId: 1 type: prometheus url: https://www.ds.ap-northeast-1.aws.observability.tidbcloud.com/prom/self-monitoring version: 1 withCredentials: false - name: Loki-regional-ap-northeast-1 access: proxy basicAuth: false orgId: 1 type: loki url: https://www.ds.ap-northeast-1.aws.observability.tidbcloud.com/loki/self-monitoring version: 1 withCredentials: false - name: Prometheus-regional-ap-northeast-2 access: proxy basicAuth: false jsonData: httpMethod: POST timeInterval: 30s orgId: 1 type: prometheus url: https://www.ds.ap-northeast-2.aws.observability.tidbcloud.com/prom/self-monitoring version: 1 withCredentials: false - name: Loki-regional-ap-northeast-2 access: proxy basicAuth: false orgId: 1 type: loki url: https://www.ds.ap-northeast-2.aws.observability.tidbcloud.com/loki/self-monitoring version: 1 withCredentials: false - name: Prometheus-regional-ap-southeast-1 access: proxy basicAuth: false jsonData: httpMethod: POST timeInterval: 30s orgId: 1 type: prometheus url: https://www.ds.ap-southeast-1.aws.observability.tidbcloud.com/prom/self-monitoring version: 1 withCredentials: false - name: Loki-regional-ap-southeast-1 access: proxy basicAuth: false orgId: 1 type: loki url: https://www.ds.ap-southeast-1.aws.observability.tidbcloud.com/loki/self-monitoring version: 1 withCredentials: false - name: Prometheus-regional-eu-central-1 access: proxy basicAuth: false jsonData: httpMethod: POST timeInterval: 30s orgId: 1 type: prometheus url: https://www.ds.eu-central-1.aws.observability.tidbcloud.com/prom/self-monitoring version: 1 withCredentials: false - name: Loki-regional-eu-central-1 access: proxy basicAuth: false orgId: 1 type: loki url: https://www.ds.eu-central-1.aws.observability.tidbcloud.com/loki/self-monitoring version: 1 withCredentials: false - name: Prometheus-regional-us-east-1 access: proxy basicAuth: false jsonData: httpMethod: POST timeInterval: 30s orgId: 1 type: prometheus url: https://www.ds.us-east-1.aws.observability.tidbcloud.com/prom/self-monitoring version: 1 withCredentials: false - name: Loki-regional-us-east-1 access: proxy basicAuth: false orgId: 1 type: loki url: https://www.ds.us-east-1.aws.observability.tidbcloud.com/loki/self-monitoring version: 1 withCredentials: false - name: Prometheus-regional-us-west-2 access: proxy basicAuth: false jsonData: httpMethod: POST timeInterval: 30s orgId: 1 type: prometheus url: https://www.ds.us-west-2.aws.observability.tidbcloud.com/prom/self-monitoring version: 1 withCredentials: false - name: Loki-regional-us-west-2 access: proxy basicAuth: false orgId: 1 type: loki url: https://www.ds.us-west-2.aws.observability.tidbcloud.com/loki/self-monitoring version: 1 withCredentials: false - name: Prometheus-regional-us-west1 access: proxy basicAuth: false jsonData: httpMethod: POST timeInterval: 30s orgId: 1 type: prometheus url: https://www.ds.us-west1.gcp.observability.tidbcloud.com/prom/self-monitoring version: 1 withCredentials: false - name: Loki-regional-us-west1 access: proxy basicAuth: false orgId: 1 type: loki url: https://www.ds.us-west1.gcp.observability.tidbcloud.com/loki/self-monitoring version: 1 withCredentials: false - name: Prometheus-regional-us-central1 access: proxy basicAuth: false jsonData: httpMethod: POST timeInterval: 30s orgId: 1 type: prometheus url: https://www.ds.us-central1.gcp.observability.tidbcloud.com/prom/self-monitoring version: 1 withCredentials: false - name: Loki-regional-us-central1 access: proxy basicAuth: false orgId: 1 type: loki url: https://www.ds.us-central1.gcp.observability.tidbcloud.com/loki/self-monitoring version: 1 withCredentials: false - name: Prometheus-regional-asia-southeast1 access: proxy basicAuth: false jsonData: httpMethod: POST timeInterval: 30s orgId: 1 type: prometheus url: https://www.ds.asia-southeast1.gcp.observability.tidbcloud.com/prom/self-monitoring version: 1 withCredentials: false - name: Loki-regional-asia-southeast1 access: proxy basicAuth: false orgId: 1 type: loki url: https://www.ds.asia-southeast1.gcp.observability.tidbcloud.com/loki/self-monitoring version: 1 withCredentials: false - name: Prometheus-regional-asia-northeast1 access: proxy basicAuth: false jsonData: httpMethod: POST timeInterval: 30s orgId: 1 type: prometheus url: https://www.ds.asia-northeast1.gcp.observability.tidbcloud.com/prom/self-monitoring version: 1 withCredentials: false - name: Loki-regional-asia-northeast1 access: proxy basicAuth: false orgId: 1 type: loki url: https://www.ds.asia-northeast1.gcp.observability.tidbcloud.com/loki/self-monitoring version: 1 withCredentials: false - name: Prometheus-regional-asia-east1 access: proxy basicAuth: false jsonData: httpMethod: POST timeInterval: 30s orgId: 1 type: prometheus url: https://www.ds.asia-east1.gcp.observability.tidbcloud.com/prom/self-monitoring version: 1 withCredentials: false - name: Loki-regional-asia-east1 access: proxy basicAuth: false orgId: 1 type: loki url: https://www.ds.asia-east1.gcp.observability.tidbcloud.com/loki/self-monitoring version: 1 withCredentials: false - name: Prometheus-regional-us-east4 access: proxy basicAuth: false jsonData: httpMethod: POST timeInterval: 30s orgId: 1 type: prometheus url: https://www.ds.us-east4.gcp.observability.tidbcloud.com/prom/self-monitoring version: 1 withCredentials: false - name: Loki-regional-us-east4 access: proxy basicAuth: false orgId: 1 type: loki url: https://www.ds.us-east4.gcp.observability.tidbcloud.com/loki/self-monitoring version: 1 withCredentials: false - name: Prometheus-regional-ap-southeast-3 access: proxy basicAuth: false jsonData: httpMethod: POST timeInterval: 30s orgId: 1 type: prometheus url: https://www.ds.ap-southeast-3.aws.observability.tidbcloud.com/prom/self-monitoring version: 1 withCredentials: false - name: Loki-regional-ap-southeast-3 access: proxy basicAuth: false orgId: 1 type: loki url: https://www.ds.ap-southeast-3.aws.observability.tidbcloud.com/loki/self-monitoring version: 1 withCredentials: false # add seed & serverless datasource to assist with capacity planning in certain regions. - name: Prometheus-regional-ap-northeast-1-seed access: proxy jsonData: httpMethod: POST timeInterval: 30s orgId: 1 type: prometheus url: https://www.ds.ap-northeast-1.aws.observability.tidbcloud.com/internal/metrics/d5d1a915-1d37-22a7-82b8-8cb67cc57820 version: 1 - name: Prometheus-regional-ap-northeast-1-serverless access: proxy jsonData: httpMethod: POST timeInterval: 30s orgId: 1 type: prometheus url: https://www.ds.ap-northeast-1.aws.observability.tidbcloud.com/internal/metrics/541521b7-b5eb-47c7-a015-b1f4f97c2744 version: 1 - name: Prometheus-regional-ap-southeast-1-seed access: proxy jsonData: httpMethod: POST timeInterval: 30s orgId: 1 type: prometheus url: https://www.ds.ap-southeast-1.aws.observability.tidbcloud.com/internal/metrics/d5d1a915-1d37-22a7-82b8-8cb67cc57820 version: 1 - name: Prometheus-regional-ap-southeast-1-serverless access: proxy jsonData: httpMethod: POST timeInterval: 30s orgId: 1 type: prometheus url: https://www.ds.ap-southeast-1.aws.observability.tidbcloud.com/internal/metrics/d39797d5-f75d-4871-84f8-d7b4f87b5d88 version: 1 - name: Prometheus-regional-us-east-1-seed access: proxy jsonData: httpMethod: POST timeInterval: 30s orgId: 1 type: prometheus url: https://www.ds.us-east-1.aws.observability.tidbcloud.com/internal/metrics/d5d1a915-1d37-22a7-82b8-8cb67cc57820 version: 1 - name: Prometheus-regional-us-east-1-serverless access: proxy jsonData: httpMethod: POST timeInterval: 30s orgId: 1 type: prometheus url: https://www.ds.us-east-1-serverless.aws.observability.tidbcloud.com/prom/self-monitoring version: 1 - name: Prometheus-regional-us-east-1-multi access: proxy jsonData: httpMethod: POST timeInterval: 30s orgId: 1 type: prometheus url: https://www.ds.us-east-1.aws.observability.tidbcloud.com/internal/v2/metrics/multitenant - name: Prometheus-regional-us-west-2-seed access: proxy jsonData: httpMethod: POST timeInterval: 30s orgId: 1 type: prometheus url: https://www.ds.us-west-2.aws.observability.tidbcloud.com/internal/metrics/d5d1a915-1d37-22a7-82b8-8cb67cc57820 version: 1 - name: Prometheus-regional-us-west-2-serverless access: proxy jsonData: httpMethod: POST timeInterval: 30s orgId: 1 type: prometheus url: https://www.ds.us-west-2.aws.observability.tidbcloud.com/internal/metrics/7055cf36-ea60-4c77-926d-60d3178162cf version: 1 - name: Prometheus-regional-eu-central-1-seed access: proxy jsonData: httpMethod: POST timeInterval: 30s orgId: 1 type: prometheus url: https://www.ds.eu-central-1.aws.observability.tidbcloud.com/internal/metrics/d5d1a915-1d37-22a7-82b8-8cb67cc57820 version: 1 - name: Prometheus-regional-eu-central-1-serverless access: proxy jsonData: httpMethod: POST timeInterval: 30s orgId: 1 type: prometheus url: https://www.ds.eu-central-1.aws.observability.tidbcloud.com/internal/metrics/934f05ea-66b2-4040-9202-d10f35f0561b version: 1 - name: Prometheus-regional-ap-southeast-3-seed access: proxy jsonData: httpMethod: POST timeInterval: 30s orgId: 1 type: prometheus url: https://www.ds.ap-southeast-3.aws.observability.tidbcloud.com/internal/metrics/d5d1a915-1d37-22a7-82b8-8cb67cc57820 version: 1 - name: Prometheus-regional-ap-east-1 access: proxy jsonData: httpMethod: POST timeInterval: 30s orgId: 1 type: prometheus url: https://www.ds.ap-east-1.aws.observability.tidbcloud.com/prom/self-monitoring - name: Loki-regional-ap-east-1 access: proxy orgId: 1 type: loki url: https://www.ds.ap-east-1.aws.observability.tidbcloud.com/loki/self-monitoring - name: Prometheus-regional-japaneast access: proxy jsonData: httpMethod: POST timeInterval: 30s orgId: 1 type: prometheus url: https://www.ds.japaneast.azure.observability.tidbcloud.com/prom/self-monitoring - name: Loki-regional-japaneast access: proxy orgId: 1 type: loki url: https://www.ds.japaneast.azure.observability.tidbcloud.com/loki/self-monitoring - name: Prometheus-regional-sa-east-1 access: proxy jsonData: httpMethod: POST timeInterval: 30s orgId: 1 type: prometheus url: https://www.ds.sa-east-1.aws.observability.tidbcloud.com/prom/self-monitoring - name: Loki-regional-sa-east-1 access: proxy orgId: 1 type: loki url: https://www.ds.sa-east-1.aws.observability.tidbcloud.com/loki/self-monitoring - name: Prometheus-regional-eu-west-1 access: proxy jsonData: httpMethod: POST timeInterval: 30s orgId: 1 type: prometheus url: https://www.ds.eu-west-1.aws.observability.tidbcloud.com/prom/self-monitoring - name: Loki-regional-eu-west-1 access: proxy orgId: 1 type: loki url: https://www.ds.eu-west-1.aws.observability.tidbcloud.com/loki/self-monitoring - name: Prometheus-regional-asia-northeast2 access: proxy jsonData: httpMethod: POST timeInterval: 30s orgId: 1 type: prometheus url: https://www.ds.asia-northeast2.gcp.observability.tidbcloud.com/prom/self-monitoring - name: Loki-regional-asia-northeast2 access: proxy orgId: 1 type: loki url: https://www.ds.asia-northeast2.gcp.observability.tidbcloud.com/loki/self-monitoring - name: Prometheus-regional-alicloud-ap-southeast-1 access: proxy jsonData: httpMethod: POST timeInterval: 30s orgId: 1 type: prometheus url: https://www.ds.ap-southeast-1.alicloud.observability.tidbcloud.com/prom/self-monitoring - name: Loki-regional-alicloud-ap-southeast-1 access: proxy orgId: 1 type: loki url: https://www.ds.ap-southeast-1.alicloud.observability.tidbcloud.com/loki/self-monitoring - name: Prometheus-regional-alicloud-ap-northeast-1 access: proxy jsonData: httpMethod: POST timeInterval: 30s orgId: 1 type: prometheus url: https://www.ds.ap-northeast-1.alicloud.observability.tidbcloud.com/prom/self-monitoring - name: Loki-regional-alicloud-ap-northeast-1 access: proxy orgId: 1 type: loki url: https://www.ds.ap-northeast-1.alicloud.observability.tidbcloud.com/loki/self-monitoring - name: Prometheus-regional-alicloud-na-south-1 access: proxy jsonData: httpMethod: POST timeInterval: 30s orgId: 1 type: prometheus url: https://www.ds.na-south-1.alicloud.observability.tidbcloud.com/prom/self-monitoring - name: Loki-regional-alicloud-na-south-1 access: proxy orgId: 1 type: loki url: https://www.ds.na-south-1.alicloud.observability.tidbcloud.com/loki/self-monitoring - name: Prometheus-regional-alicloud-ap-southeast-5 access: proxy jsonData: httpMethod: POST timeInterval: 30s orgId: 1 type: prometheus url: https://www.ds.ap-southeast-5.alicloud.observability.tidbcloud.com/prom/self-monitoring - name: Loki-regional-alicloud-ap-southeast-5 access: proxy orgId: 1 type: loki url: https://www.ds.ap-southeast-5.alicloud.observability.tidbcloud.com/loki/self-monitoring - name: Prometheus-regional-alicloud-cn-hongkong access: proxy jsonData: httpMethod: POST timeInterval: 30s orgId: 1 type: prometheus url: https://www.ds.cn-hongkong.alicloud.observability.tidbcloud.com/prom/self-monitoring - name: Loki-regional-alicloud-cn-hongkong access: proxy orgId: 1 type: loki url: https://www.ds.cn-hongkong.alicloud.observability.tidbcloud.com/loki/self-monitoring - name: Prometheus-regional-ap-south-1 access: proxy jsonData: httpMethod: POST timeInterval: 30s orgId: 1 type: prometheus url: https://www.ds.ap-south-1.aws.observability.tidbcloud.com/prom/self-monitoring - name: Loki-regional-ap-south-1 access: proxy orgId: 1 type: loki url: https://www.ds.ap-south-1.aws.observability.tidbcloud.com/loki/self-monitoring - name: Prometheus-regional-ap-southeast-1-premium access: proxy jsonData: httpMethod: POST timeInterval: 30s orgId: 1 type: prometheus url: https://www.ds.ap-southeast-1-premium.aws.observability.tidbcloud.com/prom/self-monitoring - name: Loki-regional-ap-southeast-1-premium access: proxy orgId: 1 type: loki url: https://www.ds.ap-southeast-1-premium.aws.observability.tidbcloud.com/loki/self-monitoring - name: Prometheus-regional-ap-southeast-7 access: proxy jsonData: httpMethod: POST timeInterval: 30s orgId: 1 type: prometheus url: https://www.ds.ap-southeast-7.aws.observability.tidbcloud.com/prom/self-monitoring - name: Loki-regional-ap-southeast-7 access: proxy orgId: 1 type: loki url: https://www.ds.ap-southeast-7.aws.observability.tidbcloud.com/loki/self-monitoring - name: Prometheus-regional-us-west-2-premium access: proxy jsonData: httpMethod: POST timeInterval: 30s orgId: 1 type: prometheus url: https://www.ds.us-west-2-premium.aws.observability.tidbcloud.com/prom/self-monitoring - name: Loki-regional-us-west-2-premium access: proxy orgId: 1 type: loki url: https://www.ds.us-west-2-premium.aws.observability.tidbcloud.com/loki/self-monitoring - name: Prometheus-regional-ap-northeast-1-premium access: proxy jsonData: httpMethod: POST timeInterval: 30s orgId: 1 type: prometheus url: https://www.ds.ap-northeast-1-premium.aws.observability.tidbcloud.com/prom/self-monitoring - name: Loki-regional-ap-northeast-1-premium access: proxy orgId: 1 type: loki url: https://www.ds.ap-northeast-1-premium.aws.observability.tidbcloud.com/loki/self-monitoring - name: Prometheus-regional-ap-southeast-2-ng access: proxy jsonData: httpMethod: POST timeInterval: 30s orgId: 1 type: prometheus url: https://www.ds.ap-southeast-2-ng.aws.observability.tidbcloud.com/prom/self-monitoring - name: Loki-regional-ap-southeast-2-ng access: proxy orgId: 1 type: loki url: https://www.ds.ap-southeast-2-ng.aws.observability.tidbcloud.com/loki/self-monitoring - name: Prometheus-regional-us-west-2-staging access: proxy jsonData: httpMethod: POST timeInterval: 30s orgId: 1 type: prometheus url: https://www.ds.us-west-2-staging.aws.observability.tidbcloud.com/prom/self-monitoring - name: Loki-regional-us-west-2-staging access: proxy orgId: 1 type: loki url: https://www.ds.us-west-2-staging.aws.observability.tidbcloud.com/loki/self-monitoring - name: Prometheus-regional-us-east-1-premium access: proxy jsonData: httpMethod: POST timeInterval: 30s orgId: 1 type: prometheus url: https://www.ds.us-east-1-premium.aws.observability.tidbcloud.com/prom/self-monitoring - name: Loki-regional-us-east-1-premium access: proxy orgId: 1 type: loki url: https://www.ds.us-east-1-premium.aws.observability.tidbcloud.com/loki/self-monitoring - name: Prometheus-regional-us-east-1-byoc-poc access: proxy jsonData: httpMethod: POST timeInterval: 30s httpHeaderName1: X-Region-Id httpHeaderName2: X-Biz-Type httpHeaderName3: X-Org-Id secureJsonData: httpHeaderValue1: aws-us-east-1 httpHeaderValue2: byoc httpHeaderValue3: "1372813089209313726" orgId: 1 type: prometheus url: http://kong-kong-proxy.kong.svc.cluster.local/internal/prom/self-monitoring - name: Loki-regional-us-east-1-byoc-poc access: proxy jsonData: httpHeaderName1: X-Region-Id httpHeaderName2: X-Biz-Type httpHeaderName3: X-Org-Id secureJsonData: httpHeaderValue1: aws-us-east-1 httpHeaderValue2: byoc httpHeaderValue3: "1372813089209313726" orgId: 1 type: loki url: http://kong-kong-proxy.kong.svc.cluster.local/internal/loki/self-monitoring - name: Loki-regional-us-east-1-serverless access: proxy orgId: 1 type: loki url: https://www.ds.us-east-1-serverless.aws.observability.tidbcloud.com/loki/self-monitoring - name: Prometheus-regional-ap-east-1-premium access: proxy jsonData: httpMethod: POST timeInterval: 30s orgId: 1 type: prometheus url: https://www.ds.ap-east-1-premium.aws.observability.tidbcloud.com/prom/self-monitoring - name: Loki-regional-ap-east-1-premium access: proxy orgId: 1 type: loki url: https://www.ds.ap-east-1-premium.aws.observability.tidbcloud.com/loki/self-monitoring - name: Prometheus-regional-us-east-1-na-test access: proxy jsonData: httpMethod: POST timeInterval: 30s httpHeaderName1: X-Region-Id httpHeaderName2: X-Biz-Type httpHeaderName3: X-Org-Id secureJsonData: httpHeaderValue1: aws-us-east-1 httpHeaderValue2: byoc httpHeaderValue3: "1372813089208281461" orgId: 1 type: prometheus url: http://kong-kong-proxy.kong.svc.cluster.local/internal/prom/self-monitoring - name: Loki-regional-us-east-1-na-test access: proxy jsonData: httpHeaderName1: X-Region-Id httpHeaderName2: X-Biz-Type httpHeaderName3: X-Org-Id secureJsonData: httpHeaderValue1: aws-us-east-1 httpHeaderValue2: byoc httpHeaderValue3: "1372813089208281461" orgId: 1 type: loki url: http://kong-kong-proxy.kong.svc.cluster.local/internal/loki/self-monitoring - name: Prometheus-regional-mx-central-1 access: proxy jsonData: httpMethod: POST timeInterval: 30s orgId: 1 type: prometheus url: https://www.ds.mx-central-1.aws.observability.tidbcloud.com/prom/self-monitoring - name: Loki-regional-mx-central-1 access: proxy orgId: 1 type: loki url: https://www.ds.mx-central-1.aws.observability.tidbcloud.com/loki/self-monitoring - name: Prometheus-regional-us-west-2-na-test access: proxy jsonData: httpMethod: POST timeInterval: 30s httpHeaderName1: X-Region-Id httpHeaderName2: X-Biz-Type httpHeaderName3: X-Org-Id secureJsonData: httpHeaderValue1: aws-us-west-2 httpHeaderValue2: byoc httpHeaderValue3: "1372813089208281461" orgId: 1 type: prometheus url: http://kong-kong-proxy.kong.svc.cluster.local/internal/prom/self-monitoring - name: Loki-regional-us-west-2-na-test access: proxy jsonData: httpHeaderName1: X-Region-Id httpHeaderName2: X-Biz-Type httpHeaderName3: X-Org-Id secureJsonData: httpHeaderValue1: aws-us-west-2 httpHeaderValue2: byoc httpHeaderValue3: "1372813089208281461" orgId: 1 type: loki url: http://kong-kong-proxy.kong.svc.cluster.local/internal/loki/self-monitoring - name: Prometheus-regional-us-west-2-byoc-fnu access: proxy jsonData: httpMethod: POST timeInterval: 30s httpHeaderName1: X-Region-Id httpHeaderName2: X-Biz-Type httpHeaderName3: X-Org-Id secureJsonData: httpHeaderValue1: aws-us-west-2 httpHeaderValue2: byoc httpHeaderValue3: "1372813089192471291" orgId: 1 type: prometheus url: http://kong-kong-proxy.kong.svc.cluster.local/internal/prom/self-monitoring - name: Loki-regional-us-west-2-byoc-fnu access: proxy jsonData: httpHeaderName1: X-Region-Id httpHeaderName2: X-Biz-Type httpHeaderName3: X-Org-Id secureJsonData: httpHeaderValue1: aws-us-west-2 httpHeaderValue2: byoc httpHeaderValue3: "1372813089192471291" orgId: 1 type: loki url: http://kong-kong-proxy.kong.svc.cluster.local/internal/loki/self-monitoring - name: Prometheus-regional-byoc-usw2-mithra access: proxy jsonData: httpMethod: POST timeInterval: 30s httpHeaderName1: X-Region-Id httpHeaderName2: X-Biz-Type httpHeaderName3: X-Org-Id secureJsonData: httpHeaderValue1: aws-us-west-2 httpHeaderValue2: byoc httpHeaderValue3: "1372813089209301084" orgId: 1 type: prometheus url: http://kong-kong-proxy.kong.svc.cluster.local/internal/prom/self-monitoring - name: Loki-regional-byoc-usw2-mithra access: proxy jsonData: httpHeaderName1: X-Region-Id httpHeaderName2: X-Biz-Type httpHeaderName3: X-Org-Id secureJsonData: httpHeaderValue1: aws-us-west-2 httpHeaderValue2: byoc httpHeaderValue3: "1372813089209301084" orgId: 1 type: loki url: http://kong-kong-proxy.kong.svc.cluster.local/internal/loki/self-monitoring - name: Prometheus-regional-us-east-1-byoc-se access: proxy jsonData: httpMethod: POST timeInterval: 30s httpHeaderName1: X-Region-Id httpHeaderName2: X-Biz-Type httpHeaderName3: X-Org-Id secureJsonData: httpHeaderValue1: aws-us-east-1 httpHeaderValue2: byoc httpHeaderValue3: "1372813089200871284" orgId: 1 type: prometheus url: http://kong-kong-proxy.kong.svc.cluster.local/internal/prom/self-monitoring - name: Loki-regional-us-east-1-byoc-se access: proxy jsonData: httpHeaderName1: X-Region-Id httpHeaderName2: X-Biz-Type httpHeaderName3: X-Org-Id secureJsonData: httpHeaderValue1: aws-us-east-1 httpHeaderValue2: byoc httpHeaderValue3: "1372813089200871284" orgId: 1 type: loki url: http://kong-kong-proxy.kong.svc.cluster.local/internal/loki/self-monitoring - name: Prometheus-regional-us-west-2-byoc-se access: proxy jsonData: httpMethod: POST timeInterval: 30s httpHeaderName1: X-Region-Id httpHeaderName2: X-Biz-Type httpHeaderName3: X-Org-Id secureJsonData: httpHeaderValue1: aws-us-west-2 httpHeaderValue2: byoc httpHeaderValue3: "1372813089200871284" orgId: 1 type: prometheus url: http://kong-kong-proxy.kong.svc.cluster.local/internal/prom/self-monitoring - name: Loki-regional-us-west-2-byoc-se access: proxy jsonData: httpHeaderName1: X-Region-Id httpHeaderName2: X-Biz-Type httpHeaderName3: X-Org-Id secureJsonData: httpHeaderValue1: aws-us-west-2 httpHeaderValue2: byoc httpHeaderValue3: "1372813089200871284" orgId: 1 type: loki url: http://kong-kong-proxy.kong.svc.cluster.local/internal/loki/self-monitoring - name: Prometheus-regional-asia-south1 access: proxy jsonData: httpMethod: POST timeInterval: 30s orgId: 1 type: prometheus url: https://www.ds.asia-south1.gcp.observability.tidbcloud.com/prom/self-monitoring - name: Loki-regional-asia-south1 access: proxy orgId: 1 type: loki url: https://www.ds.asia-south1.gcp.observability.tidbcloud.com/loki/self-monitoring - name: Prometheus-regional-use1-byoc-jpg-test access: proxy jsonData: httpMethod: POST timeInterval: 30s httpHeaderName1: X-Region-Id httpHeaderName2: X-Biz-Type httpHeaderName3: X-Org-Id secureJsonData: httpHeaderValue1: aws-us-east-1 httpHeaderValue2: byoc httpHeaderValue3: "1372813089194211283" orgId: 1 type: prometheus url: http://kong-kong-proxy.kong.svc.cluster.local/internal/prom/self-monitoring - name: Loki-regional-use1-byoc-jpg-test access: proxy jsonData: httpHeaderName1: X-Region-Id httpHeaderName2: X-Biz-Type httpHeaderName3: X-Org-Id secureJsonData: httpHeaderValue1: aws-us-east-1 httpHeaderValue2: byoc httpHeaderValue3: "1372813089194211283" orgId: 1 type: loki url: http://kong-kong-proxy.kong.svc.cluster.local/internal/loki/self-monitoring prometheusOperator: enabled: false kubeControllerManager: enabled: false kubeScheduler: enabled: false kubeProxy: enabled: false kubeApiServer: enabled: false defaultRules: disabled: KubeAggregatedAPIDown: true KubeCPUOvercommit: true #we have Autoscaler so disable this alert KubeMemoryOvercommit: true #we have Autoscaler so disable this alert CPUThrottlingHigh: true #cfs limits are not accurate prometheus: enabled: true prometheusSpec: image: tag: v2.55.1 externalUrl: https://www.gs.us-west-2.aws.observability.tidbcloud.com/prom/self-monitoring/ externalLabels: o11y_k8s_id: o11y-prod-global-us-west-2 component: observability-regional-server retention: 1y ## EXPERIMENTAL: Number of shards to distribute targets onto. ## Number of replicas multiplied by shards is the total number of Pods created. ## Note that scaling down shards will not reshard data onto remaining instances, it must be manually moved. ## Increasing shards will not reshard data either but it will continue to be available from the same instances. ## To query globally use Thanos sidecar and Thanos querier or remote write data to a central location. ## Sharding is done on the content of the `__address__` target meta-label. ## shards: 1 resources: limits: cpu: "2" memory: 8Gi requests: cpu: 500m memory: 4Gi ## Prometheus StorageSpec for persistent data ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/user-guides/storage.md ## storageSpec: volumeClaimTemplate: spec: storageClassName: gp2 accessModes: ["ReadWriteOnce"] resources: requests: storage: 100Gi ## Select all the service monitors by set this to false. serviceMonitorSelectorNilUsesHelmValues: false ## AdditionalScrapeConfigs allows specifying additional Prometheus scrape configurations. Scrape configurations ## are appended to the configurations generated by the Prometheus Operator. Job configurations must have the form ## as specified in the official Prometheus documentation: ## https://prometheus.io/docs/prometheus/latest/configuration/configuration/#scrape_config. As scrape configs are ## appended, the user is responsible to make sure it is valid. Note that using this feature may expose the possibility ## to break upgrades of Prometheus. It is advised to review Prometheus release notes to ensure that no incompatible ## scrape configs are going to break Prometheus after the upgrade. ## AdditionalScrapeConfigs can be defined as a list or as a templated string. ## ## The scrape configuration example below will find master nodes, provided they have the name .*mst.*, relabel the ## port to 2379 and allow etcd scraping provided it is running on all Kubernetes master nodes ## additionalScrapeConfigs: [] # - job_name: kube-etcd # kubernetes_sd_configs: # - role: node # scheme: https # tls_config: # ca_file: /etc/prometheus/secrets/etcd-client-cert/etcd-ca # cert_file: /etc/prometheus/secrets/etcd-client-cert/etcd-client # key_file: /etc/prometheus/secrets/etcd-client-cert/etcd-client-key # relabel_configs: # - action: labelmap # regex: __meta_kubernetes_node_label_(.+) # - source_labels: [__address__] # action: replace # targetLabel: __address__ # regex: ([^:;]+):(\d+) # replacement: ${1}:2379 # - source_labels: [__meta_kubernetes_node_name] # action: keep # regex: .*mst.* # - source_labels: [__meta_kubernetes_node_name] # action: replace # targetLabel: node # regex: (.*) # replacement: ${1} # metric_relabel_configs: # - regex: (kubernetes_io_hostname|failure_domain_beta_kubernetes_io_region|beta_kubernetes_io_os|beta_kubernetes_io_arch|beta_kubernetes_io_instance_type|failure_domain_beta_kubernetes_io_zone) # action: labeldrop # ## If scrape config contains a repetitive section, you may want to use a template. ## In the following example, you can see how to define `gce_sd_configs` for multiple zones # additionalScrapeConfigs: | # - job_name: "node-exporter" # gce_sd_configs: # {{range $zone := .Values.gcp_zones}} # - project: "project1" # zone: "{{$zone}}" # port: 9100 # {{end}} # relabel_configs: # ... ## If additional scrape configurations are already deployed in a single secret file you can use this section. ## Expected values are the secret name and key ## Cannot be used with additionalScrapeConfigs additionalScrapeConfigsSecret: {} # enabled: false # name: # key: ## AdditionalAlertManagerConfigs allows for manual configuration of alertmanager jobs in the form as specified ## in the official Prometheus documentation https://prometheus.io/docs/prometheus/latest/configuration/configuration/#. ## AlertManager configurations specified are appended to the configurations generated by the Prometheus Operator. ## As AlertManager configs are appended, the user is responsible to make sure it is valid. Note that using this ## feature may expose the possibility to break upgrades of Prometheus. It is advised to review Prometheus release ## notes to ensure that no incompatible AlertManager configs are going to break Prometheus after the upgrade. ## additionalAlertManagerConfigs: [] # - consul_sd_configs: # - server: consul.dev.test:8500 # scheme: http # datacenter: dev # tag_separator: ',' # services: # - metrics-prometheus-alertmanager ## If additional alertmanager configurations are already deployed in a single secret, or you want to manage ## them separately from the helm deployment, you can use this section. ## Expected values are the secret name and key ## Cannot be used with additionalAlertManagerConfigs additionalAlertManagerConfigsSecret: {} # name: # key: ## AdditionalAlertRelabelConfigs allows specifying Prometheus alert relabel configurations. Alert relabel configurations specified are appended ## to the configurations generated by the Prometheus Operator. Alert relabel configurations specified must have the form as specified in the ## official Prometheus documentation: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#alert_relabel_configs. ## As alert relabel configs are appended, the user is responsible to make sure it is valid. Note that using this feature may expose the ## possibility to break upgrades of Prometheus. It is advised to review Prometheus release notes to ensure that no incompatible alert relabel ## configs are going to break Prometheus after the upgrade. ## additionalAlertRelabelConfigs: [] # - separator: ; # regex: prometheus_replica # replacement: $1 # action: labeldrop ## If additional alert relabel configurations are already deployed in a single secret, or you want to manage ## them separately from the helm deployment, you can use this section. ## Expected values are the secret name and key ## Cannot be used with additionalAlertRelabelConfigs additionalAlertRelabelConfigsSecret: {} # name: # key: kube-state-metrics: image: repository: 557537366020.dkr.ecr.us-west-2.amazonaws.com/third-party/kube-state-metrics/kube-state-metrics tag: v2.16.0 metricLabelsAllowlist: - nodes=[*]