role: "Agent" rbac: create: true psp: create: false image: repository: TMPL_IMAGE_REGISTRY/tidbcloud/vector tag: "0.37.1-76b50d-debian" serviceAccount: create: true annotations: azure.workload.identity/client-id: TMPL_VECTOR_ROLE_ARN persistence: hostPath: path: "/home/vector" podLabels: vector.dev/exclude: "true" azure.workload.identity/use: "true" args: - --config-dir - "/etc/vector/" env: - name: K8S_CLUSTER_NAME value: TMPL_CLUSTER_NAME resources: limits: cpu: "1" memory: 1Gi requests: cpu: 10m memory: 100Mi customConfig: data_dir: /vector/vector-data api: enabled: true address: 0.0.0.0:8686 playground: false sources: all_logs: type: kubernetes_logs delay_deletion_ms: 180000 # Refer to https://github.com/vectordotdev/vector/issues/7840#issuecomment-921533804 glob_minimum_cooldown_ms: 5000 journald: type: journald self_metrics: type: internal_metrics self_logs: type: internal_logs transforms: ensure_component: type: remap inputs: - all_logs drop_on_error: true reroute_dropped: true source: |- if !is_null(.kubernetes.pod_labels."app.kubernetes.io/component") { .o11y.component = .kubernetes.pod_labels."app.kubernetes.io/component" } else if !is_null(.kubernetes.container_name) { .o11y.component = .kubernetes.container_name } else { .o11y.component = "unknown" } route_logs: type: route inputs: - ensure_component route: tidb_cluster_general_logs: |- includes(["tidb","tikv","pd","ticdc","tiflash","tiflash-learner","db-tidb-extra-access","backup","restore","tidb-lightning","dm-master","dm-worker","extra-tidb"], .o11y.component) infra_logs: |- !includes(["tidb","tikv","pd","ticdc","tiflash","tiflash-learner","db-tidb-extra-access","backup","restore","tidb-lightning","dm-master","dm-worker","extra-tidb"], .o11y.component) ensure_cluster_id: type: remap inputs: - route_logs.tidb_cluster_general_logs drop_on_error: true reroute_dropped: true source: |- if !is_null(.kubernetes.pod_labels."tags.tidbcloud.com/cluster") { .o11y.cluster_id = .kubernetes.pod_labels."tags.tidbcloud.com/cluster" } else { parsed, err = parse_grok(.kubernetes.pod_namespace, "tidb%{GREEDYDATA:cluster_id}") if err == null { .o11y.cluster_id = parsed.cluster_id } else { .o11y.cluster_id = "unknown" } } enrich_tidb_cluster_log_message: type: remap inputs: - ensure_cluster_id drop_on_error: true reroute_dropped: true source: |- t = to_string!(.timestamp) node = .kubernetes.pod_node_name pod_namespace = .kubernetes.pod_namespace pod_name = .kubernetes.pod_name container_name = .kubernetes.container_name prefix = join!([t, node, pod_namespace, pod_name, container_name], ";") new_message, err = prefix + " " + .message if err == null { .message = new_message } enrich_infra_log_message: type: remap inputs: - route_logs.infra_logs drop_on_error: true reroute_dropped: true source: |- t = to_string!(.timestamp) node = .kubernetes.pod_node_name pod_namespace = .kubernetes.pod_namespace pod_name = .kubernetes.pod_name container_name = .kubernetes.container_name prefix = join!([t, node, pod_namespace, pod_name, container_name], ";") new_message, err = prefix + " " + .message if err == null { .message = new_message } ensure_all_logs_labels: type: remap inputs: - all_logs drop_on_error: true reroute_dropped: true source: |- if is_null(.kubernetes.pod_labels.app) { .kubernetes.pod_labels.app = .kubernetes.pod_name } if is_null(.kubernetes.pod_labels.release) { .kubernetes.pod_labels.release = "unknown" } sinks: self_metrics_sink: type: prometheus_exporter inputs: [self_metrics] address: 0.0.0.0:9598 self_logs_sink: type: loki inputs: [self_logs] labels: cluster: TMPL_CLUSTER_IDENTITY cluster_env: TMPL_ENV node: "{{ `{{ host }}` }}" app: vector_log_agent endpoint: http://TMPL_O11Y_ENDPOINT_SERVICE_DNS_NAME/loki compression: gzip out_of_order_action: accept batch: max_bytes: 1024000 timeout_secs: 1 buffer: type: disk max_size: 1073741824 encoding: codec: json loki_sink: type: loki inputs: [ensure_all_logs_labels] labels: namespace: "{{ `{{ kubernetes.pod_namespace }}` }}" container: "{{ `{{ kubernetes.container_name }}` }}" node: "{{ `{{ kubernetes.pod_node_name }}` }}" instance: "{{ `{{ kubernetes.pod_name }}` }}" app: "{{ `{{ kubernetes.pod_labels.app }}` }}" release: "{{ `{{ kubernetes.pod_labels.release }}` }}" stream: "{{ `{{ stream }}` }}" cluster: TMPL_CLUSTER_IDENTITY cluster_env: TMPL_ENV endpoint: http://TMPL_O11Y_ENDPOINT_SERVICE_DNS_NAME/loki compression: gzip out_of_order_action: accept batch: max_bytes: 1024000 timeout_secs: 1 buffer: type: disk max_size: 1073741824 encoding: codec: text loki_sys_logs: type: loki inputs: [journald] labels: node: "{{ `{{ kubernetes.pod_node_name }}` }}" syslog: '{{ "{{ SYSLOG_IDENTIFIER }}" }}' cluster: TMPL_CLUSTER_IDENTITY cluster_env: TMPL_ENV endpoint: http://TMPL_O11Y_ENDPOINT_SERVICE_DNS_NAME/loki compression: gzip out_of_order_action: accept batch: max_bytes: 1024000 timeout_secs: 1 buffer: type: disk max_size: 1073741824 encoding: codec: text azure_infra_logs: type: azure_blob inputs: [enrich_infra_log_message] container_name: TMPL_LOG_BUCKET_NAME storage_account: TMPL_STORAGE_ACCOUNT_NAME blob_prefix: 1/TMPL_TENANT_ID/TMPL_PROJECT_ID/k8s-infra/logs/{{ `{{ o11y.component }}` }}/ compression: gzip batch: max_bytes: 104857600 timeout_secs: 180 buffer: type: disk max_size: 1073741824 encoding: codec: text azure_tidb_cluster_logs: type: azure_blob inputs: [enrich_tidb_cluster_log_message] container_name: TMPL_LOG_BUCKET_NAME storage_account: TMPL_STORAGE_ACCOUNT_NAME blob_prefix: "1/TMPL_TENANT_ID/TMPL_PROJECT_ID/{{ `{{ o11y.cluster_id }}` }}/logs/{{ `{{ o11y.component }}` }}/" compression: gzip batch: max_bytes: 104857600 timeout_secs: 180 buffer: type: disk max_size: 1073741824 encoding: codec: text azure_unmatched_kubernetes_logs: type: azure_blob inputs: - ensure_component.dropped - route_logs._unmatched - ensure_cluster_id.dropped - enrich_tidb_cluster_log_message.dropped - enrich_infra_log_message.dropped container_name: TMPL_LOG_BUCKET_NAME storage_account: TMPL_STORAGE_ACCOUNT_NAME blob_prefix: "1/TMPL_TENANT_ID/TMPL_PROJECT_ID/k8s-infra/logs/_unmatched/" compression: gzip batch: max_bytes: 104857600 timeout_secs: 180 buffer: type: disk max_size: 1073741824 encoding: codec: text azure_journald_logs: type: azure_blob inputs: [journald] container_name: TMPL_LOG_BUCKET_NAME storage_account: TMPL_STORAGE_ACCOUNT_NAME blob_prefix: "1/TMPL_TENANT_ID/TMPL_PROJECT_ID/k8s-infra/logs/journald/{{ `{{ host }}` }}/" compression: gzip batch: max_bytes: 104857600 timeout_secs: 180 buffer: type: disk max_size: 1073741824 encoding: codec: text containerPorts: - name: prom-exporter containerPort: 9598 protocol: TCP - name: api containerPort: 8686 protocol: TCP