import json import re from collections import namedtuple from analyst.libs import consts from analyst.libs import metadata from analyst.libs import util from analyst.libs.consts import ServiceType # DISK_KEYS_SYNC_FROM_VM define which keys can be synced from node(ec2,vm) DISK_KEYS_SYNC_FROM_VM = { 'env', 'tenant_id', 'project_id', 'cluster_id', 'component', 'member', 'k8s', 'vpc_id', 'vpc_name', 'vpc_type', 'shoot_id', 'usedby', 'service_type', 'network_name', 'network_namespace', 'network_status', } # key: source, value: target # sync info from cloud resource labels KEYS_SYNC_FROM_RESOURCE_LABEL = { 'usedby': 'usedby', 'servicetype': 'service_type', 'Name': 'rname', 'component': 'component', 'environment': 'env', 'tenant_id': 'tenant', 'project_id': 'project', 'cluster_id': 'cluster', } K8S_NAME_PARSER_RE = re.compile(r'((nightly-ms|staging|prod)-\d+)-(eks-\w+-\w+-\d|gke-\w+-\w+\d).*') def update_by_keyword(resource: dict): keyword_items = [] k8s_keyword_items = [ { 'keyword': 'base', 'keyspace': 'k8s', 'update_items': { 'env': resource['k8s'].split('-')[0], 'vpc_type': 'base', 'usedby': 'dbaas-control-plane-base', 'tenant_id': 'admin', 'project_id': 'admin', 'cluster_id': 'admin', 'service_type': 'dedicated', 'component': 'dbaas-control-plane-base', }, }, { 'keyword': 'seed', 'keyspace': 'k8s', 'update_items': { 'env': resource['k8s'].split('-')[0], 'vpc_type': 'seed', 'usedby': 'dbaas-control-plane-seed', 'tenant_id': 'admin', 'project_id': 'admin', 'cluster_id': 'admin', 'service_type': 'dedicated', 'component': 'dbaas-control-plane-seed', }, }, { 'keyword': 'dr-rich', 'keyspace': 'k8s', 'update_items': { 'component': 'dr_worker', 'usedby': 'dr-rich', 'service_type': 'dedicated' } }, { 'keyword': 'community', 'keyspace': 'k8s', 'update_items': { 'component': 'community', 'usedby': 'community', 'service_type': 'dedicated' } }, { 'keyword': 'it-', 'keyspace': 'k8s', 'update_items': { 'component': 'it-vpc', 'usedby': 'dbaas-control-plane-base', 'service_type': 'dedicated', 'tenant_id': 'admin', 'project_id': 'admin', 'cluster_id': 'admin' } }, ] vpc_keyword_items = [ { 'keyword': '-f01-', 'keyspace': 'vpc_name', 'update_items': { 'service_type': 'serverless', 'usedby': 'serverless', }, }, { 'keyword': '-f02-', 'keyspace': 'vpc_name', 'update_items': { 'service_type': 'serverless', 'usedby': 'serverless', }, }, { 'keyword': '-f03-', 'keyspace': 'vpc_name', 'update_items': { 'service_type': 'serverless', 'usedby': 'serverless', }, }, { 'keyword': 'seed', 'keyspace': 'vpc_name', 'update_items': { 'env': resource['vpc_name'].split('-')[0], 'vpc_type': 'seed', 'usedby': 'dbaas-control-plane-seed', 'tenant_id': 'admin', 'project_id': 'admin', 'cluster_id': 'admin', 'service_type': 'dedicated', 'component': 'dbaas-control-plane-seed' }, }, { 'keyword': 'base', 'keyspace': 'vpc_name', 'update_items': { 'env': resource['vpc_name'].split('-')[0], 'vpc_type': 'base', 'usedby': 'dbaas-control-plane-base', 'tenant_id': 'admin', 'project_id': 'admin', 'cluster_id': 'admin', 'service_type': 'dedicated', 'component': 'dbaas-control-plane-base' }, }, { 'keyword': 'it-', 'keyspace': 'vpc_name', 'update_items': { 'component': 'it-vpc', 'usedby': 'dbaas-control-plane-base', 'service_type': 'dedicated', 'tenant_id': 'admin', 'project_id': 'admin', 'cluster_id': 'admin' } }, { 'keyword': 'bastion', 'keyspace': 'vpc_name', 'update_items': { 'usedby': 'dbaas-control-plane-base', 'component': 'bastion', 'service_type': 'dedicated', 'tenant_id': 'admin', 'project_id': 'admin', 'cluster_id': 'admin' } }, ] name_keyword_items = [ { 'keyword': 'master-monitor', 'keyspace': 'name', 'update_items': {'component': 'master-monitor'} }, { 'keyword': 'jira', 'keyspace': 'name', 'update_items': { 'component': 'jira', 'usedby': 'dbaas-control-plane-base', 'service_type': 'dedicated', 'tenant_id': 'admin', 'project_id': 'admin', 'cluster_id': 'admin' } }, { 'keyword': 'Jira', 'keyspace': 'name', 'update_items': { 'component': 'jira', 'usedby': 'dbaas-control-plane-base', 'service_type': 'dedicated', 'tenant_id': 'admin', 'project_id': 'admin', 'cluster_id': 'admin' } }, { 'keyword': 'atlassian', 'keyspace': 'name', 'update_items': { 'component': 'atlassian+metabase', 'usedby': 'dbaas-control-plane-base', 'service_type': 'dedicated', 'tenant_id': 'admin', 'project_id': 'admin', 'cluster_id': 'admin' } }, { 'keyword': 'it-worker', 'keyspace': 'name', 'update_items': { 'component': 'it-worker', 'usedby': 'dbaas-control-plane-base', 'service_type': 'dedicated', 'tenant_id': 'admin', 'project_id': 'admin', 'cluster_id': 'admin' } }, { 'keyword': 'it-vpc', 'keyspace': 'name', 'update_items': { 'component': 'it-vpc', 'usedby': 'dbaas-control-plane-base', 'service_type': 'dedicated', 'tenant_id': 'admin', 'project_id': 'admin', 'cluster_id': 'admin' } }, { 'keyword': 'bastion', 'keyspace': 'name', 'update_items': { 'usedby': 'dbaas-control-plane-base', 'component': 'bastion', 'service_type': 'dedicated', 'tenant_id': 'admin', 'project_id': 'admin', 'cluster_id': 'admin' } }, { 'keyword': 'null', 'keyspace': 'name', 'update_items': {'name': resource['id']} }, ] usage_keyword_items = [ { 'keyword': 'dbaas-control-plane-base', 'keyspace': 'component', 'update_items': { 'vpc_type': 'base', 'usedby': 'dbaas-control-plane-base', 'tenant_id': 'admin', 'project_id': 'admin', 'cluster_id': 'admin', 'service_type': 'dedicated', 'component': 'dbaas-control-plane-base', }, }, { 'keyword': 'dbaas-control-plane-seed', 'keyspace': 'component', 'update_items': { 'vpc_type': 'seed', 'usedby': 'dbaas-control-plane-seed', 'tenant_id': 'admin', 'project_id': 'admin', 'cluster_id': 'admin', 'service_type': 'dedicated', 'component': 'dbaas-control-plane-seed', }, }, { 'keyword': 'it-vpc', 'keyspace': 'component', 'update_items': { 'component': 'it-vpc', 'usedby': 'dbaas-control-plane-base', 'service_type': 'dedicated', 'tenant_id': 'admin', 'project_id': 'admin', 'cluster_id': 'admin' } }, { 'keyword': 'bastion', 'keyspace': 'component', 'update_items': { 'component': 'bastion', 'usedby': 'dbaas-control-plane-base', 'service_type': 'dedicated', 'tenant_id': 'admin', 'project_id': 'admin', 'cluster_id': 'admin' } }, ] ec2_wg_id_keyword_items = [ { 'keyword': 'master', 'keyspace': 'ec2_wg_id', 'update_items': {'component': 'dbaas-control-plane-shoot', 'cluster_id': 'admin'} }, { 'keyword': 'bastion', 'keyspace': 'ec2_wg_id', 'update_items': { 'usedby': 'dbaas-control-plane-base', 'component': 'bastion', 'service_type': 'dedicated', 'tenant_id': 'admin', 'project_id': 'admin', 'cluster_id': 'admin' } }, { 'keyword': 'master2', 'keyspace': 'ec2_wg_id', 'update_items': {'component': 'dbaas-control-plane-shoot', 'cluster_id': 'admin'} }, { 'keyword': 'master-extra', 'keyspace': 'ec2_wg_id', 'update_items': {'component': 'dbaas-control-plane-shoot', 'cluster_id': 'admin'} }, { 'keyword': 'master-monitor', 'keyspace': 'ec2_wg_id', 'update_items': {'component': 'ngm'} }, ] ebs_pvc_name_keyword_items = [ { 'keyword': 'metadb', 'keyspace': 'ebs_pvc_name', 'update_items': { 'component': 'metadb', 'usedby': 'dbaas-control-plane-base', 'tenant_id': 'admin', 'project_id': 'admin', 'cluster_id': 'admin', 'service_type': 'dedicated', }, }, { 'keyword': 'etcd-events', 'keyspace': 'ebs_pvc_name', 'update_items': { 'component': 'shoot_etcd', 'usedby': 'dbaas-control-plane-shoot', }, }, { 'keyword': 'main-etcd', 'keyspace': 'ebs_pvc_name', 'update_items': { 'component': 'shoot_etcd', 'usedby': 'dbaas-control-plane-shoot', }, }, { 'keyword': 'data-etcd', 'keyspace': 'ebs_pvc_name', 'update_items': { 'component': 'infra_apiserver_etcd', 'usedby': 'dbaas-control-plane-base', 'service_type': 'dedicated', }, }, { 'keyword': 'db-monitor', 'keyspace': 'ebs_pvc_name', 'update_items': { 'component': 'cluster_monitor', }, }, { 'keyword': 'loki', 'keyspace': 'ebs_pvc_name', 'update_items': { 'component': 'loki', 'service_type': 'dedicated' }, }, { 'keyword': 'thanos', 'keyspace': 'ebs_pvc_name', 'update_items': { 'component': 'thanos', 'usedby': 'dbaas-control-plane-base', 'service_type': 'dedicated' } }, { 'keyword': 'prometheus', 'keyspace': 'ebs_pvc_name', 'update_items': { 'component': 'prometheus' } }, { 'keyword': 'grafana', 'keyspace': 'ebs_pvc_name', 'update_items': { 'component': 'grafana', 'usedby': 'dbaas-control-plane-base', 'service_type': 'dedicated' } }, { 'keyword': 'mysql', 'keyspace': 'ebs_pvc_name', 'update_items': { 'component': 'mysql', 'usedby': 'dbaas-control-plane-base', 'service_type': 'dedicated' } }, { 'keyword': 'o11y-vmagent-victoria-metrics-agent', 'keyspace': 'ebs_pvc_name', 'update_items': { 'component': 'o11y-vmagent-victoria-metrics-agent', 'usedby': 'dbaas-control-plane-shoot', 'service_type': 'dedicated' } }, { 'keyword': 'ng-monitoring', 'keyspace': 'ebs_pvc_name', 'update_items': { 'component': 'ngm', 'usedby': 'dbaas-control-plane-shoot', 'service_type': 'dedicated' } }, { 'keyword': 'profile', 'keyspace': 'ebs_pvc_name', 'update_items': { 'component': 'profile', 'usedby': 'dbaas-control-plane-shoot', 'service_type': 'dedicated' } }, { 'keyword': 'jira', 'keyspace': 'ebs_pvc_name', 'update_items': { 'component': 'jira', 'usedby': 'dbaas-control-plane-base', 'service_type': 'dedicated', 'tenant_id': 'admin', 'project_id': 'admin', 'cluster_id': 'admin' } }, ] elb_service_name_keyword_items = [ { 'keyword': '-tidb', 'keyspace': 'elb_service_name', 'update_items': { 'component': 'tidb', 'cluster_id': resource['elb_service_name'].split( '/')[0].replace('tidb', ''), 'usedby': 'customer-tidb', }, }, { 'keyword': 'coredns-lb', 'keyspace': 'elb_service_name', 'update_items': { 'component': 'coredns-lb', }, }, { 'keyword': 'dataflow-agent', 'keyspace': 'elb_service_name', 'update_items': { 'component': 'dataflow-agent', }, }, { 'keyword': 'db-prometheus', 'keyspace': 'elb_service_name', 'update_items': { 'component': 'db-prometheus', }, }, { 'keyword': 'ti-community-bots', 'keyspace': 'elb_service_name', 'update_items': { 'component': 'ti-community-bots', }, }, { 'keyword': 'central', 'keyspace': 'elb_service_name', 'update_items': { 'component': 'central', }, }, { 'keyword': 'vpn-shoot', 'keyspace': 'elb_service_name', 'update_items': { 'component': 'vpn-shoot', 'usedby': 'dbaas-control-plane-shoot', }, }, { 'keyword': 'grafana', 'keyspace': 'elb_service_name', 'update_items': { 'component': 'grafana', }, }, { 'keyword': 'ingress-controller', 'keyspace': 'elb_service_name', 'update_items': { 'component': 'ingress-controller', }, }, { 'keyword': 'ingress-nginx-controller', 'keyspace': 'elb_service_name', 'update_items': { 'component': 'ingress-controller', }, }, { 'keyword': 'dbaas-proxy', 'keyspace': 'elb_service_name', 'update_items': { 'component': 'dbaas-proxy', 'usedby': 'dbaas-control-plane-shoot', }, }, ] keyword_items.extend(k8s_keyword_items) keyword_items.extend(vpc_keyword_items) keyword_items.extend(name_keyword_items) keyword_items.extend(usage_keyword_items) keyword_items.extend(ec2_wg_id_keyword_items) keyword_items.extend(ebs_pvc_name_keyword_items) keyword_items.extend(elb_service_name_keyword_items) for item in keyword_items: if item['keyword'] in resource[item['keyspace']]: resource.update(item['update_items']) # analyze by the relationship between pod and node def analyze_by_pod_node_relation(result: dict, node_name: str, md: metadata.Metadata): pods = md.get_pods_by_node(result['k8s'], node_name) for pod in pods or []: # find cluster id by pod cluster_id = result['cluster_id'] member = '' if pod.namespace == consts.TIDB_ADMIN_NS: cluster_id = 'admin' elif pod.namespace.startswith(consts.TIDB_CLUSTER_NS_PREFIX): cluster_id = pod.namespace.lstrip(consts.TIDB_CLUSTER_NS_PREFIX) if pod.pod.startswith(consts.TIDB_CLUSTER_POD_PREFIX): member = pod.pod else: continue util.set_if_empty(result, 'cluster_id', cluster_id, extra_empty_values={'shared', 'admin'}) util.set_if_empty(result, 'member', member) # sync info from resource labels def sync_from_resource_labels(resource: object, label_str: str): tags = json.loads(label_str) for src_key, target_key in KEYS_SYNC_FROM_RESOURCE_LABEL.items(): value = tags.get(src_key) if value: setattr(resource, target_key, value) def sync_from_node_to_ebs(ebs: dict, node: dict): for key in DISK_KEYS_SYNC_FROM_VM: util.set_if_empty(ebs, key, node.get(key, ''), extra_empty_values={'shared'}) def sync_from_network(resource: dict, network: metadata.Network): util.set_if_empty(resource, 'env', network.env, extra_empty_values={'shared'}) util.set_if_empty(resource, 'tenant_id', network.tenant_id, extra_empty_values={'shared'}) util.set_if_empty(resource, 'project_id', network.project_id, extra_empty_values={'shared'}) util.set_if_empty(resource, 'k8s', network.dataplane, extra_empty_values={'shared'}) util.set_if_empty(resource, 'vpc_id', network.vpc, extra_empty_values={'shared'}) util.set_if_empty(resource, 'vpc_name', network.vpc, extra_empty_values={'shared'}) util.set_if_empty(resource, 'shoot_id', network.shoot_name, extra_empty_values={'shared'}) util.set_if_empty(resource, 'service_type', ServiceType.DEDICATED, extra_empty_values={'shared'}) util.set_if_empty(resource, 'network_name', network.name, extra_empty_values={'shared'}) util.set_if_empty(resource, 'network_namespace', network.namespace, extra_empty_values={'shared'}) util.set_if_empty(resource, 'network_status', network.status, extra_empty_values={'shared'}) util.set_if_empty(resource, 'region', network.region, extra_empty_values={'shared'}) def sync_from_shoot(resource: dict, shoot: metadata.Shoot): util.set_if_empty(resource, 'env', shoot.env, extra_empty_values={'shared'}) util.set_if_empty(resource, 'tenant_id', shoot.tenant, extra_empty_values={'shared'}) util.set_if_empty(resource, 'project_id', shoot.project, extra_empty_values={'shared'}) util.set_if_empty(resource, 'shoot_id', shoot.name, extra_empty_values={'shared'}) util.set_if_empty(resource, 'service_type', ServiceType.DEDICATED, extra_empty_values={'shared'}) util.set_if_empty(resource, 'network_name', shoot.network_name, extra_empty_values={'shared'}) util.set_if_empty(resource, 'network_namespace', shoot.network_namespace, extra_empty_values={'shared'}) util.set_if_empty(resource, 'region', shoot.region, extra_empty_values={'shared'}) def sync_from_network_by_vpc_name(resource: dict, vpc_name: str, md: metadata.Metadata): network = md.get_network_by_vpc_name(vpc_name) if network: sync_from_network(resource, network) def parse_from_k8s_name(k8s_name: str): result = K8S_NAME_PARSER_RE.match(k8s_name) if result: return result.groups()[0], result.groups()[2] return None assert parse_from_k8s_name('nightly-ms-73876-gke-us-west1-ca33bc82') == ('nightly-ms-73876', 'gke-us-west1') assert parse_from_k8s_name('nightly-ms-73765-eks-us-west-2-bdb813f3') == ('nightly-ms-73765', 'eks-us-west-2') assert parse_from_k8s_name('staging-123-eks-ap-northeast-2-bdb813f3') == ('staging-123', 'eks-ap-northeast-2') assert parse_from_k8s_name('shoot--nightly-123123') == None def sync_from_network_by_k8s_name(resource: dict, k8s_name: str, md: metadata.Metadata): network = md.get_network_by_k8s_name(k8s_name) if network: sync_from_network(resource, network) else: parsed = parse_from_k8s_name(k8s_name) if not parsed: return network_namespace, network_name = parsed util.set_if_empty(resource, 'network_namespace', network_namespace) util.set_if_empty(resource, 'network_name', network_name) network = md.get_network_by_ns_and_name(network_namespace, network_name) if network: sync_from_network(resource, network) def sync_from_network_by_shoot_name(resource: dict, shoot_name: str, md: metadata.Metadata): env = resource['env'] shoot = md.get_shoot(env, shoot_name) if not shoot: return sync_from_shoot(resource, shoot) network = md.get_network_by_shoot_name(env, shoot_name) if network: sync_from_network(resource, network) ParsedFromVPCName = namedtuple('result', ['env', 'vpc_type', 'usedby']) def parse_from_vpc_name(vpc_name: str) -> ParsedFromVPCName: keyword2env = { 'prod': consts.Env.Prod, 'nightly': consts.Env.Dev, 'dev': consts.Env.Dev, 'staging': consts.Env.Staging, } keyword2usedby = { '-seed-': consts.UsedBy.DBaaSControlPlaneSeed, 'shoot': consts.UsedBy.DBaaSControlPlaneShoot, 'base': consts.UsedBy.DBaaSControlPlaneBase } keyword2vpc_type = { '-seed-': consts.VPCType.Seed, 'shoot': consts.VPCType.Shoot, 'base': consts.VPCType.Base } env = '' usedby = '' vpc_type = '' for kw, value in keyword2env.items(): if kw in vpc_name: env = value break for kw, value in keyword2usedby.items(): if kw in vpc_name: usedby = value break for kw, value in keyword2vpc_type.items(): if kw in vpc_name: vpc_type = value break else: vpc_type = consts.VPCType.Shoot return ParsedFromVPCName(env=env, vpc_type=vpc_type, usedby=usedby) def set_by_vpc_name(resource: dict, md: metadata.Metadata): vpc_name = resource['vpc_name'] if not vpc_name: return parsed = parse_from_vpc_name(vpc_name) util.set_if_empty(resource, 'env', parsed.env) util.set_if_empty(resource, 'usedby', parsed.usedby) util.set_if_empty(resource, 'vpc_type', parsed.vpc_type) def get_env_by_account(account: str) -> str: # dev and staging use the same account if account in { "385595570414", # PingCAP, default account, this is also a dataplane account "200683210682", # dbaas-infra-dev "495580073302", # dp-infra-dev "676546228356", # CLould QA "448141240262", # qa-dp-testing-dev "986330900858", # qa-system-testing-dev "646528577659", # compute-dev "013690006788", # qa-perf-testing-dev "166681434446", # resilience-dev "pingcap-gardener", "tidbcloud-dev-000", "gcp-qa-dbaas-dev", "tidbcloud-dp-dev-000", "gcp-resilience-dev", "gcp-compute-dev", }: return 'dev' elif account in { '380838443567', # pingcap-dbaas '143458967504', # dbaas-prod-dataplane-000 '730335266318', # dbaas-prod-dataplane-001 "dbaas-prod", "tidbcloud-prod-000", }: return 'prod' else: return '' def parse_env_region_by_3layer_k8s_name(k8s_name: str): patterns = [ re.compile(r'^dedicated-(dev|staging|prod)-([a-z]+-[a-z]+-?\d)-'), ] for p in patterns: m = p.match(k8s_name) if m: return m.group(1), m.group(2) return None assert parse_env_region_by_3layer_k8s_name('dedicated-prod-ap-southeast-1-0b4bd00d97de5baf7776da02-7b8b16c3') == ('prod', 'ap-southeast-1') assert parse_env_region_by_3layer_k8s_name('dedicated-dev-us-west-2-8ddbbcbba5d979f4b50d72698f6ebe-b55a26f4') == ('dev', 'us-west-2') assert parse_env_region_by_3layer_k8s_name('dedicated-staging-us-east-1-c2e471173051e5eead788901d3-b08357bc') == ('staging', 'us-east-1') assert parse_env_region_by_3layer_k8s_name('dedicated-prod-us-west1-3f67af0-9413623d') == ('prod', 'us-west1') assert parse_env_region_by_3layer_k8s_name('dedicated-dev-us-west1-1b5a7ab8-cd039228') == ('dev', 'us-west1') assert parse_env_region_by_3layer_k8s_name('dedicated-prod-asia-northeast1--9f945c5b') == ('prod', 'asia-northeast1') assert parse_env_region_by_3layer_k8s_name('staging-123-eks-ap-northeast-2-bdb813f3') == None assert parse_env_region_by_3layer_k8s_name('nightly-ms-73876-gke-us-west1-ca33bc82') == None assert parse_env_region_by_3layer_k8s_name('shoot--nightly-123123') == None def set_env_by_account(resource: dict): if resource['env'] in {'', 'null'}: resource['env'] = get_env_by_account(resource['account_id']) # because 'get_env_by_account' can't distinguish between 'dev' and 'staging' if resource['env'] == 'dev': if 'stg' in resource['vpc_name'] or 'staging' in resource['vpc_name']: resource['env'] = 'staging'