import functools from kubernetes import client from src import util from src.context import env, get_infra_api_cli, get_dataplane_cli from . import base CR_GROUP = "cloud.pingcap.com" CR_VERSION = "v1" class BaseTC(base.SuspicionABC): def __init__(self, *args, **kwargs): """ :param project_id: :param kwargs: """ super().__init__(*args, **kwargs) ctx = kwargs["ctx"] self.cluster_id = ctx.cluster_id self.project_id = ctx.project_id self.infra_api_k8s_cli = get_infra_api_cli() self.infra_api_cr_api = client.CustomObjectsApi(api_client=self.infra_api_k8s_cli) self.network_name = ctx.network_name self.infra_api_namespace = f"{env.STAGE}-{self.project_id}" self.network = util.get_namespaced_cr(self.infra_api_cr_api, CR_GROUP, CR_VERSION, self.infra_api_namespace, "networks", self.network_name) self.dataplane_namespace = self._build_tc_namespace(self.cluster_id) self.dataplane_k8s_cli = None self.dataplane_cr_api = None self.tc = None if self.network.status.phase == "Normal": self.dataplane_k8s_cli = get_dataplane_cli(network=self.network) self.dataplane_cr_api = client.CustomObjectsApi(api_client=self.dataplane_k8s_cli) self.dataplane_core_api = client.CoreV1Api(api_client=self.dataplane_k8s_cli) self.tc = util.get_namespaced_cr(self.dataplane_cr_api, "pingcap.com", "v1alpha1", self.dataplane_namespace, "tidbclusters", "db") def _build_tc_namespace(self, cluster_id: str) -> str: return f'tidb{cluster_id}' def _get_nodes(self) -> list[client.V1Node]: label_selector = {} if self.network.spec.provider == "gke": pass else: label_selector = { "cluster": self.cluster_id, } label_selector_str = ', '.join([f"{key}={value}" for key, value in label_selector.items()]) nodes = self.dataplane_core_api.list_node(label_selector=label_selector_str).items return nodes class TCNotReady(BaseTC): def verify(self) -> bool: if self.tc is None: self._solution = "tc not found" return True ready = False for condition in self.tc.status.conditions: if condition.type == "Ready" and condition.status == "True": ready = True break if not ready: self._solution = "tc is not ready" self._info["cr.tc"] = self.tc self._next_suspicion_clss = [PodNotReady] return True class PodNotReady(BaseTC): # check whether all pods are ready # check restart count # check lastError def verify(self) -> bool: pods = self.dataplane_core_api.list_namespaced_pod(namespace=self.dataplane_namespace).items non_running_pods: list[client.V1Pod] = [] for pod in pods: if pod.status.phase != "Running": non_running_pods.append(pod) self._solution = [] for component in ['pd', 'tidb', 'tikv', 'tiflash']: pod_cnt = 0 for pod in pods: if pod.metadata.labels.get('app.kubernetes.io/component', '') == component: pod_cnt += 1 if pod_cnt != self.tc.spec[component].replicas: self._solution.append( f"{component} required {self.tc.spec[component].replicas} pods, but only {pod_cnt} found") if len(non_running_pods) != 0: non_running_pod_names = ', '.join([pod.metadata.name for pod in non_running_pods]) self._solution.append(f"there are some pods are not ready: {non_running_pod_names}") self._info["non_running_pods"] = non_running_pod_names self._next_suspicion_clss = [functools.partial(PodUnscheduled, pods=non_running_pods)] for pod in non_running_pods: for condition in pod.status.conditions: if condition.type == "Initialized" and condition.status == "False": self._solution.append( f"pod {pod.metadata.name} is not initialized, reason: {condition.reason}, message: {condition.message}") if condition.type == "Ready" and condition.status == "False": self._solution.append( f"pod {pod.metadata.name} is not ready, reason: {condition.reason}, message: {condition.message}") return True class PodAndPVMismatchAZ(BaseTC): def verify(self) -> bool: pass class NodeNotReady(BaseTC): def __init__(self, nodes=None, *args, **kwargs): super().__init__(*args, **kwargs) if nodes is None: nodes = self._get_nodes() self.nodes: list[client.V1Node] = nodes def verify(self) -> bool: self._solution = [] found = False for node in self.nodes: for condition in node.status.conditions: if condition.type == "Ready" and condition.status == "False": self._solution.append( f"node {node.metadata.name} is not ready, reason: {condition.reason}, message: {condition.message}") self._next_suspicion_clss = [] found = True return found class PodUnscheduled(BaseTC): def __init__(self, pods=None, *args, **kwargs): super().__init__(*args, **kwargs) if pods is None: pods = self.dataplane_core_api.list_namespaced_pod(namespace=self.dataplane_namespace).items self.pods: list[client.V1Pod] = pods def verify(self) -> bool: self._solution = [] found = False for pod in self.pods: if pod.status.phase == "Running": continue for condition in pod.status.conditions: if condition.type == "PodScheduled" and condition.status == "False": self._solution.append( f"pod {pod.metadata.name} is not scheduled, reason: {condition.reason}, message: {condition.message}") self._next_suspicion_clss = [NodeNotReady, NodeNotEnough] found = True return found class NodeNotEnough(BaseTC): def __init__(self, nodes=None, *args, **kwargs): super().__init__(*args, **kwargs) if nodes is None: nodes = self._get_nodes() self.nodes: list[client.V1Node] = nodes def verify(self) -> bool: self._solution = [] found = False for component in ['pd', 'tidb', 'tikv', 'tiflash']: node_cnt = 0 for node in self.nodes: if node.metadata.labels.component == component: node_cnt += 1 if node_cnt != self.tc.spec[component].replicas: found = True self._solution.append( f"{component} required {self.tc.spec[component].replicas} nodes, but only {node_cnt} provisioned") return found