import logging import dotmap from kubernetes import client from src import util from src.context import env, get_infra_api_cli from . import base, network, tc CR_GROUP = "cloud.pingcap.com" CR_VERSION = "v1" CR_CLUSTER = "clusters" logger = logging.getLogger("oncall_master.suspicions.cluster") class BaseCluster(base.SuspicionABC): def __init__(self, *args, **kwargs): """ :param project_id: :param kwargs: """ super().__init__(*args, **kwargs) ctx = kwargs["ctx"] self.infra_api_k8s_cli = get_infra_api_cli() self.cr_api = client.CustomObjectsApi(api_client=self.infra_api_k8s_cli) self.cluster_id = ctx.cluster_id self.project_id = ctx.project_id self.namespace = f"{env.STAGE}-{self.project_id}" self.cluster = util.get_namespaced_cr(self.cr_api, CR_GROUP, CR_VERSION, self.namespace, "clusters", self.cluster_id) self.network_name = self.cluster.spec.networkName self.network = util.get_namespaced_cr(self.cr_api, CR_GROUP, CR_VERSION, self.namespace, "networks", self.network_name) class ClusterNotReady(BaseCluster): def verify(self) -> bool: if self.cluster is None: self._solution = "clusterCR is not found, may blocking in network creation" self._next_suspicion_clss = [network.NetworkReady] return True status = self.cluster.get("status", dotmap.DotMap({})) if not status: self._solution = f"empty status, may cluster-controller is not working" self._info = { "cluster": self.cluster, "cluster_reconciler": self.cluster.metadata.labels["cloud.pingcap.com/reconciler"] or '' } self._next_suspicion_clss = [network.NetworkReady] return True if status.phase != "Running": self._solution = f"cluster not ready, phase: {status.phase or 'Unknown'}" self._info = { "cr.cluster": self.cluster, } self._next_suspicion_clss = [network.NetworkReady, tc.TCNotReady, NodeGroupNotReady, AutoScalingGroupNotReady] return True class NodeGroupNotReady(BaseCluster): def __init__(self, groups: list[dotmap.DotMap] = None, *args, **kwargs): super().__init__(*args, **kwargs) if groups is None: groups = [] if not groups: # gke uses NodePoolNotReady if self.network.spec.provider != "eks": return label_selector = { "tidbcloud-namespace": self.namespace, "tidbcloud-network": self.network_name, } label_selector_str = ', '.join([f"{key}={value}" for key, value in label_selector.items()]) groups = util.list_cluster_crs(self.cr_api, "eks.aws.crossplane.io", "v1alpha1", "nodegroups", label_selector=label_selector_str) logger.info(f"nodegroups, len: {len(groups)}, groups: {[each.metadata.name for each in groups]}") self.groups: list[dotmap.DotMap] = groups def verify(self) -> bool: # gke uses NodePoolNotReady if self.network.spec.provider != "eks": return False self._solution = [] found = False for group in self.groups: for condition in group.status.conditions: if condition.type == 'Ready' and condition.status != 'True': found = True self._solution.append( f"nodegroup {group.metadata.name} is not ready, reason: {condition.reason}, message: {condition.message}") elif condition.type == 'Synced' and condition.status != 'True': found = True self._solution.append( f"nodegroup {group.metadata.name} is not synced, reason: {condition.reason}, message: {condition.message}") return found # self-managed node group class AutoScalingGroupNotReady(BaseCluster): def __init__(self, groups: list[dotmap.DotMap] = None, *args, **kwargs): super().__init__(*args, **kwargs) if groups is None: groups = [] if not groups: # gke uses NodePoolNotReady if self.network.spec.provider != "eks": return label_selector = { "tidbcloud-namespace": self.namespace, "tidbcloud-network": self.network_name, "cluster": self.cluster_id, } label_selector_str = ', '.join([f"{key}={value}" for key, value in label_selector.items()]) groups = util.list_cluster_crs(self.cr_api, "autoscaling.aws.crossplane.io", "v1beta1", "autoscalinggroups", label_selector=label_selector_str) logger.info(f"autoScalingGroup, len: {len(groups)}, groups: {[each.metadata.name for each in groups]}") self.groups: list[dotmap.DotMap] = groups def verify(self) -> bool: # gke uses NodePoolNotReady if self.network.spec.provider != "eks": return False self._solution = [] found = False for group in self.groups: for condition in group.status.conditions: if condition.type == 'Ready' and condition.status != 'True': found = True self._solution.append( f"autoScalingGroup {group.metadata.name} is not ready, reason: {condition.reason}, message: {condition.message}") elif condition.type == 'Synced' and condition.status != 'True': found = True self._solution.append( f"autoScalingGroup {group.metadata.name} is not synced, reason: {condition.reason}, message: {condition.message}") return found