#!/usr/bin/env python3 # ngm.py - A script for deploying ngm. # # ----- # # usage: ngm.py deploy [-h] --shoot SHOOT --cluster-id CLUSTER_ID --tidb-version TIDB_VERSION # [--node-type NODE_TYPE] [--garden-ctx GARDEN_CTX] [--garden-ns GARDEN_NS] [--env ENV] # [--dry-run] # # optional arguments: # -h, --help show this help message and exit # --shoot SHOOT # --cluster-id CLUSTER_ID # --tidb-version TIDB_VERSION # e.g. v5.4.1 # --node-type NODE_TYPE # e.g. t3.medium # --garden-ctx GARDEN_CTX # e.g. garden # --garden-ns GARDEN_NS # e.g. garden-nightly # --env ENV dev/staging # --dry-run # # ----- # # Here is a sample deployment in the staging environment: # # $ ./ngm.py deploy --shoot f350adb5 \ # --cluster-id 1373933076658061947 \ # --tidb-version v6.1.0 \ # --garden-ns garden-staging \ # --env staging \ # --dry-run # # [-] checking kubectl... # [-] fetching shoot kubeconfig into /tmp/shoot-config.yaml... # [v] shoot context name: shoot--staging--f350adb5 # [-] importing shoot kubeconfig from /tmp/shoot-config.yaml... # [-] checking current nodes... # [v] current node count: 12 # [-] fetching shoot spec... # [v] shoot tenant id: 1369847559691427371 # [v] shoot project id: 1369847559691427441 # [v] az list: {'us-east-1a', 'us-east-1c', 'us-east-1b'} # [-] generating shoot patch into /tmp/shoot-patch.yaml # --- /tmp/shoot-patch.yaml --- # # spec: # provider: # workers: # - labels: # component: ngm # servicetype: dedicated # usedby: clinic # environment: staging # project: "1369847559691427441" # cluster: "1373933076658061947" # gardener.node.az.info: us-east-1a # gardener.node.machine.type: t3.medium # gardener.wg.name: tidb1373933076658061947-ngm # gardener.wg.prefix.name: tidb1373933076658061947-ngm # machine: # image: # name: centos-v20210420 # version: 7.8.2003 # type: t3.medium # maxSurge: 1 # maxUnavailable: 0 # maximum: 1 # minimum: 1 # name: tidb1373933076658061947-ngm # taints: # - effect: NoSchedule # key: dedicated # value: tidb1373933076658061947-ngm # volume: # size: 200Gi # type: gp2 # zones: # - us-east-1a # # --- /tmp/shoot-patch.yaml --- # [-] please confirm your shoot patch above (type 'y' to continue) >>> y # [-] applying shoot patch from /tmp/shoot-patch.yaml # [D] dry run: kubectl --context=garden -n garden-staging patch shoots f350adb5 --patch-file=/tmp/shoot-patch.yaml # [-] generating ngm spec into /tmp/ngm-spec.yaml # --- /tmp/ngm-spec.yaml --- # # apiVersion: pingcap.com/v1alpha1 # kind: TidbNGMonitoring # metadata: # name: db # namespace: tidb1373933076658061947 # spec: # nodeSelector: # gardener.wg.prefix.name: tidb1373933076658061947-ngm # tolerations: # - effect: NoSchedule # key: dedicated # value: tidb1373933076658061947-ngm # clusters: # - name: db # namespace: tidb1373933076658061947 # ngMonitoring: # version: v6.1.0 # baseImage: pingcap/ng-monitoring # requests: # storage: 200Gi # limits: # storage: 200Gi # # --- /tmp/ngm-spec.yaml --- # [-] please confirm your ngm spec above (type 'y' to continue) >>> y # [-] applying ngm spec from /tmp/ngm-spec.yaml # [D] dry run: kubectl --context=shoot--staging--f350adb5 -n tidb1373933076658061947 create -f /tmp/ngm-spec.yaml # [!] all steps completed import argparse import subprocess import sys import yaml def must_run(shell): r = subprocess.run(shell, shell=True, capture_output=True) if r.returncode != 0: print(f"[x] failed to run: {shell}") print("stdout:") print(r.stdout.decode()) print("stderr:") print(r.stderr.decode()) sys.exit(r.returncode) return r def shoot_patch(env, project_id, cluster_id, node_type, az): return f""" spec: provider: workers: - labels: component: ngm servicetype: dedicated usedby: clinic environment: {env} project: "{project_id}" cluster: "{cluster_id}" gardener.node.az.info: {az} gardener.node.machine.type: {node_type} gardener.wg.name: tidb{cluster_id}-ngm gardener.wg.prefix.name: tidb{cluster_id}-ngm machine: image: name: centos-v20210420 version: 7.8.2003 type: {node_type} maxSurge: 1 maxUnavailable: 0 maximum: 1 minimum: 1 name: tidb{cluster_id}-ngm taints: - effect: NoSchedule key: dedicated value: tidb{cluster_id}-ngm volume: size: 200Gi type: gp2 zones: - {az} """ def ngm_spec(cluster_id, tidb_version): return f""" apiVersion: pingcap.com/v1alpha1 kind: TidbNGMonitoring metadata: name: db namespace: tidb{cluster_id} spec: nodeSelector: gardener.wg.prefix.name: tidb{cluster_id}-ngm tolerations: - effect: NoSchedule key: dedicated value: tidb{cluster_id}-ngm clusters: - name: db namespace: tidb{cluster_id} ngMonitoring: version: {tidb_version} baseImage: pingcap/ng-monitoring requests: storage: 200Gi limits: storage: 200Gi """ def command_gen_patch(args): print(shoot_patch(args.env, args.project_id, args.cluster_id, args.node_type, args.az)) def command_gen_spec(args): print(ngm_spec(args.cluster_id, args.tidb_version)) def command_deploy(args): print("[-] checking kubectl...") must_run(f"kubectl --context={args.garden_ctx} -n {args.garden_ns} get shoots") print("[-] fetching shoot kubeconfig into /tmp/shoot-config.yaml...") r = must_run(f"""kubectl --context={args.garden_ctx} -n {args.garden_ns} get secret {args.shoot}.kubeconfig \ --template='{{{{ .data.kubeconfig }}}}' \ | base64 -d \ | tee /tmp/shoot-config.yaml""") stdout = r.stdout.decode() if not stdout: print("[x] failed to get shoot kubeconfig, stderr:") print(r.stderr.decode()) sys.exit(-1) shoot_config = yaml.full_load(stdout) try: shoot_context_name = shoot_config["contexts"][0]["name"] except (TypeError, KeyError, IndexError): print("[x] failed to read shoot context name from shoot kubeconfig, raw data:") print(stdout) sys.exit(-1) print("[v] shoot context name:", shoot_context_name) print("[-] importing shoot kubeconfig from /tmp/shoot-config.yaml...") must_run("konfig import --save /tmp/shoot-config.yaml") print("[-] checking current nodes...") r = must_run(f"kubectl --context={shoot_context_name} get nodes -o yaml") stdout = r.stdout.decode() if not stdout: print("[x] failed to get nodes, stderr:") print(r.stderr.decode()) sys.exit(-1) nodes = yaml.full_load(stdout) try: node_count = len(nodes["items"]) except (TypeError, KeyError): print("[x] failed to read node count from nodes, raw data:") print(stdout) sys.exit(-1) print("[v] current node count:", node_count) print("[-] fetching shoot spec...") r = must_run(f"kubectl --context={args.garden_ctx} -n {args.garden_ns} get shoots {args.shoot} -o yaml") stdout = r.stdout.decode() if not stdout: print("[x] failed to fetch shoot spec, stderr:") print(r.stderr.decode()) sys.exit(-1) shoot_spec = yaml.full_load(stdout) try: tenant_id = shoot_spec["metadata"]["labels"]["tenant"] project_id = shoot_spec["metadata"]["labels"]["project"] az = set() for worker in shoot_spec["spec"]["provider"]["workers"]: for zone in worker["zones"]: az.add(zone) if not az: raise KeyError() except (TypeError, KeyError, IndexError): print("[x] failed to read value from shoot spec, raw data:") print(stdout) sys.exit(-1) print("[v] shoot tenant id:", tenant_id) print("[v] shoot project id:", project_id) print("[v] az list:", az) print("[-] generating shoot patch into /tmp/shoot-patch.yaml") patch = shoot_patch(args.env, project_id, args.cluster_id, args.node_type, list(az)[0]) print("--- /tmp/shoot-patch.yaml ---") print(patch) print("--- /tmp/shoot-patch.yaml ---") c = input("[-] please confirm your shoot patch above (type 'y' to continue) >>> ") if c != "y": print("[x] interrupted") sys.exit(0) with open("/tmp/shoot-patch.yaml", "w") as f: f.write(patch) print("[-] applying shoot patch from /tmp/shoot-patch.yaml") cmd = f"kubectl --context={args.garden_ctx} -n {args.garden_ns} patch shoots {args.shoot} \ --patch-file=/tmp/shoot-patch.yaml" if args.dry_run: print("[D] dry run:", cmd) else: must_run(cmd) print("[-] generating ngm spec into /tmp/ngm-spec.yaml") spec = ngm_spec(args.cluster_id, args.tidb_version) print("--- /tmp/ngm-spec.yaml ---") print(spec) print("--- /tmp/ngm-spec.yaml ---") c = input("[-] please confirm your ngm spec above (type 'y' to continue) >>> ") if c != "y": print("[x] interrupted") sys.exit(0) with open("/tmp/ngm-spec.yaml", "w") as f: f.write(spec) print("[-] applying ngm spec from /tmp/ngm-spec.yaml") cmd = f"kubectl --context={shoot_context_name} -n tidb{args.cluster_id} create -f /tmp/ngm-spec.yaml" if args.dry_run: print("[D] dry run:", cmd) else: must_run(cmd) print("[!] all steps completed") def main(): parser = argparse.ArgumentParser(description="Deploy ngm on TiDB Cloud") subparsers = parser.add_subparsers(dest="subcommand") parser_gen_patch = subparsers.add_parser("gen-patch", help="Generate garden patch") parser_gen_patch.add_argument("--project-id", required=True) parser_gen_patch.add_argument("--cluster-id", required=True) parser_gen_patch.add_argument("--az", required=True, help="e.g. us-west-2a") parser_gen_patch.add_argument("--env", default="", help="dev/staging") parser_gen_patch.add_argument("--node-type", default="t3.medium", help="e.g. t3.medium") parser_gen_spec = subparsers.add_parser("gen-spec", help="Generate ngm spec") parser_gen_spec.add_argument("--cluster-id", required=True) parser_gen_spec.add_argument("--tidb-version", required=True, help="e.g. v5.4.1") parser_deploy = subparsers.add_parser("deploy", help="Deploy ngm") parser_deploy.add_argument("--shoot", required=True) parser_deploy.add_argument("--cluster-id", required=True) parser_deploy.add_argument("--tidb-version", required=True, help="e.g. v5.4.1") parser_deploy.add_argument("--node-type", default="t3.medium", help="e.g. t3.medium") parser_deploy.add_argument("--garden-ctx", default="garden", help="e.g. garden") parser_deploy.add_argument("--garden-ns", default="garden-nightly", help="e.g. garden-nightly") parser_deploy.add_argument("--env", default="", help="dev/staging") parser_deploy.add_argument("--dry-run", action="store_true") args = parser.parse_args() if args.subcommand == "gen-patch": command_gen_patch(args) elif args.subcommand == "gen-spec": command_gen_spec(args) elif args.subcommand == "deploy": command_deploy(args) else: parser.print_help() if __name__ == "__main__": main()