apiVersion: v1
kind: Service
metadata:
  labels:
    app: etcd
  annotations:
    pulumi.com/skipAwait: "true"
  name: etcd-headless
  namespace: tidb-admin
spec:
  clusterIP: None
  ports:
    - name: peer
      port: 2380
      protocol: TCP
      targetPort: 2380
    - name: client
      port: 2379
      protocol: TCP
      targetPort: 2379
  selector:
    app: etcd
  type: ClusterIP
---
apiVersion: v1
kind: Service
metadata:
  labels:
    app: etcd
  annotations:
    pulumi.com/skipAwait: "true"
  name: etcd-svc
  namespace: tidb-admin
spec:
  ports:
    - name: etcd-cluster
      port: 2379
      protocol: TCP
      targetPort: 2379
  selector:
    app: etcd
  type: ClusterIP
---
apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
  name: etcd
  namespace: tidb-admin
spec:
  endpoints:
  - honorLabels: true
    path: /metrics
    port: etcd-cluster
  selector:
    matchLabels:
      app: etcd
---
apiVersion: apps/v1
kind: StatefulSet
metadata:
  labels:
    app: etcd
  name: etcd
  namespace: tidb-admin
spec:
  updateStrategy:
    type: RollingUpdate
    rollingUpdate:
      partition: 3
  podManagementPolicy: OrderedReady
  replicas: 3
  selector:
    matchLabels:
      app: etcd
  serviceName: etcd-headless
  template:
    metadata:
      labels:
        app: etcd
        zr-evicter.tidbcloud.com/ignored: "true"
      annotations:
        cluster-autoscaler.kubernetes.io/safe-to-evict: "false"
    spec:
      nodeSelector:
        serverless.tidbcloud.com/node: critical-services
      tolerations:
      - key: serverless.tidbcloud.com/node
        operator: Equal
        value: critical-services
        effect: NoSchedule
      - key: status-taint.cluster-autoscaler.kubernetes.io/zr-evicter
        operator: Exists
        effect: NoSchedule
      topologySpreadConstraints:
        - labelSelector:
            matchLabels:
              app: etcd
          maxSkew: 1
          topologyKey: topology.kubernetes.io/zone
          whenUnsatisfiable: DoNotSchedule
      containers:
        - command:
            - /bin/sh
            - -ec
            - etcd --data-dir=/var/lib/etcd --name=${MY_POD_NAME} --listen-peer-urls=http://0.0.0.0:2380
              --listen-client-urls=http://0.0.0.0:2379 --advertise-client-urls=http://${MY_POD_NAME}.${SERVICE_NAME}.${CLUSTER_NAMESPACE}:2379
              --initial-advertise-peer-urls=http://${MY_POD_NAME}.${SERVICE_NAME}.${CLUSTER_NAMESPACE}:2380
              --initial-cluster-state='new' --initial-cluster-token='etcd-cluster-token'
              --initial-cluster=${INITIAL_CLUSTER} --auto-compaction-retention=1 --auto-compaction-mode='periodic'
              --quota-backend-bytes=8589934592
          env:
            - name: MY_POD_NAME
              valueFrom:
                fieldRef:
                  apiVersion: v1
                  fieldPath: metadata.name
            - name: CLUSTER_NAMESPACE
              valueFrom:
                fieldRef:
                  apiVersion: v1
                  fieldPath: metadata.namespace
            - name: SERVICE_NAME
              value: etcd-headless
            - name: INITIAL_CLUSTER
              value: etcd-0=http://etcd-0.etcd-headless.tidb-admin:2380,etcd-1=http://etcd-1.etcd-headless.tidb-admin:2380,etcd-2=http://etcd-2.etcd-headless.tidb-admin:2380
          image: gcr.io/etcd-development/etcd:v3.5.4
          imagePullPolicy: IfNotPresent
          name: etcd
          ports:
            - containerPort: 2380
              name: peer
              protocol: TCP
            - containerPort: 2379
              name: client
              protocol: TCP
          resources:
            limits:
              cpu: "2"
              memory: 8Gi
            requests:
              cpu: "1"
              memory: 8Gi
          volumeMounts:
            - mountPath: /var/lib/etcd
              name: datadir
  volumeClaimTemplates:
    - apiVersion: v1
      kind: PersistentVolumeClaim
      metadata:
        name: datadir
      spec:
        accessModes:
          - ReadWriteOnce
        resources:
          requests:
            storage: 32Gi
        storageClassName: xxxx-sc
---
apiVersion: v1
kind: ServiceAccount
metadata:
  name: etcd-backup
  namespace: tidb-admin
---
apiVersion: batch/v1
kind: CronJob
metadata:
  name: etcd-backup
  namespace: tidb-admin
spec:
  schedule: "0 2 * * *"             # Run every day at 02:00
  concurrencyPolicy: Forbid         # Do not run a new job if the previous one is still running
  successfulJobsHistoryLimit: 1     # Keep at most 1 successful Job
  failedJobsHistoryLimit: 2         # Keep at most 1 failed Job
  jobTemplate:
    spec:
      backoffLimit: 3               # Retry a failed Job up to 3 times
      template:
        spec:
          serviceAccountName: etcd-backup
          restartPolicy: OnFailure
          initContainers:
            - name: rclone-config-init
              image: xxxx-image
              command:
                - xxxx-rclone-config-init-sh
              volumeMounts:
                - name: rclone-config
                  mountPath: /etc/rclone
            # rclone sidecar implemented as an initContainer with restartPolicy=Always
            - name: rclone-sidecar
              image: ghcr.io/rclone/rclone:1.71.2
              restartPolicy: Always
              securityContext:
                privileged: true
              env:
                - name: RCLONE_CONFIG
                  value: /etc/rclone/rclone.conf
              command:
                - rclone
                - mount
                - --allow-non-empty
                - "s3-etcd-backup:"
                - /backup
              volumeMounts:
                - name: backup
                  mountPath: /backup
                  mountPropagation: Bidirectional
                - name: rclone-config
                  mountPath: /etc/rclone
                  readOnly: true
          containers:
            - name: etcd-backup
              image: gcr.io/etcd-development/etcd:v3.5.4
              env:
                - name: ETCDCTL_API
                  value: "3"
              # Use /bin/sh -ec so that any command failure causes the container to exit
              # and the Job to be marked as failed (subject to backoffLimit retries).
              command:
                - /bin/sh
                - -ec
                - |
                  TS=$(date +%Y%m%d%H%M%S)
                  SNAPSHOT="etcd_snapshot_${TS}.db"

                  # Take an etcd snapshot via the service endpoint.
                  etcdctl --endpoints=http://etcd-svc:2379 snapshot save "/backup/${SNAPSHOT}"
              volumeMounts:
                - name: backup
                  mountPath: /backup
                  mountPropagation: HostToContainer
          volumes:
            - name: backup
              emptyDir: {}
            - name: rclone-config
              emptyDir: {}