apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
  name: "local-storage"
provisioner: "kubernetes.io/no-provisioner"
volumeBindingMode: "WaitForFirstConsumer"

---
apiVersion: v1
kind: ConfigMap
metadata:
  name: local-provisioner-config
  namespace: kube-system
data:
  nodeLabelsForPV: |
    - kubernetes.io/hostname
  storageClassMap: |
    local-storage:
      hostDir: /mnt/local-ssd
      mountDir: /mnt/local-ssd

---
# Local SSD provisioner
# Remount disks with a UUID. Ensure the nobarrier options is set.
# This will combine all disks with mdadm or LVM.
# If you don't want to combine disks, you can set NO_COMBINE_LOCAL_SSD=1
# mdadm is preferred over LVM, to use LVM set USE_LVM=1
apiVersion: apps/v1
kind: DaemonSet
metadata:
  name: local-volume-provisioner
  namespace: kube-system
  labels:
    app: local-volume-provisioner
spec:
  selector:
    matchLabels:
      app: local-volume-provisioner
  template:
    metadata:
      labels:
        app: local-volume-provisioner
    spec:
      hostPID: true
      nodeSelector:
        pingcap.com/aws-local-ssd: "true"
      serviceAccountName: local-storage-admin
      initContainers:
      - name: local-ssd-startup
        image: alpine
        command: ['/bin/sh', '-c', 'nsenter -t 1 -m -u -i -n -p -- bash -c "${STARTUP_SCRIPT}"']
        securityContext:
          privileged: true
        volumeMounts:
        - mountPath: /mnt/local-ssd
          name: local-disks
          mountPropagation: Bidirectional
        env:
        #- name: NO_COMBINE_LOCAL_SSD
        #  value: "1"
        #- name: USE_LVM
        #  value: "1"
        - name: STARTUP_SCRIPT
          value: |
            #!/usr/bin/env bash
            set -euo pipefail
            set -x

            devs=$(ls -lh /dev/nvme0n* | awk '{print $10}')
            # Don't combine if there is 1 disk.
            if [ "$(echo "$devs" | wc -l)" -eq 1 ] ; then
              exit 0
            fi

            raid_dev=/dev/md0
            if test -e $raid_dev ; then
              exit 0
            fi


            mnt_opts="defaults,nodelalloc,noatime,discard,nobarrier"
            ssd_mounts=$(findmnt -n -a -l --nofsroot | grep /mnt/local-ssd)

            # umount nvme0n1 because of ubuntu-tikv, this will be removed soon.
            if echo "$ssd_mounts" | grep nvme ; then
              if [ "$(ls -lh /dev/nvme0n* | wc -l)" -gt 1 ] ; then
                # Re-mount all disks as a single logical volume with a UUID
                if old_mounts=$(findmnt -n -a -l --nofsroot | grep /mnt/local-ssd) ; then
                  echo "$old_mounts" | awk '{print $1}' | while read -r ssd ; do
                    umount "$ssd"
                    echo "umount $ssd"
                  done
                fi
              fi
            fi

            new_dev=
            if ! test -e $raid_dev ; then
              echo y | /sbin/mdadm --create $raid_dev --level=0 --raid-devices=$(echo "$devs" | wc -l) $devs --force
              sudo mkfs.ext4 -F $raid_dev
              new_dev=$raid_dev
            fi

            if ! uuid=$(blkid -s UUID -o value $new_dev) ; then
              mkfs.ext4 $new_dev
              uuid=$(blkid -s UUID -o value $new_dev)
            fi

            mnt_dir="/mnt/local-ssd/$uuid"
            mkdir -p "$mnt_dir"

            if ! grep "$uuid" /etc/fstab ; then
              echo "UUID=$uuid $mnt_dir ext4 $mnt_opts" >> /etc/fstab
            fi
            mount -U "$uuid" -t ext4 --target "$mnt_dir" --options "$mnt_opts"
            chmod a+w "$mnt_dir"
      containers:
      - image: "quay.io/external_storage/local-volume-provisioner:v2.3.2"
        name: provisioner
        securityContext:
          privileged: true
        resources:
          requests:
            cpu: 100m
            memory: 100Mi
          limits:
            cpu: 100m
            memory: 100Mi
        env:
        - name: MY_NODE_NAME
          valueFrom:
            fieldRef:
              fieldPath: spec.nodeName
        - name: MY_NAMESPACE
          valueFrom:
            fieldRef:
              fieldPath: metadata.namespace
        - name: JOB_CONTAINER_IMAGE
          value: "quay.io/external_storage/local-volume-provisioner:v2.3.2"
        volumeMounts:
        - mountPath: /etc/provisioner/config
          name: provisioner-config
          readOnly: true
        # mounting /dev in DinD environment would fail
        # - mountPath: /dev
        #   name: provisioner-dev
        - mountPath: /mnt/local-ssd
          name: local-disks
          mountPropagation: "HostToContainer"
      tolerations:
      - effect: NoSchedule
        operator: Exists
      volumes:
      - name: provisioner-config
        configMap:
          name: local-provisioner-config
      # - name: provisioner-dev
      #   hostPath:
      #     path: /dev
      - name: local-disks
        hostPath:
          path: /mnt/local-ssd

---
apiVersion: v1
kind: ServiceAccount
metadata:
  name: local-storage-admin
  namespace: kube-system

---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: local-storage-provisioner-pv-binding
  namespace: kube-system
subjects:
- kind: ServiceAccount
  name: local-storage-admin
  namespace: kube-system
roleRef:
  kind: ClusterRole
  name: system:persistent-volume-provisioner
  apiGroup: rbac.authorization.k8s.io
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  name: local-storage-provisioner-node-clusterrole
  namespace: kube-system
rules:
- apiGroups: [""]
  resources: ["nodes"]
  verbs: ["get"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: local-storage-provisioner-node-binding
  namespace: kube-system
subjects:
- kind: ServiceAccount
  name: local-storage-admin
  namespace: kube-system
roleRef:
  kind: ClusterRole
  name: local-storage-provisioner-node-clusterrole
  apiGroup: rbac.authorization.k8s.io