昨天將 kubernetes 集群中的 pki 及 etcd 資料庫進行備份，今天則進行還原的動作。

1. 首先，將 master 節點恢復成乾淨的虛擬主機。

NODE=master1

VBoxManage controlvm ${NODE} poweroff
sleep 2
VBoxManage unregistervm --delete ${NODE}

VBoxManage createvm --name ${NODE} --register
VBoxManage modifyvm ${NODE} \
      --cpus 2 \
      --memory 2048 \
      --acpi on \
      --boot2 dvd \
      --nic1 nat --nic2 intnet \
      --ioapic on \
      --ostype Ubuntu_64
VBoxManage storagectl ${NODE} --name "SATA Controller" --add sata
VBoxManage storagectl ${NODE} --name "IDE Controller" --add ide

for id in 1 2
do
  DISK="${HOME}/VirtualBox VMs/${NODE}/disk-${id}.vdi"
  VBoxManage createvdi --filename "${DISK}" --size 10240
  VBoxManage storageattach ${NODE} \
          --storagectl "SATA Controller" --port ${id} --device 0 \
          --type hdd --medium "${DISK}"
done

2. 重新在 master1 上安裝作業系統。

ISO=~/Documents/iso/ubuntu-18.04.2-server-amd64.iso
USERNAME=`whoami`
PASSWORD=iamironman

VBoxManage storageattach ${NODE} \
        --storagectl "IDE Controller" --port 0 --device 0 \ 
        --type dvddrive --medium ${ISO}

# unattended installation https://www.debian.org/releases/etch/ia64/apbs04.html.zh_CN
VBoxManage unattended install \
        ${NODE} \
        --iso=${ISO} \
        --hostname=${NODE}.localhost \
        --user=${USERNAME} --password=${PASSWORD} \
        --locale=en_US --country=TW --time-zone=UTC \
        --script-template=${PWD}/ubuntu-preseed.cfg \
        --post-install-template=${PWD}/ubuntu-postinstall.sh \
        --start-vm=headless

記得安裝必要套件，如 docker。

3. 將備份的 etcd 及 certificates 復原到 master1 上。

ETCDIMAGE=k8s.gcr.io/etcd:3.3.10

ssh master1 "sudo mkdir -p /etc/kubernetes/pki"
cat kubepki.tgz | ssh master1 "sudo tar zxpf - -C /etc/kubernetes/pki"

cat etcd-snapshot-latest.tgz | ssh master1 "tar zxf -"
ssh master1 "
        sudo mkdir -p /var/lib/etcd ;
        docker run --rm \
                -v \$(pwd):/backup \
                -v '/var/lib/etcd:/var/lib/etcd' \
                --env ETCDCTL_API=3 \
                ${ETCDIMAGE} \
                /bin/sh -c 'etcdctl \
                        --endpoints=https://127.0.0.1:2379 \
                        --cacert=/etc/kubernetes/pki/etcd/ca.crt \
                        --cert=/etc/kubernetes/pki/etcd/healthcheck-client.crt \
                        --key=/etc/kubernetes/pki/etcd/healthcheck-client.key \
                        snapshot restore '/backup/etcd-snapshot-latest.db' ; mv /default.etcd/member/ /var/lib/etcd/'"

ssh master1 "rm etcd-snapshot-latest.db"

4. 重新初始化 master1 節點；使用先前備份的 etcd。

KUBERNETES_VERSION='=1.15.0-00'
NETWORK=10.244.0.0/16
MASTERIP=10.13.13.101

NODE=master1

ssh ${NODE} "
sudo apt update -y;
sudo apt install -y apt-transport-https curl gpg;
curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo apt-key add - ;
echo 'deb https://apt.kubernetes.io/ kubernetes-xenial main' | sudo tee /etc/apt/sources.list.d/kubernetes.list ;
sudo apt update -y ;
sudo apt install -y kubelet${KUBERNETES_VERSION} kubeadm${KUBERNETES_VERSION} kubectl${KUBERNETES_VERSION} ;
sudo apt-mark hold kubelet kubeadm kubectl ;
sudo swapoff -a ;
sudo sed -i -e 's/^\\([^#].*swap.*\\)$/#\\1/g' /etc/fstab ;
"

ssh master1 "sudo kubeadm init --pod-network-cidr ${NETWORK} --apiserver-advertise-address ${MASTERIP} --ignore-preflight-errors=DirAvailable--var-lib-etcd" 2>&1 > init

ssh master1 "
        mkdir -p \${HOME}/.kube ;
        sudo cp -f /etc/kubernetes/admin.conf \${HOME}/.kube/config ;
        sudo chown \$(id -u):\$(id -g) \${HOME}/.kube/config ;
        "

ssh master1 "kubectl apply -f https://docs.projectcalico.org/v3.8/manifests/canal.yaml"

5. 還原成功。

$ kubectl get nodes
NAME      STATUS   ROLES    AGE    VERSION
master1   Ready    master   116m   v1.15.0
worker1   Ready    <none>   112m   v1.15.0

參考

1. https://labs.consol.de/kubernetes/2018/05/25/kubeadm-backup.html