iT邦幫忙

1

二、K8S etcd架設

kubernetes devops
DK 2018-08-21 23:23:39 ‧ 8027 瀏覽

etcd會紀錄你pod、service deployments 的那些資訊，etcd死了，你master活著也沒用。(etcd不用安裝kubelet)

IP hostname
192.168.3.21 k8s-etcd-01
192.168.3.22 k8s-etcd-02
192.168.3.23 k8s-etcd-03

3台etcd主機 安裝etcd CA認證

# echo "net.ipv4.ip_forward = 1" >> /etc/sysctl.conf ; sysctl -p
# curl -o /usr/local/bin/cfssl https://pkg.cfssl.org/R1.2/cfssl_linux-amd64
# curl -o /usr/local/bin/cfssljson https://pkg.cfssl.org/R1.2/cfssljson_linux-amd64
# chmod +x /usr/local/bin/cfssl*
# echo "192.168.3.21  k8s-etcd-01" >> /etc/hosts
# echo "192.168.3.22  k8s-etcd-02" >> /etc/hosts
# echo "192.168.3.23  k8s-etcd-03" >> /etc/hosts

建立CA證書

先登入 192.168.3.21 k8s-etcd-01

# mkdir -p /etc/kubernetes/etcd ; cd /etc/kubernetes/etcd
# cat >ca-config.json <<EOF
{
    "signing": {
        "default": {
            "expiry": "43800h"
        },
        "profiles": {
            "server": {
                "expiry": "43800h",
                "usages": [
                    "signing",
                    "key encipherment",
                    "server auth",
                    "client auth"
                ]
            },
            "client": {
                "expiry": "43800h",
                "usages": [
                    "signing",
                    "key encipherment",
                    "client auth"
                ]
            },
            "peer": {
                "expiry": "43800h",
                "usages": [
                    "signing",
                    "key encipherment",
                    "server auth",
                    "client auth"
                ]
            }
        }
    }
 }
EOF

# cat >ca-csr.json <<EOF
{
    "CN": "etcd",
    "key": {
        "algo": "rsa",
        "size": 2048
    }
}
EOF

生成CA證書

# cfssl gencert -initca ca-csr.json | cfssljson -bare ca -

生成etcd客戶端CA證書

cat > client.json <<EOF
{
  "CN": "client",
  "key": {
      "algo": "ecdsa",
      "size": 256
  }
}
EOF

創立CA證書

# cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=client client.json | cfssljson -bare client

ls 看一下 client.pem和client-key.pem都被創建。

把CA認證複製到其他的etcd

登入 192.168.3.22 k8s-etcd-02 和 192.168.3.23 k8s-etcd-03 下以下的command

# mkdir -p /etc/kubernetes/etcd ; cd /etc/kubernetes/etcd
# scp 192.168.3.21:/etc/kubernetes/etcd/ca.pem   /etc/kubernetes/etcd
# scp 192.168.3.21:/etc/kubernetes/etcd/ca-key.pem   /etc/kubernetes/etcd
# scp 192.168.3.21:/etc/kubernetes/etcd/client.pem   /etc/kubernetes/etcd
# scp 192.168.3.21:/etc/kubernetes/etcd/client-key.pem   /etc/kubernetes/etcd
# scp 192.168.3.21:/etc/kubernetes/etcd/ca-config.json   /etc/kubernetes/etcd

3台etcd都要設定

# cd /etc/kubernetes/etcd
# export PEER_NAME=$(hostname)
# export PRIVATE_IP=$(ip addr show eth0 | grep -Po 'inet \K[\d.]+')
注意網卡 因為我用eth0跟其他etcd server做溝通
# cfssl print-defaults csr > config.json
# sed -i '0,/CN/{s/example\.net/'"$PEER_NAME"'/}' config.json
# sed -i 's/www\.example\.net/'"$PRIVATE_IP"'/' config.json
# sed -i 's/example\.net/'"$PEER_NAME"'/' config.json
# cat config.json 
{
    "CN": "K8S-etcd-01",
    "hosts": [
            "K8S-etcd-1",
            "192.168.3.21"
    ],
    "key": {
            "algo": "ecdsa",
                "size": 256
    },
    "names": [
            {
                "C": "US",
                "L": "CA",
                "ST": "San Francisco"
            }
        ]
}

# cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=server config.json | cfssljson -bare server
# cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=peer config.json | cfssljson -bare peer

會建立檔案:peer.pem, peer-key.pem, server.pem, server-key.pem

安裝etcd service 3台etcd都安裝及設定

# ETCD_VERSION="v3.3.9" 
# curl -sSL https://github.com/coreos/etcd/releases/download/${ETCD_VERSION}/etcd-${ETCD_VERSION}-linux-amd64.tar.gz | tar -xzv --strip-components=1 -C /usr/local/bin/
# touch /etc/etcd.env
# export PEER_NAME=$(hostname)
# export PRIVATE_IP=$(ip addr show eth0 | grep -Po 'inet \K[\d.]+')
# echo "PEER_NAME=${PEER_NAME}" >> /etc/etcd.env
# echo "PRIVATE_IP=${PRIVATE_IP}" >> /etc/etcd.env
# echo "ETCD_CLIENT_CERT_AUTH="true"" >> /etc/etcd.env
# echo "ETCD_AUTO_TLS="true""   >> /etc/etcd.env
# echo "ETCD_PEER_CLIENT_CERT_AUTH="true""  >> /etc/etcd.env
# echo "ETCD_PEER_AUTO_TLS="true""  >> /etc/etcd.env

192.168.3.21 k8s-etcd-01

# cat /etc/etcd.env 
PEER_NAME=K8S-etcd-01
PRIVATE_IP=192.168.3.21
ETCD_CLIENT_CERT_AUTH="true"
ETCD_AUTO_TLS="true"
ETCD_PEER_CLIENT_CERT_AUTH="true"
ETCD_PEER_AUTO_TLS="true"

192.168.3.22 k8s-etcd-02

# cat /etc/etcd.env 
PEER_NAME=K8S-etcd-02
PRIVATE_IP=192.168.3.22
ETCD_CLIENT_CERT_AUTH="true"
ETCD_AUTO_TLS="true"
ETCD_PEER_CLIENT_CERT_AUTH="true"
ETCD_PEER_AUTO_TLS="true"

192.168.3.23 k8s-etcd-03

# cat /etc/etcd.env 
PEER_NAME=K8S-etcd-03
PRIVATE_IP=192.168.3.23
ETCD_CLIENT_CERT_AUTH="true"
ETCD_AUTO_TLS="true"
ETCD_PEER_CLIENT_CERT_AUTH="true"
ETCD_PEER_AUTO_TLS="true"

設定K8s-etcd-01

# vim /etc/systemd/system/etcd.service

[Unit]
Description=etcd
Documentation=https://github.com/coreos/etcd
Conflicts=etcd.service
Conflicts=etcd2.service

[Service]
EnvironmentFile=/etc/etcd.env
Type=notify
Restart=always
RestartSec=5s
LimitNOFILE=40000
TimeoutStartSec=0

ExecStart=/usr/local/bin/etcd \
--name k8s-etcd-01 \
--data-dir /var/lib/etcd \
--listen-client-urls https://192.168.3.21:2379 \
--advertise-client-urls https://192.168.3.21:2379 \
--listen-peer-urls https://192.168.3.21:2380 \
--initial-advertise-peer-urls https://192.168.3.21:2380 \
--cert-file=/etc/kubernetes/etcd/server.pem \
--key-file=/etc/kubernetes/etcd/server-key.pem \
--client-cert-auth \
--trusted-ca-file=/etc/kubernetes/etcd/ca.pem \
--peer-cert-file=/etc/kubernetes/etcd/peer.pem \
--peer-key-file=/etc/kubernetes/etcd/peer-key.pem \
--peer-client-cert-auth \
--peer-trusted-ca-file=/etc/kubernetes/etcd/ca.pem \
--initial-cluster k8s-etcd-01=https://192.168.3.21:2380,k8s-etcd-02=https://192.168.3.22:2380,k8s-etcd-03=https://192.168.3.23:2380 \
--initial-cluster-token etcd-cluster \
--initial-cluster-state new
[Install]
WantedBy=multi-user.target

# systemctl daemon-reload
# systemctl start etcd
第一台start時，會卡住不動，因為在等第二台start

設定K8s-etcd-02

# vim /etc/systemd/system/etcd.service

[Unit]
Description=etcd
Documentation=https://github.com/coreos/etcd
Conflicts=etcd.service
Conflicts=etcd2.service

[Service]
EnvironmentFile=/etc/etcd.env
Type=notify
Restart=always
RestartSec=5s
LimitNOFILE=40000
TimeoutStartSec=0

ExecStart=/usr/local/bin/etcd \
--name k8s-etcd-02 \
--data-dir /var/lib/etcd \
--listen-client-urls https://192.168.3.22:2379 \
--advertise-client-urls https://192.168.3.22:2379 \
--listen-peer-urls https://192.168.3.22:2380 \
--initial-advertise-peer-urls https://192.168.3.22:2380 \
--cert-file=/etc/kubernetes/etcd/server.pem \
--key-file=/etc/kubernetes/etcd/server-key.pem \
--client-cert-auth \
--trusted-ca-file=/etc/kubernetes/etcd/ca.pem \
--peer-cert-file=/etc/kubernetes/etcd/peer.pem \
--peer-key-file=/etc/kubernetes/etcd/peer-key.pem \
--peer-client-cert-auth \
--peer-trusted-ca-file=/etc/kubernetes/etcd/ca.pem \
--initial-cluster k8s-etcd-01=https://192.168.3.21:2380,k8s-etcd-02=https://192.168.3.22:2380,k8s-etcd-03=https://192.168.3.23:2380 \
--initial-cluster-token etcd-cluster \
--initial-cluster-state new
[Install]
WantedBy=multi-user.target

# systemctl daemon-reload
# systemctl start etcd
# systemctl status etcd

設定K8s-etcd-03

# vim /etc/systemd/system/etcd.service

[Unit]
Description=etcd
Documentation=https://github.com/coreos/etcd
Conflicts=etcd.service
Conflicts=etcd2.service

[Service]
EnvironmentFile=/etc/etcd.env
Type=notify
Restart=always
RestartSec=5s
LimitNOFILE=40000
TimeoutStartSec=0

ExecStart=/usr/local/bin/etcd \
--name k8s-etcd-02 \
--data-dir /var/lib/etcd \
--listen-client-urls https://192.168.3.23:2379 \
--advertise-client-urls https://192.168.3.23:2379 \
--listen-peer-urls https://192.168.3.23:2380 \
--initial-advertise-peer-urls https://192.168.3.23:2380 \
--cert-file=/etc/kubernetes/etcd/server.pem \
--key-file=/etc/kubernetes/etcd/server-key.pem \
--client-cert-auth \
--trusted-ca-file=/etc/kubernetes/etcd/ca.pem \
--peer-cert-file=/etc/kubernetes/etcd/peer.pem \
--peer-key-file=/etc/kubernetes/etcd/peer-key.pem \
--peer-client-cert-auth \
--peer-trusted-ca-file=/etc/kubernetes/etcd/ca.pem \
--initial-cluster k8s-etcd-01=https://192.168.3.21:2380,k8s-etcd-02=https://192.168.3.22:2380,k8s-etcd-03=https://192.168.3.23:2380 \
--initial-cluster-token etcd-cluster \
--initial-cluster-state new
[Install]
WantedBy=multi-user.target

# systemctl daemon-reload
# systemctl start etcd
# systemctl status etcd

製作看etcd cluster狀態

# cat  > /root/etc-cluster-list <<EOF
PRIVATE_IP=\$(ip addr show eth0 | grep -Po 'inet \K[\d.]+')
etcdctl --endpoints=https://\${PRIVATE_IP}:2379  \
--ca-file=/etc/kubernetes/etcd/ca.pem  \
--cert-file=/etc/kubernetes/etcd/client.pem  \
--key-file=/etc/kubernetes/etcd/client-key.pem cluster-health
EOF

# chmod 755 /root/etc-cluster-list   
# ./etc-cluster-list 
member 4ffe8d328fb0d962 is healthy: got healthy result from https://192.168.3.21:2379
member cf954255bd8aa390 is healthy: got healthy result from https://192.168.3.22:2379
member f15f66968af37a67 is healthy: got healthy result from https://192.168.3.22:2379
cluster is healthy

參考文件: https://kubernetes.io/docs/setup/independent/high-availability/

END


圖片
  直播研討會
圖片
{{ item.channelVendor }} {{ item.webinarstarted }} |
{{ formatDate(item.duration) }}
直播中

尚未有邦友留言

立即登入留言
iThome鐵人賽
參賽組數
1123 組
團體組數
52 組
累計文章數
23096 篇
完賽人數
656 人
iT+看影片追技術 看影片追技術 看更多
熱門tag 看更多
15th鐵人賽 13th鐵人賽 14th鐵人賽 12th鐵人賽 11th鐵人賽 鐵人賽 2019鐵人賽 2018鐵人賽 javascript 2017鐵人賽 python windows php c# windows server linux css 程式設計 react vue.js
熱門問題
熱門回答
熱門文章
IT邦幫忙