etcd會紀錄你pod、service deployments 的那些資訊,etcd死了,你master活著也沒用。(etcd不用安裝kubelet)
IP | hostname |
---|---|
192.168.3.21 | k8s-etcd-01 |
192.168.3.22 | k8s-etcd-02 |
192.168.3.23 | k8s-etcd-03 |
# echo "net.ipv4.ip_forward = 1" >> /etc/sysctl.conf ; sysctl -p
# curl -o /usr/local/bin/cfssl https://pkg.cfssl.org/R1.2/cfssl_linux-amd64
# curl -o /usr/local/bin/cfssljson https://pkg.cfssl.org/R1.2/cfssljson_linux-amd64
# chmod +x /usr/local/bin/cfssl*
# echo "192.168.3.21 k8s-etcd-01" >> /etc/hosts
# echo "192.168.3.22 k8s-etcd-02" >> /etc/hosts
# echo "192.168.3.23 k8s-etcd-03" >> /etc/hosts
# mkdir -p /etc/kubernetes/etcd ; cd /etc/kubernetes/etcd
# cat >ca-config.json <<EOF
{
"signing": {
"default": {
"expiry": "43800h"
},
"profiles": {
"server": {
"expiry": "43800h",
"usages": [
"signing",
"key encipherment",
"server auth",
"client auth"
]
},
"client": {
"expiry": "43800h",
"usages": [
"signing",
"key encipherment",
"client auth"
]
},
"peer": {
"expiry": "43800h",
"usages": [
"signing",
"key encipherment",
"server auth",
"client auth"
]
}
}
}
}
EOF
# cat >ca-csr.json <<EOF
{
"CN": "etcd",
"key": {
"algo": "rsa",
"size": 2048
}
}
EOF
# cfssl gencert -initca ca-csr.json | cfssljson -bare ca -
cat > client.json <<EOF
{
"CN": "client",
"key": {
"algo": "ecdsa",
"size": 256
}
}
EOF
# cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=client client.json | cfssljson -bare client
# mkdir -p /etc/kubernetes/etcd ; cd /etc/kubernetes/etcd
# scp 192.168.3.21:/etc/kubernetes/etcd/ca.pem /etc/kubernetes/etcd
# scp 192.168.3.21:/etc/kubernetes/etcd/ca-key.pem /etc/kubernetes/etcd
# scp 192.168.3.21:/etc/kubernetes/etcd/client.pem /etc/kubernetes/etcd
# scp 192.168.3.21:/etc/kubernetes/etcd/client-key.pem /etc/kubernetes/etcd
# scp 192.168.3.21:/etc/kubernetes/etcd/ca-config.json /etc/kubernetes/etcd
# cd /etc/kubernetes/etcd
# export PEER_NAME=$(hostname)
# export PRIVATE_IP=$(ip addr show eth0 | grep -Po 'inet \K[\d.]+')
注意網卡 因為我用eth0跟其他etcd server做溝通
# cfssl print-defaults csr > config.json
# sed -i '0,/CN/{s/example\.net/'"$PEER_NAME"'/}' config.json
# sed -i 's/www\.example\.net/'"$PRIVATE_IP"'/' config.json
# sed -i 's/example\.net/'"$PEER_NAME"'/' config.json
# cat config.json
{
"CN": "K8S-etcd-01",
"hosts": [
"K8S-etcd-1",
"192.168.3.21"
],
"key": {
"algo": "ecdsa",
"size": 256
},
"names": [
{
"C": "US",
"L": "CA",
"ST": "San Francisco"
}
]
}
# cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=server config.json | cfssljson -bare server
# cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=peer config.json | cfssljson -bare peer
會建立檔案:peer.pem, peer-key.pem, server.pem, server-key.pem
# ETCD_VERSION="v3.3.9"
# curl -sSL https://github.com/coreos/etcd/releases/download/${ETCD_VERSION}/etcd-${ETCD_VERSION}-linux-amd64.tar.gz | tar -xzv --strip-components=1 -C /usr/local/bin/
# touch /etc/etcd.env
# export PEER_NAME=$(hostname)
# export PRIVATE_IP=$(ip addr show eth0 | grep -Po 'inet \K[\d.]+')
# echo "PEER_NAME=${PEER_NAME}" >> /etc/etcd.env
# echo "PRIVATE_IP=${PRIVATE_IP}" >> /etc/etcd.env
# echo "ETCD_CLIENT_CERT_AUTH="true"" >> /etc/etcd.env
# echo "ETCD_AUTO_TLS="true"" >> /etc/etcd.env
# echo "ETCD_PEER_CLIENT_CERT_AUTH="true"" >> /etc/etcd.env
# echo "ETCD_PEER_AUTO_TLS="true"" >> /etc/etcd.env
192.168.3.21 k8s-etcd-01
# cat /etc/etcd.env
PEER_NAME=K8S-etcd-01
PRIVATE_IP=192.168.3.21
ETCD_CLIENT_CERT_AUTH="true"
ETCD_AUTO_TLS="true"
ETCD_PEER_CLIENT_CERT_AUTH="true"
ETCD_PEER_AUTO_TLS="true"
192.168.3.22 k8s-etcd-02
# cat /etc/etcd.env
PEER_NAME=K8S-etcd-02
PRIVATE_IP=192.168.3.22
ETCD_CLIENT_CERT_AUTH="true"
ETCD_AUTO_TLS="true"
ETCD_PEER_CLIENT_CERT_AUTH="true"
ETCD_PEER_AUTO_TLS="true"
192.168.3.23 k8s-etcd-03
# cat /etc/etcd.env
PEER_NAME=K8S-etcd-03
PRIVATE_IP=192.168.3.23
ETCD_CLIENT_CERT_AUTH="true"
ETCD_AUTO_TLS="true"
ETCD_PEER_CLIENT_CERT_AUTH="true"
ETCD_PEER_AUTO_TLS="true"
# vim /etc/systemd/system/etcd.service
[Unit]
Description=etcd
Documentation=https://github.com/coreos/etcd
Conflicts=etcd.service
Conflicts=etcd2.service
[Service]
EnvironmentFile=/etc/etcd.env
Type=notify
Restart=always
RestartSec=5s
LimitNOFILE=40000
TimeoutStartSec=0
ExecStart=/usr/local/bin/etcd \
--name k8s-etcd-01 \
--data-dir /var/lib/etcd \
--listen-client-urls https://192.168.3.21:2379 \
--advertise-client-urls https://192.168.3.21:2379 \
--listen-peer-urls https://192.168.3.21:2380 \
--initial-advertise-peer-urls https://192.168.3.21:2380 \
--cert-file=/etc/kubernetes/etcd/server.pem \
--key-file=/etc/kubernetes/etcd/server-key.pem \
--client-cert-auth \
--trusted-ca-file=/etc/kubernetes/etcd/ca.pem \
--peer-cert-file=/etc/kubernetes/etcd/peer.pem \
--peer-key-file=/etc/kubernetes/etcd/peer-key.pem \
--peer-client-cert-auth \
--peer-trusted-ca-file=/etc/kubernetes/etcd/ca.pem \
--initial-cluster k8s-etcd-01=https://192.168.3.21:2380,k8s-etcd-02=https://192.168.3.22:2380,k8s-etcd-03=https://192.168.3.23:2380 \
--initial-cluster-token etcd-cluster \
--initial-cluster-state new
[Install]
WantedBy=multi-user.target
# systemctl daemon-reload
# systemctl start etcd
第一台start時,會卡住不動,因為在等第二台start
# vim /etc/systemd/system/etcd.service
[Unit]
Description=etcd
Documentation=https://github.com/coreos/etcd
Conflicts=etcd.service
Conflicts=etcd2.service
[Service]
EnvironmentFile=/etc/etcd.env
Type=notify
Restart=always
RestartSec=5s
LimitNOFILE=40000
TimeoutStartSec=0
ExecStart=/usr/local/bin/etcd \
--name k8s-etcd-02 \
--data-dir /var/lib/etcd \
--listen-client-urls https://192.168.3.22:2379 \
--advertise-client-urls https://192.168.3.22:2379 \
--listen-peer-urls https://192.168.3.22:2380 \
--initial-advertise-peer-urls https://192.168.3.22:2380 \
--cert-file=/etc/kubernetes/etcd/server.pem \
--key-file=/etc/kubernetes/etcd/server-key.pem \
--client-cert-auth \
--trusted-ca-file=/etc/kubernetes/etcd/ca.pem \
--peer-cert-file=/etc/kubernetes/etcd/peer.pem \
--peer-key-file=/etc/kubernetes/etcd/peer-key.pem \
--peer-client-cert-auth \
--peer-trusted-ca-file=/etc/kubernetes/etcd/ca.pem \
--initial-cluster k8s-etcd-01=https://192.168.3.21:2380,k8s-etcd-02=https://192.168.3.22:2380,k8s-etcd-03=https://192.168.3.23:2380 \
--initial-cluster-token etcd-cluster \
--initial-cluster-state new
[Install]
WantedBy=multi-user.target
# systemctl daemon-reload
# systemctl start etcd
# systemctl status etcd
# vim /etc/systemd/system/etcd.service
[Unit]
Description=etcd
Documentation=https://github.com/coreos/etcd
Conflicts=etcd.service
Conflicts=etcd2.service
[Service]
EnvironmentFile=/etc/etcd.env
Type=notify
Restart=always
RestartSec=5s
LimitNOFILE=40000
TimeoutStartSec=0
ExecStart=/usr/local/bin/etcd \
--name k8s-etcd-02 \
--data-dir /var/lib/etcd \
--listen-client-urls https://192.168.3.23:2379 \
--advertise-client-urls https://192.168.3.23:2379 \
--listen-peer-urls https://192.168.3.23:2380 \
--initial-advertise-peer-urls https://192.168.3.23:2380 \
--cert-file=/etc/kubernetes/etcd/server.pem \
--key-file=/etc/kubernetes/etcd/server-key.pem \
--client-cert-auth \
--trusted-ca-file=/etc/kubernetes/etcd/ca.pem \
--peer-cert-file=/etc/kubernetes/etcd/peer.pem \
--peer-key-file=/etc/kubernetes/etcd/peer-key.pem \
--peer-client-cert-auth \
--peer-trusted-ca-file=/etc/kubernetes/etcd/ca.pem \
--initial-cluster k8s-etcd-01=https://192.168.3.21:2380,k8s-etcd-02=https://192.168.3.22:2380,k8s-etcd-03=https://192.168.3.23:2380 \
--initial-cluster-token etcd-cluster \
--initial-cluster-state new
[Install]
WantedBy=multi-user.target
# systemctl daemon-reload
# systemctl start etcd
# systemctl status etcd
# cat > /root/etc-cluster-list <<EOF
PRIVATE_IP=\$(ip addr show eth0 | grep -Po 'inet \K[\d.]+')
etcdctl --endpoints=https://\${PRIVATE_IP}:2379 \
--ca-file=/etc/kubernetes/etcd/ca.pem \
--cert-file=/etc/kubernetes/etcd/client.pem \
--key-file=/etc/kubernetes/etcd/client-key.pem cluster-health
EOF
# chmod 755 /root/etc-cluster-list
# ./etc-cluster-list
member 4ffe8d328fb0d962 is healthy: got healthy result from https://192.168.3.21:2379
member cf954255bd8aa390 is healthy: got healthy result from https://192.168.3.22:2379
member f15f66968af37a67 is healthy: got healthy result from https://192.168.3.22:2379
cluster is healthy
參考文件: https://kubernetes.io/docs/setup/independent/high-availability/