一、环境说明

  • 主机规划

    序号 IP 主机名 操作系统
    1 192.168.233.12 gateway01 Ubuntu 22.04
    2 192.168.233.13 gateway02 Ubuntu 22.04
    3 192.168.233.14 gateway03 Ubuntu 22.04
  • etcd证书目录

/etc/etcd/cert
  • etcd数据目录
/data/etcd

  • 高可用的三种形式
    • 静态配置:
      预先已经知道 etcd 集群有哪些节点,在启动时通过 --initial-cluster 参数直接指定好etcd的各个节点地址。
    • etcd动态发现:
      通过已经搭建好的etcd来辅助搭建新的etcd集群,已有的etcd集群作为数据交互点,然后在扩展新的集群时,实现通过已有集群进行服务发现的机制,如官方提供的:discovery.etcd.io
    • DNS动态发现:
      通过DNS查询方式获取其它节点地址信息。

二、部署

  1. 配置/etc/hosts(节点1、2、3执行)
echo "192.168.233.12 gateway01" | sudo tee -a /etc/hosts && \
echo "192.168.233.13 gateway02" | sudo tee -a /etc/hosts && \
echo "192.168.233.14 gateway03" | sudo tee -a /etc/hosts
  1. 安装cfssl证书工具(节点1执行)
wget https://github.com/cloudflare/cfssl/releases/download/v1.6.5/cfssl_1.6.5_linux_amd64  -O   /usr/local/bin/cfssl && \
wget https://github.com/cloudflare/cfssl/releases/download/v1.6.5/cfssljson_1.6.5_linux_amd64 -O  /usr/local/bin/cfssljson && \
wget https://github.com/cloudflare/cfssl/releases/download/v1.6.5/cfssl-certinfo_1.6.5_linux_amd64   -O /usr/local/bin/cfssl-certinfo && \
chmod +x  /usr/local/bin/cfssl*
  1. 创建工作目录 (节点1、2、3执行)
mkdir -p /etc/etcd/cert
  1. 关闭防火墙 (节点1、2、3执行)
systemctl disable ufw && \
systemctl stop ufw
  1. 生成默认的CA配置 (节点1执行)
cfssl print-defaults  config > /etc/etcd/cert/ca-config.json
  1. 写入具体的证书配置 (节点1执行)
cat <<EOF > /etc/etcd/cert/ca-config.json
{
    "signing": {
        "default": {
            "expiry": "87600h"
        },
        "profiles": {
            "etcd": {
                "expiry": "87600h",
                "usages": [
                    "signing",
                    "key encipherment",
                    "server auth",
                    "client auth"
                ]
            }
        }
    }
}
EOF
  1. 生成默认的csr请求文件 (节点1执行)
cfssl  print-defaults csr  > /etc/etcd/cert/ca-csr.json
  1. 写入具体的配置 (节点1执行)
cat <<EOF > /etc/etcd/cert/ca-csr.json
{
    "CN": "etcd",
    "key": {
        "algo": "rsa",
        "size": 2048
    },
    "names": [
        {
            "C": "CN",
            "ST": "BeiJing",
            "L": "BeiJing",
            "O": "etcd",
            "OU": "system"
        }
    ]
}
EOF
  1. 创建ca证书 (节点1执行)
cfssl gencert -initca /etc/etcd/cert/ca-csr.json | cfssljson -bare /etc/etcd/cert/etcd-ca
  1. 配置etcd请求csr文件 (节点1执行,hosts:所有etcd节点的地址列表和本地回环地址)
cfssl  print-defaults csr  > /etc/etcd/cert/etcd-csr.json && \
cat <<EOF > /etc/etcd/cert/etcd-csr.json
{
    "CN": "etcd",
    "hosts": [
        "127.0.0.1",
        "192.168.233.12",
        "192.168.233.13",
        "192.168.233.14"
    ],
    "key": {
        "algo": "rsa",
        "size": 2048
    },
    "names": [
        {
            "C": "CN",
            "ST": "BeiJing",
            "L": "BeiJing",
            "O": "etcd",
            "OU": "system"
        }
    ]
}
EOF
  1. 生成etcd证书 (节点1执行)
cfssl  gencert  -ca=/etc/etcd/cert/etcd-ca.pem  \
 -ca-key=/etc/etcd/cert/etcd-ca-key.pem \
 -config=/etc/etcd/cert/ca-config.json  -profile=etcd  \
 /etc/etcd/cert/etcd-csr.json  |  cfssljson -bare  /etc/etcd/cert/etcd
  1. 证书分发至节点2 (节点1执行)
scp /etc/etcd/cert/{etcd-ca.pem,etcd.pem,etcd-key.pem} 192.168.233.13:/etc/etcd/cert/
  1. 证书分发至节点3 (节点1执行)
scp /etc/etcd/cert/{etcd-ca.pem,etcd.pem,etcd-key.pem} 192.168.233.14:/etc/etcd/cert/
  1. 下载安装etcd(节点1、2、3执行)
wget -c https://github.com/etcd-io/etcd/releases/download/v3.5.12/etcd-v3.5.12-linux-amd64.tar.gz -k && \
tar -xf etcd-v3.5.12-linux-amd64.tar.gz && \
cp -p  etcd-v3.5.12-linux-amd64/{etcd,etcdctl,etcdutl} /usr/local/bin/
  1. 确认安装版本(节点1、2、3执行)
etcd -version
  1. 创建节点1的etcd配置文件(节点1执行)
cat <<EOF > /etc/etcd/etcd.conf
# Member(成员):
ETCD_NAME="gateway01"
ETCD_DATA_DIR="/data/etcd"
ETCD_SNAPSHOT_COUNT="5000"
ETCD_HEARTBEAT_INTERVAL="100"
ETCD_ELECTION_TIMEOUT="500"
ETCD_LISTEN_PEER_URLS="https://192.168.233.12:2380"
ETCD_LISTEN_CLIENT_URLS="https://192.168.233.12:2379,https://127.0.0.1:2379"
# Clustering(集群):
ETCD_INITIAL_ADVERTISE_PEER_URLS="https://192.168.233.12:2380"
ETCD_INITIAL_CLUSTER_STATE="new"
ETCD_INITIAL_CLUSTER="gateway01=https://192.168.233.12:2380,gateway02=https://192.168.233.13:2380,gateway03=https://192.168.233.14:2380"
ETCD_INITIAL_CLUSTER_TOKEN="etcd-cluster-apisix"
ETCD_ADVERTISE_CLIENT_URLS="https://192.168.233.12:2379"
# Security(安全):
ETCD_CLIENT_CERT_AUTH="true"
ETCD_CERT_FILE="/etc/etcd/cert/etcd.pem"
ETCD_KEY_FILE="/etc/etcd/cert/etcd-key.pem"
ETCD_TRUSTED_CA_FILE="/etc/etcd/cert/etcd-ca.pem"
ETCD_PEER_CLIENT_CERT_AUTH="true"
ETCD_PEER_CERT_FILE="/etc/etcd/cert/etcd.pem"
ETCD_PEER_KEY_FILE="/etc/etcd/cert/etcd-key.pem"
ETCD_PEER_TRUSTED_CA_FILE="/etc/etcd/cert/etcd-ca.pem"
EOF
  1. 创建节点2的etcd配置文件(节点2执行)
cat <<EOF > /etc/etcd/etcd.conf
# Member(成员):
ETCD_NAME="gateway02"
ETCD_DATA_DIR="/data/etcd"
ETCD_SNAPSHOT_COUNT="5000"
ETCD_HEARTBEAT_INTERVAL="100"
ETCD_ELECTION_TIMEOUT="500"
ETCD_LISTEN_PEER_URLS="https://192.168.233.13:2380"
ETCD_LISTEN_CLIENT_URLS="https://192.168.233.13:2379,https://127.0.0.1:2379"
# Clustering(集群):
ETCD_INITIAL_ADVERTISE_PEER_URLS="https://192.168.233.13:2380"
ETCD_INITIAL_CLUSTER_STATE="new"
ETCD_INITIAL_CLUSTER="gateway01=https://192.168.233.12:2380,gateway02=https://192.168.233.13:2380,gateway03=https://192.168.233.14:2380"
ETCD_INITIAL_CLUSTER_TOKEN="etcd-cluster-apisix"
ETCD_ADVERTISE_CLIENT_URLS="https://192.168.233.13:2379"
# Security(安全):
ETCD_CLIENT_CERT_AUTH="true"
ETCD_CERT_FILE="/etc/etcd/cert/etcd.pem"
ETCD_KEY_FILE="/etc/etcd/cert/etcd-key.pem"
ETCD_TRUSTED_CA_FILE="/etc/etcd/cert/etcd-ca.pem"
ETCD_PEER_CLIENT_CERT_AUTH="true"
ETCD_PEER_CERT_FILE="/etc/etcd/cert/etcd.pem"
ETCD_PEER_KEY_FILE="/etc/etcd/cert/etcd-key.pem"
ETCD_PEER_TRUSTED_CA_FILE="/etc/etcd/cert/etcd-ca.pem"
EOF
  1. 创建节点3的etcd配置文件(节点3执行)
cat <<EOF > /etc/etcd/etcd.conf
# Member(成员):
ETCD_NAME="gateway03"
ETCD_DATA_DIR="/data/etcd"
ETCD_SNAPSHOT_COUNT="5000"
ETCD_HEARTBEAT_INTERVAL="100"
ETCD_ELECTION_TIMEOUT="500"
ETCD_LISTEN_PEER_URLS="https://192.168.233.14:2380"
ETCD_LISTEN_CLIENT_URLS="https://192.168.233.14:2379,https://127.0.0.1:2379"
# Clustering(集群):
ETCD_INITIAL_ADVERTISE_PEER_URLS="https://192.168.233.14:2380"
ETCD_INITIAL_CLUSTER_STATE="new"
ETCD_INITIAL_CLUSTER="gateway01=https://192.168.233.12:2380,gateway02=https://192.168.233.13:2380,gateway03=https://192.168.233.14:2380"
ETCD_INITIAL_CLUSTER_TOKEN="etcd-cluster-apisix"
ETCD_ADVERTISE_CLIENT_URLS="https://192.168.233.14:2379"
# Security(安全):
ETCD_CLIENT_CERT_AUTH="true"
ETCD_CERT_FILE="/etc/etcd/cert/etcd.pem"
ETCD_KEY_FILE="/etc/etcd/cert/etcd-key.pem"
ETCD_TRUSTED_CA_FILE="/etc/etcd/cert/etcd-ca.pem"
ETCD_PEER_CLIENT_CERT_AUTH="true"
ETCD_PEER_CERT_FILE="/etc/etcd/cert/etcd.pem"
ETCD_PEER_KEY_FILE="/etc/etcd/cert/etcd-key.pem"
ETCD_PEER_TRUSTED_CA_FILE="/etc/etcd/cert/etcd-ca.pem"
EOF
  1. 创建服务启动文件 (节点1、2、3执行)
cat <<EOF > /usr/lib/systemd/system/etcd.service
[Unit]
Description=Etcd Server
After=network.target
After=network-online.target
Wants=network-online.target

[Service]
Type=notify
EnvironmentFile=/etc/etcd/etcd.conf
ExecStart=/usr/local/bin/etcd  
Restart=on-failure
RestartSec=5
LimitNOFILE=65536

[Install]
WantedBy=multi-user.target
EOF
  1. 重新加载服务配置 (节点1、2、3执行)
systemctl daemon-reload
  1. 启动etcd服务 (节点1、2、3执行)
systemctl start etcd.service
  1. 确认etcd运行状态(节点1、2、3执行)
systemctl status etcd
  1. 配置etcd自启动(节点1、2、3执行)
systemctl enable etcd.service
  1. 创建etcdctl别名,指定监听地址,和证书(节点1、2、3执行)
cat <<EOF >> ~/.bashrc
# Set etcdctl alias for etcd cluster
alias etcdctl='etcdctl --write-out=table --endpoints=https://192.168.233.12:2379,https://192.168.233.13:2379,https://192.168.233.14:2379 --cacert=/etc/etcd/cert/etcd-ca.pem --cert=/etc/etcd/cert/etcd.pem --key=/etc/etcd/cert/etcd-key.pem'
EOF
  1. 重载bashrc
source ~/.bashrc
  1. 查看集群状态
etcdctl
+-----------------------------+--------+-------------+-------+
|          ENDPOINT           | HEALTH |    TOOK     | ERROR |
+-----------------------------+--------+-------------+-------+
| https://192.168.233.12:2379 |   true | 11.546232ms |       |
| https://192.168.233.13:2379 |   true | 23.809094ms |       |
| https://192.168.233.14:2379 |   true | 38.027802ms |       |
+-----------------------------+--------+-------------+-------+