环境介绍

安装 GlusterFS(所有节点)

配置 hosts 解析

cat >> /etc/hosts <<EOF
10.1.30.30    gluster-server1
10.1.30.32    gluster-server2
EOF

所有节点安装 glusterfs

# 安装最新版本的gluster软件仓库
yum install -y centos-release-gluster
 
# 安装 glusterfs 相关软件
yum install glusterfs glusterfs-server glusterfs-cli glusterfs-geo-replication glusterfs-rdma -y
 
# 启动 gluster 服务
systemctl enable --now glusterd.service

配置可信池

# 在 gluster-server1 上执行以下命令
gluster peer probe gluster-server2
 
# 查看可信池节点列表
gluster pool list
 
# 查看节点状态
gluster peer status

部署 Heketi (gluster-server1 上安装）

下载安装 Heketi

# 下载二进制包
wget https://github.com/heketi/heketi/releases/download/v10.4.0/heketi-v10.4.0-release-10.linux.amd64.tar.gz
 
# 创建安装目录
mkdir -p /usr/local/heketi/{bin,conf,data}
 
# 解压二进制包
tar xf heketi-v10.4.0-release-10.linux.amd64.tar.gz
 
# 拷贝文件
mv heketi/{heketi,heketi-cli} /usr/local/heketi/bin/
mv heketi/heketi.json /usr/local/heketi/conf/
 
# 创建免密登录
ssh-keygen -f /usr/local/heketi/conf/heketi_key -t rsa -N ''
 
ssh-copy-id -i /usr/local/heketi/conf/heketi_key.pub deployer@gluster-server1
ssh-copy-id -i /usr/local/heketi/conf/heketi_key.pub deployer@gluster-server2
 
# 修改配置文件
## 主要修改的地方有以下几个地方
 
"use_auth": true                  # 默认是 false, 这里改为 true
"jwt"."admin"."key": "adminkey"    # 添加一个 adminkey
"jwt"."user"."key": "userkey"      # 添加一个 userkey
"glusterfs"."executor": "ssh"      # 修改 exector 模式为 ssh
 
## 修改 ssh exector 相关配置
"sshexec": {
      "keyfile": "/usr/local/heketi/conf/heketi_key",
      "user": "deployer",
      "port": "22",
      "sudo": true,    # 由于 root 用户被禁止远程登录，所以需要使用 sudo
      "fstab": "/etc/fstab",
 
## 修改 db 配置
"db": "/usr/local/heketi/data/heketi.db",
 
## 删除以下配置，ssh 模块以及 kubernetes 模块都要删除
"xfs_sw": "Optional: Specify number of data disks in the underlying RAID device.",
"xfs_su": "Optional: Specifies a stripe unit or RAID chunk size.",
"gluster_cli_timeout": "Optional: Timeout, in seconds, passed to the gluster cli invocations",
 
# 创建 systemd 服务管理文件
cat > /usr/lib/systemd/system/heketi.service <<EOF
[Unit]
Description=RESTful based volume management framework for GlusterFS
Before=network-online.target
After=network-online.target
Documentation=https://github.com/heketi/heketi
 
[Service]
Type=simple
LimitNOFILE=65536
ExecStart=/usr/local/heketi/bin/heketi --config=/usr/local/heketi/conf/heketi.json
KillMode=process
Restart=on-failure
RestartSec=5
SuccessExitStatus=15
StandardOutput=syslog
StandardError=syslog
 
[Install]
WantedBy=multi-user.target
EOF
 
# 启动 Heketi
systemctl enable --now heketi.service
 
# 查看服务状态
systemctl status heketi.service
 
# 创建命令快捷方式
ln -s /usr/local/heketi/bin/heketi-cli /usr/local/bin/

管理 GlusterFS

# 创建集群
heketi-cli cluster create \
  --server=http://10.1.30.30:8080 \
  --user=admin \
  --secret=adminkey \
  --json
 
# 输出信息如下，主机其中的 id 为集群ID
{"id":"034f10398853b67682761d318f624d33","nodes":[],"volumes":[],"block":true,"file":true,"blockvolumes":[]}
 
# 将 GlusterFS 节点作为 node 添加到 cluster
 
heketi-cli node add \
  --cluster="034f10398853b67682761d318f624d33" \
  --management-host-name="gluster-server1" \
  --storage-host-name="10.1.30.30" \
  --user=admin \
  --secret=adminkey \
  --zone=1 \
  --json
 
 
heketi-cli node add \
  --cluster="034f10398853b67682761d318f624d33" \
  --management-host-name="gluster-server2" \
  --storage-host-name="10.1.30.32" \
  --user=admin \
  --secret=adminkey \
  --zone=1 \
  --json
 
 
# 查看节点信息
heketi-cli node list --user=admin --secret=adminkey

添加 Device

# 首先确保节点上有裸磁盘（未格式化文件系统的）,如下面的 sdb
# lsblk -f
NAME            FSTYPE      LABEL           UUID                                   MOUNTPOINT
sda                                                                               
├─sda1          xfs                         b2436d95-4d1f-4ead-95a4-a845a0eb4bd1   /boot
└─sda2          LVM2_member                 vAjm5R-RDpp-qW26-ifx6-OhMx-Z6Ep-os073F
  ├─centos-root ext4                        7d244150-ffee-4c51-a0a5-c9e57bc5f594   /
  └─centos-swap swap                        f3f341ad-40f7-4ba7-ad43-c87be91a39a8   [SWAP]
sdb                                                                               
sr0             iso9660     CentOS 7 x86_64 2019-09-11-19-02-53-00
 
# 执行以下命令添加 Device
heketi-cli device add --name="/dev/sdb" --node="203f32473c504e38a8bc135dd4d1c729" --user=admin --secret=adminkey
heketi-cli device add --name="/dev/sdb" --node="2187edc77262198616f98a00587d2081" --user=admin --secret=adminkey

创建 volume

# heketi-cli volume create --size=100 --replica=2 --user=admin --secret=adminkey
Name: vol_d20e9a92d61bab31b8daaf57c069a441
Size: 100
Volume Id: d20e9a92d61bab31b8daaf57c069a441
Cluster Id: 034f10398853b67682761d318f624d33
Mount: 10.1.30.30:vol_d20e9a92d61bab31b8daaf57c069a441
Mount Options: backup-volfile-servers=10.1.30.32
Block: false
Free Size: 0
Reserved Size: 0
Block Hosting Restriction: (none)
Block Volumes: []
Durability Type: replicate
Distribute Count: 1
Replica Count: 2