ceph集群搭建

#生产最好使用2个ip段

#系统
Ubuntu 20.04.3 
#192.168.10.20x集群通讯事业
#192.168.10.10x客户端和各种应用使用
192.168.10.200/192.168.10.100 ceph-deploy
192.168.10.201/192.168.10.101 ceph-mgr1-ceph-rgw1
192.168.10.202/192.168.10.102 ceph-mgr2-ceph-rgw2
192.168.10.203/192.168.10.103 ceph-mon1-ceph-mds1-node1
192.168.10.204/192.168.10.104 ceph-mon2-ceph-mds2-node2
192.168.10.205/192.168.10.105 ceph-mon3-ceph-mds3-node3

系统基础优化

#配置root远程登录
vim /etc/ssh/sshd_config
PermitRootLogin yes
UseDNS no
#重启sshd
systemctl restart sshd.service

#所有节点配置hosts
cat > /etc/hosts <<'EOF'
127.0.0.1 localhost
127.0.1.1 t

# The following lines are desirable for IPv6 capable hosts
::1     ip6-localhost ip6-loopback
fe00::0 ip6-localnet
ff00::0 ip6-mcastprefix
ff02::1 ip6-allnodes
ff02::2 ip6-allrouters

192.168.10.200 ceph-deploy1
192.168.10.201 ceph-mgr1-ceph-rgw1
192.168.10.202 ceph-mgr2-ceph-rgw2
192.168.10.203 ceph-mon1-ceph-mds1-node1
192.168.10.204 ceph-mon2-ceph-mds2-node2
192.168.10.205 ceph-mon3-ceph-mds3-node3
EOF

#配置apt源这里使用ustc源
cat >/etc/apt/sources.list<<'EOF'
deb http://mirrors.ustc.edu.cn/ubuntu/ focal main restricted universe multiverse
deb-src http://mirrors.ustc.edu.cn/ubuntu/ focal main restricted universe multiverse

deb http://mirrors.ustc.edu.cn/ubuntu/ focal-security main restricted universe multiverse
deb-src http://mirrors.ustc.edu.cn/ubuntu/ focal-security main restricted universe multiverse

deb http://mirrors.ustc.edu.cn/ubuntu/ focal-updates main restricted universe multiverse
deb-src http://mirrors.ustc.edu.cn/ubuntu/ focal-updates main restricted universe multiverse

deb http://mirrors.ustc.edu.cn/ubuntu/ focal-backports main restricted universe multiverse
deb-src http://mirrors.ustc.edu.cn/ubuntu/ focal-backports main restricted universe multiverse

## Not recommended
# deb http://mirrors.ustc.edu.cn/ubuntu/ focal-proposed main restricted universe multiverse
# deb-src http://mirrors.ustc.edu.cn/ubuntu/ focal-proposed main restricted universe multiverse
EOF
#更新镜像源和校验安装包
wget -q -O- 'http://mirrors.ustc.edu.cn/ceph/keys/release.asc' | sudo apt-key add -
echo deb http://mirrors.ustc.edu.cn/ceph/debian-pacific/ $(lsb_release -sc) main | sudo tee /etc/apt/sources.list.d/ceph.list
apt-get update
apt update


#时间同步
apt install cron -y
systemctl status cron.service  
echo "*/5 * * * * /usr/sbin/ntpdate time1.aliyun.com &> /dev/null && hwclock -w" >> /var/spool/cron/crontabs/root
/usr/sbin/ntpdate time1.aliyun.com &> /dev/null && hwclock -w
rm -rf /etc/localtime
ln -sf /usr/share/zoneinfo/Asia/Shanghai /etc/localtime
cat >> /etc/default/locale << 'EOF'
LANG=en_US.UTF-8
LC_TIME=en_DK.UTF-8
EOF

#安装常用系统命令：
apt install iproute2  ntpdate  tcpdump telnet traceroute nfs-kernel-server nfs-common  lrzsz tree  openssl libssl-dev libpcre3 libpcre3-dev zlib1g-dev gcc openssh-server iotop unzip zip  apt-transport-https ca-certificates curl software-properties-common vim-common inetutils-ping -y



#所有节点的内核配置
cat > /etc/sysctl.conf <<'EOF'
# Controls source route verification
net.ipv4.conf.default.rp_filter = 1
net.ipv4.ip_nonlocal_bind = 1
net.ipv4.ip_forward = 1
   
# Do not accept source routing
net.ipv4.conf.default.accept_source_route = 0
   
# Controls the System Request debugging functionality of the kernel
kernel.sysrq = 0
   
# Controls whether core dumps will append the PID to the core filename.
# Useful for debugging multi-threaded
applications. kernel.core_uses_pid = 1
   
# Controls the use of TCP syncookies
net.ipv4.tcp_syncookies = 1
   
# Disable netfilter on bridges.
net.bridge.bridge-nf-call-ip6tables = 0
net.bridge.bridge-nf-call-iptables = 0
net.bridge.bridge-nf-call-arptables = 0
   
# Controls the default maxmimum size of a mesage queue
kernel.msgmnb = 65536
   
# # Controls the maximum size of a message, in bytes
kernel.msgmax = 65536
   
# Controls the maximum shared segment size, in bytes
kernel.shmmax = 68719476736
   
# # Controls the maximum number of shared memory segments, in pages
kernel.shmall = 4294967296
   
# TCP kernel paramater
net.ipv4.tcp_mem = 786432 1048576 1572864
net.ipv4.tcp_rmem = 4096        87380   4194304
net.ipv4.tcp_wmem = 4096        16384   4194304 n
et.ipv4.tcp_window_scaling = 1
net.ipv4.tcp_sack = 1
   
# socket buffer
net.core.wmem_default = 8388608
net.core.rmem_default = 8388608
net.core.rmem_max = 16777216
net.core.wmem_max = 16777216
net.core.netdev_max_backlog = 262144
net.core.somaxconn = 20480
net.core.optmem_max = 81920
   
# TCP conn
net.ipv4.tcp_max_syn_backlog = 262144
net.ipv4.tcp_syn_retries = 3
net.ipv4.tcp_retries1 = 3
net.ipv4.tcp_retries2 = 15
   
# tcp conn reuse
net.ipv4.tcp_timestamps = 0
net.ipv4.tcp_tw_reuse = 0
net.ipv4.tcp_tw_recycle = 0
net.ipv4.tcp_fin_timeout = 1
   
   
net.ipv4.tcp_max_tw_buckets = 20000
net.ipv4.tcp_max_orphans = 3276800
net.ipv4.tcp_synack_retries = 1
net.ipv4.tcp_syncookies = 1
   
# keepalive conn
net.ipv4.tcp_keepalive_time = 300
net.ipv4.tcp_keepalive_intvl = 30
net.ipv4.tcp_keepalive_probes = 3
net.ipv4.ip_local_port_range = 10001    65000
   
# swap
vm.overcommit_memory = 0
vm.swappiness = 10
   
#net.ipv4.conf.eth1.rp_filter = 0
#net.ipv4.conf.lo.arp_ignore = 1
#net.ipv4.conf.lo.arp_announce = 2
#net.ipv4.conf.all.arp_ignore = 1
#net.ipv4.conf.all.arp_announce = 2
EOF

#所有节点的文件权限配置
cat > /etc/security/limits.conf <<'EOF'
root                soft    core            unlimited
root                hard    core            unlimited
root                soft    nproc           1000000
root                hard    nproc           1000000
root                soft    nofile          1000000
root                hard    nofile          1000000
root                soft    memlock         32000
root                hard    memlock         32000
root                soft    msgqueue        8192000
root                hard    msgqueue        8192000
   
  
*                soft    core            unlimited
*                hard    core            unlimited
*                soft    nproc           1000000
*                hard    nproc           1000000
*                soft    nofile          1000000
*                hard    nofile          1000000
*                soft    memlock         32000
*                hard    memlock         32000
*                soft    msgqueue        8192000
*                hard    msgqueue        8192000
EOF

#所有节点安装python2.7
apt install python2.7 -y && ln -sv /usr/bin/python2.7 /usr/bin/python2


#所有节点创建ceph用户,并允许ceph 用户以 执行特权命令：
#推荐使用指定的普通用户部署和运行ceph 集群，普通用户只要能以非交互方式执行命令执行一些特权命令即可，新版的ceph-deploy 可以指定包含root的在内只要可以执行命令的用户，
#不过仍然推荐使用普通用户，比如ceph、cephuser、cephadmin 这样的用户去管理ceph 集群
#这里使用cephadmin用户
groupadd -r -g 2022 cephadmin && useradd -r -m -s /bin/bash -u 2022 -g 2022 cephadmin && echo cephadmin:qwe123 | chpasswd

#允许用户以特权身份执行命令
echo "cephadmin ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers

关闭swap
swapoff -a
  
vim /etc/fstab
# /etc/fstab: static file system information.
#
# Use 'blkid' to print the universally unique identifier for a
# device; this may be used with UUID= as a more robust way to name devices
# that works even if disks are added and removed. See fstab(5).
#
# <file system> <mount point>   <type>  <options>       <dump>  <pass>
# / was on /dev/sda2 during curtin installation
/dev/disk/by-uuid/d70a7e92-2d0d-4014-a9a1-4cd95db5e242 / xfs defaults 0 0
#/swap.img      none    swap    sw      0       0

#快速配置主机名
hostname ceph-deploy && sed -i "s#t#ceph-deploy#g" /etc/hostname
hostname ceph-mgr1-ceph-rgw1 && sed -i "s#t#ceph-mgr1-ceph-rgw1#g" /etc/hostname
hostname ceph-mgr2-ceph-rgw2 && sed -i "s#t#ceph-mgr2-ceph-rgw2#g" /etc/hostname
hostname ceph-mon1-ceph-mds1-node1 && sed -i "s#t#ceph-mon1-ceph-mds1-node1#g" /etc/hostname
hostname ceph-mon2-ceph-mds2-node2 && sed -i "s#t#ceph-mon2-ceph-mds2-node2#g" /etc/hostname
hostname ceph-mon3-ceph-mds3-node3 && sed -i "s#t#ceph-mon3-ceph-mds3-node3#g" /etc/hostname

#快速配置ip
sed -i "s#200#201#g" /etc/netplan/00-installer-config.yaml && sed -i "s#100#101#g" /etc/netplan/00-installer-config.yaml && netplan apply && ip a
sed -i "s#200#202#g" /etc/netplan/00-installer-config.yaml && sed -i "s#100#102#g" /etc/netplan/00-installer-config.yaml && netplan apply && ip a
sed -i "s#200#203#g" /etc/netplan/00-installer-config.yaml && sed -i "s#100#103#g" /etc/netplan/00-installer-config.yaml && netplan apply && ip a
sed -i "s#200#204#g" /etc/netplan/00-installer-config.yaml && sed -i "s#100#104#g" /etc/netplan/00-installer-config.yaml && netplan apply && ip a
sed -i "s#200#205#g" /etc/netplan/00-installer-config.yaml && sed -i "s#100#105#g" /etc/netplan/00-installer-config.yaml && netplan apply && ip a

开始安装集群

#ceph-deploy执行切换到cephadmin创建key
cephadmin@ceph-deploy:~$ ssh-keygen
Generating public/private rsa key pair.
Enter file in which to save the key (/home/cephadmin/.ssh/id_rsa): 
Created directory '/home/cephadmin/.ssh'.
Enter passphrase (empty for no passphrase): 
Enter same passphrase again: 
Your identification has been saved in /home/cephadmin/.ssh/id_rsa
Your public key has been saved in /home/cephadmin/.ssh/id_rsa.pub
The key fingerprint is:
SHA256:kkB54FFimEU/mCw0DxOm6SFwCjjRMQVq+/b3zDOdH5I cephadmin@ceph-deploy
The key's randomart image is:
+---[RSA 3072]----+
|=o%XX+.          |
|=O+@.*.          |
|*+. B.o          |
|+ o. . o         |
| o    o S        |
|  .    .    .    |
|   o      .E..   |
|  . .  .oo o. .  |
|     .. .+o ..   |
+----[SHA256]-----+



#安装sshpass
cephadmin@ceph-deploy:~$ sudo apt install sshpass


#ceph-deploy节点使用cephadmin用户分发密钥脚本
cat >ssh-scp.sh<<'EOF'
#!/bin/bash
#目标主机列表
IP="
192.168.10.201
192.168.10.202
192.168.10.203
192.168.10.204
192.168.10.205
"
for node in ${IP};do
    sshpass -p "qwe123" ssh-copy-id -i /home/cephadmin/.ssh/id_rsa.pub -o StrictHostKeyChecking=no cephadmin@${node} &> /dev/null
   if [ $? -eq 0 ];then
     echo "${node} 秘钥copy完成"
   else
     echo "${node} 秘钥copy失败"
   fi
done
EOF
  
#在ceph-deploy节点部署ceph-deploy工具包
#查看可以安装的版本
sudo apt-cache madison ceph-deploy

#执行安装命令
sudo apt install ceph-deploy=2.0.1-0ubuntu1.1


#初始化mon节点
#保存当前集群的初始化配置信息
mkdir ceph-cluster  && cd ceph-cluster/

#初始化
#--cluster-network  内部ceph服务器使用的，集群的管理，数据同步使用的网络#192.168.10.20x集群通讯事业
#--public-network   客户端使用的网络，就是应用程序使用的#192.168.10.10x客户端和各种应用使用
ceph-deploy new --cluster-network 192.168.10.0/24 --public-network 192.168.10.0/24  ceph-mon1-ceph-mds1-node1
ceph-deploy new --cluster-network 192.168.10.0/24 --public-network 192.168.10.0/24  ceph-mon2-ceph-mds2-node2
ceph-deploy new --cluster-network 192.168.10.0/24 --public-network 192.168.10.0/24  ceph-mon3-ceph-mds3-node3

#初始化ceph-node节点，执行安装软件:ceph ceph-osd ceph-mds ceph-mon radosgw
--no-adjust-repos：不推送repo仓库
--nogpgcheck：不进行包的校验
ceph-deploy install --no-adjust-repos --nogpgcheck ceph-mon1-ceph-mds1-node1 ceph-mon2-ceph-mds2-node2 ceph-mon3-ceph-mds3-node

#添加mon节点
ceph-deploy mon add ceph-mon1-ceph-mds1-node1 
ceph-deploy mon add ceph-mon2-ceph-mds2-node2 
ceph-deploy mon add ceph-mon3-ceph-mds3-node3
#ceph-mon结果验证
cephadmin@ceph-deploy:~/ceph-cluster$ ceph -s
  cluster:
    id:     511a9f2f-c2eb-4461-9472-a2074f266bba
    health: HEALTH_OK
 
  services:
    mon: 3 daemons, quorum ceph-mon1-ceph-mds1-node1,ceph-mon2-ceph-mds2-node2,ceph-mon3-ceph-mds3-node3 (age 9s)
    mgr: ceph-mgr1-ceph-rgw1(active, since 19m), standbys: ceph-mgr2-ceph-rgw2
    osd: 12 osds: 12 up (since 3m), 12 in (since 3m)
 
  data:
    pools:   1 pools, 1 pgs
    objects: 0 objects, 0 B
    usage:   74 MiB used, 480 GiB / 480 GiB avail
    pgs:     1 active+clean
 



#在mon节点安装软件，配置mon节点并生成同步密钥
apt install -y ceph-mon

#在ceph-deploy节点初始化mon节点
ceph-deploy mon create-initial


#验证mon节点,根据生成的配置文件只添加了一台mon服务器
root@ceph-mon1-ceph-mds1-node1:~# ps aux | grep ceph-mon
ceph       73614  0.5  2.1 416112 43156 ?        Ssl  13:52   0:00 /usr/bin/ceph-mon -f --cluster ceph --id ceph-mon1-ceph-mds1-node1 --setuser ceph --setgroup ceph
root       74197  0.0  0.0   8160   656 pts/1    S+   13:53   0:00 grep --color=auto ceph-mon
#检查mon的端口此步很重要后面使用cephFS和rbd都是使用此端口，监听地址错误会导致超时
root@ceph-mon1-ceph-mds1-node1:~# ss -lnt|grep 6789
LISTEN  0       512        192.168.10.103:6789           0.0.0.0:*
root@ceph-mon2-ceph-mds2-node2:~# ss -lnt|grep 6789
LISTEN  0       512        192.168.10.104:6789           0.0.0.0:* 
root@ceph-mon3-ceph-mds3-node3:~#  ss -lnt|grep 6789
LISTEN  0       512        192.168.10.105:6789           0.0.0.0:* 


#在node节点安装ceph-common
apt install ceph-common -y

#拷贝密钥到node1，node2，node3节点
ceph-deploy admin ceph-mon1-ceph-mds1-node1 ceph-mon2-ceph-mds2-node2 ceph-mon3-ceph-mds3-node3

#在ceph-node节点对密钥授权
#认证文件的属主和属组为了安全考虑，默认设置为了root 用户和root 组，如果需要ceph用户也能执行ceph 命令，那么就需要对ceph 用户进行授权
apt-get install acl
setfacl -m u:cephadmin:rw /etc/ceph/ceph.client.admin.keyring


在ceph-mgr1和ceph-mgr2，ceph-mgr2上安装ceph-mgr包
apt install -y ceph-mgr

#初始化mgr节点
ceph-deploy mgr create ceph-mgr1-ceph-rgw1 ceph-mgr2-ceph-rgw2

验证ceph-mgr1-ceph-rgw1，ceph-mgr2-ceph-rgw2节点
root@ceph-mgr1-ceph-rgw1:~# ps aux | grep ceph-mgr


配置ceph-deploy管理ceph集群
#ceph-deploy管理ceph集群环境设置
sudo apt install -y ceph-common
ceph-deploy admin ceph-deploy
sudo setfacl -m u:cephadmin:rw /etc/ceph/ceph.client.admin.keyring

#ceph-deploy管理ceph集群信息
cephadmin@ceph-deploy:~/ceph-cluster$ ceph -s
  cluster:
    id:     511a9f2f-c2eb-4461-9472-a2074f266bba
    health: HEALTH_WARN
            mon is allowing insecure global_id reclaim
            OSD count 0 < osd_pool_default_size 3
 
  services:
    mon: 1 daemons, quorum ceph-mon1-ceph-mds1-node1 (age 18m)
    mgr: ceph-mgr1-ceph-rgw1(active, since 3m), standbys: ceph-mgr2-ceph-rgw2
    osd: 0 osds: 0 up, 0 in
 
  data:
    pools:   0 pools, 0 pgs
    objects: 0 objects, 0 B
    usage:   0 B used, 0 B / 0 B avail
    pgs
	
	
#禁用非安全模式通信
ceph config set mon auth_allow_insecure_global_id_reclaim false
cephadmin@ceph-deploy:~/ceph-cluster$ ceph -s
  cluster:
    id:     511a9f2f-c2eb-4461-9472-a2074f266bba
    health: HEALTH_WARN
            OSD count 0 < osd_pool_default_size 3
 
  services:
    mon: 1 daemons, quorum ceph-mon1-ceph-mds1-node1 (age 19m)
    mgr: ceph-mgr1-ceph-rgw1(active, since 4m), standbys: ceph-mgr2-ceph-rgw2
    osd: 0 osds: 0 up, 0 in
 
  data:
    pools:   0 pools, 0 pgs
    objects: 0 objects, 0 B
    usage:   0 B used, 0 B / 0 B avail
    pgs:  

#ceph集群个组件版本
cephadmin@ceph-deploy:~/ceph-cluster$ ceph version
ceph version 16.2.11 (3cf40e2dca667f68c6ce3ff5cd94f01e711af894) pacific (stable)
cephadmin@ceph-deploy:~/ceph-cluster$ ceph versions
{
    "mon": {
        "ceph version 16.2.11 (3cf40e2dca667f68c6ce3ff5cd94f01e711af894) pacific (stable)": 1
    },
    "mgr": {
        "ceph version 16.2.11 (3cf40e2dca667f68c6ce3ff5cd94f01e711af894) pacific (stable)": 2
    },
    "osd": {},
    "mds": {},
    "overall": {
        "ceph version 16.2.11 (3cf40e2dca667f68c6ce3ff5cd94f01e711af894) pacific (stable)": 3
    }
}


#准备osd节点
#前面执行了node节点的初始化，如下步骤，后面的操作可以不执行
#擦除磁盘之前通过deploy 节点对node 节点执行安装ceph 基本运行环境。
#在ceph-deploy上操作
ceph-deploy install --release pacific ceph-mon1-ceph-mds1-node1 ceph-mon2-ceph-mds2-node2 ceph-mon3-ceph-mds3-node3

#列出ceph node 节点磁盘
ceph-deploy disk list ceph-mon1-ceph-mds1-node1 
ceph-deploy disk list ceph-mon2-ceph-mds2-node2
ceph-deploy disk list ceph-mon3-ceph-mds3-node3

#使用ceph-deploy disk zap擦除各ceph node的ceph数据磁盘
#在ceph-deploy上操作
ceph-deploy  disk zap ceph-mon1-ceph-mds1-node1  /dev/sdb
ceph-deploy  disk zap ceph-mon1-ceph-mds1-node1  /dev/sdc
ceph-deploy  disk zap ceph-mon1-ceph-mds1-node1  /dev/sdd
ceph-deploy  disk zap ceph-mon1-ceph-mds1-node1  /dev/sde
 
ceph-deploy  disk zap ceph-mon2-ceph-mds2-node2  /dev/sdb
ceph-deploy  disk zap ceph-mon2-ceph-mds2-node2  /dev/sdc
ceph-deploy  disk zap ceph-mon2-ceph-mds2-node2  /dev/sdd
ceph-deploy  disk zap ceph-mon2-ceph-mds2-node2  /dev/sde
  
ceph-deploy  disk zap ceph-mon3-ceph-mds3-node3  /dev/sdb
ceph-deploy  disk zap ceph-mon3-ceph-mds3-node3  /dev/sdc
ceph-deploy  disk zap ceph-mon3-ceph-mds3-node3  /dev/sdd
ceph-deploy  disk zap ceph-mon3-ceph-mds3-node3  /dev/sde


添加osd
#在ceph-deploy上操作
ceph-deploy osd create ceph-mon1-ceph-mds1-node1 --data /dev/sdb
ceph-deploy osd create ceph-mon1-ceph-mds1-node1 --data /dev/sdc
ceph-deploy osd create ceph-mon1-ceph-mds1-node1 --data /dev/sdd
ceph-deploy osd create ceph-mon1-ceph-mds1-node1 --data /dev/sde
 
ceph-deploy osd create ceph-mon2-ceph-mds2-node2 --data /dev/sdb
ceph-deploy osd create ceph-mon2-ceph-mds2-node2 --data /dev/sdc
ceph-deploy osd create ceph-mon2-ceph-mds2-node2 --data /dev/sdd
ceph-deploy osd create ceph-mon2-ceph-mds2-node2 --data /dev/sde
 
ceph-deploy osd create ceph-mon3-ceph-mds3-node3 --data /dev/sdb
ceph-deploy osd create ceph-mon3-ceph-mds3-node3 --data /dev/sdc
ceph-deploy osd create ceph-mon3-ceph-mds3-node3 --data /dev/sdd
ceph-deploy osd create ceph-mon3-ceph-mds3-node3 --data /dev/sde


设置osd开机自启动
systemctl enable ceph-osd@0 ceph-osd@1 ceph-osd@2 ceph-osd@3
systemctl enable ceph-osd@4 ceph-osd@5 ceph-osd@6 ceph-osd@7
systemctl enable ceph-osd@8 ceph-osd@9 ceph-osd@10 ceph-osd@11


#部署cephfs，安装ceph-mds软件包
apt install -y ceph-mds

#在ceph-deploy上把mds服务添加到ceph集群
#在ceph-deploy上操作
ceph-deploy mds create ceph-mon1-ceph-mds1-node1
ceph-deploy mds create ceph-mon2-ceph-mds2-node2
ceph-deploy mds create ceph-mon3-ceph-mds3-node3
ceph-deploy mds create ceph-mgr2-ceph-rgw2



创建CephFS metadata和cephfs-data存储池
cephfs-metadata:存放元数据的存储池，数据有几个T的话元数据大约1-2g，所有不需要太大，存放的都是数据的名字、大小，创建时间，修改时间，生产放32个pg足够了
cephfs-data存放数据的存储池
# 创建cephfs-metadata
root@ceph-mon1-ceph-mds1:~# ceph osd pool create cephfs-metadata 32 32
pool 'cephfs-metadata' created
# 创建cephfs-data
root@ceph-mon1-ceph-mds1:~# ceph osd pool create cephfs-data 64 64
pool 'cephfs-data' created
# 创建一个叫mycephfs的cephFS
root@ceph-mon1-ceph-mds1:~# ceph fs new mycephfs cephfs-metadata cephfs-data
new fs with metadata pool 3 and data pool 4

验证CephFS
root@ceph-mon1-ceph-mds1:~# ceph fs ls
name: mycephfs, metadata pool: cephfs-metadata, data pools: [cephfs-data ]
root@ceph-mon1-ceph-mds1:~# ceph fs status mycephfs
mycephfs - 0 clients
========
RANK  STATE           MDS             ACTIVITY     DNS    INOS   DIRS   CAPS  
 0    active  ceph-mon1-ceph-mds1  Reqs:    0 /s    10     13     12      0   
      POOL         TYPE     USED  AVAIL  
cephfs-metadata  metadata  96.0k  50.6G  
  cephfs-data      data       0   50.6G  
MDS version: ceph version 16.2.10 (45fa1a083152e41a408d15505f594ec5f1b4fe17) pacific (stable)


#MDS高可用优化
cephadmin@ceph-deploy:~/ceph-cluster$ cat ceph.conf 
[global]
fsid = 511a9f2f-c2eb-4461-9472-a2074f266bba
public_network = 192.168.10.0/24
cluster_network = 192.168.10.0/24
mon_initial_members = ceph-mon1-ceph-mds1-node1
mon_host = 192.168.10.103
auth_cluster_required = cephx
auth_service_required = cephx
auth_client_required = cephx

[mds.ceph-mon2-ceph-mds2-node2]
mds_standby_for_name = ceph-mon1-ceph-mds1-node1
mds_standby_replay = true
[mds.ceph-mon3-ceph-mds3-node3]
mds_standby_for_name = ceph-mgr2-ceph-rgw2
mds_standby_replay = true


#在ceph-deploy拷贝到/etc/ceph
sudo cp -ar /home/cephadmin/ceph-cluster/* /etc/ceph/

#分发配置文件并重启mds 服务:
#分发配置文件保证各mds 服务重启有效
ceph-deploy --overwrite-conf config push ceph-mon1-ceph-mds1-node1
ceph-deploy --overwrite-conf config push ceph-mon2-ceph-mds2-node2
ceph-deploy --overwrite-conf config push ceph-mon3-ceph-mds3-node3
ceph-deploy --overwrite-conf config push ceph-mgr1-ceph-rgw1
ceph-deploy --overwrite-conf config push ceph-mgr2-ceph-rgw2
 
#重启mds节点，先重启active节点ceph-mds1和ceph-mgr2,再重启standby节点ceph-mds2和ceph-mds3
systemctl restart ceph-mds@ceph-mon1-ceph-mds1-node1.service
systemctl restart ceph-mds@ceph-mon2-ceph-mds2-node2.service
systemctl restart ceph-mds@ceph-mon3-ceph-mds3-node3.service
systemctl restart ceph-mds@ceph-mgr2-ceph-rgw2.service


#在ceph-mgr1和ceph-mgr2上安装ceph-radosgw
root@ceph-mgr1:~# apt install radosgw -y
root@ceph-mgr2:~# apt install radosgw -y


#在ceph deploy 服务器将ceph-mgr1和ceph-mgr2 初始化为radosGW 服务
#注意：如果前面有做ceph.conf的配置变动，需要将ceph.conf重新拷贝到ceph-rgw1和cpeh-rgw2上
ceph-deploy --overwrite-conf rgw create ceph-mgr1-ceph-rgw1
ceph-deploy --overwrite-conf rgw create ceph-mgr2-ceph-rgw2


#在ceph-deploy上查看集群的radosgw信息
cephadmin@ceph-deploy:~/ceph-cluster$ ceph -s
  cluster:
    id:     511a9f2f-c2eb-4461-9472-a2074f266bba
    health: HEALTH_OK
 
  services:
    mon: 3 daemons, quorum ceph-mon1-ceph-mds1-node1,ceph-mon2-ceph-mds2-node2,ceph-mon3-ceph-mds3-node3 (age 16m)
    mgr: ceph-mgr1-ceph-rgw1(active, since 36m), standbys: ceph-mgr2-ceph-rgw2
    mds: 1/1 daemons up, 3 standby
    osd: 12 osds: 12 up (since 19m), 12 in (since 19m)
    rgw: 2 daemons active (2 hosts, 1 zones)
 
  data:
    volumes: 1/1 healthy
    pools:   7 pools, 225 pgs
    objects: 211 objects, 7.2 KiB
    usage:   90 MiB used, 480 GiB / 480 GiB avail
    pgs:     225 active+clean

配置时间偏差
#时钟偏差配置
mon clock drift allowed = 0.5 时钟偏差时间配置，默认为50毫秒，0.5的话就是0.5秒钟
mon clock drift warn backoff =3 时钟偏差出现多少次出现报警配置

添加配置
cat >/home/cephadmin/ceph-cluster/ceph.conf<<'EOF'
[global]
fsid = a6f25391-9efa-4a89-a7c6-663258be6321
public_network = 192.168.10.0/24
cluster_network = 192.168.10.0/24
mon_initial_members = ceph-mon1-ceph-mds1
mon_host = 192.168.10.103
auth_cluster_required = cephx
auth_service_required = cephx
auth_client_required = cephx

#时钟偏差配置
mon clock drift allowed = 0.5
mon clock drift warn backoff =3
EOF


在同步到mon节点上 
scp ceph.conf root@192.168.10.203:/etc/ceph/
scp ceph.conf root@192.168.10.204:/etc/ceph/
scp ceph.conf root@192.168.10.205:/etc/ceph/

在mon服务器检查,也可以把配置直接拷贝到mon服务器的配置文件然后重启mon
===================================================
cephadmin@ceph-deploy:~/ceph-cluster$ cat ceph.conf
[global]
fsid = a6f25391-9efa-4a89-a7c6-663258be6321
public_network = 192.168.10.0/24
cluster_network = 192.168.10.0/24
mon_initial_members = ceph-mon1-ceph-mds1
mon_host = 192.168.10.103
auth_cluster_required = cephx
auth_service_required = cephx
auth_client_required = cephx

#时钟偏差配置
mon clock drift allowed = 0.5
mon clock drift warn backoff =3
===================================================
root@ceph-mon2-ceph-mds2:~# cat /etc/ceph/ceph.conf
mon clock drift allowed = 0.5
mon clock drift warn backoff =3
cephadmin@ceph-deploy:~/ceph-cluster$ cat ceph.conf
[global]
fsid = a6f25391-9efa-4a89-a7c6-663258be6321
public_network = 192.168.10.0/24
cluster_network = 192.168.10.0/24
mon_initial_members = ceph-mon1-ceph-mds1
mon_host = 192.168.10.103
auth_cluster_required = cephx
auth_service_required = cephx
auth_client_required = cephx

#时钟偏差配置
mon clock drift allowed = 0.5
mon clock drift warn backoff =3
===================================================
root@ceph-mon3-ceph-mds3:~# cat /etc/ceph/ceph.conf
mon clock drift allowed = 0.5
mon clock drift warn backoff =3
cephadmin@ceph-deploy:~/ceph-cluster$ cat ceph.conf
[global]
fsid = a6f25391-9efa-4a89-a7c6-663258be6321
public_network = 192.168.10.0/24
cluster_network = 192.168.10.0/24
mon_initial_members = ceph-mon1-ceph-mds1
mon_host = 192.168.10.103
auth_cluster_required = cephx
auth_service_required = cephx
auth_client_required = cephx

#时钟偏差配置
mon clock drift allowed = 0.5
mon clock drift warn backoff =3
===================================================

重启mon节点
root@ceph-mon1-ceph-mds1:~# reboot
root@ceph-mon2-ceph-mds2:~# reboot
root@ceph-mon3-ceph-mds3:~# reboot

检查