集群型号与规模 CPU型号：3X24 x Intel(R) Xeon(R) CPU E5-2620 v2 @ 2.10GHz (2 插槽) 内 存：110G 磁 盘：3 X 12 X 三星850 evo 500G (osd以raid0模式加入ceph)+3X1nvme 500G
可用osd个数：33个
ceph集群副本：3副本
ceph版本：ceph version 17.2.6 (810db68029296377607028a6c6da1ec06f5a2b27) quincy (stable)
内核版本 Linux 6.2.16-3-pve #1 SMP PREEMPT_DYNAMIC PVE 6.2.16-3 (2023-06-17T05:58Z)
PVE管理器版本 pve-manager/8.0.3/bbf3993334bfa916
虚机测试
#!/usr/bin/env python
# Author: Lu Xu <oliver_lew at outlook dot com>
# License: MIT
# Repo: https://github.com/OliverLew/fio-cdm
import argparse
import configparser
import json
import logging
import os
import shutil
import sys
import tempfile
from subprocess import Popen, PIPE
class Job:
def __init__(self, args):
args.target = os.path.realpath(args.target)
# Standradize the size
args.size = self.readable2byte(args.size)
self._jobs = []
self._jobfile_id, self._jobfile_name = tempfile.mkstemp(suffix='.jobfile')
self._jobfile = os.fdopen(self._jobfile_id, 'w')
self._testfile_name = '.fio_testmark'
self._blocksize = {'seq': '1m', 'rnd': '4k'}
self._config = configparser.ConfigParser(allow_no_value=True)
self._config.read_dict({
'global': {
'ioengine': 'windowsaio' if os.name == 'nt' else 'libaio',
'filename': self._testfile_name,
# escape colons, see man page 'filename'
'directory': args.target.replace(':', r'\:'),
'size': args.size,
'direct': '1',
# borrowed configuration from shell version
'runtime': '5',
'refill_buffers': None,
'norandommap': None,
'randrepeat': '0',
'allrandrepeat': '0',
'group_reporting': None
}
})
if args.zero_buffers:
self._config.read_dict({'global': {'zero_buffers': None}})
# Windows does not support pthread mutexes, suppress the warning
if os.name == 'nt':
self._config.read_dict({'global': {'thread': None}})
# TODO: fit output of different lengths
self._header = "|Name | Read(MB/s)| Write(MB/s)|"
self._sep = "|------------|------------|------------|"
self._template_row = "|{jobname}|{read:12.2f}|{write:12.2f}|"
if args.mix:
self._header += " Mix(MB/s)|"
self._sep += "------------|"
self._template_row += "{mix:12.2f}|"
def readable2byte(self, raw):
try:
num = raw.lower().rstrip("b").rstrip("i")
units = {'k': 1, 'm': 2, 'g': 3, 't': 4, 'p': 5}
return float(num[:-1]) * 1024 ** units[num[-1]] if num[-1] in units else float(num)
except ValueError:
logging.error("Unrecognised size: %s. Need: [0-9]*[KMGTP][i][B]", raw)
exit(1)
def byte2readable(self, num):
for unit in ['B', 'KiB', 'MiB', 'GiB', 'TiB']:
if abs(num) < 1024.0:
return "{:3.1f}{}".format(num, unit)
num /= 1024.0
return "{:.1f}{}".format(num, 'PiB')
def _jobname_templ(self, job, rw):
return "{}-{rw}-{}-q{}-t{}".format(job["rnd"], job["bs"], job["q"], job["t"], rw=rw)
def _displayname(self, job):
return "{}{} Q{:<2}T{:<2}".format(job["rnd"], job["bs"], job["q"], job["t"]).upper()
def create_job(self, rnd_type, queue_size, thread_num):
try:
blocksize = self._blocksize[rnd_type]
except KeyError as e:
logging.error("Job type only accepts 'seq' and 'rnd'")
raise e
job = {"rnd": rnd_type, "bs": blocksize, "q": queue_size, "t": thread_num}
self._jobs.append(job)
for rw in ["read", "write", "rw"] if args.mix else ["read", "write"]:
self._config.read_dict({
self._jobname_templ(job, rw): {
'rw': rw if rnd_type == 'seq' else 'rand' + rw,
'bs': blocksize,
'rwmixread': args.mix,
'iodepth': queue_size,
'numjobs': thread_num,
'loops': args.number,
'stonewall': None
}
})
def run(self):
# Will exit if not enough space available
space_info = self._check_disk_space()
if args.dump_jobfile:
if args.dump_jobfile == '-':
self._config.write(sys.stdout, space_around_delimiters=False)
else:
with open(args.dump_jobfile, 'w') as fp:
self._config.write(fp, space_around_delimiters=False)
exit()
else:
self._config.write(self._jobfile, space_around_delimiters=False)
self._jobfile.close()
print("tests: {}, size: {}, target: {} {}".format(
args.number,
self.byte2readable(args.size),
args.target,
space_info
))
if args.mix:
print("Mixed rw: read {:2.0f}%, write {:2.0f}%".format(args.mix, 100 - args.mix))
FIO_COMMAND = ['fio', '--output-format', 'json', self._jobfile_name]
try:
res = Popen(FIO_COMMAND, stdout=PIPE)
output, _ = res.communicate()
except KeyboardInterrupt:
logging.info('interrupted, cleaning up before exit...')
exit()
finally:
if os.path.exists(self._jobfile_name):
os.remove(self._jobfile_name)
if os.path.exists(os.path.join(args.target, self._testfile_name)):
os.remove(os.path.join(args.target, self._testfile_name))
if res.returncode == 0:
fio_output = json.loads(output)
else:
exit()
# rearrange to make jobname as keys
info = {job.pop("jobname"): job for job in fio_output["jobs"]}
logging.debug(info)
self._print(info)
def _printline(self, info, job, name, f):
read = f(info.get(self._jobname_templ(job, "read"))['read'])
write = f(info.get(self._jobname_templ(job, "write"))['write'])
if args.mix:
mixr = f(info.get(self._jobname_templ(job, "rw"))['read'])
mixw = f(info.get(self._jobname_templ(job, "rw"))['write'])
mix = (mixw * (100 - args.mix) + mixr * args.mix) / 100.0
else:
mix = None
print(self._template_row.format(jobname=name, read=read, write=write, mix=mix))
def _info_get_bw_megabytes(self, info):
# Unit of I/O speed, use MB/s(10^6) instead of MiB/s(2^30).
return info.get('bw_bytes', info['bw'] * 1e3) / 1e6
def _print(self, info):
print(self._header)
print(self._sep)
for job in self._jobs:
self._printline(info, job, self._displayname(job), self._info_get_bw_megabytes)
if job['rnd'] == 'rnd' and args.extra_info:
self._printline(info, job, ". IOPS ", lambda d: d['iops'])
self._printline(info, job, ". latency us", lambda d: d['lat_ns']['mean'] / 1000)
def _check_disk_space(self):
if hasattr(shutil, "disk_usage"): # Python3
du = shutil.disk_usage(args.target)
free = du.free
used = du.used
total = du.total
elif hasattr(os, "statvfs"): # Python2 + Unix-like
stat = os.statvfs(args.target)
free = stat.f_bsize * stat.f_bavail
total = stat.f_bsize * stat.f_blocks
used = total - stat.f_bsize * stat.f_bfree
else: # Python2 + Windows? No way!
logging.warning("It's hard to get disk usage for current system and"
" python version,\nskipping available space checking.")
return "(disk usage not avallable)"
if free >= args.size:
return "{}/{}".format(self.byte2readable(used), self.byte2readable(total))
logging.error("Not enough space available in %s:", args.target)
logging.error("Needed: %s. Available: %s",
self.byte2readable(args.size),
self.byte2readable(free))
exit(1)
def get_parser():
parser = argparse.ArgumentParser(
description='A python script to show disk test results with fio')
parser.add_argument('target', nargs='?', default='.',
help='The path of the directory to test. '
'Default to current directory')
parser.add_argument('-0', dest='zero_buffers', action='store_true',
help='Initialize buffers with zeros. '
'Default to use random buffers.')
parser.add_argument('-a', metavar='job', dest='jobs', action="append",
help='Manually add multiple jobs. Format is '
'"seq|rnd,<queue depth>,<thread number>". '
'This overrides the preset jobs. '
'This option can be used more than once.')
parser.add_argument('-E', dest='extra_info', action='store_false',
help='Disable extra information (iops, latency) for '
'random IO tests. Default is enabled.')
parser.add_argument('-f', metavar='jobfile', dest='dump_jobfile',
help='Save jobfile and quit without running fio. '
'Use "-" to print to stdout.')
parser.add_argument('-n', metavar='number', dest='number', type=int, default=5,
help='Number of tests, default is 5.')
parser.add_argument('-s', metavar='size', dest='size', default='1G',
help='The size of file I/O. '
'It is directly passed to fio. '
'Default is 1G.')
parser.add_argument('-x', metavar='mix', dest='mix',
type=float, nargs="?", const=70, default=0,
help='Add mixed rw test. Default is disabled. '
'<mix> is read percentage. Default is 70.')
# hidden option, enable to show debug information
parser.add_argument('-g', dest='debug', action='store_true', help=argparse.SUPPRESS)
return parser
if __name__ == '__main__':
# TODO: Real-time visual feedback, with fio --debug=io? Seams hard.
# TODO: Linux: vendor and model with lsblk -o +VENDOR,MODEL or /sys/block/*/device/{vendor,model}
# TODO: Specify device instead of directory
parser = get_parser()
args = parser.parse_args()
logging.basicConfig(level=logging.DEBUG if args.debug else logging.INFO,
format="%(message)s")
fio_job = Job(args)
if args.jobs:
for job in args.jobs:
rnd_type, qd, tn = job.split(',')
fio_job.create_job(rnd_type, int(qd), int(tn))
else:
fio_job.create_job('seq', 8, 1)
fio_job.create_job('seq', 1, 1)
fio_job.create_job('rnd', 32, 16)
fio_job.create_job('rnd', 1, 1)
fio_job.run()
fio-cdm
顺序读写 8队列深度 1 并发数
[root@localhost ~]# fio -filename=./test_write-10G -direct=1 -iodepth 8 -thread -rw=write -ioengine=libaio -bs=4k -size=10G -numjobs=1 -runtime=1200 -group_reporting -name=mytest
mytest: (g=0): rw=write, bs=(R) 4096B-4096B, (W) 4096B-4096B, (T) 4096B-4096B, ioengine=libaio, iodepth=8
fio-3.7
Starting 1 thread
mytest: Laying out IO file (1 file / 10240MiB)
Jobs: 1 (f=1): [W(1)][100.0%][r=0KiB/s,w=159MiB/s][r=0,w=40.8k IOPS][eta 00m:00s]
mytest: (groupid=0, jobs=1): err= 0: pid=14326: Mon Aug 7 23:00:43 2023
write: IOPS=40.7k, BW=159MiB/s (167MB/s)(10.0GiB/64411msec)
slat (usec): min=5, max=306, avg= 8.69, stdev= 2.87
clat (usec): min=84, max=7113, avg=185.39, stdev=37.63
lat (usec): min=117, max=7122, avg=194.70, stdev=37.93
clat percentiles (usec):
| 1.00th=[ 149], 5.00th=[ 161], 10.00th=[ 165], 20.00th=[ 169],
| 30.00th=[ 174], 40.00th=[ 176], 50.00th=[ 180], 60.00th=[ 186],
| 70.00th=[ 192], 80.00th=[ 198], 90.00th=[ 210], 95.00th=[ 225],
| 99.00th=[ 269], 99.50th=[ 293], 99.90th=[ 379], 99.95th=[ 474],
| 99.99th=[ 1565]
bw ( KiB/s): min=123776, max=170408, per=100.00%, avg=162806.35, stdev=6346.37, samples=128
iops : min=30944, max=42602, avg=40701.57, stdev=1586.60, samples=128
lat (usec) : 100=0.01%, 250=98.16%, 500=1.80%, 750=0.01%, 1000=0.01%
lat (msec) : 2=0.02%, 4=0.01%, 10=0.01%
cpu : usr=9.87%, sys=52.72%, ctx=1038421, majf=0, minf=6
IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=100.0%, 16=0.0%, 32=0.0%, >=64=0.0%
submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete : 0=0.0%, 4=100.0%, 8=0.1%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
issued rwts: total=0,2621440,0,0 short=0,0,0,0 dropped=0,0,0,0
latency : target=0, window=0, percentile=100.00%, depth=8
Run status group 0 (all jobs):
WRITE: bw=159MiB/s (167MB/s), 159MiB/s-159MiB/s (167MB/s-167MB/s), io=10.0GiB (10.7GB), run=64411-64411msec
Disk stats (read/write):
vda: ios=0/2620260, merge=0/205, ticks=0/430185, in_queue=429635, util=99.86%
[root@localhost ~]#
[root@localhost ~]# fio -filename=./test_write-10G -direct=1 -iodepth 8 -thread -rw=read -ioengine=libaio -bs=4k -size=10G -numjobs=1 -runtime=1200 -group_reporting -name=mytest
mytest: (g=0): rw=read, bs=(R) 4096B-4096B, (W) 4096B-4096B, (T) 4096B-4096B, ioengine=libaio, iodepth=8
fio-3.7
Starting 1 thread
Jobs: 1 (f=1): [R(1)][100.0%][r=32.6MiB/s,w=0KiB/s][r=8336,w=0 IOPS][eta 00m:00s]
mytest: (groupid=0, jobs=1): err= 0: pid=14354: Mon Aug 7 23:07:47 2023
read: IOPS=8083, BW=31.6MiB/s (33.1MB/s)(10.0GiB/324283msec)
slat (usec): min=3, max=478, avg= 8.00, stdev= 3.84
clat (usec): min=397, max=24999, avg=978.82, stdev=355.61
lat (usec): min=408, max=25006, avg=987.44, stdev=355.69
clat percentiles (usec):
| 1.00th=[ 758], 5.00th=[ 816], 10.00th=[ 848], 20.00th=[ 881],
| 30.00th=[ 898], 40.00th=[ 914], 50.00th=[ 930], 60.00th=[ 955],
| 70.00th=[ 979], 80.00th=[ 1012], 90.00th=[ 1057], 95.00th=[ 1156],
| 99.00th=[ 2040], 99.50th=[ 2769], 99.90th=[ 3720], 99.95th=[ 5735],
| 99.99th=[18482]
bw ( KiB/s): min=28864, max=34104, per=100.00%, avg=32333.54, stdev=731.21, samples=648
iops : min= 7216, max= 8526, avg=8083.37, stdev=182.81, samples=648
lat (usec) : 500=0.01%, 750=0.82%, 1000=76.52%
lat (msec) : 2=21.60%, 4=0.98%, 10=0.04%, 20=0.02%, 50=0.01%
cpu : usr=2.61%, sys=12.45%, ctx=2456643, majf=0, minf=17
IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=100.0%, 16=0.0%, 32=0.0%, >=64=0.0%
submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete : 0=0.0%, 4=100.0%, 8=0.1%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
issued rwts: total=2621440,0,0,0 short=0,0,0,0 dropped=0,0,0,0
latency : target=0, window=0, percentile=100.00%, depth=8
Run status group 0 (all jobs):
READ: bw=31.6MiB/s (33.1MB/s), 31.6MiB/s-31.6MiB/s (33.1MB/s-33.1MB/s), io=10.0GiB (10.7GB), run=324283-324283msec
Disk stats (read/write):
vda: ios=2620420/3, merge=0/1, ticks=2556885/14, in_queue=2554998, util=99.99%
[root@localhost ~]#
标签：self,args,Ceph,job,PVE,th,RBD,fio,0.0% From： https://www.cnblogs.com/cyh00001/p/17613583.html