2 Long running fio tests on rbd mapped devices for format/features provided in config
3 Many fio parameters can be configured so that this task can be used along with thrash/power-cut tests
4 and exercise IO on full disk for all format/features
5 - This test should not be run on VM due to heavy use of resource
14 from teuthology.parallel import parallel
15 from teuthology import misc as teuthology
16 from tempfile import NamedTemporaryFile
17 from teuthology.orchestra import run
18 from teuthology.packaging import install_package, remove_package
20 log = logging.getLogger(__name__)
22 @contextlib.contextmanager
23 def task(ctx, config):
26 fio-io-size: 100g or 80% or 100m
29 features: [[layering],[striping],[layering,exclusive-lock,object-map]]
30 test-clone-io: 1 #remove this option to not run create rbd clone and not run io on clone
31 io-engine: "sync or rbd or any io-engine"
44 features: [[layering],[striping]]
47 Create rbd image + device and exercise IO for format/features provided in config file
48 Config can be per client or one config can be used for all clients, fio jobs are run in parallel for client provided
52 client_config = config['all']
53 clients = ctx.cluster.only(teuthology.is_type('client'))
54 rbd_test_dir = teuthology.get_testdir(ctx) + "/rbd_fio_test"
55 for remote,role in clients.remotes.iteritems():
56 if 'client_config' in locals():
58 p.spawn(run_fio, remote, client_config, rbd_test_dir)
60 for client_config in config:
61 if client_config in role:
63 p.spawn(run_fio, remote, config[client_config], rbd_test_dir)
68 def get_ioengine_package_name(ioengine, remote):
69 system_type = teuthology.get_system_type(remote)
71 return 'librbd1-devel' if system_type == 'rpm' else 'librbd-dev'
72 elif ioengine == 'libaio':
73 return 'libaio-devel' if system_type == 'rpm' else 'libaio-dev'
78 def run_rbd_map(remote, image, iodepth):
79 iodepth = max(iodepth, 128) # RBD_QUEUE_DEPTH_DEFAULT
80 out = StringIO.StringIO()
81 remote.run(args=['sudo', 'rbd', 'map', '-o', 'queue_depth={}'.format(iodepth), image], stdout=out)
82 dev = out.getvalue().rstrip('\n')
83 teuthology.sudo_write_file(
85 '/sys/block/{}/queue/nr_requests'.format(os.path.basename(dev)),
90 def run_fio(remote, config, rbd_test_dir):
92 create fio config file with options based on above config
93 get the fio from github, generate binary, and use it to run on
94 the generated fio config file
96 fio_config=NamedTemporaryFile(prefix='fio_rbd_', dir='/tmp/', delete=False)
97 fio_config.write('[global]\n')
98 if config.get('io-engine'):
99 ioengine=config['io-engine']
100 fio_config.write('ioengine={ioe}\n'.format(ioe=ioengine))
102 fio_config.write('ioengine=sync\n')
105 fio_config.write('bs={bs}\n'.format(bs=bs))
107 fio_config.write('bs=4k\n')
108 iodepth = config.get('io-depth', 2)
109 fio_config.write('iodepth={iod}\n'.format(iod=iodepth))
110 if config.get('fio-io-size'):
111 size=config['fio-io-size']
112 fio_config.write('size={size}\n'.format(size=size))
114 fio_config.write('size=100m\n')
116 fio_config.write('time_based\n')
117 if config.get('runtime'):
118 runtime=config['runtime']
119 fio_config.write('runtime={runtime}\n'.format(runtime=runtime))
121 fio_config.write('runtime=1800\n')
122 fio_config.write('allow_file_create=0\n')
124 if config.get('image_size'):
125 image_size=config['image_size']
128 features=[['layering'],['striping'],['exclusive-lock','object-map']]
130 if config.get('formats'):
131 formats=config['formats']
132 if config.get('features'):
133 features=config['features']
134 if config.get('fio-version'):
135 fio_version=config['fio-version']
137 # handle package required for ioengine, if any
139 ioengine_pkg = get_ioengine_package_name(ioengine, remote)
141 install_package(ioengine_pkg, remote)
143 fio_config.write('norandommap\n')
144 if ioengine == 'rbd':
145 fio_config.write('clientname=admin\n')
146 fio_config.write('pool=rbd\n')
147 fio_config.write('invalidate=0\n')
148 elif ioengine == 'libaio':
149 fio_config.write('direct=1\n')
151 for feature in features:
152 log.info("Creating rbd images on {sn}".format(sn=sn))
153 feature_name = '-'.join(feature)
154 rbd_name = 'i{i}f{f}{sn}'.format(i=frmt,f=feature_name,sn=sn)
155 rbd_snap_name = 'i{i}f{f}{sn}@i{i}f{f}{sn}Snap'.format(i=frmt,f=feature_name,sn=sn)
156 rbd_clone_name = 'i{i}f{f}{sn}Clone'.format(i=frmt,f=feature_name,sn=sn)
157 create_args=['rbd', 'create',
158 '--size', '{size}'.format(size=image_size),
160 '--image-format', '{f}'.format(f=frmt)]
161 map(lambda x: create_args.extend(['--image-feature', x]), feature)
162 remote.run(args=create_args)
163 remote.run(args=['rbd', 'info', rbd_name])
164 if ioengine != 'rbd':
165 rbd_dev = run_rbd_map(remote, rbd_name, iodepth)
166 if config.get('test-clone-io'):
167 log.info("Testing clones using fio")
168 remote.run(args=['rbd', 'snap', 'create', rbd_snap_name])
169 remote.run(args=['rbd', 'snap', 'protect', rbd_snap_name])
170 remote.run(args=['rbd', 'clone', rbd_snap_name, rbd_clone_name])
171 rbd_clone_dev = run_rbd_map(remote, rbd_clone_name, iodepth)
172 fio_config.write('[{rbd_dev}]\n'.format(rbd_dev=rbd_dev))
175 fio_config.write('rw={rw}\n'.format(rw=rw))
177 fio_config .write('rw=randrw\n')
178 fio_config.write('filename={rbd_dev}\n'.format(rbd_dev=rbd_dev))
179 if config.get('test-clone-io'):
180 fio_config.write('[{rbd_clone_dev}]\n'.format(rbd_clone_dev=rbd_clone_dev))
181 fio_config.write('rw={rw}\n'.format(rw=rw))
182 fio_config.write('filename={rbd_clone_dev}\n'.format(rbd_clone_dev=rbd_clone_dev))
184 if config.get('test-clone-io'):
185 log.info("Testing clones using fio")
186 remote.run(args=['rbd', 'snap', 'create', rbd_snap_name])
187 remote.run(args=['rbd', 'snap', 'protect', rbd_snap_name])
188 remote.run(args=['rbd', 'clone', rbd_snap_name, rbd_clone_name])
189 fio_config.write('[{img_name}]\n'.format(img_name=rbd_name))
192 fio_config.write('rw={rw}\n'.format(rw=rw))
194 fio_config.write('rw=randrw\n')
195 fio_config.write('rbdname={img_name}\n'.format(img_name=rbd_name))
196 if config.get('test-clone-io'):
197 fio_config.write('[{clone_img_name}]\n'.format(clone_img_name=rbd_clone_name))
198 fio_config.write('rw={rw}\n'.format(rw=rw))
199 fio_config.write('rbdname={clone_img_name}\n'.format(clone_img_name=rbd_clone_name))
203 remote.put_file(fio_config.name,fio_config.name)
205 log.info("Running rbd feature - fio test on {sn}".format(sn=sn))
206 fio = "https://github.com/axboe/fio/archive/fio-" + fio_version + ".tar.gz"
207 remote.run(args=['mkdir', run.Raw(rbd_test_dir),])
208 remote.run(args=['cd' , run.Raw(rbd_test_dir),
209 run.Raw(';'), 'wget' , fio , run.Raw(';'), run.Raw('tar -xvf fio*tar.gz'), run.Raw(';'),
210 run.Raw('cd fio-fio*'), 'configure', run.Raw(';') ,'make'])
211 remote.run(args=['ceph', '-s'])
212 remote.run(args=[run.Raw('{tdir}/fio-fio-{v}/fio --showcmd {f}'.format(tdir=rbd_test_dir,v=fio_version,f=fio_config.name))])
213 remote.run(args=['sudo', run.Raw('{tdir}/fio-fio-{v}/fio {f}'.format(tdir=rbd_test_dir,v=fio_version,f=fio_config.name))])
214 remote.run(args=['ceph', '-s'])
216 out=StringIO.StringIO()
217 remote.run(args=['rbd','showmapped', '--format=json'], stdout=out)
218 mapped_images = json.loads(out.getvalue())
220 log.info("Unmapping rbd images on {sn}".format(sn=sn))
221 for image in mapped_images.itervalues():
222 remote.run(args=['sudo', 'rbd', 'unmap', str(image['device'])])
223 log.info("Cleaning up fio install")
224 remote.run(args=['rm','-rf', run.Raw(rbd_test_dir)])
226 remove_package(ioengine_pkg, remote)