src/ceph/qa/tasks/rbd_fio.py

   1 """
   2  Long running fio tests on rbd mapped devices for format/features provided in config
   3  Many fio parameters can be configured so that this task can be used along with thrash/power-cut tests
   4  and exercise IO on full disk for all format/features
   5   - This test should not be run on VM due to heavy use of resource
   6
   7 """
   8 import contextlib
   9 import json
  10 import logging
  11 import os
  12 import StringIO
  13
  14 from teuthology.parallel import parallel
  15 from teuthology import misc as teuthology
  16 from tempfile import NamedTemporaryFile
  17 from teuthology.orchestra import run
  18 from teuthology.packaging import install_package, remove_package
  19
  20 log = logging.getLogger(__name__)
  21
  22 @contextlib.contextmanager
  23 def task(ctx, config):
  24     """
  25     client.0:
  26        fio-io-size: 100g or 80% or 100m
  27        fio-version: 2.2.9
  28        formats: [2]
  29        features: [[layering],[striping],[layering,exclusive-lock,object-map]]
  30        test-clone-io: 1  #remove this option to not run create rbd clone and not run io on clone
  31        io-engine: "sync or rbd or any io-engine"
  32        rw: randrw
  33     client.1:
  34        fio-io-size: 100g
  35        fio-version: 2.2.9
  36        rw: read
  37        image-size:20480
  38
  39 or
  40     all:
  41        fio-io-size: 400g
  42        rw: randrw
  43        formats: [2]
  44        features: [[layering],[striping]]
  45        io-engine: libaio
  46
  47     Create rbd image + device and exercise IO for format/features provided in config file
  48     Config can be per client or one config can be used for all clients, fio jobs are run in parallel for client provided
  49
  50     """
  51     if config.get('all'):
  52         client_config = config['all']
  53     clients = ctx.cluster.only(teuthology.is_type('client'))
  54     rbd_test_dir = teuthology.get_testdir(ctx) + "/rbd_fio_test"
  55     for remote,role in clients.remotes.iteritems():
  56         if 'client_config' in locals():
  57            with parallel() as p:
  58                p.spawn(run_fio, remote, client_config, rbd_test_dir)
  59         else:
  60            for client_config in config:
  61               if client_config in role:
  62                  with parallel() as p:
  63                      p.spawn(run_fio, remote, config[client_config], rbd_test_dir)
  64
  65     yield
  66
  67
  68 def get_ioengine_package_name(ioengine, remote):
  69     system_type = teuthology.get_system_type(remote)
  70     if ioengine == 'rbd':
  71         return 'librbd1-devel' if system_type == 'rpm' else 'librbd-dev'
  72     elif ioengine == 'libaio':
  73         return 'libaio-devel' if system_type == 'rpm' else 'libaio-dev'
  74     else:
  75         return None
  76
  77
  78 def run_rbd_map(remote, image, iodepth):
  79     iodepth = max(iodepth, 128)  # RBD_QUEUE_DEPTH_DEFAULT
  80     out = StringIO.StringIO()
  81     remote.run(args=['sudo', 'rbd', 'map', '-o', 'queue_depth={}'.format(iodepth), image], stdout=out)
  82     dev = out.getvalue().rstrip('\n')
  83     teuthology.sudo_write_file(
  84         remote,
  85         '/sys/block/{}/queue/nr_requests'.format(os.path.basename(dev)),
  86         str(iodepth))
  87     return dev
  88
  89
  90 def run_fio(remote, config, rbd_test_dir):
  91     """
  92     create fio config file with options based on above config
  93     get the fio from github, generate binary, and use it to run on
  94     the generated fio config file
  95     """
  96     fio_config=NamedTemporaryFile(prefix='fio_rbd_', dir='/tmp/', delete=False)
  97     fio_config.write('[global]\n')
  98     if config.get('io-engine'):
  99         ioengine=config['io-engine']
 100         fio_config.write('ioengine={ioe}\n'.format(ioe=ioengine))
 101     else:
 102         fio_config.write('ioengine=sync\n')
 103     if config.get('bs'):
 104         bs=config['bs']
 105         fio_config.write('bs={bs}\n'.format(bs=bs))
 106     else:
 107         fio_config.write('bs=4k\n')
 108     iodepth = config.get('io-depth', 2)
 109     fio_config.write('iodepth={iod}\n'.format(iod=iodepth))
 110     if config.get('fio-io-size'):
 111         size=config['fio-io-size']
 112         fio_config.write('size={size}\n'.format(size=size))
 113     else:
 114         fio_config.write('size=100m\n')
 115
 116     fio_config.write('time_based\n')
 117     if config.get('runtime'):
 118         runtime=config['runtime']
 119         fio_config.write('runtime={runtime}\n'.format(runtime=runtime))
 120     else:
 121         fio_config.write('runtime=1800\n')
 122     fio_config.write('allow_file_create=0\n')
 123     image_size=10240
 124     if config.get('image_size'):
 125         image_size=config['image_size']
 126
 127     formats=[1,2]
 128     features=[['layering'],['striping'],['exclusive-lock','object-map']]
 129     fio_version='2.21'
 130     if config.get('formats'):
 131         formats=config['formats']
 132     if config.get('features'):
 133         features=config['features']
 134     if config.get('fio-version'):
 135         fio_version=config['fio-version']
 136
 137     # handle package required for ioengine, if any
 138     sn=remote.shortname
 139     ioengine_pkg = get_ioengine_package_name(ioengine, remote)
 140     if ioengine_pkg:
 141         install_package(ioengine_pkg, remote)
 142
 143     fio_config.write('norandommap\n')
 144     if ioengine == 'rbd':
 145         fio_config.write('clientname=admin\n')
 146         fio_config.write('pool=rbd\n')
 147         fio_config.write('invalidate=0\n')
 148     elif ioengine == 'libaio':
 149         fio_config.write('direct=1\n')
 150     for frmt in formats:
 151         for feature in features:
 152            log.info("Creating rbd images on {sn}".format(sn=sn))
 153            feature_name = '-'.join(feature)
 154            rbd_name = 'i{i}f{f}{sn}'.format(i=frmt,f=feature_name,sn=sn)
 155            rbd_snap_name = 'i{i}f{f}{sn}@i{i}f{f}{sn}Snap'.format(i=frmt,f=feature_name,sn=sn)
 156            rbd_clone_name = 'i{i}f{f}{sn}Clone'.format(i=frmt,f=feature_name,sn=sn)
 157            create_args=['rbd', 'create',
 158                         '--size', '{size}'.format(size=image_size),
 159                         '--image', rbd_name,
 160                         '--image-format', '{f}'.format(f=frmt)]
 161            map(lambda x: create_args.extend(['--image-feature', x]), feature)
 162            remote.run(args=create_args)
 163            remote.run(args=['rbd', 'info', rbd_name])
 164            if ioengine != 'rbd':
 165                rbd_dev = run_rbd_map(remote, rbd_name, iodepth)
 166                if config.get('test-clone-io'):
 167                     log.info("Testing clones using fio")
 168                     remote.run(args=['rbd', 'snap', 'create', rbd_snap_name])
 169                     remote.run(args=['rbd', 'snap', 'protect', rbd_snap_name])
 170                     remote.run(args=['rbd', 'clone', rbd_snap_name, rbd_clone_name])
 171                     rbd_clone_dev = run_rbd_map(remote, rbd_clone_name, iodepth)
 172                fio_config.write('[{rbd_dev}]\n'.format(rbd_dev=rbd_dev))
 173                if config.get('rw'):
 174                    rw=config['rw']
 175                    fio_config.write('rw={rw}\n'.format(rw=rw))
 176                else:
 177                    fio_config .write('rw=randrw\n')
 178                fio_config.write('filename={rbd_dev}\n'.format(rbd_dev=rbd_dev))
 179                if config.get('test-clone-io'):
 180                    fio_config.write('[{rbd_clone_dev}]\n'.format(rbd_clone_dev=rbd_clone_dev))
 181                    fio_config.write('rw={rw}\n'.format(rw=rw))
 182                    fio_config.write('filename={rbd_clone_dev}\n'.format(rbd_clone_dev=rbd_clone_dev))
 183            else:
 184                if config.get('test-clone-io'):
 185                     log.info("Testing clones using fio")
 186                     remote.run(args=['rbd', 'snap', 'create', rbd_snap_name])
 187                     remote.run(args=['rbd', 'snap', 'protect', rbd_snap_name])
 188                     remote.run(args=['rbd', 'clone', rbd_snap_name, rbd_clone_name])
 189                fio_config.write('[{img_name}]\n'.format(img_name=rbd_name))
 190                if config.get('rw'):
 191                    rw=config['rw']
 192                    fio_config.write('rw={rw}\n'.format(rw=rw))
 193                else:
 194                    fio_config.write('rw=randrw\n')
 195                fio_config.write('rbdname={img_name}\n'.format(img_name=rbd_name))
 196                if config.get('test-clone-io'):
 197                    fio_config.write('[{clone_img_name}]\n'.format(clone_img_name=rbd_clone_name))
 198                    fio_config.write('rw={rw}\n'.format(rw=rw))
 199                    fio_config.write('rbdname={clone_img_name}\n'.format(clone_img_name=rbd_clone_name))
 200
 201
 202     fio_config.close()
 203     remote.put_file(fio_config.name,fio_config.name)
 204     try:
 205         log.info("Running rbd feature - fio test on {sn}".format(sn=sn))
 206         fio = "https://github.com/axboe/fio/archive/fio-" + fio_version + ".tar.gz"
 207         remote.run(args=['mkdir', run.Raw(rbd_test_dir),])
 208         remote.run(args=['cd' , run.Raw(rbd_test_dir),
 209                          run.Raw(';'), 'wget' , fio , run.Raw(';'), run.Raw('tar -xvf fio*tar.gz'), run.Raw(';'),
 210                          run.Raw('cd fio-fio*'), 'configure', run.Raw(';') ,'make'])
 211         remote.run(args=['ceph', '-s'])
 212         remote.run(args=[run.Raw('{tdir}/fio-fio-{v}/fio --showcmd {f}'.format(tdir=rbd_test_dir,v=fio_version,f=fio_config.name))])
 213         remote.run(args=['sudo', run.Raw('{tdir}/fio-fio-{v}/fio {f}'.format(tdir=rbd_test_dir,v=fio_version,f=fio_config.name))])
 214         remote.run(args=['ceph', '-s'])
 215     finally:
 216         out=StringIO.StringIO()
 217         remote.run(args=['rbd','showmapped', '--format=json'], stdout=out)
 218         mapped_images = json.loads(out.getvalue())
 219         if mapped_images:
 220             log.info("Unmapping rbd images on {sn}".format(sn=sn))
 221             for image in mapped_images.itervalues():
 222                 remote.run(args=['sudo', 'rbd', 'unmap', str(image['device'])])
 223         log.info("Cleaning up fio install")
 224         remote.run(args=['rm','-rf', run.Raw(rbd_test_dir)])
 225         if ioengine_pkg:
 226             remove_package(ioengine_pkg, remote)