4 from cStringIO import StringIO
11 from teuthology import misc as teuthology
12 from teuthology import contextutil
14 from teuthology.orchestra import run
15 from teuthology.config import config as teuth_config
17 log = logging.getLogger(__name__)
20 DEFAULT_IMAGE_URL = 'http://download.ceph.com/qa/ubuntu-12.04.qcow2'
21 DEFAULT_IMAGE_SIZE = 10240 # in megabytes
23 DEFAULT_MEM = 4096 # in megabytes
25 def create_images(ctx, config, managers):
26 for client, client_config in config.iteritems():
27 disks = client_config.get('disks', DEFAULT_NUM_DISKS)
28 if not isinstance(disks, list):
29 disks = [{} for n in range(int(disks))]
30 clone = client_config.get('clone', False)
31 assert disks, 'at least one rbd device must be used'
32 for i, disk in enumerate(disks[1:]):
35 'image_name': '{client}.{num}'.format(client=client,
37 'image_format': 2 if clone else 1,
38 'image_size': (disk or {}).get('image_size',
43 lambda create_config=create_config:
44 rbd.create_image(ctx=ctx, config=create_config)
47 def create_clones(ctx, config, managers):
48 for client, client_config in config.iteritems():
49 clone = client_config.get('clone', False)
51 num_disks = client_config.get('disks', DEFAULT_NUM_DISKS)
52 if isinstance(num_disks, list):
53 num_disks = len(num_disks)
54 for i in xrange(num_disks):
58 '{client}.{num}-clone'.format(client=client, num=i),
60 '{client}.{num}'.format(client=client, num=i),
64 lambda create_config=create_config:
65 rbd.clone_image(ctx=ctx, config=create_config)
68 @contextlib.contextmanager
69 def create_dirs(ctx, config):
71 Handle directory creation and cleanup
73 testdir = teuthology.get_testdir(ctx)
74 for client, client_config in config.iteritems():
75 assert 'test' in client_config, 'You must specify a test to run'
76 (remote,) = ctx.cluster.only(client).remotes.keys()
79 'install', '-d', '-m0755', '--',
80 '{tdir}/qemu'.format(tdir=testdir),
81 '{tdir}/archive/qemu'.format(tdir=testdir),
87 for client, client_config in config.iteritems():
88 assert 'test' in client_config, 'You must specify a test to run'
89 (remote,) = ctx.cluster.only(client).remotes.keys()
92 'rmdir', '{tdir}/qemu'.format(tdir=testdir), run.Raw('||'), 'true',
96 @contextlib.contextmanager
97 def generate_iso(ctx, config):
98 """Execute system commands to generate iso"""
99 log.info('generating iso...')
100 testdir = teuthology.get_testdir(ctx)
102 # use ctx.config instead of config, because config has been
103 # through teuthology.replace_all_with_clients()
104 refspec = ctx.config.get('branch')
106 refspec = ctx.config.get('tag')
108 refspec = ctx.config.get('sha1')
112 # hack: the git_url is always ceph-ci or ceph
113 git_url = teuth_config.get_ceph_git_url()
114 repo_name = 'ceph.git'
115 if git_url.count('ceph-ci'):
116 repo_name = 'ceph-ci.git'
118 for client, client_config in config.iteritems():
119 assert 'test' in client_config, 'You must specify a test to run'
120 test_url = client_config['test'].format(repo=repo_name, branch=refspec)
121 (remote,) = ctx.cluster.only(client).remotes.keys()
122 src_dir = os.path.dirname(__file__)
123 userdata_path = os.path.join(testdir, 'qemu', 'userdata.' + client)
124 metadata_path = os.path.join(testdir, 'qemu', 'metadata.' + client)
126 with file(os.path.join(src_dir, 'userdata_setup.yaml'), 'rb') as f:
127 test_setup = ''.join(f.readlines())
128 # configuring the commands to setup the nfs mount
129 mnt_dir = "/export/{client}".format(client=client)
130 test_setup = test_setup.format(
134 with file(os.path.join(src_dir, 'userdata_teardown.yaml'), 'rb') as f:
135 test_teardown = ''.join(f.readlines())
137 user_data = test_setup
138 if client_config.get('type', 'filesystem') == 'filesystem':
139 num_disks = client_config.get('disks', DEFAULT_NUM_DISKS)
140 if isinstance(num_disks, list):
141 num_disks = len(num_disks)
142 for i in xrange(1, num_disks):
143 dev_letter = chr(ord('a') + i)
147 mkdir /mnt/test_{dev_letter}
148 mkfs -t xfs /dev/vd{dev_letter}
149 mount -t xfs /dev/vd{dev_letter} /mnt/test_{dev_letter}
150 """.format(dev_letter=dev_letter)
155 test -d /etc/ceph || mkdir /etc/ceph
156 cp /mnt/cdrom/ceph.* /etc/ceph/
159 cloud_config_archive = client_config.get('cloud_config_archive', [])
160 if cloud_config_archive:
161 user_data += yaml.safe_dump(cloud_config_archive, default_style='|',
162 default_flow_style=False)
164 # this may change later to pass the directories as args to the
165 # script or something. xfstests needs that.
169 test -d /mnt/test_b && cd /mnt/test_b
170 /mnt/cdrom/test.sh > /mnt/log/test.log 2>&1 && touch /mnt/log/success
173 user_data = user_data.format(
174 ceph_branch=ctx.config.get('branch'),
175 ceph_sha1=ctx.config.get('sha1'))
176 teuthology.write_file(remote, userdata_path, StringIO(user_data))
178 with file(os.path.join(src_dir, 'metadata.yaml'), 'rb') as f:
179 teuthology.write_file(remote, metadata_path, f)
181 test_file = '{tdir}/qemu/{client}.test.sh'.format(tdir=testdir, client=client)
183 log.info('fetching test %s for %s', test_url, client)
186 'wget', '-nv', '-O', test_file,
189 'chmod', '755', test_file,
194 'genisoimage', '-quiet', '-input-charset', 'utf-8',
195 '-volid', 'cidata', '-joliet', '-rock',
196 '-o', '{tdir}/qemu/{client}.iso'.format(tdir=testdir, client=client),
198 'user-data={userdata}'.format(userdata=userdata_path),
199 'meta-data={metadata}'.format(metadata=metadata_path),
200 'ceph.conf=/etc/ceph/ceph.conf',
201 'ceph.keyring=/etc/ceph/ceph.keyring',
202 'test.sh={file}'.format(file=test_file),
208 for client in config.iterkeys():
209 (remote,) = ctx.cluster.only(client).remotes.keys()
213 '{tdir}/qemu/{client}.iso'.format(tdir=testdir, client=client),
214 os.path.join(testdir, 'qemu', 'userdata.' + client),
215 os.path.join(testdir, 'qemu', 'metadata.' + client),
216 '{tdir}/qemu/{client}.test.sh'.format(tdir=testdir, client=client),
220 @contextlib.contextmanager
221 def download_image(ctx, config):
222 """Downland base image, remove image file when done"""
223 log.info('downloading base image')
224 testdir = teuthology.get_testdir(ctx)
225 for client, client_config in config.iteritems():
226 (remote,) = ctx.cluster.only(client).remotes.keys()
227 base_file = '{tdir}/qemu/base.{client}.qcow2'.format(tdir=testdir, client=client)
228 image_url = client_config.get('image_url', DEFAULT_IMAGE_URL)
231 'wget', '-nv', '-O', base_file, image_url,
235 disks = client_config.get('disks', None)
236 if not isinstance(disks, list):
238 image_name = '{client}.0'.format(client=client)
239 image_size = (disks[0] or {}).get('image_size', DEFAULT_IMAGE_SIZE)
242 'qemu-img', 'convert', '-f', 'qcow2', '-O', 'raw',
243 base_file, 'rbd:rbd/{image_name}'.format(image_name=image_name)
249 '--size={image_size}M'.format(image_size=image_size),
256 log.debug('cleaning up base image files')
257 for client in config.iterkeys():
258 base_file = '{tdir}/qemu/base.{client}.qcow2'.format(
262 (remote,) = ctx.cluster.only(client).remotes.keys()
265 'rm', '-f', base_file,
270 def _setup_nfs_mount(remote, client, mount_dir):
272 Sets up an nfs mount on the remote that the guest can use to
273 store logs. This nfs mount is also used to touch a file
274 at the end of the test to indiciate if the test was successful
277 export_dir = "/export/{client}".format(client=client)
278 log.info("Creating the nfs export directory...")
280 'sudo', 'mkdir', '-p', export_dir,
282 log.info("Mounting the test directory...")
284 'sudo', 'mount', '--bind', mount_dir, export_dir,
286 log.info("Adding mount to /etc/exports...")
287 export = "{dir} *(rw,no_root_squash,no_subtree_check,insecure)".format(
291 'sudo', 'sed', '-i', '/^\/export\//d', "/etc/exports",
294 'echo', export, run.Raw("|"),
295 'sudo', 'tee', '-a', "/etc/exports",
297 log.info("Restarting NFS...")
298 if remote.os.package_type == "deb":
299 remote.run(args=['sudo', 'service', 'nfs-kernel-server', 'restart'])
301 remote.run(args=['sudo', 'systemctl', 'restart', 'nfs'])
304 def _teardown_nfs_mount(remote, client):
306 Tears down the nfs mount on the remote used for logging and reporting the
307 status of the tests being ran in the guest.
309 log.info("Tearing down the nfs mount for {remote}".format(remote=remote))
310 export_dir = "/export/{client}".format(client=client)
311 log.info("Stopping NFS...")
312 if remote.os.package_type == "deb":
314 'sudo', 'service', 'nfs-kernel-server', 'stop'
318 'sudo', 'systemctl', 'stop', 'nfs'
320 log.info("Unmounting exported directory...")
322 'sudo', 'umount', export_dir
324 log.info("Deleting exported directory...")
326 'sudo', 'rm', '-r', '/export'
328 log.info("Deleting export from /etc/exports...")
330 'sudo', 'sed', '-i', '$ d', '/etc/exports'
332 log.info("Starting NFS...")
333 if remote.os.package_type == "deb":
335 'sudo', 'service', 'nfs-kernel-server', 'start'
339 'sudo', 'systemctl', 'start', 'nfs'
343 @contextlib.contextmanager
344 def run_qemu(ctx, config):
345 """Setup kvm environment and start qemu"""
347 testdir = teuthology.get_testdir(ctx)
348 for client, client_config in config.iteritems():
349 (remote,) = ctx.cluster.only(client).remotes.keys()
350 log_dir = '{tdir}/archive/qemu/{client}'.format(tdir=testdir, client=client)
353 'mkdir', log_dir, run.Raw('&&'),
354 'sudo', 'modprobe', 'kvm',
358 # make an nfs mount to use for logging and to
359 # allow to test to tell teuthology the tests outcome
360 _setup_nfs_mount(remote, client, log_dir)
362 # Hack to make sure /dev/kvm permissions are set correctly
363 # See http://tracker.ceph.com/issues/17977 and
364 # https://bugzilla.redhat.com/show_bug.cgi?id=1333159
365 remote.run(args='sudo udevadm control --reload')
366 remote.run(args='sudo udevadm trigger /dev/kvm')
367 remote.run(args='ls -l /dev/kvm')
369 qemu_cmd = 'qemu-system-x86_64'
370 if remote.os.package_type == "rpm":
371 qemu_cmd = "/usr/libexec/qemu-kvm"
375 '{tdir}/archive/coverage'.format(tdir=testdir),
378 qemu_cmd, '-enable-kvm', '-nographic', '-cpu', 'host',
379 '-smp', str(client_config.get('cpus', DEFAULT_CPUS)),
380 '-m', str(client_config.get('memory', DEFAULT_MEM)),
381 # cd holding metadata for cloud-init
382 '-cdrom', '{tdir}/qemu/{client}.iso'.format(tdir=testdir, client=client),
386 ceph_config = ctx.ceph['ceph'].conf.get('global', {})
387 ceph_config.update(ctx.ceph['ceph'].conf.get('client', {}))
388 ceph_config.update(ctx.ceph['ceph'].conf.get(client, {}))
389 if ceph_config.get('rbd cache', True):
390 if ceph_config.get('rbd cache max dirty', 1) > 0:
391 cachemode = 'writeback'
393 cachemode = 'writethrough'
395 clone = client_config.get('clone', False)
396 num_disks = client_config.get('disks', DEFAULT_NUM_DISKS)
397 if isinstance(num_disks, list):
398 num_disks = len(num_disks)
399 for i in xrange(num_disks):
400 suffix = '-clone' if clone else ''
403 'file=rbd:rbd/{img}:id={id},format=raw,if=virtio,cache={cachemode}'.format(
404 img='{client}.{num}{suffix}'.format(client=client, num=i,
406 id=client[len('client.'):],
411 log.info('starting qemu...')
415 logger=log.getChild(client),
424 log.info('waiting for qemu tests to finish...')
427 log.debug('checking that qemu tests succeeded...')
428 for client in config.iterkeys():
429 (remote,) = ctx.cluster.only(client).remotes.keys()
431 # ensure we have permissions to all the logs
432 log_dir = '{tdir}/archive/qemu/{client}'.format(tdir=testdir,
436 'sudo', 'chmod', 'a+rw', '-R', log_dir
441 _teardown_nfs_mount(remote, client)
442 # check for test status
446 '{tdir}/archive/qemu/{client}/success'.format(
454 @contextlib.contextmanager
455 def task(ctx, config):
457 Run a test inside of QEMU on top of rbd. Only one test
458 is supported per client.
460 For example, you can specify which clients to run on::
466 test: http://download.ceph.com/qa/test.sh
468 test: http://download.ceph.com/qa/test2.sh
470 Or use the same settings on all clients:
476 test: http://download.ceph.com/qa/test.sh
478 For tests that don't need a filesystem, set type to block::
484 test: http://download.ceph.com/qa/test.sh
487 The test should be configured to run on /dev/vdb and later
490 If you want to run a test that uses more than one rbd image,
491 specify how many images to use::
497 test: http://download.ceph.com/qa/test.sh
507 test: http://ceph.com/qa/test.sh
513 You can set the amount of CPUs and memory the VM has (default is 1 CPU and
520 test: http://download.ceph.com/qa/test.sh
522 memory: 512 # megabytes
524 If you want to run a test against a cloned rbd image, set clone to true::
530 test: http://download.ceph.com/qa/test.sh
533 If you need to configure additional cloud-config options, set cloud_config
534 to the required data set::
540 test: http://ceph.com/qa/test.sh
541 cloud_config_archive:
550 If you need to override the default cloud image, set image_url:
556 test: http://ceph.com/qa/test.sh
557 image_url: https://cloud-images.ubuntu.com/releases/16.04/release/ubuntu-16.04-server-cloudimg-amd64-disk1.img
559 assert isinstance(config, dict), \
560 "task qemu only supports a dictionary for configuration"
562 config = teuthology.replace_all_with_clients(ctx.cluster, config)
565 create_images(ctx=ctx, config=config, managers=managers)
567 lambda: create_dirs(ctx=ctx, config=config),
568 lambda: generate_iso(ctx=ctx, config=config),
569 lambda: download_image(ctx=ctx, config=config),
571 create_clones(ctx=ctx, config=config, managers=managers)
573 lambda: run_qemu(ctx=ctx, config=config),
576 with contextutil.nested(*managers):