src/ceph/qa/tasks/swift.py

   1 """
   2 Test Swift API
   3 """
   4 from cStringIO import StringIO
   5 from configobj import ConfigObj
   6 import base64
   7 import contextlib
   8 import logging
   9 import os
  10
  11 from teuthology import misc as teuthology
  12 from teuthology import contextutil
  13 from teuthology.config import config as teuth_config
  14 from teuthology.orchestra import run
  15 from teuthology.orchestra.connection import split_user
  16
  17 log = logging.getLogger(__name__)
  18
  19
  20 @contextlib.contextmanager
  21 def download(ctx, config):
  22     """
  23     Download the Swift API.
  24     """
  25     testdir = teuthology.get_testdir(ctx)
  26     assert isinstance(config, list)
  27     log.info('Downloading swift...')
  28     for client in config:
  29         ctx.cluster.only(client).run(
  30             args=[
  31                 'git', 'clone',
  32                 teuth_config.ceph_git_base_url + 'swift.git',
  33                 '{tdir}/swift'.format(tdir=testdir),
  34                 ],
  35             )
  36     try:
  37         yield
  38     finally:
  39         log.info('Removing swift...')
  40         testdir = teuthology.get_testdir(ctx)
  41         for client in config:
  42             ctx.cluster.only(client).run(
  43                 args=[
  44                     'rm',
  45                     '-rf',
  46                     '{tdir}/swift'.format(tdir=testdir),
  47                     ],
  48                 )
  49
  50 def _config_user(testswift_conf, account, user, suffix):
  51     """
  52     Configure a swift user
  53
  54     :param account: Swift account
  55     :param user: User name
  56     :param suffix: user name and email suffixes.
  57     """
  58     testswift_conf['func_test'].setdefault('account{s}'.format(s=suffix), account)
  59     testswift_conf['func_test'].setdefault('username{s}'.format(s=suffix), user)
  60     testswift_conf['func_test'].setdefault('email{s}'.format(s=suffix), '{account}+test@test.test'.format(account=account))
  61     testswift_conf['func_test'].setdefault('display_name{s}'.format(s=suffix), 'Mr. {account} {user}'.format(account=account, user=user))
  62     testswift_conf['func_test'].setdefault('password{s}'.format(s=suffix), base64.b64encode(os.urandom(40)))
  63
  64 @contextlib.contextmanager
  65 def create_users(ctx, config):
  66     """
  67     Create rgw users to interact with the swift interface.
  68     """
  69     assert isinstance(config, dict)
  70     log.info('Creating rgw users...')
  71     testdir = teuthology.get_testdir(ctx)
  72     users = {'': 'foo', '2': 'bar'}
  73     for client in config['clients']:
  74         cluster_name, daemon_type, client_id = teuthology.split_role(client)
  75         testswift_conf = config['testswift_conf'][client]
  76         for suffix, user in users.iteritems():
  77             _config_user(testswift_conf, '{user}.{client}'.format(user=user, client=client), user, suffix)
  78             ctx.cluster.only(client).run(
  79                 args=[
  80                     'adjust-ulimits',
  81                     'ceph-coverage',
  82                     '{tdir}/archive/coverage'.format(tdir=testdir),
  83                     'radosgw-admin',
  84                     '-n', client,
  85                     '--cluster', cluster_name,
  86                     'user', 'create',
  87                     '--subuser', '{account}:{user}'.format(account=testswift_conf['func_test']['account{s}'.format(s=suffix)],user=user),
  88                     '--display-name', testswift_conf['func_test']['display_name{s}'.format(s=suffix)],
  89                     '--secret', testswift_conf['func_test']['password{s}'.format(s=suffix)],
  90                     '--email', testswift_conf['func_test']['email{s}'.format(s=suffix)],
  91                     '--key-type', 'swift',
  92                     '--access', 'full',
  93                 ],
  94             )
  95     try:
  96         yield
  97     finally:
  98         for client in config['clients']:
  99             for user in users.itervalues():
 100                 uid = '{user}.{client}'.format(user=user, client=client)
 101                 cluster_name, daemon_type, client_id = teuthology.split_role(client)
 102                 ctx.cluster.only(client).run(
 103                     args=[
 104                         'adjust-ulimits',
 105                         'ceph-coverage',
 106                         '{tdir}/archive/coverage'.format(tdir=testdir),
 107                         'radosgw-admin',
 108                         '-n', client,
 109                         '--cluster', cluster_name,
 110                         'user', 'rm',
 111                         '--uid', uid,
 112                         '--purge-data',
 113                         ],
 114                     )
 115
 116 @contextlib.contextmanager
 117 def configure(ctx, config):
 118     """
 119     Configure rgw and Swift
 120     """
 121     assert isinstance(config, dict)
 122     log.info('Configuring testswift...')
 123     testdir = teuthology.get_testdir(ctx)
 124     for client, properties in config['clients'].iteritems():
 125         log.info('client={c}'.format(c=client))
 126         log.info('config={c}'.format(c=config))
 127         testswift_conf = config['testswift_conf'][client]
 128         if properties is not None and 'rgw_server' in properties:
 129             host = None
 130             for target, roles in zip(ctx.config['targets'].iterkeys(), ctx.config['roles']):
 131                 log.info('roles: ' + str(roles))
 132                 log.info('target: ' + str(target))
 133                 if properties['rgw_server'] in roles:
 134                     _, host = split_user(target)
 135             assert host is not None, "Invalid client specified as the rgw_server"
 136             testswift_conf['func_test']['auth_host'] = host
 137         else:
 138             testswift_conf['func_test']['auth_host'] = 'localhost'
 139
 140         log.info(client)
 141         (remote,) = ctx.cluster.only(client).remotes.keys()
 142         remote.run(
 143             args=[
 144                 'cd',
 145                 '{tdir}/swift'.format(tdir=testdir),
 146                 run.Raw('&&'),
 147                 './bootstrap',
 148                 ],
 149             )
 150         conf_fp = StringIO()
 151         testswift_conf.write(conf_fp)
 152         teuthology.write_file(
 153             remote=remote,
 154             path='{tdir}/archive/testswift.{client}.conf'.format(tdir=testdir, client=client),
 155             data=conf_fp.getvalue(),
 156             )
 157     yield
 158
 159
 160 @contextlib.contextmanager
 161 def run_tests(ctx, config):
 162     """
 163     Run an individual Swift test.
 164     """
 165     assert isinstance(config, dict)
 166     testdir = teuthology.get_testdir(ctx)
 167     for client, client_config in config.iteritems():
 168         args = [
 169                 'SWIFT_TEST_CONFIG_FILE={tdir}/archive/testswift.{client}.conf'.format(tdir=testdir, client=client),
 170                 '{tdir}/swift/virtualenv/bin/nosetests'.format(tdir=testdir),
 171                 '-w',
 172                 '{tdir}/swift/test/functional'.format(tdir=testdir),
 173                 '-v',
 174                 '-a', '!fails_on_rgw',
 175                 ]
 176         if client_config is not None and 'extra_args' in client_config:
 177             args.extend(client_config['extra_args'])
 178
 179         ctx.cluster.only(client).run(
 180             args=args,
 181             )
 182     yield
 183
 184 @contextlib.contextmanager
 185 def task(ctx, config):
 186     """
 187     Run the testswift suite against rgw.
 188
 189     To run all tests on all clients::
 190
 191         tasks:
 192         - ceph:
 193         - rgw:
 194         - testswift:
 195
 196     To restrict testing to particular clients::
 197
 198         tasks:
 199         - ceph:
 200         - rgw: [client.0]
 201         - testswift: [client.0]
 202
 203     To run against a server on client.1::
 204
 205         tasks:
 206         - ceph:
 207         - rgw: [client.1]
 208         - testswift:
 209             client.0:
 210               rgw_server: client.1
 211
 212     To pass extra arguments to nose (e.g. to run a certain test)::
 213
 214         tasks:
 215         - ceph:
 216         - rgw: [client.0]
 217         - testswift:
 218             client.0:
 219               extra_args: ['test.functional.tests:TestFileUTF8', '-m', 'testCopy']
 220             client.1:
 221               extra_args: ['--exclude', 'TestFile']
 222     """
 223     assert config is None or isinstance(config, list) \
 224         or isinstance(config, dict), \
 225         "task testswift only supports a list or dictionary for configuration"
 226     all_clients = ['client.{id}'.format(id=id_)
 227                    for id_ in teuthology.all_roles_of_type(ctx.cluster, 'client')]
 228     if config is None:
 229         config = all_clients
 230     if isinstance(config, list):
 231         config = dict.fromkeys(config)
 232     clients = config.keys()
 233
 234     log.info('clients={c}'.format(c=clients))
 235
 236     testswift_conf = {}
 237     for client in clients:
 238         testswift_conf[client] = ConfigObj(
 239                 indent_type='',
 240                 infile={
 241                     'func_test':
 242                         {
 243                         'auth_port'      : 7280,
 244                         'auth_ssl' : 'no',
 245                         'auth_prefix' : '/auth/',
 246                         },
 247                     }
 248                 )
 249
 250     with contextutil.nested(
 251         lambda: download(ctx=ctx, config=clients),
 252         lambda: create_users(ctx=ctx, config=dict(
 253                 clients=clients,
 254                 testswift_conf=testswift_conf,
 255                 )),
 256         lambda: configure(ctx=ctx, config=dict(
 257                 clients=config,
 258                 testswift_conf=testswift_conf,
 259                 )),
 260         lambda: run_tests(ctx=ctx, config=config),
 261         ):
 262         pass
 263     yield