2 Run rgw roundtrip message tests
4 from cStringIO import StringIO
13 from teuthology import misc as teuthology
14 from teuthology import contextutil
15 from teuthology.config import config as teuth_config
16 from teuthology.orchestra import run
17 from teuthology.orchestra.connection import split_user
19 log = logging.getLogger(__name__)
22 @contextlib.contextmanager
23 def download(ctx, config):
25 Download the s3 tests from the git builder.
26 Remove downloaded s3 file upon exit.
28 The context passed in should be identical to the context
29 passed in to the main task.
31 assert isinstance(config, dict)
32 log.info('Downloading s3-tests...')
33 testdir = teuthology.get_testdir(ctx)
34 for (client, cconf) in config.iteritems():
35 branch = cconf.get('force-branch', None)
37 branch = cconf.get('branch', 'master')
38 ctx.cluster.only(client).run(
42 teuth_config.ceph_git_base_url + 's3-tests.git',
43 '{tdir}/s3-tests'.format(tdir=testdir),
49 log.info('Removing s3-tests...')
51 ctx.cluster.only(client).run(
55 '{tdir}/s3-tests'.format(tdir=testdir),
59 def _config_user(s3tests_conf, section, user):
61 Configure users for this section by stashing away keys, ids, and
64 s3tests_conf[section].setdefault('user_id', user)
65 s3tests_conf[section].setdefault('email', '{user}+test@test.test'.format(user=user))
66 s3tests_conf[section].setdefault('display_name', 'Mr. {user}'.format(user=user))
67 s3tests_conf[section].setdefault('access_key', ''.join(random.choice(string.uppercase) for i in xrange(20)))
68 s3tests_conf[section].setdefault('secret_key', base64.b64encode(os.urandom(40)))
70 @contextlib.contextmanager
71 def create_users(ctx, config):
73 Create a default s3 user.
75 assert isinstance(config, dict)
76 log.info('Creating rgw users...')
77 testdir = teuthology.get_testdir(ctx)
79 for client in config['clients']:
80 s3tests_conf = config['s3tests_conf'][client]
81 s3tests_conf.setdefault('roundtrip', {})
82 s3tests_conf['roundtrip'].setdefault('bucket', 'rttest-' + client + '-{random}-')
83 s3tests_conf['roundtrip'].setdefault('readers', 10)
84 s3tests_conf['roundtrip'].setdefault('writers', 3)
85 s3tests_conf['roundtrip'].setdefault('duration', 300)
86 s3tests_conf['roundtrip'].setdefault('files', {})
87 rtconf = s3tests_conf['roundtrip']
88 rtconf['files'].setdefault('num', 10)
89 rtconf['files'].setdefault('size', 2000)
90 rtconf['files'].setdefault('stddev', 500)
91 for section, user in [('s3', 'foo')]:
92 _config_user(s3tests_conf, section, '{user}.{client}'.format(user=user, client=client))
93 ctx.cluster.only(client).run(
97 '{tdir}/archive/coverage'.format(tdir=testdir),
101 '--uid', s3tests_conf[section]['user_id'],
102 '--display-name', s3tests_conf[section]['display_name'],
103 '--access-key', s3tests_conf[section]['access_key'],
104 '--secret', s3tests_conf[section]['secret_key'],
105 '--email', s3tests_conf[section]['email'],
111 for client in config['clients']:
112 for user in users.itervalues():
113 uid = '{user}.{client}'.format(user=user, client=client)
114 ctx.cluster.only(client).run(
118 '{tdir}/archive/coverage'.format(tdir=testdir),
127 @contextlib.contextmanager
128 def configure(ctx, config):
130 Configure the s3-tests. This includes the running of the
131 bootstrap code and the updating of local conf files.
133 assert isinstance(config, dict)
134 log.info('Configuring s3-roundtrip-tests...')
135 testdir = teuthology.get_testdir(ctx)
136 for client, properties in config['clients'].iteritems():
137 s3tests_conf = config['s3tests_conf'][client]
138 if properties is not None and 'rgw_server' in properties:
140 for target, roles in zip(ctx.config['targets'].iterkeys(), ctx.config['roles']):
141 log.info('roles: ' + str(roles))
142 log.info('target: ' + str(target))
143 if properties['rgw_server'] in roles:
144 _, host = split_user(target)
145 assert host is not None, "Invalid client specified as the rgw_server"
146 s3tests_conf['s3']['host'] = host
148 s3tests_conf['s3']['host'] = 'localhost'
150 def_conf = s3tests_conf['DEFAULT']
151 s3tests_conf['s3'].setdefault('port', def_conf['port'])
152 s3tests_conf['s3'].setdefault('is_secure', def_conf['is_secure'])
154 (remote,) = ctx.cluster.only(client).remotes.keys()
158 '{tdir}/s3-tests'.format(tdir=testdir),
165 s3=s3tests_conf['s3'],
166 roundtrip=s3tests_conf['roundtrip'],
168 yaml.safe_dump(conf, conf_fp, default_flow_style=False)
169 teuthology.write_file(
171 path='{tdir}/archive/s3roundtrip.{client}.config.yaml'.format(tdir=testdir, client=client),
172 data=conf_fp.getvalue(),
177 @contextlib.contextmanager
178 def run_tests(ctx, config):
180 Run the s3 roundtrip after everything is set up.
182 :param ctx: Context passed to task
183 :param config: specific configuration information
185 assert isinstance(config, dict)
186 testdir = teuthology.get_testdir(ctx)
187 for client, client_config in config.iteritems():
188 (remote,) = ctx.cluster.only(client).remotes.keys()
189 conf = teuthology.get_file(remote, '{tdir}/archive/s3roundtrip.{client}.config.yaml'.format(tdir=testdir, client=client))
191 '{tdir}/s3-tests/virtualenv/bin/s3tests-test-roundtrip'.format(tdir=testdir),
193 if client_config is not None and 'extra_args' in client_config:
194 args.extend(client_config['extra_args'])
196 ctx.cluster.only(client).run(
203 @contextlib.contextmanager
204 def task(ctx, config):
206 Run the s3tests-test-roundtrip suite against rgw.
208 To run all tests on all clients::
215 To restrict testing to particular clients::
220 - s3roundtrip: [client.0]
222 To run against a server on client.1::
231 To pass extra test arguments
250 To override s3 configuration
261 access_key: myaccesskey
262 secret_key: mysecretkey
265 assert config is None or isinstance(config, list) \
266 or isinstance(config, dict), \
267 "task s3tests only supports a list or dictionary for configuration"
268 all_clients = ['client.{id}'.format(id=id_)
269 for id_ in teuthology.all_roles_of_type(ctx.cluster, 'client')]
272 if isinstance(config, list):
273 config = dict.fromkeys(config)
274 clients = config.keys()
277 for client in clients:
278 if config[client] is None:
280 config[client].setdefault('s3', {})
281 config[client].setdefault('roundtrip', {})
283 s3tests_conf[client] = ({
289 'roundtrip' : config[client]['roundtrip'],
290 's3' : config[client]['s3'],
293 with contextutil.nested(
294 lambda: download(ctx=ctx, config=config),
295 lambda: create_users(ctx=ctx, config=dict(
297 s3tests_conf=s3tests_conf,
299 lambda: configure(ctx=ctx, config=dict(
301 s3tests_conf=s3tests_conf,
303 lambda: run_tests(ctx=ctx, config=config),