2 rgw multisite configuration routines
9 from copy import deepcopy
10 from util.rgw import rgwadmin, wait_for_radosgw
11 from util.rados import create_ec_pool, create_replicated_pool
12 from rgw_multi import multisite
13 from rgw_multi.zone_rados import RadosZone as RadosZone
15 from teuthology.orchestra import run
16 from teuthology import misc
17 from teuthology.exceptions import ConfigError
18 from teuthology.task import Task
20 log = logging.getLogger(__name__)
22 class RGWMultisite(Task):
24 Performs rgw multisite configuration to match the given realm definition.
31 List one or more zonegroup definitions. These are provided as json
32 input to `radosgw-admin zonegroup set`, with the exception of these keys:
34 * 'is_master' is passed on the command line as --master
35 * 'is_default' is passed on the command line as --default
36 * 'endpoints' given as client names are replaced with actual endpoints
39 - name: test-zonegroup
43 endpoints: [c1.client.0]
45 List each of the zones to be created in this zonegroup.
51 endpoints: [c1.client.0]
54 endpoints: [c2.client.0]
70 - name: test-zonegroup
77 endpoints: [c1.client.0]
80 endpoints: [c2.client.0]
83 def __init__(self, ctx, config):
84 super(RGWMultisite, self).__init__(ctx, config)
87 super(RGWMultisite, self).setup()
89 overrides = self.ctx.config.get('overrides', {})
90 misc.deep_merge(self.config, overrides.get('rgw-multisite', {}))
93 raise ConfigError('rgw-multisite must run after the rgw task')
94 role_endpoints = self.ctx.rgw.role_endpoints
96 # construct Clusters and Gateways for each client in the rgw task
97 clusters, gateways = extract_clusters_and_gateways(self.ctx,
100 # get the master zone and zonegroup configuration
101 mz, mzg = extract_master_zone_zonegroup(self.config['zonegroups'])
102 cluster1 = cluster_for_zone(clusters, mz)
104 # create the realm and period on the master zone's cluster
105 log.info('creating realm..')
106 realm = create_realm(cluster1, self.config['realm'])
107 period = realm.current_period
109 creds = gen_credentials()
111 # create the master zonegroup and its master zone
112 log.info('creating master zonegroup..')
113 master_zonegroup = create_zonegroup(cluster1, gateways, period,
115 period.master_zonegroup = master_zonegroup
117 log.info('creating master zone..')
118 master_zone = create_zone(self.ctx, cluster1, gateways, creds,
119 master_zonegroup, deepcopy(mz))
120 master_zonegroup.master_zone = master_zone
122 period.update(master_zone, commit=True)
123 restart_zone_gateways(master_zone) # restart with --rgw-zone
125 # create the admin user on the master zone
126 log.info('creating admin user..')
127 user_args = ['--display-name', 'Realm Admin', '--system']
128 user_args += creds.credential_args()
129 admin_user = multisite.User('realm-admin')
130 admin_user.create(master_zone, user_args)
132 # process 'zonegroups'
133 for zg_config in self.config['zonegroups']:
134 zones_config = zg_config.pop('zones')
137 for zone_config in zones_config:
138 # get the cluster for this zone
139 cluster = cluster_for_zone(clusters, zone_config)
141 if cluster != cluster1: # already created on master cluster
142 log.info('pulling realm configuration to %s', cluster.name)
143 realm.pull(cluster, master_zone.gateways[0], creds)
145 # use the first zone's cluster to create the zonegroup
147 if zg_config['name'] == master_zonegroup.name:
148 zonegroup = master_zonegroup
150 log.info('creating zonegroup..')
151 zonegroup = create_zonegroup(cluster, gateways,
154 if zone_config['name'] == master_zone.name:
155 # master zone was already created
158 # create the zone and commit the period
159 log.info('creating zone..')
160 zone = create_zone(self.ctx, cluster, gateways, creds,
161 zonegroup, zone_config)
162 period.update(zone, commit=True)
164 restart_zone_gateways(zone) # restart with --rgw-zone
166 # attach configuration to the ctx for other tasks
167 self.ctx.rgw_multisite = argparse.Namespace()
168 self.ctx.rgw_multisite.clusters = clusters
169 self.ctx.rgw_multisite.gateways = gateways
170 self.ctx.rgw_multisite.realm = realm
171 self.ctx.rgw_multisite.admin_user = admin_user
173 log.info('rgw multisite configuration completed')
176 del self.ctx.rgw_multisite
178 class Cluster(multisite.Cluster):
179 """ Issues 'radosgw-admin' commands with the rgwadmin() helper """
180 def __init__(self, ctx, name, client):
181 super(Cluster, self).__init__()
186 def admin(self, args = None, **kwargs):
187 """ radosgw-admin command """
189 args += ['--cluster', self.name]
190 args += ['--debug-rgw', '0']
191 if kwargs.pop('read_only', False):
192 args += ['--rgw-cache-enabled', 'false']
193 kwargs['decode'] = False
194 check_retcode = kwargs.pop('check_retcode', True)
195 r, s = rgwadmin(self.ctx, self.client, args, **kwargs)
200 class Gateway(multisite.Gateway):
201 """ Controls a radosgw instance using its daemon """
202 def __init__(self, role, remote, daemon, *args, **kwargs):
203 super(Gateway, self).__init__(*args, **kwargs)
208 def set_zone(self, zone):
209 """ set the zone and add its args to the daemon's command line """
210 assert self.zone is None, 'zone can only be set once'
212 # daemon.restart_with_args() would be perfect for this, except that
213 # radosgw args likely include a pipe and redirect. zone arguments at
214 # the end won't actually apply to radosgw
215 args = self.daemon.command_kwargs.get('args', [])
217 # insert zone args before the first |
218 pipe = args.index(run.Raw('|'))
219 args = args[0:pipe] + zone.zone_args() + args[pipe:]
220 except ValueError, e:
221 args += zone.zone_args()
222 self.daemon.command_kwargs['args'] = args
224 def start(self, args = None):
225 """ (re)start the daemon """
226 self.daemon.restart()
227 # wait until startup completes
228 wait_for_radosgw(self.endpoint())
231 """ stop the daemon """
234 def extract_clusters_and_gateways(ctx, role_endpoints):
235 """ create cluster and gateway instances for all of the radosgw roles """
238 for role, (host, port) in role_endpoints.iteritems():
239 cluster_name, daemon_type, client_id = misc.split_role(role)
240 # find or create the cluster by name
241 cluster = clusters.get(cluster_name)
243 clusters[cluster_name] = cluster = Cluster(ctx, cluster_name, role)
244 # create a gateway for this daemon
245 client_with_id = daemon_type + '.' + client_id # match format from rgw.py
246 daemon = ctx.daemons.get_daemon('rgw', client_with_id, cluster_name)
248 raise ConfigError('no daemon for role=%s cluster=%s type=rgw id=%s' % \
249 (role, cluster_name, client_id))
250 (remote,) = ctx.cluster.only(role).remotes.keys()
251 gateways[role] = Gateway(role, remote, daemon, host, port, cluster)
252 return clusters, gateways
254 def create_realm(cluster, config):
255 """ create a realm from configuration and initialize its first period """
256 realm = multisite.Realm(config['name'])
258 if config.get('is_default', False):
259 args += ['--default']
260 realm.create(cluster, args)
261 realm.current_period = multisite.Period(realm)
264 def extract_user_credentials(config):
265 """ extract keys from configuration """
266 return multisite.Credentials(config['access_key'], config['secret_key'])
268 def extract_master_zone(zonegroup_config):
269 """ find and return the master zone definition """
271 for zone in zonegroup_config['zones']:
272 if not zone.get('is_master', False):
275 raise ConfigError('zones %s and %s cannot both set \'is_master\'' % \
276 (master['name'], zone['name']))
278 # continue the loop so we can detect duplicates
280 raise ConfigError('one zone must set \'is_master\' in zonegroup %s' % \
281 zonegroup_config['name'])
284 def extract_master_zone_zonegroup(zonegroups_config):
285 """ find and return the master zone and zonegroup definitions """
286 master_zone, master_zonegroup = (None, None)
287 for zonegroup in zonegroups_config:
288 # verify that all zonegroups have a master zone set, even if they
289 # aren't in the master zonegroup
290 zone = extract_master_zone(zonegroup)
291 if not zonegroup.get('is_master', False):
294 raise ConfigError('zonegroups %s and %s cannot both set \'is_master\'' % \
295 (master_zonegroup['name'], zonegroup['name']))
296 master_zonegroup = zonegroup
298 # continue the loop so we can detect duplicates
299 if not master_zonegroup:
300 raise ConfigError('one zonegroup must set \'is_master\'')
301 return master_zone, master_zonegroup
303 def extract_zone_cluster_name(zone_config):
304 """ return the cluster (must be common to all zone endpoints) """
306 endpoints = zone_config.get('endpoints')
308 raise ConfigError('zone %s missing \'endpoints\' list' % \
310 for role in endpoints:
311 name, _, _ = misc.split_role(role)
314 elif cluster_name != name:
315 raise ConfigError('all zone %s endpoints must be in the same cluster' % \
319 def cluster_for_zone(clusters, zone_config):
320 """ return the cluster entry for the given zone """
321 name = extract_zone_cluster_name(zone_config)
323 return clusters[name]
325 raise ConfigError('no cluster %s found' % name)
327 def gen_access_key():
328 return ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(16))
331 return ''.join(random.choice(string.ascii_uppercase + string.ascii_lowercase + string.digits) for _ in range(32))
333 def gen_credentials():
334 return multisite.Credentials(gen_access_key(), gen_secret())
336 def extract_gateway_endpoints(gateways, endpoints_config):
337 """ return a list of gateway endpoints associated with the given roles """
339 for role in endpoints_config:
341 # replace role names with their gateway's endpoint
342 endpoints.append(gateways[role].endpoint())
344 raise ConfigError('no radosgw endpoint found for role %s' % role)
347 def is_default_arg(config):
348 return ['--default'] if config.pop('is_default', False) else []
350 def is_master_arg(config):
351 return ['--master'] if config.pop('is_master', False) else []
353 def create_zonegroup(cluster, gateways, period, config):
354 """ pass the zonegroup configuration to `zonegroup set` """
355 config.pop('zones', None) # remove 'zones' from input to `zonegroup set`
356 endpoints = config.get('endpoints')
358 # replace client names with their gateway endpoints
359 config['endpoints'] = extract_gateway_endpoints(gateways, endpoints)
360 zonegroup = multisite.ZoneGroup(config['name'], period)
361 # `zonegroup set` needs --default on command line, and 'is_master' in json
362 args = is_default_arg(config)
363 zonegroup.set(cluster, config, args)
364 period.zonegroups.append(zonegroup)
367 def create_zone(ctx, cluster, gateways, creds, zonegroup, config):
368 """ create a zone with the given configuration """
369 zone = multisite.Zone(config['name'], zonegroup, cluster)
370 zone = RadosZone(config['name'], zonegroup, cluster)
372 # collect Gateways for the zone's endpoints
373 endpoints = config.get('endpoints')
375 raise ConfigError('no \'endpoints\' for zone %s' % config['name'])
376 zone.gateways = [gateways[role] for role in endpoints]
377 for gateway in zone.gateways:
378 gateway.set_zone(zone)
380 # format the gateway endpoints
381 endpoints = [g.endpoint() for g in zone.gateways]
383 args = is_default_arg(config)
384 args += is_master_arg(config)
385 args += creds.credential_args()
387 args += ['--endpoints', ','.join(endpoints)]
388 zone.create(cluster, args)
389 zonegroup.zones.append(zone)
391 create_zone_pools(ctx, zone)
392 if ctx.rgw.compression_type:
393 configure_zone_compression(zone, ctx.rgw.compression_type)
395 zonegroup.zones_by_type.setdefault(zone.tier_type(), []).append(zone)
397 if zone.is_read_only():
398 zonegroup.ro_zones.append(zone)
400 zonegroup.rw_zones.append(zone)
404 def create_zone_pools(ctx, zone):
405 """ Create the data_pool for each placement type """
406 gateway = zone.gateways[0]
407 cluster = zone.cluster
408 for pool_config in zone.data.get('placement_pools', []):
409 pool_name = pool_config['val']['data_pool']
410 if ctx.rgw.ec_data_pool:
411 create_ec_pool(gateway.remote, pool_name, zone.name, 64,
412 ctx.rgw.erasure_code_profile, cluster.name, 'rgw')
414 create_replicated_pool(gateway.remote, pool_name, 64, cluster.name, 'rgw')
416 def configure_zone_compression(zone, compression):
417 """ Set compression type in the zone's default-placement """
418 zone.json_command(zone.cluster, 'placement', ['modify',
419 '--placement-id', 'default-placement',
420 '--compression', compression
423 def restart_zone_gateways(zone):