src/ceph/src/test/rgw/test_multi.py

   1 import subprocess
   2 import os
   3 import random
   4 import string
   5 import argparse
   6 import sys
   7 import logging
   8 try:
   9     import configparser
  10 except ImportError:
  11     import ConfigParser as configparser
  12
  13 import nose.core
  14
  15 from rgw_multi import multisite
  16 from rgw_multi.zone_rados import RadosZone as RadosZone
  17 from rgw_multi.zone_es  import ESZone as ESZone
  18
  19 # make tests from rgw_multi.tests available to nose
  20 from rgw_multi.tests import *
  21 from rgw_multi.tests_es import *
  22
  23 mstart_path = os.getenv('MSTART_PATH')
  24 if mstart_path is None:
  25     mstart_path = os.path.normpath(os.path.dirname(os.path.realpath(__file__)) + '/../..') + '/'
  26
  27 test_path = os.path.normpath(os.path.dirname(os.path.realpath(__file__))) + '/'
  28
  29 # configure logging for the tests module
  30 log = logging.getLogger('rgw_multi.tests')
  31
  32 def bash(cmd, **kwargs):
  33     log.debug('running cmd: %s', ' '.join(cmd))
  34     check_retcode = kwargs.pop('check_retcode', True)
  35     kwargs['stdout'] = subprocess.PIPE
  36     process = subprocess.Popen(cmd, **kwargs)
  37     s = process.communicate()[0]
  38     log.debug('command returned status=%d stdout=%s', process.returncode, s.decode('utf-8'))
  39     if check_retcode:
  40         assert(process.returncode == 0)
  41     return (s, process.returncode)
  42
  43 class Cluster(multisite.Cluster):
  44     """ cluster implementation based on mstart/mrun scripts """
  45     def __init__(self, cluster_id):
  46         super(Cluster, self).__init__()
  47         self.cluster_id = cluster_id
  48         self.needs_reset = True
  49
  50     def admin(self, args = None, **kwargs):
  51         """ radosgw-admin command """
  52         cmd = [test_path + 'test-rgw-call.sh', 'call_rgw_admin', self.cluster_id]
  53         if args:
  54             cmd += args
  55         if kwargs.pop('read_only', False):
  56             cmd += ['--rgw-cache-enabled', 'false']
  57         return bash(cmd, **kwargs)
  58
  59     def start(self):
  60         cmd = [mstart_path + 'mstart.sh', self.cluster_id]
  61         if self.needs_reset:
  62             cmd += ['-n', '--mds_num', '0']
  63         bash(cmd)
  64         self.needs_reset = False
  65
  66     def stop(self):
  67         cmd = [mstart_path + 'mstop.sh', self.cluster_id]
  68         bash(cmd)
  69
  70 class Gateway(multisite.Gateway):
  71     """ gateway implementation based on mrgw/mstop scripts """
  72     def __init__(self, client_id = None, *args, **kwargs):
  73         super(Gateway, self).__init__(*args, **kwargs)
  74         self.id = client_id
  75
  76     def start(self, args = None):
  77         """ start the gateway """
  78         assert(self.cluster)
  79         cmd = [mstart_path + 'mrgw.sh', self.cluster.cluster_id, str(self.port)]
  80         if self.id:
  81             cmd += ['-i', self.id]
  82         cmd += ['--debug-rgw=20', '--debug-ms=1']
  83         if args:
  84             cmd += args
  85         bash(cmd)
  86
  87     def stop(self):
  88         """ stop the gateway """
  89         assert(self.cluster)
  90         cmd = [mstart_path + 'mstop.sh', self.cluster.cluster_id, 'radosgw', self.id]
  91         bash(cmd)
  92
  93 def gen_access_key():
  94     return ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(16))
  95
  96 def gen_secret():
  97     return ''.join(random.choice(string.ascii_uppercase + string.ascii_lowercase + string.digits) for _ in range(32))
  98
  99 def gen_credentials():
 100     return multisite.Credentials(gen_access_key(), gen_secret())
 101
 102 def cluster_name(cluster_num):
 103     return 'c' + str(cluster_num)
 104
 105 def zonegroup_name(zonegroup_num):
 106     return string.ascii_lowercase[zonegroup_num]
 107
 108 def zone_name(zonegroup_num, zone_num):
 109     return zonegroup_name(zonegroup_num) + str(zone_num + 1)
 110
 111 def gateway_port(zonegroup_num, gateway_num):
 112     return 8000 + 100 * zonegroup_num + gateway_num
 113
 114 def gateway_name(zonegroup_num, zone_num, gateway_num):
 115     return zone_name(zonegroup_num, zone_num) + '-' + str(gateway_num + 1)
 116
 117 def zone_endpoints(zonegroup_num, zone_num, gateways_per_zone):
 118     endpoints = []
 119     base = gateway_port(zonegroup_num, zone_num * gateways_per_zone)
 120     for i in range(0, gateways_per_zone):
 121         endpoints.append('http://localhost:' + str(base + i))
 122     return endpoints
 123
 124 def get_log_level(log_level):
 125     if log_level >= 20:
 126         return logging.DEBUG
 127     if log_level >= 10:
 128         return logging.INFO
 129     if log_level >= 5:
 130         return logging.WARN
 131     if log_level >= 1:
 132         return logging.ERROR
 133     return logging.CRITICAL
 134
 135 def setup_logging(log_level_console, log_file, log_level_file):
 136     if log_file:
 137         formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s')
 138         fh = logging.FileHandler(log_file)
 139         fh.setFormatter(formatter)
 140         fh.setLevel(get_log_level(log_level_file))
 141         log.addHandler(fh)
 142
 143     formatter = logging.Formatter('%(levelname)s %(message)s')
 144     ch = logging.StreamHandler()
 145     ch.setFormatter(formatter)
 146     ch.setLevel(get_log_level(log_level_console))
 147     log.addHandler(ch)
 148
 149 def init(parse_args):
 150     cfg = configparser.RawConfigParser({
 151                                          'num_zonegroups': 1,
 152                                          'num_zones': 3,
 153                                          'num_es_zones': 0,
 154                                          'gateways_per_zone': 2,
 155                                          'no_bootstrap': 'false',
 156                                          'log_level': 20,
 157                                          'log_file': None,
 158                                          'file_log_level': 20,
 159                                          'tenant': None,
 160                                          'checkpoint_retries': 60,
 161                                          'checkpoint_delay': 5,
 162                                          'reconfigure_delay': 5,
 163                                          'es_endpoint': None,
 164                                          })
 165     try:
 166         path = os.environ['RGW_MULTI_TEST_CONF']
 167     except KeyError:
 168         path = test_path + 'test_multi.conf'
 169
 170     try:
 171         with open(path) as f:
 172             cfg.readfp(f)
 173     except:
 174         print('WARNING: error reading test config. Path can be set through the RGW_MULTI_TEST_CONF env variable')
 175         pass
 176
 177     parser = argparse.ArgumentParser(
 178             description='Run rgw multi-site tests',
 179             usage='test_multi [--num-zonegroups <num>] [--num-zones <num>] [--no-bootstrap]')
 180
 181     section = 'DEFAULT'
 182     parser.add_argument('--num-zonegroups', type=int, default=cfg.getint(section, 'num_zonegroups'))
 183     parser.add_argument('--num-zones', type=int, default=cfg.getint(section, 'num_zones'))
 184     parser.add_argument('--num-es-zones', type=int, default=cfg.getint(section, 'num_es_zones'))
 185     parser.add_argument('--gateways-per-zone', type=int, default=cfg.getint(section, 'gateways_per_zone'))
 186     parser.add_argument('--no-bootstrap', action='store_true', default=cfg.getboolean(section, 'no_bootstrap'))
 187     parser.add_argument('--log-level', type=int, default=cfg.getint(section, 'log_level'))
 188     parser.add_argument('--log-file', type=str, default=cfg.get(section, 'log_file'))
 189     parser.add_argument('--file-log-level', type=int, default=cfg.getint(section, 'file_log_level'))
 190     parser.add_argument('--tenant', type=str, default=cfg.get(section, 'tenant'))
 191     parser.add_argument('--checkpoint-retries', type=int, default=cfg.getint(section, 'checkpoint_retries'))
 192     parser.add_argument('--checkpoint-delay', type=int, default=cfg.getint(section, 'checkpoint_delay'))
 193     parser.add_argument('--reconfigure-delay', type=int, default=cfg.getint(section, 'reconfigure_delay'))
 194     parser.add_argument('--es-endpoint', type=str, default=cfg.get(section, 'es_endpoint'))
 195
 196     argv = []
 197
 198     if parse_args:
 199         argv = sys.argv[1:]
 200
 201     args = parser.parse_args(argv)
 202     bootstrap = not args.no_bootstrap
 203
 204     # if num_es_zones is defined, need to have es_endpoint defined too
 205     assert(args.num_es_zones == 0 or args.es_endpoint)
 206
 207     setup_logging(args.log_level, args.log_file, args.file_log_level)
 208
 209     # start first cluster
 210     c1 = Cluster(cluster_name(1))
 211     if bootstrap:
 212         c1.start()
 213     clusters = []
 214     clusters.append(c1)
 215
 216     admin_creds = gen_credentials()
 217     admin_user = multisite.User('zone.user')
 218
 219     user_creds = gen_credentials()
 220     user = multisite.User('tester')
 221
 222     realm = multisite.Realm('r')
 223     if bootstrap:
 224         # create the realm on c1
 225         realm.create(c1)
 226     else:
 227         realm.get(c1)
 228     period = multisite.Period(realm=realm)
 229     realm.current_period = period
 230
 231     num_zones = args.num_zones + args.num_es_zones
 232
 233     for zg in range(0, args.num_zonegroups):
 234         zonegroup = multisite.ZoneGroup(zonegroup_name(zg), period)
 235         period.zonegroups.append(zonegroup)
 236
 237         is_master_zg = zg == 0
 238         if is_master_zg:
 239             period.master_zonegroup = zonegroup
 240
 241         for z in range(0, num_zones):
 242             is_master = z == 0
 243             # start a cluster, or use c1 for first zone
 244             cluster = None
 245             if is_master_zg and is_master:
 246                 cluster = c1
 247             else:
 248                 cluster = Cluster(cluster_name(len(clusters) + 1))
 249                 clusters.append(cluster)
 250                 if bootstrap:
 251                     cluster.start()
 252                     # pull realm configuration from the master's gateway
 253                     gateway = realm.meta_master_zone().gateways[0]
 254                     realm.pull(cluster, gateway, admin_creds)
 255
 256             endpoints = zone_endpoints(zg, z, args.gateways_per_zone)
 257             if is_master:
 258                 if bootstrap:
 259                     # create the zonegroup on its first zone's cluster
 260                     arg = []
 261                     if is_master_zg:
 262                         arg += ['--master']
 263                     if len(endpoints): # use master zone's endpoints
 264                         arg += ['--endpoints', ','.join(endpoints)]
 265                     zonegroup.create(cluster, arg)
 266                 else:
 267                     zonegroup.get(cluster)
 268
 269             es_zone = (z >= args.num_zones)
 270
 271             # create the zone in its zonegroup
 272             zone = multisite.Zone(zone_name(zg, z), zonegroup, cluster)
 273             if es_zone:
 274                 zone = ESZone(zone_name(zg, z), args.es_endpoint, zonegroup, cluster)
 275             else:
 276                 zone = RadosZone(zone_name(zg, z), zonegroup, cluster)
 277
 278             if bootstrap:
 279                 arg = admin_creds.credential_args()
 280                 if is_master:
 281                     arg += ['--master']
 282                 if len(endpoints):
 283                     arg += ['--endpoints', ','.join(endpoints)]
 284                 zone.create(cluster, arg)
 285             else:
 286                 zone.get(cluster)
 287             zonegroup.zones.append(zone)
 288             if is_master:
 289                 zonegroup.master_zone = zone
 290
 291             zonegroup.zones_by_type.setdefault(zone.tier_type(), []).append(zone)
 292
 293             if zone.is_read_only():
 294                 zonegroup.ro_zones.append(zone)
 295             else:
 296                 zonegroup.rw_zones.append(zone)
 297
 298             # update/commit the period
 299             if bootstrap:
 300                 period.update(zone, commit=True)
 301
 302             # start the gateways
 303             for g in range(0, args.gateways_per_zone):
 304                 port = gateway_port(zg, g + z * args.gateways_per_zone)
 305                 client_id = gateway_name(zg, z, g)
 306                 gateway = Gateway(client_id, 'localhost', port, cluster, zone)
 307                 if bootstrap:
 308                     gateway.start()
 309                 zone.gateways.append(gateway)
 310
 311             if is_master_zg and is_master:
 312                 if bootstrap:
 313                     # create admin user
 314                     arg = ['--display-name', '"Zone User"', '--system']
 315                     arg += admin_creds.credential_args()
 316                     admin_user.create(zone, arg)
 317                     # create test user
 318                     arg = ['--display-name', '"Test User"']
 319                     arg += user_creds.credential_args()
 320                     if args.tenant:
 321                         cmd += ['--tenant', args.tenant]
 322                     user.create(zone, arg)
 323                 else:
 324                     # read users and update keys
 325                     admin_user.info(zone)
 326                     admin_creds = admin_user.credentials[0]
 327                     user.info(zone)
 328                     user_creds = user.credentials[0]
 329
 330     if not bootstrap:
 331         period.get(c1)
 332
 333     config = Config(checkpoint_retries=args.checkpoint_retries,
 334                     checkpoint_delay=args.checkpoint_delay,
 335                     reconfigure_delay=args.reconfigure_delay)
 336     init_multi(realm, user, config)
 337
 338 def setup_module():
 339     init(False)
 340
 341 if __name__ == "__main__":
 342     init(True)