testcases/config_functest.py

   1 #!/usr/bin/env python
   2 #
   3 # Copyright (c) 2015 Ericsson
   4 # jose.lausuch@ericsson.com
   5 # All rights reserved. This program and the accompanying materials
   6 # are made available under the terms of the Apache License, Version 2.0
   7 # which accompanies this distribution, and is available at
   8 # http://www.apache.org/licenses/LICENSE-2.0
   9 #
  10
  11 import re, json, os, urllib2, argparse, logging, shutil, subprocess, yaml
  12 from git import Repo
  13
  14 from neutronclient.v2_0 import client
  15
  16 actions = ['start', 'check', 'clean']
  17 parser = argparse.ArgumentParser()
  18 parser.add_argument("repo_path", help="Path to the repository")
  19 parser.add_argument("action", help="Possible actions are: '{d[0]}|{d[1]}|{d[2]}' ".format(d=actions))
  20 parser.add_argument("-d", "--debug", help="Debug mode",  action="store_true")
  21 parser.add_argument("-f", "--force", help="Force",  action="store_true")
  22 args = parser.parse_args()
  23
  24
  25 """ logging configuration """
  26 logger = logging.getLogger('config_functest')
  27 logger.setLevel(logging.DEBUG)
  28
  29 ch = logging.StreamHandler()
  30 if args.debug:
  31     ch.setLevel(logging.DEBUG)
  32 else:
  33     ch.setLevel(logging.INFO)
  34
  35 formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
  36 ch.setFormatter(formatter)
  37 logger.addHandler(ch)
  38
  39 if not os.path.exists(args.repo_path):
  40     logger.error("Repo directory not found '%s'" % args.repo_path)
  41     exit(-1)
  42
  43 with open(args.repo_path+"testcases/config_functest.yaml") as f:
  44     functest_yaml = yaml.safe_load(f)
  45 f.close()
  46
  47
  48
  49
  50 """ global variables """
  51 # Directories
  52 HOME = os.environ['HOME']+"/"
  53 REPO_PATH = args.repo_path
  54 RALLY_DIR = REPO_PATH + functest_yaml.get("general").get("directories").get("dir_rally")
  55 RALLY_REPO_DIR = HOME + functest_yaml.get("general").get("directories").get("dir_rally_repo")
  56 RALLY_INSTALLATION_DIR = HOME + functest_yaml.get("general").get("directories").get("dir_rally_inst")
  57 VPING_DIR = REPO_PATH + functest_yaml.get("general").get("directories").get("dir_vping")
  58 ODL_DIR = REPO_PATH + functest_yaml.get("general").get("directories").get("dir_odl")
  59
  60 # NEUTRON Private Network parameters
  61 NEUTRON_PRIVATE_NET_NAME = functest_yaml.get("general").get("openstack").get("neutron_private_net_name")
  62 NEUTRON_PRIVATE_SUBNET_NAME = functest_yaml.get("general").get("openstack").get("neutron_private_subnet_name")
  63 NEUTRON_PRIVATE_SUBNET_CIDR = functest_yaml.get("general").get("openstack").get("neutron_private_subnet_cidr")
  64 ROUTER_NAME = functest_yaml.get("general").get("openstack").get("neutron_router_name")
  65
  66 #GLANCE image parameters
  67 IMAGE_URL = functest_yaml.get("general").get("openstack").get("image_url")
  68 IMAGE_DISK_FORMAT = functest_yaml.get("general").get("openstack").get("image_disk_format")
  69 IMAGE_NAME = functest_yaml.get("general").get("openstack").get("image_name")
  70 IMAGE_FILE_NAME = IMAGE_URL.rsplit('/')[-1]
  71 IMAGE_DIR = HOME + functest_yaml.get("general").get("openstack").get("image_download_path")
  72 IMAGE_PATH = IMAGE_DIR + IMAGE_FILE_NAME
  73
  74
  75 credentials = None
  76 neutron_client = None
  77
  78 def action_start():
  79     """
  80     Start the functest environment installation
  81     """
  82     if not check_internet_connectivity():
  83         logger.error("There is no Internet connectivity. Please check the network configuration.")
  84         exit(-1)
  85
  86     if action_check():
  87         logger.info("Functest environment already installed. Nothing to do.")
  88         exit(0)
  89
  90     else:
  91         # Clean in case there are left overs
  92         logger.debug("Cleaning possible functest environment leftovers.")
  93         action_clean()
  94
  95         logger.info("Starting installation of functest environment")
  96         logger.info("Installing Rally...")
  97         if not install_rally():
  98             logger.error("There has been a problem while installing Rally.")
  99             action_clean()
 100             exit(-1)
 101
 102         logger.info("Installing ODL environment...")
 103         if not install_odl():
 104             logger.error("There has been a problem while installing Robot.")
 105             action_clean()
 106             exit(-1)
 107
 108         credentials = get_credentials()
 109         neutron_client = client.Client(**credentials)
 110
 111         logger.info("Configuring Neutron...")
 112         logger.info("Checking if private network '%s' exists..." % NEUTRON_PRIVATE_NET_NAME)
 113         #Now: if exists we don't create it again (the clean command does not clean the neutron networks yet)
 114         if check_neutron_net(neutron_client, NEUTRON_PRIVATE_NET_NAME):
 115             logger.info("Private network '%s' found. No need to create another one." % NEUTRON_PRIVATE_NET_NAME)
 116         else:
 117             logger.info("Private network '%s' not found. Creating..." % NEUTRON_PRIVATE_NET_NAME)
 118             if not create_private_neutron_net(neutron_client):
 119                 logger.error("There has been a problem while creating the Neutron network.")
 120                 #action_clean()
 121                 exit(-1)
 122
 123
 124         logger.info("Donwloading image...")
 125         if not download_url(IMAGE_URL, IMAGE_DIR):
 126             logger.error("There has been a problem while downloading the image.")
 127             action_clean()
 128             exit(-1)
 129
 130         logger.info("Creating Glance image: %s ..." %IMAGE_NAME)
 131         if not create_glance_image(IMAGE_PATH,IMAGE_NAME,IMAGE_DISK_FORMAT):
 132             logger.error("There has been a problem while creating the Glance image.")
 133             action_clean()
 134             exit(-1)
 135
 136         exit(0)
 137
 138
 139 def action_check():
 140     """
 141     Check if the functest environment is properly installed
 142     """
 143     errors_all = False
 144
 145     logger.info("Checking current functest configuration...")
 146     credentials = get_credentials()
 147     neutron_client = client.Client(**credentials)
 148
 149     logger.debug("Checking directories...")
 150     errors = False
 151     dirs = [RALLY_DIR, RALLY_INSTALLATION_DIR, VPING_DIR, ODL_DIR]
 152     for dir in dirs:
 153         if not os.path.exists(dir):
 154             logger.debug("The directory '%s' does NOT exist." % dir)
 155             errors = True
 156             errors_all = True
 157         else:
 158             logger.debug("   %s found" % dir)
 159     if not errors:
 160         logger.debug("...OK")
 161     else:
 162         logger.debug("...FAIL")
 163
 164
 165     logger.debug("Checking Rally deployment...")
 166     if not check_rally():
 167         logger.debug("   Rally deployment NOT found.")
 168         errors_all = True
 169         logger.debug("...FAIL")
 170     else:
 171         logger.debug("...OK")
 172
 173
 174     logger.debug("Checking Neutron...")
 175     if not check_neutron_net(neutron_client, NEUTRON_PRIVATE_NET_NAME):
 176         logger.debug("   Private network '%s' NOT found." % NEUTRON_PRIVATE_NET_NAME)
 177         logger.debug("...FAIL")
 178         errors_all = True
 179     else:
 180         logger.debug("   Private network '%s' found." % NEUTRON_PRIVATE_NET_NAME)
 181         logger.debug("...OK")
 182
 183
 184     logger.debug("Checking Image...")
 185     errors = False
 186     if not os.path.isfile(IMAGE_PATH):
 187         logger.debug("   Image file '%s' NOT found." % IMAGE_PATH)
 188         errors = True
 189         errors_all = True
 190     else:
 191         logger.debug("   Image file found in %s" % IMAGE_PATH)
 192
 193     cmd="glance image-list | grep " + IMAGE_NAME
 194     FNULL = open(os.devnull, 'w');
 195     logger.debug('   Executing command : {}'.format(cmd))
 196     p=subprocess.Popen(cmd,shell=True,stdout=subprocess.PIPE,stderr=FNULL);
 197     #if the command does not exist or there is no glance image
 198     line = p.stdout.readline()
 199     if line == "":
 200         logger.debug("   Glance image NOT found.")
 201         errors = True
 202         errors_all = True
 203     else:
 204         logger.debug("   Glance image found.")
 205
 206     if not errors:
 207         logger.debug("...OK")
 208     else:
 209         logger.debug("...FAIL")
 210
 211     #TODO: check OLD environment setup
 212     if errors_all:
 213         return False
 214     else:
 215         return True
 216
 217
 218
 219
 220 def action_clean():
 221     """
 222     Clean the existing functest environment
 223     """
 224     logger.info("Removing current functest environment...")
 225     if os.path.exists(RALLY_INSTALLATION_DIR):
 226         logger.debug("Removing Rally installation directory %s" % RALLY_INSTALLATION_DIR)
 227         shutil.rmtree(RALLY_INSTALLATION_DIR,ignore_errors=True)
 228
 229     if os.path.exists(RALLY_REPO_DIR):
 230         logger.debug("Removing Rally repository %s" % RALLY_REPO_DIR)
 231         cmd = "sudo rm -rf " + RALLY_REPO_DIR #need to be sudo, not possible with rmtree
 232         execute_command(cmd)
 233     #logger.debug("Deleting Neutron network %s" % NEUTRON_PRIVATE_NET_NAME)
 234     #if not delete_neutron_net() :
 235     #    logger.error("Error deleting the network. Remove it manually.")
 236
 237     logger.debug("Deleting glance images")
 238     if os.path.exists(IMAGE_PATH):
 239         os.remove(IMAGE_PATH)
 240
 241     cmd = "glance image-list | grep "+IMAGE_NAME+" | cut -c3-38"
 242     p = os.popen(cmd,"r")
 243
 244     #while image_id = p.readline()
 245     for image_id in p.readlines():
 246         cmd = "glance image-delete " + image_id
 247         execute_command(cmd)
 248
 249     return True
 250
 251
 252
 253
 254
 255 def install_rally():
 256     if check_rally():
 257         logger.info("Rally is already installed.")
 258     else:
 259         logger.debug("Cloning repository...")
 260         url = "https://git.openstack.org/openstack/rally"
 261         Repo.clone_from(url, RALLY_REPO_DIR)
 262
 263         logger.debug("Executing %s./install_rally.sh..." %RALLY_REPO_DIR)
 264         install_script = RALLY_REPO_DIR + "install_rally.sh"
 265         cmd = 'sudo ' + install_script
 266         execute_command(cmd)
 267         #subprocess.call(['sudo', install_script])
 268
 269         logger.debug("Creating Rally environment...")
 270         cmd = "rally deployment create --fromenv --name=opnfv-arno-rally"
 271         execute_command(cmd)
 272
 273         logger.debug("Installing tempest...")
 274         cmd = "rally-manage tempest install"
 275         execute_command(cmd)
 276
 277         cmd = "rally deployment check"
 278         execute_command(cmd)
 279         #TODO: check that everything is 'Available' and warn if not
 280
 281         cmd = "rally show images"
 282         execute_command(cmd)
 283
 284         cmd = "rally show flavors"
 285         execute_command(cmd)
 286
 287     return True
 288
 289
 290
 291 def check_rally():
 292     """
 293     Check if Rally is installed and properly configured
 294     """
 295     if os.path.exists(RALLY_INSTALLATION_DIR):
 296         logger.debug("   Rally installation directory found in %s" % RALLY_INSTALLATION_DIR)
 297         FNULL = open(os.devnull, 'w');
 298         cmd="rally deployment list | grep opnfv";
 299         logger.debug('   Executing command : {}'.format(cmd))
 300         p=subprocess.Popen(cmd,shell=True,stdout=subprocess.PIPE,stderr=FNULL);
 301         #if the command does not exist or there is no deployment
 302         line = p.stdout.readline()
 303         if line == "":
 304             logger.debug("   Rally deployment not found")
 305             return False
 306         logger.debug("   Rally deployment found")
 307         return True
 308     else:
 309         logger.debug("   Rally installation directory not found")
 310         return False
 311
 312
 313 def install_odl():
 314     cmd = "chmod +x " + ODL_DIR + "start_tests.sh"
 315     execute_command(cmd)
 316     cmd = "chmod +x " + ODL_DIR + "create_venv.sh"
 317     execute_command(cmd)
 318     cmd = ODL_DIR + "create_venv.sh"
 319     execute_command(cmd)
 320     return True
 321
 322
 323 def check_credentials():
 324     """
 325     Check if the OpenStack credentials (openrc) are sourced
 326     """
 327     #TODO: there must be a short way to do this, doing if os.environ["something"] == "" throws an error
 328     try:
 329        os.environ['OS_AUTH_URL']
 330     except KeyError:
 331         return False
 332     try:
 333        os.environ['OS_USERNAME']
 334     except KeyError:
 335         return False
 336     try:
 337        os.environ['OS_PASSWORD']
 338     except KeyError:
 339         return False
 340     try:
 341        os.environ['OS_TENANT_NAME']
 342     except KeyError:
 343         return False
 344     try:
 345        os.environ['OS_REGION_NAME']
 346     except KeyError:
 347         return False
 348     return True
 349
 350
 351 def get_credentials():
 352     d = {}
 353     d['username'] = os.environ['OS_USERNAME']
 354     d['password'] = os.environ['OS_PASSWORD']
 355     d['auth_url'] = os.environ['OS_AUTH_URL']
 356     d['tenant_name'] = os.environ['OS_TENANT_NAME']
 357     return d
 358
 359 def get_nova_credentials():
 360     d = {}
 361     d['username'] = os.environ['OS_USERNAME']
 362     d['api_key'] = os.environ['OS_PASSWORD']
 363     d['auth_url'] = os.environ['OS_AUTH_URL']
 364     d['project_id'] = os.environ['OS_TENANT_NAME']
 365     return d
 366
 367
 368
 369 def create_private_neutron_net(neutron):
 370     try:
 371         neutron.format = 'json'
 372         logger.debug('Creating Neutron network %s...' % NEUTRON_PRIVATE_NET_NAME)
 373         json_body = {'network': {'name': NEUTRON_PRIVATE_NET_NAME,
 374                     'admin_state_up': True}}
 375         netw = neutron.create_network(body=json_body)
 376         net_dict = netw['network']
 377         network_id = net_dict['id']
 378         logger.debug("Network '%s' created successfully" % network_id)
 379
 380         logger.debug('Creating Subnet....')
 381         json_body = {'subnets': [{'name': NEUTRON_PRIVATE_SUBNET_NAME, 'cidr': NEUTRON_PRIVATE_SUBNET_CIDR,
 382                            'ip_version': 4, 'network_id': network_id}]}
 383
 384         subnet = neutron.create_subnet(body=json_body)
 385         subnet_id = subnet['subnets'][0]['id']
 386         logger.debug("Subnet '%s' created successfully" % subnet_id)
 387
 388
 389         logger.debug('Creating Router...')
 390         json_body = {'router': {'name': ROUTER_NAME, 'admin_state_up': True}}
 391         router = neutron.create_router(json_body)
 392         router_id = router['router']['id']
 393         logger.debug("Router '%s' created successfully" % router_id)
 394
 395         logger.debug('Adding router to subnet...')
 396         json_body = {"subnet_id": subnet_id}
 397         neutron.add_interface_router(router=router_id, body=json_body)
 398         logger.debug("Interface added successfully.")
 399
 400     except:
 401         print "Error:", sys.exc_info()[0]
 402         return False
 403
 404     logger.info("Private Neutron network created successfully.")
 405     return True
 406
 407 def get_network_id(neutron, network_name):
 408     networks = neutron.list_networks()['networks']
 409     id  = ''
 410     for n in networks:
 411         if n['name'] == network_name:
 412             id = n['id']
 413             break
 414     return id
 415
 416 def check_neutron_net(neutron, net_name):
 417     for network in neutron.list_networks()['networks']:
 418         if network['name'] == net_name :
 419             for subnet in network['subnets']:
 420                 return True
 421     return False
 422
 423 def delete_neutron_net(neutron):
 424     #TODO: remove router, ports
 425     try:
 426         #https://github.com/isginf/openstack_tools/blob/master/openstack_remove_tenant.py
 427         for network in neutron.list_networks()['networks']:
 428             if network['name'] == NEUTRON_PRIVATE_NET_NAME :
 429                 for subnet in network['subnets']:
 430                     print "Deleting subnet " + subnet
 431                     neutron.delete_subnet(subnet)
 432                 print "Deleting network " + network['name']
 433                 neutron.delete_neutron_net(network['id'])
 434     finally:
 435         return True
 436     return False
 437
 438
 439
 440
 441
 442 def create_glance_image(path,name,disk_format):
 443     """
 444     Create a glance image given the absolute path of the image, its name and the disk format
 445     """
 446     cmd = "glance image-create --name "+name+" --is-public true --disk-format "+disk_format+" --container-format bare --file "+path
 447     execute_command(cmd)
 448     return True
 449
 450
 451
 452
 453
 454 def download_url(url, dest_path):
 455     """
 456     Download a file to a destination path given a URL
 457     """
 458     name = url.rsplit('/')[-1]
 459     dest = dest_path + name
 460     try:
 461         response = urllib2.urlopen(url)
 462     except (urllib2.HTTPError, urllib2.URLError):
 463         logger.error("Error in fetching %s" %url)
 464         return False
 465
 466     with open(dest, 'wb') as f:
 467         f.write(response.read())
 468     return True
 469
 470
 471
 472 def check_internet_connectivity(url='http://www.google.com/'):
 473     """
 474     Check if there is access to the internet
 475     """
 476     try:
 477         urllib2.urlopen(url, timeout=5)
 478         return True
 479     except urllib.request.URLError:
 480         return False
 481
 482 def execute_command(cmd):
 483     """
 484     Execute Linux command
 485     """
 486     logger.debug('Executing command : {}'.format(cmd))
 487     #p = os.popen(cmd,"r")
 488     #logger.debug(p.read())
 489     output_file = "output.txt"
 490     f = open(output_file, 'w+')
 491     p = subprocess.call(cmd,shell=True, stdout=f, stderr=subprocess.STDOUT)
 492     f.close()
 493     f = open(output_file, 'r')
 494     result = f.read()
 495     if result != "":
 496         logger.debug(result)
 497     #p = subprocess.call(cmd,shell=True);
 498     if p == 0 :
 499         return True
 500     else:
 501         logger.error("Error when executing command %s" %cmd)
 502         exit(-1)
 503
 504
 505
 506
 507 def main():
 508     if not (args.action in actions):
 509         logger.error('argument not valid')
 510         exit(-1)
 511
 512
 513     if not check_credentials():
 514         logger.error("Please source the openrc credentials and run the script again.")
 515         #TODO: source the credentials in this script
 516         exit(-1)
 517
 518     if args.action == "start":
 519         action_start()
 520
 521     if args.action == "check":
 522         if action_check():
 523             logger.info("Functest environment correctly installed")
 524         else:
 525             logger.info("Functest environment faulty")
 526
 527     if args.action == "clean":
 528         if args.force :
 529             action_clean()
 530         else :
 531             while True:
 532                 print("Are you sure? [y|n]")
 533                 answer = raw_input("")
 534                 if answer == "y":
 535                     action_clean()
 536                     break
 537                 elif answer == "n":
 538                     break
 539                 else:
 540                     print("Invalid option.")
 541     exit(0)
 542
 543
 544 if __name__ == '__main__':
 545     main()
 546