X-Git-Url: https://gerrit.opnfv.org/gerrit/gitweb?a=blobdiff_plain;f=testcases%2Fconfig_functest.py;h=21c569cb00d65966edaffe7ce67774c510ac0d76;hb=75bfd472302483a201a48eb18c69447343f83b75;hp=ad804d146bee060ca760db929719251048398b6c;hpb=84f07e9d8453e8d7eb6b99cd6988d27fb4fa456c;p=functest.git diff --git a/testcases/config_functest.py b/testcases/config_functest.py old mode 100644 new mode 100755 index ad804d146..21c569cb0 --- a/testcases/config_functest.py +++ b/testcases/config_functest.py @@ -8,30 +8,18 @@ # http://www.apache.org/licenses/LICENSE-2.0 # -import re, json, os, urllib2, argparse, logging, shutil, subprocess +import re, json, os, urllib2, argparse, logging, shutil, subprocess, yaml, sys, getpass +import functest_utils from git import Repo +from os import stat +from pwd import getpwuid +from neutronclient.v2_0 import client as neutronclient actions = ['start', 'check', 'clean'] - -""" global variables """ -functest_dir = os.environ['HOME'] + '/.functest/' -#image_url = 'https://cloud-images.ubuntu.com/trusty/current/trusty-server-cloudimg-amd64-disk1.img' -image_url = 'http://download.cirros-cloud.net/0.3.0/cirros-0.3.0-i386-disk.img' - -image_disk_format = 'raw' -image_name = image_url.rsplit('/')[-1] -image_path = functest_dir + image_name -rally_repo_dir = functest_dir + "Rally_repo/" -rally_test_dir = functest_dir + "Rally_test/" -bench_tests_dir = rally_test_dir + "scenarios/" -rally_installation_dir = os.environ['HOME'] + "/.rally" -vPing_dir = functest_dir + "vPing/" -odl_dir = functest_dir + "ODL/" - - parser = argparse.ArgumentParser() parser.add_argument("action", help="Possible actions are: '{d[0]}|{d[1]}|{d[2]}' ".format(d=actions)) parser.add_argument("-d", "--debug", help="Debug mode", action="store_true") +parser.add_argument("-f", "--force", help="Force", action="store_true") args = parser.parse_args() @@ -49,386 +37,328 @@ formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(messag ch.setFormatter(formatter) logger.addHandler(ch) +REPOS_DIR=os.environ['repos_dir'] +FUNCTEST_REPO=REPOS_DIR+'/functest/' +if not os.path.exists(FUNCTEST_REPO): + logger.error("Functest repository directory not found '%s'" % FUNCTEST_REPO) + exit(-1) +sys.path.append(FUNCTEST_REPO + "testcases/") + +with open("/home/opnfv/functest/conf/config_functest.yaml") as f: + functest_yaml = yaml.safe_load(f) +f.close() -def config_functest_start(): +""" global variables """ +# Directories +RALLY_DIR = FUNCTEST_REPO + functest_yaml.get("general").get("directories").get("dir_rally") +RALLY_REPO_DIR = functest_yaml.get("general").get("directories").get("dir_repo_rally") +RALLY_INSTALLATION_DIR = functest_yaml.get("general").get("directories").get("dir_rally_inst") +RALLY_RESULT_DIR = functest_yaml.get("general").get("directories").get("dir_rally_res") +VPING_DIR = FUNCTEST_REPO + functest_yaml.get("general").get("directories").get("dir_vping") +VIMS_TEST_DIR = functest_yaml.get("general").get("directories").get("dir_repo_vims_test") +ODL_DIR = FUNCTEST_REPO + functest_yaml.get("general").get("directories").get("dir_odl") +DATA_DIR = functest_yaml.get("general").get("directories").get("dir_functest_data") + +# Tempest/Rally configuration details +DEPLOYMENT_MAME = functest_yaml.get("rally").get("deployment_name") +RALLY_COMMIT = functest_yaml.get("general").get("repositories").get("rally_commit") + +#Image (cirros) +IMAGE_FILE_NAME = functest_yaml.get("general").get("openstack").get("image_file_name") +IMAGE_PATH = DATA_DIR + "/" + IMAGE_FILE_NAME + +# NEUTRON Private Network parameters +NEUTRON_PRIVATE_NET_NAME = functest_yaml.get("general"). \ + get("openstack").get("neutron_private_net_name") +NEUTRON_PRIVATE_SUBNET_NAME = functest_yaml.get("general"). \ + get("openstack").get("neutron_private_subnet_name") +NEUTRON_PRIVATE_SUBNET_CIDR = functest_yaml.get("general"). \ + get("openstack").get("neutron_private_subnet_cidr") +NEUTRON_ROUTER_NAME = functest_yaml.get("general"). \ + get("openstack").get("neutron_router_name") + +creds_neutron = functest_utils.get_credentials("neutron") +neutron_client = neutronclient.Client(**creds_neutron) + +def action_start(): """ Start the functest environment installation """ - if config_functest_check(): - logger.info("Functest environment already installed in %s. Nothing to do." %functest_dir) - exit(0) - elif not check_internet_connectivity(): + if not functest_utils.check_internet_connectivity(): logger.error("There is no Internet connectivity. Please check the network configuration.") exit(-1) - elif not check_credentials(): - logger.error("Please source the openrc credentials and run the script again.") - #TODO: source the credentials in this script - exit(-1) - else: - config_functest_clean() - logger.info("Starting installationg of functest environment in %s" %functest_dir) - os.makedirs(functest_dir) - if not os.path.exists(functest_dir): - logger.error("There has been a problem while creating the environment directory.") - exit(-1) + if action_check(): + logger.info("Functest environment already installed. Nothing to do.") + exit(0) - logger.info("Donwloading test scripts and scenarios...") - if not download_tests(): - logger.error("There has been a problem while downloading the test scripts and scenarios.") - config_functest_clean() - exit(-1) + else: + # Clean in case there are left overs + logger.debug("Cleaning possible functest environment leftovers.") + action_clean() + logger.info("Starting installation of functest environment") + + private_net = functest_utils.get_private_net(neutron_client) + if private_net is None: + # If there is no private network in the deployment we create one + if not create_private_neutron_net(neutron_client): + logger.error("There has been a problem while creating the functest network.") + action_clean() + exit(-1) + else: + logger.info("Private network '%s' already existing in the deployment." + % private_net['name']) logger.info("Installing Rally...") if not install_rally(): logger.error("There has been a problem while installing Rally.") - config_functest_clean() + action_clean() exit(-1) - logger.info("Installing ODL environment...") - if not install_odl(): - logger.error("There has been a problem while installing Robot.") - config_functest_clean() - exit(-1) + logger.info("Installing Ruby libraries for vIMS testcase...") + # Install ruby libraries for vims test-case + script = 'source /etc/profile.d/rvm.sh; ' + script += 'cd ' + VIMS_TEST_DIR + '; ' + script += 'rvm autolibs enable ;' + script += 'rvm install 1.9.3; ' + script += 'rvm use 1.9.3;' + script += 'bundle install' - logger.info("Donwloading image...") - if not download_url_with_progress(image_url, functest_dir): - logger.error("There has been a problem while downloading the image.") - config_functest_clean() - exit(-1) + logger_debug = None + CI_DEBUG = os.environ.get("CI_DEBUG") + if CI_DEBUG == "true" or CI_DEBUG == "True": + logger_debug = logger - logger.info("Creating Glance image: %s ..." %image_name) - if not create_glance_image(image_path,image_name,image_disk_format): - logger.error("There has been a problem while creating the Glance image.") - config_functest_clean() - exit(-1) + cmd = "/bin/bash -c '" + script + "'" + functest_utils.execute_command(cmd, logger = logger_debug, exit_on_error=False) - exit(0) + install_promise(logger_debug) + # Create result folder under functest if necessary + if not os.path.exists(RALLY_RESULT_DIR): + os.makedirs(RALLY_RESULT_DIR) + + try: + logger.info("CI: Generate the list of executable tests.") + runnable_test = functest_utils.generateTestcaseList(functest_yaml) + logger.info("List of runnable tests generated: %s" % runnable_test) + except: + logger.error("Impossible to generate the list of runnable tests") + + exit(0) -def config_functest_check(): +def action_check(): """ Check if the functest environment is properly installed """ + errors_all = False + errors = False logger.info("Checking current functest configuration...") - logger.debug("Checking directories...") - dirs = [functest_dir, rally_installation_dir, rally_repo_dir, rally_test_dir, bench_tests_dir, vPing_dir, odl_dir] + logger.debug("Checking script directories...") + + dirs = [RALLY_DIR, RALLY_INSTALLATION_DIR, VPING_DIR, ODL_DIR] for dir in dirs: if not os.path.exists(dir): - logger.debug("The directory %s does not exist." %dir) - return False - logger.debug(" %s found" % dir) + logger.debug(" %s NOT found" % dir) + errors = True + errors_all = True + else: + logger.debug(" %s found" % dir) - logger.debug("...OK") logger.debug("Checking Rally deployment...") if not check_rally(): - logger.debug("Rally deployment not found.") - return False - logger.debug("...OK") + logger.debug(" Rally deployment NOT installed.") + errors_all = True logger.debug("Checking Image...") - if not os.path.isfile(image_path): - return False - logger.debug(" Image file found in %s" %image_path) - - cmd="glance image-list | grep " + image_name - FNULL = open(os.devnull, 'w'); - logger.debug(' Executing command : {}'.format(cmd)) - p=subprocess.Popen(cmd,shell=True,stdout=subprocess.PIPE,stderr=FNULL); - #if the command does not exist or there is no glance image - line = p.stdout.readline() - if line == "": - logger.debug(" Glance image not found") - return False - logger.debug(" Glance image found") - logger.debug("...OK") - - #TODO: check OLD environment setup + errors = False + if not os.path.isfile(IMAGE_PATH): + logger.debug(" Image file '%s' NOT found." % IMAGE_PATH) + errors = True + errors_all = True + else: + logger.debug(" Image file found in %s" % IMAGE_PATH) - return True + #TODO: check OLD environment setup + return not errors_all -def config_functest_clean(): +def action_clean(): """ Clean the existing functest environment """ logger.info("Removing current functest environment...") - if os.path.exists(rally_installation_dir): - logger.debug("Removing rally installation directory %s" % rally_installation_dir) - shutil.rmtree(rally_installation_dir,ignore_errors=True) - - if os.path.exists(functest_dir): - logger.debug("Removing functest directory %s" % functest_dir) - cmd = "sudo rm -rf " + functest_dir #need to be sudo, not possible with rmtree - execute_command(cmd) + if os.path.exists(RALLY_INSTALLATION_DIR): + logger.debug("Removing Rally installation directory %s" % RALLY_INSTALLATION_DIR) + shutil.rmtree(RALLY_INSTALLATION_DIR,ignore_errors=True) - logger.debug("Deleting glance images") - cmd = "glance image-list | grep "+image_name+" | cut -c3-38" - p = os.popen(cmd,"r") + if os.path.exists(RALLY_RESULT_DIR): + logger.debug("Removing Result directory") + shutil.rmtree(RALLY_RESULT_DIR,ignore_errors=True) - #while image_id = p.readline() - for image_id in p.readlines(): - cmd = "glance image-delete " + image_id - execute_command(cmd) + logger.info("Functest environment clean!") - return True def install_rally(): if check_rally(): logger.info("Rally is already installed.") else: - logger.debug("Cloning repository...") - url = "https://git.openstack.org/openstack/rally" - Repo.clone_from(url, rally_repo_dir) - - logger.debug("Executing %s./install_rally.sh..." %rally_repo_dir) - install_script = rally_repo_dir + "install_rally.sh" + logger.debug("Executing %s/install_rally.sh..." %RALLY_REPO_DIR) + install_script = RALLY_REPO_DIR + "/install_rally.sh --yes" cmd = 'sudo ' + install_script - execute_command(cmd) - #subprocess.call(['sudo', install_script]) + if os.environ.get("CI_DEBUG") == "false": + functest_utils.execute_command(cmd) + else: + functest_utils.execute_command(cmd,logger) logger.debug("Creating Rally environment...") - cmd = "rally deployment create --fromenv --name=opnfv-arno-rally" - execute_command(cmd) + cmd = "rally deployment create --fromenv --name="+DEPLOYMENT_MAME + functest_utils.execute_command(cmd,logger) logger.debug("Installing tempest...") - cmd = "rally-manage tempest install" - execute_command(cmd) + cmd = "rally verify install" + functest_utils.execute_command(cmd,logger) cmd = "rally deployment check" - execute_command(cmd) + functest_utils.execute_command(cmd,logger) #TODO: check that everything is 'Available' and warn if not cmd = "rally show images" - execute_command(cmd) + functest_utils.execute_command(cmd,logger) cmd = "rally show flavors" - execute_command(cmd) + functest_utils.execute_command(cmd,logger) return True +def install_promise(logger_debug): + logger.info("Installing dependencies for Promise testcase...") + current_dir = os.getcwd() + os.chdir(REPOS_DIR+'/promise/') + + cmd = 'curl -sL https://deb.nodesource.com/setup_4.x | sudo -E bash -' + functest_utils.execute_command(cmd,logger = logger_debug, exit_on_error=False) + + cmd = 'sudo apt-get install -y nodejs' + functest_utils.execute_command(cmd,logger = logger_debug, exit_on_error=False) + + cmd = 'sudo npm -g install npm@latest' + functest_utils.execute_command(cmd,logger = logger_debug, exit_on_error=False) + + cmd = 'npm install' + functest_utils.execute_command(cmd,logger = logger_debug, exit_on_error=False) + os.chdir(current_dir) def check_rally(): """ Check if Rally is installed and properly configured """ - if os.path.exists(rally_installation_dir): - logger.debug(" Rally installation directory found in %s" % rally_installation_dir) + if os.path.exists(RALLY_INSTALLATION_DIR): + logger.debug(" Rally installation directory found in %s" % RALLY_INSTALLATION_DIR) FNULL = open(os.devnull, 'w'); - cmd="rally deployment list | grep opnfv"; + cmd="rally deployment list | grep "+DEPLOYMENT_MAME logger.debug(' Executing command : {}'.format(cmd)) p=subprocess.Popen(cmd,shell=True,stdout=subprocess.PIPE,stderr=FNULL); #if the command does not exist or there is no deployment line = p.stdout.readline() if line == "": - logger.debug(" Rally deployment not found") + logger.debug(" Rally deployment NOT found") return False logger.debug(" Rally deployment found") return True else: - logger.debug(" Rally installation directory not found") - return False - - -def install_odl(): - cmd = "chmod +x " + odl_dir + "create_venv.sh" - execute_command(cmd) - cmd = odl_dir + "create_venv.sh" - execute_command(cmd) - return True - - -def check_credentials(): - """ - Check if the OpenStack credentials (openrc) are sourced - """ - #TODO: there must be a short way to do this, doing if os.environ["something"] == "" throws an error - try: - os.environ['OS_AUTH_URL'] - except KeyError: - return False - try: - os.environ['OS_USERNAME'] - except KeyError: - return False - try: - os.environ['OS_PASSWORD'] - except KeyError: - return False - try: - os.environ['OS_TENANT_NAME'] - except KeyError: return False - try: - os.environ['OS_REGION_NAME'] - except KeyError: - return False - return True -def download_tests(): - os.makedirs(vPing_dir) - os.makedirs(odl_dir) - os.makedirs(bench_tests_dir) +def create_private_neutron_net(neutron): + neutron.format = 'json' + logger.info("Creating network '%s'..." % NEUTRON_PRIVATE_NET_NAME) + network_id = functest_utils. \ + create_neutron_net(neutron, NEUTRON_PRIVATE_NET_NAME) - logger.info("Downloading vPing test...") - vPing_url = 'https://git.opnfv.org/cgit/functest/plain/testcases/vPing/CI/libraries/vPing.py' - if not download_url(vPing_url,vPing_dir): + if not network_id: return False + logger.debug("Network '%s' created successfully." % network_id) - - logger.info("Downloading Rally bench tests...") - run_rally_url = 'https://git.opnfv.org/cgit/functest/plain/testcases/VIM/OpenStack/CI/libraries/run_rally.py' - if not download_url(run_rally_url,rally_test_dir): + logger.info("Updating network '%s' with shared=True..." % NEUTRON_PRIVATE_NET_NAME) + if functest_utils.update_neutron_net(neutron, network_id, shared=True): + logger.debug("Network '%s' updated successfully." % network_id) + else: + logger.info("Updating neutron network '%s' failed" % network_id) + + logger.info("Creating Subnet....") + subnet_id = functest_utils. \ + create_neutron_subnet(neutron, + NEUTRON_PRIVATE_SUBNET_NAME, + NEUTRON_PRIVATE_SUBNET_CIDR, + network_id) + if not subnet_id: return False + logger.debug("Subnet '%s' created successfully." % subnet_id) + logger.info("Creating Router...") + router_id = functest_utils. \ + create_neutron_router(neutron, NEUTRON_ROUTER_NAME) - rally_bench_base_url = 'https://git.opnfv.org/cgit/functest/plain/testcases/VIM/OpenStack/CI/suites/' - bench_tests = ['authenticate', 'cinder', 'glance', 'heat', 'keystone', 'neutron', 'nova', 'quotas', 'requests', 'tempest', 'vm'] - for i in bench_tests: - rally_bench_url = rally_bench_base_url + "opnfv-" + i + ".json" - logger.debug("Downloading %s" %rally_bench_url) - if not download_url(rally_bench_url,bench_tests_dir): - return False - - logger.info("Downloading OLD tests...") - odl_base_url = 'https://git.opnfv.org/cgit/functest/plain/testcases/Controllers/ODL/CI/' - odl_tests = ['create_venv.sh', 'requirements.pip', 'start_tests.sh', 'test_list.txt'] - for i in odl_tests: - odl_url = odl_base_url + i - logger.debug("Downloading %s" %odl_url) - if not download_url(odl_url,odl_dir): - return False - - return True - - - -def create_glance_image(path,name,disk_format): - """ - Create a glance image given the absolute path of the image, its name and the disk format - """ - cmd = "glance image-create --name "+name+" --is-public true --disk-format "+disk_format+" --container-format bare --file "+path - execute_command(cmd) - return True - - -def download_url(url, dest_path): - """ - Download a file to a destination path given a URL - """ - name = url.rsplit('/')[-1] - dest = dest_path + name - try: - response = urllib2.urlopen(url) - except (urllib2.HTTPError, urllib2.URLError): - logger.error("Error in fetching %s" %url) + if not router_id: return False - with open(dest, 'wb') as f: - f.write(response.read()) - return True + logger.debug("Router '%s' created successfully." % router_id) + logger.info("Adding router to subnet...") + result = functest_utils.add_interface_router(neutron, router_id, subnet_id) -def download_url_with_progress(url, dest_path): - """ - Download a file to a destination path given a URL showing the progress - """ - name = url.rsplit('/')[-1] - dest = dest_path + name - try: - response = urllib2.urlopen(url) - except (urllib2.HTTPError, urllib2.URLError): - logger.error("Error in fetching %s" %url) + if not result: return False - f = open(dest, 'wb') - meta = response.info() - file_size = int(meta.getheaders("Content-Length")[0]) - logger.info("Downloading: %s Bytes: %s" %(dest, file_size)) - - file_size_dl = 0 - block_sz = 8192 - while True: - buffer = response.read(block_sz) - if not buffer: - break - - file_size_dl += len(buffer) - f.write(buffer) - status = r"%10d [%3.2f%%]" % (file_size_dl, file_size_dl * 100. / file_size) - status = status + chr(8)*(len(status)+1) - print status, - - f.close() - print("\n") + logger.debug("Interface added successfully.") + network_dic = {'net_id': network_id, + 'subnet_id': subnet_id, + 'router_id': router_id} return True -def check_internet_connectivity(url='http://www.google.com/'): - """ - Check if there is access to the internet - """ - try: - urllib2.urlopen(url, timeout=5) - return True - except urllib.request.URLError: - return False - -def execute_command(cmd): - """ - Execute Linux command - """ - logger.debug('Executing command : {}'.format(cmd)) - #p = os.popen(cmd,"r") - #logger.debug(p.read()) - output_file = "/tmp/output.txt" - f = open(output_file, 'w+') - p = subprocess.call(cmd,shell=True, stdout=f, stderr=subprocess.STDOUT) - f.close() - f = open(output_file, 'r') - logger.debug(f.read()) - #p = subprocess.call(cmd,shell=True); - if p == 0 : - return True - else: - logger.error("Error when executing command %s" %cmd) - exit(-1) - - - - def main(): if not (args.action in actions): logger.error('argument not valid') exit(-1) + + if not functest_utils.check_credentials(): + logger.error("Please source the openrc credentials and run the script again.") + #TODO: source the credentials in this script + exit(-1) + + if args.action == "start": - config_functest_start() + action_start() if args.action == "check": - if config_functest_check(): + if action_check(): logger.info("Functest environment correctly installed") else: logger.info("Functest environment not found or faulty") if args.action == "clean": - while True: - print("Are you sure? [y|n]") - answer = raw_input("") - if answer == "y": - config_functest_clean() - break - elif answer == "n": - break - else: - print("Invalid option.") + if args.force : + action_clean() + else : + while True: + print("Are you sure? [y|n]") + answer = raw_input("") + if answer == "y": + action_clean() + break + elif answer == "n": + break + else: + print("Invalid option.") exit(0)