X-Git-Url: https://gerrit.opnfv.org/gerrit/gitweb?a=blobdiff_plain;f=functest%2Futils%2Ffunctest_utils.py;h=f07f59d7fd83796160f6a8e3be8b12aadd369730;hb=842d0b381755176a63b55d045325ed8c1d662407;hp=e5e755d7f23a2c7ea7742d69d6a6828a1d116401;hpb=5f82f17a633674c19b69cca877569fd845253343;p=functest.git diff --git a/functest/utils/functest_utils.py b/functest/utils/functest_utils.py index e5e755d7f..f07f59d7f 100644 --- a/functest/utils/functest_utils.py +++ b/functest/utils/functest_utils.py @@ -9,24 +9,26 @@ # import functools import json +import logging import os +import pkg_resources import re import shutil import subprocess import sys import time -import urllib2 from datetime import datetime as dt import dns.resolver import requests +from six.moves import urllib import yaml -from git import Repo +from functest.utils import constants from functest.utils import decorators -import functest.utils.functest_logger as ft_logger +from functest.utils.constants import CONST -logger = ft_logger.Logger("functest_utils").getLogger() +logger = logging.getLogger(__name__) # ---------------------------------------------------------- @@ -39,9 +41,9 @@ def check_internet_connectivity(url='http://www.opnfv.org/'): Check if there is access to the internet """ try: - urllib2.urlopen(url, timeout=5) + urllib.request.urlopen(url, timeout=5) return True - except urllib2.URLError: + except urllib.error.URLError: return False @@ -52,8 +54,8 @@ def download_url(url, dest_path): name = url.rsplit('/')[-1] dest = dest_path + "/" + name try: - response = urllib2.urlopen(url) - except (urllib2.HTTPError, urllib2.URLError): + response = urllib.request.urlopen(url) + except (urllib.error.HTTPError, urllib.error.URLError): return False with open(dest, 'wb') as f: @@ -66,42 +68,6 @@ def download_url(url, dest_path): # CI UTILS # # ----------------------------------------------------------- -def get_git_branch(repo_path): - """ - Get git branch name - """ - repo = Repo(repo_path) - branch = repo.active_branch - return branch.name - - -def get_installer_type(): - """ - Get installer type (fuel, apex, joid, compass) - """ - try: - installer = os.environ['INSTALLER_TYPE'] - except KeyError: - logger.error("Impossible to retrieve the installer type") - installer = "Unknown_installer" - - return installer - - -def get_scenario(): - """ - Get scenario - """ - try: - scenario = os.environ['DEPLOY_SCENARIO'] - except KeyError: - logger.info("Impossible to retrieve the scenario." - "Use default os-nosdn-nofeature-noha") - scenario = "os-nosdn-nofeature-noha" - - return scenario - - def get_version(): """ Get version @@ -111,62 +77,28 @@ def get_version(): # if launched through CI the build tag has the following format # jenkins------ # e.g. jenkins-functest-fuel-opnfv-jump-2-daily-master-190 + # jenkins-functest-fuel-baremetal-weekly-master-8 # use regex to match branch info - rule = "daily-(.+?)-[0-9]*" - build_tag = get_build_tag() + rule = "(dai|week)ly-(.+?)-[0-9]*" + build_tag = CONST.__getattribute__('BUILD_TAG') + if not build_tag: + build_tag = 'none' m = re.search(rule, build_tag) if m: - return m.group(1) + return m.group(2) else: return "unknown" -def get_pod_name(): - """ - Get PoD Name from env variable NODE_NAME - """ - try: - return os.environ['NODE_NAME'] - except KeyError: - logger.info( - "Unable to retrieve the POD name from environment. " + - "Using pod name 'unknown-pod'") - return "unknown-pod" - - -def get_build_tag(): - """ - Get build tag of jenkins jobs - """ - try: - build_tag = os.environ['BUILD_TAG'] - except KeyError: - logger.info("Impossible to retrieve the build tag") - build_tag = "none" - - return build_tag - - -def get_db_url(): +def logger_test_results(project, case_name, status, details): """ - Returns DB URL + Format test case results for the logger """ - # TODO use CONST mechanism - try: - # if TEST_DB_URL declared in env variable, use it! - db_url = os.environ['TEST_DB_URL'] - except KeyError: - logger.info("DB URL not declared as env variable," - "use local configuration") - db_url = get_functest_config('results.test_db_url') - return db_url - - -def logger_test_results(project, case_name, status, details): - pod_name = get_pod_name() - scenario = get_scenario() + pod_name = CONST.__getattribute__('NODE_NAME') + scenario = CONST.__getattribute__('DEPLOY_SCENARIO') version = get_version() - build_tag = get_build_tag() + build_tag = CONST.__getattribute__('BUILD_TAG') + db_url = CONST.__getattribute__("results_test_db_url") logger.info( "\n" @@ -182,7 +114,7 @@ def logger_test_results(project, case_name, status, details): "details:\t%(d)s\n" % {'p': project, 'n': case_name, - 'db': get_db_url(), + 'db': db_url, 'pod': pod_name, 'v': version, 's': scenario, @@ -193,12 +125,12 @@ def logger_test_results(project, case_name, status, details): @decorators.can_dump_request_to_file def push_results_to_db(project, case_name, - start_date, stop_date, criteria, details): + start_date, stop_date, result, details): """ POST results to the Result target DB """ # Retrieve params from CI and conf - url = get_db_url() + "/results" + url = CONST.__getattribute__("results_test_db_url") try: installer = os.environ['INSTALLER_TYPE'] @@ -208,26 +140,21 @@ def push_results_to_db(project, case_name, except KeyError as e: logger.error("Please set env var: " + str(e)) return False - rule = "daily-(.+?)-[0-9]*" - m = re.search(rule, build_tag) - if m: - version = m.group(1) - else: - logger.error("Please fix BUILD_TAG env var: " + build_tag) - return False + version = get_version() test_start = dt.fromtimestamp(start_date).strftime('%Y-%m-%d %H:%M:%S') test_stop = dt.fromtimestamp(stop_date).strftime('%Y-%m-%d %H:%M:%S') params = {"project_name": project, "case_name": case_name, "pod_name": pod_name, "installer": installer, - "version": version, "scenario": scenario, "criteria": criteria, + "version": version, "scenario": scenario, "criteria": result, "build_tag": build_tag, "start_date": test_start, "stop_date": test_stop, "details": details} error = None headers = {'Content-Type': 'application/json'} try: - r = requests.post(url, data=json.dumps(params), headers=headers) + r = requests.post(url, data=json.dumps(params, sort_keys=True), + headers=headers) logger.debug(r) r.raise_for_status() except requests.RequestException as exc: @@ -255,7 +182,7 @@ def push_results_to_db(project, case_name, 'pod': pod_name, 'v': version, 's': scenario, - 'c': criteria, + 'c': result, 't': build_tag, 'd': details, 'error': e @@ -325,7 +252,7 @@ def execute_command(cmd, info=False, error_msg="", f.write(line) else: line = line.replace('\n', '') - print line + print(line) sys.stdout.flush() if output_file: f.close() @@ -339,12 +266,13 @@ def execute_command(cmd, info=False, error_msg="", def get_dict_by_test(testname): - with open(get_testcases_file_dir()) as f: + with open(pkg_resources.resource_filename( + 'functest', 'ci/testcases.yaml')) as f: testcases_yaml = yaml.safe_load(f) for dic_tier in testcases_yaml.get("tiers"): for dic_testcase in dic_tier['testcases']: - if dic_testcase['name'] == testname: + if dic_testcase['case_name'] == testname: return dic_testcase logger.error('Project %s is not defined in testcases.yaml' % testname) @@ -382,29 +310,10 @@ def get_parameter_from_yaml(parameter, file): def get_functest_config(parameter): - yaml_ = os.environ["CONFIG_FUNCTEST_YAML"] + yaml_ = constants.CONST.__getattribute__('CONFIG_FUNCTEST_YAML') return get_parameter_from_yaml(parameter, yaml_) -def check_success_rate(case_name, success_rate): - success_rate = float(success_rate) - criteria = get_criteria_by_test(case_name) - - def get_criteria_value(op): - return float(criteria.split(op)[1].rstrip('%')) - - status = 'FAIL' - ops = ['==', '>='] - for op in ops: - if op in criteria: - c_value = get_criteria_value(op) - if eval("%s %s %s" % (success_rate, op, c_value)): - status = 'PASS' - break - - return status - - def merge_dicts(dict1, dict2): for k in set(dict1.keys()).union(dict2.keys()): if k in dict1 and k in dict2: @@ -418,12 +327,8 @@ def merge_dicts(dict1, dict2): yield (k, dict2[k]) -def get_testcases_file_dir(): - return get_functest_config('general.functest.testcases_yaml') - - def get_functest_yaml(): - with open(os.environ["CONFIG_FUNCTEST_YAML"]) as f: + with open(constants.CONST.__getattribute__('CONFIG_FUNCTEST_YAML')) as f: functest_yaml = yaml.safe_load(f) f.close() return functest_yaml