3 # jose.lausuch@ericsson.com
4 # valentin.boucher@orange.com
5 # All rights reserved. This program and the accompanying materials
6 # are made available under the terms of the Apache License, Version 2.0
7 # which accompanies this distribution, and is available at
8 # http://www.apache.org/licenses/LICENSE-2.0
19 from datetime import datetime as dt
26 from functest.utils.constants import CONST
27 import functest.utils.functest_logger as ft_logger
30 logger = ft_logger.Logger("functest_utils").getLogger()
33 # ----------------------------------------------------------
37 # -----------------------------------------------------------
38 def check_internet_connectivity(url='http://www.opnfv.org/'):
40 Check if there is access to the internet
43 urllib2.urlopen(url, timeout=5)
45 except urllib2.URLError:
49 def download_url(url, dest_path):
51 Download a file to a destination path given a URL
53 name = url.rsplit('/')[-1]
54 dest = dest_path + "/" + name
56 response = urllib2.urlopen(url)
57 except (urllib2.HTTPError, urllib2.URLError):
60 with open(dest, 'wb') as f:
61 shutil.copyfileobj(response, f)
65 # ----------------------------------------------------------
69 # -----------------------------------------------------------
70 def get_git_branch(repo_path):
74 repo = Repo(repo_path)
75 branch = repo.active_branch
79 def get_installer_type():
81 Get installer type (fuel, apex, joid, compass)
84 installer = os.environ['INSTALLER_TYPE']
86 logger.error("Impossible to retrieve the installer type")
87 installer = "Unknown_installer"
97 scenario = os.environ['DEPLOY_SCENARIO']
99 logger.error("Impossible to retrieve the scenario")
100 scenario = "Unknown_scenario"
109 # Use the build tag to retrieve the version
110 # By default version is unknown
111 # if launched through CI the build tag has the following format
112 # jenkins-<project>-<installer>-<pod>-<job>-<branch>-<id>
113 # e.g. jenkins-functest-fuel-opnfv-jump-2-daily-master-190
114 # use regex to match branch info
115 rule = "daily-(.+?)-[0-9]*"
116 build_tag = get_build_tag()
117 m = re.search(rule, build_tag)
126 Get PoD Name from env variable NODE_NAME
129 return os.environ['NODE_NAME']
132 "Unable to retrieve the POD name from environment. " +
133 "Using pod name 'unknown-pod'")
139 Get build tag of jenkins jobs
142 build_tag = os.environ['BUILD_TAG']
144 logger.error("Impossible to retrieve the build tag")
145 build_tag = "unknown_build_tag"
154 return get_functest_config('results.test_db_url')
157 def logger_test_results(project, case_name, status, details):
158 pod_name = get_pod_name()
159 scenario = get_scenario()
160 version = get_version()
161 build_tag = get_build_tag()
165 "****************************************\n"
166 "\t %(p)s/%(n)s results \n\n"
167 "****************************************\n"
173 "build tag:\t%(b)s\n"
186 def write_results_to_file(project, case_name, start_date,
187 stop_date, criteria, details):
188 file_path = re.split(r'://', CONST.results_test_db_url)[1]
191 installer = os.environ['INSTALLER_TYPE']
192 scenario = os.environ['DEPLOY_SCENARIO']
193 pod_name = os.environ['NODE_NAME']
194 except KeyError as e:
195 logger.error("Please set env var: " + str(e))
198 test_start = dt.fromtimestamp(start_date).strftime('%Y-%m-%d %H:%M:%S')
199 test_stop = dt.fromtimestamp(stop_date).strftime('%Y-%m-%d %H:%M:%S')
201 params = {"project_name": project, "case_name": case_name,
202 "pod_name": pod_name, "installer": installer,
203 "scenario": scenario, "criteria": criteria,
204 "start_date": test_start, "stop_date": test_stop,
207 with open(file_path, "a+w") as outfile:
208 json.dump(params, outfile)
211 except Exception as e:
212 logger.error("write result data into a file failed: %s" % e)
216 def push_results_to_db(project, case_name,
217 start_date, stop_date, criteria, details):
219 POST results to the Result target DB
221 # Retrieve params from CI and conf
222 url = CONST.results_test_db_url + "/results"
225 installer = os.environ['INSTALLER_TYPE']
226 scenario = os.environ['DEPLOY_SCENARIO']
227 pod_name = os.environ['NODE_NAME']
228 build_tag = os.environ['BUILD_TAG']
229 except KeyError as e:
230 logger.error("Please set env var: " + str(e))
232 rule = "daily-(.+?)-[0-9]*"
233 m = re.search(rule, build_tag)
237 logger.error("Please fix BUILD_TAG env var: " + build_tag)
239 test_start = dt.fromtimestamp(start_date).strftime('%Y-%m-%d %H:%M:%S')
240 test_stop = dt.fromtimestamp(stop_date).strftime('%Y-%m-%d %H:%M:%S')
242 params = {"project_name": project, "case_name": case_name,
243 "pod_name": pod_name, "installer": installer,
244 "version": version, "scenario": scenario, "criteria": criteria,
245 "build_tag": build_tag, "start_date": test_start,
246 "stop_date": test_stop, "details": details}
249 headers = {'Content-Type': 'application/json'}
251 r = requests.post(url, data=json.dumps(params), headers=headers)
254 except requests.RequestException as exc:
256 error = ("Pushing Result to DB(%s) failed: %s" %
259 error = ("Pushing Result to DB(%s) failed: %s" % (url, exc))
260 except Exception as e:
261 error = ("Error [push_results_to_db("
263 "project: '%(project)s', "
267 "scenario: '%(s)s', "
268 "criteria: '%(c)s', "
269 "build_tag: '%(t)s', "
270 "details: '%(d)s')]: "
291 def get_resolvconf_ns():
293 Get nameservers from current resolv.conf
296 rconf = open("/etc/resolv.conf", "r")
297 line = rconf.readline()
298 resolver = dns.resolver.Resolver()
300 ip = re.search(r"\b(?:[0-9]{1,3}\.){3}[0-9]{1,3}\b", line)
302 resolver.nameservers = [str(ip)]
304 result = resolver.query('opnfv.org')[0]
306 nameservers.append(ip.group())
307 except dns.exception.Timeout:
309 line = rconf.readline()
313 def get_ci_envvars():
315 Get the CI env variables
318 "installer": os.environ.get('INSTALLER_TYPE'),
319 "scenario": os.environ.get('DEPLOY_SCENARIO')}
323 def execute_command(cmd, info=False, error_msg="",
324 verbose=True, output_file=None):
326 error_msg = ("The command '%s' failed." % cmd)
327 msg_exec = ("Executing command: '%s'" % cmd)
330 logger.info(msg_exec)
332 logger.debug(msg_exec)
333 p = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE,
334 stderr=subprocess.STDOUT)
336 f = open(output_file, "w")
337 for line in iter(p.stdout.readline, b''):
341 line = line.replace('\n', '')
347 returncode = p.wait()
350 logger.error(error_msg)
355 def get_dict_by_test(testname):
356 with open(get_testcases_file_dir()) as f:
357 testcases_yaml = yaml.safe_load(f)
359 for dic_tier in testcases_yaml.get("tiers"):
360 for dic_testcase in dic_tier['testcases']:
361 if dic_testcase['name'] == testname:
364 logger.error('Project %s is not defined in testcases.yaml' % testname)
368 def get_criteria_by_test(testname):
369 dict = get_dict_by_test(testname)
371 return dict['criteria']
375 # ----------------------------------------------------------
379 # -----------------------------------------------------------
380 def get_parameter_from_yaml(parameter, file):
382 Returns the value of a given parameter in file.yaml
383 parameter must be given in string format with dots
384 Example: general.openstack.image_name
386 with open(file) as f:
387 file_yaml = yaml.safe_load(f)
390 for element in parameter.split("."):
391 value = value.get(element)
393 raise ValueError("The parameter %s is not defined in"
394 " config_functest.yaml" % parameter)
398 def get_functest_config(parameter):
399 yaml_ = os.environ["CONFIG_FUNCTEST_YAML"]
400 return get_parameter_from_yaml(parameter, yaml_)
403 def check_success_rate(case_name, success_rate):
404 success_rate = float(success_rate)
405 criteria = get_criteria_by_test(case_name)
407 def get_criteria_value(op):
408 return float(criteria.split(op)[1].rstrip('%'))
414 c_value = get_criteria_value(op)
415 if eval("%s %s %s" % (success_rate, op, c_value)):
422 def merge_dicts(dict1, dict2):
423 for k in set(dict1.keys()).union(dict2.keys()):
424 if k in dict1 and k in dict2:
425 if isinstance(dict1[k], dict) and isinstance(dict2[k], dict):
426 yield (k, dict(merge_dicts(dict1[k], dict2[k])))
435 def get_testcases_file_dir():
436 return get_functest_config('general.functest.testcases_yaml')
439 def get_functest_yaml():
440 with open(os.environ["CONFIG_FUNCTEST_YAML"]) as f:
441 functest_yaml = yaml.safe_load(f)
446 def print_separator():
447 logger.info("==============================================")
451 """Measure the time it takes for a function to complete"""
452 @functools.wraps(func)
453 def timed(*args, **kwargs):
455 result = func(*args, **kwargs)
457 elapsed = '{0}'.format(te - ts)
458 logger.info('{f}(*{a}, **{kw}) took: {t} sec'.format(
459 f=func.__name__, a=args, kw=kwargs, t=elapsed))
460 return result, elapsed