3 # jose.lausuch@ericsson.com
4 # valentin.boucher@orange.com
5 # All rights reserved. This program and the accompanying materials
6 # are made available under the terms of the Apache License, Version 2.0
7 # which accompanies this distribution, and is available at
8 # http://www.apache.org/licenses/LICENSE-2.0
17 from datetime import datetime as dt
24 import functest.utils.functest_logger as ft_logger
26 logger = ft_logger.Logger("functest_utils").getLogger()
29 # ----------------------------------------------------------
33 # -----------------------------------------------------------
34 def check_internet_connectivity(url='http://www.opnfv.org/'):
36 Check if there is access to the internet
39 urllib2.urlopen(url, timeout=5)
41 except urllib2.URLError:
45 def download_url(url, dest_path):
47 Download a file to a destination path given a URL
49 name = url.rsplit('/')[-1]
50 dest = dest_path + "/" + name
52 response = urllib2.urlopen(url)
53 except (urllib2.HTTPError, urllib2.URLError):
56 with open(dest, 'wb') as f:
57 shutil.copyfileobj(response, f)
61 # ----------------------------------------------------------
65 # -----------------------------------------------------------
66 def get_git_branch(repo_path):
70 repo = Repo(repo_path)
71 branch = repo.active_branch
75 def get_installer_type():
77 Get installer type (fuel, apex, joid, compass)
80 installer = os.environ['INSTALLER_TYPE']
82 logger.error("Impossible to retrieve the installer type")
83 installer = "Unknown_installer"
93 scenario = os.environ['DEPLOY_SCENARIO']
95 logger.error("Impossible to retrieve the scenario")
96 scenario = "Unknown_scenario"
105 # Use the build tag to retrieve the version
106 # By default version is unknown
107 # if launched through CI the build tag has the following format
108 # jenkins-<project>-<installer>-<pod>-<job>-<branch>-<id>
109 # e.g. jenkins-functest-fuel-opnfv-jump-2-daily-master-190
110 # use regex to match branch info
111 rule = "daily-(.+?)-[0-9]*"
112 build_tag = get_build_tag()
113 m = re.search(rule, build_tag)
122 Get PoD Name from env variable NODE_NAME
125 return os.environ['NODE_NAME']
128 "Unable to retrieve the POD name from environment. " +
129 "Using pod name 'unknown-pod'")
135 Get build tag of jenkins jobs
138 build_tag = os.environ['BUILD_TAG']
140 logger.error("Impossible to retrieve the build tag")
141 build_tag = "unknown_build_tag"
150 return get_functest_config('results.test_db_url')
153 def logger_test_results(project, case_name, status, details):
154 pod_name = get_pod_name()
155 scenario = get_scenario()
156 version = get_version()
157 build_tag = get_build_tag()
161 "****************************************\n"
162 "\t %(p)s/%(n)s results \n\n"
163 "****************************************\n"
169 "build tag:\t%(b)s\n"
182 def push_results_to_db(project, case_name,
183 start_date, stop_date, criteria, details):
185 POST results to the Result target DB
187 # Retrieve params from CI and conf
188 url = get_db_url() + "/results"
191 installer = os.environ['INSTALLER_TYPE']
192 scenario = os.environ['DEPLOY_SCENARIO']
193 pod_name = os.environ['NODE_NAME']
194 build_tag = os.environ['BUILD_TAG']
195 except KeyError as e:
196 logger.error("Please set env var: " + str(e))
198 rule = "daily-(.+?)-[0-9]*"
199 m = re.search(rule, build_tag)
203 logger.error("Please fix BUILD_TAG env var: " + build_tag)
205 test_start = dt.fromtimestamp(start_date).strftime('%Y-%m-%d %H:%M:%S')
206 test_stop = dt.fromtimestamp(stop_date).strftime('%Y-%m-%d %H:%M:%S')
208 params = {"project_name": project, "case_name": case_name,
209 "pod_name": pod_name, "installer": installer,
210 "version": version, "scenario": scenario, "criteria": criteria,
211 "build_tag": build_tag, "start_date": test_start,
212 "stop_date": test_stop, "details": details}
215 headers = {'Content-Type': 'application/json'}
217 r = requests.post(url, data=json.dumps(params), headers=headers)
220 except requests.RequestException as exc:
222 error = ("Pushing Result to DB(%s) failed: %s" %
225 error = ("Pushing Result to DB(%s) failed: %s" % (url, exc))
226 except Exception as e:
227 error = ("Error [push_results_to_db("
229 "project: '%(project)s', "
233 "scenario: '%(s)s', "
234 "criteria: '%(c)s', "
235 "build_tag: '%(t)s', "
236 "details: '%(d)s')]: "
257 def get_resolvconf_ns():
259 Get nameservers from current resolv.conf
262 rconf = open("/etc/resolv.conf", "r")
263 line = rconf.readline()
264 resolver = dns.resolver.Resolver()
266 ip = re.search(r"\b(?:[0-9]{1,3}\.){3}[0-9]{1,3}\b", line)
268 resolver.nameservers = [str(ip)]
270 result = resolver.query('opnfv.org')[0]
272 nameservers.append(ip.group())
273 except dns.exception.Timeout:
275 line = rconf.readline()
279 def get_ci_envvars():
281 Get the CI env variables
284 "installer": os.environ.get('INSTALLER_TYPE'),
285 "scenario": os.environ.get('DEPLOY_SCENARIO')}
289 def execute_command(cmd, info=False, error_msg="",
290 verbose=True, output_file=None):
292 error_msg = ("The command '%s' failed." % cmd)
293 msg_exec = ("Executing command: '%s'" % cmd)
296 logger.info(msg_exec)
298 logger.debug(msg_exec)
299 p = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE,
300 stderr=subprocess.STDOUT)
302 f = open(output_file, "w")
303 for line in iter(p.stdout.readline, b''):
307 line = line.replace('\n', '')
313 returncode = p.wait()
316 logger.error(error_msg)
321 def get_deployment_dir():
323 Returns current Rally deployment directory
325 deployment_name = get_functest_config('rally.deployment_name')
326 rally_dir = get_functest_config('general.directories.dir_rally_inst')
327 cmd = ("rally deployment list | awk '/" + deployment_name +
329 p = subprocess.Popen(cmd, shell=True,
330 stdout=subprocess.PIPE,
331 stderr=subprocess.STDOUT)
332 deployment_uuid = p.stdout.readline().rstrip()
333 if deployment_uuid == "":
334 logger.error("Rally deployment not found.")
336 deployment_dir = (rally_dir + "/tempest/for-deployment-" +
338 return deployment_dir
341 def get_dict_by_test(testname):
342 with open(get_testcases_file_dir()) as f:
343 testcases_yaml = yaml.safe_load(f)
345 for dic_tier in testcases_yaml.get("tiers"):
346 for dic_testcase in dic_tier['testcases']:
347 if dic_testcase['name'] == testname:
350 logger.error('Project %s is not defined in testcases.yaml' % testname)
354 def get_criteria_by_test(testname):
355 dict = get_dict_by_test(testname)
357 return dict['criteria']
361 # ----------------------------------------------------------
365 # -----------------------------------------------------------
366 def get_parameter_from_yaml(parameter, file):
368 Returns the value of a given parameter in file.yaml
369 parameter must be given in string format with dots
370 Example: general.openstack.image_name
372 with open(file) as f:
373 file_yaml = yaml.safe_load(f)
376 for element in parameter.split("."):
377 value = value.get(element)
379 raise ValueError("The parameter %s is not defined in"
380 " config_functest.yaml" % parameter)
384 def get_functest_config(parameter):
385 yaml_ = os.environ["CONFIG_FUNCTEST_YAML"]
386 return get_parameter_from_yaml(parameter, yaml_)
389 def check_success_rate(case_name, success_rate):
390 success_rate = float(success_rate)
391 criteria = get_criteria_by_test(case_name)
393 def get_criteria_value(op):
394 return float(criteria.split(op)[1].rstrip('%'))
400 c_value = get_criteria_value(op)
401 if eval("%s %s %s" % (success_rate, op, c_value)):
408 def merge_dicts(dict1, dict2):
409 for k in set(dict1.keys()).union(dict2.keys()):
410 if k in dict1 and k in dict2:
411 if isinstance(dict1[k], dict) and isinstance(dict2[k], dict):
412 yield (k, dict(merge_dicts(dict1[k], dict2[k])))
421 def check_test_result(test_name, ret, start_time, stop_time):
422 def get_criteria_value():
423 return get_criteria_by_test(test_name).split('==')[1].strip()
426 if str(ret) == get_criteria_value():
430 'timestart': start_time,
431 'duration': round(stop_time - start_time, 1),
435 return status, details
438 def get_testcases_file_dir():
439 return "/home/opnfv/repos/functest/functest/ci/testcases.yaml"
442 def get_functest_yaml():
443 with open(os.environ["CONFIG_FUNCTEST_YAML"]) as f:
444 functest_yaml = yaml.safe_load(f)
449 def print_separator():
450 logger.info("==============================================")