utils/test/reporting/functest/reporting-tempest.py

   1 #!/usr/bin/env python
   2
   3 # Copyright (c) 2017 Orange and others.
   4 #
   5 # All rights reserved. This program and the accompanying materials
   6 # are made available under the terms of the Apache License, Version 2.0
   7 # which accompanies this distribution, and is available at
   8 # http://www.apache.org/licenses/LICENSE-2.0
   9 # SPDX-license-identifier: Apache-2.0
  10
  11 from urllib2 import Request, urlopen, URLError
  12 from datetime import datetime
  13 import json
  14 import jinja2
  15 import os
  16
  17 # manage conf
  18 import utils.reporting_utils as rp_utils
  19
  20 installers = rp_utils.get_config('general.installers')
  21 items = ["tests", "Success rate", "duration"]
  22
  23 CURRENT_DIR = os.getcwd()
  24
  25 PERIOD = rp_utils.get_config('general.period')
  26 criteria_nb_test = 165
  27 criteria_duration = 1800
  28 criteria_success_rate = 90
  29
  30 logger = rp_utils.getLogger("Tempest")
  31 logger.info("************************************************")
  32 logger.info("*   Generating reporting Tempest_smoke_serial  *")
  33 logger.info("*   Data retention = %s days                   *" % PERIOD)
  34 logger.info("*                                              *")
  35 logger.info("************************************************")
  36
  37 logger.info("Success criteria:")
  38 logger.info("nb tests executed > %s s " % criteria_nb_test)
  39 logger.info("test duration < %s s " % criteria_duration)
  40 logger.info("success rate > %s " % criteria_success_rate)
  41
  42 # For all the versions
  43 for version in rp_utils.get_config('general.versions'):
  44     for installer in installers:
  45         # we consider the Tempest results of the last PERIOD days
  46         url = ("http://" + rp_utils.get_config('testapi.url') +
  47                "?case=tempest_smoke_serial")
  48         request = Request(url + '&period=' + str(PERIOD) +
  49                           '&installer=' + installer +
  50                           '&version=' + version)
  51         logger.info("Search tempest_smoke_serial results for installer %s"
  52                     " for version %s"
  53                     % (installer, version))
  54         try:
  55             response = urlopen(request)
  56             k = response.read()
  57             results = json.loads(k)
  58         except URLError as e:
  59             logger.error("Error code: %s" % e)
  60
  61         test_results = results['results']
  62
  63         scenario_results = {}
  64         criteria = {}
  65         errors = {}
  66
  67         for r in test_results:
  68             # Retrieve all the scenarios per installer
  69             # In Brahmaputra use version
  70             # Since Colorado use scenario
  71             if not r['scenario'] in scenario_results.keys():
  72                 scenario_results[r['scenario']] = []
  73             scenario_results[r['scenario']].append(r)
  74
  75         for s, s_result in scenario_results.items():
  76             scenario_results[s] = s_result[0:5]
  77             # For each scenario, we build a result object to deal with
  78             # results, criteria and error handling
  79             for result in scenario_results[s]:
  80                 result["start_date"] = result["start_date"].split(".")[0]
  81
  82                 # retrieve results
  83                 # ****************
  84                 nb_tests_run = result['details']['tests']
  85                 nb_tests_failed = result['details']['failures']
  86                 if nb_tests_run != 0:
  87                     success_rate = 100 * ((int(nb_tests_run) -
  88                                            int(nb_tests_failed)) /
  89                                           int(nb_tests_run))
  90                 else:
  91                     success_rate = 0
  92
  93                 result['details']["tests"] = nb_tests_run
  94                 result['details']["Success rate"] = str(success_rate) + "%"
  95
  96                 # Criteria management
  97                 # *******************
  98                 crit_tests = False
  99                 crit_rate = False
 100                 crit_time = False
 101
 102                 # Expect that at least 165 tests are run
 103                 if nb_tests_run >= criteria_nb_test:
 104                     crit_tests = True
 105
 106                 # Expect that at least 90% of success
 107                 if success_rate >= criteria_success_rate:
 108                     crit_rate = True
 109
 110                 # Expect that the suite duration is inferior to 30m
 111                 stop_date = datetime.strptime(result['stop_date'],
 112                                               '%Y-%m-%d %H:%M:%S')
 113                 start_date = datetime.strptime(result['start_date'],
 114                                                '%Y-%m-%d %H:%M:%S')
 115
 116                 delta = stop_date - start_date
 117                 if (delta.total_seconds() < criteria_duration):
 118                     crit_time = True
 119
 120                 result['criteria'] = {'tests': crit_tests,
 121                                       'Success rate': crit_rate,
 122                                       'duration': crit_time}
 123                 try:
 124                     logger.debug("Scenario %s, Installer %s"
 125                                  % (s_result[1]['scenario'], installer))
 126                     logger.debug("Nb Test run: %s" % nb_tests_run)
 127                     logger.debug("Test duration: %s"
 128                                  % result['details']['duration'])
 129                     logger.debug("Success rate: %s" % success_rate)
 130                 except:
 131                     logger.error("Data format error")
 132
 133                 # Error management
 134                 # ****************
 135                 try:
 136                     errors = result['details']['errors']
 137                     result['errors'] = errors.replace('{0}', '')
 138                 except:
 139                     logger.error("Error field not present (Brahamputra runs?)")
 140
 141         templateLoader = jinja2.FileSystemLoader(".")
 142         templateEnv = jinja2.Environment(loader=templateLoader,
 143                                          autoescape=True)
 144
 145         TEMPLATE_FILE = "./functest/template/index-tempest-tmpl.html"
 146         template = templateEnv.get_template(TEMPLATE_FILE)
 147
 148         outputText = template.render(scenario_results=scenario_results,
 149                                      items=items,
 150                                      installer=installer)
 151
 152         with open("./display/" + version +
 153                   "/functest/tempest-" + installer + ".html", "wb") as fh:
 154             fh.write(outputText)
 155 logger.info("Tempest automatic reporting succesfully generated.")