utils/test/reporting/functest/reporting-status.py

   1 #!/usr/bin/python
   2 #
   3 # This program and the accompanying materials
   4 # are made available under the terms of the Apache License, Version 2.0
   5 # which accompanies this distribution, and is available at
   6 #
   7 # http://www.apache.org/licenses/LICENSE-2.0
   8 #
   9 import datetime
  10 import jinja2
  11 import os
  12 import requests
  13 import sys
  14 import time
  15 import yaml
  16
  17 import testCase as tc
  18 import scenarioResult as sr
  19
  20 # manage conf
  21 import utils.reporting_utils as rp_utils
  22
  23 # Logger
  24 logger = rp_utils.getLogger("Functest-Status")
  25
  26 # Initialization
  27 testValid = []
  28 otherTestCases = []
  29 reportingDate = datetime.datetime.now().strftime("%Y-%m-%d %H:%M")
  30
  31 # init just connection_check to get the list of scenarios
  32 # as all the scenarios run connection_check
  33 healthcheck = tc.TestCase("connection_check", "functest", -1)
  34
  35 # Retrieve the Functest configuration to detect which tests are relevant
  36 # according to the installer, scenario
  37 cf = rp_utils.get_config('functest.test_conf')
  38 period = rp_utils.get_config('general.period')
  39 versions = rp_utils.get_config('general.versions')
  40 installers = rp_utils.get_config('general.installers')
  41 blacklist = rp_utils.get_config('functest.blacklist')
  42 log_level = rp_utils.get_config('general.log.log_level')
  43 exclude_noha = rp_utils.get_config('functest.exclude_noha')
  44 exclude_virtual = rp_utils.get_config('functest.exclude_virtual')
  45
  46 response = requests.get(cf)
  47
  48 functest_yaml_config = yaml.safe_load(response.text)
  49
  50 logger.info("*******************************************")
  51 logger.info("*                                         *")
  52 logger.info("*   Generating reporting scenario status  *")
  53 logger.info("*   Data retention: %s days               *" % period)
  54 logger.info("*   Log level: %s                         *" % log_level)
  55 logger.info("*                                         *")
  56 logger.info("*   Virtual PODs exluded: %s              *" % exclude_virtual)
  57 logger.info("*   NOHA scenarios excluded: %s           *" % exclude_noha)
  58 logger.info("*                                         *")
  59 logger.info("*******************************************")
  60
  61 # Retrieve test cases of Tier 1 (smoke)
  62 config_tiers = functest_yaml_config.get("tiers")
  63
  64 # we consider Tier 0 (Healthcheck), Tier 1 (smoke),2 (features)
  65 # to validate scenarios
  66 # Tier > 2 are not used to validate scenarios but we display the results anyway
  67 # tricky thing for the API as some tests are Functest tests
  68 # other tests are declared directly in the feature projects
  69 for tier in config_tiers:
  70     if tier['order'] >= 0 and tier['order'] < 2:
  71         for case in tier['testcases']:
  72             if case['name'] not in blacklist:
  73                 testValid.append(tc.TestCase(case['name'],
  74                                              "functest",
  75                                              case['dependencies']))
  76     elif tier['order'] == 2:
  77         for case in tier['testcases']:
  78             if case['name'] not in blacklist:
  79                 testValid.append(tc.TestCase(case['name'],
  80                                              case['name'],
  81                                              case['dependencies']))
  82     elif tier['order'] > 2:
  83         for case in tier['testcases']:
  84             if case['name'] not in blacklist:
  85                 otherTestCases.append(tc.TestCase(case['name'],
  86                                                   "functest",
  87                                                   case['dependencies']))
  88
  89 logger.debug("Functest reporting start")
  90 # For all the versions
  91 for version in versions:
  92     # For all the installers
  93     for installer in installers:
  94         # get scenarios
  95         scenario_results = rp_utils.getScenarios(healthcheck,
  96                                                  installer,
  97                                                  version)
  98         scenario_stats = rp_utils.getScenarioStats(scenario_results)
  99         items = {}
 100         scenario_result_criteria = {}
 101         scenario_directory = "./display/" + version + "/functest/"
 102         scenario_file_name = scenario_directory + "scenario_history.txt"
 103
 104         # check that the directory exists, if not create it
 105         # (first run on new version)
 106         if not os.path.exists(scenario_directory):
 107             os.makedirs(scenario_directory)
 108
 109         # initiate scenario file if it does not exist
 110         if not os.path.isfile(scenario_file_name):
 111             with open(scenario_file_name, "a") as my_file:
 112                 logger.debug("Create scenario file: %s" % scenario_file_name)
 113                 my_file.write("date,scenario,installer,detail,score\n")
 114
 115         # For all the scenarios get results
 116         for s, s_result in scenario_results.items():
 117             logger.info("---------------------------------")
 118             logger.info("installer %s, version %s, scenario %s:" %
 119                         (installer, version, s))
 120             logger.debug("Scenario results: %s" % s_result)
 121
 122             # Green or Red light for a given scenario
 123             nb_test_runnable_for_this_scenario = 0
 124             scenario_score = 0
 125             # url of the last jenkins log corresponding to a given
 126             # scenario
 127             s_url = ""
 128             if len(s_result) > 0:
 129                 build_tag = s_result[len(s_result)-1]['build_tag']
 130                 logger.debug("Build tag: %s" % build_tag)
 131                 s_url = rp_utils.getJenkinsUrl(build_tag)
 132                 if s_url is None:
 133                     s_url = "http://testresultS.opnfv.org/reporting"
 134                 logger.info("last jenkins url: %s" % s_url)
 135             testCases2BeDisplayed = []
 136             # Check if test case is runnable / installer, scenario
 137             # for the test case used for Scenario validation
 138             try:
 139                 # 1) Manage the test cases for the scenario validation
 140                 # concretely Tiers 0-3
 141                 for test_case in testValid:
 142                     test_case.checkRunnable(installer, s,
 143                                             test_case.getConstraints())
 144                     logger.debug("testcase %s (%s) is %s" %
 145                                  (test_case.getDisplayName(),
 146                                   test_case.getName(),
 147                                   test_case.isRunnable))
 148                     time.sleep(1)
 149                     if test_case.isRunnable:
 150                         dbName = test_case.getDbName()
 151                         name = test_case.getName()
 152                         displayName = test_case.getDisplayName()
 153                         project = test_case.getProject()
 154                         nb_test_runnable_for_this_scenario += 1
 155                         logger.info(" Searching results for case %s " %
 156                                     (displayName))
 157                         result = rp_utils.getResult(dbName, installer,
 158                                                     s, version)
 159                         # if no result set the value to 0
 160                         if result < 0:
 161                             result = 0
 162                         logger.info(" >>>> Test score = " + str(result))
 163                         test_case.setCriteria(result)
 164                         test_case.setIsRunnable(True)
 165                         testCases2BeDisplayed.append(tc.TestCase(name,
 166                                                                  project,
 167                                                                  "",
 168                                                                  result,
 169                                                                  True,
 170                                                                  1))
 171                         scenario_score = scenario_score + result
 172
 173                 # 2) Manage the test cases for the scenario qualification
 174                 # concretely Tiers > 3
 175                 for test_case in otherTestCases:
 176                     test_case.checkRunnable(installer, s,
 177                                             test_case.getConstraints())
 178                     logger.debug("testcase %s (%s) is %s" %
 179                                  (test_case.getDisplayName(),
 180                                   test_case.getName(),
 181                                   test_case.isRunnable))
 182                     time.sleep(1)
 183                     if test_case.isRunnable:
 184                         dbName = test_case.getDbName()
 185                         name = test_case.getName()
 186                         displayName = test_case.getDisplayName()
 187                         project = test_case.getProject()
 188                         logger.info(" Searching results for case %s " %
 189                                     (displayName))
 190                         result = rp_utils.getResult(dbName, installer,
 191                                                     s, version)
 192                         # at least 1 result for the test
 193                         if result > -1:
 194                             test_case.setCriteria(result)
 195                             test_case.setIsRunnable(True)
 196                             testCases2BeDisplayed.append(tc.TestCase(name,
 197                                                                      project,
 198                                                                      "",
 199                                                                      result,
 200                                                                      True,
 201                                                                      4))
 202                         else:
 203                             logger.debug("No results found")
 204
 205                     items[s] = testCases2BeDisplayed
 206             except:
 207                 logger.error("Error: installer %s, version %s, scenario %s" %
 208                              (installer, version, s))
 209                 logger.error("No data available: %s " % (sys.exc_info()[0]))
 210
 211             # **********************************************
 212             # Evaluate the results for scenario validation
 213             # **********************************************
 214             # the validation criteria = nb runnable tests x 3
 215             # because each test case = 0,1,2 or 3
 216             scenario_criteria = nb_test_runnable_for_this_scenario * 3
 217             # if 0 runnable tests set criteria at a high value
 218             if scenario_criteria < 1:
 219                 scenario_criteria = 50  # conf.MAX_SCENARIO_CRITERIA
 220
 221             s_score = str(scenario_score) + "/" + str(scenario_criteria)
 222             s_score_percent = rp_utils.getScenarioPercent(scenario_score,
 223                                                           scenario_criteria)
 224
 225             s_status = "KO"
 226             if scenario_score < scenario_criteria:
 227                 logger.info(">>>> scenario not OK, score = %s/%s" %
 228                             (scenario_score, scenario_criteria))
 229                 s_status = "KO"
 230             else:
 231                 logger.info(">>>>> scenario OK, save the information")
 232                 s_status = "OK"
 233                 path_validation_file = ("./display/" + version +
 234                                         "/functest/" +
 235                                         "validated_scenario_history.txt")
 236                 with open(path_validation_file, "a") as f:
 237                     time_format = "%Y-%m-%d %H:%M"
 238                     info = (datetime.datetime.now().strftime(time_format) +
 239                             ";" + installer + ";" + s + "\n")
 240                     f.write(info)
 241
 242             # Save daily results in a file
 243             with open(scenario_file_name, "a") as f:
 244                 info = (reportingDate + "," + s + "," + installer +
 245                         "," + s_score + "," +
 246                         str(round(s_score_percent)) + "\n")
 247                 f.write(info)
 248
 249             scenario_result_criteria[s] = sr.ScenarioResult(s_status,
 250                                                             s_score,
 251                                                             s_score_percent,
 252                                                             s_url)
 253             logger.info("--------------------------")
 254
 255         templateLoader = jinja2.FileSystemLoader(".")
 256         templateEnv = jinja2.Environment(
 257             loader=templateLoader, autoescape=True)
 258
 259         TEMPLATE_FILE = "./functest/template/index-status-tmpl.html"
 260         template = templateEnv.get_template(TEMPLATE_FILE)
 261
 262         outputText = template.render(scenario_stats=scenario_stats,
 263                                      scenario_results=scenario_result_criteria,
 264                                      items=items,
 265                                      installer=installer,
 266                                      period=period,
 267                                      version=version,
 268                                      date=reportingDate)
 269
 270         with open("./display/" + version +
 271                   "/functest/status-" + installer + ".html", "wb") as fh:
 272             fh.write(outputText)
 273
 274         logger.info("Manage export CSV & PDF")
 275         rp_utils.export_csv(scenario_file_name, installer, version)
 276         logger.error("CSV generated...")
 277
 278         # Generate outputs for export
 279         # pdf
 280         # TODO Change once web site updated...use the current one
 281         # to test pdf production
 282         url_pdf = rp_utils.get_config('general.url')
 283         pdf_path = ("./display/" + version +
 284                     "/functest/status-" + installer + ".html")
 285         pdf_doc_name = ("./display/" + version +
 286                         "/functest/status-" + installer + ".pdf")
 287         rp_utils.export_pdf(pdf_path, pdf_doc_name)
 288         logger.info("PDF generated...")