utils/test/reporting/reporting/functest/reporting-status.py

   1 #!/usr/bin/python
   2 #
   3 # This program and the accompanying materials
   4 # are made available under the terms of the Apache License, Version 2.0
   5 # which accompanies this distribution, and is available at
   6 #
   7 # http://www.apache.org/licenses/LICENSE-2.0
   8 #
   9 import datetime
  10 import os
  11 import sys
  12 import time
  13
  14 import jinja2
  15
  16 import reporting.functest.testCase as tc
  17 import reporting.functest.scenarioResult as sr
  18 import reporting.utils.reporting_utils as rp_utils
  19
  20 """
  21 Functest reporting status
  22 """
  23
  24 # Logger
  25 LOGGER = rp_utils.getLogger("Functest-Status")
  26
  27 # Initialization
  28 testValid = []
  29 otherTestCases = []
  30 reportingDate = datetime.datetime.now().strftime("%Y-%m-%d %H:%M")
  31
  32 # init just connection_check to get the list of scenarios
  33 # as all the scenarios run connection_check
  34 healthcheck = tc.TestCase("connection_check", "functest", -1)
  35
  36 # Retrieve the Functest configuration to detect which tests are relevant
  37 # according to the installer, scenario
  38 cf = rp_utils.get_config('functest.test_conf')
  39 period = rp_utils.get_config('general.period')
  40 versions = rp_utils.get_config('general.versions')
  41 installers = rp_utils.get_config('general.installers')
  42 blacklist = rp_utils.get_config('functest.blacklist')
  43 log_level = rp_utils.get_config('general.log.log_level')
  44 exclude_noha = rp_utils.get_config('functest.exclude_noha')
  45 exclude_virtual = rp_utils.get_config('functest.exclude_virtual')
  46
  47 functest_yaml_config = rp_utils.getFunctestConfig()
  48
  49 LOGGER.info("*******************************************")
  50 LOGGER.info("*                                         *")
  51 LOGGER.info("*   Generating reporting scenario status  *")
  52 LOGGER.info("*   Data retention: %s days               *", period)
  53 LOGGER.info("*   Log level: %s                         *", log_level)
  54 LOGGER.info("*                                         *")
  55 LOGGER.info("*   Virtual PODs exluded: %s              *", exclude_virtual)
  56 LOGGER.info("*   NOHA scenarios excluded: %s           *", exclude_noha)
  57 LOGGER.info("*                                         *")
  58 LOGGER.info("*******************************************")
  59
  60 # Retrieve test cases of Tier 1 (smoke)
  61 config_tiers = functest_yaml_config.get("tiers")
  62
  63 # we consider Tier 0 (Healthcheck), Tier 1 (smoke),2 (features)
  64 # to validate scenarios
  65 # Tier > 2 are not used to validate scenarios but we display the results anyway
  66 # tricky thing for the API as some tests are Functest tests
  67 # other tests are declared directly in the feature projects
  68 for tier in config_tiers:
  69     if tier['order'] >= 0 and tier['order'] < 2:
  70         for case in tier['testcases']:
  71             if case['case_name'] not in blacklist:
  72                 testValid.append(tc.TestCase(case['case_name'],
  73                                              "functest",
  74                                              case['dependencies']))
  75     elif tier['order'] == 2:
  76         for case in tier['testcases']:
  77             if case['case_name'] not in blacklist:
  78                 otherTestCases.append(tc.TestCase(case['case_name'],
  79                                                   case['case_name'],
  80                                                   case['dependencies']))
  81     elif tier['order'] > 2:
  82         for case in tier['testcases']:
  83             if case['case_name'] not in blacklist:
  84                 otherTestCases.append(tc.TestCase(case['case_name'],
  85                                                   "functest",
  86                                                   case['dependencies']))
  87
  88 LOGGER.debug("Functest reporting start")
  89
  90 # For all the versions
  91 for version in versions:
  92     # For all the installers
  93     scenario_directory = "./display/" + version + "/functest/"
  94     scenario_file_name = scenario_directory + "scenario_history.txt"
  95
  96     # check that the directory exists, if not create it
  97     # (first run on new version)
  98     if not os.path.exists(scenario_directory):
  99         os.makedirs(scenario_directory)
 100
 101     # initiate scenario file if it does not exist
 102     if not os.path.isfile(scenario_file_name):
 103         with open(scenario_file_name, "a") as my_file:
 104             LOGGER.debug("Create scenario file: %s", scenario_file_name)
 105             my_file.write("date,scenario,installer,detail,score\n")
 106
 107     for installer in installers:
 108
 109         # get scenarios
 110         scenario_results = rp_utils.getScenarios("functest",
 111                                                  "connection_check",
 112                                                  installer,
 113                                                  version)
 114         # get nb of supported architecture (x86, aarch64)
 115         architectures = rp_utils.getArchitectures(scenario_results)
 116         LOGGER.info("Supported architectures: %s", architectures)
 117
 118         for architecture in architectures:
 119             LOGGER.info("Architecture: %s", architecture)
 120             # Consider only the results for the selected architecture
 121             # i.e drop x86 for aarch64 and vice versa
 122             filter_results = rp_utils.filterArchitecture(scenario_results,
 123                                                          architecture)
 124             scenario_stats = rp_utils.getScenarioStats(filter_results)
 125             items = {}
 126             scenario_result_criteria = {}
 127
 128             # in case of more than 1 architecture supported
 129             # precise the architecture
 130             installer_display = installer
 131             if "fuel" in installer:
 132                 installer_display = installer + "@" + architecture
 133
 134             # For all the scenarios get results
 135             for s, s_result in filter_results.items():
 136                 LOGGER.info("---------------------------------")
 137                 LOGGER.info("installer %s, version %s, scenario %s:",
 138                             installer, version, s)
 139                 LOGGER.debug("Scenario results: %s", s_result)
 140
 141                 # Green or Red light for a given scenario
 142                 nb_test_runnable_for_this_scenario = 0
 143                 scenario_score = 0
 144                 # url of the last jenkins log corresponding to a given
 145                 # scenario
 146                 s_url = ""
 147                 if len(s_result) > 0:
 148                     build_tag = s_result[len(s_result)-1]['build_tag']
 149                     LOGGER.debug("Build tag: %s", build_tag)
 150                     s_url = rp_utils.getJenkinsUrl(build_tag)
 151                     if s_url is None:
 152                         s_url = "http://testresultS.opnfv.org/reporting"
 153                     LOGGER.info("last jenkins url: %s", s_url)
 154                 testCases2BeDisplayed = []
 155                 # Check if test case is runnable / installer, scenario
 156                 # for the test case used for Scenario validation
 157                 try:
 158                     # 1) Manage the test cases for the scenario validation
 159                     # concretely Tiers 0-3
 160                     for test_case in testValid:
 161                         test_case.checkRunnable(installer, s,
 162                                                 test_case.getConstraints())
 163                         LOGGER.debug("testcase %s (%s) is %s",
 164                                      test_case.getDisplayName(),
 165                                      test_case.getName(),
 166                                      test_case.isRunnable)
 167                         time.sleep(1)
 168                         if test_case.isRunnable:
 169                             name = test_case.getName()
 170                             displayName = test_case.getDisplayName()
 171                             project = test_case.getProject()
 172                             nb_test_runnable_for_this_scenario += 1
 173                             LOGGER.info(" Searching results for case %s ",
 174                                         displayName)
 175                             if "fuel" in installer:
 176                                 result = rp_utils.getCaseScoreFromBuildTag(
 177                                     name,
 178                                     s_result)
 179                             else:
 180                                 result = rp_utils.getCaseScore(name, installer,
 181                                                                s, version)
 182                             # if no result set the value to 0
 183                             if result < 0:
 184                                 result = 0
 185                             LOGGER.info(" >>>> Test score = " + str(result))
 186                             test_case.setCriteria(result)
 187                             test_case.setIsRunnable(True)
 188                             testCases2BeDisplayed.append(tc.TestCase(name,
 189                                                                      project,
 190                                                                      "",
 191                                                                      result,
 192                                                                      True,
 193                                                                      1))
 194                             scenario_score = scenario_score + result
 195
 196                     # 2) Manage the test cases for the scenario qualification
 197                     # concretely Tiers > 3
 198                     for test_case in otherTestCases:
 199                         test_case.checkRunnable(installer, s,
 200                                                 test_case.getConstraints())
 201                         LOGGER.debug("testcase %s (%s) is %s",
 202                                      test_case.getDisplayName(),
 203                                      test_case.getName(),
 204                                      test_case.isRunnable)
 205                         time.sleep(1)
 206                         if test_case.isRunnable:
 207                             name = test_case.getName()
 208                             displayName = test_case.getDisplayName()
 209                             project = test_case.getProject()
 210                             LOGGER.info(" Searching results for case %s ",
 211                                         displayName)
 212                             if "fuel" in installer:
 213                                 result = rp_utils.getCaseScoreFromBuildTag(
 214                                     name,
 215                                     s_result)
 216                             else:
 217                                 result = rp_utils.getCaseScore(name, installer,
 218                                                                s, version)
 219                             # at least 1 result for the test
 220                             if result > -1:
 221                                 test_case.setCriteria(result)
 222                                 test_case.setIsRunnable(True)
 223                                 testCases2BeDisplayed.append(tc.TestCase(
 224                                     name,
 225                                     project,
 226                                     "",
 227                                     result,
 228                                     True,
 229                                     4))
 230                             else:
 231                                 LOGGER.debug("No results found")
 232
 233                         items[s] = testCases2BeDisplayed
 234                 except Exception:  # pylint: disable=broad-except
 235                     LOGGER.error("Error installer %s, version %s, scenario %s",
 236                                  installer, version, s)
 237                     LOGGER.error("No data available: %s", sys.exc_info()[0])
 238
 239                 # **********************************************
 240                 # Evaluate the results for scenario validation
 241                 # **********************************************
 242                 # the validation criteria = nb runnable tests x 3
 243                 # because each test case can get
 244                 # 0 point (never PASS)
 245                 # 1 point at least (PASS once over the time window)
 246                 # 2 points (PASS more than once but 1 FAIL on the last 4)
 247                 # 3 points PASS on the last 4 iterations
 248                 # e.g. 1 scenario = 10 cases
 249                 # 1 iteration : max score = 10 (10x1)
 250                 # 2 iterations : max score = 20 (10x2)
 251                 # 3 iterations : max score = 20
 252                 # 4 or more iterations : max score = 30 (1x30)
 253                 LOGGER.info("Number of iterations for this scenario: %s",
 254                             len(s_result))
 255                 if len(s_result) > 3:
 256                     k_score = 3
 257                 elif len(s_result) < 2:
 258                     k_score = 1
 259                 else:
 260                     k_score = 2
 261
 262                 scenario_criteria = nb_test_runnable_for_this_scenario*k_score
 263
 264                 # score for reporting
 265                 s_score = str(scenario_score) + "/" + str(scenario_criteria)
 266                 s_score_percent = rp_utils.getScenarioPercent(
 267                     scenario_score,
 268                     scenario_criteria)
 269
 270                 s_status = "KO"
 271                 if scenario_score < scenario_criteria:
 272                     LOGGER.info(">>>> scenario not OK, score = %s/%s",
 273                                 scenario_score, scenario_criteria)
 274                     s_status = "KO"
 275                 else:
 276                     LOGGER.info(">>>>> scenario OK, save the information")
 277                     s_status = "OK"
 278                     path_validation_file = ("./display/" + version +
 279                                             "/functest/" +
 280                                             "validated_scenario_history.txt")
 281                     with open(path_validation_file, "a") as f:
 282                         time_format = "%Y-%m-%d %H:%M"
 283                         info = (datetime.datetime.now().strftime(time_format) +
 284                                 ";" + installer_display + ";" + s + "\n")
 285                         f.write(info)
 286
 287                 # Save daily results in a file
 288                 with open(scenario_file_name, "a") as f:
 289                     info = (reportingDate + "," + s + "," + installer_display +
 290                             "," + s_score + "," +
 291                             str(round(s_score_percent)) + "\n")
 292                     f.write(info)
 293
 294                 scenario_result_criteria[s] = sr.ScenarioResult(
 295                     s_status,
 296                     s_score,
 297                     s_score_percent,
 298                     s_url)
 299                 LOGGER.info("--------------------------")
 300
 301             templateLoader = jinja2.FileSystemLoader(".")
 302             templateEnv = jinja2.Environment(
 303                 loader=templateLoader, autoescape=True)
 304
 305             TEMPLATE_FILE = ("./reporting/functest/template"
 306                              "/index-status-tmpl.html")
 307             template = templateEnv.get_template(TEMPLATE_FILE)
 308
 309             outputText = template.render(
 310                 scenario_stats=scenario_stats,
 311                 scenario_results=scenario_result_criteria,
 312                 items=items,
 313                 installer=installer_display,
 314                 period=period,
 315                 version=version,
 316                 date=reportingDate)
 317
 318             with open("./display/" + version +
 319                       "/functest/status-" +
 320                       installer_display + ".html", "wb") as fh:
 321                 fh.write(outputText)
 322
 323             LOGGER.info("Manage export CSV & PDF")
 324             rp_utils.export_csv(scenario_file_name, installer_display, version)
 325             LOGGER.error("CSV generated...")
 326
 327             # Generate outputs for export
 328             # pdf
 329             url_pdf = rp_utils.get_config('general.url')
 330             pdf_path = ("./display/" + version +
 331                         "/functest/status-" + installer_display + ".html")
 332             pdf_doc_name = ("./display/" + version +
 333                             "/functest/status-" + installer_display + ".pdf")
 334             rp_utils.export_pdf(pdf_path, pdf_doc_name)
 335             LOGGER.info("PDF generated...")