utils/test/reporting/functest/reporting-status.py

   1 #!/usr/bin/python
   2 #
   3 # This program and the accompanying materials
   4 # are made available under the terms of the Apache License, Version 2.0
   5 # which accompanies this distribution, and is available at
   6 #
   7 # http://www.apache.org/licenses/LICENSE-2.0
   8 #
   9 import datetime
  10 import jinja2
  11 import requests
  12 import sys
  13 import time
  14 import yaml
  15
  16 import reportingUtils as utils
  17 import reportingConf as conf
  18 import testCase as tc
  19 import scenarioResult as sr
  20
  21 # Logger
  22 logger = utils.getLogger("Status")
  23
  24 # Initialization
  25 testValid = []
  26 otherTestCases = []
  27
  28 # init just tempest to get the list of scenarios
  29 # as all the scenarios run Tempest
  30 tempest = tc.TestCase("tempest_smoke_serial", "functest", -1)
  31
  32 # Retrieve the Functest configuration to detect which tests are relevant
  33 # according to the installer, scenario
  34 cf = conf.TEST_CONF
  35 response = requests.get(cf)
  36
  37 functest_yaml_config = yaml.load(response.text)
  38
  39 logger.info("*******************************************")
  40 logger.info("*   Generating reporting scenario status  *")
  41 logger.info("*   Data retention = %s days              *" % conf.PERIOD)
  42 logger.info("*                                         *")
  43 logger.info("*******************************************")
  44
  45 # Retrieve test cases of Tier 1 (smoke)
  46 config_tiers = functest_yaml_config.get("tiers")
  47
  48 # we consider Tier 1 (smoke),2 (sdn suites) and 3 (features)
  49 # to validate scenarios
  50 # Tier > 4 are not used to validate scenarios but we display the results anyway
  51 # tricky thing for the API as some tests are Functest tests
  52 # other tests are declared directly in the feature projects
  53 for tier in config_tiers:
  54     if tier['order'] > 0 and tier['order'] < 3:
  55         for case in tier['testcases']:
  56             if case['name'] not in conf.blacklist:
  57                 testValid.append(tc.TestCase(case['name'],
  58                                              "functest",
  59                                              case['dependencies']))
  60     elif tier['order'] == 3:
  61         for case in tier['testcases']:
  62             if case['name'] not in conf.blacklist:
  63                 testValid.append(tc.TestCase(case['name'],
  64                                              case['name'],
  65                                              case['dependencies']))
  66     elif tier['order'] > 3:
  67         for case in tier['testcases']:
  68             if case['name'] not in conf.blacklist:
  69                 otherTestCases.append(tc.TestCase(case['name'],
  70                                                   "functest",
  71                                                   case['dependencies']))
  72
  73 # For all the versions
  74 for version in conf.versions:
  75     # For all the installers
  76     for installer in conf.installers:
  77         # get scenarios
  78         scenario_results = utils.getScenarios(tempest, installer, version)
  79         scenario_stats = utils.getScenarioStats(scenario_results)
  80         items = {}
  81         scenario_result_criteria = {}
  82
  83         # For all the scenarios get results
  84         for s, s_result in scenario_results.items():
  85             # Green or Red light for a given scenario
  86             nb_test_runnable_for_this_scenario = 0
  87             scenario_score = 0
  88
  89             testCases2BeDisplayed = []
  90             # Check if test case is runnable / installer, scenario
  91             # for the test case used for Scenario validation
  92             try:
  93                 logger.info("---------------------------------")
  94                 logger.info("installer %s, version %s, scenario %s:" %
  95                             (installer, version, s))
  96
  97                 # 1) Manage the test cases for the scenario validation
  98                 # concretely Tiers 0-3
  99                 for test_case in testValid:
 100                     test_case.checkRunnable(installer, s,
 101                                             test_case.getConstraints())
 102                     logger.debug("testcase %s (%s) is %s" %
 103                                  (test_case.getDisplayName(),
 104                                   test_case.getName(),
 105                                   test_case.isRunnable))
 106                     time.sleep(1)
 107                     if test_case.isRunnable:
 108                         dbName = test_case.getDbName()
 109                         name = test_case.getName()
 110                         displayName = test_case.getDisplayName()
 111                         project = test_case.getProject()
 112                         nb_test_runnable_for_this_scenario += 1
 113                         logger.info(" Searching results for case %s " %
 114                                     (displayName))
 115                         result = utils.getResult(dbName, installer, s, version)
 116                         # if no result set the value to 0
 117                         if result < 0:
 118                             result = 0
 119                         logger.info(" >>>> Test score = " + str(result))
 120                         test_case.setCriteria(result)
 121                         test_case.setIsRunnable(True)
 122                         testCases2BeDisplayed.append(tc.TestCase(name,
 123                                                                  project,
 124                                                                  "",
 125                                                                  result,
 126                                                                  True,
 127                                                                  1))
 128                         scenario_score = scenario_score + result
 129
 130                 # 2) Manage the test cases for the scenario qualification
 131                 # concretely Tiers > 3
 132                 for test_case in otherTestCases:
 133                     test_case.checkRunnable(installer, s,
 134                                             test_case.getConstraints())
 135                     logger.debug("testcase %s (%s) is %s" %
 136                                  (test_case.getDisplayName(),
 137                                   test_case.getName(),
 138                                   test_case.isRunnable))
 139                     time.sleep(1)
 140                     if test_case.isRunnable:
 141                         dbName = test_case.getDbName()
 142                         name = test_case.getName()
 143                         displayName = test_case.getDisplayName()
 144                         project = test_case.getProject()
 145                         logger.info(" Searching results for case %s " %
 146                                     (displayName))
 147                         result = utils.getResult(dbName, installer, s, version)
 148                         # at least 1 result for the test
 149                         if result > -1:
 150                             test_case.setCriteria(result)
 151                             test_case.setIsRunnable(True)
 152                             testCases2BeDisplayed.append(tc.TestCase(name,
 153                                                                      project,
 154                                                                      "",
 155                                                                      result,
 156                                                                      True,
 157                                                                      4))
 158                         else:
 159                             logger.debug("No results found")
 160
 161                     items[s] = testCases2BeDisplayed
 162             except:
 163                 logger.error("Error: installer %s, version %s, scenario %s" %
 164                              (installer, version, s))
 165                 logger.error("No data available: %s " % (sys.exc_info()[0]))
 166
 167             # **********************************************
 168             # Evaluate the results for scenario validation
 169             # **********************************************
 170             # the validation criteria = nb runnable tests x 3
 171             # because each test case = 0,1,2 or 3
 172             scenario_criteria = nb_test_runnable_for_this_scenario * 3
 173             # if 0 runnable tests set criteria at a high value
 174             if scenario_criteria < 1:
 175                 scenario_criteria = conf.MAX_SCENARIO_CRITERIA
 176
 177             s_score = str(scenario_score) + "/" + str(scenario_criteria)
 178             s_status = "KO"
 179             if scenario_score < scenario_criteria:
 180                 logger.info(">>>> scenario not OK, score = %s/%s" %
 181                             (scenario_score, scenario_criteria))
 182                 s_status = "KO"
 183             else:
 184                 logger.info(">>>>> scenario OK, save the information")
 185                 s_status = "OK"
 186                 path_validation_file = (conf.REPORTING_PATH + "/release/" + version +
 187                                         "/validated_scenario_history.txt")
 188                 with open(path_validation_file, "a") as f:
 189                     time_format = "%Y-%m-%d %H:%M"
 190                     info = (datetime.datetime.now().strftime(time_format) +
 191                             ";" + installer + ";" + s + "\n")
 192                     f.write(info)
 193
 194             scenario_result_criteria[s] = sr.ScenarioResult(s_status, s_score)
 195             logger.info("--------------------------")
 196
 197         templateLoader = jinja2.FileSystemLoader(conf.REPORTING_PATH)
 198         templateEnv = jinja2.Environment(loader=templateLoader, autoescape=True)
 199
 200         TEMPLATE_FILE = "/template/index-status-tmpl.html"
 201         template = templateEnv.get_template(TEMPLATE_FILE)
 202
 203         outputText = template.render(scenario_stats=scenario_stats,
 204                                      scenario_results=scenario_result_criteria,
 205                                      items=items,
 206                                      installer=installer,
 207                                      period=conf.PERIOD,
 208                                      version=version)
 209
 210         with open(conf.REPORTING_PATH + "/release/" + version +
 211                   "/index-status-" + installer + ".html", "wb") as fh:
 212             fh.write(outputText)