utils/test/reporting/functest/reporting-status.py

   1 #!/usr/bin/python
   2 #
   3 # This program and the accompanying materials
   4 # are made available under the terms of the Apache License, Version 2.0
   5 # which accompanies this distribution, and is available at
   6 #
   7 # http://www.apache.org/licenses/LICENSE-2.0
   8 #
   9 import datetime
  10 import jinja2
  11 import requests
  12 import sys
  13 import time
  14 import yaml
  15
  16 import reportingUtils as utils
  17 import reportingConf as conf
  18 import testCase as tc
  19 import scenarioResult as sr
  20
  21 # Logger
  22 logger = utils.getLogger("Status")
  23
  24 # Initialization
  25 testValid = []
  26 otherTestCases = []
  27
  28 # init just tempest to get the list of scenarios
  29 # as all the scenarios run Tempest
  30 tempest = tc.TestCase("tempest_smoke_serial", "functest", -1)
  31
  32 # Retrieve the Functest configuration to detect which tests are relevant
  33 # according to the installer, scenario
  34 cf = conf.TEST_CONF
  35 response = requests.get(cf)
  36
  37 functest_yaml_config = yaml.load(response.text)
  38
  39 logger.info("*******************************************")
  40 logger.info("*   Generating reporting scenario status  *")
  41 logger.info("*   Data retention = %s days              *" % conf.PERIOD)
  42 logger.info("*                                         *")
  43 logger.info("*******************************************")
  44
  45 # Retrieve test cases of Tier 1 (smoke)
  46 config_tiers = functest_yaml_config.get("tiers")
  47
  48 # we consider Tier 1 (smoke),2 (sdn suites) and 3 (features)
  49 # to validate scenarios
  50 # Tier > 4 are not used to validate scenarios but we display the results anyway
  51 # tricky thing for the API as some tests are Functest tests
  52 # other tests are declared directly in the feature projects
  53 for tier in config_tiers:
  54     if tier['order'] > 0 and tier['order'] < 3:
  55         for case in tier['testcases']:
  56             if case['name'] not in conf.blacklist:
  57                 testValid.append(tc.TestCase(case['name'],
  58                                              "functest",
  59                                              case['dependencies']))
  60     elif tier['order'] == 3:
  61         for case in tier['testcases']:
  62             if case['name'] not in conf.blacklist:
  63                 testValid.append(tc.TestCase(case['name'],
  64                                              case['name'],
  65                                              case['dependencies']))
  66     elif tier['order'] > 3:
  67         for case in tier['testcases']:
  68             if case['name'] not in conf.blacklist:
  69                 otherTestCases.append(tc.TestCase(case['name'],
  70                                                   "functest",
  71                                                   case['dependencies']))
  72
  73 # For all the versions
  74 for version in conf.versions:
  75     # For all the installers
  76     for installer in conf.installers:
  77         # get scenarios
  78         scenario_results = utils.getScenarios(tempest, installer, version)
  79         scenario_stats = utils.getScenarioStats(scenario_results)
  80         items = {}
  81         scenario_result_criteria = {}
  82
  83         # For all the scenarios get results
  84         for s, s_result in scenario_results.items():
  85             # Green or Red light for a given scenario
  86             nb_test_runnable_for_this_scenario = 0
  87             scenario_score = 0
  88
  89             testCases2BeDisplayed = []
  90             # Check if test case is runnable / installer, scenario
  91             # for the test case used for Scenario validation
  92             try:
  93                 logger.info("---------------------------------")
  94                 logger.info("installer %s, version %s, scenario %s:" %
  95                             (installer, version, s))
  96
  97                 # 1) Manage the test cases for the scenario validation
  98                 # concretely Tiers 0-3
  99                 for test_case in testValid:
 100                     test_case.checkRunnable(installer, s,
 101                                             test_case.getConstraints())
 102                     logger.debug("testcase %s is %s" %
 103                                  (test_case.getDisplayName(),
 104                                   test_case.isRunnable))
 105                     time.sleep(1)
 106                     if test_case.isRunnable:
 107                         dbName = test_case.getDbName()
 108                         name = test_case.getName()
 109                         displayName = test_case.getDisplayName()
 110                         project = test_case.getProject()
 111                         nb_test_runnable_for_this_scenario += 1
 112                         logger.info(" Searching results for case %s " %
 113                                     (displayName))
 114                         result = utils.getResult(dbName, installer, s, version)
 115                         # at least 1 result for the test
 116                         if result > -1:
 117                             logger.info(" >>>> Test score = " + str(result))
 118                             test_case.setCriteria(result)
 119                             test_case.setIsRunnable(True)
 120                             testCases2BeDisplayed.append(tc.TestCase(name,
 121                                                                      project,
 122                                                                      "",
 123                                                                      result,
 124                                                                      True,
 125                                                                      1))
 126                             scenario_score = scenario_score + result
 127                         else:
 128                             logger.debug("No results found")
 129
 130                 # 2) Manage the test cases for the scenario qualification
 131                 # concretely Tiers > 3
 132                 for test_case in otherTestCases:
 133                     test_case.checkRunnable(installer, s,
 134                                             test_case.getConstraints())
 135                     logger.info("testcase %s is %s" %
 136                                 (test_case.getName(), test_case.isRunnable))
 137                     time.sleep(1)
 138                     if test_case.isRunnable:
 139                         dbName = test_case.getDbName()
 140                         name = test_case.getName()
 141                         displayName = test_case.getDisplayName()
 142                         project = test_case.getProject()
 143                         logger.info(" Searching results for case %s " %
 144                                     (displayName))
 145                         result = utils.getResult(dbName, installer, s, version)
 146                         # at least 1 result for the test
 147                         if result > -1:
 148                             test_case.setCriteria(result)
 149                             test_case.setIsRunnable(True)
 150                             testCases2BeDisplayed.append(tc.TestCase(name,
 151                                                                      project,
 152                                                                      "",
 153                                                                      result,
 154                                                                      True,
 155                                                                      4))
 156                         else:
 157                             logger.debug("No results found")
 158
 159                     items[s] = testCases2BeDisplayed
 160             except:
 161                 logger.error("Error: installer %s, version %s, scenario %s" %
 162                              (installer, version, s))
 163                 logger.error("No data available: %s " % (sys.exc_info()[0]))
 164
 165             # **********************************************
 166             # Evaluate the results for scenario validation
 167             # **********************************************
 168             # the validation criteria = nb runnable tests x 3
 169             # because each test case = 0,1,2 or 3
 170             scenario_criteria = nb_test_runnable_for_this_scenario * 3
 171             # if 0 runnable tests set criteria at a high value
 172             if scenario_criteria < 1:
 173                 scenario_criteria = conf.MAX_SCENARIO_CRITERIA
 174
 175             s_score = str(scenario_score) + "/" + str(scenario_criteria)
 176             s_status = "KO"
 177             if scenario_score < scenario_criteria:
 178                 logger.info(">>>> scenario not OK, score = %s/%s" %
 179                             (scenario_score, scenario_criteria))
 180                 s_status = "KO"
 181             else:
 182                 logger.info(">>>>> scenario OK, save the information")
 183                 s_status = "OK"
 184                 path_validation_file = ("./release/" + version +
 185                                         "/validated_scenario_history.txt")
 186                 with open(path_validation_file, "a") as f:
 187                     time_format = "%Y-%m-%d %H:%M"
 188                     info = (datetime.datetime.now().strftime(time_format) +
 189                             ";" + installer + ";" + s + "\n")
 190                     f.write(info)
 191
 192             scenario_result_criteria[s] = sr.ScenarioResult(s_status, s_score)
 193             logger.info("--------------------------")
 194
 195         templateLoader = jinja2.FileSystemLoader(conf.REPORTING_PATH)
 196         templateEnv = jinja2.Environment(loader=templateLoader)
 197
 198         TEMPLATE_FILE = "/template/index-status-tmpl.html"
 199         template = templateEnv.get_template(TEMPLATE_FILE)
 200
 201         outputText = template.render(scenario_stats=scenario_stats,
 202                                      scenario_results=scenario_result_criteria,
 203                                      items=items,
 204                                      installer=installer,
 205                                      period=conf.PERIOD,
 206                                      version=version)
 207
 208         with open(conf.REPORTING_PATH + "/release/" + version +
 209                   "/index-status-" + installer + ".html", "wb") as fh:
 210             fh.write(outputText)