utils/test/reporting/functest/reporting-status.py

   1 #!/usr/bin/python
   2 #
   3 # This program and the accompanying materials
   4 # are made available under the terms of the Apache License, Version 2.0
   5 # which accompanies this distribution, and is available at
   6 #
   7 # http://www.apache.org/licenses/LICENSE-2.0
   8 #
   9 import datetime
  10 import jinja2
  11 import requests
  12 import sys
  13 import time
  14 import yaml
  15
  16 import reportingUtils as utils
  17 import reportingConf as conf
  18 import testCase as tc
  19 import scenarioResult as sr
  20
  21 # Logger
  22 logger = utils.getLogger("Status")
  23
  24 # Initialization
  25 testValid = []
  26 otherTestCases = []
  27
  28 # init just tempest to get the list of scenarios
  29 # as all the scenarios run Tempest
  30 tempest = tc.TestCase("tempest_smoke_serial", "functest", -1)
  31
  32 # Retrieve the Functest configuration to detect which tests are relevant
  33 # according to the installer, scenario
  34 cf = conf.TEST_CONF
  35 response = requests.get(cf)
  36
  37 functest_yaml_config = yaml.load(response.text)
  38
  39 logger.info("*******************************************")
  40 logger.info("*   Generating reporting scenario status  *")
  41 logger.info("*   Data retention = %s days              *" % conf.PERIOD)
  42 logger.info("*                                         *")
  43 logger.info("*******************************************")
  44
  45 # Retrieve test cases of Tier 1 (smoke)
  46 config_tiers = functest_yaml_config.get("tiers")
  47
  48 # we consider Tier 1 (smoke),2 (sdn suites) and 3 (features)
  49 # to validate scenarios
  50 # Tier > 4 are not used to validate scenarios but we display the results anyway
  51 # tricky thing for the API as some tests are Functest tests
  52 # other tests are declared directly in the feature projects
  53 for tier in config_tiers:
  54     if tier['order'] > 0 and tier['order'] < 3:
  55         for case in tier['testcases']:
  56             if case['name'] not in conf.blacklist:
  57                 testValid.append(tc.TestCase(case['name'],
  58                                              "functest",
  59                                              case['dependencies']))
  60     elif tier['order'] == 3:
  61         for case in tier['testcases']:
  62             if case['name'] not in conf.blacklist:
  63                 testValid.append(tc.TestCase(case['name'],
  64                                              case['name'],
  65                                              case['dependencies']))
  66     elif tier['order'] > 3:
  67         for case in tier['testcases']:
  68             if case['name'] not in conf.blacklist:
  69                 otherTestCases.append(tc.TestCase(case['name'],
  70                                                   "functest",
  71                                                   case['dependencies']))
  72
  73 # For all the versions
  74 for version in conf.versions:
  75     # For all the installers
  76     for installer in conf.installers:
  77         # get scenarios
  78         scenario_results = utils.getScenarios(tempest, installer, version)
  79         scenario_stats = utils.getScenarioStats(scenario_results)
  80         items = {}
  81         scenario_result_criteria = {}
  82
  83         # For all the scenarios get results
  84         for s, s_result in scenario_results.items():
  85             # Green or Red light for a given scenario
  86             nb_test_runnable_for_this_scenario = 0
  87             scenario_score = 0
  88
  89             testCases2BeDisplayed = []
  90             # Check if test case is runnable / installer, scenario
  91             # for the test case used for Scenario validation
  92             try:
  93                 logger.info("---------------------------------")
  94                 logger.info("installer %s, version %s, scenario %s:" %
  95                             (installer, version, s))
  96
  97                 # 1) Manage the test cases for the scenario validation
  98                 # concretely Tiers 0-3
  99                 for test_case in testValid:
 100                     test_case.checkRunnable(installer, s,
 101                                             test_case.getConstraints())
 102                     logger.debug("testcase %s is %s" %
 103                                  (test_case.getDisplayName(),
 104                                   test_case.isRunnable))
 105                     time.sleep(1)
 106                     if test_case.isRunnable:
 107                         dbName = test_case.getDbName()
 108                         name = test_case.getName()
 109                         displayName = test_case.getDisplayName()
 110                         project = test_case.getProject()
 111                         nb_test_runnable_for_this_scenario += 1
 112                         logger.info(" Searching results for case %s " %
 113                                     (displayName))
 114                         result = utils.getResult(dbName, installer, s, version)
 115                         # if no result set the value to 0
 116                         if result < 0:
 117                             result = 0
 118                         logger.info(" >>>> Test score = " + str(result))
 119                         test_case.setCriteria(result)
 120                         test_case.setIsRunnable(True)
 121                         testCases2BeDisplayed.append(tc.TestCase(name,
 122                                                                  project,
 123                                                                  "",
 124                                                                  result,
 125                                                                  True,
 126                                                                  1))
 127                         scenario_score = scenario_score + result
 128
 129                 # 2) Manage the test cases for the scenario qualification
 130                 # concretely Tiers > 3
 131                 for test_case in otherTestCases:
 132                     test_case.checkRunnable(installer, s,
 133                                             test_case.getConstraints())
 134                     logger.info("testcase %s is %s" %
 135                                 (test_case.getName(), test_case.isRunnable))
 136                     time.sleep(1)
 137                     if test_case.isRunnable:
 138                         dbName = test_case.getDbName()
 139                         name = test_case.getName()
 140                         displayName = test_case.getDisplayName()
 141                         project = test_case.getProject()
 142                         logger.info(" Searching results for case %s " %
 143                                     (displayName))
 144                         result = utils.getResult(dbName, installer, s, version)
 145                         # at least 1 result for the test
 146                         if result > -1:
 147                             test_case.setCriteria(result)
 148                             test_case.setIsRunnable(True)
 149                             testCases2BeDisplayed.append(tc.TestCase(name,
 150                                                                      project,
 151                                                                      "",
 152                                                                      result,
 153                                                                      True,
 154                                                                      4))
 155                         else:
 156                             logger.debug("No results found")
 157
 158                     items[s] = testCases2BeDisplayed
 159             except:
 160                 logger.error("Error: installer %s, version %s, scenario %s" %
 161                              (installer, version, s))
 162                 logger.error("No data available: %s " % (sys.exc_info()[0]))
 163
 164             # **********************************************
 165             # Evaluate the results for scenario validation
 166             # **********************************************
 167             # the validation criteria = nb runnable tests x 3
 168             # because each test case = 0,1,2 or 3
 169             scenario_criteria = nb_test_runnable_for_this_scenario * 3
 170             # if 0 runnable tests set criteria at a high value
 171             if scenario_criteria < 1:
 172                 scenario_criteria = conf.MAX_SCENARIO_CRITERIA
 173
 174             s_score = str(scenario_score) + "/" + str(scenario_criteria)
 175             s_status = "KO"
 176             if scenario_score < scenario_criteria:
 177                 logger.info(">>>> scenario not OK, score = %s/%s" %
 178                             (scenario_score, scenario_criteria))
 179                 s_status = "KO"
 180             else:
 181                 logger.info(">>>>> scenario OK, save the information")
 182                 s_status = "OK"
 183                 path_validation_file = (conf.REPORTING_PATH + "/release/" + version +
 184                                         "/validated_scenario_history.txt")
 185                 with open(path_validation_file, "a") as f:
 186                     time_format = "%Y-%m-%d %H:%M"
 187                     info = (datetime.datetime.now().strftime(time_format) +
 188                             ";" + installer + ";" + s + "\n")
 189                     f.write(info)
 190
 191             scenario_result_criteria[s] = sr.ScenarioResult(s_status, s_score)
 192             logger.info("--------------------------")
 193
 194         templateLoader = jinja2.FileSystemLoader(conf.REPORTING_PATH)
 195         templateEnv = jinja2.Environment(loader=templateLoader)
 196
 197         TEMPLATE_FILE = "/template/index-status-tmpl.html"
 198         template = templateEnv.get_template(TEMPLATE_FILE)
 199
 200         outputText = template.render(scenario_stats=scenario_stats,
 201                                      scenario_results=scenario_result_criteria,
 202                                      items=items,
 203                                      installer=installer,
 204                                      period=conf.PERIOD,
 205                                      version=version)
 206
 207         with open(conf.REPORTING_PATH + "/release/" + version +
 208                   "/index-status-" + installer + ".html", "wb") as fh:
 209             fh.write(outputText)