utils/test/reporting/functest/reporting-status.py

   1 #!/usr/bin/python
   2 #
   3 # This program and the accompanying materials
   4 # are made available under the terms of the Apache License, Version 2.0
   5 # which accompanies this distribution, and is available at
   6 #
   7 # http://www.apache.org/licenses/LICENSE-2.0
   8 #
   9 import datetime
  10 import jinja2
  11 import logging
  12 import os
  13 import requests
  14 import sys
  15 import time
  16 import yaml
  17
  18 import reportingUtils as utils
  19 import reportingConf as conf
  20 import testCase as tc
  21 import scenarioResult as sr
  22
  23 # Logger
  24 logFormatter = logging.Formatter("%(asctime)s [%(threadName)-12.12s] [%(levelname)-5.5s]  %(message)s")
  25 logger = logging.getLogger()
  26
  27 fileHandler = logging.FileHandler("{0}/{1}".format('.', conf.LOG_FILE))
  28 fileHandler.setFormatter(logFormatter)
  29 logger.addHandler(fileHandler)
  30
  31 consoleHandler = logging.StreamHandler()
  32 consoleHandler.setFormatter(logFormatter)
  33 logger.addHandler(consoleHandler)
  34 logger.setLevel(conf.LOG_LEVEL)
  35
  36 # Initialization
  37 testValid = []
  38 otherTestCases = []
  39
  40 # init just tempest to get the list of scenarios
  41 # as all the scenarios run Tempest
  42 tempest = tc.TestCase("tempest_smoke_serial", "functest", -1)
  43
  44 # Retrieve the Functest configuration to detect which tests are relevant
  45 # according to the installer, scenario
  46 cf = conf.TEST_CONF
  47 response = requests.get(cf)
  48
  49 functest_yaml_config = yaml.load(response.text)
  50
  51 logger.info("****************************************")
  52 logger.info("*   Generating reporting.....          *")
  53 logger.info("*   Data retention = %s days           *" % conf.PERIOD)
  54 logger.info("*                                      *")
  55 logger.info("****************************************")
  56
  57 # Retrieve test cases of Tier 1 (smoke)
  58 config_tiers = functest_yaml_config.get("tiers")
  59
  60 # we consider Tier 1 (smoke),2 (sdn suites) and 3 (features)
  61 # to validate scenarios
  62 # Tier > 4 are not used to validate scenarios but we display the results anyway
  63 # tricky thing for the API as some tests are Functest tests
  64 # other tests are declared directly in the feature projects
  65 for tier in config_tiers:
  66     if tier['order'] > 0 and tier['order'] < 3:
  67         for case in tier['testcases']:
  68             if case['name'] not in conf.blacklist:
  69                 testValid.append(tc.TestCase(case['name'],
  70                                              "functest",
  71                                              case['dependencies']))
  72     elif tier['order'] == 3:
  73         for case in tier['testcases']:
  74             if case['name'] not in conf.blacklist:
  75                 testValid.append(tc.TestCase(case['name'],
  76                                              case['name'],
  77                                              case['dependencies']))
  78     elif tier['order'] > 3:
  79         for case in tier['testcases']:
  80             if case['name'] not in conf.blacklist:
  81                 otherTestCases.append(tc.TestCase(case['name'],
  82                                                   "functest",
  83                                                   case['dependencies']))
  84
  85 # For all the versions
  86 for version in conf.versions:
  87     # For all the installers
  88     for installer in conf.installers:
  89         # get scenarios
  90         scenario_results = utils.getScenarios(tempest, installer, version)
  91         scenario_stats = utils.getScenarioStats(scenario_results)
  92         items = {}
  93         scenario_result_criteria = {}
  94
  95         # For all the scenarios get results
  96         for s, s_result in scenario_results.items():
  97             # Green or Red light for a given scenario
  98             nb_test_runnable_for_this_scenario = 0
  99             scenario_score = 0
 100
 101             testCases2BeDisplayed = []
 102             # Check if test case is runnable / installer, scenario
 103             # for the test case used for Scenario validation
 104             try:
 105                 logger.info("---------------------------------")
 106                 logger.info("installer %s, version %s, scenario %s:" %
 107                             (installer, version, s))
 108
 109                 # 1) Manage the test cases for the scenario validation
 110                 # concretely Tiers 0-3
 111                 for test_case in testValid:
 112                     test_case.checkRunnable(installer, s,
 113                                             test_case.getConstraints())
 114                     logger.debug("testcase %s is %s" % (test_case.getName(),
 115                                                         test_case.isRunnable))
 116                     time.sleep(1)
 117                     if test_case.isRunnable:
 118                         dbName = test_case.getDbName()
 119                         name = test_case.getName()
 120                         project = test_case.getProject()
 121                         nb_test_runnable_for_this_scenario += 1
 122                         logger.info(" Searching results for case %s " %
 123                                     (dbName))
 124                         result = utils.getResult(dbName, installer, s, version)
 125                         logger.info(" >>>> Test score = " + str(result))
 126                         test_case.setCriteria(result)
 127                         test_case.setIsRunnable(True)
 128                         testCases2BeDisplayed.append(tc.TestCase(name,
 129                                                                  project,
 130                                                                  "",
 131                                                                  result,
 132                                                                  True,
 133                                                                  1))
 134                         scenario_score = scenario_score + result
 135
 136                 # 2) Manage the test cases for the scenario qualification
 137                 # concretely Tiers > 3
 138                 for test_case in otherTestCases:
 139                     test_case.checkRunnable(installer, s,
 140                                             test_case.getConstraints())
 141                     logger.info("testcase %s is %s" %
 142                                 (test_case.getName(), test_case.isRunnable))
 143                     time.sleep(1)
 144                     if test_case.isRunnable:
 145                         dbName = test_case.getDbName()
 146                         name = test_case.getName()
 147                         project = test_case.getProject()
 148                         logger.info(" Searching results for case %s " %
 149                                     (dbName))
 150                         result = utils.getResult(dbName, installer, s, version)
 151                         test_case.setCriteria(result)
 152                         test_case.setIsRunnable(True)
 153                         testCases2BeDisplayed.append(tc.TestCase(name,
 154                                                                  project,
 155                                                                  "",
 156                                                                  result,
 157                                                                  True,
 158                                                                  4))
 159
 160                     items[s] = testCases2BeDisplayed
 161             except:
 162                 logger.error("Error: installer %s, version %s, scenario %s" %
 163                              (installer, version, s))
 164                 logger.error("No data available: %s " % (sys.exc_info()[0]))
 165
 166             # **********************************************
 167             # Evaluate the results for scenario validation
 168             # **********************************************
 169             # the validation criteria = nb runnable tests x 3
 170             # because each test case = 0,1,2 or 3
 171             scenario_criteria = nb_test_runnable_for_this_scenario * 3
 172             # if 0 runnable tests set criteria at a high value
 173             if scenario_criteria < 1:
 174                 scenario_criteria = conf.MAX_SCENARIO_CRITERIA
 175
 176             s_score = str(scenario_score) + "/" + str(scenario_criteria)
 177             s_status = "KO"
 178             if scenario_score < scenario_criteria:
 179                 logger.info(">>>> scenario not OK, score = %s/%s" %
 180                             (scenario_score, scenario_criteria))
 181                 s_status = "KO"
 182             else:
 183                 logger.info(">>>>> scenario OK, save the information")
 184                 s_status = "OK"
 185                 path_validation_file = ("./release/" + version +
 186                                         "/validated_scenario_history.txt")
 187                 with open(path_validation_file, "a") as f:
 188                     time_format = "%Y-%m-%d %H:%M"
 189                     info = (datetime.datetime.now().strftime(time_format) +
 190                             ";" + installer + ";" + s + "\n")
 191                     f.write(info)
 192
 193             scenario_result_criteria[s] = sr.ScenarioResult(s_status, s_score)
 194             logger.info("--------------------------")
 195
 196         templateLoader = jinja2.FileSystemLoader(os.path.dirname
 197                                                  (os.path.abspath
 198                                                   (__file__)))
 199         templateEnv = jinja2.Environment(loader=templateLoader)
 200
 201         TEMPLATE_FILE = "./template/index-status-tmpl.html"
 202         template = templateEnv.get_template(TEMPLATE_FILE)
 203
 204         outputText = template.render(scenario_stats=scenario_stats,
 205                                      scenario_results=scenario_result_criteria,
 206                                      items=items,
 207                                      installer=installer,
 208                                      period=conf.PERIOD,
 209                                      version=version)
 210
 211         with open("./release/" + version +
 212                   "/index-status-" + installer + ".html", "wb") as fh:
 213             fh.write(outputText)