3 # This program and the accompanying materials
4 # are made available under the terms of the Apache License, Version 2.0
5 # which accompanies this distribution, and is available at
7 # http://www.apache.org/licenses/LICENSE-2.0
18 import reportingUtils as utils
19 import reportingConf as conf
21 import scenarioResult as sr
24 logFormatter = logging.Formatter("%(asctime)s [%(threadName)-12.12s] [%(levelname)-5.5s] %(message)s")
25 logger = logging.getLogger()
27 fileHandler = logging.FileHandler("{0}/{1}".format('.', conf.LOG_FILE))
28 fileHandler.setFormatter(logFormatter)
29 logger.addHandler(fileHandler)
31 consoleHandler = logging.StreamHandler()
32 consoleHandler.setFormatter(logFormatter)
33 logger.addHandler(consoleHandler)
34 logger.setLevel(conf.LOG_LEVEL)
40 # init just tempest to get the list of scenarios
41 # as all the scenarios run Tempest
42 tempest = tc.TestCase("tempest_smoke_serial", "functest", -1)
44 # Retrieve the Functest configuration to detect which tests are relevant
45 # according to the installer, scenario
47 response = requests.get(cf)
49 functest_yaml_config = yaml.load(response.text)
51 logger.info("****************************************")
52 logger.info("* Generating reporting..... *")
53 logger.info("* Data retention = %s days *" % conf.PERIOD)
55 logger.info("****************************************")
57 # Retrieve test cases of Tier 1 (smoke)
58 config_tiers = functest_yaml_config.get("tiers")
60 # we consider Tier 1 (smoke),2 (sdn suites) and 3 (features)
61 # to validate scenarios
62 # Tier > 4 are not used to validate scenarios but we display the results anyway
63 # tricky thing for the API as some tests are Functest tests
64 # other tests are declared directly in the feature projects
65 for tier in config_tiers:
66 if tier['order'] > 0 and tier['order'] < 3:
67 for case in tier['testcases']:
68 if case['name'] not in conf.blacklist:
69 testValid.append(tc.TestCase(case['name'],
71 case['dependencies']))
72 elif tier['order'] == 3:
73 for case in tier['testcases']:
74 if case['name'] not in conf.blacklist:
75 testValid.append(tc.TestCase(case['name'],
77 case['dependencies']))
78 elif tier['order'] > 3:
79 for case in tier['testcases']:
80 if case['name'] not in conf.blacklist:
81 otherTestCases.append(tc.TestCase(case['name'],
83 case['dependencies']))
85 # For all the versions
86 for version in conf.versions:
87 # For all the installers
88 for installer in conf.installers:
90 scenario_results = utils.getScenarios(tempest, installer, version)
91 scenario_stats = utils.getScenarioStats(scenario_results)
93 scenario_result_criteria = {}
95 # For all the scenarios get results
96 for s, s_result in scenario_results.items():
97 # Green or Red light for a given scenario
98 nb_test_runnable_for_this_scenario = 0
101 testCases2BeDisplayed = []
102 # Check if test case is runnable / installer, scenario
103 # for the test case used for Scenario validation
105 logger.info("---------------------------------")
106 logger.info("installer %s, version %s, scenario %s:" %
107 (installer, version, s))
109 # 1) Manage the test cases for the scenario validation
110 # concretely Tiers 0-3
111 for test_case in testValid:
112 test_case.checkRunnable(installer, s,
113 test_case.getConstraints())
114 logger.debug("testcase %s is %s" % (test_case.getName(),
115 test_case.isRunnable))
117 if test_case.isRunnable:
118 dbName = test_case.getDbName()
119 name = test_case.getName()
120 project = test_case.getProject()
121 nb_test_runnable_for_this_scenario += 1
122 logger.info(" Searching results for case %s " %
124 result = utils.getResult(dbName, installer, s, version)
125 logger.info(" >>>> Test score = " + str(result))
126 test_case.setCriteria(result)
127 test_case.setIsRunnable(True)
128 testCases2BeDisplayed.append(tc.TestCase(name,
134 scenario_score = scenario_score + result
136 # 2) Manage the test cases for the scenario qualification
137 # concretely Tiers > 3
138 for test_case in otherTestCases:
139 test_case.checkRunnable(installer, s,
140 test_case.getConstraints())
141 logger.info("testcase %s is %s" %
142 (test_case.getName(), test_case.isRunnable))
144 if test_case.isRunnable:
145 dbName = test_case.getDbName()
146 name = test_case.getName()
147 project = test_case.getProject()
148 logger.info(" Searching results for case %s " %
150 result = utils.getResult(dbName, installer, s, version)
151 test_case.setCriteria(result)
152 test_case.setIsRunnable(True)
153 testCases2BeDisplayed.append(tc.TestCase(name,
160 items[s] = testCases2BeDisplayed
162 logger.error("Error: installer %s, version %s, scenario %s" %
163 (installer, version, s))
164 logger.error("No data available: %s " % (sys.exc_info()[0]))
166 # **********************************************
167 # Evaluate the results for scenario validation
168 # **********************************************
169 # the validation criteria = nb runnable tests x 3
170 # because each test case = 0,1,2 or 3
171 scenario_criteria = nb_test_runnable_for_this_scenario * 3
172 # if 0 runnable tests set criteria at a high value
173 if scenario_criteria < 1:
174 scenario_criteria = conf.MAX_SCENARIO_CRITERIA
176 s_score = str(scenario_score) + "/" + str(scenario_criteria)
178 if scenario_score < scenario_criteria:
179 logger.info(">>>> scenario not OK, score = %s/%s" %
180 (scenario_score, scenario_criteria))
183 logger.info(">>>>> scenario OK, save the information")
185 path_validation_file = ("./release/" + version +
186 "/validated_scenario_history.txt")
187 with open(path_validation_file, "a") as f:
188 time_format = "%Y-%m-%d %H:%M"
189 info = (datetime.datetime.now().strftime(time_format) +
190 ";" + installer + ";" + s + "\n")
193 scenario_result_criteria[s] = sr.ScenarioResult(s_status, s_score)
194 logger.info("--------------------------")
196 templateLoader = jinja2.FileSystemLoader(os.path.dirname
199 templateEnv = jinja2.Environment(loader=templateLoader)
201 TEMPLATE_FILE = "./template/index-status-tmpl.html"
202 template = templateEnv.get_template(TEMPLATE_FILE)
204 outputText = template.render(scenario_stats=scenario_stats,
205 scenario_results=scenario_result_criteria,
211 with open("./release/" + version +
212 "/index-status-" + installer + ".html", "wb") as fh: