3 # This program and the accompanying materials
4 # are made available under the terms of the Apache License, Version 2.0
5 # which accompanies this distribution, and is available at
7 # http://www.apache.org/licenses/LICENSE-2.0
18 import scenarioResult as sr
21 import utils.reporting_utils as rp_utils
24 logger = rp_utils.getLogger("Functest-Status")
29 reportingDate = datetime.datetime.now().strftime("%Y-%m-%d %H:%M")
31 # init just tempest to get the list of scenarios
32 # as all the scenarios run Tempest
33 tempest = tc.TestCase("tempest_smoke_serial", "functest", -1)
35 # Retrieve the Functest configuration to detect which tests are relevant
36 # according to the installer, scenario
37 cf = rp_utils.get_config('functest.test_conf')
38 period = rp_utils.get_config('general.period')
39 versions = rp_utils.get_config('general.versions')
40 installers = rp_utils.get_config('general.installers')
41 blacklist = rp_utils.get_config('functest.blacklist')
42 log_level = rp_utils.get_config('general.log.log_level')
43 exclude_noha = rp_utils.get_config('functest.exclude_noha')
44 exclude_virtual = rp_utils.get_config('functest.exclude_virtual')
46 response = requests.get(cf)
48 functest_yaml_config = yaml.safe_load(response.text)
50 logger.info("*******************************************")
52 logger.info("* Generating reporting scenario status *")
53 logger.info("* Data retention: %s days *" % period)
54 logger.info("* Log level: %s *" % log_level)
56 logger.info("* Virtual PODs exluded: %s *" % exclude_virtual)
57 logger.info("* NOHA scenarios excluded: %s *" % exclude_noha)
59 logger.info("*******************************************")
61 # Retrieve test cases of Tier 1 (smoke)
62 config_tiers = functest_yaml_config.get("tiers")
64 # we consider Tier 0 (Healthcheck), Tier 1 (smoke),2 (features)
65 # to validate scenarios
66 # Tier > 2 are not used to validate scenarios but we display the results anyway
67 # tricky thing for the API as some tests are Functest tests
68 # other tests are declared directly in the feature projects
69 for tier in config_tiers:
70 if tier['order'] >= 0 and tier['order'] < 2:
71 for case in tier['testcases']:
72 if case['name'] not in blacklist:
73 testValid.append(tc.TestCase(case['name'],
75 case['dependencies']))
76 elif tier['order'] == 2:
77 for case in tier['testcases']:
78 if case['name'] not in blacklist:
79 testValid.append(tc.TestCase(case['name'],
81 case['dependencies']))
82 elif tier['order'] > 2:
83 for case in tier['testcases']:
84 if case['name'] not in blacklist:
85 otherTestCases.append(tc.TestCase(case['name'],
87 case['dependencies']))
89 logger.debug("Functest reporting start")
90 # For all the versions
91 for version in versions:
92 # For all the installers
93 for installer in installers:
95 scenario_results = rp_utils.getScenarios(tempest, installer, version)
96 scenario_stats = rp_utils.getScenarioStats(scenario_results)
98 scenario_result_criteria = {}
99 scenario_file_name = ("./display/" + version +
100 "/functest/scenario_history.txt")
101 # initiate scenario file if it does not exist
102 if not os.path.isfile(scenario_file_name):
103 with open(scenario_file_name, "a") as my_file:
104 logger.debug("Create scenario file: %s" % scenario_file_name)
105 my_file.write("date,scenario,installer,detail,score\n")
107 # For all the scenarios get results
108 for s, s_result in scenario_results.items():
109 logger.info("---------------------------------")
110 logger.info("installer %s, version %s, scenario %s:" %
111 (installer, version, s))
112 logger.debug("Scenario results: %s" % s_result)
114 # Green or Red light for a given scenario
115 nb_test_runnable_for_this_scenario = 0
117 # url of the last jenkins log corresponding to a given
120 if len(s_result) > 0:
121 build_tag = s_result[len(s_result)-1]['build_tag']
122 logger.debug("Build tag: %s" % build_tag)
123 s_url = s_url = rp_utils.getJenkinsUrl(build_tag)
124 logger.info("last jenkins url: %s" % s_url)
125 testCases2BeDisplayed = []
126 # Check if test case is runnable / installer, scenario
127 # for the test case used for Scenario validation
129 # 1) Manage the test cases for the scenario validation
130 # concretely Tiers 0-3
131 for test_case in testValid:
132 test_case.checkRunnable(installer, s,
133 test_case.getConstraints())
134 logger.debug("testcase %s (%s) is %s" %
135 (test_case.getDisplayName(),
137 test_case.isRunnable))
139 if test_case.isRunnable:
140 dbName = test_case.getDbName()
141 name = test_case.getName()
142 displayName = test_case.getDisplayName()
143 project = test_case.getProject()
144 nb_test_runnable_for_this_scenario += 1
145 logger.info(" Searching results for case %s " %
147 result = rp_utils.getResult(dbName, installer,
149 # if no result set the value to 0
152 logger.info(" >>>> Test score = " + str(result))
153 test_case.setCriteria(result)
154 test_case.setIsRunnable(True)
155 testCases2BeDisplayed.append(tc.TestCase(name,
161 scenario_score = scenario_score + result
163 # 2) Manage the test cases for the scenario qualification
164 # concretely Tiers > 3
165 for test_case in otherTestCases:
166 test_case.checkRunnable(installer, s,
167 test_case.getConstraints())
168 logger.debug("testcase %s (%s) is %s" %
169 (test_case.getDisplayName(),
171 test_case.isRunnable))
173 if test_case.isRunnable:
174 dbName = test_case.getDbName()
175 name = test_case.getName()
176 displayName = test_case.getDisplayName()
177 project = test_case.getProject()
178 logger.info(" Searching results for case %s " %
180 result = rp_utils.getResult(dbName, installer,
182 # at least 1 result for the test
184 test_case.setCriteria(result)
185 test_case.setIsRunnable(True)
186 testCases2BeDisplayed.append(tc.TestCase(name,
193 logger.debug("No results found")
195 items[s] = testCases2BeDisplayed
197 logger.error("Error: installer %s, version %s, scenario %s" %
198 (installer, version, s))
199 logger.error("No data available: %s " % (sys.exc_info()[0]))
201 # **********************************************
202 # Evaluate the results for scenario validation
203 # **********************************************
204 # the validation criteria = nb runnable tests x 3
205 # because each test case = 0,1,2 or 3
206 scenario_criteria = nb_test_runnable_for_this_scenario * 3
207 # if 0 runnable tests set criteria at a high value
208 if scenario_criteria < 1:
209 scenario_criteria = 50 # conf.MAX_SCENARIO_CRITERIA
211 s_score = str(scenario_score) + "/" + str(scenario_criteria)
212 s_score_percent = rp_utils.getScenarioPercent(scenario_score,
216 if scenario_score < scenario_criteria:
217 logger.info(">>>> scenario not OK, score = %s/%s" %
218 (scenario_score, scenario_criteria))
221 logger.info(">>>>> scenario OK, save the information")
223 path_validation_file = ("./display/" + version +
225 "validated_scenario_history.txt")
226 with open(path_validation_file, "a") as f:
227 time_format = "%Y-%m-%d %H:%M"
228 info = (datetime.datetime.now().strftime(time_format) +
229 ";" + installer + ";" + s + "\n")
232 # Save daily results in a file
233 with open(scenario_file_name, "a") as f:
234 info = (reportingDate + "," + s + "," + installer +
235 "," + s_score + "," +
236 str(round(s_score_percent)) + "\n")
239 scenario_result_criteria[s] = sr.ScenarioResult(s_status,
243 logger.info("--------------------------")
245 templateLoader = jinja2.FileSystemLoader(".")
246 templateEnv = jinja2.Environment(
247 loader=templateLoader, autoescape=True)
249 TEMPLATE_FILE = "./functest/template/index-status-tmpl.html"
250 template = templateEnv.get_template(TEMPLATE_FILE)
252 outputText = template.render(scenario_stats=scenario_stats,
253 scenario_results=scenario_result_criteria,
260 with open("./display/" + version +
261 "/functest/status-" + installer + ".html", "wb") as fh:
264 logger.info("Manage export CSV & PDF")
265 rp_utils.export_csv(scenario_file_name, installer, version)
266 logger.error("CSV generated...")
268 # Generate outputs for export
270 # TODO Change once web site updated...use the current one
271 # to test pdf production
272 url_pdf = rp_utils.get_config('general.url')
273 pdf_path = ("./display/" + version +
274 "/functest/status-" + installer + ".html")
275 pdf_doc_name = ("./display/" + version +
276 "/functest/status-" + installer + ".pdf")
277 rp_utils.export_pdf(pdf_path, pdf_doc_name)
278 logger.info("PDF generated...")