3 # This program and the accompanying materials
4 # are made available under the terms of the Apache License, Version 2.0
5 # which accompanies this distribution, and is available at
7 # http://www.apache.org/licenses/LICENSE-2.0
16 import reporting.functest.testCase as tc
17 import reporting.functest.scenarioResult as sr
18 import reporting.utils.reporting_utils as rp_utils
21 Functest reporting status
25 LOGGER = rp_utils.getLogger("Functest-Status")
30 reportingDate = datetime.datetime.now().strftime("%Y-%m-%d %H:%M")
32 # init just connection_check to get the list of scenarios
33 # as all the scenarios run connection_check
34 healthcheck = tc.TestCase("connection_check", "functest", -1)
36 # Retrieve the Functest configuration to detect which tests are relevant
37 # according to the installer, scenario
38 cf = rp_utils.get_config('functest.test_conf')
39 period = rp_utils.get_config('general.period')
40 versions = rp_utils.get_config('general.versions')
41 installers = rp_utils.get_config('general.installers')
42 blacklist = rp_utils.get_config('functest.blacklist')
43 log_level = rp_utils.get_config('general.log.log_level')
44 exclude_noha = rp_utils.get_config('functest.exclude_noha')
45 exclude_virtual = rp_utils.get_config('functest.exclude_virtual')
47 functest_yaml_config = rp_utils.getFunctestConfig()
49 LOGGER.info("*******************************************")
51 LOGGER.info("* Generating reporting scenario status *")
52 LOGGER.info("* Data retention: %s days *", period)
53 LOGGER.info("* Log level: %s *", log_level)
55 LOGGER.info("* Virtual PODs exluded: %s *", exclude_virtual)
56 LOGGER.info("* NOHA scenarios excluded: %s *", exclude_noha)
58 LOGGER.info("*******************************************")
60 # Retrieve test cases of Tier 1 (smoke)
61 config_tiers = functest_yaml_config.get("tiers")
63 # we consider Tier 0 (Healthcheck), Tier 1 (smoke),2 (features)
64 # to validate scenarios
65 # Tier > 2 are not used to validate scenarios but we display the results anyway
66 # tricky thing for the API as some tests are Functest tests
67 # other tests are declared directly in the feature projects
68 for tier in config_tiers:
69 if tier['order'] >= 0 and tier['order'] < 2:
70 for case in tier['testcases']:
71 if case['case_name'] not in blacklist:
72 testValid.append(tc.TestCase(case['case_name'],
74 case['dependencies']))
75 elif tier['order'] == 2:
76 for case in tier['testcases']:
77 if case['case_name'] not in blacklist:
78 otherTestCases.append(tc.TestCase(case['case_name'],
80 case['dependencies']))
81 elif tier['order'] > 2:
82 for case in tier['testcases']:
83 if case['case_name'] not in blacklist:
84 otherTestCases.append(tc.TestCase(case['case_name'],
86 case['dependencies']))
88 LOGGER.debug("Functest reporting start")
90 # For all the versions
91 for version in versions:
92 # For all the installers
93 scenario_directory = "./display/" + version + "/functest/"
94 scenario_file_name = scenario_directory + "scenario_history.txt"
96 # check that the directory exists, if not create it
97 # (first run on new version)
98 if not os.path.exists(scenario_directory):
99 os.makedirs(scenario_directory)
101 # initiate scenario file if it does not exist
102 if not os.path.isfile(scenario_file_name):
103 with open(scenario_file_name, "a") as my_file:
104 LOGGER.debug("Create scenario file: %s", scenario_file_name)
105 my_file.write("date,scenario,installer,detail,score\n")
107 for installer in installers:
110 scenario_results = rp_utils.getScenarios("functest",
114 # get nb of supported architecture (x86, aarch64)
115 architectures = rp_utils.getArchitectures(scenario_results)
116 LOGGER.info("Supported architectures: %s", architectures)
118 for architecture in architectures:
119 LOGGER.info("Architecture: %s", architecture)
120 # Consider only the results for the selected architecture
121 # i.e drop x86 for aarch64 and vice versa
122 filter_results = rp_utils.filterArchitecture(scenario_results,
124 scenario_stats = rp_utils.getScenarioStats(filter_results)
126 scenario_result_criteria = {}
128 # in case of more than 1 architecture supported
129 # precise the architecture
130 installer_display = installer
131 if "fuel" in installer:
132 installer_display = installer + "@" + architecture
134 # For all the scenarios get results
135 for s, s_result in filter_results.items():
136 LOGGER.info("---------------------------------")
137 LOGGER.info("installer %s, version %s, scenario %s:",
138 installer, version, s)
139 LOGGER.debug("Scenario results: %s", s_result)
141 # Green or Red light for a given scenario
142 nb_test_runnable_for_this_scenario = 0
144 # url of the last jenkins log corresponding to a given
147 if len(s_result) > 0:
148 build_tag = s_result[len(s_result)-1]['build_tag']
149 LOGGER.debug("Build tag: %s", build_tag)
150 s_url = rp_utils.getJenkinsUrl(build_tag)
152 s_url = "http://testresultS.opnfv.org/reporting"
153 LOGGER.info("last jenkins url: %s", s_url)
154 testCases2BeDisplayed = []
155 # Check if test case is runnable / installer, scenario
156 # for the test case used for Scenario validation
158 # 1) Manage the test cases for the scenario validation
159 # concretely Tiers 0-3
160 for test_case in testValid:
161 test_case.checkRunnable(installer, s,
162 test_case.getConstraints())
163 LOGGER.debug("testcase %s (%s) is %s",
164 test_case.getDisplayName(),
166 test_case.isRunnable)
168 if test_case.isRunnable:
169 name = test_case.getName()
170 displayName = test_case.getDisplayName()
171 project = test_case.getProject()
172 nb_test_runnable_for_this_scenario += 1
173 LOGGER.info(" Searching results for case %s ",
175 if "fuel" in installer:
176 result = rp_utils.getCaseScoreFromBuildTag(
180 result = rp_utils.getCaseScore(name, installer,
182 # if no result set the value to 0
185 LOGGER.info(" >>>> Test score = " + str(result))
186 test_case.setCriteria(result)
187 test_case.setIsRunnable(True)
188 testCases2BeDisplayed.append(tc.TestCase(name,
194 scenario_score = scenario_score + result
196 # 2) Manage the test cases for the scenario qualification
197 # concretely Tiers > 3
198 for test_case in otherTestCases:
199 test_case.checkRunnable(installer, s,
200 test_case.getConstraints())
201 LOGGER.debug("testcase %s (%s) is %s",
202 test_case.getDisplayName(),
204 test_case.isRunnable)
206 if test_case.isRunnable:
207 name = test_case.getName()
208 displayName = test_case.getDisplayName()
209 project = test_case.getProject()
210 LOGGER.info(" Searching results for case %s ",
212 if "fuel" in installer:
213 result = rp_utils.getCaseScoreFromBuildTag(
217 result = rp_utils.getCaseScore(name, installer,
219 # at least 1 result for the test
221 test_case.setCriteria(result)
222 test_case.setIsRunnable(True)
223 testCases2BeDisplayed.append(tc.TestCase(
231 LOGGER.debug("No results found")
233 items[s] = testCases2BeDisplayed
234 except Exception: # pylint: disable=broad-except
235 LOGGER.error("Error installer %s, version %s, scenario %s",
236 installer, version, s)
237 LOGGER.error("No data available: %s", sys.exc_info()[0])
239 # **********************************************
240 # Evaluate the results for scenario validation
241 # **********************************************
242 # the validation criteria = nb runnable tests x 3
243 # because each test case can get
244 # 0 point (never PASS)
245 # 1 point at least (PASS once over the time window)
246 # 2 points (PASS more than once but 1 FAIL on the last 4)
247 # 3 points PASS on the last 4 iterations
248 # e.g. 1 scenario = 10 cases
249 # 1 iteration : max score = 10 (10x1)
250 # 2 iterations : max score = 20 (10x2)
251 # 3 iterations : max score = 20
252 # 4 or more iterations : max score = 30 (1x30)
253 LOGGER.info("Number of iterations for this scenario: %s",
255 if len(s_result) > 3:
257 elif len(s_result) < 2:
262 scenario_criteria = nb_test_runnable_for_this_scenario*k_score
264 # score for reporting
265 s_score = str(scenario_score) + "/" + str(scenario_criteria)
266 s_score_percent = rp_utils.getScenarioPercent(
271 if scenario_score < scenario_criteria:
272 LOGGER.info(">>>> scenario not OK, score = %s/%s",
273 scenario_score, scenario_criteria)
276 LOGGER.info(">>>>> scenario OK, save the information")
278 path_validation_file = ("./display/" + version +
280 "validated_scenario_history.txt")
281 with open(path_validation_file, "a") as f:
282 time_format = "%Y-%m-%d %H:%M"
283 info = (datetime.datetime.now().strftime(time_format) +
284 ";" + installer_display + ";" + s + "\n")
287 # Save daily results in a file
288 with open(scenario_file_name, "a") as f:
289 info = (reportingDate + "," + s + "," + installer_display +
290 "," + s_score + "," +
291 str(round(s_score_percent)) + "\n")
294 scenario_result_criteria[s] = sr.ScenarioResult(
299 LOGGER.info("--------------------------")
301 templateLoader = jinja2.FileSystemLoader(".")
302 templateEnv = jinja2.Environment(
303 loader=templateLoader, autoescape=True)
305 TEMPLATE_FILE = ("./reporting/functest/template"
306 "/index-status-tmpl.html")
307 template = templateEnv.get_template(TEMPLATE_FILE)
309 outputText = template.render(
310 scenario_stats=scenario_stats,
311 scenario_results=scenario_result_criteria,
313 installer=installer_display,
318 with open("./display/" + version +
319 "/functest/status-" +
320 installer_display + ".html", "wb") as fh:
323 LOGGER.info("Manage export CSV & PDF")
324 rp_utils.export_csv(scenario_file_name, installer_display, version)
325 LOGGER.error("CSV generated...")
327 # Generate outputs for export
329 url_pdf = rp_utils.get_config('general.url')
330 pdf_path = ("./display/" + version +
331 "/functest/status-" + installer_display + ".html")
332 pdf_doc_name = ("./display/" + version +
333 "/functest/status-" + installer_display + ".pdf")
334 rp_utils.export_pdf(pdf_path, pdf_doc_name)
335 LOGGER.info("PDF generated...")