3 # This program and the accompanying materials
4 # are made available under the terms of the Apache License, Version 2.0
5 # which accompanies this distribution, and is available at
7 # http://www.apache.org/licenses/LICENSE-2.0
16 import scenarioResult as sr
19 import utils.reporting_utils as rp_utils
22 logger = rp_utils.getLogger("Functest-Status")
27 reportingDate = datetime.datetime.now().strftime("%Y-%m-%d %H:%M")
29 # init just connection_check to get the list of scenarios
30 # as all the scenarios run connection_check
31 healthcheck = tc.TestCase("connection_check", "functest", -1)
33 # Retrieve the Functest configuration to detect which tests are relevant
34 # according to the installer, scenario
35 cf = rp_utils.get_config('functest.test_conf')
36 period = rp_utils.get_config('general.period')
37 versions = rp_utils.get_config('general.versions')
38 installers = rp_utils.get_config('general.installers')
39 blacklist = rp_utils.get_config('functest.blacklist')
40 log_level = rp_utils.get_config('general.log.log_level')
41 exclude_noha = rp_utils.get_config('functest.exclude_noha')
42 exclude_virtual = rp_utils.get_config('functest.exclude_virtual')
44 functest_yaml_config = rp_utils.getFunctestConfig()
46 logger.info("*******************************************")
48 logger.info("* Generating reporting scenario status *")
49 logger.info("* Data retention: %s days *" % period)
50 logger.info("* Log level: %s *" % log_level)
52 logger.info("* Virtual PODs exluded: %s *" % exclude_virtual)
53 logger.info("* NOHA scenarios excluded: %s *" % exclude_noha)
55 logger.info("*******************************************")
57 # Retrieve test cases of Tier 1 (smoke)
58 config_tiers = functest_yaml_config.get("tiers")
60 # we consider Tier 0 (Healthcheck), Tier 1 (smoke),2 (features)
61 # to validate scenarios
62 # Tier > 2 are not used to validate scenarios but we display the results anyway
63 # tricky thing for the API as some tests are Functest tests
64 # other tests are declared directly in the feature projects
65 for tier in config_tiers:
66 if tier['order'] >= 0 and tier['order'] < 2:
67 for case in tier['testcases']:
68 if case['case_name'] not in blacklist:
69 testValid.append(tc.TestCase(case['case_name'],
71 case['dependencies']))
72 elif tier['order'] == 2:
73 for case in tier['testcases']:
74 if case['case_name'] not in blacklist:
75 testValid.append(tc.TestCase(case['case_name'],
77 case['dependencies']))
78 elif tier['order'] > 2:
79 for case in tier['testcases']:
80 if case['case_name'] not in blacklist:
81 otherTestCases.append(tc.TestCase(case['case_name'],
83 case['dependencies']))
85 logger.debug("Functest reporting start")
87 # For all the versions
88 for version in versions:
89 # For all the installers
90 scenario_directory = "./display/" + version + "/functest/"
91 scenario_file_name = scenario_directory + "scenario_history.txt"
93 # check that the directory exists, if not create it
94 # (first run on new version)
95 if not os.path.exists(scenario_directory):
96 os.makedirs(scenario_directory)
98 # initiate scenario file if it does not exist
99 if not os.path.isfile(scenario_file_name):
100 with open(scenario_file_name, "a") as my_file:
101 logger.debug("Create scenario file: %s" % scenario_file_name)
102 my_file.write("date,scenario,installer,detail,score\n")
104 for installer in installers:
107 scenario_results = rp_utils.getScenarios(healthcheck,
111 # get nb of supported architecture (x86, aarch64)
112 architectures = rp_utils.getArchitectures(scenario_results)
113 logger.info("Supported architectures: {}".format(architectures))
115 for architecture in architectures:
116 logger.info("architecture: {}".format(architecture))
117 # Consider only the results for the selected architecture
118 # i.e drop x86 for aarch64 and vice versa
119 filter_results = rp_utils.filterArchitecture(scenario_results,
121 scenario_stats = rp_utils.getScenarioStats(filter_results)
123 scenario_result_criteria = {}
125 # in case of more than 1 architecture supported
126 # precise the architecture
127 installer_display = installer
128 if (len(architectures) > 1):
129 installer_display = installer + "@" + architecture
131 # For all the scenarios get results
132 for s, s_result in filter_results.items():
133 logger.info("---------------------------------")
134 logger.info("installer %s, version %s, scenario %s:" %
135 (installer, version, s))
136 logger.debug("Scenario results: %s" % s_result)
138 # Green or Red light for a given scenario
139 nb_test_runnable_for_this_scenario = 0
141 # url of the last jenkins log corresponding to a given
144 if len(s_result) > 0:
145 build_tag = s_result[len(s_result)-1]['build_tag']
146 logger.debug("Build tag: %s" % build_tag)
147 s_url = rp_utils.getJenkinsUrl(build_tag)
149 s_url = "http://testresultS.opnfv.org/reporting"
150 logger.info("last jenkins url: %s" % s_url)
151 testCases2BeDisplayed = []
152 # Check if test case is runnable / installer, scenario
153 # for the test case used for Scenario validation
155 # 1) Manage the test cases for the scenario validation
156 # concretely Tiers 0-3
157 for test_case in testValid:
158 test_case.checkRunnable(installer, s,
159 test_case.getConstraints())
160 logger.debug("testcase %s (%s) is %s" %
161 (test_case.getDisplayName(),
163 test_case.isRunnable))
165 if test_case.isRunnable:
166 dbName = test_case.getDbName()
167 name = test_case.getName()
168 displayName = test_case.getDisplayName()
169 project = test_case.getProject()
170 nb_test_runnable_for_this_scenario += 1
171 logger.info(" Searching results for case %s " %
173 result = rp_utils.getResult(dbName, installer,
175 # if no result set the value to 0
178 logger.info(" >>>> Test score = " + str(result))
179 test_case.setCriteria(result)
180 test_case.setIsRunnable(True)
181 testCases2BeDisplayed.append(tc.TestCase(name,
187 scenario_score = scenario_score + result
189 # 2) Manage the test cases for the scenario qualification
190 # concretely Tiers > 3
191 for test_case in otherTestCases:
192 test_case.checkRunnable(installer, s,
193 test_case.getConstraints())
194 logger.debug("testcase %s (%s) is %s" %
195 (test_case.getDisplayName(),
197 test_case.isRunnable))
199 if test_case.isRunnable:
200 dbName = test_case.getDbName()
201 name = test_case.getName()
202 displayName = test_case.getDisplayName()
203 project = test_case.getProject()
204 logger.info(" Searching results for case %s " %
206 result = rp_utils.getResult(dbName, installer,
208 # at least 1 result for the test
210 test_case.setCriteria(result)
211 test_case.setIsRunnable(True)
212 testCases2BeDisplayed.append(tc.TestCase(
220 logger.debug("No results found")
222 items[s] = testCases2BeDisplayed
224 logger.error("Error: installer %s, version %s, scenario %s"
225 % (installer, version, s))
226 logger.error("No data available: %s" % (sys.exc_info()[0]))
228 # **********************************************
229 # Evaluate the results for scenario validation
230 # **********************************************
231 # the validation criteria = nb runnable tests x 3
232 # because each test case = 0,1,2 or 3
233 scenario_criteria = nb_test_runnable_for_this_scenario * 3
234 # if 0 runnable tests set criteria at a high value
235 if scenario_criteria < 1:
236 scenario_criteria = 50 # conf.MAX_SCENARIO_CRITERIA
238 s_score = str(scenario_score) + "/" + str(scenario_criteria)
239 s_score_percent = rp_utils.getScenarioPercent(
244 if scenario_score < scenario_criteria:
245 logger.info(">>>> scenario not OK, score = %s/%s" %
246 (scenario_score, scenario_criteria))
249 logger.info(">>>>> scenario OK, save the information")
251 path_validation_file = ("./display/" + version +
253 "validated_scenario_history.txt")
254 with open(path_validation_file, "a") as f:
255 time_format = "%Y-%m-%d %H:%M"
256 info = (datetime.datetime.now().strftime(time_format) +
257 ";" + installer_display + ";" + s + "\n")
260 # Save daily results in a file
261 with open(scenario_file_name, "a") as f:
262 info = (reportingDate + "," + s + "," + installer_display +
263 "," + s_score + "," +
264 str(round(s_score_percent)) + "\n")
267 scenario_result_criteria[s] = sr.ScenarioResult(
272 logger.info("--------------------------")
274 templateLoader = jinja2.FileSystemLoader(".")
275 templateEnv = jinja2.Environment(
276 loader=templateLoader, autoescape=True)
278 TEMPLATE_FILE = "./functest/template/index-status-tmpl.html"
279 template = templateEnv.get_template(TEMPLATE_FILE)
281 outputText = template.render(
282 scenario_stats=scenario_stats,
283 scenario_results=scenario_result_criteria,
285 installer=installer_display,
290 with open("./display/" + version +
291 "/functest/status-" +
292 installer_display + ".html", "wb") as fh:
295 logger.info("Manage export CSV & PDF")
296 rp_utils.export_csv(scenario_file_name, installer_display, version)
297 logger.error("CSV generated...")
299 # Generate outputs for export
301 # TODO Change once web site updated...use the current one
302 # to test pdf production
303 url_pdf = rp_utils.get_config('general.url')
304 pdf_path = ("./display/" + version +
305 "/functest/status-" + installer_display + ".html")
306 pdf_doc_name = ("./display/" + version +
307 "/functest/status-" + installer_display + ".pdf")
308 rp_utils.export_pdf(pdf_path, pdf_doc_name)
309 logger.info("PDF generated...")