3 # This program and the accompanying materials
4 # are made available under the terms of the Apache License, Version 2.0
5 # which accompanies this distribution, and is available at
7 # http://www.apache.org/licenses/LICENSE-2.0
16 import reportingUtils as utils
17 import reportingConf as conf
19 import scenarioResult as sr
22 logger = utils.getLogger("Status")
27 reportingDate = datetime.datetime.now().strftime("%Y-%m-%d %H:%M")
29 # init just tempest to get the list of scenarios
30 # as all the scenarios run Tempest
31 tempest = tc.TestCase("tempest_smoke_serial", "functest", -1)
33 # Retrieve the Functest configuration to detect which tests are relevant
34 # according to the installer, scenario
36 response = requests.get(cf)
38 functest_yaml_config = yaml.safe_load(response.text)
40 logger.info("*******************************************")
41 logger.info("* Generating reporting scenario status *")
42 logger.info("* Data retention = %s days *" % conf.PERIOD)
44 logger.info("*******************************************")
46 # Retrieve test cases of Tier 1 (smoke)
47 config_tiers = functest_yaml_config.get("tiers")
49 # we consider Tier 1 (smoke),2 (sdn suites) and 3 (features)
50 # to validate scenarios
51 # Tier > 4 are not used to validate scenarios but we display the results anyway
52 # tricky thing for the API as some tests are Functest tests
53 # other tests are declared directly in the feature projects
54 for tier in config_tiers:
55 if tier['order'] > 0 and tier['order'] < 3:
56 for case in tier['testcases']:
57 if case['name'] not in conf.blacklist:
58 testValid.append(tc.TestCase(case['name'],
60 case['dependencies']))
61 elif tier['order'] == 3:
62 for case in tier['testcases']:
63 if case['name'] not in conf.blacklist:
64 testValid.append(tc.TestCase(case['name'],
66 case['dependencies']))
67 elif tier['order'] > 3:
68 for case in tier['testcases']:
69 if case['name'] not in conf.blacklist:
70 otherTestCases.append(tc.TestCase(case['name'],
72 case['dependencies']))
74 # For all the versions
75 for version in conf.versions:
76 # For all the installers
77 for installer in conf.installers:
79 scenario_results = utils.getScenarios(tempest, installer, version)
80 scenario_stats = utils.getScenarioStats(scenario_results)
82 scenario_result_criteria = {}
84 # For all the scenarios get results
85 for s, s_result in scenario_results.items():
86 logger.info("---------------------------------")
87 logger.info("installer %s, version %s, scenario %s:" %
88 (installer, version, s))
89 logger.debug("Scenario results: %s" % s_result)
91 # Green or Red light for a given scenario
92 nb_test_runnable_for_this_scenario = 0
94 # url of the last jenkins log corresponding to a given
98 build_tag = s_result[len(s_result)-1]['build_tag']
99 logger.debug("Build tag: %s" % build_tag)
100 s_url = s_url = utils.getJenkinsUrl(build_tag)
101 logger.info("last jenkins url: %s" % s_url)
102 testCases2BeDisplayed = []
103 # Check if test case is runnable / installer, scenario
104 # for the test case used for Scenario validation
106 # 1) Manage the test cases for the scenario validation
107 # concretely Tiers 0-3
108 for test_case in testValid:
109 test_case.checkRunnable(installer, s,
110 test_case.getConstraints())
111 logger.debug("testcase %s (%s) is %s" %
112 (test_case.getDisplayName(),
114 test_case.isRunnable))
116 if test_case.isRunnable:
117 dbName = test_case.getDbName()
118 name = test_case.getName()
119 displayName = test_case.getDisplayName()
120 project = test_case.getProject()
121 nb_test_runnable_for_this_scenario += 1
122 logger.info(" Searching results for case %s " %
124 result = utils.getResult(dbName, installer, s, version)
125 # if no result set the value to 0
128 logger.info(" >>>> Test score = " + str(result))
129 test_case.setCriteria(result)
130 test_case.setIsRunnable(True)
131 testCases2BeDisplayed.append(tc.TestCase(name,
137 scenario_score = scenario_score + result
139 # 2) Manage the test cases for the scenario qualification
140 # concretely Tiers > 3
141 for test_case in otherTestCases:
142 test_case.checkRunnable(installer, s,
143 test_case.getConstraints())
144 logger.debug("testcase %s (%s) is %s" %
145 (test_case.getDisplayName(),
147 test_case.isRunnable))
149 if test_case.isRunnable:
150 dbName = test_case.getDbName()
151 name = test_case.getName()
152 displayName = test_case.getDisplayName()
153 project = test_case.getProject()
154 logger.info(" Searching results for case %s " %
156 result = utils.getResult(dbName, installer, s, version)
157 # at least 1 result for the test
159 test_case.setCriteria(result)
160 test_case.setIsRunnable(True)
161 testCases2BeDisplayed.append(tc.TestCase(name,
168 logger.debug("No results found")
170 items[s] = testCases2BeDisplayed
172 logger.error("Error: installer %s, version %s, scenario %s" %
173 (installer, version, s))
174 logger.error("No data available: %s " % (sys.exc_info()[0]))
176 # **********************************************
177 # Evaluate the results for scenario validation
178 # **********************************************
179 # the validation criteria = nb runnable tests x 3
180 # because each test case = 0,1,2 or 3
181 scenario_criteria = nb_test_runnable_for_this_scenario * 3
182 # if 0 runnable tests set criteria at a high value
183 if scenario_criteria < 1:
184 scenario_criteria = conf.MAX_SCENARIO_CRITERIA
186 s_score = str(scenario_score) + "/" + str(scenario_criteria)
187 s_score_percent = 0.0
189 s_score_percent = float(
190 scenario_score) / float(scenario_criteria) * 100
192 logger.error("cannot calculate the score percent")
195 if scenario_score < scenario_criteria:
196 logger.info(">>>> scenario not OK, score = %s/%s" %
197 (scenario_score, scenario_criteria))
200 logger.info(">>>>> scenario OK, save the information")
202 path_validation_file = (conf.REPORTING_PATH +
203 "/release/" + version +
204 "/validated_scenario_history.txt")
205 with open(path_validation_file, "a") as f:
206 time_format = "%Y-%m-%d %H:%M"
207 info = (datetime.datetime.now().strftime(time_format) +
208 ";" + installer + ";" + s + "\n")
211 # Save daily results in a file
212 path_validation_file = (conf.REPORTING_PATH +
213 "/release/" + version +
214 "/scenario_history.txt")
215 with open(path_validation_file, "a") as f:
216 info = (reportingDate + "," + s + "," + installer +
217 "," + s_score + "," +
218 str(round(s_score_percent)) + "\n")
221 scenario_result_criteria[s] = sr.ScenarioResult(s_status,
225 logger.info("--------------------------")
227 templateLoader = jinja2.FileSystemLoader(conf.REPORTING_PATH)
228 templateEnv = jinja2.Environment(
229 loader=templateLoader, autoescape=True)
231 TEMPLATE_FILE = "/template/index-status-tmpl.html"
232 template = templateEnv.get_template(TEMPLATE_FILE)
234 outputText = template.render(scenario_stats=scenario_stats,
235 scenario_results=scenario_result_criteria,
242 with open(conf.REPORTING_PATH + "/release/" + version +
243 "/index-status-" + installer + ".html", "wb") as fh: