3 # This program and the accompanying materials
4 # are made available under the terms of the Apache License, Version 2.0
5 # which accompanies this distribution, and is available at
7 # http://www.apache.org/licenses/LICENSE-2.0
16 import reportingUtils as utils
17 import reportingConf as conf
19 import scenarioResult as sr
22 logger = utils.getLogger("Status")
28 # init just tempest to get the list of scenarios
29 # as all the scenarios run Tempest
30 tempest = tc.TestCase("tempest_smoke_serial", "functest", -1)
32 # Retrieve the Functest configuration to detect which tests are relevant
33 # according to the installer, scenario
35 response = requests.get(cf)
37 functest_yaml_config = yaml.load(response.text)
39 logger.info("*******************************************")
40 logger.info("* Generating reporting scenario status *")
41 logger.info("* Data retention = %s days *" % conf.PERIOD)
43 logger.info("*******************************************")
45 # Retrieve test cases of Tier 1 (smoke)
46 config_tiers = functest_yaml_config.get("tiers")
48 # we consider Tier 1 (smoke),2 (sdn suites) and 3 (features)
49 # to validate scenarios
50 # Tier > 4 are not used to validate scenarios but we display the results anyway
51 # tricky thing for the API as some tests are Functest tests
52 # other tests are declared directly in the feature projects
53 for tier in config_tiers:
54 if tier['order'] > 0 and tier['order'] < 3:
55 for case in tier['testcases']:
56 if case['name'] not in conf.blacklist:
57 testValid.append(tc.TestCase(case['name'],
59 case['dependencies']))
60 elif tier['order'] == 3:
61 for case in tier['testcases']:
62 if case['name'] not in conf.blacklist:
63 testValid.append(tc.TestCase(case['name'],
65 case['dependencies']))
66 elif tier['order'] > 3:
67 for case in tier['testcases']:
68 if case['name'] not in conf.blacklist:
69 otherTestCases.append(tc.TestCase(case['name'],
71 case['dependencies']))
73 # For all the versions
74 for version in conf.versions:
75 # For all the installers
76 for installer in conf.installers:
78 scenario_results = utils.getScenarios(tempest, installer, version)
79 scenario_stats = utils.getScenarioStats(scenario_results)
81 scenario_result_criteria = {}
83 # For all the scenarios get results
84 for s, s_result in scenario_results.items():
85 # Green or Red light for a given scenario
86 nb_test_runnable_for_this_scenario = 0
89 testCases2BeDisplayed = []
90 # Check if test case is runnable / installer, scenario
91 # for the test case used for Scenario validation
93 logger.info("---------------------------------")
94 logger.info("installer %s, version %s, scenario %s:" %
95 (installer, version, s))
97 # 1) Manage the test cases for the scenario validation
98 # concretely Tiers 0-3
99 for test_case in testValid:
100 test_case.checkRunnable(installer, s,
101 test_case.getConstraints())
102 logger.debug("testcase %s (%s) is %s" %
103 (test_case.getDisplayName(),
105 test_case.isRunnable))
107 if test_case.isRunnable:
108 dbName = test_case.getDbName()
109 name = test_case.getName()
110 displayName = test_case.getDisplayName()
111 project = test_case.getProject()
112 nb_test_runnable_for_this_scenario += 1
113 logger.info(" Searching results for case %s " %
115 result = utils.getResult(dbName, installer, s, version)
116 # if no result set the value to 0
119 logger.info(" >>>> Test score = " + str(result))
120 test_case.setCriteria(result)
121 test_case.setIsRunnable(True)
122 testCases2BeDisplayed.append(tc.TestCase(name,
128 scenario_score = scenario_score + result
130 # 2) Manage the test cases for the scenario qualification
131 # concretely Tiers > 3
132 for test_case in otherTestCases:
133 test_case.checkRunnable(installer, s,
134 test_case.getConstraints())
135 logger.debug("testcase %s (%s) is %s" %
136 (test_case.getDisplayName(),
138 test_case.isRunnable))
140 if test_case.isRunnable:
141 dbName = test_case.getDbName()
142 name = test_case.getName()
143 displayName = test_case.getDisplayName()
144 project = test_case.getProject()
145 logger.info(" Searching results for case %s " %
147 result = utils.getResult(dbName, installer, s, version)
148 # at least 1 result for the test
150 test_case.setCriteria(result)
151 test_case.setIsRunnable(True)
152 testCases2BeDisplayed.append(tc.TestCase(name,
159 logger.debug("No results found")
161 items[s] = testCases2BeDisplayed
163 logger.error("Error: installer %s, version %s, scenario %s" %
164 (installer, version, s))
165 logger.error("No data available: %s " % (sys.exc_info()[0]))
167 # **********************************************
168 # Evaluate the results for scenario validation
169 # **********************************************
170 # the validation criteria = nb runnable tests x 3
171 # because each test case = 0,1,2 or 3
172 scenario_criteria = nb_test_runnable_for_this_scenario * 3
173 # if 0 runnable tests set criteria at a high value
174 if scenario_criteria < 1:
175 scenario_criteria = conf.MAX_SCENARIO_CRITERIA
177 s_score = str(scenario_score) + "/" + str(scenario_criteria)
179 if scenario_score < scenario_criteria:
180 logger.info(">>>> scenario not OK, score = %s/%s" %
181 (scenario_score, scenario_criteria))
184 logger.info(">>>>> scenario OK, save the information")
186 path_validation_file = (conf.REPORTING_PATH + "/release/" + version +
187 "/validated_scenario_history.txt")
188 with open(path_validation_file, "a") as f:
189 time_format = "%Y-%m-%d %H:%M"
190 info = (datetime.datetime.now().strftime(time_format) +
191 ";" + installer + ";" + s + "\n")
194 scenario_result_criteria[s] = sr.ScenarioResult(s_status, s_score)
195 logger.info("--------------------------")
197 templateLoader = jinja2.FileSystemLoader(conf.REPORTING_PATH)
198 templateEnv = jinja2.Environment(loader=templateLoader, autoescape=True)
200 TEMPLATE_FILE = "/template/index-status-tmpl.html"
201 template = templateEnv.get_template(TEMPLATE_FILE)
203 outputText = template.render(scenario_stats=scenario_stats,
204 scenario_results=scenario_result_criteria,
210 with open(conf.REPORTING_PATH + "/release/" + version +
211 "/index-status-" + installer + ".html", "wb") as fh: