1 from urllib2 import Request, urlopen, URLError
11 # Declaration of the variables
12 functest_test_list = ['vPing', 'vPing_userdata',
14 'ODL', 'ONOS', 'vIMS']
15 # functest_test_list = ['vPing']
16 companion_test_list = ['doctor/doctor-notification', 'promise/promise']
17 # companion_test_list = []
18 installers = ["apex", "compass", "fuel", "joid"]
19 # installers = ["fuel"]
20 versions = ["brahmaputra", "master"]
21 # versions = ["master"]
24 # Correspondance between the name of the test case and the name in the DB
25 # ideally we should modify the DB to avoid such interface....
26 # '<name in the DB':'<name in the config'>
27 # I know it is uggly...
28 test_match_matrix = {'vPing': 'vping_ssh',
29 'vPing_userdata': 'vping_userdata',
35 'doctor-notification': 'doctor',
39 class TestCase(object):
40 def __init__(self, name, project, criteria=-1, isRunnable=True):
42 self.project = project
43 self.criteria = criteria
44 self.isRunnable = isRunnable
52 def getCriteria(self):
55 def setCriteria(self, criteria):
56 self.criteria = criteria
58 def setIsRunnable(self, isRunnable):
59 self.isRunnable = isRunnable
61 def checkRunnable(self, installer, scenario, config):
62 # Re-use Functest declaration
63 # Retrieve Functest configuration file functest_config.yaml
66 TEST_ENV = functest_yaml_config.get("test-dependencies")
68 # print " *********************** "
70 # print " ---------------------- "
71 # print "case = " + self.name
72 # print "installer = " + installer
73 # print "scenario = " + scenario
74 # print "project = " + self.project
76 # Retrieve test constraints
77 case_name_formated = test_match_matrix[self.name]
80 config_test = TEST_ENV[self.project][case_name_formated]
82 # if not defined in dependencies => no dependencies
83 config_test = TEST_ENV[case_name_formated]
85 print "Error [getTestEnv]:", e
87 # Retrieve test execution param
88 test_execution_context = {"installer": installer,
90 # By default we assume that all the tests are always runnable...
91 # if test_env not empty => dependencies to be checked
92 if config_test is not None and len(config_test) > 0:
93 # possible criteria = ["installer", "scenario"]
94 # consider test criteria from config file
95 # compare towards CI env through CI en variable
96 for criteria in config_test:
97 if re.search(config_test[criteria],
98 test_execution_context[criteria]) is None:
99 # print "Test "+ test + " cannot be run on the environment"
102 self.isRunnable = is_runnable
105 def getApiResults(case, installer, scenario, version):
106 case = case.getName()
107 results = json.dumps([])
108 # to remove proxy (to be removed at the end for local test only)
109 # proxy_handler = urllib2.ProxyHandler({})
110 # opener = urllib2.build_opener(proxy_handler)
111 # urllib2.install_opener(opener)
112 # url = "http://127.0.0.1:8000/results?case=" + case + \
113 # "&period=30&installer=" + installer
114 url = "http://testresults.opnfv.org/testapi/results?case=" + case + \
115 "&period=" + str(PERIOD) + "&installer=" + installer + \
116 "&scenario=" + scenario + "&version=" + version
117 request = Request(url)
120 response = urlopen(request)
122 results = json.loads(k)
124 print 'No kittez. Got an error code:', e
129 def getScenarios(case, installer, version):
131 case = case.getName()
132 url = "http://testresults.opnfv.org/testapi/results?case=" + case + \
133 "&period=" + str(PERIOD) + "&installer=" + installer + \
134 "&version=" + version
135 request = Request(url)
138 response = urlopen(request)
140 results = json.loads(k)
142 print 'Got an error code:', e
144 test_results = results['test_results']
146 if test_results is not None:
147 test_results.reverse()
149 scenario_results = {}
151 for r in test_results:
152 # Retrieve all the scenarios per installer
153 if not r['scenario'] in scenario_results.keys():
154 scenario_results[r['scenario']] = []
155 scenario_results[r['scenario']].append(r)
157 return scenario_results
160 def getScenarioStats(scenario_results):
162 for k, v in scenario_results.iteritems():
163 scenario_stats[k] = len(v)
165 return scenario_stats
168 def getNbtestOk(results):
171 for k, v in r.iteritems():
176 print "Cannot retrieve test status"
180 def getResult(testCase, installer, scenario, version):
182 # retrieve raw results
183 results = getApiResults(testCase, installer, scenario, version)
184 # let's concentrate on test results only
185 test_results = results['test_results']
187 # if results found, analyze them
188 if test_results is not None:
189 test_results.reverse()
191 scenario_results = []
193 # print " ---------------- "
195 # print " ---------------- "
196 # print "nb of results:" + str(len(test_results))
198 for r in test_results:
199 # print r["creation_date"]
200 # print r["criteria"]
201 scenario_results.append({r["creation_date"]: r["criteria"]})
203 scenario_results.sort()
204 # 4 levels for the results
205 # 3: 4+ consecutive runs passing the success criteria
206 # 2: <4 successful consecutive runs but passing the criteria
207 # 1: close to pass the success criteria
208 # 0: 0% success, not passing
209 test_result_indicator = 0
210 nbTestOk = getNbtestOk(scenario_results)
211 # print "Nb test OK:"+ str(nbTestOk)
212 # check that we have at least 4 runs
214 test_result_indicator = 0
216 test_result_indicator = 1
218 # Test the last 4 run
219 if (len(scenario_results) > 3):
220 last4runResults = scenario_results[-4:]
221 if getNbtestOk(last4runResults):
222 test_result_indicator = 3
224 test_result_indicator = 2
226 test_result_indicator = 2
227 print " >>>> Test indicator:" + str(test_result_indicator)
228 return test_result_indicator
230 # ******************************************************************************
231 # ******************************************************************************
232 # ******************************************************************************
233 # ******************************************************************************
234 # ******************************************************************************
236 # init just tempest to get the list of scenarios
237 # as all the scenarios run Tempest
238 tempest = TestCase("Tempest", "functest", -1)
240 # Retrieve the Functest configuration to detect which tests are relevant
241 # according to the installer, scenario
242 response = requests.get('https://git.opnfv.org/cgit/functest/plain/testcases/config_functest.yaml')
243 functest_yaml_config = yaml.load(response.text)
245 print "****************************************"
246 print "* Generating reporting..... *"
247 print "****************************************"
248 # For all the versions
249 for version in versions:
250 # For all the installers
251 for installer in installers:
253 scenario_results = getScenarios(tempest, installer, version)
254 scenario_stats = getScenarioStats(scenario_results)
257 # For all the scenarios get results
258 for s, s_result in scenario_results.items():
260 # For each scenario declare the test cases
262 for test_case in functest_test_list:
263 testCases.append(TestCase(test_case, "functest"))
266 for test_case in companion_test_list:
267 test_split = test_case.split("/")
268 test_project = test_split[0]
269 test_case = test_split[1]
270 testCases.append(TestCase(test_case, test_project))
272 # Check if test case is runnable / installer, scenario
274 for test_case in testCases:
275 test_case.checkRunnable(installer, s, functest_yaml_config)
276 # print "testcase %s is %s" % (test_case.getName(),
277 # test_case.isRunnable)
278 print "--------------------------"
279 print "installer %s, version %s, scenario %s:" % (installer, version, s)
280 for testCase in testCases:
282 if testCase.isRunnable:
283 print " Searching results for case %s " % (testCase.getName())
284 result = getResult(testCase, installer, s, version)
285 testCase.setCriteria(result)
287 print "--------------------------"
289 print "installer %s, version %s, scenario %s" % (installer, version, s)
290 print "No data available , error %s " % (sys.exc_info()[0])
292 print "****************************************"
293 templateLoader = jinja2.FileSystemLoader(os.path.dirname(os.path.abspath(__file__)))
294 templateEnv = jinja2.Environment(loader=templateLoader)
296 TEMPLATE_FILE = "./template/index-status-tmpl.html"
297 template = templateEnv.get_template(TEMPLATE_FILE)
299 outputText = template.render(scenario_stats=scenario_stats,
305 with open("./release/" + version +
306 "/index-status-" + installer + ".html", "wb") as fh: