12 logger = logging.getLogger('mongo_to_elasticsearch')
13 logger.setLevel(logging.DEBUG)
14 file_handler = logging.FileHandler('/var/log/{}.log'.format('mongo_to_elasticsearch'))
15 file_handler.setFormatter(logging.Formatter('%(asctime)s %(levelname)s: %(message)s'))
16 logger.addHandler(file_handler)
19 def _get_dicts_from_list(testcase, dict_list, keys):
21 for dictionary in dict_list:
22 # iterate over dictionaries in input list
23 if not isinstance(dictionary, dict):
24 logger.info("Skipping non-dict details testcase '{}'".format(testcase))
26 if keys == set(dictionary.keys()):
27 # check the dictionary structure
28 dicts.append(dictionary)
32 def _get_results_from_list_of_dicts(list_of_dict_statuses, dict_indexes, expected_results=None):
34 for test_status in list_of_dict_statuses:
36 for index in dict_indexes:
37 status = status[index]
38 if status in test_results:
39 test_results[status] += 1
41 test_results[status] = 1
43 if expected_results is not None:
44 for expected_result in expected_results:
45 if expected_result not in test_results:
46 test_results[expected_result] = 0
51 def _convert_value(value):
52 return value if value != '' else 0
55 def _convert_duration(duration):
56 if (isinstance(duration, str) or isinstance(duration, unicode)) and ':' in duration:
57 hours, minutes, seconds = duration.split(":")
58 hours = _convert_value(hours)
59 minutes = _convert_value(minutes)
60 seconds = _convert_value(seconds)
61 int_duration = 3600 * int(hours) + 60 * int(minutes) + float(seconds)
63 int_duration = duration
67 def modify_functest_tempest(testcase):
68 if modify_default_entry(testcase):
69 testcase_details = testcase['details']
70 testcase_tests = float(testcase_details['tests'])
71 testcase_failures = float(testcase_details['failures'])
72 if testcase_tests != 0:
73 testcase_details['success_percentage'] = 100 * (testcase_tests - testcase_failures) / testcase_tests
75 testcase_details['success_percentage'] = 0
81 def modify_functest_vims(testcase):
84 details.sig_test.result.[{result}]
85 details.sig_test.duration
87 details.orchestrator.duration
89 Find data for these fields
90 -> details.sig_test.duration
91 -> details.sig_test.tests
92 -> details.sig_test.failures
93 -> details.sig_test.passed
94 -> details.sig_test.skipped
95 -> details.vIMS.duration
96 -> details.orchestrator.duration
98 testcase_details = testcase['details']
99 sig_test_results = _get_dicts_from_list(testcase, testcase_details['sig_test']['result'],
100 {'duration', 'result', 'name', 'error'})
101 if len(sig_test_results) < 1:
102 logger.info("No 'result' from 'sig_test' found in vIMS details, skipping")
105 test_results = _get_results_from_list_of_dicts(sig_test_results, ('result',), ('Passed', 'Skipped', 'Failed'))
106 passed = test_results['Passed']
107 skipped = test_results['Skipped']
108 failures = test_results['Failed']
109 all_tests = passed + skipped + failures
110 testcase['details'] = {
112 'duration': testcase_details['sig_test']['duration'],
114 'failures': failures,
119 'duration': testcase_details['vIMS']['duration']
122 'duration': testcase_details['orchestrator']['duration']
128 def modify_functest_onos(testcase):
131 details.FUNCvirNet.duration
132 details.FUNCvirNet.status.[{Case result}]
133 details.FUNCvirNetL3.duration
134 details.FUNCvirNetL3.status.[{Case result}]
136 Find data for these fields
137 -> details.FUNCvirNet.duration
138 -> details.FUNCvirNet.tests
139 -> details.FUNCvirNet.failures
140 -> details.FUNCvirNetL3.duration
141 -> details.FUNCvirNetL3.tests
142 -> details.FUNCvirNetL3.failures
144 testcase_details = testcase['details']
146 if 'FUNCvirNet' not in testcase_details:
147 return modify_default_entry(testcase)
149 funcvirnet_details = testcase_details['FUNCvirNet']['status']
150 funcvirnet_statuses = _get_dicts_from_list(testcase, funcvirnet_details, {'Case result', 'Case name:'})
152 funcvirnetl3_details = testcase_details['FUNCvirNetL3']['status']
153 funcvirnetl3_statuses = _get_dicts_from_list(testcase, funcvirnetl3_details, {'Case result', 'Case name:'})
155 if len(funcvirnet_statuses) < 0:
156 logger.info("No results found in 'FUNCvirNet' part of ONOS results")
158 elif len(funcvirnetl3_statuses) < 0:
159 logger.info("No results found in 'FUNCvirNetL3' part of ONOS results")
162 funcvirnet_results = _get_results_from_list_of_dicts(funcvirnet_statuses,
163 ('Case result',), ('PASS', 'FAIL'))
164 funcvirnetl3_results = _get_results_from_list_of_dicts(funcvirnetl3_statuses,
165 ('Case result',), ('PASS', 'FAIL'))
167 funcvirnet_passed = funcvirnet_results['PASS']
168 funcvirnet_failed = funcvirnet_results['FAIL']
169 funcvirnet_all = funcvirnet_passed + funcvirnet_failed
171 funcvirnetl3_passed = funcvirnetl3_results['PASS']
172 funcvirnetl3_failed = funcvirnetl3_results['FAIL']
173 funcvirnetl3_all = funcvirnetl3_passed + funcvirnetl3_failed
175 testcase_details['FUNCvirNet'] = {
176 'duration': _convert_duration(testcase_details['FUNCvirNet']['duration']),
177 'tests': funcvirnet_all,
178 'failures': funcvirnet_failed
181 testcase_details['FUNCvirNetL3'] = {
182 'duration': _convert_duration(testcase_details['FUNCvirNetL3']['duration']),
183 'tests': funcvirnetl3_all,
184 'failures': funcvirnetl3_failed
190 def modify_functest_rally(testcase):
193 details.[{summary.duration}]
194 details.[{summary.nb success}]
195 details.[{summary.nb tests}]
197 Find data for these fields
200 -> details.success_percentage
202 summaries = _get_dicts_from_list(testcase, testcase['details'], {'summary'})
204 if len(summaries) != 1:
205 logger.info("Found zero or more than one 'summaries' in Rally details, skipping")
208 summary = summaries[0]['summary']
209 testcase['details'] = {
210 'duration': summary['duration'],
211 'tests': summary['nb tests'],
212 'success_percentage': summary['nb success']
217 def modify_functest_odl(testcase):
220 details.details.[{test_status.@status}]
222 Find data for these fields
225 -> details.success_percentage?
227 test_statuses = _get_dicts_from_list(testcase, testcase['details']['details'],
228 {'test_status', 'test_doc', 'test_name'})
229 if len(test_statuses) < 1:
230 logger.info("No 'test_status' found in ODL details, skipping")
233 test_results = _get_results_from_list_of_dicts(test_statuses, ('test_status', '@status'), ('PASS', 'FAIL'))
235 passed_tests = test_results['PASS']
236 failed_tests = test_results['FAIL']
237 all_tests = passed_tests + failed_tests
239 testcase['details'] = {
241 'failures': failed_tests,
242 'success_percentage': 100 * passed_tests / float(all_tests)
244 logger.debug("Modified odl testcase: '{}'".format(json.dumps(testcase, indent=2)))
248 def modify_default_entry(testcase):
250 Look for these and leave any of those:
255 If none are present, then return False
258 testcase_details = testcase['details']
259 fields = ['duration', 'tests', 'failures']
260 if isinstance(testcase_details, dict):
261 for key, value in testcase_details.items():
264 if key == 'duration':
265 testcase_details[key] = _convert_duration(value)
267 del testcase_details[key]
272 def _fix_date(date_string):
273 if isinstance(date_string, dict):
274 return date_string['$date']
276 return date_string[:-3].replace(' ', 'T') + 'Z'
279 def verify_mongo_entry(testcase):
290 these fields must be present and must NOT be None
295 these fields will be preserved if the are NOT None
297 mandatory_fields = ['installer',
303 mandatory_fields_to_modify = {'start_date': _fix_date}
304 fields_to_swap_or_add = {'scenario': 'version'}
305 if '_id' in testcase:
306 mongo_id = testcase['_id']
309 optional_fields = ['description']
310 for key, value in testcase.items():
311 if key in mandatory_fields:
313 # empty mandatory field, invalid input
314 logger.info("Skipping testcase with mongo _id '{}' because the testcase was missing value"
315 " for mandatory field '{}'".format(mongo_id, key))
318 mandatory_fields.remove(key)
319 elif key in mandatory_fields_to_modify:
321 # empty mandatory field, invalid input
322 logger.info("Skipping testcase with mongo _id '{}' because the testcase was missing value"
323 " for mandatory field '{}'".format(mongo_id, key))
326 testcase[key] = mandatory_fields_to_modify[key](value)
327 del mandatory_fields_to_modify[key]
328 elif key in fields_to_swap_or_add:
330 swapped_key = fields_to_swap_or_add[key]
331 swapped_value = testcase[swapped_key]
332 logger.info("Swapping field '{}' with value None for '{}' with value '{}'.".format(key, swapped_key, swapped_value))
333 testcase[key] = swapped_value
334 del fields_to_swap_or_add[key]
336 del fields_to_swap_or_add[key]
337 elif key in optional_fields:
339 # empty optional field, remove
341 optional_fields.remove(key)
346 if len(mandatory_fields) > 0:
347 # some mandatory fields are missing
348 logger.info("Skipping testcase with mongo _id '{}' because the testcase was missing"
349 " mandatory field(s) '{}'".format(mongo_id, mandatory_fields))
351 elif len(mandatory_fields_to_modify) > 0:
352 # some mandatory fields are missing
353 logger.info("Skipping testcase with mongo _id '{}' because the testcase was missing"
354 " mandatory field(s) '{}'".format(mongo_id, mandatory_fields_to_modify.keys()))
357 if len(fields_to_swap_or_add) > 0:
358 for key, swap_key in fields_to_swap_or_add.iteritems():
359 testcase[key] = testcase[swap_key]
364 def modify_mongo_entry(testcase):
365 # 1. verify and identify the testcase
366 # 2. if modification is implemented, then use that
367 # 3. if not, try to use default
368 # 4. if 2 or 3 is successful, return True, otherwise return False
369 if verify_mongo_entry(testcase):
370 project = testcase['project_name']
371 case_name = testcase['case_name']
372 logger.info("Processing mongo test case '{}'".format(case_name))
373 if project == 'functest':
374 if case_name == 'rally_sanity':
375 return modify_functest_rally(testcase)
376 elif case_name.lower() == 'odl':
377 return modify_functest_odl(testcase)
378 elif case_name.lower() == 'onos':
379 return modify_functest_onos(testcase)
380 elif case_name.lower() == 'vims':
381 return modify_functest_vims(testcase)
382 elif case_name == 'tempest_smoke_serial':
383 return modify_functest_tempest(testcase)
384 return modify_default_entry(testcase)
389 def publish_mongo_data(output_destination):
390 tmp_filename = 'mongo-{}.log'.format(uuid.uuid4())
392 subprocess.check_call(['mongoexport', '--db', 'test_results_collection', '-c', 'results', '--out',
394 with open(tmp_filename) as fobj:
395 for mongo_json_line in fobj:
396 test_result = json.loads(mongo_json_line)
397 if modify_mongo_entry(test_result):
398 status, data = shared_utils.publish_json(test_result, es_creds, output_destination)
400 project = test_result['project_name']
401 case_name = test_result['case_name']
402 logger.info('project {} case {} publish failed, due to [{}]'
403 .format(project, case_name, json.loads(data)['error']['reason']))
405 if os.path.exists(tmp_filename):
406 os.remove(tmp_filename)
409 def get_mongo_data(days):
410 past_time = datetime.datetime.today() - datetime.timedelta(days=days)
411 mongo_json_lines = subprocess.check_output(['mongoexport', '--db', 'test_results_collection', '-c', 'results',
412 '--query', '{{"start_date":{{$gt:"{}"}}}}'
413 .format(past_time)]).splitlines()
416 for mongo_json_line in mongo_json_lines:
417 test_result = json.loads(mongo_json_line)
418 if modify_mongo_entry(test_result):
419 # if the modification could be applied, append the modified result
420 mongo_data.append(test_result)
424 def publish_difference(mongo_data, elastic_data, output_destination, es_creds):
425 for elastic_entry in elastic_data:
426 if elastic_entry in mongo_data:
427 mongo_data.remove(elastic_entry)
429 logger.info('number of parsed test results: {}'.format(len(mongo_data)))
431 for parsed_test_result in mongo_data:
432 shared_utils.publish_json(parsed_test_result, es_creds, output_destination)
435 if __name__ == '__main__':
436 parser = argparse.ArgumentParser(description='Modify and filter mongo json data for elasticsearch')
437 parser.add_argument('-od', '--output-destination',
438 default='elasticsearch',
439 choices=('elasticsearch', 'stdout'),
440 help='defaults to elasticsearch')
442 parser.add_argument('-ml', '--merge-latest', default=0, type=int, metavar='N',
443 help='get entries old at most N days from mongodb and'
444 ' parse those that are not already in elasticsearch.'
445 ' If not present, will get everything from mongodb, which is the default')
447 parser.add_argument('-e', '--elasticsearch-url', default='http://localhost:9200',
448 help='the url of elasticsearch, defaults to http://localhost:9200')
450 parser.add_argument('-u', '--elasticsearch-username', default=None,
451 help='The username with password for elasticsearch in format username:password')
453 args = parser.parse_args()
454 base_elastic_url = urlparse.urljoin(args.elasticsearch_url, '/test_results/mongo2elastic')
455 output_destination = args.output_destination
456 days = args.merge_latest
457 es_creds = args.elasticsearch_username
459 if output_destination == 'elasticsearch':
460 output_destination = base_elastic_url
462 # parsed_test_results will be printed/sent to elasticsearch
464 publish_mongo_data(output_destination)
475 elastic_data = shared_utils.get_elastic_data(base_elastic_url, es_creds, body)
476 logger.info('number of hits in elasticsearch for now-{}d: {}'.format(days, len(elastic_data)))
477 mongo_data = get_mongo_data(days)
478 publish_difference(mongo_data, elastic_data, output_destination, es_creds)
480 raise Exception('Update must be non-negative')