utils/test/scripts/mongo_to_elasticsearch.py

   1 #! /usr/bin/env python
   2 import logging
   3 import argparse
   4 import shared_utils
   5 import json
   6 import urlparse
   7 import uuid
   8 import os
   9 import subprocess
  10 import datetime
  11
  12 logger = logging.getLogger('mongo_to_elasticsearch')
  13 logger.setLevel(logging.DEBUG)
  14 file_handler = logging.FileHandler('/var/log/{}.log'.format(__name__))
  15 file_handler.setFormatter(logging.Formatter('%(asctime)s %(levelname)s: %(message)s'))
  16 logger.addHandler(file_handler)
  17
  18
  19 def _get_dicts_from_list(testcase, dict_list, keys):
  20     dicts = []
  21     for dictionary in dict_list:
  22         # iterate over dictionaries in input list
  23         if not isinstance(dictionary, dict):
  24             logger.info("Skipping non-dict details testcase [{}]".format(testcase))
  25             continue
  26         if keys == set(dictionary.keys()):
  27             # check the dictionary structure
  28             dicts.append(dictionary)
  29     return dicts
  30
  31
  32 def _get_results_from_list_of_dicts(list_of_dict_statuses, dict_indexes, expected_results=None):
  33     test_results = {}
  34     for test_status in list_of_dict_statuses:
  35         status = test_status
  36         for index in dict_indexes:
  37             status = status[index]
  38         if status in test_results:
  39             test_results[status] += 1
  40         else:
  41             test_results[status] = 1
  42
  43     if expected_results is not None:
  44         for expected_result in expected_results:
  45             if expected_result not in test_results:
  46                 test_results[expected_result] = 0
  47
  48     return test_results
  49
  50
  51 def _convert_duration(duration):
  52     if (isinstance(duration, str) or isinstance(duration, unicode)) and ':' in duration:
  53         hours, minutes, seconds = duration.split(":")
  54         int_duration = 3600 * int(hours) + 60 * int(minutes) + float(seconds)
  55     else:
  56         int_duration = duration
  57     return int_duration
  58
  59
  60 def modify_functest_tempest(testcase):
  61     if modify_default_entry(testcase):
  62         testcase_details = testcase['details']
  63         testcase_tests = float(testcase_details['tests'])
  64         testcase_failures = float(testcase_details['failures'])
  65         if testcase_tests != 0:
  66             testcase_details['success_percentage'] = 100 * (testcase_tests - testcase_failures) / testcase_tests
  67         else:
  68             testcase_details['success_percentage'] = 0
  69         return True
  70     else:
  71         return False
  72
  73
  74 def modify_functest_vims(testcase):
  75     """
  76     Structure:
  77         details.sig_test.result.[{result}]
  78         details.sig_test.duration
  79         details.vIMS.duration
  80         details.orchestrator.duration
  81
  82     Find data for these fields
  83         -> details.sig_test.duration
  84         -> details.sig_test.tests
  85         -> details.sig_test.failures
  86         -> details.sig_test.passed
  87         -> details.sig_test.skipped
  88         -> details.vIMS.duration
  89         -> details.orchestrator.duration
  90     """
  91     testcase_details = testcase['details']
  92     sig_test_results = _get_dicts_from_list(testcase, testcase_details['sig_test']['result'],
  93                                             {'duration', 'result', 'name', 'error'})
  94     if len(sig_test_results) < 1:
  95         logger.info("No 'result' from 'sig_test' found in vIMS details, skipping")
  96         return False
  97     else:
  98         test_results = _get_results_from_list_of_dicts(sig_test_results, ('result',), ('Passed', 'Skipped', 'Failed'))
  99         passed = test_results['Passed']
 100         skipped = test_results['Skipped']
 101         failures = test_results['Failed']
 102         all_tests = passed + skipped + failures
 103         testcase['details'] = {
 104             'sig_test': {
 105                 'duration': testcase_details['sig_test']['duration'],
 106                 'tests': all_tests,
 107                 'failures': failures,
 108                 'passed': passed,
 109                 'skipped': skipped
 110             },
 111             'vIMS': {
 112                 'duration': testcase_details['vIMS']['duration']
 113             },
 114             'orchestrator': {
 115                 'duration': testcase_details['orchestrator']['duration']
 116             }
 117         }
 118         return True
 119
 120
 121 def modify_functest_onos(testcase):
 122     """
 123     Structure:
 124         details.FUNCvirNet.duration
 125         details.FUNCvirNet.status.[{Case result}]
 126         details.FUNCvirNetL3.duration
 127         details.FUNCvirNetL3.status.[{Case result}]
 128
 129     Find data for these fields
 130         -> details.FUNCvirNet.duration
 131         -> details.FUNCvirNet.tests
 132         -> details.FUNCvirNet.failures
 133         -> details.FUNCvirNetL3.duration
 134         -> details.FUNCvirNetL3.tests
 135         -> details.FUNCvirNetL3.failures
 136     """
 137     testcase_details = testcase['details']
 138
 139     funcvirnet_details = testcase_details['FUNCvirNet']['status']
 140     funcvirnet_statuses = _get_dicts_from_list(testcase, funcvirnet_details, {'Case result', 'Case name:'})
 141
 142     funcvirnetl3_details = testcase_details['FUNCvirNetL3']['status']
 143     funcvirnetl3_statuses = _get_dicts_from_list(testcase, funcvirnetl3_details, {'Case result', 'Case name:'})
 144
 145     if len(funcvirnet_statuses) < 0:
 146         logger.info("No results found in 'FUNCvirNet' part of ONOS results")
 147         return False
 148     elif len(funcvirnetl3_statuses) < 0:
 149         logger.info("No results found in 'FUNCvirNetL3' part of ONOS results")
 150         return False
 151     else:
 152         funcvirnet_results = _get_results_from_list_of_dicts(funcvirnet_statuses,
 153                                                              ('Case result',), ('PASS', 'FAIL'))
 154         funcvirnetl3_results = _get_results_from_list_of_dicts(funcvirnetl3_statuses,
 155                                                                ('Case result',), ('PASS', 'FAIL'))
 156
 157         funcvirnet_passed = funcvirnet_results['PASS']
 158         funcvirnet_failed = funcvirnet_results['FAIL']
 159         funcvirnet_all = funcvirnet_passed + funcvirnet_failed
 160
 161         funcvirnetl3_passed = funcvirnetl3_results['PASS']
 162         funcvirnetl3_failed = funcvirnetl3_results['FAIL']
 163         funcvirnetl3_all = funcvirnetl3_passed + funcvirnetl3_failed
 164
 165         testcase_details['FUNCvirNet'] = {
 166             'duration': _convert_duration(testcase_details['FUNCvirNet']['duration']),
 167             'tests': funcvirnet_all,
 168             'failures': funcvirnet_failed
 169         }
 170
 171         testcase_details['FUNCvirNetL3'] = {
 172             'duration': _convert_duration(testcase_details['FUNCvirNetL3']['duration']),
 173             'tests': funcvirnetl3_all,
 174             'failures': funcvirnetl3_failed
 175         }
 176
 177         return True
 178
 179
 180 def modify_functest_rally(testcase):
 181     """
 182     Structure:
 183         details.[{summary.duration}]
 184         details.[{summary.nb success}]
 185         details.[{summary.nb tests}]
 186
 187     Find data for these fields
 188         -> details.duration
 189         -> details.tests
 190         -> details.success_percentage
 191     """
 192     summaries = _get_dicts_from_list(testcase, testcase['details'], {'summary'})
 193
 194     if len(summaries) != 1:
 195         logger.info("Found zero or more than one 'summaries' in Rally details, skipping")
 196         return False
 197     else:
 198         summary = summaries[0]['summary']
 199         testcase['details'] = {
 200             'duration': summary['duration'],
 201             'tests': summary['nb tests'],
 202             'success_percentage': summary['nb success']
 203         }
 204         return True
 205
 206
 207 def modify_functest_odl(testcase):
 208     """
 209     Structure:
 210         details.details.[{test_status.@status}]
 211
 212     Find data for these fields
 213         -> details.tests
 214         -> details.failures
 215         -> details.success_percentage?
 216     """
 217     test_statuses = _get_dicts_from_list(testcase, testcase['details']['details'],
 218                                          {'test_status', 'test_doc', 'test_name'})
 219     if len(test_statuses) < 1:
 220         logger.info("No 'test_status' found in ODL details, skipping")
 221         return False
 222     else:
 223         test_results = _get_results_from_list_of_dicts(test_statuses, ('test_status', '@status'), ('PASS', 'FAIL'))
 224
 225         passed_tests = test_results['PASS']
 226         failed_tests = test_results['FAIL']
 227         all_tests = passed_tests + failed_tests
 228
 229         testcase['details'] = {
 230             'tests': all_tests,
 231             'failures': failed_tests,
 232             'success_percentage': 100 * passed_tests / float(all_tests)
 233         }
 234         return True
 235
 236
 237 def modify_default_entry(testcase):
 238     """
 239     Look for these and leave any of those:
 240         details.duration
 241         details.tests
 242         details.failures
 243
 244     If none are present, then return False
 245     """
 246     found = False
 247     testcase_details = testcase['details']
 248     fields = ['duration', 'tests', 'failures']
 249     if isinstance(testcase_details, dict):
 250         for key, value in testcase_details.items():
 251             if key in fields:
 252                 found = True
 253                 if key == 'duration':
 254                     testcase_details[key] = _convert_duration(value)
 255             else:
 256                 del testcase_details[key]
 257
 258     return found
 259
 260
 261 def _fix_date(date_string):
 262     if isinstance(date_string, dict):
 263         return date_string['$date']
 264     else:
 265         return date_string[:-3].replace(' ', 'T') + 'Z'
 266
 267
 268 def verify_mongo_entry(testcase):
 269     """
 270     Mandatory fields:
 271         installer
 272         pod_name
 273         version
 274         case_name
 275         date
 276         project
 277         details
 278
 279         these fields must be present and must NOT be None
 280
 281     Optional fields:
 282         description
 283
 284         these fields will be preserved if the are NOT None
 285     """
 286     mandatory_fields = ['installer',
 287                         'pod_name',
 288                         'version',
 289                         'case_name',
 290                         'project_name',
 291                         'details']
 292     mandatory_fields_to_modify = {'creation_date': _fix_date}
 293     if '_id' in testcase:
 294         mongo_id = testcase['_id']
 295     else:
 296         mongo_id = None
 297     optional_fields = ['description']
 298     for key, value in testcase.items():
 299         if key in mandatory_fields:
 300             if value is None:
 301                 # empty mandatory field, invalid input
 302                 logger.info("Skipping testcase with mongo _id '{}' because the testcase was missing value"
 303                             " for mandatory field '{}'".format(mongo_id, key))
 304                 return False
 305             else:
 306                 mandatory_fields.remove(key)
 307         elif key in mandatory_fields_to_modify:
 308             if value is None:
 309                 # empty mandatory field, invalid input
 310                 logger.info("Skipping testcase with mongo _id '{}' because the testcase was missing value"
 311                             " for mandatory field '{}'".format(mongo_id, key))
 312                 return False
 313             else:
 314                 testcase[key] = mandatory_fields_to_modify[key](value)
 315                 del mandatory_fields_to_modify[key]
 316         elif key in optional_fields:
 317             if value is None:
 318                 # empty optional field, remove
 319                 del testcase[key]
 320             optional_fields.remove(key)
 321         else:
 322             # unknown field
 323             del testcase[key]
 324
 325     if len(mandatory_fields) > 0:
 326         # some mandatory fields are missing
 327         logger.info("Skipping testcase with mongo _id '{}' because the testcase was missing"
 328                     " mandatory field(s) '{}'".format(mongo_id, mandatory_fields))
 329         return False
 330     else:
 331         return True
 332
 333
 334 def modify_mongo_entry(testcase):
 335     # 1. verify and identify the testcase
 336     # 2. if modification is implemented, then use that
 337     # 3. if not, try to use default
 338     # 4. if 2 or 3 is successful, return True, otherwise return False
 339     if verify_mongo_entry(testcase):
 340         project = testcase['project_name']
 341         case_name = testcase['case_name']
 342         if project == 'functest':
 343             if case_name == 'Rally':
 344                 return modify_functest_rally(testcase)
 345             elif case_name == 'ODL':
 346                 return modify_functest_odl(testcase)
 347             elif case_name == 'ONOS':
 348                 return modify_functest_onos(testcase)
 349             elif case_name == 'vIMS':
 350                 return modify_functest_vims(testcase)
 351             elif case_name == 'Tempest':
 352                 return modify_functest_tempest(testcase)
 353         return modify_default_entry(testcase)
 354     else:
 355         return False
 356
 357
 358 def publish_mongo_data(output_destination):
 359     tmp_filename = 'mongo-{}.log'.format(uuid.uuid4())
 360     try:
 361         subprocess.check_call(['mongoexport', '--db', 'test_results_collection', '-c', 'test_results', '--out',
 362                                tmp_filename])
 363         with open(tmp_filename) as fobj:
 364             for mongo_json_line in fobj:
 365                 test_result = json.loads(mongo_json_line)
 366                 if modify_mongo_entry(test_result):
 367                     shared_utils.publish_json(test_result, output_destination, es_user, es_passwd)
 368     finally:
 369         if os.path.exists(tmp_filename):
 370             os.remove(tmp_filename)
 371
 372
 373 def get_mongo_data(days):
 374     past_time = datetime.datetime.today() - datetime.timedelta(days=days)
 375     mongo_json_lines = subprocess.check_output(['mongoexport', '--db', 'test_results_collection', '-c', 'test_results',
 376                                                 '--query', '{{"creation_date":{{$gt:"{}"}}}}'
 377                                                .format(past_time)]).splitlines()
 378
 379     mongo_data = []
 380     for mongo_json_line in mongo_json_lines:
 381         test_result = json.loads(mongo_json_line)
 382         if modify_mongo_entry(test_result):
 383             # if the modification could be applied, append the modified result
 384             mongo_data.append(test_result)
 385     return mongo_data
 386
 387
 388 def publish_difference(mongo_data, elastic_data, output_destination, es_user, es_passwd):
 389     for elastic_entry in elastic_data:
 390         if elastic_entry in mongo_data:
 391             mongo_data.remove(elastic_entry)
 392
 393     logger.info('number of parsed test results: {}'.format(len(mongo_data)))
 394
 395     for parsed_test_result in mongo_data:
 396         shared_utils.publish_json(parsed_test_result, es_user, es_passwd, output_destination)
 397
 398
 399 if __name__ == '__main__':
 400     parser = argparse.ArgumentParser(description='Modify and filter mongo json data for elasticsearch')
 401     parser.add_argument('-od', '--output-destination',
 402                         default='elasticsearch',
 403                         choices=('elasticsearch', 'stdout'),
 404                         help='defaults to elasticsearch')
 405
 406     parser.add_argument('-ml', '--merge-latest', default=0, type=int, metavar='N',
 407                         help='get entries old at most N days from mongodb and'
 408                              ' parse those that are not already in elasticsearch.'
 409                              ' If not present, will get everything from mongodb, which is the default')
 410
 411     parser.add_argument('-e', '--elasticsearch-url', default='http://localhost:9200',
 412                         help='the url of elasticsearch, defaults to http://localhost:9200')
 413
 414     parser.add_argument('-u', '--elasticsearch-username',
 415                         help='the username for elasticsearch')
 416
 417     parser.add_argument('-p', '--elasticsearch-password',
 418                         help='the password for elasticsearch')
 419
 420     parser.add_argument('-m', '--mongodb-url', default='http://localhost:8082',
 421                         help='the url of mongodb, defaults to http://localhost:8082')
 422
 423     args = parser.parse_args()
 424     base_elastic_url = urlparse.urljoin(args.elasticsearch_url, '/test_results/mongo2elastic')
 425     output_destination = args.output_destination
 426     days = args.merge_latest
 427     es_user = args.elasticsearch_username
 428     es_passwd = args.elasticsearch_password
 429
 430     if output_destination == 'elasticsearch':
 431         output_destination = base_elastic_url
 432
 433     # parsed_test_results will be printed/sent to elasticsearch
 434     if days == 0:
 435         # TODO get everything from mongo
 436         publish_mongo_data(output_destination)
 437     elif days > 0:
 438         body = '''{{
 439     "query" : {{
 440         "range" : {{
 441             "creation_date" : {{
 442                 "gte" : "now-{}d"
 443             }}
 444         }}
 445     }}
 446 }}'''.format(days)
 447         elastic_data = shared_utils.get_elastic_data(base_elastic_url, es_user, es_passwd, body)
 448         logger.info('number of hits in elasticsearch for now-{}d: {}'.format(days, len(elastic_data)))
 449         mongo_data = get_mongo_data(days)
 450         publish_difference(mongo_data, elastic_data, output_destination, es_user, es_passwd)
 451     else:
 452         raise Exception('Update must be non-negative')