utils/test/result_collection_api/dashboard/functest2Dashboard.py

   1 #!/usr/bin/python
   2 #
   3 # Copyright (c) 2015 Orange
   4 # morgan.richomme@orange.com
   5 #
   6 # This program and the accompanying materials
   7 # are made available under the terms of the Apache License, Version 2.0
   8 # which accompanies this distribution, and is available at
   9 #
  10 # http://www.apache.org/licenses/LICENSE-2.0
  11 #
  12 # This script is used to build dashboard ready json results
  13 # It may be used for all the test case of the Functest project
  14 # a new method format_<Test_case>_for_dashboard(results)
  15 # v0.1: basic example with methods for odl, Tempest, Rally and vPing
  16 #
  17 import datetime
  18 import re
  19
  20
  21 def get_functest_cases():
  22     """
  23     get the list of the supported test cases
  24     TODO: update the list when adding a new test case for the dashboard
  25     """
  26     return ["status", "vPing", "vPing_userdata", "vIMS", "Tempest", "ODL",
  27             "ONOS", "Rally"]
  28
  29
  30 def format_functest_for_dashboard(case, results):
  31     """
  32     generic method calling the method corresponding to the test case
  33     check that the testcase is properly declared first
  34     then build the call to the specific method
  35     """
  36     if check_functest_case_exist(case):
  37         cmd = "format_" + case + "_for_dashboard(results)"
  38         res = eval(cmd)
  39     else:
  40         res = []
  41         print "Test cases not declared"
  42     return res
  43
  44
  45 def check_functest_case_exist(case):
  46     """
  47     check if the testcase exists
  48     if the test case is not defined or not declared in the list
  49     return False
  50     """
  51     functest_cases = get_functest_cases()
  52
  53     if (case is None or case not in functest_cases):
  54         return False
  55     else:
  56         return True
  57
  58
  59 def format_status_for_dashboard(results):
  60     test_data = [{'description': 'Functest status'}]
  61
  62     # define magic equation for the status....
  63     # 5 suites: vPing, odl, Tempest, vIMS, Rally
  64     # Which overall KPI make sense...
  65
  66     # TODO to be done and discussed
  67     testcases = get_functest_cases()
  68     test_data.append({'nb test suite(s) run': len(testcases)-1})
  69     test_data.append({'vPing': '100%'})
  70     test_data.append({'VIM status': '82%'})
  71     test_data.append({'SDN Controllers': {'odl': '92%',
  72                                           'onos': '95%',
  73                                           'ocl': '93%'}})
  74     test_data.append({'VNF deployment': '95%'})
  75
  76     return test_data
  77
  78
  79 def format_vIMS_for_dashboard(results):
  80     """
  81     Post processing for the vIMS test case
  82     """
  83     test_data = [{'description': 'vIMS results for Dashboard'}]
  84
  85     # Graph 1: (duration_deployment_orchestrator,
  86     #            duration_deployment_vnf,
  87     #             duration_test) = f(time)
  88     # ********************************
  89     new_element = []
  90
  91     for data in results:
  92         new_element.append({'x': data['creation_date'],
  93                             'y1': data['details']['orchestrator']['duration'],
  94                             'y2': data['details']['vIMS']['duration'],
  95                             'y3': data['details']['sig_test']['duration']})
  96
  97     test_data.append({'name': "vIMS orchestrator/VNF/test duration",
  98                       'info': {'type': "graph",
  99                                'xlabel': 'time',
 100                                'y1label': 'orchestation deployment duration',
 101                                'y2label': 'vIMS deployment duration',
 102                                'y3label': 'vIMS test duration'},
 103                       'data_set': new_element})
 104
 105     # Graph 2: (Nb test, nb failure, nb skipped)=f(time)
 106     # **************************************************
 107     new_element = []
 108
 109     for data in results:
 110         # Retrieve all the tests
 111         nbTests = 0
 112         nbFailures = 0
 113         nbSkipped = 0
 114         vIMS_test = data['details']['sig_test']['result']
 115
 116         for data_test in vIMS_test:
 117             # Calculate nb of tests run and nb of tests failed
 118             # vIMS_results = get_vIMSresults(vIMS_test)
 119             # print vIMS_results
 120             if data_test['result'] == "Passed":
 121                 nbTests += 1
 122             elif data_test['result'] == "Failed":
 123                 nbFailures += 1
 124             elif data_test['result'] == "Skipped":
 125                 nbSkipped += 1
 126
 127         new_element.append({'x': data['creation_date'],
 128                             'y1': nbTests,
 129                             'y2': nbFailures,
 130                             'y3': nbSkipped})
 131
 132     test_data.append({'name': "vIMS nb tests passed/failed/skipped",
 133                       'info': {'type': "graph",
 134                                'xlabel': 'time',
 135                                'y1label': 'Number of tests passed',
 136                                'y2label': 'Number of tests failed',
 137                                'y3label': 'Number of tests skipped'},
 138                       'data_set': new_element})
 139
 140     # Graph 3: bar graph Summ(nb tests run), Sum (nb tests failed)
 141     # ********************************************************
 142     nbTests = 0
 143     nbFailures = 0
 144
 145     for data in results:
 146         vIMS_test = data['details']['sig_test']['result']
 147
 148         for data_test in vIMS_test:
 149             nbTestsOK = 0
 150             nbTestsKO = 0
 151
 152             if data_test['result'] == "Passed":
 153                 nbTestsOK += 1
 154             elif data_test['result'] == "Failed":
 155                 nbTestsKO += 1
 156
 157             nbTests += nbTestsOK + nbTestsKO
 158             nbFailures += nbTestsKO
 159
 160     test_data.append({'name': "Total number of tests run/failure tests",
 161                       'info': {"type": "bar"},
 162                       'data_set': [{'Run': nbTests,
 163                                     'Failed': nbFailures}]})
 164
 165     return test_data
 166
 167
 168 def format_Tempest_for_dashboard(results):
 169     """
 170     Post processing for the Tempest test case
 171     """
 172     test_data = [{'description': 'Tempest results for Dashboard'}]
 173
 174     # Graph 1: Test_Duration = f(time)
 175     # ********************************
 176     new_element = []
 177     for data in results:
 178         new_element.append({'x': data['creation_date'],
 179                             'y': data['details']['duration']})
 180
 181     test_data.append({'name': "Tempest duration",
 182                       'info': {'type': "graph",
 183                                'xlabel': 'time',
 184                                'ylabel': 'duration (s)'},
 185                       'data_set': new_element})
 186
 187     # Graph 2: (Nb test, nb failure)=f(time)
 188     # ***************************************
 189     new_element = []
 190     for data in results:
 191         new_element.append({'x': data['creation_date'],
 192                             'y1': data['details']['tests'],
 193                             'y2': data['details']['failures']})
 194
 195     test_data.append({'name': "Tempest nb tests/nb failures",
 196                       'info': {'type': "graph",
 197                                'xlabel': 'time',
 198                                'y1label': 'Number of tests',
 199                                'y2label': 'Number of failures'},
 200                       'data_set': new_element})
 201
 202     # Graph 3: bar graph Summ(nb tests run), Sum (nb tests failed)
 203     # ********************************************************
 204     nbTests = 0
 205     nbFailures = 0
 206
 207     for data in results:
 208         nbTests += data['details']['tests']
 209         nbFailures += data['details']['failures']
 210
 211     test_data.append({'name': "Total number of tests run/failure tests",
 212                       'info': {"type": "bar"},
 213                       'data_set': [{'Run': nbTests,
 214                                     'Failed': nbFailures}]})
 215
 216     return test_data
 217
 218
 219 def format_ODL_for_dashboard(results):
 220     """
 221     Post processing for the ODL test case
 222     """
 223     test_data = [{'description': 'ODL results for Dashboard'}]
 224
 225     # Graph 1: (Nb test, nb failure)=f(time)
 226     # ***************************************
 227     new_element = []
 228
 229     for data in results:
 230         odl_results = data['details']['details']
 231         nbFailures = 0
 232         for odl in odl_results:
 233             if (odl['test_status']['@status'] == "FAIL"):
 234                 nbFailures += 1
 235         new_element.append({'x': data['creation_date'],
 236                             'y1': len(odl_results),
 237                             'y2': nbFailures})
 238
 239     test_data.append({'name': "ODL nb tests/nb failures",
 240                       'info': {'type': "graph",
 241                                'xlabel': 'time',
 242                                'y1label': 'Number of tests',
 243                                'y2label': 'Number of failures'},
 244                       'data_set': new_element})
 245     return test_data
 246
 247
 248 def format_ONOS_for_dashboard(results):
 249     """
 250     Post processing for the odl test case
 251     """
 252     test_data = [{'description': 'ONOS results for Dashboard'}]
 253     # Graph 1: (duration FUNCvirtNet)=f(time)
 254     # ***************************************
 255     new_element = []
 256
 257     # default duration 0:00:08.999904
 258     # consider only seconds => 09
 259     for data in results:
 260         t = data['details']['FUNCvirNet']['duration']
 261         h, m, s = re.split(':', t)
 262         s = round(float(s))
 263         new_duration = int(datetime.timedelta(hours=int(h),
 264                                               minutes=int(m),
 265                                               seconds=int(s)).total_seconds())
 266         new_element.append({'x': data['creation_date'],
 267                             'y': new_duration})
 268
 269     test_data.append({'name': "ONOS FUNCvirNet duration ",
 270                       'info': {'type': "graph",
 271                                'xlabel': 'time (s)',
 272                                'ylabel': 'duration (s)'},
 273                       'data_set': new_element})
 274
 275     # Graph 2: (Nb test, nb failure)FuncvirtNet=f(time)
 276     # ***************************************
 277     new_element = []
 278
 279     for data in results:
 280         onos_results = data['details']['FUNCvirNet']['status']
 281         nbFailures = 0
 282         for onos in onos_results:
 283             if (onos['Case result'] == "FAIL"):
 284                 nbFailures += 1
 285         new_element.append({'x': data['creation_date'],
 286                             'y1': len(onos_results),
 287                             'y2': nbFailures})
 288
 289     test_data.append({'name': "ONOS FUNCvirNet nb tests/nb failures",
 290                       'info': {'type': "graph",
 291                                'xlabel': 'time',
 292                                'y1label': 'Number of tests',
 293                                'y2label': 'Number of failures'},
 294                       'data_set': new_element})
 295
 296     # Graph 3: (duration FUNCvirtNetL3)=f(time)
 297     # ***************************************
 298     new_element = []
 299
 300     # default duration 0:00:08.999904
 301     # consider only seconds => 09
 302     for data in results:
 303         t = data['details']['FUNCvirNetL3']['duration']
 304         h, m, s = re.split(':', t)
 305         s = round(float(s))
 306         new_duration = int(datetime.timedelta(hours=int(h),
 307                                               minutes=int(m),
 308                                               seconds=int(s)).total_seconds())
 309         new_element.append({'x': data['creation_date'],
 310                             'y': new_duration})
 311
 312     test_data.append({'name': "ONOS FUNCvirNetL3 duration",
 313                       'info': {'type': "graph",
 314                                'xlabel': 'time (s)',
 315                                'ylabel': 'duration (s)'},
 316                       'data_set': new_element})
 317
 318     # Graph 4: (Nb test, nb failure)FuncvirtNetL3=f(time)
 319     # ***************************************
 320     new_element = []
 321
 322     for data in results:
 323         onos_results = data['details']['FUNCvirNetL3']['status']
 324         nbFailures = 0
 325         for onos in onos_results:
 326             if (onos['Case result'] == "FAIL"):
 327                 nbFailures += 1
 328         new_element.append({'x': data['creation_date'],
 329                             'y1': len(onos_results),
 330                             'y2': nbFailures})
 331
 332     test_data.append({'name': "ONOS FUNCvirNetL3 nb tests/nb failures",
 333                       'info': {'type': "graph",
 334                                'xlabel': 'time',
 335                                'y1label': 'Number of tests',
 336                                'y2label': 'Number of failures'},
 337                       'data_set': new_element})
 338     return test_data
 339
 340
 341 def format_Rally_for_dashboard(results):
 342     """
 343     Post processing for the Rally test case
 344     """
 345     test_data = [{'description': 'Rally results for Dashboard'}]
 346     # Graph 1: Test_Duration = f(time)
 347     # ********************************
 348     new_element = []
 349     for data in results:
 350         summary_cursor = len(data)
 351         new_element.append({'x': data['creation_date'],
 352                             'y': int(data['details'][summary_cursor]['summary']['duration'])})
 353
 354     test_data.append({'name': "rally duration",
 355                       'info': {'type': "graph",
 356                                'xlabel': 'time',
 357                                'ylabel': 'duration (s)'},
 358                       'data_set': new_element})
 359
 360     # Graph 2: Success rate = f(time)
 361     # ********************************
 362     new_element = []
 363     for data in results:
 364         new_element.append({'x': data['creation_date'],
 365                             'y': float(data['details'][summary_cursor]['summary']['nb success'])})
 366
 367     test_data.append({'name': "rally success rate",
 368                       'info': {'type': "graph",
 369                                'xlabel': 'time',
 370                                'ylabel': 'success rate (%)'},
 371                       'data_set': new_element})
 372
 373     return test_data
 374
 375
 376 def format_vPing_for_dashboard(results):
 377     """
 378     Post processing for the vPing test case
 379     """
 380     test_data = [{'description': 'vPing results for Dashboard'}]
 381
 382     # Graph 1: Test_Duration = f(time)
 383     # ********************************
 384     new_element = []
 385     for data in results:
 386         new_element.append({'x': data['creation_date'],
 387                             'y': data['details']['duration']})
 388
 389     test_data.append({'name': "vPing duration",
 390                       'info': {'type': "graph",
 391                                'xlabel': 'time',
 392                                'ylabel': 'duration (s)'},
 393                       'data_set': new_element})
 394
 395     # Graph 2: bar
 396     # ************
 397     nbTest = 0
 398     nbTestOk = 0
 399
 400     for data in results:
 401         nbTest += 1
 402         if data['details']['status'] == "OK":
 403             nbTestOk += 1
 404
 405     test_data.append({'name': "vPing status",
 406                       'info': {"type": "bar"},
 407                       'data_set': [{'Nb tests': nbTest,
 408                                     'Nb Success': nbTestOk}]})
 409
 410     return test_data
 411
 412
 413 def format_vPing_userdata_for_dashboard(results):
 414     """
 415     Post processing for the vPing_userdata test case
 416     """
 417     test_data = [{'description': 'vPing_userdata results for Dashboard'}]
 418
 419     # Graph 1: Test_Duration = f(time)
 420     # ********************************
 421     new_element = []
 422     for data in results:
 423         new_element.append({'x': data['creation_date'],
 424                             'y': data['details']['duration']})
 425
 426     test_data.append({'name': "vPing_userdata duration",
 427                       'info': {'type': "graph",
 428                                'xlabel': 'time',
 429                                'ylabel': 'duration (s)'},
 430                       'data_set': new_element})
 431
 432     # Graph 2: bar
 433     # ************
 434     nbTest = 0
 435     nbTestOk = 0
 436
 437     for data in results:
 438         nbTest += 1
 439         if data['details']['status'] == "OK":
 440             nbTestOk += 1
 441
 442     test_data.append({'name': "vPing_userdata status",
 443                       'info': {"type": "bar"},
 444                       'data_set': [{'Nb tests': nbTest,
 445                                     'Nb Success': nbTestOk}]})
 446
 447     return test_data