utils/test/result_collection_api/opnfv_testapi/dashboard/functest2Dashboard.py

   1 #!/usr/bin/python
   2 #
   3 # Copyright (c) 2015 Orange
   4 # morgan.richomme@orange.com
   5 #
   6 # This program and the accompanying materials
   7 # are made available under the terms of the Apache License, Version 2.0
   8 # which accompanies this distribution, and is available at
   9 #
  10 # http://www.apache.org/licenses/LICENSE-2.0
  11 #
  12 # This script is used to build dashboard ready json results
  13 # It may be used for all the test case of the Functest project
  14 # a new method format_<Test_case>_for_dashboard(results)
  15 # v0.1: basic example with methods for odl, Tempest, Rally and vPing
  16 #
  17 import datetime
  18 import re
  19
  20
  21 def get_functest_cases():
  22     """
  23     get the list of the supported test cases
  24     TODO: update the list when adding a new test case for the dashboard
  25     """
  26     return ["status", "vPing", "vPing_userdata", "vIMS", "Tempest", "ODL",
  27             "ONOS", "Rally"]
  28
  29
  30 def format_functest_for_dashboard(case, results):
  31     """
  32     generic method calling the method corresponding to the test case
  33     check that the testcase is properly declared first
  34     then build the call to the specific method
  35     """
  36     if check_functest_case_exist(case):
  37         cmd = "format_" + case + "_for_dashboard"
  38         res = globals()[cmd](results)
  39     else:
  40         res = []
  41         print "Test cases not declared"
  42     return res
  43
  44
  45 def check_functest_case_exist(case):
  46     """
  47     check if the testcase exists
  48     if the test case is not defined or not declared in the list
  49     return False
  50     """
  51     functest_cases = get_functest_cases()
  52
  53     if (case is None or case not in functest_cases):
  54         return False
  55     else:
  56         return True
  57
  58
  59 def format_status_for_dashboard(results):
  60     test_data = [{'description': 'Functest status'}]
  61
  62     # define magic equation for the status....
  63     # 5 suites: vPing, odl, Tempest, vIMS, Rally
  64     # Which overall KPI make sense...
  65
  66     # TODO to be done and discussed
  67     testcases = get_functest_cases()
  68     test_data.append({'nb test suite(s) run': len(testcases)-1})
  69     test_data.append({'vPing': '100%'})
  70     test_data.append({'VIM status': '82%'})
  71     test_data.append({'SDN Controllers': {'odl': '92%',
  72                                           'onos': '95%',
  73                                           'ocl': '93%'}})
  74     test_data.append({'VNF deployment': '95%'})
  75
  76     return test_data
  77
  78
  79 def format_vIMS_for_dashboard(results):
  80     """
  81     Post processing for the vIMS test case
  82     """
  83     test_data = [{'description': 'vIMS results for Dashboard'}]
  84
  85     # Graph 1: (duration_deployment_orchestrator,
  86     #            duration_deployment_vnf,
  87     #             duration_test) = f(time)
  88     # ********************************
  89     new_element = []
  90
  91     for data in results:
  92         new_element.append({'x': data['start_date'],
  93                             'y1': data['details']['orchestrator']['duration'],
  94                             'y2': data['details']['vIMS']['duration'],
  95                             'y3': data['details']['sig_test']['duration']})
  96
  97     test_data.append({'name': "vIMS orchestrator/VNF/test duration",
  98                       'info': {'type': "graph",
  99                                'xlabel': 'time',
 100                                'y1label': 'orchestation deployment duration',
 101                                'y2label': 'vIMS deployment duration',
 102                                'y3label': 'vIMS test duration'},
 103                       'data_set': new_element})
 104
 105     # Graph 2: (Nb test, nb failure, nb skipped)=f(time)
 106     # **************************************************
 107     new_element = []
 108
 109     for data in results:
 110         # Retrieve all the tests
 111         nbTests = 0
 112         nbFailures = 0
 113         nbSkipped = 0
 114         vIMS_test = data['details']['sig_test']['result']
 115
 116         for data_test in vIMS_test:
 117             # Calculate nb of tests run and nb of tests failed
 118             # vIMS_results = get_vIMSresults(vIMS_test)
 119             # print vIMS_results
 120             try:
 121                 if data_test['result'] == "Passed":
 122                     nbTests += 1
 123                 elif data_test['result'] == "Failed":
 124                     nbFailures += 1
 125                 elif data_test['result'] == "Skipped":
 126                     nbSkipped += 1
 127             except:
 128                 nbTests = 0
 129
 130         new_element.append({'x': data['start_date'],
 131                             'y1': nbTests,
 132                             'y2': nbFailures,
 133                             'y3': nbSkipped})
 134
 135     test_data.append({'name': "vIMS nb tests passed/failed/skipped",
 136                       'info': {'type': "graph",
 137                                'xlabel': 'time',
 138                                'y1label': 'Number of tests passed',
 139                                'y2label': 'Number of tests failed',
 140                                'y3label': 'Number of tests skipped'},
 141                       'data_set': new_element})
 142
 143     # Graph 3: bar graph Summ(nb tests run), Sum (nb tests failed)
 144     # ********************************************************
 145     nbTests = 0
 146     nbFailures = 0
 147
 148     for data in results:
 149         vIMS_test = data['details']['sig_test']['result']
 150
 151         for data_test in vIMS_test:
 152             nbTestsOK = 0
 153             nbTestsKO = 0
 154
 155             try:
 156                 if data_test['result'] == "Passed":
 157                     nbTestsOK += 1
 158                 elif data_test['result'] == "Failed":
 159                     nbTestsKO += 1
 160             except:
 161                 nbTestsOK = 0
 162
 163             nbTests += nbTestsOK + nbTestsKO
 164             nbFailures += nbTestsKO
 165
 166     test_data.append({'name': "Total number of tests run/failure tests",
 167                       'info': {"type": "bar"},
 168                       'data_set': [{'Run': nbTests,
 169                                     'Failed': nbFailures}]})
 170
 171     return test_data
 172
 173
 174 def format_Tempest_for_dashboard(results):
 175     """
 176     Post processing for the Tempest test case
 177     """
 178     test_data = [{'description': 'Tempest results for Dashboard'}]
 179
 180     # Graph 1: Test_Duration = f(time)
 181     # ********************************
 182     new_element = []
 183     for data in results:
 184         new_element.append({'x': data['start_date'],
 185                             'y': data['details']['duration']})
 186
 187     test_data.append({'name': "Tempest duration",
 188                       'info': {'type': "graph",
 189                                'xlabel': 'time',
 190                                'ylabel': 'duration (s)'},
 191                       'data_set': new_element})
 192
 193     # Graph 2: (Nb test, nb failure)=f(time)
 194     # ***************************************
 195     new_element = []
 196     for data in results:
 197         new_element.append({'x': data['start_date'],
 198                             'y1': data['details']['tests'],
 199                             'y2': data['details']['failures']})
 200
 201     test_data.append({'name': "Tempest nb tests/nb failures",
 202                       'info': {'type': "graph",
 203                                'xlabel': 'time',
 204                                'y1label': 'Number of tests',
 205                                'y2label': 'Number of failures'},
 206                       'data_set': new_element})
 207
 208     # Graph 3: bar graph Summ(nb tests run), Sum (nb tests failed)
 209     # ********************************************************
 210     nbTests = 0
 211     nbFailures = 0
 212
 213     for data in results:
 214         nbTests += data['details']['tests']
 215         nbFailures += data['details']['failures']
 216
 217     test_data.append({'name': "Total number of tests run/failure tests",
 218                       'info': {"type": "bar"},
 219                       'data_set': [{'Run': nbTests,
 220                                     'Failed': nbFailures}]})
 221
 222     # Graph 4: (Success rate)=f(time)
 223     # ***************************************
 224     new_element = []
 225     for data in results:
 226         try:
 227             diff = (int(data['details']['tests']) - int(data['details']['failures']))
 228             success_rate = 100*diff/int(data['details']['tests'])
 229         except:
 230             success_rate = 0
 231
 232         new_element.append({'x': data['start_date'],
 233                             'y1': success_rate})
 234
 235     test_data.append({'name': "Tempest success rate",
 236                       'info': {'type': "graph",
 237                                'xlabel': 'time',
 238                                'y1label': 'Success rate'},
 239                       'data_set': new_element})
 240
 241     return test_data
 242
 243
 244 def format_ODL_for_dashboard(results):
 245     """
 246     Post processing for the ODL test case
 247     """
 248     test_data = [{'description': 'ODL results for Dashboard'}]
 249
 250     # Graph 1: (Nb test, nb failure)=f(time)
 251     # ***************************************
 252     new_element = []
 253
 254     for data in results:
 255         odl_results = data['details']['details']
 256         nbFailures = 0
 257         for odl in odl_results:
 258             if (odl['test_status']['@status'] == "FAIL"):
 259                 nbFailures += 1
 260         new_element.append({'x': data['start_date'],
 261                             'y1': len(odl_results),
 262                             'y2': nbFailures})
 263
 264     test_data.append({'name': "ODL nb tests/nb failures",
 265                       'info': {'type': "graph",
 266                                'xlabel': 'time',
 267                                'y1label': 'Number of tests',
 268                                'y2label': 'Number of failures'},
 269                       'data_set': new_element})
 270     return test_data
 271
 272
 273 def format_ONOS_for_dashboard(results):
 274     """
 275     Post processing for the odl test case
 276     """
 277     test_data = [{'description': 'ONOS results for Dashboard'}]
 278     # Graph 1: (duration FUNCvirtNet)=f(time)
 279     # ***************************************
 280     new_element = []
 281
 282     # default duration 0:00:08.999904
 283     # consider only seconds => 09
 284     for data in results:
 285         t = data['details']['FUNCvirNet']['duration']
 286         h, m, s = re.split(':', t)
 287         s = round(float(s))
 288         new_duration = int(datetime.timedelta(hours=int(h),
 289                                               minutes=int(m),
 290                                               seconds=int(s)).total_seconds())
 291         new_element.append({'x': data['start_date'],
 292                             'y': new_duration})
 293
 294     test_data.append({'name': "ONOS FUNCvirNet duration ",
 295                       'info': {'type': "graph",
 296                                'xlabel': 'time (s)',
 297                                'ylabel': 'duration (s)'},
 298                       'data_set': new_element})
 299
 300     # Graph 2: (Nb test, nb failure)FuncvirtNet=f(time)
 301     # ***************************************
 302     new_element = []
 303
 304     for data in results:
 305         onos_results = data['details']['FUNCvirNet']['status']
 306         nbFailures = 0
 307         for onos in onos_results:
 308             if (onos['Case result'] == "FAIL"):
 309                 nbFailures += 1
 310         new_element.append({'x': data['start_date'],
 311                             'y1': len(onos_results),
 312                             'y2': nbFailures})
 313
 314     test_data.append({'name': "ONOS FUNCvirNet nb tests/nb failures",
 315                       'info': {'type': "graph",
 316                                'xlabel': 'time',
 317                                'y1label': 'Number of tests',
 318                                'y2label': 'Number of failures'},
 319                       'data_set': new_element})
 320
 321     # Graph 3: (duration FUNCvirtNetL3)=f(time)
 322     # ***************************************
 323     new_element = []
 324
 325     # default duration 0:00:08.999904
 326     # consider only seconds => 09
 327     for data in results:
 328         t = data['details']['FUNCvirNetL3']['duration']
 329         h, m, s = re.split(':', t)
 330         s = round(float(s))
 331         new_duration = int(datetime.timedelta(hours=int(h),
 332                                               minutes=int(m),
 333                                               seconds=int(s)).total_seconds())
 334         new_element.append({'x': data['start_date'],
 335                             'y': new_duration})
 336
 337     test_data.append({'name': "ONOS FUNCvirNetL3 duration",
 338                       'info': {'type': "graph",
 339                                'xlabel': 'time (s)',
 340                                'ylabel': 'duration (s)'},
 341                       'data_set': new_element})
 342
 343     # Graph 4: (Nb test, nb failure)FuncvirtNetL3=f(time)
 344     # ***************************************
 345     new_element = []
 346
 347     for data in results:
 348         onos_results = data['details']['FUNCvirNetL3']['status']
 349         nbFailures = 0
 350         for onos in onos_results:
 351             if (onos['Case result'] == "FAIL"):
 352                 nbFailures += 1
 353         new_element.append({'x': data['start_date'],
 354                             'y1': len(onos_results),
 355                             'y2': nbFailures})
 356
 357     test_data.append({'name': "ONOS FUNCvirNetL3 nb tests/nb failures",
 358                       'info': {'type': "graph",
 359                                'xlabel': 'time',
 360                                'y1label': 'Number of tests',
 361                                'y2label': 'Number of failures'},
 362                       'data_set': new_element})
 363     return test_data
 364
 365
 366 def format_Rally_for_dashboard(results):
 367     """
 368     Post processing for the Rally test case
 369     """
 370     test_data = [{'description': 'Rally results for Dashboard'}]
 371     # Graph 1: Test_Duration = f(time)
 372     # ********************************
 373     new_element = []
 374     for data in results:
 375         summary_cursor = len(data['details']) - 1
 376         new_element.append({'x': data['start_date'],
 377                             'y': int(data['details'][summary_cursor]['summary']['duration'])})
 378
 379     test_data.append({'name': "rally duration",
 380                       'info': {'type': "graph",
 381                                'xlabel': 'time',
 382                                'ylabel': 'duration (s)'},
 383                       'data_set': new_element})
 384
 385     # Graph 2: Success rate = f(time)
 386     # ********************************
 387     new_element = []
 388     for data in results:
 389         new_element.append({'x': data['start_date'],
 390                             'y': float(data['details'][summary_cursor]['summary']['nb success'])})
 391
 392     test_data.append({'name': "rally success rate",
 393                       'info': {'type': "graph",
 394                                'xlabel': 'time',
 395                                'ylabel': 'success rate (%)'},
 396                       'data_set': new_element})
 397
 398     return test_data
 399
 400
 401 def format_vPing_for_dashboard(results):
 402     """
 403     Post processing for the vPing test case
 404     """
 405     test_data = [{'description': 'vPing results for Dashboard'}]
 406
 407     # Graph 1: Test_Duration = f(time)
 408     # ********************************
 409     new_element = []
 410     for data in results:
 411         new_element.append({'x': data['start_date'],
 412                             'y': data['details']['duration']})
 413
 414     test_data.append({'name': "vPing duration",
 415                       'info': {'type': "graph",
 416                                'xlabel': 'time',
 417                                'ylabel': 'duration (s)'},
 418                       'data_set': new_element})
 419
 420     # Graph 2: bar
 421     # ************
 422     nbTest = 0
 423     nbTestOk = 0
 424
 425     for data in results:
 426         nbTest += 1
 427         if data['details']['status'] == "OK":
 428             nbTestOk += 1
 429
 430     test_data.append({'name': "vPing status",
 431                       'info': {"type": "bar"},
 432                       'data_set': [{'Nb tests': nbTest,
 433                                     'Nb Success': nbTestOk}]})
 434
 435     return test_data
 436
 437
 438 def format_vPing_userdata_for_dashboard(results):
 439     """
 440     Post processing for the vPing_userdata test case
 441     """
 442     test_data = [{'description': 'vPing_userdata results for Dashboard'}]
 443
 444     # Graph 1: Test_Duration = f(time)
 445     # ********************************
 446     new_element = []
 447     for data in results:
 448         new_element.append({'x': data['start_date'],
 449                             'y': data['details']['duration']})
 450
 451     test_data.append({'name': "vPing_userdata duration",
 452                       'info': {'type': "graph",
 453                                'xlabel': 'time',
 454                                'ylabel': 'duration (s)'},
 455                       'data_set': new_element})
 456
 457     # Graph 2: bar
 458     # ************
 459     nbTest = 0
 460     nbTestOk = 0
 461
 462     for data in results:
 463         nbTest += 1
 464         if data['details']['status'] == "OK":
 465             nbTestOk += 1
 466
 467     test_data.append({'name': "vPing_userdata status",
 468                       'info': {"type": "bar"},
 469                       'data_set': [{'Nb tests': nbTest,
 470                                     'Nb Success': nbTestOk}]})
 471
 472     return test_data