functest2Dashboard.py

   1 #!/usr/bin/python
   2 #
   3 # Copyright (c) 2015 Orange
   4 # morgan.richomme@orange.com
   5 #
   6 # This program and the accompanying materials
   7 # are made available under the terms of the Apache License, Version 2.0
   8 # which accompanies this distribution, and is available at
   9 #
  10 # http://www.apache.org/licenses/LICENSE-2.0
  11 #
  12 # This script is used to build dashboard ready json results
  13 # It may be used for all the test case of the Functest project
  14 # a new method format_<Test_case>_for_dashboard(results)
  15 # v0.1: basic example with methods for odl, Tempest, Rally and vPing
  16 #
  17 import datetime
  18 import re
  19
  20
  21 def get_functest_cases():
  22     """
  23     get the list of the supported test cases
  24     TODO: update the list when adding a new test case for the dashboard
  25     """
  26     return ["status", "vPing", "vPing_userdata", "vIMS", "Tempest", "ODL",
  27             "ONOS", "Rally"]
  28
  29
  30 def format_functest_for_dashboard(case, results):
  31     """
  32     generic method calling the method corresponding to the test case
  33     check that the testcase is properly declared first
  34     then build the call to the specific method
  35     """
  36     if check_functest_case_exist(case):
  37         cmd = "format_" + case + "_for_dashboard(results)"
  38         res = eval(cmd)
  39     else:
  40         res = []
  41         print "Test cases not declared"
  42     return res
  43
  44
  45 def check_functest_case_exist(case):
  46     """
  47     check if the testcase exists
  48     if the test case is not defined or not declared in the list
  49     return False
  50     """
  51     functest_cases = get_functest_cases()
  52
  53     if (case is None or case not in functest_cases):
  54         return False
  55     else:
  56         return True
  57
  58
  59 def format_status_for_dashboard(results):
  60     test_data = [{'description': 'Functest status'}]
  61
  62     # define magic equation for the status....
  63     # 5 suites: vPing, odl, Tempest, vIMS, Rally
  64     # Which overall KPI make sense...
  65
  66     # TODO to be done and discussed
  67     testcases = get_functest_cases()
  68     test_data.append({'nb test suite(s) run': len(testcases)-1})
  69     test_data.append({'vPing': '100%'})
  70     test_data.append({'VIM status': '82%'})
  71     test_data.append({'SDN Controllers': {'odl': '92%',
  72                                           'onos': '95%',
  73                                           'ocl': '93%'}})
  74     test_data.append({'VNF deployment': '95%'})
  75
  76     return test_data
  77
  78
  79 def format_vIMS_for_dashboard(results):
  80     """
  81     Post processing for the vIMS test case
  82     """
  83     test_data = [{'description': 'vIMS results for Dashboard'}]
  84
  85     # Graph 1: (duration_deployment_orchestrator,
  86     #            duration_deployment_vnf,
  87     #             duration_test) = f(time)
  88     # ********************************
  89     new_element = []
  90
  91     for data in results:
  92         new_element.append({'x': data['creation_date'],
  93                             'y1': data['details']['orchestrator']['duration'],
  94                             'y2': data['details']['vIMS']['duration'],
  95                             'y3': data['details']['sig_test']['duration']})
  96
  97     test_data.append({'name': "vIMS orchestrator/VNF/test duration",
  98                       'info': {'type': "graph",
  99                                'xlabel': 'time',
 100                                'y1label': 'orchestation deployment duration',
 101                                'y2label': 'vIMS deployment duration',
 102                                'y3label': 'vIMS test duration'},
 103                       'data_set': new_element})
 104
 105     # Graph 2: (Nb test, nb failure, nb skipped)=f(time)
 106     # **************************************************
 107     new_element = []
 108
 109     for data in results:
 110         # Retrieve all the tests
 111         nbTests = 0
 112         nbFailures = 0
 113         nbSkipped = 0
 114         vIMS_test = data['details']['sig_test']['result']
 115
 116         for data_test in vIMS_test:
 117             # Calculate nb of tests run and nb of tests failed
 118             # vIMS_results = get_vIMSresults(vIMS_test)
 119             # print vIMS_results
 120             if data_test['result'] == "Passed":
 121                 nbTests += 1
 122             elif data_test['result'] == "Failed":
 123                 nbFailures += 1
 124             elif data_test['result'] == "Skipped":
 125                 nbSkipped += 1
 126
 127         new_element.append({'x': data['creation_date'],
 128                             'y1': nbTests,
 129                             'y2': nbFailures,
 130                             'y3': nbSkipped})
 131
 132     test_data.append({'name': "vIMS nb tests passed/failed/skipped",
 133                       'info': {'type': "graph",
 134                                'xlabel': 'time',
 135                                'y1label': 'Number of tests passed',
 136                                'y2label': 'Number of tests failed',
 137                                'y3label': 'Number of tests skipped'},
 138                       'data_set': new_element})
 139
 140     # Graph 3: bar graph Summ(nb tests run), Sum (nb tests failed)
 141     # ********************************************************
 142     nbTests = 0
 143     nbFailures = 0
 144
 145     for data in results:
 146         vIMS_test = data['details']['sig_test']['result']
 147
 148         for data_test in vIMS_test:
 149             nbTestsOK = 0
 150             nbTestsKO = 0
 151
 152             if data_test['result'] == "Passed":
 153                 nbTestsOK += 1
 154             elif data_test['result'] == "Failed":
 155                 nbTestsKO += 1
 156
 157             nbTests += nbTestsOK + nbTestsKO
 158             nbFailures += nbTestsKO
 159
 160     test_data.append({'name': "Total number of tests run/failure tests",
 161                       'info': {"type": "bar"},
 162                       'data_set': [{'Run': nbTests,
 163                                     'Failed': nbFailures}]})
 164
 165     return test_data
 166
 167
 168 def format_Tempest_for_dashboard(results):
 169     """
 170     Post processing for the Tempest test case
 171     """
 172     test_data = [{'description': 'Tempest results for Dashboard'}]
 173
 174     # Graph 1: Test_Duration = f(time)
 175     # ********************************
 176     new_element = []
 177     for data in results:
 178         new_element.append({'x': data['creation_date'],
 179                             'y': data['details']['duration']})
 180
 181     test_data.append({'name': "Tempest duration",
 182                       'info': {'type': "graph",
 183                                'xlabel': 'time',
 184                                'ylabel': 'duration (s)'},
 185                       'data_set': new_element})
 186
 187     # Graph 2: (Nb test, nb failure)=f(time)
 188     # ***************************************
 189     new_element = []
 190     for data in results:
 191         new_element.append({'x': data['creation_date'],
 192                             'y1': data['details']['tests'],
 193                             'y2': data['details']['failures']})
 194
 195     test_data.append({'name': "Tempest nb tests/nb failures",
 196                       'info': {'type': "graph",
 197                                'xlabel': 'time',
 198                                'y1label': 'Number of tests',
 199                                'y2label': 'Number of failures'},
 200                       'data_set': new_element})
 201
 202     # Graph 3: bar graph Summ(nb tests run), Sum (nb tests failed)
 203     # ********************************************************
 204     nbTests = 0
 205     nbFailures = 0
 206
 207     for data in results:
 208         nbTests += data['details']['tests']
 209         nbFailures += data['details']['failures']
 210
 211     test_data.append({'name': "Total number of tests run/failure tests",
 212                       'info': {"type": "bar"},
 213                       'data_set': [{'Run': nbTests,
 214                                     'Failed': nbFailures}]})
 215
 216     # Graph 4: (Success rate)=f(time)
 217     # ***************************************
 218     new_element = []
 219     for data in results:
 220         try:
 221             diff = (int(data['details']['tests']) - int(data['details']['failures']))
 222             success_rate = 100*diff/int(data['details']['tests'])
 223         except:
 224             success_rate = 0
 225
 226         new_element.append({'x': data['creation_date'],
 227                             'y1': success_rate})
 228
 229     test_data.append({'name': "Tempest success rate",
 230                       'info': {'type': "graph",
 231                                'xlabel': 'time',
 232                                'y1label': 'Success rate'},
 233                       'data_set': new_element})
 234
 235     return test_data
 236
 237
 238 def format_ODL_for_dashboard(results):
 239     """
 240     Post processing for the ODL test case
 241     """
 242     test_data = [{'description': 'ODL results for Dashboard'}]
 243
 244     # Graph 1: (Nb test, nb failure)=f(time)
 245     # ***************************************
 246     new_element = []
 247
 248     for data in results:
 249         odl_results = data['details']['details']
 250         nbFailures = 0
 251         for odl in odl_results:
 252             if (odl['test_status']['@status'] == "FAIL"):
 253                 nbFailures += 1
 254         new_element.append({'x': data['creation_date'],
 255                             'y1': len(odl_results),
 256                             'y2': nbFailures})
 257
 258     test_data.append({'name': "ODL nb tests/nb failures",
 259                       'info': {'type': "graph",
 260                                'xlabel': 'time',
 261                                'y1label': 'Number of tests',
 262                                'y2label': 'Number of failures'},
 263                       'data_set': new_element})
 264     return test_data
 265
 266
 267 def format_ONOS_for_dashboard(results):
 268     """
 269     Post processing for the odl test case
 270     """
 271     test_data = [{'description': 'ONOS results for Dashboard'}]
 272     # Graph 1: (duration FUNCvirtNet)=f(time)
 273     # ***************************************
 274     new_element = []
 275
 276     # default duration 0:00:08.999904
 277     # consider only seconds => 09
 278     for data in results:
 279         t = data['details']['FUNCvirNet']['duration']
 280         h, m, s = re.split(':', t)
 281         s = round(float(s))
 282         new_duration = int(datetime.timedelta(hours=int(h),
 283                                               minutes=int(m),
 284                                               seconds=int(s)).total_seconds())
 285         new_element.append({'x': data['creation_date'],
 286                             'y': new_duration})
 287
 288     test_data.append({'name': "ONOS FUNCvirNet duration ",
 289                       'info': {'type': "graph",
 290                                'xlabel': 'time (s)',
 291                                'ylabel': 'duration (s)'},
 292                       'data_set': new_element})
 293
 294     # Graph 2: (Nb test, nb failure)FuncvirtNet=f(time)
 295     # ***************************************
 296     new_element = []
 297
 298     for data in results:
 299         onos_results = data['details']['FUNCvirNet']['status']
 300         nbFailures = 0
 301         for onos in onos_results:
 302             if (onos['Case result'] == "FAIL"):
 303                 nbFailures += 1
 304         new_element.append({'x': data['creation_date'],
 305                             'y1': len(onos_results),
 306                             'y2': nbFailures})
 307
 308     test_data.append({'name': "ONOS FUNCvirNet nb tests/nb failures",
 309                       'info': {'type': "graph",
 310                                'xlabel': 'time',
 311                                'y1label': 'Number of tests',
 312                                'y2label': 'Number of failures'},
 313                       'data_set': new_element})
 314
 315     # Graph 3: (duration FUNCvirtNetL3)=f(time)
 316     # ***************************************
 317     new_element = []
 318
 319     # default duration 0:00:08.999904
 320     # consider only seconds => 09
 321     for data in results:
 322         t = data['details']['FUNCvirNetL3']['duration']
 323         h, m, s = re.split(':', t)
 324         s = round(float(s))
 325         new_duration = int(datetime.timedelta(hours=int(h),
 326                                               minutes=int(m),
 327                                               seconds=int(s)).total_seconds())
 328         new_element.append({'x': data['creation_date'],
 329                             'y': new_duration})
 330
 331     test_data.append({'name': "ONOS FUNCvirNetL3 duration",
 332                       'info': {'type': "graph",
 333                                'xlabel': 'time (s)',
 334                                'ylabel': 'duration (s)'},
 335                       'data_set': new_element})
 336
 337     # Graph 4: (Nb test, nb failure)FuncvirtNetL3=f(time)
 338     # ***************************************
 339     new_element = []
 340
 341     for data in results:
 342         onos_results = data['details']['FUNCvirNetL3']['status']
 343         nbFailures = 0
 344         for onos in onos_results:
 345             if (onos['Case result'] == "FAIL"):
 346                 nbFailures += 1
 347         new_element.append({'x': data['creation_date'],
 348                             'y1': len(onos_results),
 349                             'y2': nbFailures})
 350
 351     test_data.append({'name': "ONOS FUNCvirNetL3 nb tests/nb failures",
 352                       'info': {'type': "graph",
 353                                'xlabel': 'time',
 354                                'y1label': 'Number of tests',
 355                                'y2label': 'Number of failures'},
 356                       'data_set': new_element})
 357     return test_data
 358
 359
 360 def format_Rally_for_dashboard(results):
 361     """
 362     Post processing for the Rally test case
 363     """
 364     test_data = [{'description': 'Rally results for Dashboard'}]
 365     # Graph 1: Test_Duration = f(time)
 366     # ********************************
 367     new_element = []
 368     for data in results:
 369         summary_cursor = len(data)
 370         new_element.append({'x': data['creation_date'],
 371                             'y': int(data['details'][summary_cursor]['summary']['duration'])})
 372
 373     test_data.append({'name': "rally duration",
 374                       'info': {'type': "graph",
 375                                'xlabel': 'time',
 376                                'ylabel': 'duration (s)'},
 377                       'data_set': new_element})
 378
 379     # Graph 2: Success rate = f(time)
 380     # ********************************
 381     new_element = []
 382     for data in results:
 383         new_element.append({'x': data['creation_date'],
 384                             'y': float(data['details'][summary_cursor]['summary']['nb success'])})
 385
 386     test_data.append({'name': "rally success rate",
 387                       'info': {'type': "graph",
 388                                'xlabel': 'time',
 389                                'ylabel': 'success rate (%)'},
 390                       'data_set': new_element})
 391
 392     return test_data
 393
 394
 395 def format_vPing_for_dashboard(results):
 396     """
 397     Post processing for the vPing test case
 398     """
 399     test_data = [{'description': 'vPing results for Dashboard'}]
 400
 401     # Graph 1: Test_Duration = f(time)
 402     # ********************************
 403     new_element = []
 404     for data in results:
 405         new_element.append({'x': data['creation_date'],
 406                             'y': data['details']['duration']})
 407
 408     test_data.append({'name': "vPing duration",
 409                       'info': {'type': "graph",
 410                                'xlabel': 'time',
 411                                'ylabel': 'duration (s)'},
 412                       'data_set': new_element})
 413
 414     # Graph 2: bar
 415     # ************
 416     nbTest = 0
 417     nbTestOk = 0
 418
 419     for data in results:
 420         nbTest += 1
 421         if data['details']['status'] == "OK":
 422             nbTestOk += 1
 423
 424     test_data.append({'name': "vPing status",
 425                       'info': {"type": "bar"},
 426                       'data_set': [{'Nb tests': nbTest,
 427                                     'Nb Success': nbTestOk}]})
 428
 429     return test_data
 430
 431
 432 def format_vPing_userdata_for_dashboard(results):
 433     """
 434     Post processing for the vPing_userdata test case
 435     """
 436     test_data = [{'description': 'vPing_userdata results for Dashboard'}]
 437
 438     # Graph 1: Test_Duration = f(time)
 439     # ********************************
 440     new_element = []
 441     for data in results:
 442         new_element.append({'x': data['creation_date'],
 443                             'y': data['details']['duration']})
 444
 445     test_data.append({'name': "vPing_userdata duration",
 446                       'info': {'type': "graph",
 447                                'xlabel': 'time',
 448                                'ylabel': 'duration (s)'},
 449                       'data_set': new_element})
 450
 451     # Graph 2: bar
 452     # ************
 453     nbTest = 0
 454     nbTestOk = 0
 455
 456     for data in results:
 457         nbTest += 1
 458         if data['details']['status'] == "OK":
 459             nbTestOk += 1
 460
 461     test_data.append({'name': "vPing_userdata status",
 462                       'info': {"type": "bar"},
 463                       'data_set': [{'Nb tests': nbTest,
 464                                     'Nb Success': nbTestOk}]})
 465
 466     return test_data