utils/test/reporting/functest/reporting-tempest.py

   1 from urllib2 import Request, urlopen, URLError
   2 import json
   3 import jinja2
   4 import os
   5
   6 installers = ["apex", "compass", "fuel", "joid"]
   7 items = ["tests", "Success rate", "duration"]
   8
   9 PERIOD = 7
  10 print "Generate Tempest automatic reporting"
  11 for installer in installers:
  12     # we consider the Tempest results of the last PERIOD days
  13     url = "http://testresults.opnfv.org/test/api/v1/results?case=Tempest"
  14     request = Request(url + '&period=' + str(PERIOD)
  15                       + '&installer=' + installer + '&version=master')
  16
  17     try:
  18         response = urlopen(request)
  19         k = response.read()
  20         results = json.loads(k)
  21     except URLError, e:
  22         print 'No kittez. Got an error code:', e
  23
  24     test_results = results['results']
  25     test_results.reverse()
  26
  27     scenario_results = {}
  28     criteria = {}
  29     errors = {}
  30
  31     for r in test_results:
  32         # Retrieve all the scenarios per installer
  33         # In Brahmaputra use version
  34         # Since Colorado use scenario
  35         if not r['scenario'] in scenario_results.keys():
  36             scenario_results[r['scenario']] = []
  37         scenario_results[r['scenario']].append(r)
  38
  39     for s, s_result in scenario_results.items():
  40         scenario_results[s] = s_result[0:5]
  41         # For each scenario, we build a result object to deal with
  42         # results, criteria and error handling
  43         for result in scenario_results[s]:
  44             result["start_date"] = result["start_date"].split(".")[0]
  45
  46             # retrieve results
  47             # ****************
  48             nb_tests_run = result['details']['tests']
  49             nb_tests_failed = result['details']['failures']
  50             if nb_tests_run != 0:
  51                 success_rate = 100*(int(nb_tests_run)
  52                                     - int(nb_tests_failed))/int(nb_tests_run)
  53             else:
  54                 success_rate = 0
  55
  56             result['details']["tests"] = nb_tests_run
  57             result['details']["Success rate"] = str(success_rate) + "%"
  58
  59             # Criteria management
  60             # *******************
  61             crit_tests = False
  62             crit_rate = False
  63             crit_time = False
  64
  65             # Expect that at least 165 tests are run
  66             if nb_tests_run >= 165:
  67                 crit_tests = True
  68
  69             # Expect that at least 90% of success
  70             if success_rate >= 90:
  71                 crit_rate = True
  72
  73             # Expect that the suite duration is inferior to 30m
  74             if result['details']['duration'] < 1800:
  75                 crit_time = True
  76
  77             result['criteria'] = {'tests': crit_tests,
  78                                   'Success rate': crit_rate,
  79                                   'duration': crit_time}
  80             # error management
  81             # ****************
  82             try:
  83                 errors = result['details']['errors']
  84                 result['errors'] = errors.replace('{0}', '')
  85             except:
  86                 print "Error field not present (Brahamputra runs?)"
  87
  88     mypath = os.path.abspath(__file__)
  89     tplLoader = jinja2.FileSystemLoader(os.path.dirname(mypath))
  90     templateEnv = jinja2.Environment(loader=tplLoader)
  91
  92     TEMPLATE_FILE = "./template/index-tempest-tmpl.html"
  93     template = templateEnv.get_template(TEMPLATE_FILE)
  94
  95     outputText = template.render(scenario_results=scenario_results,
  96                                  items=items,
  97                                  installer=installer)
  98
  99     with open("./release/master/index-tempest-" +
 100               installer + ".html", "wb") as fh:
 101         fh.write(outputText)
 102 print "Tempest automatic reporting Done"