From 42c0c75a9e3564758aaaccadec9e1bad42b283eb Mon Sep 17 00:00:00 2001 From: Morgan Richomme Date: Tue, 26 Apr 2016 14:24:56 +0200 Subject: [PATCH] Functest reporting refactoring integrate the notion of version (brahmaputra, master, ..) change dir structure Change-Id: Ieb8aed811ed4e1ab8738fb02e4db411da3d07ca2 Signed-off-by: Morgan Richomme --- utils/test/reporting/{ => functest}/default.css | 0 .../reporting/{ => functest}/img/weather-clear.png | Bin .../{ => functest}/img/weather-few-clouds.png | Bin .../{ => functest}/img/weather-overcast.png | Bin .../reporting/{ => functest}/img/weather-storm.png | Bin utils/test/reporting/{ => functest}/index.html | 0 .../reporting/{ => functest}/reporting-status.py | 147 +++++++++++---------- .../reporting/{ => functest}/reporting-tempest.py | 4 +- .../reporting/{ => functest}/reporting-vims.py | 4 +- .../{ => functest/template}/index-status-tmpl.html | 12 +- .../template}/index-tempest-tmpl.html | 0 .../{ => functest/template}/index-vims-tmpl.html | 0 12 files changed, 84 insertions(+), 83 deletions(-) rename utils/test/reporting/{ => functest}/default.css (100%) rename utils/test/reporting/{ => functest}/img/weather-clear.png (100%) rename utils/test/reporting/{ => functest}/img/weather-few-clouds.png (100%) rename utils/test/reporting/{ => functest}/img/weather-overcast.png (100%) rename utils/test/reporting/{ => functest}/img/weather-storm.png (100%) rename utils/test/reporting/{ => functest}/index.html (100%) rename utils/test/reporting/{ => functest}/reporting-status.py (69%) rename utils/test/reporting/{ => functest}/reporting-tempest.py (96%) rename utils/test/reporting/{ => functest}/reporting-vims.py (95%) rename utils/test/reporting/{ => functest/template}/index-status-tmpl.html (88%) rename utils/test/reporting/{ => functest/template}/index-tempest-tmpl.html (100%) rename utils/test/reporting/{ => functest/template}/index-vims-tmpl.html (100%) diff --git a/utils/test/reporting/default.css b/utils/test/reporting/functest/default.css similarity index 100% rename from utils/test/reporting/default.css rename to utils/test/reporting/functest/default.css diff --git a/utils/test/reporting/img/weather-clear.png b/utils/test/reporting/functest/img/weather-clear.png similarity index 100% rename from utils/test/reporting/img/weather-clear.png rename to utils/test/reporting/functest/img/weather-clear.png diff --git a/utils/test/reporting/img/weather-few-clouds.png b/utils/test/reporting/functest/img/weather-few-clouds.png similarity index 100% rename from utils/test/reporting/img/weather-few-clouds.png rename to utils/test/reporting/functest/img/weather-few-clouds.png diff --git a/utils/test/reporting/img/weather-overcast.png b/utils/test/reporting/functest/img/weather-overcast.png similarity index 100% rename from utils/test/reporting/img/weather-overcast.png rename to utils/test/reporting/functest/img/weather-overcast.png diff --git a/utils/test/reporting/img/weather-storm.png b/utils/test/reporting/functest/img/weather-storm.png similarity index 100% rename from utils/test/reporting/img/weather-storm.png rename to utils/test/reporting/functest/img/weather-storm.png diff --git a/utils/test/reporting/index.html b/utils/test/reporting/functest/index.html similarity index 100% rename from utils/test/reporting/index.html rename to utils/test/reporting/functest/index.html diff --git a/utils/test/reporting/reporting-status.py b/utils/test/reporting/functest/reporting-status.py similarity index 69% rename from utils/test/reporting/reporting-status.py rename to utils/test/reporting/functest/reporting-status.py index e15bac9f7..9e6aeb1ad 100644 --- a/utils/test/reporting/reporting-status.py +++ b/utils/test/reporting/functest/reporting-status.py @@ -4,6 +4,7 @@ import jinja2 import os import re import requests +import sys import time import yaml @@ -12,11 +13,12 @@ functest_test_list = ['vPing', 'vPing_userdata', 'Tempest', 'Rally', 'ODL', 'ONOS', 'vIMS'] # functest_test_list = ['vPing'] -# functest_test_list = [] companion_test_list = ['doctor/doctor-notification', 'promise/promise'] # companion_test_list = [] installers = ["apex", "compass", "fuel", "joid"] -# installers = ["apex"] +# installers = ["fuel"] +versions = ["brahmaputra", "master"] +# versions = ["master"] PERIOD = 10 # Correspondance between the name of the test case and the name in the DB @@ -100,7 +102,7 @@ class TestCase(object): self.isRunnable = is_runnable -def getApiResults(case, installer, scenario): +def getApiResults(case, installer, scenario, version): case = case.getName() results = json.dumps([]) # to remove proxy (to be removed at the end for local test only) @@ -111,7 +113,7 @@ def getApiResults(case, installer, scenario): # "&period=30&installer=" + installer url = "http://testresults.opnfv.org/testapi/results?case=" + case + \ "&period=" + str(PERIOD) + "&installer=" + installer + \ - "&scenario=" + scenario + "&scenario=" + scenario + "&version=" + version request = Request(url) try: @@ -124,11 +126,12 @@ def getApiResults(case, installer, scenario): return results -def getScenarios(case, installer): +def getScenarios(case, installer, version): case = case.getName() url = "http://testresults.opnfv.org/testapi/results?case=" + case + \ - "&period=" + str(PERIOD) + "&installer=" + installer + "&period=" + str(PERIOD) + "&installer=" + installer + \ + "&version=" + version request = Request(url) try: @@ -147,9 +150,9 @@ def getScenarios(case, installer): for r in test_results: # Retrieve all the scenarios per installer - if not r['version'] in scenario_results.keys(): - scenario_results[r['version']] = [] - scenario_results[r['version']].append(r) + if not r['scenario'] in scenario_results.keys(): + scenario_results[r['scenario']] = [] + scenario_results[r['scenario']].append(r) return scenario_results @@ -174,10 +177,10 @@ def getNbtestOk(results): return nb_test_ok -def getResult(testCase, installer, scenario): +def getResult(testCase, installer, scenario, version): # retrieve raw results - results = getApiResults(testCase, installer, scenario) + results = getApiResults(testCase, installer, scenario, version) # let's concentrate on test results only test_results = results['test_results'] @@ -230,16 +233,6 @@ def getResult(testCase, installer, scenario): # ****************************************************************************** # ****************************************************************************** -# as the criteria are all difference, we shall use a common way to indicate -# the criteria -# 100 = 100% = all the test must be OK -# 90 = 90% = all the test must be above 90% of success rate -# TODO harmonize success criteria -# some criteria could be the duration, the success rate, the packet loss,... -# to be done case by case -# TODo create TestCriteria Object - - # init just tempest to get the list of scenarios # as all the scenarios run Tempest tempest = TestCase("Tempest", "functest", -1) @@ -252,55 +245,63 @@ functest_yaml_config = yaml.load(response.text) print "****************************************" print "* Generating reporting..... *" print "****************************************" -# For all the installers -for installer in installers: - # get scenarios - scenario_results = getScenarios(tempest, installer) - scenario_stats = getScenarioStats(scenario_results) - - items = {} - # For all the scenarios get results - for s, s_result in scenario_results.items(): - testCases = [] - # For each scenario declare the test cases - # Functest cases - for test_case in functest_test_list: - testCases.append(TestCase(test_case, "functest")) - - # project/case - for test_case in companion_test_list: - test_split = test_case.split("/") - test_project = test_split[0] - test_case = test_split[1] - testCases.append(TestCase(test_case, test_project)) - - # Check if test case is runnable according to the installer, scenario - for test_case in testCases: - test_case.checkRunnable(installer, s, functest_yaml_config) - # print "testcase %s is %s" % (test_case.getName(), - # test_case.isRunnable) - - print "--------------------------" - print "%s / %s:" % (installer, s) - for testCase in testCases: - time.sleep(1) - if testCase.isRunnable: - print " Searching results for case %s " % testCase.getName() - result = getResult(testCase, installer, s) - testCase.setCriteria(result) - items[s] = testCases - print "--------------------------" - print "****************************************" - templateLoader = jinja2.FileSystemLoader(os.path.dirname(os.path.abspath(__file__))) - templateEnv = jinja2.Environment(loader=templateLoader) - - TEMPLATE_FILE = "index-status-tmpl.html" - template = templateEnv.get_template(TEMPLATE_FILE) - - outputText = template.render(scenario_stats=scenario_stats, - items=items, - installer=installer, - period=PERIOD) - - with open("index-status-" + installer + ".html", "wb") as fh: - fh.write(outputText) +# For all the versions +for version in versions: + # For all the installers + for installer in installers: + # get scenarios + scenario_results = getScenarios(tempest, installer, version) + scenario_stats = getScenarioStats(scenario_results) + + items = {} + # For all the scenarios get results + for s, s_result in scenario_results.items(): + testCases = [] + # For each scenario declare the test cases + # Functest cases + for test_case in functest_test_list: + testCases.append(TestCase(test_case, "functest")) + + # project/case + for test_case in companion_test_list: + test_split = test_case.split("/") + test_project = test_split[0] + test_case = test_split[1] + testCases.append(TestCase(test_case, test_project)) + + # Check if test case is runnable / installer, scenario + try: + for test_case in testCases: + test_case.checkRunnable(installer, s, functest_yaml_config) + # print "testcase %s is %s" % (test_case.getName(), + # test_case.isRunnable) + print "--------------------------" + print "installer %s, version %s, scenario %s:" % (installer, version, s) + for testCase in testCases: + time.sleep(1) + if testCase.isRunnable: + print " Searching results for case %s " % (testCase.getName()) + result = getResult(testCase, installer, s, version) + testCase.setCriteria(result) + items[s] = testCases + print "--------------------------" + except: + print "installer %s, version %s, scenario %s" % (installer, version, s) + print "No data available , error %s " % (sys.exc_info()[0]) + + print "****************************************" + templateLoader = jinja2.FileSystemLoader(os.path.dirname(os.path.abspath(__file__))) + templateEnv = jinja2.Environment(loader=templateLoader) + + TEMPLATE_FILE = "./template/index-status-tmpl.html" + template = templateEnv.get_template(TEMPLATE_FILE) + + outputText = template.render(scenario_stats=scenario_stats, + items=items, + installer=installer, + period=PERIOD, + version=version) + + with open("./release/" + version + + "/index-status-" + installer + ".html", "wb") as fh: + fh.write(outputText) diff --git a/utils/test/reporting/reporting-tempest.py b/utils/test/reporting/functest/reporting-tempest.py similarity index 96% rename from utils/test/reporting/reporting-tempest.py rename to utils/test/reporting/functest/reporting-tempest.py index 944b42809..563e53010 100644 --- a/utils/test/reporting/reporting-tempest.py +++ b/utils/test/reporting/functest/reporting-tempest.py @@ -88,12 +88,12 @@ for installer in installers: templateLoader = jinja2.FileSystemLoader(os.path.dirname(os.path.abspath(__file__))) templateEnv = jinja2.Environment(loader=templateLoader) - TEMPLATE_FILE = "index-tempest-tmpl.html" + TEMPLATE_FILE = "./template/index-tempest-tmpl.html" template = templateEnv.get_template(TEMPLATE_FILE) outputText = template.render(scenario_results=scenario_results, items=items, installer=installer) - with open("index-tempest-" + installer + ".html", "wb") as fh: + with open("./release/index-tempest-" + installer + ".html", "wb") as fh: fh.write(outputText) diff --git a/utils/test/reporting/reporting-vims.py b/utils/test/reporting/functest/reporting-vims.py similarity index 95% rename from utils/test/reporting/reporting-vims.py rename to utils/test/reporting/functest/reporting-vims.py index cf43f3ebc..78ca9f5b3 100644 --- a/utils/test/reporting/reporting-vims.py +++ b/utils/test/reporting/functest/reporting-vims.py @@ -72,12 +72,12 @@ for installer in installers: templateLoader = jinja2.FileSystemLoader(os.path.dirname(os.path.abspath(__file__))) templateEnv = jinja2.Environment( loader=templateLoader ) - TEMPLATE_FILE = "index-vims-tmpl.html" + TEMPLATE_FILE = "./template/index-vims-tmpl.html" template = templateEnv.get_template( TEMPLATE_FILE ) outputText = template.render( scenario_results = scenario_results, step_order = step_order, installer = installer) - with open("index-vims" + installer + ".html", "wb") as fh: + with open("./release/index-vims" + installer + ".html", "wb") as fh: fh.write(outputText) diff --git a/utils/test/reporting/index-status-tmpl.html b/utils/test/reporting/functest/template/index-status-tmpl.html similarity index 88% rename from utils/test/reporting/index-status-tmpl.html rename to utils/test/reporting/functest/template/index-status-tmpl.html index 0fd470d6f..604f2c8e4 100644 --- a/utils/test/reporting/index-status-tmpl.html +++ b/utils/test/reporting/functest/template/index-status-tmpl.html @@ -18,7 +18,7 @@
-

Functest status page

+

Functest status page ({{version}})

-

List of last scenarios run over the last {{period}} days

+

List of last scenarios ({{version}}) run over the last {{period}} days

@@ -75,13 +75,13 @@ {% if test.isRunnable is sameas false -%} {% elif test.getCriteria() > 2 -%} - + {%- elif test.getCriteria() > 1 -%} - + {%- elif test.getCriteria() > 0 -%} - + {%- else -%} - + {%- endif %} {%- endfor %} diff --git a/utils/test/reporting/index-tempest-tmpl.html b/utils/test/reporting/functest/template/index-tempest-tmpl.html similarity index 100% rename from utils/test/reporting/index-tempest-tmpl.html rename to utils/test/reporting/functest/template/index-tempest-tmpl.html diff --git a/utils/test/reporting/index-vims-tmpl.html b/utils/test/reporting/functest/template/index-vims-tmpl.html similarity index 100% rename from utils/test/reporting/index-vims-tmpl.html rename to utils/test/reporting/functest/template/index-vims-tmpl.html -- 2.16.6
ScenarioN.R