#
import datetime
import jinja2
-import logging
-import os
import requests
import sys
import time
import scenarioResult as sr
# Logger
-logFormatter = logging.Formatter("%(asctime)s [%(threadName)-12.12s] [%(levelname)-5.5s] %(message)s")
-logger = logging.getLogger()
-
-fileHandler = logging.FileHandler("{0}/{1}".format('.', conf.LOG_FILE))
-fileHandler.setFormatter(logFormatter)
-logger.addHandler(fileHandler)
-
-consoleHandler = logging.StreamHandler()
-consoleHandler.setFormatter(logFormatter)
-logger.addHandler(consoleHandler)
-logger.setLevel(conf.LOG_LEVEL)
+logger = utils.getLogger("Status")
# Initialization
testValid = []
functest_yaml_config = yaml.load(response.text)
-logger.info("****************************************")
-logger.info("* Generating reporting..... *")
-logger.info("* Data retention = %s days *" % conf.PERIOD)
-logger.info("* *")
-logger.info("****************************************")
+logger.info("*******************************************")
+logger.info("* Generating reporting scenario status *")
+logger.info("* Data retention = %s days *" % conf.PERIOD)
+logger.info("* *")
+logger.info("*******************************************")
# Retrieve test cases of Tier 1 (smoke)
config_tiers = functest_yaml_config.get("tiers")
for test_case in testValid:
test_case.checkRunnable(installer, s,
test_case.getConstraints())
- logger.debug("testcase %s is %s" % (test_case.getName(),
- test_case.isRunnable))
+ logger.debug("testcase %s is %s" %
+ (test_case.getDisplayName(),
+ test_case.isRunnable))
time.sleep(1)
if test_case.isRunnable:
dbName = test_case.getDbName()
name = test_case.getName()
+ displayName = test_case.getDisplayName()
project = test_case.getProject()
nb_test_runnable_for_this_scenario += 1
logger.info(" Searching results for case %s " %
- (dbName))
+ (displayName))
result = utils.getResult(dbName, installer, s, version)
- logger.info(" >>>> Test score = " + str(result))
- test_case.setCriteria(result)
- test_case.setIsRunnable(True)
- testCases2BeDisplayed.append(tc.TestCase(name,
- project,
- "",
- result,
- True,
- 1))
- scenario_score = scenario_score + result
+ # at least 1 result for the test
+ if result > -1:
+ logger.info(" >>>> Test score = " + str(result))
+ test_case.setCriteria(result)
+ test_case.setIsRunnable(True)
+ testCases2BeDisplayed.append(tc.TestCase(name,
+ project,
+ "",
+ result,
+ True,
+ 1))
+ scenario_score = scenario_score + result
+ else:
+ logger.debug("No results found")
# 2) Manage the test cases for the scenario qualification
# concretely Tiers > 3
if test_case.isRunnable:
dbName = test_case.getDbName()
name = test_case.getName()
+ displayName = test_case.getDisplayName()
project = test_case.getProject()
logger.info(" Searching results for case %s " %
- (dbName))
+ (displayName))
result = utils.getResult(dbName, installer, s, version)
- test_case.setCriteria(result)
- test_case.setIsRunnable(True)
- testCases2BeDisplayed.append(tc.TestCase(name,
- project,
- "",
- result,
- True,
- 4))
+ # at least 1 result for the test
+ if result > -1:
+ test_case.setCriteria(result)
+ test_case.setIsRunnable(True)
+ testCases2BeDisplayed.append(tc.TestCase(name,
+ project,
+ "",
+ result,
+ True,
+ 4))
+ else:
+ logger.debug("No results found")
items[s] = testCases2BeDisplayed
except:
scenario_result_criteria[s] = sr.ScenarioResult(s_status, s_score)
logger.info("--------------------------")
- templateLoader = jinja2.FileSystemLoader(os.path.dirname
- (os.path.abspath
- (__file__)))
+ templateLoader = jinja2.FileSystemLoader(conf.REPORTING_PATH)
templateEnv = jinja2.Environment(loader=templateLoader)
- TEMPLATE_FILE = "./template/index-status-tmpl.html"
+ TEMPLATE_FILE = "/template/index-status-tmpl.html"
template = templateEnv.get_template(TEMPLATE_FILE)
outputText = template.render(scenario_stats=scenario_stats,
period=conf.PERIOD,
version=version)
- with open("./release/" + version +
+ with open(conf.REPORTING_PATH + "/release/" + version +
"/index-status-" + installer + ".html", "wb") as fh:
fh.write(outputText)
from urllib2 import Request, urlopen, URLError
import json
import jinja2
-import os
+import reportingConf as conf
+import reportingUtils as utils
-installers = ["apex", "compass", "fuel", "joid"]
+installers = conf.installers
items = ["tests", "Success rate", "duration"]
-PERIOD = 7
-print "Generate Tempest automatic reporting"
+PERIOD = conf.PERIOD
+criteria_nb_test = 165
+criteria_duration = 1800
+criteria_success_rate = 90
+
+logger = utils.getLogger("Tempest")
+logger.info("************************************************")
+logger.info("* Generating reporting Tempest_smoke_serial *")
+logger.info("* Data retention = %s days *" % PERIOD)
+logger.info("* *")
+logger.info("************************************************")
+
+logger.info("Success criteria: nb tests executed > %s s," +
+ "test duration < %s s," +
+ "success rate > %s " % (criteria_nb_test,
+ criteria_duration,
+ criteria_success_rate))
+
for installer in installers:
# we consider the Tempest results of the last PERIOD days
- url = "http://testresults.opnfv.org/test/api/v1/results?case=tempest_smoke_serial"
- request = Request(url + '&period=' + str(PERIOD)
- + '&installer=' + installer + '&version=master')
-
+ url = conf.URL_BASE + "?case=tempest_smoke_serial"
+ request = Request(url + '&period=' + str(PERIOD) +
+ '&installer=' + installer + '&version=master')
+ logger.info("Search tempest_smoke_serial results for installer %s"
+ % installer)
try:
response = urlopen(request)
k = response.read()
results = json.loads(k)
except URLError, e:
- print 'No kittez. Got an error code:', e
+ logger.error("Error code: %s" % e)
test_results = results['results']
test_results.reverse()
nb_tests_run = result['details']['tests']
nb_tests_failed = result['details']['failures']
if nb_tests_run != 0:
- success_rate = 100*(int(nb_tests_run)
- - int(nb_tests_failed))/int(nb_tests_run)
+ success_rate = 100*(int(nb_tests_run) -
+ int(nb_tests_failed)) / int(nb_tests_run)
else:
success_rate = 0
crit_time = False
# Expect that at least 165 tests are run
- if nb_tests_run >= 165:
+ if nb_tests_run >= criteria_nb_test:
crit_tests = True
# Expect that at least 90% of success
- if success_rate >= 90:
+ if success_rate >= criteria_success_rate:
crit_rate = True
# Expect that the suite duration is inferior to 30m
- if result['details']['duration'] < 1800:
+ if result['details']['duration'] < criteria_duration:
crit_time = True
result['criteria'] = {'tests': crit_tests,
'Success rate': crit_rate,
'duration': crit_time}
- # error management
+ try:
+ logger.debug("Scenario %s, Installer %s"
+ % (s_result[1]['scenario'], installer))
+ logger.debug("Nb Test run: %s" % nb_tests_run)
+ logger.debug("Test duration: %s"
+ % result['details']['duration'])
+ logger.debug("Success rate: %s" % success_rate)
+ except:
+ logger.error("Data format error")
+
+ # Error management
# ****************
try:
errors = result['details']['errors']
result['errors'] = errors.replace('{0}', '')
except:
- print "Error field not present (Brahamputra runs?)"
+ logger.error("Error field not present (Brahamputra runs?)")
- mypath = os.path.abspath(__file__)
- tplLoader = jinja2.FileSystemLoader(os.path.dirname(mypath))
- templateEnv = jinja2.Environment(loader=tplLoader)
+ templateLoader = jinja2.FileSystemLoader(conf.REPORTING_PATH)
+ templateEnv = jinja2.Environment(loader=templateLoader)
- TEMPLATE_FILE = "./template/index-tempest-tmpl.html"
+ TEMPLATE_FILE = "/template/index-tempest-tmpl.html"
template = templateEnv.get_template(TEMPLATE_FILE)
outputText = template.render(scenario_results=scenario_results,
items=items,
installer=installer)
- with open("./release/master/index-tempest-" +
+ with open(conf.REPORTING_PATH + "/release/master/index-tempest-" +
installer + ".html", "wb") as fh:
fh.write(outputText)
-print "Tempest automatic reporting Done"
+logger.info("Tempest automatic reporting succesfully generated.")
from urllib2 import Request, urlopen, URLError
import json
import jinja2
-import os
+import reportingConf as conf
+import reportingUtils as utils
+
+logger = utils.getLogger("vIMS")
+
def sig_test_format(sig_test):
nbPassed = 0
nbSkipped = 0
for data_test in sig_test:
if data_test['result'] == "Passed":
- nbPassed+= 1
+ nbPassed += 1
elif data_test['result'] == "Failed":
nbFailures += 1
elif data_test['result'] == "Skipped":
total_sig_test_result['skipped'] = nbSkipped
return total_sig_test_result
-installers = ["fuel", "compass", "joid", "apex"]
-step_order = ["initialisation", "orchestrator", "vIMS", "sig_test"]
+logger.info("****************************************")
+logger.info("* Generating reporting vIMS *")
+logger.info("* Data retention = %s days *" % conf.PERIOD)
+logger.info("* *")
+logger.info("****************************************")
+installers = conf.installers
+step_order = ["initialisation", "orchestrator", "vIMS", "sig_test"]
+logger.info("Start processing....")
for installer in installers:
- request = Request('http://testresults.opnfv.org/test/api/v1/results?case=vims&installer=' + installer)
+ logger.info("Search vIMS results for installer %s" % installer)
+ request = Request(conf.URL_BASE + '?case=vims&installer=' + installer)
try:
response = urlopen(request)
k = response.read()
results = json.loads(k)
except URLError, e:
- print 'No kittez. Got an error code:', e
+ logger.error("Error code: %s" % e)
test_results = results['results']
- test_results.reverse()
+
+ logger.debug("Results found: %s" % test_results)
scenario_results = {}
for r in test_results:
for s, s_result in scenario_results.items():
scenario_results[s] = s_result[0:5]
+ logger.debug("Search for success criteria")
for result in scenario_results[s]:
result["start_date"] = result["start_date"].split(".")[0]
sig_test = result['details']['sig_test']['result']
result['pr_step_ok'] = 0
if nb_step != 0:
result['pr_step_ok'] = (float(nb_step_ok)/nb_step)*100
-
-
- templateLoader = jinja2.FileSystemLoader(os.path.dirname(os.path.abspath(__file__)))
- templateEnv = jinja2.Environment( loader=templateLoader )
-
- TEMPLATE_FILE = "./template/index-vims-tmpl.html"
- template = templateEnv.get_template( TEMPLATE_FILE )
-
- outputText = template.render( scenario_results = scenario_results, step_order = step_order, installer = installer)
-
- with open("./release/master/index-vims-" + installer + ".html", "wb") as fh:
+ try:
+ logger.debug("Scenario %s, Installer %s"
+ % (s_result[1]['scenario'], installer))
+ logger.debug("Orchestrator deployment: %s s"
+ % result['details']['orchestrator']['duration'])
+ logger.debug("vIMS deployment: %s s"
+ % result['details']['vIMS']['duration'])
+ logger.debug("Signaling testing: %s s"
+ % result['details']['sig_test']['duration'])
+ logger.debug("Signaling testing results: %s"
+ % format_result)
+ except:
+ logger.error("Data badly formatted")
+ logger.debug("------------------------------------------------")
+
+ templateLoader = jinja2.FileSystemLoader(conf.REPORTING_PATH)
+ templateEnv = jinja2.Environment(loader=templateLoader)
+
+ TEMPLATE_FILE = "/template/index-vims-tmpl.html"
+ template = templateEnv.get_template(TEMPLATE_FILE)
+
+ outputText = template.render(scenario_results=scenario_results,
+ step_order=step_order,
+ installer=installer)
+
+ with open(conf.REPORTING_PATH +
+ "/release/master/index-vims-" +
+ installer + ".html", "wb") as fh:
fh.write(outputText)
-
+logger.info("vIMS report succesfully generated")
# installers = ["apex"]
# list of test cases declared in testcases.yaml but that must not be
# taken into account for the scoring
-blacklist = ["odl", "ovno", "security_scan"]
+blacklist = ["odl", "ovno", "security_scan", "copper", "moon"]
# versions = ["brahmaputra", "master"]
versions = ["master"]
PERIOD = 10
MAX_SCENARIO_CRITERIA = 18
# get the last 5 test results to determinate the success criteria
NB_TESTS = 5
+# REPORTING_PATH = "/usr/share/nginx/html/reporting/functest"
+REPORTING_PATH = "."
URL_BASE = 'http://testresults.opnfv.org/test/api/v1/results'
TEST_CONF = "https://git.opnfv.org/cgit/functest/plain/ci/testcases.yaml"
-LOG_LEVEL = "INFO"
-LOG_FILE = "reporting.log"
+LOG_LEVEL = "ERROR"
+LOG_FILE = REPORTING_PATH + "/reporting.log"
# http://www.apache.org/licenses/LICENSE-2.0
#
from urllib2 import Request, urlopen, URLError
+import logging
import json
-import reportingConf
+import reportingConf as conf
+
+
+def getLogger(module):
+ logFormatter = logging.Formatter("%(asctime)s [" +
+ module +
+ "] [%(levelname)-5.5s] %(message)s")
+ logger = logging.getLogger()
+
+ fileHandler = logging.FileHandler("{0}/{1}".format('.', conf.LOG_FILE))
+ fileHandler.setFormatter(logFormatter)
+ logger.addHandler(fileHandler)
+
+ consoleHandler = logging.StreamHandler()
+ consoleHandler.setFormatter(logFormatter)
+ logger.addHandler(consoleHandler)
+ logger.setLevel(conf.LOG_LEVEL)
+ return logger
def getApiResults(case, installer, scenario, version):
# urllib2.install_opener(opener)
# url = "http://127.0.0.1:8000/results?case=" + case + \
# "&period=30&installer=" + installer
- url = (reportingConf.URL_BASE + "?case=" + case +
- "&period=" + str(reportingConf.PERIOD) + "&installer=" + installer +
+ url = (conf.URL_BASE + "?case=" + case +
+ "&period=" + str(conf.PERIOD) + "&installer=" + installer +
"&scenario=" + scenario + "&version=" + version +
- "&last=" + str(reportingConf.NB_TESTS))
+ "&last=" + str(conf.NB_TESTS))
request = Request(url)
try:
def getScenarios(case, installer, version):
case = case.getName()
- print case
- url = (reportingConf.URL_BASE + "?case=" + case +
- "&period=" + str(reportingConf.PERIOD) + "&installer=" + installer +
+ url = (conf.URL_BASE + "?case=" + case +
+ "&period=" + str(conf.PERIOD) + "&installer=" + installer +
"&version=" + version)
request = Request(url)
# 2: <4 successful consecutive runs but passing the criteria
# 1: close to pass the success criteria
# 0: 0% success, not passing
+ # -1: no run available
test_result_indicator = 0
nbTestOk = getNbtestOk(scenario_results)
+
# print "Nb test OK (last 10 days):"+ str(nbTestOk)
# check that we have at least 4 runs
- if nbTestOk < 1:
+ if len(scenario_results) < 1:
+ # No results available
+ test_result_indicator = -1
+ elif nbTestOk < 1:
test_result_indicator = 0
elif nbTestOk < 2:
test_result_indicator = 1
{% for test in items[scenario] -%}
<th>
{% if test.getCriteria() > -1 -%}
- {{test.getDbName() }}
+ {{test.getDisplayName() }}
{%- endif %}
{% if test.getTier() > 3 -%}
*
self.criteria = criteria
self.isRunnable = isRunnable
self.tier = tier
+ display_name_matrix = {'healthcheck': 'healthcheck',
+ 'vping_ssh': 'vPing (ssh)',
+ 'vping_userdata': 'vPing (userdata)',
+ 'odl': 'ODL',
+ 'onos': 'ONOS',
+ 'ocl': 'OCL',
+ 'tempest_smoke_serial': 'Tempest (smoke)',
+ 'tempest_full_parallel': 'Tempest (full)',
+ 'rally_sanity': 'Rally (smoke)',
+ 'bgpvpn': 'bgpvpn',
+ 'rally_full': 'Rally (full)',
+ 'vims': 'vIMS',
+ 'doctor': 'Doctor',
+ 'promise': 'Promise',
+ 'moon': 'moon',
+ 'copper': 'copper',
+ 'security_scan': 'security'
+ }
+ try:
+ self.displayName = display_name_matrix[self.name]
+ except:
+ self.displayName = "unknown"
def getName(self):
return self.name
self.isRunnable = is_runnable
def toString(self):
- testcase = ("Name=" + self.name + ";Criteria=" + str(self.criteria)
- + ";Project=" + self.project + ";Constraints="
- + str(self.constraints) + ";IsRunnable"
- + str(self.isRunnable))
+ testcase = ("Name=" + self.name + ";Criteria=" +
+ str(self.criteria) + ";Project=" + self.project +
+ ";Constraints=" + str(self.constraints) +
+ ";IsRunnable" + str(self.isRunnable))
return testcase
def getDbName(self):
'rally_full': 'rally_full',
'vims': 'vims',
'doctor': 'doctor-notification',
- 'promise': 'promise'
+ 'promise': 'promise',
+ 'moon': 'moon',
+ 'copper': 'copper',
+ 'security_scan': 'security'
}
try:
return test_match_matrix[self.name]
except:
return "unknown"
- def getTestDisplayName(self):
- # Correspondance name of the test case / name in the DB
- test_match_matrix = {'healthcheck': 'healthcheck',
- 'vping_ssh': 'vPing (ssh)',
- 'vping_userdata': 'vPing (userdata)',
- 'odl': 'ODL',
- 'onos': 'ONOS',
- 'ocl': 'OCL',
- 'tempest_smoke_serial': 'Tempest (smoke)',
- 'tempest_full_parallel': 'Tempest (full)',
- 'rally_sanity': 'Rally (smoke)',
- 'bgpvpn': 'bgpvpn',
- 'rally_full': 'Rally (full)',
- 'vims': 'vIMS',
- 'doctor': 'Doctor',
- 'promise': 'Promise'
- }
- try:
- return test_match_matrix[self.name]
- except:
- return "unknown"
+ def getDisplayName(self):
+ return self.displayName
Code Review / releng.git / commitdiff