Compliance and certification test cases
----------------------------------------
-The compliance and certification test cases can be defined under the ``/dovetail/cert``
+The compliance and certification test cases can be defined under the ``/dovetail/compliance``
directory, which is defined in yaml format.
A sample file named ``compliance_set.yml`` is provided as follows:
::
- certification_compliance_set:
- name: certification_compliance_set
+ compliance_set:
+ name: compliance_set
testcases_list:
- dovetail.ipv6.tc001
The testcase listed here is dovetail.ipv6.tc001, defined within ``dovetail/testcase``.
-Note: if a new test case yaml file is created, its name should start with ``certification_``,
-in similiar fashion as the sample file ``certification_compliance_set``.
-
Running Dovetail tool
---------------------
::
- python run.py --scenario compliance_set
+ python run.py --testsuite compliance_set
+
+The value ``compliance_set`` passed to the ``testsuite`` flag can be replaced with the test cases yaml file.
+If not argument is given, the compliance_set testsuite will be run as the default.
+
+Moreover, the testcases in given testarea can be run with ``testarea`` command line argument, such as
+testarea ``ipv6`` in ``compliance_set``
+
+::
-The value ``compliance_set`` passed to the ``scenario`` flag can be replaced with the test cases yaml file.
-If not argument is given, the compliance_set scenario will be run as the default.
+ python run.py --testsuite compliance_set --testarea ipv6
Running Dovetail in a Docker container
########################################
Inside the container the following commands can be executed to trigger the testcases ::
cd /home/opnfv/dovetail/dovetail
- python run.py --scenario compliance_set
+ python run.py --testsuite compliance_set
Results Output
###############
The running log is stored in ``/home/opnfv/dovetail/results/dovetail.log``.
-The certification report is stored in ``/home/opnfv/dovetail/results/dovetail_report.txt``.
+The compliance report is stored in ``/home/opnfv/dovetail/results/dovetail_report.txt``.
What is Dovetail tool
#####################
-A toolset for providing an evaluation of a deployed cloud environment for OPNFV certification.
+A toolset for providing an evaluation of a deployed cloud environment for OPNFV compliance.
Overview
########
-certification_compliance_set:
- name: certification_compliance_set
+compliance_set:
+ name: compliance_set
testcases_list:
# Temporarily, one test case kept here as default to run
# for use of software development/debug
-certification_proposed_tests:
- name: certification_proposed_tests
+proposed_tests:
+ name: proposed_tests
testcases_list:
# TO DO: will adjust the dovetail tool to support in later patches
# run.py --name1 {**/proposed/compliance} --name2 {**/vim/ipv6,etc}
- '-d'
help: 'CI_DEBUG for showing debug log.'
non-envs:
- scenario:
+ testsuite:
flags:
- - '--scenario'
- - '-s'
+ - '--testsuite'
default: 'compliance_set'
- help: 'certification scenario.'
+ help: 'compliance testsuite.'
+ testarea:
+ flags:
+ - '--testarea'
+ default: 'full'
+ help: 'compliance testarea within testsuite'
tag:
flags:
- '--tag'
class DovetailConfig:
- CERT_PATH = './cert/'
+ COMPLIANCE_PATH = './compliance/'
TESTCASE_PATH = './testcase/'
- SCENARIO_NAMING_FMT = 'certification_%s'
+ # testsuite supported tuple, should adjust accordingly
+ testsuite_supported = ('compliance_set', 'proposed_tests')
+ # testarea supported tuple, should adjust accordingly
+ testarea_supported = ('vimops', 'nfvi', 'ipv6')
curr_path = os.path.dirname(os.path.abspath(__file__))
checker.check(testcase, db_result)
@classmethod
- def generate_json(cls, scenario_yaml):
+ def generate_json(cls, testsuite_yaml, testarea):
report_obj = {}
- report_obj['scenario'] = scenario_yaml['name']
+ report_obj['testsuite'] = testsuite_yaml['name']
report_obj['testcases_list'] = []
- for testcase_name in scenario_yaml['testcases_list']:
+ testarea_list = []
+ for value in testsuite_yaml['testcases_list']:
+ if value is not None and (testarea == 'full' or testarea in value):
+ testarea_list.append(value)
+ for testcase_name in testarea_list:
testcase = Testcase.get(testcase_name)
testcase_in_rpt = {}
testcase_in_rpt['name'] = testcase_name
return report_obj
@classmethod
- def generate(cls, scenario_yaml):
- rpt_data = cls.generate_json(scenario_yaml)
+ def generate(cls, testsuite_yaml, testarea):
+ rpt_data = cls.generate_json(testsuite_yaml, testarea)
rpt_text = ''
split_line = '+-----------------------------------------------------'
split_line += '---------------------+\n'
+==========================================================================+\n\
| report |\n'
rpt_text += split_line
- rpt_text += '|scenario: %s\n' % rpt_data['scenario']
+ rpt_text += '|testsuite: %s\n' % rpt_data['testsuite']
for testcase in rpt_data['testcases_list']:
rpt_text += '| [testcase]: %s\t\t\t\t[%s]\n' % \
(testcase['name'], testcase['result'])
from container import Container
from testcase import Testcase
-from testcase import Scenario
+from testcase import Testsuite
from report import Report
from conf.dovetail_config import DovetailConfig as dt_config
logger = dt_logger.Logger('run.py').getLogger()
-def load_scenario(scenario):
- Scenario.load()
- return Scenario.get(dt_config.SCENARIO_NAMING_FMT % scenario)
+def load_testsuite(testsuite):
+ Testsuite.load()
+ return Testsuite.get(testsuite)
def set_container_tags(option_str):
Testcase.load()
-def run_test(scenario):
- for testcase_name in scenario['testcases_list']:
+def run_test(testsuite, testarea):
+ testarea_list = []
+ for value in testsuite['testcases_list']:
+ if value is not None and (testarea == 'full' or testarea in value):
+ testarea_list.append(value)
+
+ for testcase_name in testarea_list:
logger.info('>>[testcase]: %s' % (testcase_name))
testcase = Testcase.get(testcase_name)
if testcase is None:
def main(*args, **kwargs):
- """Dovetail certification test entry!"""
- logger.info('=======================================')
- logger.info('Dovetail certification: %s!' % (kwargs['scenario']))
- logger.info('=======================================')
+ """Dovetail compliance test entry!"""
+ logger.info('================================================')
+ logger.info('Dovetail compliance: %s!' % (kwargs['testsuite']))
+ logger.info('================================================')
validate_options(kwargs)
envs_options = filter_env_options(kwargs)
dt_config.update_envs(envs_options)
dt_config.dovetail_config['functest']['envs'])
logger.info('Your new envs for yardstick: %s' %
dt_config.dovetail_config['yardstick']['envs'])
- load_testcase()
- scenario_yaml = load_scenario(kwargs['scenario'])
+
if 'tag' in kwargs and kwargs['tag'] is not None:
set_container_tags(kwargs['tag'])
- run_test(scenario_yaml)
- Report.generate(scenario_yaml)
+
+ load_testcase()
+ testsuite_yaml = load_testsuite(kwargs['testsuite'])
+ testarea = kwargs['testarea']
+ testsuite_validation = False
+ testarea_validation = False
+ if (testarea == 'full') or (testarea in dt_config.testarea_supported):
+ testarea_validation = True
+ if kwargs['testsuite'] in dt_config.testsuite_supported:
+ testsuite_validation = True
+ if testsuite_validation and testarea_validation:
+ run_test(testsuite_yaml, testarea)
+ Report.generate(testsuite_yaml, testarea)
+ else:
+ logger.error('invalid input commands, testsuite %s testarea %s' %
+ (kwargs['testsuite'], testarea))
CONTEXT_SETTINGS = dict(help_option_names=['-h', '--help'])
return None
-class Scenario:
+class Testsuite:
- def __init__(self, scenario):
- self.scenario = scenario
+ def __init__(self, testsuite):
+ self.testsuite = testsuite
self.testcase_list = {}
def get_test(self, testcase_name):
return self.testcase_list[testcase_name]
return None
- scenario_list = {}
+ testsuite_list = {}
@classmethod
def load(cls):
- for root, dirs, files in os.walk(dt_config.CERT_PATH):
- for scenario_yaml in files:
- with open(os.path.join(root, scenario_yaml)) as f:
- scenario_yaml = yaml.safe_load(f)
- cls.scenario_list.update(scenario_yaml)
+ for root, dirs, files in os.walk(dt_config.COMPLIANCE_PATH):
+ for testsuite_yaml in files:
+ with open(os.path.join(root, testsuite_yaml)) as f:
+ testsuite_yaml = yaml.safe_load(f)
+ cls.testsuite_list.update(testsuite_yaml)
- logger.debug(cls.scenario_list)
+ logger.debug(cls.testsuite_list)
@classmethod
- def get(cls, scenario_name):
- if scenario_name in cls.scenario_list:
- return cls.scenario_list[scenario_name]
+ def get(cls, testsuite_name):
+ if testsuite_name in cls.testsuite_list:
+ return cls.testsuite_list[testsuite_name]
return None
Code Review / dovetail.git / commitdiff