# NOTE: Still not 100% working when running the tests
offline=false
report=""
-arr_test=(vping odl tempest vims rally)
+# Get the list of runnable tests
+# Check if we are in CI mode
+if [ -n "$DEPLOY_SCENARIO" ]; then
+ testcase=`cat /home/opnfv/functest/conf/testcase-list.txt`
+ arr_test=("$testcase")
+elif
+ arr_test=(vping odl tempest vims rally)
+fi
function clean_openstack(){
python ${FUNCTEST_REPO_DIR}/testcases/VIM/OpenStack/CI/libraries/clean_openstack.py \
if not os.path.exists(RALLY_RESULT_DIR):
os.makedirs(RALLY_RESULT_DIR)
+ try:
+ functest_utils.generateTestcaseList(functest_yaml)
+ except:
+ logger.info("Not CI mode. Test cases list not generated.")
+
exit(0)
results:
test_db_url: http://213.77.62.197
+# to be maintained...
+# the execution order is important as some tests may be more destructive than others
+# and if vPing is failing is usually not needed to continue...
+test_exec_priority:
+ 1: vping
+ 2: tempest
+ 3: odl
+ #4: onos
+ #5: ovno
+ #6: doctor
+ #7: promise
+ #8: policy-test
+ #9: odl-vpn_service-tests
+ #10: opnfv-yardstick-tc026-sdnvpn
+ #11: openstack-neutron-bgpvpn-api-extension-tests
+ 12: vims
+ 13: rally
+
+
########################################################################
# This part lists the dependencies of the tests
#
-# it is used to managed the complexity of the possible combination
+# it is used to manage the complexity of the possible combinations
#
# 17 projects have been declared for Brahmaputra (D Milestone)
# 89 testcases are associated with these 17 projects
#
# By default we consider that all the tests can be run on any configuration
#
-# we defined 3 constraints
+# we defined 2 constraints
# - installer (e.g. my test can be run only with installer Compass)
# possible values: apex, compass, fuel, joid
#
-# - the controller (odl, onos, opencontrail)
+# - the scenario: it described a specif installation
+# os-<controller>-<nfvfeature>-<mode>[-<extrastuff>]
+# With parameters:
+# controller=(nosdn|odl_l3|odl_l2|onos|ocl)
+# No odl_l3 today
+# nfvfeature=(kvm|ovs|dpdk|nofeature)
+# '_' list separated.
+# mode=(ha|noha)
+# extrastuff=(none)
+# Optional field - Not used today#
#
-# - the scenario (ovs, kvm, QEMU, proc) based on Fuel proposal
-# see https://git.opnfv.org/cgit/fuel/tree/deploy/scenario/scenario.yaml
+# ref:https://gerrit.opnfv.org/gerrit/#/c/6323/7/jjb/joid/joid-deploy.sh (L72-82)
# e.g my test is only possible with OVS 2.3.2 on odl
-# not fully clear as the controller may be included with the scenario
#
-# In a first step, our main need is to trigger ad hox controller suite
-# In second step we may focus with scenario parameter
-# but no so far
-# - either controller suite
-# or
-# - suites that should be runnable on any conf, any scenario
+# in functest, we indicate the regex pattern to be checked towards the scenario
+# e.g. odl-vpn_service-tests can be run if and only if
+# - installer is fuel
+# - scenario contains the name ovs and odl
#
#######################################################################
test-dependencies:
doctor:
+ installer: 'fuel'
functest:
- vping:
vims:
+ vping:
tempest:
rally:
odl:
- controller: 'odl'
+ scenario: 'odl'
onos:
- controller: 'onos'
- onos-ovsdb:
- controller: 'onos'
+ scenario: 'onos'
promise:
+ installer: '(fuel)|(joid)'
ovno:
- controller: 'opencontrail'
+ scenario: 'ocl'
policy-test:
- controller: 'odl'
+ scenario: 'odl'
sdnvpn:
opnfv-yardstick-tc026-sdnvpn:
- controller: 'nosdn'
- scenario: 'os_ovh_ha'
+ installer: 'fuel'
+ scenario: '(ovs)*(nosdn)'
odl-vpn_service-tests:
- controller: 'odl'
- scenario: 'os_ovh_ha'
+ installer: 'fuel'
+ scenario: '(ovs)*(odl)'
openstack-neutron-bgpvpn-api-extension-tests:
- controller: 'nosdn'
- scenario: 'os_ovh_ha'
-
+ installer: 'fuel'
+ scenario: '(ovs)*(nosdn)'
import json
import shutil
import re
+import yaml
from git import Repo
return False
+def get_resolvconf_ns():
+ nameservers = []
+ rconf = open("/etc/resolv.conf", "r")
+ line = rconf.readline()
+ while line:
+ ip = re.search(r"\b(?:[0-9]{1,3}\.){3}[0-9]{1,3}\b", line)
+ if ip:
+ nameservers.append(ip.group())
+ line = rconf.readline()
+ return nameservers
+
def getTestEnv(test, functest_yaml):
# get the config of the testcase based on functest_config.yaml
# 2 options
"""
ci_env_var = {
"installer": os.environ.get('INSTALLER_TYPE'),
- "controller": os.environ.get('SDN_CONTROLLER'),
- "options": os.environ.get("OPNFV_FEATURE")}
+ "scenario": os.environ.get('DEPLOY_SCENARIO')}
return ci_env_var
-def get_resolvconf_ns():
- nameservers = []
- rconf = open("/etc/resolv.conf", "r")
- line = rconf.readline()
- while line:
- ip = re.search(r"\b(?:[0-9]{1,3}\.){3}[0-9]{1,3}\b", line)
- if ip:
- nameservers.append(ip.group())
- line = rconf.readline()
- return nameservers
-
-
def isTestRunnable(test, functest_yaml):
- # check getTestEnv(test) and CI env var
- # check installer, controller and options
- # e.g. if test needs onos => do not run odl suite
- try:
- # By default we assume that all the tests are always runnable...
- is_runnable = True
- # Retrieve CI environment
- ci_env = get_ci_envvars()
-
- # Retrieve test environement from config file
- test_env = getTestEnv(test, functest_yaml)
-
- # if test_env not empty => dependencies to be checked
- if test_env is not None and len(test_env) > 0:
- # possible criteria = ["installer", "controller", "options"]
- # consider test criteria from config file
- # compare towards CI env through CI en variable
- for criteria in test_env:
- if test_env[criteria] != ci_env[criteria]:
- # print "Test "+ test + " cannot be run on the environment"
- is_runnable = False
- except:
- print "Error isTestRunnable:", sys.exc_info()[0]
+ # By default we assume that all the tests are always runnable...
+ is_runnable = True
+ # Retrieve CI environment
+ ci_env = get_ci_envvars()
+ # Retrieve test environement from config file
+ test_env = getTestEnv(test, functest_yaml)
+
+ # if test_env not empty => dependencies to be checked
+ if test_env is not None and len(test_env) > 0:
+ # possible criteria = ["installer", "scenario"]
+ # consider test criteria from config file
+ # compare towards CI env through CI en variable
+ for criteria in test_env:
+ if re.search(test_env[criteria], ci_env[criteria]) is None:
+ # print "Test "+ test + " cannot be run on the environment"
+ is_runnable = False
return is_runnable
def generateTestcaseList(functest_yaml):
- try:
- test_list = ""
- # Retrieve CI environment
- get_ci_envvars()
-
- # get testcases
- testcase_list = functest_yaml.get("test-dependencies")
- projects = testcase_list.keys()
- for project in projects:
- testcases = testcase_list[project]
- # 1 or 2 levels for testcases project[/case]
- # if only project name without controller or scenario
- # => shall be runnable on any controller/scenario
- if testcases is None:
- test_list += project + " "
- else:
- for testcase in testcases:
- if testcase == "controller" or testcase == "scenario":
- # project (1 level)
- if isTestRunnable(project, functest_yaml):
- test_list += project + " "
- else:
- # project/testcase (2 levels)
- thetest = project + "/" + testcase
- if isTestRunnable(thetest, functest_yaml):
- test_list += testcase + " "
-
- # create a file that could be consumed by run-test.sh
- file = open("testcase-list.txt", 'w')
- file.write(test_list)
- file.close()
-
- return test_list
-
- # test for each testcase if it is runnable
- # towards the declared configuration
- # generate the test config file
- except:
- print "Error generateTestcaseList:", sys.exc_info()[0]
+ test_list = ""
+ # get testcases
+ testcase_list = functest_yaml.get("test-dependencies")
+ projects = testcase_list.keys()
+
+ for project in projects:
+ testcases = testcase_list[project]
+ # 1 or 2 levels for testcases project[/case]
+ # if only project name without controller or scenario
+ # => shall be runnable on any controller/scenario
+ if testcases is None:
+ test_list += project + " "
+ else:
+ for testcase in testcases:
+ if testcase == "installer" or testcase == "scenario":
+ # project (1 level)
+ if isTestRunnable(project, functest_yaml):
+ test_list += project + " "
+ else:
+ # project/testcase (2 levels)
+ thetest = project + "/" + testcase
+ if isTestRunnable(thetest, functest_yaml):
+ test_list += testcase + " "
+
+ # sort the list to execute the test in the right order
+ test_order_list = functest_yaml.get("test_exec_priority")
+ test_sorted_list = ""
+ for test in test_order_list:
+ if test_order_list[test] in test_list:
+ test_sorted_list += test_order_list[test] + " "
+
+ # create a file that could be consumed by run-test.sh
+ # this method is used only for CI
+ # so it can be run only in container
+ # reuse default conf directory to store the list of runnable tests
+ file = open("/home/opnfv/functest/conf/testcase-list.txt", 'w')
+ file.write(test_sorted_list)
+ file.close()
+
+ return test_sorted_list
+
def setUp(self):
os.environ["INSTALLER_TYPE"] = "fuel"
- os.environ["SDN_CONTROLLER"] = "odl"
- os.environ["OPNFV_FEATURE"] = "ovs2.4"
+ os.environ["DEPLOY_SCENARIO"] = "os-odl_l3-ovs-ha"
global functest_yaml
- with open("/home/opnfv/functest/conf/config_functest.yaml") as f:
+ with open("../config_functest.yaml") as f:
functest_yaml = yaml.safe_load(f)
f.close()
def test_getTestEnv(self):
env_test = getTestEnv('ovno', functest_yaml)
- self.assertEqual(env_test, {'controller': 'opencontrail'})
+ self.assertEqual(env_test, {'scenario': 'ocl'})
env_test = getTestEnv('doctor', functest_yaml)
- self.assertEqual(env_test, None)
+ self.assertEqual(env_test, {'installer': 'fuel'})
env_test = getTestEnv('promise', functest_yaml)
- self.assertEqual(env_test, None)
+ self.assertEqual(env_test, {'installer': '(fuel)|(joid)'})
env_test = getTestEnv('functest/tempest', functest_yaml)
self.assertEqual(env_test, None)
self.assertEqual(env_test, None)
env_test = getTestEnv('functest/odl', functest_yaml)
- self.assertEqual(env_test, {'controller': 'odl'})
+ self.assertEqual(env_test, {'scenario': 'odl'})
env_test = getTestEnv('functest/onos', functest_yaml)
- self.assertEqual(env_test, {'controller': 'onos'})
-
- env_test = getTestEnv('functest/onos-ovsdb', functest_yaml)
- self.assertEqual(env_test, {'controller': 'onos'})
+ self.assertEqual(env_test, {'scenario': 'onos'})
env_test = getTestEnv('policy-test', functest_yaml)
- self.assertEqual(env_test, {'controller': 'odl'})
+ self.assertEqual(env_test, {'scenario': 'odl'})
env_test = getTestEnv('sdnvpn/odl-vpn_service-tests', functest_yaml)
self.assertEqual(env_test,
- {'controller': 'odl', 'scenario': 'os_ovh_ha'})
+ {'installer': 'fuel', 'scenario': '(ovs)*(odl)'})
env_test = getTestEnv('sdnvpn/opnfv-yardstick-tc026-sdnvpn',
functest_yaml)
self.assertEqual(env_test,
- {'controller': 'nosdn', 'scenario': 'os_ovh_ha'})
-
- env_test = getTestEnv('sdnvpn/openstack-neutron-bgpvpn-api-extension-tests',
- functest_yaml)
- self.assertEqual(env_test,
- {'controller': 'nosdn', 'scenario': 'os_ovh_ha'})
+ {'installer': 'fuel', 'scenario': '(ovs)*(nosdn)'})
env_test = getTestEnv('foo', functest_yaml)
self.assertEqual(env_test, '')
def test_generateTestcaseList(self):
test = generateTestcaseList(functest_yaml)
- test = sorted(test.split(' '))
- expected_list = "doctor vims odl rally vping tempest promise policy-test odl-vpn_service-tests "
- expected_list_array = sorted(expected_list.split(' '))
-
- self.assertEqual(test, expected_list_array)
+
+ expected_list = "vping tempest odl doctor promise policy-test odl-vpn_service-tests vims rally "
+ self.assertEqual(test, expected_list)
def tearDown(self):
os.environ["INSTALLER_TYPE"] = ""
- os.environ["SDN_CONTROLLER"] = ""
- os.environ["OPNFV_FEATURE"] = ""
+ os.environ["DEPLOY_SCENARIO"] = ""
if __name__ == '__main__':
unittest.main()
+
Code Review / functest-xtesting.git / commitdiff