Allow regex for blacklist scenarios/installers

Reasoning: Some blacklisted test cases apply to a range of scenarios,
e.g. both 'ha' and 'noha' and possibly even to all scenarios for
a specific installer -- therefore the list of excluded scenarios can
get long and unclear. For consistency with rally test case scenarios
also allow regex for installers specification.

Additionally, log the excluded test cases (if any) to debug log.

Change-Id: Ie0bec35d1a6443203d4719ea5428c437c12e5e12
Signed-off-by: Martin Kulhavy <martin.kulhavy@nokia.com>

diff --git a/functest/opnfv_tests/openstack/rally/blacklist.txt b/functest/opnfv_tests/openstack/rally/blacklist.txt
index 3a17fa6..95bea2b 100644 (file)
--- a/functest/opnfv_tests/openstack/rally/blacklist.txt
+++ b/functest/opnfv_tests/openstack/rally/blacklist.txt
@@ -1,8 +1,7 @@
 scenario:
     -
         scenarios:
-            - os-nosdn-lxd-ha
-            - os-nosdn-lxd-noha
+            - '^os-nosdn-lxd-(no)?ha$'
         installers:
             - joid
         tests:

diff --git a/functest/opnfv_tests/openstack/rally/rally.py b/functest/opnfv_tests/openstack/rally/rally.py
index 24c9147..d5acb1b 100644 (file)
--- a/functest/opnfv_tests/openstack/rally/rally.py
+++ b/functest/opnfv_tests/openstack/rally/rally.py
@@ -188,20 +188,43 @@ class RallyBase(testcase.OSGCTestCase):
 
             installer_type = CONST.__getattribute__('INSTALLER_TYPE')
             deploy_scenario = CONST.__getattribute__('DEPLOY_SCENARIO')
-            if (bool(installer_type) * bool(deploy_scenario)):
-                if 'scenario' in black_list_yaml.keys():
-                    for item in black_list_yaml['scenario']:
-                        scenarios = item['scenarios']
-                        installers = item['installers']
-                        if (deploy_scenario in scenarios and
-                                installer_type in installers):
-                            tests = item['tests']
-                            black_tests.extend(tests)
+            if (bool(installer_type) and bool(deploy_scenario) and
+                    'scenario' in black_list_yaml.keys()):
+                for item in black_list_yaml['scenario']:
+                    scenarios = item['scenarios']
+                    installers = item['installers']
+                    in_it = RallyBase.in_iterable_re
+                    if (in_it(deploy_scenario, scenarios) and
+                            in_it(installer_type, installers)):
+                        tests = item['tests']
+                        black_tests.extend(tests)
         except Exception:
             logger.debug("Scenario exclusion not applied.")
 
         return black_tests
 
+    @staticmethod
+    def in_iterable_re(needle, haystack):
+        """
+        Check if given needle is in the iterable haystack, using regex.
+
+        :param needle: string to be matched
+        :param haystack: iterable of strings (optionally regex patterns)
+        :return: True if needle is eqial to any of the elements in haystack,
+                 or if a nonempty regex pattern in haystack is found in needle.
+        """
+
+        # match without regex
+        if needle in haystack:
+            return True
+
+        for pattern in haystack:
+            # match if regex pattern is set and found in the needle
+            if pattern and re.search(pattern, needle) is not None:
+                return True
+        else:
+            return False
+
     @staticmethod
     def excl_func():
         black_tests = []
@@ -235,6 +258,9 @@ class RallyBase(testcase.OSGCTestCase):
         black_tests = list(set(RallyBase.excl_func() +
                            RallyBase.excl_scenario()))
 
+        if black_tests:
+            logger.debug("Blacklisted tests: " + str(black_tests))
+
         include = True
         for cases_line in cases_file:
             if include:

diff --git a/functest/tests/unit/openstack/rally/test_rally.py b/functest/tests/unit/openstack/rally/test_rally.py
index 8845f66..def9c93 100644 (file)
--- a/functest/tests/unit/openstack/rally/test_rally.py
+++ b/functest/tests/unit/openstack/rally/test_rally.py
@@ -130,7 +130,10 @@ class OSRallyTesting(unittest.TestCase):
         CONST.__setattr__('DEPLOY_SCENARIO', 'test_scenario')
         dic = {'scenario': [{'scenarios': ['test_scenario'],
                              'installers': ['test_installer'],
-                             'tests': ['test']}]}
+                             'tests': ['test']},
+                            {'scenarios': ['other_scenario'],
+                             'installers': ['test_installer'],
+                             'tests': ['other_test']}]}
         with mock.patch('__builtin__.open', mock.mock_open()), \
             mock.patch('functest.opnfv_tests.openstack.rally.rally.'
                        'yaml.safe_load',
@@ -138,6 +141,34 @@ class OSRallyTesting(unittest.TestCase):
                 self.assertEqual(self.rally_base.excl_scenario(),
                                  ['test'])
 
+    def test_excl_scenario_regex(self):
+        CONST.__setattr__('INSTALLER_TYPE', 'test_installer')
+        CONST.__setattr__('DEPLOY_SCENARIO', 'os-ctrlT-featT-modeT')
+        dic = {'scenario': [{'scenarios': ['^os-[^-]+-featT-modeT$'],
+                             'installers': ['test_installer'],
+                             'tests': ['test1']},
+                            {'scenarios': ['^os-ctrlT-[^-]+-modeT$'],
+                             'installers': ['test_installer'],
+                             'tests': ['test2']},
+                            {'scenarios': ['^os-ctrlT-featT-[^-]+$'],
+                             'installers': ['test_installer'],
+                             'tests': ['test3']},
+                            {'scenarios': ['^os-'],
+                             'installers': ['test_installer'],
+                             'tests': ['test4']},
+                            {'scenarios': ['other_scenario'],
+                             'installers': ['test_installer'],
+                             'tests': ['test0a']},
+                            {'scenarios': [''],  # empty scenario
+                             'installers': ['test_installer'],
+                             'tests': ['test0b']}]}
+        with mock.patch('__builtin__.open', mock.mock_open()), \
+            mock.patch('functest.opnfv_tests.openstack.rally.rally.'
+                       'yaml.safe_load',
+                       return_value=dic):
+            self.assertEqual(self.rally_base.excl_scenario(),
+                             ['test1', 'test2', 'test3', 'test4'])
+
     def test_excl_scenario_exception(self):
         with mock.patch('__builtin__.open', side_effect=Exception):
                 self.assertEqual(self.rally_base.excl_scenario(),