All patching config files in memory
authorLinda Wang <wangwulin@huawei.com>
Tue, 5 Dec 2017 10:20:55 +0000 (10:20 +0000)
committerCédric Ollivier <cedric.ollivier@orange.com>
Tue, 12 Dec 2017 20:43:12 +0000 (21:43 +0100)
It adds the yaml content as Config attribute and allow patching it
without modifying the file (and reimport constants or config).

It also removes obsolete data in config_patch.yaml and moves several
functions in the right modules (Config)

JIRA: ARMBAND-334

Co-Authored-By: Cédric Ollivier <cedric.ollivier@orange.com>
Change-Id: I5c9a574d3283828063154c977cdfbc2abfdc6777
Signed-off-by: Linda Wang <wangwulin@huawei.com>
Signed-off-by: Cédric Ollivier <cedric.ollivier@orange.com>
functest/ci/config_patch.yaml
functest/ci/run_tests.py
functest/tests/unit/ci/test_run_tests.py
functest/utils/config.py
functest/utils/functest_utils.py

index f54c2d7..b6ba979 100644 (file)
@@ -6,9 +6,6 @@ lxd:
             image_file_name: cirros-0.3.5-x86_64-lxc.tar.gz
             image_disk_format: raw
 
-    healthcheck:
-        disk_image: /home/opnfv/functest/data/cirros-0.3.5-x86_64-lxc.tar.gz
-        disk_format: raw
 fdio:
     general:
         flavor_extra_specs: {'hw:mem_page_size':'large'}
index 67a6f14..b8da3be 100644 (file)
@@ -20,7 +20,6 @@ import textwrap
 
 import prettytable
 import six
-import yaml
 
 import functest.ci.tier_builder as tb
 import functest.core.testcase as testcase
@@ -33,13 +32,6 @@ logger = logging.getLogger('functest.ci.run_tests')
 
 CONFIG_FUNCTEST_PATH = pkg_resources.resource_filename(
     'functest', 'ci/config_functest.yaml')
-CONFIG_PATCH_PATH = pkg_resources.resource_filename(
-    'functest', 'ci/config_patch.yaml')
-CONFIG_AARCH64_PATCH_PATH = pkg_resources.resource_filename(
-    'functest', 'ci/config_aarch64_patch.yaml')
-# set the architecture to default
-pod_arch = os.getenv("POD_ARCH", None)
-arch_filter = ['aarch64']
 
 
 class Result(enum.Enum):
@@ -87,31 +79,6 @@ class Runner(object):
             CONST.__getattribute__('DEPLOY_SCENARIO'),
             pkg_resources.resource_filename('functest', 'ci/testcases.yaml'))
 
-    @staticmethod
-    def update_config_file():
-        Runner.patch_file(CONFIG_PATCH_PATH)
-
-        if pod_arch and pod_arch in arch_filter:
-            Runner.patch_file(CONFIG_AARCH64_PATCH_PATH)
-
-    @staticmethod
-    def patch_file(patch_file_path):
-        logger.debug('Updating file: %s', patch_file_path)
-        with open(patch_file_path) as f:
-            patch_file = yaml.safe_load(f)
-
-        updated = False
-        for key in patch_file:
-            if key in CONST.__getattribute__('DEPLOY_SCENARIO'):
-                new_functest_yaml = dict(ft_utils.merge_dicts(
-                    ft_utils.get_functest_yaml(), patch_file[key]))
-                updated = True
-
-        if updated:
-            os.remove(CONFIG_FUNCTEST_PATH)
-            with open(CONFIG_FUNCTEST_PATH, "w") as f:
-                f.write(yaml.dump(new_functest_yaml, default_style='"'))
-
     @staticmethod
     def source_rc_file():
         rc_file = CONST.__getattribute__('openstack_creds')
@@ -229,8 +196,6 @@ class Runner(object):
             self.run_tier(tier)
 
     def main(self, **kwargs):
-        Runner.update_config_file()
-
         if 'noclean' in kwargs:
             self.clean_flag = not kwargs['noclean']
         if 'report' in kwargs:
index b5af18c..bc95f8f 100644 (file)
@@ -58,56 +58,6 @@ class RunTestsTesting(unittest.TestCase):
 
         self.run_tests_parser = run_tests.RunTestsParser()
 
-        self.config_file_yaml = {'general': {
-                                     'openstack': {
-                                         'image_name': 'test_image_name'}},
-                                 'results': {
-                                     'test_db_url': 'url1'}}
-        self.config_file_patch_yaml = {'fdio': {'general': {
-                                       'openstack': {
-                                           'image_name':
-                                           'test_image_name_2'}}}}
-        self.config_file_aarcg64_patch_yaml = {'os': {'general': {
-                    'openstack': {'image_name': 'test_image_name_3'}}}}
-
-    @mock.patch('functest.ci.run_tests.Runner.patch_file')
-    def test_update_config_file_default(self, mock_patch):
-        self.runner.update_config_file()
-        mock_patch.assert_called()
-
-    def test_patch_file_missing_file(self):
-        patch_file_path = "unexisting_file"
-        with self.assertRaises(IOError):
-            self.runner.patch_file(patch_file_path)
-
-    @mock.patch('functest.ci.run_tests.ft_utils.merge_dicts')
-    @mock.patch('functest.ci.run_tests.ft_utils.get_functest_yaml')
-    def test_patch_file_default(self, *mock_methods):
-        CONST.__setattr__('DEPLOY_SCENARIO', 'os-nosdn-nofeature-noha')
-        with mock.patch(
-               'six.moves.builtins.open', mock.mock_open()), mock.patch(
-               'functest.ci.run_tests.yaml.safe_load') as yaml1, mock.patch(
-               'functest.ci.run_tests.ft_utils.get_functest_yaml') as yaml2:
-            yaml1.return_value = self.config_file_patch_yaml
-            yaml2.return_value = self.config_file_yaml
-            self.runner.patch_file(yaml1)
-            mock_methods[1].assert_not_called()
-            mock_methods[0].assert_not_called()
-
-    @mock.patch('functest.ci.run_tests.ft_utils.merge_dicts')
-    @mock.patch('functest.ci.run_tests.os.remove')
-    def test_patch_file_match_scenario(self, *mock_methods):
-        CONST.__setattr__('DEPLOY_SCENARIO', 'os-nosdn-fdio-noha')
-        with mock.patch(
-               'six.moves.builtins.open', mock.mock_open()), mock.patch(
-               'functest.ci.run_tests.yaml.safe_load') as yaml1, mock.patch(
-               'functest.ci.run_tests.ft_utils.get_functest_yaml') as yaml2:
-            yaml1.return_value = self.config_file_patch_yaml
-            yaml2.return_value = self.config_file_yaml
-            self.runner.patch_file(yaml2)
-            mock_methods[1].assert_called()
-            mock_methods[0].assert_called()
-
     @mock.patch('functest.ci.run_tests.logger.error')
     def test_source_rc_file_missing_file(self, mock_logger_error):
         with mock.patch('functest.ci.run_tests.os.path.isfile',
index 6bb4f58..f4749a7 100644 (file)
@@ -1,21 +1,45 @@
 #!/usr/bin/env python
 
+# pylint: disable=missing-docstring
+
+import os
+import pkg_resources
 import yaml
 
 import six
 
-from functest.utils import env
-
 
 class Config(object):
     def __init__(self):
         try:
-            with open(env.ENV.CONFIG_FUNCTEST_YAML) as f:
-                self.functest_yaml = yaml.safe_load(f)
-                self._parse(None, self.functest_yaml)
+            with open(pkg_resources.resource_filename(
+                    'functest', 'ci/config_functest.yaml')) as yfile:
+                self.functest_yaml = yaml.safe_load(yfile)
         except Exception as error:
             raise Exception('Parse config failed: {}'.format(str(error)))
 
+    @staticmethod
+    def _merge_dicts(dict1, dict2):
+        for k in set(dict1.keys()).union(dict2.keys()):
+            if k in dict1 and k in dict2:
+                if isinstance(dict1[k], dict) and isinstance(dict2[k], dict):
+                    yield (k, dict(Config._merge_dicts(dict1[k], dict2[k])))
+                else:
+                    yield (k, dict2[k])
+            elif k in dict1:
+                yield (k, dict1[k])
+            else:
+                yield (k, dict2[k])
+
+    def patch_file(self, patch_file_path):
+        with open(patch_file_path) as yfile:
+            patch_file = yaml.safe_load(yfile)
+
+        for key in patch_file:
+            if key in os.environ.get('DEPLOY_SCENARIO', ""):
+                self.functest_yaml = dict(Config._merge_dicts(
+                    self.functest_yaml, patch_file[key]))
+
     def _parse(self, attr_now, left_parametes):
         for param_n, param_v in six.iteritems(left_parametes):
             attr_further = self._get_attr_further(attr_now, param_n)
@@ -24,9 +48,22 @@ class Config(object):
             if isinstance(param_v, dict):
                 self._parse(attr_further, param_v)
 
-    def _get_attr_further(self, attr_now, next):
+    @staticmethod
+    def _get_attr_further(attr_now, next):  # pylint: disable=redefined-builtin
         return attr_now if next == 'general' else (
             '{}_{}'.format(attr_now, next) if attr_now else next)
 
+    def fill(self):
+        try:
+            self._parse(None, self.functest_yaml)
+        except Exception as error:
+            raise Exception('Parse config failed: {}'.format(str(error)))
+
 
 CONF = Config()
+CONF.patch_file(pkg_resources.resource_filename(
+    'functest', 'ci/config_patch.yaml'))
+if os.getenv("POD_ARCH", None) and os.getenv("POD_ARCH", None) in ['aarch64']:
+    CONF.patch_file(pkg_resources.resource_filename(
+        'functest', 'ci/config_aarch64_patch.yaml'))
+CONF.fill()
index bbb6b63..6c502eb 100644 (file)
@@ -317,19 +317,6 @@ def get_functest_config(parameter):
     return get_parameter_from_yaml(parameter, yaml_)
 
 
-def merge_dicts(dict1, dict2):
-    for k in set(dict1.keys()).union(dict2.keys()):
-        if k in dict1 and k in dict2:
-            if isinstance(dict1[k], dict) and isinstance(dict2[k], dict):
-                yield (k, dict(merge_dicts(dict1[k], dict2[k])))
-            else:
-                yield (k, dict2[k])
-        elif k in dict1:
-            yield (k, dict1[k])
-        else:
-            yield (k, dict2[k])
-
-
 def get_functest_yaml():
     with open(constants.CONST.__getattribute__('CONFIG_FUNCTEST_YAML')) as f:
         functest_yaml = yaml.safe_load(f)