Add ONAP VNF SDK verification tests 77/65377/8
authorxudan <xudan16@huawei.com>
Sat, 24 Nov 2018 06:44:02 +0000 (01:44 -0500)
committerxudan <xudan16@huawei.com>
Fri, 4 Jan 2019 03:15:40 +0000 (22:15 -0500)
Please refer to env_config.sh.onap.sample when testing it.
The guide of preparing local env for onap package validation can be found here
https://gerrit.onap.org/r/#/c/75119

Change-Id: I39d6b11544847756126623a7eb3b953dc800c470
Signed-off-by: xudan <xudan16@huawei.com>
dovetail/report.py
dovetail/run.py
dovetail/test_runner.py
dovetail/testcase.py
dovetail/tests/unit/test_report.py
dovetail/tests/unit/test_test_runner.py
dovetail/tests/unit/test_testcase.py
dovetail/utils/dovetail_utils.py
etc/conf/onap-vtp_config.yml [new file with mode: 0644]
etc/testcase/onap-vtp.validate.csar.yml [new file with mode: 0644]
etc/userconfig/env_config.sh.onap.sample [new file with mode: 0644]

index f11bb24..ce88ed1 100644 (file)
@@ -29,7 +29,7 @@ from testcase import Testcase
 class Report(object):
 
     results = {'functest': {}, 'yardstick': {}, 'functest-k8s': {},
-               'bottlenecks': {}, 'shell': {}, 'vnftest': {}}
+               'bottlenecks': {}, 'shell': {}, 'vnftest': {}, 'onap-vtp': {}}
 
     logger = None
 
@@ -431,6 +431,63 @@ class VnftestCrawler(Crawler):
         return json_results
 
 
+class OnapVtpCrawler(Crawler):
+
+    logger = None
+
+    def __init__(self):
+        self.type = 'onap-vtp'
+        self.logger.debug('Create crawler: {}'.format(self.type))
+
+    @classmethod
+    def create_log(cls):
+        cls.logger = dt_logger.Logger(__name__ + '.OnapVtpCrawler').getLogger()
+
+    def crawl(self, testcase, file_path):
+        return self.crawl_from_file(testcase, file_path)
+
+    # The pass result looks like
+    # {
+    #   "results": [
+    #     {"property": "results", "value": "{value=SUCCESS}"},
+    #     {"property": "build_tag", "value": "test"},
+    #     {"property": "criteria", "value": "PASS"}
+    #   ]
+    # }
+    # The fail result looks like
+    # {
+    #   "results": [
+    #     {"property": "results", "value": "{value=file doesn't exists}"},
+    #     {"property": "build_tag", "value": "test"},
+    #     {"property": "criteria", "value": "FAILED"}
+    #   ]
+    # }
+    def crawl_from_file(self, testcase, file_path):
+        if not os.path.exists(file_path):
+            self.logger.error('Result file not found: {}'.format(file_path))
+            return None
+        criteria = 'FAIL'
+        with open(file_path, 'r') as f:
+            for jsonfile in f:
+                try:
+                    data = json.loads(jsonfile)
+                    for item in data['results']:
+                        if 'criteria' == item['property']:
+                            if 'PASS' == item['value']:
+                                criteria = 'PASS'
+                            break
+                    else:
+                        self.logger.error('There is no property criteria.')
+                except KeyError as e:
+                    self.logger.exception('Pass flag not found {}'.format(e))
+                except ValueError:
+                    continue
+        json_results = {'criteria': criteria}
+
+        testcase.set_results(json_results)
+        return json_results
+
+
 class CrawlerFactory(object):
 
     CRAWLER_MAP = {'functest': FunctestCrawler,
@@ -438,7 +495,8 @@ class CrawlerFactory(object):
                    'bottlenecks': BottlenecksCrawler,
                    'vnftest': VnftestCrawler,
                    'shell': ShellCrawler,
-                   'functest-k8s': FunctestK8sCrawler}
+                   'functest-k8s': FunctestK8sCrawler,
+                   'onap-vtp': OnapVtpCrawler}
 
     @classmethod
     def create(cls, type):
@@ -592,6 +650,23 @@ class VnftestChecker(object):
         return
 
 
+class OnapVtpChecker(object):
+
+    logger = None
+
+    @classmethod
+    def create_log(cls):
+        cls.logger = dt_logger.Logger(__name__ + '.OnapVtpChecker').getLogger()
+
+    @staticmethod
+    def check(testcase, result):
+        if not result:
+            testcase.passed('FAIL')
+        else:
+            testcase.passed(result['criteria'])
+        return
+
+
 class CheckerFactory(object):
 
     CHECKER_MAP = {'functest': FunctestChecker,
@@ -599,7 +674,8 @@ class CheckerFactory(object):
                    'bottlenecks': BottlenecksChecker,
                    'shell': ShellChecker,
                    'vnftest': VnftestChecker,
-                   'functest-k8s': FunctestK8sChecker}
+                   'functest-k8s': FunctestK8sChecker,
+                   'onap-vtp': OnapVtpChecker}
 
     @classmethod
     def create(cls, type):
index 1579ff6..7bb707f 100755 (executable)
@@ -111,11 +111,13 @@ def create_logs():
     dt_report.YardstickCrawler.create_log()
     dt_report.VnftestCrawler.create_log()
     dt_report.BottlenecksCrawler.create_log()
+    dt_report.OnapVtpCrawler.create_log()
     dt_report.FunctestChecker.create_log()
     dt_report.FunctestK8sChecker.create_log()
     dt_report.YardstickChecker.create_log()
     dt_report.VnftestChecker.create_log()
     dt_report.BottlenecksChecker.create_log()
+    dt_report.OnapVtpChecker.create_log()
     dt_testcase.Testcase.create_log()
     dt_testcase.Testsuite.create_log()
     dt_test_runner.DockerRunner.create_log()
index 2fa1eee..228432a 100644 (file)
@@ -130,6 +130,8 @@ class DockerRunner(Runner):
         config_item['debug'] = os.getenv('DEBUG')
         config_item['build_tag'] = dt_cfg.dovetail_config['build_tag']
         config_item['cacert'] = os.getenv('OS_CACERT')
+        config_item['host_url'] = os.getenv('HOST_URL')
+        config_item['csar_file'] = os.getenv('CSAR_FILE')
         return config_item
 
     def _update_config(self, testcase, update_pod=True):
@@ -274,6 +276,22 @@ class VnftestRunner(DockerRunner):
         super(VnftestRunner, self).__init__(testcase)
 
 
+class OnapVtpRunner(DockerRunner):
+
+    config_file_name = 'onap-vtp_config.yml'
+
+    def __init__(self, testcase):
+        self.type = 'onap-vtp'
+        super(OnapVtpRunner, self).__init__(testcase)
+        env_file = os.path.join(dt_cfg.dovetail_config['config_dir'],
+                                dt_cfg.dovetail_config['env_file'])
+        if not os.path.isfile(env_file):
+            self.logger.error('File {} does not exist.'.format(env_file))
+            return
+        dt_utils.source_env(env_file)
+        self._update_config(testcase, update_pod=False)
+
+
 class TestRunnerFactory(object):
 
     TEST_RUNNER_MAP = {
@@ -282,7 +300,8 @@ class TestRunnerFactory(object):
         "bottlenecks": BottlenecksRunner,
         "shell": ShellRunner,
         "vnftest": VnftestRunner,
-        "functest-k8s": FunctestK8sRunner
+        "functest-k8s": FunctestK8sRunner,
+        "onap-vtp": OnapVtpRunner
     }
 
     @classmethod
index 74fbbea..04d95ca 100644 (file)
@@ -349,6 +349,15 @@ class VnftestTestcase(Testcase):
         self.type = 'vnftest'
 
 
+class OnapVtpTestcase(Testcase):
+
+    validate_testcase_list = {}
+
+    def __init__(self, testcase_yaml):
+        super(OnapVtpTestcase, self).__init__(testcase_yaml)
+        self.type = 'onap-vtp'
+
+
 class TestcaseFactory(object):
     TESTCASE_TYPE_MAP = {
         'functest': FunctestTestcase,
@@ -356,7 +365,8 @@ class TestcaseFactory(object):
         'bottlenecks': BottlenecksTestcase,
         'shell': ShellTestcase,
         'vnftest': VnftestTestcase,
-        'functest-k8s': FunctestK8sTestcase
+        'functest-k8s': FunctestK8sTestcase,
+        'onap-vtp': OnapVtpTestcase
     }
 
     @classmethod
index 0f0d3ee..9f5369e 100644 (file)
@@ -34,15 +34,17 @@ class ReportTesting(unittest.TestCase):
         dt_report.YardstickCrawler.logger = None
         dt_report.BottlenecksCrawler.logger = None
         dt_report.VnftestCrawler.logger = None
+        dt_report.OnapVtpCrawler.logger = None
         dt_report.FunctestChecker.logger = None
         dt_report.FunctestK8sChecker.logger = None
         dt_report.YardstickChecker.logger = None
         dt_report.BottlenecksChecker.logger = None
         dt_report.VnftestChecker.logger = None
+        dt_report.OnapVtpChecker.logger = None
         dt_report.Report.logger = None
         dt_report.Report.results = {
             'functest': {}, 'yardstick': {}, 'functest-k8s': {},
-            'bottlenecks': {}, 'shell': {}, 'vnftest': {}}
+            'bottlenecks': {}, 'shell': {}, 'vnftest': {}, 'onap-vtp': {}}
 
     def _produce_report_initial_text(self, report_data):
         report_txt = ''
@@ -947,6 +949,179 @@ class ReportTesting(unittest.TestCase):
             "Pass flag not found 'result'")
         self.assertEquals(expected, result)
 
+    @patch('dovetail.report.dt_logger')
+    def test_onapvtp_crawler_create_log(self, mock_logger):
+        getlogger_obj = Mock()
+        logger_obj = Mock()
+        logger_obj.getLogger.return_value = getlogger_obj
+        mock_logger.Logger.return_value = logger_obj
+
+        dt_report.OnapVtpCrawler.create_log()
+
+        self.assertEquals(getlogger_obj, dt_report.OnapVtpCrawler.logger)
+
+    @patch('dovetail.report.os.path')
+    def test_onapvtp_crawler_crawl_not_exists(self, mock_path):
+        logger_obj = Mock()
+        dt_report.OnapVtpCrawler.logger = logger_obj
+        mock_path.exists.return_value = False
+        file_path = 'file_path'
+
+        crawler = dt_report.OnapVtpCrawler()
+        result = crawler.crawl(None, file_path)
+
+        mock_path.exists.assert_called_once_with(file_path)
+        logger_obj.error.assert_called_once_with(
+            'Result file not found: {}'.format(file_path))
+        self.assertEquals(None, result)
+
+    @patch('__builtin__.open')
+    @patch('dovetail.report.json.loads')
+    @patch('dovetail.report.os.path')
+    def test_onapvtp_crawler_crawl_pass(self, mock_path, mock_loads,
+                                        mock_open):
+        dt_report.OnapVtpCrawler.logger = Mock()
+        mock_path.exists.return_value = True
+        file_path = 'file_path'
+        testcase_obj = Mock()
+        file_obj = Mock()
+        mock_open.return_value.__enter__.return_value = [file_obj]
+        data_dict = {
+            'results': [
+                {"property": "results", "value": "{value=SUCCESS}"},
+                {"property": "build_tag", "value": "test-name"},
+                {"property": "criteria", "value": "PASS"}
+            ]
+        }
+        mock_loads.return_value = data_dict
+
+        crawler = dt_report.OnapVtpCrawler()
+        result = crawler.crawl(testcase_obj, file_path)
+        expected = {'criteria': 'PASS'}
+
+        mock_path.exists.assert_called_once_with(file_path)
+        mock_open.assert_called_once_with(file_path, 'r')
+        mock_loads.assert_called_once_with(file_obj)
+        testcase_obj.set_results.assert_called_once_with(expected)
+        self.assertEquals(expected, result)
+
+    @patch('__builtin__.open')
+    @patch('dovetail.report.json.loads')
+    @patch('dovetail.report.os.path')
+    def test_onapvtp_crawler_crawl_fail(self, mock_path, mock_loads,
+                                        mock_open):
+        dt_report.OnapVtpCrawler.logger = Mock()
+        mock_path.exists.return_value = True
+        file_path = 'file_path'
+        testcase_obj = Mock()
+        file_obj = Mock()
+        mock_open.return_value.__enter__.return_value = [file_obj]
+        data_dict = {
+            'results': [
+                {"property": "results", "value": "{value=file doesn't exist}"},
+                {"property": "build_tag", "value": "test-name"},
+                {"property": "criteria", "value": "FAILED"}
+            ]
+        }
+        mock_loads.return_value = data_dict
+
+        crawler = dt_report.OnapVtpCrawler()
+        result = crawler.crawl(testcase_obj, file_path)
+        expected = {'criteria': 'FAIL'}
+
+        mock_path.exists.assert_called_once_with(file_path)
+        mock_open.assert_called_once_with(file_path, 'r')
+        mock_loads.assert_called_once_with(file_obj)
+        testcase_obj.set_results.assert_called_once_with(expected)
+        self.assertEquals(expected, result)
+
+    @patch('__builtin__.open')
+    @patch('dovetail.report.json.loads')
+    @patch('dovetail.report.os.path')
+    def test_onapvtp_crawler_crawl_no_criteria(self, mock_path, mock_loads,
+                                               mock_open):
+        dt_report.OnapVtpCrawler.logger = Mock()
+        mock_path.exists.return_value = True
+        file_path = 'file_path'
+        testcase_obj = Mock()
+        file_obj = Mock()
+        mock_open.return_value.__enter__.return_value = [file_obj]
+        data_dict = {
+            'results': [
+                {"property": "results", "value": "{value=file doesn't exist}"},
+                {"property": "build_tag", "value": "test-name"},
+                {"property": "error_criteria", "value": "FAILED"}
+            ]
+        }
+        mock_loads.return_value = data_dict
+
+        crawler = dt_report.OnapVtpCrawler()
+        result = crawler.crawl(testcase_obj, file_path)
+        expected = {'criteria': 'FAIL'}
+
+        mock_path.exists.assert_called_once_with(file_path)
+        mock_open.assert_called_once_with(file_path, 'r')
+        mock_loads.assert_called_once_with(file_obj)
+        dt_report.OnapVtpCrawler.logger.error.assert_called_once_with(
+            'There is no property criteria.')
+        testcase_obj.set_results.assert_called_once_with(expected)
+        self.assertEquals(expected, result)
+
+    @patch('__builtin__.open')
+    @patch('dovetail.report.json.loads')
+    @patch('dovetail.report.os.path')
+    def test_onapvtp_crawler_crawl_exception(self, mock_path, mock_loads,
+                                             mock_open):
+        dt_report.OnapVtpCrawler.logger = Mock()
+        mock_path.exists.return_value = True
+        file_path = 'file_path'
+        testcase_obj = Mock()
+        file_obj = Mock()
+        mock_open.return_value.__enter__.return_value = [file_obj]
+        data_dict = {
+            'error_results': [
+                {"property": "results", "value": "{value=file doesn't exist}"},
+                {"property": "build_tag", "value": "test-name"},
+                {"property": "error_criteria", "value": "FAILED"}
+            ]
+        }
+        mock_loads.return_value = data_dict
+
+        crawler = dt_report.OnapVtpCrawler()
+        result = crawler.crawl(testcase_obj, file_path)
+        expected = {'criteria': 'FAIL'}
+
+        mock_path.exists.assert_called_once_with(file_path)
+        mock_open.assert_called_once_with(file_path, 'r')
+        mock_loads.assert_called_once_with(file_obj)
+        dt_report.OnapVtpCrawler.logger.exception.assert_called_once_with(
+            "Pass flag not found 'results'")
+        testcase_obj.set_results.assert_called_once_with(expected)
+        self.assertEquals(expected, result)
+
+    @patch('__builtin__.open')
+    @patch('dovetail.report.json.loads')
+    @patch('dovetail.report.os.path')
+    def test_onapvtp_crawler_crawl_value_error(self, mock_path, mock_loads,
+                                               mock_open):
+        dt_report.OnapVtpCrawler.logger = Mock()
+        mock_path.exists.return_value = True
+        file_path = 'file_path'
+        testcase_obj = Mock()
+        file_obj = Mock()
+        mock_open.return_value.__enter__.return_value = [file_obj]
+        mock_loads.side_effect = ValueError('No JSON object could be decoded')
+
+        crawler = dt_report.OnapVtpCrawler()
+        result = crawler.crawl(testcase_obj, file_path)
+        expected = {'criteria': 'FAIL'}
+
+        mock_path.exists.assert_called_once_with(file_path)
+        mock_open.assert_called_once_with(file_path, 'r')
+        mock_loads.assert_called_once_with(file_obj)
+        testcase_obj.set_results.assert_called_once_with(expected)
+        self.assertEquals(expected, result)
+
     def test_crawler_factory(self):
         result = dt_report.CrawlerFactory.create('shell')
         self.assertEquals(dt_report.ShellCrawler, result.__class__)
@@ -1157,3 +1332,30 @@ class ReportTesting(unittest.TestCase):
 
     def test_checker_factory_none(self):
         self.assertEquals(None, dt_report.CheckerFactory.create('other'))
+
+    @patch('dovetail.report.dt_logger')
+    def test_onapvtp_checker_create_log(self, mock_logger):
+        getlogger_obj = Mock()
+        logger_obj = Mock()
+        logger_obj.getLogger.return_value = getlogger_obj
+        mock_logger.Logger.return_value = logger_obj
+
+        dt_report.OnapVtpChecker.create_log()
+
+        self.assertEquals(getlogger_obj, dt_report.OnapVtpChecker.logger)
+
+    def test_onapvtp_check_result_none(self):
+        testcase_obj = Mock()
+        result = {}
+
+        dt_report.OnapVtpChecker.check(testcase_obj, result)
+
+        testcase_obj.passed.assert_called_once_with('FAIL')
+
+    def test_onapvtp_check_result(self):
+        testcase_obj = Mock()
+        result = {'criteria': 'PASS'}
+
+        dt_report.OnapVtpChecker.check(testcase_obj, result)
+
+        testcase_obj.passed.assert_called_once_with('PASS')
index 2570ec7..4b5c00b 100644 (file)
@@ -323,7 +323,7 @@ class TestRunnerTesting(unittest.TestCase):
     @patch('dovetail.test_runner.os')
     def test_add_testcase_info(self, mock_os, mock_config):
         mock_os.getenv.side_effect = ['os_insecure', 'dovetail_home', 'debug',
-                                      'os_cacert']
+                                      'os_cacert', 'host_url', 'csar_file']
         mock_os.environ = {'DEPLOY_SCENARIO': 'deploy_scenario'}
         mock_config.dovetail_config = {'build_tag': 'build_tag'}
 
@@ -332,7 +332,8 @@ class TestRunnerTesting(unittest.TestCase):
             'testcase': 'testcase_name', 'os_insecure': 'os_insecure',
             'deploy_scenario': 'deploy_scenario',
             'dovetail_home': 'dovetail_home', 'debug': 'debug',
-            'build_tag': 'build_tag', 'cacert': 'os_cacert'}
+            'build_tag': 'build_tag', 'cacert': 'os_cacert',
+            'host_url': 'host_url', 'csar_file': 'csar_file'}
         result = t_runner.FunctestRunner._add_testcase_info(self.testcase)
 
         self.testcase.validate_testcase.assert_called_once_with()
@@ -639,3 +640,35 @@ class TestRunnerTesting(unittest.TestCase):
              'pod_file': 'two',
              'full_task_yaml': 'full_value'},
             result)
+
+    @patch('dovetail.test_runner.dt_utils')
+    @patch('dovetail.test_runner.os.path')
+    @patch('dovetail.test_runner.dt_cfg')
+    def test_init_onapvtprunner_no_env_file(self, mock_config, mock_path,
+                                            mock_utils):
+        t_runner.OnapVtpRunner.create_log()
+        mock_path.join.side_effect = ['env_file']
+        mock_config.dovetail_config = {'config_dir': 'one', 'env_file': 'two'}
+        mock_path.isfile.return_value = False
+
+        docker_runner = t_runner.OnapVtpRunner(self.testcase)
+
+        mock_path.join.assert_has_calls([call('one', 'two')])
+        mock_path.isfile.assert_called_once()
+        docker_runner.logger.error.assert_called_once_with(
+            'File env_file does not exist.')
+
+    @patch('dovetail.test_runner.dt_utils')
+    @patch('dovetail.test_runner.os.path')
+    @patch('dovetail.test_runner.dt_cfg')
+    def test_init_onapvtprunner(self, mock_config, mock_path, mock_utils):
+        t_runner.OnapVtpRunner.create_log()
+        mock_path.join.side_effect = ['env_file']
+        mock_config.dovetail_config = {'config_dir': 'one', 'env_file': 'two'}
+        mock_path.isfile.return_value = True
+
+        t_runner.OnapVtpRunner(self.testcase)
+
+        mock_path.join.assert_has_calls([call('one', 'two')])
+        mock_path.isfile.assert_called_once()
+        mock_utils.source_env.assert_called_once_with('env_file')
index c3eb683..7224c1a 100644 (file)
@@ -113,7 +113,7 @@ class TestcaseTesting(unittest.TestCase):
         self.assertEquals(True, result)
 
     def test_str(self):
-        testcase = tcase.Testcase(self.testcase_yaml)
+        testcase = tcase.OnapVtpTestcase(self.testcase_yaml)
 
         result = testcase.__str__()
 
index 2e7b610..0f26eb2 100644 (file)
@@ -77,7 +77,7 @@ def exec_cmd(cmd, logger=None, exit_on_error=False, info=False,
 
 # walkthrough the object, yield path and value
 
-# dual python 2/3 compatability, inspired by the "six" library
+# dual python 2/3 compatibility, inspired by the "six" library
 string_types = (str, unicode) if str is bytes else (str, bytes)
 # iteritems = lambda mapping: getattr(mapping, 'iteritems', mapping.items)()
 
diff --git a/etc/conf/onap-vtp_config.yml b/etc/conf/onap-vtp_config.yml
new file mode 100644 (file)
index 0000000..85944c3
--- /dev/null
@@ -0,0 +1,25 @@
+---
+
+{% set validate_testcase = validate_testcase or '' %}
+{% set testcase = testcase or '' %}
+{% set build_tag = build_tag or '' %}
+{% set csar_file = csar_file or '' %}
+{% set host_url = host_url or '' %}
+{% set result_dir = '/tmp/onap' %}
+
+onap-vtp:
+  image_name: nexus3.onap.org:10001/onap/cli
+  docker_tag: 2.0.5
+  opts: '-td '
+  envs: '-e OPEN_CLI_MODE=daemon -e BUILD_TAG={{build_tag}}-{{testcase}}
+         -e OPEN_CLI_PRODUCT_IN_USE=onap-vtp'
+  volumes:
+    - '-v {{dovetail_home}}/pre_config/{{csar_file}}:/{{csar_file}}'
+    - '-v {{dovetail_home}}/results:{{result_dir}}'
+  pre_condition:
+    - 'echo this is pre_condition'
+  cmds:
+    - "oclip vnftest-run --name {{validate_testcase}} --param csar=/{{csar_file}}
+       --host-url {{host_url}} --format json > {{result_dir}}/{{testcase}}.out"
+  post_condition:
+    - 'echo this is post_condition'
diff --git a/etc/testcase/onap-vtp.validate.csar.yml b/etc/testcase/onap-vtp.validate.csar.yml
new file mode 100644 (file)
index 0000000..4a9d0eb
--- /dev/null
@@ -0,0 +1,14 @@
+---
+onap-vtp.validate.csar:
+  name: onap-vtp.validate.csar
+  objective: onap csar validation
+  validate:
+    type: onap-vtp
+    testcase: csar-validate
+  report:
+    source_archive_files:
+      - onap-vtp.validate.csar.out
+    dest_archive_files:
+      - onap-vtp_logs/onap-vtp.validate.csar.out
+    check_results_file: onap-vtp_logs/onap-vtp.validate.csar.out
+    sub_testcase_list:
diff --git a/etc/userconfig/env_config.sh.onap.sample b/etc/userconfig/env_config.sh.onap.sample
new file mode 100644 (file)
index 0000000..d983b46
--- /dev/null
@@ -0,0 +1,4 @@
+export HOST_URL="http://<docker host ip>:8702"
+
+# Absolute path of CSAR file, and should be copied to vtp container.
+export CSAR_FILE="/opt/test.csar"