Update to Python3 12/67912/1
authorCédric Ollivier <cedric.ollivier@orange.com>
Tue, 7 May 2019 09:02:43 +0000 (11:02 +0200)
committerCédric Ollivier <ollivier.cedric@gmail.com>
Thu, 23 May 2019 09:03:17 +0000 (11:03 +0200)
Functest containers leverage on Python3 instead of python2.
https://mail.python.org/pipermail/python-dev/2018-March/152348.html

It also updates robotframework librairies to latest release and Vmtp
to master ([1] is needed)

It patches cloudify rest client to support python3.

Vmtp is currently disabled because it currently supports python2 only.

[1] https://github.com/openstack/vmtp/commit/a5d062881d91bf4f547d92c6e289bea30feb5d6e#diff-b4ef698db8ca845e5845c4618278f29a

Change-Id: I39964a212ec2d4dbf53c9ea7c63f02e0b6a05b48
Signed-off-by: Cédric Ollivier <cedric.ollivier@orange.com>
(cherry picked from commit 2a0702ea914a574ffa7c6d6f83a5c606e24ebd0a)

functest/opnfv_tests/openstack/barbican/barbican.py
functest/opnfv_tests/openstack/patrole/patrole.py
functest/opnfv_tests/openstack/rally/rally.py
functest/opnfv_tests/openstack/refstack/refstack.py
functest/opnfv_tests/openstack/tempest/tempest.py
functest/opnfv_tests/vnf/epc/juju_epc.py
functest/opnfv_tests/vnf/router/vnf_controller/ssh_client.py
functest/tests/unit/openstack/rally/test_rally.py
functest/tests/unit/openstack/tempest/test_tempest.py
functest/tests/unit/utils/test_functest_utils.py
functest/utils/functest_utils.py

index b9488c2..7b1bb24 100644 (file)
@@ -36,6 +36,6 @@ class Barbican(tempest.TempestCommon):
         if not rconfig.has_section('image-feature-enabled'):
             rconfig.add_section('image-feature-enabled')
         rconfig.set('image-feature-enabled', 'api_v1', False)
-        with open(self.conf_file, 'wb') as config_file:
+        with open(self.conf_file, 'w') as config_file:
             rconfig.write(config_file)
         self.backup_tempest_config(self.conf_file, self.res_dir)
index 45378b6..9bd877b 100644 (file)
@@ -27,7 +27,7 @@ class Patrole(tempest.TempestCommon):
         rconfig.add_section('rbac')
         rconfig.set('rbac', 'enable_rbac', True)
         rconfig.set('rbac', 'rbac_test_role', kwargs.get('role', 'admin'))
-        with open(self.conf_file, 'wb') as config_file:
+        with open(self.conf_file, 'w') as config_file:
             rconfig.write(config_file)
         self.backup_tempest_config(self.conf_file, self.res_dir)
 
index 25de3cf..674c742 100644 (file)
@@ -166,7 +166,7 @@ class RallyBase(singlevm.VmReady2):
                                 stdout=subprocess.PIPE,
                                 stderr=subprocess.STDOUT)
         deployment_uuid = proc.stdout.readline().rstrip()
-        return deployment_uuid
+        return deployment_uuid.decode()
 
     @staticmethod
     def create_rally_deployment(environ=None):
@@ -215,7 +215,7 @@ class RallyBase(singlevm.VmReady2):
                 rconfig.add_section('openstack')
             rconfig.set(
                 'openstack', 'keystone_default_role', env.get("NEW_USER_ROLE"))
-            with open(rally_conf, 'wb') as config_file:
+            with open(rally_conf, 'w') as config_file:
                 rconfig.write(config_file)
 
     @staticmethod
@@ -226,7 +226,7 @@ class RallyBase(singlevm.VmReady2):
             rconfig.read(rally_conf)
             if rconfig.has_option('openstack', 'keystone_default_role'):
                 rconfig.remove_option('openstack', 'keystone_default_role')
-            with open(rally_conf, 'wb') as config_file:
+            with open(rally_conf, 'w') as config_file:
                 rconfig.write(config_file)
 
     @staticmethod
@@ -240,7 +240,7 @@ class RallyBase(singlevm.VmReady2):
         taskid_re = re.compile('^Task +(.*): started$')
         for line in cmd_raw.splitlines(True):
             line = line.strip()
-            match = taskid_re.match(line)
+            match = taskid_re.match(line.decode())
             if match:
                 return match.group(1)
         return None
@@ -657,8 +657,8 @@ class RallyBase(singlevm.VmReady2):
                 "{}/{}.xml".format(self.results_dir, self.case_name),
                 export_type="junit-xml")
             res = testcase.TestCase.EX_OK
-        except Exception as exc:   # pylint: disable=broad-except
-            LOGGER.error('Error with run: %s', exc)
+        except Exception:   # pylint: disable=broad-except
+            LOGGER.exception('Error with run:')
             self.result = 0
             res = testcase.TestCase.EX_RUN_ERROR
         self.stop_time = time.time()
index e80e5c2..49fa658 100644 (file)
@@ -52,7 +52,7 @@ class Refstack(tempest.TempestCommon):
         yaml_data2 = ""
         for line in output.splitlines():
             try:
-                grp = re.search(r'^([^\[]*)(\[.*\])\n*$', line)
+                grp = re.search(r'^([^\[]*)(\[.*\])\n*$', line.decode())
                 yaml_data2 = "{}\n{}: {}".format(
                     yaml_data2, grp.group(1), grp.group(2))
             except Exception:  # pylint: disable=broad-except
index 0874e6d..30e9ce6 100644 (file)
@@ -143,7 +143,7 @@ class TempestCommon(singlevm.VmReady2):
                                 stderr=subprocess.STDOUT)
         for line in proc.stdout:
             LOGGER.info(line.rstrip())
-            new_line = line.replace(' ', '').split('|')
+            new_line = line.decode().replace(' ', '').split('|')
             if 'Tests' in new_line:
                 break
             if 'Testscount' in new_line:
@@ -199,7 +199,7 @@ class TempestCommon(singlevm.VmReady2):
                                 stdout=subprocess.PIPE,
                                 stderr=subprocess.STDOUT)
         verifier_uuid = proc.stdout.readline().rstrip()
-        return verifier_uuid
+        return verifier_uuid.decode()
 
     @staticmethod
     def get_verifier_repo_dir(verifier_id):
@@ -235,7 +235,7 @@ class TempestCommon(singlevm.VmReady2):
                 for key, value in sub_conf.items():
                     rconfig.set(section, key, value)
 
-        with open(conf_file, 'wb') as config_file:
+        with open(conf_file, 'w') as config_file:
             rconfig.write(config_file)
 
     @staticmethod
@@ -403,12 +403,12 @@ class TempestCommon(singlevm.VmReady2):
 
         with proc.stdout:
             for line in iter(proc.stdout.readline, b''):
-                if re.search(r"\} tempest\.", line):
+                if re.search(r"\} tempest\.", line.decode()):
                     LOGGER.info(line.rstrip())
-                elif re.search(r'(?=\(UUID=(.*)\))', line):
+                elif re.search(r'(?=\(UUID=(.*)\))', line.decode()):
                     self.verification_id = re.search(
-                        r'(?=\(UUID=(.*)\))', line).group(1)
-                f_stdout.write(line)
+                        r'(?=\(UUID=(.*)\))', line.decode()).group(1)
+                f_stdout.write(line.decode())
         proc.wait()
         f_stdout.close()
 
@@ -473,7 +473,7 @@ class TempestCommon(singlevm.VmReady2):
             rconfig.add_section('openstack')
         rconfig.set('openstack', 'img_name_regex', '^{}$'.format(
             self.image.name))
-        with open(rally_conf, 'wb') as config_file:
+        with open(rally_conf, 'w') as config_file:
             rconfig.write(config_file)
 
     def update_default_role(self, rally_conf='/etc/rally/rally.conf'):
@@ -486,7 +486,7 @@ class TempestCommon(singlevm.VmReady2):
         if not rconfig.has_section('openstack'):
             rconfig.add_section('openstack')
         rconfig.set('openstack', 'swift_operator_role', role.name)
-        with open(rally_conf, 'wb') as config_file:
+        with open(rally_conf, 'w') as config_file:
             rconfig.write(config_file)
 
     def update_rally_logs(self, rally_conf='/etc/rally/rally.conf'):
@@ -499,7 +499,7 @@ class TempestCommon(singlevm.VmReady2):
         rconfig.set('DEFAULT', 'use_stderr', False)
         rconfig.set('DEFAULT', 'log-file', 'rally.log')
         rconfig.set('DEFAULT', 'log_dir', self.res_dir)
-        with open(rally_conf, 'wb') as config_file:
+        with open(rally_conf, 'w') as config_file:
             rconfig.write(config_file)
 
     @staticmethod
@@ -519,7 +519,7 @@ class TempestCommon(singlevm.VmReady2):
             rconfig.remove_option('DEFAULT', 'log-file')
         if rconfig.has_option('DEFAULT', 'log_dir'):
             rconfig.remove_option('DEFAULT', 'log_dir')
-        with open(rally_conf, 'wb') as config_file:
+        with open(rally_conf, 'w') as config_file:
             rconfig.write(config_file)
 
     def update_network_section(self):
@@ -530,7 +530,7 @@ class TempestCommon(singlevm.VmReady2):
             rconfig.add_section('network')
         rconfig.set('network', 'public_network_id', self.ext_net.id)
         rconfig.set('network', 'floating_network_name', self.ext_net.name)
-        with open(self.conf_file, 'wb') as config_file:
+        with open(self.conf_file, 'w') as config_file:
             rconfig.write(config_file)
 
     def update_compute_section(self):
@@ -540,7 +540,7 @@ class TempestCommon(singlevm.VmReady2):
         if not rconfig.has_section('compute'):
             rconfig.add_section('compute')
         rconfig.set('compute', 'fixed_network_name', self.network.name)
-        with open(self.conf_file, 'wb') as config_file:
+        with open(self.conf_file, 'w') as config_file:
             rconfig.write(config_file)
 
     def update_scenario_section(self):
@@ -567,7 +567,7 @@ class TempestCommon(singlevm.VmReady2):
         rconfig.set(
             'scenario', 'img_properties',
             functest_utils.convert_dict_to_ini(extra_properties))
-        with open(self.conf_file, 'wb') as config_file:
+        with open(self.conf_file, 'w') as config_file:
             rconfig.write(config_file)
 
     def configure(self, **kwargs):  # pylint: disable=unused-argument
index cbf531b..93961ad 100644 (file)
@@ -273,7 +273,8 @@ class JujuEpc(singlevm.VmReady2):
         for i in range(10):
             output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
             self.__logger.info("%s\n%s", " ".join(cmd), output)
-            ret = re.search(r'(?=workload:({})\))'.format(status), output)
+            ret = re.search(
+                r'(?=workload:({})\))'.format(status), output.decode())
             if ret:
                 self.__logger.info("%s workload is %s", name, status)
                 break
index c5f554c..67f1e0d 100644 (file)
@@ -110,7 +110,7 @@ class SshClient(object):  # pylint: disable=too-many-instance-attributes
                                       cmd)
                     break
 
-                res_buff += res
+                res_buff += res.decode()
 
             self.logger.debug("Response : '%s'", res_buff)
             return res_buff
index 5dc38a2..1b790a0 100644 (file)
@@ -101,12 +101,12 @@ class OSRallyTesting(unittest.TestCase):
         mock_os_makedirs.assert_called()
 
     def test_get_task_id_default(self):
-        cmd_raw = 'Task 1: started'
+        cmd_raw = b'Task 1: started'
         self.assertEqual(self.rally_base.get_task_id(cmd_raw),
                          '1')
 
     def test_get_task_id_missing_id(self):
-        cmd_raw = ''
+        cmd_raw = b''
         self.assertEqual(self.rally_base.get_task_id(cmd_raw),
                          None)
 
index 1199596..a500a37 100644 (file)
@@ -271,7 +271,7 @@ class OSTempestTesting(unittest.TestCase):
         with mock.patch('functest.opnfv_tests.openstack.tempest.'
                         'tempest.subprocess.Popen') as mock_popen:
             mock_stdout = mock.Mock()
-            attrs = {'stdout.readline.return_value': 'test_deploy_id'}
+            attrs = {'stdout.readline.return_value': b'test_deploy_id'}
             mock_stdout.configure_mock(**attrs)
             mock_popen.return_value = mock_stdout
 
@@ -283,7 +283,7 @@ class OSTempestTesting(unittest.TestCase):
         with mock.patch('functest.opnfv_tests.openstack.tempest.'
                         'tempest.subprocess.Popen') as mock_popen:
             mock_stdout = mock.Mock()
-            attrs = {'stdout.readline.return_value': 'test_deploy_id'}
+            attrs = {'stdout.readline.return_value': b'test_deploy_id'}
             mock_stdout.configure_mock(**attrs)
             mock_popen.return_value = mock_stdout
 
index da3b03c..ba022e3 100644 (file)
@@ -15,6 +15,7 @@ import unittest
 
 import mock
 import pkg_resources
+import six
 
 from functest.utils import functest_utils
 
@@ -97,23 +98,15 @@ class FunctestUtilsTesting(unittest.TestCase):
                 as mock_subproc_open, \
                 mock.patch('six.moves.builtins.open',
                            mock.mock_open()) as mopen:
-
-            FunctestUtilsTesting.readline = 0
-
-            mock_obj = mock.Mock()
-            attrs = {'readline.side_effect': self.cmd_readline()}
-            mock_obj.configure_mock(**attrs)
-
+            stream = six.BytesIO()
+            stream.write(self.cmd_readline().encode())
             mock_obj2 = mock.Mock()
-            attrs = {'stdout': mock_obj, 'wait.return_value': 1}
+            attrs = {'stdout': stream, 'wait.return_value': 1}
             mock_obj2.configure_mock(**attrs)
-
             mock_subproc_open.return_value = mock_obj2
-
-            resp = functest_utils.execute_command(self.cmd, info=True,
-                                                  error_msg=self.error_msg,
-                                                  verbose=True,
-                                                  output_file=self.output_file)
+            resp = functest_utils.execute_command(
+                self.cmd, info=True, error_msg=self.error_msg, verbose=True,
+                output_file=self.output_file)
             self.assertEqual(resp, 1)
             msg_exec = ("Executing command: '%s'" % self.cmd)
             mock_logger_info.assert_called_once_with(msg_exec)
@@ -126,23 +119,15 @@ class FunctestUtilsTesting(unittest.TestCase):
                 as mock_subproc_open, \
                 mock.patch('six.moves.builtins.open',
                            mock.mock_open()) as mopen:
-
-            FunctestUtilsTesting.readline = 0
-
-            mock_obj = mock.Mock()
-            attrs = {'readline.side_effect': self.cmd_readline()}
-            mock_obj.configure_mock(**attrs)
-
+            stream = six.BytesIO()
+            stream.write(self.cmd_readline().encode())
             mock_obj2 = mock.Mock()
-            attrs = {'stdout': mock_obj, 'wait.return_value': 0}
+            attrs = {'stdout': stream, 'wait.return_value': 0}
             mock_obj2.configure_mock(**attrs)
-
             mock_subproc_open.return_value = mock_obj2
-
-            resp = functest_utils.execute_command(self.cmd, info=True,
-                                                  error_msg=self.error_msg,
-                                                  verbose=True,
-                                                  output_file=self.output_file)
+            resp = functest_utils.execute_command(
+                self.cmd, info=True, error_msg=self.error_msg, verbose=True,
+                output_file=self.output_file)
             self.assertEqual(resp, 0)
             msg_exec = ("Executing command: '%s'" % self.cmd)
             mock_logger_info.assert_called_once_with(msg_exec)
@@ -153,23 +138,15 @@ class FunctestUtilsTesting(unittest.TestCase):
         # pylint: disable=unused-argument
         with mock.patch('functest.utils.functest_utils.subprocess.Popen') \
                 as mock_subproc_open:
-
-            FunctestUtilsTesting.readline = 2
-
-            mock_obj = mock.Mock()
-            attrs = {'readline.side_effect': self.cmd_readline()}
-            mock_obj.configure_mock(**attrs)
-
+            stream = six.BytesIO()
+            stream.write(self.cmd_readline().encode())
             mock_obj2 = mock.Mock()
-            attrs = {'stdout': mock_obj, 'wait.return_value': 0}
+            attrs = {'stdout': stream, 'wait.return_value': 0}
             mock_obj2.configure_mock(**attrs)
-
             mock_subproc_open.return_value = mock_obj2
-
-            resp = functest_utils.execute_command(self.cmd, info=False,
-                                                  error_msg="",
-                                                  verbose=False,
-                                                  output_file=None)
+            resp = functest_utils.execute_command(
+                self.cmd, info=False, error_msg="", verbose=False,
+                output_file=None)
             self.assertEqual(resp, 0)
 
     @mock.patch('sys.stdout')
@@ -177,22 +154,15 @@ class FunctestUtilsTesting(unittest.TestCase):
         # pylint: disable=unused-argument
         with mock.patch('functest.utils.functest_utils.subprocess.Popen') \
                 as mock_subproc_open:
-
-            FunctestUtilsTesting.readline = 2
-            mock_obj = mock.Mock()
-            attrs = {'readline.side_effect': self.cmd_readline()}
-            mock_obj.configure_mock(**attrs)
-
+            stream = six.BytesIO()
+            stream.write(self.cmd_readline().encode())
             mock_obj2 = mock.Mock()
-            attrs = {'stdout': mock_obj, 'wait.return_value': 1}
+            attrs = {'stdout': stream, 'wait.return_value': 1}
             mock_obj2.configure_mock(**attrs)
-
             mock_subproc_open.return_value = mock_obj2
-
-            resp = functest_utils.execute_command(self.cmd, info=False,
-                                                  error_msg="",
-                                                  verbose=False,
-                                                  output_file=None)
+            resp = functest_utils.execute_command(
+                self.cmd, info=False, error_msg="", verbose=False,
+                output_file=None)
             self.assertEqual(resp, 1)
 
     def test_get_param_from_yaml_failed(self):
index 19d5baf..4bc417d 100644 (file)
@@ -43,9 +43,9 @@ def execute_command(cmd, info=False, error_msg="",
         ofd = open(output_file, "w")
     for line in iter(popen.stdout.readline, b''):
         if output_file:
-            ofd.write(line)
+            ofd.write(line.decode())
         else:
-            line = line.replace('\n', '')
+            line = line.decode().replace('\n', '')
             print(line)
             sys.stdout.flush()
     if output_file: