Merge "Consider criteria as exit condition in odl test"

diff --combined functest/opnfv_tests/Controllers/ODL/OpenDaylightTesting.py
index 0ddb58c,b755c40..b78db8b
--- 1/functest/opnfv_tests/Controllers/ODL/OpenDaylightTesting.py
--- 2/functest/opnfv_tests/Controllers/ODL/OpenDaylightTesting.py
+++ b/functest/opnfv_tests/Controllers/ODL/OpenDaylightTesting.py
@@@ -23,7 -23,6 +23,7 @@@ from robot.utils.robottime import times
  from functest.core import TestCasesBase
  import functest.utils.functest_logger as ft_logger
  import functest.utils.openstack_utils as op_utils
 +import functest.utils.functest_constants as ft_constants
  
  
  class ODLResultVisitor(ResultVisitor):
@@@ -49,14 -48,11 +49,14 @@@
  
  class ODLTestCases(TestCasesBase.TestCasesBase):
  
 -    repos = "/home/opnfv/repos/"
 -    odl_test_repo = repos + "odl_test/"
 -    neutron_suite_dir = odl_test_repo + "csit/suites/openstack/neutron/"
 -    basic_suite_dir = odl_test_repo + "csit/suites/integration/basic/"
 -    res_dir = '/home/opnfv/functest/results/odl/'
 +    repos = ft_constants.REPOS_DIR
 +    odl_test_repo = os.path.join(repos, "odl_test")
 +    neutron_suite_dir = os.path.join(odl_test_repo,
 +                                     "csit/suites/openstack/neutron")
 +    basic_suite_dir = os.path.join(odl_test_repo,
 +                                   "csit/suites/integration/basic")
 +    res_dir = os.path.join(ft_constants.FUNCTEST_RESULTS_DIR, "odl")
 +
      logger = ft_logger.Logger("opendaylight").getLogger()
  
      def __init__(self):
@@@ -64,8 -60,7 +64,8 @@@
  
      @classmethod
      def set_robotframework_vars(cls, odlusername="admin", odlpassword="admin"):
 -        odl_variables_files = cls.odl_test_repo + 'csit/variables/Variables.py'
 +        odl_variables_files = os.path.join(cls.odl_test_repo,
 +                                           'csit/variables/Variables.py')
          try:
              for line in fileinput.input(odl_variables_files,
                                          inplace=True):
@@@ -79,8 -74,7 +79,8 @@@
              return False
  
      def parse_results(self):
 -        result = ExecutionResult(self.res_dir + 'output.xml')
 +        output_dir = os.path.join(self.res_dir, 'output.xml')
 +        result = ExecutionResult(output_dir)
          visitor = ODLResultVisitor()
          result.visit(visitor)
          self.criteria = result.suite.status
@@@ -89,6 -83,7 +89,7 @@@
          self.details = {}
          self.details['description'] = result.suite.name
          self.details['tests'] = visitor.get_data()
+         return self.criteria
  
      def main(self, **kwargs):
          dirs = [self.basic_suite_dir, self.neutron_suite_dir]
@@@ -115,11 -110,10 +116,11 @@@
                      self.logger.exception(
                          "Cannot create {}".format(self.res_dir))
                      return self.EX_RUN_ERROR
 -            stdout_file = self.res_dir + 'stdout.txt'
 +            stdout_file = os.path.join(self.res_dir, 'stdout.txt')
 +            output_dir = os.path.join(self.res_dir, 'output.xml')
              with open(stdout_file, 'w+') as stdout:
                  robot.run(*dirs, variable=variables,
 -                          output=self.res_dir + 'output.xml',
 +                          output=output_dir,
                            log='NONE',
                            report='NONE',
                            stdout=stdout)
@@@ -127,8 -121,10 +128,10 @@@
                  self.logger.info("\n" + stdout.read())
              self.logger.info("ODL results were successfully generated")
              try:
-                 self.parse_results()
+                 test_res = self.parse_results()
                  self.logger.info("ODL results were successfully parsed")
+                 if test_res is not "PASS":
+                     return self.EX_RUN_ERROR
              except RobotError as e:
                  self.logger.error("Run tests before publishing: %s" %
                                    e.message)
@@@ -155,29 -151,23 +158,29 @@@
              kwargs['odlrestconfport'] = '8181'
              kwargs['odlusername'] = 'admin'
              kwargs['odlpassword'] = 'admin'
 -            installer_type = None
 -            if 'INSTALLER_TYPE' in os.environ:
 -                installer_type = os.environ['INSTALLER_TYPE']
 -            kwargs['osusername'] = os.environ['OS_USERNAME']
 -            kwargs['ostenantname'] = os.environ['OS_TENANT_NAME']
 -            kwargs['ospassword'] = os.environ['OS_PASSWORD']
 +
 +            installer_type = ft_constants.CI_INSTALLER_TYPE
 +            kwargs['osusername'] = ft_constants.OS_USERNAME
 +            kwargs['ostenantname'] = ft_constants.OS_TENANT_NAME
 +            kwargs['ospassword'] = ft_constants.OS_PASSWORD
 +
              if installer_type == 'fuel':
                  kwargs['odlwebport'] = '8282'
              elif installer_type == 'apex':
 -                kwargs['odlip'] = os.environ['SDN_CONTROLLER_IP']
 +                if ft_constants.SDN_CONTROLLER_IP is None:
 +                    return self.EX_RUN_ERROR
 +                kwargs['odlip'] = ft_constants.SDN_CONTROLLER_IP
                  kwargs['odlwebport'] = '8181'
              elif installer_type == 'joid':
 -                kwargs['odlip'] = os.environ['SDN_CONTROLLER']
 +                if ft_constants.SDN_CONTROLLER is None:
 +                    return self.EX_RUN_ERROR
 +                kwargs['odlip'] = ft_constants.SDN_CONTROLLER
              elif installer_type == 'compass':
                  kwargs['odlwebport'] = '8181'
              else:
 -                kwargs['odlip'] = os.environ['SDN_CONTROLLER_IP']
 +                if ft_constants.SDN_CONTROLLER_IP is None:
 +                    return self.EX_RUN_ERROR
 +                kwargs['odlip'] = ft_constants.SDN_CONTROLLER_IP
          except KeyError as e:
              self.logger.error("Cannot run ODL testcases. "
                                "Please check env var: "
@@@ -189,7 -179,6 +192,7 @@@
  
          return self.main(**kwargs)
  
 +
  if __name__ == '__main__':
      parser = argparse.ArgumentParser()
      parser.add_argument('-k', '--keystoneip',

diff --combined functest/tests/unit/odl/test_odl.py
index 065298d,a62bf39..fdd4576
--- 1/functest/tests/unit/odl/test_odl.py
--- 2/functest/tests/unit/odl/test_odl.py
+++ b/functest/tests/unit/odl/test_odl.py
@@@ -17,7 -17,6 +17,7 @@@ from robot.errors import RobotErro
  
  from functest.core import TestCasesBase
  from functest.opnfv_tests.Controllers.ODL import OpenDaylightTesting
 +from functest.utils import functest_constants as ft_constants
  
  
  class ODLTestCasesTesting(unittest.TestCase):
@@@ -36,9 -35,12 +36,9 @@@
      _odl_password = "admin"
  
      def setUp(self):
 -        for var in ("INSTALLER_TYPE", "SDN_CONTROLLER", "SDN_CONTROLLER_IP"):
 -            if var in os.environ:
 -                del os.environ[var]
 -        os.environ["OS_USERNAME"] = self._os_username
 -        os.environ["OS_PASSWORD"] = self._os_password
 -        os.environ["OS_TENANT_NAME"] = self._os_tenantname
 +        ft_constants.OS_USERNAME = self._os_username
 +        ft_constants.OS_PASSWORD = self._os_password
 +        ft_constants.OS_TENANT_NAME = self._os_tenantname
          self.test = OpenDaylightTesting.ODLTestCases()
  
      @mock.patch('fileinput.input', side_effect=Exception())
@@@ -85,9 -87,9 +85,9 @@@
      def _test_main(self, status, *args):
          kwargs = self._get_main_kwargs()
          self.assertEqual(self.test.main(**kwargs), status)
 +        odl_res_dir = OpenDaylightTesting.ODLTestCases.res_dir
          if len(args) > 0:
 -            args[0].assert_called_once_with(
 -                OpenDaylightTesting.ODLTestCases.res_dir)
 +            args[0].assert_called_once_with(odl_res_dir)
          if len(args) > 1:
              variable = ['KEYSTONE:{}'.format(self._keystone_ip),
                          'NEUTRON:{}'.format(self._neutron_ip),
@@@ -97,18 -99,17 +97,18 @@@
                          'ODL_SYSTEM_IP:{}'.format(self._sdn_controller_ip),
                          'PORT:{}'.format(self._odl_webport),
                          'RESTCONFPORT:{}'.format(self._odl_restconfport)]
 +            output_file = os.path.join(odl_res_dir, 'output.xml')
              args[1].assert_called_once_with(
                  OpenDaylightTesting.ODLTestCases.basic_suite_dir,
                  OpenDaylightTesting.ODLTestCases.neutron_suite_dir,
                  log='NONE',
 -                output=OpenDaylightTesting.ODLTestCases.res_dir + 'output.xml',
 +                output=output_file,
                  report='NONE',
                  stdout=mock.ANY,
                  variable=variable)
          if len(args) > 2:
 -            args[2].assert_called_with(
 -                OpenDaylightTesting.ODLTestCases.res_dir + 'stdout.txt')
 +            stdout_file = os.path.join(odl_res_dir, 'stdout.txt')
 +            args[2].assert_called_with(stdout_file)
  
      def _test_main_missing_keyword(self, key):
          kwargs = self._get_main_kwargs(key)
@@@ -200,7 -201,8 +200,8 @@@
      def test_main(self, *args):
          with mock.patch.object(self.test, 'set_robotframework_vars',
                                 return_value=True), \
-                 mock.patch.object(self.test, 'parse_results'):
+                 mock.patch.object(self.test, 'parse_results',
+                                   return_value="PASS"):
              self._test_main(TestCasesBase.TestCasesBase.EX_OK, *args)
  
      @mock.patch('os.remove')
@@@ -209,7 -211,8 +210,8 @@@
      def test_main_makedirs_oserror17(self, *args):
          with mock.patch.object(self.test, 'set_robotframework_vars',
                                 return_value=True), \
-                 mock.patch.object(self.test, 'parse_results'):
+                 mock.patch.object(self.test, 'parse_results',
+                                   return_value="PASS"):
              self._test_main(TestCasesBase.TestCasesBase.EX_OK, *args)
  
      @mock.patch('os.remove')
@@@ -218,7 -221,8 +220,8 @@@
      def test_main_testcases_in_failure(self, *args):
          with mock.patch.object(self.test, 'set_robotframework_vars',
                                 return_value=True), \
-                 mock.patch.object(self.test, 'parse_results'):
+                 mock.patch.object(self.test, 'parse_results',
+                                   return_value="PASS"):
              self._test_main(TestCasesBase.TestCasesBase.EX_OK, *args)
  
      @mock.patch('os.remove', side_effect=OSError)
@@@ -227,17 -231,12 +230,18 @@@
      def test_main_remove_oserror(self, *args):
          with mock.patch.object(self.test, 'set_robotframework_vars',
                                 return_value=True), \
-                 mock.patch.object(self.test, 'parse_results'):
+                 mock.patch.object(self.test, 'parse_results',
+                                   return_value="PASS"):
              self._test_main(TestCasesBase.TestCasesBase.EX_OK, *args)
  
      def _test_run_missing_env_var(self, var):
 -        del os.environ[var]
 +        if var == 'OS_USERNAME':
 +            ft_constants.OS_USERNAME = None
 +        elif var == 'OS_PASSWORD':
 +            ft_constants.OS_PASSWORD = None
 +        elif var == 'OS_TENANT_NAME':
 +            ft_constants.OS_TENANT_NAME = None
 +
          self.assertEqual(self.test.run(),
                           TestCasesBase.TestCasesBase.EX_RUN_ERROR)
  
@@@ -268,15 -267,13 +272,15 @@@
          self._test_run_missing_env_var("OS_TENANT_NAME")
  
      def test_run_main_false(self):
 -        os.environ["SDN_CONTROLLER_IP"] = self._sdn_controller_ip
 +        ft_constants.CI_INSTALLER_TYPE = None
 +        ft_constants.SDN_CONTROLLER_IP = self._sdn_controller_ip
          self._test_run(TestCasesBase.TestCasesBase.EX_RUN_ERROR,
                         odlip=self._sdn_controller_ip,
                         odlwebport=self._odl_webport)
  
      def test_run_main_exception(self):
 -        os.environ["SDN_CONTROLLER_IP"] = self._sdn_controller_ip
 +        ft_constants.CI_INSTALLER_TYPE = None
 +        ft_constants.SDN_CONTROLLER_IP = self._sdn_controller_ip
          with self.assertRaises(Exception):
              self._test_run(status=TestCasesBase.TestCasesBase.EX_RUN_ERROR,
                             exception=Exception(),
@@@ -286,54 -283,49 +290,54 @@@
      def test_run_missing_sdn_controller_ip(self):
          with mock.patch('functest.utils.openstack_utils.get_keystone_client',
                          return_value=self._get_fake_keystone_client()):
 +            ft_constants.CI_INSTALLER_TYPE = None
 +            ft_constants.SDN_CONTROLLER_IP = None
              self.assertEqual(self.test.run(),
                               TestCasesBase.TestCasesBase.EX_RUN_ERROR)
  
      def test_run_without_installer_type(self):
 -        os.environ["SDN_CONTROLLER_IP"] = self._sdn_controller_ip
 +        ft_constants.SDN_CONTROLLER_IP = self._sdn_controller_ip
 +        ft_constants.CI_INSTALLER_TYPE = None
          self._test_run(TestCasesBase.TestCasesBase.EX_OK,
                         odlip=self._sdn_controller_ip,
                         odlwebport=self._odl_webport)
  
      def test_run_fuel(self):
 -        os.environ["INSTALLER_TYPE"] = "fuel"
 +        ft_constants.CI_INSTALLER_TYPE = "fuel"
          self._test_run(TestCasesBase.TestCasesBase.EX_OK,
                         odlip=self._neutron_ip, odlwebport='8282')
  
      def test_run_apex_missing_sdn_controller_ip(self):
          with mock.patch('functest.utils.openstack_utils.get_keystone_client',
                          return_value=self._get_fake_keystone_client()):
 -            os.environ["INSTALLER_TYPE"] = "apex"
 +            ft_constants.CI_INSTALLER_TYPE = "apex"
 +            ft_constants.SDN_CONTROLLER_IP = None
              self.assertEqual(self.test.run(),
                               TestCasesBase.TestCasesBase.EX_RUN_ERROR)
  
      def test_run_apex(self):
 -        os.environ["SDN_CONTROLLER_IP"] = self._sdn_controller_ip
 -        os.environ["INSTALLER_TYPE"] = "apex"
 +        ft_constants.SDN_CONTROLLER_IP = self._sdn_controller_ip
 +        ft_constants.CI_INSTALLER_TYPE = "apex"
          self._test_run(TestCasesBase.TestCasesBase.EX_OK,
                         odlip=self._sdn_controller_ip, odlwebport='8181')
  
      def test_run_joid_missing_sdn_controller(self):
          with mock.patch('functest.utils.openstack_utils.get_keystone_client',
                          return_value=self._get_fake_keystone_client()):
 -            os.environ["INSTALLER_TYPE"] = "joid"
 +            ft_constants.CI_INSTALLER_TYPE = "joid"
 +            ft_constants.SDN_CONTROLLER = None
              self.assertEqual(self.test.run(),
                               TestCasesBase.TestCasesBase.EX_RUN_ERROR)
  
      def test_run_joid(self):
 -        os.environ["SDN_CONTROLLER"] = self._sdn_controller_ip
 -        os.environ["INSTALLER_TYPE"] = "joid"
 +        ft_constants.SDN_CONTROLLER = self._sdn_controller_ip
 +        ft_constants.CI_INSTALLER_TYPE = "joid"
          self._test_run(TestCasesBase.TestCasesBase.EX_OK,
                         odlip=self._sdn_controller_ip,
                         odlwebport=self._odl_webport)
  
      def test_run_compass(self, *args):
 -        os.environ["INSTALLER_TYPE"] = "compass"
 +        ft_constants.CI_INSTALLER_TYPE = "compass"
          self._test_run(TestCasesBase.TestCasesBase.EX_OK,
                         odlip=self._neutron_ip, odlwebport='8181')