Merge "unify test result check for feature project and apply to parser"

diff --git a/ci/tier_handler.py b/ci/tier_handler.py
index e50fbe6..1eadfba 100644 (file)
--- a/ci/tier_handler.py
+++ b/ci/tier_handler.py
@@ -111,17 +111,21 @@ class TestCase:
         self.criteria = criteria
         self.blocking = blocking
 
+    @staticmethod
+    def is_none(item):
+        return item is None or item is ""
+
     def is_compatible(self, ci_installer, ci_scenario):
         try:
-            if ci_installer is not None:
+            if not self.is_none(ci_installer):
                 if re.search(self.dependency.get_installer(),
                              ci_installer) is None:
                     return False
-            if ci_scenario is not None:
+            if not self.is_none(ci_scenario):
                 if re.search(self.dependency.get_scenario(),
                              ci_scenario) is None:
                     return False
-            return not (ci_scenario is None and ci_installer is None)
+            return True
         except TypeError:
             return False
 

diff --git a/docker/requirements.pip b/docker/requirements.pip
index f590b2a..52d0a9c 100644 (file)
--- a/docker/requirements.pip
+++ b/docker/requirements.pip
@@ -25,3 +25,4 @@ paramiko==1.16.0
 python-congressclient==1.3.0
 subprocess32
 shyaml
+dnspython

diff --git a/testcases/OpenStack/tempest/custom_tests/blacklist.txt b/testcases/OpenStack/tempest/custom_tests/blacklist.txt
index 49605ca..5dd8fe2 100644 (file)
--- a/testcases/OpenStack/tempest/custom_tests/blacklist.txt
+++ b/testcases/OpenStack/tempest/custom_tests/blacklist.txt
@@ -6,12 +6,28 @@
         - fuel
         - apex
     tests:
-        - tempest.api.compute.servers.test_create_server.ServersTestJSON.test_list_servers[id-9a438d88-10c6-4bcd-8b5b-5b6e25e1346f,smoke]
-        - tempest.api.compute.servers.test_create_server.ServersTestJSON.test_verify_server_details[id-5de47127-9977-400a-936f-abcfbec1218f,smoke]
-        - tempest.api.compute.servers.test_create_server.ServersTestManualDisk.test_list_servers[id-9a438d88-10c6-4bcd-8b5b-5b6e25e1346f,smoke]
-        - tempest.api.compute.servers.test_create_server.ServersTestManualDisk.test_verify_server_details[id-5de47127-9977-400a-936f-abcfbec1218f,smoke]
-        - tempest.api.compute.servers.test_server_actions.ServerActionsTestJSON.test_reboot_server_hard[id-2cb1baf6-ac8d-4429-bf0d-ba8a0ba53e32,smoke]
-        - tempest.scenario.test_network_basic_ops.TestNetworkBasicOps.test_network_basic_ops[compute,id-f323b3ba-82f8-4db7-8ea6-6a895869ec49,network,smoke]
-        - tempest.scenario.test_server_basic_ops.TestServerBasicOps.test_server_basic_ops[compute,id-7fff3fb3-91d8-4fd0-bd7d-0204f1f180ba,network,smoke]
-        - tempest.scenario.test_volume_boot_pattern.TestVolumeBootPattern.test_volume_boot_pattern[compute,id-557cd2c2-4eb8-4dce-98be-f86765ff311b,image,smoke,volume]
-        - tempest.scenario.test_volume_boot_pattern.TestVolumeBootPatternV2.test_volume_boot_pattern[compute,id-557cd2c2-4eb8-4dce-98be-f86765ff311b,image,smoke,volume]
+        - tempest.api.compute.servers.test_create_server.ServersTestJSON.test_list_servers
+        - tempest.api.compute.servers.test_create_server.ServersTestJSON.test_verify_server_details
+        - tempest.api.compute.servers.test_create_server.ServersTestManualDisk.test_list_servers
+        - tempest.api.compute.servers.test_create_server.ServersTestManualDisk.test_verify_server_details
+        - tempest.api.compute.servers.test_server_actions.ServerActionsTestJSON.test_reboot_server_hard
+        - tempest.scenario.test_network_basic_ops.TestNetworkBasicOps.test_network_basic_ops
+        - tempest.scenario.test_server_basic_ops.TestServerBasicOps.test_server_basic_ops
+        - tempest.scenario.test_volume_boot_pattern.TestVolumeBootPattern.test_volume_boot_pattern
+        - tempest.scenario.test_volume_boot_pattern.TestVolumeBootPatternV2.test_volume_boot_pattern
+
+    scenarios:
+        - os-odl_l2-nofeature-ha
+        - os-odl_l2-nofeature-noha
+        - os-nosdn-nofeature-ha
+        - os-nosdn-nofeature-noha
+        - os-onos-nofeature-ha
+        - os-onos-nofeature-noha
+        - os-onos-sfc-ha
+        - os-onos-sfc-noha
+        - os-nosdn-lxd-ha
+        - os-nosdn-lxd-noha
+    installers:
+        - joid
+    tests:
+        - tempest.api.object_storage

diff --git a/testcases/OpenStack/tempest/run_tempest.py b/testcases/OpenStack/tempest/run_tempest.py
index 3ee10b7..53a70b7 100755 (executable)
--- a/testcases/OpenStack/tempest/run_tempest.py
+++ b/testcases/OpenStack/tempest/run_tempest.py
@@ -214,8 +214,8 @@ def configure_tempest(deployment_dir):
 
 
 def configure_tempest_feature(deployment_dir, mode):
-    """
-    Add/update needed parameters into tempest.conf file generated by Rally
+    """Add/update needed parameters into tempest.conf file generated by Rally
+
     """
 
     logger.debug("Finding tempest.conf file...")
@@ -231,56 +231,11 @@ def configure_tempest_feature(deployment_dir, mode):
     if mode == 'feature_multisite':
         config.set('service_available', 'kingbird', 'true')
         cmd = "openstack endpoint show kingbird | grep publicurl |\
-               awk '{print $4}' | awk -F '/' '{print $4}'"
-        kingbird_api_version = os.popen(cmd).read()
-        if os.environ.get("INSTALLER_TYPE") == 'fuel':
-            # For MOS based setup, the service is accessible
-            # via bind host
-            kingbird_conf_path = "/etc/kingbird/kingbird.conf"
-            installer_type = os.getenv('INSTALLER_TYPE', 'Unknown')
-            installer_ip = os.getenv('INSTALLER_IP', 'Unknown')
-            installer_username = ft_utils.get_parameter_from_yaml(
-                "multisite." + installer_type +
-                "_environment.installer_username")
-            installer_password = ft_utils.get_parameter_from_yaml(
-                "multisite." + installer_type +
-                "_environment.installer_password")
-
-            ssh_options = "-o UserKnownHostsFile=/dev/null -o \
-                StrictHostKeyChecking=no"
-
-            # Get the controller IP from the fuel node
-            cmd = 'sshpass -p %s ssh 2>/dev/null %s %s@%s \
-                    \'fuel node --env 1| grep controller | grep "True\|  1" \
-                    | awk -F\| "{print \$5}"\'' % (installer_password,
-                                                   ssh_options,
-                                                   installer_username,
-                                                   installer_ip)
-            multisite_controller_ip = \
-                "".join(os.popen(cmd).read().split())
-
-            # Login to controller and get bind host details
-            cmd = 'sshpass -p %s ssh 2>/dev/null  %s %s@%s "ssh %s \\" \
-                grep -e "^bind_" %s  \\""' % (installer_password,
-                                              ssh_options,
-                                              installer_username,
-                                              installer_ip,
-                                              multisite_controller_ip,
-                                              kingbird_conf_path)
-            bind_details = os.popen(cmd).read()
-            bind_details = "".join(bind_details.split())
-            # Extract port number from the bind details
-            bind_port = re.findall(r"\D(\d{4})", bind_details)[0]
-            # Extract ip address from the bind details
-            bind_host = re.findall(r"\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}",
-                                   bind_details)[0]
-            kingbird_endpoint_url = "http://" + bind_host + ":" + bind_port + \
-                                    "/"
-        else:
-            cmd = "openstack endpoint show kingbird | grep publicurl |\
-                   awk '{print $4}' | awk -F '/' '{print $3}'"
-            kingbird_endpoint_url = os.popen(cmd).read()
-
+               awk '{print $4}'"
+        kingbird_endpoint_details = "".join(os.popen(cmd).read().split())
+        kingbird_endpoint_url = kingbird_endpoint_details.rsplit('/', 1)[0] + \
+            '/'
+        kingbird_api_version = kingbird_endpoint_details.rsplit('/', 1)[1]
         try:
             config.add_section("kingbird")
         except Exception:
@@ -353,9 +308,12 @@ def apply_tempest_blacklist():
         black_tests = []
         logger.debug("Tempest blacklist file does not exist.")
 
-    for line in cases_file:
-        if line not in black_tests:
-            result_file.write(str(line) + '\n')
+    for cases_line in cases_file:
+        for black_tests_line in black_tests:
+            if black_tests_line in cases_line:
+                break
+        else:
+            result_file.write(str(cases_line) + '\n')
     result_file.close()
 
 
@@ -427,34 +385,20 @@ def run_tempest(OPTION):
     dur_sec_int = dur_sec_int + 60 * dur_min
     stop_time = time.time()
 
-    status = "FAIL"
     try:
         diff = (int(num_tests) - int(num_failures))
         success_rate = 100 * diff / int(num_tests)
     except:
         success_rate = 0
 
-    # For Tempest we assume that the success rate is above 90%
-    if "smoke" in args.mode:
-        case_name = "tempest_smoke_serial"
-        # Note criteria hardcoded...TODO read it from testcases.yaml
-        success_criteria = 100
-        if success_rate >= success_criteria:
-            status = "PASS"
-        else:
-            logger.info("Tempest success rate: %s%%. The success criteria to "
-                        "pass this test is %s%%. Marking the test as FAILED." %
-                        (success_rate, success_criteria))
+    if 'smoke' in args.mode:
+        case_name = 'tempest_smoke_serial'
     else:
-        case_name = "tempest_full_parallel"
-        # Note criteria hardcoded...TODO read it from testcases.yaml
-        success_criteria = 80
-        if success_rate >= success_criteria:
-            status = "PASS"
-        else:
-            logger.info("Tempest success rate: %s%%. The success criteria to "
-                        "pass this test is %s%%. Marking the test as FAILED." %
-                        (success_rate, success_criteria))
+        case_name = 'tempest_full_parallel'
+
+    status = ft_utils.check_success_rate(case_name, success_rate)
+    logger.info("Tempest %s success_rate is %s%%, is marked as %s"
+                % (case_name, success_rate, status))
 
     # Push results in payload of testcase
     if args.report:

diff --git a/testcases/features/doctor.py b/testcases/features/doctor.py
index bdf3ddc..184ab03 100755 (executable)
--- a/testcases/features/doctor.py
+++ b/testcases/features/doctor.py
@@ -37,7 +37,8 @@ def main():
     cmd = 'cd %s/tests && ./run.sh' % DOCTOR_REPO
     start_time = time.time()
 
-    ret = functest_utils.execute_command(cmd, logger, exit_on_error=False)
+    ret = functest_utils.execute_command(cmd, logger, info=True,
+                                         exit_on_error=False)
 
     stop_time = time.time()
     duration = round(stop_time - start_time, 1)

diff --git a/testcases/vnf/vIMS/orchestrator.py b/testcases/vnf/vIMS/orchestrator.py
index d88efe9..3da3fb0 100644 (file)
--- a/testcases/vnf/vIMS/orchestrator.py
+++ b/testcases/vnf/vIMS/orchestrator.py
@@ -55,8 +55,6 @@ class orchestrator:
     def set_nameservers(self, nameservers):
         if 0 < len(nameservers):
             self.config['dns_subnet_1'] = nameservers[0]
-        if 1 < len(nameservers):
-            self.config['dns_subnet_2'] = nameservers[1]
 
     def set_logger(self, logger):
         self.logger = logger

diff --git a/testcases/vnf/vIMS/vIMS.py b/testcases/vnf/vIMS/vIMS.py
index 790bfa0..dfbb675 100755 (executable)
--- a/testcases/vnf/vIMS/vIMS.py
+++ b/testcases/vnf/vIMS/vIMS.py
@@ -441,9 +441,9 @@ def main():
     flavor_name = "m1.small"
     flavor_id = os_utils.get_flavor_id(nova, flavor_name)
     for requirement in CW_REQUIERMENTS:
-        if requirement == 'ram_min':
+        if requirement == 'ram_min' and flavor_id == '':
             flavor_id = os_utils.get_flavor_id_by_ram_range(
-                nova, CW_REQUIERMENTS['ram_min'], 8196)
+                nova, CW_REQUIERMENTS['ram_min'], 4500)
 
     if flavor_id == '':
         logger.error(

diff --git a/utils/functest_utils.py b/utils/functest_utils.py
index 4373152..cb2333d 100644 (file)
--- a/utils/functest_utils.py
+++ b/utils/functest_utils.py
@@ -16,10 +16,10 @@ import os
 import os.path
 import re
 import shutil
-import socket
 import subprocess
 import sys
 import urllib2
+import dns.resolver
 
 import functest.ci.tier_builder as tb
 from git import Repo
@@ -226,13 +226,17 @@ def get_resolvconf_ns():
     nameservers = []
     rconf = open("/etc/resolv.conf", "r")
     line = rconf.readline()
+    resolver = dns.resolver.Resolver()
     while line:
         ip = re.search(r"\b(?:[0-9]{1,3}\.){3}[0-9]{1,3}\b", line)
-        sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
         if ip:
-            result = sock.connect_ex((ip.group(), 53))
-            if result == 0:
-                nameservers.append(ip.group())
+            resolver.nameservers = [str(ip)]
+            try:
+                result = resolver.query('opnfv.org')[0]
+                if result != "":
+                    nameservers.append(ip.group())
+            except dns.exception.Timeout:
+                pass
         line = rconf.readline()
     return nameservers