HA testcase improvement

This patch improve HA test case in the following aspects:
1. the "GeneralHA" type now will check if the target service process in the controller node.
2. support ignore server certificate
3. add debug log for recovering service failed
4. improve method to kill keystone process

Change-Id: I9ae7ab54391fe41d5d7f3e4951a7ac2e3ba75968
Signed-off-by: JingLu5 <lvjing5@huawei.com>
(cherry picked from commit 5b99f1532ec4d15258ec86e970acd2904954b3bc)

diff --git a/tests/ci/clean_images.sh b/tests/ci/clean_images.sh
index 27da9e2..cb5410a 100755 (executable)
--- a/tests/ci/clean_images.sh
+++ b/tests/ci/clean_images.sh
@@ -15,18 +15,24 @@ cleanup()
     echo
     echo "========== Cleanup =========="
 
-    if ! openstack image list; then
+    if [ $OS_CACERT ] && [ "$(echo $OS_CACERT | tr '[:upper:]' '[:lower:]')" = "false" ]; then
+        SECURE="--insecure"
+    else
+        SECURE=""
+    fi
+
+    if ! openstack "${SECURE}" image list; then
         return
     fi
 
-    for image in $(openstack image list | grep -e cirros-0.3.5 -e yardstick-image -e Ubuntu-16.04 \
+    for image in $(openstack "${SECURE}" image list | grep -e cirros-0.3.5 -e yardstick-image -e Ubuntu-16.04 \
         | awk '{print $2}'); do
         echo "Deleting image $image..."
-        openstack image delete $image || true
+        openstack "${SECURE}" image delete $image || true
     done
 
-    openstack flavor delete yardstick-flavor &> /dev/null || true
-    openstack flavor delete storperf &> /dev/null || true
+    openstack "${SECURE}" flavor delete yardstick-flavor &> /dev/null || true
+    openstack "${SECURE}" flavor delete storperf &> /dev/null || true
 }
 
 main()

diff --git a/tests/ci/load_images.sh b/tests/ci/load_images.sh
index 0ebe9b7..a5114f4 100755 (executable)
--- a/tests/ci/load_images.sh
+++ b/tests/ci/load_images.sh
@@ -88,7 +88,7 @@ load_yardstick_image()
         if [ ! -f "${CLOUD_KERNEL}" ]; then
             tar xf "${CLOUD_IMAGE}" "${CLOUD_KERNEL##**/}"
         fi
-        create_kernel=$(openstack image create \
+        create_kernel=$(openstack "${SECURE}" image create \
                 --public \
                 --disk-format qcow2 \
                 --container-format bare \
@@ -119,7 +119,7 @@ load_yardstick_image()
     fi
 
     if [[ "$DEPLOY_SCENARIO" == *"-lxd-"* ]]; then
-        output=$(eval openstack image create \
+        output=$(eval openstack "${SECURE}" image create \
             --public \
             --disk-format raw \
             --container-format bare \
@@ -127,7 +127,7 @@ load_yardstick_image()
             --file ${RAW_IMAGE} \
             yardstick-image)
     else
-        output=$(eval openstack image create \
+        output=$(eval openstack "${SECURE}" image create \
             --public \
             --disk-format qcow2 \
             --container-format bare \
@@ -156,7 +156,7 @@ load_yardstick_image()
 
 load_cirros_image()
 {
-    if [[ -n $(openstack image list | grep -e Cirros-0.3.5) ]]; then
+    if [[ -n $(openstack "${SECURE}" image list | grep -e Cirros-0.3.5) ]]; then
         echo "Cirros-0.3.5 image already exist, skip loading cirros image"
     else
         echo
@@ -170,7 +170,7 @@ load_cirros_image()
             EXTRA_PARAMS=$EXTRA_PARAMS" --property hw_mem_page_size=large"
         fi
 
-        output=$(openstack image create \
+        output=$(openstack "${SECURE}" image create \
             --disk-format qcow2 \
             --container-format bare \
             ${EXTRA_PARAMS} \
@@ -201,7 +201,7 @@ load_ubuntu_image()
         EXTRA_PARAMS=$EXTRA_PARAMS" --property hw_mem_page_size=large"
     fi
 
-    output=$(openstack image create \
+    output=$(openstack "${SECURE}" image create \
         --disk-format qcow2 \
         --container-format bare \
         $EXTRA_PARAMS \
@@ -221,26 +221,26 @@ load_ubuntu_image()
 
 create_nova_flavor()
 {
-    if ! openstack flavor list | grep -q yardstick-flavor; then
+    if ! openstack "${SECURE}" flavor list | grep -q yardstick-flavor; then
         echo
         echo "========== Creating yardstick-flavor =========="
         # Create the nova flavor used by some sample test cases
-        openstack flavor create --id 100 --ram 1024 --disk 3 --vcpus 1 yardstick-flavor
+        openstack "${SECURE}" flavor create --id 100 --ram 1024 --disk 3 --vcpus 1 yardstick-flavor
         # DPDK-enabled OVS requires guest memory to be backed by large pages
         if [[ $DEPLOY_SCENARIO == *[_-]ovs[_-]* ]]; then
-            openstack flavor set --property hw:mem_page_size=large yardstick-flavor
+            openstack "${SECURE}" flavor set --property hw:mem_page_size=large yardstick-flavor
         fi
         # VPP requires guest memory to be backed by large pages
         if [[ "$DEPLOY_SCENARIO" == *"-fdio-"* ]]; then
-            openstack flavor set --property hw:mem_page_size=large yardstick-flavor
+            openstack "${SECURE}" flavor set --property hw:mem_page_size=large yardstick-flavor
         fi
     fi
 
-    if ! openstack flavor list | grep -q storperf; then
+    if ! openstack "${SECURE}" flavor list | grep -q storperf; then
         echo
         echo "========== Creating storperf flavor =========="
         # Create the nova flavor used by storperf test case
-        openstack flavor create --id auto --ram 8192 --disk 4 --vcpus 2 storperf
+        openstack "${SECURE}" flavor create --id auto --ram 8192 --disk 4 --vcpus 2 storperf
     fi
 }
 
@@ -256,6 +256,12 @@ main()
         RAW_IMAGE='/home/opnfv/images/yardstick-image.tar.gz'
     fi
 
+    if [ $OS_CACERT ] && [ "$(echo $OS_CACERT | tr '[:upper:]' '[:lower:]')" = "false" ]; then
+        SECURE="--insecure"
+    else
+        SECURE=""
+    fi
+
     build_yardstick_image
     load_yardstick_image
     if [ "${YARD_IMG_ARCH}" == "arm64" ]; then

diff --git a/tests/unit/benchmark/scenarios/availability/test_scenario_general.py b/tests/unit/benchmark/scenarios/availability/test_scenario_general.py
index ea54fbb..de2170b 100644 (file)
--- a/tests/unit/benchmark/scenarios/availability/test_scenario_general.py
+++ b/tests/unit/benchmark/scenarios/availability/test_scenario_general.py
@@ -56,6 +56,7 @@ class ScenarioGeneralTestCase(unittest.TestCase):
         mock_obj = mock.Mock()
         mock_obj.createActionPlayer.side_effect = KeyError('Wrong')
         ins.director = mock_obj
+        ins.director.data = {}
         ins.run({})
         ins.teardown()
 
@@ -64,5 +65,6 @@ class ScenarioGeneralTestCase(unittest.TestCase):
         mock_obj = mock.Mock()
         mock_obj.verify.return_value = False
         ins.director = mock_obj
+        ins.director.data = {}
         ins.run({})
         ins.teardown()

diff --git a/yardstick/benchmark/scenarios/availability/attacker/attacker_process.py b/yardstick/benchmark/scenarios/availability/attacker/attacker_process.py
index e0e6cf3..f7ab23d 100644 (file)
--- a/yardstick/benchmark/scenarios/availability/attacker/attacker_process.py
+++ b/yardstick/benchmark/scenarios/availability/attacker/attacker_process.py
@@ -66,3 +66,5 @@ class ProcessAttacker(BaseAttacker):
             exit_status, stdout, stderr = self.connection.execute(
                 "sudo /bin/bash -s {0} ".format(self.service_name),
                 stdin=stdin_file)
+        if exit_status:
+            LOG.info("Fail to restart service!")

diff --git a/yardstick/benchmark/scenarios/availability/ha_tools/fault_process_kill.bash b/yardstick/benchmark/scenarios/availability/ha_tools/fault_process_kill.bash
index a6a3e96..a865b65 100755 (executable)
--- a/yardstick/benchmark/scenarios/availability/ha_tools/fault_process_kill.bash
+++ b/yardstick/benchmark/scenarios/availability/ha_tools/fault_process_kill.bash
@@ -16,10 +16,13 @@ set -e
 process_name=$1
 
 if [ "$process_name" = "keystone" ]; then
-    killall -9 -u $process_name
+    for pid in $(ps aux | grep "keystone" | grep -iv heartbeat | grep -iv monitor | grep -v grep | grep -v /bin/sh | awk '{print $2}'); \
+        do
+            kill -9 "${pid}"
+        done
 else
-    for pid in `ps aux | grep "/usr/.*/${process_name}" | grep -v grep | grep -v /bin/sh | awk '{print $2}'`; \
+    for pid in $(pgrep -f "/usr/.*/${process_name}");
         do
-            kill -9 ${pid}
+            kill -9 "${pid}"
         done
 fi

diff --git a/yardstick/benchmark/scenarios/availability/ha_tools/nova/create_flavor.bash b/yardstick/benchmark/scenarios/availability/ha_tools/nova/create_flavor.bash
index 941563e..8737836 100644 (file)
--- a/yardstick/benchmark/scenarios/availability/ha_tools/nova/create_flavor.bash
+++ b/yardstick/benchmark/scenarios/availability/ha_tools/nova/create_flavor.bash
@@ -14,4 +14,10 @@
 
 set -e
 
-openstack flavor create $1 --id $2 --ram $3 --disk $4 --vcpus $5
+if [ $OS_CACERT ] && [ "$(echo $OS_CACERT | tr '[:upper:]' '[:lower:]')" = "false" ]; then
+    SECURE="--insecure"
+else
+    SECURE=""
+fi
+
+openstack "${SECURE}" flavor create $1 --id $2 --ram $3 --disk $4 --vcpus $5

diff --git a/yardstick/benchmark/scenarios/availability/ha_tools/nova/delete_flavor.bash b/yardstick/benchmark/scenarios/availability/ha_tools/nova/delete_flavor.bash
index e998464..617dcf8 100644 (file)
--- a/yardstick/benchmark/scenarios/availability/ha_tools/nova/delete_flavor.bash
+++ b/yardstick/benchmark/scenarios/availability/ha_tools/nova/delete_flavor.bash
@@ -14,4 +14,10 @@
 
 set -e
 
-openstack flavor delete $1
+if [ $OS_CACERT ] && [ "$(echo $OS_CACERT | tr '[:upper:]' '[:lower:]')" = "false" ]; then
+    SECURE="--insecure"
+else
+    SECURE=""
+fi
+
+openstack "${SECURE}" flavor delete $1

diff --git a/yardstick/benchmark/scenarios/availability/ha_tools/nova/show_flavors.bash b/yardstick/benchmark/scenarios/availability/ha_tools/nova/show_flavors.bash
index 1b07396..9b413c9 100644 (file)
--- a/yardstick/benchmark/scenarios/availability/ha_tools/nova/show_flavors.bash
+++ b/yardstick/benchmark/scenarios/availability/ha_tools/nova/show_flavors.bash
@@ -13,4 +13,10 @@
 
 set -e
 
-nova flavor-list
\ No newline at end of file
+if [ $OS_CACERT ] && [ "$(echo $OS_CACERT | tr '[:upper:]' '[:lower:]')" = "false" ]; then
+    SECURE="--insecure"
+else
+    SECURE=""
+fi
+
+openstack "${SECURE}" flavor list

diff --git a/yardstick/benchmark/scenarios/availability/monitor/monitor_command.py b/yardstick/benchmark/scenarios/availability/monitor/monitor_command.py
index 033a2d7..d757bd8 100644 (file)
--- a/yardstick/benchmark/scenarios/availability/monitor/monitor_command.py
+++ b/yardstick/benchmark/scenarios/availability/monitor/monitor_command.py
@@ -7,6 +7,8 @@
 # http://www.apache.org/licenses/LICENSE-2.0
 ##############################################################################
 from __future__ import absolute_import
+
+import os
 import logging
 import subprocess
 import traceback
@@ -53,6 +55,14 @@ class MonitorOpenstackCmd(basemonitor.BaseMonitor):
 
         self.cmd = self._config["command_name"]
 
+        try:
+            cacert = os.environ['OS_CACERT']
+        except KeyError:
+            pass
+        else:
+            if cacert.lower() == "false":
+                self.cmd = self.cmd + " --insecure"
+
     def monitor_func(self):
         exit_status = 0
         exit_status, stdout = _execute_shell_command(self.cmd)

diff --git a/yardstick/benchmark/scenarios/availability/scenario_general.py b/yardstick/benchmark/scenarios/availability/scenario_general.py
index 689d33a..28bec8a 100644 (file)
--- a/yardstick/benchmark/scenarios/availability/scenario_general.py
+++ b/yardstick/benchmark/scenarios/availability/scenario_general.py
@@ -54,7 +54,18 @@ class ScenarioGeneral(base.Scenario):
                 pass
 
         self.director.stopMonitors()
-        if self.director.verify():
+
+        verify_result = self.director.verify()
+
+        for k, v in self.director.data.items():
+            if v == 0:
+                result['sla_pass'] = 0
+                verify_result = False
+                LOG.info(
+                    "\033[92m The service process not found in the host \
+envrioment, the HA test case NOT pass")
+
+        if verify_result:
             result['sla_pass'] = 1
             LOG.info(
                 "\033[92m Congratulations, "

diff --git a/yardstick/benchmark/scenarios/availability/serviceha.py b/yardstick/benchmark/scenarios/availability/serviceha.py
index 2e82971..2f0012e 100755 (executable)
--- a/yardstick/benchmark/scenarios/availability/serviceha.py
+++ b/yardstick/benchmark/scenarios/availability/serviceha.py
@@ -71,7 +71,7 @@ class ServiceHA(base.Scenario):
 
         sla_pass = self.monitorMgr.verify_SLA()
         for k, v in self.data.items():
-            if self.data[k] == 0:
+            if v == 0:
                 result['sla_pass'] = 0
                 LOG.info("The service process not found in the host envrioment, \
 the HA test case NOT pass")