StorPerf Daily Job

Helper script that determines what installer was used.
Script to generate RC file for container.
Script to determine number of Cinder nodes.
Daily job to pull new container, create stack based on
number of Cinder nodes, issue a warm up, and then run
a series of performance tests against the stack.  Deletes
stack once complete.

JIRA: STORPERF-87

Signed-off-by: mbeierl <mark.beierl@dell.com>
Change-Id: I551f21ffc85f11b0c0724205e952551022a94f5b

diff --git a/ci/create_stack.sh b/ci/create_stack.sh
index 152f1db..68c4874 100755 (executable)
--- a/ci/create_stack.sh
+++ b/ci/create_stack.sh
@@ -11,8 +11,8 @@
 cat << EOF > body.json
 {
   "agent_count": $1,
-  "agent_image": "Trusty x86_64",
-  "public_network": "ext-net",
+  "agent_image": "$3",
+  "public_network": "$4",
   "volume_size": $2
 }
 EOF

diff --git a/ci/daily.sh b/ci/daily.sh
index bdfe746..9ee6afb 100755 (executable)
--- a/ci/daily.sh
+++ b/ci/daily.sh
@@ -14,6 +14,21 @@ then
     WORKSPACE=`pwd`
 fi
 
+if [ -d $WORKSPACE/ci/job ]
+then
+    sudo rm -rf $WORKSPACE/ci/job
+fi
+
+virtualenv $WORKSPACE/ci/job/storperf_daily_venv
+source $WORKSPACE/ci/job/storperf_daily_venv/bin/activate
+
+pip install --upgrade setuptools
+pip install functools32
+pip install pytz
+pip install osc_lib
+pip install python-openstackclient
+pip install python-heatclient
+
 # This is set by Jenkins, but if we are running manually, just use the
 # current hostname.
 if [ -z "$NODE_NAME" ]
@@ -22,12 +37,10 @@ then
 fi
 export POD_NAME=$NODE_NAME
 
-if [ -d $WORKSPACE/ci/job ]
-then
-    sudo rm -rf $WORKSPACE/ci/job
-fi
 sudo find $WORKSPACE/ -name '*.db' -exec rm -fv {} \;
 
+export INSTALLER=`./detect_installer.sh`
+
 $WORKSPACE/ci/generate-admin-rc.sh
 $WORKSPACE/ci/generate-environment.sh
 
@@ -38,16 +51,20 @@ do
 done
 
 echo "Checking for an existing stack"
-STACK_ID=`heat stack-list | grep StorPerfAgentGroup | awk '{print $2}'`
+STACK_ID=`openstack stack list | grep StorPerfAgentGroup | awk '{print $2}'`
 if [ ! -z $STACK_ID ]
 then
-    heat stack-delete -y StorPerfAgentGroup
+    openstack stack delete --yes --wait StorPerfAgentGroup
 fi
 
-while [ ! -z $STACK_ID ]
-do
-    STACK_ID=`heat stack-list | grep StorPerfAgentGroup | awk '{print $2}'`
-done
+echo Checking for Ubuntu 14.04 image in Glance
+IMAGE=`openstack image list | grep "Trusty x86_64"`
+if [ -z $IMAGE ]
+then
+    wget https://cloud-images.ubuntu.com/releases/14.04/release/ubuntu-14.04-server-cloudimg-amd64-disk1.img
+    openstack image create "Trusty x86_64" --disk-format qcow2 --public \
+    --container-format bare --file ubuntu-14.04-server-cloudimg-amd64-disk1.img
+fi
 
 echo "TEST_DB_URL=http://testresults.opnfv.org/test/api/v1" >> $WORKSPACE/ci/job/admin.rc
 echo "INSTALLER_TYPE=${INSTALLER}" >> $WORKSPACE/ci/job/admin.rc
@@ -60,11 +77,11 @@ do
 done
 
 echo Creating 1:1 stack
-$WORKSPACE/ci/create_stack.sh $CINDER_NODES 10
+$WORKSPACE/ci/create_stack.sh $CINDER_NODES 10 "Trusty x86_64" $NETWORK
 
 export QUEUE_DEPTH=8
 export BLOCK_SIZE=16384
-export WORKLOAD=ws
+export WORKLOAD=_warm_up
 export SCENARIO_NAME="${CINDER_BACKEND}_${WORKLOAD}"
 WARM_UP=`$WORKSPACE/ci/start_job.sh | awk '/job_id/ {print $2}' | sed 's/"//g'`
 
@@ -77,7 +94,6 @@ do
     | awk '/Status/ {print $2}' | sed 's/"//g'`
 done
 
-
 for WORKLOAD in ws wr rs rr rw
 do
     for BLOCK_SIZE in 2048 8192 16384

diff --git a/ci/detect_installer.sh b/ci/detect_installer.sh
new file mode 100755 (executable)
index 0000000..035b7e2
--- /dev/null
+++ b/ci/detect_installer.sh
@@ -0,0 +1,22 @@
+#!/bin/bash -x
+##############################################################################
+# Copyright (c) 2015 EMC and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+which juju 2>/dev/null
+if [ $? -eq 0 ]
+then
+    INSTALLER=joid
+fi
+sudo virsh list --all | grep undercloud >/dev/null
+if [ $? -eq 0 ]
+then
+    INSTALLER=apex
+fi
+
+echo $INSTALLER
\ No newline at end of file

diff --git a/ci/generate-admin-rc.sh b/ci/generate-admin-rc.sh
index 6c5b162..efe8b94 100755 (executable)
--- a/ci/generate-admin-rc.sh
+++ b/ci/generate-admin-rc.sh
@@ -17,18 +17,29 @@ then
     mkdir job
 fi
 
-export OS_AUTH_URL=http://`juju status keystone | grep public | awk '{print $2}'`:5000/v2.0
-export OS_USERNAME=admin
-export OS_PASSWORD=openstack
-export OS_TENANT_ID=`openstack project list|awk '/admin/ {print $2}'`
-
-cat << EOF > job/admin.rc
-OS_AUTH_URL=$OS_AUTH_URL
-OS_USERNAME=$OS_USERNAME
-OS_PASSWORD=$OS_PASSWORD
-OS_TENANT_ID=$OS_TENANT_ID
-OS_TENANT_NAME=admin
-OS_PROJECT_NAME=admin
-OS_REGION_NAME=RegionOne
+export INSTALLER=`./detect_installer.sh`
+case $INSTALLER in
+    joid)
+        export OS_AUTH_URL=http://`juju status keystone | grep public | awk '{print $2}'`:5000/v2.0
+        export OS_USERNAME=admin
+        export OS_PASSWORD=openstack
+        cat << EOF > job/openstack.rc
+export OS_AUTH_URL=$OS_AUTH_URL
+export OS_USERNAME=$OS_USERNAME
+export OS_PASSWORD=$OS_PASSWORD
+export OS_TENANT_NAME=admin
+export OS_PROJECT_NAME=admin
 EOF
+        ;;
+    apex)
+        INSTALLER_IP=`sudo virsh domifaddr undercloud | grep ipv4 | awk '{print $4}' | cut -d/ -f1`
+        ;;
+esac
+
+if [ ! -z $INSTALLER_IP ]
+then
+    ../../releng/utils/fetch_os_creds.sh -i $INSTALLER -a $INSTALLER_IP -d job/openstack.rc
+    echo export OS_PROJECT_NAME=admin >> job/openstack.rc
+fi
 
+sed "s/export //" job/openstack.rc > job/admin.rc
\ No newline at end of file

diff --git a/ci/generate-environment.sh b/ci/generate-environment.sh
index 6b44313..c65dc17 100755 (executable)
--- a/ci/generate-environment.sh
+++ b/ci/generate-environment.sh
@@ -13,13 +13,12 @@ if [ ! -d job ]
 then
     mkdir job
 fi
+ssh_options="-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no"
 
-# TODO: This assumes JOID.  Need to make this programmatic
-
-INSTALLER=JOID
+export INSTALLER=`./detect_installer.sh`
 
 case $INSTALLER in
-    JOID)
+    joid)
         # Determine Cinder backend
         if [ ! -z "$(juju status | grep ceph)" ]
         then
@@ -31,18 +30,28 @@ case $INSTALLER in
             CINDER_BACKEND=scaleio
             JUJU_CHARM=scaleio-sds
         fi
-            # Determine how many storage blades we have
-            CINDER_NODES=`juju status | grep "$JUJU_CHARM/" | wc -l`
-            # Later, collect info about each node:
-            # juju status | grep hardware: | grep tags | grep -v virtual
+        # Determine how many storage blades we have
+        CINDER_NODES=`juju status | grep "$JUJU_CHARM/" | wc -l`
+        # Later, collect info about each node:
+        # juju status | grep hardware: | grep tags | grep -v virtual
+        NETWORK=ext-net
+        ;;
+    apex)
+        INSTALLER_IP=`sudo virsh domifaddr undercloud | grep ipv4 | awk '{print $4}' | cut -d/ -f1`
+        CINDER_BACKEND=ceph
+        sudo scp $ssh_options root@$INSTALLER_IP:/home/stack/instackenv.json job/
+        CINDER_NODES=`grep capabilities job/instackenv.json | wc -l`
+        NETWORK=external
         ;;
     *)
         CINDER_BACKEND=ceph
         CINDER_NODES=4
+        NETWORK=external
 esac
 
 cat << EOF > job/environment.rc
 export CINDER_BACKEND=$CINDER_BACKEND
 export CINDER_NODES=$CINDER_NODES
 export INSTALLER=$INSTALLER
+export NETWORK=$NETWORK
 EOF
\ No newline at end of file

diff --git a/ci/launch_docker_container.sh b/ci/launch_docker_container.sh
index 60d1a07..824fade 100755 (executable)
--- a/ci/launch_docker_container.sh
+++ b/ci/launch_docker_container.sh
@@ -1,3 +1,4 @@
+emc.)(165
 #!/bin/bash
 ##############################################################################
 # Copyright (c) 2016 EMC and others.
@@ -29,11 +30,12 @@ then
     sudo chown 33:33 job/carbon
 fi
 
+docker pull opnfv/storperf:latest
+
 docker run -d --env-file `pwd`/job/admin.rc \
     -p 5000:5000 \
     -p 8000:8000 \
     -v `pwd`/job/carbon:/opt/graphite/storage/whisper \
-    -v `pwd`/../../storperf:/home/opnfv/repos/storperf \
     --name storperf opnfv/storperf