Apex,CSIT: Fixes slaves and other bugs

Changes Include:
 - Fixes phase jobs to run on the same node
 - Removes using WORKSPACE/.tmp dir for unpacking snapshot since other
   phase jobs will need access to these files.  Now just unpacks to cache
   directory
 - Fixes download for ODL.  The previous way ODL was being downloaded
   was only if a manual patch-set job was run on the ODL gerrit.  This
   patch changes the download to use the zip artifact automatically
   produced.
 - The zip artifact is converted to tar.gz and stored in /tmp/ for the
   install job to consume. Note: the conversion is done because the
   odl-pipeline install scripts only currently accept tar.gz.
 - Extra checks in install job to make sure required files exist and
   have been previously unpacked by deploy job.

Change-Id: I38aad98cac247eb4af2839811388b119079bc7ba
Signed-off-by: Tim Rozet <trozet@redhat.com>

diff --git a/jjb/3rd_party_ci/download-netvirt-artifact.sh b/jjb/3rd_party_ci/download-netvirt-artifact.sh
index fe8066c..88456e9 100755 (executable)
--- a/jjb/3rd_party_ci/download-netvirt-artifact.sh
+++ b/jjb/3rd_party_ci/download-netvirt-artifact.sh
@@ -6,17 +6,23 @@ set -o pipefail
 echo "Attempting to fetch the artifact location from ODL Jenkins"
 CHANGE_DETAILS_URL="https://git.opendaylight.org/gerrit/changes/netvirt~master~$GERRIT_CHANGE_ID/detail"
 # due to limitation with the Jenkins Gerrit Trigger, we need to use Gerrit REST API to get the change details
 echo "Attempting to fetch the artifact location from ODL Jenkins"
 CHANGE_DETAILS_URL="https://git.opendaylight.org/gerrit/changes/netvirt~master~$GERRIT_CHANGE_ID/detail"
 # due to limitation with the Jenkins Gerrit Trigger, we need to use Gerrit REST API to get the change details

-ODL_JOB_URL=$(curl -s $CHANGE_DETAILS_URL | grep netvirt-patch-test-current-carbon | tail -1 | \
-    sed 's/\\n//g' | awk '{print $6}')
-NETVIRT_ARTIFACT_URL="${ODL_JOB_URL}org.opendaylight.integration\$distribution-karaf/artifact/org.opendaylight.integration/distribution-karaf/0.6.0-SNAPSHOT/distribution-karaf-0.6.0-SNAPSHOT.tar.gz"
+ODL_BUILD_JOB_NUM=$(curl -s $CHANGE_DETAILS_URL | grep -Eo 'netvirt-distribution-check-carbon/[0-9]+' | tail -1 | grep -Eo [0-9]+)
+
+NETVIRT_ARTIFACT_URL="https://jenkins.opendaylight.org/releng/job/netvirt-distribution-check-carbon/${ODL_BUILD_JOB_NUM}/artifact/distribution-karaf-0.6.0-SNAPSHOT.zip"

 echo -e "URL to artifact is\n\t$NETVIRT_ARTIFACT_URL"
 
 echo "Downloading the artifact. This could take time..."
 echo -e "URL to artifact is\n\t$NETVIRT_ARTIFACT_URL"
 
 echo "Downloading the artifact. This could take time..."

-wget -q -O $NETVIRT_ARTIFACT $NETVIRT_ARTIFACT_URL
+wget -q -O distribution-karaf-0.6.0-SNAPSHOT.zip $NETVIRT_ARTIFACT_URL

 if [[ $? -ne 0 ]]; then
     echo "The artifact does not exist! Probably removed due to ODL Jenkins artifact retention policy."
     echo "Rerun netvirt-patch-test-current-carbon to get artifact rebuilt."
     exit 1
 fi
 if [[ $? -ne 0 ]]; then
     echo "The artifact does not exist! Probably removed due to ODL Jenkins artifact retention policy."
     echo "Rerun netvirt-patch-test-current-carbon to get artifact rebuilt."
     exit 1
 fi

+
+#TODO(trozet) remove this once odl-pipeline accepts zip files
+echo "Converting artifact zip to tar.gz"
+sudo pip3 install ruamel.zip2tar
+zip2tar distribution-karaf-0.6.0-SNAPSHOT.zip --gz --tar-file-name /tmp/${NETVIRT_ARTIFACT}
+

 echo "Download complete"
 echo "Download complete"

-ls -al $NETVIRT_ARTIFACT
+ls -al /tmp/${NETVIRT_ARTIFACT}

diff --git a/jjb/3rd_party_ci/install-netvirt.sh b/jjb/3rd_party_ci/install-netvirt.sh
index ce2a50c..8658019 100755 (executable)
--- a/jjb/3rd_party_ci/install-netvirt.sh
+++ b/jjb/3rd_party_ci/install-netvirt.sh
@@ -7,8 +7,18 @@ SNAP_CACHE=$HOME/snap_cache
 # clone opnfv sdnvpn repo
 git clone https://gerrit.opnfv.org/gerrit/p/sdnvpn.git $WORKSPACE/sdnvpn
 
 # clone opnfv sdnvpn repo
 git clone https://gerrit.opnfv.org/gerrit/p/sdnvpn.git $WORKSPACE/sdnvpn
 

-if [ ! -f "$NETVIRT_ARTIFACT" ]; then
-  echo "ERROR: ${NETVIRT_ARTIFACT} specified as NetVirt Artifact, but file does not exist"
+if [ ! -f "/tmp/${NETVIRT_ARTIFACT}" ]; then
+  echo "ERROR: /tmp/${NETVIRT_ARTIFACT} specified as NetVirt Artifact, but file does not exist"
+  exit 1
+fi
+
+if [ ! -f "${SNAP_CACHE}/node.yaml" ]; then
+  echo "ERROR: node.yaml pod config missing in ${SNAP_CACHE}"
+  exit 1
+fi
+
+if [ ! -f "${SNAP_CACHE}/id_rsa" ]; then
+  echo "ERROR: id_rsa ssh creds missing in ${SNAP_CACHE}"

   exit 1
 fi
 
   exit 1
 fi
 


@@ -17,5 +27,5 @@ fi
 # for when we support multiple jobs on a single slave
 pushd sdnvpn/odl-pipeline/lib > /dev/null
 ./odl_reinstaller.sh --pod-config ${SNAP_CACHE}/node.yaml \
 # for when we support multiple jobs on a single slave
 pushd sdnvpn/odl-pipeline/lib > /dev/null
 ./odl_reinstaller.sh --pod-config ${SNAP_CACHE}/node.yaml \

-  --odl-artifact ${NETVIRT_ARTIFACT} --ssh-key-file ${SNAP_CACHE}/id_rsa
+  --odl-artifact /tmp/${NETVIRT_ARTIFACT} --ssh-key-file ${SNAP_CACHE}/id_rsa

 popd > /dev/null
 popd > /dev/null

diff --git a/jjb/3rd_party_ci/odl-netvirt.yml b/jjb/3rd_party_ci/odl-netvirt.yml
index f3a4c02..bf2ba6c 100644 (file)
--- a/jjb/3rd_party_ci/odl-netvirt.yml
+++ b/jjb/3rd_party_ci/odl-netvirt.yml
@@ -63,7 +63,7 @@
         - string:
             name: NETVIRT_ARTIFACT
             default: $WORKSPACE/distribution-karaf.tar.gz
         - string:
             name: NETVIRT_ARTIFACT
             default: $WORKSPACE/distribution-karaf.tar.gz

-        - 'odl-netvirt-virtual-defaults'
+        - 'odl-netvirt-virtual-intel-defaults'

 
     triggers:
         - gerrit:
 
     triggers:
         - gerrit:


@@ -101,7 +101,7 @@
                     GERRIT_PATCHSET_REVISION=$GERRIT_PATCHSET_REVISION
                     NETVIRT_ARTIFACT=$NETVIRT_ARTIFACT
                     APEX_ENV_NUMBER=$APEX_ENV_NUMBER
                     GERRIT_PATCHSET_REVISION=$GERRIT_PATCHSET_REVISION
                     NETVIRT_ARTIFACT=$NETVIRT_ARTIFACT
                     APEX_ENV_NUMBER=$APEX_ENV_NUMBER

-                  node-parameters: false
+                  node-parameters: true

                   kill-phase-on: FAILURE
                   abort-all-job: true
         - multijob:
                   kill-phase-on: FAILURE
                   abort-all-job: true
         - multijob:

diff --git a/jjb/apex/apex-snapshot-deploy.sh b/jjb/apex/apex-snapshot-deploy.sh
index b77755f..773edd2 100644 (file)
--- a/jjb/apex/apex-snapshot-deploy.sh
+++ b/jjb/apex/apex-snapshot-deploy.sh
@@ -69,10 +69,9 @@ fi
 
 echo "INFO: Snapshot to be used is ${snap_tar}"
 
 
 echo "INFO: Snapshot to be used is ${snap_tar}"
 

-# create tmp directory and unpack snap
-mkdir -p ./tmp
-pushd ./tmp > /dev/null
-tar xvf ${SNAP_CACHE}/${snap_tar} -C ./
+# move to snap cache dir and unpack
+pushd ${SNAP_CACHE} > /dev/null
+tar xvf ${snap_tar}

 
 # create each network
 virsh_networks=$(ls *.xml | grep -v baremetal)
 
 # create each network
 virsh_networks=$(ls *.xml | grep -v baremetal)