Merge "Initial Release files from Euphrates Release"

diff --git a/jjb/dovetail/dovetail-run.sh b/jjb/dovetail/dovetail-run.sh
index a5a95f4..b8c4324 100755 (executable)
--- a/jjb/dovetail/dovetail-run.sh
+++ b/jjb/dovetail/dovetail-run.sh
@@ -154,6 +154,13 @@ if [[ ! -f ${DOVETAIL_CONFIG}/pod.yaml ]]; then
 fi
 
 if [ -f ${DOVETAIL_CONFIG}/pod.yaml ]; then
+    echo "Adapt process info for $INSTALLER_TYPE ..."
+    attack_process='rabbitmq'
+    cat << EOF >> ${DOVETAIL_CONFIG}/pod.yaml
+process_info:
+- {testcase_name: dovetail.ha.tc010, attack_process: ${attack_process}}
+
+EOF
     echo "file ${DOVETAIL_CONFIG}/pod.yaml:"
     cat ${DOVETAIL_CONFIG}/pod.yaml
 else

diff --git a/jjb/functest/functest-alpine.sh b/jjb/functest/functest-alpine.sh
index 788a4cd..bdd79c4 100755 (executable)
--- a/jjb/functest/functest-alpine.sh
+++ b/jjb/functest/functest-alpine.sh
@@ -161,7 +161,19 @@ if [ "${INSTALLER_TYPE}" == 'fuel' ]; then
     envs="${envs} -e POD_ARCH=${COMPUTE_ARCH}"
 fi
 
-volumes="${images_vol} ${results_vol} ${sshkey_vol} ${rc_file_vol} ${cacert_file_vol}"
+
+if [[ ${INSTALLER_TYPE} == 'compass' && ${DEPLOY_SCENARIO} =~ 'odl.*sfc' ]]; then
+    ssh_key="/tmp/id_rsa"
+    user_config="/tmp/openstack_user_config.yml"
+    docker cp compass-tasks:/root/.ssh/id_rsa $ssh_key
+    docker cp compass-tasks:/etc/openstack_deploy/openstack_user_config.yml $user_config
+    sshkey_vol="-v ${ssh_key}:/root/.ssh/id_rsa"
+    userconfig_vol="-v ${user_config}:${user_config}"
+    envs="${envs} -e EXTERNAL_NETWORK=${EXTERNAL_NETWORK}"
+fi
+
+
+volumes="${images_vol} ${results_vol} ${sshkey_vol} ${userconfig_vol} ${rc_file_vol} ${cacert_file_vol}"
 
 set +e
 

diff --git a/jjb/functest/functest-k8.sh b/jjb/functest/functest-k8.sh
index 115ff42..49d7163 100755 (executable)
--- a/jjb/functest/functest-k8.sh
+++ b/jjb/functest/functest-k8.sh
@@ -19,7 +19,7 @@ if [[ ${INSTALLER_TYPE} == 'compass' ]]; then
     KUBE_MASTER_IP=$(echo $KUBE_MASTER_URL|awk -F'https://|:[0-9]+' '$0=$2')
     echo "export KUBE_MASTER_IP=$KUBE_MASTER_IP" >> $rc_file
 elif [[ ${INSTALLER_TYPE} == 'joid' && ${BRANCH} == 'master' ]]; then
-    admin_conf_file_vol="-v ${HOME}/joid_config/config:/root/joid_config/config"
+    admin_conf_file_vol="-v ${HOME}/joid_config/config:/root/.kube/config"
     rc_file=${HOME}/joid_config/k8config
 else
     echo "Not supported by other installers yet"
@@ -35,11 +35,6 @@ results_vol="-v ${dir_result}:${FUNCTEST_DIR}/results"
 
 volumes="${rc_file_vol} ${results_vol} ${admin_conf_file_vol}"
 
-# Set iptables rule to allow forwarding return traffic for container
-if ! sudo iptables -C FORWARD -j RETURN 2> ${redirect} || ! sudo iptables -L FORWARD | awk 'NR==3' | grep RETURN 2> ${redirect}; then
-    sudo iptables -I FORWARD -j RETURN
-fi
-
 envs="-e INSTALLER_TYPE=${INSTALLER_TYPE} \
     -e NODE_NAME=${NODE_NAME} -e DEPLOY_SCENARIO=${DEPLOY_SCENARIO} \
     -e BUILD_TAG=${BUILD_TAG} -e DEPLOY_TYPE=${DEPLOY_TYPE}"
@@ -52,10 +47,13 @@ FUNCTEST_IMAGE=opnfv/functest-kubernetes:${DOCKER_TAG}
 echo "Pulling Docker image ${FUNCTEST_IMAGE} ..."
 docker pull ${FUNCTEST_IMAGE}>/dev/null
 cmd_opt="run_tests -r -t all"
-cmd="docker run --rm --privileged=true ${envs} ${volumes} ${FUNCTEST_IMAGE} /bin/bash -c '${cmd_opt}'"
+cmd="docker run --rm ${envs} ${volumes} ${FUNCTEST_IMAGE} /bin/bash -c '${cmd_opt}'"
 echo "Running Functest k8s test cases, CMD: ${cmd}"
 eval ${cmd}
 ret_value=$?
+
+ret_val_file="${HOME}/opnfv/functest/results/${BRANCH##*/}/return_value"
+echo 0 > ${ret_val_file}
 if [ ${ret_value} != 0 ]; then
   echo ${ret_value} > ${ret_val_file}
 fi

diff --git a/jjb/xci/xci-merge-jobs.yml b/jjb/xci/xci-merge-jobs.yml
index 8a3a7cf..a986aa5 100644 (file)
--- a/jjb/xci/xci-merge-jobs.yml
+++ b/jjb/xci/xci-merge-jobs.yml
@@ -20,17 +20,6 @@
       - opensuse:
           disabled: false
     # -------------------------------
-    # scenarios
-    # -------------------------------
-    scenario:
-      - 'os-odl-sfc':
-          project: sfc
-    # -------------------------------
-    # type
-    # -------------------------------
-    type:
-      - virtual
-    # -------------------------------
     # postmerge promotion phases
     # -------------------------------
     phase:
@@ -40,19 +29,19 @@
     # jobs
     # -------------------------------
     jobs:
-      - 'xci-{scenario}-{type}-merge-{stream}'
-      - 'xci-{scenario}-{type}-promote-merge-{stream}'
-      - 'xci-merge-{distro}-{type}-{stream}'
-      - 'xci-merge-{distro}-{phase}-{type}-{stream}'
+      - 'xci-merge-virtual-{stream}'
+      - 'xci-merge-promote-virtual-{stream}'
+      - 'xci-merge-{distro}-virtual-{stream}'
+      - 'xci-merge-{distro}-{phase}-virtual-{stream}'
 # -------------------------------
 # job templates
 # -------------------------------
 - job-template:
-    name: 'xci-{scenario}-{type}-merge-{stream}'
+    name: 'xci-merge-virtual-{stream}'
 
     project-type: multijob
 
-    disabled: '{obj:disabled}'
+    disabled: false
 
     concurrent: false
 
@@ -76,8 +65,8 @@
             - comment-added-contains-event:
                 comment-contains-value: 'remerge'
           projects:
-            - project-compare-type: 'ANT'
-              project-pattern: '{obj:project}'
+            - project-compare-type: 'REG_EXP'
+              project-pattern: 'sfc|sdnvpn'
               branches:
                 - branch-compare-type: 'ANT'
                   branch-pattern: '**/{branch}'
@@ -107,14 +96,16 @@
           description: 'Git URL to use on this Jenkins Slave'
 
     builders:
-      - description-setter:
-          description: "Node: $NODE_NAME"
       - 'xci-merge-set-scenario-macro'
+      - inject:
+          properties-file: "/tmp/$GERRIT_CHANGE_NUMBER/$DISTRO/scenario.properties"
+      - description-setter:
+          description: "Scenario: $DEPLOY_SCENARIO | Node: $NODE_NAME"
       - multijob:
           name: deploy and test
           condition: SUCCESSFUL
           projects:
-            - name: 'xci-merge-opensuse-{type}-{stream}'
+            - name: 'xci-merge-opensuse-virtual-{stream}'
               current-parameters: false
               predefined-parameters: |
                 DISTRO=opensuse
@@ -127,7 +118,7 @@
               node-parameters: false
               kill-phase-on: FAILURE
               abort-all-job: true
-            - name: 'xci-merge-ubuntu-{type}-{stream}'
+            - name: 'xci-merge-ubuntu-virtual-{stream}'
               current-parameters: false
               predefined-parameters: |
                 DISTRO=ubuntu
@@ -140,7 +131,7 @@
               node-parameters: false
               kill-phase-on: FAILURE
               abort-all-job: true
-            - name: 'xci-merge-centos-{type}-{stream}'
+            - name: 'xci-merge-centos-virtual-{stream}'
               current-parameters: false
               predefined-parameters: |
                 DISTRO=centos
@@ -159,7 +150,7 @@
           name: promote
           condition: SUCCESSFUL
           projects:
-            - name: 'xci-{scenario}-{type}-promote-merge-{stream}'
+            - name: 'xci-merge-promote-virtual-{stream}'
               current-parameters: true
               predefined-parameters: |
                 GERRIT_PROJECT=$GERRIT_PROJECT
@@ -172,13 +163,13 @@
               abort-all-job: true
 
 - job-template:
-    name: 'xci-merge-{distro}-{type}-{stream}'
+    name: 'xci-merge-{distro}-virtual-{stream}'
 
     project-type: multijob
 
-    disabled: '{obj:disabled}'
+    disabled: false
 
-    concurrent: true
+    concurrent: false
 
     properties:
       - logrotate-default
@@ -235,7 +226,7 @@
           name: deploy
           condition: SUCCESSFUL
           projects:
-            - name: 'xci-merge-{distro}-deploy-{type}-{stream}'
+            - name: 'xci-merge-{distro}-deploy-virtual-{stream}'
               current-parameters: true
               predefined-parameters: |
                 DISTRO={distro}
@@ -252,7 +243,7 @@
           name: healthcheck
           condition: SUCCESSFUL
           projects:
-            - name: 'xci-merge-{distro}-healthcheck-{type}-{stream}'
+            - name: 'xci-merge-{distro}-healthcheck-virtual-{stream}'
               current-parameters: true
               predefined-parameters: |
                 DISTRO={distro}
@@ -277,11 +268,11 @@
                 !include-raw: ./xci-cleanup.sh
 
 - job-template:
-    name: 'xci-merge-{distro}-{phase}-{type}-{stream}'
+    name: 'xci-merge-{distro}-{phase}-virtual-{stream}'
 
     disabled: false
 
-    concurrent: true
+    concurrent: false
 
     properties:
       - logrotate-default
@@ -338,7 +329,7 @@
       - 'xci-merge-{phase}-macro'
 
 - job-template:
-    name: 'xci-{scenario}-{type}-promote-merge-{stream}'
+    name: 'xci-merge-promote-virtual-{stream}'
 
     disabled: false
 
@@ -357,6 +348,12 @@
       - string:
           name: OPNFV_RELENG_DEV_PATH
           default: $WORKSPACE/
+      - string:
+          name: LOCAL_PROMOTION_METADATA_FILE
+          default: "/tmp/$GERRIT_CHANGE_NUMBER/$DISTRO/scenario.properties"
+      - string:
+          name: REMOTE_PROMOTION_METADATA_FILE
+          default: "gs://artifacts.opnfv.org/xci/pipeline/merge/$DEPLOY_SCENARIO.properties"
       - string:
           name: GIT_BASE
           default: https://gerrit.opnfv.org/gerrit/$PROJECT
@@ -387,20 +384,20 @@
 - builder:
     name: 'xci-merge-deploy-macro'
     builders:
-      - shell: |
-          #!/bin/bash
-          echo "Hello World"
+      - shell:
+          !include-raw: ./xci-start-new-vm.sh
+
+      - shell:
+          !include-raw: ./xci-start-deployment.sh
 
 - builder:
     name: 'xci-merge-healthcheck-macro'
     builders:
-      - shell: |
-          #!/bin/bash
-          echo "Hello World"
+      - shell:
+          !include-raw: ./xci-run-functest.sh
 
 - builder:
     name: 'xci-merge-promote-macro'
     builders:
-      - shell: |
-          #!/bin/bash
-          echo "Hello World"
+      - shell:
+          !include-raw: ./xci-promote.sh

diff --git a/jjb/xci/xci-promote.sh b/jjb/xci/xci-promote.sh
index 62818ab..fb7e0e6 100755 (executable)
--- a/jjb/xci/xci-promote.sh
+++ b/jjb/xci/xci-promote.sh
@@ -23,11 +23,24 @@ if [[ "$GERRIT_TOPIC" =~ skip-verify|skip-deployment ]]; then
     exit 0
 fi
 
-# skip the deployment if the scenario is not supported on this distro
-OPNFV_SCENARIO_REQUIREMENTS=$WORKSPACE/xci/opnfv-scenario-requirements.yml
-if ! sed -n "/^- scenario: $DEPLOY_SCENARIO$/,/^$/p" $OPNFV_SCENARIO_REQUIREMENTS | grep -q $DISTRO; then
-    echo "# SKIPPED: Scenario $DEPLOY_SCENARIO is NOT supported on $DISTRO"
-    exit 0
+# fail if promotion metadata file doesn't exist
+if [ ! -f $LOCAL_PROMOTION_METADATA_FILE ]; then
+    echo "Unable to find promotion metadata file $LOCAL_PROMOTION_METADATA_FILE"
+    echo "Skipping promotion!"
+    exit 1
 fi
 
-echo "Hello World!"
+# upload promotion metadata file to OPNFV artifact repo
+echo "Storing promotion metadata as $REMOTE_PROMOTION_METADATA_FILE"
+gsutil cp $LOCAL_PROMOTION_METADATA_FILE $REMOTE_PROMOTION_METADATA_FILE > /dev/null 2>&1
+
+# update the file metadata on gs to prevent the use of cached version of the file
+gsutil -m setmeta -r -h "Cache-Control:private, max-age=0, no-transform" \
+    $REMOTE_PROMOTION_METADATA_FILE > /dev/null 2>&1
+
+# log the metadata to console
+echo "Stored the metadata for $DEPLOY_SCENARIO"
+echo "---------------------------------------------------------------------------------"
+gsutil cat $REMOTE_PROMOTION_METADATA_FILE
+echo "---------------------------------------------------------------------------------"
+echo "Scenario $DEPLOY_SCENARIO has successfully been promoted!"

diff --git a/jjb/xci/xci-run-functest.sh b/jjb/xci/xci-run-functest.sh
index 60b48cf..2e34363 100755 (executable)
--- a/jjb/xci/xci-run-functest.sh
+++ b/jjb/xci/xci-run-functest.sh
@@ -7,9 +7,7 @@
 # which accompanies this distribution, and is available at
 # http://www.apache.org/licenses/LICENSE-2.0
 ##############################################################################
-set -o errexit
 set -o nounset
-set -o pipefail
 
 #----------------------------------------------------------------------
 # This script is used by CI and executed by Jenkins jobs.
@@ -58,3 +56,8 @@ echo "Functest log"
 echo "---------------------------------------------------------------------------------"
 ssh -F $HOME/.ssh/${DISTRO}-xci-vm-config ${DISTRO}_xci_vm_opnfv "cat /root/results/functest.log"
 echo "---------------------------------------------------------------------------------"
+# check the log to see if we have any error
+if ssh -F $HOME/.ssh/${DISTRO}-xci-vm-config ${DISTRO}_xci_vm_opnfv "grep -q 'FAIL' /root/results/functest.log"; then
+    echo "Error: Functest failed!"
+    exit 1
+fi

diff --git a/jjb/xci/xci-set-scenario.sh b/jjb/xci/xci-set-scenario.sh
index c602957..c3012cd 100755 (executable)
--- a/jjb/xci/xci-set-scenario.sh
+++ b/jjb/xci/xci-set-scenario.sh
@@ -42,20 +42,22 @@ set -o pipefail
 function override_generic_scenario() {
     echo "Processing $GERRIT_PROJECT patchset $GERRIT_REFSPEC"
 
+    # ensure the metadata we record is consistent for all types of patches including skipped ones
+    # extract releng-xci sha
+    XCI_SHA=$(cd $WORKSPACE && git rev-parse HEAD)
+
+    # extract scenario sha which is same as releng-xci sha for generic scenarios
+    SCENARIO_SHA=$XCI_SHA
+
     # process topic branch names
-    if [[ "$GERRIT_TOPIC" =~ skip-verify|skip-deployment ]]; then
-        # skip the real verification
-        echo "Skipping verify!"
-        echo "INSTALLER_TYPE=osa" > $WORK_DIRECTORY/scenario.properties
-        echo "DEPLOY_SCENARIO=os-nosdn-nofeature" >> $WORK_DIRECTORY/scenario.properties
-        exit 0
-    elif [[ "$GERRIT_TOPIC" =~ 'force-verify' ]]; then
-        # Run the deployment with default installer and scenario when multiple things change
-        # and we want to force that.
-        echo "Recording the installer 'osa' and scenario 'os-nosdn-nofeature' for downstream jobs"
-        echo "Forcing CI verification of default scenario and installer!"
+    if [[ "$GERRIT_TOPIC" =~ skip-verify|skip-deployment|force-verify ]]; then
+        [[ "$GERRIT_TOPIC" =~ force-verify ]] && echo "Forcing CI verification using default scenario and installer!"
+        [[ "$GERRIT_TOPIC" =~ skip-verify|skip-deployment ]] && echo "Skipping verification!"
         echo "INSTALLER_TYPE=osa" > $WORK_DIRECTORY/scenario.properties
         echo "DEPLOY_SCENARIO=os-nosdn-nofeature" >> $WORK_DIRECTORY/scenario.properties
+        echo "XCI_SHA=$XCI_SHA" >> $WORK_DIRECTORY/scenario.properties
+        echo "SCENARIO_SHA=$SCENARIO_SHA" >> $WORK_DIRECTORY/scenario.properties
+        echo "PROJECT_NAME=$GERRIT_PROJECT" >> $WORK_DIRECTORY/scenario.properties
         exit 0
     fi
 
@@ -70,6 +72,9 @@ function override_generic_scenario() {
             echo "Recording the installer '$INSTALLER_TYPE' and scenario '$DEPLOY_SCENARIO' for downstream jobs"
             echo "INSTALLER_TYPE=$INSTALLER_TYPE" > $WORK_DIRECTORY/scenario.properties
             echo "DEPLOY_SCENARIO=$DEPLOY_SCENARIO" >> $WORK_DIRECTORY/scenario.properties
+            echo "XCI_SHA=$XCI_SHA" >> $WORK_DIRECTORY/scenario.properties
+            echo "SCENARIO_SHA=$SCENARIO_SHA" >> $WORK_DIRECTORY/scenario.properties
+            echo "PROJECT_NAME=$GERRIT_PROJECT" >> $WORK_DIRECTORY/scenario.properties
             exit 0
         fi
     else
@@ -110,10 +115,10 @@ function determine_generic_scenario() {
     done
 
     # extract releng-xci sha
-    RELENG_XCI_SHA=$(cd $WORKSPACE && git rev-parse HEAD)
+    XCI_SHA=$(cd $WORKSPACE && git rev-parse HEAD)
 
     # extract scenario sha which is same as releng-xci sha for generic scenarios
-    SCENARIO_SHA=$RELENG_XCI_SHA
+    SCENARIO_SHA=$XCI_SHA
 }
 
 # This function determines the impacted external scenario by processing the Gerrit
@@ -142,7 +147,7 @@ function determine_external_scenario() {
     DEPLOY_SCENARIO+=$(git diff HEAD^..HEAD --name-only | grep scenarios | awk -F '[/|/]' '{print $2}' | uniq)
 
     # extract releng-xci sha
-    RELENG_XCI_SHA=$(cd $WORKSPACE && git rev-parse HEAD)
+    XCI_SHA=$(cd $WORKSPACE && git rev-parse HEAD)
 
     # extract scenario sha
     SCENARIO_SHA=$(cd $WORK_DIRECTORY/$GERRIT_PROJECT && git rev-parse HEAD)
@@ -194,8 +199,9 @@ esac
 echo "Recording the installer '$INSTALLER_TYPE' and scenario '${DEPLOY_SCENARIO[0]}' and SHAs for downstream jobs"
 echo "INSTALLER_TYPE=$INSTALLER_TYPE" > $WORK_DIRECTORY/scenario.properties
 echo "DEPLOY_SCENARIO=$DEPLOY_SCENARIO" >> $WORK_DIRECTORY/scenario.properties
-echo "RELENG_XCI_SHA=$RELENG_XCI_SHA" >> $WORK_DIRECTORY/scenario.properties
+echo "XCI_SHA=$XCI_SHA" >> $WORK_DIRECTORY/scenario.properties
 echo "SCENARIO_SHA=$SCENARIO_SHA" >> $WORK_DIRECTORY/scenario.properties
+echo "PROJECT_NAME=$GERRIT_PROJECT" >> $WORK_DIRECTORY/scenario.properties
 
 # skip scenario support check if the job is promotion job
 if [[ "$JOB_NAME" =~ (os|k8) ]]; then