From: Tim Rozet <trozet@redhat.com>
Date: Thu, 15 Nov 2018 22:28:18 +0000 (-0500)
Subject: Apex: Refactor snapshot pipeline for other types
X-Git-Url: https://gerrit.opnfv.org/gerrit/gitweb?p=releng.git;a=commitdiff_plain;h=fbe5dfee30f419fece03fb4bf5c27bcfae7abd8c

Apex: Refactor snapshot pipeline for other types

This patch changes snapshot related scripts to allow for more than just
the current snapshot type of "csit". This allows us to add other
snapshot types, including the functest type. The same
snapshot.properties file will be used to store information for all types
of snapshots. The snap cache folder structure will remain the same, but
hold different types of snapshots in the same folders.

These changes include renaming the URLs for the snapshot artifacts to
reflect the snapshot type, but remains backwards compatible until enough
promotions have occurred and other apex snapshot deployment scripts have
been migrated to the new format.

Change-Id: If3e47e700b5ad9c95418179042b85814fe389233
Signed-off-by: Tim Rozet <trozet@redhat.com>
---

diff --git a/jjb/apex/apex-deploy.sh b/jjb/apex/apex-deploy.sh
index 4876b8504..8f7af7ff9 100755
--- a/jjb/apex/apex-deploy.sh
+++ b/jjb/apex/apex-deploy.sh
@@ -6,6 +6,7 @@ set -o pipefail
 IPV6_FLAG=False
 ALLINONE_FLAG=False
 CSIT_ENV_FLAG=False
+FUNCTEST_ENV_FLAG=False
 
 # log info to console
 echo "Starting the Apex deployment."
@@ -119,6 +120,10 @@ if echo ${DEPLOY_SCENARIO} | grep csit; then
   CSIT_ENV_FLAG=True
   DEPLOY_SCENARIO=$(echo ${DEPLOY_SCENARIO} |  sed 's/-csit//')
   echo "INFO: CSIT env requested in deploy scenario"
+elif echo ${DEPLOY_SCENARIO} | grep functest; then
+  FUNCTEST_ENV_FLAG=True
+  DEPLOY_SCENARIO=$(echo ${DEPLOY_SCENARIO} |  sed 's/-functest//')
+  echo "INFO: Functest env requested in deploy scenario"
 fi
 
 echo "Deploy Scenario set to ${DEPLOY_SCENARIO}"
@@ -140,13 +145,18 @@ if [[ "$JOB_NAME" =~ "virtual" ]]; then
     DEPLOY_CMD="${DEPLOY_CMD} --virtual-computes 2"
   fi
 
-  if [[ "$PROMOTE" == "True"  || "$CSIT_ENV_FLAG" == "True" ]]; then
+  if [[ "$FUNCTEST_ENV_FLAG" == "True"  || "$CSIT_ENV_FLAG" == "True" ]]; then
+    if [[ "$CSIT_ENV_FLAG" == "True" ]]; then
+      ENV_TYPE="csit"
+    else
+      ENV_TYPE="functest"
+    fi
     if [[ "$OS_VERSION" != "master" ]]; then
-      CSIT_ENV="csit-${OS_VERSION}-environment.yaml"
+      SNAP_ENV="${ENV_TYPE}-${OS_VERSION}-environment.yaml"
     else
-      CSIT_ENV="csit-environment.yaml"
+      SNAP_ENV="${ENV_TYPE}-environment.yaml"
     fi
-    DEPLOY_CMD="${DEPLOY_CMD} -e ${CSIT_ENV}"
+    DEPLOY_CMD="${DEPLOY_CMD} -e ${SNAP_ENV}"
   fi
 else
   # settings for bare metal deployment
@@ -163,7 +173,8 @@ fi
 
 if [ "$IPV6_FLAG" == "True" ]; then
   NETWORK_FILE="${NETWORK_SETTINGS_DIR}/network_settings_v6.yaml"
-elif [[ "$PROMOTE" == "True" ]]; then
+elif [[ "$CSIT_ENV_FLAG" == "True"  || "$FUNCTEST_ENV_FLAG" == "True" ]]; then
+  # We use csit network settings which is single network for snapshots
   NETWORK_FILE="${NETWORK_SETTINGS_DIR}/network_settings_csit.yaml"
 else
   NETWORK_FILE="${NETWORK_SETTINGS_DIR}/network_settings.yaml"
diff --git a/jjb/apex/apex-fetch-snap-info.sh b/jjb/apex/apex-fetch-snap-info.sh
index 2cfaa59d7..3324aca4f 100755
--- a/jjb/apex/apex-fetch-snap-info.sh
+++ b/jjb/apex/apex-fetch-snap-info.sh
@@ -17,7 +17,7 @@ echo "Fetching overcloudrc, ssh key, and node.yaml from deployment..."
 
 SSH_OPTIONS=(-o StrictHostKeyChecking=no -o GlobalKnownHostsFile=/dev/null -o UserKnownHostsFile=/dev/null -o LogLevel=error)
 
-tmp_dir=/tmp/csit
+tmp_dir=/tmp/snap
 rm -rf ${tmp_dir}
 mkdir -p ${tmp_dir}
 
diff --git a/jjb/apex/apex-snapshot-create.sh b/jjb/apex/apex-snapshot-create.sh
index f91720883..e8bf60bd2 100644
--- a/jjb/apex/apex-snapshot-create.sh
+++ b/jjb/apex/apex-snapshot-create.sh
@@ -28,7 +28,7 @@ tmp_dir=$(pwd)/.tmp
 mkdir -p ${tmp_dir}
 
 # info should have already been collected in apex-fetch-snap-info so copy it
-cp -r /tmp/csit/* ${tmp_dir}/
+cp -r /tmp/snap/* ${tmp_dir}/
 
 echo "Shutting down nodes"
 # Shut down nodes
@@ -76,19 +76,30 @@ sudo rm -rf ${tmp_dir}
 echo "Snapshot saved as apex-${SNAP_TYPE}-snap-${DATE}.tar.gz"
 
 # update opnfv properties file
-if [ "$SNAP_TYPE" == 'csit' ]; then
-  snap_sha=$(sha512sum apex-csit-snap-${DATE}.tar.gz | cut -d' ' -f1)
-  if curl --fail -O -L http://$GS_URL/snapshot.properties; then
+snap_sha=$(sha512sum apex-${SNAP_TYPE}-snap-${DATE}.tar.gz | cut -d' ' -f1)
+
+if curl --fail -O -L http://$GS_URL/snapshot.properties; then
+  # TODO(trozet): deprecate OPNFV_SNAP_URL for CSIT_SNAP_URL
+  if [ "$SNAP_TYPE" == 'csit' ]; then
     sed -i '/^OPNFV_SNAP_URL=/{h;s#=.*#='${GS_URL}'/apex-csit-snap-'${DATE}'.tar.gz#};${x;/^$/{s##OPNFV_SNAP_URL='${GS_URL}'/apex-csit-snap-'${DATE}'.tar.gz#;H};x}' snapshot.properties
     sed -i '/^OPNFV_SNAP_SHA512SUM=/{h;s/=.*/='${snap_sha}'/};${x;/^$/{s//OPNFV_SNAP_SHA512SUM='${snap_sha}'/;H};x}' snapshot.properties
-  else
-    cat << EOF > snapshot.properties
+  fi
+  sed -i '/^'${SNAP_TYPE}'_SNAP_URL=/{h;s#=.*#='${GS_URL}'/apex-'${SNAP_TYPE}'-snap-'${DATE}'.tar.gz#};${x;/^$/{s##'${SNAP_TYPE}'_SNAP_URL='${GS_URL}'/apex-'${SNAP_TYPE}'-snap-'${DATE}'.tar.gz#;H};x}' snapshot.properties
+  sed -i '/^'${SNAP_TYPE}'_SNAP_SHA512SUM=/{h;s/=.*/='${snap_sha}'/};${x;/^$/{s//'${SNAP_TYPE}'_SNAP_SHA512SUM='${snap_sha}'/;H};x}' snapshot.properties
+else
+  cat << EOF > snapshot.properties
+${SNAP_TYPE}_SNAP_URL=${GS_URL}/apex-${SNAP_TYPE}-snap-${DATE}.tar.gz
+${SNAP_TYPE}_SNAP_SHA512SUM=${snap_sha}
+EOF
+  # TODO(trozet): deprecate OPNFV_SNAP_URL for CSIT_SNAP_URL
+  if [ "$SNAP_TYPE" == 'csit' ]; then
+    cat << EOF >> snapshot.properties
 OPNFV_SNAP_URL=${GS_URL}/apex-csit-snap-${DATE}.tar.gz
 OPNFV_SNAP_SHA512SUM=${snap_sha}
 EOF
   fi
-  echo "OPNFV_SNAP_URL=$GS_URL/apex-csit-snap-${DATE}.tar.gz"
-  echo "OPNFV_SNAP_SHA512SUM=$(sha512sum apex-csit-snap-${DATE}.tar.gz | cut -d' ' -f1)"
-  echo "Updated properties file: "
-  cat snapshot.properties
 fi
+echo "${SNAP_TYPE}_SNAP_URL=$GS_URL/apex-${SNAP_TYPE}-snap-${DATE}.tar.gz"
+echo "${SNAP_TYPE}_SNAP_SHA512SUM=$(sha512sum apex-${SNAP_TYPE}-snap-${DATE}.tar.gz | cut -d' ' -f1)"
+echo "Updated properties file: "
+cat snapshot.properties
diff --git a/jjb/apex/apex-snapshot-deploy.sh b/jjb/apex/apex-snapshot-deploy.sh
index bd023f558..dd69df3fc 100644
--- a/jjb/apex/apex-snapshot-deploy.sh
+++ b/jjb/apex/apex-snapshot-deploy.sh
@@ -20,6 +20,11 @@ echo "Deploying Apex snapshot..."
 echo "--------------------------"
 echo
 
+if [ -z "$SNAP_TYPE" ]; then
+  echo "ERROR: SNAP_TYPE not provided...exiting"
+  exit 1
+fi
+
 echo "Cleaning server"
 pushd ci > /dev/null
 sudo opnfv-clean
@@ -37,7 +42,7 @@ echo "Properties contents:"
 cat ${WORKSPACE}/opnfv.properties
 
 # find latest check sum
-latest_snap_checksum=$(cat ${WORKSPACE}/opnfv.properties | grep OPNFV_SNAP_SHA512SUM | awk -F "=" '{print $2}')
+latest_snap_checksum=$(cat ${WORKSPACE}/opnfv.properties | grep ${SNAP_TYPE}_SNAP_SHA512SUM | awk -F "=" '{print $2}')
 if [ -z "$latest_snap_checksum" ]; then
   echo "ERROR: checksum of latest snapshot from snapshot.properties is null!"
   exit 1
@@ -49,7 +54,7 @@ SNAP_CACHE=${SNAP_CACHE}/${OS_VERSION}/${TOPOLOGY}
 # check snap cache directory exists
 # if snapshot cache exists, find the checksum
 if [ -d "$SNAP_CACHE" ]; then
-  latest_snap=$(ls ${SNAP_CACHE} | grep tar.gz | tail -n 1)
+  latest_snap=$(ls ${SNAP_CACHE} | grep tar.gz | grep $SNAP_TYPE | tail -n 1)
   if [ -n "$latest_snap" ]; then
     local_snap_checksum=$(sha512sum ${SNAP_CACHE}/${latest_snap} | cut -d' ' -f1)
     echo "Local snap checksum is: ${local_snap_checksum}"
@@ -60,7 +65,12 @@ fi
 
 # compare check sum and download latest snap if not up to date
 if [ "$local_snap_checksum" != "$latest_snap_checksum" ]; then
-  snap_url=$(cat opnfv.properties | grep OPNFV_SNAP_URL | awk -F "=" '{print $2}')
+  snap_url=$(cat opnfv.properties | grep ${SNAP_TYPE}_SNAP_URL | awk -F "=" '{print $2}')
+  # TODO(trozet): Remove this once OPNFV url is deprecated
+  if [[ -z "$snap_url" && "$SNAP_TYPE" == 'csit' ]]; then
+      echo "WARN: Unable to find snap url for ${SNAP_TYPE}, attempting to use OPNFV"
+      snap_url=$(cat opnfv.properties | grep OPNFV_SNAP_URL | awk -F "=" '{print $2}')
+  fi
   if [ -z "$snap_url" ]; then
     echo "ERROR: Snap URL from snapshot.properties is null!"
     exit 1
diff --git a/jjb/apex/apex-upload-artifact.sh b/jjb/apex/apex-upload-artifact.sh
index 07198b188..00a0a1c78 100755
--- a/jjb/apex/apex-upload-artifact.sh
+++ b/jjb/apex/apex-upload-artifact.sh
@@ -85,9 +85,7 @@ uploadsnap () {
     exit 1
   fi
   gsutil cp ${snapshot_dir}/apex-${SNAP_TYPE}-snap-`date +%Y-%m-%d`.tar.gz gs://$GS_URL/ > gsutil.iso.log
-  if [ "$SNAP_TYPE" == 'csit' ]; then
-    gsutil cp ${snapshot_dir}/snapshot.properties gs://$GS_URL/snapshot.properties > gsutil.latest.log
-  fi
+  gsutil cp ${snapshot_dir}/snapshot.properties gs://$GS_URL/snapshot.properties > gsutil.latest.log
   echo "Upload complete for Snapshot"
 }
 
diff --git a/jjb/apex/apex.yaml b/jjb/apex/apex.yaml
index c7c938569..80a4d82f5 100644
--- a/jjb/apex/apex.yaml
+++ b/jjb/apex/apex.yaml
@@ -8,7 +8,7 @@
       - 'apex-virtual-{stream}'
       - 'apex-deploy-{platform}-{stream}'
       - 'apex-daily-{stream}'
-      - 'apex-csit-promote-daily-{stream}-os-{os_version}-{topology}'
+      - 'apex-{snap_type}-promote-daily-{stream}-os-{os_version}-{topology}'
       - 'apex-fdio-promote-daily-{stream}'
       - 'apex-{scenario}-baremetal-{scenario_stream}'
       - 'apex-testsuite-{scenario}-baremetal-{scenario_stream}'
@@ -254,6 +254,11 @@
       - 'ha'
       - 'noha-allinone'
 
+    snap_type:
+      - csit:
+          sdn: 'odl'
+      - functest:
+          sdn: 'nosdn'
 # Fetch Logs Job
 - job-template:
     name: 'apex-fetch-logs-{stream}'
@@ -839,7 +844,7 @@
 
 # CSIT promote
 - job-template:
-    name: 'apex-csit-promote-daily-{stream}-os-{os_version}-{topology}'
+    name: 'apex-{snap_type}-promote-daily-{stream}-os-{os_version}-{topology}'
 
     # Job template for promoting CSIT Snapshots
     #
@@ -883,6 +888,10 @@
           name: FORCE_PROMOTE
           default: 'False'
           description: "Used to force promotion and skip CSIT"
+      - string:
+          name: SNAP_TYPE
+          default: '{snap_type}'
+          description: Type of snapshot to promote
     properties:
       - build-blocker:
           use-build-blocker: true
@@ -891,7 +900,7 @@
             - 'apex-verify.*'
             - 'apex-runner.*'
             - 'apex-daily.*'
-            - 'apex-csit-promote.*'
+            - 'apex-.*-promote.*'
             - 'odl-netvirt.*'
       - throttle:
           max-per-node: 1
@@ -899,7 +908,7 @@
           option: 'project'
 
     triggers:
-      - '{stream}-{os_version}'
+      - '{stream}-{snap_type}-{os_version}'
 
     builders:
       - multijob:
@@ -909,7 +918,7 @@
             - name: 'apex-deploy-virtual-{stream}'
               current-parameters: true
               predefined-parameters: |
-                DEPLOY_SCENARIO=os-odl-{os_scenario}-{topology}
+                DEPLOY_SCENARIO=os-{sdn}-{os_scenario}-{topology}
                 OPNFV_CLEAN=yes
                 GERRIT_BRANCH=$GERRIT_BRANCH
                 GERRIT_REFSPEC=$GERRIT_REFSPEC
@@ -930,7 +939,7 @@
               abort-all-job: true
               git-revision: false
       - multijob:
-          name: csit
+          name: test phase
           condition: SUCCESSFUL
           execution-type: SEQUENTIALLY
           projects:
@@ -943,16 +952,36 @@
                 ODL_CONTAINERIZED=true
                 OS_VERSION=$OS_VERSION
                 SKIP_CSIT=$FORCE_PROMOTE
+                SNAP_TYPE=$SNAP_TYPE
               node-parameters: true
               kill-phase-on: NEVER
               abort-all-job: false
+              enable-condition: "def m = '$SNAP_TYPE' ==~ /csit/"
             - name: cperf-upload-logs-csit
               predefined-parameters: |
                 ODL_BRANCH=$ODL_BRANCH
                 OS_VERSION=$OS_VERSION
+                SNAP_TYPE=$SNAP_TYPE
               node-parameters: true
               kill-phase-on: FAILURE
               abort-all-job: false
+              enable-condition: "def m = '$SNAP_TYPE' ==~ /csit/"
+            - name: 'functest-apex-virtual-suite-{stream}'
+              current-parameters: false
+              predefined-parameters: |
+                DEPLOY_SCENARIO=$DEPLOY_SCENARIO
+                DOCKER_TAG=$DOCKER_TAG
+                FUNCTEST_SUITE_NAME=tempest_smoke
+                FUNCTEST_MODE=testcase
+                GERRIT_BRANCH=$GERRIT_BRANCH
+                GERRIT_REFSPEC=$GERRIT_REFSPEC
+                GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER
+                GERRIT_CHANGE_COMMIT_MESSAGE=$GERRIT_CHANGE_COMMIT_MESSAGE
+              node-parameters: true
+              kill-phase-on: NEVER
+              enable-condition: "def m = '$SNAP_TYPE' ==~ /functest/"
+              abort-all-job: true
+              git-revision: false
             - name: 'apex-fetch-logs-{stream}'
               current-parameters: false
               predefined-parameters: |
@@ -970,8 +999,6 @@
           projects:
             - name: 'apex-create-snapshot'
               current-parameters: true
-              predefined-parameters: |
-                SNAP_TYPE=csit
               node-parameters: true
               kill-phase-on: FAILURE
               abort-all-job: true
@@ -982,8 +1009,6 @@
           projects:
             - name: 'apex-upload-snapshot'
               current-parameters: true
-              predefined-parameters: |
-                SNAP_TYPE=csit
               node-parameters: true
               kill-phase-on: FAILURE
               abort-all-job: true
@@ -2014,76 +2039,150 @@
       - timed: '0 3 1 1 7'
 
 - trigger:
-    name: 'master-master'
+    name: 'master-csit-master'
     triggers:
       - timed: '0 5 * * *'
 
 - trigger:
-    name: 'master-rocky'
+    name: 'master-csit-rocky'
     triggers:
       - timed: '0 5 * * *'
 
 - trigger:
-    name: 'master-queens'
+    name: 'master-csit-queens'
     triggers:
       - timed: ''
 
 - trigger:
-    name: 'gambia-master'
+    name: 'gambia-csit-master'
     triggers:
       - timed: ''
 
 - trigger:
-    name: 'gambia-rocky'
+    name: 'gambia-csit-rocky'
     triggers:
       - timed: ''
 
 - trigger:
-    name: 'gambia-queens'
+    name: 'gambia-csit-queens'
     triggers:
       - timed: '0 5 * * *'
 
 - trigger:
-    name: 'fraser-master'
+    name: 'fraser-csit-master'
+    triggers:
+      - timed: ''
+
+- trigger:
+    name: 'fraser-csit-rocky'
+    triggers:
+      - timed: ''
+
+- trigger:
+    name: 'fraser-csit-queens'
+    triggers:
+      - timed: ''
+
+- trigger:
+    name: 'euphrates-csit-master'
+    triggers:
+      - timed: ''
+
+- trigger:
+    name: 'euphrates-csit-rocky'
+    triggers:
+      - timed: ''
+
+- trigger:
+    name: 'euphrates-csit-queens'
+    triggers:
+      - timed: ''
+
+- trigger:
+    name: 'danube-csit-master'
+    triggers:
+      - timed: ''
+
+- trigger:
+    name: 'danube-csit-rocky'
+    triggers:
+      - timed: ''
+
+- trigger:
+    name: 'danube-csit-queens'
+    triggers:
+      - timed: ''
+- trigger:
+    name: 'master-functest-master'
+    triggers:
+      - timed: '0 3 * * *'
+
+- trigger:
+    name: 'master-functest-rocky'
+    triggers:
+      - timed: '0 3 * * *'
+
+- trigger:
+    name: 'master-functest-queens'
+    triggers:
+      - timed: ''
+
+- trigger:
+    name: 'gambia-functest-master'
+    triggers:
+      - timed: ''
+
+- trigger:
+    name: 'gambia-functest-rocky'
+    triggers:
+      - timed: ''
+
+- trigger:
+    name: 'gambia-functest-queens'
+    triggers:
+      - timed: '0 3 * * *'
+
+- trigger:
+    name: 'fraser-functest-master'
     triggers:
       - timed: ''
 
 - trigger:
-    name: 'fraser-rocky'
+    name: 'fraser-functest-rocky'
     triggers:
       - timed: ''
 
 - trigger:
-    name: 'fraser-queens'
+    name: 'fraser-functest-queens'
     triggers:
       - timed: ''
 
 - trigger:
-    name: 'euphrates-master'
+    name: 'euphrates-functest-master'
     triggers:
       - timed: ''
 
 - trigger:
-    name: 'euphrates-rocky'
+    name: 'euphrates-functest-rocky'
     triggers:
       - timed: ''
 
 - trigger:
-    name: 'euphrates-queens'
+    name: 'euphrates-functest-queens'
     triggers:
       - timed: ''
 
 - trigger:
-    name: 'danube-master'
+    name: 'danube-functest-master'
     triggers:
       - timed: ''
 
 - trigger:
-    name: 'danube-rocky'
+    name: 'danube-functest-rocky'
     triggers:
       - timed: ''
 
 - trigger:
-    name: 'danube-queens'
+    name: 'danube-functest-queens'
     triggers:
       - timed: ''
diff --git a/jjb/apex/apex.yaml.j2 b/jjb/apex/apex.yaml.j2
index 8edcc2742..7203cb4b9 100644
--- a/jjb/apex/apex.yaml.j2
+++ b/jjb/apex/apex.yaml.j2
@@ -8,7 +8,7 @@
       - 'apex-virtual-{stream}'
       - 'apex-deploy-{platform}-{stream}'
       - 'apex-daily-{stream}'
-      - 'apex-csit-promote-daily-{stream}-os-{os_version}-{topology}'
+      - 'apex-{snap_type}-promote-daily-{stream}-os-{os_version}-{topology}'
       - 'apex-fdio-promote-daily-{stream}'
       - 'apex-{scenario}-baremetal-{scenario_stream}'
       - 'apex-testsuite-{scenario}-baremetal-{scenario_stream}'
@@ -100,6 +100,11 @@
       - 'ha'
       - 'noha-allinone'
 
+    snap_type:
+      - csit:
+          sdn: 'odl'
+      - functest:
+          sdn: 'nosdn'
 # Fetch Logs Job
 - job-template:
     name: 'apex-fetch-logs-{stream}'
@@ -685,7 +690,7 @@
 
 # CSIT promote
 - job-template:
-    name: 'apex-csit-promote-daily-{stream}-os-{os_version}-{topology}'
+    name: 'apex-{snap_type}-promote-daily-{stream}-os-{os_version}-{topology}'
 
     # Job template for promoting CSIT Snapshots
     #
@@ -729,6 +734,10 @@
           name: FORCE_PROMOTE
           default: 'False'
           description: "Used to force promotion and skip CSIT"
+      - string:
+          name: SNAP_TYPE
+          default: '{snap_type}'
+          description: Type of snapshot to promote
     properties:
       - build-blocker:
           use-build-blocker: true
@@ -737,7 +746,7 @@
             - 'apex-verify.*'
             - 'apex-runner.*'
             - 'apex-daily.*'
-            - 'apex-csit-promote.*'
+            - 'apex-.*-promote.*'
             - 'odl-netvirt.*'
       - throttle:
           max-per-node: 1
@@ -745,7 +754,7 @@
           option: 'project'
 
     triggers:
-      - '{stream}-{os_version}'
+      - '{stream}-{snap_type}-{os_version}'
 
     builders:
       - multijob:
@@ -755,7 +764,7 @@
             - name: 'apex-deploy-virtual-{stream}'
               current-parameters: true
               predefined-parameters: |
-                DEPLOY_SCENARIO=os-odl-{os_scenario}-{topology}
+                DEPLOY_SCENARIO=os-{sdn}-{os_scenario}-{topology}
                 OPNFV_CLEAN=yes
                 GERRIT_BRANCH=$GERRIT_BRANCH
                 GERRIT_REFSPEC=$GERRIT_REFSPEC
@@ -776,7 +785,7 @@
               abort-all-job: true
               git-revision: false
       - multijob:
-          name: csit
+          name: test phase
           condition: SUCCESSFUL
           execution-type: SEQUENTIALLY
           projects:
@@ -789,16 +798,36 @@
                 ODL_CONTAINERIZED=true
                 OS_VERSION=$OS_VERSION
                 SKIP_CSIT=$FORCE_PROMOTE
+                SNAP_TYPE=$SNAP_TYPE
               node-parameters: true
               kill-phase-on: NEVER
               abort-all-job: false
+              enable-condition: "def m = '$SNAP_TYPE' ==~ /csit/"
             - name: cperf-upload-logs-csit
               predefined-parameters: |
                 ODL_BRANCH=$ODL_BRANCH
                 OS_VERSION=$OS_VERSION
+                SNAP_TYPE=$SNAP_TYPE
               node-parameters: true
               kill-phase-on: FAILURE
               abort-all-job: false
+              enable-condition: "def m = '$SNAP_TYPE' ==~ /csit/"
+            - name: 'functest-apex-virtual-suite-{stream}'
+              current-parameters: false
+              predefined-parameters: |
+                DEPLOY_SCENARIO=$DEPLOY_SCENARIO
+                DOCKER_TAG=$DOCKER_TAG
+                FUNCTEST_SUITE_NAME=tempest_smoke
+                FUNCTEST_MODE=testcase
+                GERRIT_BRANCH=$GERRIT_BRANCH
+                GERRIT_REFSPEC=$GERRIT_REFSPEC
+                GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER
+                GERRIT_CHANGE_COMMIT_MESSAGE=$GERRIT_CHANGE_COMMIT_MESSAGE
+              node-parameters: true
+              kill-phase-on: NEVER
+              enable-condition: "def m = '$SNAP_TYPE' ==~ /functest/"
+              abort-all-job: true
+              git-revision: false
             - name: 'apex-fetch-logs-{stream}'
               current-parameters: false
               predefined-parameters: |
@@ -816,8 +845,6 @@
           projects:
             - name: 'apex-create-snapshot'
               current-parameters: true
-              predefined-parameters: |
-                SNAP_TYPE=csit
               node-parameters: true
               kill-phase-on: FAILURE
               abort-all-job: true
@@ -828,8 +855,6 @@
           projects:
             - name: 'apex-upload-snapshot'
               current-parameters: true
-              predefined-parameters: |
-                SNAP_TYPE=csit
               node-parameters: true
               kill-phase-on: FAILURE
               abort-all-job: true
@@ -1196,76 +1221,150 @@
       - timed: '0 3 1 1 7'
 
 - trigger:
-    name: 'master-master'
+    name: 'master-csit-master'
     triggers:
       - timed: '0 5 * * *'
 
 - trigger:
-    name: 'master-rocky'
+    name: 'master-csit-rocky'
     triggers:
       - timed: '0 5 * * *'
 
 - trigger:
-    name: 'master-queens'
+    name: 'master-csit-queens'
     triggers:
       - timed: ''
 
 - trigger:
-    name: 'gambia-master'
+    name: 'gambia-csit-master'
     triggers:
       - timed: ''
 
 - trigger:
-    name: 'gambia-rocky'
+    name: 'gambia-csit-rocky'
     triggers:
       - timed: ''
 
 - trigger:
-    name: 'gambia-queens'
+    name: 'gambia-csit-queens'
     triggers:
       - timed: '0 5 * * *'
 
 - trigger:
-    name: 'fraser-master'
+    name: 'fraser-csit-master'
+    triggers:
+      - timed: ''
+
+- trigger:
+    name: 'fraser-csit-rocky'
+    triggers:
+      - timed: ''
+
+- trigger:
+    name: 'fraser-csit-queens'
+    triggers:
+      - timed: ''
+
+- trigger:
+    name: 'euphrates-csit-master'
+    triggers:
+      - timed: ''
+
+- trigger:
+    name: 'euphrates-csit-rocky'
+    triggers:
+      - timed: ''
+
+- trigger:
+    name: 'euphrates-csit-queens'
+    triggers:
+      - timed: ''
+
+- trigger:
+    name: 'danube-csit-master'
+    triggers:
+      - timed: ''
+
+- trigger:
+    name: 'danube-csit-rocky'
+    triggers:
+      - timed: ''
+
+- trigger:
+    name: 'danube-csit-queens'
+    triggers:
+      - timed: ''
+- trigger:
+    name: 'master-functest-master'
+    triggers:
+      - timed: '0 3 * * *'
+
+- trigger:
+    name: 'master-functest-rocky'
+    triggers:
+      - timed: '0 3 * * *'
+
+- trigger:
+    name: 'master-functest-queens'
+    triggers:
+      - timed: ''
+
+- trigger:
+    name: 'gambia-functest-master'
+    triggers:
+      - timed: ''
+
+- trigger:
+    name: 'gambia-functest-rocky'
+    triggers:
+      - timed: ''
+
+- trigger:
+    name: 'gambia-functest-queens'
+    triggers:
+      - timed: '0 3 * * *'
+
+- trigger:
+    name: 'fraser-functest-master'
     triggers:
       - timed: ''
 
 - trigger:
-    name: 'fraser-rocky'
+    name: 'fraser-functest-rocky'
     triggers:
       - timed: ''
 
 - trigger:
-    name: 'fraser-queens'
+    name: 'fraser-functest-queens'
     triggers:
       - timed: ''
 
 - trigger:
-    name: 'euphrates-master'
+    name: 'euphrates-functest-master'
     triggers:
       - timed: ''
 
 - trigger:
-    name: 'euphrates-rocky'
+    name: 'euphrates-functest-rocky'
     triggers:
       - timed: ''
 
 - trigger:
-    name: 'euphrates-queens'
+    name: 'euphrates-functest-queens'
     triggers:
       - timed: ''
 
 - trigger:
-    name: 'danube-master'
+    name: 'danube-functest-master'
     triggers:
       - timed: ''
 
 - trigger:
-    name: 'danube-rocky'
+    name: 'danube-functest-rocky'
     triggers:
       - timed: ''
 
 - trigger:
-    name: 'danube-queens'
+    name: 'danube-functest-queens'
     triggers:
       - timed: ''