Merge "[multisite] trigger functest followed daily deployment job"
authorFatih Degirmenci <fatih.degirmenci@ericsson.com>
Tue, 26 Jul 2016 07:43:44 +0000 (07:43 +0000)
committerGerrit Code Review <gerrit@172.30.200.206>
Tue, 26 Jul 2016 07:43:45 +0000 (07:43 +0000)
12 files changed:
jjb/apex/apex.yml
jjb/fuel/fuel-upload-artifact.sh
jjb/functest/functest-ci-jobs.yml
jjb/functest/set-functest-env.sh
jjb/joid/joid-ci-jobs.yml
jjb/opnfv/opnfv-docs.yml
jjb/opnfvdocs/opnfvdocs.yml
jjb/releng-macros.yaml
jjb/yardstick/yardstick-project-jobs.yml
utils/fetch_os_creds.sh
utils/push-test-logs.sh
utils/retention_script.sh

index a3f2da6..f380233 100644 (file)
@@ -59,6 +59,7 @@
          - 'os-odl_l2-sfc-noha'
          - 'os-odl_l3-nofeature-ha'
          - 'os-onos-nofeature-ha'
+         - 'os-onos-sfc-ha'
          - 'os-ocl-nofeature-ha'
 
     platform:
                 build-step-failure-threshold: 'never'
                 failure-threshold: 'never'
                 unstable-threshold: 'FAILURE'
+        - trigger-builds:
+          - project: 'apex-deploy-baremetal-os-onos-sfc-ha-{stream1}'
+            predefined-parameters: |
+              BUILD_DIRECTORY=apex-build-{stream1}/build
+              OPNFV_CLEAN=yes
+            git-revision: true
+            same-node: true
+            block-thresholds:
+                build-step-failure-threshold: 'never'
+            block: true
+        - trigger-builds:
+          - project: 'functest-apex-{daily-slave}-daily-{stream1}'
+            predefined-parameters:
+              DEPLOY_SCENARIO=os-onos-sfc-ha
+            block: true
+            same-node: true
+            block-thresholds:
+                build-step-failure-threshold: 'never'
+                failure-threshold: 'never'
+                unstable-threshold: 'FAILURE'
+        - trigger-builds:
+          - project: 'yardstick-apex-{slave}-daily-{stream1}'
+            predefined-parameters:
+              DEPLOY_SCENARIO=os-onos-sfc-ha
+            block: true
+            same-node: true
+            block-thresholds:
+                build-step-failure-threshold: 'never'
+                failure-threshold: 'never'
+                unstable-threshold: 'FAILURE'
 
 - job-template:
     name: 'apex-gs-clean-{stream}'
index 47cf898..1ccd328 100755 (executable)
@@ -19,6 +19,7 @@ fi
 # source the opnfv.properties to get ARTIFACT_VERSION
 source $WORKSPACE/opnfv.properties
 
+nfsstore () {
 # storing ISOs for verify & merge jobs will be done once we get the disk array
 if [[ ! "$JOB_NAME" =~ (verify|merge) ]]; then
     # store ISO locally on NFS first
@@ -34,7 +35,30 @@ if [[ ! "$JOB_NAME" =~ (verify|merge) ]]; then
             $ISOSTORE/opnfv-$OPNFV_ARTIFACT_VERSION.iso
     fi
 fi
+}
 
+importkey () {
+# clone releng repository
+echo "Cloning releng repository..."
+[ -d releng ] && rm -rf releng
+git clone https://gerrit.opnfv.org/gerrit/releng $WORKSPACE/releng/ &> /dev/null
+#this is where we import the siging key
+if [ -f $WORKSPACE/releng/utils/gpg_import_key.sh ]; then
+  source $WORKSPACE/releng/utils/gpg_import_key.sh
+fi
+}
+
+signiso () {
+gpg2 -vvv --batch --yes --no-tty \
+  --default-key opnfv-helpdesk@rt.linuxfoundation.org  \
+  --passphrase besteffort \
+  --detach-sig $BUILD_DIRECTORY/opnfv-$OPNFV_ARTIFACT_VERSION.iso
+
+gsutil cp $BUILD_DIRECTORY/opnfv-$OPNFV_ARTIFACT_VERSION.iso.sig gs://$GS_URL/opnfv-$OPNFV_ARTIFACT_VERSION.iso.sig
+echo "ISO signature Upload Complete!"
+}
+
+uploadiso () {
 # log info to console
 echo "Uploading $INSTALLER_TYPE artifact. This could take some time..."
 echo
@@ -80,3 +104,17 @@ echo "Artifact is available as http://$GS_URL/opnfv-$OPNFV_ARTIFACT_VERSION.iso"
 echo
 echo "--------------------------------------------------------"
 echo
+}
+
+nfsstore
+
+if [[ ! "$JOB_NAME" =~ merge ]]; then
+    importkey
+    signiso
+    uploadiso
+fi
+
+if [[ ! "$JOB_NAME" =~ verify ]]; then
+    uploadiso
+fi
+
index 28ea643..6a571ab 100644 (file)
 #--------------------------------
 
     testsuite:
-        - 'daily'
-        - 'weekly'
-        - 'suite'
+        - 'suite':
+            job-timeout: 60
+        - 'daily':
+            job-timeout: 120
+        - 'weekly':
+            job-timeout: 400
 
     jobs:
         - 'functest-{installer}-{pod}-{testsuite}-{stream}'
         - build-name:
             name: '$BUILD_NUMBER Suite: $FUNCTEST_SUITE_NAME Scenario: $DEPLOY_SCENARIO'
         - timeout:
-            timeout: 400
+            timeout: '{job-timeout}'
             abort: true
 
     parameters:
index d2e232d..b7d745a 100755 (executable)
@@ -41,10 +41,13 @@ if ! sudo iptables -C FORWARD -j RETURN 2> ${redirect} || ! sudo iptables -L FOR
     sudo iptables -I FORWARD -j RETURN
 fi
 
+DEPLOY_TYPE=baremetal
+[[ $BUILD_TAG =~ "virtual" ]] && DEPLOY_TYPE=virt
+
 echo "Functest: Start Docker and prepare environment"
 envs="-e INSTALLER_TYPE=${INSTALLER_TYPE} -e INSTALLER_IP=${INSTALLER_IP} \
     -e NODE_NAME=${NODE_NAME} -e DEPLOY_SCENARIO=${DEPLOY_SCENARIO} \
-    -e BUILD_TAG=${BUILD_TAG} -e CI_DEBUG=${CI_DEBUG}"
+    -e BUILD_TAG=${BUILD_TAG} -e CI_DEBUG=${CI_DEBUG} -e DEPLOY_TYPE=${DEPLOY_TYPE}"
 branch=${GIT_BRANCH##*/}
 dir_result="${HOME}/opnfv/functest/results/${branch}"
 mkdir -p ${dir_result}
index 61c1f82..ae03eab 100644 (file)
@@ -70,7 +70,7 @@
         - 'os-onos-nofeature-noha':
             auto-trigger-name: 'daily-trigger-disabled'
         - 'os-onos-sfc-ha':
-            auto-trigger-name: 'daily-trigger-disabled'
+            auto-trigger-name: 'joid-{scenario}-{pod}-{stream}-trigger'
         - 'os-ocl-nofeature-ha':
             auto-trigger-name: 'daily-trigger-disabled'
         - 'os-ocl-nofeature-noha':
 - trigger:
     name: 'joid-os-onos-sfc-ha-baremetal-master-trigger'
     triggers:
-        - timed: '0 21 * * *'
+        - timed: '0 22 * * *'
+- trigger:
+    name: 'joid-os-onos-sfc-ha-virtual-master-trigger'
+    triggers:
+        - timed: ''
+- trigger:
+    name: 'joid-os-onos-sfc-ha-orange-pod2-master-trigger'
+    triggers:
+        - timed: ''
+- trigger:
+    name: 'joid-os-onos-sfc-ha-juniper-pod1-master-trigger'
+    triggers:
+        - timed: ''
 
 # os-onos-nofeature-ha trigger - branch: stable/brahmaputra
 - trigger:
     name: 'joid-os-onos-nofeature-ha-juniper-pod1-brahmaputra-trigger'
     triggers:
         - timed: ''
+- trigger:
+    name: 'joid-os-onos-sfc-ha-baremetal-brahmaputra-trigger'
+    triggers:
+        - timed: ''
+- trigger:
+    name: 'joid-os-onos-sfc-ha-virtual-brahmaputra-trigger'
+    triggers:
+        - timed: ''
+- trigger:
+    name: 'joid-os-onos-sfc-ha-orange-pod2-brahmaputra-trigger'
+    triggers:
+        - timed: ''
+- trigger:
+    name: 'joid-os-onos-sfc-ha-juniper-pod1-brahmaputra-trigger'
+    triggers:
+        - timed: ''
 
 # os-nosdn-lxd-noha trigger - branch: master
 - trigger:
index 2b3f98a..0c11757 100644 (file)
                 - comment-added-contains-event:
                     comment-contains-value: 'reverify'
             projects:
-              - project-compare-type: 'REG_EXP'
-#Left out arno projects oscar|octopus|functest|genesis
-                project-pattern: 'vswitchperf|apex|armband|availability|bottlenecks|compass4nfv|conductor|copper|doctor|dovetail|dpacc|escalator|fastpathmetrics|fuel|inspector|ipv6|joid|kvmfornfv|lsoapi|moon|movie|multisite|octopus|onosfw|ovsnfv|parser|prediction|promise|qtip|releng|rs|sdnvpn|sfc|vnf_forwarding_graph|enfv|pinpoint|pharos|cperf|functest|genesisreq|models|domino|netready'
+              - project-compare-type: 'ANT'
+                project-pattern: '(?!yardstick)'
                 branches:
                   - branch-compare-type: 'ANT'
                     branch-pattern: '**/{branch}'
                 file-paths:
                   - compare-type: ANT
                     pattern: 'docs/**'
+              - project-compare-type: 'ANT'
+                project-pattern: 'yardstick'
+                branches:
+                  - branch-compare-type: 'ANT'
+                    branch-pattern: '**/{branch}'
 
     builders:
         - upload-review-docs
                 - comment-added-contains-event:
                     comment-contains-value: 'remerge'
             projects:
-              - project-compare-type: 'REG_EXP'
-                project-pattern: 'vswitchperf|apex|armband|availability|bottlenecks|compass4nfv|conductor|copper|doctor|dovetail|dpacc|escalator|fastpathmetrics|fuel|inspector|ipv6|joid|kvmfornfv|lsoapi|moon|movie|multisite|octopus|onosfw|ovsnfv|parser|prediction|promise|qtip|releng|rs|sdnvpn|sfc|vnf_forwarding_graph|enfv|pinpoint|pharos|cperf|functest|genesisreq|models|domino|netready'
+              - project-compare-type: 'ANT'
+                project-pattern: '(?!yardstick)'
                 branches:
                   - branch-compare-type: 'ANT'
                     branch-pattern: '**/{branch}'
                 file-paths:
                   - compare-type: ANT
                     pattern: 'docs/**'
+              - project-compare-type: 'ANT'
+                project-pattern: 'yardstick'
+                branches:
+                  - branch-compare-type: 'ANT'
+                    branch-pattern: '**/{branch}'
 
     builders:
         - upload-merged-docs
index cbb8f95..f3e776c 100644 (file)
                 branches:
                   - branch-compare-type: 'ANT'
                     branch-pattern: '**/{branch}'
-                file-paths:
-                  - compare-type: ANT
-                    pattern: 'docs/**'
 
     builders:
-        - build-opnfv-composite-docs
-        - upload-under-review-docs-to-opnfv-artifacts
-        - report-docs-build-result-to-gerrit
+        - check-bash-syntax
 
 - job-template:
     name: 'opnfvdocs-merge-{stream}'
                 branches:
                   - branch-compare-type: 'ANT'
                     branch-pattern: '**/{branch}'
-                file-paths:
-                  - compare-type: ANT
-                    pattern: 'docs/**'
 
     builders:
-        - build-opnfv-composite-docs
-        - upload-generated-docs-to-opnfv-artifacts
-        - report-docs-build-result-to-gerrit
-        - remove-old-docs-from-opnfv-artifacts
+        - check-bash-syntax
 
 - job-template:
     name: 'opnfvdocs-daily-{stream}'
         - timed: '0 H/6 * * *'
 
     builders:
-        - build-opnfv-composite-docs
+        - build-html-and-pdf-docs-output
 #        - upload-generated-docs-to-opnfv-artifacts
 
-- builder:
-    name: build-opnfv-composite-docs
-    builders:
-        - shell: |
-            #!/bin/bash -e
-            export PATH=$PATH:/usr/local/bin/
-            GERRIT_COMMENT=gerrit_comment.txt ./build-composite.sh
index a69badb..0c4cb8f 100644 (file)
         - upload-generated-docs-to-opnfv-artifacts
         - report-docs-build-result-to-gerrit
         - remove-old-docs-from-opnfv-artifacts
+
+- builder:
+    name: check-bash-syntax
+    builders:
+        - shell: "find . -name '*.sh' | xargs bash -n"
index 7656c92..64031b7 100644 (file)
@@ -59,8 +59,6 @@
                     branch-pattern: '**/{branch}'
     builders:
         - yardstick-unit-tests-and-docs-build
-        - upload-under-review-docs-to-opnfv-artifacts
-        - report-docs-build-result-to-gerrit
 
 - job-template:
     name: 'yardstick-merge-{stream}'
@@ -97,9 +95,6 @@
 
     builders:
         - yardstick-unit-tests-and-docs-build
-        - upload-generated-docs-to-opnfv-artifacts
-        - report-docs-build-result-to-gerrit
-        - remove-old-docs-from-opnfv-artifacts
 
 ################################
 # job builders
             # unit tests
             ./run_tests.sh
 
-            pip install Sphinx==1.3.1 doc8 docutils sphinxcontrib-httpdomain
-
-            # generate api doc sources
-            sphinx-apidoc -o docs/apidocs yardstick
-
-            # build docs
-            git clone ssh://gerrit.opnfv.org:29418/opnfvdocs docs_build/_opnfvdocs
-            GERRIT_COMMENT=gerrit_comment.txt ./docs_build/_opnfvdocs/scripts/docs-build.sh
-
             deactivate
index c240976..34c81af 100755 (executable)
@@ -80,9 +80,16 @@ if [ "$installer_type" == "fuel" ]; then
     #ip_fuel="10.20.0.2"
     verify_connectivity $installer_ip
 
+    env=$(sshpass -p r00tme ssh 2>/dev/null $ssh_options root@${installer_ip} \
+        'fuel env'|grep operational|tail -1|awk '{print $1}') &> /dev/null
+    if [ -z $env ]; then
+        error "No operational environment detected in Fuel"
+    fi
+    env_id="${FUEL_ENV:-$env}"
+
     # Check if controller is alive (online='True')
     controller_ip=$(sshpass -p r00tme ssh 2>/dev/null $ssh_options root@${installer_ip} \
-        'fuel node | grep controller | grep "True\|  1" | awk -F\| "{print \$5}" | tail -1' | \
+        'fuel node --env ${env_id} | grep controller | grep "True\|  1" | awk -F\| "{print \$5}" | tail -1' | \
         sed 's/ //g') &> /dev/null
 
     if [ -z $controller_ip ]; then
index b385fa8..d36561d 100644 (file)
@@ -54,7 +54,7 @@ if [ -d "$dir_result" ]; then
                 echo "Not possible to push results to artifact: gsutil not installed.";
             else
                 echo "copy result files to artifact $project_artifact"
-                gsutil -m cp -r "$dir_result" gs://artifacts.opnfv.org/"$project_artifact"/
+                gsutil -m cp -r "$dir_result" gs://artifacts.opnfv.org/"$project_artifact"/ >/dev/null 2>&1
             fi
         fi
     else
index 7e50623..b217793 100755 (executable)
@@ -22,6 +22,7 @@ for x in armband ovsnfv fuel apex compass4nfv
 do
 
   echo "Looking at artifacts for project $x"
+  echo "In path gs://artifacts.opnfv.org/$x"
 
   while IFS= read -r artifact; do
 
@@ -31,6 +32,7 @@ do
 
     if [[ "$daysold" -gt "10" ]]; then
       echo "$daysold Days old deleting: $(basename $artifact)"
+      gsutil rm "$artifact"
     else
       echo "$daysold Days old retaining: $(basename $artifact)"
     fi