Redirect gsutil log to file(s) to have cleaner Jenkins log
[releng.git] / jjb / genesis / genesis-foreman.yml
index 8395d6a..faa4d47 100644 (file)
@@ -3,13 +3,17 @@
 
     name: genesis-foreman
 
+    project: 'genesis'
+
     installer:
         - foreman
+
     jobs:
-        - 'genesis-foreman-verify'
-        - 'genesis-foreman-merge'
-        - 'genesis-foreman-daily-{stream}'
-        - 'genesis-foreman-deploy'
+        - 'genesis-{installer}-verify'
+        - 'genesis-{installer}-merge'
+        - 'genesis-{installer}-daily-{stream}'
+        - 'genesis-{installer}-build'
+        - 'genesis-{installer}-deploy'
 
     # stream:    branch with - in place of / (eg. stable-helium)
     # branch:    branch (eg. stable/helium)
         - master:
             branch: 'master'
 
-    project: 'genesis'
 
 ########################
 # job templates
 ########################
 
 - job-template:
-    name: 'genesis-foreman-verify'
+    name: 'genesis-{installer}-verify'
 
     project-type: freestyle
 
         artifactNumToKeep: -1
 
     parameters:
+        - string:
+            name: INSTALLER
+            default: '{installer}'
+            description: "Installer to use."
         - string:
             name: BUILD_DIRECTORY
             default: $WORKSPACE/build_output
+        - string:
+            name: CACHE_DIRECTORY
+            default: '$HOME/opnfv/cache/genesis-{installer}'
         - string:
             name: GIT_BASE
             default: https://gerrit.opnfv.org/gerrit/genesis
                   - compare-type: ANT
                     pattern: 'common/**'
                   - compare-type: ANT
-                    pattern: 'foreman/**'
+                    pattern: '{installer}/**'
 
 
     builders:
-        - 'foreman-verify'
+        - 'verify'
 
 - job-template:
-    name: 'genesis-foreman-merge'
+    name: 'genesis-{installer}-merge'
 
     # builder-merge job to run JJB update
     #
         artifactNumToKeep: 5
 
     parameters:
+        - string:
+            name: INSTALLER
+            default: '{installer}'
+            description: "Installer to use."
         - string:
             name: BUILD_DIRECTORY
             default: $WORKSPACE/build_output
+        - string:
+            name: CACHE_DIRECTORY
+            default: '$HOME/opnfv/cache/genesis-{installer}'
         - string:
             name: GIT_BASE
             default: https://gerrit.opnfv.org/gerrit/genesis
                   - compare-type: ANT
                     pattern: 'common/**'
                   - compare-type: ANT
-                    pattern: 'foreman/**'
+                    pattern: '{installer}/**'
 
     builders:
-        - 'foreman-merge'
+        - 'merge'
 
 - job-template:
-    name: 'genesis-foreman-daily-{stream}'
+    name: 'genesis-{installer}-daily-{stream}'
 
     project-type: freestyle
 
     node: ericsson-build
 
     parameters:
-        - string:
-            name: BUILD_DIRECTORY
-            default: $WORKSPACE/build_output
-            description: "Directory where the build artifact will be located upon the completion of the build."
-        - string:
-            name: GS_URL
-            default: 'artifacts.opnfv.org/genesis/foreman'
-            description: "URL to Google Storage."
-        - string:
-            name: INSTALLER
-            default: 'foreman'
-            description: "Installer to use."
-        - string:
-            name: GIT_BASE
-            default: https://gerrit.opnfv.org/gerrit/genesis
-        - string:
-            name: GERRIT_BRANCH
-            default: origin/master
-            description: "Branch to build, deploy and test."
-        - string:
-            name: GERRIT_REFSPEC
-            default: refs/heads/master
-            description: "Refspec to retrieve."
+        - genesis-parameters:
+            installer: '{installer}'
 
     scm:
         - git:
         artifactNumToKeep: -1
 
     builders:
-        - 'foreman-daily-master'
+        - trigger-builds:
+          - project: 'genesis-{installer}-build'
+            git-revision: true
+            block: true
+        - trigger-builds:
+          - project: 'genesis-{installer}-deploy'
+            git-revision: true
+            block: true
+        - trigger-builds:
+          - project: 'functest-opnfv-jump-2'
+            block: true
 
+- job-template:
+    name: 'genesis-{installer}-build'
+
+    project-type: freestyle
+
+    node: ericsson-build
+
+    logrotate:
+        daysToKeep: 30
+        numToKeep: 10
+        artifactDaysToKeep: -1
+        artifactNumToKeep: -1
+
+    parameters:
+        - genesis-parameters:
+            installer: '{installer}'
+
+    scm:
+        - git:
+            skip-tag: true
+            url: $GIT_BASE
+            branches:
+                - $GERRIT_BRANCH
+            refspec: $GERRIT_REFSPEC
+
+    builders:
+        - 'build'
+        - 'upload-artifact'
 
 - job-template:
-    name: 'genesis-foreman-deploy'
+    name: 'genesis-{installer}-deploy'
 
     project-type: freestyle
 
         artifactDaysToKeep: -1
         artifactNumToKeep: -1
 
+    parameters:
+        - genesis-parameters:
+            installer: '{installer}'
+
+    scm:
+        - git:
+            skip-tag: true
+            url: $GIT_BASE
+            branches:
+                - $GERRIT_BRANCH
+            refspec: $GERRIT_REFSPEC
+
+    builders:
+        - 'deploy'
+
+########################
+# parameter macros
+########################
+- parameter:
+    name: genesis-parameters
     parameters:
+        - string:
+            name: INSTALLER
+            default: '{installer}'
+            description: "Installer to use."
         - string:
             name: BUILD_DIRECTORY
             default: $WORKSPACE/build_output
+            description: "Directory where the build artifact will be located upon the completion of the build."
+        - string:
+            name: CACHE_DIRECTORY
+            default: '$HOME/opnfv/cache/genesis-{installer}'
+            description: "Directory where the cache to be used during the build is located."
         - string:
             name: GIT_BASE
-            default: https://github.com/trozet/bgs_vagrant.git
-            description: "This is a temporary location to clone the needed scripts. Will be replaced with OPNFV Git URL."
+            default: https://gerrit.opnfv.org/gerrit/genesis
+            description: "Used for overriding the GIT URL coming from Global Jenkins configuration in case if the stuff is done on none-LF HW."
         - string:
             name: GERRIT_BRANCH
             default: origin/master
             name: GERRIT_REFSPEC
             default: refs/heads/master
             description: "Refspec to retrieve."
-
-    scm:
-        - git:
-            skip-tag: true
-            url: $GIT_BASE
-            branches:
-                - $GERRIT_BRANCH
-            refspec: $GERRIT_REFSPEC
-
-    wrappers:
-        - ssh-agent-credentials:
-            user: '{ssh-credentials}'
-
-    builders:
-        - 'foreman-deploy'
+        - string:
+            name: GS_URL
+            default: 'artifacts.opnfv.org/genesis/{installer}'
+            description: "URL to Google Storage."
 
 ########################
 # builder macros
 ########################
 - builder:
-    name: foreman-verify
+    name: 'verify'
     builders:
         - shell: |
             #!/bin/bash
             set -x
 
             # get the dependencies - temporary fix for RC0
-            [[ -d $WORKSPACE/foreman/build ]] || mkdir -p $WORKSPACE/foreman/build
-            cp /home/jenkins/Downloads/opnfv_iso/* $WORKSPACE/foreman/build/
+            [[ -d $WORKSPACE/$INSTALLER/build ]] || mkdir -p $WORKSPACE/$INSTALLER/build
+            cp /home/jenkins/Downloads/opnfv_iso/* $WORKSPACE/$INSTALLER/build/
 
             # below lines perhaps need to be enabled in order to fetch images from internet
             # as we are having disk space issues on ericsson-build slave
             #wget http://repos.duss-janser.ch/vagrant/vagrant_1.7.2_x86_64.rpm
 
             # do the build
-            cd $WORKSPACE/foreman/ci
+            cd $WORKSPACE/$INSTALLER/ci
             ./build.sh $BUILD_DIRECTORY
 
             # list the build artifacts
             /bin/rm -rf $BUILD_DIRECTORY
 
 - builder:
-    name: foreman-merge
+    name: 'merge'
     builders:
         - shell: |
             #!/bin/bash
             set -x
 
             # get the dependencies - temporary fix for RC0
-            [[ -d $WORKSPACE/foreman/build ]] || mkdir -p $WORKSPACE/foreman/build
-            cp /home/jenkins/Downloads/opnfv_iso/* $WORKSPACE/foreman/build/
+            [[ -d $WORKSPACE/$INSTALLER/build ]] || mkdir -p $WORKSPACE/$INSTALLER/build
+            cp /home/jenkins/Downloads/opnfv_iso/* $WORKSPACE/$INSTALLER/build/
 
             # below lines perhaps need to be enabled in order to fetch images from internet
             # as we are having disk space issues on ericsson-build slave
             #wget http://repos.duss-janser.ch/vagrant/vagrant_1.7.2_x86_64.rpm
 
             # do the build
-            cd $WORKSPACE/foreman/ci
+            cd $WORKSPACE/$INSTALLER/ci
             ./build.sh $BUILD_DIRECTORY
 
             # list the build artifacts
             /bin/rm -rf $BUILD_DIRECTORY
 
 - builder:
-    name: foreman-daily-master
+    name: 'build'
     builders:
         - shell: |
             #!/bin/bash
             set -o errexit
             set -o nounset
             set -o pipefail
-            set -x
 
-            # get the dependencies - temporary fix for RC0
-            [[ -d $WORKSPACE/foreman/build ]] || mkdir -p $WORKSPACE/foreman/build
-            cp /home/jenkins/Downloads/opnfv_iso/* $WORKSPACE/foreman/build/
+            # log info to console
+            echo "Starting build of $INSTALLER. This could take some time..."
+            echo "--------------------------------------------------------"
+            echo
 
-            # below lines perhaps need to be enabled in order to fetch images from internet
-            # as we are having disk space issues on ericsson-build slave
-            #wget http://ftp.riken.jp/Linux/centos/7.0.1406/isos/x86_64/CentOS-7.0-1406-x86_64-Minimal.iso
-            #wget http://download.virtualbox.org/virtualbox/rpm/fedora/20/x86_64/VirtualBox-4.3-4.3.26_98988_fedora18-1.x86_64.rpm
-            #wget http://repos.duss-janser.ch/vagrant/vagrant_1.7.2_x86_64.rpm
-
-            # do the build
-            cd $WORKSPACE/foreman/ci
-            ./build.sh -v $BUILD_ID $BUILD_DIRECTORY
-
-        - shell: |
-            #!/bin/bash
-            set -o errexit
-            set -o nounset
-            set -o pipefail
-            set -x
+            # create the cache directory if it doesn't exist
+            [[ -d $CACHE_DIRECTORY ]] || mkdir -p $CACHE_DIRECTORY
 
-            # list the build artifacts
-            ls -al $BUILD_DIRECTORY
+            # set OPNFV_ARTIFACT_VERSION
+            export OPNFV_ARTIFACT_VERSION=$(date -u +"%Y-%m-%d_%H-%M-%S")
 
-            cd $BUILD_DIRECTORY
+            # start the build
+            cd $WORKSPACE/$INSTALLER/ci
+            ./build.sh -v $OPNFV_ARTIFACT_VERSION -c file://$CACHE_DIRECTORY $BUILD_DIRECTORY
 
             # save information regarding artifact into file
             (
+                echo "OPNFV_ARTIFACT_VERSION=$OPNFV_ARTIFACT_VERSION"
                 echo "OPNFV_GIT_URL=$(git config --get remote.origin.url)"
                 echo "OPNFV_GIT_SHA1=$(git rev-parse HEAD)"
-                echo "OPNFV_ARTIFACT_URL=$GS_URL/opnfv-$BUILD_ID.iso"
-                echo "OPNFV_ARTIFACT_MD5SUM=$(md5sum opnfv-$ARTIFACT_VERSION.iso | cut -d' ' -f1)"
+                echo "OPNFV_ARTIFACT_URL=$GS_URL/opnfv-$OPNFV_ARTIFACT_VERSION.iso"
+                echo "OPNFV_ARTIFACT_MD5SUM=$(md5sum $BUILD_DIRECTORY/OPNFV-CentOS-7-x86_64-$OPNFV_ARTIFACT_VERSION.iso | cut -d' ' -f1)"
                 echo "OPNFV_BUILD_URL=$BUILD_URL"
                 echo "OPNFV_BUILD=OK"
-            ) > opnfv-$BUILD_ID.properties
+            ) > $WORKSPACE/opnfv.properties
 
-            # copy artifact property file as latest.properties
-            # so we can directly get info regarding latest artifact
-            /bin/cp -f opnfv-$BUILD_ID.properties latest.properties
-
-            # upload artifact and additional files to google storage
-            gsutil cp OPNFV-CentOS-7-x86_64-$BUILD_ID.iso gs://$GS_URL/opnfv-$BUILD_ID.iso
-            gsutil cp opnfv-$BUILD_ID.properties gs://$GS_URL/opnfv-$BUILD_ID.properties
-            gsutil cp latest.properties gs://$GS_URL/latest.properties
+            echo
+            echo "--------------------------------------------------------"
+            echo "Done!"
 
+- builder:
+    name: 'deploy'
+    builders:
         - shell: |
             #!/bin/bash
             set -o errexit
             set -o nounset
             set -o pipefail
-            set -x
-
-            # get the latest.properties file in order to get info regarding latest artifact
-            gsutil cp gs://$GS_URL/latest.properties latest.properties
 
-            # check if we got the file
-            [[ -f latest.properties ]] || exit 1
+            # log info to console
+            echo "Startint the deployment. This could take some time..."
+            echo "--------------------------------------------------------"
+            echo
 
-            # source the file so we get OPNFV vars
-            source latest.properties
-
-            # download the file
-            gsutil cp gs://$OPNFV_ARTIFACT_URL $WORKSPACE/opnfv.iso
+            # cleanup first
+            sudo $WORKSPACE/$INSTALLER/ci/clean.sh -base_config $WORKSPACE/$INSTALLER/ci/inventory/lf_pod2_ksgen_settings.yml
 
-            # list the file
-            ls -al $WORKSPACE/opnfv.iso
+            # and then initiate deployment
+            sudo $WORKSPACE/$INSTALLER/ci/deploy.sh -base_config $WORKSPACE/$INSTALLER/ci/inventory/lf_pod2_ksgen_settings.yml
 
-            # remove the BUILD_DIRECTORY and downlaoded artifact to reclaim some space
-            /bin/rm -rf $BUILD_DIRECTORY
-            /bin/rm -rf $WORKSPACE/opnfv.iso
+            echo
+            echo "--------------------------------------------------------"
+            echo "Done!"
 
 - builder:
-    name: foreman-deploy
+    name: 'upload-artifact'
     builders:
         - shell: |
             #!/bin/bash
             set -o errexit
             set -o nounset
             set -o pipefail
-            set -x
 
-            cd $WORKSPACE
+            # log info to console
+            echo "Uploading artifact. This could take some time..."
+            echo "--------------------------------------------------------"
+            echo
 
-            # fix PATH
-            export PATH=/usr/sbin:$PATH
-            echo $PATH
+            # list the build output
+            ls -al $BUILD_DIRECTORY
 
-            # messy workaround
-            git clone https://gist.github.com/4f2eb87d8c9937ee19f0.git lf_settings
-            ls -al $WORKSPACE/lf_settings
+            # source the opnfv.properties to get ARTIFACT_VERSION
+            source $WORKSPACE/opnfv.properties
 
-            # cleanup first
-            ssh -t -t jenkins-ci@localhost "sudo $WORKSPACE/clean.sh -base_config $WORKSPACE/lf_settings/lf_pod2_ksgen_settings.yml"
+            # copy artifact property file as latest.properties
+            # so we can directly get info regarding latest artifact
+            /bin/cp -f $WORKSPACE/opnfv.properties $WORKSPACE/latest.properties
 
-            # and then initiate deployment
-            ssh -t -t jenkins-ci@localhost "sudo $WORKSPACE/deploy.sh -base_config $WORKSPACE/lf_settings/lf_pod2_ksgen_settings.yml"
+            # upload artifact and additional files to google storage
+            gsutil cp $BUILD_DIRECTORY/OPNFV-CentOS-7-x86_64-$OPNFV_ARTIFACT_VERSION.iso gs://$GS_URL/opnfv-$OPNFV_ARTIFACT_VERSION.iso > gsutil.iso.log 2>&1
+            gsutil cp $WORKSPACE/opnfv.properties gs://$GS_URL/opnfv-$OPNFV_ARTIFACT_VERSION.properties > gsutil.properties.log 2>&1
+            gsutil cp $WORKSPACE/latest.properties gs://$GS_URL/latest.properties gsutil.latest.log 2>&1
+
+            echo
+            echo "--------------------------------------------------------"
+            echo "Done!"