Bugfix: Fix lab reconfig issue by moving fuel dependency installation to master node
[releng.git] / jjb / genesis / genesis-fuel.yml
1 ########################
2 # Job configuration for genesis/fuel
3 ########################
4 - project:
5
6     name: genesis-fuel
7
8     project: 'genesis'
9
10     installer: 'fuel'
11
12     jobs:
13         - 'genesis-fuel-verify-build-{stream}'
14         - 'genesis-fuel-merge-{stream}'
15         - 'genesis-fuel-daily-{stream}'
16         - 'genesis-fuel-build-{stream}'
17         - 'genesis-fuel-deploy-{stream}'
18         - 'genesis-fuel-deploy-runner-{stream}'
19         - 'genesis-fuel-lab-reconfig-{stream}'
20         - 'genesis-fuel-verify-deploy-virtual-{stream}'
21
22     # stream:    branch with - in place of / (eg. stable-arno)
23     # branch:    branch (eg. stable/arno)
24     stream:
25         - master:
26             branch: 'master'
27             gs-pathname: ''
28         - stable-arno:
29             branch: 'stable/arno'
30             gs-pathname: '/arno'
31
32 ########################
33 # job templates
34 ########################
35
36 - job-template:
37     name: 'genesis-fuel-verify-build-{stream}'
38
39     project-type: freestyle
40
41     node: ericsson-build
42
43     concurrent: true
44
45     properties:
46         - throttle:
47             enabled: true
48             max-total: 3
49
50     logrotate:
51         daysToKeep: 30
52         numToKeep: 10
53         artifactDaysToKeep: -1
54         artifactNumToKeep: -1
55
56     parameters:
57         - project-parameter:
58             project: '{project}'
59         - gerrit-parameter:
60             branch: '{branch}'
61         - fuel-parameter:
62             installer: '{installer}'
63             gs-pathname: '{gs-pathname}'
64
65     scm:
66         - gerrit-trigger-scm:
67             credentials-id: '{ssh-credentials}'
68             refspec: '$GERRIT_REFSPEC'
69             choosing-strategy: 'gerrit'
70
71     wrappers:
72         - ssh-agent-credentials:
73             user: '{ssh-credentials}'
74
75     triggers:
76         - gerrit:
77             trigger-on:
78                 - patchset-created-event:
79                     exclude-drafts: 'false'
80                     exclude-trivial-rebase: 'false'
81                     exclude-no-code-change: 'false'
82                 - draft-published-event
83                 - comment-added-contains-event:
84                     comment-contains-value: 'recheck'
85                 - comment-added-contains-event:
86                     comment-contains-value: 'reverify'
87             projects:
88               - project-compare-type: 'ANT'
89                 project-pattern: 'genesis'
90                 branches:
91                   - branch-compare-type: 'ANT'
92                     branch-pattern: '**/{branch}'
93                 file-paths:
94                   - compare-type: ANT
95                     pattern: 'common/ci/**'
96                   - compare-type: ANT
97                     pattern: 'common/manifests/**'
98                   - compare-type: ANT
99                     pattern: 'common/puppet-opnfv/**'
100                   - compare-type: ANT
101                     pattern: 'common/tools/**'
102                   - compare-type: ANT
103                     pattern: '{installer}/build/**'
104                   - compare-type: ANT
105                     pattern: '{installer}/ci/**'
106                   - compare-type: ANT
107                     pattern: '{installer}/deploy/**'
108                   - compare-type: ANT
109                     pattern: '{installer}/include/**'
110                   - compare-type: ANT
111                     pattern: '{installer}/prototypes/**'
112                 forbidden-file-paths:
113                   - compare-type: ANT
114                     pattern: 'common/docs/**'
115                   - compare-type: ANT
116                     pattern: '{installer}/docs/**'
117
118     builders:
119         - 'fuel-build'
120         - 'fuel-workspace-cleanup'
121
122 - job-template:
123     name: 'genesis-fuel-merge-{stream}'
124
125     project-type: freestyle
126
127     node: ericsson-build
128
129     concurrent: true
130
131     properties:
132         - throttle:
133             enabled: true
134             max-total: 2
135
136     logrotate:
137         daysToKeep: 30
138         numToKeep: 40
139         artifactDaysToKeep: -1
140         artifactNumToKeep: 5
141
142     parameters:
143         - project-parameter:
144             project: '{project}'
145         - gerrit-parameter:
146             branch: '{branch}'
147         - fuel-parameter:
148             installer: '{installer}'
149             gs-pathname: ''
150
151     scm:
152         - gerrit-trigger-scm:
153             credentials-id: '{ssh-credentials}'
154             refspec: ''
155             choosing-strategy: 'default'
156
157     wrappers:
158         - ssh-agent-credentials:
159             user: '{ssh-credentials}'
160
161     triggers:
162         - gerrit:
163             trigger-on:
164                 - change-merged-event
165                 - comment-added-contains-event:
166                     comment-contains-value: 'remerge'
167             projects:
168               - project-compare-type: 'ANT'
169                 project-pattern: 'genesis'
170                 branches:
171                   - branch-compare-type: 'ANT'
172                     branch-pattern: '**/{branch}'
173                 file-paths:
174                   - compare-type: ANT
175                     pattern: 'common/ci/**'
176                   - compare-type: ANT
177                     pattern: 'common/manifests/**'
178                   - compare-type: ANT
179                     pattern: 'common/puppet-opnfv/**'
180                   - compare-type: ANT
181                     pattern: 'common/tools/**'
182                   - compare-type: ANT
183                     pattern: '{installer}/build/**'
184                   - compare-type: ANT
185                     pattern: '{installer}/ci/**'
186                   - compare-type: ANT
187                     pattern: '{installer}/deploy/**'
188                   - compare-type: ANT
189                     pattern: '{installer}/include/**'
190                   - compare-type: ANT
191                     pattern: '{installer}/prototypes/**'
192                 forbidden-file-paths:
193                   - compare-type: ANT
194                     pattern: 'common/docs/**'
195                   - compare-type: ANT
196                     pattern: '{installer}/docs/**'
197
198     builders:
199         - 'fuel-build'
200         - 'fuel-workspace-cleanup'
201
202 - job-template:
203     name: 'genesis-fuel-daily-{stream}'
204
205     project-type: freestyle
206
207     node: ericsson-build
208
209     disabled: false
210
211     triggers:
212         - 'fuel-{stream}'
213
214     parameters:
215         - project-parameter:
216             project: '{project}'
217         - fuel-parameter:
218             installer: '{installer}'
219             gs-pathname: '{gs-pathname}'
220
221     scm:
222         - git-scm:
223             credentials-id: '{ssh-credentials}'
224             refspec: ''
225             branch: '{branch}'
226
227     logrotate:
228         daysToKeep: 30
229         numToKeep: 10
230         artifactDaysToKeep: -1
231         artifactNumToKeep: -1
232
233     builders:
234         - trigger-builds:
235           - project: 'genesis-fuel-build-{stream}'
236             git-revision: true
237             block: true
238         - trigger-builds:
239           - project: 'genesis-fuel-lab-reconfig-{stream}'
240             git-revision: true
241             block: true
242         - trigger-builds:
243           - project: 'genesis-fuel-deploy-{stream}'
244             git-revision: true
245             block: true
246         - trigger-builds:
247           - project: 'functest-fuel-{stream}'
248             block: true
249             block-thresholds:
250                 build-step-failure-threshold: 'never'
251                 failure-threshold: 'never'
252                 unstable-threshold: 'FAILURE'
253         - trigger-builds:
254           - project: 'yardstick-fuel-{stream}'
255             block: true
256             block-thresholds:
257                 build-step-failure-threshold: 'never'
258                 failure-threshold: 'never'
259                 unstable-threshold: 'FAILURE'
260
261 - job-template:
262     name: 'genesis-fuel-deploy-runner-{stream}'
263
264     project-type: freestyle
265
266     disabled: false
267
268     node: master
269
270     logrotate:
271         daysToKeep: 30
272         numToKeep: 10
273         artifactDaysToKeep: -1
274         artifactNumToKeep: -1
275
276     parameters:
277         - project-parameter:
278             project: '{project}'
279         - foreman-parameter:
280             installer: '{installer}'
281             gs-pathname: '{gs-pathname}'
282         - string:
283             name: GIT_BASE
284             default: ssh://gerrit.opnfv.org:29418/$PROJECT
285             description: "Used for overriding the GIT URL coming from parameters macro."
286
287     scm:
288         - git-scm:
289             credentials-id: '{ssh-credentials}'
290             refspec: ''
291             branch: '{branch}'
292
293     properties:
294         - build-blocker:
295             use-build-blocker: true
296             blocking-jobs:
297                 - "genesis-foreman-daily.*"
298                 - "genesis-foreman-deploy-runner-{stream}"
299                 - "genesis-fuel-daily.*"
300
301     builders:
302         - 'fuel-lab-reconfig'
303         - trigger-builds:
304           - project: 'genesis-fuel-deploy-{stream}'
305             git-revision: true
306             block: true
307
308 - job-template:
309     name: 'genesis-fuel-build-{stream}'
310
311     project-type: freestyle
312
313     node: ericsson-build
314
315     logrotate:
316         daysToKeep: 30
317         numToKeep: 10
318         artifactDaysToKeep: -1
319         artifactNumToKeep: -1
320
321     parameters:
322         - project-parameter:
323             project: '{project}'
324         - fuel-parameter:
325             installer: '{installer}'
326             gs-pathname: '{gs-pathname}'
327
328     scm:
329         - git-scm:
330             credentials-id: '{ssh-credentials}'
331             refspec: ''
332             branch: '{branch}'
333
334     builders:
335         - 'fuel-build'
336         - 'fuel-upload-artifact'
337         - 'fuel-workspace-cleanup'
338
339     publishers:
340         - email:
341             recipients: jonas.bjurel@ericsson.com stefan.k.berg@ericsson.com
342
343 - job-template:
344     name: 'genesis-fuel-deploy-{stream}'
345
346     project-type: freestyle
347
348     disabled: false
349
350     node: opnfv-jump-2
351
352     parameters:
353         - project-parameter:
354             project: '{project}'
355         - fuel-parameter:
356             installer: '{installer}'
357             gs-pathname: '{gs-pathname}'
358         - string:
359             name: GIT_BASE
360             default: ssh://gerrit.opnfv.org:29418/$PROJECT
361             description: "POD2 has some issues with cloning using https so that's why GIT_BASE is overriden here again."
362
363     scm:
364         - git-scm:
365             credentials-id: '{ssh-credentials}'
366             refspec: ''
367             branch: '{branch}'
368
369     logrotate:
370         daysToKeep: 30
371         numToKeep: 10
372         artifactDaysToKeep: -1
373         artifactNumToKeep: -1
374
375     builders:
376         - 'fuel-download-artifact'
377         - 'fuel-opendaylight-plugin-build'
378         - 'fuel-deploy'
379
380     publishers:
381         - email:
382             recipients: jonas.bjurel@ericsson.com stefan.k.berg@ericsson.com
383
384 - job-template:
385     name: 'genesis-fuel-lab-reconfig-{stream}'
386
387     project-type: freestyle
388
389     disabled: false
390
391     node: master
392
393     logrotate:
394         daysToKeep: 30
395         numToKeep: 10
396         artifactDaysToKeep: -1
397         artifactNumToKeep: -1
398
399     parameters:
400         - project-parameter:
401             project: '{project}'
402         - fuel-parameter:
403             installer: '{installer}'
404             gs-pathname: '{gs-pathname}'
405         - string:
406             name: GIT_BASE
407             default: ssh://gerrit.opnfv.org:29418/$PROJECT
408             description: "Used for overriding the GIT URL coming from parameters macro."
409
410     scm:
411         - git-scm:
412             credentials-id: '{ssh-credentials}'
413             refspec: ''
414             branch: '{branch}'
415
416     properties:
417         - build-blocker:
418             use-build-blocker: true
419             blocking-jobs:
420                 - "genesis-foreman-daily.*"
421
422     builders:
423         - 'fuel-lab-reconfig'
424
425 - job-template:
426     name: 'genesis-fuel-verify-deploy-virtual-{stream}'
427
428     project-type: freestyle
429
430     disabled: true
431
432     node: ericsson-build
433
434     parameters:
435         - project-parameter:
436             project: '{project}'
437         - fuel-parameter:
438             installer: '{installer}'
439             gs-pathname: ''
440
441     scm:
442         - git-scm:
443             credentials-id: '{ssh-credentials}'
444             refspec: ''
445             branch: 'master'
446
447     logrotate:
448         daysToKeep: 30
449         numToKeep: 10
450         artifactDaysToKeep: -1
451         artifactNumToKeep: -1
452
453     builders:
454         - 'fuel-deploy-virtual'
455
456 ########################
457 # parameter macros
458 ########################
459 - parameter:
460     name: fuel-parameter
461     parameters:
462         - string:
463             name: INSTALLER
464             default: '{installer}'
465             description: "Installer to use."
466         - string:
467             name: BUILD_DIRECTORY
468             default: $WORKSPACE/build_output
469             description: "Directory where the build artifact will be located upon the completion of the build."
470         - string:
471             name: CACHE_DIRECTORY
472             default: $HOME/opnfv/cache/genesis-$INSTALLER
473             description: "Directory where the cache to be used during the build is located."
474         - string:
475             name: GIT_BASE
476             default: https://gerrit.opnfv.org/gerrit/$PROJECT
477             description: "Used for overriding the GIT URL coming from Global Jenkins configuration in case if the stuff is done on none-LF HW."
478         - string:
479             name: GS_URL
480             default: artifacts.opnfv.org/$PROJECT/$INSTALLER{gs-pathname}
481             description: "URL to Google Storage."
482
483 ########################
484 # builder macros
485 ########################
486 - builder:
487     name: 'fuel-build'
488     builders:
489         - shell: |
490             #!/bin/bash
491             set -o errexit
492             set -o nounset
493             set -o pipefail
494
495             # log info to console
496             echo "Starting the build of $INSTALLER. This could take some time..."
497             echo "--------------------------------------------------------"
498             echo
499
500             # create the cache directory if it doesn't exist
501             [[ -d $CACHE_DIRECTORY ]] || mkdir -p $CACHE_DIRECTORY
502
503             # set OPNFV_ARTIFACT_VERSION
504             export OPNFV_ARTIFACT_VERSION=$(date -u +"%Y-%m-%d_%H-%M-%S")
505
506             # start the build
507             cd $WORKSPACE/$INSTALLER/ci
508             ./build.sh -v $OPNFV_ARTIFACT_VERSION -c file://$CACHE_DIRECTORY $BUILD_DIRECTORY
509
510             # list the build artifacts
511             ls -al $BUILD_DIRECTORY
512
513             # save information regarding artifact into file
514             (
515                 echo "OPNFV_ARTIFACT_VERSION=$OPNFV_ARTIFACT_VERSION"
516                 echo "OPNFV_GIT_URL=$(git config --get remote.origin.url)"
517                 echo "OPNFV_GIT_SHA1=$(git rev-parse HEAD)"
518                 echo "OPNFV_ARTIFACT_URL=$GS_URL/opnfv-$OPNFV_ARTIFACT_VERSION.iso"
519                 echo "OPNFV_ARTIFACT_MD5SUM=$(md5sum $BUILD_DIRECTORY/opnfv-$OPNFV_ARTIFACT_VERSION.iso | cut -d' ' -f1)"
520                 echo "OPNFV_BUILD_URL=$BUILD_URL"
521             ) > $WORKSPACE/opnfv.properties
522
523             echo
524             echo "--------------------------------------------------------"
525             echo "Done!"
526
527 - builder:
528     name: 'fuel-opendaylight-plugin-build'
529     builders:
530         - shell: |
531             #!/bin/bash
532             set -o errexit
533             set -o nounset
534             set -o pipefail
535
536             if [[ ! $(rpm -qa | grep rubygems) ]]; then
537                 echo "Rubygems not found, but required for Fuel Plugin build ... attempting to install"
538                 sudo yum install -y rubygems ruby-devel
539             fi
540
541             if [[ ! $(gem list | grep fpm) ]]; then
542                 echo "fpm gem not found, but required for Fuel Plugin build ... attempting to install"
543                 sudo gem install fpm
544             fi
545
546             # log info to console
547             echo "Starting the build of Fuel Opendaylight Plugin. This could take some time..."
548             echo "--------------------------------------------------------"
549             echo
550
551             export ODL_PLUGIN_BRANCH="juno/lithium-sr1"
552
553             cd $WORKSPACE
554             git clone https://github.com/stackforge/fuel-plugin-opendaylight
555
556             pushd $WORKSPACE/fuel-plugin-opendaylight
557             git checkout $ODL_PLUGIN_BRANCH
558             popd
559
560             fpb --build fuel-plugin-opendaylight/
561
562             echo
563             echo "--------------------------------------------------------"
564             echo "Done!"
565
566 - builder:
567     name: 'fuel-deploy'
568     builders:
569         - shell: |
570             #!/bin/bash
571             set -o errexit
572             set -o nounset
573             set -o pipefail
574
575             # source the file so we get OPNFV vars
576             source latest.properties
577
578             # echo the info about artifact that is used during the deployment
579             echo "Using $(echo $OPNFV_ARTIFACT_URL | cut -d'/' -f4) for deployment"
580
581             # create TMPDIR if it doesn't exist
582             export TMPDIR=$HOME/tmpdir
583             [[ -d $TMPDIR ]] || mkdir -p $TMPDIR
584
585             # change permissions down to TMPDIR
586             chmod a+x $HOME
587             chmod a+x $TMPDIR
588
589             # set CONFDIR, BRIDGE
590             export CONFDIR=$WORKSPACE/fuel/deploy/templates/hardware_environment/conf/linux_foundation_lab/pod2
591             export BRIDGE=pxebr
592             export PLUGIN=$WORKSPACE/fuel-plugin-opendaylight
593
594             # cleanup first
595             sudo $WORKSPACE/common/ci/clean.sh -base_config $WORKSPACE/foreman/ci/inventory/lf_pod2_ksgen_settings.yml
596
597             # prepare for Fuel Deployment
598             sudo $WORKSPACE/common/ci/setup.sh
599
600             # log info to console
601             echo "Starting the deployment using $INSTALLER. This could take some time..."
602             echo "--------------------------------------------------------"
603             echo
604
605             # start the deployment
606             echo "Issuing command"
607             echo "sudo $WORKSPACE/fuel/ci/deploy.sh -iso $WORKSPACE/opnfv.iso -dea $CONFDIR/dea.yaml -dha $CONFDIR/dha.yaml -s $TMPDIR -b $BRIDGE -p $PLUGIN"
608             sudo $WORKSPACE/fuel/ci/deploy.sh -iso $WORKSPACE/opnfv.iso -dea $CONFDIR/dea.yaml -dha $CONFDIR/dha.yaml -s $TMPDIR -b $BRIDGE -p $PLUGIN
609
610             echo
611             echo "--------------------------------------------------------"
612             echo "Done!"
613
614 - builder:
615     name: 'fuel-deploy-virtual'
616     builders:
617         - shell: |
618             #!/bin/bash
619             set -o errexit
620             set -o nounset
621             set -o pipefail
622
623             # log info to console
624             echo "Starting the deployment on virtual environment using $INSTALLER. This could take some time..."
625             echo "--------------------------------------------------------"
626             echo
627
628             ssh -o BatchMode=yes -o TCPKeepAlive=yes cideploy@10.118.34.205 ./cideploy.sh
629
630             echo
631             echo "--------------------------------------------------------"
632             echo "Done!"
633
634 - builder:
635     name: 'fuel-upload-artifact'
636     builders:
637         - shell: |
638             #!/bin/bash
639             set -o errexit
640             set -o nounset
641             set -o pipefail
642
643             # log info to console
644             echo "Uploading the $INSTALLER artifact. This could take some time..."
645             echo "--------------------------------------------------------"
646             echo
647
648             # source the opnfv.properties to get ARTIFACT_VERSION
649             source $WORKSPACE/opnfv.properties
650
651             # upload artifact and additional files to google storage
652             gsutil cp $BUILD_DIRECTORY/opnfv-$OPNFV_ARTIFACT_VERSION.iso gs://$GS_URL/opnfv-$OPNFV_ARTIFACT_VERSION.iso > gsutil.iso.log 2>&1
653             gsutil cp $WORKSPACE/opnfv.properties gs://$GS_URL/opnfv-$OPNFV_ARTIFACT_VERSION.properties > gsutil.properties.log 2>&1
654             gsutil cp $WORKSPACE/opnfv.properties gs://$GS_URL/latest.properties > gsutil.latest.log 2>&1
655
656             echo
657             echo "--------------------------------------------------------"
658             echo "Done!"
659             echo "Artifact is available as http://$GS_URL/opnfv-$OPNFV_ARTIFACT_VERSION.iso"
660
661 - builder:
662     name: 'fuel-download-artifact'
663     builders:
664         - shell: |
665             #!/bin/bash
666             set -o errexit
667             set -o nounset
668             set -o pipefail
669
670             # get the latest.properties file in order to get info regarding latest artifact
671             curl -s -o $WORKSPACE/latest.properties http://$GS_URL/latest.properties
672
673             # check if we got the file
674             [[ -f latest.properties ]] || exit 1
675
676             # source the file so we get OPNFV vars
677             source latest.properties
678
679             # log info to console
680             echo "Downloading the $INSTALLER artifact using URL http://$OPNFV_ARTIFACT_URL"
681             echo "This could take some time..."
682             echo "--------------------------------------------------------"
683             echo
684
685             # download the file
686             curl -s -o $WORKSPACE/opnfv.iso http://$OPNFV_ARTIFACT_URL > gsutil.iso.log 2>&1
687
688             # list the file
689             ls -al $WORKSPACE/opnfv.iso
690
691             echo
692             echo "--------------------------------------------------------"
693             echo "Done!"
694
695 - builder:
696     name: 'fuel-workspace-cleanup'
697     builders:
698         - shell: |
699             #!/bin/bash
700             set -o errexit
701             set -o nounset
702             set -o pipefail
703
704             # delete everything that is in $WORKSPACE
705             /bin/rm -rf $WORKSPACE
706
707 - builder:
708     name: 'fuel-lab-reconfig'
709     builders:
710         - shell: |
711             #!/bin/bash
712             set -o errexit
713             set -o nounset
714             set -o pipefail
715
716             # check to see if ucs login info file exists
717             if [ -e ~/.ssh/ucs_creds ];then
718               source ~/.ssh/ucs_creds
719             else
720               echo "Unable to find UCS credentials for LF lab reconfiguration...Exiting"
721               exit 1
722             fi
723
724             # clone releng
725             echo "Cloning releng repo..."
726             if ! GIT_SSL_NO_VERIFY=true git clone https://gerrit.opnfv.org/gerrit/releng; then
727               echo "Unable to clone releng repo...Exiting"
728               exit 1
729             fi
730
731             # log info to console
732             echo "Starting the lab reconfiguration for $INSTALLER..."
733             echo "--------------------------------------------------------"
734             echo
735
736             # create venv
737             $WORKSPACE/releng/utils/lab-reconfiguration/create_venv.sh
738
739             # disable nounset because 'activate' script contains unbound variable(s)
740             set +o nounset
741             # enter venv
742             source $WORKSPACE/releng/utils/lab-reconfiguration/venv/bin/activate
743             # set nounset back again
744             set -o nounset
745
746             # verify we are in venv
747             if [[ ! $(which python | grep venv) ]]; then
748               echo "Unable to activate venv...Exiting"
749               exit 1
750             fi
751
752             python $WORKSPACE/releng/utils/lab-reconfiguration/reconfigUcsNet.py -i $ucs_host -u $ucs_user -p $ucs_password -f $WORKSPACE/releng/utils/lab-reconfiguration/fuel.yaml
753
754             # while undergoing reboot
755             sleep 30
756
757             # check to see if slave is back up
758             ping_counter=0
759             ping_flag=0
760             while [ "$ping_counter" -lt 20 ]; do
761               if [[ $(ping -c 5 172.30.10.72) ]]; then
762                 ping_flag=1
763                 break
764               fi
765               ((ping_counter++))
766               sleep 10
767             done
768
769             if [ "$ping_flag" -eq 1 ]; then
770               echo "Slave is pingable, now wait 180 seconds for services to start"
771               sleep 180
772             else
773               echo "Slave did not come back up after reboot: please check opnfv-jump-2"
774               exit 1
775             fi
776
777             set +o nounset
778             deactivate
779
780             echo
781             echo "--------------------------------------------------------"
782             echo "Done!"
783
784 ########################
785 # trigger macros
786 ########################
787 - trigger:
788     name: 'fuel-master'
789     triggers:
790         - timed: '0 3 * * *'
791
792 - trigger:
793     name: 'fuel-stable-arno'
794     triggers:
795         - timed: '0 7 * * *'