New job for Fuel Plugin Build
[releng.git] / jjb / genesis / genesis-fuel.yml
1 ########################
2 # Job configuration for genesis/fuel
3 ########################
4 - project:
5
6     name: genesis-fuel
7
8     project: 'genesis'
9
10     installer: 'fuel'
11
12     jobs:
13         - 'genesis-fuel-verify-build-{stream}'
14         - 'genesis-fuel-merge-{stream}'
15         - 'genesis-fuel-daily-{stream}'
16         - 'genesis-fuel-build-{stream}'
17         - 'genesis-fuel-deploy-{stream}'
18         - 'genesis-fuel-deploy-runner-{stream}'
19         - 'genesis-fuel-lab-reconfig-{stream}'
20         - 'genesis-fuel-verify-deploy-virtual-{stream}'
21
22     # stream:    branch with - in place of / (eg. stable-arno)
23     # branch:    branch (eg. stable/arno)
24     stream:
25         - master:
26             branch: 'master'
27             gs-pathname: ''
28         - stable-arno:
29             branch: 'stable/arno'
30             gs-pathname: '/arno'
31
32 ########################
33 # job templates
34 ########################
35
36 - job-template:
37     name: 'genesis-fuel-verify-build-{stream}'
38
39     project-type: freestyle
40
41     node: ericsson-build
42
43     concurrent: true
44
45     properties:
46         - throttle:
47             enabled: true
48             max-total: 3
49
50     logrotate:
51         daysToKeep: 30
52         numToKeep: 10
53         artifactDaysToKeep: -1
54         artifactNumToKeep: -1
55
56     parameters:
57         - project-parameter:
58             project: '{project}'
59         - gerrit-parameter:
60             branch: '{branch}'
61         - fuel-parameter:
62             installer: '{installer}'
63             gs-pathname: '{gs-pathname}'
64
65     scm:
66         - gerrit-trigger-scm:
67             credentials-id: '{ssh-credentials}'
68             refspec: '$GERRIT_REFSPEC'
69             choosing-strategy: 'gerrit'
70
71     wrappers:
72         - ssh-agent-credentials:
73             user: '{ssh-credentials}'
74
75     triggers:
76         - gerrit:
77             trigger-on:
78                 - patchset-created-event:
79                     exclude-drafts: 'false'
80                     exclude-trivial-rebase: 'false'
81                     exclude-no-code-change: 'false'
82                 - draft-published-event
83                 - comment-added-contains-event:
84                     comment-contains-value: 'recheck'
85                 - comment-added-contains-event:
86                     comment-contains-value: 'reverify'
87             projects:
88               - project-compare-type: 'ANT'
89                 project-pattern: 'genesis'
90                 branches:
91                   - branch-compare-type: 'ANT'
92                     branch-pattern: '**/{branch}'
93                 file-paths:
94                   - compare-type: ANT
95                     pattern: 'common/ci/**'
96                   - compare-type: ANT
97                     pattern: 'common/manifests/**'
98                   - compare-type: ANT
99                     pattern: 'common/puppet-opnfv/**'
100                   - compare-type: ANT
101                     pattern: 'common/tools/**'
102                   - compare-type: ANT
103                     pattern: '{installer}/build/**'
104                   - compare-type: ANT
105                     pattern: '{installer}/ci/**'
106                   - compare-type: ANT
107                     pattern: '{installer}/deploy/**'
108                   - compare-type: ANT
109                     pattern: '{installer}/include/**'
110                   - compare-type: ANT
111                     pattern: '{installer}/prototypes/**'
112                 forbidden-file-paths:
113                   - compare-type: ANT
114                     pattern: 'common/docs/**'
115                   - compare-type: ANT
116                     pattern: '{installer}/docs/**'
117
118     builders:
119         - 'fuel-build'
120         - 'fuel-workspace-cleanup'
121
122 - job-template:
123     name: 'genesis-fuel-merge-{stream}'
124
125     project-type: freestyle
126
127     node: ericsson-build
128
129     concurrent: true
130
131     properties:
132         - throttle:
133             enabled: true
134             max-total: 2
135
136     logrotate:
137         daysToKeep: 30
138         numToKeep: 40
139         artifactDaysToKeep: -1
140         artifactNumToKeep: 5
141
142     parameters:
143         - project-parameter:
144             project: '{project}'
145         - gerrit-parameter:
146             branch: '{branch}'
147         - fuel-parameter:
148             installer: '{installer}'
149             gs-pathname: ''
150
151     scm:
152         - gerrit-trigger-scm:
153             credentials-id: '{ssh-credentials}'
154             refspec: ''
155             choosing-strategy: 'default'
156
157     wrappers:
158         - ssh-agent-credentials:
159             user: '{ssh-credentials}'
160
161     triggers:
162         - gerrit:
163             trigger-on:
164                 - change-merged-event
165                 - comment-added-contains-event:
166                     comment-contains-value: 'remerge'
167             projects:
168               - project-compare-type: 'ANT'
169                 project-pattern: 'genesis'
170                 branches:
171                   - branch-compare-type: 'ANT'
172                     branch-pattern: '**/{branch}'
173                 file-paths:
174                   - compare-type: ANT
175                     pattern: 'common/ci/**'
176                   - compare-type: ANT
177                     pattern: 'common/manifests/**'
178                   - compare-type: ANT
179                     pattern: 'common/puppet-opnfv/**'
180                   - compare-type: ANT
181                     pattern: 'common/tools/**'
182                   - compare-type: ANT
183                     pattern: '{installer}/build/**'
184                   - compare-type: ANT
185                     pattern: '{installer}/ci/**'
186                   - compare-type: ANT
187                     pattern: '{installer}/deploy/**'
188                   - compare-type: ANT
189                     pattern: '{installer}/include/**'
190                   - compare-type: ANT
191                     pattern: '{installer}/prototypes/**'
192                 forbidden-file-paths:
193                   - compare-type: ANT
194                     pattern: 'common/docs/**'
195                   - compare-type: ANT
196                     pattern: '{installer}/docs/**'
197
198     builders:
199         - 'fuel-build'
200         - 'fuel-workspace-cleanup'
201
202 - job-template:
203     name: 'genesis-fuel-daily-{stream}'
204
205     project-type: freestyle
206
207     node: ericsson-build
208
209     disabled: false
210
211     triggers:
212         - 'fuel-{stream}'
213
214     parameters:
215         - project-parameter:
216             project: '{project}'
217         - fuel-parameter:
218             installer: '{installer}'
219             gs-pathname: '{gs-pathname}'
220
221     scm:
222         - git-scm:
223             credentials-id: '{ssh-credentials}'
224             refspec: ''
225             branch: '{branch}'
226
227     logrotate:
228         daysToKeep: 30
229         numToKeep: 10
230         artifactDaysToKeep: -1
231         artifactNumToKeep: -1
232
233     builders:
234         - trigger-builds:
235           - project: 'genesis-fuel-build-{stream}'
236             git-revision: true
237             block: true
238         - trigger-builds:
239           - project: 'genesis-fuel-lab-reconfig-{stream}'
240             git-revision: true
241             block: true
242         - trigger-builds:
243           - project: 'genesis-fuel-deploy-{stream}'
244             git-revision: true
245             block: true
246         - trigger-builds:
247           - project: 'functest-fuel-{stream}'
248             block: true
249             block-thresholds:
250                 build-step-failure-threshold: 'never'
251                 failure-threshold: 'never'
252                 unstable-threshold: 'FAILURE'
253         - trigger-builds:
254           - project: 'yardstick-fuel-{stream}'
255             block: true
256             block-thresholds:
257                 build-step-failure-threshold: 'never'
258                 failure-threshold: 'never'
259                 unstable-threshold: 'FAILURE'
260
261 - job-template:
262     name: 'genesis-fuel-deploy-runner-{stream}'
263
264     project-type: freestyle
265
266     disabled: false
267
268     node: master
269
270     logrotate:
271         daysToKeep: 30
272         numToKeep: 10
273         artifactDaysToKeep: -1
274         artifactNumToKeep: -1
275
276     parameters:
277         - project-parameter:
278             project: '{project}'
279         - foreman-parameter:
280             installer: '{installer}'
281             gs-pathname: '{gs-pathname}'
282         - string:
283             name: GIT_BASE
284             default: ssh://gerrit.opnfv.org:29418/$PROJECT
285             description: "Used for overriding the GIT URL coming from parameters macro."
286
287     scm:
288         - git-scm:
289             credentials-id: '{ssh-credentials}'
290             refspec: ''
291             branch: '{branch}'
292
293     properties:
294         - build-blocker:
295             use-build-blocker: true
296             blocking-jobs:
297                 - "genesis-foreman-daily.*"
298                 - "genesis-foreman-deploy-runner-{stream}"
299                 - "genesis-fuel-daily.*"
300
301     builders:
302         - 'fuel-lab-reconfig'
303         - trigger-builds:
304           - project: 'genesis-fuel-deploy-{stream}'
305             git-revision: true
306             block: true
307
308 - job-template:
309     name: 'genesis-fuel-build-{stream}'
310
311     project-type: freestyle
312
313     node: ericsson-build
314
315     logrotate:
316         daysToKeep: 30
317         numToKeep: 10
318         artifactDaysToKeep: -1
319         artifactNumToKeep: -1
320
321     parameters:
322         - project-parameter:
323             project: '{project}'
324         - fuel-parameter:
325             installer: '{installer}'
326             gs-pathname: '{gs-pathname}'
327
328     scm:
329         - git-scm:
330             credentials-id: '{ssh-credentials}'
331             refspec: ''
332             branch: '{branch}'
333
334     builders:
335         - 'fuel-build'
336         - 'fuel-upload-artifact'
337         - 'fuel-workspace-cleanup'
338
339     publishers:
340         - email:
341             recipients: jonas.bjurel@ericsson.com stefan.k.berg@ericsson.com
342
343 - job-template:
344     name: 'genesis-fuel-deploy-{stream}'
345
346     project-type: freestyle
347
348     disabled: false
349
350     node: opnfv-jump-2
351
352     parameters:
353         - project-parameter:
354             project: '{project}'
355         - fuel-parameter:
356             installer: '{installer}'
357             gs-pathname: '{gs-pathname}'
358         - string:
359             name: GIT_BASE
360             default: ssh://gerrit.opnfv.org:29418/$PROJECT
361             description: "POD2 has some issues with cloning using https so that's why GIT_BASE is overriden here again."
362
363     scm:
364         - git-scm:
365             credentials-id: '{ssh-credentials}'
366             refspec: ''
367             branch: '{branch}'
368
369     logrotate:
370         daysToKeep: 30
371         numToKeep: 10
372         artifactDaysToKeep: -1
373         artifactNumToKeep: -1
374
375     builders:
376         - 'fuel-download-artifact'
377         - 'fuel-opendaylight-plugin-build'
378         - 'fuel-deploy'
379
380     publishers:
381         - email:
382             recipients: jonas.bjurel@ericsson.com stefan.k.berg@ericsson.com
383
384 - job-template:
385     name: 'genesis-fuel-lab-reconfig-{stream}'
386
387     project-type: freestyle
388
389     disabled: false
390
391     node: master
392
393     logrotate:
394         daysToKeep: 30
395         numToKeep: 10
396         artifactDaysToKeep: -1
397         artifactNumToKeep: -1
398
399     parameters:
400         - project-parameter:
401             project: '{project}'
402         - fuel-parameter:
403             installer: '{installer}'
404             gs-pathname: '{gs-pathname}'
405         - string:
406             name: GIT_BASE
407             default: ssh://gerrit.opnfv.org:29418/$PROJECT
408             description: "Used for overriding the GIT URL coming from parameters macro."
409
410     scm:
411         - git-scm:
412             credentials-id: '{ssh-credentials}'
413             refspec: ''
414             branch: '{branch}'
415
416     properties:
417         - build-blocker:
418             use-build-blocker: true
419             blocking-jobs:
420                 - "genesis-foreman-daily.*"
421
422     builders:
423         - 'fuel-lab-reconfig'
424
425 - job-template:
426     name: 'genesis-fuel-verify-deploy-virtual-{stream}'
427
428     project-type: freestyle
429
430     disabled: true
431
432     node: ericsson-build
433
434     parameters:
435         - project-parameter:
436             project: '{project}'
437         - fuel-parameter:
438             installer: '{installer}'
439             gs-pathname: ''
440
441     scm:
442         - git-scm:
443             credentials-id: '{ssh-credentials}'
444             refspec: ''
445             branch: 'master'
446
447     logrotate:
448         daysToKeep: 30
449         numToKeep: 10
450         artifactDaysToKeep: -1
451         artifactNumToKeep: -1
452
453     builders:
454         - 'fuel-deploy-virtual'
455
456 ########################
457 # parameter macros
458 ########################
459 - parameter:
460     name: fuel-parameter
461     parameters:
462         - string:
463             name: INSTALLER
464             default: '{installer}'
465             description: "Installer to use."
466         - string:
467             name: BUILD_DIRECTORY
468             default: $WORKSPACE/build_output
469             description: "Directory where the build artifact will be located upon the completion of the build."
470         - string:
471             name: CACHE_DIRECTORY
472             default: $HOME/opnfv/cache/genesis-$INSTALLER
473             description: "Directory where the cache to be used during the build is located."
474         - string:
475             name: GIT_BASE
476             default: https://gerrit.opnfv.org/gerrit/$PROJECT
477             description: "Used for overriding the GIT URL coming from Global Jenkins configuration in case if the stuff is done on none-LF HW."
478         - string:
479             name: GS_URL
480             default: artifacts.opnfv.org/$PROJECT/$INSTALLER{gs-pathname}
481             description: "URL to Google Storage."
482
483 ########################
484 # builder macros
485 ########################
486 - builder:
487     name: 'fuel-build'
488     builders:
489         - shell: |
490             #!/bin/bash
491             set -o errexit
492             set -o nounset
493             set -o pipefail
494
495             # log info to console
496             echo "Starting the build of $INSTALLER. This could take some time..."
497             echo "--------------------------------------------------------"
498             echo
499
500             # create the cache directory if it doesn't exist
501             [[ -d $CACHE_DIRECTORY ]] || mkdir -p $CACHE_DIRECTORY
502
503             # set OPNFV_ARTIFACT_VERSION
504             export OPNFV_ARTIFACT_VERSION=$(date -u +"%Y-%m-%d_%H-%M-%S")
505
506             # start the build
507             cd $WORKSPACE/$INSTALLER/ci
508             ./build.sh -v $OPNFV_ARTIFACT_VERSION -c file://$CACHE_DIRECTORY $BUILD_DIRECTORY
509
510             # list the build artifacts
511             ls -al $BUILD_DIRECTORY
512
513             # save information regarding artifact into file
514             (
515                 echo "OPNFV_ARTIFACT_VERSION=$OPNFV_ARTIFACT_VERSION"
516                 echo "OPNFV_GIT_URL=$(git config --get remote.origin.url)"
517                 echo "OPNFV_GIT_SHA1=$(git rev-parse HEAD)"
518                 echo "OPNFV_ARTIFACT_URL=$GS_URL/opnfv-$OPNFV_ARTIFACT_VERSION.iso"
519                 echo "OPNFV_ARTIFACT_MD5SUM=$(md5sum $BUILD_DIRECTORY/opnfv-$OPNFV_ARTIFACT_VERSION.iso | cut -d' ' -f1)"
520                 echo "OPNFV_BUILD_URL=$BUILD_URL"
521             ) > $WORKSPACE/opnfv.properties
522
523             echo
524             echo "--------------------------------------------------------"
525             echo "Done!"
526
527 - builder:
528     name: 'fuel-opendaylight-plugin-build'
529     builders:
530         - shell: |
531             #!/bin/bash
532             set -o errexit
533             set -o nounset
534             set -o pipefail
535
536             # log info to console
537             echo "Starting the build of Fuel Opendaylight Plugin. This could take some time..."
538             echo "--------------------------------------------------------"
539             echo
540
541             export ODL_PLUGIN_BRANCH="juno/lithium-sr1"
542
543             cd $WORKSPACE
544             git clone https://github.com/stackforge/fuel-plugin-opendaylight
545
546             pushd $WORKSPACE/fuel-plugin-opendaylight
547             git checkout $ODL_PLUGIN_BRANCH
548             popd
549
550             fpb --build fuel-plugin-opendaylight/
551
552             echo
553             echo "--------------------------------------------------------"
554             echo "Done!"
555
556 - builder:
557     name: 'fuel-deploy'
558     builders:
559         - shell: |
560             #!/bin/bash
561             set -o errexit
562             set -o nounset
563             set -o pipefail
564
565             # source the file so we get OPNFV vars
566             source latest.properties
567
568             # echo the info about artifact that is used during the deployment
569             echo "Using $(echo $OPNFV_ARTIFACT_URL | cut -d'/' -f4) for deployment"
570
571             # create TMPDIR if it doesn't exist
572             export TMPDIR=$HOME/tmpdir
573             [[ -d $TMPDIR ]] || mkdir -p $TMPDIR
574
575             # change permissions down to TMPDIR
576             chmod a+x $HOME
577             chmod a+x $TMPDIR
578
579             # set CONFDIR, BRIDGE
580             export CONFDIR=$WORKSPACE/fuel/deploy/templates/hardware_environment/conf/linux_foundation_lab/pod2
581             export BRIDGE=pxebr
582             export PLUGIN=$WORKSPACE/fuel-plugin-opendaylight
583
584             # cleanup first
585             sudo $WORKSPACE/common/ci/clean.sh -base_config $WORKSPACE/foreman/ci/inventory/lf_pod2_ksgen_settings.yml
586
587             # prepare for Fuel Deployment
588             sudo $WORKSPACE/common/ci/setup.sh
589
590             # log info to console
591             echo "Starting the deployment using $INSTALLER. This could take some time..."
592             echo "--------------------------------------------------------"
593             echo
594
595             # start the deployment
596             echo "Issuing command"
597             echo "sudo $WORKSPACE/fuel/ci/deploy.sh -iso $WORKSPACE/opnfv.iso -dea $CONFDIR/dea.yaml -dha $CONFDIR/dha.yaml -s $TMPDIR -b $BRIDGE -p $PLUGIN"
598             sudo $WORKSPACE/fuel/ci/deploy.sh -iso $WORKSPACE/opnfv.iso -dea $CONFDIR/dea.yaml -dha $CONFDIR/dha.yaml -s $TMPDIR -b $BRIDGE -p $PLUGIN
599
600             echo
601             echo "--------------------------------------------------------"
602             echo "Done!"
603
604 - builder:
605     name: 'fuel-deploy-virtual'
606     builders:
607         - shell: |
608             #!/bin/bash
609             set -o errexit
610             set -o nounset
611             set -o pipefail
612
613             # log info to console
614             echo "Starting the deployment on virtual environment using $INSTALLER. This could take some time..."
615             echo "--------------------------------------------------------"
616             echo
617
618             ssh -o BatchMode=yes -o TCPKeepAlive=yes cideploy@10.118.34.205 ./cideploy.sh
619
620             echo
621             echo "--------------------------------------------------------"
622             echo "Done!"
623
624 - builder:
625     name: 'fuel-upload-artifact'
626     builders:
627         - shell: |
628             #!/bin/bash
629             set -o errexit
630             set -o nounset
631             set -o pipefail
632
633             # log info to console
634             echo "Uploading the $INSTALLER artifact. This could take some time..."
635             echo "--------------------------------------------------------"
636             echo
637
638             # source the opnfv.properties to get ARTIFACT_VERSION
639             source $WORKSPACE/opnfv.properties
640
641             # upload artifact and additional files to google storage
642             gsutil cp $BUILD_DIRECTORY/opnfv-$OPNFV_ARTIFACT_VERSION.iso gs://$GS_URL/opnfv-$OPNFV_ARTIFACT_VERSION.iso > gsutil.iso.log 2>&1
643             gsutil cp $WORKSPACE/opnfv.properties gs://$GS_URL/opnfv-$OPNFV_ARTIFACT_VERSION.properties > gsutil.properties.log 2>&1
644             gsutil cp $WORKSPACE/opnfv.properties gs://$GS_URL/latest.properties > gsutil.latest.log 2>&1
645
646             echo
647             echo "--------------------------------------------------------"
648             echo "Done!"
649             echo "Artifact is available as http://$GS_URL/opnfv-$OPNFV_ARTIFACT_VERSION.iso"
650
651 - builder:
652     name: 'fuel-download-artifact'
653     builders:
654         - shell: |
655             #!/bin/bash
656             set -o errexit
657             set -o nounset
658             set -o pipefail
659
660             # get the latest.properties file in order to get info regarding latest artifact
661             curl -s -o $WORKSPACE/latest.properties http://$GS_URL/latest.properties
662
663             # check if we got the file
664             [[ -f latest.properties ]] || exit 1
665
666             # source the file so we get OPNFV vars
667             source latest.properties
668
669             # log info to console
670             echo "Downloading the $INSTALLER artifact using URL http://$OPNFV_ARTIFACT_URL"
671             echo "This could take some time..."
672             echo "--------------------------------------------------------"
673             echo
674
675             # download the file
676             curl -s -o $WORKSPACE/opnfv.iso http://$OPNFV_ARTIFACT_URL > gsutil.iso.log 2>&1
677
678             # list the file
679             ls -al $WORKSPACE/opnfv.iso
680
681             echo
682             echo "--------------------------------------------------------"
683             echo "Done!"
684
685 - builder:
686     name: 'fuel-workspace-cleanup'
687     builders:
688         - shell: |
689             #!/bin/bash
690             set -o errexit
691             set -o nounset
692             set -o pipefail
693
694             # delete everything that is in $WORKSPACE
695             /bin/rm -rf $WORKSPACE
696
697 - builder:
698     name: 'fuel-lab-reconfig'
699     builders:
700         - shell: |
701             #!/bin/bash
702             set -o errexit
703             set -o nounset
704             set -o pipefail
705
706             # check to see if ucs login info file exists
707             if [ -e ~/.ssh/ucs_creds ];then
708               source ~/.ssh/ucs_creds
709             else
710               echo "Unable to find UCS credentials for LF lab reconfiguration...Exiting"
711               exit 1
712             fi
713
714             # clone releng
715             echo "Cloning releng repo..."
716             if ! GIT_SSL_NO_VERIFY=true git clone https://gerrit.opnfv.org/gerrit/releng; then
717               echo "Unable to clone releng repo...Exiting"
718               exit 1
719             fi
720
721             # log info to console
722             echo "Starting the lab reconfiguration for $INSTALLER..."
723             echo "--------------------------------------------------------"
724             echo
725
726             # create venv
727             $WORKSPACE/releng/utils/lab-reconfiguration/create_venv.sh
728
729             # disable nounset because 'activate' script contains unbound variable(s)
730             set +o nounset
731             # enter venv
732             source $WORKSPACE/releng/utils/lab-reconfiguration/venv/bin/activate
733             # set nounset back again
734             set -o nounset
735
736             # verify we are in venv
737             if [[ ! $(which python | grep venv) ]]; then
738               echo "Unable to activate venv...Exiting"
739               exit 1
740             fi
741
742             python $WORKSPACE/releng/utils/lab-reconfiguration/reconfigUcsNet.py -i $ucs_host -u $ucs_user -p $ucs_password -f $WORKSPACE/releng/utils/lab-reconfiguration/fuel.yaml
743
744             # while undergoing reboot
745             sleep 30
746
747             # check to see if slave is back up
748             ping_counter=0
749             ping_flag=0
750             while [ "$ping_counter" -lt 20 ]; do
751               if [[ $(ping -c 5 172.30.10.72) ]]; then
752                 ping_flag=1
753                 break
754               fi
755               ((ping_counter++))
756               sleep 10
757             done
758
759             if [ "$ping_flag" -eq 1 ]; then
760               echo "Slave is pingable, now wait 180 seconds for services to start"
761               sleep 180
762             else
763               echo "Slave did not come back up after reboot: please check opnfv-jump-2"
764               exit 1
765             fi
766
767             set +o nounset
768             deactivate
769
770             echo
771             echo "--------------------------------------------------------"
772             echo "Done!"
773
774 ########################
775 # trigger macros
776 ########################
777 - trigger:
778     name: 'fuel-master'
779     triggers:
780         - timed: '0 3 * * *'
781
782 - trigger:
783     name: 'fuel-stable-arno'
784     triggers:
785         - timed: '0 7 * * *'