04dcfcd9047386214b362d5b724f600f90d64bbb
[releng.git] / jjb / genesis / genesis-fuel.yml
1 ########################
2 # Job configuration for genesis/fuel
3 ########################
4 - project:
5
6     name: genesis-fuel
7
8     project: 'genesis'
9
10     installer: 'fuel'
11
12     jobs:
13         - 'genesis-fuel-verify-build-{stream}'
14         - 'genesis-fuel-merge-{stream}'
15         - 'genesis-fuel-daily-{stream}'
16         - 'genesis-fuel-build-{stream}'
17         - 'genesis-fuel-deploy-{stream}'
18         - 'genesis-fuel-deploy-runner-{stream}'
19         - 'genesis-fuel-lab-reconfig-{stream}'
20         - 'genesis-fuel-verify-deploy-virtual-{stream}'
21
22     # stream:    branch with - in place of / (eg. stable-arno)
23     # branch:    branch (eg. stable/arno)
24     stream:
25         - master:
26             branch: 'master'
27             gs-pathname: ''
28             disabled: true
29         - stable-arno:
30             branch: 'stable/arno'
31             gs-pathname: '/arno'
32             disabled: true
33
34 ########################
35 # job templates
36 ########################
37
38 - job-template:
39     name: 'genesis-fuel-verify-build-{stream}'
40
41     project-type: freestyle
42
43     node: ericsson-build
44
45     concurrent: true
46
47     properties:
48         - throttle:
49             enabled: true
50             max-total: 3
51
52     logrotate:
53         daysToKeep: 30
54         numToKeep: 10
55         artifactDaysToKeep: -1
56         artifactNumToKeep: -1
57
58     parameters:
59         - project-parameter:
60             project: '{project}'
61         - gerrit-parameter:
62             branch: '{branch}'
63         - genesis-fuel-parameter:
64             installer: '{installer}'
65             gs-pathname: '{gs-pathname}'
66
67     scm:
68         - gerrit-trigger-scm:
69             credentials-id: '{ssh-credentials}'
70             refspec: '$GERRIT_REFSPEC'
71             choosing-strategy: 'gerrit'
72
73     triggers:
74         - gerrit:
75             trigger-on:
76                 - patchset-created-event:
77                     exclude-drafts: 'false'
78                     exclude-trivial-rebase: 'false'
79                     exclude-no-code-change: 'false'
80                 - draft-published-event
81                 - comment-added-contains-event:
82                     comment-contains-value: 'recheck'
83                 - comment-added-contains-event:
84                     comment-contains-value: 'reverify'
85             projects:
86               - project-compare-type: 'ANT'
87                 project-pattern: 'genesis'
88                 branches:
89                   - branch-compare-type: 'ANT'
90                     branch-pattern: '**/{branch}'
91                 file-paths:
92                   - compare-type: ANT
93                     pattern: 'common/ci/**'
94                   - compare-type: ANT
95                     pattern: 'common/manifests/**'
96                   - compare-type: ANT
97                     pattern: 'common/puppet-opnfv/**'
98                   - compare-type: ANT
99                     pattern: 'common/tools/**'
100                   - compare-type: ANT
101                     pattern: '{installer}/build/**'
102                   - compare-type: ANT
103                     pattern: '{installer}/ci/**'
104                   - compare-type: ANT
105                     pattern: '{installer}/deploy/**'
106                   - compare-type: ANT
107                     pattern: '{installer}/include/**'
108                   - compare-type: ANT
109                     pattern: '{installer}/prototypes/**'
110                 forbidden-file-paths:
111                   - compare-type: ANT
112                     pattern: 'common/docs/**'
113                   - compare-type: ANT
114                     pattern: '{installer}/docs/**'
115
116     builders:
117         - 'fuel-build'
118         - 'fuel-workspace-cleanup'
119
120 - job-template:
121     name: 'genesis-fuel-merge-{stream}'
122
123     project-type: freestyle
124
125     node: ericsson-build
126
127     concurrent: true
128
129     properties:
130         - throttle:
131             enabled: true
132             max-total: 2
133
134     logrotate:
135         daysToKeep: 30
136         numToKeep: 40
137         artifactDaysToKeep: -1
138         artifactNumToKeep: 5
139
140     parameters:
141         - project-parameter:
142             project: '{project}'
143         - gerrit-parameter:
144             branch: '{branch}'
145         - genesis-fuel-parameter:
146             installer: '{installer}'
147             gs-pathname: ''
148
149     scm:
150         - gerrit-trigger-scm:
151             credentials-id: '{ssh-credentials}'
152             refspec: ''
153             choosing-strategy: 'default'
154
155     triggers:
156         - gerrit:
157             trigger-on:
158                 - change-merged-event
159                 - comment-added-contains-event:
160                     comment-contains-value: 'remerge'
161             projects:
162               - project-compare-type: 'ANT'
163                 project-pattern: 'genesis'
164                 branches:
165                   - branch-compare-type: 'ANT'
166                     branch-pattern: '**/{branch}'
167                 file-paths:
168                   - compare-type: ANT
169                     pattern: 'common/ci/**'
170                   - compare-type: ANT
171                     pattern: 'common/manifests/**'
172                   - compare-type: ANT
173                     pattern: 'common/puppet-opnfv/**'
174                   - compare-type: ANT
175                     pattern: 'common/tools/**'
176                   - compare-type: ANT
177                     pattern: '{installer}/build/**'
178                   - compare-type: ANT
179                     pattern: '{installer}/ci/**'
180                   - compare-type: ANT
181                     pattern: '{installer}/deploy/**'
182                   - compare-type: ANT
183                     pattern: '{installer}/include/**'
184                   - compare-type: ANT
185                     pattern: '{installer}/prototypes/**'
186                 forbidden-file-paths:
187                   - compare-type: ANT
188                     pattern: 'common/docs/**'
189                   - compare-type: ANT
190                     pattern: '{installer}/docs/**'
191
192     builders:
193         - 'fuel-build'
194         - 'fuel-workspace-cleanup'
195
196 - job-template:
197     name: 'genesis-fuel-daily-{stream}'
198
199     project-type: freestyle
200
201     node: ericsson-build
202
203     disabled: '{obj:disabled}'
204
205     triggers:
206         - 'genesis-fuel-{stream}-trigger'
207
208     parameters:
209         - project-parameter:
210             project: '{project}'
211         - genesis-fuel-parameter:
212             installer: '{installer}'
213             gs-pathname: '{gs-pathname}'
214
215     scm:
216         - git-scm:
217             credentials-id: '{ssh-credentials}'
218             refspec: ''
219             branch: '{branch}'
220
221     logrotate:
222         daysToKeep: 30
223         numToKeep: 10
224         artifactDaysToKeep: -1
225         artifactNumToKeep: -1
226
227     builders:
228         - trigger-builds:
229           - project: 'genesis-fuel-build-{stream}'
230             git-revision: true
231             block: true
232         - trigger-builds:
233           - project: 'genesis-fuel-lab-reconfig-{stream}'
234             git-revision: true
235             block: true
236         - trigger-builds:
237           - project: 'genesis-fuel-deploy-{stream}'
238             git-revision: true
239             block: true
240         - trigger-builds:
241           - project: 'functest-fuel-{stream}'
242             block: true
243             block-thresholds:
244                 build-step-failure-threshold: 'never'
245                 failure-threshold: 'never'
246                 unstable-threshold: 'FAILURE'
247         # - trigger-builds:
248           # - project: 'yardstick-fuel-{stream}'
249             # block: true
250             # block-thresholds:
251                 # build-step-failure-threshold: 'never'
252                 # failure-threshold: 'never'
253                 # unstable-threshold: 'FAILURE'
254
255     publishers:
256         - trigger-parameterized-builds:
257             - project: 'genesis-foreman-daily-{stream}'
258               trigger-with-no-params: true
259
260 - job-template:
261     name: 'genesis-fuel-deploy-runner-{stream}'
262
263     project-type: freestyle
264
265     disabled: false
266
267     node: master
268
269     logrotate:
270         daysToKeep: 30
271         numToKeep: 10
272         artifactDaysToKeep: -1
273         artifactNumToKeep: -1
274
275     parameters:
276         - project-parameter:
277             project: '{project}'
278         - genesis-fuel-parameter:
279             installer: '{installer}'
280             gs-pathname: '{gs-pathname}'
281         - string:
282             name: GIT_BASE
283             default: ssh://gerrit.opnfv.org:29418/$PROJECT
284             description: "Used for overriding the GIT URL coming from parameters macro."
285
286     scm:
287         - git-scm:
288             credentials-id: '{ssh-credentials}'
289             refspec: ''
290             branch: '{branch}'
291
292     properties:
293         - build-blocker:
294             use-build-blocker: true
295             blocking-jobs:
296                 - "genesis-foreman-daily.*"
297                 - "genesis-foreman-deploy-runner-{stream}"
298                 - "genesis-fuel-daily.*"
299
300     builders:
301         - 'fuel-lab-reconfig'
302         - trigger-builds:
303           - project: 'genesis-fuel-deploy-{stream}'
304             git-revision: true
305             block: true
306
307 - job-template:
308     name: 'genesis-fuel-build-{stream}'
309
310     project-type: freestyle
311
312     node: ericsson-build
313
314     logrotate:
315         daysToKeep: 30
316         numToKeep: 10
317         artifactDaysToKeep: -1
318         artifactNumToKeep: -1
319
320     parameters:
321         - project-parameter:
322             project: '{project}'
323         - genesis-fuel-parameter:
324             installer: '{installer}'
325             gs-pathname: '{gs-pathname}'
326
327     scm:
328         - git-scm:
329             credentials-id: '{ssh-credentials}'
330             refspec: ''
331             branch: '{branch}'
332
333     builders:
334         - 'fuel-build'
335         - 'fuel-upload-artifact'
336         - 'fuel-workspace-cleanup'
337
338     publishers:
339         - email:
340             recipients: jonas.bjurel@ericsson.com stefan.k.berg@ericsson.com
341
342 - job-template:
343     name: 'genesis-fuel-deploy-{stream}'
344
345     project-type: freestyle
346
347     disabled: false
348
349     node: opnfv-jump-2
350
351     parameters:
352         - project-parameter:
353             project: '{project}'
354         - genesis-fuel-parameter:
355             installer: '{installer}'
356             gs-pathname: '{gs-pathname}'
357         - string:
358             name: GIT_BASE
359             default: ssh://gerrit.opnfv.org:29418/$PROJECT
360             description: "POD2 has some issues with cloning using https so that's why GIT_BASE is overriden here again."
361
362     scm:
363         - git-scm:
364             credentials-id: '{ssh-credentials}'
365             refspec: ''
366             branch: '{branch}'
367
368     logrotate:
369         daysToKeep: 30
370         numToKeep: 10
371         artifactDaysToKeep: -1
372         artifactNumToKeep: -1
373
374     builders:
375         - 'fuel-download-artifact'
376         - 'fuel-deploy'
377
378     publishers:
379         - email:
380             recipients: jonas.bjurel@ericsson.com stefan.k.berg@ericsson.com
381
382 - job-template:
383     name: 'genesis-fuel-lab-reconfig-{stream}'
384
385     project-type: freestyle
386
387     disabled: false
388
389     node: master
390
391     logrotate:
392         daysToKeep: 30
393         numToKeep: 10
394         artifactDaysToKeep: -1
395         artifactNumToKeep: -1
396
397     parameters:
398         - project-parameter:
399             project: '{project}'
400         - genesis-fuel-parameter:
401             installer: '{installer}'
402             gs-pathname: '{gs-pathname}'
403         - string:
404             name: GIT_BASE
405             default: ssh://gerrit.opnfv.org:29418/$PROJECT
406             description: "Used for overriding the GIT URL coming from parameters macro."
407
408     scm:
409         - git-scm:
410             credentials-id: '{ssh-credentials}'
411             refspec: ''
412             branch: '{branch}'
413
414     properties:
415         - build-blocker:
416             use-build-blocker: true
417             blocking-jobs:
418                 - "genesis-foreman-daily.*"
419
420     builders:
421         - 'fuel-lab-reconfig'
422
423 - job-template:
424     name: 'genesis-fuel-verify-deploy-virtual-{stream}'
425
426     project-type: freestyle
427
428     disabled: true
429
430     node: ericsson-build
431
432     parameters:
433         - project-parameter:
434             project: '{project}'
435         - genesis-fuel-parameter:
436             installer: '{installer}'
437             gs-pathname: ''
438
439     scm:
440         - git-scm:
441             credentials-id: '{ssh-credentials}'
442             refspec: ''
443             branch: 'master'
444
445     logrotate:
446         daysToKeep: 30
447         numToKeep: 10
448         artifactDaysToKeep: -1
449         artifactNumToKeep: -1
450
451     builders:
452         - 'fuel-deploy-virtual'
453
454 ########################
455 # parameter macros
456 ########################
457 - parameter:
458     name: genesis-fuel-parameter
459     parameters:
460         - string:
461             name: INSTALLER
462             default: '{installer}'
463             description: "Installer to use."
464         - string:
465             name: BUILD_DIRECTORY
466             default: $WORKSPACE/build_output
467             description: "Directory where the build artifact will be located upon the completion of the build."
468         - string:
469             name: CACHE_DIRECTORY
470             default: $HOME/opnfv/cache/genesis-$INSTALLER
471             description: "Directory where the cache to be used during the build is located."
472         - string:
473             name: GIT_BASE
474             default: https://gerrit.opnfv.org/gerrit/$PROJECT
475             description: "Used for overriding the GIT URL coming from Global Jenkins configuration in case if the stuff is done on none-LF HW."
476         - string:
477             name: GS_URL
478             default: artifacts.opnfv.org/$PROJECT/$INSTALLER{gs-pathname}
479             description: "URL to Google Storage."
480
481 ########################
482 # builder macros
483 ########################
484 - builder:
485     name: 'fuel-build'
486     builders:
487         - shell: |
488             #!/bin/bash
489             set -o errexit
490             set -o nounset
491             set -o pipefail
492
493             # log info to console
494             echo "Starting the build of $INSTALLER. This could take some time..."
495             echo "--------------------------------------------------------"
496             echo
497
498             # create the cache directory if it doesn't exist
499             [[ -d $CACHE_DIRECTORY ]] || mkdir -p $CACHE_DIRECTORY
500
501             # set OPNFV_ARTIFACT_VERSION
502             export OPNFV_ARTIFACT_VERSION=$(date -u +"%Y-%m-%d_%H-%M-%S")
503
504             # start the build
505             cd $WORKSPACE/$INSTALLER/ci
506             ./build.sh -v $OPNFV_ARTIFACT_VERSION -c file://$CACHE_DIRECTORY $BUILD_DIRECTORY
507
508             # list the build artifacts
509             ls -al $BUILD_DIRECTORY
510
511             # save information regarding artifact into file
512             (
513                 echo "OPNFV_ARTIFACT_VERSION=$OPNFV_ARTIFACT_VERSION"
514                 echo "OPNFV_GIT_URL=$(git config --get remote.origin.url)"
515                 echo "OPNFV_GIT_SHA1=$(git rev-parse HEAD)"
516                 echo "OPNFV_ARTIFACT_URL=$GS_URL/opnfv-$OPNFV_ARTIFACT_VERSION.iso"
517                 echo "OPNFV_ARTIFACT_MD5SUM=$(md5sum $BUILD_DIRECTORY/opnfv-$OPNFV_ARTIFACT_VERSION.iso | cut -d' ' -f1)"
518                 echo "OPNFV_BUILD_URL=$BUILD_URL"
519             ) > $WORKSPACE/opnfv.properties
520
521             echo
522             echo "--------------------------------------------------------"
523             echo "Done!"
524
525 - builder:
526     name: 'fuel-deploy'
527     builders:
528         - shell: |
529             #!/bin/bash
530             set -o errexit
531             set -o nounset
532             set -o pipefail
533
534             # source the file so we get OPNFV vars
535             source latest.properties
536
537             # echo the info about artifact that is used during the deployment
538             echo "Using $(echo $OPNFV_ARTIFACT_URL | cut -d'/' -f4) for deployment"
539
540             # create TMPDIR if it doesn't exist
541             export TMPDIR=$HOME/tmpdir
542             [[ -d $TMPDIR ]] || mkdir -p $TMPDIR
543
544             # change permissions down to TMPDIR
545             chmod a+x $HOME
546             chmod a+x $TMPDIR
547
548             # set CONFDIR, BRIDGE
549             export CONFDIR=$WORKSPACE/fuel/deploy/templates/hardware_environment/conf/linux_foundation_lab/pod2
550             export BRIDGE=pxebr
551
552             # cleanup first
553             sudo $WORKSPACE/common/ci/clean.sh -base_config $WORKSPACE/foreman/ci/inventory/lf_pod2_ksgen_settings.yml
554
555             # prepare for Fuel Deployment
556             sudo $WORKSPACE/common/ci/setup.sh
557
558             # log info to console
559             echo "Starting the deployment using $INSTALLER. This could take some time..."
560             echo "--------------------------------------------------------"
561             echo
562
563             # start the deployment
564             echo "Issuing command"
565             echo "sudo $WORKSPACE/fuel/ci/deploy.sh -iso $WORKSPACE/opnfv.iso -dea $CONFDIR/dea.yaml -dha $CONFDIR/dha.yaml -s $TMPDIR -b $BRIDGE -nh"
566             sudo $WORKSPACE/fuel/ci/deploy.sh -iso $WORKSPACE/opnfv.iso -dea $CONFDIR/dea.yaml -dha $CONFDIR/dha.yaml -s $TMPDIR -b $BRIDGE -nh
567
568             echo
569             echo "--------------------------------------------------------"
570             echo "Done!"
571
572 - builder:
573     name: 'fuel-deploy-virtual'
574     builders:
575         - shell: |
576             #!/bin/bash
577             set -o errexit
578             set -o nounset
579             set -o pipefail
580
581             # log info to console
582             echo "Starting the deployment on virtual environment using $INSTALLER. This could take some time..."
583             echo "--------------------------------------------------------"
584             echo
585
586             ssh -o BatchMode=yes -o TCPKeepAlive=yes cideploy@10.118.34.205 ./cideploy.sh
587
588             echo
589             echo "--------------------------------------------------------"
590             echo "Done!"
591
592 - builder:
593     name: 'fuel-upload-artifact'
594     builders:
595         - shell: |
596             #!/bin/bash
597             set -o errexit
598             set -o nounset
599             set -o pipefail
600
601             # log info to console
602             echo "Uploading the $INSTALLER artifact. This could take some time..."
603             echo "--------------------------------------------------------"
604             echo
605
606             # source the opnfv.properties to get ARTIFACT_VERSION
607             source $WORKSPACE/opnfv.properties
608
609             # upload artifact and additional files to google storage
610             gsutil cp $BUILD_DIRECTORY/opnfv-$OPNFV_ARTIFACT_VERSION.iso gs://$GS_URL/opnfv-$OPNFV_ARTIFACT_VERSION.iso > gsutil.iso.log 2>&1
611             gsutil cp $WORKSPACE/opnfv.properties gs://$GS_URL/opnfv-$OPNFV_ARTIFACT_VERSION.properties > gsutil.properties.log 2>&1
612             gsutil cp $WORKSPACE/opnfv.properties gs://$GS_URL/latest.properties > gsutil.latest.log 2>&1
613
614             echo
615             echo "--------------------------------------------------------"
616             echo "Done!"
617             echo "Artifact is available as http://$GS_URL/opnfv-$OPNFV_ARTIFACT_VERSION.iso"
618
619 - builder:
620     name: 'fuel-download-artifact'
621     builders:
622         - shell: |
623             #!/bin/bash
624             set -o errexit
625             set -o nounset
626             set -o pipefail
627
628             # get the latest.properties file in order to get info regarding latest artifact
629             curl -s -o $WORKSPACE/latest.properties http://$GS_URL/latest.properties
630
631             # check if we got the file
632             [[ -f latest.properties ]] || exit 1
633
634             # source the file so we get OPNFV vars
635             source latest.properties
636
637             # log info to console
638             echo "Downloading the $INSTALLER artifact using URL http://$OPNFV_ARTIFACT_URL"
639             echo "This could take some time..."
640             echo "--------------------------------------------------------"
641             echo
642
643             # download the file
644             curl -s -o $WORKSPACE/opnfv.iso http://$OPNFV_ARTIFACT_URL > gsutil.iso.log 2>&1
645
646             # list the file
647             ls -al $WORKSPACE/opnfv.iso
648
649             echo
650             echo "--------------------------------------------------------"
651             echo "Done!"
652
653 - builder:
654     name: 'fuel-workspace-cleanup'
655     builders:
656         - shell: |
657             #!/bin/bash
658             set -o errexit
659             set -o nounset
660             set -o pipefail
661
662             # delete everything that is in $WORKSPACE
663             /bin/rm -rf $WORKSPACE
664
665 - builder:
666     name: 'fuel-lab-reconfig'
667     builders:
668         - shell: |
669             #!/bin/bash
670             set -o errexit
671             set -o nounset
672             set -o pipefail
673
674             # check to see if ucs login info file exists
675             if [ -e ~/.ssh/ucs_creds ];then
676               source ~/.ssh/ucs_creds
677             else
678               echo "Unable to find UCS credentials for LF lab reconfiguration...Exiting"
679               exit 1
680             fi
681
682             # clone releng
683             echo "Cloning releng repo..."
684             if ! GIT_SSL_NO_VERIFY=true git clone https://gerrit.opnfv.org/gerrit/releng; then
685               echo "Unable to clone releng repo...Exiting"
686               exit 1
687             fi
688
689             # log info to console
690             echo "Starting the lab reconfiguration for $INSTALLER..."
691             echo "--------------------------------------------------------"
692             echo
693
694             # create venv
695             $WORKSPACE/releng/utils/lab-reconfiguration/create_venv.sh
696
697             # disable nounset because 'activate' script contains unbound variable(s)
698             set +o nounset
699             # enter venv
700             source $WORKSPACE/releng/utils/lab-reconfiguration/venv/bin/activate
701             # set nounset back again
702             set -o nounset
703
704             # verify we are in venv
705             if [[ ! $(which python | grep venv) ]]; then
706               echo "Unable to activate venv...Exiting"
707               exit 1
708             fi
709
710             python $WORKSPACE/releng/utils/lab-reconfiguration/reconfigUcsNet.py -i $ucs_host -u $ucs_user -p $ucs_password -f $WORKSPACE/releng/utils/lab-reconfiguration/fuel.yaml
711
712             # while undergoing reboot
713             sleep 30
714
715             # check to see if slave is back up
716             ping_counter=0
717             ping_flag=0
718             while [ "$ping_counter" -lt 20 ]; do
719               if [[ $(ping -c 5 172.30.10.72) ]]; then
720                 ping_flag=1
721                 break
722               fi
723               ((ping_counter++))
724               sleep 10
725             done
726
727             if [ "$ping_flag" -eq 1 ]; then
728               echo "Slave is pingable, now wait 180 seconds for services to start"
729               sleep 180
730             else
731               echo "Slave did not come back up after reboot: please check opnfv-jump-2"
732               exit 1
733             fi
734
735             set +o nounset
736             deactivate
737
738             echo
739             echo "--------------------------------------------------------"
740             echo "Done!"
741
742 ########################
743 # trigger macros
744 ########################
745 - trigger:
746     name: 'genesis-fuel-master-trigger'
747     triggers:
748         - timed: '0 5 * * *'
749
750 - trigger:
751     name: 'genesis-fuel-stable-arno-trigger'
752     triggers:
753         - timed: '0 7 * * *'