a957a693baf20eba14b1f756f21bda696c0c7dce
[releng.git] / jjb / genesis / genesis-fuel.yml
1 ########################
2 # Job configuration for genesis/fuel
3 ########################
4 - project:
5
6     name: genesis-fuel
7
8     project: 'genesis'
9
10     installer: 'fuel'
11
12     jobs:
13         - 'genesis-fuel-verify-build-{stream}'
14         - 'genesis-fuel-merge-{stream}'
15         - 'genesis-fuel-daily-{stream}'
16         - 'genesis-fuel-build-{stream}'
17         - 'genesis-fuel-deploy-{stream}'
18         - 'genesis-fuel-deploy-runner-{stream}'
19         - 'genesis-fuel-lab-reconfig-{stream}'
20         - 'genesis-fuel-verify-deploy-virtual-{stream}'
21
22     # stream:    branch with - in place of / (eg. stable-arno)
23     # branch:    branch (eg. stable/arno)
24     stream:
25         - master:
26             branch: 'master'
27             gs-pathname: ''
28             disabled: true
29         - stable-arno:
30             branch: 'stable/arno'
31             gs-pathname: '/arno'
32             disabled: true
33
34 ########################
35 # job templates
36 ########################
37
38 - job-template:
39     name: 'genesis-fuel-verify-build-{stream}'
40
41     project-type: freestyle
42
43     node: ericsson-build
44
45     concurrent: true
46
47     properties:
48         - throttle:
49             enabled: true
50             max-total: 3
51
52     logrotate:
53         daysToKeep: 30
54         numToKeep: 10
55         artifactDaysToKeep: -1
56         artifactNumToKeep: -1
57
58     parameters:
59         - project-parameter:
60             project: '{project}'
61         - gerrit-parameter:
62             branch: '{branch}'
63         - genesis-fuel-parameter:
64             installer: '{installer}'
65             gs-pathname: '{gs-pathname}'
66
67     scm:
68         - gerrit-trigger-scm:
69             credentials-id: '{ssh-credentials}'
70             refspec: '$GERRIT_REFSPEC'
71             choosing-strategy: 'gerrit'
72
73     wrappers:
74         - ssh-agent-credentials:
75             user: '{ssh-credentials}'
76
77     triggers:
78         - gerrit:
79             trigger-on:
80                 - patchset-created-event:
81                     exclude-drafts: 'false'
82                     exclude-trivial-rebase: 'false'
83                     exclude-no-code-change: 'false'
84                 - draft-published-event
85                 - comment-added-contains-event:
86                     comment-contains-value: 'recheck'
87                 - comment-added-contains-event:
88                     comment-contains-value: 'reverify'
89             projects:
90               - project-compare-type: 'ANT'
91                 project-pattern: 'genesis'
92                 branches:
93                   - branch-compare-type: 'ANT'
94                     branch-pattern: '**/{branch}'
95                 file-paths:
96                   - compare-type: ANT
97                     pattern: 'common/ci/**'
98                   - compare-type: ANT
99                     pattern: 'common/manifests/**'
100                   - compare-type: ANT
101                     pattern: 'common/puppet-opnfv/**'
102                   - compare-type: ANT
103                     pattern: 'common/tools/**'
104                   - compare-type: ANT
105                     pattern: '{installer}/build/**'
106                   - compare-type: ANT
107                     pattern: '{installer}/ci/**'
108                   - compare-type: ANT
109                     pattern: '{installer}/deploy/**'
110                   - compare-type: ANT
111                     pattern: '{installer}/include/**'
112                   - compare-type: ANT
113                     pattern: '{installer}/prototypes/**'
114                 forbidden-file-paths:
115                   - compare-type: ANT
116                     pattern: 'common/docs/**'
117                   - compare-type: ANT
118                     pattern: '{installer}/docs/**'
119
120     builders:
121         - 'fuel-build'
122         - 'fuel-workspace-cleanup'
123
124 - job-template:
125     name: 'genesis-fuel-merge-{stream}'
126
127     project-type: freestyle
128
129     node: ericsson-build
130
131     concurrent: true
132
133     properties:
134         - throttle:
135             enabled: true
136             max-total: 2
137
138     logrotate:
139         daysToKeep: 30
140         numToKeep: 40
141         artifactDaysToKeep: -1
142         artifactNumToKeep: 5
143
144     parameters:
145         - project-parameter:
146             project: '{project}'
147         - gerrit-parameter:
148             branch: '{branch}'
149         - genesis-fuel-parameter:
150             installer: '{installer}'
151             gs-pathname: ''
152
153     scm:
154         - gerrit-trigger-scm:
155             credentials-id: '{ssh-credentials}'
156             refspec: ''
157             choosing-strategy: 'default'
158
159     wrappers:
160         - ssh-agent-credentials:
161             user: '{ssh-credentials}'
162
163     triggers:
164         - gerrit:
165             trigger-on:
166                 - change-merged-event
167                 - comment-added-contains-event:
168                     comment-contains-value: 'remerge'
169             projects:
170               - project-compare-type: 'ANT'
171                 project-pattern: 'genesis'
172                 branches:
173                   - branch-compare-type: 'ANT'
174                     branch-pattern: '**/{branch}'
175                 file-paths:
176                   - compare-type: ANT
177                     pattern: 'common/ci/**'
178                   - compare-type: ANT
179                     pattern: 'common/manifests/**'
180                   - compare-type: ANT
181                     pattern: 'common/puppet-opnfv/**'
182                   - compare-type: ANT
183                     pattern: 'common/tools/**'
184                   - compare-type: ANT
185                     pattern: '{installer}/build/**'
186                   - compare-type: ANT
187                     pattern: '{installer}/ci/**'
188                   - compare-type: ANT
189                     pattern: '{installer}/deploy/**'
190                   - compare-type: ANT
191                     pattern: '{installer}/include/**'
192                   - compare-type: ANT
193                     pattern: '{installer}/prototypes/**'
194                 forbidden-file-paths:
195                   - compare-type: ANT
196                     pattern: 'common/docs/**'
197                   - compare-type: ANT
198                     pattern: '{installer}/docs/**'
199
200     builders:
201         - 'fuel-build'
202         - 'fuel-workspace-cleanup'
203
204 - job-template:
205     name: 'genesis-fuel-daily-{stream}'
206
207     project-type: freestyle
208
209     node: ericsson-build
210
211     disabled: '{obj:disabled}'
212
213     triggers:
214         - 'genesis-fuel-{stream}-trigger'
215
216     parameters:
217         - project-parameter:
218             project: '{project}'
219         - genesis-fuel-parameter:
220             installer: '{installer}'
221             gs-pathname: '{gs-pathname}'
222
223     scm:
224         - git-scm:
225             credentials-id: '{ssh-credentials}'
226             refspec: ''
227             branch: '{branch}'
228
229     logrotate:
230         daysToKeep: 30
231         numToKeep: 10
232         artifactDaysToKeep: -1
233         artifactNumToKeep: -1
234
235     builders:
236         - trigger-builds:
237           - project: 'genesis-fuel-build-{stream}'
238             git-revision: true
239             block: true
240         - trigger-builds:
241           - project: 'genesis-fuel-lab-reconfig-{stream}'
242             git-revision: true
243             block: true
244         - trigger-builds:
245           - project: 'genesis-fuel-deploy-{stream}'
246             git-revision: true
247             block: true
248         - trigger-builds:
249           - project: 'functest-fuel-{stream}'
250             block: true
251             block-thresholds:
252                 build-step-failure-threshold: 'never'
253                 failure-threshold: 'never'
254                 unstable-threshold: 'FAILURE'
255         # - trigger-builds:
256           # - project: 'yardstick-fuel-{stream}'
257             # block: true
258             # block-thresholds:
259                 # build-step-failure-threshold: 'never'
260                 # failure-threshold: 'never'
261                 # unstable-threshold: 'FAILURE'
262
263     publishers:
264         - trigger-parameterized-builds:
265             - project: 'genesis-foreman-daily-{stream}'
266               trigger-with-no-params: true
267
268 - job-template:
269     name: 'genesis-fuel-deploy-runner-{stream}'
270
271     project-type: freestyle
272
273     disabled: false
274
275     node: master
276
277     logrotate:
278         daysToKeep: 30
279         numToKeep: 10
280         artifactDaysToKeep: -1
281         artifactNumToKeep: -1
282
283     parameters:
284         - project-parameter:
285             project: '{project}'
286         - genesis-fuel-parameter:
287             installer: '{installer}'
288             gs-pathname: '{gs-pathname}'
289         - string:
290             name: GIT_BASE
291             default: ssh://gerrit.opnfv.org:29418/$PROJECT
292             description: "Used for overriding the GIT URL coming from parameters macro."
293
294     scm:
295         - git-scm:
296             credentials-id: '{ssh-credentials}'
297             refspec: ''
298             branch: '{branch}'
299
300     properties:
301         - build-blocker:
302             use-build-blocker: true
303             blocking-jobs:
304                 - "genesis-foreman-daily.*"
305                 - "genesis-foreman-deploy-runner-{stream}"
306                 - "genesis-fuel-daily.*"
307
308     builders:
309         - 'fuel-lab-reconfig'
310         - trigger-builds:
311           - project: 'genesis-fuel-deploy-{stream}'
312             git-revision: true
313             block: true
314
315 - job-template:
316     name: 'genesis-fuel-build-{stream}'
317
318     project-type: freestyle
319
320     node: ericsson-build
321
322     logrotate:
323         daysToKeep: 30
324         numToKeep: 10
325         artifactDaysToKeep: -1
326         artifactNumToKeep: -1
327
328     parameters:
329         - project-parameter:
330             project: '{project}'
331         - genesis-fuel-parameter:
332             installer: '{installer}'
333             gs-pathname: '{gs-pathname}'
334
335     scm:
336         - git-scm:
337             credentials-id: '{ssh-credentials}'
338             refspec: ''
339             branch: '{branch}'
340
341     builders:
342         - 'fuel-build'
343         - 'fuel-upload-artifact'
344         - 'fuel-workspace-cleanup'
345
346     publishers:
347         - email:
348             recipients: jonas.bjurel@ericsson.com stefan.k.berg@ericsson.com
349
350 - job-template:
351     name: 'genesis-fuel-deploy-{stream}'
352
353     project-type: freestyle
354
355     disabled: false
356
357     node: opnfv-jump-2
358
359     parameters:
360         - project-parameter:
361             project: '{project}'
362         - genesis-fuel-parameter:
363             installer: '{installer}'
364             gs-pathname: '{gs-pathname}'
365         - string:
366             name: GIT_BASE
367             default: ssh://gerrit.opnfv.org:29418/$PROJECT
368             description: "POD2 has some issues with cloning using https so that's why GIT_BASE is overriden here again."
369
370     scm:
371         - git-scm:
372             credentials-id: '{ssh-credentials}'
373             refspec: ''
374             branch: '{branch}'
375
376     logrotate:
377         daysToKeep: 30
378         numToKeep: 10
379         artifactDaysToKeep: -1
380         artifactNumToKeep: -1
381
382     builders:
383         - 'fuel-download-artifact'
384         - 'fuel-deploy'
385
386     publishers:
387         - email:
388             recipients: jonas.bjurel@ericsson.com stefan.k.berg@ericsson.com
389
390 - job-template:
391     name: 'genesis-fuel-lab-reconfig-{stream}'
392
393     project-type: freestyle
394
395     disabled: false
396
397     node: master
398
399     logrotate:
400         daysToKeep: 30
401         numToKeep: 10
402         artifactDaysToKeep: -1
403         artifactNumToKeep: -1
404
405     parameters:
406         - project-parameter:
407             project: '{project}'
408         - genesis-fuel-parameter:
409             installer: '{installer}'
410             gs-pathname: '{gs-pathname}'
411         - string:
412             name: GIT_BASE
413             default: ssh://gerrit.opnfv.org:29418/$PROJECT
414             description: "Used for overriding the GIT URL coming from parameters macro."
415
416     scm:
417         - git-scm:
418             credentials-id: '{ssh-credentials}'
419             refspec: ''
420             branch: '{branch}'
421
422     properties:
423         - build-blocker:
424             use-build-blocker: true
425             blocking-jobs:
426                 - "genesis-foreman-daily.*"
427
428     builders:
429         - 'fuel-lab-reconfig'
430
431 - job-template:
432     name: 'genesis-fuel-verify-deploy-virtual-{stream}'
433
434     project-type: freestyle
435
436     disabled: true
437
438     node: ericsson-build
439
440     parameters:
441         - project-parameter:
442             project: '{project}'
443         - genesis-fuel-parameter:
444             installer: '{installer}'
445             gs-pathname: ''
446
447     scm:
448         - git-scm:
449             credentials-id: '{ssh-credentials}'
450             refspec: ''
451             branch: 'master'
452
453     logrotate:
454         daysToKeep: 30
455         numToKeep: 10
456         artifactDaysToKeep: -1
457         artifactNumToKeep: -1
458
459     builders:
460         - 'fuel-deploy-virtual'
461
462 ########################
463 # parameter macros
464 ########################
465 - parameter:
466     name: genesis-fuel-parameter
467     parameters:
468         - string:
469             name: INSTALLER
470             default: '{installer}'
471             description: "Installer to use."
472         - string:
473             name: BUILD_DIRECTORY
474             default: $WORKSPACE/build_output
475             description: "Directory where the build artifact will be located upon the completion of the build."
476         - string:
477             name: CACHE_DIRECTORY
478             default: $HOME/opnfv/cache/genesis-$INSTALLER
479             description: "Directory where the cache to be used during the build is located."
480         - string:
481             name: GIT_BASE
482             default: https://gerrit.opnfv.org/gerrit/$PROJECT
483             description: "Used for overriding the GIT URL coming from Global Jenkins configuration in case if the stuff is done on none-LF HW."
484         - string:
485             name: GS_URL
486             default: artifacts.opnfv.org/$PROJECT/$INSTALLER{gs-pathname}
487             description: "URL to Google Storage."
488
489 ########################
490 # builder macros
491 ########################
492 - builder:
493     name: 'fuel-build'
494     builders:
495         - shell: |
496             #!/bin/bash
497             set -o errexit
498             set -o nounset
499             set -o pipefail
500
501             # log info to console
502             echo "Starting the build of $INSTALLER. This could take some time..."
503             echo "--------------------------------------------------------"
504             echo
505
506             # create the cache directory if it doesn't exist
507             [[ -d $CACHE_DIRECTORY ]] || mkdir -p $CACHE_DIRECTORY
508
509             # set OPNFV_ARTIFACT_VERSION
510             export OPNFV_ARTIFACT_VERSION=$(date -u +"%Y-%m-%d_%H-%M-%S")
511
512             # start the build
513             cd $WORKSPACE/$INSTALLER/ci
514             ./build.sh -v $OPNFV_ARTIFACT_VERSION -c file://$CACHE_DIRECTORY $BUILD_DIRECTORY
515
516             # list the build artifacts
517             ls -al $BUILD_DIRECTORY
518
519             # save information regarding artifact into file
520             (
521                 echo "OPNFV_ARTIFACT_VERSION=$OPNFV_ARTIFACT_VERSION"
522                 echo "OPNFV_GIT_URL=$(git config --get remote.origin.url)"
523                 echo "OPNFV_GIT_SHA1=$(git rev-parse HEAD)"
524                 echo "OPNFV_ARTIFACT_URL=$GS_URL/opnfv-$OPNFV_ARTIFACT_VERSION.iso"
525                 echo "OPNFV_ARTIFACT_MD5SUM=$(md5sum $BUILD_DIRECTORY/opnfv-$OPNFV_ARTIFACT_VERSION.iso | cut -d' ' -f1)"
526                 echo "OPNFV_BUILD_URL=$BUILD_URL"
527             ) > $WORKSPACE/opnfv.properties
528
529             echo
530             echo "--------------------------------------------------------"
531             echo "Done!"
532
533 - builder:
534     name: 'fuel-deploy'
535     builders:
536         - shell: |
537             #!/bin/bash
538             set -o errexit
539             set -o nounset
540             set -o pipefail
541
542             # source the file so we get OPNFV vars
543             source latest.properties
544
545             # echo the info about artifact that is used during the deployment
546             echo "Using $(echo $OPNFV_ARTIFACT_URL | cut -d'/' -f4) for deployment"
547
548             # create TMPDIR if it doesn't exist
549             export TMPDIR=$HOME/tmpdir
550             [[ -d $TMPDIR ]] || mkdir -p $TMPDIR
551
552             # change permissions down to TMPDIR
553             chmod a+x $HOME
554             chmod a+x $TMPDIR
555
556             # set CONFDIR, BRIDGE
557             export CONFDIR=$WORKSPACE/fuel/deploy/templates/hardware_environment/conf/linux_foundation_lab/pod2
558             export BRIDGE=pxebr
559
560             # cleanup first
561             sudo $WORKSPACE/common/ci/clean.sh -base_config $WORKSPACE/foreman/ci/inventory/lf_pod2_ksgen_settings.yml
562
563             # prepare for Fuel Deployment
564             sudo $WORKSPACE/common/ci/setup.sh
565
566             # log info to console
567             echo "Starting the deployment using $INSTALLER. This could take some time..."
568             echo "--------------------------------------------------------"
569             echo
570
571             # start the deployment
572             echo "Issuing command"
573             echo "sudo $WORKSPACE/fuel/ci/deploy.sh -iso $WORKSPACE/opnfv.iso -dea $CONFDIR/dea.yaml -dha $CONFDIR/dha.yaml -s $TMPDIR -b $BRIDGE -nh"
574             sudo $WORKSPACE/fuel/ci/deploy.sh -iso $WORKSPACE/opnfv.iso -dea $CONFDIR/dea.yaml -dha $CONFDIR/dha.yaml -s $TMPDIR -b $BRIDGE -nh
575
576             echo
577             echo "--------------------------------------------------------"
578             echo "Done!"
579
580 - builder:
581     name: 'fuel-deploy-virtual'
582     builders:
583         - shell: |
584             #!/bin/bash
585             set -o errexit
586             set -o nounset
587             set -o pipefail
588
589             # log info to console
590             echo "Starting the deployment on virtual environment using $INSTALLER. This could take some time..."
591             echo "--------------------------------------------------------"
592             echo
593
594             ssh -o BatchMode=yes -o TCPKeepAlive=yes cideploy@10.118.34.205 ./cideploy.sh
595
596             echo
597             echo "--------------------------------------------------------"
598             echo "Done!"
599
600 - builder:
601     name: 'fuel-upload-artifact'
602     builders:
603         - shell: |
604             #!/bin/bash
605             set -o errexit
606             set -o nounset
607             set -o pipefail
608
609             # log info to console
610             echo "Uploading the $INSTALLER artifact. This could take some time..."
611             echo "--------------------------------------------------------"
612             echo
613
614             # source the opnfv.properties to get ARTIFACT_VERSION
615             source $WORKSPACE/opnfv.properties
616
617             # upload artifact and additional files to google storage
618             gsutil cp $BUILD_DIRECTORY/opnfv-$OPNFV_ARTIFACT_VERSION.iso gs://$GS_URL/opnfv-$OPNFV_ARTIFACT_VERSION.iso > gsutil.iso.log 2>&1
619             gsutil cp $WORKSPACE/opnfv.properties gs://$GS_URL/opnfv-$OPNFV_ARTIFACT_VERSION.properties > gsutil.properties.log 2>&1
620             gsutil cp $WORKSPACE/opnfv.properties gs://$GS_URL/latest.properties > gsutil.latest.log 2>&1
621
622             echo
623             echo "--------------------------------------------------------"
624             echo "Done!"
625             echo "Artifact is available as http://$GS_URL/opnfv-$OPNFV_ARTIFACT_VERSION.iso"
626
627 - builder:
628     name: 'fuel-download-artifact'
629     builders:
630         - shell: |
631             #!/bin/bash
632             set -o errexit
633             set -o nounset
634             set -o pipefail
635
636             # get the latest.properties file in order to get info regarding latest artifact
637             curl -s -o $WORKSPACE/latest.properties http://$GS_URL/latest.properties
638
639             # check if we got the file
640             [[ -f latest.properties ]] || exit 1
641
642             # source the file so we get OPNFV vars
643             source latest.properties
644
645             # log info to console
646             echo "Downloading the $INSTALLER artifact using URL http://$OPNFV_ARTIFACT_URL"
647             echo "This could take some time..."
648             echo "--------------------------------------------------------"
649             echo
650
651             # download the file
652             curl -s -o $WORKSPACE/opnfv.iso http://$OPNFV_ARTIFACT_URL > gsutil.iso.log 2>&1
653
654             # list the file
655             ls -al $WORKSPACE/opnfv.iso
656
657             echo
658             echo "--------------------------------------------------------"
659             echo "Done!"
660
661 - builder:
662     name: 'fuel-workspace-cleanup'
663     builders:
664         - shell: |
665             #!/bin/bash
666             set -o errexit
667             set -o nounset
668             set -o pipefail
669
670             # delete everything that is in $WORKSPACE
671             /bin/rm -rf $WORKSPACE
672
673 - builder:
674     name: 'fuel-lab-reconfig'
675     builders:
676         - shell: |
677             #!/bin/bash
678             set -o errexit
679             set -o nounset
680             set -o pipefail
681
682             # check to see if ucs login info file exists
683             if [ -e ~/.ssh/ucs_creds ];then
684               source ~/.ssh/ucs_creds
685             else
686               echo "Unable to find UCS credentials for LF lab reconfiguration...Exiting"
687               exit 1
688             fi
689
690             # clone releng
691             echo "Cloning releng repo..."
692             if ! GIT_SSL_NO_VERIFY=true git clone https://gerrit.opnfv.org/gerrit/releng; then
693               echo "Unable to clone releng repo...Exiting"
694               exit 1
695             fi
696
697             # log info to console
698             echo "Starting the lab reconfiguration for $INSTALLER..."
699             echo "--------------------------------------------------------"
700             echo
701
702             # create venv
703             $WORKSPACE/releng/utils/lab-reconfiguration/create_venv.sh
704
705             # disable nounset because 'activate' script contains unbound variable(s)
706             set +o nounset
707             # enter venv
708             source $WORKSPACE/releng/utils/lab-reconfiguration/venv/bin/activate
709             # set nounset back again
710             set -o nounset
711
712             # verify we are in venv
713             if [[ ! $(which python | grep venv) ]]; then
714               echo "Unable to activate venv...Exiting"
715               exit 1
716             fi
717
718             python $WORKSPACE/releng/utils/lab-reconfiguration/reconfigUcsNet.py -i $ucs_host -u $ucs_user -p $ucs_password -f $WORKSPACE/releng/utils/lab-reconfiguration/fuel.yaml
719
720             # while undergoing reboot
721             sleep 30
722
723             # check to see if slave is back up
724             ping_counter=0
725             ping_flag=0
726             while [ "$ping_counter" -lt 20 ]; do
727               if [[ $(ping -c 5 172.30.10.72) ]]; then
728                 ping_flag=1
729                 break
730               fi
731               ((ping_counter++))
732               sleep 10
733             done
734
735             if [ "$ping_flag" -eq 1 ]; then
736               echo "Slave is pingable, now wait 180 seconds for services to start"
737               sleep 180
738             else
739               echo "Slave did not come back up after reboot: please check opnfv-jump-2"
740               exit 1
741             fi
742
743             set +o nounset
744             deactivate
745
746             echo
747             echo "--------------------------------------------------------"
748             echo "Done!"
749
750 ########################
751 # trigger macros
752 ########################
753 - trigger:
754     name: 'genesis-fuel-master-trigger'
755     triggers:
756         - timed: '0 5 * * *'
757
758 - trigger:
759     name: 'genesis-fuel-stable-arno-trigger'
760     triggers:
761         - timed: '0 7 * * *'