jjb/genesis/genesis-foreman.yml

   1 ########################
   2 # Job configuration for genesis/foreman
   3 ########################
   4 - project:
   5
   6     name: genesis-foreman
   7
   8     project: 'genesis'
   9
  10     installer: 'foreman'
  11
  12     jobs:
  13         - 'genesis-foreman-verify-build-{stream}'
  14         - 'genesis-foreman-merge-{stream}'
  15         - 'genesis-foreman-daily-{stream}'
  16         - 'genesis-foreman-build-{stream}'
  17         - 'genesis-foreman-deploy-{stream}'
  18         - 'genesis-foreman-lab-reconfig-{stream}'
  19         - 'genesis-foreman-deploy-runner-{stream}'
  20         - 'genesis-foreman-verify-deploy-virtual-{stream}'
  21
  22     # stream:    branch with - in place of / (eg. stable-arno)
  23     # branch:    branch (eg. stable/arno)
  24     stream:
  25         - master:
  26             branch: 'master'
  27             gs-pathname: ''
  28             job-disabled: true
  29         - stable-arno:
  30             branch: 'stable/arno'
  31             gs-pathname: '/arno'
  32             job-disabled: false
  33 ########################
  34 # job templates
  35 ########################
  36
  37 - job-template:
  38     name: 'genesis-foreman-verify-build-{stream}'
  39
  40     project-type: freestyle
  41
  42     node: ericsson-build
  43
  44     concurrent: true
  45
  46     properties:
  47         - throttle:
  48             enabled: true
  49             max-total: 1
  50
  51     logrotate:
  52         daysToKeep: 30
  53         numToKeep: 10
  54         artifactDaysToKeep: -1
  55         artifactNumToKeep: -1
  56
  57     parameters:
  58         - project-parameter:
  59             project: '{project}'
  60         - gerrit-parameter:
  61             branch: '{branch}'
  62         - foreman-parameter:
  63             installer: '{installer}'
  64             gs-pathname: '{gs-pathname}'
  65     scm:
  66         - gerrit-trigger-scm:
  67             credentials-id: '{ssh-credentials}'
  68             refspec: '$GERRIT_REFSPEC'
  69             choosing-strategy: 'gerrit'
  70
  71     wrappers:
  72         - ssh-agent-credentials:
  73             user: '{ssh-credentials}'
  74
  75     triggers:
  76         - gerrit:
  77             trigger-on:
  78                 - patchset-created-event:
  79                     exclude-drafts: 'false'
  80                     exclude-trivial-rebase: 'false'
  81                     exclude-no-code-change: 'false'
  82                 - draft-published-event
  83                 - comment-added-contains-event:
  84                     comment-contains-value: 'recheck'
  85                 - comment-added-contains-event:
  86                     comment-contains-value: 'reverify'
  87             projects:
  88               - project-compare-type: 'ANT'
  89                 project-pattern: 'genesis'
  90                 branches:
  91                   - branch-compare-type: 'ANT'
  92                     branch-pattern: '**/{branch}'
  93                 file-paths:
  94                   - compare-type: ANT
  95                     pattern: 'common/ci/**'
  96                   - compare-type: ANT
  97                     pattern: 'common/manifests/**'
  98                   - compare-type: ANT
  99                     pattern: 'common/puppet-opnfv/**'
 100                   - compare-type: ANT
 101                     pattern: 'common/tools/**'
 102                   - compare-type: ANT
 103                     pattern: '{installer}/build/**'
 104                   - compare-type: ANT
 105                     pattern: '{installer}/ci/**'
 106                   - compare-type: ANT
 107                     pattern: '{installer}/include/**'
 108                 forbidden-file-paths:
 109                   - compare-type: ANT
 110                     pattern: 'common/docs/**'
 111                   - compare-type: ANT
 112                     pattern: '{installer}/docs/**'
 113
 114     builders:
 115         - 'foreman-build'
 116         - 'foreman-workspace-cleanup'
 117
 118 - job-template:
 119     name: 'genesis-foreman-merge-{stream}'
 120
 121     project-type: freestyle
 122
 123     node: ericsson-build
 124
 125     concurrent: true
 126
 127     properties:
 128         - throttle:
 129             enabled: true
 130             max-total: 2
 131
 132     logrotate:
 133         daysToKeep: 30
 134         numToKeep: 40
 135         artifactDaysToKeep: -1
 136         artifactNumToKeep: 5
 137
 138     parameters:
 139         - project-parameter:
 140             project: '{project}'
 141         - gerrit-parameter:
 142             branch: '{branch}'
 143         - foreman-parameter:
 144             installer: '{installer}'
 145             gs-pathname: ''
 146     scm:
 147         - gerrit-trigger-scm:
 148             credentials-id: '{ssh-credentials}'
 149             refspec: ''
 150             choosing-strategy: 'default'
 151
 152     wrappers:
 153         - ssh-agent-credentials:
 154             user: '{ssh-credentials}'
 155
 156     triggers:
 157         - gerrit:
 158             trigger-on:
 159                 - change-merged-event
 160                 - comment-added-contains-event:
 161                     comment-contains-value: 'remerge'
 162             projects:
 163               - project-compare-type: 'ANT'
 164                 project-pattern: 'genesis'
 165                 branches:
 166                   - branch-compare-type: 'ANT'
 167                     branch-pattern: '**/{branch}'
 168                 file-paths:
 169                   - compare-type: ANT
 170                     pattern: 'common/ci/**'
 171                   - compare-type: ANT
 172                     pattern: 'common/manifests/**'
 173                   - compare-type: ANT
 174                     pattern: 'common/puppet-opnfv/**'
 175                   - compare-type: ANT
 176                     pattern: 'common/tools/**'
 177                   - compare-type: ANT
 178                     pattern: '{installer}/build/**'
 179                   - compare-type: ANT
 180                     pattern: '{installer}/ci/**'
 181                   - compare-type: ANT
 182                     pattern: '{installer}/include/**'
 183                 forbidden-file-paths:
 184                   - compare-type: ANT
 185                     pattern: 'common/docs/**'
 186                   - compare-type: ANT
 187                     pattern: '{installer}/docs/**'
 188
 189     builders:
 190         - 'foreman-build'
 191         - 'foreman-workspace-cleanup'
 192
 193 - job-template:
 194     name: 'genesis-foreman-daily-{stream}'
 195
 196     project-type: freestyle
 197
 198     node: ericsson-build
 199
 200     disabled: '{job-disabled}'
 201
 202     triggers:
 203         - 'foreman-{stream}'
 204
 205     parameters:
 206         - project-parameter:
 207             project: '{project}'
 208         - foreman-parameter:
 209             installer: '{installer}'
 210             gs-pathname: '{gs-pathname}'
 211
 212     scm:
 213         - git-scm:
 214             credentials-id: '{ssh-credentials}'
 215             refspec: ''
 216             branch: '{branch}'
 217
 218     logrotate:
 219         daysToKeep: 30
 220         numToKeep: 10
 221         artifactDaysToKeep: -1
 222         artifactNumToKeep: -1
 223
 224     builders:
 225         - trigger-builds:
 226           - project: 'genesis-foreman-build-{stream}'
 227             git-revision: true
 228             block: true
 229         - trigger-builds:
 230           - project: 'genesis-foreman-lab-reconfig-{stream}'
 231             git-revision: true
 232             block: true
 233         - trigger-builds:
 234           - project: 'genesis-foreman-deploy-{stream}'
 235             git-revision: true
 236             block: true
 237         - trigger-builds:
 238           - project: 'functest-foreman-{stream}'
 239             block: true
 240             block-thresholds:
 241                 build-step-failure-threshold: 'never'
 242                 failure-threshold: 'never'
 243                 unstable-threshold: 'FAILURE'
 244         # - trigger-builds:
 245           # - project: 'yardstick-foreman-{stream}'
 246             # block: true
 247             # block-thresholds:
 248                 # build-step-failure-threshold: 'never'
 249                 # failure-threshold: 'never'
 250                 # unstable-threshold: 'FAILURE'
 251
 252 - job-template:
 253     name: 'genesis-foreman-deploy-runner-{stream}'
 254
 255     project-type: freestyle
 256
 257     disabled: false
 258
 259     node: master
 260
 261     logrotate:
 262         daysToKeep: 30
 263         numToKeep: 10
 264         artifactDaysToKeep: -1
 265         artifactNumToKeep: -1
 266
 267     parameters:
 268         - project-parameter:
 269             project: '{project}'
 270         - foreman-parameter:
 271             installer: '{installer}'
 272             gs-pathname: '{gs-pathname}'
 273         - string:
 274             name: GIT_BASE
 275             default: ssh://gerrit.opnfv.org:29418/$PROJECT
 276             description: "Used for overriding the GIT URL coming from parameters macro."
 277
 278     scm:
 279         - git-scm:
 280             credentials-id: '{ssh-credentials}'
 281             refspec: ''
 282             branch: '{branch}'
 283
 284     properties:
 285         - build-blocker:
 286             use-build-blocker: true
 287             blocking-jobs:
 288                 - "genesis-fuel-daily.*"
 289                 - "genesis-fuel-deploy-runner-{stream}"
 290                 - "genesis-foreman-daily.*"
 291
 292     builders:
 293         - 'foreman-lab-reconfig'
 294         - trigger-builds:
 295           - project: 'genesis-foreman-deploy-{stream}'
 296             git-revision: true
 297             block: true
 298
 299 - job-template:
 300     name: 'genesis-foreman-build-{stream}'
 301
 302     project-type: freestyle
 303
 304     node: ericsson-build
 305
 306     logrotate:
 307         daysToKeep: 30
 308         numToKeep: 10
 309         artifactDaysToKeep: -1
 310         artifactNumToKeep: -1
 311
 312     parameters:
 313         - project-parameter:
 314             project: '{project}'
 315         - foreman-parameter:
 316             installer: '{installer}'
 317             gs-pathname: '{gs-pathname}'
 318     scm:
 319         - git-scm:
 320             credentials-id: '{ssh-credentials}'
 321             refspec: ''
 322             branch: '{branch}'
 323
 324     builders:
 325         - 'foreman-build'
 326         - 'foreman-upload-artifact'
 327         - 'foreman-workspace-cleanup'
 328
 329 - job-template:
 330     name: 'genesis-foreman-deploy-{stream}'
 331
 332     project-type: freestyle
 333
 334     disabled: false
 335
 336     node: opnfv-jump-2
 337
 338     logrotate:
 339         daysToKeep: 30
 340         numToKeep: 10
 341         artifactDaysToKeep: -1
 342         artifactNumToKeep: -1
 343
 344     parameters:
 345         - project-parameter:
 346             project: '{project}'
 347         - foreman-parameter:
 348             installer: '{installer}'
 349             gs-pathname: '{gs-pathname}'
 350         - string:
 351             name: GIT_BASE
 352             default: ssh://gerrit.opnfv.org:29418/$PROJECT
 353             description: "Used for overriding the GIT URL coming from parameters macro."
 354
 355     scm:
 356         - git-scm:
 357             credentials-id: '{ssh-credentials}'
 358             refspec: ''
 359             branch: '{branch}'
 360
 361     builders:
 362         - 'foreman-deploy'
 363
 364 - job-template:
 365     name: 'genesis-foreman-lab-reconfig-{stream}'
 366
 367     project-type: freestyle
 368
 369     disabled: false
 370
 371     node: master
 372
 373     logrotate:
 374         daysToKeep: 30
 375         numToKeep: 10
 376         artifactDaysToKeep: -1
 377         artifactNumToKeep: -1
 378
 379     parameters:
 380         - project-parameter:
 381             project: '{project}'
 382         - foreman-parameter:
 383             installer: '{installer}'
 384             gs-pathname: '{gs-pathname}'
 385         - string:
 386             name: GIT_BASE
 387             default: ssh://gerrit.opnfv.org:29418/$PROJECT
 388             description: "Used for overriding the GIT URL coming from parameters macro."
 389
 390     scm:
 391         - git-scm:
 392             credentials-id: '{ssh-credentials}'
 393             refspec: ''
 394             branch: '{branch}'
 395
 396     properties:
 397         - build-blocker:
 398             use-build-blocker: true
 399             blocking-jobs:
 400                 - "genesis-fuel-daily.*"
 401
 402     builders:
 403         - 'foreman-lab-reconfig'
 404
 405 - job-template:
 406     name: 'genesis-foreman-verify-deploy-virtual-{stream}'
 407
 408     project-type: freestyle
 409
 410     node: opnfv-jump-1
 411
 412     logrotate:
 413         daysToKeep: 30
 414         numToKeep: 10
 415         artifactDaysToKeep: -1
 416         artifactNumToKeep: -1
 417
 418     parameters:
 419         - project-parameter:
 420             project: '{project}'
 421         - gerrit-parameter:
 422             branch: '{branch}'
 423         - foreman-parameter:
 424             installer: '{installer}'
 425             gs-pathname: '{gs-pathname}'
 426         - string:
 427             name: GIT_BASE
 428             default: ssh://gerrit.opnfv.org:29418/$PROJECT
 429             description: "Used for overriding the GIT URL coming from parameters macro."
 430     scm:
 431         - gerrit-trigger-scm:
 432             credentials-id: '{ssh-credentials}'
 433             refspec: '$GERRIT_REFSPEC'
 434             choosing-strategy: 'gerrit'
 435
 436     wrappers:
 437         - ssh-agent-credentials:
 438             user: '{ssh-credentials}'
 439
 440     triggers:
 441         - gerrit:
 442             trigger-on:
 443                 - patchset-created-event:
 444                     exclude-drafts: 'false'
 445                     exclude-trivial-rebase: 'false'
 446                     exclude-no-code-change: 'false'
 447                 - draft-published-event
 448                 - comment-added-contains-event:
 449                     comment-contains-value: 'recheck'
 450                 - comment-added-contains-event:
 451                     comment-contains-value: 'reverify'
 452             projects:
 453               - project-compare-type: 'ANT'
 454                 project-pattern: 'genesis'
 455                 branches:
 456                   - branch-compare-type: 'ANT'
 457                     branch-pattern: '**/{branch}'
 458                 file-paths:
 459                   - compare-type: ANT
 460                     pattern: 'common/ci/**'
 461                   - compare-type: ANT
 462                     pattern: 'common/manifests/**'
 463                   - compare-type: ANT
 464                     pattern: 'common/puppet-opnfv/**'
 465                   - compare-type: ANT
 466                     pattern: 'common/tools/**'
 467                   - compare-type: ANT
 468                     pattern: '{installer}/build/**'
 469                   - compare-type: ANT
 470                     pattern: '{installer}/ci/**'
 471                   - compare-type: ANT
 472                     pattern: '{installer}/include/**'
 473                 forbidden-file-paths:
 474                   - compare-type: ANT
 475                     pattern: 'common/docs/**'
 476                   - compare-type: ANT
 477                     pattern: '{installer}/docs/**'
 478             dependency-jobs: 'genesis-foreman-verify-build-{stream}'
 479
 480
 481     builders:
 482         - 'foreman-deploy-virtual'
 483
 484 ########################
 485 # parameter macros
 486 ########################
 487 - parameter:
 488     name: foreman-parameter
 489     parameters:
 490         - string:
 491             name: INSTALLER
 492             default: '{installer}'
 493             description: "Installer to use."
 494         - string:
 495             name: BUILD_DIRECTORY
 496             default: $WORKSPACE/build_output
 497             description: "Directory where the build artifact will be located upon the completion of the build."
 498         - string:
 499             name: CACHE_DIRECTORY
 500             default: $HOME/opnfv/cache/genesis-$INSTALLER{gs-pathname}
 501             description: "Directory where the cache to be used during the build is located."
 502         - string:
 503             name: GIT_BASE
 504             default: https://gerrit.opnfv.org/gerrit/$PROJECT
 505             description: "Used for overriding the GIT URL coming from Global Jenkins configuration in case if the stuff is done on none-LF HW."
 506         - string:
 507             name: GS_URL
 508             default: artifacts.opnfv.org/$PROJECT/$INSTALLER{gs-pathname}
 509             description: "URL to Google Storage."
 510
 511 ########################
 512 # builder macros
 513 ########################
 514 - builder:
 515     name: 'foreman-lab-reconfig'
 516     builders:
 517         - shell: |
 518             #!/bin/bash
 519             set -o errexit
 520             set -o nounset
 521             set -o pipefail
 522
 523             # check to see if ucs login info file exists
 524             if [ -e ~/.ssh/ucs_creds ];then
 525               source ~/.ssh/ucs_creds
 526             else
 527               echo "Unable to find UCS credentials for LF lab reconfiguration...Exiting"
 528               exit 1
 529             fi
 530
 531             # clone releng
 532             echo "Cloning releng repo..."
 533             if ! GIT_SSL_NO_VERIFY=true git clone https://gerrit.opnfv.org/gerrit/releng; then
 534               echo "Unable to clone releng repo...Exiting"
 535               exit 1
 536             fi
 537
 538             # log info to console
 539             echo "Starting the lab reconfiguration for $INSTALLER..."
 540             echo "--------------------------------------------------------"
 541             echo
 542
 543             # create venv
 544             $WORKSPACE/releng/utils/lab-reconfiguration/create_venv.sh
 545
 546             # disable nounset because 'activate' script contains unbound variable(s)
 547             set +o nounset
 548             # enter venv
 549             source $WORKSPACE/releng/utils/lab-reconfiguration/venv/bin/activate
 550             # set nounset back again
 551             set -o nounset
 552
 553             # verify we are in venv
 554             if [[ ! $(which python | grep venv) ]]; then
 555               echo "Unable to activate venv...Exiting"
 556               exit 1
 557             fi
 558
 559             python $WORKSPACE/releng/utils/lab-reconfiguration/reconfigUcsNet.py -i $ucs_host -u $ucs_user -p $ucs_password -f $WORKSPACE/releng/utils/lab-reconfiguration/foreman.yaml
 560
 561             # while undergoing reboot
 562             sleep 30
 563
 564             # check to see if slave is back up
 565             ping_counter=0
 566             ping_flag=0
 567             while [ "$ping_counter" -lt 20 ]; do
 568               if [[ $(ping -c 5 172.30.10.72) ]]; then
 569                 ping_flag=1
 570                 break
 571               fi
 572               ((ping_counter++))
 573               sleep 10
 574             done
 575
 576             if [ "$ping_flag" -eq 1 ]; then
 577               echo "Slave is pingable, now wait 180 seconds for services to start"
 578               sleep 180
 579             else
 580               echo "Slave did not come back up after reboot: please check opnfv-jump-2"
 581               exit 1
 582             fi
 583
 584             set +o nounset
 585             deactivate
 586
 587             echo
 588             echo "--------------------------------------------------------"
 589             echo "Done!"
 590
 591 - builder:
 592     name: 'foreman-build'
 593     builders:
 594         - shell: |
 595             #!/bin/bash
 596             set -o errexit
 597             set -o nounset
 598             set -o pipefail
 599
 600             # log info to console
 601             echo "Starting the build of $INSTALLER. This could take some time..."
 602             echo "--------------------------------------------------------"
 603             echo
 604
 605             # create the cache directory if it doesn't exist
 606             [[ -d $CACHE_DIRECTORY ]] || mkdir -p $CACHE_DIRECTORY
 607
 608             # set OPNFV_ARTIFACT_VERSION
 609             export OPNFV_ARTIFACT_VERSION=$(date -u +"%Y-%m-%d_%H-%M-%S")
 610
 611             # start the build
 612             cd $WORKSPACE/$INSTALLER/ci
 613             ./build.sh -v $OPNFV_ARTIFACT_VERSION -c file://$CACHE_DIRECTORY $BUILD_DIRECTORY
 614
 615             # list the contents of BUILD_OUTPUT directory
 616             ls -al $BUILD_DIRECTORY
 617
 618             # save information regarding artifact into file
 619             (
 620                 echo "OPNFV_ARTIFACT_VERSION=$OPNFV_ARTIFACT_VERSION"
 621                 echo "OPNFV_GIT_URL=$(git config --get remote.origin.url)"
 622                 echo "OPNFV_GIT_SHA1=$(git rev-parse HEAD)"
 623                 echo "OPNFV_ARTIFACT_URL=$GS_URL/opnfv-$OPNFV_ARTIFACT_VERSION.iso"
 624                 echo "OPNFV_ARTIFACT_MD5SUM=$(md5sum $BUILD_DIRECTORY/OPNFV-CentOS-7-x86_64-$OPNFV_ARTIFACT_VERSION.iso | cut -d' ' -f1)"
 625                 echo "OPNFV_BUILD_URL=$BUILD_URL"
 626             ) > $WORKSPACE/opnfv.properties
 627
 628             echo
 629             echo "--------------------------------------------------------"
 630             echo "Done!"
 631
 632 - builder:
 633     name: 'foreman-deploy'
 634     builders:
 635         - shell: |
 636             #!/bin/bash
 637             set -o errexit
 638             set -o nounset
 639             set -o pipefail
 640
 641             # log info to console
 642             echo "Starting the deployment using $INSTALLER. This could take some time..."
 643             echo "--------------------------------------------------------"
 644             echo
 645
 646             # cleanup first
 647             sudo $WORKSPACE/common/ci/clean.sh -base_config $WORKSPACE/$INSTALLER/ci/inventory/lf_pod2_ksgen_settings.yml
 648
 649             echo "Sleeping for 60 seconds between clean and deploy to allow nodes to power off"
 650             sleep 60
 651
 652             # and then initiate deployment
 653             sudo $WORKSPACE/$INSTALLER/ci/deploy.sh -base_config $WORKSPACE/$INSTALLER/ci/inventory/lf_pod2_ksgen_settings.yml
 654
 655             echo
 656             echo "--------------------------------------------------------"
 657             echo "Done!"
 658
 659 - builder:
 660     name: 'foreman-deploy-virtual'
 661     builders:
 662         - shell: |
 663             #!/bin/bash
 664             set -o errexit
 665             set -o nounset
 666             set -o pipefail
 667
 668             # log info to console
 669             echo "Starting the virtual deployment using $INSTALLER. This could take some time..."
 670             echo "--------------------------------------------------------"
 671             echo
 672
 673             # cleanup baremetal first (in case baremetal was being used on pod2, remove when moved to isolated Virtual pod)
 674             sudo $WORKSPACE/common/ci/clean.sh
 675
 676             # initiate virtual deployment
 677             sudo $WORKSPACE/$INSTALLER/ci/deploy.sh -virtual
 678
 679             echo
 680             echo "--------------------------------------------------------"
 681             echo "Done!"
 682
 683 - builder:
 684     name: 'foreman-upload-artifact'
 685     builders:
 686         - shell: |
 687             #!/bin/bash
 688             set -o errexit
 689             set -o nounset
 690             set -o pipefail
 691
 692             # log info to console
 693             echo "Uploading the $INSTALLER artifact. This could take some time..."
 694             echo "--------------------------------------------------------"
 695             echo
 696
 697             # source the opnfv.properties to get ARTIFACT_VERSION
 698             source $WORKSPACE/opnfv.properties
 699
 700             # upload artifact and additional files to google storage
 701             gsutil cp $BUILD_DIRECTORY/OPNFV-CentOS-7-x86_64-$OPNFV_ARTIFACT_VERSION.iso gs://$GS_URL/opnfv-$OPNFV_ARTIFACT_VERSION.iso > gsutil.iso.log 2>&1
 702             gsutil cp $WORKSPACE/opnfv.properties gs://$GS_URL/opnfv-$OPNFV_ARTIFACT_VERSION.properties > gsutil.properties.log 2>&1
 703             gsutil cp $WORKSPACE/opnfv.properties gs://$GS_URL/latest.properties > gsutil.latest.log 2>&1
 704
 705             echo
 706             echo "--------------------------------------------------------"
 707             echo "Done!"
 708             echo "Artifact is available as http://$GS_URL/opnfv-$OPNFV_ARTIFACT_VERSION.iso"
 709
 710 - builder:
 711     name: 'foreman-workspace-cleanup'
 712     builders:
 713         - shell: |
 714             #!/bin/bash
 715             set -o errexit
 716             set -o nounset
 717             set -o pipefail
 718
 719             # delete everything that is in $WORKSPACE
 720             /bin/rm -rf $WORKSPACE
 721
 722 ########################
 723 # trigger macros
 724 ########################
 725 - trigger:
 726     name: 'foreman-master'
 727     triggers:
 728         - timed: '0 11 * * *'
 729
 730 - trigger:
 731     name: 'foreman-stable-arno'
 732     triggers:
 733         - timed: '0 12 * * *'