jjb/apex/apex.yml

   1 - project:
   2     name: apex
   3     jobs:
   4         - 'apex-verify-{stream}'
   5         - 'apex-merge-{stream}'
   6         - 'apex-build-{stream}'
   7         - 'apex-deploy-virtual-{scenario}-{stream}'
   8         - 'apex-deploy-baremetal-{scenario}-{stream}'
   9         - 'apex-daily-{stream}'
  10
  11     # stream:    branch with - in place of / (eg. stable-arno)
  12     # branch:    branch (eg. stable/arno)
  13     stream:
  14         - master:
  15             branch: 'master'
  16             gs-pathname: ''
  17         - brahmaputra:
  18             branch: 'stable/brahmaputra'
  19             gs-pathname: '/brahmaputra'
  20             disabled: true
  21
  22     project: 'apex'
  23
  24     scenario:
  25          - 'os-odl_l2-nofeature-ha'
  26          - 'os-odl_l2-sfc-ha'
  27          - 'os-odl_l3-nofeature-ha'
  28          - 'os-onos-nofeature-ha'
  29          - 'os-opencontrail-nofeature-ha'
  30
  31 - job-template:
  32     name: 'apex-verify-{stream}'
  33
  34     node: opnfv-jump-1
  35
  36     parameters:
  37         - apex-parameter:
  38             gs-pathname: '{gs-pathname}'
  39         - project-parameter:
  40             project: '{project}'
  41         - gerrit-parameter:
  42             branch: '{branch}'
  43         - string:
  44             name: GIT_BASE
  45             default: https://gerrit.opnfv.org/gerrit/$PROJECT
  46             description: "Used for overriding the GIT URL coming from parameters macro."
  47
  48     scm:
  49         - gerrit-trigger-scm:
  50             credentials-id: '{ssh-credentials}'
  51             refspec: '$GERRIT_REFSPEC'
  52             choosing-strategy: 'gerrit'
  53
  54     triggers:
  55         - gerrit:
  56             trigger-on:
  57                 - patchset-created-event:
  58                     exclude-drafts: 'false'
  59                     exclude-trivial-rebase: 'false'
  60                     exclude-no-code-change: 'false'
  61                 - draft-published-event
  62                 - comment-added-contains-event:
  63                     comment-contains-value: 'recheck'
  64                 - comment-added-contains-event:
  65                     comment-contains-value: 'reverify'
  66             projects:
  67               - project-compare-type: 'ANT'
  68                 project-pattern: 'apex'
  69                 branches:
  70                   - branch-compare-type: 'ANT'
  71                     branch-pattern: '**/{branch}'
  72                 file-paths:
  73                   - compare-type: ANT
  74                     pattern: 'ci/**'
  75                   - compare-type: ANT
  76                     pattern: 'build/**'
  77                   - compare-type: ANT
  78                     pattern: 'lib/**'
  79                   - compare-type: ANT
  80                     pattern: 'config/**'
  81                 forbidden-file-paths:
  82                   - compare-type: ANT
  83                     pattern: 'docs/**'
  84
  85     properties:
  86         - build-blocker:
  87             use-build-blocker: true
  88             blocking-jobs:
  89                 - "apex-daily.*"
  90                 - "apex-deploy.*"
  91                 - "apex-build.*"
  92
  93     builders:
  94         - 'apex-build'
  95         - trigger-builds:
  96           - project: 'apex-deploy-virtual-os-odl_l2-nofeature-ha-{stream}'
  97             predefined-parameters:
  98               BUILD_DIRECTORY=apex-verify-master/build_output
  99             git-revision: false
 100             block: true
 101         - trigger-builds:
 102           - project: 'apex-deploy-virtual-os-onos-nofeature-ha-{stream}'
 103             predefined-parameters:
 104               BUILD_DIRECTORY=apex-verify-master/build_output
 105             git-revision: false
 106             block: true
 107         - 'apex-workspace-cleanup'
 108
 109 - job-template:
 110     name: 'apex-merge-{stream}'
 111
 112     # builder-merge job to run JJB update
 113     #
 114     # This job's purpose is to update all the JJB
 115
 116     node: opnfv-jump-1
 117
 118     disabled: true
 119
 120     parameters:
 121         - apex-parameter:
 122             gs-pathname: '{gs-pathname}'
 123         - project-parameter:
 124             project: '{project}'
 125         - gerrit-parameter:
 126             branch: '{branch}'
 127         - string:
 128             name: GIT_BASE
 129             default: https://gerrit.opnfv.org/gerrit/$PROJECT
 130             description: "Used for overriding the GIT URL coming from parameters macro."
 131
 132     scm:
 133         - gerrit-trigger-scm:
 134             credentials-id: '{ssh-credentials}'
 135             refspec: ''
 136             choosing-strategy: 'default'
 137
 138     triggers:
 139         - gerrit:
 140             trigger-on:
 141                 - change-merged-event
 142                 - comment-added-contains-event:
 143                     comment-contains-value: 'remerge'
 144             projects:
 145               - project-compare-type: 'ANT'
 146                 project-pattern: 'apex'
 147                 branches:
 148                     - branch-compare-type: 'ANT'
 149                       branch-pattern: '**/master'
 150                 forbidden-file-paths:
 151                   - compare-type: ANT
 152                     pattern: 'docs/**'
 153
 154     builders:
 155         - 'apex-build'
 156         - 'apex-deploy-virtual'
 157         - 'apex-workspace-cleanup'
 158
 159 - job-template:
 160     name: 'apex-build-{stream}'
 161
 162     # Job template for builds
 163     #
 164     # Required Variables:
 165     #     stream:    branch with - in place of / (eg. stable)
 166     #     branch:    branch (eg. stable)
 167     node: opnfv-jump-1
 168
 169     disabled: false
 170
 171     parameters:
 172         - project-parameter:
 173             project: '{project}'
 174         - apex-parameter:
 175             gs-pathname: '{gs-pathname}'
 176         - gerrit-parameter:
 177             branch: '{branch}'
 178         - string:
 179             name: GIT_BASE
 180             default: https://gerrit.opnfv.org/gerrit/$PROJECT
 181             description: "Used for overriding the GIT URL coming from parameters macro."
 182
 183     scm:
 184         - git-scm:
 185             credentials-id: '{ssh-credentials}'
 186             refspec: ''
 187             branch: '{branch}'
 188
 189     properties:
 190         - build-blocker:
 191             use-build-blocker: true
 192             blocking-jobs:
 193                 - "apex-deploy.*"
 194
 195     builders:
 196         - 'apex-build'
 197         - 'apex-deploy-virtual'
 198         - 'apex-upload-artifact'
 199         - 'apex-workspace-cleanup'
 200
 201 - job-template:
 202     name: 'apex-deploy-virtual-{scenario}-{stream}'
 203
 204     # Job template for virtual deployment
 205     #
 206     # Required Variables:
 207     #     stream:    branch with - in place of / (eg. stable)
 208     #     branch:    branch (eg. stable)
 209     node: opnfv-jump-1
 210
 211     disabled: false
 212
 213     scm:
 214         - git-scm:
 215             credentials-id: '{ssh-credentials}'
 216             refspec: ''
 217             branch: '{branch}'
 218
 219     parameters:
 220         - project-parameter:
 221             project: '{project}'
 222         - apex-parameter:
 223             gs-pathname: '{gs-pathname}'
 224         - string:
 225             name: DEPLOY_SCENARIO
 226             default: '{scenario}'
 227             description: "Scenario to deploy with."
 228
 229     properties:
 230         - build-blocker:
 231             use-build-blocker: true
 232             blocking-jobs:
 233                 - "apex-deploy.*"
 234                 - "apex-build.*"
 235
 236     builders:
 237         - 'apex-deploy-virtual'
 238         - 'apex-workspace-cleanup'
 239
 240 - job-template:
 241     name: 'apex-deploy-baremetal-{scenario}-{stream}'
 242
 243     # Job template for baremetal deployment
 244     #
 245     # Required Variables:
 246     #     stream:    branch with - in place of / (eg. stable)
 247     #     branch:    branch (eg. stable)
 248     node: opnfv-jump-1
 249
 250     disabled: false
 251
 252     scm:
 253         - git-scm:
 254             credentials-id: '{ssh-credentials}'
 255             refspec: ''
 256             branch: '{branch}'
 257
 258     parameters:
 259         - project-parameter:
 260             project: '{project}'
 261         - apex-parameter:
 262             gs-pathname: '{gs-pathname}'
 263         - string:
 264             name: DEPLOY_SCENARIO
 265             default: '{scenario}'
 266             description: "Scenario to deploy with."
 267
 268     properties:
 269         - build-blocker:
 270             use-build-blocker: true
 271             blocking-jobs:
 272                 - "apex-verify.*"
 273                 - "apex-deploy.*"
 274                 - "apex-build.*"
 275
 276
 277     builders:
 278         - 'apex-deploy-baremetal'
 279         - 'apex-workspace-cleanup'
 280
 281 - job-template:
 282     name: 'apex-daily-{stream}'
 283
 284     # Job template for daily build
 285     #
 286     # Required Variables:
 287     #     stream:    branch with - in place of / (eg. stable)
 288     #     branch:    branch (eg. stable)
 289     node: opnfv-jump-1
 290
 291     disabled: false
 292
 293     scm:
 294         - git-scm:
 295             credentials-id: '{ssh-credentials}'
 296             refspec: ''
 297             branch: '{branch}'
 298
 299     parameters:
 300         - project-parameter:
 301             project: '{project}'
 302         - apex-parameter:
 303             gs-pathname: '{gs-pathname}'
 304
 305     properties:
 306         - build-blocker:
 307             use-build-blocker: true
 308             blocking-jobs:
 309                 - "apex-verify.*"
 310                 - "apex-deploy.*"
 311                 - "apex-build.*"
 312
 313     triggers:
 314         - 'apex-master'
 315
 316     builders:
 317         - trigger-builds:
 318           - project: 'apex-build-{stream}'
 319             git-revision: true
 320             current-parameters: true
 321             block: true
 322         - trigger-builds:
 323           - project: 'apex-deploy-baremetal-os-odl_l2-nofeature-ha-{stream}'
 324             predefined-parameters:
 325               DEPLOY_SCENARIO="os-odl_l2-nofeature-ha"
 326             git-revision: true
 327             block: true
 328         - trigger-builds:
 329           - project: 'functest-apex-opnfv-jump-1-daily-{stream}'
 330             block: true
 331             block-thresholds:
 332                 build-step-failure-threshold: 'never'
 333                 failure-threshold: 'never'
 334                 unstable-threshold: 'FAILURE'
 335         - trigger-builds:
 336           - project: 'yardstick-apex-opnfv-jump-1-daily-{stream}'
 337             block: true
 338             block-thresholds:
 339                 build-step-failure-threshold: 'never'
 340                 failure-threshold: 'never'
 341                 unstable-threshold: 'FAILURE'
 342
 343 ########################
 344 # parameter macros
 345 ########################
 346 - parameter:
 347     name: apex-parameter
 348     parameters:
 349         - string:
 350             name: ARTIFACT_NAME
 351             default: 'latest'
 352             description: "RPM Artifact name that will be appended to GS_URL to deploy a specific artifact"
 353         - string:
 354             name: ARTIFACT_VERSION
 355             default: 'daily'
 356             description: "Artifact version type"
 357         - string:
 358             name: BUILD_DIRECTORY
 359             default: $WORKSPACE/build_output
 360             description: "Directory where the build artifact will be located upon the completion of the build."
 361         - string:
 362             name: CACHE_DIRECTORY
 363             default: $HOME/opnfv/cache{gs-pathname}
 364             description: "Directory where the cache to be used during the build is located."
 365         - string:
 366             name: GIT_BASE
 367             default: https://gerrit.opnfv.org/gerrit/$PROJECT
 368             description: "Used for overriding the GIT URL coming from Global Jenkins configuration in case if the stuff is done on none-LF HW."
 369         - string:
 370             name: GS_URL
 371             default: artifacts.opnfv.org/$PROJECT{gs-pathname}
 372             description: "URL to Google Storage."
 373
 374 ########################
 375 # builder macros
 376 ########################
 377 - builder:
 378     name: 'apex-build'
 379     builders:
 380         - shell: |
 381             #!/bin/bash
 382             set -o errexit
 383             set -o nounset
 384             set -o pipefail
 385             # log info to console
 386             echo "Starting the build of Apex using OpenStack Master packages. This will take some time..."
 387             echo "---------------------------------------------------------------------------------------"
 388             echo
 389             # create the cache directory if it doesn't exist
 390             [[ -d $CACHE_DIRECTORY ]] || mkdir -p $CACHE_DIRECTORY
 391             # set OPNFV_ARTIFACT_VERSION
 392             if echo $GERRIT_BRANCH | grep "brahmaputra" 1> /dev/null; then
 393               export OPNFV_ARTIFACT_VERSION="bramaputra.1.rc0"
 394             else
 395               if echo $BUILD_TAG | grep "apex-verify" 1> /dev/null; then
 396                 export OPNFV_ARTIFACT_VERSION=dev${BUILD_NUMBER}
 397               elif [ "$ARTIFACT_VERSION" == "daily" ]; then
 398                 export OPNFV_ARTIFACT_VERSION=$(date -u +"%Y-%m-%d")
 399               else
 400                 export OPNFV_ARTIFACT_VERSION=${ARTIFACT_VERSION}
 401               fi
 402             fi
 403             # start the build
 404             cd $WORKSPACE/ci
 405             ./build.sh -v $OPNFV_ARTIFACT_VERSION -c file://$CACHE_DIRECTORY $BUILD_DIRECTORY
 406             RPM_VERSION=$(grep Version $BUILD_DIRECTORY/../build/opnfv-apex.spec | awk '{ print $2 }')-$(echo $OPNFV_ARTIFACT_VERSION | tr -d '_-')
 407             # list the contents of BUILD_OUTPUT directory
 408             ls -al $BUILD_DIRECTORY
 409             # save information regarding artifact into file
 410             (
 411                 echo "OPNFV_ARTIFACT_VERSION=$OPNFV_ARTIFACT_VERSION"
 412                 echo "OPNFV_GIT_URL=$(git config --get remote.origin.url)"
 413                 echo "OPNFV_GIT_SHA1=$(git rev-parse HEAD)"
 414                 echo "OPNFV_ARTIFACT_URL=$GS_URL/opnfv-$OPNFV_ARTIFACT_VERSION.iso"
 415                 echo "OPNFV_ARTIFACT_MD5SUM=$(md5sum $BUILD_DIRECTORY/OPNFV-CentOS-7-x86_64-$OPNFV_ARTIFACT_VERSION.iso | cut -d' ' -f1)"
 416                 echo "OPNFV_SRPM_URL=$GS_URL/opnfv-apex-$RPM_VERSION.src.rpm"
 417                 echo "OPNFV_RPM_URL=$GS_URL/opnfv-apex-$RPM_VERSION.noarch.rpm"
 418                 echo "OPNFV_RPM_MD5SUM=$(md5sum $BUILD_DIRECTORY/opnfv-apex-$RPM_VERSION.noarch.rpm | cut -d' ' -f1)"
 419                 echo "OPNFV_BUILD_URL=$BUILD_URL"
 420             ) > $WORKSPACE/opnfv.properties
 421             echo
 422             echo "--------------------------------------------------------"
 423
 424 - builder:
 425     name: 'apex-workspace-cleanup'
 426     builders:
 427         - shell: |
 428             #!/bin/bash
 429             set -o errexit
 430             set -o nounset
 431             set -o pipefail
 432
 433             # delete everything that is in $WORKSPACE
 434             /bin/rm -rf $WORKSPACE
 435
 436 - builder:
 437     name: 'apex-upload-artifact'
 438     builders:
 439         - shell: |
 440             #!/bin/bash
 441             set -o errexit
 442             set -o nounset
 443             set -o pipefail
 444
 445             # log info to console
 446             echo "Uploading the Apex artifact. This could take some time..."
 447             echo "--------------------------------------------------------"
 448             echo
 449
 450             # source the opnfv.properties to get ARTIFACT_VERSION
 451             source $WORKSPACE/opnfv.properties
 452
 453             # upload artifact and additional files to google storage
 454             gsutil cp $BUILD_DIRECTORY/OPNFV-CentOS-7-x86_64-$OPNFV_ARTIFACT_VERSION.iso gs://$GS_URL/opnfv-$OPNFV_ARTIFACT_VERSION.iso > gsutil.iso.log 2>&1
 455             gsutil cp $BUILD_DIRECTORY/$(basename $OPNFV_RPM_URL) gs://$GS_URL/$(basename $OPNFV_RPM_URL) > gsutil.iso.log 2>&1
 456             gsutil cp $BUILD_DIRECTORY/$(basename $OPNFV_SRPM_URL) gs://$GS_URL/$(basename $OPNFV_SRPM_URL) > gsutil.iso.log 2>&1
 457             gsutil cp $WORKSPACE/opnfv.properties gs://$GS_URL/opnfv-$OPNFV_ARTIFACT_VERSION.properties > gsutil.properties.log 2>&1
 458             gsutil cp $WORKSPACE/opnfv.properties gs://$GS_URL/latest.properties > gsutil.latest.log 2>&1
 459
 460             echo
 461             echo "--------------------------------------------------------"
 462             echo "Done!"
 463             echo "ISO Artifact is available as http://$GS_URL/opnfv-$OPNFV_ARTIFACT_VERSION.iso"
 464             echo "RPM Artifact is available as http://$GS_URL/$(basename $OPNFV_RPM_URL)"
 465
 466 - builder:
 467     name: 'apex-deploy-virtual'
 468     builders:
 469         - shell: |
 470             #!/bin/bash
 471             set -o errexit
 472             set -o nounset
 473             set -o pipefail
 474
 475             # log info to console
 476             echo "Starting the Apex virtual deployment."
 477             echo "--------------------------------------------------------"
 478             echo
 479
 480             if [[ ! "$ARTIFACT_NAME" == "latest" ]]; then
 481                 # if artifact name is passed the pull a
 482                 # specific artifact from artifacts.opnfv.org
 483                 RPM_INSTALL_PATH=$GS_URL/$ARTIFACT_NAME
 484             else
 485                 if [[ -f ${BUILD_DIRECTORY}/../opnfv.properties ]]; then
 486                     # if opnfv.properties exists then use the
 487                     # local build. Source the file so we get local OPNFV vars
 488                     source ${BUILD_DIRECTORY}/../opnfv.properties
 489                     RPM_INSTALL_PATH=${BUILD_DIRECTORY}/$(basename $OPNFV_RPM_URL)
 490                 else
 491                     if [[ $BUILD_DIRECTORY == *verify* ]]; then
 492                       echo "BUILD_DIRECTORY is from a verify job, so will not use latest from URL"
 493                       echo "Check that the slave has opnfv.properties in $BUILD_DIRECTORY"
 494                       exit 1
 495                     fi
 496                     # no opnfv.properties means use the latest from artifacts.opnfv.org
 497                     # get the latest.properties to get the link to the latest artifact
 498                     curl -s -o $WORKSPACE/opnfv.properties http://$GS_URL/latest.properties
 499                     [[ -f opnfv.properties ]] || exit 1
 500                     # source the file so we get OPNFV vars
 501                     source opnfv.properties
 502                     RPM_INSTALL_PATH=$OPNFV_RPM_URL
 503                 fi
 504             fi
 505
 506             RPM_LIST=$RPM_INSTALL_PATH
 507             for pkg in common undercloud; do
 508                 RPM_LIST+=" ${RPM_INSTALL_PATH/opnfv-apex/opnfv-apex-${pkg}}"
 509             done
 510
 511             # update / install the new rpm
 512             if rpm -q opnfv-apex > /dev/null; then
 513                if [ $(basename $OPNFV_RPM_URL) == $(rpm -q opnfv-apex).rpm ]; then
 514                  echo "RPM is already installed"
 515                elif sudo yum update -y $RPM_LIST | grep "does not update installed package"; then
 516                    if ! sudo yum downgrade -y $RPM_LIST; then
 517                      sudo yum remove -y opnfv-undercloud opnfv-common
 518                      sudo yum downgrade -y $RPM_INSTALL_PATH
 519                    fi
 520                fi
 521             else
 522                sudo yum install -y $RPM_LIST;
 523             fi
 524
 525             # cleanup virtual machines before we start
 526             sudo opnfv-clean
 527             # initiate virtual deployment
 528             if [ -e /etc/opnfv-apex/network_settings.yaml ]; then
 529               if [ -n "$DEPLOY_SCENARIO" ]; then
 530                 echo "Deploy Scenario set to ${DEPLOY_SCENARIO}"
 531                 if [ -e /etc/opnfv-apex/${DEPLOY_SCENARIO}.yaml ]; then
 532                   sudo opnfv-deploy -v -d /etc/opnfv-apex/${DEPLOY_SCENARIO}.yaml -n /etc/opnfv-apex/network_settings.yaml
 533                 else
 534                   echo "File does not exist /etc/opnfv-apex/${DEPLOY_SCENARIO}.yaml"
 535                   exit 1
 536                 fi
 537               else
 538                 echo "Deploy scenario not set!"
 539                 exit 1
 540             else
 541               sudo opnfv-deploy -v
 542             fi
 543             echo
 544             echo "--------------------------------------------------------"
 545             echo "Done!"
 546
 547 - builder:
 548     name: 'apex-deploy-baremetal'
 549     builders:
 550         - shell: |
 551             #!/bin/bash
 552             set -o errexit
 553             set -o nounset
 554             set -o pipefail
 555
 556             # log info to console
 557             echo "Starting the Apex baremetal deployment."
 558             echo "--------------------------------------------------------"
 559             echo
 560
 561             if [[ ! "$ARTIFACT_NAME" == "latest" ]]; then
 562                 # if artifact name is passed the pull a
 563                 # specific artifact from artifacts.opnfv.org
 564                 RPM_INSTALL_PATH=$GS_URL/$ARTIFACT_NAME
 565             else
 566                 if [[ -f opnfv.properties ]]; then
 567                     # if opnfv.properties exists then use the
 568                     # local build. Source the file so we get local OPNFV vars
 569                     source opnfv.properties
 570                     RPM_INSTALL_PATH=build_output/$(basename $OPNFV_RPM_URL)
 571                 else
 572                     # no opnfv.properties means use the latest from artifacts.opnfv.org
 573                     # get the latest.properties to get the link to the latest artifact
 574                     curl -s -o $WORKSPACE/opnfv.properties http://$GS_URL/latest.properties
 575                     [[ -f opnfv.properties ]] || exit 1
 576                     # source the file so we get OPNFV vars
 577                     source opnfv.properties
 578                     RPM_INSTALL_PATH=$OPNFV_RPM_URL
 579                 fi
 580             fi
 581
 582             source opnfv.properties
 583             RPM_INSTALL_PATH=build_output/$(basename $OPNFV_RPM_URL)
 584             if [ ! -e "$RPM_INSTALL_PATH" ]; then
 585                RPM_INSTALL_PATH=http://${OPNFV_RPM_URL}
 586             fi
 587
 588             RPM_LIST=$RPM_INSTALL_PATH
 589             for pkg in common undercloud; do
 590                 RPM_LIST+=" ${RPM_INSTALL_PATH/opnfv-apex/opnfv-apex-${pkg}}"
 591             done
 592
 593             # update / install the new rpm
 594             if rpm -q opnfv-apex > /dev/null; then
 595                if [ $(basename $OPNFV_RPM_URL) == $(rpm -q opnfv-apex).rpm ]; then
 596                  echo "RPM is already installed"
 597                elif sudo yum update -y $RPM_LIST | grep "does not update installed package"; then
 598                    if ! sudo yum downgrade -y $RPM_LIST; then
 599                      sudo yum remove -y opnfv-undercloud opnfv-common
 600                      sudo yum downgrade -y $RPM_INSTALL_PATH
 601                    fi
 602                fi
 603             else
 604                sudo yum install -y $RPM_LIST;
 605             fi
 606
 607             # cleanup environment before we start
 608             sudo opnfv-clean
 609             # initiate baremetal deployment
 610             if [ -e /etc/opnfv-apex/network_settings.yaml ]; then
 611               if [ -n "$DEPLOY_SCENARIO" ]; then
 612                 echo "Deploy Scenario set to ${DEPLOY_SCENARIO}"
 613                 if [ -e /etc/opnfv-apex/${DEPLOY_SCENARIO}.yaml ]; then
 614                   sudo opnfv-deploy -i  /root/inventory/pod_settings.yaml \
 615                   -d /etc/opnfv-apex/${DEPLOY_SCENARIO}.yaml \
 616                   -n /root/network_settings.yaml
 617                 else
 618                   echo "File does not exist /etc/opnfv-apex/${DEPLOY_SCENARIO}.yaml"
 619                   exit 1
 620                 fi
 621               else
 622                 echo "Deploy scenario not set!"
 623                 exit 1
 624             else
 625               echo "File /etc/opnfv-apex/network_settings.yaml does not exist!"
 626               exit 1
 627             fi
 628
 629             echo
 630             echo "--------------------------------------------------------"
 631             echo "Done!"
 632
 633 #######################
 634 # trigger macros
 635 ########################
 636 - trigger:
 637     name: 'apex-master'
 638     triggers:
 639         - timed: '0 3 * * *'