jjb/apex/apex.yml

   1 - project:
   2     name: apex
   3     jobs:
   4         - 'apex-verify-{stream}'
   5         - 'apex-merge-{stream}'
   6         - 'apex-build-{stream}'
   7         - 'apex-deploy-virtual-{scenario}-{stream}'
   8         - 'apex-deploy-baremetal-{scenario}-{stream}'
   9         - 'apex-daily-{stream}'
  10
  11     # stream:    branch with - in place of / (eg. stable-arno)
  12     # branch:    branch (eg. stable/arno)
  13     stream:
  14         - master:
  15             branch: 'master'
  16             gs-pathname: ''
  17         - brahmaputra:
  18             branch: 'stable/brahmaputra'
  19             gs-pathname: '/brahmaputra'
  20             disabled: true
  21
  22     project: 'apex'
  23
  24     scenario:
  25          - 'os-odl_l2-nofeature-ha'
  26          - 'os-odl_l2-sfc-ha'
  27          - 'os-odl_l3-nofeature-ha'
  28          - 'os-onos-nofeature-ha'
  29          - 'os-opencontrail-nofeature-ha'
  30
  31 - job-template:
  32     name: 'apex-verify-{stream}'
  33
  34     node: opnfv-jump-1
  35
  36     parameters:
  37         - apex-parameter:
  38             gs-pathname: '{gs-pathname}'
  39         - project-parameter:
  40             project: '{project}'
  41         - gerrit-parameter:
  42             branch: '{branch}'
  43         - string:
  44             name: GIT_BASE
  45             default: https://gerrit.opnfv.org/gerrit/$PROJECT
  46             description: "Used for overriding the GIT URL coming from parameters macro."
  47
  48     scm:
  49         - gerrit-trigger-scm:
  50             credentials-id: '{ssh-credentials}'
  51             refspec: '$GERRIT_REFSPEC'
  52             choosing-strategy: 'gerrit'
  53
  54     triggers:
  55         - gerrit:
  56             trigger-on:
  57                 - patchset-created-event:
  58                     exclude-drafts: 'false'
  59                     exclude-trivial-rebase: 'false'
  60                     exclude-no-code-change: 'false'
  61                 - draft-published-event
  62                 - comment-added-contains-event:
  63                     comment-contains-value: 'recheck'
  64                 - comment-added-contains-event:
  65                     comment-contains-value: 'reverify'
  66             projects:
  67               - project-compare-type: 'ANT'
  68                 project-pattern: 'apex'
  69                 branches:
  70                   - branch-compare-type: 'ANT'
  71                     branch-pattern: '**/{branch}'
  72                 file-paths:
  73                   - compare-type: ANT
  74                     pattern: 'ci/**'
  75                   - compare-type: ANT
  76                     pattern: 'build/**'
  77                   - compare-type: ANT
  78                     pattern: 'lib/**'
  79                   - compare-type: ANT
  80                     pattern: 'config/**'
  81                 forbidden-file-paths:
  82                   - compare-type: ANT
  83                     pattern: 'docs/**'
  84
  85     properties:
  86         - build-blocker:
  87             use-build-blocker: true
  88             blocking-jobs:
  89                 - "apex-daily.*"
  90                 - "apex-deploy.*"
  91                 - "apex-build.*"
  92
  93     builders:
  94         - 'apex-build'
  95         - trigger-builds:
  96           - project: 'apex-deploy-virtual-os-odl_l2-nofeature-ha-{stream}'
  97             predefined-parameters:
  98               DEPLOY_SCENARIO="os-odl_l2-nofeature-ha"
  99             git-revision: true
 100             block: true
 101         - trigger-builds:
 102           - project: 'apex-deploy-virtual-os-onos-nofeature-ha-{stream}'
 103             predefined-parameters:
 104               DEPLOY_SCENARIO="os-onos-nofeature-ha"
 105             git-revision: true
 106             block: true
 107         - 'apex-workspace-cleanup'
 108
 109 - job-template:
 110     name: 'apex-merge-{stream}'
 111
 112     # builder-merge job to run JJB update
 113     #
 114     # This job's purpose is to update all the JJB
 115
 116     node: opnfv-jump-1
 117
 118     disabled: true
 119
 120     parameters:
 121         - apex-parameter:
 122             gs-pathname: '{gs-pathname}'
 123         - project-parameter:
 124             project: '{project}'
 125         - gerrit-parameter:
 126             branch: '{branch}'
 127         - string:
 128             name: GIT_BASE
 129             default: https://gerrit.opnfv.org/gerrit/$PROJECT
 130             description: "Used for overriding the GIT URL coming from parameters macro."
 131
 132     scm:
 133         - gerrit-trigger-scm:
 134             credentials-id: '{ssh-credentials}'
 135             refspec: ''
 136             choosing-strategy: 'default'
 137
 138     triggers:
 139         - gerrit:
 140             trigger-on:
 141                 - change-merged-event
 142                 - comment-added-contains-event:
 143                     comment-contains-value: 'remerge'
 144             projects:
 145               - project-compare-type: 'ANT'
 146                 project-pattern: 'apex'
 147                 branches:
 148                     - branch-compare-type: 'ANT'
 149                       branch-pattern: '**/master'
 150                 forbidden-file-paths:
 151                   - compare-type: ANT
 152                     pattern: 'docs/**'
 153
 154     builders:
 155         - 'apex-build'
 156         - 'apex-deploy-virtual'
 157         - 'apex-workspace-cleanup'
 158
 159 - job-template:
 160     name: 'apex-build-{stream}'
 161
 162     # Job template for builds
 163     #
 164     # Required Variables:
 165     #     stream:    branch with - in place of / (eg. stable)
 166     #     branch:    branch (eg. stable)
 167     node: opnfv-jump-1
 168
 169     disabled: false
 170
 171     parameters:
 172         - project-parameter:
 173             project: '{project}'
 174         - apex-parameter:
 175             gs-pathname: '{gs-pathname}'
 176         - gerrit-parameter:
 177             branch: '{branch}'
 178         - string:
 179             name: GIT_BASE
 180             default: https://gerrit.opnfv.org/gerrit/$PROJECT
 181             description: "Used for overriding the GIT URL coming from parameters macro."
 182
 183     scm:
 184         - git-scm:
 185             credentials-id: '{ssh-credentials}'
 186             refspec: ''
 187             branch: '{branch}'
 188
 189     properties:
 190         - build-blocker:
 191             use-build-blocker: true
 192             blocking-jobs:
 193                 - "apex-deploy.*"
 194
 195     builders:
 196         - 'apex-build'
 197         - 'apex-deploy-virtual'
 198         - 'apex-upload-artifact'
 199         - 'apex-workspace-cleanup'
 200
 201 - job-template:
 202     name: 'apex-deploy-virtual-{scenario}-{stream}'
 203
 204     # Job template for virtual deployment
 205     #
 206     # Required Variables:
 207     #     stream:    branch with - in place of / (eg. stable)
 208     #     branch:    branch (eg. stable)
 209     node: opnfv-jump-1
 210
 211     disabled: false
 212
 213     scm:
 214         - git-scm:
 215             credentials-id: '{ssh-credentials}'
 216             refspec: ''
 217             branch: '{branch}'
 218
 219     parameters:
 220         - project-parameter:
 221             project: '{project}'
 222         - apex-parameter:
 223             gs-pathname: '{gs-pathname}'
 224         - string:
 225             name: DEPLOY_SCENARIO
 226             default: '{scenario}'
 227             description: "Scenario to deploy with."
 228
 229     properties:
 230         - build-blocker:
 231             use-build-blocker: true
 232             blocking-jobs:
 233                 - "apex-verify.*"
 234                 - "apex-deploy.*"
 235                 - "apex-build.*"
 236
 237     builders:
 238         - 'apex-deploy-virtual'
 239         - 'apex-workspace-cleanup'
 240
 241 - job-template:
 242     name: 'apex-deploy-baremetal-{scenario}-{stream}'
 243
 244     # Job template for baremetal deployment
 245     #
 246     # Required Variables:
 247     #     stream:    branch with - in place of / (eg. stable)
 248     #     branch:    branch (eg. stable)
 249     node: opnfv-jump-1
 250
 251     disabled: false
 252
 253     scm:
 254         - git-scm:
 255             credentials-id: '{ssh-credentials}'
 256             refspec: ''
 257             branch: '{branch}'
 258
 259     parameters:
 260         - project-parameter:
 261             project: '{project}'
 262         - apex-parameter:
 263             gs-pathname: '{gs-pathname}'
 264         - string:
 265             name: DEPLOY_SCENARIO
 266             default: '{scenario}'
 267             description: "Scenario to deploy with."
 268
 269     properties:
 270         - build-blocker:
 271             use-build-blocker: true
 272             blocking-jobs:
 273                 - "apex-verify.*"
 274                 - "apex-deploy.*"
 275                 - "apex-build.*"
 276
 277
 278     builders:
 279         - 'apex-deploy-baremetal'
 280         - 'apex-workspace-cleanup'
 281
 282 - job-template:
 283     name: 'apex-daily-{stream}'
 284
 285     # Job template for daily build
 286     #
 287     # Required Variables:
 288     #     stream:    branch with - in place of / (eg. stable)
 289     #     branch:    branch (eg. stable)
 290     node: opnfv-jump-1
 291
 292     disabled: false
 293
 294     scm:
 295         - git-scm:
 296             credentials-id: '{ssh-credentials}'
 297             refspec: ''
 298             branch: '{branch}'
 299
 300     parameters:
 301         - project-parameter:
 302             project: '{project}'
 303         - apex-parameter:
 304             gs-pathname: '{gs-pathname}'
 305
 306     properties:
 307         - build-blocker:
 308             use-build-blocker: true
 309             blocking-jobs:
 310                 - "apex-verify.*"
 311                 - "apex-deploy.*"
 312                 - "apex-build.*"
 313
 314     triggers:
 315         - 'apex-master'
 316
 317     builders:
 318         - trigger-builds:
 319           - project: 'apex-build-{stream}'
 320             git-revision: true
 321             current-parameters: true
 322             block: true
 323         - trigger-builds:
 324           - project: 'apex-deploy-baremetal-os-odl_l2-nofeature-ha-{stream}'
 325             predefined-parameters:
 326               DEPLOY_SCENARIO="os-odl_l2-nofeature-ha"
 327             git-revision: true
 328             block: true
 329         - trigger-builds:
 330           - project: 'functest-apex-opnfv-jump-1-daily-{stream}'
 331             block: true
 332             block-thresholds:
 333                 build-step-failure-threshold: 'never'
 334                 failure-threshold: 'never'
 335                 unstable-threshold: 'FAILURE'
 336         - trigger-builds:
 337           - project: 'yardstick-apex-opnfv-jump-1-daily-{stream}'
 338             block: true
 339             block-thresholds:
 340                 build-step-failure-threshold: 'never'
 341                 failure-threshold: 'never'
 342                 unstable-threshold: 'FAILURE'
 343
 344 ########################
 345 # parameter macros
 346 ########################
 347 - parameter:
 348     name: apex-parameter
 349     parameters:
 350         - string:
 351             name: ARTIFACT_NAME
 352             default: 'latest'
 353             description: "RPM Artifact name that will be appended to GS_URL to deploy a specific artifact"
 354         - string:
 355             name: ARTIFACT_VERSION
 356             default: 'daily'
 357             description: "Artifact version type"
 358         - string:
 359             name: BUILD_DIRECTORY
 360             default: $WORKSPACE/build_output
 361             description: "Directory where the build artifact will be located upon the completion of the build."
 362         - string:
 363             name: CACHE_DIRECTORY
 364             default: $HOME/opnfv/cache{gs-pathname}
 365             description: "Directory where the cache to be used during the build is located."
 366         - string:
 367             name: GIT_BASE
 368             default: https://gerrit.opnfv.org/gerrit/$PROJECT
 369             description: "Used for overriding the GIT URL coming from Global Jenkins configuration in case if the stuff is done on none-LF HW."
 370         - string:
 371             name: GS_URL
 372             default: artifacts.opnfv.org/$PROJECT{gs-pathname}
 373             description: "URL to Google Storage."
 374
 375 ########################
 376 # builder macros
 377 ########################
 378 - builder:
 379     name: 'apex-build'
 380     builders:
 381         - shell: |
 382             #!/bin/bash
 383             set -o errexit
 384             set -o nounset
 385             set -o pipefail
 386             # log info to console
 387             echo "Starting the build of Apex using OpenStack Master packages. This will take some time..."
 388             echo "---------------------------------------------------------------------------------------"
 389             echo
 390             # create the cache directory if it doesn't exist
 391             [[ -d $CACHE_DIRECTORY ]] || mkdir -p $CACHE_DIRECTORY
 392             # set OPNFV_ARTIFACT_VERSION
 393             if echo $GERRIT_BRANCH | grep "brahmaputra" 1> /dev/null; then
 394               export OPNFV_ARTIFACT_VERSION="bramaputra.1.rc0"
 395             else
 396               if echo $BUILD_TAG | grep "apex-verify" 1> /dev/null; then
 397                 export OPNFV_ARTIFACT_VERSION=dev${BUILD_NUMBER}
 398               elif [ "$ARTIFACT_VERSION" == "daily" ]; then
 399                 export OPNFV_ARTIFACT_VERSION=$(date -u +"%Y-%m-%d")
 400               else
 401                 export OPNFV_ARTIFACT_VERSION=${ARTIFACT_VERSION}
 402               fi
 403             fi
 404             # start the build
 405             cd $WORKSPACE/ci
 406             ./build.sh -v $OPNFV_ARTIFACT_VERSION -c file://$CACHE_DIRECTORY $BUILD_DIRECTORY
 407             RPM_VERSION=$(grep Version $BUILD_DIRECTORY/../build/opnfv-apex.spec | awk '{ print $2 }')-$(echo $OPNFV_ARTIFACT_VERSION | tr -d '_-')
 408             # list the contents of BUILD_OUTPUT directory
 409             ls -al $BUILD_DIRECTORY
 410             # save information regarding artifact into file
 411             (
 412                 echo "OPNFV_ARTIFACT_VERSION=$OPNFV_ARTIFACT_VERSION"
 413                 echo "OPNFV_GIT_URL=$(git config --get remote.origin.url)"
 414                 echo "OPNFV_GIT_SHA1=$(git rev-parse HEAD)"
 415                 echo "OPNFV_ARTIFACT_URL=$GS_URL/opnfv-$OPNFV_ARTIFACT_VERSION.iso"
 416                 echo "OPNFV_ARTIFACT_MD5SUM=$(md5sum $BUILD_DIRECTORY/OPNFV-CentOS-7-x86_64-$OPNFV_ARTIFACT_VERSION.iso | cut -d' ' -f1)"
 417                 echo "OPNFV_SRPM_URL=$GS_URL/opnfv-apex-$RPM_VERSION.src.rpm"
 418                 echo "OPNFV_RPM_URL=$GS_URL/opnfv-apex-$RPM_VERSION.noarch.rpm"
 419                 echo "OPNFV_RPM_MD5SUM=$(md5sum $BUILD_DIRECTORY/opnfv-apex-$RPM_VERSION.noarch.rpm | cut -d' ' -f1)"
 420                 echo "OPNFV_BUILD_URL=$BUILD_URL"
 421             ) > $WORKSPACE/opnfv.properties
 422             echo
 423             echo "--------------------------------------------------------"
 424
 425 - builder:
 426     name: 'apex-workspace-cleanup'
 427     builders:
 428         - shell: |
 429             #!/bin/bash
 430             set -o errexit
 431             set -o nounset
 432             set -o pipefail
 433
 434             # delete everything that is in $WORKSPACE
 435             /bin/rm -rf $WORKSPACE
 436
 437 - builder:
 438     name: 'apex-upload-artifact'
 439     builders:
 440         - shell: |
 441             #!/bin/bash
 442             set -o errexit
 443             set -o nounset
 444             set -o pipefail
 445
 446             # log info to console
 447             echo "Uploading the Apex artifact. This could take some time..."
 448             echo "--------------------------------------------------------"
 449             echo
 450
 451             # source the opnfv.properties to get ARTIFACT_VERSION
 452             source $WORKSPACE/opnfv.properties
 453
 454             # upload artifact and additional files to google storage
 455             gsutil cp $BUILD_DIRECTORY/OPNFV-CentOS-7-x86_64-$OPNFV_ARTIFACT_VERSION.iso gs://$GS_URL/opnfv-$OPNFV_ARTIFACT_VERSION.iso > gsutil.iso.log 2>&1
 456             gsutil cp $BUILD_DIRECTORY/$(basename $OPNFV_RPM_URL) gs://$GS_URL/$(basename $OPNFV_RPM_URL) > gsutil.iso.log 2>&1
 457             gsutil cp $BUILD_DIRECTORY/$(basename $OPNFV_SRPM_URL) gs://$GS_URL/$(basename $OPNFV_SRPM_URL) > gsutil.iso.log 2>&1
 458             gsutil cp $WORKSPACE/opnfv.properties gs://$GS_URL/opnfv-$OPNFV_ARTIFACT_VERSION.properties > gsutil.properties.log 2>&1
 459             gsutil cp $WORKSPACE/opnfv.properties gs://$GS_URL/latest.properties > gsutil.latest.log 2>&1
 460
 461             echo
 462             echo "--------------------------------------------------------"
 463             echo "Done!"
 464             echo "ISO Artifact is available as http://$GS_URL/opnfv-$OPNFV_ARTIFACT_VERSION.iso"
 465             echo "RPM Artifact is available as http://$GS_URL/$(basename $OPNFV_RPM_URL)"
 466
 467 - builder:
 468     name: 'apex-deploy-virtual'
 469     builders:
 470         - shell: |
 471             #!/bin/bash
 472             set -o errexit
 473             set -o nounset
 474             set -o pipefail
 475
 476             # log info to console
 477             echo "Starting the Apex virtual deployment."
 478             echo "--------------------------------------------------------"
 479             echo
 480
 481             if [[ ! "$ARTIFACT_NAME" == "latest" ]]; then
 482                 # if artifact name is passed the pull a
 483                 # specific artifact from artifacts.opnfv.org
 484                 RPM_INSTALL_PATH=$GS_URL/$ARTIFACT_NAME
 485             else
 486                 if [[ -f opnfv.properties ]]; then
 487                     # if opnfv.properties exists then use the
 488                     # local build. Source the file so we get local OPNFV vars
 489                     source opnfv.properties
 490                     RPM_INSTALL_PATH=build_output/$(basename $OPNFV_RPM_URL)
 491                 else
 492                     # no opnfv.properties means use the latest from artifacts.opnfv.org
 493                     # get the latest.properties to get the link to the latest artifact
 494                     curl -s -o $WORKSPACE/opnfv.properties http://$GS_URL/latest.properties
 495                     [[ -f opnfv.properties ]] || exit 1
 496                     # source the file so we get OPNFV vars
 497                     source opnfv.properties
 498                     RPM_INSTALL_PATH=$OPNFV_RPM_URL
 499                 fi
 500             fi
 501
 502             source opnfv.properties
 503             RPM_INSTALL_PATH=build_output/$(basename $OPNFV_RPM_URL)
 504             if [ ! -e "$RPM_INSTALL_PATH" ]; then
 505                RPM_INSTALL_PATH=http://${OPNFV_RPM_URL}
 506             fi
 507
 508             RPM_LIST=$RPM_INSTALL_PATH
 509             for pkg in common undercloud; do
 510                 RPM_LIST+=" ${RPM_INSTALL_PATH/opnfv-apex/opnfv-apex-${pkg}}"
 511             done
 512
 513             # update / install the new rpm
 514             if rpm -q opnfv-apex > /dev/null; then
 515                if [ $(basename $OPNFV_RPM_URL) == $(rpm -q opnfv-apex).rpm ]; then
 516                  echo "RPM is already installed"
 517                elif sudo yum update -y $RPM_LIST | grep "does not update installed package"; then
 518                    if ! sudo yum downgrade -y $RPM_LIST; then
 519                      sudo yum remove -y opnfv-undercloud opnfv-common
 520                      sudo yum downgrade -y $RPM_INSTALL_PATH
 521                    fi
 522                fi
 523             else
 524                sudo yum install -y $RPM_LIST;
 525             fi
 526
 527             # cleanup virtual machines before we start
 528             sudo opnfv-clean
 529             # initiate virtual deployment
 530             if [ -e /etc/opnfv-apex/network_settings.yaml ]; then
 531               if [ -n "$DEPLOY_SCENARIO" ]; then
 532                 echo "Deploy Scenario set to ${DEPLOY_SCENARIO}"
 533                 if [ -e /etc/opnfv-apex/${DEPLOY_SCENARIO}.yaml ]; then
 534                   sudo opnfv-deploy -v -d /etc/opnfv-apex/${DEPLOY_SCENARIO}.yaml -n /etc/opnfv-apex/network_settings.yaml
 535                 else
 536                   echo "File does not exist /etc/opnfv-apex/${DEPLOY_SCENARIO}.yaml"
 537                   exit 1
 538                 fi
 539               else
 540                 echo "Deploy scenario not set!"
 541                 exit 1
 542             else
 543               sudo opnfv-deploy -v
 544             fi
 545             echo
 546             echo "--------------------------------------------------------"
 547             echo "Done!"
 548
 549 - builder:
 550     name: 'apex-deploy-baremetal'
 551     builders:
 552         - shell: |
 553             #!/bin/bash
 554             set -o errexit
 555             set -o nounset
 556             set -o pipefail
 557
 558             # log info to console
 559             echo "Starting the Apex baremetal deployment."
 560             echo "--------------------------------------------------------"
 561             echo
 562
 563             if [[ ! "$ARTIFACT_NAME" == "latest" ]]; then
 564                 # if artifact name is passed the pull a
 565                 # specific artifact from artifacts.opnfv.org
 566                 RPM_INSTALL_PATH=$GS_URL/$ARTIFACT_NAME
 567             else
 568                 if [[ -f opnfv.properties ]]; then
 569                     # if opnfv.properties exists then use the
 570                     # local build. Source the file so we get local OPNFV vars
 571                     source opnfv.properties
 572                     RPM_INSTALL_PATH=build_output/$(basename $OPNFV_RPM_URL)
 573                 else
 574                     # no opnfv.properties means use the latest from artifacts.opnfv.org
 575                     # get the latest.properties to get the link to the latest artifact
 576                     curl -s -o $WORKSPACE/opnfv.properties http://$GS_URL/latest.properties
 577                     [[ -f opnfv.properties ]] || exit 1
 578                     # source the file so we get OPNFV vars
 579                     source opnfv.properties
 580                     RPM_INSTALL_PATH=$OPNFV_RPM_URL
 581                 fi
 582             fi
 583
 584             source opnfv.properties
 585             RPM_INSTALL_PATH=build_output/$(basename $OPNFV_RPM_URL)
 586             if [ ! -e "$RPM_INSTALL_PATH" ]; then
 587                RPM_INSTALL_PATH=http://${OPNFV_RPM_URL}
 588             fi
 589
 590             RPM_LIST=$RPM_INSTALL_PATH
 591             for pkg in common undercloud; do
 592                 RPM_LIST+=" ${RPM_INSTALL_PATH/opnfv-apex/opnfv-apex-${pkg}}"
 593             done
 594
 595             # update / install the new rpm
 596             if rpm -q opnfv-apex > /dev/null; then
 597                if [ $(basename $OPNFV_RPM_URL) == $(rpm -q opnfv-apex).rpm ]; then
 598                  echo "RPM is already installed"
 599                elif sudo yum update -y $RPM_LIST | grep "does not update installed package"; then
 600                    if ! sudo yum downgrade -y $RPM_LIST; then
 601                      sudo yum remove -y opnfv-undercloud opnfv-common
 602                      sudo yum downgrade -y $RPM_INSTALL_PATH
 603                    fi
 604                fi
 605             else
 606                sudo yum install -y $RPM_LIST;
 607             fi
 608
 609             # cleanup environment before we start
 610             sudo opnfv-clean
 611             # initiate baremetal deployment
 612             if [ -e /etc/opnfv-apex/network_settings.yaml ]; then
 613               if [ -n "$DEPLOY_SCENARIO" ]; then
 614                 echo "Deploy Scenario set to ${DEPLOY_SCENARIO}"
 615                 if [ -e /etc/opnfv-apex/${DEPLOY_SCENARIO}.yaml ]; then
 616                   sudo opnfv-deploy -i  /root/inventory/pod_settings.yaml \
 617                   -d /etc/opnfv-apex/${DEPLOY_SCENARIO}.yaml \
 618                   -n /root/network_settings.yaml
 619                 else
 620                   echo "File does not exist /etc/opnfv-apex/${DEPLOY_SCENARIO}.yaml"
 621                   exit 1
 622                 fi
 623               else
 624                 echo "Deploy scenario not set!"
 625                 exit 1
 626             else
 627               echo "File /etc/opnfv-apex/network_settings.yaml does not exist!"
 628               exit 1
 629             fi
 630
 631             echo
 632             echo "--------------------------------------------------------"
 633             echo "Done!"
 634
 635 #######################
 636 # trigger macros
 637 ########################
 638 - trigger:
 639     name: 'apex-master'
 640     triggers:
 641         - timed: '0 3 * * *'