jjb/genesis/genesis-fuel.yml

   1 ########################
   2 # Job configuration for genesis/fuel
   3 ########################
   4 - project:
   5
   6     name: genesis-fuel
   7
   8     project: 'genesis'
   9
  10     installer: 'fuel'
  11
  12     jobs:
  13         - 'genesis-fuel-verify-build-{stream}'
  14         - 'genesis-fuel-merge-{stream}'
  15         - 'genesis-fuel-daily-{stream}'
  16         - 'genesis-fuel-build-{stream}'
  17         - 'genesis-fuel-deploy-{stream}'
  18         - 'genesis-fuel-deploy-runner-{stream}'
  19         - 'genesis-fuel-lab-reconfig-{stream}'
  20         - 'genesis-fuel-verify-deploy-virtual-{stream}'
  21
  22     # stream:    branch with - in place of / (eg. stable-arno)
  23     # branch:    branch (eg. stable/arno)
  24     stream:
  25         - master:
  26             branch: 'master'
  27             gs-pathname: ''
  28         - stable-arno:
  29             branch: 'stable/arno'
  30             gs-pathname: '/arno'
  31
  32 ########################
  33 # job templates
  34 ########################
  35
  36 - job-template:
  37     name: 'genesis-fuel-verify-build-{stream}'
  38
  39     project-type: freestyle
  40
  41     node: ericsson-build
  42
  43     concurrent: true
  44
  45     properties:
  46         - throttle:
  47             enabled: true
  48             max-total: 3
  49
  50     logrotate:
  51         daysToKeep: 30
  52         numToKeep: 10
  53         artifactDaysToKeep: -1
  54         artifactNumToKeep: -1
  55
  56     parameters:
  57         - project-parameter:
  58             project: '{project}'
  59         - gerrit-parameter:
  60             branch: '{branch}'
  61         - fuel-parameter:
  62             installer: '{installer}'
  63             gs-pathname: '{gs-pathname}'
  64
  65     scm:
  66         - gerrit-trigger-scm:
  67             credentials-id: '{ssh-credentials}'
  68             refspec: '$GERRIT_REFSPEC'
  69             choosing-strategy: 'gerrit'
  70
  71     wrappers:
  72         - ssh-agent-credentials:
  73             user: '{ssh-credentials}'
  74
  75     triggers:
  76         - gerrit:
  77             trigger-on:
  78                 - patchset-created-event:
  79                     exclude-drafts: 'false'
  80                     exclude-trivial-rebase: 'false'
  81                     exclude-no-code-change: 'false'
  82                 - draft-published-event
  83                 - comment-added-contains-event:
  84                     comment-contains-value: 'recheck'
  85                 - comment-added-contains-event:
  86                     comment-contains-value: 'reverify'
  87             projects:
  88               - project-compare-type: 'ANT'
  89                 project-pattern: 'genesis'
  90                 branches:
  91                   - branch-compare-type: 'ANT'
  92                     branch-pattern: '**/{branch}'
  93                 file-paths:
  94                   - compare-type: ANT
  95                     pattern: 'common/ci/**'
  96                   - compare-type: ANT
  97                     pattern: 'common/manifests/**'
  98                   - compare-type: ANT
  99                     pattern: 'common/puppet-opnfv/**'
 100                   - compare-type: ANT
 101                     pattern: 'common/tools/**'
 102                   - compare-type: ANT
 103                     pattern: '{installer}/build/**'
 104                   - compare-type: ANT
 105                     pattern: '{installer}/ci/**'
 106                   - compare-type: ANT
 107                     pattern: '{installer}/deploy/**'
 108                   - compare-type: ANT
 109                     pattern: '{installer}/include/**'
 110                   - compare-type: ANT
 111                     pattern: '{installer}/prototypes/**'
 112                 forbidden-file-paths:
 113                   - compare-type: ANT
 114                     pattern: 'common/docs/**'
 115                   - compare-type: ANT
 116                     pattern: '{installer}/docs/**'
 117
 118     builders:
 119         - 'fuel-build'
 120         - 'fuel-workspace-cleanup'
 121
 122 - job-template:
 123     name: 'genesis-fuel-merge-{stream}'
 124
 125     project-type: freestyle
 126
 127     node: ericsson-build
 128
 129     concurrent: true
 130
 131     properties:
 132         - throttle:
 133             enabled: true
 134             max-total: 2
 135
 136     logrotate:
 137         daysToKeep: 30
 138         numToKeep: 40
 139         artifactDaysToKeep: -1
 140         artifactNumToKeep: 5
 141
 142     parameters:
 143         - project-parameter:
 144             project: '{project}'
 145         - gerrit-parameter:
 146             branch: '{branch}'
 147         - fuel-parameter:
 148             installer: '{installer}'
 149             gs-pathname: ''
 150
 151     scm:
 152         - gerrit-trigger-scm:
 153             credentials-id: '{ssh-credentials}'
 154             refspec: ''
 155             choosing-strategy: 'default'
 156
 157     wrappers:
 158         - ssh-agent-credentials:
 159             user: '{ssh-credentials}'
 160
 161     triggers:
 162         - gerrit:
 163             trigger-on:
 164                 - change-merged-event
 165                 - comment-added-contains-event:
 166                     comment-contains-value: 'remerge'
 167             projects:
 168               - project-compare-type: 'ANT'
 169                 project-pattern: 'genesis'
 170                 branches:
 171                   - branch-compare-type: 'ANT'
 172                     branch-pattern: '**/{branch}'
 173                 file-paths:
 174                   - compare-type: ANT
 175                     pattern: 'common/ci/**'
 176                   - compare-type: ANT
 177                     pattern: 'common/manifests/**'
 178                   - compare-type: ANT
 179                     pattern: 'common/puppet-opnfv/**'
 180                   - compare-type: ANT
 181                     pattern: 'common/tools/**'
 182                   - compare-type: ANT
 183                     pattern: '{installer}/build/**'
 184                   - compare-type: ANT
 185                     pattern: '{installer}/ci/**'
 186                   - compare-type: ANT
 187                     pattern: '{installer}/deploy/**'
 188                   - compare-type: ANT
 189                     pattern: '{installer}/include/**'
 190                   - compare-type: ANT
 191                     pattern: '{installer}/prototypes/**'
 192                 forbidden-file-paths:
 193                   - compare-type: ANT
 194                     pattern: 'common/docs/**'
 195                   - compare-type: ANT
 196                     pattern: '{installer}/docs/**'
 197
 198     builders:
 199         - 'fuel-build'
 200         - 'fuel-workspace-cleanup'
 201
 202 - job-template:
 203     name: 'genesis-fuel-daily-{stream}'
 204
 205     project-type: freestyle
 206
 207     node: ericsson-build
 208
 209     disabled: false
 210
 211     triggers:
 212         - 'fuel-{stream}'
 213
 214     parameters:
 215         - project-parameter:
 216             project: '{project}'
 217         - fuel-parameter:
 218             installer: '{installer}'
 219             gs-pathname: '{gs-pathname}'
 220
 221     scm:
 222         - git-scm:
 223             credentials-id: '{ssh-credentials}'
 224             refspec: ''
 225             branch: '{branch}'
 226
 227     logrotate:
 228         daysToKeep: 30
 229         numToKeep: 10
 230         artifactDaysToKeep: -1
 231         artifactNumToKeep: -1
 232
 233     builders:
 234         - trigger-builds:
 235           - project: 'genesis-fuel-build-{stream}'
 236             git-revision: true
 237             block: true
 238         - trigger-builds:
 239           - project: 'genesis-fuel-lab-reconfig-{stream}'
 240             git-revision: true
 241             block: true
 242         - trigger-builds:
 243           - project: 'genesis-fuel-deploy-{stream}'
 244             git-revision: true
 245             block: true
 246         - trigger-builds:
 247           - project: 'functest-fuel-{stream}'
 248             block: true
 249             block-thresholds:
 250                 build-step-failure-threshold: 'never'
 251                 failure-threshold: 'never'
 252                 unstable-threshold: 'FAILURE'
 253         - trigger-builds:
 254           - project: 'yardstick-fuel-{stream}'
 255             block: true
 256             block-thresholds:
 257                 build-step-failure-threshold: 'never'
 258                 failure-threshold: 'never'
 259                 unstable-threshold: 'FAILURE'
 260
 261 - job-template:
 262     name: 'genesis-fuel-deploy-runner-{stream}'
 263
 264     project-type: freestyle
 265
 266     disabled: false
 267
 268     node: master
 269
 270     logrotate:
 271         daysToKeep: 30
 272         numToKeep: 10
 273         artifactDaysToKeep: -1
 274         artifactNumToKeep: -1
 275
 276     parameters:
 277         - project-parameter:
 278             project: '{project}'
 279         - foreman-parameter:
 280             installer: '{installer}'
 281             gs-pathname: '{gs-pathname}'
 282         - string:
 283             name: GIT_BASE
 284             default: ssh://gerrit.opnfv.org:29418/$PROJECT
 285             description: "Used for overriding the GIT URL coming from parameters macro."
 286
 287     scm:
 288         - git-scm:
 289             credentials-id: '{ssh-credentials}'
 290             refspec: ''
 291             branch: '{branch}'
 292
 293     properties:
 294         - build-blocker:
 295             use-build-blocker: true
 296             blocking-jobs:
 297                 - "genesis-foreman-daily.*"
 298                 - "genesis-foreman-deploy-runner-{stream}"
 299                 - "genesis-fuel-daily.*"
 300
 301     builders:
 302         - 'fuel-lab-reconfig'
 303         - trigger-builds:
 304           - project: 'genesis-fuel-deploy-{stream}'
 305             git-revision: true
 306             block: true
 307
 308 - job-template:
 309     name: 'genesis-fuel-build-{stream}'
 310
 311     project-type: freestyle
 312
 313     node: ericsson-build
 314
 315     logrotate:
 316         daysToKeep: 30
 317         numToKeep: 10
 318         artifactDaysToKeep: -1
 319         artifactNumToKeep: -1
 320
 321     parameters:
 322         - project-parameter:
 323             project: '{project}'
 324         - fuel-parameter:
 325             installer: '{installer}'
 326             gs-pathname: '{gs-pathname}'
 327
 328     scm:
 329         - git-scm:
 330             credentials-id: '{ssh-credentials}'
 331             refspec: ''
 332             branch: '{branch}'
 333
 334     builders:
 335         - 'fuel-build'
 336         - 'fuel-upload-artifact'
 337         - 'fuel-workspace-cleanup'
 338
 339     publishers:
 340         - email:
 341             recipients: jonas.bjurel@ericsson.com stefan.k.berg@ericsson.com
 342
 343 - job-template:
 344     name: 'genesis-fuel-deploy-{stream}'
 345
 346     project-type: freestyle
 347
 348     disabled: false
 349
 350     node: opnfv-jump-2
 351
 352     parameters:
 353         - project-parameter:
 354             project: '{project}'
 355         - fuel-parameter:
 356             installer: '{installer}'
 357             gs-pathname: '{gs-pathname}'
 358         - string:
 359             name: GIT_BASE
 360             default: ssh://gerrit.opnfv.org:29418/$PROJECT
 361             description: "POD2 has some issues with cloning using https so that's why GIT_BASE is overriden here again."
 362
 363     scm:
 364         - git-scm:
 365             credentials-id: '{ssh-credentials}'
 366             refspec: ''
 367             branch: '{branch}'
 368
 369     logrotate:
 370         daysToKeep: 30
 371         numToKeep: 10
 372         artifactDaysToKeep: -1
 373         artifactNumToKeep: -1
 374
 375     builders:
 376         - 'fuel-download-artifact'
 377         - 'fuel-deploy'
 378
 379     publishers:
 380         - email:
 381             recipients: jonas.bjurel@ericsson.com stefan.k.berg@ericsson.com
 382
 383 - job-template:
 384     name: 'genesis-fuel-lab-reconfig-{stream}'
 385
 386     project-type: freestyle
 387
 388     disabled: false
 389
 390     node: master
 391
 392     logrotate:
 393         daysToKeep: 30
 394         numToKeep: 10
 395         artifactDaysToKeep: -1
 396         artifactNumToKeep: -1
 397
 398     parameters:
 399         - project-parameter:
 400             project: '{project}'
 401         - fuel-parameter:
 402             installer: '{installer}'
 403             gs-pathname: '{gs-pathname}'
 404         - string:
 405             name: GIT_BASE
 406             default: ssh://gerrit.opnfv.org:29418/$PROJECT
 407             description: "Used for overriding the GIT URL coming from parameters macro."
 408
 409     scm:
 410         - git-scm:
 411             credentials-id: '{ssh-credentials}'
 412             refspec: ''
 413             branch: '{branch}'
 414
 415     properties:
 416         - build-blocker:
 417             use-build-blocker: true
 418             blocking-jobs:
 419                 - "genesis-foreman-daily.*"
 420
 421     builders:
 422         - 'fuel-lab-reconfig'
 423
 424 - job-template:
 425     name: 'genesis-fuel-verify-deploy-virtual-{stream}'
 426
 427     project-type: freestyle
 428
 429     disabled: true
 430
 431     node: ericsson-build
 432
 433     parameters:
 434         - project-parameter:
 435             project: '{project}'
 436         - fuel-parameter:
 437             installer: '{installer}'
 438             gs-pathname: ''
 439
 440     scm:
 441         - git-scm:
 442             credentials-id: '{ssh-credentials}'
 443             refspec: ''
 444             branch: 'master'
 445
 446     logrotate:
 447         daysToKeep: 30
 448         numToKeep: 10
 449         artifactDaysToKeep: -1
 450         artifactNumToKeep: -1
 451
 452     builders:
 453         - 'fuel-deploy-virtual'
 454
 455 ########################
 456 # parameter macros
 457 ########################
 458 - parameter:
 459     name: fuel-parameter
 460     parameters:
 461         - string:
 462             name: INSTALLER
 463             default: '{installer}'
 464             description: "Installer to use."
 465         - string:
 466             name: BUILD_DIRECTORY
 467             default: $WORKSPACE/build_output
 468             description: "Directory where the build artifact will be located upon the completion of the build."
 469         - string:
 470             name: CACHE_DIRECTORY
 471             default: $HOME/opnfv/cache/genesis-$INSTALLER
 472             description: "Directory where the cache to be used during the build is located."
 473         - string:
 474             name: GIT_BASE
 475             default: https://gerrit.opnfv.org/gerrit/$PROJECT
 476             description: "Used for overriding the GIT URL coming from Global Jenkins configuration in case if the stuff is done on none-LF HW."
 477         - string:
 478             name: GS_URL
 479             default: artifacts.opnfv.org/$PROJECT/$INSTALLER{gs-pathname}
 480             description: "URL to Google Storage."
 481
 482 ########################
 483 # builder macros
 484 ########################
 485 - builder:
 486     name: 'fuel-build'
 487     builders:
 488         - shell: |
 489             #!/bin/bash
 490             set -o errexit
 491             set -o nounset
 492             set -o pipefail
 493
 494             # log info to console
 495             echo "Starting the build of $INSTALLER. This could take some time..."
 496             echo "--------------------------------------------------------"
 497             echo
 498
 499             # create the cache directory if it doesn't exist
 500             [[ -d $CACHE_DIRECTORY ]] || mkdir -p $CACHE_DIRECTORY
 501
 502             # set OPNFV_ARTIFACT_VERSION
 503             export OPNFV_ARTIFACT_VERSION=$(date -u +"%Y-%m-%d_%H-%M-%S")
 504
 505             # start the build
 506             cd $WORKSPACE/$INSTALLER/ci
 507             ./build.sh -v $OPNFV_ARTIFACT_VERSION -c file://$CACHE_DIRECTORY $BUILD_DIRECTORY
 508
 509             # list the build artifacts
 510             ls -al $BUILD_DIRECTORY
 511
 512             # save information regarding artifact into file
 513             (
 514                 echo "OPNFV_ARTIFACT_VERSION=$OPNFV_ARTIFACT_VERSION"
 515                 echo "OPNFV_GIT_URL=$(git config --get remote.origin.url)"
 516                 echo "OPNFV_GIT_SHA1=$(git rev-parse HEAD)"
 517                 echo "OPNFV_ARTIFACT_URL=$GS_URL/opnfv-$OPNFV_ARTIFACT_VERSION.iso"
 518                 echo "OPNFV_ARTIFACT_MD5SUM=$(md5sum $BUILD_DIRECTORY/opnfv-$OPNFV_ARTIFACT_VERSION.iso | cut -d' ' -f1)"
 519                 echo "OPNFV_BUILD_URL=$BUILD_URL"
 520             ) > $WORKSPACE/opnfv.properties
 521
 522             echo
 523             echo "--------------------------------------------------------"
 524             echo "Done!"
 525
 526 - builder:
 527     name: 'fuel-deploy'
 528     builders:
 529         - shell: |
 530             #!/bin/bash
 531             set -o errexit
 532             set -o nounset
 533             set -o pipefail
 534
 535             # source the file so we get OPNFV vars
 536             source latest.properties
 537
 538             # echo the info about artifact that is used during the deployment
 539             echo "Using $(echo $OPNFV_ARTIFACT_URL | cut -d'/' -f4) for deployment"
 540
 541             # create TMPDIR if it doesn't exist
 542             export TMPDIR=$HOME/tmpdir
 543             [[ -d $TMPDIR ]] || mkdir -p $TMPDIR
 544
 545             # change permissions down to TMPDIR
 546             chmod a+x $HOME
 547             chmod a+x $TMPDIR
 548
 549             # set CONFDIR, BRIDGE
 550             export CONFDIR=$WORKSPACE/fuel/deploy/templates/hardware_environment/old_conf/linux_foundation_lab/pod2/ha
 551             export BRIDGE=pxebr
 552
 553             # cleanup first
 554             sudo $WORKSPACE/common/ci/clean.sh -base_config $WORKSPACE/foreman/ci/inventory/lf_pod2_ksgen_settings.yml
 555
 556             # prepare for Fuel Deployment
 557             sudo $WORKSPACE/common/ci/setup.sh
 558
 559             # log info to console
 560             echo "Starting the deployment using $INSTALLER. This could take some time..."
 561             echo "--------------------------------------------------------"
 562             echo
 563
 564             # start the deployment
 565             echo "Issuing command"
 566             echo "sudo $WORKSPACE/fuel/ci/deploy.sh -iso $WORKSPACE/opnfv.iso -dea $CONFDIR/dea.yaml -dha $CONFDIR/dha.yaml -s $TMPDIR -b $BRIDGE"
 567             sudo $WORKSPACE/fuel/ci/deploy.sh -iso $WORKSPACE/opnfv.iso -dea $CONFDIR/dea.yaml -dha $CONFDIR/dha.yaml -s $TMPDIR -b $BRIDGE
 568
 569             echo
 570             echo "--------------------------------------------------------"
 571             echo "Done!"
 572
 573 - builder:
 574     name: 'fuel-deploy-virtual'
 575     builders:
 576         - shell: |
 577             #!/bin/bash
 578             set -o errexit
 579             set -o nounset
 580             set -o pipefail
 581
 582             # log info to console
 583             echo "Starting the deployment on virtual environment using $INSTALLER. This could take some time..."
 584             echo "--------------------------------------------------------"
 585             echo
 586
 587             ssh -o BatchMode=yes -o TCPKeepAlive=yes cideploy@10.118.34.205 ./cideploy.sh
 588
 589             echo
 590             echo "--------------------------------------------------------"
 591             echo "Done!"
 592
 593 - builder:
 594     name: 'fuel-upload-artifact'
 595     builders:
 596         - shell: |
 597             #!/bin/bash
 598             set -o errexit
 599             set -o nounset
 600             set -o pipefail
 601
 602             # log info to console
 603             echo "Uploading the $INSTALLER artifact. This could take some time..."
 604             echo "--------------------------------------------------------"
 605             echo
 606
 607             # source the opnfv.properties to get ARTIFACT_VERSION
 608             source $WORKSPACE/opnfv.properties
 609
 610             # upload artifact and additional files to google storage
 611             gsutil cp $BUILD_DIRECTORY/opnfv-$OPNFV_ARTIFACT_VERSION.iso gs://$GS_URL/opnfv-$OPNFV_ARTIFACT_VERSION.iso > gsutil.iso.log 2>&1
 612             gsutil cp $WORKSPACE/opnfv.properties gs://$GS_URL/opnfv-$OPNFV_ARTIFACT_VERSION.properties > gsutil.properties.log 2>&1
 613             gsutil cp $WORKSPACE/opnfv.properties gs://$GS_URL/latest.properties > gsutil.latest.log 2>&1
 614
 615             echo
 616             echo "--------------------------------------------------------"
 617             echo "Done!"
 618             echo "Artifact is available as http://$GS_URL/opnfv-$OPNFV_ARTIFACT_VERSION.iso"
 619
 620 - builder:
 621     name: 'fuel-download-artifact'
 622     builders:
 623         - shell: |
 624             #!/bin/bash
 625             set -o errexit
 626             set -o nounset
 627             set -o pipefail
 628
 629             # get the latest.properties file in order to get info regarding latest artifact
 630             curl -s -o $WORKSPACE/latest.properties http://$GS_URL/latest.properties
 631
 632             # check if we got the file
 633             [[ -f latest.properties ]] || exit 1
 634
 635             # source the file so we get OPNFV vars
 636             source latest.properties
 637
 638             # log info to console
 639             echo "Downloading the $INSTALLER artifact using URL http://$OPNFV_ARTIFACT_URL"
 640             echo "This could take some time..."
 641             echo "--------------------------------------------------------"
 642             echo
 643
 644             # download the file
 645             curl -s -o $WORKSPACE/opnfv.iso http://$OPNFV_ARTIFACT_URL > gsutil.iso.log 2>&1
 646
 647             # list the file
 648             ls -al $WORKSPACE/opnfv.iso
 649
 650             echo
 651             echo "--------------------------------------------------------"
 652             echo "Done!"
 653
 654 - builder:
 655     name: 'fuel-workspace-cleanup'
 656     builders:
 657         - shell: |
 658             #!/bin/bash
 659             set -o errexit
 660             set -o nounset
 661             set -o pipefail
 662
 663             # delete everything that is in $WORKSPACE
 664             /bin/rm -rf $WORKSPACE
 665
 666 - builder:
 667     name: 'fuel-lab-reconfig'
 668     builders:
 669         - shell: |
 670             #!/bin/bash
 671             set -o errexit
 672             set -o nounset
 673             set -o pipefail
 674
 675             # check to see if ucs login info file exists
 676             if [ -e ~/.ssh/ucs_creds ];then
 677               source ~/.ssh/ucs_creds
 678             else
 679               echo "Unable to find UCS credentials for LF lab reconfiguration...Exiting"
 680               exit 1
 681             fi
 682
 683             # clone releng
 684             echo "Cloning releng repo..."
 685             if ! GIT_SSL_NO_VERIFY=true git clone https://gerrit.opnfv.org/gerrit/releng; then
 686               echo "Unable to clone releng repo...Exiting"
 687               exit 1
 688             fi
 689
 690             # log info to console
 691             echo "Starting the lab reconfiguration for $INSTALLER..."
 692             echo "--------------------------------------------------------"
 693             echo
 694
 695             # create venv
 696             $WORKSPACE/releng/utils/lab-reconfiguration/create_venv.sh
 697
 698             # disable nounset because 'activate' script contains unbound variable(s)
 699             set +o nounset
 700             # enter venv
 701             source $WORKSPACE/releng/utils/lab-reconfiguration/venv/bin/activate
 702             # set nounset back again
 703             set -o nounset
 704
 705             # verify we are in venv
 706             if [[ ! $(which python | grep venv) ]]; then
 707               echo "Unable to activate venv...Exiting"
 708               exit 1
 709             fi
 710
 711             python $WORKSPACE/releng/utils/lab-reconfiguration/reconfigUcsNet.py -i $ucs_host -u $ucs_user -p $ucs_password -f $WORKSPACE/releng/utils/lab-reconfiguration/fuel.yaml
 712
 713             # while undergoing reboot
 714             sleep 30
 715
 716             # check to see if slave is back up
 717             ping_counter=0
 718             ping_flag=0
 719             while [ "$ping_counter" -lt 20 ]; do
 720               if [[ $(ping -c 5 172.30.10.72) ]]; then
 721                 ping_flag=1
 722                 break
 723               fi
 724               ((ping_counter++))
 725               sleep 10
 726             done
 727
 728             if [ "$ping_flag" -eq 1 ]; then
 729               echo "Slave is pingable, now wait 180 seconds for services to start"
 730               sleep 180
 731             else
 732               echo "Slave did not come back up after reboot: please check opnfv-jump-2"
 733               exit 1
 734             fi
 735
 736             set +o nounset
 737             deactivate
 738
 739             echo
 740             echo "--------------------------------------------------------"
 741             echo "Done!"
 742
 743 ########################
 744 # trigger macros
 745 ########################
 746 - trigger:
 747     name: 'fuel-master'
 748     triggers:
 749         - timed: '0 3 * * *'
 750
 751 - trigger:
 752     name: 'fuel-stable-arno'
 753     triggers:
 754         - timed: '0 7 * * *'