jjb/global/releng-macros.yml

   1 ---
   2 # Releng macros
   3 #
   4 # NOTE: make sure macros are listed in execution ordered.
   5 #
   6 # 1. parameters/properties
   7 # 2. scm
   8 # 3. triggers
   9 # 4. wrappers
  10 # 5. prebuilders (maven only, configured like Builders)
  11 # 6. builders (maven, freestyle, matrix, etc..)
  12 # 7. postbuilders (maven only, configured like Builders)
  13 # 8. publishers/reporters/notifications
  14
  15 - parameter:
  16     name: project-parameter
  17     parameters:
  18       - string:
  19           name: PROJECT
  20           default: '{project}'
  21           description: "JJB configured PROJECT parameter to identify an opnfv Gerrit project"
  22       - string:
  23           name: GS_BASE
  24           default: artifacts.opnfv.org/$PROJECT
  25           description: "URL to Google Storage."
  26       - string:
  27           name: GS_BASE_PROXY
  28           default: build.opnfv.org/artifacts.opnfv.org/$PROJECT
  29           description: "URL to Google Storage proxy"
  30       - string:
  31           name: BRANCH
  32           default: '{branch}'
  33           description: "JJB configured BRANCH parameter (e.g. master, stable/danube)"
  34       - string:
  35           name: GERRIT_BRANCH
  36           default: '{branch}'
  37           description: "JJB configured GERRIT_BRANCH parameter (deprecated)"
  38
  39 - property:
  40     name: logrotate-default
  41     properties:
  42       - build-discarder:
  43           days-to-keep: 60
  44           num-to-keep: 200
  45           artifact-days-to-keep: 60
  46           artifact-num-to-keep: 200
  47
  48 - scm:
  49     name: git-scm
  50     scm:
  51       - git: &git-scm-defaults
  52           credentials-id: '$SSH_CREDENTIAL_ID'
  53           url: '$GIT_BASE'
  54           branches:
  55             - 'origin/$BRANCH'
  56           timeout: 15
  57
  58 - scm:
  59     name: git-scm-gerrit
  60     scm:
  61       - git:
  62           choosing-strategy: 'gerrit'
  63           refspec: '$GERRIT_REFSPEC'
  64           <<: *git-scm-defaults
  65 - scm:
  66     name: git-scm-with-submodules
  67     scm:
  68       - git:
  69           credentials-id: '$SSH_CREDENTIAL_ID'
  70           url: '$GIT_BASE'
  71           refspec: ''
  72           branches:
  73             - 'refs/heads/{branch}'
  74           skip-tag: true
  75           wipe-workspace: true
  76           submodule:
  77             recursive: true
  78             timeout: 20
  79 - trigger:
  80     name: 'daily-trigger-disabled'
  81     triggers:
  82       - timed: ''
  83
  84 - trigger:
  85     name: 'weekly-trigger-disabled'
  86     triggers:
  87       - timed: ''
  88
  89 - trigger:
  90     name: gerrit-trigger-patchset-created
  91     triggers:
  92       - gerrit:
  93           server-name: 'gerrit.opnfv.org'
  94           trigger-on:
  95             - patchset-created-event:
  96                 exclude-drafts: 'false'
  97                 exclude-trivial-rebase: 'false'
  98                 exclude-no-code-change: 'false'
  99             - draft-published-event
 100             - comment-added-contains-event:
 101                 comment-contains-value: 'recheck'
 102             - comment-added-contains-event:
 103                 comment-contains-value: 'reverify'
 104           projects:
 105             - project-compare-type: 'ANT'
 106               project-pattern: '{project}'
 107               branches:
 108                 - branch-compare-type: 'ANT'
 109                   branch-pattern: '**/{branch}'
 110               file-paths:
 111                 - compare-type: 'ANT'
 112                   pattern: '{files}'
 113           skip-vote:
 114             successful: false
 115             failed: false
 116             unstable: false
 117             notbuilt: false
 118
 119 - trigger:
 120     name: gerrit-trigger-change-merged
 121     triggers:
 122       - gerrit:
 123           server-name: 'gerrit.opnfv.org'
 124           trigger-on:
 125             - change-merged-event
 126             - comment-added-contains-event:
 127                 comment-contains-value: 'remerge'
 128           projects:
 129             - project-compare-type: 'ANT'
 130               project-pattern: '{project}'
 131               branches:
 132                 - branch-compare-type: 'ANT'
 133                   branch-pattern: '**/{branch}'
 134               file-paths:
 135                 - compare-type: 'ANT'
 136                   pattern: '{files}'
 137
 138 - trigger:
 139     name: 'experimental'
 140     triggers:
 141       - gerrit:
 142           server-name: 'gerrit.opnfv.org'
 143           trigger-on:
 144             - comment-added-contains-event:
 145                 comment-contains-value: 'check-experimental'
 146           projects:
 147             - project-compare-type: 'ANT'
 148               project-pattern: '{project}'
 149               branches:
 150                 - branch-compare-type: 'ANT'
 151                   branch-pattern: '**/{branch}'
 152               file-paths:
 153                 - compare-type: 'ANT'
 154                   pattern: '{files}'
 155           skip-vote:
 156             successful: true
 157             failed: true
 158             unstable: true
 159             notbuilt: true
 160
 161 - wrapper:
 162     name: ssh-agent-wrapper
 163     wrappers:
 164       - ssh-agent-credentials:
 165           users:
 166             - 'd42411ac011ad6f3dd2e1fa34eaa5d87f910eb2e'
 167
 168 - wrapper:
 169     name: build-timeout
 170     wrappers:
 171       - timeout:
 172           timeout: '{timeout}'
 173           timeout-var: 'BUILD_TIMEOUT'
 174           fail: true
 175
 176 - wrapper:
 177     name: fix-workspace-permissions
 178     wrappers:
 179       - pre-scm-buildstep:
 180           - shell: |
 181              #!/bin/bash
 182              sudo chown -R $USER:$USER $WORKSPACE || exit 1
 183
 184 - builder:
 185     name: build-html-and-pdf-docs-output
 186     builders:
 187       - shell: |
 188           #!/bin/bash
 189           set -o errexit
 190           set -o xtrace
 191           export PATH=$PATH:/usr/local/bin/
 192           git clone ssh://gerrit.opnfv.org:29418/opnfvdocs docs_build/_opnfvdocs
 193           GERRIT_COMMENT=gerrit_comment.txt ./docs_build/_opnfvdocs/scripts/docs-build.sh
 194
 195 - builder:
 196     name: upload-under-review-docs-to-opnfv-artifacts
 197     builders:
 198       - shell: |
 199           #!/bin/bash
 200           set -o errexit
 201           set -o pipefail
 202           set -o xtrace
 203           export PATH=$PATH:/usr/local/bin/
 204
 205           [[ $GERRIT_CHANGE_NUMBER =~ .+ ]]
 206           [[ -d docs_output ]] || exit 0
 207
 208           echo
 209           echo "###########################"
 210           echo "UPLOADING DOCS UNDER REVIEW"
 211           echo "###########################"
 212           echo
 213
 214           gs_base="artifacts.opnfv.org/$PROJECT/review"
 215           gs_path="$gs_base/$GERRIT_CHANGE_NUMBER"
 216           local_path="upload/$GERRIT_CHANGE_NUMBER"
 217
 218           mkdir -p upload
 219           mv docs_output "$local_path"
 220           gsutil -m cp -r "$local_path" "gs://$gs_base"
 221
 222           gsutil -m setmeta \
 223               -h "Content-Type:text/html" \
 224               -h "Cache-Control:private, max-age=0, no-transform" \
 225               "gs://$gs_path"/**.html > /dev/null 2>&1
 226
 227           echo "Document link(s):" >> gerrit_comment.txt
 228           find "$local_path" | grep -e 'index.html$' -e 'pdf$' | \
 229               sed -e "s|^$local_path|    http://$gs_path|" >> gerrit_comment.txt
 230
 231 - builder:
 232     name: upload-generated-docs-to-opnfv-artifacts
 233     builders:
 234       - shell: |
 235           #!/bin/bash
 236           set -o errexit
 237           set -o pipefail
 238           set -o xtrace
 239           export PATH=$PATH:/usr/local/bin/
 240
 241           [[ -d docs_output ]] || exit 0
 242
 243           echo
 244           echo "########################"
 245           echo "UPLOADING GENERATED DOCS"
 246           echo "########################"
 247           echo
 248
 249           echo "gs_path="$GS_URL/docs""
 250           echo "local_path="upload/docs""
 251
 252           gs_path="$GS_URL/docs"
 253           local_path="upload/docs"
 254
 255           mkdir -p upload
 256           mv docs_output "$local_path"
 257           ls "$local_path"
 258
 259           echo "gsutil -m cp -r "$local_path"/* "gs://$gs_path""
 260           gsutil -m cp -r "$local_path"/* "gs://$gs_path"
 261
 262           gsutil -m setmeta \
 263               -h "Content-Type:text/html" \
 264               -h "Cache-Control:private, max-age=0, no-transform" \
 265               "gs://$gs_path"/**.html > /dev/null 2>&1
 266
 267           echo "Document link(s):" >> gerrit_comment.txt
 268           find "$local_path" | grep -e 'index.html$' -e 'pdf$' | \
 269               sed -e "s|^$local_path|    http://$gs_path|" >> gerrit_comment.txt
 270
 271 # To take advantage of this macro, have your build write
 272 # out the file 'gerrit_comment.txt' with information to post
 273 # back to gerrit and include this macro in the list of builders.
 274 - builder:
 275     name: report-build-result-to-gerrit
 276     builders:
 277       - shell: |
 278           #!/bin/bash
 279           set -o errexit
 280           set -o pipefail
 281           set -o xtrace
 282           export PATH=$PATH:/usr/local/bin/
 283           if [[ -e gerrit_comment.txt ]] ; then
 284               echo
 285               echo "posting review comment to gerrit..."
 286               echo
 287               cat gerrit_comment.txt
 288               echo
 289               ssh -p 29418 gerrit.opnfv.org \
 290                   "gerrit review -p $GERRIT_PROJECT \
 291                    -m '$(cat gerrit_comment.txt)' \
 292                    $GERRIT_PATCHSET_REVISION \
 293                    --notify NONE"
 294           fi
 295
 296 - builder:
 297     name: remove-old-docs-from-opnfv-artifacts
 298     builders:
 299       - shell: |
 300           #!/bin/bash
 301           set -o errexit
 302           set -o pipefail
 303           set -o xtrace
 304           export PATH=$PATH:/usr/local/bin/
 305
 306           [[ $GERRIT_CHANGE_NUMBER =~ .+ ]]
 307
 308           gs_path="artifacts.opnfv.org/$PROJECT/review/$GERRIT_CHANGE_NUMBER"
 309
 310           if gsutil ls "gs://$gs_path" > /dev/null 2>&1 ; then
 311               echo
 312               echo "Deleting Out-of-dated Documents..."
 313               gsutil -m rm -r "gs://$gs_path"
 314           fi
 315           gs_path="artifacts.opnfv.org/review/$GERRIT_CHANGE_NUMBER"
 316
 317           if gsutil ls "gs://$gs_path" > /dev/null 2>&1 ; then
 318               echo
 319               echo "Deleting Out-of-dated Documents..."
 320               gsutil -m rm -r "gs://$gs_path"
 321           fi
 322
 323 - builder:
 324     name: build-and-upload-artifacts-json-api
 325     builders:
 326       - shell: |
 327           #!/bin/bash
 328           set -o errexit
 329           set -o pipefail
 330           export PATH=$PATH:/usr/local/bin/
 331
 332           virtualenv -p python2.7 $WORKSPACE/releng_artifacts
 333           source $WORKSPACE/releng_artifacts/bin/activate
 334
 335           # install python packages
 336           pip install google-api-python-client
 337
 338           # generate and upload index file
 339           echo "Generating Artifacts API ..."
 340           python $WORKSPACE/utils/opnfv-artifacts.py > index.json
 341           gsutil cp index.json gs://artifacts.opnfv.org/index.json
 342
 343           deactivate
 344
 345 - builder:
 346     name: lint-python-code
 347     builders:
 348       - shell: |
 349           #!/bin/bash
 350           set -o errexit
 351           set -o pipefail
 352           set -o xtrace
 353           export PATH=$PATH:/usr/local/bin/
 354
 355           virtualenv -p python2.7 $WORKSPACE/releng_flake8
 356           source $WORKSPACE/releng_flake8/bin/activate
 357
 358           # install python packages
 359           pip install "flake8==2.6.2"
 360
 361           # generate and upload lint log
 362           echo "Running flake8 code on $PROJECT ..."
 363
 364           # Get number of flake8 violations. If none, this will be an
 365           # empty string: ""
 366           FLAKE_COUNT="$(find . \
 367               -path './releng_flake8' -prune -o \
 368               -path './.tox' -prune -o \
 369               -type f -name "*.py" -print | \
 370               xargs flake8 --exit-zero -qq --count 2>&1)"
 371
 372           # Ensure we start with a clean environment
 373           rm -f lint.log
 374
 375           if [ ! -z $FLAKE_COUNT ]; then
 376             echo "Flake8 Violations: $FLAKE_COUNT" > lint.log
 377             find . \
 378                 -path './releng_flake8' -prune -o \
 379                 -path './.tox' -prune -o \
 380                 -type f -name "*.py" -print | \
 381                 xargs flake8 --exit-zero --first >> violation.log
 382             SHOWN=$(wc -l violation.log | cut -d' ' -f1)
 383             echo -e "First $SHOWN shown\n---" >> lint.log
 384             cat violation.log >> lint.log
 385             sed -r -i '4,$s/^/ /g' lint.log
 386             rm violation.log
 387           fi
 388
 389           deactivate
 390
 391 - builder:
 392     name: report-lint-result-to-gerrit
 393     builders:
 394       - shell: |
 395           #!/bin/bash
 396           set -o errexit
 397           set -o pipefail
 398           set -o xtrace
 399           export PATH=$PATH:/usr/local/bin/
 400
 401           # If no violations were found, no lint log will exist.
 402           if [[ -e lint.log ]] ; then
 403               echo -e "\nposting linting report to gerrit...\n"
 404
 405               cat lint.log
 406               echo
 407
 408               ssh -p 29418 gerrit.opnfv.org \
 409                   "gerrit review -p $GERRIT_PROJECT \
 410                    -m \"$(cat lint.log)\" \
 411                    $GERRIT_PATCHSET_REVISION \
 412                    --notify NONE"
 413
 414               exit 1
 415           fi
 416
 417 - builder:
 418     name: upload-review-docs
 419     builders:
 420       - build-html-and-pdf-docs-output
 421       - upload-under-review-docs-to-opnfv-artifacts
 422       - report-build-result-to-gerrit
 423
 424 - builder:
 425     name: upload-merged-docs
 426     builders:
 427       - build-html-and-pdf-docs-output
 428       - upload-generated-docs-to-opnfv-artifacts
 429       - report-build-result-to-gerrit
 430       - remove-old-docs-from-opnfv-artifacts
 431
 432 - builder:
 433     name: check-bash-syntax
 434     builders:
 435       - shell: "find . -name '*.sh' | xargs bash -n"
 436
 437 - builder:
 438     name: lint-yaml-code
 439     builders:
 440       - shell: |
 441           #!/bin/bash
 442           set -o errexit
 443           set -o pipefail
 444           set -o xtrace
 445           export PATH=$PATH:/usr/local/bin/
 446
 447           # install python packages
 448           pip install "yamllint==1.6.0"
 449
 450           # generate and upload lint log
 451           echo "Running yaml code on $PROJECT ..."
 452
 453           # Ensure we start with a clean environment
 454           rm -f yaml-violation.log lint.log
 455
 456           # Get number of yaml violations. If none, this will be an
 457           # empty string: ""
 458           find . \
 459               -type f -name "*.yml" -print \
 460               -o -name "*.yaml" -print | \
 461               xargs yamllint > yaml-violation.log || true
 462
 463           if [ -s "yaml-violation.log" ]; then
 464             SHOWN=$(cat yaml-violation.log| grep -v "^$" |wc -l)
 465             echo -e "First $SHOWN shown\n---" > lint.log
 466             cat yaml-violation.log >> lint.log
 467             sed -r -i '4,$s/^/ /g' lint.log
 468           fi
 469
 470 - builder:
 471     name: clean-workspace
 472     builders:
 473       - shell: |
 474           #!/bin/bash
 475           set -o errexit
 476           set -o nounset
 477           set -o pipefail
 478           sudo /bin/rm -rf "$WORKSPACE"
 479
 480 - builder:
 481     name: clean-workspace-log
 482     builders:
 483       - shell: |
 484           find $WORKSPACE -type f -name '*.log' | xargs rm -f
 485
 486 - publisher:
 487     name: archive-artifacts
 488     publishers:
 489       - archive:
 490           artifacts: '{artifacts}'
 491           allow-empty: true
 492           fingerprint: true
 493           latest-only: true
 494
 495 - publisher:
 496     name: publish-coverage
 497     publishers:
 498       - cobertura:
 499           report-file: "coverage.xml"
 500           only-stable: "true"
 501           health-auto-update: "false"
 502           stability-auto-update: "false"
 503           zoom-coverage-chart: "true"
 504           targets:
 505             - files:
 506                 healthy: 10
 507                 unhealthy: 20
 508                 failing: 30
 509             - method:
 510                 healthy: 50
 511                 unhealthy: 40
 512                 failing: 30
 513
 514 # The majority of the email-ext plugin options are set to the default
 515 # for this macro so they can be managed through Jenkins' global
 516 # settings.
 517 - publisher:
 518     name: email-jenkins-admins-on-failure
 519     publishers:
 520       - email-ext:
 521           content-type: text
 522           attach-build-log: true
 523           compress-log: true
 524           always: false
 525           failure: true
 526           send-to:
 527             - recipients