jjb/global/releng-macros.yml

   1 # Releng macros
   2 #
   3 # NOTE: make sure macros are listed in execution ordered.
   4 #
   5 # 1. parameters/properties
   6 # 2. scm
   7 # 3. triggers
   8 # 4. wrappers
   9 # 5. prebuilders (maven only, configured like Builders)
  10 # 6. builders (maven, freestyle, matrix, etc..)
  11 # 7. postbuilders (maven only, configured like Builders)
  12 # 8. publishers/reporters/notifications
  13
  14 - parameter:
  15     name: project-parameter
  16     parameters:
  17         - string:
  18             name: PROJECT
  19             default: '{project}'
  20             description: "JJB configured PROJECT parameter to identify an opnfv Gerrit project"
  21         - string:
  22             name: GS_BASE
  23             default: artifacts.opnfv.org/$PROJECT
  24             description: "URL to Google Storage."
  25         - string:
  26             name: GS_BASE_PROXY
  27             default: build.opnfv.org/artifacts.opnfv.org/$PROJECT
  28             description: "URL to Google Storage proxy"
  29
  30 - parameter:
  31     name: gerrit-parameter
  32     parameters:
  33         - string:
  34             name: GERRIT_BRANCH
  35             default: '{branch}'
  36             description: "JJB configured GERRIT_BRANCH parameter"
  37
  38 - scm:
  39     name: git-scm
  40     scm:
  41         - git: &git-scm-defaults
  42             credentials-id: '$SSH_CREDENTIAL_ID'
  43             url: '$GIT_BASE'
  44             branches:
  45                 - 'origin/$GERRIT_BRANCH'
  46             timeout: 15
  47
  48 - scm:
  49     name: git-scm-gerrit
  50     scm:
  51         - git:
  52             choosing-strategy: 'gerrit'
  53             refspec: '$GERRIT_REFSPEC'
  54             <<: *git-scm-defaults
  55
  56 - trigger:
  57     name: 'daily-trigger-disabled'
  58     triggers:
  59         - timed: ''
  60
  61 # NOTE: unused macro, but we may use this for some jobs.
  62 - trigger:
  63     name: gerrit-trigger-patch-submitted
  64     triggers:
  65         - gerrit:
  66             server-name: 'gerrit.opnfv.org'
  67             trigger-on:
  68                 - patchset-created-event:
  69                     exclude-drafts: 'false'
  70                     exclude-trivial-rebase: 'false'
  71                     exclude-no-code-change: 'false'
  72                 - draft-published-event
  73                 - comment-added-contains-event:
  74                     comment-contains-value: 'recheck'
  75             projects:
  76               - project-compare-type: 'ANT'
  77                 project-pattern: '{name}'
  78                 branches:
  79                   - branch-compare-type: 'ANT'
  80                     branch-pattern: '**/{branch}'
  81
  82 # NOTE: unused macro, but we may use this for some jobs.
  83 - trigger:
  84     name: gerrit-trigger-patch-merged
  85     triggers:
  86         - gerrit:
  87             server-name: 'gerrit.opnfv.org'
  88             trigger-on:
  89                 - change-merged-event
  90                 - comment-added-contains-event:
  91                     comment-contains-value: 'remerge'
  92             projects:
  93               - project-compare-type: 'ANT'
  94                 project-pattern: '{name}'
  95                 branches:
  96                   - branch-compare-type: 'ANT'
  97                     branch-pattern: '**/{branch}'
  98
  99 - wrapper:
 100     name: ssh-agent-wrapper
 101     wrappers:
 102         - ssh-agent-credentials:
 103             users:
 104                 - 'd42411ac011ad6f3dd2e1fa34eaa5d87f910eb2e'
 105
 106 - builder:
 107     name: build-html-and-pdf-docs-output
 108     builders:
 109         - shell: |
 110             #!/bin/bash
 111             set -o errexit
 112             set -o xtrace
 113             export PATH=$PATH:/usr/local/bin/
 114             git clone ssh://gerrit.opnfv.org:29418/opnfvdocs docs_build/_opnfvdocs
 115             GERRIT_COMMENT=gerrit_comment.txt ./docs_build/_opnfvdocs/scripts/docs-build.sh
 116
 117 - builder:
 118     name: upload-under-review-docs-to-opnfv-artifacts
 119     builders:
 120         - shell: |
 121             #!/bin/bash
 122             set -o errexit
 123             set -o pipefail
 124             set -o xtrace
 125             export PATH=$PATH:/usr/local/bin/
 126
 127             [[ $GERRIT_CHANGE_NUMBER =~ .+ ]]
 128             [[ -d docs_output ]] || exit 0
 129
 130             echo
 131             echo "###########################"
 132             echo "UPLOADING DOCS UNDER REVIEW"
 133             echo "###########################"
 134             echo
 135
 136             gs_base="artifacts.opnfv.org/$PROJECT/review"
 137             gs_path="$gs_base/$GERRIT_CHANGE_NUMBER"
 138             local_path="upload/$GERRIT_CHANGE_NUMBER"
 139
 140             mkdir -p upload
 141             mv docs_output "$local_path"
 142             gsutil -m cp -r "$local_path" "gs://$gs_base"
 143
 144             gsutil -m setmeta \
 145                 -h "Content-Type:text/html" \
 146                 -h "Cache-Control:private, max-age=0, no-transform" \
 147                 "gs://$gs_path"/**.html > /dev/null 2>&1
 148
 149             echo "Document link(s):" >> gerrit_comment.txt
 150             find "$local_path" | grep -e 'index.html$' -e 'pdf$' | \
 151                 sed -e "s|^$local_path|    http://$gs_path|" >> gerrit_comment.txt
 152
 153 - builder:
 154     name: upload-generated-docs-to-opnfv-artifacts
 155     builders:
 156         - shell: |
 157             #!/bin/bash
 158             set -o errexit
 159             set -o pipefail
 160             set -o xtrace
 161             export PATH=$PATH:/usr/local/bin/
 162
 163             [[ -d docs_output ]] || exit 0
 164
 165             echo
 166             echo "########################"
 167             echo "UPLOADING GENERATED DOCS"
 168             echo "########################"
 169             echo
 170
 171             echo "gs_path="$GS_URL/docs""
 172             echo "local_path="upload/docs""
 173
 174             gs_path="$GS_URL/docs"
 175             local_path="upload/docs"
 176
 177             mkdir -p upload
 178             mv docs_output "$local_path"
 179             ls "$local_path"
 180
 181             echo "gsutil -m cp -r "$local_path"/* "gs://$gs_path""
 182             gsutil -m cp -r "$local_path"/* "gs://$gs_path"
 183
 184             gsutil -m setmeta \
 185                 -h "Content-Type:text/html" \
 186                 -h "Cache-Control:private, max-age=0, no-transform" \
 187                 "gs://$gs_path"/**.html > /dev/null 2>&1
 188
 189             echo "Document link(s):" >> gerrit_comment.txt
 190             find "$local_path" | grep -e 'index.html$' -e 'pdf$' | \
 191                 sed -e "s|^$local_path|    http://$gs_path|" >> gerrit_comment.txt
 192
 193 - builder:
 194     name: report-docs-build-result-to-gerrit
 195     builders:
 196         - shell: |
 197             #!/bin/bash
 198             set -o errexit
 199             set -o pipefail
 200             set -o xtrace
 201             export PATH=$PATH:/usr/local/bin/
 202             if [[ -e gerrit_comment.txt ]] ; then
 203                 echo
 204                 echo "posting review comment to gerrit..."
 205                 echo
 206                 cat gerrit_comment.txt
 207                 echo
 208                 ssh -p 29418 gerrit.opnfv.org \
 209                     "gerrit review -p $GERRIT_PROJECT \
 210                      -m '$(cat gerrit_comment.txt)' \
 211                      $GERRIT_PATCHSET_REVISION \
 212                      --notify NONE"
 213             fi
 214
 215 - builder:
 216     name: remove-old-docs-from-opnfv-artifacts
 217     builders:
 218         - shell: |
 219             #!/bin/bash
 220             set -o errexit
 221             set -o pipefail
 222             set -o xtrace
 223             export PATH=$PATH:/usr/local/bin/
 224
 225             [[ $GERRIT_CHANGE_NUMBER =~ .+ ]]
 226
 227             gs_path="artifacts.opnfv.org/$PROJECT/review/$GERRIT_CHANGE_NUMBER"
 228
 229             if gsutil ls "gs://$gs_path" > /dev/null 2>&1 ; then
 230                 echo
 231                 echo "Deleting Out-of-dated Documents..."
 232                 gsutil -m rm -r "gs://$gs_path"
 233             fi
 234             gs_path="artifacts.opnfv.org/review/$GERRIT_CHANGE_NUMBER"
 235
 236             if gsutil ls "gs://$gs_path" > /dev/null 2>&1 ; then
 237                 echo
 238                 echo "Deleting Out-of-dated Documents..."
 239                 gsutil -m rm -r "gs://$gs_path"
 240             fi
 241
 242 - builder:
 243     name: build-and-upload-artifacts-json-api
 244     builders:
 245         - shell: |
 246             #!/bin/bash
 247             set -o errexit
 248             set -o pipefail
 249             export PATH=$PATH:/usr/local/bin/
 250
 251             virtualenv -p python2.7 $WORKSPACE/releng_artifacts
 252             source $WORKSPACE/releng_artifacts/bin/activate
 253
 254             # install python packages
 255             pip install google-api-python-client
 256
 257             # generate and upload index file
 258             echo "Generating Artifacts API ..."
 259             python $WORKSPACE/utils/opnfv-artifacts.py > index.json
 260             gsutil cp index.json gs://artifacts.opnfv.org/index.json
 261
 262             deactivate
 263
 264 - builder:
 265     name: lint-python-code
 266     builders:
 267         - shell: |
 268             #!/bin/bash
 269             set -o errexit
 270             set -o pipefail
 271             set -o xtrace
 272             export PATH=$PATH:/usr/local/bin/
 273
 274             virtualenv -p python2.7 $WORKSPACE/releng_flake8
 275             source $WORKSPACE/releng_flake8/bin/activate
 276
 277             # install python packages
 278             pip install "flake8==2.6.2"
 279
 280             # generate and upload lint log
 281             echo "Running flake8 code on $PROJECT ..."
 282
 283             # Get number of flake8 violations. If none, this will be an
 284             # empty string: ""
 285             FLAKE_COUNT="$(find . \
 286                 -path './releng_flake8' -prune -o \
 287                 -path './.tox' -prune -o \
 288                 -type f -name "*.py" -print | \
 289                 xargs flake8 --exit-zero -qq --count 2>&1)"
 290
 291             # Ensure we start with a clean environment
 292             rm -f lint.log
 293
 294             if [ ! -z $FLAKE_COUNT ]; then
 295               echo "Flake8 Violations: $FLAKE_COUNT" > lint.log
 296               find . \
 297                   -path './releng_flake8' -prune -o \
 298                   -path './.tox' -prune -o \
 299                   -type f -name "*.py" -print | \
 300                   xargs flake8 --exit-zero --first >> violation.log
 301               SHOWN=$(wc -l violation.log | cut -d' ' -f1)
 302               echo -e "First $SHOWN shown\n---" >> lint.log
 303               cat violation.log >> lint.log
 304               sed -r -i '4,$s/^/ /g' lint.log
 305               rm violation.log
 306             fi
 307
 308             deactivate
 309
 310 - builder:
 311     name: report-lint-result-to-gerrit
 312     builders:
 313         - shell: |
 314             #!/bin/bash
 315             set -o errexit
 316             set -o pipefail
 317             set -o xtrace
 318             export PATH=$PATH:/usr/local/bin/
 319
 320             # If no violations were found, no lint log will exist.
 321             if [[ -e lint.log ]] ; then
 322                 echo -e "\nposting linting report to gerrit...\n"
 323
 324                 cat lint.log
 325                 echo
 326
 327                 ssh -p 29418 gerrit.opnfv.org \
 328                     "gerrit review -p $GERRIT_PROJECT \
 329                      -m \"$(cat lint.log)\" \
 330                      $GERRIT_PATCHSET_REVISION \
 331                      --notify NONE"
 332
 333                 exit 1
 334             fi
 335
 336 - builder:
 337     name: upload-review-docs
 338     builders:
 339         - build-html-and-pdf-docs-output
 340         - upload-under-review-docs-to-opnfv-artifacts
 341         - report-docs-build-result-to-gerrit
 342
 343 - builder:
 344     name: upload-merged-docs
 345     builders:
 346         - build-html-and-pdf-docs-output
 347         - upload-generated-docs-to-opnfv-artifacts
 348         - report-docs-build-result-to-gerrit
 349         - remove-old-docs-from-opnfv-artifacts
 350
 351 - builder:
 352     name: check-bash-syntax
 353     builders:
 354         - shell: "find . -name '*.sh' | xargs bash -n"
 355
 356 - builder:
 357     name: lint-yaml-code
 358     builders:
 359         - shell: |
 360             #!/bin/bash
 361             set -o errexit
 362             set -o pipefail
 363             set -o xtrace
 364             export PATH=$PATH:/usr/local/bin/
 365
 366             # generate and upload lint log
 367             echo "Running yaml code on $PROJECT ..."
 368
 369             # Ensure we start with a clean environment
 370             rm -f yaml-violation.log lint.log
 371
 372             # Get number of yaml violations. If none, this will be an
 373             # empty string: ""
 374             find . \
 375                 -path './releng_yamllint' -prune -o \
 376                 -path './.tox' -prune -o \
 377                 -type f -name "*.yml" -print \
 378                 -o -name "*.yaml" -print | \
 379                 xargs yamllint > yaml-violation.log || true
 380
 381             if [ -s "yaml-violation.log" ]; then
 382               SHOWN=$(cat yaml-violation.log| grep -v "^$" |wc -l)
 383               echo -e "First $SHOWN shown\n---" > lint.log
 384               cat yaml-violation.log >> lint.log
 385               sed -r -i '4,$s/^/ /g' lint.log
 386             fi
 387
 388 - publisher:
 389     name: archive-artifacts
 390     publishers:
 391         - archive:
 392             artifacts: '{artifacts}'
 393             allow-empty: true
 394             fingerprint: true
 395             latest-only: true