From: mei mei <meimei@huawei.com>
Date: Fri, 2 Feb 2018 06:34:35 +0000 (+0000)
Subject: Merge "container4nfv: add arm64 test scenarios"
X-Git-Url: https://gerrit.opnfv.org/gerrit/gitweb?a=commitdiff_plain;h=f5559f5115d1167a1bc97e50fd373f7c3c3545c1;hp=d3ba2c736c8f88a561c54e741088b72d60c61655;p=releng.git

Merge "container4nfv: add arm64 test scenarios"
---

diff --git a/.yamllint b/.yamllint
index 4402f17fa..6ac3724dd 100644
--- a/.yamllint
+++ b/.yamllint
@@ -2,7 +2,9 @@
 extends: default
 
 rules:
-  # 120 chars should be enough and don't fail if a line is longer
+  # ONLY 'max' length is configure, other parameters are default values.
   line-length:
     max: 120
     level: warning
+    allow-non-breakable-words: true
+    allow-non-breakable-inline-mappings: false
diff --git a/jjb/compass4nfv/compass-ci-jobs.yml b/jjb/compass4nfv/compass-ci-jobs.yml
index 0790b83ca..70296bda1 100644
--- a/jjb/compass4nfv/compass-ci-jobs.yml
+++ b/jjb/compass4nfv/compass-ci-jobs.yml
@@ -127,6 +127,7 @@
     jobs:
       - 'compass-{scenario}-{pod}-daily-{stream}'
       - 'compass-deploy-{pod}-daily-{stream}'
+      - 'compass-collect-logs-{pod}-daily-{stream}'
 
 ########################
 # job templates
@@ -281,6 +282,25 @@
                     build-step-failure-threshold: 'never'
                     failure-threshold: 'never'
                     unstable-threshold: 'FAILURE'
+      - conditional-step:
+          condition-kind: and
+          condition-operands:
+            - condition-kind: regex-match
+              regex: master
+              label: '{stream}'
+          steps:
+            - trigger-builds:
+                - project: 'compass-collect-logs-{pod}-daily-{stream}'
+                  current-parameters: false
+                  predefined-parameters:
+                    DEPLOY_SCENARIO={scenario}
+                  block: true
+                  same-node: true
+                  block-thresholds:
+                    build-step-failure-threshold: 'never'
+                    failure-threshold: 'never'
+                    unstable-threshold: 'FAILURE'
+
 
 - job-template:
     name: 'compass-deploy-{pod}-daily-{stream}'
@@ -346,6 +366,41 @@
             - shell:
                 !include-raw-escape: ./compass-deploy.sh
 
+- job-template:
+    name: 'compass-collect-logs-{pod}-daily-{stream}'
+
+    disabled: '{obj:disabled}'
+
+    concurrent: true
+
+    properties:
+      - logrotate-default
+
+    parameters:
+      - project-parameter:
+          project: '{project}'
+          branch: '{branch}'
+      - compass-ci-parameter:
+          installer: '{installer}'
+          gs-pathname: '{gs-pathname}'
+          ppa-pathname: '{ppa-pathname}'
+      - '{slave-label}-defaults'
+      - '{installer}-defaults'
+
+    scm:
+      - git-scm
+
+    wrappers:
+      - build-name:
+          name: '$BUILD_NUMBER - Scenario: $DEPLOY_SCENARIO'
+      - fix-workspace-permissions
+
+    builders:
+      - description-setter:
+          description: "Built on $NODE_NAME"
+      - shell:
+          !include-raw-escape: ./compass-logs.sh
+
 ########################
 # parameter macros
 ########################
@@ -370,6 +425,10 @@
       - string:
           name: PPA_CACHE
           default: "$WORKSPACE/work/repo/"
+      - string:
+          name: LOG_DIRECTORY
+          default: $WORKSPACE/log_output
+          description: "Directory where the logs will be located upon the completion of the collection."
 
 ########################
 # trigger macros
diff --git a/jjb/compass4nfv/compass-logs.sh b/jjb/compass4nfv/compass-logs.sh
new file mode 100644
index 000000000..03c8b20ac
--- /dev/null
+++ b/jjb/compass4nfv/compass-logs.sh
@@ -0,0 +1,28 @@
+#!/bin/bash
+set -o nounset
+set -o pipefail
+
+# log info to console
+echo "Uploading the logs $INSTALLER_TYPE artifact. This could take some time..."
+echo "--------------------------------------------------------"
+echo
+
+# create the log directory if it doesn't exist
+[[ -d $LOG_DIRECTORY ]] || mkdir -p $LOG_DIRECTORY
+
+OPNFV_ARTIFACT_VERSION=$(date -u +"%Y-%m-%d_%H-%M-%S")
+COMPASS_LOG_FILENAME="${JOB_NAME}_${BUILD_NUMBER}_${OPNFV_ARTIFACT_VERSION}.log.tar.gz"
+
+
+sudo docker exec -it compass-tasks /bin/bash /opt/collect-log.sh
+sudo docker cp compass-tasks:/opt/log.tar.gz ${LOG_DIRECTORY}/${COMPASS_LOG_FILENAME}
+
+sudo chown $(whoami):$(whoami) ${LOG_DIRECTORY}/${COMPASS_LOG_FILENAME}
+
+gsutil cp "${LOG_DIRECTORY}/${COMPASS_LOG_FILENAME}" \
+     "gs://${GS_URL}/logs/${COMPASS_LOG_FILENAME}" > /dev/null 2>&1
+
+echo
+echo "--------------------------------------------------------"
+echo "Done!"
+echo "Artifact is available as http://${GS_URL}/logs/${COMPASS_LOG_FILENAME}"
diff --git a/jjb/compass4nfv/compass-verify-jobs.yml b/jjb/compass4nfv/compass-verify-jobs.yml
index 7024dad58..75d1653f9 100644
--- a/jjb/compass4nfv/compass-verify-jobs.yml
+++ b/jjb/compass4nfv/compass-verify-jobs.yml
@@ -136,11 +136,6 @@
               node-parameters: true
               kill-phase-on: FAILURE
               abort-all-job: true
-            - name: 'opnfv-yamllint-verify-{stream}'
-              current-parameters: true
-              node-parameters: true
-              kill-phase-on: FAILURE
-              abort-all-job: true
       - multijob:
           name: deploy-virtual
           condition: SUCCESSFUL
@@ -261,11 +256,6 @@
               node-parameters: true
               kill-phase-on: FAILURE
               abort-all-job: true
-            - name: 'opnfv-yamllint-verify-{stream}'
-              current-parameters: true
-              node-parameters: true
-              kill-phase-on: FAILURE
-              abort-all-job: true
       - multijob:
           name: deploy-virtual
           condition: SUCCESSFUL
diff --git a/jjb/fuel/fuel-deploy.sh b/jjb/fuel/fuel-deploy.sh
index cb9febe2a..35bf3dc01 100755
--- a/jjb/fuel/fuel-deploy.sh
+++ b/jjb/fuel/fuel-deploy.sh
@@ -38,7 +38,6 @@ fi
 
 # set deployment parameters
 export TMPDIR=${HOME}/tmpdir
-BRIDGE=${BRIDGE:-pxebr}
 # shellcheck disable=SC2153
 LAB_NAME=${NODE_NAME/-*}
 # shellcheck disable=SC2153
@@ -49,10 +48,6 @@ LAB_CONFIG_URL=${LAB_CONFIG_URL:-'ssh://jenkins-ericsson@gerrit.opnfv.org:29418/
 # Fuel requires deploy script to be ran with sudo, Armband does not
 SUDO='sudo -E'
 if [ "${PROJECT}" = 'fuel' ]; then
-    # Fuel does not use any POD-specific configuration for virtual deploys
-    if [[ "${NODE_NAME}" =~ "virtual" ]]; then
-        POD_NAME="virtual_kvm"
-    fi
     # Fuel currently supports ericsson, intel, lf and zte labs
     if [[ ! "${LAB_NAME}" =~ (ericsson|intel|lf|zte) ]]; then
         echo "Unsupported/unidentified lab ${LAB_NAME}. Cannot continue!"
@@ -81,6 +76,7 @@ if [[ "$BRANCH" =~ (danube|euphrates) ]]; then
         rm -rf "${LOCAL_CFG}"
         git clone --quiet --branch "${BRANCH}" "${LAB_CONFIG_URL}" "${LOCAL_CFG}"
         LAB_CONFIG_ARG="-b file://${LOCAL_CFG}"
+        BRIDGE_ARG="-B ${BRIDGE:-pxebr}"
     else
         LAB_CONFIG_ARG="-b ${LAB_CONFIG_URL}"
     fi
@@ -92,7 +88,7 @@ FUEL_LOG_FILENAME="${JOB_NAME}_${BUILD_NUMBER}.log.tar.gz"
 # construct the command
 DEPLOY_COMMAND="${SUDO} ${WORKSPACE}/ci/deploy.sh ${LAB_CONFIG_ARG:-} \
     -l ${LAB_NAME} -p ${POD_NAME} -s ${DEPLOY_SCENARIO} ${ISO_FILE_ARG:-} \
-    -B ${DEFAULT_BRIDGE:-${BRIDGE}} -S ${TMPDIR} \
+    -S ${TMPDIR} ${BRIDGE_ARG:-} \
     -L ${WORKSPACE}/${FUEL_LOG_FILENAME}"
 
 # log info to console
diff --git a/jjb/functest/functest-alpine.sh b/jjb/functest/functest-alpine.sh
index 735ca6f7f..0f4e80f24 100755
--- a/jjb/functest/functest-alpine.sh
+++ b/jjb/functest/functest-alpine.sh
@@ -6,7 +6,10 @@ set +o pipefail
 
 check_os_deployment() {
     FUNCTEST_IMAGE=opnfv/functest-healthcheck:${DOCKER_TAG}
+    echo "Functest: Pulling Functest Docker image ${FUNCTEST_IMAGE} ..."
+    docker pull ${FUNCTEST_IMAGE}>/dev/null
     cmd="docker run --rm --privileged=true ${volumes} ${FUNCTEST_IMAGE} check_deployment"
+    echo "Checking deployment, CMD: ${cmd}"
     eval ${cmd}
     ret_value=$?
     if [ ${ret_value} != 0 ]; then
@@ -107,7 +110,7 @@ elif [[ ${INSTALLER_TYPE} == 'compass' ]]; then
 elif [[ ${INSTALLER_TYPE} == 'fuel' && ${DEPLOY_TYPE} == 'baremetal' ]]; then
     cacert_file_vol="-v ${HOME}/os_cacert:/etc/ssl/certs/mcp_os_cacert"
 fi
-rc_file_vol="-v ${rc_file}:${FUNCTEST_DIR}/conf/openstack.creds"
+rc_file_vol="-v ${rc_file}:${FUNCTEST_DIR}/conf/env_file"
 
 
 # Set iptables rule to allow forwarding return traffic for container
diff --git a/jjb/global/installer-params.yml b/jjb/global/installer-params.yml
index 916db808d..683ef78ba 100644
--- a/jjb/global/installer-params.yml
+++ b/jjb/global/installer-params.yml
@@ -50,10 +50,6 @@
           name: EXTERNAL_NETWORK
           default: 'floating_net'
           description: 'external network for test'
-      - string:
-          name: BRIDGE
-          default: 'pxebr'
-          description: 'Bridge(s) to be used by salt master'
       - string:
           name: GS_URL
           default: '$GS_BASE{gs-pathname}'
@@ -80,8 +76,8 @@
           description: 'Model to deploy (os|k8)'
       - string:
           name: OS_RELEASE
-          default: 'ocata'
-          description: 'OpenStack release (mitaka|newton|ocata)'
+          default: 'pike'
+          description: 'OpenStack release (mitaka|ocata|pike)'
       - string:
           name: EXTERNAL_NETWORK
           default: ext-net
diff --git a/jjb/global/releng-macros.yml b/jjb/global/releng-macros.yml
index c6a65ca21..a5a83e23f 100644
--- a/jjb/global/releng-macros.yml
+++ b/jjb/global/releng-macros.yml
@@ -298,127 +298,91 @@
           fi
 
 - builder:
-    name: lint-python-code
+    name: upload-review-docs
     builders:
-      - shell: |
-          #!/bin/bash
-          set -o errexit
-          set -o pipefail
-          set -o xtrace
-          export PATH=$PATH:/usr/local/bin/
-
-          virtualenv -p python2.7 $WORKSPACE/releng_flake8
-          source $WORKSPACE/releng_flake8/bin/activate
-
-          # install python packages
-          pip install "flake8==2.6.2"
-
-          # generate and upload lint log
-          echo "Running flake8 code on $PROJECT ..."
-
-          # Get number of flake8 violations. If none, this will be an
-          # empty string: ""
-          FLAKE_COUNT="$(find . \
-              -path './releng_flake8' -prune -o \
-              -path './.tox' -prune -o \
-              -type f -name "*.py" -print | \
-              xargs flake8 --exit-zero -qq --count 2>&1)"
+      - upload-under-review-docs-to-opnfv-artifacts
+      - report-build-result-to-gerrit
 
+- builder:
+    name: lint-init
+    builders:
+      - shell: |
           # Ensure we start with a clean environment
-          rm -f lint.log
-
-          if [ ! -z $FLAKE_COUNT ]; then
-            echo "Flake8 Violations: $FLAKE_COUNT" > lint.log
-            find . \
-                -path './releng_flake8' -prune -o \
-                -path './.tox' -prune -o \
-                -type f -name "*.py" -print | \
-                xargs flake8 --exit-zero --first >> violation.log
-            SHOWN=$(wc -l violation.log | cut -d' ' -f1)
-            echo -e "First $SHOWN shown\n---" >> lint.log
-            cat violation.log >> lint.log
-            sed -r -i '4,$s/^/ /g' lint.log
-            rm violation.log
-          fi
-
-          deactivate
+          rm -f bash-violation.log python-violation.log yaml-violation.log violation.log
+          git --no-pager diff --diff-filter=MCRAT --name-only HEAD^1 > modified_files
 
 - builder:
-    name: report-lint-result-to-gerrit
+    name: lint-report
     builders:
       - shell: |
-          #!/bin/bash
-          set -o errexit
-          set -o pipefail
-          set -o xtrace
-          export PATH=$PATH:/usr/local/bin/
-
-          # If no violations were found, no lint log will exist.
-          if [[ -e lint.log ]] ; then
-              echo -e "\nposting linting report to gerrit...\n"
-
-              cat lint.log
-              echo
-
-              ssh -p 29418 gerrit.opnfv.org \
-                  "gerrit review -p $GERRIT_PROJECT \
-                   -m \"$(cat lint.log)\" \
-                   $GERRIT_PATCHSET_REVISION \
-                   --notify NONE"
-
+          if [[ -s violation.log ]]; then
+              echo "Reporting lint result..."
+              msg="Found syntax error and/or coding style violation(s) in the files modified by your patchset."
+              sed -i -e '1s/^//$msg\n\n/' violation.log
+              cmd="gerrit review -p $GERRIT_PROJECT -m \"$(cat violation.log)\" $GERRIT_PATCHSET_REVISION --notify NONE"
+              ssh -p 29418 gerrit.opnfv.org "$cmd"
+
+              # Make sure the caller job failed
               exit 1
           fi
 
 - builder:
-    name: upload-review-docs
+    name: lint-bash-code
     builders:
-      - upload-under-review-docs-to-opnfv-artifacts
-      - report-build-result-to-gerrit
+      - shell: |
+          echo "Checking bash code..."
+          for f in $(egrep '\.sh$' modified_files)
+          do
+              bash -n "$f" 2>> bash-violation.log
+          done
+          if [[ -s bash-violation.log ]]; then
+              echo -e "Bash syntax error(s)\n---" >> violation.log
+              sed -e 's/^/ /g' bash-violation.log >> violation.log
+          fi
 
 - builder:
-    name: check-bash-syntax
+    name: lint-python-code
     builders:
-      - shell: "find . -name '*.sh' | xargs bash -n"
+      - shell: |
+          # Install python package 
+          sudo pip install "flake8==2.6.2"
+
+          echo "Checking python code..."
+          for f in $(egrep '\.py$' modified_files)
+          do
+              flake8 "$f" >> python-violation.log
+          done
+          if [[ -s python-violation.log ]]; then
+              echo -e "Python violation(s)\n---" >> violation.log
+              sed -e 's/^/ /g' python-violation.log >> violation.log
+          fi
 
 - builder:
     name: lint-yaml-code
     builders:
       - shell: |
-          #!/bin/bash
-          set -o errexit
-          set -o pipefail
-          set -o xtrace
-          export PATH=$PATH:/usr/local/bin/
-
-          # install python packages
+          # sudo Install python packages
           sudo pip install "yamllint==1.8.2"
 
-          # generate and upload lint log
-          echo "Running yaml code on $PROJECT ..."
-
-          # Get list of yaml files
-          YAML_FILES=$(git --no-pager diff --diff-filter=MCRAT --name-only HEAD^1 | egrep "ya?ml$") || true
-
-          #If YAML_FILES is none exit with 0
-          if [ -z "$YAML_FILES" ]; then
-              exit 0
-          fi
-
-          # Ensure we start with a clean environment
-          rm -f yaml-violation.log lint.log
-
-          # Yamllint files only in patchset
-          for yamlfile in $YAML_FILES; do
-            yamllint $yamlfile >> yaml-violation.log || true
+          echo "Checking yaml file..."
+          for f in $(egrep '\.ya?ml$' modified_files)
+          do
+              yamllint "$f" >> yaml-violation.log
           done
-
-          if [ -s "yaml-violation.log" ]; then
-            SHOWN=$(grep -c -v "^$" yaml-violation.log)
-            echo -e "First $SHOWN shown\n---" > lint.log
-            cat yaml-violation.log >> lint.log
-            sed -r -i '4,$s/^/ /g' lint.log
+          if [[ -s yaml-violation.log ]]; then
+              echo -e "YAML violation(s)\n---" >> violation.log
+              sed -e 's/^/ /g' yaml-violation.log >> violation.log
           fi
 
+- builder:
+    name: lint-all-code
+    builders:
+      - lint-init
+      - lint-bash-code
+      - lint-python-code
+      - lint-yaml-code
+      - lint-report
+
 - builder:
     name: clean-workspace
     builders:
diff --git a/jjb/opnfvdocs/opnfvdocs.yml b/jjb/opnfvdocs/opnfvdocs.yml
index 3dfb9d155..908ddba9d 100644
--- a/jjb/opnfvdocs/opnfvdocs.yml
+++ b/jjb/opnfvdocs/opnfvdocs.yml
@@ -69,7 +69,7 @@
             notbuilt: true
 
     builders:
-      - check-bash-syntax
+      - lint-bash-code
 
 - job-template:
     name: 'opnfvdocs-merge-shellcheck-{stream}'
@@ -107,4 +107,4 @@
                   branch-pattern: '**/{branch}'
 
     builders:
-      - check-bash-syntax
+      - lint-bash-code
diff --git a/jjb/pharos/check-jinja2.yml b/jjb/pharos/check-jinja2.yml
index 49a2ba539..7251d3f61 100644
--- a/jjb/pharos/check-jinja2.yml
+++ b/jjb/pharos/check-jinja2.yml
@@ -74,10 +74,10 @@
                 - compare-type: ANT
                   pattern: 'check-schema.sh'
           skip-vote:
-            successful: true
-            failed: true
-            unstable: true
-            notbuilt: true
+            successful: false
+            failed: false
+            unstable: false
+            notbuilt: false
 
     builders:
       - check-pdf-jinja
diff --git a/jjb/releng/automate.yml b/jjb/releng/automate.yml
index afeb35821..6bd952a1d 100644
--- a/jjb/releng/automate.yml
+++ b/jjb/releng/automate.yml
@@ -46,6 +46,12 @@
     builders:
       - mongodb-backup
 
+    publishers:
+      - email:
+          recipients: serena.feng.711@gmail.com
+          notify-every-unstable-build: true
+          send-to-individuals: true
+
 - job-template:
     name: '{module}-verify-{stream}'
 
diff --git a/jjb/releng/opnfv-lint.yml b/jjb/releng/opnfv-lint.yml
index d566430d6..668b49021 100644
--- a/jjb/releng/opnfv-lint.yml
+++ b/jjb/releng/opnfv-lint.yml
@@ -31,10 +31,73 @@
 
     disabled: '{obj:disabled}'
 
+    concurrent: true
+
     parameters:
       - project-parameter:
           project: $GERRIT_PROJECT
           branch: '{branch}'
+      - node:
+          name: SLAVE_NAME
+          description: Slaves to execute yamllint
+          default-slaves:
+            - lf-build1
+          allowed-multiselect: true
+          ignore-offline-nodes: true
+
+    scm:
+      - git-scm-gerrit
+
+    triggers:
+      - gerrit:
+          server-name: 'gerrit.opnfv.org'
+          trigger-on:
+            - patchset-created-event:
+                exclude-drafts: 'false'
+                exclude-trivial-rebase: 'false'
+                exclude-no-code-change: 'false'
+            - draft-published-event
+            - comment-added-contains-event:
+                comment-contains-value: 'recheck'
+            - comment-added-contains-event:
+                comment-contains-value: 'reverify'
+          projects:
+            - project-compare-type: 'REG_EXP'
+              project-pattern: 'releng|doctor'
+              branches:
+                - branch-compare-type: 'ANT'
+                  branch-pattern: '**/{branch}'
+              file-paths:
+                - compare-type: ANT
+                  pattern: '**/*.py'
+                - compare-type: ANT
+                  pattern: '**/*.sh'
+                - compare-type: ANT
+                  pattern: '**/*.yml'
+                - compare-type: ANT
+                  pattern: '**/*.yaml'
+
+    builders:
+      - lint-all-code
+
+- job-template:
+    name: 'opnfv-pylint-verify-{stream}'
+
+    disabled: '{obj:disabled}'
+
+    concurrent: true
+
+    parameters:
+      - project-parameter:
+          project: $GERRIT_PROJECT
+          branch: '{branch}'
+      - node:
+          name: SLAVE_NAME
+          description: Slaves to execute yamllint
+          default-slaves:
+            - lf-build1
+          allowed-multiselect: true
+          ignore-offline-nodes: true
 
     scm:
       - git-scm-gerrit
@@ -54,7 +117,7 @@
                 comment-contains-value: 'reverify'
           projects:
             - project-compare-type: 'REG_EXP'
-              project-pattern: 'functest|sdnvpn|qtip|daisy|sfc|escalator|releng'
+              project-pattern: 'functest|sdnvpn|qtip|daisy|sfc|escalator'
               branches:
                 - branch-compare-type: 'ANT'
                   branch-pattern: '**/{branch}'
@@ -63,8 +126,9 @@
                   pattern: '**/*.py'
 
     builders:
+      - lint-init
       - lint-python-code
-      - report-lint-result-to-gerrit
+      - lint-report
 
 - job-template:
     name: 'opnfv-yamllint-verify-{stream}'
@@ -103,7 +167,7 @@
                 comment-contains-value: 'reverify'
           projects:
             - project-compare-type: 'REG_EXP'
-              project-pattern: 'armband|fuel|octopus|pharos|releng|releng-anteater'
+              project-pattern: 'armband|fuel|octopus|pharos|releng-anteater'
               branches:
                 - branch-compare-type: 'ANT'
                   branch-pattern: '**/{branch}'
@@ -114,5 +178,6 @@
                   pattern: '**/*.yaml'
 
     builders:
+      - lint-init
       - lint-yaml-code
-      - report-lint-result-to-gerrit
+      - lint-report
diff --git a/jjb/xci/xci-verify-jobs.yml b/jjb/xci/xci-verify-jobs.yml
index 3c314568f..28bc317bd 100644
--- a/jjb/xci/xci-verify-jobs.yml
+++ b/jjb/xci/xci-verify-jobs.yml
@@ -21,10 +21,10 @@
           notbuilt: false
       - centos:
           disabled: false
-          successful: true
-          failed: true
-          unstable: true
-          notbuilt: true
+          successful: false
+          failed: false
+          unstable: false
+          notbuilt: false
       - opensuse:
           disabled: false
           successful: false