Merge "Remove Restriction on Total Docker Builds per Node"

diff --git a/jjb/functest/functest-alpine.sh b/jjb/functest/functest-alpine.sh
index fe1888f..8ede529 100755 (executable)
--- a/jjb/functest/functest-alpine.sh
+++ b/jjb/functest/functest-alpine.sh
@@ -6,6 +6,7 @@ set +o pipefail
 
 run_tiers() {
     cmd_opt='prepare_env start && run_tests -r -t all'
+    [[ $BUILD_TAG =~ "suite" ]] && cmd_opt='prepare_env start && run_tests -t all'
     ret_val_file="${HOME}/opnfv/functest/results/${BRANCH##*/}/return_value"
     echo 0 > ${ret_val_file}
 
@@ -26,25 +27,27 @@ run_tiers() {
 run_test() {
     test_name=$1
     cmd_opt='prepare_env start && run_tests -r -t $test_name'
+    [[ $BUILD_TAG =~ "suite" ]] && cmd_opt='prepare_env start && run_tests -t $test_name'
     ret_val_file="${HOME}/opnfv/functest/results/${BRANCH##*/}/return_value"
     echo 0 > ${ret_val_file}
     # Determine which Functest image should be used for the test case
     case ${test_name} in
         connection_check|api_check|snaps_health_check)
-            FUNCTEST_IMAGE=opnfv/functest-healthcheck
+            FUNCTEST_IMAGE=opnfv/functest-healthcheck ;;
         vping_ssh|vping_userdata|tempest_smoke_serial|rally_sanity|refstack_defcore|odl|odl_netvirt|fds|snaps_smoke)
-            FUNCTEST_IMAGE=opnfv/functest-smoke
+            FUNCTEST_IMAGE=opnfv/functest-smoke ;;
         tempest_full_parallel|tempest_custom|rally_full)
-            FUNCTEST_IMAGE=opnfv/functest-components
+            FUNCTEST_IMAGE=opnfv/functest-components ;;
         cloudify_ims|orchestra_openims|orchestra_clearwaterims|vyos_vrouter)
-            FUNCTEST_IMAGE=opnfv/functest-vnf
+            FUNCTEST_IMAGE=opnfv/functest-vnf ;;
         promise|doctor-notification|bgpvpn|functest-odl-sfc|domino-multinode|barometercollectd)
-            FUNCTEST_IMAGE=opnfv/functest-features
+            FUNCTEST_IMAGE=opnfv/functest-features ;;
         parser)
-            FUNCTEST_IMAGE=opnfv/functest-parser
+            FUNCTEST_IMAGE=opnfv/functest-parser ;;
         *)
             echo "Unkown test case $test_name"
             exit 1
+            ;;
     esac
     echo "Functest: Pulling Functest Docker image ${FUNCTEST_IMAGE} ..."
     docker pull ${FUNCTEST_IMAGE}>/dev/null

diff --git a/jjb/joid/joid-daily-jobs.yml b/jjb/joid/joid-daily-jobs.yml
index ac04962..b28efc3 100644 (file)
--- a/jjb/joid/joid-daily-jobs.yml
+++ b/jjb/joid/joid-daily-jobs.yml
@@ -13,11 +13,11 @@
     # -------------------------------
     # BRANCH ANCHORS
     # -------------------------------
-    master: &master
-      stream: master
-      branch: '{stream}'
-      disabled: false
-      gs-pathname: ''
+    euphrates: &euphrates
+        stream: euphrates
+        branch: 'stable/{stream}'
+        disabled: false
+        gs-pathname: '/{stream}'
     # -------------------------------
     # POD, INSTALLER, AND BRANCH MAPPING
     # -------------------------------
@@ -26,19 +26,8 @@
     pod:
       - baremetal:
           slave-label: joid-baremetal
-          <<: *master
-      - virtual:
-          slave-label: joid-virtual
-          <<: *master
-      # -------------------------------
-      #        None-CI PODs
-      # -------------------------------
-      - orange-pod1:
-          slave-label: orange-pod1
-          <<: *master
-      - cengn-pod1:
-          slave-label: cengn-pod1
-          <<: *master
+          <<: *euphrates
+
     # -------------------------------
     # scenarios
     # -------------------------------
@@ -53,18 +42,12 @@
           auto-trigger-name: 'joid-{scenario}-{pod}-{stream}-trigger'
       - 'os-odl_l2-nofeature-ha':
           auto-trigger-name: 'daily-trigger-disabled'
-      - 'os-onos-nofeature-ha':
-          auto-trigger-name: 'daily-trigger-disabled'
       - 'os-odl_l2-nofeature-noha':
           auto-trigger-name: 'daily-trigger-disabled'
-      - 'os-onos-nofeature-noha':
-          auto-trigger-name: 'daily-trigger-disabled'
-      - 'os-onos-sfc-ha':
-          auto-trigger-name: 'daily-trigger-disabled'
       - 'os-ocl-nofeature-ha':
-          auto-trigger-name: 'daily-trigger-disabled'
+          auto-trigger-name: 'joid-{scenario}-{pod}-{stream}-trigger'
       - 'os-ocl-nofeature-noha':
-          auto-trigger-name: 'daily-trigger-disabled'
+          auto-trigger-name: 'joid-{scenario}-{pod}-{stream}-trigger'
       - 'k8-nosdn-nofeature-noha':
           auto-trigger-name: 'daily-trigger-disabled'
       - 'k8-nosdn-lb-noha':
@@ -151,7 +134,7 @@
               build-step-failure-threshold: 'never'
               failure-threshold: 'never'
               unstable-threshold: 'FAILURE'
-# 1.dovetail only master by now, not sync with A/B/C branches
+# 1.dovetail only euphrates by now, not sync with A/B/C branches
 # 2.here the stream means the SUT stream, dovetail stream is defined in
 #   its own job
 # 3.only debug testsuite here(includes basic testcase,
@@ -234,191 +217,58 @@
 ########################
 # trigger macros
 ########################
-# os-nosdn-nofeature-ha trigger - branch: master
+# os-nosdn-nofeature-ha trigger - branch: euphrates
 - trigger:
-    name: 'joid-os-nosdn-nofeature-ha-baremetal-master-trigger'
+    name: 'joid-os-nosdn-nofeature-ha-baremetal-euphrates-trigger'
     triggers:
       - timed: '5 2 * * *'
+# os-odl_l2-nofeature-ha trigger - branch: euphrates
 - trigger:
-    name: 'joid-os-nosdn-nofeature-ha-virtual-master-trigger'
-    triggers:
-      - timed: ''
-- trigger:
-    name: 'joid-os-nosdn-nofeature-ha-orange-pod1-master-trigger'
-    triggers:
-      - timed: ''
-- trigger:
-    name: 'joid-os-nosdn-nofeature-ha-cengn-pod1-master-trigger'
-    triggers:
-      - timed: ''
-# os-odl_l2-nofeature-ha trigger - branch: master
-- trigger:
-    name: 'joid-os-odl_l2-nofeature-ha-baremetal-master-trigger'
+    name: 'joid-os-odl_l2-nofeature-ha-baremetal-euphrates-trigger'
     triggers:
       - timed: '5 7 * * *'
+# os-nosdn-lxd-noha trigger - branch: euphrates
 - trigger:
-    name: 'joid-os-odl_l2-nofeature-ha-virtual-master-trigger'
-    triggers:
-      - timed: ''
-- trigger:
-    name: 'joid-os-odl_l2-nofeature-ha-orange-pod1-master-trigger'
-    triggers:
-      - timed: ''
-- trigger:
-    name: 'joid-os-odl_l2-nofeature-ha-cengn-pod1-master-trigger'
-    triggers:
-      - timed: ''
-# os-onos-nofeature-ha trigger - branch: master
-- trigger:
-    name: 'joid-os-onos-nofeature-ha-baremetal-master-trigger'
-    triggers:
-      - timed: '5 12 * * *'
-- trigger:
-    name: 'joid-os-onos-nofeature-ha-virtual-master-trigger'
-    triggers:
-      - timed: ''
-- trigger:
-    name: 'joid-os-onos-nofeature-ha-orange-pod1-master-trigger'
-    triggers:
-      - timed: ''
-- trigger:
-    name: 'joid-os-onos-nofeature-ha-cengn-pod1-master-trigger'
-    triggers:
-      - timed: ''
-# os-onos-sfc-ha trigger - branch: master
-- trigger:
-    name: 'joid-os-onos-sfc-ha-baremetal-master-trigger'
-    triggers:
-      - timed: '5 17 * * *'
-- trigger:
-    name: 'joid-os-onos-sfc-ha-virtual-master-trigger'
-    triggers:
-      - timed: ''
-- trigger:
-    name: 'joid-os-onos-sfc-ha-orange-pod1-master-trigger'
-    triggers:
-      - timed: ''
-- trigger:
-    name: 'joid-os-onos-sfc-ha-cengn-pod1-master-trigger'
-    triggers:
-      - timed: ''
-# os-nosdn-lxd-noha trigger - branch: master
-- trigger:
-    name: 'joid-os-nosdn-lxd-noha-baremetal-master-trigger'
+    name: 'joid-os-nosdn-lxd-noha-baremetal-euphrates-trigger'
     triggers:
       - timed: '5 22 * * *'
+# os-nosdn-lxd-ha trigger - branch: euphrates
 - trigger:
-    name: 'joid-os-nosdn-lxd-noha-virtual-master-trigger'
-    triggers:
-      - timed: ''
-- trigger:
-    name: 'joid-os-nosdn-lxd-noha-orange-pod1-master-trigger'
-    triggers:
-      - timed: ''
-- trigger:
-    name: 'joid-os-nosdn-lxd-noha-cengn-pod1-master-trigger'
-    triggers:
-      - timed: ''
-# os-nosdn-lxd-ha trigger - branch: master
-- trigger:
-    name: 'joid-os-nosdn-lxd-ha-baremetal-master-trigger'
+    name: 'joid-os-nosdn-lxd-ha-baremetal-euphrates-trigger'
     triggers:
       - timed: '5 10 * * *'
+# os-nosdn-nofeature-noha trigger - branch: euphrates
 - trigger:
-    name: 'joid-os-nosdn-lxd-ha-virtual-master-trigger'
-    triggers:
-      - timed: ''
-- trigger:
-    name: 'joid-os-nosdn-lxd-ha-orange-pod1-master-trigger'
-    triggers:
-      - timed: ''
-- trigger:
-    name: 'joid-os-nosdn-lxd-ha-cengn-pod1-master-trigger'
-    triggers:
-      - timed: ''
-# os-nosdn-nofeature-noha trigger - branch: master
-- trigger:
-    name: 'joid-os-nosdn-nofeature-noha-baremetal-master-trigger'
+    name: 'joid-os-nosdn-nofeature-noha-baremetal-euphrates-trigger'
     triggers:
       - timed: '5 4 * * *'
+# k8-nosdn-nofeature-noha trigger - branch: euphrates
 - trigger:
-    name: 'joid-os-nosdn-nofeature-noha-virtual-master-trigger'
-    triggers:
-      - timed: ''
-- trigger:
-    name: 'joid-os-nosdn-nofeature-noha-orange-pod1-master-trigger'
-    triggers:
-      - timed: ''
-- trigger:
-    name: 'joid-os-nosdn-nofeature-noha-cengn-pod1-master-trigger'
-    triggers:
-      - timed: ''
-# k8-nosdn-nofeature-noha trigger - branch: master
-- trigger:
-    name: 'joid-k8-nosdn-nofeature-noha-baremetal-master-trigger'
+    name: 'joid-k8-nosdn-nofeature-noha-baremetal-euphrates-trigger'
     triggers:
       - timed: '5 15 * * *'
+# k8-nosdn-lb-noha trigger - branch: euphrates
 - trigger:
-    name: 'joid-k8-nosdn-nofeature-noha-virtual-master-trigger'
-    triggers:
-      - timed: ''
-- trigger:
-    name: 'joid-k8-nosdn-nofeature-noha-orange-pod1-master-trigger'
-    triggers:
-      - timed: ''
-- trigger:
-    name: 'joid-k8-nosdn-nofeature-noha-cengn-pod1-master-trigger'
-    triggers:
-      - timed: ''
-# k8-nosdn-lb-noha trigger - branch: master
-- trigger:
-    name: 'joid-k8-nosdn-lb-noha-baremetal-master-trigger'
+    name: 'joid-k8-nosdn-lb-noha-baremetal-euphrates-trigger'
     triggers:
       - timed: '5 20 * * *'
+# k8-ovn-lb-noha trigger - branch: euphrates
 - trigger:
-    name: 'joid-k8-nosdn-lb-noha-virtual-master-trigger'
-    triggers:
-      - timed: ''
-- trigger:
-    name: 'joid-k8-nosdn-lb-noha-orange-pod1-master-trigger'
-    triggers:
-      - timed: ''
-- trigger:
-    name: 'joid-k8-nosdn-lb-noha-cengn-pod1-master-trigger'
-    triggers:
-      - timed: ''
-# k8-ovn-lb-noha trigger - branch: master
-- trigger:
-    name: 'joid-k8-ovn-lb-noha-baremetal-master-trigger'
+    name: 'joid-k8-ovn-lb-noha-baremetal-euphrates-trigger'
     triggers:
       - timed: '5 17 * * *'
+# os-nosdn-openbaton-ha trigger - branch: euphrates
 - trigger:
-    name: 'joid-k8-ovn-lb-noha-virtual-master-trigger'
-    triggers:
-      - timed: ''
-- trigger:
-    name: 'joid-k8-ovn-lb-noha-orange-pod1-master-trigger'
-    triggers:
-      - timed: ''
-- trigger:
-    name: 'joid-k8-ovn-lb-noha-cengn-pod1-master-trigger'
-    triggers:
-      - timed: ''
-
-# os-nosdn-openbaton-ha trigger - branch: master
-- trigger:
-    name: 'joid-os-nosdn-openbaton-ha-baremetal-master-trigger'
+    name: 'joid-os-nosdn-openbaton-ha-baremetal-euphrates-trigger'
     triggers:
       - timed: '5 25 * * *'
+# os-ocl-nofeature-ha trigger - branch: euphrates
 - trigger:
-    name: 'joid-os-nosdn-openbaton-ha-virtual-master-trigger'
-    triggers:
-      - timed: ''
-- trigger:
-    name: 'joid-os-nosdn-openbaton-ha-orange-pod1-master-trigger'
+    name: 'joid-os-ocl-nofeature-ha-baremetal-euphrates-trigger'
     triggers:
-      - timed: ''
+      - timed: '5 13 * * *'
+# os-ocl-nofeature-ha trigger - branch: euphrates
 - trigger:
-    name: 'joid-os-nosdn-openbaton-ha-cengn-pod1-master-trigger'
+    name: 'joid-os-ocl-nofeature-noha-baremetal-euphrates-trigger'
     triggers:
-      - timed: ''
+      - timed: '5 17 * * *'

diff --git a/jjb/qtip/qtip-verify-jobs.yml b/jjb/qtip/qtip-verify-jobs.yml
index a919728..047d7f2 100644 (file)
--- a/jjb/qtip/qtip-verify-jobs.yml
+++ b/jjb/qtip/qtip-verify-jobs.yml
@@ -8,8 +8,8 @@
     project: qtip
     jobs:
       - 'qtip-verify-{stream}'
-      - 'qtip-review-notebook-{stream}'
-      - 'qtip-merge-{stream}'
+      - 'qtip-verify-notebook-{stream}'
+      - 'qtip-merged-notebook-{stream}'
     stream:
       - master:
           branch: '{stream}'
@@ -69,7 +69,7 @@
 
 # upload juypter notebook to artifacts for review
 - job-template:
-    name: 'qtip-review-notebook-{stream}'
+    name: 'qtip-verify-notebook-{stream}'
 
     disabled: '{obj:disabled}'
 
@@ -104,14 +104,14 @@
               disable-strict-forbidden-file-verification: 'true'
               file-paths:
                 - compare-type: ANT
-                  pattern: 'examples/**'
+                  pattern: '**/*.ipynb'
 
     builders:
       - upload-under-review-notebooks-to-opnfv-artifacts
       - report-build-result-to-gerrit
 
 - job-template:
-    name: 'qtip-merge-{stream}'
+    name: 'qtip-merged-notebook-{stream}'
 
     disabled: '{obj:disabled}'
 
@@ -146,7 +146,7 @@
                   branch-pattern: '**/{branch}'
               file-paths:
                 - compare-type: ANT
-                  pattern: examples/**
+                  pattern: '**/*ipynb'
 
     builders:
       - remove-old-docs-from-opnfv-artifacts
@@ -178,7 +178,6 @@
           export PATH=$PATH:/usr/local/bin/
 
           [[ $GERRIT_CHANGE_NUMBER =~ .+ ]]
-          [[ -d examples ]] || exit 0
 
           echo
           echo "###########################"
@@ -190,8 +189,9 @@
           gs_path="$gs_base/$GERRIT_CHANGE_NUMBER"
           local_path="upload/$GERRIT_CHANGE_NUMBER"
 
-          mkdir -p upload
-          cp -r examples "$local_path"
+          mkdir -p $local_path
+
+          git diff HEAD~1 --name-only | grep -E ".+\.ipynb$" | xargs -I '{}' cp '{}' $local_path
           gsutil -m cp -r "$local_path" "gs://$gs_base/"
 
           echo "Document link(s):" >> gerrit_comment.txt

diff --git a/jjb/releng/verify-releng.sh b/jjb/releng/verify-releng.sh
index 682a8be..a6eaa0f 100755 (executable)
--- a/jjb/releng/verify-releng.sh
+++ b/jjb/releng/verify-releng.sh
@@ -8,7 +8,7 @@
 # http://www.apache.org/licenses/LICENSE-2.0
 ##############################################################################
 #test for non-ascii characters, these can pass the test and end up breaking things in production
-for x in $(find . -name *\.yml -or -name *\.yaml); do
+for x in $(find . -name *\.yml -or -name *\.yaml -or -name *\.sh); do
 
   if LC_ALL=C grep -q '[^[:print:][:space:]]' "$x"; then
     echo "file "$x" contains non-ascii characters"

diff --git a/jjb/securedlab/check-jinja2.sh b/jjb/securedlab/check-jinja2.sh
deleted file mode 100755 (executable)
index 4c1927d..0000000
--- a/jjb/securedlab/check-jinja2.sh
+++ /dev/null
@@ -1,9 +0,0 @@
-#!/bin/bash
-set +x
-set -o errexit
-for lab_configs in $(find labs/ -name 'pod*.yaml'); do
-        while IFS= read -r jinja_templates; do
-          echo "./utils/generate_config.py -y $lab_configs -j $jinja_templates"
-          ./utils/generate_config.py -y $lab_configs -j $jinja_templates
-        done < <(find installers/ -name '*.j2')
-done

diff --git a/jjb/securedlab/check-jinja2.yml b/jjb/securedlab/check-jinja2.yml
index 430ced5..155be17 100644 (file)
--- a/jjb/securedlab/check-jinja2.yml
+++ b/jjb/securedlab/check-jinja2.yml
@@ -1,5 +1,6 @@
+---
 ########################
-# Job configuration to validate jninja2 files
+# Job configuration to validate jinja2 files
 ########################
 - project:
 
@@ -8,15 +9,15 @@
     project: 'securedlab'
 
     jobs:
-        - 'validate-jinja2-templates-{stream}'
+      - 'validate-jinja2-templates-{stream}'
 
     stream:
-        - master:
-            branch: '{stream}'
-            disabled: false
-        - danube:
-            branch: 'stable/{stream}'
-            disabled: false
+      - master:
+          branch: '{stream}'
+          disabled: false
+      - euphrates:
+          branch: 'stable/{stream}'
+          disabled: false
 
 ########################
 # job templates
@@ -30,57 +31,55 @@
     concurrent: true
 
     parameters:
-        - project-parameter:
-            project: $GERRIT_PROJECT
-            branch: '{branch}'
-        - node:
-            name: SLAVE_NAME
-            description: Slave to execute jnija template test
-            default-slaves:
-                - lf-build1
-            allowed-multiselect: true
-            ignore-offline-nodes: true
+      - project-parameter:
+          project: $GERRIT_PROJECT
+          branch: '{branch}'
+      - node:
+          name: SLAVE_NAME
+          description: Slave to execute jnija template test
+          default-slaves:
+            - lf-build1
+          allowed-multiselect: true
+          ignore-offline-nodes: true
 
     scm:
-        - git-scm-gerrit
+      - git-scm-gerrit
 
     triggers:
-        - gerrit:
-            server-name: 'gerrit.opnfv.org'
-            trigger-on:
-                - patchset-created-event:
-                    exclude-drafts: 'false'
-                    exclude-trivial-rebase: 'false'
-                    exclude-no-code-change: 'false'
-                - draft-published-event
-                - comment-added-contains-event:
-                    comment-contains-value: 'recheck'
-                - comment-added-contains-event:
-                    comment-contains-value: 'reverify'
-            projects:
-              - project-compare-type: 'REG_EXP'
-                project-pattern: '{project}'
-                branches:
-                  - branch-compare-type: 'ANT'
-                    branch-pattern: '**/{branch}'
-                file-paths:
-                  - compare-type: ANT
-                    pattern: 'utils/generate_config.yml'
-                  - compare-type: ANT
-                    pattern: '**/*.jinja2'
-                  - compare-type: ANT
-                    pattern: '**/*.yaml'
-            skip-vote:
-                successful: true
-                failed: true
-                unstable: true
-                notbuilt: true
+      - gerrit:
+          server-name: 'gerrit.opnfv.org'
+          trigger-on:
+            - patchset-created-event:
+                exclude-drafts: 'false'
+                exclude-trivial-rebase: 'false'
+                exclude-no-code-change: 'false'
+            - draft-published-event
+            - comment-added-contains-event:
+                comment-contains-value: 'recheck'
+            - comment-added-contains-event:
+                comment-contains-value: 'reverify'
+          projects:
+            - project-compare-type: 'REG_EXP'
+              project-pattern: '{project}'
+              branches:
+                - branch-compare-type: 'ANT'
+                  branch-pattern: '**/{branch}'
+              file-paths:
+                - compare-type: ANT
+                  pattern: '**/*.yaml'
+                - compare-type: ANT
+                  pattern: 'check-jinja2.sh'
+          skip-vote:
+            successful: true
+            failed: true
+            unstable: true
+            notbuilt: true
 
     builders:
-        - check-jinja
+      - check-jinja
 
 - builder:
     name: check-jinja
     builders:
-        - shell:
-            !include-raw-escape: ./check-jinja2.sh
+      - shell: |
+          $WORKSPACE/check-jinja2.sh

diff --git a/jjb/xci/xci-verify-jobs.yml b/jjb/xci/xci-verify-jobs.yml
index efc8995..0e79bd9 100644 (file)
--- a/jjb/xci/xci-verify-jobs.yml
+++ b/jjb/xci/xci-verify-jobs.yml
@@ -231,6 +231,29 @@
             fi
 
             cd $WORKSPACE
+
+            # The start-new-vm.sh script will copy the entire releng-xci directory
+            # so lets prepare the test script now so it can be copied by the script.
+            # Please do not move it elsewhere or you would have to move it to the VM
+            # yourself.
+            cat > xci_test.sh<<EOF
+            #!/bin/bash
+            export DISTRO=$DISTRO
+            export DEPLOY_SCENARIO=$DEPLOY_SCENARIO
+            export FUNCTEST_MODE=$FUNCTEST_MODE
+            export FUNCTEST_SUITE_NAME=$FUNCTEST_SUITE_NAME
+            export XCI_FLAVOR=$XCI_FLAVOR
+            export CLEAN_DIB_IMAGES=$CLEAN_DIB_IMAGES
+            export OPNFV_RELENG_DEV_PATH=/home/devuser/releng-xci/
+            export INSTALLER_TYPE=$INSTALLER_TYPE
+            export GIT_BASE=$GIT_BASE
+            export JENKINS_HOME=$JENKINS_HOME
+
+            cd xci
+            ./xci-deploy.sh
+            EOF
+            chmod a+x xci_test.sh
+
             ./xci/scripts/vm/start-new-vm.sh $DISTRO
         - shell: |
             #!/bin/bash
@@ -241,7 +264,7 @@
                 exit 0
             fi
 
-            ssh ${DISTRO}_xci_vm "cd releng-xci/xci && ./xci-deploy.sh"
+            ssh ${DISTRO}_xci_vm "cd releng-xci && ./xci_test.sh"
 
 
 - builder:
@@ -266,8 +289,8 @@
                 exit 0
             fi
 
-            sudo virsh destroy $DISTRO
-            sudo virsh undefine $DISTRO
+            sudo virsh destroy ${DISTRO}_xci_vm
+            sudo virsh undefine ${DISTRO}_xci_vm
 
 # this will be enabled once the xci is prepared
 #- builder: