Merge "Making docker deploy script generic"
authorMorgan Richomme <morgan.richomme@orange.com>
Thu, 13 Jul 2017 07:22:41 +0000 (07:22 +0000)
committerGerrit Code Review <gerrit@opnfv.org>
Thu, 13 Jul 2017 07:22:41 +0000 (07:22 +0000)
26 files changed:
jjb/armband/armband-ci-jobs.yml
jjb/armband/armband-deploy.sh
jjb/armband/armband-download-artifact.sh
jjb/armband/build.sh
jjb/ci_gate_security/opnfv-ci-gate-security.yml
jjb/compass4nfv/compass-ci-jobs.yml
jjb/qtip/qtip-validate-jobs.yml
jjb/qtip/qtip-weekly-jobs.yml [new file with mode: 0644]
jjb/yardstick/yardstick-daily-jobs.yml
jjb/yardstick/yardstick-daily.sh
prototypes/xci/playbooks/configure-opnfvhost.yml
prototypes/xci/var/Debian.yml
prototypes/xci/var/RedHat.yml
prototypes/xci/var/Suse.yml
utils/test/testapi/opnfv_testapi/tests/unit/resources/__init__.py [new file with mode: 0644]
utils/test/testapi/opnfv_testapi/tests/unit/resources/scenario-c1.json [moved from utils/test/testapi/opnfv_testapi/tests/unit/scenario-c1.json with 100% similarity]
utils/test/testapi/opnfv_testapi/tests/unit/resources/scenario-c2.json [moved from utils/test/testapi/opnfv_testapi/tests/unit/scenario-c2.json with 100% similarity]
utils/test/testapi/opnfv_testapi/tests/unit/resources/test_base.py [moved from utils/test/testapi/opnfv_testapi/tests/unit/test_base.py with 97% similarity]
utils/test/testapi/opnfv_testapi/tests/unit/resources/test_fake_pymongo.py [moved from utils/test/testapi/opnfv_testapi/tests/unit/test_fake_pymongo.py with 100% similarity]
utils/test/testapi/opnfv_testapi/tests/unit/resources/test_pod.py [moved from utils/test/testapi/opnfv_testapi/tests/unit/test_pod.py with 97% similarity]
utils/test/testapi/opnfv_testapi/tests/unit/resources/test_project.py [moved from utils/test/testapi/opnfv_testapi/tests/unit/test_project.py with 98% similarity]
utils/test/testapi/opnfv_testapi/tests/unit/resources/test_result.py [moved from utils/test/testapi/opnfv_testapi/tests/unit/test_result.py with 99% similarity]
utils/test/testapi/opnfv_testapi/tests/unit/resources/test_scenario.py [moved from utils/test/testapi/opnfv_testapi/tests/unit/test_scenario.py with 99% similarity]
utils/test/testapi/opnfv_testapi/tests/unit/resources/test_testcase.py [moved from utils/test/testapi/opnfv_testapi/tests/unit/test_testcase.py with 99% similarity]
utils/test/testapi/opnfv_testapi/tests/unit/resources/test_token.py [moved from utils/test/testapi/opnfv_testapi/tests/unit/test_token.py with 98% similarity]
utils/test/testapi/opnfv_testapi/tests/unit/resources/test_version.py [moved from utils/test/testapi/opnfv_testapi/tests/unit/test_version.py with 94% similarity]

index 55d8ff9..e7de319 100644 (file)
             name: GS_URL
             default: artifacts.opnfv.org/$PROJECT{gs-pathname}
             description: "URL to Google Storage."
+        - string:
+            name: SSH_KEY
+            default: "/tmp/mcp.rsa"
+            description: "Path to private SSH key to access environment nodes. For MCP deployments only."
 
 ########################
 # trigger macros
index e445e08..9964ed5 100755 (executable)
@@ -2,7 +2,7 @@
 # SPDX-license-identifier: Apache-2.0
 ##############################################################################
 # Copyright (c) 2016 Ericsson AB and others.
-#           (c) 2016 Enea Software AB
+#           (c) 2017 Enea Software AB
 # All rights reserved. This program and the accompanying materials
 # are made available under the terms of the Apache License, Version 2.0
 # which accompanies this distribution, and is available at
@@ -13,16 +13,18 @@ set -o pipefail
 
 export TERM="vt220"
 
-# source the file so we get OPNFV vars
-source latest.properties
+if [[ "$BRANCH" != 'master' ]]; then
+    # source the file so we get OPNFV vars
+    source latest.properties
 
-# echo the info about artifact that is used during the deployment
-echo "Using ${OPNFV_ARTIFACT_URL/*\/} for deployment"
+    # echo the info about artifact that is used during the deployment
+    echo "Using ${OPNFV_ARTIFACT_URL/*\/} for deployment"
+fi
 
 if [[ "$JOB_NAME" =~ "merge" ]]; then
     # set simplest scenario for virtual deploys to run for merges
     DEPLOY_SCENARIO="os-nosdn-nofeature-ha"
-else
+elif [[ "$BRANCH" != 'master' ]]; then
     # for none-merge deployments
     # checkout the commit that was used for building the downloaded artifact
     # to make sure the ISO and deployment mechanism uses same versions
@@ -102,7 +104,7 @@ echo "--------------------------------------------------------"
 echo "Scenario: $DEPLOY_SCENARIO"
 echo "Lab: $LAB_NAME"
 echo "POD: $POD_NAME"
-echo "ISO: ${OPNFV_ARTIFACT_URL/*\/}"
+[[ "$BRANCH" != 'master' ]] && echo "ISO: ${OPNFV_ARTIFACT_URL/*\/}"
 echo
 echo "Starting the deployment using $INSTALLER_TYPE. This could take some time..."
 echo "--------------------------------------------------------"
index e2dd097..4f83305 100755 (executable)
@@ -2,6 +2,7 @@
 # SPDX-license-identifier: Apache-2.0
 ##############################################################################
 # Copyright (c) 2016 Ericsson AB and others.
+#           (c) 2017 Enea AB
 # All rights reserved. This program and the accompanying materials
 # are made available under the terms of the Apache License, Version 2.0
 # which accompanies this distribution, and is available at
@@ -10,6 +11,9 @@
 set -o errexit
 set -o pipefail
 
+# disable Fuel ISO download for master branch
+[[ "$BRANCH" == 'master' ]] && exit 0
+
 echo "Host info: $(hostname) $(hostname -I)"
 
 # Configurable environment variables:
index a71cf11..29c01bb 100755 (executable)
@@ -2,12 +2,21 @@
 # SPDX-license-identifier: Apache-2.0
 ##############################################################################
 # Copyright (c) 2016 Ericsson AB and others.
-# Copyright (c) 2016 Enea AB.
+# Copyright (c) 2017 Enea AB.
 # All rights reserved. This program and the accompanying materials
 # are made available under the terms of the Apache License, Version 2.0
 # which accompanies this distribution, and is available at
 # http://www.apache.org/licenses/LICENSE-2.0
 ##############################################################################
+
+# disable Armband iso build for master branch
+if [[ "$BRANCH" == 'master' ]]; then
+    touch $WORKSPACE/.noupload
+    echo "--------------------------------------------------------"
+    echo "Done!"
+    exit 0
+fi
+
 set -o errexit
 set -o nounset
 set -o pipefail
index e2f6ceb..d991573 100644 (file)
@@ -77,7 +77,7 @@
                     comment-contains-value: 'reverify'
             projects:
               - project-compare-type: 'REG_EXP'
-                project-pattern: 'apex|armband|bamboo|barometer|bottlenecks|calipso|compass4nfv|conductor|cooper|functest|octopus|pharos|releng|sandbox'
+                project-pattern: 'apex|armband|bamboo|barometer|bottlenecks|calipso|compass4nfv|conductor|cooper|cperf|daisy|doctor|dovetail|dpacc|enfv|escalator|fds|functest|octopus|pharos|releng|sandbox|yardstick'
                 branches:
                   - branch-compare-type: 'ANT'
                     branch-pattern: '**/{branch}'
index ddd90b6..3335391 100644 (file)
             steps:
                 - trigger-builds:
                     - project: 'dovetail-compass-{pod}-proposed_tests-{stream}'
+                      current-parameters: false
+                      predefined-parameters:
+                        DEPLOY_SCENARIO={scenario}
+                      block: true
+                      same-node: true
+                      block-thresholds:
+                        build-step-failure-threshold: 'never'
+                        failure-threshold: 'never'
+                        unstable-threshold: 'FAILURE'
 
 - job-template:
     name: 'compass-deploy-{pod}-daily-{stream}'
index 8dd97de..9f1ccec 100644 (file)
         branch: '{stream}'
         gs-pathname: ''
         docker-tag: latest
-    danube: &danube
-        stream: danube
-        branch: 'stable/{stream}'
-        gs-pathname: '/{stream}'
-        docker-tag: 'stable'
 
 #--------------------------------
 # JOB VARIABLES
             installer: fuel
             scenario: os-nosdn-kvm-ha
             <<: *master
-        - zte-pod1:
-            installer: fuel
-            scenario: os-odl_l2-nofeature-ha
-            <<: *danube
-        - zte-pod3:
-            installer: fuel
-            scenario: os-nosdn-nofeature-ha
-            <<: *danube
-        - zte-pod3:
-            installer: fuel
-            scenario: os-nosdn-kvm-ha
-            <<: *danube
 
 #--------------------------------
 # JOB LIST
     name: 'qtip-os-nosdn-kvm-ha-zte-pod3-daily-master-trigger'
     triggers:
         - timed: '0 15 * * *'
-
-- trigger:
-    name: 'qtip-os-odl_l2-nofeature-ha-zte-pod1-daily-danube-trigger'
-    triggers:
-        - timed: '0 7 * * *'
-
-- trigger:
-    name: 'qtip-os-nosdn-kvm-ha-zte-pod3-daily-danube-trigger'
-    triggers:
-        - timed: '0 7 * * *'
-
-- trigger:
-    name: 'qtip-os-nosdn-nofeature-ha-zte-pod3-daily-danube-trigger'
-    triggers:
-        - timed: '30 0 * * *'
diff --git a/jjb/qtip/qtip-weekly-jobs.yml b/jjb/qtip/qtip-weekly-jobs.yml
new file mode 100644 (file)
index 0000000..2170de7
--- /dev/null
@@ -0,0 +1,107 @@
+#################
+# QTIP weekly job
+#################
+- project:
+    name: qtip-weekly
+    project: qtip
+
+#--------------------------------
+# BRANCH ANCHORS
+#--------------------------------
+    danube: &danube
+        stream: danube
+        branch: 'stable/{stream}'
+        gs-pathname: '/{stream}'
+        docker-tag: 'stable'
+
+#--------------------------------
+# JOB VARIABLES
+#--------------------------------
+    pod:
+        - zte-pod1:
+            installer: fuel
+            scenario: os-odl_l2-nofeature-ha
+            <<: *danube
+        - zte-pod3:
+            installer: fuel
+            scenario: os-nosdn-nofeature-ha
+            <<: *danube
+        - zte-pod3:
+            installer: fuel
+            scenario: os-nosdn-kvm-ha
+            <<: *danube
+
+#--------------------------------
+# JOB LIST
+#--------------------------------
+    jobs:
+        - 'qtip-{scenario}-{pod}-weekly-{stream}'
+
+################################
+# job templates
+################################
+- job-template:
+    name: 'qtip-{scenario}-{pod}-weekly-{stream}'
+    disabled: false
+    parameters:
+        - project-parameter:
+            project: '{project}'
+            branch: '{branch}'
+        - '{installer}-defaults'
+        - '{pod}-defaults'
+        - string:
+            name: DEPLOY_SCENARIO
+            default: '{scenario}'
+        - string:
+            name: DOCKER_TAG
+            default: '{docker-tag}'
+            description: 'Tag to pull docker image'
+        - string:
+            name: CI_DEBUG
+            default: 'false'
+            description: "Show debug output information"
+    scm:
+        - git-scm
+    triggers:
+        - 'qtip-{scenario}-{pod}-weekly-{stream}-trigger'
+    builders:
+        - description-setter:
+            description: "POD: $NODE_NAME"
+        - qtip-validate-deploy-weekly
+    publishers:
+        - email:
+            recipients: wu.zhihui1@zte.com.cn, zhang.yujunz@zte.com.cn
+
+################
+# MARCOS
+################
+
+#---------
+# builder
+#---------
+- builder:
+    name: qtip-validate-deploy-weekly
+    builders:
+        - shell:
+            !include-raw: ./helpers/cleanup-deploy.sh
+        - shell:
+            !include-raw: ./helpers/validate-deploy.sh
+
+#---------
+# trigger
+#---------
+
+- trigger:
+    name: 'qtip-os-odl_l2-nofeature-ha-zte-pod1-weekly-danube-trigger'
+    triggers:
+        - timed: '0 7 * * 0'
+
+- trigger:
+    name: 'qtip-os-nosdn-kvm-ha-zte-pod3-weekly-danube-trigger'
+    triggers:
+        - timed: '0 7 * * 0'
+
+- trigger:
+    name: 'qtip-os-nosdn-nofeature-ha-zte-pod3-weekly-danube-trigger'
+    triggers:
+        - timed: '30 0 * * 0'
index ff1d47e..41840d5 100644 (file)
         - description-setter:
             description: "POD: $NODE_NAME"
         - 'yardstick-cleanup'
-        #- 'yardstick-fetch-os-creds'
+        - 'yardstick-fetch-os-creds'
         - 'yardstick-{testsuite}'
         - 'yardstick-store-results'
 
index 1c2abad..cf37ac2 100755 (executable)
@@ -2,9 +2,9 @@
 set -e
 [[ $CI_DEBUG == true ]] && redirect="/dev/stdout" || redirect="/dev/null"
 
-# labconfig is used only for joid
-labconfig=""
+rc_file_vol=""
 sshkey=""
+
 if [[ ${INSTALLER_TYPE} == 'apex' ]]; then
     instack_mac=$(sudo virsh domiflist undercloud | grep default | \
                   grep -Eo "[0-9a-f]+:[0-9a-f]+:[0-9a-f]+:[0-9a-f]+:[0-9a-f]+:[0-9a-f]+")
@@ -15,12 +15,16 @@ if [[ ${INSTALLER_TYPE} == 'apex' ]]; then
         sudo iptables -D FORWARD -o virbr0 -j REJECT --reject-with icmp-port-unreachable
         sudo iptables -D FORWARD -i virbr0 -j REJECT --reject-with icmp-port-unreachable
     fi
-elif [[ ${INSTALLER_TYPE} == 'joid' ]]; then
+fi
+
+if [[ ${INSTALLER_TYPE} == 'joid' ]]; then
     # If production lab then creds may be retrieved dynamically
     # creds are on the jumphost, always in the same folder
-    labconfig="-v $LAB_CONFIG/admin-openrc:/etc/yardstick/openstack.creds"
+    rc_file_vol="-v $LAB_CONFIG/admin-openrc:/etc/yardstick/openstack.creds"
     # If dev lab, credentials may not be the default ones, just provide a path to put them into docker
     # replace the default one by the customized one provided by jenkins config
+else
+    rc_file_vol="-v ${HOME}/opnfv-openrc.sh:/etc/yardstick/openstack.creds"
 fi
 
 # Set iptables rule to allow forwarding return traffic for container
@@ -46,7 +50,7 @@ sudo rm -rf ${dir_result}/*
 map_log_dir="-v ${dir_result}:/tmp/yardstick"
 
 # Run docker
-cmd="sudo docker run ${opts} ${envs} ${labconfig} ${map_log_dir} ${sshkey} opnfv/yardstick:${DOCKER_TAG} \
+cmd="sudo docker run ${opts} ${envs} ${rc_file_vol} ${map_log_dir} ${sshkey} opnfv/yardstick:${DOCKER_TAG} \
     exec_tests.sh ${YARDSTICK_DB_BACKEND} ${YARDSTICK_SCENARIO_SUITE_NAME}"
 echo "Yardstick: Running docker cmd: ${cmd}"
 ${cmd}
index 8656ff9..a7ce521 100644 (file)
       command: "/bin/bash ./scripts/bootstrap-ansible.sh"
       args:
         chdir: "{{OPENSTACK_OSA_PATH}}"
+    - name: install python Crypto module
+      package:
+        name: "{{ python_crypto_package_name }}"
+    - name: install PyYAML
+      pip:
+        name: pyyaml
+        state: present
     - name: generate password token
       command: "python pw-token-gen.py --file {{OPENSTACK_OSA_ETC_PATH}}/user_secrets.yml"
       args:
index d13d080..33f1105 100644 (file)
@@ -9,3 +9,4 @@
 ##############################################################################
 # this is the interface the VM nodes are connected to libvirt network "default"
 interface: "ens3"
+python_crypto_package_name: python-crypto
index 6d03e0f..eae7d12 100644 (file)
@@ -8,3 +8,4 @@
 # http://www.apache.org/licenses/LICENSE-2.0
 ##############################################################################
 # this is placeholder and left blank intentionally to complete later on
+python_crypto_package_name: python-crypto
index 6d03e0f..9674ed2 100644 (file)
@@ -8,3 +8,4 @@
 # http://www.apache.org/licenses/LICENSE-2.0
 ##############################################################################
 # this is placeholder and left blank intentionally to complete later on
+python_crypto_package_name: python-pycrypto
diff --git a/utils/test/testapi/opnfv_testapi/tests/unit/resources/__init__.py b/utils/test/testapi/opnfv_testapi/tests/unit/resources/__init__.py
new file mode 100644 (file)
index 0000000..e69de29
@@ -17,7 +17,7 @@ from opnfv_testapi.resources import models
 from opnfv_testapi.tests.unit import fake_pymongo
 
 config.Config.CONFIG = path.join(path.dirname(__file__),
-                                 '../../../etc/config.ini')
+                                 '../../../../etc/config.ini')
 
 
 class TestBase(testing.AsyncHTTPTestCase):
@@ -42,7 +42,7 @@ class TestBase(testing.AsyncHTTPTestCase):
         from opnfv_testapi.cmd import server
         server.parse_config([
             '--config-file',
-            path.join(path.dirname(__file__), 'common/normal.ini')
+            path.join(path.dirname(__file__), path.pardir, 'common/normal.ini')
         ])
         self.db_patcher = mock.patch('opnfv_testapi.cmd.server.get_db',
                                      self._fake_pymongo)
@@ -12,7 +12,7 @@ import unittest
 from opnfv_testapi.common import message
 from opnfv_testapi.resources import pod_models
 from opnfv_testapi.tests.unit import executor
-from opnfv_testapi.tests.unit import test_base as base
+from opnfv_testapi.tests.unit.resources import test_base as base
 
 
 class TestPodBase(base.TestBase):
@@ -4,7 +4,7 @@ import unittest
 from opnfv_testapi.common import message
 from opnfv_testapi.resources import project_models
 from opnfv_testapi.tests.unit import executor
-from opnfv_testapi.tests.unit import test_base as base
+from opnfv_testapi.tests.unit.resources import test_base as base
 
 
 class TestProjectBase(base.TestBase):
@@ -7,17 +7,17 @@
 # http://www.apache.org/licenses/LICENSE-2.0
 ##############################################################################
 import copy
-from datetime import datetime, timedelta
 import httplib
 import unittest
+from datetime import datetime, timedelta
 
 from opnfv_testapi.common import message
 from opnfv_testapi.resources import pod_models
 from opnfv_testapi.resources import project_models
 from opnfv_testapi.resources import result_models
 from opnfv_testapi.resources import testcase_models
-from opnfv_testapi.tests.unit import test_base as base
 from opnfv_testapi.tests.unit import executor
+from opnfv_testapi.tests.unit.resources import test_base as base
 
 
 class Details(object):
@@ -1,13 +1,13 @@
-from copy import deepcopy
-from datetime import datetime
 import functools
 import httplib
 import json
 import os
+from copy import deepcopy
+from datetime import datetime
 
-from opnfv_testapi.common import message
 import opnfv_testapi.resources.scenario_models as models
-from opnfv_testapi.tests.unit import test_base as base
+from opnfv_testapi.common import message
+from opnfv_testapi.tests.unit.resources import test_base as base
 
 
 class TestScenarioBase(base.TestBase):
@@ -13,8 +13,8 @@ import unittest
 from opnfv_testapi.common import message
 from opnfv_testapi.resources import project_models
 from opnfv_testapi.resources import testcase_models
-from opnfv_testapi.tests.unit import test_base as base
 from opnfv_testapi.tests.unit import executor
+from opnfv_testapi.tests.unit.resources import test_base as base
 
 
 class TestCaseBase(base.TestBase):
@@ -13,7 +13,7 @@ from opnfv_testapi.resources import project_models
 from opnfv_testapi.router import url_mappings
 from opnfv_testapi.tests.unit import executor
 from opnfv_testapi.tests.unit import fake_pymongo
-from opnfv_testapi.tests.unit import test_base as base
+from opnfv_testapi.tests.unit.resources import test_base as base
 
 
 class TestToken(base.TestBase):
@@ -11,7 +11,7 @@ import unittest
 
 from opnfv_testapi.resources import models
 from opnfv_testapi.tests.unit import executor
-from opnfv_testapi.tests.unit import test_base as base
+from opnfv_testapi.tests.unit.resources import test_base as base
 
 
 class TestVersionBase(base.TestBase):