From: Trevor Bramwell Date: Thu, 6 Sep 2018 07:32:30 +0000 (+0000) Subject: Merge "Set NODE_NAME as the description for ansible runs" X-Git-Url: https://gerrit.opnfv.org/gerrit/gitweb?a=commitdiff_plain;h=395cb6b211fd075709a7bd84b3b68de52c362185;hp=7602314a628fa75604de2b15b21e2184fe04dee9;p=releng.git Merge "Set NODE_NAME as the description for ansible runs" --- diff --git a/docs/ci/resources.rst b/docs/ci/resources.rst index 572852c48..b4efe9704 100644 --- a/docs/ci/resources.rst +++ b/docs/ci/resources.rst @@ -13,49 +13,81 @@ verification needs. Each resource must meet a set of criteria in order to be part of CI for an OPNFV release. There are three types of resources: - - Baremetal PODs (PODs) - - Virtual PODs (vPODs) - - Build Servers +- Baremetal PODs (PODs) +- Virtual PODs (vPODs) +- Build Servers + + +.. _ci-resources-baremetal-pods: --------------- Baremetal PODs -------------- -TBD +Baremetal PODs are used to deploy OPNFV on to baremetal hardware through +one of the installer projects. They enable the full range of scenarios +to be deployed and tested. -~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Baremetal PODs Requirements -~~~~~~~~~~~~~~~~~~~~~~~~~~~ +**Requirements** In order of a POD to be considered CI-Ready the following requirements must be met: - #. Pharos Compliant and has a PDF - #. Connected to Jenkins - #. 24/7 Uptime - #. No Development - #. No manual intervention +#. Pharos Compliant and has a PDF +#. Connected to Jenkins +#. 24/7 Uptime +#. No Development +#. No manual intervention ------------- -Virtual PODS +.. include:: tables/ci-baremetal-servers.rst + + +.. _ci-resources-virtual-pods: + +Virtual PODs ------------ -TBD +Virtual PODs are used to deploy OPNFV in a virtualized environment +generally on top of KVM through libvirt. -~~~~~~~~~~~~ -Requirements -~~~~~~~~~~~~ +**Requirements** -TBD +#. Have required virtualization packages installed +#. Meet the Pharos resource specification for virtual PODs +#. Connected to Jenkins +#. 24/7 Uptime + +.. include:: tables/ci-virtual-servers.rst + +.. _ci-resources-build-servers: -------------- Build Servers ------------- -TBD +Build servers are used to build project, run basic verifications (such +as unit tests and linting), and generate documentation. + +**Requirements** + +#. Have required `packages_` installed +#. 24/7 Uptime +#. Connected to Jenkins + +.. include:: tables/ci-build-servers.rst + +.. _dev-resources: + +===================== +Development Resources +===================== + +.. include:: tables/none-ci-servers.rst + +.. _ci-lables: + +=================== +CI Resources Labels +=================== -~~~~~~~~~~~~ -Requirements -~~~~~~~~~~~~ +.. include:: tables/ci-labels.rst -TBD +.. _packages: https://wiki.opnfv.org/display/INF/Continuous+Integration#ContinuousIntegration-BuildServers diff --git a/docs/ci/tables/ci-baremetal-servers.rst b/docs/ci/tables/ci-baremetal-servers.rst new file mode 100644 index 000000000..2efea0ac2 --- /dev/null +++ b/docs/ci/tables/ci-baremetal-servers.rst @@ -0,0 +1,143 @@ +.. This work is licensed under a Creative Commons Attribution 4.0 International License. +.. SPDX-License-Identifier: CC-BY-4.0 +.. (c) Open Platform for NFV Project, Inc. and its contributors + +.. list-table:: CI Servers for Baremetal Deployment + :header-rows: 1 + :stub-columns: 1 + + * - Node + - Usage + - Jumphost OS / Version + - PDF + - IDF + * - `arm-pod6 `_ + - Armband + - Ubuntu 16.04 + - `PDF `__ + - `IDF `__ + * - `ericsson-pod1 `_ + - Fuel + - Ubuntu 16.04 + - `PDF `__ + - `IDF `__ + * - `ericsson-pod2 `_ + - XCI + - Ubuntu 16.04 + - `PDF `__ + - `IDF `__ + * - `flex-pod1 `_ + - Yardstick + - + - PDF + - IDF + * - `flex-pod2 `_ + - Apex + - + - PDF + - IDF + * - `huawei-pod1 `_ + - Compass4NFV + - + - `PDF `__ + - `IDF `__ + * - `huawei-pod2 `_ + - Compass4NFV + - Ubuntu 14.04 + - PDF + - IDF + * - `huawei-pod3 `_ + - Yardstick + - Ubuntu 14.04 + - PDF + - IDF + * - `huawei-pod4 `_ + - Dovetail + - + - PDF + - IDF + * - `huawei-pod6 `_ + - + - Ubuntu 14.04 + - PDF + - IDF + * - `huawei-pod7 `_ + - Dovetail + - Ubuntu 14.04 + - PDF + - IDF + * - `huawei-pod8 `_ + - Compass4NFV + - Ubuntu 16.04 (aarch64) + - PDF + - IDF + * - `huawei-pod12 `_ + - JOID + - Ubuntu 16.04 + - PDF + - IDF + * - `intel-pod10 `_ + - KVMforNFV + - CentOS 7 + - PDF + - IDF + * - `intel-pod11 `_ + - Apex + - + - PDF + - IDF + * - `intel-pod12 `_ + - VSPerf + - CentOS 7 + - PDF + - IDF + * - `intel-pod17 `_ + - Compass4NFV + - + - PDF + - IDF + * - `intel-pod18 `_ + - JOID + - Ubuntu 16.04 + - `PDF `__ + - `IDF `__ + * - `lf-pod1 `_ + - Apex + - CentOS 7 + - `PDF `__ + - IDF + * - `lf-pod2 `_ + - Fuel + - CentOS 7 + - `PDF `__ + - `IDF `__ + * - `unh-pod1 `_ + - Auto + - Ubuntu 16.04 (aarch64) + - PDF + - IDF + * - `zte-pod1 `_ + - + - + - `PDF `__ + - `IDF `__ + * - `zte-pod2 `_ + - + - + - `PDF `__ + - `IDF `__ + * - `zte-pod3 `_ + - + - + - `PDF `__ + - `IDF `__ + * - `zte-pod4 `_ + - + - + - PDF + - IDF + * - `zte-pod9 `_ + - + - + - `PDF `__ + - `IDF `__ diff --git a/docs/ci/tables/ci-build-servers.rst b/docs/ci/tables/ci-build-servers.rst new file mode 100644 index 000000000..b58b0fc9b --- /dev/null +++ b/docs/ci/tables/ci-build-servers.rst @@ -0,0 +1,48 @@ +.. This work is licensed under a Creative Commons Attribution 4.0 International License. +.. SPDX-License-Identifier: CC-BY-4.0 +.. (c) Open Platform for NFV Project, Inc. and its contributors + +.. list-table:: CI Build Servers + :header-rows: 1 + :stub-columns: 1 + + * - Node + - Architecture + - OS + - Contact + * - `arm-build3 `_ + - aarch64 + - CentOS 7.4 + - `Armband ENEA Team`_ + * - `arm-build4 `_ + - aarch64 + - CentOS 7.4 + - `Armband ENEA Team`_ + * - `arm-build5 `_ + - aarch64 + - CentOS 7.4 + - `Armband ENEA Team`_ + * - `arm-build6 `_ + - aarch64 + - CentOS 7.4 + - `Armband ENEA Team`_ + * - `ericsson-build3 `_ + - x86_64 + - Ubuntu 16.04 + - `Dianfeng Du`_ + * - `ericsson-build4 `_ + - x86_64 + - Ubuntu 16.04 + - `Dianfeng Du`_ + * - `lf-build1 `_ + - x86_64 + - CentOS 7.4 + - `Linux Foundation`_ + * - `lf-build2 `_ + - x86_64 + - Ubuntu 16.10 + - `Linux Foundation`_ + +.. _Linux Foundation: helpdesk@opnfv.org +.. _Dianfeng Du: dianfeng.du@ericsson.com +.. _Armband ENEA Team: armband@enea.com diff --git a/docs/ci/tables/ci-labels.rst b/docs/ci/tables/ci-labels.rst new file mode 100644 index 000000000..2865cc22f --- /dev/null +++ b/docs/ci/tables/ci-labels.rst @@ -0,0 +1,13 @@ +.. This work is licensed under a Creative Commons Attribution 4.0 International License. +.. SPDX-License-Identifier: CC-BY-4.0 +.. (c) Open Platform for NFV Project, Inc. and its contributors + +:ci-resource: Resource devoted to CI +:ci-pod: POD devoted to CI +:opnfv-build: Node is for builds - independent of OS +:opnfv-build-centos: Node is for builds needing CentOS +:opnfv-build-centos-arm: Node is for ARM builds on CentOS +:opnfv-build-ubuntu: Node is for builds needing Ubuntu +:opnfv-build-ubuntu-arm: Node is for ARM builds on Ubuntu +:{installer}-baremetal: POD is devoted to {installer} for baremetal deployments +:{installer}-virtual: Server is devoted to {installer} for virtual deployments diff --git a/docs/ci/tables/ci-virtual-servers.rst b/docs/ci/tables/ci-virtual-servers.rst new file mode 100644 index 000000000..e87c46396 --- /dev/null +++ b/docs/ci/tables/ci-virtual-servers.rst @@ -0,0 +1,164 @@ +.. This work is licensed under a Creative Commons Attribution 4.0 International License. +.. SPDX-License-Identifier: CC-BY-4.0 +.. (c) Open Platform for NFV Project, Inc. and its contributors + +.. list-table:: CI Servers for Virtual Deployment + :header-rows: 1 + :stub-columns: 1 + + * - Node + - Architecture + - OS + - Contact + * - `arm-virtual2 `_ + - aarch64 + - Ubuntu 16.04 + - `Armband ENEA Team`_ + * - `arm-virtual3 `_ + - aarch64 + - Ubuntu 16.04 + - `Xuan Jia` + * - `arm-virtual4 `_ + - aarch64 + - Ubuntu 16.04 + - `Xuan Jia` + * - `ericsson-virtual-pod1bl01 `_ + - x86_64 + - CentOS 7 + - + * - `ericsson-virtual1 `_ + - x86_64 + - Ubuntu 16.04 + - + * - `ericsson-virtual2 `_ + - x86_64 + - Ubuntu 16.04 + - + * - `ericsson-virtual3 `_ + - x86_64 + - Ubuntu 16.04 + - + * - `ericsson-virtual4 `_ + - x86_64 + - Ubuntu 16.04 + - + * - `ericsson-virtual5 `_ + - x86_64 + - Ubuntu 16.04 + - + * - `huawei-virtual1 `_ + - x86_64 + - Ubuntu 14.04 + - + * - `huawei-virtual2 `_ + - x86_64 + - Ubuntu 14.04 + - + * - `huawei-virtual3 `_ + - x86_64 + - Ubuntu 14.04 + - + * - `huawei-virtual4 `_ + - x86_64 + - Ubuntu 14.04 + - + * - `huawei-virtual5 `_ + - x86_64 + - + - + * - `huawei-virtual6 `_ + - x86_64 + - Ubuntu 16.04 + - + * - `huawei-virtual7 `_ + - x86_64 + - Ubuntu 14.04 + - + * - `huawei-virtual8 `_ + - x86_64 + - Ubuntu 14.04 + - + * - `huawei-virtual9 `_ + - x86_64 + - Ubuntu 14.04 + - + * - `intel-virtual3 `_ + - x86_64 + - + - + * - `intel-virtual11 `_ + - x86_64 + - + - + * - `intel-virtual12 `_ + - x86_64 + - + - + * - `intel-virtual13 `_ + - x86_64 + - + - + * - `intel-virtual14 `_ + - x86_64 + - + - + * - `intel-virtual15 `_ + - x86_64 + - + - + * - `intel-virtual16 `_ + - x86_64 + - + - + * - `lf-virtual1 `_ + - x86_64 + - Ubuntu 14.04 + - `Linux Foundation`_ + * - `lf-virtual2 `_ + - x86_64 + - CentOS 7 + - `Linux Foundation`_ + * - `lf-virtual3 `_ + - x86_64 + - CentOS 7 + - `Linux Foundation`_ + * - `ool-virtual1 `_ + - x86_64 + - + - + * - `ool-virtual2 `_ + - x86_64 + - + - + * - `ool-virtual3 `_ + - x86_64 + - + - + * - `zte-virtual1 `_ + - x86_64 + - + - + * - `zte-virtual2 `_ + - x86_64 + - + - + * - `zte-virtual3 `_ + - x86_64 + - + - + * - `zte-virtual4 `_ + - x86_64 + - + - + * - `zte-virtual5 `_ + - x86_64 + - + - + * - `zte-virtual6 `_ + - x86_64 + - + - + +.. _Armband ENEA Team: armband@enea.com +.. _Linux Foundation: helpdesk@opnfv.org +.. _Xuan Jia: jason.jiaxuan@gmail.com' diff --git a/docs/ci/tables/none-ci-servers.rst b/docs/ci/tables/none-ci-servers.rst new file mode 100644 index 000000000..c4f403a64 --- /dev/null +++ b/docs/ci/tables/none-ci-servers.rst @@ -0,0 +1,58 @@ +.. This work is licensed under a Creative Commons Attribution 4.0 International License. +.. SPDX-License-Identifier: CC-BY-4.0 +.. (c) Open Platform for NFV Project, Inc. and its contributors + +.. list-table:: Baremetal Development Servers + :header-rows: 1 + :stub-columns: 1 + + * - Node + - Usage + - Jumphost OS / Version + - PDF + - IDF + * - `arm-pod5 `_ + - Armband + - Ubuntu 16.04 + - `PDF `__ + - `IDF `__ + * - cacti-pod1 + - + - + - + - + * - cengn-pod1 + - + - + - + - + * - itri-pod1 + - + - + - + - + * - lf-pod4 + - + - + - + - + * - lf-pod5 + - + - + - + - + * - nokia-pod1 + - + - + - + - + * - ool-pod1 + - + - + - + - + * - bii-pod1 + - + - + - + - diff --git a/jjb/3rd_party_ci/detect-snapshot.sh b/jjb/3rd_party_ci/detect-snapshot.sh new file mode 100755 index 000000000..77788aa2c --- /dev/null +++ b/jjb/3rd_party_ci/detect-snapshot.sh @@ -0,0 +1,30 @@ +#!/bin/bash +set -o errexit +set -o nounset +set -o pipefail + +echo "Detecting requested OpenStack branch and topology type in gerrit comment" +parsed_comment=$(echo $GERRIT_EVENT_COMMENT_TEXT | sed -n 's/^.*check-opnfv\s*//p') +parsed_comment=$(echo $parsed_comment | sed -n 's/\s*$//p') +if [ ! -z "$parsed_comment" ]; then + if echo $parsed_comment | grep -E '^[a-z]+-(no)?ha'; then + os_version=${parsed_comment%%"-"*} + topo=${parsed_comment#*"-"} + echo "OS version detected in gerrit comment: ${os_version}" + echo "Topology type detected in gerrit comment: ${topo}" + else + echo "Invalid format given for scenario in gerrit comment: ${parsed_comment}...aborting" + exit 1 + fi +else + echo "No scenario given in gerrit comment, will use default (master OpenStack, noha)" + os_version='master' + topo='noha' +fi + +echo "Writing variables to file" +cat > detected_snapshot << EOI +OS_VERSION=$os_version +TOPOLOGY=$topo +SNAP_CACHE=$HOME/snap_cache/$os_version/$topo +EOI diff --git a/jjb/3rd_party_ci/download-netvirt-artifact.sh b/jjb/3rd_party_ci/download-netvirt-artifact.sh index b1f977a6f..ac7f76ccd 100755 --- a/jjb/3rd_party_ci/download-netvirt-artifact.sh +++ b/jjb/3rd_party_ci/download-netvirt-artifact.sh @@ -8,24 +8,23 @@ if [ "$ODL_BRANCH" != 'master' ]; then DIST=$(echo ${ODL_BRANCH} | sed -rn 's#([a-zA-Z]+)/([a-zA-Z]+)#\2#p') ODL_BRANCH=$(echo ${ODL_BRANCH} | sed -rn 's#([a-zA-Z]+)/([a-zA-Z]+)#\1%2F\2#p') else - DIST='fluorine' + DIST='neon' fi echo "ODL Distribution is ${DIST}" ODL_ZIP="karaf-SNAPSHOT.zip" -CHANGE_DETAILS_URL="https://git.opendaylight.org/gerrit/changes/netvirt~${ODL_BRANCH}~${GERRIT_CHANGE_ID}/detail" +CHANGE_DETAILS_URL="https://git.opendaylight.org/gerrit/changes/${GERRIT_PROJECT}~${ODL_BRANCH}~${GERRIT_CHANGE_ID}/detail" # due to limitation with the Jenkins Gerrit Trigger, we need to use Gerrit REST API to get the change details -ODL_BUILD_JOB_NUM=$(curl --fail ${CHANGE_DETAILS_URL} | grep -Eo "netvirt-distribution-check-${DIST}/[0-9]+" | tail -1 | grep -Eo [0-9]+) -DISTRO_CHECK_CONSOLE_LOG="https://logs.opendaylight.org/releng/vex-yul-odl-jenkins-1/netvirt-distribution-check-${DIST}/${ODL_BUILD_JOB_NUM}/console.log.gz" +ODL_BUILD_JOB_NUM=$(curl --fail ${CHANGE_DETAILS_URL} | grep -Eo "${GERRIT_PROJECT}-distribution-check-${DIST}/[0-9]+" | tail -1 | grep -Eo [0-9]+) +DISTRO_CHECK_CONSOLE_LOG="https://logs.opendaylight.org/releng/vex-yul-odl-jenkins-1/${GERRIT_PROJECT}-distribution-check-${DIST}/${ODL_BUILD_JOB_NUM}/console.log.gz" NETVIRT_ARTIFACT_URL=$(curl --fail --compressed ${DISTRO_CHECK_CONSOLE_LOG} | grep 'BUNDLE_URL' | cut -d = -f 2) echo -e "URL to artifact is\n\t$NETVIRT_ARTIFACT_URL" echo "Downloading the artifact. This could take time..." -wget -q -O $ODL_ZIP $NETVIRT_ARTIFACT_URL -if [[ $? -ne 0 ]]; then +if ! wget -q -O $ODL_ZIP $NETVIRT_ARTIFACT_URL; then echo "The artifact does not exist! Probably removed due to ODL Jenkins artifact retention policy." - echo "Rerun netvirt-patch-test-current-carbon to get artifact rebuilt." + echo "Use 'recheck' on the gerrit to get artifact rebuilt." exit 1 fi diff --git a/jjb/3rd_party_ci/install-netvirt.sh b/jjb/3rd_party_ci/install-netvirt.sh index 07bbe7739..232d60e5c 100755 --- a/jjb/3rd_party_ci/install-netvirt.sh +++ b/jjb/3rd_party_ci/install-netvirt.sh @@ -3,7 +3,7 @@ set -o errexit set -o nounset set -o pipefail -SNAP_CACHE=$HOME/snap_cache +SNAP_CACHE=$HOME/snap_cache/$OS_VERSION/$TOPOLOGY # clone opnfv sdnvpn repo git clone https://gerrit.opnfv.org/gerrit/p/sdnvpn.git $WORKSPACE/sdnvpn diff --git a/jjb/3rd_party_ci/odl-netvirt.yaml b/jjb/3rd_party_ci/odl-netvirt.yaml index c78de9bdf..b810a2998 100644 --- a/jjb/3rd_party_ci/odl-netvirt.yaml +++ b/jjb/3rd_party_ci/odl-netvirt.yaml @@ -95,7 +95,7 @@ comment-contains-value: 'check-opnfv' projects: - project-compare-type: 'ANT' - project-pattern: '{project}' + project-pattern: '*' branches: - branch-compare-type: 'ANT' branch-pattern: '**/{branch}' @@ -104,6 +104,9 @@ builders: - description-setter: description: "Built on $NODE_NAME" + - detect-opnfv-snapshot + - inject: + properties-file: detected_snapshot - multijob: name: create-apex-vms condition: SUCCESSFUL @@ -120,6 +123,8 @@ NETVIRT_ARTIFACT=$NETVIRT_ARTIFACT APEX_ENV_NUMBER=$APEX_ENV_NUMBER GERRIT_EVENT_COMMENT_TEXT=$GERRIT_EVENT_COMMENT_TEXT + TOPOLOGY=$TOPOLOGY + OS_VERSION=$OS_VERSION node-parameters: true kill-phase-on: FAILURE abort-all-job: true @@ -130,30 +135,56 @@ - name: 'odl-netvirt-verify-virtual-install-netvirt-{stream}' current-parameters: false predefined-parameters: | - ODL_BRANCH={branch} + ODL_BRANCH=$BRANCH BRANCH=$BRANCH GERRIT_REFSPEC=$GERRIT_REFSPEC GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER GERRIT_CHANGE_ID=$GERRIT_CHANGE_ID GERRIT_PATCHSET_NUMBER=$GERRIT_PATCHSET_NUMBER GERRIT_PATCHSET_REVISION=$GERRIT_PATCHSET_REVISION + GERRIT_PROJECT=$GERRIT_PROJECT NETVIRT_ARTIFACT=$NETVIRT_ARTIFACT + TOPOLOGY=$TOPOLOGY + OS_VERSION=$OS_VERSION node-parameters: true kill-phase-on: FAILURE abort-all-job: true - multijob: - name: functest - condition: SUCCESSFUL + name: csit + condition: ALWAYS + projects: + - name: cperf-apex-csit-master + predefined-parameters: | + ODL_BRANCH=$BRANCH + RC_FILE_PATH=$SNAP_CACHE/overcloudrc + NODE_FILE_PATH=$SNAP_CACHE/node.yaml + SSH_KEY_PATH=$SNAP_CACHE/id_rsa + ODL_CONTAINERIZED=false + OS_VERSION=$OS_VERSION + node-parameters: true + kill-phase-on: FAILURE + abort-all-job: false + - multijob: + name: csit-collect-logs + condition: ALWAYS projects: - - name: 'functest-netvirt-virtual-suite-master' + - name: cperf-upload-logs-csit predefined-parameters: | - DEPLOY_SCENARIO=os-odl-nofeature-ha - FUNCTEST_MODE=testcase - FUNCTEST_SUITE_NAME=tempest_smoke - RC_FILE_PATH=$HOME/cloner-info/overcloudrc + ODL_BRANCH=$BRANCH + OS_VERSION=$OS_VERSION node-parameters: true kill-phase-on: FAILURE abort-all-job: false + - multijob: + name: apex-fetch-logs + condition: ALWAYS + projects: + - name: 'apex-fetch-logs-{stream}' + current-parameters: false + node-parameters: true + kill-phase-on: NEVER + abort-all-job: true + git-revision: false - multijob: name: postprocess condition: ALWAYS @@ -248,3 +279,9 @@ builders: - shell: !include-raw: ./postprocess-netvirt.sh + +- builder: + name: 'detect-opnfv-snapshot' + builders: + - shell: + !include-raw: ./detect-snapshot.sh diff --git a/jjb/apex/apex-deploy.sh b/jjb/apex/apex-deploy.sh index f5c6ab108..441bf9059 100755 --- a/jjb/apex/apex-deploy.sh +++ b/jjb/apex/apex-deploy.sh @@ -4,6 +4,8 @@ set -o nounset set -o pipefail IPV6_FLAG=False +ALLINONE_FLAG=False +CSIT_ENV_FLAG=False # log info to console echo "Starting the Apex deployment." @@ -99,12 +101,26 @@ if [ "$OPNFV_CLEAN" == 'yes' ]; then sudo ${CLEAN_CMD} ${clean_opts} fi +# These are add-ons to regular scenarios where you can do like +# os-nosdn-nofeature-noha-ipv6, or os-nosdn-nofeature-noha-allinone if echo ${DEPLOY_SCENARIO} | grep ipv6; then IPV6_FLAG=True DEPLOY_SCENARIO=$(echo ${DEPLOY_SCENARIO} | sed 's/-ipv6//') echo "INFO: IPV6 Enabled" fi +if echo ${DEPLOY_SCENARIO} | grep allinone; then + ALLINONE_FLAG=True + DEPLOY_SCENARIO=$(echo ${DEPLOY_SCENARIO} | sed 's/-allinone//') + echo "INFO: All in one deployment detected" +fi + +if echo ${DEPLOY_SCENARIO} | grep csit; then + CSIT_ENV_FLAG=True + DEPLOY_SCENARIO=$(echo ${DEPLOY_SCENARIO} | sed 's/-csit//') + echo "INFO: CSIT env requested in deploy scenario" +fi + echo "Deploy Scenario set to ${DEPLOY_SCENARIO}" DEPLOY_FILE="${DEPLOY_SETTINGS_DIR}/${DEPLOY_SCENARIO}.yaml" @@ -118,8 +134,19 @@ if [[ "$JOB_NAME" =~ "virtual" ]]; then if [[ "${DEPLOY_SCENARIO}" =~ fdio|ovs ]]; then DEPLOY_CMD="${DEPLOY_CMD} --virtual-default-ram 12 --virtual-compute-ram 7" fi - if [[ "$PROMOTE" == "True" ]]; then - DEPLOY_CMD="${DEPLOY_CMD} --virtual-computes 2 -e csit-environment.yaml" + if [[ "$ALLINONE_FLAG" == "True" ]]; then + DEPLOY_CMD="${DEPLOY_CMD} --virtual-computes 0" + elif [[ "$PROMOTE" == "True" ]]; then + DEPLOY_CMD="${DEPLOY_CMD} --virtual-computes 2" + fi + + if [[ "$PROMOTE" == "True" || "$CSIT_ENV_FLAG" == "True" ]]; then + if [[ "$DEPLOY_SCENARIO" =~ "queens" ]]; then + CSIT_ENV="csit-queens-environment.yaml" + else + CSIT_ENV="csit-environment.yaml" + fi + DEPLOY_CMD="${DEPLOY_CMD} -e ${CSIT_ENV}" fi else # settings for bare metal deployment diff --git a/jjb/apex/apex-fetch-snap-info.sh b/jjb/apex/apex-fetch-snap-info.sh new file mode 100755 index 000000000..2cfaa59d7 --- /dev/null +++ b/jjb/apex/apex-fetch-snap-info.sh @@ -0,0 +1,46 @@ +#!/usr/bin/env bash + +############################################################################## +# Copyright (c) 2018 Tim Rozet (Red Hat) and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +set -o errexit +set -o nounset +set -o pipefail + +echo "Fetching overcloudrc, ssh key, and node.yaml from deployment..." + +SSH_OPTIONS=(-o StrictHostKeyChecking=no -o GlobalKnownHostsFile=/dev/null -o UserKnownHostsFile=/dev/null -o LogLevel=error) + +tmp_dir=/tmp/csit +rm -rf ${tmp_dir} +mkdir -p ${tmp_dir} + +# TODO(trozet) remove this after fix goes in for tripleo_inspector to copy these +pushd ${tmp_dir} > /dev/null +echo "Copying overcloudrc and ssh key from Undercloud..." +# Store overcloudrc +UNDERCLOUD=$(sudo virsh domifaddr undercloud | grep -Eo '[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+') +sudo scp ${SSH_OPTIONS[@]} stack@${UNDERCLOUD}:overcloudrc ./ +# Copy out ssh key of stack from undercloud +sudo scp ${SSH_OPTIONS[@]} stack@${UNDERCLOUD}:.ssh/id_rsa ./ +sudo chmod 0600 id_rsa +popd > /dev/null + +echo "Gathering introspection information" +git clone https://gerrit.opnfv.org/gerrit/sdnvpn.git +pushd sdnvpn/odl-pipeline/lib > /dev/null +sudo ./tripleo_introspector.sh --out-file ${tmp_dir}/node.yaml +popd > /dev/null +sudo rm -rf sdnvpn + +sudo chown jenkins-ci:jenkins-ci ${tmp_dir}/* + +ls -lrt ${tmp_dir} + +echo "Fetch complete" diff --git a/jjb/apex/apex-jjb-renderer.py b/jjb/apex/apex-jjb-renderer.py index 0da47b518..58dc4fff9 100644 --- a/jjb/apex/apex-jjb-renderer.py +++ b/jjb/apex/apex-jjb-renderer.py @@ -20,7 +20,8 @@ def render_jjb(): gspathname = dict() branch = dict() build_slave = dict() - env = Environment(loader=FileSystemLoader('./'), autoescape=True) + env = Environment(loader=FileSystemLoader('./'), autoescape=True, + keep_trailing_newline=True) with open('scenarios.yaml.hidden') as _: scenarios = yaml.safe_load(_) @@ -45,5 +46,6 @@ def render_jjb(): with open('./apex.yaml', 'w') as fh: fh.write(output) + if __name__ == "__main__": render_jjb() diff --git a/jjb/apex/apex-snapshot-create.sh b/jjb/apex/apex-snapshot-create.sh index b73931470..f91720883 100644 --- a/jjb/apex/apex-snapshot-create.sh +++ b/jjb/apex/apex-snapshot-create.sh @@ -27,22 +27,8 @@ echo tmp_dir=$(pwd)/.tmp mkdir -p ${tmp_dir} -# TODO(trozet) remove this after fix goes in for tripleo_inspector to copy these -pushd ${tmp_dir} > /dev/null -echo "Copying overcloudrc and ssh key from Undercloud..." -# Store overcloudrc -UNDERCLOUD=$(sudo virsh domifaddr undercloud | grep -Eo '[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+') -sudo scp ${SSH_OPTIONS[@]} stack@${UNDERCLOUD}:overcloudrc ./ -# Copy out ssh key of stack from undercloud -sudo scp ${SSH_OPTIONS[@]} stack@${UNDERCLOUD}:.ssh/id_rsa ./ -popd > /dev/null - -echo "Gathering introspection information" -git clone https://gerrit.opnfv.org/gerrit/sdnvpn.git -pushd sdnvpn/odl-pipeline/lib > /dev/null -sudo ./tripleo_introspector.sh --out-file ${tmp_dir}/node.yaml -popd > /dev/null -sudo rm -rf sdnvpn +# info should have already been collected in apex-fetch-snap-info so copy it +cp -r /tmp/csit/* ${tmp_dir}/ echo "Shutting down nodes" # Shut down nodes diff --git a/jjb/apex/apex-snapshot-deploy.sh b/jjb/apex/apex-snapshot-deploy.sh index 0a475062c..bd023f558 100644 --- a/jjb/apex/apex-snapshot-deploy.sh +++ b/jjb/apex/apex-snapshot-deploy.sh @@ -25,43 +25,26 @@ pushd ci > /dev/null sudo opnfv-clean popd > /dev/null -echo "Detecting requested OpenStack branch and topology type in gerrit comment" -parsed_comment=$(echo $GERRIT_EVENT_COMMENT_TEXT | sed -n 's/^opnfv-check\s*//p') -parsed_comment=$(echo $parsed_comment | sed -n 's/\s*$//p') -if [ ! -z "$parsed_comment" ]; then - if echo $parsed_comment | grep -E '^[a-z]+-(no)?ha'; then - IFS='-' read -r -a array <<< "$parsed_comment" - os_version=${array[0]} - topo=${array[1]} - echo "OS version detected in gerrit comment: ${os_version}" - echo "Topology type detected in gerrit comment: ${topo}" - else - echo "Invalid format given for scenario in gerrit comment: ${parsed_comment}...aborting" - exit 1 - fi -else - echo "No scenario given in gerrit comment, will use default (master OpenStack, noha)" - os_version='master' - topo='noha' -fi - -full_snap_url=http://$GS_URL/${os_version}/${topo} +full_snap_url="gs://${GS_URL}/${OS_VERSION}/${TOPOLOGY}" echo "Downloading latest snapshot properties file" -if ! wget -O $WORKSPACE/opnfv.properties ${full_snap_url}/snapshot.properties; then +if ! gsutil cp ${full_snap_url}/snapshot.properties $WORKSPACE/opnfv.properties; then echo "ERROR: Unable to find snapshot.properties at ${full_snap_url}...exiting" exit 1 fi +echo "Properties contents:" +cat ${WORKSPACE}/opnfv.properties + # find latest check sum -latest_snap_checksum=$(cat opnfv.properties | grep OPNFV_SNAP_SHA512SUM | awk -F "=" '{print $2}') +latest_snap_checksum=$(cat ${WORKSPACE}/opnfv.properties | grep OPNFV_SNAP_SHA512SUM | awk -F "=" '{print $2}') if [ -z "$latest_snap_checksum" ]; then echo "ERROR: checksum of latest snapshot from snapshot.properties is null!" exit 1 fi local_snap_checksum="" -SNAP_CACHE=${SNAP_CACHE}/${os_version}/${topo} +SNAP_CACHE=${SNAP_CACHE}/${OS_VERSION}/${TOPOLOGY} # check snap cache directory exists # if snapshot cache exists, find the checksum @@ -69,6 +52,7 @@ if [ -d "$SNAP_CACHE" ]; then latest_snap=$(ls ${SNAP_CACHE} | grep tar.gz | tail -n 1) if [ -n "$latest_snap" ]; then local_snap_checksum=$(sha512sum ${SNAP_CACHE}/${latest_snap} | cut -d' ' -f1) + echo "Local snap checksum is: ${local_snap_checksum}" fi else mkdir -p ${SNAP_CACHE} @@ -84,7 +68,7 @@ if [ "$local_snap_checksum" != "$latest_snap_checksum" ]; then echo "INFO: SHA mismatch, will download latest snapshot" # wipe cache rm -rf ${SNAP_CACHE}/* - wget --directory-prefix=${SNAP_CACHE}/ ${snap_url} + gsutil cp "gs://${snap_url}" ${SNAP_CACHE}/ snap_tar=$(basename ${snap_url}) else snap_tar=${latest_snap} @@ -156,10 +140,6 @@ for node_def in ${virsh_vm_defs}; do sudo virsh define ${node_def} node=$(echo ${node_def} | awk -F '.' '{print $1}') sudo cp -f ${node}.qcow2 /var/lib/libvirt/images/ - # FIXME (trozet) install java on each disk image as required to upgrade ODL - # should be added to Apex as part of the deployment. Remove this after that - # is complete - sudo LIBGUESTFS_BACKEND=direct virt-customize --install java-1.8.0-openjdk -a /var/lib/libvirt/images/${node}.qcow2 sudo virsh start ${node} echo "Node: ${node} started" done @@ -177,7 +157,7 @@ while [ "$counter" -le 10 ]; do echo "Checking if OpenStack is up" if nc -z ${admin_controller_ip} 9696 > /dev/null; then echo "Overcloud Neutron is up...Checking if OpenDaylight NetVirt is up..." - if curl --fail --silent -u admin:admin ${netvirt_url} > /dev/null; then + if curl --fail --silent -u admin:${SDN_CONTROLLER_PASSWORD} ${netvirt_url} > /dev/null; then echo "OpenDaylight is up. Overcloud deployment complete" exit 0 else diff --git a/jjb/apex/apex-verify-jobs.yaml b/jjb/apex/apex-verify-jobs.yaml index 819e17b76..f0f99fc78 100644 --- a/jjb/apex/apex-verify-jobs.yaml +++ b/jjb/apex/apex-verify-jobs.yaml @@ -10,7 +10,7 @@ - master: &master branch: '{stream}' gs-pathname: '' - verify-scenario: 'os-nosdn-nofeature-ha' + verify-scenario: 'os-nosdn-nofeature-noha' disabled: false - fraser: &fraser branch: 'stable/{stream}' @@ -194,7 +194,11 @@ max-per-node: 3 max-total: 10 option: 'project' - + - build-blocker: + use-build-blocker: true + block-level: 'NODE' + blocking-jobs: + - 'apex-.*-promote.*' builders: - description-setter: description: "Built on $NODE_NAME" @@ -245,7 +249,7 @@ GERRIT_CHANGE_COMMIT_MESSAGE=$GERRIT_CHANGE_COMMIT_MESSAGE FUNCTEST_SUITE_NAME=tempest_smoke FUNCTEST_MODE=testcase - node-parameters: false + node-parameters: true kill-phase-on: FAILURE abort-all-job: true git-revision: true @@ -369,7 +373,7 @@ GERRIT_CHANGE_COMMIT_MESSAGE=$GERRIT_CHANGE_COMMIT_MESSAGE node-parameters: true kill-phase-on: NEVER - enable-condition: "def m = '$DEPLOY_SCENARIO' != k8s-nosdn-nofeature-noha" + enable-condition: "def m = '$DEPLOY_SCENARIO' != 'k8s-nosdn-nofeature-noha'" abort-all-job: true git-revision: false - multijob: diff --git a/jjb/apex/apex.yaml b/jjb/apex/apex.yaml index 45f6fd8d2..82f766e20 100644 --- a/jjb/apex/apex.yaml +++ b/jjb/apex/apex.yaml @@ -14,6 +14,7 @@ - 'apex-testsuite-{scenario}-baremetal-{scenario_stream}' - 'apex-upload-snapshot' - 'apex-create-snapshot' + - 'apex-fetch-snap-info' - 'apex-flex-daily-os-nosdn-nofeature-ha-{stream}' - 'apex-dovetail-daily-os-nosdn-nofeature-ha-baremetal-danube' # stream: branch with - in place of / (eg. stable-arno) @@ -25,7 +26,7 @@ build-slave: 'apex-build-master' virtual-slave: 'apex-virtual-master' baremetal-slave: 'apex-baremetal-master' - verify-scenario: 'os-nosdn-nofeature-ha' + verify-scenario: 'os-nosdn-nofeature-noha' scenario_stream: 'master' disable_daily: false disable_promote: false @@ -37,7 +38,7 @@ baremetal-slave: 'apex-baremetal-master' verify-scenario: 'os-nosdn-nofeature-ha' scenario_stream: 'fraser' - disable_daily: true + disable_daily: false disable_promote: true - euphrates: &euphrates branch: 'stable/euphrates' @@ -62,36 +63,10 @@ disable_promote: true scenario: - - 'os-nosdn-nofeature-noha': - <<: *fraser - 'os-nosdn-nofeature-ha': <<: *fraser - - 'os-odl-nofeature-ha': - <<: *fraser - - 'os-odl-nofeature-noha': - <<: *fraser - 'os-odl-bgpvpn-ha': <<: *fraser - - 'os-ovn-nofeature-noha': - <<: *fraser - - 'os-nosdn-fdio-noha': - <<: *fraser - - 'os-nosdn-fdio-ha': - <<: *fraser - - 'os-nosdn-bar-ha': - <<: *fraser - - 'os-nosdn-bar-noha': - <<: *fraser - - 'os-nosdn-ovs_dpdk-noha': - <<: *fraser - - 'os-nosdn-ovs_dpdk-ha': - <<: *fraser - - 'os-odl-sfc-noha': - <<: *fraser - - 'os-odl-sfc-ha': - <<: *fraser - - 'os-nosdn-calipso-noha': - <<: *fraser - 'os-nosdn-nofeature-noha': <<: *danube - 'os-nosdn-nofeature-ha': @@ -148,6 +123,8 @@ <<: *master - 'os-nosdn-nofeature-ha': <<: *master + - 'os-nosdn-nofeature-ha-ipv6': + <<: *master - 'os-odl-nofeature-noha': <<: *master - 'os-odl-nofeature-ha': @@ -156,10 +133,30 @@ <<: *master - 'os-nosdn-queens-ha': <<: *master + - 'os-nosdn-queens-ha-ipv6': + <<: *master - 'os-odl-queens-noha': <<: *master - 'os-odl-queens-ha': <<: *master + - 'k8s-nosdn-nofeature-noha': + <<: *master + - 'os-odl-bgpvpn-ha': + <<: *master + - 'os-odl-bgpvpn-noha': + <<: *master + - 'os-odl-bgpvpn_queens-ha': + <<: *master + - 'os-odl-bgpvpn_queens-noha': + <<: *master + - 'os-odl-sfc-ha': + <<: *master + - 'os-odl-sfc-noha': + <<: *master + - 'os-odl-sfc_queens-ha': + <<: *master + - 'os-odl-sfc_queens-noha': + <<: *master - 'os-nosdn-nofeature-noha': <<: *euphrates - 'os-nosdn-nofeature-ha': @@ -204,14 +201,18 @@ os_version: - 'pike': os_scenario: 'pike' + odl_branch: 'stable/carbon' - 'queens': os_scenario: 'queens' + odl_branch: 'stable/oxygen' - 'master': os_scenario: 'nofeature' + odl_branch: 'stable/oxygen' topology: - 'noha' - 'ha' + - 'noha-allinone' # Fetch Logs Job - job-template: @@ -543,6 +544,24 @@ kill-phase-on: NEVER abort-all-job: true git-revision: false + - shell: | + if echo $DEPLOY_SCENARIO | grep queens &> /dev/null; then + echo 'REPO=opnfv' > functest_properties + else + echo 'REPO=ollivier' > functest_properties + fi + features=$(echo $DEPLOY_SCENARIO | sed -r -n 's/os-.+-(.+)-(noha|ha)/\1/p') + if [ "$features" == 'queens' ]; then + functest_scenario=$(echo $DEPLOY_SCENARIO | sed -r -n 's/(os-.+?)-queens-(noha|ha)/\1-nofeature-\2/p') + elif [[ "$features" =~ 'queens' ]]; then + functest_scenario=$(echo $DEPLOY_SCENARIO | sed -r -n 's/(os-.+?)-(.+)_queens-(noha|ha)/\1-\2-\3/p') + else + functest_scenario=$DEPLOY_SCENARIO + fi + echo "DEPLOY_SCENARIO=$functest_scenario" >> functest_properties + - inject: + properties-file: functest_properties + override-build-parameters: true - multijob: name: 'OPNFV Test Suite' condition: ALWAYS @@ -639,7 +658,7 @@ name: Dovetail-proposed_tests condition: ALWAYS projects: - - name: 'dovetail-apex-baremetal-proposed_tests-{scenario_stream}' + - name: 'dovetail-apex-baremetal-default-mandatory-{scenario_stream}' node-parameters: true current-parameters: false predefined-parameters: @@ -652,7 +671,7 @@ name: Dovetail-default condition: ALWAYS projects: - - name: 'dovetail-apex-baremetal-default-{scenario_stream}' + - name: 'dovetail-apex-baremetal-default-optional-{scenario_stream}' node-parameters: true current-parameters: false predefined-parameters: @@ -753,6 +772,19 @@ git-revision: true - apex-builder-{stream} +# snapshot info fetch +- job-template: + name: 'apex-fetch-snap-info' + + disabled: false + + parameters: + - '{project}-defaults' + + builders: + - shell: + !include-raw-escape: ./apex-fetch-snap-info.sh + # snapshot create - job-template: name: 'apex-create-snapshot' @@ -814,6 +846,14 @@ name: GS_URL default: $GS_BASE{gs-pathname}/{os_version}/{topology} description: "User for overriding GS_URL from apex params" + - string: + name: OS_VERSION + default: '{os_version}' + description: OpenStack version short name + - string: + name: ODL_BRANCH + default: '{odl_branch}' + description: ODL branch being used properties: - build-blocker: @@ -824,6 +864,7 @@ - 'apex-runner.*' - 'apex-daily.*' - 'apex-csit-promote.*' + - 'odl-netvirt.*' - throttle: max-per-node: 1 max-total: 10 @@ -834,10 +875,10 @@ builders: - multijob: - name: apex-virtual-deploy-test + name: apex-virtual-deploy condition: SUCCESSFUL projects: - - name: 'apex-virtual-{stream}' + - name: 'apex-deploy-virtual-{stream}' current-parameters: true predefined-parameters: | DEPLOY_SCENARIO=os-odl-{os_scenario}-{topology} @@ -846,12 +887,54 @@ GERRIT_REFSPEC=$GERRIT_REFSPEC GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER GERRIT_CHANGE_COMMIT_MESSAGE=$GERRIT_CHANGE_COMMIT_MESSAGE - FUNCTEST_SUITE_NAME=tempest_smoke - FUNCTEST_MODE=testcase node-parameters: true kill-phase-on: FAILURE abort-all-job: true git-revision: true + - multijob: + name: fetch snapshot info + condition: SUCCESSFUL + projects: + - name: 'apex-fetch-snap-info' + current-parameters: true + node-parameters: true + kill-phase-on: FAILURE + abort-all-job: true + git-revision: false + - multijob: + name: csit + condition: SUCCESSFUL + execution-type: SEQUENTIALLY + projects: + - name: cperf-apex-csit-master + predefined-parameters: | + ODL_BRANCH=$ODL_BRANCH + RC_FILE_PATH=/tmp/csit/overcloudrc + NODE_FILE_PATH=/tmp/csit/node.yaml + SSH_KEY_PATH=/tmp/csit/id_rsa + ODL_CONTAINERIZED=true + OS_VERSION=$OS_VERSION + node-parameters: true + kill-phase-on: NEVER + abort-all-job: false + - name: cperf-upload-logs-csit + predefined-parameters: | + ODL_BRANCH=$ODL_BRANCH + OS_VERSION=$OS_VERSION + node-parameters: true + kill-phase-on: FAILURE + abort-all-job: false + - name: 'apex-fetch-logs-{stream}' + current-parameters: false + predefined-parameters: | + GERRIT_BRANCH=$GERRIT_BRANCH + GERRIT_REFSPEC=$GERRIT_REFSPEC + GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER + GERRIT_CHANGE_COMMIT_MESSAGE=$GERRIT_CHANGE_COMMIT_MESSAGE + node-parameters: true + kill-phase-on: NEVER + abort-all-job: true + git-revision: false - multijob: name: create snapshot condition: SUCCESSFUL @@ -1172,14 +1255,6 @@ name: Baremetal Deploy and Test Phase condition: SUCCESSFUL projects: - - name: 'apex-os-nosdn-nofeature-noha-baremetal-fraser' - node-parameters: false - current-parameters: false - predefined-parameters: | - OPNFV_CLEAN=yes - kill-phase-on: NEVER - abort-all-job: true - git-revision: false - name: 'apex-os-nosdn-nofeature-ha-baremetal-fraser' node-parameters: false current-parameters: false @@ -1188,15 +1263,7 @@ kill-phase-on: NEVER abort-all-job: true git-revision: false - - name: 'apex-os-odl-nofeature-ha-baremetal-fraser' - node-parameters: false - current-parameters: false - predefined-parameters: | - OPNFV_CLEAN=yes - kill-phase-on: NEVER - abort-all-job: true - git-revision: false - - name: 'apex-os-odl-nofeature-noha-baremetal-fraser' + - name: 'apex-os-odl-bgpvpn-ha-baremetal-fraser' node-parameters: false current-parameters: false predefined-parameters: | @@ -1204,7 +1271,16 @@ kill-phase-on: NEVER abort-all-job: true git-revision: false - - name: 'apex-os-odl-bgpvpn-ha-baremetal-fraser' + +# danube Builder +- builder: + name: apex-builder-danube + builders: + - multijob: + name: Baremetal Deploy and Test Phase + condition: SUCCESSFUL + projects: + - name: 'apex-os-nosdn-nofeature-noha-baremetal-danube' node-parameters: false current-parameters: false predefined-parameters: | @@ -1212,7 +1288,7 @@ kill-phase-on: NEVER abort-all-job: true git-revision: false - - name: 'apex-os-ovn-nofeature-noha-baremetal-fraser' + - name: 'apex-os-nosdn-nofeature-ha-baremetal-danube' node-parameters: false current-parameters: false predefined-parameters: | @@ -1220,7 +1296,7 @@ kill-phase-on: NEVER abort-all-job: true git-revision: false - - name: 'apex-os-nosdn-fdio-noha-baremetal-fraser' + - name: 'apex-os-nosdn-nofeature-ha-ipv6-baremetal-danube' node-parameters: false current-parameters: false predefined-parameters: | @@ -1228,7 +1304,7 @@ kill-phase-on: NEVER abort-all-job: true git-revision: false - - name: 'apex-os-nosdn-fdio-ha-baremetal-fraser' + - name: 'apex-os-nosdn-ovs-noha-baremetal-danube' node-parameters: false current-parameters: false predefined-parameters: | @@ -1236,7 +1312,7 @@ kill-phase-on: NEVER abort-all-job: true git-revision: false - - name: 'apex-os-nosdn-bar-ha-baremetal-fraser' + - name: 'apex-os-nosdn-ovs-ha-baremetal-danube' node-parameters: false current-parameters: false predefined-parameters: | @@ -1244,7 +1320,7 @@ kill-phase-on: NEVER abort-all-job: true git-revision: false - - name: 'apex-os-nosdn-bar-noha-baremetal-fraser' + - name: 'apex-os-nosdn-fdio-noha-baremetal-danube' node-parameters: false current-parameters: false predefined-parameters: | @@ -1252,7 +1328,7 @@ kill-phase-on: NEVER abort-all-job: true git-revision: false - - name: 'apex-os-nosdn-ovs_dpdk-noha-baremetal-fraser' + - name: 'apex-os-nosdn-fdio-ha-baremetal-danube' node-parameters: false current-parameters: false predefined-parameters: | @@ -1260,7 +1336,7 @@ kill-phase-on: NEVER abort-all-job: true git-revision: false - - name: 'apex-os-nosdn-ovs_dpdk-ha-baremetal-fraser' + - name: 'apex-os-nosdn-kvm-ha-baremetal-danube' node-parameters: false current-parameters: false predefined-parameters: | @@ -1268,7 +1344,7 @@ kill-phase-on: NEVER abort-all-job: true git-revision: false - - name: 'apex-os-odl-sfc-noha-baremetal-fraser' + - name: 'apex-os-nosdn-kvm-noha-baremetal-danube' node-parameters: false current-parameters: false predefined-parameters: | @@ -1276,7 +1352,7 @@ kill-phase-on: NEVER abort-all-job: true git-revision: false - - name: 'apex-os-odl-sfc-ha-baremetal-fraser' + - name: 'apex-os-odl_l2-fdio-noha-baremetal-danube' node-parameters: false current-parameters: false predefined-parameters: | @@ -1284,7 +1360,7 @@ kill-phase-on: NEVER abort-all-job: true git-revision: false - - name: 'apex-os-nosdn-calipso-noha-baremetal-fraser' + - name: 'apex-os-odl_l2-fdio-ha-baremetal-danube' node-parameters: false current-parameters: false predefined-parameters: | @@ -1292,16 +1368,7 @@ kill-phase-on: NEVER abort-all-job: true git-revision: false - -# danube Builder -- builder: - name: apex-builder-danube - builders: - - multijob: - name: Baremetal Deploy and Test Phase - condition: SUCCESSFUL - projects: - - name: 'apex-os-nosdn-nofeature-noha-baremetal-danube' + - name: 'apex-os-odl_netvirt-fdio-noha-baremetal-danube' node-parameters: false current-parameters: false predefined-parameters: | @@ -1309,7 +1376,7 @@ kill-phase-on: NEVER abort-all-job: true git-revision: false - - name: 'apex-os-nosdn-nofeature-ha-baremetal-danube' + - name: 'apex-os-odl_l2-sfc-noha-baremetal-danube' node-parameters: false current-parameters: false predefined-parameters: | @@ -1317,7 +1384,7 @@ kill-phase-on: NEVER abort-all-job: true git-revision: false - - name: 'apex-os-nosdn-nofeature-ha-ipv6-baremetal-danube' + - name: 'apex-os-odl_l3-nofeature-noha-baremetal-danube' node-parameters: false current-parameters: false predefined-parameters: | @@ -1325,7 +1392,7 @@ kill-phase-on: NEVER abort-all-job: true git-revision: false - - name: 'apex-os-nosdn-ovs-noha-baremetal-danube' + - name: 'apex-os-odl_l3-nofeature-ha-baremetal-danube' node-parameters: false current-parameters: false predefined-parameters: | @@ -1333,7 +1400,7 @@ kill-phase-on: NEVER abort-all-job: true git-revision: false - - name: 'apex-os-nosdn-ovs-ha-baremetal-danube' + - name: 'apex-os-odl_l3-ovs-noha-baremetal-danube' node-parameters: false current-parameters: false predefined-parameters: | @@ -1341,7 +1408,7 @@ kill-phase-on: NEVER abort-all-job: true git-revision: false - - name: 'apex-os-nosdn-fdio-noha-baremetal-danube' + - name: 'apex-os-odl_l3-ovs-ha-baremetal-danube' node-parameters: false current-parameters: false predefined-parameters: | @@ -1349,7 +1416,7 @@ kill-phase-on: NEVER abort-all-job: true git-revision: false - - name: 'apex-os-nosdn-fdio-ha-baremetal-danube' + - name: 'apex-os-odl-bgpvpn-ha-baremetal-danube' node-parameters: false current-parameters: false predefined-parameters: | @@ -1357,7 +1424,7 @@ kill-phase-on: NEVER abort-all-job: true git-revision: false - - name: 'apex-os-nosdn-kvm-ha-baremetal-danube' + - name: 'apex-os-odl-gluon-noha-baremetal-danube' node-parameters: false current-parameters: false predefined-parameters: | @@ -1365,7 +1432,7 @@ kill-phase-on: NEVER abort-all-job: true git-revision: false - - name: 'apex-os-nosdn-kvm-noha-baremetal-danube' + - name: 'apex-os-odl_l3-fdio-noha-baremetal-danube' node-parameters: false current-parameters: false predefined-parameters: | @@ -1373,7 +1440,7 @@ kill-phase-on: NEVER abort-all-job: true git-revision: false - - name: 'apex-os-odl_l2-fdio-noha-baremetal-danube' + - name: 'apex-os-odl_l3-fdio-ha-baremetal-danube' node-parameters: false current-parameters: false predefined-parameters: | @@ -1381,7 +1448,7 @@ kill-phase-on: NEVER abort-all-job: true git-revision: false - - name: 'apex-os-odl_l2-fdio-ha-baremetal-danube' + - name: 'apex-os-odl_l3-fdio_dvr-noha-baremetal-danube' node-parameters: false current-parameters: false predefined-parameters: | @@ -1389,7 +1456,7 @@ kill-phase-on: NEVER abort-all-job: true git-revision: false - - name: 'apex-os-odl_netvirt-fdio-noha-baremetal-danube' + - name: 'apex-os-odl_l3-fdio_dvr-ha-baremetal-danube' node-parameters: false current-parameters: false predefined-parameters: | @@ -1397,7 +1464,7 @@ kill-phase-on: NEVER abort-all-job: true git-revision: false - - name: 'apex-os-odl_l2-sfc-noha-baremetal-danube' + - name: 'apex-os-odl_l3-csit-noha-baremetal-danube' node-parameters: false current-parameters: false predefined-parameters: | @@ -1405,7 +1472,7 @@ kill-phase-on: NEVER abort-all-job: true git-revision: false - - name: 'apex-os-odl_l3-nofeature-noha-baremetal-danube' + - name: 'apex-os-onos-nofeature-ha-baremetal-danube' node-parameters: false current-parameters: false predefined-parameters: | @@ -1413,7 +1480,7 @@ kill-phase-on: NEVER abort-all-job: true git-revision: false - - name: 'apex-os-odl_l3-nofeature-ha-baremetal-danube' + - name: 'apex-os-ovn-nofeature-noha-baremetal-danube' node-parameters: false current-parameters: false predefined-parameters: | @@ -1421,7 +1488,16 @@ kill-phase-on: NEVER abort-all-job: true git-revision: false - - name: 'apex-os-odl_l3-ovs-noha-baremetal-danube' + +# master Builder +- builder: + name: apex-builder-master + builders: + - multijob: + name: Baremetal Deploy and Test Phase + condition: SUCCESSFUL + projects: + - name: 'apex-os-nosdn-nofeature-noha-baremetal-master' node-parameters: false current-parameters: false predefined-parameters: | @@ -1429,7 +1505,7 @@ kill-phase-on: NEVER abort-all-job: true git-revision: false - - name: 'apex-os-odl_l3-ovs-ha-baremetal-danube' + - name: 'apex-os-nosdn-nofeature-ha-baremetal-master' node-parameters: false current-parameters: false predefined-parameters: | @@ -1437,7 +1513,7 @@ kill-phase-on: NEVER abort-all-job: true git-revision: false - - name: 'apex-os-odl-bgpvpn-ha-baremetal-danube' + - name: 'apex-os-nosdn-nofeature-ha-ipv6-baremetal-master' node-parameters: false current-parameters: false predefined-parameters: | @@ -1445,7 +1521,7 @@ kill-phase-on: NEVER abort-all-job: true git-revision: false - - name: 'apex-os-odl-gluon-noha-baremetal-danube' + - name: 'apex-os-odl-nofeature-noha-baremetal-master' node-parameters: false current-parameters: false predefined-parameters: | @@ -1453,7 +1529,7 @@ kill-phase-on: NEVER abort-all-job: true git-revision: false - - name: 'apex-os-odl_l3-fdio-noha-baremetal-danube' + - name: 'apex-os-odl-nofeature-ha-baremetal-master' node-parameters: false current-parameters: false predefined-parameters: | @@ -1461,7 +1537,7 @@ kill-phase-on: NEVER abort-all-job: true git-revision: false - - name: 'apex-os-odl_l3-fdio-ha-baremetal-danube' + - name: 'apex-os-nosdn-queens-noha-baremetal-master' node-parameters: false current-parameters: false predefined-parameters: | @@ -1469,7 +1545,7 @@ kill-phase-on: NEVER abort-all-job: true git-revision: false - - name: 'apex-os-odl_l3-fdio_dvr-noha-baremetal-danube' + - name: 'apex-os-nosdn-queens-ha-baremetal-master' node-parameters: false current-parameters: false predefined-parameters: | @@ -1477,7 +1553,7 @@ kill-phase-on: NEVER abort-all-job: true git-revision: false - - name: 'apex-os-odl_l3-fdio_dvr-ha-baremetal-danube' + - name: 'apex-os-nosdn-queens-ha-ipv6-baremetal-master' node-parameters: false current-parameters: false predefined-parameters: | @@ -1485,7 +1561,7 @@ kill-phase-on: NEVER abort-all-job: true git-revision: false - - name: 'apex-os-odl_l3-csit-noha-baremetal-danube' + - name: 'apex-os-odl-queens-noha-baremetal-master' node-parameters: false current-parameters: false predefined-parameters: | @@ -1493,7 +1569,7 @@ kill-phase-on: NEVER abort-all-job: true git-revision: false - - name: 'apex-os-onos-nofeature-ha-baremetal-danube' + - name: 'apex-os-odl-queens-ha-baremetal-master' node-parameters: false current-parameters: false predefined-parameters: | @@ -1501,7 +1577,7 @@ kill-phase-on: NEVER abort-all-job: true git-revision: false - - name: 'apex-os-ovn-nofeature-noha-baremetal-danube' + - name: 'apex-k8s-nosdn-nofeature-noha-baremetal-master' node-parameters: false current-parameters: false predefined-parameters: | @@ -1509,16 +1585,7 @@ kill-phase-on: NEVER abort-all-job: true git-revision: false - -# master Builder -- builder: - name: apex-builder-master - builders: - - multijob: - name: Baremetal Deploy and Test Phase - condition: SUCCESSFUL - projects: - - name: 'apex-os-nosdn-nofeature-noha-baremetal-master' + - name: 'apex-os-odl-bgpvpn-ha-baremetal-master' node-parameters: false current-parameters: false predefined-parameters: | @@ -1526,7 +1593,7 @@ kill-phase-on: NEVER abort-all-job: true git-revision: false - - name: 'apex-os-nosdn-nofeature-ha-baremetal-master' + - name: 'apex-os-odl-bgpvpn-noha-baremetal-master' node-parameters: false current-parameters: false predefined-parameters: | @@ -1534,7 +1601,7 @@ kill-phase-on: NEVER abort-all-job: true git-revision: false - - name: 'apex-os-odl-nofeature-noha-baremetal-master' + - name: 'apex-os-odl-bgpvpn_queens-ha-baremetal-master' node-parameters: false current-parameters: false predefined-parameters: | @@ -1542,7 +1609,7 @@ kill-phase-on: NEVER abort-all-job: true git-revision: false - - name: 'apex-os-odl-nofeature-ha-baremetal-master' + - name: 'apex-os-odl-bgpvpn_queens-noha-baremetal-master' node-parameters: false current-parameters: false predefined-parameters: | @@ -1550,7 +1617,7 @@ kill-phase-on: NEVER abort-all-job: true git-revision: false - - name: 'apex-os-nosdn-queens-noha-baremetal-master' + - name: 'apex-os-odl-sfc-ha-baremetal-master' node-parameters: false current-parameters: false predefined-parameters: | @@ -1558,7 +1625,7 @@ kill-phase-on: NEVER abort-all-job: true git-revision: false - - name: 'apex-os-nosdn-queens-ha-baremetal-master' + - name: 'apex-os-odl-sfc-noha-baremetal-master' node-parameters: false current-parameters: false predefined-parameters: | @@ -1566,7 +1633,7 @@ kill-phase-on: NEVER abort-all-job: true git-revision: false - - name: 'apex-os-odl-queens-noha-baremetal-master' + - name: 'apex-os-odl-sfc_queens-ha-baremetal-master' node-parameters: false current-parameters: false predefined-parameters: | @@ -1574,7 +1641,7 @@ kill-phase-on: NEVER abort-all-job: true git-revision: false - - name: 'apex-os-odl-queens-ha-baremetal-master' + - name: 'apex-os-odl-sfc_queens-noha-baremetal-master' node-parameters: false current-parameters: false predefined-parameters: | diff --git a/jjb/apex/apex.yaml.j2 b/jjb/apex/apex.yaml.j2 index ffe8c5780..31e3bd8f3 100644 --- a/jjb/apex/apex.yaml.j2 +++ b/jjb/apex/apex.yaml.j2 @@ -14,6 +14,7 @@ - 'apex-testsuite-{scenario}-baremetal-{scenario_stream}' - 'apex-upload-snapshot' - 'apex-create-snapshot' + - 'apex-fetch-snap-info' - 'apex-flex-daily-os-nosdn-nofeature-ha-{stream}' - 'apex-dovetail-daily-os-nosdn-nofeature-ha-baremetal-danube' # stream: branch with - in place of / (eg. stable-arno) @@ -25,7 +26,7 @@ build-slave: 'apex-build-master' virtual-slave: 'apex-virtual-master' baremetal-slave: 'apex-baremetal-master' - verify-scenario: 'os-nosdn-nofeature-ha' + verify-scenario: 'os-nosdn-nofeature-noha' scenario_stream: 'master' disable_daily: false disable_promote: false @@ -37,7 +38,7 @@ baremetal-slave: 'apex-baremetal-master' verify-scenario: 'os-nosdn-nofeature-ha' scenario_stream: 'fraser' - disable_daily: true + disable_daily: false disable_promote: true - euphrates: &euphrates branch: 'stable/euphrates' @@ -76,14 +77,18 @@ os_version: - 'pike': os_scenario: 'pike' + odl_branch: 'stable/carbon' - 'queens': os_scenario: 'queens' + odl_branch: 'stable/oxygen' - 'master': os_scenario: 'nofeature' + odl_branch: 'stable/oxygen' topology: - 'noha' - 'ha' + - 'noha-allinone' # Fetch Logs Job - job-template: @@ -415,6 +420,24 @@ kill-phase-on: NEVER abort-all-job: true git-revision: false + - shell: | + if echo $DEPLOY_SCENARIO | grep queens &> /dev/null; then + echo 'REPO=opnfv' > functest_properties + else + echo 'REPO=ollivier' > functest_properties + fi + features=$(echo $DEPLOY_SCENARIO | sed -r -n 's/os-.+-(.+)-(noha|ha)/\1/p') + if [ "$features" == 'queens' ]; then + functest_scenario=$(echo $DEPLOY_SCENARIO | sed -r -n 's/(os-.+?)-queens-(noha|ha)/\1-nofeature-\2/p') + elif [[ "$features" =~ 'queens' ]]; then + functest_scenario=$(echo $DEPLOY_SCENARIO | sed -r -n 's/(os-.+?)-(.+)_queens-(noha|ha)/\1-\2-\3/p') + else + functest_scenario=$DEPLOY_SCENARIO + fi + echo "DEPLOY_SCENARIO=$functest_scenario" >> functest_properties + - inject: + properties-file: functest_properties + override-build-parameters: true - multijob: name: 'OPNFV Test Suite' condition: ALWAYS @@ -511,7 +534,7 @@ name: Dovetail-proposed_tests condition: ALWAYS projects: - - name: 'dovetail-apex-baremetal-proposed_tests-{scenario_stream}' + - name: 'dovetail-apex-baremetal-default-mandatory-{scenario_stream}' node-parameters: true current-parameters: false predefined-parameters: @@ -524,7 +547,7 @@ name: Dovetail-default condition: ALWAYS projects: - - name: 'dovetail-apex-baremetal-default-{scenario_stream}' + - name: 'dovetail-apex-baremetal-default-optional-{scenario_stream}' node-parameters: true current-parameters: false predefined-parameters: @@ -625,6 +648,19 @@ git-revision: true - apex-builder-{stream} +# snapshot info fetch +- job-template: + name: 'apex-fetch-snap-info' + + disabled: false + + parameters: + - '{project}-defaults' + + builders: + - shell: + !include-raw-escape: ./apex-fetch-snap-info.sh + # snapshot create - job-template: name: 'apex-create-snapshot' @@ -686,6 +722,14 @@ name: GS_URL default: $GS_BASE{gs-pathname}/{os_version}/{topology} description: "User for overriding GS_URL from apex params" + - string: + name: OS_VERSION + default: '{os_version}' + description: OpenStack version short name + - string: + name: ODL_BRANCH + default: '{odl_branch}' + description: ODL branch being used properties: - build-blocker: @@ -696,6 +740,7 @@ - 'apex-runner.*' - 'apex-daily.*' - 'apex-csit-promote.*' + - 'odl-netvirt.*' - throttle: max-per-node: 1 max-total: 10 @@ -706,10 +751,10 @@ builders: - multijob: - name: apex-virtual-deploy-test + name: apex-virtual-deploy condition: SUCCESSFUL projects: - - name: 'apex-virtual-{stream}' + - name: 'apex-deploy-virtual-{stream}' current-parameters: true predefined-parameters: | DEPLOY_SCENARIO=os-odl-{os_scenario}-{topology} @@ -718,12 +763,54 @@ GERRIT_REFSPEC=$GERRIT_REFSPEC GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER GERRIT_CHANGE_COMMIT_MESSAGE=$GERRIT_CHANGE_COMMIT_MESSAGE - FUNCTEST_SUITE_NAME=tempest_smoke - FUNCTEST_MODE=testcase node-parameters: true kill-phase-on: FAILURE abort-all-job: true git-revision: true + - multijob: + name: fetch snapshot info + condition: SUCCESSFUL + projects: + - name: 'apex-fetch-snap-info' + current-parameters: true + node-parameters: true + kill-phase-on: FAILURE + abort-all-job: true + git-revision: false + - multijob: + name: csit + condition: SUCCESSFUL + execution-type: SEQUENTIALLY + projects: + - name: cperf-apex-csit-master + predefined-parameters: | + ODL_BRANCH=$ODL_BRANCH + RC_FILE_PATH=/tmp/csit/overcloudrc + NODE_FILE_PATH=/tmp/csit/node.yaml + SSH_KEY_PATH=/tmp/csit/id_rsa + ODL_CONTAINERIZED=true + OS_VERSION=$OS_VERSION + node-parameters: true + kill-phase-on: NEVER + abort-all-job: false + - name: cperf-upload-logs-csit + predefined-parameters: | + ODL_BRANCH=$ODL_BRANCH + OS_VERSION=$OS_VERSION + node-parameters: true + kill-phase-on: FAILURE + abort-all-job: false + - name: 'apex-fetch-logs-{stream}' + current-parameters: false + predefined-parameters: | + GERRIT_BRANCH=$GERRIT_BRANCH + GERRIT_REFSPEC=$GERRIT_REFSPEC + GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER + GERRIT_CHANGE_COMMIT_MESSAGE=$GERRIT_CHANGE_COMMIT_MESSAGE + node-parameters: true + kill-phase-on: NEVER + abort-all-job: true + git-revision: false - multijob: name: create snapshot condition: SUCCESSFUL @@ -1102,4 +1189,3 @@ name: 'apex-danube' triggers: - timed: '0 3 1 1 7' - diff --git a/jjb/apex/scenarios.yaml.hidden b/jjb/apex/scenarios.yaml.hidden index 71a6c3dbe..b032613a3 100644 --- a/jjb/apex/scenarios.yaml.hidden +++ b/jjb/apex/scenarios.yaml.hidden @@ -1,28 +1,26 @@ master: - 'os-nosdn-nofeature-noha' - 'os-nosdn-nofeature-ha' + - 'os-nosdn-nofeature-ha-ipv6' - 'os-odl-nofeature-noha' - 'os-odl-nofeature-ha' - 'os-nosdn-queens-noha' - 'os-nosdn-queens-ha' + - 'os-nosdn-queens-ha-ipv6' - 'os-odl-queens-noha' - 'os-odl-queens-ha' + - 'k8s-nosdn-nofeature-noha' + - 'os-odl-bgpvpn-ha' + - 'os-odl-bgpvpn-noha' + - 'os-odl-bgpvpn_queens-ha' + - 'os-odl-bgpvpn_queens-noha' + - 'os-odl-sfc-ha' + - 'os-odl-sfc-noha' + - 'os-odl-sfc_queens-ha' + - 'os-odl-sfc_queens-noha' fraser: - - 'os-nosdn-nofeature-noha' - 'os-nosdn-nofeature-ha' - - 'os-odl-nofeature-ha' - - 'os-odl-nofeature-noha' - 'os-odl-bgpvpn-ha' - - 'os-ovn-nofeature-noha' - - 'os-nosdn-fdio-noha' - - 'os-nosdn-fdio-ha' - - 'os-nosdn-bar-ha' - - 'os-nosdn-bar-noha' - - 'os-nosdn-ovs_dpdk-noha' - - 'os-nosdn-ovs_dpdk-ha' - - 'os-odl-sfc-noha' - - 'os-odl-sfc-ha' - - 'os-nosdn-calipso-noha' euphrates: - 'os-nosdn-nofeature-noha' - 'os-nosdn-nofeature-ha' diff --git a/jjb/armband/armband-ci-jobs.yaml b/jjb/armband/armband-ci-jobs.yaml index 140f84b03..959115ac2 100644 --- a/jjb/armband/armband-ci-jobs.yaml +++ b/jjb/armband/armband-ci-jobs.yaml @@ -66,11 +66,17 @@ auto-trigger-name: '{installer}-{scenario}-{pod}-{stream}-trigger' - 'os-nosdn-ovs-ha': auto-trigger-name: '{installer}-{scenario}-{pod}-{stream}-trigger' + - 'os-nosdn-vpp-ha': + auto-trigger-name: '{installer}-{scenario}-{pod}-{stream}-trigger' # NOHA scenarios - 'os-nosdn-nofeature-noha': auto-trigger-name: '{installer}-{scenario}-{pod}-{stream}-trigger' + exclude: + - scenario: os-nosdn-vpp-ha + stream: fraser + jobs: - '{installer}-{scenario}-{pod}-daily-{stream}' - '{installer}-deploy-{pod}-daily-{stream}' @@ -284,7 +290,7 @@ - trigger: name: 'fuel-os-nosdn-nofeature-ha-armband-baremetal-master-trigger' triggers: - - timed: '0 1 * * 1' + - timed: '10 1 * * 2,4,6,7' - trigger: name: 'fuel-os-nosdn-nofeature-noha-armband-baremetal-master-trigger' triggers: @@ -292,22 +298,26 @@ - trigger: name: 'fuel-os-odl-nofeature-ha-armband-baremetal-master-trigger' triggers: - - timed: '0 1 * * 3,7' + - timed: '0 1 * * 2,4,6,7' - trigger: name: 'fuel-os-ovn-nofeature-ha-armband-baremetal-master-trigger' triggers: - - timed: '' + - timed: '10 1 * * 1,3,5' - trigger: name: 'fuel-os-nosdn-ovs-ha-armband-baremetal-master-trigger' triggers: - - timed: '0 1 * * 4' + - timed: '0 1 * * 1,3,5' +- trigger: + name: 'fuel-os-nosdn-vpp-ha-armband-baremetal-master-trigger' + triggers: + - timed: '' # --------------------------------------------------------------------- # Enea Armband CI Baremetal Triggers running against fraser branch # --------------------------------------------------------------------- - trigger: name: 'fuel-os-nosdn-nofeature-ha-armband-baremetal-fraser-trigger' triggers: - - timed: '0 1 * * 2' + - timed: '' - trigger: name: 'fuel-os-nosdn-nofeature-noha-armband-baremetal-fraser-trigger' triggers: @@ -315,7 +325,7 @@ - trigger: name: 'fuel-os-odl-nofeature-ha-armband-baremetal-fraser-trigger' triggers: - - timed: '0 1 * * 5' + - timed: '' - trigger: name: 'fuel-os-ovn-nofeature-ha-armband-baremetal-fraser-trigger' triggers: @@ -323,8 +333,7 @@ - trigger: name: 'fuel-os-nosdn-ovs-ha-armband-baremetal-fraser-trigger' triggers: - - timed: '0 1 * * 6' - + - timed: '' # -------------------------------------------------------------- # Enea Armband CI Virtual Triggers running against master branch # -------------------------------------------------------------- @@ -348,6 +357,10 @@ name: 'fuel-os-nosdn-ovs-ha-armband-virtual-master-trigger' triggers: - timed: '' +- trigger: + name: 'fuel-os-nosdn-vpp-ha-armband-virtual-master-trigger' + triggers: + - timed: '' # ------------------------------------------------------------------- # Enea Armband CI Virtual Triggers running against fraser branch # ------------------------------------------------------------------- diff --git a/jjb/armband/armband-verify-jobs.yaml b/jjb/armband/armband-verify-jobs.yaml index 6f0016348..6add90704 100644 --- a/jjb/armband/armband-verify-jobs.yaml +++ b/jjb/armband/armband-verify-jobs.yaml @@ -142,20 +142,6 @@ node-parameters: true kill-phase-on: NEVER abort-all-job: true - - name: 'functest-fuel-armband-virtual-suite-{stream}' - current-parameters: false - predefined-parameters: | - FUNCTEST_MODE=testcase - FUNCTEST_SUITE_NAME=vping_ssh - # Should be in sync with fuel-deploy.sh default scenario - DEPLOY_SCENARIO=$DEPLOY_SCENARIO - BRANCH=$BRANCH - GERRIT_REFSPEC=$GERRIT_REFSPEC - GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER - GERRIT_CHANGE_COMMIT_MESSAGE=$GERRIT_CHANGE_COMMIT_MESSAGE - node-parameters: true - kill-phase-on: NEVER - abort-all-job: true - job-template: name: 'armband-verify-{phase}-{stream}' diff --git a/jjb/auto/auto.yaml b/jjb/auto/auto.yaml index ee7208205..62f6ec29d 100644 --- a/jjb/auto/auto.yaml +++ b/jjb/auto/auto.yaml @@ -7,16 +7,15 @@ # ------------------------------- # BRANCH ANCHORS # ------------------------------- - master: &master - stream: master - branch: '{stream}' - gs-pathname: '' - disabled: false - fraser: &fraser - stream: fraser - branch: 'stable/{stream}' - gs-pathname: '/{stream}' - disabled: false + stream: + - master: + branch: '{stream}' + gs-pathname: '' + disabled: false + - fraser: &fraser + branch: 'stable/{stream}' + gs-pathname: '/{stream}' + disabled: false # ------------------------------- # DEPLOY TYPE ANCHORS # ------------------------------- @@ -28,20 +27,9 @@ # ------------------------------- # CI POD's # ------------------------------- - # fraser - # ------------------------------- pod: - # yamllint disable rule:key-duplicates - - auto-baremetal: - <<: *baremetal - <<: *fraser - # ------------------------------- - # master - # ------------------------------- - auto-baremetal: <<: *baremetal - <<: *master - # yamllint enable rule:key-duplicates # ------------------------------- # scenarios # ------------------------------- @@ -53,7 +41,8 @@ jobs: - '{installer}-{scenario}-{pod}-auto-daily-{stream}' - '{installer}-deploy-{pod}-daily-{stream}' - - '{project}-verify-basic' + - 'auto-verify-{stream}' + - 'auto-merge-{stream}' ######################## # job templates @@ -107,6 +96,98 @@ same-node: true block: true +- job-template: + name: 'auto-verify-{stream}' + + disabled: '{obj:disabled}' + + project-type: freestyle + + concurrent: true + + properties: + - logrotate-default + + parameters: + - project-parameter: + project: '{project}' + branch: '{branch}' + - 'opnfv-build-ubuntu-defaults' + + scm: + - git-scm-gerrit + + triggers: + - gerrit: + server-name: 'gerrit.opnfv.org' + trigger-on: + - patchset-created-event: + exclude-drafts: 'false' + exclude-trivial-rebase: 'false' + exclude-no-code-change: 'false' + - draft-published-event + - comment-added-contains-event: + comment-contains-value: 'recheck' + - comment-added-contains-event: + comment-contains-value: 'reverify' + projects: + - project-compare-type: 'ANT' + project-pattern: '{project}' + branches: + - branch-compare-type: 'ANT' + branch-pattern: '**/{branch}' + forbidden-file-paths: + - compare-type: ANT + pattern: 'docs/**' + + builders: + - shell: | + pwd + ./ci/build-auto.sh verify + +- job-template: + name: 'auto-merge-{stream}' + + disabled: '{obj:disabled}' + + project-type: freestyle + + concurrent: true + + properties: + - logrotate-default + + parameters: + - project-parameter: + project: '{project}' + branch: '{branch}' + - 'opnfv-build-ubuntu-defaults' + + scm: + - git-scm + + triggers: + - gerrit: + server-name: 'gerrit.opnfv.org' + trigger-on: + - change-merged-event + - comment-added-contains-event: + comment-contains-value: 'remerge' + projects: + - project-compare-type: 'ANT' + project-pattern: '{project}' + branches: + - branch-compare-type: 'ANT' + branch-pattern: '**/{branch}' + forbidden-file-paths: + - compare-type: ANT + pattern: 'docs/**' + + builders: + - shell: | + pwd + ./ci/build-auto.sh merge + ######################## # trigger macros ######################## diff --git a/jjb/compass4nfv/compass-ci-jobs.yaml b/jjb/compass4nfv/compass-ci-jobs.yaml index 298173e85..328e9e3a2 100644 --- a/jjb/compass4nfv/compass-ci-jobs.yaml +++ b/jjb/compass4nfv/compass-ci-jobs.yaml @@ -129,6 +129,9 @@ - 'k8-nosdn-stor4nfv-noha': disabled: false auto-trigger-name: 'compass-{scenario}-{pod}-{stream}-trigger' + - 'os-nosdn-stor4nfv-ha': + disabled: false + auto-trigger-name: 'compass-{scenario}-{pod}-{stream}-trigger' jobs: - 'compass-{scenario}-{pod}-daily-{stream}' @@ -214,8 +217,8 @@ unstable-threshold: 'FAILURE' # here the stream means the SUT stream, dovetail stream is defined in its own job # only run on os-(nosdn|odl_l3)-nofeature-ha scenario - # run with testsuite default, dovetail docker image with latest tag(Monday, Tuesday) - # run with testsuite proposed_tests, dovetail docker image with latest tag(Thursday, Friday) + # run with testsuite default, testarea mandatory, dovetail docker image with latest tag + # run with testsuite default, testarea optional, dovetail docker image with latest tag - conditional-step: condition-kind: and condition-operands: @@ -224,7 +227,7 @@ label: '{scenario}' steps: - trigger-builds: - - project: 'dovetail-compass-{pod}-default-{stream}' + - project: 'dovetail-compass-{pod}-default-mandatory-{stream}' current-parameters: false predefined-parameters: | DOCKER_TAG=latest @@ -243,9 +246,10 @@ label: '{scenario}' steps: - trigger-builds: - - project: 'dovetail-compass-{pod}-proposed_tests-{stream}' + - project: 'dovetail-compass-{pod}-default-optional-{stream}' current-parameters: false - predefined-parameters: + predefined-parameters: | + DOCKER_TAG=latest DEPLOY_SCENARIO={scenario} block: true same-node: true @@ -431,7 +435,7 @@ description: "Directory where the cache to be used during the build is located." - string: name: PPA_REPO - default: "http://artifacts.opnfv.org/compass4nfv/package{ppa-pathname}" + default: "https://artifacts.opnfv.org/compass4nfv/package{ppa-pathname}" - string: name: PPA_CACHE default: "$WORKSPACE/work/repo/" @@ -510,6 +514,10 @@ name: 'compass-k8-nosdn-stor4nfv-ha-baremetal-centos-master-trigger' triggers: - timed: '' +- trigger: + name: 'compass-os-nosdn-stor4nfv-ha-baremetal-centos-master-trigger' + triggers: + - timed: '' # ---------------------------- # noha-baremetal-centos-master @@ -603,6 +611,10 @@ name: 'compass-k8-nosdn-stor4nfv-ha-huawei-pod7-danube-trigger' triggers: - timed: '' +- trigger: + name: 'compass-os-nosdn-stor4nfv-ha-huawei-pod7-danube-trigger' + triggers: + - timed: '' # ---------------------------- # noha-huawei-pod7-danube @@ -695,6 +707,10 @@ name: 'compass-k8-nosdn-stor4nfv-ha-baremetal-master-trigger' triggers: - timed: '0 16 1-29/2 * *' +- trigger: + name: 'compass-os-nosdn-stor4nfv-ha-baremetal-master-trigger' + triggers: + - timed: '0 20 1-29/2 * *' # --------------------- # noha-baremetal-master @@ -787,6 +803,10 @@ name: 'compass-k8-nosdn-stor4nfv-ha-baremetal-fraser-trigger' triggers: - timed: '0 7 2-30/2 * *' +- trigger: + name: 'compass-os-nosdn-stor4nfv-ha-baremetal-fraser-trigger' + triggers: + - timed: '' # --------------------- # noha-baremetal-fraser @@ -879,6 +899,10 @@ name: 'compass-k8-nosdn-stor4nfv-ha-virtual-master-trigger' triggers: - timed: '0 15 2-30/2 * *' +- trigger: + name: 'compass-os-nosdn-stor4nfv-ha-virtual-master-trigger' + triggers: + - timed: '0 19 2-30/2 * *' # ------------------- # noha-virtual-master @@ -971,6 +995,10 @@ name: 'compass-k8-nosdn-stor4nfv-ha-virtual-fraser-trigger' triggers: - timed: '0 15 1-29/2 * *' +- trigger: + name: 'compass-os-nosdn-stor4nfv-ha-virtual-fraser-trigger' + triggers: + - timed: '' # ------------------- # noha-virtual-fraser diff --git a/jjb/compass4nfv/compass-deploy.sh b/jjb/compass4nfv/compass-deploy.sh index ac649b992..4c5448e33 100644 --- a/jjb/compass4nfv/compass-deploy.sh +++ b/jjb/compass4nfv/compass-deploy.sh @@ -35,6 +35,8 @@ export OPENSTACK_VERSION=${COMPASS_OPENSTACK_VERSION} if [[ "${DEPLOY_SCENARIO}" =~ "-ocl" ]]; then export NETWORK_CONF_FILE=network_ocl.yml +elif [[ "${DEPLOY_SCENARIO}" =~ "-odl" ]]; then + export NETWORK_CONF_FILE=network_odl.yml elif [[ "${DEPLOY_SCENARIO}" =~ "-onos" ]]; then export NETWORK_CONF_FILE=network_onos.yml elif [[ "${DEPLOY_SCENARIO}" =~ "-openo" ]]; then diff --git a/jjb/compass4nfv/compass-project-jobs.yaml b/jjb/compass4nfv/compass-project-jobs.yaml index dc935f024..b85773405 100644 --- a/jjb/compass4nfv/compass-project-jobs.yaml +++ b/jjb/compass4nfv/compass-project-jobs.yaml @@ -129,7 +129,7 @@ description: "URL to Google Storage." - string: name: PPA_REPO - default: "http://artifacts.opnfv.org/compass4nfv/package{ppa-pathname}" + default: "https://artifacts.opnfv.org/compass4nfv/package{ppa-pathname}" - string: name: PPA_CACHE default: "$WORKSPACE/work/repo/" diff --git a/jjb/compass4nfv/compass-verify-jobs.yaml b/jjb/compass4nfv/compass-verify-jobs.yaml index beb857d94..f0cda8273 100644 --- a/jjb/compass4nfv/compass-verify-jobs.yaml +++ b/jjb/compass4nfv/compass-verify-jobs.yaml @@ -334,7 +334,7 @@ description: "URL to Google Storage." - string: name: PPA_REPO - default: "http://artifacts.opnfv.org/compass4nfv/package{ppa-pathname}" + default: "https://artifacts.opnfv.org/compass4nfv/package{ppa-pathname}" - string: name: PPA_CACHE default: "$WORKSPACE/work/repo/" diff --git a/jjb/container4nfv/arm64/compass-build.sh b/jjb/container4nfv/arm64/compass-build.sh index 696f7ff76..48381cb00 100755 --- a/jjb/container4nfv/arm64/compass-build.sh +++ b/jjb/container4nfv/arm64/compass-build.sh @@ -7,8 +7,8 @@ COMPASS_WORK_DIR=$WORKSPACE/../compass-work mkdir -p $COMPASS_WORK_DIR ln -s $COMPASS_WORK_DIR work -#TODO: remove workaround after all arm64 patches merged -curl -s http://people.linaro.org/~yibo.cai/compass/compass4nfv-arm64-fixup.sh | bash -s {scenario} +sudo docker rm -f `docker ps | grep compass | cut -f1 -d' '` || true -# build tarball -COMPASS_ISO_REPO='http://people.linaro.org/~yibo.cai/compass' ./build.sh +curl -s http://people.linaro.org/~yibo.cai/compass/compass4nfv-arm64-fixup.sh | bash || true + +./build.sh diff --git a/jjb/container4nfv/arm64/compass-deploy.sh b/jjb/container4nfv/arm64/compass-deploy.sh index a22af09ba..7889c3bba 100755 --- a/jjb/container4nfv/arm64/compass-deploy.sh +++ b/jjb/container4nfv/arm64/compass-deploy.sh @@ -5,11 +5,11 @@ cd compass4nfv export ADAPTER_OS_PATTERN='(?i)CentOS-7.*arm.*' export OS_VERSION="centos7" -export KUBERNETES_VERSION="v1.7.5" +export KUBERNETES_VERSION="v1.9.1" if [[ "$NODE_NAME" =~ "-virtual" ]]; then export DHA="deploy/conf/vm_environment/k8-nosdn-nofeature-noha.yml" export NETWORK="deploy/conf/vm_environment/network.yml" - export VIRT_NUMBER=2 VIRT_CPUS=4 VIRT_MEM=8192 VIRT_DISK=50G + export VIRT_NUMBER=2 VIRT_CPUS=8 VIRT_MEM=8192 VIRT_DISK=50G else export DHA="deploy/conf/hardware_environment/huawei-pod8/k8-nosdn-nofeature-noha.yml" export NETWORK="deploy/conf/hardware_environment/huawei-pod8/network.yml" diff --git a/jjb/container4nfv/arm64/deploy-cni.sh b/jjb/container4nfv/arm64/deploy-cni.sh new file mode 100755 index 000000000..9afb98048 --- /dev/null +++ b/jjb/container4nfv/arm64/deploy-cni.sh @@ -0,0 +1,17 @@ +#!/bin/bash -e + +cd container4nfv/src/arm/cni-deploy + +DEPLOY_SCENARIO={scenario} + +virtualenv .venv +source .venv/bin/activate +pip install ansible==2.6.1 + +ansible-playbook -i inventory/inventory.cfg deploy.yml --tags flannel,multus + +if [ "$DEPLOY_SCENARIO" == "k8-sriov-nofeature-noha" ]; then + ansible-playbook -i inventory/inventory.cfg deploy.yml --tags sriov +elif [ "$DEPLOY_SCENARIO" == "k8-vpp-nofeature-noha" ]; then + ansible-playbook -i inventory/inventory.cfg deploy.yml --tags vhost-vpp +fi diff --git a/jjb/container4nfv/container4nfv-arm64.yaml b/jjb/container4nfv/container4nfv-arm64.yaml index 5f5bc8627..b66667471 100644 --- a/jjb/container4nfv/container4nfv-arm64.yaml +++ b/jjb/container4nfv/container4nfv-arm64.yaml @@ -14,21 +14,28 @@ disabled: false - 'k8-sriov-nofeature-noha': disabled: false + - 'k8-vpp-nofeature-noha': + disabled: false pod: - virtual: slave-label: arm-packet01 - baremetal: slave-label: compass-baremetal-arm jobs: + - 'container4nfv-arm-deploy-{pod}-daily-{stream}' - 'container4nfv-{scenario}-{pod}-daily-{stream}' - job-template: - name: 'container4nfv-{scenario}-{pod}-daily-{stream}' - disabled: '{obj:disabled}' - concurrent: false + name: 'container4nfv-arm-deploy-{pod}-daily-{stream}' + disabled: false node: '{slave-label}' + wrappers: + - timeout: + timeout: 210 + fail: true + scm: - git: url: https://gerrit.opnfv.org/gerrit/compass4nfv @@ -38,12 +45,7 @@ wipe-workspace: true triggers: - - 'trigger-{scenario}-{pod}-{stream}' - - wrappers: - - timeout: - timeout: 150 - fail: true + - 'trigger-deploy-{pod}-{stream}' builders: - shell: @@ -54,42 +56,68 @@ - project: yardstick-arm64-compass-arm-virtual03-daily-master current-parameters: false same-node: true - block: false + block: true + block-thresholds: + build-step-failure-threshold: 'never' + failure-threshold: 'never' + unstable-threshold: 'Failure' + - project: container4nfv-k8-multus-nofeature-noha-{pod}-daily-{stream} + current-parameters: true + same-node: true + block: true + block-thresholds: + build-step-failure-threshold: 'never' + failure-threshold: 'never' + unstable-threshold: 'Failure' + - project: container4nfv-k8-sriov-nofeature-noha-{pod}-daily-{stream} + current-parameters: true + same-node: true + block: true + block-thresholds: + build-step-failure-threshold: 'never' + failure-threshold: 'never' + unstable-threshold: 'Failure' + - project: container4nfv-k8-vpp-nofeature-noha-{pod}-daily-{stream} + current-parameters: true + same-node: true + block: true + block-thresholds: + build-step-failure-threshold: 'never' + failure-threshold: 'never' + unstable-threshold: 'never' -- trigger: - name: 'trigger-k8-multus-nofeature-noha-virtual-master' - triggers: - - timed: '0 12 * * *' -- trigger: - name: 'trigger-k8-sriov-nofeature-noha-virtual-master' - triggers: - - timed: '0 15 * * *' +- job-template: + name: 'container4nfv-{scenario}-{pod}-daily-{stream}' + disabled: '{obj:disabled}' + node: '{slave-label}' -- trigger: - name: 'trigger-k8-multus-nofeature-noha-virtual-fraser' - triggers: - - timed: '0 18 * * *' -- trigger: - name: 'trigger-k8-sriov-nofeature-noha-virtual-fraser' - triggers: - - timed: '0 21 * * *' + scm: + - git: + url: https://gerrit.opnfv.org/gerrit/container4nfv + branches: + - master + basedir: container4nfv + wipe-workspace: true + + builders: + - shell: + !include-raw: arm64/deploy-cni.sh - trigger: - name: 'trigger-k8-multus-nofeature-noha-baremetal-master' + name: 'trigger-deploy-virtual-master' triggers: - timed: '0 12 * * *' - trigger: - name: 'trigger-k8-sriov-nofeature-noha-baremetal-master' + name: 'trigger-deploy-virtual-fraser' triggers: - - timed: '0 15 * * *' - + - timed: '0 18 * * *' - trigger: - name: 'trigger-k8-multus-nofeature-noha-baremetal-fraser' + name: 'trigger-deploy-baremetal-master' triggers: - - timed: '0 18 * * *' + - timed: '0 12 * * *' - trigger: - name: 'trigger-k8-sriov-nofeature-noha-baremetal-fraser' + name: 'trigger-deploy-baremetal-fraser' triggers: - - timed: '0 21 * * *' + - timed: '0 18 * * *' diff --git a/jjb/cperf/cirros-upload.yaml.ansible b/jjb/cperf/cirros-upload.yaml.ansible new file mode 100644 index 000000000..855bb1f3d --- /dev/null +++ b/jjb/cperf/cirros-upload.yaml.ansible @@ -0,0 +1,39 @@ +--- +- hosts: all + tasks: + - copy: + src: "{{ lookup('env', 'WORKSPACE') }}/{{ item }}" + dest: "/home/heat-admin/{{ item }}" + owner: heat-admin + group: heat-admin + mode: 0775 + with_items: + - cirros-0.3.5-x86_64-disk.img + - overcloudrc + - name: Upload cirros glance image + shell: > + source /home/heat-admin/overcloudrc && openstack image create + cirros-0.3.5-x86_64-disk --public + --file /home/heat-admin/cirros-0.3.5-x86_64-disk.img + --disk-format qcow2 --container-format bare + - name: Create nano flavor + shell: > + source /home/heat-admin/overcloudrc && openstack flavor create + --id 42 --ram 64 --disk 0 --vcpus 1 m1.nano + - name: Open CSIT TCP port for netcat + iptables: + chain: INPUT + action: insert + protocol: tcp + destination_port: 12345 + jump: ACCEPT + become: yes + - name: Open CSIT UDP port for netcat + iptables: + chain: INPUT + action: insert + protocol: udp + destination_port: 12345 + jump: ACCEPT + become: yes + diff --git a/jjb/cperf/cperf-ci-jobs.yaml b/jjb/cperf/cperf-ci-jobs.yaml index fdd3509d1..efcf51faa 100644 --- a/jjb/cperf/cperf-ci-jobs.yaml +++ b/jjb/cperf/cperf-ci-jobs.yaml @@ -9,47 +9,30 @@ # ------------------------------- # BRANCH ANCHORS # ------------------------------- - master: &master - stream: master - branch: '{stream}' - gs-pathname: '' - docker-tag: 'latest' - danube: &danube - stream: danube - branch: 'stable/{stream}' - gs-pathname: '/{stream}' - docker-tag: 'stable' + stream: master + branch: '{stream}' + gs-pathname: '' + docker-tag: 'latest' - # ------------------------------- - # POD, INSTALLER, AND BRANCH MAPPING - # ------------------------------- - pod: - # ------------------------------- - # master - # ------------------------------- - - intel-pod2: - installer: apex - <<: *master - - intel-pod2: - installer: apex - <<: *danube + installer: apex testsuite: - - 'daily' + - csit + - cbench jobs: - - 'cperf-{installer}-{pod}-{testsuite}-{stream}' + - 'cperf-{installer}-{testsuite}-{stream}' + - 'cperf-upload-logs-csit' ################################ # job template ################################ - job-template: - name: 'cperf-{installer}-{pod}-{testsuite}-{stream}' + name: 'cperf-{installer}-{testsuite}-{stream}' concurrent: true properties: - - logrotate-default - throttle: enabled: true max-per-node: 1 @@ -57,28 +40,46 @@ wrappers: - build-name: - name: '$BUILD_NUMBER Suite: $CPERF_SUITE_NAME Scenario: $DEPLOY_SCENARIO' + name: '$BUILD_NUMBER Suite: $CPERF_SUITE_NAME ODL BRANCH: $ODL_BRANCH' - timeout: timeout: 400 abort: true parameters: - - project-parameter: - project: '{project}' - branch: '{branch}' - - '{pod}-defaults' - - '{installer}-defaults' - cperf-parameter: testsuite: '{testsuite}' gs-pathname: '{gs-pathname}' docker-tag: '{docker-tag}' - - scm: - - git-scm + stream: '{stream}' builders: - 'cperf-{testsuite}-builder' +- job-template: + name: 'cperf-upload-logs-csit' + + concurrent: true + + disabled: false + + parameters: + - cperf-parameter: + testsuite: 'csit' + gs-pathname: '{gs-pathname}' + docker-tag: '{docker-tag}' + stream: '{stream}' + + # yamllint enable rule:line-length + properties: + - logrotate-default + - throttle: + max-per-node: 1 + max-total: 10 + option: 'project' + + builders: + - 'cperf-upload-logs-csit' + ######################## # parameter macros ######################## @@ -89,6 +90,14 @@ name: CPERF_SUITE_NAME default: '{testsuite}' description: "Suite name to run" + - string: + name: ODL_BRANCH + default: 'master' + description: "Branch that OpenDaylight is running" + - string: + name: OS_VERSION + default: 'master' + description: "OpenStack version (short name, no stable/ prefix)" - string: name: GS_PATHNAME default: '{gs-pathname}' @@ -101,6 +110,22 @@ name: DOCKER_TAG default: '{docker-tag}' description: 'Tag to pull docker image' + - string: + name: RC_FILE_PATH + default: '' + description: "Path to the OS credentials file if given" + - string: + name: SSH_KEY_PATH + default: '' + description: "Path to the private SSH key to access OPNFV nodes" + - string: + name: NODE_FILE_PATH + default: '' + description: "Path to the yaml file describing overcloud nodes" + - string: + name: ODL_CONTAINERIZED + default: 'true' + description: "boolean set true if ODL on overcloud is a container" ######################## # trigger macros @@ -110,11 +135,25 @@ # builder macros ######################## - builder: - name: cperf-daily-builder + name: cperf-csit-builder + builders: + - 'cperf-cleanup' + - 'cperf-prepare-robot' + - 'cperf-robot-netvirt-csit' + +- builder: + name: cperf-cbench-builder builders: - 'cperf-cleanup' + - 'cperf-prepare-robot' - 'cperf-robot-cbench' +- builder: + name: cperf-prepare-robot + builders: + - shell: + !include-raw: ./cperf-prepare-robot.sh + - builder: name: cperf-robot-cbench builders: @@ -123,23 +162,6 @@ set -o errexit set -o nounset set -o pipefail - undercloud_mac=$(sudo virsh domiflist undercloud | grep default | \ - grep -Eo "[0-9a-f]+:[0-9a-f]+:[0-9a-f]+:[0-9a-f]+:[0-9a-f]+:[0-9a-f]+") - INSTALLER_IP=$(/usr/sbin/arp -e | grep ${undercloud_mac} | awk {'print $1'}) - - sudo scp -o StrictHostKeyChecking=no root@$INSTALLER_IP:/home/stack/overcloudrc /tmp/overcloudrc - sudo chmod 755 /tmp/overcloudrc - source /tmp/overcloudrc - - # robot suites need the ssh key to log in to controller nodes, so throwing it - # in tmp, and mounting /tmp as $HOME as far as robot is concerned - sudo rm -rf /tmp/.ssh - sudo mkdir /tmp/.ssh - sudo chmod 0700 /tmp/.ssh - sudo scp -o StrictHostKeyChecking=no root@$INSTALLER_IP:/home/stack/.ssh/id_rsa /tmp/.ssh/ - sudo chown -R jenkins-ci:jenkins-ci /tmp/.ssh - # done with sudo. jenkins-ci is the user from this point - chmod 0600 /tmp/.ssh/id_rsa # cbench requires the openflow drop test feature to be installed. sshpass -p karaf ssh -o StrictHostKeyChecking=no \ @@ -148,8 +170,6 @@ -p 8101 karaf@$SDN_CONTROLLER_IP \ feature:install odl-openflowplugin-flow-services-ui odl-openflowplugin-drop-test - docker pull opnfv/cperf:$DOCKER_TAG - robot_cmd="pybot -e exclude -L TRACE -d /tmp \ -v ODL_SYSTEM_1_IP:${SDN_CONTROLLER_IP} \ -v ODL_SYSTEM_IP:${SDN_CONTROLLER_IP} \ @@ -164,6 +184,12 @@ docker run -i -v /tmp:/tmp opnfv/cperf:$DOCKER_TAG ${robot_cmd} ${robot_suite} +- builder: + name: cperf-robot-netvirt-csit + builders: + - shell: + !include-raw: ./cperf-robot-netvirt-csit.sh + - builder: name: cperf-cleanup builders: @@ -188,3 +214,8 @@ docker rmi opnfv/cperf:$tag >/dev/null done fi + +- builder: + name: cperf-upload-logs-csit + builders: + - shell: !include-raw: ./cperf-upload-logs-csit.sh diff --git a/jjb/cperf/cperf-prepare-robot.sh b/jjb/cperf/cperf-prepare-robot.sh new file mode 100755 index 000000000..d88c6d510 --- /dev/null +++ b/jjb/cperf/cperf-prepare-robot.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash + +set -o errexit +set -o nounset +set -o pipefail + +if [ -z ${RC_FILE_PATH+x} ]; then + undercloud_mac=$(sudo virsh domiflist undercloud | grep default | \ + grep -Eo "[0-9a-f]+:[0-9a-f]+:[0-9a-f]+:[0-9a-f]+:[0-9a-f]+:[0-9a-f]+") + INSTALLER_IP=$(/usr/sbin/arp -e | grep ${undercloud_mac} | awk {'print $1'}) + sudo scp -o StrictHostKeyChecking=no root@$INSTALLER_IP:/home/stack/overcloudrc /tmp/overcloudrc +else + cp -f $RC_FILE_PATH ${WORKSPACE}/overcloudrc +fi + +sudo chmod 755 ${WORKSPACE}/overcloudrc +source ${WORKSPACE}/overcloudrc + +# copy ssh key for robot + +if [ -z ${SSH_KEY_PATH+x} ]; then + sudo scp -o StrictHostKeyChecking=no root@$INSTALLER_IP:/home/stack/.ssh/id_rsa ${WORKSPACE}/ + sudo chown -R jenkins-ci:jenkins-ci ${WORKSPACE}/ + # done with sudo. jenkins-ci is the user from this point + chmod 0600 ${WORKSPACE}/id_rsa +else + cp -f ${SSH_KEY_PATH} ${WORKSPACE}/ +fi + +docker pull opnfv/cperf:$DOCKER_TAG + +sudo mkdir -p /tmp/robot_results diff --git a/jjb/cperf/cperf-robot-netvirt-csit.sh b/jjb/cperf/cperf-robot-netvirt-csit.sh new file mode 100755 index 000000000..4fdcd102b --- /dev/null +++ b/jjb/cperf/cperf-robot-netvirt-csit.sh @@ -0,0 +1,174 @@ +#!/usr/bin/env bash + +set -o errexit +set -o nounset +set -o pipefail + +if [ "$OS_VERSION" == 'master' ]; then + FULL_OS_VER='master' +else + FULL_OS_VER="stable/${OS_VERSION}" +fi + +if [ "$ODL_BRANCH" == 'master' ]; then + ODL_STREAM='neon' +else + ODL_STREAM=${ODL_BRANCH#"stable/"} +fi + +echo "ODL Stream set: ${ODL_STREAM} and OS Version is ${FULL_OS_VER}" + +sudo rm -rf releng +git clone https://gerrit.opnfv.org/gerrit/releng.git +REL_PATH='releng/jjb/cperf' + +# NOTE: sourcing overcloudrc unsets any variable with OS_ prefix +source ${WORKSPACE}/overcloudrc +# note SDN_CONTROLLER_IP is set in overcloudrc, which is the VIP +# for admin/public network (since we are running single network deployment) + +NUM_CONTROL_NODES=$(python ${REL_PATH}/parse-node-yaml.py num_nodes --file $NODE_FILE_PATH) +NUM_COMPUTE_NODES=$(python ${REL_PATH}/parse-node-yaml.py num_nodes --node-type compute --file $NODE_FILE_PATH) + +echo "Number of Control nodes found: ${NUM_CONTROL_NODES}" +echo "Number of Compute nodes found: ${NUM_COMPUTE_NODES}" + +# Only 1 combo or ctrl node is specified, even for OS HA deployments +# Currently supported combinations are: +# 0cmb-1ctl-2cmp +# 1cmb-0ctl-0cmp +# 1cmb-0ctl-1cmp +if [ "$NUM_COMPUTE_NODES" -eq 0 ]; then + OPENSTACK_TOPO="1cmb-0ctl-0cmp" +else + OPENSTACK_TOPO="0cmb-1ctl-2cmp" +fi + +idx=1 +EXTRA_ROBOT_ARGS="" +for idx in `seq 1 $NUM_CONTROL_NODES`; do + CONTROLLER_IP=$(python ${REL_PATH}/parse-node-yaml.py get_value -k address --node-number ${idx} --file $NODE_FILE_PATH) + EXTRA_ROBOT_ARGS+=" -v ODL_SYSTEM_${idx}_IP:${CONTROLLER_IP} \ + -v OS_CONTROL_NODE_${idx}_IP:${CONTROLLER_IP} \ + -v ODL_SYSTEM_${idx}_IP:${CONTROLLER_IP} \ + -v HA_PROXY_${idx}_IP:${SDN_CONTROLLER_IP}" +done + +# In all-in-one these Compute IPs still need to be passed to robot +if [ "$NUM_COMPUTE_NODES" -eq 0 ]; then + EXTRA_ROBOT_ARGS+=" -v OS_COMPUTE_1_IP:'' -v OS_COMPUTE_2_IP:''" +else + idx=1 + for idx in `seq 1 $NUM_COMPUTE_NODES`; do + COMPUTE_IP=$(python ${REL_PATH}/parse-node-yaml.py get_value -k address --node-type compute --node-number ${idx} --file $NODE_FILE_PATH) + EXTRA_ROBOT_ARGS+=" -v OS_COMPUTE_${idx}_IP:${COMPUTE_IP}" + done +fi + +CONTROLLER_1_IP=$(python ${REL_PATH}/parse-node-yaml.py get_value -k address --node-number 1 --file $NODE_FILE_PATH) + +if [ "$ODL_CONTAINERIZED" == 'false' ]; then + EXTRA_ROBOT_ARGS+=" -v NODE_KARAF_COUNT_COMMAND:'ps axf | grep org.apache.karaf | grep -v grep | wc -l || echo 0' \ + -v NODE_START_COMMAND:'sudo systemctl start opendaylight_api' \ + -v NODE_KILL_COMMAND:'sudo systemctl stop opendaylight_api' \ + -v NODE_STOP_COMMAND:'sudo systemctl stop opendaylight_api' \ + -v NODE_FREEZE_COMMAND:'sudo systemctl stop opendaylight_api' " +else + EXTRA_ROBOT_ARGS+=" -v NODE_KARAF_COUNT_COMMAND:'sudo docker ps | grep opendaylight_api | wc -l || echo 0' \ + -v NODE_START_COMMAND:'sudo docker start opendaylight_api' \ + -v NODE_KILL_COMMAND:'sudo docker stop opendaylight_api' \ + -v NODE_STOP_COMMAND:'sudo docker stop opendaylight_api' \ + -v NODE_FREEZE_COMMAND:'sudo docker stop opendaylight_api' " +fi + +# FIXME(trozet) remove this once it is fixed in csit +# Upload glance image into openstack +wget -O ${WORKSPACE}/cirros-0.3.5-x86_64-disk.img http://download.cirros-cloud.net/0.3.5/cirros-0.3.5-x86_64-disk.img +export ANSIBLE_HOST_KEY_CHECKING=False +ansible-playbook -i ${CONTROLLER_1_IP}, -u heat-admin --key-file ${WORKSPACE}/id_rsa ${REL_PATH}/cirros-upload.yaml.ansible -vvv + +LOGS_LOCATION=/tmp/robot_results + +robot_cmd="pybot \ + --removekeywords wuks \ + --xunit robotxunit.xml \ + --name 'CSIT' \ + -e exclude \ + -d $LOGS_LOCATION \ + -v BUNDLEFOLDER:/opt/opendaylight \ + -v CONTROLLER_USER:heat-admin \ + -v DEFAULT_LINUX_PROMPT:\$ \ + -v DEFAULT_LINUX_PROMPT_STRICT:]\$ \ + -v DEFAULT_USER:heat-admin \ + -v DEVSTACK_DEPLOY_PATH:/tmp \ + -v EXTERNAL_GATEWAY:$CONTROLLER_1_IP \ + -v EXTERNAL_PNF:$CONTROLLER_1_IP \ + -v EXTERNAL_SUBNET:192.0.2.0/24 \ + -v EXTERNAL_SUBNET_ALLOCATION_POOL:start=192.0.2.100,end=192.0.2.200 \ + -v EXTERNAL_INTERNET_ADDR:$CONTROLLER_1_IP \ + -v HA_PROXY_IP:$SDN_CONTROLLER_IP \ + -v NUM_ODL_SYSTEM:$NUM_CONTROL_NODES \ + -v NUM_OS_SYSTEM:$(($NUM_CONTROL_NODES + $NUM_COMPUTE_NODES)) \ + -v NUM_TOOLS_SYSTEM:0 \ + -v ODL_SNAT_MODE:conntrack \ + -v ODL_STREAM:$ODL_STREAM \ + -v ODL_SYSTEM_IP:$CONTROLLER_1_IP \ + -v OS_CONTROL_NODE_IP:$CONTROLLER_1_IP \ + -v OPENSTACK_BRANCH:$FULL_OS_VER \ + -v OPENSTACK_TOPO:$OPENSTACK_TOPO \ + -v OS_USER:heat-admin \ + -v ODL_ENABLE_L3_FWD:yes \ + -v ODL_SYSTEM_USER:heat-admin \ + -v ODL_SYSTEM_PROMPT:\$ \ + -v PRE_CLEAN_OPENSTACK_ALL:True \ + -v PUBLIC_PHYSICAL_NETWORK:datacentre \ + -v RESTCONFPORT:8081 \ + -v ODL_RESTCONF_USER:admin \ + -v ODL_RESTCONF_PASSWORD:$SDN_CONTROLLER_PASSWORD \ + -v KARAF_PROMPT_LOGIN:'opendaylight-user' \ + -v KARAF_PROMPT:'opendaylight-user.*root.*>' \ + -v SECURITY_GROUP_MODE:stateful \ + -v USER:heat-admin \ + -v USER_HOME:\$HOME \ + -v TOOLS_SYSTEM_IP:'' \ + -v NODE_ROLE_INDEX_START:0 \ + -v WORKSPACE:/tmp \ + $EXTRA_ROBOT_ARGS \ + -v of_port:6653 " + +SUITE_HOME='/home/opnfv/repos/odl_test/csit/suites' + +# Disabled suites +# +# ${SUITE_HOME}/openstack/connectivity/live_migration.robot +# Live migration will not work unless we use a shared storage backend like +# Ceph which we do not currently use with CSIT images +# + +suites="${SUITE_HOME}/openstack/connectivity/l2.robot \ + ${SUITE_HOME}/openstack/connectivity/l3.robot \ + ${SUITE_HOME}/openstack/connectivity/external_network.robot \ + ${SUITE_HOME}/openstack/connectivity/security_group.robot \ + ${SUITE_HOME}/openstack/securitygroup/neutron_security_group.robot \ + ${SUITE_HOME}/openstack/securitygroup/security_group_l3bcast.robot \ + ${SUITE_HOME}/netvirt/vpnservice/vpn_basic.robot \ + ${SUITE_HOME}/netvirt/vpnservice/vpn_basic_ipv6.robot \ + ${SUITE_HOME}/netvirt/elan/elan.robot \ + ${SUITE_HOME}/netvirt/vpnservice/arp_learning.robot \ + ${SUITE_HOME}/netvirt/l2l3_gatewaymac_arp.robot \ + ${SUITE_HOME}/integration/Create_JVM_Plots.robot" + +echo "Robot command set: ${robot_cmd}" +echo "Running robot..." +docker run -i --net=host \ + -v ${LOGS_LOCATION}:${LOGS_LOCATION} \ + -v ${WORKSPACE}/id_rsa:/tmp/id_rsa \ + -v ${WORKSPACE}/overcloudrc:/tmp/overcloudrc \ + opnfv/cperf:$DOCKER_TAG \ + /bin/bash -c "source /tmp/overcloudrc; mkdir -p \$HOME/.ssh; cp /tmp/id_rsa \$HOME/.ssh; \ + cd /home/opnfv/repos/odl_test/ && git pull origin master; \ + pip install odltools; \ + ${robot_cmd} ${suites};" + +echo "Running post CSIT clean" +ansible-playbook -i ${CONTROLLER_1_IP}, -u heat-admin --key-file ${WORKSPACE}/id_rsa ${REL_PATH}/csit-clean.yaml.ansible -vvv diff --git a/jjb/cperf/cperf-upload-logs-csit.sh b/jjb/cperf/cperf-upload-logs-csit.sh new file mode 100644 index 000000000..bd8680406 --- /dev/null +++ b/jjb/cperf/cperf-upload-logs-csit.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash + +set -o errexit +set -o nounset +set -o pipefail + +ODL_STREAM=${ODL_BRANCH#"stable/"} + +LOGS_LOCATION=/tmp/robot_results +UPLOAD_LOCATION=artifacts.opnfv.org/cperf/cperf-apex-csit-${ODL_STREAM}/${BUILD_NUMBER}/ +echo "Uploading robot logs to ${UPLOAD_LOCATION}" +gsutil -m cp -r -v ${LOGS_LOCATION} gs://${UPLOAD_LOCATION} > gsutil.latest_logs.log diff --git a/jjb/cperf/csit-clean.yaml.ansible b/jjb/cperf/csit-clean.yaml.ansible new file mode 100644 index 000000000..0151dd824 --- /dev/null +++ b/jjb/cperf/csit-clean.yaml.ansible @@ -0,0 +1,11 @@ +--- +- hosts: all + tasks: + - name: Delete cirros glance image + shell: > + source /home/heat-admin/overcloudrc && openstack image delete + cirros-0.3.5-x86_64-disk + - name: Delete nano flavor + shell: > + source /home/heat-admin/overcloudrc && openstack flavor delete + m1.nano diff --git a/jjb/cperf/parse-node-yaml.py b/jjb/cperf/parse-node-yaml.py new file mode 100644 index 000000000..5a7575540 --- /dev/null +++ b/jjb/cperf/parse-node-yaml.py @@ -0,0 +1,71 @@ +############################################################################## +# Copyright (c) 2018 Tim Rozet (trozet@redhat.com) and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +import argparse +import sys +import yaml + + +def get_node_data_by_number(node_type, node_number): + node_idx = 1 + for node_name, node_data in data['servers'].items(): + if node_type == node_data['type']: + if node_idx == node_number: + return node_name, node_data + else: + node_idx += 1 + + +def get_node_value(node_type, node_number, key): + node_name, node_data = get_node_data_by_number(node_type, node_number) + if not key and node_name is not None: + return node_name + elif node_data and isinstance(node_data, dict) and key in node_data: + return node_data[key] + + +def get_number_of_nodes(node_type): + nodes = data['servers'] + num_nodes = 0 + for node_name, node_data in nodes.items(): + if node_data['type'] == node_type: + num_nodes += 1 + return num_nodes + + +FUNCTION_MAP = {'num_nodes': + {'func': get_number_of_nodes, + 'args': ['node_type']}, + 'get_value': + {'func': get_node_value, + 'args': ['node_type', 'node_number', 'key']}, + } + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument('command', choices=FUNCTION_MAP.keys()) + parser.add_argument('-f', '--file', + dest='node_file', + required=True) + parser.add_argument('--node-type', + default='controller', + required=False) + parser.add_argument('--node-number', + default=1, + type=int, + required=False) + parser.add_argument('-k', '--key', + required=False) + args = parser.parse_args(sys.argv[1:]) + with open(args.node_file, 'r') as fh: + data = yaml.safe_load(fh) + assert 'servers' in data + func = FUNCTION_MAP[args.command]['func'] + args = [getattr(args, x) for x in FUNCTION_MAP[args.command]['args']] + print(func(*args)) diff --git a/jjb/doctor/doctor-env-presetup.sh b/jjb/doctor/doctor-env-presetup.sh new file mode 100755 index 000000000..61e65c792 --- /dev/null +++ b/jjb/doctor/doctor-env-presetup.sh @@ -0,0 +1,62 @@ +#!/usr/bin/env bash +set -o errexit +set -o pipefail + +# set vars from env if not provided by user as options +installer_key_file=${installer_key_file:-$HOME/installer_key_file} +opnfv_installer=${opnfv_installer:-$HOME/opnfv-installer.sh} + +# Fetch INSTALLER_IP for APEX deployments +if [[ ${INSTALLER_TYPE} == 'apex' ]]; then + + echo "Gathering IP information for Apex installer VM" + ssh_options="-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no" + if sudo virsh list | grep undercloud; then + echo "Installer VM detected" + undercloud_mac=$(sudo virsh domiflist undercloud | grep default | \ + grep -Eo "[0-9a-f]+:[0-9a-f]+:[0-9a-f]+:[0-9a-f]+:[0-9a-f]+:[0-9a-f]+") + export INSTALLER_IP=$(/usr/sbin/arp -e | grep ${undercloud_mac} | awk {'print $1'}) + echo "Installer ip is ${INSTALLER_IP}" + else + echo "No available installer VM exists and no credentials provided...exiting" + exit 1 + fi + + sudo cp /root/.ssh/id_rsa ${installer_key_file} + sudo chown `whoami`:`whoami` ${installer_key_file} + +elif [[ ${INSTALLER_TYPE} == 'daisy' ]]; then + echo "Gathering IP information for Daisy installer VM" + if sudo virsh list | grep daisy; then + echo "Installer VM detected" + + bridge_name=$(sudo virsh domiflist daisy | grep vnet | awk '{print $3}') + echo "Bridge is $bridge_name" + + installer_mac=$(sudo virsh domiflist daisy | grep vnet | \ + grep -Eo "[0-9a-f]+:[0-9a-f]+:[0-9a-f]+:[0-9a-f]+:[0-9a-f]+:[0-9a-f]+") + export INSTALLER_IP=$(/usr/sbin/arp -e -i $bridge_name | grep ${installer_mac} | head -n 1 | awk {'print $1'}) + + echo "Installer ip is ${INSTALLER_IP}" + else + echo "No available installer VM exists...exiting" + exit 1 + fi +fi + + +# Checking if destination path is valid +if [ -d $opnfv_installer ]; then + error "Please provide the full destination path for the installer ip file including the filename" +else + # Check if we can create the file (e.g. path is correct) + touch $opnfv_installer || error "Cannot create the file specified. Check that the path is correct and run the script again." +fi + + +# Write the installer info to the file +echo export INSTALLER_TYPE=${INSTALLER_TYPE} > $opnfv_installer +echo export INSTALLER_IP=${INSTALLER_IP} >> $opnfv_installer +if [ -e ${installer_key_file} ]; then + echo export SSH_KEY=${installer_key_file} >> $opnfv_installer +fi diff --git a/jjb/doctor/doctor.yaml b/jjb/doctor/doctor.yaml index 0d7b781a7..469023037 100644 --- a/jjb/doctor/doctor.yaml +++ b/jjb/doctor/doctor.yaml @@ -8,23 +8,16 @@ - master: branch: '{stream}' gs-pathname: '' - docker-tag: 'latest' disabled: false - fraser: &fraser branch: 'stable/{stream}' gs-pathname: '/{stream}' - docker-tag: 'stable' disabled: false - # feature projects' tests are not triggered by functest - # doctor verify Pods need to deploy with these scenario installer: - - 'apex': - scenario: 'os-nosdn-kvm-ha' - - 'fuel': - scenario: 'os-nosdn-ovs-ha' - - 'daisy': - scenario: 'os-nosdn-ovs_dpdk-noha' + - 'apex' + - 'fuel' + - 'daisy' arch: - 'x86_64' @@ -43,15 +36,13 @@ # these would beenabled again once the PoDs are ready - installer: 'fuel' arch: 'x86_64' - - installer: 'daisy' - arch: 'x86_64' jobs: - - 'doctor-verify-{inspector}-{stream}' + - 'doctor-verify-{stream}' - 'doctor-verify-{installer}-{inspector}-{arch}-{stream}' - job-template: - name: 'doctor-verify-{inspector}-{stream}' + name: 'doctor-verify-{stream}' disabled: '{obj:disabled}' project-type: 'multijob' parameters: @@ -74,10 +65,10 @@ # in phase jobs echo "Triggering phase jobs!" - multijob: - name: 'doctor-verify' + name: 'doctor-verify-sample' execution-type: PARALLEL projects: - - name: 'doctor-verify-apex-{inspector}-x86_64-{stream}' + - name: 'doctor-verify-apex-sample-x86_64-{stream}' predefined-parameters: | PROJECT=$PROJECT GERRIT_BRANCH=$GERRIT_BRANCH @@ -86,7 +77,7 @@ GERRIT_CHANGE_COMMIT_MESSAGE=$GERRIT_CHANGE_COMMIT_MESSAGE kill-phase-on: FAILURE git-revision: true - - name: 'doctor-verify-fuel-{inspector}-x86_64-{stream}' + - name: 'doctor-verify-daisy-sample-x86_64-{stream}' predefined-parameters: | PROJECT=$PROJECT GERRIT_BRANCH=$GERRIT_BRANCH @@ -95,15 +86,11 @@ GERRIT_CHANGE_COMMIT_MESSAGE=$GERRIT_CHANGE_COMMIT_MESSAGE kill-phase-on: FAILURE git-revision: true - - name: 'doctor-verify-fuel-{inspector}-aarch64-{stream}' - predefined-parameters: | - GERRIT_BRANCH=$GERRIT_BRANCH - GERRIT_REFSPEC=$GERRIT_REFSPEC - GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER - GERRIT_CHANGE_COMMIT_MESSAGE=$GERRIT_CHANGE_COMMIT_MESSAGE - kill-phase-on: FAILURE - git-revision: true - - name: 'doctor-verify-daisy-{inspector}-x86_64-{stream}' + - multijob: + name: 'doctor-verify-congress' + execution-type: PARALLEL + projects: + - name: 'doctor-verify-apex-congress-x86_64-{stream}' predefined-parameters: | PROJECT=$PROJECT GERRIT_BRANCH=$GERRIT_BRANCH @@ -128,10 +115,6 @@ - '{installer}-defaults' - 'doctor-slave-parameter' - 'doctor-parameter': - docker-tag: '{docker-tag}' - scenario: '{scenario}' - - 'doctor-functest-parameter': - gs-pathname: '{gs-pathname}' inspector: '{inspector}' scm: - git-scm-gerrit @@ -148,57 +131,10 @@ name: 'doctor-parameter' parameters: - string: - name: OS_CREDS - default: /home/jenkins/openstack.creds - description: 'OpenStack credentials' - - string: - name: DOCKER_TAG - default: '{docker-tag}' - description: 'Tag to pull docker image' - - string: - name: CLEAN_DOCKER_IMAGES - default: 'false' - description: 'Remove downloaded docker images (opnfv/functest:*)' - - string: - name: DEPLOY_SCENARIO - default: '{scenario}' - description: 'Scenario to deploy and test' + name: INSPECTOR_TYPE + default: '{inspector}' + description: 'inspector component' -- parameter: - name: 'doctor-functest-parameter' - parameters: - # functest-suite-parameter - - string: - name: FUNCTEST_MODE - default: 'testcase' - - string: - name: FUNCTEST_SUITE_NAME - default: 'doctor-notification' - - string: - name: TESTCASE_OPTIONS - # yamllint disable rule:line-length - default: '-e INSPECTOR_TYPE={inspector} -v $WORKSPACE:/home/opnfv/repos/doctor' - # yamllint enable rule:line-length - description: 'Addtional parameters specific to test case(s)' - # functest-parameter - - string: - name: GS_PATHNAME - default: '{gs-pathname}' - # yamllint disable rule:line-length - description: "Version directory where the opnfv documents will be stored in gs repository" - # yamllint enable rule:line-length - - string: - name: FUNCTEST_REPO_DIR - default: "/home/opnfv/repos/functest" - description: "Directory where the Functest repository is cloned" - - string: - name: PUSH_RESULTS_TO_DB - default: "true" - description: "Push the results of all the tests to the resultDB" - - string: - name: CI_DEBUG - default: 'true' - description: "Show debug output information" # ------------------------------- # builder macros # ------------------------------- @@ -206,27 +142,23 @@ - builder: name: 'doctor-verify-installer-inspector-builders-macro' builders: - - 'clean-workspace-log' - # yamllint disable rule:line-length + # yamllint disable rule:indentation + - shell: + !include-raw: + - ./doctor-env-presetup.sh + - ../../utils/fetch_os_creds.sh - shell: | - # NOTE: Create symbolic link, so that we can archive file outside - # of $WORKSPACE . - # NOTE: We are printing all logs under 'tests/' during test run, - # so this symbolic link should not be in 'tests/'. Otherwise, - # we'll have the same log twice in jenkins console log. - ln -sfn $HOME/opnfv/functest/results/{stream} functest_results - # NOTE: Get functest script in $WORKSPACE. This functest script is - # needed to perform VM image download in set-functest-env.sh - # from E release cycle. - mkdir -p functest/ci - wget https://git.opnfv.org/functest/plain/functest/ci/download_images.sh -O functest/ci/download_images.sh - - 'functest-suite-builder' - - shell: | - functest_log="$HOME/opnfv/functest/results/{stream}/$FUNCTEST_SUITE_NAME.log" - # NOTE: checking the test result, as the previous job could return - # 0 regardless the result of doctor test scenario. - grep -e 'doctor test successfully' $functest_log || exit 1 - # yamllint enable rule:line-length + # prepare the env for test + source $HOME/opnfv-openrc.sh + if [ -f $HOME/os_cacert ]; then + export OS_CACERT=$HOME/os_cacert + fi + source $HOME/opnfv-installer.sh + + # run tox to trigger the test + # As Jenkins user, it has no permission to send ICMP package + sudo -E tox -e py34 + # ------------------------------- # publisher macros @@ -236,10 +168,8 @@ publishers: - archive: artifacts: 'doctor_tests/*.log' - - archive: - artifacts: 'functest_results/$FUNCTEST_SUITE_NAME.log' - email-jenkins-admins-on-failure - + - workspace-cleanup ##################################### # trigger macros diff --git a/jjb/dovetail-webportal/dovetail-webportal-project-jobs.yaml b/jjb/dovetail-webportal/dovetail-webportal-project-jobs.yaml new file mode 100644 index 000000000..f7d3213c4 --- /dev/null +++ b/jjb/dovetail-webportal/dovetail-webportal-project-jobs.yaml @@ -0,0 +1,101 @@ +--- +################################################### +# Non-ci jobs for Dovetail project +# They will only be enabled on request by projects! +################################################### +- project: + name: dovetail-webportal-project-jobs + + project: 'dovetail-webportal' + + jobs: + - 'dovetail-webportal-verify-{stream}' + - 'dovetail-webportal-merge-{stream}' + + stream: + - master: + branch: '{stream}' + disabled: false + +################################ +# job templates +################################ + +- job-template: + name: 'dovetail-webportal-verify-{stream}' + + disabled: '{obj:disabled}' + + parameters: + - project-parameter: + project: '{project}' + branch: '{branch}' + - 'opnfv-build-ubuntu-defaults' + + scm: + - git-scm-gerrit + + triggers: + - gerrit: + server-name: 'gerrit.opnfv.org' + trigger-on: + - patchset-created-event: + exclude-drafts: 'false' + exclude-trivial-rebase: 'false' + exclude-no-code-change: 'false' + - draft-published-event + - comment-added-contains-event: + comment-contains-value: 'recheck' + - comment-added-contains-event: + comment-contains-value: 'reverify' + projects: + - project-compare-type: 'ANT' + project-pattern: '{project}' + branches: + - branch-compare-type: 'ANT' + branch-pattern: '**/{branch}' + builders: + - dovetail-webportal-hello-world + +- job-template: + name: 'dovetail-webportal-merge-{stream}' + + disabled: '{obj:disabled}' + + parameters: + - project-parameter: + project: '{project}' + branch: '{branch}' + - 'opnfv-build-ubuntu-defaults' + + scm: + - git-scm + + triggers: + - gerrit: + server-name: 'gerrit.opnfv.org' + trigger-on: + - change-merged-event + - comment-added-contains-event: + comment-contains-value: 'remerge' + projects: + - project-compare-type: 'ANT' + project-pattern: '{project}' + branches: + - branch-compare-type: 'ANT' + branch-pattern: '**/{branch}' + + builders: + - dovetail-webportal-hello-world + +################################ +# builders for dovetail project +############################### +- builder: + name: dovetail-webportal-hello-world + builders: + - shell: | + #!/bin/bash + set -o errexit + + echo "hello world" diff --git a/jjb/dovetail/dovetail-ci-jobs.yaml b/jjb/dovetail/dovetail-ci-jobs.yaml index a95617ae4..8d47616d8 100644 --- a/jjb/dovetail/dovetail-ci-jobs.yaml +++ b/jjb/dovetail/dovetail-ci-jobs.yaml @@ -178,8 +178,13 @@ - 'default' - 'proposed_tests' + testarea: + - 'mandatory' + - 'optional' + jobs: - 'dovetail-{SUT}-{pod}-{testsuite}-{stream}' + - 'dovetail-{SUT}-{pod}-{testsuite}-{testarea}-{stream}' ################################ # job templates @@ -230,6 +235,85 @@ name: TESTSUITE default: '{testsuite}' description: "dovetail testsuite to run" + - string: + name: TESTAREA + default: 'all' + description: "dovetail testarea to run" + - string: + name: DOVETAIL_REPO_DIR + default: "/home/opnfv/dovetail" + description: "Directory where the dovetail repository is cloned" + - string: + name: SUT_BRANCH + default: '{branch}' + description: "SUT branch" + + scm: + - git-scm + + builders: + - description-setter: + description: "POD: $NODE_NAME" + - 'dovetail-cleanup' + - 'dovetail-run' + + publishers: + - archive: + artifacts: 'results/**/*' + allow-empty: true + fingerprint: true + - email-jenkins-admins-on-failure + +- job-template: + name: 'dovetail-{SUT}-{pod}-{testsuite}-{testarea}-{stream}' + + disabled: false + + concurrent: true + + properties: + - logrotate-default + - throttle: + enabled: true + max-per-node: 1 + option: 'project' + + wrappers: + - build-name: + name: '$BUILD_NUMBER - Scenario: $DEPLOY_SCENARIO' + - timeout: + timeout: 300 + abort: true + - fix-workspace-permissions + + triggers: + - '{auto-trigger-name}' + + parameters: + - project-parameter: + project: '{project}' + branch: '{dovetail-branch}' + - '{SUT}-defaults' + - '{slave-label}-defaults' + - string: + name: DEPLOY_SCENARIO + default: 'os-nosdn-nofeature-ha' + - string: + name: DOCKER_TAG + default: '{docker-tag}' + description: 'Tag to pull dovetail docker image' + - string: + name: CI_DEBUG + default: 'true' + description: "Show debug output information" + - string: + name: TESTSUITE + default: '{testsuite}' + description: "dovetail testsuite to run" + - string: + name: TESTAREA + default: '{testarea}' + description: "dovetail testarea to run" - string: name: DOVETAIL_REPO_DIR default: "/home/opnfv/dovetail" diff --git a/jjb/dovetail/dovetail-run.sh b/jjb/dovetail/dovetail-run.sh index 2a9c73c2f..739da9eac 100755 --- a/jjb/dovetail/dovetail-run.sh +++ b/jjb/dovetail/dovetail-run.sh @@ -130,8 +130,18 @@ if [[ ! "${SUT_BRANCH}" =~ "danube" && ${INSTALLER_TYPE} == 'fuel' && ${DEPLOY_T fuel_ctl_ip=$(ssh 2>/dev/null ${fuel_ctl_ssh_options} "${ssh_user}@${INSTALLER_IP}" \ "sudo salt 'cfg*' pillar.get _param:openstack_control_address --out text| \ cut -f2 -d' '") + fuel_cmp_ip=$(ssh 2>/dev/null ${fuel_ctl_ssh_options} "${ssh_user}@${INSTALLER_IP}" \ + "sudo salt 'cmp001*' pillar.get _param:openstack_control_address --out text| \ + cut -f2 -d' '") + fuel_dbs_ip=$(ssh 2>/dev/null ${fuel_ctl_ssh_options} "${ssh_user}@${INSTALLER_IP}" \ + "sudo salt 'dbs01*' pillar.get _param:openstack_database_node01_address --out text| \ + cut -f2 -d' '") + fuel_msg_ip=$(ssh 2>/dev/null ${fuel_ctl_ssh_options} "${ssh_user}@${INSTALLER_IP}" \ + "sudo salt 'msg01*' pillar.get _param:openstack_message_queue_node01_address --out text| \ + cut -f2 -d' '") ipmi_index=$(ssh 2>/dev/null ${fuel_ctl_ssh_options} "${ssh_user}@${INSTALLER_IP}" \ "sudo salt 'ctl*' network.ip_addrs cidr=${fuel_ctl_ip} --out text | grep ${fuel_ctl_ip} | cut -c 5") + organization="$(cut -d'-' -f1 <<< "${NODE_NAME}")" pod_name="$(cut -d'-' -f2 <<< "${NODE_NAME}")" pdf_file=${pharos_repo}/labs/${organization}/${pod_name}.yaml @@ -148,7 +158,9 @@ nodes: role: Jumpserver, user: ${ssh_user}} - {ip: ${fuel_ctl_ip}, name: node1, key_filename: /home/opnfv/userconfig/pre_config/id_rsa, role: controller, user: ${ssh_user}, ipmi_ip: ${ipmiIp}, ipmi_user: ${ipmiUser}, ipmi_password: ${ipmiPass}} - +- {ip: ${fuel_msg_ip}, name: msg01, key_filename: /home/opnfv/userconfig/pre_config/id_rsa, role: controller, user: ${ssh_user}} +- {ip: ${fuel_cmp_ip}, name: cmp01, key_filename: /home/opnfv/userconfig/pre_config/id_rsa, role: controller, user: ${ssh_user}} +- {ip: ${fuel_dbs_ip}, name: dbs01, key_filename: /home/opnfv/userconfig/pre_config/id_rsa, role: controller, user: ${ssh_user}} EOF fi @@ -193,15 +205,21 @@ if [ -f ${DOVETAIL_CONFIG}/pod.yaml ]; then sudo chmod 666 ${DOVETAIL_CONFIG}/pod.yaml echo "Adapt process info for $INSTALLER_TYPE ..." if [ "$INSTALLER_TYPE" == "apex" ]; then - attack_process='rabbitmq_server' - else - attack_process='rabbitmq' - fi - cat << EOF >> ${DOVETAIL_CONFIG}/pod.yaml + cat << EOF >> ${DOVETAIL_CONFIG}/pod.yaml process_info: -- {testcase_name: dovetail.ha.rabbitmq, attack_process: ${attack_process}} - +- {testcase_name: yardstick.ha.rabbitmq, attack_process: rabbitmq_server} +- {testcase_name: yardstick.ha.cinder_api, attack_process: cinder_wsgi} EOF + elif [ "$INSTALLER_TYPE" == "fuel" ]; then + cat << EOF >> ${DOVETAIL_CONFIG}/pod.yaml +process_info: +- {testcase_name: yardstick.ha.cinder_api, attack_process: cinder-wsgi} +- {testcase_name: yardstick.ha.rabbitmq, attack_process: rabbitmq_server, attack_host: msg01} +- {testcase_name: yardstick.ha.neutron_l3_agent, attack_process: neutron-l3-agent, attack_host: cmp01} +- {testcase_name: yardstick.ha.database, attack_process: mysqld, attack_host: dbs01} +EOF + fi + echo "file ${DOVETAIL_CONFIG}/pod.yaml:" cat ${DOVETAIL_CONFIG}/pod.yaml else @@ -238,7 +256,7 @@ fi ubuntu_image=${image_path}/ubuntu-16.04-server-cloudimg-amd64-disk1.img if [[ ! -f ${ubuntu_image} ]]; then echo "Download image ubuntu-16.04-server-cloudimg-amd64-disk1.img ..." - wget -q -nc http://artifacts.opnfv.org/sdnvpn/ubuntu-16.04-server-cloudimg-amd64-disk1.img -P ${image_path} + wget -q -nc https://artifacts.opnfv.org/sdnvpn/ubuntu-16.04-server-cloudimg-amd64-disk1.img -P ${image_path} fi sudo cp ${ubuntu_image} ${DOVETAIL_IMAGES} @@ -345,11 +363,28 @@ else testsuite="--testsuite ${TESTSUITE}" fi -run_cmd="dovetail run ${testsuite} -d" +if [[ ${TESTAREA} == 'mandatory' ]]; then + testarea='--mandatory' +elif [[ ${TESTAREA} == 'optional' ]]; then + testarea="--optional" +elif [[ ${TESTAREA} == 'all' ]]; then + testarea="" +else + testarea="--testarea ${TESTAREA}" +fi + +run_cmd="dovetail run ${testsuite} ${testarea} --deploy-scenario ${DEPLOY_SCENARIO} -d -r" echo "Container exec command: ${run_cmd}" docker exec $container_id ${run_cmd} sudo cp -r ${DOVETAIL_HOME}/results ./ +result_package=$(find ${DOVETAIL_HOME} -name 'logs_*') +echo "Results package is ${result_package}" +for item in ${result_package}; +do + sudo mv ${item} ./results +done + # To make sure the file owner is the current user, for the copied results files in the above line echo "Change owner of result files ..." CURRENT_USER=${SUDO_USER:-$USER} @@ -360,7 +395,7 @@ sudo chown -R ${CURRENT_USER}:${PRIMARY_GROUP} ./results #remove useless files to save disk space sudo rm -rf ./results/workspace sudo rm -f ./results/yardstick.img -sudo rm -f ./results/tmp* +sudo rm -f ./results/bottlenecks/tmp* echo "Dovetail: done!" diff --git a/jjb/fuel/fuel-daily-jobs.yaml b/jjb/fuel/fuel-daily-jobs.yaml index e636538ef..07cdb7345 100644 --- a/jjb/fuel/fuel-daily-jobs.yaml +++ b/jjb/fuel/fuel-daily-jobs.yaml @@ -75,6 +75,8 @@ auto-trigger-name: 'fuel-{scenario}-{pod}-daily-{stream}-trigger' - 'os-nosdn-kvm_ovs_dpdk_bar-ha': auto-trigger-name: 'fuel-{scenario}-{pod}-daily-{stream}-trigger' + - 'os-nosdn-vpp-ha': + auto-trigger-name: 'fuel-{scenario}-{pod}-daily-{stream}-trigger' # NOHA scenarios - 'os-nosdn-nofeature-noha': auto-trigger-name: 'fuel-{scenario}-{pod}-daily-{stream}-trigger' @@ -96,6 +98,8 @@ auto-trigger-name: 'fuel-{scenario}-{pod}-daily-{stream}-trigger' - 'os-nosdn-kvm_ovs_dpdk_bar-noha': auto-trigger-name: 'fuel-{scenario}-{pod}-daily-{stream}-trigger' + - 'os-nosdn-vpp-noha': + auto-trigger-name: 'fuel-{scenario}-{pod}-daily-{stream}-trigger' # danube scenario for Dovetail only - 'os-odl_l2-bgpvpn-ha': auto-trigger-name: 'fuel-{scenario}-{pod}-daily-{stream}-trigger' @@ -111,6 +115,14 @@ stream: danube - scenario: os-odl-ovs-noha stream: fraser + - scenario: os-nosdn-vpp-ha + stream: fraser + - scenario: os-nosdn-vpp-noha + stream: fraser + - scenario: os-nosdn-vpp-ha + stream: danube + - scenario: os-nosdn-vpp-noha + stream: danube jobs: - 'fuel-{scenario}-{pod}-daily-{stream}' @@ -194,27 +206,19 @@ failure-threshold: 'never' unstable-threshold: 'FAILURE' # 1.here the stream means the SUT stream, dovetail stream is defined in its own job - # 2.testsuite proposed_tests here is for new test cases planning to add into OVP - # 3.run proposed_tests on Monday, Wednesday and Friday against ha scenario - # 4.testsuite default here is for the test cases already added into OVP - # 5.run default testsuite on Tuesday against ha scenario - # 6.not used for release criteria or compliance, only to debug the dovetail tool bugs + # 2.testsuite default here is for the test cases already added into OVP + # 3.run default testsuite mandatory test cases against ha scenario + # 4.run default testsuite optional test cases against ha scenario + # 5.not used for release criteria or compliance, only to debug the dovetail tool bugs - conditional-step: condition-kind: and condition-operands: - condition-kind: regex-match regex: '.*-ha' label: '{scenario}' - - condition-kind: day-of-week - day-selector: select-days - days: - MON: true - WED: true - FRI: true - use-build-time: true steps: - trigger-builds: - - project: 'dovetail-fuel-{pod}-proposed_tests-{stream}' + - project: 'dovetail-fuel-{pod}-default-mandatory-{stream}' current-parameters: false predefined-parameters: DEPLOY_SCENARIO={scenario} @@ -230,14 +234,9 @@ - condition-kind: regex-match regex: '.*-ha' label: '{scenario}' - - condition-kind: day-of-week - day-selector: select-days - days: - TUES: true - use-build-time: true steps: - trigger-builds: - - project: 'dovetail-fuel-{pod}-default-{stream}' + - project: 'dovetail-fuel-{pod}-default-optional-{stream}' current-parameters: false predefined-parameters: DEPLOY_SCENARIO={scenario} @@ -267,8 +266,7 @@ unstable-threshold: 'FAILURE' publishers: - - email: - recipients: peter.barabas@ericsson.com fzhadaev@mirantis.com + - email-fuel-ptl - email-jenkins-admins-on-failure - job-template: @@ -324,8 +322,7 @@ !include-raw-escape: ./fuel-deploy.sh publishers: - - email: - recipients: peter.barabas@ericsson.com fzhadaev@mirantis.com + - email-fuel-ptl - email-jenkins-admins-on-failure - report-provision-result @@ -410,6 +407,11 @@ name: 'fuel-os-nosdn-kvm_ovs_dpdk_bar-ha-baremetal-daily-master-trigger' triggers: - timed: '' # '30 8 * * *' +- trigger: + name: 'fuel-os-nosdn-vpp-ha-baremetal-daily-master-trigger' + triggers: + - timed: '' + # NOHA Scenarios - trigger: name: 'fuel-os-nosdn-nofeature-noha-baremetal-daily-master-trigger' @@ -451,6 +453,10 @@ name: 'fuel-os-nosdn-kvm_ovs_dpdk_bar-noha-baremetal-daily-master-trigger' triggers: - timed: '' +- trigger: + name: 'fuel-os-nosdn-vpp-noha-baremetal-daily-master-trigger' + triggers: + - timed: '' # ---------------------------------------------- # Triggers for job running on fuel-baremetal against fraser branch # ---------------------------------------------- @@ -458,7 +464,7 @@ - trigger: name: 'fuel-os-nosdn-nofeature-ha-baremetal-daily-fraser-trigger' triggers: - - timed: '' # '0 20 * * *' + - timed: '0 20 * * 1,3,7' - trigger: name: 'fuel-os-odl-nofeature-ha-baremetal-daily-fraser-trigger' triggers: @@ -567,6 +573,10 @@ name: 'fuel-os-nosdn-kvm_ovs_dpdk_bar-ha-virtual-daily-master-trigger' triggers: - timed: '' +- trigger: + name: 'fuel-os-nosdn-vpp-ha-virtual-daily-master-trigger' + triggers: + - timed: '' # NOHA Scenarios - trigger: name: 'fuel-os-nosdn-nofeature-noha-virtual-daily-master-trigger' @@ -608,6 +618,10 @@ name: 'fuel-os-nosdn-kvm_ovs_dpdk_bar-noha-virtual-daily-master-trigger' triggers: - timed: '' # '30 20 * * *' +- trigger: + name: 'fuel-os-nosdn-vpp-noha-virtual-daily-master-trigger' + triggers: + - timed: '' # ---------------------------------------------- # Triggers for job running on fuel-virtual against fraser branch # ---------------------------------------------- @@ -723,6 +737,10 @@ name: 'fuel-os-nosdn-kvm_ovs_dpdk_bar-ha-zte-pod1-daily-master-trigger' triggers: - timed: '' +- trigger: + name: 'fuel-os-nosdn-vpp-ha-zte-pod1-daily-master-trigger' + triggers: + - timed: '' # NOHA Scenarios - trigger: name: 'fuel-os-nosdn-nofeature-noha-zte-pod1-daily-master-trigger' @@ -764,3 +782,7 @@ name: 'fuel-os-nosdn-kvm_ovs_dpdk_bar-noha-zte-pod1-daily-master-trigger' triggers: - timed: '' +- trigger: + name: 'fuel-os-nosdn-vpp-noha-zte-pod1-daily-master-trigger' + triggers: + - timed: '' diff --git a/jjb/fuel/fuel-docker-jobs.yaml b/jjb/fuel/fuel-docker-jobs.yaml new file mode 100644 index 000000000..98ed66da7 --- /dev/null +++ b/jjb/fuel/fuel-docker-jobs.yaml @@ -0,0 +1,200 @@ +--- +############################################## +# job configuration for docker build and push +############################################## +- project: + + name: fuel-docker + + project: fuel + + stream: + - master: + branch: '{stream}' + disabled: false + - gambia: + branch: 'stable/{stream}' + disabled: true + + arch_tag: + - 'amd64': + slave_label: 'opnfv-build-ubuntu' + - 'arm64': + slave_label: 'opnfv-build-ubuntu-arm' + + # settings for jobs run in multijob phases + build-job-settings: &build-job-settings + current-parameters: false + git-revision: true + node-parameters: false + predefined-parameters: | + PUSH_IMAGE=$PUSH_IMAGE + COMMIT_ID=$COMMIT_ID + GERRIT_REFNAME=$GERRIT_REFNAME + kill-phase-on: FAILURE + abort-all-jobs: false + + manifest-job-settings: &manifest-job-settings + current-parameters: false + git-revision: true + node-parameters: false + predefined-parameters: + GERRIT_REFNAME=$GERRIT_REFNAME + kill-phase-on: FAILURE + abort-all-jobs: false + + jobs: + - "fuel-docker-{stream}" + - "fuel-docker-build-{arch_tag}-{stream}" + - "fuel-docker-manifest-{stream}" + +######################## +# job templates +######################## +- job-template: + name: 'fuel-docker-{stream}' + + project-type: multijob + + disabled: '{obj:disabled}' + + parameters: + - fuel-job-parameters: + project: '{project}' + branch: '{branch}' + slave_label: 'opnfv-build-ubuntu' + arch_tag: 'amd64' + + properties: + - throttle: + max-per-node: 1 + option: 'project' + + scm: + - git-scm + + triggers: + - pollscm: + cron: "*/25 * * * *" + - gerrit-trigger-tag-created: + project: '{project}' + + builders: + - multijob: + name: 'build fuel images' + execution-type: PARALLEL + projects: + - name: 'fuel-docker-build-amd64-{stream}' + <<: *build-job-settings + - name: 'fuel-docker-build-arm64-{stream}' + <<: *build-job-settings + - multijob: + name: 'publish fuel manifests' + condition: SUCCESSFUL + execution-type: PARALLEL + projects: + - name: 'fuel-docker-manifest-{stream}' + <<: *manifest-job-settings + + publishers: + - email-fuel-ptl + +- job-template: + name: 'fuel-docker-build-{arch_tag}-{stream}' + disabled: '{obj:disabled}' + parameters: + - fuel-job-parameters: + project: '{project}' + branch: '{branch}' + slave_label: '{slave_label}' + arch_tag: '{arch_tag}' + scm: + - git-scm + builders: + - shell: | + #!/bin/bash -ex + case "{stream}" in + "master") + tag="latest" ;; + *) + tag="{stream}" ;; + esac + sudo ./ci/build.sh $tag + +- job-template: + name: 'fuel-docker-manifest-{stream}' + + parameters: + - project-parameter: + project: '{project}' + branch: '{branch}' + - label: + name: SLAVE_LABEL + default: 'opnfv-build-ubuntu' + description: 'Slave label on Jenkins' + - string: + name: PROJECT + default: "{project}" + description: "Project name used to enable job conditions" + - string: + name: GIT_BASE + default: https://gerrit.opnfv.org/gerrit/$PROJECT + description: 'Git URL to use on this Jenkins Slave' + - string: + name: REPO + default: "opnfv" + description: "Repository name for fuel-docker images" + + + disabled: '{obj:disabled}' + + builders: + - shell: | + #!/bin/bash -ex + case "{stream}" in + "master") + tag="latest" ;; + *) + tag="{stream}" ;; + esac + sudo manifest-tool push from-args \ + --platforms linux/amd64,linux/arm64 \ + --template $REPO/fuel:saltmaster-reclass-ARCH-$tag \ + --target $REPO/fuel:saltmaster-reclass-$tag + exit $? + +# parameter macro +- parameter: + name: fuel-job-parameters + parameters: + - project-parameter: + project: '{project}' + branch: '{branch}' + - label: + name: SLAVE_LABEL + default: '{slave_label}' + description: 'Slave label on Jenkins' + - string: + name: GIT_BASE + default: https://gerrit.opnfv.org/gerrit/$PROJECT + description: 'Git URL to use on this Jenkins Slave' + - string: + name: PUSH_IMAGE + default: "true" + description: "To enable/disable pushing the image to Dockerhub." + - string: + name: COMMIT_ID + default: "" + description: "commit id to make a snapshot docker image" + - string: + name: GERRIT_REFNAME + default: "" + description: "Docker tag to be built, e.g. refs/tags/5.0.0, refs/tags/opnfv-5.0.0, refs/tags/5.0.RC1" + - string: + name: PROJECT + default: "{project}" + description: "Project name used to enable job conditions" + - string: + name: REPO + default: "opnfv" + description: "Repository name for fuel-docker images" diff --git a/jjb/fuel/fuel-verify-jobs.yaml b/jjb/fuel/fuel-verify-jobs.yaml index f48822d7f..8eb5ab0cd 100644 --- a/jjb/fuel/fuel-verify-jobs.yaml +++ b/jjb/fuel/fuel-verify-jobs.yaml @@ -141,19 +141,6 @@ node-parameters: true kill-phase-on: NEVER abort-all-job: true - - name: 'functest-fuel-virtual-suite-{stream}' - current-parameters: false - predefined-parameters: | - FUNCTEST_MODE=testcase - FUNCTEST_SUITE_NAME=vping_ssh - DEPLOY_SCENARIO=$DEPLOY_SCENARIO - BRANCH=$BRANCH - GERRIT_REFSPEC=$GERRIT_REFSPEC - GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER - GERRIT_CHANGE_COMMIT_MESSAGE=$GERRIT_CHANGE_COMMIT_MESSAGE - node-parameters: true - kill-phase-on: NEVER - abort-all-job: true - job-template: name: 'fuel-verify-{phase}-{stream}' diff --git a/jjb/fuel/fuel-weekly-jobs.yaml b/jjb/fuel/fuel-weekly-jobs.yaml index aca625d2b..c6b921729 100644 --- a/jjb/fuel/fuel-weekly-jobs.yaml +++ b/jjb/fuel/fuel-weekly-jobs.yaml @@ -118,8 +118,7 @@ unstable-threshold: 'FAILURE' publishers: - - email: - recipients: peter.barabas@ericsson.com fzhadaev@mirantis.com + - email-fuel-ptl - email-jenkins-admins-on-failure - job-template: @@ -173,8 +172,7 @@ !include-raw-escape: ./fuel-deploy.sh publishers: - - email: - recipients: peter.barabas@ericsson.com fzhadaev@mirantis.com + - email-fuel-ptl - email-jenkins-admins-on-failure - report-provision-result diff --git a/jjb/functest/functest-alpine.sh b/jjb/functest/functest-alpine.sh index 7adfdffc7..744a7de4e 100755 --- a/jjb/functest/functest-alpine.sh +++ b/jjb/functest/functest-alpine.sh @@ -61,8 +61,10 @@ run_test() { case ${test_name} in connection_check|tenantnetwork1|tenantnetwork2|vmready1|vmready2|singlevm1|singlevm2|vping_ssh|vping_userdata|cinder_test|odl|api_check|snaps_health_check) FUNCTEST_IMAGE=${REPO}/functest-healthcheck:${DOCKER_TAG} ;; - tempest_smoke|rally_sanity|refstack_defcore|patrole|shaker|snaps_smoke|neutron_trunk|barbican) + tempest_smoke|rally_sanity|refstack_defcore|patrole|snaps_smoke|neutron_trunk|barbican) FUNCTEST_IMAGE=${REPO}/functest-smoke:${DOCKER_TAG} ;; + shaker|vmtp) + FUNCTEST_IMAGE=${REPO}/functest-benchmarking:${DOCKER_TAG} ;; tempest_full|rally_full) FUNCTEST_IMAGE=${REPO}/functest-components:${DOCKER_TAG} ;; cloudify_ims|vyos_vrouter|juju_epc) @@ -170,13 +172,13 @@ else if [[ ${BRANCH} == "stable/fraser" ]]; then tiers=(healthcheck smoke features vnf parser) else - tiers=(healthcheck smoke features vnf) + tiers=(healthcheck smoke benchmarking features vnf) fi else if [[ ${BRANCH} == "stable/fraser" ]]; then tiers=(healthcheck smoke features parser) else - tiers=(healthcheck smoke features) + tiers=(healthcheck smoke benchmarking features) fi fi run_tiers ${tiers} diff --git a/jjb/functest/functest-daily-jobs.yaml b/jjb/functest/functest-daily-jobs.yaml index 1239db5af..763d1d4f0 100644 --- a/jjb/functest/functest-daily-jobs.yaml +++ b/jjb/functest/functest-daily-jobs.yaml @@ -290,6 +290,8 @@ - 'patrole' - 'odl' - 'snaps_smoke' + - 'shaker' + - 'vmtp' - 'neutron_trunk' - 'doctor-notification' - 'bgpvpn' @@ -308,6 +310,7 @@ choices: - 'healthcheck' - 'smoke' + - 'benchmarking' - 'features' - 'components' - 'vnf' diff --git a/jjb/functest/functest-docker.yaml b/jjb/functest/functest-docker.yaml index 74738f280..fc2fd3cae 100644 --- a/jjb/functest/functest-docker.yaml +++ b/jjb/functest/functest-docker.yaml @@ -31,6 +31,7 @@ - 'components' - 'parser' - 'smoke' + - 'benchmarking' - 'vnf' exclude: @@ -38,6 +39,8 @@ image: 'parser' - stream: 'fraser' image: 'tempest' + - stream: 'fraser' + image: 'benchmarking' # settings for jobs run in multijob phases build-job-settings: &build-job-settings @@ -152,6 +155,10 @@ <<: *build-job-settings - name: 'functest-smoke-docker-build-arm64-{stream}' <<: *build-job-settings + - name: 'functest-benchmarking-docker-build-amd64-{stream}' + <<: *build-job-settings + - name: 'functest-benchmarking-docker-build-arm64-{stream}' + <<: *build-job-settings - name: 'functest-vnf-docker-build-amd64-{stream}' <<: *build-job-settings - name: 'functest-vnf-docker-build-arm64-{stream}' @@ -171,6 +178,8 @@ <<: *manifest-job-settings - name: 'functest-smoke-docker-manifest-{stream}' <<: *manifest-job-settings + - name: 'functest-benchmarking-docker-manifest-{stream}' + <<: *manifest-job-settings - name: 'functest-vnf-docker-manifest-{stream}' <<: *manifest-job-settings diff --git a/jjb/functest/functest-kubernetes-docker.yaml b/jjb/functest/functest-kubernetes-docker.yaml index 2085251c6..5fbb3be7e 100644 --- a/jjb/functest/functest-kubernetes-docker.yaml +++ b/jjb/functest/functest-kubernetes-docker.yaml @@ -19,13 +19,15 @@ arch_tag: - 'amd64': slave_label: 'opnfv-build-ubuntu' + - 'arm64': + slave_label: 'opnfv-build-ubuntu-arm' # yamllint disable rule:key-duplicates image: - 'core' - 'healthcheck' - - 'smoke' - 'features' + - 'smoke' # settings for jobs run in multijob phases build-job-settings: &build-job-settings @@ -53,6 +55,7 @@ jobs: - "functest-kubernetes-docker-{stream}" - "functest-kubernetes-{image}-docker-build-{arch_tag}-{stream}" + - "functest-kubernetes-{image}-docker-manifest-{stream}" ######################## # job templates @@ -92,23 +95,54 @@ projects: - name: 'functest-kubernetes-core-docker-build-amd64-{stream}' <<: *build-job-settings + - name: 'functest-kubernetes-core-docker-build-arm64-{stream}' + <<: *build-job-settings - multijob: - name: 'build functest-kubernetes-[healthcheck,features] image' + name: 'publish functest-kubernetes-core manifests' + execution-type: PARALLEL + projects: + - name: 'functest-kubernetes-core-docker-manifest-{stream}' + <<: *manifest-job-settings + - multijob: + name: 'build functest-kubernetes-healthcheck images' execution-type: PARALLEL projects: - name: 'functest-kubernetes-healthcheck-docker-build-amd64-{stream}' <<: *build-job-settings - - name: 'functest-kubernetes-features-docker-build-amd64-{stream}' + - name: 'functest-kubernetes-healthcheck-docker-build-arm64-{stream}' <<: *build-job-settings - multijob: - name: 'build functest-kubernetes-smoke image' + name: 'publish functest-kubernetes-healthcheck manifests' execution-type: PARALLEL projects: + - name: 'functest-kubernetes-healthcheck-docker-manifest-{stream}' + <<: *manifest-job-settings + - multijob: + name: 'build all functest-kubernetes images' + condition: SUCCESSFUL + execution-type: PARALLEL + projects: + - name: 'functest-kubernetes-features-docker-build-amd64-{stream}' + <<: *build-job-settings + - name: 'functest-kubernetes-features-docker-build-arm64-{stream}' + <<: *build-job-settings - name: 'functest-kubernetes-smoke-docker-build-amd64-{stream}' <<: *build-job-settings + - name: 'functest-kubernetes-smoke-docker-build-arm64-{stream}' + <<: *build-job-settings + - multijob: + name: 'publish all manifests' + condition: SUCCESSFUL + execution-type: PARALLEL + projects: + - name: 'functest-kubernetes-features-docker-manifest-{stream}' + <<: *manifest-job-settings + - name: 'functest-kubernetes-smoke-docker-manifest-{stream}' + <<: *manifest-job-settings publishers: - 'functest-kubernetes-amd64-recipients' + - 'functest-kubernetes-arm64-recipients' - job-template: name: 'functest-kubernetes-{image}-docker-build-{arch_tag}-{stream}' @@ -124,7 +158,54 @@ builders: - shell: | #!/bin/bash -ex - sudo amd64_dirs=docker/{image} bash ./build.sh + case "{arch_tag}" in + "arm64") + sudo amd64_dirs= arm64_dirs=docker/{image} bash ./build.sh ;; + *) + sudo amd64_dirs=docker/{image} arm64_dirs= bash ./build.sh ;; + esac + exit $? + +- job-template: + name: 'functest-kubernetes-{image}-docker-manifest-{stream}' + + parameters: + - project-parameter: + project: '{project}' + branch: '{branch}' + - label: + name: SLAVE_LABEL + default: 'opnfv-build-ubuntu' + description: 'Slave label on Jenkins' + - string: + name: PROJECT + default: "{project}" + description: "Project name used to enable job conditions" + - string: + name: GIT_BASE + default: https://gerrit.opnfv.org/gerrit/$PROJECT + description: 'Git URL to use on this Jenkins Slave' + - string: + name: REPO + default: "opnfv" + description: "Repository name for functest-kubernetes images" + + + disabled: '{obj:disabled}' + + builders: + - shell: | + #!/bin/bash -ex + case "{stream}" in + "master") + tag="latest" ;; + *) + tag="{stream}" ;; + esac + sudo manifest-tool push from-args \ + --platforms linux/amd64,linux/arm64 \ + --template $REPO/functest-kubernetes-{image}:ARCH-$tag \ + --target $REPO/functest-kubernetes-{image}:$tag exit $? # parameter macro @@ -169,7 +250,17 @@ - string: name: REPO default: "opnfv" - description: "Repository name for functest images" + description: "Repository name for functest-kubernetes images" + +# publisher macros +- publisher: + name: 'functest-kubernetes-arm64-recipients' + publishers: + - email: + recipients: > + cristina.pauna@enea.com + alexandru.avadanii@enea.com + delia.popescu@enea.com - publisher: name: 'functest-kubernetes-amd64-recipients' diff --git a/jjb/global/releng-macros.yaml b/jjb/global/releng-macros.yaml index b282cff4f..8055fdbdf 100644 --- a/jjb/global/releng-macros.yaml +++ b/jjb/global/releng-macros.yaml @@ -720,7 +720,7 @@ - email-ext: <<: *email_ptl_defaults recipients: > - gelkinbard@mirantis.com + Alexandru.Avadanii@enea.com - publisher: name: 'email-functest-ptl' @@ -883,7 +883,7 @@ - email-ext: <<: *email_ptl_defaults recipients: > - fatih.degirmenci@ericsson.com + tbramwell@linuxfoundation.org - publisher: name: 'email-releng-anteater-ptl' <<: *email_releng_ptl_defaults @@ -900,7 +900,11 @@ <<: *email_releng_ptl_defaults - publisher: name: 'email-releng-xci-ptl' - <<: *email_releng_ptl_defaults + publishers: + - email-ext: + <<: *email_ptl_defaults + recipients: > + fatih.degirmenci@ericsson.com - publisher: name: 'email-samplevnf-ptl' diff --git a/jjb/releng/opnfv-docker.yaml b/jjb/releng/opnfv-docker.yaml index c73bae371..c31860a16 100644 --- a/jjb/releng/opnfv-docker.yaml +++ b/jjb/releng/opnfv-docker.yaml @@ -107,6 +107,12 @@ dockerfile: 'Dockerfile' <<: *master <<: *other-receivers + - 'clover-sample-app': + project: 'clover' + dockerdir: 'samples/scenarios/sample_app' + dockerfile: 'Dockerfile' + <<: *master + <<: *other-receivers - 'cperf': project: 'cperf' <<: *master @@ -115,6 +121,16 @@ project: 'dovetail' <<: *master <<: *other-receivers + - 'dovetail-webportal-api': + project: 'dovetail-webportal' + dockerfile: 'Dockerfile.api' + <<: *master + <<: *other-receivers + - 'dovetail-webportal-web': + project: 'dovetail-webportal' + dockerfile: 'Dockerfile.web' + <<: *master + <<: *other-receivers - 'nfvbench': project: 'nfvbench' <<: *master @@ -243,11 +259,6 @@ project: 'yardstick' <<: *fraser <<: *other-receivers - # projects with jobs for danube - - 'dovetail': - project: 'dovetail' - <<: *danube - <<: *other-receivers # projects with jobs for fraser - 'bottlenecks': project: 'bottlenecks' diff --git a/jjb/releng/opnfv-utils.yaml b/jjb/releng/opnfv-utils.yaml index c80766a80..71013f4ff 100644 --- a/jjb/releng/opnfv-utils.yaml +++ b/jjb/releng/opnfv-utils.yaml @@ -41,6 +41,7 @@ # yamllint disable rule:line-length - shell: | #!/bin/bash + sudo systemctl restart docker (docker ps -q; docker ps -aq) | sort | uniq -u | xargs --no-run-if-empty docker rm docker images -f dangling=true -q | xargs --no-run-if-empty docker rmi diff --git a/jjb/xci/xci-daily-jobs.yaml b/jjb/xci/xci-daily-jobs.yaml index c1132ab85..5f75b0e28 100644 --- a/jjb/xci/xci-daily-jobs.yaml +++ b/jjb/xci/xci-daily-jobs.yaml @@ -1,15 +1,9 @@ --- -# ------------------------------- -# These jobs run on a daily basis and deploy OpenStack -# using the pinned versions of opnfv/releng, openstack/bifrost -# and openstack/openstack-ansible. Due to this, there is no -# version/branch is set/passed to jobs and instead the versions -# are checked out based on what is configured. -# ------------------------------- - project: project: 'releng-xci' name: 'xci-daily' + # ------------------------------- # Branch Anchors # ------------------------------- @@ -17,55 +11,87 @@ stream: master opnfv-releng-version: master gs-pathname: '' + # ------------------------------- # Scenarios # ------------------------------- scenario: - - 'os-nosdn-nofeature-ha': - auto-trigger-name: 'daily-trigger-disabled' - xci-flavor: 'ha' - 'os-nosdn-nofeature-noha': auto-trigger-name: 'daily-trigger-disabled' - xci-flavor: 'noha' - - 'os-odl-sfc-ha': + deploy-scenario: 'os-nosdn-nofeature' + installer-type: 'osa' + xci-flavor: 'mini' + - 'os-nosdn-osm-noha': auto-trigger-name: 'daily-trigger-disabled' - xci-flavor: 'ha' + deploy-scenario: 'os-nosdn-osm' + installer-type: 'osa' + xci-flavor: 'mini' + - 'os-odl-nofeature-noha': + auto-trigger-name: 'daily-trigger-disabled' + deploy-scenario: 'os-odl-nofeature' + installer-type: 'osa' + xci-flavor: 'mini' + - 'os-odl-bgpvpn-noha': + auto-trigger-name: 'daily-trigger-disabled' + deploy-scenario: 'os-odl-bgpvpn' + installer-type: 'osa' + xci-flavor: 'mini' - 'os-odl-sfc-noha': auto-trigger-name: 'daily-trigger-disabled' - xci-flavor: 'noha' + deploy-scenario: 'os-odl-sfc' + installer-type: 'osa' + xci-flavor: 'mini' + - 'k8-nosdn-nofeature-noha': + auto-trigger-name: 'daily-trigger-disabled' + deploy-scenario: 'k8-nosdn-nofeature' + installer-type: 'kubespray' + xci-flavor: 'mini' + - 'k8-canal-nofeature-noha': + auto-trigger-name: 'daily-trigger-disabled' + deploy-scenario: 'k8-canal-nofeature' + installer-type: 'kubespray' + xci-flavor: 'mini' + - 'k8-calico-nofeature-noha': + auto-trigger-name: 'daily-trigger-disabled' + deploy-scenario: 'k8-calico-nofeature' + installer-type: 'kubespray' + xci-flavor: 'mini' + - 'k8-contiv-nofeature-noha': + auto-trigger-name: 'daily-trigger-disabled' + deploy-scenario: 'k8-contiv-nofeature' + installer-type: 'kubespray' + xci-flavor: 'mini' + - 'k8-flannel-nofeature-noha': + auto-trigger-name: 'daily-trigger-disabled' + deploy-scenario: 'k8-flannel-nofeature' + installer-type: 'kubespray' + xci-flavor: 'mini' + - 'k8-nosdn-istio-noha': + auto-trigger-name: 'daily-trigger-disabled' + deploy-scenario: 'k8-nosdn-istio' + installer-type: 'kubespray' + xci-flavor: 'mini' + # ------------------------------- # XCI PODs # ------------------------------- pod: - virtual: <<: *master + # ------------------------------- # Supported Distros # ------------------------------- distro: - - 'xenial': + - 'ubuntu': disabled: false - slave-label: xci-xenial-virtual - dib-os-release: 'xenial' - dib-os-element: 'ubuntu-minimal' - # yamllint disable rule:line-length - dib-os-packages: 'vlan,vim,less,bridge-utils,sudo,language-pack-en,iputils-ping,rsyslog,curl,python,debootstrap,ifenslave,ifenslave-2.6,lsof,lvm2,tcpdump,nfs-kernel-server,chrony,iptabls' - # yamllint enable rule:line-length - extra-dib-elements: 'openssh-server' - - 'centos7': + slave-label: xci-virtual + - 'centos': disabled: true - slave-label: xci-centos7-virtual - dib-os-release: '7' - dib-os-element: 'centos7' - dib-os-packages: 'vim,less,bridge-utils,iputils,rsyslog,curl' - extra-dib-elements: 'openssh-server' - - 'suse': - disabled: true - slave-label: xci-suse-virtual - dib-os-release: '42.3' - dib-os-element: 'opensuse-minimal' - dib-os-packages: 'vim,less,bridge-utils,iputils,rsyslog,curl' - extra-dib-elements: 'openssh-server' + slave-label: xci-virtual + - 'opensuse': + disabled: false + slave-label: xci-virtual # ------------------------------- # Phases @@ -73,6 +99,8 @@ phase: - 'deploy' - 'functest' + - 'yardstick' + # ------------------------------- # jobs # ------------------------------- @@ -94,7 +122,10 @@ - build-blocker: use-build-blocker: true blocking-jobs: + - '^xci-verify.*' + - '^xci-merge.*' - '^xci-os.*' + - '^xci-k8.*' - '^xci-deploy.*' - '^xci-functest.*' - '^bifrost-.*periodic.*' @@ -105,33 +136,56 @@ parameters: - string: name: DEPLOY_SCENARIO - default: '{scenario}' + default: '{deploy-scenario}' + - string: + name: INSTALLER_TYPE + default: '{installer-type}' - string: name: XCI_FLAVOR default: '{xci-flavor}' - label: name: SLAVE_LABEL default: '{slave-label}' + - string: + name: XCI_DISTRO + default: '{distro}' + - string: + name: FUNCTEST_MODE + default: 'tier' + - string: + name: FUNCTEST_SUITE_NAME + default: 'smoke' - string: name: CI_LOOP default: 'daily' + - string: + name: GIT_BASE + default: https://gerrit.opnfv.org/gerrit/$PROJECT + description: 'Git URL to use on this Jenkins Slave' triggers: - '{auto-trigger-name}' wrappers: + - ssh-agent-wrapper + - build-timeout: + timeout: 240 - fix-workspace-permissions builders: - description-setter: - description: "Built on $NODE_NAME" + description: "Scenario: $DEPLOY_SCENARIO | Node: $NODE_NAME" - trigger-builds: - project: 'xci-deploy-{pod}-{distro}-daily-{stream}' current-parameters: false predefined-parameters: | DEPLOY_SCENARIO=$DEPLOY_SCENARIO + INSTALLER_TYPE=$INSTALLER_TYPE XCI_FLAVOR=$XCI_FLAVOR CI_LOOP=$CI_LOOP + XCI_DISTRO=$XCI_DISTRO + FUNCTEST_MODE=$FUNCTEST_MODE + FUNCTEST_SUITE_NAME=$FUNCTEST_SUITE_NAME same-node: true block: true - trigger-builds: @@ -139,8 +193,27 @@ current-parameters: false predefined-parameters: | DEPLOY_SCENARIO=$DEPLOY_SCENARIO + INSTALLER_TYPE=$INSTALLER_TYPE + XCI_FLAVOR=$XCI_FLAVOR + CI_LOOP=$CI_LOOP + XCI_DISTRO=$XCI_DISTRO + FUNCTEST_MODE=$FUNCTEST_MODE + FUNCTEST_SUITE_NAME=$FUNCTEST_SUITE_NAME + same-node: true + block: true + block-thresholds: + build-step-failure-threshold: 'never' + failure-threshold: 'never' + unstable-threshold: 'FAILURE' + - trigger-builds: + - project: 'xci-yardstick-{pod}-{distro}-daily-{stream}' + current-parameters: false + predefined-parameters: | + DEPLOY_SCENARIO=$DEPLOY_SCENARIO + INSTALLER_TYPE=$INSTALLER_TYPE XCI_FLAVOR=$XCI_FLAVOR CI_LOOP=$CI_LOOP + XCI_DISTRO=$XCI_DISTRO same-node: true block: true block-thresholds: @@ -150,42 +223,37 @@ publishers: # yamllint disable rule:line-length - - email: - recipients: fatih.degirmenci@ericsson.com yroblamo@redhat.com mchandras@suse.de jack.morgan@intel.com julienjut@gmail.com # yamllint enable rule:line-length - email-jenkins-admins-on-failure - - postbuildscript: - builders: - - role: BOTH - build-on: - - ABORTED - - FAILURE - - NOT_BUILT - - SUCCESS - - UNSTABLE - build-steps: - - shell: !include-raw: ./xci-cleanup.sh - mark-unstable-if-failed: true - job-template: name: 'xci-{phase}-{pod}-{distro}-daily-{stream}' - disabled: '{obj:disabled}' + disabled: false - concurrent: false + concurrent: true properties: - build-blocker: use-build-blocker: true blocking-jobs: + - '^xci-verify.*' + - '^xci-merge.*' - '^xci-deploy.*' - '^xci-functest.*' - '^bifrost-.*periodic.*' - '^osa-.*periodic.*' block-level: 'NODE' + - throttle: + enabled: true + max-per-node: 1 + option: 'project' - logrotate-default wrappers: + - ssh-agent-wrapper + - build-timeout: + timeout: 240 - fix-workspace-permissions scm: @@ -195,77 +263,119 @@ - project-parameter: project: '{project}' branch: '{opnfv-releng-version}' - - string: - name: GIT_BASE - default: https://gerrit.opnfv.org/gerrit/$PROJECT - string: name: DEPLOY_SCENARIO - default: 'os-nosdn-nofeature-ha' - - string: - name: XCI_FLAVOR - default: 'ha' - - string: - name: DISTRO - default: '{distro}' - - string: - name: DIB_OS_RELEASE - default: '{dib-os-release}' - - string: - name: DIB_OS_ELEMENT - default: '{dib-os-element}' + default: 'os-nosdn-nofeature-noha' - string: - name: DIB_OS_PACKAGES - default: '{dib-os-packages}' + name: INSTALLER_TYPE + default: 'osa' - string: - name: EXTRA_DIB_ELEMENTS - default: '{extra-dib-elements}' + name: XCI_FLAVOR + default: 'mini' - string: - name: CLEAN_DIB_IMAGES - default: 'true' + name: XCI_DISTRO + default: 'ubuntu' - label: name: SLAVE_LABEL default: '{slave-label}' - - string: - name: CI_LOOP - default: 'daily' - - string: - name: INSTALLER_TYPE - default: 'osa' - string: name: FUNCTEST_MODE - default: 'daily' + default: 'tier' - string: name: FUNCTEST_SUITE_NAME + default: 'smoke' + - string: + name: CI_LOOP default: 'daily' - description: "Daily suite name to run" + - string: + name: GIT_BASE + default: https://gerrit.opnfv.org/gerrit/$PROJECT builders: - description-setter: - description: "Built on $NODE_NAME - Scenario: $DEPLOY_SCENARIO" - - 'xci-{phase}-builder' + description: "Scenario: $DEPLOY_SCENARIO | Node: $NODE_NAME" + - 'xci-daily-{phase}-macro' # -------------------------- # builder macros # -------------------------- +# These need to be properly fixed once the basic deployment, functest, and +# yardstick jobs are working outside of clean vm. +# One of the ugly fixes is moving test preparation step into the +# deployment job itself since test preparation requires some of the +# things from deployment job. With clean VM, this wasn't an issue +# since everything was on clean VM. When we move things out of clean +# VM, things are done in workspaces of the jobs that are different. +# +# Apart from these things, we will need to go through the scripts +# used for verify jobs and make them updated in order to be able to +# use them for jobs that don't use clean VM. - builder: - name: xci-deploy-builder + name: 'xci-daily-deploy-macro' builders: - - shell: - !include-raw: ./xci-deploy.sh + - shell: | + #!/bin/bash + set -o errexit + set -o pipefail + + echo "Cleaning the leftovers from the earlier run" + echo "---------------------------------------------------------------------------------" + for vm in $(sudo virsh list --all --name | grep -v xci_vm); do + echo "Removing $vm" + sudo virsh destroy $vm > /dev/null 2>&1 || true + sudo virsh undefine $vm > /dev/null 2>&1 || true + done + echo "---------------------------------------------------------------------------------" + + cd $WORKSPACE/xci && ./xci-deploy.sh + + echo "Prepare OPNFV VM for Tests" + echo "---------------------------------------------------------------------------------" + export XCI_PATH=$WORKSPACE + export XCI_VENV=${XCI_PATH}/venv + source $XCI_VENV/bin/activate + while read var; do + declare -x "\${var}" 2>/dev/null + echo $var + done < ${XCI_PATH}/.cache/xci.env && cd ${XCI_PATH}/xci && \ + ansible-playbook -i playbooks/dynamic_inventory.py playbooks/prepare-tests.yml + ssh root@192.168.122.2 "/root/prepare-tests.sh" + echo "---------------------------------------------------------------------------------" - builder: - name: xci-functest-builder + name: 'xci-daily-functest-macro' builders: - shell: | #!/bin/bash + set -o pipefail + + ssh root@192.168.122.2 "/root/run-functest.sh" + functest_exit=$? + + case ${DEPLOY_SCENARIO[0]} in + os-*) + FUNCTEST_LOG=/root/functest-results/functest.log + ;; + k8-*) + FUNCTEST_LOG=/root/functest-results/functest-kubernetes.log + ;; + *) + echo "Unable to determine the installer. Exiting!" + exit $functest_exit + ;; + esac - echo "Hello World!" + echo "Functest log" + echo "---------------------------------------------------------------------------------" + ssh root@192.168.122.2 "cat $FUNCTEST_LOG" + echo "---------------------------------------------------------------------------------" + exit ${functest_exit} +- builder: + name: 'xci-daily-yardstick-macro' + builders: + - shell: | + #!/bin/bash + set -o errexit + set -o pipefail -# this will be enabled once the xci is prepared -# - builder: -# name: xci-functest-builder -# builders: -# - shell: -# !include-raw: -# - ../../utils/fetch_os_creds.sh -# - ../functest/functest-alpine.sh + ssh root@192.168.122.2 "/root/run-yardstick.sh" diff --git a/jjb/xci/xci-merge-jobs.yaml b/jjb/xci/xci-merge-jobs.yaml index cb438ad5d..afa88a224 100644 --- a/jjb/xci/xci-merge-jobs.yaml +++ b/jjb/xci/xci-merge-jobs.yaml @@ -223,6 +223,8 @@ blocking-jobs: - 'xci-verify-{distro}-.*' - 'xci-merge-{distro}-.*' + - 'xci-os.*' + - 'xci-k8.*' - 'openstack-bifrost-verify-{distro}-.*' - 'xci-osa-verify-{distro}-.*' - 'xci-osa-periodic-{distro}-.*' @@ -290,6 +292,8 @@ GERRIT_REFSPEC=$GERRIT_REFSPEC GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER GERRIT_CHANGE_COMMIT_MESSAGE=$GERRIT_CHANGE_COMMIT_MESSAGE + FUNCTEST_MODE=$FUNCTEST_MODE + FUNCTEST_SUITE_NAME=$FUNCTEST_SUITE_NAME node-parameters: true kill-phase-on: '{kill-phase-on}' abort-all-job: '{abort-all-job}' diff --git a/jjb/xci/xci-run-functest.sh b/jjb/xci/xci-run-functest.sh index f46abe043..992194ca3 100755 --- a/jjb/xci/xci-run-functest.sh +++ b/jjb/xci/xci-run-functest.sh @@ -47,18 +47,22 @@ fi export XCI_PATH=/home/devuser/releng-xci export XCI_VENV=${XCI_PATH}/venv -ssh -F $HOME/.ssh/${DISTRO}-xci-vm-config ${DISTRO}_xci_vm "source $XCI_VENV/bin/activate; while read var; do declare -x \"\${var}\" 2>/dev/null; done < ${XCI_PATH}/.cache/xci.env && cd releng-xci/xci && ansible-playbook -i playbooks/dynamic_inventory.py playbooks/prepare-functest.yml" -echo "Running functest" +ssh -F $HOME/.ssh/${DISTRO}-xci-vm-config ${DISTRO}_xci_vm "source $XCI_VENV/bin/activate; \ + while read var; do declare -x \"\${var}\" 2>/dev/null; done < ${XCI_PATH}/.cache/xci.env && \ + cd releng-xci/xci && ansible-playbook -i playbooks/dynamic_inventory.py playbooks/prepare-tests.yml" +echo "Prepare OPNFV VM for Tests" +ssh -F $HOME/.ssh/${DISTRO}-xci-vm-config ${DISTRO}_xci_vm_opnfv "/root/prepare-tests.sh" +echo "Running Functest" ssh -F $HOME/.ssh/${DISTRO}-xci-vm-config ${DISTRO}_xci_vm_opnfv "/root/run-functest.sh" # Record exit code functest_exit=$? case ${DEPLOY_SCENARIO[0]} in os-*) - FUNCTEST_LOG=/root/results/functest.log + FUNCTEST_LOG=/root/functest-results/functest.log ;; k8-*) - FUNCTEST_LOG=/root/results/functest-kubernetes.log + FUNCTEST_LOG=/root/functest-results/functest-kubernetes.log ;; *) echo "Unable to determine the installer. Exiting!" diff --git a/jjb/xci/xci-set-scenario.sh b/jjb/xci/xci-set-scenario.sh index 7bc45f1c2..59c1ebf6e 100755 --- a/jjb/xci/xci-set-scenario.sh +++ b/jjb/xci/xci-set-scenario.sh @@ -8,6 +8,7 @@ # http://www.apache.org/licenses/LICENSE-2.0 ############################################################################## set -o pipefail +set -x #---------------------------------------------------------------------- # This script is used by CI and executed by Jenkins jobs. @@ -166,9 +167,10 @@ WORK_DIRECTORY=/tmp/$GERRIT_CHANGE_NUMBER/$DISTRO if [[ $GERRIT_PROJECT == "releng-xci" ]]; then determine_default_scenario +else + determine_scenario fi override_scenario -determine_scenario # ensure single scenario is impacted if [[ $(IFS=$'\n' echo ${DEPLOY_SCENARIO[@]} | wc -w) != 1 ]]; then diff --git a/jjb/xci/xci-verify-jobs.yaml b/jjb/xci/xci-verify-jobs.yaml index baff0c99e..020ec8bac 100644 --- a/jjb/xci/xci-verify-jobs.yaml +++ b/jjb/xci/xci-verify-jobs.yaml @@ -67,6 +67,8 @@ blocking-jobs: - 'xci-verify-{distro}-.*' - 'xci-merge-{distro}-.*' + - 'xci-os-.*' + - 'xci-k8-.*' - 'openstack-bifrost-verify-{distro}-.*' - 'xci-osa-verify-{distro}-.*' - 'xci-osa-periodic-{distro}-.*' @@ -179,6 +181,8 @@ GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER GERRIT_CHANGE_COMMIT_MESSAGE=$GERRIT_CHANGE_COMMIT_MESSAGE CI_LOOP=$CI_LOOP + FUNCTEST_MODE=$FUNCTEST_MODE + FUNCTEST_SUITE_NAME=$FUNCTEST_SUITE_NAME node-parameters: true kill-phase-on: FAILURE abort-all-job: true diff --git a/utils/artifacts.opnfv.org.sh b/utils/artifacts.opnfv.org.sh index 1984b4992..2b35006d4 100755 --- a/utils/artifacts.opnfv.org.sh +++ b/utils/artifacts.opnfv.org.sh @@ -31,7 +31,7 @@ do for index in $(gsutil ls -l gs://artifacts.opnfv.org/logs/"$project"/ |awk 'NF==1'| sed s,gs://artifacts.opnfv.org/,, ) do index="$(echo ${index%/*} | sed s,/,_,g)" - echo "
  • "$index"
    • " >> $OUTPUT + echo "
  • "$index"
    • " >> $OUTPUT done done @@ -55,7 +55,7 @@ rm -f $OUTPUT echo "
  • $path
    • " >> $OUTPUT echo "" >> $OUTPUT else - echo "
  • "$filepath"
    • " >> $OUTPUT + echo "
  • "$filepath"
    • " >> $OUTPUT fi done @@ -85,7 +85,7 @@ rm -f $OUTPUT echo "
  • $path
    • " >> $OUTPUT echo "" >> $OUTPUT else - echo "
  • "$filepath"
    • " >> $OUTPUT + echo "
  • "$filepath"
    • " >> $OUTPUT fi done @@ -119,7 +119,7 @@ do echo "
  • $path
    • " >> $OUTPUT echo "" >> $OUTPUT else - echo "
  • "$filepath"
    • " >> $OUTPUT + echo "
  • "$filepath"
    • " >> $OUTPUT fi diff --git a/utils/push-test-logs.sh b/utils/push-test-logs.sh index 9debb45b7..d5ab8b91c 100644 --- a/utils/push-test-logs.sh +++ b/utils/push-test-logs.sh @@ -64,11 +64,11 @@ if [ -d "$dir_result" ]; then else echo "Uploading logs to artifact $project_artifact" gsutil -m cp -r "$dir_result"/* gs://artifacts.opnfv.org/"$project_artifact"/ >/dev/null 2>&1 - echo "Logs can be found in http://artifacts.opnfv.org/logs_${project}_${testbed}.html" + echo "Logs can be found in https://artifacts.opnfv.org/logs_${project}_${testbed}.html" cd $dir_result files=($(find . -name \* -print|sed 's/^\.//'|sed '/^\s*$/d')) for f in ${files[@]}; do - echo "http://artifacts.opnfv.org/${project_artifact}${f}" + echo "https://artifacts.opnfv.org/${project_artifact}${f}" done fi fi