- REPO="${DOCKER_USERNAME}"
- amd64_dirs="docker/features"
- arm64_dirs=""
- - script: sudo -E bash build.sh
- env:
- - REPO="${DOCKER_USERNAME}"
- - amd64_dirs="docker/components"
- - arm64_dirs=""
- script: sudo -E bash build.sh
env:
- REPO="${DOCKER_USERNAME}"
--platforms linux/amd64 \
--template ${DOCKER_USERNAME}/functest-features:ARCH-latest \
--target ${DOCKER_USERNAME}/functest-features:latest
- - script: >
- sudo manifest-tool push from-args \
- --platforms linux/amd64 \
- --template ${DOCKER_USERNAME}/functest-components:ARCH-latest \
- --target ${DOCKER_USERNAME}/functest-components:latest
- script: >
sudo manifest-tool push from-args \
--platforms linux/amd64 \
- name: functest-benchmarking
ref_arg: BRANCH
path: docker/benchmarking
- - name: functest-components
- ref_arg:
- path: docker/components
- name: functest-vnf
ref_arg:
path: docker/vnf
- odl
- api_check
- snaps_health_check
+ - tempest_smoke
- container: functest-smoke
tests:
- - tempest_smoke
- neutron-tempest-plugin-api
- rally_sanity
- - rally_jobs
- refstack_defcore
+ - tempest_full
+ - tempest_scenario
- patrole
- snaps_smoke
- neutron_trunk
- barbican
- container: functest-benchmarking
tests:
+ - rally_full
+ - rally_jobs
- vmtp
- shaker
- - container: functest-components
- tests:
- - tempest_full
- - tempest_scenario
- - rally_full
- container: functest-vnf
tests:
- cloudify
docker/smoke \
docker/benchmarking \
docker/features \
-docker/components \
docker/vnf"}
arm64_dirs=${arm64_dirs-${amd64_dirs}}
build_opts=(--pull=true --no-cache --force-rm=true)
-FROM opnfv/functest-core
+FROM opnfv/functest-tempest
ARG BRANCH=master
ARG OPENSTACK_TAG=master
apk --no-cache add --virtual .build-deps --update \
python-dev build-base linux-headers libffi-dev \
openssl-dev libjpeg-turbo-dev libxml2-dev libxslt-dev && \
- wget -q -O- https://git.openstack.org/cgit/openstack/requirements/plain/upper-constraints.txt?h=$OPENSTACK_TAG | \
- sed -E s/^tempest==+\(.*\)$/-e\ git+https:\\/\\/git.openstack.org\\/openstack\\/tempest@\\1#egg=tempest/ \
- > upper-constraints.txt && \
- wget -q -O- https://git.opnfv.org/functest/plain/upper-constraints.txt?h=$BRANCH | \
- sed -E /#egg=functest/d > upper-constraints.opnfv.txt && \
+ wget -q -O- https://git.openstack.org/cgit/openstack/requirements/plain/upper-constraints.txt?h=$OPENSTACK_TAG > upper-constraints.txt && \
+ sed -i -E s/^tempest==+.*$/-e\ git+https:\\/\\/git.openstack.org\\/openstack\\/tempest#egg=tempest/ upper-constraints.txt && \
+ wget -q -O- https://git.opnfv.org/functest/plain/upper-constraints.txt?h=$BRANCH > upper-constraints.opnfv.txt && \
+ sed -i -E /#egg=functest/d upper-constraints.opnfv.txt && \
git init /src/vmtp && \
(cd /src/vmtp && \
git fetch --tags https://git.openstack.org/openstack/vmtp.git $VMTP_TAG && \
update-requirements -s --source /src/openstack-requirements /src/vmtp/ && \
pip install --no-cache-dir --src /src -cupper-constraints.txt -cupper-constraints.opnfv.txt \
/src/vmtp && \
- rm -r upper-constraints.txt upper-constraints.opnfv.txt src/vmtp && \
+ mkdir -p /home/opnfv/functest/data/rally/neutron && \
+ git init /src/neutron && \
+ (cd /src/neutron && \
+ git fetch --tags https://git.openstack.org/openstack/neutron.git $OPENSTACK_TAG && \
+ git checkout FETCH_HEAD) && \
+ cp -r /src/neutron/rally-jobs /home/opnfv/functest/data/rally/neutron/rally-jobs && \
+ rm -r upper-constraints.txt upper-constraints.opnfv.txt src/vmtp /src/neutron && \
apk del .build-deps
COPY testcases.yaml /usr/lib/python2.7/site-packages/xtesting/ci/testcases.yaml
CMD ["run_tests", "-t", "all"]
Run several OpenStack performance tools
https://docs.openstack.org/performance-docs/latest/methodologies/tools.html
testcases:
+ -
+ case_name: rally_full
+ project_name: functest
+ criteria: 100
+ blocking: false
+ description: >-
+ This test case runs the full suite of scenarios of the
+ OpenStack Rally suite using several threads and iterations.
+ run:
+ name: rally_full
+ args:
+ optional:
+ - 'gnocchi'
+
+ -
+ case_name: rally_jobs
+ project_name: functest
+ criteria: 100
+ blocking: false
+ description: >-
+ This test case runs a group of Rally jobs used in
+ OpenStack gating
+ run:
+ name: rally_jobs
+ args:
+ optional:
+ - 'gnocchi'
+
-
case_name: vmtp
project_name: functest
+++ /dev/null
-FROM opnfv/functest-tempest
-
-COPY testcases.yaml /usr/lib/python2.7/site-packages/xtesting/ci/testcases.yaml
-CMD ["run_tests", "-t", "all"]
+++ /dev/null
-#!/bin/bash
-
-from="${DOCKER_REPO%/*}/functest-core:${DOCKER_TAG}"
-sed -i "s|^FROM.*$|FROM ${from}|" Dockerfile
-
-exit $?
+++ /dev/null
----
-tiers:
- -
- name: components
- order: 4
- description: >-
- Extensive testing of OpenStack API.
- testcases:
- -
- case_name: tempest_full
- project_name: functest
- criteria: 100
- blocking: false
- description: >-
- The list of test cases is generated by
- Tempest automatically and depends on the parameters of
- the OpenStack deployment.
- https://github.com/openstack/tempest/blob/18.0.0/tox.ini#L83
- run:
- name: tempest_common
- args:
- mode: '(?!.*\[.*\bslow\b.*\])(^tempest\.api)'
- option:
- - '--concurrency=4'
-
- -
- case_name: tempest_scenario
- project_name: functest
- criteria: 100
- blocking: false
- description: >-
- The list of test cases is generated by
- Tempest automatically and depends on the parameters of
- the OpenStack deployment.
- https://github.com/openstack/tempest/blob/18.0.0/tox.ini#L84
- run:
- name: tempest_common
- args:
- mode: '(?!.*\[.*\bslow\b.*\])(^tempest\.scenario)'
- option:
- - '--concurrency=1'
-
- -
- case_name: rally_full
- project_name: functest
- criteria: 100
- blocking: false
- description: >-
- This test case runs the full suite of scenarios of the
- OpenStack Rally suite using several threads and iterations.
- run:
- name: rally_full
- args:
- optional:
- - 'gnocchi'
apk --no-cache add --virtual .build-deps --update \
python-dev build-base linux-headers libffi-dev \
openssl-dev libjpeg-turbo-dev && \
- wget -q -O- https://git.opnfv.org/functest/plain/upper-constraints.txt?h=$BRANCH | \
- sed -E /#egg=functest/d > upper-constraints.txt && \
- pip install --no-cache-dir --src /src -cupper-constraints.txt \
+ wget -q -O- https://git.opnfv.org/functest/plain/upper-constraints.txt?h=$BRANCH > upper-constraints.opnfv.txt && \
+ sed -i -E /#egg=functest/d upper-constraints.opnfv.txt && \
+ pip install --no-cache-dir --src /src -cupper-constraints.opnfv.txt \
-chttps://git.openstack.org/cgit/openstack/requirements/plain/upper-constraints.txt?h=$OPENSTACK_TAG \
pip==$PIP_TAG && \
- pip install --no-cache-dir --src /src -cupper-constraints.txt \
+ pip install --no-cache-dir --src /src -cupper-constraints.opnfv.txt \
-chttps://git.openstack.org/cgit/openstack/requirements/plain/upper-constraints.txt?h=$OPENSTACK_TAG \
-e git+https://git.openstack.org/openstack/requirements@$OPENSTACK_TAG#egg=openstack_requirements && \
git init /src/functest && \
git fetch --tags https://gerrit.opnfv.org/gerrit/functest $BRANCH && \
git checkout FETCH_HEAD) && \
update-requirements -s --source /src/openstack-requirements /src/functest && \
- pip install --no-cache-dir --src /src -cupper-constraints.txt \
+ pip install --no-cache-dir --src /src -cupper-constraints.opnfv.txt \
-chttps://git.openstack.org/cgit/openstack/requirements/plain/upper-constraints.txt?h=$OPENSTACK_TAG \
/src/functest && \
- rm -r upper-constraints.txt /src/functest && \
+ rm -r upper-constraints.opnfv.txt /src/functest && \
cp /usr/lib/python2.7/site-packages/functest/ci/logging.ini /usr/lib/python2.7/site-packages/xtesting/ci/ && \
bash -c "mkdir -p /var/lib/xtesting /home/opnfv" && \
ln -s /var/lib/xtesting /home/opnfv/functest && \
apk --no-cache add --virtual .build-deps --update \
python-dev python3-dev build-base linux-headers libffi-dev \
openssl-dev libjpeg-turbo-dev && \
- wget -q -O- https://git.openstack.org/cgit/openstack/requirements/plain/upper-constraints.txt?h=$OPENSTACK_TAG | \
- sed -E s/^tempest==+.*$/-e\ git+https:\\/\\/git.openstack.org\\/openstack\\/tempest#egg=tempest/ \
- > upper-constraints.txt && \
- wget -q -O- https://git.opnfv.org/functest/plain/upper-constraints.txt?h=$BRANCH | \
- sed -E /#egg=functest/d > upper-constraints.opnfv.txt && \
+ wget -q -O- https://git.openstack.org/cgit/openstack/requirements/plain/upper-constraints.txt?h=$OPENSTACK_TAG > upper-constraints.txt && \
+ sed -i -E s/^tempest==+.*$/-e\ git+https:\\/\\/git.openstack.org\\/openstack\\/tempest#egg=tempest/ upper-constraints.txt && \
+ wget -q -O- https://git.opnfv.org/functest/plain/upper-constraints.txt?h=$BRANCH > upper-constraints.opnfv.txt && \
+ sed -i -E /#egg=functest/d upper-constraints.opnfv.txt && \
pip install --no-cache-dir --src /src -cupper-constraints.txt \
-cupper-constraints.opnfv.txt \
-rthirdparty-requirements.txt && \
-FROM opnfv/functest-core
+FROM opnfv/functest-tempest
ARG BRANCH=master
ARG OPENSTACK_TAG=master
COPY thirdparty-requirements.txt thirdparty-requirements.txt
RUN apk --no-cache add --virtual .build-deps --update \
python-dev build-base linux-headers libffi-dev openssl-dev && \
- wget -q -O- https://git.openstack.org/cgit/openstack/requirements/plain/upper-constraints.txt?h=$OPENSTACK_TAG | \
- sed -E s/^tempest==+\(.*\)$/-e\ git+https:\\/\\/git.openstack.org\\/openstack\\/tempest@\\1#egg=tempest/ \
- > upper-constraints.txt && \
- wget -q -O- https://git.opnfv.org/functest/plain/upper-constraints.txt?h=$BRANCH | \
- sed -E /#egg=functest/d > upper-constraints.opnfv.txt && \
+ wget -q -O- https://git.openstack.org/cgit/openstack/requirements/plain/upper-constraints.txt?h=$OPENSTACK_TAG > upper-constraints.txt && \
+ sed -i -E s/^tempest==+.*$/-e\ git+https:\\/\\/git.openstack.org\\/openstack\\/tempest#egg=tempest/ upper-constraints.txt && \
+ wget -q -O- https://git.opnfv.org/functest/plain/upper-constraints.txt?h=$BRANCH > upper-constraints.opnfv.txt && \
+ sed -i -E /#egg=functest/d upper-constraints.opnfv.txt && \
pip install --no-cache-dir --src /src -cupper-constraints.txt -cupper-constraints.opnfv.txt \
-rthirdparty-requirements.txt && \
git init /src/odl_test && \
validate the single port obtains the correct IP address.
run:
name: snaps_health_check
+
+ -
+ case_name: tempest_smoke
+ project_name: functest
+ criteria: 100
+ blocking: false
+ description: >-
+ This test case runs the smoke subset of the OpenStack
+ Tempest suite. The list of test cases is generated by
+ Tempest automatically and depends on the parameters of
+ the OpenStack deplopyment.
+ https://github.com/openstack/tempest/blob/18.0.0/tox.ini#L114
+ run:
+ name: tempest_common
+ args:
+ mode: '(?=.*\[.*\bsmoke\b.*\])(^tempest\.api)'
+ option:
+ - '--concurrency=4'
ARG OPENSTACK_TAG=master
ARG REFSTACK_TARGET=2018.02
ARG PATROLE_TAG=master
-ARG NEUTRON_TAG=master
ARG BARBICAN_TAG=master
RUN apk --no-cache add --virtual .build-deps --update \
python-dev build-base linux-headers libffi-dev \
openssl-dev libjpeg-turbo-dev libxml2-dev libxslt-dev && \
- wget -q -O- https://git.openstack.org/cgit/openstack/requirements/plain/upper-constraints.txt?h=$OPENSTACK_TAG | \
- sed -E s/^tempest==+.*$/-e\ git+https:\\/\\/git.openstack.org\\/openstack\\/tempest#egg=tempest/ \
- > upper-constraints.txt && \
- wget -q -O- https://git.opnfv.org/functest/plain/upper-constraints.txt?h=$BRANCH | \
- sed -E /#egg=functest/d > upper-constraints.opnfv.txt && \
+ wget -q -O- https://git.openstack.org/cgit/openstack/requirements/plain/upper-constraints.txt?h=$OPENSTACK_TAG > upper-constraints.txt && \
+ sed -i -E s/^tempest==+.*$/-e\ git+https:\\/\\/git.openstack.org\\/openstack\\/tempest#egg=tempest/ upper-constraints.txt && \
+ wget -q -O- https://git.opnfv.org/functest/plain/upper-constraints.txt?h=$BRANCH > upper-constraints.opnfv.txt && \
+ sed -i -E /#egg=functest/d upper-constraints.opnfv.txt && \
git init /src/patrole && \
(cd /src/patrole && \
git fetch --tags https://git.openstack.org/openstack/patrole.git $PATROLE_TAG && \
git checkout FETCH_HEAD) && \
update-requirements -s --source /src/openstack-requirements /src/patrole/ && \
- git init /src/neutron-tempest-plugin && \
- (cd /src/neutron-tempest-plugin && \
- git fetch --tags https://git.openstack.org/openstack/neutron-tempest-plugin.git $NEUTRON_TAG && \
- git checkout FETCH_HEAD) && \
- update-requirements -s --source /src/openstack-requirements /src/neutron-tempest-plugin && \
git init /src/barbican-tempest-plugin && \
(cd /src/barbican-tempest-plugin && \
git fetch --tags https://git.openstack.org/openstack/barbican-tempest-plugin.git $BARBICAN_TAG && \
git checkout FETCH_HEAD) && \
update-requirements -s --source /src/openstack-requirements /src/barbican-tempest-plugin/ && \
pip install --no-cache-dir --src /src -cupper-constraints.txt -cupper-constraints.opnfv.txt \
- /src/patrole /src/barbican-tempest-plugin /src/neutron-tempest-plugin \
+ /src/patrole /src/barbican-tempest-plugin neutron-tempest-plugin \
networking-bgpvpn networking-sfc && \
mkdir -p /home/opnfv/functest/data/refstack && \
wget "https://refstack.openstack.org/api/v1/guidelines/${REFSTACK_TARGET}/tests?target=compute&type=required&alias=true&flag=false"Â \
oslopolicy-sample-generator --format json --output-file /etc/neutron/policy.json --namespace neutron && \
oslopolicy-sample-generator --format json --output-file /etc/nova/policy.json --namespace nova && \
deactivate && \
- mkdir -p /home/opnfv/functest/data/rally/neutron && \
- cp -r /src/neutron/rally-jobs /home/opnfv/functest/data/rally/neutron/rally-jobs && \
rm -r oslo upper-constraints.txt upper-constraints.opnfv.txt \
- /src/neutron /src/patrole /src/barbican-tempest-plugin /src/neutron-tempest-plugin && \
+ /src/patrole /src/barbican-tempest-plugin && \
apk del .build-deps
COPY testcases.yaml /usr/lib/python2.7/site-packages/xtesting/ci/testcases.yaml
CMD ["run_tests", "-t", "all"]
description: >-
Set of basic Functional tests to validate the OPNFV scenarios.
testcases:
- -
- case_name: tempest_smoke
- project_name: functest
- criteria: 100
- blocking: false
- description: >-
- This test case runs the smoke subset of the OpenStack
- Tempest suite. The list of test cases is generated by
- Tempest automatically and depends on the parameters of
- the OpenStack deplopyment.
- https://github.com/openstack/tempest/blob/18.0.0/tox.ini#L114
- run:
- name: tempest_common
- args:
- mode: '(?=.*\[.*\bsmoke\b.*\])(^tempest\.api)'
- option:
- - '--concurrency=4'
-
-
case_name: neutron-tempest-plugin-api
project_name: functest
- 'gnocchi'
-
- case_name: rally_jobs
+ case_name: refstack_defcore
project_name: functest
criteria: 100
blocking: false
description: >-
- This test case runs a group of Rally jobs used in
- OpenStack gating
+ This test case runs a sub group of tests of the OpenStack
+ Defcore testcases.
run:
- name: rally_jobs
+ name: refstack_defcore
args:
- optional:
- - 'gnocchi'
+ option:
+ - '--concurrency=4'
-
- case_name: refstack_defcore
+ case_name: tempest_full
project_name: functest
criteria: 100
blocking: false
description: >-
- This test case runs a sub group of tests of the OpenStack
- Defcore testcases.
+ The list of test cases is generated by
+ Tempest automatically and depends on the parameters of
+ the OpenStack deployment.
+ https://github.com/openstack/tempest/blob/18.0.0/tox.ini#L83
run:
- name: refstack_defcore
+ name: tempest_common
args:
+ mode: '(?!.*\[.*\bslow\b.*\])(^tempest\.api)'
option:
- '--concurrency=4'
+ -
+ case_name: tempest_scenario
+ project_name: functest
+ criteria: 100
+ blocking: false
+ description: >-
+ The list of test cases is generated by
+ Tempest automatically and depends on the parameters of
+ the OpenStack deployment.
+ https://github.com/openstack/tempest/blob/18.0.0/tox.ini#L84
+ run:
+ name: tempest_common
+ args:
+ mode: '(?!.*\[.*\bslow\b.*\])(^tempest\.scenario)'
+ option:
+ - '--concurrency=1'
+
-
case_name: patrole
project_name: functest
run:
name: tempest_common
args:
- mode: '^networking_bgpvpn_tempest\.'
+ mode: '^networking_bgpvpn_tempest.tests.api\.'
neutron_extensions:
- bgpvpn
option:
RUN apk --no-cache add --virtual .build-deps --update \
python-dev build-base linux-headers libffi-dev \
openssl-dev libjpeg-turbo-dev && \
- wget -q -O- https://git.openstack.org/cgit/openstack/requirements/plain/upper-constraints.txt?h=$OPENSTACK_TAG | \
- sed -E s/^tempest==+.*$/-e\ git+https:\\/\\/git.openstack.org\\/openstack\\/tempest#egg=tempest/ \
- > upper-constraints.txt && \
- wget -q -O- https://git.opnfv.org/functest/plain/upper-constraints.txt?h=$BRANCH | \
- sed -E /#egg=functest/d > upper-constraints.opnfv.txt && \
+ wget -q -O- https://git.openstack.org/cgit/openstack/requirements/plain/upper-constraints.txt?h=$OPENSTACK_TAG > upper-constraints.txt && \
+ sed -i -E s/^tempest==+.*$/-e\ git+https:\\/\\/git.openstack.org\\/openstack\\/tempest#egg=tempest/ upper-constraints.txt && \
+ wget -q -O- https://git.opnfv.org/functest/plain/upper-constraints.txt?h=$BRANCH > upper-constraints.opnfv.txt && \
+ sed -i -E /#egg=functest/d upper-constraints.opnfv.txt && \
git init /src/os-faults && \
(cd /src/os-faults && \
git fetch --tags https://git.openstack.org/openstack/os-faults.git $OS_FAULTS_TAG && \
ARG OPENSTACK_TAG=master
ARG VIMS_TEST_TAG=release-130
ARG QUAFF_TAG=59213d6d8ee29433552bb75f505cdc96b0b18909
-ARG CLOUDIFY_VIMS_TAG=fraser
+ARG CLOUDIFY_VIMS_TAG=gambia
ARG HEAT_VIMS_TAG=release-130
ARG VROUTER_TAG=fraser
+ARG VROUTER_BP_TAG=9b76d46a388d32d4985797620e67c2ed3315b3e4
ARG JUJU_TAG=tags/juju-2.3.9
ARG JUJU_WAIT_TAG=2.6.4
ARG ABOT_CHARM=opnfv-fraser
ENV GOBIN /src/epc-requirements/go/bin
ENV PATH $GOBIN:$PATH
+COPY clearwater-heat-add-deps.patch /tmp/clearwater-heat-add-deps.patch
RUN apk --no-cache add --update \
ruby ruby-bundler ruby-irb ruby-rdoc dnsmasq \
procps libxslt libxml2 zlib libffi python3 go musl-dev && \
apk --no-cache add --virtual .build-deps --update \
ruby-dev g++ make libxslt-dev libxml2-dev zlib-dev libffi-dev g++ make && \
- wget -q -O- https://git.openstack.org/cgit/openstack/requirements/plain/upper-constraints.txt?h=$OPENSTACK_TAG | \
- sed -E s/^tempest==+\(.*\)$/-e\ git+https:\\/\\/git.openstack.org\\/openstack\\/tempest@\\1#egg=tempest/ \
- > upper-constraints.txt && \
- wget -q -O- https://git.opnfv.org/functest/plain/upper-constraints.txt?h=$BRANCH | \
- sed -E /#egg=functest/d > upper-constraints.opnfv.txt && \
+ wget -q -O- https://git.openstack.org/cgit/openstack/requirements/plain/upper-constraints.txt?h=$OPENSTACK_TAG > upper-constraints.txt && \
+ sed -i -E s/^tempest==+.*$/-e\ git+https:\\/\\/git.openstack.org\\/openstack\\/tempest#egg=tempest/ upper-constraints.txt && \
+ wget -q -O- https://git.opnfv.org/functest/plain/upper-constraints.txt?h=$BRANCH > upper-constraints.opnfv.txt && \
+ sed -i -E /#egg=functest/d upper-constraints.opnfv.txt && \
git clone --depth 1 -b $VIMS_TEST_TAG https://github.com/Metaswitch/clearwater-live-test /src/vims-test && \
sed -i s/unf_ext\ \(.*\)/unf_ext\ \(0.0.7.4\)/g /src/vims-test/Gemfile.lock && \
git init /src/vims-test/quaff && \
git init /src/heat_vims && \
(cd /src/heat_vims && \
git fetch --tags https://github.com/Metaswitch/clearwater-heat.git $HEAT_VIMS_TAG && \
- git checkout FETCH_HEAD) && \
+ git checkout FETCH_HEAD && \
+ patch -p1 < /tmp/clearwater-heat-add-deps.patch) && \
git init /src/opnfv-vnf-vyos-blueprint && \
(cd /src/opnfv-vnf-vyos-blueprint && \
- git fetch --tags https://github.com/oolorg/opnfv-vnf-vyos-blueprint.git $VROUTER_TAG && \
+ git fetch --tags https://github.com/oolorg/opnfv-vnf-vyos-blueprint.git $VROUTER_BP_TAG && \
git checkout FETCH_HEAD) && \
mkdir -p /home/opnfv/functest/data/router && \
git init /home/opnfv/functest/data/router/opnfv-vnf-data && \
(cd /src/vims-test && bundle config build.nokogiri --use-system-libraries && bundle install --system) && \
rm -r upper-constraints.txt upper-constraints.opnfv.txt /src/vims-test/.git /src/cloudify_vims/.git /src/heat_vims/.git /src/vims-test/quaff/.git \
/src/vims-test/build-infra/.git /src/opnfv-vnf-vyos-blueprint/.git \
- /src/epc-requirements/abot_charm/.git && \
+ /src/epc-requirements/abot_charm/.git /root/.cache/go-build \
+ /tmp/clearwater-heat-add-deps.patch && \
apk del .build-deps
COPY testcases.yaml /usr/lib/python2.7/site-packages/xtesting/ci/testcases.yaml
CMD ["run_tests", "-t", "all"]
--- /dev/null
+diff --git a/clearwater.yaml b/clearwater.yaml
+index a155c60..1de2e0f 100644
+--- a/clearwater.yaml
++++ b/clearwater.yaml
+@@ -185,6 +185,7 @@ resources:
+
+ ellis:
+ type: ./ellis.yaml
++ depends_on: dns
+ properties:
+ public_mgmt_net_id: { get_param: public_mgmt_net_id }
+ private_mgmt_net_id: { get_attr: [ mgmt_network, private_net ] }
+@@ -202,6 +203,7 @@ resources:
+
+ bono:
+ type: OS::Heat::ResourceGroup
++ depends_on: ellis
+ properties:
+ count: { get_param: bono_cluster_size }
+ index_var: "__index__"
+@@ -229,6 +231,7 @@ resources:
+
+ sprout:
+ type: OS::Heat::ResourceGroup
++ depends_on: ellis
+ properties:
+ count: { get_param: sprout_cluster_size }
+ index_var: __index__
+@@ -257,6 +260,7 @@ resources:
+
+ homer:
+ type: OS::Heat::ResourceGroup
++ depends_on: ellis
+ properties:
+ count: { get_param: homer_cluster_size }
+ index_var: __index__
+@@ -285,6 +289,7 @@ resources:
+
+ dime:
+ type: OS::Heat::ResourceGroup
++ depends_on: ellis
+ properties:
+ count: { get_param: dime_cluster_size }
+ index_var: __index__
+@@ -313,6 +318,7 @@ resources:
+
+ vellum:
+ type: OS::Heat::ResourceGroup
++ depends_on: ellis
+ properties:
+ count: { get_param: vellum_cluster_size }
+ index_var: __index__
+diff --git a/dns.yaml b/dns.yaml
+index 825ede1..3e6c938 100644
+--- a/dns.yaml
++++ b/dns.yaml
+@@ -91,6 +91,16 @@ resources:
+ security_groups:
+ - { get_param: dns_security_group }
+
++ wait_condition:
++ type: OS::Heat::WaitCondition
++ properties:
++ handle: {get_resource: wait_handle}
++ count: 1
++ timeout: 600
++
++ wait_handle:
++ type: OS::Heat::WaitConditionHandle
++
+ server:
+ type: OS::Nova::Server
+ properties:
+@@ -110,6 +120,7 @@ resources:
+ __public_ip__: { get_attr: [ mgmt_floating_ip, floating_ip_address ] }
+ __private_sig_ip__: { get_attr: [ sig_port, fixed_ips, 0, ip_address ] }
+ __private_sig_cidr__: { get_param: private_sig_net_cidr }
++ wc_notify: { get_attr: [wait_handle, curl_cli] }
+ template: |
+ #!/bin/bash
+
+@@ -162,6 +173,7 @@ resources:
+
+ # Now that BIND configuration is correct, kick it to reload.
+ service bind9 reload
++ wc_notify --data-binary '{"status": "SUCCESS"}'
+
+ outputs:
+ public_mgmt_ip:
+diff --git a/ellis.yaml b/ellis.yaml
+index 963352d..d39c235 100644
+--- a/ellis.yaml
++++ b/ellis.yaml
+@@ -103,6 +103,16 @@ resources:
+ floating_network_id: { get_param: public_mgmt_net_id }
+ port_id: { get_resource: mgmt_port }
+
++ wait_condition:
++ type: OS::Heat::WaitCondition
++ properties:
++ handle: {get_resource: wait_handle}
++ count: 1
++ timeout: 600
++
++ wait_handle:
++ type: OS::Heat::WaitConditionHandle
++
+ server:
+ type: OS::Nova::Server
+ properties:
+@@ -126,6 +136,7 @@ resources:
+ __dnssec_key__: { get_param: dnssec_key }
+ __etcd_ip__ : { get_param: etcd_ip }
+ __index__ : { get_param: index }
++ wc_notify: { get_attr: [wait_handle, curl_cli] }
+ template: |
+ #!/bin/bash
+
+@@ -227,6 +238,7 @@ resources:
+ echo 'nameserver __dns_mgmt_ip__' > /etc/dnsmasq.resolv.conf
+ echo 'RESOLV_CONF=/etc/dnsmasq.resolv.conf' >> /etc/default/dnsmasq
+ service dnsmasq force-reload
++ wc_notify --data-binary '{"status": "SUCCESS"}'
+
+ outputs:
+ public_mgmt_ip:
tiers:
-
name: vnf
- order: 5
+ order: 4
description: >-
Collection of VNF test cases.
testcases:
https://cloud-images.ubuntu.com/releases/14.04/release/ubuntu-14.04-server-cloudimg-amd64-disk1.img
https://cloud.centos.org/centos/7/images/CentOS-7-x86_64-GenericCloud.qcow2
https://cloud-images.ubuntu.com/releases/16.04/release/ubuntu-16.04-server-cloudimg-amd64-disk1.img
-http://repository.cloudifysource.org/cloudify/4.0.1/sp-release/cloudify-manager-premium-4.0.1.qcow2
http://download.cirros-cloud.net/0.4.0/cirros-0.4.0-aarch64-disk.img
https://cloud-images.ubuntu.com/releases/14.04/release/ubuntu-14.04-server-cloudimg-arm64-uefi1.img
http://cloud.centos.org/altarch/7/images/aarch64/CentOS-7-aarch64-GenericCloud.qcow2.xz
-https://sourceforge.net/projects/ool-opnfv/files/vyos-1.1.7.img
+http://repository.cloudifysource.org/cloudify/19.01.24/community-release/cloudify-docker-manager-community-19.01.24.tar
+http://testresults.opnfv.org/functest/vyos-1.1.8-amd64.qcow2
http://testresults.opnfv.org/functest/shaker-image.qcow2
http://testresults.opnfv.org/functest/shaker-image-arm64.qcow2
EOF
run:
name: snaps_health_check
- -
- name: smoke
- order: 1
- description: >-
- Set of basic Functional tests to validate the OPNFV scenarios.
- testcases:
-
case_name: tempest_smoke
project_name: functest
option:
- '--concurrency=4'
+ -
+ name: smoke
+ order: 1
+ description: >-
+ Set of basic Functional tests to validate the OPNFV scenarios.
+ testcases:
-
case_name: neutron-tempest-plugin-api
project_name: functest
Rally suite in smoke mode.
run:
name: rally_sanity
+ args:
+ optional:
+ - 'gnocchi'
-
- case_name: rally_jobs
+ case_name: refstack_defcore
project_name: functest
criteria: 100
blocking: false
description: >-
- This test case runs a group of Rally jobs used in
- OpenStack gating
+ This test case runs a sub group of tests of the OpenStack
+ Defcore testcases.
run:
- name: rally_jobs
+ name: refstack_defcore
+ args:
+ option:
+ - '--concurrency=4'
-
- case_name: refstack_defcore
+ case_name: tempest_full
project_name: functest
criteria: 100
blocking: false
description: >-
- This test case runs a sub group of tests of the OpenStack
- Defcore testcases.
+ The list of test cases is generated by
+ Tempest automatically and depends on the parameters of
+ the OpenStack deployment.
+ https://github.com/openstack/tempest/blob/18.0.0/tox.ini#L83
run:
- name: refstack_defcore
+ name: tempest_common
args:
+ mode: '(?!.*\[.*\bslow\b.*\])(^tempest\.api)'
option:
- '--concurrency=4'
+ -
+ case_name: tempest_scenario
+ project_name: functest
+ criteria: 100
+ blocking: false
+ description: >-
+ The list of test cases is generated by
+ Tempest automatically and depends on the parameters of
+ the OpenStack deployment.
+ https://github.com/openstack/tempest/blob/18.0.0/tox.ini#L84
+ run:
+ name: tempest_common
+ args:
+ mode: '(?!.*\[.*\bslow\b.*\])(^tempest\.scenario)'
+ option:
+ - '--concurrency=1'
+
-
case_name: patrole
project_name: functest
run:
name: tempest_common
args:
- mode: '^networking_bgpvpn_tempest\.'
+ mode: '^networking_bgpvpn_tempest.tests.api\.'
neutron_extensions:
- bgpvpn
option:
Run several OpenStack performance tools
https://docs.openstack.org/performance-docs/latest/methodologies/tools.html
testcases:
+ -
+ case_name: rally_full
+ project_name: functest
+ criteria: 100
+ blocking: false
+ description: >-
+ This test case runs the full suite of scenarios of the
+ OpenStack Rally suite using several threads and iterations.
+ run:
+ name: rally_full
+ args:
+ optional:
+ - 'gnocchi'
+
+ -
+ case_name: rally_jobs
+ project_name: functest
+ criteria: 100
+ blocking: false
+ description: >-
+ This test case runs a group of Rally jobs used in
+ OpenStack gating
+ run:
+ name: rally_jobs
+ args:
+ optional:
+ - 'gnocchi'
+
-
case_name: vmtp
project_name: functest
run:
name: stor4nfv_os
- -
- name: components
- order: 4
- description: >-
- Extensive testing of OpenStack API.
- testcases:
- -
- case_name: tempest_full
- project_name: functest
- criteria: 100
- blocking: false
- description: >-
- The list of test cases is generated by
- Tempest automatically and depends on the parameters of
- the OpenStack deployment.
- https://github.com/openstack/tempest/blob/18.0.0/tox.ini#L83
- run:
- name: tempest_common
- args:
- mode: '(?!.*\[.*\bslow\b.*\])(^tempest\.api)'
- option:
- - '--concurrency=4'
-
- -
- case_name: tempest_scenario
- project_name: functest
- criteria: 100
- blocking: false
- description: >-
- The list of test cases is generated by
- Tempest automatically and depends on the parameters of
- the OpenStack deployment.
- https://github.com/openstack/tempest/blob/18.0.0/tox.ini#L84
- run:
- name: tempest_common
- args:
- mode: '(?!.*\[.*\bslow\b.*\])(^tempest\.scenario)'
- option:
- - '--concurrency=1'
-
- -
- case_name: rally_full
- project_name: functest
- criteria: 100
- blocking: false
- description: >-
- This test case runs the full suite of scenarios of the
- OpenStack Rally suite using several threads and iterations.
- run:
- name: rally_full
-
-
name: vnf
- order: 5
+ order: 4
description: >-
Collection of VNF test cases.
testcases:
from __future__ import division
import logging
+import os
import time
import traceback
from cloudify_rest_client import CloudifyClient
from cloudify_rest_client.executions import Execution
+import scp
from functest.core import singlevm
__logger = logging.getLogger(__name__)
filename = ('/home/opnfv/functest/images/'
- 'cloudify-manager-premium-4.0.1.qcow2')
+ 'ubuntu-16.04-server-cloudimg-amd64-disk1.img')
flavor_ram = 4096
flavor_vcpus = 2
flavor_disk = 40
- username = 'centos'
+ username = 'ubuntu'
ssh_connect_loops = 12
create_server_timeout = 600
ports = [80, 443, 5671, 53333]
+ cloudify_archive = ('/home/opnfv/functest/images/'
+ 'cloudify-docker-manager-community-19.01.24.tar')
+ cloudify_container = "docker-cfy-manager:latest"
+
def __init__(self, **kwargs):
"""Initialize Cloudify testcase object."""
if "case_name" not in kwargs:
"""
Deploy Cloudify Manager.
"""
+ scpc = scp.SCPClient(self.ssh.get_transport())
+ scpc.put(self.cloudify_archive,
+ remote_path=os.path.basename(self.cloudify_archive))
+ (_, stdout, stderr) = self.ssh.exec_command(
+ "sudo wget https://get.docker.com/ -O script.sh && "
+ "sudo chmod +x script.sh && "
+ "sudo ./script.sh && "
+ "sudo docker load -i ~/{} && "
+ "sudo docker run --name cfy_manager_local -d "
+ "--restart unless-stopped -v /sys/fs/cgroup:/sys/fs/cgroup:ro "
+ "--tmpfs /run --tmpfs /run/lock --security-opt seccomp:unconfined "
+ "--cap-add SYS_ADMIN --network=host {}".format(
+ os.path.basename(self.cloudify_archive),
+ self.cloudify_container))
+ self.__logger.debug("output:\n%s", stdout.read())
+ self.__logger.debug("error:\n%s", stderr.read())
self.cfy_client = CloudifyClient(
host=self.fip.floating_ip_address,
- username='admin', password='admin', tenant='default_tenant',
- api_version='v3')
+ username='admin', password='admin', tenant='default_tenant')
self.__logger.info("Attemps running status of the Manager")
secret_key = "foo"
secret_value = "bar"
- for loop in range(10):
+ for loop in range(20):
try:
self.__logger.debug(
"status %s", self.cfy_client.manager.get_status())
self.keypair = self.cloud.create_keypair(
'{}-kp_{}'.format(self.case_name, self.guid))
self.__logger.debug("keypair: %s", self.keypair)
- self.__logger.debug("private_key: %s", self.keypair.private_key)
+ self.__logger.debug("private_key:\n%s", self.keypair.private_key)
with open(self.key_filename, 'w') as private_key_file:
private_key_file.write(self.keypair.private_key)
self.sec = self.cloud.create_security_group(
"""Rally testcases implementation."""
from __future__ import division
+from __future__ import print_function
+import fileinput
import json
import logging
import os
import yaml
from functest.core import singlevm
-from functest.opnfv_tests.openstack.tempest import conf_utils
from functest.utils import config
from functest.utils import env
class RallyBase(singlevm.VmReady2):
"""Base class form Rally testcases implementation."""
- # pylint: disable=too-many-instance-attributes
+ # pylint: disable=too-many-instance-attributes, too-many-public-methods
TESTS = ['authenticate', 'glance', 'cinder', 'gnocchi', 'heat',
'keystone', 'neutron', 'nova', 'quotas']
+ RALLY_CONF_PATH = "/etc/rally/rally.conf"
+ RALLY_AARCH64_PATCH_PATH = pkg_resources.resource_filename(
+ 'functest', 'ci/rally_aarch64_patch.conf')
RALLY_DIR = pkg_resources.resource_filename(
'functest', 'opnfv_tests/openstack/rally')
RALLY_SCENARIO_DIR = pkg_resources.resource_filename(
self.summary = []
self.scenario_dir = ''
self.smoke = None
- self.test_name = None
self.start_time = None
self.result = None
self.details = None
self.apply_blacklist(scenario_file_name, test_file_name)
return test_file_name
+ @staticmethod
+ def get_verifier_deployment_id():
+ """
+ Returns deployment id for active Rally deployment
+ """
+ cmd = ("rally deployment list | awk '/" +
+ getattr(config.CONF, 'rally_deployment_name') +
+ "/ {print $2}'")
+ proc = subprocess.Popen(cmd, shell=True,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.STDOUT)
+ deployment_uuid = proc.stdout.readline().rstrip()
+ return deployment_uuid
+
+ @staticmethod
+ def create_rally_deployment(environ=None):
+ """Create new rally deployment"""
+ # set the architecture to default
+ pod_arch = env.get("POD_ARCH")
+ arch_filter = ['aarch64']
+
+ if pod_arch and pod_arch in arch_filter:
+ LOGGER.info("Apply aarch64 specific to rally config...")
+ with open(RallyBase.RALLY_AARCH64_PATCH_PATH, "r") as pfile:
+ rally_patch_conf = pfile.read()
+
+ for line in fileinput.input(RallyBase.RALLY_CONF_PATH):
+ print(line, end=' ')
+ if "cirros|testvm" in line:
+ print(rally_patch_conf)
+
+ LOGGER.info("Creating Rally environment...")
+ try:
+ cmd = ['rally', 'deployment', 'destroy',
+ '--deployment',
+ str(getattr(config.CONF, 'rally_deployment_name'))]
+ output = subprocess.check_output(cmd)
+ LOGGER.info("%s\n%s", " ".join(cmd), output)
+ except subprocess.CalledProcessError:
+ pass
+
+ cmd = ['rally', 'deployment', 'create', '--fromenv',
+ '--name', str(getattr(config.CONF, 'rally_deployment_name'))]
+ output = subprocess.check_output(cmd, env=environ)
+ LOGGER.info("%s\n%s", " ".join(cmd), output)
+
+ cmd = ['rally', 'deployment', 'check']
+ output = subprocess.check_output(cmd)
+ LOGGER.info("%s\n%s", " ".join(cmd), output)
+ return RallyBase.get_verifier_deployment_id()
+
@staticmethod
def update_keystone_default_role(rally_conf='/etc/rally/rally.conf'):
"""Set keystone_default_role in rally.conf"""
output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
LOGGER.info("%s\n%s", " ".join(cmd), output)
- # save report as HTML
- report_html_name = '{}.html'.format(test_name)
- report_html_dir = os.path.join(self.results_dir, report_html_name)
- cmd = (["rally", "task", "report", "--html", "--uuid", task_id,
- "--out", report_html_dir])
- LOGGER.debug('running command: %s', cmd)
- output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
- LOGGER.info("%s\n%s", " ".join(cmd), output)
-
json_results = open(report_json_dir).read()
self._append_summary(json_results, test_name)
nb_tests = 0
nb_success = 0
overall_duration = 0.0
+ success = []
+ failures = []
rally_report = json.loads(json_raw)
for task in rally_report.get('tasks'):
for subtask in task.get('subtasks'):
+ has_errors = False
for workload in subtask.get('workloads'):
if workload.get('full_duration'):
overall_duration += workload.get('full_duration')
for result in workload.get('data'):
if not result.get('error'):
nb_success += 1
+ else:
+ has_errors = True
+
+ if has_errors:
+ failures.append(subtask['title'])
+ else:
+ success.append(subtask['title'])
scenario_summary = {'test_name': test_name,
'overall_duration': overall_duration,
'nb_tests': nb_tests,
'nb_success': nb_success,
+ 'success': success,
+ 'failures': failures,
'task_status': self.task_succeed(json_raw)}
self.summary.append(scenario_summary)
- def prepare_run(self):
+ def prepare_run(self, **kwargs):
"""Prepare resources needed by test scenarios."""
assert self.cloud
- LOGGER.debug('Validating the test name...')
- if self.test_name == 'all':
- self.tests = self.TESTS
- elif self.test_name in self.TESTS:
- self.tests = [self.test_name]
- else:
- raise Exception("Test name '%s' is invalid" % self.test_name)
+ LOGGER.debug('Validating run tests...')
+ for test in kwargs.get('tests', self.TESTS):
+ if test in self.TESTS:
+ self.tests.append(test)
+ else:
+ raise Exception("Test name '%s' is invalid" % test)
if not os.path.exists(self.TASK_DIR):
os.makedirs(self.TASK_DIR)
payload.append({'module': item['test_name'],
'details': {'duration': item['overall_duration'],
'nb tests': item['nb_tests'],
- 'success': success_str}})
+ 'success rate': success_str,
+ 'success': item['success'],
+ 'failures': item['failures']}})
total_duration_str = time.strftime("%H:%M:%S",
time.gmtime(total_duration))
'nb success': success_rate}})
self.details = payload
+ @staticmethod
+ def export_task(file_name, export_type="html"):
+ """Export all task results (e.g. html or xunit report)
+
+ Raises:
+ subprocess.CalledProcessError: if Rally doesn't return 0
+
+ Returns:
+ None
+ """
+ cmd = ["rally", "task", "export", "--type", export_type,
+ "--deployment",
+ str(getattr(config.CONF, 'rally_deployment_name')),
+ "--to", file_name]
+ LOGGER.debug('running command: %s', cmd)
+ output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
+ LOGGER.info("%s\n%s", " ".join(cmd), output)
+
+ @staticmethod
+ def verify_report(file_name, uuid, export_type="html"):
+ """Generate the verifier report (e.g. html or xunit report)
+
+ Raises:
+ subprocess.CalledProcessError: if Rally doesn't return 0
+
+ Returns:
+ None
+ """
+ cmd = ["rally", "verify", "report", "--type", export_type,
+ "--uuid", uuid, "--to", file_name]
+ LOGGER.debug('running command: %s', cmd)
+ output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
+ LOGGER.info("%s\n%s", " ".join(cmd), output)
+
def clean(self):
"""Cleanup of OpenStack resources. Should be called on completion."""
self.clean_rally_conf()
del environ['OS_TENANT_ID']
except Exception: # pylint: disable=broad-except
pass
- conf_utils.create_rally_deployment(environ=environ)
- self.prepare_run()
+ self.create_rally_deployment(environ=environ)
+ self.prepare_run(**kwargs)
self.run_tests(**kwargs)
self._generate_report()
+ self.export_task(
+ "{}/{}.html".format(self.results_dir, self.case_name))
+ self.export_task(
+ "{}/{}.xml".format(self.results_dir, self.case_name),
+ export_type="junit-xml")
res = testcase.TestCase.EX_OK
except Exception as exc: # pylint: disable=broad-except
LOGGER.error('Error with run: %s', exc)
if "case_name" not in kwargs:
kwargs["case_name"] = "rally_sanity"
super(RallySanity, self).__init__(**kwargs)
- self.test_name = 'all'
self.smoke = True
self.scenario_dir = os.path.join(self.RALLY_SCENARIO_DIR, 'sanity')
if "case_name" not in kwargs:
kwargs["case_name"] = "rally_full"
super(RallyFull, self).__init__(**kwargs)
- self.test_name = 'all'
self.smoke = False
self.scenario_dir = os.path.join(self.RALLY_SCENARIO_DIR, 'full')
if "case_name" not in kwargs:
kwargs["case_name"] = "rally_jobs"
super(RallyJobs, self).__init__(**kwargs)
- self.test_name = 'all'
self.task_file = os.path.join(self.RALLY_DIR, 'rally_jobs.yaml')
self.task_yaml = None
- def prepare_run(self):
+ def prepare_run(self, **kwargs):
"""Create resources needed by test scenarios."""
- super(RallyJobs, self).prepare_run()
+ super(RallyJobs, self).prepare_run(**kwargs)
with open(os.path.join(self.RALLY_DIR,
'rally_jobs.yaml'), 'r') as task_file:
self.task_yaml = yaml.safe_load(task_file)
- if not all(task in self.task_yaml for task in self.tests):
- raise Exception("Test '%s' not in '%s'" %
- (self.test_name, self.tests))
+ for task in self.task_yaml:
+ if task not in self.tests:
+ raise Exception("Test '%s' not in '%s'" %
+ (task, self.tests))
def apply_blacklist(self, case_file_name, result_file_name):
# pylint: disable=too-many-branches
"""Tempest configuration utilities."""
-from __future__ import print_function
-
import json
import logging
-import fileinput
import os
import subprocess
from functest.utils import functest_utils
-RALLY_CONF_PATH = "/etc/rally/rally.conf"
-RALLY_AARCH64_PATCH_PATH = pkg_resources.resource_filename(
- 'functest', 'ci/rally_aarch64_patch.conf')
GLANCE_IMAGE_PATH = os.path.join(
getattr(config.CONF, 'dir_functest_images'),
getattr(config.CONF, 'openstack_image_file_name'))
LOGGER = logging.getLogger(__name__)
-def create_rally_deployment(environ=None):
- """Create new rally deployment"""
- # set the architecture to default
- pod_arch = env.get("POD_ARCH")
- arch_filter = ['aarch64']
-
- if pod_arch and pod_arch in arch_filter:
- LOGGER.info("Apply aarch64 specific to rally config...")
- with open(RALLY_AARCH64_PATCH_PATH, "r") as pfile:
- rally_patch_conf = pfile.read()
-
- for line in fileinput.input(RALLY_CONF_PATH):
- print(line, end=' ')
- if "cirros|testvm" in line:
- print(rally_patch_conf)
-
- LOGGER.info("Creating Rally environment...")
-
- try:
- cmd = ['rally', 'deployment', 'destroy',
- '--deployment',
- str(getattr(config.CONF, 'rally_deployment_name'))]
- output = subprocess.check_output(cmd)
- LOGGER.info("%s\n%s", " ".join(cmd), output)
- except subprocess.CalledProcessError:
- pass
-
- cmd = ['rally', 'deployment', 'create', '--fromenv',
- '--name', str(getattr(config.CONF, 'rally_deployment_name'))]
- output = subprocess.check_output(cmd, env=environ)
- LOGGER.info("%s\n%s", " ".join(cmd), output)
-
- cmd = ['rally', 'deployment', 'check']
- output = subprocess.check_output(cmd)
- LOGGER.info("%s\n%s", " ".join(cmd), output)
- return get_verifier_deployment_id()
-
-
def create_verifier():
"""Create new verifier"""
LOGGER.info("Create verifier from existing repo...")
return verifier_uuid
-def get_verifier_deployment_id():
- """
- Returns deployment id for active Rally deployment
- """
- cmd = ("rally deployment list | awk '/" +
- getattr(config.CONF, 'rally_deployment_name') +
- "/ {print $2}'")
- proc = subprocess.Popen(cmd, shell=True,
- stdout=subprocess.PIPE,
- stderr=subprocess.STDOUT)
- deployment_uuid = proc.stdout.readline().rstrip()
- return deployment_uuid
-
-
def get_verifier_repo_dir(verifier_id):
"""
Returns installed verifier repo directory for Tempest
import yaml
from functest.core import singlevm
+from functest.opnfv_tests.openstack.rally import rally
from functest.opnfv_tests.openstack.tempest import conf_utils
from functest.utils import config
from functest.utils import env
return
with open(os.path.join(self.res_dir,
- "tempest.log"), 'r') as logfile:
+ "rally.log"), 'r') as logfile:
output = logfile.read()
success_testcases = []
output):
failed_testcases.append(match)
skipped_testcases = []
- for match in re.findall(r'.*\{\d{1,2}\} (.*?) \.{3} skip:',
+ for match in re.findall(r'.*\{\d{1,2}\} (.*?) \.{3} skip(?::| )',
output):
skipped_testcases.append(match)
LOGGER.info("Tempest %s success_rate is %s%%",
self.case_name, self.result)
- def generate_report(self):
- """Generate verification report."""
- html_file = os.path.join(self.res_dir,
- "tempest-report.html")
- cmd = ["rally", "verify", "report", "--type", "html", "--uuid",
- self.verification_id, "--to", str(html_file)]
- subprocess.check_output(cmd)
-
def update_rally_regex(self, rally_conf='/etc/rally/rally.conf'):
"""Set image name as tempest img_name_regex"""
rconfig = configparser.RawConfigParser()
del environ['OS_TENANT_ID']
except Exception: # pylint: disable=broad-except
pass
- self.deployment_id = conf_utils.create_rally_deployment(
+ self.deployment_id = rally.RallyBase.create_rally_deployment(
environ=environ)
if not self.deployment_id:
raise Exception("Deployment create failed")
self.apply_tempest_blacklist()
self.run_verifier_tests(**kwargs)
self.parse_verifier_result()
- self.generate_report()
+ rally.RallyBase.verify_report(
+ os.path.join(self.res_dir, "tempest-report.html"),
+ self.verification_id)
+ rally.RallyBase.verify_report(
+ os.path.join(self.res_dir, "tempest-report.xml"),
+ self.verification_id, "junit-xml")
res = testcase.TestCase.EX_OK
except Exception: # pylint: disable=broad-except
LOGGER.exception('Error with run')
flavor_alt_vcpus = 1
flavor_alt_disk = 10
- juju_timeout = '3600'
+ juju_timeout = '4800'
def __init__(self, **kwargs):
if "case_name" not in kwargs:
return output_dict
def _create_ellis_account(self, account_url, params):
- i = 50
+ i = 80
for iloop in range(i):
try:
req = requests.post(account_url, data=params)
except Exception: # pylint: disable=broad-except
self.logger.info(
"try %s: cannot create ellis account", iloop + 1)
- time.sleep(25)
+ time.sleep(30)
raise Exception(
"Unable to create an account {}".format(
params.get('full_name')))
filename_alt = ('/home/opnfv/functest/images/'
'ubuntu-14.04-server-cloudimg-amd64-disk1.img')
- flavor_alt_ram = 2048
- flavor_alt_vcpus = 2
- flavor_alt_disk = 25
+ flavor_alt_ram = 1024
+ flavor_alt_vcpus = 1
+ flavor_alt_disk = 3
quota_security_group = 20
quota_security_group_rule = 100
quota_port = 50
+ cop_yaml = ("https://github.com/cloudify-cosmo/cloudify-openstack-plugin/"
+ "releases/download/2.14.7/plugin.yaml")
+ cop_wgn = ("https://github.com/cloudify-cosmo/cloudify-openstack-plugin/"
+ "releases/download/2.14.7/cloudify_openstack_plugin-2.14.7-py27"
+ "-none-linux_x86_64-centos-Core.wgn")
+
def __init__(self, **kwargs):
"""Initialize CloudifyIms testcase object."""
if "case_name" not in kwargs:
scpc = scp.SCPClient(self.ssh.get_transport())
scpc.put(self.key_filename, remote_path='~/cloudify_ims.pem')
(_, stdout, stderr) = self.ssh.exec_command(
- "sudo cp ~/cloudify_ims.pem /etc/cloudify/ && "
- "sudo chmod 444 /etc/cloudify/cloudify_ims.pem && "
- "sudo yum install -y gcc python-devel python-cmd2 && "
- "cfy status")
+ "sudo docker exec cfy_manager_local "
+ "cfy plugins upload -y {} {} && "
+ "sudo docker cp ~/cloudify_ims.pem "
+ "cfy_manager_local:/etc/cloudify/ && "
+ "sudo docker exec cfy_manager_local "
+ "chmod 444 /etc/cloudify/cloudify_ims.pem && "
+ "sudo docker exec cfy_manager_local cfy status".format(
+ self.cop_yaml, self.cop_wgn))
self.__logger.info("output:\n%s", stdout.read())
self.__logger.info("error:\n%s", stderr.read())
self.cfy_client, execution, self.__logger)
self.cfy_client.deployments.delete(
self.vnf['descriptor'].get('name'))
+ time.sleep(10)
self.cfy_client.blueprints.delete(
self.vnf['descriptor'].get('name'))
except Exception: # pylint: disable=broad-except
filename = ('/home/opnfv/functest/images/'
'ubuntu-14.04-server-cloudimg-amd64-disk1.img')
- flavor_ram = 2048
- flavor_vcpus = 2
- flavor_disk = 25
+ flavor_ram = 1024
+ flavor_vcpus = 1
+ flavor_disk = 3
quota_security_group = 20
quota_security_group_rule = 100
quota_port = 50
+ parameters = {
+ 'private_mgmt_net_cidr': '192.168.100.0/24',
+ 'private_mgmt_net_gateway': '192.168.100.254',
+ 'private_mgmt_net_pool_start': '192.168.100.1',
+ 'private_mgmt_net_pool_end': '192.168.100.253',
+ 'private_sig_net_cidr': '192.168.101.0/24',
+ 'private_sig_net_gateway': '192.168.101.254',
+ 'private_sig_net_pool_start': '192.168.101.1',
+ 'private_sig_net_pool_end': '192.168.101.253'}
+
def __init__(self, **kwargs):
"""Initialize HeatIms testcase object."""
if "case_name" not in kwargs:
domain=self.project.domain.id)
self.keypair = self.cloud.create_keypair(
'{}-kp_{}'.format(self.case_name, self.guid))
- self.__logger.info("keypair: %s", self.keypair.private_key)
+ self.__logger.info("keypair:\n%s", self.keypair.private_key)
if self.deploy_vnf() and self.test_vnf():
self.result = 100
parameters['key_name'] = self.keypair.name
parameters['external_mgmt_dns_ip'] = env.get('NAMESERVER')
parameters['external_sig_dns_ip'] = env.get('NAMESERVER')
+ parameters.update(self.parameters)
self.__logger.info("Create Heat Stack")
self.stack = self.cloud.create_stack(
__logger = logging.getLogger(__name__)
- filename_alt = '/home/opnfv/functest/images/vyos-1.1.7.img'
+ filename_alt = '/home/opnfv/functest/images/vyos-1.1.8-amd64.qcow2'
- flavor_alt_ram = 2048
+ flavor_alt_ram = 1024
flavor_alt_vcpus = 1
- flavor_alt_disk = 50
+ flavor_alt_disk = 3
+
+ cop_yaml = ("https://github.com/cloudify-cosmo/cloudify-openstack-plugin/"
+ "releases/download/2.14.7/plugin.yaml")
+ cop_wgn = ("https://github.com/cloudify-cosmo/cloudify-openstack-plugin/"
+ "releases/download/2.14.7/cloudify_openstack_plugin-2.14.7-py27"
+ "-none-linux_x86_64-centos-Core.wgn")
def __init__(self, **kwargs):
if "case_name" not in kwargs:
scpc = scp.SCPClient(self.ssh.get_transport())
scpc.put(self.key_filename, remote_path='~/cloudify_ims.pem')
(_, stdout, stderr) = self.ssh.exec_command(
- "sudo cp ~/cloudify_ims.pem /etc/cloudify/ && "
- "sudo chmod 444 /etc/cloudify/cloudify_ims.pem && "
- "sudo yum install -y gcc python-devel python-cmd2 && "
- "cfy status")
+ "sudo docker exec cfy_manager_local "
+ "cfy plugins upload -y {} {} && "
+ "sudo docker cp ~/cloudify_ims.pem "
+ "cfy_manager_local:/etc/cloudify/ && "
+ "sudo docker exec cfy_manager_local "
+ "chmod 444 /etc/cloudify/cloudify_ims.pem && "
+ "sudo docker exec cfy_manager_local cfy status".format(
+ self.cop_yaml, self.cop_wgn))
self.__logger.info("output:\n%s", stdout.read())
self.__logger.info("error:\n%s", stderr.read())
def test_vnf(self):
start_time = time.time()
- testing = vrouter_base.VrouterOnBoardingBase(
- self.case_name, self.util, self.util_info)
+ testing = vrouter_base.VrouterOnBoardingBase(self.util, self.util_info)
result, test_result_data = testing.test_vnf()
duration = time.time() - start_time
if result:
self.cfy_client, execution, self.__logger)
self.cfy_client.deployments.delete(
self.vnf['descriptor'].get('name'))
+ time.sleep(10)
self.cfy_client.blueprints.delete(
self.vnf['descriptor'].get('name'))
except Exception: # pylint: disable=broad-except
import pkg_resources
-from functest.utils import config
from functest.opnfv_tests.vnf.router.test_controller import function_test_exec
__author__ = "Shuya Nakama <shuya.nakama@okinawaopenlabs.org>"
class VrouterOnBoardingBase(object):
"""vrouter testing base class"""
- def __init__(self, case_name, util, util_info):
+ def __init__(self, util, util_info):
self.logger = logging.getLogger(__name__)
self.case_dir = pkg_resources.resource_filename(
'functest', 'opnfv_tests/vnf/router')
- self.result_dir = os.path.join(
- getattr(config.CONF, 'dir_results'), case_name)
self.util = util
self.util_info = util_info
self.vnf_list = []
- if not os.path.exists(self.result_dir):
- os.makedirs(self.result_dir)
-
def test_vnf(self):
"""vrouter test execution"""
result = False
import json
import logging
import os
+import subprocess
import unittest
import mock
from xtesting.core import testcase
from functest.opnfv_tests.openstack.rally import rally
+from functest.utils import config
class OSRallyTesting(unittest.TestCase):
self.rally_base.image = munch.Munch(name='foo')
self.rally_base.flavor = munch.Munch(name='foo')
self.rally_base.flavor_alt = munch.Munch(name='bar')
- self.rally_base.test_name = 'all'
self.assertTrue(mock_get_config.called)
self.assertTrue(mock_shade.called)
self.assertTrue(mock_new_project.called)
return True
return False
+ @mock.patch('functest.opnfv_tests.openstack.rally.rally.'
+ 'RallyBase.get_verifier_deployment_id', return_value='foo')
+ @mock.patch('subprocess.check_output')
+ def test_create_rally_deployment(self, mock_exec, mock_get_id):
+ # pylint: disable=unused-argument
+ self.assertEqual(rally.RallyBase.create_rally_deployment(), 'foo')
+ calls = [
+ mock.call(['rally', 'deployment', 'destroy', '--deployment',
+ str(getattr(config.CONF, 'rally_deployment_name'))]),
+ mock.call(['rally', 'deployment', 'create', '--fromenv', '--name',
+ str(getattr(config.CONF, 'rally_deployment_name'))],
+ env=None),
+ mock.call(['rally', 'deployment', 'check'])]
+ mock_exec.assert_has_calls(calls)
+
@mock.patch('functest.opnfv_tests.openstack.rally.rally.os.path.exists')
@mock.patch('functest.opnfv_tests.openstack.rally.rally.os.makedirs')
@mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
def test_prepare_run_testname_invalid(self):
self.rally_base.TESTS = ['test1', 'test2']
- self.rally_base.test_name = 'test'
with self.assertRaises(Exception):
- self.rally_base.prepare_run()
+ self.rally_base.prepare_run(tests=['test'])
@mock.patch('functest.opnfv_tests.openstack.rally.rally.os.path.exists')
@mock.patch('functest.opnfv_tests.openstack.rally.rally.shutil.copyfile')
def test_prepare_run_flavor_alt_creation_failed(self, *args):
# pylint: disable=unused-argument
self.rally_base.TESTS = ['test1', 'test2']
- self.rally_base.test_name = 'test1'
with mock.patch.object(self.rally_base.cloud,
'list_hypervisors') as mock_list_hyperv, \
mock.patch.object(self.rally_base, 'create_flavor_alt',
side_effect=Exception) \
as mock_create_flavor:
with self.assertRaises(Exception):
- self.rally_base.prepare_run()
+ self.rally_base.prepare_run(tests=['test1'])
mock_list_hyperv.assert_called_once()
mock_create_flavor.assert_called_once()
'run_task')
def test_run_tests_all(self, mock_run_task, mock_prepare_task):
self.rally_base.tests = ['test1', 'test2']
- self.rally_base.test_name = 'all'
self.rally_base.run_tests()
mock_prepare_task.assert_any_call('test1')
mock_prepare_task.assert_any_call('test2')
self.rally_base.clean()
self.assertEqual(mock_delete_flavor.call_count, 1)
- @mock.patch('functest.opnfv_tests.openstack.tempest.conf_utils.'
+ @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
'create_rally_deployment')
@mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
'prepare_run')
'run_tests')
@mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
'_generate_report')
+ @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
+ 'export_task')
def test_run_default(self, *args):
self.assertEqual(self.rally_base.run(), testcase.TestCase.EX_OK)
for func in args:
func.assert_called()
- @mock.patch('functest.opnfv_tests.openstack.tempest.conf_utils.'
+ @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
'create_rally_deployment', side_effect=Exception)
def test_run_exception_create_rally_dep(self, mock_create_rally_dep):
self.assertEqual(self.rally_base.run(), testcase.TestCase.EX_RUN_ERROR)
mock_create_rally_dep.assert_called()
- @mock.patch('functest.opnfv_tests.openstack.tempest.conf_utils.'
+ @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
'create_rally_deployment', return_value=mock.Mock())
@mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
'prepare_run', side_effect=Exception)
mock_prep_env.assert_called()
def test_append_summary(self):
- text = '{"tasks": [{"subtasks": [{"workloads": [{"full_duration": ' \
- '1.23,"data": [{"error": []}]}]},{"workloads": ' \
- '[{"full_duration": 2.78, "data": [{"error": ["err"]}]}]}]}]}'
- self.rally_base._append_summary(text, "foo_test")
+ json_dict = {
+ 'tasks': [{
+ 'subtasks': [{
+ 'title': 'sub_task',
+ 'workloads': [{
+ 'full_duration': 1.23,
+ 'data': [{
+ 'error': []
+ }]
+ }, {
+ 'full_duration': 2.78,
+ 'data': [{
+ 'error': ['err']
+ }]
+ }]
+ }]
+ }]
+ }
+ self.rally_base._append_summary(json.dumps(json_dict), "foo_test")
self.assertEqual(self.rally_base.summary[0]['test_name'], "foo_test")
self.assertEqual(self.rally_base.summary[0]['overall_duration'], 4.01)
self.assertEqual(self.rally_base.summary[0]['nb_tests'], 2)
self.assertEqual(self.rally_base.summary[0]['nb_success'], 1)
+ self.assertEqual(self.rally_base.summary[0]['success'], [])
+ self.assertEqual(self.rally_base.summary[0]['failures'], ['sub_task'])
def test_is_successful_false(self):
with mock.patch('six.moves.builtins.super') as mock_super:
self.assertEqual(self.rally_base.is_successful(), 424)
mock_super(rally.RallyBase, self).is_successful.assert_called()
+ @mock.patch('subprocess.check_output',
+ side_effect=subprocess.CalledProcessError('', ''))
+ def test_export_task_ko(self, *args):
+ file_name = "{}/{}.html".format(
+ self.rally_base.results_dir, self.rally_base.case_name)
+ with self.assertRaises(subprocess.CalledProcessError):
+ self.rally_base.export_task(file_name)
+ cmd = ["rally", "task", "export", "--type", "html", "--deployment",
+ str(getattr(config.CONF, 'rally_deployment_name')),
+ "--to", file_name]
+ args[0].assert_called_with(cmd, stderr=subprocess.STDOUT)
+
+ @mock.patch('subprocess.check_output', return_value=None)
+ def test_export_task(self, *args):
+ file_name = "{}/{}.html".format(
+ self.rally_base.results_dir, self.rally_base.case_name)
+ self.assertEqual(self.rally_base.export_task(file_name), None)
+ cmd = ["rally", "task", "export", "--type", "html", "--deployment",
+ str(getattr(config.CONF, 'rally_deployment_name')),
+ "--to", file_name]
+ args[0].assert_called_with(cmd, stderr=subprocess.STDOUT)
+
+ @mock.patch('subprocess.check_output',
+ side_effect=subprocess.CalledProcessError('', ''))
+ def test_verify_report_ko(self, *args):
+ file_name = "{}/{}.html".format(
+ self.rally_base.results_dir, self.rally_base.case_name)
+ with self.assertRaises(subprocess.CalledProcessError):
+ self.rally_base.verify_report(file_name, "1")
+ cmd = ["rally", "verify", "report", "--type", "html", "--uuid", "1",
+ "--to", file_name]
+ args[0].assert_called_with(cmd, stderr=subprocess.STDOUT)
+
+ @mock.patch('subprocess.check_output', return_value=None)
+ def test_verify_report(self, *args):
+ file_name = "{}/{}.html".format(
+ self.rally_base.results_dir, self.rally_base.case_name)
+ self.assertEqual(self.rally_base.verify_report(file_name, "1"), None)
+ cmd = ["rally", "verify", "report", "--type", "html", "--uuid", "1",
+ "--to", file_name]
+ args[0].assert_called_with(cmd, stderr=subprocess.STDOUT)
+
if __name__ == "__main__":
logging.disable(logging.CRITICAL)
import mock
+from functest.opnfv_tests.openstack.rally import rally
from functest.opnfv_tests.openstack.tempest import conf_utils
from functest.utils import config
class OSTempestConfUtilsTesting(unittest.TestCase):
# pylint: disable=too-many-public-methods
- @mock.patch('functest.opnfv_tests.openstack.tempest.conf_utils'
- '.get_verifier_deployment_id', return_value='foo')
- @mock.patch('subprocess.check_output')
- def test_create_rally_deployment(self, mock_exec, mock_get_id):
- # pylint: disable=unused-argument
- self.assertEqual(conf_utils.create_rally_deployment(), 'foo')
- calls = [
- mock.call(['rally', 'deployment', 'destroy', '--deployment',
- str(getattr(config.CONF, 'rally_deployment_name'))]),
- mock.call(['rally', 'deployment', 'create', '--fromenv', '--name',
- str(getattr(config.CONF, 'rally_deployment_name'))],
- env=None),
- mock.call(['rally', 'deployment', 'check'])]
- mock_exec.assert_has_calls(calls)
-
@mock.patch('functest.opnfv_tests.openstack.tempest.conf_utils'
'.LOGGER.debug')
def test_create_verifier(self, mock_logger_debug):
@mock.patch('functest.opnfv_tests.openstack.tempest.conf_utils.'
'create_verifier', return_value=mock.Mock())
- @mock.patch('functest.opnfv_tests.openstack.tempest.conf_utils.'
- 'create_rally_deployment', return_value=mock.Mock())
+ @mock.patch('functest.opnfv_tests.openstack.rally.rally.'
+ 'RallyBase.create_rally_deployment', return_value=mock.Mock())
def test_get_verifier_id_default(self, mock_rally, mock_tempest):
# pylint: disable=unused-argument
setattr(config.CONF, 'tempest_verifier_name', 'test_verifier_name')
mock_stdout.configure_mock(**attrs)
mock_popen.return_value = mock_stdout
- self.assertEqual(conf_utils.get_verifier_deployment_id(),
+ self.assertEqual(rally.RallyBase.get_verifier_deployment_id(),
'test_deploy_id')
def test_get_verif_repo_dir_default(self):
from functest.opnfv_tests.openstack.tempest import tempest
from functest.opnfv_tests.openstack.tempest import conf_utils
-from functest.utils import config
class OSTempestTesting(unittest.TestCase):
with mock.patch('os_client_config.get_config'), \
mock.patch('shade.OpenStackCloud'), \
mock.patch('functest.core.tenantnetwork.NewProject'), \
- mock.patch('functest.opnfv_tests.openstack.tempest.tempest.'
- 'conf_utils.create_rally_deployment'), \
+ mock.patch('functest.opnfv_tests.openstack.rally.rally.'
+ 'RallyBase.create_rally_deployment'), \
mock.patch('functest.opnfv_tests.openstack.tempest.tempest.'
'conf_utils.create_verifier'), \
mock.patch('functest.opnfv_tests.openstack.tempest.tempest.'
'conf_utils.get_verifier_id',
return_value='test_deploy_id'), \
- mock.patch('functest.opnfv_tests.openstack.tempest.tempest.'
- 'conf_utils.get_verifier_deployment_id',
+ mock.patch('functest.opnfv_tests.openstack.rally.rally.'
+ 'RallyBase.get_verifier_deployment_id',
return_value='test_deploy_id'), \
mock.patch('functest.opnfv_tests.openstack.tempest.tempest.'
'conf_utils.get_verifier_repo_dir',
mock_logger_info. \
assert_any_call("Starting Tempest test suite: '%s'.", cmd)
- @mock.patch('subprocess.check_output')
- def test_generate_report(self, mock_popen):
- self.tempestcommon.verification_id = "1234"
- html_file = os.path.join(
- os.path.join(
- getattr(config.CONF, 'dir_results'),
- self.tempestcommon.case_name),
- "tempest-report.html")
- cmd = ["rally", "verify", "report", "--type", "html", "--uuid",
- "1234", "--to", html_file]
- self.tempestcommon.generate_report()
- mock_popen.assert_called_once_with(cmd)
-
@mock.patch('functest.opnfv_tests.openstack.tempest.tempest.'
'os.path.exists', return_value=False)
@mock.patch('functest.opnfv_tests.openstack.tempest.tempest.os.makedirs',
'apply_tempest_blacklist'), \
mock.patch.object(self.tempestcommon, 'run_verifier_tests'), \
mock.patch.object(self.tempestcommon,
- 'parse_verifier_result'), \
- mock.patch.object(self.tempestcommon, 'generate_report'):
+ 'parse_verifier_result'):
self._test_run(testcase.TestCase.EX_OK)
args[0].assert_called_once_with()
git+https://gerrit.opnfv.org/gerrit/parser#egg=nfv-heattranslator&subdirectory=tosca2heat/heat-translator
git+https://gerrit.opnfv.org/gerrit/parser#egg=nfv-toscaparser&subdirectory=tosca2heat/tosca-parser
-e git+https://gerrit.opnfv.org/gerrit/parser#egg=nfv-parser
-cloudify-rest-client===4.3.2
+cloudify-rest-client===4.3.3
robotframework===3.0.2
robotframework-httplibrary===0.4.2
robotframework-requests===0.4.7
Code Review / functest.git / commitdiff