From a0f76d884c1f13c27e906136d3418f12b5342fc4 Mon Sep 17 00:00:00 2001 From: =?utf8?q?C=C3=A9dric=20Ollivier?= Date: Sat, 22 Aug 2020 16:36:19 +0200 Subject: [PATCH] Add xrally_kubernetes_full MIME-Version: 1.0 Content-Type: text/plain; charset=utf8 Content-Transfer-Encoding: 8bit It iterates 10 times all operations run by xrally_kubernetes. Now times, concurrenty and count can be set via testcases.yaml. It creates a new containers functest-kubernetes-benchamrking as Functest. Change-Id: Ibfd493b987a359a3f16c2d8db0aef6b8e90b7e92 Signed-off-by: Cédric Ollivier --- .travis.yml | 26 ++++++ ansible/site.yml | 12 ++- build.sh | 9 ++- docker/benchmarking/Dockerfile | 4 + docker/benchmarking/hooks/post_checkout | 6 ++ docker/benchmarking/testcases.yaml | 25 ++++++ functest_kubernetes/rally/all-in-one.yaml | 112 +++++++++++++------------- functest_kubernetes/rally/rally_kubernetes.py | 17 +++- requirements.txt | 1 + tox.ini | 1 - 10 files changed, 149 insertions(+), 64 deletions(-) create mode 100644 docker/benchmarking/Dockerfile create mode 100644 docker/benchmarking/hooks/post_checkout create mode 100644 docker/benchmarking/testcases.yaml diff --git a/.travis.yml b/.travis.yml index aa8a2b67..46664fb8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -146,3 +146,29 @@ jobs: --platforms linux/amd64,linux/arm,linux/arm64 \ --template ${DOCKER_USERNAME}/functest-kubernetes-smoke:ARCH-latest \ --target ${DOCKER_USERNAME}/functest-kubernetes-smoke:latest + - stage: build functest-kubernetes-benchmarking image + script: bash build.sh + env: + - REPO="${DOCKER_USERNAME}" + - amd64_dirs="docker/benchmarking" + - arm_dirs="" + - arm64_dirs="" + - script: bash build.sh + env: + - REPO="${DOCKER_USERNAME}" + - amd64_dirs="" + - arm_dirs="docker/benchmarking" + - arm64_dirs="" + - script: bash build.sh + env: + - REPO="${DOCKER_USERNAME}" + - amd64_dirs="" + - arm_dirs="" + - arm64_dirs="docker/benchmarking" + - stage: publish functest-kubernetes-benchmarking manifests + script: > + manifest-tool push from-args \ + --platforms linux/amd64,linux/arm,linux/arm64 \ + --template \ + ${DOCKER_USERNAME}/functest-kubernetes-benchmarking:ARCH-latest \ + --target ${DOCKER_USERNAME}/functest-kubernetes-benchmarking:latest diff --git a/ansible/site.yml b/ansible/site.yml index b1294951..939ab915 100644 --- a/ansible/site.yml +++ b/ansible/site.yml @@ -17,7 +17,7 @@ - name: functest-kubernetes-core ref_arg: BRANCH path: docker/core - - name: build opnfv/functest-kubernetes-healthcheck + - name: build containers containers: - name: functest-kubernetes-healthcheck ref_arg: @@ -28,11 +28,16 @@ - name: functest-kubernetes-security ref_arg: BRANCH path: docker/security - - name: build containers + - name: build opnfv/functest-kubernetes-smoke containers: - name: functest-kubernetes-smoke ref_arg: path: docker/smoke + - name: build opnfv/functest-kubernetes-benchmarking + containers: + - name: functest-kubernetes-benchmarking + ref_arg: + path: docker/benchmarking suites: - container: functest-kubernetes-healthcheck tests: @@ -45,6 +50,9 @@ tests: - kube_hunter - kube_bench + - container: functest-kubernetes-benchmarking + tests: + - xrally_kubernetes_full - container: functest-kubernetes-cnf tests: - k8s_vims diff --git a/build.sh b/build.sh index 8fb23242..fbd5001e 100644 --- a/build.sh +++ b/build.sh @@ -8,7 +8,8 @@ docker/core \ docker/healthcheck \ docker/smoke \ docker/cnf \ -docker/security"} +docker/security \ +docker/benchmarking"} arm_dirs=${arm_dirs-${amd64_dirs}} arm64_dirs=${arm64_dirs-${amd64_dirs}} build_opts=(--pull=true --no-cache --force-rm=true) @@ -42,6 +43,9 @@ ${repo}/functest-kubernetes-core:arm64-latest|g" {} + find . -name Dockerfile -exec sed -i \ -e "s|opnfv/functest-kubernetes-healthcheck|\ ${repo}/functest-kubernetes-healthcheck:arm64-latest|g" {} + +find . -name Dockerfile -exec sed -i \ + -e "s|opnfv/functest-kubernetes-smoke|\ +${repo}/functest-kubernetes-smoke:arm64-latest|g" {} + for dir in ${arm64_dirs}; do (cd "${dir}" && docker build "${build_opts[@]}" \ -t "${repo}/functest-kubernetes-${dir##**/}:arm64-latest" .) @@ -63,6 +67,9 @@ ${repo}/functest-kubernetes-core:arm-latest|g" {} + find . -name Dockerfile -exec sed -i \ -e "s|opnfv/functest-kubernetes-healthcheck|\ ${repo}/functest-kubernetes-healthcheck:arm-latest|g" {} + +find . -name Dockerfile -exec sed -i \ + -e "s|opnfv/functest-kubernetes-smoke|\ +${repo}/functest-kubernetes-smoke:arm-latest|g" {} + for dir in ${arm_dirs}; do (cd "${dir}" && docker build "${build_opts[@]}" \ -t "${repo}/functest-kubernetes-${dir##**/}:arm-latest" .) diff --git a/docker/benchmarking/Dockerfile b/docker/benchmarking/Dockerfile new file mode 100644 index 00000000..6625322c --- /dev/null +++ b/docker/benchmarking/Dockerfile @@ -0,0 +1,4 @@ +FROM opnfv/functest-kubernetes-smoke + +COPY testcases.yaml /usr/lib/python3.8/site-packages/xtesting/ci/testcases.yaml +CMD ["run_tests", "-t", "all"] diff --git a/docker/benchmarking/hooks/post_checkout b/docker/benchmarking/hooks/post_checkout new file mode 100644 index 00000000..6598f040 --- /dev/null +++ b/docker/benchmarking/hooks/post_checkout @@ -0,0 +1,6 @@ +#!/bin/bash + +from="${DOCKER_REPO%/*}/functest-kubernetes-smoke:${DOCKER_TAG}" +sed -i "s|^FROM.*$|FROM ${from}|" Dockerfile + +exit $? diff --git a/docker/benchmarking/testcases.yaml b/docker/benchmarking/testcases.yaml new file mode 100644 index 00000000..f4800095 --- /dev/null +++ b/docker/benchmarking/testcases.yaml @@ -0,0 +1,25 @@ +--- +tiers: + - + name: benchmarking + order: 1 + description: >- + A set of benchmarking tests integrated from xRally for Kubernetes + platform. + testcases: + - + case_name: xrally_kubernetes_full + project_name: functest + criteria: 100 + blocking: false + description: >- + All in one tasks for checking basic functionality of + Kubernetes cluster. + dependencies: + - DEPLOY_SCENARIO: 'k8-*' + run: + name: xrally_kubernetes + args: + times: 10 + concurrency: 4 + namespaces_count: 3 diff --git a/functest_kubernetes/rally/all-in-one.yaml b/functest_kubernetes/rally/all-in-one.yaml index 9437f760..ae4157ff 100644 --- a/functest_kubernetes/rally/all-in-one.yaml +++ b/functest_kubernetes/rally/all-in-one.yaml @@ -8,16 +8,16 @@ subtasks: Kubernetes.list_namespaces: {} runner: constant: - concurrency: 1 - times: 1 + concurrency: {{ concurrency }} + times: {{ times }} - title: Run a single workload with create/read/delete namespace scenario: Kubernetes.create_and_delete_namespace: {} runner: constant: - concurrency: 1 - times: 1 + concurrency: {{ concurrency }} + times: {{ times }} - title: Run a single workload with create/read/delete pod scenario: @@ -25,11 +25,11 @@ subtasks: image: kubernetes/pause runner: constant: - concurrency: 1 - times: 1 + concurrency: {{ concurrency }} + times: {{ times }} contexts: namespaces: - count: 3 + count: {{ namespaces_count }} with_serviceaccount: true - title: Run a single workload with create/read/delete replication controller @@ -39,11 +39,11 @@ subtasks: replicas: 2 runner: constant: - concurrency: 1 - times: 1 + concurrency: {{ concurrency }} + times: {{ times }} contexts: namespaces: - count: 3 + count: {{ namespaces_count }} with_serviceaccount: true - title: Run a single workload with create/scale/delete replication controller @@ -54,11 +54,11 @@ subtasks: scale_replicas: 3 runner: constant: - concurrency: 1 - times: 1 + concurrency: {{ concurrency }} + times: {{ times }} contexts: namespaces: - count: 3 + count: {{ namespaces_count }} with_serviceaccount: true - title: Run a single workload with create/read/delete replicaset @@ -68,11 +68,11 @@ subtasks: replicas: 1 runner: constant: - concurrency: 1 - times: 1 + concurrency: {{ concurrency }} + times: {{ times }} contexts: namespaces: - count: 3 + count: {{ namespaces_count }} with_serviceaccount: true - title: Run a single workload with create/scale/delete replicaset @@ -83,11 +83,11 @@ subtasks: scale_replicas: 2 runner: constant: - concurrency: 1 - times: 1 + concurrency: {{ concurrency }} + times: {{ times }} contexts: namespaces: - count: 3 + count: {{ namespaces_count }} with_serviceaccount: true - title: @@ -98,11 +98,11 @@ subtasks: mount_path: /opt/check runner: constant: - concurrency: 1 - times: 1 + concurrency: {{ concurrency }} + times: {{ times }} contexts: namespaces: - count: 3 + count: {{ namespaces_count }} with_serviceaccount: true - title: >- @@ -125,7 +125,7 @@ subtasks: times: 1 contexts: namespaces: - count: 3 + count: {{ namespaces_count }} with_serviceaccount: true - title: Run a single workload with create/read/delete pod with secret volume @@ -138,11 +138,11 @@ subtasks: mount_path: /opt/check runner: constant: - concurrency: 1 - times: 1 + concurrency: {{ concurrency }} + times: {{ times }} contexts: namespaces: - count: 3 + count: {{ namespaces_count }} with_serviceaccount: true - title: Run a single workload with create/check/delete pod with secret volume @@ -159,11 +159,11 @@ subtasks: error_regexp: No such file runner: constant: - concurrency: 1 - times: 1 + concurrency: {{ concurrency }} + times: {{ times }} contexts: namespaces: - count: 3 + count: {{ namespaces_count }} with_serviceaccount: true - title: >- @@ -188,7 +188,7 @@ subtasks: times: 1 contexts: namespaces: - count: 3 + count: {{ namespaces_count }} with_serviceaccount: true - title: @@ -206,11 +206,11 @@ subtasks: test runner: constant: - concurrency: 1 - times: 1 + concurrency: {{ concurrency }} + times: {{ times }} contexts: namespaces: - count: 3 + count: {{ namespaces_count }} with_serviceaccount: true - title: >- @@ -237,7 +237,7 @@ subtasks: times: 1 contexts: namespaces: - count: 3 + count: {{ namespaces_count }} with_serviceaccount: true - title: Run a single workload with create/read/delete deployment @@ -247,11 +247,11 @@ subtasks: replicas: 2 runner: constant: - concurrency: 1 - times: 1 + concurrency: {{ concurrency }} + times: {{ times }} contexts: namespaces: - count: 3 + count: {{ namespaces_count }} with_serviceaccount: true - title: Run a single workload with create/rollout/delete deployment @@ -271,11 +271,11 @@ subtasks: value: "true" runner: constant: - concurrency: 1 - times: 1 + concurrency: {{ concurrency }} + times: {{ times }} contexts: namespaces: - count: 3 + count: {{ namespaces_count }} with_serviceaccount: true - title: Run a single workload with create/read/delete statefulset @@ -285,11 +285,11 @@ subtasks: replicas: 2 runner: constant: - concurrency: 1 - times: 1 + concurrency: {{ concurrency }} + times: {{ times }} contexts: namespaces: - count: 3 + count: {{ namespaces_count }} with_serviceaccount: true - title: Run a single workload with create/scale/delete statefulset @@ -300,11 +300,11 @@ subtasks: scale_replicas: 2 runner: constant: - concurrency: 1 - times: 1 + concurrency: {{ concurrency }} + times: {{ times }} contexts: namespaces: - count: 3 + count: {{ namespaces_count }} with_serviceaccount: true - title: Run a single workload with create/read/delete job @@ -316,11 +316,11 @@ subtasks: - "SUCCESS" runner: constant: - concurrency: 1 - times: 1 + concurrency: {{ concurrency }} + times: {{ times }} contexts: namespaces: - count: 3 + count: {{ namespaces_count }} with_serviceaccount: true - title: Run a single workload with create/check/delete clusterIP service @@ -331,11 +331,11 @@ subtasks: protocol: TCP runner: constant: - concurrency: 1 - times: 2 + concurrency: {{ concurrency }} + times: {{ times }} contexts: namespaces: - count: 3 + count: {{ namespaces_count }} with_serviceaccount: true - title: >- @@ -353,7 +353,7 @@ subtasks: times: 1 contexts: namespaces: - count: 3 + count: {{ namespaces_count }} with_serviceaccount: true - title: Run a single workload with create/check/delete NodePort service @@ -365,9 +365,9 @@ subtasks: request_timeout: 10 runner: constant: - concurrency: 1 - times: 1 + concurrency: {{ concurrency }} + times: {{ times }} contexts: namespaces: - count: 3 + count: {{ namespaces_count }} with_serviceaccount: true diff --git a/functest_kubernetes/rally/rally_kubernetes.py b/functest_kubernetes/rally/rally_kubernetes.py index e7a903d9..28415579 100644 --- a/functest_kubernetes/rally/rally_kubernetes.py +++ b/functest_kubernetes/rally/rally_kubernetes.py @@ -11,6 +11,7 @@ import logging import os import time +from jinja2 import Template import pkg_resources from rally import api from rally import exceptions @@ -26,6 +27,10 @@ class RallyKubernetes(testcase.TestCase): __logger = logging.getLogger(__name__) + concurrency = 1 + times = 1 + namespaces_count = 1 + def __init__(self, **kwargs): super(RallyKubernetes, self).__init__(**kwargs) self.dir_results = "/home/opnfv/functest/results" @@ -53,10 +58,14 @@ class RallyKubernetes(testcase.TestCase): "Cannot check env heath: %s", result['existing@kubernetes']['message']) return - input_task = open( - pkg_resources.resource_filename( - 'functest_kubernetes', 'rally/all-in-one.yaml')).read() - task = yaml.safe_load(input_task) + with open(pkg_resources.resource_filename( + 'functest_kubernetes', 'rally/all-in-one.yaml')) as file: + template = Template(file.read()) + task = yaml.safe_load(template.render( + concurrency=kwargs.get("concurrency", self.concurrency), + times=kwargs.get("times", self.times), + namespaces_count=kwargs.get( + "namespaces_count", self.namespaces_count))) rapi.task.validate(deployment='my-kubernetes', config=task) task_instance = rapi.task.create(deployment='my-kubernetes') rapi.task.start( diff --git a/requirements.txt b/requirements.txt index 742dcbf3..60c38eb2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,3 +6,4 @@ xtesting # Apache-2.0 rally xrally-kubernetes kubernetes # Apache-2.0 +Jinja2 # BSD License (3 clause) diff --git a/tox.ini b/tox.ini index d055b8f1..fb27ac71 100644 --- a/tox.ini +++ b/tox.ini @@ -32,7 +32,6 @@ basepython = python3.8 files = .travis.yml docker - functest_kubernetes/rally/all-in-one.yaml commands = yamllint {[testenv:yamllint]files} -- 2.16.6