.. This work is licensed under a Creative Commons Attribution 4.0 International License.
.. http://creativecommons.org/licenses/by/4.0
-.. (c) OPNFV, Intel Corporation, AT&T, Red Hat, Spirent, Ixia and others.
+.. (c) OPNFV, Spirent, AT&T, Ixia and others.
.. OPNFV VSPERF Documentation master file.
-**************************
-OPNFV VSPERF K8S CNB Guide
-**************************
+=========================================================
+OPNFV VSPERF Kubernetes Container Networking Benchmarking
+=========================================================
+VSPERF supports testing and benchmarking of kubernetes container networking solution, referred as kubernetes Container Networking Benchmarking (CNB). The process can be broadly classified into following four operations.
-============
-Introduction
-============
+1. Setting up of Kubernetes Cluster.
+2. Deploying container networking solution.
+3. Deploying pod(s).
+4. Running tests.
+First step is achieved through the tool present in *tools/k8s/cluster-deployment* folder. Please refer to the documentation present in that folder for automated kubernetes cluster setup. To perform the remaining steps, the user has to run the following command.
+
+.. code-block:: console
+
+ vsperf --k8s --conf-file k8s.conf pcp_tput
+
+************************
+Important Configurations
+************************
+
+VSPERF has introduced a new configuration parameters, as listed below, for kubernetes CNB. The file *12_k8s.conf*, present in conf folder provides sample values. User has to modify these parameters to suit their environment before running the above command.
+
+1. K8S_CONFIG_FILEPATH - location of the kubernetes-cluster access file. This will be used to connect to the cluster.
+2. PLUGIN - The plugin to use. Allowed values are OvsDPDK, VPP, and SRIOV.
+3. NETWORK_ATTACHMENT_FILEPATH - location of the network attachment definition file.
+4. CONFIGMAP_FILEPATH - location of the config-map file. This will be used only for SRIOV plugin.
+5. POD_MANIFEST_FILEPATH - location of the POD definition file.
+6. APP_NAME - Application to run in the pod. Options - l2fwd, testpmd, and l3fwd.
+
+
+*********
+Testcases
+*********
+Kubernetes CNB will be done through new testcases. For Jerma release, only pcp_tput will be supported. This testcase, will be similar to pvp_tput, where VNF is replaced with a pod/container. The pcp_tput testcase, will still use phy2phy as deployment. In future releases, a new deployment model will be added to support more testcases for kubernetes
--- /dev/null
+#!/usr/bin/env bash
+##
+## Copyright (c) 2010-2020 Intel Corporation
+##
+## Licensed under the Apache License, Version 2.0 (the "License");
+## you may not use this file except in compliance with the License.
+## You may obtain a copy of the License at
+##
+## http://www.apache.org/licenses/LICENSE-2.0
+##
+## Unless required by applicable law or agreed to in writing, software
+## distributed under the License is distributed on an "AS IS" BASIS,
+## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+## See the License for the specific language governing permissions and
+## limitations under the License.
+##
+
+# Directory for package build
+BUILD_DIR="/opt/rapid"
+TREX_DIR="/opt/trex"
+RAMSPEED_DIR="/opt/ramspeed"
+STRESSNG_DIR="/opt/stressng"
+UNIXBENCH_DIR="/opt/unixbench"
+DPDK_VERSION="20.05"
+PROX_COMMIT="80dfeb5c734cc4d681f467e853a541a8a91fe1cf"
+PROX_CHECKOUT="git checkout ${PROX_COMMIT}"
+## Next line is overruling the PROX_COMMIT and will replace the version with a very specific patch. Should be commented out
+## if you want to use a committed version of PROX with the COMMIT ID specified above
+## As an example: Following line has the commit for testing IMIX, IPV6, ... It is the merge of all PROX commits on May 27th 2020
+#PROX_CHECKOUT="git fetch \"https://gerrit.opnfv.org/gerrit/samplevnf\" refs/changes/23/70223/1 && git checkout FETCH_HEAD"
+MULTI_BUFFER_LIB_VER="0.52"
+export RTE_SDK="${BUILD_DIR}/dpdk-${DPDK_VERSION}"
+export RTE_TARGET="x86_64-native-linuxapp-gcc"
+
+# By default, do not update OS
+OS_UPDATE="n"
+# By default, asumming that we are in the VM
+K8S_ENV="n"
+
+# If already running from root, no need for sudo
+SUDO=""
+[ $(id -u) -ne 0 ] && SUDO="sudo"
+
+function os_pkgs_install()
+{
+ ${SUDO} yum install -y deltarpm yum-utils
+
+ # NASM repository for AESNI MB library
+ #${SUDO} yum-config-manager --add-repo http://www.nasm.us/nasm.repo
+
+ [ "${OS_UPDATE}" == "y" ] && ${SUDO} yum update -y
+ ${SUDO} yum install -y git wget gcc unzip libpcap-devel ncurses-devel \
+ libedit-devel lua-devel kernel-devel iperf3 pciutils \
+ numactl-devel vim tuna openssl-devel wireshark \
+ make driverctl
+
+ ${SUDO} wget https://www.nasm.us/pub/nasm/releasebuilds/2.14.02/linux/nasm-2.14.02-0.fc27.x86_64.rpm
+ ${SUDO} rpm -ivh nasm-2.14.02-0.fc27.x86_64.rpm
+}
+
+function k8s_os_pkgs_runtime_install()
+{
+ [ "${OS_UPDATE}" == "y" ] && ${SUDO} yum update -y
+
+ # Install required dynamically linked libraries + required packages
+ ${SUDO} yum install -y numactl-libs libpcap openssh openssh-server \
+ openssh-clients sudo
+}
+
+function os_cfg()
+{
+ # huge pages to be used by DPDK
+ ${SUDO} sh -c '(echo "vm.nr_hugepages = 1024") > /etc/sysctl.conf'
+
+ ${SUDO} sh -c '(echo "options vfio enable_unsafe_noiommu_mode=1") > /etc/modprobe.d/vfio.conf'
+ ${SUDO} sh -c '(echo "vfio") > /etc/modules-load.d/vfio.conf'
+ ${SUDO} sh -c '(echo "vfio-pci") > /etc/modules-load.d/vfio.conf'
+ # Enabling tuned with the realtime-virtual-guest profile
+ pushd ${BUILD_DIR} > /dev/null 2>&1
+ wget http://linuxsoft.cern.ch/cern/centos/7/rt/x86_64/Packages/tuned-profiles-realtime-2.8.0-5.el7_4.2.noarch.rpm
+ wget http://linuxsoft.cern.ch/cern/centos/7/rt/x86_64/Packages/tuned-profiles-nfv-guest-2.8.0-5.el7_4.2.noarch.rpm
+ # Install with --nodeps. The latest CentOS cloud images come with a tuned version higher than 2.8. These 2 packages however
+ # do not depend on v2.8 and also work with tuned 2.9. Need to be careful in the future
+ ${SUDO} rpm -ivh ${BUILD_DIR}/tuned-profiles-realtime-2.8.0-5.el7_4.2.noarch.rpm --nodeps
+ ${SUDO} rpm -ivh ${BUILD_DIR}/tuned-profiles-nfv-guest-2.8.0-5.el7_4.2.noarch.rpm --nodeps
+ # Although we do no know how many cores the VM will have when begin deployed for real testing, we already put a number for the
+ # isolated CPUs so we can start the realtime-virtual-guest profile. If we don't, that command will fail.
+ # When the VM will be instantiated, the check_kernel_params service will check for the real number of cores available to this VM
+ # and update the realtime-virtual-guest-variables.conf accordingly.
+ echo "isolated_cores=1-3" | ${SUDO} tee -a /etc/tuned/realtime-virtual-guest-variables.conf
+ ${SUDO} tuned-adm profile realtime-virtual-guest
+
+ # Install the check_tuned_params service to make sure that the grub cmd line has the right cpus in isolcpu. The actual number of cpu's
+ # assigned to this VM depends on the flavor used. We don't know at this time what that will be.
+ ${SUDO} chmod +x ${BUILD_DIR}/check_prox_system_setup.sh
+ ${SUDO} mv ${BUILD_DIR}/check_prox_system_setup.sh /usr/local/libexec/
+ ${SUDO} mv ${BUILD_DIR}/check-prox-system-setup.service /etc/systemd/system/
+ ${SUDO} systemctl daemon-reload
+ ${SUDO} systemctl enable check-prox-system-setup.service
+ popd > /dev/null 2>&1
+}
+
+function k8s_os_cfg()
+{
+ [ ! -f /etc/ssh/ssh_host_rsa_key ] && ssh-keygen -t rsa -f /etc/ssh/ssh_host_rsa_key -N ''
+ [ ! -f /etc/ssh/ssh_host_ecdsa_key ] && ssh-keygen -t ecdsa -f /etc/ssh/ssh_host_ecdsa_key -N ''
+ [ ! -f /etc/ssh/ssh_host_ed25519_key ] && ssh-keygen -t ed25519 -f /etc/ssh/ssh_host_ed25519_key -N ''
+
+ [ ! -d /var/run/sshd ] && mkdir -p /var/run/sshd
+
+ USER_NAME="centos"
+ USER_PWD="centos"
+
+ useradd -m -d /home/${USER_NAME} -s /bin/bash -U ${USER_NAME}
+ echo "${USER_NAME}:${USER_PWD}" | chpasswd
+ usermod -aG wheel ${USER_NAME}
+
+ echo "%wheel ALL=(ALL) NOPASSWD: ALL" > /etc/sudoers.d/wheelnopass
+}
+
+function mblib_install()
+{
+ export AESNI_MULTI_BUFFER_LIB_PATH="${BUILD_DIR}/intel-ipsec-mb-${MULTI_BUFFER_LIB_VER}"
+
+ # Downloading the Multi-buffer library. Note that the version to download is linked to the DPDK version being used
+ pushd ${BUILD_DIR} > /dev/null 2>&1
+ wget https://github.com/01org/intel-ipsec-mb/archive/v${MULTI_BUFFER_LIB_VER}.zip
+ unzip v${MULTI_BUFFER_LIB_VER}.zip
+ pushd ${AESNI_MULTI_BUFFER_LIB_PATH}
+ make -j`getconf _NPROCESSORS_ONLN`
+ ${SUDO} make install
+ popd > /dev/null 2>&1
+ popd > /dev/null 2>&1
+}
+
+function trex_install()
+{
+ pushd ${TREX_DIR} > /dev/null 2>&1
+ wget --no-cache https://trex-tgen.cisco.com/trex/release/latest
+ tar -xzvf latest
+ popd > /dev/null 2>&1
+}
+
+function unixbench_install()
+{
+ pushd ${UNIXBENCH_DIR} > /dev/null 2>&1
+ git clone https://github.com/kdlucas/byte-unixbench
+ popd > /dev/null 2>&1
+}
+
+function ramspeed_install()
+{
+ RAMSPEED_BUILD_DIR = "${RAMSPEED_DIR}/ramspeed-smp"
+ pushd ${RAMSPEED_DIR} > /dev/null 2>&1
+ git clone https://github.com/cruvolo/ramspeed-smp
+ pushd ${RAMSPEED_BUILD_DIR} > /dev/null 2>&1
+ chmod 766 build.sh
+ source build.sh
+ popd > /dev/null 2>&1
+ popd > /dev/null 2>&1
+}
+
+function collectd_install()
+{
+ ${SUDO} yum -y install collectd
+}
+
+function fio_install()
+{
+ ${SUDO} yum -y install fio
+}
+
+function stressng_install()
+{
+ STRESSNG_BUILD_DIR = "${STRESSNG_DIR}/stress-ng"
+ pushd ${STRESSNG_DIR} > /dev/null 2>&1
+ git clone https://github.com/ColinIanKing/stress-ng
+ ${SUDO} yum -y install libaio-devel libbsd-devel libcap-devel libattr-devel libgcrypt-devel
+ ${SUDO} yum -y install Judy-devel keyutils-libs-devel lksctp-tools-devel libatomic zlib-devel
+ pushd ${STRESSNG_BUILD_DIR} > /dev/null 2>&1
+ make clean
+ make
+ ${SUDO} make install
+ popd > /dev/null 2>&1
+ popd > /dev/null 2>&1
+}
+
+function dpdk_install()
+{
+ # Build DPDK for the latest kernel installed
+ LATEST_KERNEL_INSTALLED=`ls -v1 /lib/modules/ | tail -1`
+ export RTE_KERNELDIR="/lib/modules/${LATEST_KERNEL_INSTALLED}/build"
+
+ # Get and compile DPDK
+ pushd ${BUILD_DIR} > /dev/null 2>&1
+ wget http://fast.dpdk.org/rel/dpdk-${DPDK_VERSION}.tar.xz
+ tar -xf ./dpdk-${DPDK_VERSION}.tar.xz
+ popd > /dev/null 2>&1
+
+ ${SUDO} ln -s ${RTE_SDK} ${BUILD_DIR}/dpdk
+
+ pushd ${RTE_SDK} > /dev/null 2>&1
+ make config T=${RTE_TARGET}
+ # Starting from DPDK 20.05, the IGB_UIO driver is not compiled by default.
+ # Uncomment the sed command to enable the driver compilation
+ #${SUDO} sed -i 's/CONFIG_RTE_EAL_IGB_UIO=n/c\/CONFIG_RTE_EAL_IGB_UIO=y' ${RTE_SDK}/build/.config
+
+ # For Kubernetes environment we use host vfio module
+ if [ "${K8S_ENV}" == "y" ]; then
+ sed -i 's/CONFIG_RTE_EAL_IGB_UIO=y/CONFIG_RTE_EAL_IGB_UIO=n/g' ${RTE_SDK}/build/.config
+ sed -i 's/CONFIG_RTE_LIBRTE_KNI=y/CONFIG_RTE_LIBRTE_KNI=n/g' ${RTE_SDK}/build/.config
+ sed -i 's/CONFIG_RTE_KNI_KMOD=y/CONFIG_RTE_KNI_KMOD=n/g' ${RTE_SDK}/build/.config
+ fi
+
+ # Compile with MB library
+ sed -i '/CONFIG_RTE_LIBRTE_PMD_AESNI_MB=n/c\CONFIG_RTE_LIBRTE_PMD_AESNI_MB=y' ${RTE_SDK}/build/.config
+ make -j`getconf _NPROCESSORS_ONLN`
+ ln -s ${RTE_SDK}/build ${RTE_SDK}/${RTE_TARGET}
+ popd > /dev/null 2>&1
+}
+
+function prox_compile()
+{
+ # Compile PROX
+ pushd ${BUILD_DIR}/samplevnf/VNFs/DPPD-PROX
+ make -j`getconf _NPROCESSORS_ONLN`
+ ${SUDO} cp ${BUILD_DIR}/samplevnf/VNFs/DPPD-PROX/build/app/prox ${BUILD_DIR}/prox
+ popd > /dev/null 2>&1
+}
+
+function prox_install()
+{
+ # Clone and compile PROX
+ pushd ${BUILD_DIR} > /dev/null 2>&1
+ git clone https://git.opnfv.org/samplevnf
+ pushd ${BUILD_DIR}/samplevnf/VNFs/DPPD-PROX > /dev/null 2>&1
+ bash -c "${PROX_CHECKOUT}"
+ popd > /dev/null 2>&1
+ prox_compile
+ popd > /dev/null 2>&1
+}
+
+function port_info_build()
+{
+ [ ! -d ${BUILD_DIR}/port_info ] && echo "Skipping port_info compilation..." && return
+
+ pushd ${BUILD_DIR}/port_info > /dev/null 2>&1
+ make
+ ${SUDO} cp ${BUILD_DIR}/port_info/build/app/port_info_app ${BUILD_DIR}/port_info_app
+ popd > /dev/null 2>&1
+}
+
+function create_minimal_install()
+{
+ ldd ${BUILD_DIR}/prox | awk '{ if ($(NF-1) != "=>") print $(NF-1) }' >> ${BUILD_DIR}/list_of_install_components
+
+ echo "${BUILD_DIR}/prox" >> ${BUILD_DIR}/list_of_install_components
+ echo "${BUILD_DIR}/port_info_app" >> ${BUILD_DIR}/list_of_install_components
+
+ tar -czvhf ${BUILD_DIR}/install_components.tgz -T ${BUILD_DIR}/list_of_install_components
+}
+
+function cleanup()
+{
+ ${SUDO} yum autoremove -y
+ ${SUDO} yum clean all
+ ${SUDO} rm -rf /var/cache/yum
+}
+
+function k8s_runtime_image()
+{
+ k8s_os_pkgs_runtime_install
+ k8s_os_cfg
+ cleanup
+
+ pushd / > /dev/null 2>&1
+ tar -xvf ${BUILD_DIR}/install_components.tgz --skip-old-files
+ popd > /dev/null 2>&1
+
+ ldconfig
+
+ #rm -rf ${BUILD_DIR}/install_components.tgz
+}
+
+function print_usage()
+{
+ echo "Usage: ${0} [OPTIONS] [COMMAND]"
+ echo "Options:"
+ echo " -u, --update Full OS update"
+ echo " -k, --kubernetes Build for Kubernetes environment"
+ echo "Commands:"
+ echo " deploy Run through all deployment steps"
+ echo " compile PROX compile only"
+ echo " runtime_image Apply runtime configuration only"
+}
+
+COMMAND=""
+# Parse options and comman
+for opt in "$@"; do
+ case ${opt} in
+ -u|--update)
+ echo 'Full OS update will be done!'
+ OS_UPDATE="y"
+ ;;
+ -k|--kubernetes)
+ echo "Kubernetes environment is set!"
+ K8S_ENV="y"
+ ;;
+ compile)
+ COMMAND="compile"
+ ;;
+ runtime_image)
+ COMMAND="runtime_image"
+ ;;
+ deploy)
+ COMMAND="deploy"
+ ;;
+ *)
+ echo "Unknown option/command ${opt}"
+ print_usage
+ exit 1
+ ;;
+ esac
+done
+
+if [ "${COMMAND}" == "compile" ]; then
+ echo "PROX compile only..."
+ prox_compile
+elif [ "${COMMAND}" == "runtime_image" ]; then
+ echo "Runtime image intallation and configuration..."
+ k8s_runtime_image
+elif [ "${COMMAND}" == "deploy" ]; then
+ [ ! -d ${BUILD_DIR} ] && ${SUDO} mkdir -p ${BUILD_DIR}
+ ${SUDO} chmod 0777 ${BUILD_DIR}
+
+ os_pkgs_install
+
+ if [ "${K8S_ENV}" == "y" ]; then
+ k8s_os_cfg
+ else
+ os_cfg
+ fi
+
+ mblib_install
+ dpdk_install
+ prox_install
+ trex_install
+ collectd_install
+ stressng_install
+ fio_install
+ unixbench_install
+ ramspeed_install
+
+
+
+ if [ "${K8S_ENV}" == "y" ]; then
+ port_info_build
+ create_minimal_install
+ fi
+
+ cleanup
+else
+ print_usage
+fi
--- /dev/null
+{
+ "builders": [
+ {
+ "boot_command": [
+ "<tab> text biosdevname=0 net.ifnames=0 ",
+ "ks=http://{{ .HTTPIP }}:{{ .HTTPPort }}/ks.cfg<enter><wait>"
+ ],
+ "accelerator": "kvm",
+ "boot_wait": "10s",
+ "disk_size": 2048,
+ "disk_interface": "virtio-scsi",
+ "http_directory": "http",
+ "iso_checksum": "101bc813d2af9ccf534d112cbe8670e6d900425b297d1a4d2529c5ad5f226372",
+ "iso_checksum_type": "sha256",
+ "iso_url": "http://centos.osuosl.org/7.8.2003/isos/x86_64/CentOS-7-x86_64-NetInstall-2003.iso",
+ "output_directory": "image",
+ "qemuargs": [ [ "-m", "1024M" ]],
+ "shutdown_command": "echo 'centos'|sudo -S /sbin/halt -h -p",
+ "ssh_password": "centos",
+ "ssh_port": 22,
+ "ssh_username": "centos",
+ "ssh_wait_timeout": "10000s",
+ "type": "qemu",
+ "vm_name": "packer-centos-7-x86_64-openstack",
+ "vnc_bind_address": "0.0.0.0",
+ "Headless": "true"
+ },
+ {
+ "name": "openstack",
+ "type": "openstack",
+ "image_name": "testvnf_image",
+ "identity_endpoint": "{{user `identiy_endpoint`}}",
+ "username": "{{user `username`}}",
+ "password": "{{user `password`}}",
+ "ssh_username": "centos",
+ "networks" : "{{user `networks`}}",
+ "source_image": "{{user `source_image`}}",
+ "flavor": "{{user `flavor`}}",
+ "domain_name": "{{user `domain_name`}}",
+ "ssh_timeout": "15m",
+ "use_floating_ip": "true",
+ "floating_ip_network" : "{{user `floating_ip_network`}}"
+ }
+],
+ "provisioners": [
+ {
+ "type": "shell",
+ "execute_command": "echo testvnf | {{.Vars}} sudo -S -E bash '{{.Path}}'",
+ "script": "scripts/ansible.sh"
+ },
+ {
+ "type": "file",
+ "source": "scripts/deploycentostools.sh",
+ "destination": "deploycentostools.sh"
+ },
+ {
+ "type": "ansible-local",
+ "playbook_file": "playbook.yml"
+ },
+ {
+ "type": "file",
+ "source": "{{user `ssh_path`}}",
+ "destination": "temp"
+ },
+ {
+ "type": "shell",
+ "execute_command": "echo testvnf | {{.Vars}} sudo -S -E bash '{{.Path}}'",
+ "script": "scripts/sshConfig.sh"
+ }
+ ]
+}
+
Code Review / vswitchperf.git / commitdiff