From: Yichen Wang Date: Thu, 23 Nov 2017 00:26:29 +0000 (-0800) Subject: Add support to use vswitch to handle V2V in PVVP SRIOV scenario X-Git-Tag: 1.2.0~1 X-Git-Url: https://gerrit.opnfv.org/gerrit/gitweb?a=commitdiff_plain;h=1ce98c7510f810a5ee790523a0dcbf429961adda;p=nfvbench.git Add support to use vswitch to handle V2V in PVVP SRIOV scenario 1. Add support to use vswitch to handle V2V in PVVP SRIOV scenario 2. Update nfvbenchvm to 0.5: (1) Update VPP to 17.10; (2) Update DPDK testpmd to 17.08; (3) Change kernel to based on longterm lineup; Change-Id: I944489579a4cd92d17075e80870bbdb32512a150 Signed-off-by: Yichen Wang --- diff --git a/nfvbench/cfg.default.yaml b/nfvbench/cfg.default.yaml index 83dd5ac..a8bdc2b 100644 --- a/nfvbench/cfg.default.yaml +++ b/nfvbench/cfg.default.yaml @@ -253,6 +253,11 @@ internal_networks: segmentation_id: physical_network: +# In the scenario of PVVP + SRIOV, there is choice of how the traffic will be +# handled in the middle network. The default (false) will use vswitch, while +# SRIOV can be used by toggling below setting. +use_sriov_middle_net: false + # EXT chain only. Names of edge networks which will be used to send traffic via traffic generator. external_networks: left: 'nfvbench-net0' @@ -400,4 +405,4 @@ factory_class: 'BasicFactory' # Custom label added for every perf record generated during this run. # Can be overriden by --user-label -user_label: \ No newline at end of file +user_label: diff --git a/nfvbench/chain_clients.py b/nfvbench/chain_clients.py index d9a39af..57b15ee 100644 --- a/nfvbench/chain_clients.py +++ b/nfvbench/chain_clients.py @@ -145,11 +145,11 @@ class BasicStageClient(object): LOG.info('Created network: %s.', name) return network - def _create_port(self, net): + def _create_port(self, net, vnic_type='normal'): body = { "port": { 'network_id': net['id'], - 'binding:vnic_type': 'direct' if self.config.sriov else 'normal' + 'binding:vnic_type': vnic_type } } port = self.neutron.create_port(body) @@ -305,7 +305,7 @@ class BasicStageClient(object): else: LOG.error('Unable to delete flavor: %s', self.config.flavor_type) - def get_config_file(self, chain_index, src_mac, dst_mac): + def get_config_file(self, chain_index, src_mac, dst_mac, intf_mac1, intf_mac2): boot_script_file = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'nfvbenchvm/', self.nfvbenchvm_config_name) @@ -317,6 +317,8 @@ class BasicStageClient(object): vm_config = { 'forwarder': self.config.vm_forwarder, + 'intf_mac1': intf_mac1, + 'intf_mac2': intf_mac2, 'tg_gateway1_ip': self.config.traffic_generator.tg_gateway_ip_addrs[0], 'tg_gateway2_ip': self.config.traffic_generator.tg_gateway_ip_addrs[1], 'tg_net1': self.config.traffic_generator.ip_addrs[0], @@ -479,11 +481,13 @@ class PVPStageClient(BasicStageClient): if reusable_vm: self.vms.append(reusable_vm) else: + vnic_type = 'direct' if self.config.sriov else 'normal' + ports = [self._create_port(net, vnic_type) for net in self.nets] config_file = self.get_config_file(chain_index, self.config.generator_config.src_device.mac, - self.config.generator_config.dst_device.mac) - - ports = [self._create_port(net) for net in self.nets] + self.config.generator_config.dst_device.mac, + ports[0]['mac_address'], + ports[1]['mac_address']) self.created_ports.extend(ports) self.vms.append(self._create_server(name, ports, az, config_file)) self._ensure_vms_active() @@ -542,11 +546,15 @@ class PVVPStageClient(BasicStageClient): if reusable_vm0 and reusable_vm1: self.vms.extend([reusable_vm0, reusable_vm1]) else: - vm0_port_net0 = self._create_port(vm0_nets[0]) - vm0_port_net2 = self._create_port(vm0_nets[1]) + edge_vnic_type = 'direct' if self.config.sriov else 'normal' + middle_vnic_type = 'direct' \ + if self.config.sriov and self.config.use_sriov_middle_net \ + else 'normal' + vm0_port_net0 = self._create_port(vm0_nets[0], edge_vnic_type) + vm0_port_net2 = self._create_port(vm0_nets[1], middle_vnic_type) - vm1_port_net2 = self._create_port(vm1_nets[1]) - vm1_port_net1 = self._create_port(vm1_nets[0]) + vm1_port_net2 = self._create_port(vm1_nets[1], middle_vnic_type) + vm1_port_net1 = self._create_port(vm1_nets[0], edge_vnic_type) self.created_ports.extend([vm0_port_net0, vm0_port_net2, @@ -558,10 +566,14 @@ class PVVPStageClient(BasicStageClient): # TG0 (net0) -> VM0 (net2) -> VM1 (net2) -> TG1 (net1) config_file0 = self.get_config_file(chain_index, self.config.generator_config.src_device.mac, - vm1_port_net2['mac_address']) + vm1_port_net2['mac_address'], + vm0_port_net0['mac_address'], + vm0_port_net2['mac_address']) config_file1 = self.get_config_file(chain_index, vm0_port_net2['mac_address'], - self.config.generator_config.dst_device.mac) + self.config.generator_config.dst_device.mac, + vm1_port_net2['mac_address'], + vm1_port_net1['mac_address']) self.vms.append(self._create_server(name0, [vm0_port_net0, vm0_port_net2], diff --git a/nfvbench/nfvbench.py b/nfvbench/nfvbench.py index d1bd0d9..4c9f56c 100644 --- a/nfvbench/nfvbench.py +++ b/nfvbench/nfvbench.py @@ -284,6 +284,12 @@ def parse_opts_from_cli(): action='store_true', help='Use SRIOV (no vswitch - requires SRIOV support in compute nodes)') + parser.add_argument('--use-sriov-middle-net', dest='use_sriov_middle_net', + default=None, + action='store_true', + help='Use SRIOV to handle the middle network traffic ' + '(PVVP with SRIOV only)') + parser.add_argument('-d', '--debug', dest='debug', action='store_true', default=None, @@ -491,20 +497,29 @@ def main(): config.sriov = True if opts.log_file: config.log_file = opts.log_file + if opts.service_chain: + config.service_chain = opts.service_chain + if opts.service_chain_count: + config.service_chain_count = opts.service_chain_count - # show running config in json format - if opts.show_config: - print json.dumps(config, sort_keys=True, indent=4) - sys.exit(0) + if opts.use_sriov_middle_net: + if (not config.sriov) or (not config.service_chain == ChainType.PVVP): + raise Exception("--use-sriov-middle-net is only valid for PVVP with SRIOV") + config.use_sriov_middle_net = True if config.sriov and config.service_chain != ChainType.EXT: # if sriov is requested (does not apply to ext chains) # make sure the physnet names are specified check_physnet("left", config.internal_networks.left) check_physnet("right", config.internal_networks.right) - if config.service_chain == ChainType.PVVP: + if config.service_chain == ChainType.PVVP and config.use_sriov_middle_net: check_physnet("middle", config.internal_networks.middle) + # show running config in json format + if opts.show_config: + print json.dumps(config, sort_keys=True, indent=4) + sys.exit(0) + # update the config in the config plugin as it might have changed # in a copy of the dict (config plugin still holds the original dict) config_plugin.set_config(config) diff --git a/nfvbench/nfvbenchvm/nfvbenchvm.conf b/nfvbench/nfvbenchvm/nfvbenchvm.conf index 0b76244..3bc6ace 100644 --- a/nfvbench/nfvbenchvm/nfvbenchvm.conf +++ b/nfvbench/nfvbenchvm/nfvbenchvm.conf @@ -1,4 +1,6 @@ FORWARDER={forwarder} +INTF_MAC1={intf_mac1} +INTF_MAC2={intf_mac2} TG_MAC1={tg_mac1} TG_MAC2={tg_mac2} VNF_GATEWAY1_CIDR={vnf_gateway1_cidr} diff --git a/nfvbenchvm/README.rst b/nfvbenchvm/README.rst index baa6996..1bf0bbf 100644 --- a/nfvbenchvm/README.rst +++ b/nfvbenchvm/README.rst @@ -1,4 +1,4 @@ -NFVBENCH VM IMAGE FOR OPENSTACK +NFVBENCH VM IMAGE FOR OPENSTACK +++++++++++++++++++++++++++++++ This repo will build a centos 7 image with testpmd and VPP installed. @@ -18,7 +18,7 @@ Pre-requisites Build the image --------------- - cd dib -- update the version number for the image (if needed) by modifying __version__ in build-image.sh +- update the version number for the image (if needed) by modifying __version__ in build-image.sh - setup your http_proxy if needed - bash build-image.sh @@ -48,15 +48,17 @@ nfvbenchvm config file is located at ``/etc/nfvbenchvm.conf``. .. code-block:: bash - FORWARDER=VPP - TG_MAC0=00:10:94:00:0A:00 - TG_MAC1=00:11:94:00:0A:00 - VNF1_GATEWAY_CIDR=1.1.0.2/8 - VNF2_GATEWAY_CIDR=2.2.0.2/8 - TG1_NET=10.0.0.0/8 - TG2_NET=20.0.0.0/8 - TG1_GATEWAY_IP=1.1.0.100 - TG1_GATEWAY_IP=2.2.0.100 + FORWARDER=testpmd + INTF_MAC1=FA:16:3E:A2:30:41 + INTF_MAC2=FA:16:3E:10:DA:10 + TG_MAC1=00:10:94:00:0A:00 + TG_MAC2=00:11:94:00:0A:00 + VNF_GATEWAY1_CIDR=1.1.0.2/8 + VNF_GATEWAY2_CIDR=2.2.0.2/8 + TG_NET1=10.0.0.0/8 + TG_NET2=20.0.0.0/8 + TG_GATEWAY1_IP=1.1.0.100 + TG_GATEWAY2_IP=2.2.0.100 Launching nfvbenchvm VM @@ -79,6 +81,6 @@ To check if VPP is running, you can run this command in VNC console: Hardcoded Username and Password -------------------------------- -- Username: nfvbench -- Password: nfvbench +- Username: nfvbench +- Password: nfvbench diff --git a/nfvbenchvm/dib/build-image.sh b/nfvbenchvm/dib/build-image.sh index 9326762..605db54 100755 --- a/nfvbenchvm/dib/build-image.sh +++ b/nfvbenchvm/dib/build-image.sh @@ -11,7 +11,7 @@ set -e gs_url=artifacts.opnfv.org/nfvbench/images # image version number -__version__=0.4 +__version__=0.5 image_name=nfvbenchvm_centos-$__version__ # if image exists skip building diff --git a/nfvbenchvm/dib/elements/nfvbenchvm/fdio-release.repo b/nfvbenchvm/dib/elements/nfvbenchvm/fdio-release.repo index a26aa42..3ad12fb 100644 --- a/nfvbenchvm/dib/elements/nfvbenchvm/fdio-release.repo +++ b/nfvbenchvm/dib/elements/nfvbenchvm/fdio-release.repo @@ -1,5 +1,5 @@ [fdio-release] name=fd.io release branch latest merge -baseurl=https://nexus.fd.io/content/repositories/fd.io.stable.1707.centos7/ +baseurl=https://nexus.fd.io/content/repositories/fd.io.stable.1710.centos7/ enabled=1 gpgcheck=0 diff --git a/nfvbenchvm/dib/elements/nfvbenchvm/package-installs.yaml b/nfvbenchvm/dib/elements/nfvbenchvm/package-installs.yaml index 36e0196..e3184c7 100644 --- a/nfvbenchvm/dib/elements/nfvbenchvm/package-installs.yaml +++ b/nfvbenchvm/dib/elements/nfvbenchvm/package-installs.yaml @@ -6,6 +6,8 @@ screen: telnet: python-devel: libyaml-devel: +numactl-libs: +numactl-devel: vpp: vpp-plugins: kernel-firmware: diff --git a/nfvbenchvm/dib/elements/nfvbenchvm/post-install.d/01-update-kernel b/nfvbenchvm/dib/elements/nfvbenchvm/post-install.d/01-update-kernel index d884f79..8094006 100755 --- a/nfvbenchvm/dib/elements/nfvbenchvm/post-install.d/01-update-kernel +++ b/nfvbenchvm/dib/elements/nfvbenchvm/post-install.d/01-update-kernel @@ -8,7 +8,7 @@ fi rpm --import https://www.elrepo.org/RPM-GPG-KEY-elrepo.org rpm -Uvh http://www.elrepo.org/elrepo-release-7.0-2.el7.elrepo.noarch.rpm yum remove -y kernel-firmware kernel-headers kernel-devel -yum install -y --enablerepo=elrepo-kernel kernel-ml kernel-ml-headers kernel-ml-devel +yum install -y --enablerepo=elrepo-kernel kernel-lt kernel-lt-headers kernel-lt-devel # gcc will be removed with old kernel as dependency, so reinstalling it back yum install -y gcc diff --git a/nfvbenchvm/dib/elements/nfvbenchvm/post-install.d/02-testpmd-script b/nfvbenchvm/dib/elements/nfvbenchvm/post-install.d/02-testpmd-script index acdc6a3..2136c3a 100755 --- a/nfvbenchvm/dib/elements/nfvbenchvm/post-install.d/02-testpmd-script +++ b/nfvbenchvm/dib/elements/nfvbenchvm/post-install.d/02-testpmd-script @@ -1,7 +1,7 @@ #!/bin/bash -DPDK=dpdk-17.05.2 -DPDK_UNTAR=dpdk-stable-17.05.2 +DPDK=dpdk-17.08 +DPDK_UNTAR=dpdk-17.08 # pick up the kernel version for the target image kernel_version=`ls -t /lib/modules | awk 'NR==1 {print}'` @@ -19,6 +19,7 @@ cp usertools/dpdk-devbind.py ../dpdk # cp tools/dpdk_nic_bind.py ../dpdk/dpdk-devbind.py cp x86_64-native-linuxapp-gcc/app/testpmd ../dpdk cp x86_64-native-linuxapp-gcc/kmod/igb_uio.ko ../dpdk +echo "set promisc all off" > /dpdk/testpmd_cmd.txt cd .. rm -f $DPDK.tar.xz diff --git a/nfvbenchvm/dib/elements/nfvbenchvm/post-install.d/99-cleanup b/nfvbenchvm/dib/elements/nfvbenchvm/post-install.d/99-cleanup index 83d4fc5..14e9f27 100755 --- a/nfvbenchvm/dib/elements/nfvbenchvm/post-install.d/99-cleanup +++ b/nfvbenchvm/dib/elements/nfvbenchvm/post-install.d/99-cleanup @@ -1,3 +1,3 @@ #!/bin/bash -yum erase -y python-devel libyaml-devel kernel-devel kernel-headers kernel-ml-headers kernel-ml-devel gcc +yum erase -y python-devel libyaml-devel numactl-devel kernel-devel kernel-headers kernel-lt-headers kernel-lt-devel gcc diff --git a/nfvbenchvm/dib/elements/nfvbenchvm/static/etc/rc.d/rc.local b/nfvbenchvm/dib/elements/nfvbenchvm/static/etc/rc.d/rc.local index 596cfdb..8aab1ae 100644 --- a/nfvbenchvm/dib/elements/nfvbenchvm/static/etc/rc.d/rc.local +++ b/nfvbenchvm/dib/elements/nfvbenchvm/static/etc/rc.d/rc.local @@ -34,6 +34,25 @@ for irq in `ls /proc/irq/`; do done tuna -c $(seq -s, 1 1 $WORKER_CORES) --isolate +# Sometimes the interfaces on the loopback VM will use different drivers, e.g. +# one from vswitch which is virtio based, one is from SRIOV VF. In this case, +# we have to make sure the forwarder uses them in the right order, which is +# especially important if the VM is in a PVVP chain. +if [ -z $INTF_MAC1 ] && [ -z $INTF_MAC2 ]; then + NET_PATH=/sys/class/net + EXP_INTF_1=$(for f in $(ls $NET_PATH/); do if [ ! $(grep -o "$INTF_MAC1" $NET_PATH/$f/address) ]; then break; fi; done) + EXP_PCI_ADDRESS_1=$(basename $(readlink $NET_PATH/$EXP_INTF_1/device)) + EXP_INTF_2=$(for f in $(ls $NET_PATH/); do if [ ! $(grep -o "$INTF_MAC2" $NET_PATH/$f/address) ]; then break; fi; done) + EXP_PCI_ADDRESS_2=$(basename $(readlink $NET_PATH/$EXP_INTF_2/device)) + if [ "$PCI_ADDRESS_1" == "$EXP_PCI_ADDRESS_2" ] && [ "$PCI_ADDRESS_2" == "$EXP_PCI_ADDRESS_1" ]; then + # Interfaces are not coming in the expected order: + # (1) Swap the MAC in the case of testpmd; + # (2) Swap the interface order in the case of VPP; + TEMP=$PCI_ADDRESS_1; PCI_ADDRESS_1=$PCI_ADDRESS_2; PCI_ADDRESS_2=$TEMP + TEMP=$TG_MAC1; TG_MAC1=$TG_MAC2; TG_MAC2=$TEMP + fi +fi + # Configure the forwarder if [ "$FORWARDER" == "testpmd" ]; then echo "Configuring testpmd..." @@ -56,8 +75,9 @@ if [ "$FORWARDER" == "testpmd" ]; then --eth-peer=0,$TG_MAC1 \ --eth-peer=1,$TG_MAC2 \ --forward-mode=mac \ - --nb-cores=$WORKER_CORES\ - --max-pkt-len 9000 + --nb-cores=$WORKER_CORES \ + --max-pkt-len=9000 \ + --cmdline-file=/dpdk/testpmd_cmd.txt else echo "Configuring vpp..." cp /vpp/startup.conf /etc/vpp/startup.conf @@ -81,6 +101,5 @@ else sed -i "s/{{TG_NET2}}/${TG_NET2//\//\/}/g" /etc/vpp/vm.conf sed -i "s/{{TG_GATEWAY1_IP}}/${TG_GATEWAY1_IP}/g" /etc/vpp/vm.conf sed -i "s/{{TG_GATEWAY2_IP}}/${TG_GATEWAY2_IP}/g" /etc/vpp/vm.conf - service vpp restart fi diff --git a/nfvbenchvm/dib/elements/nfvbenchvm/static/vpp/startup.conf b/nfvbenchvm/dib/elements/nfvbenchvm/static/vpp/startup.conf index 811eee1..4306fe9 100644 --- a/nfvbenchvm/dib/elements/nfvbenchvm/static/vpp/startup.conf +++ b/nfvbenchvm/dib/elements/nfvbenchvm/static/vpp/startup.conf @@ -3,7 +3,7 @@ unix { log /tmp/vpp.log full-coredump startup-config /etc/vpp/vm.conf - cli-listen localhost:5002 + cli-listen /run/vpp/cli.sock } api-trace {