-# Copyright 2015 Intel Corporation.
+# Copyright 2015-2016 Intel Corporation.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# See the License for the specific language governing permissions and
# limitations under the License.
-"""Automation of QEMU hypervisor for launching vhost-cuse enabled guests.
+"""Automation of QEMU hypervisor for launching guests.
"""
import os
import locale
import re
import subprocess
+import time
+import pexpect
from conf import settings as S
from conf import get_test_param
Abstract class for controling an instance of QEMU
"""
_cmd = None
- _expect = S.getValue('GUEST_PROMPT_LOGIN')
+ _expect = None
_proc_name = 'qemu'
class GuestCommandFilter(logging.Filter):
Initialisation function.
"""
super(IVnfQemu, self).__init__()
+
+ self._expect = S.getValue('GUEST_PROMPT_LOGIN')[self._number]
self._logger = logging.getLogger(__name__)
self._logfile = os.path.join(
S.getValue('LOG_DIR'),
S.getValue('LOG_FILE_QEMU')) + str(self._number)
self._timeout = S.getValue('GUEST_TIMEOUT')[self._number]
self._monitor = '%s/vm%dmonitor' % ('/tmp', self._number)
- self._net1 = S.getValue('VANILLA_NIC1_NAME')[self._number]
- self._net2 = S.getValue('VANILLA_NIC2_NAME')[self._number]
+ # read GUEST NICs configuration and use only defined NR of NICS
+ nics_nr = S.getValue('GUEST_NICS_NR')[self._number]
+ # and inform user about missconfiguration
+ if nics_nr < 1:
+ raise RuntimeError('At least one VM NIC is mandotory, but {} '
+ 'NICs are configured'.format(nics_nr))
+ elif nics_nr > 1 and nics_nr % 2:
+ nics_nr = int(nics_nr / 2) * 2
+ self._logger.warning('Odd number of NICs is configured, only '
+ '%s NICs will be used', nics_nr)
+
+ self._nics = S.getValue('GUEST_NICS')[self._number][:nics_nr]
+
+ # set guest loopback application based on VNF configuration
+ # cli option take precedence to config file values
+ self._guest_loopback = S.getValue('GUEST_LOOPBACK')[self._number]
+
+ self._testpmd_fwd_mode = S.getValue('GUEST_TESTPMD_FWD_MODE')[self._number]
+ # in case of SRIOV we must ensure, that MAC addresses are not swapped
+ if S.getValue('SRIOV_ENABLED') and self._testpmd_fwd_mode.startswith('mac') and \
+ not S.getValue('VNF').endswith('PciPassthrough'):
+
+ self._logger.info("SRIOV detected, forwarding mode of testpmd was changed from '%s' to '%s'",
+ self._testpmd_fwd_mode, 'io')
+ self._testpmd_fwd_mode = 'io'
+
+ guest_smp = int(get_test_param('guest_smp', 0))
+ if guest_smp:
+ override_list = [guest_smp] * (self._number + 1)
+ S.setValue('GUEST_SMP', override_list)
name = 'Client%d' % self._number
vnc = ':%d' % self._number
- # don't use taskset to affinize main qemu process; It causes hangup
- # of 2nd VM in case of DPDK. It also slows down VM responsivnes.
- self._cmd = ['sudo', '-E', S.getValue('QEMU_BIN'),
+ # NOTE: affinization of main qemu process can cause hangup of 2nd VM
+ # in case of DPDK usage. It can also slow down VM response time.
+ cpumask = ",".join(S.getValue('GUEST_CORE_BINDING')[self._number])
+ self._cmd = ['sudo', '-E', 'taskset', '-c', cpumask,
+ S.getValue('TOOLS')['qemu-system'],
'-m', S.getValue('GUEST_MEMORY')[self._number],
'-smp', str(S.getValue('GUEST_SMP')[self._number]),
- '-cpu', 'host',
- '-drive', 'if=ide,file=' +
+ '-cpu', 'host,migratable=off',
+ '-drive', 'if={},file='.format(S.getValue(
+ 'GUEST_BOOT_DRIVE_TYPE')[self._number]) +
S.getValue('GUEST_IMAGE')[self._number],
'-boot', 'c', '--enable-kvm',
'-monitor', 'unix:%s,server,nowait' % self._monitor,
'-nographic', '-vnc', str(vnc), '-name', name,
'-snapshot', '-net none', '-no-reboot',
'-drive',
- 'if=ide,file=fat:rw:%s,snapshot=off' %
- S.getValue('GUEST_SHARE_DIR')[self._number],
+ 'if=%s,format=raw,file=fat:rw:%s,snapshot=off' %
+ (S.getValue('GUEST_SHARED_DRIVE_TYPE')[self._number],
+ S.getValue('GUEST_SHARE_DIR')[self._number]),
]
self._configure_logging()
if S.getValue('VNF_AFFINITIZATION_ON'):
self._affinitize()
+ if S.getValue('VSWITCH_VHOST_NET_AFFINITIZATION') and S.getValue(
+ 'VNF') == 'QemuVirtioNet':
+ self._affinitize_vhost_net()
+
if self._timeout:
self._config_guest_loopback()
+ def stop(self):
+ """
+ Stops VNF instance gracefully first.
+ """
+ try:
+ # exit testpmd if needed
+ if self._guest_loopback == 'testpmd':
+ self.execute_and_wait('stop', 120, "Done")
+ self.execute_and_wait('quit', 120, "[bB]ye")
+
+ # turn off VM
+ self.execute_and_wait('poweroff', 120, "Power down")
+
+ except pexpect.TIMEOUT:
+ self.kill()
+
+ # wait until qemu shutdowns
+ self._logger.debug('Wait for QEMU to terminate')
+ for dummy in range(30):
+ time.sleep(1)
+ if not self.is_running():
+ break
+
+ # just for case that graceful shutdown failed
+ super(IVnfQemu, self).stop()
+
# helper functions
def _login(self, timeout=120):
if not self._timeout:
self._expect_process(timeout=timeout)
- self._child.sendline(S.getValue('GUEST_USERNAME'))
- self._child.expect(S.getValue('GUEST_PROMPT_PASSWORD'), timeout=5)
- self._child.sendline(S.getValue('GUEST_PASSWORD'))
+ self._child.sendline(S.getValue('GUEST_USERNAME')[self._number])
+ self._child.expect(S.getValue('GUEST_PROMPT_PASSWORD')[self._number], timeout=5)
+ self._child.sendline(S.getValue('GUEST_PASSWORD')[self._number])
- self._expect_process(S.getValue('GUEST_PROMPT'), timeout=5)
+ self._expect_process(S.getValue('GUEST_PROMPT')[self._number], timeout=5)
def send_and_pass(self, cmd, timeout=30):
"""
:returns: None
"""
self.execute(cmd)
- self.wait(S.getValue('GUEST_PROMPT'), timeout=timeout)
+ self.wait(S.getValue('GUEST_PROMPT')[self._number], timeout=timeout)
self.execute('echo $?')
self._child.expect('^0$', timeout=1) # expect a 0
- self.wait(S.getValue('GUEST_PROMPT'), timeout=timeout)
+ self.wait(S.getValue('GUEST_PROMPT')[self._number], timeout=timeout)
def _affinitize(self):
"""
stdin=proc.stdout)
proc.wait()
+ guest_core_binding = int(get_test_param('guest_core_binding', 0))
for cpu in range(0, int(S.getValue('GUEST_SMP')[self._number])):
match = None
for line in output.decode(cur_locale).split('\n'):
match = re.search(thread_id % cpu, line)
if match:
- self._affinitize_pid(
- S.getValue('GUEST_CORE_BINDING')[self._number][cpu],
- match.group(1))
+ if guest_core_binding:
+ self._affinitize_pid(guest_core_binding, match.group(1))
+ else:
+ self._affinitize_pid(
+ S.getValue('GUEST_CORE_BINDING')[self._number][cpu],
+ match.group(1))
break
if not match:
self._logger.error('Failed to affinitize guest core #%d. Could'
' not parse tid.', cpu)
+ def _affinitize_vhost_net(self):
+ """
+ Affinitize the vhost net threads for Vanilla OVS and guest nic queues.
+
+ :return: None
+ """
+ self._logger.info('Affinitizing VHOST Net threads.')
+ args1 = ['pgrep', 'vhost-']
+ process1 = subprocess.Popen(args1, stdout=subprocess.PIPE,
+ shell=False)
+ out = process1.communicate()[0]
+ processes = out.decode(locale.getdefaultlocale()[1]).split('\n')
+ if processes[-1] == '':
+ processes.pop() # pgrep may return an extra line with no data
+ self._logger.info('Found %s vhost net threads...', len(processes))
+
+ cpumap = S.getValue('VSWITCH_VHOST_CPU_MAP')
+ mapcount = 0
+ for proc in processes:
+ self._affinitize_pid(cpumap[mapcount], proc)
+ mapcount += 1
+ if mapcount + 1 > len(cpumap):
+ # Not enough cpus were given in the mapping to cover all the
+ # threads on a 1 to 1 ratio with cpus so reset the list counter
+ # to 0.
+ mapcount = 0
+
def _config_guest_loopback(self):
"""
- Configure VM to run VNF (e.g. port forwarding application)
+ Configure VM to run VNF, e.g. port forwarding application based on the configuration
"""
- # set guest loopback application based on VNF configuration
- # cli option take precedence to config file values
- guest_loopback = get_test_param('guest_loopback', S.getValue('GUEST_LOOPBACK')[self._number])
- if guest_loopback == 'testpmd':
+ if self._guest_loopback == 'testpmd':
self._login()
self._configure_testpmd()
- elif guest_loopback == 'l2fwd':
+ elif self._guest_loopback == 'l2fwd':
self._login()
self._configure_l2fwd()
- elif guest_loopback == 'linux_bridge':
+ elif self._guest_loopback == 'linux_bridge':
self._login()
self._configure_linux_bridge()
- elif guest_loopback != 'buildin':
+ elif self._guest_loopback != 'buildin':
self._logger.error('Unsupported guest loopback method "%s" was specified. Option'
- ' "buildin" will be used as a fallback.', guest_loopback)
+ ' "buildin" will be used as a fallback.', self._guest_loopback)
- def wait(self, prompt=S.getValue('GUEST_PROMPT'), timeout=30):
+ def wait(self, prompt=None, timeout=30):
+ if prompt is None:
+ prompt = S.getValue('GUEST_PROMPT')[self._number]
super(IVnfQemu, self).wait(prompt=prompt, timeout=timeout)
- def execute_and_wait(self, cmd, timeout=30,
- prompt=S.getValue('GUEST_PROMPT')):
+ def execute_and_wait(self, cmd, timeout=30, prompt=None):
+ if prompt is None:
+ prompt = S.getValue('GUEST_PROMPT')[self._number]
super(IVnfQemu, self).execute_and_wait(cmd, timeout=timeout,
prompt=prompt)
def _modify_dpdk_makefile(self):
"""
- Modifies DPDK makefile in Guest before compilation
+ Modifies DPDK makefile in Guest before compilation if needed
"""
pass
Mount shared directory and copy DPDK and l2fwd sources
"""
# mount shared directory
- self.execute_and_wait('umount ' + S.getValue('OVS_DPDK_SHARE'))
- self.execute_and_wait('rm -rf ' + S.getValue('GUEST_OVS_DPDK_DIR'))
- self.execute_and_wait('mkdir -p ' + S.getValue('OVS_DPDK_SHARE'))
- self.execute_and_wait('mount -o iocharset=utf8 /dev/sdb1 ' +
- S.getValue('OVS_DPDK_SHARE'))
- self.execute_and_wait('mkdir -p ' + S.getValue('GUEST_OVS_DPDK_DIR'))
- self.execute_and_wait('cp -ra ' + os.path.join(S.getValue('OVS_DPDK_SHARE'), dirname) +
- ' ' + S.getValue('GUEST_OVS_DPDK_DIR'))
+ self.execute_and_wait('umount /dev/sdb1')
+ self.execute_and_wait('rm -rf ' + S.getValue('GUEST_OVS_DPDK_DIR')[self._number])
+ self.execute_and_wait('mkdir -p ' + S.getValue('GUEST_OVS_DPDK_SHARE')[self._number])
+ self.execute_and_wait('mount -o ro,iocharset=utf8 /dev/sdb1 ' +
+ S.getValue('GUEST_OVS_DPDK_SHARE')[self._number])
+ self.execute_and_wait('mkdir -p ' + S.getValue('GUEST_OVS_DPDK_DIR')[self._number])
+ self.execute_and_wait('cp -r ' + os.path.join(S.getValue('GUEST_OVS_DPDK_SHARE')[self._number], dirname) +
+ ' ' + S.getValue('GUEST_OVS_DPDK_DIR')[self._number])
+ self.execute_and_wait('umount /dev/sdb1')
def _configure_disable_firewall(self):
"""
Disable firewall in VM
"""
- # Disable services (F16)
- self.execute_and_wait('systemctl status iptables.service')
- self.execute_and_wait('systemctl stop iptables.service')
+ for iptables in ['iptables', 'ip6tables']:
+ # filter table
+ for chain in ['INPUT', 'FORWARD', 'OUTPUT']:
+ self.execute_and_wait("{} -t filter -P {} ACCEPT".format(iptables, chain))
+ # mangle table
+ for chain in ['PREROUTING', 'INPUT', 'FORWARD', 'OUTPUT', 'POSTROUTING']:
+ self.execute_and_wait("{} -t mangle -P {} ACCEPT".format(iptables, chain))
+ # nat table
+ for chain in ['PREROUTING', 'INPUT', 'OUTPUT', 'POSTROUTING']:
+ self.execute_and_wait("{} -t nat -P {} ACCEPT".format(iptables, chain))
+
+ # flush rules and delete chains created by user
+ for table in ['filter', 'mangle', 'nat']:
+ self.execute_and_wait("{} -t {} -F".format(iptables, table))
+ self.execute_and_wait("{} -t {} -X".format(iptables, table))
+
def _configure_testpmd(self):
"""
# Guest images _should_ have 1024 hugepages by default,
# but just in case:'''
- self.execute_and_wait('sysctl vm.nr_hugepages=1024')
+ self.execute_and_wait('sysctl vm.nr_hugepages={}'.format(S.getValue('GUEST_HUGEPAGES_NR')[self._number]))
# Mount hugepages
self.execute_and_wait('mkdir -p /dev/hugepages')
'mount -t hugetlbfs hugetlbfs /dev/hugepages')
# build and configure system for dpdk
- self.execute_and_wait('cd ' + S.getValue('GUEST_OVS_DPDK_DIR') +
+ self.execute_and_wait('cd ' + S.getValue('GUEST_OVS_DPDK_DIR')[self._number] +
'/DPDK')
self.execute_and_wait('export CC=gcc')
self.execute_and_wait('export RTE_SDK=' +
- S.getValue('GUEST_OVS_DPDK_DIR') + '/DPDK')
+ S.getValue('GUEST_OVS_DPDK_DIR')[self._number] + '/DPDK')
self.execute_and_wait('export RTE_TARGET=%s' % S.getValue('RTE_TARGET'))
# modify makefile if needed
self._modify_dpdk_makefile()
+ # disable network interfaces, so DPDK can take care of them
+ for nic in self._nics:
+ self.execute_and_wait('ifdown ' + nic['device'])
+
+ # build and insert igb_uio and rebind interfaces to it
self.execute_and_wait('make RTE_OUTPUT=$RTE_SDK/$RTE_TARGET -C '
'$RTE_SDK/lib/librte_eal/linuxapp/igb_uio')
self.execute_and_wait('modprobe uio')
self.execute_and_wait('insmod %s/kmod/igb_uio.ko' %
S.getValue('RTE_TARGET'))
- self.execute_and_wait('./tools/dpdk_nic_bind.py --status')
- self.execute_and_wait(
- './tools/dpdk_nic_bind.py -b igb_uio' ' ' +
- S.getValue('GUEST_NET1_PCI_ADDRESS')[self._number] + ' ' +
- S.getValue('GUEST_NET2_PCI_ADDRESS')[self._number])
+ self.execute_and_wait('./tools/dpdk*bind.py --status')
+ pci_list = ' '.join([nic['pci'] for nic in self._nics])
+ self.execute_and_wait('./tools/dpdk*bind.py -u ' + pci_list)
+ self.execute_and_wait('./tools/dpdk*bind.py -b igb_uio ' + pci_list)
+ self.execute_and_wait('./tools/dpdk*bind.py --status')
# build and run 'test-pmd'
- self.execute_and_wait('cd ' + S.getValue('GUEST_OVS_DPDK_DIR') +
+ self.execute_and_wait('cd ' + S.getValue('GUEST_OVS_DPDK_DIR')[self._number] +
'/DPDK/app/test-pmd')
self.execute_and_wait('make clean')
self.execute_and_wait('make')
- self.execute_and_wait('./testpmd -c 0x3 -n 4 --socket-mem 512 --'
- ' --burst=64 -i --txqflags=0xf00 ' +
- '--disable-hw-vlan', 60, "Done")
- self.execute('set fwd mac_retry', 1)
- self.execute_and_wait('start', 20,
- 'TX RS bit threshold=0 - TXQ flags=0xf00')
+
+ # get testpmd settings from CLI
+ testpmd_params = get_test_param('guest_testpmd_params',
+ S.getValue('GUEST_TESTPMD_PARAMS')[self._number])
+
+ self.execute_and_wait( './testpmd {}'.format(testpmd_params), 60, "Done")
+ self.execute('set fwd ' + self._testpmd_fwd_mode, 1)
+ self.execute_and_wait('start', 20, 'testpmd>')
def _configure_l2fwd(self):
"""
Configure VM to perform L2 forwarding between NICs by l2fwd module
"""
+ if int(S.getValue('GUEST_NIC_QUEUES')[self._number]):
+ self._set_multi_queue_nic()
self._configure_copy_sources('l2fwd')
self._configure_disable_firewall()
+ # configure all interfaces
+ for nic in self._nics:
+ self.execute('ip addr add ' +
+ nic['ip'] + ' dev ' + nic['device'])
+ self.execute('ip link set dev ' + nic['device'] + ' up')
+
# build and configure system for l2fwd
- self.execute_and_wait('cd ' + S.getValue('GUEST_OVS_DPDK_DIR') +
+ self.execute_and_wait('cd ' + S.getValue('GUEST_OVS_DPDK_DIR')[self._number] +
'/l2fwd')
self.execute_and_wait('export CC=gcc')
self.execute_and_wait('make')
- self.execute_and_wait('insmod ' + S.getValue('GUEST_OVS_DPDK_DIR') +
- '/l2fwd' + '/l2fwd.ko net1=' + self._net1 +
- ' net2=' + self._net2)
+ if len(self._nics) == 2:
+ self.execute_and_wait('insmod ' + S.getValue('GUEST_OVS_DPDK_DIR')[self._number] +
+ '/l2fwd' + '/l2fwd.ko net1=' + self._nics[0]['device'] +
+ ' net2=' + self._nics[1]['device'])
+ else:
+ raise RuntimeError('l2fwd can forward only between 2 NICs, but {} NICs are '
+ 'configured inside GUEST'.format(len(self._nics)))
def _configure_linux_bridge(self):
"""
Configure VM to perform L2 forwarding between NICs by linux bridge
"""
+ if int(S.getValue('GUEST_NIC_QUEUES')[self._number]):
+ self._set_multi_queue_nic()
self._configure_disable_firewall()
- nic1_name = get_test_param('vanilla_nic1_name', self._net1)
- self.execute('ifconfig ' + nic1_name + ' ' +
- S.getValue('VANILLA_NIC1_IP_CIDR')[self._number])
-
- nic2_name = get_test_param('vanilla_nic2_name', self._net2)
- self.execute('ifconfig ' + nic2_name + ' ' +
- S.getValue('VANILLA_NIC2_IP_CIDR')[self._number])
# configure linux bridge
self.execute('brctl addbr br0')
- self.execute('brctl addif br0 ' + self._net1 + ' ' + self._net2)
- self.execute('ifconfig br0 ' +
- S.getValue('VANILLA_BRIDGE_IP')[self._number])
+
+ # add all NICs into the bridge
+ for nic in self._nics:
+ self.execute('ip addr add ' +
+ nic['ip'] + ' dev ' + nic['device'])
+ self.execute('ip link set dev ' + nic['device'] + ' up')
+ self.execute('brctl addif br0 ' + nic['device'])
+
+ self.execute('ip addr add ' +
+ S.getValue('GUEST_BRIDGE_IP')[self._number] +
+ ' dev br0')
+ self.execute('ip link set dev br0 up')
# Add the arp entries for the IXIA ports and the bridge you are using.
# Use command line values if provided.
# Controls source route verification
# 0 means no source validation
self.execute('sysctl -w net.ipv4.conf.all.rp_filter=0')
- self.execute('sysctl -w net.ipv4.conf.' + self._net1 + '.rp_filter=0')
- self.execute('sysctl -w net.ipv4.conf.' + self._net2 + '.rp_filter=0')
+ for nic in self._nics:
+ self.execute('sysctl -w net.ipv4.conf.' + nic['device'] + '.rp_filter=0')
+
+ def _set_multi_queue_nic(self):
+ """
+ Enable multi-queue in guest kernel with ethool.
+ :return: None
+ """
+ for nic in self._nics:
+ self.execute_and_wait('ethtool -L {} combined {}'.format(
+ nic['device'], S.getValue('GUEST_NIC_QUEUES')[self._number]))
+ self.execute_and_wait('ethtool -l {}'.format(nic['device']))
Code Review / vswitchperf.git / blobdiff