X-Git-Url: https://gerrit.opnfv.org/gerrit/gitweb?a=blobdiff_plain;f=vnfs%2Fqemu%2Fqemu.py;h=8e3d44de5658ae406b9448710a09b2527066acc4;hb=b1534957e463b5e34957a8d48ce5c6b0552ffbb4;hp=01c16a0f9be21018ae0a679ad96a2218de2aaf56;hpb=6dfabefd6cac3cd4b8fd29abf5a5b6ab9e6ae956;p=vswitchperf.git diff --git a/vnfs/qemu/qemu.py b/vnfs/qemu/qemu.py index 01c16a0f..8e3d44de 100644 --- a/vnfs/qemu/qemu.py +++ b/vnfs/qemu/qemu.py @@ -1,4 +1,4 @@ -# Copyright 2015-2016 Intel Corporation. +# Copyright 2015-2017 Intel Corporation. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -24,7 +24,6 @@ import time import pexpect from conf import settings as S -from conf import get_test_param from vnfs.vnf.vnf import IVnf class IVnfQemu(IVnf): @@ -69,13 +68,12 @@ class IVnfQemu(IVnf): self._nics = S.getValue('GUEST_NICS')[self._number][:nics_nr] # set guest loopback application based on VNF configuration - # cli option take precedence to config file values self._guest_loopback = S.getValue('GUEST_LOOPBACK')[self._number] self._testpmd_fwd_mode = S.getValue('GUEST_TESTPMD_FWD_MODE')[self._number] # in case of SRIOV we must ensure, that MAC addresses are not swapped if S.getValue('SRIOV_ENABLED') and self._testpmd_fwd_mode.startswith('mac') and \ - not S.getValue('VNF').endswith('PciPassthrough'): + not str(S.getValue('VNF')).endswith('PciPassthrough'): self._logger.info("SRIOV detected, forwarding mode of testpmd was changed from '%s' to '%s'", self._testpmd_fwd_mode, 'io') @@ -83,12 +81,14 @@ class IVnfQemu(IVnf): name = 'Client%d' % self._number vnc = ':%d' % self._number - # don't use taskset to affinize main qemu process; It causes hangup - # of 2nd VM in case of DPDK. It also slows down VM responsivnes. - self._cmd = ['sudo', '-E', S.getValue('TOOLS')['qemu-system'], + # NOTE: affinization of main qemu process can cause hangup of 2nd VM + # in case of DPDK usage. It can also slow down VM response time. + cpumask = ",".join(S.getValue('GUEST_CORE_BINDING')[self._number]) + self._cmd = ['sudo', '-E', 'taskset', '-c', cpumask, + S.getValue('TOOLS')['qemu-system'], '-m', S.getValue('GUEST_MEMORY')[self._number], '-smp', str(S.getValue('GUEST_SMP')[self._number]), - '-cpu', 'host,migratable=off', + '-cpu', str(S.getValue('GUEST_CPU_OPTIONS')[self._number]), '-drive', 'if={},file='.format(S.getValue( 'GUEST_BOOT_DRIVE_TYPE')[self._number]) + S.getValue('GUEST_IMAGE')[self._number], @@ -144,31 +144,33 @@ class IVnfQemu(IVnf): """ Stops VNF instance gracefully first. """ - try: - # exit testpmd if needed - if self._guest_loopback == 'testpmd': - self.execute_and_wait('stop', 120, "Done") - self.execute_and_wait('quit', 120, "[bB]ye") - - # turn off VM - self.execute_and_wait('poweroff', 120, "Power down") - - except pexpect.TIMEOUT: - self.kill() - - # wait until qemu shutdowns - self._logger.debug('Wait for QEMU to terminate') - for dummy in range(30): - time.sleep(1) - if not self.is_running(): - break + if self.is_running(): + try: + if self._login_active: + # exit testpmd if needed + if self._guest_loopback == 'testpmd': + self.execute_and_wait('stop', 120, "Done") + self.execute_and_wait('quit', 120, "[bB]ye") + + # turn off VM + self.execute_and_wait('poweroff', 120, "Power down") + + except pexpect.TIMEOUT: + self.kill() + + # wait until qemu shutdowns + self._logger.debug('Wait for QEMU to terminate') + for dummy in range(30): + time.sleep(1) + if not self.is_running(): + break - # just for case that graceful shutdown failed - super(IVnfQemu, self).stop() + # just for case that graceful shutdown failed + super(IVnfQemu, self).stop() # helper functions - def _login(self, timeout=120): + def login(self, timeout=120): """ Login to QEMU instance. @@ -177,8 +179,11 @@ class IVnfQemu(IVnf): :param timeout: Timeout to wait for login to complete. - :returns: None + :returns: True if login is active """ + if self._login_active: + return self._login_active + # if no timeout was set, we likely started QEMU without waiting for it # to boot. This being the case, we best check that it has finished # first. @@ -190,21 +195,8 @@ class IVnfQemu(IVnf): self._child.sendline(S.getValue('GUEST_PASSWORD')[self._number]) self._expect_process(S.getValue('GUEST_PROMPT')[self._number], timeout=5) - - def send_and_pass(self, cmd, timeout=30): - """ - Send ``cmd`` and wait ``timeout`` seconds for it to pass. - - :param cmd: Command to send to guest. - :param timeout: Time to wait for prompt before checking return code. - - :returns: None - """ - self.execute(cmd) - self.wait(S.getValue('GUEST_PROMPT')[self._number], timeout=timeout) - self.execute('echo $?') - self._child.expect('^0$', timeout=1) # expect a 0 - self.wait(S.getValue('GUEST_PROMPT')[self._number], timeout=timeout) + self._login_active = True + return self._login_active def _affinitize(self): """ @@ -232,12 +224,13 @@ class IVnfQemu(IVnf): for cpu in range(0, int(S.getValue('GUEST_SMP')[self._number])): match = None + guest_thread_binding = S.getValue('GUEST_THREAD_BINDING')[self._number] + if guest_thread_binding is None: + guest_thread_binding = S.getValue('GUEST_CORE_BINDING')[self._number] for line in output.decode(cur_locale).split('\n'): match = re.search(thread_id % cpu, line) if match: - self._affinitize_pid( - S.getValue('GUEST_CORE_BINDING')[self._number][cpu], - match.group(1)) + self._affinitize_pid(guest_thread_binding[cpu], match.group(1)) break if not match: @@ -275,29 +268,31 @@ class IVnfQemu(IVnf): """ Configure VM to run VNF, e.g. port forwarding application based on the configuration """ + if self._guest_loopback == 'buildin': + return + + self.login() + if self._guest_loopback == 'testpmd': - self._login() self._configure_testpmd() elif self._guest_loopback == 'l2fwd': - self._login() self._configure_l2fwd() elif self._guest_loopback == 'linux_bridge': - self._login() self._configure_linux_bridge() - elif self._guest_loopback != 'buildin': - self._logger.error('Unsupported guest loopback method "%s" was specified. Option' - ' "buildin" will be used as a fallback.', self._guest_loopback) + elif self._guest_loopback != 'clean': + raise RuntimeError('Unsupported guest loopback method "%s" was specified.', + self._guest_loopback) def wait(self, prompt=None, timeout=30): if prompt is None: prompt = S.getValue('GUEST_PROMPT')[self._number] - super(IVnfQemu, self).wait(prompt=prompt, timeout=timeout) + return super(IVnfQemu, self).wait(prompt=prompt, timeout=timeout) def execute_and_wait(self, cmd, timeout=30, prompt=None): if prompt is None: prompt = S.getValue('GUEST_PROMPT')[self._number] - super(IVnfQemu, self).execute_and_wait(cmd, timeout=timeout, - prompt=prompt) + return super(IVnfQemu, self).execute_and_wait(cmd, timeout=timeout, + prompt=prompt) def _modify_dpdk_makefile(self): """ @@ -340,7 +335,6 @@ class IVnfQemu(IVnf): self.execute_and_wait("{} -t {} -F".format(iptables, table)) self.execute_and_wait("{} -t {} -X".format(iptables, table)) - def _configure_testpmd(self): """ Configure VM to perform L2 forwarding between NICs by DPDK's testpmd @@ -372,42 +366,28 @@ class IVnfQemu(IVnf): for nic in self._nics: self.execute_and_wait('ifdown ' + nic['device']) - # build and insert igb_uio and rebind interfaces to it - self.execute_and_wait('make RTE_OUTPUT=$RTE_SDK/$RTE_TARGET -C ' - '$RTE_SDK/lib/librte_eal/linuxapp/igb_uio') - self.execute_and_wait('modprobe uio') - self.execute_and_wait('insmod %s/kmod/igb_uio.ko' % - S.getValue('RTE_TARGET')) - self.execute_and_wait('./tools/dpdk*bind.py --status') + self.execute_and_wait('./*tools/dpdk*bind.py --status') pci_list = ' '.join([nic['pci'] for nic in self._nics]) - self.execute_and_wait('./tools/dpdk*bind.py -u ' + pci_list) - self.execute_and_wait('./tools/dpdk*bind.py -b igb_uio ' + pci_list) - self.execute_and_wait('./tools/dpdk*bind.py --status') + self.execute_and_wait('./*tools/dpdk*bind.py -u ' + pci_list) + self._bind_dpdk_driver(S.getValue( + 'GUEST_DPDK_BIND_DRIVER')[self._number], pci_list) + self.execute_and_wait('./*tools/dpdk*bind.py --status') # build and run 'test-pmd' self.execute_and_wait('cd ' + S.getValue('GUEST_OVS_DPDK_DIR')[self._number] + '/DPDK/app/test-pmd') self.execute_and_wait('make clean') self.execute_and_wait('make') - if int(S.getValue('GUEST_NIC_QUEUES')[self._number]): - self.execute_and_wait( - './testpmd {} -n4 --socket-mem 512 --'.format( - S.getValue('GUEST_TESTPMD_CPU_MASK')[self._number]) + - ' --burst=64 -i --txqflags=0xf00 ' + - '--nb-cores={} --rxq={} --txq={} '.format( - S.getValue('GUEST_TESTPMD_NB_CORES')[self._number], - S.getValue('GUEST_TESTPMD_TXQ')[self._number], - S.getValue('GUEST_TESTPMD_RXQ')[self._number]) + - '--disable-hw-vlan', 60, "Done") - else: - self.execute_and_wait( - './testpmd {} -n 4 --socket-mem 512 --'.format( - S.getValue('GUEST_TESTPMD_CPU_MASK')[self._number]) + - ' --burst=64 -i --txqflags=0xf00 ' + - '--disable-hw-vlan', 60, "Done") - self.execute('set fwd ' + self._testpmd_fwd_mode, 1) - self.execute_and_wait('start', 20, - 'TX RS bit threshold=.+ - TXQ flags=0xf00') + + # get testpmd settings from CLI + testpmd_params = S.getValue('GUEST_TESTPMD_PARAMS')[self._number] + if S.getValue('VSWITCH_JUMBO_FRAMES_ENABLED'): + testpmd_params += ' --max-pkt-len={}'.format(S.getValue( + 'VSWITCH_JUMBO_FRAMES_SIZE')) + + self.execute_and_wait('./testpmd {}'.format(testpmd_params), 60, "Done") + self.execute_and_wait('set fwd ' + self._testpmd_fwd_mode, 20, 'testpmd>') + self.execute_and_wait('start', 20, 'testpmd>') def _configure_l2fwd(self): """ @@ -420,9 +400,12 @@ class IVnfQemu(IVnf): # configure all interfaces for nic in self._nics: - self.execute('ip addr add ' + - nic['ip'] + ' dev ' + nic['device']) - self.execute('ip link set dev ' + nic['device'] + ' up') + self.execute_and_wait('ip addr add ' + + nic['ip'] + ' dev ' + nic['device']) + if S.getValue('VSWITCH_JUMBO_FRAMES_ENABLED'): + self.execute_and_wait('ifconfig {} mtu {}'.format( + nic['device'], S.getValue('VSWITCH_JUMBO_FRAMES_SIZE'))) + self.execute_and_wait('ip link set dev ' + nic['device'] + ' up') # build and configure system for l2fwd self.execute_and_wait('cd ' + S.getValue('GUEST_OVS_DPDK_DIR')[self._number] + @@ -447,44 +430,78 @@ class IVnfQemu(IVnf): self._configure_disable_firewall() # configure linux bridge - self.execute('brctl addbr br0') + self.execute_and_wait('brctl addbr br0') # add all NICs into the bridge for nic in self._nics: - self.execute('ip addr add ' + - nic['ip'] + ' dev ' + nic['device']) - self.execute('ip link set dev ' + nic['device'] + ' up') - self.execute('brctl addif br0 ' + nic['device']) + self.execute_and_wait('ip addr add ' + nic['ip'] + ' dev ' + nic['device']) + if S.getValue('VSWITCH_JUMBO_FRAMES_ENABLED'): + self.execute_and_wait('ifconfig {} mtu {}'.format( + nic['device'], S.getValue('VSWITCH_JUMBO_FRAMES_SIZE'))) + self.execute_and_wait('ip link set dev ' + nic['device'] + ' up') + self.execute_and_wait('brctl addif br0 ' + nic['device']) - self.execute('ip addr add ' + - S.getValue('GUEST_BRIDGE_IP')[self._number] + - ' dev br0') - self.execute('ip link set dev br0 up') + self.execute_and_wait('ip addr add {} dev br0'.format( + S.getValue('GUEST_BRIDGE_IP')[self._number])) + self.execute_and_wait('ip link set dev br0 up') # Add the arp entries for the IXIA ports and the bridge you are using. # Use command line values if provided. - trafficgen_mac = get_test_param('vanilla_tgen_port1_mac', - S.getValue('VANILLA_TGEN_PORT1_MAC')) - trafficgen_ip = get_test_param('vanilla_tgen_port1_ip', - S.getValue('VANILLA_TGEN_PORT1_IP')) + trafficgen_mac = S.getValue('VANILLA_TGEN_PORT1_MAC') + trafficgen_ip = S.getValue('VANILLA_TGEN_PORT1_IP') - self.execute('arp -s ' + trafficgen_ip + ' ' + trafficgen_mac) + self.execute_and_wait('arp -s ' + trafficgen_ip + ' ' + trafficgen_mac) - trafficgen_mac = get_test_param('vanilla_tgen_port2_mac', - S.getValue('VANILLA_TGEN_PORT2_MAC')) - trafficgen_ip = get_test_param('vanilla_tgen_port2_ip', - S.getValue('VANILLA_TGEN_PORT2_IP')) + trafficgen_mac = S.getValue('VANILLA_TGEN_PORT2_MAC') + trafficgen_ip = S.getValue('VANILLA_TGEN_PORT2_IP') - self.execute('arp -s ' + trafficgen_ip + ' ' + trafficgen_mac) + self.execute_and_wait('arp -s ' + trafficgen_ip + ' ' + trafficgen_mac) # Enable forwarding - self.execute('sysctl -w net.ipv4.ip_forward=1') + self.execute_and_wait('sysctl -w net.ipv4.ip_forward=1') # Controls source route verification # 0 means no source validation - self.execute('sysctl -w net.ipv4.conf.all.rp_filter=0') + self.execute_and_wait('sysctl -w net.ipv4.conf.all.rp_filter=0') for nic in self._nics: - self.execute('sysctl -w net.ipv4.conf.' + nic['device'] + '.rp_filter=0') + self.execute_and_wait('sysctl -w net.ipv4.conf.' + nic['device'] + + '.rp_filter=0') + + def _bind_dpdk_driver(self, driver, pci_slots): + """ + Bind the virtual nics to the driver specific in the conf file + :return: None + """ + if driver == 'uio_pci_generic': + if S.getValue('VNF') == 'QemuPciPassthrough': + # unsupported config, bind to igb_uio instead and exit the + # outer function after completion. + self._logger.error('SR-IOV does not support uio_pci_generic. ' + 'Igb_uio will be used instead.') + self._bind_dpdk_driver('igb_uio_from_src', pci_slots) + return + self.execute_and_wait('modprobe uio_pci_generic') + self.execute_and_wait('./*tools/dpdk*bind.py -b uio_pci_generic '+ + pci_slots) + elif driver == 'vfio_no_iommu': + self.execute_and_wait('modprobe -r vfio') + self.execute_and_wait('modprobe -r vfio_iommu_type1') + self.execute_and_wait('modprobe vfio enable_unsafe_noiommu_mode=Y') + self.execute_and_wait('modprobe vfio-pci') + self.execute_and_wait('./*tools/dpdk*bind.py -b vfio-pci ' + + pci_slots) + elif driver == 'igb_uio_from_src': + # build and insert igb_uio and rebind interfaces to it + self.execute_and_wait('make RTE_OUTPUT=$RTE_SDK/$RTE_TARGET -C ' + '$RTE_SDK/lib/librte_eal/linuxapp/igb_uio') + self.execute_and_wait('modprobe uio') + self.execute_and_wait('insmod %s/kmod/igb_uio.ko' % + S.getValue('RTE_TARGET')) + self.execute_and_wait('./*tools/dpdk*bind.py -b igb_uio ' + pci_slots) + else: + self._logger.error( + 'Unknown driver for binding specified, defaulting to igb_uio') + self._bind_dpdk_driver('igb_uio_from_src', pci_slots) def _set_multi_queue_nic(self): """