X-Git-Url: https://gerrit.opnfv.org/gerrit/gitweb?a=blobdiff_plain;f=vnfs%2Fqemu%2Fqemu.py;h=67dbfab412546c0ae8d84a8cc7ec1ce1e6a5509a;hb=7bae8ffe122b7d62eae03a2393182342f1f8fe85;hp=cb6d9ecc007fcf8c9d8aa549d58b14678219839f;hpb=5853025b59cb6e44f23a1d245a58a2cd286e03dc;p=vswitchperf.git diff --git a/vnfs/qemu/qemu.py b/vnfs/qemu/qemu.py index cb6d9ecc..67dbfab4 100644 --- a/vnfs/qemu/qemu.py +++ b/vnfs/qemu/qemu.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Automation of QEMU hypervisor for launching vhost-cuse enabled guests. +"""Automation of QEMU hypervisor for launching guests. """ import os @@ -20,6 +20,8 @@ import logging import locale import re import subprocess +import time +import pexpect from conf import settings as S from conf import get_test_param @@ -30,7 +32,7 @@ class IVnfQemu(IVnf): Abstract class for controling an instance of QEMU """ _cmd = None - _expect = S.getValue('GUEST_PROMPT_LOGIN') + _expect = None _proc_name = 'qemu' class GuestCommandFilter(logging.Filter): @@ -45,33 +47,57 @@ class IVnfQemu(IVnf): Initialisation function. """ super(IVnfQemu, self).__init__() + + self._expect = S.getValue('GUEST_PROMPT_LOGIN')[self._number] self._logger = logging.getLogger(__name__) self._logfile = os.path.join( S.getValue('LOG_DIR'), S.getValue('LOG_FILE_QEMU')) + str(self._number) self._timeout = S.getValue('GUEST_TIMEOUT')[self._number] self._monitor = '%s/vm%dmonitor' % ('/tmp', self._number) - self._net1 = get_test_param('guest_nic1_name', None) - if self._net1 == None: - self._net1 = S.getValue('GUEST_NIC1_NAME')[self._number] - else: - self._net1 = self._net1.split(',')[self._number] - self._net2 = get_test_param('guest_nic2_name', None) - if self._net2 == None: - self._net2 = S.getValue('GUEST_NIC2_NAME')[self._number] - else: - self._net2 = self._net2.split(',')[self._number] + # read GUEST NICs configuration and use only defined NR of NICS + nics_nr = S.getValue('GUEST_NICS_NR')[self._number] + # and inform user about missconfiguration + if nics_nr < 1: + raise RuntimeError('At least one VM NIC is mandotory, but {} ' + 'NICs are configured'.format(nics_nr)) + elif nics_nr > 1 and nics_nr % 2: + nics_nr = int(nics_nr / 2) * 2 + self._logger.warning('Odd number of NICs is configured, only ' + '%s NICs will be used', nics_nr) + + self._nics = S.getValue('GUEST_NICS')[self._number][:nics_nr] + # set guest loopback application based on VNF configuration + # cli option take precedence to config file values + self._guest_loopback = S.getValue('GUEST_LOOPBACK')[self._number] + + self._testpmd_fwd_mode = S.getValue('GUEST_TESTPMD_FWD_MODE')[self._number] + # in case of SRIOV we must ensure, that MAC addresses are not swapped + if S.getValue('SRIOV_ENABLED') and self._testpmd_fwd_mode.startswith('mac') and \ + not S.getValue('VNF').endswith('PciPassthrough'): + + self._logger.info("SRIOV detected, forwarding mode of testpmd was changed from '%s' to '%s'", + self._testpmd_fwd_mode, 'io') + self._testpmd_fwd_mode = 'io' + + guest_smp = int(get_test_param('guest_smp', 0)) + if guest_smp: + override_list = [guest_smp] * (self._number + 1) + S.setValue('GUEST_SMP', override_list) name = 'Client%d' % self._number vnc = ':%d' % self._number - # don't use taskset to affinize main qemu process; It causes hangup - # of 2nd VM in case of DPDK. It also slows down VM responsivnes. - self._cmd = ['sudo', '-E', S.getValue('QEMU_BIN'), + # NOTE: affinization of main qemu process can cause hangup of 2nd VM + # in case of DPDK usage. It can also slow down VM response time. + cpumask = ",".join(S.getValue('GUEST_CORE_BINDING')[self._number]) + self._cmd = ['sudo', '-E', 'taskset', '-c', cpumask, + S.getValue('TOOLS')['qemu-system'], '-m', S.getValue('GUEST_MEMORY')[self._number], '-smp', str(S.getValue('GUEST_SMP')[self._number]), '-cpu', 'host,migratable=off', - '-drive', 'if=scsi,file=' + + '-drive', 'if={},file='.format(S.getValue( + 'GUEST_BOOT_DRIVE_TYPE')[self._number]) + S.getValue('GUEST_IMAGE')[self._number], '-boot', 'c', '--enable-kvm', '-monitor', 'unix:%s,server,nowait' % self._monitor, @@ -83,8 +109,9 @@ class IVnfQemu(IVnf): '-nographic', '-vnc', str(vnc), '-name', name, '-snapshot', '-net none', '-no-reboot', '-drive', - 'if=scsi,format=raw,file=fat:rw:%s,snapshot=off' % - S.getValue('GUEST_SHARE_DIR')[self._number], + 'if=%s,format=raw,file=fat:rw:%s,snapshot=off' % + (S.getValue('GUEST_SHARED_DRIVE_TYPE')[self._number], + S.getValue('GUEST_SHARE_DIR')[self._number]), ] self._configure_logging() @@ -113,9 +140,39 @@ class IVnfQemu(IVnf): if S.getValue('VNF_AFFINITIZATION_ON'): self._affinitize() + if S.getValue('VSWITCH_VHOST_NET_AFFINITIZATION') and S.getValue( + 'VNF') == 'QemuVirtioNet': + self._affinitize_vhost_net() + if self._timeout: self._config_guest_loopback() + def stop(self): + """ + Stops VNF instance gracefully first. + """ + try: + # exit testpmd if needed + if self._guest_loopback == 'testpmd': + self.execute_and_wait('stop', 120, "Done") + self.execute_and_wait('quit', 120, "[bB]ye") + + # turn off VM + self.execute_and_wait('poweroff', 120, "Power down") + + except pexpect.TIMEOUT: + self.kill() + + # wait until qemu shutdowns + self._logger.debug('Wait for QEMU to terminate') + for dummy in range(30): + time.sleep(1) + if not self.is_running(): + break + + # just for case that graceful shutdown failed + super(IVnfQemu, self).stop() + # helper functions def _login(self, timeout=120): @@ -135,11 +192,11 @@ class IVnfQemu(IVnf): if not self._timeout: self._expect_process(timeout=timeout) - self._child.sendline(S.getValue('GUEST_USERNAME')) - self._child.expect(S.getValue('GUEST_PROMPT_PASSWORD'), timeout=5) - self._child.sendline(S.getValue('GUEST_PASSWORD')) + self._child.sendline(S.getValue('GUEST_USERNAME')[self._number]) + self._child.expect(S.getValue('GUEST_PROMPT_PASSWORD')[self._number], timeout=5) + self._child.sendline(S.getValue('GUEST_PASSWORD')[self._number]) - self._expect_process(S.getValue('GUEST_PROMPT'), timeout=5) + self._expect_process(S.getValue('GUEST_PROMPT')[self._number], timeout=5) def send_and_pass(self, cmd, timeout=30): """ @@ -151,10 +208,10 @@ class IVnfQemu(IVnf): :returns: None """ self.execute(cmd) - self.wait(S.getValue('GUEST_PROMPT'), timeout=timeout) + self.wait(S.getValue('GUEST_PROMPT')[self._number], timeout=timeout) self.execute('echo $?') self._child.expect('^0$', timeout=1) # expect a 0 - self.wait(S.getValue('GUEST_PROMPT'), timeout=timeout) + self.wait(S.getValue('GUEST_PROMPT')[self._number], timeout=timeout) def _affinitize(self): """ @@ -180,52 +237,82 @@ class IVnfQemu(IVnf): stdin=proc.stdout) proc.wait() + guest_core_binding = int(get_test_param('guest_core_binding', 0)) for cpu in range(0, int(S.getValue('GUEST_SMP')[self._number])): match = None for line in output.decode(cur_locale).split('\n'): match = re.search(thread_id % cpu, line) if match: - self._affinitize_pid( - S.getValue('GUEST_CORE_BINDING')[self._number][cpu], - match.group(1)) + if guest_core_binding: + self._affinitize_pid(guest_core_binding, match.group(1)) + else: + self._affinitize_pid( + S.getValue('GUEST_CORE_BINDING')[self._number][cpu], + match.group(1)) break if not match: self._logger.error('Failed to affinitize guest core #%d. Could' ' not parse tid.', cpu) - def _config_guest_loopback(self): + def _affinitize_vhost_net(self): """ - Configure VM to run VNF (e.g. port forwarding application) + Affinitize the vhost net threads for Vanilla OVS and guest nic queues. + + :return: None """ - # set guest loopback application based on VNF configuration - # cli option take precedence to config file values - guest_loopback = S.getValue('GUEST_LOOPBACK')[self._number] + self._logger.info('Affinitizing VHOST Net threads.') + args1 = ['pgrep', 'vhost-'] + process1 = subprocess.Popen(args1, stdout=subprocess.PIPE, + shell=False) + out = process1.communicate()[0] + processes = out.decode(locale.getdefaultlocale()[1]).split('\n') + if processes[-1] == '': + processes.pop() # pgrep may return an extra line with no data + self._logger.info('Found %s vhost net threads...', len(processes)) + + cpumap = S.getValue('VSWITCH_VHOST_CPU_MAP') + mapcount = 0 + for proc in processes: + self._affinitize_pid(cpumap[mapcount], proc) + mapcount += 1 + if mapcount + 1 > len(cpumap): + # Not enough cpus were given in the mapping to cover all the + # threads on a 1 to 1 ratio with cpus so reset the list counter + # to 0. + mapcount = 0 - if guest_loopback == 'testpmd': + def _config_guest_loopback(self): + """ + Configure VM to run VNF, e.g. port forwarding application based on the configuration + """ + if self._guest_loopback == 'testpmd': self._login() self._configure_testpmd() - elif guest_loopback == 'l2fwd': + elif self._guest_loopback == 'l2fwd': self._login() self._configure_l2fwd() - elif guest_loopback == 'linux_bridge': + elif self._guest_loopback == 'linux_bridge': self._login() self._configure_linux_bridge() - elif guest_loopback != 'buildin': + elif self._guest_loopback != 'buildin': self._logger.error('Unsupported guest loopback method "%s" was specified. Option' - ' "buildin" will be used as a fallback.', guest_loopback) + ' "buildin" will be used as a fallback.', self._guest_loopback) - def wait(self, prompt=S.getValue('GUEST_PROMPT'), timeout=30): + def wait(self, prompt=None, timeout=30): + if prompt is None: + prompt = S.getValue('GUEST_PROMPT')[self._number] super(IVnfQemu, self).wait(prompt=prompt, timeout=timeout) - def execute_and_wait(self, cmd, timeout=30, - prompt=S.getValue('GUEST_PROMPT')): + def execute_and_wait(self, cmd, timeout=30, prompt=None): + if prompt is None: + prompt = S.getValue('GUEST_PROMPT')[self._number] super(IVnfQemu, self).execute_and_wait(cmd, timeout=timeout, prompt=prompt) def _modify_dpdk_makefile(self): """ - Modifies DPDK makefile in Guest before compilation + Modifies DPDK makefile in Guest before compilation if needed """ pass @@ -234,14 +321,15 @@ class IVnfQemu(IVnf): Mount shared directory and copy DPDK and l2fwd sources """ # mount shared directory - self.execute_and_wait('umount ' + S.getValue('OVS_DPDK_SHARE')) - self.execute_and_wait('rm -rf ' + S.getValue('GUEST_OVS_DPDK_DIR')) - self.execute_and_wait('mkdir -p ' + S.getValue('OVS_DPDK_SHARE')) - self.execute_and_wait('mount -o iocharset=utf8 /dev/sdb1 ' + - S.getValue('OVS_DPDK_SHARE')) - self.execute_and_wait('mkdir -p ' + S.getValue('GUEST_OVS_DPDK_DIR')) - self.execute_and_wait('cp -ra ' + os.path.join(S.getValue('OVS_DPDK_SHARE'), dirname) + - ' ' + S.getValue('GUEST_OVS_DPDK_DIR')) + self.execute_and_wait('umount /dev/sdb1') + self.execute_and_wait('rm -rf ' + S.getValue('GUEST_OVS_DPDK_DIR')[self._number]) + self.execute_and_wait('mkdir -p ' + S.getValue('GUEST_OVS_DPDK_SHARE')[self._number]) + self.execute_and_wait('mount -o ro,iocharset=utf8 /dev/sdb1 ' + + S.getValue('GUEST_OVS_DPDK_SHARE')[self._number]) + self.execute_and_wait('mkdir -p ' + S.getValue('GUEST_OVS_DPDK_DIR')[self._number]) + self.execute_and_wait('cp -r ' + os.path.join(S.getValue('GUEST_OVS_DPDK_SHARE')[self._number], dirname) + + ' ' + S.getValue('GUEST_OVS_DPDK_DIR')[self._number]) + self.execute_and_wait('umount /dev/sdb1') def _configure_disable_firewall(self): """ @@ -273,7 +361,7 @@ class IVnfQemu(IVnf): # Guest images _should_ have 1024 hugepages by default, # but just in case:''' - self.execute_and_wait('sysctl vm.nr_hugepages=1024') + self.execute_and_wait('sysctl vm.nr_hugepages={}'.format(S.getValue('GUEST_HUGEPAGES_NR')[self._number])) # Mount hugepages self.execute_and_wait('mkdir -p /dev/hugepages') @@ -281,73 +369,97 @@ class IVnfQemu(IVnf): 'mount -t hugetlbfs hugetlbfs /dev/hugepages') # build and configure system for dpdk - self.execute_and_wait('cd ' + S.getValue('GUEST_OVS_DPDK_DIR') + + self.execute_and_wait('cd ' + S.getValue('GUEST_OVS_DPDK_DIR')[self._number] + '/DPDK') self.execute_and_wait('export CC=gcc') self.execute_and_wait('export RTE_SDK=' + - S.getValue('GUEST_OVS_DPDK_DIR') + '/DPDK') + S.getValue('GUEST_OVS_DPDK_DIR')[self._number] + '/DPDK') self.execute_and_wait('export RTE_TARGET=%s' % S.getValue('RTE_TARGET')) # modify makefile if needed self._modify_dpdk_makefile() + # disable network interfaces, so DPDK can take care of them + for nic in self._nics: + self.execute_and_wait('ifdown ' + nic['device']) + + # build and insert igb_uio and rebind interfaces to it self.execute_and_wait('make RTE_OUTPUT=$RTE_SDK/$RTE_TARGET -C ' '$RTE_SDK/lib/librte_eal/linuxapp/igb_uio') self.execute_and_wait('modprobe uio') self.execute_and_wait('insmod %s/kmod/igb_uio.ko' % S.getValue('RTE_TARGET')) - self.execute_and_wait('./tools/dpdk_nic_bind.py --status') - self.execute_and_wait( - './tools/dpdk_nic_bind.py -b igb_uio' ' ' + - S.getValue('GUEST_NET1_PCI_ADDRESS')[self._number] + ' ' + - S.getValue('GUEST_NET2_PCI_ADDRESS')[self._number]) + self.execute_and_wait('./tools/dpdk*bind.py --status') + pci_list = ' '.join([nic['pci'] for nic in self._nics]) + self.execute_and_wait('./tools/dpdk*bind.py -u ' + pci_list) + self.execute_and_wait('./tools/dpdk*bind.py -b igb_uio ' + pci_list) + self.execute_and_wait('./tools/dpdk*bind.py --status') # build and run 'test-pmd' - self.execute_and_wait('cd ' + S.getValue('GUEST_OVS_DPDK_DIR') + + self.execute_and_wait('cd ' + S.getValue('GUEST_OVS_DPDK_DIR')[self._number] + '/DPDK/app/test-pmd') self.execute_and_wait('make clean') self.execute_and_wait('make') - self.execute_and_wait('./testpmd -c 0x3 -n 4 --socket-mem 512 --' - ' --burst=64 -i --txqflags=0xf00 ' + - '--disable-hw-vlan', 60, "Done") - self.execute('set fwd mac_retry', 1) - self.execute_and_wait('start', 20, - 'TX RS bit threshold=0 - TXQ flags=0xf00') + + # get testpmd settings from CLI + testpmd_params = get_test_param('guest_testpmd_params', + S.getValue('GUEST_TESTPMD_PARAMS')[self._number]) + + self.execute_and_wait( './testpmd {}'.format(testpmd_params), 60, "Done") + self.execute('set fwd ' + self._testpmd_fwd_mode, 1) + self.execute_and_wait('start', 20, 'testpmd>') def _configure_l2fwd(self): """ Configure VM to perform L2 forwarding between NICs by l2fwd module """ + if int(S.getValue('GUEST_NIC_QUEUES')[self._number]): + self._set_multi_queue_nic() self._configure_copy_sources('l2fwd') self._configure_disable_firewall() + # configure all interfaces + for nic in self._nics: + self.execute('ip addr add ' + + nic['ip'] + ' dev ' + nic['device']) + self.execute('ip link set dev ' + nic['device'] + ' up') + # build and configure system for l2fwd - self.execute_and_wait('cd ' + S.getValue('GUEST_OVS_DPDK_DIR') + + self.execute_and_wait('cd ' + S.getValue('GUEST_OVS_DPDK_DIR')[self._number] + '/l2fwd') self.execute_and_wait('export CC=gcc') self.execute_and_wait('make') - self.execute_and_wait('insmod ' + S.getValue('GUEST_OVS_DPDK_DIR') + - '/l2fwd' + '/l2fwd.ko net1=' + self._net1 + - ' net2=' + self._net2) + if len(self._nics) == 2: + self.execute_and_wait('insmod ' + S.getValue('GUEST_OVS_DPDK_DIR')[self._number] + + '/l2fwd' + '/l2fwd.ko net1=' + self._nics[0]['device'] + + ' net2=' + self._nics[1]['device']) + else: + raise RuntimeError('l2fwd can forward only between 2 NICs, but {} NICs are ' + 'configured inside GUEST'.format(len(self._nics))) def _configure_linux_bridge(self): """ Configure VM to perform L2 forwarding between NICs by linux bridge """ + if int(S.getValue('GUEST_NIC_QUEUES')[self._number]): + self._set_multi_queue_nic() self._configure_disable_firewall() - self.execute('ifconfig ' + self._net1 + ' ' + - S.getValue('VANILLA_NIC1_IP_CIDR')[self._number]) - - self.execute('ifconfig ' + self._net2 + ' ' + - S.getValue('VANILLA_NIC2_IP_CIDR')[self._number]) - # configure linux bridge self.execute('brctl addbr br0') - self.execute('brctl addif br0 ' + self._net1 + ' ' + self._net2) - self.execute('ifconfig br0 ' + - S.getValue('VANILLA_BRIDGE_IP')[self._number]) + + # add all NICs into the bridge + for nic in self._nics: + self.execute('ip addr add ' + + nic['ip'] + ' dev ' + nic['device']) + self.execute('ip link set dev ' + nic['device'] + ' up') + self.execute('brctl addif br0 ' + nic['device']) + + self.execute('ip addr add ' + + S.getValue('GUEST_BRIDGE_IP')[self._number] + + ' dev br0') + self.execute('ip link set dev br0 up') # Add the arp entries for the IXIA ports and the bridge you are using. # Use command line values if provided. @@ -371,5 +483,15 @@ class IVnfQemu(IVnf): # Controls source route verification # 0 means no source validation self.execute('sysctl -w net.ipv4.conf.all.rp_filter=0') - self.execute('sysctl -w net.ipv4.conf.' + self._net1 + '.rp_filter=0') - self.execute('sysctl -w net.ipv4.conf.' + self._net2 + '.rp_filter=0') + for nic in self._nics: + self.execute('sysctl -w net.ipv4.conf.' + nic['device'] + '.rp_filter=0') + + def _set_multi_queue_nic(self): + """ + Enable multi-queue in guest kernel with ethool. + :return: None + """ + for nic in self._nics: + self.execute_and_wait('ethtool -L {} combined {}'.format( + nic['device'], S.getValue('GUEST_NIC_QUEUES')[self._number])) + self.execute_and_wait('ethtool -l {}'.format(nic['device']))