1 # Copyright 2015-2016 Intel Corporation.
3 # Licensed under the Apache License, Version 2.0 (the "License");
4 # you may not use this file except in compliance with the License.
5 # You may obtain a copy of the License at
7 # http://www.apache.org/licenses/LICENSE-2.0
9 # Unless required by applicable law or agreed to in writing, software
10 # distributed under the License is distributed on an "AS IS" BASIS,
11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 # See the License for the specific language governing permissions and
13 # limitations under the License.
15 """Automation of QEMU hypervisor for launching guests.
26 from conf import settings as S
27 from conf import get_test_param
28 from vnfs.vnf.vnf import IVnf
32 Abstract class for controling an instance of QEMU
35 _expect = S.getValue('GUEST_PROMPT_LOGIN')
38 class GuestCommandFilter(logging.Filter):
40 Filter out strings beginning with 'guestcmd :'.
42 def filter(self, record):
43 return record.getMessage().startswith(self.prefix)
47 Initialisation function.
49 super(IVnfQemu, self).__init__()
50 self._logger = logging.getLogger(__name__)
51 self._logfile = os.path.join(
52 S.getValue('LOG_DIR'),
53 S.getValue('LOG_FILE_QEMU')) + str(self._number)
54 self._timeout = S.getValue('GUEST_TIMEOUT')[self._number]
55 self._monitor = '%s/vm%dmonitor' % ('/tmp', self._number)
56 self._net1 = get_test_param('guest_nic1_name', None)
57 if self._net1 == None:
58 self._net1 = S.getValue('GUEST_NIC1_NAME')[self._number]
60 self._net1 = self._net1.split(',')[self._number]
61 self._net2 = get_test_param('guest_nic2_name', None)
62 if self._net2 == None:
63 self._net2 = S.getValue('GUEST_NIC2_NAME')[self._number]
65 self._net2 = self._net2.split(',')[self._number]
67 # set guest loopback application based on VNF configuration
68 # cli option take precedence to config file values
69 self._guest_loopback = S.getValue('GUEST_LOOPBACK')[self._number]
71 self._testpmd_fwd_mode = S.getValue('GUEST_TESTPMD_FWD_MODE')
72 # in case of SRIOV we must ensure, that MAC addresses are not swapped
73 if S.getValue('SRIOV_ENABLED') and self._testpmd_fwd_mode.startswith('mac') and \
74 not S.getValue('VNF').endswith('PciPassthrough'):
76 self._logger.info("SRIOV detected, forwarding mode of testpmd was changed from '%s' to '%s'",
77 self._testpmd_fwd_mode, 'io')
78 self._testpmd_fwd_mode = 'io'
80 name = 'Client%d' % self._number
81 vnc = ':%d' % self._number
82 # don't use taskset to affinize main qemu process; It causes hangup
83 # of 2nd VM in case of DPDK. It also slows down VM responsivnes.
84 self._cmd = ['sudo', '-E', S.getValue('QEMU_BIN'),
85 '-m', S.getValue('GUEST_MEMORY')[self._number],
86 '-smp', str(S.getValue('GUEST_SMP')[self._number]),
87 '-cpu', 'host,migratable=off',
88 '-drive', 'if={},file='.format(S.getValue(
89 'GUEST_BOOT_DRIVE_TYPE')) +
90 S.getValue('GUEST_IMAGE')[self._number],
91 '-boot', 'c', '--enable-kvm',
92 '-monitor', 'unix:%s,server,nowait' % self._monitor,
94 'memory-backend-file,id=mem,size=' +
95 str(S.getValue('GUEST_MEMORY')[self._number]) + 'M,' +
96 'mem-path=' + S.getValue('HUGEPAGE_DIR') + ',share=on',
97 '-numa', 'node,memdev=mem -mem-prealloc',
98 '-nographic', '-vnc', str(vnc), '-name', name,
99 '-snapshot', '-net none', '-no-reboot',
101 'if=%s,format=raw,file=fat:rw:%s,snapshot=off' %
102 (S.getValue('GUEST_SHARED_DRIVE_TYPE'),
103 S.getValue('GUEST_SHARE_DIR')[self._number]),
105 self._configure_logging()
107 def _configure_logging(self):
111 self.GuestCommandFilter.prefix = self._log_prefix
113 logger = logging.getLogger()
114 cmd_logger = logging.FileHandler(
115 filename=os.path.join(S.getValue('LOG_DIR'),
116 S.getValue('LOG_FILE_GUEST_CMDS')) +
118 cmd_logger.setLevel(logging.DEBUG)
119 cmd_logger.addFilter(self.GuestCommandFilter())
120 logger.addHandler(cmd_logger)
126 Start QEMU instance, login and prepare for commands.
128 super(IVnfQemu, self).start()
129 if S.getValue('VNF_AFFINITIZATION_ON'):
132 if S.getValue('VSWITCH_VHOST_NET_AFFINITIZATION') and S.getValue(
133 'VNF') == 'QemuVirtioNet':
134 self._affinitize_vhost_net()
137 self._config_guest_loopback()
141 Stops VNF instance gracefully first.
144 # exit testpmd if needed
145 if self._guest_loopback == 'testpmd':
146 self.execute_and_wait('stop', 120, "Done")
147 self.execute_and_wait('quit', 120, "[bB]ye")
150 self.execute_and_wait('poweroff', 120, "Power down")
152 except pexpect.TIMEOUT:
155 # wait until qemu shutdowns
156 self._logger.debug('Wait for QEMU to terminate')
157 for dummy in range(30):
159 if not self.is_running():
162 # just for case that graceful shutdown failed
163 super(IVnfQemu, self).stop()
167 def _login(self, timeout=120):
169 Login to QEMU instance.
171 This can be used immediately after booting the machine, provided a
172 sufficiently long ``timeout`` is given.
174 :param timeout: Timeout to wait for login to complete.
178 # if no timeout was set, we likely started QEMU without waiting for it
179 # to boot. This being the case, we best check that it has finished
181 if not self._timeout:
182 self._expect_process(timeout=timeout)
184 self._child.sendline(S.getValue('GUEST_USERNAME'))
185 self._child.expect(S.getValue('GUEST_PROMPT_PASSWORD'), timeout=5)
186 self._child.sendline(S.getValue('GUEST_PASSWORD'))
188 self._expect_process(S.getValue('GUEST_PROMPT'), timeout=5)
190 def send_and_pass(self, cmd, timeout=30):
192 Send ``cmd`` and wait ``timeout`` seconds for it to pass.
194 :param cmd: Command to send to guest.
195 :param timeout: Time to wait for prompt before checking return code.
200 self.wait(S.getValue('GUEST_PROMPT'), timeout=timeout)
201 self.execute('echo $?')
202 self._child.expect('^0$', timeout=1) # expect a 0
203 self.wait(S.getValue('GUEST_PROMPT'), timeout=timeout)
205 def _affinitize(self):
207 Affinitize the SMP cores of a QEMU instance.
209 This is a bit of a hack. The 'socat' utility is used to
210 interact with the QEMU HMP. This is necessary due to the lack
211 of QMP in older versions of QEMU, like v1.6.2. In future
212 releases, this should be replaced with calls to libvirt or
213 another Python-QEMU wrapper library.
217 thread_id = (r'.* CPU #%d: .* thread_id=(\d+)')
219 self._logger.info('Affinitizing guest...')
221 cur_locale = locale.getdefaultlocale()[1]
222 proc = subprocess.Popen(
223 ('echo', 'info cpus'), stdout=subprocess.PIPE)
224 output = subprocess.check_output(
225 ('sudo', 'socat', '-', 'UNIX-CONNECT:%s' % self._monitor),
229 for cpu in range(0, int(S.getValue('GUEST_SMP')[self._number])):
231 for line in output.decode(cur_locale).split('\n'):
232 match = re.search(thread_id % cpu, line)
234 self._affinitize_pid(
235 S.getValue('GUEST_CORE_BINDING')[self._number][cpu],
240 self._logger.error('Failed to affinitize guest core #%d. Could'
241 ' not parse tid.', cpu)
243 def _affinitize_vhost_net(self):
245 Affinitize the vhost net threads for Vanilla OVS and guest nic queues.
249 self._logger.info('Affinitizing VHOST Net threads.')
251 process1 = subprocess.Popen(args1, stdout=subprocess.PIPE,
253 out = process1.communicate()[0]
255 for line in out.decode(locale.getdefaultlocale()[1]).split('\n'):
256 if re.search('\[vhost-(\d+)', line):
257 processes.append(re.match('\s*(\d+)', line).group(1))
258 self._logger.info('Found %s vhost net threads...', len(processes))
260 cpumap = S.getValue('VSWITCH_VHOST_CPU_MAP')
262 for proc in processes:
263 self._affinitize_pid(cpumap[mapcount], proc)
265 if mapcount + 1 > len(cpumap):
266 # Not enough cpus were given in the mapping to cover all the
267 # threads on a 1 to 1 ratio with cpus so reset the list counter
271 def _config_guest_loopback(self):
273 Configure VM to run VNF, e.g. port forwarding application based on the configuration
275 if self._guest_loopback == 'testpmd':
277 self._configure_testpmd()
278 elif self._guest_loopback == 'l2fwd':
280 self._configure_l2fwd()
281 elif self._guest_loopback == 'linux_bridge':
283 self._configure_linux_bridge()
284 elif self._guest_loopback != 'buildin':
285 self._logger.error('Unsupported guest loopback method "%s" was specified. Option'
286 ' "buildin" will be used as a fallback.', self._guest_loopback)
288 def wait(self, prompt=S.getValue('GUEST_PROMPT'), timeout=30):
289 super(IVnfQemu, self).wait(prompt=prompt, timeout=timeout)
291 def execute_and_wait(self, cmd, timeout=30,
292 prompt=S.getValue('GUEST_PROMPT')):
293 super(IVnfQemu, self).execute_and_wait(cmd, timeout=timeout,
296 def _modify_dpdk_makefile(self):
298 Modifies DPDK makefile in Guest before compilation if needed
302 def _configure_copy_sources(self, dirname):
304 Mount shared directory and copy DPDK and l2fwd sources
306 # mount shared directory
307 self.execute_and_wait('umount /dev/sdb1')
308 self.execute_and_wait('rm -rf ' + S.getValue('GUEST_OVS_DPDK_DIR'))
309 self.execute_and_wait('mkdir -p ' + S.getValue('OVS_DPDK_SHARE'))
310 self.execute_and_wait('mount -o ro,iocharset=utf8 /dev/sdb1 ' +
311 S.getValue('OVS_DPDK_SHARE'))
312 self.execute_and_wait('mkdir -p ' + S.getValue('GUEST_OVS_DPDK_DIR'))
313 self.execute_and_wait('cp -r ' + os.path.join(S.getValue('OVS_DPDK_SHARE'), dirname) +
314 ' ' + S.getValue('GUEST_OVS_DPDK_DIR'))
315 self.execute_and_wait('umount /dev/sdb1')
317 def _configure_disable_firewall(self):
319 Disable firewall in VM
321 for iptables in ['iptables', 'ip6tables']:
323 for chain in ['INPUT', 'FORWARD', 'OUTPUT']:
324 self.execute_and_wait("{} -t filter -P {} ACCEPT".format(iptables, chain))
326 for chain in ['PREROUTING', 'INPUT', 'FORWARD', 'OUTPUT', 'POSTROUTING']:
327 self.execute_and_wait("{} -t mangle -P {} ACCEPT".format(iptables, chain))
329 for chain in ['PREROUTING', 'INPUT', 'OUTPUT', 'POSTROUTING']:
330 self.execute_and_wait("{} -t nat -P {} ACCEPT".format(iptables, chain))
332 # flush rules and delete chains created by user
333 for table in ['filter', 'mangle', 'nat']:
334 self.execute_and_wait("{} -t {} -F".format(iptables, table))
335 self.execute_and_wait("{} -t {} -X".format(iptables, table))
338 def _configure_testpmd(self):
340 Configure VM to perform L2 forwarding between NICs by DPDK's testpmd
342 self._configure_copy_sources('DPDK')
343 self._configure_disable_firewall()
345 # Guest images _should_ have 1024 hugepages by default,
346 # but just in case:'''
347 self.execute_and_wait('sysctl vm.nr_hugepages=1024')
350 self.execute_and_wait('mkdir -p /dev/hugepages')
351 self.execute_and_wait(
352 'mount -t hugetlbfs hugetlbfs /dev/hugepages')
354 # build and configure system for dpdk
355 self.execute_and_wait('cd ' + S.getValue('GUEST_OVS_DPDK_DIR') +
357 self.execute_and_wait('export CC=gcc')
358 self.execute_and_wait('export RTE_SDK=' +
359 S.getValue('GUEST_OVS_DPDK_DIR') + '/DPDK')
360 self.execute_and_wait('export RTE_TARGET=%s' % S.getValue('RTE_TARGET'))
362 # modify makefile if needed
363 self._modify_dpdk_makefile()
365 # disable network interfaces, so DPDK can take care of them
366 self.execute_and_wait('ifdown ' + self._net1)
367 self.execute_and_wait('ifdown ' + self._net2)
369 # build and insert igb_uio and rebind interfaces to it
370 self.execute_and_wait('make RTE_OUTPUT=$RTE_SDK/$RTE_TARGET -C '
371 '$RTE_SDK/lib/librte_eal/linuxapp/igb_uio')
372 self.execute_and_wait('modprobe uio')
373 self.execute_and_wait('insmod %s/kmod/igb_uio.ko' %
374 S.getValue('RTE_TARGET'))
375 self.execute_and_wait('./tools/dpdk*bind.py --status')
376 self.execute_and_wait(
377 './tools/dpdk*bind.py -u' ' ' +
378 S.getValue('GUEST_NET1_PCI_ADDRESS')[self._number] + ' ' +
379 S.getValue('GUEST_NET2_PCI_ADDRESS')[self._number])
380 self.execute_and_wait(
381 './tools/dpdk*bind.py -b igb_uio' ' ' +
382 S.getValue('GUEST_NET1_PCI_ADDRESS')[self._number] + ' ' +
383 S.getValue('GUEST_NET2_PCI_ADDRESS')[self._number])
384 self.execute_and_wait('./tools/dpdk*bind.py --status')
386 # build and run 'test-pmd'
387 self.execute_and_wait('cd ' + S.getValue('GUEST_OVS_DPDK_DIR') +
388 '/DPDK/app/test-pmd')
389 self.execute_and_wait('make clean')
390 self.execute_and_wait('make')
391 if int(S.getValue('GUEST_NIC_QUEUES')):
392 self.execute_and_wait(
393 './testpmd {} -n4 --socket-mem 512 --'.format(
394 S.getValue('GUEST_TESTPMD_CPU_MASK')) +
395 ' --burst=64 -i --txqflags=0xf00 ' +
396 '--nb-cores={} --rxq={} --txq={} '.format(
397 S.getValue('GUEST_TESTPMD_NB_CORES'),
398 S.getValue('GUEST_TESTPMD_TXQ'),
399 S.getValue('GUEST_TESTPMD_RXQ')) +
400 '--disable-hw-vlan', 60, "Done")
402 self.execute_and_wait(
403 './testpmd {} -n 4 --socket-mem 512 --'.format(
404 S.getValue('GUEST_TESTPMD_CPU_MASK')) +
405 ' --burst=64 -i --txqflags=0xf00 ' +
406 '--disable-hw-vlan', 60, "Done")
407 self.execute('set fwd ' + self._testpmd_fwd_mode, 1)
408 self.execute_and_wait('start', 20,
409 'TX RS bit threshold=.+ - TXQ flags=0xf00')
411 def _configure_l2fwd(self):
413 Configure VM to perform L2 forwarding between NICs by l2fwd module
415 if int(S.getValue('GUEST_NIC_QUEUES')):
416 self._set_multi_queue_nic()
417 self._configure_copy_sources('l2fwd')
418 self._configure_disable_firewall()
420 # build and configure system for l2fwd
421 self.execute_and_wait('cd ' + S.getValue('GUEST_OVS_DPDK_DIR') +
423 self.execute_and_wait('export CC=gcc')
425 self.execute_and_wait('make')
426 self.execute_and_wait('insmod ' + S.getValue('GUEST_OVS_DPDK_DIR') +
427 '/l2fwd' + '/l2fwd.ko net1=' + self._net1 +
428 ' net2=' + self._net2)
430 def _configure_linux_bridge(self):
432 Configure VM to perform L2 forwarding between NICs by linux bridge
434 if int(S.getValue('GUEST_NIC_QUEUES')):
435 self._set_multi_queue_nic()
436 self._configure_disable_firewall()
438 self.execute('ip addr add ' +
439 S.getValue('VANILLA_NIC1_IP_CIDR')[self._number] +
440 ' dev ' + self._net1)
441 self.execute('ip link set dev ' + self._net1 + ' up')
443 self.execute('ip addr add ' +
444 S.getValue('VANILLA_NIC2_IP_CIDR')[self._number] +
445 ' dev ' + self._net2)
446 self.execute('ip link set dev ' + self._net2 + ' up')
448 # configure linux bridge
449 self.execute('brctl addbr br0')
450 self.execute('brctl addif br0 ' + self._net1 + ' ' + self._net2)
451 self.execute('ip addr add ' +
452 S.getValue('VANILLA_BRIDGE_IP')[self._number] +
454 self.execute('ip link set dev br0 up')
456 # Add the arp entries for the IXIA ports and the bridge you are using.
457 # Use command line values if provided.
458 trafficgen_mac = get_test_param('vanilla_tgen_port1_mac',
459 S.getValue('VANILLA_TGEN_PORT1_MAC'))
460 trafficgen_ip = get_test_param('vanilla_tgen_port1_ip',
461 S.getValue('VANILLA_TGEN_PORT1_IP'))
463 self.execute('arp -s ' + trafficgen_ip + ' ' + trafficgen_mac)
465 trafficgen_mac = get_test_param('vanilla_tgen_port2_mac',
466 S.getValue('VANILLA_TGEN_PORT2_MAC'))
467 trafficgen_ip = get_test_param('vanilla_tgen_port2_ip',
468 S.getValue('VANILLA_TGEN_PORT2_IP'))
470 self.execute('arp -s ' + trafficgen_ip + ' ' + trafficgen_mac)
473 self.execute('sysctl -w net.ipv4.ip_forward=1')
475 # Controls source route verification
476 # 0 means no source validation
477 self.execute('sysctl -w net.ipv4.conf.all.rp_filter=0')
478 self.execute('sysctl -w net.ipv4.conf.' + self._net1 + '.rp_filter=0')
479 self.execute('sysctl -w net.ipv4.conf.' + self._net2 + '.rp_filter=0')
481 def _set_multi_queue_nic(self):
483 Enable multi-queue in guest kernel with ethool.
486 self.execute_and_wait('ethtool -L {} combined {}'.format(
487 self._net1, S.getValue('GUEST_NIC_QUEUES')))
488 self.execute_and_wait('ethtool -l {}'.format(self._net1))
489 self.execute_and_wait('ethtool -L {} combined {}'.format(
490 self._net2, S.getValue('GUEST_NIC_QUEUES')))
491 self.execute_and_wait('ethtool -l {}'.format(self._net2))