1 # Copyright 2015-2016 Intel Corporation.
3 # Licensed under the Apache License, Version 2.0 (the "License");
4 # you may not use this file except in compliance with the License.
5 # You may obtain a copy of the License at
7 # http://www.apache.org/licenses/LICENSE-2.0
9 # Unless required by applicable law or agreed to in writing, software
10 # distributed under the License is distributed on an "AS IS" BASIS,
11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 # See the License for the specific language governing permissions and
13 # limitations under the License.
15 """Automation of QEMU hypervisor for launching guests.
26 from conf import settings as S
27 from conf import get_test_param
28 from vnfs.vnf.vnf import IVnf
32 Abstract class for controling an instance of QEMU
35 _expect = S.getValue('GUEST_PROMPT_LOGIN')
38 class GuestCommandFilter(logging.Filter):
40 Filter out strings beginning with 'guestcmd :'.
42 def filter(self, record):
43 return record.getMessage().startswith(self.prefix)
47 Initialisation function.
49 super(IVnfQemu, self).__init__()
50 self._logger = logging.getLogger(__name__)
51 self._logfile = os.path.join(
52 S.getValue('LOG_DIR'),
53 S.getValue('LOG_FILE_QEMU')) + str(self._number)
54 self._timeout = S.getValue('GUEST_TIMEOUT')[self._number]
55 self._monitor = '%s/vm%dmonitor' % ('/tmp', self._number)
56 self._net1 = get_test_param('guest_nic1_name', None)
57 if self._net1 == None:
58 self._net1 = S.getValue('GUEST_NIC1_NAME')[self._number]
60 self._net1 = self._net1.split(',')[self._number]
61 self._net2 = get_test_param('guest_nic2_name', None)
62 if self._net2 == None:
63 self._net2 = S.getValue('GUEST_NIC2_NAME')[self._number]
65 self._net2 = self._net2.split(',')[self._number]
67 # set guest loopback application based on VNF configuration
68 # cli option take precedence to config file values
69 self._guest_loopback = S.getValue('GUEST_LOOPBACK')[self._number]
71 self._testpmd_fwd_mode = S.getValue('GUEST_TESTPMD_FWD_MODE')
72 # in case of SRIOV we must ensure, that MAC addresses are not swapped
73 if S.getValue('SRIOV_ENABLED') and self._testpmd_fwd_mode.startswith('mac') and \
74 not S.getValue('VNF').endswith('PciPassthrough'):
76 self._logger.info("SRIOV detected, forwarding mode of testpmd was changed from '%s' to '%s'",
77 self._testpmd_fwd_mode, 'io')
78 self._testpmd_fwd_mode = 'io'
80 name = 'Client%d' % self._number
81 vnc = ':%d' % self._number
82 # don't use taskset to affinize main qemu process; It causes hangup
83 # of 2nd VM in case of DPDK. It also slows down VM responsivnes.
84 self._cmd = ['sudo', '-E', S.getValue('QEMU_BIN'),
85 '-m', S.getValue('GUEST_MEMORY')[self._number],
86 '-smp', str(S.getValue('GUEST_SMP')[self._number]),
87 '-cpu', 'host,migratable=off',
88 '-drive', 'if={},file='.format(S.getValue(
89 'GUEST_BOOT_DRIVE_TYPE')) +
90 S.getValue('GUEST_IMAGE')[self._number],
91 '-boot', 'c', '--enable-kvm',
92 '-monitor', 'unix:%s,server,nowait' % self._monitor,
94 'memory-backend-file,id=mem,size=' +
95 str(S.getValue('GUEST_MEMORY')[self._number]) + 'M,' +
96 'mem-path=' + S.getValue('HUGEPAGE_DIR') + ',share=on',
97 '-numa', 'node,memdev=mem -mem-prealloc',
98 '-nographic', '-vnc', str(vnc), '-name', name,
99 '-snapshot', '-net none', '-no-reboot',
101 'if=%s,format=raw,file=fat:rw:%s,snapshot=off' %
102 (S.getValue('GUEST_SHARED_DRIVE_TYPE'),
103 S.getValue('GUEST_SHARE_DIR')[self._number]),
105 self._configure_logging()
107 def _configure_logging(self):
111 self.GuestCommandFilter.prefix = self._log_prefix
113 logger = logging.getLogger()
114 cmd_logger = logging.FileHandler(
115 filename=os.path.join(S.getValue('LOG_DIR'),
116 S.getValue('LOG_FILE_GUEST_CMDS')) +
118 cmd_logger.setLevel(logging.DEBUG)
119 cmd_logger.addFilter(self.GuestCommandFilter())
120 logger.addHandler(cmd_logger)
126 Start QEMU instance, login and prepare for commands.
128 super(IVnfQemu, self).start()
129 if S.getValue('VNF_AFFINITIZATION_ON'):
132 if S.getValue('VSWITCH_VHOST_NET_AFFINITIZATION') and S.getValue(
133 'VNF') == 'QemuVirtioNet':
134 self._affinitize_vhost_net()
137 self._config_guest_loopback()
141 Stops VNF instance gracefully first.
144 # exit testpmd if needed
145 if self._guest_loopback == 'testpmd':
146 self.execute_and_wait('stop', 120, "Done")
147 self.execute_and_wait('quit', 120, "[bB]ye")
150 self.execute_and_wait('poweroff', 120, "Power down")
152 except pexpect.TIMEOUT:
155 # wait until qemu shutdowns
156 self._logger.debug('Wait for QEMU to terminate')
157 for dummy in range(30):
159 if not self.is_running():
162 # just for case that graceful shutdown failed
163 super(IVnfQemu, self).stop()
167 def _login(self, timeout=120):
169 Login to QEMU instance.
171 This can be used immediately after booting the machine, provided a
172 sufficiently long ``timeout`` is given.
174 :param timeout: Timeout to wait for login to complete.
178 # if no timeout was set, we likely started QEMU without waiting for it
179 # to boot. This being the case, we best check that it has finished
181 if not self._timeout:
182 self._expect_process(timeout=timeout)
184 self._child.sendline(S.getValue('GUEST_USERNAME'))
185 self._child.expect(S.getValue('GUEST_PROMPT_PASSWORD'), timeout=5)
186 self._child.sendline(S.getValue('GUEST_PASSWORD'))
188 self._expect_process(S.getValue('GUEST_PROMPT'), timeout=5)
190 def send_and_pass(self, cmd, timeout=30):
192 Send ``cmd`` and wait ``timeout`` seconds for it to pass.
194 :param cmd: Command to send to guest.
195 :param timeout: Time to wait for prompt before checking return code.
200 self.wait(S.getValue('GUEST_PROMPT'), timeout=timeout)
201 self.execute('echo $?')
202 self._child.expect('^0$', timeout=1) # expect a 0
203 self.wait(S.getValue('GUEST_PROMPT'), timeout=timeout)
205 def _affinitize(self):
207 Affinitize the SMP cores of a QEMU instance.
209 This is a bit of a hack. The 'socat' utility is used to
210 interact with the QEMU HMP. This is necessary due to the lack
211 of QMP in older versions of QEMU, like v1.6.2. In future
212 releases, this should be replaced with calls to libvirt or
213 another Python-QEMU wrapper library.
217 thread_id = (r'.* CPU #%d: .* thread_id=(\d+)')
219 self._logger.info('Affinitizing guest...')
221 cur_locale = locale.getdefaultlocale()[1]
222 proc = subprocess.Popen(
223 ('echo', 'info cpus'), stdout=subprocess.PIPE)
224 output = subprocess.check_output(
225 ('sudo', 'socat', '-', 'UNIX-CONNECT:%s' % self._monitor),
229 for cpu in range(0, int(S.getValue('GUEST_SMP')[self._number])):
231 for line in output.decode(cur_locale).split('\n'):
232 match = re.search(thread_id % cpu, line)
234 self._affinitize_pid(
235 S.getValue('GUEST_CORE_BINDING')[self._number][cpu],
240 self._logger.error('Failed to affinitize guest core #%d. Could'
241 ' not parse tid.', cpu)
243 def _affinitize_vhost_net(self):
245 Affinitize the vhost net threads for Vanilla OVS and guest nic queues.
249 self._logger.info('Affinitizing VHOST Net threads.')
250 args1 = ['pgrep', 'vhost-']
251 process1 = subprocess.Popen(args1, stdout=subprocess.PIPE,
253 out = process1.communicate()[0]
254 processes = out.decode(locale.getdefaultlocale()[1]).split('\n')
255 if processes[-1] == '':
256 processes.pop() # pgrep may return an extra line with no data
257 self._logger.info('Found %s vhost net threads...', len(processes))
259 cpumap = S.getValue('VSWITCH_VHOST_CPU_MAP')
261 for proc in processes:
262 self._affinitize_pid(cpumap[mapcount], proc)
264 if mapcount + 1 > len(cpumap):
265 # Not enough cpus were given in the mapping to cover all the
266 # threads on a 1 to 1 ratio with cpus so reset the list counter
270 def _config_guest_loopback(self):
272 Configure VM to run VNF, e.g. port forwarding application based on the configuration
274 if self._guest_loopback == 'testpmd':
276 self._configure_testpmd()
277 elif self._guest_loopback == 'l2fwd':
279 self._configure_l2fwd()
280 elif self._guest_loopback == 'linux_bridge':
282 self._configure_linux_bridge()
283 elif self._guest_loopback != 'buildin':
284 self._logger.error('Unsupported guest loopback method "%s" was specified. Option'
285 ' "buildin" will be used as a fallback.', self._guest_loopback)
287 def wait(self, prompt=S.getValue('GUEST_PROMPT'), timeout=30):
288 super(IVnfQemu, self).wait(prompt=prompt, timeout=timeout)
290 def execute_and_wait(self, cmd, timeout=30,
291 prompt=S.getValue('GUEST_PROMPT')):
292 super(IVnfQemu, self).execute_and_wait(cmd, timeout=timeout,
295 def _modify_dpdk_makefile(self):
297 Modifies DPDK makefile in Guest before compilation if needed
301 def _configure_copy_sources(self, dirname):
303 Mount shared directory and copy DPDK and l2fwd sources
305 # mount shared directory
306 self.execute_and_wait('umount /dev/sdb1')
307 self.execute_and_wait('rm -rf ' + S.getValue('GUEST_OVS_DPDK_DIR'))
308 self.execute_and_wait('mkdir -p ' + S.getValue('OVS_DPDK_SHARE'))
309 self.execute_and_wait('mount -o ro,iocharset=utf8 /dev/sdb1 ' +
310 S.getValue('OVS_DPDK_SHARE'))
311 self.execute_and_wait('mkdir -p ' + S.getValue('GUEST_OVS_DPDK_DIR'))
312 self.execute_and_wait('cp -r ' + os.path.join(S.getValue('OVS_DPDK_SHARE'), dirname) +
313 ' ' + S.getValue('GUEST_OVS_DPDK_DIR'))
314 self.execute_and_wait('umount /dev/sdb1')
316 def _configure_disable_firewall(self):
318 Disable firewall in VM
320 for iptables in ['iptables', 'ip6tables']:
322 for chain in ['INPUT', 'FORWARD', 'OUTPUT']:
323 self.execute_and_wait("{} -t filter -P {} ACCEPT".format(iptables, chain))
325 for chain in ['PREROUTING', 'INPUT', 'FORWARD', 'OUTPUT', 'POSTROUTING']:
326 self.execute_and_wait("{} -t mangle -P {} ACCEPT".format(iptables, chain))
328 for chain in ['PREROUTING', 'INPUT', 'OUTPUT', 'POSTROUTING']:
329 self.execute_and_wait("{} -t nat -P {} ACCEPT".format(iptables, chain))
331 # flush rules and delete chains created by user
332 for table in ['filter', 'mangle', 'nat']:
333 self.execute_and_wait("{} -t {} -F".format(iptables, table))
334 self.execute_and_wait("{} -t {} -X".format(iptables, table))
337 def _configure_testpmd(self):
339 Configure VM to perform L2 forwarding between NICs by DPDK's testpmd
341 self._configure_copy_sources('DPDK')
342 self._configure_disable_firewall()
344 # Guest images _should_ have 1024 hugepages by default,
345 # but just in case:'''
346 self.execute_and_wait('sysctl vm.nr_hugepages=1024')
349 self.execute_and_wait('mkdir -p /dev/hugepages')
350 self.execute_and_wait(
351 'mount -t hugetlbfs hugetlbfs /dev/hugepages')
353 # build and configure system for dpdk
354 self.execute_and_wait('cd ' + S.getValue('GUEST_OVS_DPDK_DIR') +
356 self.execute_and_wait('export CC=gcc')
357 self.execute_and_wait('export RTE_SDK=' +
358 S.getValue('GUEST_OVS_DPDK_DIR') + '/DPDK')
359 self.execute_and_wait('export RTE_TARGET=%s' % S.getValue('RTE_TARGET'))
361 # modify makefile if needed
362 self._modify_dpdk_makefile()
364 # disable network interfaces, so DPDK can take care of them
365 self.execute_and_wait('ifdown ' + self._net1)
366 self.execute_and_wait('ifdown ' + self._net2)
368 # build and insert igb_uio and rebind interfaces to it
369 self.execute_and_wait('make RTE_OUTPUT=$RTE_SDK/$RTE_TARGET -C '
370 '$RTE_SDK/lib/librte_eal/linuxapp/igb_uio')
371 self.execute_and_wait('modprobe uio')
372 self.execute_and_wait('insmod %s/kmod/igb_uio.ko' %
373 S.getValue('RTE_TARGET'))
374 self.execute_and_wait('./tools/dpdk*bind.py --status')
375 self.execute_and_wait(
376 './tools/dpdk*bind.py -u' ' ' +
377 S.getValue('GUEST_NET1_PCI_ADDRESS')[self._number] + ' ' +
378 S.getValue('GUEST_NET2_PCI_ADDRESS')[self._number])
379 self.execute_and_wait(
380 './tools/dpdk*bind.py -b igb_uio' ' ' +
381 S.getValue('GUEST_NET1_PCI_ADDRESS')[self._number] + ' ' +
382 S.getValue('GUEST_NET2_PCI_ADDRESS')[self._number])
383 self.execute_and_wait('./tools/dpdk*bind.py --status')
385 # build and run 'test-pmd'
386 self.execute_and_wait('cd ' + S.getValue('GUEST_OVS_DPDK_DIR') +
387 '/DPDK/app/test-pmd')
388 self.execute_and_wait('make clean')
389 self.execute_and_wait('make')
390 if int(S.getValue('GUEST_NIC_QUEUES')):
391 self.execute_and_wait(
392 './testpmd {} -n4 --socket-mem 512 --'.format(
393 S.getValue('GUEST_TESTPMD_CPU_MASK')) +
394 ' --burst=64 -i --txqflags=0xf00 ' +
395 '--nb-cores={} --rxq={} --txq={} '.format(
396 S.getValue('GUEST_TESTPMD_NB_CORES'),
397 S.getValue('GUEST_TESTPMD_TXQ'),
398 S.getValue('GUEST_TESTPMD_RXQ')) +
399 '--disable-hw-vlan', 60, "Done")
401 self.execute_and_wait(
402 './testpmd {} -n 4 --socket-mem 512 --'.format(
403 S.getValue('GUEST_TESTPMD_CPU_MASK')) +
404 ' --burst=64 -i --txqflags=0xf00 ' +
405 '--disable-hw-vlan', 60, "Done")
406 self.execute('set fwd ' + self._testpmd_fwd_mode, 1)
407 self.execute_and_wait('start', 20,
408 'TX RS bit threshold=.+ - TXQ flags=0xf00')
410 def _configure_l2fwd(self):
412 Configure VM to perform L2 forwarding between NICs by l2fwd module
414 if int(S.getValue('GUEST_NIC_QUEUES')):
415 self._set_multi_queue_nic()
416 self._configure_copy_sources('l2fwd')
417 self._configure_disable_firewall()
419 # build and configure system for l2fwd
420 self.execute_and_wait('cd ' + S.getValue('GUEST_OVS_DPDK_DIR') +
422 self.execute_and_wait('export CC=gcc')
424 self.execute_and_wait('make')
425 self.execute_and_wait('insmod ' + S.getValue('GUEST_OVS_DPDK_DIR') +
426 '/l2fwd' + '/l2fwd.ko net1=' + self._net1 +
427 ' net2=' + self._net2)
429 def _configure_linux_bridge(self):
431 Configure VM to perform L2 forwarding between NICs by linux bridge
433 if int(S.getValue('GUEST_NIC_QUEUES')):
434 self._set_multi_queue_nic()
435 self._configure_disable_firewall()
437 self.execute('ip addr add ' +
438 S.getValue('VANILLA_NIC1_IP_CIDR')[self._number] +
439 ' dev ' + self._net1)
440 self.execute('ip link set dev ' + self._net1 + ' up')
442 self.execute('ip addr add ' +
443 S.getValue('VANILLA_NIC2_IP_CIDR')[self._number] +
444 ' dev ' + self._net2)
445 self.execute('ip link set dev ' + self._net2 + ' up')
447 # configure linux bridge
448 self.execute('brctl addbr br0')
449 self.execute('brctl addif br0 ' + self._net1 + ' ' + self._net2)
450 self.execute('ip addr add ' +
451 S.getValue('VANILLA_BRIDGE_IP')[self._number] +
453 self.execute('ip link set dev br0 up')
455 # Add the arp entries for the IXIA ports and the bridge you are using.
456 # Use command line values if provided.
457 trafficgen_mac = get_test_param('vanilla_tgen_port1_mac',
458 S.getValue('VANILLA_TGEN_PORT1_MAC'))
459 trafficgen_ip = get_test_param('vanilla_tgen_port1_ip',
460 S.getValue('VANILLA_TGEN_PORT1_IP'))
462 self.execute('arp -s ' + trafficgen_ip + ' ' + trafficgen_mac)
464 trafficgen_mac = get_test_param('vanilla_tgen_port2_mac',
465 S.getValue('VANILLA_TGEN_PORT2_MAC'))
466 trafficgen_ip = get_test_param('vanilla_tgen_port2_ip',
467 S.getValue('VANILLA_TGEN_PORT2_IP'))
469 self.execute('arp -s ' + trafficgen_ip + ' ' + trafficgen_mac)
472 self.execute('sysctl -w net.ipv4.ip_forward=1')
474 # Controls source route verification
475 # 0 means no source validation
476 self.execute('sysctl -w net.ipv4.conf.all.rp_filter=0')
477 self.execute('sysctl -w net.ipv4.conf.' + self._net1 + '.rp_filter=0')
478 self.execute('sysctl -w net.ipv4.conf.' + self._net2 + '.rp_filter=0')
480 def _set_multi_queue_nic(self):
482 Enable multi-queue in guest kernel with ethool.
485 self.execute_and_wait('ethtool -L {} combined {}'.format(
486 self._net1, S.getValue('GUEST_NIC_QUEUES')))
487 self.execute_and_wait('ethtool -l {}'.format(self._net1))
488 self.execute_and_wait('ethtool -L {} combined {}'.format(
489 self._net2, S.getValue('GUEST_NIC_QUEUES')))
490 self.execute_and_wait('ethtool -l {}'.format(self._net2))