1 # Copyright 2015-2016 Intel Corporation.
3 # Licensed under the Apache License, Version 2.0 (the "License");
4 # you may not use this file except in compliance with the License.
5 # You may obtain a copy of the License at
7 # http://www.apache.org/licenses/LICENSE-2.0
9 # Unless required by applicable law or agreed to in writing, software
10 # distributed under the License is distributed on an "AS IS" BASIS,
11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 # See the License for the specific language governing permissions and
13 # limitations under the License.
15 """Automation of QEMU hypervisor for launching guests.
26 from conf import settings as S
27 from conf import get_test_param
28 from vnfs.vnf.vnf import IVnf
32 Abstract class for controling an instance of QEMU
38 class GuestCommandFilter(logging.Filter):
40 Filter out strings beginning with 'guestcmd :'.
42 def filter(self, record):
43 return record.getMessage().startswith(self.prefix)
47 Initialisation function.
49 super(IVnfQemu, self).__init__()
51 self._expect = S.getValue('GUEST_PROMPT_LOGIN')[self._number]
52 self._logger = logging.getLogger(__name__)
53 self._logfile = os.path.join(
54 S.getValue('LOG_DIR'),
55 S.getValue('LOG_FILE_QEMU')) + str(self._number)
56 self._timeout = S.getValue('GUEST_TIMEOUT')[self._number]
57 self._monitor = '%s/vm%dmonitor' % ('/tmp', self._number)
58 # read GUEST NICs configuration and use only defined NR of NICS
59 nics_nr = S.getValue('GUEST_NICS_NR')[self._number]
60 # and inform user about missconfiguration
62 raise RuntimeError('At least one VM NIC is mandotory, but {} '
63 'NICs are configured'.format(nics_nr))
64 elif nics_nr > 1 and nics_nr % 2:
65 nics_nr = int(nics_nr / 2) * 2
66 self._logger.warning('Odd number of NICs is configured, only '
67 '%s NICs will be used', nics_nr)
69 self._nics = S.getValue('GUEST_NICS')[self._number][:nics_nr]
71 # set guest loopback application based on VNF configuration
72 # cli option take precedence to config file values
73 self._guest_loopback = S.getValue('GUEST_LOOPBACK')[self._number]
75 self._testpmd_fwd_mode = S.getValue('GUEST_TESTPMD_FWD_MODE')[self._number]
76 # in case of SRIOV we must ensure, that MAC addresses are not swapped
77 if S.getValue('SRIOV_ENABLED') and self._testpmd_fwd_mode.startswith('mac') and \
78 not S.getValue('VNF').endswith('PciPassthrough'):
80 self._logger.info("SRIOV detected, forwarding mode of testpmd was changed from '%s' to '%s'",
81 self._testpmd_fwd_mode, 'io')
82 self._testpmd_fwd_mode = 'io'
84 name = 'Client%d' % self._number
85 vnc = ':%d' % self._number
86 # don't use taskset to affinize main qemu process; It causes hangup
87 # of 2nd VM in case of DPDK. It also slows down VM responsivnes.
88 self._cmd = ['sudo', '-E', S.getValue('TOOLS')['qemu-system'],
89 '-m', S.getValue('GUEST_MEMORY')[self._number],
90 '-smp', str(S.getValue('GUEST_SMP')[self._number]),
91 '-cpu', 'host,migratable=off',
92 '-drive', 'if={},file='.format(S.getValue(
93 'GUEST_BOOT_DRIVE_TYPE')[self._number]) +
94 S.getValue('GUEST_IMAGE')[self._number],
95 '-boot', 'c', '--enable-kvm',
96 '-monitor', 'unix:%s,server,nowait' % self._monitor,
98 'memory-backend-file,id=mem,size=' +
99 str(S.getValue('GUEST_MEMORY')[self._number]) + 'M,' +
100 'mem-path=' + S.getValue('HUGEPAGE_DIR') + ',share=on',
101 '-numa', 'node,memdev=mem -mem-prealloc',
102 '-nographic', '-vnc', str(vnc), '-name', name,
103 '-snapshot', '-net none', '-no-reboot',
105 'if=%s,format=raw,file=fat:rw:%s,snapshot=off' %
106 (S.getValue('GUEST_SHARED_DRIVE_TYPE')[self._number],
107 S.getValue('GUEST_SHARE_DIR')[self._number]),
109 self._configure_logging()
111 def _configure_logging(self):
115 self.GuestCommandFilter.prefix = self._log_prefix
117 logger = logging.getLogger()
118 cmd_logger = logging.FileHandler(
119 filename=os.path.join(S.getValue('LOG_DIR'),
120 S.getValue('LOG_FILE_GUEST_CMDS')) +
122 cmd_logger.setLevel(logging.DEBUG)
123 cmd_logger.addFilter(self.GuestCommandFilter())
124 logger.addHandler(cmd_logger)
130 Start QEMU instance, login and prepare for commands.
132 super(IVnfQemu, self).start()
133 if S.getValue('VNF_AFFINITIZATION_ON'):
136 if S.getValue('VSWITCH_VHOST_NET_AFFINITIZATION') and S.getValue(
137 'VNF') == 'QemuVirtioNet':
138 self._affinitize_vhost_net()
141 self._config_guest_loopback()
145 Stops VNF instance gracefully first.
148 # exit testpmd if needed
149 if self._guest_loopback == 'testpmd':
150 self.execute_and_wait('stop', 120, "Done")
151 self.execute_and_wait('quit', 120, "[bB]ye")
154 self.execute_and_wait('poweroff', 120, "Power down")
156 except pexpect.TIMEOUT:
159 # wait until qemu shutdowns
160 self._logger.debug('Wait for QEMU to terminate')
161 for dummy in range(30):
163 if not self.is_running():
166 # just for case that graceful shutdown failed
167 super(IVnfQemu, self).stop()
171 def _login(self, timeout=120):
173 Login to QEMU instance.
175 This can be used immediately after booting the machine, provided a
176 sufficiently long ``timeout`` is given.
178 :param timeout: Timeout to wait for login to complete.
182 # if no timeout was set, we likely started QEMU without waiting for it
183 # to boot. This being the case, we best check that it has finished
185 if not self._timeout:
186 self._expect_process(timeout=timeout)
188 self._child.sendline(S.getValue('GUEST_USERNAME')[self._number])
189 self._child.expect(S.getValue('GUEST_PROMPT_PASSWORD')[self._number], timeout=5)
190 self._child.sendline(S.getValue('GUEST_PASSWORD')[self._number])
192 self._expect_process(S.getValue('GUEST_PROMPT')[self._number], timeout=5)
194 def send_and_pass(self, cmd, timeout=30):
196 Send ``cmd`` and wait ``timeout`` seconds for it to pass.
198 :param cmd: Command to send to guest.
199 :param timeout: Time to wait for prompt before checking return code.
204 self.wait(S.getValue('GUEST_PROMPT')[self._number], timeout=timeout)
205 self.execute('echo $?')
206 self._child.expect('^0$', timeout=1) # expect a 0
207 self.wait(S.getValue('GUEST_PROMPT')[self._number], timeout=timeout)
209 def _affinitize(self):
211 Affinitize the SMP cores of a QEMU instance.
213 This is a bit of a hack. The 'socat' utility is used to
214 interact with the QEMU HMP. This is necessary due to the lack
215 of QMP in older versions of QEMU, like v1.6.2. In future
216 releases, this should be replaced with calls to libvirt or
217 another Python-QEMU wrapper library.
221 thread_id = (r'.* CPU #%d: .* thread_id=(\d+)')
223 self._logger.info('Affinitizing guest...')
225 cur_locale = locale.getdefaultlocale()[1]
226 proc = subprocess.Popen(
227 ('echo', 'info cpus'), stdout=subprocess.PIPE)
228 output = subprocess.check_output(
229 ('sudo', 'socat', '-', 'UNIX-CONNECT:%s' % self._monitor),
233 for cpu in range(0, int(S.getValue('GUEST_SMP')[self._number])):
235 for line in output.decode(cur_locale).split('\n'):
236 match = re.search(thread_id % cpu, line)
238 self._affinitize_pid(
239 S.getValue('GUEST_CORE_BINDING')[self._number][cpu],
244 self._logger.error('Failed to affinitize guest core #%d. Could'
245 ' not parse tid.', cpu)
247 def _affinitize_vhost_net(self):
249 Affinitize the vhost net threads for Vanilla OVS and guest nic queues.
253 self._logger.info('Affinitizing VHOST Net threads.')
254 args1 = ['pgrep', 'vhost-']
255 process1 = subprocess.Popen(args1, stdout=subprocess.PIPE,
257 out = process1.communicate()[0]
258 processes = out.decode(locale.getdefaultlocale()[1]).split('\n')
259 if processes[-1] == '':
260 processes.pop() # pgrep may return an extra line with no data
261 self._logger.info('Found %s vhost net threads...', len(processes))
263 cpumap = S.getValue('VSWITCH_VHOST_CPU_MAP')
265 for proc in processes:
266 self._affinitize_pid(cpumap[mapcount], proc)
268 if mapcount + 1 > len(cpumap):
269 # Not enough cpus were given in the mapping to cover all the
270 # threads on a 1 to 1 ratio with cpus so reset the list counter
274 def _config_guest_loopback(self):
276 Configure VM to run VNF, e.g. port forwarding application based on the configuration
278 if self._guest_loopback == 'testpmd':
280 self._configure_testpmd()
281 elif self._guest_loopback == 'l2fwd':
283 self._configure_l2fwd()
284 elif self._guest_loopback == 'linux_bridge':
286 self._configure_linux_bridge()
287 elif self._guest_loopback != 'buildin':
288 self._logger.error('Unsupported guest loopback method "%s" was specified. Option'
289 ' "buildin" will be used as a fallback.', self._guest_loopback)
291 def wait(self, prompt=None, timeout=30):
293 prompt = S.getValue('GUEST_PROMPT')[self._number]
294 super(IVnfQemu, self).wait(prompt=prompt, timeout=timeout)
296 def execute_and_wait(self, cmd, timeout=30, prompt=None):
298 prompt = S.getValue('GUEST_PROMPT')[self._number]
299 super(IVnfQemu, self).execute_and_wait(cmd, timeout=timeout,
302 def _modify_dpdk_makefile(self):
304 Modifies DPDK makefile in Guest before compilation if needed
308 def _configure_copy_sources(self, dirname):
310 Mount shared directory and copy DPDK and l2fwd sources
312 # mount shared directory
313 self.execute_and_wait('umount /dev/sdb1')
314 self.execute_and_wait('rm -rf ' + S.getValue('GUEST_OVS_DPDK_DIR')[self._number])
315 self.execute_and_wait('mkdir -p ' + S.getValue('GUEST_OVS_DPDK_SHARE')[self._number])
316 self.execute_and_wait('mount -o ro,iocharset=utf8 /dev/sdb1 ' +
317 S.getValue('GUEST_OVS_DPDK_SHARE')[self._number])
318 self.execute_and_wait('mkdir -p ' + S.getValue('GUEST_OVS_DPDK_DIR')[self._number])
319 self.execute_and_wait('cp -r ' + os.path.join(S.getValue('GUEST_OVS_DPDK_SHARE')[self._number], dirname) +
320 ' ' + S.getValue('GUEST_OVS_DPDK_DIR')[self._number])
321 self.execute_and_wait('umount /dev/sdb1')
323 def _configure_disable_firewall(self):
325 Disable firewall in VM
327 for iptables in ['iptables', 'ip6tables']:
329 for chain in ['INPUT', 'FORWARD', 'OUTPUT']:
330 self.execute_and_wait("{} -t filter -P {} ACCEPT".format(iptables, chain))
332 for chain in ['PREROUTING', 'INPUT', 'FORWARD', 'OUTPUT', 'POSTROUTING']:
333 self.execute_and_wait("{} -t mangle -P {} ACCEPT".format(iptables, chain))
335 for chain in ['PREROUTING', 'INPUT', 'OUTPUT', 'POSTROUTING']:
336 self.execute_and_wait("{} -t nat -P {} ACCEPT".format(iptables, chain))
338 # flush rules and delete chains created by user
339 for table in ['filter', 'mangle', 'nat']:
340 self.execute_and_wait("{} -t {} -F".format(iptables, table))
341 self.execute_and_wait("{} -t {} -X".format(iptables, table))
344 def _configure_testpmd(self):
346 Configure VM to perform L2 forwarding between NICs by DPDK's testpmd
348 self._configure_copy_sources('DPDK')
349 self._configure_disable_firewall()
351 # Guest images _should_ have 1024 hugepages by default,
352 # but just in case:'''
353 self.execute_and_wait('sysctl vm.nr_hugepages={}'.format(S.getValue('GUEST_HUGEPAGES_NR')[self._number]))
356 self.execute_and_wait('mkdir -p /dev/hugepages')
357 self.execute_and_wait(
358 'mount -t hugetlbfs hugetlbfs /dev/hugepages')
360 # build and configure system for dpdk
361 self.execute_and_wait('cd ' + S.getValue('GUEST_OVS_DPDK_DIR')[self._number] +
363 self.execute_and_wait('export CC=gcc')
364 self.execute_and_wait('export RTE_SDK=' +
365 S.getValue('GUEST_OVS_DPDK_DIR')[self._number] + '/DPDK')
366 self.execute_and_wait('export RTE_TARGET=%s' % S.getValue('RTE_TARGET'))
368 # modify makefile if needed
369 self._modify_dpdk_makefile()
371 # disable network interfaces, so DPDK can take care of them
372 for nic in self._nics:
373 self.execute_and_wait('ifdown ' + nic['device'])
375 # build and insert igb_uio and rebind interfaces to it
376 self.execute_and_wait('make RTE_OUTPUT=$RTE_SDK/$RTE_TARGET -C '
377 '$RTE_SDK/lib/librte_eal/linuxapp/igb_uio')
378 self.execute_and_wait('modprobe uio')
379 self.execute_and_wait('insmod %s/kmod/igb_uio.ko' %
380 S.getValue('RTE_TARGET'))
381 self.execute_and_wait('./tools/dpdk*bind.py --status')
382 pci_list = ' '.join([nic['pci'] for nic in self._nics])
383 self.execute_and_wait('./tools/dpdk*bind.py -u ' + pci_list)
384 self.execute_and_wait('./tools/dpdk*bind.py -b igb_uio ' + pci_list)
385 self.execute_and_wait('./tools/dpdk*bind.py --status')
387 # build and run 'test-pmd'
388 self.execute_and_wait('cd ' + S.getValue('GUEST_OVS_DPDK_DIR')[self._number] +
389 '/DPDK/app/test-pmd')
390 self.execute_and_wait('make clean')
391 self.execute_and_wait('make')
392 if int(S.getValue('GUEST_NIC_QUEUES')[self._number]):
393 self.execute_and_wait(
394 './testpmd {} -n4 --socket-mem 512 --'.format(
395 S.getValue('GUEST_TESTPMD_CPU_MASK')[self._number]) +
396 ' --burst=64 -i --txqflags=0xf00 ' +
397 '--nb-cores={} --rxq={} --txq={} '.format(
398 S.getValue('GUEST_TESTPMD_NB_CORES')[self._number],
399 S.getValue('GUEST_TESTPMD_TXQ')[self._number],
400 S.getValue('GUEST_TESTPMD_RXQ')[self._number]) +
401 '--disable-hw-vlan', 60, "Done")
403 self.execute_and_wait(
404 './testpmd {} -n 4 --socket-mem 512 --'.format(
405 S.getValue('GUEST_TESTPMD_CPU_MASK')[self._number]) +
406 ' --burst=64 -i --txqflags=0xf00 ' +
407 '--disable-hw-vlan', 60, "Done")
408 self.execute('set fwd ' + self._testpmd_fwd_mode, 1)
409 self.execute_and_wait('start', 20,
410 'TX RS bit threshold=.+ - TXQ flags=0xf00')
412 def _configure_l2fwd(self):
414 Configure VM to perform L2 forwarding between NICs by l2fwd module
416 if int(S.getValue('GUEST_NIC_QUEUES')[self._number]):
417 self._set_multi_queue_nic()
418 self._configure_copy_sources('l2fwd')
419 self._configure_disable_firewall()
421 # configure all interfaces
422 for nic in self._nics:
423 self.execute('ip addr add ' +
424 nic['ip'] + ' dev ' + nic['device'])
425 self.execute('ip link set dev ' + nic['device'] + ' up')
427 # build and configure system for l2fwd
428 self.execute_and_wait('cd ' + S.getValue('GUEST_OVS_DPDK_DIR')[self._number] +
430 self.execute_and_wait('export CC=gcc')
432 self.execute_and_wait('make')
433 if len(self._nics) == 2:
434 self.execute_and_wait('insmod ' + S.getValue('GUEST_OVS_DPDK_DIR')[self._number] +
435 '/l2fwd' + '/l2fwd.ko net1=' + self._nics[0]['device'] +
436 ' net2=' + self._nics[1]['device'])
438 raise RuntimeError('l2fwd can forward only between 2 NICs, but {} NICs are '
439 'configured inside GUEST'.format(len(self._nics)))
441 def _configure_linux_bridge(self):
443 Configure VM to perform L2 forwarding between NICs by linux bridge
445 if int(S.getValue('GUEST_NIC_QUEUES')[self._number]):
446 self._set_multi_queue_nic()
447 self._configure_disable_firewall()
449 # configure linux bridge
450 self.execute('brctl addbr br0')
452 # add all NICs into the bridge
453 for nic in self._nics:
454 self.execute('ip addr add ' +
455 nic['ip'] + ' dev ' + nic['device'])
456 self.execute('ip link set dev ' + nic['device'] + ' up')
457 self.execute('brctl addif br0 ' + nic['device'])
459 self.execute('ip addr add ' +
460 S.getValue('GUEST_BRIDGE_IP')[self._number] +
462 self.execute('ip link set dev br0 up')
464 # Add the arp entries for the IXIA ports and the bridge you are using.
465 # Use command line values if provided.
466 trafficgen_mac = get_test_param('vanilla_tgen_port1_mac',
467 S.getValue('VANILLA_TGEN_PORT1_MAC'))
468 trafficgen_ip = get_test_param('vanilla_tgen_port1_ip',
469 S.getValue('VANILLA_TGEN_PORT1_IP'))
471 self.execute('arp -s ' + trafficgen_ip + ' ' + trafficgen_mac)
473 trafficgen_mac = get_test_param('vanilla_tgen_port2_mac',
474 S.getValue('VANILLA_TGEN_PORT2_MAC'))
475 trafficgen_ip = get_test_param('vanilla_tgen_port2_ip',
476 S.getValue('VANILLA_TGEN_PORT2_IP'))
478 self.execute('arp -s ' + trafficgen_ip + ' ' + trafficgen_mac)
481 self.execute('sysctl -w net.ipv4.ip_forward=1')
483 # Controls source route verification
484 # 0 means no source validation
485 self.execute('sysctl -w net.ipv4.conf.all.rp_filter=0')
486 for nic in self._nics:
487 self.execute('sysctl -w net.ipv4.conf.' + nic['device'] + '.rp_filter=0')
489 def _set_multi_queue_nic(self):
491 Enable multi-queue in guest kernel with ethool.
494 for nic in self._nics:
495 self.execute_and_wait('ethtool -L {} combined {}'.format(
496 nic['device'], S.getValue('GUEST_NIC_QUEUES')[self._number]))
497 self.execute_and_wait('ethtool -l {}'.format(nic['device']))