1 # Copyright 2015-2016 Intel Corporation.
3 # Licensed under the Apache License, Version 2.0 (the "License");
4 # you may not use this file except in compliance with the License.
5 # You may obtain a copy of the License at
7 # http://www.apache.org/licenses/LICENSE-2.0
9 # Unless required by applicable law or agreed to in writing, software
10 # distributed under the License is distributed on an "AS IS" BASIS,
11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 # See the License for the specific language governing permissions and
13 # limitations under the License.
15 """Automation of QEMU hypervisor for launching vhost-cuse enabled guests.
26 from conf import settings as S
27 from conf import get_test_param
28 from vnfs.vnf.vnf import IVnf
32 Abstract class for controling an instance of QEMU
35 _expect = S.getValue('GUEST_PROMPT_LOGIN')
38 class GuestCommandFilter(logging.Filter):
40 Filter out strings beginning with 'guestcmd :'.
42 def filter(self, record):
43 return record.getMessage().startswith(self.prefix)
47 Initialisation function.
49 super(IVnfQemu, self).__init__()
50 self._logger = logging.getLogger(__name__)
51 self._logfile = os.path.join(
52 S.getValue('LOG_DIR'),
53 S.getValue('LOG_FILE_QEMU')) + str(self._number)
54 self._timeout = S.getValue('GUEST_TIMEOUT')[self._number]
55 self._monitor = '%s/vm%dmonitor' % ('/tmp', self._number)
56 self._net1 = get_test_param('guest_nic1_name', None)
57 if self._net1 == None:
58 self._net1 = S.getValue('GUEST_NIC1_NAME')[self._number]
60 self._net1 = self._net1.split(',')[self._number]
61 self._net2 = get_test_param('guest_nic2_name', None)
62 if self._net2 == None:
63 self._net2 = S.getValue('GUEST_NIC2_NAME')[self._number]
65 self._net2 = self._net2.split(',')[self._number]
67 # set guest loopback application based on VNF configuration
68 # cli option take precedence to config file values
69 self._guest_loopback = S.getValue('GUEST_LOOPBACK')[self._number]
71 self._testpmd_fwd_mode = S.getValue('GUEST_TESTPMD_FWD_MODE')
72 # in case of SRIOV we must ensure, that MAC addresses are not swapped
73 if S.getValue('SRIOV_ENABLED') and self._testpmd_fwd_mode.startswith('mac') and \
74 not S.getValue('VNF').endswith('PciPassthrough'):
76 self._logger.info("SRIOV detected, forwarding mode of testpmd was changed from '%s' to '%s'",
77 self._testpmd_fwd_mode, 'io')
78 self._testpmd_fwd_mode = 'io'
80 name = 'Client%d' % self._number
81 vnc = ':%d' % self._number
82 # don't use taskset to affinize main qemu process; It causes hangup
83 # of 2nd VM in case of DPDK. It also slows down VM responsivnes.
84 self._cmd = ['sudo', '-E', S.getValue('QEMU_BIN'),
85 '-m', S.getValue('GUEST_MEMORY')[self._number],
86 '-smp', str(S.getValue('GUEST_SMP')[self._number]),
87 '-cpu', 'host,migratable=off',
88 '-drive', 'if=scsi,file=' +
89 S.getValue('GUEST_IMAGE')[self._number],
90 '-boot', 'c', '--enable-kvm',
91 '-monitor', 'unix:%s,server,nowait' % self._monitor,
93 'memory-backend-file,id=mem,size=' +
94 str(S.getValue('GUEST_MEMORY')[self._number]) + 'M,' +
95 'mem-path=' + S.getValue('HUGEPAGE_DIR') + ',share=on',
96 '-numa', 'node,memdev=mem -mem-prealloc',
97 '-nographic', '-vnc', str(vnc), '-name', name,
98 '-snapshot', '-net none', '-no-reboot',
100 'if=scsi,format=raw,file=fat:rw:%s,snapshot=off' %
101 S.getValue('GUEST_SHARE_DIR')[self._number],
103 self._configure_logging()
105 def _configure_logging(self):
109 self.GuestCommandFilter.prefix = self._log_prefix
111 logger = logging.getLogger()
112 cmd_logger = logging.FileHandler(
113 filename=os.path.join(S.getValue('LOG_DIR'),
114 S.getValue('LOG_FILE_GUEST_CMDS')) +
116 cmd_logger.setLevel(logging.DEBUG)
117 cmd_logger.addFilter(self.GuestCommandFilter())
118 logger.addHandler(cmd_logger)
124 Start QEMU instance, login and prepare for commands.
126 super(IVnfQemu, self).start()
127 if S.getValue('VNF_AFFINITIZATION_ON'):
131 self._config_guest_loopback()
135 Stops VNF instance gracefully first.
138 # exit testpmd if needed
139 if self._guest_loopback == 'testpmd':
140 self.execute_and_wait('stop', 120, "Done")
141 self.execute_and_wait('quit', 120, "[bB]ye")
144 self.execute_and_wait('poweroff', 120, "Power down")
146 except pexpect.TIMEOUT:
149 # wait until qemu shutdowns
150 self._logger.debug('Wait for QEMU to terminate')
151 for dummy in range(30):
153 if not self.is_running():
156 # just for case that graceful shutdown failed
157 super(IVnfQemu, self).stop()
161 def _login(self, timeout=120):
163 Login to QEMU instance.
165 This can be used immediately after booting the machine, provided a
166 sufficiently long ``timeout`` is given.
168 :param timeout: Timeout to wait for login to complete.
172 # if no timeout was set, we likely started QEMU without waiting for it
173 # to boot. This being the case, we best check that it has finished
175 if not self._timeout:
176 self._expect_process(timeout=timeout)
178 self._child.sendline(S.getValue('GUEST_USERNAME'))
179 self._child.expect(S.getValue('GUEST_PROMPT_PASSWORD'), timeout=5)
180 self._child.sendline(S.getValue('GUEST_PASSWORD'))
182 self._expect_process(S.getValue('GUEST_PROMPT'), timeout=5)
184 def send_and_pass(self, cmd, timeout=30):
186 Send ``cmd`` and wait ``timeout`` seconds for it to pass.
188 :param cmd: Command to send to guest.
189 :param timeout: Time to wait for prompt before checking return code.
194 self.wait(S.getValue('GUEST_PROMPT'), timeout=timeout)
195 self.execute('echo $?')
196 self._child.expect('^0$', timeout=1) # expect a 0
197 self.wait(S.getValue('GUEST_PROMPT'), timeout=timeout)
199 def _affinitize(self):
201 Affinitize the SMP cores of a QEMU instance.
203 This is a bit of a hack. The 'socat' utility is used to
204 interact with the QEMU HMP. This is necessary due to the lack
205 of QMP in older versions of QEMU, like v1.6.2. In future
206 releases, this should be replaced with calls to libvirt or
207 another Python-QEMU wrapper library.
211 thread_id = (r'.* CPU #%d: .* thread_id=(\d+)')
213 self._logger.info('Affinitizing guest...')
215 cur_locale = locale.getdefaultlocale()[1]
216 proc = subprocess.Popen(
217 ('echo', 'info cpus'), stdout=subprocess.PIPE)
218 output = subprocess.check_output(
219 ('sudo', 'socat', '-', 'UNIX-CONNECT:%s' % self._monitor),
223 for cpu in range(0, int(S.getValue('GUEST_SMP')[self._number])):
225 for line in output.decode(cur_locale).split('\n'):
226 match = re.search(thread_id % cpu, line)
228 self._affinitize_pid(
229 S.getValue('GUEST_CORE_BINDING')[self._number][cpu],
234 self._logger.error('Failed to affinitize guest core #%d. Could'
235 ' not parse tid.', cpu)
237 def _config_guest_loopback(self):
239 Configure VM to run VNF, e.g. port forwarding application based on the configuration
241 if self._guest_loopback == 'testpmd':
243 self._configure_testpmd()
244 elif self._guest_loopback == 'l2fwd':
246 self._configure_l2fwd()
247 elif self._guest_loopback == 'linux_bridge':
249 self._configure_linux_bridge()
250 elif self._guest_loopback != 'buildin':
251 self._logger.error('Unsupported guest loopback method "%s" was specified. Option'
252 ' "buildin" will be used as a fallback.', self._guest_loopback)
254 def wait(self, prompt=S.getValue('GUEST_PROMPT'), timeout=30):
255 super(IVnfQemu, self).wait(prompt=prompt, timeout=timeout)
257 def execute_and_wait(self, cmd, timeout=30,
258 prompt=S.getValue('GUEST_PROMPT')):
259 super(IVnfQemu, self).execute_and_wait(cmd, timeout=timeout,
262 def _modify_dpdk_makefile(self):
264 Modifies DPDK makefile in Guest before compilation if needed
268 def _configure_copy_sources(self, dirname):
270 Mount shared directory and copy DPDK and l2fwd sources
272 # mount shared directory
273 self.execute_and_wait('umount /dev/sdb1')
274 self.execute_and_wait('rm -rf ' + S.getValue('GUEST_OVS_DPDK_DIR'))
275 self.execute_and_wait('mkdir -p ' + S.getValue('OVS_DPDK_SHARE'))
276 self.execute_and_wait('mount -o ro,iocharset=utf8 /dev/sdb1 ' +
277 S.getValue('OVS_DPDK_SHARE'))
278 self.execute_and_wait('mkdir -p ' + S.getValue('GUEST_OVS_DPDK_DIR'))
279 self.execute_and_wait('cp -r ' + os.path.join(S.getValue('OVS_DPDK_SHARE'), dirname) +
280 ' ' + S.getValue('GUEST_OVS_DPDK_DIR'))
281 self.execute_and_wait('umount /dev/sdb1')
283 def _configure_disable_firewall(self):
285 Disable firewall in VM
287 for iptables in ['iptables', 'ip6tables']:
289 for chain in ['INPUT', 'FORWARD', 'OUTPUT']:
290 self.execute_and_wait("{} -t filter -P {} ACCEPT".format(iptables, chain))
292 for chain in ['PREROUTING', 'INPUT', 'FORWARD', 'OUTPUT', 'POSTROUTING']:
293 self.execute_and_wait("{} -t mangle -P {} ACCEPT".format(iptables, chain))
295 for chain in ['PREROUTING', 'INPUT', 'OUTPUT', 'POSTROUTING']:
296 self.execute_and_wait("{} -t nat -P {} ACCEPT".format(iptables, chain))
298 # flush rules and delete chains created by user
299 for table in ['filter', 'mangle', 'nat']:
300 self.execute_and_wait("{} -t {} -F".format(iptables, table))
301 self.execute_and_wait("{} -t {} -X".format(iptables, table))
304 def _configure_testpmd(self):
306 Configure VM to perform L2 forwarding between NICs by DPDK's testpmd
308 self._configure_copy_sources('DPDK')
309 self._configure_disable_firewall()
311 # Guest images _should_ have 1024 hugepages by default,
312 # but just in case:'''
313 self.execute_and_wait('sysctl vm.nr_hugepages=1024')
316 self.execute_and_wait('mkdir -p /dev/hugepages')
317 self.execute_and_wait(
318 'mount -t hugetlbfs hugetlbfs /dev/hugepages')
320 # build and configure system for dpdk
321 self.execute_and_wait('cd ' + S.getValue('GUEST_OVS_DPDK_DIR') +
323 self.execute_and_wait('export CC=gcc')
324 self.execute_and_wait('export RTE_SDK=' +
325 S.getValue('GUEST_OVS_DPDK_DIR') + '/DPDK')
326 self.execute_and_wait('export RTE_TARGET=%s' % S.getValue('RTE_TARGET'))
328 # modify makefile if needed
329 self._modify_dpdk_makefile()
331 # disable network interfaces, so DPDK can take care of them
332 self.execute_and_wait('ifdown ' + self._net1)
333 self.execute_and_wait('ifdown ' + self._net2)
335 # build and insert igb_uio and rebind interfaces to it
336 self.execute_and_wait('make RTE_OUTPUT=$RTE_SDK/$RTE_TARGET -C '
337 '$RTE_SDK/lib/librte_eal/linuxapp/igb_uio')
338 self.execute_and_wait('modprobe uio')
339 self.execute_and_wait('insmod %s/kmod/igb_uio.ko' %
340 S.getValue('RTE_TARGET'))
341 self.execute_and_wait('./tools/dpdk_nic_bind.py --status')
342 self.execute_and_wait(
343 './tools/dpdk_nic_bind.py -u' ' ' +
344 S.getValue('GUEST_NET1_PCI_ADDRESS')[self._number] + ' ' +
345 S.getValue('GUEST_NET2_PCI_ADDRESS')[self._number])
346 self.execute_and_wait(
347 './tools/dpdk_nic_bind.py -b igb_uio' ' ' +
348 S.getValue('GUEST_NET1_PCI_ADDRESS')[self._number] + ' ' +
349 S.getValue('GUEST_NET2_PCI_ADDRESS')[self._number])
350 self.execute_and_wait('./tools/dpdk_nic_bind.py --status')
352 # build and run 'test-pmd'
353 self.execute_and_wait('cd ' + S.getValue('GUEST_OVS_DPDK_DIR') +
354 '/DPDK/app/test-pmd')
355 self.execute_and_wait('make clean')
356 self.execute_and_wait('make')
357 if int(S.getValue('GUEST_NIC_QUEUES')):
358 self.execute_and_wait(
359 './testpmd {} -n4 --socket-mem 512 --'.format(
360 S.getValue('GUEST_TESTPMD_CPU_MASK')) +
361 ' --burst=64 -i --txqflags=0xf00 ' +
362 '--nb-cores={} --rxq={} --txq={} '.format(
363 S.getValue('GUEST_TESTPMD_NB_CORES'),
364 S.getValue('GUEST_TESTPMD_TXQ'),
365 S.getValue('GUEST_TESTPMD_RXQ')) +
366 '--disable-hw-vlan', 60, "Done")
368 self.execute_and_wait(
369 './testpmd {} -n 4 --socket-mem 512 --'.format(
370 S.getValue('GUEST_TESTPMD_CPU_MASK')) +
371 ' --burst=64 -i --txqflags=0xf00 ' +
372 '--disable-hw-vlan', 60, "Done")
373 self.execute('set fwd ' + self._testpmd_fwd_mode, 1)
374 self.execute_and_wait('start', 20,
375 'TX RS bit threshold=.+ - TXQ flags=0xf00')
377 def _configure_l2fwd(self):
379 Configure VM to perform L2 forwarding between NICs by l2fwd module
381 self._configure_copy_sources('l2fwd')
382 self._configure_disable_firewall()
384 # build and configure system for l2fwd
385 self.execute_and_wait('cd ' + S.getValue('GUEST_OVS_DPDK_DIR') +
387 self.execute_and_wait('export CC=gcc')
389 self.execute_and_wait('make')
390 self.execute_and_wait('insmod ' + S.getValue('GUEST_OVS_DPDK_DIR') +
391 '/l2fwd' + '/l2fwd.ko net1=' + self._net1 +
392 ' net2=' + self._net2)
394 def _configure_linux_bridge(self):
396 Configure VM to perform L2 forwarding between NICs by linux bridge
398 self._configure_disable_firewall()
400 self.execute('ip addr add ' +
401 S.getValue('VANILLA_NIC1_IP_CIDR')[self._number] +
402 ' dev ' + self._net1)
403 self.execute('ip link set dev ' + self._net1 + ' up')
405 self.execute('ip addr add ' +
406 S.getValue('VANILLA_NIC2_IP_CIDR')[self._number] +
407 ' dev ' + self._net2)
408 self.execute('ip link set dev ' + self._net2 + ' up')
410 # configure linux bridge
411 self.execute('brctl addbr br0')
412 self.execute('brctl addif br0 ' + self._net1 + ' ' + self._net2)
413 self.execute('ip addr add ' +
414 S.getValue('VANILLA_BRIDGE_IP')[self._number] +
416 self.execute('ip link set dev br0 up')
418 # Add the arp entries for the IXIA ports and the bridge you are using.
419 # Use command line values if provided.
420 trafficgen_mac = get_test_param('vanilla_tgen_port1_mac',
421 S.getValue('VANILLA_TGEN_PORT1_MAC'))
422 trafficgen_ip = get_test_param('vanilla_tgen_port1_ip',
423 S.getValue('VANILLA_TGEN_PORT1_IP'))
425 self.execute('arp -s ' + trafficgen_ip + ' ' + trafficgen_mac)
427 trafficgen_mac = get_test_param('vanilla_tgen_port2_mac',
428 S.getValue('VANILLA_TGEN_PORT2_MAC'))
429 trafficgen_ip = get_test_param('vanilla_tgen_port2_ip',
430 S.getValue('VANILLA_TGEN_PORT2_IP'))
432 self.execute('arp -s ' + trafficgen_ip + ' ' + trafficgen_mac)
435 self.execute('sysctl -w net.ipv4.ip_forward=1')
437 # Controls source route verification
438 # 0 means no source validation
439 self.execute('sysctl -w net.ipv4.conf.all.rp_filter=0')
440 self.execute('sysctl -w net.ipv4.conf.' + self._net1 + '.rp_filter=0')
441 self.execute('sysctl -w net.ipv4.conf.' + self._net2 + '.rp_filter=0')