1 # Copyright 2015-2016 Intel Corporation.
3 # Licensed under the Apache License, Version 2.0 (the "License");
4 # you may not use this file except in compliance with the License.
5 # You may obtain a copy of the License at
7 # http://www.apache.org/licenses/LICENSE-2.0
9 # Unless required by applicable law or agreed to in writing, software
10 # distributed under the License is distributed on an "AS IS" BASIS,
11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 # See the License for the specific language governing permissions and
13 # limitations under the License.
15 """Automation of QEMU hypervisor for launching vhost-cuse enabled guests.
25 from conf import settings as S
26 from conf import get_test_param
27 from vnfs.vnf.vnf import IVnf
31 Abstract class for controling an instance of QEMU
34 _expect = S.getValue('GUEST_PROMPT_LOGIN')
37 class GuestCommandFilter(logging.Filter):
39 Filter out strings beginning with 'guestcmd :'.
41 def filter(self, record):
42 return record.getMessage().startswith(self.prefix)
46 Initialisation function.
48 super(IVnfQemu, self).__init__()
49 self._logger = logging.getLogger(__name__)
50 self._logfile = os.path.join(
51 S.getValue('LOG_DIR'),
52 S.getValue('LOG_FILE_QEMU')) + str(self._number)
53 self._timeout = S.getValue('GUEST_TIMEOUT')[self._number]
54 self._monitor = '%s/vm%dmonitor' % ('/tmp', self._number)
55 self._net1 = get_test_param('guest_nic1_name', None)
56 if self._net1 == None:
57 self._net1 = S.getValue('GUEST_NIC1_NAME')[self._number]
59 self._net1 = self._net1.split(',')[self._number]
60 self._net2 = get_test_param('guest_nic2_name', None)
61 if self._net2 == None:
62 self._net2 = S.getValue('GUEST_NIC2_NAME')[self._number]
64 self._net2 = self._net2.split(',')[self._number]
66 # set guest loopback application based on VNF configuration
67 # cli option take precedence to config file values
68 self._guest_loopback = S.getValue('GUEST_LOOPBACK')[self._number]
70 self._testpmd_fwd_mode = S.getValue('GUEST_TESTPMD_FWD_MODE')
71 # in case of SRIOV we must ensure, that MAC addresses are not swapped
72 if S.getValue('SRIOV_ENABLED') and self._testpmd_fwd_mode.startswith('mac') and \
73 not S.getValue('VNF').endswith('PciPassthrough'):
75 self._logger.info("SRIOV detected, forwarding mode of testpmd was changed from '%s' to '%s'",
76 self._testpmd_fwd_mode, 'io')
77 self._testpmd_fwd_mode = 'io'
79 name = 'Client%d' % self._number
80 vnc = ':%d' % self._number
81 # don't use taskset to affinize main qemu process; It causes hangup
82 # of 2nd VM in case of DPDK. It also slows down VM responsivnes.
83 self._cmd = ['sudo', '-E', S.getValue('QEMU_BIN'),
84 '-m', S.getValue('GUEST_MEMORY')[self._number],
85 '-smp', str(S.getValue('GUEST_SMP')[self._number]),
86 '-cpu', 'host,migratable=off',
87 '-drive', 'if=scsi,file=' +
88 S.getValue('GUEST_IMAGE')[self._number],
89 '-boot', 'c', '--enable-kvm',
90 '-monitor', 'unix:%s,server,nowait' % self._monitor,
92 'memory-backend-file,id=mem,size=' +
93 str(S.getValue('GUEST_MEMORY')[self._number]) + 'M,' +
94 'mem-path=' + S.getValue('HUGEPAGE_DIR') + ',share=on',
95 '-numa', 'node,memdev=mem -mem-prealloc',
96 '-nographic', '-vnc', str(vnc), '-name', name,
97 '-snapshot', '-net none', '-no-reboot',
99 'if=scsi,format=raw,file=fat:rw:%s,snapshot=off' %
100 S.getValue('GUEST_SHARE_DIR')[self._number],
102 self._configure_logging()
104 def _configure_logging(self):
108 self.GuestCommandFilter.prefix = self._log_prefix
110 logger = logging.getLogger()
111 cmd_logger = logging.FileHandler(
112 filename=os.path.join(S.getValue('LOG_DIR'),
113 S.getValue('LOG_FILE_GUEST_CMDS')) +
115 cmd_logger.setLevel(logging.DEBUG)
116 cmd_logger.addFilter(self.GuestCommandFilter())
117 logger.addHandler(cmd_logger)
123 Start QEMU instance, login and prepare for commands.
125 super(IVnfQemu, self).start()
126 if S.getValue('VNF_AFFINITIZATION_ON'):
130 self._config_guest_loopback()
134 Stops VNF instance gracefully first.
136 # exit testpmd if needed
137 if self._guest_loopback == 'testpmd':
138 self.execute_and_wait('stop', 120, "Done")
139 self.execute_and_wait('quit', 120, "bye")
142 self.execute_and_wait('poweroff', 120, "Power down")
143 # VM OS is off, but wait until qemu shutdowns
146 # just for case that graceful shutdown failed
147 super(IVnfQemu, self).stop()
151 def _login(self, timeout=120):
153 Login to QEMU instance.
155 This can be used immediately after booting the machine, provided a
156 sufficiently long ``timeout`` is given.
158 :param timeout: Timeout to wait for login to complete.
162 # if no timeout was set, we likely started QEMU without waiting for it
163 # to boot. This being the case, we best check that it has finished
165 if not self._timeout:
166 self._expect_process(timeout=timeout)
168 self._child.sendline(S.getValue('GUEST_USERNAME'))
169 self._child.expect(S.getValue('GUEST_PROMPT_PASSWORD'), timeout=5)
170 self._child.sendline(S.getValue('GUEST_PASSWORD'))
172 self._expect_process(S.getValue('GUEST_PROMPT'), timeout=5)
174 def send_and_pass(self, cmd, timeout=30):
176 Send ``cmd`` and wait ``timeout`` seconds for it to pass.
178 :param cmd: Command to send to guest.
179 :param timeout: Time to wait for prompt before checking return code.
184 self.wait(S.getValue('GUEST_PROMPT'), timeout=timeout)
185 self.execute('echo $?')
186 self._child.expect('^0$', timeout=1) # expect a 0
187 self.wait(S.getValue('GUEST_PROMPT'), timeout=timeout)
189 def _affinitize(self):
191 Affinitize the SMP cores of a QEMU instance.
193 This is a bit of a hack. The 'socat' utility is used to
194 interact with the QEMU HMP. This is necessary due to the lack
195 of QMP in older versions of QEMU, like v1.6.2. In future
196 releases, this should be replaced with calls to libvirt or
197 another Python-QEMU wrapper library.
201 thread_id = (r'.* CPU #%d: .* thread_id=(\d+)')
203 self._logger.info('Affinitizing guest...')
205 cur_locale = locale.getdefaultlocale()[1]
206 proc = subprocess.Popen(
207 ('echo', 'info cpus'), stdout=subprocess.PIPE)
208 output = subprocess.check_output(
209 ('sudo', 'socat', '-', 'UNIX-CONNECT:%s' % self._monitor),
213 for cpu in range(0, int(S.getValue('GUEST_SMP')[self._number])):
215 for line in output.decode(cur_locale).split('\n'):
216 match = re.search(thread_id % cpu, line)
218 self._affinitize_pid(
219 S.getValue('GUEST_CORE_BINDING')[self._number][cpu],
224 self._logger.error('Failed to affinitize guest core #%d. Could'
225 ' not parse tid.', cpu)
227 def _config_guest_loopback(self):
229 Configure VM to run VNF, e.g. port forwarding application based on the configuration
231 if self._guest_loopback == 'testpmd':
233 self._configure_testpmd()
234 elif self._guest_loopback == 'l2fwd':
236 self._configure_l2fwd()
237 elif self._guest_loopback == 'linux_bridge':
239 self._configure_linux_bridge()
240 elif self._guest_loopback != 'buildin':
241 self._logger.error('Unsupported guest loopback method "%s" was specified. Option'
242 ' "buildin" will be used as a fallback.', self._guest_loopback)
244 def wait(self, prompt=S.getValue('GUEST_PROMPT'), timeout=30):
245 super(IVnfQemu, self).wait(prompt=prompt, timeout=timeout)
247 def execute_and_wait(self, cmd, timeout=30,
248 prompt=S.getValue('GUEST_PROMPT')):
249 super(IVnfQemu, self).execute_and_wait(cmd, timeout=timeout,
252 def _modify_dpdk_makefile(self):
254 Modifies DPDK makefile in Guest before compilation if needed
258 def _configure_copy_sources(self, dirname):
260 Mount shared directory and copy DPDK and l2fwd sources
262 # mount shared directory
263 self.execute_and_wait('umount /dev/sdb1')
264 self.execute_and_wait('rm -rf ' + S.getValue('GUEST_OVS_DPDK_DIR'))
265 self.execute_and_wait('mkdir -p ' + S.getValue('OVS_DPDK_SHARE'))
266 self.execute_and_wait('mount -o ro,iocharset=utf8 /dev/sdb1 ' +
267 S.getValue('OVS_DPDK_SHARE'))
268 self.execute_and_wait('mkdir -p ' + S.getValue('GUEST_OVS_DPDK_DIR'))
269 self.execute_and_wait('cp -ra ' + os.path.join(S.getValue('OVS_DPDK_SHARE'), dirname) +
270 ' ' + S.getValue('GUEST_OVS_DPDK_DIR'))
271 self.execute_and_wait('umount /dev/sdb1')
273 def _configure_disable_firewall(self):
275 Disable firewall in VM
277 for iptables in ['iptables', 'ip6tables']:
279 for chain in ['INPUT', 'FORWARD', 'OUTPUT']:
280 self.execute_and_wait("{} -t filter -P {} ACCEPT".format(iptables, chain))
282 for chain in ['PREROUTING', 'INPUT', 'FORWARD', 'OUTPUT', 'POSTROUTING']:
283 self.execute_and_wait("{} -t mangle -P {} ACCEPT".format(iptables, chain))
285 for chain in ['PREROUTING', 'INPUT', 'OUTPUT', 'POSTROUTING']:
286 self.execute_and_wait("{} -t nat -P {} ACCEPT".format(iptables, chain))
288 # flush rules and delete chains created by user
289 for table in ['filter', 'mangle', 'nat']:
290 self.execute_and_wait("{} -t {} -F".format(iptables, table))
291 self.execute_and_wait("{} -t {} -X".format(iptables, table))
294 def _configure_testpmd(self):
296 Configure VM to perform L2 forwarding between NICs by DPDK's testpmd
298 self._configure_copy_sources('DPDK')
299 self._configure_disable_firewall()
301 # Guest images _should_ have 1024 hugepages by default,
302 # but just in case:'''
303 self.execute_and_wait('sysctl vm.nr_hugepages=1024')
306 self.execute_and_wait('mkdir -p /dev/hugepages')
307 self.execute_and_wait(
308 'mount -t hugetlbfs hugetlbfs /dev/hugepages')
310 # build and configure system for dpdk
311 self.execute_and_wait('cd ' + S.getValue('GUEST_OVS_DPDK_DIR') +
313 self.execute_and_wait('export CC=gcc')
314 self.execute_and_wait('export RTE_SDK=' +
315 S.getValue('GUEST_OVS_DPDK_DIR') + '/DPDK')
316 self.execute_and_wait('export RTE_TARGET=%s' % S.getValue('RTE_TARGET'))
318 # modify makefile if needed
319 self._modify_dpdk_makefile()
321 # disable network interfaces, so DPDK can take care of them
322 self.execute_and_wait('ifdown ' + self._net1)
323 self.execute_and_wait('ifdown ' + self._net2)
325 # build and insert igb_uio and rebind interfaces to it
326 self.execute_and_wait('make RTE_OUTPUT=$RTE_SDK/$RTE_TARGET -C '
327 '$RTE_SDK/lib/librte_eal/linuxapp/igb_uio')
328 self.execute_and_wait('modprobe uio')
329 self.execute_and_wait('insmod %s/kmod/igb_uio.ko' %
330 S.getValue('RTE_TARGET'))
331 self.execute_and_wait('./tools/dpdk_nic_bind.py --status')
332 self.execute_and_wait(
333 './tools/dpdk_nic_bind.py -u' ' ' +
334 S.getValue('GUEST_NET1_PCI_ADDRESS')[self._number] + ' ' +
335 S.getValue('GUEST_NET2_PCI_ADDRESS')[self._number])
336 self.execute_and_wait(
337 './tools/dpdk_nic_bind.py -b igb_uio' ' ' +
338 S.getValue('GUEST_NET1_PCI_ADDRESS')[self._number] + ' ' +
339 S.getValue('GUEST_NET2_PCI_ADDRESS')[self._number])
340 self.execute_and_wait('./tools/dpdk_nic_bind.py --status')
342 # build and run 'test-pmd'
343 self.execute_and_wait('cd ' + S.getValue('GUEST_OVS_DPDK_DIR') +
344 '/DPDK/app/test-pmd')
345 self.execute_and_wait('make clean')
346 self.execute_and_wait('make')
347 self.execute_and_wait('./testpmd -c 0x3 -n 4 --socket-mem 512 --'
348 ' --burst=64 -i --txqflags=0xf00 ' +
349 '--disable-hw-vlan', 60, "Done")
350 self.execute('set fwd ' + self._testpmd_fwd_mode, 1)
351 self.execute_and_wait('start', 20,
352 'TX RS bit threshold=.+ - TXQ flags=0xf00')
354 def _configure_l2fwd(self):
356 Configure VM to perform L2 forwarding between NICs by l2fwd module
358 self._configure_copy_sources('l2fwd')
359 self._configure_disable_firewall()
361 # build and configure system for l2fwd
362 self.execute_and_wait('cd ' + S.getValue('GUEST_OVS_DPDK_DIR') +
364 self.execute_and_wait('export CC=gcc')
366 self.execute_and_wait('make')
367 self.execute_and_wait('insmod ' + S.getValue('GUEST_OVS_DPDK_DIR') +
368 '/l2fwd' + '/l2fwd.ko net1=' + self._net1 +
369 ' net2=' + self._net2)
371 def _configure_linux_bridge(self):
373 Configure VM to perform L2 forwarding between NICs by linux bridge
375 self._configure_disable_firewall()
377 self.execute('ifconfig ' + self._net1 + ' ' +
378 S.getValue('VANILLA_NIC1_IP_CIDR')[self._number])
380 self.execute('ifconfig ' + self._net2 + ' ' +
381 S.getValue('VANILLA_NIC2_IP_CIDR')[self._number])
383 # configure linux bridge
384 self.execute('brctl addbr br0')
385 self.execute('brctl addif br0 ' + self._net1 + ' ' + self._net2)
386 self.execute('ifconfig br0 ' +
387 S.getValue('VANILLA_BRIDGE_IP')[self._number])
389 # Add the arp entries for the IXIA ports and the bridge you are using.
390 # Use command line values if provided.
391 trafficgen_mac = get_test_param('vanilla_tgen_port1_mac',
392 S.getValue('VANILLA_TGEN_PORT1_MAC'))
393 trafficgen_ip = get_test_param('vanilla_tgen_port1_ip',
394 S.getValue('VANILLA_TGEN_PORT1_IP'))
396 self.execute('arp -s ' + trafficgen_ip + ' ' + trafficgen_mac)
398 trafficgen_mac = get_test_param('vanilla_tgen_port2_mac',
399 S.getValue('VANILLA_TGEN_PORT2_MAC'))
400 trafficgen_ip = get_test_param('vanilla_tgen_port2_ip',
401 S.getValue('VANILLA_TGEN_PORT2_IP'))
403 self.execute('arp -s ' + trafficgen_ip + ' ' + trafficgen_mac)
406 self.execute('sysctl -w net.ipv4.ip_forward=1')
408 # Controls source route verification
409 # 0 means no source validation
410 self.execute('sysctl -w net.ipv4.conf.all.rp_filter=0')
411 self.execute('sysctl -w net.ipv4.conf.' + self._net1 + '.rp_filter=0')
412 self.execute('sysctl -w net.ipv4.conf.' + self._net2 + '.rp_filter=0')