1 # Copyright 2015-2016 Intel Corporation.
3 # Licensed under the Apache License, Version 2.0 (the "License");
4 # you may not use this file except in compliance with the License.
5 # You may obtain a copy of the License at
7 # http://www.apache.org/licenses/LICENSE-2.0
9 # Unless required by applicable law or agreed to in writing, software
10 # distributed under the License is distributed on an "AS IS" BASIS,
11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 # See the License for the specific language governing permissions and
13 # limitations under the License.
15 """Automation of QEMU hypervisor for launching guests.
26 from conf import settings as S
27 from vnfs.vnf.vnf import IVnf
31 Abstract class for controling an instance of QEMU
37 class GuestCommandFilter(logging.Filter):
39 Filter out strings beginning with 'guestcmd :'.
41 def filter(self, record):
42 return record.getMessage().startswith(self.prefix)
46 Initialisation function.
48 super(IVnfQemu, self).__init__()
50 self._expect = S.getValue('GUEST_PROMPT_LOGIN')[self._number]
51 self._logger = logging.getLogger(__name__)
52 self._logfile = os.path.join(
53 S.getValue('LOG_DIR'),
54 S.getValue('LOG_FILE_QEMU')) + str(self._number)
55 self._timeout = S.getValue('GUEST_TIMEOUT')[self._number]
56 self._monitor = '%s/vm%dmonitor' % ('/tmp', self._number)
57 # read GUEST NICs configuration and use only defined NR of NICS
58 nics_nr = S.getValue('GUEST_NICS_NR')[self._number]
59 # and inform user about missconfiguration
61 raise RuntimeError('At least one VM NIC is mandotory, but {} '
62 'NICs are configured'.format(nics_nr))
63 elif nics_nr > 1 and nics_nr % 2:
64 nics_nr = int(nics_nr / 2) * 2
65 self._logger.warning('Odd number of NICs is configured, only '
66 '%s NICs will be used', nics_nr)
68 self._nics = S.getValue('GUEST_NICS')[self._number][:nics_nr]
70 # set guest loopback application based on VNF configuration
71 self._guest_loopback = S.getValue('GUEST_LOOPBACK')[self._number]
73 self._testpmd_fwd_mode = S.getValue('GUEST_TESTPMD_FWD_MODE')[self._number]
74 # in case of SRIOV we must ensure, that MAC addresses are not swapped
75 if S.getValue('SRIOV_ENABLED') and self._testpmd_fwd_mode.startswith('mac') and \
76 not S.getValue('VNF').endswith('PciPassthrough'):
78 self._logger.info("SRIOV detected, forwarding mode of testpmd was changed from '%s' to '%s'",
79 self._testpmd_fwd_mode, 'io')
80 self._testpmd_fwd_mode = 'io'
82 name = 'Client%d' % self._number
83 vnc = ':%d' % self._number
84 # NOTE: affinization of main qemu process can cause hangup of 2nd VM
85 # in case of DPDK usage. It can also slow down VM response time.
86 cpumask = ",".join(S.getValue('GUEST_CORE_BINDING')[self._number])
87 self._cmd = ['sudo', '-E', 'taskset', '-c', cpumask,
88 S.getValue('TOOLS')['qemu-system'],
89 '-m', S.getValue('GUEST_MEMORY')[self._number],
90 '-smp', str(S.getValue('GUEST_SMP')[self._number]),
91 '-cpu', str(S.getValue('GUEST_CPU_OPTIONS')[self._number]),
92 '-drive', 'if={},file='.format(S.getValue(
93 'GUEST_BOOT_DRIVE_TYPE')[self._number]) +
94 S.getValue('GUEST_IMAGE')[self._number],
95 '-boot', 'c', '--enable-kvm',
96 '-monitor', 'unix:%s,server,nowait' % self._monitor,
98 'memory-backend-file,id=mem,size=' +
99 str(S.getValue('GUEST_MEMORY')[self._number]) + 'M,' +
100 'mem-path=' + S.getValue('HUGEPAGE_DIR') + ',share=on',
101 '-numa', 'node,memdev=mem -mem-prealloc',
102 '-nographic', '-vnc', str(vnc), '-name', name,
103 '-snapshot', '-net none', '-no-reboot',
105 'if=%s,format=raw,file=fat:rw:%s,snapshot=off' %
106 (S.getValue('GUEST_SHARED_DRIVE_TYPE')[self._number],
107 S.getValue('GUEST_SHARE_DIR')[self._number]),
109 self._configure_logging()
111 def _configure_logging(self):
115 self.GuestCommandFilter.prefix = self._log_prefix
117 logger = logging.getLogger()
118 cmd_logger = logging.FileHandler(
119 filename=os.path.join(S.getValue('LOG_DIR'),
120 S.getValue('LOG_FILE_GUEST_CMDS')) +
122 cmd_logger.setLevel(logging.DEBUG)
123 cmd_logger.addFilter(self.GuestCommandFilter())
124 logger.addHandler(cmd_logger)
130 Start QEMU instance, login and prepare for commands.
132 super(IVnfQemu, self).start()
133 if S.getValue('VNF_AFFINITIZATION_ON'):
136 if S.getValue('VSWITCH_VHOST_NET_AFFINITIZATION') and S.getValue(
137 'VNF') == 'QemuVirtioNet':
138 self._affinitize_vhost_net()
141 self._config_guest_loopback()
145 Stops VNF instance gracefully first.
147 if self.is_running():
149 # exit testpmd if needed
150 if self._guest_loopback == 'testpmd':
151 self.execute_and_wait('stop', 120, "Done")
152 self.execute_and_wait('quit', 120, "[bB]ye")
155 self.execute_and_wait('poweroff', 120, "Power down")
157 except pexpect.TIMEOUT:
160 # wait until qemu shutdowns
161 self._logger.debug('Wait for QEMU to terminate')
162 for dummy in range(30):
164 if not self.is_running():
167 # just for case that graceful shutdown failed
168 super(IVnfQemu, self).stop()
172 def _login(self, timeout=120):
174 Login to QEMU instance.
176 This can be used immediately after booting the machine, provided a
177 sufficiently long ``timeout`` is given.
179 :param timeout: Timeout to wait for login to complete.
183 # if no timeout was set, we likely started QEMU without waiting for it
184 # to boot. This being the case, we best check that it has finished
186 if not self._timeout:
187 self._expect_process(timeout=timeout)
189 self._child.sendline(S.getValue('GUEST_USERNAME')[self._number])
190 self._child.expect(S.getValue('GUEST_PROMPT_PASSWORD')[self._number], timeout=5)
191 self._child.sendline(S.getValue('GUEST_PASSWORD')[self._number])
193 self._expect_process(S.getValue('GUEST_PROMPT')[self._number], timeout=5)
195 def send_and_pass(self, cmd, timeout=30):
197 Send ``cmd`` and wait ``timeout`` seconds for it to pass.
199 :param cmd: Command to send to guest.
200 :param timeout: Time to wait for prompt before checking return code.
205 self.wait(S.getValue('GUEST_PROMPT')[self._number], timeout=timeout)
206 self.execute('echo $?')
207 self._child.expect('^0$', timeout=1) # expect a 0
208 self.wait(S.getValue('GUEST_PROMPT')[self._number], timeout=timeout)
210 def _affinitize(self):
212 Affinitize the SMP cores of a QEMU instance.
214 This is a bit of a hack. The 'socat' utility is used to
215 interact with the QEMU HMP. This is necessary due to the lack
216 of QMP in older versions of QEMU, like v1.6.2. In future
217 releases, this should be replaced with calls to libvirt or
218 another Python-QEMU wrapper library.
222 thread_id = (r'.* CPU #%d: .* thread_id=(\d+)')
224 self._logger.info('Affinitizing guest...')
226 cur_locale = locale.getdefaultlocale()[1]
227 proc = subprocess.Popen(
228 ('echo', 'info cpus'), stdout=subprocess.PIPE)
229 output = subprocess.check_output(
230 ('sudo', 'socat', '-', 'UNIX-CONNECT:%s' % self._monitor),
234 for cpu in range(0, int(S.getValue('GUEST_SMP')[self._number])):
236 guest_thread_binding = S.getValue('GUEST_THREAD_BINDING')[self._number]
237 if guest_thread_binding is None:
238 guest_thread_binding = S.getValue('GUEST_CORE_BINDING')[self._number]
239 for line in output.decode(cur_locale).split('\n'):
240 match = re.search(thread_id % cpu, line)
242 self._affinitize_pid(guest_thread_binding[cpu], match.group(1))
246 self._logger.error('Failed to affinitize guest core #%d. Could'
247 ' not parse tid.', cpu)
249 def _affinitize_vhost_net(self):
251 Affinitize the vhost net threads for Vanilla OVS and guest nic queues.
255 self._logger.info('Affinitizing VHOST Net threads.')
256 args1 = ['pgrep', 'vhost-']
257 process1 = subprocess.Popen(args1, stdout=subprocess.PIPE,
259 out = process1.communicate()[0]
260 processes = out.decode(locale.getdefaultlocale()[1]).split('\n')
261 if processes[-1] == '':
262 processes.pop() # pgrep may return an extra line with no data
263 self._logger.info('Found %s vhost net threads...', len(processes))
265 cpumap = S.getValue('VSWITCH_VHOST_CPU_MAP')
267 for proc in processes:
268 self._affinitize_pid(cpumap[mapcount], proc)
270 if mapcount + 1 > len(cpumap):
271 # Not enough cpus were given in the mapping to cover all the
272 # threads on a 1 to 1 ratio with cpus so reset the list counter
276 def _config_guest_loopback(self):
278 Configure VM to run VNF, e.g. port forwarding application based on the configuration
280 if self._guest_loopback == 'testpmd':
282 self._configure_testpmd()
283 elif self._guest_loopback == 'l2fwd':
285 self._configure_l2fwd()
286 elif self._guest_loopback == 'linux_bridge':
288 self._configure_linux_bridge()
289 elif self._guest_loopback != 'buildin':
290 self._logger.error('Unsupported guest loopback method "%s" was specified. Option'
291 ' "buildin" will be used as a fallback.', self._guest_loopback)
293 def wait(self, prompt=None, timeout=30):
295 prompt = S.getValue('GUEST_PROMPT')[self._number]
296 super(IVnfQemu, self).wait(prompt=prompt, timeout=timeout)
298 def execute_and_wait(self, cmd, timeout=30, prompt=None):
300 prompt = S.getValue('GUEST_PROMPT')[self._number]
301 super(IVnfQemu, self).execute_and_wait(cmd, timeout=timeout,
304 def _modify_dpdk_makefile(self):
306 Modifies DPDK makefile in Guest before compilation if needed
310 def _configure_copy_sources(self, dirname):
312 Mount shared directory and copy DPDK and l2fwd sources
314 # mount shared directory
315 self.execute_and_wait('umount /dev/sdb1')
316 self.execute_and_wait('rm -rf ' + S.getValue('GUEST_OVS_DPDK_DIR')[self._number])
317 self.execute_and_wait('mkdir -p ' + S.getValue('GUEST_OVS_DPDK_SHARE')[self._number])
318 self.execute_and_wait('mount -o ro,iocharset=utf8 /dev/sdb1 ' +
319 S.getValue('GUEST_OVS_DPDK_SHARE')[self._number])
320 self.execute_and_wait('mkdir -p ' + S.getValue('GUEST_OVS_DPDK_DIR')[self._number])
321 self.execute_and_wait('cp -r ' + os.path.join(S.getValue('GUEST_OVS_DPDK_SHARE')[self._number], dirname) +
322 ' ' + S.getValue('GUEST_OVS_DPDK_DIR')[self._number])
323 self.execute_and_wait('umount /dev/sdb1')
325 def _configure_disable_firewall(self):
327 Disable firewall in VM
329 for iptables in ['iptables', 'ip6tables']:
331 for chain in ['INPUT', 'FORWARD', 'OUTPUT']:
332 self.execute_and_wait("{} -t filter -P {} ACCEPT".format(iptables, chain))
334 for chain in ['PREROUTING', 'INPUT', 'FORWARD', 'OUTPUT', 'POSTROUTING']:
335 self.execute_and_wait("{} -t mangle -P {} ACCEPT".format(iptables, chain))
337 for chain in ['PREROUTING', 'INPUT', 'OUTPUT', 'POSTROUTING']:
338 self.execute_and_wait("{} -t nat -P {} ACCEPT".format(iptables, chain))
340 # flush rules and delete chains created by user
341 for table in ['filter', 'mangle', 'nat']:
342 self.execute_and_wait("{} -t {} -F".format(iptables, table))
343 self.execute_and_wait("{} -t {} -X".format(iptables, table))
345 def _configure_testpmd(self):
347 Configure VM to perform L2 forwarding between NICs by DPDK's testpmd
349 self._configure_copy_sources('DPDK')
350 self._configure_disable_firewall()
352 # Guest images _should_ have 1024 hugepages by default,
353 # but just in case:'''
354 self.execute_and_wait('sysctl vm.nr_hugepages={}'.format(S.getValue('GUEST_HUGEPAGES_NR')[self._number]))
357 self.execute_and_wait('mkdir -p /dev/hugepages')
358 self.execute_and_wait(
359 'mount -t hugetlbfs hugetlbfs /dev/hugepages')
361 # build and configure system for dpdk
362 self.execute_and_wait('cd ' + S.getValue('GUEST_OVS_DPDK_DIR')[self._number] +
364 self.execute_and_wait('export CC=gcc')
365 self.execute_and_wait('export RTE_SDK=' +
366 S.getValue('GUEST_OVS_DPDK_DIR')[self._number] + '/DPDK')
367 self.execute_and_wait('export RTE_TARGET=%s' % S.getValue('RTE_TARGET'))
369 # modify makefile if needed
370 self._modify_dpdk_makefile()
372 # disable network interfaces, so DPDK can take care of them
373 for nic in self._nics:
374 self.execute_and_wait('ifdown ' + nic['device'])
376 self.execute_and_wait('./tools/dpdk*bind.py --status')
377 pci_list = ' '.join([nic['pci'] for nic in self._nics])
378 self.execute_and_wait('./tools/dpdk*bind.py -u ' + pci_list)
379 self._bind_dpdk_driver(S.getValue(
380 'GUEST_DPDK_BIND_DRIVER')[self._number], pci_list)
381 self.execute_and_wait('./tools/dpdk*bind.py --status')
383 # build and run 'test-pmd'
384 self.execute_and_wait('cd ' + S.getValue('GUEST_OVS_DPDK_DIR')[self._number] +
385 '/DPDK/app/test-pmd')
386 self.execute_and_wait('make clean')
387 self.execute_and_wait('make')
389 # get testpmd settings from CLI
390 testpmd_params = S.getValue('GUEST_TESTPMD_PARAMS')[self._number]
391 if S.getValue('VSWITCH_JUMBO_FRAMES_ENABLED'):
392 testpmd_params += ' --max-pkt-len={}'.format(S.getValue(
393 'VSWITCH_JUMBO_FRAMES_SIZE'))
395 self.execute_and_wait('./testpmd {}'.format(testpmd_params), 60, "Done")
396 self.execute('set fwd ' + self._testpmd_fwd_mode, 1)
397 self.execute_and_wait('start', 20, 'testpmd>')
399 def _configure_l2fwd(self):
401 Configure VM to perform L2 forwarding between NICs by l2fwd module
403 if int(S.getValue('GUEST_NIC_QUEUES')[self._number]):
404 self._set_multi_queue_nic()
405 self._configure_copy_sources('l2fwd')
406 self._configure_disable_firewall()
408 # configure all interfaces
409 for nic in self._nics:
410 self.execute('ip addr add ' +
411 nic['ip'] + ' dev ' + nic['device'])
412 if S.getValue('VSWITCH_JUMBO_FRAMES_ENABLED'):
413 self.execute('ifconfig {} mtu {}'.format(
414 nic['device'], S.getValue('VSWITCH_JUMBO_FRAMES_SIZE')))
415 self.execute('ip link set dev ' + nic['device'] + ' up')
417 # build and configure system for l2fwd
418 self.execute_and_wait('cd ' + S.getValue('GUEST_OVS_DPDK_DIR')[self._number] +
420 self.execute_and_wait('export CC=gcc')
422 self.execute_and_wait('make')
423 if len(self._nics) == 2:
424 self.execute_and_wait('insmod ' + S.getValue('GUEST_OVS_DPDK_DIR')[self._number] +
425 '/l2fwd' + '/l2fwd.ko net1=' + self._nics[0]['device'] +
426 ' net2=' + self._nics[1]['device'])
428 raise RuntimeError('l2fwd can forward only between 2 NICs, but {} NICs are '
429 'configured inside GUEST'.format(len(self._nics)))
431 def _configure_linux_bridge(self):
433 Configure VM to perform L2 forwarding between NICs by linux bridge
435 if int(S.getValue('GUEST_NIC_QUEUES')[self._number]):
436 self._set_multi_queue_nic()
437 self._configure_disable_firewall()
439 # configure linux bridge
440 self.execute('brctl addbr br0')
442 # add all NICs into the bridge
443 for nic in self._nics:
444 self.execute('ip addr add ' +
445 nic['ip'] + ' dev ' + nic['device'])
446 if S.getValue('VSWITCH_JUMBO_FRAMES_ENABLED'):
447 self.execute('ifconfig {} mtu {}'.format(
448 nic['device'], S.getValue('VSWITCH_JUMBO_FRAMES_SIZE')))
449 self.execute('ip link set dev ' + nic['device'] + ' up')
450 self.execute('brctl addif br0 ' + nic['device'])
452 self.execute('ip addr add ' +
453 S.getValue('GUEST_BRIDGE_IP')[self._number] +
455 self.execute('ip link set dev br0 up')
457 # Add the arp entries for the IXIA ports and the bridge you are using.
458 # Use command line values if provided.
459 trafficgen_mac = S.getValue('VANILLA_TGEN_PORT1_MAC')
460 trafficgen_ip = S.getValue('VANILLA_TGEN_PORT1_IP')
462 self.execute('arp -s ' + trafficgen_ip + ' ' + trafficgen_mac)
464 trafficgen_mac = S.getValue('VANILLA_TGEN_PORT2_MAC')
465 trafficgen_ip = S.getValue('VANILLA_TGEN_PORT2_IP')
467 self.execute('arp -s ' + trafficgen_ip + ' ' + trafficgen_mac)
470 self.execute('sysctl -w net.ipv4.ip_forward=1')
472 # Controls source route verification
473 # 0 means no source validation
474 self.execute('sysctl -w net.ipv4.conf.all.rp_filter=0')
475 for nic in self._nics:
476 self.execute('sysctl -w net.ipv4.conf.' + nic['device'] + '.rp_filter=0')
478 def _bind_dpdk_driver(self, driver, pci_slots):
480 Bind the virtual nics to the driver specific in the conf file
483 if driver == 'uio_pci_generic':
484 if S.getValue('VNF') == 'QemuPciPassthrough':
485 # unsupported config, bind to igb_uio instead and exit the
486 # outer function after completion.
487 self._logger.error('SR-IOV does not support uio_pci_generic. '
488 'Igb_uio will be used instead.')
489 self._bind_dpdk_driver('igb_uio_from_src', pci_slots)
491 self.execute_and_wait('modprobe uio_pci_generic')
492 self.execute_and_wait('./tools/dpdk*bind.py -b uio_pci_generic '+
494 elif driver == 'vfio_no_iommu':
495 self.execute_and_wait('modprobe -r vfio')
496 self.execute_and_wait('modprobe -r vfio_iommu_type1')
497 self.execute_and_wait('modprobe vfio enable_unsafe_noiommu_mode=Y')
498 self.execute_and_wait('modprobe vfio-pci')
499 self.execute_and_wait('./tools/dpdk*bind.py -b vfio-pci ' +
501 elif driver == 'igb_uio_from_src':
502 # build and insert igb_uio and rebind interfaces to it
503 self.execute_and_wait('make RTE_OUTPUT=$RTE_SDK/$RTE_TARGET -C '
504 '$RTE_SDK/lib/librte_eal/linuxapp/igb_uio')
505 self.execute_and_wait('modprobe uio')
506 self.execute_and_wait('insmod %s/kmod/igb_uio.ko' %
507 S.getValue('RTE_TARGET'))
508 self.execute_and_wait('./tools/dpdk*bind.py -b igb_uio ' + pci_slots)
511 'Unknown driver for binding specified, defaulting to igb_uio')
512 self._bind_dpdk_driver('igb_uio_from_src', pci_slots)
514 def _set_multi_queue_nic(self):
516 Enable multi-queue in guest kernel with ethool.
519 for nic in self._nics:
520 self.execute_and_wait('ethtool -L {} combined {}'.format(
521 nic['device'], S.getValue('GUEST_NIC_QUEUES')[self._number]))
522 self.execute_and_wait('ethtool -l {}'.format(nic['device']))