1 # Copyright 2015-2017 Intel Corporation.
3 # Licensed under the Apache License, Version 2.0 (the "License");
4 # you may not use this file except in compliance with the License.
5 # You may obtain a copy of the License at
7 # http://www.apache.org/licenses/LICENSE-2.0
9 # Unless required by applicable law or agreed to in writing, software
10 # distributed under the License is distributed on an "AS IS" BASIS,
11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 # See the License for the specific language governing permissions and
13 # limitations under the License.
15 """Automation of QEMU hypervisor for launching guests.
26 from conf import settings as S
27 from vnfs.vnf.vnf import IVnf
31 Abstract class for controling an instance of QEMU
37 class GuestCommandFilter(logging.Filter):
39 Filter out strings beginning with 'guestcmd :'.
41 def filter(self, record):
42 return record.getMessage().startswith(self.prefix)
46 Initialisation function.
48 super(IVnfQemu, self).__init__()
49 name, ext = os.path.splitext(S.getValue('LOG_FILE_QEMU'))
50 name = name + str(self._number)
51 rename_qemu = "{name}_{uid}{ex}".format(name=name,
52 uid=S.getValue('LOG_TIMESTAMP'),
54 self._expect = S.getValue('GUEST_PROMPT_LOGIN')[self._number]
55 self._logger = logging.getLogger(__name__)
56 self._logfile = os.path.join(S.getValue('RESULTS_PATH'), rename_qemu)
57 self._timeout = S.getValue('GUEST_TIMEOUT')[self._number]
58 self._monitor = '%s/vm%dmonitor' % ('/tmp', self._number)
59 # read GUEST NICs configuration and use only defined NR of NICS
60 nics_nr = S.getValue('GUEST_NICS_NR')[self._number]
61 # and inform user about missconfiguration
63 raise RuntimeError('At least one VM NIC is mandotory, but {} '
64 'NICs are configured'.format(nics_nr))
65 elif nics_nr > 1 and nics_nr % 2:
66 nics_nr = int(nics_nr / 2) * 2
67 self._logger.warning('Odd number of NICs is configured, only '
68 '%s NICs will be used', nics_nr)
70 self._nics = S.getValue('GUEST_NICS')[self._number][:nics_nr]
72 # set guest loopback application based on VNF configuration
73 self._guest_loopback = S.getValue('GUEST_LOOPBACK')[self._number]
75 self._testpmd_fwd_mode = S.getValue('GUEST_TESTPMD_FWD_MODE')[self._number]
76 # in case of SRIOV we must ensure, that MAC addresses are not swapped
77 if S.getValue('SRIOV_ENABLED') and self._testpmd_fwd_mode.startswith('mac') and \
78 not str(S.getValue('VNF')).endswith('PciPassthrough'):
80 self._logger.info("SRIOV detected, forwarding mode of testpmd was changed from '%s' to '%s'",
81 self._testpmd_fwd_mode, 'io')
82 self._testpmd_fwd_mode = 'io'
84 name = 'Client%d' % self._number
85 vnc = ':%d' % self._number
86 # NOTE: affinization of main qemu process can cause hangup of 2nd VM
87 # in case of DPDK usage. It can also slow down VM response time.
88 cpumask = ",".join(S.getValue('GUEST_CORE_BINDING')[self._number])
89 self._cmd = ['sudo', '-E', 'taskset', '-c', cpumask,
90 S.getValue('TOOLS')['qemu-system'],
91 '-m', S.getValue('GUEST_MEMORY')[self._number],
92 '-smp', str(S.getValue('GUEST_SMP')[self._number]),
93 '-cpu', str(S.getValue('GUEST_CPU_OPTIONS')[self._number]),
94 '-drive', 'if={},file='.format(S.getValue(
95 'GUEST_BOOT_DRIVE_TYPE')[self._number]) +
96 S.getValue('GUEST_IMAGE')[self._number],
97 '-boot', 'c', '--enable-kvm',
98 '-monitor', 'unix:%s,server,nowait' % self._monitor,
100 'memory-backend-file,id=mem,size=' +
101 str(S.getValue('GUEST_MEMORY')[self._number]) + 'M,' +
102 'mem-path=' + S.getValue('HUGEPAGE_DIR') + ',share=on',
103 '-numa', 'node,memdev=mem -mem-prealloc',
104 '-nographic', '-vnc', str(vnc), '-name', name,
105 '-snapshot', '-net none', '-no-reboot',
107 'if=%s,format=raw,file=fat:rw:%s,snapshot=off' %
108 (S.getValue('GUEST_SHARED_DRIVE_TYPE')[self._number],
109 S.getValue('GUEST_SHARE_DIR')[self._number]),
111 self._configure_logging()
113 def _configure_logging(self):
117 self.GuestCommandFilter.prefix = self._log_prefix
119 logger = logging.getLogger()
120 name, ext = os.path.splitext(S.getValue('LOG_FILE_GUEST_CMDS'))
121 name = name + str(self._number)
122 rename_gcmd = "{name}_{uid}{ex}".format(name=name,
123 uid=S.getValue('LOG_TIMESTAMP'),
125 cmd_logger = logging.FileHandler(
126 filename=os.path.join(S.getValue('LOG_DIR'), rename_gcmd))
127 cmd_logger.setLevel(logging.DEBUG)
128 cmd_logger.addFilter(self.GuestCommandFilter())
129 logger.addHandler(cmd_logger)
135 Start QEMU instance, login and prepare for commands.
137 super(IVnfQemu, self).start()
138 if S.getValue('VNF_AFFINITIZATION_ON'):
141 if S.getValue('VSWITCH_VHOST_NET_AFFINITIZATION') and S.getValue(
142 'VNF') == 'QemuVirtioNet':
143 self._affinitize_vhost_net()
146 self._config_guest_loopback()
150 Stops VNF instance gracefully first.
152 if self.is_running():
154 if self._login_active:
155 # exit testpmd if needed
156 if self._guest_loopback == 'testpmd':
157 self.execute_and_wait('stop', 120, "Done")
158 self.execute_and_wait('quit', 120, "[bB]ye")
161 self.execute_and_wait('poweroff', 120, "Power down")
163 except pexpect.TIMEOUT:
166 # wait until qemu shutdowns
167 self._logger.debug('Wait for QEMU to terminate')
168 for dummy in range(30):
170 if not self.is_running():
173 # just for case that graceful shutdown failed
174 super(IVnfQemu, self).stop()
178 def login(self, timeout=120):
180 Login to QEMU instance.
182 This can be used immediately after booting the machine, provided a
183 sufficiently long ``timeout`` is given.
185 :param timeout: Timeout to wait for login to complete.
187 :returns: True if login is active
189 if self._login_active:
190 return self._login_active
192 # if no timeout was set, we likely started QEMU without waiting for it
193 # to boot. This being the case, we best check that it has finished
195 if not self._timeout:
196 self._expect_process(timeout=timeout)
198 self._child.sendline(S.getValue('GUEST_USERNAME')[self._number])
199 self._child.expect(S.getValue('GUEST_PROMPT_PASSWORD')[self._number], timeout=5)
200 self._child.sendline(S.getValue('GUEST_PASSWORD')[self._number])
202 self._expect_process(S.getValue('GUEST_PROMPT')[self._number], timeout=5)
203 self._login_active = True
204 return self._login_active
206 def _affinitize(self):
208 Affinitize the SMP cores of a QEMU instance.
210 This is a bit of a hack. The 'socat' utility is used to
211 interact with the QEMU HMP. This is necessary due to the lack
212 of QMP in older versions of QEMU, like v1.6.2. In future
213 releases, this should be replaced with calls to libvirt or
214 another Python-QEMU wrapper library.
218 thread_id = (r'.* CPU #%d:.*? thread_id=(\d+)')
220 self._logger.info('Affinitizing guest...')
222 cur_locale = locale.getdefaultlocale()[1]
223 proc = subprocess.Popen(
224 ('echo', 'info cpus'), stdout=subprocess.PIPE)
225 output = subprocess.check_output(
226 ('sudo', 'socat', '-', 'UNIX-CONNECT:%s' % self._monitor),
230 # calculate the number of CPUs in SMP topology specified by GUEST_SMP
231 # e.g. "sockets=2,cores=3", "4", etc.
233 for i in re.findall(r'\d', S.getValue('GUEST_SMP')[self._number]):
234 cpu_nr = cpu_nr * int(i)
235 # pin each GUEST's core to host core based on configured BINDING
236 for cpu in range(0, cpu_nr):
238 guest_thread_binding = S.getValue('GUEST_THREAD_BINDING')[self._number]
239 if guest_thread_binding is None:
240 guest_thread_binding = S.getValue('GUEST_CORE_BINDING')[self._number]
241 for line in output.decode(cur_locale).split('\n'):
242 match = re.search(thread_id % cpu, line)
244 self._affinitize_pid(guest_thread_binding[cpu], match.group(1))
248 self._logger.error('Failed to affinitize guest core #%d. Could'
249 ' not parse tid.', cpu)
251 def _affinitize_vhost_net(self):
253 Affinitize the vhost net threads for Vanilla OVS and guest nic queues.
257 self._logger.info('Affinitizing VHOST Net threads.')
258 args1 = ['pgrep', 'vhost-']
259 process1 = subprocess.Popen(args1, stdout=subprocess.PIPE,
261 out = process1.communicate()[0]
262 processes = out.decode(locale.getdefaultlocale()[1]).split('\n')
263 if processes[-1] == '':
264 processes.pop() # pgrep may return an extra line with no data
265 self._logger.info('Found %s vhost net threads...', len(processes))
267 cpumap = S.getValue('VSWITCH_VHOST_CPU_MAP')
269 for proc in processes:
270 self._affinitize_pid(cpumap[mapcount], proc)
272 if mapcount + 1 > len(cpumap):
273 # Not enough cpus were given in the mapping to cover all the
274 # threads on a 1 to 1 ratio with cpus so reset the list counter
278 def _config_guest_loopback(self):
280 Configure VM to run VNF, e.g. port forwarding application based on the configuration
282 if self._guest_loopback == 'buildin':
287 if self._guest_loopback == 'testpmd':
288 self._configure_testpmd()
289 elif self._guest_loopback == 'l2fwd':
290 self._configure_l2fwd()
291 elif self._guest_loopback == 'linux_bridge':
292 self._configure_linux_bridge()
293 elif self._guest_loopback != 'clean':
294 raise RuntimeError('Unsupported guest loopback method "%s" was specified.' %
295 self._guest_loopback)
297 def wait(self, prompt=None, timeout=30):
299 prompt = S.getValue('GUEST_PROMPT')[self._number]
300 return super(IVnfQemu, self).wait(prompt=prompt, timeout=timeout)
302 def execute_and_wait(self, cmd, timeout=30, prompt=None):
304 prompt = S.getValue('GUEST_PROMPT')[self._number]
305 return super(IVnfQemu, self).execute_and_wait(cmd, timeout=timeout,
308 def _modify_dpdk_makefile(self):
310 Modifies DPDK makefile in Guest before compilation if needed
314 def _configure_copy_sources(self, dirname):
316 Mount shared directory and copy DPDK and l2fwd sources
318 # mount shared directory
319 self.execute_and_wait('umount /dev/sdb1')
320 self.execute_and_wait('rm -rf ' + S.getValue('GUEST_OVS_DPDK_DIR')[self._number])
321 self.execute_and_wait('mkdir -p ' + S.getValue('GUEST_OVS_DPDK_SHARE')[self._number])
322 self.execute_and_wait('mount -o ro,iocharset=utf8 /dev/sdb1 ' +
323 S.getValue('GUEST_OVS_DPDK_SHARE')[self._number])
324 self.execute_and_wait('mkdir -p ' + S.getValue('GUEST_OVS_DPDK_DIR')[self._number])
325 self.execute_and_wait('cp -r ' + os.path.join(S.getValue('GUEST_OVS_DPDK_SHARE')[self._number], dirname) +
326 ' ' + S.getValue('GUEST_OVS_DPDK_DIR')[self._number])
327 self.execute_and_wait('umount /dev/sdb1')
329 def _configure_disable_firewall(self):
331 Disable firewall in VM
333 for iptables in ['iptables', 'ip6tables']:
335 for chain in ['INPUT', 'FORWARD', 'OUTPUT']:
336 self.execute_and_wait("{} -t filter -P {} ACCEPT".format(iptables, chain))
338 for chain in ['PREROUTING', 'INPUT', 'FORWARD', 'OUTPUT', 'POSTROUTING']:
339 self.execute_and_wait("{} -t mangle -P {} ACCEPT".format(iptables, chain))
341 for chain in ['PREROUTING', 'INPUT', 'OUTPUT', 'POSTROUTING']:
342 self.execute_and_wait("{} -t nat -P {} ACCEPT".format(iptables, chain))
344 # flush rules and delete chains created by user
345 for table in ['filter', 'mangle', 'nat']:
346 self.execute_and_wait("{} -t {} -F".format(iptables, table))
347 self.execute_and_wait("{} -t {} -X".format(iptables, table))
349 def _configure_testpmd(self):
351 Configure VM to perform L2 forwarding between NICs by DPDK's testpmd
353 self._configure_copy_sources('DPDK')
354 self._configure_disable_firewall()
356 # Guest images _should_ have 1024 hugepages by default,
357 # but just in case:'''
358 self.execute_and_wait('sysctl vm.nr_hugepages={}'.format(S.getValue('GUEST_HUGEPAGES_NR')[self._number]))
361 self.execute_and_wait('mkdir -p /dev/hugepages')
362 self.execute_and_wait(
363 'mount -t hugetlbfs hugetlbfs /dev/hugepages')
365 # build and configure system for dpdk
366 self.execute_and_wait('cd ' + S.getValue('GUEST_OVS_DPDK_DIR')[self._number] +
368 self.execute_and_wait('export CC=gcc')
369 self.execute_and_wait('export RTE_SDK=' +
370 S.getValue('GUEST_OVS_DPDK_DIR')[self._number] + '/DPDK')
371 self.execute_and_wait('export RTE_TARGET=%s' % S.getValue('RTE_TARGET'))
373 # modify makefile if needed
374 self._modify_dpdk_makefile()
376 # disable network interfaces, so DPDK can take care of them
377 for nic in self._nics:
378 self.execute_and_wait('ifdown ' + nic['device'])
380 self.execute_and_wait('./*tools/dpdk*bind.py --status')
381 pci_list = ' '.join([nic['pci'] for nic in self._nics])
382 self.execute_and_wait('./*tools/dpdk*bind.py -u ' + pci_list)
383 self._bind_dpdk_driver(S.getValue(
384 'GUEST_DPDK_BIND_DRIVER')[self._number], pci_list)
385 self.execute_and_wait('./*tools/dpdk*bind.py --status')
387 # build and run 'test-pmd'
388 self.execute_and_wait('cd ' + S.getValue('GUEST_OVS_DPDK_DIR')[self._number] +
389 '/DPDK/app/test-pmd')
390 self.execute_and_wait('make clean')
391 self.execute_and_wait('make')
393 # get testpmd settings from CLI
394 testpmd_params = S.getValue('GUEST_TESTPMD_PARAMS')[self._number]
395 if S.getValue('VSWITCH_JUMBO_FRAMES_ENABLED'):
396 testpmd_params += ' --max-pkt-len={}'.format(S.getValue(
397 'VSWITCH_JUMBO_FRAMES_SIZE'))
399 self.execute_and_wait('./testpmd {}'.format(testpmd_params), 60, "Done")
400 self.execute_and_wait('set fwd ' + self._testpmd_fwd_mode, 20, 'testpmd>')
401 self.execute_and_wait('start', 20, 'testpmd>')
403 def _configure_l2fwd(self):
405 Configure VM to perform L2 forwarding between NICs by l2fwd module
407 if int(S.getValue('GUEST_NIC_QUEUES')[self._number]):
408 self._set_multi_queue_nic()
409 self._configure_copy_sources('l2fwd')
410 self._configure_disable_firewall()
412 # configure all interfaces
413 for nic in self._nics:
414 self.execute_and_wait('ip addr add ' +
415 nic['ip'] + ' dev ' + nic['device'])
416 if S.getValue('VSWITCH_JUMBO_FRAMES_ENABLED'):
417 self.execute_and_wait('ifconfig {} mtu {}'.format(
418 nic['device'], S.getValue('VSWITCH_JUMBO_FRAMES_SIZE')))
419 self.execute_and_wait('ip link set dev ' + nic['device'] + ' up')
421 # build and configure system for l2fwd
422 self.execute_and_wait('cd ' + S.getValue('GUEST_OVS_DPDK_DIR')[self._number] +
424 self.execute_and_wait('export CC=gcc')
426 self.execute_and_wait('make')
427 if len(self._nics) == 2:
428 self.execute_and_wait('insmod ' + S.getValue('GUEST_OVS_DPDK_DIR')[self._number] +
429 '/l2fwd' + '/l2fwd.ko net1=' + self._nics[0]['device'] +
430 ' net2=' + self._nics[1]['device'])
432 raise RuntimeError('l2fwd can forward only between 2 NICs, but {} NICs are '
433 'configured inside GUEST'.format(len(self._nics)))
435 def _configure_linux_bridge(self):
437 Configure VM to perform L2 forwarding between NICs by linux bridge
439 if int(S.getValue('GUEST_NIC_QUEUES')[self._number]):
440 self._set_multi_queue_nic()
441 self._configure_disable_firewall()
443 # configure linux bridge
444 self.execute_and_wait('brctl addbr br0')
446 # add all NICs into the bridge
447 for nic in self._nics:
448 self.execute_and_wait('ip addr add ' + nic['ip'] + ' dev ' + nic['device'])
449 if S.getValue('VSWITCH_JUMBO_FRAMES_ENABLED'):
450 self.execute_and_wait('ifconfig {} mtu {}'.format(
451 nic['device'], S.getValue('VSWITCH_JUMBO_FRAMES_SIZE')))
452 self.execute_and_wait('ip link set dev ' + nic['device'] + ' up')
453 self.execute_and_wait('brctl addif br0 ' + nic['device'])
455 self.execute_and_wait('ip addr add {} dev br0'.format(
456 S.getValue('GUEST_BRIDGE_IP')[self._number]))
457 self.execute_and_wait('ip link set dev br0 up')
459 # Add the arp entries for the IXIA ports and the bridge you are using.
460 # Use command line values if provided.
461 trafficgen_mac = S.getValue('VANILLA_TGEN_PORT1_MAC')
462 trafficgen_ip = S.getValue('VANILLA_TGEN_PORT1_IP')
464 self.execute_and_wait('arp -s ' + trafficgen_ip + ' ' + trafficgen_mac)
466 trafficgen_mac = S.getValue('VANILLA_TGEN_PORT2_MAC')
467 trafficgen_ip = S.getValue('VANILLA_TGEN_PORT2_IP')
469 self.execute_and_wait('arp -s ' + trafficgen_ip + ' ' + trafficgen_mac)
472 self.execute_and_wait('sysctl -w net.ipv4.ip_forward=1')
474 # Controls source route verification
475 # 0 means no source validation
476 self.execute_and_wait('sysctl -w net.ipv4.conf.all.rp_filter=0')
477 for nic in self._nics:
478 self.execute_and_wait('sysctl -w net.ipv4.conf.' + nic['device'] +
481 def _bind_dpdk_driver(self, driver, pci_slots):
483 Bind the virtual nics to the driver specific in the conf file
486 if driver == 'uio_pci_generic':
487 if S.getValue('VNF') == 'QemuPciPassthrough':
488 # unsupported config, bind to igb_uio instead and exit the
489 # outer function after completion.
490 self._logger.error('SR-IOV does not support uio_pci_generic. '
491 'Igb_uio will be used instead.')
492 self._bind_dpdk_driver('igb_uio_from_src', pci_slots)
494 self.execute_and_wait('modprobe uio_pci_generic')
495 self.execute_and_wait('./*tools/dpdk*bind.py -b uio_pci_generic '+
497 elif driver == 'vfio_no_iommu':
498 self.execute_and_wait('modprobe -r vfio')
499 self.execute_and_wait('modprobe -r vfio_iommu_type1')
500 self.execute_and_wait('modprobe vfio enable_unsafe_noiommu_mode=Y')
501 self.execute_and_wait('modprobe vfio-pci')
502 self.execute_and_wait('./*tools/dpdk*bind.py -b vfio-pci ' +
504 elif driver == 'igb_uio_from_src':
505 # build and insert igb_uio and rebind interfaces to it
506 self.execute_and_wait('make RTE_OUTPUT=$RTE_SDK/$RTE_TARGET -C '
507 '$RTE_SDK/lib/librte_eal/linuxapp/igb_uio')
508 self.execute_and_wait('modprobe uio')
509 self.execute_and_wait('insmod %s/kmod/igb_uio.ko' %
510 S.getValue('RTE_TARGET'))
511 self.execute_and_wait('./*tools/dpdk*bind.py -b igb_uio ' + pci_slots)
514 'Unknown driver for binding specified, defaulting to igb_uio')
515 self._bind_dpdk_driver('igb_uio_from_src', pci_slots)
517 def _set_multi_queue_nic(self):
519 Enable multi-queue in guest kernel with ethool.
522 for nic in self._nics:
523 self.execute_and_wait('ethtool -L {} combined {}'.format(
524 nic['device'], S.getValue('GUEST_NIC_QUEUES')[self._number]))
525 self.execute_and_wait('ethtool -l {}'.format(nic['device']))