1 # Copyright 2015-2016 Intel Corporation.
3 # Licensed under the Apache License, Version 2.0 (the "License");
4 # you may not use this file except in compliance with the License.
5 # You may obtain a copy of the License at
7 # http://www.apache.org/licenses/LICENSE-2.0
9 # Unless required by applicable law or agreed to in writing, software
10 # distributed under the License is distributed on an "AS IS" BASIS,
11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 # See the License for the specific language governing permissions and
13 # limitations under the License.
15 """Automation of QEMU hypervisor for launching vhost-cuse enabled guests.
24 from conf import settings as S
25 from conf import get_test_param
26 from vnfs.vnf.vnf import IVnf
30 Abstract class for controling an instance of QEMU
33 _expect = S.getValue('GUEST_PROMPT_LOGIN')
36 class GuestCommandFilter(logging.Filter):
38 Filter out strings beginning with 'guestcmd :'.
40 def filter(self, record):
41 return record.getMessage().startswith(self.prefix)
45 Initialisation function.
47 super(IVnfQemu, self).__init__()
48 self._logger = logging.getLogger(__name__)
49 self._logfile = os.path.join(
50 S.getValue('LOG_DIR'),
51 S.getValue('LOG_FILE_QEMU')) + str(self._number)
52 self._timeout = S.getValue('GUEST_TIMEOUT')[self._number]
53 self._monitor = '%s/vm%dmonitor' % ('/tmp', self._number)
54 self._net1 = get_test_param('guest_nic1_name', None)
55 if self._net1 == None:
56 self._net1 = S.getValue('GUEST_NIC1_NAME')[self._number]
58 self._net1 = self._net1.split(',')[self._number]
59 self._net2 = get_test_param('guest_nic2_name', None)
60 if self._net2 == None:
61 self._net2 = S.getValue('GUEST_NIC2_NAME')[self._number]
63 self._net2 = self._net2.split(',')[self._number]
66 name = 'Client%d' % self._number
67 vnc = ':%d' % self._number
68 # don't use taskset to affinize main qemu process; It causes hangup
69 # of 2nd VM in case of DPDK. It also slows down VM responsivnes.
70 self._cmd = ['sudo', '-E', S.getValue('QEMU_BIN'),
71 '-m', S.getValue('GUEST_MEMORY')[self._number],
72 '-smp', str(S.getValue('GUEST_SMP')[self._number]),
73 '-cpu', 'host,migratable=off',
74 '-drive', 'if=scsi,file=' +
75 S.getValue('GUEST_IMAGE')[self._number],
76 '-boot', 'c', '--enable-kvm',
77 '-monitor', 'unix:%s,server,nowait' % self._monitor,
79 'memory-backend-file,id=mem,size=' +
80 str(S.getValue('GUEST_MEMORY')[self._number]) + 'M,' +
81 'mem-path=' + S.getValue('HUGEPAGE_DIR') + ',share=on',
82 '-numa', 'node,memdev=mem -mem-prealloc',
83 '-nographic', '-vnc', str(vnc), '-name', name,
84 '-snapshot', '-net none', '-no-reboot',
86 'if=scsi,format=raw,file=fat:rw:%s,snapshot=off' %
87 S.getValue('GUEST_SHARE_DIR')[self._number],
89 self._configure_logging()
91 def _configure_logging(self):
95 self.GuestCommandFilter.prefix = self._log_prefix
97 logger = logging.getLogger()
98 cmd_logger = logging.FileHandler(
99 filename=os.path.join(S.getValue('LOG_DIR'),
100 S.getValue('LOG_FILE_GUEST_CMDS')) +
102 cmd_logger.setLevel(logging.DEBUG)
103 cmd_logger.addFilter(self.GuestCommandFilter())
104 logger.addHandler(cmd_logger)
110 Start QEMU instance, login and prepare for commands.
112 super(IVnfQemu, self).start()
113 if S.getValue('VNF_AFFINITIZATION_ON'):
117 self._config_guest_loopback()
121 def _login(self, timeout=120):
123 Login to QEMU instance.
125 This can be used immediately after booting the machine, provided a
126 sufficiently long ``timeout`` is given.
128 :param timeout: Timeout to wait for login to complete.
132 # if no timeout was set, we likely started QEMU without waiting for it
133 # to boot. This being the case, we best check that it has finished
135 if not self._timeout:
136 self._expect_process(timeout=timeout)
138 self._child.sendline(S.getValue('GUEST_USERNAME'))
139 self._child.expect(S.getValue('GUEST_PROMPT_PASSWORD'), timeout=5)
140 self._child.sendline(S.getValue('GUEST_PASSWORD'))
142 self._expect_process(S.getValue('GUEST_PROMPT'), timeout=5)
144 def send_and_pass(self, cmd, timeout=30):
146 Send ``cmd`` and wait ``timeout`` seconds for it to pass.
148 :param cmd: Command to send to guest.
149 :param timeout: Time to wait for prompt before checking return code.
154 self.wait(S.getValue('GUEST_PROMPT'), timeout=timeout)
155 self.execute('echo $?')
156 self._child.expect('^0$', timeout=1) # expect a 0
157 self.wait(S.getValue('GUEST_PROMPT'), timeout=timeout)
159 def _affinitize(self):
161 Affinitize the SMP cores of a QEMU instance.
163 This is a bit of a hack. The 'socat' utility is used to
164 interact with the QEMU HMP. This is necessary due to the lack
165 of QMP in older versions of QEMU, like v1.6.2. In future
166 releases, this should be replaced with calls to libvirt or
167 another Python-QEMU wrapper library.
171 thread_id = (r'.* CPU #%d: .* thread_id=(\d+)')
173 self._logger.info('Affinitizing guest...')
175 cur_locale = locale.getdefaultlocale()[1]
176 proc = subprocess.Popen(
177 ('echo', 'info cpus'), stdout=subprocess.PIPE)
178 output = subprocess.check_output(
179 ('sudo', 'socat', '-', 'UNIX-CONNECT:%s' % self._monitor),
183 for cpu in range(0, int(S.getValue('GUEST_SMP')[self._number])):
185 for line in output.decode(cur_locale).split('\n'):
186 match = re.search(thread_id % cpu, line)
188 self._affinitize_pid(
189 S.getValue('GUEST_CORE_BINDING')[self._number][cpu],
194 self._logger.error('Failed to affinitize guest core #%d. Could'
195 ' not parse tid.', cpu)
197 def _config_guest_loopback(self):
199 Configure VM to run VNF (e.g. port forwarding application)
201 # set guest loopback application based on VNF configuration
202 # cli option take precedence to config file values
203 guest_loopback = S.getValue('GUEST_LOOPBACK')[self._number]
205 if guest_loopback == 'testpmd':
207 self._configure_testpmd()
208 elif guest_loopback == 'l2fwd':
210 self._configure_l2fwd()
211 elif guest_loopback == 'linux_bridge':
213 self._configure_linux_bridge()
214 elif guest_loopback != 'buildin':
215 self._logger.error('Unsupported guest loopback method "%s" was specified. Option'
216 ' "buildin" will be used as a fallback.', guest_loopback)
218 def wait(self, prompt=S.getValue('GUEST_PROMPT'), timeout=30):
219 super(IVnfQemu, self).wait(prompt=prompt, timeout=timeout)
221 def execute_and_wait(self, cmd, timeout=30,
222 prompt=S.getValue('GUEST_PROMPT')):
223 super(IVnfQemu, self).execute_and_wait(cmd, timeout=timeout,
226 def _modify_dpdk_makefile(self):
228 Modifies DPDK makefile in Guest before compilation
232 def _configure_copy_sources(self, dirname):
234 Mount shared directory and copy DPDK and l2fwd sources
236 # mount shared directory
237 self.execute_and_wait('umount ' + S.getValue('OVS_DPDK_SHARE'))
238 self.execute_and_wait('rm -rf ' + S.getValue('GUEST_OVS_DPDK_DIR'))
239 self.execute_and_wait('mkdir -p ' + S.getValue('OVS_DPDK_SHARE'))
240 self.execute_and_wait('mount -o iocharset=utf8 /dev/sdb1 ' +
241 S.getValue('OVS_DPDK_SHARE'))
242 self.execute_and_wait('mkdir -p ' + S.getValue('GUEST_OVS_DPDK_DIR'))
243 self.execute_and_wait('cp -ra ' + os.path.join(S.getValue('OVS_DPDK_SHARE'), dirname) +
244 ' ' + S.getValue('GUEST_OVS_DPDK_DIR'))
246 def _configure_disable_firewall(self):
248 Disable firewall in VM
250 for iptables in ['iptables', 'ip6tables']:
252 for chain in ['INPUT', 'FORWARD', 'OUTPUT']:
253 self.execute_and_wait("{} -t filter -P {} ACCEPT".format(iptables, chain))
255 for chain in ['PREROUTING', 'INPUT', 'FORWARD', 'OUTPUT', 'POSTROUTING']:
256 self.execute_and_wait("{} -t mangle -P {} ACCEPT".format(iptables, chain))
258 for chain in ['PREROUTING', 'INPUT', 'OUTPUT', 'POSTROUTING']:
259 self.execute_and_wait("{} -t nat -P {} ACCEPT".format(iptables, chain))
261 # flush rules and delete chains created by user
262 for table in ['filter', 'mangle', 'nat']:
263 self.execute_and_wait("{} -t {} -F".format(iptables, table))
264 self.execute_and_wait("{} -t {} -X".format(iptables, table))
267 def _configure_testpmd(self):
269 Configure VM to perform L2 forwarding between NICs by DPDK's testpmd
271 self._configure_copy_sources('DPDK')
272 self._configure_disable_firewall()
274 # Guest images _should_ have 1024 hugepages by default,
275 # but just in case:'''
276 self.execute_and_wait('sysctl vm.nr_hugepages=1024')
279 self.execute_and_wait('mkdir -p /dev/hugepages')
280 self.execute_and_wait(
281 'mount -t hugetlbfs hugetlbfs /dev/hugepages')
283 # build and configure system for dpdk
284 self.execute_and_wait('cd ' + S.getValue('GUEST_OVS_DPDK_DIR') +
286 self.execute_and_wait('export CC=gcc')
287 self.execute_and_wait('export RTE_SDK=' +
288 S.getValue('GUEST_OVS_DPDK_DIR') + '/DPDK')
289 self.execute_and_wait('export RTE_TARGET=%s' % S.getValue('RTE_TARGET'))
291 # modify makefile if needed
292 self._modify_dpdk_makefile()
294 self.execute_and_wait('make RTE_OUTPUT=$RTE_SDK/$RTE_TARGET -C '
295 '$RTE_SDK/lib/librte_eal/linuxapp/igb_uio')
296 self.execute_and_wait('modprobe uio')
297 self.execute_and_wait('insmod %s/kmod/igb_uio.ko' %
298 S.getValue('RTE_TARGET'))
299 self.execute_and_wait('./tools/dpdk_nic_bind.py --status')
300 self.execute_and_wait(
301 './tools/dpdk_nic_bind.py -b igb_uio' ' ' +
302 S.getValue('GUEST_NET1_PCI_ADDRESS')[self._number] + ' ' +
303 S.getValue('GUEST_NET2_PCI_ADDRESS')[self._number])
305 # build and run 'test-pmd'
306 self.execute_and_wait('cd ' + S.getValue('GUEST_OVS_DPDK_DIR') +
307 '/DPDK/app/test-pmd')
308 self.execute_and_wait('make clean')
309 self.execute_and_wait('make')
310 self.execute_and_wait('./testpmd -c 0x3 -n 4 --socket-mem 512 --'
311 ' --burst=64 -i --txqflags=0xf00 ' +
312 '--disable-hw-vlan', 60, "Done")
313 self.execute('set fwd mac_retry', 1)
314 self.execute_and_wait('start', 20,
315 'TX RS bit threshold=0 - TXQ flags=0xf00')
317 def _configure_l2fwd(self):
319 Configure VM to perform L2 forwarding between NICs by l2fwd module
321 self._configure_copy_sources('l2fwd')
322 self._configure_disable_firewall()
324 # build and configure system for l2fwd
325 self.execute_and_wait('cd ' + S.getValue('GUEST_OVS_DPDK_DIR') +
327 self.execute_and_wait('export CC=gcc')
329 self.execute_and_wait('make')
330 self.execute_and_wait('insmod ' + S.getValue('GUEST_OVS_DPDK_DIR') +
331 '/l2fwd' + '/l2fwd.ko net1=' + self._net1 +
332 ' net2=' + self._net2)
334 def _configure_linux_bridge(self):
336 Configure VM to perform L2 forwarding between NICs by linux bridge
338 self._configure_disable_firewall()
340 self.execute('ifconfig ' + self._net1 + ' ' +
341 S.getValue('VANILLA_NIC1_IP_CIDR')[self._number])
343 self.execute('ifconfig ' + self._net2 + ' ' +
344 S.getValue('VANILLA_NIC2_IP_CIDR')[self._number])
346 # configure linux bridge
347 self.execute('brctl addbr br0')
348 self.execute('brctl addif br0 ' + self._net1 + ' ' + self._net2)
349 self.execute('ifconfig br0 ' +
350 S.getValue('VANILLA_BRIDGE_IP')[self._number])
352 # Add the arp entries for the IXIA ports and the bridge you are using.
353 # Use command line values if provided.
354 trafficgen_mac = get_test_param('vanilla_tgen_port1_mac',
355 S.getValue('VANILLA_TGEN_PORT1_MAC'))
356 trafficgen_ip = get_test_param('vanilla_tgen_port1_ip',
357 S.getValue('VANILLA_TGEN_PORT1_IP'))
359 self.execute('arp -s ' + trafficgen_ip + ' ' + trafficgen_mac)
361 trafficgen_mac = get_test_param('vanilla_tgen_port2_mac',
362 S.getValue('VANILLA_TGEN_PORT2_MAC'))
363 trafficgen_ip = get_test_param('vanilla_tgen_port2_ip',
364 S.getValue('VANILLA_TGEN_PORT2_IP'))
366 self.execute('arp -s ' + trafficgen_ip + ' ' + trafficgen_mac)
369 self.execute('sysctl -w net.ipv4.ip_forward=1')
371 # Controls source route verification
372 # 0 means no source validation
373 self.execute('sysctl -w net.ipv4.conf.all.rp_filter=0')
374 self.execute('sysctl -w net.ipv4.conf.' + self._net1 + '.rp_filter=0')
375 self.execute('sysctl -w net.ipv4.conf.' + self._net2 + '.rp_filter=0')