vnfs/qemu/qemu.py

   1 # Copyright 2015-2016 Intel Corporation.
   2 #
   3 # Licensed under the Apache License, Version 2.0 (the "License");
   4 # you may not use this file except in compliance with the License.
   5 # You may obtain a copy of the License at
   6 #
   7 #   http://www.apache.org/licenses/LICENSE-2.0
   8 #
   9 # Unless required by applicable law or agreed to in writing, software
  10 # distributed under the License is distributed on an "AS IS" BASIS,
  11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12 # See the License for the specific language governing permissions and
  13 # limitations under the License.
  14
  15 """Automation of QEMU hypervisor for launching guests.
  16 """
  17
  18 import os
  19 import logging
  20 import locale
  21 import re
  22 import subprocess
  23 import time
  24 import pexpect
  25
  26 from conf import settings as S
  27 from conf import get_test_param
  28 from vnfs.vnf.vnf import IVnf
  29
  30 class IVnfQemu(IVnf):
  31     """
  32     Abstract class for controling an instance of QEMU
  33     """
  34     _cmd = None
  35     _expect = S.getValue('GUEST_PROMPT_LOGIN')
  36     _proc_name = 'qemu'
  37
  38     class GuestCommandFilter(logging.Filter):
  39         """
  40         Filter out strings beginning with 'guestcmd :'.
  41         """
  42         def filter(self, record):
  43             return record.getMessage().startswith(self.prefix)
  44
  45     def __init__(self):
  46         """
  47         Initialisation function.
  48         """
  49         super(IVnfQemu, self).__init__()
  50         self._logger = logging.getLogger(__name__)
  51         self._logfile = os.path.join(
  52             S.getValue('LOG_DIR'),
  53             S.getValue('LOG_FILE_QEMU')) + str(self._number)
  54         self._timeout = S.getValue('GUEST_TIMEOUT')[self._number]
  55         self._monitor = '%s/vm%dmonitor' % ('/tmp', self._number)
  56         self._net1 = get_test_param('guest_nic1_name', None)
  57         if self._net1 == None:
  58             self._net1 = S.getValue('GUEST_NIC1_NAME')[self._number]
  59         else:
  60             self._net1 = self._net1.split(',')[self._number]
  61         self._net2 = get_test_param('guest_nic2_name', None)
  62         if self._net2 == None:
  63             self._net2 = S.getValue('GUEST_NIC2_NAME')[self._number]
  64         else:
  65             self._net2 = self._net2.split(',')[self._number]
  66
  67         # set guest loopback application based on VNF configuration
  68         # cli option take precedence to config file values
  69         self._guest_loopback = S.getValue('GUEST_LOOPBACK')[self._number]
  70
  71         self._testpmd_fwd_mode = S.getValue('GUEST_TESTPMD_FWD_MODE')
  72         # in case of SRIOV we must ensure, that MAC addresses are not swapped
  73         if S.getValue('SRIOV_ENABLED') and self._testpmd_fwd_mode.startswith('mac') and \
  74            not S.getValue('VNF').endswith('PciPassthrough'):
  75
  76             self._logger.info("SRIOV detected, forwarding mode of testpmd was changed from '%s' to '%s'",
  77                               self._testpmd_fwd_mode, 'io')
  78             self._testpmd_fwd_mode = 'io'
  79
  80         name = 'Client%d' % self._number
  81         vnc = ':%d' % self._number
  82         # don't use taskset to affinize main qemu process; It causes hangup
  83         # of 2nd VM in case of DPDK. It also slows down VM responsivnes.
  84         self._cmd = ['sudo', '-E', S.getValue('QEMU_BIN'),
  85                      '-m', S.getValue('GUEST_MEMORY')[self._number],
  86                      '-smp', str(S.getValue('GUEST_SMP')[self._number]),
  87                      '-cpu', 'host,migratable=off',
  88                      '-drive', 'if={},file='.format(S.getValue(
  89                         'GUEST_BOOT_DRIVE_TYPE')) +
  90                      S.getValue('GUEST_IMAGE')[self._number],
  91                      '-boot', 'c', '--enable-kvm',
  92                      '-monitor', 'unix:%s,server,nowait' % self._monitor,
  93                      '-object',
  94                      'memory-backend-file,id=mem,size=' +
  95                      str(S.getValue('GUEST_MEMORY')[self._number]) + 'M,' +
  96                      'mem-path=' + S.getValue('HUGEPAGE_DIR') + ',share=on',
  97                      '-numa', 'node,memdev=mem -mem-prealloc',
  98                      '-nographic', '-vnc', str(vnc), '-name', name,
  99                      '-snapshot', '-net none', '-no-reboot',
 100                      '-drive',
 101                      'if=%s,format=raw,file=fat:rw:%s,snapshot=off' %
 102                      (S.getValue('GUEST_SHARED_DRIVE_TYPE'),
 103                       S.getValue('GUEST_SHARE_DIR')[self._number]),
 104                     ]
 105         self._configure_logging()
 106
 107     def _configure_logging(self):
 108         """
 109         Configure logging.
 110         """
 111         self.GuestCommandFilter.prefix = self._log_prefix
 112
 113         logger = logging.getLogger()
 114         cmd_logger = logging.FileHandler(
 115             filename=os.path.join(S.getValue('LOG_DIR'),
 116                                   S.getValue('LOG_FILE_GUEST_CMDS')) +
 117             str(self._number))
 118         cmd_logger.setLevel(logging.DEBUG)
 119         cmd_logger.addFilter(self.GuestCommandFilter())
 120         logger.addHandler(cmd_logger)
 121
 122     # startup/Shutdown
 123
 124     def start(self):
 125         """
 126         Start QEMU instance, login and prepare for commands.
 127         """
 128         super(IVnfQemu, self).start()
 129         if S.getValue('VNF_AFFINITIZATION_ON'):
 130             self._affinitize()
 131
 132         if S.getValue('VSWITCH_VHOST_NET_AFFINITIZATION') and S.getValue(
 133                 'VNF') == 'QemuVirtioNet':
 134             self._affinitize_vhost_net()
 135
 136         if self._timeout:
 137             self._config_guest_loopback()
 138
 139     def stop(self):
 140         """
 141         Stops VNF instance gracefully first.
 142         """
 143         try:
 144             # exit testpmd if needed
 145             if self._guest_loopback == 'testpmd':
 146                 self.execute_and_wait('stop', 120, "Done")
 147                 self.execute_and_wait('quit', 120, "[bB]ye")
 148
 149             # turn off VM
 150             self.execute_and_wait('poweroff', 120, "Power down")
 151
 152         except pexpect.TIMEOUT:
 153             self.kill()
 154
 155         # wait until qemu shutdowns
 156         self._logger.debug('Wait for QEMU to terminate')
 157         for dummy in range(30):
 158             time.sleep(1)
 159             if not self.is_running():
 160                 break
 161
 162         # just for case that graceful shutdown failed
 163         super(IVnfQemu, self).stop()
 164
 165     # helper functions
 166
 167     def _login(self, timeout=120):
 168         """
 169         Login to QEMU instance.
 170
 171         This can be used immediately after booting the machine, provided a
 172         sufficiently long ``timeout`` is given.
 173
 174         :param timeout: Timeout to wait for login to complete.
 175
 176         :returns: None
 177         """
 178         # if no timeout was set, we likely started QEMU without waiting for it
 179         # to boot. This being the case, we best check that it has finished
 180         # first.
 181         if not self._timeout:
 182             self._expect_process(timeout=timeout)
 183
 184         self._child.sendline(S.getValue('GUEST_USERNAME'))
 185         self._child.expect(S.getValue('GUEST_PROMPT_PASSWORD'), timeout=5)
 186         self._child.sendline(S.getValue('GUEST_PASSWORD'))
 187
 188         self._expect_process(S.getValue('GUEST_PROMPT'), timeout=5)
 189
 190     def send_and_pass(self, cmd, timeout=30):
 191         """
 192         Send ``cmd`` and wait ``timeout`` seconds for it to pass.
 193
 194         :param cmd: Command to send to guest.
 195         :param timeout: Time to wait for prompt before checking return code.
 196
 197         :returns: None
 198         """
 199         self.execute(cmd)
 200         self.wait(S.getValue('GUEST_PROMPT'), timeout=timeout)
 201         self.execute('echo $?')
 202         self._child.expect('^0$', timeout=1)  # expect a 0
 203         self.wait(S.getValue('GUEST_PROMPT'), timeout=timeout)
 204
 205     def _affinitize(self):
 206         """
 207         Affinitize the SMP cores of a QEMU instance.
 208
 209         This is a bit of a hack. The 'socat' utility is used to
 210         interact with the QEMU HMP. This is necessary due to the lack
 211         of QMP in older versions of QEMU, like v1.6.2. In future
 212         releases, this should be replaced with calls to libvirt or
 213         another Python-QEMU wrapper library.
 214
 215         :returns: None
 216         """
 217         thread_id = (r'.* CPU #%d: .* thread_id=(\d+)')
 218
 219         self._logger.info('Affinitizing guest...')
 220
 221         cur_locale = locale.getdefaultlocale()[1]
 222         proc = subprocess.Popen(
 223             ('echo', 'info cpus'), stdout=subprocess.PIPE)
 224         output = subprocess.check_output(
 225             ('sudo', 'socat', '-', 'UNIX-CONNECT:%s' % self._monitor),
 226             stdin=proc.stdout)
 227         proc.wait()
 228
 229         for cpu in range(0, int(S.getValue('GUEST_SMP')[self._number])):
 230             match = None
 231             for line in output.decode(cur_locale).split('\n'):
 232                 match = re.search(thread_id % cpu, line)
 233                 if match:
 234                     self._affinitize_pid(
 235                         S.getValue('GUEST_CORE_BINDING')[self._number][cpu],
 236                         match.group(1))
 237                     break
 238
 239             if not match:
 240                 self._logger.error('Failed to affinitize guest core #%d. Could'
 241                                    ' not parse tid.', cpu)
 242
 243     def _affinitize_vhost_net(self):
 244         """
 245         Affinitize the vhost net threads for Vanilla OVS and guest nic queues.
 246
 247         :return: None
 248         """
 249         self._logger.info('Affinitizing VHOST Net threads.')
 250         args1 = ['ps', 'ax']
 251         process1 = subprocess.Popen(args1, stdout=subprocess.PIPE,
 252                                     shell=False)
 253         out = process1.communicate()[0]
 254         processes = list()
 255         for line in out.decode(locale.getdefaultlocale()[1]).split('\n'):
 256             if re.search('\[vhost-(\d+)', line):
 257                 processes.append(re.match('\s*(\d+)', line).group(1))
 258         self._logger.info('Found %s vhost net threads...', len(processes))
 259
 260         cpumap = S.getValue('VSWITCH_VHOST_CPU_MAP')
 261         mapcount = 0
 262         for proc in processes:
 263             self._affinitize_pid(cpumap[mapcount], proc)
 264             mapcount += 1
 265             if mapcount + 1 > len(cpumap):
 266                 # Not enough cpus were given in the mapping to cover all the
 267                 # threads on a 1 to 1 ratio with cpus so reset the list counter
 268                 #  to 0.
 269                 mapcount = 0
 270
 271     def _config_guest_loopback(self):
 272         """
 273         Configure VM to run VNF, e.g. port forwarding application based on the configuration
 274         """
 275         if self._guest_loopback == 'testpmd':
 276             self._login()
 277             self._configure_testpmd()
 278         elif self._guest_loopback == 'l2fwd':
 279             self._login()
 280             self._configure_l2fwd()
 281         elif self._guest_loopback == 'linux_bridge':
 282             self._login()
 283             self._configure_linux_bridge()
 284         elif self._guest_loopback != 'buildin':
 285             self._logger.error('Unsupported guest loopback method "%s" was specified. Option'
 286                                ' "buildin" will be used as a fallback.', self._guest_loopback)
 287
 288     def wait(self, prompt=S.getValue('GUEST_PROMPT'), timeout=30):
 289         super(IVnfQemu, self).wait(prompt=prompt, timeout=timeout)
 290
 291     def execute_and_wait(self, cmd, timeout=30,
 292                          prompt=S.getValue('GUEST_PROMPT')):
 293         super(IVnfQemu, self).execute_and_wait(cmd, timeout=timeout,
 294                                                prompt=prompt)
 295
 296     def _modify_dpdk_makefile(self):
 297         """
 298         Modifies DPDK makefile in Guest before compilation if needed
 299         """
 300         pass
 301
 302     def _configure_copy_sources(self, dirname):
 303         """
 304         Mount shared directory and copy DPDK and l2fwd sources
 305         """
 306         # mount shared directory
 307         self.execute_and_wait('umount /dev/sdb1')
 308         self.execute_and_wait('rm -rf ' + S.getValue('GUEST_OVS_DPDK_DIR'))
 309         self.execute_and_wait('mkdir -p ' + S.getValue('OVS_DPDK_SHARE'))
 310         self.execute_and_wait('mount -o ro,iocharset=utf8 /dev/sdb1 ' +
 311                               S.getValue('OVS_DPDK_SHARE'))
 312         self.execute_and_wait('mkdir -p ' + S.getValue('GUEST_OVS_DPDK_DIR'))
 313         self.execute_and_wait('cp -r ' + os.path.join(S.getValue('OVS_DPDK_SHARE'), dirname) +
 314                               ' ' + S.getValue('GUEST_OVS_DPDK_DIR'))
 315         self.execute_and_wait('umount /dev/sdb1')
 316
 317     def _configure_disable_firewall(self):
 318         """
 319         Disable firewall in VM
 320         """
 321         for iptables in ['iptables', 'ip6tables']:
 322             # filter table
 323             for chain in ['INPUT', 'FORWARD', 'OUTPUT']:
 324                 self.execute_and_wait("{} -t filter -P {} ACCEPT".format(iptables, chain))
 325             # mangle table
 326             for chain in ['PREROUTING', 'INPUT', 'FORWARD', 'OUTPUT', 'POSTROUTING']:
 327                 self.execute_and_wait("{} -t mangle -P {} ACCEPT".format(iptables, chain))
 328             # nat table
 329             for chain in ['PREROUTING', 'INPUT', 'OUTPUT', 'POSTROUTING']:
 330                 self.execute_and_wait("{} -t nat -P {} ACCEPT".format(iptables, chain))
 331
 332             # flush rules and delete chains created by user
 333             for table in ['filter', 'mangle', 'nat']:
 334                 self.execute_and_wait("{} -t {} -F".format(iptables, table))
 335                 self.execute_and_wait("{} -t {} -X".format(iptables, table))
 336
 337
 338     def _configure_testpmd(self):
 339         """
 340         Configure VM to perform L2 forwarding between NICs by DPDK's testpmd
 341         """
 342         self._configure_copy_sources('DPDK')
 343         self._configure_disable_firewall()
 344
 345         # Guest images _should_ have 1024 hugepages by default,
 346         # but just in case:'''
 347         self.execute_and_wait('sysctl vm.nr_hugepages=1024')
 348
 349         # Mount hugepages
 350         self.execute_and_wait('mkdir -p /dev/hugepages')
 351         self.execute_and_wait(
 352             'mount -t hugetlbfs hugetlbfs /dev/hugepages')
 353
 354         # build and configure system for dpdk
 355         self.execute_and_wait('cd ' + S.getValue('GUEST_OVS_DPDK_DIR') +
 356                               '/DPDK')
 357         self.execute_and_wait('export CC=gcc')
 358         self.execute_and_wait('export RTE_SDK=' +
 359                               S.getValue('GUEST_OVS_DPDK_DIR') + '/DPDK')
 360         self.execute_and_wait('export RTE_TARGET=%s' % S.getValue('RTE_TARGET'))
 361
 362         # modify makefile if needed
 363         self._modify_dpdk_makefile()
 364
 365         # disable network interfaces, so DPDK can take care of them
 366         self.execute_and_wait('ifdown ' + self._net1)
 367         self.execute_and_wait('ifdown ' + self._net2)
 368
 369         # build and insert igb_uio and rebind interfaces to it
 370         self.execute_and_wait('make RTE_OUTPUT=$RTE_SDK/$RTE_TARGET -C '
 371                               '$RTE_SDK/lib/librte_eal/linuxapp/igb_uio')
 372         self.execute_and_wait('modprobe uio')
 373         self.execute_and_wait('insmod %s/kmod/igb_uio.ko' %
 374                               S.getValue('RTE_TARGET'))
 375         self.execute_and_wait('./tools/dpdk*bind.py --status')
 376         self.execute_and_wait(
 377             './tools/dpdk*bind.py -u' ' ' +
 378             S.getValue('GUEST_NET1_PCI_ADDRESS')[self._number] + ' ' +
 379             S.getValue('GUEST_NET2_PCI_ADDRESS')[self._number])
 380         self.execute_and_wait(
 381             './tools/dpdk*bind.py -b igb_uio' ' ' +
 382             S.getValue('GUEST_NET1_PCI_ADDRESS')[self._number] + ' ' +
 383             S.getValue('GUEST_NET2_PCI_ADDRESS')[self._number])
 384         self.execute_and_wait('./tools/dpdk*bind.py --status')
 385
 386         # build and run 'test-pmd'
 387         self.execute_and_wait('cd ' + S.getValue('GUEST_OVS_DPDK_DIR') +
 388                               '/DPDK/app/test-pmd')
 389         self.execute_and_wait('make clean')
 390         self.execute_and_wait('make')
 391         if int(S.getValue('GUEST_NIC_QUEUES')):
 392             self.execute_and_wait(
 393                 './testpmd {} -n4 --socket-mem 512 --'.format(
 394                     S.getValue('GUEST_TESTPMD_CPU_MASK')) +
 395                 ' --burst=64 -i --txqflags=0xf00 ' +
 396                 '--nb-cores={} --rxq={} --txq={} '.format(
 397                     S.getValue('GUEST_TESTPMD_NB_CORES'),
 398                     S.getValue('GUEST_TESTPMD_TXQ'),
 399                     S.getValue('GUEST_TESTPMD_RXQ')) +
 400                 '--disable-hw-vlan', 60, "Done")
 401         else:
 402             self.execute_and_wait(
 403                 './testpmd {} -n 4 --socket-mem 512 --'.format(
 404                     S.getValue('GUEST_TESTPMD_CPU_MASK')) +
 405                 ' --burst=64 -i --txqflags=0xf00 ' +
 406                 '--disable-hw-vlan', 60, "Done")
 407         self.execute('set fwd ' + self._testpmd_fwd_mode, 1)
 408         self.execute_and_wait('start', 20,
 409                               'TX RS bit threshold=.+ - TXQ flags=0xf00')
 410
 411     def _configure_l2fwd(self):
 412         """
 413         Configure VM to perform L2 forwarding between NICs by l2fwd module
 414         """
 415         if int(S.getValue('GUEST_NIC_QUEUES')):
 416             self._set_multi_queue_nic()
 417         self._configure_copy_sources('l2fwd')
 418         self._configure_disable_firewall()
 419
 420         # build and configure system for l2fwd
 421         self.execute_and_wait('cd ' + S.getValue('GUEST_OVS_DPDK_DIR') +
 422                               '/l2fwd')
 423         self.execute_and_wait('export CC=gcc')
 424
 425         self.execute_and_wait('make')
 426         self.execute_and_wait('insmod ' + S.getValue('GUEST_OVS_DPDK_DIR') +
 427                               '/l2fwd' + '/l2fwd.ko net1=' + self._net1 +
 428                               ' net2=' + self._net2)
 429
 430     def _configure_linux_bridge(self):
 431         """
 432         Configure VM to perform L2 forwarding between NICs by linux bridge
 433         """
 434         if int(S.getValue('GUEST_NIC_QUEUES')):
 435             self._set_multi_queue_nic()
 436         self._configure_disable_firewall()
 437
 438         self.execute('ip addr add ' +
 439                      S.getValue('VANILLA_NIC1_IP_CIDR')[self._number] +
 440                      ' dev ' + self._net1)
 441         self.execute('ip link set dev ' + self._net1 + ' up')
 442
 443         self.execute('ip addr add ' +
 444                      S.getValue('VANILLA_NIC2_IP_CIDR')[self._number] +
 445                      ' dev ' + self._net2)
 446         self.execute('ip link set dev ' + self._net2 + ' up')
 447
 448         # configure linux bridge
 449         self.execute('brctl addbr br0')
 450         self.execute('brctl addif br0 ' + self._net1 + ' ' + self._net2)
 451         self.execute('ip addr add ' +
 452                      S.getValue('VANILLA_BRIDGE_IP')[self._number] +
 453                      ' dev br0')
 454         self.execute('ip link set dev br0 up')
 455
 456         # Add the arp entries for the IXIA ports and the bridge you are using.
 457         # Use command line values if provided.
 458         trafficgen_mac = get_test_param('vanilla_tgen_port1_mac',
 459                                         S.getValue('VANILLA_TGEN_PORT1_MAC'))
 460         trafficgen_ip = get_test_param('vanilla_tgen_port1_ip',
 461                                        S.getValue('VANILLA_TGEN_PORT1_IP'))
 462
 463         self.execute('arp -s ' + trafficgen_ip + ' ' + trafficgen_mac)
 464
 465         trafficgen_mac = get_test_param('vanilla_tgen_port2_mac',
 466                                         S.getValue('VANILLA_TGEN_PORT2_MAC'))
 467         trafficgen_ip = get_test_param('vanilla_tgen_port2_ip',
 468                                        S.getValue('VANILLA_TGEN_PORT2_IP'))
 469
 470         self.execute('arp -s ' + trafficgen_ip + ' ' + trafficgen_mac)
 471
 472         # Enable forwarding
 473         self.execute('sysctl -w net.ipv4.ip_forward=1')
 474
 475         # Controls source route verification
 476         # 0 means no source validation
 477         self.execute('sysctl -w net.ipv4.conf.all.rp_filter=0')
 478         self.execute('sysctl -w net.ipv4.conf.' + self._net1 + '.rp_filter=0')
 479         self.execute('sysctl -w net.ipv4.conf.' + self._net2 + '.rp_filter=0')
 480
 481     def _set_multi_queue_nic(self):
 482         """
 483         Enable multi-queue in guest kernel with ethool.
 484         :return: None
 485         """
 486         self.execute_and_wait('ethtool -L {} combined {}'.format(
 487             self._net1, S.getValue('GUEST_NIC_QUEUES')))
 488         self.execute_and_wait('ethtool -l {}'.format(self._net1))
 489         self.execute_and_wait('ethtool -L {} combined {}'.format(
 490             self._net2, S.getValue('GUEST_NIC_QUEUES')))
 491         self.execute_and_wait('ethtool -l {}'.format(self._net2))