vnfs/qemu/qemu.py

   1 # Copyright 2015-2016 Intel Corporation.
   2 #
   3 # Licensed under the Apache License, Version 2.0 (the "License");
   4 # you may not use this file except in compliance with the License.
   5 # You may obtain a copy of the License at
   6 #
   7 #   http://www.apache.org/licenses/LICENSE-2.0
   8 #
   9 # Unless required by applicable law or agreed to in writing, software
  10 # distributed under the License is distributed on an "AS IS" BASIS,
  11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12 # See the License for the specific language governing permissions and
  13 # limitations under the License.
  14
  15 """Automation of QEMU hypervisor for launching guests.
  16 """
  17
  18 import os
  19 import logging
  20 import locale
  21 import re
  22 import subprocess
  23 import time
  24 import pexpect
  25
  26 from conf import settings as S
  27 from conf import get_test_param
  28 from vnfs.vnf.vnf import IVnf
  29
  30 class IVnfQemu(IVnf):
  31     """
  32     Abstract class for controling an instance of QEMU
  33     """
  34     _cmd = None
  35     _expect = None
  36     _proc_name = 'qemu'
  37
  38     class GuestCommandFilter(logging.Filter):
  39         """
  40         Filter out strings beginning with 'guestcmd :'.
  41         """
  42         def filter(self, record):
  43             return record.getMessage().startswith(self.prefix)
  44
  45     def __init__(self):
  46         """
  47         Initialisation function.
  48         """
  49         super(IVnfQemu, self).__init__()
  50
  51         self._expect = S.getValue('GUEST_PROMPT_LOGIN')[self._number]
  52         self._logger = logging.getLogger(__name__)
  53         self._logfile = os.path.join(
  54             S.getValue('LOG_DIR'),
  55             S.getValue('LOG_FILE_QEMU')) + str(self._number)
  56         self._timeout = S.getValue('GUEST_TIMEOUT')[self._number]
  57         self._monitor = '%s/vm%dmonitor' % ('/tmp', self._number)
  58         # read GUEST NICs configuration and use only defined NR of NICS
  59         nics_nr = S.getValue('GUEST_NICS_NR')[self._number]
  60         # and inform user about missconfiguration
  61         if nics_nr < 1:
  62             raise RuntimeError('At least one VM NIC is mandotory, but {} '
  63                                'NICs are configured'.format(nics_nr))
  64         elif nics_nr > 1 and nics_nr % 2:
  65             nics_nr = int(nics_nr / 2) * 2
  66             self._logger.warning('Odd number of NICs is configured, only '
  67                                  '%s NICs will be used', nics_nr)
  68
  69         self._nics = S.getValue('GUEST_NICS')[self._number][:nics_nr]
  70
  71         # set guest loopback application based on VNF configuration
  72         # cli option take precedence to config file values
  73         self._guest_loopback = S.getValue('GUEST_LOOPBACK')[self._number]
  74
  75         self._testpmd_fwd_mode = S.getValue('GUEST_TESTPMD_FWD_MODE')[self._number]
  76         # in case of SRIOV we must ensure, that MAC addresses are not swapped
  77         if S.getValue('SRIOV_ENABLED') and self._testpmd_fwd_mode.startswith('mac') and \
  78            not S.getValue('VNF').endswith('PciPassthrough'):
  79
  80             self._logger.info("SRIOV detected, forwarding mode of testpmd was changed from '%s' to '%s'",
  81                               self._testpmd_fwd_mode, 'io')
  82             self._testpmd_fwd_mode = 'io'
  83
  84         name = 'Client%d' % self._number
  85         vnc = ':%d' % self._number
  86         # don't use taskset to affinize main qemu process; It causes hangup
  87         # of 2nd VM in case of DPDK. It also slows down VM responsivnes.
  88         self._cmd = ['sudo', '-E', S.getValue('QEMU_BIN'),
  89                      '-m', S.getValue('GUEST_MEMORY')[self._number],
  90                      '-smp', str(S.getValue('GUEST_SMP')[self._number]),
  91                      '-cpu', 'host,migratable=off',
  92                      '-drive', 'if={},file='.format(S.getValue(
  93                          'GUEST_BOOT_DRIVE_TYPE')[self._number]) +
  94                      S.getValue('GUEST_IMAGE')[self._number],
  95                      '-boot', 'c', '--enable-kvm',
  96                      '-monitor', 'unix:%s,server,nowait' % self._monitor,
  97                      '-object',
  98                      'memory-backend-file,id=mem,size=' +
  99                      str(S.getValue('GUEST_MEMORY')[self._number]) + 'M,' +
 100                      'mem-path=' + S.getValue('HUGEPAGE_DIR') + ',share=on',
 101                      '-numa', 'node,memdev=mem -mem-prealloc',
 102                      '-nographic', '-vnc', str(vnc), '-name', name,
 103                      '-snapshot', '-net none', '-no-reboot',
 104                      '-drive',
 105                      'if=%s,format=raw,file=fat:rw:%s,snapshot=off' %
 106                      (S.getValue('GUEST_SHARED_DRIVE_TYPE')[self._number],
 107                       S.getValue('GUEST_SHARE_DIR')[self._number]),
 108                     ]
 109         self._configure_logging()
 110
 111     def _configure_logging(self):
 112         """
 113         Configure logging.
 114         """
 115         self.GuestCommandFilter.prefix = self._log_prefix
 116
 117         logger = logging.getLogger()
 118         cmd_logger = logging.FileHandler(
 119             filename=os.path.join(S.getValue('LOG_DIR'),
 120                                   S.getValue('LOG_FILE_GUEST_CMDS')) +
 121             str(self._number))
 122         cmd_logger.setLevel(logging.DEBUG)
 123         cmd_logger.addFilter(self.GuestCommandFilter())
 124         logger.addHandler(cmd_logger)
 125
 126     # startup/Shutdown
 127
 128     def start(self):
 129         """
 130         Start QEMU instance, login and prepare for commands.
 131         """
 132         super(IVnfQemu, self).start()
 133         if S.getValue('VNF_AFFINITIZATION_ON'):
 134             self._affinitize()
 135
 136         if S.getValue('VSWITCH_VHOST_NET_AFFINITIZATION') and S.getValue(
 137                 'VNF') == 'QemuVirtioNet':
 138             self._affinitize_vhost_net()
 139
 140         if self._timeout:
 141             self._config_guest_loopback()
 142
 143     def stop(self):
 144         """
 145         Stops VNF instance gracefully first.
 146         """
 147         try:
 148             # exit testpmd if needed
 149             if self._guest_loopback == 'testpmd':
 150                 self.execute_and_wait('stop', 120, "Done")
 151                 self.execute_and_wait('quit', 120, "[bB]ye")
 152
 153             # turn off VM
 154             self.execute_and_wait('poweroff', 120, "Power down")
 155
 156         except pexpect.TIMEOUT:
 157             self.kill()
 158
 159         # wait until qemu shutdowns
 160         self._logger.debug('Wait for QEMU to terminate')
 161         for dummy in range(30):
 162             time.sleep(1)
 163             if not self.is_running():
 164                 break
 165
 166         # just for case that graceful shutdown failed
 167         super(IVnfQemu, self).stop()
 168
 169     # helper functions
 170
 171     def _login(self, timeout=120):
 172         """
 173         Login to QEMU instance.
 174
 175         This can be used immediately after booting the machine, provided a
 176         sufficiently long ``timeout`` is given.
 177
 178         :param timeout: Timeout to wait for login to complete.
 179
 180         :returns: None
 181         """
 182         # if no timeout was set, we likely started QEMU without waiting for it
 183         # to boot. This being the case, we best check that it has finished
 184         # first.
 185         if not self._timeout:
 186             self._expect_process(timeout=timeout)
 187
 188         self._child.sendline(S.getValue('GUEST_USERNAME')[self._number])
 189         self._child.expect(S.getValue('GUEST_PROMPT_PASSWORD')[self._number], timeout=5)
 190         self._child.sendline(S.getValue('GUEST_PASSWORD')[self._number])
 191
 192         self._expect_process(S.getValue('GUEST_PROMPT')[self._number], timeout=5)
 193
 194     def send_and_pass(self, cmd, timeout=30):
 195         """
 196         Send ``cmd`` and wait ``timeout`` seconds for it to pass.
 197
 198         :param cmd: Command to send to guest.
 199         :param timeout: Time to wait for prompt before checking return code.
 200
 201         :returns: None
 202         """
 203         self.execute(cmd)
 204         self.wait(S.getValue('GUEST_PROMPT')[self._number], timeout=timeout)
 205         self.execute('echo $?')
 206         self._child.expect('^0$', timeout=1)  # expect a 0
 207         self.wait(S.getValue('GUEST_PROMPT')[self._number], timeout=timeout)
 208
 209     def _affinitize(self):
 210         """
 211         Affinitize the SMP cores of a QEMU instance.
 212
 213         This is a bit of a hack. The 'socat' utility is used to
 214         interact with the QEMU HMP. This is necessary due to the lack
 215         of QMP in older versions of QEMU, like v1.6.2. In future
 216         releases, this should be replaced with calls to libvirt or
 217         another Python-QEMU wrapper library.
 218
 219         :returns: None
 220         """
 221         thread_id = (r'.* CPU #%d: .* thread_id=(\d+)')
 222
 223         self._logger.info('Affinitizing guest...')
 224
 225         cur_locale = locale.getdefaultlocale()[1]
 226         proc = subprocess.Popen(
 227             ('echo', 'info cpus'), stdout=subprocess.PIPE)
 228         output = subprocess.check_output(
 229             ('sudo', 'socat', '-', 'UNIX-CONNECT:%s' % self._monitor),
 230             stdin=proc.stdout)
 231         proc.wait()
 232
 233         for cpu in range(0, int(S.getValue('GUEST_SMP')[self._number])):
 234             match = None
 235             for line in output.decode(cur_locale).split('\n'):
 236                 match = re.search(thread_id % cpu, line)
 237                 if match:
 238                     self._affinitize_pid(
 239                         S.getValue('GUEST_CORE_BINDING')[self._number][cpu],
 240                         match.group(1))
 241                     break
 242
 243             if not match:
 244                 self._logger.error('Failed to affinitize guest core #%d. Could'
 245                                    ' not parse tid.', cpu)
 246
 247     def _affinitize_vhost_net(self):
 248         """
 249         Affinitize the vhost net threads for Vanilla OVS and guest nic queues.
 250
 251         :return: None
 252         """
 253         self._logger.info('Affinitizing VHOST Net threads.')
 254         args1 = ['ps', 'ax']
 255         process1 = subprocess.Popen(args1, stdout=subprocess.PIPE,
 256                                     shell=False)
 257         out = process1.communicate()[0]
 258         processes = list()
 259         for line in out.decode(locale.getdefaultlocale()[1]).split('\n'):
 260             if re.search('\[vhost-(\d+)', line):
 261                 processes.append(re.match('\s*(\d+)', line).group(1))
 262         self._logger.info('Found %s vhost net threads...', len(processes))
 263
 264         cpumap = S.getValue('VSWITCH_VHOST_CPU_MAP')
 265         mapcount = 0
 266         for proc in processes:
 267             self._affinitize_pid(cpumap[mapcount], proc)
 268             mapcount += 1
 269             if mapcount + 1 > len(cpumap):
 270                 # Not enough cpus were given in the mapping to cover all the
 271                 # threads on a 1 to 1 ratio with cpus so reset the list counter
 272                 #  to 0.
 273                 mapcount = 0
 274
 275     def _config_guest_loopback(self):
 276         """
 277         Configure VM to run VNF, e.g. port forwarding application based on the configuration
 278         """
 279         if self._guest_loopback == 'testpmd':
 280             self._login()
 281             self._configure_testpmd()
 282         elif self._guest_loopback == 'l2fwd':
 283             self._login()
 284             self._configure_l2fwd()
 285         elif self._guest_loopback == 'linux_bridge':
 286             self._login()
 287             self._configure_linux_bridge()
 288         elif self._guest_loopback != 'buildin':
 289             self._logger.error('Unsupported guest loopback method "%s" was specified. Option'
 290                                ' "buildin" will be used as a fallback.', self._guest_loopback)
 291
 292     def wait(self, prompt=None, timeout=30):
 293         if prompt is None:
 294             prompt = S.getValue('GUEST_PROMPT')[self._number]
 295         super(IVnfQemu, self).wait(prompt=prompt, timeout=timeout)
 296
 297     def execute_and_wait(self, cmd, timeout=30, prompt=None):
 298         if prompt is None:
 299             prompt = S.getValue('GUEST_PROMPT')[self._number]
 300         super(IVnfQemu, self).execute_and_wait(cmd, timeout=timeout,
 301                                                prompt=prompt)
 302
 303     def _modify_dpdk_makefile(self):
 304         """
 305         Modifies DPDK makefile in Guest before compilation if needed
 306         """
 307         pass
 308
 309     def _configure_copy_sources(self, dirname):
 310         """
 311         Mount shared directory and copy DPDK and l2fwd sources
 312         """
 313         # mount shared directory
 314         self.execute_and_wait('umount /dev/sdb1')
 315         self.execute_and_wait('rm -rf ' + S.getValue('GUEST_OVS_DPDK_DIR')[self._number])
 316         self.execute_and_wait('mkdir -p ' + S.getValue('GUEST_OVS_DPDK_SHARE')[self._number])
 317         self.execute_and_wait('mount -o ro,iocharset=utf8 /dev/sdb1 ' +
 318                               S.getValue('GUEST_OVS_DPDK_SHARE')[self._number])
 319         self.execute_and_wait('mkdir -p ' + S.getValue('GUEST_OVS_DPDK_DIR')[self._number])
 320         self.execute_and_wait('cp -r ' + os.path.join(S.getValue('GUEST_OVS_DPDK_SHARE')[self._number], dirname) +
 321                               ' ' + S.getValue('GUEST_OVS_DPDK_DIR')[self._number])
 322         self.execute_and_wait('umount /dev/sdb1')
 323
 324     def _configure_disable_firewall(self):
 325         """
 326         Disable firewall in VM
 327         """
 328         for iptables in ['iptables', 'ip6tables']:
 329             # filter table
 330             for chain in ['INPUT', 'FORWARD', 'OUTPUT']:
 331                 self.execute_and_wait("{} -t filter -P {} ACCEPT".format(iptables, chain))
 332             # mangle table
 333             for chain in ['PREROUTING', 'INPUT', 'FORWARD', 'OUTPUT', 'POSTROUTING']:
 334                 self.execute_and_wait("{} -t mangle -P {} ACCEPT".format(iptables, chain))
 335             # nat table
 336             for chain in ['PREROUTING', 'INPUT', 'OUTPUT', 'POSTROUTING']:
 337                 self.execute_and_wait("{} -t nat -P {} ACCEPT".format(iptables, chain))
 338
 339             # flush rules and delete chains created by user
 340             for table in ['filter', 'mangle', 'nat']:
 341                 self.execute_and_wait("{} -t {} -F".format(iptables, table))
 342                 self.execute_and_wait("{} -t {} -X".format(iptables, table))
 343
 344
 345     def _configure_testpmd(self):
 346         """
 347         Configure VM to perform L2 forwarding between NICs by DPDK's testpmd
 348         """
 349         self._configure_copy_sources('DPDK')
 350         self._configure_disable_firewall()
 351
 352         # Guest images _should_ have 1024 hugepages by default,
 353         # but just in case:'''
 354         self.execute_and_wait('sysctl vm.nr_hugepages={}'.format(S.getValue('GUEST_HUGEPAGES_NR')[self._number]))
 355
 356         # Mount hugepages
 357         self.execute_and_wait('mkdir -p /dev/hugepages')
 358         self.execute_and_wait(
 359             'mount -t hugetlbfs hugetlbfs /dev/hugepages')
 360
 361         # build and configure system for dpdk
 362         self.execute_and_wait('cd ' + S.getValue('GUEST_OVS_DPDK_DIR')[self._number] +
 363                               '/DPDK')
 364         self.execute_and_wait('export CC=gcc')
 365         self.execute_and_wait('export RTE_SDK=' +
 366                               S.getValue('GUEST_OVS_DPDK_DIR')[self._number] + '/DPDK')
 367         self.execute_and_wait('export RTE_TARGET=%s' % S.getValue('RTE_TARGET'))
 368
 369         # modify makefile if needed
 370         self._modify_dpdk_makefile()
 371
 372         # disable network interfaces, so DPDK can take care of them
 373         for nic in self._nics:
 374             self.execute_and_wait('ifdown ' + nic['device'])
 375
 376         # build and insert igb_uio and rebind interfaces to it
 377         self.execute_and_wait('make RTE_OUTPUT=$RTE_SDK/$RTE_TARGET -C '
 378                               '$RTE_SDK/lib/librte_eal/linuxapp/igb_uio')
 379         self.execute_and_wait('modprobe uio')
 380         self.execute_and_wait('insmod %s/kmod/igb_uio.ko' %
 381                               S.getValue('RTE_TARGET'))
 382         self.execute_and_wait('./tools/dpdk*bind.py --status')
 383         pci_list = ' '.join([nic['pci'] for nic in self._nics])
 384         self.execute_and_wait('./tools/dpdk*bind.py -u ' + pci_list)
 385         self.execute_and_wait('./tools/dpdk*bind.py -b igb_uio ' + pci_list)
 386         self.execute_and_wait('./tools/dpdk*bind.py --status')
 387
 388         # build and run 'test-pmd'
 389         self.execute_and_wait('cd ' + S.getValue('GUEST_OVS_DPDK_DIR')[self._number] +
 390                               '/DPDK/app/test-pmd')
 391         self.execute_and_wait('make clean')
 392         self.execute_and_wait('make')
 393         if int(S.getValue('GUEST_NIC_QUEUES')[self._number]):
 394             self.execute_and_wait(
 395                 './testpmd {} -n4 --socket-mem 512 --'.format(
 396                     S.getValue('GUEST_TESTPMD_CPU_MASK')[self._number]) +
 397                 ' --burst=64 -i --txqflags=0xf00 ' +
 398                 '--nb-cores={} --rxq={} --txq={} '.format(
 399                     S.getValue('GUEST_TESTPMD_NB_CORES')[self._number],
 400                     S.getValue('GUEST_TESTPMD_TXQ')[self._number],
 401                     S.getValue('GUEST_TESTPMD_RXQ')[self._number]) +
 402                 '--disable-hw-vlan', 60, "Done")
 403         else:
 404             self.execute_and_wait(
 405                 './testpmd {} -n 4 --socket-mem 512 --'.format(
 406                     S.getValue('GUEST_TESTPMD_CPU_MASK')[self._number]) +
 407                 ' --burst=64 -i --txqflags=0xf00 ' +
 408                 '--disable-hw-vlan', 60, "Done")
 409         self.execute('set fwd ' + self._testpmd_fwd_mode, 1)
 410         self.execute_and_wait('start', 20,
 411                               'TX RS bit threshold=.+ - TXQ flags=0xf00')
 412
 413     def _configure_l2fwd(self):
 414         """
 415         Configure VM to perform L2 forwarding between NICs by l2fwd module
 416         """
 417         if int(S.getValue('GUEST_NIC_QUEUES')[self._number]):
 418             self._set_multi_queue_nic()
 419         self._configure_copy_sources('l2fwd')
 420         self._configure_disable_firewall()
 421
 422         # configure all interfaces
 423         for nic in self._nics:
 424             self.execute('ip addr add ' +
 425                          nic['ip'] + ' dev ' + nic['device'])
 426             self.execute('ip link set dev ' + nic['device'] + ' up')
 427
 428         # build and configure system for l2fwd
 429         self.execute_and_wait('cd ' + S.getValue('GUEST_OVS_DPDK_DIR')[self._number] +
 430                               '/l2fwd')
 431         self.execute_and_wait('export CC=gcc')
 432
 433         self.execute_and_wait('make')
 434         if len(self._nics) == 2:
 435             self.execute_and_wait('insmod ' + S.getValue('GUEST_OVS_DPDK_DIR')[self._number] +
 436                                   '/l2fwd' + '/l2fwd.ko net1=' + self._nics[0]['device'] +
 437                                   ' net2=' + self._nics[1]['device'])
 438         else:
 439             raise RuntimeError('l2fwd can forward only between 2 NICs, but {} NICs are '
 440                                'configured inside GUEST'.format(len(self._nics)))
 441
 442     def _configure_linux_bridge(self):
 443         """
 444         Configure VM to perform L2 forwarding between NICs by linux bridge
 445         """
 446         if int(S.getValue('GUEST_NIC_QUEUES')[self._number]):
 447             self._set_multi_queue_nic()
 448         self._configure_disable_firewall()
 449
 450         # configure linux bridge
 451         self.execute('brctl addbr br0')
 452
 453         # add all NICs into the bridge
 454         for nic in self._nics:
 455             self.execute('ip addr add ' +
 456                          nic['ip'] + ' dev ' + nic['device'])
 457             self.execute('ip link set dev ' + nic['device'] + ' up')
 458             self.execute('brctl addif br0 ' + nic['device'])
 459
 460         self.execute('ip addr add ' +
 461                      S.getValue('GUEST_BRIDGE_IP')[self._number] +
 462                      ' dev br0')
 463         self.execute('ip link set dev br0 up')
 464
 465         # Add the arp entries for the IXIA ports and the bridge you are using.
 466         # Use command line values if provided.
 467         trafficgen_mac = get_test_param('vanilla_tgen_port1_mac',
 468                                         S.getValue('VANILLA_TGEN_PORT1_MAC'))
 469         trafficgen_ip = get_test_param('vanilla_tgen_port1_ip',
 470                                        S.getValue('VANILLA_TGEN_PORT1_IP'))
 471
 472         self.execute('arp -s ' + trafficgen_ip + ' ' + trafficgen_mac)
 473
 474         trafficgen_mac = get_test_param('vanilla_tgen_port2_mac',
 475                                         S.getValue('VANILLA_TGEN_PORT2_MAC'))
 476         trafficgen_ip = get_test_param('vanilla_tgen_port2_ip',
 477                                        S.getValue('VANILLA_TGEN_PORT2_IP'))
 478
 479         self.execute('arp -s ' + trafficgen_ip + ' ' + trafficgen_mac)
 480
 481         # Enable forwarding
 482         self.execute('sysctl -w net.ipv4.ip_forward=1')
 483
 484         # Controls source route verification
 485         # 0 means no source validation
 486         self.execute('sysctl -w net.ipv4.conf.all.rp_filter=0')
 487         for nic in self._nics:
 488             self.execute('sysctl -w net.ipv4.conf.' + nic['device'] + '.rp_filter=0')
 489
 490     def _set_multi_queue_nic(self):
 491         """
 492         Enable multi-queue in guest kernel with ethool.
 493         :return: None
 494         """
 495         for nic in self._nics:
 496             self.execute_and_wait('ethtool -L {} combined {}'.format(
 497                 nic['device'], S.getValue('GUEST_NIC_QUEUES')[self._number]))
 498             self.execute_and_wait('ethtool -l {}'.format(nic['device']))