vnfs/qemu/qemu.py

   1 # Copyright 2015-2017 Intel Corporation.
   2 #
   3 # Licensed under the Apache License, Version 2.0 (the "License");
   4 # you may not use this file except in compliance with the License.
   5 # You may obtain a copy of the License at
   6 #
   7 #   http://www.apache.org/licenses/LICENSE-2.0
   8 #
   9 # Unless required by applicable law or agreed to in writing, software
  10 # distributed under the License is distributed on an "AS IS" BASIS,
  11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12 # See the License for the specific language governing permissions and
  13 # limitations under the License.
  14
  15 """Automation of QEMU hypervisor for launching guests.
  16 """
  17
  18 import os
  19 import logging
  20 import locale
  21 import re
  22 import subprocess
  23 import time
  24 import pexpect
  25
  26 from conf import settings as S
  27 from vnfs.vnf.vnf import IVnf
  28
  29 class IVnfQemu(IVnf):
  30     """
  31     Abstract class for controling an instance of QEMU
  32     """
  33     _cmd = None
  34     _expect = None
  35     _proc_name = 'qemu'
  36
  37     class GuestCommandFilter(logging.Filter):
  38         """
  39         Filter out strings beginning with 'guestcmd :'.
  40         """
  41         def filter(self, record):
  42             return record.getMessage().startswith(self.prefix)
  43
  44     def __init__(self):
  45         """
  46         Initialisation function.
  47         """
  48         super(IVnfQemu, self).__init__()
  49
  50         self._expect = S.getValue('GUEST_PROMPT_LOGIN')[self._number]
  51         self._logger = logging.getLogger(__name__)
  52         self._logfile = os.path.join(
  53             S.getValue('LOG_DIR'),
  54             S.getValue('LOG_FILE_QEMU')) + str(self._number)
  55         self._timeout = S.getValue('GUEST_TIMEOUT')[self._number]
  56         self._monitor = '%s/vm%dmonitor' % ('/tmp', self._number)
  57         # read GUEST NICs configuration and use only defined NR of NICS
  58         nics_nr = S.getValue('GUEST_NICS_NR')[self._number]
  59         # and inform user about missconfiguration
  60         if nics_nr < 1:
  61             raise RuntimeError('At least one VM NIC is mandotory, but {} '
  62                                'NICs are configured'.format(nics_nr))
  63         elif nics_nr > 1 and nics_nr % 2:
  64             nics_nr = int(nics_nr / 2) * 2
  65             self._logger.warning('Odd number of NICs is configured, only '
  66                                  '%s NICs will be used', nics_nr)
  67
  68         self._nics = S.getValue('GUEST_NICS')[self._number][:nics_nr]
  69
  70         # set guest loopback application based on VNF configuration
  71         self._guest_loopback = S.getValue('GUEST_LOOPBACK')[self._number]
  72
  73         self._testpmd_fwd_mode = S.getValue('GUEST_TESTPMD_FWD_MODE')[self._number]
  74         # in case of SRIOV we must ensure, that MAC addresses are not swapped
  75         if S.getValue('SRIOV_ENABLED') and self._testpmd_fwd_mode.startswith('mac') and \
  76            not str(S.getValue('VNF')).endswith('PciPassthrough'):
  77
  78             self._logger.info("SRIOV detected, forwarding mode of testpmd was changed from '%s' to '%s'",
  79                               self._testpmd_fwd_mode, 'io')
  80             self._testpmd_fwd_mode = 'io'
  81
  82         name = 'Client%d' % self._number
  83         vnc = ':%d' % self._number
  84         # NOTE: affinization of main qemu process can cause hangup of 2nd VM
  85         # in case of DPDK usage. It can also slow down VM response time.
  86         cpumask = ",".join(S.getValue('GUEST_CORE_BINDING')[self._number])
  87         self._cmd = ['sudo', '-E', 'taskset', '-c', cpumask,
  88                      S.getValue('TOOLS')['qemu-system'],
  89                      '-m', S.getValue('GUEST_MEMORY')[self._number],
  90                      '-smp', str(S.getValue('GUEST_SMP')[self._number]),
  91                      '-cpu', str(S.getValue('GUEST_CPU_OPTIONS')[self._number]),
  92                      '-drive', 'if={},file='.format(S.getValue(
  93                          'GUEST_BOOT_DRIVE_TYPE')[self._number]) +
  94                      S.getValue('GUEST_IMAGE')[self._number],
  95                      '-boot', 'c', '--enable-kvm',
  96                      '-monitor', 'unix:%s,server,nowait' % self._monitor,
  97                      '-object',
  98                      'memory-backend-file,id=mem,size=' +
  99                      str(S.getValue('GUEST_MEMORY')[self._number]) + 'M,' +
 100                      'mem-path=' + S.getValue('HUGEPAGE_DIR') + ',share=on',
 101                      '-numa', 'node,memdev=mem -mem-prealloc',
 102                      '-nographic', '-vnc', str(vnc), '-name', name,
 103                      '-snapshot', '-net none', '-no-reboot',
 104                      '-drive',
 105                      'if=%s,format=raw,file=fat:rw:%s,snapshot=off' %
 106                      (S.getValue('GUEST_SHARED_DRIVE_TYPE')[self._number],
 107                       S.getValue('GUEST_SHARE_DIR')[self._number]),
 108                     ]
 109         self._configure_logging()
 110
 111     def _configure_logging(self):
 112         """
 113         Configure logging.
 114         """
 115         self.GuestCommandFilter.prefix = self._log_prefix
 116
 117         logger = logging.getLogger()
 118         cmd_logger = logging.FileHandler(
 119             filename=os.path.join(S.getValue('LOG_DIR'),
 120                                   S.getValue('LOG_FILE_GUEST_CMDS')) +
 121             str(self._number))
 122         cmd_logger.setLevel(logging.DEBUG)
 123         cmd_logger.addFilter(self.GuestCommandFilter())
 124         logger.addHandler(cmd_logger)
 125
 126     # startup/Shutdown
 127
 128     def start(self):
 129         """
 130         Start QEMU instance, login and prepare for commands.
 131         """
 132         super(IVnfQemu, self).start()
 133         if S.getValue('VNF_AFFINITIZATION_ON'):
 134             self._affinitize()
 135
 136         if S.getValue('VSWITCH_VHOST_NET_AFFINITIZATION') and S.getValue(
 137                 'VNF') == 'QemuVirtioNet':
 138             self._affinitize_vhost_net()
 139
 140         if self._timeout:
 141             self._config_guest_loopback()
 142
 143     def stop(self):
 144         """
 145         Stops VNF instance gracefully first.
 146         """
 147         if self.is_running():
 148             try:
 149                 if self._login_active:
 150                     # exit testpmd if needed
 151                     if self._guest_loopback == 'testpmd':
 152                         self.execute_and_wait('stop', 120, "Done")
 153                         self.execute_and_wait('quit', 120, "[bB]ye")
 154
 155                     # turn off VM
 156                     self.execute_and_wait('poweroff', 120, "Power down")
 157
 158             except pexpect.TIMEOUT:
 159                 self.kill()
 160
 161             # wait until qemu shutdowns
 162             self._logger.debug('Wait for QEMU to terminate')
 163             for dummy in range(30):
 164                 time.sleep(1)
 165                 if not self.is_running():
 166                     break
 167
 168             # just for case that graceful shutdown failed
 169             super(IVnfQemu, self).stop()
 170
 171     # helper functions
 172
 173     def login(self, timeout=120):
 174         """
 175         Login to QEMU instance.
 176
 177         This can be used immediately after booting the machine, provided a
 178         sufficiently long ``timeout`` is given.
 179
 180         :param timeout: Timeout to wait for login to complete.
 181
 182         :returns: True if login is active
 183         """
 184         if self._login_active:
 185             return self._login_active
 186
 187         # if no timeout was set, we likely started QEMU without waiting for it
 188         # to boot. This being the case, we best check that it has finished
 189         # first.
 190         if not self._timeout:
 191             self._expect_process(timeout=timeout)
 192
 193         self._child.sendline(S.getValue('GUEST_USERNAME')[self._number])
 194         self._child.expect(S.getValue('GUEST_PROMPT_PASSWORD')[self._number], timeout=5)
 195         self._child.sendline(S.getValue('GUEST_PASSWORD')[self._number])
 196
 197         self._expect_process(S.getValue('GUEST_PROMPT')[self._number], timeout=5)
 198         self._login_active = True
 199         return self._login_active
 200
 201     def _affinitize(self):
 202         """
 203         Affinitize the SMP cores of a QEMU instance.
 204
 205         This is a bit of a hack. The 'socat' utility is used to
 206         interact with the QEMU HMP. This is necessary due to the lack
 207         of QMP in older versions of QEMU, like v1.6.2. In future
 208         releases, this should be replaced with calls to libvirt or
 209         another Python-QEMU wrapper library.
 210
 211         :returns: None
 212         """
 213         thread_id = (r'.* CPU #%d: .* thread_id=(\d+)')
 214
 215         self._logger.info('Affinitizing guest...')
 216
 217         cur_locale = locale.getdefaultlocale()[1]
 218         proc = subprocess.Popen(
 219             ('echo', 'info cpus'), stdout=subprocess.PIPE)
 220         output = subprocess.check_output(
 221             ('sudo', 'socat', '-', 'UNIX-CONNECT:%s' % self._monitor),
 222             stdin=proc.stdout)
 223         proc.wait()
 224
 225         for cpu in range(0, int(S.getValue('GUEST_SMP')[self._number])):
 226             match = None
 227             guest_thread_binding = S.getValue('GUEST_THREAD_BINDING')[self._number]
 228             if guest_thread_binding is None:
 229                 guest_thread_binding = S.getValue('GUEST_CORE_BINDING')[self._number]
 230             for line in output.decode(cur_locale).split('\n'):
 231                 match = re.search(thread_id % cpu, line)
 232                 if match:
 233                     self._affinitize_pid(guest_thread_binding[cpu], match.group(1))
 234                     break
 235
 236             if not match:
 237                 self._logger.error('Failed to affinitize guest core #%d. Could'
 238                                    ' not parse tid.', cpu)
 239
 240     def _affinitize_vhost_net(self):
 241         """
 242         Affinitize the vhost net threads for Vanilla OVS and guest nic queues.
 243
 244         :return: None
 245         """
 246         self._logger.info('Affinitizing VHOST Net threads.')
 247         args1 = ['pgrep', 'vhost-']
 248         process1 = subprocess.Popen(args1, stdout=subprocess.PIPE,
 249                                     shell=False)
 250         out = process1.communicate()[0]
 251         processes = out.decode(locale.getdefaultlocale()[1]).split('\n')
 252         if processes[-1] == '':
 253             processes.pop() # pgrep may return an extra line with no data
 254         self._logger.info('Found %s vhost net threads...', len(processes))
 255
 256         cpumap = S.getValue('VSWITCH_VHOST_CPU_MAP')
 257         mapcount = 0
 258         for proc in processes:
 259             self._affinitize_pid(cpumap[mapcount], proc)
 260             mapcount += 1
 261             if mapcount + 1 > len(cpumap):
 262                 # Not enough cpus were given in the mapping to cover all the
 263                 # threads on a 1 to 1 ratio with cpus so reset the list counter
 264                 #  to 0.
 265                 mapcount = 0
 266
 267     def _config_guest_loopback(self):
 268         """
 269         Configure VM to run VNF, e.g. port forwarding application based on the configuration
 270         """
 271         if self._guest_loopback == 'buildin':
 272             return
 273
 274         self.login()
 275
 276         if self._guest_loopback == 'testpmd':
 277             self._configure_testpmd()
 278         elif self._guest_loopback == 'l2fwd':
 279             self._configure_l2fwd()
 280         elif self._guest_loopback == 'linux_bridge':
 281             self._configure_linux_bridge()
 282         elif self._guest_loopback != 'clean':
 283             raise RuntimeError('Unsupported guest loopback method "%s" was specified.',
 284                                self._guest_loopback)
 285
 286     def wait(self, prompt=None, timeout=30):
 287         if prompt is None:
 288             prompt = S.getValue('GUEST_PROMPT')[self._number]
 289         return super(IVnfQemu, self).wait(prompt=prompt, timeout=timeout)
 290
 291     def execute_and_wait(self, cmd, timeout=30, prompt=None):
 292         if prompt is None:
 293             prompt = S.getValue('GUEST_PROMPT')[self._number]
 294         return super(IVnfQemu, self).execute_and_wait(cmd, timeout=timeout,
 295                                                       prompt=prompt)
 296
 297     def _modify_dpdk_makefile(self):
 298         """
 299         Modifies DPDK makefile in Guest before compilation if needed
 300         """
 301         pass
 302
 303     def _configure_copy_sources(self, dirname):
 304         """
 305         Mount shared directory and copy DPDK and l2fwd sources
 306         """
 307         # mount shared directory
 308         self.execute_and_wait('umount /dev/sdb1')
 309         self.execute_and_wait('rm -rf ' + S.getValue('GUEST_OVS_DPDK_DIR')[self._number])
 310         self.execute_and_wait('mkdir -p ' + S.getValue('GUEST_OVS_DPDK_SHARE')[self._number])
 311         self.execute_and_wait('mount -o ro,iocharset=utf8 /dev/sdb1 ' +
 312                               S.getValue('GUEST_OVS_DPDK_SHARE')[self._number])
 313         self.execute_and_wait('mkdir -p ' + S.getValue('GUEST_OVS_DPDK_DIR')[self._number])
 314         self.execute_and_wait('cp -r ' + os.path.join(S.getValue('GUEST_OVS_DPDK_SHARE')[self._number], dirname) +
 315                               ' ' + S.getValue('GUEST_OVS_DPDK_DIR')[self._number])
 316         self.execute_and_wait('umount /dev/sdb1')
 317
 318     def _configure_disable_firewall(self):
 319         """
 320         Disable firewall in VM
 321         """
 322         for iptables in ['iptables', 'ip6tables']:
 323             # filter table
 324             for chain in ['INPUT', 'FORWARD', 'OUTPUT']:
 325                 self.execute_and_wait("{} -t filter -P {} ACCEPT".format(iptables, chain))
 326             # mangle table
 327             for chain in ['PREROUTING', 'INPUT', 'FORWARD', 'OUTPUT', 'POSTROUTING']:
 328                 self.execute_and_wait("{} -t mangle -P {} ACCEPT".format(iptables, chain))
 329             # nat table
 330             for chain in ['PREROUTING', 'INPUT', 'OUTPUT', 'POSTROUTING']:
 331                 self.execute_and_wait("{} -t nat -P {} ACCEPT".format(iptables, chain))
 332
 333             # flush rules and delete chains created by user
 334             for table in ['filter', 'mangle', 'nat']:
 335                 self.execute_and_wait("{} -t {} -F".format(iptables, table))
 336                 self.execute_and_wait("{} -t {} -X".format(iptables, table))
 337
 338     def _configure_testpmd(self):
 339         """
 340         Configure VM to perform L2 forwarding between NICs by DPDK's testpmd
 341         """
 342         self._configure_copy_sources('DPDK')
 343         self._configure_disable_firewall()
 344
 345         # Guest images _should_ have 1024 hugepages by default,
 346         # but just in case:'''
 347         self.execute_and_wait('sysctl vm.nr_hugepages={}'.format(S.getValue('GUEST_HUGEPAGES_NR')[self._number]))
 348
 349         # Mount hugepages
 350         self.execute_and_wait('mkdir -p /dev/hugepages')
 351         self.execute_and_wait(
 352             'mount -t hugetlbfs hugetlbfs /dev/hugepages')
 353
 354         # build and configure system for dpdk
 355         self.execute_and_wait('cd ' + S.getValue('GUEST_OVS_DPDK_DIR')[self._number] +
 356                               '/DPDK')
 357         self.execute_and_wait('export CC=gcc')
 358         self.execute_and_wait('export RTE_SDK=' +
 359                               S.getValue('GUEST_OVS_DPDK_DIR')[self._number] + '/DPDK')
 360         self.execute_and_wait('export RTE_TARGET=%s' % S.getValue('RTE_TARGET'))
 361
 362         # modify makefile if needed
 363         self._modify_dpdk_makefile()
 364
 365         # disable network interfaces, so DPDK can take care of them
 366         for nic in self._nics:
 367             self.execute_and_wait('ifdown ' + nic['device'])
 368
 369         self.execute_and_wait('./*tools/dpdk*bind.py --status')
 370         pci_list = ' '.join([nic['pci'] for nic in self._nics])
 371         self.execute_and_wait('./*tools/dpdk*bind.py -u ' + pci_list)
 372         self._bind_dpdk_driver(S.getValue(
 373             'GUEST_DPDK_BIND_DRIVER')[self._number], pci_list)
 374         self.execute_and_wait('./*tools/dpdk*bind.py --status')
 375
 376         # build and run 'test-pmd'
 377         self.execute_and_wait('cd ' + S.getValue('GUEST_OVS_DPDK_DIR')[self._number] +
 378                               '/DPDK/app/test-pmd')
 379         self.execute_and_wait('make clean')
 380         self.execute_and_wait('make')
 381
 382         # get testpmd settings from CLI
 383         testpmd_params = S.getValue('GUEST_TESTPMD_PARAMS')[self._number]
 384         if S.getValue('VSWITCH_JUMBO_FRAMES_ENABLED'):
 385             testpmd_params += ' --max-pkt-len={}'.format(S.getValue(
 386                 'VSWITCH_JUMBO_FRAMES_SIZE'))
 387
 388         self.execute_and_wait('./testpmd {}'.format(testpmd_params), 60, "Done")
 389         self.execute_and_wait('set fwd ' + self._testpmd_fwd_mode, 20, 'testpmd>')
 390         self.execute_and_wait('start', 20, 'testpmd>')
 391
 392     def _configure_l2fwd(self):
 393         """
 394         Configure VM to perform L2 forwarding between NICs by l2fwd module
 395         """
 396         if int(S.getValue('GUEST_NIC_QUEUES')[self._number]):
 397             self._set_multi_queue_nic()
 398         self._configure_copy_sources('l2fwd')
 399         self._configure_disable_firewall()
 400
 401         # configure all interfaces
 402         for nic in self._nics:
 403             self.execute_and_wait('ip addr add ' +
 404                                   nic['ip'] + ' dev ' + nic['device'])
 405             if S.getValue('VSWITCH_JUMBO_FRAMES_ENABLED'):
 406                 self.execute_and_wait('ifconfig {} mtu {}'.format(
 407                     nic['device'], S.getValue('VSWITCH_JUMBO_FRAMES_SIZE')))
 408             self.execute_and_wait('ip link set dev ' + nic['device'] + ' up')
 409
 410         # build and configure system for l2fwd
 411         self.execute_and_wait('cd ' + S.getValue('GUEST_OVS_DPDK_DIR')[self._number] +
 412                               '/l2fwd')
 413         self.execute_and_wait('export CC=gcc')
 414
 415         self.execute_and_wait('make')
 416         if len(self._nics) == 2:
 417             self.execute_and_wait('insmod ' + S.getValue('GUEST_OVS_DPDK_DIR')[self._number] +
 418                                   '/l2fwd' + '/l2fwd.ko net1=' + self._nics[0]['device'] +
 419                                   ' net2=' + self._nics[1]['device'])
 420         else:
 421             raise RuntimeError('l2fwd can forward only between 2 NICs, but {} NICs are '
 422                                'configured inside GUEST'.format(len(self._nics)))
 423
 424     def _configure_linux_bridge(self):
 425         """
 426         Configure VM to perform L2 forwarding between NICs by linux bridge
 427         """
 428         if int(S.getValue('GUEST_NIC_QUEUES')[self._number]):
 429             self._set_multi_queue_nic()
 430         self._configure_disable_firewall()
 431
 432         # configure linux bridge
 433         self.execute_and_wait('brctl addbr br0')
 434
 435         # add all NICs into the bridge
 436         for nic in self._nics:
 437             self.execute_and_wait('ip addr add ' + nic['ip'] + ' dev ' + nic['device'])
 438             if S.getValue('VSWITCH_JUMBO_FRAMES_ENABLED'):
 439                 self.execute_and_wait('ifconfig {} mtu {}'.format(
 440                     nic['device'], S.getValue('VSWITCH_JUMBO_FRAMES_SIZE')))
 441             self.execute_and_wait('ip link set dev ' + nic['device'] + ' up')
 442             self.execute_and_wait('brctl addif br0 ' + nic['device'])
 443
 444         self.execute_and_wait('ip addr add {} dev br0'.format(
 445             S.getValue('GUEST_BRIDGE_IP')[self._number]))
 446         self.execute_and_wait('ip link set dev br0 up')
 447
 448         # Add the arp entries for the IXIA ports and the bridge you are using.
 449         # Use command line values if provided.
 450         trafficgen_mac = S.getValue('VANILLA_TGEN_PORT1_MAC')
 451         trafficgen_ip = S.getValue('VANILLA_TGEN_PORT1_IP')
 452
 453         self.execute_and_wait('arp -s ' + trafficgen_ip + ' ' + trafficgen_mac)
 454
 455         trafficgen_mac = S.getValue('VANILLA_TGEN_PORT2_MAC')
 456         trafficgen_ip = S.getValue('VANILLA_TGEN_PORT2_IP')
 457
 458         self.execute_and_wait('arp -s ' + trafficgen_ip + ' ' + trafficgen_mac)
 459
 460         # Enable forwarding
 461         self.execute_and_wait('sysctl -w net.ipv4.ip_forward=1')
 462
 463         # Controls source route verification
 464         # 0 means no source validation
 465         self.execute_and_wait('sysctl -w net.ipv4.conf.all.rp_filter=0')
 466         for nic in self._nics:
 467             self.execute_and_wait('sysctl -w net.ipv4.conf.' + nic['device'] +
 468                                   '.rp_filter=0')
 469
 470     def _bind_dpdk_driver(self, driver, pci_slots):
 471         """
 472         Bind the virtual nics to the driver specific in the conf file
 473         :return: None
 474         """
 475         if driver == 'uio_pci_generic':
 476             if S.getValue('VNF') == 'QemuPciPassthrough':
 477                 # unsupported config, bind to igb_uio instead and exit the
 478                 # outer function after completion.
 479                 self._logger.error('SR-IOV does not support uio_pci_generic. '
 480                                    'Igb_uio will be used instead.')
 481                 self._bind_dpdk_driver('igb_uio_from_src', pci_slots)
 482                 return
 483             self.execute_and_wait('modprobe uio_pci_generic')
 484             self.execute_and_wait('./*tools/dpdk*bind.py -b uio_pci_generic '+
 485                                   pci_slots)
 486         elif driver == 'vfio_no_iommu':
 487             self.execute_and_wait('modprobe -r vfio')
 488             self.execute_and_wait('modprobe -r vfio_iommu_type1')
 489             self.execute_and_wait('modprobe vfio enable_unsafe_noiommu_mode=Y')
 490             self.execute_and_wait('modprobe vfio-pci')
 491             self.execute_and_wait('./*tools/dpdk*bind.py -b vfio-pci ' +
 492                                   pci_slots)
 493         elif driver == 'igb_uio_from_src':
 494             # build and insert igb_uio and rebind interfaces to it
 495             self.execute_and_wait('make RTE_OUTPUT=$RTE_SDK/$RTE_TARGET -C '
 496                                   '$RTE_SDK/lib/librte_eal/linuxapp/igb_uio')
 497             self.execute_and_wait('modprobe uio')
 498             self.execute_and_wait('insmod %s/kmod/igb_uio.ko' %
 499                                   S.getValue('RTE_TARGET'))
 500             self.execute_and_wait('./*tools/dpdk*bind.py -b igb_uio ' + pci_slots)
 501         else:
 502             self._logger.error(
 503                 'Unknown driver for binding specified, defaulting to igb_uio')
 504             self._bind_dpdk_driver('igb_uio_from_src', pci_slots)
 505
 506     def _set_multi_queue_nic(self):
 507         """
 508         Enable multi-queue in guest kernel with ethool.
 509         :return: None
 510         """
 511         for nic in self._nics:
 512             self.execute_and_wait('ethtool -L {} combined {}'.format(
 513                 nic['device'], S.getValue('GUEST_NIC_QUEUES')[self._number]))
 514             self.execute_and_wait('ethtool -l {}'.format(nic['device']))