vnfs/qemu/qemu.py

   1 # Copyright 2015-2017 Intel Corporation.
   2 #
   3 # Licensed under the Apache License, Version 2.0 (the "License");
   4 # you may not use this file except in compliance with the License.
   5 # You may obtain a copy of the License at
   6 #
   7 #   http://www.apache.org/licenses/LICENSE-2.0
   8 #
   9 # Unless required by applicable law or agreed to in writing, software
  10 # distributed under the License is distributed on an "AS IS" BASIS,
  11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12 # See the License for the specific language governing permissions and
  13 # limitations under the License.
  14
  15 """Automation of QEMU hypervisor for launching guests.
  16 """
  17
  18 import os
  19 import logging
  20 import locale
  21 import re
  22 import subprocess
  23 import time
  24 import pexpect
  25
  26 from conf import settings as S
  27 from vnfs.vnf.vnf import IVnf
  28
  29 class IVnfQemu(IVnf):
  30     """
  31     Abstract class for controling an instance of QEMU
  32     """
  33     _cmd = None
  34     _expect = None
  35     _proc_name = 'qemu'
  36
  37     class GuestCommandFilter(logging.Filter):
  38         """
  39         Filter out strings beginning with 'guestcmd :'.
  40         """
  41         def filter(self, record):
  42             return record.getMessage().startswith(self.prefix)
  43
  44     def __init__(self):
  45         """
  46         Initialisation function.
  47         """
  48         super(IVnfQemu, self).__init__()
  49
  50         self._expect = S.getValue('GUEST_PROMPT_LOGIN')[self._number]
  51         self._logger = logging.getLogger(__name__)
  52         self._logfile = os.path.join(
  53             S.getValue('LOG_DIR'),
  54             S.getValue('LOG_FILE_QEMU')) + str(self._number)
  55         self._timeout = S.getValue('GUEST_TIMEOUT')[self._number]
  56         self._monitor = '%s/vm%dmonitor' % ('/tmp', self._number)
  57         # read GUEST NICs configuration and use only defined NR of NICS
  58         nics_nr = S.getValue('GUEST_NICS_NR')[self._number]
  59         # and inform user about missconfiguration
  60         if nics_nr < 1:
  61             raise RuntimeError('At least one VM NIC is mandotory, but {} '
  62                                'NICs are configured'.format(nics_nr))
  63         elif nics_nr > 1 and nics_nr % 2:
  64             nics_nr = int(nics_nr / 2) * 2
  65             self._logger.warning('Odd number of NICs is configured, only '
  66                                  '%s NICs will be used', nics_nr)
  67
  68         self._nics = S.getValue('GUEST_NICS')[self._number][:nics_nr]
  69
  70         # set guest loopback application based on VNF configuration
  71         self._guest_loopback = S.getValue('GUEST_LOOPBACK')[self._number]
  72
  73         self._testpmd_fwd_mode = S.getValue('GUEST_TESTPMD_FWD_MODE')[self._number]
  74         # in case of SRIOV we must ensure, that MAC addresses are not swapped
  75         if S.getValue('SRIOV_ENABLED') and self._testpmd_fwd_mode.startswith('mac') and \
  76            not str(S.getValue('VNF')).endswith('PciPassthrough'):
  77
  78             self._logger.info("SRIOV detected, forwarding mode of testpmd was changed from '%s' to '%s'",
  79                               self._testpmd_fwd_mode, 'io')
  80             self._testpmd_fwd_mode = 'io'
  81
  82         name = 'Client%d' % self._number
  83         vnc = ':%d' % self._number
  84         # NOTE: affinization of main qemu process can cause hangup of 2nd VM
  85         # in case of DPDK usage. It can also slow down VM response time.
  86         cpumask = ",".join(S.getValue('GUEST_CORE_BINDING')[self._number])
  87         self._cmd = ['sudo', '-E', 'taskset', '-c', cpumask,
  88                      S.getValue('TOOLS')['qemu-system'],
  89                      '-m', S.getValue('GUEST_MEMORY')[self._number],
  90                      '-smp', str(S.getValue('GUEST_SMP')[self._number]),
  91                      '-cpu', str(S.getValue('GUEST_CPU_OPTIONS')[self._number]),
  92                      '-drive', 'if={},file='.format(S.getValue(
  93                          'GUEST_BOOT_DRIVE_TYPE')[self._number]) +
  94                      S.getValue('GUEST_IMAGE')[self._number],
  95                      '-boot', 'c', '--enable-kvm',
  96                      '-monitor', 'unix:%s,server,nowait' % self._monitor,
  97                      '-object',
  98                      'memory-backend-file,id=mem,size=' +
  99                      str(S.getValue('GUEST_MEMORY')[self._number]) + 'M,' +
 100                      'mem-path=' + S.getValue('HUGEPAGE_DIR') + ',share=on',
 101                      '-numa', 'node,memdev=mem -mem-prealloc',
 102                      '-nographic', '-vnc', str(vnc), '-name', name,
 103                      '-snapshot', '-net none', '-no-reboot',
 104                      '-drive',
 105                      'if=%s,format=raw,file=fat:rw:%s,snapshot=off' %
 106                      (S.getValue('GUEST_SHARED_DRIVE_TYPE')[self._number],
 107                       S.getValue('GUEST_SHARE_DIR')[self._number]),
 108                     ]
 109         self._configure_logging()
 110
 111     def _configure_logging(self):
 112         """
 113         Configure logging.
 114         """
 115         self.GuestCommandFilter.prefix = self._log_prefix
 116
 117         logger = logging.getLogger()
 118         cmd_logger = logging.FileHandler(
 119             filename=os.path.join(S.getValue('LOG_DIR'),
 120                                   S.getValue('LOG_FILE_GUEST_CMDS')) +
 121             str(self._number))
 122         cmd_logger.setLevel(logging.DEBUG)
 123         cmd_logger.addFilter(self.GuestCommandFilter())
 124         logger.addHandler(cmd_logger)
 125
 126     # startup/Shutdown
 127
 128     def start(self):
 129         """
 130         Start QEMU instance, login and prepare for commands.
 131         """
 132         super(IVnfQemu, self).start()
 133         if S.getValue('VNF_AFFINITIZATION_ON'):
 134             self._affinitize()
 135
 136         if S.getValue('VSWITCH_VHOST_NET_AFFINITIZATION') and S.getValue(
 137                 'VNF') == 'QemuVirtioNet':
 138             self._affinitize_vhost_net()
 139
 140         if self._timeout:
 141             self._config_guest_loopback()
 142
 143     def stop(self):
 144         """
 145         Stops VNF instance gracefully first.
 146         """
 147         if self.is_running():
 148             try:
 149                 # exit testpmd if needed
 150                 if self._guest_loopback == 'testpmd':
 151                     self.execute_and_wait('stop', 120, "Done")
 152                     self.execute_and_wait('quit', 120, "[bB]ye")
 153
 154                 # turn off VM
 155                 self.execute_and_wait('poweroff', 120, "Power down")
 156
 157             except pexpect.TIMEOUT:
 158                 self.kill()
 159
 160             # wait until qemu shutdowns
 161             self._logger.debug('Wait for QEMU to terminate')
 162             for dummy in range(30):
 163                 time.sleep(1)
 164                 if not self.is_running():
 165                     break
 166
 167             # just for case that graceful shutdown failed
 168             super(IVnfQemu, self).stop()
 169
 170     # helper functions
 171
 172     def _login(self, timeout=120):
 173         """
 174         Login to QEMU instance.
 175
 176         This can be used immediately after booting the machine, provided a
 177         sufficiently long ``timeout`` is given.
 178
 179         :param timeout: Timeout to wait for login to complete.
 180
 181         :returns: None
 182         """
 183         # if no timeout was set, we likely started QEMU without waiting for it
 184         # to boot. This being the case, we best check that it has finished
 185         # first.
 186         if not self._timeout:
 187             self._expect_process(timeout=timeout)
 188
 189         self._child.sendline(S.getValue('GUEST_USERNAME')[self._number])
 190         self._child.expect(S.getValue('GUEST_PROMPT_PASSWORD')[self._number], timeout=5)
 191         self._child.sendline(S.getValue('GUEST_PASSWORD')[self._number])
 192
 193         self._expect_process(S.getValue('GUEST_PROMPT')[self._number], timeout=5)
 194
 195     def send_and_pass(self, cmd, timeout=30):
 196         """
 197         Send ``cmd`` and wait ``timeout`` seconds for it to pass.
 198
 199         :param cmd: Command to send to guest.
 200         :param timeout: Time to wait for prompt before checking return code.
 201
 202         :returns: None
 203         """
 204         self.execute(cmd)
 205         self.wait(S.getValue('GUEST_PROMPT')[self._number], timeout=timeout)
 206         self.execute('echo $?')
 207         self._child.expect('^0$', timeout=1)  # expect a 0
 208         self.wait(S.getValue('GUEST_PROMPT')[self._number], timeout=timeout)
 209
 210     def _affinitize(self):
 211         """
 212         Affinitize the SMP cores of a QEMU instance.
 213
 214         This is a bit of a hack. The 'socat' utility is used to
 215         interact with the QEMU HMP. This is necessary due to the lack
 216         of QMP in older versions of QEMU, like v1.6.2. In future
 217         releases, this should be replaced with calls to libvirt or
 218         another Python-QEMU wrapper library.
 219
 220         :returns: None
 221         """
 222         thread_id = (r'.* CPU #%d: .* thread_id=(\d+)')
 223
 224         self._logger.info('Affinitizing guest...')
 225
 226         cur_locale = locale.getdefaultlocale()[1]
 227         proc = subprocess.Popen(
 228             ('echo', 'info cpus'), stdout=subprocess.PIPE)
 229         output = subprocess.check_output(
 230             ('sudo', 'socat', '-', 'UNIX-CONNECT:%s' % self._monitor),
 231             stdin=proc.stdout)
 232         proc.wait()
 233
 234         for cpu in range(0, int(S.getValue('GUEST_SMP')[self._number])):
 235             match = None
 236             guest_thread_binding = S.getValue('GUEST_THREAD_BINDING')[self._number]
 237             if guest_thread_binding is None:
 238                 guest_thread_binding = S.getValue('GUEST_CORE_BINDING')[self._number]
 239             for line in output.decode(cur_locale).split('\n'):
 240                 match = re.search(thread_id % cpu, line)
 241                 if match:
 242                     self._affinitize_pid(guest_thread_binding[cpu], match.group(1))
 243                     break
 244
 245             if not match:
 246                 self._logger.error('Failed to affinitize guest core #%d. Could'
 247                                    ' not parse tid.', cpu)
 248
 249     def _affinitize_vhost_net(self):
 250         """
 251         Affinitize the vhost net threads for Vanilla OVS and guest nic queues.
 252
 253         :return: None
 254         """
 255         self._logger.info('Affinitizing VHOST Net threads.')
 256         args1 = ['pgrep', 'vhost-']
 257         process1 = subprocess.Popen(args1, stdout=subprocess.PIPE,
 258                                     shell=False)
 259         out = process1.communicate()[0]
 260         processes = out.decode(locale.getdefaultlocale()[1]).split('\n')
 261         if processes[-1] == '':
 262             processes.pop() # pgrep may return an extra line with no data
 263         self._logger.info('Found %s vhost net threads...', len(processes))
 264
 265         cpumap = S.getValue('VSWITCH_VHOST_CPU_MAP')
 266         mapcount = 0
 267         for proc in processes:
 268             self._affinitize_pid(cpumap[mapcount], proc)
 269             mapcount += 1
 270             if mapcount + 1 > len(cpumap):
 271                 # Not enough cpus were given in the mapping to cover all the
 272                 # threads on a 1 to 1 ratio with cpus so reset the list counter
 273                 #  to 0.
 274                 mapcount = 0
 275
 276     def _config_guest_loopback(self):
 277         """
 278         Configure VM to run VNF, e.g. port forwarding application based on the configuration
 279         """
 280         if self._guest_loopback == 'testpmd':
 281             self._login()
 282             self._configure_testpmd()
 283         elif self._guest_loopback == 'l2fwd':
 284             self._login()
 285             self._configure_l2fwd()
 286         elif self._guest_loopback == 'linux_bridge':
 287             self._login()
 288             self._configure_linux_bridge()
 289         elif self._guest_loopback != 'buildin':
 290             self._logger.error('Unsupported guest loopback method "%s" was specified. Option'
 291                                ' "buildin" will be used as a fallback.', self._guest_loopback)
 292
 293     def wait(self, prompt=None, timeout=30):
 294         if prompt is None:
 295             prompt = S.getValue('GUEST_PROMPT')[self._number]
 296         super(IVnfQemu, self).wait(prompt=prompt, timeout=timeout)
 297
 298     def execute_and_wait(self, cmd, timeout=30, prompt=None):
 299         if prompt is None:
 300             prompt = S.getValue('GUEST_PROMPT')[self._number]
 301         super(IVnfQemu, self).execute_and_wait(cmd, timeout=timeout,
 302                                                prompt=prompt)
 303
 304     def _modify_dpdk_makefile(self):
 305         """
 306         Modifies DPDK makefile in Guest before compilation if needed
 307         """
 308         pass
 309
 310     def _configure_copy_sources(self, dirname):
 311         """
 312         Mount shared directory and copy DPDK and l2fwd sources
 313         """
 314         # mount shared directory
 315         self.execute_and_wait('umount /dev/sdb1')
 316         self.execute_and_wait('rm -rf ' + S.getValue('GUEST_OVS_DPDK_DIR')[self._number])
 317         self.execute_and_wait('mkdir -p ' + S.getValue('GUEST_OVS_DPDK_SHARE')[self._number])
 318         self.execute_and_wait('mount -o ro,iocharset=utf8 /dev/sdb1 ' +
 319                               S.getValue('GUEST_OVS_DPDK_SHARE')[self._number])
 320         self.execute_and_wait('mkdir -p ' + S.getValue('GUEST_OVS_DPDK_DIR')[self._number])
 321         self.execute_and_wait('cp -r ' + os.path.join(S.getValue('GUEST_OVS_DPDK_SHARE')[self._number], dirname) +
 322                               ' ' + S.getValue('GUEST_OVS_DPDK_DIR')[self._number])
 323         self.execute_and_wait('umount /dev/sdb1')
 324
 325     def _configure_disable_firewall(self):
 326         """
 327         Disable firewall in VM
 328         """
 329         for iptables in ['iptables', 'ip6tables']:
 330             # filter table
 331             for chain in ['INPUT', 'FORWARD', 'OUTPUT']:
 332                 self.execute_and_wait("{} -t filter -P {} ACCEPT".format(iptables, chain))
 333             # mangle table
 334             for chain in ['PREROUTING', 'INPUT', 'FORWARD', 'OUTPUT', 'POSTROUTING']:
 335                 self.execute_and_wait("{} -t mangle -P {} ACCEPT".format(iptables, chain))
 336             # nat table
 337             for chain in ['PREROUTING', 'INPUT', 'OUTPUT', 'POSTROUTING']:
 338                 self.execute_and_wait("{} -t nat -P {} ACCEPT".format(iptables, chain))
 339
 340             # flush rules and delete chains created by user
 341             for table in ['filter', 'mangle', 'nat']:
 342                 self.execute_and_wait("{} -t {} -F".format(iptables, table))
 343                 self.execute_and_wait("{} -t {} -X".format(iptables, table))
 344
 345     def _configure_testpmd(self):
 346         """
 347         Configure VM to perform L2 forwarding between NICs by DPDK's testpmd
 348         """
 349         self._configure_copy_sources('DPDK')
 350         self._configure_disable_firewall()
 351
 352         # Guest images _should_ have 1024 hugepages by default,
 353         # but just in case:'''
 354         self.execute_and_wait('sysctl vm.nr_hugepages={}'.format(S.getValue('GUEST_HUGEPAGES_NR')[self._number]))
 355
 356         # Mount hugepages
 357         self.execute_and_wait('mkdir -p /dev/hugepages')
 358         self.execute_and_wait(
 359             'mount -t hugetlbfs hugetlbfs /dev/hugepages')
 360
 361         # build and configure system for dpdk
 362         self.execute_and_wait('cd ' + S.getValue('GUEST_OVS_DPDK_DIR')[self._number] +
 363                               '/DPDK')
 364         self.execute_and_wait('export CC=gcc')
 365         self.execute_and_wait('export RTE_SDK=' +
 366                               S.getValue('GUEST_OVS_DPDK_DIR')[self._number] + '/DPDK')
 367         self.execute_and_wait('export RTE_TARGET=%s' % S.getValue('RTE_TARGET'))
 368
 369         # modify makefile if needed
 370         self._modify_dpdk_makefile()
 371
 372         # disable network interfaces, so DPDK can take care of them
 373         for nic in self._nics:
 374             self.execute_and_wait('ifdown ' + nic['device'])
 375
 376         self.execute_and_wait('./tools/dpdk*bind.py --status')
 377         pci_list = ' '.join([nic['pci'] for nic in self._nics])
 378         self.execute_and_wait('./tools/dpdk*bind.py -u ' + pci_list)
 379         self._bind_dpdk_driver(S.getValue(
 380             'GUEST_DPDK_BIND_DRIVER')[self._number], pci_list)
 381         self.execute_and_wait('./tools/dpdk*bind.py --status')
 382
 383         # build and run 'test-pmd'
 384         self.execute_and_wait('cd ' + S.getValue('GUEST_OVS_DPDK_DIR')[self._number] +
 385                               '/DPDK/app/test-pmd')
 386         self.execute_and_wait('make clean')
 387         self.execute_and_wait('make')
 388
 389         # get testpmd settings from CLI
 390         testpmd_params = S.getValue('GUEST_TESTPMD_PARAMS')[self._number]
 391         if S.getValue('VSWITCH_JUMBO_FRAMES_ENABLED'):
 392             testpmd_params += ' --max-pkt-len={}'.format(S.getValue(
 393                 'VSWITCH_JUMBO_FRAMES_SIZE'))
 394
 395         self.execute_and_wait('./testpmd {}'.format(testpmd_params), 60, "Done")
 396         self.execute('set fwd ' + self._testpmd_fwd_mode, 1)
 397         self.execute_and_wait('start', 20, 'testpmd>')
 398
 399     def _configure_l2fwd(self):
 400         """
 401         Configure VM to perform L2 forwarding between NICs by l2fwd module
 402         """
 403         if int(S.getValue('GUEST_NIC_QUEUES')[self._number]):
 404             self._set_multi_queue_nic()
 405         self._configure_copy_sources('l2fwd')
 406         self._configure_disable_firewall()
 407
 408         # configure all interfaces
 409         for nic in self._nics:
 410             self.execute('ip addr add ' +
 411                          nic['ip'] + ' dev ' + nic['device'])
 412             if S.getValue('VSWITCH_JUMBO_FRAMES_ENABLED'):
 413                 self.execute('ifconfig {} mtu {}'.format(
 414                     nic['device'], S.getValue('VSWITCH_JUMBO_FRAMES_SIZE')))
 415             self.execute('ip link set dev ' + nic['device'] + ' up')
 416
 417         # build and configure system for l2fwd
 418         self.execute_and_wait('cd ' + S.getValue('GUEST_OVS_DPDK_DIR')[self._number] +
 419                               '/l2fwd')
 420         self.execute_and_wait('export CC=gcc')
 421
 422         self.execute_and_wait('make')
 423         if len(self._nics) == 2:
 424             self.execute_and_wait('insmod ' + S.getValue('GUEST_OVS_DPDK_DIR')[self._number] +
 425                                   '/l2fwd' + '/l2fwd.ko net1=' + self._nics[0]['device'] +
 426                                   ' net2=' + self._nics[1]['device'])
 427         else:
 428             raise RuntimeError('l2fwd can forward only between 2 NICs, but {} NICs are '
 429                                'configured inside GUEST'.format(len(self._nics)))
 430
 431     def _configure_linux_bridge(self):
 432         """
 433         Configure VM to perform L2 forwarding between NICs by linux bridge
 434         """
 435         if int(S.getValue('GUEST_NIC_QUEUES')[self._number]):
 436             self._set_multi_queue_nic()
 437         self._configure_disable_firewall()
 438
 439         # configure linux bridge
 440         self.execute('brctl addbr br0')
 441
 442         # add all NICs into the bridge
 443         for nic in self._nics:
 444             self.execute('ip addr add ' +
 445                          nic['ip'] + ' dev ' + nic['device'])
 446             if S.getValue('VSWITCH_JUMBO_FRAMES_ENABLED'):
 447                 self.execute('ifconfig {} mtu {}'.format(
 448                     nic['device'], S.getValue('VSWITCH_JUMBO_FRAMES_SIZE')))
 449             self.execute('ip link set dev ' + nic['device'] + ' up')
 450             self.execute('brctl addif br0 ' + nic['device'])
 451
 452         self.execute('ip addr add ' +
 453                      S.getValue('GUEST_BRIDGE_IP')[self._number] +
 454                      ' dev br0')
 455         self.execute('ip link set dev br0 up')
 456
 457         # Add the arp entries for the IXIA ports and the bridge you are using.
 458         # Use command line values if provided.
 459         trafficgen_mac = S.getValue('VANILLA_TGEN_PORT1_MAC')
 460         trafficgen_ip = S.getValue('VANILLA_TGEN_PORT1_IP')
 461
 462         self.execute('arp -s ' + trafficgen_ip + ' ' + trafficgen_mac)
 463
 464         trafficgen_mac = S.getValue('VANILLA_TGEN_PORT2_MAC')
 465         trafficgen_ip = S.getValue('VANILLA_TGEN_PORT2_IP')
 466
 467         self.execute('arp -s ' + trafficgen_ip + ' ' + trafficgen_mac)
 468
 469         # Enable forwarding
 470         self.execute('sysctl -w net.ipv4.ip_forward=1')
 471
 472         # Controls source route verification
 473         # 0 means no source validation
 474         self.execute('sysctl -w net.ipv4.conf.all.rp_filter=0')
 475         for nic in self._nics:
 476             self.execute('sysctl -w net.ipv4.conf.' + nic['device'] + '.rp_filter=0')
 477
 478     def _bind_dpdk_driver(self, driver, pci_slots):
 479         """
 480         Bind the virtual nics to the driver specific in the conf file
 481         :return: None
 482         """
 483         if driver == 'uio_pci_generic':
 484             if S.getValue('VNF') == 'QemuPciPassthrough':
 485                 # unsupported config, bind to igb_uio instead and exit the
 486                 # outer function after completion.
 487                 self._logger.error('SR-IOV does not support uio_pci_generic. '
 488                                    'Igb_uio will be used instead.')
 489                 self._bind_dpdk_driver('igb_uio_from_src', pci_slots)
 490                 return
 491             self.execute_and_wait('modprobe uio_pci_generic')
 492             self.execute_and_wait('./tools/dpdk*bind.py -b uio_pci_generic '+
 493                                   pci_slots)
 494         elif driver == 'vfio_no_iommu':
 495             self.execute_and_wait('modprobe -r vfio')
 496             self.execute_and_wait('modprobe -r vfio_iommu_type1')
 497             self.execute_and_wait('modprobe vfio enable_unsafe_noiommu_mode=Y')
 498             self.execute_and_wait('modprobe vfio-pci')
 499             self.execute_and_wait('./tools/dpdk*bind.py -b vfio-pci ' +
 500                                   pci_slots)
 501         elif driver == 'igb_uio_from_src':
 502             # build and insert igb_uio and rebind interfaces to it
 503             self.execute_and_wait('make RTE_OUTPUT=$RTE_SDK/$RTE_TARGET -C '
 504                                   '$RTE_SDK/lib/librte_eal/linuxapp/igb_uio')
 505             self.execute_and_wait('modprobe uio')
 506             self.execute_and_wait('insmod %s/kmod/igb_uio.ko' %
 507                                   S.getValue('RTE_TARGET'))
 508             self.execute_and_wait('./tools/dpdk*bind.py -b igb_uio ' + pci_slots)
 509         else:
 510             self._logger.error(
 511                 'Unknown driver for binding specified, defaulting to igb_uio')
 512             self._bind_dpdk_driver('igb_uio_from_src', pci_slots)
 513
 514     def _set_multi_queue_nic(self):
 515         """
 516         Enable multi-queue in guest kernel with ethool.
 517         :return: None
 518         """
 519         for nic in self._nics:
 520             self.execute_and_wait('ethtool -L {} combined {}'.format(
 521                 nic['device'], S.getValue('GUEST_NIC_QUEUES')[self._number]))
 522             self.execute_and_wait('ethtool -l {}'.format(nic['device']))