vnfs/qemu/qemu.py

   1 # Copyright 2015-2016 Intel Corporation.
   2 #
   3 # Licensed under the Apache License, Version 2.0 (the "License");
   4 # you may not use this file except in compliance with the License.
   5 # You may obtain a copy of the License at
   6 #
   7 #   http://www.apache.org/licenses/LICENSE-2.0
   8 #
   9 # Unless required by applicable law or agreed to in writing, software
  10 # distributed under the License is distributed on an "AS IS" BASIS,
  11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12 # See the License for the specific language governing permissions and
  13 # limitations under the License.
  14
  15 """Automation of QEMU hypervisor for launching guests.
  16 """
  17
  18 import os
  19 import logging
  20 import locale
  21 import re
  22 import subprocess
  23 import time
  24 import pexpect
  25
  26 from conf import settings as S
  27 from conf import get_test_param
  28 from vnfs.vnf.vnf import IVnf
  29
  30 class IVnfQemu(IVnf):
  31     """
  32     Abstract class for controling an instance of QEMU
  33     """
  34     _cmd = None
  35     _expect = None
  36     _proc_name = 'qemu'
  37
  38     class GuestCommandFilter(logging.Filter):
  39         """
  40         Filter out strings beginning with 'guestcmd :'.
  41         """
  42         def filter(self, record):
  43             return record.getMessage().startswith(self.prefix)
  44
  45     def __init__(self):
  46         """
  47         Initialisation function.
  48         """
  49         super(IVnfQemu, self).__init__()
  50
  51         self._expect = S.getValue('GUEST_PROMPT_LOGIN')[self._number]
  52         self._logger = logging.getLogger(__name__)
  53         self._logfile = os.path.join(
  54             S.getValue('LOG_DIR'),
  55             S.getValue('LOG_FILE_QEMU')) + str(self._number)
  56         self._timeout = S.getValue('GUEST_TIMEOUT')[self._number]
  57         self._monitor = '%s/vm%dmonitor' % ('/tmp', self._number)
  58         # read GUEST NICs configuration and use only defined NR of NICS
  59         nics_nr = S.getValue('GUEST_NICS_NR')[self._number]
  60         # and inform user about missconfiguration
  61         if nics_nr < 1:
  62             raise RuntimeError('At least one VM NIC is mandotory, but {} '
  63                                'NICs are configured'.format(nics_nr))
  64         elif nics_nr > 1 and nics_nr % 2:
  65             nics_nr = int(nics_nr / 2) * 2
  66             self._logger.warning('Odd number of NICs is configured, only '
  67                                  '%s NICs will be used', nics_nr)
  68
  69         self._nics = S.getValue('GUEST_NICS')[self._number][:nics_nr]
  70
  71         # set guest loopback application based on VNF configuration
  72         # cli option take precedence to config file values
  73         self._guest_loopback = S.getValue('GUEST_LOOPBACK')[self._number]
  74
  75         self._testpmd_fwd_mode = S.getValue('GUEST_TESTPMD_FWD_MODE')[self._number]
  76         # in case of SRIOV we must ensure, that MAC addresses are not swapped
  77         if S.getValue('SRIOV_ENABLED') and self._testpmd_fwd_mode.startswith('mac') and \
  78            not S.getValue('VNF').endswith('PciPassthrough'):
  79
  80             self._logger.info("SRIOV detected, forwarding mode of testpmd was changed from '%s' to '%s'",
  81                               self._testpmd_fwd_mode, 'io')
  82             self._testpmd_fwd_mode = 'io'
  83
  84         name = 'Client%d' % self._number
  85         vnc = ':%d' % self._number
  86         # NOTE: affinization of main qemu process can cause hangup of 2nd VM
  87         # in case of DPDK usage. It can also slow down VM response time.
  88         cpumask = ",".join(S.getValue('GUEST_CORE_BINDING')[self._number])
  89         self._cmd = ['sudo', '-E', 'taskset', '-c', cpumask,
  90                      S.getValue('TOOLS')['qemu-system'],
  91                      '-m', S.getValue('GUEST_MEMORY')[self._number],
  92                      '-smp', str(S.getValue('GUEST_SMP')[self._number]),
  93                      '-cpu', 'host,migratable=off',
  94                      '-drive', 'if={},file='.format(S.getValue(
  95                          'GUEST_BOOT_DRIVE_TYPE')[self._number]) +
  96                      S.getValue('GUEST_IMAGE')[self._number],
  97                      '-boot', 'c', '--enable-kvm',
  98                      '-monitor', 'unix:%s,server,nowait' % self._monitor,
  99                      '-object',
 100                      'memory-backend-file,id=mem,size=' +
 101                      str(S.getValue('GUEST_MEMORY')[self._number]) + 'M,' +
 102                      'mem-path=' + S.getValue('HUGEPAGE_DIR') + ',share=on',
 103                      '-numa', 'node,memdev=mem -mem-prealloc',
 104                      '-nographic', '-vnc', str(vnc), '-name', name,
 105                      '-snapshot', '-net none', '-no-reboot',
 106                      '-drive',
 107                      'if=%s,format=raw,file=fat:rw:%s,snapshot=off' %
 108                      (S.getValue('GUEST_SHARED_DRIVE_TYPE')[self._number],
 109                       S.getValue('GUEST_SHARE_DIR')[self._number]),
 110                     ]
 111         self._configure_logging()
 112
 113     def _configure_logging(self):
 114         """
 115         Configure logging.
 116         """
 117         self.GuestCommandFilter.prefix = self._log_prefix
 118
 119         logger = logging.getLogger()
 120         cmd_logger = logging.FileHandler(
 121             filename=os.path.join(S.getValue('LOG_DIR'),
 122                                   S.getValue('LOG_FILE_GUEST_CMDS')) +
 123             str(self._number))
 124         cmd_logger.setLevel(logging.DEBUG)
 125         cmd_logger.addFilter(self.GuestCommandFilter())
 126         logger.addHandler(cmd_logger)
 127
 128     # startup/Shutdown
 129
 130     def start(self):
 131         """
 132         Start QEMU instance, login and prepare for commands.
 133         """
 134         super(IVnfQemu, self).start()
 135         if S.getValue('VNF_AFFINITIZATION_ON'):
 136             self._affinitize()
 137
 138         if S.getValue('VSWITCH_VHOST_NET_AFFINITIZATION') and S.getValue(
 139                 'VNF') == 'QemuVirtioNet':
 140             self._affinitize_vhost_net()
 141
 142         if self._timeout:
 143             self._config_guest_loopback()
 144
 145     def stop(self):
 146         """
 147         Stops VNF instance gracefully first.
 148         """
 149         try:
 150             # exit testpmd if needed
 151             if self._guest_loopback == 'testpmd':
 152                 self.execute_and_wait('stop', 120, "Done")
 153                 self.execute_and_wait('quit', 120, "[bB]ye")
 154
 155             # turn off VM
 156             self.execute_and_wait('poweroff', 120, "Power down")
 157
 158         except pexpect.TIMEOUT:
 159             self.kill()
 160
 161         # wait until qemu shutdowns
 162         self._logger.debug('Wait for QEMU to terminate')
 163         for dummy in range(30):
 164             time.sleep(1)
 165             if not self.is_running():
 166                 break
 167
 168         # just for case that graceful shutdown failed
 169         super(IVnfQemu, self).stop()
 170
 171     # helper functions
 172
 173     def _login(self, timeout=120):
 174         """
 175         Login to QEMU instance.
 176
 177         This can be used immediately after booting the machine, provided a
 178         sufficiently long ``timeout`` is given.
 179
 180         :param timeout: Timeout to wait for login to complete.
 181
 182         :returns: None
 183         """
 184         # if no timeout was set, we likely started QEMU without waiting for it
 185         # to boot. This being the case, we best check that it has finished
 186         # first.
 187         if not self._timeout:
 188             self._expect_process(timeout=timeout)
 189
 190         self._child.sendline(S.getValue('GUEST_USERNAME')[self._number])
 191         self._child.expect(S.getValue('GUEST_PROMPT_PASSWORD')[self._number], timeout=5)
 192         self._child.sendline(S.getValue('GUEST_PASSWORD')[self._number])
 193
 194         self._expect_process(S.getValue('GUEST_PROMPT')[self._number], timeout=5)
 195
 196     def send_and_pass(self, cmd, timeout=30):
 197         """
 198         Send ``cmd`` and wait ``timeout`` seconds for it to pass.
 199
 200         :param cmd: Command to send to guest.
 201         :param timeout: Time to wait for prompt before checking return code.
 202
 203         :returns: None
 204         """
 205         self.execute(cmd)
 206         self.wait(S.getValue('GUEST_PROMPT')[self._number], timeout=timeout)
 207         self.execute('echo $?')
 208         self._child.expect('^0$', timeout=1)  # expect a 0
 209         self.wait(S.getValue('GUEST_PROMPT')[self._number], timeout=timeout)
 210
 211     def _affinitize(self):
 212         """
 213         Affinitize the SMP cores of a QEMU instance.
 214
 215         This is a bit of a hack. The 'socat' utility is used to
 216         interact with the QEMU HMP. This is necessary due to the lack
 217         of QMP in older versions of QEMU, like v1.6.2. In future
 218         releases, this should be replaced with calls to libvirt or
 219         another Python-QEMU wrapper library.
 220
 221         :returns: None
 222         """
 223         thread_id = (r'.* CPU #%d: .* thread_id=(\d+)')
 224
 225         self._logger.info('Affinitizing guest...')
 226
 227         cur_locale = locale.getdefaultlocale()[1]
 228         proc = subprocess.Popen(
 229             ('echo', 'info cpus'), stdout=subprocess.PIPE)
 230         output = subprocess.check_output(
 231             ('sudo', 'socat', '-', 'UNIX-CONNECT:%s' % self._monitor),
 232             stdin=proc.stdout)
 233         proc.wait()
 234
 235         for cpu in range(0, int(S.getValue('GUEST_SMP')[self._number])):
 236             match = None
 237             for line in output.decode(cur_locale).split('\n'):
 238                 match = re.search(thread_id % cpu, line)
 239                 if match:
 240                     self._affinitize_pid(
 241                         S.getValue('GUEST_CORE_BINDING')[self._number][cpu],
 242                         match.group(1))
 243                     break
 244
 245             if not match:
 246                 self._logger.error('Failed to affinitize guest core #%d. Could'
 247                                    ' not parse tid.', cpu)
 248
 249     def _affinitize_vhost_net(self):
 250         """
 251         Affinitize the vhost net threads for Vanilla OVS and guest nic queues.
 252
 253         :return: None
 254         """
 255         self._logger.info('Affinitizing VHOST Net threads.')
 256         args1 = ['pgrep', 'vhost-']
 257         process1 = subprocess.Popen(args1, stdout=subprocess.PIPE,
 258                                     shell=False)
 259         out = process1.communicate()[0]
 260         processes = out.decode(locale.getdefaultlocale()[1]).split('\n')
 261         if processes[-1] == '':
 262             processes.pop() # pgrep may return an extra line with no data
 263         self._logger.info('Found %s vhost net threads...', len(processes))
 264
 265         cpumap = S.getValue('VSWITCH_VHOST_CPU_MAP')
 266         mapcount = 0
 267         for proc in processes:
 268             self._affinitize_pid(cpumap[mapcount], proc)
 269             mapcount += 1
 270             if mapcount + 1 > len(cpumap):
 271                 # Not enough cpus were given in the mapping to cover all the
 272                 # threads on a 1 to 1 ratio with cpus so reset the list counter
 273                 #  to 0.
 274                 mapcount = 0
 275
 276     def _config_guest_loopback(self):
 277         """
 278         Configure VM to run VNF, e.g. port forwarding application based on the configuration
 279         """
 280         if self._guest_loopback == 'testpmd':
 281             self._login()
 282             self._configure_testpmd()
 283         elif self._guest_loopback == 'l2fwd':
 284             self._login()
 285             self._configure_l2fwd()
 286         elif self._guest_loopback == 'linux_bridge':
 287             self._login()
 288             self._configure_linux_bridge()
 289         elif self._guest_loopback != 'buildin':
 290             self._logger.error('Unsupported guest loopback method "%s" was specified. Option'
 291                                ' "buildin" will be used as a fallback.', self._guest_loopback)
 292
 293     def wait(self, prompt=None, timeout=30):
 294         if prompt is None:
 295             prompt = S.getValue('GUEST_PROMPT')[self._number]
 296         super(IVnfQemu, self).wait(prompt=prompt, timeout=timeout)
 297
 298     def execute_and_wait(self, cmd, timeout=30, prompt=None):
 299         if prompt is None:
 300             prompt = S.getValue('GUEST_PROMPT')[self._number]
 301         super(IVnfQemu, self).execute_and_wait(cmd, timeout=timeout,
 302                                                prompt=prompt)
 303
 304     def _modify_dpdk_makefile(self):
 305         """
 306         Modifies DPDK makefile in Guest before compilation if needed
 307         """
 308         pass
 309
 310     def _configure_copy_sources(self, dirname):
 311         """
 312         Mount shared directory and copy DPDK and l2fwd sources
 313         """
 314         # mount shared directory
 315         self.execute_and_wait('umount /dev/sdb1')
 316         self.execute_and_wait('rm -rf ' + S.getValue('GUEST_OVS_DPDK_DIR')[self._number])
 317         self.execute_and_wait('mkdir -p ' + S.getValue('GUEST_OVS_DPDK_SHARE')[self._number])
 318         self.execute_and_wait('mount -o ro,iocharset=utf8 /dev/sdb1 ' +
 319                               S.getValue('GUEST_OVS_DPDK_SHARE')[self._number])
 320         self.execute_and_wait('mkdir -p ' + S.getValue('GUEST_OVS_DPDK_DIR')[self._number])
 321         self.execute_and_wait('cp -r ' + os.path.join(S.getValue('GUEST_OVS_DPDK_SHARE')[self._number], dirname) +
 322                               ' ' + S.getValue('GUEST_OVS_DPDK_DIR')[self._number])
 323         self.execute_and_wait('umount /dev/sdb1')
 324
 325     def _configure_disable_firewall(self):
 326         """
 327         Disable firewall in VM
 328         """
 329         for iptables in ['iptables', 'ip6tables']:
 330             # filter table
 331             for chain in ['INPUT', 'FORWARD', 'OUTPUT']:
 332                 self.execute_and_wait("{} -t filter -P {} ACCEPT".format(iptables, chain))
 333             # mangle table
 334             for chain in ['PREROUTING', 'INPUT', 'FORWARD', 'OUTPUT', 'POSTROUTING']:
 335                 self.execute_and_wait("{} -t mangle -P {} ACCEPT".format(iptables, chain))
 336             # nat table
 337             for chain in ['PREROUTING', 'INPUT', 'OUTPUT', 'POSTROUTING']:
 338                 self.execute_and_wait("{} -t nat -P {} ACCEPT".format(iptables, chain))
 339
 340             # flush rules and delete chains created by user
 341             for table in ['filter', 'mangle', 'nat']:
 342                 self.execute_and_wait("{} -t {} -F".format(iptables, table))
 343                 self.execute_and_wait("{} -t {} -X".format(iptables, table))
 344
 345
 346     def _configure_testpmd(self):
 347         """
 348         Configure VM to perform L2 forwarding between NICs by DPDK's testpmd
 349         """
 350         self._configure_copy_sources('DPDK')
 351         self._configure_disable_firewall()
 352
 353         # Guest images _should_ have 1024 hugepages by default,
 354         # but just in case:'''
 355         self.execute_and_wait('sysctl vm.nr_hugepages={}'.format(S.getValue('GUEST_HUGEPAGES_NR')[self._number]))
 356
 357         # Mount hugepages
 358         self.execute_and_wait('mkdir -p /dev/hugepages')
 359         self.execute_and_wait(
 360             'mount -t hugetlbfs hugetlbfs /dev/hugepages')
 361
 362         # build and configure system for dpdk
 363         self.execute_and_wait('cd ' + S.getValue('GUEST_OVS_DPDK_DIR')[self._number] +
 364                               '/DPDK')
 365         self.execute_and_wait('export CC=gcc')
 366         self.execute_and_wait('export RTE_SDK=' +
 367                               S.getValue('GUEST_OVS_DPDK_DIR')[self._number] + '/DPDK')
 368         self.execute_and_wait('export RTE_TARGET=%s' % S.getValue('RTE_TARGET'))
 369
 370         # modify makefile if needed
 371         self._modify_dpdk_makefile()
 372
 373         # disable network interfaces, so DPDK can take care of them
 374         for nic in self._nics:
 375             self.execute_and_wait('ifdown ' + nic['device'])
 376
 377         # build and insert igb_uio and rebind interfaces to it
 378         self.execute_and_wait('make RTE_OUTPUT=$RTE_SDK/$RTE_TARGET -C '
 379                               '$RTE_SDK/lib/librte_eal/linuxapp/igb_uio')
 380         self.execute_and_wait('modprobe uio')
 381         self.execute_and_wait('insmod %s/kmod/igb_uio.ko' %
 382                               S.getValue('RTE_TARGET'))
 383         self.execute_and_wait('./tools/dpdk*bind.py --status')
 384         pci_list = ' '.join([nic['pci'] for nic in self._nics])
 385         self.execute_and_wait('./tools/dpdk*bind.py -u ' + pci_list)
 386         self.execute_and_wait('./tools/dpdk*bind.py -b igb_uio ' + pci_list)
 387         self.execute_and_wait('./tools/dpdk*bind.py --status')
 388
 389         # build and run 'test-pmd'
 390         self.execute_and_wait('cd ' + S.getValue('GUEST_OVS_DPDK_DIR')[self._number] +
 391                               '/DPDK/app/test-pmd')
 392         self.execute_and_wait('make clean')
 393         self.execute_and_wait('make')
 394         if int(S.getValue('GUEST_NIC_QUEUES')[self._number]):
 395             self.execute_and_wait(
 396                 './testpmd {} -n4 --socket-mem 512 --'.format(
 397                     S.getValue('GUEST_TESTPMD_CPU_MASK')[self._number]) +
 398                 ' --burst=64 -i --txqflags=0xf00 ' +
 399                 '--nb-cores={} --rxq={} --txq={} '.format(
 400                     S.getValue('GUEST_TESTPMD_NB_CORES')[self._number],
 401                     S.getValue('GUEST_TESTPMD_TXQ')[self._number],
 402                     S.getValue('GUEST_TESTPMD_RXQ')[self._number]) +
 403                 '--disable-hw-vlan', 60, "Done")
 404         else:
 405             self.execute_and_wait(
 406                 './testpmd {} -n 4 --socket-mem 512 --'.format(
 407                     S.getValue('GUEST_TESTPMD_CPU_MASK')[self._number]) +
 408                 ' --burst=64 -i --txqflags=0xf00 ' +
 409                 '--disable-hw-vlan', 60, "Done")
 410         self.execute('set fwd ' + self._testpmd_fwd_mode, 1)
 411         self.execute_and_wait('start', 20,
 412                               'TX RS bit threshold=.+ - TXQ flags=0xf00')
 413
 414     def _configure_l2fwd(self):
 415         """
 416         Configure VM to perform L2 forwarding between NICs by l2fwd module
 417         """
 418         if int(S.getValue('GUEST_NIC_QUEUES')[self._number]):
 419             self._set_multi_queue_nic()
 420         self._configure_copy_sources('l2fwd')
 421         self._configure_disable_firewall()
 422
 423         # configure all interfaces
 424         for nic in self._nics:
 425             self.execute('ip addr add ' +
 426                          nic['ip'] + ' dev ' + nic['device'])
 427             self.execute('ip link set dev ' + nic['device'] + ' up')
 428
 429         # build and configure system for l2fwd
 430         self.execute_and_wait('cd ' + S.getValue('GUEST_OVS_DPDK_DIR')[self._number] +
 431                               '/l2fwd')
 432         self.execute_and_wait('export CC=gcc')
 433
 434         self.execute_and_wait('make')
 435         if len(self._nics) == 2:
 436             self.execute_and_wait('insmod ' + S.getValue('GUEST_OVS_DPDK_DIR')[self._number] +
 437                                   '/l2fwd' + '/l2fwd.ko net1=' + self._nics[0]['device'] +
 438                                   ' net2=' + self._nics[1]['device'])
 439         else:
 440             raise RuntimeError('l2fwd can forward only between 2 NICs, but {} NICs are '
 441                                'configured inside GUEST'.format(len(self._nics)))
 442
 443     def _configure_linux_bridge(self):
 444         """
 445         Configure VM to perform L2 forwarding between NICs by linux bridge
 446         """
 447         if int(S.getValue('GUEST_NIC_QUEUES')[self._number]):
 448             self._set_multi_queue_nic()
 449         self._configure_disable_firewall()
 450
 451         # configure linux bridge
 452         self.execute('brctl addbr br0')
 453
 454         # add all NICs into the bridge
 455         for nic in self._nics:
 456             self.execute('ip addr add ' +
 457                          nic['ip'] + ' dev ' + nic['device'])
 458             self.execute('ip link set dev ' + nic['device'] + ' up')
 459             self.execute('brctl addif br0 ' + nic['device'])
 460
 461         self.execute('ip addr add ' +
 462                      S.getValue('GUEST_BRIDGE_IP')[self._number] +
 463                      ' dev br0')
 464         self.execute('ip link set dev br0 up')
 465
 466         # Add the arp entries for the IXIA ports and the bridge you are using.
 467         # Use command line values if provided.
 468         trafficgen_mac = get_test_param('vanilla_tgen_port1_mac',
 469                                         S.getValue('VANILLA_TGEN_PORT1_MAC'))
 470         trafficgen_ip = get_test_param('vanilla_tgen_port1_ip',
 471                                        S.getValue('VANILLA_TGEN_PORT1_IP'))
 472
 473         self.execute('arp -s ' + trafficgen_ip + ' ' + trafficgen_mac)
 474
 475         trafficgen_mac = get_test_param('vanilla_tgen_port2_mac',
 476                                         S.getValue('VANILLA_TGEN_PORT2_MAC'))
 477         trafficgen_ip = get_test_param('vanilla_tgen_port2_ip',
 478                                        S.getValue('VANILLA_TGEN_PORT2_IP'))
 479
 480         self.execute('arp -s ' + trafficgen_ip + ' ' + trafficgen_mac)
 481
 482         # Enable forwarding
 483         self.execute('sysctl -w net.ipv4.ip_forward=1')
 484
 485         # Controls source route verification
 486         # 0 means no source validation
 487         self.execute('sysctl -w net.ipv4.conf.all.rp_filter=0')
 488         for nic in self._nics:
 489             self.execute('sysctl -w net.ipv4.conf.' + nic['device'] + '.rp_filter=0')
 490
 491     def _set_multi_queue_nic(self):
 492         """
 493         Enable multi-queue in guest kernel with ethool.
 494         :return: None
 495         """
 496         for nic in self._nics:
 497             self.execute_and_wait('ethtool -L {} combined {}'.format(
 498                 nic['device'], S.getValue('GUEST_NIC_QUEUES')[self._number]))
 499             self.execute_and_wait('ethtool -l {}'.format(nic['device']))