vnfs/qemu/qemu.py

   1 # Copyright 2015-2016 Intel Corporation.
   2 #
   3 # Licensed under the Apache License, Version 2.0 (the "License");
   4 # you may not use this file except in compliance with the License.
   5 # You may obtain a copy of the License at
   6 #
   7 #   http://www.apache.org/licenses/LICENSE-2.0
   8 #
   9 # Unless required by applicable law or agreed to in writing, software
  10 # distributed under the License is distributed on an "AS IS" BASIS,
  11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12 # See the License for the specific language governing permissions and
  13 # limitations under the License.
  14
  15 """Automation of QEMU hypervisor for launching guests.
  16 """
  17
  18 import os
  19 import logging
  20 import locale
  21 import re
  22 import subprocess
  23 import time
  24 import pexpect
  25
  26 from conf import settings as S
  27 from conf import get_test_param
  28 from vnfs.vnf.vnf import IVnf
  29
  30 class IVnfQemu(IVnf):
  31     """
  32     Abstract class for controling an instance of QEMU
  33     """
  34     _cmd = None
  35     _expect = None
  36     _proc_name = 'qemu'
  37
  38     class GuestCommandFilter(logging.Filter):
  39         """
  40         Filter out strings beginning with 'guestcmd :'.
  41         """
  42         def filter(self, record):
  43             return record.getMessage().startswith(self.prefix)
  44
  45     def __init__(self):
  46         """
  47         Initialisation function.
  48         """
  49         super(IVnfQemu, self).__init__()
  50
  51         self._expect = S.getValue('GUEST_PROMPT_LOGIN')[self._number]
  52         self._logger = logging.getLogger(__name__)
  53         self._logfile = os.path.join(
  54             S.getValue('LOG_DIR'),
  55             S.getValue('LOG_FILE_QEMU')) + str(self._number)
  56         self._timeout = S.getValue('GUEST_TIMEOUT')[self._number]
  57         self._monitor = '%s/vm%dmonitor' % ('/tmp', self._number)
  58         # read GUEST NICs configuration and use only defined NR of NICS
  59         nics_nr = S.getValue('GUEST_NICS_NR')[self._number]
  60         # and inform user about missconfiguration
  61         if nics_nr < 1:
  62             raise RuntimeError('At least one VM NIC is mandotory, but {} '
  63                                'NICs are configured'.format(nics_nr))
  64         elif nics_nr > 1 and nics_nr % 2:
  65             nics_nr = int(nics_nr / 2) * 2
  66             self._logger.warning('Odd number of NICs is configured, only '
  67                                  '%s NICs will be used', nics_nr)
  68
  69         self._nics = S.getValue('GUEST_NICS')[self._number][:nics_nr]
  70
  71         # set guest loopback application based on VNF configuration
  72         # cli option take precedence to config file values
  73         self._guest_loopback = S.getValue('GUEST_LOOPBACK')[self._number]
  74
  75         self._testpmd_fwd_mode = S.getValue('GUEST_TESTPMD_FWD_MODE')[self._number]
  76         # in case of SRIOV we must ensure, that MAC addresses are not swapped
  77         if S.getValue('SRIOV_ENABLED') and self._testpmd_fwd_mode.startswith('mac') and \
  78            not S.getValue('VNF').endswith('PciPassthrough'):
  79
  80             self._logger.info("SRIOV detected, forwarding mode of testpmd was changed from '%s' to '%s'",
  81                               self._testpmd_fwd_mode, 'io')
  82             self._testpmd_fwd_mode = 'io'
  83
  84         name = 'Client%d' % self._number
  85         vnc = ':%d' % self._number
  86         # don't use taskset to affinize main qemu process; It causes hangup
  87         # of 2nd VM in case of DPDK. It also slows down VM responsivnes.
  88         self._cmd = ['sudo', '-E', S.getValue('TOOLS')['qemu-system'],
  89                      '-m', S.getValue('GUEST_MEMORY')[self._number],
  90                      '-smp', str(S.getValue('GUEST_SMP')[self._number]),
  91                      '-cpu', 'host,migratable=off',
  92                      '-drive', 'if={},file='.format(S.getValue(
  93                          'GUEST_BOOT_DRIVE_TYPE')[self._number]) +
  94                      S.getValue('GUEST_IMAGE')[self._number],
  95                      '-boot', 'c', '--enable-kvm',
  96                      '-monitor', 'unix:%s,server,nowait' % self._monitor,
  97                      '-object',
  98                      'memory-backend-file,id=mem,size=' +
  99                      str(S.getValue('GUEST_MEMORY')[self._number]) + 'M,' +
 100                      'mem-path=' + S.getValue('HUGEPAGE_DIR') + ',share=on',
 101                      '-numa', 'node,memdev=mem -mem-prealloc',
 102                      '-nographic', '-vnc', str(vnc), '-name', name,
 103                      '-snapshot', '-net none', '-no-reboot',
 104                      '-drive',
 105                      'if=%s,format=raw,file=fat:rw:%s,snapshot=off' %
 106                      (S.getValue('GUEST_SHARED_DRIVE_TYPE')[self._number],
 107                       S.getValue('GUEST_SHARE_DIR')[self._number]),
 108                     ]
 109         self._configure_logging()
 110
 111     def _configure_logging(self):
 112         """
 113         Configure logging.
 114         """
 115         self.GuestCommandFilter.prefix = self._log_prefix
 116
 117         logger = logging.getLogger()
 118         cmd_logger = logging.FileHandler(
 119             filename=os.path.join(S.getValue('LOG_DIR'),
 120                                   S.getValue('LOG_FILE_GUEST_CMDS')) +
 121             str(self._number))
 122         cmd_logger.setLevel(logging.DEBUG)
 123         cmd_logger.addFilter(self.GuestCommandFilter())
 124         logger.addHandler(cmd_logger)
 125
 126     # startup/Shutdown
 127
 128     def start(self):
 129         """
 130         Start QEMU instance, login and prepare for commands.
 131         """
 132         super(IVnfQemu, self).start()
 133         if S.getValue('VNF_AFFINITIZATION_ON'):
 134             self._affinitize()
 135
 136         if S.getValue('VSWITCH_VHOST_NET_AFFINITIZATION') and S.getValue(
 137                 'VNF') == 'QemuVirtioNet':
 138             self._affinitize_vhost_net()
 139
 140         if self._timeout:
 141             self._config_guest_loopback()
 142
 143     def stop(self):
 144         """
 145         Stops VNF instance gracefully first.
 146         """
 147         try:
 148             # exit testpmd if needed
 149             if self._guest_loopback == 'testpmd':
 150                 self.execute_and_wait('stop', 120, "Done")
 151                 self.execute_and_wait('quit', 120, "[bB]ye")
 152
 153             # turn off VM
 154             self.execute_and_wait('poweroff', 120, "Power down")
 155
 156         except pexpect.TIMEOUT:
 157             self.kill()
 158
 159         # wait until qemu shutdowns
 160         self._logger.debug('Wait for QEMU to terminate')
 161         for dummy in range(30):
 162             time.sleep(1)
 163             if not self.is_running():
 164                 break
 165
 166         # just for case that graceful shutdown failed
 167         super(IVnfQemu, self).stop()
 168
 169     # helper functions
 170
 171     def _login(self, timeout=120):
 172         """
 173         Login to QEMU instance.
 174
 175         This can be used immediately after booting the machine, provided a
 176         sufficiently long ``timeout`` is given.
 177
 178         :param timeout: Timeout to wait for login to complete.
 179
 180         :returns: None
 181         """
 182         # if no timeout was set, we likely started QEMU without waiting for it
 183         # to boot. This being the case, we best check that it has finished
 184         # first.
 185         if not self._timeout:
 186             self._expect_process(timeout=timeout)
 187
 188         self._child.sendline(S.getValue('GUEST_USERNAME')[self._number])
 189         self._child.expect(S.getValue('GUEST_PROMPT_PASSWORD')[self._number], timeout=5)
 190         self._child.sendline(S.getValue('GUEST_PASSWORD')[self._number])
 191
 192         self._expect_process(S.getValue('GUEST_PROMPT')[self._number], timeout=5)
 193
 194     def send_and_pass(self, cmd, timeout=30):
 195         """
 196         Send ``cmd`` and wait ``timeout`` seconds for it to pass.
 197
 198         :param cmd: Command to send to guest.
 199         :param timeout: Time to wait for prompt before checking return code.
 200
 201         :returns: None
 202         """
 203         self.execute(cmd)
 204         self.wait(S.getValue('GUEST_PROMPT')[self._number], timeout=timeout)
 205         self.execute('echo $?')
 206         self._child.expect('^0$', timeout=1)  # expect a 0
 207         self.wait(S.getValue('GUEST_PROMPT')[self._number], timeout=timeout)
 208
 209     def _affinitize(self):
 210         """
 211         Affinitize the SMP cores of a QEMU instance.
 212
 213         This is a bit of a hack. The 'socat' utility is used to
 214         interact with the QEMU HMP. This is necessary due to the lack
 215         of QMP in older versions of QEMU, like v1.6.2. In future
 216         releases, this should be replaced with calls to libvirt or
 217         another Python-QEMU wrapper library.
 218
 219         :returns: None
 220         """
 221         thread_id = (r'.* CPU #%d: .* thread_id=(\d+)')
 222
 223         self._logger.info('Affinitizing guest...')
 224
 225         cur_locale = locale.getdefaultlocale()[1]
 226         proc = subprocess.Popen(
 227             ('echo', 'info cpus'), stdout=subprocess.PIPE)
 228         output = subprocess.check_output(
 229             ('sudo', 'socat', '-', 'UNIX-CONNECT:%s' % self._monitor),
 230             stdin=proc.stdout)
 231         proc.wait()
 232
 233         for cpu in range(0, int(S.getValue('GUEST_SMP')[self._number])):
 234             match = None
 235             for line in output.decode(cur_locale).split('\n'):
 236                 match = re.search(thread_id % cpu, line)
 237                 if match:
 238                     self._affinitize_pid(
 239                         S.getValue('GUEST_CORE_BINDING')[self._number][cpu],
 240                         match.group(1))
 241                     break
 242
 243             if not match:
 244                 self._logger.error('Failed to affinitize guest core #%d. Could'
 245                                    ' not parse tid.', cpu)
 246
 247     def _affinitize_vhost_net(self):
 248         """
 249         Affinitize the vhost net threads for Vanilla OVS and guest nic queues.
 250
 251         :return: None
 252         """
 253         self._logger.info('Affinitizing VHOST Net threads.')
 254         args1 = ['pgrep', 'vhost-']
 255         process1 = subprocess.Popen(args1, stdout=subprocess.PIPE,
 256                                     shell=False)
 257         out = process1.communicate()[0]
 258         processes = out.decode(locale.getdefaultlocale()[1]).split('\n')
 259         if processes[-1] == '':
 260             processes.pop() # pgrep may return an extra line with no data
 261         self._logger.info('Found %s vhost net threads...', len(processes))
 262
 263         cpumap = S.getValue('VSWITCH_VHOST_CPU_MAP')
 264         mapcount = 0
 265         for proc in processes:
 266             self._affinitize_pid(cpumap[mapcount], proc)
 267             mapcount += 1
 268             if mapcount + 1 > len(cpumap):
 269                 # Not enough cpus were given in the mapping to cover all the
 270                 # threads on a 1 to 1 ratio with cpus so reset the list counter
 271                 #  to 0.
 272                 mapcount = 0
 273
 274     def _config_guest_loopback(self):
 275         """
 276         Configure VM to run VNF, e.g. port forwarding application based on the configuration
 277         """
 278         if self._guest_loopback == 'testpmd':
 279             self._login()
 280             self._configure_testpmd()
 281         elif self._guest_loopback == 'l2fwd':
 282             self._login()
 283             self._configure_l2fwd()
 284         elif self._guest_loopback == 'linux_bridge':
 285             self._login()
 286             self._configure_linux_bridge()
 287         elif self._guest_loopback != 'buildin':
 288             self._logger.error('Unsupported guest loopback method "%s" was specified. Option'
 289                                ' "buildin" will be used as a fallback.', self._guest_loopback)
 290
 291     def wait(self, prompt=None, timeout=30):
 292         if prompt is None:
 293             prompt = S.getValue('GUEST_PROMPT')[self._number]
 294         super(IVnfQemu, self).wait(prompt=prompt, timeout=timeout)
 295
 296     def execute_and_wait(self, cmd, timeout=30, prompt=None):
 297         if prompt is None:
 298             prompt = S.getValue('GUEST_PROMPT')[self._number]
 299         super(IVnfQemu, self).execute_and_wait(cmd, timeout=timeout,
 300                                                prompt=prompt)
 301
 302     def _modify_dpdk_makefile(self):
 303         """
 304         Modifies DPDK makefile in Guest before compilation if needed
 305         """
 306         pass
 307
 308     def _configure_copy_sources(self, dirname):
 309         """
 310         Mount shared directory and copy DPDK and l2fwd sources
 311         """
 312         # mount shared directory
 313         self.execute_and_wait('umount /dev/sdb1')
 314         self.execute_and_wait('rm -rf ' + S.getValue('GUEST_OVS_DPDK_DIR')[self._number])
 315         self.execute_and_wait('mkdir -p ' + S.getValue('GUEST_OVS_DPDK_SHARE')[self._number])
 316         self.execute_and_wait('mount -o ro,iocharset=utf8 /dev/sdb1 ' +
 317                               S.getValue('GUEST_OVS_DPDK_SHARE')[self._number])
 318         self.execute_and_wait('mkdir -p ' + S.getValue('GUEST_OVS_DPDK_DIR')[self._number])
 319         self.execute_and_wait('cp -r ' + os.path.join(S.getValue('GUEST_OVS_DPDK_SHARE')[self._number], dirname) +
 320                               ' ' + S.getValue('GUEST_OVS_DPDK_DIR')[self._number])
 321         self.execute_and_wait('umount /dev/sdb1')
 322
 323     def _configure_disable_firewall(self):
 324         """
 325         Disable firewall in VM
 326         """
 327         for iptables in ['iptables', 'ip6tables']:
 328             # filter table
 329             for chain in ['INPUT', 'FORWARD', 'OUTPUT']:
 330                 self.execute_and_wait("{} -t filter -P {} ACCEPT".format(iptables, chain))
 331             # mangle table
 332             for chain in ['PREROUTING', 'INPUT', 'FORWARD', 'OUTPUT', 'POSTROUTING']:
 333                 self.execute_and_wait("{} -t mangle -P {} ACCEPT".format(iptables, chain))
 334             # nat table
 335             for chain in ['PREROUTING', 'INPUT', 'OUTPUT', 'POSTROUTING']:
 336                 self.execute_and_wait("{} -t nat -P {} ACCEPT".format(iptables, chain))
 337
 338             # flush rules and delete chains created by user
 339             for table in ['filter', 'mangle', 'nat']:
 340                 self.execute_and_wait("{} -t {} -F".format(iptables, table))
 341                 self.execute_and_wait("{} -t {} -X".format(iptables, table))
 342
 343
 344     def _configure_testpmd(self):
 345         """
 346         Configure VM to perform L2 forwarding between NICs by DPDK's testpmd
 347         """
 348         self._configure_copy_sources('DPDK')
 349         self._configure_disable_firewall()
 350
 351         # Guest images _should_ have 1024 hugepages by default,
 352         # but just in case:'''
 353         self.execute_and_wait('sysctl vm.nr_hugepages={}'.format(S.getValue('GUEST_HUGEPAGES_NR')[self._number]))
 354
 355         # Mount hugepages
 356         self.execute_and_wait('mkdir -p /dev/hugepages')
 357         self.execute_and_wait(
 358             'mount -t hugetlbfs hugetlbfs /dev/hugepages')
 359
 360         # build and configure system for dpdk
 361         self.execute_and_wait('cd ' + S.getValue('GUEST_OVS_DPDK_DIR')[self._number] +
 362                               '/DPDK')
 363         self.execute_and_wait('export CC=gcc')
 364         self.execute_and_wait('export RTE_SDK=' +
 365                               S.getValue('GUEST_OVS_DPDK_DIR')[self._number] + '/DPDK')
 366         self.execute_and_wait('export RTE_TARGET=%s' % S.getValue('RTE_TARGET'))
 367
 368         # modify makefile if needed
 369         self._modify_dpdk_makefile()
 370
 371         # disable network interfaces, so DPDK can take care of them
 372         for nic in self._nics:
 373             self.execute_and_wait('ifdown ' + nic['device'])
 374
 375         # build and insert igb_uio and rebind interfaces to it
 376         self.execute_and_wait('make RTE_OUTPUT=$RTE_SDK/$RTE_TARGET -C '
 377                               '$RTE_SDK/lib/librte_eal/linuxapp/igb_uio')
 378         self.execute_and_wait('modprobe uio')
 379         self.execute_and_wait('insmod %s/kmod/igb_uio.ko' %
 380                               S.getValue('RTE_TARGET'))
 381         self.execute_and_wait('./tools/dpdk*bind.py --status')
 382         pci_list = ' '.join([nic['pci'] for nic in self._nics])
 383         self.execute_and_wait('./tools/dpdk*bind.py -u ' + pci_list)
 384         self.execute_and_wait('./tools/dpdk*bind.py -b igb_uio ' + pci_list)
 385         self.execute_and_wait('./tools/dpdk*bind.py --status')
 386
 387         # build and run 'test-pmd'
 388         self.execute_and_wait('cd ' + S.getValue('GUEST_OVS_DPDK_DIR')[self._number] +
 389                               '/DPDK/app/test-pmd')
 390         self.execute_and_wait('make clean')
 391         self.execute_and_wait('make')
 392         if int(S.getValue('GUEST_NIC_QUEUES')[self._number]):
 393             self.execute_and_wait(
 394                 './testpmd {} -n4 --socket-mem 512 --'.format(
 395                     S.getValue('GUEST_TESTPMD_CPU_MASK')[self._number]) +
 396                 ' --burst=64 -i --txqflags=0xf00 ' +
 397                 '--nb-cores={} --rxq={} --txq={} '.format(
 398                     S.getValue('GUEST_TESTPMD_NB_CORES')[self._number],
 399                     S.getValue('GUEST_TESTPMD_TXQ')[self._number],
 400                     S.getValue('GUEST_TESTPMD_RXQ')[self._number]) +
 401                 '--disable-hw-vlan', 60, "Done")
 402         else:
 403             self.execute_and_wait(
 404                 './testpmd {} -n 4 --socket-mem 512 --'.format(
 405                     S.getValue('GUEST_TESTPMD_CPU_MASK')[self._number]) +
 406                 ' --burst=64 -i --txqflags=0xf00 ' +
 407                 '--disable-hw-vlan', 60, "Done")
 408         self.execute('set fwd ' + self._testpmd_fwd_mode, 1)
 409         self.execute_and_wait('start', 20,
 410                               'TX RS bit threshold=.+ - TXQ flags=0xf00')
 411
 412     def _configure_l2fwd(self):
 413         """
 414         Configure VM to perform L2 forwarding between NICs by l2fwd module
 415         """
 416         if int(S.getValue('GUEST_NIC_QUEUES')[self._number]):
 417             self._set_multi_queue_nic()
 418         self._configure_copy_sources('l2fwd')
 419         self._configure_disable_firewall()
 420
 421         # configure all interfaces
 422         for nic in self._nics:
 423             self.execute('ip addr add ' +
 424                          nic['ip'] + ' dev ' + nic['device'])
 425             self.execute('ip link set dev ' + nic['device'] + ' up')
 426
 427         # build and configure system for l2fwd
 428         self.execute_and_wait('cd ' + S.getValue('GUEST_OVS_DPDK_DIR')[self._number] +
 429                               '/l2fwd')
 430         self.execute_and_wait('export CC=gcc')
 431
 432         self.execute_and_wait('make')
 433         if len(self._nics) == 2:
 434             self.execute_and_wait('insmod ' + S.getValue('GUEST_OVS_DPDK_DIR')[self._number] +
 435                                   '/l2fwd' + '/l2fwd.ko net1=' + self._nics[0]['device'] +
 436                                   ' net2=' + self._nics[1]['device'])
 437         else:
 438             raise RuntimeError('l2fwd can forward only between 2 NICs, but {} NICs are '
 439                                'configured inside GUEST'.format(len(self._nics)))
 440
 441     def _configure_linux_bridge(self):
 442         """
 443         Configure VM to perform L2 forwarding between NICs by linux bridge
 444         """
 445         if int(S.getValue('GUEST_NIC_QUEUES')[self._number]):
 446             self._set_multi_queue_nic()
 447         self._configure_disable_firewall()
 448
 449         # configure linux bridge
 450         self.execute('brctl addbr br0')
 451
 452         # add all NICs into the bridge
 453         for nic in self._nics:
 454             self.execute('ip addr add ' +
 455                          nic['ip'] + ' dev ' + nic['device'])
 456             self.execute('ip link set dev ' + nic['device'] + ' up')
 457             self.execute('brctl addif br0 ' + nic['device'])
 458
 459         self.execute('ip addr add ' +
 460                      S.getValue('GUEST_BRIDGE_IP')[self._number] +
 461                      ' dev br0')
 462         self.execute('ip link set dev br0 up')
 463
 464         # Add the arp entries for the IXIA ports and the bridge you are using.
 465         # Use command line values if provided.
 466         trafficgen_mac = get_test_param('vanilla_tgen_port1_mac',
 467                                         S.getValue('VANILLA_TGEN_PORT1_MAC'))
 468         trafficgen_ip = get_test_param('vanilla_tgen_port1_ip',
 469                                        S.getValue('VANILLA_TGEN_PORT1_IP'))
 470
 471         self.execute('arp -s ' + trafficgen_ip + ' ' + trafficgen_mac)
 472
 473         trafficgen_mac = get_test_param('vanilla_tgen_port2_mac',
 474                                         S.getValue('VANILLA_TGEN_PORT2_MAC'))
 475         trafficgen_ip = get_test_param('vanilla_tgen_port2_ip',
 476                                        S.getValue('VANILLA_TGEN_PORT2_IP'))
 477
 478         self.execute('arp -s ' + trafficgen_ip + ' ' + trafficgen_mac)
 479
 480         # Enable forwarding
 481         self.execute('sysctl -w net.ipv4.ip_forward=1')
 482
 483         # Controls source route verification
 484         # 0 means no source validation
 485         self.execute('sysctl -w net.ipv4.conf.all.rp_filter=0')
 486         for nic in self._nics:
 487             self.execute('sysctl -w net.ipv4.conf.' + nic['device'] + '.rp_filter=0')
 488
 489     def _set_multi_queue_nic(self):
 490         """
 491         Enable multi-queue in guest kernel with ethool.
 492         :return: None
 493         """
 494         for nic in self._nics:
 495             self.execute_and_wait('ethtool -L {} combined {}'.format(
 496                 nic['device'], S.getValue('GUEST_NIC_QUEUES')[self._number]))
 497             self.execute_and_wait('ethtool -l {}'.format(nic['device']))