vnfs/qemu/qemu.py

   1 # Copyright 2015-2016 Intel Corporation.
   2 #
   3 # Licensed under the Apache License, Version 2.0 (the "License");
   4 # you may not use this file except in compliance with the License.
   5 # You may obtain a copy of the License at
   6 #
   7 #   http://www.apache.org/licenses/LICENSE-2.0
   8 #
   9 # Unless required by applicable law or agreed to in writing, software
  10 # distributed under the License is distributed on an "AS IS" BASIS,
  11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12 # See the License for the specific language governing permissions and
  13 # limitations under the License.
  14
  15 """Automation of QEMU hypervisor for launching guests.
  16 """
  17
  18 import os
  19 import logging
  20 import locale
  21 import re
  22 import subprocess
  23 import time
  24 import pexpect
  25
  26 from conf import settings as S
  27 from conf import get_test_param
  28 from vnfs.vnf.vnf import IVnf
  29
  30 class IVnfQemu(IVnf):
  31     """
  32     Abstract class for controling an instance of QEMU
  33     """
  34     _cmd = None
  35     _expect = None
  36     _proc_name = 'qemu'
  37
  38     class GuestCommandFilter(logging.Filter):
  39         """
  40         Filter out strings beginning with 'guestcmd :'.
  41         """
  42         def filter(self, record):
  43             return record.getMessage().startswith(self.prefix)
  44
  45     def __init__(self):
  46         """
  47         Initialisation function.
  48         """
  49         super(IVnfQemu, self).__init__()
  50
  51         self._expect = S.getValue('GUEST_PROMPT_LOGIN')[self._number]
  52         self._logger = logging.getLogger(__name__)
  53         self._logfile = os.path.join(
  54             S.getValue('LOG_DIR'),
  55             S.getValue('LOG_FILE_QEMU')) + str(self._number)
  56         self._timeout = S.getValue('GUEST_TIMEOUT')[self._number]
  57         self._monitor = '%s/vm%dmonitor' % ('/tmp', self._number)
  58         # read GUEST NICs configuration and use only defined NR of NICS
  59         nics_nr = S.getValue('GUEST_NICS_NR')[self._number]
  60         # and inform user about missconfiguration
  61         if nics_nr < 1:
  62             raise RuntimeError('At least one VM NIC is mandotory, but {} '
  63                                'NICs are configured'.format(nics_nr))
  64         elif nics_nr > 1 and nics_nr % 2:
  65             nics_nr = int(nics_nr / 2) * 2
  66             self._logger.warning('Odd number of NICs is configured, only '
  67                                  '%s NICs will be used', nics_nr)
  68
  69         self._nics = S.getValue('GUEST_NICS')[self._number][:nics_nr]
  70
  71         # set guest loopback application based on VNF configuration
  72         # cli option take precedence to config file values
  73         self._guest_loopback = S.getValue('GUEST_LOOPBACK')[self._number]
  74
  75         self._testpmd_fwd_mode = S.getValue('GUEST_TESTPMD_FWD_MODE')[self._number]
  76         # in case of SRIOV we must ensure, that MAC addresses are not swapped
  77         if S.getValue('SRIOV_ENABLED') and self._testpmd_fwd_mode.startswith('mac') and \
  78            not S.getValue('VNF').endswith('PciPassthrough'):
  79
  80             self._logger.info("SRIOV detected, forwarding mode of testpmd was changed from '%s' to '%s'",
  81                               self._testpmd_fwd_mode, 'io')
  82             self._testpmd_fwd_mode = 'io'
  83
  84         guest_smp = int(get_test_param('guest_smp', 0))
  85         if guest_smp:
  86             override_list = [guest_smp] * (self._number + 1)
  87             S.setValue('GUEST_SMP', override_list)
  88
  89         name = 'Client%d' % self._number
  90         vnc = ':%d' % self._number
  91         # don't use taskset to affinize main qemu process; It causes hangup
  92         # of 2nd VM in case of DPDK. It also slows down VM responsivnes.
  93         self._cmd = ['sudo', '-E', S.getValue('TOOLS')['qemu-system'],
  94                      '-m', S.getValue('GUEST_MEMORY')[self._number],
  95                      '-smp', str(S.getValue('GUEST_SMP')[self._number]),
  96                      '-cpu', 'host,migratable=off',
  97                      '-drive', 'if={},file='.format(S.getValue(
  98                          'GUEST_BOOT_DRIVE_TYPE')[self._number]) +
  99                      S.getValue('GUEST_IMAGE')[self._number],
 100                      '-boot', 'c', '--enable-kvm',
 101                      '-monitor', 'unix:%s,server,nowait' % self._monitor,
 102                      '-object',
 103                      'memory-backend-file,id=mem,size=' +
 104                      str(S.getValue('GUEST_MEMORY')[self._number]) + 'M,' +
 105                      'mem-path=' + S.getValue('HUGEPAGE_DIR') + ',share=on',
 106                      '-numa', 'node,memdev=mem -mem-prealloc',
 107                      '-nographic', '-vnc', str(vnc), '-name', name,
 108                      '-snapshot', '-net none', '-no-reboot',
 109                      '-drive',
 110                      'if=%s,format=raw,file=fat:rw:%s,snapshot=off' %
 111                      (S.getValue('GUEST_SHARED_DRIVE_TYPE')[self._number],
 112                       S.getValue('GUEST_SHARE_DIR')[self._number]),
 113                     ]
 114         self._configure_logging()
 115
 116     def _configure_logging(self):
 117         """
 118         Configure logging.
 119         """
 120         self.GuestCommandFilter.prefix = self._log_prefix
 121
 122         logger = logging.getLogger()
 123         cmd_logger = logging.FileHandler(
 124             filename=os.path.join(S.getValue('LOG_DIR'),
 125                                   S.getValue('LOG_FILE_GUEST_CMDS')) +
 126             str(self._number))
 127         cmd_logger.setLevel(logging.DEBUG)
 128         cmd_logger.addFilter(self.GuestCommandFilter())
 129         logger.addHandler(cmd_logger)
 130
 131     # startup/Shutdown
 132
 133     def start(self):
 134         """
 135         Start QEMU instance, login and prepare for commands.
 136         """
 137         super(IVnfQemu, self).start()
 138         if S.getValue('VNF_AFFINITIZATION_ON'):
 139             self._affinitize()
 140
 141         if S.getValue('VSWITCH_VHOST_NET_AFFINITIZATION') and S.getValue(
 142                 'VNF') == 'QemuVirtioNet':
 143             self._affinitize_vhost_net()
 144
 145         if self._timeout:
 146             self._config_guest_loopback()
 147
 148     def stop(self):
 149         """
 150         Stops VNF instance gracefully first.
 151         """
 152         try:
 153             # exit testpmd if needed
 154             if self._guest_loopback == 'testpmd':
 155                 self.execute_and_wait('stop', 120, "Done")
 156                 self.execute_and_wait('quit', 120, "[bB]ye")
 157
 158             # turn off VM
 159             self.execute_and_wait('poweroff', 120, "Power down")
 160
 161         except pexpect.TIMEOUT:
 162             self.kill()
 163
 164         # wait until qemu shutdowns
 165         self._logger.debug('Wait for QEMU to terminate')
 166         for dummy in range(30):
 167             time.sleep(1)
 168             if not self.is_running():
 169                 break
 170
 171         # just for case that graceful shutdown failed
 172         super(IVnfQemu, self).stop()
 173
 174     # helper functions
 175
 176     def _login(self, timeout=120):
 177         """
 178         Login to QEMU instance.
 179
 180         This can be used immediately after booting the machine, provided a
 181         sufficiently long ``timeout`` is given.
 182
 183         :param timeout: Timeout to wait for login to complete.
 184
 185         :returns: None
 186         """
 187         # if no timeout was set, we likely started QEMU without waiting for it
 188         # to boot. This being the case, we best check that it has finished
 189         # first.
 190         if not self._timeout:
 191             self._expect_process(timeout=timeout)
 192
 193         self._child.sendline(S.getValue('GUEST_USERNAME')[self._number])
 194         self._child.expect(S.getValue('GUEST_PROMPT_PASSWORD')[self._number], timeout=5)
 195         self._child.sendline(S.getValue('GUEST_PASSWORD')[self._number])
 196
 197         self._expect_process(S.getValue('GUEST_PROMPT')[self._number], timeout=5)
 198
 199     def send_and_pass(self, cmd, timeout=30):
 200         """
 201         Send ``cmd`` and wait ``timeout`` seconds for it to pass.
 202
 203         :param cmd: Command to send to guest.
 204         :param timeout: Time to wait for prompt before checking return code.
 205
 206         :returns: None
 207         """
 208         self.execute(cmd)
 209         self.wait(S.getValue('GUEST_PROMPT')[self._number], timeout=timeout)
 210         self.execute('echo $?')
 211         self._child.expect('^0$', timeout=1)  # expect a 0
 212         self.wait(S.getValue('GUEST_PROMPT')[self._number], timeout=timeout)
 213
 214     def _affinitize(self):
 215         """
 216         Affinitize the SMP cores of a QEMU instance.
 217
 218         This is a bit of a hack. The 'socat' utility is used to
 219         interact with the QEMU HMP. This is necessary due to the lack
 220         of QMP in older versions of QEMU, like v1.6.2. In future
 221         releases, this should be replaced with calls to libvirt or
 222         another Python-QEMU wrapper library.
 223
 224         :returns: None
 225         """
 226         thread_id = (r'.* CPU #%d: .* thread_id=(\d+)')
 227
 228         self._logger.info('Affinitizing guest...')
 229
 230         cur_locale = locale.getdefaultlocale()[1]
 231         proc = subprocess.Popen(
 232             ('echo', 'info cpus'), stdout=subprocess.PIPE)
 233         output = subprocess.check_output(
 234             ('sudo', 'socat', '-', 'UNIX-CONNECT:%s' % self._monitor),
 235             stdin=proc.stdout)
 236         proc.wait()
 237
 238         guest_core_binding = int(get_test_param('guest_core_binding', 0))
 239         for cpu in range(0, int(S.getValue('GUEST_SMP')[self._number])):
 240             match = None
 241             for line in output.decode(cur_locale).split('\n'):
 242                 match = re.search(thread_id % cpu, line)
 243                 if match:
 244                     if guest_core_binding:
 245                         self._affinitize_pid(guest_core_binding, match.group(1))
 246                     else:
 247                         self._affinitize_pid(
 248                             S.getValue('GUEST_CORE_BINDING')[self._number][cpu],
 249                             match.group(1))
 250                     break
 251
 252             if not match:
 253                 self._logger.error('Failed to affinitize guest core #%d. Could'
 254                                    ' not parse tid.', cpu)
 255
 256     def _affinitize_vhost_net(self):
 257         """
 258         Affinitize the vhost net threads for Vanilla OVS and guest nic queues.
 259
 260         :return: None
 261         """
 262         self._logger.info('Affinitizing VHOST Net threads.')
 263         args1 = ['pgrep', 'vhost-']
 264         process1 = subprocess.Popen(args1, stdout=subprocess.PIPE,
 265                                     shell=False)
 266         out = process1.communicate()[0]
 267         processes = out.decode(locale.getdefaultlocale()[1]).split('\n')
 268         if processes[-1] == '':
 269             processes.pop() # pgrep may return an extra line with no data
 270         self._logger.info('Found %s vhost net threads...', len(processes))
 271
 272         cpumap = S.getValue('VSWITCH_VHOST_CPU_MAP')
 273         mapcount = 0
 274         for proc in processes:
 275             self._affinitize_pid(cpumap[mapcount], proc)
 276             mapcount += 1
 277             if mapcount + 1 > len(cpumap):
 278                 # Not enough cpus were given in the mapping to cover all the
 279                 # threads on a 1 to 1 ratio with cpus so reset the list counter
 280                 #  to 0.
 281                 mapcount = 0
 282
 283     def _config_guest_loopback(self):
 284         """
 285         Configure VM to run VNF, e.g. port forwarding application based on the configuration
 286         """
 287         if self._guest_loopback == 'testpmd':
 288             self._login()
 289             self._configure_testpmd()
 290         elif self._guest_loopback == 'l2fwd':
 291             self._login()
 292             self._configure_l2fwd()
 293         elif self._guest_loopback == 'linux_bridge':
 294             self._login()
 295             self._configure_linux_bridge()
 296         elif self._guest_loopback != 'buildin':
 297             self._logger.error('Unsupported guest loopback method "%s" was specified. Option'
 298                                ' "buildin" will be used as a fallback.', self._guest_loopback)
 299
 300     def wait(self, prompt=None, timeout=30):
 301         if prompt is None:
 302             prompt = S.getValue('GUEST_PROMPT')[self._number]
 303         super(IVnfQemu, self).wait(prompt=prompt, timeout=timeout)
 304
 305     def execute_and_wait(self, cmd, timeout=30, prompt=None):
 306         if prompt is None:
 307             prompt = S.getValue('GUEST_PROMPT')[self._number]
 308         super(IVnfQemu, self).execute_and_wait(cmd, timeout=timeout,
 309                                                prompt=prompt)
 310
 311     def _modify_dpdk_makefile(self):
 312         """
 313         Modifies DPDK makefile in Guest before compilation if needed
 314         """
 315         pass
 316
 317     def _configure_copy_sources(self, dirname):
 318         """
 319         Mount shared directory and copy DPDK and l2fwd sources
 320         """
 321         # mount shared directory
 322         self.execute_and_wait('umount /dev/sdb1')
 323         self.execute_and_wait('rm -rf ' + S.getValue('GUEST_OVS_DPDK_DIR')[self._number])
 324         self.execute_and_wait('mkdir -p ' + S.getValue('GUEST_OVS_DPDK_SHARE')[self._number])
 325         self.execute_and_wait('mount -o ro,iocharset=utf8 /dev/sdb1 ' +
 326                               S.getValue('GUEST_OVS_DPDK_SHARE')[self._number])
 327         self.execute_and_wait('mkdir -p ' + S.getValue('GUEST_OVS_DPDK_DIR')[self._number])
 328         self.execute_and_wait('cp -r ' + os.path.join(S.getValue('GUEST_OVS_DPDK_SHARE')[self._number], dirname) +
 329                               ' ' + S.getValue('GUEST_OVS_DPDK_DIR')[self._number])
 330         self.execute_and_wait('umount /dev/sdb1')
 331
 332     def _configure_disable_firewall(self):
 333         """
 334         Disable firewall in VM
 335         """
 336         for iptables in ['iptables', 'ip6tables']:
 337             # filter table
 338             for chain in ['INPUT', 'FORWARD', 'OUTPUT']:
 339                 self.execute_and_wait("{} -t filter -P {} ACCEPT".format(iptables, chain))
 340             # mangle table
 341             for chain in ['PREROUTING', 'INPUT', 'FORWARD', 'OUTPUT', 'POSTROUTING']:
 342                 self.execute_and_wait("{} -t mangle -P {} ACCEPT".format(iptables, chain))
 343             # nat table
 344             for chain in ['PREROUTING', 'INPUT', 'OUTPUT', 'POSTROUTING']:
 345                 self.execute_and_wait("{} -t nat -P {} ACCEPT".format(iptables, chain))
 346
 347             # flush rules and delete chains created by user
 348             for table in ['filter', 'mangle', 'nat']:
 349                 self.execute_and_wait("{} -t {} -F".format(iptables, table))
 350                 self.execute_and_wait("{} -t {} -X".format(iptables, table))
 351
 352
 353     def _configure_testpmd(self):
 354         """
 355         Configure VM to perform L2 forwarding between NICs by DPDK's testpmd
 356         """
 357         self._configure_copy_sources('DPDK')
 358         self._configure_disable_firewall()
 359
 360         # Guest images _should_ have 1024 hugepages by default,
 361         # but just in case:'''
 362         self.execute_and_wait('sysctl vm.nr_hugepages={}'.format(S.getValue('GUEST_HUGEPAGES_NR')[self._number]))
 363
 364         # Mount hugepages
 365         self.execute_and_wait('mkdir -p /dev/hugepages')
 366         self.execute_and_wait(
 367             'mount -t hugetlbfs hugetlbfs /dev/hugepages')
 368
 369         # build and configure system for dpdk
 370         self.execute_and_wait('cd ' + S.getValue('GUEST_OVS_DPDK_DIR')[self._number] +
 371                               '/DPDK')
 372         self.execute_and_wait('export CC=gcc')
 373         self.execute_and_wait('export RTE_SDK=' +
 374                               S.getValue('GUEST_OVS_DPDK_DIR')[self._number] + '/DPDK')
 375         self.execute_and_wait('export RTE_TARGET=%s' % S.getValue('RTE_TARGET'))
 376
 377         # modify makefile if needed
 378         self._modify_dpdk_makefile()
 379
 380         # disable network interfaces, so DPDK can take care of them
 381         for nic in self._nics:
 382             self.execute_and_wait('ifdown ' + nic['device'])
 383
 384         # build and insert igb_uio and rebind interfaces to it
 385         self.execute_and_wait('make RTE_OUTPUT=$RTE_SDK/$RTE_TARGET -C '
 386                               '$RTE_SDK/lib/librte_eal/linuxapp/igb_uio')
 387         self.execute_and_wait('modprobe uio')
 388         self.execute_and_wait('insmod %s/kmod/igb_uio.ko' %
 389                               S.getValue('RTE_TARGET'))
 390         self.execute_and_wait('./tools/dpdk*bind.py --status')
 391         pci_list = ' '.join([nic['pci'] for nic in self._nics])
 392         self.execute_and_wait('./tools/dpdk*bind.py -u ' + pci_list)
 393         self.execute_and_wait('./tools/dpdk*bind.py -b igb_uio ' + pci_list)
 394         self.execute_and_wait('./tools/dpdk*bind.py --status')
 395
 396         # build and run 'test-pmd'
 397         self.execute_and_wait('cd ' + S.getValue('GUEST_OVS_DPDK_DIR')[self._number] +
 398                               '/DPDK/app/test-pmd')
 399         self.execute_and_wait('make clean')
 400         self.execute_and_wait('make')
 401
 402         # get multi-queue settings from CLI
 403         guest_testpmd_txq = int(get_test_param('guest_testpmd_txq', 0))
 404         if guest_testpmd_txq:
 405             override_list = [guest_testpmd_txq] * (self._number + 1)
 406             S.setValue('GUEST_TESTPMD_TXQ', override_list)
 407
 408         guest_testpmd_rxq = int(get_test_param('guest_testpmd_rxq', 0))
 409         if guest_testpmd_rxq:
 410             override_list = [guest_testpmd_rxq] * (self._number + 1)
 411             S.setValue('GUEST_TESTPMD_RXQ', override_list)
 412
 413         guest_testpmd_nb_cores = \
 414             int(get_test_param('guest_testpmd_nb_cores', 0))
 415         if guest_testpmd_nb_cores:
 416             override_list = [guest_testpmd_nb_cores] * (self._number + 1)
 417             S.setValue('GUEST_TESTPMD_NB_CORES', override_list)
 418
 419         guest_testpmd_cpu_mask = \
 420             int(get_test_param('guest_testpmd_cpu_mask', 0))
 421         if guest_testpmd_cpu_mask:
 422             override_list = [guest_testpmd_cpu_mask] * (self._number + 1)
 423             S.setValue('GUEST_TESTPMD_CPU_MASK', override_list)
 424
 425         if int(S.getValue('GUEST_NIC_QUEUES')[self._number]):
 426             self.execute_and_wait(
 427                 './testpmd {} -n4 --socket-mem 512 --'.format(
 428                     S.getValue('GUEST_TESTPMD_CPU_MASK')[self._number]) +
 429                 ' --burst=64 -i --txqflags=0xf00 ' +
 430                 '--nb-cores={} --rxq={} --txq={} '.format(
 431                     S.getValue('GUEST_TESTPMD_NB_CORES')[self._number],
 432                     S.getValue('GUEST_TESTPMD_TXQ')[self._number],
 433                     S.getValue('GUEST_TESTPMD_RXQ')[self._number]) +
 434                 '--disable-hw-vlan', 60, "Done")
 435         else:
 436             self.execute_and_wait(
 437                 './testpmd {} -n 4 --socket-mem 512 --'.format(
 438                     S.getValue('GUEST_TESTPMD_CPU_MASK')[self._number]) +
 439                 ' --burst=64 -i --txqflags=0xf00 ' +
 440                 '--disable-hw-vlan', 60, "Done")
 441         self.execute('set fwd ' + self._testpmd_fwd_mode, 1)
 442         self.execute_and_wait('start', 20,
 443                               'TX RS bit threshold=.+ - TXQ flags=0xf00')
 444
 445     def _configure_l2fwd(self):
 446         """
 447         Configure VM to perform L2 forwarding between NICs by l2fwd module
 448         """
 449         if int(S.getValue('GUEST_NIC_QUEUES')[self._number]):
 450             self._set_multi_queue_nic()
 451         self._configure_copy_sources('l2fwd')
 452         self._configure_disable_firewall()
 453
 454         # configure all interfaces
 455         for nic in self._nics:
 456             self.execute('ip addr add ' +
 457                          nic['ip'] + ' dev ' + nic['device'])
 458             self.execute('ip link set dev ' + nic['device'] + ' up')
 459
 460         # build and configure system for l2fwd
 461         self.execute_and_wait('cd ' + S.getValue('GUEST_OVS_DPDK_DIR')[self._number] +
 462                               '/l2fwd')
 463         self.execute_and_wait('export CC=gcc')
 464
 465         self.execute_and_wait('make')
 466         if len(self._nics) == 2:
 467             self.execute_and_wait('insmod ' + S.getValue('GUEST_OVS_DPDK_DIR')[self._number] +
 468                                   '/l2fwd' + '/l2fwd.ko net1=' + self._nics[0]['device'] +
 469                                   ' net2=' + self._nics[1]['device'])
 470         else:
 471             raise RuntimeError('l2fwd can forward only between 2 NICs, but {} NICs are '
 472                                'configured inside GUEST'.format(len(self._nics)))
 473
 474     def _configure_linux_bridge(self):
 475         """
 476         Configure VM to perform L2 forwarding between NICs by linux bridge
 477         """
 478         if int(S.getValue('GUEST_NIC_QUEUES')[self._number]):
 479             self._set_multi_queue_nic()
 480         self._configure_disable_firewall()
 481
 482         # configure linux bridge
 483         self.execute('brctl addbr br0')
 484
 485         # add all NICs into the bridge
 486         for nic in self._nics:
 487             self.execute('ip addr add ' +
 488                          nic['ip'] + ' dev ' + nic['device'])
 489             self.execute('ip link set dev ' + nic['device'] + ' up')
 490             self.execute('brctl addif br0 ' + nic['device'])
 491
 492         self.execute('ip addr add ' +
 493                      S.getValue('GUEST_BRIDGE_IP')[self._number] +
 494                      ' dev br0')
 495         self.execute('ip link set dev br0 up')
 496
 497         # Add the arp entries for the IXIA ports and the bridge you are using.
 498         # Use command line values if provided.
 499         trafficgen_mac = get_test_param('vanilla_tgen_port1_mac',
 500                                         S.getValue('VANILLA_TGEN_PORT1_MAC'))
 501         trafficgen_ip = get_test_param('vanilla_tgen_port1_ip',
 502                                        S.getValue('VANILLA_TGEN_PORT1_IP'))
 503
 504         self.execute('arp -s ' + trafficgen_ip + ' ' + trafficgen_mac)
 505
 506         trafficgen_mac = get_test_param('vanilla_tgen_port2_mac',
 507                                         S.getValue('VANILLA_TGEN_PORT2_MAC'))
 508         trafficgen_ip = get_test_param('vanilla_tgen_port2_ip',
 509                                        S.getValue('VANILLA_TGEN_PORT2_IP'))
 510
 511         self.execute('arp -s ' + trafficgen_ip + ' ' + trafficgen_mac)
 512
 513         # Enable forwarding
 514         self.execute('sysctl -w net.ipv4.ip_forward=1')
 515
 516         # Controls source route verification
 517         # 0 means no source validation
 518         self.execute('sysctl -w net.ipv4.conf.all.rp_filter=0')
 519         for nic in self._nics:
 520             self.execute('sysctl -w net.ipv4.conf.' + nic['device'] + '.rp_filter=0')
 521
 522     def _set_multi_queue_nic(self):
 523         """
 524         Enable multi-queue in guest kernel with ethool.
 525         :return: None
 526         """
 527         for nic in self._nics:
 528             self.execute_and_wait('ethtool -L {} combined {}'.format(
 529                 nic['device'], S.getValue('GUEST_NIC_QUEUES')[self._number]))
 530             self.execute_and_wait('ethtool -l {}'.format(nic['device']))