vnfs/qemu/qemu.py

   1 # Copyright 2015-2016 Intel Corporation.
   2 #
   3 # Licensed under the Apache License, Version 2.0 (the "License");
   4 # you may not use this file except in compliance with the License.
   5 # You may obtain a copy of the License at
   6 #
   7 #   http://www.apache.org/licenses/LICENSE-2.0
   8 #
   9 # Unless required by applicable law or agreed to in writing, software
  10 # distributed under the License is distributed on an "AS IS" BASIS,
  11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12 # See the License for the specific language governing permissions and
  13 # limitations under the License.
  14
  15 """Automation of QEMU hypervisor for launching guests.
  16 """
  17
  18 import os
  19 import logging
  20 import locale
  21 import re
  22 import subprocess
  23 import time
  24 import pexpect
  25
  26 from conf import settings as S
  27 from conf import get_test_param
  28 from vnfs.vnf.vnf import IVnf
  29
  30 class IVnfQemu(IVnf):
  31     """
  32     Abstract class for controling an instance of QEMU
  33     """
  34     _cmd = None
  35     _expect = S.getValue('GUEST_PROMPT_LOGIN')
  36     _proc_name = 'qemu'
  37
  38     class GuestCommandFilter(logging.Filter):
  39         """
  40         Filter out strings beginning with 'guestcmd :'.
  41         """
  42         def filter(self, record):
  43             return record.getMessage().startswith(self.prefix)
  44
  45     def __init__(self):
  46         """
  47         Initialisation function.
  48         """
  49         super(IVnfQemu, self).__init__()
  50         self._logger = logging.getLogger(__name__)
  51         self._logfile = os.path.join(
  52             S.getValue('LOG_DIR'),
  53             S.getValue('LOG_FILE_QEMU')) + str(self._number)
  54         self._timeout = S.getValue('GUEST_TIMEOUT')[self._number]
  55         self._monitor = '%s/vm%dmonitor' % ('/tmp', self._number)
  56         self._net1 = get_test_param('guest_nic1_name', None)
  57         if self._net1 == None:
  58             self._net1 = S.getValue('GUEST_NIC1_NAME')[self._number]
  59         else:
  60             self._net1 = self._net1.split(',')[self._number]
  61         self._net2 = get_test_param('guest_nic2_name', None)
  62         if self._net2 == None:
  63             self._net2 = S.getValue('GUEST_NIC2_NAME')[self._number]
  64         else:
  65             self._net2 = self._net2.split(',')[self._number]
  66
  67         # set guest loopback application based on VNF configuration
  68         # cli option take precedence to config file values
  69         self._guest_loopback = S.getValue('GUEST_LOOPBACK')[self._number]
  70
  71         self._testpmd_fwd_mode = S.getValue('GUEST_TESTPMD_FWD_MODE')
  72         # in case of SRIOV we must ensure, that MAC addresses are not swapped
  73         if S.getValue('SRIOV_ENABLED') and self._testpmd_fwd_mode.startswith('mac') and \
  74            not S.getValue('VNF').endswith('PciPassthrough'):
  75
  76             self._logger.info("SRIOV detected, forwarding mode of testpmd was changed from '%s' to '%s'",
  77                               self._testpmd_fwd_mode, 'io')
  78             self._testpmd_fwd_mode = 'io'
  79
  80         name = 'Client%d' % self._number
  81         vnc = ':%d' % self._number
  82         # don't use taskset to affinize main qemu process; It causes hangup
  83         # of 2nd VM in case of DPDK. It also slows down VM responsivnes.
  84         self._cmd = ['sudo', '-E', S.getValue('QEMU_BIN'),
  85                      '-m', S.getValue('GUEST_MEMORY')[self._number],
  86                      '-smp', str(S.getValue('GUEST_SMP')[self._number]),
  87                      '-cpu', 'host,migratable=off',
  88                      '-drive', 'if={},file='.format(S.getValue(
  89                         'GUEST_BOOT_DRIVE_TYPE')) +
  90                      S.getValue('GUEST_IMAGE')[self._number],
  91                      '-boot', 'c', '--enable-kvm',
  92                      '-monitor', 'unix:%s,server,nowait' % self._monitor,
  93                      '-object',
  94                      'memory-backend-file,id=mem,size=' +
  95                      str(S.getValue('GUEST_MEMORY')[self._number]) + 'M,' +
  96                      'mem-path=' + S.getValue('HUGEPAGE_DIR') + ',share=on',
  97                      '-numa', 'node,memdev=mem -mem-prealloc',
  98                      '-nographic', '-vnc', str(vnc), '-name', name,
  99                      '-snapshot', '-net none', '-no-reboot',
 100                      '-drive',
 101                      'if=%s,format=raw,file=fat:rw:%s,snapshot=off' %
 102                      (S.getValue('GUEST_SHARED_DRIVE_TYPE'),
 103                       S.getValue('GUEST_SHARE_DIR')[self._number]),
 104                     ]
 105         self._configure_logging()
 106
 107     def _configure_logging(self):
 108         """
 109         Configure logging.
 110         """
 111         self.GuestCommandFilter.prefix = self._log_prefix
 112
 113         logger = logging.getLogger()
 114         cmd_logger = logging.FileHandler(
 115             filename=os.path.join(S.getValue('LOG_DIR'),
 116                                   S.getValue('LOG_FILE_GUEST_CMDS')) +
 117             str(self._number))
 118         cmd_logger.setLevel(logging.DEBUG)
 119         cmd_logger.addFilter(self.GuestCommandFilter())
 120         logger.addHandler(cmd_logger)
 121
 122     # startup/Shutdown
 123
 124     def start(self):
 125         """
 126         Start QEMU instance, login and prepare for commands.
 127         """
 128         super(IVnfQemu, self).start()
 129         if S.getValue('VNF_AFFINITIZATION_ON'):
 130             self._affinitize()
 131
 132         if S.getValue('VSWITCH_VHOST_NET_AFFINITIZATION') and S.getValue(
 133                 'VNF') == 'QemuVirtioNet':
 134             self._affinitize_vhost_net()
 135
 136         if self._timeout:
 137             self._config_guest_loopback()
 138
 139     def stop(self):
 140         """
 141         Stops VNF instance gracefully first.
 142         """
 143         try:
 144             # exit testpmd if needed
 145             if self._guest_loopback == 'testpmd':
 146                 self.execute_and_wait('stop', 120, "Done")
 147                 self.execute_and_wait('quit', 120, "[bB]ye")
 148
 149             # turn off VM
 150             self.execute_and_wait('poweroff', 120, "Power down")
 151
 152         except pexpect.TIMEOUT:
 153             self.kill()
 154
 155         # wait until qemu shutdowns
 156         self._logger.debug('Wait for QEMU to terminate')
 157         for dummy in range(30):
 158             time.sleep(1)
 159             if not self.is_running():
 160                 break
 161
 162         # just for case that graceful shutdown failed
 163         super(IVnfQemu, self).stop()
 164
 165     # helper functions
 166
 167     def _login(self, timeout=120):
 168         """
 169         Login to QEMU instance.
 170
 171         This can be used immediately after booting the machine, provided a
 172         sufficiently long ``timeout`` is given.
 173
 174         :param timeout: Timeout to wait for login to complete.
 175
 176         :returns: None
 177         """
 178         # if no timeout was set, we likely started QEMU without waiting for it
 179         # to boot. This being the case, we best check that it has finished
 180         # first.
 181         if not self._timeout:
 182             self._expect_process(timeout=timeout)
 183
 184         self._child.sendline(S.getValue('GUEST_USERNAME'))
 185         self._child.expect(S.getValue('GUEST_PROMPT_PASSWORD'), timeout=5)
 186         self._child.sendline(S.getValue('GUEST_PASSWORD'))
 187
 188         self._expect_process(S.getValue('GUEST_PROMPT'), timeout=5)
 189
 190     def send_and_pass(self, cmd, timeout=30):
 191         """
 192         Send ``cmd`` and wait ``timeout`` seconds for it to pass.
 193
 194         :param cmd: Command to send to guest.
 195         :param timeout: Time to wait for prompt before checking return code.
 196
 197         :returns: None
 198         """
 199         self.execute(cmd)
 200         self.wait(S.getValue('GUEST_PROMPT'), timeout=timeout)
 201         self.execute('echo $?')
 202         self._child.expect('^0$', timeout=1)  # expect a 0
 203         self.wait(S.getValue('GUEST_PROMPT'), timeout=timeout)
 204
 205     def _affinitize(self):
 206         """
 207         Affinitize the SMP cores of a QEMU instance.
 208
 209         This is a bit of a hack. The 'socat' utility is used to
 210         interact with the QEMU HMP. This is necessary due to the lack
 211         of QMP in older versions of QEMU, like v1.6.2. In future
 212         releases, this should be replaced with calls to libvirt or
 213         another Python-QEMU wrapper library.
 214
 215         :returns: None
 216         """
 217         thread_id = (r'.* CPU #%d: .* thread_id=(\d+)')
 218
 219         self._logger.info('Affinitizing guest...')
 220
 221         cur_locale = locale.getdefaultlocale()[1]
 222         proc = subprocess.Popen(
 223             ('echo', 'info cpus'), stdout=subprocess.PIPE)
 224         output = subprocess.check_output(
 225             ('sudo', 'socat', '-', 'UNIX-CONNECT:%s' % self._monitor),
 226             stdin=proc.stdout)
 227         proc.wait()
 228
 229         for cpu in range(0, int(S.getValue('GUEST_SMP')[self._number])):
 230             match = None
 231             for line in output.decode(cur_locale).split('\n'):
 232                 match = re.search(thread_id % cpu, line)
 233                 if match:
 234                     self._affinitize_pid(
 235                         S.getValue('GUEST_CORE_BINDING')[self._number][cpu],
 236                         match.group(1))
 237                     break
 238
 239             if not match:
 240                 self._logger.error('Failed to affinitize guest core #%d. Could'
 241                                    ' not parse tid.', cpu)
 242
 243     def _affinitize_vhost_net(self):
 244         """
 245         Affinitize the vhost net threads for Vanilla OVS and guest nic queues.
 246
 247         :return: None
 248         """
 249         self._logger.info('Affinitizing VHOST Net threads.')
 250         args1 = ['pgrep', 'vhost-']
 251         process1 = subprocess.Popen(args1, stdout=subprocess.PIPE,
 252                                     shell=False)
 253         out = process1.communicate()[0]
 254         processes = out.decode(locale.getdefaultlocale()[1]).split('\n')
 255         if processes[-1] == '':
 256             processes.pop() # pgrep may return an extra line with no data
 257         self._logger.info('Found %s vhost net threads...', len(processes))
 258
 259         cpumap = S.getValue('VSWITCH_VHOST_CPU_MAP')
 260         mapcount = 0
 261         for proc in processes:
 262             self._affinitize_pid(cpumap[mapcount], proc)
 263             mapcount += 1
 264             if mapcount + 1 > len(cpumap):
 265                 # Not enough cpus were given in the mapping to cover all the
 266                 # threads on a 1 to 1 ratio with cpus so reset the list counter
 267                 #  to 0.
 268                 mapcount = 0
 269
 270     def _config_guest_loopback(self):
 271         """
 272         Configure VM to run VNF, e.g. port forwarding application based on the configuration
 273         """
 274         if self._guest_loopback == 'testpmd':
 275             self._login()
 276             self._configure_testpmd()
 277         elif self._guest_loopback == 'l2fwd':
 278             self._login()
 279             self._configure_l2fwd()
 280         elif self._guest_loopback == 'linux_bridge':
 281             self._login()
 282             self._configure_linux_bridge()
 283         elif self._guest_loopback != 'buildin':
 284             self._logger.error('Unsupported guest loopback method "%s" was specified. Option'
 285                                ' "buildin" will be used as a fallback.', self._guest_loopback)
 286
 287     def wait(self, prompt=S.getValue('GUEST_PROMPT'), timeout=30):
 288         super(IVnfQemu, self).wait(prompt=prompt, timeout=timeout)
 289
 290     def execute_and_wait(self, cmd, timeout=30,
 291                          prompt=S.getValue('GUEST_PROMPT')):
 292         super(IVnfQemu, self).execute_and_wait(cmd, timeout=timeout,
 293                                                prompt=prompt)
 294
 295     def _modify_dpdk_makefile(self):
 296         """
 297         Modifies DPDK makefile in Guest before compilation if needed
 298         """
 299         pass
 300
 301     def _configure_copy_sources(self, dirname):
 302         """
 303         Mount shared directory and copy DPDK and l2fwd sources
 304         """
 305         # mount shared directory
 306         self.execute_and_wait('umount /dev/sdb1')
 307         self.execute_and_wait('rm -rf ' + S.getValue('GUEST_OVS_DPDK_DIR'))
 308         self.execute_and_wait('mkdir -p ' + S.getValue('OVS_DPDK_SHARE'))
 309         self.execute_and_wait('mount -o ro,iocharset=utf8 /dev/sdb1 ' +
 310                               S.getValue('OVS_DPDK_SHARE'))
 311         self.execute_and_wait('mkdir -p ' + S.getValue('GUEST_OVS_DPDK_DIR'))
 312         self.execute_and_wait('cp -r ' + os.path.join(S.getValue('OVS_DPDK_SHARE'), dirname) +
 313                               ' ' + S.getValue('GUEST_OVS_DPDK_DIR'))
 314         self.execute_and_wait('umount /dev/sdb1')
 315
 316     def _configure_disable_firewall(self):
 317         """
 318         Disable firewall in VM
 319         """
 320         for iptables in ['iptables', 'ip6tables']:
 321             # filter table
 322             for chain in ['INPUT', 'FORWARD', 'OUTPUT']:
 323                 self.execute_and_wait("{} -t filter -P {} ACCEPT".format(iptables, chain))
 324             # mangle table
 325             for chain in ['PREROUTING', 'INPUT', 'FORWARD', 'OUTPUT', 'POSTROUTING']:
 326                 self.execute_and_wait("{} -t mangle -P {} ACCEPT".format(iptables, chain))
 327             # nat table
 328             for chain in ['PREROUTING', 'INPUT', 'OUTPUT', 'POSTROUTING']:
 329                 self.execute_and_wait("{} -t nat -P {} ACCEPT".format(iptables, chain))
 330
 331             # flush rules and delete chains created by user
 332             for table in ['filter', 'mangle', 'nat']:
 333                 self.execute_and_wait("{} -t {} -F".format(iptables, table))
 334                 self.execute_and_wait("{} -t {} -X".format(iptables, table))
 335
 336
 337     def _configure_testpmd(self):
 338         """
 339         Configure VM to perform L2 forwarding between NICs by DPDK's testpmd
 340         """
 341         self._configure_copy_sources('DPDK')
 342         self._configure_disable_firewall()
 343
 344         # Guest images _should_ have 1024 hugepages by default,
 345         # but just in case:'''
 346         self.execute_and_wait('sysctl vm.nr_hugepages=1024')
 347
 348         # Mount hugepages
 349         self.execute_and_wait('mkdir -p /dev/hugepages')
 350         self.execute_and_wait(
 351             'mount -t hugetlbfs hugetlbfs /dev/hugepages')
 352
 353         # build and configure system for dpdk
 354         self.execute_and_wait('cd ' + S.getValue('GUEST_OVS_DPDK_DIR') +
 355                               '/DPDK')
 356         self.execute_and_wait('export CC=gcc')
 357         self.execute_and_wait('export RTE_SDK=' +
 358                               S.getValue('GUEST_OVS_DPDK_DIR') + '/DPDK')
 359         self.execute_and_wait('export RTE_TARGET=%s' % S.getValue('RTE_TARGET'))
 360
 361         # modify makefile if needed
 362         self._modify_dpdk_makefile()
 363
 364         # disable network interfaces, so DPDK can take care of them
 365         self.execute_and_wait('ifdown ' + self._net1)
 366         self.execute_and_wait('ifdown ' + self._net2)
 367
 368         # build and insert igb_uio and rebind interfaces to it
 369         self.execute_and_wait('make RTE_OUTPUT=$RTE_SDK/$RTE_TARGET -C '
 370                               '$RTE_SDK/lib/librte_eal/linuxapp/igb_uio')
 371         self.execute_and_wait('modprobe uio')
 372         self.execute_and_wait('insmod %s/kmod/igb_uio.ko' %
 373                               S.getValue('RTE_TARGET'))
 374         self.execute_and_wait('./tools/dpdk*bind.py --status')
 375         self.execute_and_wait(
 376             './tools/dpdk*bind.py -u' ' ' +
 377             S.getValue('GUEST_NET1_PCI_ADDRESS')[self._number] + ' ' +
 378             S.getValue('GUEST_NET2_PCI_ADDRESS')[self._number])
 379         self.execute_and_wait(
 380             './tools/dpdk*bind.py -b igb_uio' ' ' +
 381             S.getValue('GUEST_NET1_PCI_ADDRESS')[self._number] + ' ' +
 382             S.getValue('GUEST_NET2_PCI_ADDRESS')[self._number])
 383         self.execute_and_wait('./tools/dpdk*bind.py --status')
 384
 385         # build and run 'test-pmd'
 386         self.execute_and_wait('cd ' + S.getValue('GUEST_OVS_DPDK_DIR') +
 387                               '/DPDK/app/test-pmd')
 388         self.execute_and_wait('make clean')
 389         self.execute_and_wait('make')
 390         if int(S.getValue('GUEST_NIC_QUEUES')):
 391             self.execute_and_wait(
 392                 './testpmd {} -n4 --socket-mem 512 --'.format(
 393                     S.getValue('GUEST_TESTPMD_CPU_MASK')) +
 394                 ' --burst=64 -i --txqflags=0xf00 ' +
 395                 '--nb-cores={} --rxq={} --txq={} '.format(
 396                     S.getValue('GUEST_TESTPMD_NB_CORES'),
 397                     S.getValue('GUEST_TESTPMD_TXQ'),
 398                     S.getValue('GUEST_TESTPMD_RXQ')) +
 399                 '--disable-hw-vlan', 60, "Done")
 400         else:
 401             self.execute_and_wait(
 402                 './testpmd {} -n 4 --socket-mem 512 --'.format(
 403                     S.getValue('GUEST_TESTPMD_CPU_MASK')) +
 404                 ' --burst=64 -i --txqflags=0xf00 ' +
 405                 '--disable-hw-vlan', 60, "Done")
 406         self.execute('set fwd ' + self._testpmd_fwd_mode, 1)
 407         self.execute_and_wait('start', 20,
 408                               'TX RS bit threshold=.+ - TXQ flags=0xf00')
 409
 410     def _configure_l2fwd(self):
 411         """
 412         Configure VM to perform L2 forwarding between NICs by l2fwd module
 413         """
 414         if int(S.getValue('GUEST_NIC_QUEUES')):
 415             self._set_multi_queue_nic()
 416         self._configure_copy_sources('l2fwd')
 417         self._configure_disable_firewall()
 418
 419         # build and configure system for l2fwd
 420         self.execute_and_wait('cd ' + S.getValue('GUEST_OVS_DPDK_DIR') +
 421                               '/l2fwd')
 422         self.execute_and_wait('export CC=gcc')
 423
 424         self.execute_and_wait('make')
 425         self.execute_and_wait('insmod ' + S.getValue('GUEST_OVS_DPDK_DIR') +
 426                               '/l2fwd' + '/l2fwd.ko net1=' + self._net1 +
 427                               ' net2=' + self._net2)
 428
 429     def _configure_linux_bridge(self):
 430         """
 431         Configure VM to perform L2 forwarding between NICs by linux bridge
 432         """
 433         if int(S.getValue('GUEST_NIC_QUEUES')):
 434             self._set_multi_queue_nic()
 435         self._configure_disable_firewall()
 436
 437         self.execute('ip addr add ' +
 438                      S.getValue('VANILLA_NIC1_IP_CIDR')[self._number] +
 439                      ' dev ' + self._net1)
 440         self.execute('ip link set dev ' + self._net1 + ' up')
 441
 442         self.execute('ip addr add ' +
 443                      S.getValue('VANILLA_NIC2_IP_CIDR')[self._number] +
 444                      ' dev ' + self._net2)
 445         self.execute('ip link set dev ' + self._net2 + ' up')
 446
 447         # configure linux bridge
 448         self.execute('brctl addbr br0')
 449         self.execute('brctl addif br0 ' + self._net1 + ' ' + self._net2)
 450         self.execute('ip addr add ' +
 451                      S.getValue('VANILLA_BRIDGE_IP')[self._number] +
 452                      ' dev br0')
 453         self.execute('ip link set dev br0 up')
 454
 455         # Add the arp entries for the IXIA ports and the bridge you are using.
 456         # Use command line values if provided.
 457         trafficgen_mac = get_test_param('vanilla_tgen_port1_mac',
 458                                         S.getValue('VANILLA_TGEN_PORT1_MAC'))
 459         trafficgen_ip = get_test_param('vanilla_tgen_port1_ip',
 460                                        S.getValue('VANILLA_TGEN_PORT1_IP'))
 461
 462         self.execute('arp -s ' + trafficgen_ip + ' ' + trafficgen_mac)
 463
 464         trafficgen_mac = get_test_param('vanilla_tgen_port2_mac',
 465                                         S.getValue('VANILLA_TGEN_PORT2_MAC'))
 466         trafficgen_ip = get_test_param('vanilla_tgen_port2_ip',
 467                                        S.getValue('VANILLA_TGEN_PORT2_IP'))
 468
 469         self.execute('arp -s ' + trafficgen_ip + ' ' + trafficgen_mac)
 470
 471         # Enable forwarding
 472         self.execute('sysctl -w net.ipv4.ip_forward=1')
 473
 474         # Controls source route verification
 475         # 0 means no source validation
 476         self.execute('sysctl -w net.ipv4.conf.all.rp_filter=0')
 477         self.execute('sysctl -w net.ipv4.conf.' + self._net1 + '.rp_filter=0')
 478         self.execute('sysctl -w net.ipv4.conf.' + self._net2 + '.rp_filter=0')
 479
 480     def _set_multi_queue_nic(self):
 481         """
 482         Enable multi-queue in guest kernel with ethool.
 483         :return: None
 484         """
 485         self.execute_and_wait('ethtool -L {} combined {}'.format(
 486             self._net1, S.getValue('GUEST_NIC_QUEUES')))
 487         self.execute_and_wait('ethtool -l {}'.format(self._net1))
 488         self.execute_and_wait('ethtool -L {} combined {}'.format(
 489             self._net2, S.getValue('GUEST_NIC_QUEUES')))
 490         self.execute_and_wait('ethtool -l {}'.format(self._net2))