vnfs/qemu/qemu.py

   1 # Copyright 2015-2016 Intel Corporation.
   2 #
   3 # Licensed under the Apache License, Version 2.0 (the "License");
   4 # you may not use this file except in compliance with the License.
   5 # You may obtain a copy of the License at
   6 #
   7 #   http://www.apache.org/licenses/LICENSE-2.0
   8 #
   9 # Unless required by applicable law or agreed to in writing, software
  10 # distributed under the License is distributed on an "AS IS" BASIS,
  11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12 # See the License for the specific language governing permissions and
  13 # limitations under the License.
  14
  15 """Automation of QEMU hypervisor for launching vhost-cuse enabled guests.
  16 """
  17
  18 import os
  19 import logging
  20 import locale
  21 import re
  22 import subprocess
  23
  24 from conf import settings as S
  25 from conf import get_test_param
  26 from vnfs.vnf.vnf import IVnf
  27
  28 class IVnfQemu(IVnf):
  29     """
  30     Abstract class for controling an instance of QEMU
  31     """
  32     _cmd = None
  33     _expect = S.getValue('GUEST_PROMPT_LOGIN')
  34     _proc_name = 'qemu'
  35
  36     class GuestCommandFilter(logging.Filter):
  37         """
  38         Filter out strings beginning with 'guestcmd :'.
  39         """
  40         def filter(self, record):
  41             return record.getMessage().startswith(self.prefix)
  42
  43     def __init__(self):
  44         """
  45         Initialisation function.
  46         """
  47         super(IVnfQemu, self).__init__()
  48         self._logger = logging.getLogger(__name__)
  49         self._logfile = os.path.join(
  50             S.getValue('LOG_DIR'),
  51             S.getValue('LOG_FILE_QEMU')) + str(self._number)
  52         self._timeout = S.getValue('GUEST_TIMEOUT')[self._number]
  53         self._monitor = '%s/vm%dmonitor' % ('/tmp', self._number)
  54         self._net1 = get_test_param('guest_nic1_name', None)
  55         if self._net1 == None:
  56             self._net1 = S.getValue('GUEST_NIC1_NAME')[self._number]
  57         else:
  58             self._net1 = self._net1.split(',')[self._number]
  59         self._net2 = get_test_param('guest_nic2_name', None)
  60         if self._net2 == None:
  61             self._net2 = S.getValue('GUEST_NIC2_NAME')[self._number]
  62         else:
  63             self._net2 = self._net2.split(',')[self._number]
  64
  65         # set guest loopback application based on VNF configuration
  66         # cli option take precedence to config file values
  67         self._guest_loopback = S.getValue('GUEST_LOOPBACK')[self._number]
  68
  69
  70         name = 'Client%d' % self._number
  71         vnc = ':%d' % self._number
  72         # don't use taskset to affinize main qemu process; It causes hangup
  73         # of 2nd VM in case of DPDK. It also slows down VM responsivnes.
  74         self._cmd = ['sudo', '-E', S.getValue('QEMU_BIN'),
  75                      '-m', S.getValue('GUEST_MEMORY')[self._number],
  76                      '-smp', str(S.getValue('GUEST_SMP')[self._number]),
  77                      '-cpu', 'host,migratable=off',
  78                      '-drive', 'if=scsi,file=' +
  79                      S.getValue('GUEST_IMAGE')[self._number],
  80                      '-boot', 'c', '--enable-kvm',
  81                      '-monitor', 'unix:%s,server,nowait' % self._monitor,
  82                      '-object',
  83                      'memory-backend-file,id=mem,size=' +
  84                      str(S.getValue('GUEST_MEMORY')[self._number]) + 'M,' +
  85                      'mem-path=' + S.getValue('HUGEPAGE_DIR') + ',share=on',
  86                      '-numa', 'node,memdev=mem -mem-prealloc',
  87                      '-nographic', '-vnc', str(vnc), '-name', name,
  88                      '-snapshot', '-net none', '-no-reboot',
  89                      '-drive',
  90                      'if=scsi,format=raw,file=fat:rw:%s,snapshot=off' %
  91                      S.getValue('GUEST_SHARE_DIR')[self._number],
  92                     ]
  93         self._configure_logging()
  94
  95     def _configure_logging(self):
  96         """
  97         Configure logging.
  98         """
  99         self.GuestCommandFilter.prefix = self._log_prefix
 100
 101         logger = logging.getLogger()
 102         cmd_logger = logging.FileHandler(
 103             filename=os.path.join(S.getValue('LOG_DIR'),
 104                                   S.getValue('LOG_FILE_GUEST_CMDS')) +
 105             str(self._number))
 106         cmd_logger.setLevel(logging.DEBUG)
 107         cmd_logger.addFilter(self.GuestCommandFilter())
 108         logger.addHandler(cmd_logger)
 109
 110     # startup/Shutdown
 111
 112     def start(self):
 113         """
 114         Start QEMU instance, login and prepare for commands.
 115         """
 116         super(IVnfQemu, self).start()
 117         if S.getValue('VNF_AFFINITIZATION_ON'):
 118             self._affinitize()
 119
 120         if self._timeout:
 121             self._config_guest_loopback()
 122
 123     def stop(self):
 124         """
 125         Stops VNF instance gracefully first.
 126         """
 127         # exit testpmd if needed
 128         if self._guest_loopback == 'testpmd':
 129             self.execute_and_wait('stop', 120, "Done")
 130             self.execute_and_wait('quit', 120, "bye")
 131
 132         # turn off VM
 133         self.execute_and_wait('poweroff', 120, "Power down")
 134
 135         # just for case that graceful shutdown failed
 136         super(IVnfQemu, self).stop()
 137
 138     # helper functions
 139
 140     def _login(self, timeout=120):
 141         """
 142         Login to QEMU instance.
 143
 144         This can be used immediately after booting the machine, provided a
 145         sufficiently long ``timeout`` is given.
 146
 147         :param timeout: Timeout to wait for login to complete.
 148
 149         :returns: None
 150         """
 151         # if no timeout was set, we likely started QEMU without waiting for it
 152         # to boot. This being the case, we best check that it has finished
 153         # first.
 154         if not self._timeout:
 155             self._expect_process(timeout=timeout)
 156
 157         self._child.sendline(S.getValue('GUEST_USERNAME'))
 158         self._child.expect(S.getValue('GUEST_PROMPT_PASSWORD'), timeout=5)
 159         self._child.sendline(S.getValue('GUEST_PASSWORD'))
 160
 161         self._expect_process(S.getValue('GUEST_PROMPT'), timeout=5)
 162
 163     def send_and_pass(self, cmd, timeout=30):
 164         """
 165         Send ``cmd`` and wait ``timeout`` seconds for it to pass.
 166
 167         :param cmd: Command to send to guest.
 168         :param timeout: Time to wait for prompt before checking return code.
 169
 170         :returns: None
 171         """
 172         self.execute(cmd)
 173         self.wait(S.getValue('GUEST_PROMPT'), timeout=timeout)
 174         self.execute('echo $?')
 175         self._child.expect('^0$', timeout=1)  # expect a 0
 176         self.wait(S.getValue('GUEST_PROMPT'), timeout=timeout)
 177
 178     def _affinitize(self):
 179         """
 180         Affinitize the SMP cores of a QEMU instance.
 181
 182         This is a bit of a hack. The 'socat' utility is used to
 183         interact with the QEMU HMP. This is necessary due to the lack
 184         of QMP in older versions of QEMU, like v1.6.2. In future
 185         releases, this should be replaced with calls to libvirt or
 186         another Python-QEMU wrapper library.
 187
 188         :returns: None
 189         """
 190         thread_id = (r'.* CPU #%d: .* thread_id=(\d+)')
 191
 192         self._logger.info('Affinitizing guest...')
 193
 194         cur_locale = locale.getdefaultlocale()[1]
 195         proc = subprocess.Popen(
 196             ('echo', 'info cpus'), stdout=subprocess.PIPE)
 197         output = subprocess.check_output(
 198             ('sudo', 'socat', '-', 'UNIX-CONNECT:%s' % self._monitor),
 199             stdin=proc.stdout)
 200         proc.wait()
 201
 202         for cpu in range(0, int(S.getValue('GUEST_SMP')[self._number])):
 203             match = None
 204             for line in output.decode(cur_locale).split('\n'):
 205                 match = re.search(thread_id % cpu, line)
 206                 if match:
 207                     self._affinitize_pid(
 208                         S.getValue('GUEST_CORE_BINDING')[self._number][cpu],
 209                         match.group(1))
 210                     break
 211
 212             if not match:
 213                 self._logger.error('Failed to affinitize guest core #%d. Could'
 214                                    ' not parse tid.', cpu)
 215
 216     def _config_guest_loopback(self):
 217         """
 218         Configure VM to run VNF, e.g. port forwarding application based on the configuration
 219         """
 220         if self._guest_loopback == 'testpmd':
 221             self._login()
 222             self._configure_testpmd()
 223         elif self._guest_loopback == 'l2fwd':
 224             self._login()
 225             self._configure_l2fwd()
 226         elif self._guest_loopback == 'linux_bridge':
 227             self._login()
 228             self._configure_linux_bridge()
 229         elif self._guest_loopback != 'buildin':
 230             self._logger.error('Unsupported guest loopback method "%s" was specified. Option'
 231                                ' "buildin" will be used as a fallback.', self._guest_loopback)
 232
 233     def wait(self, prompt=S.getValue('GUEST_PROMPT'), timeout=30):
 234         super(IVnfQemu, self).wait(prompt=prompt, timeout=timeout)
 235
 236     def execute_and_wait(self, cmd, timeout=30,
 237                          prompt=S.getValue('GUEST_PROMPT')):
 238         super(IVnfQemu, self).execute_and_wait(cmd, timeout=timeout,
 239                                                prompt=prompt)
 240
 241     def _modify_dpdk_makefile(self):
 242         """
 243         Modifies DPDK makefile in Guest before compilation
 244         """
 245         pass
 246
 247     def _configure_copy_sources(self, dirname):
 248         """
 249         Mount shared directory and copy DPDK and l2fwd sources
 250         """
 251         # mount shared directory
 252         self.execute_and_wait('umount ' + S.getValue('OVS_DPDK_SHARE'))
 253         self.execute_and_wait('rm -rf ' + S.getValue('GUEST_OVS_DPDK_DIR'))
 254         self.execute_and_wait('mkdir -p ' + S.getValue('OVS_DPDK_SHARE'))
 255         self.execute_and_wait('mount -o iocharset=utf8 /dev/sdb1 ' +
 256                               S.getValue('OVS_DPDK_SHARE'))
 257         self.execute_and_wait('mkdir -p ' + S.getValue('GUEST_OVS_DPDK_DIR'))
 258         self.execute_and_wait('cp -ra ' + os.path.join(S.getValue('OVS_DPDK_SHARE'), dirname) +
 259                               ' ' + S.getValue('GUEST_OVS_DPDK_DIR'))
 260
 261     def _configure_disable_firewall(self):
 262         """
 263         Disable firewall in VM
 264         """
 265         for iptables in ['iptables', 'ip6tables']:
 266             # filter table
 267             for chain in ['INPUT', 'FORWARD', 'OUTPUT']:
 268                 self.execute_and_wait("{} -t filter -P {} ACCEPT".format(iptables, chain))
 269             # mangle table
 270             for chain in ['PREROUTING', 'INPUT', 'FORWARD', 'OUTPUT', 'POSTROUTING']:
 271                 self.execute_and_wait("{} -t mangle -P {} ACCEPT".format(iptables, chain))
 272             # nat table
 273             for chain in ['PREROUTING', 'INPUT', 'OUTPUT', 'POSTROUTING']:
 274                 self.execute_and_wait("{} -t nat -P {} ACCEPT".format(iptables, chain))
 275
 276             # flush rules and delete chains created by user
 277             for table in ['filter', 'mangle', 'nat']:
 278                 self.execute_and_wait("{} -t {} -F".format(iptables, table))
 279                 self.execute_and_wait("{} -t {} -X".format(iptables, table))
 280
 281
 282     def _configure_testpmd(self):
 283         """
 284         Configure VM to perform L2 forwarding between NICs by DPDK's testpmd
 285         """
 286         self._configure_copy_sources('DPDK')
 287         self._configure_disable_firewall()
 288
 289         # Guest images _should_ have 1024 hugepages by default,
 290         # but just in case:'''
 291         self.execute_and_wait('sysctl vm.nr_hugepages=1024')
 292
 293         # Mount hugepages
 294         self.execute_and_wait('mkdir -p /dev/hugepages')
 295         self.execute_and_wait(
 296             'mount -t hugetlbfs hugetlbfs /dev/hugepages')
 297
 298         # build and configure system for dpdk
 299         self.execute_and_wait('cd ' + S.getValue('GUEST_OVS_DPDK_DIR') +
 300                               '/DPDK')
 301         self.execute_and_wait('export CC=gcc')
 302         self.execute_and_wait('export RTE_SDK=' +
 303                               S.getValue('GUEST_OVS_DPDK_DIR') + '/DPDK')
 304         self.execute_and_wait('export RTE_TARGET=%s' % S.getValue('RTE_TARGET'))
 305
 306         # modify makefile if needed
 307         self._modify_dpdk_makefile()
 308
 309         self.execute_and_wait('make RTE_OUTPUT=$RTE_SDK/$RTE_TARGET -C '
 310                               '$RTE_SDK/lib/librte_eal/linuxapp/igb_uio')
 311         self.execute_and_wait('modprobe uio')
 312         self.execute_and_wait('insmod %s/kmod/igb_uio.ko' %
 313                               S.getValue('RTE_TARGET'))
 314         self.execute_and_wait('./tools/dpdk_nic_bind.py --status')
 315         self.execute_and_wait(
 316             './tools/dpdk_nic_bind.py -b igb_uio' ' ' +
 317             S.getValue('GUEST_NET1_PCI_ADDRESS')[self._number] + ' ' +
 318             S.getValue('GUEST_NET2_PCI_ADDRESS')[self._number])
 319
 320         # build and run 'test-pmd'
 321         self.execute_and_wait('cd ' + S.getValue('GUEST_OVS_DPDK_DIR') +
 322                               '/DPDK/app/test-pmd')
 323         self.execute_and_wait('make clean')
 324         self.execute_and_wait('make')
 325         self.execute_and_wait('./testpmd -c 0x3 -n 4 --socket-mem 512 --'
 326                               ' --burst=64 -i --txqflags=0xf00 ' +
 327                               '--disable-hw-vlan', 60, "Done")
 328         self.execute('set fwd mac_retry', 1)
 329         self.execute_and_wait('start', 20,
 330                               'TX RS bit threshold=0 - TXQ flags=0xf00')
 331
 332     def _configure_l2fwd(self):
 333         """
 334         Configure VM to perform L2 forwarding between NICs by l2fwd module
 335         """
 336         self._configure_copy_sources('l2fwd')
 337         self._configure_disable_firewall()
 338
 339         # build and configure system for l2fwd
 340         self.execute_and_wait('cd ' + S.getValue('GUEST_OVS_DPDK_DIR') +
 341                               '/l2fwd')
 342         self.execute_and_wait('export CC=gcc')
 343
 344         self.execute_and_wait('make')
 345         self.execute_and_wait('insmod ' + S.getValue('GUEST_OVS_DPDK_DIR') +
 346                               '/l2fwd' + '/l2fwd.ko net1=' + self._net1 +
 347                               ' net2=' + self._net2)
 348
 349     def _configure_linux_bridge(self):
 350         """
 351         Configure VM to perform L2 forwarding between NICs by linux bridge
 352         """
 353         self._configure_disable_firewall()
 354
 355         self.execute('ifconfig ' + self._net1 + ' ' +
 356                      S.getValue('VANILLA_NIC1_IP_CIDR')[self._number])
 357
 358         self.execute('ifconfig ' + self._net2 + ' ' +
 359                      S.getValue('VANILLA_NIC2_IP_CIDR')[self._number])
 360
 361         # configure linux bridge
 362         self.execute('brctl addbr br0')
 363         self.execute('brctl addif br0 ' + self._net1 + ' ' + self._net2)
 364         self.execute('ifconfig br0 ' +
 365                      S.getValue('VANILLA_BRIDGE_IP')[self._number])
 366
 367         # Add the arp entries for the IXIA ports and the bridge you are using.
 368         # Use command line values if provided.
 369         trafficgen_mac = get_test_param('vanilla_tgen_port1_mac',
 370                                         S.getValue('VANILLA_TGEN_PORT1_MAC'))
 371         trafficgen_ip = get_test_param('vanilla_tgen_port1_ip',
 372                                        S.getValue('VANILLA_TGEN_PORT1_IP'))
 373
 374         self.execute('arp -s ' + trafficgen_ip + ' ' + trafficgen_mac)
 375
 376         trafficgen_mac = get_test_param('vanilla_tgen_port2_mac',
 377                                         S.getValue('VANILLA_TGEN_PORT2_MAC'))
 378         trafficgen_ip = get_test_param('vanilla_tgen_port2_ip',
 379                                        S.getValue('VANILLA_TGEN_PORT2_IP'))
 380
 381         self.execute('arp -s ' + trafficgen_ip + ' ' + trafficgen_mac)
 382
 383         # Enable forwarding
 384         self.execute('sysctl -w net.ipv4.ip_forward=1')
 385
 386         # Controls source route verification
 387         # 0 means no source validation
 388         self.execute('sysctl -w net.ipv4.conf.all.rp_filter=0')
 389         self.execute('sysctl -w net.ipv4.conf.' + self._net1 + '.rp_filter=0')
 390         self.execute('sysctl -w net.ipv4.conf.' + self._net2 + '.rp_filter=0')