vnfs/qemu/qemu.py

   1 # Copyright 2015-2016 Intel Corporation.
   2 #
   3 # Licensed under the Apache License, Version 2.0 (the "License");
   4 # you may not use this file except in compliance with the License.
   5 # You may obtain a copy of the License at
   6 #
   7 #   http://www.apache.org/licenses/LICENSE-2.0
   8 #
   9 # Unless required by applicable law or agreed to in writing, software
  10 # distributed under the License is distributed on an "AS IS" BASIS,
  11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12 # See the License for the specific language governing permissions and
  13 # limitations under the License.
  14
  15 """Automation of QEMU hypervisor for launching vhost-cuse enabled guests.
  16 """
  17
  18 import os
  19 import logging
  20 import locale
  21 import re
  22 import subprocess
  23 import time
  24
  25 from conf import settings as S
  26 from conf import get_test_param
  27 from vnfs.vnf.vnf import IVnf
  28
  29 class IVnfQemu(IVnf):
  30     """
  31     Abstract class for controling an instance of QEMU
  32     """
  33     _cmd = None
  34     _expect = S.getValue('GUEST_PROMPT_LOGIN')
  35     _proc_name = 'qemu'
  36
  37     class GuestCommandFilter(logging.Filter):
  38         """
  39         Filter out strings beginning with 'guestcmd :'.
  40         """
  41         def filter(self, record):
  42             return record.getMessage().startswith(self.prefix)
  43
  44     def __init__(self):
  45         """
  46         Initialisation function.
  47         """
  48         super(IVnfQemu, self).__init__()
  49         self._logger = logging.getLogger(__name__)
  50         self._logfile = os.path.join(
  51             S.getValue('LOG_DIR'),
  52             S.getValue('LOG_FILE_QEMU')) + str(self._number)
  53         self._timeout = S.getValue('GUEST_TIMEOUT')[self._number]
  54         self._monitor = '%s/vm%dmonitor' % ('/tmp', self._number)
  55         self._net1 = get_test_param('guest_nic1_name', None)
  56         if self._net1 == None:
  57             self._net1 = S.getValue('GUEST_NIC1_NAME')[self._number]
  58         else:
  59             self._net1 = self._net1.split(',')[self._number]
  60         self._net2 = get_test_param('guest_nic2_name', None)
  61         if self._net2 == None:
  62             self._net2 = S.getValue('GUEST_NIC2_NAME')[self._number]
  63         else:
  64             self._net2 = self._net2.split(',')[self._number]
  65
  66         # set guest loopback application based on VNF configuration
  67         # cli option take precedence to config file values
  68         self._guest_loopback = S.getValue('GUEST_LOOPBACK')[self._number]
  69
  70         self._testpmd_fwd_mode = S.getValue('GUEST_TESTPMD_FWD_MODE')
  71         # in case of SRIOV we must ensure, that MAC addresses are not swapped
  72         if S.getValue('SRIOV_ENABLED') and self._testpmd_fwd_mode.startswith('mac') and \
  73            not S.getValue('VNF').endswith('PciPassthrough'):
  74
  75             self._logger.info("SRIOV detected, forwarding mode of testpmd was changed from '%s' to '%s'",
  76                               self._testpmd_fwd_mode, 'io')
  77             self._testpmd_fwd_mode = 'io'
  78
  79         name = 'Client%d' % self._number
  80         vnc = ':%d' % self._number
  81         # don't use taskset to affinize main qemu process; It causes hangup
  82         # of 2nd VM in case of DPDK. It also slows down VM responsivnes.
  83         self._cmd = ['sudo', '-E', S.getValue('QEMU_BIN'),
  84                      '-m', S.getValue('GUEST_MEMORY')[self._number],
  85                      '-smp', str(S.getValue('GUEST_SMP')[self._number]),
  86                      '-cpu', 'host,migratable=off',
  87                      '-drive', 'if=scsi,file=' +
  88                      S.getValue('GUEST_IMAGE')[self._number],
  89                      '-boot', 'c', '--enable-kvm',
  90                      '-monitor', 'unix:%s,server,nowait' % self._monitor,
  91                      '-object',
  92                      'memory-backend-file,id=mem,size=' +
  93                      str(S.getValue('GUEST_MEMORY')[self._number]) + 'M,' +
  94                      'mem-path=' + S.getValue('HUGEPAGE_DIR') + ',share=on',
  95                      '-numa', 'node,memdev=mem -mem-prealloc',
  96                      '-nographic', '-vnc', str(vnc), '-name', name,
  97                      '-snapshot', '-net none', '-no-reboot',
  98                      '-drive',
  99                      'if=scsi,format=raw,file=fat:rw:%s,snapshot=off' %
 100                      S.getValue('GUEST_SHARE_DIR')[self._number],
 101                     ]
 102         self._configure_logging()
 103
 104     def _configure_logging(self):
 105         """
 106         Configure logging.
 107         """
 108         self.GuestCommandFilter.prefix = self._log_prefix
 109
 110         logger = logging.getLogger()
 111         cmd_logger = logging.FileHandler(
 112             filename=os.path.join(S.getValue('LOG_DIR'),
 113                                   S.getValue('LOG_FILE_GUEST_CMDS')) +
 114             str(self._number))
 115         cmd_logger.setLevel(logging.DEBUG)
 116         cmd_logger.addFilter(self.GuestCommandFilter())
 117         logger.addHandler(cmd_logger)
 118
 119     # startup/Shutdown
 120
 121     def start(self):
 122         """
 123         Start QEMU instance, login and prepare for commands.
 124         """
 125         super(IVnfQemu, self).start()
 126         if S.getValue('VNF_AFFINITIZATION_ON'):
 127             self._affinitize()
 128
 129         if self._timeout:
 130             self._config_guest_loopback()
 131
 132     def stop(self):
 133         """
 134         Stops VNF instance gracefully first.
 135         """
 136         # exit testpmd if needed
 137         if self._guest_loopback == 'testpmd':
 138             self.execute_and_wait('stop', 120, "Done")
 139             self.execute_and_wait('quit', 120, "bye")
 140
 141         # turn off VM
 142         self.execute_and_wait('poweroff', 120, "Power down")
 143         # VM OS is off, but wait until qemu shutdowns
 144         time.sleep(2)
 145
 146         # just for case that graceful shutdown failed
 147         super(IVnfQemu, self).stop()
 148
 149     # helper functions
 150
 151     def _login(self, timeout=120):
 152         """
 153         Login to QEMU instance.
 154
 155         This can be used immediately after booting the machine, provided a
 156         sufficiently long ``timeout`` is given.
 157
 158         :param timeout: Timeout to wait for login to complete.
 159
 160         :returns: None
 161         """
 162         # if no timeout was set, we likely started QEMU without waiting for it
 163         # to boot. This being the case, we best check that it has finished
 164         # first.
 165         if not self._timeout:
 166             self._expect_process(timeout=timeout)
 167
 168         self._child.sendline(S.getValue('GUEST_USERNAME'))
 169         self._child.expect(S.getValue('GUEST_PROMPT_PASSWORD'), timeout=5)
 170         self._child.sendline(S.getValue('GUEST_PASSWORD'))
 171
 172         self._expect_process(S.getValue('GUEST_PROMPT'), timeout=5)
 173
 174     def send_and_pass(self, cmd, timeout=30):
 175         """
 176         Send ``cmd`` and wait ``timeout`` seconds for it to pass.
 177
 178         :param cmd: Command to send to guest.
 179         :param timeout: Time to wait for prompt before checking return code.
 180
 181         :returns: None
 182         """
 183         self.execute(cmd)
 184         self.wait(S.getValue('GUEST_PROMPT'), timeout=timeout)
 185         self.execute('echo $?')
 186         self._child.expect('^0$', timeout=1)  # expect a 0
 187         self.wait(S.getValue('GUEST_PROMPT'), timeout=timeout)
 188
 189     def _affinitize(self):
 190         """
 191         Affinitize the SMP cores of a QEMU instance.
 192
 193         This is a bit of a hack. The 'socat' utility is used to
 194         interact with the QEMU HMP. This is necessary due to the lack
 195         of QMP in older versions of QEMU, like v1.6.2. In future
 196         releases, this should be replaced with calls to libvirt or
 197         another Python-QEMU wrapper library.
 198
 199         :returns: None
 200         """
 201         thread_id = (r'.* CPU #%d: .* thread_id=(\d+)')
 202
 203         self._logger.info('Affinitizing guest...')
 204
 205         cur_locale = locale.getdefaultlocale()[1]
 206         proc = subprocess.Popen(
 207             ('echo', 'info cpus'), stdout=subprocess.PIPE)
 208         output = subprocess.check_output(
 209             ('sudo', 'socat', '-', 'UNIX-CONNECT:%s' % self._monitor),
 210             stdin=proc.stdout)
 211         proc.wait()
 212
 213         for cpu in range(0, int(S.getValue('GUEST_SMP')[self._number])):
 214             match = None
 215             for line in output.decode(cur_locale).split('\n'):
 216                 match = re.search(thread_id % cpu, line)
 217                 if match:
 218                     self._affinitize_pid(
 219                         S.getValue('GUEST_CORE_BINDING')[self._number][cpu],
 220                         match.group(1))
 221                     break
 222
 223             if not match:
 224                 self._logger.error('Failed to affinitize guest core #%d. Could'
 225                                    ' not parse tid.', cpu)
 226
 227     def _config_guest_loopback(self):
 228         """
 229         Configure VM to run VNF, e.g. port forwarding application based on the configuration
 230         """
 231         if self._guest_loopback == 'testpmd':
 232             self._login()
 233             self._configure_testpmd()
 234         elif self._guest_loopback == 'l2fwd':
 235             self._login()
 236             self._configure_l2fwd()
 237         elif self._guest_loopback == 'linux_bridge':
 238             self._login()
 239             self._configure_linux_bridge()
 240         elif self._guest_loopback != 'buildin':
 241             self._logger.error('Unsupported guest loopback method "%s" was specified. Option'
 242                                ' "buildin" will be used as a fallback.', self._guest_loopback)
 243
 244     def wait(self, prompt=S.getValue('GUEST_PROMPT'), timeout=30):
 245         super(IVnfQemu, self).wait(prompt=prompt, timeout=timeout)
 246
 247     def execute_and_wait(self, cmd, timeout=30,
 248                          prompt=S.getValue('GUEST_PROMPT')):
 249         super(IVnfQemu, self).execute_and_wait(cmd, timeout=timeout,
 250                                                prompt=prompt)
 251
 252     def _modify_dpdk_makefile(self):
 253         """
 254         Modifies DPDK makefile in Guest before compilation
 255         """
 256         pass
 257
 258     def _configure_copy_sources(self, dirname):
 259         """
 260         Mount shared directory and copy DPDK and l2fwd sources
 261         """
 262         # mount shared directory
 263         self.execute_and_wait('umount /dev/sdb1')
 264         self.execute_and_wait('rm -rf ' + S.getValue('GUEST_OVS_DPDK_DIR'))
 265         self.execute_and_wait('mkdir -p ' + S.getValue('OVS_DPDK_SHARE'))
 266         self.execute_and_wait('mount -o ro,iocharset=utf8 /dev/sdb1 ' +
 267                               S.getValue('OVS_DPDK_SHARE'))
 268         self.execute_and_wait('mkdir -p ' + S.getValue('GUEST_OVS_DPDK_DIR'))
 269         self.execute_and_wait('cp -ra ' + os.path.join(S.getValue('OVS_DPDK_SHARE'), dirname) +
 270                               ' ' + S.getValue('GUEST_OVS_DPDK_DIR'))
 271         self.execute_and_wait('umount /dev/sdb1')
 272
 273     def _configure_disable_firewall(self):
 274         """
 275         Disable firewall in VM
 276         """
 277         for iptables in ['iptables', 'ip6tables']:
 278             # filter table
 279             for chain in ['INPUT', 'FORWARD', 'OUTPUT']:
 280                 self.execute_and_wait("{} -t filter -P {} ACCEPT".format(iptables, chain))
 281             # mangle table
 282             for chain in ['PREROUTING', 'INPUT', 'FORWARD', 'OUTPUT', 'POSTROUTING']:
 283                 self.execute_and_wait("{} -t mangle -P {} ACCEPT".format(iptables, chain))
 284             # nat table
 285             for chain in ['PREROUTING', 'INPUT', 'OUTPUT', 'POSTROUTING']:
 286                 self.execute_and_wait("{} -t nat -P {} ACCEPT".format(iptables, chain))
 287
 288             # flush rules and delete chains created by user
 289             for table in ['filter', 'mangle', 'nat']:
 290                 self.execute_and_wait("{} -t {} -F".format(iptables, table))
 291                 self.execute_and_wait("{} -t {} -X".format(iptables, table))
 292
 293
 294     def _configure_testpmd(self):
 295         """
 296         Configure VM to perform L2 forwarding between NICs by DPDK's testpmd
 297         """
 298         self._configure_copy_sources('DPDK')
 299         self._configure_disable_firewall()
 300
 301         # Guest images _should_ have 1024 hugepages by default,
 302         # but just in case:'''
 303         self.execute_and_wait('sysctl vm.nr_hugepages=1024')
 304
 305         # Mount hugepages
 306         self.execute_and_wait('mkdir -p /dev/hugepages')
 307         self.execute_and_wait(
 308             'mount -t hugetlbfs hugetlbfs /dev/hugepages')
 309
 310         # build and configure system for dpdk
 311         self.execute_and_wait('cd ' + S.getValue('GUEST_OVS_DPDK_DIR') +
 312                               '/DPDK')
 313         self.execute_and_wait('export CC=gcc')
 314         self.execute_and_wait('export RTE_SDK=' +
 315                               S.getValue('GUEST_OVS_DPDK_DIR') + '/DPDK')
 316         self.execute_and_wait('export RTE_TARGET=%s' % S.getValue('RTE_TARGET'))
 317
 318         # modify makefile if needed
 319         self._modify_dpdk_makefile()
 320
 321         # disable network interfaces, so DPDK can take care of them
 322         self.execute_and_wait('ifdown ' + self._net1)
 323         self.execute_and_wait('ifdown ' + self._net2)
 324
 325         # build and insert igb_uio and rebind interfaces to it
 326         self.execute_and_wait('make RTE_OUTPUT=$RTE_SDK/$RTE_TARGET -C '
 327                               '$RTE_SDK/lib/librte_eal/linuxapp/igb_uio')
 328         self.execute_and_wait('modprobe uio')
 329         self.execute_and_wait('insmod %s/kmod/igb_uio.ko' %
 330                               S.getValue('RTE_TARGET'))
 331         self.execute_and_wait('./tools/dpdk_nic_bind.py --status')
 332         self.execute_and_wait(
 333             './tools/dpdk_nic_bind.py -u' ' ' +
 334             S.getValue('GUEST_NET1_PCI_ADDRESS')[self._number] + ' ' +
 335             S.getValue('GUEST_NET2_PCI_ADDRESS')[self._number])
 336         self.execute_and_wait(
 337             './tools/dpdk_nic_bind.py -b igb_uio' ' ' +
 338             S.getValue('GUEST_NET1_PCI_ADDRESS')[self._number] + ' ' +
 339             S.getValue('GUEST_NET2_PCI_ADDRESS')[self._number])
 340         self.execute_and_wait('./tools/dpdk_nic_bind.py --status')
 341
 342         # build and run 'test-pmd'
 343         self.execute_and_wait('cd ' + S.getValue('GUEST_OVS_DPDK_DIR') +
 344                               '/DPDK/app/test-pmd')
 345         self.execute_and_wait('make clean')
 346         self.execute_and_wait('make')
 347         self.execute_and_wait('./testpmd -c 0x3 -n 4 --socket-mem 512 --'
 348                               ' --burst=64 -i --txqflags=0xf00 ' +
 349                               '--disable-hw-vlan', 60, "Done")
 350         self.execute('set fwd ' + self._testpmd_fwd_mode, 1)
 351         self.execute_and_wait('start', 20,
 352                               'TX RS bit threshold=.+ - TXQ flags=0xf00')
 353
 354     def _configure_l2fwd(self):
 355         """
 356         Configure VM to perform L2 forwarding between NICs by l2fwd module
 357         """
 358         self._configure_copy_sources('l2fwd')
 359         self._configure_disable_firewall()
 360
 361         # build and configure system for l2fwd
 362         self.execute_and_wait('cd ' + S.getValue('GUEST_OVS_DPDK_DIR') +
 363                               '/l2fwd')
 364         self.execute_and_wait('export CC=gcc')
 365
 366         self.execute_and_wait('make')
 367         self.execute_and_wait('insmod ' + S.getValue('GUEST_OVS_DPDK_DIR') +
 368                               '/l2fwd' + '/l2fwd.ko net1=' + self._net1 +
 369                               ' net2=' + self._net2)
 370
 371     def _configure_linux_bridge(self):
 372         """
 373         Configure VM to perform L2 forwarding between NICs by linux bridge
 374         """
 375         self._configure_disable_firewall()
 376
 377         self.execute('ifconfig ' + self._net1 + ' ' +
 378                      S.getValue('VANILLA_NIC1_IP_CIDR')[self._number])
 379
 380         self.execute('ifconfig ' + self._net2 + ' ' +
 381                      S.getValue('VANILLA_NIC2_IP_CIDR')[self._number])
 382
 383         # configure linux bridge
 384         self.execute('brctl addbr br0')
 385         self.execute('brctl addif br0 ' + self._net1 + ' ' + self._net2)
 386         self.execute('ifconfig br0 ' +
 387                      S.getValue('VANILLA_BRIDGE_IP')[self._number])
 388
 389         # Add the arp entries for the IXIA ports and the bridge you are using.
 390         # Use command line values if provided.
 391         trafficgen_mac = get_test_param('vanilla_tgen_port1_mac',
 392                                         S.getValue('VANILLA_TGEN_PORT1_MAC'))
 393         trafficgen_ip = get_test_param('vanilla_tgen_port1_ip',
 394                                        S.getValue('VANILLA_TGEN_PORT1_IP'))
 395
 396         self.execute('arp -s ' + trafficgen_ip + ' ' + trafficgen_mac)
 397
 398         trafficgen_mac = get_test_param('vanilla_tgen_port2_mac',
 399                                         S.getValue('VANILLA_TGEN_PORT2_MAC'))
 400         trafficgen_ip = get_test_param('vanilla_tgen_port2_ip',
 401                                        S.getValue('VANILLA_TGEN_PORT2_IP'))
 402
 403         self.execute('arp -s ' + trafficgen_ip + ' ' + trafficgen_mac)
 404
 405         # Enable forwarding
 406         self.execute('sysctl -w net.ipv4.ip_forward=1')
 407
 408         # Controls source route verification
 409         # 0 means no source validation
 410         self.execute('sysctl -w net.ipv4.conf.all.rp_filter=0')
 411         self.execute('sysctl -w net.ipv4.conf.' + self._net1 + '.rp_filter=0')
 412         self.execute('sysctl -w net.ipv4.conf.' + self._net2 + '.rp_filter=0')