vnfs/qemu/qemu.py

   1 # Copyright 2015 Intel Corporation.
   2 #
   3 # Licensed under the Apache License, Version 2.0 (the "License");
   4 # you may not use this file except in compliance with the License.
   5 # You may obtain a copy of the License at
   6 #
   7 #   http://www.apache.org/licenses/LICENSE-2.0
   8 #
   9 # Unless required by applicable law or agreed to in writing, software
  10 # distributed under the License is distributed on an "AS IS" BASIS,
  11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12 # See the License for the specific language governing permissions and
  13 # limitations under the License.
  14
  15 """Automation of QEMU hypervisor for launching vhost-cuse enabled guests.
  16 """
  17
  18 import os
  19 import logging
  20 import locale
  21 import re
  22 import subprocess
  23
  24 from conf import settings as S
  25 from conf import get_test_param
  26 from vnfs.vnf.vnf import IVnf
  27
  28 class IVnfQemu(IVnf):
  29     """
  30     Abstract class for controling an instance of QEMU
  31     """
  32     _cmd = None
  33     _expect = S.getValue('GUEST_PROMPT_LOGIN')
  34     _proc_name = 'qemu'
  35
  36     class GuestCommandFilter(logging.Filter):
  37         """
  38         Filter out strings beginning with 'guestcmd :'.
  39         """
  40         def filter(self, record):
  41             return record.getMessage().startswith(self.prefix)
  42
  43     def __init__(self):
  44         """
  45         Initialisation function.
  46         """
  47         super(IVnfQemu, self).__init__()
  48         self._logger = logging.getLogger(__name__)
  49         self._logfile = os.path.join(
  50             S.getValue('LOG_DIR'),
  51             S.getValue('LOG_FILE_QEMU')) + str(self._number)
  52         self._timeout = S.getValue('GUEST_TIMEOUT')[self._number]
  53         self._monitor = '%s/vm%dmonitor' % ('/tmp', self._number)
  54         self._net1 = S.getValue('VANILLA_NIC1_NAME')[self._number]
  55         self._net2 = S.getValue('VANILLA_NIC2_NAME')[self._number]
  56
  57         name = 'Client%d' % self._number
  58         vnc = ':%d' % self._number
  59         # don't use taskset to affinize main qemu process; It causes hangup
  60         # of 2nd VM in case of DPDK. It also slows down VM responsivnes.
  61         self._cmd = ['sudo', '-E', S.getValue('QEMU_BIN'),
  62                      '-m', S.getValue('GUEST_MEMORY')[self._number],
  63                      '-smp', str(S.getValue('GUEST_SMP')[self._number]),
  64                      '-cpu', 'host',
  65                      '-drive', 'if=ide,file=' +
  66                      S.getValue('GUEST_IMAGE')[self._number],
  67                      '-boot', 'c', '--enable-kvm',
  68                      '-monitor', 'unix:%s,server,nowait' % self._monitor,
  69                      '-object',
  70                      'memory-backend-file,id=mem,size=' +
  71                      str(S.getValue('GUEST_MEMORY')[self._number]) + 'M,' +
  72                      'mem-path=' + S.getValue('HUGEPAGE_DIR') + ',share=on',
  73                      '-numa', 'node,memdev=mem -mem-prealloc',
  74                      '-nographic', '-vnc', str(vnc), '-name', name,
  75                      '-snapshot', '-net none', '-no-reboot',
  76                      '-drive',
  77                      'if=ide,file=fat:rw:%s,snapshot=off' %
  78                      S.getValue('GUEST_SHARE_DIR')[self._number],
  79                     ]
  80         self._configure_logging()
  81
  82     def _configure_logging(self):
  83         """
  84         Configure logging.
  85         """
  86         self.GuestCommandFilter.prefix = self._log_prefix
  87
  88         logger = logging.getLogger()
  89         cmd_logger = logging.FileHandler(
  90             filename=os.path.join(S.getValue('LOG_DIR'),
  91                                   S.getValue('LOG_FILE_GUEST_CMDS')) +
  92             str(self._number))
  93         cmd_logger.setLevel(logging.DEBUG)
  94         cmd_logger.addFilter(self.GuestCommandFilter())
  95         logger.addHandler(cmd_logger)
  96
  97     # startup/Shutdown
  98
  99     def start(self):
 100         """
 101         Start QEMU instance, login and prepare for commands.
 102         """
 103         super(IVnfQemu, self).start()
 104         if S.getValue('VNF_AFFINITIZATION_ON'):
 105             self._affinitize()
 106
 107         if self._timeout:
 108             self._config_guest_loopback()
 109
 110     # helper functions
 111
 112     def _login(self, timeout=120):
 113         """
 114         Login to QEMU instance.
 115
 116         This can be used immediately after booting the machine, provided a
 117         sufficiently long ``timeout`` is given.
 118
 119         :param timeout: Timeout to wait for login to complete.
 120
 121         :returns: None
 122         """
 123         # if no timeout was set, we likely started QEMU without waiting for it
 124         # to boot. This being the case, we best check that it has finished
 125         # first.
 126         if not self._timeout:
 127             self._expect_process(timeout=timeout)
 128
 129         self._child.sendline(S.getValue('GUEST_USERNAME'))
 130         self._child.expect(S.getValue('GUEST_PROMPT_PASSWORD'), timeout=5)
 131         self._child.sendline(S.getValue('GUEST_PASSWORD'))
 132
 133         self._expect_process(S.getValue('GUEST_PROMPT'), timeout=5)
 134
 135     def send_and_pass(self, cmd, timeout=30):
 136         """
 137         Send ``cmd`` and wait ``timeout`` seconds for it to pass.
 138
 139         :param cmd: Command to send to guest.
 140         :param timeout: Time to wait for prompt before checking return code.
 141
 142         :returns: None
 143         """
 144         self.execute(cmd)
 145         self.wait(S.getValue('GUEST_PROMPT'), timeout=timeout)
 146         self.execute('echo $?')
 147         self._child.expect('^0$', timeout=1)  # expect a 0
 148         self.wait(S.getValue('GUEST_PROMPT'), timeout=timeout)
 149
 150     def _affinitize(self):
 151         """
 152         Affinitize the SMP cores of a QEMU instance.
 153
 154         This is a bit of a hack. The 'socat' utility is used to
 155         interact with the QEMU HMP. This is necessary due to the lack
 156         of QMP in older versions of QEMU, like v1.6.2. In future
 157         releases, this should be replaced with calls to libvirt or
 158         another Python-QEMU wrapper library.
 159
 160         :returns: None
 161         """
 162         thread_id = (r'.* CPU #%d: .* thread_id=(\d+)')
 163
 164         self._logger.info('Affinitizing guest...')
 165
 166         cur_locale = locale.getlocale()[1]
 167         proc = subprocess.Popen(
 168             ('echo', 'info cpus'), stdout=subprocess.PIPE)
 169         output = subprocess.check_output(
 170             ('sudo', 'socat', '-', 'UNIX-CONNECT:%s' % self._monitor),
 171             stdin=proc.stdout)
 172         proc.wait()
 173
 174         for cpu in range(0, int(S.getValue('GUEST_SMP')[self._number])):
 175             match = None
 176             for line in output.decode(cur_locale).split('\n'):
 177                 match = re.search(thread_id % cpu, line)
 178                 if match:
 179                     self._affinitize_pid(
 180                         S.getValue('GUEST_CORE_BINDING')[self._number][cpu],
 181                         match.group(1))
 182                     break
 183
 184             if not match:
 185                 self._logger.error('Failed to affinitize guest core #%d. Could'
 186                                    ' not parse tid.', cpu)
 187
 188     def _config_guest_loopback(self):
 189         """
 190         Configure VM to run VNF (e.g. port forwarding application)
 191         """
 192         # set guest loopback application based on VNF configuration
 193         # cli option take precedence to config file values
 194         guest_loopback = get_test_param('guest_loopback', S.getValue('GUEST_LOOPBACK')[self._number])
 195         if guest_loopback == 'testpmd':
 196             self._login()
 197             self._configure_testpmd()
 198         elif guest_loopback == 'l2fwd':
 199             self._login()
 200             self._configure_l2fwd()
 201         elif guest_loopback == 'linux_bridge':
 202             self._login()
 203             self._configure_linux_bridge()
 204         elif guest_loopback != 'buildin':
 205             self._logger.error('Unsupported guest loopback method "%s" was specified. Option'
 206                                ' "buildin" will be used as a fallback.', guest_loopback)
 207
 208     def wait(self, prompt=S.getValue('GUEST_PROMPT'), timeout=30):
 209         super(IVnfQemu, self).wait(prompt=prompt, timeout=timeout)
 210
 211     def execute_and_wait(self, cmd, timeout=30,
 212                          prompt=S.getValue('GUEST_PROMPT')):
 213         super(IVnfQemu, self).execute_and_wait(cmd, timeout=timeout,
 214                                                prompt=prompt)
 215
 216     def _modify_dpdk_makefile(self):
 217         """
 218         Modifies DPDK makefile in Guest before compilation
 219         """
 220         pass
 221
 222     def _configure_copy_sources(self, dirname):
 223         """
 224         Mount shared directory and copy DPDK and l2fwd sources
 225         """
 226         # mount shared directory
 227         self.execute_and_wait('umount ' + S.getValue('OVS_DPDK_SHARE'))
 228         self.execute_and_wait('rm -rf ' + S.getValue('GUEST_OVS_DPDK_DIR'))
 229         self.execute_and_wait('mkdir -p ' + S.getValue('OVS_DPDK_SHARE'))
 230         self.execute_and_wait('mount -o iocharset=utf8 /dev/sdb1 ' +
 231                               S.getValue('OVS_DPDK_SHARE'))
 232         self.execute_and_wait('mkdir -p ' + S.getValue('GUEST_OVS_DPDK_DIR'))
 233         self.execute_and_wait('cp -ra ' + os.path.join(S.getValue('OVS_DPDK_SHARE'), dirname) +
 234                               ' ' + S.getValue('GUEST_OVS_DPDK_DIR'))
 235
 236     def _configure_disable_firewall(self):
 237         """
 238         Disable firewall in VM
 239         """
 240         # Disable services (F16)
 241         self.execute_and_wait('systemctl status iptables.service')
 242         self.execute_and_wait('systemctl stop iptables.service')
 243
 244     def _configure_testpmd(self):
 245         """
 246         Configure VM to perform L2 forwarding between NICs by DPDK's testpmd
 247         """
 248         self._configure_copy_sources('DPDK')
 249         self._configure_disable_firewall()
 250
 251         # Guest images _should_ have 1024 hugepages by default,
 252         # but just in case:'''
 253         self.execute_and_wait('sysctl vm.nr_hugepages=1024')
 254
 255         # Mount hugepages
 256         self.execute_and_wait('mkdir -p /dev/hugepages')
 257         self.execute_and_wait(
 258             'mount -t hugetlbfs hugetlbfs /dev/hugepages')
 259
 260         # build and configure system for dpdk
 261         self.execute_and_wait('cd ' + S.getValue('GUEST_OVS_DPDK_DIR') +
 262                               '/DPDK')
 263         self.execute_and_wait('export CC=gcc')
 264         self.execute_and_wait('export RTE_SDK=' +
 265                               S.getValue('GUEST_OVS_DPDK_DIR') + '/DPDK')
 266         self.execute_and_wait('export RTE_TARGET=%s' % S.getValue('RTE_TARGET'))
 267
 268         # modify makefile if needed
 269         self._modify_dpdk_makefile()
 270
 271         self.execute_and_wait('make RTE_OUTPUT=$RTE_SDK/$RTE_TARGET -C '
 272                               '$RTE_SDK/lib/librte_eal/linuxapp/igb_uio')
 273         self.execute_and_wait('modprobe uio')
 274         self.execute_and_wait('insmod %s/kmod/igb_uio.ko' %
 275                               S.getValue('RTE_TARGET'))
 276         self.execute_and_wait('./tools/dpdk_nic_bind.py --status')
 277         self.execute_and_wait(
 278             './tools/dpdk_nic_bind.py -b igb_uio' ' ' +
 279             S.getValue('GUEST_NET1_PCI_ADDRESS')[self._number] + ' ' +
 280             S.getValue('GUEST_NET2_PCI_ADDRESS')[self._number])
 281
 282         # build and run 'test-pmd'
 283         self.execute_and_wait('cd ' + S.getValue('GUEST_OVS_DPDK_DIR') +
 284                               '/DPDK/app/test-pmd')
 285         self.execute_and_wait('make clean')
 286         self.execute_and_wait('make')
 287         self.execute_and_wait('./testpmd -c 0x3 -n 4 --socket-mem 512 --'
 288                               ' --burst=64 -i --txqflags=0xf00 ' +
 289                               '--disable-hw-vlan', 60, "Done")
 290         self.execute('set fwd mac_retry', 1)
 291         self.execute_and_wait('start', 20,
 292                               'TX RS bit threshold=0 - TXQ flags=0xf00')
 293
 294     def _configure_l2fwd(self):
 295         """
 296         Configure VM to perform L2 forwarding between NICs by l2fwd module
 297         """
 298         self._configure_copy_sources('l2fwd')
 299         self._configure_disable_firewall()
 300
 301         # build and configure system for l2fwd
 302         self.execute_and_wait('cd ' + S.getValue('GUEST_OVS_DPDK_DIR') +
 303                               '/l2fwd')
 304         self.execute_and_wait('export CC=gcc')
 305
 306         self.execute_and_wait('make')
 307         self.execute_and_wait('insmod ' + S.getValue('GUEST_OVS_DPDK_DIR') +
 308                               '/l2fwd' + '/l2fwd.ko net1=' + self._net1 +
 309                               ' net2=' + self._net2)
 310
 311     def _configure_linux_bridge(self):
 312         """
 313         Configure VM to perform L2 forwarding between NICs by linux bridge
 314         """
 315         self._configure_disable_firewall()
 316         nic1_name = get_test_param('vanilla_nic1_name', self._net1)
 317         self.execute('ifconfig ' + nic1_name + ' ' +
 318                      S.getValue('VANILLA_NIC1_IP_CIDR')[self._number])
 319
 320         nic2_name = get_test_param('vanilla_nic2_name', self._net2)
 321         self.execute('ifconfig ' + nic2_name + ' ' +
 322                      S.getValue('VANILLA_NIC2_IP_CIDR')[self._number])
 323
 324         # configure linux bridge
 325         self.execute('brctl addbr br0')
 326         self.execute('brctl addif br0 ' + self._net1 + ' ' + self._net2)
 327         self.execute('ifconfig br0 ' +
 328                      S.getValue('VANILLA_BRIDGE_IP')[self._number])
 329
 330         # Add the arp entries for the IXIA ports and the bridge you are using.
 331         # Use command line values if provided.
 332         trafficgen_mac = get_test_param('vanilla_tgen_port1_mac',
 333                                         S.getValue('VANILLA_TGEN_PORT1_MAC'))
 334         trafficgen_ip = get_test_param('vanilla_tgen_port1_ip',
 335                                        S.getValue('VANILLA_TGEN_PORT1_IP'))
 336
 337         self.execute('arp -s ' + trafficgen_ip + ' ' + trafficgen_mac)
 338
 339         trafficgen_mac = get_test_param('vanilla_tgen_port2_mac',
 340                                         S.getValue('VANILLA_TGEN_PORT2_MAC'))
 341         trafficgen_ip = get_test_param('vanilla_tgen_port2_ip',
 342                                        S.getValue('VANILLA_TGEN_PORT2_IP'))
 343
 344         self.execute('arp -s ' + trafficgen_ip + ' ' + trafficgen_mac)
 345
 346         # Enable forwarding
 347         self.execute('sysctl -w net.ipv4.ip_forward=1')
 348
 349         # Controls source route verification
 350         # 0 means no source validation
 351         self.execute('sysctl -w net.ipv4.conf.all.rp_filter=0')
 352         self.execute('sysctl -w net.ipv4.conf.' + self._net1 + '.rp_filter=0')
 353         self.execute('sysctl -w net.ipv4.conf.' + self._net2 + '.rp_filter=0')