vnfs/qemu/qemu_dpdk_vhost_cuse.py

   1 # Copyright 2015 Intel Corporation.
   2 #
   3 # Licensed under the Apache License, Version 2.0 (the "License");
   4 # you may not use this file except in compliance with the License.
   5 # You may obtain a copy of the License at
   6 #
   7 #   http://www.apache.org/licenses/LICENSE-2.0
   8 #
   9 # Unless required by applicable law or agreed to in writing, software
  10 # distributed under the License is distributed on an "AS IS" BASIS,
  11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12 # See the License for the specific language governing permissions and
  13 # limitations under the License.
  14
  15 """Automation of QEMU hypervisor for launching vhost-cuse enabled guests.
  16 """
  17
  18 import os
  19 import time
  20 import logging
  21 import locale
  22 import re
  23 import subprocess
  24
  25 from tools import tasks
  26 from conf import settings
  27 from vnfs.vnf.vnf import IVnf
  28
  29 _QEMU_BIN = settings.getValue('QEMU_BIN')
  30 _RTE_TARGET = settings.getValue('RTE_TARGET')
  31
  32 _GUEST_MEMORY = settings.getValue('GUEST_MEMORY')
  33 _GUEST_SMP = settings.getValue('GUEST_SMP')
  34 _GUEST_CORE_BINDING = settings.getValue('GUEST_CORE_BINDING')
  35 _QEMU_CORE = settings.getValue('QEMU_CORE')
  36
  37 _GUEST_IMAGE = settings.getValue('GUEST_IMAGE')
  38 _GUEST_SHARE_DIR = settings.getValue('GUEST_SHARE_DIR')
  39
  40 _GUEST_USERNAME = settings.getValue('GUEST_USERNAME')
  41 _GUEST_PASSWORD = settings.getValue('GUEST_PASSWORD')
  42
  43 _GUEST_PROMPT_LOGIN = settings.getValue('GUEST_PROMPT_LOGIN')
  44 _GUEST_PROMPT_PASSWORD = settings.getValue('GUEST_PROMPT_PASSWORD')
  45 _GUEST_PROMPT = settings.getValue('GUEST_PROMPT')
  46
  47 _QEMU_GUEST_DPDK_PROMPT = settings.getValue('QEMU_GUEST_DPDK_PROMPT')
  48 _QEMU_GUEST_TEST_PMD_PROMPT = settings.getValue('QEMU_GUEST_TEST_PMD_PROMPT')
  49 _HUGEPAGE_DIR = settings.getValue('HUGEPAGE_DIR')
  50
  51 _GUEST_OVS_DPDK_DIR = settings.getValue('GUEST_OVS_DPDK_DIR')
  52 _OVS_DPDK_SHARE = settings.getValue('OVS_DPDK_SHARE')
  53
  54 _LOG_FILE_QEMU = os.path.join(
  55     settings.getValue('LOG_DIR'), settings.getValue('LOG_FILE_QEMU'))
  56 _LOG_FILE_GUEST_CMDS = os.path.join(
  57     settings.getValue('LOG_DIR'), settings.getValue('LOG_FILE_GUEST_CMDS'))
  58
  59 _OVS_VAR_DIR = settings.getValue('OVS_VAR_DIR')
  60 _GUEST_NET1_MAC = settings.getValue('GUEST_NET1_MAC')
  61 _GUEST_NET2_MAC = settings.getValue('GUEST_NET2_MAC')
  62 _GUEST_NET1_PCI_ADDRESS = settings.getValue('GUEST_NET1_PCI_ADDRESS')
  63 _GUEST_NET2_PCI_ADDRESS = settings.getValue('GUEST_NET2_PCI_ADDRESS')
  64
  65 class QemuDpdkVhostCuse(tasks.Process, IVnf):
  66     """
  67     Control an instance of QEMU with vHost user guest communication.
  68     """
  69     _bin = _QEMU_BIN
  70     _logfile = _LOG_FILE_QEMU
  71     _cmd = None
  72     _expect = _GUEST_PROMPT_LOGIN
  73     _proc_name = 'qemu'
  74     _number_vnfs = 0
  75
  76     class GuestCommandFilter(logging.Filter):
  77         """
  78         Filter out strings beginning with 'guestcmd :'.
  79         """
  80         def filter(self, record):
  81             return record.getMessage().startswith(self.prefix)
  82
  83     def __init__(self, memory=_GUEST_MEMORY, cpus=_GUEST_SMP,
  84                  monitor_path='/tmp', shared_path_host=_GUEST_SHARE_DIR,
  85                  args='', timeout=120, deployment="PVP"):
  86         """
  87         Initialisation function.
  88
  89         :param timeout: Time to wait for login prompt. If set to
  90             0 do not wait.
  91         :param number: Number of QEMU instance, used when multiple QEMU
  92             instances are started at once.
  93         :param args: Arguments to pass to QEMU.
  94
  95         :returns: None
  96         """
  97         self._logger = logging.getLogger(__name__)
  98         self._number = self._number_vnfs
  99         self._number_vnfs = self._number_vnfs + 1
 100         self._logfile = self._logfile + str(self._number)
 101         self._log_prefix = 'guest_%d_cmd : ' % self._number
 102         self._timeout = timeout
 103         self._monitor = '%s/vm%dmonitor' % (monitor_path, self._number)
 104
 105         name = 'Client%d' % self._number
 106         vnc = ':%d' % self._number
 107         self._cmd = ['sudo', '-E', 'taskset ' + str(_QEMU_CORE), self._bin,
 108                      '-m', str(memory), '-smp', str(cpus), '-cpu', 'host',
 109                      '-drive', 'if=scsi,file='+_GUEST_IMAGE,
 110                      '-drive',
 111                      'if=scsi,file=fat:rw:%s,snapshot=off' % shared_path_host,
 112                      '-boot', 'c', '--enable-kvm',
 113                      '-monitor', 'unix:%s,server,nowait' % self._monitor,
 114                      '-object',
 115                      'memory-backend-file,id=mem,size=' + str(memory) + 'M,' +
 116                      'mem-path=' + _HUGEPAGE_DIR + ',share=on',
 117                      '-numa', 'node,memdev=mem -mem-prealloc',
 118                      '-net', 'none', '-no-reboot',
 119                      '-netdev',
 120                      'type=tap,id=net1,script=no,downscript=no,' +
 121                      'ifname=dpdkvhostcuse0,vhost=on',
 122                      '-device',
 123                      'virtio-net-pci,netdev=net1,mac=' + _GUEST_NET1_MAC,
 124                      '-netdev',
 125                      'type=tap,id=net2,script=no,downscript=no,' +
 126                      'ifname=dpdkvhostcuse1,vhost=on',
 127                      '-device',
 128                      'virtio-net-pci,netdev=net2,mac=' + _GUEST_NET2_MAC,
 129                      '-nographic', '-vnc', str(vnc), '-name', name,
 130                      '-snapshot',
 131                     ]
 132         self._cmd.extend(args)
 133         self._configure_logging()
 134
 135     def _configure_logging(self):
 136         """
 137         Configure logging.
 138         """
 139         self.GuestCommandFilter.prefix = self._log_prefix
 140
 141         logger = logging.getLogger()
 142         cmd_logger = logging.FileHandler(
 143             filename=_LOG_FILE_GUEST_CMDS + str(self._number))
 144         cmd_logger.setLevel(logging.DEBUG)
 145         cmd_logger.addFilter(self.GuestCommandFilter())
 146         logger.addHandler(cmd_logger)
 147
 148     # startup/Shutdown
 149
 150     def start(self):
 151         """
 152         Start QEMU instance, login and prepare for commands.
 153         """
 154         super(QemuDpdkVhostCuse, self).start()
 155         self._affinitize()
 156
 157         if self._timeout:
 158             self._login()
 159             self._config_guest_loopback()
 160
 161     def stop(self):
 162         """
 163         Kill QEMU instance if it is alive.
 164         """
 165         self._logger.info('Killing QEMU...')
 166
 167         super(QemuDpdkVhostCuse, self).kill()
 168
 169     # helper functions
 170
 171     def _login(self, timeout=120):
 172         """
 173         Login to QEMU instance.
 174
 175         This can be used immediately after booting the machine, provided a
 176         sufficiently long ``timeout`` is given.
 177
 178         :param timeout: Timeout to wait for login to complete.
 179
 180         :returns: None
 181         """
 182         # if no timeout was set, we likely started QEMU without waiting for it
 183         # to boot. This being the case, we best check that it has finished
 184         # first.
 185         if not self._timeout:
 186             self._expect_process(timeout=timeout)
 187
 188         self._child.sendline(_GUEST_USERNAME)
 189         self._child.expect(_GUEST_PROMPT_PASSWORD, timeout=5)
 190         self._child.sendline(_GUEST_PASSWORD)
 191
 192         self._expect_process(_GUEST_PROMPT, timeout=5)
 193
 194     def execute(self, cmd, delay=0):
 195         """
 196         Send ``cmd`` with no wait.
 197
 198         Useful for asynchronous commands.
 199
 200         :param cmd: Command to send to guest.
 201         :param timeout: Delay to wait after sending command before returning.
 202
 203         :returns: None
 204         """
 205         self._logger.debug('%s%s', self._log_prefix, cmd)
 206         self._child.sendline(cmd)
 207         time.sleep(delay)
 208
 209     def wait(self, msg=_GUEST_PROMPT, timeout=30):
 210         """
 211         Wait for ``msg``.
 212
 213         :param msg: Message to wait for from guest.
 214         :param timeout: Time to wait for message.
 215
 216         :returns: None
 217         """
 218         self._child.expect(msg, timeout=timeout)
 219
 220     def execute_and_wait(self, cmd, timeout=30, prompt=_GUEST_PROMPT):
 221         """
 222         Send ``cmd`` and wait ``timeout`` seconds for prompt.
 223
 224         :param cmd: Command to send to guest.
 225         :param timeout: Time to wait for prompt.
 226
 227         :returns: None
 228         """
 229         self.execute(cmd)
 230         self.wait(prompt, timeout=timeout)
 231
 232     def send_and_pass(self, cmd, timeout=30):
 233         """
 234         Send ``cmd`` and wait ``timeout`` seconds for it to pass.
 235
 236         :param cmd: Command to send to guest.
 237         :param timeout: Time to wait for prompt before checking return code.
 238
 239         :returns: None
 240         """
 241         self.execute(cmd)
 242         self.wait(_GUEST_PROMPT, timeout=timeout)
 243         self.execute('echo $?')
 244         self._child.expect('^0$', timeout=1)  # expect a 0
 245         self.wait(_GUEST_PROMPT, timeout=timeout)
 246
 247     def _affinitize(self):
 248         """
 249         Affinitize the SMP cores of a QEMU instance.
 250
 251         This is a bit of a hack. The 'socat' utility is used to
 252         interact with the QEMU HMP. This is necessary due to the lack
 253         of QMP in older versions of QEMU, like v1.6.2. In future
 254         releases, this should be replaced with calls to libvirt or
 255         another Python-QEMU wrapper library.
 256
 257         :returns: None
 258         """
 259         thread_id = (r'.* CPU #%d: .* thread_id=(\d+)')
 260
 261         self._logger.info('Affinitizing guest...')
 262
 263         cur_locale = locale.getlocale()[1]
 264         proc = subprocess.Popen(
 265             ('echo', 'info cpus'), stdout=subprocess.PIPE)
 266         output = subprocess.check_output(
 267             ('sudo', 'socat', '-', 'UNIX-CONNECT:%s' % self._monitor),
 268             stdin=proc.stdout)
 269         proc.wait()
 270
 271         for cpu in range(0, int(_GUEST_SMP)):
 272             match = None
 273             for line in output.decode(cur_locale).split('\n'):
 274                 match = re.search(thread_id % cpu, line)
 275                 if match:
 276                     self._affinitize_pid(
 277                         _GUEST_CORE_BINDING[self._number - 1][cpu],
 278                         match.group(1))
 279                     break
 280
 281             if not match:
 282                 self._logger.error('Failed to affinitize guest core #%d. Could'
 283                                    ' not parse tid.', cpu)
 284
 285     def _config_guest_loopback(self):
 286         """
 287         Configure VM to run testpmd
 288
 289         Configure performs the following:
 290         * Mount hugepages
 291         * mount shared directory for copying DPDK
 292         * Disable firewall
 293         * Compile DPDK
 294         * DPDK NIC bind
 295         * Run testpmd
 296         """
 297
 298         # Guest images _should_ have 1024 hugepages by default,
 299         # but just in case:'''
 300         self.execute_and_wait('sysctl vm.nr_hugepages=1024')
 301
 302         # Mount hugepages
 303         self.execute_and_wait('mkdir -p /dev/hugepages')
 304         self.execute_and_wait(
 305             'mount -t hugetlbfs hugetlbfs /dev/hugepages')
 306
 307         # mount shared directory
 308         self.execute_and_wait('umount ' + _OVS_DPDK_SHARE)
 309         self.execute_and_wait('rm -rf ' + _GUEST_OVS_DPDK_DIR)
 310         self.execute_and_wait('mkdir -p ' + _OVS_DPDK_SHARE)
 311         self.execute_and_wait('mount -o iocharset=utf8 /dev/sdb1 ' +
 312                               _OVS_DPDK_SHARE)
 313         self.execute_and_wait('mkdir -p ' + _GUEST_OVS_DPDK_DIR)
 314         self.execute_and_wait('cp -a ' + _OVS_DPDK_SHARE + '/* ' + _GUEST_OVS_DPDK_DIR)
 315
 316         # Disable services (F16)
 317         self.execute_and_wait('systemctl status iptables.service')
 318         self.execute_and_wait('systemctl stop iptables.service')
 319
 320         # build and configure system for dpdk
 321         self.execute_and_wait('cd ' + _GUEST_OVS_DPDK_DIR + '/DPDK',
 322                               prompt=_QEMU_GUEST_DPDK_PROMPT)
 323
 324         self.execute_and_wait('export CC=gcc', prompt=_QEMU_GUEST_DPDK_PROMPT)
 325         self.execute_and_wait('export RTE_SDK=' + _GUEST_OVS_DPDK_DIR + '/DPDK',
 326                               prompt=_QEMU_GUEST_DPDK_PROMPT)
 327         self.execute_and_wait('export RTE_TARGET=%s' % _RTE_TARGET,
 328                               prompt=_QEMU_GUEST_DPDK_PROMPT)
 329
 330         self.execute_and_wait("sed -i -e 's/CONFIG_RTE_LIBRTE_VHOST_USER=n/" +
 331                               "CONFIG_RTE_LIBRTE_VHOST_USER=y/g' config/common_linuxapp",
 332                               prompt=_QEMU_GUEST_DPDK_PROMPT)
 333
 334         self.execute_and_wait('make uninstall', prompt=_QEMU_GUEST_DPDK_PROMPT)
 335         self.execute_and_wait('make install T=%s -j 2' % _RTE_TARGET,
 336                               timeout=300, prompt=_QEMU_GUEST_DPDK_PROMPT)
 337
 338         self.execute_and_wait('modprobe uio', prompt=_QEMU_GUEST_DPDK_PROMPT)
 339         self.execute_and_wait('insmod %s/kmod/igb_uio.ko' % _RTE_TARGET,
 340                               prompt=_QEMU_GUEST_DPDK_PROMPT)
 341         self.execute_and_wait('./tools/dpdk_nic_bind.py --status',
 342                               prompt=_QEMU_GUEST_DPDK_PROMPT)
 343         self.execute_and_wait('./tools/dpdk_nic_bind.py -b igb_uio'
 344                               ' ' + _GUEST_NET1_PCI_ADDRESS + ' '
 345                               + _GUEST_NET2_PCI_ADDRESS,
 346                               prompt=_QEMU_GUEST_DPDK_PROMPT)
 347
 348         self.execute_and_wait('./tools/dpdk_nic_bind.py --status',
 349                               prompt=_QEMU_GUEST_DPDK_PROMPT)
 350
 351         self.execute_and_wait('cd ' +  _RTE_TARGET + '/build/app/test-pmd',
 352                               prompt=_QEMU_GUEST_TEST_PMD_PROMPT)
 353
 354         self.execute_and_wait('./testpmd -c 0x3 -n 4 --socket-mem 512 --'
 355                               ' --burst=64 -i --txqflags=0xf00 ' +
 356                               '--disable-hw-vlan', 20, "Done")
 357         self.execute('set fwd mac_retry', 1)
 358         self.execute_and_wait('start', 20,
 359                               'TX RS bit threshold=0 - TXQ flags=0xf00')
 360
 361
 362 if __name__ == '__main__':
 363     import sys
 364
 365     with QemuDpdkVhostCuse() as vm1:
 366         print(
 367             '\n\n************************\n'
 368             'Basic command line suitable for ls, cd, grep and cat.\n If you'
 369             ' try to run Vim from here you\'re going to have a bad time.\n'
 370             'For more complex tasks please use \'vncviewer :1\' to connect to'
 371             ' this VM\nUsername: %s Password: %s\nPress ctrl-C to quit\n'
 372             '************************\n' % (_GUEST_USERNAME, _GUEST_PASSWORD))
 373
 374         if sys.argv[1]:
 375             with open(sys.argv[1], 'r') as file_:
 376                 for logline in file_:
 377                     # lines are of format:
 378                     #   guest_N_cmd : <command>
 379                     # and we only want the <command> piece
 380                     cmdline = logline.split(':')[1].strip()
 381
 382                     # use a no timeout since we don't know how long we
 383                     # should wait
 384                     vm1.send_and_wait(cmdline, timeout=-1)
 385
 386         while True:
 387             USER_INPUT = input()
 388             vm1.send_and_wait(USER_INPUT, timeout=5)