-# Copyright 2015 Intel Corporation.
+# Copyright 2015-2016 Intel Corporation.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
"""Automation of system configuration for DPDK use.
-Parts of this based on ``tools/pci_unbind.py`` script from Intel(R) DPDK.
+Parts of this based on ``tools/dpdk_nic_bind.py`` script from Intel(R)
+DPDK.
"""
from sys import platform as _platform
import os
-import re
import subprocess
import logging
-import locale
from tools import tasks
from conf import settings
-from tools.module_manager import ModuleManager, KernelModuleInsertMode
+from tools.module_manager import ModuleManager
_LOGGER = logging.getLogger(__name__)
RTE_PCI_TOOL = os.path.join(
- settings.getValue('RTE_SDK'), 'tools', 'dpdk_nic_bind.py')
+ settings.getValue('RTE_SDK_USER'), 'tools', 'dpdk_nic_bind.py')
-_DPDK_MODULE_MANAGER = ModuleManager(KernelModuleInsertMode.MODPROBE)
+_DPDK_MODULE_MANAGER = ModuleManager()
+
+# declare global NIC variables only as their content might not be known yet
+_NICS = []
+_NICS_PCI = []
#
# system management
#
-
def init():
"""Setup system for DPDK.
"""
+ global _NICS
+ global _NICS_PCI
+ _NICS = settings.getValue('NICS')
+ _NICS_PCI = list(nic['pci'] for nic in _NICS)
if not _is_linux():
_LOGGER.error('Not running on a compatible Linux version. Exiting...')
return
-
- _mount_hugepages()
_insert_modules()
_remove_vhost_net()
_bind_nics()
- _copy_dpdk_for_guest()
def cleanup():
_unbind_nics()
_remove_modules()
- _umount_hugepages()
_vhost_user_cleanup()
-
-#
-# vhost specific modules management
-#
-
-
-def insert_vhost_modules():
- """Inserts VHOST related kernel modules
- """
- mod_path_prefix = os.path.join(settings.getValue('RTE_SDK'),
- 'lib',
- 'librte_vhost')
- _insert_module_group('VHOST_MODULE', mod_path_prefix)
-
-
-def remove_vhost_modules():
- """Removes all VHOST related kernel modules
- """
- _remove_module_group('VHOST_MODULE')
-
#
# basic compatibility test
#
-
def _is_linux():
"""Check if running on Linux.
"""
return _platform.startswith('linux') and os.path.isdir('/proc')
-#
-# hugepage management
-#
-
-
-def _is_hugepage_available():
- """Check if hugepages are available on the system.
- """
- hugepage_re = re.compile(r'^HugePages_Free:\s+(?P<num_hp>\d+)$')
-
- # read in meminfo
- with open('/proc/meminfo') as mem_file:
- mem_info = mem_file.readlines()
-
- # first check if module is loaded
- for line in mem_info:
- result = hugepage_re.match(line)
- if not result:
- continue
-
- num_huge = result.group('num_hp')
- if not num_huge:
- _LOGGER.info('No free hugepages.')
- else:
- _LOGGER.info('Found \'%s\' free hugepage(s).', num_huge)
- return True
-
- return False
-
-
-def _is_hugepage_mounted():
- """Check if hugepages are mounted.
- """
- output = subprocess.check_output(['mount'], shell=True)
- my_encoding = locale.getdefaultlocale()[1]
- for line in output.decode(my_encoding).split('\n'):
- if 'hugetlbfs' in line:
- return True
-
- return False
-
-
-def _mount_hugepages():
- """Ensure hugepages are mounted.
- """
- if not _is_hugepage_available():
- return
-
- if _is_hugepage_mounted():
- return
-
- if not os.path.exists(settings.getValue('HUGEPAGE_DIR')):
- os.makedirs(settings.getValue('HUGEPAGE_DIR'))
- try:
- tasks.run_task(['sudo', 'mount', '-t', 'hugetlbfs', 'nodev',
- settings.getValue('HUGEPAGE_DIR')],
- _LOGGER, 'Mounting hugepages...', True)
- except subprocess.CalledProcessError:
- _LOGGER.error('Unable to mount hugepages.')
-
-
-def _umount_hugepages():
- """Ensure hugepages are unmounted.
- """
- if not _is_hugepage_mounted():
- return
-
- try:
- tasks.run_task(['sudo', 'umount', settings.getValue('HUGEPAGE_DIR')],
- _LOGGER, 'Unmounting hugepages...', True)
- except subprocess.CalledProcessError:
- _LOGGER.error('Unable to umount hugepages.')
-
#
# module management
#
-
-def _is_module_inserted(module):
- """Check if a module is inserted on system.
- """
- with open('/proc/modules') as mod_file:
- loaded_mods = mod_file.readlines()
-
- # first check if module is loaded
- for line in loaded_mods:
- if line.startswith(module):
- return True
- return False
-
-
def _insert_modules():
"""Ensure required modules are inserted on system.
"""
_DPDK_MODULE_MANAGER.insert_modules(settings.getValue('SYS_MODULES'))
mod_path_prefix = settings.getValue('OVS_DIR')
- _insert_module_group('OVS_MODULES', mod_path_prefix)
- mod_path_prefix = os.path.join(settings.getValue('RTE_SDK'),
- settings.getValue('RTE_TARGET'))
- _insert_module_group('DPDK_MODULES', mod_path_prefix)
-
-
-def _insert_module_group(module_group, group_path_prefix):
- """Ensure all modules in a group are inserted into the system.
-
- :param module_group: A name of configuration item containing a list
- of module names
- """
- for module in settings.getValue(module_group):
- # first check if module is loaded
- if _is_module_inserted(module[1]):
- continue
-
- try:
- mod_path = os.path.join(group_path_prefix, module[0],
- '%s.ko' % module[1])
- tasks.run_task(['sudo', 'insmod', mod_path], _LOGGER,
- 'Inserting module \'%s\'...' % module[1], True)
- except subprocess.CalledProcessError:
- _LOGGER.error('Unable to insert module \'%s\'.', module[1])
- raise # fail catastrophically
-
+ _DPDK_MODULE_MANAGER.insert_module_group(settings.getValue('OVS_MODULES'),
+ mod_path_prefix)
+ if 'vfio-pci' not in settings.getValue('DPDK_MODULES'):
+ mod_path_prefix = os.path.join(settings.getValue('RTE_SDK'),
+ settings.getValue('RTE_TARGET'))
+ _DPDK_MODULE_MANAGER.insert_module_group(settings.getValue('DPDK_MODULES'),
+ mod_path_prefix)
+ else:
+ _DPDK_MODULE_MANAGER.insert_modules(settings.getValue('DPDK_MODULES'))
def _remove_modules():
"""Ensure required modules are removed from system.
"""
- _remove_module_group('OVS_MODULES')
- _remove_module_group('DPDK_MODULES')
-
_DPDK_MODULE_MANAGER.remove_modules()
-def _remove_module_group(module_group):
- """Ensure all modules in a group are removed from the system.
+#
+# vhost specific modules management
+#
- :param module_group: A name of configuration item containing a list
- of module names
+def insert_vhost_modules():
+ """Inserts VHOST related kernel modules
"""
- for module in settings.getValue(module_group):
- # first check if module is loaded
- if not _is_module_inserted(module[1]):
- continue
+ mod_path_prefix = os.path.join(settings.getValue('RTE_SDK'),
+ 'lib',
+ 'librte_vhost')
+ _DPDK_MODULE_MANAGER.insert_module_group(settings.getValue('VHOST_MODULE'), mod_path_prefix)
- try:
- tasks.run_task(['sudo', 'rmmod', module[1]], _LOGGER,
- 'Removing module \'%s\'...' % module[1], True)
- except subprocess.CalledProcessError:
- _LOGGER.error('Unable to remove module \'%s\'.', module[1])
- continue
+
+def remove_vhost_modules():
+ """Removes all VHOST related kernel modules
+ """
+ # all modules are removed automatically by _remove_modules() method
+ pass
#
-# 'vhost-net' module management
+# 'vhost-net' module cleanup
#
def _remove_vhost_net():
"""Remove vhost-net driver and file.
"""
- if _is_module_inserted('vhost_net'):
- try:
- tasks.run_task(['sudo', 'rmmod', 'vhost_net'], _LOGGER,
- 'Removing \'/dev/vhost-net\' directory...', True)
- except subprocess.CalledProcessError:
- _LOGGER.error('Unable to remove module \'vhost_net\'.')
-
+ _DPDK_MODULE_MANAGER.remove_module('vhost-net')
try:
tasks.run_task(['sudo', 'rm', '-f', '/dev/vhost-net'], _LOGGER,
'Removing \'/dev/vhost-net\' directory...', True)
except subprocess.CalledProcessError:
_LOGGER.error('Unable to remove directory \'/dev/vhost-net\'.')
-#
-# NIC management
-#
-
-
-def _bind_nics():
- """Bind NICs using the Intel DPDK ``pci_unbind.py`` tool.
- """
- try:
- tasks.run_task(['sudo', RTE_PCI_TOOL, '--bind', 'igb_uio'] +
- settings.getValue('WHITELIST_NICS'), _LOGGER,
- 'Binding NICs %s...' %
- settings.getValue('WHITELIST_NICS'),
- True)
- except subprocess.CalledProcessError:
- _LOGGER.error('Unable to bind NICs %s',
- str(settings.getValue('WHITELIST_NICS')))
-
-
-def _unbind_nics():
- """Unbind NICs using the Intel DPDK ``pci_unbind.py`` tool.
- """
- try:
- tasks.run_task(['sudo', RTE_PCI_TOOL, '--unbind'] +
- settings.getValue('WHITELIST_NICS'), _LOGGER,
- 'Unbinding NICs %s...' %
- str(settings.getValue('WHITELIST_NICS')),
- True)
- except subprocess.CalledProcessError:
- _LOGGER.error('Unable to unbind NICs %s',
- str(settings.getValue('WHITELIST_NICS')))
-
-
-def _copy_dpdk_for_guest():
- """Copy dpdk code to GUEST_SHARE_DIR for use by guests.
- """
- guest_share_dir = os.path.join(
- settings.getValue('GUEST_SHARE_DIR'), 'DPDK')
-
- if not os.path.exists(guest_share_dir):
- os.makedirs(guest_share_dir)
-
- try:
- tasks.run_task(['rsync', '-a', '-r', '-l', r'--exclude="\.git"',
- os.path.join(settings.getValue('RTE_SDK'), ''),
- guest_share_dir],
- _LOGGER,
- 'Copying DPDK to shared directory...',
- True)
- except subprocess.CalledProcessError:
- _LOGGER.error('Unable to copy DPDK to shared directory')
-
-
#
# Vhost-user cleanup
#
_LOGGER.error('Unable to delete vhost-user socket \'%s\'.',
sock)
continue
+#
+# NIC management
+#
+def _bind_nics():
+ """Bind NICs using the Intel DPDK ``dpdk_nic_bind.py`` tool.
+ """
+ try:
+ _driver = 'igb_uio'
+ if 'vfio-pci' in settings.getValue('DPDK_MODULES'):
+ _driver = 'vfio-pci'
+ tasks.run_task(['sudo', 'chmod', 'a+x', '/dev/vfio'],
+ _LOGGER, 'Setting VFIO permissions .. a+x',
+ True)
+ tasks.run_task(['sudo', 'chmod', '-R', '666', '/dev/vfio/'],
+ _LOGGER, 'Setting VFIO permissions .. 0666',
+ True)
+
+ tasks.run_task(['sudo', RTE_PCI_TOOL, '--bind=' + _driver] +
+ _NICS_PCI, _LOGGER,
+ 'Binding NICs %s...' % _NICS_PCI,
+ True)
+ except subprocess.CalledProcessError:
+ _LOGGER.error('Unable to bind NICs %s', str(_NICS_PCI))
+
+def _unbind_nics():
+ """Unbind NICs using the Intel DPDK ``dpdk_nic_bind.py`` tool.
+ """
+ try:
+ tasks.run_task(['sudo', RTE_PCI_TOOL, '--unbind'] +
+ _NICS_PCI, _LOGGER,
+ 'Unbinding NICs %s...' % str(_NICS_PCI),
+ True)
+ except subprocess.CalledProcessError:
+ _LOGGER.error('Unable to unbind NICs %s', str(_NICS_PCI))
+ # Rebind NICs to their original drivers
+ # using the Intel DPDK ``dpdk_nic_bind.py`` tool.
+ for nic in _NICS:
+ try:
+ if nic['driver']:
+ tasks.run_task(['sudo', RTE_PCI_TOOL, '--bind',
+ nic['driver'], nic['pci']],
+ _LOGGER, 'Binding NIC %s to %s...' %
+ (nic['pci'], nic['driver']),
+ True)
+ except subprocess.CalledProcessError:
+ _LOGGER.error('Unable to bind NIC %s to driver %s',
+ nic['pci'], nic['driver'])
+
class Dpdk(object):
"""A context manager for the system init/cleanup.
"""
Code Review / vswitchperf.git / blobdiff