yardstick/network_services/helpers/dpdkbindnic_helper.py

   1 # Copyright (c) 2016-2018 Intel Corporation
   2 #
   3 # Licensed under the Apache License, Version 2.0 (the "License");
   4 # you may not use this file except in compliance with the License.
   5 # You may obtain a copy of the License at
   6 #
   7 #      http://www.apache.org/licenses/LICENSE-2.0
   8 #
   9 # Unless required by applicable law or agreed to in writing, software
  10 # distributed under the License is distributed on an "AS IS" BASIS,
  11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12 # See the License for the specific language governing permissions and
  13 # limitations under the License.
  14 import logging
  15 import os
  16 import re
  17 from collections import defaultdict
  18 from itertools import chain
  19
  20 from yardstick.common import exceptions
  21 from yardstick.common.utils import validate_non_string_sequence
  22
  23 NETWORK_KERNEL = 'network_kernel'
  24 NETWORK_DPDK = 'network_dpdk'
  25 NETWORK_OTHER = 'network_other'
  26 CRYPTO_KERNEL = 'crypto_kernel'
  27 CRYPTO_DPDK = 'crypto_dpdk'
  28 CRYPTO_OTHER = 'crypto_other'
  29
  30 LOG = logging.getLogger(__name__)
  31
  32
  33 class DpdkBindHelperException(Exception):
  34     pass
  35
  36
  37 class DpdkInterface(object):
  38     TOPOLOGY_REQUIRED_KEYS = frozenset({
  39         "vpci", "local_ip", "netmask", "local_mac", "driver"})
  40
  41     def __init__(self, dpdk_node, interface):
  42         super(DpdkInterface, self).__init__()
  43         self.dpdk_node = dpdk_node
  44         self.interface = interface
  45
  46         try:
  47             assert self.local_mac
  48         except (AssertionError, KeyError):
  49             raise exceptions.IncorrectConfig(error_msg='')
  50
  51     @property
  52     def local_mac(self):
  53         return self.interface['local_mac']
  54
  55     @property
  56     def mac_lower(self):
  57         return self.local_mac.lower()
  58
  59     @property
  60     def missing_fields(self):
  61         return self.TOPOLOGY_REQUIRED_KEYS.difference(self.interface)
  62
  63     @staticmethod
  64     def _detect_socket(netdev):
  65         try:
  66             socket = netdev['numa_node']
  67         except KeyError:
  68             # Where is this documented?
  69             # It seems for dual-sockets systems the second socket PCI bridge
  70             # will have an address > 0x0f, e.g.
  71             # Bridge PCI->PCI (P#524320 busid=0000:80:02.0 id=8086:6f04
  72             if netdev['pci_bus_id'][5] == "0":
  73                 socket = 0
  74             else:
  75                 # this doesn't handle quad-sockets
  76                 # TODO: fix this for quad-socket
  77                 socket = 1
  78         return socket
  79
  80     def probe_missing_values(self):
  81         try:
  82             for netdev in self.dpdk_node.netdevs.values():
  83                 if netdev['address'].lower() == self.mac_lower:
  84                     socket = self._detect_socket(netdev)
  85                     self.interface.update({
  86                         'vpci': netdev['pci_bus_id'],
  87                         'driver': netdev['driver'],
  88                         'socket': socket,
  89                         # don't need ifindex
  90                     })
  91
  92         except KeyError:
  93             # if we don't find all the keys then don't update
  94             pass
  95
  96         except (exceptions.IncorrectNodeSetup, exceptions.SSHError,
  97                 exceptions.SSHTimeout):
  98             message = ('Unable to probe missing interface fields "%s", on '
  99                        'node %s SSH Error' % (', '.join(self.missing_fields),
 100                                               self.dpdk_node.node_key))
 101             raise exceptions.IncorrectConfig(error_msg=message)
 102
 103
 104 class DpdkNode(object):
 105
 106     def __init__(self, node_name, interfaces, ssh_helper, timeout=120):
 107         super(DpdkNode, self).__init__()
 108         self.interfaces = interfaces
 109         self.ssh_helper = ssh_helper
 110         self.node_key = node_name
 111         self.timeout = timeout
 112         self._dpdk_helper = None
 113         self.netdevs = {}
 114
 115         try:
 116             self.dpdk_interfaces = {intf['name']: DpdkInterface(self, intf['virtual-interface'])
 117                                     for intf in self.interfaces}
 118         except exceptions.IncorrectConfig:
 119             template = "MAC address is required for all interfaces, missing on: {}"
 120             errors = (intf['name'] for intf in self.interfaces if
 121                       'local_mac' not in intf['virtual-interface'])
 122             raise exceptions.IncorrectSetup(
 123                 error_msg=template.format(", ".join(errors)))
 124
 125     @property
 126     def dpdk_helper(self):
 127         if not isinstance(self._dpdk_helper, DpdkBindHelper):
 128             self._dpdk_helper = DpdkBindHelper(self.ssh_helper)
 129         return self._dpdk_helper
 130
 131     @property
 132     def _interface_missing_iter(self):
 133         return chain.from_iterable(self._interface_missing_map.values())
 134
 135     @property
 136     def _interface_missing_map(self):
 137         return {name: intf.missing_fields for name, intf in self.dpdk_interfaces.items()}
 138
 139     def _probe_netdevs(self):
 140         self.netdevs.update(self.dpdk_helper.find_net_devices())
 141
 142     def _force_rebind(self):
 143         return self.dpdk_helper.force_dpdk_rebind()
 144
 145     def _probe_dpdk_drivers(self):
 146         self.dpdk_helper.probe_real_kernel_drivers()
 147         for pci, driver in self.dpdk_helper.real_kernel_interface_driver_map.items():
 148             for intf in self.interfaces:
 149                 vintf = intf['virtual-interface']
 150                 # stupid substring matches
 151                 # don't use netdev use interface
 152                 if vintf['vpci'].endswith(pci):
 153                     vintf['driver'] = driver
 154                     # we can't update netdevs because we may not have netdev info
 155
 156     def _probe_missing_values(self):
 157         for intf in self.dpdk_interfaces.values():
 158             intf.probe_missing_values()
 159
 160     def check(self):
 161         # only ssh probe if there are missing values
 162         # ssh probe won't work on Ixia, so we had better define all our values
 163         try:
 164             missing_fields_set = set(self._interface_missing_iter)
 165
 166             # if we are only missing driver then maybe we can get kernel module
 167             # this requires vpci
 168             if missing_fields_set == {'driver'}:
 169                 self._probe_dpdk_drivers()
 170                 # we can't reprobe missing values because we may not have netdev info
 171
 172             # if there are any other missing then we have to netdev probe
 173             if missing_fields_set.difference({'driver'}):
 174                 self._probe_netdevs()
 175                 try:
 176                     self._probe_missing_values()
 177                 except exceptions.IncorrectConfig:
 178                     # ignore for now
 179                     pass
 180
 181                 # check again and verify we have all the fields
 182                 if set(self._interface_missing_iter):
 183                     # last chance fallback, rebind everything and probe
 184                     # this probably won't work
 185                     self._force_rebind()
 186                     self._probe_netdevs()
 187                     self._probe_missing_values()
 188
 189             errors = ("{} missing: {}".format(name, ", ".join(missing_fields)) for
 190                       name, missing_fields in self._interface_missing_map.items() if
 191                       missing_fields)
 192             errors = "\n".join(errors)
 193             if errors:
 194                 raise exceptions.IncorrectSetup(error_msg=errors)
 195
 196         finally:
 197             self._dpdk_helper = None
 198
 199
 200 class DpdkBindHelper(object):
 201     DPDK_STATUS_CMD = "{dpdk_devbind} --status"
 202     DPDK_BIND_CMD = "sudo {dpdk_devbind} {force} -b {driver} {vpci}"
 203
 204     NIC_ROW_RE = re.compile(r"([^ ]+) '([^']+)' (?:if=([^ ]+) )?drv=([^ ]+) "
 205                             r"unused=([^ ]*)(?: (\*Active\*))?")
 206     SKIP_RE = re.compile('(====|<none>|^$)')
 207     NIC_ROW_FIELDS = ['vpci', 'dev_type', 'iface', 'driver', 'unused', 'active']
 208
 209     UIO_DRIVER = "uio"
 210
 211     HEADER_DICT_PAIRS = [
 212         (re.compile('^Network.*DPDK.*$'), NETWORK_DPDK),
 213         (re.compile('^Network.*kernel.*$'), NETWORK_KERNEL),
 214         (re.compile('^Other network.*$'), NETWORK_OTHER),
 215         (re.compile('^Crypto.*DPDK.*$'), CRYPTO_DPDK),
 216         (re.compile('^Crypto.*kernel$'), CRYPTO_KERNEL),
 217         (re.compile('^Other crypto.*$'), CRYPTO_OTHER),
 218     ]
 219
 220     FIND_NETDEVICE_STRING = r"""\
 221 find /sys/devices/pci* -type d -name net -exec sh -c '{ grep -sH ^ \
 222 $1/ifindex $1/address $1/operstate $1/device/vendor $1/device/device \
 223 $1/device/subsystem_vendor $1/device/subsystem_device $1/device/numa_node ; \
 224 printf "%s/driver:" $1 ; basename $(readlink -s $1/device/driver); } \
 225 ' sh  \{\}/* \;
 226 """
 227
 228     BASE_ADAPTER_RE = re.compile('^/sys/devices/(.*)/net/([^/]*)/([^:]*):(.*)$', re.M)
 229     DPDK_DEVBIND = "dpdk-devbind.py"
 230
 231     @classmethod
 232     def parse_netdev_info(cls, stdout):
 233         network_devices = defaultdict(dict)
 234         match_iter = (match.groups() for match in cls.BASE_ADAPTER_RE.finditer(stdout))
 235         for bus_path, interface_name, name, value in match_iter:
 236             dir_name, bus_id = os.path.split(bus_path)
 237             if 'virtio' in bus_id:
 238                 # for some stupid reason VMs include virtio1/
 239                 # in PCI device path
 240                 bus_id = os.path.basename(dir_name)
 241
 242             # remove extra 'device/' from 'device/vendor,
 243             # device/subsystem_vendor', etc.
 244             if 'device' in name:
 245                 name = name.split('/')[1]
 246
 247             network_devices[interface_name].update({
 248                 name: value,
 249                 'interface_name': interface_name,
 250                 'pci_bus_id': bus_id,
 251             })
 252
 253         # convert back to regular dict
 254         return dict(network_devices)
 255
 256     def clean_status(self):
 257         self.dpdk_status = {
 258             NETWORK_KERNEL: [],
 259             NETWORK_DPDK: [],
 260             CRYPTO_KERNEL: [],
 261             CRYPTO_DPDK: [],
 262             NETWORK_OTHER: [],
 263             CRYPTO_OTHER: [],
 264         }
 265
 266     # TODO: add support for driver other than igb_uio
 267     def __init__(self, ssh_helper, dpdk_driver="igb_uio"):
 268         self.ssh_helper = ssh_helper
 269         self.real_kernel_interface_driver_map = {}
 270         self.dpdk_driver = dpdk_driver
 271         self.dpdk_status = None
 272         self.status_nic_row_re = None
 273         self.dpdk_devbind = self.ssh_helper.join_bin_path(self.DPDK_DEVBIND)
 274         self._status_cmd_attr = None
 275         self.used_drivers = None
 276         self.real_kernel_drivers = {}
 277
 278         self.ssh_helper = ssh_helper
 279         self.clean_status()
 280
 281     def _dpdk_execute(self, *args, **kwargs):
 282         res = self.ssh_helper.execute(*args, **kwargs)
 283         if res[0] != 0:
 284             template = '{} command failed with rc={}'
 285             LOG.critical("DPDK_DEVBIND Failure %s", res[1])
 286             raise DpdkBindHelperException(template.format(self.dpdk_devbind, res[0]))
 287         return res
 288
 289     def load_dpdk_driver(self, dpdk_driver=None):
 290         if dpdk_driver is None:
 291             dpdk_driver = self.dpdk_driver
 292         cmd_template = "sudo modprobe {} && sudo modprobe {}"
 293         self.ssh_helper.execute(
 294             cmd_template.format(self.UIO_DRIVER, dpdk_driver))
 295
 296     def check_dpdk_driver(self, dpdk_driver=None):
 297         if dpdk_driver is None:
 298             dpdk_driver = self.dpdk_driver
 299         return \
 300         self.ssh_helper.execute("lsmod | grep -i {}".format(dpdk_driver))[0]
 301
 302     @property
 303     def _status_cmd(self):
 304         if self._status_cmd_attr is None:
 305             self._status_cmd_attr = self.DPDK_STATUS_CMD.format(dpdk_devbind=self.dpdk_devbind)
 306         return self._status_cmd_attr
 307
 308     def _add_line(self, active_list, line):
 309         if active_list is None:
 310             return
 311
 312         res = self.NIC_ROW_RE.match(line)
 313         if res is None:
 314             return
 315
 316         new_data = {k: v for k, v in zip(self.NIC_ROW_FIELDS, res.groups())}
 317         new_data['active'] = bool(new_data['active'])
 318         self.dpdk_status[active_list].append(new_data)
 319
 320     @classmethod
 321     def _switch_active_dict(cls, a_row, active_dict):
 322         for regexp, a_dict in cls.HEADER_DICT_PAIRS:
 323             if regexp.match(a_row):
 324                 return a_dict
 325         return active_dict
 326
 327     def _parse_dpdk_status_output(self, output):
 328         active_dict = None
 329         self.clean_status()
 330         for a_row in output.splitlines():
 331             if self.SKIP_RE.match(a_row):
 332                 continue
 333             active_dict = self._switch_active_dict(a_row, active_dict)
 334             self._add_line(active_dict, a_row)
 335         return self.dpdk_status
 336
 337     def _get_bound_pci_addresses(self, active_dict):
 338         return [iface['vpci'] for iface in self.dpdk_status[active_dict]]
 339
 340     @property
 341     def dpdk_bound_pci_addresses(self):
 342         return self._get_bound_pci_addresses(NETWORK_DPDK)
 343
 344     @property
 345     def kernel_bound_pci_addresses(self):
 346         return self._get_bound_pci_addresses(NETWORK_KERNEL)
 347
 348     @property
 349     def interface_driver_map(self):
 350         return {interface['vpci']: interface['driver']
 351                 for interface in chain.from_iterable(self.dpdk_status.values())}
 352
 353     def read_status(self):
 354         return self._parse_dpdk_status_output(self._dpdk_execute(self._status_cmd)[1])
 355
 356     def find_net_devices(self):
 357         exit_status, stdout, _ = self.ssh_helper.execute(self.FIND_NETDEVICE_STRING)
 358         if exit_status != 0:
 359             return {}
 360
 361         return self.parse_netdev_info(stdout)
 362
 363     def bind(self, pci_addresses, driver, force=True):
 364         # accept single PCI or sequence of PCI
 365         pci_addresses = validate_non_string_sequence(pci_addresses, [pci_addresses])
 366
 367         cmd = self.DPDK_BIND_CMD.format(dpdk_devbind=self.dpdk_devbind,
 368                                         driver=driver,
 369                                         vpci=' '.join(list(pci_addresses)),
 370                                         force='--force' if force else '')
 371         LOG.debug(cmd)
 372         self._dpdk_execute(cmd)
 373
 374         # update the inner status dict
 375         self.read_status()
 376
 377     def probe_real_kernel_drivers(self):
 378         self.read_status()
 379         self.save_real_kernel_interface_driver_map()
 380
 381     def force_dpdk_rebind(self):
 382         self.load_dpdk_driver()
 383         self.read_status()
 384         self.save_real_kernel_interface_driver_map()
 385         self.save_used_drivers()
 386
 387         real_driver_map = {}
 388         # only rebind devices that are bound to DPDK
 389         for pci in self.dpdk_bound_pci_addresses:
 390             # messy
 391             real_driver = self.real_kernel_interface_driver_map[pci]
 392             real_driver_map.setdefault(real_driver, []).append(pci)
 393         for real_driver, pcis in real_driver_map.items():
 394             self.bind(pcis, real_driver, force=True)
 395
 396     def save_used_drivers(self):
 397         # invert the map, so we can bind by driver type
 398         self.used_drivers = {}
 399         # sort for stability
 400         for vpci, driver in sorted(self.interface_driver_map.items()):
 401             self.used_drivers.setdefault(driver, []).append(vpci)
 402
 403     KERNEL_DRIVER_RE = re.compile(r"Kernel modules: (\S+)", re.M)
 404     VIRTIO_DRIVER_RE = re.compile(r"Ethernet.*Virtio network device", re.M)
 405     VIRTIO_DRIVER = "virtio-pci"
 406
 407     def save_real_kernel_drivers(self):
 408         # invert the map, so we can bind by driver type
 409         self.real_kernel_drivers = {}
 410         # sort for stability
 411         for vpci, driver in sorted(self.real_kernel_interface_driver_map.items()):
 412             self.used_drivers.setdefault(driver, []).append(vpci)
 413
 414     def get_real_kernel_driver(self, pci):
 415         out = self.ssh_helper.execute('lspci -k -s %s' % pci)[1]
 416         match = self.KERNEL_DRIVER_RE.search(out)
 417         if match:
 418             return match.group(1)
 419
 420         match = self.VIRTIO_DRIVER_RE.search(out)
 421         if match:
 422             return self.VIRTIO_DRIVER
 423
 424         return None
 425
 426     def save_real_kernel_interface_driver_map(self):
 427         iter1 = ((pci, self.get_real_kernel_driver(pci)) for pci in self.interface_driver_map)
 428         self.real_kernel_interface_driver_map = {pci: driver for pci, driver in iter1 if driver}
 429
 430     def rebind_drivers(self, force=True):
 431         for driver, vpcis in self.used_drivers.items():
 432             self.bind(vpcis, driver, force)