yardstick/network_services/helpers/dpdkbindnic_helper.py

   1 # Copyright (c) 2016-2018 Intel Corporation
   2 #
   3 # Licensed under the Apache License, Version 2.0 (the "License");
   4 # you may not use this file except in compliance with the License.
   5 # You may obtain a copy of the License at
   6 #
   7 #      http://www.apache.org/licenses/LICENSE-2.0
   8 #
   9 # Unless required by applicable law or agreed to in writing, software
  10 # distributed under the License is distributed on an "AS IS" BASIS,
  11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12 # See the License for the specific language governing permissions and
  13 # limitations under the License.
  14 import logging
  15 import os
  16
  17 import re
  18 from collections import defaultdict
  19 from itertools import chain
  20
  21 from yardstick.common import exceptions
  22 from yardstick.common.utils import validate_non_string_sequence
  23
  24
  25 NETWORK_KERNEL = 'network_kernel'
  26 NETWORK_DPDK = 'network_dpdk'
  27 NETWORK_OTHER = 'network_other'
  28 CRYPTO_KERNEL = 'crypto_kernel'
  29 CRYPTO_DPDK = 'crypto_dpdk'
  30 CRYPTO_OTHER = 'crypto_other'
  31
  32 LOG = logging.getLogger(__name__)
  33
  34
  35 class DpdkBindHelperException(Exception):
  36     pass
  37
  38
  39 class DpdkInterface(object):
  40     TOPOLOGY_REQUIRED_KEYS = frozenset({
  41         "vpci", "local_ip", "netmask", "local_mac", "driver"})
  42
  43     def __init__(self, dpdk_node, interface):
  44         super(DpdkInterface, self).__init__()
  45         self.dpdk_node = dpdk_node
  46         self.interface = interface
  47
  48         try:
  49             assert self.local_mac
  50         except (AssertionError, KeyError):
  51             raise exceptions.IncorrectConfig(error_msg='')
  52
  53     @property
  54     def local_mac(self):
  55         return self.interface['local_mac']
  56
  57     @property
  58     def mac_lower(self):
  59         return self.local_mac.lower()
  60
  61     @property
  62     def missing_fields(self):
  63         return self.TOPOLOGY_REQUIRED_KEYS.difference(self.interface)
  64
  65     @staticmethod
  66     def _detect_socket(netdev):
  67         try:
  68             socket = netdev['numa_node']
  69         except KeyError:
  70             # Where is this documented?
  71             # It seems for dual-sockets systems the second socket PCI bridge
  72             # will have an address > 0x0f, e.g.
  73             # Bridge PCI->PCI (P#524320 busid=0000:80:02.0 id=8086:6f04
  74             if netdev['pci_bus_id'][5] == "0":
  75                 socket = 0
  76             else:
  77                 # this doesn't handle quad-sockets
  78                 # TODO: fix this for quad-socket
  79                 socket = 1
  80         return socket
  81
  82     def probe_missing_values(self):
  83         try:
  84             for netdev in self.dpdk_node.netdevs.values():
  85                 if netdev['address'].lower() == self.mac_lower:
  86                     socket = self._detect_socket(netdev)
  87                     self.interface.update({
  88                         'vpci': netdev['pci_bus_id'],
  89                         'driver': netdev['driver'],
  90                         'socket': socket,
  91                         # don't need ifindex
  92                     })
  93
  94         except KeyError:
  95             # if we don't find all the keys then don't update
  96             pass
  97
  98         except (exceptions.IncorrectNodeSetup, exceptions.SSHError,
  99                 exceptions.SSHTimeout):
 100             message = ('Unable to probe missing interface fields "%s", on '
 101                        'node %s SSH Error' % (', '.join(self.missing_fields),
 102                                               self.dpdk_node.node_key))
 103             raise exceptions.IncorrectConfig(error_msg=message)
 104
 105
 106 class DpdkNode(object):
 107
 108     def __init__(self, node_name, interfaces, ssh_helper, timeout=120):
 109         super(DpdkNode, self).__init__()
 110         self.interfaces = interfaces
 111         self.ssh_helper = ssh_helper
 112         self.node_key = node_name
 113         self.timeout = timeout
 114         self._dpdk_helper = None
 115         self.netdevs = {}
 116
 117         try:
 118             self.dpdk_interfaces = {intf['name']: DpdkInterface(self, intf['virtual-interface'])
 119                                     for intf in self.interfaces}
 120         except exceptions.IncorrectConfig:
 121             template = "MAC address is required for all interfaces, missing on: {}"
 122             errors = (intf['name'] for intf in self.interfaces if
 123                       'local_mac' not in intf['virtual-interface'])
 124             raise exceptions.IncorrectSetup(
 125                 error_msg=template.format(", ".join(errors)))
 126
 127     @property
 128     def dpdk_helper(self):
 129         if not isinstance(self._dpdk_helper, DpdkBindHelper):
 130             self._dpdk_helper = DpdkBindHelper(self.ssh_helper)
 131         return self._dpdk_helper
 132
 133     @property
 134     def _interface_missing_iter(self):
 135         return chain.from_iterable(self._interface_missing_map.values())
 136
 137     @property
 138     def _interface_missing_map(self):
 139         return {name: intf.missing_fields for name, intf in self.dpdk_interfaces.items()}
 140
 141     def _probe_netdevs(self):
 142         self.netdevs.update(self.dpdk_helper.find_net_devices())
 143
 144     def _force_rebind(self):
 145         return self.dpdk_helper.force_dpdk_rebind()
 146
 147     def _probe_dpdk_drivers(self):
 148         self.dpdk_helper.probe_real_kernel_drivers()
 149         for pci, driver in self.dpdk_helper.real_kernel_interface_driver_map.items():
 150             for intf in self.interfaces:
 151                 vintf = intf['virtual-interface']
 152                 # stupid substring matches
 153                 # don't use netdev use interface
 154                 if vintf['vpci'].endswith(pci):
 155                     vintf['driver'] = driver
 156                     # we can't update netdevs because we may not have netdev info
 157
 158     def _probe_missing_values(self):
 159         for intf in self.dpdk_interfaces.values():
 160             intf.probe_missing_values()
 161
 162     def check(self):
 163         # only ssh probe if there are missing values
 164         # ssh probe won't work on Ixia, so we had better define all our values
 165         try:
 166             missing_fields_set = set(self._interface_missing_iter)
 167
 168             # if we are only missing driver then maybe we can get kernel module
 169             # this requires vpci
 170             if missing_fields_set == {'driver'}:
 171                 self._probe_dpdk_drivers()
 172                 # we can't reprobe missing values because we may not have netdev info
 173
 174             # if there are any other missing then we have to netdev probe
 175             if missing_fields_set.difference({'driver'}):
 176                 self._probe_netdevs()
 177                 try:
 178                     self._probe_missing_values()
 179                 except exceptions.IncorrectConfig:
 180                     # ignore for now
 181                     pass
 182
 183                 # check again and verify we have all the fields
 184                 if set(self._interface_missing_iter):
 185                     # last chance fallback, rebind everything and probe
 186                     # this probably won't work
 187                     self._force_rebind()
 188                     self._probe_netdevs()
 189                     self._probe_missing_values()
 190
 191             errors = ("{} missing: {}".format(name, ", ".join(missing_fields)) for
 192                       name, missing_fields in self._interface_missing_map.items() if
 193                       missing_fields)
 194             errors = "\n".join(errors)
 195             if errors:
 196                 raise exceptions.IncorrectSetup(error_msg=errors)
 197
 198         finally:
 199             self._dpdk_helper = None
 200
 201
 202 class DpdkBindHelper(object):
 203     DPDK_STATUS_CMD = "{dpdk_devbind} --status"
 204     DPDK_BIND_CMD = "sudo {dpdk_devbind} {force} -b {driver} {vpci}"
 205
 206     NIC_ROW_RE = re.compile(r"([^ ]+) '([^']+)' (?:if=([^ ]+) )?drv=([^ ]+) "
 207                             r"unused=([^ ]*)(?: (\*Active\*))?")
 208     SKIP_RE = re.compile('(====|<none>|^$)')
 209     NIC_ROW_FIELDS = ['vpci', 'dev_type', 'iface', 'driver', 'unused', 'active']
 210
 211     UIO_DRIVER = "uio"
 212
 213     HEADER_DICT_PAIRS = [
 214         (re.compile('^Network.*DPDK.*$'), NETWORK_DPDK),
 215         (re.compile('^Network.*kernel.*$'), NETWORK_KERNEL),
 216         (re.compile('^Other network.*$'), NETWORK_OTHER),
 217         (re.compile('^Crypto.*DPDK.*$'), CRYPTO_DPDK),
 218         (re.compile('^Crypto.*kernel$'), CRYPTO_KERNEL),
 219         (re.compile('^Other crypto.*$'), CRYPTO_OTHER),
 220     ]
 221
 222     FIND_NETDEVICE_STRING = r"""\
 223 find /sys/devices/pci* -type d -name net -exec sh -c '{ grep -sH ^ \
 224 $1/ifindex $1/address $1/operstate $1/device/vendor $1/device/device \
 225 $1/device/subsystem_vendor $1/device/subsystem_device $1/device/numa_node ; \
 226 printf "%s/driver:" $1 ; basename $(readlink -s $1/device/driver); } \
 227 ' sh  \{\}/* \;
 228 """
 229
 230     BASE_ADAPTER_RE = re.compile('^/sys/devices/(.*)/net/([^/]*)/([^:]*):(.*)$', re.M)
 231     DPDK_DEVBIND = "dpdk-devbind.py"
 232
 233     @classmethod
 234     def parse_netdev_info(cls, stdout):
 235         network_devices = defaultdict(dict)
 236         match_iter = (match.groups() for match in cls.BASE_ADAPTER_RE.finditer(stdout))
 237         for bus_path, interface_name, name, value in match_iter:
 238             dir_name, bus_id = os.path.split(bus_path)
 239             if 'virtio' in bus_id:
 240                 # for some stupid reason VMs include virtio1/
 241                 # in PCI device path
 242                 bus_id = os.path.basename(dir_name)
 243
 244             # remove extra 'device/' from 'device/vendor,
 245             # device/subsystem_vendor', etc.
 246             if 'device' in name:
 247                 name = name.split('/')[1]
 248
 249             network_devices[interface_name].update({
 250                 name: value,
 251                 'interface_name': interface_name,
 252                 'pci_bus_id': bus_id,
 253             })
 254
 255         # convert back to regular dict
 256         return dict(network_devices)
 257
 258     def clean_status(self):
 259         self.dpdk_status = {
 260             NETWORK_KERNEL: [],
 261             NETWORK_DPDK: [],
 262             CRYPTO_KERNEL: [],
 263             CRYPTO_DPDK: [],
 264             NETWORK_OTHER: [],
 265             CRYPTO_OTHER: [],
 266         }
 267
 268     # TODO: add support for driver other than igb_uio
 269     def __init__(self, ssh_helper, dpdk_driver="igb_uio"):
 270         self.ssh_helper = ssh_helper
 271         self.real_kernel_interface_driver_map = {}
 272         self.dpdk_driver = dpdk_driver
 273         self.dpdk_status = None
 274         self.status_nic_row_re = None
 275         self.dpdk_devbind = self.ssh_helper.join_bin_path(self.DPDK_DEVBIND)
 276         self._status_cmd_attr = None
 277         self.used_drivers = None
 278         self.real_kernel_drivers = {}
 279
 280         self.ssh_helper = ssh_helper
 281         self.clean_status()
 282
 283     def _dpdk_execute(self, *args, **kwargs):
 284         res = self.ssh_helper.execute(*args, **kwargs)
 285         if res[0] != 0:
 286             template = '{} command failed with rc={}'
 287             raise DpdkBindHelperException(template.format(self.dpdk_devbind, res[0]))
 288         return res
 289
 290     def load_dpdk_driver(self):
 291         cmd_template = "sudo modprobe {} && sudo modprobe {}"
 292         self.ssh_helper.execute(cmd_template.format(self.UIO_DRIVER, self.dpdk_driver))
 293
 294     def check_dpdk_driver(self):
 295         return self.ssh_helper.execute("lsmod | grep -i {}".format(self.dpdk_driver))[0]
 296
 297     @property
 298     def _status_cmd(self):
 299         if self._status_cmd_attr is None:
 300             self._status_cmd_attr = self.DPDK_STATUS_CMD.format(dpdk_devbind=self.dpdk_devbind)
 301         return self._status_cmd_attr
 302
 303     def _add_line(self, active_list, line):
 304         if active_list is None:
 305             return
 306
 307         res = self.NIC_ROW_RE.match(line)
 308         if res is None:
 309             return
 310
 311         new_data = {k: v for k, v in zip(self.NIC_ROW_FIELDS, res.groups())}
 312         new_data['active'] = bool(new_data['active'])
 313         self.dpdk_status[active_list].append(new_data)
 314
 315     @classmethod
 316     def _switch_active_dict(cls, a_row, active_dict):
 317         for regexp, a_dict in cls.HEADER_DICT_PAIRS:
 318             if regexp.match(a_row):
 319                 return a_dict
 320         return active_dict
 321
 322     def _parse_dpdk_status_output(self, output):
 323         active_dict = None
 324         self.clean_status()
 325         for a_row in output.splitlines():
 326             if self.SKIP_RE.match(a_row):
 327                 continue
 328             active_dict = self._switch_active_dict(a_row, active_dict)
 329             self._add_line(active_dict, a_row)
 330         return self.dpdk_status
 331
 332     def _get_bound_pci_addresses(self, active_dict):
 333         return [iface['vpci'] for iface in self.dpdk_status[active_dict]]
 334
 335     @property
 336     def dpdk_bound_pci_addresses(self):
 337         return self._get_bound_pci_addresses(NETWORK_DPDK)
 338
 339     @property
 340     def kernel_bound_pci_addresses(self):
 341         return self._get_bound_pci_addresses(NETWORK_KERNEL)
 342
 343     @property
 344     def interface_driver_map(self):
 345         return {interface['vpci']: interface['driver']
 346                 for interface in chain.from_iterable(self.dpdk_status.values())}
 347
 348     def read_status(self):
 349         return self._parse_dpdk_status_output(self._dpdk_execute(self._status_cmd)[1])
 350
 351     def find_net_devices(self):
 352         exit_status, stdout, _ = self.ssh_helper.execute(self.FIND_NETDEVICE_STRING)
 353         if exit_status != 0:
 354             return {}
 355
 356         return self.parse_netdev_info(stdout)
 357
 358     def bind(self, pci_addresses, driver, force=True):
 359         # accept single PCI or sequence of PCI
 360         pci_addresses = validate_non_string_sequence(pci_addresses, [pci_addresses])
 361
 362         cmd = self.DPDK_BIND_CMD.format(dpdk_devbind=self.dpdk_devbind,
 363                                         driver=driver,
 364                                         vpci=' '.join(list(pci_addresses)),
 365                                         force='--force' if force else '')
 366         LOG.debug(cmd)
 367         self._dpdk_execute(cmd)
 368
 369         # update the inner status dict
 370         self.read_status()
 371
 372     def probe_real_kernel_drivers(self):
 373         self.read_status()
 374         self.save_real_kernel_interface_driver_map()
 375
 376     def force_dpdk_rebind(self):
 377         self.load_dpdk_driver()
 378         self.read_status()
 379         self.save_real_kernel_interface_driver_map()
 380         self.save_used_drivers()
 381
 382         real_driver_map = {}
 383         # only rebind devices that are bound to DPDK
 384         for pci in self.dpdk_bound_pci_addresses:
 385             # messy
 386             real_driver = self.real_kernel_interface_driver_map[pci]
 387             real_driver_map.setdefault(real_driver, []).append(pci)
 388         for real_driver, pcis in real_driver_map.items():
 389             self.bind(pcis, real_driver, force=True)
 390
 391     def save_used_drivers(self):
 392         # invert the map, so we can bind by driver type
 393         self.used_drivers = {}
 394         # sort for stability
 395         for vpci, driver in sorted(self.interface_driver_map.items()):
 396             self.used_drivers.setdefault(driver, []).append(vpci)
 397
 398     KERNEL_DRIVER_RE = re.compile(r"Kernel modules: (\S+)", re.M)
 399     VIRTIO_DRIVER_RE = re.compile(r"Ethernet.*Virtio network device", re.M)
 400     VIRTIO_DRIVER = "virtio-pci"
 401
 402     def save_real_kernel_drivers(self):
 403         # invert the map, so we can bind by driver type
 404         self.real_kernel_drivers = {}
 405         # sort for stability
 406         for vpci, driver in sorted(self.real_kernel_interface_driver_map.items()):
 407             self.used_drivers.setdefault(driver, []).append(vpci)
 408
 409     def get_real_kernel_driver(self, pci):
 410         out = self.ssh_helper.execute('lspci -k -s %s' % pci)[1]
 411         match = self.KERNEL_DRIVER_RE.search(out)
 412         if match:
 413             return match.group(1)
 414
 415         match = self.VIRTIO_DRIVER_RE.search(out)
 416         if match:
 417             return self.VIRTIO_DRIVER
 418
 419         return None
 420
 421     def save_real_kernel_interface_driver_map(self):
 422         iter1 = ((pci, self.get_real_kernel_driver(pci)) for pci in self.interface_driver_map)
 423         self.real_kernel_interface_driver_map = {pci: driver for pci, driver in iter1 if driver}
 424
 425     def rebind_drivers(self, force=True):
 426         for driver, vpcis in self.used_drivers.items():
 427             self.bind(vpcis, driver, force)