yardstick/network_services/helpers/dpdkbindnic_helper.py

   1 # Copyright (c) 2016-2018 Intel Corporation
   2 #
   3 # Licensed under the Apache License, Version 2.0 (the "License");
   4 # you may not use this file except in compliance with the License.
   5 # You may obtain a copy of the License at
   6 #
   7 #      http://www.apache.org/licenses/LICENSE-2.0
   8 #
   9 # Unless required by applicable law or agreed to in writing, software
  10 # distributed under the License is distributed on an "AS IS" BASIS,
  11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12 # See the License for the specific language governing permissions and
  13 # limitations under the License.
  14 import logging
  15 import os
  16
  17 import re
  18 from collections import defaultdict
  19 from itertools import chain
  20
  21 from yardstick.common import exceptions
  22 from yardstick.common.utils import validate_non_string_sequence
  23
  24
  25 NETWORK_KERNEL = 'network_kernel'
  26 NETWORK_DPDK = 'network_dpdk'
  27 NETWORK_OTHER = 'network_other'
  28 CRYPTO_KERNEL = 'crypto_kernel'
  29 CRYPTO_DPDK = 'crypto_dpdk'
  30 CRYPTO_OTHER = 'crypto_other'
  31
  32 LOG = logging.getLogger(__name__)
  33
  34
  35 class DpdkBindHelperException(Exception):
  36     pass
  37
  38
  39 class DpdkInterface(object):
  40     TOPOLOGY_REQUIRED_KEYS = frozenset({
  41         "vpci", "local_ip", "netmask", "local_mac", "driver"})
  42
  43     def __init__(self, dpdk_node, interface):
  44         super(DpdkInterface, self).__init__()
  45         self.dpdk_node = dpdk_node
  46         self.interface = interface
  47
  48         try:
  49             assert self.local_mac
  50         except (AssertionError, KeyError):
  51             raise exceptions.IncorrectConfig(error_msg='')
  52
  53     @property
  54     def local_mac(self):
  55         return self.interface['local_mac']
  56
  57     @property
  58     def mac_lower(self):
  59         return self.local_mac.lower()
  60
  61     @property
  62     def missing_fields(self):
  63         return self.TOPOLOGY_REQUIRED_KEYS.difference(self.interface)
  64
  65     @staticmethod
  66     def _detect_socket(netdev):
  67         try:
  68             socket = netdev['numa_node']
  69         except KeyError:
  70             # Where is this documented?
  71             # It seems for dual-sockets systems the second socket PCI bridge
  72             # will have an address > 0x0f, e.g.
  73             # Bridge PCI->PCI (P#524320 busid=0000:80:02.0 id=8086:6f04
  74             if netdev['pci_bus_id'][5] == "0":
  75                 socket = 0
  76             else:
  77                 # this doesn't handle quad-sockets
  78                 # TODO: fix this for quad-socket
  79                 socket = 1
  80         return socket
  81
  82     def probe_missing_values(self):
  83         try:
  84             for netdev in self.dpdk_node.netdevs.values():
  85                 if netdev['address'].lower() == self.mac_lower:
  86                     socket = self._detect_socket(netdev)
  87                     self.interface.update({
  88                         'vpci': netdev['pci_bus_id'],
  89                         'driver': netdev['driver'],
  90                         'socket': socket,
  91                         # don't need ifindex
  92                     })
  93
  94         except KeyError:
  95             # if we don't find all the keys then don't update
  96             pass
  97
  98         except (exceptions.IncorrectNodeSetup, exceptions.SSHError,
  99                 exceptions.SSHTimeout):
 100             message = ('Unable to probe missing interface fields "%s", on '
 101                        'node %s SSH Error' % (', '.join(self.missing_fields),
 102                                               self.dpdk_node.node_key))
 103             raise exceptions.IncorrectConfig(error_msg=message)
 104
 105
 106 class DpdkNode(object):
 107
 108     def __init__(self, node_name, interfaces, ssh_helper, timeout=120):
 109         super(DpdkNode, self).__init__()
 110         self.interfaces = interfaces
 111         self.ssh_helper = ssh_helper
 112         self.node_key = node_name
 113         self.timeout = timeout
 114         self._dpdk_helper = None
 115         self.netdevs = {}
 116
 117         try:
 118             self.dpdk_interfaces = {intf['name']: DpdkInterface(self, intf['virtual-interface'])
 119                                     for intf in self.interfaces}
 120         except exceptions.IncorrectConfig:
 121             template = "MAC address is required for all interfaces, missing on: {}"
 122             errors = (intf['name'] for intf in self.interfaces if
 123                       'local_mac' not in intf['virtual-interface'])
 124             raise exceptions.IncorrectSetup(
 125                 error_msg=template.format(", ".join(errors)))
 126
 127     @property
 128     def dpdk_helper(self):
 129         if not isinstance(self._dpdk_helper, DpdkBindHelper):
 130             self._dpdk_helper = DpdkBindHelper(self.ssh_helper)
 131         return self._dpdk_helper
 132
 133     @property
 134     def _interface_missing_iter(self):
 135         return chain.from_iterable(self._interface_missing_map.values())
 136
 137     @property
 138     def _interface_missing_map(self):
 139         return {name: intf.missing_fields for name, intf in self.dpdk_interfaces.items()}
 140
 141     def _probe_netdevs(self):
 142         self.netdevs.update(self.dpdk_helper.find_net_devices())
 143
 144     def _force_rebind(self):
 145         return self.dpdk_helper.force_dpdk_rebind()
 146
 147     def _probe_dpdk_drivers(self):
 148         self.dpdk_helper.probe_real_kernel_drivers()
 149         for pci, driver in self.dpdk_helper.real_kernel_interface_driver_map.items():
 150             for intf in self.interfaces:
 151                 vintf = intf['virtual-interface']
 152                 # stupid substring matches
 153                 # don't use netdev use interface
 154                 if vintf['vpci'].endswith(pci):
 155                     vintf['driver'] = driver
 156                     # we can't update netdevs because we may not have netdev info
 157
 158     def _probe_missing_values(self):
 159         for intf in self.dpdk_interfaces.values():
 160             intf.probe_missing_values()
 161
 162     def check(self):
 163         # only ssh probe if there are missing values
 164         # ssh probe won't work on Ixia, so we had better define all our values
 165         try:
 166             missing_fields_set = set(self._interface_missing_iter)
 167
 168             # if we are only missing driver then maybe we can get kernel module
 169             # this requires vpci
 170             if missing_fields_set == {'driver'}:
 171                 self._probe_dpdk_drivers()
 172                 # we can't reprobe missing values because we may not have netdev info
 173
 174             # if there are any other missing then we have to netdev probe
 175             if missing_fields_set.difference({'driver'}):
 176                 self._probe_netdevs()
 177                 try:
 178                     self._probe_missing_values()
 179                 except exceptions.IncorrectConfig:
 180                     # ignore for now
 181                     pass
 182
 183                 # check again and verify we have all the fields
 184                 if set(self._interface_missing_iter):
 185                     # last chance fallback, rebind everything and probe
 186                     # this probably won't work
 187                     self._force_rebind()
 188                     self._probe_netdevs()
 189                     self._probe_missing_values()
 190
 191             errors = ("{} missing: {}".format(name, ", ".join(missing_fields)) for
 192                       name, missing_fields in self._interface_missing_map.items() if
 193                       missing_fields)
 194             errors = "\n".join(errors)
 195             if errors:
 196                 raise exceptions.IncorrectSetup(error_msg=errors)
 197
 198         finally:
 199             self._dpdk_helper = None
 200
 201
 202 class DpdkBindHelper(object):
 203     DPDK_STATUS_CMD = "{dpdk_devbind} --status"
 204     DPDK_BIND_CMD = "sudo {dpdk_devbind} {force} -b {driver} {vpci}"
 205
 206     NIC_ROW_RE = re.compile(r"([^ ]+) '([^']+)' (?:if=([^ ]+) )?drv=([^ ]+) "
 207                             r"unused=([^ ]*)(?: (\*Active\*))?")
 208     SKIP_RE = re.compile('(====|<none>|^$)')
 209     NIC_ROW_FIELDS = ['vpci', 'dev_type', 'iface', 'driver', 'unused', 'active']
 210
 211     UIO_DRIVER = "uio"
 212
 213     HEADER_DICT_PAIRS = [
 214         (re.compile('^Network.*DPDK.*$'), NETWORK_DPDK),
 215         (re.compile('^Network.*kernel.*$'), NETWORK_KERNEL),
 216         (re.compile('^Other network.*$'), NETWORK_OTHER),
 217         (re.compile('^Crypto.*DPDK.*$'), CRYPTO_DPDK),
 218         (re.compile('^Crypto.*kernel$'), CRYPTO_KERNEL),
 219         (re.compile('^Other crypto.*$'), CRYPTO_OTHER),
 220     ]
 221
 222     FIND_NETDEVICE_STRING = r"""\
 223 find /sys/devices/pci* -type d -name net -exec sh -c '{ grep -sH ^ \
 224 $1/ifindex $1/address $1/operstate $1/device/vendor $1/device/device \
 225 $1/device/subsystem_vendor $1/device/subsystem_device $1/device/numa_node ; \
 226 printf "%s/driver:" $1 ; basename $(readlink -s $1/device/driver); } \
 227 ' sh  \{\}/* \;
 228 """
 229
 230     BASE_ADAPTER_RE = re.compile('^/sys/devices/(.*)/net/([^/]*)/([^:]*):(.*)$', re.M)
 231     DPDK_DEVBIND = "dpdk-devbind.py"
 232
 233     @classmethod
 234     def parse_netdev_info(cls, stdout):
 235         network_devices = defaultdict(dict)
 236         match_iter = (match.groups() for match in cls.BASE_ADAPTER_RE.finditer(stdout))
 237         for bus_path, interface_name, name, value in match_iter:
 238             dir_name, bus_id = os.path.split(bus_path)
 239             if 'virtio' in bus_id:
 240                 # for some stupid reason VMs include virtio1/
 241                 # in PCI device path
 242                 bus_id = os.path.basename(dir_name)
 243
 244             # remove extra 'device/' from 'device/vendor,
 245             # device/subsystem_vendor', etc.
 246             if 'device' in name:
 247                 name = name.split('/')[1]
 248
 249             network_devices[interface_name].update({
 250                 name: value,
 251                 'interface_name': interface_name,
 252                 'pci_bus_id': bus_id,
 253             })
 254
 255         # convert back to regular dict
 256         return dict(network_devices)
 257
 258     def clean_status(self):
 259         self.dpdk_status = {
 260             NETWORK_KERNEL: [],
 261             NETWORK_DPDK: [],
 262             CRYPTO_KERNEL: [],
 263             CRYPTO_DPDK: [],
 264             NETWORK_OTHER: [],
 265             CRYPTO_OTHER: [],
 266         }
 267
 268     # TODO: add support for driver other than igb_uio
 269     def __init__(self, ssh_helper, dpdk_driver="igb_uio"):
 270         self.ssh_helper = ssh_helper
 271         self.real_kernel_interface_driver_map = {}
 272         self.dpdk_driver = dpdk_driver
 273         self.dpdk_status = None
 274         self.status_nic_row_re = None
 275         self.dpdk_devbind = self.ssh_helper.join_bin_path(self.DPDK_DEVBIND)
 276         self._status_cmd_attr = None
 277         self.used_drivers = None
 278         self.real_kernel_drivers = {}
 279
 280         self.ssh_helper = ssh_helper
 281         self.clean_status()
 282
 283     def _dpdk_execute(self, *args, **kwargs):
 284         res = self.ssh_helper.execute(*args, **kwargs)
 285         if res[0] != 0:
 286             template = '{} command failed with rc={}'
 287             LOG.critical("DPDK_DEVBIND Failure %s", res[1])
 288             raise DpdkBindHelperException(template.format(self.dpdk_devbind, res[0]))
 289         return res
 290
 291     def load_dpdk_driver(self):
 292         cmd_template = "sudo modprobe {} && sudo modprobe {}"
 293         self.ssh_helper.execute(cmd_template.format(self.UIO_DRIVER, self.dpdk_driver))
 294
 295     def check_dpdk_driver(self):
 296         return self.ssh_helper.execute("lsmod | grep -i {}".format(self.dpdk_driver))[0]
 297
 298     @property
 299     def _status_cmd(self):
 300         if self._status_cmd_attr is None:
 301             self._status_cmd_attr = self.DPDK_STATUS_CMD.format(dpdk_devbind=self.dpdk_devbind)
 302         return self._status_cmd_attr
 303
 304     def _add_line(self, active_list, line):
 305         if active_list is None:
 306             return
 307
 308         res = self.NIC_ROW_RE.match(line)
 309         if res is None:
 310             return
 311
 312         new_data = {k: v for k, v in zip(self.NIC_ROW_FIELDS, res.groups())}
 313         new_data['active'] = bool(new_data['active'])
 314         self.dpdk_status[active_list].append(new_data)
 315
 316     @classmethod
 317     def _switch_active_dict(cls, a_row, active_dict):
 318         for regexp, a_dict in cls.HEADER_DICT_PAIRS:
 319             if regexp.match(a_row):
 320                 return a_dict
 321         return active_dict
 322
 323     def _parse_dpdk_status_output(self, output):
 324         active_dict = None
 325         self.clean_status()
 326         for a_row in output.splitlines():
 327             if self.SKIP_RE.match(a_row):
 328                 continue
 329             active_dict = self._switch_active_dict(a_row, active_dict)
 330             self._add_line(active_dict, a_row)
 331         return self.dpdk_status
 332
 333     def _get_bound_pci_addresses(self, active_dict):
 334         return [iface['vpci'] for iface in self.dpdk_status[active_dict]]
 335
 336     @property
 337     def dpdk_bound_pci_addresses(self):
 338         return self._get_bound_pci_addresses(NETWORK_DPDK)
 339
 340     @property
 341     def kernel_bound_pci_addresses(self):
 342         return self._get_bound_pci_addresses(NETWORK_KERNEL)
 343
 344     @property
 345     def interface_driver_map(self):
 346         return {interface['vpci']: interface['driver']
 347                 for interface in chain.from_iterable(self.dpdk_status.values())}
 348
 349     def read_status(self):
 350         return self._parse_dpdk_status_output(self._dpdk_execute(self._status_cmd)[1])
 351
 352     def find_net_devices(self):
 353         exit_status, stdout, _ = self.ssh_helper.execute(self.FIND_NETDEVICE_STRING)
 354         if exit_status != 0:
 355             return {}
 356
 357         return self.parse_netdev_info(stdout)
 358
 359     def bind(self, pci_addresses, driver, force=True):
 360         # accept single PCI or sequence of PCI
 361         pci_addresses = validate_non_string_sequence(pci_addresses, [pci_addresses])
 362
 363         cmd = self.DPDK_BIND_CMD.format(dpdk_devbind=self.dpdk_devbind,
 364                                         driver=driver,
 365                                         vpci=' '.join(list(pci_addresses)),
 366                                         force='--force' if force else '')
 367         LOG.debug(cmd)
 368         self._dpdk_execute(cmd)
 369
 370         # update the inner status dict
 371         self.read_status()
 372
 373     def probe_real_kernel_drivers(self):
 374         self.read_status()
 375         self.save_real_kernel_interface_driver_map()
 376
 377     def force_dpdk_rebind(self):
 378         self.load_dpdk_driver()
 379         self.read_status()
 380         self.save_real_kernel_interface_driver_map()
 381         self.save_used_drivers()
 382
 383         real_driver_map = {}
 384         # only rebind devices that are bound to DPDK
 385         for pci in self.dpdk_bound_pci_addresses:
 386             # messy
 387             real_driver = self.real_kernel_interface_driver_map[pci]
 388             real_driver_map.setdefault(real_driver, []).append(pci)
 389         for real_driver, pcis in real_driver_map.items():
 390             self.bind(pcis, real_driver, force=True)
 391
 392     def save_used_drivers(self):
 393         # invert the map, so we can bind by driver type
 394         self.used_drivers = {}
 395         # sort for stability
 396         for vpci, driver in sorted(self.interface_driver_map.items()):
 397             self.used_drivers.setdefault(driver, []).append(vpci)
 398
 399     KERNEL_DRIVER_RE = re.compile(r"Kernel modules: (\S+)", re.M)
 400     VIRTIO_DRIVER_RE = re.compile(r"Ethernet.*Virtio network device", re.M)
 401     VIRTIO_DRIVER = "virtio-pci"
 402
 403     def save_real_kernel_drivers(self):
 404         # invert the map, so we can bind by driver type
 405         self.real_kernel_drivers = {}
 406         # sort for stability
 407         for vpci, driver in sorted(self.real_kernel_interface_driver_map.items()):
 408             self.used_drivers.setdefault(driver, []).append(vpci)
 409
 410     def get_real_kernel_driver(self, pci):
 411         out = self.ssh_helper.execute('lspci -k -s %s' % pci)[1]
 412         match = self.KERNEL_DRIVER_RE.search(out)
 413         if match:
 414             return match.group(1)
 415
 416         match = self.VIRTIO_DRIVER_RE.search(out)
 417         if match:
 418             return self.VIRTIO_DRIVER
 419
 420         return None
 421
 422     def save_real_kernel_interface_driver_map(self):
 423         iter1 = ((pci, self.get_real_kernel_driver(pci)) for pci in self.interface_driver_map)
 424         self.real_kernel_interface_driver_map = {pci: driver for pci, driver in iter1 if driver}
 425
 426     def rebind_drivers(self, force=True):
 427         for driver, vpcis in self.used_drivers.items():
 428             self.bind(vpcis, driver, force)