yardstick/network_services/helpers/dpdkbindnic_helper.py

   1 # Copyright (c) 2016-2018 Intel Corporation
   2 #
   3 # Licensed under the Apache License, Version 2.0 (the "License");
   4 # you may not use this file except in compliance with the License.
   5 # You may obtain a copy of the License at
   6 #
   7 #      http://www.apache.org/licenses/LICENSE-2.0
   8 #
   9 # Unless required by applicable law or agreed to in writing, software
  10 # distributed under the License is distributed on an "AS IS" BASIS,
  11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12 # See the License for the specific language governing permissions and
  13 # limitations under the License.
  14 import logging
  15 import os
  16
  17 import re
  18 from collections import defaultdict
  19 from itertools import chain
  20
  21 from yardstick.common.utils import validate_non_string_sequence
  22 from yardstick.error import IncorrectConfig
  23 from yardstick.error import IncorrectSetup
  24 from yardstick.error import IncorrectNodeSetup
  25 from yardstick.error import SSHTimeout
  26 from yardstick.error import SSHError
  27
  28 NETWORK_KERNEL = 'network_kernel'
  29 NETWORK_DPDK = 'network_dpdk'
  30 NETWORK_OTHER = 'network_other'
  31 CRYPTO_KERNEL = 'crypto_kernel'
  32 CRYPTO_DPDK = 'crypto_dpdk'
  33 CRYPTO_OTHER = 'crypto_other'
  34
  35 LOG = logging.getLogger(__name__)
  36
  37
  38 class DpdkBindHelperException(Exception):
  39     pass
  40
  41
  42 class DpdkInterface(object):
  43     TOPOLOGY_REQUIRED_KEYS = frozenset({
  44         "vpci", "local_ip", "netmask", "local_mac", "driver"})
  45
  46     def __init__(self, dpdk_node, interface):
  47         super(DpdkInterface, self).__init__()
  48         self.dpdk_node = dpdk_node
  49         self.interface = interface
  50
  51         try:
  52             assert self.local_mac
  53         except (AssertionError, KeyError):
  54             raise IncorrectConfig
  55
  56     @property
  57     def local_mac(self):
  58         return self.interface['local_mac']
  59
  60     @property
  61     def mac_lower(self):
  62         return self.local_mac.lower()
  63
  64     @property
  65     def missing_fields(self):
  66         return self.TOPOLOGY_REQUIRED_KEYS.difference(self.interface)
  67
  68     @staticmethod
  69     def _detect_socket(netdev):
  70         try:
  71             socket = netdev['numa_node']
  72         except KeyError:
  73             # Where is this documented?
  74             # It seems for dual-sockets systems the second socket PCI bridge
  75             # will have an address > 0x0f, e.g.
  76             # Bridge PCI->PCI (P#524320 busid=0000:80:02.0 id=8086:6f04
  77             if netdev['pci_bus_id'][5] == "0":
  78                 socket = 0
  79             else:
  80                 # this doesn't handle quad-sockets
  81                 # TODO: fix this for quad-socket
  82                 socket = 1
  83         return socket
  84
  85     def probe_missing_values(self):
  86         try:
  87             for netdev in self.dpdk_node.netdevs.values():
  88                 if netdev['address'].lower() == self.mac_lower:
  89                     socket = self._detect_socket(netdev)
  90                     self.interface.update({
  91                         'vpci': netdev['pci_bus_id'],
  92                         'driver': netdev['driver'],
  93                         'socket': socket,
  94                         # don't need ifindex
  95                     })
  96
  97         except KeyError:
  98             # if we don't find all the keys then don't update
  99             pass
 100
 101         except (IncorrectNodeSetup, SSHError, SSHTimeout):
 102             raise IncorrectConfig(
 103                 "Unable to probe missing interface fields '%s', on node %s "
 104                 "SSH Error" % (', '.join(self.missing_fields), self.dpdk_node.node_key))
 105
 106
 107 class DpdkNode(object):
 108
 109     def __init__(self, node_name, interfaces, ssh_helper, timeout=120):
 110         super(DpdkNode, self).__init__()
 111         self.interfaces = interfaces
 112         self.ssh_helper = ssh_helper
 113         self.node_key = node_name
 114         self.timeout = timeout
 115         self._dpdk_helper = None
 116         self.netdevs = {}
 117
 118         try:
 119             self.dpdk_interfaces = {intf['name']: DpdkInterface(self, intf['virtual-interface'])
 120                                     for intf in self.interfaces}
 121         except IncorrectConfig:
 122             template = "MAC address is required for all interfaces, missing on: {}"
 123             errors = (intf['name'] for intf in self.interfaces if
 124                       'local_mac' not in intf['virtual-interface'])
 125             raise IncorrectSetup(template.format(", ".join(errors)))
 126
 127     @property
 128     def dpdk_helper(self):
 129         if not isinstance(self._dpdk_helper, DpdkBindHelper):
 130             self._dpdk_helper = DpdkBindHelper(self.ssh_helper)
 131         return self._dpdk_helper
 132
 133     @property
 134     def _interface_missing_iter(self):
 135         return chain.from_iterable(self._interface_missing_map.values())
 136
 137     @property
 138     def _interface_missing_map(self):
 139         return {name: intf.missing_fields for name, intf in self.dpdk_interfaces.items()}
 140
 141     def _probe_netdevs(self):
 142         self.netdevs.update(self.dpdk_helper.find_net_devices())
 143
 144     def _force_rebind(self):
 145         return self.dpdk_helper.force_dpdk_rebind()
 146
 147     def _probe_dpdk_drivers(self):
 148         self.dpdk_helper.probe_real_kernel_drivers()
 149         for pci, driver in self.dpdk_helper.real_kernel_interface_driver_map.items():
 150             for intf in self.interfaces:
 151                 vintf = intf['virtual-interface']
 152                 # stupid substring matches
 153                 # don't use netdev use interface
 154                 if vintf['vpci'].endswith(pci):
 155                     vintf['driver'] = driver
 156                     # we can't update netdevs because we may not have netdev info
 157
 158     def _probe_missing_values(self):
 159         for intf in self.dpdk_interfaces.values():
 160             intf.probe_missing_values()
 161
 162     def check(self):
 163         # only ssh probe if there are missing values
 164         # ssh probe won't work on Ixia, so we had better define all our values
 165         try:
 166             missing_fields_set = set(self._interface_missing_iter)
 167
 168             # if we are only missing driver then maybe we can get kernel module
 169             # this requires vpci
 170             if missing_fields_set == {'driver'}:
 171                 self._probe_dpdk_drivers()
 172                 # we can't reprobe missing values because we may not have netdev info
 173
 174             # if there are any other missing then we have to netdev probe
 175             if missing_fields_set.difference({'driver'}):
 176                 self._probe_netdevs()
 177                 try:
 178                     self._probe_missing_values()
 179                 except IncorrectConfig:
 180                     # ignore for now
 181                     pass
 182
 183                 # check again and verify we have all the fields
 184                 if set(self._interface_missing_iter):
 185                     # last chance fallback, rebind everything and probe
 186                     # this probably won't work
 187                     self._force_rebind()
 188                     self._probe_netdevs()
 189                     self._probe_missing_values()
 190
 191             errors = ("{} missing: {}".format(name, ", ".join(missing_fields)) for
 192                       name, missing_fields in self._interface_missing_map.items() if
 193                       missing_fields)
 194             errors = "\n".join(errors)
 195             if errors:
 196                 raise IncorrectSetup(errors)
 197
 198         finally:
 199             self._dpdk_helper = None
 200
 201
 202 class DpdkBindHelper(object):
 203     DPDK_STATUS_CMD = "{dpdk_devbind} --status"
 204     DPDK_BIND_CMD = "sudo {dpdk_devbind} {force} -b {driver} {vpci}"
 205
 206     NIC_ROW_RE = re.compile(r"([^ ]+) '([^']+)' (?:if=([^ ]+) )?drv=([^ ]+) "
 207                             r"unused=([^ ]*)(?: (\*Active\*))?")
 208     SKIP_RE = re.compile('(====|<none>|^$)')
 209     NIC_ROW_FIELDS = ['vpci', 'dev_type', 'iface', 'driver', 'unused', 'active']
 210
 211     UIO_DRIVER = "uio"
 212
 213     HEADER_DICT_PAIRS = [
 214         (re.compile('^Network.*DPDK.*$'), NETWORK_DPDK),
 215         (re.compile('^Network.*kernel.*$'), NETWORK_KERNEL),
 216         (re.compile('^Other network.*$'), NETWORK_OTHER),
 217         (re.compile('^Crypto.*DPDK.*$'), CRYPTO_DPDK),
 218         (re.compile('^Crypto.*kernel$'), CRYPTO_KERNEL),
 219         (re.compile('^Other crypto.*$'), CRYPTO_OTHER),
 220     ]
 221
 222     FIND_NETDEVICE_STRING = r"""\
 223 find /sys/devices/pci* -type d -name net -exec sh -c '{ grep -sH ^ \
 224 $1/ifindex $1/address $1/operstate $1/device/vendor $1/device/device \
 225 $1/device/subsystem_vendor $1/device/subsystem_device $1/device/numa_node ; \
 226 printf "%s/driver:" $1 ; basename $(readlink -s $1/device/driver); } \
 227 ' sh  \{\}/* \;
 228 """
 229
 230     BASE_ADAPTER_RE = re.compile('^/sys/devices/(.*)/net/([^/]*)/([^:]*):(.*)$', re.M)
 231     DPDK_DEVBIND = "dpdk-devbind.py"
 232
 233     @classmethod
 234     def parse_netdev_info(cls, stdout):
 235         network_devices = defaultdict(dict)
 236         match_iter = (match.groups() for match in cls.BASE_ADAPTER_RE.finditer(stdout))
 237         for bus_path, interface_name, name, value in match_iter:
 238             dir_name, bus_id = os.path.split(bus_path)
 239             if 'virtio' in bus_id:
 240                 # for some stupid reason VMs include virtio1/
 241                 # in PCI device path
 242                 bus_id = os.path.basename(dir_name)
 243
 244             # remove extra 'device/' from 'device/vendor,
 245             # device/subsystem_vendor', etc.
 246             if 'device' in name:
 247                 name = name.split('/')[1]
 248
 249             network_devices[interface_name].update({
 250                 name: value,
 251                 'interface_name': interface_name,
 252                 'pci_bus_id': bus_id,
 253             })
 254
 255         # convert back to regular dict
 256         return dict(network_devices)
 257
 258     def clean_status(self):
 259         self.dpdk_status = {
 260             NETWORK_KERNEL: [],
 261             NETWORK_DPDK: [],
 262             CRYPTO_KERNEL: [],
 263             CRYPTO_DPDK: [],
 264             NETWORK_OTHER: [],
 265             CRYPTO_OTHER: [],
 266         }
 267
 268     # TODO: add support for driver other than igb_uio
 269     def __init__(self, ssh_helper, dpdk_driver="igb_uio"):
 270         self.ssh_helper = ssh_helper
 271         self.real_kernel_interface_driver_map = {}
 272         self.dpdk_driver = dpdk_driver
 273         self.dpdk_status = None
 274         self.status_nic_row_re = None
 275         self.dpdk_devbind = self.ssh_helper.join_bin_path(self.DPDK_DEVBIND)
 276         self._status_cmd_attr = None
 277         self.used_drivers = None
 278         self.real_kernel_drivers = {}
 279
 280         self.ssh_helper = ssh_helper
 281         self.clean_status()
 282
 283     def _dpdk_execute(self, *args, **kwargs):
 284         res = self.ssh_helper.execute(*args, **kwargs)
 285         if res[0] != 0:
 286             template = '{} command failed with rc={}'
 287             raise DpdkBindHelperException(template.format(self.dpdk_devbind, res[0]))
 288         return res
 289
 290     def load_dpdk_driver(self):
 291         cmd_template = "sudo modprobe {} && sudo modprobe {}"
 292         self.ssh_helper.execute(cmd_template.format(self.UIO_DRIVER, self.dpdk_driver))
 293
 294     def check_dpdk_driver(self):
 295         return self.ssh_helper.execute("lsmod | grep -i {}".format(self.dpdk_driver))[0]
 296
 297     @property
 298     def _status_cmd(self):
 299         if self._status_cmd_attr is None:
 300             self._status_cmd_attr = self.DPDK_STATUS_CMD.format(dpdk_devbind=self.dpdk_devbind)
 301         return self._status_cmd_attr
 302
 303     def _add_line(self, active_list, line):
 304         if active_list is None:
 305             return
 306
 307         res = self.NIC_ROW_RE.match(line)
 308         if res is None:
 309             return
 310
 311         new_data = {k: v for k, v in zip(self.NIC_ROW_FIELDS, res.groups())}
 312         new_data['active'] = bool(new_data['active'])
 313         self.dpdk_status[active_list].append(new_data)
 314
 315     @classmethod
 316     def _switch_active_dict(cls, a_row, active_dict):
 317         for regexp, a_dict in cls.HEADER_DICT_PAIRS:
 318             if regexp.match(a_row):
 319                 return a_dict
 320         return active_dict
 321
 322     def _parse_dpdk_status_output(self, output):
 323         active_dict = None
 324         self.clean_status()
 325         for a_row in output.splitlines():
 326             if self.SKIP_RE.match(a_row):
 327                 continue
 328             active_dict = self._switch_active_dict(a_row, active_dict)
 329             self._add_line(active_dict, a_row)
 330         return self.dpdk_status
 331
 332     def _get_bound_pci_addresses(self, active_dict):
 333         return [iface['vpci'] for iface in self.dpdk_status[active_dict]]
 334
 335     @property
 336     def dpdk_bound_pci_addresses(self):
 337         return self._get_bound_pci_addresses(NETWORK_DPDK)
 338
 339     @property
 340     def kernel_bound_pci_addresses(self):
 341         return self._get_bound_pci_addresses(NETWORK_KERNEL)
 342
 343     @property
 344     def interface_driver_map(self):
 345         return {interface['vpci']: interface['driver']
 346                 for interface in chain.from_iterable(self.dpdk_status.values())}
 347
 348     def read_status(self):
 349         return self._parse_dpdk_status_output(self._dpdk_execute(self._status_cmd)[1])
 350
 351     def find_net_devices(self):
 352         exit_status, stdout, _ = self.ssh_helper.execute(self.FIND_NETDEVICE_STRING)
 353         if exit_status != 0:
 354             return {}
 355
 356         return self.parse_netdev_info(stdout)
 357
 358     def bind(self, pci_addresses, driver, force=True):
 359         # accept single PCI or sequence of PCI
 360         pci_addresses = validate_non_string_sequence(pci_addresses, [pci_addresses])
 361
 362         cmd = self.DPDK_BIND_CMD.format(dpdk_devbind=self.dpdk_devbind,
 363                                         driver=driver,
 364                                         vpci=' '.join(list(pci_addresses)),
 365                                         force='--force' if force else '')
 366         LOG.debug(cmd)
 367         self._dpdk_execute(cmd)
 368
 369         # update the inner status dict
 370         self.read_status()
 371
 372     def probe_real_kernel_drivers(self):
 373         self.read_status()
 374         self.save_real_kernel_interface_driver_map()
 375
 376     def force_dpdk_rebind(self):
 377         self.load_dpdk_driver()
 378         self.read_status()
 379         self.save_real_kernel_interface_driver_map()
 380         self.save_used_drivers()
 381
 382         real_driver_map = {}
 383         # only rebind devices that are bound to DPDK
 384         for pci in self.dpdk_bound_pci_addresses:
 385             # messy
 386             real_driver = self.real_kernel_interface_driver_map[pci]
 387             real_driver_map.setdefault(real_driver, []).append(pci)
 388         for real_driver, pcis in real_driver_map.items():
 389             self.bind(pcis, real_driver, force=True)
 390
 391     def save_used_drivers(self):
 392         # invert the map, so we can bind by driver type
 393         self.used_drivers = {}
 394         # sort for stability
 395         for vpci, driver in sorted(self.interface_driver_map.items()):
 396             self.used_drivers.setdefault(driver, []).append(vpci)
 397
 398     KERNEL_DRIVER_RE = re.compile(r"Kernel modules: (\S+)", re.M)
 399     VIRTIO_DRIVER_RE = re.compile(r"Ethernet.*Virtio network device", re.M)
 400     VIRTIO_DRIVER = "virtio-pci"
 401
 402     def save_real_kernel_drivers(self):
 403         # invert the map, so we can bind by driver type
 404         self.real_kernel_drivers = {}
 405         # sort for stability
 406         for vpci, driver in sorted(self.real_kernel_interface_driver_map.items()):
 407             self.used_drivers.setdefault(driver, []).append(vpci)
 408
 409     def get_real_kernel_driver(self, pci):
 410         out = self.ssh_helper.execute('lspci -k -s %s' % pci)[1]
 411         match = self.KERNEL_DRIVER_RE.search(out)
 412         if match:
 413             return match.group(1)
 414
 415         match = self.VIRTIO_DRIVER_RE.search(out)
 416         if match:
 417             return self.VIRTIO_DRIVER
 418
 419         return None
 420
 421     def save_real_kernel_interface_driver_map(self):
 422         iter1 = ((pci, self.get_real_kernel_driver(pci)) for pci in self.interface_driver_map)
 423         self.real_kernel_interface_driver_map = {pci: driver for pci, driver in iter1 if driver}
 424
 425     def rebind_drivers(self, force=True):
 426         for driver, vpcis in self.used_drivers.items():
 427             self.bind(vpcis, driver, force)