yardstick/network_services/helpers/dpdkbindnic_helper.py

   1 # Copyright (c) 2016-2018 Intel Corporation
   2 #
   3 # Licensed under the Apache License, Version 2.0 (the "License");
   4 # you may not use this file except in compliance with the License.
   5 # You may obtain a copy of the License at
   6 #
   7 #      http://www.apache.org/licenses/LICENSE-2.0
   8 #
   9 # Unless required by applicable law or agreed to in writing, software
  10 # distributed under the License is distributed on an "AS IS" BASIS,
  11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12 # See the License for the specific language governing permissions and
  13 # limitations under the License.
  14 import logging
  15 import os
  16
  17 import re
  18 from collections import defaultdict
  19 from itertools import chain
  20
  21 from yardstick.common import exceptions
  22 from yardstick.common.utils import validate_non_string_sequence
  23 from yardstick.error import IncorrectConfig
  24 from yardstick.error import IncorrectSetup
  25 from yardstick.error import IncorrectNodeSetup
  26
  27
  28 NETWORK_KERNEL = 'network_kernel'
  29 NETWORK_DPDK = 'network_dpdk'
  30 NETWORK_OTHER = 'network_other'
  31 CRYPTO_KERNEL = 'crypto_kernel'
  32 CRYPTO_DPDK = 'crypto_dpdk'
  33 CRYPTO_OTHER = 'crypto_other'
  34
  35 LOG = logging.getLogger(__name__)
  36
  37
  38 class DpdkBindHelperException(Exception):
  39     pass
  40
  41
  42 class DpdkInterface(object):
  43     TOPOLOGY_REQUIRED_KEYS = frozenset({
  44         "vpci", "local_ip", "netmask", "local_mac", "driver"})
  45
  46     def __init__(self, dpdk_node, interface):
  47         super(DpdkInterface, self).__init__()
  48         self.dpdk_node = dpdk_node
  49         self.interface = interface
  50
  51         try:
  52             assert self.local_mac
  53         except (AssertionError, KeyError):
  54             raise IncorrectConfig
  55
  56     @property
  57     def local_mac(self):
  58         return self.interface['local_mac']
  59
  60     @property
  61     def mac_lower(self):
  62         return self.local_mac.lower()
  63
  64     @property
  65     def missing_fields(self):
  66         return self.TOPOLOGY_REQUIRED_KEYS.difference(self.interface)
  67
  68     @staticmethod
  69     def _detect_socket(netdev):
  70         try:
  71             socket = netdev['numa_node']
  72         except KeyError:
  73             # Where is this documented?
  74             # It seems for dual-sockets systems the second socket PCI bridge
  75             # will have an address > 0x0f, e.g.
  76             # Bridge PCI->PCI (P#524320 busid=0000:80:02.0 id=8086:6f04
  77             if netdev['pci_bus_id'][5] == "0":
  78                 socket = 0
  79             else:
  80                 # this doesn't handle quad-sockets
  81                 # TODO: fix this for quad-socket
  82                 socket = 1
  83         return socket
  84
  85     def probe_missing_values(self):
  86         try:
  87             for netdev in self.dpdk_node.netdevs.values():
  88                 if netdev['address'].lower() == self.mac_lower:
  89                     socket = self._detect_socket(netdev)
  90                     self.interface.update({
  91                         'vpci': netdev['pci_bus_id'],
  92                         'driver': netdev['driver'],
  93                         'socket': socket,
  94                         # don't need ifindex
  95                     })
  96
  97         except KeyError:
  98             # if we don't find all the keys then don't update
  99             pass
 100
 101         except (IncorrectNodeSetup, exceptions.SSHError,
 102                 exceptions.SSHTimeout):
 103             raise IncorrectConfig(
 104                 "Unable to probe missing interface fields '%s', on node %s "
 105                 "SSH Error" % (', '.join(self.missing_fields), self.dpdk_node.node_key))
 106
 107
 108 class DpdkNode(object):
 109
 110     def __init__(self, node_name, interfaces, ssh_helper, timeout=120):
 111         super(DpdkNode, self).__init__()
 112         self.interfaces = interfaces
 113         self.ssh_helper = ssh_helper
 114         self.node_key = node_name
 115         self.timeout = timeout
 116         self._dpdk_helper = None
 117         self.netdevs = {}
 118
 119         try:
 120             self.dpdk_interfaces = {intf['name']: DpdkInterface(self, intf['virtual-interface'])
 121                                     for intf in self.interfaces}
 122         except IncorrectConfig:
 123             template = "MAC address is required for all interfaces, missing on: {}"
 124             errors = (intf['name'] for intf in self.interfaces if
 125                       'local_mac' not in intf['virtual-interface'])
 126             raise IncorrectSetup(template.format(", ".join(errors)))
 127
 128     @property
 129     def dpdk_helper(self):
 130         if not isinstance(self._dpdk_helper, DpdkBindHelper):
 131             self._dpdk_helper = DpdkBindHelper(self.ssh_helper)
 132         return self._dpdk_helper
 133
 134     @property
 135     def _interface_missing_iter(self):
 136         return chain.from_iterable(self._interface_missing_map.values())
 137
 138     @property
 139     def _interface_missing_map(self):
 140         return {name: intf.missing_fields for name, intf in self.dpdk_interfaces.items()}
 141
 142     def _probe_netdevs(self):
 143         self.netdevs.update(self.dpdk_helper.find_net_devices())
 144
 145     def _force_rebind(self):
 146         return self.dpdk_helper.force_dpdk_rebind()
 147
 148     def _probe_dpdk_drivers(self):
 149         self.dpdk_helper.probe_real_kernel_drivers()
 150         for pci, driver in self.dpdk_helper.real_kernel_interface_driver_map.items():
 151             for intf in self.interfaces:
 152                 vintf = intf['virtual-interface']
 153                 # stupid substring matches
 154                 # don't use netdev use interface
 155                 if vintf['vpci'].endswith(pci):
 156                     vintf['driver'] = driver
 157                     # we can't update netdevs because we may not have netdev info
 158
 159     def _probe_missing_values(self):
 160         for intf in self.dpdk_interfaces.values():
 161             intf.probe_missing_values()
 162
 163     def check(self):
 164         # only ssh probe if there are missing values
 165         # ssh probe won't work on Ixia, so we had better define all our values
 166         try:
 167             missing_fields_set = set(self._interface_missing_iter)
 168
 169             # if we are only missing driver then maybe we can get kernel module
 170             # this requires vpci
 171             if missing_fields_set == {'driver'}:
 172                 self._probe_dpdk_drivers()
 173                 # we can't reprobe missing values because we may not have netdev info
 174
 175             # if there are any other missing then we have to netdev probe
 176             if missing_fields_set.difference({'driver'}):
 177                 self._probe_netdevs()
 178                 try:
 179                     self._probe_missing_values()
 180                 except IncorrectConfig:
 181                     # ignore for now
 182                     pass
 183
 184                 # check again and verify we have all the fields
 185                 if set(self._interface_missing_iter):
 186                     # last chance fallback, rebind everything and probe
 187                     # this probably won't work
 188                     self._force_rebind()
 189                     self._probe_netdevs()
 190                     self._probe_missing_values()
 191
 192             errors = ("{} missing: {}".format(name, ", ".join(missing_fields)) for
 193                       name, missing_fields in self._interface_missing_map.items() if
 194                       missing_fields)
 195             errors = "\n".join(errors)
 196             if errors:
 197                 raise IncorrectSetup(errors)
 198
 199         finally:
 200             self._dpdk_helper = None
 201
 202
 203 class DpdkBindHelper(object):
 204     DPDK_STATUS_CMD = "{dpdk_devbind} --status"
 205     DPDK_BIND_CMD = "sudo {dpdk_devbind} {force} -b {driver} {vpci}"
 206
 207     NIC_ROW_RE = re.compile(r"([^ ]+) '([^']+)' (?:if=([^ ]+) )?drv=([^ ]+) "
 208                             r"unused=([^ ]*)(?: (\*Active\*))?")
 209     SKIP_RE = re.compile('(====|<none>|^$)')
 210     NIC_ROW_FIELDS = ['vpci', 'dev_type', 'iface', 'driver', 'unused', 'active']
 211
 212     UIO_DRIVER = "uio"
 213
 214     HEADER_DICT_PAIRS = [
 215         (re.compile('^Network.*DPDK.*$'), NETWORK_DPDK),
 216         (re.compile('^Network.*kernel.*$'), NETWORK_KERNEL),
 217         (re.compile('^Other network.*$'), NETWORK_OTHER),
 218         (re.compile('^Crypto.*DPDK.*$'), CRYPTO_DPDK),
 219         (re.compile('^Crypto.*kernel$'), CRYPTO_KERNEL),
 220         (re.compile('^Other crypto.*$'), CRYPTO_OTHER),
 221     ]
 222
 223     FIND_NETDEVICE_STRING = r"""\
 224 find /sys/devices/pci* -type d -name net -exec sh -c '{ grep -sH ^ \
 225 $1/ifindex $1/address $1/operstate $1/device/vendor $1/device/device \
 226 $1/device/subsystem_vendor $1/device/subsystem_device $1/device/numa_node ; \
 227 printf "%s/driver:" $1 ; basename $(readlink -s $1/device/driver); } \
 228 ' sh  \{\}/* \;
 229 """
 230
 231     BASE_ADAPTER_RE = re.compile('^/sys/devices/(.*)/net/([^/]*)/([^:]*):(.*)$', re.M)
 232     DPDK_DEVBIND = "dpdk-devbind.py"
 233
 234     @classmethod
 235     def parse_netdev_info(cls, stdout):
 236         network_devices = defaultdict(dict)
 237         match_iter = (match.groups() for match in cls.BASE_ADAPTER_RE.finditer(stdout))
 238         for bus_path, interface_name, name, value in match_iter:
 239             dir_name, bus_id = os.path.split(bus_path)
 240             if 'virtio' in bus_id:
 241                 # for some stupid reason VMs include virtio1/
 242                 # in PCI device path
 243                 bus_id = os.path.basename(dir_name)
 244
 245             # remove extra 'device/' from 'device/vendor,
 246             # device/subsystem_vendor', etc.
 247             if 'device' in name:
 248                 name = name.split('/')[1]
 249
 250             network_devices[interface_name].update({
 251                 name: value,
 252                 'interface_name': interface_name,
 253                 'pci_bus_id': bus_id,
 254             })
 255
 256         # convert back to regular dict
 257         return dict(network_devices)
 258
 259     def clean_status(self):
 260         self.dpdk_status = {
 261             NETWORK_KERNEL: [],
 262             NETWORK_DPDK: [],
 263             CRYPTO_KERNEL: [],
 264             CRYPTO_DPDK: [],
 265             NETWORK_OTHER: [],
 266             CRYPTO_OTHER: [],
 267         }
 268
 269     # TODO: add support for driver other than igb_uio
 270     def __init__(self, ssh_helper, dpdk_driver="igb_uio"):
 271         self.ssh_helper = ssh_helper
 272         self.real_kernel_interface_driver_map = {}
 273         self.dpdk_driver = dpdk_driver
 274         self.dpdk_status = None
 275         self.status_nic_row_re = None
 276         self.dpdk_devbind = self.ssh_helper.join_bin_path(self.DPDK_DEVBIND)
 277         self._status_cmd_attr = None
 278         self.used_drivers = None
 279         self.real_kernel_drivers = {}
 280
 281         self.ssh_helper = ssh_helper
 282         self.clean_status()
 283
 284     def _dpdk_execute(self, *args, **kwargs):
 285         res = self.ssh_helper.execute(*args, **kwargs)
 286         if res[0] != 0:
 287             template = '{} command failed with rc={}'
 288             raise DpdkBindHelperException(template.format(self.dpdk_devbind, res[0]))
 289         return res
 290
 291     def load_dpdk_driver(self):
 292         cmd_template = "sudo modprobe {} && sudo modprobe {}"
 293         self.ssh_helper.execute(cmd_template.format(self.UIO_DRIVER, self.dpdk_driver))
 294
 295     def check_dpdk_driver(self):
 296         return self.ssh_helper.execute("lsmod | grep -i {}".format(self.dpdk_driver))[0]
 297
 298     @property
 299     def _status_cmd(self):
 300         if self._status_cmd_attr is None:
 301             self._status_cmd_attr = self.DPDK_STATUS_CMD.format(dpdk_devbind=self.dpdk_devbind)
 302         return self._status_cmd_attr
 303
 304     def _add_line(self, active_list, line):
 305         if active_list is None:
 306             return
 307
 308         res = self.NIC_ROW_RE.match(line)
 309         if res is None:
 310             return
 311
 312         new_data = {k: v for k, v in zip(self.NIC_ROW_FIELDS, res.groups())}
 313         new_data['active'] = bool(new_data['active'])
 314         self.dpdk_status[active_list].append(new_data)
 315
 316     @classmethod
 317     def _switch_active_dict(cls, a_row, active_dict):
 318         for regexp, a_dict in cls.HEADER_DICT_PAIRS:
 319             if regexp.match(a_row):
 320                 return a_dict
 321         return active_dict
 322
 323     def _parse_dpdk_status_output(self, output):
 324         active_dict = None
 325         self.clean_status()
 326         for a_row in output.splitlines():
 327             if self.SKIP_RE.match(a_row):
 328                 continue
 329             active_dict = self._switch_active_dict(a_row, active_dict)
 330             self._add_line(active_dict, a_row)
 331         return self.dpdk_status
 332
 333     def _get_bound_pci_addresses(self, active_dict):
 334         return [iface['vpci'] for iface in self.dpdk_status[active_dict]]
 335
 336     @property
 337     def dpdk_bound_pci_addresses(self):
 338         return self._get_bound_pci_addresses(NETWORK_DPDK)
 339
 340     @property
 341     def kernel_bound_pci_addresses(self):
 342         return self._get_bound_pci_addresses(NETWORK_KERNEL)
 343
 344     @property
 345     def interface_driver_map(self):
 346         return {interface['vpci']: interface['driver']
 347                 for interface in chain.from_iterable(self.dpdk_status.values())}
 348
 349     def read_status(self):
 350         return self._parse_dpdk_status_output(self._dpdk_execute(self._status_cmd)[1])
 351
 352     def find_net_devices(self):
 353         exit_status, stdout, _ = self.ssh_helper.execute(self.FIND_NETDEVICE_STRING)
 354         if exit_status != 0:
 355             return {}
 356
 357         return self.parse_netdev_info(stdout)
 358
 359     def bind(self, pci_addresses, driver, force=True):
 360         # accept single PCI or sequence of PCI
 361         pci_addresses = validate_non_string_sequence(pci_addresses, [pci_addresses])
 362
 363         cmd = self.DPDK_BIND_CMD.format(dpdk_devbind=self.dpdk_devbind,
 364                                         driver=driver,
 365                                         vpci=' '.join(list(pci_addresses)),
 366                                         force='--force' if force else '')
 367         LOG.debug(cmd)
 368         self._dpdk_execute(cmd)
 369
 370         # update the inner status dict
 371         self.read_status()
 372
 373     def probe_real_kernel_drivers(self):
 374         self.read_status()
 375         self.save_real_kernel_interface_driver_map()
 376
 377     def force_dpdk_rebind(self):
 378         self.load_dpdk_driver()
 379         self.read_status()
 380         self.save_real_kernel_interface_driver_map()
 381         self.save_used_drivers()
 382
 383         real_driver_map = {}
 384         # only rebind devices that are bound to DPDK
 385         for pci in self.dpdk_bound_pci_addresses:
 386             # messy
 387             real_driver = self.real_kernel_interface_driver_map[pci]
 388             real_driver_map.setdefault(real_driver, []).append(pci)
 389         for real_driver, pcis in real_driver_map.items():
 390             self.bind(pcis, real_driver, force=True)
 391
 392     def save_used_drivers(self):
 393         # invert the map, so we can bind by driver type
 394         self.used_drivers = {}
 395         # sort for stability
 396         for vpci, driver in sorted(self.interface_driver_map.items()):
 397             self.used_drivers.setdefault(driver, []).append(vpci)
 398
 399     KERNEL_DRIVER_RE = re.compile(r"Kernel modules: (\S+)", re.M)
 400     VIRTIO_DRIVER_RE = re.compile(r"Ethernet.*Virtio network device", re.M)
 401     VIRTIO_DRIVER = "virtio-pci"
 402
 403     def save_real_kernel_drivers(self):
 404         # invert the map, so we can bind by driver type
 405         self.real_kernel_drivers = {}
 406         # sort for stability
 407         for vpci, driver in sorted(self.real_kernel_interface_driver_map.items()):
 408             self.used_drivers.setdefault(driver, []).append(vpci)
 409
 410     def get_real_kernel_driver(self, pci):
 411         out = self.ssh_helper.execute('lspci -k -s %s' % pci)[1]
 412         match = self.KERNEL_DRIVER_RE.search(out)
 413         if match:
 414             return match.group(1)
 415
 416         match = self.VIRTIO_DRIVER_RE.search(out)
 417         if match:
 418             return self.VIRTIO_DRIVER
 419
 420         return None
 421
 422     def save_real_kernel_interface_driver_map(self):
 423         iter1 = ((pci, self.get_real_kernel_driver(pci)) for pci in self.interface_driver_map)
 424         self.real_kernel_interface_driver_map = {pci: driver for pci, driver in iter1 if driver}
 425
 426     def rebind_drivers(self, force=True):
 427         for driver, vpcis in self.used_drivers.items():
 428             self.bind(vpcis, driver, force)