1 # Copyright (c) 2016-2018 Intel Corporation
3 # Licensed under the Apache License, Version 2.0 (the "License");
4 # you may not use this file except in compliance with the License.
5 # You may obtain a copy of the License at
7 # http://www.apache.org/licenses/LICENSE-2.0
9 # Unless required by applicable law or agreed to in writing, software
10 # distributed under the License is distributed on an "AS IS" BASIS,
11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 # See the License for the specific language governing permissions and
13 # limitations under the License.
18 from collections import defaultdict
19 from itertools import chain
21 from yardstick.common import exceptions
22 from yardstick.common.utils import validate_non_string_sequence
25 NETWORK_KERNEL = 'network_kernel'
26 NETWORK_DPDK = 'network_dpdk'
27 NETWORK_OTHER = 'network_other'
28 CRYPTO_KERNEL = 'crypto_kernel'
29 CRYPTO_DPDK = 'crypto_dpdk'
30 CRYPTO_OTHER = 'crypto_other'
32 LOG = logging.getLogger(__name__)
35 class DpdkBindHelperException(Exception):
39 class DpdkInterface(object):
40 TOPOLOGY_REQUIRED_KEYS = frozenset({
41 "vpci", "local_ip", "netmask", "local_mac", "driver"})
43 def __init__(self, dpdk_node, interface):
44 super(DpdkInterface, self).__init__()
45 self.dpdk_node = dpdk_node
46 self.interface = interface
50 except (AssertionError, KeyError):
51 raise exceptions.IncorrectConfig(error_msg='')
55 return self.interface['local_mac']
59 return self.local_mac.lower()
62 def missing_fields(self):
63 return self.TOPOLOGY_REQUIRED_KEYS.difference(self.interface)
66 def _detect_socket(netdev):
68 socket = netdev['numa_node']
70 # Where is this documented?
71 # It seems for dual-sockets systems the second socket PCI bridge
72 # will have an address > 0x0f, e.g.
73 # Bridge PCI->PCI (P#524320 busid=0000:80:02.0 id=8086:6f04
74 if netdev['pci_bus_id'][5] == "0":
77 # this doesn't handle quad-sockets
78 # TODO: fix this for quad-socket
82 def probe_missing_values(self):
84 for netdev in self.dpdk_node.netdevs.values():
85 if netdev['address'].lower() == self.mac_lower:
86 socket = self._detect_socket(netdev)
87 self.interface.update({
88 'vpci': netdev['pci_bus_id'],
89 'driver': netdev['driver'],
95 # if we don't find all the keys then don't update
98 except (exceptions.IncorrectNodeSetup, exceptions.SSHError,
99 exceptions.SSHTimeout):
100 message = ('Unable to probe missing interface fields "%s", on '
101 'node %s SSH Error' % (', '.join(self.missing_fields),
102 self.dpdk_node.node_key))
103 raise exceptions.IncorrectConfig(error_msg=message)
106 class DpdkNode(object):
108 def __init__(self, node_name, interfaces, ssh_helper, timeout=120):
109 super(DpdkNode, self).__init__()
110 self.interfaces = interfaces
111 self.ssh_helper = ssh_helper
112 self.node_key = node_name
113 self.timeout = timeout
114 self._dpdk_helper = None
118 self.dpdk_interfaces = {intf['name']: DpdkInterface(self, intf['virtual-interface'])
119 for intf in self.interfaces}
120 except exceptions.IncorrectConfig:
121 template = "MAC address is required for all interfaces, missing on: {}"
122 errors = (intf['name'] for intf in self.interfaces if
123 'local_mac' not in intf['virtual-interface'])
124 raise exceptions.IncorrectSetup(
125 error_msg=template.format(", ".join(errors)))
128 def dpdk_helper(self):
129 if not isinstance(self._dpdk_helper, DpdkBindHelper):
130 self._dpdk_helper = DpdkBindHelper(self.ssh_helper)
131 return self._dpdk_helper
134 def _interface_missing_iter(self):
135 return chain.from_iterable(self._interface_missing_map.values())
138 def _interface_missing_map(self):
139 return {name: intf.missing_fields for name, intf in self.dpdk_interfaces.items()}
141 def _probe_netdevs(self):
142 self.netdevs.update(self.dpdk_helper.find_net_devices())
144 def _force_rebind(self):
145 return self.dpdk_helper.force_dpdk_rebind()
147 def _probe_dpdk_drivers(self):
148 self.dpdk_helper.probe_real_kernel_drivers()
149 for pci, driver in self.dpdk_helper.real_kernel_interface_driver_map.items():
150 for intf in self.interfaces:
151 vintf = intf['virtual-interface']
152 # stupid substring matches
153 # don't use netdev use interface
154 if vintf['vpci'].endswith(pci):
155 vintf['driver'] = driver
156 # we can't update netdevs because we may not have netdev info
158 def _probe_missing_values(self):
159 for intf in self.dpdk_interfaces.values():
160 intf.probe_missing_values()
163 # only ssh probe if there are missing values
164 # ssh probe won't work on Ixia, so we had better define all our values
166 missing_fields_set = set(self._interface_missing_iter)
168 # if we are only missing driver then maybe we can get kernel module
170 if missing_fields_set == {'driver'}:
171 self._probe_dpdk_drivers()
172 # we can't reprobe missing values because we may not have netdev info
174 # if there are any other missing then we have to netdev probe
175 if missing_fields_set.difference({'driver'}):
176 self._probe_netdevs()
178 self._probe_missing_values()
179 except exceptions.IncorrectConfig:
183 # check again and verify we have all the fields
184 if set(self._interface_missing_iter):
185 # last chance fallback, rebind everything and probe
186 # this probably won't work
188 self._probe_netdevs()
189 self._probe_missing_values()
191 errors = ("{} missing: {}".format(name, ", ".join(missing_fields)) for
192 name, missing_fields in self._interface_missing_map.items() if
194 errors = "\n".join(errors)
196 raise exceptions.IncorrectSetup(error_msg=errors)
199 self._dpdk_helper = None
202 class DpdkBindHelper(object):
203 DPDK_STATUS_CMD = "{dpdk_devbind} --status"
204 DPDK_BIND_CMD = "sudo {dpdk_devbind} {force} -b {driver} {vpci}"
206 NIC_ROW_RE = re.compile(r"([^ ]+) '([^']+)' (?:if=([^ ]+) )?drv=([^ ]+) "
207 r"unused=([^ ]*)(?: (\*Active\*))?")
208 SKIP_RE = re.compile('(====|<none>|^$)')
209 NIC_ROW_FIELDS = ['vpci', 'dev_type', 'iface', 'driver', 'unused', 'active']
213 HEADER_DICT_PAIRS = [
214 (re.compile('^Network.*DPDK.*$'), NETWORK_DPDK),
215 (re.compile('^Network.*kernel.*$'), NETWORK_KERNEL),
216 (re.compile('^Other network.*$'), NETWORK_OTHER),
217 (re.compile('^Crypto.*DPDK.*$'), CRYPTO_DPDK),
218 (re.compile('^Crypto.*kernel$'), CRYPTO_KERNEL),
219 (re.compile('^Other crypto.*$'), CRYPTO_OTHER),
222 FIND_NETDEVICE_STRING = r"""\
223 find /sys/devices/pci* -type d -name net -exec sh -c '{ grep -sH ^ \
224 $1/ifindex $1/address $1/operstate $1/device/vendor $1/device/device \
225 $1/device/subsystem_vendor $1/device/subsystem_device $1/device/numa_node ; \
226 printf "%s/driver:" $1 ; basename $(readlink -s $1/device/driver); } \
230 BASE_ADAPTER_RE = re.compile('^/sys/devices/(.*)/net/([^/]*)/([^:]*):(.*)$', re.M)
231 DPDK_DEVBIND = "dpdk-devbind.py"
234 def parse_netdev_info(cls, stdout):
235 network_devices = defaultdict(dict)
236 match_iter = (match.groups() for match in cls.BASE_ADAPTER_RE.finditer(stdout))
237 for bus_path, interface_name, name, value in match_iter:
238 dir_name, bus_id = os.path.split(bus_path)
239 if 'virtio' in bus_id:
240 # for some stupid reason VMs include virtio1/
242 bus_id = os.path.basename(dir_name)
244 # remove extra 'device/' from 'device/vendor,
245 # device/subsystem_vendor', etc.
247 name = name.split('/')[1]
249 network_devices[interface_name].update({
251 'interface_name': interface_name,
252 'pci_bus_id': bus_id,
255 # convert back to regular dict
256 return dict(network_devices)
258 def clean_status(self):
268 # TODO: add support for driver other than igb_uio
269 def __init__(self, ssh_helper, dpdk_driver="igb_uio"):
270 self.ssh_helper = ssh_helper
271 self.real_kernel_interface_driver_map = {}
272 self.dpdk_driver = dpdk_driver
273 self.dpdk_status = None
274 self.status_nic_row_re = None
275 self.dpdk_devbind = self.ssh_helper.join_bin_path(self.DPDK_DEVBIND)
276 self._status_cmd_attr = None
277 self.used_drivers = None
278 self.real_kernel_drivers = {}
280 self.ssh_helper = ssh_helper
283 def _dpdk_execute(self, *args, **kwargs):
284 res = self.ssh_helper.execute(*args, **kwargs)
286 template = '{} command failed with rc={}'
287 raise DpdkBindHelperException(template.format(self.dpdk_devbind, res[0]))
290 def load_dpdk_driver(self):
291 cmd_template = "sudo modprobe {} && sudo modprobe {}"
292 self.ssh_helper.execute(cmd_template.format(self.UIO_DRIVER, self.dpdk_driver))
294 def check_dpdk_driver(self):
295 return self.ssh_helper.execute("lsmod | grep -i {}".format(self.dpdk_driver))[0]
298 def _status_cmd(self):
299 if self._status_cmd_attr is None:
300 self._status_cmd_attr = self.DPDK_STATUS_CMD.format(dpdk_devbind=self.dpdk_devbind)
301 return self._status_cmd_attr
303 def _add_line(self, active_list, line):
304 if active_list is None:
307 res = self.NIC_ROW_RE.match(line)
311 new_data = {k: v for k, v in zip(self.NIC_ROW_FIELDS, res.groups())}
312 new_data['active'] = bool(new_data['active'])
313 self.dpdk_status[active_list].append(new_data)
316 def _switch_active_dict(cls, a_row, active_dict):
317 for regexp, a_dict in cls.HEADER_DICT_PAIRS:
318 if regexp.match(a_row):
322 def _parse_dpdk_status_output(self, output):
325 for a_row in output.splitlines():
326 if self.SKIP_RE.match(a_row):
328 active_dict = self._switch_active_dict(a_row, active_dict)
329 self._add_line(active_dict, a_row)
330 return self.dpdk_status
332 def _get_bound_pci_addresses(self, active_dict):
333 return [iface['vpci'] for iface in self.dpdk_status[active_dict]]
336 def dpdk_bound_pci_addresses(self):
337 return self._get_bound_pci_addresses(NETWORK_DPDK)
340 def kernel_bound_pci_addresses(self):
341 return self._get_bound_pci_addresses(NETWORK_KERNEL)
344 def interface_driver_map(self):
345 return {interface['vpci']: interface['driver']
346 for interface in chain.from_iterable(self.dpdk_status.values())}
348 def read_status(self):
349 return self._parse_dpdk_status_output(self._dpdk_execute(self._status_cmd)[1])
351 def find_net_devices(self):
352 exit_status, stdout, _ = self.ssh_helper.execute(self.FIND_NETDEVICE_STRING)
356 return self.parse_netdev_info(stdout)
358 def bind(self, pci_addresses, driver, force=True):
359 # accept single PCI or sequence of PCI
360 pci_addresses = validate_non_string_sequence(pci_addresses, [pci_addresses])
362 cmd = self.DPDK_BIND_CMD.format(dpdk_devbind=self.dpdk_devbind,
364 vpci=' '.join(list(pci_addresses)),
365 force='--force' if force else '')
367 self._dpdk_execute(cmd)
369 # update the inner status dict
372 def probe_real_kernel_drivers(self):
374 self.save_real_kernel_interface_driver_map()
376 def force_dpdk_rebind(self):
377 self.load_dpdk_driver()
379 self.save_real_kernel_interface_driver_map()
380 self.save_used_drivers()
383 # only rebind devices that are bound to DPDK
384 for pci in self.dpdk_bound_pci_addresses:
386 real_driver = self.real_kernel_interface_driver_map[pci]
387 real_driver_map.setdefault(real_driver, []).append(pci)
388 for real_driver, pcis in real_driver_map.items():
389 self.bind(pcis, real_driver, force=True)
391 def save_used_drivers(self):
392 # invert the map, so we can bind by driver type
393 self.used_drivers = {}
395 for vpci, driver in sorted(self.interface_driver_map.items()):
396 self.used_drivers.setdefault(driver, []).append(vpci)
398 KERNEL_DRIVER_RE = re.compile(r"Kernel modules: (\S+)", re.M)
399 VIRTIO_DRIVER_RE = re.compile(r"Ethernet.*Virtio network device", re.M)
400 VIRTIO_DRIVER = "virtio-pci"
402 def save_real_kernel_drivers(self):
403 # invert the map, so we can bind by driver type
404 self.real_kernel_drivers = {}
406 for vpci, driver in sorted(self.real_kernel_interface_driver_map.items()):
407 self.used_drivers.setdefault(driver, []).append(vpci)
409 def get_real_kernel_driver(self, pci):
410 out = self.ssh_helper.execute('lspci -k -s %s' % pci)[1]
411 match = self.KERNEL_DRIVER_RE.search(out)
413 return match.group(1)
415 match = self.VIRTIO_DRIVER_RE.search(out)
417 return self.VIRTIO_DRIVER
421 def save_real_kernel_interface_driver_map(self):
422 iter1 = ((pci, self.get_real_kernel_driver(pci)) for pci in self.interface_driver_map)
423 self.real_kernel_interface_driver_map = {pci: driver for pci, driver in iter1 if driver}
425 def rebind_drivers(self, force=True):
426 for driver, vpcis in self.used_drivers.items():
427 self.bind(vpcis, driver, force)