yardstick/benchmark/contexts/standalone/ovs_dpdk.py

   1 # Copyright (c) 2016-2017 Intel Corporation
   2 #
   3 # Licensed under the Apache License, Version 2.0 (the "License");
   4 # you may not use this file except in compliance with the License.
   5 # You may obtain a copy of the License at
   6 #
   7 #      http://www.apache.org/licenses/LICENSE-2.0
   8 #
   9 # Unless required by applicable law or agreed to in writing, software
  10 # distributed under the License is distributed on an "AS IS" BASIS,
  11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12 # See the License for the specific language governing permissions and
  13 # limitations under the License.
  14
  15 import io
  16 import collections
  17 import logging
  18 import os
  19 import re
  20 import time
  21
  22 from yardstick import ssh
  23 from yardstick.benchmark.contexts.base import Context
  24 from yardstick.benchmark.contexts.standalone import model
  25 from yardstick.common import exceptions
  26 from yardstick.network_services import utils
  27
  28
  29 LOG = logging.getLogger(__name__)
  30
  31 MAIN_BRIDGE = 'br0'
  32
  33
  34 class OvsDpdkContext(Context):
  35     """ This class handles OVS standalone nodes - VM running on Non-Managed NFVi
  36     Configuration: ovs_dpdk
  37     """
  38
  39     __context_type__ = "StandaloneOvsDpdk"
  40
  41     SUPPORTED_OVS_TO_DPDK_MAP = {
  42         '2.6.0': '16.07.1',
  43         '2.6.1': '16.07.2',
  44         '2.7.0': '16.11.1',
  45         '2.7.1': '16.11.2',
  46         '2.7.2': '16.11.3',
  47         '2.8.0': '17.05.2'
  48     }
  49
  50     DEFAULT_OVS = '2.6.0'
  51     PKILL_TEMPLATE = "pkill %s %s"
  52     CMD_TIMEOUT = 30
  53
  54     def __init__(self):
  55         self.file_path = None
  56         self.sriov = []
  57         self.first_run = True
  58         self.dpdk_devbind = ''
  59         self.vm_names = []
  60         self.nfvi_host = []
  61         self.nodes = []
  62         self.networks = {}
  63         self.attrs = {}
  64         self.vm_flavor = None
  65         self.servers = None
  66         self.helper = model.StandaloneContextHelper()
  67         self.vnf_node = model.Server()
  68         self.ovs_properties = {}
  69         self.wait_for_vswitchd = 10
  70         super(OvsDpdkContext, self).__init__()
  71
  72     def init(self, attrs):
  73         """initializes itself from the supplied arguments"""
  74         super(OvsDpdkContext, self).init(attrs)
  75
  76         self.file_path = attrs.get("file", "pod.yaml")
  77
  78         self.nodes, self.nfvi_host, self.host_mgmt = \
  79             self.helper.parse_pod_file(self.file_path, 'OvsDpdk')
  80
  81         self.attrs = attrs
  82         self.vm_flavor = attrs.get('flavor', {})
  83         self.servers = attrs.get('servers', {})
  84         self.vm_deploy = attrs.get("vm_deploy", True)
  85         self.ovs_properties = attrs.get('ovs_properties', {})
  86         # add optional static network definition
  87         self.networks = attrs.get("networks", {})
  88
  89         LOG.debug("Nodes: %r", self.nodes)
  90         LOG.debug("NFVi Node: %r", self.nfvi_host)
  91         LOG.debug("Networks: %r", self.networks)
  92
  93     def setup_ovs(self):
  94         vpath = self.ovs_properties.get("vpath", "/usr/local")
  95         xargs_kill_cmd = self.PKILL_TEMPLATE % ('-9', 'ovs')
  96
  97         create_from = os.path.join(vpath, 'etc/openvswitch/conf.db')
  98         create_to = os.path.join(vpath, 'share/openvswitch/vswitch.ovsschema')
  99
 100         cmd_list = [
 101             "chmod 0666 /dev/vfio/*",
 102             "chmod a+x /dev/vfio",
 103             "pkill -9 ovs",
 104             xargs_kill_cmd,
 105             "killall -r 'ovs*'",
 106             "mkdir -p {0}/etc/openvswitch".format(vpath),
 107             "mkdir -p {0}/var/run/openvswitch".format(vpath),
 108             "rm {0}/etc/openvswitch/conf.db".format(vpath),
 109             "ovsdb-tool create {0} {1}".format(create_from, create_to),
 110             "modprobe vfio-pci",
 111             "chmod a+x /dev/vfio",
 112             "chmod 0666 /dev/vfio/*",
 113         ]
 114         for cmd in cmd_list:
 115             self.connection.execute(cmd)
 116         bind_cmd = "{dpdk_devbind} --force -b {driver} {port}"
 117         phy_driver = "vfio-pci"
 118         for port in self.networks.values():
 119             vpci = port.get("phy_port")
 120             self.connection.execute(bind_cmd.format(
 121                 dpdk_devbind=self.dpdk_devbind, driver=phy_driver, port=vpci))
 122
 123     def start_ovs_serverswitch(self):
 124         vpath = self.ovs_properties.get("vpath")
 125         pmd_nums = int(self.ovs_properties.get("pmd_threads", 2))
 126         ovs_sock_path = '/var/run/openvswitch/db.sock'
 127         log_path = '/var/log/openvswitch/ovs-vswitchd.log'
 128
 129         pmd_cpu_mask = self.ovs_properties.get("pmd_cpu_mask", '')
 130         pmd_mask = hex(sum(2 ** num for num in range(pmd_nums)) << 1)
 131         if pmd_cpu_mask:
 132             pmd_mask = pmd_cpu_mask
 133
 134         socket0 = self.ovs_properties.get("ram", {}).get("socket_0", "2048")
 135         socket1 = self.ovs_properties.get("ram", {}).get("socket_1", "2048")
 136
 137         ovs_other_config = "ovs-vsctl {0}set Open_vSwitch . other_config:{1}"
 138         detach_cmd = "ovs-vswitchd unix:{0}{1} --pidfile --detach --log-file={2}"
 139
 140         lcore_mask = self.ovs_properties.get("lcore_mask", '')
 141         if lcore_mask:
 142             lcore_mask = ovs_other_config.format("--no-wait ", "dpdk-lcore-mask='%s'" % lcore_mask)
 143
 144         cmd_list = [
 145             "mkdir -p /usr/local/var/run/openvswitch",
 146             "mkdir -p {}".format(os.path.dirname(log_path)),
 147             "ovsdb-server --remote=punix:/{0}/{1}  --pidfile --detach".format(vpath,
 148                                                                               ovs_sock_path),
 149             ovs_other_config.format("--no-wait ", "dpdk-init=true"),
 150             ovs_other_config.format("--no-wait ", "dpdk-socket-mem='%s,%s'" % (socket0, socket1)),
 151             lcore_mask,
 152             detach_cmd.format(vpath, ovs_sock_path, log_path),
 153             ovs_other_config.format("", "pmd-cpu-mask=%s" % pmd_mask),
 154         ]
 155
 156         for cmd in cmd_list:
 157             LOG.info(cmd)
 158             self.connection.execute(cmd)
 159         time.sleep(self.wait_for_vswitchd)
 160
 161     def setup_ovs_bridge_add_flows(self):
 162         dpdk_args = ""
 163         dpdk_list = []
 164         vpath = self.ovs_properties.get("vpath", "/usr/local")
 165         version = self.ovs_properties.get('version', {})
 166         ovs_ver = [int(x) for x in version.get('ovs', self.DEFAULT_OVS).split('.')]
 167         ovs_add_port = ('ovs-vsctl add-port {br} {port} -- '
 168                         'set Interface {port} type={type_}{dpdk_args}')
 169         ovs_add_queue = 'ovs-vsctl set Interface {port} options:n_rxq={queue}'
 170         chmod_vpath = 'chmod 0777 {0}/var/run/openvswitch/dpdkvhostuser*'
 171
 172         cmd_list = [
 173             'ovs-vsctl --if-exists del-br {0}'.format(MAIN_BRIDGE),
 174             'rm -rf {0}/var/run/openvswitch/dpdkvhostuser*'.format(vpath),
 175             'ovs-vsctl add-br {0} -- set bridge {0} datapath_type=netdev'.
 176             format(MAIN_BRIDGE)
 177         ]
 178
 179         ordered_network = collections.OrderedDict(self.networks)
 180         for index, vnf in enumerate(ordered_network.values()):
 181             if ovs_ver >= [2, 7, 0]:
 182                 dpdk_args = " options:dpdk-devargs=%s" % vnf.get("phy_port")
 183             dpdk_list.append(ovs_add_port.format(
 184                 br=MAIN_BRIDGE, port='dpdk%s' % vnf.get("port_num", 0),
 185                 type_='dpdk', dpdk_args=dpdk_args))
 186             dpdk_list.append(ovs_add_queue.format(
 187                 port='dpdk%s' % vnf.get("port_num", 0),
 188                 queue=self.ovs_properties.get("queues", 1)))
 189
 190         # Sorting the array to make sure we execute dpdk0... in the order
 191         list.sort(dpdk_list)
 192         cmd_list.extend(dpdk_list)
 193
 194         # Need to do two for loop to maintain the dpdk/vhost ports.
 195         for index, _ in enumerate(ordered_network):
 196             cmd_list.append(ovs_add_port.format(
 197                 br=MAIN_BRIDGE, port='dpdkvhostuser%s' % index,
 198                 type_='dpdkvhostuser', dpdk_args=""))
 199
 200         ovs_flow = ("ovs-ofctl add-flow {0} in_port=%s,action=output:%s".
 201                     format(MAIN_BRIDGE))
 202         network_count = len(ordered_network) + 1
 203         for in_port, out_port in zip(range(1, network_count),
 204                                      range(network_count, network_count * 2)):
 205             cmd_list.append(ovs_flow % (in_port, out_port))
 206             cmd_list.append(ovs_flow % (out_port, in_port))
 207
 208         cmd_list.append(chmod_vpath.format(vpath))
 209
 210         for cmd in cmd_list:
 211             LOG.info(cmd)
 212             exit_status, _, stderr = self.connection.execute(
 213                 cmd, timeout=self.CMD_TIMEOUT)
 214             if exit_status:
 215                 raise exceptions.OVSSetupError(command=cmd, error=stderr)
 216
 217     def _check_hugepages(self):
 218         meminfo = io.BytesIO()
 219         self.connection.get_file_obj('/proc/meminfo', meminfo)
 220         regex = re.compile(r"HugePages_Total:\s+(?P<hp_total>\d+)[\n\r]"
 221                            r"HugePages_Free:\s+(?P<hp_free>\d+)")
 222         match = regex.search(meminfo.getvalue().decode('utf-8'))
 223         if not match:
 224             raise exceptions.OVSHugepagesInfoError()
 225         if int(match.group('hp_total')) == 0:
 226             raise exceptions.OVSHugepagesNotConfigured()
 227         if int(match.group('hp_free')) == 0:
 228             raise exceptions.OVSHugepagesZeroFree(
 229                 total_hugepages=int(match.group('hp_total')))
 230
 231     def cleanup_ovs_dpdk_env(self):
 232         self.connection.execute(
 233             'ovs-vsctl --if-exists del-br {0}'.format(MAIN_BRIDGE))
 234         self.connection.execute("pkill -9 ovs")
 235
 236     def check_ovs_dpdk_env(self):
 237         self.cleanup_ovs_dpdk_env()
 238         self._check_hugepages()
 239
 240         version = self.ovs_properties.get("version", {})
 241         ovs_ver = version.get("ovs", self.DEFAULT_OVS)
 242         dpdk_ver = version.get("dpdk", "16.07.2").split('.')
 243
 244         supported_version = self.SUPPORTED_OVS_TO_DPDK_MAP.get(ovs_ver, None)
 245         if supported_version is None or supported_version.split('.')[:2] != dpdk_ver[:2]:
 246             raise exceptions.OVSUnsupportedVersion(
 247                 ovs_version=ovs_ver,
 248                 ovs_to_dpdk_map=self.SUPPORTED_OVS_TO_DPDK_MAP)
 249
 250         status = self.connection.execute("ovs-vsctl -V | grep -i '%s'" % ovs_ver)[0]
 251         if status:
 252             deploy = model.OvsDeploy(self.connection,
 253                                      utils.get_nsb_option("bin_path"),
 254                                      self.ovs_properties)
 255             deploy.ovs_deploy()
 256
 257     def deploy(self):
 258         """don't need to deploy"""
 259
 260         # Todo: NFVi deploy (sriov, vswitch, ovs etc) based on the config.
 261         if not self.vm_deploy:
 262             return
 263
 264         self.connection = ssh.SSH.from_node(self.host_mgmt)
 265         self.dpdk_devbind = utils.provision_tool(
 266             self.connection,
 267             os.path.join(utils.get_nsb_option('bin_path'), 'dpdk-devbind.py'))
 268
 269         # Check dpdk/ovs version, if not present install
 270         self.check_ovs_dpdk_env()
 271         #    Todo: NFVi deploy (sriov, vswitch, ovs etc) based on the config.
 272         model.StandaloneContextHelper.install_req_libs(self.connection)
 273         self.networks = model.StandaloneContextHelper.get_nic_details(
 274             self.connection, self.networks, self.dpdk_devbind)
 275
 276         self.setup_ovs()
 277         self.start_ovs_serverswitch()
 278         self.setup_ovs_bridge_add_flows()
 279         self.nodes = self.setup_ovs_dpdk_context()
 280         LOG.debug("Waiting for VM to come up...")
 281         self.nodes = model.StandaloneContextHelper.wait_for_vnfs_to_start(
 282             self.connection, self.servers, self.nodes)
 283
 284     def undeploy(self):
 285
 286         if not self.vm_deploy:
 287             return
 288
 289         # Cleanup the ovs installation...
 290         self.cleanup_ovs_dpdk_env()
 291
 292         # Bind nics back to kernel
 293         bind_cmd = "{dpdk_devbind} --force -b {driver} {port}"
 294         for port in self.networks.values():
 295             vpci = port.get("phy_port")
 296             phy_driver = port.get("driver")
 297             self.connection.execute(bind_cmd.format(
 298                 dpdk_devbind=self.dpdk_devbind, driver=phy_driver, port=vpci))
 299
 300         # Todo: NFVi undeploy (sriov, vswitch, ovs etc) based on the config.
 301         for vm in self.vm_names:
 302             model.Libvirt.check_if_vm_exists_and_delete(vm, self.connection)
 303
 304     def _get_server(self, attr_name):
 305         """lookup server info by name from context
 306
 307         Keyword arguments:
 308         attr_name -- A name for a server listed in nodes config file
 309         """
 310         node_name, name = self.split_name(attr_name)
 311         if name is None or self.name != name:
 312             return None
 313
 314         matching_nodes = (n for n in self.nodes if n["name"] == node_name)
 315         try:
 316             # A clone is created in order to avoid affecting the
 317             # original one.
 318             node = dict(next(matching_nodes))
 319         except StopIteration:
 320             return None
 321
 322         try:
 323             duplicate = next(matching_nodes)
 324         except StopIteration:
 325             pass
 326         else:
 327             raise ValueError("Duplicate nodes!!! Nodes: %s %s" % (node, duplicate))
 328
 329         node["name"] = attr_name
 330         return node
 331
 332     def _get_network(self, attr_name):
 333         if not isinstance(attr_name, collections.Mapping):
 334             network = self.networks.get(attr_name)
 335
 336         else:
 337             # Don't generalize too much  Just support vld_id
 338             vld_id = attr_name.get('vld_id', {})
 339             # for standalone context networks are dicts
 340             iter1 = (n for n in self.networks.values() if n.get('vld_id') == vld_id)
 341             network = next(iter1, None)
 342
 343         if network is None:
 344             return None
 345
 346         result = {
 347             # name is required
 348             "name": network["name"],
 349             "vld_id": network.get("vld_id"),
 350             "segmentation_id": network.get("segmentation_id"),
 351             "network_type": network.get("network_type"),
 352             "physical_network": network.get("physical_network"),
 353         }
 354         return result
 355
 356     def configure_nics_for_ovs_dpdk(self):
 357         portlist = collections.OrderedDict(self.networks)
 358         for key in portlist:
 359             mac = model.StandaloneContextHelper.get_mac_address()
 360             portlist[key].update({'mac': mac})
 361         self.networks = portlist
 362         LOG.info("Ports %s", self.networks)
 363
 364     def _enable_interfaces(self, index, vfs, cfg):
 365         vpath = self.ovs_properties.get("vpath", "/usr/local")
 366         vf = self.networks[vfs[0]]
 367         port_num = vf.get('port_num', 0)
 368         vpci = utils.PciAddress(vf['vpci'].strip())
 369         # Generate the vpci for the interfaces
 370         slot = index + port_num + 10
 371         vf['vpci'] = \
 372             "{}:{}:{:02x}.{}".format(vpci.domain, vpci.bus, slot, vpci.function)
 373         model.Libvirt.add_ovs_interface(
 374             vpath, port_num, vf['vpci'], vf['mac'], str(cfg))
 375
 376     def setup_ovs_dpdk_context(self):
 377         nodes = []
 378
 379         self.configure_nics_for_ovs_dpdk()
 380
 381         for index, (key, vnf) in enumerate(collections.OrderedDict(
 382                 self.servers).items()):
 383             cfg = '/tmp/vm_ovs_%d.xml' % index
 384             vm_name = "vm_%d" % index
 385
 386             # 1. Check and delete VM if already exists
 387             model.Libvirt.check_if_vm_exists_and_delete(vm_name,
 388                                                         self.connection)
 389
 390             _, mac = model.Libvirt.build_vm_xml(
 391                 self.connection, self.vm_flavor, cfg, vm_name, index)
 392             # 2: Cleanup already available VMs
 393             for vkey, vfs in collections.OrderedDict(
 394                     vnf["network_ports"]).items():
 395                 if vkey == "mgmt":
 396                     continue
 397                 self._enable_interfaces(index, vfs, cfg)
 398
 399             # copy xml to target...
 400             self.connection.put(cfg, cfg)
 401
 402             # NOTE: launch through libvirt
 403             LOG.info("virsh create ...")
 404             model.Libvirt.virsh_create_vm(self.connection, cfg)
 405
 406             self.vm_names.append(vm_name)
 407
 408             # build vnf node details
 409             nodes.append(self.vnf_node.generate_vnf_instance(self.vm_flavor,
 410                                                              self.networks,
 411                                                              self.host_mgmt.get('ip'),
 412                                                              key, vnf, mac))
 413
 414         return nodes