yardstick/benchmark/contexts/standalone/ovs_dpdk.py

   1 # Copyright (c) 2016-2017 Intel Corporation
   2 #
   3 # Licensed under the Apache License, Version 2.0 (the "License");
   4 # you may not use this file except in compliance with the License.
   5 # You may obtain a copy of the License at
   6 #
   7 #      http://www.apache.org/licenses/LICENSE-2.0
   8 #
   9 # Unless required by applicable law or agreed to in writing, software
  10 # distributed under the License is distributed on an "AS IS" BASIS,
  11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12 # See the License for the specific language governing permissions and
  13 # limitations under the License.
  14
  15 from __future__ import absolute_import
  16 import os
  17 import logging
  18 import collections
  19 import time
  20
  21 from collections import OrderedDict
  22
  23 from yardstick import ssh
  24 from yardstick.network_services.utils import get_nsb_option
  25 from yardstick.network_services.utils import provision_tool
  26 from yardstick.benchmark.contexts.base import Context
  27 from yardstick.benchmark.contexts.standalone.model import Libvirt
  28 from yardstick.benchmark.contexts.standalone.model import StandaloneContextHelper
  29 from yardstick.benchmark.contexts.standalone.model import Server
  30 from yardstick.benchmark.contexts.standalone.model import OvsDeploy
  31 from yardstick.network_services.utils import PciAddress
  32
  33 LOG = logging.getLogger(__name__)
  34
  35
  36 class OvsDpdkContext(Context):
  37     """ This class handles OVS standalone nodes - VM running on Non-Managed NFVi
  38     Configuration: ovs_dpdk
  39     """
  40
  41     __context_type__ = "StandaloneOvsDpdk"
  42
  43     SUPPORTED_OVS_TO_DPDK_MAP = {
  44         '2.6.0': '16.07.1',
  45         '2.6.1': '16.07.2',
  46         '2.7.0': '16.11.1',
  47         '2.7.1': '16.11.2',
  48         '2.7.2': '16.11.3',
  49         '2.8.0': '17.05.2'
  50     }
  51
  52     DEFAULT_OVS = '2.6.0'
  53
  54     PKILL_TEMPLATE = "pkill %s %s"
  55
  56     def __init__(self):
  57         self.file_path = None
  58         self.sriov = []
  59         self.first_run = True
  60         self.dpdk_devbind = ''
  61         self.vm_names = []
  62         self.name = None
  63         self.nfvi_host = []
  64         self.nodes = []
  65         self.networks = {}
  66         self.attrs = {}
  67         self.vm_flavor = None
  68         self.servers = None
  69         self.helper = StandaloneContextHelper()
  70         self.vnf_node = Server()
  71         self.ovs_properties = {}
  72         self.wait_for_vswitchd = 10
  73         super(OvsDpdkContext, self).__init__()
  74
  75     def init(self, attrs):
  76         """initializes itself from the supplied arguments"""
  77
  78         self.name = attrs["name"]
  79         self.file_path = attrs.get("file", "pod.yaml")
  80
  81         self.nodes, self.nfvi_host, self.host_mgmt = \
  82             self.helper.parse_pod_file(self.file_path, 'OvsDpdk')
  83
  84         self.attrs = attrs
  85         self.vm_flavor = attrs.get('flavor', {})
  86         self.servers = attrs.get('servers', {})
  87         self.vm_deploy = attrs.get("vm_deploy", True)
  88         self.ovs_properties = attrs.get('ovs_properties', {})
  89         # add optional static network definition
  90         self.networks = attrs.get("networks", {})
  91
  92         LOG.debug("Nodes: %r", self.nodes)
  93         LOG.debug("NFVi Node: %r", self.nfvi_host)
  94         LOG.debug("Networks: %r", self.networks)
  95
  96     def setup_ovs(self):
  97         vpath = self.ovs_properties.get("vpath", "/usr/local")
  98         xargs_kill_cmd = self.PKILL_TEMPLATE % ('-9', 'ovs')
  99
 100         create_from = os.path.join(vpath, 'etc/openvswitch/conf.db')
 101         create_to = os.path.join(vpath, 'share/openvswitch/vswitch.ovsschema')
 102
 103         cmd_list = [
 104             "chmod 0666 /dev/vfio/*",
 105             "chmod a+x /dev/vfio",
 106             "pkill -9 ovs",
 107             xargs_kill_cmd,
 108             "killall -r 'ovs*'",
 109             "mkdir -p {0}/etc/openvswitch".format(vpath),
 110             "mkdir -p {0}/var/run/openvswitch".format(vpath),
 111             "rm {0}/etc/openvswitch/conf.db".format(vpath),
 112             "ovsdb-tool create {0} {1}".format(create_from, create_to),
 113             "modprobe vfio-pci",
 114             "chmod a+x /dev/vfio",
 115             "chmod 0666 /dev/vfio/*",
 116         ]
 117         for cmd in cmd_list:
 118             self.connection.execute(cmd)
 119         bind_cmd = "{dpdk_devbind} --force -b {driver} {port}"
 120         phy_driver = "vfio-pci"
 121         for port in self.networks.values():
 122             vpci = port.get("phy_port")
 123             self.connection.execute(bind_cmd.format(
 124                 dpdk_devbind=self.dpdk_devbind, driver=phy_driver, port=vpci))
 125
 126     def start_ovs_serverswitch(self):
 127         vpath = self.ovs_properties.get("vpath")
 128         pmd_nums = int(self.ovs_properties.get("pmd_threads", 2))
 129         ovs_sock_path = '/var/run/openvswitch/db.sock'
 130         log_path = '/var/log/openvswitch/ovs-vswitchd.log'
 131
 132         pmd_cpu_mask = self.ovs_properties.get("pmd_cpu_mask", '')
 133         pmd_mask = hex(sum(2 ** num for num in range(pmd_nums)) << 1)
 134         if pmd_cpu_mask:
 135             pmd_mask = pmd_cpu_mask
 136
 137         socket0 = self.ovs_properties.get("ram", {}).get("socket_0", "2048")
 138         socket1 = self.ovs_properties.get("ram", {}).get("socket_1", "2048")
 139
 140         ovs_other_config = "ovs-vsctl {0}set Open_vSwitch . other_config:{1}"
 141         detach_cmd = "ovs-vswitchd unix:{0}{1} --pidfile --detach --log-file={2}"
 142
 143         lcore_mask = self.ovs_properties.get("lcore_mask", '')
 144         if lcore_mask:
 145             lcore_mask = ovs_other_config.format("--no-wait ", "dpdk-lcore-mask='%s'" % lcore_mask)
 146
 147         cmd_list = [
 148             "mkdir -p /usr/local/var/run/openvswitch",
 149             "mkdir -p {}".format(os.path.dirname(log_path)),
 150             "ovsdb-server --remote=punix:/{0}/{1}  --pidfile --detach".format(vpath,
 151                                                                               ovs_sock_path),
 152             ovs_other_config.format("--no-wait ", "dpdk-init=true"),
 153             ovs_other_config.format("--no-wait ", "dpdk-socket-mem='%s,%s'" % (socket0, socket1)),
 154             lcore_mask,
 155             detach_cmd.format(vpath, ovs_sock_path, log_path),
 156             ovs_other_config.format("", "pmd-cpu-mask=%s" % pmd_mask),
 157         ]
 158
 159         for cmd in cmd_list:
 160             LOG.info(cmd)
 161             self.connection.execute(cmd)
 162         time.sleep(self.wait_for_vswitchd)
 163
 164     def setup_ovs_bridge_add_flows(self):
 165         dpdk_args = ""
 166         dpdk_list = []
 167         vpath = self.ovs_properties.get("vpath", "/usr/local")
 168         version = self.ovs_properties.get('version', {})
 169         ovs_ver = [int(x) for x in version.get('ovs', self.DEFAULT_OVS).split('.')]
 170         ovs_add_port = \
 171             "ovs-vsctl add-port {br} {port} -- set Interface {port} type={type_}{dpdk_args}"
 172         ovs_add_queue = "ovs-vsctl set Interface {port} options:n_rxq={queue}"
 173         chmod_vpath = "chmod 0777 {0}/var/run/openvswitch/dpdkvhostuser*"
 174
 175         cmd_dpdk_list = [
 176             "ovs-vsctl del-br br0",
 177             "rm -rf {0}/var/run/openvswitch/dpdkvhostuser*".format(vpath),
 178             "ovs-vsctl add-br br0 -- set bridge br0 datapath_type=netdev",
 179         ]
 180
 181         ordered_network = OrderedDict(self.networks)
 182         for index, vnf in enumerate(ordered_network.values()):
 183             if ovs_ver >= [2, 7, 0]:
 184                 dpdk_args = " options:dpdk-devargs=%s" % vnf.get("phy_port")
 185             dpdk_list.append(ovs_add_port.format(br='br0', port='dpdk%s' % vnf.get("port_num", 0),
 186                                                  type_='dpdk', dpdk_args=dpdk_args))
 187             dpdk_list.append(ovs_add_queue.format(port='dpdk%s' % vnf.get("port_num", 0),
 188                                                   queue=self.ovs_properties.get("queues", 1)))
 189
 190         # Sorting the array to make sure we execute dpdk0... in the order
 191         list.sort(dpdk_list)
 192         cmd_dpdk_list.extend(dpdk_list)
 193
 194         # Need to do two for loop to maintain the dpdk/vhost ports.
 195         for index, _ in enumerate(ordered_network):
 196             cmd_dpdk_list.append(ovs_add_port.format(br='br0', port='dpdkvhostuser%s' % index,
 197                                                      type_='dpdkvhostuser', dpdk_args=""))
 198
 199         for cmd in cmd_dpdk_list:
 200             LOG.info(cmd)
 201             self.connection.execute(cmd)
 202
 203         # Fixme: add flows code
 204         ovs_flow = "ovs-ofctl add-flow br0 in_port=%s,action=output:%s"
 205
 206         network_count = len(ordered_network) + 1
 207         for in_port, out_port in zip(range(1, network_count),
 208                                      range(network_count, network_count * 2)):
 209             self.connection.execute(ovs_flow % (in_port, out_port))
 210             self.connection.execute(ovs_flow % (out_port, in_port))
 211
 212         self.connection.execute(chmod_vpath.format(vpath))
 213
 214     def cleanup_ovs_dpdk_env(self):
 215         self.connection.execute("ovs-vsctl del-br br0")
 216         self.connection.execute("pkill -9 ovs")
 217
 218     def check_ovs_dpdk_env(self):
 219         self.cleanup_ovs_dpdk_env()
 220
 221         version = self.ovs_properties.get("version", {})
 222         ovs_ver = version.get("ovs", self.DEFAULT_OVS)
 223         dpdk_ver = version.get("dpdk", "16.07.2").split('.')
 224
 225         supported_version = self.SUPPORTED_OVS_TO_DPDK_MAP.get(ovs_ver, None)
 226         if supported_version is None or supported_version.split('.')[:2] != dpdk_ver[:2]:
 227             raise Exception("Unsupported ovs '{}'. Please check the config...".format(ovs_ver))
 228
 229         status = self.connection.execute("ovs-vsctl -V | grep -i '%s'" % ovs_ver)[0]
 230         if status:
 231             deploy = OvsDeploy(self.connection,
 232                                get_nsb_option("bin_path"),
 233                                self.ovs_properties)
 234             deploy.ovs_deploy()
 235
 236     def deploy(self):
 237         """don't need to deploy"""
 238
 239         # Todo: NFVi deploy (sriov, vswitch, ovs etc) based on the config.
 240         if not self.vm_deploy:
 241             return
 242
 243         self.connection = ssh.SSH.from_node(self.host_mgmt)
 244         self.dpdk_devbind = provision_tool(
 245             self.connection,
 246             os.path.join(get_nsb_option("bin_path"), "dpdk-devbind.py"))
 247
 248         # Check dpdk/ovs version, if not present install
 249         self.check_ovs_dpdk_env()
 250         #    Todo: NFVi deploy (sriov, vswitch, ovs etc) based on the config.
 251         StandaloneContextHelper.install_req_libs(self.connection)
 252         self.networks = StandaloneContextHelper.get_nic_details(
 253             self.connection, self.networks, self.dpdk_devbind)
 254
 255         self.setup_ovs()
 256         self.start_ovs_serverswitch()
 257         self.setup_ovs_bridge_add_flows()
 258         self.nodes = self.setup_ovs_dpdk_context()
 259         LOG.debug("Waiting for VM to come up...")
 260         self.nodes = StandaloneContextHelper.wait_for_vnfs_to_start(self.connection,
 261                                                                     self.servers,
 262                                                                     self.nodes)
 263
 264     def undeploy(self):
 265
 266         if not self.vm_deploy:
 267             return
 268
 269         # Cleanup the ovs installation...
 270         self.cleanup_ovs_dpdk_env()
 271
 272         # Bind nics back to kernel
 273         bind_cmd = "{dpdk_devbind} --force -b {driver} {port}"
 274         for port in self.networks.values():
 275             vpci = port.get("phy_port")
 276             phy_driver = port.get("driver")
 277             self.connection.execute(bind_cmd.format(
 278                 dpdk_devbind=self.dpdk_devbind, driver=phy_driver, port=vpci))
 279
 280         # Todo: NFVi undeploy (sriov, vswitch, ovs etc) based on the config.
 281         for vm in self.vm_names:
 282             Libvirt.check_if_vm_exists_and_delete(vm, self.connection)
 283
 284     def _get_server(self, attr_name):
 285         """lookup server info by name from context
 286
 287         Keyword arguments:
 288         attr_name -- A name for a server listed in nodes config file
 289         """
 290         node_name, name = self.split_name(attr_name)
 291         if name is None or self.name != name:
 292             return None
 293
 294         matching_nodes = (n for n in self.nodes if n["name"] == node_name)
 295         try:
 296             # A clone is created in order to avoid affecting the
 297             # original one.
 298             node = dict(next(matching_nodes))
 299         except StopIteration:
 300             return None
 301
 302         try:
 303             duplicate = next(matching_nodes)
 304         except StopIteration:
 305             pass
 306         else:
 307             raise ValueError("Duplicate nodes!!! Nodes: %s %s" % (node, duplicate))
 308
 309         node["name"] = attr_name
 310         return node
 311
 312     def _get_network(self, attr_name):
 313         if not isinstance(attr_name, collections.Mapping):
 314             network = self.networks.get(attr_name)
 315
 316         else:
 317             # Don't generalize too much  Just support vld_id
 318             vld_id = attr_name.get('vld_id', {})
 319             # for standalone context networks are dicts
 320             iter1 = (n for n in self.networks.values() if n.get('vld_id') == vld_id)
 321             network = next(iter1, None)
 322
 323         if network is None:
 324             return None
 325
 326         result = {
 327             # name is required
 328             "name": network["name"],
 329             "vld_id": network.get("vld_id"),
 330             "segmentation_id": network.get("segmentation_id"),
 331             "network_type": network.get("network_type"),
 332             "physical_network": network.get("physical_network"),
 333         }
 334         return result
 335
 336     def configure_nics_for_ovs_dpdk(self):
 337         portlist = OrderedDict(self.networks)
 338         for key in portlist:
 339             mac = StandaloneContextHelper.get_mac_address()
 340             portlist[key].update({'mac': mac})
 341         self.networks = portlist
 342         LOG.info("Ports %s", self.networks)
 343
 344     def _enable_interfaces(self, index, vfs, cfg):
 345         vpath = self.ovs_properties.get("vpath", "/usr/local")
 346         vf = self.networks[vfs[0]]
 347         port_num = vf.get('port_num', 0)
 348         vpci = PciAddress(vf['vpci'].strip())
 349         # Generate the vpci for the interfaces
 350         slot = index + port_num + 10
 351         vf['vpci'] = \
 352             "{}:{}:{:02x}.{}".format(vpci.domain, vpci.bus, slot, vpci.function)
 353         Libvirt.add_ovs_interface(vpath, port_num, vf['vpci'], vf['mac'], str(cfg))
 354
 355     def setup_ovs_dpdk_context(self):
 356         nodes = []
 357
 358         self.configure_nics_for_ovs_dpdk()
 359
 360         for index, (key, vnf) in enumerate(OrderedDict(self.servers).items()):
 361             cfg = '/tmp/vm_ovs_%d.xml' % index
 362             vm_name = "vm_%d" % index
 363
 364             # 1. Check and delete VM if already exists
 365             Libvirt.check_if_vm_exists_and_delete(vm_name, self.connection)
 366
 367             _, mac = Libvirt.build_vm_xml(self.connection, self.vm_flavor,
 368                                           cfg, vm_name, index)
 369             # 2: Cleanup already available VMs
 370             for vkey, vfs in OrderedDict(vnf["network_ports"]).items():
 371                 if vkey == "mgmt":
 372                     continue
 373                 self._enable_interfaces(index, vfs, cfg)
 374
 375             # copy xml to target...
 376             self.connection.put(cfg, cfg)
 377
 378             # NOTE: launch through libvirt
 379             LOG.info("virsh create ...")
 380             Libvirt.virsh_create_vm(self.connection, cfg)
 381
 382             self.vm_names.append(vm_name)
 383
 384             # build vnf node details
 385             nodes.append(self.vnf_node.generate_vnf_instance(self.vm_flavor,
 386                                                              self.networks,
 387                                                              self.host_mgmt.get('ip'),
 388                                                              key, vnf, mac))
 389
 390         return nodes