yardstick/benchmark/contexts/standalone/ovs_dpdk.py

   1 # Copyright (c) 2016-2017 Intel Corporation
   2 #
   3 # Licensed under the Apache License, Version 2.0 (the "License");
   4 # you may not use this file except in compliance with the License.
   5 # You may obtain a copy of the License at
   6 #
   7 #      http://www.apache.org/licenses/LICENSE-2.0
   8 #
   9 # Unless required by applicable law or agreed to in writing, software
  10 # distributed under the License is distributed on an "AS IS" BASIS,
  11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12 # See the License for the specific language governing permissions and
  13 # limitations under the License.
  14
  15 import io
  16 import collections
  17 import logging
  18 import os
  19 import re
  20 import time
  21
  22 from yardstick import ssh
  23 from yardstick.benchmark import contexts
  24 from yardstick.benchmark.contexts import base
  25 from yardstick.benchmark.contexts.standalone import model
  26 from yardstick.common import exceptions
  27 from yardstick.network_services import utils
  28 from yardstick.network_services.utils import get_nsb_option
  29
  30
  31 LOG = logging.getLogger(__name__)
  32
  33 MAIN_BRIDGE = 'br0'
  34
  35
  36 class OvsDpdkContext(base.Context):
  37     """ This class handles OVS standalone nodes - VM running on Non-Managed NFVi
  38     Configuration: ovs_dpdk
  39     """
  40
  41     __context_type__ = contexts.CONTEXT_STANDALONEOVSDPDK
  42
  43     SUPPORTED_OVS_TO_DPDK_MAP = {
  44         '2.6.0': '16.07.1',
  45         '2.6.1': '16.07.2',
  46         '2.7.0': '16.11.1',
  47         '2.7.1': '16.11.2',
  48         '2.7.2': '16.11.3',
  49         '2.8.0': '17.05.2',
  50         '2.8.1': '17.05.2'
  51     }
  52
  53     DEFAULT_OVS = '2.6.0'
  54     CMD_TIMEOUT = 30
  55     DEFAULT_USER_PATH = '/usr/local'
  56
  57     def __init__(self):
  58         self.file_path = None
  59         self.sriov = []
  60         self.first_run = True
  61         self.dpdk_devbind = os.path.join(get_nsb_option('bin_path'),
  62                                          'dpdk-devbind.py')
  63         self.vm_names = []
  64         self.nfvi_host = []
  65         self.nodes = []
  66         self.networks = {}
  67         self.attrs = {}
  68         self.vm_flavor = None
  69         self.servers = None
  70         self.helper = model.StandaloneContextHelper()
  71         self.vnf_node = model.Server()
  72         self.ovs_properties = {}
  73         self.wait_for_vswitchd = 10
  74         super(OvsDpdkContext, self).__init__()
  75
  76     def init(self, attrs):
  77         """initializes itself from the supplied arguments"""
  78         super(OvsDpdkContext, self).init(attrs)
  79
  80         self.file_path = attrs.get("file", "pod.yaml")
  81
  82         self.nodes, self.nfvi_host, self.host_mgmt = \
  83             self.helper.parse_pod_file(self.file_path, 'OvsDpdk')
  84
  85         self.attrs = attrs
  86         self.vm_flavor = attrs.get('flavor', {})
  87         self.servers = attrs.get('servers', {})
  88         self.vm_deploy = attrs.get("vm_deploy", True)
  89         self.ovs_properties = attrs.get('ovs_properties', {})
  90         # add optional static network definition
  91         self.networks = attrs.get("networks", {})
  92
  93         LOG.debug("Nodes: %r", self.nodes)
  94         LOG.debug("NFVi Node: %r", self.nfvi_host)
  95         LOG.debug("Networks: %r", self.networks)
  96
  97     def setup_ovs(self):
  98         """Initialize OVS-DPDK"""
  99         vpath = self.ovs_properties.get('vpath', self.DEFAULT_USER_PATH)
 100         create_from = os.path.join(vpath, 'etc/openvswitch/conf.db')
 101         create_to = os.path.join(vpath, 'share/openvswitch/vswitch.ovsschema')
 102
 103         cmd_list = [
 104             'killall -r "ovs.*" -q | true',
 105             'mkdir -p {0}/etc/openvswitch'.format(vpath),
 106             'mkdir -p {0}/var/run/openvswitch'.format(vpath),
 107             'rm {0}/etc/openvswitch/conf.db | true'.format(vpath),
 108             'ovsdb-tool create {0} {1}'.format(create_from, create_to),
 109             'modprobe vfio-pci',
 110             'chmod a+x /dev/vfio',
 111             'chmod 0666 /dev/vfio/*',
 112         ]
 113
 114         bind_cmd = '%s --force -b vfio-pci {port}' % self.dpdk_devbind
 115         for port in self.networks.values():
 116             cmd_list.append(bind_cmd.format(port=port.get('phy_port')))
 117
 118         for cmd in cmd_list:
 119             LOG.info(cmd)
 120             exit_status, _, stderr = self.connection.execute(
 121                 cmd, timeout=self.CMD_TIMEOUT)
 122             if exit_status:
 123                 raise exceptions.OVSSetupError(command=cmd, error=stderr)
 124
 125     def start_ovs_serverswitch(self):
 126         vpath = self.ovs_properties.get("vpath")
 127         pmd_nums = int(self.ovs_properties.get("pmd_threads", 2))
 128         ovs_sock_path = '/var/run/openvswitch/db.sock'
 129         log_path = '/var/log/openvswitch/ovs-vswitchd.log'
 130
 131         pmd_cpu_mask = self.ovs_properties.get("pmd_cpu_mask", '')
 132         pmd_mask = hex(sum(2 ** num for num in range(pmd_nums)) << 1)
 133         if pmd_cpu_mask:
 134             pmd_mask = pmd_cpu_mask
 135
 136         socket0 = self.ovs_properties.get("ram", {}).get("socket_0", "2048")
 137         socket1 = self.ovs_properties.get("ram", {}).get("socket_1", "2048")
 138
 139         ovs_other_config = "ovs-vsctl {0}set Open_vSwitch . other_config:{1}"
 140         detach_cmd = "ovs-vswitchd unix:{0}{1} --pidfile --detach --log-file={2}"
 141
 142         lcore_mask = self.ovs_properties.get("lcore_mask", '')
 143         if lcore_mask:
 144             lcore_mask = ovs_other_config.format("--no-wait ", "dpdk-lcore-mask='%s'" % lcore_mask)
 145
 146         cmd_list = [
 147             "mkdir -p /usr/local/var/run/openvswitch",
 148             "mkdir -p {}".format(os.path.dirname(log_path)),
 149             ("ovsdb-server --remote=punix:/{0}/{1} --remote=ptcp:6640"
 150              " --pidfile --detach").format(vpath, ovs_sock_path),
 151             ovs_other_config.format("--no-wait ", "dpdk-init=true"),
 152             ovs_other_config.format("--no-wait ", "dpdk-socket-mem='%s,%s'" % (socket0, socket1)),
 153             lcore_mask,
 154             detach_cmd.format(vpath, ovs_sock_path, log_path),
 155             ovs_other_config.format("", "pmd-cpu-mask=%s" % pmd_mask),
 156         ]
 157
 158         for cmd in cmd_list:
 159             LOG.info(cmd)
 160             self.connection.execute(cmd)
 161         time.sleep(self.wait_for_vswitchd)
 162
 163     def setup_ovs_bridge_add_flows(self):
 164         dpdk_args = ""
 165         dpdk_list = []
 166         vpath = self.ovs_properties.get("vpath", "/usr/local")
 167         version = self.ovs_properties.get('version', {})
 168         ovs_ver = [int(x) for x in version.get('ovs', self.DEFAULT_OVS).split('.')]
 169         ovs_add_port = ('ovs-vsctl add-port {br} {port} -- '
 170                         'set Interface {port} type={type_}{dpdk_args}{dpdk_rxq}')
 171         chmod_vpath = 'chmod 0777 {0}/var/run/openvswitch/dpdkvhostuser*'
 172
 173         cmd_list = [
 174             'ovs-vsctl --if-exists del-br {0}'.format(MAIN_BRIDGE),
 175             'rm -rf {0}/var/run/openvswitch/dpdkvhostuser*'.format(vpath),
 176             'ovs-vsctl add-br {0} -- set bridge {0} datapath_type=netdev'.
 177             format(MAIN_BRIDGE)
 178         ]
 179         dpdk_rxq = " options:n_rxq={queue}".format(
 180             queue=self.ovs_properties.get("queues", 1))
 181
 182         ordered_network = collections.OrderedDict(self.networks)
 183         for index, vnf in enumerate(ordered_network.values()):
 184             if ovs_ver >= [2, 7, 0]:
 185                 dpdk_args = " options:dpdk-devargs=%s" % vnf.get("phy_port")
 186             dpdk_list.append(ovs_add_port.format(
 187                 br=MAIN_BRIDGE, port='dpdk%s' % vnf.get("port_num", 0),
 188                 type_='dpdk', dpdk_args=dpdk_args, dpdk_rxq=dpdk_rxq))
 189
 190         # Sorting the array to make sure we execute dpdk0... in the order
 191         list.sort(dpdk_list)
 192         cmd_list.extend(dpdk_list)
 193
 194         # Need to do two for loop to maintain the dpdk/vhost ports.
 195         for index, _ in enumerate(ordered_network):
 196             cmd_list.append(ovs_add_port.format(
 197                 br=MAIN_BRIDGE, port='dpdkvhostuser%s' % index,
 198                 type_='dpdkvhostuser', dpdk_args="", dpdk_rxq=""))
 199
 200         ovs_flow = ("ovs-ofctl add-flow {0} in_port=%s,action=output:%s".
 201                     format(MAIN_BRIDGE))
 202         network_count = len(ordered_network) + 1
 203         for in_port, out_port in zip(range(1, network_count),
 204                                      range(network_count, network_count * 2)):
 205             cmd_list.append(ovs_flow % (in_port, out_port))
 206             cmd_list.append(ovs_flow % (out_port, in_port))
 207
 208         cmd_list.append(chmod_vpath.format(vpath))
 209
 210         for cmd in cmd_list:
 211             LOG.info(cmd)
 212             exit_status, _, stderr = self.connection.execute(
 213                 cmd, timeout=self.CMD_TIMEOUT)
 214             if exit_status:
 215                 raise exceptions.OVSSetupError(command=cmd, error=stderr)
 216
 217     def _check_hugepages(self):
 218         meminfo = io.BytesIO()
 219         self.connection.get_file_obj('/proc/meminfo', meminfo)
 220         regex = re.compile(r"HugePages_Total:\s+(?P<hp_total>\d+)[\n\r]"
 221                            r"HugePages_Free:\s+(?P<hp_free>\d+)")
 222         match = regex.search(meminfo.getvalue().decode('utf-8'))
 223         if not match:
 224             raise exceptions.OVSHugepagesInfoError()
 225         if int(match.group('hp_total')) == 0:
 226             raise exceptions.OVSHugepagesNotConfigured()
 227         if int(match.group('hp_free')) == 0:
 228             raise exceptions.OVSHugepagesZeroFree(
 229                 total_hugepages=int(match.group('hp_total')))
 230
 231     def cleanup_ovs_dpdk_env(self):
 232         self.connection.execute(
 233             'ovs-vsctl --if-exists del-br {0}'.format(MAIN_BRIDGE))
 234         self.connection.execute("pkill -9 ovs")
 235
 236     def check_ovs_dpdk_env(self):
 237         self.cleanup_ovs_dpdk_env()
 238         self._check_hugepages()
 239
 240         version = self.ovs_properties.get("version", {})
 241         ovs_ver = version.get("ovs", self.DEFAULT_OVS)
 242         dpdk_ver = version.get("dpdk", "16.07.2").split('.')
 243
 244         supported_version = self.SUPPORTED_OVS_TO_DPDK_MAP.get(ovs_ver, None)
 245         if supported_version is None or supported_version.split('.')[:2] != dpdk_ver[:2]:
 246             raise exceptions.OVSUnsupportedVersion(
 247                 ovs_version=ovs_ver,
 248                 ovs_to_dpdk_map=self.SUPPORTED_OVS_TO_DPDK_MAP)
 249
 250         status = self.connection.execute("ovs-vsctl -V | grep -i '%s'" % ovs_ver)[0]
 251         if status:
 252             deploy = model.OvsDeploy(self.connection,
 253                                      utils.get_nsb_option("bin_path"),
 254                                      self.ovs_properties)
 255             deploy.ovs_deploy()
 256
 257     def deploy(self):
 258         """don't need to deploy"""
 259
 260         # Todo: NFVi deploy (sriov, vswitch, ovs etc) based on the config.
 261         if not self.vm_deploy:
 262             return
 263
 264         self.connection = ssh.SSH.from_node(self.host_mgmt)
 265
 266         # Check dpdk/ovs version, if not present install
 267         self.check_ovs_dpdk_env()
 268         #    Todo: NFVi deploy (sriov, vswitch, ovs etc) based on the config.
 269         model.StandaloneContextHelper.install_req_libs(self.connection)
 270         self.networks = model.StandaloneContextHelper.get_nic_details(
 271             self.connection, self.networks, self.dpdk_devbind)
 272
 273         self.setup_ovs()
 274         self.start_ovs_serverswitch()
 275         self.setup_ovs_bridge_add_flows()
 276         self.nodes = self.setup_ovs_dpdk_context()
 277         LOG.debug("Waiting for VM to come up...")
 278         self.nodes = model.StandaloneContextHelper.wait_for_vnfs_to_start(
 279             self.connection, self.servers, self.nodes)
 280
 281     def undeploy(self):
 282
 283         if not self.vm_deploy:
 284             return
 285
 286         # Cleanup the ovs installation...
 287         self.cleanup_ovs_dpdk_env()
 288
 289         # Bind nics back to kernel
 290         bind_cmd = "{dpdk_devbind} --force -b {driver} {port}"
 291         for port in self.networks.values():
 292             vpci = port.get("phy_port")
 293             phy_driver = port.get("driver")
 294             self.connection.execute(bind_cmd.format(
 295                 dpdk_devbind=self.dpdk_devbind, driver=phy_driver, port=vpci))
 296
 297         # Todo: NFVi undeploy (sriov, vswitch, ovs etc) based on the config.
 298         for vm in self.vm_names:
 299             model.Libvirt.check_if_vm_exists_and_delete(vm, self.connection)
 300
 301     def _get_physical_nodes(self):
 302         return self.nfvi_host
 303
 304     def _get_physical_node_for_server(self, server_name):
 305         node_name, ctx_name = self.split_host_name(server_name)
 306         if ctx_name is None or self.name != ctx_name:
 307             return None
 308
 309         matching_nodes = [s for s in self.servers if s == node_name]
 310         if len(matching_nodes) == 0:
 311             return None
 312
 313         # self.nfvi_host always contain only one host
 314         return "{}.{}".format(self.nfvi_host[0]["name"], self._name)
 315
 316     def _get_server(self, attr_name):
 317         """lookup server info by name from context
 318
 319         Keyword arguments:
 320         attr_name -- A name for a server listed in nodes config file
 321         """
 322         node_name, name = self.split_host_name(attr_name)
 323         if name is None or self.name != name:
 324             return None
 325
 326         matching_nodes = (n for n in self.nodes if n["name"] == node_name)
 327         try:
 328             # A clone is created in order to avoid affecting the
 329             # original one.
 330             node = dict(next(matching_nodes))
 331         except StopIteration:
 332             return None
 333
 334         try:
 335             duplicate = next(matching_nodes)
 336         except StopIteration:
 337             pass
 338         else:
 339             raise ValueError("Duplicate nodes!!! Nodes: %s %s" % (node, duplicate))
 340
 341         node["name"] = attr_name
 342         return node
 343
 344     def _get_network(self, attr_name):
 345         if not isinstance(attr_name, collections.Mapping):
 346             network = self.networks.get(attr_name)
 347
 348         else:
 349             # Don't generalize too much  Just support vld_id
 350             vld_id = attr_name.get('vld_id', {})
 351             # for standalone context networks are dicts
 352             iter1 = (n for n in self.networks.values() if n.get('vld_id') == vld_id)
 353             network = next(iter1, None)
 354
 355         if network is None:
 356             return None
 357
 358         result = {
 359             # name is required
 360             "name": network["name"],
 361             "vld_id": network.get("vld_id"),
 362             "segmentation_id": network.get("segmentation_id"),
 363             "network_type": network.get("network_type"),
 364             "physical_network": network.get("physical_network"),
 365         }
 366         return result
 367
 368     def configure_nics_for_ovs_dpdk(self):
 369         portlist = collections.OrderedDict(self.networks)
 370         for key in portlist:
 371             mac = model.StandaloneContextHelper.get_mac_address()
 372             portlist[key].update({'mac': mac})
 373         self.networks = portlist
 374         LOG.info("Ports %s", self.networks)
 375
 376     def _enable_interfaces(self, index, vfs, xml_str):
 377         vpath = self.ovs_properties.get("vpath", "/usr/local")
 378         vf = self.networks[vfs[0]]
 379         port_num = vf.get('port_num', 0)
 380         vpci = utils.PciAddress(vf['vpci'].strip())
 381         # Generate the vpci for the interfaces
 382         slot = index + port_num + 10
 383         vf['vpci'] = \
 384             "{}:{}:{:02x}.{}".format(vpci.domain, vpci.bus, slot, vpci.function)
 385         return model.Libvirt.add_ovs_interface(
 386             vpath, port_num, vf['vpci'], vf['mac'], xml_str)
 387
 388     def setup_ovs_dpdk_context(self):
 389         nodes = []
 390
 391         self.configure_nics_for_ovs_dpdk()
 392
 393         for index, (key, vnf) in enumerate(collections.OrderedDict(
 394                 self.servers).items()):
 395             cfg = '/tmp/vm_ovs_%d.xml' % index
 396             vm_name = "vm-%d" % index
 397             cdrom_img = "/var/lib/libvirt/images/cdrom-%d.img" % index
 398
 399             # 1. Check and delete VM if already exists
 400             model.Libvirt.check_if_vm_exists_and_delete(vm_name,
 401                                                         self.connection)
 402             xml_str, mac = model.Libvirt.build_vm_xml(
 403                 self.connection, self.vm_flavor, vm_name, index, cdrom_img)
 404
 405             # 2: Cleanup already available VMs
 406             for vfs in [vfs for vfs_name, vfs in vnf["network_ports"].items()
 407                         if vfs_name != 'mgmt']:
 408                 xml_str = self._enable_interfaces(index, vfs, xml_str)
 409
 410             # copy xml to target...
 411             model.Libvirt.write_file(cfg, xml_str)
 412             self.connection.put(cfg, cfg)
 413
 414             node = self.vnf_node.generate_vnf_instance(self.vm_flavor,
 415                                                        self.networks,
 416                                                        self.host_mgmt.get('ip'),
 417                                                        key, vnf, mac)
 418             # Generate public/private keys if password or private key file is not provided
 419             node = model.StandaloneContextHelper.check_update_key(self.connection,
 420                                                                   node,
 421                                                                   vm_name,
 422                                                                   self.name,
 423                                                                   cdrom_img,
 424                                                                   mac)
 425
 426             # store vnf node details
 427             nodes.append(node)
 428
 429             # NOTE: launch through libvirt
 430             LOG.info("virsh create ...")
 431             model.Libvirt.virsh_create_vm(self.connection, cfg)
 432
 433             self.vm_names.append(vm_name)
 434
 435         return nodes