yardstick/benchmark/contexts/standalone/ovs_dpdk.py

   1 # Copyright (c) 2016-2017 Intel Corporation
   2 #
   3 # Licensed under the Apache License, Version 2.0 (the "License");
   4 # you may not use this file except in compliance with the License.
   5 # You may obtain a copy of the License at
   6 #
   7 #      http://www.apache.org/licenses/LICENSE-2.0
   8 #
   9 # Unless required by applicable law or agreed to in writing, software
  10 # distributed under the License is distributed on an "AS IS" BASIS,
  11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12 # See the License for the specific language governing permissions and
  13 # limitations under the License.
  14
  15 import io
  16 import collections
  17 import logging
  18 import os
  19 import re
  20 import time
  21
  22 from yardstick import ssh
  23 from yardstick.benchmark import contexts
  24 from yardstick.benchmark.contexts import base
  25 from yardstick.benchmark.contexts.standalone import model
  26 from yardstick.common import exceptions
  27 from yardstick.network_services import utils
  28 from yardstick.network_services.utils import get_nsb_option
  29
  30
  31 LOG = logging.getLogger(__name__)
  32
  33 MAIN_BRIDGE = 'br0'
  34
  35
  36 class OvsDpdkContext(base.Context):
  37     """ This class handles OVS standalone nodes - VM running on Non-Managed NFVi
  38     Configuration: ovs_dpdk
  39     """
  40
  41     __context_type__ = contexts.CONTEXT_STANDALONEOVSDPDK
  42
  43     SUPPORTED_OVS_TO_DPDK_MAP = {
  44         '2.6.0': '16.07.1',
  45         '2.6.1': '16.07.2',
  46         '2.7.0': '16.11.1',
  47         '2.7.1': '16.11.2',
  48         '2.7.2': '16.11.3',
  49         '2.8.0': '17.05.2'
  50     }
  51
  52     DEFAULT_OVS = '2.6.0'
  53     CMD_TIMEOUT = 30
  54     DEFAULT_USER_PATH = '/usr/local'
  55
  56     def __init__(self):
  57         self.file_path = None
  58         self.sriov = []
  59         self.first_run = True
  60         self.dpdk_devbind = os.path.join(get_nsb_option('bin_path'),
  61                                          'dpdk-devbind.py')
  62         self.vm_names = []
  63         self.nfvi_host = []
  64         self.nodes = []
  65         self.networks = {}
  66         self.attrs = {}
  67         self.vm_flavor = None
  68         self.servers = None
  69         self.helper = model.StandaloneContextHelper()
  70         self.vnf_node = model.Server()
  71         self.ovs_properties = {}
  72         self.wait_for_vswitchd = 10
  73         super(OvsDpdkContext, self).__init__()
  74
  75     def init(self, attrs):
  76         """initializes itself from the supplied arguments"""
  77         super(OvsDpdkContext, self).init(attrs)
  78
  79         self.file_path = attrs.get("file", "pod.yaml")
  80
  81         self.nodes, self.nfvi_host, self.host_mgmt = \
  82             self.helper.parse_pod_file(self.file_path, 'OvsDpdk')
  83
  84         self.attrs = attrs
  85         self.vm_flavor = attrs.get('flavor', {})
  86         self.servers = attrs.get('servers', {})
  87         self.vm_deploy = attrs.get("vm_deploy", True)
  88         self.ovs_properties = attrs.get('ovs_properties', {})
  89         # add optional static network definition
  90         self.networks = attrs.get("networks", {})
  91
  92         LOG.debug("Nodes: %r", self.nodes)
  93         LOG.debug("NFVi Node: %r", self.nfvi_host)
  94         LOG.debug("Networks: %r", self.networks)
  95
  96     def setup_ovs(self):
  97         """Initialize OVS-DPDK"""
  98         vpath = self.ovs_properties.get('vpath', self.DEFAULT_USER_PATH)
  99         create_from = os.path.join(vpath, 'etc/openvswitch/conf.db')
 100         create_to = os.path.join(vpath, 'share/openvswitch/vswitch.ovsschema')
 101
 102         cmd_list = [
 103             'killall -r "ovs.*" -q | true',
 104             'mkdir -p {0}/etc/openvswitch'.format(vpath),
 105             'mkdir -p {0}/var/run/openvswitch'.format(vpath),
 106             'rm {0}/etc/openvswitch/conf.db | true'.format(vpath),
 107             'ovsdb-tool create {0} {1}'.format(create_from, create_to),
 108             'modprobe vfio-pci',
 109             'chmod a+x /dev/vfio',
 110             'chmod 0666 /dev/vfio/*',
 111         ]
 112
 113         bind_cmd = '%s --force -b vfio-pci {port}' % self.dpdk_devbind
 114         for port in self.networks.values():
 115             cmd_list.append(bind_cmd.format(port=port.get('phy_port')))
 116
 117         for cmd in cmd_list:
 118             LOG.info(cmd)
 119             exit_status, _, stderr = self.connection.execute(
 120                 cmd, timeout=self.CMD_TIMEOUT)
 121             if exit_status:
 122                 raise exceptions.OVSSetupError(command=cmd, error=stderr)
 123
 124     def start_ovs_serverswitch(self):
 125         vpath = self.ovs_properties.get("vpath")
 126         pmd_nums = int(self.ovs_properties.get("pmd_threads", 2))
 127         ovs_sock_path = '/var/run/openvswitch/db.sock'
 128         log_path = '/var/log/openvswitch/ovs-vswitchd.log'
 129
 130         pmd_cpu_mask = self.ovs_properties.get("pmd_cpu_mask", '')
 131         pmd_mask = hex(sum(2 ** num for num in range(pmd_nums)) << 1)
 132         if pmd_cpu_mask:
 133             pmd_mask = pmd_cpu_mask
 134
 135         socket0 = self.ovs_properties.get("ram", {}).get("socket_0", "2048")
 136         socket1 = self.ovs_properties.get("ram", {}).get("socket_1", "2048")
 137
 138         ovs_other_config = "ovs-vsctl {0}set Open_vSwitch . other_config:{1}"
 139         detach_cmd = "ovs-vswitchd unix:{0}{1} --pidfile --detach --log-file={2}"
 140
 141         lcore_mask = self.ovs_properties.get("lcore_mask", '')
 142         if lcore_mask:
 143             lcore_mask = ovs_other_config.format("--no-wait ", "dpdk-lcore-mask='%s'" % lcore_mask)
 144
 145         cmd_list = [
 146             "mkdir -p /usr/local/var/run/openvswitch",
 147             "mkdir -p {}".format(os.path.dirname(log_path)),
 148             "ovsdb-server --remote=punix:/{0}/{1}  --pidfile --detach".format(vpath,
 149                                                                               ovs_sock_path),
 150             ovs_other_config.format("--no-wait ", "dpdk-init=true"),
 151             ovs_other_config.format("--no-wait ", "dpdk-socket-mem='%s,%s'" % (socket0, socket1)),
 152             lcore_mask,
 153             detach_cmd.format(vpath, ovs_sock_path, log_path),
 154             ovs_other_config.format("", "pmd-cpu-mask=%s" % pmd_mask),
 155         ]
 156
 157         for cmd in cmd_list:
 158             LOG.info(cmd)
 159             self.connection.execute(cmd)
 160         time.sleep(self.wait_for_vswitchd)
 161
 162     def setup_ovs_bridge_add_flows(self):
 163         dpdk_args = ""
 164         dpdk_list = []
 165         vpath = self.ovs_properties.get("vpath", "/usr/local")
 166         version = self.ovs_properties.get('version', {})
 167         ovs_ver = [int(x) for x in version.get('ovs', self.DEFAULT_OVS).split('.')]
 168         ovs_add_port = ('ovs-vsctl add-port {br} {port} -- '
 169                         'set Interface {port} type={type_}{dpdk_args}')
 170         ovs_add_queue = 'ovs-vsctl set Interface {port} options:n_rxq={queue}'
 171         chmod_vpath = 'chmod 0777 {0}/var/run/openvswitch/dpdkvhostuser*'
 172
 173         cmd_list = [
 174             'ovs-vsctl --if-exists del-br {0}'.format(MAIN_BRIDGE),
 175             'rm -rf {0}/var/run/openvswitch/dpdkvhostuser*'.format(vpath),
 176             'ovs-vsctl add-br {0} -- set bridge {0} datapath_type=netdev'.
 177             format(MAIN_BRIDGE)
 178         ]
 179
 180         ordered_network = collections.OrderedDict(self.networks)
 181         for index, vnf in enumerate(ordered_network.values()):
 182             if ovs_ver >= [2, 7, 0]:
 183                 dpdk_args = " options:dpdk-devargs=%s" % vnf.get("phy_port")
 184             dpdk_list.append(ovs_add_port.format(
 185                 br=MAIN_BRIDGE, port='dpdk%s' % vnf.get("port_num", 0),
 186                 type_='dpdk', dpdk_args=dpdk_args))
 187             dpdk_list.append(ovs_add_queue.format(
 188                 port='dpdk%s' % vnf.get("port_num", 0),
 189                 queue=self.ovs_properties.get("queues", 1)))
 190
 191         # Sorting the array to make sure we execute dpdk0... in the order
 192         list.sort(dpdk_list)
 193         cmd_list.extend(dpdk_list)
 194
 195         # Need to do two for loop to maintain the dpdk/vhost ports.
 196         for index, _ in enumerate(ordered_network):
 197             cmd_list.append(ovs_add_port.format(
 198                 br=MAIN_BRIDGE, port='dpdkvhostuser%s' % index,
 199                 type_='dpdkvhostuser', dpdk_args=""))
 200
 201         ovs_flow = ("ovs-ofctl add-flow {0} in_port=%s,action=output:%s".
 202                     format(MAIN_BRIDGE))
 203         network_count = len(ordered_network) + 1
 204         for in_port, out_port in zip(range(1, network_count),
 205                                      range(network_count, network_count * 2)):
 206             cmd_list.append(ovs_flow % (in_port, out_port))
 207             cmd_list.append(ovs_flow % (out_port, in_port))
 208
 209         cmd_list.append(chmod_vpath.format(vpath))
 210
 211         for cmd in cmd_list:
 212             LOG.info(cmd)
 213             exit_status, _, stderr = self.connection.execute(
 214                 cmd, timeout=self.CMD_TIMEOUT)
 215             if exit_status:
 216                 raise exceptions.OVSSetupError(command=cmd, error=stderr)
 217
 218     def _check_hugepages(self):
 219         meminfo = io.BytesIO()
 220         self.connection.get_file_obj('/proc/meminfo', meminfo)
 221         regex = re.compile(r"HugePages_Total:\s+(?P<hp_total>\d+)[\n\r]"
 222                            r"HugePages_Free:\s+(?P<hp_free>\d+)")
 223         match = regex.search(meminfo.getvalue().decode('utf-8'))
 224         if not match:
 225             raise exceptions.OVSHugepagesInfoError()
 226         if int(match.group('hp_total')) == 0:
 227             raise exceptions.OVSHugepagesNotConfigured()
 228         if int(match.group('hp_free')) == 0:
 229             raise exceptions.OVSHugepagesZeroFree(
 230                 total_hugepages=int(match.group('hp_total')))
 231
 232     def cleanup_ovs_dpdk_env(self):
 233         self.connection.execute(
 234             'ovs-vsctl --if-exists del-br {0}'.format(MAIN_BRIDGE))
 235         self.connection.execute("pkill -9 ovs")
 236
 237     def check_ovs_dpdk_env(self):
 238         self.cleanup_ovs_dpdk_env()
 239         self._check_hugepages()
 240
 241         version = self.ovs_properties.get("version", {})
 242         ovs_ver = version.get("ovs", self.DEFAULT_OVS)
 243         dpdk_ver = version.get("dpdk", "16.07.2").split('.')
 244
 245         supported_version = self.SUPPORTED_OVS_TO_DPDK_MAP.get(ovs_ver, None)
 246         if supported_version is None or supported_version.split('.')[:2] != dpdk_ver[:2]:
 247             raise exceptions.OVSUnsupportedVersion(
 248                 ovs_version=ovs_ver,
 249                 ovs_to_dpdk_map=self.SUPPORTED_OVS_TO_DPDK_MAP)
 250
 251         status = self.connection.execute("ovs-vsctl -V | grep -i '%s'" % ovs_ver)[0]
 252         if status:
 253             deploy = model.OvsDeploy(self.connection,
 254                                      utils.get_nsb_option("bin_path"),
 255                                      self.ovs_properties)
 256             deploy.ovs_deploy()
 257
 258     def deploy(self):
 259         """don't need to deploy"""
 260
 261         # Todo: NFVi deploy (sriov, vswitch, ovs etc) based on the config.
 262         if not self.vm_deploy:
 263             return
 264
 265         self.connection = ssh.SSH.from_node(self.host_mgmt)
 266
 267         # Check dpdk/ovs version, if not present install
 268         self.check_ovs_dpdk_env()
 269         #    Todo: NFVi deploy (sriov, vswitch, ovs etc) based on the config.
 270         model.StandaloneContextHelper.install_req_libs(self.connection)
 271         self.networks = model.StandaloneContextHelper.get_nic_details(
 272             self.connection, self.networks, self.dpdk_devbind)
 273
 274         self.setup_ovs()
 275         self.start_ovs_serverswitch()
 276         self.setup_ovs_bridge_add_flows()
 277         self.nodes = self.setup_ovs_dpdk_context()
 278         LOG.debug("Waiting for VM to come up...")
 279         self.nodes = model.StandaloneContextHelper.wait_for_vnfs_to_start(
 280             self.connection, self.servers, self.nodes)
 281
 282     def undeploy(self):
 283
 284         if not self.vm_deploy:
 285             return
 286
 287         # Cleanup the ovs installation...
 288         self.cleanup_ovs_dpdk_env()
 289
 290         # Bind nics back to kernel
 291         bind_cmd = "{dpdk_devbind} --force -b {driver} {port}"
 292         for port in self.networks.values():
 293             vpci = port.get("phy_port")
 294             phy_driver = port.get("driver")
 295             self.connection.execute(bind_cmd.format(
 296                 dpdk_devbind=self.dpdk_devbind, driver=phy_driver, port=vpci))
 297
 298         # Todo: NFVi undeploy (sriov, vswitch, ovs etc) based on the config.
 299         for vm in self.vm_names:
 300             model.Libvirt.check_if_vm_exists_and_delete(vm, self.connection)
 301
 302     def _get_physical_nodes(self):
 303         return self.nfvi_host
 304
 305     def _get_physical_node_for_server(self, server_name):
 306         node_name, ctx_name = self.split_host_name(server_name)
 307         if ctx_name is None or self.name != ctx_name:
 308             return None
 309
 310         matching_nodes = [s for s in self.servers if s == node_name]
 311         if len(matching_nodes) == 0:
 312             return None
 313
 314         # self.nfvi_host always contain only one host
 315         return "{}.{}".format(self.nfvi_host[0]["name"], self._name)
 316
 317     def _get_server(self, attr_name):
 318         """lookup server info by name from context
 319
 320         Keyword arguments:
 321         attr_name -- A name for a server listed in nodes config file
 322         """
 323         node_name, name = self.split_host_name(attr_name)
 324         if name is None or self.name != name:
 325             return None
 326
 327         matching_nodes = (n for n in self.nodes if n["name"] == node_name)
 328         try:
 329             # A clone is created in order to avoid affecting the
 330             # original one.
 331             node = dict(next(matching_nodes))
 332         except StopIteration:
 333             return None
 334
 335         try:
 336             duplicate = next(matching_nodes)
 337         except StopIteration:
 338             pass
 339         else:
 340             raise ValueError("Duplicate nodes!!! Nodes: %s %s" % (node, duplicate))
 341
 342         node["name"] = attr_name
 343         return node
 344
 345     def _get_network(self, attr_name):
 346         if not isinstance(attr_name, collections.Mapping):
 347             network = self.networks.get(attr_name)
 348
 349         else:
 350             # Don't generalize too much  Just support vld_id
 351             vld_id = attr_name.get('vld_id', {})
 352             # for standalone context networks are dicts
 353             iter1 = (n for n in self.networks.values() if n.get('vld_id') == vld_id)
 354             network = next(iter1, None)
 355
 356         if network is None:
 357             return None
 358
 359         result = {
 360             # name is required
 361             "name": network["name"],
 362             "vld_id": network.get("vld_id"),
 363             "segmentation_id": network.get("segmentation_id"),
 364             "network_type": network.get("network_type"),
 365             "physical_network": network.get("physical_network"),
 366         }
 367         return result
 368
 369     def configure_nics_for_ovs_dpdk(self):
 370         portlist = collections.OrderedDict(self.networks)
 371         for key in portlist:
 372             mac = model.StandaloneContextHelper.get_mac_address()
 373             portlist[key].update({'mac': mac})
 374         self.networks = portlist
 375         LOG.info("Ports %s", self.networks)
 376
 377     def _enable_interfaces(self, index, vfs, xml_str):
 378         vpath = self.ovs_properties.get("vpath", "/usr/local")
 379         vf = self.networks[vfs[0]]
 380         port_num = vf.get('port_num', 0)
 381         vpci = utils.PciAddress(vf['vpci'].strip())
 382         # Generate the vpci for the interfaces
 383         slot = index + port_num + 10
 384         vf['vpci'] = \
 385             "{}:{}:{:02x}.{}".format(vpci.domain, vpci.bus, slot, vpci.function)
 386         return model.Libvirt.add_ovs_interface(
 387             vpath, port_num, vf['vpci'], vf['mac'], xml_str)
 388
 389     def setup_ovs_dpdk_context(self):
 390         nodes = []
 391
 392         self.configure_nics_for_ovs_dpdk()
 393
 394         for index, (key, vnf) in enumerate(collections.OrderedDict(
 395                 self.servers).items()):
 396             cfg = '/tmp/vm_ovs_%d.xml' % index
 397             vm_name = "vm_%d" % index
 398
 399             # 1. Check and delete VM if already exists
 400             model.Libvirt.check_if_vm_exists_and_delete(vm_name,
 401                                                         self.connection)
 402             xml_str, mac = model.Libvirt.build_vm_xml(
 403                 self.connection, self.vm_flavor, vm_name, index)
 404
 405             # 2: Cleanup already available VMs
 406             for vfs in [vfs for vfs_name, vfs in vnf["network_ports"].items()
 407                         if vfs_name != 'mgmt']:
 408                 xml_str = self._enable_interfaces(index, vfs, xml_str)
 409
 410             # copy xml to target...
 411             model.Libvirt.write_file(cfg, xml_str)
 412             self.connection.put(cfg, cfg)
 413
 414             # NOTE: launch through libvirt
 415             LOG.info("virsh create ...")
 416             model.Libvirt.virsh_create_vm(self.connection, cfg)
 417
 418             self.vm_names.append(vm_name)
 419
 420             # build vnf node details
 421             nodes.append(self.vnf_node.generate_vnf_instance(self.vm_flavor,
 422                                                              self.networks,
 423                                                              self.host_mgmt.get('ip'),
 424                                                              key, vnf, mac))
 425
 426         return nodes