yardstick/benchmark/contexts/standalone/ovs_dpdk.py

   1 # Copyright (c) 2016-2017 Intel Corporation
   2 #
   3 # Licensed under the Apache License, Version 2.0 (the "License");
   4 # you may not use this file except in compliance with the License.
   5 # You may obtain a copy of the License at
   6 #
   7 #      http://www.apache.org/licenses/LICENSE-2.0
   8 #
   9 # Unless required by applicable law or agreed to in writing, software
  10 # distributed under the License is distributed on an "AS IS" BASIS,
  11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12 # See the License for the specific language governing permissions and
  13 # limitations under the License.
  14
  15 import io
  16 import collections
  17 import logging
  18 import os
  19 import re
  20 import time
  21
  22 from yardstick import ssh
  23 from yardstick.benchmark import contexts
  24 from yardstick.benchmark.contexts import base
  25 from yardstick.benchmark.contexts.standalone import model
  26 from yardstick.common import exceptions
  27 from yardstick.network_services import utils
  28 from yardstick.network_services.utils import get_nsb_option
  29
  30
  31 LOG = logging.getLogger(__name__)
  32
  33 MAIN_BRIDGE = 'br0'
  34
  35
  36 class OvsDpdkContext(base.Context):
  37     """ This class handles OVS standalone nodes - VM running on Non-Managed NFVi
  38     Configuration: ovs_dpdk
  39     """
  40
  41     __context_type__ = contexts.CONTEXT_STANDALONEOVSDPDK
  42
  43     SUPPORTED_OVS_TO_DPDK_MAP = {
  44         '2.6.0': '16.07.1',
  45         '2.6.1': '16.07.2',
  46         '2.7.0': '16.11.1',
  47         '2.7.1': '16.11.2',
  48         '2.7.2': '16.11.3',
  49         '2.8.0': '17.05.2'
  50     }
  51
  52     DEFAULT_OVS = '2.6.0'
  53     CMD_TIMEOUT = 30
  54     DEFAULT_USER_PATH = '/usr/local'
  55
  56     def __init__(self):
  57         self.file_path = None
  58         self.sriov = []
  59         self.first_run = True
  60         self.dpdk_devbind = os.path.join(get_nsb_option('bin_path'),
  61                                          'dpdk-devbind.py')
  62         self.vm_names = []
  63         self.nfvi_host = []
  64         self.nodes = []
  65         self.networks = {}
  66         self.attrs = {}
  67         self.vm_flavor = None
  68         self.servers = None
  69         self.helper = model.StandaloneContextHelper()
  70         self.vnf_node = model.Server()
  71         self.ovs_properties = {}
  72         self.wait_for_vswitchd = 10
  73         super(OvsDpdkContext, self).__init__()
  74
  75     def init(self, attrs):
  76         """initializes itself from the supplied arguments"""
  77         super(OvsDpdkContext, self).init(attrs)
  78
  79         self.file_path = attrs.get("file", "pod.yaml")
  80
  81         self.nodes, self.nfvi_host, self.host_mgmt = \
  82             self.helper.parse_pod_file(self.file_path, 'OvsDpdk')
  83
  84         self.attrs = attrs
  85         self.vm_flavor = attrs.get('flavor', {})
  86         self.servers = attrs.get('servers', {})
  87         self.vm_deploy = attrs.get("vm_deploy", True)
  88         self.ovs_properties = attrs.get('ovs_properties', {})
  89         # add optional static network definition
  90         self.networks = attrs.get("networks", {})
  91
  92         LOG.debug("Nodes: %r", self.nodes)
  93         LOG.debug("NFVi Node: %r", self.nfvi_host)
  94         LOG.debug("Networks: %r", self.networks)
  95
  96     def setup_ovs(self):
  97         """Initialize OVS-DPDK"""
  98         vpath = self.ovs_properties.get('vpath', self.DEFAULT_USER_PATH)
  99         create_from = os.path.join(vpath, 'etc/openvswitch/conf.db')
 100         create_to = os.path.join(vpath, 'share/openvswitch/vswitch.ovsschema')
 101
 102         cmd_list = [
 103             'killall -r "ovs.*" -q | true',
 104             'mkdir -p {0}/etc/openvswitch'.format(vpath),
 105             'mkdir -p {0}/var/run/openvswitch'.format(vpath),
 106             'rm {0}/etc/openvswitch/conf.db | true'.format(vpath),
 107             'ovsdb-tool create {0} {1}'.format(create_from, create_to),
 108             'modprobe vfio-pci',
 109             'chmod a+x /dev/vfio',
 110             'chmod 0666 /dev/vfio/*',
 111         ]
 112
 113         bind_cmd = '%s --force -b vfio-pci {port}' % self.dpdk_devbind
 114         for port in self.networks.values():
 115             cmd_list.append(bind_cmd.format(port=port.get('phy_port')))
 116
 117         for cmd in cmd_list:
 118             LOG.info(cmd)
 119             exit_status, _, stderr = self.connection.execute(
 120                 cmd, timeout=self.CMD_TIMEOUT)
 121             if exit_status:
 122                 raise exceptions.OVSSetupError(command=cmd, error=stderr)
 123
 124     def start_ovs_serverswitch(self):
 125         vpath = self.ovs_properties.get("vpath")
 126         pmd_nums = int(self.ovs_properties.get("pmd_threads", 2))
 127         ovs_sock_path = '/var/run/openvswitch/db.sock'
 128         log_path = '/var/log/openvswitch/ovs-vswitchd.log'
 129
 130         pmd_cpu_mask = self.ovs_properties.get("pmd_cpu_mask", '')
 131         pmd_mask = hex(sum(2 ** num for num in range(pmd_nums)) << 1)
 132         if pmd_cpu_mask:
 133             pmd_mask = pmd_cpu_mask
 134
 135         socket0 = self.ovs_properties.get("ram", {}).get("socket_0", "2048")
 136         socket1 = self.ovs_properties.get("ram", {}).get("socket_1", "2048")
 137
 138         ovs_other_config = "ovs-vsctl {0}set Open_vSwitch . other_config:{1}"
 139         detach_cmd = "ovs-vswitchd unix:{0}{1} --pidfile --detach --log-file={2}"
 140
 141         lcore_mask = self.ovs_properties.get("lcore_mask", '')
 142         if lcore_mask:
 143             lcore_mask = ovs_other_config.format("--no-wait ", "dpdk-lcore-mask='%s'" % lcore_mask)
 144
 145         cmd_list = [
 146             "mkdir -p /usr/local/var/run/openvswitch",
 147             "mkdir -p {}".format(os.path.dirname(log_path)),
 148             "ovsdb-server --remote=punix:/{0}/{1}  --pidfile --detach".format(vpath,
 149                                                                               ovs_sock_path),
 150             ovs_other_config.format("--no-wait ", "dpdk-init=true"),
 151             ovs_other_config.format("--no-wait ", "dpdk-socket-mem='%s,%s'" % (socket0, socket1)),
 152             lcore_mask,
 153             detach_cmd.format(vpath, ovs_sock_path, log_path),
 154             ovs_other_config.format("", "pmd-cpu-mask=%s" % pmd_mask),
 155         ]
 156
 157         for cmd in cmd_list:
 158             LOG.info(cmd)
 159             self.connection.execute(cmd)
 160         time.sleep(self.wait_for_vswitchd)
 161
 162     def setup_ovs_bridge_add_flows(self):
 163         dpdk_args = ""
 164         dpdk_list = []
 165         vpath = self.ovs_properties.get("vpath", "/usr/local")
 166         version = self.ovs_properties.get('version', {})
 167         ovs_ver = [int(x) for x in version.get('ovs', self.DEFAULT_OVS).split('.')]
 168         ovs_add_port = ('ovs-vsctl add-port {br} {port} -- '
 169                         'set Interface {port} type={type_}{dpdk_args}{dpdk_rxq}')
 170         chmod_vpath = 'chmod 0777 {0}/var/run/openvswitch/dpdkvhostuser*'
 171
 172         cmd_list = [
 173             'ovs-vsctl --if-exists del-br {0}'.format(MAIN_BRIDGE),
 174             'rm -rf {0}/var/run/openvswitch/dpdkvhostuser*'.format(vpath),
 175             'ovs-vsctl add-br {0} -- set bridge {0} datapath_type=netdev'.
 176             format(MAIN_BRIDGE)
 177         ]
 178         dpdk_rxq = " options:n_rxq={queue}".format(
 179             queue=self.ovs_properties.get("queues", 1))
 180
 181         ordered_network = collections.OrderedDict(self.networks)
 182         for index, vnf in enumerate(ordered_network.values()):
 183             if ovs_ver >= [2, 7, 0]:
 184                 dpdk_args = " options:dpdk-devargs=%s" % vnf.get("phy_port")
 185             dpdk_list.append(ovs_add_port.format(
 186                 br=MAIN_BRIDGE, port='dpdk%s' % vnf.get("port_num", 0),
 187                 type_='dpdk', dpdk_args=dpdk_args, dpdk_rxq=dpdk_rxq))
 188
 189         # Sorting the array to make sure we execute dpdk0... in the order
 190         list.sort(dpdk_list)
 191         cmd_list.extend(dpdk_list)
 192
 193         # Need to do two for loop to maintain the dpdk/vhost ports.
 194         for index, _ in enumerate(ordered_network):
 195             cmd_list.append(ovs_add_port.format(
 196                 br=MAIN_BRIDGE, port='dpdkvhostuser%s' % index,
 197                 type_='dpdkvhostuser', dpdk_args="", dpdk_rxq=""))
 198
 199         ovs_flow = ("ovs-ofctl add-flow {0} in_port=%s,action=output:%s".
 200                     format(MAIN_BRIDGE))
 201         network_count = len(ordered_network) + 1
 202         for in_port, out_port in zip(range(1, network_count),
 203                                      range(network_count, network_count * 2)):
 204             cmd_list.append(ovs_flow % (in_port, out_port))
 205             cmd_list.append(ovs_flow % (out_port, in_port))
 206
 207         cmd_list.append(chmod_vpath.format(vpath))
 208
 209         for cmd in cmd_list:
 210             LOG.info(cmd)
 211             exit_status, _, stderr = self.connection.execute(
 212                 cmd, timeout=self.CMD_TIMEOUT)
 213             if exit_status:
 214                 raise exceptions.OVSSetupError(command=cmd, error=stderr)
 215
 216     def _check_hugepages(self):
 217         meminfo = io.BytesIO()
 218         self.connection.get_file_obj('/proc/meminfo', meminfo)
 219         regex = re.compile(r"HugePages_Total:\s+(?P<hp_total>\d+)[\n\r]"
 220                            r"HugePages_Free:\s+(?P<hp_free>\d+)")
 221         match = regex.search(meminfo.getvalue().decode('utf-8'))
 222         if not match:
 223             raise exceptions.OVSHugepagesInfoError()
 224         if int(match.group('hp_total')) == 0:
 225             raise exceptions.OVSHugepagesNotConfigured()
 226         if int(match.group('hp_free')) == 0:
 227             raise exceptions.OVSHugepagesZeroFree(
 228                 total_hugepages=int(match.group('hp_total')))
 229
 230     def cleanup_ovs_dpdk_env(self):
 231         self.connection.execute(
 232             'ovs-vsctl --if-exists del-br {0}'.format(MAIN_BRIDGE))
 233         self.connection.execute("pkill -9 ovs")
 234
 235     def check_ovs_dpdk_env(self):
 236         self.cleanup_ovs_dpdk_env()
 237         self._check_hugepages()
 238
 239         version = self.ovs_properties.get("version", {})
 240         ovs_ver = version.get("ovs", self.DEFAULT_OVS)
 241         dpdk_ver = version.get("dpdk", "16.07.2").split('.')
 242
 243         supported_version = self.SUPPORTED_OVS_TO_DPDK_MAP.get(ovs_ver, None)
 244         if supported_version is None or supported_version.split('.')[:2] != dpdk_ver[:2]:
 245             raise exceptions.OVSUnsupportedVersion(
 246                 ovs_version=ovs_ver,
 247                 ovs_to_dpdk_map=self.SUPPORTED_OVS_TO_DPDK_MAP)
 248
 249         status = self.connection.execute("ovs-vsctl -V | grep -i '%s'" % ovs_ver)[0]
 250         if status:
 251             deploy = model.OvsDeploy(self.connection,
 252                                      utils.get_nsb_option("bin_path"),
 253                                      self.ovs_properties)
 254             deploy.ovs_deploy()
 255
 256     def deploy(self):
 257         """don't need to deploy"""
 258
 259         # Todo: NFVi deploy (sriov, vswitch, ovs etc) based on the config.
 260         if not self.vm_deploy:
 261             return
 262
 263         self.connection = ssh.SSH.from_node(self.host_mgmt)
 264
 265         # Check dpdk/ovs version, if not present install
 266         self.check_ovs_dpdk_env()
 267         #    Todo: NFVi deploy (sriov, vswitch, ovs etc) based on the config.
 268         model.StandaloneContextHelper.install_req_libs(self.connection)
 269         self.networks = model.StandaloneContextHelper.get_nic_details(
 270             self.connection, self.networks, self.dpdk_devbind)
 271
 272         self.setup_ovs()
 273         self.start_ovs_serverswitch()
 274         self.setup_ovs_bridge_add_flows()
 275         self.nodes = self.setup_ovs_dpdk_context()
 276         LOG.debug("Waiting for VM to come up...")
 277         self.nodes = model.StandaloneContextHelper.wait_for_vnfs_to_start(
 278             self.connection, self.servers, self.nodes)
 279
 280     def undeploy(self):
 281
 282         if not self.vm_deploy:
 283             return
 284
 285         # Cleanup the ovs installation...
 286         self.cleanup_ovs_dpdk_env()
 287
 288         # Bind nics back to kernel
 289         bind_cmd = "{dpdk_devbind} --force -b {driver} {port}"
 290         for port in self.networks.values():
 291             vpci = port.get("phy_port")
 292             phy_driver = port.get("driver")
 293             self.connection.execute(bind_cmd.format(
 294                 dpdk_devbind=self.dpdk_devbind, driver=phy_driver, port=vpci))
 295
 296         # Todo: NFVi undeploy (sriov, vswitch, ovs etc) based on the config.
 297         for vm in self.vm_names:
 298             model.Libvirt.check_if_vm_exists_and_delete(vm, self.connection)
 299
 300     def _get_physical_nodes(self):
 301         return self.nfvi_host
 302
 303     def _get_physical_node_for_server(self, server_name):
 304         node_name, ctx_name = self.split_host_name(server_name)
 305         if ctx_name is None or self.name != ctx_name:
 306             return None
 307
 308         matching_nodes = [s for s in self.servers if s == node_name]
 309         if len(matching_nodes) == 0:
 310             return None
 311
 312         # self.nfvi_host always contain only one host
 313         return "{}.{}".format(self.nfvi_host[0]["name"], self._name)
 314
 315     def _get_server(self, attr_name):
 316         """lookup server info by name from context
 317
 318         Keyword arguments:
 319         attr_name -- A name for a server listed in nodes config file
 320         """
 321         node_name, name = self.split_host_name(attr_name)
 322         if name is None or self.name != name:
 323             return None
 324
 325         matching_nodes = (n for n in self.nodes if n["name"] == node_name)
 326         try:
 327             # A clone is created in order to avoid affecting the
 328             # original one.
 329             node = dict(next(matching_nodes))
 330         except StopIteration:
 331             return None
 332
 333         try:
 334             duplicate = next(matching_nodes)
 335         except StopIteration:
 336             pass
 337         else:
 338             raise ValueError("Duplicate nodes!!! Nodes: %s %s" % (node, duplicate))
 339
 340         node["name"] = attr_name
 341         return node
 342
 343     def _get_network(self, attr_name):
 344         if not isinstance(attr_name, collections.Mapping):
 345             network = self.networks.get(attr_name)
 346
 347         else:
 348             # Don't generalize too much  Just support vld_id
 349             vld_id = attr_name.get('vld_id', {})
 350             # for standalone context networks are dicts
 351             iter1 = (n for n in self.networks.values() if n.get('vld_id') == vld_id)
 352             network = next(iter1, None)
 353
 354         if network is None:
 355             return None
 356
 357         result = {
 358             # name is required
 359             "name": network["name"],
 360             "vld_id": network.get("vld_id"),
 361             "segmentation_id": network.get("segmentation_id"),
 362             "network_type": network.get("network_type"),
 363             "physical_network": network.get("physical_network"),
 364         }
 365         return result
 366
 367     def configure_nics_for_ovs_dpdk(self):
 368         portlist = collections.OrderedDict(self.networks)
 369         for key in portlist:
 370             mac = model.StandaloneContextHelper.get_mac_address()
 371             portlist[key].update({'mac': mac})
 372         self.networks = portlist
 373         LOG.info("Ports %s", self.networks)
 374
 375     def _enable_interfaces(self, index, vfs, xml_str):
 376         vpath = self.ovs_properties.get("vpath", "/usr/local")
 377         vf = self.networks[vfs[0]]
 378         port_num = vf.get('port_num', 0)
 379         vpci = utils.PciAddress(vf['vpci'].strip())
 380         # Generate the vpci for the interfaces
 381         slot = index + port_num + 10
 382         vf['vpci'] = \
 383             "{}:{}:{:02x}.{}".format(vpci.domain, vpci.bus, slot, vpci.function)
 384         return model.Libvirt.add_ovs_interface(
 385             vpath, port_num, vf['vpci'], vf['mac'], xml_str)
 386
 387     def setup_ovs_dpdk_context(self):
 388         nodes = []
 389
 390         self.configure_nics_for_ovs_dpdk()
 391
 392         for index, (key, vnf) in enumerate(collections.OrderedDict(
 393                 self.servers).items()):
 394             cfg = '/tmp/vm_ovs_%d.xml' % index
 395             vm_name = "vm-%d" % index
 396             cdrom_img = "/var/lib/libvirt/images/cdrom-%d.img" % index
 397
 398             # 1. Check and delete VM if already exists
 399             model.Libvirt.check_if_vm_exists_and_delete(vm_name,
 400                                                         self.connection)
 401             xml_str, mac = model.Libvirt.build_vm_xml(
 402                 self.connection, self.vm_flavor, vm_name, index, cdrom_img)
 403
 404             # 2: Cleanup already available VMs
 405             for vfs in [vfs for vfs_name, vfs in vnf["network_ports"].items()
 406                         if vfs_name != 'mgmt']:
 407                 xml_str = self._enable_interfaces(index, vfs, xml_str)
 408
 409             # copy xml to target...
 410             model.Libvirt.write_file(cfg, xml_str)
 411             self.connection.put(cfg, cfg)
 412
 413             node = self.vnf_node.generate_vnf_instance(self.vm_flavor,
 414                                                        self.networks,
 415                                                        self.host_mgmt.get('ip'),
 416                                                        key, vnf, mac)
 417             # Generate public/private keys if password or private key file is not provided
 418             node = model.StandaloneContextHelper.check_update_key(self.connection,
 419                                                                   node,
 420                                                                   vm_name,
 421                                                                   self.name,
 422                                                                   cdrom_img,
 423                                                                   mac)
 424
 425             # store vnf node details
 426             nodes.append(node)
 427
 428             # NOTE: launch through libvirt
 429             LOG.info("virsh create ...")
 430             model.Libvirt.virsh_create_vm(self.connection, cfg)
 431
 432             self.vm_names.append(vm_name)
 433
 434         return nodes