yardstick/benchmark/contexts/standalone/ovs_dpdk.py

   1 # Copyright (c) 2016-2017 Intel Corporation
   2 #
   3 # Licensed under the Apache License, Version 2.0 (the "License");
   4 # you may not use this file except in compliance with the License.
   5 # You may obtain a copy of the License at
   6 #
   7 #      http://www.apache.org/licenses/LICENSE-2.0
   8 #
   9 # Unless required by applicable law or agreed to in writing, software
  10 # distributed under the License is distributed on an "AS IS" BASIS,
  11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12 # See the License for the specific language governing permissions and
  13 # limitations under the License.
  14
  15 import io
  16 import collections
  17 import logging
  18 import os
  19 import re
  20 import time
  21
  22 from yardstick import ssh
  23 from yardstick.benchmark.contexts.base import Context
  24 from yardstick.benchmark.contexts.standalone import model
  25 from yardstick.common import exceptions
  26 from yardstick.network_services import utils
  27
  28
  29 LOG = logging.getLogger(__name__)
  30
  31 MAIN_BRIDGE = 'br0'
  32
  33
  34 class OvsDpdkContext(Context):
  35     """ This class handles OVS standalone nodes - VM running on Non-Managed NFVi
  36     Configuration: ovs_dpdk
  37     """
  38
  39     __context_type__ = "StandaloneOvsDpdk"
  40
  41     SUPPORTED_OVS_TO_DPDK_MAP = {
  42         '2.6.0': '16.07.1',
  43         '2.6.1': '16.07.2',
  44         '2.7.0': '16.11.1',
  45         '2.7.1': '16.11.2',
  46         '2.7.2': '16.11.3',
  47         '2.8.0': '17.05.2'
  48     }
  49
  50     DEFAULT_OVS = '2.6.0'
  51     CMD_TIMEOUT = 30
  52     DEFAULT_USER_PATH = '/usr/local'
  53
  54     def __init__(self):
  55         self.file_path = None
  56         self.sriov = []
  57         self.first_run = True
  58         self.dpdk_devbind = ''
  59         self.vm_names = []
  60         self.nfvi_host = []
  61         self.nodes = []
  62         self.networks = {}
  63         self.attrs = {}
  64         self.vm_flavor = None
  65         self.servers = None
  66         self.helper = model.StandaloneContextHelper()
  67         self.vnf_node = model.Server()
  68         self.ovs_properties = {}
  69         self.wait_for_vswitchd = 10
  70         super(OvsDpdkContext, self).__init__()
  71
  72     def init(self, attrs):
  73         """initializes itself from the supplied arguments"""
  74         super(OvsDpdkContext, self).init(attrs)
  75
  76         self.file_path = attrs.get("file", "pod.yaml")
  77
  78         self.nodes, self.nfvi_host, self.host_mgmt = \
  79             self.helper.parse_pod_file(self.file_path, 'OvsDpdk')
  80
  81         self.attrs = attrs
  82         self.vm_flavor = attrs.get('flavor', {})
  83         self.servers = attrs.get('servers', {})
  84         self.vm_deploy = attrs.get("vm_deploy", True)
  85         self.ovs_properties = attrs.get('ovs_properties', {})
  86         # add optional static network definition
  87         self.networks = attrs.get("networks", {})
  88
  89         LOG.debug("Nodes: %r", self.nodes)
  90         LOG.debug("NFVi Node: %r", self.nfvi_host)
  91         LOG.debug("Networks: %r", self.networks)
  92
  93     def setup_ovs(self):
  94         """Initialize OVS-DPDK"""
  95         vpath = self.ovs_properties.get('vpath', self.DEFAULT_USER_PATH)
  96         create_from = os.path.join(vpath, 'etc/openvswitch/conf.db')
  97         create_to = os.path.join(vpath, 'share/openvswitch/vswitch.ovsschema')
  98
  99         cmd_list = [
 100             'killall -r "ovs.*" -q | true',
 101             'mkdir -p {0}/etc/openvswitch'.format(vpath),
 102             'mkdir -p {0}/var/run/openvswitch'.format(vpath),
 103             'rm {0}/etc/openvswitch/conf.db | true'.format(vpath),
 104             'ovsdb-tool create {0} {1}'.format(create_from, create_to),
 105             'modprobe vfio-pci',
 106             'chmod a+x /dev/vfio',
 107             'chmod 0666 /dev/vfio/*',
 108         ]
 109
 110         bind_cmd = '%s --force -b vfio-pci {port}' % self.dpdk_devbind
 111         for port in self.networks.values():
 112             cmd_list.append(bind_cmd.format(port=port.get('phy_port')))
 113
 114         for cmd in cmd_list:
 115             LOG.info(cmd)
 116             exit_status, _, stderr = self.connection.execute(
 117                 cmd, timeout=self.CMD_TIMEOUT)
 118             if exit_status:
 119                 raise exceptions.OVSSetupError(command=cmd, error=stderr)
 120
 121     def start_ovs_serverswitch(self):
 122         vpath = self.ovs_properties.get("vpath")
 123         pmd_nums = int(self.ovs_properties.get("pmd_threads", 2))
 124         ovs_sock_path = '/var/run/openvswitch/db.sock'
 125         log_path = '/var/log/openvswitch/ovs-vswitchd.log'
 126
 127         pmd_cpu_mask = self.ovs_properties.get("pmd_cpu_mask", '')
 128         pmd_mask = hex(sum(2 ** num for num in range(pmd_nums)) << 1)
 129         if pmd_cpu_mask:
 130             pmd_mask = pmd_cpu_mask
 131
 132         socket0 = self.ovs_properties.get("ram", {}).get("socket_0", "2048")
 133         socket1 = self.ovs_properties.get("ram", {}).get("socket_1", "2048")
 134
 135         ovs_other_config = "ovs-vsctl {0}set Open_vSwitch . other_config:{1}"
 136         detach_cmd = "ovs-vswitchd unix:{0}{1} --pidfile --detach --log-file={2}"
 137
 138         lcore_mask = self.ovs_properties.get("lcore_mask", '')
 139         if lcore_mask:
 140             lcore_mask = ovs_other_config.format("--no-wait ", "dpdk-lcore-mask='%s'" % lcore_mask)
 141
 142         cmd_list = [
 143             "mkdir -p /usr/local/var/run/openvswitch",
 144             "mkdir -p {}".format(os.path.dirname(log_path)),
 145             "ovsdb-server --remote=punix:/{0}/{1}  --pidfile --detach".format(vpath,
 146                                                                               ovs_sock_path),
 147             ovs_other_config.format("--no-wait ", "dpdk-init=true"),
 148             ovs_other_config.format("--no-wait ", "dpdk-socket-mem='%s,%s'" % (socket0, socket1)),
 149             lcore_mask,
 150             detach_cmd.format(vpath, ovs_sock_path, log_path),
 151             ovs_other_config.format("", "pmd-cpu-mask=%s" % pmd_mask),
 152         ]
 153
 154         for cmd in cmd_list:
 155             LOG.info(cmd)
 156             self.connection.execute(cmd)
 157         time.sleep(self.wait_for_vswitchd)
 158
 159     def setup_ovs_bridge_add_flows(self):
 160         dpdk_args = ""
 161         dpdk_list = []
 162         vpath = self.ovs_properties.get("vpath", "/usr/local")
 163         version = self.ovs_properties.get('version', {})
 164         ovs_ver = [int(x) for x in version.get('ovs', self.DEFAULT_OVS).split('.')]
 165         ovs_add_port = ('ovs-vsctl add-port {br} {port} -- '
 166                         'set Interface {port} type={type_}{dpdk_args}')
 167         ovs_add_queue = 'ovs-vsctl set Interface {port} options:n_rxq={queue}'
 168         chmod_vpath = 'chmod 0777 {0}/var/run/openvswitch/dpdkvhostuser*'
 169
 170         cmd_list = [
 171             'ovs-vsctl --if-exists del-br {0}'.format(MAIN_BRIDGE),
 172             'rm -rf {0}/var/run/openvswitch/dpdkvhostuser*'.format(vpath),
 173             'ovs-vsctl add-br {0} -- set bridge {0} datapath_type=netdev'.
 174             format(MAIN_BRIDGE)
 175         ]
 176
 177         ordered_network = collections.OrderedDict(self.networks)
 178         for index, vnf in enumerate(ordered_network.values()):
 179             if ovs_ver >= [2, 7, 0]:
 180                 dpdk_args = " options:dpdk-devargs=%s" % vnf.get("phy_port")
 181             dpdk_list.append(ovs_add_port.format(
 182                 br=MAIN_BRIDGE, port='dpdk%s' % vnf.get("port_num", 0),
 183                 type_='dpdk', dpdk_args=dpdk_args))
 184             dpdk_list.append(ovs_add_queue.format(
 185                 port='dpdk%s' % vnf.get("port_num", 0),
 186                 queue=self.ovs_properties.get("queues", 1)))
 187
 188         # Sorting the array to make sure we execute dpdk0... in the order
 189         list.sort(dpdk_list)
 190         cmd_list.extend(dpdk_list)
 191
 192         # Need to do two for loop to maintain the dpdk/vhost ports.
 193         for index, _ in enumerate(ordered_network):
 194             cmd_list.append(ovs_add_port.format(
 195                 br=MAIN_BRIDGE, port='dpdkvhostuser%s' % index,
 196                 type_='dpdkvhostuser', dpdk_args=""))
 197
 198         ovs_flow = ("ovs-ofctl add-flow {0} in_port=%s,action=output:%s".
 199                     format(MAIN_BRIDGE))
 200         network_count = len(ordered_network) + 1
 201         for in_port, out_port in zip(range(1, network_count),
 202                                      range(network_count, network_count * 2)):
 203             cmd_list.append(ovs_flow % (in_port, out_port))
 204             cmd_list.append(ovs_flow % (out_port, in_port))
 205
 206         cmd_list.append(chmod_vpath.format(vpath))
 207
 208         for cmd in cmd_list:
 209             LOG.info(cmd)
 210             exit_status, _, stderr = self.connection.execute(
 211                 cmd, timeout=self.CMD_TIMEOUT)
 212             if exit_status:
 213                 raise exceptions.OVSSetupError(command=cmd, error=stderr)
 214
 215     def _check_hugepages(self):
 216         meminfo = io.BytesIO()
 217         self.connection.get_file_obj('/proc/meminfo', meminfo)
 218         regex = re.compile(r"HugePages_Total:\s+(?P<hp_total>\d+)[\n\r]"
 219                            r"HugePages_Free:\s+(?P<hp_free>\d+)")
 220         match = regex.search(meminfo.getvalue().decode('utf-8'))
 221         if not match:
 222             raise exceptions.OVSHugepagesInfoError()
 223         if int(match.group('hp_total')) == 0:
 224             raise exceptions.OVSHugepagesNotConfigured()
 225         if int(match.group('hp_free')) == 0:
 226             raise exceptions.OVSHugepagesZeroFree(
 227                 total_hugepages=int(match.group('hp_total')))
 228
 229     def cleanup_ovs_dpdk_env(self):
 230         self.connection.execute(
 231             'ovs-vsctl --if-exists del-br {0}'.format(MAIN_BRIDGE))
 232         self.connection.execute("pkill -9 ovs")
 233
 234     def check_ovs_dpdk_env(self):
 235         self.cleanup_ovs_dpdk_env()
 236         self._check_hugepages()
 237
 238         version = self.ovs_properties.get("version", {})
 239         ovs_ver = version.get("ovs", self.DEFAULT_OVS)
 240         dpdk_ver = version.get("dpdk", "16.07.2").split('.')
 241
 242         supported_version = self.SUPPORTED_OVS_TO_DPDK_MAP.get(ovs_ver, None)
 243         if supported_version is None or supported_version.split('.')[:2] != dpdk_ver[:2]:
 244             raise exceptions.OVSUnsupportedVersion(
 245                 ovs_version=ovs_ver,
 246                 ovs_to_dpdk_map=self.SUPPORTED_OVS_TO_DPDK_MAP)
 247
 248         status = self.connection.execute("ovs-vsctl -V | grep -i '%s'" % ovs_ver)[0]
 249         if status:
 250             deploy = model.OvsDeploy(self.connection,
 251                                      utils.get_nsb_option("bin_path"),
 252                                      self.ovs_properties)
 253             deploy.ovs_deploy()
 254
 255     def deploy(self):
 256         """don't need to deploy"""
 257
 258         # Todo: NFVi deploy (sriov, vswitch, ovs etc) based on the config.
 259         if not self.vm_deploy:
 260             return
 261
 262         self.connection = ssh.SSH.from_node(self.host_mgmt)
 263         self.dpdk_devbind = utils.provision_tool(
 264             self.connection,
 265             os.path.join(utils.get_nsb_option('bin_path'), 'dpdk-devbind.py'))
 266
 267         # Check dpdk/ovs version, if not present install
 268         self.check_ovs_dpdk_env()
 269         #    Todo: NFVi deploy (sriov, vswitch, ovs etc) based on the config.
 270         model.StandaloneContextHelper.install_req_libs(self.connection)
 271         self.networks = model.StandaloneContextHelper.get_nic_details(
 272             self.connection, self.networks, self.dpdk_devbind)
 273
 274         self.setup_ovs()
 275         self.start_ovs_serverswitch()
 276         self.setup_ovs_bridge_add_flows()
 277         self.nodes = self.setup_ovs_dpdk_context()
 278         LOG.debug("Waiting for VM to come up...")
 279         self.nodes = model.StandaloneContextHelper.wait_for_vnfs_to_start(
 280             self.connection, self.servers, self.nodes)
 281
 282     def undeploy(self):
 283
 284         if not self.vm_deploy:
 285             return
 286
 287         # Cleanup the ovs installation...
 288         self.cleanup_ovs_dpdk_env()
 289
 290         # Bind nics back to kernel
 291         bind_cmd = "{dpdk_devbind} --force -b {driver} {port}"
 292         for port in self.networks.values():
 293             vpci = port.get("phy_port")
 294             phy_driver = port.get("driver")
 295             self.connection.execute(bind_cmd.format(
 296                 dpdk_devbind=self.dpdk_devbind, driver=phy_driver, port=vpci))
 297
 298         # Todo: NFVi undeploy (sriov, vswitch, ovs etc) based on the config.
 299         for vm in self.vm_names:
 300             model.Libvirt.check_if_vm_exists_and_delete(vm, self.connection)
 301
 302     def _get_server(self, attr_name):
 303         """lookup server info by name from context
 304
 305         Keyword arguments:
 306         attr_name -- A name for a server listed in nodes config file
 307         """
 308         node_name, name = self.split_name(attr_name)
 309         if name is None or self.name != name:
 310             return None
 311
 312         matching_nodes = (n for n in self.nodes if n["name"] == node_name)
 313         try:
 314             # A clone is created in order to avoid affecting the
 315             # original one.
 316             node = dict(next(matching_nodes))
 317         except StopIteration:
 318             return None
 319
 320         try:
 321             duplicate = next(matching_nodes)
 322         except StopIteration:
 323             pass
 324         else:
 325             raise ValueError("Duplicate nodes!!! Nodes: %s %s" % (node, duplicate))
 326
 327         node["name"] = attr_name
 328         return node
 329
 330     def _get_network(self, attr_name):
 331         if not isinstance(attr_name, collections.Mapping):
 332             network = self.networks.get(attr_name)
 333
 334         else:
 335             # Don't generalize too much  Just support vld_id
 336             vld_id = attr_name.get('vld_id', {})
 337             # for standalone context networks are dicts
 338             iter1 = (n for n in self.networks.values() if n.get('vld_id') == vld_id)
 339             network = next(iter1, None)
 340
 341         if network is None:
 342             return None
 343
 344         result = {
 345             # name is required
 346             "name": network["name"],
 347             "vld_id": network.get("vld_id"),
 348             "segmentation_id": network.get("segmentation_id"),
 349             "network_type": network.get("network_type"),
 350             "physical_network": network.get("physical_network"),
 351         }
 352         return result
 353
 354     def configure_nics_for_ovs_dpdk(self):
 355         portlist = collections.OrderedDict(self.networks)
 356         for key in portlist:
 357             mac = model.StandaloneContextHelper.get_mac_address()
 358             portlist[key].update({'mac': mac})
 359         self.networks = portlist
 360         LOG.info("Ports %s", self.networks)
 361
 362     def _enable_interfaces(self, index, vfs, xml_str):
 363         vpath = self.ovs_properties.get("vpath", "/usr/local")
 364         vf = self.networks[vfs[0]]
 365         port_num = vf.get('port_num', 0)
 366         vpci = utils.PciAddress(vf['vpci'].strip())
 367         # Generate the vpci for the interfaces
 368         slot = index + port_num + 10
 369         vf['vpci'] = \
 370             "{}:{}:{:02x}.{}".format(vpci.domain, vpci.bus, slot, vpci.function)
 371         return model.Libvirt.add_ovs_interface(
 372             vpath, port_num, vf['vpci'], vf['mac'], xml_str)
 373
 374     def setup_ovs_dpdk_context(self):
 375         nodes = []
 376
 377         self.configure_nics_for_ovs_dpdk()
 378
 379         for index, (key, vnf) in enumerate(collections.OrderedDict(
 380                 self.servers).items()):
 381             cfg = '/tmp/vm_ovs_%d.xml' % index
 382             vm_name = "vm_%d" % index
 383
 384             # 1. Check and delete VM if already exists
 385             model.Libvirt.check_if_vm_exists_and_delete(vm_name,
 386                                                         self.connection)
 387             xml_str, mac = model.Libvirt.build_vm_xml(
 388                 self.connection, self.vm_flavor, vm_name, index)
 389
 390             # 2: Cleanup already available VMs
 391             for vfs in [vfs for vfs_name, vfs in vnf["network_ports"].items()
 392                         if vfs_name != 'mgmt']:
 393                 xml_str = self._enable_interfaces(index, vfs, xml_str)
 394
 395             # copy xml to target...
 396             model.Libvirt.write_file(cfg, xml_str)
 397             self.connection.put(cfg, cfg)
 398
 399             # NOTE: launch through libvirt
 400             LOG.info("virsh create ...")
 401             model.Libvirt.virsh_create_vm(self.connection, cfg)
 402
 403             self.vm_names.append(vm_name)
 404
 405             # build vnf node details
 406             nodes.append(self.vnf_node.generate_vnf_instance(self.vm_flavor,
 407                                                              self.networks,
 408                                                              self.host_mgmt.get('ip'),
 409                                                              key, vnf, mac))
 410
 411         return nodes