1 # Copyright (c) 2016-2017 Intel Corporation
3 # Licensed under the Apache License, Version 2.0 (the "License");
4 # you may not use this file except in compliance with the License.
5 # You may obtain a copy of the License at
7 # http://www.apache.org/licenses/LICENSE-2.0
9 # Unless required by applicable law or agreed to in writing, software
10 # distributed under the License is distributed on an "AS IS" BASIS,
11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 # See the License for the specific language governing permissions and
13 # limitations under the License.
22 from yardstick import ssh
23 from yardstick.benchmark import contexts
24 from yardstick.benchmark.contexts import base
25 from yardstick.benchmark.contexts.standalone import model
26 from yardstick.common import exceptions
27 from yardstick.network_services import utils
28 from yardstick.network_services.utils import get_nsb_option
31 LOG = logging.getLogger(__name__)
36 class OvsDpdkContext(base.Context):
37 """ This class handles OVS standalone nodes - VM running on Non-Managed NFVi
38 Configuration: ovs_dpdk
41 __context_type__ = contexts.CONTEXT_STANDALONEOVSDPDK
43 SUPPORTED_OVS_TO_DPDK_MAP = {
55 DEFAULT_USER_PATH = '/usr/local'
61 self.dpdk_devbind = os.path.join(get_nsb_option('bin_path'),
70 self.helper = model.StandaloneContextHelper()
71 self.vnf_node = model.Server()
72 self.ovs_properties = {}
73 self.wait_for_vswitchd = 10
74 super(OvsDpdkContext, self).__init__()
76 def init(self, attrs):
77 """initializes itself from the supplied arguments"""
78 super(OvsDpdkContext, self).init(attrs)
80 self.file_path = attrs.get("file", "pod.yaml")
82 self.nodes, self.nfvi_host, self.host_mgmt = \
83 self.helper.parse_pod_file(self.file_path, 'OvsDpdk')
86 self.vm_flavor = attrs.get('flavor', {})
87 self.servers = attrs.get('servers', {})
88 self.vm_deploy = attrs.get("vm_deploy", True)
89 self.ovs_properties = attrs.get('ovs_properties', {})
90 # add optional static network definition
91 self.networks = attrs.get("networks", {})
93 LOG.debug("Nodes: %r", self.nodes)
94 LOG.debug("NFVi Node: %r", self.nfvi_host)
95 LOG.debug("Networks: %r", self.networks)
98 """Initialize OVS-DPDK"""
99 vpath = self.ovs_properties.get('vpath', self.DEFAULT_USER_PATH)
100 create_from = os.path.join(vpath, 'etc/openvswitch/conf.db')
101 create_to = os.path.join(vpath, 'share/openvswitch/vswitch.ovsschema')
104 'killall -r "ovs.*" -q | true',
105 'mkdir -p {0}/etc/openvswitch'.format(vpath),
106 'mkdir -p {0}/var/run/openvswitch'.format(vpath),
107 'rm {0}/etc/openvswitch/conf.db | true'.format(vpath),
108 'ovsdb-tool create {0} {1}'.format(create_from, create_to),
110 'chmod a+x /dev/vfio',
111 'chmod 0666 /dev/vfio/*',
114 bind_cmd = '%s --force -b vfio-pci {port}' % self.dpdk_devbind
115 for port in self.networks.values():
116 cmd_list.append(bind_cmd.format(port=port.get('phy_port')))
120 exit_status, _, stderr = self.connection.execute(
121 cmd, timeout=self.CMD_TIMEOUT)
123 raise exceptions.OVSSetupError(command=cmd, error=stderr)
125 def start_ovs_serverswitch(self):
126 vpath = self.ovs_properties.get("vpath")
127 pmd_nums = int(self.ovs_properties.get("pmd_threads", 2))
128 ovs_sock_path = '/var/run/openvswitch/db.sock'
129 log_path = '/var/log/openvswitch/ovs-vswitchd.log'
131 pmd_cpu_mask = self.ovs_properties.get("pmd_cpu_mask", '')
132 pmd_mask = hex(sum(2 ** num for num in range(pmd_nums)) << 1)
134 pmd_mask = pmd_cpu_mask
136 socket0 = self.ovs_properties.get("ram", {}).get("socket_0", "2048")
137 socket1 = self.ovs_properties.get("ram", {}).get("socket_1", "2048")
139 ovs_other_config = "ovs-vsctl {0}set Open_vSwitch . other_config:{1}"
140 detach_cmd = "ovs-vswitchd unix:{0}{1} --pidfile --detach --log-file={2}"
142 lcore_mask = self.ovs_properties.get("lcore_mask", '')
144 lcore_mask = ovs_other_config.format("--no-wait ", "dpdk-lcore-mask='%s'" % lcore_mask)
147 "mkdir -p /usr/local/var/run/openvswitch",
148 "mkdir -p {}".format(os.path.dirname(log_path)),
149 "ovsdb-server --remote=punix:/{0}/{1} --pidfile --detach".format(vpath,
151 ovs_other_config.format("--no-wait ", "dpdk-init=true"),
152 ovs_other_config.format("--no-wait ", "dpdk-socket-mem='%s,%s'" % (socket0, socket1)),
154 detach_cmd.format(vpath, ovs_sock_path, log_path),
155 ovs_other_config.format("", "pmd-cpu-mask=%s" % pmd_mask),
160 self.connection.execute(cmd)
161 time.sleep(self.wait_for_vswitchd)
163 def setup_ovs_bridge_add_flows(self):
166 vpath = self.ovs_properties.get("vpath", "/usr/local")
167 version = self.ovs_properties.get('version', {})
168 ovs_ver = [int(x) for x in version.get('ovs', self.DEFAULT_OVS).split('.')]
169 ovs_add_port = ('ovs-vsctl add-port {br} {port} -- '
170 'set Interface {port} type={type_}{dpdk_args}{dpdk_rxq}')
171 chmod_vpath = 'chmod 0777 {0}/var/run/openvswitch/dpdkvhostuser*'
174 'ovs-vsctl --if-exists del-br {0}'.format(MAIN_BRIDGE),
175 'rm -rf {0}/var/run/openvswitch/dpdkvhostuser*'.format(vpath),
176 'ovs-vsctl add-br {0} -- set bridge {0} datapath_type=netdev'.
179 dpdk_rxq = " options:n_rxq={queue}".format(
180 queue=self.ovs_properties.get("queues", 1))
182 ordered_network = collections.OrderedDict(self.networks)
183 for index, vnf in enumerate(ordered_network.values()):
184 if ovs_ver >= [2, 7, 0]:
185 dpdk_args = " options:dpdk-devargs=%s" % vnf.get("phy_port")
186 dpdk_list.append(ovs_add_port.format(
187 br=MAIN_BRIDGE, port='dpdk%s' % vnf.get("port_num", 0),
188 type_='dpdk', dpdk_args=dpdk_args, dpdk_rxq=dpdk_rxq))
190 # Sorting the array to make sure we execute dpdk0... in the order
192 cmd_list.extend(dpdk_list)
194 # Need to do two for loop to maintain the dpdk/vhost ports.
195 for index, _ in enumerate(ordered_network):
196 cmd_list.append(ovs_add_port.format(
197 br=MAIN_BRIDGE, port='dpdkvhostuser%s' % index,
198 type_='dpdkvhostuser', dpdk_args="", dpdk_rxq=""))
200 ovs_flow = ("ovs-ofctl add-flow {0} in_port=%s,action=output:%s".
202 network_count = len(ordered_network) + 1
203 for in_port, out_port in zip(range(1, network_count),
204 range(network_count, network_count * 2)):
205 cmd_list.append(ovs_flow % (in_port, out_port))
206 cmd_list.append(ovs_flow % (out_port, in_port))
208 cmd_list.append(chmod_vpath.format(vpath))
212 exit_status, _, stderr = self.connection.execute(
213 cmd, timeout=self.CMD_TIMEOUT)
215 raise exceptions.OVSSetupError(command=cmd, error=stderr)
217 def _check_hugepages(self):
218 meminfo = io.BytesIO()
219 self.connection.get_file_obj('/proc/meminfo', meminfo)
220 regex = re.compile(r"HugePages_Total:\s+(?P<hp_total>\d+)[\n\r]"
221 r"HugePages_Free:\s+(?P<hp_free>\d+)")
222 match = regex.search(meminfo.getvalue().decode('utf-8'))
224 raise exceptions.OVSHugepagesInfoError()
225 if int(match.group('hp_total')) == 0:
226 raise exceptions.OVSHugepagesNotConfigured()
227 if int(match.group('hp_free')) == 0:
228 raise exceptions.OVSHugepagesZeroFree(
229 total_hugepages=int(match.group('hp_total')))
231 def cleanup_ovs_dpdk_env(self):
232 self.connection.execute(
233 'ovs-vsctl --if-exists del-br {0}'.format(MAIN_BRIDGE))
234 self.connection.execute("pkill -9 ovs")
236 def check_ovs_dpdk_env(self):
237 self.cleanup_ovs_dpdk_env()
238 self._check_hugepages()
240 version = self.ovs_properties.get("version", {})
241 ovs_ver = version.get("ovs", self.DEFAULT_OVS)
242 dpdk_ver = version.get("dpdk", "16.07.2").split('.')
244 supported_version = self.SUPPORTED_OVS_TO_DPDK_MAP.get(ovs_ver, None)
245 if supported_version is None or supported_version.split('.')[:2] != dpdk_ver[:2]:
246 raise exceptions.OVSUnsupportedVersion(
248 ovs_to_dpdk_map=self.SUPPORTED_OVS_TO_DPDK_MAP)
250 status = self.connection.execute("ovs-vsctl -V | grep -i '%s'" % ovs_ver)[0]
252 deploy = model.OvsDeploy(self.connection,
253 utils.get_nsb_option("bin_path"),
258 """don't need to deploy"""
260 # Todo: NFVi deploy (sriov, vswitch, ovs etc) based on the config.
261 if not self.vm_deploy:
264 self.connection = ssh.SSH.from_node(self.host_mgmt)
266 # Check dpdk/ovs version, if not present install
267 self.check_ovs_dpdk_env()
268 # Todo: NFVi deploy (sriov, vswitch, ovs etc) based on the config.
269 model.StandaloneContextHelper.install_req_libs(self.connection)
270 self.networks = model.StandaloneContextHelper.get_nic_details(
271 self.connection, self.networks, self.dpdk_devbind)
274 self.start_ovs_serverswitch()
275 self.setup_ovs_bridge_add_flows()
276 self.nodes = self.setup_ovs_dpdk_context()
277 LOG.debug("Waiting for VM to come up...")
278 self.nodes = model.StandaloneContextHelper.wait_for_vnfs_to_start(
279 self.connection, self.servers, self.nodes)
283 if not self.vm_deploy:
286 # Cleanup the ovs installation...
287 self.cleanup_ovs_dpdk_env()
289 # Bind nics back to kernel
290 bind_cmd = "{dpdk_devbind} --force -b {driver} {port}"
291 for port in self.networks.values():
292 vpci = port.get("phy_port")
293 phy_driver = port.get("driver")
294 self.connection.execute(bind_cmd.format(
295 dpdk_devbind=self.dpdk_devbind, driver=phy_driver, port=vpci))
297 # Todo: NFVi undeploy (sriov, vswitch, ovs etc) based on the config.
298 for vm in self.vm_names:
299 model.Libvirt.check_if_vm_exists_and_delete(vm, self.connection)
301 def _get_physical_nodes(self):
302 return self.nfvi_host
304 def _get_physical_node_for_server(self, server_name):
305 node_name, ctx_name = self.split_host_name(server_name)
306 if ctx_name is None or self.name != ctx_name:
309 matching_nodes = [s for s in self.servers if s == node_name]
310 if len(matching_nodes) == 0:
313 # self.nfvi_host always contain only one host
314 return "{}.{}".format(self.nfvi_host[0]["name"], self._name)
316 def _get_server(self, attr_name):
317 """lookup server info by name from context
320 attr_name -- A name for a server listed in nodes config file
322 node_name, name = self.split_host_name(attr_name)
323 if name is None or self.name != name:
326 matching_nodes = (n for n in self.nodes if n["name"] == node_name)
328 # A clone is created in order to avoid affecting the
330 node = dict(next(matching_nodes))
331 except StopIteration:
335 duplicate = next(matching_nodes)
336 except StopIteration:
339 raise ValueError("Duplicate nodes!!! Nodes: %s %s" % (node, duplicate))
341 node["name"] = attr_name
344 def _get_network(self, attr_name):
345 if not isinstance(attr_name, collections.Mapping):
346 network = self.networks.get(attr_name)
349 # Don't generalize too much Just support vld_id
350 vld_id = attr_name.get('vld_id', {})
351 # for standalone context networks are dicts
352 iter1 = (n for n in self.networks.values() if n.get('vld_id') == vld_id)
353 network = next(iter1, None)
360 "name": network["name"],
361 "vld_id": network.get("vld_id"),
362 "segmentation_id": network.get("segmentation_id"),
363 "network_type": network.get("network_type"),
364 "physical_network": network.get("physical_network"),
368 def configure_nics_for_ovs_dpdk(self):
369 portlist = collections.OrderedDict(self.networks)
371 mac = model.StandaloneContextHelper.get_mac_address()
372 portlist[key].update({'mac': mac})
373 self.networks = portlist
374 LOG.info("Ports %s", self.networks)
376 def _enable_interfaces(self, index, vfs, xml_str):
377 vpath = self.ovs_properties.get("vpath", "/usr/local")
378 vf = self.networks[vfs[0]]
379 port_num = vf.get('port_num', 0)
380 vpci = utils.PciAddress(vf['vpci'].strip())
381 # Generate the vpci for the interfaces
382 slot = index + port_num + 10
384 "{}:{}:{:02x}.{}".format(vpci.domain, vpci.bus, slot, vpci.function)
385 return model.Libvirt.add_ovs_interface(
386 vpath, port_num, vf['vpci'], vf['mac'], xml_str)
388 def setup_ovs_dpdk_context(self):
391 self.configure_nics_for_ovs_dpdk()
393 for index, (key, vnf) in enumerate(collections.OrderedDict(
394 self.servers).items()):
395 cfg = '/tmp/vm_ovs_%d.xml' % index
396 vm_name = "vm-%d" % index
397 cdrom_img = "/var/lib/libvirt/images/cdrom-%d.img" % index
399 # 1. Check and delete VM if already exists
400 model.Libvirt.check_if_vm_exists_and_delete(vm_name,
402 xml_str, mac = model.Libvirt.build_vm_xml(
403 self.connection, self.vm_flavor, vm_name, index, cdrom_img)
405 # 2: Cleanup already available VMs
406 for vfs in [vfs for vfs_name, vfs in vnf["network_ports"].items()
407 if vfs_name != 'mgmt']:
408 xml_str = self._enable_interfaces(index, vfs, xml_str)
410 # copy xml to target...
411 model.Libvirt.write_file(cfg, xml_str)
412 self.connection.put(cfg, cfg)
414 node = self.vnf_node.generate_vnf_instance(self.vm_flavor,
416 self.host_mgmt.get('ip'),
418 # Generate public/private keys if password or private key file is not provided
419 node = model.StandaloneContextHelper.check_update_key(self.connection,
426 # store vnf node details
429 # NOTE: launch through libvirt
430 LOG.info("virsh create ...")
431 model.Libvirt.virsh_create_vm(self.connection, cfg)
433 self.vm_names.append(vm_name)