1 # Copyright (c) 2016-2017 Intel Corporation
3 # Licensed under the Apache License, Version 2.0 (the "License");
4 # you may not use this file except in compliance with the License.
5 # You may obtain a copy of the License at
7 # http://www.apache.org/licenses/LICENSE-2.0
9 # Unless required by applicable law or agreed to in writing, software
10 # distributed under the License is distributed on an "AS IS" BASIS,
11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 # See the License for the specific language governing permissions and
13 # limitations under the License.
15 from __future__ import absolute_import
21 from collections import OrderedDict
23 from yardstick import ssh
24 from yardstick.network_services.utils import get_nsb_option
25 from yardstick.network_services.utils import provision_tool
26 from yardstick.benchmark.contexts.base import Context
27 from yardstick.benchmark.contexts.standalone.model import Libvirt
28 from yardstick.benchmark.contexts.standalone.model import StandaloneContextHelper
29 from yardstick.benchmark.contexts.standalone.model import Server
30 from yardstick.benchmark.contexts.standalone.model import OvsDeploy
31 from yardstick.network_services.utils import PciAddress
33 LOG = logging.getLogger(__name__)
36 class OvsDpdkContext(Context):
37 """ This class handles OVS standalone nodes - VM running on Non-Managed NFVi
38 Configuration: ovs_dpdk
41 __context_type__ = "StandaloneOvsDpdk"
43 SUPPORTED_OVS_TO_DPDK_MAP = {
54 PKILL_TEMPLATE = "pkill %s %s"
60 self.dpdk_devbind = ''
69 self.helper = StandaloneContextHelper()
70 self.vnf_node = Server()
71 self.ovs_properties = {}
72 self.wait_for_vswitchd = 10
73 super(OvsDpdkContext, self).__init__()
75 def init(self, attrs):
76 """initializes itself from the supplied arguments"""
78 self.name = attrs["name"]
79 self.file_path = attrs.get("file", "pod.yaml")
81 self.nodes, self.nfvi_host, self.host_mgmt = \
82 self.helper.parse_pod_file(self.file_path, 'OvsDpdk')
85 self.vm_flavor = attrs.get('flavor', {})
86 self.servers = attrs.get('servers', {})
87 self.vm_deploy = attrs.get("vm_deploy", True)
88 self.ovs_properties = attrs.get('ovs_properties', {})
89 # add optional static network definition
90 self.networks = attrs.get("networks", {})
92 LOG.debug("Nodes: %r", self.nodes)
93 LOG.debug("NFVi Node: %r", self.nfvi_host)
94 LOG.debug("Networks: %r", self.networks)
97 vpath = self.ovs_properties.get("vpath", "/usr/local")
98 xargs_kill_cmd = self.PKILL_TEMPLATE % ('-9', 'ovs')
100 create_from = os.path.join(vpath, 'etc/openvswitch/conf.db')
101 create_to = os.path.join(vpath, 'share/openvswitch/vswitch.ovsschema')
104 "chmod 0666 /dev/vfio/*",
105 "chmod a+x /dev/vfio",
109 "mkdir -p {0}/etc/openvswitch".format(vpath),
110 "mkdir -p {0}/var/run/openvswitch".format(vpath),
111 "rm {0}/etc/openvswitch/conf.db".format(vpath),
112 "ovsdb-tool create {0} {1}".format(create_from, create_to),
114 "chmod a+x /dev/vfio",
115 "chmod 0666 /dev/vfio/*",
118 self.connection.execute(cmd)
119 bind_cmd = "{dpdk_devbind} --force -b {driver} {port}"
120 phy_driver = "vfio-pci"
121 for port in self.networks.values():
122 vpci = port.get("phy_port")
123 self.connection.execute(bind_cmd.format(
124 dpdk_devbind=self.dpdk_devbind, driver=phy_driver, port=vpci))
126 def start_ovs_serverswitch(self):
127 vpath = self.ovs_properties.get("vpath")
128 pmd_nums = int(self.ovs_properties.get("pmd_threads", 2))
129 ovs_sock_path = '/var/run/openvswitch/db.sock'
130 log_path = '/var/log/openvswitch/ovs-vswitchd.log'
132 pmd_cpu_mask = self.ovs_properties.get("pmd_cpu_mask", '')
133 pmd_mask = hex(sum(2 ** num for num in range(pmd_nums)) << 1)
135 pmd_mask = pmd_cpu_mask
137 socket0 = self.ovs_properties.get("ram", {}).get("socket_0", "2048")
138 socket1 = self.ovs_properties.get("ram", {}).get("socket_1", "2048")
140 ovs_other_config = "ovs-vsctl {0}set Open_vSwitch . other_config:{1}"
141 detach_cmd = "ovs-vswitchd unix:{0}{1} --pidfile --detach --log-file={2}"
143 lcore_mask = self.ovs_properties.get("lcore_mask", '')
145 lcore_mask = ovs_other_config.format("--no-wait ", "dpdk-lcore-mask='%s'" % lcore_mask)
148 "mkdir -p /usr/local/var/run/openvswitch",
149 "mkdir -p {}".format(os.path.dirname(log_path)),
150 "ovsdb-server --remote=punix:/{0}/{1} --pidfile --detach".format(vpath,
152 ovs_other_config.format("--no-wait ", "dpdk-init=true"),
153 ovs_other_config.format("--no-wait ", "dpdk-socket-mem='%s,%s'" % (socket0, socket1)),
155 detach_cmd.format(vpath, ovs_sock_path, log_path),
156 ovs_other_config.format("", "pmd-cpu-mask=%s" % pmd_mask),
161 self.connection.execute(cmd)
162 time.sleep(self.wait_for_vswitchd)
164 def setup_ovs_bridge_add_flows(self):
167 vpath = self.ovs_properties.get("vpath", "/usr/local")
168 version = self.ovs_properties.get('version', {})
169 ovs_ver = [int(x) for x in version.get('ovs', self.DEFAULT_OVS).split('.')]
171 "ovs-vsctl add-port {br} {port} -- set Interface {port} type={type_}{dpdk_args}"
172 ovs_add_queue = "ovs-vsctl set Interface {port} options:n_rxq={queue}"
173 chmod_vpath = "chmod 0777 {0}/var/run/openvswitch/dpdkvhostuser*"
176 "ovs-vsctl del-br br0",
177 "rm -rf {0}/var/run/openvswitch/dpdkvhostuser*".format(vpath),
178 "ovs-vsctl add-br br0 -- set bridge br0 datapath_type=netdev",
181 ordered_network = OrderedDict(self.networks)
182 for index, vnf in enumerate(ordered_network.values()):
183 if ovs_ver >= [2, 7, 0]:
184 dpdk_args = " options:dpdk-devargs=%s" % vnf.get("phy_port")
185 dpdk_list.append(ovs_add_port.format(br='br0', port='dpdk%s' % vnf.get("port_num", 0),
186 type_='dpdk', dpdk_args=dpdk_args))
187 dpdk_list.append(ovs_add_queue.format(port='dpdk%s' % vnf.get("port_num", 0),
188 queue=self.ovs_properties.get("queues", 1)))
190 # Sorting the array to make sure we execute dpdk0... in the order
192 cmd_dpdk_list.extend(dpdk_list)
194 # Need to do two for loop to maintain the dpdk/vhost ports.
195 for index, _ in enumerate(ordered_network):
196 cmd_dpdk_list.append(ovs_add_port.format(br='br0', port='dpdkvhostuser%s' % index,
197 type_='dpdkvhostuser', dpdk_args=""))
199 for cmd in cmd_dpdk_list:
201 self.connection.execute(cmd)
203 # Fixme: add flows code
204 ovs_flow = "ovs-ofctl add-flow br0 in_port=%s,action=output:%s"
206 network_count = len(ordered_network) + 1
207 for in_port, out_port in zip(range(1, network_count),
208 range(network_count, network_count * 2)):
209 self.connection.execute(ovs_flow % (in_port, out_port))
210 self.connection.execute(ovs_flow % (out_port, in_port))
212 self.connection.execute(chmod_vpath.format(vpath))
214 def cleanup_ovs_dpdk_env(self):
215 self.connection.execute("ovs-vsctl del-br br0")
216 self.connection.execute("pkill -9 ovs")
218 def check_ovs_dpdk_env(self):
219 self.cleanup_ovs_dpdk_env()
221 version = self.ovs_properties.get("version", {})
222 ovs_ver = version.get("ovs", self.DEFAULT_OVS)
223 dpdk_ver = version.get("dpdk", "16.07.2").split('.')
225 supported_version = self.SUPPORTED_OVS_TO_DPDK_MAP.get(ovs_ver, None)
226 if supported_version is None or supported_version.split('.')[:2] != dpdk_ver[:2]:
227 raise Exception("Unsupported ovs '{}'. Please check the config...".format(ovs_ver))
229 status = self.connection.execute("ovs-vsctl -V | grep -i '%s'" % ovs_ver)[0]
231 deploy = OvsDeploy(self.connection,
232 get_nsb_option("bin_path"),
237 """don't need to deploy"""
239 # Todo: NFVi deploy (sriov, vswitch, ovs etc) based on the config.
240 if not self.vm_deploy:
243 self.connection = ssh.SSH.from_node(self.host_mgmt)
244 self.dpdk_devbind = provision_tool(
246 os.path.join(get_nsb_option("bin_path"), "dpdk-devbind.py"))
248 # Check dpdk/ovs version, if not present install
249 self.check_ovs_dpdk_env()
250 # Todo: NFVi deploy (sriov, vswitch, ovs etc) based on the config.
251 StandaloneContextHelper.install_req_libs(self.connection)
252 self.networks = StandaloneContextHelper.get_nic_details(
253 self.connection, self.networks, self.dpdk_devbind)
256 self.start_ovs_serverswitch()
257 self.setup_ovs_bridge_add_flows()
258 self.nodes = self.setup_ovs_dpdk_context()
259 LOG.debug("Waiting for VM to come up...")
260 self.nodes = StandaloneContextHelper.wait_for_vnfs_to_start(self.connection,
266 if not self.vm_deploy:
269 # Cleanup the ovs installation...
270 self.cleanup_ovs_dpdk_env()
272 # Bind nics back to kernel
273 bind_cmd = "{dpdk_devbind} --force -b {driver} {port}"
274 for port in self.networks.values():
275 vpci = port.get("phy_port")
276 phy_driver = port.get("driver")
277 self.connection.execute(bind_cmd.format(
278 dpdk_devbind=self.dpdk_devbind, driver=phy_driver, port=vpci))
280 # Todo: NFVi undeploy (sriov, vswitch, ovs etc) based on the config.
281 for vm in self.vm_names:
282 Libvirt.check_if_vm_exists_and_delete(vm, self.connection)
284 def _get_server(self, attr_name):
285 """lookup server info by name from context
288 attr_name -- A name for a server listed in nodes config file
290 node_name, name = self.split_name(attr_name)
291 if name is None or self.name != name:
294 matching_nodes = (n for n in self.nodes if n["name"] == node_name)
296 # A clone is created in order to avoid affecting the
298 node = dict(next(matching_nodes))
299 except StopIteration:
303 duplicate = next(matching_nodes)
304 except StopIteration:
307 raise ValueError("Duplicate nodes!!! Nodes: %s %s" % (node, duplicate))
309 node["name"] = attr_name
312 def _get_network(self, attr_name):
313 if not isinstance(attr_name, collections.Mapping):
314 network = self.networks.get(attr_name)
317 # Don't generalize too much Just support vld_id
318 vld_id = attr_name.get('vld_id', {})
319 # for standalone context networks are dicts
320 iter1 = (n for n in self.networks.values() if n.get('vld_id') == vld_id)
321 network = next(iter1, None)
328 "name": network["name"],
329 "vld_id": network.get("vld_id"),
330 "segmentation_id": network.get("segmentation_id"),
331 "network_type": network.get("network_type"),
332 "physical_network": network.get("physical_network"),
336 def configure_nics_for_ovs_dpdk(self):
337 portlist = OrderedDict(self.networks)
339 mac = StandaloneContextHelper.get_mac_address()
340 portlist[key].update({'mac': mac})
341 self.networks = portlist
342 LOG.info("Ports %s", self.networks)
344 def _enable_interfaces(self, index, vfs, cfg):
345 vpath = self.ovs_properties.get("vpath", "/usr/local")
346 vf = self.networks[vfs[0]]
347 port_num = vf.get('port_num', 0)
348 vpci = PciAddress(vf['vpci'].strip())
349 # Generate the vpci for the interfaces
350 slot = index + port_num + 10
352 "{}:{}:{:02x}.{}".format(vpci.domain, vpci.bus, slot, vpci.function)
353 Libvirt.add_ovs_interface(vpath, port_num, vf['vpci'], vf['mac'], str(cfg))
355 def setup_ovs_dpdk_context(self):
358 self.configure_nics_for_ovs_dpdk()
360 for index, (key, vnf) in enumerate(OrderedDict(self.servers).items()):
361 cfg = '/tmp/vm_ovs_%d.xml' % index
362 vm_name = "vm_%d" % index
364 # 1. Check and delete VM if already exists
365 Libvirt.check_if_vm_exists_and_delete(vm_name, self.connection)
367 _, mac = Libvirt.build_vm_xml(self.connection, self.vm_flavor,
369 # 2: Cleanup already available VMs
370 for vkey, vfs in OrderedDict(vnf["network_ports"]).items():
373 self._enable_interfaces(index, vfs, cfg)
375 # copy xml to target...
376 self.connection.put(cfg, cfg)
378 # NOTE: launch through libvirt
379 LOG.info("virsh create ...")
380 Libvirt.virsh_create_vm(self.connection, cfg)
382 self.vm_names.append(vm_name)
384 # build vnf node details
385 nodes.append(self.vnf_node.generate_vnf_instance(self.vm_flavor,
387 self.host_mgmt.get('ip'),