1 # Copyright (c) 2016-2017 Intel Corporation
3 # Licensed under the Apache License, Version 2.0 (the "License");
4 # you may not use this file except in compliance with the License.
5 # You may obtain a copy of the License at
7 # http://www.apache.org/licenses/LICENSE-2.0
9 # Unless required by applicable law or agreed to in writing, software
10 # distributed under the License is distributed on an "AS IS" BASIS,
11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 # See the License for the specific language governing permissions and
13 # limitations under the License.
22 from yardstick import ssh
23 from yardstick.benchmark.contexts.base import Context
24 from yardstick.benchmark.contexts.standalone import model
25 from yardstick.common import exceptions
26 from yardstick.network_services import utils
29 LOG = logging.getLogger(__name__)
34 class OvsDpdkContext(Context):
35 """ This class handles OVS standalone nodes - VM running on Non-Managed NFVi
36 Configuration: ovs_dpdk
39 __context_type__ = "StandaloneOvsDpdk"
41 SUPPORTED_OVS_TO_DPDK_MAP = {
52 DEFAULT_USER_PATH = '/usr/local'
58 self.dpdk_devbind = ''
66 self.helper = model.StandaloneContextHelper()
67 self.vnf_node = model.Server()
68 self.ovs_properties = {}
69 self.wait_for_vswitchd = 10
70 super(OvsDpdkContext, self).__init__()
72 def init(self, attrs):
73 """initializes itself from the supplied arguments"""
74 super(OvsDpdkContext, self).init(attrs)
76 self.file_path = attrs.get("file", "pod.yaml")
78 self.nodes, self.nfvi_host, self.host_mgmt = \
79 self.helper.parse_pod_file(self.file_path, 'OvsDpdk')
82 self.vm_flavor = attrs.get('flavor', {})
83 self.servers = attrs.get('servers', {})
84 self.vm_deploy = attrs.get("vm_deploy", True)
85 self.ovs_properties = attrs.get('ovs_properties', {})
86 # add optional static network definition
87 self.networks = attrs.get("networks", {})
89 LOG.debug("Nodes: %r", self.nodes)
90 LOG.debug("NFVi Node: %r", self.nfvi_host)
91 LOG.debug("Networks: %r", self.networks)
94 """Initialize OVS-DPDK"""
95 vpath = self.ovs_properties.get('vpath', self.DEFAULT_USER_PATH)
96 create_from = os.path.join(vpath, 'etc/openvswitch/conf.db')
97 create_to = os.path.join(vpath, 'share/openvswitch/vswitch.ovsschema')
100 'killall -r "ovs.*" -q | true',
101 'mkdir -p {0}/etc/openvswitch'.format(vpath),
102 'mkdir -p {0}/var/run/openvswitch'.format(vpath),
103 'rm {0}/etc/openvswitch/conf.db | true'.format(vpath),
104 'ovsdb-tool create {0} {1}'.format(create_from, create_to),
106 'chmod a+x /dev/vfio',
107 'chmod 0666 /dev/vfio/*',
110 bind_cmd = '%s --force -b vfio-pci {port}' % self.dpdk_devbind
111 for port in self.networks.values():
112 cmd_list.append(bind_cmd.format(port=port.get('phy_port')))
116 exit_status, _, stderr = self.connection.execute(
117 cmd, timeout=self.CMD_TIMEOUT)
119 raise exceptions.OVSSetupError(command=cmd, error=stderr)
121 def start_ovs_serverswitch(self):
122 vpath = self.ovs_properties.get("vpath")
123 pmd_nums = int(self.ovs_properties.get("pmd_threads", 2))
124 ovs_sock_path = '/var/run/openvswitch/db.sock'
125 log_path = '/var/log/openvswitch/ovs-vswitchd.log'
127 pmd_cpu_mask = self.ovs_properties.get("pmd_cpu_mask", '')
128 pmd_mask = hex(sum(2 ** num for num in range(pmd_nums)) << 1)
130 pmd_mask = pmd_cpu_mask
132 socket0 = self.ovs_properties.get("ram", {}).get("socket_0", "2048")
133 socket1 = self.ovs_properties.get("ram", {}).get("socket_1", "2048")
135 ovs_other_config = "ovs-vsctl {0}set Open_vSwitch . other_config:{1}"
136 detach_cmd = "ovs-vswitchd unix:{0}{1} --pidfile --detach --log-file={2}"
138 lcore_mask = self.ovs_properties.get("lcore_mask", '')
140 lcore_mask = ovs_other_config.format("--no-wait ", "dpdk-lcore-mask='%s'" % lcore_mask)
143 "mkdir -p /usr/local/var/run/openvswitch",
144 "mkdir -p {}".format(os.path.dirname(log_path)),
145 "ovsdb-server --remote=punix:/{0}/{1} --pidfile --detach".format(vpath,
147 ovs_other_config.format("--no-wait ", "dpdk-init=true"),
148 ovs_other_config.format("--no-wait ", "dpdk-socket-mem='%s,%s'" % (socket0, socket1)),
150 detach_cmd.format(vpath, ovs_sock_path, log_path),
151 ovs_other_config.format("", "pmd-cpu-mask=%s" % pmd_mask),
156 self.connection.execute(cmd)
157 time.sleep(self.wait_for_vswitchd)
159 def setup_ovs_bridge_add_flows(self):
162 vpath = self.ovs_properties.get("vpath", "/usr/local")
163 version = self.ovs_properties.get('version', {})
164 ovs_ver = [int(x) for x in version.get('ovs', self.DEFAULT_OVS).split('.')]
165 ovs_add_port = ('ovs-vsctl add-port {br} {port} -- '
166 'set Interface {port} type={type_}{dpdk_args}')
167 ovs_add_queue = 'ovs-vsctl set Interface {port} options:n_rxq={queue}'
168 chmod_vpath = 'chmod 0777 {0}/var/run/openvswitch/dpdkvhostuser*'
171 'ovs-vsctl --if-exists del-br {0}'.format(MAIN_BRIDGE),
172 'rm -rf {0}/var/run/openvswitch/dpdkvhostuser*'.format(vpath),
173 'ovs-vsctl add-br {0} -- set bridge {0} datapath_type=netdev'.
177 ordered_network = collections.OrderedDict(self.networks)
178 for index, vnf in enumerate(ordered_network.values()):
179 if ovs_ver >= [2, 7, 0]:
180 dpdk_args = " options:dpdk-devargs=%s" % vnf.get("phy_port")
181 dpdk_list.append(ovs_add_port.format(
182 br=MAIN_BRIDGE, port='dpdk%s' % vnf.get("port_num", 0),
183 type_='dpdk', dpdk_args=dpdk_args))
184 dpdk_list.append(ovs_add_queue.format(
185 port='dpdk%s' % vnf.get("port_num", 0),
186 queue=self.ovs_properties.get("queues", 1)))
188 # Sorting the array to make sure we execute dpdk0... in the order
190 cmd_list.extend(dpdk_list)
192 # Need to do two for loop to maintain the dpdk/vhost ports.
193 for index, _ in enumerate(ordered_network):
194 cmd_list.append(ovs_add_port.format(
195 br=MAIN_BRIDGE, port='dpdkvhostuser%s' % index,
196 type_='dpdkvhostuser', dpdk_args=""))
198 ovs_flow = ("ovs-ofctl add-flow {0} in_port=%s,action=output:%s".
200 network_count = len(ordered_network) + 1
201 for in_port, out_port in zip(range(1, network_count),
202 range(network_count, network_count * 2)):
203 cmd_list.append(ovs_flow % (in_port, out_port))
204 cmd_list.append(ovs_flow % (out_port, in_port))
206 cmd_list.append(chmod_vpath.format(vpath))
210 exit_status, _, stderr = self.connection.execute(
211 cmd, timeout=self.CMD_TIMEOUT)
213 raise exceptions.OVSSetupError(command=cmd, error=stderr)
215 def _check_hugepages(self):
216 meminfo = io.BytesIO()
217 self.connection.get_file_obj('/proc/meminfo', meminfo)
218 regex = re.compile(r"HugePages_Total:\s+(?P<hp_total>\d+)[\n\r]"
219 r"HugePages_Free:\s+(?P<hp_free>\d+)")
220 match = regex.search(meminfo.getvalue().decode('utf-8'))
222 raise exceptions.OVSHugepagesInfoError()
223 if int(match.group('hp_total')) == 0:
224 raise exceptions.OVSHugepagesNotConfigured()
225 if int(match.group('hp_free')) == 0:
226 raise exceptions.OVSHugepagesZeroFree(
227 total_hugepages=int(match.group('hp_total')))
229 def cleanup_ovs_dpdk_env(self):
230 self.connection.execute(
231 'ovs-vsctl --if-exists del-br {0}'.format(MAIN_BRIDGE))
232 self.connection.execute("pkill -9 ovs")
234 def check_ovs_dpdk_env(self):
235 self.cleanup_ovs_dpdk_env()
236 self._check_hugepages()
238 version = self.ovs_properties.get("version", {})
239 ovs_ver = version.get("ovs", self.DEFAULT_OVS)
240 dpdk_ver = version.get("dpdk", "16.07.2").split('.')
242 supported_version = self.SUPPORTED_OVS_TO_DPDK_MAP.get(ovs_ver, None)
243 if supported_version is None or supported_version.split('.')[:2] != dpdk_ver[:2]:
244 raise exceptions.OVSUnsupportedVersion(
246 ovs_to_dpdk_map=self.SUPPORTED_OVS_TO_DPDK_MAP)
248 status = self.connection.execute("ovs-vsctl -V | grep -i '%s'" % ovs_ver)[0]
250 deploy = model.OvsDeploy(self.connection,
251 utils.get_nsb_option("bin_path"),
256 """don't need to deploy"""
258 # Todo: NFVi deploy (sriov, vswitch, ovs etc) based on the config.
259 if not self.vm_deploy:
262 self.connection = ssh.SSH.from_node(self.host_mgmt)
263 self.dpdk_devbind = utils.provision_tool(
265 os.path.join(utils.get_nsb_option('bin_path'), 'dpdk-devbind.py'))
267 # Check dpdk/ovs version, if not present install
268 self.check_ovs_dpdk_env()
269 # Todo: NFVi deploy (sriov, vswitch, ovs etc) based on the config.
270 model.StandaloneContextHelper.install_req_libs(self.connection)
271 self.networks = model.StandaloneContextHelper.get_nic_details(
272 self.connection, self.networks, self.dpdk_devbind)
275 self.start_ovs_serverswitch()
276 self.setup_ovs_bridge_add_flows()
277 self.nodes = self.setup_ovs_dpdk_context()
278 LOG.debug("Waiting for VM to come up...")
279 self.nodes = model.StandaloneContextHelper.wait_for_vnfs_to_start(
280 self.connection, self.servers, self.nodes)
284 if not self.vm_deploy:
287 # Cleanup the ovs installation...
288 self.cleanup_ovs_dpdk_env()
290 # Bind nics back to kernel
291 bind_cmd = "{dpdk_devbind} --force -b {driver} {port}"
292 for port in self.networks.values():
293 vpci = port.get("phy_port")
294 phy_driver = port.get("driver")
295 self.connection.execute(bind_cmd.format(
296 dpdk_devbind=self.dpdk_devbind, driver=phy_driver, port=vpci))
298 # Todo: NFVi undeploy (sriov, vswitch, ovs etc) based on the config.
299 for vm in self.vm_names:
300 model.Libvirt.check_if_vm_exists_and_delete(vm, self.connection)
302 def _get_server(self, attr_name):
303 """lookup server info by name from context
306 attr_name -- A name for a server listed in nodes config file
308 node_name, name = self.split_name(attr_name)
309 if name is None or self.name != name:
312 matching_nodes = (n for n in self.nodes if n["name"] == node_name)
314 # A clone is created in order to avoid affecting the
316 node = dict(next(matching_nodes))
317 except StopIteration:
321 duplicate = next(matching_nodes)
322 except StopIteration:
325 raise ValueError("Duplicate nodes!!! Nodes: %s %s" % (node, duplicate))
327 node["name"] = attr_name
330 def _get_network(self, attr_name):
331 if not isinstance(attr_name, collections.Mapping):
332 network = self.networks.get(attr_name)
335 # Don't generalize too much Just support vld_id
336 vld_id = attr_name.get('vld_id', {})
337 # for standalone context networks are dicts
338 iter1 = (n for n in self.networks.values() if n.get('vld_id') == vld_id)
339 network = next(iter1, None)
346 "name": network["name"],
347 "vld_id": network.get("vld_id"),
348 "segmentation_id": network.get("segmentation_id"),
349 "network_type": network.get("network_type"),
350 "physical_network": network.get("physical_network"),
354 def configure_nics_for_ovs_dpdk(self):
355 portlist = collections.OrderedDict(self.networks)
357 mac = model.StandaloneContextHelper.get_mac_address()
358 portlist[key].update({'mac': mac})
359 self.networks = portlist
360 LOG.info("Ports %s", self.networks)
362 def _enable_interfaces(self, index, vfs, cfg):
363 vpath = self.ovs_properties.get("vpath", "/usr/local")
364 vf = self.networks[vfs[0]]
365 port_num = vf.get('port_num', 0)
366 vpci = utils.PciAddress(vf['vpci'].strip())
367 # Generate the vpci for the interfaces
368 slot = index + port_num + 10
370 "{}:{}:{:02x}.{}".format(vpci.domain, vpci.bus, slot, vpci.function)
371 model.Libvirt.add_ovs_interface(
372 vpath, port_num, vf['vpci'], vf['mac'], str(cfg))
374 def setup_ovs_dpdk_context(self):
377 self.configure_nics_for_ovs_dpdk()
379 for index, (key, vnf) in enumerate(collections.OrderedDict(
380 self.servers).items()):
381 cfg = '/tmp/vm_ovs_%d.xml' % index
382 vm_name = "vm_%d" % index
384 # 1. Check and delete VM if already exists
385 model.Libvirt.check_if_vm_exists_and_delete(vm_name,
388 _, mac = model.Libvirt.build_vm_xml(
389 self.connection, self.vm_flavor, cfg, vm_name, index)
390 # 2: Cleanup already available VMs
391 for vkey, vfs in collections.OrderedDict(
392 vnf["network_ports"]).items():
395 self._enable_interfaces(index, vfs, cfg)
397 # copy xml to target...
398 self.connection.put(cfg, cfg)
400 # NOTE: launch through libvirt
401 LOG.info("virsh create ...")
402 model.Libvirt.virsh_create_vm(self.connection, cfg)
404 self.vm_names.append(vm_name)
406 # build vnf node details
407 nodes.append(self.vnf_node.generate_vnf_instance(self.vm_flavor,
409 self.host_mgmt.get('ip'),