1 # Copyright (c) 2016-2017 Intel Corporation
3 # Licensed under the Apache License, Version 2.0 (the "License");
4 # you may not use this file except in compliance with the License.
5 # You may obtain a copy of the License at
7 # http://www.apache.org/licenses/LICENSE-2.0
9 # Unless required by applicable law or agreed to in writing, software
10 # distributed under the License is distributed on an "AS IS" BASIS,
11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 # See the License for the specific language governing permissions and
13 # limitations under the License.
22 from yardstick import ssh
23 from yardstick.benchmark import contexts
24 from yardstick.benchmark.contexts import base
25 from yardstick.benchmark.contexts.standalone import model
26 from yardstick.common import exceptions
27 from yardstick.network_services import utils
28 from yardstick.network_services.utils import get_nsb_option
31 LOG = logging.getLogger(__name__)
36 class OvsDpdkContext(base.Context):
37 """ This class handles OVS standalone nodes - VM running on Non-Managed NFVi
38 Configuration: ovs_dpdk
41 __context_type__ = contexts.CONTEXT_STANDALONEOVSDPDK
43 SUPPORTED_OVS_TO_DPDK_MAP = {
54 DEFAULT_USER_PATH = '/usr/local'
60 self.dpdk_devbind = os.path.join(get_nsb_option('bin_path'),
69 self.helper = model.StandaloneContextHelper()
70 self.vnf_node = model.Server()
71 self.ovs_properties = {}
72 self.wait_for_vswitchd = 10
73 super(OvsDpdkContext, self).__init__()
75 def init(self, attrs):
76 """initializes itself from the supplied arguments"""
77 super(OvsDpdkContext, self).init(attrs)
79 self.file_path = attrs.get("file", "pod.yaml")
81 self.nodes, self.nfvi_host, self.host_mgmt = \
82 self.helper.parse_pod_file(self.file_path, 'OvsDpdk')
85 self.vm_flavor = attrs.get('flavor', {})
86 self.servers = attrs.get('servers', {})
87 self.vm_deploy = attrs.get("vm_deploy", True)
88 self.ovs_properties = attrs.get('ovs_properties', {})
89 # add optional static network definition
90 self.networks = attrs.get("networks", {})
92 LOG.debug("Nodes: %r", self.nodes)
93 LOG.debug("NFVi Node: %r", self.nfvi_host)
94 LOG.debug("Networks: %r", self.networks)
97 """Initialize OVS-DPDK"""
98 vpath = self.ovs_properties.get('vpath', self.DEFAULT_USER_PATH)
99 create_from = os.path.join(vpath, 'etc/openvswitch/conf.db')
100 create_to = os.path.join(vpath, 'share/openvswitch/vswitch.ovsschema')
103 'killall -r "ovs.*" -q | true',
104 'mkdir -p {0}/etc/openvswitch'.format(vpath),
105 'mkdir -p {0}/var/run/openvswitch'.format(vpath),
106 'rm {0}/etc/openvswitch/conf.db | true'.format(vpath),
107 'ovsdb-tool create {0} {1}'.format(create_from, create_to),
109 'chmod a+x /dev/vfio',
110 'chmod 0666 /dev/vfio/*',
113 bind_cmd = '%s --force -b vfio-pci {port}' % self.dpdk_devbind
114 for port in self.networks.values():
115 cmd_list.append(bind_cmd.format(port=port.get('phy_port')))
119 exit_status, _, stderr = self.connection.execute(
120 cmd, timeout=self.CMD_TIMEOUT)
122 raise exceptions.OVSSetupError(command=cmd, error=stderr)
124 def start_ovs_serverswitch(self):
125 vpath = self.ovs_properties.get("vpath")
126 pmd_nums = int(self.ovs_properties.get("pmd_threads", 2))
127 ovs_sock_path = '/var/run/openvswitch/db.sock'
128 log_path = '/var/log/openvswitch/ovs-vswitchd.log'
130 pmd_cpu_mask = self.ovs_properties.get("pmd_cpu_mask", '')
131 pmd_mask = hex(sum(2 ** num for num in range(pmd_nums)) << 1)
133 pmd_mask = pmd_cpu_mask
135 socket0 = self.ovs_properties.get("ram", {}).get("socket_0", "2048")
136 socket1 = self.ovs_properties.get("ram", {}).get("socket_1", "2048")
138 ovs_other_config = "ovs-vsctl {0}set Open_vSwitch . other_config:{1}"
139 detach_cmd = "ovs-vswitchd unix:{0}{1} --pidfile --detach --log-file={2}"
141 lcore_mask = self.ovs_properties.get("lcore_mask", '')
143 lcore_mask = ovs_other_config.format("--no-wait ", "dpdk-lcore-mask='%s'" % lcore_mask)
146 "mkdir -p /usr/local/var/run/openvswitch",
147 "mkdir -p {}".format(os.path.dirname(log_path)),
148 "ovsdb-server --remote=punix:/{0}/{1} --pidfile --detach".format(vpath,
150 ovs_other_config.format("--no-wait ", "dpdk-init=true"),
151 ovs_other_config.format("--no-wait ", "dpdk-socket-mem='%s,%s'" % (socket0, socket1)),
153 detach_cmd.format(vpath, ovs_sock_path, log_path),
154 ovs_other_config.format("", "pmd-cpu-mask=%s" % pmd_mask),
159 self.connection.execute(cmd)
160 time.sleep(self.wait_for_vswitchd)
162 def setup_ovs_bridge_add_flows(self):
165 vpath = self.ovs_properties.get("vpath", "/usr/local")
166 version = self.ovs_properties.get('version', {})
167 ovs_ver = [int(x) for x in version.get('ovs', self.DEFAULT_OVS).split('.')]
168 ovs_add_port = ('ovs-vsctl add-port {br} {port} -- '
169 'set Interface {port} type={type_}{dpdk_args}')
170 ovs_add_queue = 'ovs-vsctl set Interface {port} options:n_rxq={queue}'
171 chmod_vpath = 'chmod 0777 {0}/var/run/openvswitch/dpdkvhostuser*'
174 'ovs-vsctl --if-exists del-br {0}'.format(MAIN_BRIDGE),
175 'rm -rf {0}/var/run/openvswitch/dpdkvhostuser*'.format(vpath),
176 'ovs-vsctl add-br {0} -- set bridge {0} datapath_type=netdev'.
180 ordered_network = collections.OrderedDict(self.networks)
181 for index, vnf in enumerate(ordered_network.values()):
182 if ovs_ver >= [2, 7, 0]:
183 dpdk_args = " options:dpdk-devargs=%s" % vnf.get("phy_port")
184 dpdk_list.append(ovs_add_port.format(
185 br=MAIN_BRIDGE, port='dpdk%s' % vnf.get("port_num", 0),
186 type_='dpdk', dpdk_args=dpdk_args))
187 dpdk_list.append(ovs_add_queue.format(
188 port='dpdk%s' % vnf.get("port_num", 0),
189 queue=self.ovs_properties.get("queues", 1)))
191 # Sorting the array to make sure we execute dpdk0... in the order
193 cmd_list.extend(dpdk_list)
195 # Need to do two for loop to maintain the dpdk/vhost ports.
196 for index, _ in enumerate(ordered_network):
197 cmd_list.append(ovs_add_port.format(
198 br=MAIN_BRIDGE, port='dpdkvhostuser%s' % index,
199 type_='dpdkvhostuser', dpdk_args=""))
201 ovs_flow = ("ovs-ofctl add-flow {0} in_port=%s,action=output:%s".
203 network_count = len(ordered_network) + 1
204 for in_port, out_port in zip(range(1, network_count),
205 range(network_count, network_count * 2)):
206 cmd_list.append(ovs_flow % (in_port, out_port))
207 cmd_list.append(ovs_flow % (out_port, in_port))
209 cmd_list.append(chmod_vpath.format(vpath))
213 exit_status, _, stderr = self.connection.execute(
214 cmd, timeout=self.CMD_TIMEOUT)
216 raise exceptions.OVSSetupError(command=cmd, error=stderr)
218 def _check_hugepages(self):
219 meminfo = io.BytesIO()
220 self.connection.get_file_obj('/proc/meminfo', meminfo)
221 regex = re.compile(r"HugePages_Total:\s+(?P<hp_total>\d+)[\n\r]"
222 r"HugePages_Free:\s+(?P<hp_free>\d+)")
223 match = regex.search(meminfo.getvalue().decode('utf-8'))
225 raise exceptions.OVSHugepagesInfoError()
226 if int(match.group('hp_total')) == 0:
227 raise exceptions.OVSHugepagesNotConfigured()
228 if int(match.group('hp_free')) == 0:
229 raise exceptions.OVSHugepagesZeroFree(
230 total_hugepages=int(match.group('hp_total')))
232 def cleanup_ovs_dpdk_env(self):
233 self.connection.execute(
234 'ovs-vsctl --if-exists del-br {0}'.format(MAIN_BRIDGE))
235 self.connection.execute("pkill -9 ovs")
237 def check_ovs_dpdk_env(self):
238 self.cleanup_ovs_dpdk_env()
239 self._check_hugepages()
241 version = self.ovs_properties.get("version", {})
242 ovs_ver = version.get("ovs", self.DEFAULT_OVS)
243 dpdk_ver = version.get("dpdk", "16.07.2").split('.')
245 supported_version = self.SUPPORTED_OVS_TO_DPDK_MAP.get(ovs_ver, None)
246 if supported_version is None or supported_version.split('.')[:2] != dpdk_ver[:2]:
247 raise exceptions.OVSUnsupportedVersion(
249 ovs_to_dpdk_map=self.SUPPORTED_OVS_TO_DPDK_MAP)
251 status = self.connection.execute("ovs-vsctl -V | grep -i '%s'" % ovs_ver)[0]
253 deploy = model.OvsDeploy(self.connection,
254 utils.get_nsb_option("bin_path"),
259 """don't need to deploy"""
261 # Todo: NFVi deploy (sriov, vswitch, ovs etc) based on the config.
262 if not self.vm_deploy:
265 self.connection = ssh.SSH.from_node(self.host_mgmt)
267 # Check dpdk/ovs version, if not present install
268 self.check_ovs_dpdk_env()
269 # Todo: NFVi deploy (sriov, vswitch, ovs etc) based on the config.
270 model.StandaloneContextHelper.install_req_libs(self.connection)
271 self.networks = model.StandaloneContextHelper.get_nic_details(
272 self.connection, self.networks, self.dpdk_devbind)
275 self.start_ovs_serverswitch()
276 self.setup_ovs_bridge_add_flows()
277 self.nodes = self.setup_ovs_dpdk_context()
278 LOG.debug("Waiting for VM to come up...")
279 self.nodes = model.StandaloneContextHelper.wait_for_vnfs_to_start(
280 self.connection, self.servers, self.nodes)
284 if not self.vm_deploy:
287 # Cleanup the ovs installation...
288 self.cleanup_ovs_dpdk_env()
290 # Bind nics back to kernel
291 bind_cmd = "{dpdk_devbind} --force -b {driver} {port}"
292 for port in self.networks.values():
293 vpci = port.get("phy_port")
294 phy_driver = port.get("driver")
295 self.connection.execute(bind_cmd.format(
296 dpdk_devbind=self.dpdk_devbind, driver=phy_driver, port=vpci))
298 # Todo: NFVi undeploy (sriov, vswitch, ovs etc) based on the config.
299 for vm in self.vm_names:
300 model.Libvirt.check_if_vm_exists_and_delete(vm, self.connection)
302 def _get_physical_nodes(self):
303 return self.nfvi_host
305 def _get_physical_node_for_server(self, server_name):
306 node_name, ctx_name = self.split_host_name(server_name)
307 if ctx_name is None or self.name != ctx_name:
310 matching_nodes = [s for s in self.servers if s == node_name]
311 if len(matching_nodes) == 0:
314 # self.nfvi_host always contain only one host
315 return "{}.{}".format(self.nfvi_host[0]["name"], self._name)
317 def _get_server(self, attr_name):
318 """lookup server info by name from context
321 attr_name -- A name for a server listed in nodes config file
323 node_name, name = self.split_host_name(attr_name)
324 if name is None or self.name != name:
327 matching_nodes = (n for n in self.nodes if n["name"] == node_name)
329 # A clone is created in order to avoid affecting the
331 node = dict(next(matching_nodes))
332 except StopIteration:
336 duplicate = next(matching_nodes)
337 except StopIteration:
340 raise ValueError("Duplicate nodes!!! Nodes: %s %s" % (node, duplicate))
342 node["name"] = attr_name
345 def _get_network(self, attr_name):
346 if not isinstance(attr_name, collections.Mapping):
347 network = self.networks.get(attr_name)
350 # Don't generalize too much Just support vld_id
351 vld_id = attr_name.get('vld_id', {})
352 # for standalone context networks are dicts
353 iter1 = (n for n in self.networks.values() if n.get('vld_id') == vld_id)
354 network = next(iter1, None)
361 "name": network["name"],
362 "vld_id": network.get("vld_id"),
363 "segmentation_id": network.get("segmentation_id"),
364 "network_type": network.get("network_type"),
365 "physical_network": network.get("physical_network"),
369 def configure_nics_for_ovs_dpdk(self):
370 portlist = collections.OrderedDict(self.networks)
372 mac = model.StandaloneContextHelper.get_mac_address()
373 portlist[key].update({'mac': mac})
374 self.networks = portlist
375 LOG.info("Ports %s", self.networks)
377 def _enable_interfaces(self, index, vfs, xml_str):
378 vpath = self.ovs_properties.get("vpath", "/usr/local")
379 vf = self.networks[vfs[0]]
380 port_num = vf.get('port_num', 0)
381 vpci = utils.PciAddress(vf['vpci'].strip())
382 # Generate the vpci for the interfaces
383 slot = index + port_num + 10
385 "{}:{}:{:02x}.{}".format(vpci.domain, vpci.bus, slot, vpci.function)
386 return model.Libvirt.add_ovs_interface(
387 vpath, port_num, vf['vpci'], vf['mac'], xml_str)
389 def setup_ovs_dpdk_context(self):
392 self.configure_nics_for_ovs_dpdk()
394 for index, (key, vnf) in enumerate(collections.OrderedDict(
395 self.servers).items()):
396 cfg = '/tmp/vm_ovs_%d.xml' % index
397 vm_name = "vm_%d" % index
399 # 1. Check and delete VM if already exists
400 model.Libvirt.check_if_vm_exists_and_delete(vm_name,
402 xml_str, mac = model.Libvirt.build_vm_xml(
403 self.connection, self.vm_flavor, vm_name, index)
405 # 2: Cleanup already available VMs
406 for vfs in [vfs for vfs_name, vfs in vnf["network_ports"].items()
407 if vfs_name != 'mgmt']:
408 xml_str = self._enable_interfaces(index, vfs, xml_str)
410 # copy xml to target...
411 model.Libvirt.write_file(cfg, xml_str)
412 self.connection.put(cfg, cfg)
414 # NOTE: launch through libvirt
415 LOG.info("virsh create ...")
416 model.Libvirt.virsh_create_vm(self.connection, cfg)
418 self.vm_names.append(vm_name)
420 # build vnf node details
421 nodes.append(self.vnf_node.generate_vnf_instance(self.vm_flavor,
423 self.host_mgmt.get('ip'),