1 # Copyright (c) 2016-2017 Intel Corporation
3 # Licensed under the Apache License, Version 2.0 (the "License");
4 # you may not use this file except in compliance with the License.
5 # You may obtain a copy of the License at
7 # http://www.apache.org/licenses/LICENSE-2.0
9 # Unless required by applicable law or agreed to in writing, software
10 # distributed under the License is distributed on an "AS IS" BASIS,
11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 # See the License for the specific language governing permissions and
13 # limitations under the License.
15 from __future__ import absolute_import
24 from netaddr import IPNetwork
25 import xml.etree.ElementTree as ET
27 from yardstick import ssh
28 from yardstick.common.constants import YARDSTICK_ROOT_PATH
29 from yardstick.common.yaml_loader import yaml_load
30 from yardstick.network_services.utils import PciAddress
31 from yardstick.network_services.helpers.cpu import CpuSysCores
32 from yardstick.common.utils import write_file
34 LOG = logging.getLogger(__name__)
38 <name>{vm_name}</name>
39 <uuid>{random_uuid}</uuid>
40 <memory unit="MB">{memory}</memory>
41 <currentMemory unit="MB">{memory}</currentMemory>
45 <vcpu cpuset='{cpuset}'>{vcpu}</vcpu>
48 <type arch="x86_64" machine="pc-i440fx-utopic">hvm</type>
56 <cpu mode='host-passthrough'>
57 <topology cores="{cpu}" sockets="{socket}" threads="{threads}" />
59 <cell id='0' cpus='{numa_cpus}' memory='{memory}' unit='MB' memAccess='shared'/>
63 <timer name="rtc" tickpolicy="catchup" />
64 <timer name="pit" tickpolicy="delay" />
65 <timer name="hpet" present="no" />
67 <on_poweroff>destroy</on_poweroff>
68 <on_reboot>restart</on_reboot>
69 <on_crash>restart</on_crash>
71 <emulator>/usr/bin/kvm-spice</emulator>
72 <disk device="disk" type="file">
73 <driver name="qemu" type="qcow2" />
74 <source file="{vm_image}"/>
75 <target bus="virtio" dev="vda" />
77 <graphics autoport="yes" listen="0.0.0.0" port="-1" type="vnc" />
78 <interface type="bridge">
79 <mac address='{mac_addr}'/>
80 <source bridge="br-int" />
81 <model type='virtio'/>
87 <target type='serial' port='0'/>
95 class Libvirt(object):
96 """ This class handles all the libvirt updates to lauch VM
100 def check_if_vm_exists_and_delete(vm_name, connection):
101 cmd_template = "virsh list --name | grep -i %s"
102 status = connection.execute(cmd_template % vm_name)[0]
104 LOG.info("VM '%s' is already present... destroying", vm_name)
105 connection.execute("virsh destroy %s" % vm_name)
108 def virsh_create_vm(connection, cfg):
109 err = connection.execute("virsh create %s" % cfg)[0]
110 LOG.info("VM create status: %s", err)
113 def virsh_destroy_vm(vm_name, connection):
114 connection.execute("virsh destroy %s" % vm_name)
117 def _add_interface_address(interface, pci_address):
118 """Add a PCI 'address' XML node
120 <address type='pci' domain='0x0000' bus='0x00' slot='0x08'
123 Refence: https://software.intel.com/en-us/articles/
124 configure-sr-iov-network-virtual-functions-in-linux-kvm
126 vm_pci = ET.SubElement(interface, 'address')
127 vm_pci.set('type', 'pci')
128 vm_pci.set('domain', '0x{}'.format(pci_address.domain))
129 vm_pci.set('bus', '0x{}'.format(pci_address.bus))
130 vm_pci.set('slot', '0x{}'.format(pci_address.slot))
131 vm_pci.set('function', '0x{}'.format(pci_address.function))
135 def add_ovs_interface(cls, vpath, port_num, vpci, vports_mac, xml):
136 """Add a DPDK OVS 'interface' XML node in 'devices' node
139 <interface type='vhostuser'>
140 <mac address='00:00:00:00:00:01'/>
141 <source type='unix' path='/usr/local/var/run/openvswitch/
142 dpdkvhostuser0' mode='client'/>
143 <model type='virtio'/>
145 <host mrg_rxbuf='off'/>
147 <address type='pci' domain='0x0000' bus='0x00' slot='0x03'
153 Reference: http://docs.openvswitch.org/en/latest/topics/dpdk/
157 vhost_path = ('{0}/var/run/openvswitch/dpdkvhostuser{1}'.
158 format(vpath, port_num))
160 pci_address = PciAddress(vpci.strip())
161 device = root.find('devices')
163 interface = ET.SubElement(device, 'interface')
164 interface.set('type', 'vhostuser')
165 mac = ET.SubElement(interface, 'mac')
166 mac.set('address', vports_mac)
168 source = ET.SubElement(interface, 'source')
169 source.set('type', 'unix')
170 source.set('path', vhost_path)
171 source.set('mode', 'client')
173 model = ET.SubElement(interface, 'model')
174 model.set('type', 'virtio')
176 driver = ET.SubElement(interface, 'driver')
177 driver.set('queues', '4')
179 host = ET.SubElement(driver, 'host')
180 host.set('mrg_rxbuf', 'off')
182 cls._add_interface_address(interface, pci_address)
187 def add_sriov_interfaces(cls, vm_pci, vf_pci, vf_mac, xml):
188 """Add a SR-IOV 'interface' XML node in 'devices' node
191 <interface type='hostdev' managed='yes'>
193 <address type='pci' domain='0x0000' bus='0x00' slot='0x03'
196 <mac address='52:54:00:6d:90:02'>
197 <address type='pci' domain='0x0000' bus='0x02' slot='0x04'
203 Reference: https://access.redhat.com/documentation/en-us/
204 red_hat_enterprise_linux/6/html/
205 virtualization_host_configuration_and_guest_installation_guide/
206 sect-virtualization_host_configuration_and_guest_installation_guide
207 -sr_iov-how_sr_iov_libvirt_works
211 device = root.find('devices')
213 interface = ET.SubElement(device, 'interface')
214 interface.set('managed', 'yes')
215 interface.set('type', 'hostdev')
217 mac = ET.SubElement(interface, 'mac')
218 mac.set('address', vf_mac)
220 source = ET.SubElement(interface, 'source')
221 pci_address = PciAddress(vf_pci.strip())
222 cls._add_interface_address(source, pci_address)
224 pci_vm_address = PciAddress(vm_pci.strip())
225 cls._add_interface_address(interface, pci_vm_address)
230 def create_snapshot_qemu(connection, index, vm_image):
231 # build snapshot image
232 image = "/var/lib/libvirt/images/%s.qcow2" % index
233 connection.execute("rm %s" % image)
234 qemu_template = "qemu-img create -f qcow2 -o backing_file=%s %s"
235 connection.execute(qemu_template % (vm_image, image))
240 def build_vm_xml(cls, connection, flavor, cfg, vm_name, index):
241 memory = flavor.get('ram', '4096')
242 extra_spec = flavor.get('extra_specs', {})
243 cpu = extra_spec.get('hw:cpu_cores', '2')
244 socket = extra_spec.get('hw:cpu_sockets', '1')
245 threads = extra_spec.get('hw:cpu_threads', '2')
246 vcpu = int(cpu) * int(threads)
247 numa_cpus = '0-%s' % (vcpu - 1)
248 hw_socket = flavor.get('hw_socket', '0')
249 cpuset = Libvirt.pin_vcpu_for_perf(connection, hw_socket)
251 cputune = extra_spec.get('cputune', '')
252 mac = StandaloneContextHelper.get_mac_address(0x00)
253 image = cls.create_snapshot_qemu(connection, index,
254 flavor.get("images", None))
255 vm_xml = VM_TEMPLATE.format(
257 random_uuid=uuid.uuid4(),
259 memory=memory, vcpu=vcpu, cpu=cpu,
261 socket=socket, threads=threads,
262 vm_image=image, cpuset=cpuset, cputune=cputune)
264 write_file(cfg, vm_xml)
269 def update_interrupts_hugepages_perf(connection):
270 connection.execute("echo 1 > /sys/module/kvm/parameters/allow_unsafe_assigned_interrupts")
271 connection.execute("echo never > /sys/kernel/mm/transparent_hugepage/enabled")
274 def pin_vcpu_for_perf(cls, connection, socket='0'):
276 sys_obj = CpuSysCores(connection)
277 soc_cpu = sys_obj.get_core_socket()
278 sys_cpu = int(soc_cpu["cores_per_socket"])
280 cores = "%s-%s" % (soc_cpu[socket][0], soc_cpu[socket][sys_cpu - 1])
281 if int(soc_cpu["thread_per_core"]) > 1:
282 threads = "%s-%s" % (soc_cpu[socket][sys_cpu], soc_cpu[socket][-1])
283 cpuset = "%s,%s" % (cores, threads)
287 class StandaloneContextHelper(object):
288 """ This class handles all the common code for standalone
291 self.file_path = None
292 super(StandaloneContextHelper, self).__init__()
295 def install_req_libs(connection, extra_pkgs=None):
296 extra_pkgs = extra_pkgs or []
297 pkgs = ["qemu-kvm", "libvirt-bin", "bridge-utils", "numactl", "fping"]
298 pkgs.extend(extra_pkgs)
299 cmd_template = "dpkg-query -W --showformat='${Status}\\n' \"%s\"|grep 'ok installed'"
301 if connection.execute(cmd_template % pkg)[0]:
302 connection.execute("apt-get update")
303 connection.execute("apt-get -y install %s" % pkg)
306 def get_kernel_module(connection, pci, driver):
308 out = connection.execute("lspci -k -s %s" % pci)[1]
309 driver = out.split("Kernel modules:").pop().strip()
313 def get_nic_details(cls, connection, networks, dpdk_devbind):
314 for key, ports in networks.items():
318 phy_ports = ports['phy_port']
319 phy_driver = ports.get('phy_driver', None)
320 driver = cls.get_kernel_module(connection, phy_ports, phy_driver)
322 # Make sure that ports are bound to kernel drivers e.g. i40e/ixgbe
323 bind_cmd = "{dpdk_devbind} --force -b {driver} {port}"
324 lshw_cmd = "lshw -c network -businfo | grep '{port}'"
325 link_show_cmd = "ip -s link show {interface}"
327 cmd = bind_cmd.format(dpdk_devbind=dpdk_devbind,
328 driver=driver, port=ports['phy_port'])
329 connection.execute(cmd)
331 out = connection.execute(lshw_cmd.format(port=phy_ports))[1]
332 interface = out.split()[1]
334 connection.execute(link_show_cmd.format(interface=interface))
337 'interface': str(interface),
345 def get_virtual_devices(connection, pci):
346 cmd = "cat /sys/bus/pci/devices/{0}/virtfn0/uevent"
347 output = connection.execute(cmd.format(pci))[1]
349 pattern = "PCI_SLOT_NAME=({})".format(PciAddress.PCI_PATTERN_STR)
350 m = re.search(pattern, output, re.MULTILINE)
354 pf_vfs = {pci: m.group(1).rstrip()}
356 LOG.info("pf_vfs:\n%s", pf_vfs)
360 def read_config_file(self):
361 """Read from config file"""
363 with open(self.file_path) as stream:
364 LOG.info("Parsing pod file: %s", self.file_path)
365 cfg = yaml_load(stream)
368 def parse_pod_file(self, file_path, nfvi_role='Sriov'):
369 self.file_path = file_path
373 cfg = self.read_config_file()
374 except IOError as io_error:
375 if io_error.errno != errno.ENOENT:
377 self.file_path = os.path.join(YARDSTICK_ROOT_PATH, file_path)
378 cfg = self.read_config_file()
380 nodes.extend([node for node in cfg["nodes"] if str(node["role"]) != nfvi_role])
381 nfvi_host.extend([node for node in cfg["nodes"] if str(node["role"]) == nfvi_role])
383 raise("Node role is other than SRIOV")
385 host_mgmt = {'user': nfvi_host[0]['user'],
386 'ip': str(IPNetwork(nfvi_host[0]['ip']).ip),
387 'password': nfvi_host[0]['password'],
388 'ssh_port': nfvi_host[0].get('ssh_port', 22),
389 'key_filename': nfvi_host[0].get('key_filename')}
391 return [nodes, nfvi_host, host_mgmt]
394 def get_mac_address(end=0x7f):
395 mac = [0x52, 0x54, 0x00,
396 random.randint(0x00, end),
397 random.randint(0x00, 0xff),
398 random.randint(0x00, 0xff)]
399 mac_address = ':'.join('%02x' % x for x in mac)
403 def get_mgmt_ip(connection, mac, cidr, node):
406 while not mgmtip and times:
407 connection.execute("fping -c 1 -g %s > /dev/null 2>&1" % cidr)
408 out = connection.execute("ip neighbor | grep '%s'" % mac)[1]
409 LOG.info("fping -c 1 -g %s > /dev/null 2>&1", cidr)
411 mgmtip = str(out.split(" ")[0]).strip()
412 client = ssh.SSH.from_node(node, overrides={"ip": mgmtip})
416 time.sleep(WAIT_FOR_BOOT) # FixMe: How to find if VM is booted?
421 def wait_for_vnfs_to_start(cls, connection, servers, nodes):
423 vnf = servers[node["name"]]
424 mgmtip = vnf["network_ports"]["mgmt"]["cidr"]
425 ip = cls.get_mgmt_ip(connection, node["mac"], mgmtip, node)
431 class Server(object):
432 """ This class handles geting vnf nodes
436 def build_vnf_interfaces(vnf, ports):
440 for key, vfs in vnf["network_ports"].items():
442 mgmtip = str(IPNetwork(vfs['cidr']).ip)
446 ip = IPNetwork(vf['cidr'])
450 'driver': "%svf" % vf['driver'],
451 'local_mac': vf['mac'],
452 'dpdk_port_num': index,
453 'local_ip': str(ip.ip),
454 'netmask': str(ip.netmask)
459 return mgmtip, interfaces
462 def generate_vnf_instance(cls, flavor, ports, ip, key, vnf, mac):
463 mgmtip, interfaces = cls.build_vnf_interfaces(vnf, ports)
469 "user": flavor.get('user', 'root'),
470 "interfaces": interfaces,
472 # empty IPv6 routing table
474 "name": key, "role": key
478 result['key_filename'] = flavor['key_filename']
483 result['password'] = flavor['password']
490 class OvsDeploy(object):
491 """ This class handles deploy of ovs dpdk
492 Configuration: ovs_dpdk
495 OVS_DEPLOY_SCRIPT = "ovs_deploy.bash"
497 def __init__(self, connection, bin_path, ovs_properties):
498 self.connection = connection
499 self.bin_path = bin_path
500 self.ovs_properties = ovs_properties
502 def prerequisite(self):
503 pkgs = ["git", "build-essential", "pkg-config", "automake",
504 "autotools-dev", "libltdl-dev", "cmake", "libnuma-dev",
506 StandaloneContextHelper.install_req_libs(self.connection, pkgs)
508 def ovs_deploy(self):
509 ovs_deploy = os.path.join(YARDSTICK_ROOT_PATH,
510 "yardstick/resources/scripts/install/",
511 self.OVS_DEPLOY_SCRIPT)
512 if os.path.isfile(ovs_deploy):
514 remote_ovs_deploy = os.path.join(self.bin_path, self.OVS_DEPLOY_SCRIPT)
515 LOG.info(remote_ovs_deploy)
516 self.connection.put(ovs_deploy, remote_ovs_deploy)
518 http_proxy = os.environ.get('http_proxy', '')
519 ovs_details = self.ovs_properties.get("version", {})
520 ovs = ovs_details.get("ovs", "2.6.0")
521 dpdk = ovs_details.get("dpdk", "16.11.1")
523 cmd = "sudo -E %s --ovs='%s' --dpdk='%s' -p='%s'" % (remote_ovs_deploy,
524 ovs, dpdk, http_proxy)
525 self.connection.execute(cmd)