1 # Copyright (c) 2016-2017 Intel Corporation
3 # Licensed under the Apache License, Version 2.0 (the "License");
4 # you may not use this file except in compliance with the License.
5 # You may obtain a copy of the License at
7 # http://www.apache.org/licenses/LICENSE-2.0
9 # Unless required by applicable law or agreed to in writing, software
10 # distributed under the License is distributed on an "AS IS" BASIS,
11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 # See the License for the specific language governing permissions and
13 # limitations under the License.
23 from netaddr import IPNetwork
24 import xml.etree.ElementTree as ET
26 from yardstick import ssh
27 from yardstick.common import constants
28 from yardstick.common import exceptions
29 from yardstick.common.yaml_loader import yaml_load
30 from yardstick.network_services.utils import PciAddress
31 from yardstick.network_services.helpers.cpu import CpuSysCores
34 LOG = logging.getLogger(__name__)
38 <name>{vm_name}</name>
39 <uuid>{random_uuid}</uuid>
40 <memory unit="MB">{memory}</memory>
41 <currentMemory unit="MB">{memory}</currentMemory>
45 <vcpu cpuset='{cpuset}'>{vcpu}</vcpu>
48 <type arch="x86_64" machine="pc-i440fx-utopic">hvm</type>
56 <cpu mode='host-passthrough'>
57 <topology cores="{cpu}" sockets="{socket}" threads="{threads}" />
59 <cell id='0' cpus='{numa_cpus}' memory='{memory}' unit='MB' memAccess='shared'/>
63 <timer name="rtc" tickpolicy="catchup" />
64 <timer name="pit" tickpolicy="delay" />
65 <timer name="hpet" present="no" />
67 <on_poweroff>destroy</on_poweroff>
68 <on_reboot>restart</on_reboot>
69 <on_crash>restart</on_crash>
71 <emulator>/usr/bin/kvm-spice</emulator>
72 <disk device="disk" type="file">
73 <driver name="qemu" type="qcow2" />
74 <source file="{vm_image}"/>
75 <target bus="virtio" dev="vda" />
77 <graphics autoport="yes" listen="0.0.0.0" port="-1" type="vnc" />
78 <interface type="bridge">
79 <mac address='{mac_addr}'/>
80 <source bridge="br-int" />
81 <model type='virtio'/>
87 <target type='serial' port='0'/>
95 class Libvirt(object):
96 """ This class handles all the libvirt updates to lauch VM
100 def check_if_vm_exists_and_delete(vm_name, connection):
101 cmd_template = "virsh list --name | grep -i %s"
102 status = connection.execute(cmd_template % vm_name)[0]
104 LOG.info("VM '%s' is already present... destroying", vm_name)
105 connection.execute("virsh destroy %s" % vm_name)
108 def virsh_create_vm(connection, cfg):
109 LOG.info('VM create, XML config: %s', cfg)
110 status, _, error = connection.execute('virsh create %s' % cfg)
112 raise exceptions.LibvirtCreateError(error=error)
115 def virsh_destroy_vm(vm_name, connection):
116 connection.execute("virsh destroy %s" % vm_name)
119 def _add_interface_address(interface, pci_address):
120 """Add a PCI 'address' XML node
122 <address type='pci' domain='0x0000' bus='0x00' slot='0x08'
125 Refence: https://software.intel.com/en-us/articles/
126 configure-sr-iov-network-virtual-functions-in-linux-kvm
128 vm_pci = ET.SubElement(interface, 'address')
129 vm_pci.set('type', 'pci')
130 vm_pci.set('domain', '0x{}'.format(pci_address.domain))
131 vm_pci.set('bus', '0x{}'.format(pci_address.bus))
132 vm_pci.set('slot', '0x{}'.format(pci_address.slot))
133 vm_pci.set('function', '0x{}'.format(pci_address.function))
137 def add_ovs_interface(cls, vpath, port_num, vpci, vports_mac, xml_str):
138 """Add a DPDK OVS 'interface' XML node in 'devices' node
141 <interface type='vhostuser'>
142 <mac address='00:00:00:00:00:01'/>
143 <source type='unix' path='/usr/local/var/run/openvswitch/
144 dpdkvhostuser0' mode='client'/>
145 <model type='virtio'/>
147 <host mrg_rxbuf='off'/>
149 <address type='pci' domain='0x0000' bus='0x00' slot='0x03'
155 Reference: http://docs.openvswitch.org/en/latest/topics/dpdk/
159 vhost_path = ('{0}/var/run/openvswitch/dpdkvhostuser{1}'.
160 format(vpath, port_num))
161 root = ET.fromstring(xml_str)
162 pci_address = PciAddress(vpci.strip())
163 device = root.find('devices')
165 interface = ET.SubElement(device, 'interface')
166 interface.set('type', 'vhostuser')
167 mac = ET.SubElement(interface, 'mac')
168 mac.set('address', vports_mac)
170 source = ET.SubElement(interface, 'source')
171 source.set('type', 'unix')
172 source.set('path', vhost_path)
173 source.set('mode', 'client')
175 model = ET.SubElement(interface, 'model')
176 model.set('type', 'virtio')
178 driver = ET.SubElement(interface, 'driver')
179 driver.set('queues', '4')
181 host = ET.SubElement(driver, 'host')
182 host.set('mrg_rxbuf', 'off')
184 cls._add_interface_address(interface, pci_address)
186 return ET.tostring(root)
189 def add_sriov_interfaces(cls, vm_pci, vf_pci, vf_mac, xml_str):
190 """Add a SR-IOV 'interface' XML node in 'devices' node
193 <interface type='hostdev' managed='yes'>
195 <address type='pci' domain='0x0000' bus='0x00' slot='0x03'
198 <mac address='52:54:00:6d:90:02'>
199 <address type='pci' domain='0x0000' bus='0x02' slot='0x04'
205 Reference: https://access.redhat.com/documentation/en-us/
206 red_hat_enterprise_linux/6/html/
207 virtualization_host_configuration_and_guest_installation_guide/
208 sect-virtualization_host_configuration_and_guest_installation_guide
209 -sr_iov-how_sr_iov_libvirt_works
212 root = ET.fromstring(xml_str)
213 device = root.find('devices')
215 interface = ET.SubElement(device, 'interface')
216 interface.set('managed', 'yes')
217 interface.set('type', 'hostdev')
219 mac = ET.SubElement(interface, 'mac')
220 mac.set('address', vf_mac)
222 source = ET.SubElement(interface, 'source')
223 pci_address = PciAddress(vf_pci.strip())
224 cls._add_interface_address(source, pci_address)
226 pci_vm_address = PciAddress(vm_pci.strip())
227 cls._add_interface_address(interface, pci_vm_address)
229 return ET.tostring(root)
232 def create_snapshot_qemu(connection, index, vm_image):
233 # build snapshot image
234 image = "/var/lib/libvirt/images/%s.qcow2" % index
235 connection.execute("rm %s" % image)
236 qemu_template = "qemu-img create -f qcow2 -o backing_file=%s %s"
237 connection.execute(qemu_template % (vm_image, image))
242 def build_vm_xml(cls, connection, flavor, vm_name, index):
243 """Build the XML from the configuration parameters"""
244 memory = flavor.get('ram', '4096')
245 extra_spec = flavor.get('extra_specs', {})
246 cpu = extra_spec.get('hw:cpu_cores', '2')
247 socket = extra_spec.get('hw:cpu_sockets', '1')
248 threads = extra_spec.get('hw:cpu_threads', '2')
249 vcpu = int(cpu) * int(threads)
250 numa_cpus = '0-%s' % (vcpu - 1)
251 hw_socket = flavor.get('hw_socket', '0')
252 cpuset = Libvirt.pin_vcpu_for_perf(connection, hw_socket)
254 cputune = extra_spec.get('cputune', '')
255 mac = StandaloneContextHelper.get_mac_address(0x00)
256 image = cls.create_snapshot_qemu(connection, index,
257 flavor.get("images", None))
258 vm_xml = VM_TEMPLATE.format(
260 random_uuid=uuid.uuid4(),
262 memory=memory, vcpu=vcpu, cpu=cpu,
264 socket=socket, threads=threads,
265 vm_image=image, cpuset=cpuset, cputune=cputune)
270 def update_interrupts_hugepages_perf(connection):
271 connection.execute("echo 1 > /sys/module/kvm/parameters/allow_unsafe_assigned_interrupts")
272 connection.execute("echo never > /sys/kernel/mm/transparent_hugepage/enabled")
275 def pin_vcpu_for_perf(cls, connection, socket='0'):
277 sys_obj = CpuSysCores(connection)
278 soc_cpu = sys_obj.get_core_socket()
279 sys_cpu = int(soc_cpu["cores_per_socket"])
281 cores = "%s-%s" % (soc_cpu[socket][0], soc_cpu[socket][sys_cpu - 1])
282 if int(soc_cpu["thread_per_core"]) > 1:
283 threads = "%s-%s" % (soc_cpu[socket][sys_cpu], soc_cpu[socket][-1])
284 cpuset = "%s,%s" % (cores, threads)
288 def write_file(cls, file_name, xml_str):
289 """Dump a XML string to a file"""
290 root = ET.fromstring(xml_str)
291 et = ET.ElementTree(element=root)
292 et.write(file_name, encoding='utf-8', method='xml')
295 class StandaloneContextHelper(object):
296 """ This class handles all the common code for standalone
299 self.file_path = None
300 super(StandaloneContextHelper, self).__init__()
303 def install_req_libs(connection, extra_pkgs=None):
304 extra_pkgs = extra_pkgs or []
305 pkgs = ["qemu-kvm", "libvirt-bin", "bridge-utils", "numactl", "fping"]
306 pkgs.extend(extra_pkgs)
307 cmd_template = "dpkg-query -W --showformat='${Status}\\n' \"%s\"|grep 'ok installed'"
309 if connection.execute(cmd_template % pkg)[0]:
310 connection.execute("apt-get update")
311 connection.execute("apt-get -y install %s" % pkg)
314 def get_kernel_module(connection, pci, driver):
316 out = connection.execute("lspci -k -s %s" % pci)[1]
317 driver = out.split("Kernel modules:").pop().strip()
321 def get_nic_details(cls, connection, networks, dpdk_devbind):
322 for key, ports in networks.items():
326 phy_ports = ports['phy_port']
327 phy_driver = ports.get('phy_driver', None)
328 driver = cls.get_kernel_module(connection, phy_ports, phy_driver)
330 # Make sure that ports are bound to kernel drivers e.g. i40e/ixgbe
331 bind_cmd = "{dpdk_devbind} --force -b {driver} {port}"
332 lshw_cmd = "lshw -c network -businfo | grep '{port}'"
333 link_show_cmd = "ip -s link show {interface}"
335 cmd = bind_cmd.format(dpdk_devbind=dpdk_devbind,
336 driver=driver, port=ports['phy_port'])
337 connection.execute(cmd)
339 out = connection.execute(lshw_cmd.format(port=phy_ports))[1]
340 interface = out.split()[1]
342 connection.execute(link_show_cmd.format(interface=interface))
345 'interface': str(interface),
353 def get_virtual_devices(connection, pci):
354 cmd = "cat /sys/bus/pci/devices/{0}/virtfn0/uevent"
355 output = connection.execute(cmd.format(pci))[1]
357 pattern = "PCI_SLOT_NAME=({})".format(PciAddress.PCI_PATTERN_STR)
358 m = re.search(pattern, output, re.MULTILINE)
362 pf_vfs = {pci: m.group(1).rstrip()}
364 LOG.info("pf_vfs:\n%s", pf_vfs)
368 def read_config_file(self):
369 """Read from config file"""
371 with open(self.file_path) as stream:
372 LOG.info("Parsing pod file: %s", self.file_path)
373 cfg = yaml_load(stream)
376 def parse_pod_file(self, file_path, nfvi_role='Sriov'):
377 self.file_path = file_path
381 cfg = self.read_config_file()
382 except IOError as io_error:
383 if io_error.errno != errno.ENOENT:
385 self.file_path = os.path.join(constants.YARDSTICK_ROOT_PATH,
387 cfg = self.read_config_file()
389 nodes.extend([node for node in cfg["nodes"] if str(node["role"]) != nfvi_role])
390 nfvi_host.extend([node for node in cfg["nodes"] if str(node["role"]) == nfvi_role])
392 raise("Node role is other than SRIOV")
394 host_mgmt = {'user': nfvi_host[0]['user'],
395 'ip': str(IPNetwork(nfvi_host[0]['ip']).ip),
396 'password': nfvi_host[0]['password'],
397 'ssh_port': nfvi_host[0].get('ssh_port', 22),
398 'key_filename': nfvi_host[0].get('key_filename')}
400 return [nodes, nfvi_host, host_mgmt]
403 def get_mac_address(end=0x7f):
404 mac = [0x52, 0x54, 0x00,
405 random.randint(0x00, end),
406 random.randint(0x00, 0xff),
407 random.randint(0x00, 0xff)]
408 mac_address = ':'.join('%02x' % x for x in mac)
412 def get_mgmt_ip(connection, mac, cidr, node):
415 while not mgmtip and times:
416 connection.execute("fping -c 1 -g %s > /dev/null 2>&1" % cidr)
417 out = connection.execute("ip neighbor | grep '%s'" % mac)[1]
418 LOG.info("fping -c 1 -g %s > /dev/null 2>&1", cidr)
420 mgmtip = str(out.split(" ")[0]).strip()
421 client = ssh.SSH.from_node(node, overrides={"ip": mgmtip})
425 time.sleep(WAIT_FOR_BOOT) # FixMe: How to find if VM is booted?
430 def wait_for_vnfs_to_start(cls, connection, servers, nodes):
432 vnf = servers[node["name"]]
433 mgmtip = vnf["network_ports"]["mgmt"]["cidr"]
434 ip = cls.get_mgmt_ip(connection, node["mac"], mgmtip, node)
440 class Server(object):
441 """ This class handles geting vnf nodes
445 def build_vnf_interfaces(vnf, ports):
449 for key, vfs in vnf["network_ports"].items():
451 mgmtip = str(IPNetwork(vfs['cidr']).ip)
455 ip = IPNetwork(vf['cidr'])
459 'driver': "%svf" % vf['driver'],
460 'local_mac': vf['mac'],
461 'dpdk_port_num': index,
462 'local_ip': str(ip.ip),
463 'netmask': str(ip.netmask)
468 return mgmtip, interfaces
471 def generate_vnf_instance(cls, flavor, ports, ip, key, vnf, mac):
472 mgmtip, interfaces = cls.build_vnf_interfaces(vnf, ports)
478 "user": flavor.get('user', 'root'),
479 "interfaces": interfaces,
481 # empty IPv6 routing table
483 "name": key, "role": key
487 result['key_filename'] = flavor['key_filename']
492 result['password'] = flavor['password']
499 class OvsDeploy(object):
500 """ This class handles deploy of ovs dpdk
501 Configuration: ovs_dpdk
504 OVS_DEPLOY_SCRIPT = "ovs_deploy.bash"
506 def __init__(self, connection, bin_path, ovs_properties):
507 self.connection = connection
508 self.bin_path = bin_path
509 self.ovs_properties = ovs_properties
511 def prerequisite(self):
512 pkgs = ["git", "build-essential", "pkg-config", "automake",
513 "autotools-dev", "libltdl-dev", "cmake", "libnuma-dev",
515 StandaloneContextHelper.install_req_libs(self.connection, pkgs)
517 def ovs_deploy(self):
518 ovs_deploy = os.path.join(constants.YARDSTICK_ROOT_PATH,
519 "yardstick/resources/scripts/install/",
520 self.OVS_DEPLOY_SCRIPT)
521 if os.path.isfile(ovs_deploy):
523 remote_ovs_deploy = os.path.join(self.bin_path, self.OVS_DEPLOY_SCRIPT)
524 LOG.info(remote_ovs_deploy)
525 self.connection.put(ovs_deploy, remote_ovs_deploy)
527 http_proxy = os.environ.get('http_proxy', '')
528 ovs_details = self.ovs_properties.get("version", {})
529 ovs = ovs_details.get("ovs", "2.6.0")
530 dpdk = ovs_details.get("dpdk", "16.11.1")
532 cmd = "sudo -E %s --ovs='%s' --dpdk='%s' -p='%s'" % (remote_ovs_deploy,
533 ovs, dpdk, http_proxy)
534 exit_status, _, stderr = self.connection.execute(cmd)
536 raise exceptions.OVSDeployError(stderr=stderr)