1 # Copyright (c) 2016-2017 Intel Corporation
3 # Licensed under the Apache License, Version 2.0 (the "License");
4 # you may not use this file except in compliance with the License.
5 # You may obtain a copy of the License at
7 # http://www.apache.org/licenses/LICENSE-2.0
9 # Unless required by applicable law or agreed to in writing, software
10 # distributed under the License is distributed on an "AS IS" BASIS,
11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 # See the License for the specific language governing permissions and
13 # limitations under the License.
23 from netaddr import IPNetwork
24 import xml.etree.ElementTree as ET
26 from yardstick import ssh
27 from yardstick.common import constants
28 from yardstick.common import exceptions
29 from yardstick.common import utils as common_utils
30 from yardstick.common import yaml_loader
31 from yardstick.network_services.utils import PciAddress
32 from yardstick.network_services.helpers.cpu import CpuSysCores
35 LOG = logging.getLogger(__name__)
39 <name>{vm_name}</name>
40 <uuid>{random_uuid}</uuid>
41 <memory unit="MB">{memory}</memory>
42 <currentMemory unit="MB">{memory}</currentMemory>
46 <vcpu cpuset='{cpuset}'>{vcpu}</vcpu>
49 <type arch="x86_64" machine="{machine}">hvm</type>
57 <cpu mode='host-passthrough'>
58 <topology cores="{cpu}" sockets="{socket}" threads="{threads}" />
60 <cell id='0' cpus='{numa_cpus}' memory='{memory}' unit='MB' memAccess='shared'/>
64 <timer name="rtc" tickpolicy="catchup" />
65 <timer name="pit" tickpolicy="delay" />
66 <timer name="hpet" present="no" />
68 <on_poweroff>destroy</on_poweroff>
69 <on_reboot>restart</on_reboot>
70 <on_crash>restart</on_crash>
72 <emulator>/usr/bin/kvm-spice</emulator>
73 <disk device="disk" type="file">
74 <driver name="qemu" type="qcow2" />
75 <source file="{vm_image}"/>
76 <target bus="virtio" dev="vda" />
78 <graphics autoport="yes" listen="0.0.0.0" port="-1" type="vnc" />
79 <interface type="bridge">
80 <mac address='{mac_addr}'/>
81 <source bridge="br-int" />
82 <model type='virtio'/>
88 <target type='serial' port='0'/>
94 USER_DATA_TEMPLATE = """
95 cat > {user_file} <<EOF
97 preserve_hostname: false
104 NETWORK_DATA_TEMPLATE = """
105 cat > {network_file} <<EOF
111 macaddress: {mac_address}
120 class Libvirt(object):
121 """ This class handles all the libvirt updates to lauch VM
125 def check_if_vm_exists_and_delete(vm_name, connection):
126 cmd_template = "virsh list --name | grep -i %s"
127 status = connection.execute(cmd_template % vm_name)[0]
129 LOG.info("VM '%s' is already present... destroying", vm_name)
130 connection.execute("virsh destroy %s" % vm_name)
133 def virsh_create_vm(connection, cfg):
134 LOG.info('VM create, XML config: %s', cfg)
135 status, _, error = connection.execute('virsh create %s' % cfg)
137 raise exceptions.LibvirtCreateError(error=error)
140 def virsh_destroy_vm(vm_name, connection):
141 LOG.info('VM destroy, VM name: %s', vm_name)
142 status, _, error = connection.execute('virsh destroy %s' % vm_name)
144 LOG.warning('Error destroying VM %s. Error: %s', vm_name, error)
147 def _add_interface_address(interface, pci_address):
148 """Add a PCI 'address' XML node
150 <address type='pci' domain='0x0000' bus='0x00' slot='0x08'
153 Refence: https://software.intel.com/en-us/articles/
154 configure-sr-iov-network-virtual-functions-in-linux-kvm
156 vm_pci = ET.SubElement(interface, 'address')
157 vm_pci.set('type', 'pci')
158 vm_pci.set('domain', '0x{}'.format(pci_address.domain))
159 vm_pci.set('bus', '0x{}'.format(pci_address.bus))
160 vm_pci.set('slot', '0x{}'.format(pci_address.slot))
161 vm_pci.set('function', '0x{}'.format(pci_address.function))
165 def add_ovs_interface(cls, vpath, port_num, vpci, vports_mac, xml_str,
167 """Add a DPDK OVS 'interface' XML node in 'devices' node
170 <interface type='vhostuser'>
171 <mac address='00:00:00:00:00:01'/>
172 <source type='unix' path='/usr/local/var/run/openvswitch/
173 dpdkvhostuser0' mode='client'/>
174 <model type='virtio'/>
176 <host mrg_rxbuf='off'/>
178 <address type='pci' domain='0x0000' bus='0x00' slot='0x03'
184 Reference: http://docs.openvswitch.org/en/latest/topics/dpdk/
188 vhost_path = ('{0}/var/run/openvswitch/dpdkvhostuser{1}'.
189 format(vpath, port_num))
190 root = ET.fromstring(xml_str)
191 pci_address = PciAddress(vpci.strip())
192 device = root.find('devices')
194 interface = ET.SubElement(device, 'interface')
195 interface.set('type', 'vhostuser')
196 mac = ET.SubElement(interface, 'mac')
197 mac.set('address', vports_mac)
199 source = ET.SubElement(interface, 'source')
200 source.set('type', 'unix')
201 source.set('path', vhost_path)
202 source.set('mode', 'client')
204 model = ET.SubElement(interface, 'model')
205 model.set('type', 'virtio')
207 driver = ET.SubElement(interface, 'driver')
208 driver.set('queues', str(queues))
210 host = ET.SubElement(driver, 'host')
211 host.set('mrg_rxbuf', 'off')
213 cls._add_interface_address(interface, pci_address)
215 return ET.tostring(root)
218 def add_sriov_interfaces(cls, vm_pci, vf_pci, vf_mac, xml_str):
219 """Add a SR-IOV 'interface' XML node in 'devices' node
222 <interface type='hostdev' managed='yes'>
224 <address type='pci' domain='0x0000' bus='0x00' slot='0x03'
227 <mac address='52:54:00:6d:90:02'>
228 <address type='pci' domain='0x0000' bus='0x02' slot='0x04'
234 Reference: https://access.redhat.com/documentation/en-us/
235 red_hat_enterprise_linux/6/html/
236 virtualization_host_configuration_and_guest_installation_guide/
237 sect-virtualization_host_configuration_and_guest_installation_guide
238 -sr_iov-how_sr_iov_libvirt_works
241 root = ET.fromstring(xml_str)
242 device = root.find('devices')
244 interface = ET.SubElement(device, 'interface')
245 interface.set('managed', 'yes')
246 interface.set('type', 'hostdev')
248 mac = ET.SubElement(interface, 'mac')
249 mac.set('address', vf_mac)
251 source = ET.SubElement(interface, 'source')
252 pci_address = PciAddress(vf_pci.strip())
253 cls._add_interface_address(source, pci_address)
255 pci_vm_address = PciAddress(vm_pci.strip())
256 cls._add_interface_address(interface, pci_vm_address)
258 return ET.tostring(root)
261 def create_snapshot_qemu(connection, index, base_image):
262 """Create the snapshot image for a VM using a base image
264 :param connection: SSH connection to the remote host
265 :param index: index of the VM to be spawn
266 :param base_image: path of the VM base image in the remote host
267 :return: snapshot image path
269 vm_image = '/var/lib/libvirt/images/%s.qcow2' % index
270 connection.execute('rm -- "%s"' % vm_image)
271 status, _, _ = connection.execute('test -r %s' % base_image)
273 if not os.access(base_image, os.R_OK):
274 raise exceptions.LibvirtQemuImageBaseImageNotPresent(
275 vm_image=vm_image, base_image=base_image)
276 # NOTE(ralonsoh): done in two steps to avoid root permission
278 LOG.info('Copy %s from execution host to remote host', base_image)
279 file_name = os.path.basename(os.path.normpath(base_image))
280 connection.put_file(base_image, '/tmp/%s' % file_name)
281 status, _, error = connection.execute(
282 'mv -- "/tmp/%s" "%s"' % (file_name, base_image))
284 raise exceptions.LibvirtQemuImageCreateError(
285 vm_image=vm_image, base_image=base_image, error=error)
287 LOG.info('Convert image %s to %s', base_image, vm_image)
288 qemu_cmd = ('qemu-img create -f qcow2 -o backing_file=%s %s' %
289 (base_image, vm_image))
290 status, _, error = connection.execute(qemu_cmd)
292 raise exceptions.LibvirtQemuImageCreateError(
293 vm_image=vm_image, base_image=base_image, error=error)
297 def build_vm_xml(cls, connection, flavor, vm_name, index, cdrom_img):
298 """Build the XML from the configuration parameters"""
299 memory = flavor.get('ram', '4096')
300 extra_spec = flavor.get('extra_specs', {})
301 cpu = extra_spec.get('hw:cpu_cores', '2')
302 socket = extra_spec.get('hw:cpu_sockets', '1')
303 threads = extra_spec.get('hw:cpu_threads', '2')
304 vcpu = int(cpu) * int(threads)
305 numa_cpus = '0-%s' % (vcpu - 1)
306 hw_socket = flavor.get('hw_socket', '0')
307 cpuset = Libvirt.pin_vcpu_for_perf(connection, hw_socket)
309 cputune = extra_spec.get('cputune', '')
310 machine = extra_spec.get('machine_type', 'pc-i440fx-xenial')
311 mac = StandaloneContextHelper.get_mac_address(0x00)
312 image = cls.create_snapshot_qemu(connection, index,
313 flavor.get("images", None))
314 vm_xml = VM_TEMPLATE.format(
316 random_uuid=uuid.uuid4(),
318 memory=memory, vcpu=vcpu, cpu=cpu,
320 socket=socket, threads=threads,
321 vm_image=image, cpuset=cpuset,
322 machine=machine, cputune=cputune)
325 vm_xml = Libvirt.add_cdrom(cdrom_img, vm_xml)
330 def update_interrupts_hugepages_perf(connection):
331 connection.execute("echo 1 > /sys/module/kvm/parameters/allow_unsafe_assigned_interrupts")
332 connection.execute("echo never > /sys/kernel/mm/transparent_hugepage/enabled")
335 def pin_vcpu_for_perf(cls, connection, socket='0'):
337 sys_obj = CpuSysCores(connection)
338 soc_cpu = sys_obj.get_core_socket()
339 sys_cpu = int(soc_cpu["cores_per_socket"])
341 cores = "%s-%s" % (soc_cpu[socket][0], soc_cpu[socket][sys_cpu - 1])
342 if int(soc_cpu["thread_per_core"]) > 1:
343 threads = "%s-%s" % (soc_cpu[socket][sys_cpu], soc_cpu[socket][-1])
344 cpuset = "%s,%s" % (cores, threads)
348 def write_file(cls, file_name, xml_str):
349 """Dump a XML string to a file"""
350 root = ET.fromstring(xml_str)
351 et = ET.ElementTree(element=root)
352 et.write(file_name, encoding='utf-8', method='xml')
355 def add_cdrom(cls, file_path, xml_str):
356 """Add a CD-ROM disk XML node in 'devices' node
359 <disk type='file' device='cdrom'>
360 <driver name='qemu' type='raw'/>
361 <source file='/var/lib/libvirt/images/data.img'/>
369 root = ET.fromstring(xml_str)
370 device = root.find('devices')
372 disk = ET.SubElement(device, 'disk')
373 disk.set('type', 'file')
374 disk.set('device', 'cdrom')
376 driver = ET.SubElement(disk, 'driver')
377 driver.set('name', 'qemu')
378 driver.set('type', 'raw')
380 source = ET.SubElement(disk, 'source')
381 source.set('file', file_path)
383 target = ET.SubElement(disk, 'target')
384 target.set('dev', 'hdb')
386 ET.SubElement(disk, 'readonly')
387 return ET.tostring(root)
390 def gen_cdrom_image(connection, file_path, vm_name, vm_user, key_filename, mac, ip):
391 """Generate ISO image for CD-ROM """
393 user_config = [" - name: {user_name}",
394 " ssh_authorized_keys:",
396 if vm_user != "root":
397 user_config.append(" sudo: ALL=(ALL) NOPASSWD:ALL")
399 meta_data = "/tmp/meta-data"
400 user_data = "/tmp/user-data"
401 network_data = "/tmp/network-config"
402 with open(".".join([key_filename, "pub"]), "r") as pub_key_file:
403 pub_key_str = pub_key_file.read().rstrip()
404 user_conf = os.linesep.join(user_config).format(pub_key_str=pub_key_str, user_name=vm_user)
407 "touch %s" % meta_data,
408 USER_DATA_TEMPLATE.format(user_file=user_data, host=vm_name, user_config=user_conf),
409 NETWORK_DATA_TEMPLATE.format(network_file=network_data, mac_address=mac,
411 "genisoimage -output {0} -volid cidata -joliet -r {1} {2} {3}".format(file_path,
415 "rm {0} {1} {2}".format(meta_data, user_data, network_data),
419 status, _, error = connection.execute(cmd)
421 raise exceptions.LibvirtQemuImageCreateError(error=error)
424 class StandaloneContextHelper(object):
425 """ This class handles all the common code for standalone
428 self.file_path = None
429 super(StandaloneContextHelper, self).__init__()
432 def install_req_libs(connection, extra_pkgs=None):
433 extra_pkgs = extra_pkgs or []
434 pkgs = ["qemu-kvm", "libvirt-bin", "bridge-utils", "numactl", "fping", "genisoimage"]
435 pkgs.extend(extra_pkgs)
436 cmd_template = "dpkg-query -W --showformat='${Status}\\n' \"%s\"|grep 'ok installed'"
438 if connection.execute(cmd_template % pkg)[0]:
439 connection.execute("apt-get update")
440 connection.execute("apt-get -y install %s" % pkg)
443 def get_kernel_module(connection, pci, driver):
445 out = connection.execute("lspci -k -s %s" % pci)[1]
446 driver = out.split("Kernel modules:").pop().strip()
450 def get_nic_details(cls, connection, networks, dpdk_devbind):
451 for key, ports in networks.items():
455 phy_ports = ports['phy_port']
456 phy_driver = ports.get('phy_driver', None)
457 driver = cls.get_kernel_module(connection, phy_ports, phy_driver)
459 # Make sure that ports are bound to kernel drivers e.g. i40e/ixgbe
460 bind_cmd = "{dpdk_devbind} --force -b {driver} {port}"
461 lshw_cmd = "lshw -c network -businfo | grep '{port}'"
462 link_show_cmd = "ip -s link show {interface}"
464 cmd = bind_cmd.format(dpdk_devbind=dpdk_devbind,
465 driver=driver, port=ports['phy_port'])
466 connection.execute(cmd)
468 out = connection.execute(lshw_cmd.format(port=phy_ports))[1]
469 interface = out.split()[1]
471 connection.execute(link_show_cmd.format(interface=interface))
474 'interface': str(interface),
482 def get_virtual_devices(connection, pci):
483 cmd = "cat /sys/bus/pci/devices/{0}/virtfn0/uevent"
484 output = connection.execute(cmd.format(pci))[1]
486 pattern = "PCI_SLOT_NAME=({})".format(PciAddress.PCI_PATTERN_STR)
487 m = re.search(pattern, output, re.MULTILINE)
491 pf_vfs = {pci: m.group(1).rstrip()}
493 LOG.info("pf_vfs:\n%s", pf_vfs)
497 def parse_pod_file(self, file_path, nfvi_role='Sriov'):
498 self.file_path = file_path
502 cfg = yaml_loader.read_yaml_file(self.file_path)
503 except IOError as io_error:
504 if io_error.errno != errno.ENOENT:
506 self.file_path = os.path.join(constants.YARDSTICK_ROOT_PATH,
508 cfg = yaml_loader.read_yaml_file(self.file_path)
510 nodes.extend([node for node in cfg["nodes"] if str(node["role"]) != nfvi_role])
511 nfvi_host.extend([node for node in cfg["nodes"] if str(node["role"]) == nfvi_role])
513 raise("Node role is other than SRIOV")
515 host_mgmt = {'user': nfvi_host[0]['user'],
516 'ip': str(IPNetwork(nfvi_host[0]['ip']).ip),
517 'password': nfvi_host[0]['password'],
518 'ssh_port': nfvi_host[0].get('ssh_port', 22),
519 'key_filename': nfvi_host[0].get('key_filename')}
521 return [nodes, nfvi_host, host_mgmt]
524 def get_mac_address(end=0x7f):
525 mac = [0x52, 0x54, 0x00,
526 random.randint(0x00, end),
527 random.randint(0x00, 0xff),
528 random.randint(0x00, 0xff)]
529 mac_address = ':'.join('%02x' % x for x in mac)
533 def get_mgmt_ip(connection, mac, cidr, node):
536 while not mgmtip and times:
537 connection.execute("fping -c 1 -g %s > /dev/null 2>&1" % cidr)
538 out = connection.execute("ip neighbor | grep '%s'" % mac)[1]
539 LOG.info("fping -c 1 -g %s > /dev/null 2>&1", cidr)
541 mgmtip = str(out.split(" ")[0]).strip()
542 client = ssh.SSH.from_node(node, overrides={"ip": mgmtip})
546 time.sleep(WAIT_FOR_BOOT) # FixMe: How to find if VM is booted?
551 def wait_for_vnfs_to_start(cls, connection, servers, nodes):
553 vnf = servers[node["name"]]
554 mgmtip = vnf["network_ports"]["mgmt"]["cidr"]
555 ip = cls.get_mgmt_ip(connection, node["mac"], mgmtip, node)
558 client = ssh.SSH.from_node(node)
559 LOG.debug("OS version: %s",
560 common_utils.get_os_version(client))
561 LOG.debug("Kernel version: %s",
562 common_utils.get_kernel_version(client))
563 vnfs_data = common_utils.get_sample_vnf_info(client)
564 for vnf_name, vnf_data in vnfs_data.items():
565 LOG.debug("VNF name: '%s', commit ID/branch: '%s'",
566 vnf_name, vnf_data["branch_commit"])
567 LOG.debug("%s", vnf_data["md5_result"])
571 def check_update_key(cls, connection, node, vm_name, id_name, cdrom_img, mac):
572 # Generate public/private keys if private key file is not provided
573 user_name = node.get('user')
575 node['user'] = 'root'
576 user_name = node.get('user')
577 if not node.get('key_filename'):
578 key_filename = ''.join(
579 [constants.YARDSTICK_ROOT_PATH,
580 'yardstick/resources/files/yardstick_key-',
581 id_name, '-', vm_name])
582 ssh.SSH.gen_keys(key_filename)
583 node['key_filename'] = key_filename
584 # Update image with public key
585 key_filename = node.get('key_filename')
586 ip_netmask = "{0}/{1}".format(node.get('ip'), node.get('netmask'))
587 ip_netmask = "{0}/{1}".format(node.get('ip'),
588 IPNetwork(ip_netmask).prefixlen)
589 Libvirt.gen_cdrom_image(connection, cdrom_img, vm_name, user_name, key_filename, mac,
594 class Server(object):
595 """ This class handles geting vnf nodes
599 def build_vnf_interfaces(vnf, ports):
603 for key, vfs in vnf["network_ports"].items():
605 mgmt_cidr = IPNetwork(vfs['cidr'])
609 ip = IPNetwork(vf['cidr'])
613 'driver': "%svf" % vf['driver'],
614 'local_mac': vf['mac'],
615 'dpdk_port_num': index,
616 'local_ip': str(ip.ip),
617 'netmask': str(ip.netmask)
622 return mgmt_cidr, interfaces
625 def generate_vnf_instance(cls, flavor, ports, ip, key, vnf, mac):
626 mgmt_cidr, interfaces = cls.build_vnf_interfaces(vnf, ports)
629 "ip": str(mgmt_cidr.ip),
630 "netmask": str(mgmt_cidr.netmask),
633 "user": flavor.get('user', 'root'),
634 "interfaces": interfaces,
636 # empty IPv6 routing table
638 "name": key, "role": key
642 result['key_filename'] = flavor['key_filename']
647 result['password'] = flavor['password']
654 class OvsDeploy(object):
655 """ This class handles deploy of ovs dpdk
656 Configuration: ovs_dpdk
659 OVS_DEPLOY_SCRIPT = "ovs_deploy.bash"
661 def __init__(self, connection, bin_path, ovs_properties):
662 self.connection = connection
663 self.bin_path = bin_path
664 self.ovs_properties = ovs_properties
666 def prerequisite(self):
667 pkgs = ["git", "build-essential", "pkg-config", "automake",
668 "autotools-dev", "libltdl-dev", "cmake", "libnuma-dev",
670 StandaloneContextHelper.install_req_libs(self.connection, pkgs)
672 def ovs_deploy(self):
673 ovs_deploy = os.path.join(constants.YARDSTICK_ROOT_PATH,
674 "yardstick/resources/scripts/install/",
675 self.OVS_DEPLOY_SCRIPT)
676 if os.path.isfile(ovs_deploy):
678 remote_ovs_deploy = os.path.join(self.bin_path, self.OVS_DEPLOY_SCRIPT)
679 LOG.info(remote_ovs_deploy)
680 self.connection.put(ovs_deploy, remote_ovs_deploy)
682 http_proxy = os.environ.get('http_proxy', '')
683 ovs_details = self.ovs_properties.get("version", {})
684 ovs = ovs_details.get("ovs", "2.6.0")
685 dpdk = ovs_details.get("dpdk", "16.11.1")
687 cmd = "sudo -E %s --ovs='%s' --dpdk='%s' -p='%s'" % (remote_ovs_deploy,
688 ovs, dpdk, http_proxy)
689 exit_status, _, stderr = self.connection.execute(cmd)
691 raise exceptions.OVSDeployError(stderr=stderr)