1 # Copyright (c) 2016-2017 Intel Corporation
3 # Licensed under the Apache License, Version 2.0 (the "License");
4 # you may not use this file except in compliance with the License.
5 # You may obtain a copy of the License at
7 # http://www.apache.org/licenses/LICENSE-2.0
9 # Unless required by applicable law or agreed to in writing, software
10 # distributed under the License is distributed on an "AS IS" BASIS,
11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 # See the License for the specific language governing permissions and
13 # limitations under the License.
23 from netaddr import IPNetwork
24 import xml.etree.ElementTree as ET
26 from yardstick import ssh
27 from yardstick.common import constants
28 from yardstick.common import exceptions
29 from yardstick.common import yaml_loader
30 from yardstick.network_services.utils import PciAddress
31 from yardstick.network_services.helpers.cpu import CpuSysCores
34 LOG = logging.getLogger(__name__)
38 <name>{vm_name}</name>
39 <uuid>{random_uuid}</uuid>
40 <memory unit="MB">{memory}</memory>
41 <currentMemory unit="MB">{memory}</currentMemory>
45 <vcpu cpuset='{cpuset}'>{vcpu}</vcpu>
48 <type arch="x86_64" machine="{machine}">hvm</type>
56 <cpu mode='host-passthrough'>
57 <topology cores="{cpu}" sockets="{socket}" threads="{threads}" />
59 <cell id='0' cpus='{numa_cpus}' memory='{memory}' unit='MB' memAccess='shared'/>
63 <timer name="rtc" tickpolicy="catchup" />
64 <timer name="pit" tickpolicy="delay" />
65 <timer name="hpet" present="no" />
67 <on_poweroff>destroy</on_poweroff>
68 <on_reboot>restart</on_reboot>
69 <on_crash>restart</on_crash>
71 <emulator>/usr/bin/kvm-spice</emulator>
72 <disk device="disk" type="file">
73 <driver name="qemu" type="qcow2" />
74 <source file="{vm_image}"/>
75 <target bus="virtio" dev="vda" />
77 <graphics autoport="yes" listen="0.0.0.0" port="-1" type="vnc" />
78 <interface type="bridge">
79 <mac address='{mac_addr}'/>
80 <source bridge="br-int" />
81 <model type='virtio'/>
87 <target type='serial' port='0'/>
93 USER_DATA_TEMPLATE = """
94 cat > {user_file} <<EOF
96 preserve_hostname: false
103 NETWORK_DATA_TEMPLATE = """
104 cat > {network_file} <<EOF
110 macaddress: {mac_address}
119 class Libvirt(object):
120 """ This class handles all the libvirt updates to lauch VM
124 def check_if_vm_exists_and_delete(vm_name, connection):
125 cmd_template = "virsh list --name | grep -i %s"
126 status = connection.execute(cmd_template % vm_name)[0]
128 LOG.info("VM '%s' is already present... destroying", vm_name)
129 connection.execute("virsh destroy %s" % vm_name)
132 def virsh_create_vm(connection, cfg):
133 LOG.info('VM create, XML config: %s', cfg)
134 status, _, error = connection.execute('virsh create %s' % cfg)
136 raise exceptions.LibvirtCreateError(error=error)
139 def virsh_destroy_vm(vm_name, connection):
140 LOG.info('VM destroy, VM name: %s', vm_name)
141 status, _, error = connection.execute('virsh destroy %s' % vm_name)
143 LOG.warning('Error destroying VM %s. Error: %s', vm_name, error)
146 def _add_interface_address(interface, pci_address):
147 """Add a PCI 'address' XML node
149 <address type='pci' domain='0x0000' bus='0x00' slot='0x08'
152 Refence: https://software.intel.com/en-us/articles/
153 configure-sr-iov-network-virtual-functions-in-linux-kvm
155 vm_pci = ET.SubElement(interface, 'address')
156 vm_pci.set('type', 'pci')
157 vm_pci.set('domain', '0x{}'.format(pci_address.domain))
158 vm_pci.set('bus', '0x{}'.format(pci_address.bus))
159 vm_pci.set('slot', '0x{}'.format(pci_address.slot))
160 vm_pci.set('function', '0x{}'.format(pci_address.function))
164 def add_ovs_interface(cls, vpath, port_num, vpci, vports_mac, xml_str,
166 """Add a DPDK OVS 'interface' XML node in 'devices' node
169 <interface type='vhostuser'>
170 <mac address='00:00:00:00:00:01'/>
171 <source type='unix' path='/usr/local/var/run/openvswitch/
172 dpdkvhostuser0' mode='client'/>
173 <model type='virtio'/>
175 <host mrg_rxbuf='off'/>
177 <address type='pci' domain='0x0000' bus='0x00' slot='0x03'
183 Reference: http://docs.openvswitch.org/en/latest/topics/dpdk/
187 vhost_path = ('{0}/var/run/openvswitch/dpdkvhostuser{1}'.
188 format(vpath, port_num))
189 root = ET.fromstring(xml_str)
190 pci_address = PciAddress(vpci.strip())
191 device = root.find('devices')
193 interface = ET.SubElement(device, 'interface')
194 interface.set('type', 'vhostuser')
195 mac = ET.SubElement(interface, 'mac')
196 mac.set('address', vports_mac)
198 source = ET.SubElement(interface, 'source')
199 source.set('type', 'unix')
200 source.set('path', vhost_path)
201 source.set('mode', 'client')
203 model = ET.SubElement(interface, 'model')
204 model.set('type', 'virtio')
206 driver = ET.SubElement(interface, 'driver')
207 driver.set('queues', str(queues))
209 host = ET.SubElement(driver, 'host')
210 host.set('mrg_rxbuf', 'off')
212 cls._add_interface_address(interface, pci_address)
214 return ET.tostring(root)
217 def add_sriov_interfaces(cls, vm_pci, vf_pci, vf_mac, xml_str):
218 """Add a SR-IOV 'interface' XML node in 'devices' node
221 <interface type='hostdev' managed='yes'>
223 <address type='pci' domain='0x0000' bus='0x00' slot='0x03'
226 <mac address='52:54:00:6d:90:02'>
227 <address type='pci' domain='0x0000' bus='0x02' slot='0x04'
233 Reference: https://access.redhat.com/documentation/en-us/
234 red_hat_enterprise_linux/6/html/
235 virtualization_host_configuration_and_guest_installation_guide/
236 sect-virtualization_host_configuration_and_guest_installation_guide
237 -sr_iov-how_sr_iov_libvirt_works
240 root = ET.fromstring(xml_str)
241 device = root.find('devices')
243 interface = ET.SubElement(device, 'interface')
244 interface.set('managed', 'yes')
245 interface.set('type', 'hostdev')
247 mac = ET.SubElement(interface, 'mac')
248 mac.set('address', vf_mac)
250 source = ET.SubElement(interface, 'source')
251 pci_address = PciAddress(vf_pci.strip())
252 cls._add_interface_address(source, pci_address)
254 pci_vm_address = PciAddress(vm_pci.strip())
255 cls._add_interface_address(interface, pci_vm_address)
257 return ET.tostring(root)
260 def create_snapshot_qemu(connection, index, base_image):
261 """Create the snapshot image for a VM using a base image
263 :param connection: SSH connection to the remote host
264 :param index: index of the VM to be spawn
265 :param base_image: path of the VM base image in the remote host
266 :return: snapshot image path
268 vm_image = '/var/lib/libvirt/images/%s.qcow2' % index
269 connection.execute('rm -- "%s"' % vm_image)
270 status, _, _ = connection.execute('test -r %s' % base_image)
272 if not os.access(base_image, os.R_OK):
273 raise exceptions.LibvirtQemuImageBaseImageNotPresent(
274 vm_image=vm_image, base_image=base_image)
275 # NOTE(ralonsoh): done in two steps to avoid root permission
277 LOG.info('Copy %s from execution host to remote host', base_image)
278 file_name = os.path.basename(os.path.normpath(base_image))
279 connection.put_file(base_image, '/tmp/%s' % file_name)
280 status, _, error = connection.execute(
281 'mv -- "/tmp/%s" "%s"' % (file_name, base_image))
283 raise exceptions.LibvirtQemuImageCreateError(
284 vm_image=vm_image, base_image=base_image, error=error)
286 LOG.info('Convert image %s to %s', base_image, vm_image)
287 qemu_cmd = ('qemu-img create -f qcow2 -o backing_file=%s %s' %
288 (base_image, vm_image))
289 status, _, error = connection.execute(qemu_cmd)
291 raise exceptions.LibvirtQemuImageCreateError(
292 vm_image=vm_image, base_image=base_image, error=error)
296 def build_vm_xml(cls, connection, flavor, vm_name, index, cdrom_img):
297 """Build the XML from the configuration parameters"""
298 memory = flavor.get('ram', '4096')
299 extra_spec = flavor.get('extra_specs', {})
300 cpu = extra_spec.get('hw:cpu_cores', '2')
301 socket = extra_spec.get('hw:cpu_sockets', '1')
302 threads = extra_spec.get('hw:cpu_threads', '2')
303 vcpu = int(cpu) * int(threads)
304 numa_cpus = '0-%s' % (vcpu - 1)
305 hw_socket = flavor.get('hw_socket', '0')
306 cpuset = Libvirt.pin_vcpu_for_perf(connection, hw_socket)
308 cputune = extra_spec.get('cputune', '')
309 machine = extra_spec.get('machine_type', 'pc-i440fx-xenial')
310 mac = StandaloneContextHelper.get_mac_address(0x00)
311 image = cls.create_snapshot_qemu(connection, index,
312 flavor.get("images", None))
313 vm_xml = VM_TEMPLATE.format(
315 random_uuid=uuid.uuid4(),
317 memory=memory, vcpu=vcpu, cpu=cpu,
319 socket=socket, threads=threads,
320 vm_image=image, cpuset=cpuset,
321 machine=machine, cputune=cputune)
324 vm_xml = Libvirt.add_cdrom(cdrom_img, vm_xml)
329 def update_interrupts_hugepages_perf(connection):
330 connection.execute("echo 1 > /sys/module/kvm/parameters/allow_unsafe_assigned_interrupts")
331 connection.execute("echo never > /sys/kernel/mm/transparent_hugepage/enabled")
334 def pin_vcpu_for_perf(cls, connection, socket='0'):
336 sys_obj = CpuSysCores(connection)
337 soc_cpu = sys_obj.get_core_socket()
338 sys_cpu = int(soc_cpu["cores_per_socket"])
340 cores = "%s-%s" % (soc_cpu[socket][0], soc_cpu[socket][sys_cpu - 1])
341 if int(soc_cpu["thread_per_core"]) > 1:
342 threads = "%s-%s" % (soc_cpu[socket][sys_cpu], soc_cpu[socket][-1])
343 cpuset = "%s,%s" % (cores, threads)
347 def write_file(cls, file_name, xml_str):
348 """Dump a XML string to a file"""
349 root = ET.fromstring(xml_str)
350 et = ET.ElementTree(element=root)
351 et.write(file_name, encoding='utf-8', method='xml')
354 def add_cdrom(cls, file_path, xml_str):
355 """Add a CD-ROM disk XML node in 'devices' node
358 <disk type='file' device='cdrom'>
359 <driver name='qemu' type='raw'/>
360 <source file='/var/lib/libvirt/images/data.img'/>
368 root = ET.fromstring(xml_str)
369 device = root.find('devices')
371 disk = ET.SubElement(device, 'disk')
372 disk.set('type', 'file')
373 disk.set('device', 'cdrom')
375 driver = ET.SubElement(disk, 'driver')
376 driver.set('name', 'qemu')
377 driver.set('type', 'raw')
379 source = ET.SubElement(disk, 'source')
380 source.set('file', file_path)
382 target = ET.SubElement(disk, 'target')
383 target.set('dev', 'hdb')
385 ET.SubElement(disk, 'readonly')
386 return ET.tostring(root)
389 def gen_cdrom_image(connection, file_path, vm_name, vm_user, key_filename, mac, ip):
390 """Generate ISO image for CD-ROM """
392 user_config = [" - name: {user_name}",
393 " ssh_authorized_keys:",
395 if vm_user != "root":
396 user_config.append(" sudo: ALL=(ALL) NOPASSWD:ALL")
398 meta_data = "/tmp/meta-data"
399 user_data = "/tmp/user-data"
400 network_data = "/tmp/network-config"
401 with open(".".join([key_filename, "pub"]), "r") as pub_key_file:
402 pub_key_str = pub_key_file.read().rstrip()
403 user_conf = os.linesep.join(user_config).format(pub_key_str=pub_key_str, user_name=vm_user)
406 "touch %s" % meta_data,
407 USER_DATA_TEMPLATE.format(user_file=user_data, host=vm_name, user_config=user_conf),
408 NETWORK_DATA_TEMPLATE.format(network_file=network_data, mac_address=mac,
410 "genisoimage -output {0} -volid cidata -joliet -r {1} {2} {3}".format(file_path,
414 "rm {0} {1} {2}".format(meta_data, user_data, network_data),
418 status, _, error = connection.execute(cmd)
420 raise exceptions.LibvirtQemuImageCreateError(error=error)
423 class StandaloneContextHelper(object):
424 """ This class handles all the common code for standalone
427 self.file_path = None
428 super(StandaloneContextHelper, self).__init__()
431 def install_req_libs(connection, extra_pkgs=None):
432 extra_pkgs = extra_pkgs or []
433 pkgs = ["qemu-kvm", "libvirt-bin", "bridge-utils", "numactl", "fping", "genisoimage"]
434 pkgs.extend(extra_pkgs)
435 cmd_template = "dpkg-query -W --showformat='${Status}\\n' \"%s\"|grep 'ok installed'"
437 if connection.execute(cmd_template % pkg)[0]:
438 connection.execute("apt-get update")
439 connection.execute("apt-get -y install %s" % pkg)
442 def get_kernel_module(connection, pci, driver):
444 out = connection.execute("lspci -k -s %s" % pci)[1]
445 driver = out.split("Kernel modules:").pop().strip()
449 def get_nic_details(cls, connection, networks, dpdk_devbind):
450 for key, ports in networks.items():
454 phy_ports = ports['phy_port']
455 phy_driver = ports.get('phy_driver', None)
456 driver = cls.get_kernel_module(connection, phy_ports, phy_driver)
458 # Make sure that ports are bound to kernel drivers e.g. i40e/ixgbe
459 bind_cmd = "{dpdk_devbind} --force -b {driver} {port}"
460 lshw_cmd = "lshw -c network -businfo | grep '{port}'"
461 link_show_cmd = "ip -s link show {interface}"
463 cmd = bind_cmd.format(dpdk_devbind=dpdk_devbind,
464 driver=driver, port=ports['phy_port'])
465 connection.execute(cmd)
467 out = connection.execute(lshw_cmd.format(port=phy_ports))[1]
468 interface = out.split()[1]
470 connection.execute(link_show_cmd.format(interface=interface))
473 'interface': str(interface),
481 def get_virtual_devices(connection, pci):
482 cmd = "cat /sys/bus/pci/devices/{0}/virtfn0/uevent"
483 output = connection.execute(cmd.format(pci))[1]
485 pattern = "PCI_SLOT_NAME=({})".format(PciAddress.PCI_PATTERN_STR)
486 m = re.search(pattern, output, re.MULTILINE)
490 pf_vfs = {pci: m.group(1).rstrip()}
492 LOG.info("pf_vfs:\n%s", pf_vfs)
496 def parse_pod_file(self, file_path, nfvi_role='Sriov'):
497 self.file_path = file_path
501 cfg = yaml_loader.read_yaml_file(self.file_path)
502 except IOError as io_error:
503 if io_error.errno != errno.ENOENT:
505 self.file_path = os.path.join(constants.YARDSTICK_ROOT_PATH,
507 cfg = yaml_loader.read_yaml_file(self.file_path)
509 nodes.extend([node for node in cfg["nodes"] if str(node["role"]) != nfvi_role])
510 nfvi_host.extend([node for node in cfg["nodes"] if str(node["role"]) == nfvi_role])
512 raise("Node role is other than SRIOV")
514 host_mgmt = {'user': nfvi_host[0]['user'],
515 'ip': str(IPNetwork(nfvi_host[0]['ip']).ip),
516 'password': nfvi_host[0]['password'],
517 'ssh_port': nfvi_host[0].get('ssh_port', 22),
518 'key_filename': nfvi_host[0].get('key_filename')}
520 return [nodes, nfvi_host, host_mgmt]
523 def get_mac_address(end=0x7f):
524 mac = [0x52, 0x54, 0x00,
525 random.randint(0x00, end),
526 random.randint(0x00, 0xff),
527 random.randint(0x00, 0xff)]
528 mac_address = ':'.join('%02x' % x for x in mac)
532 def get_mgmt_ip(connection, mac, cidr, node):
535 while not mgmtip and times:
536 connection.execute("fping -c 1 -g %s > /dev/null 2>&1" % cidr)
537 out = connection.execute("ip neighbor | grep '%s'" % mac)[1]
538 LOG.info("fping -c 1 -g %s > /dev/null 2>&1", cidr)
540 mgmtip = str(out.split(" ")[0]).strip()
541 client = ssh.SSH.from_node(node, overrides={"ip": mgmtip})
545 time.sleep(WAIT_FOR_BOOT) # FixMe: How to find if VM is booted?
550 def wait_for_vnfs_to_start(cls, connection, servers, nodes):
552 vnf = servers[node["name"]]
553 mgmtip = vnf["network_ports"]["mgmt"]["cidr"]
554 ip = cls.get_mgmt_ip(connection, node["mac"], mgmtip, node)
560 def check_update_key(cls, connection, node, vm_name, id_name, cdrom_img, mac):
561 # Generate public/private keys if private key file is not provided
562 user_name = node.get('user')
564 node['user'] = 'root'
565 user_name = node.get('user')
566 if not node.get('key_filename'):
567 key_filename = ''.join(
568 [constants.YARDSTICK_ROOT_PATH,
569 'yardstick/resources/files/yardstick_key-',
570 id_name, '-', vm_name])
571 ssh.SSH.gen_keys(key_filename)
572 node['key_filename'] = key_filename
573 # Update image with public key
574 key_filename = node.get('key_filename')
575 ip_netmask = "{0}/{1}".format(node.get('ip'), node.get('netmask'))
576 ip_netmask = "{0}/{1}".format(node.get('ip'),
577 IPNetwork(ip_netmask).prefixlen)
578 Libvirt.gen_cdrom_image(connection, cdrom_img, vm_name, user_name, key_filename, mac,
583 class Server(object):
584 """ This class handles geting vnf nodes
588 def build_vnf_interfaces(vnf, ports):
592 for key, vfs in vnf["network_ports"].items():
594 mgmt_cidr = IPNetwork(vfs['cidr'])
598 ip = IPNetwork(vf['cidr'])
602 'driver': "%svf" % vf['driver'],
603 'local_mac': vf['mac'],
604 'dpdk_port_num': index,
605 'local_ip': str(ip.ip),
606 'netmask': str(ip.netmask)
611 return mgmt_cidr, interfaces
614 def generate_vnf_instance(cls, flavor, ports, ip, key, vnf, mac):
615 mgmt_cidr, interfaces = cls.build_vnf_interfaces(vnf, ports)
618 "ip": str(mgmt_cidr.ip),
619 "netmask": str(mgmt_cidr.netmask),
622 "user": flavor.get('user', 'root'),
623 "interfaces": interfaces,
625 # empty IPv6 routing table
627 "name": key, "role": key
631 result['key_filename'] = flavor['key_filename']
636 result['password'] = flavor['password']
643 class OvsDeploy(object):
644 """ This class handles deploy of ovs dpdk
645 Configuration: ovs_dpdk
648 OVS_DEPLOY_SCRIPT = "ovs_deploy.bash"
650 def __init__(self, connection, bin_path, ovs_properties):
651 self.connection = connection
652 self.bin_path = bin_path
653 self.ovs_properties = ovs_properties
655 def prerequisite(self):
656 pkgs = ["git", "build-essential", "pkg-config", "automake",
657 "autotools-dev", "libltdl-dev", "cmake", "libnuma-dev",
659 StandaloneContextHelper.install_req_libs(self.connection, pkgs)
661 def ovs_deploy(self):
662 ovs_deploy = os.path.join(constants.YARDSTICK_ROOT_PATH,
663 "yardstick/resources/scripts/install/",
664 self.OVS_DEPLOY_SCRIPT)
665 if os.path.isfile(ovs_deploy):
667 remote_ovs_deploy = os.path.join(self.bin_path, self.OVS_DEPLOY_SCRIPT)
668 LOG.info(remote_ovs_deploy)
669 self.connection.put(ovs_deploy, remote_ovs_deploy)
671 http_proxy = os.environ.get('http_proxy', '')
672 ovs_details = self.ovs_properties.get("version", {})
673 ovs = ovs_details.get("ovs", "2.6.0")
674 dpdk = ovs_details.get("dpdk", "16.11.1")
676 cmd = "sudo -E %s --ovs='%s' --dpdk='%s' -p='%s'" % (remote_ovs_deploy,
677 ovs, dpdk, http_proxy)
678 exit_status, _, stderr = self.connection.execute(cmd)
680 raise exceptions.OVSDeployError(stderr=stderr)