1 # Copyright (c) 2016-2017 Intel Corporation
3 # Licensed under the Apache License, Version 2.0 (the "License");
4 # you may not use this file except in compliance with the License.
5 # You may obtain a copy of the License at
7 # http://www.apache.org/licenses/LICENSE-2.0
9 # Unless required by applicable law or agreed to in writing, software
10 # distributed under the License is distributed on an "AS IS" BASIS,
11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 # See the License for the specific language governing permissions and
13 # limitations under the License.
23 from netaddr import IPNetwork
24 import xml.etree.ElementTree as ET
26 from yardstick import ssh
27 from yardstick.common import constants
28 from yardstick.common import exceptions
29 from yardstick.common import yaml_loader
30 from yardstick.network_services.utils import PciAddress
31 from yardstick.network_services.helpers.cpu import CpuSysCores
34 LOG = logging.getLogger(__name__)
38 <name>{vm_name}</name>
39 <uuid>{random_uuid}</uuid>
40 <memory unit="MB">{memory}</memory>
41 <currentMemory unit="MB">{memory}</currentMemory>
45 <vcpu cpuset='{cpuset}'>{vcpu}</vcpu>
48 <type arch="x86_64" machine="pc-i440fx-xenial">hvm</type>
56 <cpu mode='host-passthrough'>
57 <topology cores="{cpu}" sockets="{socket}" threads="{threads}" />
59 <cell id='0' cpus='{numa_cpus}' memory='{memory}' unit='MB' memAccess='shared'/>
63 <timer name="rtc" tickpolicy="catchup" />
64 <timer name="pit" tickpolicy="delay" />
65 <timer name="hpet" present="no" />
67 <on_poweroff>destroy</on_poweroff>
68 <on_reboot>restart</on_reboot>
69 <on_crash>restart</on_crash>
71 <emulator>/usr/bin/kvm-spice</emulator>
72 <disk device="disk" type="file">
73 <driver name="qemu" type="qcow2" />
74 <source file="{vm_image}"/>
75 <target bus="virtio" dev="vda" />
77 <graphics autoport="yes" listen="0.0.0.0" port="-1" type="vnc" />
78 <interface type="bridge">
79 <mac address='{mac_addr}'/>
80 <source bridge="br-int" />
81 <model type='virtio'/>
87 <target type='serial' port='0'/>
93 USER_DATA_TEMPLATE = """
94 cat > {user_file} <<EOF
96 preserve_hostname: false
103 NETWORK_DATA_TEMPLATE = """
104 cat > {network_file} <<EOF
110 mac_address: {mac_address}
119 class Libvirt(object):
120 """ This class handles all the libvirt updates to lauch VM
124 def check_if_vm_exists_and_delete(vm_name, connection):
125 cmd_template = "virsh list --name | grep -i %s"
126 status = connection.execute(cmd_template % vm_name)[0]
128 LOG.info("VM '%s' is already present... destroying", vm_name)
129 connection.execute("virsh destroy %s" % vm_name)
132 def virsh_create_vm(connection, cfg):
133 LOG.info('VM create, XML config: %s', cfg)
134 status, _, error = connection.execute('virsh create %s' % cfg)
136 raise exceptions.LibvirtCreateError(error=error)
139 def virsh_destroy_vm(vm_name, connection):
140 LOG.info('VM destroy, VM name: %s', vm_name)
141 status, _, error = connection.execute('virsh destroy %s' % vm_name)
143 LOG.warning('Error destroying VM %s. Error: %s', vm_name, error)
146 def _add_interface_address(interface, pci_address):
147 """Add a PCI 'address' XML node
149 <address type='pci' domain='0x0000' bus='0x00' slot='0x08'
152 Refence: https://software.intel.com/en-us/articles/
153 configure-sr-iov-network-virtual-functions-in-linux-kvm
155 vm_pci = ET.SubElement(interface, 'address')
156 vm_pci.set('type', 'pci')
157 vm_pci.set('domain', '0x{}'.format(pci_address.domain))
158 vm_pci.set('bus', '0x{}'.format(pci_address.bus))
159 vm_pci.set('slot', '0x{}'.format(pci_address.slot))
160 vm_pci.set('function', '0x{}'.format(pci_address.function))
164 def add_ovs_interface(cls, vpath, port_num, vpci, vports_mac, xml_str):
165 """Add a DPDK OVS 'interface' XML node in 'devices' node
168 <interface type='vhostuser'>
169 <mac address='00:00:00:00:00:01'/>
170 <source type='unix' path='/usr/local/var/run/openvswitch/
171 dpdkvhostuser0' mode='client'/>
172 <model type='virtio'/>
174 <host mrg_rxbuf='off'/>
176 <address type='pci' domain='0x0000' bus='0x00' slot='0x03'
182 Reference: http://docs.openvswitch.org/en/latest/topics/dpdk/
186 vhost_path = ('{0}/var/run/openvswitch/dpdkvhostuser{1}'.
187 format(vpath, port_num))
188 root = ET.fromstring(xml_str)
189 pci_address = PciAddress(vpci.strip())
190 device = root.find('devices')
192 interface = ET.SubElement(device, 'interface')
193 interface.set('type', 'vhostuser')
194 mac = ET.SubElement(interface, 'mac')
195 mac.set('address', vports_mac)
197 source = ET.SubElement(interface, 'source')
198 source.set('type', 'unix')
199 source.set('path', vhost_path)
200 source.set('mode', 'client')
202 model = ET.SubElement(interface, 'model')
203 model.set('type', 'virtio')
205 driver = ET.SubElement(interface, 'driver')
206 driver.set('queues', '4')
208 host = ET.SubElement(driver, 'host')
209 host.set('mrg_rxbuf', 'off')
211 cls._add_interface_address(interface, pci_address)
213 return ET.tostring(root)
216 def add_sriov_interfaces(cls, vm_pci, vf_pci, vf_mac, xml_str):
217 """Add a SR-IOV 'interface' XML node in 'devices' node
220 <interface type='hostdev' managed='yes'>
222 <address type='pci' domain='0x0000' bus='0x00' slot='0x03'
225 <mac address='52:54:00:6d:90:02'>
226 <address type='pci' domain='0x0000' bus='0x02' slot='0x04'
232 Reference: https://access.redhat.com/documentation/en-us/
233 red_hat_enterprise_linux/6/html/
234 virtualization_host_configuration_and_guest_installation_guide/
235 sect-virtualization_host_configuration_and_guest_installation_guide
236 -sr_iov-how_sr_iov_libvirt_works
239 root = ET.fromstring(xml_str)
240 device = root.find('devices')
242 interface = ET.SubElement(device, 'interface')
243 interface.set('managed', 'yes')
244 interface.set('type', 'hostdev')
246 mac = ET.SubElement(interface, 'mac')
247 mac.set('address', vf_mac)
249 source = ET.SubElement(interface, 'source')
250 pci_address = PciAddress(vf_pci.strip())
251 cls._add_interface_address(source, pci_address)
253 pci_vm_address = PciAddress(vm_pci.strip())
254 cls._add_interface_address(interface, pci_vm_address)
256 return ET.tostring(root)
259 def create_snapshot_qemu(connection, index, base_image):
260 """Create the snapshot image for a VM using a base image
262 :param connection: SSH connection to the remote host
263 :param index: index of the VM to be spawn
264 :param base_image: path of the VM base image in the remote host
265 :return: snapshot image path
267 vm_image = '/var/lib/libvirt/images/%s.qcow2' % index
268 connection.execute('rm -- "%s"' % vm_image)
269 status, _, _ = connection.execute('test -r %s' % base_image)
271 if not os.access(base_image, os.R_OK):
272 raise exceptions.LibvirtQemuImageBaseImageNotPresent(
273 vm_image=vm_image, base_image=base_image)
274 # NOTE(ralonsoh): done in two steps to avoid root permission
276 LOG.info('Copy %s from execution host to remote host', base_image)
277 file_name = os.path.basename(os.path.normpath(base_image))
278 connection.put_file(base_image, '/tmp/%s' % file_name)
279 status, _, error = connection.execute(
280 'mv -- "/tmp/%s" "%s"' % (file_name, base_image))
282 raise exceptions.LibvirtQemuImageCreateError(
283 vm_image=vm_image, base_image=base_image, error=error)
285 LOG.info('Convert image %s to %s', base_image, vm_image)
286 qemu_cmd = ('qemu-img create -f qcow2 -o backing_file=%s %s' %
287 (base_image, vm_image))
288 status, _, error = connection.execute(qemu_cmd)
290 raise exceptions.LibvirtQemuImageCreateError(
291 vm_image=vm_image, base_image=base_image, error=error)
295 def build_vm_xml(cls, connection, flavor, vm_name, index, cdrom_img):
296 """Build the XML from the configuration parameters"""
297 memory = flavor.get('ram', '4096')
298 extra_spec = flavor.get('extra_specs', {})
299 cpu = extra_spec.get('hw:cpu_cores', '2')
300 socket = extra_spec.get('hw:cpu_sockets', '1')
301 threads = extra_spec.get('hw:cpu_threads', '2')
302 vcpu = int(cpu) * int(threads)
303 numa_cpus = '0-%s' % (vcpu - 1)
304 hw_socket = flavor.get('hw_socket', '0')
305 cpuset = Libvirt.pin_vcpu_for_perf(connection, hw_socket)
307 cputune = extra_spec.get('cputune', '')
308 mac = StandaloneContextHelper.get_mac_address(0x00)
309 image = cls.create_snapshot_qemu(connection, index,
310 flavor.get("images", None))
311 vm_xml = VM_TEMPLATE.format(
313 random_uuid=uuid.uuid4(),
315 memory=memory, vcpu=vcpu, cpu=cpu,
317 socket=socket, threads=threads,
318 vm_image=image, cpuset=cpuset, cputune=cputune)
321 vm_xml = Libvirt.add_cdrom(cdrom_img, vm_xml)
326 def update_interrupts_hugepages_perf(connection):
327 connection.execute("echo 1 > /sys/module/kvm/parameters/allow_unsafe_assigned_interrupts")
328 connection.execute("echo never > /sys/kernel/mm/transparent_hugepage/enabled")
331 def pin_vcpu_for_perf(cls, connection, socket='0'):
333 sys_obj = CpuSysCores(connection)
334 soc_cpu = sys_obj.get_core_socket()
335 sys_cpu = int(soc_cpu["cores_per_socket"])
337 cores = "%s-%s" % (soc_cpu[socket][0], soc_cpu[socket][sys_cpu - 1])
338 if int(soc_cpu["thread_per_core"]) > 1:
339 threads = "%s-%s" % (soc_cpu[socket][sys_cpu], soc_cpu[socket][-1])
340 cpuset = "%s,%s" % (cores, threads)
344 def write_file(cls, file_name, xml_str):
345 """Dump a XML string to a file"""
346 root = ET.fromstring(xml_str)
347 et = ET.ElementTree(element=root)
348 et.write(file_name, encoding='utf-8', method='xml')
351 def add_cdrom(cls, file_path, xml_str):
352 """Add a CD-ROM disk XML node in 'devices' node
355 <disk type='file' device='cdrom'>
356 <driver name='qemu' type='raw'/>
357 <source file='/var/lib/libvirt/images/data.img'/>
365 root = ET.fromstring(xml_str)
366 device = root.find('devices')
368 disk = ET.SubElement(device, 'disk')
369 disk.set('type', 'file')
370 disk.set('device', 'cdrom')
372 driver = ET.SubElement(disk, 'driver')
373 driver.set('name', 'qemu')
374 driver.set('type', 'raw')
376 source = ET.SubElement(disk, 'source')
377 source.set('file', file_path)
379 target = ET.SubElement(disk, 'target')
380 target.set('dev', 'hdb')
382 ET.SubElement(disk, 'readonly')
383 return ET.tostring(root)
386 def gen_cdrom_image(connection, file_path, vm_name, vm_user, key_filename, mac, ip):
387 """Generate ISO image for CD-ROM """
389 user_config = [" - name: {user_name}",
390 " ssh_authorized_keys:",
392 if vm_user != "root":
393 user_config.append(" sudo: ALL=(ALL) NOPASSWD:ALL")
395 meta_data = "/tmp/meta-data"
396 user_data = "/tmp/user-data"
397 network_data = "/tmp/network-config"
398 with open(".".join([key_filename, "pub"]), "r") as pub_key_file:
399 pub_key_str = pub_key_file.read().rstrip()
400 user_conf = os.linesep.join(user_config).format(pub_key_str=pub_key_str, user_name=vm_user)
403 "touch %s" % meta_data,
404 USER_DATA_TEMPLATE.format(user_file=user_data, host=vm_name, user_config=user_conf),
405 NETWORK_DATA_TEMPLATE.format(network_file=network_data, mac_address=mac,
407 "genisoimage -output {0} -volid cidata -joliet -r {1} {2} {3}".format(file_path,
411 "rm {0} {1} {2}".format(meta_data, user_data, network_data),
415 status, _, error = connection.execute(cmd)
417 raise exceptions.LibvirtQemuImageCreateError(error=error)
420 class StandaloneContextHelper(object):
421 """ This class handles all the common code for standalone
424 self.file_path = None
425 super(StandaloneContextHelper, self).__init__()
428 def install_req_libs(connection, extra_pkgs=None):
429 extra_pkgs = extra_pkgs or []
430 pkgs = ["qemu-kvm", "libvirt-bin", "bridge-utils", "numactl", "fping", "genisoimage"]
431 pkgs.extend(extra_pkgs)
432 cmd_template = "dpkg-query -W --showformat='${Status}\\n' \"%s\"|grep 'ok installed'"
434 if connection.execute(cmd_template % pkg)[0]:
435 connection.execute("apt-get update")
436 connection.execute("apt-get -y install %s" % pkg)
439 def get_kernel_module(connection, pci, driver):
441 out = connection.execute("lspci -k -s %s" % pci)[1]
442 driver = out.split("Kernel modules:").pop().strip()
446 def get_nic_details(cls, connection, networks, dpdk_devbind):
447 for key, ports in networks.items():
451 phy_ports = ports['phy_port']
452 phy_driver = ports.get('phy_driver', None)
453 driver = cls.get_kernel_module(connection, phy_ports, phy_driver)
455 # Make sure that ports are bound to kernel drivers e.g. i40e/ixgbe
456 bind_cmd = "{dpdk_devbind} --force -b {driver} {port}"
457 lshw_cmd = "lshw -c network -businfo | grep '{port}'"
458 link_show_cmd = "ip -s link show {interface}"
460 cmd = bind_cmd.format(dpdk_devbind=dpdk_devbind,
461 driver=driver, port=ports['phy_port'])
462 connection.execute(cmd)
464 out = connection.execute(lshw_cmd.format(port=phy_ports))[1]
465 interface = out.split()[1]
467 connection.execute(link_show_cmd.format(interface=interface))
470 'interface': str(interface),
478 def get_virtual_devices(connection, pci):
479 cmd = "cat /sys/bus/pci/devices/{0}/virtfn0/uevent"
480 output = connection.execute(cmd.format(pci))[1]
482 pattern = "PCI_SLOT_NAME=({})".format(PciAddress.PCI_PATTERN_STR)
483 m = re.search(pattern, output, re.MULTILINE)
487 pf_vfs = {pci: m.group(1).rstrip()}
489 LOG.info("pf_vfs:\n%s", pf_vfs)
493 def parse_pod_file(self, file_path, nfvi_role='Sriov'):
494 self.file_path = file_path
498 cfg = yaml_loader.read_yaml_file(self.file_path)
499 except IOError as io_error:
500 if io_error.errno != errno.ENOENT:
502 self.file_path = os.path.join(constants.YARDSTICK_ROOT_PATH,
504 cfg = yaml_loader.read_yaml_file(self.file_path)
506 nodes.extend([node for node in cfg["nodes"] if str(node["role"]) != nfvi_role])
507 nfvi_host.extend([node for node in cfg["nodes"] if str(node["role"]) == nfvi_role])
509 raise("Node role is other than SRIOV")
511 host_mgmt = {'user': nfvi_host[0]['user'],
512 'ip': str(IPNetwork(nfvi_host[0]['ip']).ip),
513 'password': nfvi_host[0]['password'],
514 'ssh_port': nfvi_host[0].get('ssh_port', 22),
515 'key_filename': nfvi_host[0].get('key_filename')}
517 return [nodes, nfvi_host, host_mgmt]
520 def get_mac_address(end=0x7f):
521 mac = [0x52, 0x54, 0x00,
522 random.randint(0x00, end),
523 random.randint(0x00, 0xff),
524 random.randint(0x00, 0xff)]
525 mac_address = ':'.join('%02x' % x for x in mac)
529 def get_mgmt_ip(connection, mac, cidr, node):
532 while not mgmtip and times:
533 connection.execute("fping -c 1 -g %s > /dev/null 2>&1" % cidr)
534 out = connection.execute("ip neighbor | grep '%s'" % mac)[1]
535 LOG.info("fping -c 1 -g %s > /dev/null 2>&1", cidr)
537 mgmtip = str(out.split(" ")[0]).strip()
538 client = ssh.SSH.from_node(node, overrides={"ip": mgmtip})
542 time.sleep(WAIT_FOR_BOOT) # FixMe: How to find if VM is booted?
547 def wait_for_vnfs_to_start(cls, connection, servers, nodes):
549 vnf = servers[node["name"]]
550 mgmtip = vnf["network_ports"]["mgmt"]["cidr"]
551 ip = cls.get_mgmt_ip(connection, node["mac"], mgmtip, node)
557 def check_update_key(cls, connection, node, vm_name, id_name, cdrom_img, mac):
558 # Generate public/private keys if private key file is not provided
559 user_name = node.get('user')
561 node['user'] = 'root'
562 user_name = node.get('user')
563 if not node.get('key_filename'):
564 key_filename = ''.join(
565 [constants.YARDSTICK_ROOT_PATH,
566 'yardstick/resources/files/yardstick_key-',
567 id_name, '-', vm_name])
568 ssh.SSH.gen_keys(key_filename)
569 node['key_filename'] = key_filename
570 # Update image with public key
571 key_filename = node.get('key_filename')
572 ip_netmask = "{0}/{1}".format(node.get('ip'), node.get('netmask'))
573 Libvirt.gen_cdrom_image(connection, cdrom_img, vm_name, user_name, key_filename, mac,
578 class Server(object):
579 """ This class handles geting vnf nodes
583 def build_vnf_interfaces(vnf, ports):
587 for key, vfs in vnf["network_ports"].items():
589 mgmt_cidr = IPNetwork(vfs['cidr'])
593 ip = IPNetwork(vf['cidr'])
597 'driver': "%svf" % vf['driver'],
598 'local_mac': vf['mac'],
599 'dpdk_port_num': index,
600 'local_ip': str(ip.ip),
601 'netmask': str(ip.netmask)
606 return mgmt_cidr, interfaces
609 def generate_vnf_instance(cls, flavor, ports, ip, key, vnf, mac):
610 mgmt_cidr, interfaces = cls.build_vnf_interfaces(vnf, ports)
613 "ip": str(mgmt_cidr.ip),
614 "netmask": str(mgmt_cidr.netmask),
617 "user": flavor.get('user', 'root'),
618 "interfaces": interfaces,
620 # empty IPv6 routing table
622 "name": key, "role": key
626 result['key_filename'] = flavor['key_filename']
631 result['password'] = flavor['password']
638 class OvsDeploy(object):
639 """ This class handles deploy of ovs dpdk
640 Configuration: ovs_dpdk
643 OVS_DEPLOY_SCRIPT = "ovs_deploy.bash"
645 def __init__(self, connection, bin_path, ovs_properties):
646 self.connection = connection
647 self.bin_path = bin_path
648 self.ovs_properties = ovs_properties
650 def prerequisite(self):
651 pkgs = ["git", "build-essential", "pkg-config", "automake",
652 "autotools-dev", "libltdl-dev", "cmake", "libnuma-dev",
654 StandaloneContextHelper.install_req_libs(self.connection, pkgs)
656 def ovs_deploy(self):
657 ovs_deploy = os.path.join(constants.YARDSTICK_ROOT_PATH,
658 "yardstick/resources/scripts/install/",
659 self.OVS_DEPLOY_SCRIPT)
660 if os.path.isfile(ovs_deploy):
662 remote_ovs_deploy = os.path.join(self.bin_path, self.OVS_DEPLOY_SCRIPT)
663 LOG.info(remote_ovs_deploy)
664 self.connection.put(ovs_deploy, remote_ovs_deploy)
666 http_proxy = os.environ.get('http_proxy', '')
667 ovs_details = self.ovs_properties.get("version", {})
668 ovs = ovs_details.get("ovs", "2.6.0")
669 dpdk = ovs_details.get("dpdk", "16.11.1")
671 cmd = "sudo -E %s --ovs='%s' --dpdk='%s' -p='%s'" % (remote_ovs_deploy,
672 ovs, dpdk, http_proxy)
673 exit_status, _, stderr = self.connection.execute(cmd)
675 raise exceptions.OVSDeployError(stderr=stderr)