1 # Copyright (c) 2016-2017 Intel Corporation
3 # Licensed under the Apache License, Version 2.0 (the "License");
4 # you may not use this file except in compliance with the License.
5 # You may obtain a copy of the License at
7 # http://www.apache.org/licenses/LICENSE-2.0
9 # Unless required by applicable law or agreed to in writing, software
10 # distributed under the License is distributed on an "AS IS" BASIS,
11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 # See the License for the specific language governing permissions and
13 # limitations under the License.
23 from netaddr import IPNetwork
24 import xml.etree.ElementTree as ET
26 from yardstick import ssh
27 from yardstick.common import constants
28 from yardstick.common import exceptions
29 from yardstick.common import yaml_loader
30 from yardstick.network_services.utils import PciAddress
31 from yardstick.network_services.helpers.cpu import CpuSysCores
34 LOG = logging.getLogger(__name__)
38 <name>{vm_name}</name>
39 <uuid>{random_uuid}</uuid>
40 <memory unit="MB">{memory}</memory>
41 <currentMemory unit="MB">{memory}</currentMemory>
45 <vcpu cpuset='{cpuset}'>{vcpu}</vcpu>
48 <type arch="x86_64" machine="pc-i440fx-xenial">hvm</type>
56 <cpu mode='host-passthrough'>
57 <topology cores="{cpu}" sockets="{socket}" threads="{threads}" />
59 <cell id='0' cpus='{numa_cpus}' memory='{memory}' unit='MB' memAccess='shared'/>
63 <timer name="rtc" tickpolicy="catchup" />
64 <timer name="pit" tickpolicy="delay" />
65 <timer name="hpet" present="no" />
67 <on_poweroff>destroy</on_poweroff>
68 <on_reboot>restart</on_reboot>
69 <on_crash>restart</on_crash>
71 <emulator>/usr/bin/kvm-spice</emulator>
72 <disk device="disk" type="file">
73 <driver name="qemu" type="qcow2" />
74 <source file="{vm_image}"/>
75 <target bus="virtio" dev="vda" />
77 <graphics autoport="yes" listen="0.0.0.0" port="-1" type="vnc" />
78 <interface type="bridge">
79 <mac address='{mac_addr}'/>
80 <source bridge="br-int" />
81 <model type='virtio'/>
87 <target type='serial' port='0'/>
93 USER_DATA_TEMPLATE = """
94 cat > {user_file} <<EOF
96 preserve_hostname: false
106 class Libvirt(object):
107 """ This class handles all the libvirt updates to lauch VM
111 def check_if_vm_exists_and_delete(vm_name, connection):
112 cmd_template = "virsh list --name | grep -i %s"
113 status = connection.execute(cmd_template % vm_name)[0]
115 LOG.info("VM '%s' is already present... destroying", vm_name)
116 connection.execute("virsh destroy %s" % vm_name)
119 def virsh_create_vm(connection, cfg):
120 LOG.info('VM create, XML config: %s', cfg)
121 status, _, error = connection.execute('virsh create %s' % cfg)
123 raise exceptions.LibvirtCreateError(error=error)
126 def virsh_destroy_vm(vm_name, connection):
127 LOG.info('VM destroy, VM name: %s', vm_name)
128 status, _, error = connection.execute('virsh destroy %s' % vm_name)
130 LOG.warning('Error destroying VM %s. Error: %s', vm_name, error)
133 def _add_interface_address(interface, pci_address):
134 """Add a PCI 'address' XML node
136 <address type='pci' domain='0x0000' bus='0x00' slot='0x08'
139 Refence: https://software.intel.com/en-us/articles/
140 configure-sr-iov-network-virtual-functions-in-linux-kvm
142 vm_pci = ET.SubElement(interface, 'address')
143 vm_pci.set('type', 'pci')
144 vm_pci.set('domain', '0x{}'.format(pci_address.domain))
145 vm_pci.set('bus', '0x{}'.format(pci_address.bus))
146 vm_pci.set('slot', '0x{}'.format(pci_address.slot))
147 vm_pci.set('function', '0x{}'.format(pci_address.function))
151 def add_ovs_interface(cls, vpath, port_num, vpci, vports_mac, xml_str):
152 """Add a DPDK OVS 'interface' XML node in 'devices' node
155 <interface type='vhostuser'>
156 <mac address='00:00:00:00:00:01'/>
157 <source type='unix' path='/usr/local/var/run/openvswitch/
158 dpdkvhostuser0' mode='client'/>
159 <model type='virtio'/>
161 <host mrg_rxbuf='off'/>
163 <address type='pci' domain='0x0000' bus='0x00' slot='0x03'
169 Reference: http://docs.openvswitch.org/en/latest/topics/dpdk/
173 vhost_path = ('{0}/var/run/openvswitch/dpdkvhostuser{1}'.
174 format(vpath, port_num))
175 root = ET.fromstring(xml_str)
176 pci_address = PciAddress(vpci.strip())
177 device = root.find('devices')
179 interface = ET.SubElement(device, 'interface')
180 interface.set('type', 'vhostuser')
181 mac = ET.SubElement(interface, 'mac')
182 mac.set('address', vports_mac)
184 source = ET.SubElement(interface, 'source')
185 source.set('type', 'unix')
186 source.set('path', vhost_path)
187 source.set('mode', 'client')
189 model = ET.SubElement(interface, 'model')
190 model.set('type', 'virtio')
192 driver = ET.SubElement(interface, 'driver')
193 driver.set('queues', '4')
195 host = ET.SubElement(driver, 'host')
196 host.set('mrg_rxbuf', 'off')
198 cls._add_interface_address(interface, pci_address)
200 return ET.tostring(root)
203 def add_sriov_interfaces(cls, vm_pci, vf_pci, vf_mac, xml_str):
204 """Add a SR-IOV 'interface' XML node in 'devices' node
207 <interface type='hostdev' managed='yes'>
209 <address type='pci' domain='0x0000' bus='0x00' slot='0x03'
212 <mac address='52:54:00:6d:90:02'>
213 <address type='pci' domain='0x0000' bus='0x02' slot='0x04'
219 Reference: https://access.redhat.com/documentation/en-us/
220 red_hat_enterprise_linux/6/html/
221 virtualization_host_configuration_and_guest_installation_guide/
222 sect-virtualization_host_configuration_and_guest_installation_guide
223 -sr_iov-how_sr_iov_libvirt_works
226 root = ET.fromstring(xml_str)
227 device = root.find('devices')
229 interface = ET.SubElement(device, 'interface')
230 interface.set('managed', 'yes')
231 interface.set('type', 'hostdev')
233 mac = ET.SubElement(interface, 'mac')
234 mac.set('address', vf_mac)
236 source = ET.SubElement(interface, 'source')
237 pci_address = PciAddress(vf_pci.strip())
238 cls._add_interface_address(source, pci_address)
240 pci_vm_address = PciAddress(vm_pci.strip())
241 cls._add_interface_address(interface, pci_vm_address)
243 return ET.tostring(root)
246 def create_snapshot_qemu(connection, index, base_image):
247 """Create the snapshot image for a VM using a base image
249 :param connection: SSH connection to the remote host
250 :param index: index of the VM to be spawn
251 :param base_image: path of the VM base image in the remote host
252 :return: snapshot image path
254 vm_image = '/var/lib/libvirt/images/%s.qcow2' % index
255 connection.execute('rm -- "%s"' % vm_image)
256 status, _, _ = connection.execute('test -r %s' % base_image)
258 if not os.access(base_image, os.R_OK):
259 raise exceptions.LibvirtQemuImageBaseImageNotPresent(
260 vm_image=vm_image, base_image=base_image)
261 # NOTE(ralonsoh): done in two steps to avoid root permission
263 LOG.info('Copy %s from execution host to remote host', base_image)
264 file_name = os.path.basename(os.path.normpath(base_image))
265 connection.put_file(base_image, '/tmp/%s' % file_name)
266 status, _, error = connection.execute(
267 'mv -- "/tmp/%s" "%s"' % (file_name, base_image))
269 raise exceptions.LibvirtQemuImageCreateError(
270 vm_image=vm_image, base_image=base_image, error=error)
272 LOG.info('Convert image %s to %s', base_image, vm_image)
273 qemu_cmd = ('qemu-img create -f qcow2 -o backing_file=%s %s' %
274 (base_image, vm_image))
275 status, _, error = connection.execute(qemu_cmd)
277 raise exceptions.LibvirtQemuImageCreateError(
278 vm_image=vm_image, base_image=base_image, error=error)
282 def build_vm_xml(cls, connection, flavor, vm_name, index, cdrom_img):
283 """Build the XML from the configuration parameters"""
284 memory = flavor.get('ram', '4096')
285 extra_spec = flavor.get('extra_specs', {})
286 cpu = extra_spec.get('hw:cpu_cores', '2')
287 socket = extra_spec.get('hw:cpu_sockets', '1')
288 threads = extra_spec.get('hw:cpu_threads', '2')
289 vcpu = int(cpu) * int(threads)
290 numa_cpus = '0-%s' % (vcpu - 1)
291 hw_socket = flavor.get('hw_socket', '0')
292 cpuset = Libvirt.pin_vcpu_for_perf(connection, hw_socket)
294 cputune = extra_spec.get('cputune', '')
295 mac = StandaloneContextHelper.get_mac_address(0x00)
296 image = cls.create_snapshot_qemu(connection, index,
297 flavor.get("images", None))
298 vm_xml = VM_TEMPLATE.format(
300 random_uuid=uuid.uuid4(),
302 memory=memory, vcpu=vcpu, cpu=cpu,
304 socket=socket, threads=threads,
305 vm_image=image, cpuset=cpuset, cputune=cputune)
308 vm_xml = Libvirt.add_cdrom(cdrom_img, vm_xml)
313 def update_interrupts_hugepages_perf(connection):
314 connection.execute("echo 1 > /sys/module/kvm/parameters/allow_unsafe_assigned_interrupts")
315 connection.execute("echo never > /sys/kernel/mm/transparent_hugepage/enabled")
318 def pin_vcpu_for_perf(cls, connection, socket='0'):
320 sys_obj = CpuSysCores(connection)
321 soc_cpu = sys_obj.get_core_socket()
322 sys_cpu = int(soc_cpu["cores_per_socket"])
324 cores = "%s-%s" % (soc_cpu[socket][0], soc_cpu[socket][sys_cpu - 1])
325 if int(soc_cpu["thread_per_core"]) > 1:
326 threads = "%s-%s" % (soc_cpu[socket][sys_cpu], soc_cpu[socket][-1])
327 cpuset = "%s,%s" % (cores, threads)
331 def write_file(cls, file_name, xml_str):
332 """Dump a XML string to a file"""
333 root = ET.fromstring(xml_str)
334 et = ET.ElementTree(element=root)
335 et.write(file_name, encoding='utf-8', method='xml')
338 def add_cdrom(cls, file_path, xml_str):
339 """Add a CD-ROM disk XML node in 'devices' node
342 <disk type='file' device='cdrom'>
343 <driver name='qemu' type='raw'/>
344 <source file='/var/lib/libvirt/images/data.img'/>
352 root = ET.fromstring(xml_str)
353 device = root.find('devices')
355 disk = ET.SubElement(device, 'disk')
356 disk.set('type', 'file')
357 disk.set('device', 'cdrom')
359 driver = ET.SubElement(disk, 'driver')
360 driver.set('name', 'qemu')
361 driver.set('type', 'raw')
363 source = ET.SubElement(disk, 'source')
364 source.set('file', file_path)
366 target = ET.SubElement(disk, 'target')
367 target.set('dev', 'hdb')
369 ET.SubElement(disk, 'readonly')
370 return ET.tostring(root)
373 def gen_cdrom_image(connection, file_path, vm_name, vm_user, key_filename):
374 """Generate ISO image for CD-ROM """
376 user_config = [" - name: {user_name}",
377 " ssh_authorized_keys:",
379 if vm_user != "root":
380 user_config.append(" sudo: ALL=(ALL) NOPASSWD:ALL")
382 meta_data = "/tmp/meta-data"
383 user_data = "/tmp/user-data"
384 with open(".".join([key_filename, "pub"]), "r") as pub_key_file:
385 pub_key_str = pub_key_file.read().rstrip()
386 user_conf = os.linesep.join(user_config).format(pub_key_str=pub_key_str, user_name=vm_user)
389 "touch %s" % meta_data,
390 USER_DATA_TEMPLATE.format(user_file=user_data, host=vm_name, user_config=user_conf),
391 "genisoimage -output {0} -volid cidata -joliet -r {1} {2}".format(file_path,
394 "rm {0} {1}".format(meta_data, user_data),
398 status, _, error = connection.execute(cmd)
400 raise exceptions.LibvirtQemuImageCreateError(error=error)
403 class StandaloneContextHelper(object):
404 """ This class handles all the common code for standalone
407 self.file_path = None
408 super(StandaloneContextHelper, self).__init__()
411 def install_req_libs(connection, extra_pkgs=None):
412 extra_pkgs = extra_pkgs or []
413 pkgs = ["qemu-kvm", "libvirt-bin", "bridge-utils", "numactl", "fping", "genisoimage"]
414 pkgs.extend(extra_pkgs)
415 cmd_template = "dpkg-query -W --showformat='${Status}\\n' \"%s\"|grep 'ok installed'"
417 if connection.execute(cmd_template % pkg)[0]:
418 connection.execute("apt-get update")
419 connection.execute("apt-get -y install %s" % pkg)
422 def get_kernel_module(connection, pci, driver):
424 out = connection.execute("lspci -k -s %s" % pci)[1]
425 driver = out.split("Kernel modules:").pop().strip()
429 def get_nic_details(cls, connection, networks, dpdk_devbind):
430 for key, ports in networks.items():
434 phy_ports = ports['phy_port']
435 phy_driver = ports.get('phy_driver', None)
436 driver = cls.get_kernel_module(connection, phy_ports, phy_driver)
438 # Make sure that ports are bound to kernel drivers e.g. i40e/ixgbe
439 bind_cmd = "{dpdk_devbind} --force -b {driver} {port}"
440 lshw_cmd = "lshw -c network -businfo | grep '{port}'"
441 link_show_cmd = "ip -s link show {interface}"
443 cmd = bind_cmd.format(dpdk_devbind=dpdk_devbind,
444 driver=driver, port=ports['phy_port'])
445 connection.execute(cmd)
447 out = connection.execute(lshw_cmd.format(port=phy_ports))[1]
448 interface = out.split()[1]
450 connection.execute(link_show_cmd.format(interface=interface))
453 'interface': str(interface),
461 def get_virtual_devices(connection, pci):
462 cmd = "cat /sys/bus/pci/devices/{0}/virtfn0/uevent"
463 output = connection.execute(cmd.format(pci))[1]
465 pattern = "PCI_SLOT_NAME=({})".format(PciAddress.PCI_PATTERN_STR)
466 m = re.search(pattern, output, re.MULTILINE)
470 pf_vfs = {pci: m.group(1).rstrip()}
472 LOG.info("pf_vfs:\n%s", pf_vfs)
476 def parse_pod_file(self, file_path, nfvi_role='Sriov'):
477 self.file_path = file_path
481 cfg = yaml_loader.read_yaml_file(self.file_path)
482 except IOError as io_error:
483 if io_error.errno != errno.ENOENT:
485 self.file_path = os.path.join(constants.YARDSTICK_ROOT_PATH,
487 cfg = yaml_loader.read_yaml_file(self.file_path)
489 nodes.extend([node for node in cfg["nodes"] if str(node["role"]) != nfvi_role])
490 nfvi_host.extend([node for node in cfg["nodes"] if str(node["role"]) == nfvi_role])
492 raise("Node role is other than SRIOV")
494 host_mgmt = {'user': nfvi_host[0]['user'],
495 'ip': str(IPNetwork(nfvi_host[0]['ip']).ip),
496 'password': nfvi_host[0]['password'],
497 'ssh_port': nfvi_host[0].get('ssh_port', 22),
498 'key_filename': nfvi_host[0].get('key_filename')}
500 return [nodes, nfvi_host, host_mgmt]
503 def get_mac_address(end=0x7f):
504 mac = [0x52, 0x54, 0x00,
505 random.randint(0x00, end),
506 random.randint(0x00, 0xff),
507 random.randint(0x00, 0xff)]
508 mac_address = ':'.join('%02x' % x for x in mac)
512 def get_mgmt_ip(connection, mac, cidr, node):
515 while not mgmtip and times:
516 connection.execute("fping -c 1 -g %s > /dev/null 2>&1" % cidr)
517 out = connection.execute("ip neighbor | grep '%s'" % mac)[1]
518 LOG.info("fping -c 1 -g %s > /dev/null 2>&1", cidr)
520 mgmtip = str(out.split(" ")[0]).strip()
521 client = ssh.SSH.from_node(node, overrides={"ip": mgmtip})
525 time.sleep(WAIT_FOR_BOOT) # FixMe: How to find if VM is booted?
530 def wait_for_vnfs_to_start(cls, connection, servers, nodes):
532 vnf = servers[node["name"]]
533 mgmtip = vnf["network_ports"]["mgmt"]["cidr"]
534 ip = cls.get_mgmt_ip(connection, node["mac"], mgmtip, node)
540 def check_update_key(cls, connection, node, vm_name, id_name, cdrom_img):
541 # Generate public/private keys if private key file is not provided
542 user_name = node.get('user')
544 node['user'] = 'root'
545 user_name = node.get('user')
546 if not node.get('key_filename'):
547 key_filename = ''.join(
548 [constants.YARDSTICK_ROOT_PATH,
549 'yardstick/resources/files/yardstick_key-',
551 ssh.SSH.gen_keys(key_filename)
552 node['key_filename'] = key_filename
553 # Update image with public key
554 key_filename = node.get('key_filename')
555 Libvirt.gen_cdrom_image(connection, cdrom_img, vm_name, user_name, key_filename)
559 class Server(object):
560 """ This class handles geting vnf nodes
564 def build_vnf_interfaces(vnf, ports):
568 for key, vfs in vnf["network_ports"].items():
570 mgmtip = str(IPNetwork(vfs['cidr']).ip)
574 ip = IPNetwork(vf['cidr'])
578 'driver': "%svf" % vf['driver'],
579 'local_mac': vf['mac'],
580 'dpdk_port_num': index,
581 'local_ip': str(ip.ip),
582 'netmask': str(ip.netmask)
587 return mgmtip, interfaces
590 def generate_vnf_instance(cls, flavor, ports, ip, key, vnf, mac):
591 mgmtip, interfaces = cls.build_vnf_interfaces(vnf, ports)
597 "user": flavor.get('user', 'root'),
598 "interfaces": interfaces,
600 # empty IPv6 routing table
602 "name": key, "role": key
606 result['key_filename'] = flavor['key_filename']
611 result['password'] = flavor['password']
618 class OvsDeploy(object):
619 """ This class handles deploy of ovs dpdk
620 Configuration: ovs_dpdk
623 OVS_DEPLOY_SCRIPT = "ovs_deploy.bash"
625 def __init__(self, connection, bin_path, ovs_properties):
626 self.connection = connection
627 self.bin_path = bin_path
628 self.ovs_properties = ovs_properties
630 def prerequisite(self):
631 pkgs = ["git", "build-essential", "pkg-config", "automake",
632 "autotools-dev", "libltdl-dev", "cmake", "libnuma-dev",
634 StandaloneContextHelper.install_req_libs(self.connection, pkgs)
636 def ovs_deploy(self):
637 ovs_deploy = os.path.join(constants.YARDSTICK_ROOT_PATH,
638 "yardstick/resources/scripts/install/",
639 self.OVS_DEPLOY_SCRIPT)
640 if os.path.isfile(ovs_deploy):
642 remote_ovs_deploy = os.path.join(self.bin_path, self.OVS_DEPLOY_SCRIPT)
643 LOG.info(remote_ovs_deploy)
644 self.connection.put(ovs_deploy, remote_ovs_deploy)
646 http_proxy = os.environ.get('http_proxy', '')
647 ovs_details = self.ovs_properties.get("version", {})
648 ovs = ovs_details.get("ovs", "2.6.0")
649 dpdk = ovs_details.get("dpdk", "16.11.1")
651 cmd = "sudo -E %s --ovs='%s' --dpdk='%s' -p='%s'" % (remote_ovs_deploy,
652 ovs, dpdk, http_proxy)
653 exit_status, _, stderr = self.connection.execute(cmd)
655 raise exceptions.OVSDeployError(stderr=stderr)