1 # Copyright (c) 2016-2017 Intel Corporation
3 # Licensed under the Apache License, Version 2.0 (the "License");
4 # you may not use this file except in compliance with the License.
5 # You may obtain a copy of the License at
7 # http://www.apache.org/licenses/LICENSE-2.0
9 # Unless required by applicable law or agreed to in writing, software
10 # distributed under the License is distributed on an "AS IS" BASIS,
11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 # See the License for the specific language governing permissions and
13 # limitations under the License.
23 from netaddr import IPNetwork
24 import xml.etree.ElementTree as ET
26 from yardstick import ssh
27 from yardstick.common import constants
28 from yardstick.common import exceptions
29 from yardstick.common import yaml_loader
30 from yardstick.network_services.utils import PciAddress
31 from yardstick.network_services.helpers.cpu import CpuSysCores
34 LOG = logging.getLogger(__name__)
38 <name>{vm_name}</name>
39 <uuid>{random_uuid}</uuid>
40 <memory unit="MB">{memory}</memory>
41 <currentMemory unit="MB">{memory}</currentMemory>
45 <vcpu cpuset='{cpuset}'>{vcpu}</vcpu>
48 <type arch="x86_64" machine="pc-i440fx-xenial">hvm</type>
56 <cpu mode='host-passthrough'>
57 <topology cores="{cpu}" sockets="{socket}" threads="{threads}" />
59 <cell id='0' cpus='{numa_cpus}' memory='{memory}' unit='MB' memAccess='shared'/>
63 <timer name="rtc" tickpolicy="catchup" />
64 <timer name="pit" tickpolicy="delay" />
65 <timer name="hpet" present="no" />
67 <on_poweroff>destroy</on_poweroff>
68 <on_reboot>restart</on_reboot>
69 <on_crash>restart</on_crash>
71 <emulator>/usr/bin/kvm-spice</emulator>
72 <disk device="disk" type="file">
73 <driver name="qemu" type="qcow2" />
74 <source file="{vm_image}"/>
75 <target bus="virtio" dev="vda" />
77 <graphics autoport="yes" listen="0.0.0.0" port="-1" type="vnc" />
78 <interface type="bridge">
79 <mac address='{mac_addr}'/>
80 <source bridge="br-int" />
81 <model type='virtio'/>
87 <target type='serial' port='0'/>
95 class Libvirt(object):
96 """ This class handles all the libvirt updates to lauch VM
100 def check_if_vm_exists_and_delete(vm_name, connection):
101 cmd_template = "virsh list --name | grep -i %s"
102 status = connection.execute(cmd_template % vm_name)[0]
104 LOG.info("VM '%s' is already present... destroying", vm_name)
105 connection.execute("virsh destroy %s" % vm_name)
108 def virsh_create_vm(connection, cfg):
109 LOG.info('VM create, XML config: %s', cfg)
110 status, _, error = connection.execute('virsh create %s' % cfg)
112 raise exceptions.LibvirtCreateError(error=error)
115 def virsh_destroy_vm(vm_name, connection):
116 LOG.info('VM destroy, VM name: %s', vm_name)
117 status, _, error = connection.execute('virsh destroy %s' % vm_name)
119 LOG.warning('Error destroying VM %s. Error: %s', vm_name, error)
122 def _add_interface_address(interface, pci_address):
123 """Add a PCI 'address' XML node
125 <address type='pci' domain='0x0000' bus='0x00' slot='0x08'
128 Refence: https://software.intel.com/en-us/articles/
129 configure-sr-iov-network-virtual-functions-in-linux-kvm
131 vm_pci = ET.SubElement(interface, 'address')
132 vm_pci.set('type', 'pci')
133 vm_pci.set('domain', '0x{}'.format(pci_address.domain))
134 vm_pci.set('bus', '0x{}'.format(pci_address.bus))
135 vm_pci.set('slot', '0x{}'.format(pci_address.slot))
136 vm_pci.set('function', '0x{}'.format(pci_address.function))
140 def add_ovs_interface(cls, vpath, port_num, vpci, vports_mac, xml_str):
141 """Add a DPDK OVS 'interface' XML node in 'devices' node
144 <interface type='vhostuser'>
145 <mac address='00:00:00:00:00:01'/>
146 <source type='unix' path='/usr/local/var/run/openvswitch/
147 dpdkvhostuser0' mode='client'/>
148 <model type='virtio'/>
150 <host mrg_rxbuf='off'/>
152 <address type='pci' domain='0x0000' bus='0x00' slot='0x03'
158 Reference: http://docs.openvswitch.org/en/latest/topics/dpdk/
162 vhost_path = ('{0}/var/run/openvswitch/dpdkvhostuser{1}'.
163 format(vpath, port_num))
164 root = ET.fromstring(xml_str)
165 pci_address = PciAddress(vpci.strip())
166 device = root.find('devices')
168 interface = ET.SubElement(device, 'interface')
169 interface.set('type', 'vhostuser')
170 mac = ET.SubElement(interface, 'mac')
171 mac.set('address', vports_mac)
173 source = ET.SubElement(interface, 'source')
174 source.set('type', 'unix')
175 source.set('path', vhost_path)
176 source.set('mode', 'client')
178 model = ET.SubElement(interface, 'model')
179 model.set('type', 'virtio')
181 driver = ET.SubElement(interface, 'driver')
182 driver.set('queues', '4')
184 host = ET.SubElement(driver, 'host')
185 host.set('mrg_rxbuf', 'off')
187 cls._add_interface_address(interface, pci_address)
189 return ET.tostring(root)
192 def add_sriov_interfaces(cls, vm_pci, vf_pci, vf_mac, xml_str):
193 """Add a SR-IOV 'interface' XML node in 'devices' node
196 <interface type='hostdev' managed='yes'>
198 <address type='pci' domain='0x0000' bus='0x00' slot='0x03'
201 <mac address='52:54:00:6d:90:02'>
202 <address type='pci' domain='0x0000' bus='0x02' slot='0x04'
208 Reference: https://access.redhat.com/documentation/en-us/
209 red_hat_enterprise_linux/6/html/
210 virtualization_host_configuration_and_guest_installation_guide/
211 sect-virtualization_host_configuration_and_guest_installation_guide
212 -sr_iov-how_sr_iov_libvirt_works
215 root = ET.fromstring(xml_str)
216 device = root.find('devices')
218 interface = ET.SubElement(device, 'interface')
219 interface.set('managed', 'yes')
220 interface.set('type', 'hostdev')
222 mac = ET.SubElement(interface, 'mac')
223 mac.set('address', vf_mac)
225 source = ET.SubElement(interface, 'source')
226 pci_address = PciAddress(vf_pci.strip())
227 cls._add_interface_address(source, pci_address)
229 pci_vm_address = PciAddress(vm_pci.strip())
230 cls._add_interface_address(interface, pci_vm_address)
232 return ET.tostring(root)
235 def create_snapshot_qemu(connection, index, base_image):
236 """Create the snapshot image for a VM using a base image
238 :param connection: SSH connection to the remote host
239 :param index: index of the VM to be spawn
240 :param base_image: path of the VM base image in the remote host
241 :return: snapshot image path
243 vm_image = '/var/lib/libvirt/images/%s.qcow2' % index
244 connection.execute('rm -- "%s"' % vm_image)
245 status, _, _ = connection.execute('test -r %s' % base_image)
247 if not os.access(base_image, os.R_OK):
248 raise exceptions.LibvirtQemuImageBaseImageNotPresent(
249 vm_image=vm_image, base_image=base_image)
250 # NOTE(ralonsoh): done in two steps to avoid root permission
252 LOG.info('Copy %s from execution host to remote host', base_image)
253 file_name = os.path.basename(os.path.normpath(base_image))
254 connection.put_file(base_image, '/tmp/%s' % file_name)
255 status, _, error = connection.execute(
256 'mv -- "/tmp/%s" "%s"' % (file_name, base_image))
258 raise exceptions.LibvirtQemuImageCreateError(
259 vm_image=vm_image, base_image=base_image, error=error)
261 LOG.info('Convert image %s to %s', base_image, vm_image)
262 qemu_cmd = ('qemu-img create -f qcow2 -o backing_file=%s %s' %
263 (base_image, vm_image))
264 status, _, error = connection.execute(qemu_cmd)
266 raise exceptions.LibvirtQemuImageCreateError(
267 vm_image=vm_image, base_image=base_image, error=error)
271 def build_vm_xml(cls, connection, flavor, vm_name, index):
272 """Build the XML from the configuration parameters"""
273 memory = flavor.get('ram', '4096')
274 extra_spec = flavor.get('extra_specs', {})
275 cpu = extra_spec.get('hw:cpu_cores', '2')
276 socket = extra_spec.get('hw:cpu_sockets', '1')
277 threads = extra_spec.get('hw:cpu_threads', '2')
278 vcpu = int(cpu) * int(threads)
279 numa_cpus = '0-%s' % (vcpu - 1)
280 hw_socket = flavor.get('hw_socket', '0')
281 cpuset = Libvirt.pin_vcpu_for_perf(connection, hw_socket)
283 cputune = extra_spec.get('cputune', '')
284 mac = StandaloneContextHelper.get_mac_address(0x00)
285 image = cls.create_snapshot_qemu(connection, index,
286 flavor.get("images", None))
287 vm_xml = VM_TEMPLATE.format(
289 random_uuid=uuid.uuid4(),
291 memory=memory, vcpu=vcpu, cpu=cpu,
293 socket=socket, threads=threads,
294 vm_image=image, cpuset=cpuset, cputune=cputune)
299 def update_interrupts_hugepages_perf(connection):
300 connection.execute("echo 1 > /sys/module/kvm/parameters/allow_unsafe_assigned_interrupts")
301 connection.execute("echo never > /sys/kernel/mm/transparent_hugepage/enabled")
304 def pin_vcpu_for_perf(cls, connection, socket='0'):
306 sys_obj = CpuSysCores(connection)
307 soc_cpu = sys_obj.get_core_socket()
308 sys_cpu = int(soc_cpu["cores_per_socket"])
310 cores = "%s-%s" % (soc_cpu[socket][0], soc_cpu[socket][sys_cpu - 1])
311 if int(soc_cpu["thread_per_core"]) > 1:
312 threads = "%s-%s" % (soc_cpu[socket][sys_cpu], soc_cpu[socket][-1])
313 cpuset = "%s,%s" % (cores, threads)
317 def write_file(cls, file_name, xml_str):
318 """Dump a XML string to a file"""
319 root = ET.fromstring(xml_str)
320 et = ET.ElementTree(element=root)
321 et.write(file_name, encoding='utf-8', method='xml')
324 class StandaloneContextHelper(object):
325 """ This class handles all the common code for standalone
328 self.file_path = None
329 super(StandaloneContextHelper, self).__init__()
332 def install_req_libs(connection, extra_pkgs=None):
333 extra_pkgs = extra_pkgs or []
334 pkgs = ["qemu-kvm", "libvirt-bin", "bridge-utils", "numactl", "fping"]
335 pkgs.extend(extra_pkgs)
336 cmd_template = "dpkg-query -W --showformat='${Status}\\n' \"%s\"|grep 'ok installed'"
338 if connection.execute(cmd_template % pkg)[0]:
339 connection.execute("apt-get update")
340 connection.execute("apt-get -y install %s" % pkg)
343 def get_kernel_module(connection, pci, driver):
345 out = connection.execute("lspci -k -s %s" % pci)[1]
346 driver = out.split("Kernel modules:").pop().strip()
350 def get_nic_details(cls, connection, networks, dpdk_devbind):
351 for key, ports in networks.items():
355 phy_ports = ports['phy_port']
356 phy_driver = ports.get('phy_driver', None)
357 driver = cls.get_kernel_module(connection, phy_ports, phy_driver)
359 # Make sure that ports are bound to kernel drivers e.g. i40e/ixgbe
360 bind_cmd = "{dpdk_devbind} --force -b {driver} {port}"
361 lshw_cmd = "lshw -c network -businfo | grep '{port}'"
362 link_show_cmd = "ip -s link show {interface}"
364 cmd = bind_cmd.format(dpdk_devbind=dpdk_devbind,
365 driver=driver, port=ports['phy_port'])
366 connection.execute(cmd)
368 out = connection.execute(lshw_cmd.format(port=phy_ports))[1]
369 interface = out.split()[1]
371 connection.execute(link_show_cmd.format(interface=interface))
374 'interface': str(interface),
382 def get_virtual_devices(connection, pci):
383 cmd = "cat /sys/bus/pci/devices/{0}/virtfn0/uevent"
384 output = connection.execute(cmd.format(pci))[1]
386 pattern = "PCI_SLOT_NAME=({})".format(PciAddress.PCI_PATTERN_STR)
387 m = re.search(pattern, output, re.MULTILINE)
391 pf_vfs = {pci: m.group(1).rstrip()}
393 LOG.info("pf_vfs:\n%s", pf_vfs)
397 def parse_pod_file(self, file_path, nfvi_role='Sriov'):
398 self.file_path = file_path
402 cfg = yaml_loader.read_yaml_file(self.file_path)
403 except IOError as io_error:
404 if io_error.errno != errno.ENOENT:
406 self.file_path = os.path.join(constants.YARDSTICK_ROOT_PATH,
408 cfg = yaml_loader.read_yaml_file(self.file_path)
410 nodes.extend([node for node in cfg["nodes"] if str(node["role"]) != nfvi_role])
411 nfvi_host.extend([node for node in cfg["nodes"] if str(node["role"]) == nfvi_role])
413 raise("Node role is other than SRIOV")
415 host_mgmt = {'user': nfvi_host[0]['user'],
416 'ip': str(IPNetwork(nfvi_host[0]['ip']).ip),
417 'password': nfvi_host[0]['password'],
418 'ssh_port': nfvi_host[0].get('ssh_port', 22),
419 'key_filename': nfvi_host[0].get('key_filename')}
421 return [nodes, nfvi_host, host_mgmt]
424 def get_mac_address(end=0x7f):
425 mac = [0x52, 0x54, 0x00,
426 random.randint(0x00, end),
427 random.randint(0x00, 0xff),
428 random.randint(0x00, 0xff)]
429 mac_address = ':'.join('%02x' % x for x in mac)
433 def get_mgmt_ip(connection, mac, cidr, node):
436 while not mgmtip and times:
437 connection.execute("fping -c 1 -g %s > /dev/null 2>&1" % cidr)
438 out = connection.execute("ip neighbor | grep '%s'" % mac)[1]
439 LOG.info("fping -c 1 -g %s > /dev/null 2>&1", cidr)
441 mgmtip = str(out.split(" ")[0]).strip()
442 client = ssh.SSH.from_node(node, overrides={"ip": mgmtip})
446 time.sleep(WAIT_FOR_BOOT) # FixMe: How to find if VM is booted?
451 def wait_for_vnfs_to_start(cls, connection, servers, nodes):
453 vnf = servers[node["name"]]
454 mgmtip = vnf["network_ports"]["mgmt"]["cidr"]
455 ip = cls.get_mgmt_ip(connection, node["mac"], mgmtip, node)
461 class Server(object):
462 """ This class handles geting vnf nodes
466 def build_vnf_interfaces(vnf, ports):
470 for key, vfs in vnf["network_ports"].items():
472 mgmtip = str(IPNetwork(vfs['cidr']).ip)
476 ip = IPNetwork(vf['cidr'])
480 'driver': "%svf" % vf['driver'],
481 'local_mac': vf['mac'],
482 'dpdk_port_num': index,
483 'local_ip': str(ip.ip),
484 'netmask': str(ip.netmask)
489 return mgmtip, interfaces
492 def generate_vnf_instance(cls, flavor, ports, ip, key, vnf, mac):
493 mgmtip, interfaces = cls.build_vnf_interfaces(vnf, ports)
499 "user": flavor.get('user', 'root'),
500 "interfaces": interfaces,
502 # empty IPv6 routing table
504 "name": key, "role": key
508 result['key_filename'] = flavor['key_filename']
513 result['password'] = flavor['password']
520 class OvsDeploy(object):
521 """ This class handles deploy of ovs dpdk
522 Configuration: ovs_dpdk
525 OVS_DEPLOY_SCRIPT = "ovs_deploy.bash"
527 def __init__(self, connection, bin_path, ovs_properties):
528 self.connection = connection
529 self.bin_path = bin_path
530 self.ovs_properties = ovs_properties
532 def prerequisite(self):
533 pkgs = ["git", "build-essential", "pkg-config", "automake",
534 "autotools-dev", "libltdl-dev", "cmake", "libnuma-dev",
536 StandaloneContextHelper.install_req_libs(self.connection, pkgs)
538 def ovs_deploy(self):
539 ovs_deploy = os.path.join(constants.YARDSTICK_ROOT_PATH,
540 "yardstick/resources/scripts/install/",
541 self.OVS_DEPLOY_SCRIPT)
542 if os.path.isfile(ovs_deploy):
544 remote_ovs_deploy = os.path.join(self.bin_path, self.OVS_DEPLOY_SCRIPT)
545 LOG.info(remote_ovs_deploy)
546 self.connection.put(ovs_deploy, remote_ovs_deploy)
548 http_proxy = os.environ.get('http_proxy', '')
549 ovs_details = self.ovs_properties.get("version", {})
550 ovs = ovs_details.get("ovs", "2.6.0")
551 dpdk = ovs_details.get("dpdk", "16.11.1")
553 cmd = "sudo -E %s --ovs='%s' --dpdk='%s' -p='%s'" % (remote_ovs_deploy,
554 ovs, dpdk, http_proxy)
555 exit_status, _, stderr = self.connection.execute(cmd)
557 raise exceptions.OVSDeployError(stderr=stderr)