1 # Copyright (c) 2016-2017 Intel Corporation
3 # Licensed under the Apache License, Version 2.0 (the "License");
4 # you may not use this file except in compliance with the License.
5 # You may obtain a copy of the License at
7 # http://www.apache.org/licenses/LICENSE-2.0
9 # Unless required by applicable law or agreed to in writing, software
10 # distributed under the License is distributed on an "AS IS" BASIS,
11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 # See the License for the specific language governing permissions and
13 # limitations under the License.
23 from netaddr import IPNetwork
24 import xml.etree.ElementTree as ET
26 from yardstick import ssh
27 from yardstick.common import constants
28 from yardstick.common import exceptions
29 from yardstick.common.yaml_loader import yaml_load
30 from yardstick.network_services.utils import PciAddress
31 from yardstick.network_services.helpers.cpu import CpuSysCores
34 LOG = logging.getLogger(__name__)
38 <name>{vm_name}</name>
39 <uuid>{random_uuid}</uuid>
40 <memory unit="MB">{memory}</memory>
41 <currentMemory unit="MB">{memory}</currentMemory>
45 <vcpu cpuset='{cpuset}'>{vcpu}</vcpu>
48 <type arch="x86_64" machine="pc-i440fx-utopic">hvm</type>
56 <cpu mode='host-passthrough'>
57 <topology cores="{cpu}" sockets="{socket}" threads="{threads}" />
59 <cell id='0' cpus='{numa_cpus}' memory='{memory}' unit='MB' memAccess='shared'/>
63 <timer name="rtc" tickpolicy="catchup" />
64 <timer name="pit" tickpolicy="delay" />
65 <timer name="hpet" present="no" />
67 <on_poweroff>destroy</on_poweroff>
68 <on_reboot>restart</on_reboot>
69 <on_crash>restart</on_crash>
71 <emulator>/usr/bin/kvm-spice</emulator>
72 <disk device="disk" type="file">
73 <driver name="qemu" type="qcow2" />
74 <source file="{vm_image}"/>
75 <target bus="virtio" dev="vda" />
77 <graphics autoport="yes" listen="0.0.0.0" port="-1" type="vnc" />
78 <interface type="bridge">
79 <mac address='{mac_addr}'/>
80 <source bridge="br-int" />
81 <model type='virtio'/>
87 <target type='serial' port='0'/>
95 class Libvirt(object):
96 """ This class handles all the libvirt updates to lauch VM
100 def check_if_vm_exists_and_delete(vm_name, connection):
101 cmd_template = "virsh list --name | grep -i %s"
102 status = connection.execute(cmd_template % vm_name)[0]
104 LOG.info("VM '%s' is already present... destroying", vm_name)
105 connection.execute("virsh destroy %s" % vm_name)
108 def virsh_create_vm(connection, cfg):
109 LOG.info('VM create, XML config: %s', cfg)
110 status, _, error = connection.execute('virsh create %s' % cfg)
112 raise exceptions.LibvirtCreateError(error=error)
115 def virsh_destroy_vm(vm_name, connection):
116 LOG.info('VM destroy, VM name: %s', vm_name)
117 status, _, error = connection.execute('virsh destroy %s' % vm_name)
119 LOG.warning('Error destroying VM %s. Error: %s', vm_name, error)
122 def _add_interface_address(interface, pci_address):
123 """Add a PCI 'address' XML node
125 <address type='pci' domain='0x0000' bus='0x00' slot='0x08'
128 Refence: https://software.intel.com/en-us/articles/
129 configure-sr-iov-network-virtual-functions-in-linux-kvm
131 vm_pci = ET.SubElement(interface, 'address')
132 vm_pci.set('type', 'pci')
133 vm_pci.set('domain', '0x{}'.format(pci_address.domain))
134 vm_pci.set('bus', '0x{}'.format(pci_address.bus))
135 vm_pci.set('slot', '0x{}'.format(pci_address.slot))
136 vm_pci.set('function', '0x{}'.format(pci_address.function))
140 def add_ovs_interface(cls, vpath, port_num, vpci, vports_mac, xml_str):
141 """Add a DPDK OVS 'interface' XML node in 'devices' node
144 <interface type='vhostuser'>
145 <mac address='00:00:00:00:00:01'/>
146 <source type='unix' path='/usr/local/var/run/openvswitch/
147 dpdkvhostuser0' mode='client'/>
148 <model type='virtio'/>
150 <host mrg_rxbuf='off'/>
152 <address type='pci' domain='0x0000' bus='0x00' slot='0x03'
158 Reference: http://docs.openvswitch.org/en/latest/topics/dpdk/
162 vhost_path = ('{0}/var/run/openvswitch/dpdkvhostuser{1}'.
163 format(vpath, port_num))
164 root = ET.fromstring(xml_str)
165 pci_address = PciAddress(vpci.strip())
166 device = root.find('devices')
168 interface = ET.SubElement(device, 'interface')
169 interface.set('type', 'vhostuser')
170 mac = ET.SubElement(interface, 'mac')
171 mac.set('address', vports_mac)
173 source = ET.SubElement(interface, 'source')
174 source.set('type', 'unix')
175 source.set('path', vhost_path)
176 source.set('mode', 'client')
178 model = ET.SubElement(interface, 'model')
179 model.set('type', 'virtio')
181 driver = ET.SubElement(interface, 'driver')
182 driver.set('queues', '4')
184 host = ET.SubElement(driver, 'host')
185 host.set('mrg_rxbuf', 'off')
187 cls._add_interface_address(interface, pci_address)
189 return ET.tostring(root)
192 def add_sriov_interfaces(cls, vm_pci, vf_pci, vf_mac, xml_str):
193 """Add a SR-IOV 'interface' XML node in 'devices' node
196 <interface type='hostdev' managed='yes'>
198 <address type='pci' domain='0x0000' bus='0x00' slot='0x03'
201 <mac address='52:54:00:6d:90:02'>
202 <address type='pci' domain='0x0000' bus='0x02' slot='0x04'
208 Reference: https://access.redhat.com/documentation/en-us/
209 red_hat_enterprise_linux/6/html/
210 virtualization_host_configuration_and_guest_installation_guide/
211 sect-virtualization_host_configuration_and_guest_installation_guide
212 -sr_iov-how_sr_iov_libvirt_works
215 root = ET.fromstring(xml_str)
216 device = root.find('devices')
218 interface = ET.SubElement(device, 'interface')
219 interface.set('managed', 'yes')
220 interface.set('type', 'hostdev')
222 mac = ET.SubElement(interface, 'mac')
223 mac.set('address', vf_mac)
225 source = ET.SubElement(interface, 'source')
226 pci_address = PciAddress(vf_pci.strip())
227 cls._add_interface_address(source, pci_address)
229 pci_vm_address = PciAddress(vm_pci.strip())
230 cls._add_interface_address(interface, pci_vm_address)
232 return ET.tostring(root)
235 def create_snapshot_qemu(connection, index, vm_image):
236 # build snapshot image
237 image = "/var/lib/libvirt/images/%s.qcow2" % index
238 connection.execute("rm %s" % image)
239 qemu_template = "qemu-img create -f qcow2 -o backing_file=%s %s"
240 connection.execute(qemu_template % (vm_image, image))
245 def build_vm_xml(cls, connection, flavor, vm_name, index):
246 """Build the XML from the configuration parameters"""
247 memory = flavor.get('ram', '4096')
248 extra_spec = flavor.get('extra_specs', {})
249 cpu = extra_spec.get('hw:cpu_cores', '2')
250 socket = extra_spec.get('hw:cpu_sockets', '1')
251 threads = extra_spec.get('hw:cpu_threads', '2')
252 vcpu = int(cpu) * int(threads)
253 numa_cpus = '0-%s' % (vcpu - 1)
254 hw_socket = flavor.get('hw_socket', '0')
255 cpuset = Libvirt.pin_vcpu_for_perf(connection, hw_socket)
257 cputune = extra_spec.get('cputune', '')
258 mac = StandaloneContextHelper.get_mac_address(0x00)
259 image = cls.create_snapshot_qemu(connection, index,
260 flavor.get("images", None))
261 vm_xml = VM_TEMPLATE.format(
263 random_uuid=uuid.uuid4(),
265 memory=memory, vcpu=vcpu, cpu=cpu,
267 socket=socket, threads=threads,
268 vm_image=image, cpuset=cpuset, cputune=cputune)
273 def update_interrupts_hugepages_perf(connection):
274 connection.execute("echo 1 > /sys/module/kvm/parameters/allow_unsafe_assigned_interrupts")
275 connection.execute("echo never > /sys/kernel/mm/transparent_hugepage/enabled")
278 def pin_vcpu_for_perf(cls, connection, socket='0'):
280 sys_obj = CpuSysCores(connection)
281 soc_cpu = sys_obj.get_core_socket()
282 sys_cpu = int(soc_cpu["cores_per_socket"])
284 cores = "%s-%s" % (soc_cpu[socket][0], soc_cpu[socket][sys_cpu - 1])
285 if int(soc_cpu["thread_per_core"]) > 1:
286 threads = "%s-%s" % (soc_cpu[socket][sys_cpu], soc_cpu[socket][-1])
287 cpuset = "%s,%s" % (cores, threads)
291 def write_file(cls, file_name, xml_str):
292 """Dump a XML string to a file"""
293 root = ET.fromstring(xml_str)
294 et = ET.ElementTree(element=root)
295 et.write(file_name, encoding='utf-8', method='xml')
298 class StandaloneContextHelper(object):
299 """ This class handles all the common code for standalone
302 self.file_path = None
303 super(StandaloneContextHelper, self).__init__()
306 def install_req_libs(connection, extra_pkgs=None):
307 extra_pkgs = extra_pkgs or []
308 pkgs = ["qemu-kvm", "libvirt-bin", "bridge-utils", "numactl", "fping"]
309 pkgs.extend(extra_pkgs)
310 cmd_template = "dpkg-query -W --showformat='${Status}\\n' \"%s\"|grep 'ok installed'"
312 if connection.execute(cmd_template % pkg)[0]:
313 connection.execute("apt-get update")
314 connection.execute("apt-get -y install %s" % pkg)
317 def get_kernel_module(connection, pci, driver):
319 out = connection.execute("lspci -k -s %s" % pci)[1]
320 driver = out.split("Kernel modules:").pop().strip()
324 def get_nic_details(cls, connection, networks, dpdk_devbind):
325 for key, ports in networks.items():
329 phy_ports = ports['phy_port']
330 phy_driver = ports.get('phy_driver', None)
331 driver = cls.get_kernel_module(connection, phy_ports, phy_driver)
333 # Make sure that ports are bound to kernel drivers e.g. i40e/ixgbe
334 bind_cmd = "{dpdk_devbind} --force -b {driver} {port}"
335 lshw_cmd = "lshw -c network -businfo | grep '{port}'"
336 link_show_cmd = "ip -s link show {interface}"
338 cmd = bind_cmd.format(dpdk_devbind=dpdk_devbind,
339 driver=driver, port=ports['phy_port'])
340 connection.execute(cmd)
342 out = connection.execute(lshw_cmd.format(port=phy_ports))[1]
343 interface = out.split()[1]
345 connection.execute(link_show_cmd.format(interface=interface))
348 'interface': str(interface),
356 def get_virtual_devices(connection, pci):
357 cmd = "cat /sys/bus/pci/devices/{0}/virtfn0/uevent"
358 output = connection.execute(cmd.format(pci))[1]
360 pattern = "PCI_SLOT_NAME=({})".format(PciAddress.PCI_PATTERN_STR)
361 m = re.search(pattern, output, re.MULTILINE)
365 pf_vfs = {pci: m.group(1).rstrip()}
367 LOG.info("pf_vfs:\n%s", pf_vfs)
371 def read_config_file(self):
372 """Read from config file"""
374 with open(self.file_path) as stream:
375 LOG.info("Parsing pod file: %s", self.file_path)
376 cfg = yaml_load(stream)
379 def parse_pod_file(self, file_path, nfvi_role='Sriov'):
380 self.file_path = file_path
384 cfg = self.read_config_file()
385 except IOError as io_error:
386 if io_error.errno != errno.ENOENT:
388 self.file_path = os.path.join(constants.YARDSTICK_ROOT_PATH,
390 cfg = self.read_config_file()
392 nodes.extend([node for node in cfg["nodes"] if str(node["role"]) != nfvi_role])
393 nfvi_host.extend([node for node in cfg["nodes"] if str(node["role"]) == nfvi_role])
395 raise("Node role is other than SRIOV")
397 host_mgmt = {'user': nfvi_host[0]['user'],
398 'ip': str(IPNetwork(nfvi_host[0]['ip']).ip),
399 'password': nfvi_host[0]['password'],
400 'ssh_port': nfvi_host[0].get('ssh_port', 22),
401 'key_filename': nfvi_host[0].get('key_filename')}
403 return [nodes, nfvi_host, host_mgmt]
406 def get_mac_address(end=0x7f):
407 mac = [0x52, 0x54, 0x00,
408 random.randint(0x00, end),
409 random.randint(0x00, 0xff),
410 random.randint(0x00, 0xff)]
411 mac_address = ':'.join('%02x' % x for x in mac)
415 def get_mgmt_ip(connection, mac, cidr, node):
418 while not mgmtip and times:
419 connection.execute("fping -c 1 -g %s > /dev/null 2>&1" % cidr)
420 out = connection.execute("ip neighbor | grep '%s'" % mac)[1]
421 LOG.info("fping -c 1 -g %s > /dev/null 2>&1", cidr)
423 mgmtip = str(out.split(" ")[0]).strip()
424 client = ssh.SSH.from_node(node, overrides={"ip": mgmtip})
428 time.sleep(WAIT_FOR_BOOT) # FixMe: How to find if VM is booted?
433 def wait_for_vnfs_to_start(cls, connection, servers, nodes):
435 vnf = servers[node["name"]]
436 mgmtip = vnf["network_ports"]["mgmt"]["cidr"]
437 ip = cls.get_mgmt_ip(connection, node["mac"], mgmtip, node)
443 class Server(object):
444 """ This class handles geting vnf nodes
448 def build_vnf_interfaces(vnf, ports):
452 for key, vfs in vnf["network_ports"].items():
454 mgmtip = str(IPNetwork(vfs['cidr']).ip)
458 ip = IPNetwork(vf['cidr'])
462 'driver': "%svf" % vf['driver'],
463 'local_mac': vf['mac'],
464 'dpdk_port_num': index,
465 'local_ip': str(ip.ip),
466 'netmask': str(ip.netmask)
471 return mgmtip, interfaces
474 def generate_vnf_instance(cls, flavor, ports, ip, key, vnf, mac):
475 mgmtip, interfaces = cls.build_vnf_interfaces(vnf, ports)
481 "user": flavor.get('user', 'root'),
482 "interfaces": interfaces,
484 # empty IPv6 routing table
486 "name": key, "role": key
490 result['key_filename'] = flavor['key_filename']
495 result['password'] = flavor['password']
502 class OvsDeploy(object):
503 """ This class handles deploy of ovs dpdk
504 Configuration: ovs_dpdk
507 OVS_DEPLOY_SCRIPT = "ovs_deploy.bash"
509 def __init__(self, connection, bin_path, ovs_properties):
510 self.connection = connection
511 self.bin_path = bin_path
512 self.ovs_properties = ovs_properties
514 def prerequisite(self):
515 pkgs = ["git", "build-essential", "pkg-config", "automake",
516 "autotools-dev", "libltdl-dev", "cmake", "libnuma-dev",
518 StandaloneContextHelper.install_req_libs(self.connection, pkgs)
520 def ovs_deploy(self):
521 ovs_deploy = os.path.join(constants.YARDSTICK_ROOT_PATH,
522 "yardstick/resources/scripts/install/",
523 self.OVS_DEPLOY_SCRIPT)
524 if os.path.isfile(ovs_deploy):
526 remote_ovs_deploy = os.path.join(self.bin_path, self.OVS_DEPLOY_SCRIPT)
527 LOG.info(remote_ovs_deploy)
528 self.connection.put(ovs_deploy, remote_ovs_deploy)
530 http_proxy = os.environ.get('http_proxy', '')
531 ovs_details = self.ovs_properties.get("version", {})
532 ovs = ovs_details.get("ovs", "2.6.0")
533 dpdk = ovs_details.get("dpdk", "16.11.1")
535 cmd = "sudo -E %s --ovs='%s' --dpdk='%s' -p='%s'" % (remote_ovs_deploy,
536 ovs, dpdk, http_proxy)
537 exit_status, _, stderr = self.connection.execute(cmd)
539 raise exceptions.OVSDeployError(stderr=stderr)