1 # Copyright (c) 2016-2017 Intel Corporation
3 # Licensed under the Apache License, Version 2.0 (the "License");
4 # you may not use this file except in compliance with the License.
5 # You may obtain a copy of the License at
7 # http://www.apache.org/licenses/LICENSE-2.0
9 # Unless required by applicable law or agreed to in writing, software
10 # distributed under the License is distributed on an "AS IS" BASIS,
11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 # See the License for the specific language governing permissions and
13 # limitations under the License.
23 from netaddr import IPNetwork
24 import xml.etree.ElementTree as ET
26 from yardstick import ssh
27 from yardstick.common import constants
28 from yardstick.common import exceptions
29 from yardstick.common.yaml_loader import yaml_load
30 from yardstick.network_services.utils import PciAddress
31 from yardstick.network_services.helpers.cpu import CpuSysCores
34 LOG = logging.getLogger(__name__)
38 <name>{vm_name}</name>
39 <uuid>{random_uuid}</uuid>
40 <memory unit="MB">{memory}</memory>
41 <currentMemory unit="MB">{memory}</currentMemory>
45 <vcpu cpuset='{cpuset}'>{vcpu}</vcpu>
48 <type arch="x86_64" machine="pc-i440fx-utopic">hvm</type>
56 <cpu mode='host-passthrough'>
57 <topology cores="{cpu}" sockets="{socket}" threads="{threads}" />
59 <cell id='0' cpus='{numa_cpus}' memory='{memory}' unit='MB' memAccess='shared'/>
63 <timer name="rtc" tickpolicy="catchup" />
64 <timer name="pit" tickpolicy="delay" />
65 <timer name="hpet" present="no" />
67 <on_poweroff>destroy</on_poweroff>
68 <on_reboot>restart</on_reboot>
69 <on_crash>restart</on_crash>
71 <emulator>/usr/bin/kvm-spice</emulator>
72 <disk device="disk" type="file">
73 <driver name="qemu" type="qcow2" />
74 <source file="{vm_image}"/>
75 <target bus="virtio" dev="vda" />
77 <graphics autoport="yes" listen="0.0.0.0" port="-1" type="vnc" />
78 <interface type="bridge">
79 <mac address='{mac_addr}'/>
80 <source bridge="br-int" />
81 <model type='virtio'/>
87 <target type='serial' port='0'/>
95 class Libvirt(object):
96 """ This class handles all the libvirt updates to lauch VM
100 def check_if_vm_exists_and_delete(vm_name, connection):
101 cmd_template = "virsh list --name | grep -i %s"
102 status = connection.execute(cmd_template % vm_name)[0]
104 LOG.info("VM '%s' is already present... destroying", vm_name)
105 connection.execute("virsh destroy %s" % vm_name)
108 def virsh_create_vm(connection, cfg):
109 err = connection.execute("virsh create %s" % cfg)[0]
110 LOG.info("VM create status: %s", err)
113 def virsh_destroy_vm(vm_name, connection):
114 connection.execute("virsh destroy %s" % vm_name)
117 def _add_interface_address(interface, pci_address):
118 """Add a PCI 'address' XML node
120 <address type='pci' domain='0x0000' bus='0x00' slot='0x08'
123 Refence: https://software.intel.com/en-us/articles/
124 configure-sr-iov-network-virtual-functions-in-linux-kvm
126 vm_pci = ET.SubElement(interface, 'address')
127 vm_pci.set('type', 'pci')
128 vm_pci.set('domain', '0x{}'.format(pci_address.domain))
129 vm_pci.set('bus', '0x{}'.format(pci_address.bus))
130 vm_pci.set('slot', '0x{}'.format(pci_address.slot))
131 vm_pci.set('function', '0x{}'.format(pci_address.function))
135 def add_ovs_interface(cls, vpath, port_num, vpci, vports_mac, xml_str):
136 """Add a DPDK OVS 'interface' XML node in 'devices' node
139 <interface type='vhostuser'>
140 <mac address='00:00:00:00:00:01'/>
141 <source type='unix' path='/usr/local/var/run/openvswitch/
142 dpdkvhostuser0' mode='client'/>
143 <model type='virtio'/>
145 <host mrg_rxbuf='off'/>
147 <address type='pci' domain='0x0000' bus='0x00' slot='0x03'
153 Reference: http://docs.openvswitch.org/en/latest/topics/dpdk/
157 vhost_path = ('{0}/var/run/openvswitch/dpdkvhostuser{1}'.
158 format(vpath, port_num))
159 root = ET.fromstring(xml_str)
160 pci_address = PciAddress(vpci.strip())
161 device = root.find('devices')
163 interface = ET.SubElement(device, 'interface')
164 interface.set('type', 'vhostuser')
165 mac = ET.SubElement(interface, 'mac')
166 mac.set('address', vports_mac)
168 source = ET.SubElement(interface, 'source')
169 source.set('type', 'unix')
170 source.set('path', vhost_path)
171 source.set('mode', 'client')
173 model = ET.SubElement(interface, 'model')
174 model.set('type', 'virtio')
176 driver = ET.SubElement(interface, 'driver')
177 driver.set('queues', '4')
179 host = ET.SubElement(driver, 'host')
180 host.set('mrg_rxbuf', 'off')
182 cls._add_interface_address(interface, pci_address)
184 return ET.tostring(root)
187 def add_sriov_interfaces(cls, vm_pci, vf_pci, vf_mac, xml_str):
188 """Add a SR-IOV 'interface' XML node in 'devices' node
191 <interface type='hostdev' managed='yes'>
193 <address type='pci' domain='0x0000' bus='0x00' slot='0x03'
196 <mac address='52:54:00:6d:90:02'>
197 <address type='pci' domain='0x0000' bus='0x02' slot='0x04'
203 Reference: https://access.redhat.com/documentation/en-us/
204 red_hat_enterprise_linux/6/html/
205 virtualization_host_configuration_and_guest_installation_guide/
206 sect-virtualization_host_configuration_and_guest_installation_guide
207 -sr_iov-how_sr_iov_libvirt_works
210 root = ET.fromstring(xml_str)
211 device = root.find('devices')
213 interface = ET.SubElement(device, 'interface')
214 interface.set('managed', 'yes')
215 interface.set('type', 'hostdev')
217 mac = ET.SubElement(interface, 'mac')
218 mac.set('address', vf_mac)
220 source = ET.SubElement(interface, 'source')
221 pci_address = PciAddress(vf_pci.strip())
222 cls._add_interface_address(source, pci_address)
224 pci_vm_address = PciAddress(vm_pci.strip())
225 cls._add_interface_address(interface, pci_vm_address)
227 return ET.tostring(root)
230 def create_snapshot_qemu(connection, index, vm_image):
231 # build snapshot image
232 image = "/var/lib/libvirt/images/%s.qcow2" % index
233 connection.execute("rm %s" % image)
234 qemu_template = "qemu-img create -f qcow2 -o backing_file=%s %s"
235 connection.execute(qemu_template % (vm_image, image))
240 def build_vm_xml(cls, connection, flavor, vm_name, index):
241 """Build the XML from the configuration parameters"""
242 memory = flavor.get('ram', '4096')
243 extra_spec = flavor.get('extra_specs', {})
244 cpu = extra_spec.get('hw:cpu_cores', '2')
245 socket = extra_spec.get('hw:cpu_sockets', '1')
246 threads = extra_spec.get('hw:cpu_threads', '2')
247 vcpu = int(cpu) * int(threads)
248 numa_cpus = '0-%s' % (vcpu - 1)
249 hw_socket = flavor.get('hw_socket', '0')
250 cpuset = Libvirt.pin_vcpu_for_perf(connection, hw_socket)
252 cputune = extra_spec.get('cputune', '')
253 mac = StandaloneContextHelper.get_mac_address(0x00)
254 image = cls.create_snapshot_qemu(connection, index,
255 flavor.get("images", None))
256 vm_xml = VM_TEMPLATE.format(
258 random_uuid=uuid.uuid4(),
260 memory=memory, vcpu=vcpu, cpu=cpu,
262 socket=socket, threads=threads,
263 vm_image=image, cpuset=cpuset, cputune=cputune)
268 def update_interrupts_hugepages_perf(connection):
269 connection.execute("echo 1 > /sys/module/kvm/parameters/allow_unsafe_assigned_interrupts")
270 connection.execute("echo never > /sys/kernel/mm/transparent_hugepage/enabled")
273 def pin_vcpu_for_perf(cls, connection, socket='0'):
275 sys_obj = CpuSysCores(connection)
276 soc_cpu = sys_obj.get_core_socket()
277 sys_cpu = int(soc_cpu["cores_per_socket"])
279 cores = "%s-%s" % (soc_cpu[socket][0], soc_cpu[socket][sys_cpu - 1])
280 if int(soc_cpu["thread_per_core"]) > 1:
281 threads = "%s-%s" % (soc_cpu[socket][sys_cpu], soc_cpu[socket][-1])
282 cpuset = "%s,%s" % (cores, threads)
286 def write_file(cls, file_name, xml_str):
287 """Dump a XML string to a file"""
288 root = ET.fromstring(xml_str)
289 et = ET.ElementTree(element=root)
290 et.write(file_name, encoding='utf-8', method='xml')
293 class StandaloneContextHelper(object):
294 """ This class handles all the common code for standalone
297 self.file_path = None
298 super(StandaloneContextHelper, self).__init__()
301 def install_req_libs(connection, extra_pkgs=None):
302 extra_pkgs = extra_pkgs or []
303 pkgs = ["qemu-kvm", "libvirt-bin", "bridge-utils", "numactl", "fping"]
304 pkgs.extend(extra_pkgs)
305 cmd_template = "dpkg-query -W --showformat='${Status}\\n' \"%s\"|grep 'ok installed'"
307 if connection.execute(cmd_template % pkg)[0]:
308 connection.execute("apt-get update")
309 connection.execute("apt-get -y install %s" % pkg)
312 def get_kernel_module(connection, pci, driver):
314 out = connection.execute("lspci -k -s %s" % pci)[1]
315 driver = out.split("Kernel modules:").pop().strip()
319 def get_nic_details(cls, connection, networks, dpdk_devbind):
320 for key, ports in networks.items():
324 phy_ports = ports['phy_port']
325 phy_driver = ports.get('phy_driver', None)
326 driver = cls.get_kernel_module(connection, phy_ports, phy_driver)
328 # Make sure that ports are bound to kernel drivers e.g. i40e/ixgbe
329 bind_cmd = "{dpdk_devbind} --force -b {driver} {port}"
330 lshw_cmd = "lshw -c network -businfo | grep '{port}'"
331 link_show_cmd = "ip -s link show {interface}"
333 cmd = bind_cmd.format(dpdk_devbind=dpdk_devbind,
334 driver=driver, port=ports['phy_port'])
335 connection.execute(cmd)
337 out = connection.execute(lshw_cmd.format(port=phy_ports))[1]
338 interface = out.split()[1]
340 connection.execute(link_show_cmd.format(interface=interface))
343 'interface': str(interface),
351 def get_virtual_devices(connection, pci):
352 cmd = "cat /sys/bus/pci/devices/{0}/virtfn0/uevent"
353 output = connection.execute(cmd.format(pci))[1]
355 pattern = "PCI_SLOT_NAME=({})".format(PciAddress.PCI_PATTERN_STR)
356 m = re.search(pattern, output, re.MULTILINE)
360 pf_vfs = {pci: m.group(1).rstrip()}
362 LOG.info("pf_vfs:\n%s", pf_vfs)
366 def read_config_file(self):
367 """Read from config file"""
369 with open(self.file_path) as stream:
370 LOG.info("Parsing pod file: %s", self.file_path)
371 cfg = yaml_load(stream)
374 def parse_pod_file(self, file_path, nfvi_role='Sriov'):
375 self.file_path = file_path
379 cfg = self.read_config_file()
380 except IOError as io_error:
381 if io_error.errno != errno.ENOENT:
383 self.file_path = os.path.join(constants.YARDSTICK_ROOT_PATH,
385 cfg = self.read_config_file()
387 nodes.extend([node for node in cfg["nodes"] if str(node["role"]) != nfvi_role])
388 nfvi_host.extend([node for node in cfg["nodes"] if str(node["role"]) == nfvi_role])
390 raise("Node role is other than SRIOV")
392 host_mgmt = {'user': nfvi_host[0]['user'],
393 'ip': str(IPNetwork(nfvi_host[0]['ip']).ip),
394 'password': nfvi_host[0]['password'],
395 'ssh_port': nfvi_host[0].get('ssh_port', 22),
396 'key_filename': nfvi_host[0].get('key_filename')}
398 return [nodes, nfvi_host, host_mgmt]
401 def get_mac_address(end=0x7f):
402 mac = [0x52, 0x54, 0x00,
403 random.randint(0x00, end),
404 random.randint(0x00, 0xff),
405 random.randint(0x00, 0xff)]
406 mac_address = ':'.join('%02x' % x for x in mac)
410 def get_mgmt_ip(connection, mac, cidr, node):
413 while not mgmtip and times:
414 connection.execute("fping -c 1 -g %s > /dev/null 2>&1" % cidr)
415 out = connection.execute("ip neighbor | grep '%s'" % mac)[1]
416 LOG.info("fping -c 1 -g %s > /dev/null 2>&1", cidr)
418 mgmtip = str(out.split(" ")[0]).strip()
419 client = ssh.SSH.from_node(node, overrides={"ip": mgmtip})
423 time.sleep(WAIT_FOR_BOOT) # FixMe: How to find if VM is booted?
428 def wait_for_vnfs_to_start(cls, connection, servers, nodes):
430 vnf = servers[node["name"]]
431 mgmtip = vnf["network_ports"]["mgmt"]["cidr"]
432 ip = cls.get_mgmt_ip(connection, node["mac"], mgmtip, node)
438 class Server(object):
439 """ This class handles geting vnf nodes
443 def build_vnf_interfaces(vnf, ports):
447 for key, vfs in vnf["network_ports"].items():
449 mgmtip = str(IPNetwork(vfs['cidr']).ip)
453 ip = IPNetwork(vf['cidr'])
457 'driver': "%svf" % vf['driver'],
458 'local_mac': vf['mac'],
459 'dpdk_port_num': index,
460 'local_ip': str(ip.ip),
461 'netmask': str(ip.netmask)
466 return mgmtip, interfaces
469 def generate_vnf_instance(cls, flavor, ports, ip, key, vnf, mac):
470 mgmtip, interfaces = cls.build_vnf_interfaces(vnf, ports)
476 "user": flavor.get('user', 'root'),
477 "interfaces": interfaces,
479 # empty IPv6 routing table
481 "name": key, "role": key
485 result['key_filename'] = flavor['key_filename']
490 result['password'] = flavor['password']
497 class OvsDeploy(object):
498 """ This class handles deploy of ovs dpdk
499 Configuration: ovs_dpdk
502 OVS_DEPLOY_SCRIPT = "ovs_deploy.bash"
504 def __init__(self, connection, bin_path, ovs_properties):
505 self.connection = connection
506 self.bin_path = bin_path
507 self.ovs_properties = ovs_properties
509 def prerequisite(self):
510 pkgs = ["git", "build-essential", "pkg-config", "automake",
511 "autotools-dev", "libltdl-dev", "cmake", "libnuma-dev",
513 StandaloneContextHelper.install_req_libs(self.connection, pkgs)
515 def ovs_deploy(self):
516 ovs_deploy = os.path.join(constants.YARDSTICK_ROOT_PATH,
517 "yardstick/resources/scripts/install/",
518 self.OVS_DEPLOY_SCRIPT)
519 if os.path.isfile(ovs_deploy):
521 remote_ovs_deploy = os.path.join(self.bin_path, self.OVS_DEPLOY_SCRIPT)
522 LOG.info(remote_ovs_deploy)
523 self.connection.put(ovs_deploy, remote_ovs_deploy)
525 http_proxy = os.environ.get('http_proxy', '')
526 ovs_details = self.ovs_properties.get("version", {})
527 ovs = ovs_details.get("ovs", "2.6.0")
528 dpdk = ovs_details.get("dpdk", "16.11.1")
530 cmd = "sudo -E %s --ovs='%s' --dpdk='%s' -p='%s'" % (remote_ovs_deploy,
531 ovs, dpdk, http_proxy)
532 exit_status, _, stderr = self.connection.execute(cmd)
534 raise exceptions.OVSDeployError(stderr=stderr)