yardstick/benchmark/contexts/standalone/model.py

   1 # Copyright (c) 2016-2017 Intel Corporation
   2 #
   3 # Licensed under the Apache License, Version 2.0 (the "License");
   4 # you may not use this file except in compliance with the License.
   5 # You may obtain a copy of the License at
   6 #
   7 #      http://www.apache.org/licenses/LICENSE-2.0
   8 #
   9 # Unless required by applicable law or agreed to in writing, software
  10 # distributed under the License is distributed on an "AS IS" BASIS,
  11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12 # See the License for the specific language governing permissions and
  13 # limitations under the License.
  14
  15 import os
  16 import re
  17 import time
  18 import uuid
  19 import random
  20 import logging
  21 import errno
  22
  23 from netaddr import IPNetwork
  24 import xml.etree.ElementTree as ET
  25
  26 from yardstick import ssh
  27 from yardstick.common import constants
  28 from yardstick.common import exceptions
  29 from yardstick.common.yaml_loader import yaml_load
  30 from yardstick.network_services.utils import PciAddress
  31 from yardstick.network_services.helpers.cpu import CpuSysCores
  32 from yardstick.common.utils import write_file
  33
  34 LOG = logging.getLogger(__name__)
  35
  36 VM_TEMPLATE = """
  37 <domain type="kvm">
  38   <name>{vm_name}</name>
  39   <uuid>{random_uuid}</uuid>
  40   <memory unit="MB">{memory}</memory>
  41   <currentMemory unit="MB">{memory}</currentMemory>
  42   <memoryBacking>
  43     <hugepages />
  44   </memoryBacking>
  45   <vcpu cpuset='{cpuset}'>{vcpu}</vcpu>
  46  {cputune}
  47   <os>
  48     <type arch="x86_64" machine="pc-i440fx-utopic">hvm</type>
  49     <boot dev="hd" />
  50   </os>
  51   <features>
  52     <acpi />
  53     <apic />
  54     <pae />
  55   </features>
  56   <cpu mode='host-passthrough'>
  57     <topology cores="{cpu}" sockets="{socket}" threads="{threads}" />
  58     <numa>
  59        <cell id='0' cpus='{numa_cpus}' memory='{memory}' unit='MB' memAccess='shared'/>
  60     </numa>
  61   </cpu>
  62   <clock offset="utc">
  63     <timer name="rtc" tickpolicy="catchup" />
  64     <timer name="pit" tickpolicy="delay" />
  65     <timer name="hpet" present="no" />
  66   </clock>
  67   <on_poweroff>destroy</on_poweroff>
  68   <on_reboot>restart</on_reboot>
  69   <on_crash>restart</on_crash>
  70   <devices>
  71     <emulator>/usr/bin/kvm-spice</emulator>
  72     <disk device="disk" type="file">
  73       <driver name="qemu" type="qcow2" />
  74       <source file="{vm_image}"/>
  75       <target bus="virtio" dev="vda" />
  76     </disk>
  77     <graphics autoport="yes" listen="0.0.0.0" port="-1" type="vnc" />
  78     <interface type="bridge">
  79       <mac address='{mac_addr}'/>
  80       <source bridge="br-int" />
  81       <model type='virtio'/>
  82     </interface>
  83     <serial type='pty'>
  84       <target port='0'/>
  85     </serial>
  86     <console type='pty'>
  87       <target type='serial' port='0'/>
  88     </console>
  89   </devices>
  90 </domain>
  91 """
  92 WAIT_FOR_BOOT = 30
  93
  94
  95 class Libvirt(object):
  96     """ This class handles all the libvirt updates to lauch VM
  97     """
  98
  99     @staticmethod
 100     def check_if_vm_exists_and_delete(vm_name, connection):
 101         cmd_template = "virsh list --name | grep -i %s"
 102         status = connection.execute(cmd_template % vm_name)[0]
 103         if status == 0:
 104             LOG.info("VM '%s' is already present... destroying", vm_name)
 105             connection.execute("virsh destroy %s" % vm_name)
 106
 107     @staticmethod
 108     def virsh_create_vm(connection, cfg):
 109         err = connection.execute("virsh create %s" % cfg)[0]
 110         LOG.info("VM create status: %s", err)
 111
 112     @staticmethod
 113     def virsh_destroy_vm(vm_name, connection):
 114         connection.execute("virsh destroy %s" % vm_name)
 115
 116     @staticmethod
 117     def _add_interface_address(interface, pci_address):
 118         """Add a PCI 'address' XML node
 119
 120         <address type='pci' domain='0x0000' bus='0x00' slot='0x08'
 121          function='0x0'/>
 122
 123         Refence: https://software.intel.com/en-us/articles/
 124                  configure-sr-iov-network-virtual-functions-in-linux-kvm
 125         """
 126         vm_pci = ET.SubElement(interface, 'address')
 127         vm_pci.set('type', 'pci')
 128         vm_pci.set('domain', '0x{}'.format(pci_address.domain))
 129         vm_pci.set('bus', '0x{}'.format(pci_address.bus))
 130         vm_pci.set('slot', '0x{}'.format(pci_address.slot))
 131         vm_pci.set('function', '0x{}'.format(pci_address.function))
 132         return vm_pci
 133
 134     @classmethod
 135     def add_ovs_interface(cls, vpath, port_num, vpci, vports_mac, xml):
 136         """Add a DPDK OVS 'interface' XML node in 'devices' node
 137
 138         <devices>
 139             <interface type='vhostuser'>
 140                 <mac address='00:00:00:00:00:01'/>
 141                 <source type='unix' path='/usr/local/var/run/openvswitch/
 142                  dpdkvhostuser0' mode='client'/>
 143                 <model type='virtio'/>
 144                 <driver queues='4'>
 145                     <host mrg_rxbuf='off'/>
 146                 </driver>
 147                 <address type='pci' domain='0x0000' bus='0x00' slot='0x03'
 148                  function='0x0'/>
 149             </interface>
 150             ...
 151         </devices>
 152
 153         Reference: http://docs.openvswitch.org/en/latest/topics/dpdk/
 154                    vhost-user/
 155         """
 156
 157         vhost_path = ('{0}/var/run/openvswitch/dpdkvhostuser{1}'.
 158                       format(vpath, port_num))
 159         root = ET.parse(xml)
 160         pci_address = PciAddress(vpci.strip())
 161         device = root.find('devices')
 162
 163         interface = ET.SubElement(device, 'interface')
 164         interface.set('type', 'vhostuser')
 165         mac = ET.SubElement(interface, 'mac')
 166         mac.set('address', vports_mac)
 167
 168         source = ET.SubElement(interface, 'source')
 169         source.set('type', 'unix')
 170         source.set('path', vhost_path)
 171         source.set('mode', 'client')
 172
 173         model = ET.SubElement(interface, 'model')
 174         model.set('type', 'virtio')
 175
 176         driver = ET.SubElement(interface, 'driver')
 177         driver.set('queues', '4')
 178
 179         host = ET.SubElement(driver, 'host')
 180         host.set('mrg_rxbuf', 'off')
 181
 182         cls._add_interface_address(interface, pci_address)
 183
 184         root.write(xml)
 185
 186     @classmethod
 187     def add_sriov_interfaces(cls, vm_pci, vf_pci, vf_mac, xml):
 188         """Add a SR-IOV 'interface' XML node in 'devices' node
 189
 190         <devices>
 191            <interface type='hostdev' managed='yes'>
 192              <source>
 193                <address type='pci' domain='0x0000' bus='0x00' slot='0x03'
 194                 function='0x0'/>
 195              </source>
 196              <mac address='52:54:00:6d:90:02'>
 197              <address type='pci' domain='0x0000' bus='0x02' slot='0x04'
 198               function='0x1'/>
 199            </interface>
 200            ...
 201          </devices>
 202
 203         Reference: https://access.redhat.com/documentation/en-us/
 204             red_hat_enterprise_linux/6/html/
 205             virtualization_host_configuration_and_guest_installation_guide/
 206             sect-virtualization_host_configuration_and_guest_installation_guide
 207             -sr_iov-how_sr_iov_libvirt_works
 208         """
 209
 210         root = ET.parse(xml)
 211         device = root.find('devices')
 212
 213         interface = ET.SubElement(device, 'interface')
 214         interface.set('managed', 'yes')
 215         interface.set('type', 'hostdev')
 216
 217         mac = ET.SubElement(interface, 'mac')
 218         mac.set('address', vf_mac)
 219
 220         source = ET.SubElement(interface, 'source')
 221         pci_address = PciAddress(vf_pci.strip())
 222         cls._add_interface_address(source, pci_address)
 223
 224         pci_vm_address = PciAddress(vm_pci.strip())
 225         cls._add_interface_address(interface, pci_vm_address)
 226
 227         root.write(xml)
 228
 229     @staticmethod
 230     def create_snapshot_qemu(connection, index, vm_image):
 231         # build snapshot image
 232         image = "/var/lib/libvirt/images/%s.qcow2" % index
 233         connection.execute("rm %s" % image)
 234         qemu_template = "qemu-img create -f qcow2 -o backing_file=%s %s"
 235         connection.execute(qemu_template % (vm_image, image))
 236
 237         return image
 238
 239     @classmethod
 240     def build_vm_xml(cls, connection, flavor, cfg, vm_name, index):
 241         memory = flavor.get('ram', '4096')
 242         extra_spec = flavor.get('extra_specs', {})
 243         cpu = extra_spec.get('hw:cpu_cores', '2')
 244         socket = extra_spec.get('hw:cpu_sockets', '1')
 245         threads = extra_spec.get('hw:cpu_threads', '2')
 246         vcpu = int(cpu) * int(threads)
 247         numa_cpus = '0-%s' % (vcpu - 1)
 248         hw_socket = flavor.get('hw_socket', '0')
 249         cpuset = Libvirt.pin_vcpu_for_perf(connection, hw_socket)
 250
 251         cputune = extra_spec.get('cputune', '')
 252         mac = StandaloneContextHelper.get_mac_address(0x00)
 253         image = cls.create_snapshot_qemu(connection, index,
 254                                          flavor.get("images", None))
 255         vm_xml = VM_TEMPLATE.format(
 256             vm_name=vm_name,
 257             random_uuid=uuid.uuid4(),
 258             mac_addr=mac,
 259             memory=memory, vcpu=vcpu, cpu=cpu,
 260             numa_cpus=numa_cpus,
 261             socket=socket, threads=threads,
 262             vm_image=image, cpuset=cpuset, cputune=cputune)
 263
 264         write_file(cfg, vm_xml)
 265
 266         return [vcpu, mac]
 267
 268     @staticmethod
 269     def update_interrupts_hugepages_perf(connection):
 270         connection.execute("echo 1 > /sys/module/kvm/parameters/allow_unsafe_assigned_interrupts")
 271         connection.execute("echo never > /sys/kernel/mm/transparent_hugepage/enabled")
 272
 273     @classmethod
 274     def pin_vcpu_for_perf(cls, connection, socket='0'):
 275         threads = ""
 276         sys_obj = CpuSysCores(connection)
 277         soc_cpu = sys_obj.get_core_socket()
 278         sys_cpu = int(soc_cpu["cores_per_socket"])
 279         socket = str(socket)
 280         cores = "%s-%s" % (soc_cpu[socket][0], soc_cpu[socket][sys_cpu - 1])
 281         if int(soc_cpu["thread_per_core"]) > 1:
 282             threads = "%s-%s" % (soc_cpu[socket][sys_cpu], soc_cpu[socket][-1])
 283         cpuset = "%s,%s" % (cores, threads)
 284         return cpuset
 285
 286
 287 class StandaloneContextHelper(object):
 288     """ This class handles all the common code for standalone
 289     """
 290     def __init__(self):
 291         self.file_path = None
 292         super(StandaloneContextHelper, self).__init__()
 293
 294     @staticmethod
 295     def install_req_libs(connection, extra_pkgs=None):
 296         extra_pkgs = extra_pkgs or []
 297         pkgs = ["qemu-kvm", "libvirt-bin", "bridge-utils", "numactl", "fping"]
 298         pkgs.extend(extra_pkgs)
 299         cmd_template = "dpkg-query -W --showformat='${Status}\\n' \"%s\"|grep 'ok installed'"
 300         for pkg in pkgs:
 301             if connection.execute(cmd_template % pkg)[0]:
 302                 connection.execute("apt-get update")
 303                 connection.execute("apt-get -y install %s" % pkg)
 304
 305     @staticmethod
 306     def get_kernel_module(connection, pci, driver):
 307         if not driver:
 308             out = connection.execute("lspci -k -s %s" % pci)[1]
 309             driver = out.split("Kernel modules:").pop().strip()
 310         return driver
 311
 312     @classmethod
 313     def get_nic_details(cls, connection, networks, dpdk_devbind):
 314         for key, ports in networks.items():
 315             if key == "mgmt":
 316                 continue
 317
 318             phy_ports = ports['phy_port']
 319             phy_driver = ports.get('phy_driver', None)
 320             driver = cls.get_kernel_module(connection, phy_ports, phy_driver)
 321
 322             # Make sure that ports are bound to kernel drivers e.g. i40e/ixgbe
 323             bind_cmd = "{dpdk_devbind} --force -b {driver} {port}"
 324             lshw_cmd = "lshw -c network -businfo | grep '{port}'"
 325             link_show_cmd = "ip -s link show {interface}"
 326
 327             cmd = bind_cmd.format(dpdk_devbind=dpdk_devbind,
 328                                   driver=driver, port=ports['phy_port'])
 329             connection.execute(cmd)
 330
 331             out = connection.execute(lshw_cmd.format(port=phy_ports))[1]
 332             interface = out.split()[1]
 333
 334             connection.execute(link_show_cmd.format(interface=interface))
 335
 336             ports.update({
 337                 'interface': str(interface),
 338                 'driver': driver
 339             })
 340         LOG.info(networks)
 341
 342         return networks
 343
 344     @staticmethod
 345     def get_virtual_devices(connection, pci):
 346         cmd = "cat /sys/bus/pci/devices/{0}/virtfn0/uevent"
 347         output = connection.execute(cmd.format(pci))[1]
 348
 349         pattern = "PCI_SLOT_NAME=({})".format(PciAddress.PCI_PATTERN_STR)
 350         m = re.search(pattern, output, re.MULTILINE)
 351
 352         pf_vfs = {}
 353         if m:
 354             pf_vfs = {pci: m.group(1).rstrip()}
 355
 356         LOG.info("pf_vfs:\n%s", pf_vfs)
 357
 358         return pf_vfs
 359
 360     def read_config_file(self):
 361         """Read from config file"""
 362
 363         with open(self.file_path) as stream:
 364             LOG.info("Parsing pod file: %s", self.file_path)
 365             cfg = yaml_load(stream)
 366         return cfg
 367
 368     def parse_pod_file(self, file_path, nfvi_role='Sriov'):
 369         self.file_path = file_path
 370         nodes = []
 371         nfvi_host = []
 372         try:
 373             cfg = self.read_config_file()
 374         except IOError as io_error:
 375             if io_error.errno != errno.ENOENT:
 376                 raise
 377             self.file_path = os.path.join(constants.YARDSTICK_ROOT_PATH,
 378                                           file_path)
 379             cfg = self.read_config_file()
 380
 381         nodes.extend([node for node in cfg["nodes"] if str(node["role"]) != nfvi_role])
 382         nfvi_host.extend([node for node in cfg["nodes"] if str(node["role"]) == nfvi_role])
 383         if not nfvi_host:
 384             raise("Node role is other than SRIOV")
 385
 386         host_mgmt = {'user': nfvi_host[0]['user'],
 387                      'ip': str(IPNetwork(nfvi_host[0]['ip']).ip),
 388                      'password': nfvi_host[0]['password'],
 389                      'ssh_port': nfvi_host[0].get('ssh_port', 22),
 390                      'key_filename': nfvi_host[0].get('key_filename')}
 391
 392         return [nodes, nfvi_host, host_mgmt]
 393
 394     @staticmethod
 395     def get_mac_address(end=0x7f):
 396         mac = [0x52, 0x54, 0x00,
 397                random.randint(0x00, end),
 398                random.randint(0x00, 0xff),
 399                random.randint(0x00, 0xff)]
 400         mac_address = ':'.join('%02x' % x for x in mac)
 401         return mac_address
 402
 403     @staticmethod
 404     def get_mgmt_ip(connection, mac, cidr, node):
 405         mgmtip = None
 406         times = 10
 407         while not mgmtip and times:
 408             connection.execute("fping -c 1 -g %s > /dev/null 2>&1" % cidr)
 409             out = connection.execute("ip neighbor | grep '%s'" % mac)[1]
 410             LOG.info("fping -c 1 -g %s > /dev/null 2>&1", cidr)
 411             if out.strip():
 412                 mgmtip = str(out.split(" ")[0]).strip()
 413                 client = ssh.SSH.from_node(node, overrides={"ip": mgmtip})
 414                 client.wait()
 415                 break
 416
 417             time.sleep(WAIT_FOR_BOOT)  # FixMe: How to find if VM is booted?
 418             times = times - 1
 419         return mgmtip
 420
 421     @classmethod
 422     def wait_for_vnfs_to_start(cls, connection, servers, nodes):
 423         for node in nodes:
 424             vnf = servers[node["name"]]
 425             mgmtip = vnf["network_ports"]["mgmt"]["cidr"]
 426             ip = cls.get_mgmt_ip(connection, node["mac"], mgmtip, node)
 427             if ip:
 428                 node["ip"] = ip
 429         return nodes
 430
 431
 432 class Server(object):
 433     """ This class handles geting vnf nodes
 434     """
 435
 436     @staticmethod
 437     def build_vnf_interfaces(vnf, ports):
 438         interfaces = {}
 439         index = 0
 440
 441         for key, vfs in vnf["network_ports"].items():
 442             if key == "mgmt":
 443                 mgmtip = str(IPNetwork(vfs['cidr']).ip)
 444                 continue
 445
 446             vf = ports[vfs[0]]
 447             ip = IPNetwork(vf['cidr'])
 448             interfaces.update({
 449                 key: {
 450                     'vpci': vf['vpci'],
 451                     'driver': "%svf" % vf['driver'],
 452                     'local_mac': vf['mac'],
 453                     'dpdk_port_num': index,
 454                     'local_ip': str(ip.ip),
 455                     'netmask': str(ip.netmask)
 456                     },
 457             })
 458             index = index + 1
 459
 460         return mgmtip, interfaces
 461
 462     @classmethod
 463     def generate_vnf_instance(cls, flavor, ports, ip, key, vnf, mac):
 464         mgmtip, interfaces = cls.build_vnf_interfaces(vnf, ports)
 465
 466         result = {
 467             "ip": mgmtip,
 468             "mac": mac,
 469             "host": ip,
 470             "user": flavor.get('user', 'root'),
 471             "interfaces": interfaces,
 472             "routing_table": [],
 473             # empty IPv6 routing table
 474             "nd_route_tbl": [],
 475             "name": key, "role": key
 476         }
 477
 478         try:
 479             result['key_filename'] = flavor['key_filename']
 480         except KeyError:
 481             pass
 482
 483         try:
 484             result['password'] = flavor['password']
 485         except KeyError:
 486             pass
 487         LOG.info(result)
 488         return result
 489
 490
 491 class OvsDeploy(object):
 492     """ This class handles deploy of ovs dpdk
 493     Configuration: ovs_dpdk
 494     """
 495
 496     OVS_DEPLOY_SCRIPT = "ovs_deploy.bash"
 497
 498     def __init__(self, connection, bin_path, ovs_properties):
 499         self.connection = connection
 500         self.bin_path = bin_path
 501         self.ovs_properties = ovs_properties
 502
 503     def prerequisite(self):
 504         pkgs = ["git", "build-essential", "pkg-config", "automake",
 505                 "autotools-dev", "libltdl-dev", "cmake", "libnuma-dev",
 506                 "libpcap-dev"]
 507         StandaloneContextHelper.install_req_libs(self.connection, pkgs)
 508
 509     def ovs_deploy(self):
 510         ovs_deploy = os.path.join(constants.YARDSTICK_ROOT_PATH,
 511                                   "yardstick/resources/scripts/install/",
 512                                   self.OVS_DEPLOY_SCRIPT)
 513         if os.path.isfile(ovs_deploy):
 514             self.prerequisite()
 515             remote_ovs_deploy = os.path.join(self.bin_path, self.OVS_DEPLOY_SCRIPT)
 516             LOG.info(remote_ovs_deploy)
 517             self.connection.put(ovs_deploy, remote_ovs_deploy)
 518
 519             http_proxy = os.environ.get('http_proxy', '')
 520             ovs_details = self.ovs_properties.get("version", {})
 521             ovs = ovs_details.get("ovs", "2.6.0")
 522             dpdk = ovs_details.get("dpdk", "16.11.1")
 523
 524             cmd = "sudo -E %s --ovs='%s' --dpdk='%s' -p='%s'" % (remote_ovs_deploy,
 525                                                                  ovs, dpdk, http_proxy)
 526             exit_status, _, stderr = self.connection.execute(cmd)
 527             if exit_status:
 528                 raise exceptions.OVSDeployError(stderr=stderr)