yardstick/benchmark/contexts/standalone/model.py

   1 # Copyright (c) 2016-2017 Intel Corporation
   2 #
   3 # Licensed under the Apache License, Version 2.0 (the "License");
   4 # you may not use this file except in compliance with the License.
   5 # You may obtain a copy of the License at
   6 #
   7 #      http://www.apache.org/licenses/LICENSE-2.0
   8 #
   9 # Unless required by applicable law or agreed to in writing, software
  10 # distributed under the License is distributed on an "AS IS" BASIS,
  11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12 # See the License for the specific language governing permissions and
  13 # limitations under the License.
  14
  15 import os
  16 import re
  17 import time
  18 import uuid
  19 import random
  20 import logging
  21 import errno
  22
  23 from netaddr import IPNetwork
  24 import xml.etree.ElementTree as ET
  25
  26 from yardstick import ssh
  27 from yardstick.common import constants
  28 from yardstick.common import exceptions
  29 from yardstick.common.yaml_loader import yaml_load
  30 from yardstick.network_services.utils import PciAddress
  31 from yardstick.network_services.helpers.cpu import CpuSysCores
  32
  33
  34 LOG = logging.getLogger(__name__)
  35
  36 VM_TEMPLATE = """
  37 <domain type="kvm">
  38   <name>{vm_name}</name>
  39   <uuid>{random_uuid}</uuid>
  40   <memory unit="MB">{memory}</memory>
  41   <currentMemory unit="MB">{memory}</currentMemory>
  42   <memoryBacking>
  43     <hugepages />
  44   </memoryBacking>
  45   <vcpu cpuset='{cpuset}'>{vcpu}</vcpu>
  46  {cputune}
  47   <os>
  48     <type arch="x86_64" machine="pc-i440fx-utopic">hvm</type>
  49     <boot dev="hd" />
  50   </os>
  51   <features>
  52     <acpi />
  53     <apic />
  54     <pae />
  55   </features>
  56   <cpu mode='host-passthrough'>
  57     <topology cores="{cpu}" sockets="{socket}" threads="{threads}" />
  58     <numa>
  59        <cell id='0' cpus='{numa_cpus}' memory='{memory}' unit='MB' memAccess='shared'/>
  60     </numa>
  61   </cpu>
  62   <clock offset="utc">
  63     <timer name="rtc" tickpolicy="catchup" />
  64     <timer name="pit" tickpolicy="delay" />
  65     <timer name="hpet" present="no" />
  66   </clock>
  67   <on_poweroff>destroy</on_poweroff>
  68   <on_reboot>restart</on_reboot>
  69   <on_crash>restart</on_crash>
  70   <devices>
  71     <emulator>/usr/bin/kvm-spice</emulator>
  72     <disk device="disk" type="file">
  73       <driver name="qemu" type="qcow2" />
  74       <source file="{vm_image}"/>
  75       <target bus="virtio" dev="vda" />
  76     </disk>
  77     <graphics autoport="yes" listen="0.0.0.0" port="-1" type="vnc" />
  78     <interface type="bridge">
  79       <mac address='{mac_addr}'/>
  80       <source bridge="br-int" />
  81       <model type='virtio'/>
  82     </interface>
  83     <serial type='pty'>
  84       <target port='0'/>
  85     </serial>
  86     <console type='pty'>
  87       <target type='serial' port='0'/>
  88     </console>
  89   </devices>
  90 </domain>
  91 """
  92 WAIT_FOR_BOOT = 30
  93
  94
  95 class Libvirt(object):
  96     """ This class handles all the libvirt updates to lauch VM
  97     """
  98
  99     @staticmethod
 100     def check_if_vm_exists_and_delete(vm_name, connection):
 101         cmd_template = "virsh list --name | grep -i %s"
 102         status = connection.execute(cmd_template % vm_name)[0]
 103         if status == 0:
 104             LOG.info("VM '%s' is already present... destroying", vm_name)
 105             connection.execute("virsh destroy %s" % vm_name)
 106
 107     @staticmethod
 108     def virsh_create_vm(connection, cfg):
 109         err = connection.execute("virsh create %s" % cfg)[0]
 110         LOG.info("VM create status: %s", err)
 111
 112     @staticmethod
 113     def virsh_destroy_vm(vm_name, connection):
 114         connection.execute("virsh destroy %s" % vm_name)
 115
 116     @staticmethod
 117     def _add_interface_address(interface, pci_address):
 118         """Add a PCI 'address' XML node
 119
 120         <address type='pci' domain='0x0000' bus='0x00' slot='0x08'
 121          function='0x0'/>
 122
 123         Refence: https://software.intel.com/en-us/articles/
 124                  configure-sr-iov-network-virtual-functions-in-linux-kvm
 125         """
 126         vm_pci = ET.SubElement(interface, 'address')
 127         vm_pci.set('type', 'pci')
 128         vm_pci.set('domain', '0x{}'.format(pci_address.domain))
 129         vm_pci.set('bus', '0x{}'.format(pci_address.bus))
 130         vm_pci.set('slot', '0x{}'.format(pci_address.slot))
 131         vm_pci.set('function', '0x{}'.format(pci_address.function))
 132         return vm_pci
 133
 134     @classmethod
 135     def add_ovs_interface(cls, vpath, port_num, vpci, vports_mac, xml_str):
 136         """Add a DPDK OVS 'interface' XML node in 'devices' node
 137
 138         <devices>
 139             <interface type='vhostuser'>
 140                 <mac address='00:00:00:00:00:01'/>
 141                 <source type='unix' path='/usr/local/var/run/openvswitch/
 142                  dpdkvhostuser0' mode='client'/>
 143                 <model type='virtio'/>
 144                 <driver queues='4'>
 145                     <host mrg_rxbuf='off'/>
 146                 </driver>
 147                 <address type='pci' domain='0x0000' bus='0x00' slot='0x03'
 148                  function='0x0'/>
 149             </interface>
 150             ...
 151         </devices>
 152
 153         Reference: http://docs.openvswitch.org/en/latest/topics/dpdk/
 154                    vhost-user/
 155         """
 156
 157         vhost_path = ('{0}/var/run/openvswitch/dpdkvhostuser{1}'.
 158                       format(vpath, port_num))
 159         root = ET.fromstring(xml_str)
 160         pci_address = PciAddress(vpci.strip())
 161         device = root.find('devices')
 162
 163         interface = ET.SubElement(device, 'interface')
 164         interface.set('type', 'vhostuser')
 165         mac = ET.SubElement(interface, 'mac')
 166         mac.set('address', vports_mac)
 167
 168         source = ET.SubElement(interface, 'source')
 169         source.set('type', 'unix')
 170         source.set('path', vhost_path)
 171         source.set('mode', 'client')
 172
 173         model = ET.SubElement(interface, 'model')
 174         model.set('type', 'virtio')
 175
 176         driver = ET.SubElement(interface, 'driver')
 177         driver.set('queues', '4')
 178
 179         host = ET.SubElement(driver, 'host')
 180         host.set('mrg_rxbuf', 'off')
 181
 182         cls._add_interface_address(interface, pci_address)
 183
 184         return ET.tostring(root)
 185
 186     @classmethod
 187     def add_sriov_interfaces(cls, vm_pci, vf_pci, vf_mac, xml_str):
 188         """Add a SR-IOV 'interface' XML node in 'devices' node
 189
 190         <devices>
 191            <interface type='hostdev' managed='yes'>
 192              <source>
 193                <address type='pci' domain='0x0000' bus='0x00' slot='0x03'
 194                 function='0x0'/>
 195              </source>
 196              <mac address='52:54:00:6d:90:02'>
 197              <address type='pci' domain='0x0000' bus='0x02' slot='0x04'
 198               function='0x1'/>
 199            </interface>
 200            ...
 201          </devices>
 202
 203         Reference: https://access.redhat.com/documentation/en-us/
 204             red_hat_enterprise_linux/6/html/
 205             virtualization_host_configuration_and_guest_installation_guide/
 206             sect-virtualization_host_configuration_and_guest_installation_guide
 207             -sr_iov-how_sr_iov_libvirt_works
 208         """
 209
 210         root = ET.fromstring(xml_str)
 211         device = root.find('devices')
 212
 213         interface = ET.SubElement(device, 'interface')
 214         interface.set('managed', 'yes')
 215         interface.set('type', 'hostdev')
 216
 217         mac = ET.SubElement(interface, 'mac')
 218         mac.set('address', vf_mac)
 219
 220         source = ET.SubElement(interface, 'source')
 221         pci_address = PciAddress(vf_pci.strip())
 222         cls._add_interface_address(source, pci_address)
 223
 224         pci_vm_address = PciAddress(vm_pci.strip())
 225         cls._add_interface_address(interface, pci_vm_address)
 226
 227         return ET.tostring(root)
 228
 229     @staticmethod
 230     def create_snapshot_qemu(connection, index, vm_image):
 231         # build snapshot image
 232         image = "/var/lib/libvirt/images/%s.qcow2" % index
 233         connection.execute("rm %s" % image)
 234         qemu_template = "qemu-img create -f qcow2 -o backing_file=%s %s"
 235         connection.execute(qemu_template % (vm_image, image))
 236
 237         return image
 238
 239     @classmethod
 240     def build_vm_xml(cls, connection, flavor, vm_name, index):
 241         """Build the XML from the configuration parameters"""
 242         memory = flavor.get('ram', '4096')
 243         extra_spec = flavor.get('extra_specs', {})
 244         cpu = extra_spec.get('hw:cpu_cores', '2')
 245         socket = extra_spec.get('hw:cpu_sockets', '1')
 246         threads = extra_spec.get('hw:cpu_threads', '2')
 247         vcpu = int(cpu) * int(threads)
 248         numa_cpus = '0-%s' % (vcpu - 1)
 249         hw_socket = flavor.get('hw_socket', '0')
 250         cpuset = Libvirt.pin_vcpu_for_perf(connection, hw_socket)
 251
 252         cputune = extra_spec.get('cputune', '')
 253         mac = StandaloneContextHelper.get_mac_address(0x00)
 254         image = cls.create_snapshot_qemu(connection, index,
 255                                          flavor.get("images", None))
 256         vm_xml = VM_TEMPLATE.format(
 257             vm_name=vm_name,
 258             random_uuid=uuid.uuid4(),
 259             mac_addr=mac,
 260             memory=memory, vcpu=vcpu, cpu=cpu,
 261             numa_cpus=numa_cpus,
 262             socket=socket, threads=threads,
 263             vm_image=image, cpuset=cpuset, cputune=cputune)
 264
 265         return vm_xml, mac
 266
 267     @staticmethod
 268     def update_interrupts_hugepages_perf(connection):
 269         connection.execute("echo 1 > /sys/module/kvm/parameters/allow_unsafe_assigned_interrupts")
 270         connection.execute("echo never > /sys/kernel/mm/transparent_hugepage/enabled")
 271
 272     @classmethod
 273     def pin_vcpu_for_perf(cls, connection, socket='0'):
 274         threads = ""
 275         sys_obj = CpuSysCores(connection)
 276         soc_cpu = sys_obj.get_core_socket()
 277         sys_cpu = int(soc_cpu["cores_per_socket"])
 278         socket = str(socket)
 279         cores = "%s-%s" % (soc_cpu[socket][0], soc_cpu[socket][sys_cpu - 1])
 280         if int(soc_cpu["thread_per_core"]) > 1:
 281             threads = "%s-%s" % (soc_cpu[socket][sys_cpu], soc_cpu[socket][-1])
 282         cpuset = "%s,%s" % (cores, threads)
 283         return cpuset
 284
 285     @classmethod
 286     def write_file(cls, file_name, xml_str):
 287         """Dump a XML string to a file"""
 288         root = ET.fromstring(xml_str)
 289         et = ET.ElementTree(element=root)
 290         et.write(file_name, encoding='utf-8', method='xml')
 291
 292
 293 class StandaloneContextHelper(object):
 294     """ This class handles all the common code for standalone
 295     """
 296     def __init__(self):
 297         self.file_path = None
 298         super(StandaloneContextHelper, self).__init__()
 299
 300     @staticmethod
 301     def install_req_libs(connection, extra_pkgs=None):
 302         extra_pkgs = extra_pkgs or []
 303         pkgs = ["qemu-kvm", "libvirt-bin", "bridge-utils", "numactl", "fping"]
 304         pkgs.extend(extra_pkgs)
 305         cmd_template = "dpkg-query -W --showformat='${Status}\\n' \"%s\"|grep 'ok installed'"
 306         for pkg in pkgs:
 307             if connection.execute(cmd_template % pkg)[0]:
 308                 connection.execute("apt-get update")
 309                 connection.execute("apt-get -y install %s" % pkg)
 310
 311     @staticmethod
 312     def get_kernel_module(connection, pci, driver):
 313         if not driver:
 314             out = connection.execute("lspci -k -s %s" % pci)[1]
 315             driver = out.split("Kernel modules:").pop().strip()
 316         return driver
 317
 318     @classmethod
 319     def get_nic_details(cls, connection, networks, dpdk_devbind):
 320         for key, ports in networks.items():
 321             if key == "mgmt":
 322                 continue
 323
 324             phy_ports = ports['phy_port']
 325             phy_driver = ports.get('phy_driver', None)
 326             driver = cls.get_kernel_module(connection, phy_ports, phy_driver)
 327
 328             # Make sure that ports are bound to kernel drivers e.g. i40e/ixgbe
 329             bind_cmd = "{dpdk_devbind} --force -b {driver} {port}"
 330             lshw_cmd = "lshw -c network -businfo | grep '{port}'"
 331             link_show_cmd = "ip -s link show {interface}"
 332
 333             cmd = bind_cmd.format(dpdk_devbind=dpdk_devbind,
 334                                   driver=driver, port=ports['phy_port'])
 335             connection.execute(cmd)
 336
 337             out = connection.execute(lshw_cmd.format(port=phy_ports))[1]
 338             interface = out.split()[1]
 339
 340             connection.execute(link_show_cmd.format(interface=interface))
 341
 342             ports.update({
 343                 'interface': str(interface),
 344                 'driver': driver
 345             })
 346         LOG.info(networks)
 347
 348         return networks
 349
 350     @staticmethod
 351     def get_virtual_devices(connection, pci):
 352         cmd = "cat /sys/bus/pci/devices/{0}/virtfn0/uevent"
 353         output = connection.execute(cmd.format(pci))[1]
 354
 355         pattern = "PCI_SLOT_NAME=({})".format(PciAddress.PCI_PATTERN_STR)
 356         m = re.search(pattern, output, re.MULTILINE)
 357
 358         pf_vfs = {}
 359         if m:
 360             pf_vfs = {pci: m.group(1).rstrip()}
 361
 362         LOG.info("pf_vfs:\n%s", pf_vfs)
 363
 364         return pf_vfs
 365
 366     def read_config_file(self):
 367         """Read from config file"""
 368
 369         with open(self.file_path) as stream:
 370             LOG.info("Parsing pod file: %s", self.file_path)
 371             cfg = yaml_load(stream)
 372         return cfg
 373
 374     def parse_pod_file(self, file_path, nfvi_role='Sriov'):
 375         self.file_path = file_path
 376         nodes = []
 377         nfvi_host = []
 378         try:
 379             cfg = self.read_config_file()
 380         except IOError as io_error:
 381             if io_error.errno != errno.ENOENT:
 382                 raise
 383             self.file_path = os.path.join(constants.YARDSTICK_ROOT_PATH,
 384                                           file_path)
 385             cfg = self.read_config_file()
 386
 387         nodes.extend([node for node in cfg["nodes"] if str(node["role"]) != nfvi_role])
 388         nfvi_host.extend([node for node in cfg["nodes"] if str(node["role"]) == nfvi_role])
 389         if not nfvi_host:
 390             raise("Node role is other than SRIOV")
 391
 392         host_mgmt = {'user': nfvi_host[0]['user'],
 393                      'ip': str(IPNetwork(nfvi_host[0]['ip']).ip),
 394                      'password': nfvi_host[0]['password'],
 395                      'ssh_port': nfvi_host[0].get('ssh_port', 22),
 396                      'key_filename': nfvi_host[0].get('key_filename')}
 397
 398         return [nodes, nfvi_host, host_mgmt]
 399
 400     @staticmethod
 401     def get_mac_address(end=0x7f):
 402         mac = [0x52, 0x54, 0x00,
 403                random.randint(0x00, end),
 404                random.randint(0x00, 0xff),
 405                random.randint(0x00, 0xff)]
 406         mac_address = ':'.join('%02x' % x for x in mac)
 407         return mac_address
 408
 409     @staticmethod
 410     def get_mgmt_ip(connection, mac, cidr, node):
 411         mgmtip = None
 412         times = 10
 413         while not mgmtip and times:
 414             connection.execute("fping -c 1 -g %s > /dev/null 2>&1" % cidr)
 415             out = connection.execute("ip neighbor | grep '%s'" % mac)[1]
 416             LOG.info("fping -c 1 -g %s > /dev/null 2>&1", cidr)
 417             if out.strip():
 418                 mgmtip = str(out.split(" ")[0]).strip()
 419                 client = ssh.SSH.from_node(node, overrides={"ip": mgmtip})
 420                 client.wait()
 421                 break
 422
 423             time.sleep(WAIT_FOR_BOOT)  # FixMe: How to find if VM is booted?
 424             times = times - 1
 425         return mgmtip
 426
 427     @classmethod
 428     def wait_for_vnfs_to_start(cls, connection, servers, nodes):
 429         for node in nodes:
 430             vnf = servers[node["name"]]
 431             mgmtip = vnf["network_ports"]["mgmt"]["cidr"]
 432             ip = cls.get_mgmt_ip(connection, node["mac"], mgmtip, node)
 433             if ip:
 434                 node["ip"] = ip
 435         return nodes
 436
 437
 438 class Server(object):
 439     """ This class handles geting vnf nodes
 440     """
 441
 442     @staticmethod
 443     def build_vnf_interfaces(vnf, ports):
 444         interfaces = {}
 445         index = 0
 446
 447         for key, vfs in vnf["network_ports"].items():
 448             if key == "mgmt":
 449                 mgmtip = str(IPNetwork(vfs['cidr']).ip)
 450                 continue
 451
 452             vf = ports[vfs[0]]
 453             ip = IPNetwork(vf['cidr'])
 454             interfaces.update({
 455                 key: {
 456                     'vpci': vf['vpci'],
 457                     'driver': "%svf" % vf['driver'],
 458                     'local_mac': vf['mac'],
 459                     'dpdk_port_num': index,
 460                     'local_ip': str(ip.ip),
 461                     'netmask': str(ip.netmask)
 462                     },
 463             })
 464             index = index + 1
 465
 466         return mgmtip, interfaces
 467
 468     @classmethod
 469     def generate_vnf_instance(cls, flavor, ports, ip, key, vnf, mac):
 470         mgmtip, interfaces = cls.build_vnf_interfaces(vnf, ports)
 471
 472         result = {
 473             "ip": mgmtip,
 474             "mac": mac,
 475             "host": ip,
 476             "user": flavor.get('user', 'root'),
 477             "interfaces": interfaces,
 478             "routing_table": [],
 479             # empty IPv6 routing table
 480             "nd_route_tbl": [],
 481             "name": key, "role": key
 482         }
 483
 484         try:
 485             result['key_filename'] = flavor['key_filename']
 486         except KeyError:
 487             pass
 488
 489         try:
 490             result['password'] = flavor['password']
 491         except KeyError:
 492             pass
 493         LOG.info(result)
 494         return result
 495
 496
 497 class OvsDeploy(object):
 498     """ This class handles deploy of ovs dpdk
 499     Configuration: ovs_dpdk
 500     """
 501
 502     OVS_DEPLOY_SCRIPT = "ovs_deploy.bash"
 503
 504     def __init__(self, connection, bin_path, ovs_properties):
 505         self.connection = connection
 506         self.bin_path = bin_path
 507         self.ovs_properties = ovs_properties
 508
 509     def prerequisite(self):
 510         pkgs = ["git", "build-essential", "pkg-config", "automake",
 511                 "autotools-dev", "libltdl-dev", "cmake", "libnuma-dev",
 512                 "libpcap-dev"]
 513         StandaloneContextHelper.install_req_libs(self.connection, pkgs)
 514
 515     def ovs_deploy(self):
 516         ovs_deploy = os.path.join(constants.YARDSTICK_ROOT_PATH,
 517                                   "yardstick/resources/scripts/install/",
 518                                   self.OVS_DEPLOY_SCRIPT)
 519         if os.path.isfile(ovs_deploy):
 520             self.prerequisite()
 521             remote_ovs_deploy = os.path.join(self.bin_path, self.OVS_DEPLOY_SCRIPT)
 522             LOG.info(remote_ovs_deploy)
 523             self.connection.put(ovs_deploy, remote_ovs_deploy)
 524
 525             http_proxy = os.environ.get('http_proxy', '')
 526             ovs_details = self.ovs_properties.get("version", {})
 527             ovs = ovs_details.get("ovs", "2.6.0")
 528             dpdk = ovs_details.get("dpdk", "16.11.1")
 529
 530             cmd = "sudo -E %s --ovs='%s' --dpdk='%s' -p='%s'" % (remote_ovs_deploy,
 531                                                                  ovs, dpdk, http_proxy)
 532             exit_status, _, stderr = self.connection.execute(cmd)
 533             if exit_status:
 534                 raise exceptions.OVSDeployError(stderr=stderr)