yardstick/benchmark/contexts/standalone/model.py

   1 # Copyright (c) 2016-2017 Intel Corporation
   2 #
   3 # Licensed under the Apache License, Version 2.0 (the "License");
   4 # you may not use this file except in compliance with the License.
   5 # You may obtain a copy of the License at
   6 #
   7 #      http://www.apache.org/licenses/LICENSE-2.0
   8 #
   9 # Unless required by applicable law or agreed to in writing, software
  10 # distributed under the License is distributed on an "AS IS" BASIS,
  11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12 # See the License for the specific language governing permissions and
  13 # limitations under the License.
  14
  15 import os
  16 import re
  17 import time
  18 import uuid
  19 import random
  20 import logging
  21 import errno
  22
  23 from netaddr import IPNetwork
  24 import xml.etree.ElementTree as ET
  25
  26 from yardstick import ssh
  27 from yardstick.common import constants
  28 from yardstick.common import exceptions
  29 from yardstick.common.yaml_loader import yaml_load
  30 from yardstick.network_services.utils import PciAddress
  31 from yardstick.network_services.helpers.cpu import CpuSysCores
  32
  33
  34 LOG = logging.getLogger(__name__)
  35
  36 VM_TEMPLATE = """
  37 <domain type="kvm">
  38   <name>{vm_name}</name>
  39   <uuid>{random_uuid}</uuid>
  40   <memory unit="MB">{memory}</memory>
  41   <currentMemory unit="MB">{memory}</currentMemory>
  42   <memoryBacking>
  43     <hugepages />
  44   </memoryBacking>
  45   <vcpu cpuset='{cpuset}'>{vcpu}</vcpu>
  46  {cputune}
  47   <os>
  48     <type arch="x86_64" machine="pc-i440fx-xenial">hvm</type>
  49     <boot dev="hd" />
  50   </os>
  51   <features>
  52     <acpi />
  53     <apic />
  54     <pae />
  55   </features>
  56   <cpu mode='host-passthrough'>
  57     <topology cores="{cpu}" sockets="{socket}" threads="{threads}" />
  58     <numa>
  59        <cell id='0' cpus='{numa_cpus}' memory='{memory}' unit='MB' memAccess='shared'/>
  60     </numa>
  61   </cpu>
  62   <clock offset="utc">
  63     <timer name="rtc" tickpolicy="catchup" />
  64     <timer name="pit" tickpolicy="delay" />
  65     <timer name="hpet" present="no" />
  66   </clock>
  67   <on_poweroff>destroy</on_poweroff>
  68   <on_reboot>restart</on_reboot>
  69   <on_crash>restart</on_crash>
  70   <devices>
  71     <emulator>/usr/bin/kvm-spice</emulator>
  72     <disk device="disk" type="file">
  73       <driver name="qemu" type="qcow2" />
  74       <source file="{vm_image}"/>
  75       <target bus="virtio" dev="vda" />
  76     </disk>
  77     <graphics autoport="yes" listen="0.0.0.0" port="-1" type="vnc" />
  78     <interface type="bridge">
  79       <mac address='{mac_addr}'/>
  80       <source bridge="br-int" />
  81       <model type='virtio'/>
  82     </interface>
  83     <serial type='pty'>
  84       <target port='0'/>
  85     </serial>
  86     <console type='pty'>
  87       <target type='serial' port='0'/>
  88     </console>
  89   </devices>
  90 </domain>
  91 """
  92 WAIT_FOR_BOOT = 30
  93
  94
  95 class Libvirt(object):
  96     """ This class handles all the libvirt updates to lauch VM
  97     """
  98
  99     @staticmethod
 100     def check_if_vm_exists_and_delete(vm_name, connection):
 101         cmd_template = "virsh list --name | grep -i %s"
 102         status = connection.execute(cmd_template % vm_name)[0]
 103         if status == 0:
 104             LOG.info("VM '%s' is already present... destroying", vm_name)
 105             connection.execute("virsh destroy %s" % vm_name)
 106
 107     @staticmethod
 108     def virsh_create_vm(connection, cfg):
 109         LOG.info('VM create, XML config: %s', cfg)
 110         status, _, error = connection.execute('virsh create %s' % cfg)
 111         if status:
 112             raise exceptions.LibvirtCreateError(error=error)
 113
 114     @staticmethod
 115     def virsh_destroy_vm(vm_name, connection):
 116         LOG.info('VM destroy, VM name: %s', vm_name)
 117         status, _, error = connection.execute('virsh destroy %s' % vm_name)
 118         if status:
 119             LOG.warning('Error destroying VM %s. Error: %s', vm_name, error)
 120
 121     @staticmethod
 122     def _add_interface_address(interface, pci_address):
 123         """Add a PCI 'address' XML node
 124
 125         <address type='pci' domain='0x0000' bus='0x00' slot='0x08'
 126          function='0x0'/>
 127
 128         Refence: https://software.intel.com/en-us/articles/
 129                  configure-sr-iov-network-virtual-functions-in-linux-kvm
 130         """
 131         vm_pci = ET.SubElement(interface, 'address')
 132         vm_pci.set('type', 'pci')
 133         vm_pci.set('domain', '0x{}'.format(pci_address.domain))
 134         vm_pci.set('bus', '0x{}'.format(pci_address.bus))
 135         vm_pci.set('slot', '0x{}'.format(pci_address.slot))
 136         vm_pci.set('function', '0x{}'.format(pci_address.function))
 137         return vm_pci
 138
 139     @classmethod
 140     def add_ovs_interface(cls, vpath, port_num, vpci, vports_mac, xml_str):
 141         """Add a DPDK OVS 'interface' XML node in 'devices' node
 142
 143         <devices>
 144             <interface type='vhostuser'>
 145                 <mac address='00:00:00:00:00:01'/>
 146                 <source type='unix' path='/usr/local/var/run/openvswitch/
 147                  dpdkvhostuser0' mode='client'/>
 148                 <model type='virtio'/>
 149                 <driver queues='4'>
 150                     <host mrg_rxbuf='off'/>
 151                 </driver>
 152                 <address type='pci' domain='0x0000' bus='0x00' slot='0x03'
 153                  function='0x0'/>
 154             </interface>
 155             ...
 156         </devices>
 157
 158         Reference: http://docs.openvswitch.org/en/latest/topics/dpdk/
 159                    vhost-user/
 160         """
 161
 162         vhost_path = ('{0}/var/run/openvswitch/dpdkvhostuser{1}'.
 163                       format(vpath, port_num))
 164         root = ET.fromstring(xml_str)
 165         pci_address = PciAddress(vpci.strip())
 166         device = root.find('devices')
 167
 168         interface = ET.SubElement(device, 'interface')
 169         interface.set('type', 'vhostuser')
 170         mac = ET.SubElement(interface, 'mac')
 171         mac.set('address', vports_mac)
 172
 173         source = ET.SubElement(interface, 'source')
 174         source.set('type', 'unix')
 175         source.set('path', vhost_path)
 176         source.set('mode', 'client')
 177
 178         model = ET.SubElement(interface, 'model')
 179         model.set('type', 'virtio')
 180
 181         driver = ET.SubElement(interface, 'driver')
 182         driver.set('queues', '4')
 183
 184         host = ET.SubElement(driver, 'host')
 185         host.set('mrg_rxbuf', 'off')
 186
 187         cls._add_interface_address(interface, pci_address)
 188
 189         return ET.tostring(root)
 190
 191     @classmethod
 192     def add_sriov_interfaces(cls, vm_pci, vf_pci, vf_mac, xml_str):
 193         """Add a SR-IOV 'interface' XML node in 'devices' node
 194
 195         <devices>
 196            <interface type='hostdev' managed='yes'>
 197              <source>
 198                <address type='pci' domain='0x0000' bus='0x00' slot='0x03'
 199                 function='0x0'/>
 200              </source>
 201              <mac address='52:54:00:6d:90:02'>
 202              <address type='pci' domain='0x0000' bus='0x02' slot='0x04'
 203               function='0x1'/>
 204            </interface>
 205            ...
 206          </devices>
 207
 208         Reference: https://access.redhat.com/documentation/en-us/
 209             red_hat_enterprise_linux/6/html/
 210             virtualization_host_configuration_and_guest_installation_guide/
 211             sect-virtualization_host_configuration_and_guest_installation_guide
 212             -sr_iov-how_sr_iov_libvirt_works
 213         """
 214
 215         root = ET.fromstring(xml_str)
 216         device = root.find('devices')
 217
 218         interface = ET.SubElement(device, 'interface')
 219         interface.set('managed', 'yes')
 220         interface.set('type', 'hostdev')
 221
 222         mac = ET.SubElement(interface, 'mac')
 223         mac.set('address', vf_mac)
 224
 225         source = ET.SubElement(interface, 'source')
 226         pci_address = PciAddress(vf_pci.strip())
 227         cls._add_interface_address(source, pci_address)
 228
 229         pci_vm_address = PciAddress(vm_pci.strip())
 230         cls._add_interface_address(interface, pci_vm_address)
 231
 232         return ET.tostring(root)
 233
 234     @staticmethod
 235     def create_snapshot_qemu(connection, index, vm_image):
 236         # build snapshot image
 237         image = "/var/lib/libvirt/images/%s.qcow2" % index
 238         connection.execute("rm %s" % image)
 239         qemu_template = "qemu-img create -f qcow2 -o backing_file=%s %s"
 240         connection.execute(qemu_template % (vm_image, image))
 241
 242         return image
 243
 244     @classmethod
 245     def build_vm_xml(cls, connection, flavor, vm_name, index):
 246         """Build the XML from the configuration parameters"""
 247         memory = flavor.get('ram', '4096')
 248         extra_spec = flavor.get('extra_specs', {})
 249         cpu = extra_spec.get('hw:cpu_cores', '2')
 250         socket = extra_spec.get('hw:cpu_sockets', '1')
 251         threads = extra_spec.get('hw:cpu_threads', '2')
 252         vcpu = int(cpu) * int(threads)
 253         numa_cpus = '0-%s' % (vcpu - 1)
 254         hw_socket = flavor.get('hw_socket', '0')
 255         cpuset = Libvirt.pin_vcpu_for_perf(connection, hw_socket)
 256
 257         cputune = extra_spec.get('cputune', '')
 258         mac = StandaloneContextHelper.get_mac_address(0x00)
 259         image = cls.create_snapshot_qemu(connection, index,
 260                                          flavor.get("images", None))
 261         vm_xml = VM_TEMPLATE.format(
 262             vm_name=vm_name,
 263             random_uuid=uuid.uuid4(),
 264             mac_addr=mac,
 265             memory=memory, vcpu=vcpu, cpu=cpu,
 266             numa_cpus=numa_cpus,
 267             socket=socket, threads=threads,
 268             vm_image=image, cpuset=cpuset, cputune=cputune)
 269
 270         return vm_xml, mac
 271
 272     @staticmethod
 273     def update_interrupts_hugepages_perf(connection):
 274         connection.execute("echo 1 > /sys/module/kvm/parameters/allow_unsafe_assigned_interrupts")
 275         connection.execute("echo never > /sys/kernel/mm/transparent_hugepage/enabled")
 276
 277     @classmethod
 278     def pin_vcpu_for_perf(cls, connection, socket='0'):
 279         threads = ""
 280         sys_obj = CpuSysCores(connection)
 281         soc_cpu = sys_obj.get_core_socket()
 282         sys_cpu = int(soc_cpu["cores_per_socket"])
 283         socket = str(socket)
 284         cores = "%s-%s" % (soc_cpu[socket][0], soc_cpu[socket][sys_cpu - 1])
 285         if int(soc_cpu["thread_per_core"]) > 1:
 286             threads = "%s-%s" % (soc_cpu[socket][sys_cpu], soc_cpu[socket][-1])
 287         cpuset = "%s,%s" % (cores, threads)
 288         return cpuset
 289
 290     @classmethod
 291     def write_file(cls, file_name, xml_str):
 292         """Dump a XML string to a file"""
 293         root = ET.fromstring(xml_str)
 294         et = ET.ElementTree(element=root)
 295         et.write(file_name, encoding='utf-8', method='xml')
 296
 297
 298 class StandaloneContextHelper(object):
 299     """ This class handles all the common code for standalone
 300     """
 301     def __init__(self):
 302         self.file_path = None
 303         super(StandaloneContextHelper, self).__init__()
 304
 305     @staticmethod
 306     def install_req_libs(connection, extra_pkgs=None):
 307         extra_pkgs = extra_pkgs or []
 308         pkgs = ["qemu-kvm", "libvirt-bin", "bridge-utils", "numactl", "fping"]
 309         pkgs.extend(extra_pkgs)
 310         cmd_template = "dpkg-query -W --showformat='${Status}\\n' \"%s\"|grep 'ok installed'"
 311         for pkg in pkgs:
 312             if connection.execute(cmd_template % pkg)[0]:
 313                 connection.execute("apt-get update")
 314                 connection.execute("apt-get -y install %s" % pkg)
 315
 316     @staticmethod
 317     def get_kernel_module(connection, pci, driver):
 318         if not driver:
 319             out = connection.execute("lspci -k -s %s" % pci)[1]
 320             driver = out.split("Kernel modules:").pop().strip()
 321         return driver
 322
 323     @classmethod
 324     def get_nic_details(cls, connection, networks, dpdk_devbind):
 325         for key, ports in networks.items():
 326             if key == "mgmt":
 327                 continue
 328
 329             phy_ports = ports['phy_port']
 330             phy_driver = ports.get('phy_driver', None)
 331             driver = cls.get_kernel_module(connection, phy_ports, phy_driver)
 332
 333             # Make sure that ports are bound to kernel drivers e.g. i40e/ixgbe
 334             bind_cmd = "{dpdk_devbind} --force -b {driver} {port}"
 335             lshw_cmd = "lshw -c network -businfo | grep '{port}'"
 336             link_show_cmd = "ip -s link show {interface}"
 337
 338             cmd = bind_cmd.format(dpdk_devbind=dpdk_devbind,
 339                                   driver=driver, port=ports['phy_port'])
 340             connection.execute(cmd)
 341
 342             out = connection.execute(lshw_cmd.format(port=phy_ports))[1]
 343             interface = out.split()[1]
 344
 345             connection.execute(link_show_cmd.format(interface=interface))
 346
 347             ports.update({
 348                 'interface': str(interface),
 349                 'driver': driver
 350             })
 351         LOG.info(networks)
 352
 353         return networks
 354
 355     @staticmethod
 356     def get_virtual_devices(connection, pci):
 357         cmd = "cat /sys/bus/pci/devices/{0}/virtfn0/uevent"
 358         output = connection.execute(cmd.format(pci))[1]
 359
 360         pattern = "PCI_SLOT_NAME=({})".format(PciAddress.PCI_PATTERN_STR)
 361         m = re.search(pattern, output, re.MULTILINE)
 362
 363         pf_vfs = {}
 364         if m:
 365             pf_vfs = {pci: m.group(1).rstrip()}
 366
 367         LOG.info("pf_vfs:\n%s", pf_vfs)
 368
 369         return pf_vfs
 370
 371     def read_config_file(self):
 372         """Read from config file"""
 373
 374         with open(self.file_path) as stream:
 375             LOG.info("Parsing pod file: %s", self.file_path)
 376             cfg = yaml_load(stream)
 377         return cfg
 378
 379     def parse_pod_file(self, file_path, nfvi_role='Sriov'):
 380         self.file_path = file_path
 381         nodes = []
 382         nfvi_host = []
 383         try:
 384             cfg = self.read_config_file()
 385         except IOError as io_error:
 386             if io_error.errno != errno.ENOENT:
 387                 raise
 388             self.file_path = os.path.join(constants.YARDSTICK_ROOT_PATH,
 389                                           file_path)
 390             cfg = self.read_config_file()
 391
 392         nodes.extend([node for node in cfg["nodes"] if str(node["role"]) != nfvi_role])
 393         nfvi_host.extend([node for node in cfg["nodes"] if str(node["role"]) == nfvi_role])
 394         if not nfvi_host:
 395             raise("Node role is other than SRIOV")
 396
 397         host_mgmt = {'user': nfvi_host[0]['user'],
 398                      'ip': str(IPNetwork(nfvi_host[0]['ip']).ip),
 399                      'password': nfvi_host[0]['password'],
 400                      'ssh_port': nfvi_host[0].get('ssh_port', 22),
 401                      'key_filename': nfvi_host[0].get('key_filename')}
 402
 403         return [nodes, nfvi_host, host_mgmt]
 404
 405     @staticmethod
 406     def get_mac_address(end=0x7f):
 407         mac = [0x52, 0x54, 0x00,
 408                random.randint(0x00, end),
 409                random.randint(0x00, 0xff),
 410                random.randint(0x00, 0xff)]
 411         mac_address = ':'.join('%02x' % x for x in mac)
 412         return mac_address
 413
 414     @staticmethod
 415     def get_mgmt_ip(connection, mac, cidr, node):
 416         mgmtip = None
 417         times = 10
 418         while not mgmtip and times:
 419             connection.execute("fping -c 1 -g %s > /dev/null 2>&1" % cidr)
 420             out = connection.execute("ip neighbor | grep '%s'" % mac)[1]
 421             LOG.info("fping -c 1 -g %s > /dev/null 2>&1", cidr)
 422             if out.strip():
 423                 mgmtip = str(out.split(" ")[0]).strip()
 424                 client = ssh.SSH.from_node(node, overrides={"ip": mgmtip})
 425                 client.wait()
 426                 break
 427
 428             time.sleep(WAIT_FOR_BOOT)  # FixMe: How to find if VM is booted?
 429             times = times - 1
 430         return mgmtip
 431
 432     @classmethod
 433     def wait_for_vnfs_to_start(cls, connection, servers, nodes):
 434         for node in nodes:
 435             vnf = servers[node["name"]]
 436             mgmtip = vnf["network_ports"]["mgmt"]["cidr"]
 437             ip = cls.get_mgmt_ip(connection, node["mac"], mgmtip, node)
 438             if ip:
 439                 node["ip"] = ip
 440         return nodes
 441
 442
 443 class Server(object):
 444     """ This class handles geting vnf nodes
 445     """
 446
 447     @staticmethod
 448     def build_vnf_interfaces(vnf, ports):
 449         interfaces = {}
 450         index = 0
 451
 452         for key, vfs in vnf["network_ports"].items():
 453             if key == "mgmt":
 454                 mgmtip = str(IPNetwork(vfs['cidr']).ip)
 455                 continue
 456
 457             vf = ports[vfs[0]]
 458             ip = IPNetwork(vf['cidr'])
 459             interfaces.update({
 460                 key: {
 461                     'vpci': vf['vpci'],
 462                     'driver': "%svf" % vf['driver'],
 463                     'local_mac': vf['mac'],
 464                     'dpdk_port_num': index,
 465                     'local_ip': str(ip.ip),
 466                     'netmask': str(ip.netmask)
 467                     },
 468             })
 469             index = index + 1
 470
 471         return mgmtip, interfaces
 472
 473     @classmethod
 474     def generate_vnf_instance(cls, flavor, ports, ip, key, vnf, mac):
 475         mgmtip, interfaces = cls.build_vnf_interfaces(vnf, ports)
 476
 477         result = {
 478             "ip": mgmtip,
 479             "mac": mac,
 480             "host": ip,
 481             "user": flavor.get('user', 'root'),
 482             "interfaces": interfaces,
 483             "routing_table": [],
 484             # empty IPv6 routing table
 485             "nd_route_tbl": [],
 486             "name": key, "role": key
 487         }
 488
 489         try:
 490             result['key_filename'] = flavor['key_filename']
 491         except KeyError:
 492             pass
 493
 494         try:
 495             result['password'] = flavor['password']
 496         except KeyError:
 497             pass
 498         LOG.info(result)
 499         return result
 500
 501
 502 class OvsDeploy(object):
 503     """ This class handles deploy of ovs dpdk
 504     Configuration: ovs_dpdk
 505     """
 506
 507     OVS_DEPLOY_SCRIPT = "ovs_deploy.bash"
 508
 509     def __init__(self, connection, bin_path, ovs_properties):
 510         self.connection = connection
 511         self.bin_path = bin_path
 512         self.ovs_properties = ovs_properties
 513
 514     def prerequisite(self):
 515         pkgs = ["git", "build-essential", "pkg-config", "automake",
 516                 "autotools-dev", "libltdl-dev", "cmake", "libnuma-dev",
 517                 "libpcap-dev"]
 518         StandaloneContextHelper.install_req_libs(self.connection, pkgs)
 519
 520     def ovs_deploy(self):
 521         ovs_deploy = os.path.join(constants.YARDSTICK_ROOT_PATH,
 522                                   "yardstick/resources/scripts/install/",
 523                                   self.OVS_DEPLOY_SCRIPT)
 524         if os.path.isfile(ovs_deploy):
 525             self.prerequisite()
 526             remote_ovs_deploy = os.path.join(self.bin_path, self.OVS_DEPLOY_SCRIPT)
 527             LOG.info(remote_ovs_deploy)
 528             self.connection.put(ovs_deploy, remote_ovs_deploy)
 529
 530             http_proxy = os.environ.get('http_proxy', '')
 531             ovs_details = self.ovs_properties.get("version", {})
 532             ovs = ovs_details.get("ovs", "2.6.0")
 533             dpdk = ovs_details.get("dpdk", "16.11.1")
 534
 535             cmd = "sudo -E %s --ovs='%s' --dpdk='%s' -p='%s'" % (remote_ovs_deploy,
 536                                                                  ovs, dpdk, http_proxy)
 537             exit_status, _, stderr = self.connection.execute(cmd)
 538             if exit_status:
 539                 raise exceptions.OVSDeployError(stderr=stderr)