yardstick/benchmark/contexts/standalone/model.py

   1 # Copyright (c) 2016-2017 Intel Corporation
   2 #
   3 # Licensed under the Apache License, Version 2.0 (the "License");
   4 # you may not use this file except in compliance with the License.
   5 # You may obtain a copy of the License at
   6 #
   7 #      http://www.apache.org/licenses/LICENSE-2.0
   8 #
   9 # Unless required by applicable law or agreed to in writing, software
  10 # distributed under the License is distributed on an "AS IS" BASIS,
  11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12 # See the License for the specific language governing permissions and
  13 # limitations under the License.
  14
  15 import os
  16 import re
  17 import time
  18 import uuid
  19 import random
  20 import logging
  21 import errno
  22
  23 from netaddr import IPNetwork
  24 import xml.etree.ElementTree as ET
  25
  26 from yardstick import ssh
  27 from yardstick.common import constants
  28 from yardstick.common import exceptions
  29 from yardstick.common import yaml_loader
  30 from yardstick.network_services.utils import PciAddress
  31 from yardstick.network_services.helpers.cpu import CpuSysCores
  32
  33
  34 LOG = logging.getLogger(__name__)
  35
  36 VM_TEMPLATE = """
  37 <domain type="kvm">
  38   <name>{vm_name}</name>
  39   <uuid>{random_uuid}</uuid>
  40   <memory unit="MB">{memory}</memory>
  41   <currentMemory unit="MB">{memory}</currentMemory>
  42   <memoryBacking>
  43     <hugepages />
  44   </memoryBacking>
  45   <vcpu cpuset='{cpuset}'>{vcpu}</vcpu>
  46  {cputune}
  47   <os>
  48     <type arch="x86_64" machine="pc-i440fx-xenial">hvm</type>
  49     <boot dev="hd" />
  50   </os>
  51   <features>
  52     <acpi />
  53     <apic />
  54     <pae />
  55   </features>
  56   <cpu mode='host-passthrough'>
  57     <topology cores="{cpu}" sockets="{socket}" threads="{threads}" />
  58     <numa>
  59        <cell id='0' cpus='{numa_cpus}' memory='{memory}' unit='MB' memAccess='shared'/>
  60     </numa>
  61   </cpu>
  62   <clock offset="utc">
  63     <timer name="rtc" tickpolicy="catchup" />
  64     <timer name="pit" tickpolicy="delay" />
  65     <timer name="hpet" present="no" />
  66   </clock>
  67   <on_poweroff>destroy</on_poweroff>
  68   <on_reboot>restart</on_reboot>
  69   <on_crash>restart</on_crash>
  70   <devices>
  71     <emulator>/usr/bin/kvm-spice</emulator>
  72     <disk device="disk" type="file">
  73       <driver name="qemu" type="qcow2" />
  74       <source file="{vm_image}"/>
  75       <target bus="virtio" dev="vda" />
  76     </disk>
  77     <graphics autoport="yes" listen="0.0.0.0" port="-1" type="vnc" />
  78     <interface type="bridge">
  79       <mac address='{mac_addr}'/>
  80       <source bridge="br-int" />
  81       <model type='virtio'/>
  82     </interface>
  83     <serial type='pty'>
  84       <target port='0'/>
  85     </serial>
  86     <console type='pty'>
  87       <target type='serial' port='0'/>
  88     </console>
  89   </devices>
  90 </domain>
  91 """
  92 WAIT_FOR_BOOT = 30
  93
  94
  95 class Libvirt(object):
  96     """ This class handles all the libvirt updates to lauch VM
  97     """
  98
  99     @staticmethod
 100     def check_if_vm_exists_and_delete(vm_name, connection):
 101         cmd_template = "virsh list --name | grep -i %s"
 102         status = connection.execute(cmd_template % vm_name)[0]
 103         if status == 0:
 104             LOG.info("VM '%s' is already present... destroying", vm_name)
 105             connection.execute("virsh destroy %s" % vm_name)
 106
 107     @staticmethod
 108     def virsh_create_vm(connection, cfg):
 109         LOG.info('VM create, XML config: %s', cfg)
 110         status, _, error = connection.execute('virsh create %s' % cfg)
 111         if status:
 112             raise exceptions.LibvirtCreateError(error=error)
 113
 114     @staticmethod
 115     def virsh_destroy_vm(vm_name, connection):
 116         LOG.info('VM destroy, VM name: %s', vm_name)
 117         status, _, error = connection.execute('virsh destroy %s' % vm_name)
 118         if status:
 119             LOG.warning('Error destroying VM %s. Error: %s', vm_name, error)
 120
 121     @staticmethod
 122     def _add_interface_address(interface, pci_address):
 123         """Add a PCI 'address' XML node
 124
 125         <address type='pci' domain='0x0000' bus='0x00' slot='0x08'
 126          function='0x0'/>
 127
 128         Refence: https://software.intel.com/en-us/articles/
 129                  configure-sr-iov-network-virtual-functions-in-linux-kvm
 130         """
 131         vm_pci = ET.SubElement(interface, 'address')
 132         vm_pci.set('type', 'pci')
 133         vm_pci.set('domain', '0x{}'.format(pci_address.domain))
 134         vm_pci.set('bus', '0x{}'.format(pci_address.bus))
 135         vm_pci.set('slot', '0x{}'.format(pci_address.slot))
 136         vm_pci.set('function', '0x{}'.format(pci_address.function))
 137         return vm_pci
 138
 139     @classmethod
 140     def add_ovs_interface(cls, vpath, port_num, vpci, vports_mac, xml_str):
 141         """Add a DPDK OVS 'interface' XML node in 'devices' node
 142
 143         <devices>
 144             <interface type='vhostuser'>
 145                 <mac address='00:00:00:00:00:01'/>
 146                 <source type='unix' path='/usr/local/var/run/openvswitch/
 147                  dpdkvhostuser0' mode='client'/>
 148                 <model type='virtio'/>
 149                 <driver queues='4'>
 150                     <host mrg_rxbuf='off'/>
 151                 </driver>
 152                 <address type='pci' domain='0x0000' bus='0x00' slot='0x03'
 153                  function='0x0'/>
 154             </interface>
 155             ...
 156         </devices>
 157
 158         Reference: http://docs.openvswitch.org/en/latest/topics/dpdk/
 159                    vhost-user/
 160         """
 161
 162         vhost_path = ('{0}/var/run/openvswitch/dpdkvhostuser{1}'.
 163                       format(vpath, port_num))
 164         root = ET.fromstring(xml_str)
 165         pci_address = PciAddress(vpci.strip())
 166         device = root.find('devices')
 167
 168         interface = ET.SubElement(device, 'interface')
 169         interface.set('type', 'vhostuser')
 170         mac = ET.SubElement(interface, 'mac')
 171         mac.set('address', vports_mac)
 172
 173         source = ET.SubElement(interface, 'source')
 174         source.set('type', 'unix')
 175         source.set('path', vhost_path)
 176         source.set('mode', 'client')
 177
 178         model = ET.SubElement(interface, 'model')
 179         model.set('type', 'virtio')
 180
 181         driver = ET.SubElement(interface, 'driver')
 182         driver.set('queues', '4')
 183
 184         host = ET.SubElement(driver, 'host')
 185         host.set('mrg_rxbuf', 'off')
 186
 187         cls._add_interface_address(interface, pci_address)
 188
 189         return ET.tostring(root)
 190
 191     @classmethod
 192     def add_sriov_interfaces(cls, vm_pci, vf_pci, vf_mac, xml_str):
 193         """Add a SR-IOV 'interface' XML node in 'devices' node
 194
 195         <devices>
 196            <interface type='hostdev' managed='yes'>
 197              <source>
 198                <address type='pci' domain='0x0000' bus='0x00' slot='0x03'
 199                 function='0x0'/>
 200              </source>
 201              <mac address='52:54:00:6d:90:02'>
 202              <address type='pci' domain='0x0000' bus='0x02' slot='0x04'
 203               function='0x1'/>
 204            </interface>
 205            ...
 206          </devices>
 207
 208         Reference: https://access.redhat.com/documentation/en-us/
 209             red_hat_enterprise_linux/6/html/
 210             virtualization_host_configuration_and_guest_installation_guide/
 211             sect-virtualization_host_configuration_and_guest_installation_guide
 212             -sr_iov-how_sr_iov_libvirt_works
 213         """
 214
 215         root = ET.fromstring(xml_str)
 216         device = root.find('devices')
 217
 218         interface = ET.SubElement(device, 'interface')
 219         interface.set('managed', 'yes')
 220         interface.set('type', 'hostdev')
 221
 222         mac = ET.SubElement(interface, 'mac')
 223         mac.set('address', vf_mac)
 224
 225         source = ET.SubElement(interface, 'source')
 226         pci_address = PciAddress(vf_pci.strip())
 227         cls._add_interface_address(source, pci_address)
 228
 229         pci_vm_address = PciAddress(vm_pci.strip())
 230         cls._add_interface_address(interface, pci_vm_address)
 231
 232         return ET.tostring(root)
 233
 234     @staticmethod
 235     def create_snapshot_qemu(connection, index, base_image):
 236         """Create the snapshot image for a VM using a base image
 237
 238         :param connection: SSH connection to the remote host
 239         :param index: index of the VM to be spawn
 240         :param base_image: path of the VM base image in the remote host
 241         :return: snapshot image path
 242         """
 243         vm_image = '/var/lib/libvirt/images/%s.qcow2' % index
 244         connection.execute('rm -- "%s"' % vm_image)
 245         status, _, _ = connection.execute('test -r %s' % base_image)
 246         if status:
 247             if not os.access(base_image, os.R_OK):
 248                 raise exceptions.LibvirtQemuImageBaseImageNotPresent(
 249                     vm_image=vm_image, base_image=base_image)
 250             # NOTE(ralonsoh): done in two steps to avoid root permission
 251             # issues.
 252             LOG.info('Copy %s from execution host to remote host', base_image)
 253             file_name = os.path.basename(os.path.normpath(base_image))
 254             connection.put_file(base_image, '/tmp/%s' % file_name)
 255             status, _, error = connection.execute(
 256                 'mv -- "/tmp/%s" "%s"' % (file_name, base_image))
 257             if status:
 258                 raise exceptions.LibvirtQemuImageCreateError(
 259                     vm_image=vm_image, base_image=base_image, error=error)
 260
 261         LOG.info('Convert image %s to %s', base_image, vm_image)
 262         qemu_cmd = ('qemu-img create -f qcow2 -o backing_file=%s %s' %
 263                     (base_image, vm_image))
 264         status, _, error = connection.execute(qemu_cmd)
 265         if status:
 266             raise exceptions.LibvirtQemuImageCreateError(
 267                 vm_image=vm_image, base_image=base_image, error=error)
 268         return vm_image
 269
 270     @classmethod
 271     def build_vm_xml(cls, connection, flavor, vm_name, index):
 272         """Build the XML from the configuration parameters"""
 273         memory = flavor.get('ram', '4096')
 274         extra_spec = flavor.get('extra_specs', {})
 275         cpu = extra_spec.get('hw:cpu_cores', '2')
 276         socket = extra_spec.get('hw:cpu_sockets', '1')
 277         threads = extra_spec.get('hw:cpu_threads', '2')
 278         vcpu = int(cpu) * int(threads)
 279         numa_cpus = '0-%s' % (vcpu - 1)
 280         hw_socket = flavor.get('hw_socket', '0')
 281         cpuset = Libvirt.pin_vcpu_for_perf(connection, hw_socket)
 282
 283         cputune = extra_spec.get('cputune', '')
 284         mac = StandaloneContextHelper.get_mac_address(0x00)
 285         image = cls.create_snapshot_qemu(connection, index,
 286                                          flavor.get("images", None))
 287         vm_xml = VM_TEMPLATE.format(
 288             vm_name=vm_name,
 289             random_uuid=uuid.uuid4(),
 290             mac_addr=mac,
 291             memory=memory, vcpu=vcpu, cpu=cpu,
 292             numa_cpus=numa_cpus,
 293             socket=socket, threads=threads,
 294             vm_image=image, cpuset=cpuset, cputune=cputune)
 295
 296         return vm_xml, mac
 297
 298     @staticmethod
 299     def update_interrupts_hugepages_perf(connection):
 300         connection.execute("echo 1 > /sys/module/kvm/parameters/allow_unsafe_assigned_interrupts")
 301         connection.execute("echo never > /sys/kernel/mm/transparent_hugepage/enabled")
 302
 303     @classmethod
 304     def pin_vcpu_for_perf(cls, connection, socket='0'):
 305         threads = ""
 306         sys_obj = CpuSysCores(connection)
 307         soc_cpu = sys_obj.get_core_socket()
 308         sys_cpu = int(soc_cpu["cores_per_socket"])
 309         socket = str(socket)
 310         cores = "%s-%s" % (soc_cpu[socket][0], soc_cpu[socket][sys_cpu - 1])
 311         if int(soc_cpu["thread_per_core"]) > 1:
 312             threads = "%s-%s" % (soc_cpu[socket][sys_cpu], soc_cpu[socket][-1])
 313         cpuset = "%s,%s" % (cores, threads)
 314         return cpuset
 315
 316     @classmethod
 317     def write_file(cls, file_name, xml_str):
 318         """Dump a XML string to a file"""
 319         root = ET.fromstring(xml_str)
 320         et = ET.ElementTree(element=root)
 321         et.write(file_name, encoding='utf-8', method='xml')
 322
 323
 324 class StandaloneContextHelper(object):
 325     """ This class handles all the common code for standalone
 326     """
 327     def __init__(self):
 328         self.file_path = None
 329         super(StandaloneContextHelper, self).__init__()
 330
 331     @staticmethod
 332     def install_req_libs(connection, extra_pkgs=None):
 333         extra_pkgs = extra_pkgs or []
 334         pkgs = ["qemu-kvm", "libvirt-bin", "bridge-utils", "numactl", "fping"]
 335         pkgs.extend(extra_pkgs)
 336         cmd_template = "dpkg-query -W --showformat='${Status}\\n' \"%s\"|grep 'ok installed'"
 337         for pkg in pkgs:
 338             if connection.execute(cmd_template % pkg)[0]:
 339                 connection.execute("apt-get update")
 340                 connection.execute("apt-get -y install %s" % pkg)
 341
 342     @staticmethod
 343     def get_kernel_module(connection, pci, driver):
 344         if not driver:
 345             out = connection.execute("lspci -k -s %s" % pci)[1]
 346             driver = out.split("Kernel modules:").pop().strip()
 347         return driver
 348
 349     @classmethod
 350     def get_nic_details(cls, connection, networks, dpdk_devbind):
 351         for key, ports in networks.items():
 352             if key == "mgmt":
 353                 continue
 354
 355             phy_ports = ports['phy_port']
 356             phy_driver = ports.get('phy_driver', None)
 357             driver = cls.get_kernel_module(connection, phy_ports, phy_driver)
 358
 359             # Make sure that ports are bound to kernel drivers e.g. i40e/ixgbe
 360             bind_cmd = "{dpdk_devbind} --force -b {driver} {port}"
 361             lshw_cmd = "lshw -c network -businfo | grep '{port}'"
 362             link_show_cmd = "ip -s link show {interface}"
 363
 364             cmd = bind_cmd.format(dpdk_devbind=dpdk_devbind,
 365                                   driver=driver, port=ports['phy_port'])
 366             connection.execute(cmd)
 367
 368             out = connection.execute(lshw_cmd.format(port=phy_ports))[1]
 369             interface = out.split()[1]
 370
 371             connection.execute(link_show_cmd.format(interface=interface))
 372
 373             ports.update({
 374                 'interface': str(interface),
 375                 'driver': driver
 376             })
 377         LOG.info(networks)
 378
 379         return networks
 380
 381     @staticmethod
 382     def get_virtual_devices(connection, pci):
 383         cmd = "cat /sys/bus/pci/devices/{0}/virtfn0/uevent"
 384         output = connection.execute(cmd.format(pci))[1]
 385
 386         pattern = "PCI_SLOT_NAME=({})".format(PciAddress.PCI_PATTERN_STR)
 387         m = re.search(pattern, output, re.MULTILINE)
 388
 389         pf_vfs = {}
 390         if m:
 391             pf_vfs = {pci: m.group(1).rstrip()}
 392
 393         LOG.info("pf_vfs:\n%s", pf_vfs)
 394
 395         return pf_vfs
 396
 397     def parse_pod_file(self, file_path, nfvi_role='Sriov'):
 398         self.file_path = file_path
 399         nodes = []
 400         nfvi_host = []
 401         try:
 402             cfg = yaml_loader.read_yaml_file(self.file_path)
 403         except IOError as io_error:
 404             if io_error.errno != errno.ENOENT:
 405                 raise
 406             self.file_path = os.path.join(constants.YARDSTICK_ROOT_PATH,
 407                                           file_path)
 408             cfg = yaml_loader.read_yaml_file(self.file_path)
 409
 410         nodes.extend([node for node in cfg["nodes"] if str(node["role"]) != nfvi_role])
 411         nfvi_host.extend([node for node in cfg["nodes"] if str(node["role"]) == nfvi_role])
 412         if not nfvi_host:
 413             raise("Node role is other than SRIOV")
 414
 415         host_mgmt = {'user': nfvi_host[0]['user'],
 416                      'ip': str(IPNetwork(nfvi_host[0]['ip']).ip),
 417                      'password': nfvi_host[0]['password'],
 418                      'ssh_port': nfvi_host[0].get('ssh_port', 22),
 419                      'key_filename': nfvi_host[0].get('key_filename')}
 420
 421         return [nodes, nfvi_host, host_mgmt]
 422
 423     @staticmethod
 424     def get_mac_address(end=0x7f):
 425         mac = [0x52, 0x54, 0x00,
 426                random.randint(0x00, end),
 427                random.randint(0x00, 0xff),
 428                random.randint(0x00, 0xff)]
 429         mac_address = ':'.join('%02x' % x for x in mac)
 430         return mac_address
 431
 432     @staticmethod
 433     def get_mgmt_ip(connection, mac, cidr, node):
 434         mgmtip = None
 435         times = 10
 436         while not mgmtip and times:
 437             connection.execute("fping -c 1 -g %s > /dev/null 2>&1" % cidr)
 438             out = connection.execute("ip neighbor | grep '%s'" % mac)[1]
 439             LOG.info("fping -c 1 -g %s > /dev/null 2>&1", cidr)
 440             if out.strip():
 441                 mgmtip = str(out.split(" ")[0]).strip()
 442                 client = ssh.SSH.from_node(node, overrides={"ip": mgmtip})
 443                 client.wait()
 444                 break
 445
 446             time.sleep(WAIT_FOR_BOOT)  # FixMe: How to find if VM is booted?
 447             times = times - 1
 448         return mgmtip
 449
 450     @classmethod
 451     def wait_for_vnfs_to_start(cls, connection, servers, nodes):
 452         for node in nodes:
 453             vnf = servers[node["name"]]
 454             mgmtip = vnf["network_ports"]["mgmt"]["cidr"]
 455             ip = cls.get_mgmt_ip(connection, node["mac"], mgmtip, node)
 456             if ip:
 457                 node["ip"] = ip
 458         return nodes
 459
 460
 461 class Server(object):
 462     """ This class handles geting vnf nodes
 463     """
 464
 465     @staticmethod
 466     def build_vnf_interfaces(vnf, ports):
 467         interfaces = {}
 468         index = 0
 469
 470         for key, vfs in vnf["network_ports"].items():
 471             if key == "mgmt":
 472                 mgmtip = str(IPNetwork(vfs['cidr']).ip)
 473                 continue
 474
 475             vf = ports[vfs[0]]
 476             ip = IPNetwork(vf['cidr'])
 477             interfaces.update({
 478                 key: {
 479                     'vpci': vf['vpci'],
 480                     'driver': "%svf" % vf['driver'],
 481                     'local_mac': vf['mac'],
 482                     'dpdk_port_num': index,
 483                     'local_ip': str(ip.ip),
 484                     'netmask': str(ip.netmask)
 485                     },
 486             })
 487             index = index + 1
 488
 489         return mgmtip, interfaces
 490
 491     @classmethod
 492     def generate_vnf_instance(cls, flavor, ports, ip, key, vnf, mac):
 493         mgmtip, interfaces = cls.build_vnf_interfaces(vnf, ports)
 494
 495         result = {
 496             "ip": mgmtip,
 497             "mac": mac,
 498             "host": ip,
 499             "user": flavor.get('user', 'root'),
 500             "interfaces": interfaces,
 501             "routing_table": [],
 502             # empty IPv6 routing table
 503             "nd_route_tbl": [],
 504             "name": key, "role": key
 505         }
 506
 507         try:
 508             result['key_filename'] = flavor['key_filename']
 509         except KeyError:
 510             pass
 511
 512         try:
 513             result['password'] = flavor['password']
 514         except KeyError:
 515             pass
 516         LOG.info(result)
 517         return result
 518
 519
 520 class OvsDeploy(object):
 521     """ This class handles deploy of ovs dpdk
 522     Configuration: ovs_dpdk
 523     """
 524
 525     OVS_DEPLOY_SCRIPT = "ovs_deploy.bash"
 526
 527     def __init__(self, connection, bin_path, ovs_properties):
 528         self.connection = connection
 529         self.bin_path = bin_path
 530         self.ovs_properties = ovs_properties
 531
 532     def prerequisite(self):
 533         pkgs = ["git", "build-essential", "pkg-config", "automake",
 534                 "autotools-dev", "libltdl-dev", "cmake", "libnuma-dev",
 535                 "libpcap-dev"]
 536         StandaloneContextHelper.install_req_libs(self.connection, pkgs)
 537
 538     def ovs_deploy(self):
 539         ovs_deploy = os.path.join(constants.YARDSTICK_ROOT_PATH,
 540                                   "yardstick/resources/scripts/install/",
 541                                   self.OVS_DEPLOY_SCRIPT)
 542         if os.path.isfile(ovs_deploy):
 543             self.prerequisite()
 544             remote_ovs_deploy = os.path.join(self.bin_path, self.OVS_DEPLOY_SCRIPT)
 545             LOG.info(remote_ovs_deploy)
 546             self.connection.put(ovs_deploy, remote_ovs_deploy)
 547
 548             http_proxy = os.environ.get('http_proxy', '')
 549             ovs_details = self.ovs_properties.get("version", {})
 550             ovs = ovs_details.get("ovs", "2.6.0")
 551             dpdk = ovs_details.get("dpdk", "16.11.1")
 552
 553             cmd = "sudo -E %s --ovs='%s' --dpdk='%s' -p='%s'" % (remote_ovs_deploy,
 554                                                                  ovs, dpdk, http_proxy)
 555             exit_status, _, stderr = self.connection.execute(cmd)
 556             if exit_status:
 557                 raise exceptions.OVSDeployError(stderr=stderr)