yardstick/benchmark/contexts/standalone/model.py

   1 # Copyright (c) 2016-2017 Intel Corporation
   2 #
   3 # Licensed under the Apache License, Version 2.0 (the "License");
   4 # you may not use this file except in compliance with the License.
   5 # You may obtain a copy of the License at
   6 #
   7 #      http://www.apache.org/licenses/LICENSE-2.0
   8 #
   9 # Unless required by applicable law or agreed to in writing, software
  10 # distributed under the License is distributed on an "AS IS" BASIS,
  11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12 # See the License for the specific language governing permissions and
  13 # limitations under the License.
  14
  15 import os
  16 import re
  17 import time
  18 import uuid
  19 import random
  20 import logging
  21 import errno
  22
  23 from netaddr import IPNetwork
  24 import xml.etree.ElementTree as ET
  25
  26 from yardstick import ssh
  27 from yardstick.common import constants
  28 from yardstick.common import exceptions
  29 from yardstick.common.yaml_loader import yaml_load
  30 from yardstick.network_services.utils import PciAddress
  31 from yardstick.network_services.helpers.cpu import CpuSysCores
  32
  33
  34 LOG = logging.getLogger(__name__)
  35
  36 VM_TEMPLATE = """
  37 <domain type="kvm">
  38   <name>{vm_name}</name>
  39   <uuid>{random_uuid}</uuid>
  40   <memory unit="MB">{memory}</memory>
  41   <currentMemory unit="MB">{memory}</currentMemory>
  42   <memoryBacking>
  43     <hugepages />
  44   </memoryBacking>
  45   <vcpu cpuset='{cpuset}'>{vcpu}</vcpu>
  46  {cputune}
  47   <os>
  48     <type arch="x86_64" machine="pc-i440fx-utopic">hvm</type>
  49     <boot dev="hd" />
  50   </os>
  51   <features>
  52     <acpi />
  53     <apic />
  54     <pae />
  55   </features>
  56   <cpu mode='host-passthrough'>
  57     <topology cores="{cpu}" sockets="{socket}" threads="{threads}" />
  58     <numa>
  59        <cell id='0' cpus='{numa_cpus}' memory='{memory}' unit='MB' memAccess='shared'/>
  60     </numa>
  61   </cpu>
  62   <clock offset="utc">
  63     <timer name="rtc" tickpolicy="catchup" />
  64     <timer name="pit" tickpolicy="delay" />
  65     <timer name="hpet" present="no" />
  66   </clock>
  67   <on_poweroff>destroy</on_poweroff>
  68   <on_reboot>restart</on_reboot>
  69   <on_crash>restart</on_crash>
  70   <devices>
  71     <emulator>/usr/bin/kvm-spice</emulator>
  72     <disk device="disk" type="file">
  73       <driver name="qemu" type="qcow2" />
  74       <source file="{vm_image}"/>
  75       <target bus="virtio" dev="vda" />
  76     </disk>
  77     <graphics autoport="yes" listen="0.0.0.0" port="-1" type="vnc" />
  78     <interface type="bridge">
  79       <mac address='{mac_addr}'/>
  80       <source bridge="br-int" />
  81       <model type='virtio'/>
  82     </interface>
  83     <serial type='pty'>
  84       <target port='0'/>
  85     </serial>
  86     <console type='pty'>
  87       <target type='serial' port='0'/>
  88     </console>
  89   </devices>
  90 </domain>
  91 """
  92 WAIT_FOR_BOOT = 30
  93
  94
  95 class Libvirt(object):
  96     """ This class handles all the libvirt updates to lauch VM
  97     """
  98
  99     @staticmethod
 100     def check_if_vm_exists_and_delete(vm_name, connection):
 101         cmd_template = "virsh list --name | grep -i %s"
 102         status = connection.execute(cmd_template % vm_name)[0]
 103         if status == 0:
 104             LOG.info("VM '%s' is already present... destroying", vm_name)
 105             connection.execute("virsh destroy %s" % vm_name)
 106
 107     @staticmethod
 108     def virsh_create_vm(connection, cfg):
 109         LOG.info('VM create, XML config: %s', cfg)
 110         status, _, error = connection.execute('virsh create %s' % cfg)
 111         if status:
 112             raise exceptions.LibvirtCreateError(error=error)
 113
 114     @staticmethod
 115     def virsh_destroy_vm(vm_name, connection):
 116         LOG.info('VM destroy, VM name: %s', vm_name)
 117         status, _, error = connection.execute('virsh destroy %s' % vm_name)
 118         if status:
 119             LOG.warning('Error destroying VM %s. Error: %s', vm_name, error)
 120
 121     @staticmethod
 122     def _add_interface_address(interface, pci_address):
 123         """Add a PCI 'address' XML node
 124
 125         <address type='pci' domain='0x0000' bus='0x00' slot='0x08'
 126          function='0x0'/>
 127
 128         Refence: https://software.intel.com/en-us/articles/
 129                  configure-sr-iov-network-virtual-functions-in-linux-kvm
 130         """
 131         vm_pci = ET.SubElement(interface, 'address')
 132         vm_pci.set('type', 'pci')
 133         vm_pci.set('domain', '0x{}'.format(pci_address.domain))
 134         vm_pci.set('bus', '0x{}'.format(pci_address.bus))
 135         vm_pci.set('slot', '0x{}'.format(pci_address.slot))
 136         vm_pci.set('function', '0x{}'.format(pci_address.function))
 137         return vm_pci
 138
 139     @classmethod
 140     def add_ovs_interface(cls, vpath, port_num, vpci, vports_mac, xml_str):
 141         """Add a DPDK OVS 'interface' XML node in 'devices' node
 142
 143         <devices>
 144             <interface type='vhostuser'>
 145                 <mac address='00:00:00:00:00:01'/>
 146                 <source type='unix' path='/usr/local/var/run/openvswitch/
 147                  dpdkvhostuser0' mode='client'/>
 148                 <model type='virtio'/>
 149                 <driver queues='4'>
 150                     <host mrg_rxbuf='off'/>
 151                 </driver>
 152                 <address type='pci' domain='0x0000' bus='0x00' slot='0x03'
 153                  function='0x0'/>
 154             </interface>
 155             ...
 156         </devices>
 157
 158         Reference: http://docs.openvswitch.org/en/latest/topics/dpdk/
 159                    vhost-user/
 160         """
 161
 162         vhost_path = ('{0}/var/run/openvswitch/dpdkvhostuser{1}'.
 163                       format(vpath, port_num))
 164         root = ET.fromstring(xml_str)
 165         pci_address = PciAddress(vpci.strip())
 166         device = root.find('devices')
 167
 168         interface = ET.SubElement(device, 'interface')
 169         interface.set('type', 'vhostuser')
 170         mac = ET.SubElement(interface, 'mac')
 171         mac.set('address', vports_mac)
 172
 173         source = ET.SubElement(interface, 'source')
 174         source.set('type', 'unix')
 175         source.set('path', vhost_path)
 176         source.set('mode', 'client')
 177
 178         model = ET.SubElement(interface, 'model')
 179         model.set('type', 'virtio')
 180
 181         driver = ET.SubElement(interface, 'driver')
 182         driver.set('queues', '4')
 183
 184         host = ET.SubElement(driver, 'host')
 185         host.set('mrg_rxbuf', 'off')
 186
 187         cls._add_interface_address(interface, pci_address)
 188
 189         return ET.tostring(root)
 190
 191     @classmethod
 192     def add_sriov_interfaces(cls, vm_pci, vf_pci, vf_mac, xml_str):
 193         """Add a SR-IOV 'interface' XML node in 'devices' node
 194
 195         <devices>
 196            <interface type='hostdev' managed='yes'>
 197              <source>
 198                <address type='pci' domain='0x0000' bus='0x00' slot='0x03'
 199                 function='0x0'/>
 200              </source>
 201              <mac address='52:54:00:6d:90:02'>
 202              <address type='pci' domain='0x0000' bus='0x02' slot='0x04'
 203               function='0x1'/>
 204            </interface>
 205            ...
 206          </devices>
 207
 208         Reference: https://access.redhat.com/documentation/en-us/
 209             red_hat_enterprise_linux/6/html/
 210             virtualization_host_configuration_and_guest_installation_guide/
 211             sect-virtualization_host_configuration_and_guest_installation_guide
 212             -sr_iov-how_sr_iov_libvirt_works
 213         """
 214
 215         root = ET.fromstring(xml_str)
 216         device = root.find('devices')
 217
 218         interface = ET.SubElement(device, 'interface')
 219         interface.set('managed', 'yes')
 220         interface.set('type', 'hostdev')
 221
 222         mac = ET.SubElement(interface, 'mac')
 223         mac.set('address', vf_mac)
 224
 225         source = ET.SubElement(interface, 'source')
 226         pci_address = PciAddress(vf_pci.strip())
 227         cls._add_interface_address(source, pci_address)
 228
 229         pci_vm_address = PciAddress(vm_pci.strip())
 230         cls._add_interface_address(interface, pci_vm_address)
 231
 232         return ET.tostring(root)
 233
 234     @staticmethod
 235     def create_snapshot_qemu(connection, index, base_image):
 236         """Create the snapshot image for a VM using a base image
 237
 238         :param connection: SSH connection to the remote host
 239         :param index: index of the VM to be spawn
 240         :param base_image: path of the VM base image in the remote host
 241         :return: snapshot image path
 242         """
 243         vm_image = '/var/lib/libvirt/images/%s.qcow2' % index
 244         connection.execute('rm -- "%s"' % vm_image)
 245         status, _, _ = connection.execute('test -r %s' % base_image)
 246         if status:
 247             if not os.access(base_image, os.R_OK):
 248                 raise exceptions.LibvirtQemuImageBaseImageNotPresent(
 249                     vm_image=vm_image, base_image=base_image)
 250             # NOTE(ralonsoh): done in two steps to avoid root permission
 251             # issues.
 252             LOG.info('Copy %s from execution host to remote host', base_image)
 253             file_name = os.path.basename(os.path.normpath(base_image))
 254             connection.put_file(base_image, '/tmp/%s' % file_name)
 255             status, _, error = connection.execute(
 256                 'mv -- "/tmp/%s" "%s"' % (file_name, base_image))
 257             if status:
 258                 raise exceptions.LibvirtQemuImageCreateError(
 259                     vm_image=vm_image, base_image=base_image, error=error)
 260
 261         LOG.info('Convert image %s to %s', base_image, vm_image)
 262         qemu_cmd = ('qemu-img create -f qcow2 -o backing_file=%s %s' %
 263                     (base_image, vm_image))
 264         status, _, error = connection.execute(qemu_cmd)
 265         if status:
 266             raise exceptions.LibvirtQemuImageCreateError(
 267                 vm_image=vm_image, base_image=base_image, error=error)
 268         return vm_image
 269
 270     @classmethod
 271     def build_vm_xml(cls, connection, flavor, vm_name, index):
 272         """Build the XML from the configuration parameters"""
 273         memory = flavor.get('ram', '4096')
 274         extra_spec = flavor.get('extra_specs', {})
 275         cpu = extra_spec.get('hw:cpu_cores', '2')
 276         socket = extra_spec.get('hw:cpu_sockets', '1')
 277         threads = extra_spec.get('hw:cpu_threads', '2')
 278         vcpu = int(cpu) * int(threads)
 279         numa_cpus = '0-%s' % (vcpu - 1)
 280         hw_socket = flavor.get('hw_socket', '0')
 281         cpuset = Libvirt.pin_vcpu_for_perf(connection, hw_socket)
 282
 283         cputune = extra_spec.get('cputune', '')
 284         mac = StandaloneContextHelper.get_mac_address(0x00)
 285         image = cls.create_snapshot_qemu(connection, index,
 286                                          flavor.get("images", None))
 287         vm_xml = VM_TEMPLATE.format(
 288             vm_name=vm_name,
 289             random_uuid=uuid.uuid4(),
 290             mac_addr=mac,
 291             memory=memory, vcpu=vcpu, cpu=cpu,
 292             numa_cpus=numa_cpus,
 293             socket=socket, threads=threads,
 294             vm_image=image, cpuset=cpuset, cputune=cputune)
 295
 296         return vm_xml, mac
 297
 298     @staticmethod
 299     def update_interrupts_hugepages_perf(connection):
 300         connection.execute("echo 1 > /sys/module/kvm/parameters/allow_unsafe_assigned_interrupts")
 301         connection.execute("echo never > /sys/kernel/mm/transparent_hugepage/enabled")
 302
 303     @classmethod
 304     def pin_vcpu_for_perf(cls, connection, socket='0'):
 305         threads = ""
 306         sys_obj = CpuSysCores(connection)
 307         soc_cpu = sys_obj.get_core_socket()
 308         sys_cpu = int(soc_cpu["cores_per_socket"])
 309         socket = str(socket)
 310         cores = "%s-%s" % (soc_cpu[socket][0], soc_cpu[socket][sys_cpu - 1])
 311         if int(soc_cpu["thread_per_core"]) > 1:
 312             threads = "%s-%s" % (soc_cpu[socket][sys_cpu], soc_cpu[socket][-1])
 313         cpuset = "%s,%s" % (cores, threads)
 314         return cpuset
 315
 316     @classmethod
 317     def write_file(cls, file_name, xml_str):
 318         """Dump a XML string to a file"""
 319         root = ET.fromstring(xml_str)
 320         et = ET.ElementTree(element=root)
 321         et.write(file_name, encoding='utf-8', method='xml')
 322
 323
 324 class StandaloneContextHelper(object):
 325     """ This class handles all the common code for standalone
 326     """
 327     def __init__(self):
 328         self.file_path = None
 329         super(StandaloneContextHelper, self).__init__()
 330
 331     @staticmethod
 332     def install_req_libs(connection, extra_pkgs=None):
 333         extra_pkgs = extra_pkgs or []
 334         pkgs = ["qemu-kvm", "libvirt-bin", "bridge-utils", "numactl", "fping"]
 335         pkgs.extend(extra_pkgs)
 336         cmd_template = "dpkg-query -W --showformat='${Status}\\n' \"%s\"|grep 'ok installed'"
 337         for pkg in pkgs:
 338             if connection.execute(cmd_template % pkg)[0]:
 339                 connection.execute("apt-get update")
 340                 connection.execute("apt-get -y install %s" % pkg)
 341
 342     @staticmethod
 343     def get_kernel_module(connection, pci, driver):
 344         if not driver:
 345             out = connection.execute("lspci -k -s %s" % pci)[1]
 346             driver = out.split("Kernel modules:").pop().strip()
 347         return driver
 348
 349     @classmethod
 350     def get_nic_details(cls, connection, networks, dpdk_devbind):
 351         for key, ports in networks.items():
 352             if key == "mgmt":
 353                 continue
 354
 355             phy_ports = ports['phy_port']
 356             phy_driver = ports.get('phy_driver', None)
 357             driver = cls.get_kernel_module(connection, phy_ports, phy_driver)
 358
 359             # Make sure that ports are bound to kernel drivers e.g. i40e/ixgbe
 360             bind_cmd = "{dpdk_devbind} --force -b {driver} {port}"
 361             lshw_cmd = "lshw -c network -businfo | grep '{port}'"
 362             link_show_cmd = "ip -s link show {interface}"
 363
 364             cmd = bind_cmd.format(dpdk_devbind=dpdk_devbind,
 365                                   driver=driver, port=ports['phy_port'])
 366             connection.execute(cmd)
 367
 368             out = connection.execute(lshw_cmd.format(port=phy_ports))[1]
 369             interface = out.split()[1]
 370
 371             connection.execute(link_show_cmd.format(interface=interface))
 372
 373             ports.update({
 374                 'interface': str(interface),
 375                 'driver': driver
 376             })
 377         LOG.info(networks)
 378
 379         return networks
 380
 381     @staticmethod
 382     def get_virtual_devices(connection, pci):
 383         cmd = "cat /sys/bus/pci/devices/{0}/virtfn0/uevent"
 384         output = connection.execute(cmd.format(pci))[1]
 385
 386         pattern = "PCI_SLOT_NAME=({})".format(PciAddress.PCI_PATTERN_STR)
 387         m = re.search(pattern, output, re.MULTILINE)
 388
 389         pf_vfs = {}
 390         if m:
 391             pf_vfs = {pci: m.group(1).rstrip()}
 392
 393         LOG.info("pf_vfs:\n%s", pf_vfs)
 394
 395         return pf_vfs
 396
 397     def read_config_file(self):
 398         """Read from config file"""
 399
 400         with open(self.file_path) as stream:
 401             LOG.info("Parsing pod file: %s", self.file_path)
 402             cfg = yaml_load(stream)
 403         return cfg
 404
 405     def parse_pod_file(self, file_path, nfvi_role='Sriov'):
 406         self.file_path = file_path
 407         nodes = []
 408         nfvi_host = []
 409         try:
 410             cfg = self.read_config_file()
 411         except IOError as io_error:
 412             if io_error.errno != errno.ENOENT:
 413                 raise
 414             self.file_path = os.path.join(constants.YARDSTICK_ROOT_PATH,
 415                                           file_path)
 416             cfg = self.read_config_file()
 417
 418         nodes.extend([node for node in cfg["nodes"] if str(node["role"]) != nfvi_role])
 419         nfvi_host.extend([node for node in cfg["nodes"] if str(node["role"]) == nfvi_role])
 420         if not nfvi_host:
 421             raise("Node role is other than SRIOV")
 422
 423         host_mgmt = {'user': nfvi_host[0]['user'],
 424                      'ip': str(IPNetwork(nfvi_host[0]['ip']).ip),
 425                      'password': nfvi_host[0]['password'],
 426                      'ssh_port': nfvi_host[0].get('ssh_port', 22),
 427                      'key_filename': nfvi_host[0].get('key_filename')}
 428
 429         return [nodes, nfvi_host, host_mgmt]
 430
 431     @staticmethod
 432     def get_mac_address(end=0x7f):
 433         mac = [0x52, 0x54, 0x00,
 434                random.randint(0x00, end),
 435                random.randint(0x00, 0xff),
 436                random.randint(0x00, 0xff)]
 437         mac_address = ':'.join('%02x' % x for x in mac)
 438         return mac_address
 439
 440     @staticmethod
 441     def get_mgmt_ip(connection, mac, cidr, node):
 442         mgmtip = None
 443         times = 10
 444         while not mgmtip and times:
 445             connection.execute("fping -c 1 -g %s > /dev/null 2>&1" % cidr)
 446             out = connection.execute("ip neighbor | grep '%s'" % mac)[1]
 447             LOG.info("fping -c 1 -g %s > /dev/null 2>&1", cidr)
 448             if out.strip():
 449                 mgmtip = str(out.split(" ")[0]).strip()
 450                 client = ssh.SSH.from_node(node, overrides={"ip": mgmtip})
 451                 client.wait()
 452                 break
 453
 454             time.sleep(WAIT_FOR_BOOT)  # FixMe: How to find if VM is booted?
 455             times = times - 1
 456         return mgmtip
 457
 458     @classmethod
 459     def wait_for_vnfs_to_start(cls, connection, servers, nodes):
 460         for node in nodes:
 461             vnf = servers[node["name"]]
 462             mgmtip = vnf["network_ports"]["mgmt"]["cidr"]
 463             ip = cls.get_mgmt_ip(connection, node["mac"], mgmtip, node)
 464             if ip:
 465                 node["ip"] = ip
 466         return nodes
 467
 468
 469 class Server(object):
 470     """ This class handles geting vnf nodes
 471     """
 472
 473     @staticmethod
 474     def build_vnf_interfaces(vnf, ports):
 475         interfaces = {}
 476         index = 0
 477
 478         for key, vfs in vnf["network_ports"].items():
 479             if key == "mgmt":
 480                 mgmtip = str(IPNetwork(vfs['cidr']).ip)
 481                 continue
 482
 483             vf = ports[vfs[0]]
 484             ip = IPNetwork(vf['cidr'])
 485             interfaces.update({
 486                 key: {
 487                     'vpci': vf['vpci'],
 488                     'driver': "%svf" % vf['driver'],
 489                     'local_mac': vf['mac'],
 490                     'dpdk_port_num': index,
 491                     'local_ip': str(ip.ip),
 492                     'netmask': str(ip.netmask)
 493                     },
 494             })
 495             index = index + 1
 496
 497         return mgmtip, interfaces
 498
 499     @classmethod
 500     def generate_vnf_instance(cls, flavor, ports, ip, key, vnf, mac):
 501         mgmtip, interfaces = cls.build_vnf_interfaces(vnf, ports)
 502
 503         result = {
 504             "ip": mgmtip,
 505             "mac": mac,
 506             "host": ip,
 507             "user": flavor.get('user', 'root'),
 508             "interfaces": interfaces,
 509             "routing_table": [],
 510             # empty IPv6 routing table
 511             "nd_route_tbl": [],
 512             "name": key, "role": key
 513         }
 514
 515         try:
 516             result['key_filename'] = flavor['key_filename']
 517         except KeyError:
 518             pass
 519
 520         try:
 521             result['password'] = flavor['password']
 522         except KeyError:
 523             pass
 524         LOG.info(result)
 525         return result
 526
 527
 528 class OvsDeploy(object):
 529     """ This class handles deploy of ovs dpdk
 530     Configuration: ovs_dpdk
 531     """
 532
 533     OVS_DEPLOY_SCRIPT = "ovs_deploy.bash"
 534
 535     def __init__(self, connection, bin_path, ovs_properties):
 536         self.connection = connection
 537         self.bin_path = bin_path
 538         self.ovs_properties = ovs_properties
 539
 540     def prerequisite(self):
 541         pkgs = ["git", "build-essential", "pkg-config", "automake",
 542                 "autotools-dev", "libltdl-dev", "cmake", "libnuma-dev",
 543                 "libpcap-dev"]
 544         StandaloneContextHelper.install_req_libs(self.connection, pkgs)
 545
 546     def ovs_deploy(self):
 547         ovs_deploy = os.path.join(constants.YARDSTICK_ROOT_PATH,
 548                                   "yardstick/resources/scripts/install/",
 549                                   self.OVS_DEPLOY_SCRIPT)
 550         if os.path.isfile(ovs_deploy):
 551             self.prerequisite()
 552             remote_ovs_deploy = os.path.join(self.bin_path, self.OVS_DEPLOY_SCRIPT)
 553             LOG.info(remote_ovs_deploy)
 554             self.connection.put(ovs_deploy, remote_ovs_deploy)
 555
 556             http_proxy = os.environ.get('http_proxy', '')
 557             ovs_details = self.ovs_properties.get("version", {})
 558             ovs = ovs_details.get("ovs", "2.6.0")
 559             dpdk = ovs_details.get("dpdk", "16.11.1")
 560
 561             cmd = "sudo -E %s --ovs='%s' --dpdk='%s' -p='%s'" % (remote_ovs_deploy,
 562                                                                  ovs, dpdk, http_proxy)
 563             exit_status, _, stderr = self.connection.execute(cmd)
 564             if exit_status:
 565                 raise exceptions.OVSDeployError(stderr=stderr)