yardstick/benchmark/contexts/standalone/model.py

   1 # Copyright (c) 2016-2017 Intel Corporation
   2 #
   3 # Licensed under the Apache License, Version 2.0 (the "License");
   4 # you may not use this file except in compliance with the License.
   5 # You may obtain a copy of the License at
   6 #
   7 #      http://www.apache.org/licenses/LICENSE-2.0
   8 #
   9 # Unless required by applicable law or agreed to in writing, software
  10 # distributed under the License is distributed on an "AS IS" BASIS,
  11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12 # See the License for the specific language governing permissions and
  13 # limitations under the License.
  14
  15 import os
  16 import re
  17 import time
  18 import uuid
  19 import random
  20 import logging
  21 import errno
  22
  23 from netaddr import IPNetwork
  24 import xml.etree.ElementTree as ET
  25
  26 from yardstick import ssh
  27 from yardstick.common import constants
  28 from yardstick.common import exceptions
  29 from yardstick.common.yaml_loader import yaml_load
  30 from yardstick.network_services.utils import PciAddress
  31 from yardstick.network_services.helpers.cpu import CpuSysCores
  32
  33
  34 LOG = logging.getLogger(__name__)
  35
  36 VM_TEMPLATE = """
  37 <domain type="kvm">
  38   <name>{vm_name}</name>
  39   <uuid>{random_uuid}</uuid>
  40   <memory unit="MB">{memory}</memory>
  41   <currentMemory unit="MB">{memory}</currentMemory>
  42   <memoryBacking>
  43     <hugepages />
  44   </memoryBacking>
  45   <vcpu cpuset='{cpuset}'>{vcpu}</vcpu>
  46  {cputune}
  47   <os>
  48     <type arch="x86_64" machine="pc-i440fx-utopic">hvm</type>
  49     <boot dev="hd" />
  50   </os>
  51   <features>
  52     <acpi />
  53     <apic />
  54     <pae />
  55   </features>
  56   <cpu mode='host-passthrough'>
  57     <topology cores="{cpu}" sockets="{socket}" threads="{threads}" />
  58     <numa>
  59        <cell id='0' cpus='{numa_cpus}' memory='{memory}' unit='MB' memAccess='shared'/>
  60     </numa>
  61   </cpu>
  62   <clock offset="utc">
  63     <timer name="rtc" tickpolicy="catchup" />
  64     <timer name="pit" tickpolicy="delay" />
  65     <timer name="hpet" present="no" />
  66   </clock>
  67   <on_poweroff>destroy</on_poweroff>
  68   <on_reboot>restart</on_reboot>
  69   <on_crash>restart</on_crash>
  70   <devices>
  71     <emulator>/usr/bin/kvm-spice</emulator>
  72     <disk device="disk" type="file">
  73       <driver name="qemu" type="qcow2" />
  74       <source file="{vm_image}"/>
  75       <target bus="virtio" dev="vda" />
  76     </disk>
  77     <graphics autoport="yes" listen="0.0.0.0" port="-1" type="vnc" />
  78     <interface type="bridge">
  79       <mac address='{mac_addr}'/>
  80       <source bridge="br-int" />
  81       <model type='virtio'/>
  82     </interface>
  83     <serial type='pty'>
  84       <target port='0'/>
  85     </serial>
  86     <console type='pty'>
  87       <target type='serial' port='0'/>
  88     </console>
  89   </devices>
  90 </domain>
  91 """
  92 WAIT_FOR_BOOT = 30
  93
  94
  95 class Libvirt(object):
  96     """ This class handles all the libvirt updates to lauch VM
  97     """
  98
  99     @staticmethod
 100     def check_if_vm_exists_and_delete(vm_name, connection):
 101         cmd_template = "virsh list --name | grep -i %s"
 102         status = connection.execute(cmd_template % vm_name)[0]
 103         if status == 0:
 104             LOG.info("VM '%s' is already present... destroying", vm_name)
 105             connection.execute("virsh destroy %s" % vm_name)
 106
 107     @staticmethod
 108     def virsh_create_vm(connection, cfg):
 109         LOG.info('VM create, XML config: %s', cfg)
 110         status, _, error = connection.execute('virsh create %s' % cfg)
 111         if status:
 112             raise exceptions.LibvirtCreateError(error=error)
 113
 114     @staticmethod
 115     def virsh_destroy_vm(vm_name, connection):
 116         connection.execute("virsh destroy %s" % vm_name)
 117
 118     @staticmethod
 119     def _add_interface_address(interface, pci_address):
 120         """Add a PCI 'address' XML node
 121
 122         <address type='pci' domain='0x0000' bus='0x00' slot='0x08'
 123          function='0x0'/>
 124
 125         Refence: https://software.intel.com/en-us/articles/
 126                  configure-sr-iov-network-virtual-functions-in-linux-kvm
 127         """
 128         vm_pci = ET.SubElement(interface, 'address')
 129         vm_pci.set('type', 'pci')
 130         vm_pci.set('domain', '0x{}'.format(pci_address.domain))
 131         vm_pci.set('bus', '0x{}'.format(pci_address.bus))
 132         vm_pci.set('slot', '0x{}'.format(pci_address.slot))
 133         vm_pci.set('function', '0x{}'.format(pci_address.function))
 134         return vm_pci
 135
 136     @classmethod
 137     def add_ovs_interface(cls, vpath, port_num, vpci, vports_mac, xml_str):
 138         """Add a DPDK OVS 'interface' XML node in 'devices' node
 139
 140         <devices>
 141             <interface type='vhostuser'>
 142                 <mac address='00:00:00:00:00:01'/>
 143                 <source type='unix' path='/usr/local/var/run/openvswitch/
 144                  dpdkvhostuser0' mode='client'/>
 145                 <model type='virtio'/>
 146                 <driver queues='4'>
 147                     <host mrg_rxbuf='off'/>
 148                 </driver>
 149                 <address type='pci' domain='0x0000' bus='0x00' slot='0x03'
 150                  function='0x0'/>
 151             </interface>
 152             ...
 153         </devices>
 154
 155         Reference: http://docs.openvswitch.org/en/latest/topics/dpdk/
 156                    vhost-user/
 157         """
 158
 159         vhost_path = ('{0}/var/run/openvswitch/dpdkvhostuser{1}'.
 160                       format(vpath, port_num))
 161         root = ET.fromstring(xml_str)
 162         pci_address = PciAddress(vpci.strip())
 163         device = root.find('devices')
 164
 165         interface = ET.SubElement(device, 'interface')
 166         interface.set('type', 'vhostuser')
 167         mac = ET.SubElement(interface, 'mac')
 168         mac.set('address', vports_mac)
 169
 170         source = ET.SubElement(interface, 'source')
 171         source.set('type', 'unix')
 172         source.set('path', vhost_path)
 173         source.set('mode', 'client')
 174
 175         model = ET.SubElement(interface, 'model')
 176         model.set('type', 'virtio')
 177
 178         driver = ET.SubElement(interface, 'driver')
 179         driver.set('queues', '4')
 180
 181         host = ET.SubElement(driver, 'host')
 182         host.set('mrg_rxbuf', 'off')
 183
 184         cls._add_interface_address(interface, pci_address)
 185
 186         return ET.tostring(root)
 187
 188     @classmethod
 189     def add_sriov_interfaces(cls, vm_pci, vf_pci, vf_mac, xml_str):
 190         """Add a SR-IOV 'interface' XML node in 'devices' node
 191
 192         <devices>
 193            <interface type='hostdev' managed='yes'>
 194              <source>
 195                <address type='pci' domain='0x0000' bus='0x00' slot='0x03'
 196                 function='0x0'/>
 197              </source>
 198              <mac address='52:54:00:6d:90:02'>
 199              <address type='pci' domain='0x0000' bus='0x02' slot='0x04'
 200               function='0x1'/>
 201            </interface>
 202            ...
 203          </devices>
 204
 205         Reference: https://access.redhat.com/documentation/en-us/
 206             red_hat_enterprise_linux/6/html/
 207             virtualization_host_configuration_and_guest_installation_guide/
 208             sect-virtualization_host_configuration_and_guest_installation_guide
 209             -sr_iov-how_sr_iov_libvirt_works
 210         """
 211
 212         root = ET.fromstring(xml_str)
 213         device = root.find('devices')
 214
 215         interface = ET.SubElement(device, 'interface')
 216         interface.set('managed', 'yes')
 217         interface.set('type', 'hostdev')
 218
 219         mac = ET.SubElement(interface, 'mac')
 220         mac.set('address', vf_mac)
 221
 222         source = ET.SubElement(interface, 'source')
 223         pci_address = PciAddress(vf_pci.strip())
 224         cls._add_interface_address(source, pci_address)
 225
 226         pci_vm_address = PciAddress(vm_pci.strip())
 227         cls._add_interface_address(interface, pci_vm_address)
 228
 229         return ET.tostring(root)
 230
 231     @staticmethod
 232     def create_snapshot_qemu(connection, index, vm_image):
 233         # build snapshot image
 234         image = "/var/lib/libvirt/images/%s.qcow2" % index
 235         connection.execute("rm %s" % image)
 236         qemu_template = "qemu-img create -f qcow2 -o backing_file=%s %s"
 237         connection.execute(qemu_template % (vm_image, image))
 238
 239         return image
 240
 241     @classmethod
 242     def build_vm_xml(cls, connection, flavor, vm_name, index):
 243         """Build the XML from the configuration parameters"""
 244         memory = flavor.get('ram', '4096')
 245         extra_spec = flavor.get('extra_specs', {})
 246         cpu = extra_spec.get('hw:cpu_cores', '2')
 247         socket = extra_spec.get('hw:cpu_sockets', '1')
 248         threads = extra_spec.get('hw:cpu_threads', '2')
 249         vcpu = int(cpu) * int(threads)
 250         numa_cpus = '0-%s' % (vcpu - 1)
 251         hw_socket = flavor.get('hw_socket', '0')
 252         cpuset = Libvirt.pin_vcpu_for_perf(connection, hw_socket)
 253
 254         cputune = extra_spec.get('cputune', '')
 255         mac = StandaloneContextHelper.get_mac_address(0x00)
 256         image = cls.create_snapshot_qemu(connection, index,
 257                                          flavor.get("images", None))
 258         vm_xml = VM_TEMPLATE.format(
 259             vm_name=vm_name,
 260             random_uuid=uuid.uuid4(),
 261             mac_addr=mac,
 262             memory=memory, vcpu=vcpu, cpu=cpu,
 263             numa_cpus=numa_cpus,
 264             socket=socket, threads=threads,
 265             vm_image=image, cpuset=cpuset, cputune=cputune)
 266
 267         return vm_xml, mac
 268
 269     @staticmethod
 270     def update_interrupts_hugepages_perf(connection):
 271         connection.execute("echo 1 > /sys/module/kvm/parameters/allow_unsafe_assigned_interrupts")
 272         connection.execute("echo never > /sys/kernel/mm/transparent_hugepage/enabled")
 273
 274     @classmethod
 275     def pin_vcpu_for_perf(cls, connection, socket='0'):
 276         threads = ""
 277         sys_obj = CpuSysCores(connection)
 278         soc_cpu = sys_obj.get_core_socket()
 279         sys_cpu = int(soc_cpu["cores_per_socket"])
 280         socket = str(socket)
 281         cores = "%s-%s" % (soc_cpu[socket][0], soc_cpu[socket][sys_cpu - 1])
 282         if int(soc_cpu["thread_per_core"]) > 1:
 283             threads = "%s-%s" % (soc_cpu[socket][sys_cpu], soc_cpu[socket][-1])
 284         cpuset = "%s,%s" % (cores, threads)
 285         return cpuset
 286
 287     @classmethod
 288     def write_file(cls, file_name, xml_str):
 289         """Dump a XML string to a file"""
 290         root = ET.fromstring(xml_str)
 291         et = ET.ElementTree(element=root)
 292         et.write(file_name, encoding='utf-8', method='xml')
 293
 294
 295 class StandaloneContextHelper(object):
 296     """ This class handles all the common code for standalone
 297     """
 298     def __init__(self):
 299         self.file_path = None
 300         super(StandaloneContextHelper, self).__init__()
 301
 302     @staticmethod
 303     def install_req_libs(connection, extra_pkgs=None):
 304         extra_pkgs = extra_pkgs or []
 305         pkgs = ["qemu-kvm", "libvirt-bin", "bridge-utils", "numactl", "fping"]
 306         pkgs.extend(extra_pkgs)
 307         cmd_template = "dpkg-query -W --showformat='${Status}\\n' \"%s\"|grep 'ok installed'"
 308         for pkg in pkgs:
 309             if connection.execute(cmd_template % pkg)[0]:
 310                 connection.execute("apt-get update")
 311                 connection.execute("apt-get -y install %s" % pkg)
 312
 313     @staticmethod
 314     def get_kernel_module(connection, pci, driver):
 315         if not driver:
 316             out = connection.execute("lspci -k -s %s" % pci)[1]
 317             driver = out.split("Kernel modules:").pop().strip()
 318         return driver
 319
 320     @classmethod
 321     def get_nic_details(cls, connection, networks, dpdk_devbind):
 322         for key, ports in networks.items():
 323             if key == "mgmt":
 324                 continue
 325
 326             phy_ports = ports['phy_port']
 327             phy_driver = ports.get('phy_driver', None)
 328             driver = cls.get_kernel_module(connection, phy_ports, phy_driver)
 329
 330             # Make sure that ports are bound to kernel drivers e.g. i40e/ixgbe
 331             bind_cmd = "{dpdk_devbind} --force -b {driver} {port}"
 332             lshw_cmd = "lshw -c network -businfo | grep '{port}'"
 333             link_show_cmd = "ip -s link show {interface}"
 334
 335             cmd = bind_cmd.format(dpdk_devbind=dpdk_devbind,
 336                                   driver=driver, port=ports['phy_port'])
 337             connection.execute(cmd)
 338
 339             out = connection.execute(lshw_cmd.format(port=phy_ports))[1]
 340             interface = out.split()[1]
 341
 342             connection.execute(link_show_cmd.format(interface=interface))
 343
 344             ports.update({
 345                 'interface': str(interface),
 346                 'driver': driver
 347             })
 348         LOG.info(networks)
 349
 350         return networks
 351
 352     @staticmethod
 353     def get_virtual_devices(connection, pci):
 354         cmd = "cat /sys/bus/pci/devices/{0}/virtfn0/uevent"
 355         output = connection.execute(cmd.format(pci))[1]
 356
 357         pattern = "PCI_SLOT_NAME=({})".format(PciAddress.PCI_PATTERN_STR)
 358         m = re.search(pattern, output, re.MULTILINE)
 359
 360         pf_vfs = {}
 361         if m:
 362             pf_vfs = {pci: m.group(1).rstrip()}
 363
 364         LOG.info("pf_vfs:\n%s", pf_vfs)
 365
 366         return pf_vfs
 367
 368     def read_config_file(self):
 369         """Read from config file"""
 370
 371         with open(self.file_path) as stream:
 372             LOG.info("Parsing pod file: %s", self.file_path)
 373             cfg = yaml_load(stream)
 374         return cfg
 375
 376     def parse_pod_file(self, file_path, nfvi_role='Sriov'):
 377         self.file_path = file_path
 378         nodes = []
 379         nfvi_host = []
 380         try:
 381             cfg = self.read_config_file()
 382         except IOError as io_error:
 383             if io_error.errno != errno.ENOENT:
 384                 raise
 385             self.file_path = os.path.join(constants.YARDSTICK_ROOT_PATH,
 386                                           file_path)
 387             cfg = self.read_config_file()
 388
 389         nodes.extend([node for node in cfg["nodes"] if str(node["role"]) != nfvi_role])
 390         nfvi_host.extend([node for node in cfg["nodes"] if str(node["role"]) == nfvi_role])
 391         if not nfvi_host:
 392             raise("Node role is other than SRIOV")
 393
 394         host_mgmt = {'user': nfvi_host[0]['user'],
 395                      'ip': str(IPNetwork(nfvi_host[0]['ip']).ip),
 396                      'password': nfvi_host[0]['password'],
 397                      'ssh_port': nfvi_host[0].get('ssh_port', 22),
 398                      'key_filename': nfvi_host[0].get('key_filename')}
 399
 400         return [nodes, nfvi_host, host_mgmt]
 401
 402     @staticmethod
 403     def get_mac_address(end=0x7f):
 404         mac = [0x52, 0x54, 0x00,
 405                random.randint(0x00, end),
 406                random.randint(0x00, 0xff),
 407                random.randint(0x00, 0xff)]
 408         mac_address = ':'.join('%02x' % x for x in mac)
 409         return mac_address
 410
 411     @staticmethod
 412     def get_mgmt_ip(connection, mac, cidr, node):
 413         mgmtip = None
 414         times = 10
 415         while not mgmtip and times:
 416             connection.execute("fping -c 1 -g %s > /dev/null 2>&1" % cidr)
 417             out = connection.execute("ip neighbor | grep '%s'" % mac)[1]
 418             LOG.info("fping -c 1 -g %s > /dev/null 2>&1", cidr)
 419             if out.strip():
 420                 mgmtip = str(out.split(" ")[0]).strip()
 421                 client = ssh.SSH.from_node(node, overrides={"ip": mgmtip})
 422                 client.wait()
 423                 break
 424
 425             time.sleep(WAIT_FOR_BOOT)  # FixMe: How to find if VM is booted?
 426             times = times - 1
 427         return mgmtip
 428
 429     @classmethod
 430     def wait_for_vnfs_to_start(cls, connection, servers, nodes):
 431         for node in nodes:
 432             vnf = servers[node["name"]]
 433             mgmtip = vnf["network_ports"]["mgmt"]["cidr"]
 434             ip = cls.get_mgmt_ip(connection, node["mac"], mgmtip, node)
 435             if ip:
 436                 node["ip"] = ip
 437         return nodes
 438
 439
 440 class Server(object):
 441     """ This class handles geting vnf nodes
 442     """
 443
 444     @staticmethod
 445     def build_vnf_interfaces(vnf, ports):
 446         interfaces = {}
 447         index = 0
 448
 449         for key, vfs in vnf["network_ports"].items():
 450             if key == "mgmt":
 451                 mgmtip = str(IPNetwork(vfs['cidr']).ip)
 452                 continue
 453
 454             vf = ports[vfs[0]]
 455             ip = IPNetwork(vf['cidr'])
 456             interfaces.update({
 457                 key: {
 458                     'vpci': vf['vpci'],
 459                     'driver': "%svf" % vf['driver'],
 460                     'local_mac': vf['mac'],
 461                     'dpdk_port_num': index,
 462                     'local_ip': str(ip.ip),
 463                     'netmask': str(ip.netmask)
 464                     },
 465             })
 466             index = index + 1
 467
 468         return mgmtip, interfaces
 469
 470     @classmethod
 471     def generate_vnf_instance(cls, flavor, ports, ip, key, vnf, mac):
 472         mgmtip, interfaces = cls.build_vnf_interfaces(vnf, ports)
 473
 474         result = {
 475             "ip": mgmtip,
 476             "mac": mac,
 477             "host": ip,
 478             "user": flavor.get('user', 'root'),
 479             "interfaces": interfaces,
 480             "routing_table": [],
 481             # empty IPv6 routing table
 482             "nd_route_tbl": [],
 483             "name": key, "role": key
 484         }
 485
 486         try:
 487             result['key_filename'] = flavor['key_filename']
 488         except KeyError:
 489             pass
 490
 491         try:
 492             result['password'] = flavor['password']
 493         except KeyError:
 494             pass
 495         LOG.info(result)
 496         return result
 497
 498
 499 class OvsDeploy(object):
 500     """ This class handles deploy of ovs dpdk
 501     Configuration: ovs_dpdk
 502     """
 503
 504     OVS_DEPLOY_SCRIPT = "ovs_deploy.bash"
 505
 506     def __init__(self, connection, bin_path, ovs_properties):
 507         self.connection = connection
 508         self.bin_path = bin_path
 509         self.ovs_properties = ovs_properties
 510
 511     def prerequisite(self):
 512         pkgs = ["git", "build-essential", "pkg-config", "automake",
 513                 "autotools-dev", "libltdl-dev", "cmake", "libnuma-dev",
 514                 "libpcap-dev"]
 515         StandaloneContextHelper.install_req_libs(self.connection, pkgs)
 516
 517     def ovs_deploy(self):
 518         ovs_deploy = os.path.join(constants.YARDSTICK_ROOT_PATH,
 519                                   "yardstick/resources/scripts/install/",
 520                                   self.OVS_DEPLOY_SCRIPT)
 521         if os.path.isfile(ovs_deploy):
 522             self.prerequisite()
 523             remote_ovs_deploy = os.path.join(self.bin_path, self.OVS_DEPLOY_SCRIPT)
 524             LOG.info(remote_ovs_deploy)
 525             self.connection.put(ovs_deploy, remote_ovs_deploy)
 526
 527             http_proxy = os.environ.get('http_proxy', '')
 528             ovs_details = self.ovs_properties.get("version", {})
 529             ovs = ovs_details.get("ovs", "2.6.0")
 530             dpdk = ovs_details.get("dpdk", "16.11.1")
 531
 532             cmd = "sudo -E %s --ovs='%s' --dpdk='%s' -p='%s'" % (remote_ovs_deploy,
 533                                                                  ovs, dpdk, http_proxy)
 534             exit_status, _, stderr = self.connection.execute(cmd)
 535             if exit_status:
 536                 raise exceptions.OVSDeployError(stderr=stderr)