yardstick/benchmark/contexts/standalone/model.py

   1 # Copyright (c) 2016-2017 Intel Corporation
   2 #
   3 # Licensed under the Apache License, Version 2.0 (the "License");
   4 # you may not use this file except in compliance with the License.
   5 # You may obtain a copy of the License at
   6 #
   7 #      http://www.apache.org/licenses/LICENSE-2.0
   8 #
   9 # Unless required by applicable law or agreed to in writing, software
  10 # distributed under the License is distributed on an "AS IS" BASIS,
  11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12 # See the License for the specific language governing permissions and
  13 # limitations under the License.
  14
  15 from __future__ import absolute_import
  16 import os
  17 import re
  18 import time
  19 import glob
  20 import uuid
  21 import random
  22 import logging
  23 import itertools
  24 import errno
  25
  26 from netaddr import IPNetwork
  27 import xml.etree.ElementTree as ET
  28
  29 from yardstick import ssh
  30 from yardstick.common.constants import YARDSTICK_ROOT_PATH
  31 from yardstick.common.yaml_loader import yaml_load
  32 from yardstick.network_services.utils import PciAddress
  33 from yardstick.common.utils import write_file
  34
  35 LOG = logging.getLogger(__name__)
  36
  37 VM_TEMPLATE = """
  38 <domain type="kvm">
  39  <name>{vm_name}</name>
  40   <uuid>{random_uuid}</uuid>
  41   <memory unit="MB">{memory}</memory>
  42   <currentMemory unit="MB">{memory}</currentMemory>
  43   <memoryBacking>
  44     <hugepages />
  45   </memoryBacking>
  46   <vcpu placement="static">{vcpu}</vcpu>
  47   <os>
  48     <type arch="x86_64" machine="pc-i440fx-utopic">hvm</type>
  49     <boot dev="hd" />
  50   </os>
  51   <features>
  52     <acpi />
  53     <apic />
  54     <pae />
  55   </features>
  56   <cpu mode='host-passthrough'>
  57     <topology cores="{cpu}" sockets="{socket}" threads="{threads}" />
  58     <numa>
  59        <cell id='0' cpus='{numa_cpus}' memory='{memory}' unit='MB' memAccess='shared'/>
  60     </numa>
  61   </cpu>
  62   <clock offset="utc">
  63     <timer name="rtc" tickpolicy="catchup" />
  64     <timer name="pit" tickpolicy="delay" />
  65     <timer name="hpet" present="no" />
  66   </clock>
  67   <on_poweroff>destroy</on_poweroff>
  68   <on_reboot>restart</on_reboot>
  69   <on_crash>restart</on_crash>
  70   <devices>
  71     <emulator>/usr/bin/kvm-spice</emulator>
  72     <disk device="disk" type="file">
  73       <driver name="qemu" type="qcow2" />
  74       <source file="{vm_image}"/>
  75       <target bus="virtio" dev="vda" />
  76     </disk>
  77     <graphics autoport="yes" listen="0.0.0.0" port="-1" type="vnc" />
  78     <interface type="bridge">
  79       <mac address='{mac_addr}'/>
  80       <source bridge="br-int" />
  81       <model type='virtio'/>
  82     </interface>
  83    </devices>
  84 </domain>
  85 """
  86 WAIT_FOR_BOOT = 30
  87
  88
  89 class Libvirt(object):
  90     """ This class handles all the libvirt updates to lauch VM
  91     """
  92
  93     @staticmethod
  94     def check_if_vm_exists_and_delete(vm_name, connection):
  95         cmd_template = "virsh list --name | grep -i %s"
  96         status = connection.execute(cmd_template % vm_name)[0]
  97         if status == 0:
  98             LOG.info("VM '%s' is already present.. destroying" % vm_name)
  99             connection.execute("virsh destroy %s" % vm_name)
 100
 101     @staticmethod
 102     def virsh_create_vm(connection, cfg):
 103         err = connection.execute("virsh create %s" % cfg)[0]
 104         LOG.info("VM create status: %s" % (err))
 105
 106     @staticmethod
 107     def virsh_destroy_vm(vm_name, connection):
 108         connection.execute("virsh destroy %s" % vm_name)
 109
 110     @staticmethod
 111     def add_interface_address(interface, pci_address):
 112         vm_pci = ET.SubElement(interface, 'address')
 113         vm_pci.set('type', 'pci')
 114         vm_pci.set('domain', '0x%s' % pci_address.domain)
 115         vm_pci.set('bus', '0x%s' % pci_address.bus)
 116         vm_pci.set('slot', '0x%s' % pci_address.slot)
 117         vm_pci.set('function', '0x%s' % pci_address.function)
 118         return vm_pci
 119
 120     @classmethod
 121     def add_ovs_interface(cls, vpath, port_num, vpci, vports_mac, xml):
 122         vhost_path = '{0}/var/run/openvswitch/dpdkvhostuser{1}'
 123         root = ET.parse(xml)
 124         pci_address = PciAddress.parse_address(vpci.strip(), multi_line=True)
 125         device = root.find('devices')
 126
 127         interface = ET.SubElement(device, 'interface')
 128         interface.set('type', 'vhostuser')
 129         mac = ET.SubElement(interface, 'mac')
 130         mac.set('address', vports_mac)
 131
 132         source = ET.SubElement(interface, 'source')
 133         source.set('type', 'unix')
 134         source.set('path', vhost_path.format(vpath, port_num))
 135         source.set('mode', 'client')
 136
 137         model = ET.SubElement(interface, 'model')
 138         model.set('type', 'virtio')
 139
 140         driver = ET.SubElement(interface, 'driver')
 141         driver.set('queues', '4')
 142
 143         host = ET.SubElement(driver, 'host')
 144         host.set('mrg_rxbuf', 'off')
 145
 146         cls.add_interface_address(interface, pci_address)
 147
 148         root.write(xml)
 149
 150     @classmethod
 151     def add_sriov_interfaces(cls, vm_pci, vf_pci, vfmac, xml):
 152         root = ET.parse(xml)
 153         pci_address = PciAddress.parse_address(vf_pci.strip(), multi_line=True)
 154         device = root.find('devices')
 155
 156         interface = ET.SubElement(device, 'interface')
 157         interface.set('managed', 'yes')
 158         interface.set('type', 'hostdev')
 159
 160         mac = ET.SubElement(interface, 'mac')
 161         mac.set('address', vfmac)
 162         source = ET.SubElement(interface, 'source')
 163
 164         addr = ET.SubElement(source, "address")
 165         addr.set('domain', "0x0")
 166         addr.set('bus', "{0}".format(pci_address.bus))
 167         addr.set('function', "{0}".format(pci_address.function))
 168         addr.set('slot', "0x{0}".format(pci_address.slot))
 169         addr.set('type', "pci")
 170
 171         pci_vm_address = PciAddress.parse_address(vm_pci.strip(), multi_line=True)
 172         cls.add_interface_address(interface, pci_vm_address)
 173
 174         root.write(xml)
 175
 176     @staticmethod
 177     def create_snapshot_qemu(connection, index, vm_image):
 178         # build snapshot image
 179         image = "/var/lib/libvirt/images/%s.qcow2" % index
 180         connection.execute("rm %s" % image)
 181         qemu_template = "qemu-img create -f qcow2 -o backing_file=%s %s"
 182         connection.execute(qemu_template % (vm_image, image))
 183
 184         return image
 185
 186     @classmethod
 187     def build_vm_xml(cls, connection, flavor, cfg, vm_name, index):
 188         memory = flavor.get('ram', '4096')
 189         extra_spec = flavor.get('extra_specs', {})
 190         cpu = extra_spec.get('hw:cpu_cores', '2')
 191         socket = extra_spec.get('hw:cpu_sockets', '1')
 192         threads = extra_spec.get('hw:cpu_threads', '2')
 193         vcpu = int(cpu) * int(threads)
 194         numa_cpus = '0-%s' % (vcpu - 1)
 195
 196         mac = StandaloneContextHelper.get_mac_address(0x00)
 197         image = cls.create_snapshot_qemu(connection, index,
 198                                          flavor.get("images", None))
 199         vm_xml = VM_TEMPLATE.format(
 200             vm_name=vm_name,
 201             random_uuid=uuid.uuid4(),
 202             mac_addr=mac,
 203             memory=memory, vcpu=vcpu, cpu=cpu,
 204             numa_cpus=numa_cpus,
 205             socket=socket, threads=threads,
 206             vm_image=image)
 207
 208         write_file(cfg, vm_xml)
 209
 210         return [vcpu, mac]
 211
 212     @staticmethod
 213     def split_cpu_list(cpu_list):
 214         if not cpu_list:
 215             return []
 216
 217         ranges = cpu_list.split(',')
 218         bounds = ([int(b) for b in r.split('-')] for r in ranges)
 219         range_objects = \
 220             (range(bound[0], bound[1] + 1 if len(bound) == 2
 221              else bound[0] + 1) for bound in bounds)
 222
 223         return sorted(itertools.chain.from_iterable(range_objects))
 224
 225     @classmethod
 226     def get_numa_nodes(cls):
 227         nodes_sysfs = glob.iglob("/sys/devices/system/node/node*")
 228         nodes = {}
 229         for node_sysfs in nodes_sysfs:
 230             num = os.path.basename(node_sysfs).replace("node", "")
 231             with open(os.path.join(node_sysfs, "cpulist")) as cpulist_file:
 232                 cpulist = cpulist_file.read().strip()
 233             nodes[num] = cls.split_cpu_list(cpulist)
 234         LOG.info("nodes: {0}".format(nodes))
 235         return nodes
 236
 237     @staticmethod
 238     def update_interrupts_hugepages_perf(connection):
 239         connection.execute("echo 1 > /sys/module/kvm/parameters/allow_unsafe_assigned_interrupts")
 240         connection.execute("echo never > /sys/kernel/mm/transparent_hugepage/enabled")
 241
 242     @classmethod
 243     def pin_vcpu_for_perf(cls, connection, vm_name, cpu):
 244         nodes = cls.get_numa_nodes()
 245         num_nodes = len(nodes)
 246         vcpi_pin_template = "virsh vcpupin {0} {1} {2}"
 247         for i in range(0, int(cpu)):
 248             core = nodes[str(num_nodes - 1)][i % len(nodes[str(num_nodes - 1)])]
 249             connection.execute(vcpi_pin_template.format(vm_name, i, core))
 250         cls.update_interrupts_hugepages_perf(connection)
 251
 252
 253 class StandaloneContextHelper(object):
 254     """ This class handles all the common code for standalone
 255     """
 256     def __init__(self):
 257         self.file_path = None
 258         super(StandaloneContextHelper, self).__init__()
 259
 260     @staticmethod
 261     def install_req_libs(connection, extra_pkgs=[]):
 262         pkgs = ["qemu-kvm", "libvirt-bin", "bridge-utils", "numactl", "fping"]
 263         pkgs.extend(extra_pkgs)
 264         cmd_template = "dpkg-query -W --showformat='${Status}\\n' \"%s\"|grep 'ok installed'"
 265         for pkg in pkgs:
 266             if connection.execute(cmd_template % pkg)[0]:
 267                 connection.execute("apt-get update")
 268                 connection.execute("apt-get -y install %s" % pkg)
 269         else:
 270             # all installed
 271             return
 272
 273     @staticmethod
 274     def get_kernel_module(connection, pci, driver):
 275         if not driver:
 276             out = connection.execute("lspci -k -s %s" % pci)[1]
 277             driver = out.split("Kernel modules:").pop().strip()
 278         return driver
 279
 280     @classmethod
 281     def get_nic_details(cls, connection, networks, dpdk_nic_bind):
 282         for key, ports in networks.items():
 283             if key == "mgmt":
 284                 continue
 285
 286             phy_ports = ports['phy_port']
 287             phy_driver = ports.get('phy_driver', None)
 288             driver = cls.get_kernel_module(connection, phy_ports, phy_driver)
 289
 290             # Make sure that ports are bound to kernel drivers e.g. i40e/ixgbe
 291             bind_cmd = "{dpdk_nic_bind} --force -b {driver} {port}"
 292             lshw_cmd = "lshw -c network -businfo | grep '{port}'"
 293             link_show_cmd = "ip -s link show {interface}"
 294
 295             cmd = bind_cmd.format(dpdk_nic_bind=dpdk_nic_bind,
 296                                   driver=driver, port=ports['phy_port'])
 297             connection.execute(cmd)
 298
 299             out = connection.execute(lshw_cmd.format(port=phy_ports))[1]
 300             interface = out.split()[1]
 301
 302             connection.execute(link_show_cmd.format(interface=interface))
 303
 304             ports.update({
 305                 'interface': str(interface),
 306                 'driver': driver
 307             })
 308         LOG.info("{0}".format(networks))
 309
 310         return networks
 311
 312     @staticmethod
 313     def get_virtual_devices(connection, pci):
 314         cmd = "cat /sys/bus/pci/devices/{0}/virtfn0/uevent"
 315         output = connection.execute(cmd.format(pci))[1]
 316
 317         pattern = "PCI_SLOT_NAME=({})".format(PciAddress.PCI_PATTERN_STR)
 318         m = re.search(pattern, output, re.MULTILINE)
 319
 320         pf_vfs = {}
 321         if m:
 322             pf_vfs = {pci: m.group(1).rstrip()}
 323
 324         LOG.info("pf_vfs:\n%s", pf_vfs)
 325
 326         return pf_vfs
 327
 328     def read_config_file(self):
 329         """Read from config file"""
 330
 331         with open(self.file_path) as stream:
 332             LOG.info("Parsing pod file: %s", self.file_path)
 333             cfg = yaml_load(stream)
 334         return cfg
 335
 336     def parse_pod_file(self, file_path, nfvi_role='Sriov'):
 337         self.file_path = file_path
 338         nodes = []
 339         nfvi_host = []
 340         try:
 341             cfg = self.read_config_file()
 342         except IOError as io_error:
 343             if io_error.errno != errno.ENOENT:
 344                 raise
 345             self.file_path = os.path.join(YARDSTICK_ROOT_PATH, file_path)
 346             cfg = self.read_config_file()
 347
 348         nodes.extend([node for node in cfg["nodes"] if str(node["role"]) != nfvi_role])
 349         nfvi_host.extend([node for node in cfg["nodes"] if str(node["role"]) == nfvi_role])
 350         if not nfvi_host:
 351             raise("Node role is other than SRIOV")
 352
 353         host_mgmt = {'user': nfvi_host[0]['user'],
 354                      'ip': str(IPNetwork(nfvi_host[0]['ip']).ip),
 355                      'password': nfvi_host[0]['password'],
 356                      'ssh_port': nfvi_host[0].get('ssh_port', 22),
 357                      'key_filename': nfvi_host[0].get('key_filename')}
 358
 359         return [nodes, nfvi_host, host_mgmt]
 360
 361     @staticmethod
 362     def get_mac_address(end=0x7f):
 363         mac = [0x52, 0x54, 0x00,
 364                random.randint(0x00, end),
 365                random.randint(0x00, 0xff),
 366                random.randint(0x00, 0xff)]
 367         mac_address = ':'.join('%02x' % x for x in mac)
 368         return mac_address
 369
 370     @staticmethod
 371     def get_mgmt_ip(connection, mac, cidr, node):
 372         mgmtip = None
 373         times = 10
 374         while not mgmtip and times:
 375             connection.execute("fping -c 1 -g %s > /dev/null 2>&1" % cidr)
 376             out = connection.execute("ip neighbor | grep '%s'" % mac)[1]
 377             LOG.info("fping -c 1 -g %s > /dev/null 2>&1" % cidr)
 378             if out.strip():
 379                 mgmtip = str(out.split(" ")[0]).strip()
 380                 client = ssh.SSH.from_node(node, overrides={"ip": mgmtip})
 381                 client.wait()
 382                 break
 383
 384             time.sleep(WAIT_FOR_BOOT)  # FixMe: How to find if VM is booted?
 385             times = times - 1
 386         return mgmtip
 387
 388     @classmethod
 389     def wait_for_vnfs_to_start(cls, connection, servers, nodes):
 390         for node in nodes:
 391             vnf = servers[node["name"]]
 392             mgmtip = vnf["network_ports"]["mgmt"]["cidr"]
 393             ip = cls.get_mgmt_ip(connection, node["mac"], mgmtip, node)
 394             if ip:
 395                 node["ip"] = ip
 396         return nodes
 397
 398
 399 class Server(object):
 400     """ This class handles geting vnf nodes
 401     """
 402
 403     @staticmethod
 404     def build_vnf_interfaces(vnf, ports):
 405         interfaces = {}
 406         index = 0
 407
 408         for key, vfs in vnf["network_ports"].items():
 409             if key == "mgmt":
 410                 mgmtip = str(IPNetwork(vfs['cidr']).ip)
 411                 continue
 412
 413             vf = ports[vfs[0]]
 414             ip = IPNetwork(vf['cidr'])
 415             interfaces.update({
 416                 key: {
 417                     'vpci': vf['vpci'],
 418                     'driver': "%svf" % vf['driver'],
 419                     'local_mac': vf['mac'],
 420                     'dpdk_port_num': index,
 421                     'local_ip': str(ip.ip),
 422                     'netmask': str(ip.netmask)
 423                     },
 424             })
 425             index = index + 1
 426
 427         return mgmtip, interfaces
 428
 429     @classmethod
 430     def generate_vnf_instance(cls, flavor, ports, ip, key, vnf, mac):
 431         mgmtip, interfaces = cls.build_vnf_interfaces(vnf, ports)
 432
 433         result = {
 434             "ip": mgmtip,
 435             "mac": mac,
 436             "host": ip,
 437             "user": flavor.get('user', 'root'),
 438             "interfaces": interfaces,
 439             "routing_table": [],
 440             # empty IPv6 routing table
 441             "nd_route_tbl": [],
 442             "name": key, "role": key
 443         }
 444
 445         try:
 446             result['key_filename'] = flavor['key_filename']
 447         except KeyError:
 448             pass
 449
 450         try:
 451             result['password'] = flavor['password']
 452         except KeyError:
 453             pass
 454         LOG.info(result)
 455         return result
 456
 457
 458 class OvsDeploy(object):
 459     """ This class handles deploy of ovs dpdk
 460     Configuration: ovs_dpdk
 461     """
 462
 463     OVS_DEPLOY_SCRIPT = "ovs_deploy.bash"
 464
 465     def __init__(self, connection, bin_path, ovs_properties):
 466         self.connection = connection
 467         self.bin_path = bin_path
 468         self.ovs_properties = ovs_properties
 469
 470     def prerequisite(self):
 471         pkgs = ["git", "build-essential", "pkg-config", "automake",
 472                 "autotools-dev", "libltdl-dev", "cmake", "libnuma-dev",
 473                 "libpcap-dev"]
 474         StandaloneContextHelper.install_req_libs(self.connection, pkgs)
 475
 476     def ovs_deploy(self):
 477         ovs_deploy = os.path.join(YARDSTICK_ROOT_PATH,
 478                                   "yardstick/resources/scripts/install/",
 479                                   self.OVS_DEPLOY_SCRIPT)
 480         if os.path.isfile(ovs_deploy):
 481             self.prerequisite()
 482             remote_ovs_deploy = os.path.join(self.bin_path, self.OVS_DEPLOY_SCRIPT)
 483             LOG.info(remote_ovs_deploy)
 484             self.connection.put(ovs_deploy, remote_ovs_deploy)
 485
 486             http_proxy = os.environ.get('http_proxy', '')
 487             ovs_details = self.ovs_properties.get("version", {})
 488             ovs = ovs_details.get("ovs", "2.6.0")
 489             dpdk = ovs_details.get("dpdk", "16.11.1")
 490
 491             cmd = "sudo -E %s --ovs='%s' --dpdk='%s' -p='%s'" % (remote_ovs_deploy,
 492                                                                  ovs, dpdk, http_proxy)
 493             self.connection.execute(cmd)