yardstick/benchmark/contexts/sriov.py

   1 # Copyright (c) 2016-2017 Intel Corporation
   2 #
   3 # Licensed under the Apache License, Version 2.0 (the "License");
   4 # you may not use this file except in compliance with the License.
   5 # You may obtain a copy of the License at
   6 #
   7 #      http://www.apache.org/licenses/LICENSE-2.0
   8 #
   9 # Unless required by applicable law or agreed to in writing, software
  10 # distributed under the License is distributed on an "AS IS" BASIS,
  11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12 # See the License for the specific language governing permissions and
  13 # limitations under the License.
  14
  15 from __future__ import absolute_import
  16 import os
  17 import yaml
  18 import re
  19 import time
  20 import glob
  21 import uuid
  22 import random
  23 import logging
  24 import itertools
  25 import xml.etree.ElementTree as ET
  26 from yardstick import ssh
  27 from yardstick.network_services.utils import get_nsb_option
  28 from yardstick.network_services.utils import provision_tool
  29 from yardstick.benchmark.contexts.standalone import StandaloneContext
  30
  31 log = logging.getLogger(__name__)
  32
  33 VM_TEMPLATE = """
  34 <domain type="kvm">
  35  <name>vm1</name>
  36   <uuid>{random_uuid}</uuid>
  37   <memory unit="KiB">102400</memory>
  38   <currentMemory unit="KiB">102400</currentMemory>
  39   <memoryBacking>
  40     <hugepages />
  41   </memoryBacking>
  42   <vcpu placement="static">20</vcpu>
  43   <os>
  44     <type arch="x86_64" machine="pc-i440fx-utopic">hvm</type>
  45     <boot dev="hd" />
  46   </os>
  47   <features>
  48     <acpi />
  49     <apic />
  50     <pae />
  51   </features>
  52   <cpu match="exact" mode="custom">
  53     <model fallback="allow">SandyBridge</model>
  54     <topology cores="10" sockets="1" threads="2" />
  55   </cpu>
  56   <clock offset="utc">
  57     <timer name="rtc" tickpolicy="catchup" />
  58     <timer name="pit" tickpolicy="delay" />
  59     <timer name="hpet" present="no" />
  60   </clock>
  61   <on_poweroff>destroy</on_poweroff>
  62   <on_reboot>restart</on_reboot>
  63   <on_crash>restart</on_crash>
  64   <devices>
  65     <emulator>/usr/bin/kvm-spice</emulator>
  66     <disk device="disk" type="file">
  67       <driver name="qemu" type="qcow2" />
  68       <source file="{vm_image}"/>
  69       <target bus="virtio" dev="vda" />
  70       <address bus="0x00" domain="0x0000"
  71 function="0x0" slot="0x04" type="pci" />
  72     </disk>
  73     <controller index="0" model="ich9-ehci1" type="usb">
  74       <address bus="0x00" domain="0x0000"
  75 function="0x7" slot="0x05" type="pci" />
  76     </controller>
  77     <controller index="0" model="ich9-uhci1" type="usb">
  78       <master startport="0" />
  79       <address bus="0x00" domain="0x0000" function="0x0"
  80 multifunction="on" slot="0x05" type="pci" />
  81     </controller>
  82     <controller index="0" model="ich9-uhci2" type="usb">
  83       <master startport="2" />
  84       <address bus="0x00" domain="0x0000"
  85 function="0x1" slot="0x05" type="pci" />
  86     </controller>
  87     <controller index="0" model="ich9-uhci3" type="usb">
  88       <master startport="4" />
  89       <address bus="0x00" domain="0x0000"
  90 function="0x2" slot="0x05" type="pci" />
  91     </controller>
  92     <controller index="0" model="pci-root" type="pci" />
  93       <serial type="pty">
  94       <target port="0" />
  95     </serial>
  96     <console type="pty">
  97       <target port="0" type="serial" />
  98     </console>
  99     <input bus="usb" type="tablet" />
 100     <input bus="ps2" type="mouse" />
 101     <input bus="ps2" type="keyboard" />
 102     <graphics autoport="yes" listen="0.0.0.0" port="-1" type="vnc" />
 103     <video>
 104       <model heads="1" type="cirrus" vram="16384" />
 105       <address bus="0x00" domain="0x0000"
 106 function="0x0" slot="0x02" type="pci" />
 107     </video>
 108     <memballoon model="virtio">
 109       <address bus="0x00" domain="0x0000"
 110 function="0x0" slot="0x06" type="pci" />
 111     </memballoon>
 112     <interface type="bridge">
 113       <mac address="{mac_addr}" />
 114       <source bridge="virbr0" />
 115     </interface>
 116    </devices>
 117 </domain>
 118 """
 119
 120
 121 class Sriov(StandaloneContext):
 122     def __init__(self):
 123         self.name = None
 124         self.file_path = None
 125         self.nodes = []
 126         self.vm_deploy = False
 127         self.sriov = []
 128         self.first_run = True
 129         self.dpdk_nic_bind = ""
 130         self.user = ""
 131         self.ssh_ip = ""
 132         self.passwd = ""
 133         self.ssh_port = ""
 134         self.auth_type = ""
 135
 136     def init(self):
 137         self.parse_pod_and_get_data(self.file_path)
 138
 139     def parse_pod_and_get_data(self, file_path):
 140         self.file_path = file_path
 141         log.debug("parsing pod file: {0}".format(self.file_path))
 142         try:
 143             with open(self.file_path) as stream:
 144                 cfg = yaml.load(stream)
 145         except IOError:
 146             log.error("File {0} does not exist".format(self.file_path))
 147             raise
 148
 149         self.nodes.extend([node for node in cfg["nodes"]
 150                            if node["role"] != "Sriov"])
 151         self.sriov.extend([node for node in cfg["nodes"]
 152                            if node["role"] == "Sriov"])
 153         self.user = self.sriov[0]['user']
 154         self.ssh_ip = self.sriov[0]['ip']
 155         if self.sriov[0]['auth_type'] == "password":
 156             self.passwd = self.sriov[0]['password']
 157         else:
 158             self.ssh_port = self.sriov[0]['ssh_port']
 159             self.key_filename = self.sriov[0]['key_filename']
 160
 161     def ssh_remote_machine(self):
 162         if self.sriov[0]['auth_type'] == "password":
 163             self.connection = ssh.SSH(
 164                 self.user,
 165                 self.ssh_ip,
 166                 password=self.passwd)
 167             self.connection.wait()
 168         else:
 169             if self.ssh_port is not None:
 170                 ssh_port = self.ssh_port
 171             else:
 172                 ssh_port = ssh.DEFAULT_PORT
 173             self.connection = ssh.SSH(
 174                 self.user,
 175                 self.ssh_ip,
 176                 port=ssh_port,
 177                 key_filename=self.key_filename)
 178             self.connection.wait()
 179         self.dpdk_nic_bind = provision_tool(
 180             self.connection,
 181             os.path.join(get_nsb_option("bin_path"), "dpdk_nic_bind.py"))
 182
 183     def get_nic_details(self):
 184         nic_details = {}
 185         nic_details = {
 186             'interface': {},
 187             'pci': self.sriov[0]['phy_ports'],
 188             'phy_driver': self.sriov[0]['phy_driver'],
 189             'vf_macs': self.sriov[0]['vf_macs']
 190         }
 191         #   Make sure that ports are bound to kernel drivers e.g. i40e/ixgbe
 192         for i, _ in enumerate(nic_details['pci']):
 193             err, out, _ = self.connection.execute(
 194                 "{dpdk_nic_bind} --force -b {driver} {port}".format(
 195                     dpdk_nic_bind=self.dpdk_nic_bind,
 196                     driver=self.sriov[0]['phy_driver'],
 197                     port=self.sriov[0]['phy_ports'][i]))
 198             err, out, _ = self.connection.execute(
 199                 "lshw -c network -businfo | grep '{port}'".format(
 200                     port=self.sriov[0]['phy_ports'][i]))
 201             a = out.split()[1]
 202             err, out, _ = self.connection.execute(
 203                 "ip -s link show {interface}".format(
 204                     interface=out.split()[1]))
 205             nic_details['interface'][i] = str(a)
 206         log.info("{0}".format(nic_details))
 207         return nic_details
 208
 209     def install_req_libs(self):
 210         if self.first_run:
 211             log.info("Installing required libraries...")
 212             err, out, _ = self.connection.execute("apt-get update")
 213             log.debug("{0}".format(out))
 214             err, out, _ = self.connection.execute(
 215                 "apt-get -y install qemu-kvm libvirt-bin")
 216             log.debug("{0}".format(out))
 217             err, out, _ = self.connection.execute(
 218                 "apt-get -y install libvirt-dev  bridge-utils numactl")
 219             log.debug("{0}".format(out))
 220             self.first_run = False
 221
 222     def configure_nics_for_sriov(self, host_driver, nic_details):
 223         vf_pci = [[], []]
 224         self.connection.execute(
 225             "rmmod {0}".format(host_driver))[1].splitlines()
 226         self.connection.execute(
 227             "modprobe {0} num_vfs=1".format(host_driver))[1].splitlines()
 228         nic_details['vf_pci'] = {}
 229         for i in range(len(nic_details['pci'])):
 230             self.connection.execute(
 231                 "echo 1 > /sys/bus/pci/devices/{0}/sriov_numvfs".format(
 232                     nic_details['pci'][i]))
 233             err, out, _ = self.connection.execute(
 234                 "ip link set {interface} vf 0 mac {mac}".format(
 235                     interface=nic_details['interface'][i],
 236                     mac=nic_details['vf_macs'][i]))
 237             time.sleep(3)
 238             vf_pci[i] = self.get_vf_datas(
 239                 'vf_pci',
 240                 nic_details['pci'][i],
 241                 nic_details['vf_macs'][i])
 242             nic_details['vf_pci'][i] = vf_pci[i]
 243         log.debug("NIC DETAILS : {0}".format(nic_details))
 244         return nic_details
 245
 246     def setup_sriov_context(self, pcis, nic_details, host_driver):
 247         blacklist = "/etc/modprobe.d/blacklist.conf"
 248
 249         #   1 : Blacklist the vf driver in /etc/modprobe.d/blacklist.conf
 250         vfnic = "{0}vf".format(host_driver)
 251         lines = self.read_from_file(blacklist)
 252         if vfnic not in lines:
 253             vfblacklist = "blacklist {vfnic}".format(vfnic=vfnic)
 254             self.connection.execute(
 255                 "echo {vfblacklist} >> {blacklist}".format(
 256                     vfblacklist=vfblacklist,
 257                     blacklist=blacklist))
 258
 259         #   2 : modprobe host_driver with num_vfs
 260         nic_details = self.configure_nics_for_sriov(host_driver, nic_details)
 261
 262         #   3: Setup vm_sriov.xml to launch VM
 263         cfg_sriov = '/tmp/vm_sriov.xml'
 264         mac = [0x00, 0x24, 0x81,
 265                random.randint(0x00, 0x7f),
 266                random.randint(0x00, 0xff),
 267                random.randint(0x00, 0xff)]
 268         mac_address = ':'.join(map(lambda x: "%02x" % x, mac))
 269         vm_sriov_xml = VM_TEMPLATE.format(
 270             random_uuid=uuid.uuid4(),
 271             mac_addr=mac_address,
 272             vm_image=self.sriov[0]["images"])
 273         with open(cfg_sriov, 'w') as f:
 274             f.write(vm_sriov_xml)
 275
 276         vf = nic_details['vf_pci']
 277         for index in range(len(nic_details['vf_pci'])):
 278             self.add_sriov_interface(
 279                 index,
 280                 vf[index]['vf_pci'],
 281                 mac_address,
 282                 "/tmp/vm_sriov.xml")
 283             self.connection.execute(
 284                 "ifconfig {interface} up".format(
 285                     interface=nic_details['interface'][index]))
 286
 287         #   4: Create and start the VM
 288         self.connection.put(cfg_sriov, cfg_sriov)
 289         time.sleep(10)
 290         err, out = self.check_output("virsh list --name | grep -i vm1")
 291         try:
 292             if out == "vm1":
 293                 log.info("VM is already present")
 294             else:
 295                 #    FIXME: launch through libvirt
 296                 log.info("virsh create ...")
 297                 err, out, _ = self.connection.execute(
 298                     "virsh create /tmp/vm_sriov.xml")
 299                 time.sleep(10)
 300                 log.error("err : {0}".format(err))
 301                 log.error("{0}".format(_))
 302                 log.debug("out : {0}".format(out))
 303         except ValueError:
 304                 raise
 305
 306         #    5: Tunning for better performace
 307         self.pin_vcpu(pcis)
 308         self.connection.execute(
 309             "echo 1 > /sys/module/kvm/parameters/"
 310             "allow_unsafe_assigned_interrupts")
 311         self.connection.execute(
 312             "echo never > /sys/kernel/mm/transparent_hugepage/enabled")
 313
 314     def add_sriov_interface(self, index, vf_pci, vfmac, xml):
 315         root = ET.parse(xml)
 316         pattern = "0000:(\d+):(\d+).(\d+)"
 317         m = re.search(pattern, vf_pci, re.MULTILINE)
 318         device = root.find('devices')
 319
 320         interface = ET.SubElement(device, 'interface')
 321         interface.set('managed', 'yes')
 322         interface.set('type', 'hostdev')
 323
 324         mac = ET.SubElement(interface, 'mac')
 325         mac.set('address', vfmac)
 326         source = ET.SubElement(interface, 'source')
 327
 328         addr = ET.SubElement(source, "address")
 329         addr.set('domain', "0x0")
 330         addr.set('bus', "{0}".format(m.group(1)))
 331         addr.set('function', "{0}".format(m.group(3)))
 332         addr.set('slot', "{0}".format(m.group(2)))
 333         addr.set('type', "pci")
 334
 335         vf_pci = ET.SubElement(interface, 'address')
 336         vf_pci.set('type', 'pci')
 337         vf_pci.set('domain', '0x0000')
 338         vf_pci.set('bus', '0x00')
 339         vf_pci.set('slot', '0x0{0}'.format(index + 7))
 340         vf_pci.set('function', '0x00')
 341
 342         root.write(xml)
 343
 344     #   This is roughly compatible with check_output function in subprocess
 345     #   module which is only available in python 2.7
 346     def check_output(self, cmd, stderr=None):
 347         #   Run a command and capture its output
 348         err, out, _ = self.connection.execute(cmd)
 349         return err, out
 350
 351     def get_virtual_devices(self, pci):
 352         pf_vfs = {}
 353         err, extra_info = self.check_output(
 354             "cat /sys/bus/pci/devices/{0}/virtfn0/uevent".format(pci))
 355         pattern = "PCI_SLOT_NAME=(?P<name>[0-9:.\s.]+)"
 356         m = re.search(pattern, extra_info, re.MULTILINE)
 357
 358         if m:
 359             pf_vfs.update({pci: str(m.group(1).rstrip())})
 360         log.info("pf_vfs : {0}".format(pf_vfs))
 361         return pf_vfs
 362
 363     def get_vf_datas(self, key, value, vfmac):
 364         vfret = {}
 365         pattern = "0000:(\d+):(\d+).(\d+)"
 366
 367         vfret["mac"] = vfmac
 368         vfs = self.get_virtual_devices(value)
 369         log.info("vfs: {0}".format(vfs))
 370         for k, v in vfs.items():
 371             m = re.search(pattern, k, re.MULTILINE)
 372             m1 = re.search(pattern, value, re.MULTILINE)
 373             if m.group(1) == m1.group(1):
 374                 vfret["vf_pci"] = str(v)
 375                 break
 376
 377         return vfret
 378
 379     def read_from_file(self, filename):
 380         data = ""
 381         with open(filename, 'r') as the_file:
 382             data = the_file.read()
 383         return data
 384
 385     def write_to_file(self, filename, content):
 386         with open(filename, 'w') as the_file:
 387             the_file.write(content)
 388
 389     def pin_vcpu(self, pcis):
 390         nodes = self.get_numa_nodes()
 391         log.info("{0}".format(nodes))
 392         num_nodes = len(nodes)
 393         for i in range(0, 10):
 394             self.connection.execute(
 395                 "virsh vcpupin vm1 {0} {1}".format(
 396                     i, nodes[str(num_nodes - 1)][i]))
 397
 398     def get_numa_nodes(self):
 399         nodes_sysfs = glob.iglob("/sys/devices/system/node/node*")
 400         nodes = {}
 401         for node_sysfs in nodes_sysfs:
 402             num = os.path.basename(node_sysfs).replace("node", "")
 403             with open(os.path.join(node_sysfs, "cpulist")) as cpulist_file:
 404                 cpulist = cpulist_file.read().strip()
 405             nodes[num] = self.split_cpu_list(cpulist)
 406         log.info("nodes: {0}".format(nodes))
 407         return nodes
 408
 409     def split_cpu_list(self, cpu_list):
 410         if cpu_list:
 411             ranges = cpu_list.split(',')
 412             bounds = ([int(b) for b in r.split('-')] for r in ranges)
 413             range_objects =\
 414                 (range(bound[0], bound[1] + 1 if len(bound) == 2
 415                  else bound[0] + 1) for bound in bounds)
 416
 417             return sorted(itertools.chain.from_iterable(range_objects))
 418         else:
 419             return []
 420
 421     def destroy_vm(self):
 422         host_driver = self.sriov[0]["phy_driver"]
 423         err, out = self.check_output("virsh list --name | grep -i vm1")
 424         log.info("{0}".format(out))
 425         if err == 0:
 426             self.connection.execute("virsh shutdown vm1")
 427             self.connection.execute("virsh destroy vm1")
 428             self.check_output("rmmod {0}".format(host_driver))[1].splitlines()
 429             self.check_output("modprobe {0}".format(host_driver))[
 430                 1].splitlines()
 431         else:
 432             log.error("error : {0}".format(err))