yardstick/benchmark/contexts/standalone/sriov.py

   1 # Copyright (c) 2016-2017 Intel Corporation
   2 #
   3 # Licensed under the Apache License, Version 2.0 (the "License");
   4 # you may not use this file except in compliance with the License.
   5 # You may obtain a copy of the License at
   6 #
   7 #      http://www.apache.org/licenses/LICENSE-2.0
   8 #
   9 # Unless required by applicable law or agreed to in writing, software
  10 # distributed under the License is distributed on an "AS IS" BASIS,
  11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12 # See the License for the specific language governing permissions and
  13 # limitations under the License.
  14
  15 from __future__ import absolute_import
  16 import os
  17 import yaml
  18 import re
  19 import time
  20 import glob
  21 import uuid
  22 import random
  23 import logging
  24 import itertools
  25 import xml.etree.ElementTree as ET
  26 from yardstick import ssh
  27 from yardstick.network_services.utils import get_nsb_option
  28 from yardstick.network_services.utils import provision_tool
  29 from yardstick.benchmark.contexts.standalone import StandaloneContext
  30
  31 log = logging.getLogger(__name__)
  32
  33 VM_TEMPLATE = """
  34 <domain type="kvm">
  35  <name>vm1</name>
  36   <uuid>{random_uuid}</uuid>
  37   <memory unit="KiB">102400</memory>
  38   <currentMemory unit="KiB">102400</currentMemory>
  39   <memoryBacking>
  40     <hugepages />
  41   </memoryBacking>
  42   <vcpu placement="static">20</vcpu>
  43   <os>
  44     <type arch="x86_64" machine="pc-i440fx-utopic">hvm</type>
  45     <boot dev="hd" />
  46   </os>
  47   <features>
  48     <acpi />
  49     <apic />
  50     <pae />
  51   </features>
  52   <cpu match="exact" mode="custom">
  53     <model fallback="allow">SandyBridge</model>
  54     <topology cores="10" sockets="1" threads="2" />
  55   </cpu>
  56   <clock offset="utc">
  57     <timer name="rtc" tickpolicy="catchup" />
  58     <timer name="pit" tickpolicy="delay" />
  59     <timer name="hpet" present="no" />
  60   </clock>
  61   <on_poweroff>destroy</on_poweroff>
  62   <on_reboot>restart</on_reboot>
  63   <on_crash>restart</on_crash>
  64   <devices>
  65     <emulator>/usr/bin/kvm-spice</emulator>
  66     <disk device="disk" type="file">
  67       <driver name="qemu" type="qcow2" />
  68       <source file="{vm_image}"/>
  69       <target bus="virtio" dev="vda" />
  70       <address bus="0x00" domain="0x0000"
  71 function="0x0" slot="0x04" type="pci" />
  72     </disk>
  73     <controller index="0" model="ich9-ehci1" type="usb">
  74       <address bus="0x00" domain="0x0000"
  75 function="0x7" slot="0x05" type="pci" />
  76     </controller>
  77     <controller index="0" model="ich9-uhci1" type="usb">
  78       <master startport="0" />
  79       <address bus="0x00" domain="0x0000" function="0x0"
  80 multifunction="on" slot="0x05" type="pci" />
  81     </controller>
  82     <controller index="0" model="ich9-uhci2" type="usb">
  83       <master startport="2" />
  84       <address bus="0x00" domain="0x0000"
  85 function="0x1" slot="0x05" type="pci" />
  86     </controller>
  87     <controller index="0" model="ich9-uhci3" type="usb">
  88       <master startport="4" />
  89       <address bus="0x00" domain="0x0000"
  90 function="0x2" slot="0x05" type="pci" />
  91     </controller>
  92     <controller index="0" model="pci-root" type="pci" />
  93       <serial type="pty">
  94       <target port="0" />
  95     </serial>
  96     <console type="pty">
  97       <target port="0" type="serial" />
  98     </console>
  99     <input bus="usb" type="tablet" />
 100     <input bus="ps2" type="mouse" />
 101     <input bus="ps2" type="keyboard" />
 102     <graphics autoport="yes" listen="0.0.0.0" port="-1" type="vnc" />
 103     <video>
 104       <model heads="1" type="cirrus" vram="16384" />
 105       <address bus="0x00" domain="0x0000"
 106 function="0x0" slot="0x02" type="pci" />
 107     </video>
 108     <memballoon model="virtio">
 109       <address bus="0x00" domain="0x0000"
 110 function="0x0" slot="0x06" type="pci" />
 111     </memballoon>
 112     <interface type="bridge">
 113       <mac address="{mac_addr}" />
 114       <source bridge="virbr0" />
 115     </interface>
 116    </devices>
 117 </domain>
 118 """
 119
 120
 121 class Sriov(StandaloneContext):
 122     def __init__(self):
 123         self.name = None
 124         self.file_path = None
 125         self.nodes = []
 126         self.vm_deploy = False
 127         self.sriov = []
 128         self.first_run = True
 129         self.dpdk_nic_bind = ""
 130         self.user = ""
 131         self.ssh_ip = ""
 132         self.passwd = ""
 133         self.ssh_port = ""
 134         self.auth_type = ""
 135
 136     def init(self):
 137         log.debug("In init")
 138         self.parse_pod_and_get_data(self.file_path)
 139
 140     def parse_pod_and_get_data(self, file_path):
 141         self.file_path = file_path
 142         log.debug("parsing pod file: {0}".format(self.file_path))
 143         try:
 144             with open(self.file_path) as stream:
 145                 cfg = yaml.load(stream)
 146         except IOError:
 147             log.error("File {0} does not exist".format(self.file_path))
 148             raise
 149
 150         self.sriov.extend([node for node in cfg["nodes"]
 151                            if node["role"] == "Sriov"])
 152         self.user = self.sriov[0]['user']
 153         self.ssh_ip = self.sriov[0]['ip']
 154         if self.sriov[0]['auth_type'] == "password":
 155             self.passwd = self.sriov[0]['password']
 156         else:
 157             self.ssh_port = self.sriov[0]['ssh_port']
 158             self.key_filename = self.sriov[0]['key_filename']
 159
 160     def ssh_remote_machine(self):
 161         if self.sriov[0]['auth_type'] == "password":
 162             self.connection = ssh.SSH(
 163                 self.user,
 164                 self.ssh_ip,
 165                 password=self.passwd)
 166             self.connection.wait()
 167         else:
 168             if self.ssh_port is not None:
 169                 ssh_port = self.ssh_port
 170             else:
 171                 ssh_port = ssh.DEFAULT_PORT
 172             self.connection = ssh.SSH(
 173                 self.user,
 174                 self.ssh_ip,
 175                 port=ssh_port,
 176                 key_filename=self.key_filename)
 177             self.connection.wait()
 178         self.dpdk_nic_bind = provision_tool(
 179             self.connection,
 180             os.path.join(get_nsb_option("bin_path"), "dpdk_nic_bind.py"))
 181
 182     def get_nic_details(self):
 183         nic_details = {}
 184         nic_details = {
 185             'interface': {},
 186             'pci': self.sriov[0]['phy_ports'],
 187             'phy_driver': self.sriov[0]['phy_driver'],
 188             'vf_macs': self.sriov[0]['vf_macs']
 189         }
 190         #   Make sure that ports are bound to kernel drivers e.g. i40e/ixgbe
 191         for i, _ in enumerate(nic_details['pci']):
 192             err, out, _ = self.connection.execute(
 193                 "{dpdk_nic_bind} --force -b {driver} {port}".format(
 194                     dpdk_nic_bind=self.dpdk_nic_bind,
 195                     driver=self.sriov[0]['phy_driver'],
 196                     port=self.sriov[0]['phy_ports'][i]))
 197             err, out, _ = self.connection.execute(
 198                 "lshw -c network -businfo | grep '{port}'".format(
 199                     port=self.sriov[0]['phy_ports'][i]))
 200             a = out.split()[1]
 201             err, out, _ = self.connection.execute(
 202                 "ip -s link show {interface}".format(
 203                     interface=out.split()[1]))
 204             nic_details['interface'][i] = str(a)
 205         log.info("{0}".format(nic_details))
 206         return nic_details
 207
 208     def install_req_libs(self):
 209         if self.first_run:
 210             log.info("Installing required libraries...")
 211             err, out, _ = self.connection.execute("apt-get update")
 212             log.debug("{0}".format(out))
 213             err, out, _ = self.connection.execute(
 214                 "apt-get -y install qemu-kvm libvirt-bin")
 215             log.debug("{0}".format(out))
 216             err, out, _ = self.connection.execute(
 217                 "apt-get -y install libvirt-dev  bridge-utils numactl")
 218             log.debug("{0}".format(out))
 219             self.first_run = False
 220
 221     def configure_nics_for_sriov(self, host_driver, nic_details):
 222         vf_pci = [[], []]
 223         self.connection.execute(
 224             "rmmod {0}".format(host_driver))[1].splitlines()
 225         self.connection.execute(
 226             "modprobe {0} num_vfs=1".format(host_driver))[1].splitlines()
 227         nic_details['vf_pci'] = {}
 228         for i in range(len(nic_details['pci'])):
 229             self.connection.execute(
 230                 "echo 1 > /sys/bus/pci/devices/{0}/sriov_numvfs".format(
 231                     nic_details['pci'][i]))
 232             err, out, _ = self.connection.execute(
 233                 "ip link set {interface} vf 0 mac {mac}".format(
 234                     interface=nic_details['interface'][i],
 235                     mac=nic_details['vf_macs'][i]))
 236             time.sleep(3)
 237             vf_pci[i] = self.get_vf_datas(
 238                 'vf_pci',
 239                 nic_details['pci'][i],
 240                 nic_details['vf_macs'][i])
 241             nic_details['vf_pci'][i] = vf_pci[i]
 242         log.debug("NIC DETAILS : {0}".format(nic_details))
 243         return nic_details
 244
 245     def setup_sriov_context(self, pcis, nic_details, host_driver):
 246         blacklist = "/etc/modprobe.d/blacklist.conf"
 247
 248         #   1 : Blacklist the vf driver in /etc/modprobe.d/blacklist.conf
 249         vfnic = "{0}vf".format(host_driver)
 250         lines = self.read_from_file(blacklist)
 251         if vfnic not in lines:
 252             vfblacklist = "blacklist {vfnic}".format(vfnic=vfnic)
 253             self.connection.execute(
 254                 "echo {vfblacklist} >> {blacklist}".format(
 255                     vfblacklist=vfblacklist,
 256                     blacklist=blacklist))
 257
 258         #   2 : modprobe host_driver with num_vfs
 259         nic_details = self.configure_nics_for_sriov(host_driver, nic_details)
 260
 261         #   3: Setup vm_sriov.xml to launch VM
 262         cfg_sriov = '/tmp/vm_sriov.xml'
 263         mac = [0x00, 0x24, 0x81,
 264                random.randint(0x00, 0x7f),
 265                random.randint(0x00, 0xff),
 266                random.randint(0x00, 0xff)]
 267         mac_address = ':'.join(map(lambda x: "%02x" % x, mac))
 268         vm_sriov_xml = VM_TEMPLATE.format(
 269             random_uuid=uuid.uuid4(),
 270             mac_addr=mac_address,
 271             vm_image=self.sriov[0]["images"])
 272         with open(cfg_sriov, 'w') as f:
 273             f.write(vm_sriov_xml)
 274
 275         vf = nic_details['vf_pci']
 276         for index in range(len(nic_details['vf_pci'])):
 277             self.add_sriov_interface(
 278                 index,
 279                 vf[index]['vf_pci'],
 280                 mac_address,
 281                 "/tmp/vm_sriov.xml")
 282             self.connection.execute(
 283                 "ifconfig {interface} up".format(
 284                     interface=nic_details['interface'][index]))
 285
 286         #   4: Create and start the VM
 287         self.connection.put(cfg_sriov, cfg_sriov)
 288         time.sleep(10)
 289         err, out = self.check_output("virsh list --name | grep -i vm1")
 290         try:
 291             if out == "vm1":
 292                 log.info("VM is already present")
 293             else:
 294                 #    FIXME: launch through libvirt
 295                 log.info("virsh create ...")
 296                 err, out, _ = self.connection.execute(
 297                     "virsh create /tmp/vm_sriov.xml")
 298                 time.sleep(10)
 299                 log.error("err : {0}".format(err))
 300                 log.error("{0}".format(_))
 301                 log.debug("out : {0}".format(out))
 302         except ValueError:
 303                 raise
 304
 305         #    5: Tunning for better performace
 306         self.pin_vcpu(pcis)
 307         self.connection.execute(
 308             "echo 1 > /sys/module/kvm/parameters/"
 309             "allow_unsafe_assigned_interrupts")
 310         self.connection.execute(
 311             "echo never > /sys/kernel/mm/transparent_hugepage/enabled")
 312
 313     def add_sriov_interface(self, index, vf_pci, vfmac, xml):
 314         root = ET.parse(xml)
 315         pattern = "0000:(\d+):(\d+).(\d+)"
 316         m = re.search(pattern, vf_pci, re.MULTILINE)
 317         device = root.find('devices')
 318
 319         interface = ET.SubElement(device, 'interface')
 320         interface.set('managed', 'yes')
 321         interface.set('type', 'hostdev')
 322
 323         mac = ET.SubElement(interface, 'mac')
 324         mac.set('address', vfmac)
 325         source = ET.SubElement(interface, 'source')
 326
 327         addr = ET.SubElement(source, "address")
 328         addr.set('domain', "0x0")
 329         addr.set('bus', "{0}".format(m.group(1)))
 330         addr.set('function', "{0}".format(m.group(3)))
 331         addr.set('slot', "{0}".format(m.group(2)))
 332         addr.set('type', "pci")
 333
 334         vf_pci = ET.SubElement(interface, 'address')
 335         vf_pci.set('type', 'pci')
 336         vf_pci.set('domain', '0x0000')
 337         vf_pci.set('bus', '0x00')
 338         vf_pci.set('slot', '0x0{0}'.format(index + 7))
 339         vf_pci.set('function', '0x00')
 340
 341         root.write(xml)
 342
 343     #   This is roughly compatible with check_output function in subprocess
 344     #   module which is only available in python 2.7
 345     def check_output(self, cmd, stderr=None):
 346         #   Run a command and capture its output
 347         err, out, _ = self.connection.execute(cmd)
 348         return err, out
 349
 350     def get_virtual_devices(self, pci):
 351         pf_vfs = {}
 352         err, extra_info = self.check_output(
 353             "cat /sys/bus/pci/devices/{0}/virtfn0/uevent".format(pci))
 354         pattern = "PCI_SLOT_NAME=(?P<name>[0-9:.\s.]+)"
 355         m = re.search(pattern, extra_info, re.MULTILINE)
 356
 357         if m:
 358             pf_vfs.update({pci: str(m.group(1).rstrip())})
 359         log.info("pf_vfs : {0}".format(pf_vfs))
 360         return pf_vfs
 361
 362     def get_vf_datas(self, key, value, vfmac):
 363         vfret = {}
 364         pattern = "0000:(\d+):(\d+).(\d+)"
 365
 366         vfret["mac"] = vfmac
 367         vfs = self.get_virtual_devices(value)
 368         log.info("vfs: {0}".format(vfs))
 369         for k, v in vfs.items():
 370             m = re.search(pattern, k, re.MULTILINE)
 371             m1 = re.search(pattern, value, re.MULTILINE)
 372             if m.group(1) == m1.group(1):
 373                 vfret["vf_pci"] = str(v)
 374                 break
 375
 376         return vfret
 377
 378     def read_from_file(self, filename):
 379         data = ""
 380         with open(filename, 'r') as the_file:
 381             data = the_file.read()
 382         return data
 383
 384     def write_to_file(self, filename, content):
 385         with open(filename, 'w') as the_file:
 386             the_file.write(content)
 387
 388     def pin_vcpu(self, pcis):
 389         nodes = self.get_numa_nodes()
 390         log.info("{0}".format(nodes))
 391         num_nodes = len(nodes)
 392         for i in range(0, 10):
 393             self.connection.execute(
 394                 "virsh vcpupin vm1 {0} {1}".format(
 395                     i, nodes[str(num_nodes - 1)][i % len(nodes[str(num_nodes - 1)])]))
 396
 397     def get_numa_nodes(self):
 398         nodes_sysfs = glob.iglob("/sys/devices/system/node/node*")
 399         nodes = {}
 400         for node_sysfs in nodes_sysfs:
 401             num = os.path.basename(node_sysfs).replace("node", "")
 402             with open(os.path.join(node_sysfs, "cpulist")) as cpulist_file:
 403                 cpulist = cpulist_file.read().strip()
 404             nodes[num] = self.split_cpu_list(cpulist)
 405         log.info("nodes: {0}".format(nodes))
 406         return nodes
 407
 408     def split_cpu_list(self, cpu_list):
 409         if cpu_list:
 410             ranges = cpu_list.split(',')
 411             bounds = ([int(b) for b in r.split('-')] for r in ranges)
 412             range_objects =\
 413                 (range(bound[0], bound[1] + 1 if len(bound) == 2
 414                  else bound[0] + 1) for bound in bounds)
 415
 416             return sorted(itertools.chain.from_iterable(range_objects))
 417         else:
 418             return []
 419
 420     def destroy_vm(self):
 421         host_driver = self.sriov[0]["phy_driver"]
 422         err, out = self.check_output("virsh list --name | grep -i vm1")
 423         log.info("{0}".format(out))
 424         if err == 0:
 425             self.connection.execute("virsh shutdown vm1")
 426             self.connection.execute("virsh destroy vm1")
 427             self.check_output("rmmod {0}".format(host_driver))[1].splitlines()
 428             self.check_output("modprobe {0}".format(host_driver))[
 429                 1].splitlines()
 430         else:
 431             log.error("error : {0}".format(err))