yardstick/benchmark/contexts/ovsdpdk.py

   1 # Copyright (c) 2016-2017 Intel Corporation
   2 #
   3 # Licensed under the Apache License, Version 2.0 (the "License");
   4 # you may not use this file except in compliance with the License.
   5 # You may obtain a copy of the License at
   6 #
   7 #      http://www.apache.org/licenses/LICENSE-2.0
   8 #
   9 # Unless required by applicable law or agreed to in writing, software
  10 # distributed under the License is distributed on an "AS IS" BASIS,
  11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12 # See the License for the specific language governing permissions and
  13 # limitations under the License.
  14
  15 from __future__ import absolute_import
  16 import os
  17 import yaml
  18 import time
  19 import glob
  20 import itertools
  21 import logging
  22 from yardstick import ssh
  23 from yardstick.benchmark.contexts.standalone import StandaloneContext
  24
  25 BIN_PATH = "/opt/isb_bin/"
  26 DPDK_NIC_BIND = "dpdk_nic_bind.py"
  27
  28 log = logging.getLogger(__name__)
  29
  30 VM_TEMPLATE = """
  31 <domain type='kvm'>
  32   <name>vm1</name>
  33   <uuid>18230c0c-635d-4c50-b2dc-a213d30acb34</uuid>
  34   <memory unit='KiB'>20971520</memory>
  35   <currentMemory unit="KiB">20971520</currentMemory>
  36   <memoryBacking>
  37     <hugepages/>
  38   </memoryBacking>
  39   <vcpu placement='static'>20</vcpu>
  40   <os>
  41     <type arch='x86_64' machine='pc'>hvm</type>
  42     <boot dev='hd'/>
  43   </os>
  44   <features>
  45     <acpi/>
  46     <apic/>
  47   </features>
  48   <cpu match="exact" mode='host-model'>
  49     <model fallback='allow'/>
  50     <topology sockets='1' cores='10' threads='2'/>
  51   </cpu>
  52   <on_poweroff>destroy</on_poweroff>
  53   <on_reboot>restart</on_reboot>
  54   <on_crash>destroy</on_crash>
  55   <devices>
  56     <emulator>/usr/bin/qemu-system-x86_64</emulator>
  57     <disk type='file' device='disk'>
  58       <driver name='qemu' type='qcow2' cache='none'/>
  59       <source file="{vm_image}"/>
  60       <target dev='vda' bus='virtio'/>
  61       <address bus="0x00" domain="0x0000"
  62       function="0x0" slot="0x04" type="pci" />
  63     </disk>
  64     <!--disk type='dir' device='disk'>
  65       <driver name='qemu' type='fat'/>
  66       <source dir='/opt/isb_bin/dpdk'/>
  67       <target dev='vdb' bus='virtio'/>
  68       <readonly/>
  69     </disk-->
  70     <interface type="bridge">
  71       <mac address="00:00:00:ab:cd:ef" />
  72       <source bridge="br-int" />
  73     </interface>
  74     <interface type='vhostuser'>
  75       <mac address='00:00:00:00:00:01'/>
  76       <source type='unix' path='/usr/local/var/run/openvswitch/dpdkvhostuser0' mode='client'/>
  77        <model type='virtio'/>
  78       <driver queues='4'>
  79         <host mrg_rxbuf='off'/>
  80       </driver>
  81     </interface>
  82     <interface type='vhostuser'>
  83       <mac address='00:00:00:00:00:02'/>
  84       <source type='unix' path='/usr/local/var/run/openvswitch/dpdkvhostuser1' mode='client'/>
  85       <model type='virtio'/>
  86       <driver queues='4'>
  87         <host mrg_rxbuf='off'/>
  88       </driver>
  89     </interface>
  90     <serial type='pty'>
  91       <target port='0'/>
  92     </serial>
  93     <console type='pty'>
  94       <target type='serial' port='0'/>
  95     </console>
  96     <graphics autoport="yes" listen="0.0.0.0" port="1" type="vnc" />
  97   </devices>
  98 </domain>
  99 """
 100
 101
 102 class Ovsdpdk(StandaloneContext):
 103     def __init__(self):
 104         self.name = None
 105         self.file_path = None
 106         self.nodes = []
 107         self.vm_deploy = False
 108         self.ovs = []
 109         self.first_run = True
 110         self.dpdk_nic_bind = BIN_PATH + DPDK_NIC_BIND
 111         self.user = ""
 112         self.ssh_ip = ""
 113         self.passwd = ""
 114         self.ssh_port = ""
 115         self.auth_type = ""
 116
 117     def init(self):
 118         '''initializes itself'''
 119         log.debug("In init")
 120         self.parse_pod_and_get_data()
 121
 122     def parse_pod_and_get_data(self, file_path):
 123         self.file_path = file_path
 124         print("parsing pod file: {0}".format(self.file_path))
 125         try:
 126             with open(self.file_path) as stream:
 127                 cfg = yaml.load(stream)
 128         except IOError:
 129             print("File {0} does not exist".format(self.file_path))
 130             raise
 131
 132         self.ovs.extend([node for node in cfg["nodes"]
 133                          if node["role"] == "Ovsdpdk"])
 134         self.user = self.ovs[0]['user']
 135         self.ssh_ip = self.ovs[0]['ip']
 136         if self.ovs[0]['auth_type'] == "password":
 137             self.passwd = self.ovs[0]['password']
 138         else:
 139             self.ssh_port = self.ovs[0]['ssh_port']
 140             self.key_filename = self.ovs[0]['key_filename']
 141
 142     def ssh_remote_machine(self):
 143         if self.ovs[0]['auth_type'] == "password":
 144             self.connection = ssh.SSH(
 145                 self.user,
 146                 self.ssh_ip,
 147                 password=self.passwd)
 148             self.connection.wait()
 149         else:
 150             if self.ssh_port is not None:
 151                 ssh_port = self.ssh_port
 152             else:
 153                 ssh_port = ssh.DEFAULT_PORT
 154             self.connection = ssh.SSH(
 155                 self.user,
 156                 self.ssh_ip,
 157                 port=ssh_port,
 158                 key_filename=self.key_filename)
 159             self.connection.wait()
 160
 161     def get_nic_details(self):
 162         nic_details = {}
 163         nic_details['interface'] = {}
 164         nic_details['pci'] = self.ovs[0]['phy_ports']
 165         nic_details['phy_driver'] = self.ovs[0]['phy_driver']
 166         nic_details['vports_mac'] = self.ovs[0]['vports_mac']
 167         #    Make sure that ports are bound to kernel drivers e.g. i40e/ixgbe
 168         for i, _ in enumerate(nic_details['pci']):
 169             err, out, _ = self.connection.execute(
 170                 "{dpdk_nic_bind} --force -b {driver} {port}".format(
 171                     dpdk_nic_bind=self.dpdk_nic_bind,
 172                     driver=self.ovs[0]['phy_driver'],
 173                     port=self.ovs[0]['phy_ports'][i]))
 174             err, out, _ = self.connection.execute(
 175                 "lshw -c network -businfo | grep '{port}'".format(
 176                     port=self.ovs[0]['phy_ports'][i]))
 177             a = out.split()[1]
 178             err, out, _ = self.connection.execute(
 179                 "ip -s link show {interface}".format(
 180                     interface=out.split()[1]))
 181             nic_details['interface'][i] = str(a)
 182         print("{0}".format(nic_details))
 183         return nic_details
 184
 185     def install_req_libs(self):
 186         if self.first_run:
 187             err, out, _ = self.connection.execute("apt-get update")
 188             print("{0}".format(out))
 189             err, out, _ = self.connection.execute(
 190                 "apt-get -y install qemu-kvm libvirt-bin")
 191             print("{0}".format(out))
 192             err, out, _ = self.connection.execute(
 193                 "apt-get -y install libvirt-dev  bridge-utils numactl")
 194             print("{0}".format(out))
 195             self.first_run = False
 196
 197     def setup_ovs(self, vpcis):
 198         self.connection.execute("/usr/bin/chmod 0666 /dev/vfio/*")
 199         self.connection.execute("/usr/bin/chmod a+x /dev/vfio")
 200         self.connection.execute("pkill -9 ovs")
 201         self.connection.execute("ps -ef | grep ovs | grep -v grep | "
 202                                 "awk '{print $2}' | xargs -r kill -9")
 203         self.connection.execute("killall -r 'ovs*'")
 204         self.connection.execute(
 205             "mkdir -p {0}/etc/openvswitch".format(self.ovs[0]["vpath"]))
 206         self.connection.execute(
 207             "mkdir -p {0}/var/run/openvswitch".format(self.ovs[0]["vpath"]))
 208         self.connection.execute(
 209             "rm {0}/etc/openvswitch/conf.db".format(self.ovs[0]["vpath"]))
 210         self.connection.execute(
 211             "ovsdb-tool create {0}/etc/openvswitch/conf.db "
 212             "{0}/share/openvswitch/"
 213             "vswitch.ovsschema".format(self.ovs[0]["vpath"]))
 214         self.connection.execute("modprobe vfio-pci")
 215         self.connection.execute("chmod a+x /dev/vfio")
 216         self.connection.execute("chmod 0666 /dev/vfio/*")
 217         for vpci in vpcis:
 218             self.connection.execute(
 219                 "/opt/isb_bin/dpdk_nic_bind.py "
 220                 "--bind=vfio-pci {0}".format(vpci))
 221
 222     def start_ovs_serverswitch(self):
 223             self.connection.execute("mkdir -p /usr/local/var/run/openvswitch")
 224             self.connection.execute(
 225                 "ovsdb-server --remote=punix:"
 226                 "/usr/local/var/run/openvswitch/db.sock  --pidfile --detach")
 227             self.connection.execute(
 228                 "ovs-vsctl --no-wait set "
 229                 "Open_vSwitch . other_config:dpdk-init=true")
 230             self.connection.execute(
 231                 "ovs-vsctl --no-wait set "
 232                 "Open_vSwitch . other_config:dpdk-lcore-mask=0x3")
 233             self.connection.execute(
 234                 "ovs-vsctl --no-wait set "
 235                 "Open_vSwitch . other_config:dpdk-socket-mem='2048,0'")
 236             self.connection.execute(
 237                 "ovs-vswitchd unix:{0}/"
 238                 "var/run/openvswitch/db.sock --pidfile --detach "
 239                 "--log-file=/var/log/openvswitch/"
 240                 "ovs-vswitchd.log".format(
 241                     self.ovs[0]["vpath"]))
 242             self.connection.execute(
 243                 "ovs-vsctl set Open_vSwitch . other_config:pmd-cpu-mask=2C")
 244
 245     def setup_ovs_bridge(self):
 246         self.connection.execute("ovs-vsctl del-br br0")
 247         self.connection.execute(
 248             "rm -rf /usr/local/var/run/openvswitch/dpdkvhostuser*")
 249         self.connection.execute(
 250             "ovs-vsctl add-br br0 -- set bridge br0 datapath_type=netdev")
 251         self.connection.execute(
 252             "ovs-vsctl add-port br0 dpdk0 -- set Interface dpdk0 type=dpdk")
 253         self.connection.execute(
 254             "ovs-vsctl add-port br0 dpdk1 -- set Interface dpdk1 type=dpdk")
 255         self.connection.execute(
 256             "ovs-vsctl add-port br0 dpdkvhostuser0 -- set Interface "
 257             "dpdkvhostuser0 type=dpdkvhostuser")
 258         self.connection.execute("ovs-vsctl add-port br0 dpdkvhostuser1 "
 259                                 "-- set Interface dpdkvhostuser1 "
 260                                 "type=dpdkvhostuser")
 261         self.connection.execute(
 262             "chmod 0777 {0}/var/run/"
 263             "openvswitch/dpdkvhostuser*".format(self.ovs[0]["vpath"]))
 264
 265     def add_oflows(self):
 266         self.connection.execute("ovs-ofctl del-flows br0")
 267         for flow in self.ovs[0]["flow"]:
 268             self.connection.execute(flow)
 269             self.connection.execute("ovs-ofctl dump-flows br0")
 270             self.connection.execute(
 271                 "ovs-vsctl set Interface dpdk0 options:n_rxq=4")
 272             self.connection.execute(
 273                 "ovs-vsctl set Interface dpdk1 options:n_rxq=4")
 274
 275     def setup_ovs_context(self, pcis, nic_details, host_driver):
 276
 277         ''' 1: Setup vm_ovs.xml to launch VM.'''
 278         cfg_ovs = '/tmp/vm_ovs.xml'
 279         vm_ovs_xml = VM_TEMPLATE.format(vm_image=self.ovs[0]["images"])
 280         with open(cfg_ovs, 'w') as f:
 281             f.write(vm_ovs_xml)
 282
 283         ''' 2: Create and start the VM'''
 284         self.connection.put(cfg_ovs, cfg_ovs)
 285         time.sleep(10)
 286         err, out = self.check_output("virsh list --name | grep -i vm1")
 287         if out == "vm1":
 288             print("VM is already present")
 289         else:
 290             ''' FIXME: launch through libvirt'''
 291             print("virsh create ...")
 292             err, out, _ = self.connection.execute(
 293                 "virsh create /tmp/vm_ovs.xml")
 294             time.sleep(10)
 295             print("err : {0}".format(err))
 296             print("{0}".format(_))
 297             print("out : {0}".format(out))
 298
 299         ''' 3: Tuning for better performace.'''
 300         self.pin_vcpu(pcis)
 301         self.connection.execute(
 302             "echo 1 > /sys/module/kvm/parameters/"
 303             "allow_unsafe_assigned_interrupts")
 304         self.connection.execute(
 305             "echo never > /sys/kernel/mm/transparent_hugepage/enabled")
 306         print("After tuning performance ...")
 307
 308     ''' This is roughly compatible with check_output function in subprocess
 309      module which is only available in python 2.7.'''
 310     def check_output(self, cmd, stderr=None):
 311         '''Run a command and capture its output'''
 312         err, out, _ = self.connection.execute(cmd)
 313         return err, out
 314
 315     def read_from_file(self, filename):
 316         data = ""
 317         with open(filename, 'r') as the_file:
 318             data = the_file.read()
 319         return data
 320
 321     def write_to_file(self, filename, content):
 322         with open(filename, 'w') as the_file:
 323             the_file.write(content)
 324
 325     def pin_vcpu(self, pcis):
 326         nodes = self.get_numa_nodes()
 327         print("{0}".format(nodes))
 328         num_nodes = len(nodes)
 329         for i in range(0, 10):
 330             self.connection.execute(
 331                 "virsh vcpupin vm1 {0} {1}".format(
 332                     i, nodes[str(num_nodes - 1)][i]))
 333
 334     def get_numa_nodes(self):
 335         nodes_sysfs = glob.iglob("/sys/devices/system/node/node*")
 336         nodes = {}
 337         for node_sysfs in nodes_sysfs:
 338             num = os.path.basename(node_sysfs).replace("node", "")
 339             with open(os.path.join(node_sysfs, "cpulist")) as cpulist_file:
 340                 cpulist = cpulist_file.read().strip()
 341                 print("cpulist: {0}".format(cpulist))
 342             nodes[num] = self.split_cpu_list(cpulist)
 343         print("nodes: {0}".format(nodes))
 344         return nodes
 345
 346     def split_cpu_list(self, cpu_list):
 347         if cpu_list:
 348             ranges = cpu_list.split(',')
 349             bounds = ([int(b) for b in r.split('-')] for r in ranges)
 350             range_objects =\
 351                 (range(bound[0], bound[1] + 1 if len(bound) == 2
 352                  else bound[0] + 1) for bound in bounds)
 353
 354             return sorted(itertools.chain.from_iterable(range_objects))
 355         else:
 356             return []
 357
 358     def destroy_vm(self):
 359         host_driver = self.ovs[0]['phy_driver']
 360         err, out = self.check_output("virsh list --name | grep -i vm1")
 361         print("{0}".format(out))
 362         if err == 0:
 363             self.connection.execute("virsh shutdown vm1")
 364             self.connection.execute("virsh destroy vm1")
 365             self.check_output("rmmod {0}".format(host_driver))[1].splitlines()
 366             self.check_output("modprobe {0}".format(host_driver))[
 367                 1].splitlines()
 368         else:
 369             print("error : ", err)