yardstick/benchmark/scenarios/networking/pktgen.py

   1 ##############################################################################
   2 # Copyright (c) 2015 Ericsson AB and others.
   3 #
   4 # All rights reserved. This program and the accompanying materials
   5 # are made available under the terms of the Apache License, Version 2.0
   6 # which accompanies this distribution, and is available at
   7 # http://www.apache.org/licenses/LICENSE-2.0
   8 ##############################################################################
   9 from __future__ import absolute_import
  10 from __future__ import print_function
  11
  12 import os
  13 import logging
  14 import math
  15
  16 import pkg_resources
  17 from oslo_serialization import jsonutils
  18
  19 import yardstick.ssh as ssh
  20 from yardstick.benchmark.scenarios import base
  21
  22 LOG = logging.getLogger(__name__)
  23
  24 SSH_TIMEOUT = 60
  25 VNIC_TYPE_LIST = ["ovs", "sriov"]
  26 SRIOV_DRIVER_LIST = ["ixgbevf", "i40evf"]
  27
  28
  29 class Pktgen(base.Scenario):
  30     """Execute pktgen between two hosts
  31
  32   Parameters
  33     packetsize - packet size in bytes without the CRC
  34         type:    int
  35         unit:    bytes
  36         default: 60
  37     number_of_ports - number of UDP ports to test
  38         type:    int
  39         unit:    na
  40         default: 10
  41     duration - duration of the test
  42         type:    int
  43         unit:    seconds
  44         default: 20
  45     """
  46     __scenario_type__ = "Pktgen"
  47
  48     TARGET_SCRIPT = 'pktgen_benchmark.bash'
  49
  50     def __init__(self, scenario_cfg, context_cfg):
  51         self.scenario_cfg = scenario_cfg
  52         self.context_cfg = context_cfg
  53         self.vnic_name = "eth0"
  54         self.vnic_type = "ovs"
  55         self.queue_number = 1
  56         self.setup_done = False
  57         self.multiqueue_setup_done = False
  58
  59     def setup(self):
  60         """scenario setup"""
  61         self.target_script = pkg_resources.resource_filename(
  62             'yardstick.benchmark.scenarios.networking',
  63             Pktgen.TARGET_SCRIPT)
  64         host = self.context_cfg['host']
  65         target = self.context_cfg['target']
  66
  67         LOG.info("user:%s, target:%s", target['user'], target['ip'])
  68         self.server = ssh.SSH.from_node(target, defaults={"user": "ubuntu"})
  69         self.server.wait(timeout=600)
  70
  71         LOG.info("user:%s, host:%s", host['user'], host['ip'])
  72         self.client = ssh.SSH.from_node(host, defaults={"user": "ubuntu"})
  73         self.client.wait(timeout=600)
  74
  75         # copy script to host
  76         self.client._put_file_shell(self.target_script, '~/pktgen.sh')
  77
  78         self.setup_done = True
  79
  80     def multiqueue_setup(self):
  81         # one time setup stuff
  82         cmd = "sudo sysctl -w net.core.netdev_budget=3000"
  83         self.server.send_command(cmd)
  84         self.client.send_command(cmd)
  85
  86         cmd = "sudo sysctl -w net.core.netdev_max_backlog=100000"
  87         self.server.send_command(cmd)
  88         self.client.send_command(cmd)
  89
  90         """multiqueue setup"""
  91         if not self._is_irqbalance_disabled():
  92             self._disable_irqbalance()
  93
  94         vnic_driver_name = self._get_vnic_driver_name()
  95         if vnic_driver_name in SRIOV_DRIVER_LIST:
  96             self.vnic_type = "sriov"
  97
  98             # one time setup stuff
  99             cmd = "sudo ethtool -G %s rx 4096 tx 4096" % self.vnic_name
 100             self.server.send_command(cmd)
 101             self.client.send_command(cmd)
 102
 103             self.queue_number = self._get_sriov_queue_number()
 104             self._setup_irqmapping_sriov(self.queue_number)
 105         else:
 106             self.vnic_type = "ovs"
 107             self.queue_number = self._enable_ovs_multiqueue()
 108             self._setup_irqmapping_ovs(self.queue_number)
 109
 110         self.multiqueue_setup_done = True
 111
 112     def _get_vnic_driver_name(self):
 113         cmd = "readlink /sys/class/net/%s/device/driver" % self.vnic_name
 114         LOG.debug("Executing command: %s", cmd)
 115         status, stdout, stderr = self.server.execute(cmd)
 116         if status:
 117             raise RuntimeError(stderr)
 118         return os.path.basename(stdout.strip())
 119
 120     def _is_irqbalance_disabled(self):
 121         """Did we disable irqbalance already in the guest?"""
 122         is_disabled = False
 123         cmd = "grep ENABLED /etc/default/irqbalance"
 124         status, stdout, stderr = self.server.execute(cmd)
 125         if status:
 126             raise RuntimeError(stderr)
 127         if "0" in stdout:
 128             is_disabled = True
 129
 130         return is_disabled
 131
 132     def _disable_irqbalance(self):
 133         cmd = "sudo sed -i -e 's/ENABLED=\"1\"/ENABLED=\"0\"/g' " \
 134               "/etc/default/irqbalance"
 135         status, stdout, stderr = self.server.execute(cmd)
 136         status, stdout, stderr = self.client.execute(cmd)
 137         if status:
 138             raise RuntimeError(stderr)
 139
 140         cmd = "sudo service irqbalance stop"
 141         status, stdout, stderr = self.server.execute(cmd)
 142         status, stdout, stderr = self.client.execute(cmd)
 143         if status:
 144             raise RuntimeError(stderr)
 145
 146         cmd = "sudo service irqbalance disable"
 147         status, stdout, stderr = self.server.execute(cmd)
 148         status, stdout, stderr = self.client.execute(cmd)
 149         if status:
 150             raise RuntimeError(stderr)
 151
 152     def _setup_irqmapping_ovs(self, queue_number):
 153         cmd = "grep 'virtio0-input.0' /proc/interrupts |" \
 154               "awk '{match($0,/ +[0-9]+/)} " \
 155               "{print substr($1,RSTART,RLENGTH-1)}'"
 156         status, stdout, stderr = self.server.execute(cmd)
 157         if status:
 158             raise RuntimeError(stderr)
 159
 160         cmd = "echo 1 | sudo tee /proc/irq/%s/smp_affinity" % (int(stdout))
 161         status, stdout, stderr = self.server.execute(cmd)
 162         status, stdout, stderr = self.client.execute(cmd)
 163         if status:
 164             raise RuntimeError(stderr)
 165
 166         cmd = "grep 'virtio0-output.0' /proc/interrupts |" \
 167               "awk '{match($0,/ +[0-9]+/)} " \
 168               "{print substr($1,RSTART,RLENGTH-1)}'"
 169         status, stdout, stderr = self.server.execute(cmd)
 170         if status:
 171             raise RuntimeError(stderr)
 172
 173         cmd = "echo 1 | sudo tee /proc/irq/%s/smp_affinity" % (int(stdout))
 174         status, stdout, stderr = self.server.execute(cmd)
 175         status, stdout, stderr = self.client.execute(cmd)
 176         if status:
 177             raise RuntimeError(stderr)
 178
 179         if queue_number == 1:
 180             return
 181
 182         for i in range(1, queue_number):
 183
 184             smp_affinity_mask = format(1 << i, 'x')
 185
 186             cmd = "grep 'virtio0-input.%s' /proc/interrupts |" \
 187                   "awk '{match($0,/ +[0-9]+/)} " \
 188                   "{print substr($1,RSTART,RLENGTH-1)}'" % (i)
 189             status, stdout, stderr = self.server.execute(cmd)
 190             if status:
 191                 raise RuntimeError(stderr)
 192
 193             cmd = "echo %s | sudo tee /proc/irq/%s/smp_affinity" \
 194                 % (smp_affinity_mask, int(stdout))
 195             status, stdout, stderr = self.server.execute(cmd)
 196             status, stdout, stderr = self.client.execute(cmd)
 197             if status:
 198                 raise RuntimeError(stderr)
 199
 200             cmd = "grep 'virtio0-output.%s' /proc/interrupts |" \
 201                   "awk '{match($0,/ +[0-9]+/)} " \
 202                   "{print substr($1,RSTART,RLENGTH-1)}'" % (i)
 203             status, stdout, stderr = self.server.execute(cmd)
 204             if status:
 205                 raise RuntimeError(stderr)
 206
 207             cmd = "echo %s | sudo tee /proc/irq/%s/smp_affinity" \
 208                 % (smp_affinity_mask, int(stdout))
 209             status, stdout, stderr = self.server.execute(cmd)
 210             status, stdout, stderr = self.client.execute(cmd)
 211             if status:
 212                 raise RuntimeError(stderr)
 213
 214     def _setup_irqmapping_sriov(self, queue_number):
 215         cmd = "grep '%s-TxRx-0' /proc/interrupts |" \
 216               "awk '{match($0,/ +[0-9]+/)} " \
 217               "{print substr($1,RSTART,RLENGTH-1)}'" % self.vnic_name
 218         status, stdout, stderr = self.server.execute(cmd)
 219         if status:
 220             raise RuntimeError(stderr)
 221
 222         cmd = "echo 1 | sudo tee /proc/irq/%s/smp_affinity" % (int(stdout))
 223         status, stdout, stderr = self.server.execute(cmd)
 224         status, stdout, stderr = self.client.execute(cmd)
 225         if status:
 226             raise RuntimeError(stderr)
 227
 228         if queue_number == 1:
 229             return
 230
 231         for i in range(1, queue_number):
 232             smp_affinity_mask = format(1 << i, 'x')
 233
 234             cmd = "grep '%s-TxRx-%s' /proc/interrupts |" \
 235                   "awk '{match($0,/ +[0-9]+/)} " \
 236                   "{print substr($1,RSTART,RLENGTH-1)}'" % (self.vnic_name, i)
 237             status, stdout, stderr = self.server.execute(cmd)
 238             if status:
 239                 raise RuntimeError(stderr)
 240
 241             cmd = "echo %s | sudo tee /proc/irq/%s/smp_affinity" \
 242                 % (smp_affinity_mask, int(stdout))
 243             status, stdout, stderr = self.server.execute(cmd)
 244             status, stdout, stderr = self.client.execute(cmd)
 245             if status:
 246                 raise RuntimeError(stderr)
 247
 248     def _get_sriov_queue_number(self):
 249         """Get queue number from server as both VMs are the same"""
 250         cmd = "grep %s-TxRx- /proc/interrupts | wc -l" % self.vnic_name
 251         LOG.debug("Executing command: %s", cmd)
 252         status, stdout, stderr = self.server.execute(cmd)
 253         if status:
 254             raise RuntimeError(stderr)
 255         return int(stdout)
 256
 257     def _get_available_queue_number(self):
 258         """Get queue number from client as both VMs are the same"""
 259         cmd = "sudo ethtool -l %s | grep Combined | head -1 |" \
 260               "awk '{printf $2}'" % self.vnic_name
 261         LOG.debug("Executing command: %s", cmd)
 262         status, stdout, stderr = self.server.execute(cmd)
 263         if status:
 264             raise RuntimeError(stderr)
 265         return int(stdout)
 266
 267     def _get_usable_queue_number(self):
 268         """Get queue number from client as both VMs are the same"""
 269         cmd = "sudo ethtool -l %s | grep Combined | tail -1 |" \
 270               "awk '{printf $2}'" % self.vnic_name
 271         LOG.debug("Executing command: %s", cmd)
 272         status, stdout, stderr = self.server.execute(cmd)
 273         if status:
 274             raise RuntimeError(stderr)
 275         return int(stdout)
 276
 277     def _enable_ovs_multiqueue(self):
 278         available_queue_number = self._get_available_queue_number()
 279         usable_queue_number = self._get_usable_queue_number()
 280         if available_queue_number > 1 and \
 281            available_queue_number != usable_queue_number:
 282             cmd = "sudo ethtool -L %s combined %s" % \
 283                 (self.vnic_name, available_queue_number)
 284             LOG.debug("Executing command: %s", cmd)
 285             status, stdout, stderr = self.server.execute(cmd)
 286             status, stdout, stderr = self.client.execute(cmd)
 287             if status:
 288                 raise RuntimeError(stderr)
 289         return available_queue_number
 290
 291     def _iptables_setup(self):
 292         """Setup iptables on server to monitor for received packets"""
 293         cmd = "sudo iptables -F; " \
 294               "sudo iptables -A INPUT -p udp --dport 1000:%s -j DROP" \
 295               % (1000 + self.number_of_ports)
 296         LOG.debug("Executing command: %s", cmd)
 297         status, _, stderr = self.server.execute(cmd, timeout=SSH_TIMEOUT)
 298         if status:
 299             raise RuntimeError(stderr)
 300
 301     def _iptables_get_result(self):
 302         """Get packet statistics from server"""
 303         cmd = "sudo iptables -L INPUT -vnx |" \
 304               "awk '/dpts:1000:%s/ {{printf \"%%s\", $1}}'" \
 305               % (1000 + self.number_of_ports)
 306         LOG.debug("Executing command: %s", cmd)
 307         status, stdout, stderr = self.server.execute(cmd)
 308         if status:
 309             raise RuntimeError(stderr)
 310         return int(stdout)
 311
 312     def run(self, result):
 313         """execute the benchmark"""
 314
 315         if not self.setup_done:
 316             self.setup()
 317
 318         ipaddr = self.context_cfg["target"].get("ipaddr", '127.0.0.1')
 319
 320         options = self.scenario_cfg['options']
 321         packetsize = options.get("packetsize", 60)
 322         self.number_of_ports = options.get("number_of_ports", 10)
 323         self.vnic_name = options.get("vnic_name", "eth0")
 324         ovs_dpdk = options.get("ovs_dpdk", False)
 325         pps = options.get("pps", 1000000)
 326         multiqueue = options.get("multiqueue", False)
 327
 328         if multiqueue and not self.multiqueue_setup_done:
 329             self.multiqueue_setup()
 330
 331         # if run by a duration runner
 332         duration_time = self.scenario_cfg["runner"].get("duration", None) \
 333             if "runner" in self.scenario_cfg else None
 334         # if run by an arithmetic runner
 335         arithmetic_time = options.get("duration", None)
 336
 337         if duration_time:
 338             duration = duration_time
 339         elif arithmetic_time:
 340             duration = arithmetic_time
 341         else:
 342             duration = 20
 343
 344         self._iptables_setup()
 345
 346         queue_number = self.queue_number
 347
 348         # For native OVS, half of vCPUs are used by vhost kernel threads
 349         # hence set the queue_number to half number of vCPUs
 350         # e.g. set queue_number to 2 if there are 4 vCPUs
 351         if self.vnic_type == "ovs" and not ovs_dpdk and self.queue_number > 1:
 352             queue_number = self.queue_number / 2
 353
 354         cmd = "sudo bash pktgen.sh %s %s %s %s %s %s" \
 355             % (ipaddr, self.number_of_ports, packetsize,
 356                duration, queue_number, pps)
 357
 358         LOG.debug("Executing command: %s", cmd)
 359         status, stdout, stderr = self.client.execute(cmd, timeout=SSH_TIMEOUT)
 360
 361         if status:
 362             raise RuntimeError(stderr)
 363
 364         result.update(jsonutils.loads(stdout))
 365
 366         received = result['packets_received'] = self._iptables_get_result()
 367         sent = result['packets_sent']
 368         result['packetsize'] = packetsize
 369         # compatible with python3 /
 370         ppm = math.ceil(1000000.0 * (sent - received) / sent)
 371
 372         result['ppm'] = ppm
 373
 374         if "sla" in self.scenario_cfg:
 375             LOG.debug("Lost packets %d - Lost ppm %d", (sent - received), ppm)
 376             sla_max_ppm = int(self.scenario_cfg["sla"]["max_ppm"])
 377             assert ppm <= sla_max_ppm, "ppm %d > sla_max_ppm %d; " \
 378                 % (ppm, sla_max_ppm)
 379
 380
 381 def _test():  # pragma: no cover
 382     """internal test function"""
 383     key_filename = pkg_resources.resource_filename('yardstick.resources',
 384                                                    'files/yardstick_key')
 385     ctx = {
 386         'host': {
 387             'ip': '10.229.47.137',
 388             'user': 'root',
 389             'key_filename': key_filename
 390         },
 391         'target': {
 392             'ip': '10.229.47.137',
 393             'user': 'root',
 394             'key_filename': key_filename,
 395             'ipaddr': '10.229.47.137',
 396         }
 397     }
 398
 399     logger = logging.getLogger('yardstick')
 400     logger.setLevel(logging.DEBUG)
 401
 402     options = {'packetsize': 120}
 403     args = {'options': options}
 404     result = {}
 405
 406     p = Pktgen(args, ctx)
 407     p.run(result)
 408     print(result)
 409
 410 if __name__ == '__main__':
 411     _test()