yardstick/benchmark/scenarios/networking/pktgen.py

   1 ##############################################################################
   2 # Copyright (c) 2015 Ericsson AB and others.
   3 #
   4 # All rights reserved. This program and the accompanying materials
   5 # are made available under the terms of the Apache License, Version 2.0
   6 # which accompanies this distribution, and is available at
   7 # http://www.apache.org/licenses/LICENSE-2.0
   8 ##############################################################################
   9 from __future__ import absolute_import
  10 from __future__ import print_function
  11
  12 import os
  13 import logging
  14
  15 import pkg_resources
  16 from oslo_serialization import jsonutils
  17
  18 import yardstick.ssh as ssh
  19 from yardstick.benchmark.scenarios import base
  20
  21 LOG = logging.getLogger(__name__)
  22
  23 SSH_TIMEOUT = 60
  24 VNIC_TYPE_LIST = ["ovs", "sriov"]
  25 SRIOV_DRIVER_LIST = ["ixgbevf", "i40evf"]
  26
  27
  28 class Pktgen(base.Scenario):
  29     """Execute pktgen between two hosts
  30
  31   Parameters
  32     packetsize - packet size in bytes without the CRC
  33         type:    int
  34         unit:    bytes
  35         default: 60
  36     number_of_ports - number of UDP ports to test
  37         type:    int
  38         unit:    na
  39         default: 10
  40     duration - duration of the test
  41         type:    int
  42         unit:    seconds
  43         default: 20
  44     """
  45     __scenario_type__ = "Pktgen"
  46
  47     TARGET_SCRIPT = 'pktgen_benchmark.bash'
  48
  49     def __init__(self, scenario_cfg, context_cfg):
  50         self.scenario_cfg = scenario_cfg
  51         self.context_cfg = context_cfg
  52         self.vnic_name = "eth0"
  53         self.vnic_type = "ovs"
  54         self.queue_number = 1
  55         self.setup_done = False
  56         self.multiqueue_setup_done = False
  57
  58     def setup(self):
  59         """scenario setup"""
  60         self.target_script = pkg_resources.resource_filename(
  61             'yardstick.benchmark.scenarios.networking',
  62             Pktgen.TARGET_SCRIPT)
  63         host = self.context_cfg['host']
  64         target = self.context_cfg['target']
  65
  66         LOG.info("user:%s, target:%s", target['user'], target['ip'])
  67         self.server = ssh.SSH.from_node(target, defaults={"user": "ubuntu"})
  68         self.server.wait(timeout=600)
  69
  70         LOG.info("user:%s, host:%s", host['user'], host['ip'])
  71         self.client = ssh.SSH.from_node(host, defaults={"user": "ubuntu"})
  72         self.client.wait(timeout=600)
  73
  74         # copy script to host
  75         self.client._put_file_shell(self.target_script, '~/pktgen.sh')
  76
  77         self.setup_done = True
  78
  79     def multiqueue_setup(self):
  80         # one time setup stuff
  81         cmd = "sudo sysctl -w net.core.netdev_budget=3000"
  82         self.server.send_command(cmd)
  83         self.client.send_command(cmd)
  84
  85         cmd = "sudo sysctl -w net.core.netdev_max_backlog=100000"
  86         self.server.send_command(cmd)
  87         self.client.send_command(cmd)
  88
  89         """multiqueue setup"""
  90         if not self._is_irqbalance_disabled():
  91             self._disable_irqbalance()
  92
  93         vnic_driver_name = self._get_vnic_driver_name()
  94         if vnic_driver_name in SRIOV_DRIVER_LIST:
  95             self.vnic_type = "sriov"
  96
  97             # one time setup stuff
  98             cmd = "sudo ethtool -G %s rx 4096 tx 4096" % self.vnic_name
  99             self.server.send_command(cmd)
 100             self.client.send_command(cmd)
 101
 102             self.queue_number = self._get_sriov_queue_number()
 103             self._setup_irqmapping_sriov(self.queue_number)
 104         else:
 105             self.vnic_type = "ovs"
 106             self.queue_number = self._enable_ovs_multiqueue()
 107             self._setup_irqmapping_ovs(self.queue_number)
 108
 109         self.multiqueue_setup_done = True
 110
 111     def _get_vnic_driver_name(self):
 112         cmd = "readlink /sys/class/net/%s/device/driver" % self.vnic_name
 113         LOG.debug("Executing command: %s", cmd)
 114         status, stdout, stderr = self.server.execute(cmd)
 115         if status:
 116             raise RuntimeError(stderr)
 117         return os.path.basename(stdout.strip())
 118
 119     def _is_irqbalance_disabled(self):
 120         """Did we disable irqbalance already in the guest?"""
 121         is_disabled = False
 122         cmd = "grep ENABLED /etc/default/irqbalance"
 123         status, stdout, stderr = self.server.execute(cmd)
 124         if status:
 125             raise RuntimeError(stderr)
 126         if "0" in stdout:
 127             is_disabled = True
 128
 129         return is_disabled
 130
 131     def _disable_irqbalance(self):
 132         cmd = "sudo sed -i -e 's/ENABLED=\"1\"/ENABLED=\"0\"/g' " \
 133               "/etc/default/irqbalance"
 134         status, stdout, stderr = self.server.execute(cmd)
 135         status, stdout, stderr = self.client.execute(cmd)
 136         if status:
 137             raise RuntimeError(stderr)
 138
 139         cmd = "sudo service irqbalance stop"
 140         status, stdout, stderr = self.server.execute(cmd)
 141         status, stdout, stderr = self.client.execute(cmd)
 142         if status:
 143             raise RuntimeError(stderr)
 144
 145         cmd = "sudo service irqbalance disable"
 146         status, stdout, stderr = self.server.execute(cmd)
 147         status, stdout, stderr = self.client.execute(cmd)
 148         if status:
 149             raise RuntimeError(stderr)
 150
 151     def _setup_irqmapping_ovs(self, queue_number):
 152         cmd = "grep 'virtio0-input.0' /proc/interrupts |" \
 153               "awk '{match($0,/ +[0-9]+/)} " \
 154               "{print substr($1,RSTART,RLENGTH-1)}'"
 155         status, stdout, stderr = self.server.execute(cmd)
 156         if status:
 157             raise RuntimeError(stderr)
 158
 159         cmd = "echo 1 | sudo tee /proc/irq/%s/smp_affinity" % (int(stdout))
 160         status, stdout, stderr = self.server.execute(cmd)
 161         status, stdout, stderr = self.client.execute(cmd)
 162         if status:
 163             raise RuntimeError(stderr)
 164
 165         cmd = "grep 'virtio0-output.0' /proc/interrupts |" \
 166               "awk '{match($0,/ +[0-9]+/)} " \
 167               "{print substr($1,RSTART,RLENGTH-1)}'"
 168         status, stdout, stderr = self.server.execute(cmd)
 169         if status:
 170             raise RuntimeError(stderr)
 171
 172         cmd = "echo 1 | sudo tee /proc/irq/%s/smp_affinity" % (int(stdout))
 173         status, stdout, stderr = self.server.execute(cmd)
 174         status, stdout, stderr = self.client.execute(cmd)
 175         if status:
 176             raise RuntimeError(stderr)
 177
 178         if queue_number == 1:
 179             return
 180
 181         for i in range(1, queue_number):
 182             cmd = "grep 'virtio0-input.%s' /proc/interrupts |" \
 183                   "awk '{match($0,/ +[0-9]+/)} " \
 184                   "{print substr($1,RSTART,RLENGTH-1)}'" % (i)
 185             status, stdout, stderr = self.server.execute(cmd)
 186             if status:
 187                 raise RuntimeError(stderr)
 188
 189             cmd = "echo %s | sudo tee /proc/irq/%s/smp_affinity" \
 190                 % (1 << i, int(stdout))
 191             status, stdout, stderr = self.server.execute(cmd)
 192             status, stdout, stderr = self.client.execute(cmd)
 193             if status:
 194                 raise RuntimeError(stderr)
 195
 196             cmd = "grep 'virtio0-output.%s' /proc/interrupts |" \
 197                   "awk '{match($0,/ +[0-9]+/)} " \
 198                   "{print substr($1,RSTART,RLENGTH-1)}'" % (i)
 199             status, stdout, stderr = self.server.execute(cmd)
 200             if status:
 201                 raise RuntimeError(stderr)
 202
 203             cmd = "echo %s | sudo tee /proc/irq/%s/smp_affinity" \
 204                 % (1 << i, int(stdout))
 205             status, stdout, stderr = self.server.execute(cmd)
 206             status, stdout, stderr = self.client.execute(cmd)
 207             if status:
 208                 raise RuntimeError(stderr)
 209
 210     def _setup_irqmapping_sriov(self, queue_number):
 211         cmd = "grep '%s-TxRx-0' /proc/interrupts |" \
 212               "awk '{match($0,/ +[0-9]+/)} " \
 213               "{print substr($1,RSTART,RLENGTH-1)}'" % self.vnic_name
 214         status, stdout, stderr = self.server.execute(cmd)
 215         if status:
 216             raise RuntimeError(stderr)
 217
 218         cmd = "echo 1 | sudo tee /proc/irq/%s/smp_affinity" % (int(stdout))
 219         status, stdout, stderr = self.server.execute(cmd)
 220         status, stdout, stderr = self.client.execute(cmd)
 221         if status:
 222             raise RuntimeError(stderr)
 223
 224         if queue_number == 1:
 225             return
 226
 227         for i in range(1, queue_number):
 228             cmd = "grep '%s-TxRx-%s' /proc/interrupts |" \
 229                   "awk '{match($0,/ +[0-9]+/)} " \
 230                   "{print substr($1,RSTART,RLENGTH-1)}'" % (self.vnic_name, i)
 231             status, stdout, stderr = self.server.execute(cmd)
 232             if status:
 233                 raise RuntimeError(stderr)
 234
 235             cmd = "echo %s | sudo tee /proc/irq/%s/smp_affinity" \
 236                 % (1 << i, int(stdout))
 237             status, stdout, stderr = self.server.execute(cmd)
 238             status, stdout, stderr = self.client.execute(cmd)
 239             if status:
 240                 raise RuntimeError(stderr)
 241
 242     def _get_sriov_queue_number(self):
 243         """Get queue number from server as both VMs are the same"""
 244         cmd = "grep %s-TxRx- /proc/interrupts | wc -l" % self.vnic_name
 245         LOG.debug("Executing command: %s", cmd)
 246         status, stdout, stderr = self.server.execute(cmd)
 247         if status:
 248             raise RuntimeError(stderr)
 249         return int(stdout)
 250
 251     def _get_available_queue_number(self):
 252         """Get queue number from client as both VMs are the same"""
 253         cmd = "sudo ethtool -l %s | grep Combined | head -1 |" \
 254               "awk '{printf $2}'" % self.vnic_name
 255         LOG.debug("Executing command: %s", cmd)
 256         status, stdout, stderr = self.server.execute(cmd)
 257         if status:
 258             raise RuntimeError(stderr)
 259         return int(stdout)
 260
 261     def _get_usable_queue_number(self):
 262         """Get queue number from client as both VMs are the same"""
 263         cmd = "sudo ethtool -l %s | grep Combined | tail -1 |" \
 264               "awk '{printf $2}'" % self.vnic_name
 265         LOG.debug("Executing command: %s", cmd)
 266         status, stdout, stderr = self.server.execute(cmd)
 267         if status:
 268             raise RuntimeError(stderr)
 269         return int(stdout)
 270
 271     def _enable_ovs_multiqueue(self):
 272         available_queue_number = self._get_available_queue_number()
 273         usable_queue_number = self._get_usable_queue_number()
 274         if available_queue_number > 1 and \
 275            available_queue_number != usable_queue_number:
 276             cmd = "sudo ethtool -L %s combined %s" % \
 277                 (self.vnic_name, available_queue_number)
 278             LOG.debug("Executing command: %s", cmd)
 279             status, stdout, stderr = self.server.execute(cmd)
 280             status, stdout, stderr = self.client.execute(cmd)
 281             if status:
 282                 raise RuntimeError(stderr)
 283         return available_queue_number
 284
 285     def _iptables_setup(self):
 286         """Setup iptables on server to monitor for received packets"""
 287         cmd = "sudo iptables -F; " \
 288               "sudo iptables -A INPUT -p udp --dport 1000:%s -j DROP" \
 289               % (1000 + self.number_of_ports)
 290         LOG.debug("Executing command: %s", cmd)
 291         status, _, stderr = self.server.execute(cmd, timeout=SSH_TIMEOUT)
 292         if status:
 293             raise RuntimeError(stderr)
 294
 295     def _iptables_get_result(self):
 296         """Get packet statistics from server"""
 297         cmd = "sudo iptables -L INPUT -vnx |" \
 298               "awk '/dpts:1000:%s/ {{printf \"%%s\", $1}}'" \
 299               % (1000 + self.number_of_ports)
 300         LOG.debug("Executing command: %s", cmd)
 301         status, stdout, stderr = self.server.execute(cmd)
 302         if status:
 303             raise RuntimeError(stderr)
 304         return int(stdout)
 305
 306     def run(self, result):
 307         """execute the benchmark"""
 308
 309         if not self.setup_done:
 310             self.setup()
 311
 312         ipaddr = self.context_cfg["target"].get("ipaddr", '127.0.0.1')
 313
 314         options = self.scenario_cfg['options']
 315         packetsize = options.get("packetsize", 60)
 316         self.number_of_ports = options.get("number_of_ports", 10)
 317         self.vnic_name = options.get("vnic_name", "eth0")
 318         ovs_dpdk = options.get("ovs_dpdk", False)
 319         pps = options.get("pps", 1000000)
 320         multiqueue = options.get("multiqueue", False)
 321
 322         if multiqueue and not self.multiqueue_setup_done:
 323             self.multiqueue_setup()
 324
 325         # if run by a duration runner
 326         duration_time = self.scenario_cfg["runner"].get("duration", None) \
 327             if "runner" in self.scenario_cfg else None
 328         # if run by an arithmetic runner
 329         arithmetic_time = options.get("duration", None)
 330
 331         if duration_time:
 332             duration = duration_time
 333         elif arithmetic_time:
 334             duration = arithmetic_time
 335         else:
 336             duration = 20
 337
 338         self._iptables_setup()
 339
 340         queue_number = self.queue_number
 341
 342         # For native OVS, half of vCPUs are used by vhost kernel threads
 343         # hence set the queue_number to half number of vCPUs
 344         # e.g. set queue_number to 2 if there are 4 vCPUs
 345         if self.vnic_type == "ovs" and not ovs_dpdk and self.queue_number > 1:
 346             queue_number = self.queue_number / 2
 347
 348         cmd = "sudo bash pktgen.sh %s %s %s %s %s %s" \
 349             % (ipaddr, self.number_of_ports, packetsize,
 350                duration, queue_number, pps)
 351
 352         LOG.debug("Executing command: %s", cmd)
 353         status, stdout, stderr = self.client.execute(cmd, timeout=SSH_TIMEOUT)
 354
 355         if status:
 356             raise RuntimeError(stderr)
 357
 358         result.update(jsonutils.loads(stdout))
 359
 360         result['packets_received'] = self._iptables_get_result()
 361         result['packetsize'] = packetsize
 362
 363         if "sla" in self.scenario_cfg:
 364             sent = result['packets_sent']
 365             received = result['packets_received']
 366             ppm = 1000000 * (sent - received) / sent
 367             # if ppm is 1, then 11 out of 10 million is no pass
 368             ppm += (sent - received) % sent > 0
 369             LOG.debug("Lost packets %d - Lost ppm %d", (sent - received), ppm)
 370             sla_max_ppm = int(self.scenario_cfg["sla"]["max_ppm"])
 371             assert ppm <= sla_max_ppm, "ppm %d > sla_max_ppm %d; " \
 372                 % (ppm, sla_max_ppm)
 373
 374
 375 def _test():  # pragma: no cover
 376     """internal test function"""
 377     key_filename = pkg_resources.resource_filename('yardstick.resources',
 378                                                    'files/yardstick_key')
 379     ctx = {
 380         'host': {
 381             'ip': '10.229.47.137',
 382             'user': 'root',
 383             'key_filename': key_filename
 384         },
 385         'target': {
 386             'ip': '10.229.47.137',
 387             'user': 'root',
 388             'key_filename': key_filename,
 389             'ipaddr': '10.229.47.137',
 390         }
 391     }
 392
 393     logger = logging.getLogger('yardstick')
 394     logger.setLevel(logging.DEBUG)
 395
 396     options = {'packetsize': 120}
 397     args = {'options': options}
 398     result = {}
 399
 400     p = Pktgen(args, ctx)
 401     p.run(result)
 402     print(result)
 403
 404 if __name__ == '__main__':
 405     _test()