yardstick/benchmark/scenarios/networking/pktgen.py

   1 ##############################################################################
   2 # Copyright (c) 2015 Ericsson AB and others.
   3 #
   4 # All rights reserved. This program and the accompanying materials
   5 # are made available under the terms of the Apache License, Version 2.0
   6 # which accompanies this distribution, and is available at
   7 # http://www.apache.org/licenses/LICENSE-2.0
   8 ##############################################################################
   9 from __future__ import absolute_import
  10 from __future__ import print_function
  11
  12 import os
  13 import logging
  14
  15 import pkg_resources
  16 from oslo_serialization import jsonutils
  17
  18 import yardstick.ssh as ssh
  19 from yardstick.benchmark.scenarios import base
  20
  21 LOG = logging.getLogger(__name__)
  22
  23 VNIC_TYPE_LIST = ["ovs", "sriov"]
  24 SRIOV_DRIVER_LIST = ["ixgbevf", "i40evf"]
  25
  26
  27 class Pktgen(base.Scenario):
  28     """Execute pktgen between two hosts
  29
  30   Parameters
  31     packetsize - packet size in bytes without the CRC
  32         type:    int
  33         unit:    bytes
  34         default: 60
  35     number_of_ports - number of UDP ports to test
  36         type:    int
  37         unit:    na
  38         default: 10
  39     duration - duration of the test
  40         type:    int
  41         unit:    seconds
  42         default: 20
  43     """
  44     __scenario_type__ = "Pktgen"
  45
  46     TARGET_SCRIPT = 'pktgen_benchmark.bash'
  47
  48     def __init__(self, scenario_cfg, context_cfg):
  49         self.scenario_cfg = scenario_cfg
  50         self.context_cfg = context_cfg
  51         self.vnic_name = "eth0"
  52         self.vnic_type = "ovs"
  53         self.queue_number = 1
  54         self.setup_done = False
  55         self.multiqueue_setup_done = False
  56
  57     def setup(self):
  58         """scenario setup"""
  59         self.target_script = pkg_resources.resource_filename(
  60             'yardstick.benchmark.scenarios.networking',
  61             Pktgen.TARGET_SCRIPT)
  62         host = self.context_cfg['host']
  63         target = self.context_cfg['target']
  64
  65         LOG.info("user:%s, target:%s", target['user'], target['ip'])
  66         self.server = ssh.SSH.from_node(target, defaults={"user": "ubuntu"})
  67         self.server.wait(timeout=600)
  68
  69         LOG.info("user:%s, host:%s", host['user'], host['ip'])
  70         self.client = ssh.SSH.from_node(host, defaults={"user": "ubuntu"})
  71         self.client.wait(timeout=600)
  72
  73         # copy script to host
  74         self.client._put_file_shell(self.target_script, '~/pktgen.sh')
  75
  76         self.setup_done = True
  77
  78     def multiqueue_setup(self):
  79         # one time setup stuff
  80         cmd = "sudo sysctl -w net.core.netdev_budget=3000"
  81         self.server.send_command(cmd)
  82         self.client.send_command(cmd)
  83
  84         cmd = "sudo sysctl -w net.core.netdev_max_backlog=100000"
  85         self.server.send_command(cmd)
  86         self.client.send_command(cmd)
  87
  88         """multiqueue setup"""
  89         if not self._is_irqbalance_disabled():
  90             self._disable_irqbalance()
  91
  92         vnic_driver_name = self._get_vnic_driver_name()
  93         if vnic_driver_name in SRIOV_DRIVER_LIST:
  94             self.vnic_type = "sriov"
  95
  96             # one time setup stuff
  97             cmd = "sudo ethtool -G %s rx 4096 tx 4096" % self.vnic_name
  98             self.server.send_command(cmd)
  99             self.client.send_command(cmd)
 100
 101             self.queue_number = self._get_sriov_queue_number()
 102             self._setup_irqmapping_sriov(self.queue_number)
 103         else:
 104             self.vnic_type = "ovs"
 105             self.queue_number = self._enable_ovs_multiqueue()
 106             self._setup_irqmapping_ovs(self.queue_number)
 107
 108         self.multiqueue_setup_done = True
 109
 110     def _get_vnic_driver_name(self):
 111         cmd = "readlink /sys/class/net/%s/device/driver" % self.vnic_name
 112         LOG.debug("Executing command: %s", cmd)
 113         status, stdout, stderr = self.server.execute(cmd)
 114         if status:
 115             raise RuntimeError(stderr)
 116         return os.path.basename(stdout.strip())
 117
 118     def _is_irqbalance_disabled(self):
 119         """Did we disable irqbalance already in the guest?"""
 120         is_disabled = False
 121         cmd = "grep ENABLED /etc/default/irqbalance"
 122         status, stdout, stderr = self.server.execute(cmd)
 123         if status:
 124             raise RuntimeError(stderr)
 125         if "0" in stdout:
 126             is_disabled = True
 127
 128         return is_disabled
 129
 130     def _disable_irqbalance(self):
 131         cmd = "sudo sed -i -e 's/ENABLED=\"1\"/ENABLED=\"0\"/g' " \
 132               "/etc/default/irqbalance"
 133         status, stdout, stderr = self.server.execute(cmd)
 134         status, stdout, stderr = self.client.execute(cmd)
 135         if status:
 136             raise RuntimeError(stderr)
 137
 138         cmd = "sudo service irqbalance stop"
 139         status, stdout, stderr = self.server.execute(cmd)
 140         status, stdout, stderr = self.client.execute(cmd)
 141         if status:
 142             raise RuntimeError(stderr)
 143
 144         cmd = "sudo service irqbalance disable"
 145         status, stdout, stderr = self.server.execute(cmd)
 146         status, stdout, stderr = self.client.execute(cmd)
 147         if status:
 148             raise RuntimeError(stderr)
 149
 150     def _setup_irqmapping_ovs(self, queue_number):
 151         cmd = "grep 'virtio0-input.0' /proc/interrupts |" \
 152               "awk '{match($0,/ +[0-9]+/)} " \
 153               "{print substr($1,RSTART,RLENGTH-1)}'"
 154         status, stdout, stderr = self.server.execute(cmd)
 155         if status:
 156             raise RuntimeError(stderr)
 157
 158         cmd = "echo 1 | sudo tee /proc/irq/%s/smp_affinity" % (int(stdout))
 159         status, stdout, stderr = self.server.execute(cmd)
 160         status, stdout, stderr = self.client.execute(cmd)
 161         if status:
 162             raise RuntimeError(stderr)
 163
 164         cmd = "grep 'virtio0-output.0' /proc/interrupts |" \
 165               "awk '{match($0,/ +[0-9]+/)} " \
 166               "{print substr($1,RSTART,RLENGTH-1)}'"
 167         status, stdout, stderr = self.server.execute(cmd)
 168         if status:
 169             raise RuntimeError(stderr)
 170
 171         cmd = "echo 1 | sudo tee /proc/irq/%s/smp_affinity" % (int(stdout))
 172         status, stdout, stderr = self.server.execute(cmd)
 173         status, stdout, stderr = self.client.execute(cmd)
 174         if status:
 175             raise RuntimeError(stderr)
 176
 177         if queue_number == 1:
 178             return
 179
 180         for i in range(1, queue_number):
 181             cmd = "grep 'virtio0-input.%s' /proc/interrupts |" \
 182                   "awk '{match($0,/ +[0-9]+/)} " \
 183                   "{print substr($1,RSTART,RLENGTH-1)}'" % (i)
 184             status, stdout, stderr = self.server.execute(cmd)
 185             if status:
 186                 raise RuntimeError(stderr)
 187
 188             cmd = "echo %s | sudo tee /proc/irq/%s/smp_affinity" \
 189                 % (1 << i, int(stdout))
 190             status, stdout, stderr = self.server.execute(cmd)
 191             status, stdout, stderr = self.client.execute(cmd)
 192             if status:
 193                 raise RuntimeError(stderr)
 194
 195             cmd = "grep 'virtio0-output.%s' /proc/interrupts |" \
 196                   "awk '{match($0,/ +[0-9]+/)} " \
 197                   "{print substr($1,RSTART,RLENGTH-1)}'" % (i)
 198             status, stdout, stderr = self.server.execute(cmd)
 199             if status:
 200                 raise RuntimeError(stderr)
 201
 202             cmd = "echo %s | sudo tee /proc/irq/%s/smp_affinity" \
 203                 % (1 << i, int(stdout))
 204             status, stdout, stderr = self.server.execute(cmd)
 205             status, stdout, stderr = self.client.execute(cmd)
 206             if status:
 207                 raise RuntimeError(stderr)
 208
 209     def _setup_irqmapping_sriov(self, queue_number):
 210         cmd = "grep '%s-TxRx-0' /proc/interrupts |" \
 211               "awk '{match($0,/ +[0-9]+/)} " \
 212               "{print substr($1,RSTART,RLENGTH-1)}'" % self.vnic_name
 213         status, stdout, stderr = self.server.execute(cmd)
 214         if status:
 215             raise RuntimeError(stderr)
 216
 217         cmd = "echo 1 | sudo tee /proc/irq/%s/smp_affinity" % (int(stdout))
 218         status, stdout, stderr = self.server.execute(cmd)
 219         status, stdout, stderr = self.client.execute(cmd)
 220         if status:
 221             raise RuntimeError(stderr)
 222
 223         if queue_number == 1:
 224             return
 225
 226         for i in range(1, queue_number):
 227             cmd = "grep '%s-TxRx-%s' /proc/interrupts |" \
 228                   "awk '{match($0,/ +[0-9]+/)} " \
 229                   "{print substr($1,RSTART,RLENGTH-1)}'" % (self.vnic_name, i)
 230             status, stdout, stderr = self.server.execute(cmd)
 231             if status:
 232                 raise RuntimeError(stderr)
 233
 234             cmd = "echo %s | sudo tee /proc/irq/%s/smp_affinity" \
 235                 % (1 << i, int(stdout))
 236             status, stdout, stderr = self.server.execute(cmd)
 237             status, stdout, stderr = self.client.execute(cmd)
 238             if status:
 239                 raise RuntimeError(stderr)
 240
 241     def _get_sriov_queue_number(self):
 242         """Get queue number from server as both VMs are the same"""
 243         cmd = "grep %s-TxRx- /proc/interrupts | wc -l" % self.vnic_name
 244         LOG.debug("Executing command: %s", cmd)
 245         status, stdout, stderr = self.server.execute(cmd)
 246         if status:
 247             raise RuntimeError(stderr)
 248         return int(stdout)
 249
 250     def _get_available_queue_number(self):
 251         """Get queue number from client as both VMs are the same"""
 252         cmd = "sudo ethtool -l %s | grep Combined | head -1 |" \
 253               "awk '{printf $2}'" % self.vnic_name
 254         LOG.debug("Executing command: %s", cmd)
 255         status, stdout, stderr = self.server.execute(cmd)
 256         if status:
 257             raise RuntimeError(stderr)
 258         return int(stdout)
 259
 260     def _get_usable_queue_number(self):
 261         """Get queue number from client as both VMs are the same"""
 262         cmd = "sudo ethtool -l %s | grep Combined | tail -1 |" \
 263               "awk '{printf $2}'" % self.vnic_name
 264         LOG.debug("Executing command: %s", cmd)
 265         status, stdout, stderr = self.server.execute(cmd)
 266         if status:
 267             raise RuntimeError(stderr)
 268         return int(stdout)
 269
 270     def _enable_ovs_multiqueue(self):
 271         available_queue_number = self._get_available_queue_number()
 272         usable_queue_number = self._get_usable_queue_number()
 273         if available_queue_number > 1 and \
 274            available_queue_number != usable_queue_number:
 275             cmd = "sudo ethtool -L %s combined %s" % \
 276                 (self.vnic_name, available_queue_number)
 277             LOG.debug("Executing command: %s", cmd)
 278             status, stdout, stderr = self.server.execute(cmd)
 279             status, stdout, stderr = self.client.execute(cmd)
 280             if status:
 281                 raise RuntimeError(stderr)
 282         return available_queue_number
 283
 284     def _iptables_setup(self):
 285         """Setup iptables on server to monitor for received packets"""
 286         cmd = "sudo iptables -F; " \
 287               "sudo iptables -A INPUT -p udp --dport 1000:%s -j DROP" \
 288               % (1000 + self.number_of_ports)
 289         LOG.debug("Executing command: %s", cmd)
 290         status, _, stderr = self.server.execute(cmd)
 291         if status:
 292             raise RuntimeError(stderr)
 293
 294     def _iptables_get_result(self):
 295         """Get packet statistics from server"""
 296         cmd = "sudo iptables -L INPUT -vnx |" \
 297               "awk '/dpts:1000:%s/ {{printf \"%%s\", $1}}'" \
 298               % (1000 + self.number_of_ports)
 299         LOG.debug("Executing command: %s", cmd)
 300         status, stdout, stderr = self.server.execute(cmd)
 301         if status:
 302             raise RuntimeError(stderr)
 303         return int(stdout)
 304
 305     def run(self, result):
 306         """execute the benchmark"""
 307
 308         if not self.setup_done:
 309             self.setup()
 310
 311         ipaddr = self.context_cfg["target"].get("ipaddr", '127.0.0.1')
 312
 313         options = self.scenario_cfg['options']
 314         packetsize = options.get("packetsize", 60)
 315         self.number_of_ports = options.get("number_of_ports", 10)
 316         self.vnic_name = options.get("vnic_name", "eth0")
 317         ovs_dpdk = options.get("ovs_dpdk", False)
 318         pps = options.get("pps", 1000000)
 319         multiqueue = options.get("multiqueue", False)
 320
 321         if multiqueue and not self.multiqueue_setup_done:
 322             self.multiqueue_setup()
 323
 324         # if run by a duration runner
 325         duration_time = self.scenario_cfg["runner"].get("duration", None) \
 326             if "runner" in self.scenario_cfg else None
 327         # if run by an arithmetic runner
 328         arithmetic_time = options.get("duration", None)
 329
 330         if duration_time:
 331             duration = duration_time
 332         elif arithmetic_time:
 333             duration = arithmetic_time
 334         else:
 335             duration = 20
 336
 337         self._iptables_setup()
 338
 339         queue_number = self.queue_number
 340
 341         # For native OVS, half of vCPUs are used by vhost kernel threads
 342         # hence set the queue_number to half number of vCPUs
 343         # e.g. set queue_number to 2 if there are 4 vCPUs
 344         if self.vnic_type == "ovs" and not ovs_dpdk and self.queue_number > 1:
 345             queue_number = self.queue_number / 2
 346
 347         cmd = "sudo bash pktgen.sh %s %s %s %s %s %s" \
 348             % (ipaddr, self.number_of_ports, packetsize,
 349                duration, queue_number, pps)
 350
 351         LOG.debug("Executing command: %s", cmd)
 352         status, stdout, stderr = self.client.execute(cmd)
 353
 354         if status:
 355             raise RuntimeError(stderr)
 356
 357         result.update(jsonutils.loads(stdout))
 358
 359         result['packets_received'] = self._iptables_get_result()
 360         result['packetsize'] = packetsize
 361
 362         if "sla" in self.scenario_cfg:
 363             sent = result['packets_sent']
 364             received = result['packets_received']
 365             ppm = 1000000 * (sent - received) / sent
 366             # if ppm is 1, then 11 out of 10 million is no pass
 367             ppm += (sent - received) % sent > 0
 368             LOG.debug("Lost packets %d - Lost ppm %d", (sent - received), ppm)
 369             sla_max_ppm = int(self.scenario_cfg["sla"]["max_ppm"])
 370             assert ppm <= sla_max_ppm, "ppm %d > sla_max_ppm %d; " \
 371                 % (ppm, sla_max_ppm)
 372
 373
 374 def _test():  # pragma: no cover
 375     """internal test function"""
 376     key_filename = pkg_resources.resource_filename('yardstick.resources',
 377                                                    'files/yardstick_key')
 378     ctx = {
 379         'host': {
 380             'ip': '10.229.47.137',
 381             'user': 'root',
 382             'key_filename': key_filename
 383         },
 384         'target': {
 385             'ip': '10.229.47.137',
 386             'user': 'root',
 387             'key_filename': key_filename,
 388             'ipaddr': '10.229.47.137',
 389         }
 390     }
 391
 392     logger = logging.getLogger('yardstick')
 393     logger.setLevel(logging.DEBUG)
 394
 395     options = {'packetsize': 120}
 396     args = {'options': options}
 397     result = {}
 398
 399     p = Pktgen(args, ctx)
 400     p.run(result)
 401     print(result)
 402
 403 if __name__ == '__main__':
 404     _test()