1 ##############################################################################
2 # Copyright (c) 2015 Ericsson AB and others.
4 # All rights reserved. This program and the accompanying materials
5 # are made available under the terms of the Apache License, Version 2.0
6 # which accompanies this distribution, and is available at
7 # http://www.apache.org/licenses/LICENSE-2.0
8 ##############################################################################
9 from __future__ import absolute_import
10 from __future__ import print_function
17 from oslo_serialization import jsonutils
19 import yardstick.ssh as ssh
20 from yardstick.benchmark.scenarios import base
22 LOG = logging.getLogger(__name__)
25 VNIC_TYPE_LIST = ["ovs", "sriov"]
26 SRIOV_DRIVER_LIST = ["ixgbevf", "i40evf"]
29 class Pktgen(base.Scenario):
30 """Execute pktgen between two hosts
33 packetsize - packet size in bytes without the CRC
37 number_of_ports - number of UDP ports to test
41 duration - duration of the test
46 __scenario_type__ = "Pktgen"
48 TARGET_SCRIPT = 'pktgen_benchmark.bash'
50 def __init__(self, scenario_cfg, context_cfg):
51 self.scenario_cfg = scenario_cfg
52 self.context_cfg = context_cfg
53 self.vnic_name = "eth0"
54 self.vnic_type = "ovs"
56 self.setup_done = False
57 self.multiqueue_setup_done = False
61 self.target_script = pkg_resources.resource_filename(
62 'yardstick.benchmark.scenarios.networking',
64 host = self.context_cfg['host']
65 target = self.context_cfg['target']
67 LOG.info("user:%s, target:%s", target['user'], target['ip'])
68 self.server = ssh.SSH.from_node(target, defaults={"user": "ubuntu"})
69 self.server.wait(timeout=600)
71 LOG.info("user:%s, host:%s", host['user'], host['ip'])
72 self.client = ssh.SSH.from_node(host, defaults={"user": "ubuntu"})
73 self.client.wait(timeout=600)
76 self.client._put_file_shell(self.target_script, '~/pktgen.sh')
78 self.setup_done = True
80 def multiqueue_setup(self):
81 # one time setup stuff
82 cmd = "sudo sysctl -w net.core.netdev_budget=3000"
83 self.server.send_command(cmd)
84 self.client.send_command(cmd)
86 cmd = "sudo sysctl -w net.core.netdev_max_backlog=100000"
87 self.server.send_command(cmd)
88 self.client.send_command(cmd)
91 if not self._is_irqbalance_disabled():
92 self._disable_irqbalance()
94 vnic_driver_name = self._get_vnic_driver_name()
95 if vnic_driver_name in SRIOV_DRIVER_LIST:
96 self.vnic_type = "sriov"
98 # one time setup stuff
99 cmd = "sudo ethtool -G %s rx 4096 tx 4096" % self.vnic_name
100 self.server.send_command(cmd)
101 self.client.send_command(cmd)
103 self.queue_number = self._get_sriov_queue_number()
104 self._setup_irqmapping_sriov(self.queue_number)
106 self.vnic_type = "ovs"
107 self.queue_number = self._enable_ovs_multiqueue()
108 self._setup_irqmapping_ovs(self.queue_number)
110 self.multiqueue_setup_done = True
112 def _get_vnic_driver_name(self):
113 cmd = "readlink /sys/class/net/%s/device/driver" % self.vnic_name
114 LOG.debug("Executing command: %s", cmd)
115 status, stdout, stderr = self.server.execute(cmd)
117 raise RuntimeError(stderr)
118 return os.path.basename(stdout.strip())
120 def _is_irqbalance_disabled(self):
121 """Did we disable irqbalance already in the guest?"""
123 cmd = "grep ENABLED /etc/default/irqbalance"
124 status, stdout, stderr = self.server.execute(cmd)
126 raise RuntimeError(stderr)
132 def _disable_irqbalance(self):
133 cmd = "sudo sed -i -e 's/ENABLED=\"1\"/ENABLED=\"0\"/g' " \
134 "/etc/default/irqbalance"
135 status, _, stderr = self.server.execute(cmd)
136 status, _, stderr = self.client.execute(cmd)
138 raise RuntimeError(stderr)
140 cmd = "sudo service irqbalance stop"
141 status, _, stderr = self.server.execute(cmd)
142 status, _, stderr = self.client.execute(cmd)
144 raise RuntimeError(stderr)
146 cmd = "sudo service irqbalance disable"
147 status, _, stderr = self.server.execute(cmd)
148 status, _, stderr = self.client.execute(cmd)
150 raise RuntimeError(stderr)
152 def _setup_irqmapping_ovs(self, queue_number):
153 cmd = "grep 'virtio0-input.0' /proc/interrupts |" \
154 "awk '{match($0,/ +[0-9]+/)} " \
155 "{print substr($1,RSTART,RLENGTH-1)}'"
156 status, stdout, stderr = self.server.execute(cmd)
158 raise RuntimeError(stderr)
160 cmd = "echo 1 | sudo tee /proc/irq/%s/smp_affinity" % (int(stdout))
161 status, _, stderr = self.server.execute(cmd)
162 status, _, stderr = self.client.execute(cmd)
164 raise RuntimeError(stderr)
166 cmd = "grep 'virtio0-output.0' /proc/interrupts |" \
167 "awk '{match($0,/ +[0-9]+/)} " \
168 "{print substr($1,RSTART,RLENGTH-1)}'"
169 status, stdout, stderr = self.server.execute(cmd)
171 raise RuntimeError(stderr)
173 cmd = "echo 1 | sudo tee /proc/irq/%s/smp_affinity" % (int(stdout))
174 status, _, stderr = self.server.execute(cmd)
175 status, _, stderr = self.client.execute(cmd)
177 raise RuntimeError(stderr)
179 if queue_number == 1:
182 for i in range(1, queue_number):
184 smp_affinity_mask = format(1 << i, 'x')
186 cmd = "grep 'virtio0-input.%s' /proc/interrupts |" \
187 "awk '{match($0,/ +[0-9]+/)} " \
188 "{print substr($1,RSTART,RLENGTH-1)}'" % (i)
189 status, stdout, stderr = self.server.execute(cmd)
191 raise RuntimeError(stderr)
193 cmd = "echo %s | sudo tee /proc/irq/%s/smp_affinity" \
194 % (smp_affinity_mask, int(stdout))
195 status, _, stderr = self.server.execute(cmd)
196 status, _, stderr = self.client.execute(cmd)
198 raise RuntimeError(stderr)
200 cmd = "grep 'virtio0-output.%s' /proc/interrupts |" \
201 "awk '{match($0,/ +[0-9]+/)} " \
202 "{print substr($1,RSTART,RLENGTH-1)}'" % (i)
203 status, stdout, stderr = self.server.execute(cmd)
205 raise RuntimeError(stderr)
207 cmd = "echo %s | sudo tee /proc/irq/%s/smp_affinity" \
208 % (smp_affinity_mask, int(stdout))
209 status, _, stderr = self.server.execute(cmd)
210 status, _, stderr = self.client.execute(cmd)
212 raise RuntimeError(stderr)
214 def _setup_irqmapping_sriov(self, queue_number):
215 cmd = "grep '%s-TxRx-0' /proc/interrupts |" \
216 "awk '{match($0,/ +[0-9]+/)} " \
217 "{print substr($1,RSTART,RLENGTH-1)}'" % self.vnic_name
218 status, stdout, stderr = self.server.execute(cmd)
220 raise RuntimeError(stderr)
222 cmd = "echo 1 | sudo tee /proc/irq/%s/smp_affinity" % (int(stdout))
223 status, _, stderr = self.server.execute(cmd)
224 status, _, stderr = self.client.execute(cmd)
226 raise RuntimeError(stderr)
228 if queue_number == 1:
231 for i in range(1, queue_number):
232 smp_affinity_mask = format(1 << i, 'x')
234 cmd = "grep '%s-TxRx-%s' /proc/interrupts |" \
235 "awk '{match($0,/ +[0-9]+/)} " \
236 "{print substr($1,RSTART,RLENGTH-1)}'" % (self.vnic_name, i)
237 status, stdout, stderr = self.server.execute(cmd)
239 raise RuntimeError(stderr)
241 cmd = "echo %s | sudo tee /proc/irq/%s/smp_affinity" \
242 % (smp_affinity_mask, int(stdout))
243 status, _, stderr = self.server.execute(cmd)
244 status, _, stderr = self.client.execute(cmd)
246 raise RuntimeError(stderr)
248 def _get_sriov_queue_number(self):
249 """Get queue number from server as both VMs are the same"""
250 cmd = "grep %s-TxRx- /proc/interrupts | wc -l" % self.vnic_name
251 LOG.debug("Executing command: %s", cmd)
252 status, stdout, stderr = self.server.execute(cmd)
254 raise RuntimeError(stderr)
257 def _get_available_queue_number(self):
258 """Get queue number from client as both VMs are the same"""
259 cmd = "sudo ethtool -l %s | grep Combined | head -1 |" \
260 "awk '{printf $2}'" % self.vnic_name
261 LOG.debug("Executing command: %s", cmd)
262 status, stdout, stderr = self.server.execute(cmd)
264 raise RuntimeError(stderr)
267 def _get_usable_queue_number(self):
268 """Get queue number from client as both VMs are the same"""
269 cmd = "sudo ethtool -l %s | grep Combined | tail -1 |" \
270 "awk '{printf $2}'" % self.vnic_name
271 LOG.debug("Executing command: %s", cmd)
272 status, stdout, stderr = self.server.execute(cmd)
274 raise RuntimeError(stderr)
277 def _enable_ovs_multiqueue(self):
278 available_queue_number = self._get_available_queue_number()
279 usable_queue_number = self._get_usable_queue_number()
280 if available_queue_number > 1 and \
281 available_queue_number != usable_queue_number:
282 cmd = "sudo ethtool -L %s combined %s" % \
283 (self.vnic_name, available_queue_number)
284 LOG.debug("Executing command: %s", cmd)
285 status, _, stderr = self.server.execute(cmd)
286 status, _, stderr = self.client.execute(cmd)
288 raise RuntimeError(stderr)
289 return available_queue_number
291 def _iptables_setup(self):
292 """Setup iptables on server to monitor for received packets"""
293 cmd = "sudo iptables -F; " \
294 "sudo iptables -A INPUT -p udp --dport 1000:%s -j DROP" \
295 % (1000 + self.number_of_ports)
296 LOG.debug("Executing command: %s", cmd)
297 status, _, stderr = self.server.execute(cmd, timeout=SSH_TIMEOUT)
299 raise RuntimeError(stderr)
301 def _iptables_get_result(self):
302 """Get packet statistics from server"""
303 cmd = "sudo iptables -L INPUT -vnx |" \
304 "awk '/dpts:1000:%s/ {{printf \"%%s\", $1}}'" \
305 % (1000 + self.number_of_ports)
306 LOG.debug("Executing command: %s", cmd)
307 status, stdout, stderr = self.server.execute(cmd)
309 raise RuntimeError(stderr)
312 def run(self, result):
313 """execute the benchmark"""
315 if not self.setup_done:
318 ipaddr = self.context_cfg["target"].get("ipaddr", '127.0.0.1')
320 options = self.scenario_cfg['options']
321 packetsize = options.get("packetsize", 60)
322 self.number_of_ports = options.get("number_of_ports", 10)
323 self.vnic_name = options.get("vnic_name", "eth0")
324 ovs_dpdk = options.get("ovs_dpdk", False)
325 pps = options.get("pps", 1000000)
326 multiqueue = options.get("multiqueue", False)
328 if multiqueue and not self.multiqueue_setup_done:
329 self.multiqueue_setup()
331 # if run by a duration runner
332 duration_time = self.scenario_cfg["runner"].get("duration", None) \
333 if "runner" in self.scenario_cfg else None
334 # if run by an arithmetic runner
335 arithmetic_time = options.get("duration", None)
338 duration = duration_time
339 elif arithmetic_time:
340 duration = arithmetic_time
344 self._iptables_setup()
346 queue_number = self.queue_number
348 # For native OVS, half of vCPUs are used by vhost kernel threads
349 # hence set the queue_number to half number of vCPUs
350 # e.g. set queue_number to 2 if there are 4 vCPUs
351 if self.vnic_type == "ovs" and not ovs_dpdk and self.queue_number > 1:
352 queue_number = self.queue_number / 2
354 cmd = "sudo bash pktgen.sh %s %s %s %s %s %s" \
355 % (ipaddr, self.number_of_ports, packetsize,
356 duration, queue_number, pps)
358 LOG.debug("Executing command: %s", cmd)
359 status, stdout, stderr = self.client.execute(cmd, timeout=SSH_TIMEOUT)
362 raise RuntimeError(stderr)
364 result.update(jsonutils.loads(stdout))
366 received = result['packets_received'] = self._iptables_get_result()
367 sent = result['packets_sent']
368 result['packetsize'] = packetsize
369 # compatible with python3 /
370 ppm = math.ceil(1000000.0 * (sent - received) / sent)
374 if "sla" in self.scenario_cfg:
375 LOG.debug("Lost packets %d - Lost ppm %d", (sent - received), ppm)
376 sla_max_ppm = int(self.scenario_cfg["sla"]["max_ppm"])
377 self.verify_SLA(ppm <= sla_max_ppm,
378 "ppm %d > sla_max_ppm %d; " % (ppm, sla_max_ppm))
381 def _test(): # pragma: no cover
382 """internal test function"""
383 key_filename = pkg_resources.resource_filename('yardstick.resources',
384 'files/yardstick_key')
387 'ip': '10.229.47.137',
389 'key_filename': key_filename
392 'ip': '10.229.47.137',
394 'key_filename': key_filename,
395 'ipaddr': '10.229.47.137',
399 logger = logging.getLogger('yardstick')
400 logger.setLevel(logging.DEBUG)
402 options = {'packetsize': 120}
403 args = {'options': options}
406 p = Pktgen(args, ctx)
410 if __name__ == '__main__':