1 ##############################################################################
2 # Copyright (c) 2015 Ericsson AB and others.
4 # All rights reserved. This program and the accompanying materials
5 # are made available under the terms of the Apache License, Version 2.0
6 # which accompanies this distribution, and is available at
7 # http://www.apache.org/licenses/LICENSE-2.0
8 ##############################################################################
9 from __future__ import absolute_import
10 from __future__ import print_function
16 from oslo_serialization import jsonutils
18 import yardstick.ssh as ssh
19 from yardstick.benchmark.scenarios import base
21 LOG = logging.getLogger(__name__)
24 VNIC_TYPE_LIST = ["ovs", "sriov"]
25 SRIOV_DRIVER_LIST = ["ixgbevf", "i40evf"]
28 class Pktgen(base.Scenario):
29 """Execute pktgen between two hosts
32 packetsize - packet size in bytes without the CRC
36 number_of_ports - number of UDP ports to test
40 duration - duration of the test
45 __scenario_type__ = "Pktgen"
47 TARGET_SCRIPT = 'pktgen_benchmark.bash'
49 def __init__(self, scenario_cfg, context_cfg):
50 self.scenario_cfg = scenario_cfg
51 self.context_cfg = context_cfg
52 self.vnic_name = "eth0"
53 self.vnic_type = "ovs"
55 self.setup_done = False
56 self.multiqueue_setup_done = False
60 self.target_script = pkg_resources.resource_filename(
61 'yardstick.benchmark.scenarios.networking',
63 host = self.context_cfg['host']
64 target = self.context_cfg['target']
66 LOG.info("user:%s, target:%s", target['user'], target['ip'])
67 self.server = ssh.SSH.from_node(target, defaults={"user": "ubuntu"})
68 self.server.wait(timeout=600)
70 LOG.info("user:%s, host:%s", host['user'], host['ip'])
71 self.client = ssh.SSH.from_node(host, defaults={"user": "ubuntu"})
72 self.client.wait(timeout=600)
75 self.client._put_file_shell(self.target_script, '~/pktgen.sh')
77 self.setup_done = True
79 def multiqueue_setup(self):
80 # one time setup stuff
81 cmd = "sudo sysctl -w net.core.netdev_budget=3000"
82 self.server.send_command(cmd)
83 self.client.send_command(cmd)
85 cmd = "sudo sysctl -w net.core.netdev_max_backlog=100000"
86 self.server.send_command(cmd)
87 self.client.send_command(cmd)
89 """multiqueue setup"""
90 if not self._is_irqbalance_disabled():
91 self._disable_irqbalance()
93 vnic_driver_name = self._get_vnic_driver_name()
94 if vnic_driver_name in SRIOV_DRIVER_LIST:
95 self.vnic_type = "sriov"
97 # one time setup stuff
98 cmd = "sudo ethtool -G %s rx 4096 tx 4096" % self.vnic_name
99 self.server.send_command(cmd)
100 self.client.send_command(cmd)
102 self.queue_number = self._get_sriov_queue_number()
103 self._setup_irqmapping_sriov(self.queue_number)
105 self.vnic_type = "ovs"
106 self.queue_number = self._enable_ovs_multiqueue()
107 self._setup_irqmapping_ovs(self.queue_number)
109 self.multiqueue_setup_done = True
111 def _get_vnic_driver_name(self):
112 cmd = "readlink /sys/class/net/%s/device/driver" % self.vnic_name
113 LOG.debug("Executing command: %s", cmd)
114 status, stdout, stderr = self.server.execute(cmd)
116 raise RuntimeError(stderr)
117 return os.path.basename(stdout.strip())
119 def _is_irqbalance_disabled(self):
120 """Did we disable irqbalance already in the guest?"""
122 cmd = "grep ENABLED /etc/default/irqbalance"
123 status, stdout, stderr = self.server.execute(cmd)
125 raise RuntimeError(stderr)
131 def _disable_irqbalance(self):
132 cmd = "sudo sed -i -e 's/ENABLED=\"1\"/ENABLED=\"0\"/g' " \
133 "/etc/default/irqbalance"
134 status, stdout, stderr = self.server.execute(cmd)
135 status, stdout, stderr = self.client.execute(cmd)
137 raise RuntimeError(stderr)
139 cmd = "sudo service irqbalance stop"
140 status, stdout, stderr = self.server.execute(cmd)
141 status, stdout, stderr = self.client.execute(cmd)
143 raise RuntimeError(stderr)
145 cmd = "sudo service irqbalance disable"
146 status, stdout, stderr = self.server.execute(cmd)
147 status, stdout, stderr = self.client.execute(cmd)
149 raise RuntimeError(stderr)
151 def _setup_irqmapping_ovs(self, queue_number):
152 cmd = "grep 'virtio0-input.0' /proc/interrupts |" \
153 "awk '{match($0,/ +[0-9]+/)} " \
154 "{print substr($1,RSTART,RLENGTH-1)}'"
155 status, stdout, stderr = self.server.execute(cmd)
157 raise RuntimeError(stderr)
159 cmd = "echo 1 | sudo tee /proc/irq/%s/smp_affinity" % (int(stdout))
160 status, stdout, stderr = self.server.execute(cmd)
161 status, stdout, stderr = self.client.execute(cmd)
163 raise RuntimeError(stderr)
165 cmd = "grep 'virtio0-output.0' /proc/interrupts |" \
166 "awk '{match($0,/ +[0-9]+/)} " \
167 "{print substr($1,RSTART,RLENGTH-1)}'"
168 status, stdout, stderr = self.server.execute(cmd)
170 raise RuntimeError(stderr)
172 cmd = "echo 1 | sudo tee /proc/irq/%s/smp_affinity" % (int(stdout))
173 status, stdout, stderr = self.server.execute(cmd)
174 status, stdout, stderr = self.client.execute(cmd)
176 raise RuntimeError(stderr)
178 if queue_number == 1:
181 for i in range(1, queue_number):
182 cmd = "grep 'virtio0-input.%s' /proc/interrupts |" \
183 "awk '{match($0,/ +[0-9]+/)} " \
184 "{print substr($1,RSTART,RLENGTH-1)}'" % (i)
185 status, stdout, stderr = self.server.execute(cmd)
187 raise RuntimeError(stderr)
189 cmd = "echo %s | sudo tee /proc/irq/%s/smp_affinity" \
190 % (1 << i, int(stdout))
191 status, stdout, stderr = self.server.execute(cmd)
192 status, stdout, stderr = self.client.execute(cmd)
194 raise RuntimeError(stderr)
196 cmd = "grep 'virtio0-output.%s' /proc/interrupts |" \
197 "awk '{match($0,/ +[0-9]+/)} " \
198 "{print substr($1,RSTART,RLENGTH-1)}'" % (i)
199 status, stdout, stderr = self.server.execute(cmd)
201 raise RuntimeError(stderr)
203 cmd = "echo %s | sudo tee /proc/irq/%s/smp_affinity" \
204 % (1 << i, int(stdout))
205 status, stdout, stderr = self.server.execute(cmd)
206 status, stdout, stderr = self.client.execute(cmd)
208 raise RuntimeError(stderr)
210 def _setup_irqmapping_sriov(self, queue_number):
211 cmd = "grep '%s-TxRx-0' /proc/interrupts |" \
212 "awk '{match($0,/ +[0-9]+/)} " \
213 "{print substr($1,RSTART,RLENGTH-1)}'" % self.vnic_name
214 status, stdout, stderr = self.server.execute(cmd)
216 raise RuntimeError(stderr)
218 cmd = "echo 1 | sudo tee /proc/irq/%s/smp_affinity" % (int(stdout))
219 status, stdout, stderr = self.server.execute(cmd)
220 status, stdout, stderr = self.client.execute(cmd)
222 raise RuntimeError(stderr)
224 if queue_number == 1:
227 for i in range(1, queue_number):
228 cmd = "grep '%s-TxRx-%s' /proc/interrupts |" \
229 "awk '{match($0,/ +[0-9]+/)} " \
230 "{print substr($1,RSTART,RLENGTH-1)}'" % (self.vnic_name, i)
231 status, stdout, stderr = self.server.execute(cmd)
233 raise RuntimeError(stderr)
235 cmd = "echo %s | sudo tee /proc/irq/%s/smp_affinity" \
236 % (1 << i, int(stdout))
237 status, stdout, stderr = self.server.execute(cmd)
238 status, stdout, stderr = self.client.execute(cmd)
240 raise RuntimeError(stderr)
242 def _get_sriov_queue_number(self):
243 """Get queue number from server as both VMs are the same"""
244 cmd = "grep %s-TxRx- /proc/interrupts | wc -l" % self.vnic_name
245 LOG.debug("Executing command: %s", cmd)
246 status, stdout, stderr = self.server.execute(cmd)
248 raise RuntimeError(stderr)
251 def _get_available_queue_number(self):
252 """Get queue number from client as both VMs are the same"""
253 cmd = "sudo ethtool -l %s | grep Combined | head -1 |" \
254 "awk '{printf $2}'" % self.vnic_name
255 LOG.debug("Executing command: %s", cmd)
256 status, stdout, stderr = self.server.execute(cmd)
258 raise RuntimeError(stderr)
261 def _get_usable_queue_number(self):
262 """Get queue number from client as both VMs are the same"""
263 cmd = "sudo ethtool -l %s | grep Combined | tail -1 |" \
264 "awk '{printf $2}'" % self.vnic_name
265 LOG.debug("Executing command: %s", cmd)
266 status, stdout, stderr = self.server.execute(cmd)
268 raise RuntimeError(stderr)
271 def _enable_ovs_multiqueue(self):
272 available_queue_number = self._get_available_queue_number()
273 usable_queue_number = self._get_usable_queue_number()
274 if available_queue_number > 1 and \
275 available_queue_number != usable_queue_number:
276 cmd = "sudo ethtool -L %s combined %s" % \
277 (self.vnic_name, available_queue_number)
278 LOG.debug("Executing command: %s", cmd)
279 status, stdout, stderr = self.server.execute(cmd)
280 status, stdout, stderr = self.client.execute(cmd)
282 raise RuntimeError(stderr)
283 return available_queue_number
285 def _iptables_setup(self):
286 """Setup iptables on server to monitor for received packets"""
287 cmd = "sudo iptables -F; " \
288 "sudo iptables -A INPUT -p udp --dport 1000:%s -j DROP" \
289 % (1000 + self.number_of_ports)
290 LOG.debug("Executing command: %s", cmd)
291 status, _, stderr = self.server.execute(cmd, timeout=SSH_TIMEOUT)
293 raise RuntimeError(stderr)
295 def _iptables_get_result(self):
296 """Get packet statistics from server"""
297 cmd = "sudo iptables -L INPUT -vnx |" \
298 "awk '/dpts:1000:%s/ {{printf \"%%s\", $1}}'" \
299 % (1000 + self.number_of_ports)
300 LOG.debug("Executing command: %s", cmd)
301 status, stdout, stderr = self.server.execute(cmd)
303 raise RuntimeError(stderr)
306 def run(self, result):
307 """execute the benchmark"""
309 if not self.setup_done:
312 ipaddr = self.context_cfg["target"].get("ipaddr", '127.0.0.1')
314 options = self.scenario_cfg['options']
315 packetsize = options.get("packetsize", 60)
316 self.number_of_ports = options.get("number_of_ports", 10)
317 self.vnic_name = options.get("vnic_name", "eth0")
318 ovs_dpdk = options.get("ovs_dpdk", False)
319 pps = options.get("pps", 1000000)
320 multiqueue = options.get("multiqueue", False)
322 if multiqueue and not self.multiqueue_setup_done:
323 self.multiqueue_setup()
325 # if run by a duration runner
326 duration_time = self.scenario_cfg["runner"].get("duration", None) \
327 if "runner" in self.scenario_cfg else None
328 # if run by an arithmetic runner
329 arithmetic_time = options.get("duration", None)
332 duration = duration_time
333 elif arithmetic_time:
334 duration = arithmetic_time
338 self._iptables_setup()
340 queue_number = self.queue_number
342 # For native OVS, half of vCPUs are used by vhost kernel threads
343 # hence set the queue_number to half number of vCPUs
344 # e.g. set queue_number to 2 if there are 4 vCPUs
345 if self.vnic_type == "ovs" and not ovs_dpdk and self.queue_number > 1:
346 queue_number = self.queue_number / 2
348 cmd = "sudo bash pktgen.sh %s %s %s %s %s %s" \
349 % (ipaddr, self.number_of_ports, packetsize,
350 duration, queue_number, pps)
352 LOG.debug("Executing command: %s", cmd)
353 status, stdout, stderr = self.client.execute(cmd, timeout=SSH_TIMEOUT)
356 raise RuntimeError(stderr)
358 result.update(jsonutils.loads(stdout))
360 result['packets_received'] = self._iptables_get_result()
361 result['packetsize'] = packetsize
363 if "sla" in self.scenario_cfg:
364 sent = result['packets_sent']
365 received = result['packets_received']
366 ppm = 1000000 * (sent - received) / sent
367 # if ppm is 1, then 11 out of 10 million is no pass
368 ppm += (sent - received) % sent > 0
369 LOG.debug("Lost packets %d - Lost ppm %d", (sent - received), ppm)
370 sla_max_ppm = int(self.scenario_cfg["sla"]["max_ppm"])
371 assert ppm <= sla_max_ppm, "ppm %d > sla_max_ppm %d; " \
375 def _test(): # pragma: no cover
376 """internal test function"""
377 key_filename = pkg_resources.resource_filename('yardstick.resources',
378 'files/yardstick_key')
381 'ip': '10.229.47.137',
383 'key_filename': key_filename
386 'ip': '10.229.47.137',
388 'key_filename': key_filename,
389 'ipaddr': '10.229.47.137',
393 logger = logging.getLogger('yardstick')
394 logger.setLevel(logging.DEBUG)
396 options = {'packetsize': 120}
397 args = {'options': options}
400 p = Pktgen(args, ctx)
404 if __name__ == '__main__':