1 ##############################################################################
2 # Copyright (c) 2015 Ericsson AB and others.
4 # All rights reserved. This program and the accompanying materials
5 # are made available under the terms of the Apache License, Version 2.0
6 # which accompanies this distribution, and is available at
7 # http://www.apache.org/licenses/LICENSE-2.0
8 ##############################################################################
9 from __future__ import absolute_import
10 from __future__ import print_function
16 from oslo_serialization import jsonutils
18 import yardstick.ssh as ssh
19 from yardstick.benchmark.scenarios import base
21 LOG = logging.getLogger(__name__)
23 VNIC_TYPE_LIST = ["ovs", "sriov"]
24 SRIOV_DRIVER_LIST = ["ixgbevf", "i40evf"]
27 class Pktgen(base.Scenario):
28 """Execute pktgen between two hosts
31 packetsize - packet size in bytes without the CRC
35 number_of_ports - number of UDP ports to test
39 duration - duration of the test
44 __scenario_type__ = "Pktgen"
46 TARGET_SCRIPT = 'pktgen_benchmark.bash'
48 def __init__(self, scenario_cfg, context_cfg):
49 self.scenario_cfg = scenario_cfg
50 self.context_cfg = context_cfg
51 self.vnic_name = "eth0"
52 self.vnic_type = "ovs"
54 self.setup_done = False
55 self.multiqueue_setup_done = False
59 self.target_script = pkg_resources.resource_filename(
60 'yardstick.benchmark.scenarios.networking',
62 host = self.context_cfg['host']
63 target = self.context_cfg['target']
65 LOG.info("user:%s, target:%s", target['user'], target['ip'])
66 self.server = ssh.SSH.from_node(target, defaults={"user": "ubuntu"})
67 self.server.wait(timeout=600)
69 LOG.info("user:%s, host:%s", host['user'], host['ip'])
70 self.client = ssh.SSH.from_node(host, defaults={"user": "ubuntu"})
71 self.client.wait(timeout=600)
74 self.client._put_file_shell(self.target_script, '~/pktgen.sh')
76 self.setup_done = True
78 def multiqueue_setup(self):
79 # one time setup stuff
80 cmd = "sudo sysctl -w net.core.netdev_budget=3000"
81 self.server.send_command(cmd)
82 self.client.send_command(cmd)
84 cmd = "sudo sysctl -w net.core.netdev_max_backlog=100000"
85 self.server.send_command(cmd)
86 self.client.send_command(cmd)
88 """multiqueue setup"""
89 if not self._is_irqbalance_disabled():
90 self._disable_irqbalance()
92 vnic_driver_name = self._get_vnic_driver_name()
93 if vnic_driver_name in SRIOV_DRIVER_LIST:
94 self.vnic_type = "sriov"
96 # one time setup stuff
97 cmd = "sudo ethtool -G %s rx 4096 tx 4096" % self.vnic_name
98 self.server.send_command(cmd)
99 self.client.send_command(cmd)
101 self.queue_number = self._get_sriov_queue_number()
102 self._setup_irqmapping_sriov(self.queue_number)
104 self.vnic_type = "ovs"
105 self.queue_number = self._enable_ovs_multiqueue()
106 self._setup_irqmapping_ovs(self.queue_number)
108 self.multiqueue_setup_done = True
110 def _get_vnic_driver_name(self):
111 cmd = "readlink /sys/class/net/%s/device/driver" % self.vnic_name
112 LOG.debug("Executing command: %s", cmd)
113 status, stdout, stderr = self.server.execute(cmd)
115 raise RuntimeError(stderr)
116 return os.path.basename(stdout.strip())
118 def _is_irqbalance_disabled(self):
119 """Did we disable irqbalance already in the guest?"""
121 cmd = "grep ENABLED /etc/default/irqbalance"
122 status, stdout, stderr = self.server.execute(cmd)
124 raise RuntimeError(stderr)
130 def _disable_irqbalance(self):
131 cmd = "sudo sed -i -e 's/ENABLED=\"1\"/ENABLED=\"0\"/g' " \
132 "/etc/default/irqbalance"
133 status, stdout, stderr = self.server.execute(cmd)
134 status, stdout, stderr = self.client.execute(cmd)
136 raise RuntimeError(stderr)
138 cmd = "sudo service irqbalance stop"
139 status, stdout, stderr = self.server.execute(cmd)
140 status, stdout, stderr = self.client.execute(cmd)
142 raise RuntimeError(stderr)
144 cmd = "sudo service irqbalance disable"
145 status, stdout, stderr = self.server.execute(cmd)
146 status, stdout, stderr = self.client.execute(cmd)
148 raise RuntimeError(stderr)
150 def _setup_irqmapping_ovs(self, queue_number):
151 cmd = "grep 'virtio0-input.0' /proc/interrupts |" \
152 "awk '{match($0,/ +[0-9]+/)} " \
153 "{print substr($1,RSTART,RLENGTH-1)}'"
154 status, stdout, stderr = self.server.execute(cmd)
156 raise RuntimeError(stderr)
158 cmd = "echo 1 | sudo tee /proc/irq/%s/smp_affinity" % (int(stdout))
159 status, stdout, stderr = self.server.execute(cmd)
160 status, stdout, stderr = self.client.execute(cmd)
162 raise RuntimeError(stderr)
164 cmd = "grep 'virtio0-output.0' /proc/interrupts |" \
165 "awk '{match($0,/ +[0-9]+/)} " \
166 "{print substr($1,RSTART,RLENGTH-1)}'"
167 status, stdout, stderr = self.server.execute(cmd)
169 raise RuntimeError(stderr)
171 cmd = "echo 1 | sudo tee /proc/irq/%s/smp_affinity" % (int(stdout))
172 status, stdout, stderr = self.server.execute(cmd)
173 status, stdout, stderr = self.client.execute(cmd)
175 raise RuntimeError(stderr)
177 if queue_number == 1:
180 for i in range(1, queue_number):
181 cmd = "grep 'virtio0-input.%s' /proc/interrupts |" \
182 "awk '{match($0,/ +[0-9]+/)} " \
183 "{print substr($1,RSTART,RLENGTH-1)}'" % (i)
184 status, stdout, stderr = self.server.execute(cmd)
186 raise RuntimeError(stderr)
188 cmd = "echo %s | sudo tee /proc/irq/%s/smp_affinity" \
189 % (1 << i, int(stdout))
190 status, stdout, stderr = self.server.execute(cmd)
191 status, stdout, stderr = self.client.execute(cmd)
193 raise RuntimeError(stderr)
195 cmd = "grep 'virtio0-output.%s' /proc/interrupts |" \
196 "awk '{match($0,/ +[0-9]+/)} " \
197 "{print substr($1,RSTART,RLENGTH-1)}'" % (i)
198 status, stdout, stderr = self.server.execute(cmd)
200 raise RuntimeError(stderr)
202 cmd = "echo %s | sudo tee /proc/irq/%s/smp_affinity" \
203 % (1 << i, int(stdout))
204 status, stdout, stderr = self.server.execute(cmd)
205 status, stdout, stderr = self.client.execute(cmd)
207 raise RuntimeError(stderr)
209 def _setup_irqmapping_sriov(self, queue_number):
210 cmd = "grep '%s-TxRx-0' /proc/interrupts |" \
211 "awk '{match($0,/ +[0-9]+/)} " \
212 "{print substr($1,RSTART,RLENGTH-1)}'" % self.vnic_name
213 status, stdout, stderr = self.server.execute(cmd)
215 raise RuntimeError(stderr)
217 cmd = "echo 1 | sudo tee /proc/irq/%s/smp_affinity" % (int(stdout))
218 status, stdout, stderr = self.server.execute(cmd)
219 status, stdout, stderr = self.client.execute(cmd)
221 raise RuntimeError(stderr)
223 if queue_number == 1:
226 for i in range(1, queue_number):
227 cmd = "grep '%s-TxRx-%s' /proc/interrupts |" \
228 "awk '{match($0,/ +[0-9]+/)} " \
229 "{print substr($1,RSTART,RLENGTH-1)}'" % (self.vnic_name, i)
230 status, stdout, stderr = self.server.execute(cmd)
232 raise RuntimeError(stderr)
234 cmd = "echo %s | sudo tee /proc/irq/%s/smp_affinity" \
235 % (1 << i, int(stdout))
236 status, stdout, stderr = self.server.execute(cmd)
237 status, stdout, stderr = self.client.execute(cmd)
239 raise RuntimeError(stderr)
241 def _get_sriov_queue_number(self):
242 """Get queue number from server as both VMs are the same"""
243 cmd = "grep %s-TxRx- /proc/interrupts | wc -l" % self.vnic_name
244 LOG.debug("Executing command: %s", cmd)
245 status, stdout, stderr = self.server.execute(cmd)
247 raise RuntimeError(stderr)
250 def _get_available_queue_number(self):
251 """Get queue number from client as both VMs are the same"""
252 cmd = "sudo ethtool -l %s | grep Combined | head -1 |" \
253 "awk '{printf $2}'" % self.vnic_name
254 LOG.debug("Executing command: %s", cmd)
255 status, stdout, stderr = self.server.execute(cmd)
257 raise RuntimeError(stderr)
260 def _get_usable_queue_number(self):
261 """Get queue number from client as both VMs are the same"""
262 cmd = "sudo ethtool -l %s | grep Combined | tail -1 |" \
263 "awk '{printf $2}'" % self.vnic_name
264 LOG.debug("Executing command: %s", cmd)
265 status, stdout, stderr = self.server.execute(cmd)
267 raise RuntimeError(stderr)
270 def _enable_ovs_multiqueue(self):
271 available_queue_number = self._get_available_queue_number()
272 usable_queue_number = self._get_usable_queue_number()
273 if available_queue_number > 1 and \
274 available_queue_number != usable_queue_number:
275 cmd = "sudo ethtool -L %s combined %s" % \
276 (self.vnic_name, available_queue_number)
277 LOG.debug("Executing command: %s", cmd)
278 status, stdout, stderr = self.server.execute(cmd)
279 status, stdout, stderr = self.client.execute(cmd)
281 raise RuntimeError(stderr)
282 return available_queue_number
284 def _iptables_setup(self):
285 """Setup iptables on server to monitor for received packets"""
286 cmd = "sudo iptables -F; " \
287 "sudo iptables -A INPUT -p udp --dport 1000:%s -j DROP" \
288 % (1000 + self.number_of_ports)
289 LOG.debug("Executing command: %s", cmd)
290 status, _, stderr = self.server.execute(cmd)
292 raise RuntimeError(stderr)
294 def _iptables_get_result(self):
295 """Get packet statistics from server"""
296 cmd = "sudo iptables -L INPUT -vnx |" \
297 "awk '/dpts:1000:%s/ {{printf \"%%s\", $1}}'" \
298 % (1000 + self.number_of_ports)
299 LOG.debug("Executing command: %s", cmd)
300 status, stdout, stderr = self.server.execute(cmd)
302 raise RuntimeError(stderr)
305 def run(self, result):
306 """execute the benchmark"""
308 if not self.setup_done:
311 ipaddr = self.context_cfg["target"].get("ipaddr", '127.0.0.1')
313 options = self.scenario_cfg['options']
314 packetsize = options.get("packetsize", 60)
315 self.number_of_ports = options.get("number_of_ports", 10)
316 self.vnic_name = options.get("vnic_name", "eth0")
317 ovs_dpdk = options.get("ovs_dpdk", False)
318 pps = options.get("pps", 1000000)
319 multiqueue = options.get("multiqueue", False)
321 if multiqueue and not self.multiqueue_setup_done:
322 self.multiqueue_setup()
324 # if run by a duration runner
325 duration_time = self.scenario_cfg["runner"].get("duration", None) \
326 if "runner" in self.scenario_cfg else None
327 # if run by an arithmetic runner
328 arithmetic_time = options.get("duration", None)
331 duration = duration_time
332 elif arithmetic_time:
333 duration = arithmetic_time
337 self._iptables_setup()
339 queue_number = self.queue_number
341 # For native OVS, half of vCPUs are used by vhost kernel threads
342 # hence set the queue_number to half number of vCPUs
343 # e.g. set queue_number to 2 if there are 4 vCPUs
344 if self.vnic_type == "ovs" and not ovs_dpdk and self.queue_number > 1:
345 queue_number = self.queue_number / 2
347 cmd = "sudo bash pktgen.sh %s %s %s %s %s %s" \
348 % (ipaddr, self.number_of_ports, packetsize,
349 duration, queue_number, pps)
351 LOG.debug("Executing command: %s", cmd)
352 status, stdout, stderr = self.client.execute(cmd)
355 raise RuntimeError(stderr)
357 result.update(jsonutils.loads(stdout))
359 result['packets_received'] = self._iptables_get_result()
360 result['packetsize'] = packetsize
362 if "sla" in self.scenario_cfg:
363 sent = result['packets_sent']
364 received = result['packets_received']
365 ppm = 1000000 * (sent - received) / sent
366 # if ppm is 1, then 11 out of 10 million is no pass
367 ppm += (sent - received) % sent > 0
368 LOG.debug("Lost packets %d - Lost ppm %d", (sent - received), ppm)
369 sla_max_ppm = int(self.scenario_cfg["sla"]["max_ppm"])
370 assert ppm <= sla_max_ppm, "ppm %d > sla_max_ppm %d; " \
374 def _test(): # pragma: no cover
375 """internal test function"""
376 key_filename = pkg_resources.resource_filename('yardstick.resources',
377 'files/yardstick_key')
380 'ip': '10.229.47.137',
382 'key_filename': key_filename
385 'ip': '10.229.47.137',
387 'key_filename': key_filename,
388 'ipaddr': '10.229.47.137',
392 logger = logging.getLogger('yardstick')
393 logger.setLevel(logging.DEBUG)
395 options = {'packetsize': 120}
396 args = {'options': options}
399 p = Pktgen(args, ctx)
403 if __name__ == '__main__':