2 # Copyright 2016 Cisco Systems, Inc. All rights reserved.
4 # Licensed under the Apache License, Version 2.0 (the "License"); you may
5 # not use this file except in compliance with the License. You may obtain
6 # a copy of the License at
8 # http://www.apache.org/licenses/LICENSE-2.0
10 # Unless required by applicable law or agreed to in writing, software
11 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
12 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
13 # License for the specific language governing permissions and limitations
23 from attrdict import AttrDict
24 from nfvbench.config import config_loads
25 from nfvbench.credentials import Credentials
26 from nfvbench.fluentd import FluentLogHandler
28 from nfvbench.network import Interface
29 from nfvbench.network import Network
30 from nfvbench.specs import ChainType
31 from nfvbench.specs import Encaps
32 import nfvbench.traffic_gen.traffic_utils as traffic_utils
34 __location__ = os.path.realpath(os.path.join(os.getcwd(),
35 os.path.dirname(__file__)))
39 def openstack_vxlan_spec():
42 'openstack': AttrDict({
44 'encaps': Encaps.VxLAN}),
45 'run_spec': AttrDict({
52 # =========================================================================
54 # =========================================================================
56 def test_chain_interface():
57 iface = Interface('testname', 'vpp', tx_packets=1234, rx_packets=4321)
58 assert iface.name == 'testname'
59 assert iface.device == 'vpp'
60 assert iface.get_packet_count('tx') == 1234
61 assert iface.get_packet_count('rx') == 4321
62 assert iface.get_packet_count('wrong_key') == 0
65 # pylint: disable=redefined-outer-name
66 @pytest.fixture(scope='session')
68 return Interface('iface1', 'trex', tx_packets=10000, rx_packets=1234)
71 @pytest.fixture(scope='session')
73 return Interface('iface2', 'n9k', tx_packets=1234, rx_packets=9901)
76 @pytest.fixture(scope='session')
78 return Interface('iface3', 'n9k', tx_packets=9900, rx_packets=1234)
81 @pytest.fixture(scope='session')
83 return Interface('iface4', 'vpp', tx_packets=1234, rx_packets=9801)
86 @pytest.fixture(scope='session')
87 def net1(iface1, iface2, iface3, iface4):
88 return Network([iface1, iface2, iface3, iface4], reverse=False)
91 @pytest.fixture(scope='session')
92 def net2(iface1, iface2, iface3):
93 return Network([iface1, iface2, iface3], reverse=True)
96 def test_chain_network(net1, net2, iface1, iface2, iface3, iface4):
97 assert [iface1, iface2, iface3, iface4] == net1.get_interfaces()
98 assert [iface3, iface2, iface1] == net2.get_interfaces()
99 net2.add_interface(iface4)
100 assert [iface4, iface3, iface2, iface1] == net2.get_interfaces()
103 # pylint: enable=redefined-outer-name
105 # pylint: disable=pointless-string-statement
107 def test_chain_analysis(net1, monkeypatch, openstack_vxlan_spec):
108 def mock_empty(self, *args, **kwargs):
111 monkeypatch.setattr(ServiceChain, '_setup', mock_empty)
113 f = ServiceChain(AttrDict({'service_chain': 'DUMMY'}), [], {'tor': {}}, openstack_vxlan_spec,
114 lambda x, y, z: None)
115 result = f.get_analysis([net1])
116 assert result[1]['packet_drop_count'] == 99
117 assert result[1]['packet_drop_percentage'] == 0.99
118 assert result[2]['packet_drop_count'] == 1
119 assert result[2]['packet_drop_percentage'] == 0.01
120 assert result[3]['packet_drop_count'] == 99
121 assert result[3]['packet_drop_percentage'] == 0.99
124 result = f.get_analysis([net1])
125 assert result[1]['packet_drop_count'] == 0
126 assert result[1]['packet_drop_percentage'] == 0.0
127 assert result[2]['packet_drop_count'] == 0
128 assert result[2]['packet_drop_percentage'] == 0.0
129 assert result[3]['packet_drop_count'] == 0
130 assert result[3]['packet_drop_percentage'] == 0.0
134 def pvp_chain(monkeypatch, openstack_vxlan_spec):
135 tor_vni1 = Interface('vni-4097', 'n9k', 50, 77)
136 vsw_vni1 = Interface('vxlan_tunnel0', 'vpp', 77, 48)
137 vsw_vif1 = Interface('VirtualEthernet0/0/2', 'vpp', 48, 77)
138 vsw_vif2 = Interface('VirtualEthernet0/0/3', 'vpp', 77, 47)
139 vsw_vni2 = Interface('vxlan_tunnel1', 'vpp', 43, 77)
140 tor_vni2 = Interface('vni-4098', 'n9k', 77, 40)
142 def mock_init(self, *args, **kwargs):
143 self.vni_ports = [4097, 4098]
144 self.specs = openstack_vxlan_spec
147 'set_interface_counters': lambda: None,
150 self.worker = AttrDict({
154 def mock_empty(self, *args, **kwargs):
157 def mock_get_network(self, traffic_port, vni_id, reverse=False):
159 return Network([tor_vni1, vsw_vni1, vsw_vif1], reverse)
161 return Network([tor_vni2, vsw_vni2, vsw_vif2], reverse)
163 def mock_get_data(self):
166 monkeypatch.setattr(PVPChain, '_get_network', mock_get_network)
167 monkeypatch.setattr(PVPChain, '_get_data', mock_get_data)
168 monkeypatch.setattr(PVPChain, '_setup', mock_empty)
169 monkeypatch.setattr(VxLANWorker, '_clear_interfaces', mock_empty)
170 monkeypatch.setattr(PVPChain, '_generate_traffic', mock_empty)
171 monkeypatch.setattr(PVPChain, '__init__', mock_init)
172 return PVPChain(None, None, {'vm': None, 'vpp': None, 'tor': None, 'traffic': None}, None)
175 def test_pvp_chain_run(pvp_chain):
176 result = pvp_chain.run()
181 'direction-forward': [
183 ('interface', 'vni-4097'),
188 ('interface', 'vxlan_tunnel0'),
190 ('packet_count', 48),
191 ('packet_drop_count', 2),
192 ('packet_drop_percentage', 4.0)
195 ('interface', 'VirtualEthernet0/0/2'),
197 ('packet_count', 48),
198 ('packet_drop_count', 0),
199 ('packet_drop_percentage', 0.0)
202 ('interface', 'VirtualEthernet0/0/3'),
204 ('packet_count', 47),
205 ('packet_drop_count', 1),
206 ('packet_drop_percentage', 2.0)
209 ('interface', 'vxlan_tunnel1'),
211 ('packet_count', 43),
212 ('packet_drop_count', 4),
213 ('packet_drop_percentage', 8.0)
216 ('interface', 'vni-4098'),
218 ('packet_count', 40),
219 ('packet_drop_count', 3),
220 ('packet_drop_percentage', 6.0)
223 'direction-reverse': [
225 ('interface', 'vni-4098'),
230 ('interface', 'vxlan_tunnel1'),
232 ('packet_count', 77),
233 ('packet_drop_count', 0),
234 ('packet_drop_percentage', 0.0)
237 ('interface', 'VirtualEthernet0/0/3'),
239 ('packet_count', 77),
240 ('packet_drop_count', 0),
241 ('packet_drop_percentage', 0.0)
244 ('interface', 'VirtualEthernet0/0/2'),
246 ('packet_count', 77),
247 ('packet_drop_count', 0),
248 ('packet_drop_percentage', 0.0)
251 ('interface', 'vxlan_tunnel0'),
253 ('packet_count', 77),
254 ('packet_drop_count', 0),
255 ('packet_drop_percentage', 0.0)
258 ('interface', 'vni-4097'),
260 ('packet_count', 77),
261 ('packet_drop_count', 0),
262 ('packet_drop_percentage', 0.0)
267 assert result == expected_result
270 # =========================================================================
271 # Traffic client tests
272 # =========================================================================
274 def test_parse_rate_str():
275 parse_rate_str = traffic_utils.parse_rate_str
277 assert parse_rate_str('100%') == {'rate_percent': '100.0'}
278 assert parse_rate_str('37.5%') == {'rate_percent': '37.5'}
279 assert parse_rate_str('100%') == {'rate_percent': '100.0'}
280 assert parse_rate_str('60pps') == {'rate_pps': '60'}
281 assert parse_rate_str('60kpps') == {'rate_pps': '60000'}
282 assert parse_rate_str('6Mpps') == {'rate_pps': '6000000'}
283 assert parse_rate_str('6gpps') == {'rate_pps': '6000000000'}
284 assert parse_rate_str('80bps') == {'rate_bps': '80'}
285 assert parse_rate_str('80bps') == {'rate_bps': '80'}
286 assert parse_rate_str('80kbps') == {'rate_bps': '80000'}
287 assert parse_rate_str('80kBps') == {'rate_bps': '640000'}
288 assert parse_rate_str('80Mbps') == {'rate_bps': '80000000'}
289 assert parse_rate_str('80 MBps') == {'rate_bps': '640000000'}
290 assert parse_rate_str('80Gbps') == {'rate_bps': '80000000000'}
291 except Exception as exc:
292 assert False, exc.message
294 def should_raise_error(str):
304 assert should_raise_error('101')
305 assert should_raise_error('201%')
306 assert should_raise_error('10Kbps')
307 assert should_raise_error('0kbps')
308 assert should_raise_error('0pps')
309 assert should_raise_error('-1bps')
312 def test_rate_conversion():
313 assert traffic_utils.load_to_bps(50, 10000000000) == pytest.approx(5000000000.0)
314 assert traffic_utils.load_to_bps(37, 10000000000) == pytest.approx(3700000000.0)
315 assert traffic_utils.load_to_bps(100, 10000000000) == pytest.approx(10000000000.0)
317 assert traffic_utils.bps_to_load(5000000000.0, 10000000000) == pytest.approx(50.0)
318 assert traffic_utils.bps_to_load(3700000000.0, 10000000000) == pytest.approx(37.0)
319 assert traffic_utils.bps_to_load(10000000000.0, 10000000000) == pytest.approx(100.0)
321 assert traffic_utils.bps_to_pps(500000, 64) == pytest.approx(744.047619048)
322 assert traffic_utils.bps_to_pps(388888, 1518) == pytest.approx(31.6066319896)
323 assert traffic_utils.bps_to_pps(9298322222, 340.3) == pytest.approx(3225895.85831)
325 assert traffic_utils.pps_to_bps(744.047619048, 64) == pytest.approx(500000)
326 assert traffic_utils.pps_to_bps(31.6066319896, 1518) == pytest.approx(388888)
327 assert traffic_utils.pps_to_bps(3225895.85831, 340.3) == pytest.approx(9298322222)
329 # pps at 10Gbps line rate for 64 byte frames
330 LR_64B_PPS = 14880952
331 LR_1518B_PPS = 812743
333 def assert_equivalence(reference, value, allowance_pct=1):
334 '''Asserts if a value is equivalent to a reference value with given margin
336 :param float reference: reference value to compare to
337 :param float value: value to compare to reference
338 :param float allowance_pct: max allowed percentage of margin
339 0 : requires exact match
340 1 : must be equal within 1% of the reference value
347 assert abs(value - reference) * 100 / reference <= allowance_pct
349 def test_load_from_rate():
350 assert traffic_utils.get_load_from_rate('100%') == 100
351 assert_equivalence(100, traffic_utils.get_load_from_rate(str(LR_64B_PPS) + 'pps'))
352 assert_equivalence(50, traffic_utils.get_load_from_rate(str(LR_64B_PPS / 2) + 'pps'))
353 assert_equivalence(100, traffic_utils.get_load_from_rate('10Gbps'))
354 assert_equivalence(50, traffic_utils.get_load_from_rate('5000Mbps'))
355 assert_equivalence(1, traffic_utils.get_load_from_rate('100Mbps'))
356 assert_equivalence(100, traffic_utils.get_load_from_rate(str(LR_1518B_PPS) + 'pps',
357 avg_frame_size=1518))
358 assert_equivalence(100, traffic_utils.get_load_from_rate(str(LR_1518B_PPS * 2) + 'pps',
364 def traffic_client(monkeypatch):
366 def mock_init(self, *args, **kwargs):
368 'bidirectional': False,
369 'l2frame_size': '64',
371 'rates': [{'rate_percent': '10'}, {'rate_pps': '1'}]
374 def mock_modify_load(self, load):
375 self.run_config['rates'][0] = {'rate_percent': str(load)}
376 self.current_load = load
378 monkeypatch.setattr(TrafficClient, '__init__', mock_init)
379 monkeypatch.setattr(TrafficClient, 'modify_load', mock_modify_load)
381 return TrafficClient()
385 # pylint: enable=pointless-string-statement
387 # =========================================================================
389 # =========================================================================
391 def setup_module(module):
392 nfvbench.log.setup(mute_stdout=True)
394 def test_no_credentials():
395 cred = Credentials('/completely/wrong/path/openrc', None, False)
397 # shouldn't get valid data unless user set environment variables
403 # Because trex_stl_lib may not be installed when running unit test
404 # nfvbench.traffic_client will try to import STLError:
405 # from trex_stl_lib.api import STLError
406 # will raise ImportError: No module named trex_stl_lib.api
408 import trex_stl_lib.api
410 assert trex_stl_lib.api
412 # Make up a trex_stl_lib.api.STLError class
413 class STLError(Exception):
417 from types import ModuleType
419 stl_lib_mod = ModuleType('trex_stl_lib')
420 sys.modules['trex_stl_lib'] = stl_lib_mod
421 api_mod = ModuleType('trex_stl_lib.api')
422 stl_lib_mod.api = api_mod
423 sys.modules['trex_stl_lib.api'] = api_mod
424 api_mod.STLError = STLError
426 # pylint: disable=wrong-import-position,ungrouped-imports
427 from nfvbench.traffic_client import Device
428 from nfvbench.traffic_client import IpBlock
429 from nfvbench.traffic_client import TrafficClient
430 from nfvbench.traffic_client import TrafficGeneratorFactory
432 # pylint: enable=wrong-import-position,ungrouped-imports
435 ipb = IpBlock('10.0.0.0', '0.0.0.1', 256)
436 assert ipb.get_ip() == '10.0.0.0'
437 assert ipb.get_ip(255) == '10.0.0.255'
438 with pytest.raises(IndexError):
440 # verify with step larger than 1
441 ipb = IpBlock('10.0.0.0', '0.0.0.2', 256)
442 assert ipb.get_ip() == '10.0.0.0'
443 assert ipb.get_ip(1) == '10.0.0.2'
444 assert ipb.get_ip(128) == '10.0.1.0'
445 assert ipb.get_ip(255) == '10.0.1.254'
446 with pytest.raises(IndexError):
450 def check_config(configs, cc, fc, src_ip, dst_ip, step_ip):
451 '''Verify that the range configs for each chain have adjacent IP ranges
452 of the right size and without holes between chains
454 step = Device.ip_to_int(step_ip)
456 sip = Device.ip_to_int(src_ip)
457 dip = Device.ip_to_int(dst_ip)
458 for index in range(cc):
459 config = configs[index]
460 assert config['ip_src_count'] == config['ip_dst_count']
461 assert Device.ip_to_int(config['ip_src_addr']) == sip
462 assert Device.ip_to_int(config['ip_dst_addr']) == dip
463 count = config['ip_src_count']
470 def create_device(fc, cc, ip, gip, tggip, step_ip):
471 return Device(0, 0, flow_count=fc, chain_count=cc, ip=ip, gateway_ip=gip, tg_gateway_ip=tggip,
472 ip_addrs_step=step_ip,
473 tg_gateway_ip_addrs_step=step_ip,
474 gateway_ip_addrs_step=step_ip)
477 def check_device_flow_config(step_ip):
484 dev0 = create_device(fc, cc, ip0, gip, tggip, step_ip)
485 dev1 = create_device(fc, cc, ip1, gip, tggip, step_ip)
486 dev0.set_destination(dev1)
487 configs = dev0.get_stream_configs(ChainType.EXT)
488 check_config(configs, cc, fc, ip0, ip1, step_ip)
491 def test_device_flow_config():
492 check_device_flow_config('0.0.0.1')
493 check_device_flow_config('0.0.0.2')
496 def test_device_ip_range():
497 def ip_range_overlaps(ip0, ip1, flows):
500 dev0 = create_device(flows, 10, ip0, gip, tggip, '0.0.0.1')
501 dev1 = create_device(flows, 10, ip1, gip, tggip, '0.0.0.1')
502 dev0.set_destination(dev1)
503 return dev0.ip_range_overlaps()
505 assert not ip_range_overlaps('10.0.0.0', '20.0.0.0', 10000)
506 assert ip_range_overlaps('10.0.0.0', '10.0.1.0', 10000)
507 assert ip_range_overlaps('10.0.0.0', '10.0.1.0', 257)
508 assert ip_range_overlaps('10.0.1.0', '10.0.0.0', 257)
512 refcfg = {1: 100, 2: {21: 100, 22: 200}, 3: None}
513 res1 = {1: 10, 2: {21: 100, 22: 200}, 3: None}
514 res2 = {1: 100, 2: {21: 1000, 22: 200}, 3: None}
515 res3 = {1: 100, 2: {21: 100, 22: 200}, 3: "abc"}
516 assert config_loads("{}", refcfg) == refcfg
517 assert config_loads("{1: 10}", refcfg) == res1
518 assert config_loads("{2: {21: 1000}}", refcfg) == res2
519 assert config_loads('{3: "abc"}', refcfg) == res3
522 # pairs of input string and expected subset (None if identical)
525 ["{2: {21: 100, 30: 50}}", "{2: {30: 50}}"],
526 ["{2: {0: 1, 1: 2}, 5: 5}", None],
527 ["{1: 'abc', 2: {21: 0}}", "{1: 'abc'}"],
530 for fail_pair in fail_pairs:
531 with pytest.raises(Exception) as e_info:
532 config_loads(fail_pair[0], refcfg)
533 expected = fail_pair[1]
535 expected = fail_pair[0]
536 assert expected in e_info.value.message
539 flavor = {'flavor': {'vcpus': 2, 'ram': 8192, 'disk': 0,
540 'extra_specs': {'hw:cpu_policy': 'dedicated'}}}
541 new_flavor = {'flavor': {'vcpus': 2, 'ram': 8192, 'disk': 0,
542 'extra_specs': {'hw:cpu_policy': 'dedicated', 'hw:numa_nodes': 2}}}
543 assert config_loads("{'flavor': {'extra_specs': {'hw:numa_nodes': 2}}}", flavor,
544 whitelist_keys=['alpha', 'extra_specs']) == new_flavor
548 logger = logging.getLogger('fluent-logger')
550 class FluentdConfig(dict):
551 def __getattr__(self, attr):
552 return self.get(attr)
556 'logging_tag': 'nfvbench',
557 'result_tag': 'resultnfvbench',
562 'logging_tag': 'nfvbench',
563 'result_tag': 'resultnfvbench',
569 'result_tag': 'resultnfvbench',
574 'logging_tag': 'nfvbench',
581 handler = FluentLogHandler(fluentd_configs=fluentd_configs)
582 logger.addHandler(handler)
583 logger.setLevel(logging.INFO)
585 logger.warning('test %d', 100)
588 raise Exception("test")
590 logger.exception("got exception")
592 def assert_ndr_pdr(stats, ndr, ndr_dr, pdr, pdr_dr):
593 assert stats['ndr']['rate_percent'] == ndr
594 assert stats['ndr']['stats']['overall']['drop_percentage'] == ndr_dr
595 assert_equivalence(pdr, stats['pdr']['rate_percent'])
596 assert_equivalence(pdr_dr, stats['pdr']['stats']['overall']['drop_percentage'])
598 def get_traffic_client():
600 'traffic_generator': {'host_name': 'nfvbench_tg',
601 'default_profile': 'dummy',
602 'generator_profile': [{'name': 'dummy',
605 'intf_speed': '10Gbps',
606 'interfaces': [{'port': 0, 'pci': 0},
607 {'port': 1, 'pci': 0}]}],
608 'ip_addrs_step': '0.0.0.1',
609 'ip_addrs': ['10.0.0.0/8', '20.0.0.0/8'],
610 'tg_gateway_ip_addrs': ['1.1.0.100', '2.2.0.100'],
611 'tg_gateway_ip_addrs_step': '0.0.0.1',
612 'gateway_ip_addrs': ['1.1.0.2', '2.2.0.2'],
613 'gateway_ip_addrs_step': '0.0.0.1',
614 'udp_src_port': None,
615 'udp_dst_port': None},
616 'generator_profile': 'dummy',
617 'service_chain': 'PVP',
618 'service_chain_count': 1,
620 'vlan_tagging': True,
628 'check_traffic_time_sec': 200,
629 'generic_poll_sec': 2,
630 'measurement': {'NDR': 0.001, 'PDR': 0.1, 'load_epsilon': 0.1},
632 generator_factory = TrafficGeneratorFactory(config)
633 config.generator_config = generator_factory.get_generator_config(config.generator_profile)
634 traffic_client = TrafficClient(config, skip_sleep=True)
635 traffic_client.start_traffic_generator()
636 traffic_client.set_traffic('64', True)
637 return traffic_client
639 def test_ndr_at_lr():
640 traffic_client = get_traffic_client()
641 tg = traffic_client.gen
643 # this is a perfect sut with no loss at LR
644 tg.set_response_curve(lr_dr=0, ndr=100, max_actual_tx=100, max_11_tx=100)
645 # tx packets should be line rate for 64B and no drops...
646 assert tg.get_tx_pps_dropped_pps(100) == (LR_64B_PPS, 0)
647 # NDR and PDR should be at 100%
648 traffic_client.ensure_end_to_end()
649 results = traffic_client.get_ndr_and_pdr()
651 assert_ndr_pdr(results, 200.0, 0.0, 200.0, 0.0)
653 def test_ndr_at_50():
654 traffic_client = get_traffic_client()
655 tg = traffic_client.gen
656 # this is a sut with an NDR of 50% and linear drop rate after NDR up to 20% drops at LR
657 # (meaning that if you send 100% TX, you will only receive 80% RX)
658 # the tg requested TX/actual TX ratio is 1up to 50%, after 50%
659 # is linear up 80% actuak TX when requesting 100%
660 tg.set_response_curve(lr_dr=20, ndr=50, max_actual_tx=80, max_11_tx=50)
661 # tx packets should be half line rate for 64B and no drops...
662 assert tg.get_tx_pps_dropped_pps(50) == (LR_64B_PPS / 2, 0)
663 # at 100% TX requested, actual TX is 80% where the drop rate is 3/5 of 20% of the actual TX
664 assert tg.get_tx_pps_dropped_pps(100) == (int(LR_64B_PPS * 0.8),
665 int(LR_64B_PPS * 0.8 * 0.6 * 0.2))
666 results = traffic_client.get_ndr_and_pdr()
667 assert_ndr_pdr(results, 100.0, 0.0, 100.781, 0.09374)
669 def test_ndr_pdr_low_cpu():
670 traffic_client = get_traffic_client()
671 tg = traffic_client.gen
672 # This test is for the case where the TG is underpowered and cannot send fast enough for the NDR
673 # true NDR=40%, actual TX at 50% = 30%, actual measured DR is 0%
674 # The ndr/pdr should bail out with a warning and a best effort measured NDR of 30%
675 tg.set_response_curve(lr_dr=50, ndr=40, max_actual_tx=60, max_11_tx=0)
676 # tx packets should be 30% at requested half line rate for 64B and no drops...
677 assert tg.get_tx_pps_dropped_pps(50) == (int(LR_64B_PPS * 0.3), 0)
678 results = traffic_client.get_ndr_and_pdr()
681 # pp = pprint.PrettyPrinter(indent=4)