2 # Copyright 2016 Cisco Systems, Inc. All rights reserved.
4 # Licensed under the Apache License, Version 2.0 (the "License"); you may
5 # not use this file except in compliance with the License. You may obtain
6 # a copy of the License at
8 # http://www.apache.org/licenses/LICENSE-2.0
10 # Unless required by applicable law or agreed to in writing, software
11 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
12 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
13 # License for the specific language governing permissions and limitations
23 from attrdict import AttrDict
24 from nfvbench.config import config_loads
25 from nfvbench.credentials import Credentials
26 from nfvbench.fluentd import FluentLogHandler
28 from nfvbench.network import Interface
29 from nfvbench.network import Network
30 from nfvbench.specs import ChainType
31 from nfvbench.specs import Encaps
32 import nfvbench.traffic_gen.traffic_utils as traffic_utils
34 __location__ = os.path.realpath(os.path.join(os.getcwd(),
35 os.path.dirname(__file__)))
39 def openstack_vxlan_spec():
42 'openstack': AttrDict({
44 'encaps': Encaps.VxLAN}),
45 'run_spec': AttrDict({
52 # =========================================================================
54 # =========================================================================
56 def test_chain_interface():
57 iface = Interface('testname', 'vpp', tx_packets=1234, rx_packets=4321)
58 assert iface.name == 'testname'
59 assert iface.device == 'vpp'
60 assert iface.get_packet_count('tx') == 1234
61 assert iface.get_packet_count('rx') == 4321
62 assert iface.get_packet_count('wrong_key') == 0
65 # pylint: disable=redefined-outer-name
66 @pytest.fixture(scope='session')
68 return Interface('iface1', 'trex', tx_packets=10000, rx_packets=1234)
71 @pytest.fixture(scope='session')
73 return Interface('iface2', 'n9k', tx_packets=1234, rx_packets=9901)
76 @pytest.fixture(scope='session')
78 return Interface('iface3', 'n9k', tx_packets=9900, rx_packets=1234)
81 @pytest.fixture(scope='session')
83 return Interface('iface4', 'vpp', tx_packets=1234, rx_packets=9801)
86 @pytest.fixture(scope='session')
87 def net1(iface1, iface2, iface3, iface4):
88 return Network([iface1, iface2, iface3, iface4], reverse=False)
91 @pytest.fixture(scope='session')
92 def net2(iface1, iface2, iface3):
93 return Network([iface1, iface2, iface3], reverse=True)
96 def test_chain_network(net1, net2, iface1, iface2, iface3, iface4):
97 assert [iface1, iface2, iface3, iface4] == net1.get_interfaces()
98 assert [iface3, iface2, iface1] == net2.get_interfaces()
99 net2.add_interface(iface4)
100 assert [iface4, iface3, iface2, iface1] == net2.get_interfaces()
103 # pylint: enable=redefined-outer-name
105 # pylint: disable=pointless-string-statement
107 def test_chain_analysis(net1, monkeypatch, openstack_vxlan_spec):
108 def mock_empty(self, *args, **kwargs):
111 monkeypatch.setattr(ServiceChain, '_setup', mock_empty)
113 f = ServiceChain(AttrDict({'service_chain': 'DUMMY'}), [], {'tor': {}}, openstack_vxlan_spec,
114 lambda x, y, z: None)
115 result = f.get_analysis([net1])
116 assert result[1]['packet_drop_count'] == 99
117 assert result[1]['packet_drop_percentage'] == 0.99
118 assert result[2]['packet_drop_count'] == 1
119 assert result[2]['packet_drop_percentage'] == 0.01
120 assert result[3]['packet_drop_count'] == 99
121 assert result[3]['packet_drop_percentage'] == 0.99
124 result = f.get_analysis([net1])
125 assert result[1]['packet_drop_count'] == 0
126 assert result[1]['packet_drop_percentage'] == 0.0
127 assert result[2]['packet_drop_count'] == 0
128 assert result[2]['packet_drop_percentage'] == 0.0
129 assert result[3]['packet_drop_count'] == 0
130 assert result[3]['packet_drop_percentage'] == 0.0
134 def pvp_chain(monkeypatch, openstack_vxlan_spec):
135 tor_vni1 = Interface('vni-4097', 'n9k', 50, 77)
136 vsw_vni1 = Interface('vxlan_tunnel0', 'vpp', 77, 48)
137 vsw_vif1 = Interface('VirtualEthernet0/0/2', 'vpp', 48, 77)
138 vsw_vif2 = Interface('VirtualEthernet0/0/3', 'vpp', 77, 47)
139 vsw_vni2 = Interface('vxlan_tunnel1', 'vpp', 43, 77)
140 tor_vni2 = Interface('vni-4098', 'n9k', 77, 40)
142 def mock_init(self, *args, **kwargs):
143 self.vni_ports = [4097, 4098]
144 self.specs = openstack_vxlan_spec
147 'set_interface_counters': lambda: None,
150 self.worker = AttrDict({
154 def mock_empty(self, *args, **kwargs):
157 def mock_get_network(self, traffic_port, vni_id, reverse=False):
159 return Network([tor_vni1, vsw_vni1, vsw_vif1], reverse)
161 return Network([tor_vni2, vsw_vni2, vsw_vif2], reverse)
163 def mock_get_data(self):
166 monkeypatch.setattr(PVPChain, '_get_network', mock_get_network)
167 monkeypatch.setattr(PVPChain, '_get_data', mock_get_data)
168 monkeypatch.setattr(PVPChain, '_setup', mock_empty)
169 monkeypatch.setattr(VxLANWorker, '_clear_interfaces', mock_empty)
170 monkeypatch.setattr(PVPChain, '_generate_traffic', mock_empty)
171 monkeypatch.setattr(PVPChain, '__init__', mock_init)
172 return PVPChain(None, None, {'vm': None, 'vpp': None, 'tor': None, 'traffic': None}, None)
175 def test_pvp_chain_run(pvp_chain):
176 result = pvp_chain.run()
181 'direction-forward': [
183 ('interface', 'vni-4097'),
188 ('interface', 'vxlan_tunnel0'),
190 ('packet_count', 48),
191 ('packet_drop_count', 2),
192 ('packet_drop_percentage', 4.0)
195 ('interface', 'VirtualEthernet0/0/2'),
197 ('packet_count', 48),
198 ('packet_drop_count', 0),
199 ('packet_drop_percentage', 0.0)
202 ('interface', 'VirtualEthernet0/0/3'),
204 ('packet_count', 47),
205 ('packet_drop_count', 1),
206 ('packet_drop_percentage', 2.0)
209 ('interface', 'vxlan_tunnel1'),
211 ('packet_count', 43),
212 ('packet_drop_count', 4),
213 ('packet_drop_percentage', 8.0)
216 ('interface', 'vni-4098'),
218 ('packet_count', 40),
219 ('packet_drop_count', 3),
220 ('packet_drop_percentage', 6.0)
223 'direction-reverse': [
225 ('interface', 'vni-4098'),
230 ('interface', 'vxlan_tunnel1'),
232 ('packet_count', 77),
233 ('packet_drop_count', 0),
234 ('packet_drop_percentage', 0.0)
237 ('interface', 'VirtualEthernet0/0/3'),
239 ('packet_count', 77),
240 ('packet_drop_count', 0),
241 ('packet_drop_percentage', 0.0)
244 ('interface', 'VirtualEthernet0/0/2'),
246 ('packet_count', 77),
247 ('packet_drop_count', 0),
248 ('packet_drop_percentage', 0.0)
251 ('interface', 'vxlan_tunnel0'),
253 ('packet_count', 77),
254 ('packet_drop_count', 0),
255 ('packet_drop_percentage', 0.0)
258 ('interface', 'vni-4097'),
260 ('packet_count', 77),
261 ('packet_drop_count', 0),
262 ('packet_drop_percentage', 0.0)
267 assert result == expected_result
270 # =========================================================================
271 # Traffic client tests
272 # =========================================================================
274 def test_parse_rate_str():
275 parse_rate_str = traffic_utils.parse_rate_str
277 assert parse_rate_str('100%') == {'rate_percent': '100.0'}
278 assert parse_rate_str('37.5%') == {'rate_percent': '37.5'}
279 assert parse_rate_str('100%') == {'rate_percent': '100.0'}
280 assert parse_rate_str('60pps') == {'rate_pps': '60'}
281 assert parse_rate_str('60kpps') == {'rate_pps': '60000'}
282 assert parse_rate_str('6Mpps') == {'rate_pps': '6000000'}
283 assert parse_rate_str('6gpps') == {'rate_pps': '6000000000'}
284 assert parse_rate_str('80bps') == {'rate_bps': '80'}
285 assert parse_rate_str('80bps') == {'rate_bps': '80'}
286 assert parse_rate_str('80kbps') == {'rate_bps': '80000'}
287 assert parse_rate_str('80kBps') == {'rate_bps': '640000'}
288 assert parse_rate_str('80Mbps') == {'rate_bps': '80000000'}
289 assert parse_rate_str('80 MBps') == {'rate_bps': '640000000'}
290 assert parse_rate_str('80Gbps') == {'rate_bps': '80000000000'}
291 except Exception as exc:
292 assert False, exc.message
294 def should_raise_error(str):
304 assert should_raise_error('101')
305 assert should_raise_error('201%')
306 assert should_raise_error('10Kbps')
307 assert should_raise_error('0kbps')
308 assert should_raise_error('0pps')
309 assert should_raise_error('-1bps')
312 def test_rate_conversion():
313 assert traffic_utils.load_to_bps(50, 10000000000) == pytest.approx(5000000000.0)
314 assert traffic_utils.load_to_bps(37, 10000000000) == pytest.approx(3700000000.0)
315 assert traffic_utils.load_to_bps(100, 10000000000) == pytest.approx(10000000000.0)
317 assert traffic_utils.bps_to_load(5000000000.0, 10000000000) == pytest.approx(50.0)
318 assert traffic_utils.bps_to_load(3700000000.0, 10000000000) == pytest.approx(37.0)
319 assert traffic_utils.bps_to_load(10000000000.0, 10000000000) == pytest.approx(100.0)
321 assert traffic_utils.bps_to_pps(500000, 64) == pytest.approx(744.047619048)
322 assert traffic_utils.bps_to_pps(388888, 1518) == pytest.approx(31.6066319896)
323 assert traffic_utils.bps_to_pps(9298322222, 340.3) == pytest.approx(3225895.85831)
325 assert traffic_utils.pps_to_bps(744.047619048, 64) == pytest.approx(500000)
326 assert traffic_utils.pps_to_bps(31.6066319896, 1518) == pytest.approx(388888)
327 assert traffic_utils.pps_to_bps(3225895.85831, 340.3) == pytest.approx(9298322222)
329 # pps at 10Gbps line rate for 64 byte frames
330 LR_64B_PPS = 14880952
331 LR_1518B_PPS = 812743
333 def assert_equivalence(reference, value, allowance_pct=1):
334 '''Asserts if a value is equivalent to a reference value with given margin
336 :param float reference: reference value to compare to
337 :param float value: value to compare to reference
338 :param float allowance_pct: max allowed percentage of margin
339 0 : requires exact match
340 1 : must be equal within 1% of the reference value
347 assert abs(value - reference) * 100 / reference <= allowance_pct
349 def test_load_from_rate():
350 assert traffic_utils.get_load_from_rate('100%') == 100
351 assert_equivalence(100, traffic_utils.get_load_from_rate(str(LR_64B_PPS) + 'pps'))
352 assert_equivalence(50, traffic_utils.get_load_from_rate(str(LR_64B_PPS / 2) + 'pps'))
353 assert_equivalence(100, traffic_utils.get_load_from_rate('10Gbps'))
354 assert_equivalence(50, traffic_utils.get_load_from_rate('5000Mbps'))
355 assert_equivalence(1, traffic_utils.get_load_from_rate('100Mbps'))
356 assert_equivalence(100, traffic_utils.get_load_from_rate(str(LR_1518B_PPS) + 'pps',
357 avg_frame_size=1518))
358 assert_equivalence(100, traffic_utils.get_load_from_rate(str(LR_1518B_PPS * 2) + 'pps',
364 def traffic_client(monkeypatch):
366 def mock_init(self, *args, **kwargs):
368 'bidirectional': False,
369 'l2frame_size': '64',
371 'rates': [{'rate_percent': '10'}, {'rate_pps': '1'}]
374 def mock_modify_load(self, load):
375 self.run_config['rates'][0] = {'rate_percent': str(load)}
376 self.current_load = load
378 monkeypatch.setattr(TrafficClient, '__init__', mock_init)
379 monkeypatch.setattr(TrafficClient, 'modify_load', mock_modify_load)
381 return TrafficClient()
385 # pylint: enable=pointless-string-statement
387 # =========================================================================
389 # =========================================================================
391 def setup_module(module):
392 nfvbench.log.setup(mute_stdout=True)
394 def test_no_credentials():
395 cred = Credentials('/completely/wrong/path/openrc', None, False)
397 # shouldn't get valid data unless user set environment variables
403 # Because trex_stl_lib may not be installed when running unit test
404 # nfvbench.traffic_client will try to import STLError:
405 # from trex_stl_lib.api import STLError
406 # will raise ImportError: No module named trex_stl_lib.api
408 import trex_stl_lib.api
410 assert trex_stl_lib.api
412 # Make up a trex_stl_lib.api.STLError class
413 class STLError(Exception):
417 from types import ModuleType
419 stl_lib_mod = ModuleType('trex_stl_lib')
420 sys.modules['trex_stl_lib'] = stl_lib_mod
421 api_mod = ModuleType('trex_stl_lib.api')
422 stl_lib_mod.api = api_mod
423 sys.modules['trex_stl_lib.api'] = api_mod
424 api_mod.STLError = STLError
426 # pylint: disable=wrong-import-position,ungrouped-imports
427 from nfvbench.traffic_client import Device
428 from nfvbench.traffic_client import IpBlock
429 from nfvbench.traffic_client import TrafficClient
430 from nfvbench.traffic_client import TrafficGeneratorFactory
433 ipb = IpBlock('10.0.0.0', '0.0.0.1', 256)
434 assert ipb.get_ip() == '10.0.0.0'
435 assert ipb.get_ip(255) == '10.0.0.255'
436 with pytest.raises(IndexError):
438 # verify with step larger than 1
439 ipb = IpBlock('10.0.0.0', '0.0.0.2', 256)
440 assert ipb.get_ip() == '10.0.0.0'
441 assert ipb.get_ip(1) == '10.0.0.2'
442 assert ipb.get_ip(128) == '10.0.1.0'
443 assert ipb.get_ip(255) == '10.0.1.254'
444 with pytest.raises(IndexError):
448 def check_config(configs, cc, fc, src_ip, dst_ip, step_ip):
449 '''Verify that the range configs for each chain have adjacent IP ranges
450 of the right size and without holes between chains
452 step = Device.ip_to_int(step_ip)
454 sip = Device.ip_to_int(src_ip)
455 dip = Device.ip_to_int(dst_ip)
456 for index in range(cc):
457 config = configs[index]
458 assert config['ip_src_count'] == config['ip_dst_count']
459 assert Device.ip_to_int(config['ip_src_addr']) == sip
460 assert Device.ip_to_int(config['ip_dst_addr']) == dip
461 count = config['ip_src_count']
468 def create_device(fc, cc, ip, gip, tggip, step_ip):
469 return Device(0, 0, flow_count=fc, chain_count=cc, ip=ip, gateway_ip=gip, tg_gateway_ip=tggip,
470 ip_addrs_step=step_ip,
471 tg_gateway_ip_addrs_step=step_ip,
472 gateway_ip_addrs_step=step_ip)
475 def check_device_flow_config(step_ip):
482 dev0 = create_device(fc, cc, ip0, gip, tggip, step_ip)
483 dev1 = create_device(fc, cc, ip1, gip, tggip, step_ip)
484 dev0.set_destination(dev1)
485 configs = dev0.get_stream_configs(ChainType.EXT)
486 check_config(configs, cc, fc, ip0, ip1, step_ip)
489 def test_device_flow_config():
490 check_device_flow_config('0.0.0.1')
491 check_device_flow_config('0.0.0.2')
494 def test_device_ip_range():
495 def ip_range_overlaps(ip0, ip1, flows):
498 dev0 = create_device(flows, 10, ip0, gip, tggip, '0.0.0.1')
499 dev1 = create_device(flows, 10, ip1, gip, tggip, '0.0.0.1')
500 dev0.set_destination(dev1)
501 return dev0.ip_range_overlaps()
503 assert not ip_range_overlaps('10.0.0.0', '20.0.0.0', 10000)
504 assert ip_range_overlaps('10.0.0.0', '10.0.1.0', 10000)
505 assert ip_range_overlaps('10.0.0.0', '10.0.1.0', 257)
506 assert ip_range_overlaps('10.0.1.0', '10.0.0.0', 257)
510 refcfg = {1: 100, 2: {21: 100, 22: 200}, 3: None}
511 res1 = {1: 10, 2: {21: 100, 22: 200}, 3: None}
512 res2 = {1: 100, 2: {21: 1000, 22: 200}, 3: None}
513 res3 = {1: 100, 2: {21: 100, 22: 200}, 3: "abc"}
514 assert config_loads("{}", refcfg) == refcfg
515 assert config_loads("{1: 10}", refcfg) == res1
516 assert config_loads("{2: {21: 1000}}", refcfg) == res2
517 assert config_loads('{3: "abc"}', refcfg) == res3
520 # pairs of input string and expected subset (None if identical)
523 ["{2: {21: 100, 30: 50}}", "{2: {30: 50}}"],
524 ["{2: {0: 1, 1: 2}, 5: 5}", None],
525 ["{1: 'abc', 2: {21: 0}}", "{1: 'abc'}"],
528 for fail_pair in fail_pairs:
529 with pytest.raises(Exception) as e_info:
530 config_loads(fail_pair[0], refcfg)
531 expected = fail_pair[1]
533 expected = fail_pair[0]
534 assert expected in e_info.value.message
537 flavor = {'flavor': {'vcpus': 2, 'ram': 8192, 'disk': 0,
538 'extra_specs': {'hw:cpu_policy': 'dedicated'}}}
539 new_flavor = {'flavor': {'vcpus': 2, 'ram': 8192, 'disk': 0,
540 'extra_specs': {'hw:cpu_policy': 'dedicated', 'hw:numa_nodes': 2}}}
541 assert config_loads("{'flavor': {'extra_specs': {'hw:numa_nodes': 2}}}", flavor,
542 whitelist_keys=['alpha', 'extra_specs']) == new_flavor
546 logger = logging.getLogger('fluent-logger')
548 class FluentdConfig(dict):
549 def __getattr__(self, attr):
550 return self.get(attr)
554 'logging_tag': 'nfvbench',
555 'result_tag': 'resultnfvbench',
560 'logging_tag': 'nfvbench',
561 'result_tag': 'resultnfvbench',
567 'result_tag': 'resultnfvbench',
572 'logging_tag': 'nfvbench',
579 handler = FluentLogHandler(fluentd_configs=fluentd_configs)
580 logger.addHandler(handler)
581 logger.setLevel(logging.INFO)
583 logger.warning('test %d', 100)
586 raise Exception("test")
588 logger.exception("got exception")
590 def assert_ndr_pdr(stats, ndr, ndr_dr, pdr, pdr_dr):
591 assert stats['ndr']['rate_percent'] == ndr
592 assert stats['ndr']['stats']['overall']['drop_percentage'] == ndr_dr
593 assert_equivalence(pdr, stats['pdr']['rate_percent'])
594 assert_equivalence(pdr_dr, stats['pdr']['stats']['overall']['drop_percentage'])
596 def get_dummy_tg_config(chain_type, rate):
598 'traffic_generator': {'host_name': 'nfvbench_tg',
599 'default_profile': 'dummy',
600 'generator_profile': [{'name': 'dummy',
603 'intf_speed': '10Gbps',
604 'interfaces': [{'port': 0, 'pci': '0.0'},
605 {'port': 1, 'pci': '0.0'}]}],
606 'ip_addrs_step': '0.0.0.1',
607 'ip_addrs': ['10.0.0.0/8', '20.0.0.0/8'],
608 'tg_gateway_ip_addrs': ['1.1.0.100', '2.2.0.100'],
609 'tg_gateway_ip_addrs_step': '0.0.0.1',
610 'gateway_ip_addrs': ['1.1.0.2', '2.2.0.2'],
611 'gateway_ip_addrs_step': '0.0.0.1',
612 'udp_src_port': None,
613 'udp_dst_port': None},
614 'service_chain': chain_type,
615 'service_chain_count': 1,
617 'vlan_tagging': True,
622 'check_traffic_time_sec': 200,
623 'generic_poll_sec': 2,
624 'measurement': {'NDR': 0.001, 'PDR': 0.1, 'load_epsilon': 0.1},
627 def get_traffic_client():
628 config = get_dummy_tg_config('PVP', 'ndr_pdr')
629 config['ndr_run'] = True
630 config['pdr_run'] = True
631 config['generator_profile'] = 'dummy'
632 config['single_run'] = False
633 generator_factory = TrafficGeneratorFactory(config)
634 config.generator_config = generator_factory.get_generator_config(config.generator_profile)
635 traffic_client = TrafficClient(config, skip_sleep=True)
636 traffic_client.start_traffic_generator()
637 traffic_client.set_traffic('64', True)
638 return traffic_client
640 def test_ndr_at_lr():
641 traffic_client = get_traffic_client()
642 tg = traffic_client.gen
644 # this is a perfect sut with no loss at LR
645 tg.set_response_curve(lr_dr=0, ndr=100, max_actual_tx=100, max_11_tx=100)
646 # tx packets should be line rate for 64B and no drops...
647 assert tg.get_tx_pps_dropped_pps(100) == (LR_64B_PPS, 0)
648 # NDR and PDR should be at 100%
649 traffic_client.ensure_end_to_end()
650 results = traffic_client.get_ndr_and_pdr()
652 assert_ndr_pdr(results, 200.0, 0.0, 200.0, 0.0)
654 def test_ndr_at_50():
655 traffic_client = get_traffic_client()
656 tg = traffic_client.gen
657 # this is a sut with an NDR of 50% and linear drop rate after NDR up to 20% drops at LR
658 # (meaning that if you send 100% TX, you will only receive 80% RX)
659 # the tg requested TX/actual TX ratio is 1up to 50%, after 50%
660 # is linear up 80% actuak TX when requesting 100%
661 tg.set_response_curve(lr_dr=20, ndr=50, max_actual_tx=80, max_11_tx=50)
662 # tx packets should be half line rate for 64B and no drops...
663 assert tg.get_tx_pps_dropped_pps(50) == (LR_64B_PPS / 2, 0)
664 # at 100% TX requested, actual TX is 80% where the drop rate is 3/5 of 20% of the actual TX
665 assert tg.get_tx_pps_dropped_pps(100) == (int(LR_64B_PPS * 0.8),
666 int(LR_64B_PPS * 0.8 * 0.6 * 0.2))
667 results = traffic_client.get_ndr_and_pdr()
668 assert_ndr_pdr(results, 100.0, 0.0, 100.781, 0.09374)
670 def test_ndr_pdr_low_cpu():
671 traffic_client = get_traffic_client()
672 tg = traffic_client.gen
673 # This test is for the case where the TG is underpowered and cannot send fast enough for the NDR
674 # true NDR=40%, actual TX at 50% = 30%, actual measured DR is 0%
675 # The ndr/pdr should bail out with a warning and a best effort measured NDR of 30%
676 tg.set_response_curve(lr_dr=50, ndr=40, max_actual_tx=60, max_11_tx=0)
677 # tx packets should be 30% at requested half line rate for 64B and no drops...
678 assert tg.get_tx_pps_dropped_pps(50) == (int(LR_64B_PPS * 0.3), 0)
679 results = traffic_client.get_ndr_and_pdr()
682 # pp = pprint.PrettyPrinter(indent=4)
685 import nfvbench.nfvbench
687 def test_no_openstack():
688 config = get_dummy_tg_config('EXT', '1000pps')
689 config.openrc_file = None
691 sys.argv = [old_argv[0], '-c', json.dumps(config)]
692 nfvbench.nfvbench.main()