2 # Copyright 2016 Cisco Systems, Inc. All rights reserved.
4 # Licensed under the Apache License, Version 2.0 (the "License"); you may
5 # not use this file except in compliance with the License. You may obtain
6 # a copy of the License at
8 # http://www.apache.org/licenses/LICENSE-2.0
10 # Unless required by applicable law or agreed to in writing, software
11 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
12 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
13 # License for the specific language governing permissions and limitations
23 from attrdict import AttrDict
24 from nfvbench.config import config_loads
25 from nfvbench.credentials import Credentials
26 from nfvbench.fluentd import FluentLogHandler
28 from nfvbench.network import Interface
29 from nfvbench.network import Network
30 from nfvbench.specs import ChainType
31 from nfvbench.specs import Encaps
32 import nfvbench.traffic_gen.traffic_utils as traffic_utils
34 __location__ = os.path.realpath(os.path.join(os.getcwd(),
35 os.path.dirname(__file__)))
39 def openstack_vxlan_spec():
42 'openstack': AttrDict({
44 'encaps': Encaps.VxLAN}),
45 'run_spec': AttrDict({
52 # =========================================================================
54 # =========================================================================
56 def test_chain_interface():
57 iface = Interface('testname', 'vpp', tx_packets=1234, rx_packets=4321)
58 assert iface.name == 'testname'
59 assert iface.device == 'vpp'
60 assert iface.get_packet_count('tx') == 1234
61 assert iface.get_packet_count('rx') == 4321
62 assert iface.get_packet_count('wrong_key') == 0
65 # pylint: disable=redefined-outer-name
66 @pytest.fixture(scope='session')
68 return Interface('iface1', 'trex', tx_packets=10000, rx_packets=1234)
71 @pytest.fixture(scope='session')
73 return Interface('iface2', 'n9k', tx_packets=1234, rx_packets=9901)
76 @pytest.fixture(scope='session')
78 return Interface('iface3', 'n9k', tx_packets=9900, rx_packets=1234)
81 @pytest.fixture(scope='session')
83 return Interface('iface4', 'vpp', tx_packets=1234, rx_packets=9801)
86 @pytest.fixture(scope='session')
87 def net1(iface1, iface2, iface3, iface4):
88 return Network([iface1, iface2, iface3, iface4], reverse=False)
91 @pytest.fixture(scope='session')
92 def net2(iface1, iface2, iface3):
93 return Network([iface1, iface2, iface3], reverse=True)
96 def test_chain_network(net1, net2, iface1, iface2, iface3, iface4):
97 assert [iface1, iface2, iface3, iface4] == net1.get_interfaces()
98 assert [iface3, iface2, iface1] == net2.get_interfaces()
99 net2.add_interface(iface4)
100 assert [iface4, iface3, iface2, iface1] == net2.get_interfaces()
103 # pylint: enable=redefined-outer-name
105 # pylint: disable=pointless-string-statement
107 def test_chain_analysis(net1, monkeypatch, openstack_vxlan_spec):
108 def mock_empty(self, *args, **kwargs):
111 monkeypatch.setattr(ServiceChain, '_setup', mock_empty)
113 f = ServiceChain(AttrDict({'service_chain': 'DUMMY'}), [], {'tor': {}}, openstack_vxlan_spec,
114 lambda x, y, z: None)
115 result = f.get_analysis([net1])
116 assert result[1]['packet_drop_count'] == 99
117 assert result[1]['packet_drop_percentage'] == 0.99
118 assert result[2]['packet_drop_count'] == 1
119 assert result[2]['packet_drop_percentage'] == 0.01
120 assert result[3]['packet_drop_count'] == 99
121 assert result[3]['packet_drop_percentage'] == 0.99
124 result = f.get_analysis([net1])
125 assert result[1]['packet_drop_count'] == 0
126 assert result[1]['packet_drop_percentage'] == 0.0
127 assert result[2]['packet_drop_count'] == 0
128 assert result[2]['packet_drop_percentage'] == 0.0
129 assert result[3]['packet_drop_count'] == 0
130 assert result[3]['packet_drop_percentage'] == 0.0
134 def pvp_chain(monkeypatch, openstack_vxlan_spec):
135 tor_vni1 = Interface('vni-4097', 'n9k', 50, 77)
136 vsw_vni1 = Interface('vxlan_tunnel0', 'vpp', 77, 48)
137 vsw_vif1 = Interface('VirtualEthernet0/0/2', 'vpp', 48, 77)
138 vsw_vif2 = Interface('VirtualEthernet0/0/3', 'vpp', 77, 47)
139 vsw_vni2 = Interface('vxlan_tunnel1', 'vpp', 43, 77)
140 tor_vni2 = Interface('vni-4098', 'n9k', 77, 40)
142 def mock_init(self, *args, **kwargs):
143 self.vni_ports = [4097, 4098]
144 self.specs = openstack_vxlan_spec
147 'set_interface_counters': lambda: None,
150 self.worker = AttrDict({
154 def mock_empty(self, *args, **kwargs):
157 def mock_get_network(self, traffic_port, vni_id, reverse=False):
159 return Network([tor_vni1, vsw_vni1, vsw_vif1], reverse)
161 return Network([tor_vni2, vsw_vni2, vsw_vif2], reverse)
163 def mock_get_data(self):
166 monkeypatch.setattr(PVPChain, '_get_network', mock_get_network)
167 monkeypatch.setattr(PVPChain, '_get_data', mock_get_data)
168 monkeypatch.setattr(PVPChain, '_setup', mock_empty)
169 monkeypatch.setattr(VxLANWorker, '_clear_interfaces', mock_empty)
170 monkeypatch.setattr(PVPChain, '_generate_traffic', mock_empty)
171 monkeypatch.setattr(PVPChain, '__init__', mock_init)
172 return PVPChain(None, None, {'vm': None, 'vpp': None, 'tor': None, 'traffic': None}, None)
175 def test_pvp_chain_run(pvp_chain):
176 result = pvp_chain.run()
181 'direction-forward': [
183 ('interface', 'vni-4097'),
188 ('interface', 'vxlan_tunnel0'),
190 ('packet_count', 48),
191 ('packet_drop_count', 2),
192 ('packet_drop_percentage', 4.0)
195 ('interface', 'VirtualEthernet0/0/2'),
197 ('packet_count', 48),
198 ('packet_drop_count', 0),
199 ('packet_drop_percentage', 0.0)
202 ('interface', 'VirtualEthernet0/0/3'),
204 ('packet_count', 47),
205 ('packet_drop_count', 1),
206 ('packet_drop_percentage', 2.0)
209 ('interface', 'vxlan_tunnel1'),
211 ('packet_count', 43),
212 ('packet_drop_count', 4),
213 ('packet_drop_percentage', 8.0)
216 ('interface', 'vni-4098'),
218 ('packet_count', 40),
219 ('packet_drop_count', 3),
220 ('packet_drop_percentage', 6.0)
223 'direction-reverse': [
225 ('interface', 'vni-4098'),
230 ('interface', 'vxlan_tunnel1'),
232 ('packet_count', 77),
233 ('packet_drop_count', 0),
234 ('packet_drop_percentage', 0.0)
237 ('interface', 'VirtualEthernet0/0/3'),
239 ('packet_count', 77),
240 ('packet_drop_count', 0),
241 ('packet_drop_percentage', 0.0)
244 ('interface', 'VirtualEthernet0/0/2'),
246 ('packet_count', 77),
247 ('packet_drop_count', 0),
248 ('packet_drop_percentage', 0.0)
251 ('interface', 'vxlan_tunnel0'),
253 ('packet_count', 77),
254 ('packet_drop_count', 0),
255 ('packet_drop_percentage', 0.0)
258 ('interface', 'vni-4097'),
260 ('packet_count', 77),
261 ('packet_drop_count', 0),
262 ('packet_drop_percentage', 0.0)
267 assert result == expected_result
270 # =========================================================================
271 # Traffic client tests
272 # =========================================================================
274 def test_parse_rate_str():
275 parse_rate_str = traffic_utils.parse_rate_str
277 assert parse_rate_str('100%') == {'rate_percent': '100.0'}
278 assert parse_rate_str('37.5%') == {'rate_percent': '37.5'}
279 assert parse_rate_str('100%') == {'rate_percent': '100.0'}
280 assert parse_rate_str('60pps') == {'rate_pps': '60'}
281 assert parse_rate_str('60kpps') == {'rate_pps': '60000'}
282 assert parse_rate_str('6Mpps') == {'rate_pps': '6000000'}
283 assert parse_rate_str('6gpps') == {'rate_pps': '6000000000'}
284 assert parse_rate_str('80bps') == {'rate_bps': '80'}
285 assert parse_rate_str('80bps') == {'rate_bps': '80'}
286 assert parse_rate_str('80kbps') == {'rate_bps': '80000'}
287 assert parse_rate_str('80kBps') == {'rate_bps': '640000'}
288 assert parse_rate_str('80Mbps') == {'rate_bps': '80000000'}
289 assert parse_rate_str('80 MBps') == {'rate_bps': '640000000'}
290 assert parse_rate_str('80Gbps') == {'rate_bps': '80000000000'}
291 except Exception as exc:
292 assert False, exc.message
294 def should_raise_error(str):
304 assert should_raise_error('101')
305 assert should_raise_error('201%')
306 assert should_raise_error('10Kbps')
307 assert should_raise_error('0kbps')
308 assert should_raise_error('0pps')
309 assert should_raise_error('-1bps')
312 def test_rate_conversion():
313 assert traffic_utils.load_to_bps(50, 10000000000) == pytest.approx(5000000000.0)
314 assert traffic_utils.load_to_bps(37, 10000000000) == pytest.approx(3700000000.0)
315 assert traffic_utils.load_to_bps(100, 10000000000) == pytest.approx(10000000000.0)
317 assert traffic_utils.bps_to_load(5000000000.0, 10000000000) == pytest.approx(50.0)
318 assert traffic_utils.bps_to_load(3700000000.0, 10000000000) == pytest.approx(37.0)
319 assert traffic_utils.bps_to_load(10000000000.0, 10000000000) == pytest.approx(100.0)
321 assert traffic_utils.bps_to_pps(500000, 64) == pytest.approx(744.047619048)
322 assert traffic_utils.bps_to_pps(388888, 1518) == pytest.approx(31.6066319896)
323 assert traffic_utils.bps_to_pps(9298322222, 340.3) == pytest.approx(3225895.85831)
325 assert traffic_utils.pps_to_bps(744.047619048, 64) == pytest.approx(500000)
326 assert traffic_utils.pps_to_bps(31.6066319896, 1518) == pytest.approx(388888)
327 assert traffic_utils.pps_to_bps(3225895.85831, 340.3) == pytest.approx(9298322222)
329 # pps at 10Gbps line rate for 64 byte frames
330 LR_64B_PPS = 14880952
331 LR_1518B_PPS = 812743
333 def assert_equivalence(reference, value, allowance_pct=1):
334 '''Asserts if a value is equivalent to a reference value with given margin
336 :param float reference: reference value to compare to
337 :param float value: value to compare to reference
338 :param float allowance_pct: max allowed percentage of margin
339 0 : requires exact match
340 1 : must be equal within 1% of the reference value
347 assert abs(value - reference) * 100 / reference <= allowance_pct
349 def test_load_from_rate():
350 assert traffic_utils.get_load_from_rate('100%') == 100
351 assert_equivalence(100, traffic_utils.get_load_from_rate(str(LR_64B_PPS) + 'pps'))
352 assert_equivalence(50, traffic_utils.get_load_from_rate(str(LR_64B_PPS / 2) + 'pps'))
353 assert_equivalence(100, traffic_utils.get_load_from_rate('10Gbps'))
354 assert_equivalence(50, traffic_utils.get_load_from_rate('5000Mbps'))
355 assert_equivalence(1, traffic_utils.get_load_from_rate('100Mbps'))
356 assert_equivalence(100, traffic_utils.get_load_from_rate(str(LR_1518B_PPS) + 'pps',
357 avg_frame_size=1518))
358 assert_equivalence(100, traffic_utils.get_load_from_rate(str(LR_1518B_PPS * 2) + 'pps',
364 def traffic_client(monkeypatch):
366 def mock_init(self, *args, **kwargs):
368 'bidirectional': False,
369 'l2frame_size': '64',
371 'rates': [{'rate_percent': '10'}, {'rate_pps': '1'}]
374 def mock_modify_load(self, load):
375 self.run_config['rates'][0] = {'rate_percent': str(load)}
376 self.current_load = load
378 monkeypatch.setattr(TrafficClient, '__init__', mock_init)
379 monkeypatch.setattr(TrafficClient, 'modify_load', mock_modify_load)
381 return TrafficClient()
385 # pylint: enable=pointless-string-statement
387 # =========================================================================
389 # =========================================================================
391 def setup_module(module):
392 nfvbench.log.setup(mute_stdout=True)
394 def test_no_credentials():
395 cred = Credentials('/completely/wrong/path/openrc', None, False)
397 # shouldn't get valid data unless user set environment variables
403 # Because trex_stl_lib may not be installed when running unit test
404 # nfvbench.traffic_client will try to import STLError:
405 # from trex_stl_lib.api import STLError
406 # will raise ImportError: No module named trex_stl_lib.api
408 import trex_stl_lib.api
410 assert trex_stl_lib.api
412 # Make up a trex_stl_lib.api.STLError class
413 class STLError(Exception):
417 from types import ModuleType
419 stl_lib_mod = ModuleType('trex_stl_lib')
420 sys.modules['trex_stl_lib'] = stl_lib_mod
421 api_mod = ModuleType('trex_stl_lib.api')
422 stl_lib_mod.api = api_mod
423 sys.modules['trex_stl_lib.api'] = api_mod
424 api_mod.STLError = STLError
426 # pylint: disable=wrong-import-position,ungrouped-imports
427 from nfvbench.traffic_client import Device
428 from nfvbench.traffic_client import IpBlock
429 from nfvbench.traffic_client import TrafficClient
430 from nfvbench.traffic_client import TrafficGeneratorFactory
433 ipb = IpBlock('10.0.0.0', '0.0.0.1', 256)
434 assert ipb.get_ip() == '10.0.0.0'
435 assert ipb.get_ip(255) == '10.0.0.255'
436 with pytest.raises(IndexError):
438 # verify with step larger than 1
439 ipb = IpBlock('10.0.0.0', '0.0.0.2', 256)
440 assert ipb.get_ip() == '10.0.0.0'
441 assert ipb.get_ip(1) == '10.0.0.2'
442 assert ipb.get_ip(128) == '10.0.1.0'
443 assert ipb.get_ip(255) == '10.0.1.254'
444 with pytest.raises(IndexError):
448 def check_config(configs, cc, fc, src_ip, dst_ip, step_ip):
449 '''Verify that the range configs for each chain have adjacent IP ranges
450 of the right size and without holes between chains
452 step = Device.ip_to_int(step_ip)
454 sip = Device.ip_to_int(src_ip)
455 dip = Device.ip_to_int(dst_ip)
456 for index in range(cc):
457 config = configs[index]
458 assert config['ip_src_count'] == config['ip_dst_count']
459 assert Device.ip_to_int(config['ip_src_addr']) == sip
460 assert Device.ip_to_int(config['ip_dst_addr']) == dip
461 count = config['ip_src_count']
468 def create_device(fc, cc, ip, gip, tggip, step_ip, mac):
469 return Device(0, 0, flow_count=fc, chain_count=cc, ip=ip, gateway_ip=gip, tg_gateway_ip=tggip,
470 ip_addrs_step=step_ip,
471 tg_gateway_ip_addrs_step=step_ip,
472 gateway_ip_addrs_step=step_ip,
476 def check_device_flow_config(step_ip):
483 mac = ['00:11:22:33:44:55'] * cc
484 dev0 = create_device(fc, cc, ip0, gip, tggip, step_ip, mac)
485 dev1 = create_device(fc, cc, ip1, gip, tggip, step_ip, mac)
486 dev0.set_destination(dev1)
487 configs = dev0.get_stream_configs(ChainType.EXT)
488 check_config(configs, cc, fc, ip0, ip1, step_ip)
491 def test_device_flow_config():
492 check_device_flow_config('0.0.0.1')
493 check_device_flow_config('0.0.0.2')
496 def test_device_ip_range():
497 def ip_range_overlaps(ip0, ip1, flows):
500 mac = ['00:11:22:33:44:55'] * 10
501 dev0 = create_device(flows, 10, ip0, gip, tggip, '0.0.0.1', mac)
502 dev1 = create_device(flows, 10, ip1, gip, tggip, '0.0.0.1', mac)
503 dev0.set_destination(dev1)
504 return dev0.ip_range_overlaps()
506 assert not ip_range_overlaps('10.0.0.0', '20.0.0.0', 10000)
507 assert ip_range_overlaps('10.0.0.0', '10.0.1.0', 10000)
508 assert ip_range_overlaps('10.0.0.0', '10.0.1.0', 257)
509 assert ip_range_overlaps('10.0.1.0', '10.0.0.0', 257)
513 refcfg = {1: 100, 2: {21: 100, 22: 200}, 3: None}
514 res1 = {1: 10, 2: {21: 100, 22: 200}, 3: None}
515 res2 = {1: 100, 2: {21: 1000, 22: 200}, 3: None}
516 res3 = {1: 100, 2: {21: 100, 22: 200}, 3: "abc"}
517 assert config_loads("{}", refcfg) == refcfg
518 assert config_loads("{1: 10}", refcfg) == res1
519 assert config_loads("{2: {21: 1000}}", refcfg) == res2
520 assert config_loads('{3: "abc"}', refcfg) == res3
523 # pairs of input string and expected subset (None if identical)
526 ["{2: {21: 100, 30: 50}}", "{2: {30: 50}}"],
527 ["{2: {0: 1, 1: 2}, 5: 5}", None],
528 ["{1: 'abc', 2: {21: 0}}", "{1: 'abc'}"],
531 for fail_pair in fail_pairs:
532 with pytest.raises(Exception) as e_info:
533 config_loads(fail_pair[0], refcfg)
534 expected = fail_pair[1]
536 expected = fail_pair[0]
537 assert expected in e_info.value.message
540 flavor = {'flavor': {'vcpus': 2, 'ram': 8192, 'disk': 0,
541 'extra_specs': {'hw:cpu_policy': 'dedicated'}}}
542 new_flavor = {'flavor': {'vcpus': 2, 'ram': 8192, 'disk': 0,
543 'extra_specs': {'hw:cpu_policy': 'dedicated', 'hw:numa_nodes': 2}}}
544 assert config_loads("{'flavor': {'extra_specs': {'hw:numa_nodes': 2}}}", flavor,
545 whitelist_keys=['alpha', 'extra_specs']) == new_flavor
549 logger = logging.getLogger('fluent-logger')
551 class FluentdConfig(dict):
552 def __getattr__(self, attr):
553 return self.get(attr)
557 'logging_tag': 'nfvbench',
558 'result_tag': 'resultnfvbench',
563 'logging_tag': 'nfvbench',
564 'result_tag': 'resultnfvbench',
570 'result_tag': 'resultnfvbench',
575 'logging_tag': 'nfvbench',
582 handler = FluentLogHandler(fluentd_configs=fluentd_configs)
583 logger.addHandler(handler)
584 logger.setLevel(logging.INFO)
586 logger.warning('test %d', 100)
589 raise Exception("test")
591 logger.exception("got exception")
593 def assert_ndr_pdr(stats, ndr, ndr_dr, pdr, pdr_dr):
594 assert stats['ndr']['rate_percent'] == ndr
595 assert stats['ndr']['stats']['overall']['drop_percentage'] == ndr_dr
596 assert_equivalence(pdr, stats['pdr']['rate_percent'])
597 assert_equivalence(pdr_dr, stats['pdr']['stats']['overall']['drop_percentage'])
599 def get_dummy_tg_config(chain_type, rate):
601 'traffic_generator': {'host_name': 'nfvbench_tg',
602 'default_profile': 'dummy',
603 'generator_profile': [{'name': 'dummy',
606 'intf_speed': '10Gbps',
607 'interfaces': [{'port': 0, 'pci': '0.0'},
608 {'port': 1, 'pci': '0.0'}]}],
609 'ip_addrs_step': '0.0.0.1',
610 'ip_addrs': ['10.0.0.0/8', '20.0.0.0/8'],
611 'tg_gateway_ip_addrs': ['1.1.0.100', '2.2.0.100'],
612 'tg_gateway_ip_addrs_step': '0.0.0.1',
613 'gateway_ip_addrs': ['1.1.0.2', '2.2.0.2'],
614 'gateway_ip_addrs_step': '0.0.0.1',
615 'mac_addrs_left': None,
616 'mac_addrs_right': None,
617 'udp_src_port': None,
618 'udp_dst_port': None},
619 'service_chain': chain_type,
620 'service_chain_count': 1,
622 'vlan_tagging': True,
628 'check_traffic_time_sec': 200,
629 'generic_poll_sec': 2,
630 'measurement': {'NDR': 0.001, 'PDR': 0.1, 'load_epsilon': 0.1},
634 def get_traffic_client():
635 config = get_dummy_tg_config('PVP', 'ndr_pdr')
636 config['ndr_run'] = True
637 config['pdr_run'] = True
638 config['generator_profile'] = 'dummy'
639 config['single_run'] = False
640 generator_factory = TrafficGeneratorFactory(config)
641 config.generator_config = generator_factory.get_generator_config(config.generator_profile)
642 traffic_client = TrafficClient(config, skip_sleep=True)
643 traffic_client.start_traffic_generator()
644 traffic_client.set_traffic('64', True)
645 return traffic_client
647 def test_ndr_at_lr():
648 traffic_client = get_traffic_client()
649 tg = traffic_client.gen
651 # this is a perfect sut with no loss at LR
652 tg.set_response_curve(lr_dr=0, ndr=100, max_actual_tx=100, max_11_tx=100)
653 # tx packets should be line rate for 64B and no drops...
654 assert tg.get_tx_pps_dropped_pps(100) == (LR_64B_PPS, 0)
655 # NDR and PDR should be at 100%
656 traffic_client.ensure_end_to_end()
657 results = traffic_client.get_ndr_and_pdr()
659 assert_ndr_pdr(results, 200.0, 0.0, 200.0, 0.0)
661 def test_ndr_at_50():
662 traffic_client = get_traffic_client()
663 tg = traffic_client.gen
664 # this is a sut with an NDR of 50% and linear drop rate after NDR up to 20% drops at LR
665 # (meaning that if you send 100% TX, you will only receive 80% RX)
666 # the tg requested TX/actual TX ratio is 1up to 50%, after 50%
667 # is linear up 80% actuak TX when requesting 100%
668 tg.set_response_curve(lr_dr=20, ndr=50, max_actual_tx=80, max_11_tx=50)
669 # tx packets should be half line rate for 64B and no drops...
670 assert tg.get_tx_pps_dropped_pps(50) == (LR_64B_PPS / 2, 0)
671 # at 100% TX requested, actual TX is 80% where the drop rate is 3/5 of 20% of the actual TX
672 assert tg.get_tx_pps_dropped_pps(100) == (int(LR_64B_PPS * 0.8),
673 int(LR_64B_PPS * 0.8 * 0.6 * 0.2))
674 results = traffic_client.get_ndr_and_pdr()
675 assert_ndr_pdr(results, 100.0, 0.0, 100.781, 0.09374)
677 def test_ndr_pdr_low_cpu():
678 traffic_client = get_traffic_client()
679 tg = traffic_client.gen
680 # This test is for the case where the TG is underpowered and cannot send fast enough for the NDR
681 # true NDR=40%, actual TX at 50% = 30%, actual measured DR is 0%
682 # The ndr/pdr should bail out with a warning and a best effort measured NDR of 30%
683 tg.set_response_curve(lr_dr=50, ndr=40, max_actual_tx=60, max_11_tx=0)
684 # tx packets should be 30% at requested half line rate for 64B and no drops...
685 assert tg.get_tx_pps_dropped_pps(50) == (int(LR_64B_PPS * 0.3), 0)
686 results = traffic_client.get_ndr_and_pdr()
689 # pp = pprint.PrettyPrinter(indent=4)
692 import nfvbench.nfvbench
694 def test_no_openstack():
695 config = get_dummy_tg_config('EXT', '1000pps')
696 config.openrc_file = None
698 sys.argv = [old_argv[0], '-c', json.dumps(config)]
699 nfvbench.nfvbench.main()