2 # Copyright 2016 Cisco Systems, Inc. All rights reserved.
4 # Licensed under the Apache License, Version 2.0 (the "License"); you may
5 # not use this file except in compliance with the License. You may obtain
6 # a copy of the License at
8 # http://www.apache.org/licenses/LICENSE-2.0
10 # Unless required by applicable law or agreed to in writing, software
11 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
12 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
13 # License for the specific language governing permissions and limitations
23 from attrdict import AttrDict
24 from nfvbench.config import config_loads
25 from nfvbench.credentials import Credentials
26 from nfvbench.fluentd import FluentLogHandler
28 from nfvbench.network import Interface
29 from nfvbench.network import Network
30 from nfvbench.specs import ChainType
31 from nfvbench.specs import Encaps
32 import nfvbench.traffic_gen.traffic_utils as traffic_utils
34 __location__ = os.path.realpath(os.path.join(os.getcwd(),
35 os.path.dirname(__file__)))
39 def openstack_vxlan_spec():
42 'openstack': AttrDict({
44 'encaps': Encaps.VxLAN}),
45 'run_spec': AttrDict({
52 # =========================================================================
54 # =========================================================================
56 def test_chain_interface():
57 iface = Interface('testname', 'vpp', tx_packets=1234, rx_packets=4321)
58 assert iface.name == 'testname'
59 assert iface.device == 'vpp'
60 assert iface.get_packet_count('tx') == 1234
61 assert iface.get_packet_count('rx') == 4321
62 assert iface.get_packet_count('wrong_key') == 0
65 # pylint: disable=redefined-outer-name
66 @pytest.fixture(scope='session')
68 return Interface('iface1', 'trex', tx_packets=10000, rx_packets=1234)
71 @pytest.fixture(scope='session')
73 return Interface('iface2', 'n9k', tx_packets=1234, rx_packets=9901)
76 @pytest.fixture(scope='session')
78 return Interface('iface3', 'n9k', tx_packets=9900, rx_packets=1234)
81 @pytest.fixture(scope='session')
83 return Interface('iface4', 'vpp', tx_packets=1234, rx_packets=9801)
86 @pytest.fixture(scope='session')
87 def net1(iface1, iface2, iface3, iface4):
88 return Network([iface1, iface2, iface3, iface4], reverse=False)
91 @pytest.fixture(scope='session')
92 def net2(iface1, iface2, iface3):
93 return Network([iface1, iface2, iface3], reverse=True)
96 def test_chain_network(net1, net2, iface1, iface2, iface3, iface4):
97 assert [iface1, iface2, iface3, iface4] == net1.get_interfaces()
98 assert [iface3, iface2, iface1] == net2.get_interfaces()
99 net2.add_interface(iface4)
100 assert [iface4, iface3, iface2, iface1] == net2.get_interfaces()
103 # pylint: enable=redefined-outer-name
105 # pylint: disable=pointless-string-statement
107 def test_chain_analysis(net1, monkeypatch, openstack_vxlan_spec):
108 def mock_empty(self, *args, **kwargs):
111 monkeypatch.setattr(ServiceChain, '_setup', mock_empty)
113 f = ServiceChain(AttrDict({'service_chain': 'DUMMY'}), [], {'tor': {}}, openstack_vxlan_spec,
114 lambda x, y, z: None)
115 result = f.get_analysis([net1])
116 assert result[1]['packet_drop_count'] == 99
117 assert result[1]['packet_drop_percentage'] == 0.99
118 assert result[2]['packet_drop_count'] == 1
119 assert result[2]['packet_drop_percentage'] == 0.01
120 assert result[3]['packet_drop_count'] == 99
121 assert result[3]['packet_drop_percentage'] == 0.99
124 result = f.get_analysis([net1])
125 assert result[1]['packet_drop_count'] == 0
126 assert result[1]['packet_drop_percentage'] == 0.0
127 assert result[2]['packet_drop_count'] == 0
128 assert result[2]['packet_drop_percentage'] == 0.0
129 assert result[3]['packet_drop_count'] == 0
130 assert result[3]['packet_drop_percentage'] == 0.0
134 def pvp_chain(monkeypatch, openstack_vxlan_spec):
135 tor_vni1 = Interface('vni-4097', 'n9k', 50, 77)
136 vsw_vni1 = Interface('vxlan_tunnel0', 'vpp', 77, 48)
137 vsw_vif1 = Interface('VirtualEthernet0/0/2', 'vpp', 48, 77)
138 vsw_vif2 = Interface('VirtualEthernet0/0/3', 'vpp', 77, 47)
139 vsw_vni2 = Interface('vxlan_tunnel1', 'vpp', 43, 77)
140 tor_vni2 = Interface('vni-4098', 'n9k', 77, 40)
142 def mock_init(self, *args, **kwargs):
143 self.vni_ports = [4097, 4098]
144 self.specs = openstack_vxlan_spec
147 'set_interface_counters': lambda: None,
150 self.worker = AttrDict({
154 def mock_empty(self, *args, **kwargs):
157 def mock_get_network(self, traffic_port, vni_id, reverse=False):
159 return Network([tor_vni1, vsw_vni1, vsw_vif1], reverse)
161 return Network([tor_vni2, vsw_vni2, vsw_vif2], reverse)
163 def mock_get_data(self):
166 monkeypatch.setattr(PVPChain, '_get_network', mock_get_network)
167 monkeypatch.setattr(PVPChain, '_get_data', mock_get_data)
168 monkeypatch.setattr(PVPChain, '_setup', mock_empty)
169 monkeypatch.setattr(VxLANWorker, '_clear_interfaces', mock_empty)
170 monkeypatch.setattr(PVPChain, '_generate_traffic', mock_empty)
171 monkeypatch.setattr(PVPChain, '__init__', mock_init)
172 return PVPChain(None, None, {'vm': None, 'vpp': None, 'tor': None, 'traffic': None}, None)
175 def test_pvp_chain_run(pvp_chain):
176 result = pvp_chain.run()
181 'direction-forward': [
183 ('interface', 'vni-4097'),
188 ('interface', 'vxlan_tunnel0'),
190 ('packet_count', 48),
191 ('packet_drop_count', 2),
192 ('packet_drop_percentage', 4.0)
195 ('interface', 'VirtualEthernet0/0/2'),
197 ('packet_count', 48),
198 ('packet_drop_count', 0),
199 ('packet_drop_percentage', 0.0)
202 ('interface', 'VirtualEthernet0/0/3'),
204 ('packet_count', 47),
205 ('packet_drop_count', 1),
206 ('packet_drop_percentage', 2.0)
209 ('interface', 'vxlan_tunnel1'),
211 ('packet_count', 43),
212 ('packet_drop_count', 4),
213 ('packet_drop_percentage', 8.0)
216 ('interface', 'vni-4098'),
218 ('packet_count', 40),
219 ('packet_drop_count', 3),
220 ('packet_drop_percentage', 6.0)
223 'direction-reverse': [
225 ('interface', 'vni-4098'),
230 ('interface', 'vxlan_tunnel1'),
232 ('packet_count', 77),
233 ('packet_drop_count', 0),
234 ('packet_drop_percentage', 0.0)
237 ('interface', 'VirtualEthernet0/0/3'),
239 ('packet_count', 77),
240 ('packet_drop_count', 0),
241 ('packet_drop_percentage', 0.0)
244 ('interface', 'VirtualEthernet0/0/2'),
246 ('packet_count', 77),
247 ('packet_drop_count', 0),
248 ('packet_drop_percentage', 0.0)
251 ('interface', 'vxlan_tunnel0'),
253 ('packet_count', 77),
254 ('packet_drop_count', 0),
255 ('packet_drop_percentage', 0.0)
258 ('interface', 'vni-4097'),
260 ('packet_count', 77),
261 ('packet_drop_count', 0),
262 ('packet_drop_percentage', 0.0)
267 assert result == expected_result
270 # =========================================================================
271 # Traffic client tests
272 # =========================================================================
274 def test_parse_rate_str():
275 parse_rate_str = traffic_utils.parse_rate_str
277 assert parse_rate_str('100%') == {'rate_percent': '100.0'}
278 assert parse_rate_str('37.5%') == {'rate_percent': '37.5'}
279 assert parse_rate_str('100%') == {'rate_percent': '100.0'}
280 assert parse_rate_str('60pps') == {'rate_pps': '60'}
281 assert parse_rate_str('60kpps') == {'rate_pps': '60000'}
282 assert parse_rate_str('6Mpps') == {'rate_pps': '6000000'}
283 assert parse_rate_str('6gpps') == {'rate_pps': '6000000000'}
284 assert parse_rate_str('80bps') == {'rate_bps': '80'}
285 assert parse_rate_str('80bps') == {'rate_bps': '80'}
286 assert parse_rate_str('80kbps') == {'rate_bps': '80000'}
287 assert parse_rate_str('80kBps') == {'rate_bps': '640000'}
288 assert parse_rate_str('80Mbps') == {'rate_bps': '80000000'}
289 assert parse_rate_str('80 MBps') == {'rate_bps': '640000000'}
290 assert parse_rate_str('80Gbps') == {'rate_bps': '80000000000'}
291 except Exception as exc:
292 assert False, exc.message
294 def should_raise_error(str):
302 assert should_raise_error('101')
303 assert should_raise_error('201%')
304 assert should_raise_error('10Kbps')
305 assert should_raise_error('0kbps')
306 assert should_raise_error('0pps')
307 assert should_raise_error('-1bps')
310 def test_rate_conversion():
311 assert traffic_utils.load_to_bps(50, 10000000000) == pytest.approx(5000000000.0)
312 assert traffic_utils.load_to_bps(37, 10000000000) == pytest.approx(3700000000.0)
313 assert traffic_utils.load_to_bps(100, 10000000000) == pytest.approx(10000000000.0)
315 assert traffic_utils.bps_to_load(5000000000.0, 10000000000) == pytest.approx(50.0)
316 assert traffic_utils.bps_to_load(3700000000.0, 10000000000) == pytest.approx(37.0)
317 assert traffic_utils.bps_to_load(10000000000.0, 10000000000) == pytest.approx(100.0)
319 assert traffic_utils.bps_to_pps(500000, 64) == pytest.approx(744.047619048)
320 assert traffic_utils.bps_to_pps(388888, 1518) == pytest.approx(31.6066319896)
321 assert traffic_utils.bps_to_pps(9298322222, 340.3) == pytest.approx(3225895.85831)
323 assert traffic_utils.pps_to_bps(744.047619048, 64) == pytest.approx(500000)
324 assert traffic_utils.pps_to_bps(31.6066319896, 1518) == pytest.approx(388888)
325 assert traffic_utils.pps_to_bps(3225895.85831, 340.3) == pytest.approx(9298322222)
327 # pps at 10Gbps line rate for 64 byte frames
328 LR_64B_PPS = 14880952
329 LR_1518B_PPS = 812743
331 def assert_equivalence(reference, value, allowance_pct=1):
332 '''Asserts if a value is equivalent to a reference value with given margin
334 :param float reference: reference value to compare to
335 :param float value: value to compare to reference
336 :param float allowance_pct: max allowed percentage of margin
337 0 : requires exact match
338 1 : must be equal within 1% of the reference value
345 assert abs(value - reference) * 100 / reference <= allowance_pct
347 def test_load_from_rate():
348 assert traffic_utils.get_load_from_rate('100%') == 100
349 assert_equivalence(100, traffic_utils.get_load_from_rate(str(LR_64B_PPS) + 'pps'))
350 assert_equivalence(50, traffic_utils.get_load_from_rate(str(LR_64B_PPS / 2) + 'pps'))
351 assert_equivalence(100, traffic_utils.get_load_from_rate('10Gbps'))
352 assert_equivalence(50, traffic_utils.get_load_from_rate('5000Mbps'))
353 assert_equivalence(1, traffic_utils.get_load_from_rate('100Mbps'))
354 assert_equivalence(100, traffic_utils.get_load_from_rate(str(LR_1518B_PPS) + 'pps',
355 avg_frame_size=1518))
356 assert_equivalence(100, traffic_utils.get_load_from_rate(str(LR_1518B_PPS * 2) + 'pps',
362 def traffic_client(monkeypatch):
364 def mock_init(self, *args, **kwargs):
366 'bidirectional': False,
367 'l2frame_size': '64',
369 'rates': [{'rate_percent': '10'}, {'rate_pps': '1'}]
372 def mock_modify_load(self, load):
373 self.run_config['rates'][0] = {'rate_percent': str(load)}
374 self.current_load = load
376 monkeypatch.setattr(TrafficClient, '__init__', mock_init)
377 monkeypatch.setattr(TrafficClient, 'modify_load', mock_modify_load)
379 return TrafficClient()
383 # pylint: enable=pointless-string-statement
385 # =========================================================================
387 # =========================================================================
389 def setup_module(module):
390 nfvbench.log.setup(mute_stdout=True)
392 def test_no_credentials():
393 cred = Credentials('/completely/wrong/path/openrc', None, False)
395 # shouldn't get valid data unless user set environment variables
401 # Because trex_stl_lib may not be installed when running unit test
402 # nfvbench.traffic_client will try to import STLError:
403 # from trex_stl_lib.api import STLError
404 # will raise ImportError: No module named trex_stl_lib.api
406 import trex_stl_lib.api
408 assert trex_stl_lib.api
410 # Make up a trex_stl_lib.api.STLError class
411 class STLError(Exception):
415 from types import ModuleType
417 stl_lib_mod = ModuleType('trex_stl_lib')
418 sys.modules['trex_stl_lib'] = stl_lib_mod
419 api_mod = ModuleType('trex_stl_lib.api')
420 stl_lib_mod.api = api_mod
421 sys.modules['trex_stl_lib.api'] = api_mod
422 api_mod.STLError = STLError
424 # pylint: disable=wrong-import-position,ungrouped-imports
425 from nfvbench.traffic_client import Device
426 from nfvbench.traffic_client import IpBlock
427 from nfvbench.traffic_client import TrafficClient
428 from nfvbench.traffic_client import TrafficGeneratorFactory
430 # pylint: enable=wrong-import-position,ungrouped-imports
433 ipb = IpBlock('10.0.0.0', '0.0.0.1', 256)
434 assert ipb.get_ip() == '10.0.0.0'
435 assert ipb.get_ip(255) == '10.0.0.255'
436 with pytest.raises(IndexError):
438 # verify with step larger than 1
439 ipb = IpBlock('10.0.0.0', '0.0.0.2', 256)
440 assert ipb.get_ip() == '10.0.0.0'
441 assert ipb.get_ip(1) == '10.0.0.2'
442 assert ipb.get_ip(128) == '10.0.1.0'
443 assert ipb.get_ip(255) == '10.0.1.254'
444 with pytest.raises(IndexError):
448 def check_config(configs, cc, fc, src_ip, dst_ip, step_ip):
449 '''Verify that the range configs for each chain have adjacent IP ranges
450 of the right size and without holes between chains
452 step = Device.ip_to_int(step_ip)
454 sip = Device.ip_to_int(src_ip)
455 dip = Device.ip_to_int(dst_ip)
456 for index in range(cc):
457 config = configs[index]
458 assert config['ip_src_count'] == config['ip_dst_count']
459 assert Device.ip_to_int(config['ip_src_addr']) == sip
460 assert Device.ip_to_int(config['ip_dst_addr']) == dip
461 count = config['ip_src_count']
468 def create_device(fc, cc, ip, gip, tggip, step_ip):
469 return Device(0, 0, flow_count=fc, chain_count=cc, ip=ip, gateway_ip=gip, tg_gateway_ip=tggip,
470 ip_addrs_step=step_ip,
471 tg_gateway_ip_addrs_step=step_ip,
472 gateway_ip_addrs_step=step_ip)
475 def check_device_flow_config(step_ip):
482 dev0 = create_device(fc, cc, ip0, gip, tggip, step_ip)
483 dev1 = create_device(fc, cc, ip1, gip, tggip, step_ip)
484 dev0.set_destination(dev1)
485 configs = dev0.get_stream_configs(ChainType.EXT)
486 check_config(configs, cc, fc, ip0, ip1, step_ip)
489 def test_device_flow_config():
490 check_device_flow_config('0.0.0.1')
491 check_device_flow_config('0.0.0.2')
494 def test_device_ip_range():
495 def ip_range_overlaps(ip0, ip1, flows):
498 dev0 = create_device(flows, 10, ip0, gip, tggip, '0.0.0.1')
499 dev1 = create_device(flows, 10, ip1, gip, tggip, '0.0.0.1')
500 dev0.set_destination(dev1)
501 return dev0.ip_range_overlaps()
503 assert not ip_range_overlaps('10.0.0.0', '20.0.0.0', 10000)
504 assert ip_range_overlaps('10.0.0.0', '10.0.1.0', 10000)
505 assert ip_range_overlaps('10.0.0.0', '10.0.1.0', 257)
506 assert ip_range_overlaps('10.0.1.0', '10.0.0.0', 257)
510 refcfg = {1: 100, 2: {21: 100, 22: 200}, 3: None}
511 res1 = {1: 10, 2: {21: 100, 22: 200}, 3: None}
512 res2 = {1: 100, 2: {21: 1000, 22: 200}, 3: None}
513 res3 = {1: 100, 2: {21: 100, 22: 200}, 3: "abc"}
514 assert config_loads("{}", refcfg) == refcfg
515 assert config_loads("{1: 10}", refcfg) == res1
516 assert config_loads("{2: {21: 1000}}", refcfg) == res2
517 assert config_loads('{3: "abc"}', refcfg) == res3
520 # pairs of input string and expected subset (None if identical)
523 ["{2: {21: 100, 30: 50}}", "{2: {30: 50}}"],
524 ["{2: {0: 1, 1: 2}, 5: 5}", None],
525 ["{1: 'abc', 2: {21: 0}}", "{1: 'abc'}"],
528 for fail_pair in fail_pairs:
529 with pytest.raises(Exception) as e_info:
530 config_loads(fail_pair[0], refcfg)
531 expected = fail_pair[1]
533 expected = fail_pair[0]
534 assert expected in e_info.value.message
537 flavor = {'flavor': {'vcpus': 2, 'ram': 8192, 'disk': 0,
538 'extra_specs': {'hw:cpu_policy': 'dedicated'}}}
539 new_flavor = {'flavor': {'vcpus': 2, 'ram': 8192, 'disk': 0,
540 'extra_specs': {'hw:cpu_policy': 'dedicated', 'hw:numa_nodes': 2}}}
541 assert config_loads("{'flavor': {'extra_specs': {'hw:numa_nodes': 2}}}", flavor,
542 whitelist_keys=['alpha', 'extra_specs']) == new_flavor
546 logger = logging.getLogger('fluent-logger')
548 class FluentdConfig(dict):
549 def __getattr__(self, attr):
550 return self.get(attr)
554 'logging_tag': 'nfvbench',
555 'result_tag': 'resultnfvbench',
560 'logging_tag': 'nfvbench',
561 'result_tag': 'resultnfvbench',
567 'result_tag': 'resultnfvbench',
572 'logging_tag': 'nfvbench',
579 handler = FluentLogHandler(fluentd_configs=fluentd_configs)
580 logger.addHandler(handler)
581 logger.setLevel(logging.INFO)
583 logger.warning('test %d', 100)
586 raise Exception("test")
588 logger.exception("got exception")
590 def assert_ndr_pdr(stats, ndr, ndr_dr, pdr, pdr_dr):
591 assert stats['ndr']['rate_percent'] == ndr
592 assert stats['ndr']['stats']['overall']['drop_percentage'] == ndr_dr
593 assert_equivalence(pdr, stats['pdr']['rate_percent'])
594 assert_equivalence(pdr_dr, stats['pdr']['stats']['overall']['drop_percentage'])
596 def get_traffic_client():
598 'traffic_generator': {'host_name': 'nfvbench_tg',
599 'default_profile': 'dummy',
600 'generator_profile': [{'name': 'dummy',
603 'intf_speed': '10Gbps',
604 'interfaces': [{'port': 0, 'pci': 0},
605 {'port': 1, 'pci': 0}]}],
606 'ip_addrs_step': '0.0.0.1',
607 'ip_addrs': ['10.0.0.0/8', '20.0.0.0/8'],
608 'tg_gateway_ip_addrs': ['1.1.0.100', '2.2.0.100'],
609 'tg_gateway_ip_addrs_step': '0.0.0.1',
610 'gateway_ip_addrs': ['1.1.0.2', '2.2.0.2'],
611 'gateway_ip_addrs_step': '0.0.0.1',
612 'udp_src_port': None,
613 'udp_dst_port': None},
614 'generator_profile': 'dummy',
615 'service_chain': 'PVP',
616 'service_chain_count': 1,
618 'vlan_tagging': True,
626 'check_traffic_time_sec': 200,
627 'generic_poll_sec': 2,
628 'measurement': {'NDR': 0.001, 'PDR': 0.1, 'load_epsilon': 0.1},
630 generator_factory = TrafficGeneratorFactory(config)
631 config.generator_config = generator_factory.get_generator_config(config.generator_profile)
632 traffic_client = TrafficClient(config, skip_sleep=True)
633 traffic_client.start_traffic_generator()
634 traffic_client.set_traffic('64', True)
635 return traffic_client
637 def test_ndr_at_lr():
638 traffic_client = get_traffic_client()
639 tg = traffic_client.gen
641 # this is a perfect sut with no loss at LR
642 tg.set_response_curve(lr_dr=0, ndr=100, max_actual_tx=100, max_11_tx=100)
643 # tx packets should be line rate for 64B and no drops...
644 assert tg.get_tx_pps_dropped_pps(100) == (LR_64B_PPS, 0)
645 # NDR and PDR should be at 100%
646 traffic_client.ensure_end_to_end()
647 results = traffic_client.get_ndr_and_pdr()
649 assert_ndr_pdr(results, 200.0, 0.0, 200.0, 0.0)
651 def test_ndr_at_50():
652 traffic_client = get_traffic_client()
653 tg = traffic_client.gen
654 # this is a sut with an NDR of 50% and linear drop rate after NDR up to 20% drops at LR
655 # (meaning that if you send 100% TX, you will only receive 80% RX)
656 # the tg requested TX/actual TX ratio is 1up to 50%, after 50%
657 # is linear up 80% actuak TX when requesting 100%
658 tg.set_response_curve(lr_dr=20, ndr=50, max_actual_tx=80, max_11_tx=50)
659 # tx packets should be half line rate for 64B and no drops...
660 assert tg.get_tx_pps_dropped_pps(50) == (LR_64B_PPS / 2, 0)
661 # at 100% TX requested, actual TX is 80% where the drop rate is 3/5 of 20% of the actual TX
662 assert tg.get_tx_pps_dropped_pps(100) == (int(LR_64B_PPS * 0.8),
663 int(LR_64B_PPS * 0.8 * 0.6 * 0.2))
664 results = traffic_client.get_ndr_and_pdr()
665 assert_ndr_pdr(results, 100.0, 0.0, 100.781, 0.09374)
667 def test_ndr_pdr_low_cpu():
668 traffic_client = get_traffic_client()
669 tg = traffic_client.gen
670 # This test is for the case where the TG is underpowered and cannot send fast enough for the NDR
671 # true NDR=40%, actual TX at 50% = 30%, actual measured DR is 0%
672 # The ndr/pdr should bail out with a warning and a best effort measured NDR of 30%
673 tg.set_response_curve(lr_dr=50, ndr=40, max_actual_tx=60, max_11_tx=0)
674 # tx packets should be 30% at requested half line rate for 64B and no drops...
675 assert tg.get_tx_pps_dropped_pps(50) == (int(LR_64B_PPS * 0.3), 0)
676 results = traffic_client.get_ndr_and_pdr()
679 # pp = pprint.PrettyPrinter(indent=4)