be800333adc717ba7e7c6e7adc1a1e37ae526d6c
[nfvbench.git] / test / test_nfvbench.py
1 #!/usr/bin/env python
2 # Copyright 2016 Cisco Systems, Inc.  All rights reserved.
3 #
4 #    Licensed under the Apache License, Version 2.0 (the "License"); you may
5 #    not use this file except in compliance with the License. You may obtain
6 #    a copy of the License at
7 #
8 #         http://www.apache.org/licenses/LICENSE-2.0
9 #
10 #    Unless required by applicable law or agreed to in writing, software
11 #    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
12 #    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
13 #    License for the specific language governing permissions and limitations
14 #    under the License.
15 #
16
17 from attrdict import AttrDict
18 from nfvbench.connection import SSH
19 from nfvbench.credentials import Credentials
20 from nfvbench.network import Interface
21 from nfvbench.network import Network
22 from nfvbench.specs import Encaps
23 import nfvbench.traffic_gen.traffic_utils as traffic_utils
24 import os
25 import pytest
26
27 __location__ = os.path.realpath(os.path.join(os.getcwd(),
28                                              os.path.dirname(__file__)))
29
30
31 @pytest.fixture
32 def ssh(monkeypatch):
33     def mock_init(self, ssh_access, *args, **kwargs):
34         self.ssh_access = ssh_access
35         if ssh_access.private_key:
36             self.pkey = self._get_pkey(ssh_access.private_key)
37         else:
38             self.pkey = None
39         self._client = False
40         self.connect_timeout = 2
41         self.connect_retry_count = 1
42         self.connect_retry_wait_sec = 1
43         super(SSH, self).__init__()
44
45     monkeypatch.setattr(SSH, '__init__', mock_init)
46
47
48 @pytest.fixture
49 def openstack_vxlan_spec():
50     return AttrDict(
51         {
52             'openstack': AttrDict({
53                 'vswitch': "VTS",
54                 'encaps': Encaps.VxLAN}
55             ),
56             'run_spec': AttrDict({
57                 'use_vpp': True
58             })
59         }
60     )
61
62 # =========================================================================
63 # PVP Chain tests
64 # =========================================================================
65
66 def test_chain_interface():
67     iface = Interface('testname', 'vpp', tx_packets=1234, rx_packets=4321)
68     assert iface.name == 'testname'
69     assert iface.device == 'vpp'
70     assert iface.get_packet_count('tx') == 1234
71     assert iface.get_packet_count('rx') == 4321
72     assert iface.get_packet_count('wrong_key') == 0
73
74
75 @pytest.fixture(scope='session')
76 def iface1():
77     return Interface('iface1', 'trex', tx_packets=10000, rx_packets=1234)
78
79
80 @pytest.fixture(scope='session')
81 def iface2():
82     return Interface('iface2', 'n9k', tx_packets=1234, rx_packets=9901)
83
84
85 @pytest.fixture(scope='session')
86 def iface3():
87     return Interface('iface3', 'n9k', tx_packets=9900, rx_packets=1234)
88
89
90 @pytest.fixture(scope='session')
91 def iface4():
92     return Interface('iface4', 'vpp', tx_packets=1234, rx_packets=9801)
93
94
95 @pytest.fixture(scope='session')
96 def net1(iface1, iface2, iface3, iface4):
97     return Network([iface1, iface2, iface3, iface4], reverse=False)
98
99
100 @pytest.fixture(scope='session')
101 def net2(iface1, iface2, iface3):
102     return Network([iface1, iface2, iface3], reverse=True)
103
104
105 def test_chain_network(net1, net2, iface1, iface2, iface3, iface4):
106     assert [iface1, iface2, iface3, iface4] == net1.get_interfaces()
107     assert [iface3, iface2, iface1] == net2.get_interfaces()
108     net2.add_interface(iface4)
109     assert [iface4, iface3, iface2, iface1] == net2.get_interfaces()
110
111
112 """
113 def test_chain_analysis(net1, monkeypatch, openstack_vxlan_spec):
114     def mock_empty(self, *args, **kwargs):
115         pass
116
117     monkeypatch.setattr(ServiceChain, '_setup', mock_empty)
118
119     f = ServiceChain(AttrDict({'service_chain': 'DUMMY'}), [], {'tor': {}}, openstack_vxlan_spec,
120                      lambda x, y, z: None)
121     result = f.get_analysis([net1])
122     assert result[1]['packet_drop_count'] == 99
123     assert result[1]['packet_drop_percentage'] == 0.99
124     assert result[2]['packet_drop_count'] == 1
125     assert result[2]['packet_drop_percentage'] == 0.01
126     assert result[3]['packet_drop_count'] == 99
127     assert result[3]['packet_drop_percentage'] == 0.99
128
129     net1.reverse = True
130     result = f.get_analysis([net1])
131     assert result[1]['packet_drop_count'] == 0
132     assert result[1]['packet_drop_percentage'] == 0.0
133     assert result[2]['packet_drop_count'] == 0
134     assert result[2]['packet_drop_percentage'] == 0.0
135     assert result[3]['packet_drop_count'] == 0
136     assert result[3]['packet_drop_percentage'] == 0.0
137
138
139 @pytest.fixture
140 def pvp_chain(monkeypatch, openstack_vxlan_spec):
141     tor_vni1 = Interface('vni-4097', 'n9k', 50, 77)
142     vsw_vni1 = Interface('vxlan_tunnel0', 'vpp', 77, 48)
143     vsw_vif1 = Interface('VirtualEthernet0/0/2', 'vpp', 48, 77)
144     vsw_vif2 = Interface('VirtualEthernet0/0/3', 'vpp', 77, 47)
145     vsw_vni2 = Interface('vxlan_tunnel1', 'vpp', 43, 77)
146     tor_vni2 = Interface('vni-4098', 'n9k', 77, 40)
147
148     def mock_init(self, *args, **kwargs):
149         self.vni_ports = [4097, 4098]
150         self.specs = openstack_vxlan_spec
151         self.clients = {
152             'vpp': AttrDict({
153                 'set_interface_counters': lambda: None,
154             })
155         }
156         self.worker = AttrDict({
157             'run': lambda: None,
158         })
159
160     def mock_empty(self, *args, **kwargs):
161         pass
162
163     def mock_get_network(self, traffic_port, vni_id, reverse=False):
164         if vni_id == 0:
165             return Network([tor_vni1, vsw_vni1, vsw_vif1], reverse)
166         else:
167             return Network([tor_vni2, vsw_vni2, vsw_vif2], reverse)
168
169     def mock_get_data(self):
170         return {}
171
172     monkeypatch.setattr(PVPChain, '_get_network', mock_get_network)
173     monkeypatch.setattr(PVPChain, '_get_data', mock_get_data)
174     monkeypatch.setattr(PVPChain, '_setup', mock_empty)
175     monkeypatch.setattr(VxLANWorker, '_clear_interfaces', mock_empty)
176     monkeypatch.setattr(PVPChain, '_generate_traffic', mock_empty)
177     monkeypatch.setattr(PVPChain, '__init__', mock_init)
178     return PVPChain(None, None, {'vm': None, 'vpp': None, 'tor': None, 'traffic': None}, None)
179
180
181 def test_pvp_chain_run(pvp_chain):
182     result = pvp_chain.run()
183     expected_result = {
184         'raw_data': {},
185         'stats': None,
186         'packet_analysis': {
187             'direction-forward': [
188                 OrderedDict([
189                     ('interface', 'vni-4097'),
190                     ('device', 'n9k'),
191                     ('packet_count', 50)
192                 ]),
193                 OrderedDict([
194                     ('interface', 'vxlan_tunnel0'),
195                     ('device', 'vpp'),
196                     ('packet_count', 48),
197                     ('packet_drop_count', 2),
198                     ('packet_drop_percentage', 4.0)
199                 ]),
200                 OrderedDict([
201                     ('interface', 'VirtualEthernet0/0/2'),
202                     ('device', 'vpp'),
203                     ('packet_count', 48),
204                     ('packet_drop_count', 0),
205                     ('packet_drop_percentage', 0.0)
206                 ]),
207                 OrderedDict([
208                     ('interface', 'VirtualEthernet0/0/3'),
209                     ('device', 'vpp'),
210                     ('packet_count', 47),
211                     ('packet_drop_count', 1),
212                     ('packet_drop_percentage', 2.0)
213                 ]),
214                 OrderedDict([
215                     ('interface', 'vxlan_tunnel1'),
216                     ('device', 'vpp'),
217                     ('packet_count', 43),
218                     ('packet_drop_count', 4),
219                     ('packet_drop_percentage', 8.0)
220                 ]),
221                 OrderedDict([
222                     ('interface', 'vni-4098'),
223                     ('device', 'n9k'),
224                     ('packet_count', 40),
225                     ('packet_drop_count', 3),
226                     ('packet_drop_percentage', 6.0)
227                 ])
228             ],
229             'direction-reverse': [
230                 OrderedDict([
231                     ('interface', 'vni-4098'),
232                     ('device', 'n9k'),
233                     ('packet_count', 77)
234                 ]),
235                 OrderedDict([
236                     ('interface', 'vxlan_tunnel1'),
237                     ('device', 'vpp'),
238                     ('packet_count', 77),
239                     ('packet_drop_count', 0),
240                     ('packet_drop_percentage', 0.0)
241                 ]),
242                 OrderedDict([
243                     ('interface', 'VirtualEthernet0/0/3'),
244                     ('device', 'vpp'),
245                     ('packet_count', 77),
246                     ('packet_drop_count', 0),
247                     ('packet_drop_percentage', 0.0)
248                 ]),
249                 OrderedDict([
250                     ('interface', 'VirtualEthernet0/0/2'),
251                     ('device', 'vpp'),
252                     ('packet_count', 77),
253                     ('packet_drop_count', 0),
254                     ('packet_drop_percentage', 0.0)
255                 ]),
256                 OrderedDict([
257                     ('interface', 'vxlan_tunnel0'),
258                     ('device', 'vpp'),
259                     ('packet_count', 77),
260                     ('packet_drop_count', 0),
261                     ('packet_drop_percentage', 0.0)
262                 ]),
263                 OrderedDict([
264                     ('interface', 'vni-4097'),
265                     ('device', 'n9k'),
266                     ('packet_count', 77),
267                     ('packet_drop_count', 0),
268                     ('packet_drop_percentage', 0.0)
269                 ])
270             ]
271         }
272     }
273     assert result == expected_result
274 """
275
276
277 # =========================================================================
278 # PVVP Chain tests
279 # =========================================================================
280
281 """
282 @pytest.fixture
283 def pvvp_chain(monkeypatch, openstack_vxlan_spec):
284     tor_vni1 = Interface('vni-4097', 'n9k', 50, 77)
285     vsw_vni1 = Interface('vxlan_tunnel0', 'vpp', 77, 48)
286     vsw_vif1 = Interface('VirtualEthernet0/0/2', 'vpp', 48, 77)
287     vsw_vif3 = Interface('VirtualEthernet0/0/0', 'vpp', 77, 47)
288     vsw_vif4 = Interface('VirtualEthernet0/0/1', 'vpp', 45, 77)
289     vsw_vif2 = Interface('VirtualEthernet0/0/3', 'vpp', 77, 44)
290     vsw_vni2 = Interface('vxlan_tunnel1', 'vpp', 43, 77)
291     tor_vni2 = Interface('vni-4098', 'n9k', 77, 40)
292
293     def mock_init(self, *args, **kwargs):
294         self.vni_ports = [4099, 4100]
295         self.v2vnet = V2VNetwork()
296         self.specs = openstack_vxlan_spec
297         self.clients = {
298             'vpp': AttrDict({
299                 'get_v2v_network': lambda reverse=None: Network([vsw_vif3, vsw_vif4], reverse),
300                 'set_interface_counters': lambda pvvp=None: None,
301                 'set_v2v_counters': lambda: None,
302             })
303         }
304         self.worker = AttrDict({
305             'run': lambda: None,
306         })
307
308     def mock_empty(self, *args, **kwargs):
309         pass
310
311     def mock_get_network(self, traffic_port, vni_id, reverse=False):
312         if vni_id == 0:
313             return Network([tor_vni1, vsw_vni1, vsw_vif1], reverse)
314         else:
315             return Network([tor_vni2, vsw_vni2, vsw_vif2], reverse)
316
317     def mock_get_data(self):
318         return {}
319
320     monkeypatch.setattr(PVVPChain, '_get_network', mock_get_network)
321     monkeypatch.setattr(PVVPChain, '_get_data', mock_get_data)
322     monkeypatch.setattr(PVVPChain, '_setup', mock_empty)
323     monkeypatch.setattr(VxLANWorker, '_clear_interfaces', mock_empty)
324     monkeypatch.setattr(PVVPChain, '_generate_traffic', mock_empty)
325     monkeypatch.setattr(PVVPChain, '__init__', mock_init)
326
327     return PVVPChain(None, None, {'vm': None, 'vpp': None, 'tor': None, 'traffic': None}, None)
328
329
330 def test_pvvp_chain_run(pvvp_chain):
331     result = pvvp_chain.run()
332
333     expected_result = {
334         'raw_data': {},
335         'stats': None,
336         'packet_analysis':
337             {'direction-forward': [
338                 OrderedDict([
339                     ('interface', 'vni-4097'),
340                     ('device', 'n9k'),
341                     ('packet_count', 50)
342                 ]),
343                 OrderedDict([
344                     ('interface', 'vxlan_tunnel0'),
345                     ('device', 'vpp'),
346                     ('packet_count', 48),
347                     ('packet_drop_count', 2),
348                     ('packet_drop_percentage', 4.0)
349                 ]),
350                 OrderedDict([
351                     ('interface', 'VirtualEthernet0/0/2'),
352                     ('device', 'vpp'),
353                     ('packet_count', 48),
354                     ('packet_drop_count', 0),
355                     ('packet_drop_percentage', 0.0)
356                 ]),
357                 OrderedDict([
358                     ('interface', 'VirtualEthernet0/0/0'),
359                     ('device', 'vpp'),
360                     ('packet_count', 47),
361                     ('packet_drop_count', 1),
362                     ('packet_drop_percentage', 2.0)
363                 ]),
364                 OrderedDict([
365                     ('interface', 'VirtualEthernet0/0/1'),
366                     ('device', 'vpp'),
367                     ('packet_count', 45),
368                     ('packet_drop_count', 2),
369                     ('packet_drop_percentage', 4.0)
370                 ]),
371                 OrderedDict([
372                     ('interface', 'VirtualEthernet0/0/3'),
373                     ('device', 'vpp'),
374                     ('packet_count', 44),
375                     ('packet_drop_count', 1),
376                     ('packet_drop_percentage', 2.0)
377                 ]),
378                 OrderedDict([
379                     ('interface', 'vxlan_tunnel1'),
380                     ('device', 'vpp'),
381                     ('packet_count', 43),
382                     ('packet_drop_count', 1),
383                     ('packet_drop_percentage', 2.0)
384                 ]),
385                 OrderedDict([
386                     ('interface', 'vni-4098'),
387                     ('device', 'n9k'),
388                     ('packet_count', 40),
389                     ('packet_drop_count', 3),
390                     ('packet_drop_percentage', 6.0)
391                 ])
392             ],
393             'direction-reverse': [
394                 OrderedDict([
395                     ('interface', 'vni-4098'),
396                     ('device', 'n9k'),
397                     ('packet_count', 77)
398                 ]),
399                 OrderedDict([
400                     ('interface', 'vxlan_tunnel1'),
401                     ('device', 'vpp'),
402                     ('packet_count', 77),
403                     ('packet_drop_count', 0),
404                     ('packet_drop_percentage', 0.0)
405                 ]),
406                 OrderedDict([
407                     ('interface', 'VirtualEthernet0/0/3'),
408                     ('device', 'vpp'),
409                     ('packet_count', 77),
410                     ('packet_drop_count', 0),
411                     ('packet_drop_percentage', 0.0)
412                 ]),
413                 OrderedDict([
414                     ('interface', 'VirtualEthernet0/0/1'),
415                     ('device', 'vpp'),
416                     ('packet_count', 77),
417                     ('packet_drop_count', 0),
418                     ('packet_drop_percentage', 0.0)
419                 ]),
420                 OrderedDict([
421                     ('interface', 'VirtualEthernet0/0/0'),
422                     ('device', 'vpp'),
423                     ('packet_count', 77),
424                     ('packet_drop_count', 0),
425                     ('packet_drop_percentage', 0.0)
426                 ]),
427                 OrderedDict([
428                     ('interface', 'VirtualEthernet0/0/2'),
429                     ('device', 'vpp'),
430                     ('packet_count', 77),
431                     ('packet_drop_count', 0),
432                     ('packet_drop_percentage', 0.0)
433                 ]),
434                 OrderedDict([
435                     ('interface', 'vxlan_tunnel0'),
436                     ('device', 'vpp'),
437                     ('packet_count', 77),
438                     ('packet_drop_count', 0),
439                     ('packet_drop_percentage', 0.0)
440                 ]),
441                 OrderedDict([
442                     ('interface', 'vni-4097'),
443                     ('device', 'n9k'),
444                     ('packet_count', 77),
445                     ('packet_drop_count', 0),
446                     ('packet_drop_percentage', 0.0)
447                 ])
448             ]}
449     }
450     assert result == expected_result
451 """
452
453 # =========================================================================
454 # Traffic client tests
455 # =========================================================================
456
457 def test_parse_rate_str():
458     parse_rate_str = traffic_utils.parse_rate_str
459     try:
460         assert parse_rate_str('100%') == {'rate_percent': '100.0'}
461         assert parse_rate_str('37.5%') == {'rate_percent': '37.5'}
462         assert parse_rate_str('100%') == {'rate_percent': '100.0'}
463         assert parse_rate_str('60pps') == {'rate_pps': '60'}
464         assert parse_rate_str('60kpps') == {'rate_pps': '60000'}
465         assert parse_rate_str('6Mpps') == {'rate_pps': '6000000'}
466         assert parse_rate_str('6gpps') == {'rate_pps': '6000000000'}
467         assert parse_rate_str('80bps') == {'rate_bps': '80'}
468         assert parse_rate_str('80bps') == {'rate_bps': '80'}
469         assert parse_rate_str('80kbps') == {'rate_bps': '80000'}
470         assert parse_rate_str('80kBps') == {'rate_bps': '640000'}
471         assert parse_rate_str('80Mbps') == {'rate_bps': '80000000'}
472         assert parse_rate_str('80 MBps') == {'rate_bps': '640000000'}
473         assert parse_rate_str('80Gbps') == {'rate_bps': '80000000000'}
474     except Exception as exc:
475         assert False, exc.message
476
477     def should_raise_error(str):
478         try:
479             parse_rate_str(str)
480         except Exception:
481             return True
482         else:
483             assert False
484
485     assert should_raise_error('101')
486     assert should_raise_error('201%')
487     assert should_raise_error('10Kbps')
488     assert should_raise_error('0kbps')
489     assert should_raise_error('0pps')
490     assert should_raise_error('-1bps')
491
492 def test_rate_conversion():
493     assert traffic_utils.load_to_bps(50, 10000000000) == pytest.approx(5000000000.0)
494     assert traffic_utils.load_to_bps(37, 10000000000) == pytest.approx(3700000000.0)
495     assert traffic_utils.load_to_bps(100, 10000000000) == pytest.approx(10000000000.0)
496
497     assert traffic_utils.bps_to_load(5000000000.0, 10000000000) == pytest.approx(50.0)
498     assert traffic_utils.bps_to_load(3700000000.0, 10000000000) == pytest.approx(37.0)
499     assert traffic_utils.bps_to_load(10000000000.0, 10000000000) == pytest.approx(100.0)
500
501     assert traffic_utils.bps_to_pps(500000, 64) == pytest.approx(744.047619048)
502     assert traffic_utils.bps_to_pps(388888, 1518) == pytest.approx(31.6066319896)
503     assert traffic_utils.bps_to_pps(9298322222, 340.3) == pytest.approx(3225895.85831)
504
505     assert traffic_utils.pps_to_bps(744.047619048, 64) == pytest.approx(500000)
506     assert traffic_utils.pps_to_bps(31.6066319896, 1518) == pytest.approx(388888)
507     assert traffic_utils.pps_to_bps(3225895.85831, 340.3) == pytest.approx(9298322222)
508
509
510 """
511 @pytest.fixture
512 def traffic_client(monkeypatch):
513
514     def mock_init(self, *args, **kwargs):
515         self.run_config = {
516             'bidirectional': False,
517             'l2frame_size': '64',
518             'duration_sec': 30,
519             'rates': [{'rate_percent': '10'}, {'rate_pps': '1'}]
520         }
521
522         self.config = AttrDict({
523             'generator_config': {
524                 'intf_speed': 10000000000
525             },
526             'ndr_run': True,
527             'pdr_run': True,
528             'single_run': False,
529             'attempts': 1,
530             'measurement': {
531                 'NDR': 0.0,
532                 'PDR': 0.1,
533                 'load_epsilon': 0.1
534             }
535         })
536
537         self.runner = AttrDict({
538             'time_elapsed': lambda: 30,
539             'stop': lambda: None,
540             'client': AttrDict({'get_stats': lambda: None})
541         })
542
543         self.current_load = None
544         self.dummy_stats = {
545             50.0: 72.6433562831,
546             25.0: 45.6095059858,
547             12.5: 0.0,
548             18.75: 27.218642979,
549             15.625: 12.68585861,
550             14.0625: 2.47154392563,
551             13.28125: 0.000663797066801,
552             12.890625: 0.0,
553             13.0859375: 0.0,
554             13.18359375: 0.00359387347122,
555             13.671875: 0.307939922531,
556             13.4765625: 0.0207718516156,
557             13.57421875: 0.0661795060969
558         }
559
560     def mock_modify_load(self, load):
561         self.run_config['rates'][0] = {'rate_percent': str(load)}
562         self.current_load = load
563
564     def mock_run_traffic(self):
565         yield {
566             'overall': {
567                 'drop_rate_percent': self.dummy_stats[self.current_load],
568                 'rx': {
569                     'total_pkts': 1,
570                     'avg_delay_usec': 0.0,
571                     'max_delay_usec': 0.0,
572                     'min_delay_usec': 0.0
573                 }
574             }
575         }
576
577     monkeypatch.setattr(TrafficClient, '__init__', mock_init)
578     monkeypatch.setattr(TrafficClient, 'modify_load', mock_modify_load)
579     monkeypatch.setattr(TrafficClient, 'run_traffic', mock_run_traffic)
580
581     return TrafficClient()
582
583
584 def test_ndr_pdr_search(traffic_client):
585     expected_results = {
586         'pdr': {
587             'l2frame_size': '64',
588             'initial_rate_type': 'rate_percent',
589             'stats': {
590                 'overall': {
591                     'drop_rate_percent': 0.0661795060969,
592                     'min_delay_usec': 0.0,
593                     'avg_delay_usec': 0.0,
594                     'max_delay_usec': 0.0
595                 }
596             },
597             'load_percent_per_direction': 13.57421875,
598             'rate_percent': 13.57422547,
599             'rate_bps': 1357422547.0,
600             'rate_pps': 2019974.0282738095,
601             'duration_sec': 30
602         },
603         'ndr': {
604             'l2frame_size': '64',
605             'initial_rate_type': 'rate_percent',
606             'stats': {
607                 'overall': {
608                     'drop_rate_percent': 0.0,
609                     'min_delay_usec': 0.0,
610                     'avg_delay_usec': 0.0,
611                     'max_delay_usec': 0.0
612                 }
613             },
614             'load_percent_per_direction': 13.0859375,
615             'rate_percent': 13.08594422,
616             'rate_bps': 1308594422.0,
617             'rate_pps': 1947313.1279761905,
618             'duration_sec': 30
619         }
620     }
621
622     results = traffic_client.get_ndr_and_pdr()
623     assert len(results) == 2
624     for result in results.values():
625         result.pop('timestamp_sec')
626         result.pop('time_taken_sec')
627     assert results == expected_results
628 """
629
630 # =========================================================================
631 # Other tests
632 # =========================================================================
633
634 def test_no_credentials():
635     cred = Credentials('/completely/wrong/path/openrc', None, False)
636     if cred.rc_auth_url:
637         # shouldn't get valid data unless user set environment variables
638         assert False
639     else:
640         assert True