Merge "Push yardstick debug log into the artifacts"
[yardstick.git] / tests / unit / benchmark / scenarios / networking / test_vnf_generic.py
1 #!/usr/bin/env python
2
3 # Copyright (c) 2016-2017 Intel Corporation
4 #
5 # Licensed under the Apache License, Version 2.0 (the "License");
6 # you may not use this file except in compliance with the License.
7 # You may obtain a copy of the License at
8 #
9 #      http://www.apache.org/licenses/LICENSE-2.0
10 #
11 # Unless required by applicable law or agreed to in writing, software
12 # distributed under the License is distributed on an "AS IS" BASIS,
13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 # See the License for the specific language governing permissions and
15 # limitations under the License.
16 #
17
18 # Unittest for yardstick.benchmark.scenarios.networking.test_vnf_generic
19
20 from __future__ import absolute_import
21 import unittest
22 import mock
23 import os
24
25 from yardstick.benchmark.scenarios.networking.vnf_generic import \
26     ssh_manager, NetworkServiceTestCase, IncorrectConfig, IncorrectSetup
27 from yardstick.network_services.collector.subscriber import Collector
28 from yardstick.network_services.vnf_generic.vnf.base import \
29     GenericTrafficGen, GenericVNF
30
31 STL_MOCKS = {
32     'stl': mock.MagicMock(),
33     'stl.trex_stl_lib': mock.MagicMock(),
34     'stl.trex_stl_lib.base64': mock.MagicMock(),
35     'stl.trex_stl_lib.binascii': mock.MagicMock(),
36     'stl.trex_stl_lib.collections': mock.MagicMock(),
37     'stl.trex_stl_lib.copy': mock.MagicMock(),
38     'stl.trex_stl_lib.datetime': mock.MagicMock(),
39     'stl.trex_stl_lib.functools': mock.MagicMock(),
40     'stl.trex_stl_lib.imp': mock.MagicMock(),
41     'stl.trex_stl_lib.inspect': mock.MagicMock(),
42     'stl.trex_stl_lib.json': mock.MagicMock(),
43     'stl.trex_stl_lib.linecache': mock.MagicMock(),
44     'stl.trex_stl_lib.math': mock.MagicMock(),
45     'stl.trex_stl_lib.os': mock.MagicMock(),
46     'stl.trex_stl_lib.platform': mock.MagicMock(),
47     'stl.trex_stl_lib.pprint': mock.MagicMock(),
48     'stl.trex_stl_lib.random': mock.MagicMock(),
49     'stl.trex_stl_lib.re': mock.MagicMock(),
50     'stl.trex_stl_lib.scapy': mock.MagicMock(),
51     'stl.trex_stl_lib.socket': mock.MagicMock(),
52     'stl.trex_stl_lib.string': mock.MagicMock(),
53     'stl.trex_stl_lib.struct': mock.MagicMock(),
54     'stl.trex_stl_lib.sys': mock.MagicMock(),
55     'stl.trex_stl_lib.threading': mock.MagicMock(),
56     'stl.trex_stl_lib.time': mock.MagicMock(),
57     'stl.trex_stl_lib.traceback': mock.MagicMock(),
58     'stl.trex_stl_lib.trex_stl_async_client': mock.MagicMock(),
59     'stl.trex_stl_lib.trex_stl_client': mock.MagicMock(),
60     'stl.trex_stl_lib.trex_stl_exceptions': mock.MagicMock(),
61     'stl.trex_stl_lib.trex_stl_ext': mock.MagicMock(),
62     'stl.trex_stl_lib.trex_stl_jsonrpc_client': mock.MagicMock(),
63     'stl.trex_stl_lib.trex_stl_packet_builder_interface': mock.MagicMock(),
64     'stl.trex_stl_lib.trex_stl_packet_builder_scapy': mock.MagicMock(),
65     'stl.trex_stl_lib.trex_stl_port': mock.MagicMock(),
66     'stl.trex_stl_lib.trex_stl_stats': mock.MagicMock(),
67     'stl.trex_stl_lib.trex_stl_streams': mock.MagicMock(),
68     'stl.trex_stl_lib.trex_stl_types': mock.MagicMock(),
69     'stl.trex_stl_lib.types': mock.MagicMock(),
70     'stl.trex_stl_lib.utils': mock.MagicMock(),
71     'stl.trex_stl_lib.utils.argparse': mock.MagicMock(),
72     'stl.trex_stl_lib.utils.collections': mock.MagicMock(),
73     'stl.trex_stl_lib.utils.common': mock.MagicMock(),
74     'stl.trex_stl_lib.utils.json': mock.MagicMock(),
75     'stl.trex_stl_lib.utils.os': mock.MagicMock(),
76     'stl.trex_stl_lib.utils.parsing_opts': mock.MagicMock(),
77     'stl.trex_stl_lib.utils.pwd': mock.MagicMock(),
78     'stl.trex_stl_lib.utils.random': mock.MagicMock(),
79     'stl.trex_stl_lib.utils.re': mock.MagicMock(),
80     'stl.trex_stl_lib.utils.string': mock.MagicMock(),
81     'stl.trex_stl_lib.utils.sys': mock.MagicMock(),
82     'stl.trex_stl_lib.utils.text_opts': mock.MagicMock(),
83     'stl.trex_stl_lib.utils.text_tables': mock.MagicMock(),
84     'stl.trex_stl_lib.utils.texttable': mock.MagicMock(),
85     'stl.trex_stl_lib.warnings': mock.MagicMock(),
86     'stl.trex_stl_lib.yaml': mock.MagicMock(),
87     'stl.trex_stl_lib.zlib': mock.MagicMock(),
88     'stl.trex_stl_lib.zmq': mock.MagicMock(),
89 }
90
91 COMPLETE_TREX_VNFD = \
92     {'vnfd:vnfd-catalog':
93      {'vnfd':
94       [{'benchmark':
95         {'kpi':
96          ['rx_throughput_fps',
97           'tx_throughput_fps',
98           'tx_throughput_mbps',
99           'rx_throughput_mbps',
100           'tx_throughput_pc_linerate',
101           'rx_throughput_pc_linerate',
102           'min_latency',
103           'max_latency',
104           'avg_latency']},
105         'connection-point': [{'name': 'xe0',
106                               'type': 'VPORT'},
107                              {'name': 'xe1',
108                               'type': 'VPORT'}],
109         'description': 'TRex stateless traffic generator for RFC2544',
110         'id': 'TrexTrafficGen',
111         'mgmt-interface': {'ip': '1.1.1.1',
112                            'password': 'berta',
113                            'user': 'berta',
114                            'vdu-id': 'trexgen-baremetal'},
115         'name': 'trexgen',
116         'short-name': 'trexgen',
117         'vdu': [{'description': 'TRex stateless traffic generator for RFC2544',
118                  'external-interface':
119                  [{'name': 'xe0',
120                    'virtual-interface': {'bandwidth': '10 Gbps',
121                                          'dst_ip': '1.1.1.1',
122                                          'dst_mac': '00:01:02:03:04:05',
123                                          'local_ip': '1.1.1.2',
124                                          'local_mac': '00:01:02:03:05:05',
125                                          'type': 'PCI-PASSTHROUGH',
126                                          'netmask': "255.255.255.0",
127                                          'driver': 'i40',
128                                          'vpci': '0000:00:10.2'},
129                    'vnfd-connection-point-ref': 'xe0'},
130                   {'name': 'xe1',
131                    'virtual-interface': {'bandwidth': '10 Gbps',
132                                          'dst_ip': '2.1.1.1',
133                                          'dst_mac': '00:01:02:03:04:06',
134                                          'local_ip': '2.1.1.2',
135                                          'local_mac': '00:01:02:03:05:06',
136                                          'type': 'PCI-PASSTHROUGH',
137                                          'netmask': "255.255.255.0",
138                                          'driver': 'i40',
139                                          'vpci': '0000:00:10.1'},
140                    'vnfd-connection-point-ref': 'xe1'}],
141                  'id': 'trexgen-baremetal',
142                  'name': 'trexgen-baremetal'}]}]}}
143
144 IP_ADDR_SHOW = """
145 28: eth1: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq state UP """
146 """group default qlen 1000
147     link/ether 90:e2:ba:a7:6a:c8 brd ff:ff:ff:ff:ff:ff
148     inet 1.1.1.1/8 brd 1.255.255.255 scope global eth1
149     inet6 fe80::92e2:baff:fea7:6ac8/64 scope link
150        valid_lft forever preferred_lft forever
151 29: eth5: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq state UP """
152 """group default qlen 1000
153     link/ether 90:e2:ba:a7:6a:c9 brd ff:ff:ff:ff:ff:ff
154     inet 2.1.1.1/8 brd 2.255.255.255 scope global eth5
155     inet6 fe80::92e2:baff:fea7:6ac9/64 scope link tentative
156        valid_lft forever preferred_lft forever
157 """
158
159 SYS_CLASS_NET = """
160 lrwxrwxrwx 1 root root 0 sie 10 14:16 eth1 -> """
161 """../../devices/pci0000:80/0000:80:02.2/0000:84:00.1/net/eth1
162 lrwxrwxrwx 1 root root 0 sie  3 10:37 eth2 -> """
163 """../../devices/pci0000:00/0000:00:01.1/0000:84:00.2/net/eth5
164 """
165
166 TRAFFIC_PROFILE = {
167         "schema": "isb:traffic_profile:0.1",
168         "name": "fixed",
169         "description": "Fixed traffic profile to run UDP traffic",
170         "traffic_profile": {
171             "traffic_type": "FixedTraffic",
172             "frame_rate": 100,  # pps
173             "flow_number": 10,
174             "frame_size": 64}}
175
176
177 class TestNetworkServiceTestCase(unittest.TestCase):
178     def setUp(self):
179         self.context_cfg = \
180             {'nodes':
181              {'trexgen__1': {'role': 'TrafficGen',
182                              'name': 'trafficgen_1.yardstick',
183                              'ip': '10.10.10.11',
184                              'interfaces':
185                              {'xe0':
186                               {'netmask': '255.255.255.0',
187                                'local_ip': '152.16.100.20',
188                                'local_mac': '00:00:00:00:00:01',
189                                'driver': 'i40e',
190                                'vpci': '0000:07:00.0',
191                                'dpdk_port_num': 0},
192                               'xe1':
193                               {'netmask': '255.255.255.0',
194                                'local_ip': '152.16.40.20',
195                                'local_mac': '00:00:00:00:00:02',
196                                'driver': 'i40e',
197                                'vpci': '0000:07:00.1',
198                                'dpdk_port_num': 1}},
199                              'password': 'r00t',
200                              'user': 'root'},
201               'trexvnf__1': {'name': 'vnf.yardstick',
202                              'ip': '10.10.10.12',
203                              'interfaces':
204                              {'xe0':
205                               {'netmask': '255.255.255.0',
206                                'local_ip': '152.16.100.19',
207                                'local_mac': '00:00:00:00:00:03',
208                                'driver': 'i40e',
209                                'vpci': '0000:07:00.0',
210                                'dpdk_port_num': 0},
211                               'xe1': {'netmask': '255.255.255.0',
212                                       'local_ip': '152.16.40.19',
213                                       'local_mac': '00:00:00:00:00:04',
214                                       'driver': 'i40e',
215                                       'vpci': '0000:07:00.1',
216                                       'dpdk_port_num': 1}},
217                              'routing_table': [{'netmask': '255.255.255.0',
218                                                 'gateway': '152.16.100.20',
219                                                 'network': '152.16.100.20',
220                                                 'if': 'xe0'},
221                                                {'netmask': '255.255.255.0',
222                                                 'gateway': '152.16.40.20',
223                                                 'network': '152.16.40.20',
224                                                 'if': 'xe1'}],
225                              'host': '10.223.197.164',
226                              'role': 'vnf',
227                              'user': 'root',
228                              'nd_route_tbl':
229                              [{'netmask': '112',
230                                'gateway': '0064:ff9b:0:0:0:0:9810:6414',
231                                'network': '0064:ff9b:0:0:0:0:9810:6414',
232                                'if': 'xe0'},
233                               {'netmask': '112',
234                                'gateway': '0064:ff9b:0:0:0:0:9810:2814',
235                                'network': '0064:ff9b:0:0:0:0:9810:2814',
236                                'if': 'xe1'}],
237                              'password': 'r00t'}}}
238
239         self.topology = \
240             {'short-name': 'trex-tg-topology',
241              'constituent-vnfd':
242              [{'member-vnf-index': '1',
243                'VNF model': 'tg_trex_tpl.yaml',
244                'vnfd-id-ref': 'trexgen__1'},
245               {'member-vnf-index': '2',
246                'VNF model': 'tg_trex_tpl.yaml',
247                'vnfd-id-ref': 'trexvnf__1'}],
248              'description': 'trex-tg-topology',
249              'name': 'trex-tg-topology',
250              'vld': [{'vnfd-connection-point-ref': [
251                  {'vnfd-connection-point-ref': 'xe0',
252                   'member-vnf-index-ref': '1',
253                   'vnfd-id-ref': 'trexgen'},
254                  {'vnfd-connection-point-ref': 'xe0',
255                   'member-vnf-index-ref': '2',
256                   'vnfd-id-ref': 'trexgen'}],
257                       'type': 'ELAN',
258                       'id': 'private',
259                       'name': 'trexgen__1 to trexvnf__1 link 1'},
260                      {'vnfd-connection-point-ref': [
261                          {'vnfd-connection-point-ref': 'xe1',
262                           'member-vnf-index-ref': '1',
263                           'vnfd-id-ref': 'trexgen'},
264                          {'vnfd-connection-point-ref': 'xe1',
265                           'member-vnf-index-ref': '2',
266                           'vnfd-id-ref': 'trexgen'}],
267                       'type': 'ELAN',
268                       'id': 'public',
269                       'name': 'trexvnf__1 to trexgen__1 link 2'}],
270              'id': 'trex-tg-topology'}
271
272         self.scenario_cfg = {'tc_options':
273                              {'rfc2544': {'allowed_drop_rate': '0.8 - 1'}},
274                              'task_id': 'a70bdf4a-8e67-47a3-9dc1-273c14506eb7',
275                              'tc': 'tc_ipv4_1Mflow_64B_packetsize',
276                              'runner': {'object': 'NetworkServiceTestCase',
277                                         'interval': 35,
278                                         'output_filename': 'yardstick.out',
279                                         'runner_id': 74476,
280                                         'duration': 400, 'type': 'Duration'},
281                              'traffic_profile': 'ipv4_throughput_vpe.yaml',
282                              'traffic_options':
283                              {'flow': 'ipv4_1flow_Packets_vpe.yaml',
284                               'imix': 'imix_voice.yaml'},
285                              'type': 'ISB',
286                              'nodes': {'tg__2': 'trafficgen_2.yardstick',
287                                        'tg__1': 'trafficgen_1.yardstick',
288                                        'vnf__1': 'vnf.yardstick'},
289                              'topology': 'vpe_vnf_topology.yaml'}
290
291         self.scenario_cfg["topology"] = \
292             self._get_file_abspath("vpe_vnf_topology.yaml")
293         self.s = NetworkServiceTestCase(self.scenario_cfg, self.context_cfg)
294
295     def _get_file_abspath(self, filename):
296         curr_path = os.path.dirname(os.path.abspath(__file__))
297         file_path = os.path.join(curr_path, filename)
298         return file_path
299
300     def test_ssh_manager(self):
301         with mock.patch("yardstick.ssh.SSH") as ssh:
302             ssh_mock = mock.Mock(autospec=ssh.SSH)
303             ssh_mock.execute = \
304                 mock.Mock(return_value=(0, SYS_CLASS_NET+IP_ADDR_SHOW, ""))
305             ssh.return_value = ssh_mock
306             for node, node_dict in self.context_cfg["nodes"].items():
307                 with ssh_manager(node_dict) as conn:
308                     self.assertIsNotNone(conn)
309
310     def test___init__(self):
311         assert self.topology
312
313     def test___get_traffic_flow(self):
314         self.scenario_cfg["traffic_options"]["flow"] = \
315             self._get_file_abspath("ipv4_1flow_Packets_vpe.yaml")
316         result = {'flow': {'dstip4_range': '152.40.0.20',
317                            'srcip4_range': '152.16.0.20', 'count': 1}}
318         self.assertEqual(result, self.s._get_traffic_flow(self.scenario_cfg))
319
320     def test___get_traffic_flow_error(self):
321         self.scenario_cfg["traffic_options"]["flow"] = \
322             "ipv4_1flow_Packets_vpe.yaml1"
323         self.assertEqual({}, self.s._get_traffic_flow(self.scenario_cfg))
324
325     def test_get_vnf_imp(self):
326         vnfd = COMPLETE_TREX_VNFD['vnfd:vnfd-catalog']['vnfd'][0]
327         with mock.patch.dict("sys.modules", STL_MOCKS):
328             self.assertIsNotNone(self.s.get_vnf_impl(vnfd))
329
330     def test_load_vnf_models_invalid(self):
331         self.context_cfg["nodes"]['trexgen__1']['VNF model'] = \
332             self._get_file_abspath("tg_trex_tpl.yaml")
333         self.context_cfg["nodes"]['trexvnf__1']['VNF model'] = \
334             self._get_file_abspath("tg_trex_tpl.yaml")
335
336         vnf = mock.Mock(autospec=GenericVNF)
337         self.s.get_vnf_impl = mock.Mock(return_value=vnf)
338
339         self.assertIsNotNone(self.s.load_vnf_models(self.context_cfg))
340
341     def test_map_topology_to_infrastructure(self):
342         with mock.patch("yardstick.ssh.SSH") as ssh:
343             ssh_mock = mock.Mock(autospec=ssh.SSH)
344             ssh_mock.execute = \
345                 mock.Mock(return_value=(0, SYS_CLASS_NET+IP_ADDR_SHOW, ""))
346             ssh.return_value = ssh_mock
347             self.s.map_topology_to_infrastructure(self.context_cfg,
348                                                   self.topology)
349         self.assertEqual("tg_trex_tpl.yaml",
350                          self.context_cfg["nodes"]['trexgen__1']['VNF model'])
351         self.assertEqual("tg_trex_tpl.yaml",
352                          self.context_cfg["nodes"]['trexvnf__1']['VNF model'])
353
354     def test_map_topology_to_infrastructure_insufficient_nodes(self):
355         del self.context_cfg['nodes']['trexvnf__1']
356         with mock.patch("yardstick.ssh.SSH") as ssh:
357             ssh_mock = mock.Mock(autospec=ssh.SSH)
358             ssh_mock.execute = \
359                 mock.Mock(return_value=(1, SYS_CLASS_NET+IP_ADDR_SHOW, ""))
360             ssh.return_value = ssh_mock
361
362             self.assertRaises(IncorrectSetup,
363                               self.s.map_topology_to_infrastructure,
364                               self.context_cfg, self.topology)
365
366     def test_map_topology_to_infrastructure_config_invalid(self):
367         del self.context_cfg\
368             ['nodes']['trexvnf__1']['interfaces']['xe0']['local_mac']
369         with mock.patch("yardstick.ssh.SSH") as ssh:
370             ssh_mock = mock.Mock(autospec=ssh.SSH)
371             ssh_mock.execute = \
372                 mock.Mock(return_value=(0, SYS_CLASS_NET+IP_ADDR_SHOW, ""))
373             ssh.return_value = ssh_mock
374
375             self.assertRaises(IncorrectConfig,
376                               self.s.map_topology_to_infrastructure,
377                               self.context_cfg, self.topology)
378
379     def test__resolve_topology_invalid_config(self):
380         with mock.patch("yardstick.ssh.SSH") as ssh:
381             ssh_mock = mock.Mock(autospec=ssh.SSH)
382             ssh_mock.execute = \
383                 mock.Mock(return_value=(0, SYS_CLASS_NET+IP_ADDR_SHOW, ""))
384             ssh.return_value = ssh_mock
385
386             del self.context_cfg['nodes']
387             self.assertRaises(IncorrectConfig, self.s._resolve_topology,
388                               self.context_cfg, self.topology)
389
390             self.topology['vld'][0]['vnfd-connection-point-ref'].append(
391                 self.topology['vld'][0]['vnfd-connection-point-ref'])
392             self.assertRaises(IncorrectConfig, self.s._resolve_topology,
393                               self.context_cfg, self.topology)
394
395     def test_run(self):
396         tgen = mock.Mock(autospec=GenericTrafficGen)
397         tgen.traffic_finished = True
398         verified_dict = {"verified": True}
399         tgen.verify_traffic = lambda x: verified_dict
400         tgen.name = "tgen__1"
401         vnf = mock.Mock(autospec=GenericVNF)
402         vnf.runs_traffic = False
403         self.s.vnfs = [tgen, vnf]
404         self.s.traffic_profile = mock.Mock()
405         self.s.collector = mock.Mock(autospec=Collector)
406         self.s.collector.get_kpi = \
407             mock.Mock(return_value={tgen.name: verified_dict})
408         result = {}
409         self.s.run(result)
410         self.assertDictEqual(result, {tgen.name: verified_dict})
411
412     def test_setup(self):
413         with mock.patch("yardstick.ssh.SSH") as ssh:
414             ssh_mock = mock.Mock(autospec=ssh.SSH)
415             ssh_mock.execute = \
416                 mock.Mock(return_value=(0, SYS_CLASS_NET+IP_ADDR_SHOW, ""))
417             ssh.return_value = ssh_mock
418
419             tgen = mock.Mock(autospec=GenericTrafficGen)
420             tgen.traffic_finished = True
421             verified_dict = {"verified": True}
422             tgen.verify_traffic = lambda x: verified_dict
423             tgen.terminate = mock.Mock(return_value=True)
424             tgen.name = "tgen__1"
425             vnf = mock.Mock(autospec=GenericVNF)
426             vnf.runs_traffic = False
427             vnf.terminate = mock.Mock(return_value=True)
428             self.s.vnfs = [tgen, vnf]
429             self.s.traffic_profile = mock.Mock()
430             self.s.collector = mock.Mock(autospec=Collector)
431             self.s.collector.get_kpi = \
432                 mock.Mock(return_value={tgen.name: verified_dict})
433             self.s.map_topology_to_infrastructure = mock.Mock(return_value=0)
434             self.s.load_vnf_models = mock.Mock(return_value=self.s.vnfs)
435             self.s._fill_traffic_profile = \
436                 mock.Mock(return_value=TRAFFIC_PROFILE)
437             self.assertEqual(None, self.s.setup())
438
439     def test__get_traffic_profile(self):
440         self.scenario_cfg["traffic_profile"] = \
441             self._get_file_abspath("ipv4_throughput_vpe.yaml")
442         self.assertIsNotNone(self.s._get_traffic_profile(self.scenario_cfg,
443                                                          self.context_cfg))
444
445     def test__get_traffic_profile_exception(self):
446         self.assertRaises(IOError, self.s._get_traffic_profile,
447                           self.scenario_cfg, self.context_cfg)
448
449     def test___get_traffic_imix_exception(self):
450         self.assertEqual({}, self.s._get_traffic_imix(self.scenario_cfg))
451
452     def test__fill_traffic_profile(self):
453         with mock.patch.dict("sys.modules", STL_MOCKS):
454             self.scenario_cfg["traffic_profile"] = \
455                 self._get_file_abspath("ipv4_throughput_vpe.yaml")
456             self.scenario_cfg["traffic_options"]["flow"] = \
457                 self._get_file_abspath("ipv4_1flow_Packets_vpe.yaml")
458             self.scenario_cfg["traffic_options"]["imix"] = \
459                 self._get_file_abspath("imix_voice.yaml")
460             self.assertIsNotNone(self.s._fill_traffic_profile(self.scenario_cfg,
461                                                               self.context_cfg))
462
463     def test_teardown(self):
464         vnf = mock.Mock(autospec=GenericVNF)
465         vnf.terminate = \
466             mock.Mock(return_value=True)
467         self.s.vnfs = [vnf]
468         self.s.traffic_profile = mock.Mock()
469         self.s.collector = mock.Mock(autospec=Collector)
470         self.s.collector.stop = \
471             mock.Mock(return_value=True)
472         self.assertIsNone(self.s.teardown())