1 # Copyright 2016-2017 Intel Corporation.
3 # Licensed under the Apache License, Version 2.0 (the "License");
4 # you may not use this file except in compliance with the License.
5 # You may obtain a copy of the License at
7 # http://www.apache.org/licenses/LICENSE-2.0
9 # Unless required by applicable law or agreed to in writing, software
10 # distributed under the License is distributed on an "AS IS" BASIS,
11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 # See the License for the specific language governing permissions and
13 # limitations under the License.
15 """Various helper functions
23 from conf import settings as S
30 # pylint: disable=too-many-branches
31 def settings_update_paths():
32 """ Configure paths to OVS, DPDK and QEMU sources and binaries based on
33 selected vswitch type and src/binary switch. Data are taken from
34 PATHS dictionary and after their processing they are stored inside TOOLS.
35 PATHS dictionary has specific section for 'vswitch', 'qemu' and 'dpdk'
36 Following processing is done for every item:
37 item 'type' - string, which defines the type of paths ('src' or 'bin') to be selected
39 'src' means, that VSPERF will use OVS, DPDK or QEMU built from sources
40 e.g. by execution of systems/build_base_machine.sh script during VSPERF
42 'bin' means, that VSPERF will use OVS, DPDK or QEMU binaries installed
43 in the OS, e.g. via OS specific packaging system
44 item 'path' - string with valid path; Its content is checked for existence, prefixed
45 with section name and stored into TOOLS for later use
46 e.g. TOOLS['dpdk_src'] or TOOLS['vswitch_src']
47 item 'modules' - list of strings; Every value from given list is checked for '.ko'
48 suffix. In case it matches and it is not an absolute path to the module, then
49 module name is prefixed with 'path' defined for the same section
50 e.g. TOOLS['vswitch_modules'] = [
51 '/tmp/vsperf/src_vanilla/ovs/ovs/datapath/linux/openvswitch.ko']
52 all other items - string - if given string is a relative path and item 'path'
53 is defined for a given section, then item content will be prefixed with
54 content of the 'path'. Otherwise tool name will be searched within
55 standard system directories. Also any OS filename wildcards will be
56 expanded to the real path. At the end of processing, every absolute
57 path is checked for its existence. In case that temporary path (i.e. path
58 with '_tmp' suffix) doesn't exist, then log will be written and vsperf will
59 continue. If any other path will not exist, then vsperf execution will
60 be terminated with runtime error.
62 Note: In case that 'bin' type is set for DPDK, then TOOLS['dpdk_src'] will be set to
63 the value of PATHS['dpdk']['src']['path']. The reason is, that VSPERF uses downloaded
64 DPDK sources to copy DPDK and testpmd into the GUEST, where testpmd is built. In case,
65 that DPDK sources are not available, then vsperf will continue with test execution,
66 but testpmd can't be used as a guest loopback. This is useful in case, that other guest
67 loopback applications (e.g. buildin) are used by CI jobs, etc.
69 # set dpdk and ovs paths according to VNF, VSWITCH and TRAFFICGEN selection
71 if S.getValue("mode") != 'trafficgen':
72 # VSWITCH & (probably) VNF are needed
73 vswitch_type = S.getValue('PATHS')['vswitch'][S.getValue('VSWITCH')]['type']
74 paths['vswitch'] = S.getValue('PATHS')['vswitch'][S.getValue('VSWITCH')][vswitch_type]
75 paths['dpdk'] = S.getValue('PATHS')['dpdk'][S.getValue('PATHS')['dpdk']['type']]
76 paths['qemu'] = S.getValue('PATHS')['qemu'][S.getValue('PATHS')['qemu']['type']]
78 paths['paths']['ovs_var_tmp'] = S.getValue('PATHS')['vswitch']['ovs_var_tmp']
79 paths['paths']['ovs_etc_tmp'] = S.getValue('PATHS')['vswitch']['ovs_etc_tmp']
81 if S.getValue("mode") != 'trafficgen-off':
82 # TRAFFCIGEN is required
83 if S.getValue('TRAFFICGEN') in S.getValue('PATHS')['trafficgen']:
84 tmp_trafficgen = S.getValue('PATHS')['trafficgen'][S.getValue('TRAFFICGEN')]
85 paths['trafficgen'] = tmp_trafficgen[tmp_trafficgen['type']]
88 # pylint: disable=too-many-nested-blocks
89 for path_class in paths:
90 for tool in paths[path_class]:
91 tmp_tool = paths[path_class][tool]
93 # store valid path of given class into tools dict
95 if os.path.isdir(tmp_tool):
96 tools['{}_src'.format(path_class)] = tmp_tool
99 raise RuntimeError('Path {} does not exist.'.format(tmp_tool))
101 # store list of modules of given class into tools dict
102 if tool == 'modules':
104 for module in tmp_tool:
105 # add path to the .ko modules and check it for existence
106 if module.endswith('.ko') and not os.path.isabs(module):
107 module = os.path.join(paths[path_class]['path'], module)
108 if not os.path.exists(module):
109 raise RuntimeError('Cannot locate modlue {}'.format(module))
111 tmp_modules.append(module)
113 tools['{}_modules'.format(path_class)] = tmp_modules
116 # if path to the tool is relative, then 'path' will be prefixed
117 # in case that 'path' is not defined, then tool will be searched
118 # within standard system paths
119 if not os.path.isabs(tmp_tool):
120 if 'path' in paths[path_class]:
121 tmp_tool = os.path.join(paths[path_class]['path'], tmp_tool)
122 elif shutil.which(tmp_tool):
123 tmp_tool = shutil.which(tmp_tool)
125 raise RuntimeError('Cannot locate tool {}'.format(tmp_tool))
127 # expand OS wildcards in paths if needed
128 if glob.has_magic(tmp_tool):
129 tmp_glob = glob.glob(tmp_tool)
131 raise RuntimeError('Path to the {} is not valid: {}.'.format(tool, tmp_tool))
132 elif len(tmp_glob) > 1:
133 raise RuntimeError('Path to the {} is ambiguous {}'.format(tool, tmp_glob))
134 elif len(tmp_glob) == 1:
135 tmp_tool = tmp_glob[0]
136 elif not os.path.exists(tmp_tool):
137 if tool.endswith('_tmp'):
138 logging.getLogger().debug('Temporary path to the %s does not '
139 'exist: %s', tool, tmp_tool)
141 raise RuntimeError('Path to the {} is not valid: {}'.format(tool, tmp_tool))
143 tools[tool] = tmp_tool
145 # ensure, that dpkg_src for bin will be set to downloaded DPDK sources, so it can
146 # be copied to the guest share dir and used by GUEST to build and run testpmd
147 # Validity of the path is not checked by purpose, so user can use VSPERF without
148 # downloading DPDK sources. In that case guest loopback can't be set to 'testpmd'
149 if S.getValue('PATHS')['dpdk']['type'] == 'bin':
150 tools['dpdk_src'] = S.getValue('PATHS')['dpdk']['src']['path']
152 S.setValue('TOOLS', tools)
154 def check_traffic(traffic):
155 """Check traffic definition and correct it if possible.
157 # check if requested networking layers make sense
158 if traffic['l4']['enabled']:
159 if not traffic['l3']['enabled']:
160 raise RuntimeError('TRAFFIC misconfiguration: l3 must be enabled '
163 # check if multistream configuration makes sense
164 if traffic['multistream']:
165 if traffic['stream_type'] == 'L3':
166 if not traffic['l3']['enabled']:
167 raise RuntimeError('TRAFFIC misconfiguration: l3 must be '
168 'enabled if l3 streams are requested.')
169 if traffic['stream_type'] == 'L4':
170 if not traffic['l4']['enabled']:
171 raise RuntimeError('TRAFFIC misconfiguration: l4 must be '
172 'enabled if l4 streams are requested.')
174 # in case of UDP ports we have only 65536 (0-65535) unique options
175 if traffic['multistream'] > MAX_L4_FLOWS:
176 logging.getLogger().warning(
177 'Requested amount of L4 flows %s is bigger than number of '
178 'transport protocol ports. It was set to %s.',
179 traffic['multistream'], MAX_L4_FLOWS)
180 traffic['multistream'] = MAX_L4_FLOWS
184 def filter_output(output, regex):
185 """Filter output by defined regex. Output can be either string, list or tuple.
186 Every string is split into list line by line. After that regex is applied
187 to filter only matching lines, which are returned back.
189 :returns: list of matching records
192 if isinstance(output, str):
193 for line in output.splitlines():
194 result += re.findall(regex, line)
195 elif isinstance(output, (list, tuple)):
197 result.append(filter_output(item, regex))
199 raise RuntimeError('Only strings and lists are supported by filter_output(), '
200 'but output has type {}'.format(type(output)))
203 def format_description(desc, length):
204 """ Split description into multiple lines based on given line length.
206 :param desc: A string with testcase description
207 :param length: A maximum line length
209 # split description to multiple lines
214 if len(line) + len(word) < length:
215 line += '{} '.format(word)
217 output.append(line.strip())
218 line = '{} '.format(word)
220 output.append(line.strip())