1 # Copyright (c) 2015 Intel Research and Development Ireland Ltd.
3 # Licensed under the Apache License, Version 2.0 (the "License");
4 # you may not use this file except in compliance with the License.
5 # You may obtain a copy of the License at
7 # http://www.apache.org/licenses/LICENSE-2.0
9 # Unless required by applicable law or agreed to in writing, software
10 # distributed under the License is distributed on an "AS IS" BASIS,
11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 # See the License for the specific language governing permissions and
13 # limitations under the License.
20 from experimental_framework.constants import conf_file_sections as cf
21 from experimental_framework.constants import framework_parameters as fp
24 # ------------------------------------------------------
25 # List of common variables
26 # ------------------------------------------------------
30 DEPLOYMENT_UNIT = None
37 TEMPLATE_FILE_EXTENSION = None
41 PKTGEN_DPDK_DIRECTORY = None
43 PKTGEN_COREMASK = None
44 PKTGEN_MEMCHANNEL = None
45 PKTGEN_BUS_SLOT_NIC_1 = None
46 PKTGEN_BUS_SLOT_NIC_2 = None
47 PKTGEN_NAME_NIC_1 = None
48 PKTGEN_NAME_NIC_2 = None
52 INFLUXDB_DB_NAME = None
55 # ------------------------------------------------------
56 # Initialization and Input 'heat_templates/'validation
57 # ------------------------------------------------------
61 # BASE_DIR = os.getcwd()
62 BASE_DIR = os.path.dirname(os.path.abspath(__file__))
63 BASE_DIR = BASE_DIR.replace('/experimental_framework', '')
64 BASE_DIR = InputValidation.validate_directory_exist_and_format(
65 BASE_DIR, "Error 000001")
69 init_general_vars(api)
70 if len(CONF_FILE.get_variable_list(cf.CFS_PKTGEN)) > 0:
74 def init_conf_file(api=False):
77 CONF_FILE = ConfigurationFile(cf.get_sections_api(),
78 '/tmp/apexlake/apexlake.conf')
80 CONF_FILE = ConfigurationFile(cf.get_sections(),
81 '/tmp/apexlake/apexlake.conf')
84 def init_general_vars(api=False):
85 global TEMPLATE_FILE_EXTENSION
91 TEMPLATE_FILE_EXTENSION = '.yaml'
93 # Check Section in Configuration File
95 validate_configuration_file_section(
97 "Section " + cf.CFS_GENERAL +
98 "is not present in configuration file")
100 TEMPLATE_DIR = '/tmp/apexlake/heat_templates/'
101 # if not os.path.exists(TEMPLATE_DIR):
102 # os.makedirs(TEMPLATE_DIR)
103 # cmd = "cp /tmp/apexlake/heat_templates/*.yaml {}".format(TEMPLATE_DIR)
107 # Validate template name
109 validate_configuration_file_parameter(
111 cf.CFSG_TEMPLATE_NAME,
112 "Parameter " + cf.CFSG_TEMPLATE_NAME +
113 "is not present in configuration file")
114 TEMPLATE_NAME = CONF_FILE.get_variable(cf.CFS_GENERAL,
115 cf.CFSG_TEMPLATE_NAME)
116 InputValidation.validate_file_exist(
117 TEMPLATE_DIR + TEMPLATE_NAME,
118 "The provided template file does not exist")
120 RESULT_DIR = "/tmp/apexlake/results/"
121 if not os.path.isdir(RESULT_DIR):
124 # Validate and assign Iterations
125 if cf.CFSG_ITERATIONS in CONF_FILE.get_variable_list(cf.CFS_GENERAL):
126 ITERATIONS = int(CONF_FILE.get_variable(cf.CFS_GENERAL,
134 LOG = logging.getLogger()
135 LOG.setLevel(level=logging.INFO)
136 log_formatter = logging.Formatter("%(asctime)s --- %(message)s")
137 file_handler = logging.FileHandler("{0}/{1}.log".format("./", "benchmark"))
138 file_handler.setFormatter(log_formatter)
139 file_handler.setLevel(logging.DEBUG)
140 LOG.addHandler(file_handler)
143 # ------------------------------------------------------
144 # InfluxDB conf variables
145 # ------------------------------------------------------
149 global INFLUXDB_DB_NAME
151 INFLUXDB_IP = CONF_FILE.get_variable(cf.CFS_INFLUXDB, cf.CFSI_IDB_IP)
152 INFLUXDB_PORT = CONF_FILE.get_variable(cf.CFS_INFLUXDB, cf.CFSI_IDB_PORT)
153 INFLUXDB_DB_NAME = CONF_FILE.get_variable(cf.CFS_INFLUXDB,
157 # ------------------------------------------------------
158 # Packet Generator conf variables
159 # ------------------------------------------------------
163 global PKTGEN_PROGRAM
164 global PKTGEN_COREMASK
165 global PKTGEN_MEMCHANNEL
166 global PKTGEN_BUS_SLOT_NIC_1
167 global PKTGEN_BUS_SLOT_NIC_2
168 global PKTGEN_DPDK_DIRECTORY
169 global PKTGEN_NAME_NIC_1
170 global PKTGEN_NAME_NIC_2
172 msg = "Section {} is not present in the configuration file".\
173 format(cf.CFS_PKTGEN)
174 InputValidation.validate_configuration_file_section(cf.CFS_PKTGEN, msg)
176 pktgen_var_list = CONF_FILE.get_variable_list(cf.CFS_PKTGEN)
177 PKTGEN = 'dpdk_pktgen' # default value
178 if cf.CFSP_PACKET_GENERATOR in pktgen_var_list:
179 msg = "Parameter {} is not present in section {}".format(
180 cf.CFSP_PACKET_GENERATOR, cf.CFS_PKTGEN)
181 InputValidation.validate_configuration_file_parameter(
182 cf.CFS_PKTGEN, cf.CFSP_PACKET_GENERATOR, msg)
183 PKTGEN = CONF_FILE.get_variable(
184 cf.CFS_PKTGEN, cf.CFSP_PACKET_GENERATOR)
186 if PKTGEN not in fp.get_supported_packet_generators():
187 raise ValueError('The specified packet generator is not supported '
190 # Check if the packet gen is dpdk_pktgen
191 if PKTGEN == cf.CFSP_PG_DPDK:
192 # Validation of DPDK pktgen directory
193 msg = "Parameter {} is not present in section {}".format(
194 cf.CFSP_DPDK_PKTGEN_DIRECTORY, cf.CFS_PKTGEN)
195 InputValidation.validate_configuration_file_parameter(
196 cf.CFS_PKTGEN, cf.CFSP_DPDK_PKTGEN_DIRECTORY, msg)
197 PKTGEN_DIR = CONF_FILE.get_variable(
198 cf.CFS_PKTGEN, cf.CFSP_DPDK_PKTGEN_DIRECTORY)
199 msg = "The directory {} does not exist.".format(PKTGEN_DIR)
200 PKTGEN_DIR = InputValidation.validate_directory_exist_and_format(
203 # Validation of the DPDK program name
204 msg = "Parameter {} is not present in section {}".format(
205 cf.CFSP_DPDK_PROGRAM_NAME, cf.CFS_PKTGEN)
206 InputValidation.validate_configuration_file_parameter(
207 cf.CFS_PKTGEN, cf.CFSP_DPDK_PROGRAM_NAME, msg)
208 PKTGEN_PROGRAM = CONF_FILE.get_variable(
209 cf.CFS_PKTGEN, cf.CFSP_DPDK_PROGRAM_NAME)
211 # Validation of the DPDK Coremask parameter
212 msg = "Parameter {} is not present in section {}".format(
213 cf.CFSP_DPDK_COREMASK, cf.CFS_PKTGEN)
214 InputValidation.validate_configuration_file_parameter(
215 cf.CFS_PKTGEN, cf.CFSP_DPDK_COREMASK, msg)
216 PKTGEN_COREMASK = CONF_FILE.get_variable(
217 cf.CFS_PKTGEN, cf.CFSP_DPDK_COREMASK)
219 # Validation of the DPDK Memory Channel parameter
220 msg = "Parameter {} is not present in section {}".format(
221 cf.CFSP_DPDK_MEMORY_CHANNEL, cf.CFS_PKTGEN)
222 InputValidation.validate_configuration_file_parameter(
223 cf.CFS_PKTGEN, cf.CFSP_DPDK_MEMORY_CHANNEL, msg)
224 PKTGEN_MEMCHANNEL = CONF_FILE.get_variable(
225 cf.CFS_PKTGEN, cf.CFSP_DPDK_MEMORY_CHANNEL)
227 # Validation of the DPDK Bus Slot 1
228 msg = "Parameter {} is not present in section {}".format(
229 cf.CFSP_DPDK_BUS_SLOT_NIC_1, cf.CFS_PKTGEN)
230 InputValidation.validate_configuration_file_parameter(
231 cf.CFS_PKTGEN, cf.CFSP_DPDK_BUS_SLOT_NIC_1, msg)
232 PKTGEN_BUS_SLOT_NIC_1 = CONF_FILE.get_variable(
233 cf.CFS_PKTGEN, cf.CFSP_DPDK_BUS_SLOT_NIC_1)
235 # Validation of the DPDK Bus Slot 2
236 msg = "Parameter {} is not present in section {}".format(
237 cf.CFSP_DPDK_BUS_SLOT_NIC_2, cf.CFS_PKTGEN)
238 InputValidation.validate_configuration_file_parameter(
239 cf.CFS_PKTGEN, cf.CFSP_DPDK_BUS_SLOT_NIC_2, msg)
240 PKTGEN_BUS_SLOT_NIC_2 = CONF_FILE.get_variable(
241 cf.CFS_PKTGEN, cf.CFSP_DPDK_BUS_SLOT_NIC_2)
243 # Validation of the DPDK NIC 1
244 msg = "Parameter {} is not present in section {}".format(
245 cf.CFSP_DPDK_NAME_IF_1, cf.CFS_PKTGEN)
246 InputValidation.validate_configuration_file_parameter(
247 cf.CFS_PKTGEN, cf.CFSP_DPDK_NAME_IF_1, msg)
248 PKTGEN_NAME_NIC_1 = CONF_FILE.get_variable(
249 cf.CFS_PKTGEN, cf.CFSP_DPDK_NAME_IF_1)
251 # Validation of the DPDK NIC 2
252 msg = "Parameter {} is not present in section {}".format(
253 cf.CFSP_DPDK_NAME_IF_2, cf.CFS_PKTGEN)
254 InputValidation.validate_configuration_file_parameter(
255 cf.CFS_PKTGEN, cf.CFSP_DPDK_NAME_IF_2, msg)
256 PKTGEN_NAME_NIC_2 = CONF_FILE.get_variable(
257 cf.CFS_PKTGEN, cf.CFSP_DPDK_NAME_IF_2)
259 # Validation of DPDK directory parameter
260 msg = "Parameter {} is not present in section {}".format(
261 cf.CFSP_DPDK_DPDK_DIRECTORY, cf.CFS_PKTGEN)
262 InputValidation.validate_configuration_file_parameter(
263 cf.CFS_PKTGEN, cf.CFSP_DPDK_DPDK_DIRECTORY, msg)
264 PKTGEN_DPDK_DIRECTORY = CONF_FILE.get_variable(
265 cf.CFS_PKTGEN, cf.CFSP_DPDK_DPDK_DIRECTORY)
266 msg = "Directory {} does not exist".format(
267 cf.CFSP_DPDK_DPDK_DIRECTORY)
268 PKTGEN_DPDK_DIRECTORY = InputValidation.\
269 validate_directory_exist_and_format(PKTGEN_DPDK_DIRECTORY, msg)
272 # ------------------------------------------------------
273 # Configuration file access
274 # ------------------------------------------------------
276 class ConfigurationFile:
278 Used to extract data from the configuration file
281 def __init__(self, sections, config_file='conf.cfg'):
283 Reads configuration file sections
285 :param sections: list of strings representing the sections to be
287 :param config_file: name of the configuration file (string)
290 InputValidation.validate_string(
291 config_file, "The configuration file name must be a string")
292 # config_file = BASE_DIR + config_file
293 InputValidation.validate_file_exist(
294 config_file, 'The provided configuration file does not exist')
295 self.config = ConfigParser.ConfigParser()
296 self.config.read(config_file)
297 for section in sections:
299 self, section, ConfigurationFile.
300 _config_section_map(section, self.config))
303 def _config_section_map(section, config_file):
305 Returns a dictionary with the configuration values for the specific
308 :param section: section to be loaded (string)
309 :param config_file: name of the configuration file (string)
313 options = config_file.options(section)
314 for option in options:
315 dict1[option] = config_file.get(section, option)
318 def get_variable(self, section, variable_name):
320 Returns the value correspondent to a variable
322 :param section: section to be loaded (string)
323 :param variable_name: name of the variable (string)
326 message = "The variable name must be a string"
327 InputValidation.validate_string(variable_name, message)
328 if variable_name in self.get_variable_list(section):
329 sect = getattr(self, section)
330 return sect[variable_name]
332 exc_msg = 'Parameter {} is not in the {} section of the ' \
333 'conf file'.format(variable_name, section)
334 raise ValueError(exc_msg)
336 def get_variable_list(self, section):
338 Returns the list of the available variables in a section
339 :param section: section to be loaded (string)
343 return getattr(self, section)
345 msg = 'Section {} not found in the configuration file'.\
347 raise ValueError(msg)
350 # ------------------------------------------------------
351 # Get OpenStack Credentials
352 # ------------------------------------------------------
353 def get_credentials():
355 Returns the credentials for OpenStack access from the configuration file
359 credentials[cf.CFSO_IP_CONTROLLER] = CONF_FILE.get_variable(
360 cf.CFS_OPENSTACK, cf.CFSO_IP_CONTROLLER)
361 credentials[cf.CFSO_HEAT_URL] = CONF_FILE.get_variable(
362 cf.CFS_OPENSTACK, cf.CFSO_HEAT_URL)
363 credentials[cf.CFSO_USER] = CONF_FILE.get_variable(
364 cf.CFS_OPENSTACK, cf.CFSO_USER)
365 credentials[cf.CFSO_PASSWORD] = CONF_FILE.get_variable(
366 cf.CFS_OPENSTACK, cf.CFSO_PASSWORD)
367 credentials[cf.CFSO_AUTH_URI] = CONF_FILE.get_variable(
368 cf.CFS_OPENSTACK, cf.CFSO_AUTH_URI)
369 credentials[cf.CFSO_PROJECT] = CONF_FILE.get_variable(
370 cf.CFS_OPENSTACK, cf.CFSO_PROJECT)
374 # ------------------------------------------------------
376 # ------------------------------------------------------
378 def get_heat_template_params():
380 Returns the list of deployment parameters from the configuration file
381 for the heat template
385 heat_parameters_list = CONF_FILE.get_variable_list(
386 cf.CFS_DEPLOYMENT_PARAMETERS)
387 testcase_parameters = dict()
388 for param in heat_parameters_list:
389 testcase_parameters[param] = CONF_FILE.get_variable(
390 cf.CFS_DEPLOYMENT_PARAMETERS, param)
391 return testcase_parameters
394 def get_testcase_params():
396 Returns the list of testcase parameters from the configuration file
400 testcase_parameters = dict()
401 parameters = CONF_FILE.get_variable_list(cf.CFS_TESTCASE_PARAMETERS)
402 for param in parameters:
403 testcase_parameters[param] = CONF_FILE.get_variable(
404 cf.CFS_TESTCASE_PARAMETERS, param)
405 return testcase_parameters
408 def get_file_first_line(file_name):
410 Returns the first line of a file
412 :param file_name: name of the file to be read (str)
415 message = "The name of the file must be a string"
416 InputValidation.validate_string(file_name, message)
417 message = 'The file {} does not exist'.format(file_name)
418 InputValidation.validate_file_exist(file_name, message)
419 res = open(file_name, 'r')
420 return res.readline()
423 def replace_in_file(file, text_to_search, text_to_replace):
425 Replaces a string within a file
427 :param file: name of the file (str)
428 :param text_to_search: text to be replaced
429 :param text_to_replace: new text that will replace the previous
432 message = 'The text to be replaced in the file must be a string'
433 InputValidation.validate_string(text_to_search, message)
434 message = 'The text to replace in the file must be a string'
435 InputValidation.validate_string(text_to_replace, message)
436 message = "The name of the file must be a string"
437 InputValidation.validate_string(file, message)
438 message = "The file does not exist"
439 InputValidation.validate_file_exist(file, message)
440 for line in fileinput.input(file, inplace=True):
441 print(line.replace(text_to_search, text_to_replace).rstrip())
444 # ------------------------------------------------------
446 # ------------------------------------------------------
447 def run_command(command):
448 LOG.info("Running command: {}".format(command))
449 return os.system(command)
452 def push_data_influxdb(data):
455 db_name = INFLUXDB_DB_NAME
456 command = "curl -i -XPOST 'http://{}:{}/write?db={}' " \
457 "--data-binary {}".format(ip, port, db_name, data)
461 # ------------------------------------------------------
462 # Expose variables to other modules
463 # ------------------------------------------------------
469 def get_template_dir():
473 def get_result_dir():
477 def get_dpdk_pktgen_vars():
478 if not (PKTGEN == 'dpdk_pktgen'):
481 ret_val[cf.CFSP_DPDK_PKTGEN_DIRECTORY] = PKTGEN_DIR
482 ret_val[cf.CFSP_DPDK_DPDK_DIRECTORY] = PKTGEN_DPDK_DIRECTORY
483 ret_val[cf.CFSP_DPDK_PROGRAM_NAME] = PKTGEN_PROGRAM
484 ret_val[cf.CFSP_DPDK_COREMASK] = PKTGEN_COREMASK
485 ret_val[cf.CFSP_DPDK_MEMORY_CHANNEL] = PKTGEN_MEMCHANNEL
486 ret_val[cf.CFSP_DPDK_BUS_SLOT_NIC_1] = PKTGEN_BUS_SLOT_NIC_1
487 ret_val[cf.CFSP_DPDK_BUS_SLOT_NIC_2] = PKTGEN_BUS_SLOT_NIC_2
488 ret_val[cf.CFSP_DPDK_NAME_IF_1] = PKTGEN_NAME_NIC_1
489 ret_val[cf.CFSP_DPDK_NAME_IF_2] = PKTGEN_NAME_NIC_2
493 # ------------------------------------------------------
494 # Configuration Variables from Config File
495 # ------------------------------------------------------
496 def get_deployment_configuration_variables_from_conf_file():
499 all_variables = CONF_FILE.get_variable_list(cf.CFS_EXPERIMENT_VNF)
500 for var in all_variables:
501 v = CONF_FILE.get_variable(cf.CFS_EXPERIMENT_VNF, var)
502 type = re.findall(r'@\w*', v)
503 values = re.findall(r'\"(.+?)\"', v)
504 variables[var] = values
506 types[var] = type[0][1:]
508 LOG.debug("No type has been specified for variable " + var)
512 # ------------------------------------------------------
513 # benchmarks from Config File
514 # ------------------------------------------------------
515 def get_benchmarks_from_conf_file():
516 requested_benchmarks = list()
518 CONF_FILE.get_variable(cf.CFS_GENERAL, cf.CFSG_BENCHMARKS).split(', ')
519 for benchmark in benchmarks:
520 requested_benchmarks.append(benchmark)
521 return requested_benchmarks
524 class InputValidation(object):
527 def validate_string(param, message):
528 if not isinstance(param, str):
529 raise ValueError(message)
533 def validate_integer(param, message):
534 if not isinstance(param, int):
535 raise ValueError(message)
539 def validate_dictionary(param, message):
540 if not isinstance(param, dict):
541 raise ValueError(message)
545 def validate_file_exist(file_name, message):
546 if not os.path.isfile(file_name):
547 raise ValueError(message + ' ' + file_name)
551 def validate_directory_exist_and_format(directory, message):
552 if not os.path.isdir(directory):
553 raise ValueError(message)
554 if not directory.endswith('/'):
555 return directory + '/'
559 def validate_configuration_file_parameter(section, parameter, message):
560 params = CONF_FILE.get_variable_list(section)
561 if parameter not in params:
562 raise ValueError(message)
566 def validate_configuration_file_section(section, message):
567 if section not in cf.get_sections():
568 raise ValueError(message)
572 def validate_boolean(boolean, message):
573 if isinstance(boolean, bool):
575 if isinstance(boolean, str):
576 if boolean == 'True':
578 if boolean == 'False':
580 raise ValueError(message)
583 def validate_os_credentials(credentials):
584 if not isinstance(credentials, dict):
586 'The provided openstack_credentials '
587 'variable must be in dictionary format')
589 credential_keys = ['ip_controller', 'heat_url', 'user', 'password',
590 'auth_uri', 'project']
593 for credential_key in credential_keys
594 if credential_key not in credentials.keys()
596 if len(missing) == 0:
598 msg = 'OpenStack Credentials Error! ' \
599 'The following parameters are missing: {}'.\
600 format(", ".join(missing))
601 raise ValueError(msg)