1 # Copyright (c) 2015 Intel Research and Development Ireland Ltd.
3 # Licensed under the Apache License, Version 2.0 (the "License");
4 # you may not use this file except in compliance with the License.
5 # You may obtain a copy of the License at
7 # http://www.apache.org/licenses/LICENSE-2.0
9 # Unless required by applicable law or agreed to in writing, software
10 # distributed under the License is distributed on an "AS IS" BASIS,
11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 # See the License for the specific language governing permissions and
13 # limitations under the License.
20 from experimental_framework.constants import conf_file_sections as cf
21 from experimental_framework.constants import framework_parameters as fp
24 # ------------------------------------------------------
25 # List of common variables
26 # ------------------------------------------------------
30 DEPLOYMENT_UNIT = None
37 TEMPLATE_FILE_EXTENSION = None
41 PKTGEN_DPDK_DIRECTORY = None
43 PKTGEN_COREMASK = None
44 PKTGEN_MEMCHANNEL = None
45 PKTGEN_BUS_SLOT_NIC_1 = None
46 PKTGEN_BUS_SLOT_NIC_2 = None
47 PKTGEN_NAME_NIC_1 = None
48 PKTGEN_NAME_NIC_2 = None
53 INFLUXDB_DB_NAME = None
56 # ------------------------------------------------------
57 # Initialization and Input 'heat_templates/'validation
58 # ------------------------------------------------------
62 # BASE_DIR = os.getcwd()
63 BASE_DIR = os.path.dirname(os.path.abspath(__file__))
64 BASE_DIR = BASE_DIR.replace('/experimental_framework', '')
65 BASE_DIR = InputValidation.validate_directory_exist_and_format(
66 BASE_DIR, "Error 000001")
70 init_general_vars(api)
71 if len(CONF_FILE.get_variable_list(cf.CFS_PKTGEN)) > 0:
75 def init_conf_file(api=False):
78 CONF_FILE = ConfigurationFile(cf.get_sections_api(),
79 '/etc/apexlake/apexlake.conf')
81 CONF_FILE = ConfigurationFile(cf.get_sections(),
82 '/etc/apexlake/apexlake.conf')
85 def init_general_vars(api=False):
86 global TEMPLATE_FILE_EXTENSION
92 TEMPLATE_FILE_EXTENSION = '.yaml'
94 # Check Section in Configuration File
96 validate_configuration_file_section(
98 "Section " + cf.CFS_GENERAL +
99 "is not present in configuration file")
101 TEMPLATE_DIR = '/tmp/apexlake/heat_templates/'
102 if not os.path.exists(TEMPLATE_DIR):
103 os.makedirs(TEMPLATE_DIR)
104 cmd = "cp /etc/apexlake/heat_templates/*.yaml {}".format(TEMPLATE_DIR)
108 # Validate template name
110 validate_configuration_file_parameter(
112 cf.CFSG_TEMPLATE_NAME,
113 "Parameter " + cf.CFSG_TEMPLATE_NAME +
114 "is not present in configuration file")
115 TEMPLATE_NAME = CONF_FILE.get_variable(cf.CFS_GENERAL,
116 cf.CFSG_TEMPLATE_NAME)
117 InputValidation.validate_file_exist(
118 TEMPLATE_DIR + TEMPLATE_NAME,
119 "The provided template file does not exist")
121 RESULT_DIR = "/tmp/apexlake/results/"
123 # Validate and assign Iterations
124 if cf.CFSG_ITERATIONS in CONF_FILE.get_variable_list(cf.CFS_GENERAL):
125 ITERATIONS = int(CONF_FILE.get_variable(cf.CFS_GENERAL,
133 LOG = logging.getLogger()
134 LOG.setLevel(level=logging.INFO)
135 log_formatter = logging.Formatter("%(asctime)s --- %(message)s")
136 file_handler = logging.FileHandler("{0}/{1}.log".format("./", "benchmark"))
137 file_handler.setFormatter(log_formatter)
138 file_handler.setLevel(logging.DEBUG)
139 LOG.addHandler(file_handler)
142 # ------------------------------------------------------
143 # InfluxDB conf variables
144 # ------------------------------------------------------
148 global INFLUXDB_DB_NAME
150 INFLUXDB_IP = CONF_FILE.get_variable(cf.CFS_INFLUXDB, cf.CFSI_IDB_IP)
151 INFLUXDB_PORT = CONF_FILE.get_variable(cf.CFS_INFLUXDB, cf.CFSI_IDB_PORT)
152 INFLUXDB_DB_NAME = CONF_FILE.get_variable(cf.CFS_INFLUXDB,
156 # ------------------------------------------------------
157 # Packet Generator conf variables
158 # ------------------------------------------------------
162 global PKTGEN_PROGRAM
163 global PKTGEN_COREMASK
164 global PKTGEN_MEMCHANNEL
165 global PKTGEN_BUS_SLOT_NIC_1
166 global PKTGEN_BUS_SLOT_NIC_2
167 global PKTGEN_DPDK_DIRECTORY
168 global PKTGEN_NAME_NIC_1
169 global PKTGEN_NAME_NIC_2
171 msg = "Section {} is not present in the configuration file".\
172 format(cf.CFS_PKTGEN)
173 InputValidation.validate_configuration_file_section(cf.CFS_PKTGEN, msg)
175 pktgen_var_list = CONF_FILE.get_variable_list(cf.CFS_PKTGEN)
176 PKTGEN = 'dpdk_pktgen' # default value
177 if cf.CFSP_PACKET_GENERATOR in pktgen_var_list:
178 msg = "Parameter {} is not present in section {}".format(
179 cf.CFSP_PACKET_GENERATOR, cf.CFS_PKTGEN)
180 InputValidation.validate_configuration_file_parameter(
181 cf.CFS_PKTGEN, cf.CFSP_PACKET_GENERATOR, msg)
182 PKTGEN = CONF_FILE.get_variable(
183 cf.CFS_PKTGEN, cf.CFSP_PACKET_GENERATOR)
185 if PKTGEN not in fp.get_supported_packet_generators():
186 raise ValueError('The specified packet generator is not supported '
189 # Check if the packet gen is dpdk_pktgen
190 if PKTGEN == cf.CFSP_PG_DPDK:
191 # Validation of DPDK pktgen directory
192 msg = "Parameter {} is not present in section {}".format(
193 cf.CFSP_DPDK_PKTGEN_DIRECTORY, cf.CFS_PKTGEN)
194 InputValidation.validate_configuration_file_parameter(
195 cf.CFS_PKTGEN, cf.CFSP_DPDK_PKTGEN_DIRECTORY, msg)
196 PKTGEN_DIR = CONF_FILE.get_variable(
197 cf.CFS_PKTGEN, cf.CFSP_DPDK_PKTGEN_DIRECTORY)
198 msg = "The directory {} does not exist.".format(PKTGEN_DIR)
199 PKTGEN_DIR = InputValidation.validate_directory_exist_and_format(
202 # Validation of the DPDK program name
203 msg = "Parameter {} is not present in section {}".format(
204 cf.CFSP_DPDK_PROGRAM_NAME, cf.CFS_PKTGEN)
205 InputValidation.validate_configuration_file_parameter(
206 cf.CFS_PKTGEN, cf.CFSP_DPDK_PROGRAM_NAME, msg)
207 PKTGEN_PROGRAM = CONF_FILE.get_variable(
208 cf.CFS_PKTGEN, cf.CFSP_DPDK_PROGRAM_NAME)
210 # Validation of the DPDK Coremask parameter
211 msg = "Parameter {} is not present in section {}".format(
212 cf.CFSP_DPDK_COREMASK, cf.CFS_PKTGEN)
213 InputValidation.validate_configuration_file_parameter(
214 cf.CFS_PKTGEN, cf.CFSP_DPDK_COREMASK, msg)
215 PKTGEN_COREMASK = CONF_FILE.get_variable(
216 cf.CFS_PKTGEN, cf.CFSP_DPDK_COREMASK)
218 # Validation of the DPDK Memory Channel parameter
219 msg = "Parameter {} is not present in section {}".format(
220 cf.CFSP_DPDK_MEMORY_CHANNEL, cf.CFS_PKTGEN)
221 InputValidation.validate_configuration_file_parameter(
222 cf.CFS_PKTGEN, cf.CFSP_DPDK_MEMORY_CHANNEL, msg)
223 PKTGEN_MEMCHANNEL = CONF_FILE.get_variable(
224 cf.CFS_PKTGEN, cf.CFSP_DPDK_MEMORY_CHANNEL)
226 # Validation of the DPDK Bus Slot 1
227 msg = "Parameter {} is not present in section {}".format(
228 cf.CFSP_DPDK_BUS_SLOT_NIC_1, cf.CFS_PKTGEN)
229 InputValidation.validate_configuration_file_parameter(
230 cf.CFS_PKTGEN, cf.CFSP_DPDK_BUS_SLOT_NIC_1, msg)
231 PKTGEN_BUS_SLOT_NIC_1 = CONF_FILE.get_variable(
232 cf.CFS_PKTGEN, cf.CFSP_DPDK_BUS_SLOT_NIC_1)
234 # Validation of the DPDK Bus Slot 2
235 msg = "Parameter {} is not present in section {}".format(
236 cf.CFSP_DPDK_BUS_SLOT_NIC_2, cf.CFS_PKTGEN)
237 InputValidation.validate_configuration_file_parameter(
238 cf.CFS_PKTGEN, cf.CFSP_DPDK_BUS_SLOT_NIC_2, msg)
239 PKTGEN_BUS_SLOT_NIC_2 = CONF_FILE.get_variable(
240 cf.CFS_PKTGEN, cf.CFSP_DPDK_BUS_SLOT_NIC_2)
242 # Validation of the DPDK NIC 1
243 msg = "Parameter {} is not present in section {}".format(
244 cf.CFSP_DPDK_NAME_IF_1, cf.CFS_PKTGEN)
245 InputValidation.validate_configuration_file_parameter(
246 cf.CFS_PKTGEN, cf.CFSP_DPDK_NAME_IF_1, msg)
247 PKTGEN_NAME_NIC_1 = CONF_FILE.get_variable(
248 cf.CFS_PKTGEN, cf.CFSP_DPDK_NAME_IF_1)
250 # Validation of the DPDK NIC 2
251 msg = "Parameter {} is not present in section {}".format(
252 cf.CFSP_DPDK_NAME_IF_2, cf.CFS_PKTGEN)
253 InputValidation.validate_configuration_file_parameter(
254 cf.CFS_PKTGEN, cf.CFSP_DPDK_NAME_IF_2, msg)
255 PKTGEN_NAME_NIC_2 = CONF_FILE.get_variable(
256 cf.CFS_PKTGEN, cf.CFSP_DPDK_NAME_IF_2)
258 # Validation of DPDK directory parameter
259 msg = "Parameter {} is not present in section {}".format(
260 cf.CFSP_DPDK_DPDK_DIRECTORY, cf.CFS_PKTGEN)
261 InputValidation.validate_configuration_file_parameter(
262 cf.CFS_PKTGEN, cf.CFSP_DPDK_DPDK_DIRECTORY, msg)
263 PKTGEN_DPDK_DIRECTORY = CONF_FILE.get_variable(
264 cf.CFS_PKTGEN, cf.CFSP_DPDK_DPDK_DIRECTORY)
265 msg = "Directory {} does not exist".format(
266 cf.CFSP_DPDK_DPDK_DIRECTORY)
267 PKTGEN_DPDK_DIRECTORY = InputValidation.\
268 validate_directory_exist_and_format(PKTGEN_DPDK_DIRECTORY, msg)
271 # ------------------------------------------------------
272 # Configuration file access
273 # ------------------------------------------------------
275 class ConfigurationFile:
277 Used to extract data from the configuration file
280 def __init__(self, sections, config_file='conf.cfg'):
282 Reads configuration file sections
284 :param sections: list of strings representing the sections to be
286 :param config_file: name of the configuration file (string)
289 InputValidation.validate_string(
290 config_file, "The configuration file name must be a string")
291 # config_file = BASE_DIR + config_file
292 InputValidation.validate_file_exist(
293 config_file, 'The provided configuration file does not exist')
294 self.config = ConfigParser.ConfigParser()
295 self.config.read(config_file)
296 for section in sections:
298 self, section, ConfigurationFile.
299 _config_section_map(section, self.config))
302 def _config_section_map(section, config_file):
304 Returns a dictionary with the configuration values for the specific
307 :param section: section to be loaded (string)
308 :param config_file: name of the configuration file (string)
312 options = config_file.options(section)
313 for option in options:
314 dict1[option] = config_file.get(section, option)
317 def get_variable(self, section, variable_name):
319 Returns the value correspondent to a variable
321 :param section: section to be loaded (string)
322 :param variable_name: name of the variable (string)
325 message = "The variable name must be a string"
326 InputValidation.validate_string(variable_name, message)
327 if variable_name in self.get_variable_list(section):
328 sect = getattr(self, section)
329 return sect[variable_name]
331 exc_msg = 'Parameter {} is not in the {} section of the ' \
332 'conf file'.format(variable_name, section)
333 raise ValueError(exc_msg)
335 def get_variable_list(self, section):
337 Returns the list of the available variables in a section
338 :param section: section to be loaded (string)
342 return getattr(self, section)
344 msg = 'Section {} not found in the configuration file'.\
346 raise ValueError(msg)
349 # ------------------------------------------------------
350 # Get OpenStack Credentials
351 # ------------------------------------------------------
352 def get_credentials():
354 Returns the credentials for OpenStack access from the configuration file
358 credentials[cf.CFSO_IP_CONTROLLER] = CONF_FILE.get_variable(
359 cf.CFS_OPENSTACK, cf.CFSO_IP_CONTROLLER)
360 credentials[cf.CFSO_HEAT_URL] = CONF_FILE.get_variable(
361 cf.CFS_OPENSTACK, cf.CFSO_HEAT_URL)
362 credentials[cf.CFSO_USER] = CONF_FILE.get_variable(
363 cf.CFS_OPENSTACK, cf.CFSO_USER)
364 credentials[cf.CFSO_PASSWORD] = CONF_FILE.get_variable(
365 cf.CFS_OPENSTACK, cf.CFSO_PASSWORD)
366 credentials[cf.CFSO_AUTH_URI] = CONF_FILE.get_variable(
367 cf.CFS_OPENSTACK, cf.CFSO_AUTH_URI)
368 credentials[cf.CFSO_PROJECT] = CONF_FILE.get_variable(
369 cf.CFS_OPENSTACK, cf.CFSO_PROJECT)
373 # ------------------------------------------------------
375 # ------------------------------------------------------
377 def get_heat_template_params():
379 Returns the list of deployment parameters from the configuration file
380 for the heat template
384 heat_parameters_list = CONF_FILE.get_variable_list(
385 cf.CFS_DEPLOYMENT_PARAMETERS)
386 testcase_parameters = dict()
387 for param in heat_parameters_list:
388 testcase_parameters[param] = CONF_FILE.get_variable(
389 cf.CFS_DEPLOYMENT_PARAMETERS, param)
390 return testcase_parameters
393 def get_testcase_params():
395 Returns the list of testcase parameters from the configuration file
399 testcase_parameters = dict()
400 parameters = CONF_FILE.get_variable_list(cf.CFS_TESTCASE_PARAMETERS)
401 for param in parameters:
402 testcase_parameters[param] = CONF_FILE.get_variable(
403 cf.CFS_TESTCASE_PARAMETERS, param)
404 return testcase_parameters
407 def get_file_first_line(file_name):
409 Returns the first line of a file
411 :param file_name: name of the file to be read (str)
414 message = "The name of the file must be a string"
415 InputValidation.validate_string(file_name, message)
416 message = 'The file {} does not exist'.format(file_name)
417 InputValidation.validate_file_exist(file_name, message)
418 res = open(file_name, 'r')
419 return res.readline()
422 def replace_in_file(file, text_to_search, text_to_replace):
424 Replaces a string within a file
426 :param file: name of the file (str)
427 :param text_to_search: text to be replaced
428 :param text_to_replace: new text that will replace the previous
431 message = 'The text to be replaced in the file must be a string'
432 InputValidation.validate_string(text_to_search, message)
433 message = 'The text to replace in the file must be a string'
434 InputValidation.validate_string(text_to_replace, message)
435 message = "The name of the file must be a string"
436 InputValidation.validate_string(file, message)
437 message = "The file does not exist"
438 InputValidation.validate_file_exist(file, message)
439 for line in fileinput.input(file, inplace=True):
440 print(line.replace(text_to_search, text_to_replace).rstrip())
443 # ------------------------------------------------------
445 # ------------------------------------------------------
446 def run_command(command):
447 LOG.info("Running command: {}".format(command))
448 return os.system(command)
451 def push_data_influxdb(data):
454 db_name = INFLUXDB_DB_NAME
455 command = "curl -i -XPOST 'http://{}:{}/write?db={}' " \
456 "--data-binary {}".format(ip, port, db_name, data)
460 # ------------------------------------------------------
461 # Expose variables to other modules
462 # ------------------------------------------------------
468 def get_template_dir():
472 def get_result_dir():
476 def get_dpdk_pktgen_vars():
477 if not (PKTGEN == 'dpdk_pktgen'):
480 ret_val[cf.CFSP_DPDK_PKTGEN_DIRECTORY] = PKTGEN_DIR
481 ret_val[cf.CFSP_DPDK_DPDK_DIRECTORY] = PKTGEN_DPDK_DIRECTORY
482 ret_val[cf.CFSP_DPDK_PROGRAM_NAME] = PKTGEN_PROGRAM
483 ret_val[cf.CFSP_DPDK_COREMASK] = PKTGEN_COREMASK
484 ret_val[cf.CFSP_DPDK_MEMORY_CHANNEL] = PKTGEN_MEMCHANNEL
485 ret_val[cf.CFSP_DPDK_BUS_SLOT_NIC_1] = PKTGEN_BUS_SLOT_NIC_1
486 ret_val[cf.CFSP_DPDK_BUS_SLOT_NIC_2] = PKTGEN_BUS_SLOT_NIC_2
487 ret_val[cf.CFSP_DPDK_NAME_IF_1] = PKTGEN_NAME_NIC_1
488 ret_val[cf.CFSP_DPDK_NAME_IF_2] = PKTGEN_NAME_NIC_2
492 # ------------------------------------------------------
493 # Configuration Variables from Config File
494 # ------------------------------------------------------
495 def get_deployment_configuration_variables_from_conf_file():
498 all_variables = CONF_FILE.get_variable_list(cf.CFS_EXPERIMENT_VNF)
499 for var in all_variables:
500 v = CONF_FILE.get_variable(cf.CFS_EXPERIMENT_VNF, var)
501 type = re.findall(r'@\w*', v)
502 values = re.findall(r'\"(.+?)\"', v)
503 variables[var] = values
505 types[var] = type[0][1:]
507 LOG.debug("No type has been specified for variable " + var)
511 # ------------------------------------------------------
512 # benchmarks from Config File
513 # ------------------------------------------------------
514 def get_benchmarks_from_conf_file():
515 requested_benchmarks = list()
517 CONF_FILE.get_variable(cf.CFS_GENERAL, cf.CFSG_BENCHMARKS).split(', ')
518 for benchmark in benchmarks:
519 requested_benchmarks.append(benchmark)
520 return requested_benchmarks
523 class InputValidation(object):
526 def validate_string(param, message):
527 if not isinstance(param, str):
528 raise ValueError(message)
532 def validate_integer(param, message):
533 if not isinstance(param, int):
534 raise ValueError(message)
538 def validate_dictionary(param, message):
539 if not isinstance(param, dict):
540 raise ValueError(message)
544 def validate_file_exist(file_name, message):
545 if not os.path.isfile(file_name):
546 raise ValueError(message + ' ' + file_name)
550 def validate_directory_exist_and_format(directory, message):
551 if not os.path.isdir(directory):
552 raise ValueError(message)
553 if not directory.endswith('/'):
554 return directory + '/'
558 def validate_configuration_file_parameter(section, parameter, message):
559 params = CONF_FILE.get_variable_list(section)
560 if parameter not in params:
561 raise ValueError(message)
565 def validate_configuration_file_section(section, message):
566 if section not in cf.get_sections():
567 raise ValueError(message)
571 def validate_boolean(boolean, message):
572 if isinstance(boolean, bool):
574 if isinstance(boolean, str):
575 if boolean == 'True':
577 if boolean == 'False':
579 raise ValueError(message)
582 def validate_os_credentials(credentials):
583 if not isinstance(credentials, dict):
585 'The provided openstack_credentials '
586 'variable must be in dictionary format')
588 credential_keys = ['ip_controller', 'heat_url', 'user', 'password',
589 'auth_uri', 'project']
592 for credential_key in credential_keys
593 if credential_key not in credentials.keys()
595 if len(missing) == 0:
597 msg = 'OpenStack Credentials Error! ' \
598 'The following parameters are missing: {}'.\
599 format(", ".join(missing))
600 raise ValueError(msg)