1 # Copyright (c) 2015 Intel Research and Development Ireland Ltd.
3 # Licensed under the Apache License, Version 2.0 (the "License");
4 # you may not use this file except in compliance with the License.
5 # You may obtain a copy of the License at
7 # http://www.apache.org/licenses/LICENSE-2.0
9 # Unless required by applicable law or agreed to in writing, software
10 # distributed under the License is distributed on an "AS IS" BASIS,
11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 # See the License for the specific language governing permissions and
13 # limitations under the License.
15 from __future__ import print_function
16 from __future__ import absolute_import
19 import six.moves.configparser
22 from experimental_framework.constants import conf_file_sections as cf
23 from experimental_framework.constants import framework_parameters as fp
26 # ------------------------------------------------------
27 # List of common variables
28 # ------------------------------------------------------
30 allowed_releases = ['liberty', 'kilo', 'juno']
34 DEPLOYMENT_UNIT = None
42 TEMPLATE_FILE_EXTENSION = None
46 PKTGEN_DPDK_DIRECTORY = None
48 PKTGEN_COREMASK = None
49 PKTGEN_MEMCHANNEL = None
50 PKTGEN_BUS_SLOT_NIC_1 = None
51 PKTGEN_BUS_SLOT_NIC_2 = None
52 PKTGEN_NAME_NIC_1 = None
53 PKTGEN_NAME_NIC_2 = None
57 INFLUXDB_DB_NAME = None
60 # ------------------------------------------------------
61 # Initialization and Input 'heat_templates/'validation
62 # ------------------------------------------------------
66 # BASE_DIR = os.getcwd()
67 BASE_DIR = os.path.dirname(os.path.abspath(__file__))
68 BASE_DIR = BASE_DIR.replace('/experimental_framework', '')
69 BASE_DIR = InputValidation.validate_directory_exist_and_format(
70 BASE_DIR, "Error 000001")
74 init_general_vars(api)
75 if CONF_FILE.get_variable_list(cf.CFS_PKTGEN):
79 def init_conf_file(api=False):
82 CONF_FILE = ConfigurationFile(cf.get_sections_api(),
83 '/tmp/apexlake/apexlake.conf')
85 CONF_FILE = ConfigurationFile(cf.get_sections(),
86 '/tmp/apexlake/apexlake.conf')
89 def init_general_vars(api=False):
90 global TEMPLATE_FILE_EXTENSION
97 TEMPLATE_FILE_EXTENSION = '.yaml'
99 # Check Section in Configuration File
101 validate_configuration_file_section(
103 "Section " + cf.CFS_GENERAL +
104 "is not present in configuration file")
107 validate_configuration_file_section(
109 "Section " + cf.CFS_OPENSTACK +
110 "is not present in configuration file")
112 TEMPLATE_DIR = '/tmp/apexlake/heat_templates/'
113 # if not os.path.exists(TEMPLATE_DIR):
114 # os.makedirs(TEMPLATE_DIR)
115 # cmd = "cp /tmp/apexlake/heat_templates/*.yaml {}".format(TEMPLATE_DIR)
119 # Validate template name
121 validate_configuration_file_parameter(
123 cf.CFSG_TEMPLATE_NAME,
124 "Parameter " + cf.CFSG_TEMPLATE_NAME +
125 "is not present in configuration file")
126 TEMPLATE_NAME = CONF_FILE.get_variable(cf.CFS_GENERAL,
127 cf.CFSG_TEMPLATE_NAME)
128 InputValidation.validate_file_exist(
129 TEMPLATE_DIR + TEMPLATE_NAME,
130 "The provided template file does not exist")
132 RESULT_DIR = "/tmp/apexlake/results/"
133 if not os.path.isdir(RESULT_DIR):
134 os.makedirs(RESULT_DIR)
136 if cf.CFSO_RELEASE in CONF_FILE.get_variable_list(cf.CFS_OPENSTACK):
137 RELEASE = CONF_FILE.get_variable(cf.CFS_OPENSTACK, cf.CFSO_RELEASE)
138 if RELEASE not in allowed_releases:
139 raise ValueError("Release {} is not supported".format(RELEASE))
141 # Validate and assign Iterations
142 if cf.CFSG_ITERATIONS in CONF_FILE.get_variable_list(cf.CFS_GENERAL):
143 ITERATIONS = int(CONF_FILE.get_variable(cf.CFS_GENERAL,
151 LOG = logging.getLogger()
152 debug = CONF_FILE.get_variable(cf.CFS_GENERAL, cf.CFSG_DEBUG)
153 if debug == 'true' or debug == 'True':
154 LOG.setLevel(level=logging.DEBUG)
156 LOG.setLevel(level=logging.INFO)
157 log_formatter = logging.Formatter("%(asctime)s --- %(message)s")
158 file_handler = logging.FileHandler("{0}/{1}.log".format("./", "benchmark"))
159 file_handler.setFormatter(log_formatter)
160 file_handler.setLevel(logging.DEBUG)
161 LOG.addHandler(file_handler)
164 # ------------------------------------------------------
165 # InfluxDB conf variables
166 # ------------------------------------------------------
170 global INFLUXDB_DB_NAME
172 INFLUXDB_IP = CONF_FILE.get_variable(cf.CFS_INFLUXDB, cf.CFSI_IDB_IP)
173 INFLUXDB_PORT = CONF_FILE.get_variable(cf.CFS_INFLUXDB, cf.CFSI_IDB_PORT)
174 INFLUXDB_DB_NAME = CONF_FILE.get_variable(cf.CFS_INFLUXDB,
178 # ------------------------------------------------------
179 # Packet Generator conf variables
180 # ------------------------------------------------------
184 global PKTGEN_PROGRAM
185 global PKTGEN_COREMASK
186 global PKTGEN_MEMCHANNEL
187 global PKTGEN_BUS_SLOT_NIC_1
188 global PKTGEN_BUS_SLOT_NIC_2
189 global PKTGEN_DPDK_DIRECTORY
190 global PKTGEN_NAME_NIC_1
191 global PKTGEN_NAME_NIC_2
193 msg = "Section {} is not present in the configuration file".\
194 format(cf.CFS_PKTGEN)
195 InputValidation.validate_configuration_file_section(cf.CFS_PKTGEN, msg)
197 pktgen_var_list = CONF_FILE.get_variable_list(cf.CFS_PKTGEN)
198 PKTGEN = 'dpdk_pktgen' # default value
199 if cf.CFSP_PACKET_GENERATOR in pktgen_var_list:
200 msg = "Parameter {} is not present in section {}".format(
201 cf.CFSP_PACKET_GENERATOR, cf.CFS_PKTGEN)
202 InputValidation.validate_configuration_file_parameter(
203 cf.CFS_PKTGEN, cf.CFSP_PACKET_GENERATOR, msg)
204 PKTGEN = CONF_FILE.get_variable(
205 cf.CFS_PKTGEN, cf.CFSP_PACKET_GENERATOR)
207 if PKTGEN not in fp.get_supported_packet_generators():
208 raise ValueError('The specified packet generator is not supported '
211 # Check if the packet gen is dpdk_pktgen
212 if PKTGEN == cf.CFSP_PG_DPDK:
213 # Validation of DPDK pktgen directory
214 msg = "Parameter {} is not present in section {}".format(
215 cf.CFSP_DPDK_PKTGEN_DIRECTORY, cf.CFS_PKTGEN)
216 InputValidation.validate_configuration_file_parameter(
217 cf.CFS_PKTGEN, cf.CFSP_DPDK_PKTGEN_DIRECTORY, msg)
218 PKTGEN_DIR = CONF_FILE.get_variable(
219 cf.CFS_PKTGEN, cf.CFSP_DPDK_PKTGEN_DIRECTORY)
220 msg = "The directory {} does not exist.".format(PKTGEN_DIR)
221 PKTGEN_DIR = InputValidation.validate_directory_exist_and_format(
224 # Validation of the DPDK program name
225 msg = "Parameter {} is not present in section {}".format(
226 cf.CFSP_DPDK_PROGRAM_NAME, cf.CFS_PKTGEN)
227 InputValidation.validate_configuration_file_parameter(
228 cf.CFS_PKTGEN, cf.CFSP_DPDK_PROGRAM_NAME, msg)
229 PKTGEN_PROGRAM = CONF_FILE.get_variable(
230 cf.CFS_PKTGEN, cf.CFSP_DPDK_PROGRAM_NAME)
232 # Validation of the DPDK Coremask parameter
233 msg = "Parameter {} is not present in section {}".format(
234 cf.CFSP_DPDK_COREMASK, cf.CFS_PKTGEN)
235 InputValidation.validate_configuration_file_parameter(
236 cf.CFS_PKTGEN, cf.CFSP_DPDK_COREMASK, msg)
237 PKTGEN_COREMASK = CONF_FILE.get_variable(
238 cf.CFS_PKTGEN, cf.CFSP_DPDK_COREMASK)
240 # Validation of the DPDK Memory Channel parameter
241 msg = "Parameter {} is not present in section {}".format(
242 cf.CFSP_DPDK_MEMORY_CHANNEL, cf.CFS_PKTGEN)
243 InputValidation.validate_configuration_file_parameter(
244 cf.CFS_PKTGEN, cf.CFSP_DPDK_MEMORY_CHANNEL, msg)
245 PKTGEN_MEMCHANNEL = CONF_FILE.get_variable(
246 cf.CFS_PKTGEN, cf.CFSP_DPDK_MEMORY_CHANNEL)
248 # Validation of the DPDK Bus Slot 1
249 msg = "Parameter {} is not present in section {}".format(
250 cf.CFSP_DPDK_BUS_SLOT_NIC_1, cf.CFS_PKTGEN)
251 InputValidation.validate_configuration_file_parameter(
252 cf.CFS_PKTGEN, cf.CFSP_DPDK_BUS_SLOT_NIC_1, msg)
253 PKTGEN_BUS_SLOT_NIC_1 = CONF_FILE.get_variable(
254 cf.CFS_PKTGEN, cf.CFSP_DPDK_BUS_SLOT_NIC_1)
256 # Validation of the DPDK Bus Slot 2
257 msg = "Parameter {} is not present in section {}".format(
258 cf.CFSP_DPDK_BUS_SLOT_NIC_2, cf.CFS_PKTGEN)
259 InputValidation.validate_configuration_file_parameter(
260 cf.CFS_PKTGEN, cf.CFSP_DPDK_BUS_SLOT_NIC_2, msg)
261 PKTGEN_BUS_SLOT_NIC_2 = CONF_FILE.get_variable(
262 cf.CFS_PKTGEN, cf.CFSP_DPDK_BUS_SLOT_NIC_2)
264 # Validation of the DPDK NIC 1
265 msg = "Parameter {} is not present in section {}".format(
266 cf.CFSP_DPDK_NAME_IF_1, cf.CFS_PKTGEN)
267 InputValidation.validate_configuration_file_parameter(
268 cf.CFS_PKTGEN, cf.CFSP_DPDK_NAME_IF_1, msg)
269 PKTGEN_NAME_NIC_1 = CONF_FILE.get_variable(
270 cf.CFS_PKTGEN, cf.CFSP_DPDK_NAME_IF_1)
272 # Validation of the DPDK NIC 2
273 msg = "Parameter {} is not present in section {}".format(
274 cf.CFSP_DPDK_NAME_IF_2, cf.CFS_PKTGEN)
275 InputValidation.validate_configuration_file_parameter(
276 cf.CFS_PKTGEN, cf.CFSP_DPDK_NAME_IF_2, msg)
277 PKTGEN_NAME_NIC_2 = CONF_FILE.get_variable(
278 cf.CFS_PKTGEN, cf.CFSP_DPDK_NAME_IF_2)
280 # Validation of DPDK directory parameter
281 msg = "Parameter {} is not present in section {}".format(
282 cf.CFSP_DPDK_DPDK_DIRECTORY, cf.CFS_PKTGEN)
283 InputValidation.validate_configuration_file_parameter(
284 cf.CFS_PKTGEN, cf.CFSP_DPDK_DPDK_DIRECTORY, msg)
285 PKTGEN_DPDK_DIRECTORY = CONF_FILE.get_variable(
286 cf.CFS_PKTGEN, cf.CFSP_DPDK_DPDK_DIRECTORY)
287 msg = "Directory {} does not exist".format(
288 cf.CFSP_DPDK_DPDK_DIRECTORY)
289 PKTGEN_DPDK_DIRECTORY = InputValidation.\
290 validate_directory_exist_and_format(PKTGEN_DPDK_DIRECTORY, msg)
293 # ------------------------------------------------------
294 # Configuration file access
295 # ------------------------------------------------------
297 class ConfigurationFile:
299 Used to extract data from the configuration file
302 def __init__(self, sections, config_file='conf.cfg'):
304 Reads configuration file sections
306 :param sections: list of strings representing the sections to be
308 :param config_file: name of the configuration file (string)
311 InputValidation.validate_string(
312 config_file, "The configuration file name must be a string")
313 # config_file = BASE_DIR + config_file
314 InputValidation.validate_file_exist(
315 config_file, 'The provided configuration file does not exist')
316 self.config = six.moves.configparser.ConfigParser()
317 self.config.read(config_file)
318 for section in sections:
320 self, section, ConfigurationFile.
321 _config_section_map(section, self.config))
324 def _config_section_map(section, config_file):
326 Returns a dictionary with the configuration values for the specific
329 :param section: section to be loaded (string)
330 :param config_file: name of the configuration file (string)
334 options = config_file.options(section)
335 for option in options:
336 dict1[option] = config_file.get(section, option)
339 def get_variable(self, section, variable_name):
341 Returns the value correspondent to a variable
343 :param section: section to be loaded (string)
344 :param variable_name: name of the variable (string)
347 message = "The variable name must be a string"
348 InputValidation.validate_string(variable_name, message)
349 if variable_name in self.get_variable_list(section):
350 sect = getattr(self, section)
351 return sect[variable_name]
353 exc_msg = 'Parameter {} is not in the {} section of the ' \
354 'conf file'.format(variable_name, section)
355 raise ValueError(exc_msg)
357 def get_variable_list(self, section):
359 Returns the list of the available variables in a section
360 :param section: section to be loaded (string)
364 return getattr(self, section)
366 msg = 'Section {} not found in the configuration file'.\
368 raise ValueError(msg)
371 # ------------------------------------------------------
372 # Get OpenStack Credentials
373 # ------------------------------------------------------
374 def get_credentials():
376 Returns the credentials for OpenStack access from the configuration file
380 credentials[cf.CFSO_IP_CONTROLLER] = CONF_FILE.get_variable(
381 cf.CFS_OPENSTACK, cf.CFSO_IP_CONTROLLER)
382 credentials[cf.CFSO_HEAT_URL] = CONF_FILE.get_variable(
383 cf.CFS_OPENSTACK, cf.CFSO_HEAT_URL)
384 credentials[cf.CFSO_USER] = CONF_FILE.get_variable(
385 cf.CFS_OPENSTACK, cf.CFSO_USER)
386 credentials[cf.CFSO_PASSWORD] = CONF_FILE.get_variable(
387 cf.CFS_OPENSTACK, cf.CFSO_PASSWORD)
388 credentials[cf.CFSO_AUTH_URI] = CONF_FILE.get_variable(
389 cf.CFS_OPENSTACK, cf.CFSO_AUTH_URI)
390 credentials[cf.CFSO_PROJECT] = CONF_FILE.get_variable(
391 cf.CFS_OPENSTACK, cf.CFSO_PROJECT)
395 # ------------------------------------------------------
397 # ------------------------------------------------------
399 def get_heat_template_params():
401 Returns the list of deployment parameters from the configuration file
402 for the heat template
406 heat_parameters_list = CONF_FILE.get_variable_list(
407 cf.CFS_DEPLOYMENT_PARAMETERS)
408 testcase_parameters = dict()
409 for param in heat_parameters_list:
410 testcase_parameters[param] = CONF_FILE.get_variable(
411 cf.CFS_DEPLOYMENT_PARAMETERS, param)
412 return testcase_parameters
415 def get_testcase_params():
417 Returns the list of testcase parameters from the configuration file
421 testcase_parameters = dict()
422 parameters = CONF_FILE.get_variable_list(cf.CFS_TESTCASE_PARAMETERS)
423 for param in parameters:
424 testcase_parameters[param] = CONF_FILE.get_variable(
425 cf.CFS_TESTCASE_PARAMETERS, param)
426 return testcase_parameters
429 def get_file_first_line(file_name):
431 Returns the first line of a file
433 :param file_name: name of the file to be read (str)
436 message = "The name of the file must be a string"
437 InputValidation.validate_string(file_name, message)
438 message = 'The file {} does not exist'.format(file_name)
439 InputValidation.validate_file_exist(file_name, message)
440 res = open(file_name, 'r')
441 return res.readline()
444 def replace_in_file(file, text_to_search, text_to_replace):
446 Replaces a string within a file
448 :param file: name of the file (str)
449 :param text_to_search: text to be replaced
450 :param text_to_replace: new text that will replace the previous
453 message = 'The text to be replaced in the file must be a string'
454 InputValidation.validate_string(text_to_search, message)
455 message = 'The text to replace in the file must be a string'
456 InputValidation.validate_string(text_to_replace, message)
457 message = "The name of the file must be a string"
458 InputValidation.validate_string(file, message)
459 message = "The file does not exist"
460 InputValidation.validate_file_exist(file, message)
461 for line in fileinput.input(file, inplace=True):
462 print((line.replace(text_to_search, text_to_replace).rstrip()))
465 # ------------------------------------------------------
467 # ------------------------------------------------------
468 def run_command(command):
469 LOG.info("Running command: {}".format(command))
470 return os.system(command)
473 def push_data_influxdb(data):
476 db_name = INFLUXDB_DB_NAME
477 command = "curl -i -XPOST 'http://{}:{}/write?db={}' " \
478 "--data-binary {}".format(ip, port, db_name, data)
482 # ------------------------------------------------------
483 # Expose variables to other modules
484 # ------------------------------------------------------
490 def get_template_dir():
494 def get_result_dir():
498 def get_dpdk_pktgen_vars():
499 if not (PKTGEN == 'dpdk_pktgen'):
502 ret_val[cf.CFSP_DPDK_PKTGEN_DIRECTORY] = PKTGEN_DIR
503 ret_val[cf.CFSP_DPDK_DPDK_DIRECTORY] = PKTGEN_DPDK_DIRECTORY
504 ret_val[cf.CFSP_DPDK_PROGRAM_NAME] = PKTGEN_PROGRAM
505 ret_val[cf.CFSP_DPDK_COREMASK] = PKTGEN_COREMASK
506 ret_val[cf.CFSP_DPDK_MEMORY_CHANNEL] = PKTGEN_MEMCHANNEL
507 ret_val[cf.CFSP_DPDK_BUS_SLOT_NIC_1] = PKTGEN_BUS_SLOT_NIC_1
508 ret_val[cf.CFSP_DPDK_BUS_SLOT_NIC_2] = PKTGEN_BUS_SLOT_NIC_2
509 ret_val[cf.CFSP_DPDK_NAME_IF_1] = PKTGEN_NAME_NIC_1
510 ret_val[cf.CFSP_DPDK_NAME_IF_2] = PKTGEN_NAME_NIC_2
514 # ------------------------------------------------------
515 # Configuration Variables from Config File
516 # ------------------------------------------------------
517 def get_deployment_configuration_variables_from_conf_file():
520 all_variables = CONF_FILE.get_variable_list(cf.CFS_EXPERIMENT_VNF)
521 for var in all_variables:
522 v = CONF_FILE.get_variable(cf.CFS_EXPERIMENT_VNF, var)
523 type = re.findall(r'@\w*', v)
524 values = re.findall(r'\"(.+?)\"', v)
525 variables[var] = values
527 types[var] = type[0][1:]
529 LOG.debug("No type has been specified for variable " + var)
533 # ------------------------------------------------------
534 # benchmarks from Config File
535 # ------------------------------------------------------
536 def get_benchmarks_from_conf_file():
537 requested_benchmarks = list()
539 CONF_FILE.get_variable(cf.CFS_GENERAL, cf.CFSG_BENCHMARKS).split(', ')
540 for benchmark in benchmarks:
541 requested_benchmarks.append(benchmark)
542 return requested_benchmarks
545 class InputValidation(object):
548 def validate_string(param, message):
549 if not isinstance(param, str):
550 raise ValueError(message)
554 def validate_integer(param, message):
555 if not isinstance(param, int):
556 raise ValueError(message)
560 def validate_dictionary(param, message):
561 if not isinstance(param, dict):
562 raise ValueError(message)
566 def validate_file_exist(file_name, message):
567 if not os.path.isfile(file_name):
568 raise ValueError(message + ' ' + file_name)
572 def validate_directory_exist_and_format(directory, message):
573 if not os.path.isdir(directory):
574 raise ValueError(message)
575 if not directory.endswith('/'):
576 return directory + '/'
580 def validate_configuration_file_parameter(section, parameter, message):
581 params = CONF_FILE.get_variable_list(section)
582 if parameter not in params:
583 raise ValueError(message)
587 def validate_configuration_file_section(section, message):
588 if section not in cf.get_sections():
589 raise ValueError(message)
593 def validate_boolean(boolean, message):
594 if isinstance(boolean, bool):
596 if isinstance(boolean, str):
597 if boolean == 'True':
599 if boolean == 'False':
601 raise ValueError(message)
604 def validate_os_credentials(credentials):
605 if not isinstance(credentials, dict):
607 'The provided openstack_credentials '
608 'variable must be in dictionary format')
610 credential_keys = ['ip_controller', 'heat_url', 'user', 'password',
611 'auth_uri', 'project']
614 for credential_key in credential_keys
615 if credential_key not in list(credentials.keys())
617 if len(missing) == 0:
619 msg = 'OpenStack Credentials Error! ' \
620 'The following parameters are missing: {}'.\
621 format(", ".join(missing))
622 raise ValueError(msg)