2 ###############################################################################
3 # Copyright (c) 2015 Ericsson AB and others.
4 # jonas.bjurel@ericsson.com
5 # All rights reserved. This program and the accompanying materials
6 # are made available under the terms of the Apache License, Version 2.0
7 # which accompanies this distribution, and is available at
8 # http://www.apache.org/licenses/LICENSE-2.0
9 ###############################################################################
11 ###############################################################################
13 # This script constructs the final deployment dea.yaml and dha.yaml files
14 # The dea.yaml get's constructed from (in reverse priority):
17 # 3) deployment-scenario dea-override-config section
19 # The dha.yaml get's constructed from (in reverse priority):
21 # 2) deployment-scenario dha-override-config section
22 ###############################################################################
34 from functools import reduce
35 from operator import or_
42 create_dir_if_not_exists,
49 def parse_arguments():
50 parser = ArgParser(prog='python %s' % __file__)
51 parser.add_argument('-dha', dest='dha_uri', action='store',
53 help='dha configuration file FQDN URI',
55 parser.add_argument('-deab', dest='dea_base_uri', action='store',
57 help='dea base configuration FQDN URI',
59 parser.add_argument('-deao', dest='dea_pod_override_uri',
62 help='dea POD override configuration FQDN URI',
64 parser.add_argument('-scenario-base-uri',
65 dest='scenario_base_uri',
68 help='Deployment scenario base directory URI',
70 parser.add_argument('-scenario', dest='scenario', action='store',
72 help=('Deployment scenario short-name (priority), '
73 'or base file name (in the absense of a '
74 'shortname defenition)'),
77 parser.add_argument('-plugins', dest='plugins_uri', action='store',
79 help='Plugin configurations directory URI',
81 parser.add_argument('-output', dest='output_path', action='store',
83 help='Local path for resulting output configuration files',
85 args = parser.parse_args()
86 kwargs = {'dha_uri': args.dha_uri,
87 'dea_base_uri': args.dea_base_uri,
88 'dea_pod_override_uri': args.dea_pod_override_uri,
89 'scenario_base_uri': args.scenario_base_uri,
90 'scenario': args.scenario,
91 'plugins_uri': args.plugins_uri,
92 'output_path': args.output_path}
99 print('%(red)s WARNING: %(msg)s %(NC)s' % {'red': red,
105 represent_dict_order = lambda self, data: self.represent_mapping('tag:yaml.org,2002:map', data.items())
106 yaml.add_representer(collections.OrderedDict, represent_dict_order)
110 response = urllib2.urlopen(uri)
111 data = response.read()
112 sha1 = hashlib.sha1()
114 return sha1.hexdigest()
117 def merge_fuel_plugin_version_list(list1, list2):
119 # When the plugin version in not there in list1 it will
122 plugin_version = e_l1.get('metadata', {}).get('plugin_version')
123 plugin_version_found = False
125 if plugin_version == e_l2.get('metadata', {}).get('plugin_version'):
126 final_list.append(dict(merge_dicts(e_l1, e_l2)))
127 plugin_version_found = True
128 if not plugin_version_found:
129 final_list.append(e_l1)
133 def merge_networks(list_1, list_2):
134 new_nets = {x.get('name'): x for x in list_2}
136 return [new_nets.get(net.get('name'), net) for net in list_1]
139 def merge_dicts(dict1, dict2):
140 for k in set(dict1).union(dict2):
141 if k in dict1 and k in dict2:
142 if isinstance(dict1[k], dict) and isinstance(dict2[k], dict):
143 yield (k, dict(merge_dicts(dict1[k], dict2[k])))
145 if isinstance(dict1[k], list) and isinstance(dict2[k], list):
148 merge_fuel_plugin_version_list(dict1[k], dict2[k]))
152 merge_networks(dict1[k], dict2[k]))
155 # If one of the values is not a dict nor a list,
156 # you can't continue merging it.
157 # Value from second dict overrides one in first if exists.
164 def get_node_ifaces_and_trans(nodes, nid):
166 if node['id'] == nid:
167 if 'transformations' in node and 'interfaces' in node:
168 return (node['interfaces'], node['transformations'])
175 class DeployConfig(object):
177 self.kwargs = parse_arguments()
178 self.dea_conf = dict()
179 self.dea_metadata = dict()
180 self.dea_pod_ovr_metadata = dict()
181 self.dea_pod_ovr_nodes = None
182 self.scenario_metadata = dict()
184 self.module_uris = []
185 self.module_titles = []
186 self.module_versions = []
187 self.module_createds = []
188 self.module_shas = []
189 self.module_comments = []
190 self.dha_pod_conf = dict()
191 self.dha_metadata = dict()
193 def process_dea_base(self):
194 # Generate final dea.yaml by merging following config files/fragments in reverse priority order:
195 # "dea-base", "dea-pod-override", "deplyment-scenario/module-config-override"
196 # and "deployment-scenario/dea-override"
197 print('Generating final dea.yaml configuration....')
199 # Fetch dea-base, extract and purge meta-data
200 print('Parsing dea-base from: ' + self.kwargs["dea_base_uri"] + "....")
201 response = urllib2.urlopen(self.kwargs["dea_base_uri"])
202 dea_conf = yaml.load(response.read())
204 dea_metadata = dict()
205 dea_metadata['title'] = dea_conf['dea-base-config-metadata']['title']
206 dea_metadata['version'] = dea_conf['dea-base-config-metadata']['version']
207 dea_metadata['created'] = dea_conf['dea-base-config-metadata']['created']
208 dea_metadata['sha'] = sha_uri(self.kwargs["dea_base_uri"])
209 dea_metadata['comment'] = dea_conf['dea-base-config-metadata']['comment']
210 self.dea_metadata = dea_metadata
211 dea_conf.pop('dea-base-config-metadata')
212 self.dea_conf = dea_conf
214 def process_dea_pod_override(self):
215 # Fetch dea-pod-override, extract and purge meta-data, merge with previous dea data structure
216 print('Parsing the dea-pod-override from: ' + self.kwargs["dea_pod_override_uri"] + "....")
217 response = urllib2.urlopen(self.kwargs["dea_pod_override_uri"])
218 dea_pod_override_conf = yaml.load(response.read())
220 if dea_pod_override_conf:
222 metadata['title'] = dea_pod_override_conf['dea-pod-override-config-metadata']['title']
223 metadata['version'] = dea_pod_override_conf['dea-pod-override-config-metadata']['version']
224 metadata['created'] = dea_pod_override_conf['dea-pod-override-config-metadata']['created']
225 metadata['sha'] = sha_uri(self.kwargs["dea_pod_override_uri"])
226 metadata['comment'] = dea_pod_override_conf['dea-pod-override-config-metadata']['comment']
227 self.dea_pod_ovr_metadata = metadata
229 print('Merging dea-base and dea-pod-override configuration ....')
230 dea_pod_override_conf.pop('dea-pod-override-config-metadata')
232 # Copy the list of original nodes, which holds info on their transformations
233 if 'nodes' in dea_pod_override_conf:
234 self.dea_pod_ovr_nodes = list(dea_pod_override_conf['nodes'])
235 if dea_pod_override_conf:
236 self.dea_conf = dict(merge_dicts(self.dea_conf, dea_pod_override_conf))
238 def get_scenario_uri(self):
239 response = urllib2.urlopen(self.kwargs["scenario_base_uri"] + "/scenario.yaml")
240 scenario_short_translation_conf = yaml.load(response.read())
241 if self.kwargs["scenario"] in scenario_short_translation_conf:
242 scenario_uri = (self.kwargs["scenario_base_uri"]
244 + scenario_short_translation_conf[self.kwargs["scenario"]]['configfile'])
246 scenario_uri = self.kwargs["scenario_base_uri"] + "/" + self.kwargs["scenario"]
250 def get_scenario_config(self):
251 self.scenario_metadata['uri'] = self.get_scenario_uri()
252 response = urllib2.urlopen(self.scenario_metadata['uri'])
253 return yaml.load(response.read())
255 def process_modules(self):
256 scenario_conf = self.get_scenario_config()
257 if scenario_conf["stack-extensions"]:
258 for module in scenario_conf["stack-extensions"]:
259 print('Loading configuration for module: '
261 + ' and merging it to final dea.yaml configuration....')
262 response = urllib2.urlopen(self.kwargs["plugins_uri"]
264 + module["module-config-name"]
266 + module["module-config-version"]
268 module_conf = yaml.load(response.read())
269 self.modules.append(module["module"])
270 self.module_uris.append(self.kwargs["plugins_uri"]
272 + module["module-config-name"]
274 + module["module-config-version"]
276 self.module_titles.append(str(module_conf['plugin-config-metadata']['title']))
277 self.module_versions.append(str(module_conf['plugin-config-metadata']['version']))
278 self.module_createds.append(str(module_conf['plugin-config-metadata']['created']))
279 self.module_shas.append(sha_uri(self.kwargs["plugins_uri"]
281 + module["module-config-name"]
283 + module["module-config-version"]
285 self.module_comments.append(str(module_conf['plugin-config-metadata']['comment']))
286 module_conf.pop('plugin-config-metadata')
287 self.dea_conf['settings']['editable'].update(module_conf)
289 scenario_module_override_conf = module.get('module-config-override')
290 if scenario_module_override_conf:
291 dea_scenario_module_override_conf = {}
292 dea_scenario_module_override_conf['settings'] = {}
293 dea_scenario_module_override_conf['settings']['editable'] = {}
294 dea_scenario_module_override_conf['settings']['editable'][module["module"]] = scenario_module_override_conf
295 self.dea_conf = dict(merge_dicts(self.dea_conf, dea_scenario_module_override_conf))
297 def process_scenario_config(self):
298 # Fetch deployment-scenario, extract and purge meta-data, merge deployment-scenario/
299 # dea-override-configith previous dea data structure
300 print('Parsing deployment-scenario from: ' + self.kwargs["scenario"] + "....")
302 scenario_conf = self.get_scenario_config()
306 metadata['title'] = scenario_conf['deployment-scenario-metadata']['title']
307 metadata['version'] = scenario_conf['deployment-scenario-metadata']['version']
308 metadata['created'] = scenario_conf['deployment-scenario-metadata']['created']
309 metadata['sha'] = sha_uri(self.scenario_metadata['uri'])
310 metadata['comment'] = scenario_conf['deployment-scenario-metadata']['comment']
311 self.scenario_metadata = metadata
312 scenario_conf.pop('deployment-scenario-metadata')
314 print("Deployment scenario file not found or is empty")
315 print("Cannot continue, exiting ....")
318 dea_scenario_override_conf = scenario_conf["dea-override-config"]
319 if dea_scenario_override_conf:
320 print('Merging dea-base-, dea-pod-override- and deployment-scenario '
321 'configuration into final dea.yaml configuration....')
322 self.dea_conf = dict(merge_dicts(self.dea_conf, dea_scenario_override_conf))
324 self.process_modules()
326 # Fetch plugin-configuration configuration files, extract and purge meta-data,
327 # merge/append with previous dea data structure, override plugin-configuration with
328 # deploy-scenario/module-config-override
330 if self.dea_pod_ovr_nodes:
331 for node in self.dea_conf['nodes']:
332 data = get_node_ifaces_and_trans(self.dea_pod_ovr_nodes, node['id'])
334 print("Honoring original interfaces and transformations for "
335 "node %d to %s, %s" % (node['id'], data[0], data[1]))
336 node['interfaces'] = data[0]
337 node['transformations'] = data[1]
339 def dump_dea_config(self):
340 # Dump final dea.yaml including configuration management meta-data to argument provided
342 path = self.kwargs["output_path"]
343 if not os.path.exists(path):
345 print('Dumping final dea.yaml to ' + path + '/dea.yaml....')
346 with open(path + '/dea.yaml', "w") as f:
347 f.write("\n".join([("title: DEA.yaml file automatically generated from the "
348 'configuration files stated in the "configuration-files" '
350 "version: " + str(calendar.timegm(time.gmtime())),
351 "created: " + time.strftime("%d/%m/%Y %H:%M:%S"),
354 f.write("\n".join(["configuration-files:",
356 " uri: " + self.kwargs["dea_base_uri"],
357 " title: " + str(self.dea_metadata['title']),
358 " version: " + str(self.dea_metadata['version']),
359 " created: " + str(self.dea_metadata['created']),
360 " sha1: " + sha_uri(self.kwargs["dea_base_uri"]),
361 " comment: " + str(self.dea_metadata['comment']) + "\n"]))
363 f.write("\n".join([" pod-override:",
364 " uri: " + self.kwargs["dea_pod_override_uri"],
365 " title: " + str(self.dea_pod_ovr_metadata['title']),
366 " version: " + str(self.dea_pod_ovr_metadata['version']),
367 " created: " + str(self.dea_pod_ovr_metadata['created']),
368 " sha1: " + self.dea_pod_ovr_metadata['sha'],
369 " comment: " + str(self.dea_pod_ovr_metadata['comment']) + "\n"]))
371 f.write("\n".join([" deployment-scenario:",
372 " uri: " + self.scenario_metadata['uri'],
373 " title: " + str(self.scenario_metadata['title']),
374 " version: " + str(self.scenario_metadata['version']),
375 " created: " + str(self.scenario_metadata['created']),
376 " sha1: " + self.scenario_metadata['sha'],
377 " comment: " + str(self.scenario_metadata['comment']) + "\n"]))
379 f.write(" plugin-modules:\n")
380 for k, _ in enumerate(self.modules):
381 f.write("\n".join([" - module: " + self.modules[k],
382 " uri: " + self.module_uris[k],
383 " title: " + str(self.module_titles[k]),
384 " version: " + str(self.module_versions[k]),
385 " created: " + str(self.module_createds[k]),
386 " sha-1: " + self.module_shas[k],
387 " comment: " + str(self.module_comments[k]) + "\n"]))
389 yaml.dump(self.dea_conf, f, default_flow_style=False)
391 def process_dha_pod_config(self):
392 # Load POD dha and override it with "deployment-scenario/dha-override-config" section
393 print('Generating final dha.yaml configuration....')
394 print('Parsing dha-pod yaml configuration....')
395 response = urllib2.urlopen(self.kwargs["dha_uri"])
396 dha_pod_conf = yaml.load(response.read())
398 dha_metadata = dict()
399 dha_metadata['title'] = dha_pod_conf['dha-pod-config-metadata']['title']
400 dha_metadata['version'] = dha_pod_conf['dha-pod-config-metadata']['version']
401 dha_metadata['created'] = dha_pod_conf['dha-pod-config-metadata']['created']
402 dha_metadata['sha'] = sha_uri(self.kwargs["dha_uri"])
403 dha_metadata['comment'] = dha_pod_conf['dha-pod-config-metadata']['comment']
404 self.dha_metadata = dha_metadata
405 dha_pod_conf.pop('dha-pod-config-metadata')
406 self.dha_pod_conf = dha_pod_conf
408 scenario_conf = self.get_scenario_config()
409 dha_scenario_override_conf = scenario_conf["dha-override-config"]
410 # Only virtual deploy scenarios can override dha.yaml since there
411 # is no way to programatically override a physical environment:
412 # wireing, IPMI set-up, etc.
413 # For Physical environments, dha.yaml overrides will be silently ignored
414 if dha_scenario_override_conf and (dha_pod_conf['adapter'] == 'libvirt'
415 or dha_pod_conf['adapter'] == 'esxi'
416 or dha_pod_conf['adapter'] == 'vbox'):
417 print('Merging dha-pod and deployment-scenario override information to final dha.yaml configuration....')
418 self.dha_pod_conf = dict(merge_dicts(self.dha_pod_conf, dha_scenario_override_conf))
420 def dump_dha_config(self):
421 # Dump final dha.yaml to argument provided directory
422 path = self.kwargs["output_path"]
423 print('Dumping final dha.yaml to ' + path + '/dha.yaml....')
424 with open(path + '/dha.yaml', "w") as f:
425 f.write("\n".join([("title: DHA.yaml file automatically generated from "
426 "the configuration files stated in the "
427 '"configuration-files" fragment below'),
428 "version: " + str(calendar.timegm(time.gmtime())),
429 "created: " + time.strftime("%d/%m/%Y %H:%M:%S"),
432 f.write("configuration-files:\n")
434 f.write("\n".join([" dha-pod-configuration:",
435 " uri: " + self.kwargs["dha_uri"],
436 " title: " + str(self.dha_metadata['title']),
437 " version: " + str(self.dha_metadata['version']),
438 " created: " + str(self.dha_metadata['created']),
439 " sha-1: " + self.dha_metadata['sha'],
440 " comment: " + str(self.dha_metadata['comment']) + "\n"]))
442 f.write("\n".join([" deployment-scenario:",
443 " uri: " + self.scenario_metadata['uri'],
444 " title: " + str(self.scenario_metadata['title']),
445 " version: " + str(self.scenario_metadata['version']),
446 " created: " + str(self.scenario_metadata['created']),
447 " sha-1: " + self.scenario_metadata['sha'],
448 " comment: " + str(self.scenario_metadata['comment']) + "\n"]))
450 yaml.dump(self.dha_pod_conf, f, default_flow_style=False)
456 deploy_config = DeployConfig()
457 deploy_config.process_dea_base()
458 deploy_config.process_dea_pod_override()
459 deploy_config.process_scenario_config()
460 deploy_config.dump_dea_config()
462 deploy_config.process_dha_pod_config()
463 deploy_config.dump_dha_config()
466 if __name__ == '__main__':