2 ###############################################################################
3 # Copyright (c) 2015 Ericsson AB and others.
4 # jonas.bjurel@ericsson.com
5 # All rights reserved. This program and the accompanying materials
6 # are made available under the terms of the Apache License, Version 2.0
7 # which accompanies this distribution, and is available at
8 # http://www.apache.org/licenses/LICENSE-2.0
9 ###############################################################################
11 ###############################################################################
13 # This script constructs the final deployment dea.yaml and dha.yaml files
14 # The dea.yaml get's constructed from (in reverse priority):
17 # 3) deployment-scenario dea-override-config section
19 # The dha.yaml get's constructed from (in reverse priority):
21 # 2) deployment-scenario dha-override-config section
22 ###############################################################################
34 from functools import reduce
35 from operator import or_
42 create_dir_if_not_exists,
49 def parse_arguments():
50 parser = ArgParser(prog='python %s' % __file__)
51 parser.add_argument('-dha', dest='dha_uri', action='store',
53 help='dha configuration file FQDN URI',
55 parser.add_argument('-deab', dest='dea_base_uri', action='store',
57 help='dea base configuration FQDN URI',
59 parser.add_argument('-deao', dest='dea_pod_override_uri',
62 help='dea POD override configuration FQDN URI',
64 parser.add_argument('-scenario-base-uri',
65 dest='scenario_base_uri',
68 help='Deployment scenario base directory URI',
70 parser.add_argument('-scenario', dest='scenario', action='store',
72 help=('Deployment scenario short-name (priority),'
73 'or base file name (in the absense of a'
74 'shortname defenition)'),
77 parser.add_argument('-plugins', dest='plugins_uri', action='store',
79 help='Plugin configurations directory URI',
81 parser.add_argument('-output', dest='output_path', action='store',
83 help='Local path for resulting output configuration files',
85 args = parser.parse_args()
87 kwargs = {'dha_uri': args.dha_uri,
88 'dea_base_uri': args.dea_base_uri,
89 'dea_pod_override_uri': args.dea_pod_override_uri,
90 'scenario_base_uri': args.scenario_base_uri,
91 'scenario': args.scenario,
92 'plugins_uri': args.plugins_uri,
93 'output_path': args.output_path}
100 print('%(red)s WARNING: %(msg)s %(NC)s' % {'red': red,
106 represent_dict_order = lambda self, data: self.represent_mapping('tag:yaml.org,2002:map', data.items())
107 yaml.add_representer(collections.OrderedDict, represent_dict_order)
111 response = urllib2.urlopen(uri)
112 data = response.read()
113 sha1 = hashlib.sha1()
115 return sha1.hexdigest()
118 def merge_fuel_plugin_version_list(list1, list2):
120 # When the plugin version in not there in list1 it will
123 plugin_version = e_l1.get('metadata', {}).get('plugin_version')
124 plugin_version_found = False
126 if plugin_version == e_l2.get('metadata', {}).get('plugin_version'):
127 final_list.append(dict(merge_dicts(e_l1, e_l2)))
128 plugin_version_found = True
129 if not plugin_version_found:
130 final_list.append(e_l1)
134 def merge_networks(list_1, list_2):
135 new_nets = {x.get('name'): x for x in list_2}
137 return [new_nets.get(net.get('name'), net) for net in list_1]
140 def merge_dicts(dict1, dict2):
141 for k in set(dict1).union(dict2):
142 if k in dict1 and k in dict2:
143 if isinstance(dict1[k], dict) and isinstance(dict2[k], dict):
144 yield (k, dict(merge_dicts(dict1[k], dict2[k])))
146 if isinstance(dict1[k], list) and isinstance(dict2[k], list):
149 merge_fuel_plugin_version_list(dict1[k], dict2[k]))
153 merge_networks(dict1[k], dict2[k]))
156 # If one of the values is not a dict nor a list,
157 # you can't continue merging it.
158 # Value from second dict overrides one in first if exists.
166 kwargs = parse_arguments()
168 # Generate final dea.yaml by merging following config files/fragments in revers priority order:
169 # "dea-base", "dea-pod-override", "deplyment-scenario/module-config-override"
170 # and "deployment-scenario/dea-override"
171 print('Generating final dea.yaml configuration....')
173 # Fetch dea-base, extract and purge meta-data
174 print('Parsing dea-base from: ' + kwargs["dea_base_uri"] + "....")
175 response = urllib2.urlopen(kwargs["dea_base_uri"])
176 dea_base_conf = yaml.load(response.read())
177 dea_base_title = dea_base_conf['dea-base-config-metadata']['title']
178 dea_base_version = dea_base_conf['dea-base-config-metadata']['version']
179 dea_base_creation = dea_base_conf['dea-base-config-metadata']['created']
180 dea_base_sha = sha_uri(kwargs["dea_base_uri"])
181 dea_base_comment = dea_base_conf['dea-base-config-metadata']['comment']
182 dea_base_conf.pop('dea-base-config-metadata')
183 final_dea_conf = dea_base_conf
184 dea_pod_override_nodes = None
186 # Fetch dea-pod-override, extract and purge meta-data, merge with previous dea data structure
187 print('Parsing the dea-pod-override from: ' + kwargs["dea_pod_override_uri"] + "....")
188 response = urllib2.urlopen(kwargs["dea_pod_override_uri"])
189 dea_pod_override_conf = yaml.load(response.read())
190 if dea_pod_override_conf:
191 dea_pod_title = dea_pod_override_conf['dea-pod-override-config-metadata']['title']
192 dea_pod_version = dea_pod_override_conf['dea-pod-override-config-metadata']['version']
193 dea_pod_creation = dea_pod_override_conf['dea-pod-override-config-metadata']['created']
194 dea_pod_sha = sha_uri(kwargs["dea_pod_override_uri"])
195 dea_pod_comment = dea_pod_override_conf['dea-pod-override-config-metadata']['comment']
196 print('Merging dea-base and dea-pod-override configuration ....')
197 dea_pod_override_conf.pop('dea-pod-override-config-metadata')
198 # Copy the list of original nodes, which holds info on their transformations
199 if 'nodes' in dea_pod_override_conf:
200 dea_pod_override_nodes = list(dea_pod_override_conf['nodes'])
201 if dea_pod_override_conf:
202 final_dea_conf = dict(merge_dicts(final_dea_conf, dea_pod_override_conf))
204 # Fetch deployment-scenario, extract and purge meta-data, merge deployment-scenario/
205 # dea-override-configith previous dea data structure
206 print('Parsing deployment-scenario from: ' + kwargs["scenario"] + "....")
208 response = urllib2.urlopen(kwargs["scenario_base_uri"] + "/scenario.yaml")
209 scenario_short_translation_conf = yaml.load(response.read())
210 if kwargs["scenario"] in scenario_short_translation_conf:
211 scenario_uri = (kwargs["scenario_base_uri"]
213 + scenario_short_translation_conf[kwargs["scenario"]]['configfile'])
215 scenario_uri = kwargs["scenario_base_uri"] + "/" + kwargs["scenario"]
216 response = urllib2.urlopen(scenario_uri)
217 deploy_scenario_conf = yaml.load(response.read())
219 if deploy_scenario_conf:
220 deploy_scenario_title = deploy_scenario_conf['deployment-scenario-metadata']['title']
221 deploy_scenario_version = deploy_scenario_conf['deployment-scenario-metadata']['version']
222 deploy_scenario_creation = deploy_scenario_conf['deployment-scenario-metadata']['created']
223 deploy_scenario_sha = sha_uri(scenario_uri)
224 deploy_scenario_comment = deploy_scenario_conf['deployment-scenario-metadata']['comment']
225 deploy_scenario_conf.pop('deployment-scenario-metadata')
227 print("Deployment scenario file not found or is empty")
228 print("Cannot continue, exiting ....")
231 dea_scenario_override_conf = deploy_scenario_conf["dea-override-config"]
232 if dea_scenario_override_conf:
233 print('Merging dea-base-, dea-pod-override- and deployment-scenario '
234 'configuration into final dea.yaml configuration....')
235 final_dea_conf = dict(merge_dicts(final_dea_conf, dea_scenario_override_conf))
237 # Fetch plugin-configuration configuration files, extract and purge meta-data,
238 # merge/append with previous dea data structure, override plugin-configuration with
239 # deploy-scenario/module-config-override
244 module_creations = []
247 if deploy_scenario_conf["stack-extensions"]:
248 for module in deploy_scenario_conf["stack-extensions"]:
249 print('Loading configuration for module: '
251 + ' and merging it to final dea.yaml configuration....')
252 response = urllib2.urlopen(kwargs["plugins_uri"]
254 + module["module-config-name"]
256 + module["module-config-version"]
258 module_conf = yaml.load(response.read())
259 modules.append(module["module"])
260 module_uris.append(kwargs["plugins_uri"]
262 + module["module-config-name"]
264 + module["module-config-version"]
266 module_titles.append(str(module_conf['plugin-config-metadata']['title']))
267 module_versions.append(str(module_conf['plugin-config-metadata']['version']))
268 module_creations.append(str(module_conf['plugin-config-metadata']['created']))
269 module_shas.append(sha_uri(kwargs["plugins_uri"]
271 + module["module-config-name"]
273 + module["module-config-version"]
275 module_comments.append(str(module_conf['plugin-config-metadata']['comment']))
276 module_conf.pop('plugin-config-metadata')
277 final_dea_conf['settings']['editable'].update(module_conf)
278 scenario_module_override_conf = module.get('module-config-override')
279 if scenario_module_override_conf:
280 dea_scenario_module_override_conf = {}
281 dea_scenario_module_override_conf['settings'] = {}
282 dea_scenario_module_override_conf['settings']['editable'] = {}
283 dea_scenario_module_override_conf['settings']['editable'][module["module"]] = scenario_module_override_conf
284 final_dea_conf = dict(merge_dicts(final_dea_conf, dea_scenario_module_override_conf))
286 def get_node_ifaces_and_trans(nodes, nid):
288 if node['id'] == nid:
289 if 'transformations' in node and 'interfaces' in node:
290 return (node['interfaces'], node['transformations'])
296 if dea_pod_override_nodes:
297 for node in final_dea_conf['nodes']:
298 data = get_node_ifaces_and_trans(dea_pod_override_nodes, node['id'])
300 print ("Honoring original interfaces and transformations for "
301 "node %d to %s, %s" % (node['id'], data[0], data[1]))
302 node['interfaces'] = data[0]
303 node['transformations'] = data[1]
305 # Dump final dea.yaml including configuration management meta-data to argument provided
307 if not os.path.exists(kwargs["output_path"]):
308 os.makedirs(kwargs["output_path"])
309 print('Dumping final dea.yaml to ' + kwargs["output_path"] + '/dea.yaml....')
310 with open(kwargs["output_path"] + '/dea.yaml', "w") as f:
311 f.write("\n".join([("title: DEA.yaml file automatically generated from the"
312 'configuration files stated in the "configuration-files"'
314 "version: " + str(calendar.timegm(time.gmtime())),
315 "created: " + str(time.strftime("%d/%m/%Y")) + " "
316 + str(time.strftime("%H:%M:%S")),
319 f.write("\n".join(["configuration-files:",
321 " uri: " + kwargs["dea_base_uri"],
322 " title: " + str(dea_base_title),
323 " version: " + str(dea_base_version),
324 " created: " + str(dea_base_creation),
325 " sha1: " + str(dea_base_sha),
326 " comment: " + str(dea_base_comment) + "\n"]))
328 f.write("\n".join([" pod-override:",
329 " uri: " + kwargs["dea_pod_override_uri"],
330 " title: " + str(dea_pod_title),
331 " version: " + str(dea_pod_version),
332 " created: " + str(dea_pod_creation),
333 " sha1: " + str(dea_pod_sha),
334 " comment: " + str(dea_pod_comment) + "\n"]))
336 f.write("\n".join([" deployment-scenario:",
337 " uri: " + str(scenario_uri),
338 " title: " + str(deploy_scenario_title),
339 " version: " + str(deploy_scenario_version),
340 " created: " + str(deploy_scenario_creation),
341 " sha1: " + str(deploy_scenario_sha),
342 " comment: " + str(deploy_scenario_comment) + "\n"]))
344 f.write(" plugin-modules:\n")
345 for k, _ in enumerate(modules):
346 f.write("\n".join([" - module: " + modules[k],
347 " uri: " + module_uris[k],
348 " title: " + module_titles[k],
349 " version: " + module_versions[k],
350 " created: " + module_creations[k],
351 " sha-1: " + module_shas[k],
352 " comment: " + module_comments[k] + "\n"]))
354 yaml.dump(final_dea_conf, f, default_flow_style=False)
356 # Load POD dha and override it with "deployment-scenario/dha-override-config" section
357 print('Generating final dha.yaml configuration....')
358 print('Parsing dha-pod yaml configuration....')
359 response = urllib2.urlopen(kwargs["dha_uri"])
360 dha_pod_conf = yaml.load(response.read())
361 dha_pod_title = dha_pod_conf['dha-pod-config-metadata']['title']
362 dha_pod_version = dha_pod_conf['dha-pod-config-metadata']['version']
363 dha_pod_creation = dha_pod_conf['dha-pod-config-metadata']['created']
364 dha_pod_sha = sha_uri(kwargs["dha_uri"])
365 dha_pod_comment = dha_pod_conf['dha-pod-config-metadata']['comment']
366 dha_pod_conf.pop('dha-pod-config-metadata')
367 final_dha_conf = dha_pod_conf
369 dha_scenario_override_conf = deploy_scenario_conf["dha-override-config"]
370 # Only virtual deploy scenarios can override dha.yaml since there
371 # is no way to programatically override a physical environment:
372 # wireing, IPMI set-up, etc.
373 # For Physical environments, dha.yaml overrides will be silently ignored
374 if dha_scenario_override_conf and (final_dha_conf['adapter'] == 'libvirt'
375 or final_dha_conf['adapter'] == 'esxi'
376 or final_dha_conf['adapter'] == 'vbox'):
377 print('Merging dha-pod and deployment-scenario override information to final dha.yaml configuration....')
378 final_dha_conf = dict(merge_dicts(final_dha_conf, dha_scenario_override_conf))
380 # Dump final dha.yaml to argument provided directory
381 print('Dumping final dha.yaml to ' + kwargs["output_path"] + '/dha.yaml....')
382 with open(kwargs["output_path"] + '/dha.yaml', "w") as f:
383 f.write("\n".join([("title: DHA.yaml file automatically generated from"
384 "the configuration files stated in the"
385 '"configuration-files" fragment below'),
386 "version: " + str(calendar.timegm(time.gmtime())),
387 "created: " + str(time.strftime("%d/%m/%Y")) + " "
388 + str(time.strftime("%H:%M:%S")),
391 f.write("configuration-files:\n")
393 f.write("\n".join([" dha-pod-configuration:",
394 " uri: " + kwargs["dha_uri"],
395 " title: " + str(dha_pod_title),
396 " version: " + str(dha_pod_version),
397 " created: " + str(dha_pod_creation),
398 " sha-1: " + str(dha_pod_sha),
399 " comment: " + str(dha_pod_comment) + "\n"]))
401 f.write("\n".join([" deployment-scenario:",
402 " uri: " + str(scenario_uri),
403 " title: " + str(deploy_scenario_title),
404 " version: " + str(deploy_scenario_version),
405 " created: " + str(deploy_scenario_creation),
406 " sha-1: " + str(deploy_scenario_sha),
407 " comment: " + str(deploy_scenario_comment) + "\n"]))
409 yaml.dump(final_dha_conf, f, default_flow_style=False)