2 ###############################################################################
3 # Copyright (c) 2015 Ericsson AB and others.
4 # jonas.bjurel@ericsson.com
5 # All rights reserved. This program and the accompanying materials
6 # are made available under the terms of the Apache License, Version 2.0
7 # which accompanies this distribution, and is available at
8 # http://www.apache.org/licenses/LICENSE-2.0
9 ###############################################################################
11 ###############################################################################
13 # This script constructs the final deployment dea.yaml and dha.yaml files
14 # The dea.yaml get's constructed from (in reverse priority):
17 # 3) deployment-scenario dea-override-config section
19 # The dha.yaml get's constructed from (in reverse priority):
21 # 2) deployment-scenario dha-override-config section
22 ###############################################################################
33 from functools import reduce
34 from operator import or_
41 create_dir_if_not_exists,
47 def parse_arguments():
48 parser = ArgParser(prog='python %s' % __file__)
49 parser.add_argument('-dha', dest='dha_uri', action='store',
50 default=False, help='dha configuration file FQDN URI', required=True)
51 parser.add_argument('-deab', dest='dea_base_uri', action='store',
52 default=False, help='dea base configuration FQDN URI', required=True)
53 parser.add_argument('-deao', dest='dea_pod_override_uri', action='store',
54 default=False, help='dea POD override configuration FQDN URI',
56 parser.add_argument('-scenario-base-uri', dest='scenario_base_uri', action='store',
57 default=False, help='Deploymen scenario base directory URI',
59 parser.add_argument('-scenario', dest='scenario', action='store',
60 default=False, help='Deploymen scenario short-name (priority), or base file name (in the absense of a shortname defenition)',
63 parser.add_argument('-plugins', dest='plugins_uri', action='store',
64 default=False, help='Plugin configurations directory URI',
66 parser.add_argument('-output', dest='output_path', action='store',
68 help='Local path for resulting output configuration files',
70 args = parser.parse_args()
72 kwargs = {'dha_uri': args.dha_uri,
73 'dea_base_uri': args.dea_base_uri,
74 'dea_pod_override_uri': args.dea_pod_override_uri,
75 'scenario_base_uri': args.scenario_base_uri,
76 'scenario': args.scenario,
77 'plugins_uri': args.plugins_uri,
78 'output_path': args.output_path}
84 print('%(red)s WARNING: %(msg)s %(NC)s' % {'red': red,
89 represent_dict_order = lambda self, data: self.represent_mapping('tag:yaml.org,2002:map', data.items())
90 yaml.add_representer(collections.OrderedDict, represent_dict_order)
94 response = urllib2.urlopen(uri)
95 data = response.read()
98 return sha1.hexdigest()
100 def merge_fuel_plugin_version_list(list1, list2):
102 # When the plugin version in not there in list1 it will
105 plugin_version = e_l1.get('metadata',
106 {'plugin_version', None}).get('plugin_version')
107 plugin_version_found = False
109 if plugin_version == e_l2.get('metadata',
111 None}).get('plugin_version'):
112 final_list.append(dict(mergedicts(e_l1, e_l2)))
113 plugin_version_found = True
114 if not plugin_version_found:
115 final_list.append(e_l1)
118 def merge_lists(list1, list2):
120 if isinstance(list1[0], dict):
121 if 'plugin_version' in list1[0].get('metadata', {}):
122 return merge_fuel_plugin_version_list(list1, list2)
124 warning("Lists with dictionary inside are not merge able! "
125 "List2 will overwrite List1. "
126 "List1: %s; List2: %s"
136 def mergedicts(dict1, dict2):
137 for k in set(dict1.keys()).union(dict2.keys()):
138 if k in dict1 and k in dict2:
139 if isinstance(dict1[k], dict) and isinstance(dict2[k], dict):
140 yield (k, dict(mergedicts(dict1[k], dict2[k])))
141 elif isinstance(dict1[k], list) and isinstance(dict2[k], list):
142 yield (k, merge_lists(dict1[k], dict2[k]))
144 # If one of the values is not a dict nor a list,
145 # you can't continue merging it.
146 # Value from second dict overrides one in first and we move on.
154 kwargs = parse_arguments()
156 # Generate final dea.yaml by merging following config files/fragments in revers priority order:
157 # "dea-base", "dea-pod-override", "deplyment-scenario/module-config-override"
158 # and "deployment-scenario/dea-override"
159 print 'Generating final dea.yaml configuration....'
161 # Fetch dea-base, extract and purge meta-data
162 print 'Parsing dea-base from: ' + kwargs["dea_base_uri"] + "...."
163 response = urllib2.urlopen(kwargs["dea_base_uri"])
164 dea_base_conf = yaml.load(response.read())
165 dea_base_title = dea_base_conf['dea-base-config-metadata']['title']
166 dea_base_version = dea_base_conf['dea-base-config-metadata']['version']
167 dea_base_creation = dea_base_conf['dea-base-config-metadata']['created']
168 dea_base_sha = sha_uri(kwargs["dea_base_uri"])
169 dea_base_comment = dea_base_conf['dea-base-config-metadata']['comment']
170 dea_base_conf.pop('dea-base-config-metadata')
171 final_dea_conf = dea_base_conf
173 # Fetch dea-pod-override, extract and purge meta-data, merge with previous dea data structure
174 print 'Parsing the dea-pod-override from: ' + kwargs["dea_pod_override_uri"] + "...."
175 response = urllib2.urlopen(kwargs["dea_pod_override_uri"])
176 dea_pod_override_conf = yaml.load(response.read())
177 if dea_pod_override_conf:
178 dea_pod_title = dea_pod_override_conf['dea-pod-override-config-metadata']['title']
179 dea_pod_version = dea_pod_override_conf['dea-pod-override-config-metadata']['version']
180 dea_pod_creation = dea_pod_override_conf['dea-pod-override-config-metadata']['created']
181 dea_pod_sha = sha_uri(kwargs["dea_pod_override_uri"])
182 dea_pod_comment = dea_pod_override_conf['dea-pod-override-config-metadata']['comment']
183 print 'Merging dea-base and dea-pod-override configuration ....'
184 dea_pod_override_conf.pop('dea-pod-override-config-metadata')
185 if dea_pod_override_conf:
186 final_dea_conf = dict(mergedicts(final_dea_conf, dea_pod_override_conf))
188 # Fetch deployment-scenario, extract and purge meta-data, merge deployment-scenario/
189 # dea-override-configith previous dea data structure
190 print 'Parsing deployment-scenario from: ' + kwargs["scenario"] + "...."
192 response = urllib2.urlopen(kwargs["scenario_base_uri"] + "/scenario.yaml")
193 scenario_short_translation_conf = yaml.load(response.read())
194 if kwargs["scenario"] in scenario_short_translation_conf:
195 scenario_uri = kwargs["scenario_base_uri"] + "/" + scenario_short_translation_conf[kwargs["scenario"]]['configfile']
197 scenario_uri = kwargs["scenario_base_uri"] + "/" + kwargs["scenario"]
198 response = urllib2.urlopen(scenario_uri)
199 deploy_scenario_conf = yaml.load(response.read())
201 if deploy_scenario_conf:
202 deploy_scenario_title = deploy_scenario_conf['deployment-scenario-metadata']['title']
203 deploy_scenario_version = deploy_scenario_conf['deployment-scenario-metadata']['version']
204 deploy_scenario_creation = deploy_scenario_conf['deployment-scenario-metadata']['created']
205 deploy_scenario_sha = sha_uri(scenario_uri)
206 deploy_scenario_comment = deploy_scenario_conf['deployment-scenario-metadata']['comment']
207 deploy_scenario_conf.pop('deployment-scenario-metadata')
209 print "Deployment scenario file not found or is empty"
210 print "Cannot continue, exiting ...."
213 dea_scenario_override_conf = deploy_scenario_conf["dea-override-config"]
214 if dea_scenario_override_conf:
215 print 'Merging dea-base-, dea-pod-override- and deployment-scenario configuration into final dea.yaml configuration....'
216 final_dea_conf = dict(mergedicts(final_dea_conf, dea_scenario_override_conf))
218 # Fetch plugin-configuration configuration files, extract and purge meta-data,
219 # merge/append with previous dea data structure, override plugin-configuration with
220 # deploy-scenario/module-config-override
225 module_creations = []
228 if deploy_scenario_conf["stack-extensions"]:
229 for module in deploy_scenario_conf["stack-extensions"]:
230 print 'Loading configuration for module: ' + module["module"] + ' and merging it to final dea.yaml configuration....'
231 response = urllib2.urlopen(kwargs["plugins_uri"] + '/' + module["module-config-name"] + '_' + module["module-config-version"] + '.yaml')
232 module_conf = yaml.load(response.read())
233 modules.append(module["module"])
234 module_uris.append(kwargs["plugins_uri"] + '/' + module["module-config-name"] + '_' + module["module-config-version"] + '.yaml')
235 module_titles.append(str(module_conf['plugin-config-metadata']['title']))
236 module_versions.append(str(module_conf['plugin-config-metadata']['version']))
237 module_creations.append(str(module_conf['plugin-config-metadata']['created']))
238 module_shas.append(sha_uri(kwargs["plugins_uri"] + '/' + module["module-config-name"] + '_' + module["module-config-version"] + '.yaml'))
239 module_comments.append(str(module_conf['plugin-config-metadata']['comment']))
240 module_conf.pop('plugin-config-metadata')
241 final_dea_conf['settings']['editable'].update(module_conf)
242 scenario_module_override_conf = module.get('module-config-override')
243 if scenario_module_override_conf:
244 dea_scenario_module_override_conf = {}
245 dea_scenario_module_override_conf['settings'] = {}
246 dea_scenario_module_override_conf['settings']['editable'] = {}
247 dea_scenario_module_override_conf['settings']['editable'][module["module"]] = scenario_module_override_conf
248 final_dea_conf = dict(mergedicts(final_dea_conf, dea_scenario_module_override_conf))
250 # Dump final dea.yaml including configuration management meta-data to argument provided
252 if not os.path.exists(kwargs["output_path"]):
253 os.makedirs(kwargs["output_path"])
254 print 'Dumping final dea.yaml to ' + kwargs["output_path"] + '/dea.yaml....'
255 with open(kwargs["output_path"] + '/dea.yaml', "w") as f:
256 f.write("title: DEA.yaml file automatically generated from the configuration files stated in the \"configuration-files\" fragment below\n")
257 f.write("version: " + str(calendar.timegm(time.gmtime())) + "\n")
258 f.write("created: " + str(time.strftime("%d/%m/%Y")) + " " + str(time.strftime("%H:%M:%S")) + "\n")
259 f.write("comment: none\n")
261 f.write("configuration-files:\n")
262 f.write(" dea-base:\n")
263 f.write(" uri: " + kwargs["dea_base_uri"] + "\n")
264 f.write(" title: " + str(dea_base_title) + "\n")
265 f.write(" version: " + str(dea_base_version) + "\n")
266 f.write(" created: " + str(dea_base_creation) + "\n")
267 f.write(" sha1: " + str(dea_base_sha) + "\n")
268 f.write(" comment: " + str(dea_base_comment) + "\n")
270 f.write(" pod-override:\n")
271 f.write(" uri: " + kwargs["dea_pod_override_uri"] + "\n")
272 f.write(" title: " + str(dea_pod_title) + "\n")
273 f.write(" version: " + str(dea_pod_version) + "\n")
274 f.write(" created: " + str(dea_pod_creation) + "\n")
275 f.write(" sha1: " + str(dea_pod_sha) + "\n")
276 f.write(" comment: " + str(dea_pod_comment) + "\n")
278 f.write(" deployment-scenario:\n")
279 f.write(" uri: " + str(scenario_uri) + "\n")
280 f.write(" title: " + str(deploy_scenario_title) + "\n")
281 f.write(" version: " + str(deploy_scenario_version) + "\n")
282 f.write(" created: " + str(deploy_scenario_creation) + "\n")
283 f.write(" sha1: " + str(deploy_scenario_sha) + "\n")
284 f.write(" comment: " + str(deploy_scenario_comment) + "\n")
286 f.write(" plugin-modules:\n")
287 for k in range(0,len(modules)):
288 f.write(" - module: " + modules[k] + "\n")
289 f.write(" uri: " + module_uris[k] + "\n")
290 f.write(" title: " + module_titles[k] + "\n")
291 f.write(" version: " + module_versions[k] + "\n")
292 f.write(" created: " + module_creations[k] + "\n")
293 f.write(" sha-1: " + module_shas[k] + "\n")
294 f.write(" comment: " + module_comments[k] + "\n")
296 yaml.dump(final_dea_conf, f, default_flow_style=False)
298 # Load POD dha and override it with "deployment-scenario/dha-override-config" section
299 print 'Generating final dha.yaml configuration....'
300 print 'Parsing dha-pod yaml configuration....'
301 response = urllib2.urlopen(kwargs["dha_uri"])
302 dha_pod_conf = yaml.load(response.read())
303 dha_pod_title = dha_pod_conf['dha-pod-config-metadata']['title']
304 dha_pod_version = dha_pod_conf['dha-pod-config-metadata']['version']
305 dha_pod_creation = dha_pod_conf['dha-pod-config-metadata']['created']
306 dha_pod_sha = sha_uri(kwargs["dha_uri"])
307 dha_pod_comment = dha_pod_conf['dha-pod-config-metadata']['comment']
308 dha_pod_conf.pop('dha-pod-config-metadata')
309 final_dha_conf = dha_pod_conf
311 dha_scenario_override_conf = deploy_scenario_conf["dha-override-config"]
312 # Only virtual deploy scenarios can override dha.yaml since there
313 # is no way to programatically override a physical environment:
314 # wireing, IPMI set-up, etc.
315 # For Physical environments, dha.yaml overrides will be silently ignored
316 if dha_scenario_override_conf and (final_dha_conf['adapter'] == 'libvirt' or final_dha_conf['adapter'] == 'esxi' or final_dha_conf['adapter'] == 'vbox'):
317 print 'Merging dha-pod and deployment-scenario override information to final dha.yaml configuration....'
318 final_dha_conf = dict(mergedicts(final_dha_conf, dha_scenario_override_conf))
320 # Dump final dha.yaml to argument provided directory
321 print 'Dumping final dha.yaml to ' + kwargs["output_path"] + '/dha.yaml....'
322 with open(kwargs["output_path"] + '/dha.yaml', "w") as f:
323 f.write("title: DHA.yaml file automatically generated from the configuration files stated in the \"configuration-files\" fragment below\n")
324 f.write("version: " + str(calendar.timegm(time.gmtime())) + "\n")
325 f.write("created: " + str(time.strftime("%d/%m/%Y")) + " " + str(time.strftime("%H:%M:%S")) + "\n")
326 f.write("comment: none\n")
328 f.write("configuration-files:\n")
330 f.write(" dha-pod-configuration:\n")
331 f.write(" uri: " + kwargs["dha_uri"] + "\n")
332 f.write(" title: " + str(dha_pod_title) + "\n")
333 f.write(" version: " + str(dha_pod_version) + "\n")
334 f.write(" created: " + str(dha_pod_creation) + "\n")
335 f.write(" sha-1: " + str(dha_pod_sha) + "\n")
336 f.write(" comment: " + str(dha_pod_comment) + "\n")
338 f.write(" deployment-scenario:\n")
339 f.write(" uri: " + str(scenario_uri) + "\n")
340 f.write(" title: " + str(deploy_scenario_title) + "\n")
341 f.write(" version: " + str(deploy_scenario_version) + "\n")
342 f.write(" created: " + str(deploy_scenario_creation) + "\n")
343 f.write(" sha-1: " + str(deploy_scenario_sha) + "\n")
344 f.write(" comment: " + str(deploy_scenario_comment) + "\n")
345 yaml.dump(final_dha_conf, f, default_flow_style=False)