10 'HeatTemplateFormatVersion': '2012-12-12',
15 description = 'Description'
16 parameters = 'Parameters'
18 resources = 'Resources'
20 properties = 'Properties'
22 depends_on = 'DependsOn'
23 get_attr = 'Fn::GetAtt'
29 'heat_template_version': '2013-05-23',
32 get_resource = 'get_resource'
33 get_param = 'get_param'
34 description = 'description'
35 parameters = 'parameters'
37 resources = 'resources'
39 properties = 'properties'
41 depends_on = 'depends_on'
48 def apply_maps(template):
49 """Apply Merge::Map within template.
51 Any dict {'Merge::Map': {'Foo': 'Bar', 'Baz': 'Quux'}}
52 will resolve to ['Bar', 'Quux'] - that is a dict with key
53 'Merge::Map' is replaced entirely by that dict['Merge::Map'].values().
55 if isinstance(template, dict):
56 if 'Merge::Map' in template:
58 apply_maps(value) for value in template['Merge::Map'].values()
61 return dict((key, apply_maps(value))
62 for key, value in template.items())
63 elif isinstance(template, list):
64 return [apply_maps(item) for item in template]
69 def apply_scaling(template, scaling, in_copies=None):
70 """Apply a set of scaling operations to template.
72 This is a single pass recursive function: for each call we process one
73 dict or list and recurse to handle children containers.
75 Values are handled via scale_value.
77 Keys in dicts are copied per the scaling rule.
78 Values are either replaced or copied depending on whether the given
79 scaling rule is in in_copies.
81 in_copies is reset to None when a dict {'Merge::Map': someobject} is
84 :param scaling: A dict of prefix -> (count, blacklists).
86 in_copies = dict(in_copies or {})
87 # Shouldn't be needed but to avoid unexpected side effects/bugs we short
91 if isinstance(template, dict):
92 if 'Merge::Map' in template:
95 for key, value in template.items():
96 for prefix, copy_num, new_key in scale_value(
97 key, scaling, in_copies):
99 # e.g. Compute0, 1, Compute1Foo
100 in_copies[prefix] = prefix[:-1] + str(copy_num)
101 if isinstance(value, (dict, list)):
102 new_value = apply_scaling(value, scaling, in_copies)
103 new_template[new_key] = new_value
105 new_values = list(scale_value(value, scaling, in_copies))
106 # We have nowhere to multiply a non-container value of a
107 # dict, so it may be copied or unchanged but not scaled.
108 assert len(new_values) == 1
109 new_template[new_key] = new_values[0][2]
111 del in_copies[prefix]
113 elif isinstance(template, list):
115 for value in template:
116 if isinstance(value, (dict, list)):
117 new_template.append(apply_scaling(value, scaling, in_copies))
119 for _, _, new_value in scale_value(value, scaling, in_copies):
120 new_template.append(new_value)
123 raise Exception("apply_scaling called with non-container %r" % template)
126 def scale_value(value, scaling, in_copies):
127 """Scale out a value.
129 :param value: The value to scale (not a container).
130 :param scaling: The scaling map (prefix-> (copies, blacklist) to use.
131 :param in_copies: What containers we're currently copying.
132 :return: An iterator of the new values for the value as tuples:
133 (prefix, copy_num, value). E.g. Compute0, 1, Compute1Foo
134 prefix and copy_num are only set when:
135 - a prefix in scaling matches value
136 - and that prefix is not in in_copies
138 if isinstance(value, (str, unicode)):
139 for prefix, (copies, blacklist) in scaling.items():
140 if not value.startswith(prefix):
142 suffix = value[len(prefix):]
143 if prefix in in_copies:
144 # Adjust to the copy number we're on
145 yield None, None, in_copies[prefix] + suffix
148 for n in range(copies):
149 if n not in blacklist:
150 yield prefix, n, prefix[:-1] + str(n) + suffix
152 yield None, None, value
154 yield None, None, value
157 def parse_scaling(scaling_args):
158 """Translate a list of scaling requests to a dict prefix:count."""
159 scaling_args = scaling_args or []
161 for item in scaling_args:
162 key, values = item.split('=')
163 values = values.split(',')
164 value = int(values[0])
165 blacklist = frozenset(int(v) for v in values[1:] if v)
166 result[key + '0'] = value, blacklist
170 def _translate_role(role, master_role, slave_roles):
173 if role == master_role:
175 if role not in slave_roles:
179 def translate_role(role, master_role, slave_roles):
180 r = _translate_role(role, master_role, slave_roles)
181 if not isinstance(r, basestring):
182 raise Exception('%s -> %r' % (role, r))
185 def resolve_params(item, param, value):
186 if item in ({lang.get_param: param}, {lang.get_resource: param}):
188 if isinstance(item, dict):
189 copy_item = dict(item)
190 for k, v in iter(copy_item.items()):
191 item[k] = resolve_params(v, param, value)
192 elif isinstance(item, list):
193 copy_item = list(item)
196 new_item.append(resolve_params(v, param, value))
200 MERGABLE_TYPES = {'OS::Nova::Server':
202 'AWS::EC2::Instance':
203 {'image': 'ImageId'},
204 'AWS::AutoScaling::LaunchConfiguration':
207 INCLUDED_TEMPLATE_DIR = os.getcwd()
210 def resolve_includes(template, params=None):
214 for key, value in iter(template.items()):
215 if key == '__include__':
216 new_params = dict(params) # do not propagate up the stack
217 if not isinstance(value, dict):
218 raise ValueError('__include__ must be a mapping')
219 if 'path' not in value:
220 raise ValueError('__include__ must have path')
221 if 'params' in value:
222 if not isinstance(value['params'], dict):
223 raise ValueError('__include__ params must be a mapping')
224 new_params.update(value['params'])
225 with open(value['path']) as include_file:
226 sub_template = yaml.safe_load(include_file.read())
227 if 'subkey' in value:
228 if ((not isinstance(value['subkey'], int)
229 and not isinstance(sub_template, dict))):
230 raise RuntimeError('subkey requires mapping root or'
231 ' integer for list root')
232 sub_template = sub_template[value['subkey']]
233 for k, v in iter(new_params.items()):
234 sub_template = resolve_params(sub_template, k, v)
235 new_template.update(resolve_includes(sub_template))
237 if isinstance(value, dict):
238 new_template[key] = resolve_includes(value)
240 new_template[key] = value
246 parser = argparse.ArgumentParser()
247 parser.add_argument('templates', nargs='+')
248 parser.add_argument('--master-role', nargs='?',
249 help='Translate slave_roles to this')
250 parser.add_argument('--slave-roles', nargs='*',
251 help='Translate all of these to master_role')
252 parser.add_argument('--included-template-dir', nargs='?',
253 default=INCLUDED_TEMPLATE_DIR,
254 help='Path for resolving included templates')
255 parser.add_argument('--output',
256 help='File to write output to. - for stdout',
258 parser.add_argument('--scale', action="append",
259 help="Names to scale out. Pass Prefix=2 to cause a key Prefix0Foo to "
260 "be copied to Prefix1Foo in the output, and value Prefix0Bar to be"
261 "renamed to Prefix1Bar inside that copy, or copied to Prefix1Bar "
262 "outside of any copy. Pass Prefix=3,1 to cause Prefix1* to be elided"
263 "when scaling Prefix out. Prefix=4,1,2 will likewise elide Prefix1 and"
266 '--change-image-params', action='store_true', default=False,
267 help="Change parameters in templates to match resource names. This was "
268 " the default at one time but it causes issues when parameter "
269 " names need to remain stable.")
271 '--hot', action='store_true', default=False,
272 help="Assume source templates are in the HOT format, and generate a "
273 "HOT template artifact.")
274 args = parser.parse_args(argv)
279 templates = args.templates
280 scaling = parse_scaling(args.scale)
281 merged_template = merge(templates, args.master_role, args.slave_roles,
282 args.included_template_dir, scaling=scaling,
283 change_image_params=args.change_image_params)
284 if args.output == '-':
285 out_file = sys.stdout
287 out_file = file(args.output, 'wt')
288 out_file.write(merged_template)
291 def merge(templates, master_role=None, slave_roles=None,
292 included_template_dir=INCLUDED_TEMPLATE_DIR,
293 scaling=None, change_image_params=None):
294 scaling = scaling or {}
296 end_template = dict(lang.base_template)
298 for template_path in templates:
299 template = yaml.safe_load(open(template_path))
300 # Resolve __include__ tags
301 template = resolve_includes(template)
302 end_template[lang.description].append(template.get(lang.description,
304 new_parameters = template.get(lang.parameters, {})
305 for p, pbody in sorted(new_parameters.items()):
306 if p in end_template.get(lang.parameters, {}):
307 if pbody != end_template[lang.parameters][p]:
308 errors.append('Parameter %s from %s conflicts.' % (p,
311 if lang.parameters not in end_template:
312 end_template[lang.parameters] = {}
313 end_template[lang.parameters][p] = pbody
315 new_outputs = template.get(lang.outputs, {})
316 for o, obody in sorted(new_outputs.items()):
317 if o in end_template.get(lang.outputs, {}):
318 if pbody != end_template[lang.outputs][p]:
319 errors.append('Output %s from %s conflicts.' % (o,
322 if lang.outputs not in end_template:
323 end_template[lang.outputs] = {}
324 end_template[lang.outputs][o] = obody
326 new_resources = template.get(lang.resources, {})
327 for r, rbody in sorted(new_resources.items()):
328 if rbody[lang.type] in MERGABLE_TYPES:
329 if change_image_params:
330 if 'image' in MERGABLE_TYPES[rbody[lang.type]]:
331 image_key = MERGABLE_TYPES[rbody[lang.type]]['image']
332 # XXX Assuming ImageId is always a Ref
333 ikey_val = end_template[lang.parameters][rbody[lang.properties][image_key][lang.get_param]]
334 del end_template[lang.parameters][rbody[lang.properties][image_key][lang.get_param]]
335 if 'OpenStack::Role' in rbody.get(lang.metadata, {}):
336 sys.stderr.write("WARNING: OpenStack::Role is deprecated"
337 " and will be removed in a later release\n");
338 role = rbody.get(lang.metadata, {}).get('OpenStack::Role', r)
339 role = translate_role(role, master_role, slave_roles)
341 resource_changes.append((r, role))
342 if role in end_template.get(lang.resources, {}):
343 new_metadata = rbody.get(lang.metadata, {})
344 for m, mbody in iter(new_metadata.items()):
345 if m in end_template[lang.resources][role].get(lang.metadata, {}):
346 if m == 'OpenStack::ImageBuilder::Elements':
347 end_template[lang.resources][role][lang.metadata][m].extend(mbody)
349 "WARNING: OpenStack::ImageBuilder::Elements"
350 " is deprecated and will be removed in a"
353 if mbody != end_template[lang.resources][role][lang.metadata][m]:
354 errors.append('Role %s metadata key %s conflicts.' %
357 role_res = end_template[lang.resources][role]
358 if role_res[lang.type] == 'OS::Heat::StructuredConfig':
359 end_template[lang.resources][role][lang.properties]['config'][m] = mbody
361 end_template[lang.resources][role][lang.metadata][m] = mbody
363 if lang.resources not in end_template:
364 end_template[lang.resources] = {}
365 end_template[lang.resources][role] = rbody
366 if change_image_params:
367 if 'image' in MERGABLE_TYPES[rbody[lang.type]]:
368 ikey = '%sImage' % (role)
369 end_template[lang.resources][role][lang.properties][image_key] = {lang.get_param: ikey}
370 end_template[lang.parameters][ikey] = ikey_val
371 elif rbody[lang.type] == 'FileInclude':
372 # we trust os.path.join to DTRT: if FileInclude path isn't
373 # absolute, join to included_template_dir (./)
374 with open(os.path.join(included_template_dir, rbody['Path'])) as rfile:
375 include_content = yaml.safe_load(rfile.read())
376 subkeys = rbody.get('SubKey','').split('.')
377 while len(subkeys) and subkeys[0]:
378 include_content = include_content[subkeys.pop(0)]
379 for replace_param, replace_value in iter(rbody.get(lang.parameters,
381 include_content = resolve_params(include_content,
384 if lang.resources not in end_template:
385 end_template[lang.resources] = {}
386 end_template[lang.resources][r] = include_content
388 if r in end_template.get(lang.resources, {}):
389 if rbody != end_template[lang.resources][r]:
390 errors.append('Resource %s from %s conflicts' % (r,
393 if lang.resources not in end_template:
394 end_template[lang.resources] = {}
395 end_template[lang.resources][r] = rbody
397 end_template = apply_scaling(end_template, scaling)
398 end_template = apply_maps(end_template)
400 def fix_ref(item, old, new):
401 if isinstance(item, dict):
402 copy_item = dict(item)
403 for k, v in sorted(copy_item.items()):
404 if k == lang.get_resource and v == old:
407 if k == lang.depends_on and v == old:
410 if k == lang.get_attr and isinstance(v, list) and v[0] == old:
415 if k == 'AllowedResources' and isinstance(v, list) and old in v:
421 elif isinstance(item, list):
422 copy_item = list(item)
426 for change in resource_changes:
427 fix_ref(end_template, change[0], change[1])
431 sys.stderr.write("ERROR: %s\n" % e)
432 end_template[lang.description] = ','.join(end_template[lang.description])
433 return yaml.safe_dump(end_template, default_flow_style=False)
435 if __name__ == "__main__":