7 def apply_scaling(template, scaling, in_copies=None):
8 """Apply a set of scaling operations to template.
10 This is a single pass recursive function: for each call we process one
11 dict or list and recurse to handle children containers.
13 Values are handled via scale_value.
15 Keys in dicts are always copied per the scaling rule.
16 Values are either replaced or copied depending on whether the given
17 scaling rule is in in_copies.
19 in_copies = dict(in_copies or {})
20 # Shouldn't be needed but to avoid unexpected side effects/bugs we short
24 if isinstance(template, dict):
26 for key, value in template.items():
27 for prefix, copy_num, new_key in scale_value(
28 key, scaling, in_copies):
30 # e.g. Compute0, 1, Compute1Foo
31 in_copies[prefix] = prefix[:-1] + str(copy_num)
32 if isinstance(value, (dict, list)):
33 new_value = apply_scaling(value, scaling, in_copies)
34 new_template[new_key] = new_value
36 new_values = list(scale_value(value, scaling, in_copies))
37 # We have nowhere to multiply a non-container value of a
38 # dict, so it may be copied or unchanged but not scaled.
39 assert len(new_values) == 1
40 new_template[new_key] = new_values[0][2]
44 elif isinstance(template, list):
46 for value in template:
47 if isinstance(value, (dict, list)):
48 new_template.append(apply_scaling(value, scaling, in_copies))
50 for _, _, new_value in scale_value(value, scaling, in_copies):
51 new_template.append(new_value)
54 raise Exception("apply_scaling called with non-container %r" % template)
57 def scale_value(value, scaling, in_copies):
60 :param value: The value to scale (not a container).
61 :param scaling: The scaling map to use.
62 :param in_copies: What containers we're currently copying.
63 :return: An iterator of the new values for the value as tuples:
64 (prefix, copy_num, value). E.g. Compute0, 1, Compute1Foo
65 prefix and copy_num are only set when:
66 - a prefix in scaling matches value
67 - and that prefix is not in in_copies
69 if isinstance(value, (str, unicode)):
70 for prefix, copies in scaling.items():
71 if not value.startswith(prefix):
73 suffix = value[len(prefix):]
74 if prefix in in_copies:
75 # Adjust to the copy number we're on
76 yield None, None, in_copies[prefix] + suffix
79 for n in range(copies):
80 yield prefix, n, prefix[:-1] + str(n) + suffix
82 yield None, None, value
84 yield None, None, value
87 def parse_scaling(scaling_args):
88 """Translate a list of scaling requests to a dict prefix:count."""
89 scaling_args = scaling_args or []
91 for item in scaling_args:
92 key, value = item.split('=')
94 result[key + '0'] = value
98 def _translate_role(role, master_role, slave_roles):
101 if role == master_role:
103 if role not in slave_roles:
107 def translate_role(role, master_role, slave_roles):
108 r = _translate_role(role, master_role, slave_roles)
109 if not isinstance(r, basestring):
110 raise Exception('%s -> %r' % (role, r))
113 def resolve_params(item, param, value):
114 if item == {'Ref': param}:
116 if isinstance(item, dict):
117 copy_item = dict(item)
118 for k, v in iter(copy_item.items()):
119 item[k] = resolve_params(v, param, value)
120 elif isinstance(item, list):
121 copy_item = list(item)
124 new_item.append(resolve_params(v, param, value))
128 MERGABLE_TYPES = {'OS::Nova::Server':
130 'AWS::EC2::Instance':
131 {'image': 'ImageId'},
132 'AWS::AutoScaling::LaunchConfiguration':
135 INCLUDED_TEMPLATE_DIR = os.getcwd()
138 def resolve_includes(template, params=None):
142 for key, value in iter(template.items()):
143 if key == '__include__':
144 new_params = dict(params) # do not propagate up the stack
145 if not isinstance(value, dict):
146 raise ValueError('__include__ must be a mapping')
147 if 'path' not in value:
148 raise ValueError('__include__ must have path')
149 if 'params' in value:
150 if not isinstance(value['params'], dict):
151 raise ValueError('__include__ params must be a mapping')
152 new_params.update(value['params'])
153 with open(value['path']) as include_file:
154 sub_template = yaml.safe_load(include_file.read())
155 if 'subkey' in value:
156 if ((not isinstance(value['subkey'], int)
157 and not isinstance(sub_template, dict))):
158 raise RuntimeError('subkey requires mapping root or'
159 ' integer for list root')
160 sub_template = sub_template[value['subkey']]
161 for k, v in iter(new_params.items()):
162 sub_template = resolve_params(sub_template, k, v)
163 new_template.update(resolve_includes(sub_template))
165 if isinstance(value, dict):
166 new_template[key] = resolve_includes(value)
168 new_template[key] = value
174 parser = argparse.ArgumentParser()
175 parser.add_argument('templates', nargs='+')
176 parser.add_argument('--master-role', nargs='?',
177 help='Translate slave_roles to this')
178 parser.add_argument('--slave-roles', nargs='*',
179 help='Translate all of these to master_role')
180 parser.add_argument('--included-template-dir', nargs='?',
181 default=INCLUDED_TEMPLATE_DIR,
182 help='Path for resolving included templates')
183 parser.add_argument('--output',
184 help='File to write output to. - for stdout',
186 parser.add_argument('--scale', action="append",
187 help="Names to scale out. Pass Prefix=1 to cause a key Prefix0Foo to "
188 "be copied to Prefix1Foo in the output, and value Prefix0Bar to be"
189 "renamed to Prefix1Bar inside that copy, or copied to Prefix1Bar "
190 "outside of any copy.")
192 '--change-image-params', action='store_true', default=False,
193 help="Change parameters in templates to match resource names. This was "
194 " the default at one time but it causes issues when parameter "
195 " names need to remain stable.")
196 args = parser.parse_args(argv)
197 templates = args.templates
198 scaling = parse_scaling(args.scale)
199 merged_template = merge(templates, args.master_role, args.slave_roles,
200 args.included_template_dir, scaling=scaling,
201 change_image_params=args.change_image_params)
202 if args.output == '-':
203 out_file = sys.stdout
205 out_file = file(args.output, 'wt')
206 out_file.write(merged_template)
209 def merge(templates, master_role=None, slave_roles=None,
210 included_template_dir=INCLUDED_TEMPLATE_DIR,
211 scaling=None, change_image_params=None):
212 scaling = scaling or {}
214 end_template={'HeatTemplateFormatVersion': '2012-12-12',
217 for template_path in templates:
218 template = yaml.safe_load(open(template_path))
219 # Resolve __include__ tags
220 template = resolve_includes(template)
221 end_template['Description'].append(template.get('Description',
223 new_parameters = template.get('Parameters', {})
224 for p, pbody in sorted(new_parameters.items()):
225 if p in end_template.get('Parameters', {}):
226 if pbody != end_template['Parameters'][p]:
227 errors.append('Parameter %s from %s conflicts.' % (p,
230 if 'Parameters' not in end_template:
231 end_template['Parameters'] = {}
232 end_template['Parameters'][p] = pbody
234 new_outputs = template.get('Outputs', {})
235 for o, obody in sorted(new_outputs.items()):
236 if o in end_template.get('Outputs', {}):
237 if pbody != end_template['Outputs'][p]:
238 errors.append('Output %s from %s conflicts.' % (o,
241 if 'Outputs' not in end_template:
242 end_template['Outputs'] = {}
243 end_template['Outputs'][o] = obody
245 new_resources = template.get('Resources', {})
246 for r, rbody in sorted(new_resources.items()):
247 if rbody['Type'] in MERGABLE_TYPES:
248 if change_image_params:
249 if 'image' in MERGABLE_TYPES[rbody['Type']]:
250 image_key = MERGABLE_TYPES[rbody['Type']]['image']
251 # XXX Assuming ImageId is always a Ref
252 ikey_val = end_template['Parameters'][rbody['Properties'][image_key]['Ref']]
253 del end_template['Parameters'][rbody['Properties'][image_key]['Ref']]
254 role = rbody.get('Metadata', {}).get('OpenStack::Role', r)
255 role = translate_role(role, master_role, slave_roles)
257 resource_changes.append((r, role))
258 if role in end_template.get('Resources', {}):
259 new_metadata = rbody.get('Metadata', {})
260 for m, mbody in iter(new_metadata.items()):
261 if m in end_template['Resources'][role].get('Metadata', {}):
262 if m == 'OpenStack::ImageBuilder::Elements':
263 end_template['Resources'][role]['Metadata'][m].extend(mbody)
265 if mbody != end_template['Resources'][role]['Metadata'][m]:
266 errors.append('Role %s metadata key %s conflicts.' %
269 end_template['Resources'][role]['Metadata'][m] = mbody
271 if 'Resources' not in end_template:
272 end_template['Resources'] = {}
273 end_template['Resources'][role] = rbody
274 if change_image_params:
275 if 'image' in MERGABLE_TYPES[rbody['Type']]:
276 ikey = '%sImage' % (role)
277 end_template['Resources'][role]['Properties'][image_key] = {'Ref': ikey}
278 end_template['Parameters'][ikey] = ikey_val
279 elif rbody['Type'] == 'FileInclude':
280 # we trust os.path.join to DTRT: if FileInclude path isn't
281 # absolute, join to included_template_dir (./)
282 with open(os.path.join(included_template_dir, rbody['Path'])) as rfile:
283 include_content = yaml.safe_load(rfile.read())
284 subkeys = rbody.get('SubKey','').split('.')
285 while len(subkeys) and subkeys[0]:
286 include_content = include_content[subkeys.pop(0)]
287 for replace_param, replace_value in iter(rbody.get('Parameters',
289 include_content = resolve_params(include_content,
292 end_template['Resources'][r] = include_content
294 if r in end_template.get('Resources', {}):
295 if rbody != end_template['Resources'][r]:
296 errors.append('Resource %s from %s conflicts' % (r,
299 if 'Resources' not in end_template:
300 end_template['Resources'] = {}
301 end_template['Resources'][r] = rbody
303 end_template = apply_scaling(end_template, scaling)
305 def fix_ref(item, old, new):
306 if isinstance(item, dict):
307 copy_item = dict(item)
308 for k, v in sorted(copy_item.items()):
309 if k == 'Ref' and v == old:
312 if k == 'DependsOn' and v == old:
315 if k == 'Fn::GetAtt' and isinstance(v, list) and v[0] == old:
320 if k == 'AllowedResources' and isinstance(v, list) and old in v:
326 elif isinstance(item, list):
327 copy_item = list(item)
331 for change in resource_changes:
332 fix_ref(end_template, change[0], change[1])
336 sys.stderr.write("ERROR: %s\n" % e)
337 end_template['Description'] = ','.join(end_template['Description'])
338 return yaml.safe_dump(end_template, default_flow_style=False)
340 if __name__ == "__main__":