X-Git-Url: https://gerrit.opnfv.org/gerrit/gitweb?a=blobdiff_plain;f=tripleo_heat_merge%2Fmerge.py;h=49aee6f2eece4fded370cc5e0c8638cd3590dea8;hb=cdfe285f791ede790e8ba1ab246807a9a2af1b41;hp=7b5951a32f73df35fae1d0512cc888cdb559bf41;hpb=c071255ad6bf1fb2c9db6799e96b509295f53fc9;p=apex-tripleo-heat-templates.git diff --git a/tripleo_heat_merge/merge.py b/tripleo_heat_merge/merge.py index 7b5951a3..49aee6f2 100644 --- a/tripleo_heat_merge/merge.py +++ b/tripleo_heat_merge/merge.py @@ -4,6 +4,68 @@ import yaml import argparse +class Cfn(object): + + base_template = { + 'HeatTemplateFormatVersion': '2012-12-12', + 'Description': [] + } + get_resource = 'Ref' + get_param = 'Ref' + description = 'Description' + parameters = 'Parameters' + outputs = 'Outputs' + resources = 'Resources' + type = 'Type' + properties = 'Properties' + metadata = 'Metadata' + depends_on = 'DependsOn' + get_attr = 'Fn::GetAtt' + + +class Hot(object): + + base_template = { + 'heat_template_version': '2013-05-23', + 'description': [] + } + get_resource = 'get_resource' + get_param = 'get_param' + description = 'description' + parameters = 'parameters' + outputs = 'outputs' + resources = 'resources' + type = 'type' + properties = 'properties' + metadata = 'metadata' + depends_on = 'depends_on' + get_attr = 'get_attr' + + +lang = Cfn() + + +def apply_maps(template): + """Apply Merge::Map within template. + + Any dict {'Merge::Map': {'Foo': 'Bar', 'Baz': 'Quux'}} + will resolve to ['Bar', 'Quux'] - that is a dict with key + 'Merge::Map' is replaced entirely by that dict['Merge::Map'].values(). + """ + if isinstance(template, dict): + if 'Merge::Map' in template: + return sorted( + apply_maps(value) for value in template['Merge::Map'].values() + ) + else: + return dict((key, apply_maps(value)) + for key, value in template.items()) + elif isinstance(template, list): + return [apply_maps(item) for item in template] + else: + return template + + def apply_scaling(template, scaling, in_copies=None): """Apply a set of scaling operations to template. @@ -12,9 +74,14 @@ def apply_scaling(template, scaling, in_copies=None): Values are handled via scale_value. - Keys in dicts are always copied per the scaling rule. + Keys in dicts are copied per the scaling rule. Values are either replaced or copied depending on whether the given scaling rule is in in_copies. + + in_copies is reset to None when a dict {'Merge::Map': someobject} is + encountered. + + :param scaling: A dict of prefix -> (count, blacklists). """ in_copies = dict(in_copies or {}) # Shouldn't be needed but to avoid unexpected side effects/bugs we short @@ -22,6 +89,8 @@ def apply_scaling(template, scaling, in_copies=None): if not scaling: return template if isinstance(template, dict): + if 'Merge::Map' in template: + in_copies = None new_template = {} for key, value in template.items(): for prefix, copy_num, new_key in scale_value( @@ -58,7 +127,7 @@ def scale_value(value, scaling, in_copies): """Scale out a value. :param value: The value to scale (not a container). - :param scaling: The scaling map to use. + :param scaling: The scaling map (prefix-> (copies, blacklist) to use. :param in_copies: What containers we're currently copying. :return: An iterator of the new values for the value as tuples: (prefix, copy_num, value). E.g. Compute0, 1, Compute1Foo @@ -67,7 +136,7 @@ def scale_value(value, scaling, in_copies): - and that prefix is not in in_copies """ if isinstance(value, (str, unicode)): - for prefix, copies in scaling.items(): + for prefix, (copies, blacklist) in scaling.items(): if not value.startswith(prefix): continue suffix = value[len(prefix):] @@ -77,7 +146,8 @@ def scale_value(value, scaling, in_copies): return else: for n in range(copies): - yield prefix, n, prefix[:-1] + str(n) + suffix + if n not in blacklist: + yield prefix, n, prefix[:-1] + str(n) + suffix return yield None, None, value else: @@ -89,9 +159,11 @@ def parse_scaling(scaling_args): scaling_args = scaling_args or [] result = {} for item in scaling_args: - key, value = item.split('=') - value = int(value) - result[key + '0'] = value + key, values = item.split('=') + values = values.split(',') + value = int(values[0]) + blacklist = frozenset(int(v) for v in values[1:] if v) + result[key + '0'] = value, blacklist return result @@ -111,7 +183,7 @@ def translate_role(role, master_role, slave_roles): return r def resolve_params(item, param, value): - if item == {'Ref': param}: + if item in ({lang.get_param: param}, {lang.get_resource: param}): return value if isinstance(item, dict): copy_item = dict(item) @@ -184,15 +256,31 @@ def main(argv=None): help='File to write output to. - for stdout', default='-') parser.add_argument('--scale', action="append", - help="Names to scale out. Pass Prefix=1 to cause a key Prefix0Foo to " + help="Names to scale out. Pass Prefix=2 to cause a key Prefix0Foo to " "be copied to Prefix1Foo in the output, and value Prefix0Bar to be" "renamed to Prefix1Bar inside that copy, or copied to Prefix1Bar " - "outside of any copy.") + "outside of any copy. Pass Prefix=3,1 to cause Prefix1* to be elided" + "when scaling Prefix out. Prefix=4,1,2 will likewise elide Prefix1 and" + "Prefix2.") + parser.add_argument( + '--change-image-params', action='store_true', default=False, + help="Change parameters in templates to match resource names. This was " + " the default at one time but it causes issues when parameter " + " names need to remain stable.") + parser.add_argument( + '--hot', action='store_true', default=False, + help="Assume source templates are in the HOT format, and generate a " + "HOT template artifact.") args = parser.parse_args(argv) + if args.hot: + global lang + lang = Hot() + templates = args.templates scaling = parse_scaling(args.scale) merged_template = merge(templates, args.master_role, args.slave_roles, - args.included_template_dir, scaling=scaling) + args.included_template_dir, scaling=scaling, + change_image_params=args.change_image_params) if args.output == '-': out_file = sys.stdout else: @@ -202,73 +290,78 @@ def main(argv=None): def merge(templates, master_role=None, slave_roles=None, included_template_dir=INCLUDED_TEMPLATE_DIR, - scaling=None): + scaling=None, change_image_params=None): scaling = scaling or {} errors = [] - end_template={'HeatTemplateFormatVersion': '2012-12-12', - 'Description': []} + end_template = dict(lang.base_template) resource_changes=[] for template_path in templates: template = yaml.safe_load(open(template_path)) # Resolve __include__ tags template = resolve_includes(template) - end_template['Description'].append(template.get('Description', + end_template[lang.description].append(template.get(lang.description, template_path)) - new_parameters = template.get('Parameters', {}) + new_parameters = template.get(lang.parameters, {}) for p, pbody in sorted(new_parameters.items()): - if p in end_template.get('Parameters', {}): - if pbody != end_template['Parameters'][p]: + if p in end_template.get(lang.parameters, {}): + if pbody != end_template[lang.parameters][p]: errors.append('Parameter %s from %s conflicts.' % (p, template_path)) continue - if 'Parameters' not in end_template: - end_template['Parameters'] = {} - end_template['Parameters'][p] = pbody + if lang.parameters not in end_template: + end_template[lang.parameters] = {} + end_template[lang.parameters][p] = pbody - new_outputs = template.get('Outputs', {}) + new_outputs = template.get(lang.outputs, {}) for o, obody in sorted(new_outputs.items()): - if o in end_template.get('Outputs', {}): - if pbody != end_template['Outputs'][p]: + if o in end_template.get(lang.outputs, {}): + if pbody != end_template[lang.outputs][p]: errors.append('Output %s from %s conflicts.' % (o, template_path)) continue - if 'Outputs' not in end_template: - end_template['Outputs'] = {} - end_template['Outputs'][o] = obody + if lang.outputs not in end_template: + end_template[lang.outputs] = {} + end_template[lang.outputs][o] = obody - new_resources = template.get('Resources', {}) + new_resources = template.get(lang.resources, {}) for r, rbody in sorted(new_resources.items()): - if rbody['Type'] in MERGABLE_TYPES: - if 'image' in MERGABLE_TYPES[rbody['Type']]: - image_key = MERGABLE_TYPES[rbody['Type']]['image'] - # XXX Assuming ImageId is always a Ref - ikey_val = end_template['Parameters'][rbody['Properties'][image_key]['Ref']] - del end_template['Parameters'][rbody['Properties'][image_key]['Ref']] - role = rbody.get('Metadata', {}).get('OpenStack::Role', r) + if rbody[lang.type] in MERGABLE_TYPES: + if change_image_params: + if 'image' in MERGABLE_TYPES[rbody[lang.type]]: + image_key = MERGABLE_TYPES[rbody[lang.type]]['image'] + # XXX Assuming ImageId is always a Ref + ikey_val = end_template[lang.parameters][rbody[lang.properties][image_key][lang.get_param]] + del end_template[lang.parameters][rbody[lang.properties][image_key][lang.get_param]] + role = rbody.get(lang.metadata, {}).get('OpenStack::Role', r) role = translate_role(role, master_role, slave_roles) if role != r: resource_changes.append((r, role)) - if role in end_template.get('Resources', {}): - new_metadata = rbody.get('Metadata', {}) + if role in end_template.get(lang.resources, {}): + new_metadata = rbody.get(lang.metadata, {}) for m, mbody in iter(new_metadata.items()): - if m in end_template['Resources'][role].get('Metadata', {}): + if m in end_template[lang.resources][role].get(lang.metadata, {}): if m == 'OpenStack::ImageBuilder::Elements': - end_template['Resources'][role]['Metadata'][m].extend(mbody) + end_template[lang.resources][role][lang.metadata][m].extend(mbody) continue - if mbody != end_template['Resources'][role]['Metadata'][m]: + if mbody != end_template[lang.resources][role][lang.metadata][m]: errors.append('Role %s metadata key %s conflicts.' % (role, m)) continue - end_template['Resources'][role]['Metadata'][m] = mbody + role_res = end_template[lang.resources][role] + if role_res[lang.type] == 'OS::Heat::StructuredConfig': + end_template[lang.resources][role][lang.properties]['config'][m] = mbody + else: + end_template[lang.resources][role][lang.metadata][m] = mbody continue - if 'Resources' not in end_template: - end_template['Resources'] = {} - end_template['Resources'][role] = rbody - if 'image' in MERGABLE_TYPES[rbody['Type']]: - ikey = '%sImage' % (role) - end_template['Resources'][role]['Properties'][image_key] = {'Ref': ikey} - end_template['Parameters'][ikey] = ikey_val - elif rbody['Type'] == 'FileInclude': + if lang.resources not in end_template: + end_template[lang.resources] = {} + end_template[lang.resources][role] = rbody + if change_image_params: + if 'image' in MERGABLE_TYPES[rbody[lang.type]]: + ikey = '%sImage' % (role) + end_template[lang.resources][role][lang.properties][image_key] = {lang.get_param: ikey} + end_template[lang.parameters][ikey] = ikey_val + elif rbody[lang.type] == 'FileInclude': # we trust os.path.join to DTRT: if FileInclude path isn't # absolute, join to included_template_dir (./) with open(os.path.join(included_template_dir, rbody['Path'])) as rfile: @@ -276,35 +369,38 @@ def merge(templates, master_role=None, slave_roles=None, subkeys = rbody.get('SubKey','').split('.') while len(subkeys) and subkeys[0]: include_content = include_content[subkeys.pop(0)] - for replace_param, replace_value in iter(rbody.get('Parameters', + for replace_param, replace_value in iter(rbody.get(lang.parameters, {}).items()): include_content = resolve_params(include_content, replace_param, replace_value) - end_template['Resources'][r] = include_content + if lang.resources not in end_template: + end_template[lang.resources] = {} + end_template[lang.resources][r] = include_content else: - if r in end_template.get('Resources', {}): - if rbody != end_template['Resources'][r]: + if r in end_template.get(lang.resources, {}): + if rbody != end_template[lang.resources][r]: errors.append('Resource %s from %s conflicts' % (r, template_path)) continue - if 'Resources' not in end_template: - end_template['Resources'] = {} - end_template['Resources'][r] = rbody + if lang.resources not in end_template: + end_template[lang.resources] = {} + end_template[lang.resources][r] = rbody end_template = apply_scaling(end_template, scaling) + end_template = apply_maps(end_template) def fix_ref(item, old, new): if isinstance(item, dict): copy_item = dict(item) for k, v in sorted(copy_item.items()): - if k == 'Ref' and v == old: + if k == lang.get_resource and v == old: item[k] = new continue - if k == 'DependsOn' and v == old: + if k == lang.depends_on and v == old: item[k] = new continue - if k == 'Fn::GetAtt' and isinstance(v, list) and v[0] == old: + if k == lang.get_attr and isinstance(v, list) and v[0] == old: new_list = list(v) new_list[0] = new item[k] = new_list @@ -326,7 +422,7 @@ def merge(templates, master_role=None, slave_roles=None, if errors: for e in errors: sys.stderr.write("ERROR: %s\n" % e) - end_template['Description'] = ','.join(end_template['Description']) + end_template[lang.description] = ','.join(end_template[lang.description]) return yaml.safe_dump(end_template, default_flow_style=False) if __name__ == "__main__":