X-Git-Url: https://gerrit.opnfv.org/gerrit/gitweb?a=blobdiff_plain;f=tools%2Fyaml-validate.py;h=33d12eec0c06f0fd82e8f7f83c5577eecac29e54;hb=05f773006a0f8f1ea50ba23bfaa16ef68ef6ec55;hp=ff215fbae6ae299b31e66e407cd06990d090584b;hpb=d24690765c3fd2703d5aa234767a3389ab1d28b6;p=apex-tripleo-heat-templates.git diff --git a/tools/yaml-validate.py b/tools/yaml-validate.py index ff215fba..33d12eec 100755 --- a/tools/yaml-validate.py +++ b/tools/yaml-validate.py @@ -18,7 +18,7 @@ import yaml required_params = ['EndpointMap', 'ServiceNetMap', 'DefaultPasswords', - 'RoleName', 'RoleParameters'] + 'RoleName', 'RoleParameters', 'ServiceData'] # NOTE(bnemec): The duplication in this list is intentional. For the # transition to generated environments we have two copies of these files, @@ -38,7 +38,110 @@ OPTIONAL_DOCKER_SECTIONS = ['docker_puppet_tasks', 'upgrade_tasks', 'metadata_settings', 'kolla_config'] REQUIRED_DOCKER_PUPPET_CONFIG_SECTIONS = ['config_volume', 'step_config', 'config_image'] -OPTIONAL_DOCKER_PUPPET_CONFIG_SECTIONS = [ 'puppet_tags' ] +OPTIONAL_DOCKER_PUPPET_CONFIG_SECTIONS = [ 'puppet_tags', 'volumes' ] +# Mapping of parameter names to a list of the fields we should _not_ enforce +# consistency across files on. This should only contain parameters whose +# definition we cannot change for backwards compatibility reasons. New +# parameters to the templates should not be added to this list. +PARAMETER_DEFINITION_EXCLUSIONS = {'ManagementNetCidr': ['default'], + 'ManagementAllocationPools': ['default'], + 'ExternalNetCidr': ['default'], + 'ExternalAllocationPools': ['default'], + 'StorageNetCidr': ['default'], + 'StorageAllocationPools': ['default'], + 'StorageMgmtNetCidr': ['default', + # FIXME + 'description'], + 'StorageMgmtAllocationPools': ['default'], + 'TenantNetCidr': ['default'], + 'TenantAllocationPools': ['default'], + 'InternalApiNetCidr': ['default'], + 'UpdateIdentifier': ['description'], + # TODO(bnemec): Address these existing + # inconsistencies. + 'NeutronMetadataProxySharedSecret': [ + 'description', 'hidden'], + 'ServiceNetMap': ['description', 'default'], + 'RedisPassword': ['description'], + 'EC2MetadataIp': ['default'], + 'network': ['default'], + 'ControlPlaneIP': ['default', + 'description'], + 'ControlPlaneIp': ['default', + 'description'], + 'NeutronBigswitchLLDPEnabled': ['default'], + 'NeutronEnableL2Pop': ['description'], + 'NeutronWorkers': ['description'], + 'TenantIpSubnet': ['description'], + 'ExternalNetName': ['description'], + 'AdminToken': ['description'], + 'ControlPlaneDefaultRoute': ['default'], + 'StorageMgmtNetName': ['description'], + 'ServerMetadata': ['description'], + 'InternalApiIpUri': ['description'], + 'UpgradeLevelNovaCompute': ['default'], + 'StorageMgmtIpUri': ['description'], + 'server': ['description'], + 'servers': ['description'], + 'FixedIPs': ['description'], + 'ExternalIpSubnet': ['description'], + 'NeutronBridgeMappings': ['description'], + 'ExtraConfig': ['description'], + 'InternalApiIpSubnet': ['description'], + 'DefaultPasswords': ['description', + 'default'], + 'BondInterfaceOvsOptions': ['description', + 'default', + 'constraints'], + 'KeyName': ['constraints'], + 'TenantNetName': ['description'], + 'StorageIpSubnet': ['description'], + 'OVNSouthboundServerPort': ['description'], + 'ExternalInterfaceDefaultRoute': + ['description', 'default'], + 'ExternalIpUri': ['description'], + 'IPPool': ['description'], + 'ControlPlaneNetwork': ['description'], + 'SSLCertificate': ['description', + 'default', + 'hidden'], + 'HostCpusList': ['default', 'constraints'], + 'InternalApiAllocationPools': ['default'], + 'NodeIndex': ['description'], + 'SwiftPassword': ['description'], + 'name': ['description', 'default'], + 'StorageNetName': ['description'], + 'ManagementNetName': ['description'], + 'NeutronPublicInterface': ['description'], + 'RoleParameters': ['description'], + 'AdminPassword': ['description', 'hidden'], + 'ManagementInterfaceDefaultRoute': + ['default'], + 'NovaPassword': ['description'], + 'image': ['description', 'default'], + 'NeutronBigswitchAgentEnabled': ['default'], + 'EndpointMap': ['description', 'default'], + 'DockerManilaConfigImage': ['description', + 'default'], + 'NetworkName': ['default', 'description'], + 'StorageIpUri': ['description'], + 'InternalApiNetName': ['description'], + 'NeutronTunnelTypes': ['description'], + 'replacement_policy': ['default'], + 'StorageMgmtIpSubnet': ['description'], + 'CloudDomain': ['description', 'default'], + 'key_name': ['default', 'description'], + 'EnableLoadBalancer': ['description'], + 'ControllerExtraConfig': ['description'], + 'NovaComputeExtraConfig': ['description'], + 'controllerExtraConfig': ['description'], + 'DockerSwiftConfigImage': ['default'], + } + +PREFERRED_CAMEL_CASE = { + 'ec2api': 'Ec2Api', + 'haproxy': 'HAProxy', +} def exit_usage(): @@ -46,6 +149,11 @@ def exit_usage(): sys.exit(1) +def to_camel_case(string): + return PREFERRED_CAMEL_CASE.get(string, ''.join(s.capitalize() or '_' for + s in string.split('_'))) + + def get_base_endpoint_map(filename): try: tpl = yaml.load(open(filename).read()) @@ -75,14 +183,30 @@ def validate_hci_compute_services_default(env_filename, env_tpl): env_services_list = env_tpl['parameter_defaults']['ComputeServices'] env_services_list.remove('OS::TripleO::Services::CephOSD') roles_filename = os.path.join(os.path.dirname(env_filename), - '../roles_data.yaml') + '../roles/Compute.yaml') roles_tpl = yaml.load(open(roles_filename).read()) for role in roles_tpl: if role['name'] == 'Compute': roles_services_list = role['ServicesDefault'] if sorted(env_services_list) != sorted(roles_services_list): - print('ERROR: ComputeServices in %s is different ' - 'from ServicesDefault in roles_data.yaml' % env_filename) + print('ERROR: ComputeServices in %s is different from ' + 'ServicesDefault in roles/Compute.yaml' % env_filename) + return 1 + return 0 + + +def validate_hci_computehci_role(hci_role_filename, hci_role_tpl): + compute_role_filename = os.path.join(os.path.dirname(hci_role_filename), + './Compute.yaml') + compute_role_tpl = yaml.load(open(compute_role_filename).read()) + compute_role_services = compute_role_tpl[0]['ServicesDefault'] + for role in hci_role_tpl: + if role['name'] == 'ComputeHCI': + hci_role_services = role['ServicesDefault'] + hci_role_services.remove('OS::TripleO::Services::CephOSD') + if sorted(hci_role_services) != sorted(compute_role_services): + print('ERROR: ServicesDefault in %s is different from' + 'ServicesDefault in roles/Compute.yaml' % hci_role_filename) return 1 return 0 @@ -170,6 +294,32 @@ def validate_docker_service(filename, tpl): % (key, filename)) return 1 + config_volume = puppet_config.get('config_volume') + expected_config_image_parameter = "Docker%sConfigImage" % to_camel_case(config_volume) + if config_volume and not expected_config_image_parameter in tpl.get('parameters', []): + print('ERROR: Missing %s heat parameter for %s config_volume.' + % (expected_config_image_parameter, config_volume)) + return 1 + + if 'docker_config' in role_data: + docker_config = role_data['docker_config'] + for _, step in docker_config.items(): + if not isinstance(step, dict): + # NOTE(mandre) this skips everything that is not a dict + # so we may ignore some containers definitions if they + # are in a map_merge for example + continue + for _, container in step.items(): + if not isinstance(container, dict): + continue + command = container.get('command', '') + if isinstance(command, list): + command = ' '.join(map(str, command)) + if 'bootstrap_host_exec' in command \ + and container.get('user') != 'root': + print('ERROR: bootstrap_host_exec needs to run as the root user.') + return 1 + if 'parameters' in tpl: for param in required_params: if param not in tpl['parameters']: @@ -211,7 +361,30 @@ def validate_service(filename, tpl): return 0 -def validate(filename): +def validate(filename, param_map): + """Validate a Heat template + + :param filename: The path to the file to validate + :param param_map: A dict which will be populated with the details of the + parameters in the template. The dict will have the + following structure: + + {'ParameterName': [ + {'filename': ./file1.yaml, + 'data': {'description': '', + 'type': string, + 'default': '', + ...} + }, + {'filename': ./file2.yaml, + 'data': {'description': '', + 'type': string, + 'default': '', + ...} + }, + ... + ]} + """ print('Validating %s' % filename) retval = 0 try: @@ -235,12 +408,17 @@ def validate(filename): if filename.endswith('hyperconverged-ceph.yaml'): retval = validate_hci_compute_services_default(filename, tpl) + if filename.startswith('./roles/ComputeHCI.yaml'): + retval = validate_hci_computehci_role(filename, tpl) + except Exception: print(traceback.format_exc()) return 1 # yaml is OK, now walk the parameters and output a warning for unused ones if 'heat_template_version' in tpl: - for p in tpl.get('parameters', {}): + for p, data in tpl.get('parameters', {}).items(): + definition = {'data': data, 'filename': filename} + param_map.setdefault(p, []).append(definition) if p in required_params: continue str_p = '\'%s\'' % p @@ -260,14 +438,17 @@ exit_val = 0 failed_files = [] base_endpoint_map = None env_endpoint_maps = list() +param_map = {} for base_path in path_args: if os.path.isdir(base_path): for subdir, dirs, files in os.walk(base_path): + if '.tox' in dirs: + dirs.remove('.tox') for f in files: if f.endswith('.yaml') and not f.endswith('.j2.yaml'): file_path = os.path.join(subdir, f) - failed = validate(file_path) + failed = validate(file_path, param_map) if failed: failed_files.append(file_path) exit_val |= failed @@ -278,7 +459,7 @@ for base_path in path_args: if env_endpoint_map: env_endpoint_maps.append(env_endpoint_map) elif os.path.isfile(base_path) and base_path.endswith('.yaml'): - failed = validate(base_path) + failed = validate(base_path, param_map) if failed: failed_files.append(base_path) exit_val |= failed @@ -310,6 +491,32 @@ else: failed_files.extend(set(envs_containing_endpoint_map) - matched_files) exit_val |= 1 +# Validate that duplicate parameters defined in multiple files all have the +# same definition. +mismatch_count = 0 +for p, defs in param_map.items(): + # Nothing to validate if the parameter is only defined once + if len(defs) == 1: + continue + check_data = [d['data'] for d in defs] + # Override excluded fields so they don't affect the result + exclusions = PARAMETER_DEFINITION_EXCLUSIONS.get(p, []) + ex_dict = {} + for field in exclusions: + ex_dict[field] = 'IGNORED' + for d in check_data: + d.update(ex_dict) + # If all items in the list are not == the first, then the check fails + if check_data.count(check_data[0]) != len(check_data): + mismatch_count += 1 + exit_val |= 1 + failed_files.extend([d['filename'] for d in defs]) + print('Mismatched parameter definitions found for "%s"' % p) + print('Definitions found:') + for d in defs: + print(' %s:\n %s' % (d['filename'], d['data'])) +print('Mismatched parameter definitions: %d' % mismatch_count) + if failed_files: print('Validation failed on:') for f in failed_files: