pip-log.txt
# Unit test / coverage reports
+cover
.coverage
.tox
+.testrepository
nosetests.xml
# Translations
--- /dev/null
+[DEFAULT]
+test_command=OS_STDOUT_CAPTURE=1 OS_STDERR_CAPTURE=1 OS_TEST_TIMEOUT=60 OS_LOG_CAPTURE=1 ${PYTHON:-python} -m subunit.run discover -t ./tripleo_heat_templates ./tripleo_heat_templates $LISTOPT $IDOPTION
+test_id_option=--load-list $IDFILE
+test_list_option=--list
--- /dev/null
+# *******************************************************************
+# This file was created automatically by the sample environment
+# generator. Developers should use `tox -e genconfig` to update it.
+# Users are recommended to make changes to a copy of the file instead
+# of the original, if any customizations are needed.
+# *******************************************************************
+# title: Custom Hostnames
+# description: |
+# Hostname format for each role
+# Note %index% is translated into the index of the node, e.g 0/1/2 etc
+# and %stackname% is replaced with OS::stack_name in the template below.
+# If you want to use the heat generated names, pass '' (empty string).
+parameter_defaults:
+ # Format for BlockStorage node hostnames Note %index% is translated into the index of the node, e.g 0/1/2 etc and %stackname% is replaced with the stack name e.g overcloud
+ # Type: string
+ BlockStorageHostnameFormat: '%stackname%-blockstorage-%index%'
+
+ # Format for CephStorage node hostnames Note %index% is translated into the index of the node, e.g 0/1/2 etc and %stackname% is replaced with the stack name e.g overcloud
+ # Type: string
+ CephStorageHostnameFormat: '%stackname%-cephstorage-%index%'
+
+ # Format for Compute node hostnames Note %index% is translated into the index of the node, e.g 0/1/2 etc and %stackname% is replaced with the stack name e.g overcloud
+ # Type: string
+ ComputeHostnameFormat: '%stackname%-novacompute-%index%'
+
+ # Format for Controller node hostnames Note %index% is translated into the index of the node, e.g 0/1/2 etc and %stackname% is replaced with the stack name e.g overcloud
+ # Type: string
+ ControllerHostnameFormat: '%stackname%-controller-%index%'
+
+ # Format for ObjectStorage node hostnames Note %index% is translated into the index of the node, e.g 0/1/2 etc and %stackname% is replaced with the stack name e.g overcloud
+ # Type: string
+ ObjectStorageHostnameFormat: '%stackname%-objectstorage-%index%'
+
--- /dev/null
+Sample Environment Generator
+----------------------------
+
+This is a tool to automate the generation of our sample environment
+files. It takes a yaml file as input, and based on the environments
+defined in that file generates a number of sample environment files
+from the parameters in the Heat templates.
+
+Usage
+=====
+
+The simplest case is when an existing sample environment needs to be
+updated to reflect changes in the templates. Use the tox ``genconfig``
+target to do this::
+
+ tox -e genconfig
+
+.. note:: The tool should be run from the root directory of the
+ ``tripleo-heat-templates`` project.
+
+If a new sample environment is needed, it should be added to the
+``sample-env-generator/sample-environments.yaml`` file. The existing
+entries in the file can be used as examples, and a more detailed
+explanation of the different available keys is below:
+
+- **name**: the output file will be this name + .yaml, in the
+ ``environments`` directory.
+- **title**: a human-readable title for the environment.
+- **description**: A description of the environment. Will be included
+ as a comment at the top of the sample file.
+- **files**: The Heat templates containing the parameter definitions
+ for the environment. Should be specified as a path relative to the
+ root of the ``tripleo-heat-templates`` project. For example:
+ ``puppet/extraconfig/tls/tls-cert-inject.yaml:``. Each filename
+ should be a YAML dictionary that contains a ``parameters`` entry.
+- **parameters**: There should be one ``parameters`` entry per file in the
+ ``files`` section (see the example configuration below).
+ This can be either a list of parameters related to
+ the environment, which is necessary for templates like
+ overcloud.yaml, or the string 'all', which indicates that all
+ parameters from the file should be included.
+- **static**: Can be used to specify that certain parameters must
+ not be changed. Examples would be the EnableSomething params
+ in the templates. When writing a sample config for Something,
+ ``EnableSomething: True`` would be a static param, since it
+ would be nonsense to include the environment with it set to any other
+ value.
+- **sample_values**: Sometimes it is useful to include a sample value
+ for a parameter that is not the parameter's actual default.
+ An example of this is the SSLCertificate param in the enable-tls
+ environment file.
+- **resource_registry**: Many environments also need to pass
+ resource_registry entries when they are used. This can be used
+ to specify that in the configuration file.
+
+Some behavioral notes:
+
+- Parameters without default values will be marked as mandatory to indicate
+ that the user must set a value for them.
+- It is no longer recommended to set parameters using the ``parameters``
+ section. Instead, all parameters should be set as ``parameter_defaults``
+ which will work regardless of whether the parameter is top-level or nested.
+ Therefore, the tool will always set parameters in the ``parameter_defaults``
+ section.
+- Parameters whose name begins with the _ character are treated as private.
+ This indicates that the parameter value will be passed in from another
+ template and does not need to be exposed directly to the user.
+
+If adding a new environment, don't forget to add the new file to the
+git repository so it will be included with the review.
+
+Example
+=======
+
+Given a Heat template named ``example.yaml`` that looks like::
+
+ parameters:
+ EnableExample:
+ default: False
+ description: Enable the example feature
+ type: boolean
+ ParamOne:
+ default: one
+ description: First example param
+ type: string
+ ParamTwo:
+ description: Second example param
+ type: number
+ _PrivateParam:
+ default: does not matter
+ description: Will not show up
+ type: string
+
+And an environment generator entry that looks like::
+
+ environments:
+ -
+ name: example
+ title: Example Environment
+ description: |
+ An example environment demonstrating how to use the sample
+ environment generator. This text will be included at the top
+ of the generated file as a comment.
+ files:
+ example.yaml:
+ parameters: all
+ sample_values:
+ EnableExample: True
+ static:
+ - EnableExample
+ resource_registry:
+ OS::TripleO::ExampleData: ../extraconfig/example.yaml
+
+The generated environment file would look like::
+
+ # *******************************************************************
+ # This file was created automatically by the sample environment
+ # generator. Developers should use `tox -e genconfig` to update it.
+ # Users are recommended to make changes to a copy of the file instead
+ # of the original, if any customizations are needed.
+ # *******************************************************************
+ # title: Example Environment
+ # description: |
+ # An example environment demonstrating how to use the sample
+ # environment generator. This text will be included at the top
+ # of the generated file as a comment.
+ parameter_defaults:
+ # First example param
+ # Type: string
+ ParamOne: one
+
+ # Second example param
+ # Mandatory. This parameter must be set by the user.
+ # Type: number
+ ParamTwo: <None>
+
+ # ******************************************************
+ # Static parameters - these are values that must be
+ # included in the environment but should not be changed.
+ # ******************************************************
+ # Enable the example feature
+ # Type: boolean
+ EnableExample: True
+
+ # *********************
+ # End static parameters
+ # *********************
+ resource_registry:
+ OS::TripleO::ExampleData: ../extraconfig/example.yaml
--- /dev/null
+environments:
+ -
+ name: predictable-placement/custom-hostnames
+ title: Custom Hostnames
+ files:
+ overcloud.yaml:
+ parameters:
+ - ControllerHostnameFormat
+ - ComputeHostnameFormat
+ - BlockStorageHostnameFormat
+ - ObjectStorageHostnameFormat
+ - CephStorageHostnameFormat
+ description: |
+ Hostname format for each role
+ Note %index% is translated into the index of the node, e.g 0/1/2 etc
+ and %stackname% is replaced with OS::stack_name in the template below.
+ If you want to use the heat generated names, pass '' (empty string).
sphinx!=1.6.1,>=1.5.1 # BSD
oslosphinx>=4.7.0 # Apache-2.0
reno!=2.3.1,>=1.8.0 # Apache-2.0
+coverage>=4.0,!=4.4 # Apache-2.0
+fixtures>=3.0.0 # Apache-2.0/BSD
+python-subunit>=0.0.18 # Apache-2.0/BSD
+testrepository>=0.0.18 # Apache-2.0/BSD
+testscenarios>=0.4 # Apache-2.0/BSD
+testtools>=1.4.0 # MIT
+mock>=2.0 # BSD
+oslotest>=1.10.0 # Apache-2.0
[tox]
minversion = 1.6
skipsdist = True
+envlist = py35,py27,pep8
[testenv]
usedevelop = True
install_command = pip install -c{env:UPPER_CONSTRAINTS_FILE:https://git.openstack.org/cgit/openstack/requirements/plain/upper-constraints.txt} {opts} {packages}
deps = -r{toxinidir}/requirements.txt
-r{toxinidir}/test-requirements.txt
+commands = python setup.py testr --slowest --testr-args='{posargs}'
[testenv:venv]
commands = {posargs}
[testenv:releasenotes]
commands = bash -c tools/releasenotes_tox.sh
+
+[testenv:cover]
+commands = python setup.py test --coverage --coverage-package-name=tripleo_heat_templates --testr-args='{posargs}'
+
+[testenv:genconfig]
+commands =
+ python ./tools/process-templates.py
+ python ./tripleo_heat_templates/environment_generator.py sample-env-generator/sample-environments.yaml
--- /dev/null
+#!/usr/bin/env python
+
+# Copyright 2015 Red Hat Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import errno
+import os
+import sys
+import yaml
+
+
+_PARAM_FORMAT = u""" # %(description)s
+ %(mandatory)s# Type: %(type)s
+ %(name)s: %(default)s
+"""
+_STATIC_MESSAGE_START = (
+ ' # ******************************************************\n'
+ ' # Static parameters - these are values that must be\n'
+ ' # included in the environment but should not be changed.\n'
+ ' # ******************************************************\n'
+ )
+_STATIC_MESSAGE_END = (' # *********************\n'
+ ' # End static parameters\n'
+ ' # *********************\n'
+ )
+_FILE_HEADER = (
+ '# *******************************************************************\n'
+ '# This file was created automatically by the sample environment\n'
+ '# generator. Developers should use `tox -e genconfig` to update it.\n'
+ '# Users are recommended to make changes to a copy of the file instead\n'
+ '# of the original, if any customizations are needed.\n'
+ '# *******************************************************************\n'
+ )
+# Certain parameter names can't be changed, but shouldn't be shown because
+# they are never intended for direct user input.
+_PRIVATE_OVERRIDES = ['server', 'servers', 'NodeIndex']
+
+
+def _create_output_dir(target_file):
+ try:
+ os.makedirs(os.path.dirname(target_file))
+ except OSError as e:
+ if e.errno == errno.EEXIST:
+ pass
+ else:
+ raise
+
+
+def _generate_environment(input_env, parent_env=None):
+ if parent_env is None:
+ parent_env = {}
+ env = dict(parent_env)
+ env.update(input_env)
+ parameter_defaults = {}
+ param_names = []
+ for template_file, template_data in env['files'].items():
+ with open(template_file) as f:
+ f_data = yaml.safe_load(f)
+ f_params = f_data['parameters']
+ parameter_defaults.update(f_params)
+ if template_data['parameters'] == 'all':
+ new_names = [k for k, v in f_params.items()]
+ else:
+ new_names = template_data['parameters']
+ missing_params = [name for name in new_names
+ if name not in f_params]
+ if missing_params:
+ raise RuntimeError('Did not find specified parameter names %s '
+ 'in file %s for environment %s' %
+ (missing_params, template_file,
+ env['name']))
+ param_names += new_names
+
+ static_names = env.get('static', [])
+ static_defaults = {k: v for k, v in parameter_defaults.items()
+ if k in param_names and
+ k in static_names
+ }
+ parameter_defaults = {k: v for k, v in parameter_defaults.items()
+ if k in param_names and
+ k not in _PRIVATE_OVERRIDES and
+ not k.startswith('_') and
+ k not in static_names
+ }
+ for k, v in env.get('sample_values', {}).items():
+ if k in parameter_defaults:
+ parameter_defaults[k]['sample'] = v
+ if k in static_defaults:
+ static_defaults[k]['sample'] = v
+
+ def write_sample_entry(f, name, value):
+ default = value.get('default')
+ mandatory = ''
+ if default is None:
+ mandatory = ('# Mandatory. This parameter must be set by the '
+ 'user.\n ')
+ default = '<None>'
+ if value.get('sample') is not None:
+ default = value['sample']
+ if default == '':
+ default = "''"
+ try:
+ # If the default value is something like %index%, yaml won't
+ # parse the output correctly unless we wrap it in quotes.
+ # However, not all default values can be wrapped so we need to
+ # do it conditionally.
+ if default.startswith('%'):
+ default = "'%s'" % default
+ except AttributeError:
+ pass
+
+ values = {'name': name,
+ 'type': value['type'],
+ 'description':
+ value.get('description', '').rstrip().replace('\n',
+ '\n # '),
+ 'default': default,
+ 'mandatory': mandatory,
+ }
+ f.write(_PARAM_FORMAT % values + '\n')
+
+ target_file = os.path.join('environments', env['name'] + '.yaml')
+ _create_output_dir(target_file)
+ with open(target_file, 'w') as env_file:
+ env_file.write(_FILE_HEADER)
+ # TODO(bnemec): Once Heat allows the title and description to live in
+ # the environment itself, uncomment these entries and make them
+ # top-level keys in the YAML.
+ env_title = env.get('title', '')
+ env_file.write(u'# title: %s\n' % env_title)
+ env_desc = env.get('description', '')
+ env_file.write(u'# description: |\n')
+ for line in env_desc.splitlines():
+ env_file.write(u'# %s\n' % line)
+
+ if parameter_defaults:
+ env_file.write(u'parameter_defaults:\n')
+ for name, value in sorted(parameter_defaults.items()):
+ write_sample_entry(env_file, name, value)
+ if static_defaults:
+ env_file.write(_STATIC_MESSAGE_START)
+ for name, value in sorted(static_defaults.items()):
+ write_sample_entry(env_file, name, value)
+ if static_defaults:
+ env_file.write(_STATIC_MESSAGE_END)
+
+ if env.get('resource_registry'):
+ env_file.write(u'resource_registry:\n')
+ for res, value in sorted(env.get('resource_registry', {}).items()):
+ env_file.write(u' %s: %s\n' % (res, value))
+ print('Wrote sample environment "%s"' % target_file)
+
+ for e in env.get('children', []):
+ _generate_environment(e, env)
+
+
+def generate_environments(config_file):
+ with open(config_file) as f:
+ config = yaml.safe_load(f)
+ for env in config['environments']:
+ _generate_environment(env)
+
+
+def usage(exit_code=1):
+ print('Usage: %s <filename.yaml>' % sys.argv[0])
+ sys.exit(exit_code)
+
+
+def main():
+ try:
+ config_file = sys.argv[1]
+ except IndexError:
+ usage()
+ generate_environments(config_file)
+
+
+if __name__ == '__main__':
+ main()
--- /dev/null
+# Copyright 2015 Red Hat Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import io
+import tempfile
+
+import mock
+from oslotest import base
+import six
+import testscenarios
+
+from tripleo_heat_templates import environment_generator
+
+load_tests = testscenarios.load_tests_apply_scenarios
+
+basic_template = '''
+parameters:
+ FooParam:
+ default: foo
+ description: Foo description
+ type: string
+ BarParam:
+ default: 42
+ description: Bar description
+ type: number
+resources:
+ # None
+'''
+basic_private_template = '''
+parameters:
+ FooParam:
+ default: foo
+ description: Foo description
+ type: string
+ _BarParam:
+ default: 42
+ description: Bar description
+ type: number
+resources:
+ # None
+'''
+mandatory_template = '''
+parameters:
+ FooParam:
+ description: Mandatory param
+ type: string
+resources:
+ # None
+'''
+index_template = '''
+parameters:
+ FooParam:
+ description: Param with %index% as its default
+ type: string
+ default: '%index%'
+resources:
+ # None
+'''
+multiline_template = '''
+parameters:
+ FooParam:
+ description: |
+ Parameter with
+ multi-line description
+ type: string
+ default: ''
+resources:
+ # None
+'''
+
+
+class GeneratorTestCase(base.BaseTestCase):
+ content_scenarios = [
+ ('basic',
+ {'template': basic_template,
+ 'exception': None,
+ 'input_file': '''environments:
+ -
+ name: basic
+ title: Basic Environment
+ description: Basic description
+ files:
+ foo.yaml:
+ parameters: all
+''',
+ 'expected_output': '''# title: Basic Environment
+# description: |
+# Basic description
+parameter_defaults:
+ # Bar description
+ # Type: number
+ BarParam: 42
+
+ # Foo description
+ # Type: string
+ FooParam: foo
+
+''',
+ }),
+ ('basic-one-param',
+ {'template': basic_template,
+ 'exception': None,
+ 'input_file': '''environments:
+ -
+ name: basic
+ title: Basic Environment
+ description: Basic description
+ files:
+ foo.yaml:
+ parameters:
+ - FooParam
+''',
+ 'expected_output': '''# title: Basic Environment
+# description: |
+# Basic description
+parameter_defaults:
+ # Foo description
+ # Type: string
+ FooParam: foo
+
+''',
+ }),
+ ('basic-static-param',
+ {'template': basic_template,
+ 'exception': None,
+ 'input_file': '''environments:
+ -
+ name: basic
+ title: Basic Environment
+ description: Basic description
+ files:
+ foo.yaml:
+ parameters: all
+ static:
+ - BarParam
+''',
+ 'expected_output': '''# title: Basic Environment
+# description: |
+# Basic description
+parameter_defaults:
+ # Foo description
+ # Type: string
+ FooParam: foo
+
+ # ******************************************************
+ # Static parameters - these are values that must be
+ # included in the environment but should not be changed.
+ # ******************************************************
+ # Bar description
+ # Type: number
+ BarParam: 42
+
+ # *********************
+ # End static parameters
+ # *********************
+''',
+ }),
+ ('basic-static-param-sample',
+ {'template': basic_template,
+ 'exception': None,
+ 'input_file': '''environments:
+ -
+ name: basic
+ title: Basic Environment
+ description: Basic description
+ files:
+ foo.yaml:
+ parameters: all
+ static:
+ - BarParam
+ sample_values:
+ BarParam: 1
+ FooParam: ''
+''',
+ 'expected_output': '''# title: Basic Environment
+# description: |
+# Basic description
+parameter_defaults:
+ # Foo description
+ # Type: string
+ FooParam: ''
+
+ # ******************************************************
+ # Static parameters - these are values that must be
+ # included in the environment but should not be changed.
+ # ******************************************************
+ # Bar description
+ # Type: number
+ BarParam: 1
+
+ # *********************
+ # End static parameters
+ # *********************
+''',
+ }),
+ ('basic-private',
+ {'template': basic_private_template,
+ 'exception': None,
+ 'input_file': '''environments:
+ -
+ name: basic
+ title: Basic Environment
+ description: Basic description
+ files:
+ foo.yaml:
+ parameters: all
+''',
+ 'expected_output': '''# title: Basic Environment
+# description: |
+# Basic description
+parameter_defaults:
+ # Foo description
+ # Type: string
+ FooParam: foo
+
+''',
+ }),
+ ('mandatory',
+ {'template': mandatory_template,
+ 'exception': None,
+ 'input_file': '''environments:
+ -
+ name: basic
+ title: Basic Environment
+ description: Basic description
+ files:
+ foo.yaml:
+ parameters: all
+''',
+ 'expected_output': '''# title: Basic Environment
+# description: |
+# Basic description
+parameter_defaults:
+ # Mandatory param
+ # Mandatory. This parameter must be set by the user.
+ # Type: string
+ FooParam: <None>
+
+''',
+ }),
+ ('basic-sample',
+ {'template': basic_template,
+ 'exception': None,
+ 'input_file': '''environments:
+ -
+ name: basic
+ title: Basic Environment
+ description: Basic description
+ files:
+ foo.yaml:
+ parameters: all
+ sample_values:
+ FooParam: baz
+''',
+ 'expected_output': '''# title: Basic Environment
+# description: |
+# Basic description
+parameter_defaults:
+ # Bar description
+ # Type: number
+ BarParam: 42
+
+ # Foo description
+ # Type: string
+ FooParam: baz
+
+''',
+ }),
+ ('basic-resource-registry',
+ {'template': basic_template,
+ 'exception': None,
+ 'input_file': '''environments:
+ -
+ name: basic
+ title: Basic Environment
+ description: Basic description
+ files:
+ foo.yaml:
+ parameters: all
+ resource_registry:
+ OS::TripleO::FakeResource: fake-filename.yaml
+''',
+ 'expected_output': '''# title: Basic Environment
+# description: |
+# Basic description
+parameter_defaults:
+ # Bar description
+ # Type: number
+ BarParam: 42
+
+ # Foo description
+ # Type: string
+ FooParam: foo
+
+resource_registry:
+ OS::TripleO::FakeResource: fake-filename.yaml
+''',
+ }),
+ ('missing-param',
+ {'template': basic_template,
+ 'exception': RuntimeError,
+ 'input_file': '''environments:
+ -
+ name: basic
+ title: Basic Environment
+ description: Basic description
+ files:
+ foo.yaml:
+ parameters:
+ - SomethingNonexistent
+''',
+ 'expected_output': None,
+ }),
+ ('percent-index',
+ {'template': index_template,
+ 'exception': None,
+ 'input_file': '''environments:
+ -
+ name: basic
+ title: Basic Environment
+ description: Basic description
+ files:
+ foo.yaml:
+ parameters: all
+''',
+ 'expected_output': '''# title: Basic Environment
+# description: |
+# Basic description
+parameter_defaults:
+ # Param with %index% as its default
+ # Type: string
+ FooParam: '%index%'
+
+''',
+ }),
+ ('multi-line-desc',
+ {'template': multiline_template,
+ 'exception': None,
+ 'input_file': '''environments:
+ -
+ name: basic
+ title: Basic Environment
+ description: Basic description
+ files:
+ foo.yaml:
+ parameters: all
+''',
+ 'expected_output': '''# title: Basic Environment
+# description: |
+# Basic description
+parameter_defaults:
+ # Parameter with
+ # multi-line description
+ # Type: string
+ FooParam: ''
+
+''',
+ }),
+ ]
+
+ @classmethod
+ def generate_scenarios(cls):
+ cls.scenarios = testscenarios.multiply_scenarios(
+ cls.content_scenarios)
+
+ def test_generator(self):
+ fake_input = io.StringIO(six.text_type(self.input_file))
+ fake_template = io.StringIO(six.text_type(self.template))
+ _, fake_output_path = tempfile.mkstemp()
+ fake_output = open(fake_output_path, 'w')
+ with mock.patch('tripleo_heat_templates.environment_generator.open',
+ create=True) as mock_open:
+ mock_open.side_effect = [fake_input, fake_template, fake_output]
+ if not self.exception:
+ environment_generator.generate_environments('ignored.yaml')
+ else:
+ self.assertRaises(self.exception,
+ environment_generator.generate_environments,
+ 'ignored.yaml')
+ return
+ expected = environment_generator._FILE_HEADER + self.expected_output
+ with open(fake_output_path) as f:
+ self.assertEqual(expected, f.read())
+
+GeneratorTestCase.generate_scenarios()
Code Review / apex-tripleo-heat-templates.git / commitdiff