2 # Licensed under the Apache License, Version 2.0 (the "License"); you may
3 # not use this file except in compliance with the License. You may obtain
4 # a copy of the License at
6 # http://www.apache.org/licenses/LICENSE-2.0
8 # Unless required by applicable law or agreed to in writing, software
9 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
10 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
11 # License for the specific language governing permissions and limitations
20 from collections import OrderedDict
21 from toscaparser.functions import Concat
22 from toscaparser.functions import GetAttribute
23 from toscaparser.functions import GetInput
24 from toscaparser.functions import GetOperationOutput
25 from toscaparser.functions import GetProperty
26 from toscaparser.properties import Property
27 from toscaparser.relationship_template import RelationshipTemplate
28 from toscaparser.utils.gettextutils import _
29 from translator.common.exception import ToscaClassAttributeError
30 from translator.common.exception import ToscaClassImportError
31 from translator.common.exception import ToscaModImportError
32 from translator.common.exception import UnsupportedTypeError
33 from translator.common import utils
34 from translator.conf.config import ConfigProvider as translatorConfig
35 from translator.hot.syntax.hot_resource import HotResource
36 from translator.hot.tosca.tosca_block_storage_attachment import (
37 ToscaBlockStorageAttachment
40 ###########################
41 # Module utility Functions
42 # for dynamic class loading
43 ###########################
46 def _generate_type_map():
47 '''Generate TOSCA translation types map.
49 Load user defined classes from location path specified in conf file.
50 Base classes are located within the tosca directory.
54 # Base types directory
55 BASE_PATH = 'translator/hot/tosca'
57 # Custom types directory defined in conf file
58 custom_path = translatorConfig.get_value('DEFAULT',
59 'custom_types_location')
61 # First need to load the parent module, for example 'contrib.hot',
62 # for all of the dynamically loaded classes.
64 _load_classes((BASE_PATH, custom_path), classes)
66 types_map = {clazz.toscatype: clazz for clazz in classes}
67 except AttributeError as e:
68 raise ToscaClassAttributeError(message=e.message)
73 def _load_classes(locations, classes):
74 '''Dynamically load all the classes from the given locations.'''
76 for cls_path in locations:
77 # Use the absolute path of the class path
78 abs_path = os.path.dirname(os.path.abspath(__file__))
79 abs_path = abs_path.replace('translator/hot', cls_path)
81 # Grab all the tosca type module files in the given path
82 mod_files = [f for f in os.listdir(abs_path) if f.endswith('.py')
83 and not f.startswith('__init__')
84 and f.startswith('tosca_')]
86 # For each module, pick out the target translation class
88 # NOTE: For some reason the existing code does not use the map to
89 # instantiate ToscaBlockStorageAttachment. Don't add it to the map
90 # here until the dependent code is fixed to use the map.
91 if f == 'tosca_block_storage_attachment.py':
94 mod_name = cls_path + '/' + f.strip('.py')
95 mod_name = mod_name.replace('/', '.')
97 mod = importlib.import_module(mod_name)
98 target_name = getattr(mod, 'TARGET_CLASS_NAME')
99 clazz = getattr(mod, target_name)
100 classes.append(clazz)
102 raise ToscaModImportError(mod_name=mod_name)
103 except AttributeError:
105 raise ToscaClassImportError(name=target_name,
108 # TARGET_CLASS_NAME is not defined in module.
109 # Re-raise the exception
116 SECTIONS = (TYPE, PROPERTIES, REQUIREMENTS, INTERFACES, LIFECYCLE, INPUT) = \
117 ('type', 'properties', 'requirements',
118 'interfaces', 'lifecycle', 'input')
120 # TODO(anyone): the following requirement names should not be hard-coded
121 # in the translator. Since they are basically arbitrary names, we have to get
122 # them from TOSCA type definitions.
123 # To be fixed with the blueprint:
124 # https://blueprints.launchpad.net/heat-translator/+spec/tosca-custom-types
125 REQUIRES = (CONTAINER, DEPENDENCY, DATABASE_ENDPOINT, CONNECTION, HOST) = \
126 ('container', 'dependency', 'database_endpoint',
127 'connection', 'host')
129 INTERFACES_STATE = (CREATE, START, CONFIGURE, START, DELETE) = \
130 ('create', 'stop', 'configure', 'start', 'delete')
133 TOSCA_TO_HOT_REQUIRES = {'container': 'server', 'host': 'server',
134 'dependency': 'depends_on', "connects": 'depends_on'}
136 TOSCA_TO_HOT_PROPERTIES = {'properties': 'input'}
137 log = logging.getLogger('heat-translator')
139 TOSCA_TO_HOT_TYPE = _generate_type_map()
141 BASE_TYPES = six.string_types + six.integer_types + (dict, OrderedDict)
143 HOT_SCALING_POLICY_TYPE = ["OS::Heat::AutoScalingGroup",
144 "OS::Senlin::Profile"]
147 class TranslateNodeTemplates(object):
148 '''Translate TOSCA NodeTemplates to Heat Resources.'''
150 def __init__(self, tosca, hot_template, csar_dir=None):
152 self.nodetemplates = self.tosca.nodetemplates
153 self.hot_template = hot_template
154 self.csar_dir = csar_dir
155 # list of all HOT resources generated
156 self.hot_resources = []
157 # mapping between TOSCA nodetemplate and HOT resource
158 log.debug(_('Mapping between TOSCA nodetemplate and HOT resource.'))
160 self.policies = self.tosca.topology_template.policies
161 # stores the last deploy of generated behavior for a resource
162 # useful to satisfy underlying dependencies between interfaces
163 self.last_deploy_map = {}
164 self.hot_template_version = None
165 self.processed_policy_res = []
168 return self._translate_nodetemplates()
170 def _recursive_handle_properties(self, resource):
171 '''Recursively handle the properties of the depends_on_nodes nodes.'''
172 # Use of hashtable (dict) here should be faster?
173 if resource in self.processed_resources:
175 self.processed_resources.append(resource)
176 for depend_on in resource.depends_on_nodes:
177 self._recursive_handle_properties(depend_on)
179 if resource.type == "OS::Nova::ServerGroup":
180 resource.handle_properties(self.hot_resources)
181 elif resource.type in ("OS::Heat::ScalingPolicy",
182 "OS::Senlin::Policy"):
183 if resource.name in self.processed_policy_res:
185 self.processed_policy_res.append(resource.name)
186 self.hot_resources = \
187 resource.handle_properties(self.hot_resources)
188 extra_hot_resources = []
189 for res in self.hot_resources:
190 if res.type == 'OS::Heat::ScalingPolicy':
191 extra_res = copy.deepcopy(res)
192 scaling_adjustment = res.properties['scaling_adjustment']
193 if scaling_adjustment < 0:
194 res.name = res.name + '_scale_in'
195 extra_res.name = extra_res.name + '_scale_out'
196 extra_res.properties['scaling_adjustment'] = \
197 -1 * scaling_adjustment
198 extra_hot_resources.append(extra_res)
199 self.processed_policy_res.append(res.name)
200 self.processed_policy_res.append(extra_res.name)
201 elif scaling_adjustment > 0:
202 res.name = res.name + '_scale_out'
203 extra_res.name = extra_res.name + '_scale_in'
204 extra_res.properties['scaling_adjustment'] = \
205 -1 * scaling_adjustment
206 extra_hot_resources.append(extra_res)
207 self.processed_policy_res.append(res.name)
208 self.processed_policy_res.append(extra_res.name)
211 self.hot_resources += extra_hot_resources
213 resource.handle_properties()
215 def _translate_nodetemplates(self):
216 log.debug(_('Translating the node templates.'))
218 # Copy the TOSCA graph: nodetemplate
219 for node in self.nodetemplates:
220 base_type = HotResource.get_base_type_str(node.type_definition)
221 if base_type not in TOSCA_TO_HOT_TYPE:
222 raise UnsupportedTypeError(type=_('%s') % base_type)
223 hot_node = TOSCA_TO_HOT_TYPE[base_type](node,
224 csar_dir=self.csar_dir)
225 self.hot_resources.append(hot_node)
226 self.hot_lookup[node] = hot_node
228 # BlockStorage Attachment is a special case,
229 # which doesn't match to Heat Resources 1 to 1.
230 if base_type == "tosca.nodes.Compute":
231 requirements = node.requirements
233 # Find the name of associated BlockStorage node
234 for requires in requirements:
236 for value in requires.values():
237 if isinstance(value, dict):
238 for node_name in value.values():
239 for n in self.nodetemplates:
240 if n.name == node_name and \
242 "tosca.nodes.BlockStorage"):
243 volume_name = node_name
246 for n in self.nodetemplates:
247 if n.name == value and \
249 "tosca.nodes.BlockStorage"):
250 volume_name = node_name
255 attachment_node = self._get_attachment_node(
256 node, suffix, volume_name)
258 self.hot_resources.append(attachment_node)
259 for i in self.tosca.inputs:
260 if (i.name == 'key_name' and
261 node.get_property_value('key_name') is None):
262 schema = {'type': i.type, 'default': i.default}
263 value = {"get_param": "key_name"}
264 prop = Property(i.name, value, schema)
265 node._properties.append(prop)
267 for policy in self.policies:
268 policy_type = policy.type_definition
269 if policy.is_derived_from('tosca.policies.Scaling') and \
270 policy_type.type != 'tosca.policies.Scaling.Cluster':
271 TOSCA_TO_HOT_TYPE[policy_type.type] = \
272 TOSCA_TO_HOT_TYPE['tosca.policies.Scaling']
273 if policy.is_derived_from('tosca.policies.Monitoring'):
274 TOSCA_TO_HOT_TYPE[policy_type.type] = \
275 TOSCA_TO_HOT_TYPE['tosca.policies.Monitoring']
276 if not policy.is_derived_from('tosca.policies.Monitoring') and \
277 not policy.is_derived_from('tosca.policies.Scaling') and \
278 policy_type.type not in TOSCA_TO_HOT_TYPE:
279 raise UnsupportedTypeError(type=_('%s') % policy_type.type)
280 elif policy_type.type == 'tosca.policies.Scaling.Cluster':
281 self.hot_template_version = '2016-04-08'
282 policy_node = TOSCA_TO_HOT_TYPE[policy_type.type](policy)
283 self.hot_resources.append(policy_node)
285 # Handle life cycle operations: this may expand each node
286 # into multiple HOT resources and may change their name
287 lifecycle_resources = []
288 for resource in self.hot_resources:
289 expanded_resources, deploy_lookup, last_deploy = resource.\
291 if expanded_resources:
292 lifecycle_resources += expanded_resources
294 self.hot_lookup.update(deploy_lookup)
296 self.last_deploy_map[resource] = last_deploy
297 self.hot_resources += lifecycle_resources
299 # Handle configuration from ConnectsTo relationship in the TOSCA node:
300 # this will generate multiple HOT resources, set of 2 for each
302 connectsto_resources = []
303 for node in self.nodetemplates:
304 for requirement in node.requirements:
305 for endpoint, details in requirement.items():
307 if isinstance(details, dict):
308 target = details.get('node')
309 relation = details.get('relationship')
312 if (target and relation and
313 not isinstance(relation, six.string_types)):
314 interfaces = relation.get('interfaces')
315 connectsto_resources += \
316 self._create_connect_configs(node,
319 self.hot_resources += connectsto_resources
321 # Copy the initial dependencies based on the relationship in
323 for node in self.nodetemplates:
324 for node_depend in node.related_nodes:
325 # if the source of dependency is a server and the
326 # relationship type is 'tosca.relationships.HostedOn',
327 # add dependency as properties.server
328 base_type = HotResource.get_base_type_str(
329 node_depend.type_definition)
330 if base_type == 'tosca.nodes.Compute' and \
331 node.related[node_depend].type == \
332 node.type_definition.HOSTEDON:
333 self.hot_lookup[node].properties['server'] = \
334 {'get_resource': self.hot_lookup[node_depend].name}
335 # for all others, add dependency as depends_on
337 self.hot_lookup[node].depends_on.append(
338 self.hot_lookup[node_depend].top_of_chain())
340 self.hot_lookup[node].depends_on_nodes.append(
341 self.hot_lookup[node_depend].top_of_chain())
343 last_deploy = self.last_deploy_map.get(
344 self.hot_lookup[node_depend])
346 last_deploy not in self.hot_lookup[node].depends_on:
347 self.hot_lookup[node].depends_on.append(last_deploy)
348 self.hot_lookup[node].depends_on_nodes.append(last_deploy)
350 # handle hosting relationship
351 for resource in self.hot_resources:
352 resource.handle_hosting()
354 # handle built-in properties of HOT resources
355 # if a resource depends on other resources,
356 # their properties need to be handled first.
357 # Use recursion to handle the properties of the
358 # dependent nodes in correct order
359 self.processed_resources = []
360 for resource in self.hot_resources:
361 if resource.type not in HOT_SCALING_POLICY_TYPE:
362 self._recursive_handle_properties(resource)
364 # handle resources that need to expand to more than one HOT resource
365 expansion_resources = []
366 for resource in self.hot_resources:
367 expanded = resource.handle_expansion()
369 expansion_resources += expanded
370 self.hot_resources += expansion_resources
372 # Resolve function calls: GetProperty, GetAttribute, GetInput
373 # at this point, all the HOT resources should have been created
375 for resource in self.hot_resources:
376 # traverse the reference chain to get the actual value
377 inputs = resource.properties.get('input_values')
379 for name, value in inputs.items():
380 inputs[name] = self.translate_param_value(value, resource)
382 # remove resources without type defined
383 # for example a SoftwareComponent without interfaces
384 # would fall in this case
386 for resource in self.hot_resources:
387 if resource.type is None:
388 to_remove.append(resource)
390 for resource in to_remove:
391 self.hot_resources.remove(resource)
393 return self.hot_resources
395 def translate_param_value(self, param_value, resource):
396 tosca_template = None
398 tosca_template = resource.nodetemplate
400 get_property_args = None
401 if isinstance(param_value, GetProperty):
402 get_property_args = param_value.args
403 # to remove when the parser is fixed to return GetProperty
404 elif isinstance(param_value, dict) and 'get_property' in param_value:
405 get_property_args = param_value['get_property']
406 if get_property_args is not None:
407 tosca_target, prop_name, prop_arg = \
408 self.decipher_get_operation(get_property_args,
411 prop_value = tosca_target.get_property_value(prop_name)
413 prop_value = self.translate_param_value(
414 prop_value, resource)
415 return self._unfold_value(prop_value, prop_arg)
417 if isinstance(param_value, GetAttribute):
418 get_attr_args = param_value.result().args
419 # to remove when the parser is fixed to return GetAttribute
420 elif isinstance(param_value, dict) and 'get_attribute' in param_value:
421 get_attr_args = param_value['get_attribute']
422 if get_attr_args is not None:
424 # get the proper target type to perform the translation
425 tosca_target, attr_name, attr_arg = \
426 self.decipher_get_operation(get_attr_args, tosca_template)
429 attr_args += attr_arg
431 if tosca_target in self.hot_lookup:
432 attr_value = self.hot_lookup[tosca_target].\
433 get_hot_attribute(attr_name, attr_args)
434 attr_value = self.translate_param_value(
435 attr_value, resource)
436 return self._unfold_value(attr_value, attr_arg)
437 elif isinstance(param_value, dict) and 'get_artifact' in param_value:
438 get_artifact_args = param_value['get_artifact']
439 tosca_target, artifact_name, _ = \
440 self.decipher_get_operation(get_artifact_args,
444 artifacts = HotResource.get_all_artifacts(tosca_target)
445 if artifact_name in artifacts:
447 artifact = artifacts[artifact_name]
449 os.chdir(self.csar_dir)
450 get_file = os.path.abspath(artifact.get('file'))
452 get_file = artifact.get('file')
453 if artifact.get('type', None) == 'tosca.artifacts.File':
454 return {'get_file': get_file}
456 get_input_args = None
457 if isinstance(param_value, GetInput):
458 get_input_args = param_value.args
459 elif isinstance(param_value, dict) and 'get_input' in param_value:
460 get_input_args = param_value['get_input']
461 if get_input_args is not None:
462 if isinstance(get_input_args, list) \
463 and len(get_input_args) == 1:
464 return {'get_param': self.translate_param_value(
465 get_input_args[0], resource)}
467 return {'get_param': self.translate_param_value(
468 get_input_args, resource)}
469 elif isinstance(param_value, GetOperationOutput):
470 res = self._translate_get_operation_output_function(
471 param_value.args, tosca_template)
474 elif isinstance(param_value, dict) \
475 and 'get_operation_output' in param_value:
476 res = self._translate_get_operation_output_function(
477 param_value['get_operation_output'], tosca_template)
481 if isinstance(param_value, Concat):
482 concat_list = param_value.args
483 elif isinstance(param_value, dict) and 'concat' in param_value:
484 concat_list = param_value['concat']
485 if concat_list is not None:
486 res = self._translate_concat_function(concat_list, resource)
490 if isinstance(param_value, list):
492 for elem in param_value:
493 translated_elem = self.translate_param_value(elem, resource)
495 translated_list.append(translated_elem)
496 return translated_list
498 if isinstance(param_value, BASE_TYPES):
503 def _translate_concat_function(self, concat_list, resource):
504 str_replace_template = ''
505 str_replace_params = {}
507 for elem in concat_list:
508 str_replace_template += '$s' + str(index)
509 str_replace_params['$s' + str(index)] = \
510 self.translate_param_value(elem, resource)
513 return {'str_replace': {
514 'template': str_replace_template,
515 'params': str_replace_params
518 def _translate_get_operation_output_function(self, args, tosca_template):
519 tosca_target = self._find_tosca_node(args[0],
521 if tosca_target and len(args) >= 4:
522 operations = HotResource.get_all_operations(tosca_target)
523 # ignore Standard interface name,
524 # it is the only one supported in the translator anyway
526 output_name = args[3]
527 if op_name in operations:
528 operation = operations[op_name]
529 if operation in self.hot_lookup:
530 matching_deploy = self.hot_lookup[operation]
531 matching_config_name = matching_deploy.\
532 properties['config']['get_resource']
533 matching_config = self.find_hot_resource(
534 matching_config_name)
536 outputs = matching_config.properties.get('outputs')
539 outputs.append({'name': output_name})
540 matching_config.properties['outputs'] = outputs
541 return {'get_attr': [
542 matching_deploy.name,
547 def _unfold_value(value, value_arg):
548 if value_arg is not None:
549 if isinstance(value, dict):
550 val = value.get(value_arg)
554 index = utils.str_to_num(value_arg)
555 if isinstance(value, list) and index is not None:
559 def decipher_get_operation(self, args, current_tosca_node):
560 tosca_target = self._find_tosca_node(args[0],
563 if tosca_target and len(args) > 2:
564 cap_or_req_name = args[1]
565 cap = tosca_target.get_capability(cap_or_req_name)
569 for req in tosca_target.requirements:
570 if cap_or_req_name in req:
571 new_target = self._find_tosca_node(
572 req[cap_or_req_name])
573 cap = new_target.get_capability(cap_or_req_name)
579 tosca_target = new_target
582 prop_arg = args[3] if len(args) >= 4 else None
585 prop_arg = args[2] if len(args) >= 3 else None
587 return tosca_target, prop_name, prop_arg
589 def _get_attachment_node(self, node, suffix, volume_name):
591 ntpl = self.nodetemplates
592 for key_r, value_n in node.relationships.items():
593 if key_r.is_derived_from('tosca.relationships.AttachesTo'):
594 if value_n.is_derived_from('tosca.nodes.BlockStorage'):
595 if volume_name == value_n.name:
598 relationship_tpl = None
599 for req in node.requirements:
600 for key, val in req.items():
601 if isinstance(val, dict):
602 if value_n.name != val.get('node'):
605 if value_n.name != val:
608 relship = val.get('relationship')
609 for rkey, rval in val.items():
610 if relship and isinstance(relship, dict):
611 for rkey, rval in relship.items():
613 relationship_tpl = val
615 elif rkey == 'template':
617 (self.tosca.topology_template.
618 _tpl_relationship_templates())
619 relationship_tpl = rel_tpl_list[rval]
623 elif isinstance(relship, str):
625 relationship_tpl = val
626 relationship_templates = \
627 self.tosca._tpl_relationship_templates()
628 if 'relationship' in relationship_tpl and \
630 self.tosca._tpl_relationship_types() and \
631 attach in relationship_templates:
632 relationship_tpl['relationship'] = \
633 relationship_templates[attach]
636 rval_new = attach + "_" + str(suffix)
637 att = RelationshipTemplate(
638 relationship_tpl, rval_new,
639 self.tosca._tpl_relationship_types())
640 hot_node = ToscaBlockStorageAttachment(att, ntpl,
646 def find_hot_resource(self, name):
647 for resource in self.hot_resources:
648 if resource.name == name:
651 def _find_tosca_node(self, tosca_name, current_tosca_template=None):
653 if tosca_name == 'SELF':
654 tosca_node = current_tosca_template
655 if tosca_name == 'HOST' and current_tosca_template:
656 for req in current_tosca_template.requirements:
658 tosca_node = self._find_tosca_node(req['host'])
660 if tosca_node is None:
661 for node in self.nodetemplates:
662 if node.name == tosca_name:
667 def _find_hot_resource_for_tosca(self, tosca_name,
668 current_hot_resource=None):
669 current_tosca_resource = current_hot_resource.nodetemplate \
670 if current_hot_resource else None
671 tosca_node = self._find_tosca_node(tosca_name, current_tosca_resource)
673 return self.hot_lookup[tosca_node]
677 def _create_connect_configs(self, source_node, target_name,
679 connectsto_resources = []
680 if connect_interfaces:
681 for iname, interface in connect_interfaces.items():
682 connectsto_resources += \
683 self._create_connect_config(source_node, target_name,
685 return connectsto_resources
687 def _create_connect_config(self, source_node, target_name,
689 connectsto_resources = []
690 target_node = self._find_tosca_node(target_name)
691 # the configuration can occur on the source or the target
692 connect_config = connect_interface.get('pre_configure_target')
693 if connect_config is not None:
694 config_location = 'target'
696 connect_config = connect_interface.get('pre_configure_source')
697 if connect_config is not None:
698 config_location = 'source'
700 msg = _("Template error: "
701 "no configuration found for ConnectsTo "
702 "in {1}").format(self.nodetemplate.name)
705 config_name = source_node.name + '_' + target_name + '_connect_config'
706 implement = connect_config.get('implementation')
708 if config_location == 'target':
710 os.chdir(self.csar_dir)
711 get_file = os.path.abspath(implement)
714 hot_config = HotResource(target_node,
716 'OS::Heat::SoftwareConfig',
717 {'config': {'get_file': get_file}},
718 csar_dir=self.csar_dir)
719 elif config_location == 'source':
721 os.chdir(self.csar_dir)
722 get_file = os.path.abspath(implement)
725 hot_config = HotResource(source_node,
727 'OS::Heat::SoftwareConfig',
728 {'config': {'get_file': get_file}},
729 csar_dir=self.csar_dir)
731 connectsto_resources.append(hot_config)
732 hot_target = self._find_hot_resource_for_tosca(target_name)
733 hot_source = self._find_hot_resource_for_tosca(source_node.name)
734 connectsto_resources.append(hot_config.
735 handle_connectsto(source_node,
741 return connectsto_resources