Synchronise the openstack bugs
[parser.git] / tosca2heat / heat-translator / translator / hot / translate_node_templates.py
1 #
2 # Licensed under the Apache License, Version 2.0 (the "License"); you may
3 # not use this file except in compliance with the License. You may obtain
4 # a copy of the License at
5 #
6 # http://www.apache.org/licenses/LICENSE-2.0
7 #
8 # Unless required by applicable law or agreed to in writing, software
9 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
10 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
11 # License for the specific language governing permissions and limitations
12 # under the License.
13
14 import importlib
15 import logging
16 import os
17 import six
18
19 from collections import OrderedDict
20 from toscaparser.functions import Concat
21 from toscaparser.functions import GetAttribute
22 from toscaparser.functions import GetInput
23 from toscaparser.functions import GetProperty
24 from toscaparser.properties import Property
25 from toscaparser.relationship_template import RelationshipTemplate
26 from toscaparser.utils.gettextutils import _
27 from translator.common.exception import ToscaClassAttributeError
28 from translator.common.exception import ToscaClassImportError
29 from translator.common.exception import ToscaModImportError
30 from translator.common import utils
31 from translator.conf.config import ConfigProvider as translatorConfig
32 from translator.hot.syntax.hot_resource import HotResource
33 from translator.hot.tosca.tosca_block_storage_attachment import (
34     ToscaBlockStorageAttachment
35     )
36
37 ###########################
38 # Module utility Functions
39 # for dynamic class loading
40 ###########################
41
42
43 def _generate_type_map():
44     '''Generate TOSCA translation types map.
45
46     Load user defined classes from location path specified in conf file.
47     Base classes are located within the tosca directory.
48
49     '''
50
51     # Base types directory
52     BASE_PATH = 'translator/hot/tosca'
53
54     # Custom types directory defined in conf file
55     custom_path = translatorConfig.get_value('DEFAULT',
56                                              'custom_types_location')
57
58     # First need to load the parent module, for example 'contrib.hot',
59     # for all of the dynamically loaded classes.
60     classes = []
61     _load_classes((BASE_PATH, custom_path), classes)
62     try:
63         types_map = {clazz.toscatype: clazz for clazz in classes}
64     except AttributeError as e:
65         raise ToscaClassAttributeError(message=e.message)
66
67     return types_map
68
69
70 def _load_classes(locations, classes):
71     '''Dynamically load all the classes from the given locations.'''
72
73     for cls_path in locations:
74         # Use the absolute path of the class path
75         abs_path = os.path.dirname(os.path.abspath(__file__))
76         abs_path = abs_path.replace('translator/hot', cls_path)
77
78         # Grab all the tosca type module files in the given path
79         mod_files = [f for f in os.listdir(abs_path) if f.endswith('.py')
80                      and not f.startswith('__init__')
81                      and f.startswith('tosca_')]
82
83         # For each module, pick out the target translation class
84         for f in mod_files:
85             # NOTE: For some reason the existing code does not use the map to
86             # instantiate ToscaBlockStorageAttachment. Don't add it to the map
87             # here until the dependent code is fixed to use the map.
88             if f == 'tosca_block_storage_attachment.py':
89                 continue
90
91             mod_name = cls_path + '/' + f.strip('.py')
92             mod_name = mod_name.replace('/', '.')
93             try:
94                 mod = importlib.import_module(mod_name)
95                 target_name = getattr(mod, 'TARGET_CLASS_NAME')
96                 clazz = getattr(mod, target_name)
97                 classes.append(clazz)
98             except ImportError:
99                 raise ToscaModImportError(mod_name=mod_name)
100             except AttributeError:
101                 if target_name:
102                     raise ToscaClassImportError(name=target_name,
103                                                 mod_name=mod_name)
104                 else:
105                     # TARGET_CLASS_NAME is not defined in module.
106                     # Re-raise the exception
107                     raise
108
109 ##################
110 # Module constants
111 ##################
112
113 SECTIONS = (TYPE, PROPERTIES, REQUIREMENTS, INTERFACES, LIFECYCLE, INPUT) = \
114            ('type', 'properties', 'requirements',
115             'interfaces', 'lifecycle', 'input')
116
117 # TODO(anyone):  the following requirement names should not be hard-coded
118 # in the translator.  Since they are basically arbitrary names, we have to get
119 # them from TOSCA type definitions.
120 # To be fixed with the blueprint:
121 # https://blueprints.launchpad.net/heat-translator/+spec/tosca-custom-types
122 REQUIRES = (CONTAINER, DEPENDENCY, DATABASE_ENDPOINT, CONNECTION, HOST) = \
123            ('container', 'dependency', 'database_endpoint',
124             'connection', 'host')
125
126 INTERFACES_STATE = (CREATE, START, CONFIGURE, START, DELETE) = \
127                    ('create', 'stop', 'configure', 'start', 'delete')
128
129
130 TOSCA_TO_HOT_REQUIRES = {'container': 'server', 'host': 'server',
131                          'dependency': 'depends_on', "connects": 'depends_on'}
132
133 TOSCA_TO_HOT_PROPERTIES = {'properties': 'input'}
134 log = logging.getLogger('heat-translator')
135
136 TOSCA_TO_HOT_TYPE = _generate_type_map()
137
138 BASE_TYPES = six.string_types + six.integer_types + (dict, OrderedDict)
139
140
141 class TranslateNodeTemplates(object):
142     '''Translate TOSCA NodeTemplates to Heat Resources.'''
143
144     def __init__(self, tosca, hot_template):
145         self.tosca = tosca
146         self.nodetemplates = self.tosca.nodetemplates
147         self.hot_template = hot_template
148         # list of all HOT resources generated
149         self.hot_resources = []
150         # mapping between TOSCA nodetemplate and HOT resource
151         log.debug(_('Mapping between TOSCA nodetemplate and HOT resource.'))
152         self.hot_lookup = {}
153         self.policies = self.tosca.topology_template.policies
154         # stores the last deploy of generated behavior for a resource
155         # useful to satisfy underlying dependencies between interfaces
156         self.last_deploy_map = {}
157
158     def translate(self):
159         return self._translate_nodetemplates()
160
161     def _recursive_handle_properties(self, resource):
162         '''Recursively handle the properties of the depends_on_nodes nodes.'''
163         # Use of hashtable (dict) here should be faster?
164         if resource in self.processed_resources:
165             return
166         self.processed_resources.append(resource)
167         for depend_on in resource.depends_on_nodes:
168             self._recursive_handle_properties(depend_on)
169
170         if resource.type == "OS::Nova::ServerGroup":
171             resource.handle_properties(self.hot_resources)
172         else:
173             resource.handle_properties()
174
175     def _translate_nodetemplates(self):
176
177         log.debug(_('Translating the node templates.'))
178         suffix = 0
179         # Copy the TOSCA graph: nodetemplate
180         for node in self.nodetemplates:
181             base_type = HotResource.get_base_type(node.type_definition)
182             hot_node = TOSCA_TO_HOT_TYPE[base_type.type](node)
183             self.hot_resources.append(hot_node)
184             self.hot_lookup[node] = hot_node
185
186             # BlockStorage Attachment is a special case,
187             # which doesn't match to Heat Resources 1 to 1.
188             if base_type.type == "tosca.nodes.Compute":
189                 volume_name = None
190                 requirements = node.requirements
191                 if requirements:
192                     # Find the name of associated BlockStorage node
193                     for requires in requirements:
194                         for value in requires.values():
195                             if isinstance(value, dict):
196                                 for node_name in value.values():
197                                     for n in self.nodetemplates:
198                                         if n.name == node_name:
199                                             volume_name = node_name
200                                             break
201                             else:  # unreachable code !
202                                 for n in self.nodetemplates:
203                                     if n.name == node_name:
204                                         volume_name = node_name
205                                         break
206
207                     suffix = suffix + 1
208                     attachment_node = self._get_attachment_node(node,
209                                                                 suffix,
210                                                                 volume_name)
211                     if attachment_node:
212                         self.hot_resources.append(attachment_node)
213                 for i in self.tosca.inputs:
214                     if (i.name == 'key_name' and
215                             node.get_property_value('key_name') is None):
216                         schema = {'type': i.type, 'default': i.default}
217                         value = {"get_param": "key_name"}
218                         prop = Property(i.name, value, schema)
219                         node._properties.append(prop)
220
221         for policy in self.policies:
222             policy_type = policy.type_definition
223             policy_node = TOSCA_TO_HOT_TYPE[policy_type.type](policy)
224             self.hot_resources.append(policy_node)
225
226         # Handle life cycle operations: this may expand each node
227         # into multiple HOT resources and may change their name
228         lifecycle_resources = []
229         for resource in self.hot_resources:
230             expanded_resources, deploy_lookup, last_deploy = resource.\
231                 handle_life_cycle()
232             if expanded_resources:
233                 lifecycle_resources += expanded_resources
234             if deploy_lookup:
235                 self.hot_lookup.update(deploy_lookup)
236             if last_deploy:
237                 self.last_deploy_map[resource] = last_deploy
238         self.hot_resources += lifecycle_resources
239
240         # Handle configuration from ConnectsTo relationship in the TOSCA node:
241         # this will generate multiple HOT resources, set of 2 for each
242         # configuration
243         connectsto_resources = []
244         for node in self.nodetemplates:
245             for requirement in node.requirements:
246                 for endpoint, details in six.iteritems(requirement):
247                     relation = None
248                     if isinstance(details, dict):
249                         target = details.get('node')
250                         relation = details.get('relationship')
251                     else:
252                         target = details
253                     if (target and relation and
254                             not isinstance(relation, six.string_types)):
255                         interfaces = relation.get('interfaces')
256                         connectsto_resources += \
257                             self._create_connect_configs(node,
258                                                          target,
259                                                          interfaces)
260         self.hot_resources += connectsto_resources
261
262         # Copy the initial dependencies based on the relationship in
263         # the TOSCA template
264         for node in self.nodetemplates:
265             for node_depend in node.related_nodes:
266                 # if the source of dependency is a server and the
267                 # relationship type is 'tosca.relationships.HostedOn',
268                 # add dependency as properties.server
269                 base_type = HotResource.get_base_type(
270                     node_depend.type_definition)
271                 if base_type.type == 'tosca.nodes.Compute' and \
272                    node.related[node_depend].type == \
273                    node.type_definition.HOSTEDON:
274                     self.hot_lookup[node].properties['server'] = \
275                         {'get_resource': self.hot_lookup[node_depend].name}
276                 # for all others, add dependency as depends_on
277                 else:
278                     self.hot_lookup[node].depends_on.append(
279                         self.hot_lookup[node_depend].top_of_chain())
280
281                 self.hot_lookup[node].depends_on_nodes.append(
282                     self.hot_lookup[node_depend].top_of_chain())
283
284                 last_deploy = self.last_deploy_map.get(
285                     self.hot_lookup[node_depend])
286                 if last_deploy and \
287                     last_deploy not in self.hot_lookup[node].depends_on:
288                     self.hot_lookup[node].depends_on.append(last_deploy)
289                     self.hot_lookup[node].depends_on_nodes.append(last_deploy)
290
291         # handle hosting relationship
292         for resource in self.hot_resources:
293             resource.handle_hosting()
294
295         # handle built-in properties of HOT resources
296         # if a resource depends on other resources,
297         # their properties need to be handled first.
298         # Use recursion to handle the properties of the
299         # dependent nodes in correct order
300         self.processed_resources = []
301         for resource in self.hot_resources:
302             self._recursive_handle_properties(resource)
303
304         # handle resources that need to expand to more than one HOT resource
305         expansion_resources = []
306         for resource in self.hot_resources:
307             expanded = resource.handle_expansion()
308             if expanded:
309                 expansion_resources += expanded
310         self.hot_resources += expansion_resources
311
312         # Resolve function calls:  GetProperty, GetAttribute, GetInput
313         # at this point, all the HOT resources should have been created
314         # in the graph.
315         for resource in self.hot_resources:
316             # traverse the reference chain to get the actual value
317             inputs = resource.properties.get('input_values')
318             if inputs:
319                 for name, value in six.iteritems(inputs):
320                     inputs[name] = self.translate_param_value(value, resource)
321
322         # remove resources without type defined
323         # for example a SoftwareComponent without interfaces
324         # would fall in this case
325         to_remove = []
326         for resource in self.hot_resources:
327             if resource.type is None:
328                 to_remove.append(resource)
329
330         for resource in to_remove:
331             self.hot_resources.remove(resource)
332
333         return self.hot_resources
334
335     def translate_param_value(self, param_value, resource):
336         tosca_template = None
337         if resource:
338             tosca_template = resource.nodetemplate
339
340         get_property_args = None
341         if isinstance(param_value, GetProperty):
342             get_property_args = param_value.args
343         # to remove when the parser is fixed to return GetProperty
344         elif isinstance(param_value, dict) and 'get_property' in param_value:
345             get_property_args = param_value['get_property']
346         if get_property_args is not None:
347             tosca_target, prop_name, prop_arg = \
348                 self.decipher_get_operation(get_property_args,
349                                             tosca_template)
350             if tosca_target:
351                 prop_value = tosca_target.get_property_value(prop_name)
352                 if prop_value:
353                     prop_value = self.translate_param_value(
354                         prop_value, resource)
355                     return self._unfold_value(prop_value, prop_arg)
356         get_attr_args = None
357         if isinstance(param_value, GetAttribute):
358             get_attr_args = param_value.result().args
359         # to remove when the parser is fixed to return GetAttribute
360         elif isinstance(param_value, dict) and 'get_attribute' in param_value:
361             get_attr_args = param_value['get_attribute']
362         if get_attr_args is not None:
363             # for the attribute
364             # get the proper target type to perform the translation
365             tosca_target, attr_name, attr_arg = \
366                 self.decipher_get_operation(get_attr_args, tosca_template)
367             attr_args = []
368             if attr_arg:
369                 attr_args += attr_arg
370             if tosca_target:
371                 if tosca_target in self.hot_lookup:
372                     attr_value = self.hot_lookup[tosca_target].\
373                         get_hot_attribute(attr_name, attr_args)
374                     attr_value = self.translate_param_value(
375                         attr_value, resource)
376                     return self._unfold_value(attr_value, attr_arg)
377         elif isinstance(param_value, dict) and 'get_artifact' in param_value:
378             get_artifact_args = param_value['get_artifact']
379             tosca_target, artifact_name, _ = \
380                 self.decipher_get_operation(get_artifact_args,
381                                             tosca_template)
382
383             if tosca_target:
384                 artifacts = self.get_all_artifacts(tosca_target)
385                 if artifact_name in artifacts:
386                     artifact = artifacts[artifact_name]
387                     if artifact.get('type', None) == 'tosca.artifacts.File':
388                         return {'get_file': artifact.get('file')}
389         get_input_args = None
390         if isinstance(param_value, GetInput):
391             get_input_args = param_value.args
392         elif isinstance(param_value, dict) and 'get_input' in param_value:
393             get_input_args = param_value['get_input']
394         if get_input_args is not None:
395             if isinstance(get_input_args, list) \
396                     and len(get_input_args) == 1:
397                 return {'get_param': self.translate_param_value(
398                     get_input_args[0], resource)}
399             else:
400                 return {'get_param': self.translate_param_value(
401                     get_input_args, resource)}
402         elif isinstance(param_value, dict) \
403                 and 'get_operation_output' in param_value:
404             res = self._translate_get_operation_output_function(
405                 param_value['get_operation_output'], tosca_template)
406             if res:
407                 return res
408         concat_list = None
409         if isinstance(param_value, Concat):
410             concat_list = param_value.args
411         elif isinstance(param_value, dict) and 'concat' in param_value:
412             concat_list = param_value['concat']
413         if concat_list is not None:
414             res = self._translate_concat_function(concat_list, resource)
415             if res:
416                 return res
417
418         if isinstance(param_value, list):
419             translated_list = []
420             for elem in param_value:
421                 translated_elem = self.translate_param_value(elem, resource)
422                 if translated_elem:
423                     translated_list.append(translated_elem)
424             return translated_list
425
426         if isinstance(param_value, BASE_TYPES):
427             return param_value
428
429         return None
430
431     def _translate_concat_function(self, concat_list, resource):
432         str_replace_template = ''
433         str_replace_params = {}
434         index = 0
435         for elem in concat_list:
436             str_replace_template += '$s' + str(index)
437             str_replace_params['$s' + str(index)] = \
438                 self.translate_param_value(elem, resource)
439             index += 1
440
441         return {'str_replace': {
442             'template': str_replace_template,
443             'params': str_replace_params
444         }}
445
446     def _translate_get_operation_output_function(self, args, tosca_template):
447         tosca_target = self._find_tosca_node(args[0],
448                                              tosca_template)
449         if tosca_target and len(args) >= 4:
450             operations = HotResource.get_all_operations(tosca_target)
451             # ignore Standard interface name,
452             # it is the only one supported in the translator anyway
453             op_name = args[2]
454             output_name = args[3]
455             if op_name in operations:
456                 operation = operations[op_name]
457                 if operation in self.hot_lookup:
458                     matching_deploy = self.hot_lookup[operation]
459                     matching_config_name = matching_deploy.\
460                         properties['config']['get_resource']
461                     matching_config = self.find_hot_resource(
462                         matching_config_name)
463                     if matching_config:
464                         outputs = matching_config.properties.get('outputs')
465                         if outputs is None:
466                             outputs = []
467                         outputs.append({'name': output_name})
468                         matching_config.properties['outputs'] = outputs
469                     return {'get_attr': [
470                         matching_deploy.name,
471                         output_name
472                     ]}
473
474     @staticmethod
475     def _unfold_value(value, value_arg):
476         if value_arg is not None:
477             if isinstance(value, dict):
478                 val = value.get(value_arg)
479                 if val is not None:
480                     return val
481
482             index = utils.str_to_num(value_arg)
483             if isinstance(value, list) and index is not None:
484                 return value[index]
485         return value
486
487     def decipher_get_operation(self, args, current_tosca_node):
488         tosca_target = self._find_tosca_node(args[0],
489                                              current_tosca_node)
490         new_target = None
491         if tosca_target and len(args) > 2:
492             cap_or_req_name = args[1]
493             cap = tosca_target.get_capability(cap_or_req_name)
494             if cap:
495                 new_target = cap
496             else:
497                 for req in tosca_target.requirements:
498                     if cap_or_req_name in req:
499                         new_target = self._find_tosca_node(
500                             req[cap_or_req_name])
501                         cap = new_target.get_capability(cap_or_req_name)
502                         if cap:
503                             new_target = cap
504                         break
505
506         if new_target:
507             tosca_target = new_target
508
509             prop_name = args[2]
510             prop_arg = args[3] if len(args) >= 4 else None
511         else:
512             prop_name = args[1]
513             prop_arg = args[2] if len(args) >= 3 else None
514
515         return tosca_target, prop_name, prop_arg
516
517     @staticmethod
518     def get_all_artifacts(nodetemplate):
519         artifacts = nodetemplate.type_definition.get_value('artifacts',
520                                                            parent=True)
521         if not artifacts:
522             artifacts = {}
523         tpl_artifacts = nodetemplate.entity_tpl.get('artifacts')
524         if tpl_artifacts:
525             artifacts.update(tpl_artifacts)
526
527         return artifacts
528
529     def _get_attachment_node(self, node, suffix, volume_name):
530         attach = False
531         ntpl = self.nodetemplates
532         for key, value in node.relationships.items():
533             if key.is_derived_from('tosca.relationships.AttachesTo'):
534                 if value.is_derived_from('tosca.nodes.BlockStorage'):
535                     attach = True
536             if attach:
537                 relationship_tpl = None
538                 for req in node.requirements:
539                     for key, val in req.items():
540                         attach = val
541                         relship = val.get('relationship')
542                         for rkey, rval in val.items():
543                             if relship and isinstance(relship, dict):
544                                 for rkey, rval in relship.items():
545                                     if rkey == 'type':
546                                         relationship_tpl = val
547                                         attach = rval
548                                     elif rkey == 'template':
549                                         rel_tpl_list = \
550                                             (self.tosca.topology_template.
551                                              _tpl_relationship_templates())
552                                         relationship_tpl = rel_tpl_list[rval]
553                                         attach = rval
554                                     else:
555                                         continue
556                             elif isinstance(relship, str):
557                                 attach = relship
558                                 relationship_tpl = val
559                                 relationship_templates = \
560                                     self.tosca._tpl_relationship_templates()
561                                 if 'relationship' in relationship_tpl and \
562                                    attach not in \
563                                    self.tosca._tpl_relationship_types() and \
564                                    attach in relationship_templates:
565                                     relationship_tpl['relationship'] = \
566                                         relationship_templates[attach]
567                                 break
568                         if relationship_tpl:
569                             rval_new = attach + "_" + str(suffix)
570                             att = RelationshipTemplate(
571                                 relationship_tpl, rval_new,
572                                 self.tosca._tpl_relationship_types())
573                             hot_node = ToscaBlockStorageAttachment(att, ntpl,
574                                                                    node.name,
575                                                                    volume_name
576                                                                    )
577                             return hot_node
578
579     def find_hot_resource(self, name):
580         for resource in self.hot_resources:
581             if resource.name == name:
582                 return resource
583
584     def _find_tosca_node(self, tosca_name, current_tosca_template=None):
585         tosca_node = None
586         if tosca_name == 'SELF':
587             tosca_node = current_tosca_template
588         if tosca_name == 'HOST' and current_tosca_template:
589             for req in current_tosca_template.requirements:
590                 if 'host' in req:
591                     tosca_node = self._find_tosca_node(req['host'])
592
593         if tosca_node is None:
594             for node in self.nodetemplates:
595                 if node.name == tosca_name:
596                     tosca_node = node
597                     break
598         return tosca_node
599
600     def _find_hot_resource_for_tosca(self, tosca_name,
601                                      current_hot_resource=None):
602         current_tosca_resource = current_hot_resource.nodetemplate \
603             if current_hot_resource else None
604         tosca_node = self._find_tosca_node(tosca_name, current_tosca_resource)
605         if tosca_node:
606             return self.hot_lookup[tosca_node]
607
608         return None
609
610     def _create_connect_configs(self, source_node, target_name,
611                                 connect_interfaces):
612         connectsto_resources = []
613         if connect_interfaces:
614             for iname, interface in six.iteritems(connect_interfaces):
615                 connectsto_resources += \
616                     self._create_connect_config(source_node, target_name,
617                                                 interface)
618         return connectsto_resources
619
620     def _create_connect_config(self, source_node, target_name,
621                                connect_interface):
622         connectsto_resources = []
623         target_node = self._find_tosca_node(target_name)
624         # the configuration can occur on the source or the target
625         connect_config = connect_interface.get('pre_configure_target')
626         if connect_config is not None:
627             config_location = 'target'
628         else:
629             connect_config = connect_interface.get('pre_configure_source')
630             if connect_config is not None:
631                 config_location = 'source'
632             else:
633                 msg = _("Template error:  "
634                         "no configuration found for ConnectsTo "
635                         "in {1}").format(self.nodetemplate.name)
636                 log.error(msg)
637                 raise Exception(msg)
638         config_name = source_node.name + '_' + target_name + '_connect_config'
639         implement = connect_config.get('implementation')
640         if config_location == 'target':
641             hot_config = HotResource(target_node,
642                                      config_name,
643                                      'OS::Heat::SoftwareConfig',
644                                      {'config': {'get_file': implement}})
645         elif config_location == 'source':
646             hot_config = HotResource(source_node,
647                                      config_name,
648                                      'OS::Heat::SoftwareConfig',
649                                      {'config': {'get_file': implement}})
650         connectsto_resources.append(hot_config)
651         hot_target = self._find_hot_resource_for_tosca(target_name)
652         hot_source = self._find_hot_resource_for_tosca(source_node.name)
653         connectsto_resources.append(hot_config.
654                                     handle_connectsto(source_node,
655                                                       target_node,
656                                                       hot_source,
657                                                       hot_target,
658                                                       config_location,
659                                                       connect_interface))
660         return connectsto_resources