2 # Copyright (c) 2016 Cable Television Laboratories, Inc. ("CableLabs")
3 # and others. All rights reserved.
5 # Licensed under the Apache License, Version 2.0 (the "License");
6 # you may not use this file except in compliance with the License.
7 # You may obtain a copy of the License at:
9 # http://www.apache.org/licenses/LICENSE-2.0
11 # Unless required by applicable law or agreed to in writing, software
12 # distributed under the License is distributed on an "AS IS" BASIS,
13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 # See the License for the specific language governing permissions and
15 # limitations under the License.
17 # This utility makes it easy to create OpenStack objects
24 from keystoneauth1.exceptions import Unauthorized
26 from snaps.config.flavor import FlavorConfig
27 from snaps.config.image import ImageConfig
28 from snaps.config.keypair import KeypairConfig
29 from snaps.config.network import PortConfig, NetworkConfig
30 from snaps.config.project import ProjectConfig
31 from snaps.config.qos import QoSConfig
32 from snaps.config.router import RouterConfig
33 from snaps.config.security_group import SecurityGroupConfig
34 from snaps.config.user import UserConfig
35 from snaps.config.vm_inst import VmInstanceConfig
36 from snaps.config.volume import VolumeConfig
37 from snaps.config.volume_type import VolumeTypeConfig
38 from snaps.openstack.create_flavor import OpenStackFlavor
39 from snaps.openstack.create_image import OpenStackImage
40 from snaps.openstack.create_keypairs import OpenStackKeypair
41 from snaps.openstack.create_network import OpenStackNetwork
42 from snaps.openstack.create_project import OpenStackProject
43 from snaps.openstack.create_qos import OpenStackQoS
44 from snaps.openstack.create_router import OpenStackRouter
45 from snaps.openstack.create_security_group import OpenStackSecurityGroup
46 from snaps.openstack.create_user import OpenStackUser
47 from snaps.openstack.create_volume import OpenStackVolume
48 from snaps.openstack.create_volume_type import OpenStackVolumeType
49 from snaps.openstack.os_credentials import OSCreds, ProxySettings
50 from snaps.openstack.utils import deploy_utils, neutron_utils
51 from snaps.provisioning import ansible_utils
53 logger = logging.getLogger('lanuch_utils')
54 DEFAULT_CREDS_KEY = 'admin'
57 def launch_config(config, tmplt_file, deploy, clean, clean_image):
59 Launches all objects and applies any configured ansible playbooks
60 :param config: the environment configuration dict object
61 :param tmplt_file: the path to the SNAPS-OO template file
62 :param deploy: when True deploy
63 :param clean: when True clean
64 :param clean_image: when True clean the image when clean is True
66 os_config = config.get('openstack')
72 networks_dict = dict()
74 os_creds_dict = dict()
77 os_creds_dict = __get_creds_dict(os_config)
80 projects_dict = __create_instances(
81 os_creds_dict, OpenStackProject, ProjectConfig,
82 os_config.get('projects'), 'project', clean)
83 creators.append(projects_dict)
86 users_dict = __create_instances(
87 os_creds_dict, OpenStackUser, UserConfig,
88 os_config.get('users'), 'user', clean)
89 creators.append(users_dict)
91 # Associate new users to projects
93 for project_creator in projects_dict.values():
94 users = project_creator.project_settings.users
95 for user_name in users:
96 user_creator = users_dict.get(user_name)
98 project_creator.assoc_user(
99 user_creator.get_user())
102 flavors_dict = __create_instances(
103 os_creds_dict, OpenStackFlavor, FlavorConfig,
104 os_config.get('flavors'), 'flavor', clean, users_dict)
105 creators.append(flavors_dict)
108 qos_dict = __create_instances(
109 os_creds_dict, OpenStackQoS, QoSConfig,
110 os_config.get('qos_specs'), 'qos_spec', clean, users_dict)
111 creators.append(qos_dict)
113 # Create volume types
114 vol_type_dict = __create_instances(
115 os_creds_dict, OpenStackVolumeType, VolumeTypeConfig,
116 os_config.get('volume_types'), 'volume_type', clean,
118 creators.append(vol_type_dict)
120 # Create volume types
121 vol_dict = __create_instances(
122 os_creds_dict, OpenStackVolume, VolumeConfig,
123 os_config.get('volumes'), 'volume', clean, users_dict)
124 creators.append(vol_dict)
127 images_dict = __create_instances(
128 os_creds_dict, OpenStackImage, ImageConfig,
129 os_config.get('images'), 'image', clean, users_dict)
130 creators.append(images_dict)
133 networks_dict = __create_instances(
134 os_creds_dict, OpenStackNetwork, NetworkConfig,
135 os_config.get('networks'), 'network', clean, users_dict)
136 creators.append(networks_dict)
139 routers_dict = __create_instances(
140 os_creds_dict, OpenStackRouter, RouterConfig,
141 os_config.get('routers'), 'router', clean, users_dict)
142 creators.append(routers_dict)
145 keypairs_dict = __create_instances(
146 os_creds_dict, OpenStackKeypair, KeypairConfig,
147 os_config.get('keypairs'), 'keypair', clean, users_dict)
148 creators.append(keypairs_dict)
150 # Create security groups
151 creators.append(__create_instances(
152 os_creds_dict, OpenStackSecurityGroup,
154 os_config.get('security_groups'), 'security_group', clean,
158 vm_dict = __create_vm_instances(
159 os_creds_dict, users_dict, os_config.get('instances'),
160 images_dict, keypairs_dict, clean)
161 creators.append(vm_dict)
163 'Completed creating/retrieving all configured instances')
165 # Must enter either block
167 # Cleanup Environment
168 __cleanup(creators, clean_image)
171 ansible_config = config.get('ansible')
172 if ansible_config and vm_dict:
173 if not __apply_ansible_playbooks(
174 ansible_config, os_creds_dict, vm_dict, images_dict,
175 flavors_dict, networks_dict, routers_dict, tmplt_file):
176 logger.error("Problem applying ansible playbooks")
179 def __get_creds_dict(os_conn_config):
181 Returns a dict of OSCreds where the key is the creds name.
182 For backwards compatibility, credentials not contained in a list (only
183 one) will be returned with the key of None
184 :param os_conn_config: the credential configuration
185 :return: a dict of OSCreds objects
187 if 'connection' in os_conn_config:
188 return {DEFAULT_CREDS_KEY: __get_os_credentials(os_conn_config)}
189 elif 'connections' in os_conn_config:
191 for os_conn_dict in os_conn_config['connections']:
192 config = os_conn_dict.get('connection')
194 raise Exception('Invalid connection format')
196 name = config.get('name')
198 raise Exception('Connection config requires a name field')
200 out[name] = __get_os_credentials(os_conn_dict)
204 def __get_creds(os_creds_dict, os_user_dict, inst_config):
206 Returns the appropriate credentials
207 :param os_creds_dict: a dictionary of OSCreds objects where the name is the
209 :param os_user_dict: a dictionary of OpenStackUser objects where the name
212 :return: an OSCreds instance or None
214 os_creds = os_creds_dict.get(DEFAULT_CREDS_KEY)
215 if 'os_user' in inst_config:
216 os_user_conf = inst_config['os_user']
217 if 'name' in os_user_conf:
218 user_creator = os_user_dict.get(os_user_conf['name'])
220 return user_creator.get_os_creds(
221 project_name=os_user_conf.get('project_name'))
222 elif 'os_creds_name' in inst_config:
223 if 'os_creds_name' in inst_config:
224 os_creds = os_creds_dict[inst_config['os_creds_name']]
228 def __get_os_credentials(os_conn_config):
230 Returns an object containing all of the information required to access
232 :param os_conn_config: The configuration holding the credentials
233 :return: an OSCreds instance
235 config = os_conn_config.get('connection')
237 raise Exception('Invalid connection configuration')
239 proxy_settings = None
240 http_proxy = config.get('http_proxy')
242 tokens = re.split(':', http_proxy)
243 ssh_proxy_cmd = config.get('ssh_proxy_cmd')
244 proxy_settings = ProxySettings(host=tokens[0], port=tokens[1],
245 ssh_proxy_cmd=ssh_proxy_cmd)
247 if 'proxy_settings' in config:
248 host = config['proxy_settings'].get('host')
249 port = config['proxy_settings'].get('port')
250 if host and host != 'None' and port and port != 'None':
251 proxy_settings = ProxySettings(**config['proxy_settings'])
254 config['proxy_settings'] = proxy_settings
256 if config.get('proxy_settings'):
257 del config['proxy_settings']
259 return OSCreds(**config)
262 def __parse_ports_config(config):
264 Parses the "ports" configuration
265 :param config: The dictionary to parse
266 :return: a list of PortConfig objects
269 for port_config in config:
270 out.append(PortConfig(**port_config.get('port')))
274 def __create_instances(os_creds_dict, creator_class, config_class, config,
275 config_key, cleanup=False, os_users_dict=None):
277 Returns a dictionary of SNAPS creator objects where the key is the name
278 :param os_creds_dict: Dictionary of OSCreds objects where the key is the
280 :param config: The list of configurations for the same type
281 :param config_key: The list of configurations for the same type
282 :param cleanup: Denotes whether or not this is being called for cleanup
288 for config_dict in config:
289 inst_config = config_dict.get(config_key)
291 creds = __get_creds(os_creds_dict, os_users_dict, inst_config)
293 creator = creator_class(
295 config_class(**inst_config))
301 except Unauthorized as e:
303 'Unable to initialize creator [%s] - %s',
308 out[inst_config['name']] = creator
310 logger.info('Initialized configured %ss', config_key)
315 def __create_vm_instances(os_creds_dict, os_users_dict, instances_config,
316 image_dict, keypairs_dict, cleanup=False):
318 Returns a dictionary of OpenStackVmInstance objects where the key is the
320 :param os_creds_dict: Dictionary of OSCreds objects where the key is the
322 :param os_users_dict: Dictionary of OpenStackUser objects where the key is
324 :param instances_config: The list of VM instance configurations
325 :param image_dict: A dictionary of images that will probably be used to
326 instantiate the VM instance
327 :param keypairs_dict: A dictionary of keypairs that will probably be used
328 to instantiate the VM instance
329 :param cleanup: Denotes whether or not this is being called for cleanup
335 for instance_config in instances_config:
336 conf = instance_config.get('instance')
339 image_creator = image_dict.get(conf.get('imageName'))
341 instance_settings = VmInstanceConfig(
342 **instance_config['instance'])
343 kp_creator = keypairs_dict.get(
344 conf.get('keypair_name'))
348 'name']] = deploy_utils.create_vm_instance(
350 os_creds_dict, os_users_dict, conf),
352 image_creator.image_settings,
353 keypair_creator=kp_creator,
355 except Unauthorized as e:
357 logger.warn('Unable to initialize VM - %s', e)
360 raise Exception('Image creator instance not found.'
361 ' Cannot instantiate')
364 raise Exception('Image dictionary is None. Cannot '
367 raise Exception('Instance configuration is None. Cannot '
369 logger.info('Created configured instances')
374 def __apply_ansible_playbooks(ansible_configs, os_creds_dict, vm_dict,
375 image_dict, flavor_dict, networks_dict,
376 routers_dict, tmplt_file):
378 Applies ansible playbooks to running VMs with floating IPs
379 :param ansible_configs: a list of Ansible configurations
380 :param os_creds_dict: Dictionary of OSCreds objects where the key is the
382 :param vm_dict: the dictionary of newly instantiated VMs where the name is
384 :param image_dict: the dictionary of newly instantiated images where the
386 :param flavor_dict: the dictionary of newly instantiated flavors where the
388 :param networks_dict: the dictionary of newly instantiated networks where
390 :param routers_dict: the dictionary of newly instantiated routers where
392 :param tmplt_file: the path of the SNAPS-OO template file for setting the
393 CWD so playbook location is relative to the deployment
395 :return: t/f - true if successful
397 logger.info("Applying Ansible Playbooks")
399 # Set CWD so the deployment file's playbook location can leverage
401 orig_cwd = os.getcwd()
402 env_dir = os.path.dirname(tmplt_file)
406 for ansible_config in ansible_configs:
407 # Ensure all hosts are accepting SSH session requests
408 for vm_name in ansible_config['hosts']:
409 vm_inst = vm_dict.get(vm_name)
411 if not vm_inst.vm_ssh_active(block=True):
413 'Timeout waiting for instance to respond to '
417 os_creds = os_creds_dict.get('admin-creds')
418 __apply_ansible_playbook(
419 ansible_config, os_creds, vm_dict, image_dict, flavor_dict,
420 networks_dict, routers_dict)
422 # Return to original directory
428 def __apply_ansible_playbook(ansible_config, os_creds, vm_dict, image_dict,
429 flavor_dict, networks_dict, routers_dict):
431 Applies an Ansible configuration setting
432 :param ansible_config: the configuration settings
433 :param os_creds: the OpenStack admin credentials object
434 :param vm_dict: the dictionary of newly instantiated VMs where the name is
436 :param image_dict: the dictionary of newly instantiated images where the
438 :param flavor_dict: the dictionary of newly instantiated flavors where the
440 :param networks_dict: the dictionary of newly instantiated networks where
442 :param routers_dict: the dictionary of newly instantiated routers where
446 (remote_user, floating_ips, private_key_filepath,
447 proxy_settings) = __get_connection_info(
448 ansible_config, vm_dict)
450 for key, vm_creator in vm_dict.items():
451 fip = vm_creator.get_floating_ip()
452 if fip and fip.ip in floating_ips:
453 if not vm_creator.cloud_init_complete(block=True):
455 'Cannot apply playbooks as cloud-init has not '
458 variables = __get_variables(
459 ansible_config.get('variables'), os_creds, vm_dict, image_dict,
460 flavor_dict, networks_dict, routers_dict)
462 retval = ansible_utils.apply_playbook(
463 ansible_config['playbook_location'], floating_ips, remote_user,
464 private_key_filepath,
466 proxy_setting=proxy_settings)
468 # Not a fatal type of event
470 'Unable to apply playbook found at location - %s',
471 ansible_config.get('playbook_location'))
475 def __get_connection_info(ansible_config, vm_dict):
477 Returns a tuple of data required for connecting to the running VMs
478 (remote_user, [floating_ips], private_key_filepath, proxy_settings)
479 :param ansible_config: the configuration settings
480 :param vm_dict: the dictionary of VMs where the VM name is the key
481 :return: tuple where the first element is the user and the second is a list
482 of floating IPs and the third is the
483 private key file location and the fourth is an instance of the
484 snaps.ProxySettings class
485 (note: in order to work, each of the hosts need to have the same sudo_user
486 and private key file location values)
488 if ansible_config.get('hosts'):
489 hosts = ansible_config['hosts']
491 floating_ips = list()
494 proxy_settings = None
496 vm = vm_dict.get(host)
498 fip = vm.get_floating_ip()
500 remote_user = vm.get_image_user()
503 floating_ips.append(fip.ip)
506 'Could not find floating IP for VM - ' +
509 pk_file = vm.keypair_settings.private_filepath
510 proxy_settings = vm.get_os_creds().proxy_settings
512 logger.error('Could not locate VM with name - ' + host)
514 return remote_user, floating_ips, pk_file, proxy_settings
518 def __get_variables(var_config, os_creds, vm_dict, image_dict, flavor_dict,
519 networks_dict, routers_dict):
521 Returns a dictionary of substitution variables to be used for Ansible
523 :param var_config: the variable configuration settings
524 :param os_creds: the OpenStack admin credentials object
525 :param vm_dict: the dictionary of newly instantiated VMs where the name is
527 :param image_dict: the dictionary of newly instantiated images where the
529 :param flavor_dict: the dictionary of newly instantiated flavors where the
531 :param networks_dict: the dictionary of newly instantiated networks where
533 :param routers_dict: the dictionary of newly instantiated routers where
535 :return: dictionary or None
537 if var_config and vm_dict and len(vm_dict) > 0:
539 for key, value in var_config.items():
540 value = __get_variable_value(
541 value, os_creds, vm_dict, image_dict, flavor_dict,
542 networks_dict, routers_dict)
544 variables[key] = value
546 "Set Jinga2 variable with key [%s] the value [%s]",
550 'Key - [' + str(key) + '] or Value [' + str(value)
551 + '] must not be None')
556 def __get_variable_value(var_config_values, os_creds, vm_dict, image_dict,
557 flavor_dict, networks_dict, routers_dict):
559 Returns the associated variable value for use by Ansible for substitution
561 :param var_config_values: the configuration dictionary
562 :param os_creds: the OpenStack admin credentials object
563 :param vm_dict: the dictionary of newly instantiated VMs where the name is
565 :param image_dict: the dictionary of newly instantiated images where the
567 :param flavor_dict: the dictionary of newly instantiated flavors where the
569 :param networks_dict: the dictionary of newly instantiated networks where
571 :param routers_dict: the dictionary of newly instantiated routers where
575 if var_config_values['type'] == 'string':
576 return __get_string_variable_value(var_config_values)
577 if var_config_values['type'] == 'vm-attr':
578 return __get_vm_attr_variable_value(var_config_values, vm_dict)
579 if var_config_values['type'] == 'os_creds':
580 return __get_os_creds_variable_value(var_config_values, os_creds)
581 if var_config_values['type'] == 'network':
582 return __get_network_variable_value(var_config_values, networks_dict)
583 if var_config_values['type'] == 'router':
584 return __get_router_variable_value(var_config_values, routers_dict,
586 if var_config_values['type'] == 'port':
587 return __get_vm_port_variable_value(var_config_values, vm_dict)
588 if var_config_values['type'] == 'floating_ip':
589 return __get_vm_fip_variable_value(var_config_values, vm_dict)
590 if var_config_values['type'] == 'image':
591 return __get_image_variable_value(var_config_values, image_dict)
592 if var_config_values['type'] == 'flavor':
593 return __get_flavor_variable_value(var_config_values, flavor_dict)
597 def __get_string_variable_value(var_config_values):
599 Returns the associated string value
600 :param var_config_values: the configuration dictionary
601 :return: the value contained in the dictionary with the key 'value'
603 return var_config_values['value']
606 def __get_vm_attr_variable_value(var_config_values, vm_dict):
608 Returns the associated value contained on a VM instance
609 :param var_config_values: the configuration dictionary
610 :param vm_dict: the dictionary containing all VMs where the key is the VM's
614 vm = vm_dict.get(var_config_values['vm_name'])
616 if var_config_values['value'] == 'floating_ip':
617 return vm.get_floating_ip().ip
618 if var_config_values['value'] == 'image_user':
619 return vm.get_image_user()
622 def __get_os_creds_variable_value(var_config_values, os_creds):
624 Returns the associated OS credentials value
625 :param var_config_values: the configuration dictionary
626 :param os_creds: the admin OpenStack OSCreds object
630 if var_config_values['value'] == 'username':
631 logger.info("Returning OS username")
632 return os_creds.username
633 elif var_config_values['value'] == 'password':
634 logger.info("Returning OS password")
635 return os_creds.password
636 elif var_config_values['value'] == 'auth_url':
637 logger.info("Returning OS auth_url")
638 return os_creds.auth_url
639 elif var_config_values['value'] == 'project_name':
640 logger.info("Returning OS project_name")
641 return os_creds.project_name
644 def __get_network_variable_value(var_config_values, networks_dict):
646 Returns the associated network value
647 :param var_config_values: the configuration dictionary
648 :param networks_dict: the dictionary containing all networks where the key
652 net_name = var_config_values.get('network_name')
654 if net_name and networks_dict.get(net_name):
655 network_creator = networks_dict[net_name]
657 if 'subnet_name' in var_config_values:
658 subnet_name = var_config_values.get('subnet_name')
660 for subnet in network_creator.get_network().subnets:
661 if subnet_name == subnet.name:
662 if 'value' in var_config_values:
663 if 'gateway_ip' == var_config_values['value']:
664 return subnet.gateway_ip
665 if 'ip_range' == var_config_values['value']:
666 return subnet.start + ' ' + subnet.end
667 if 'cidr_ip' == var_config_values['value']:
668 cidr_split = subnet.cidr.split('/')
670 if 'netmask' == var_config_values['value']:
671 cidr_split = subnet.cidr.split('/')
672 cidr_bits = 32 - int(cidr_split[1])
673 netmask = socket.inet_ntoa(
675 '!I', (1 << 32) - (1 << cidr_bits)))
677 if 'broadcast_ip' == var_config_values['value']:
678 end_split = subnet.end.split('.')
680 end_split[0] + '.' + end_split[1] + '.'
681 + end_split[2] + '.255')
685 def __get_router_variable_value(var_config_values, routers_dict, os_creds):
687 Returns the associated network value
688 :param var_config_values: the configuration dictionary
689 :param routers_dict: the dictionary containing all networks where the key
691 :param os_creds: the admin OpenStack credentials
694 router_name = var_config_values.get('router_name')
695 router_creator = routers_dict[router_name]
698 if 'external_fixed_ip' == var_config_values.get('attr'):
699 neutron = neutron_utils.neutron_client(os_creds)
700 ext_nets = neutron_utils.get_external_networks(neutron)
702 subnet_name = var_config_values.get('subnet_name')
704 for ext_net in ext_nets:
705 for subnet in ext_net.subnets:
706 if subnet_name == subnet.name:
707 router = router_creator.get_router()
708 for fixed_ips in router.external_fixed_ips:
709 if subnet.id == fixed_ips['subnet_id']:
710 return fixed_ips['ip_address']
713 def __get_vm_port_variable_value(var_config_values, vm_dict):
715 Returns the associated OS credentials value
716 :param var_config_values: the configuration dictionary
717 :param vm_dict: the dictionary containing all VMs where the key is the VM's
721 port_name = var_config_values.get('port_name')
722 vm_name = var_config_values.get('vm_name')
724 if port_name and vm_name:
725 vm = vm_dict.get(vm_name)
727 for vm_port in vm.get_vm_inst().ports:
728 if vm_port.name == port_name:
729 port_value_id = var_config_values.get('port_value')
731 if port_value_id == 'mac_address':
732 return vm_port.mac_address
733 if port_value_id == 'ip_address':
734 return vm_port.ips[0]['ip_address']
737 def __get_vm_fip_variable_value(var_config_values, vm_dict):
739 Returns the floating IP value if found
740 :param var_config_values: the configuration dictionary
741 :param vm_dict: the dictionary containing all VMs where the key is the VM's
743 :return: the floating IP string value or None
745 fip_name = var_config_values.get('fip_name')
746 vm_name = var_config_values.get('vm_name')
749 vm = vm_dict.get(vm_name)
751 fip = vm.get_floating_ip(fip_name)
756 def __get_image_variable_value(var_config_values, image_dict):
758 Returns the associated image value
759 :param var_config_values: the configuration dictionary
760 :param image_dict: the dictionary containing all images where the key is
765 if var_config_values.get('image_name'):
766 image_creator = image_dict.get(var_config_values['image_name'])
768 if (var_config_values.get('value')
769 and var_config_values['value'] == 'id'):
770 return image_creator.get_image().id
771 if (var_config_values.get('value')
772 and var_config_values['value'] == 'user'):
773 return image_creator.image_settings.image_user
776 def __get_flavor_variable_value(var_config_values, flavor_dict):
778 Returns the associated flavor value
779 :param var_config_values: the configuration dictionary
780 :param flavor_dict: the dictionary containing all flavor creators where the
782 :return: the value or None
785 if var_config_values.get('flavor_name'):
786 flavor_creator = flavor_dict.get(var_config_values['flavor_name'])
788 if (var_config_values.get('value')
789 and var_config_values['value'] == 'id'):
790 return flavor_creator.get_flavor().id
793 def __cleanup(creators, clean_image=False):
795 Cleans up environment
796 :param creators: the list of creators by type
797 :param clean_image: when true
800 for creator_dict in reversed(creators):
801 for key, creator in creator_dict.items():
802 if ((isinstance(creator, OpenStackImage) and clean_image)
803 or not isinstance(creator, OpenStackImage)):