2 # Copyright (c) 2016 Cable Television Laboratories, Inc. ("CableLabs")
3 # and others. All rights reserved.
5 # Licensed under the Apache License, Version 2.0 (the "License");
6 # you may not use this file except in compliance with the License.
7 # You may obtain a copy of the License at:
9 # http://www.apache.org/licenses/LICENSE-2.0
11 # Unless required by applicable law or agreed to in writing, software
12 # distributed under the License is distributed on an "AS IS" BASIS,
13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 # See the License for the specific language governing permissions and
15 # limitations under the License.
17 # This utility makes it easy to create OpenStack objects
25 from keystoneauth1.exceptions import Unauthorized
27 from snaps.config.flavor import FlavorConfig
28 from snaps.config.image import ImageConfig
29 from snaps.config.keypair import KeypairConfig
30 from snaps.config.network import PortConfig, NetworkConfig
31 from snaps.config.project import ProjectConfig
32 from snaps.config.qos import QoSConfig
33 from snaps.config.router import RouterConfig
34 from snaps.config.security_group import SecurityGroupConfig
35 from snaps.config.user import UserConfig
36 from snaps.config.vm_inst import VmInstanceConfig
37 from snaps.config.volume import VolumeConfig
38 from snaps.config.volume_type import VolumeTypeConfig
39 from snaps.openstack.create_flavor import OpenStackFlavor
40 from snaps.openstack.create_image import OpenStackImage
41 from snaps.openstack.create_keypairs import OpenStackKeypair
42 from snaps.openstack.create_network import OpenStackNetwork
43 from snaps.openstack.create_project import OpenStackProject
44 from snaps.openstack.create_qos import OpenStackQoS
45 from snaps.openstack.create_router import OpenStackRouter
46 from snaps.openstack.create_security_group import OpenStackSecurityGroup
47 from snaps.openstack.create_user import OpenStackUser
48 from snaps.openstack.create_volume import OpenStackVolume
49 from snaps.openstack.create_volume_type import OpenStackVolumeType
50 from snaps.openstack.os_credentials import OSCreds, ProxySettings
51 from snaps.openstack.utils import deploy_utils, neutron_utils
52 from snaps.provisioning import ansible_utils
54 logger = logging.getLogger('lanuch_utils')
55 DEFAULT_CREDS_KEY = 'admin'
58 def launch_config(config, tmplt_file, deploy, clean, clean_image):
60 Launches all objects and applies any configured ansible playbooks
61 :param config: the environment configuration dict object
62 :param tmplt_file: the path to the SNAPS-OO template file
63 :param deploy: when True deploy
64 :param clean: when True clean
65 :param clean_image: when True clean the image when clean is True
67 os_config = config.get('openstack')
73 networks_dict = dict()
75 os_creds_dict = dict()
78 os_creds_dict = __get_creds_dict(os_config)
81 projects_dict = __create_instances(
82 os_creds_dict, OpenStackProject, ProjectConfig,
83 os_config.get('projects'), 'project', clean)
84 creators.append(projects_dict)
87 users_dict = __create_instances(
88 os_creds_dict, OpenStackUser, UserConfig,
89 os_config.get('users'), 'user', clean)
90 creators.append(users_dict)
92 # Associate new users to projects
94 for project_creator in projects_dict.values():
95 users = project_creator.project_settings.users
96 for user_name in users:
97 user_creator = users_dict.get(user_name)
99 project_creator.assoc_user(
100 user_creator.get_user())
103 flavors_dict = __create_instances(
104 os_creds_dict, OpenStackFlavor, FlavorConfig,
105 os_config.get('flavors'), 'flavor', clean, users_dict)
106 creators.append(flavors_dict)
109 qos_dict = __create_instances(
110 os_creds_dict, OpenStackQoS, QoSConfig,
111 os_config.get('qos_specs'), 'qos_spec', clean, users_dict)
112 creators.append(qos_dict)
114 # Create volume types
115 vol_type_dict = __create_instances(
116 os_creds_dict, OpenStackVolumeType, VolumeTypeConfig,
117 os_config.get('volume_types'), 'volume_type', clean,
119 creators.append(vol_type_dict)
121 # Create volume types
122 vol_dict = __create_instances(
123 os_creds_dict, OpenStackVolume, VolumeConfig,
124 os_config.get('volumes'), 'volume', clean, users_dict)
125 creators.append(vol_dict)
128 images_dict = __create_instances(
129 os_creds_dict, OpenStackImage, ImageConfig,
130 os_config.get('images'), 'image', clean, users_dict)
131 creators.append(images_dict)
134 networks_dict = __create_instances(
135 os_creds_dict, OpenStackNetwork, NetworkConfig,
136 os_config.get('networks'), 'network', clean, users_dict)
137 creators.append(networks_dict)
140 routers_dict = __create_instances(
141 os_creds_dict, OpenStackRouter, RouterConfig,
142 os_config.get('routers'), 'router', clean, users_dict)
143 creators.append(routers_dict)
146 keypairs_dict = __create_instances(
147 os_creds_dict, OpenStackKeypair, KeypairConfig,
148 os_config.get('keypairs'), 'keypair', clean, users_dict)
149 creators.append(keypairs_dict)
151 # Create security groups
152 creators.append(__create_instances(
153 os_creds_dict, OpenStackSecurityGroup,
155 os_config.get('security_groups'), 'security_group', clean,
159 vm_dict = __create_vm_instances(
160 os_creds_dict, users_dict, os_config.get('instances'),
161 images_dict, keypairs_dict, clean)
162 creators.append(vm_dict)
164 'Completed creating/retrieving all configured instances')
166 # Must enter either block
168 # Cleanup Environment
169 __cleanup(creators, clean_image)
172 ansible_config = config.get('ansible')
173 if ansible_config and vm_dict:
174 if not __apply_ansible_playbooks(
175 ansible_config, os_creds_dict, vm_dict, images_dict,
176 flavors_dict, networks_dict, routers_dict, tmplt_file):
177 logger.error("Problem applying ansible playbooks")
180 def __get_creds_dict(os_conn_config):
182 Returns a dict of OSCreds where the key is the creds name.
183 For backwards compatibility, credentials not contained in a list (only
184 one) will be returned with the key of None
185 :param os_conn_config: the credential configuration
186 :return: a dict of OSCreds objects
188 if 'connection' in os_conn_config:
189 return {DEFAULT_CREDS_KEY: __get_os_credentials(os_conn_config)}
190 elif 'connections' in os_conn_config:
192 for os_conn_dict in os_conn_config['connections']:
193 config = os_conn_dict.get('connection')
195 raise Exception('Invalid connection format')
197 name = config.get('name')
199 raise Exception('Connection config requires a name field')
201 out[name] = __get_os_credentials(os_conn_dict)
205 def __get_creds(os_creds_dict, os_user_dict, inst_config):
207 Returns the appropriate credentials
208 :param os_creds_dict: a dictionary of OSCreds objects where the name is the
210 :param os_user_dict: a dictionary of OpenStackUser objects where the name
213 :return: an OSCreds instance or None
215 os_creds = os_creds_dict.get(DEFAULT_CREDS_KEY)
216 if 'os_user' in inst_config:
217 os_user_conf = inst_config['os_user']
218 if 'name' in os_user_conf:
219 user_creator = os_user_dict.get(os_user_conf['name'])
221 return user_creator.get_os_creds(
222 project_name=os_user_conf.get('project_name'))
223 elif 'os_creds_name' in inst_config:
224 if 'os_creds_name' in inst_config:
225 os_creds = os_creds_dict[inst_config['os_creds_name']]
229 def __get_os_credentials(os_conn_config):
231 Returns an object containing all of the information required to access
233 :param os_conn_config: The configuration holding the credentials
234 :return: an OSCreds instance
236 config = os_conn_config.get('connection')
238 raise Exception('Invalid connection configuration')
240 proxy_settings = None
241 http_proxy = config.get('http_proxy')
243 tokens = re.split(':', http_proxy)
244 ssh_proxy_cmd = config.get('ssh_proxy_cmd')
245 proxy_settings = ProxySettings(host=tokens[0], port=tokens[1],
246 ssh_proxy_cmd=ssh_proxy_cmd)
248 if 'proxy_settings' in config:
249 host = config['proxy_settings'].get('host')
250 port = config['proxy_settings'].get('port')
251 if host and host != 'None' and port and port != 'None':
252 proxy_settings = ProxySettings(**config['proxy_settings'])
255 config['proxy_settings'] = proxy_settings
257 if config.get('proxy_settings'):
258 del config['proxy_settings']
260 return OSCreds(**config)
263 def __parse_ports_config(config):
265 Parses the "ports" configuration
266 :param config: The dictionary to parse
267 :return: a list of PortConfig objects
270 for port_config in config:
271 out.append(PortConfig(**port_config.get('port')))
275 def __create_instances(os_creds_dict, creator_class, config_class, config,
276 config_key, cleanup=False, os_users_dict=None):
278 Returns a dictionary of SNAPS creator objects where the key is the name
279 :param os_creds_dict: Dictionary of OSCreds objects where the key is the
281 :param config: The list of configurations for the same type
282 :param config_key: The list of configurations for the same type
283 :param cleanup: Denotes whether or not this is being called for cleanup
289 for config_dict in config:
290 inst_config = config_dict.get(config_key)
292 creds = __get_creds(os_creds_dict, os_users_dict, inst_config)
294 creator = creator_class(
296 config_class(**inst_config))
302 except Unauthorized as e:
304 'Unable to initialize creator [%s] - %s',
309 out[inst_config['name']] = creator
311 logger.info('Initialized configured %ss', config_key)
316 def __create_vm_instances(os_creds_dict, os_users_dict, instances_config,
317 image_dict, keypairs_dict, cleanup=False):
319 Returns a dictionary of OpenStackVmInstance objects where the key is the
321 :param os_creds_dict: Dictionary of OSCreds objects where the key is the
323 :param os_users_dict: Dictionary of OpenStackUser objects where the key is
325 :param instances_config: The list of VM instance configurations
326 :param image_dict: A dictionary of images that will probably be used to
327 instantiate the VM instance
328 :param keypairs_dict: A dictionary of keypairs that will probably be used
329 to instantiate the VM instance
330 :param cleanup: Denotes whether or not this is being called for cleanup
336 for instance_config in instances_config:
337 conf = instance_config.get('instance')
340 image_creator = image_dict.get(conf.get('imageName'))
342 instance_settings = VmInstanceConfig(
343 **instance_config['instance'])
344 kp_creator = keypairs_dict.get(
345 conf.get('keypair_name'))
349 'name']] = deploy_utils.create_vm_instance(
351 os_creds_dict, os_users_dict, conf),
353 image_creator.image_settings,
354 keypair_creator=kp_creator,
356 except Unauthorized as e:
358 logger.warn('Unable to initialize VM - %s', e)
361 raise Exception('Image creator instance not found.'
362 ' Cannot instantiate')
365 raise Exception('Image dictionary is None. Cannot '
368 raise Exception('Instance configuration is None. Cannot '
370 logger.info('Created configured instances')
375 def __apply_ansible_playbooks(ansible_configs, os_creds_dict, vm_dict,
376 image_dict, flavor_dict, networks_dict,
377 routers_dict, tmplt_file):
379 Applies ansible playbooks to running VMs with floating IPs
380 :param ansible_configs: a list of Ansible configurations
381 :param os_creds_dict: Dictionary of OSCreds objects where the key is the
383 :param vm_dict: the dictionary of newly instantiated VMs where the name is
385 :param image_dict: the dictionary of newly instantiated images where the
387 :param flavor_dict: the dictionary of newly instantiated flavors where the
389 :param networks_dict: the dictionary of newly instantiated networks where
391 :param routers_dict: the dictionary of newly instantiated routers where
393 :param tmplt_file: the path of the SNAPS-OO template file for setting the
394 CWD so playbook location is relative to the deployment
396 :return: t/f - true if successful
398 logger.info("Applying Ansible Playbooks")
400 # Set CWD so the deployment file's playbook location can leverage
402 orig_cwd = os.getcwd()
403 env_dir = os.path.dirname(tmplt_file)
407 for ansible_config in ansible_configs:
408 # Ensure all hosts are accepting SSH session requests
409 for vm_name in ansible_config['hosts']:
410 vm_inst = vm_dict.get(vm_name)
412 if not vm_inst.vm_ssh_active(block=True):
414 'Timeout waiting for instance to respond to '
418 os_creds = os_creds_dict.get('admin-creds')
419 __apply_ansible_playbook(
420 ansible_config, os_creds, vm_dict, image_dict, flavor_dict,
421 networks_dict, routers_dict)
423 # Return to original directory
429 def __apply_ansible_playbook(ansible_config, os_creds, vm_dict, image_dict,
430 flavor_dict, networks_dict, routers_dict):
432 Applies an Ansible configuration setting
433 :param ansible_config: the configuration settings
434 :param os_creds: the OpenStack admin credentials object
435 :param vm_dict: the dictionary of newly instantiated VMs where the name is
437 :param image_dict: the dictionary of newly instantiated images where the
439 :param flavor_dict: the dictionary of newly instantiated flavors where the
441 :param networks_dict: the dictionary of newly instantiated networks where
443 :param routers_dict: the dictionary of newly instantiated routers where
447 (remote_user, floating_ips, private_key_filepath,
448 proxy_settings) = __get_connection_info(
449 ansible_config, vm_dict)
451 for key, vm_creator in vm_dict.items():
452 fip = vm_creator.get_floating_ip()
453 if fip and fip.ip in floating_ips:
454 if not vm_creator.cloud_init_complete(block=True):
456 'Cannot apply playbooks as cloud-init has not '
459 variables = __get_variables(
460 ansible_config.get('variables'), os_creds, vm_dict, image_dict,
461 flavor_dict, networks_dict, routers_dict)
463 retval = ansible_utils.apply_playbook(
464 ansible_config['playbook_location'], floating_ips, remote_user,
465 private_key_filepath,
467 proxy_setting=proxy_settings)
469 # Not a fatal type of event
471 'Unable to apply playbook found at location - %s',
472 ansible_config.get('playbook_location'))
473 elif ansible_config.get('post_processing'):
474 post_proc_config = ansible_config['post_processing']
475 if 'sleep' in post_proc_config:
476 time.sleep(post_proc_config['sleep'])
477 if 'reboot' in post_proc_config:
478 for vm_name in post_proc_config['reboot']:
479 if vm_name in vm_dict:
480 logger.info('Rebooting VM - %s', vm_name)
481 vm_dict[vm_name].reboot()
486 def __get_connection_info(ansible_config, vm_dict):
488 Returns a tuple of data required for connecting to the running VMs
489 (remote_user, [floating_ips], private_key_filepath, proxy_settings)
490 :param ansible_config: the configuration settings
491 :param vm_dict: the dictionary of VMs where the VM name is the key
492 :return: tuple where the first element is the user and the second is a list
493 of floating IPs and the third is the
494 private key file location and the fourth is an instance of the
495 snaps.ProxySettings class
496 (note: in order to work, each of the hosts need to have the same sudo_user
497 and private key file location values)
499 if ansible_config.get('hosts'):
500 hosts = ansible_config['hosts']
502 floating_ips = list()
505 proxy_settings = None
507 vm = vm_dict.get(host)
509 fip = vm.get_floating_ip()
511 remote_user = vm.get_image_user()
514 floating_ips.append(fip.ip)
517 'Could not find floating IP for VM - ' +
520 pk_file = vm.keypair_settings.private_filepath
521 proxy_settings = vm.get_os_creds().proxy_settings
523 logger.error('Could not locate VM with name - ' + host)
525 return remote_user, floating_ips, pk_file, proxy_settings
529 def __get_variables(var_config, os_creds, vm_dict, image_dict, flavor_dict,
530 networks_dict, routers_dict):
532 Returns a dictionary of substitution variables to be used for Ansible
534 :param var_config: the variable configuration settings
535 :param os_creds: the OpenStack admin credentials object
536 :param vm_dict: the dictionary of newly instantiated VMs where the name is
538 :param image_dict: the dictionary of newly instantiated images where the
540 :param flavor_dict: the dictionary of newly instantiated flavors where the
542 :param networks_dict: the dictionary of newly instantiated networks where
544 :param routers_dict: the dictionary of newly instantiated routers where
546 :return: dictionary or None
548 if var_config and vm_dict and len(vm_dict) > 0:
550 for key, value in var_config.items():
551 value = __get_variable_value(
552 value, os_creds, vm_dict, image_dict, flavor_dict,
553 networks_dict, routers_dict)
555 variables[key] = value
557 "Set Jinga2 variable with key [%s] the value [%s]",
561 'Key - [' + str(key) + '] or Value [' + str(value)
562 + '] must not be None')
567 def __get_variable_value(var_config_values, os_creds, vm_dict, image_dict,
568 flavor_dict, networks_dict, routers_dict):
570 Returns the associated variable value for use by Ansible for substitution
572 :param var_config_values: the configuration dictionary
573 :param os_creds: the OpenStack admin credentials object
574 :param vm_dict: the dictionary of newly instantiated VMs where the name is
576 :param image_dict: the dictionary of newly instantiated images where the
578 :param flavor_dict: the dictionary of newly instantiated flavors where the
580 :param networks_dict: the dictionary of newly instantiated networks where
582 :param routers_dict: the dictionary of newly instantiated routers where
586 if var_config_values['type'] == 'string':
587 return __get_string_variable_value(var_config_values)
588 if var_config_values['type'] == 'vm-attr':
589 return __get_vm_attr_variable_value(var_config_values, vm_dict)
590 if var_config_values['type'] == 'os_creds':
591 return __get_os_creds_variable_value(var_config_values, os_creds)
592 if var_config_values['type'] == 'network':
593 return __get_network_variable_value(var_config_values, networks_dict)
594 if var_config_values['type'] == 'router':
595 return __get_router_variable_value(var_config_values, routers_dict,
597 if var_config_values['type'] == 'port':
598 return __get_vm_port_variable_value(var_config_values, vm_dict)
599 if var_config_values['type'] == 'floating_ip':
600 return __get_vm_fip_variable_value(var_config_values, vm_dict)
601 if var_config_values['type'] == 'image':
602 return __get_image_variable_value(var_config_values, image_dict)
603 if var_config_values['type'] == 'flavor':
604 return __get_flavor_variable_value(var_config_values, flavor_dict)
608 def __get_string_variable_value(var_config_values):
610 Returns the associated string value
611 :param var_config_values: the configuration dictionary
612 :return: the value contained in the dictionary with the key 'value'
614 return var_config_values['value']
617 def __get_vm_attr_variable_value(var_config_values, vm_dict):
619 Returns the associated value contained on a VM instance
620 :param var_config_values: the configuration dictionary
621 :param vm_dict: the dictionary containing all VMs where the key is the VM's
625 vm = vm_dict.get(var_config_values['vm_name'])
627 if var_config_values['value'] == 'floating_ip':
628 return vm.get_floating_ip().ip
629 if var_config_values['value'] == 'image_user':
630 return vm.get_image_user()
633 def __get_os_creds_variable_value(var_config_values, os_creds):
635 Returns the associated OS credentials value
636 :param var_config_values: the configuration dictionary
637 :param os_creds: the admin OpenStack OSCreds object
641 if var_config_values['value'] == 'username':
642 logger.info("Returning OS username")
643 return os_creds.username
644 elif var_config_values['value'] == 'password':
645 logger.info("Returning OS password")
646 return os_creds.password
647 elif var_config_values['value'] == 'auth_url':
648 logger.info("Returning OS auth_url")
649 return os_creds.auth_url
650 elif var_config_values['value'] == 'project_name':
651 logger.info("Returning OS project_name")
652 return os_creds.project_name
655 def __get_network_variable_value(var_config_values, networks_dict):
657 Returns the associated network value
658 :param var_config_values: the configuration dictionary
659 :param networks_dict: the dictionary containing all networks where the key
663 net_name = var_config_values.get('network_name')
665 if net_name and networks_dict.get(net_name):
666 network_creator = networks_dict[net_name]
668 if 'subnet_name' in var_config_values:
669 subnet_name = var_config_values.get('subnet_name')
671 for subnet in network_creator.get_network().subnets:
672 if subnet_name == subnet.name:
673 if 'value' in var_config_values:
674 if 'gateway_ip' == var_config_values['value']:
675 return subnet.gateway_ip
676 if 'ip_range' == var_config_values['value']:
677 return subnet.start + ' ' + subnet.end
678 if 'cidr_ip' == var_config_values['value']:
679 cidr_split = subnet.cidr.split('/')
681 if 'netmask' == var_config_values['value']:
682 cidr_split = subnet.cidr.split('/')
683 cidr_bits = 32 - int(cidr_split[1])
684 netmask = socket.inet_ntoa(
686 '!I', (1 << 32) - (1 << cidr_bits)))
688 if 'broadcast_ip' == var_config_values['value']:
689 end_split = subnet.end.split('.')
691 end_split[0] + '.' + end_split[1] + '.'
692 + end_split[2] + '.255')
696 def __get_router_variable_value(var_config_values, routers_dict, os_creds):
698 Returns the associated network value
699 :param var_config_values: the configuration dictionary
700 :param routers_dict: the dictionary containing all networks where the key
702 :param os_creds: the admin OpenStack credentials
705 router_name = var_config_values.get('router_name')
706 router_creator = routers_dict[router_name]
709 if 'external_fixed_ip' == var_config_values.get('attr'):
710 neutron = neutron_utils.neutron_client(os_creds)
711 ext_nets = neutron_utils.get_external_networks(neutron)
713 subnet_name = var_config_values.get('subnet_name')
715 for ext_net in ext_nets:
716 for subnet in ext_net.subnets:
717 if subnet_name == subnet.name:
718 router = router_creator.get_router()
719 for fixed_ips in router.external_fixed_ips:
720 if subnet.id == fixed_ips['subnet_id']:
721 return fixed_ips['ip_address']
724 def __get_vm_port_variable_value(var_config_values, vm_dict):
726 Returns the associated OS credentials value
727 :param var_config_values: the configuration dictionary
728 :param vm_dict: the dictionary containing all VMs where the key is the VM's
732 port_name = var_config_values.get('port_name')
733 vm_name = var_config_values.get('vm_name')
735 if port_name and vm_name:
736 vm = vm_dict.get(vm_name)
738 for vm_port in vm.get_vm_inst().ports:
739 if vm_port.name == port_name:
740 port_value_id = var_config_values.get('port_value')
742 if port_value_id == 'mac_address':
743 return vm_port.mac_address
744 if port_value_id == 'ip_address':
745 return vm_port.ips[0]['ip_address']
748 def __get_vm_fip_variable_value(var_config_values, vm_dict):
750 Returns the floating IP value if found
751 :param var_config_values: the configuration dictionary
752 :param vm_dict: the dictionary containing all VMs where the key is the VM's
754 :return: the floating IP string value or None
756 fip_name = var_config_values.get('fip_name')
757 vm_name = var_config_values.get('vm_name')
760 vm = vm_dict.get(vm_name)
762 fip = vm.get_floating_ip(fip_name)
767 def __get_image_variable_value(var_config_values, image_dict):
769 Returns the associated image value
770 :param var_config_values: the configuration dictionary
771 :param image_dict: the dictionary containing all images where the key is
776 if var_config_values.get('image_name'):
777 image_creator = image_dict.get(var_config_values['image_name'])
779 if (var_config_values.get('value')
780 and var_config_values['value'] == 'id'):
781 return image_creator.get_image().id
782 if (var_config_values.get('value')
783 and var_config_values['value'] == 'user'):
784 return image_creator.image_settings.image_user
787 def __get_flavor_variable_value(var_config_values, flavor_dict):
789 Returns the associated flavor value
790 :param var_config_values: the configuration dictionary
791 :param flavor_dict: the dictionary containing all flavor creators where the
793 :return: the value or None
796 if var_config_values.get('flavor_name'):
797 flavor_creator = flavor_dict.get(var_config_values['flavor_name'])
799 if (var_config_values.get('value')
800 and var_config_values['value'] == 'id'):
801 return flavor_creator.get_flavor().id
804 def __cleanup(creators, clean_image=False):
806 Cleans up environment
807 :param creators: the list of creators by type
808 :param clean_image: when true
811 for creator_dict in reversed(creators):
812 for key, creator in creator_dict.items():
813 if ((isinstance(creator, OpenStackImage) and clean_image)
814 or not isinstance(creator, OpenStackImage)):