3 # Copyright (c) 2017 Cable Television Laboratories, Inc. ("CableLabs")
4 # and others. All rights reserved.
6 # Licensed under the Apache License, Version 2.0 (the "License");
7 # you may not use this file except in compliance with the License.
8 # You may obtain a copy of the License at:
10 # http://www.apache.org/licenses/LICENSE-2.0
12 # Unless required by applicable law or agreed to in writing, software
13 # distributed under the License is distributed on an "AS IS" BASIS,
14 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 # See the License for the specific language governing permissions and
16 # limitations under the License.
18 # This script is responsible for deploying virtual environments
24 from jinja2 import Environment, FileSystemLoader
28 from snaps import file_utils
29 from snaps.config.flavor import FlavorConfig
30 from snaps.config.image import ImageConfig
31 from snaps.config.keypair import KeypairConfig
32 from snaps.openstack.create_flavor import OpenStackFlavor
33 from snaps.openstack.create_image import OpenStackImage
34 from snaps.openstack.create_instance import VmInstanceSettings
35 from snaps.openstack.create_keypairs import OpenStackKeypair
36 from snaps.openstack.create_network import (
37 PortSettings, NetworkSettings, OpenStackNetwork)
38 from snaps.openstack.create_project import OpenStackProject, ProjectSettings
39 from snaps.openstack.create_qos import QoSSettings, OpenStackQoS
40 from snaps.openstack.create_router import RouterSettings, OpenStackRouter
41 from snaps.openstack.create_security_group import (
42 OpenStackSecurityGroup, SecurityGroupSettings)
43 from snaps.openstack.create_user import OpenStackUser, UserSettings
44 from snaps.openstack.create_volume import OpenStackVolume, VolumeSettings
45 from snaps.openstack.create_volume_type import (
46 OpenStackVolumeType, VolumeTypeSettings)
47 from snaps.openstack.os_credentials import OSCreds, ProxySettings
48 from snaps.openstack.utils import deploy_utils
49 from snaps.provisioning import ansible_utils
51 __author__ = 'spisarski'
53 logger = logging.getLogger('snaps_launcher')
55 ARG_NOT_SET = "argument not set"
56 DEFAULT_CREDS_KEY = 'admin'
59 def __get_creds_dict(os_conn_config):
61 Returns a dict of OSCreds where the key is the creds name.
62 For backwards compatibility, credentials not contained in a list (only
63 one) will be returned with the key of None
64 :param os_conn_config: the credential configuration
65 :return: a dict of OSCreds objects
67 if 'connection' in os_conn_config:
68 return {DEFAULT_CREDS_KEY: __get_os_credentials(os_conn_config)}
69 elif 'connections' in os_conn_config:
71 for os_conn_dict in os_conn_config['connections']:
72 config = os_conn_dict.get('connection')
74 raise Exception('Invalid connection format')
76 name = config.get('name')
78 raise Exception('Connection config requires a name field')
80 out[name] = __get_os_credentials(os_conn_dict)
84 def __get_creds(os_creds_dict, os_user_dict, inst_config):
86 Returns the appropriate credentials
87 :param os_creds_dict: a dictionary of OSCreds objects where the name is the
89 :param os_user_dict: a dictionary of OpenStackUser objects where the name
92 :return: an OSCreds instance or None
94 os_creds = os_creds_dict.get(DEFAULT_CREDS_KEY)
95 if 'os_user' in inst_config:
96 os_user_conf = inst_config['os_user']
97 if 'name' in os_user_conf:
98 user_creator = os_user_dict.get(os_user_conf['name'])
100 return user_creator.get_os_creds(
101 project_name=os_user_conf.get('project_name'))
102 elif 'os_creds_name' in inst_config:
103 if 'os_creds_name' in inst_config:
104 os_creds = os_creds_dict[inst_config['os_creds_name']]
108 def __get_os_credentials(os_conn_config):
110 Returns an object containing all of the information required to access
112 :param os_conn_config: The configuration holding the credentials
113 :return: an OSCreds instance
115 config = os_conn_config.get('connection')
117 raise Exception('Invalid connection configuration')
119 proxy_settings = None
120 http_proxy = config.get('http_proxy')
122 tokens = re.split(':', http_proxy)
123 ssh_proxy_cmd = config.get('ssh_proxy_cmd')
124 proxy_settings = ProxySettings(host=tokens[0], port=tokens[1],
125 ssh_proxy_cmd=ssh_proxy_cmd)
127 if 'proxy_settings' in config:
128 host = config['proxy_settings'].get('host')
129 port = config['proxy_settings'].get('port')
130 if host and host != 'None' and port and port != 'None':
131 proxy_settings = ProxySettings(**config['proxy_settings'])
134 config['proxy_settings'] = proxy_settings
136 if config.get('proxy_settings'):
137 del config['proxy_settings']
139 return OSCreds(**config)
142 def __parse_ports_config(config):
144 Parses the "ports" configuration
145 :param config: The dictionary to parse
146 :return: a list of PortConfig objects
149 for port_config in config:
150 out.append(PortSettings(**port_config.get('port')))
154 def __create_instances(os_creds_dict, creator_class, config_class, config,
155 config_key, cleanup=False, os_users_dict=None):
157 Returns a dictionary of SNAPS creator objects where the key is the name
158 :param os_creds_dict: Dictionary of OSCreds objects where the key is the
160 :param config: The list of configurations for the same type
161 :param config_key: The list of configurations for the same type
162 :param cleanup: Denotes whether or not this is being called for cleanup
169 for config_dict in config:
170 inst_config = config_dict.get(config_key)
172 creator = creator_class(
173 __get_creds(os_creds_dict, os_users_dict, inst_config),
174 config_class(**inst_config))
180 out[inst_config['name']] = creator
181 logger.info('Created configured %s', config_key)
182 except Exception as e:
183 logger.error('Unexpected error instantiating creator [%s] '
184 'with exception %s', creator_class, e)
189 def __create_vm_instances(os_creds_dict, os_users_dict, instances_config,
190 image_dict, keypairs_dict, cleanup=False):
192 Returns a dictionary of OpenStackVmInstance objects where the key is the
194 :param os_creds_dict: Dictionary of OSCreds objects where the key is the
196 :param os_users_dict: Dictionary of OpenStackUser objects where the key is
198 :param instances_config: The list of VM instance configurations
199 :param image_dict: A dictionary of images that will probably be used to
200 instantiate the VM instance
201 :param keypairs_dict: A dictionary of keypairs that will probably be used
202 to instantiate the VM instance
203 :param cleanup: Denotes whether or not this is being called for cleanup
210 for instance_config in instances_config:
211 conf = instance_config.get('instance')
214 image_creator = image_dict.get(conf.get('imageName'))
216 instance_settings = VmInstanceSettings(
217 **instance_config['instance'])
218 kp_creator = keypairs_dict.get(
219 conf.get('keypair_name'))
221 'name']] = deploy_utils.create_vm_instance(
223 os_creds_dict, os_users_dict, conf),
225 image_creator.image_settings,
226 keypair_creator=kp_creator,
229 raise Exception('Image creator instance not found.'
230 ' Cannot instantiate')
232 raise Exception('Image dictionary is None. Cannot '
235 raise Exception('Instance configuration is None. Cannot '
237 logger.info('Created configured instances')
238 except Exception as e:
239 logger.error('Unexpected error creating VM instances - %s', e)
243 def __apply_ansible_playbooks(ansible_configs, os_creds_dict, vm_dict,
244 image_dict, flavor_dict, env_file):
246 Applies ansible playbooks to running VMs with floating IPs
247 :param ansible_configs: a list of Ansible configurations
248 :param os_creds_dict: Dictionary of OSCreds objects where the key is the
250 :param vm_dict: the dictionary of newly instantiated VMs where the name is
252 :param image_dict: the dictionary of newly instantiated images where the
254 :param flavor_dict: the dictionary of newly instantiated flavors where the
256 :param env_file: the path of the environment for setting the CWD so
257 playbook location is relative to the deployment file
258 :return: t/f - true if successful
260 logger.info("Applying Ansible Playbooks")
262 # Ensure all hosts are accepting SSH session requests
263 for vm_inst in list(vm_dict.values()):
264 if not vm_inst.vm_ssh_active(block=True):
266 "Timeout waiting for instance to respond to SSH requests")
269 # Set CWD so the deployment file's playbook location can leverage
271 orig_cwd = os.getcwd()
272 env_dir = os.path.dirname(env_file)
276 for ansible_config in ansible_configs:
277 if 'pre_sleep_time' in ansible_config:
279 sleep_time = int(ansible_config['pre_sleep_time'])
280 logger.info('Waiting %s seconds to apply playbooks',
282 time.sleep(sleep_time)
286 os_creds = os_creds_dict.get(None, 'admin')
287 __apply_ansible_playbook(ansible_config, os_creds, vm_dict,
288 image_dict, flavor_dict)
290 # Return to original directory
296 def __apply_ansible_playbook(ansible_config, os_creds, vm_dict, image_dict,
299 Applies an Ansible configuration setting
300 :param ansible_config: the configuration settings
301 :param os_creds: the OpenStack credentials object
302 :param vm_dict: the dictionary of newly instantiated VMs where the name is
304 :param image_dict: the dictionary of newly instantiated images where the
306 :param flavor_dict: the dictionary of newly instantiated flavors where the
310 (remote_user, floating_ips, private_key_filepath,
311 proxy_settings) = __get_connection_info(
312 ansible_config, vm_dict)
314 retval = ansible_utils.apply_playbook(
315 ansible_config['playbook_location'], floating_ips, remote_user,
316 private_key_filepath,
317 variables=__get_variables(ansible_config.get('variables'),
318 os_creds, vm_dict, image_dict,
320 proxy_setting=proxy_settings)
322 # Not a fatal type of event
324 'Unable to apply playbook found at location - %s',
325 ansible_config.get('playbook_location'))
328 def __get_connection_info(ansible_config, vm_dict):
330 Returns a tuple of data required for connecting to the running VMs
331 (remote_user, [floating_ips], private_key_filepath, proxy_settings)
332 :param ansible_config: the configuration settings
333 :param vm_dict: the dictionary of VMs where the VM name is the key
334 :return: tuple where the first element is the user and the second is a list
335 of floating IPs and the third is the
336 private key file location and the fourth is an instance of the
337 snaps.ProxySettings class
338 (note: in order to work, each of the hosts need to have the same sudo_user
339 and private key file location values)
341 if ansible_config.get('hosts'):
342 hosts = ansible_config['hosts']
344 floating_ips = list()
347 proxy_settings = None
349 vm = vm_dict.get(host)
351 fip = vm.get_floating_ip()
353 remote_user = vm.get_image_user()
356 floating_ips.append(fip.ip)
359 'Could not find floating IP for VM - ' +
362 pk_file = vm.keypair_settings.private_filepath
363 proxy_settings = vm.get_os_creds().proxy_settings
365 logger.error('Could not locate VM with name - ' + host)
367 return remote_user, floating_ips, pk_file, proxy_settings
371 def __get_variables(var_config, os_creds, vm_dict, image_dict, flavor_dict):
373 Returns a dictionary of substitution variables to be used for Ansible
375 :param var_config: the variable configuration settings
376 :param os_creds: the OpenStack credentials object
377 :param vm_dict: the dictionary of newly instantiated VMs where the name is
379 :param image_dict: the dictionary of newly instantiated images where the
381 :param flavor_dict: the dictionary of newly instantiated flavors where the
383 :return: dictionary or None
385 if var_config and vm_dict and len(vm_dict) > 0:
387 for key, value in var_config.items():
388 value = __get_variable_value(value, os_creds, vm_dict, image_dict,
391 variables[key] = value
393 "Set Jinga2 variable with key [%s] the value [%s]",
396 logger.warning('Key [%s] or Value [%s] must not be None',
397 str(key), str(value))
402 def __get_variable_value(var_config_values, os_creds, vm_dict, image_dict,
405 Returns the associated variable value for use by Ansible for substitution
407 :param var_config_values: the configuration dictionary
408 :param os_creds: the OpenStack credentials object
409 :param vm_dict: the dictionary of newly instantiated VMs where the name is
411 :param image_dict: the dictionary of newly instantiated images where the
413 :param flavor_dict: the dictionary of newly instantiated flavors where the
417 if var_config_values['type'] == 'string':
418 return __get_string_variable_value(var_config_values)
419 if var_config_values['type'] == 'vm-attr':
420 return __get_vm_attr_variable_value(var_config_values, vm_dict)
421 if var_config_values['type'] == 'os_creds':
422 return __get_os_creds_variable_value(var_config_values, os_creds)
423 if var_config_values['type'] == 'port':
424 return __get_vm_port_variable_value(var_config_values, vm_dict)
425 if var_config_values['type'] == 'floating_ip':
426 return __get_vm_fip_variable_value(var_config_values, vm_dict)
427 if var_config_values['type'] == 'image':
428 return __get_image_variable_value(var_config_values, image_dict)
429 if var_config_values['type'] == 'flavor':
430 return __get_flavor_variable_value(var_config_values, flavor_dict)
434 def __get_string_variable_value(var_config_values):
436 Returns the associated string value
437 :param var_config_values: the configuration dictionary
438 :return: the value contained in the dictionary with the key 'value'
440 return var_config_values['value']
443 def __get_vm_attr_variable_value(var_config_values, vm_dict):
445 Returns the associated value contained on a VM instance
446 :param var_config_values: the configuration dictionary
447 :param vm_dict: the dictionary containing all VMs where the key is the VM's
451 vm = vm_dict.get(var_config_values['vm_name'])
453 if var_config_values['value'] == 'floating_ip':
454 return vm.get_floating_ip().ip
455 if var_config_values['value'] == 'image_user':
456 return vm.get_image_user()
459 def __get_os_creds_variable_value(var_config_values, os_creds):
461 Returns the associated OS credentials value
462 :param var_config_values: the configuration dictionary
463 :param os_creds: the credentials
466 logger.info("Retrieving OS Credentials")
468 if var_config_values['value'] == 'username':
469 logger.info("Returning OS username")
470 return os_creds.username
471 elif var_config_values['value'] == 'password':
472 logger.info("Returning OS password")
473 return os_creds.password
474 elif var_config_values['value'] == 'auth_url':
475 logger.info("Returning OS auth_url")
476 return os_creds.auth_url
477 elif var_config_values['value'] == 'project_name':
478 logger.info("Returning OS project_name")
479 return os_creds.project_name
481 logger.info("Returning none")
485 def __get_vm_port_variable_value(var_config_values, vm_dict):
487 Returns the associated OS credentials value
488 :param var_config_values: the configuration dictionary
489 :param vm_dict: the dictionary containing all VMs where the key is the VM's
493 port_name = var_config_values.get('port_name')
494 vm_name = var_config_values.get('vm_name')
496 if port_name and vm_name:
497 vm = vm_dict.get(vm_name)
499 port_value_id = var_config_values.get('port_value')
501 if port_value_id == 'mac_address':
502 return vm.get_port_mac(port_name)
503 if port_value_id == 'ip_address':
504 return vm.get_port_ip(port_name)
507 def __get_vm_fip_variable_value(var_config_values, vm_dict):
509 Returns the floating IP value if found
510 :param var_config_values: the configuration dictionary
511 :param vm_dict: the dictionary containing all VMs where the key is the VM's
513 :return: the floating IP string value or None
515 fip_name = var_config_values.get('fip_name')
516 vm_name = var_config_values.get('vm_name')
519 vm = vm_dict.get(vm_name)
521 fip = vm.get_floating_ip(fip_name)
526 def __get_image_variable_value(var_config_values, image_dict):
528 Returns the associated image value
529 :param var_config_values: the configuration dictionary
530 :param image_dict: the dictionary containing all images where the key is
534 logger.info("Retrieving image values")
537 if var_config_values.get('image_name'):
538 image_creator = image_dict.get(var_config_values['image_name'])
540 if var_config_values.get('value') and \
541 var_config_values['value'] == 'id':
542 return image_creator.get_image().id
543 if var_config_values.get('value') and \
544 var_config_values['value'] == 'user':
545 return image_creator.image_settings.image_user
547 logger.info("Returning none")
551 def __get_flavor_variable_value(var_config_values, flavor_dict):
553 Returns the associated flavor value
554 :param var_config_values: the configuration dictionary
555 :param flavor_dict: the dictionary containing all flavor creators where the
557 :return: the value or None
559 logger.info("Retrieving flavor values")
562 if var_config_values.get('flavor_name'):
563 flavor_creator = flavor_dict.get(var_config_values['flavor_name'])
565 if var_config_values.get('value') and \
566 var_config_values['value'] == 'id':
567 return flavor_creator.get_flavor().id
572 Will need to set environment variable ANSIBLE_HOST_KEY_CHECKING=False or
573 Create a file located in /etc/ansible/ansible/cfg or ~/.ansible.cfg
574 containing the following content:
577 host_key_checking = False
579 CWD must be this directory where this script is located.
583 log_level = logging.INFO
584 if arguments.log_level != 'INFO':
585 log_level = logging.DEBUG
586 logging.basicConfig(level=log_level)
588 logger.info('Starting to Deploy')
590 # Apply env_file/substitution file to template
591 env = Environment(loader=FileSystemLoader(
592 searchpath=os.path.dirname(arguments.tmplt_file)))
593 template = env.get_template(os.path.basename(arguments.tmplt_file))
596 if arguments.env_file:
597 env_dict = file_utils.read_yaml(arguments.env_file)
598 output = template.render(**env_dict)
600 config = yaml.load(output)
603 os_config = config.get('openstack')
608 flavors_dict = dict()
609 os_creds_dict = dict()
610 clean = arguments.clean is not ARG_NOT_SET
613 os_creds_dict = __get_creds_dict(os_config)
617 projects_dict = __create_instances(
618 os_creds_dict, OpenStackProject, ProjectSettings,
619 os_config.get('projects'), 'project', clean)
620 creators.append(projects_dict)
623 users_dict = __create_instances(
624 os_creds_dict, OpenStackUser, UserSettings,
625 os_config.get('users'), 'user', clean)
626 creators.append(users_dict)
628 # Associate new users to projects
630 for project_creator in projects_dict.values():
631 users = project_creator.project_settings.users
632 for user_name in users:
633 user_creator = users_dict.get(user_name)
635 project_creator.assoc_user(
636 user_creator.get_user())
639 flavors_dict = __create_instances(
640 os_creds_dict, OpenStackFlavor, FlavorConfig,
641 os_config.get('flavors'), 'flavor', clean, users_dict)
642 creators.append(flavors_dict)
645 qos_dict = __create_instances(
646 os_creds_dict, OpenStackQoS, QoSSettings,
647 os_config.get('qos_specs'), 'qos_spec', clean, users_dict)
648 creators.append(qos_dict)
650 # Create volume types
651 vol_type_dict = __create_instances(
652 os_creds_dict, OpenStackVolumeType, VolumeTypeSettings,
653 os_config.get('volume_types'), 'volume_type', clean,
655 creators.append(vol_type_dict)
657 # Create volume types
658 vol_dict = __create_instances(
659 os_creds_dict, OpenStackVolume, VolumeSettings,
660 os_config.get('volumes'), 'volume', clean, users_dict)
661 creators.append(vol_dict)
664 images_dict = __create_instances(
665 os_creds_dict, OpenStackImage, ImageConfig,
666 os_config.get('images'), 'image', clean, users_dict)
667 creators.append(images_dict)
670 creators.append(__create_instances(
671 os_creds_dict, OpenStackNetwork, NetworkSettings,
672 os_config.get('networks'), 'network', clean, users_dict))
675 creators.append(__create_instances(
676 os_creds_dict, OpenStackRouter, RouterSettings,
677 os_config.get('routers'), 'router', clean, users_dict))
680 keypairs_dict = __create_instances(
681 os_creds_dict, OpenStackKeypair, KeypairConfig,
682 os_config.get('keypairs'), 'keypair', clean, users_dict)
683 creators.append(keypairs_dict)
685 # Create security groups
686 creators.append(__create_instances(
687 os_creds_dict, OpenStackSecurityGroup,
688 SecurityGroupSettings,
689 os_config.get('security_groups'), 'security_group', clean,
693 vm_dict = __create_vm_instances(
694 os_creds_dict, users_dict, os_config.get('instances'),
695 images_dict, keypairs_dict,
696 arguments.clean is not ARG_NOT_SET)
697 creators.append(vm_dict)
699 'Completed creating/retrieving all configured instances')
700 except Exception as e:
702 'Unexpected error deploying environment. Rolling back due'
706 # Must enter either block
707 if arguments.clean is not ARG_NOT_SET:
708 # Cleanup Environment
709 __cleanup(creators, arguments.clean_image is not ARG_NOT_SET)
710 elif arguments.deploy is not ARG_NOT_SET:
711 logger.info('Configuring NICs where required')
712 for vm in vm_dict.values():
714 logger.info('Completed NIC configuration')
717 ansible_config = config.get('ansible')
718 if ansible_config and vm_dict:
719 if not __apply_ansible_playbooks(ansible_config,
720 os_creds_dict, vm_dict,
721 images_dict, flavors_dict,
722 arguments.tmplt_file):
723 logger.error("Problem applying ansible playbooks")
726 'Unable to read configuration file - ' + arguments.tmplt_file)
732 def __cleanup(creators, clean_image=False):
733 for creator_dict in reversed(creators):
734 for key, creator in creator_dict.items():
735 if ((isinstance(creator, OpenStackImage) and clean_image)
736 or not isinstance(creator, OpenStackImage)):
739 except Exception as e:
740 logger.warning('Error cleaning component - %s', e)
743 if __name__ == '__main__':
744 # To ensure any files referenced via a relative path will begin from the
745 # directory in which this file resides
746 os.chdir(os.path.dirname(os.path.realpath(__file__)))
748 parser = argparse.ArgumentParser()
750 '-d', '--deploy', dest='deploy', nargs='?', default=ARG_NOT_SET,
751 help='When used, environment will be deployed and provisioned')
753 '-c', '--clean', dest='clean', nargs='?', default=ARG_NOT_SET,
754 help='When used, the environment will be removed')
756 '-i', '--clean-image', dest='clean_image', nargs='?',
758 help='When cleaning, if this is set, the image will be cleaned too')
760 '-t', '--tmplt', dest='tmplt_file', required=True,
761 help='The SNAPS deployment template YAML file - REQUIRED')
763 '-e', '--env-file', dest='env_file',
764 help='Yaml file containing substitution values to the env file')
766 '-l', '--log-level', dest='log_level', default='INFO',
767 help='Logging Level (INFO|DEBUG)')
768 args = parser.parse_args()
770 if args.deploy is ARG_NOT_SET and args.clean is ARG_NOT_SET:
772 'Must enter either -d for deploy or -c for cleaning up and '
775 if args.deploy is not ARG_NOT_SET and args.clean is not ARG_NOT_SET:
776 print('Cannot enter both options -d/--deploy and -c/--clean')