3 # Copyright (c) 2017 Cable Television Laboratories, Inc. ("CableLabs")
4 # and others. All rights reserved.
6 # Licensed under the Apache License, Version 2.0 (the "License");
7 # you may not use this file except in compliance with the License.
8 # You may obtain a copy of the License at:
10 # http://www.apache.org/licenses/LICENSE-2.0
12 # Unless required by applicable law or agreed to in writing, software
13 # distributed under the License is distributed on an "AS IS" BASIS,
14 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 # See the License for the specific language governing permissions and
16 # limitations under the License.
18 # This script is responsible for deploying virtual environments
24 from jinja2 import Environment, FileSystemLoader
28 from snaps import file_utils
29 from snaps.config.flavor import FlavorConfig
30 from snaps.config.image import ImageConfig
31 from snaps.config.keypair import KeypairConfig
32 from snaps.config.network import PortConfig, NetworkConfig
33 from snaps.config.project import ProjectConfig
34 from snaps.config.qos import QoSConfig
35 from snaps.config.router import RouterConfig
36 from snaps.config.security_group import SecurityGroupConfig
37 from snaps.config.user import UserConfig
38 from snaps.config.vm_inst import VmInstanceConfig
39 from snaps.config.volume import VolumeConfig
40 from snaps.config.volume_type import VolumeTypeConfig
41 from snaps.openstack.create_flavor import OpenStackFlavor
42 from snaps.openstack.create_image import OpenStackImage
43 from snaps.openstack.create_keypairs import OpenStackKeypair
44 from snaps.openstack.create_network import OpenStackNetwork
45 from snaps.openstack.create_project import OpenStackProject
46 from snaps.openstack.create_qos import OpenStackQoS
47 from snaps.openstack.create_router import OpenStackRouter
48 from snaps.openstack.create_security_group import OpenStackSecurityGroup
49 from snaps.openstack.create_user import OpenStackUser
50 from snaps.openstack.create_volume import OpenStackVolume
51 from snaps.openstack.create_volume_type import OpenStackVolumeType
52 from snaps.openstack.os_credentials import OSCreds, ProxySettings
53 from snaps.openstack.utils import deploy_utils
54 from snaps.provisioning import ansible_utils
56 __author__ = 'spisarski'
58 logger = logging.getLogger('snaps_launcher')
60 ARG_NOT_SET = "argument not set"
61 DEFAULT_CREDS_KEY = 'admin'
64 def __get_creds_dict(os_conn_config):
66 Returns a dict of OSCreds where the key is the creds name.
67 For backwards compatibility, credentials not contained in a list (only
68 one) will be returned with the key of None
69 :param os_conn_config: the credential configuration
70 :return: a dict of OSCreds objects
72 if 'connection' in os_conn_config:
73 return {DEFAULT_CREDS_KEY: __get_os_credentials(os_conn_config)}
74 elif 'connections' in os_conn_config:
76 for os_conn_dict in os_conn_config['connections']:
77 config = os_conn_dict.get('connection')
79 raise Exception('Invalid connection format')
81 name = config.get('name')
83 raise Exception('Connection config requires a name field')
85 out[name] = __get_os_credentials(os_conn_dict)
89 def __get_creds(os_creds_dict, os_user_dict, inst_config):
91 Returns the appropriate credentials
92 :param os_creds_dict: a dictionary of OSCreds objects where the name is the
94 :param os_user_dict: a dictionary of OpenStackUser objects where the name
97 :return: an OSCreds instance or None
99 os_creds = os_creds_dict.get(DEFAULT_CREDS_KEY)
100 if 'os_user' in inst_config:
101 os_user_conf = inst_config['os_user']
102 if 'name' in os_user_conf:
103 user_creator = os_user_dict.get(os_user_conf['name'])
105 return user_creator.get_os_creds(
106 project_name=os_user_conf.get('project_name'))
107 elif 'os_creds_name' in inst_config:
108 if 'os_creds_name' in inst_config:
109 os_creds = os_creds_dict[inst_config['os_creds_name']]
113 def __get_os_credentials(os_conn_config):
115 Returns an object containing all of the information required to access
117 :param os_conn_config: The configuration holding the credentials
118 :return: an OSCreds instance
120 config = os_conn_config.get('connection')
122 raise Exception('Invalid connection configuration')
124 proxy_settings = None
125 http_proxy = config.get('http_proxy')
127 tokens = re.split(':', http_proxy)
128 ssh_proxy_cmd = config.get('ssh_proxy_cmd')
129 proxy_settings = ProxySettings(host=tokens[0], port=tokens[1],
130 ssh_proxy_cmd=ssh_proxy_cmd)
132 if 'proxy_settings' in config:
133 host = config['proxy_settings'].get('host')
134 port = config['proxy_settings'].get('port')
135 if host and host != 'None' and port and port != 'None':
136 proxy_settings = ProxySettings(**config['proxy_settings'])
139 config['proxy_settings'] = proxy_settings
141 if config.get('proxy_settings'):
142 del config['proxy_settings']
144 return OSCreds(**config)
147 def __parse_ports_config(config):
149 Parses the "ports" configuration
150 :param config: The dictionary to parse
151 :return: a list of PortConfig objects
154 for port_config in config:
155 out.append(PortConfig(**port_config.get('port')))
159 def __create_instances(os_creds_dict, creator_class, config_class, config,
160 config_key, cleanup=False, os_users_dict=None):
162 Returns a dictionary of SNAPS creator objects where the key is the name
163 :param os_creds_dict: Dictionary of OSCreds objects where the key is the
165 :param config: The list of configurations for the same type
166 :param config_key: The list of configurations for the same type
167 :param cleanup: Denotes whether or not this is being called for cleanup
174 for config_dict in config:
175 inst_config = config_dict.get(config_key)
177 creator = creator_class(
178 __get_creds(os_creds_dict, os_users_dict, inst_config),
179 config_class(**inst_config))
185 out[inst_config['name']] = creator
186 logger.info('Created configured %s', config_key)
187 except Exception as e:
188 logger.error('Unexpected error instantiating creator [%s] '
189 'with exception %s', creator_class, e)
194 def __create_vm_instances(os_creds_dict, os_users_dict, instances_config,
195 image_dict, keypairs_dict, cleanup=False):
197 Returns a dictionary of OpenStackVmInstance objects where the key is the
199 :param os_creds_dict: Dictionary of OSCreds objects where the key is the
201 :param os_users_dict: Dictionary of OpenStackUser objects where the key is
203 :param instances_config: The list of VM instance configurations
204 :param image_dict: A dictionary of images that will probably be used to
205 instantiate the VM instance
206 :param keypairs_dict: A dictionary of keypairs that will probably be used
207 to instantiate the VM instance
208 :param cleanup: Denotes whether or not this is being called for cleanup
215 for instance_config in instances_config:
216 conf = instance_config.get('instance')
219 image_creator = image_dict.get(conf.get('imageName'))
221 instance_settings = VmInstanceConfig(
222 **instance_config['instance'])
223 kp_creator = keypairs_dict.get(
224 conf.get('keypair_name'))
226 'name']] = deploy_utils.create_vm_instance(
228 os_creds_dict, os_users_dict, conf),
230 image_creator.image_settings,
231 keypair_creator=kp_creator,
234 raise Exception('Image creator instance not found.'
235 ' Cannot instantiate')
237 raise Exception('Image dictionary is None. Cannot '
240 raise Exception('Instance configuration is None. Cannot '
242 logger.info('Created configured instances')
243 except Exception as e:
244 logger.error('Unexpected error creating VM instances - %s', e)
248 def __apply_ansible_playbooks(ansible_configs, os_creds_dict, vm_dict,
249 image_dict, flavor_dict, env_file):
251 Applies ansible playbooks to running VMs with floating IPs
252 :param ansible_configs: a list of Ansible configurations
253 :param os_creds_dict: Dictionary of OSCreds objects where the key is the
255 :param vm_dict: the dictionary of newly instantiated VMs where the name is
257 :param image_dict: the dictionary of newly instantiated images where the
259 :param flavor_dict: the dictionary of newly instantiated flavors where the
261 :param env_file: the path of the environment for setting the CWD so
262 playbook location is relative to the deployment file
263 :return: t/f - true if successful
265 logger.info("Applying Ansible Playbooks")
267 # Ensure all hosts are accepting SSH session requests
268 for vm_inst in list(vm_dict.values()):
269 if not vm_inst.vm_ssh_active(block=True):
271 "Timeout waiting for instance to respond to SSH requests")
274 # Set CWD so the deployment file's playbook location can leverage
276 orig_cwd = os.getcwd()
277 env_dir = os.path.dirname(env_file)
281 for ansible_config in ansible_configs:
282 if 'pre_sleep_time' in ansible_config:
284 sleep_time = int(ansible_config['pre_sleep_time'])
285 logger.info('Waiting %s seconds to apply playbooks',
287 time.sleep(sleep_time)
291 os_creds = os_creds_dict.get(None, 'admin')
292 __apply_ansible_playbook(ansible_config, os_creds, vm_dict,
293 image_dict, flavor_dict)
295 # Return to original directory
301 def __apply_ansible_playbook(ansible_config, os_creds, vm_dict, image_dict,
304 Applies an Ansible configuration setting
305 :param ansible_config: the configuration settings
306 :param os_creds: the OpenStack credentials object
307 :param vm_dict: the dictionary of newly instantiated VMs where the name is
309 :param image_dict: the dictionary of newly instantiated images where the
311 :param flavor_dict: the dictionary of newly instantiated flavors where the
315 (remote_user, floating_ips, private_key_filepath,
316 proxy_settings) = __get_connection_info(
317 ansible_config, vm_dict)
319 retval = ansible_utils.apply_playbook(
320 ansible_config['playbook_location'], floating_ips, remote_user,
321 private_key_filepath,
322 variables=__get_variables(ansible_config.get('variables'),
323 os_creds, vm_dict, image_dict,
325 proxy_setting=proxy_settings)
327 # Not a fatal type of event
329 'Unable to apply playbook found at location - %s',
330 ansible_config.get('playbook_location'))
333 def __get_connection_info(ansible_config, vm_dict):
335 Returns a tuple of data required for connecting to the running VMs
336 (remote_user, [floating_ips], private_key_filepath, proxy_settings)
337 :param ansible_config: the configuration settings
338 :param vm_dict: the dictionary of VMs where the VM name is the key
339 :return: tuple where the first element is the user and the second is a list
340 of floating IPs and the third is the
341 private key file location and the fourth is an instance of the
342 snaps.ProxySettings class
343 (note: in order to work, each of the hosts need to have the same sudo_user
344 and private key file location values)
346 if ansible_config.get('hosts'):
347 hosts = ansible_config['hosts']
349 floating_ips = list()
352 proxy_settings = None
354 vm = vm_dict.get(host)
356 fip = vm.get_floating_ip()
358 remote_user = vm.get_image_user()
361 floating_ips.append(fip.ip)
364 'Could not find floating IP for VM - ' +
367 pk_file = vm.keypair_settings.private_filepath
368 proxy_settings = vm.get_os_creds().proxy_settings
370 logger.error('Could not locate VM with name - ' + host)
372 return remote_user, floating_ips, pk_file, proxy_settings
376 def __get_variables(var_config, os_creds, vm_dict, image_dict, flavor_dict):
378 Returns a dictionary of substitution variables to be used for Ansible
380 :param var_config: the variable configuration settings
381 :param os_creds: the OpenStack credentials object
382 :param vm_dict: the dictionary of newly instantiated VMs where the name is
384 :param image_dict: the dictionary of newly instantiated images where the
386 :param flavor_dict: the dictionary of newly instantiated flavors where the
388 :return: dictionary or None
390 if var_config and vm_dict and len(vm_dict) > 0:
392 for key, value in var_config.items():
393 value = __get_variable_value(value, os_creds, vm_dict, image_dict,
396 variables[key] = value
398 "Set Jinga2 variable with key [%s] the value [%s]",
401 logger.warning('Key [%s] or Value [%s] must not be None',
402 str(key), str(value))
407 def __get_variable_value(var_config_values, os_creds, vm_dict, image_dict,
410 Returns the associated variable value for use by Ansible for substitution
412 :param var_config_values: the configuration dictionary
413 :param os_creds: the OpenStack credentials object
414 :param vm_dict: the dictionary of newly instantiated VMs where the name is
416 :param image_dict: the dictionary of newly instantiated images where the
418 :param flavor_dict: the dictionary of newly instantiated flavors where the
422 if var_config_values['type'] == 'string':
423 return __get_string_variable_value(var_config_values)
424 if var_config_values['type'] == 'vm-attr':
425 return __get_vm_attr_variable_value(var_config_values, vm_dict)
426 if var_config_values['type'] == 'os_creds':
427 return __get_os_creds_variable_value(var_config_values, os_creds)
428 if var_config_values['type'] == 'port':
429 return __get_vm_port_variable_value(var_config_values, vm_dict)
430 if var_config_values['type'] == 'floating_ip':
431 return __get_vm_fip_variable_value(var_config_values, vm_dict)
432 if var_config_values['type'] == 'image':
433 return __get_image_variable_value(var_config_values, image_dict)
434 if var_config_values['type'] == 'flavor':
435 return __get_flavor_variable_value(var_config_values, flavor_dict)
439 def __get_string_variable_value(var_config_values):
441 Returns the associated string value
442 :param var_config_values: the configuration dictionary
443 :return: the value contained in the dictionary with the key 'value'
445 return var_config_values['value']
448 def __get_vm_attr_variable_value(var_config_values, vm_dict):
450 Returns the associated value contained on a VM instance
451 :param var_config_values: the configuration dictionary
452 :param vm_dict: the dictionary containing all VMs where the key is the VM's
456 vm = vm_dict.get(var_config_values['vm_name'])
458 if var_config_values['value'] == 'floating_ip':
459 return vm.get_floating_ip().ip
460 if var_config_values['value'] == 'image_user':
461 return vm.get_image_user()
464 def __get_os_creds_variable_value(var_config_values, os_creds):
466 Returns the associated OS credentials value
467 :param var_config_values: the configuration dictionary
468 :param os_creds: the credentials
471 logger.info("Retrieving OS Credentials")
473 if var_config_values['value'] == 'username':
474 logger.info("Returning OS username")
475 return os_creds.username
476 elif var_config_values['value'] == 'password':
477 logger.info("Returning OS password")
478 return os_creds.password
479 elif var_config_values['value'] == 'auth_url':
480 logger.info("Returning OS auth_url")
481 return os_creds.auth_url
482 elif var_config_values['value'] == 'project_name':
483 logger.info("Returning OS project_name")
484 return os_creds.project_name
486 logger.info("Returning none")
490 def __get_vm_port_variable_value(var_config_values, vm_dict):
492 Returns the associated OS credentials value
493 :param var_config_values: the configuration dictionary
494 :param vm_dict: the dictionary containing all VMs where the key is the VM's
498 port_name = var_config_values.get('port_name')
499 vm_name = var_config_values.get('vm_name')
501 if port_name and vm_name:
502 vm = vm_dict.get(vm_name)
504 port_value_id = var_config_values.get('port_value')
506 if port_value_id == 'mac_address':
507 return vm.get_port_mac(port_name)
508 if port_value_id == 'ip_address':
509 return vm.get_port_ip(port_name)
512 def __get_vm_fip_variable_value(var_config_values, vm_dict):
514 Returns the floating IP value if found
515 :param var_config_values: the configuration dictionary
516 :param vm_dict: the dictionary containing all VMs where the key is the VM's
518 :return: the floating IP string value or None
520 fip_name = var_config_values.get('fip_name')
521 vm_name = var_config_values.get('vm_name')
524 vm = vm_dict.get(vm_name)
526 fip = vm.get_floating_ip(fip_name)
531 def __get_image_variable_value(var_config_values, image_dict):
533 Returns the associated image value
534 :param var_config_values: the configuration dictionary
535 :param image_dict: the dictionary containing all images where the key is
539 logger.info("Retrieving image values")
542 if var_config_values.get('image_name'):
543 image_creator = image_dict.get(var_config_values['image_name'])
545 if var_config_values.get('value') and \
546 var_config_values['value'] == 'id':
547 return image_creator.get_image().id
548 if var_config_values.get('value') and \
549 var_config_values['value'] == 'user':
550 return image_creator.image_settings.image_user
552 logger.info("Returning none")
556 def __get_flavor_variable_value(var_config_values, flavor_dict):
558 Returns the associated flavor value
559 :param var_config_values: the configuration dictionary
560 :param flavor_dict: the dictionary containing all flavor creators where the
562 :return: the value or None
564 logger.info("Retrieving flavor values")
567 if var_config_values.get('flavor_name'):
568 flavor_creator = flavor_dict.get(var_config_values['flavor_name'])
570 if var_config_values.get('value') and \
571 var_config_values['value'] == 'id':
572 return flavor_creator.get_flavor().id
577 Will need to set environment variable ANSIBLE_HOST_KEY_CHECKING=False or
578 Create a file located in /etc/ansible/ansible/cfg or ~/.ansible.cfg
579 containing the following content:
582 host_key_checking = False
584 CWD must be this directory where this script is located.
588 log_level = logging.INFO
589 if arguments.log_level != 'INFO':
590 log_level = logging.DEBUG
591 logging.basicConfig(level=log_level)
593 logger.info('Starting to Deploy')
595 # Apply env_file/substitution file to template
596 env = Environment(loader=FileSystemLoader(
597 searchpath=os.path.dirname(arguments.tmplt_file)))
598 template = env.get_template(os.path.basename(arguments.tmplt_file))
601 if arguments.env_file:
602 env_dict = file_utils.read_yaml(arguments.env_file)
603 output = template.render(**env_dict)
605 config = yaml.load(output)
608 os_config = config.get('openstack')
613 flavors_dict = dict()
614 os_creds_dict = dict()
615 clean = arguments.clean is not ARG_NOT_SET
618 os_creds_dict = __get_creds_dict(os_config)
622 projects_dict = __create_instances(
623 os_creds_dict, OpenStackProject, ProjectConfig,
624 os_config.get('projects'), 'project', clean)
625 creators.append(projects_dict)
628 users_dict = __create_instances(
629 os_creds_dict, OpenStackUser, UserConfig,
630 os_config.get('users'), 'user', clean)
631 creators.append(users_dict)
633 # Associate new users to projects
635 for project_creator in projects_dict.values():
636 users = project_creator.project_settings.users
637 for user_name in users:
638 user_creator = users_dict.get(user_name)
640 project_creator.assoc_user(
641 user_creator.get_user())
644 flavors_dict = __create_instances(
645 os_creds_dict, OpenStackFlavor, FlavorConfig,
646 os_config.get('flavors'), 'flavor', clean, users_dict)
647 creators.append(flavors_dict)
650 qos_dict = __create_instances(
651 os_creds_dict, OpenStackQoS, QoSConfig,
652 os_config.get('qos_specs'), 'qos_spec', clean, users_dict)
653 creators.append(qos_dict)
655 # Create volume types
656 vol_type_dict = __create_instances(
657 os_creds_dict, OpenStackVolumeType, VolumeTypeConfig,
658 os_config.get('volume_types'), 'volume_type', clean,
660 creators.append(vol_type_dict)
662 # Create volume types
663 vol_dict = __create_instances(
664 os_creds_dict, OpenStackVolume, VolumeConfig,
665 os_config.get('volumes'), 'volume', clean, users_dict)
666 creators.append(vol_dict)
669 images_dict = __create_instances(
670 os_creds_dict, OpenStackImage, ImageConfig,
671 os_config.get('images'), 'image', clean, users_dict)
672 creators.append(images_dict)
675 creators.append(__create_instances(
676 os_creds_dict, OpenStackNetwork, NetworkConfig,
677 os_config.get('networks'), 'network', clean, users_dict))
680 creators.append(__create_instances(
681 os_creds_dict, OpenStackRouter, RouterConfig,
682 os_config.get('routers'), 'router', clean, users_dict))
685 keypairs_dict = __create_instances(
686 os_creds_dict, OpenStackKeypair, KeypairConfig,
687 os_config.get('keypairs'), 'keypair', clean, users_dict)
688 creators.append(keypairs_dict)
690 # Create security groups
691 creators.append(__create_instances(
692 os_creds_dict, OpenStackSecurityGroup,
694 os_config.get('security_groups'), 'security_group', clean,
698 vm_dict = __create_vm_instances(
699 os_creds_dict, users_dict, os_config.get('instances'),
700 images_dict, keypairs_dict,
701 arguments.clean is not ARG_NOT_SET)
702 creators.append(vm_dict)
704 'Completed creating/retrieving all configured instances')
705 except Exception as e:
707 'Unexpected error deploying environment. Rolling back due'
711 # Must enter either block
712 if arguments.clean is not ARG_NOT_SET:
713 # Cleanup Environment
714 __cleanup(creators, arguments.clean_image is not ARG_NOT_SET)
715 elif arguments.deploy is not ARG_NOT_SET:
716 logger.info('Configuring NICs where required')
717 for vm in vm_dict.values():
719 logger.info('Completed NIC configuration')
722 ansible_config = config.get('ansible')
723 if ansible_config and vm_dict:
724 if not __apply_ansible_playbooks(ansible_config,
725 os_creds_dict, vm_dict,
726 images_dict, flavors_dict,
727 arguments.tmplt_file):
728 logger.error("Problem applying ansible playbooks")
731 'Unable to read configuration file - ' + arguments.tmplt_file)
737 def __cleanup(creators, clean_image=False):
738 for creator_dict in reversed(creators):
739 for key, creator in creator_dict.items():
740 if ((isinstance(creator, OpenStackImage) and clean_image)
741 or not isinstance(creator, OpenStackImage)):
744 except Exception as e:
745 logger.warning('Error cleaning component - %s', e)
748 if __name__ == '__main__':
749 # To ensure any files referenced via a relative path will begin from the
750 # directory in which this file resides
751 os.chdir(os.path.dirname(os.path.realpath(__file__)))
753 parser = argparse.ArgumentParser()
755 '-d', '--deploy', dest='deploy', nargs='?', default=ARG_NOT_SET,
756 help='When used, environment will be deployed and provisioned')
758 '-c', '--clean', dest='clean', nargs='?', default=ARG_NOT_SET,
759 help='When used, the environment will be removed')
761 '-i', '--clean-image', dest='clean_image', nargs='?',
763 help='When cleaning, if this is set, the image will be cleaned too')
765 '-t', '--tmplt', dest='tmplt_file', required=True,
766 help='The SNAPS deployment template YAML file - REQUIRED')
768 '-e', '--env-file', dest='env_file',
769 help='Yaml file containing substitution values to the env file')
771 '-l', '--log-level', dest='log_level', default='INFO',
772 help='Logging Level (INFO|DEBUG)')
773 args = parser.parse_args()
775 if args.deploy is ARG_NOT_SET and args.clean is ARG_NOT_SET:
777 'Must enter either -d for deploy or -c for cleaning up and '
780 if args.deploy is not ARG_NOT_SET and args.clean is not ARG_NOT_SET:
781 print('Cannot enter both options -d/--deploy and -c/--clean')