3 # Copyright (c) 2017 Cable Television Laboratories, Inc. ("CableLabs")
4 # and others. All rights reserved.
6 # Licensed under the Apache License, Version 2.0 (the "License");
7 # you may not use this file except in compliance with the License.
8 # You may obtain a copy of the License at:
10 # http://www.apache.org/licenses/LICENSE-2.0
12 # Unless required by applicable law or agreed to in writing, software
13 # distributed under the License is distributed on an "AS IS" BASIS,
14 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 # See the License for the specific language governing permissions and
16 # limitations under the License.
18 # This script is responsible for deploying virtual environments
24 from snaps import file_utils
25 from snaps.openstack.create_flavor import FlavorSettings, OpenStackFlavor
26 from snaps.openstack.create_image import ImageSettings, OpenStackImage
27 from snaps.openstack.create_instance import VmInstanceSettings
28 from snaps.openstack.create_keypairs import KeypairSettings
29 from snaps.openstack.create_network import PortSettings, NetworkSettings
30 from snaps.openstack.create_router import RouterSettings
31 from snaps.openstack.os_credentials import OSCreds, ProxySettings
32 from snaps.openstack.utils import deploy_utils
33 from snaps.provisioning import ansible_utils
35 __author__ = 'spisarski'
37 logger = logging.getLogger('deploy_venv')
39 ARG_NOT_SET = "argument not set"
42 def __get_os_credentials(os_conn_config):
44 Returns an object containing all of the information required to access
46 :param os_conn_config: The configuration holding the credentials
47 :return: an OSCreds instance
50 http_proxy = os_conn_config.get('http_proxy')
52 tokens = re.split(':', http_proxy)
53 ssh_proxy_cmd = os_conn_config.get('ssh_proxy_cmd')
54 proxy_settings = ProxySettings(host=tokens[0], port=tokens[1],
55 ssh_proxy_cmd=ssh_proxy_cmd)
57 os_conn_config['proxy_settings'] = proxy_settings
59 return OSCreds(**os_conn_config)
62 def __parse_ports_config(config):
64 Parses the "ports" configuration
65 :param config: The dictionary to parse
66 :return: a list of PortConfig objects
69 for port_config in config:
70 out.append(PortSettings(**port_config.get('port')))
74 def __create_flavors(os_conn_config, flavors_config, cleanup=False):
76 Returns a dictionary of flavors where the key is the image name and the
77 value is the image object
78 :param os_conn_config: The OpenStack connection credentials
79 :param flavors_config: The list of image configurations
80 :param cleanup: Denotes whether or not this is being called for cleanup
87 for flavor_config_dict in flavors_config:
88 flavor_config = flavor_config_dict.get('flavor')
89 if flavor_config and flavor_config.get('name'):
90 flavor_creator = OpenStackFlavor(
91 __get_os_credentials(os_conn_config),
92 FlavorSettings(**flavor_config))
93 flavor_creator.create(cleanup=cleanup)
94 flavors[flavor_config['name']] = flavor_creator
95 except Exception as e:
96 for key, flavor_creator in flavors.items():
97 flavor_creator.clean()
99 logger.info('Created configured flavors')
104 def __create_images(os_conn_config, images_config, cleanup=False):
106 Returns a dictionary of images where the key is the image name and the
107 value is the image object
108 :param os_conn_config: The OpenStack connection credentials
109 :param images_config: The list of image configurations
110 :param cleanup: Denotes whether or not this is being called for cleanup
117 for image_config_dict in images_config:
118 image_config = image_config_dict.get('image')
119 if image_config and image_config.get('name'):
120 images[image_config['name']] = deploy_utils.create_image(
121 __get_os_credentials(os_conn_config),
122 ImageSettings(**image_config), cleanup)
123 except Exception as e:
124 for key, image_creator in images.items():
125 image_creator.clean()
127 logger.info('Created configured images')
132 def __create_networks(os_conn_config, network_confs, cleanup=False):
134 Returns a dictionary of networks where the key is the network name and the
135 value is the network object
136 :param os_conn_config: The OpenStack connection credentials
137 :param network_confs: The list of network configurations
138 :param cleanup: Denotes whether or not this is being called for cleanup
145 for network_conf in network_confs:
146 net_name = network_conf['network']['name']
147 os_creds = __get_os_credentials(os_conn_config)
148 network_dict[net_name] = deploy_utils.create_network(
149 os_creds, NetworkSettings(**network_conf['network']),
151 except Exception as e:
152 for key, net_creator in network_dict.items():
156 logger.info('Created configured networks')
161 def __create_routers(os_conn_config, router_confs, cleanup=False):
163 Returns a dictionary of networks where the key is the network name and the
164 value is the network object
165 :param os_conn_config: The OpenStack connection credentials
166 :param router_confs: The list of router configurations
167 :param cleanup: Denotes whether or not this is being called for cleanup
171 os_creds = __get_os_credentials(os_conn_config)
175 for router_conf in router_confs:
176 router_name = router_conf['router']['name']
177 router_dict[router_name] = deploy_utils.create_router(
178 os_creds, RouterSettings(**router_conf['router']), cleanup)
179 except Exception as e:
180 for key, router_creator in router_dict.items():
181 router_creator.clean()
184 logger.info('Created configured networks')
189 def __create_keypairs(os_conn_config, keypair_confs, cleanup=False):
191 Returns a dictionary of keypairs where the key is the keypair name and the
192 value is the keypair object
193 :param os_conn_config: The OpenStack connection credentials
194 :param keypair_confs: The list of keypair configurations
195 :param cleanup: Denotes whether or not this is being called for cleanup
201 for keypair_dict in keypair_confs:
202 keypair_config = keypair_dict['keypair']
203 kp_settings = KeypairSettings(**keypair_config)
205 keypair_config['name']] = deploy_utils.create_keypair(
206 __get_os_credentials(os_conn_config), kp_settings, cleanup)
207 except Exception as e:
208 for key, keypair_creator in keypairs_dict.items():
209 keypair_creator.clean()
212 logger.info('Created configured keypairs')
217 def __create_instances(os_conn_config, instances_config, image_dict,
218 keypairs_dict, cleanup=False):
220 Returns a dictionary of instances where the key is the instance name and
221 the value is the VM object
222 :param os_conn_config: The OpenStack connection credentials
223 :param instances_config: The list of VM instance configurations
224 :param image_dict: A dictionary of images that will probably be used to
225 instantiate the VM instance
226 :param keypairs_dict: A dictionary of keypairs that will probably be used
227 to instantiate the VM instance
228 :param cleanup: Denotes whether or not this is being called for cleanup
231 os_creds = __get_os_credentials(os_conn_config)
237 for instance_config in instances_config:
238 conf = instance_config.get('instance')
241 image_creator = image_dict.get(conf.get('imageName'))
243 instance_settings = VmInstanceSettings(
244 **instance_config['instance'])
245 kp_name = conf.get('keypair_name')
247 'name']] = deploy_utils.create_vm_instance(
248 os_creds, instance_settings,
249 image_creator.image_settings,
250 keypair_creator=keypairs_dict[kp_name],
253 raise Exception('Image creator instance not found.'
254 ' Cannot instantiate')
256 raise Exception('Image dictionary is None. Cannot '
259 raise Exception('Instance configuration is None. Cannot '
261 except Exception as e:
262 logger.error('Unexpected error creating instances. Attempting to '
263 'cleanup environment - %s', e)
264 for key, inst_creator in vm_dict.items():
268 logger.info('Created configured instances')
269 # TODO Should there be an error if there isn't an instances config
273 def __apply_ansible_playbooks(ansible_configs, os_conn_config, vm_dict,
274 image_dict, flavor_dict, env_file):
276 Applies ansible playbooks to running VMs with floating IPs
277 :param ansible_configs: a list of Ansible configurations
278 :param os_conn_config: the OpenStack connection configuration used to
279 create an OSCreds instance
280 :param vm_dict: the dictionary of newly instantiated VMs where the name is
282 :param image_dict: the dictionary of newly instantiated images where the
284 :param flavor_dict: the dictionary of newly instantiated flavors where the
286 :param env_file: the path of the environment for setting the CWD so
287 playbook location is relative to the deployment file
288 :return: t/f - true if successful
290 logger.info("Applying Ansible Playbooks")
292 # Ensure all hosts are accepting SSH session requests
293 for vm_inst in list(vm_dict.values()):
294 if not vm_inst.vm_ssh_active(block=True):
296 "Timeout waiting for instance to respond to SSH requests")
299 # Set CWD so the deployment file's playbook location can leverage
301 orig_cwd = os.getcwd()
302 env_dir = os.path.dirname(env_file)
306 for ansible_config in ansible_configs:
307 os_creds = __get_os_credentials(os_conn_config)
308 __apply_ansible_playbook(ansible_config, os_creds, vm_dict,
309 image_dict, flavor_dict)
311 # Return to original directory
317 def __apply_ansible_playbook(ansible_config, os_creds, vm_dict, image_dict,
320 Applies an Ansible configuration setting
321 :param ansible_config: the configuration settings
322 :param os_creds: the OpenStack credentials object
323 :param vm_dict: the dictionary of newly instantiated VMs where the name is
325 :param image_dict: the dictionary of newly instantiated images where the
327 :param flavor_dict: the dictionary of newly instantiated flavors where the
331 (remote_user, floating_ips, private_key_filepath,
332 proxy_settings) = __get_connection_info(
333 ansible_config, vm_dict)
335 retval = ansible_utils.apply_playbook(
336 ansible_config['playbook_location'], floating_ips, remote_user,
337 private_key_filepath,
338 variables=__get_variables(ansible_config.get('variables'),
339 os_creds, vm_dict, image_dict,
341 proxy_setting=proxy_settings)
343 # Not a fatal type of event
345 'Unable to apply playbook found at location - ' +
346 ansible_config('playbook_location'))
349 def __get_connection_info(ansible_config, vm_dict):
351 Returns a tuple of data required for connecting to the running VMs
352 (remote_user, [floating_ips], private_key_filepath, proxy_settings)
353 :param ansible_config: the configuration settings
354 :param vm_dict: the dictionary of VMs where the VM name is the key
355 :return: tuple where the first element is the user and the second is a list
356 of floating IPs and the third is the
357 private key file location and the fourth is an instance of the
358 snaps.ProxySettings class
359 (note: in order to work, each of the hosts need to have the same sudo_user
360 and private key file location values)
362 if ansible_config.get('hosts'):
363 hosts = ansible_config['hosts']
365 floating_ips = list()
368 proxy_settings = None
370 vm = vm_dict.get(host)
372 fip = vm.get_floating_ip()
374 remote_user = vm.get_image_user()
377 floating_ips.append(fip.ip)
380 'Could not find floating IP for VM - ' +
383 pk_file = vm.keypair_settings.private_filepath
384 proxy_settings = vm.get_os_creds().proxy_settings
386 logger.error('Could not locate VM with name - ' + host)
388 return remote_user, floating_ips, pk_file, proxy_settings
392 def __get_variables(var_config, os_creds, vm_dict, image_dict, flavor_dict):
394 Returns a dictionary of substitution variables to be used for Ansible
396 :param var_config: the variable configuration settings
397 :param os_creds: the OpenStack credentials object
398 :param vm_dict: the dictionary of newly instantiated VMs where the name is
400 :param image_dict: the dictionary of newly instantiated images where the
402 :param flavor_dict: the dictionary of newly instantiated flavors where the
404 :return: dictionary or None
406 if var_config and vm_dict and len(vm_dict) > 0:
408 for key, value in var_config.items():
409 value = __get_variable_value(value, os_creds, vm_dict, image_dict,
412 variables[key] = value
414 "Set Jinga2 variable with key [%s] the value [%s]",
417 logger.warning('Key [%s] or Value [%s] must not be None',
418 str(key), str(value))
423 def __get_variable_value(var_config_values, os_creds, vm_dict, image_dict,
426 Returns the associated variable value for use by Ansible for substitution
428 :param var_config_values: the configuration dictionary
429 :param os_creds: the OpenStack credentials object
430 :param vm_dict: the dictionary of newly instantiated VMs where the name is
432 :param image_dict: the dictionary of newly instantiated images where the
434 :param flavor_dict: the dictionary of newly instantiated flavors where the
438 if var_config_values['type'] == 'string':
439 return __get_string_variable_value(var_config_values)
440 if var_config_values['type'] == 'vm-attr':
441 return __get_vm_attr_variable_value(var_config_values, vm_dict)
442 if var_config_values['type'] == 'os_creds':
443 return __get_os_creds_variable_value(var_config_values, os_creds)
444 if var_config_values['type'] == 'port':
445 return __get_vm_port_variable_value(var_config_values, vm_dict)
446 if var_config_values['type'] == 'image':
447 return __get_image_variable_value(var_config_values, image_dict)
448 if var_config_values['type'] == 'flavor':
449 return __get_flavor_variable_value(var_config_values, flavor_dict)
453 def __get_string_variable_value(var_config_values):
455 Returns the associated string value
456 :param var_config_values: the configuration dictionary
457 :return: the value contained in the dictionary with the key 'value'
459 return var_config_values['value']
462 def __get_vm_attr_variable_value(var_config_values, vm_dict):
464 Returns the associated value contained on a VM instance
465 :param var_config_values: the configuration dictionary
466 :param vm_dict: the dictionary containing all VMs where the key is the VM's
470 vm = vm_dict.get(var_config_values['vm_name'])
472 if var_config_values['value'] == 'floating_ip':
473 return vm.get_floating_ip().ip
474 if var_config_values['value'] == 'image_user':
475 return vm.get_image_user()
478 def __get_os_creds_variable_value(var_config_values, os_creds):
480 Returns the associated OS credentials value
481 :param var_config_values: the configuration dictionary
482 :param os_creds: the credentials
485 logger.info("Retrieving OS Credentials")
487 if var_config_values['value'] == 'username':
488 logger.info("Returning OS username")
489 return os_creds.username
490 elif var_config_values['value'] == 'password':
491 logger.info("Returning OS password")
492 return os_creds.password
493 elif var_config_values['value'] == 'auth_url':
494 logger.info("Returning OS auth_url")
495 return os_creds.auth_url
496 elif var_config_values['value'] == 'project_name':
497 logger.info("Returning OS project_name")
498 return os_creds.project_name
500 logger.info("Returning none")
504 def __get_vm_port_variable_value(var_config_values, vm_dict):
506 Returns the associated OS credentials value
507 :param var_config_values: the configuration dictionary
508 :param vm_dict: the dictionary containing all VMs where the key is the VM's
512 port_name = var_config_values.get('port_name')
513 vm_name = var_config_values.get('vm_name')
515 if port_name and vm_name:
516 vm = vm_dict.get(vm_name)
518 port_value_id = var_config_values.get('port_value')
520 if port_value_id == 'mac_address':
521 return vm.get_port_mac(port_name)
522 if port_value_id == 'ip_address':
523 return vm.get_port_ip(port_name)
526 def __get_image_variable_value(var_config_values, image_dict):
528 Returns the associated image value
529 :param var_config_values: the configuration dictionary
530 :param image_dict: the dictionary containing all images where the key is
534 logger.info("Retrieving image values")
537 if var_config_values.get('image_name'):
538 image_creator = image_dict.get(var_config_values['image_name'])
540 if var_config_values.get('value') and \
541 var_config_values['value'] == 'id':
542 return image_creator.get_image().id
543 if var_config_values.get('value') and \
544 var_config_values['value'] == 'user':
545 return image_creator.image_settings.image_user
547 logger.info("Returning none")
551 def __get_flavor_variable_value(var_config_values, flavor_dict):
553 Returns the associated flavor value
554 :param var_config_values: the configuration dictionary
555 :param flavor_dict: the dictionary containing all flavor creators where the
557 :return: the value or None
559 logger.info("Retrieving flavor values")
562 if var_config_values.get('flavor_name'):
563 flavor_creator = flavor_dict.get(var_config_values['flavor_name'])
565 if var_config_values.get('value') and \
566 var_config_values['value'] == 'id':
567 return flavor_creator.get_flavor().id
572 Will need to set environment variable ANSIBLE_HOST_KEY_CHECKING=False or
573 Create a file located in /etc/ansible/ansible/cfg or ~/.ansible.cfg
574 containing the following content:
577 host_key_checking = False
579 CWD must be this directory where this script is located.
583 log_level = logging.INFO
584 if arguments.log_level != 'INFO':
585 log_level = logging.DEBUG
586 logging.basicConfig(level=log_level)
588 logger.info('Starting to Deploy')
589 config = file_utils.read_yaml(arguments.environment)
590 logger.debug('Read configuration file - ' + arguments.environment)
593 os_config = config.get('openstack')
595 os_conn_config = None
599 flavors_dict = dict()
603 os_conn_config = os_config.get('connection')
606 flavors_dict = __create_flavors(
607 os_conn_config, os_config.get('flavors'),
608 arguments.clean is not ARG_NOT_SET)
609 creators.append(flavors_dict)
612 images_dict = __create_images(
613 os_conn_config, os_config.get('images'),
614 arguments.clean is not ARG_NOT_SET)
615 creators.append(images_dict)
618 creators.append(__create_networks(
619 os_conn_config, os_config.get('networks'),
620 arguments.clean is not ARG_NOT_SET))
623 creators.append(__create_routers(
624 os_conn_config, os_config.get('routers'),
625 arguments.clean is not ARG_NOT_SET))
628 keypairs_dict = __create_keypairs(
629 os_conn_config, os_config.get('keypairs'),
630 arguments.clean is not ARG_NOT_SET)
631 creators.append(keypairs_dict)
634 vm_dict = __create_instances(
635 os_conn_config, os_config.get('instances'),
636 images_dict, keypairs_dict,
637 arguments.clean is not ARG_NOT_SET)
638 creators.append(vm_dict)
640 'Completed creating/retrieving all configured instances')
641 except Exception as e:
643 'Unexpected error deploying environment. Rolling back due'
648 # Must enter either block
649 if arguments.clean is not ARG_NOT_SET:
650 # Cleanup Environment
651 __cleanup(creators, arguments.clean_image is not ARG_NOT_SET)
652 elif arguments.deploy is not ARG_NOT_SET:
653 logger.info('Configuring NICs where required')
654 for vm in vm_dict.values():
656 logger.info('Completed NIC configuration')
659 ansible_config = config.get('ansible')
660 if ansible_config and vm_dict:
661 if not __apply_ansible_playbooks(ansible_config,
662 os_conn_config, vm_dict,
663 images_dict, flavors_dict,
664 arguments.environment):
665 logger.error("Problem applying ansible playbooks")
668 'Unable to read configuration file - ' + arguments.environment)
674 def __cleanup(creators, clean_image=False):
675 for creator_dict in reversed(creators):
676 for key, creator in creator_dict.items():
677 if (isinstance(creator, OpenStackImage) and clean_image) or \
678 not isinstance(creator, OpenStackImage):
681 except Exception as e:
682 logger.warning('Error cleaning component - %s', e)
685 if __name__ == '__main__':
686 # To ensure any files referenced via a relative path will begin from the
687 # directory in which this file resides
688 os.chdir(os.path.dirname(os.path.realpath(__file__)))
690 parser = argparse.ArgumentParser()
692 '-d', '--deploy', dest='deploy', nargs='?', default=ARG_NOT_SET,
693 help='When used, environment will be deployed and provisioned')
695 '-c', '--clean', dest='clean', nargs='?', default=ARG_NOT_SET,
696 help='When used, the environment will be removed')
698 '-i', '--clean-image', dest='clean_image', nargs='?',
700 help='When cleaning, if this is set, the image will be cleaned too')
702 '-e', '--env', dest='environment', required=True,
703 help='The environment configuration YAML file - REQUIRED')
705 '-l', '--log-level', dest='log_level', default='INFO',
706 help='Logging Level (INFO|DEBUG)')
707 args = parser.parse_args()
709 if args.deploy is ARG_NOT_SET and args.clean is ARG_NOT_SET:
711 'Must enter either -d for deploy or -c for cleaning up and '
714 if args.deploy is not ARG_NOT_SET and args.clean is not ARG_NOT_SET:
715 print('Cannot enter both options -d/--deploy and -c/--clean')