3 # Copyright (c) 2017 Cable Television Laboratories, Inc. ("CableLabs")
4 # and others. All rights reserved.
6 # Licensed under the Apache License, Version 2.0 (the "License");
7 # you may not use this file except in compliance with the License.
8 # You may obtain a copy of the License at:
10 # http://www.apache.org/licenses/LICENSE-2.0
12 # Unless required by applicable law or agreed to in writing, software
13 # distributed under the License is distributed on an "AS IS" BASIS,
14 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 # See the License for the specific language governing permissions and
16 # limitations under the License.
18 # This script is responsible for deploying virtual environments
24 from snaps import file_utils
25 from snaps.openstack.create_flavor import FlavorSettings, OpenStackFlavor
26 from snaps.openstack.create_image import ImageSettings, OpenStackImage
27 from snaps.openstack.create_instance import VmInstanceSettings
28 from snaps.openstack.create_keypairs import KeypairSettings
29 from snaps.openstack.create_network import PortSettings, NetworkSettings
30 from snaps.openstack.create_router import RouterSettings
31 from snaps.openstack.os_credentials import OSCreds, ProxySettings
32 from snaps.openstack.utils import deploy_utils
33 from snaps.provisioning import ansible_utils
35 __author__ = 'spisarski'
37 logger = logging.getLogger('deploy_venv')
39 ARG_NOT_SET = "argument not set"
42 def __get_os_credentials(os_conn_config):
44 Returns an object containing all of the information required to access
46 :param os_conn_config: The configuration holding the credentials
47 :return: an OSCreds instance
50 http_proxy = os_conn_config.get('http_proxy')
52 tokens = re.split(':', http_proxy)
53 ssh_proxy_cmd = os_conn_config.get('ssh_proxy_cmd')
54 proxy_settings = ProxySettings(tokens[0], tokens[1], ssh_proxy_cmd)
56 return OSCreds(username=os_conn_config.get('username'),
57 password=os_conn_config.get('password'),
58 auth_url=os_conn_config.get('auth_url'),
59 project_name=os_conn_config.get('project_name'),
60 proxy_settings=proxy_settings)
63 def __parse_ports_config(config):
65 Parses the "ports" configuration
66 :param config: The dictionary to parse
67 :return: a list of PortConfig objects
70 for port_config in config:
71 out.append(PortSettings(**port_config.get('port')))
75 def __create_flavors(os_conn_config, flavors_config, cleanup=False):
77 Returns a dictionary of flavors where the key is the image name and the
78 value is the image object
79 :param os_conn_config: The OpenStack connection credentials
80 :param flavors_config: The list of image configurations
81 :param cleanup: Denotes whether or not this is being called for cleanup
88 for flavor_config_dict in flavors_config:
89 flavor_config = flavor_config_dict.get('flavor')
90 if flavor_config and flavor_config.get('name'):
91 flavor_creator = OpenStackFlavor(
92 __get_os_credentials(os_conn_config),
93 FlavorSettings(**flavor_config))
94 flavor_creator.create(cleanup=cleanup)
95 flavors[flavor_config['name']] = flavor_creator
96 except Exception as e:
97 for key, flavor_creator in flavors.items():
98 flavor_creator.clean()
100 logger.info('Created configured flavors')
105 def __create_images(os_conn_config, images_config, cleanup=False):
107 Returns a dictionary of images where the key is the image name and the
108 value is the image object
109 :param os_conn_config: The OpenStack connection credentials
110 :param images_config: The list of image configurations
111 :param cleanup: Denotes whether or not this is being called for cleanup
118 for image_config_dict in images_config:
119 image_config = image_config_dict.get('image')
120 if image_config and image_config.get('name'):
121 images[image_config['name']] = deploy_utils.create_image(
122 __get_os_credentials(os_conn_config),
123 ImageSettings(**image_config), cleanup)
124 except Exception as e:
125 for key, image_creator in images.items():
126 image_creator.clean()
128 logger.info('Created configured images')
133 def __create_networks(os_conn_config, network_confs, cleanup=False):
135 Returns a dictionary of networks where the key is the network name and the
136 value is the network object
137 :param os_conn_config: The OpenStack connection credentials
138 :param network_confs: The list of network configurations
139 :param cleanup: Denotes whether or not this is being called for cleanup
146 for network_conf in network_confs:
147 net_name = network_conf['network']['name']
148 os_creds = __get_os_credentials(os_conn_config)
149 network_dict[net_name] = deploy_utils.create_network(
150 os_creds, NetworkSettings(**network_conf['network']),
152 except Exception as e:
153 for key, net_creator in network_dict.items():
157 logger.info('Created configured networks')
162 def __create_routers(os_conn_config, router_confs, cleanup=False):
164 Returns a dictionary of networks where the key is the network name and the
165 value is the network object
166 :param os_conn_config: The OpenStack connection credentials
167 :param router_confs: The list of router configurations
168 :param cleanup: Denotes whether or not this is being called for cleanup
172 os_creds = __get_os_credentials(os_conn_config)
176 for router_conf in router_confs:
177 router_name = router_conf['router']['name']
178 router_dict[router_name] = deploy_utils.create_router(
179 os_creds, RouterSettings(**router_conf['router']), cleanup)
180 except Exception as e:
181 for key, router_creator in router_dict.items():
182 router_creator.clean()
185 logger.info('Created configured networks')
190 def __create_keypairs(os_conn_config, keypair_confs, cleanup=False):
192 Returns a dictionary of keypairs where the key is the keypair name and the
193 value is the keypair object
194 :param os_conn_config: The OpenStack connection credentials
195 :param keypair_confs: The list of keypair configurations
196 :param cleanup: Denotes whether or not this is being called for cleanup
202 for keypair_dict in keypair_confs:
203 keypair_config = keypair_dict['keypair']
204 kp_settings = KeypairSettings(**keypair_config)
206 keypair_config['name']] = deploy_utils.create_keypair(
207 __get_os_credentials(os_conn_config), kp_settings, cleanup)
208 except Exception as e:
209 for key, keypair_creator in keypairs_dict.items():
210 keypair_creator.clean()
213 logger.info('Created configured keypairs')
218 def __create_instances(os_conn_config, instances_config, image_dict,
219 keypairs_dict, cleanup=False):
221 Returns a dictionary of instances where the key is the instance name and
222 the value is the VM object
223 :param os_conn_config: The OpenStack connection credentials
224 :param instances_config: The list of VM instance configurations
225 :param image_dict: A dictionary of images that will probably be used to
226 instantiate the VM instance
227 :param keypairs_dict: A dictionary of keypairs that will probably be used
228 to instantiate the VM instance
229 :param cleanup: Denotes whether or not this is being called for cleanup
232 os_creds = __get_os_credentials(os_conn_config)
238 for instance_config in instances_config:
239 conf = instance_config.get('instance')
242 image_creator = image_dict.get(conf.get('imageName'))
244 instance_settings = VmInstanceSettings(
245 **instance_config['instance'])
246 kp_name = conf.get('keypair_name')
248 'name']] = deploy_utils.create_vm_instance(
249 os_creds, instance_settings,
250 image_creator.image_settings,
251 keypair_creator=keypairs_dict[kp_name],
254 raise Exception('Image creator instance not found.'
255 ' Cannot instantiate')
257 raise Exception('Image dictionary is None. Cannot '
260 raise Exception('Instance configuration is None. Cannot '
262 except Exception as e:
263 logger.error('Unexpected error creating instances. Attempting to '
264 'cleanup environment - %s', e)
265 for key, inst_creator in vm_dict.items():
269 logger.info('Created configured instances')
270 # TODO Should there be an error if there isn't an instances config
274 def __apply_ansible_playbooks(ansible_configs, os_conn_config, vm_dict,
275 image_dict, flavor_dict, env_file):
277 Applies ansible playbooks to running VMs with floating IPs
278 :param ansible_configs: a list of Ansible configurations
279 :param os_conn_config: the OpenStack connection configuration used to
280 create an OSCreds instance
281 :param vm_dict: the dictionary of newly instantiated VMs where the name is
283 :param image_dict: the dictionary of newly instantiated images where the
285 :param flavor_dict: the dictionary of newly instantiated flavors where the
287 :param env_file: the path of the environment for setting the CWD so
288 playbook location is relative to the deployment file
289 :return: t/f - true if successful
291 logger.info("Applying Ansible Playbooks")
293 # Ensure all hosts are accepting SSH session requests
294 for vm_inst in list(vm_dict.values()):
295 if not vm_inst.vm_ssh_active(block=True):
297 "Timeout waiting for instance to respond to SSH requests")
300 # Set CWD so the deployment file's playbook location can leverage
302 orig_cwd = os.getcwd()
303 env_dir = os.path.dirname(env_file)
307 for ansible_config in ansible_configs:
308 os_creds = __get_os_credentials(os_conn_config)
309 __apply_ansible_playbook(ansible_config, os_creds, vm_dict,
310 image_dict, flavor_dict)
312 # Return to original directory
318 def __apply_ansible_playbook(ansible_config, os_creds, vm_dict, image_dict,
321 Applies an Ansible configuration setting
322 :param ansible_config: the configuration settings
323 :param os_creds: the OpenStack credentials object
324 :param vm_dict: the dictionary of newly instantiated VMs where the name is
326 :param image_dict: the dictionary of newly instantiated images where the
328 :param flavor_dict: the dictionary of newly instantiated flavors where the
332 (remote_user, floating_ips, private_key_filepath,
333 proxy_settings) = __get_connection_info(
334 ansible_config, vm_dict)
336 retval = ansible_utils.apply_playbook(
337 ansible_config['playbook_location'], floating_ips, remote_user,
338 private_key_filepath,
339 variables=__get_variables(ansible_config.get('variables'),
340 os_creds, vm_dict, image_dict,
342 proxy_setting=proxy_settings)
344 # Not a fatal type of event
346 'Unable to apply playbook found at location - ' +
347 ansible_config('playbook_location'))
350 def __get_connection_info(ansible_config, vm_dict):
352 Returns a tuple of data required for connecting to the running VMs
353 (remote_user, [floating_ips], private_key_filepath, proxy_settings)
354 :param ansible_config: the configuration settings
355 :param vm_dict: the dictionary of VMs where the VM name is the key
356 :return: tuple where the first element is the user and the second is a list
357 of floating IPs and the third is the
358 private key file location and the fourth is an instance of the
359 snaps.ProxySettings class
360 (note: in order to work, each of the hosts need to have the same sudo_user
361 and private key file location values)
363 if ansible_config.get('hosts'):
364 hosts = ansible_config['hosts']
366 floating_ips = list()
369 proxy_settings = None
371 vm = vm_dict.get(host)
373 fip = vm.get_floating_ip()
375 remote_user = vm.get_image_user()
378 floating_ips.append(fip.ip)
381 'Could not find floating IP for VM - ' +
384 pk_file = vm.keypair_settings.private_filepath
385 proxy_settings = vm.get_os_creds().proxy_settings
387 logger.error('Could not locate VM with name - ' + host)
389 return remote_user, floating_ips, pk_file, proxy_settings
393 def __get_variables(var_config, os_creds, vm_dict, image_dict, flavor_dict):
395 Returns a dictionary of substitution variables to be used for Ansible
397 :param var_config: the variable configuration settings
398 :param os_creds: the OpenStack credentials object
399 :param vm_dict: the dictionary of newly instantiated VMs where the name is
401 :param image_dict: the dictionary of newly instantiated images where the
403 :param flavor_dict: the dictionary of newly instantiated flavors where the
405 :return: dictionary or None
407 if var_config and vm_dict and len(vm_dict) > 0:
409 for key, value in var_config.items():
410 value = __get_variable_value(value, os_creds, vm_dict, image_dict,
413 variables[key] = value
415 "Set Jinga2 variable with key [%s] the value [%s]",
418 logger.warning('Key [%s] or Value [%s] must not be None',
419 str(key), str(value))
424 def __get_variable_value(var_config_values, os_creds, vm_dict, image_dict,
427 Returns the associated variable value for use by Ansible for substitution
429 :param var_config_values: the configuration dictionary
430 :param os_creds: the OpenStack credentials object
431 :param vm_dict: the dictionary of newly instantiated VMs where the name is
433 :param image_dict: the dictionary of newly instantiated images where the
435 :param flavor_dict: the dictionary of newly instantiated flavors where the
439 if var_config_values['type'] == 'string':
440 return __get_string_variable_value(var_config_values)
441 if var_config_values['type'] == 'vm-attr':
442 return __get_vm_attr_variable_value(var_config_values, vm_dict)
443 if var_config_values['type'] == 'os_creds':
444 return __get_os_creds_variable_value(var_config_values, os_creds)
445 if var_config_values['type'] == 'port':
446 return __get_vm_port_variable_value(var_config_values, vm_dict)
447 if var_config_values['type'] == 'image':
448 return __get_image_variable_value(var_config_values, image_dict)
449 if var_config_values['type'] == 'flavor':
450 return __get_flavor_variable_value(var_config_values, flavor_dict)
454 def __get_string_variable_value(var_config_values):
456 Returns the associated string value
457 :param var_config_values: the configuration dictionary
458 :return: the value contained in the dictionary with the key 'value'
460 return var_config_values['value']
463 def __get_vm_attr_variable_value(var_config_values, vm_dict):
465 Returns the associated value contained on a VM instance
466 :param var_config_values: the configuration dictionary
467 :param vm_dict: the dictionary containing all VMs where the key is the VM's
471 vm = vm_dict.get(var_config_values['vm_name'])
473 if var_config_values['value'] == 'floating_ip':
474 return vm.get_floating_ip().ip
475 if var_config_values['value'] == 'image_user':
476 return vm.get_image_user()
479 def __get_os_creds_variable_value(var_config_values, os_creds):
481 Returns the associated OS credentials value
482 :param var_config_values: the configuration dictionary
483 :param os_creds: the credentials
486 logger.info("Retrieving OS Credentials")
488 if var_config_values['value'] == 'username':
489 logger.info("Returning OS username")
490 return os_creds.username
491 elif var_config_values['value'] == 'password':
492 logger.info("Returning OS password")
493 return os_creds.password
494 elif var_config_values['value'] == 'auth_url':
495 logger.info("Returning OS auth_url")
496 return os_creds.auth_url
497 elif var_config_values['value'] == 'project_name':
498 logger.info("Returning OS project_name")
499 return os_creds.project_name
501 logger.info("Returning none")
505 def __get_vm_port_variable_value(var_config_values, vm_dict):
507 Returns the associated OS credentials value
508 :param var_config_values: the configuration dictionary
509 :param vm_dict: the dictionary containing all VMs where the key is the VM's
513 port_name = var_config_values.get('port_name')
514 vm_name = var_config_values.get('vm_name')
516 if port_name and vm_name:
517 vm = vm_dict.get(vm_name)
519 port_value_id = var_config_values.get('port_value')
521 if port_value_id == 'mac_address':
522 return vm.get_port_mac(port_name)
523 if port_value_id == 'ip_address':
524 return vm.get_port_ip(port_name)
527 def __get_image_variable_value(var_config_values, image_dict):
529 Returns the associated image value
530 :param var_config_values: the configuration dictionary
531 :param image_dict: the dictionary containing all images where the key is
535 logger.info("Retrieving image values")
538 if var_config_values.get('image_name'):
539 image_creator = image_dict.get(var_config_values['image_name'])
541 if var_config_values.get('value') and \
542 var_config_values['value'] == 'id':
543 return image_creator.get_image().id
544 if var_config_values.get('value') and \
545 var_config_values['value'] == 'user':
546 return image_creator.image_settings.image_user
548 logger.info("Returning none")
552 def __get_flavor_variable_value(var_config_values, flavor_dict):
554 Returns the associated flavor value
555 :param var_config_values: the configuration dictionary
556 :param flavor_dict: the dictionary containing all flavor creators where the
558 :return: the value or None
560 logger.info("Retrieving flavor values")
563 if var_config_values.get('flavor_name'):
564 flavor_creator = flavor_dict.get(var_config_values['flavor_name'])
566 if var_config_values.get('value') and \
567 var_config_values['value'] == 'id':
568 return flavor_creator.get_flavor().id
570 logger.info("Returning none")
576 Will need to set environment variable ANSIBLE_HOST_KEY_CHECKING=False or
577 Create a file located in /etc/ansible/ansible/cfg or ~/.ansible.cfg
578 containing the following content:
581 host_key_checking = False
583 CWD must be this directory where this script is located.
587 log_level = logging.INFO
588 if arguments.log_level != 'INFO':
589 log_level = logging.DEBUG
590 logging.basicConfig(level=log_level)
592 logger.info('Starting to Deploy')
593 config = file_utils.read_yaml(arguments.environment)
594 logger.debug('Read configuration file - ' + arguments.environment)
597 os_config = config.get('openstack')
599 os_conn_config = None
603 flavors_dict = dict()
607 os_conn_config = os_config.get('connection')
610 flavors_dict = __create_flavors(
611 os_conn_config, os_config.get('flavors'),
612 arguments.clean is not ARG_NOT_SET)
613 creators.append(flavors_dict)
616 images_dict = __create_images(
617 os_conn_config, os_config.get('images'),
618 arguments.clean is not ARG_NOT_SET)
619 creators.append(images_dict)
622 creators.append(__create_networks(
623 os_conn_config, os_config.get('networks'),
624 arguments.clean is not ARG_NOT_SET))
627 creators.append(__create_routers(
628 os_conn_config, os_config.get('routers'),
629 arguments.clean is not ARG_NOT_SET))
632 keypairs_dict = __create_keypairs(
633 os_conn_config, os_config.get('keypairs'),
634 arguments.clean is not ARG_NOT_SET)
635 creators.append(keypairs_dict)
638 vm_dict = __create_instances(
639 os_conn_config, os_config.get('instances'),
640 images_dict, keypairs_dict,
641 arguments.clean is not ARG_NOT_SET)
642 creators.append(vm_dict)
644 'Completed creating/retrieving all configured instances')
645 except Exception as e:
647 'Unexpected error deploying environment. Rolling back due'
652 # Must enter either block
653 if arguments.clean is not ARG_NOT_SET:
654 # Cleanup Environment
655 __cleanup(creators, arguments.clean_image is not ARG_NOT_SET)
656 elif arguments.deploy is not ARG_NOT_SET:
657 logger.info('Configuring NICs where required')
658 for vm in vm_dict.values():
660 logger.info('Completed NIC configuration')
663 ansible_config = config.get('ansible')
664 if ansible_config and vm_dict:
665 if not __apply_ansible_playbooks(ansible_config,
666 os_conn_config, vm_dict,
667 images_dict, flavors_dict,
668 arguments.environment):
669 logger.error("Problem applying ansible playbooks")
672 'Unable to read configuration file - ' + arguments.environment)
678 def __cleanup(creators, clean_image=False):
679 for creator_dict in reversed(creators):
680 for key, creator in creator_dict.items():
681 if (isinstance(creator, OpenStackImage) and clean_image) or \
682 not isinstance(creator, OpenStackImage):
685 except Exception as e:
686 logger.warning('Error cleaning component - %s', e)
689 if __name__ == '__main__':
690 # To ensure any files referenced via a relative path will begin from the
691 # directory in which this file resides
692 os.chdir(os.path.dirname(os.path.realpath(__file__)))
694 parser = argparse.ArgumentParser()
696 '-d', '--deploy', dest='deploy', nargs='?', default=ARG_NOT_SET,
697 help='When used, environment will be deployed and provisioned')
699 '-c', '--clean', dest='clean', nargs='?', default=ARG_NOT_SET,
700 help='When used, the environment will be removed')
702 '-i', '--clean-image', dest='clean_image', nargs='?',
704 help='When cleaning, if this is set, the image will be cleaned too')
706 '-e', '--env', dest='environment', required=True,
707 help='The environment configuration YAML file - REQUIRED')
709 '-l', '--log-level', dest='log_level', default='INFO',
710 help='Logging Level (INFO|DEBUG)')
711 args = parser.parse_args()
713 if args.deploy is ARG_NOT_SET and args.clean is ARG_NOT_SET:
715 'Must enter either -d for deploy or -c for cleaning up and '
718 if args.deploy is not ARG_NOT_SET and args.clean is not ARG_NOT_SET:
719 print('Cannot enter both options -d/--deploy and -c/--clean')