3 # Copyright (c) 2017 Cable Television Laboratories, Inc. ("CableLabs")
4 # and others. All rights reserved.
6 # Licensed under the Apache License, Version 2.0 (the "License");
7 # you may not use this file except in compliance with the License.
8 # You may obtain a copy of the License at:
10 # http://www.apache.org/licenses/LICENSE-2.0
12 # Unless required by applicable law or agreed to in writing, software
13 # distributed under the License is distributed on an "AS IS" BASIS,
14 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 # See the License for the specific language governing permissions and
16 # limitations under the License.
18 # This script is responsible for deploying virtual environments
24 from snaps import file_utils
25 from snaps.openstack.create_flavor import FlavorSettings, OpenStackFlavor
26 from snaps.openstack.create_image import ImageSettings
27 from snaps.openstack.create_instance import VmInstanceSettings
28 from snaps.openstack.create_network import PortSettings, NetworkSettings
29 from snaps.openstack.create_router import RouterSettings
30 from snaps.openstack.create_keypairs import KeypairSettings
31 from snaps.openstack.os_credentials import OSCreds, ProxySettings
32 from snaps.openstack.utils import deploy_utils
33 from snaps.provisioning import ansible_utils
35 __author__ = 'spisarski'
37 logger = logging.getLogger('deploy_venv')
39 ARG_NOT_SET = "argument not set"
42 def __get_os_credentials(os_conn_config):
44 Returns an object containing all of the information required to access OpenStack APIs
45 :param os_conn_config: The configuration holding the credentials
46 :return: an OSCreds instance
49 http_proxy = os_conn_config.get('http_proxy')
51 tokens = re.split(':', http_proxy)
52 ssh_proxy_cmd = os_conn_config.get('ssh_proxy_cmd')
53 proxy_settings = ProxySettings(tokens[0], tokens[1], ssh_proxy_cmd)
55 return OSCreds(username=os_conn_config.get('username'),
56 password=os_conn_config.get('password'),
57 auth_url=os_conn_config.get('auth_url'),
58 project_name=os_conn_config.get('project_name'),
59 proxy_settings=proxy_settings)
62 def __parse_ports_config(config):
64 Parses the "ports" configuration
65 :param config: The dictionary to parse
66 :return: a list of PortConfig objects
69 for port_config in config:
70 out.append(PortSettings(config=port_config.get('port')))
74 def __create_flavors(os_conn_config, flavors_config, cleanup=False):
76 Returns a dictionary of flavors where the key is the image name and the value is the image object
77 :param os_conn_config: The OpenStack connection credentials
78 :param flavors_config: The list of image configurations
79 :param cleanup: Denotes whether or not this is being called for cleanup or not
86 for flavor_config_dict in flavors_config:
87 flavor_config = flavor_config_dict.get('flavor')
88 if flavor_config and flavor_config.get('name'):
89 flavor_creator = OpenStackFlavor(__get_os_credentials(os_conn_config),
90 FlavorSettings(flavor_config))
91 flavor_creator.create(cleanup=cleanup)
92 flavors[flavor_config['name']] = flavor_creator
93 except Exception as e:
94 for key, flavor_creator in flavors.items():
95 flavor_creator.clean()
97 logger.info('Created configured flavors')
102 def __create_images(os_conn_config, images_config, cleanup=False):
104 Returns a dictionary of images where the key is the image name and the value is the image object
105 :param os_conn_config: The OpenStack connection credentials
106 :param images_config: The list of image configurations
107 :param cleanup: Denotes whether or not this is being called for cleanup or not
114 for image_config_dict in images_config:
115 image_config = image_config_dict.get('image')
116 if image_config and image_config.get('name'):
117 images[image_config['name']] = deploy_utils.create_image(__get_os_credentials(os_conn_config),
118 ImageSettings(image_config), cleanup)
119 except Exception as e:
120 for key, image_creator in images.items():
121 image_creator.clean()
123 logger.info('Created configured images')
128 def __create_networks(os_conn_config, network_confs, cleanup=False):
130 Returns a dictionary of networks where the key is the network name and the value is the network object
131 :param os_conn_config: The OpenStack connection credentials
132 :param network_confs: The list of network configurations
133 :param cleanup: Denotes whether or not this is being called for cleanup or not
140 for network_conf in network_confs:
141 net_name = network_conf['network']['name']
142 os_creds = __get_os_credentials(os_conn_config)
143 network_dict[net_name] = deploy_utils.create_network(
144 os_creds, NetworkSettings(config=network_conf['network']), cleanup)
145 except Exception as e:
146 for key, net_creator in network_dict.items():
150 logger.info('Created configured networks')
155 def __create_routers(os_conn_config, router_confs, cleanup=False):
157 Returns a dictionary of networks where the key is the network name and the value is the network object
158 :param os_conn_config: The OpenStack connection credentials
159 :param router_confs: The list of router configurations
160 :param cleanup: Denotes whether or not this is being called for cleanup or not
164 os_creds = __get_os_credentials(os_conn_config)
168 for router_conf in router_confs:
169 router_name = router_conf['router']['name']
170 router_dict[router_name] = deploy_utils.create_router(
171 os_creds, RouterSettings(config=router_conf['router']), cleanup)
172 except Exception as e:
173 for key, router_creator in router_dict.items():
174 router_creator.clean()
177 logger.info('Created configured networks')
182 def __create_keypairs(os_conn_config, keypair_confs, cleanup=False):
184 Returns a dictionary of keypairs where the key is the keypair name and the value is the keypair object
185 :param os_conn_config: The OpenStack connection credentials
186 :param keypair_confs: The list of keypair configurations
187 :param cleanup: Denotes whether or not this is being called for cleanup or not
193 for keypair_dict in keypair_confs:
194 keypair_config = keypair_dict['keypair']
195 kp_settings = KeypairSettings(keypair_config)
196 keypairs_dict[keypair_config['name']] = deploy_utils.create_keypair(
197 __get_os_credentials(os_conn_config), kp_settings, cleanup)
198 except Exception as e:
199 for key, keypair_creator in keypairs_dict.items():
200 keypair_creator.clean()
203 logger.info('Created configured keypairs')
208 def __create_instances(os_conn_config, instances_config, image_dict, keypairs_dict, cleanup=False):
210 Returns a dictionary of instances where the key is the instance name and the value is the VM object
211 :param os_conn_config: The OpenStack connection credentials
212 :param instances_config: The list of VM instance configurations
213 :param image_dict: A dictionary of images that will probably be used to instantiate the VM instance
214 :param keypairs_dict: A dictionary of keypairs that will probably be used to instantiate the VM instance
215 :param cleanup: Denotes whether or not this is being called for cleanup or not
218 os_creds = __get_os_credentials(os_conn_config)
224 for instance_config in instances_config:
225 conf = instance_config.get('instance')
228 image_creator = image_dict.get(conf.get('imageName'))
230 instance_settings = VmInstanceSettings(config=instance_config['instance'])
231 kp_name = conf.get('keypair_name')
232 vm_dict[conf['name']] = deploy_utils.create_vm_instance(
233 os_creds, instance_settings, image_creator.image_settings,
234 keypair_creator=keypairs_dict[kp_name], cleanup=cleanup)
236 raise Exception('Image creator instance not found. Cannot instantiate')
238 raise Exception('Image dictionary is None. Cannot instantiate')
240 raise Exception('Instance configuration is None. Cannot instantiate')
241 except Exception as e:
242 logger.error('Unexpected error creating instances. Attempting to cleanup environment - ' + str(e))
243 for key, inst_creator in vm_dict.items():
247 logger.info('Created configured instances')
248 # TODO Should there be an error if there isn't an instances config
252 def __apply_ansible_playbooks(ansible_configs, os_conn_config, vm_dict, image_dict, flavor_dict, env_file):
254 Applies ansible playbooks to running VMs with floating IPs
255 :param ansible_configs: a list of Ansible configurations
256 :param os_conn_config: the OpenStack connection configuration used to create an OSCreds instance
257 :param vm_dict: the dictionary of newly instantiated VMs where the name is the key
258 :param image_dict: the dictionary of newly instantiated images where the name is the key
259 :param flavor_dict: the dictionary of newly instantiated flavors where the name is the key
260 :param env_file: the path of the environment for setting the CWD so playbook location is relative to the deployment
262 :return: t/f - true if successful
264 logger.info("Applying Ansible Playbooks")
266 # Ensure all hosts are accepting SSH session requests
267 for vm_inst in list(vm_dict.values()):
268 if not vm_inst.vm_ssh_active(block=True):
269 logger.warning("Timeout waiting for instance to respond to SSH requests")
272 # Set CWD so the deployment file's playbook location can leverage relative paths
273 orig_cwd = os.getcwd()
274 env_dir = os.path.dirname(env_file)
278 for ansible_config in ansible_configs:
279 os_creds = __get_os_credentials(os_conn_config)
280 __apply_ansible_playbook(ansible_config, os_creds, vm_dict, image_dict, flavor_dict)
282 # Return to original directory
288 def __apply_ansible_playbook(ansible_config, os_creds, vm_dict, image_dict, flavor_dict):
290 Applies an Ansible configuration setting
291 :param ansible_config: the configuration settings
292 :param os_creds: the OpenStack credentials object
293 :param vm_dict: the dictionary of newly instantiated VMs where the name is the key
294 :param image_dict: the dictionary of newly instantiated images where the name is the key
295 :param flavor_dict: the dictionary of newly instantiated flavors where the name is the key
298 remote_user, floating_ips, private_key_filepath, proxy_settings = __get_connection_info(ansible_config, vm_dict)
300 retval = ansible_utils.apply_playbook(
301 ansible_config['playbook_location'], floating_ips, remote_user, private_key_filepath,
302 variables=__get_variables(ansible_config.get('variables'), os_creds, vm_dict, image_dict, flavor_dict),
303 proxy_setting=proxy_settings)
305 # Not a fatal type of event
306 logger.warning('Unable to apply playbook found at location - ' + ansible_config('playbook_location'))
309 def __get_connection_info(ansible_config, vm_dict):
311 Returns a tuple of data required for connecting to the running VMs
312 (remote_user, [floating_ips], private_key_filepath, proxy_settings)
313 :param ansible_config: the configuration settings
314 :param vm_dict: the dictionary of VMs where the VM name is the key
315 :return: tuple where the first element is the user and the second is a list of floating IPs and the third is the
316 private key file location and the fourth is an instance of the snaps.ProxySettings class
317 (note: in order to work, each of the hosts need to have the same sudo_user and private key file location values)
319 if ansible_config.get('hosts'):
320 hosts = ansible_config['hosts']
322 floating_ips = list()
324 private_key_filepath = None
325 proxy_settings = None
327 vm = vm_dict.get(host)
329 fip = vm.get_floating_ip()
331 remote_user = vm.get_image_user()
334 floating_ips.append(fip.ip)
336 raise Exception('Could not find floating IP for VM - ' + vm.name)
338 private_key_filepath = vm.keypair_settings.private_filepath
339 proxy_settings = vm.get_os_creds().proxy_settings
341 logger.error('Could not locate VM with name - ' + host)
343 return remote_user, floating_ips, private_key_filepath, proxy_settings
347 def __get_variables(var_config, os_creds, vm_dict, image_dict, flavor_dict):
349 Returns a dictionary of substitution variables to be used for Ansible templates
350 :param var_config: the variable configuration settings
351 :param os_creds: the OpenStack credentials object
352 :param vm_dict: the dictionary of newly instantiated VMs where the name is the key
353 :param image_dict: the dictionary of newly instantiated images where the name is the key
354 :param flavor_dict: the dictionary of newly instantiated flavors where the name is the key
355 :return: dictionary or None
357 if var_config and vm_dict and len(vm_dict) > 0:
359 for key, value in var_config.items():
360 value = __get_variable_value(value, os_creds, vm_dict, image_dict, flavor_dict)
362 variables[key] = value
363 logger.info("Set Jinga2 variable with key [" + key + "] the value [" + value + ']')
365 logger.warning('Key [' + str(key) + '] or Value [' + str(value) + '] must not be None')
370 def __get_variable_value(var_config_values, os_creds, vm_dict, image_dict, flavor_dict):
372 Returns the associated variable value for use by Ansible for substitution purposes
373 :param var_config_values: the configuration dictionary
374 :param os_creds: the OpenStack credentials object
375 :param vm_dict: the dictionary of newly instantiated VMs where the name is the key
376 :param image_dict: the dictionary of newly instantiated images where the name is the key
377 :param flavor_dict: the dictionary of newly instantiated flavors where the name is the key
380 if var_config_values['type'] == 'string':
381 return __get_string_variable_value(var_config_values)
382 if var_config_values['type'] == 'vm-attr':
383 return __get_vm_attr_variable_value(var_config_values, vm_dict)
384 if var_config_values['type'] == 'os_creds':
385 return __get_os_creds_variable_value(var_config_values, os_creds)
386 if var_config_values['type'] == 'port':
387 return __get_vm_port_variable_value(var_config_values, vm_dict)
388 if var_config_values['type'] == 'image':
389 return __get_image_variable_value(var_config_values, image_dict)
390 if var_config_values['type'] == 'flavor':
391 return __get_flavor_variable_value(var_config_values, flavor_dict)
395 def __get_string_variable_value(var_config_values):
397 Returns the associated string value
398 :param var_config_values: the configuration dictionary
399 :return: the value contained in the dictionary with the key 'value'
401 return var_config_values['value']
404 def __get_vm_attr_variable_value(var_config_values, vm_dict):
406 Returns the associated value contained on a VM instance
407 :param var_config_values: the configuration dictionary
408 :param vm_dict: the dictionary containing all VMs where the key is the VM's name
411 vm = vm_dict.get(var_config_values['vm_name'])
413 if var_config_values['value'] == 'floating_ip':
414 return vm.get_floating_ip().ip
415 if var_config_values['value'] == 'image_user':
416 return vm.get_image_user()
419 def __get_os_creds_variable_value(var_config_values, os_creds):
421 Returns the associated OS credentials value
422 :param var_config_values: the configuration dictionary
423 :param os_creds: the credentials
426 logger.info("Retrieving OS Credentials")
428 if var_config_values['value'] == 'username':
429 logger.info("Returning OS username")
430 return os_creds.username
431 elif var_config_values['value'] == 'password':
432 logger.info("Returning OS password")
433 return os_creds.password
434 elif var_config_values['value'] == 'auth_url':
435 logger.info("Returning OS auth_url")
436 return os_creds.auth_url
437 elif var_config_values['value'] == 'project_name':
438 logger.info("Returning OS project_name")
439 return os_creds.project_name
441 logger.info("Returning none")
445 def __get_vm_port_variable_value(var_config_values, vm_dict):
447 Returns the associated OS credentials value
448 :param var_config_values: the configuration dictionary
449 :param vm_dict: the dictionary containing all VMs where the key is the VM's name
452 port_name = var_config_values.get('port_name')
453 vm_name = var_config_values.get('vm_name')
455 if port_name and vm_name:
456 vm = vm_dict.get(vm_name)
458 port_value_id = var_config_values.get('port_value')
460 if port_value_id == 'mac_address':
461 return vm.get_port_mac(port_name)
462 if port_value_id == 'ip_address':
463 return vm.get_port_ip(port_name)
466 def __get_image_variable_value(var_config_values, image_dict):
468 Returns the associated image value
469 :param var_config_values: the configuration dictionary
470 :param image_dict: the dictionary containing all images where the key is the name
473 logger.info("Retrieving image values")
476 if var_config_values.get('image_name'):
477 image_creator = image_dict.get(var_config_values['image_name'])
479 if var_config_values.get('value') and var_config_values['value'] == 'id':
480 return image_creator.get_image().id
481 if var_config_values.get('value') and var_config_values['value'] == 'user':
482 return image_creator.image_settings.image_user
484 logger.info("Returning none")
488 def __get_flavor_variable_value(var_config_values, flavor_dict):
490 Returns the associated flavor value
491 :param var_config_values: the configuration dictionary
492 :param flavor_dict: the dictionary containing all flavor creators where the key is the name
493 :return: the value or None
495 logger.info("Retrieving flavor values")
498 if var_config_values.get('flavor_name'):
499 flavor_creator = flavor_dict.get(var_config_values['flavor_name'])
501 if var_config_values.get('value') and var_config_values['value'] == 'id':
502 return flavor_creator.get_flavor().id
504 logger.info("Returning none")
510 Will need to set environment variable ANSIBLE_HOST_KEY_CHECKING=False or ...
511 Create a file located in /etc/ansible/ansible/cfg or ~/.ansible.cfg containing the following content:
514 host_key_checking = False
516 CWD must be this directory where this script is located.
520 log_level = logging.INFO
521 if arguments.log_level != 'INFO':
522 log_level = logging.DEBUG
523 logging.basicConfig(level=log_level)
525 logger.info('Starting to Deploy')
526 config = file_utils.read_yaml(arguments.environment)
527 logger.debug('Read configuration file - ' + arguments.environment)
530 os_config = config.get('openstack')
532 os_conn_config = None
542 os_conn_config = os_config.get('connection')
545 flavor_dict = __create_flavors(os_conn_config, os_config.get('flavors'),
546 arguments.clean is not ARG_NOT_SET)
549 image_dict = __create_images(os_conn_config, os_config.get('images'),
550 arguments.clean is not ARG_NOT_SET)
553 network_dict = __create_networks(os_conn_config, os_config.get('networks'),
554 arguments.clean is not ARG_NOT_SET)
557 router_dict = __create_routers(os_conn_config, os_config.get('routers'),
558 arguments.clean is not ARG_NOT_SET)
561 keypairs_dict = __create_keypairs(os_conn_config, os_config.get('keypairs'),
562 arguments.clean is not ARG_NOT_SET)
565 vm_dict = __create_instances(os_conn_config, os_config.get('instances'), image_dict, keypairs_dict,
566 arguments.clean is not ARG_NOT_SET)
567 logger.info('Completed creating/retrieving all configured instances')
568 except Exception as e:
569 logger.error('Unexpected error deploying environment. Rolling back due to - ' + str(e))
570 __cleanup(vm_dict, keypairs_dict, router_dict, network_dict, image_dict, flavor_dict, True)
573 # Must enter either block
574 if arguments.clean is not ARG_NOT_SET:
575 # Cleanup Environment
576 __cleanup(vm_dict, keypairs_dict, router_dict, network_dict, image_dict, flavor_dict,
577 arguments.clean_image is not ARG_NOT_SET)
578 elif arguments.deploy is not ARG_NOT_SET:
579 logger.info('Configuring NICs where required')
580 for vm in vm_dict.values():
582 logger.info('Completed NIC configuration')
585 ansible_config = config.get('ansible')
586 if ansible_config and vm_dict:
587 if not __apply_ansible_playbooks(ansible_config, os_conn_config, vm_dict, image_dict, flavor_dict,
588 arguments.environment):
589 logger.error("Problem applying ansible playbooks")
591 logger.error('Unable to read configuration file - ' + arguments.environment)
597 def __cleanup(vm_dict, keypairs_dict, router_dict, network_dict, image_dict, flavor_dict, clean_image=False):
598 for key, vm_inst in vm_dict.items():
600 for key, kp_inst in keypairs_dict.items():
602 for key, router_inst in router_dict.items():
606 logger.warning("Router not found continuing to next component")
607 for key, net_inst in network_dict.items():
611 logger.warning("Network not found continuing to next component")
613 for key, image_inst in image_dict.items():
615 for key, flavor_inst in flavor_dict.items():
619 if __name__ == '__main__':
620 # To ensure any files referenced via a relative path will begin from the diectory in which this file resides
621 os.chdir(os.path.dirname(os.path.realpath(__file__)))
623 parser = argparse.ArgumentParser()
624 parser.add_argument('-d', '--deploy', dest='deploy', nargs='?', default=ARG_NOT_SET,
625 help='When used, environment will be deployed and provisioned')
626 parser.add_argument('-c', '--clean', dest='clean', nargs='?', default=ARG_NOT_SET,
627 help='When used, the environment will be removed')
628 parser.add_argument('-i', '--clean-image', dest='clean_image', nargs='?', default=ARG_NOT_SET,
629 help='When cleaning, if this is set, the image will be cleaned too')
630 parser.add_argument('-e', '--env', dest='environment', required=True,
631 help='The environment configuration YAML file - REQUIRED')
632 parser.add_argument('-l', '--log-level', dest='log_level', default='INFO', help='Logging Level (INFO|DEBUG)')
633 args = parser.parse_args()
635 if args.deploy is ARG_NOT_SET and args.clean is ARG_NOT_SET:
636 print('Must enter either -d for deploy or -c for cleaning up and environment')
638 if args.deploy is not ARG_NOT_SET and args.clean is not ARG_NOT_SET:
639 print('Cannot enter both options -d/--deploy and -c/--clean')