1 # Copyright (c) 2017 Cable Television Laboratories, Inc. ("CableLabs")
2 # and others. All rights reserved.
4 # Licensed under the Apache License, Version 2.0 (the "License");
5 # you may not use this file except in compliance with the License.
6 # You may obtain a copy of the License at:
8 # http://www.apache.org/licenses/LICENSE-2.0
10 # Unless required by applicable law or agreed to in writing, software
11 # distributed under the License is distributed on an "AS IS" BASIS,
12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 # See the License for the specific language governing permissions and
14 # limitations under the License.
18 from neutronclient.common.exceptions import PortNotFoundClient
19 from novaclient.exceptions import NotFound
21 from snaps.openstack.utils import glance_utils
22 from snaps.openstack.utils import neutron_utils
23 from snaps.openstack.create_network import PortSettings
24 from snaps.provisioning import ansible_utils
25 from snaps.openstack.utils import nova_utils
27 __author__ = 'spisarski'
29 logger = logging.getLogger('create_instance')
32 STATUS_ACTIVE = 'ACTIVE'
33 STATUS_DELETED = 'DELETED'
36 class OpenStackVmInstance:
38 Class responsible for creating a VM instance in OpenStack
41 def __init__(self, os_creds, instance_settings, image_settings, keypair_settings=None):
44 :param os_creds: The connection credentials to the OpenStack API
45 :param instance_settings: Contains the settings for this VM
46 :param image_settings: The OpenStack image object settings
47 :param keypair_settings: The keypair metadata (Optional)
50 self.__os_creds = os_creds
55 self.instance_settings = instance_settings
56 self.image_settings = image_settings
57 self.keypair_settings = keypair_settings
59 # TODO - get rid of FIP list and only use the dict(). Need to fix populating this object when already exists
60 self.__floating_ips = list()
61 self.__floating_ip_dict = dict()
63 # Instantiated in self.create()
66 # Note: this object does not change after the VM becomes active
69 def create(self, cleanup=False, block=False):
72 :param cleanup: When true, only perform lookups for OpenStack objects.
73 :param block: Thread will block until instance has either become active, error, or timeout waiting.
74 Additionally, when True, floating IPs will not be applied until VM is active.
75 :return: The VM reference object
77 self.__nova = nova_utils.nova_client(self.__os_creds)
78 self.__neutron = neutron_utils.neutron_client(self.__os_creds)
80 self.__ports = self.__setup_ports(self.instance_settings.port_settings, cleanup)
81 self.__lookup_existing_vm_by_name()
82 if not self.__vm and not cleanup:
83 self.__create_vm(block)
86 def __lookup_existing_vm_by_name(self):
88 Populates the member variables 'self.vm' and 'self.floating_ips' if a VM with the same name already exists
91 servers = nova_utils.get_servers_by_name(self.__nova, self.instance_settings.name)
92 for server in servers:
93 if server.name == self.instance_settings.name:
95 logger.info('Found existing machine with name - ' + self.instance_settings.name)
96 fips = self.__nova.floating_ips.list()
98 if fip.instance_id == server.id:
99 self.__floating_ips.append(fip)
100 # TODO - Determine a means to associate to the FIP configuration and add to FIP map
102 def __create_vm(self, block=False):
104 Responsible for creating the VM instance
105 :param block: Thread will block until instance has either become active, error, or timeout waiting.
106 Floating IPs will be assigned after active when block=True
109 for key, port in self.__ports:
111 kv['port-id'] = port['port']['id']
114 logger.info('Creating VM with name - ' + self.instance_settings.name)
116 if self.keypair_settings:
117 keypair_name = self.keypair_settings.name
119 flavor = nova_utils.get_flavor_by_name(self.__nova, self.instance_settings.flavor)
121 raise Exception('Flavor not found with name - ' + self.instance_settings.flavor)
123 image = glance_utils.get_image(glance_utils.glance_client(self.__os_creds),
124 self.image_settings.name)
126 self.__vm = self.__nova.servers.create(
127 name=self.instance_settings.name,
131 key_name=keypair_name,
132 security_groups=self.instance_settings.security_group_names,
133 userdata=self.instance_settings.userdata,
134 availability_zone=self.instance_settings.availability_zone)
137 raise Exception('Cannot create instance, image cannot be located with name ' + self.image_settings.name)
139 logger.info('Created instance with name - ' + self.instance_settings.name)
142 if not self.vm_active(block=True):
143 raise Exception('Fatal error, VM did not become ACTIVE within the alloted time')
145 # TODO - the call above should add security groups. The return object shows they exist but the association
146 # had never been made by OpenStack. This call is here to ensure they have been added
147 for sec_grp_name in self.instance_settings.security_group_names:
148 if self.vm_active(block=True):
149 nova_utils.add_security_group(self.__nova, self.__vm, sec_grp_name)
151 raise Exception('Cannot applying security group with name ' + sec_grp_name +
152 ' to VM that did not activate with name - ' + self.instance_settings.name)
154 self.__apply_floating_ips()
156 def __apply_floating_ips(self):
158 Applies the configured floating IPs to the necessary ports
161 for key, port in self.__ports:
162 port_dict[key] = port
165 for floating_ip_setting in self.instance_settings.floating_ip_settings:
166 port = port_dict.get(floating_ip_setting.port_name)
169 raise Exception('Cannot find port object with name - ' + floating_ip_setting.port_name)
171 # Setup Floating IP only if there is a router with an external gateway
172 ext_gateway = self.__ext_gateway_by_router(floating_ip_setting.router_name)
174 subnet = neutron_utils.get_subnet_by_name(self.__neutron, floating_ip_setting.subnet_name)
175 floating_ip = nova_utils.create_floating_ip(self.__nova, ext_gateway)
176 self.__floating_ips.append(floating_ip)
177 self.__floating_ip_dict[floating_ip_setting.name] = floating_ip
179 logger.info('Created floating IP ' + floating_ip.ip + ' via router - ' +
180 floating_ip_setting.router_name)
181 self.__add_floating_ip(floating_ip, port, subnet)
183 raise Exception('Unable to add floating IP to port,' +
184 ' cannot locate router with an external gateway ')
186 def __ext_gateway_by_router(self, router_name):
188 Returns network name for the external network attached to a router or None if not found
189 :param router_name: The name of the router to lookup
190 :return: the external network name or None
192 router = neutron_utils.get_router_by_name(self.__neutron, router_name)
193 if router and router['router'].get('external_gateway_info'):
194 network = neutron_utils.get_network_by_id(self.__neutron,
195 router['router']['external_gateway_info']['network_id'])
197 return network['network']['name']
202 Destroys the VM instance
205 # Cleanup floating IPs
206 for floating_ip in self.__floating_ips:
208 logger.info('Deleting Floating IP - ' + floating_ip.ip)
209 nova_utils.delete_floating_ip(self.__nova, floating_ip)
210 except Exception as e:
211 logger.error('Error deleting Floating IP - ' + str(e))
212 self.__floating_ips = list()
213 self.__floating_ip_dict = dict()
216 for name, port in self.__ports:
217 logger.info('Deleting Port - ' + name)
219 neutron_utils.delete_port(self.__neutron, port)
220 except PortNotFoundClient as e:
221 logger.warning('Unexpected error deleting port - ' + str(e))
223 self.__ports = list()
228 logger.info('Deleting VM instance - ' + self.instance_settings.name)
229 nova_utils.delete_vm_instance(self.__nova, self.__vm)
230 except Exception as e:
231 logger.error('Error deleting VM - ' + str(e))
233 # Block until instance cannot be found or returns the status of DELETED
234 logger.info('Checking deletion status')
237 if self.vm_deleted(block=True):
238 logger.info('VM has been properly deleted VM with name - ' + self.instance_settings.name)
241 logger.error('VM not deleted within the timeout period of ' +
242 str(self.instance_settings.vm_delete_timeout) + ' seconds')
243 except Exception as e:
244 logger.error('Unexpected error while checking VM instance status - ' + str(e))
246 def __setup_ports(self, port_settings, cleanup):
248 Returns the previously configured ports or creates them if they do not exist
249 :param port_settings: A list of PortSetting objects
250 :param cleanup: When true, only perform lookups for OpenStack objects.
251 :return: a list of OpenStack port tuples where the first member is the port name and the second is the port
256 for port_setting in port_settings:
257 # First check to see if network already has this port
258 # TODO/FIXME - this could potentially cause problems if another port with the same name exists
259 # VM has the same network/port name pair
262 # TODO/FIXME - should we not be iterating on ports for the specific network in question as unique port names
263 # seem to only be important by network
264 existing_ports = self.__neutron.list_ports()['ports']
265 for existing_port in existing_ports:
266 if existing_port['name'] == port_setting.name:
267 ports.append((port_setting.name, {'port': existing_port}))
271 if not found and not cleanup:
272 ports.append((port_setting.name, neutron_utils.create_port(self.__neutron, self.__os_creds,
277 def __add_floating_ip(self, floating_ip, port, subnet, timeout=30, poll_interval=POLL_INTERVAL):
279 Returns True when active else False
280 TODO - Make timeout and poll_interval configurable...
285 # Take IP of subnet if there is one configured on which to place the floating IP
286 for fixed_ip in port['port']['fixed_ips']:
287 if fixed_ip['subnet_id'] == subnet['subnet']['id']:
288 ip = fixed_ip['ip_address']
291 # Simply take the first
292 ip = port['port']['fixed_ips'][0]['ip_address']
295 count = timeout / poll_interval
297 logger.debug('Attempting to add floating IP to instance')
299 self.__vm.add_floating_ip(floating_ip, ip)
300 logger.info('Added floating IP ' + floating_ip.ip + ' to port IP - ' + ip +
301 ' on instance - ' + self.instance_settings.name)
303 except Exception as e:
304 logger.debug('Retry adding floating IP to instance. Last attempt failed with - ' + str(e))
305 time.sleep(poll_interval)
309 raise Exception('Unable find IP address on which to place the floating IP')
311 logger.error('Timeout attempting to add the floating IP to instance.')
312 raise Exception('Timeout while attempting add floating IP to instance')
314 def get_os_creds(self):
316 Returns the OpenStack credentials used to create these objects
317 :return: the credentials
319 return self.__os_creds
321 def get_vm_inst(self):
323 Returns the latest version of this server object from OpenStack
324 :return: Server object
326 return nova_utils.get_latest_server_object(self.__nova, self.__vm)
328 def get_port_ip(self, port_name, subnet_name=None):
330 Returns the first IP for the port corresponding with the port_name parameter when subnet_name is None
331 else returns the IP address that corresponds to the subnet_name parameter
332 :param port_name: the name of the port from which to return the IP
333 :param subnet_name: the name of the subnet attached to this IP
334 :return: the IP or None if not found
336 port = self.get_port_by_name(port_name)
338 port_dict = port['port']
340 subnet = neutron_utils.get_subnet_by_name(self.__neutron, subnet_name)
342 logger.warning('Cannot retrieve port IP as subnet could not be located with name - ' + subnet_name)
344 for fixed_ip in port_dict['fixed_ips']:
345 if fixed_ip['subnet_id'] == subnet['subnet']['id']:
346 return fixed_ip['ip_address']
348 fixed_ips = port_dict['fixed_ips']
349 if fixed_ips and len(fixed_ips) > 0:
350 return fixed_ips[0]['ip_address']
353 def get_port_mac(self, port_name):
355 Returns the first IP for the port corresponding with the port_name parameter
356 TODO - Add in the subnet as an additional parameter as a port may have multiple fixed_ips
357 :param port_name: the name of the port from which to return the IP
358 :return: the IP or None if not found
360 port = self.get_port_by_name(port_name)
362 port_dict = port['port']
363 return port_dict['mac_address']
366 def get_port_by_name(self, port_name):
368 Retrieves the OpenStack port object by its given name
369 :param port_name: the name of the port
370 :return: the OpenStack port object or None if not exists
372 for key, port in self.__ports:
375 logger.warning('Cannot find port with name - ' + port_name)
378 def config_nics(self):
380 Responsible for configuring NICs on RPM systems where the instance has more than one configured port
383 if len(self.__ports) > 1 and len(self.__floating_ips) > 0:
384 if self.vm_active(block=True) and self.vm_ssh_active(block=True):
385 for key, port in self.__ports:
386 port_index = self.__ports.index((key, port))
388 nic_name = 'eth' + repr(port_index)
389 self.__config_nic(nic_name, port, self.__get_first_provisioning_floating_ip().ip)
390 logger.info('Configured NIC - ' + nic_name + ' on VM - ' + self.instance_settings.name)
392 def __get_first_provisioning_floating_ip(self):
394 Returns the first floating IP tagged with the Floating IP name if exists else the first one found
397 for floating_ip_setting in self.instance_settings.floating_ip_settings:
398 if floating_ip_setting.provisioning:
399 fip = self.__floating_ip_dict.get(floating_ip_setting.name)
402 elif len(self.__floating_ips) > 0:
403 return self.__floating_ips[0]
405 def __config_nic(self, nic_name, port, floating_ip):
407 Although ports/NICs can contain multiple IPs, this code currently only supports the first.
409 Your CWD at this point must be the <repo dir>/python directory.
410 TODO - fix this restriction.
412 :param nic_name: Name of the interface
413 :param port: The port information containing the expected IP values.
414 :param floating_ip: The floating IP on which to apply the playbook.
415 :return: the return value from ansible
417 ip = port['port']['fixed_ips'][0]['ip_address']
419 'floating_ip': floating_ip,
420 'nic_name': nic_name,
424 if self.image_settings.nic_config_pb_loc and self.keypair_settings:
425 return self.apply_ansible_playbook(self.image_settings.nic_config_pb_loc, variables)
427 logger.warning('VM ' + self.instance_settings.name + ' cannot self configure NICs eth1++. ' +
428 'No playbook or keypairs found.')
430 def apply_ansible_playbook(self, pb_file_loc, variables=None, fip_name=None):
432 Applies a playbook to a VM
433 :param pb_file_loc: the file location of the playbook to be applied
434 :param variables: a dict() of substitution values required by the playbook
435 :param fip_name: the name of the floating IP to use for applying the playbook (default - will take the first)
436 :return: the return value from ansible
438 return ansible_utils.apply_playbook(pb_file_loc, [self.get_floating_ip(fip_name=fip_name).ip],
439 self.get_image_user(), self.keypair_settings.private_filepath,
440 variables, self.__os_creds.proxy_settings)
442 def get_image_user(self):
444 Returns the instance sudo_user if it has been configured in the instance_settings else it returns the
445 image_settings.image_user value
447 if self.instance_settings.sudo_user:
448 return self.instance_settings.sudo_user
450 return self.image_settings.image_user
452 def vm_deleted(self, block=False, poll_interval=POLL_INTERVAL):
454 Returns true when the VM status returns the value of expected_status_code or instance retrieval throws
455 a NotFound exception.
456 :param block: When true, thread will block until active or timeout value in seconds has been exceeded (False)
457 :param poll_interval: The polling interval in seconds
461 return self.__vm_status_check(STATUS_DELETED, block, self.instance_settings.vm_delete_timeout,
463 except NotFound as e:
464 logger.debug("Instance not found when querying status for " + STATUS_DELETED + ' with message ' + str(e))
467 def vm_active(self, block=False, poll_interval=POLL_INTERVAL):
469 Returns true when the VM status returns the value of expected_status_code
470 :param block: When true, thread will block until active or timeout value in seconds has been exceeded (False)
471 :param poll_interval: The polling interval in seconds
474 return self.__vm_status_check(STATUS_ACTIVE, block, self.instance_settings.vm_boot_timeout, poll_interval)
476 def __vm_status_check(self, expected_status_code, block, timeout, poll_interval):
478 Returns true when the VM status returns the value of expected_status_code
479 :param expected_status_code: instance status evaluated with this string value
480 :param block: When true, thread will block until active or timeout value in seconds has been exceeded (False)
481 :param timeout: The timeout value
482 :param poll_interval: The polling interval in seconds
485 # sleep and wait for VM status change
489 return self.__status(expected_status_code)
491 while timeout > time.time() - start:
492 status = self.__status(expected_status_code)
494 logger.info('VM is - ' + expected_status_code)
497 logger.debug('Retry querying VM status in ' + str(poll_interval) + ' seconds')
498 time.sleep(poll_interval)
499 logger.debug('VM status query timeout in ' + str(timeout - (time.time() - start)))
501 logger.error('Timeout checking for VM status for ' + expected_status_code)
504 def __status(self, expected_status_code):
506 Returns True when active else False
507 :param expected_status_code: instance status evaluated with this string value
513 instance = self.__nova.servers.get(self.__vm.id)
515 logger.warning('Cannot find instance with id - ' + self.__vm.id)
518 if instance.status == 'ERROR':
519 raise Exception('Instance had an error during deployment')
520 logger.debug('Instance status [' + self.instance_settings.name + '] is - ' + instance.status)
521 return instance.status == expected_status_code
523 def vm_ssh_active(self, block=False, poll_interval=POLL_INTERVAL):
525 Returns true when the VM can be accessed via SSH
526 :param block: When true, thread will block until active or timeout value in seconds has been exceeded (False)
527 :param poll_interval: The polling interval
530 # sleep and wait for VM status change
531 logger.info('Checking if VM is active')
533 timeout = self.instance_settings.ssh_connect_timeout
535 if self.vm_active(block=True):
539 start = time.time() - timeout
541 while timeout > time.time() - start:
542 status = self.__ssh_active()
544 logger.info('SSH is active for VM instance')
547 logger.debug('Retry SSH connection in ' + str(poll_interval) + ' seconds')
548 time.sleep(poll_interval)
549 logger.debug('SSH connection timeout in ' + str(timeout - (time.time() - start)))
551 logger.error('Timeout attempting to connect with VM via SSH')
554 def __ssh_active(self):
556 Returns True when can create a SSH session else False
559 if len(self.__floating_ips) > 0:
560 ssh = self.ssh_client()
565 def get_floating_ip(self, fip_name=None):
567 Returns the floating IP object byt name if found, else the first known, else None
568 :param fip_name: the name of the floating IP to return
569 :return: the SSH client or None
572 if fip_name and self.__floating_ip_dict.get(fip_name):
573 return self.__floating_ip_dict.get(fip_name)
574 if not fip and len(self.__floating_ips) > 0:
575 return self.__floating_ips[0]
578 def ssh_client(self, fip_name=None):
580 Returns an SSH client using the name or the first known floating IP if exists, else None
581 :param fip_name: the name of the floating IP to return
582 :return: the SSH client or None
584 fip = self.get_floating_ip(fip_name)
586 return ansible_utils.ssh_client(self.__floating_ips[0].ip, self.get_image_user(),
587 self.keypair_settings.private_filepath,
588 proxy_settings=self.__os_creds.proxy_settings)
590 logger.warning('Cannot return an SSH client. No Floating IP configured')
592 def add_security_group(self, security_group):
594 Adds a security group to this VM. Call will block until VM is active.
595 :param security_group: the OpenStack security group object
596 :return True if successful else False
598 self.vm_active(block=True)
600 if not security_group:
601 logger.warning('Security group object is None, cannot add')
605 nova_utils.add_security_group(self.__nova, self.get_vm_inst(), security_group['security_group']['name'])
607 except NotFound as e:
608 logger.warning('Security group not added - ' + str(e))
611 def remove_security_group(self, security_group):
613 Removes a security group to this VM. Call will block until VM is active.
614 :param security_group: the OpenStack security group object
615 :return True if successful else False
617 self.vm_active(block=True)
619 if not security_group:
620 logger.warning('Security group object is None, cannot remove')
624 nova_utils.remove_security_group(self.__nova, self.get_vm_inst(), security_group)
626 except NotFound as e:
627 logger.warning('Security group not removed - ' + str(e))
631 class VmInstanceSettings:
633 Class responsible for holding configuration setting for a VM Instance
635 def __init__(self, config=None, name=None, flavor=None, port_settings=list(), security_group_names=set(),
636 floating_ip_settings=list(), sudo_user=None, vm_boot_timeout=900,
637 vm_delete_timeout=300, ssh_connect_timeout=180, availability_zone=None, userdata=None):
640 :param config: dict() object containing the configuration settings using the attribute names below as each
641 member's the key and overrides any of the other parameters.
642 :param name: the name of the VM
643 :param flavor: the VM's flavor
644 :param port_settings: the port configuration settings (required)
645 :param security_group_names: a set of names of the security groups to add to the VM
646 :param floating_ip_settings: the floating IP configuration settings
647 :param sudo_user: the sudo user of the VM that will override the instance_settings.image_user when trying to
649 :param vm_boot_timeout: the amount of time a thread will sleep waiting for an instance to boot
650 :param vm_delete_timeout: the amount of time a thread will sleep waiting for an instance to be deleted
651 :param ssh_connect_timeout: the amount of time a thread will sleep waiting obtaining an SSH connection to a VM
652 :param availability_zone: the name of the compute server on which to deploy the VM (optional)
653 :param userdata: the cloud-init script to run after the VM has been started
656 self.name = config.get('name')
657 self.flavor = config.get('flavor')
658 self.sudo_user = config.get('sudo_user')
659 self.userdata = config.get('userdata')
661 self.port_settings = list()
662 if config.get('ports'):
663 for port_config in config['ports']:
664 if isinstance(port_config, PortSettings):
665 self.port_settings.append(port_config)
667 self.port_settings.append(PortSettings(config=port_config['port']))
669 if config.get('security_group_names'):
670 if isinstance(config['security_group_names'], list):
671 self.security_group_names = set(config['security_group_names'])
672 elif isinstance(config['security_group_names'], set):
673 self.security_group_names = config['security_group_names']
674 elif isinstance(config['security_group_names'], str):
675 self.security_group_names = [config['security_group_names']]
677 raise Exception('Invalid data type for security_group_names attribute')
679 self.security_group_names = set()
681 self.floating_ip_settings = list()
682 if config.get('floating_ips'):
683 for floating_ip_config in config['floating_ips']:
684 if isinstance(floating_ip_config, FloatingIpSettings):
685 self.floating_ip_settings.append(floating_ip_config)
687 self.floating_ip_settings.append(FloatingIpSettings(config=floating_ip_config['floating_ip']))
689 if config.get('vm_boot_timeout'):
690 self.vm_boot_timeout = config['vm_boot_timeout']
692 self.vm_boot_timeout = vm_boot_timeout
694 if config.get('vm_delete_timeout'):
695 self.vm_delete_timeout = config['vm_delete_timeout']
697 self.vm_delete_timeout = vm_delete_timeout
699 if config.get('ssh_connect_timeout'):
700 self.ssh_connect_timeout = config['ssh_connect_timeout']
702 self.ssh_connect_timeout = ssh_connect_timeout
704 if config.get('availability_zone'):
705 self.availability_zone = config['availability_zone']
707 self.availability_zone = None
711 self.port_settings = port_settings
712 self.security_group_names = security_group_names
713 self.floating_ip_settings = floating_ip_settings
714 self.sudo_user = sudo_user
715 self.vm_boot_timeout = vm_boot_timeout
716 self.vm_delete_timeout = vm_delete_timeout
717 self.ssh_connect_timeout = ssh_connect_timeout
718 self.availability_zone = availability_zone
719 self.userdata = userdata
721 if not self.name or not self.flavor:
722 raise Exception('Instance configuration requires the attributes: name, flavor')
724 if len(self.port_settings) == 0:
725 raise Exception('Instance configuration requires port settings (aka. NICS)')
728 class FloatingIpSettings:
730 Class responsible for holding configuration settings for a floating IP
732 def __init__(self, config=None, name=None, port_name=None, router_name=None, subnet_name=None, provisioning=True):
735 :param config: dict() object containing the configuration settings using the attribute names below as each
736 member's the key and overrides any of the other parameters.
737 :param name: the name of the floating IP
738 :param port_name: the name of the router to the external network
739 :param router_name: the name of the router to the external network
740 :param subnet_name: the name of the subnet on which to attach the floating IP
741 :param provisioning: when true, this floating IP can be used for provisioning
743 TODO - provisioning flag is a hack as I have only observed a single Floating IPs that actually works on
744 an instance. Multiple floating IPs placed on different subnets from the same port are especially troublesome
745 as you cannot predict which one will actually connect. For now, it is recommended not to setup multiple
746 floating IPs on an instance unless absolutely necessary.
749 self.name = config.get('name')
750 self.port_name = config.get('port_name')
751 self.router_name = config.get('router_name')
752 self.subnet_name = config.get('subnet_name')
753 if config.get('provisioning') is not None:
754 self.provisioning = config['provisioning']
756 self.provisioning = provisioning
759 self.port_name = port_name
760 self.router_name = router_name
761 self.subnet_name = subnet_name
762 self.provisioning = provisioning
764 if not self.name or not self.port_name or not self.router_name:
765 raise Exception('The attributes name, port_name and router_name are required for FloatingIPSettings')