1 # Copyright (c) 2017 Cable Television Laboratories, Inc. ("CableLabs")
2 # and others. All rights reserved.
4 # Licensed under the Apache License, Version 2.0 (the "License");
5 # you may not use this file except in compliance with the License.
6 # You may obtain a copy of the License at:
8 # http://www.apache.org/licenses/LICENSE-2.0
10 # Unless required by applicable law or agreed to in writing, software
11 # distributed under the License is distributed on an "AS IS" BASIS,
12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 # See the License for the specific language governing permissions and
14 # limitations under the License.
18 from neutronclient.common.exceptions import PortNotFoundClient
19 from novaclient.exceptions import NotFound
21 from snaps.openstack.create_network import PortSettings
22 from snaps.openstack.utils import glance_utils
23 from snaps.openstack.utils import neutron_utils
24 from snaps.openstack.utils import nova_utils
25 from snaps.provisioning import ansible_utils
27 __author__ = 'spisarski'
29 logger = logging.getLogger('create_instance')
32 STATUS_ACTIVE = 'ACTIVE'
33 STATUS_DELETED = 'DELETED'
36 class OpenStackVmInstance:
38 Class responsible for creating a VM instance in OpenStack
41 def __init__(self, os_creds, instance_settings, image_settings,
42 keypair_settings=None):
45 :param os_creds: The connection credentials to the OpenStack API
46 :param instance_settings: Contains the settings for this VM
47 :param image_settings: The OpenStack image object settings
48 :param keypair_settings: The keypair metadata (Optional)
51 self.__os_creds = os_creds
56 self.instance_settings = instance_settings
57 self.image_settings = image_settings
58 self.keypair_settings = keypair_settings
60 self.__floating_ip_dict = dict()
62 # Instantiated in self.create()
65 # Note: this object does not change after the VM becomes active
68 def create(self, cleanup=False, block=False):
71 :param cleanup: When true, only perform lookups for OpenStack objects.
72 :param block: Thread will block until instance has either become
73 active, error, or timeout waiting.
74 Additionally, when True, floating IPs will not be applied
76 :return: The VM reference object
78 self.__nova = nova_utils.nova_client(self.__os_creds)
79 self.__neutron = neutron_utils.neutron_client(self.__os_creds)
81 self.__ports = self.__setup_ports(self.instance_settings.port_settings,
83 self.__lookup_existing_vm_by_name()
84 if not self.__vm and not cleanup:
85 self.__create_vm(block)
88 def __lookup_existing_vm_by_name(self):
90 Populates the member variables 'self.vm' and 'self.floating_ips' if a
91 VM with the same name already exists
94 servers = nova_utils.get_servers_by_name(self.__nova,
95 self.instance_settings.name)
96 for server in servers:
97 if server.name == self.instance_settings.name:
100 'Found existing machine with name - %s',
101 self.instance_settings.name)
103 fips = neutron_utils.get_floating_ips(self.__neutron,
105 for port_name, fip in fips:
106 settings = self.instance_settings.floating_ip_settings
107 for fip_setting in settings:
108 if port_name == fip_setting.port_name:
109 self.__floating_ip_dict[fip_setting.name] = fip
111 def __create_vm(self, block=False):
113 Responsible for creating the VM instance
114 :param block: Thread will block until instance has either become
115 active, error, or timeout waiting. Floating IPs will be
116 assigned after active when block=True
118 glance = glance_utils.glance_client(self.__os_creds)
119 self.__vm = nova_utils.create_server(
120 self.__nova, self.__neutron, glance, self.instance_settings,
121 self.image_settings, self.keypair_settings)
122 logger.info('Created instance with name - %s',
123 self.instance_settings.name)
126 if not self.vm_active(block=True):
127 raise VmInstanceCreationError(
128 'Fatal error, VM did not become ACTIVE within the alloted '
131 # Create server should do this but found it needed to occur here
132 for sec_grp_name in self.instance_settings.security_group_names:
133 if self.vm_active(block=True):
134 nova_utils.add_security_group(self.__nova, self.__vm,
137 raise VmInstanceCreationError(
138 'Cannot applying security group with name ' +
140 ' to VM that did not activate with name - ' +
141 self.instance_settings.name)
143 self.__apply_floating_ips()
145 def __apply_floating_ips(self):
147 Applies the configured floating IPs to the necessary ports
150 for key, port in self.__ports:
151 port_dict[key] = port
154 for floating_ip_setting in self.instance_settings.floating_ip_settings:
155 port = port_dict.get(floating_ip_setting.port_name)
158 raise VmInstanceCreationError(
159 'Cannot find port object with name - ' +
160 floating_ip_setting.port_name)
162 # Setup Floating IP only if there is a router with an external
164 ext_gateway = self.__ext_gateway_by_router(
165 floating_ip_setting.router_name)
167 subnet = neutron_utils.get_subnet_by_name(
168 self.__neutron, floating_ip_setting.subnet_name)
169 floating_ip = neutron_utils.create_floating_ip(
170 self.__neutron, ext_gateway)
171 self.__floating_ip_dict[floating_ip_setting.name] = floating_ip
174 'Created floating IP %s via router - %s', floating_ip.ip,
175 floating_ip_setting.router_name)
176 self.__add_floating_ip(floating_ip, port, subnet)
178 raise VmInstanceCreationError(
179 'Unable to add floating IP to port, cannot locate router '
180 'with an external gateway ')
182 def __ext_gateway_by_router(self, router_name):
184 Returns network name for the external network attached to a router or
186 :param router_name: The name of the router to lookup
187 :return: the external network name or None
189 router = neutron_utils.get_router_by_name(self.__neutron, router_name)
190 if router and router.external_gateway_info:
191 network = neutron_utils.get_network_by_id(
193 router.external_gateway_info['network_id'])
200 Destroys the VM instance
203 # Cleanup floating IPs
204 for name, floating_ip in self.__floating_ip_dict.items():
206 logger.info('Deleting Floating IP - ' + floating_ip.ip)
207 neutron_utils.delete_floating_ip(self.__neutron, floating_ip)
208 except Exception as e:
209 logger.error('Error deleting Floating IP - ' + str(e))
210 self.__floating_ip_dict = dict()
213 for name, port in self.__ports:
214 logger.info('Deleting Port - ' + name)
216 neutron_utils.delete_port(self.__neutron, port)
217 except PortNotFoundClient as e:
218 logger.warning('Unexpected error deleting port - %s', e)
220 self.__ports = list()
226 'Deleting VM instance - ' + self.instance_settings.name)
227 nova_utils.delete_vm_instance(self.__nova, self.__vm)
228 except Exception as e:
229 logger.error('Error deleting VM - %s', e)
231 # Block until instance cannot be found or returns the status of
233 logger.info('Checking deletion status')
236 if self.vm_deleted(block=True):
238 'VM has been properly deleted VM with name - %s',
239 self.instance_settings.name)
243 'VM not deleted within the timeout period of %s '
244 'seconds', self.instance_settings.vm_delete_timeout)
245 except Exception as e:
247 'Unexpected error while checking VM instance status - %s',
250 def __setup_ports(self, port_settings, cleanup):
252 Returns the previously configured ports or creates them if they do not
254 :param port_settings: A list of PortSetting objects
255 :param cleanup: When true, only perform lookups for OpenStack objects.
256 :return: a list of OpenStack port tuples where the first member is the
257 port name and the second is the port object
261 for port_setting in port_settings:
262 port = neutron_utils.get_port_by_name(self.__neutron,
265 ports.append((port_setting.name, port))
267 # Exception will be raised when port with same name already
270 (port_setting.name, neutron_utils.create_port(
271 self.__neutron, self.__os_creds, port_setting)))
275 def __add_floating_ip(self, floating_ip, port, subnet, timeout=30,
276 poll_interval=POLL_INTERVAL):
278 Returns True when active else False
279 TODO - Make timeout and poll_interval configurable...
284 # Take IP of subnet if there is one configured on which to place
286 for fixed_ip in port.ips:
287 if fixed_ip['subnet_id'] == subnet.id:
288 ip = fixed_ip['ip_address']
291 # Simply take the first
292 ip = port.ips[0]['ip_address']
295 count = timeout / poll_interval
297 logger.debug('Attempting to add floating IP to instance')
299 nova_utils.add_floating_ip_to_server(
300 self.__nova, self.__vm, floating_ip, ip)
302 'Added floating IP %s to port IP %s on instance %s',
303 floating_ip.ip, ip, self.instance_settings.name)
305 except Exception as e:
307 'Retry adding floating IP to instance. Last attempt '
308 'failed with - %s', e)
309 time.sleep(poll_interval)
313 raise VmInstanceCreationError(
314 'Unable find IP address on which to place the floating IP')
316 logger.error('Timeout attempting to add the floating IP to instance.')
317 raise VmInstanceCreationError(
318 'Timeout while attempting add floating IP to instance')
320 def get_os_creds(self):
322 Returns the OpenStack credentials used to create these objects
323 :return: the credentials
325 return self.__os_creds
327 def get_vm_inst(self):
329 Returns the latest version of this server object from OpenStack
330 :return: Server object
334 def get_console_output(self):
336 Returns the vm console object for parsing logs
337 :return: the console output object
339 return nova_utils.get_server_console_output(self.__nova, self.__vm)
341 def get_port_ip(self, port_name, subnet_name=None):
343 Returns the first IP for the port corresponding with the port_name
344 parameter when subnet_name is None else returns the IP address that
345 corresponds to the subnet_name parameter
346 :param port_name: the name of the port from which to return the IP
347 :param subnet_name: the name of the subnet attached to this IP
348 :return: the IP or None if not found
350 port = self.get_port_by_name(port_name)
353 subnet = neutron_utils.get_subnet_by_name(self.__neutron,
356 logger.warning('Cannot retrieve port IP as subnet could '
357 'not be located with name - %s',
360 for fixed_ip in port.ips:
361 if fixed_ip['subnet_id'] == subnet.id:
362 return fixed_ip['ip_address']
364 if port.ips and len(port.ips) > 0:
365 return port.ips[0]['ip_address']
368 def get_port_mac(self, port_name):
370 Returns the first IP for the port corresponding with the port_name
372 TODO - Add in the subnet as an additional parameter as a port may have
374 :param port_name: the name of the port from which to return the IP
375 :return: the IP or None if not found
377 port = self.get_port_by_name(port_name)
379 return port.mac_address
382 def get_port_by_name(self, port_name):
384 Retrieves the OpenStack port object by its given name
385 :param port_name: the name of the port
386 :return: the OpenStack port object or None if not exists
388 for key, port in self.__ports:
391 logger.warning('Cannot find port with name - ' + port_name)
394 def get_vm_info(self):
396 Returns a dictionary of a VMs info as returned by OpenStack
399 return nova_utils.get_server_info(self.__nova, self.__vm)
401 def config_nics(self):
403 Responsible for configuring NICs on RPM systems where the instance has
404 more than one configured port
405 :return: the value returned by ansible_utils.apply_ansible_playbook()
407 if len(self.__ports) > 1 and len(self.__floating_ip_dict) > 0:
408 if self.vm_active(block=True) and self.vm_ssh_active(block=True):
409 for key, port in self.__ports:
410 port_index = self.__ports.index((key, port))
412 nic_name = 'eth' + repr(port_index)
413 retval = self.__config_nic(
415 self.__get_first_provisioning_floating_ip().ip)
416 logger.info('Configured NIC - %s on VM - %s',
417 nic_name, self.instance_settings.name)
420 def __get_first_provisioning_floating_ip(self):
422 Returns the first floating IP tagged with the Floating IP name if
423 exists else the first one found
426 for floating_ip_setting in self.instance_settings.floating_ip_settings:
427 if floating_ip_setting.provisioning:
428 fip = self.__floating_ip_dict.get(floating_ip_setting.name)
431 elif len(self.__floating_ip_dict) > 0:
432 for key, fip in self.__floating_ip_dict.items():
435 def __config_nic(self, nic_name, port, ip):
437 Although ports/NICs can contain multiple IPs, this code currently only
440 :param nic_name: Name of the interface
441 :param port: The port information containing the expected IP values.
442 :param ip: The IP on which to apply the playbook.
443 :return: the return value from ansible
445 port_ip = port.ips[0]['ip_address']
448 'nic_name': nic_name,
452 if self.image_settings.nic_config_pb_loc and self.keypair_settings:
453 return self.apply_ansible_playbook(
454 self.image_settings.nic_config_pb_loc, variables)
457 'VM %s cannot self configure NICs eth1++. No playbook or '
458 'keypairs found.', self.instance_settings.name)
460 def apply_ansible_playbook(self, pb_file_loc, variables=None,
463 Applies a playbook to a VM
464 :param pb_file_loc: the file location of the playbook to be applied
465 :param variables: a dict() of substitution values required by the
467 :param fip_name: the name of the floating IP to use for applying the
468 playbook (default - will take the first)
469 :return: the return value from ansible
471 return ansible_utils.apply_playbook(
472 pb_file_loc, [self.get_floating_ip(fip_name=fip_name).ip],
473 self.get_image_user(), self.keypair_settings.private_filepath,
474 variables, self.__os_creds.proxy_settings)
476 def get_image_user(self):
478 Returns the instance sudo_user if it has been configured in the
479 instance_settings else it returns the image_settings.image_user value
481 if self.instance_settings.sudo_user:
482 return self.instance_settings.sudo_user
484 return self.image_settings.image_user
486 def vm_deleted(self, block=False, poll_interval=POLL_INTERVAL):
488 Returns true when the VM status returns the value of
489 expected_status_code or instance retrieval throws a NotFound exception.
490 :param block: When true, thread will block until active or timeout
491 value in seconds has been exceeded (False)
492 :param poll_interval: The polling interval in seconds
496 return self.__vm_status_check(
497 STATUS_DELETED, block,
498 self.instance_settings.vm_delete_timeout, poll_interval)
499 except NotFound as e:
501 "Instance not found when querying status for %s with message "
502 "%s", STATUS_DELETED, e)
505 def vm_active(self, block=False, poll_interval=POLL_INTERVAL):
507 Returns true when the VM status returns the value of
509 :param block: When true, thread will block until active or timeout
510 value in seconds has been exceeded (False)
511 :param poll_interval: The polling interval in seconds
514 return self.__vm_status_check(STATUS_ACTIVE, block,
515 self.instance_settings.vm_boot_timeout,
518 def __vm_status_check(self, expected_status_code, block, timeout,
521 Returns true when the VM status returns the value of
523 :param expected_status_code: instance status evaluated with this
525 :param block: When true, thread will block until active or timeout
526 value in seconds has been exceeded (False)
527 :param timeout: The timeout value
528 :param poll_interval: The polling interval in seconds
531 # sleep and wait for VM status change
535 return self.__status(expected_status_code)
537 while timeout > time.time() - start:
538 status = self.__status(expected_status_code)
540 logger.info('VM is - ' + expected_status_code)
543 logger.debug('Retry querying VM status in ' + str(
544 poll_interval) + ' seconds')
545 time.sleep(poll_interval)
546 logger.debug('VM status query timeout in ' + str(
547 timeout - (time.time() - start)))
550 'Timeout checking for VM status for ' + expected_status_code)
553 def __status(self, expected_status_code):
555 Returns True when active else False
556 :param expected_status_code: instance status evaluated with this string
563 status = nova_utils.get_server_status(self.__nova, self.__vm)
565 logger.warning('Cannot find instance with id - ' + self.__vm.id)
568 if status == 'ERROR':
569 raise VmInstanceCreationError(
570 'Instance had an error during deployment')
572 'Instance status [%s] is - %s', self.instance_settings.name,
574 return status == expected_status_code
576 def vm_ssh_active(self, block=False, poll_interval=POLL_INTERVAL):
578 Returns true when the VM can be accessed via SSH
579 :param block: When true, thread will block until active or timeout
580 value in seconds has been exceeded (False)
581 :param poll_interval: The polling interval
584 # sleep and wait for VM status change
585 logger.info('Checking if VM is active')
587 timeout = self.instance_settings.ssh_connect_timeout
589 if self.vm_active(block=True):
593 start = time.time() - timeout
595 while timeout > time.time() - start:
596 status = self.__ssh_active()
598 logger.info('SSH is active for VM instance')
601 logger.debug('Retry SSH connection in ' + str(
602 poll_interval) + ' seconds')
603 time.sleep(poll_interval)
604 logger.debug('SSH connection timeout in ' + str(
605 timeout - (time.time() - start)))
607 logger.error('Timeout attempting to connect with VM via SSH')
610 def __ssh_active(self):
612 Returns True when can create a SSH session else False
615 if len(self.__floating_ip_dict) > 0:
616 ssh = self.ssh_client()
622 def get_floating_ip(self, fip_name=None):
624 Returns the floating IP object byt name if found, else the first known,
626 :param fip_name: the name of the floating IP to return
627 :return: the SSH client or None
630 if fip_name and self.__floating_ip_dict.get(fip_name):
631 return self.__floating_ip_dict.get(fip_name)
633 return self.__get_first_provisioning_floating_ip()
635 def ssh_client(self, fip_name=None):
637 Returns an SSH client using the name or the first known floating IP if
639 :param fip_name: the name of the floating IP to return
640 :return: the SSH client or None
642 fip = self.get_floating_ip(fip_name)
644 return ansible_utils.ssh_client(
645 self.__get_first_provisioning_floating_ip().ip,
646 self.get_image_user(),
647 self.keypair_settings.private_filepath,
648 proxy_settings=self.__os_creds.proxy_settings)
651 'Cannot return an SSH client. No Floating IP configured')
653 def add_security_group(self, security_group):
655 Adds a security group to this VM. Call will block until VM is active.
656 :param security_group: the SNAPS SecurityGroup domain object
657 :return True if successful else False
659 self.vm_active(block=True)
661 if not security_group:
662 logger.warning('Security group object is None, cannot add')
666 nova_utils.add_security_group(self.__nova, self.get_vm_inst(),
669 except NotFound as e:
670 logger.warning('Security group not added - ' + str(e))
673 def remove_security_group(self, security_group):
675 Removes a security group to this VM. Call will block until VM is active
676 :param security_group: the OpenStack security group object
677 :return True if successful else False
679 self.vm_active(block=True)
681 if not security_group:
682 logger.warning('Security group object is None, cannot remove')
686 nova_utils.remove_security_group(self.__nova, self.get_vm_inst(),
689 except NotFound as e:
690 logger.warning('Security group not removed - ' + str(e))
694 class VmInstanceSettings:
696 Class responsible for holding configuration setting for a VM Instance
699 def __init__(self, **kwargs):
702 :param name: the name of the VM
703 :param flavor: the VM's flavor
704 :param port_settings: the port configuration settings (required)
705 :param security_group_names: a set of names of the security groups to
707 :param floating_ip_settings: the floating IP configuration settings
708 :param sudo_user: the sudo user of the VM that will override the
709 instance_settings.image_user when trying to
711 :param vm_boot_timeout: the amount of time a thread will sleep waiting
712 for an instance to boot
713 :param vm_delete_timeout: the amount of time a thread will sleep
714 waiting for an instance to be deleted
715 :param ssh_connect_timeout: the amount of time a thread will sleep
716 waiting obtaining an SSH connection to a VM
717 :param availability_zone: the name of the compute server on which to
718 deploy the VM (optional)
719 :param userdata: the cloud-init script to run after the VM has been
722 self.name = kwargs.get('name')
723 self.flavor = kwargs.get('flavor')
724 self.sudo_user = kwargs.get('sudo_user')
725 self.userdata = kwargs.get('userdata')
727 self.port_settings = list()
728 port_settings = kwargs.get('ports')
729 if not port_settings:
730 port_settings = kwargs.get('port_settings')
732 for port_setting in port_settings:
733 if isinstance(port_setting, dict):
734 self.port_settings.append(PortSettings(**port_setting))
735 elif isinstance(port_setting, PortSettings):
736 self.port_settings.append(port_setting)
738 if kwargs.get('security_group_names'):
739 if isinstance(kwargs['security_group_names'], list):
740 self.security_group_names = kwargs['security_group_names']
741 elif isinstance(kwargs['security_group_names'], set):
742 self.security_group_names = kwargs['security_group_names']
743 elif isinstance(kwargs['security_group_names'], str):
744 self.security_group_names = [kwargs['security_group_names']]
746 raise VmInstanceSettingsError(
747 'Invalid data type for security_group_names attribute')
749 self.security_group_names = set()
751 self.floating_ip_settings = list()
752 floating_ip_settings = kwargs.get('floating_ips')
753 if not floating_ip_settings:
754 floating_ip_settings = kwargs.get('floating_ip_settings')
755 if floating_ip_settings:
756 for floating_ip_config in floating_ip_settings:
757 if isinstance(floating_ip_config, FloatingIpSettings):
758 self.floating_ip_settings.append(floating_ip_config)
760 self.floating_ip_settings.append(FloatingIpSettings(
761 **floating_ip_config['floating_ip']))
763 if kwargs.get('vm_boot_timeout'):
764 self.vm_boot_timeout = kwargs['vm_boot_timeout']
766 self.vm_boot_timeout = 900
768 if kwargs.get('vm_delete_timeout'):
769 self.vm_delete_timeout = kwargs['vm_delete_timeout']
771 self.vm_delete_timeout = 300
773 if kwargs.get('ssh_connect_timeout'):
774 self.ssh_connect_timeout = kwargs['ssh_connect_timeout']
776 self.ssh_connect_timeout = 180
778 if kwargs.get('availability_zone'):
779 self.availability_zone = kwargs['availability_zone']
781 self.availability_zone = None
783 if not self.name or not self.flavor:
784 raise VmInstanceSettingsError(
785 'Instance configuration requires the attributes: name, flavor')
787 if len(self.port_settings) == 0:
788 raise VmInstanceSettingsError(
789 'Instance configuration requires port settings (aka. NICS)')
792 class FloatingIpSettings:
794 Class responsible for holding configuration settings for a floating IP
797 def __init__(self, **kwargs):
800 :param name: the name of the floating IP
801 :param port_name: the name of the router to the external network
802 :param router_name: the name of the router to the external network
803 :param subnet_name: the name of the subnet on which to attach the
805 :param provisioning: when true, this floating IP can be used for
808 TODO - provisioning flag is a hack as I have only observed a single
809 Floating IPs that actually works on an instance. Multiple floating IPs
810 placed on different subnets from the same port are especially
811 troublesome as you cannot predict which one will actually connect.
812 For now, it is recommended not to setup multiple floating IPs on an
813 instance unless absolutely necessary.
815 self.name = kwargs.get('name')
816 self.port_name = kwargs.get('port_name')
817 self.router_name = kwargs.get('router_name')
818 self.subnet_name = kwargs.get('subnet_name')
819 if kwargs.get('provisioning') is not None:
820 self.provisioning = kwargs['provisioning']
822 self.provisioning = True
824 if not self.name or not self.port_name or not self.router_name:
825 raise FloatingIpSettingsError(
826 'The attributes name, port_name and router_name are required '
827 'for FloatingIPSettings')
830 class VmInstanceSettingsError(Exception):
832 Exception to be thrown when an VM instance settings are incorrect
836 class FloatingIpSettingsError(Exception):
838 Exception to be thrown when an VM instance settings are incorrect
842 class VmInstanceCreationError(Exception):
844 Exception to be thrown when an VM instance cannot be created