1 # Copyright (c) 2017 Cable Television Laboratories, Inc. ("CableLabs")
2 # and others. All rights reserved.
4 # Licensed under the Apache License, Version 2.0 (the "License");
5 # you may not use this file except in compliance with the License.
6 # You may obtain a copy of the License at:
8 # http://www.apache.org/licenses/LICENSE-2.0
10 # Unless required by applicable law or agreed to in writing, software
11 # distributed under the License is distributed on an "AS IS" BASIS,
12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 # See the License for the specific language governing permissions and
14 # limitations under the License.
18 from neutronclient.common.exceptions import PortNotFoundClient
19 from novaclient.exceptions import NotFound
21 from snaps.openstack.create_network import PortSettings
22 from snaps.openstack.openstack_creator import OpenStackComputeObject
23 from snaps.openstack.utils import glance_utils
24 from snaps.openstack.utils import neutron_utils
25 from snaps.openstack.utils import nova_utils
26 from snaps.provisioning import ansible_utils
28 __author__ = 'spisarski'
30 logger = logging.getLogger('create_instance')
33 STATUS_ACTIVE = 'ACTIVE'
34 STATUS_DELETED = 'DELETED'
37 class OpenStackVmInstance(OpenStackComputeObject):
39 Class responsible for managing a VM instance in OpenStack
42 def __init__(self, os_creds, instance_settings, image_settings,
43 keypair_settings=None):
46 :param os_creds: The connection credentials to the OpenStack API
47 :param instance_settings: Contains the settings for this VM
48 :param image_settings: The OpenStack image object settings
49 :param keypair_settings: The keypair metadata (Optional)
52 super(self.__class__, self).__init__(os_creds)
56 self.instance_settings = instance_settings
57 self.image_settings = image_settings
58 self.keypair_settings = keypair_settings
60 self.__floating_ip_dict = dict()
62 # Instantiated in self.create()
65 # Note: this object does not change after the VM becomes active
70 Loads the existing VMInst, Port, FloatingIps
71 :return: VMInst domain object
73 super(self.__class__, self).initialize()
75 self.__neutron = neutron_utils.neutron_client(self._os_creds)
77 self.__ports = self.__query_ports(self.instance_settings.port_settings)
78 self.__lookup_existing_vm_by_name()
80 def create(self, block=False):
82 Creates a VM instance and associated objects unless they already exist
83 :param block: Thread will block until instance has either become
84 active, error, or timeout waiting.
85 Additionally, when True, floating IPs will not be applied
87 :return: VMInst domain object
91 if len(self.__ports) == 0:
92 self.__ports = self.__create_ports(self.instance_settings.port_settings)
94 self.__create_vm(block)
98 def __lookup_existing_vm_by_name(self):
100 Populates the member variables 'self.vm' and 'self.floating_ips' if a
101 VM with the same name already exists
104 server = nova_utils.get_server(
105 self._nova, vm_inst_settings=self.instance_settings)
107 if server.name == self.instance_settings.name:
110 'Found existing machine with name - %s',
111 self.instance_settings.name)
113 fips = neutron_utils.get_floating_ips(self.__neutron,
115 for port_id, fip in fips:
116 settings = self.instance_settings.floating_ip_settings
117 for fip_setting in settings:
118 if port_id == fip_setting.port_id:
119 self.__floating_ip_dict[fip_setting.name] = fip
121 port = neutron_utils.get_port_by_id(
122 self.__neutron, port_id)
123 if port and port.name == fip_setting.port_name:
124 self.__floating_ip_dict[fip_setting.name] = fip
126 def __create_vm(self, block=False):
128 Responsible for creating the VM instance
129 :param block: Thread will block until instance has either become
130 active, error, or timeout waiting. Floating IPs will be
131 assigned after active when block=True
133 glance = glance_utils.glance_client(self._os_creds)
134 self.__vm = nova_utils.create_server(
135 self._nova, self.__neutron, glance, self.instance_settings,
136 self.image_settings, self.keypair_settings)
137 logger.info('Created instance with name - %s',
138 self.instance_settings.name)
141 if not self.vm_active(block=True):
142 raise VmInstanceCreationError(
143 'Fatal error, VM did not become ACTIVE within the alloted '
146 # Create server should do this but found it needed to occur here
147 for sec_grp_name in self.instance_settings.security_group_names:
148 if self.vm_active(block=True):
149 nova_utils.add_security_group(self._nova, self.__vm,
152 raise VmInstanceCreationError(
153 'Cannot applying security group with name ' +
155 ' to VM that did not activate with name - ' +
156 self.instance_settings.name)
158 self.__apply_floating_ips()
160 def __apply_floating_ips(self):
162 Applies the configured floating IPs to the necessary ports
165 for key, port in self.__ports:
166 port_dict[key] = port
169 for floating_ip_setting in self.instance_settings.floating_ip_settings:
170 port = port_dict.get(floating_ip_setting.port_name)
173 raise VmInstanceCreationError(
174 'Cannot find port object with name - ' +
175 floating_ip_setting.port_name)
177 # Setup Floating IP only if there is a router with an external
179 ext_gateway = self.__ext_gateway_by_router(
180 floating_ip_setting.router_name)
182 subnet = neutron_utils.get_subnet(
184 subnet_name=floating_ip_setting.subnet_name)
185 floating_ip = neutron_utils.create_floating_ip(
186 self.__neutron, ext_gateway)
187 self.__floating_ip_dict[floating_ip_setting.name] = floating_ip
190 'Created floating IP %s via router - %s', floating_ip.ip,
191 floating_ip_setting.router_name)
192 self.__add_floating_ip(floating_ip, port, subnet)
194 raise VmInstanceCreationError(
195 'Unable to add floating IP to port, cannot locate router '
196 'with an external gateway ')
198 def __ext_gateway_by_router(self, router_name):
200 Returns network name for the external network attached to a router or
202 :param router_name: The name of the router to lookup
203 :return: the external network name or None
205 router = neutron_utils.get_router(
206 self.__neutron, router_name=router_name)
207 if router and router.external_gateway_info:
208 network = neutron_utils.get_network_by_id(
210 router.external_gateway_info['network_id'])
217 Destroys the VM instance
220 # Cleanup floating IPs
221 for name, floating_ip in self.__floating_ip_dict.items():
223 logger.info('Deleting Floating IP - ' + floating_ip.ip)
224 neutron_utils.delete_floating_ip(self.__neutron, floating_ip)
225 except Exception as e:
226 logger.error('Error deleting Floating IP - ' + str(e))
227 self.__floating_ip_dict = dict()
230 for name, port in self.__ports:
231 logger.info('Deleting Port with ID - %S ' + port.id)
233 neutron_utils.delete_port(self.__neutron, port)
234 except PortNotFoundClient as e:
235 logger.warning('Unexpected error deleting port - %s', e)
237 self.__ports = list()
243 'Deleting VM instance - ' + self.instance_settings.name)
244 nova_utils.delete_vm_instance(self._nova, self.__vm)
245 except Exception as e:
246 logger.error('Error deleting VM - %s', e)
248 # Block until instance cannot be found or returns the status of
250 logger.info('Checking deletion status')
253 if self.vm_deleted(block=True):
255 'VM has been properly deleted VM with name - %s',
256 self.instance_settings.name)
260 'VM not deleted within the timeout period of %s '
261 'seconds', self.instance_settings.vm_delete_timeout)
262 except Exception as e:
264 'Unexpected error while checking VM instance status - %s',
267 def __query_ports(self, port_settings):
269 Returns the previously configured ports or an empty list if none
271 :param port_settings: A list of PortSetting objects
272 :return: a list of OpenStack port tuples where the first member is the
273 port name and the second is the port object
277 for port_setting in port_settings:
278 port = neutron_utils.get_port(
279 self.__neutron, port_settings=port_setting)
281 ports.append((port_setting.name, port))
285 def __create_ports(self, port_settings):
287 Returns the previously configured ports or creates them if they do not
289 :param port_settings: A list of PortSetting objects
290 :return: a list of OpenStack port tuples where the first member is the
291 port name and the second is the port object
295 for port_setting in port_settings:
296 port = neutron_utils.get_port(
297 self.__neutron, port_settings=port_setting)
299 port = neutron_utils.create_port(self.__neutron, self._os_creds, port_setting)
301 ports.append((port_setting.name, port))
305 def __add_floating_ip(self, floating_ip, port, subnet, timeout=30,
306 poll_interval=POLL_INTERVAL):
308 Returns True when active else False
309 TODO - Make timeout and poll_interval configurable...
314 # Take IP of subnet if there is one configured on which to place
316 for fixed_ip in port.ips:
317 if fixed_ip['subnet_id'] == subnet.id:
318 ip = fixed_ip['ip_address']
321 # Simply take the first
322 ip = port.ips[0]['ip_address']
325 count = timeout / poll_interval
327 logger.debug('Attempting to add floating IP to instance')
329 nova_utils.add_floating_ip_to_server(
330 self._nova, self.__vm, floating_ip, ip)
332 'Added floating IP %s to port IP %s on instance %s',
333 floating_ip.ip, ip, self.instance_settings.name)
335 except Exception as e:
337 'Retry adding floating IP to instance. Last attempt '
338 'failed with - %s', e)
339 time.sleep(poll_interval)
343 raise VmInstanceCreationError(
344 'Unable find IP address on which to place the floating IP')
346 logger.error('Timeout attempting to add the floating IP to instance.')
347 raise VmInstanceCreationError(
348 'Timeout while attempting add floating IP to instance')
350 def get_os_creds(self):
352 Returns the OpenStack credentials used to create these objects
353 :return: the credentials
355 return self._os_creds
357 def get_vm_inst(self):
359 Returns the latest version of this server object from OpenStack
360 :return: Server object
362 return nova_utils.get_server_object_by_id(self._nova, self.__vm.id)
364 def get_console_output(self):
366 Returns the vm console object for parsing logs
367 :return: the console output object
369 return nova_utils.get_server_console_output(self._nova, self.__vm)
371 def get_port_ip(self, port_name, subnet_name=None):
373 Returns the first IP for the port corresponding with the port_name
374 parameter when subnet_name is None else returns the IP address that
375 corresponds to the subnet_name parameter
376 :param port_name: the name of the port from which to return the IP
377 :param subnet_name: the name of the subnet attached to this IP
378 :return: the IP or None if not found
380 port = self.get_port_by_name(port_name)
383 subnet = neutron_utils.get_subnet(
384 self.__neutron, subnet_name=subnet_name)
386 logger.warning('Cannot retrieve port IP as subnet could '
387 'not be located with name - %s',
390 for fixed_ip in port.ips:
391 if fixed_ip['subnet_id'] == subnet.id:
392 return fixed_ip['ip_address']
394 if port.ips and len(port.ips) > 0:
395 return port.ips[0]['ip_address']
398 def get_port_mac(self, port_name):
400 Returns the first IP for the port corresponding with the port_name
402 TODO - Add in the subnet as an additional parameter as a port may have
404 :param port_name: the name of the port from which to return the IP
405 :return: the IP or None if not found
407 port = self.get_port_by_name(port_name)
409 return port.mac_address
412 def get_port_by_name(self, port_name):
414 Retrieves the OpenStack port object by its given name
415 :param port_name: the name of the port
416 :return: the OpenStack port object or None if not exists
418 for key, port in self.__ports:
421 logger.warning('Cannot find port with name - ' + port_name)
424 def get_vm_info(self):
426 Returns a dictionary of a VMs info as returned by OpenStack
429 return nova_utils.get_server_info(self._nova, self.__vm)
431 def config_nics(self):
433 Responsible for configuring NICs on RPM systems where the instance has
434 more than one configured port
435 :return: the value returned by ansible_utils.apply_ansible_playbook()
437 if len(self.__ports) > 1 and len(self.__floating_ip_dict) > 0:
438 if self.vm_active(block=True) and self.vm_ssh_active(block=True):
439 for key, port in self.__ports:
440 port_index = self.__ports.index((key, port))
442 nic_name = 'eth' + repr(port_index)
443 retval = self.__config_nic(
445 self.__get_first_provisioning_floating_ip().ip)
446 logger.info('Configured NIC - %s on VM - %s',
447 nic_name, self.instance_settings.name)
450 def __get_first_provisioning_floating_ip(self):
452 Returns the first floating IP tagged with the Floating IP name if
453 exists else the first one found
456 for floating_ip_setting in self.instance_settings.floating_ip_settings:
457 if floating_ip_setting.provisioning:
458 fip = self.__floating_ip_dict.get(floating_ip_setting.name)
461 elif len(self.__floating_ip_dict) > 0:
462 for key, fip in self.__floating_ip_dict.items():
465 def __config_nic(self, nic_name, port, ip):
467 Although ports/NICs can contain multiple IPs, this code currently only
470 :param nic_name: Name of the interface
471 :param port: The port information containing the expected IP values.
472 :param ip: The IP on which to apply the playbook.
473 :return: the return value from ansible
475 port_ip = port.ips[0]['ip_address']
478 'nic_name': nic_name,
482 if self.image_settings.nic_config_pb_loc and self.keypair_settings:
483 return self.apply_ansible_playbook(
484 self.image_settings.nic_config_pb_loc, variables)
487 'VM %s cannot self configure NICs eth1++. No playbook or '
488 'keypairs found.', self.instance_settings.name)
490 def apply_ansible_playbook(self, pb_file_loc, variables=None,
493 Applies a playbook to a VM
494 :param pb_file_loc: the file location of the playbook to be applied
495 :param variables: a dict() of substitution values required by the
497 :param fip_name: the name of the floating IP to use for applying the
498 playbook (default - will take the first)
499 :return: the return value from ansible
501 return ansible_utils.apply_playbook(
502 pb_file_loc, [self.get_floating_ip(fip_name=fip_name).ip],
503 self.get_image_user(), self.keypair_settings.private_filepath,
504 variables, self._os_creds.proxy_settings)
506 def get_image_user(self):
508 Returns the instance sudo_user if it has been configured in the
509 instance_settings else it returns the image_settings.image_user value
511 if self.instance_settings.sudo_user:
512 return self.instance_settings.sudo_user
514 return self.image_settings.image_user
516 def vm_deleted(self, block=False, poll_interval=POLL_INTERVAL):
518 Returns true when the VM status returns the value of
519 expected_status_code or instance retrieval throws a NotFound exception.
520 :param block: When true, thread will block until active or timeout
521 value in seconds has been exceeded (False)
522 :param poll_interval: The polling interval in seconds
526 return self.__vm_status_check(
527 STATUS_DELETED, block,
528 self.instance_settings.vm_delete_timeout, poll_interval)
529 except NotFound as e:
531 "Instance not found when querying status for %s with message "
532 "%s", STATUS_DELETED, e)
535 def vm_active(self, block=False, poll_interval=POLL_INTERVAL):
537 Returns true when the VM status returns the value of the constant
539 :param block: When true, thread will block until active or timeout
540 value in seconds has been exceeded (False)
541 :param poll_interval: The polling interval in seconds
544 return self.__vm_status_check(STATUS_ACTIVE, block,
545 self.instance_settings.vm_boot_timeout,
548 def __vm_status_check(self, expected_status_code, block, timeout,
551 Returns true when the VM status returns the value of
553 :param expected_status_code: instance status evaluated with this
555 :param block: When true, thread will block until active or timeout
556 value in seconds has been exceeded (False)
557 :param timeout: The timeout value
558 :param poll_interval: The polling interval in seconds
561 # sleep and wait for VM status change
565 return self.__status(expected_status_code)
567 while timeout > time.time() - start:
568 status = self.__status(expected_status_code)
570 logger.info('VM is - ' + expected_status_code)
573 logger.debug('Retry querying VM status in ' + str(
574 poll_interval) + ' seconds')
575 time.sleep(poll_interval)
576 logger.debug('VM status query timeout in ' + str(
577 timeout - (time.time() - start)))
580 'Timeout checking for VM status for ' + expected_status_code)
583 def __status(self, expected_status_code):
585 Returns True when active else False
586 :param expected_status_code: instance status evaluated with this string
591 if expected_status_code == STATUS_DELETED:
596 status = nova_utils.get_server_status(self._nova, self.__vm)
598 logger.warning('Cannot find instance with id - ' + self.__vm.id)
601 if status == 'ERROR':
602 raise VmInstanceCreationError(
603 'Instance had an error during deployment')
605 'Instance status [%s] is - %s', self.instance_settings.name,
607 return status == expected_status_code
609 def vm_ssh_active(self, block=False, poll_interval=POLL_INTERVAL):
611 Returns true when the VM can be accessed via SSH
612 :param block: When true, thread will block until active or timeout
613 value in seconds has been exceeded (False)
614 :param poll_interval: The polling interval
617 # sleep and wait for VM status change
618 logger.info('Checking if VM is active')
620 timeout = self.instance_settings.ssh_connect_timeout
622 if self.vm_active(block=True):
626 start = time.time() - timeout
628 while timeout > time.time() - start:
629 status = self.__ssh_active()
631 logger.info('SSH is active for VM instance')
634 logger.debug('Retry SSH connection in ' + str(
635 poll_interval) + ' seconds')
636 time.sleep(poll_interval)
637 logger.debug('SSH connection timeout in ' + str(
638 timeout - (time.time() - start)))
640 logger.error('Timeout attempting to connect with VM via SSH')
643 def __ssh_active(self):
645 Returns True when can create a SSH session else False
648 if len(self.__floating_ip_dict) > 0:
649 ssh = self.ssh_client()
655 def get_floating_ip(self, fip_name=None):
657 Returns the floating IP object byt name if found, else the first known,
659 :param fip_name: the name of the floating IP to return
660 :return: the SSH client or None
663 if fip_name and self.__floating_ip_dict.get(fip_name):
664 return self.__floating_ip_dict.get(fip_name)
666 return self.__get_first_provisioning_floating_ip()
668 def ssh_client(self, fip_name=None):
670 Returns an SSH client using the name or the first known floating IP if
672 :param fip_name: the name of the floating IP to return
673 :return: the SSH client or None
675 fip = self.get_floating_ip(fip_name)
677 return ansible_utils.ssh_client(
678 self.__get_first_provisioning_floating_ip().ip,
679 self.get_image_user(),
680 self.keypair_settings.private_filepath,
681 proxy_settings=self._os_creds.proxy_settings)
684 'Cannot return an SSH client. No Floating IP configured')
686 def add_security_group(self, security_group):
688 Adds a security group to this VM. Call will block until VM is active.
689 :param security_group: the SNAPS SecurityGroup domain object
690 :return True if successful else False
692 self.vm_active(block=True)
694 if not security_group:
695 logger.warning('Security group object is None, cannot add')
699 nova_utils.add_security_group(self._nova, self.get_vm_inst(),
702 except NotFound as e:
703 logger.warning('Security group not added - ' + str(e))
706 def remove_security_group(self, security_group):
708 Removes a security group to this VM. Call will block until VM is active
709 :param security_group: the OpenStack security group object
710 :return True if successful else False
712 self.vm_active(block=True)
714 if not security_group:
715 logger.warning('Security group object is None, cannot remove')
719 nova_utils.remove_security_group(self._nova, self.get_vm_inst(),
722 except NotFound as e:
723 logger.warning('Security group not removed - ' + str(e))
727 class VmInstanceSettings:
729 Class responsible for holding configuration setting for a VM Instance
732 def __init__(self, **kwargs):
735 :param name: the name of the VM
736 :param flavor: the VM's flavor name
737 :param port_settings: the port configuration settings (required)
738 :param security_group_names: a set of names of the security groups to
740 :param floating_ip_settings: the floating IP configuration settings
741 :param sudo_user: the sudo user of the VM that will override the
742 instance_settings.image_user when trying to
744 :param vm_boot_timeout: the amount of time a thread will sleep waiting
745 for an instance to boot
746 :param vm_delete_timeout: the amount of time a thread will sleep
747 waiting for an instance to be deleted
748 :param ssh_connect_timeout: the amount of time a thread will sleep
749 waiting obtaining an SSH connection to a VM
750 :param availability_zone: the name of the compute server on which to
751 deploy the VM (optional)
752 :param userdata: the string contents of any optional cloud-init script
753 to execute after the VM has been activated.
754 This value may also contain a dict who's key value
755 must contain the key 'cloud-init_file' which denotes
756 the location of some file containing the cloud-init
759 self.name = kwargs.get('name')
760 self.flavor = kwargs.get('flavor')
761 self.sudo_user = kwargs.get('sudo_user')
762 self.userdata = kwargs.get('userdata')
764 self.port_settings = list()
765 port_settings = kwargs.get('ports')
766 if not port_settings:
767 port_settings = kwargs.get('port_settings')
769 for port_setting in port_settings:
770 if isinstance(port_setting, dict):
771 self.port_settings.append(PortSettings(**port_setting))
772 elif isinstance(port_setting, PortSettings):
773 self.port_settings.append(port_setting)
775 if kwargs.get('security_group_names'):
776 if isinstance(kwargs['security_group_names'], list):
777 self.security_group_names = kwargs['security_group_names']
778 elif isinstance(kwargs['security_group_names'], set):
779 self.security_group_names = kwargs['security_group_names']
780 elif isinstance(kwargs['security_group_names'], str):
781 self.security_group_names = [kwargs['security_group_names']]
783 raise VmInstanceSettingsError(
784 'Invalid data type for security_group_names attribute')
786 self.security_group_names = set()
788 self.floating_ip_settings = list()
789 floating_ip_settings = kwargs.get('floating_ips')
790 if not floating_ip_settings:
791 floating_ip_settings = kwargs.get('floating_ip_settings')
792 if floating_ip_settings:
793 for floating_ip_config in floating_ip_settings:
794 if isinstance(floating_ip_config, FloatingIpSettings):
795 self.floating_ip_settings.append(floating_ip_config)
797 self.floating_ip_settings.append(FloatingIpSettings(
798 **floating_ip_config['floating_ip']))
800 if kwargs.get('vm_boot_timeout'):
801 self.vm_boot_timeout = kwargs['vm_boot_timeout']
803 self.vm_boot_timeout = 900
805 if kwargs.get('vm_delete_timeout'):
806 self.vm_delete_timeout = kwargs['vm_delete_timeout']
808 self.vm_delete_timeout = 300
810 if kwargs.get('ssh_connect_timeout'):
811 self.ssh_connect_timeout = kwargs['ssh_connect_timeout']
813 self.ssh_connect_timeout = 180
815 if kwargs.get('availability_zone'):
816 self.availability_zone = kwargs['availability_zone']
818 self.availability_zone = None
820 if not self.name or not self.flavor:
821 raise VmInstanceSettingsError(
822 'Instance configuration requires the attributes: name, flavor')
824 if len(self.port_settings) == 0:
825 raise VmInstanceSettingsError(
826 'Instance configuration requires port settings (aka. NICS)')
829 class FloatingIpSettings:
831 Class responsible for holding configuration settings for a floating IP
834 def __init__(self, **kwargs):
837 :param name: the name of the floating IP
838 :param port_name: the name of the router to the external network
839 :param router_name: the name of the router to the external network
840 :param subnet_name: the name of the subnet on which to attach the
842 :param provisioning: when true, this floating IP can be used for
845 TODO - provisioning flag is a hack as I have only observed a single
846 Floating IPs that actually works on an instance. Multiple floating IPs
847 placed on different subnets from the same port are especially
848 troublesome as you cannot predict which one will actually connect.
849 For now, it is recommended not to setup multiple floating IPs on an
850 instance unless absolutely necessary.
852 self.name = kwargs.get('name')
853 self.port_name = kwargs.get('port_name')
854 self.port_id = kwargs.get('port_id')
855 self.router_name = kwargs.get('router_name')
856 self.subnet_name = kwargs.get('subnet_name')
857 if kwargs.get('provisioning') is not None:
858 self.provisioning = kwargs['provisioning']
860 self.provisioning = True
862 # if not self.name or not self.port_name or not self.router_name:
863 if not self.name or not self.router_name:
864 raise FloatingIpSettingsError(
865 'The attributes name, port_name and router_name are required')
867 if not self.port_name and not self.port_id:
868 raise FloatingIpSettingsError(
869 'The attributes port_name or port_id are required')
872 class VmInstanceSettingsError(Exception):
874 Exception to be thrown when an VM instance settings are incorrect
878 class FloatingIpSettingsError(Exception):
880 Exception to be thrown when an VM instance settings are incorrect
884 class VmInstanceCreationError(Exception):
886 Exception to be thrown when an VM instance cannot be created