1 # Copyright (c) 2017 Cable Television Laboratories, Inc. ("CableLabs")
2 # and others. All rights reserved.
4 # Licensed under the Apache License, Version 2.0 (the "License");
5 # you may not use this file except in compliance with the License.
6 # You may obtain a copy of the License at:
8 # http://www.apache.org/licenses/LICENSE-2.0
10 # Unless required by applicable law or agreed to in writing, software
11 # distributed under the License is distributed on an "AS IS" BASIS,
12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 # See the License for the specific language governing permissions and
14 # limitations under the License.
18 from novaclient.exceptions import NotFound
20 from snaps.config.vm_inst import VmInstanceConfig, FloatingIpConfig
21 from snaps.openstack.openstack_creator import OpenStackComputeObject
22 from snaps.openstack.utils import (
23 glance_utils, cinder_utils, settings_utils, keystone_utils)
24 from snaps.openstack.utils import neutron_utils
25 from snaps.openstack.utils import nova_utils
26 from snaps.openstack.utils.nova_utils import RebootType
27 from snaps.provisioning import ansible_utils
29 __author__ = 'spisarski'
31 logger = logging.getLogger('create_instance')
34 STATUS_ACTIVE = 'ACTIVE'
35 STATUS_DELETED = 'DELETED'
38 class OpenStackVmInstance(OpenStackComputeObject):
40 Class responsible for managing a VM instance in OpenStack
43 def __init__(self, os_creds, instance_settings, image_settings,
44 keypair_settings=None):
47 :param os_creds: The connection credentials to the OpenStack API
48 :param instance_settings: Contains the settings for this VM
49 :param image_settings: The OpenStack image object settings
50 :param keypair_settings: The keypair metadata (Optional)
53 super(self.__class__, self).__init__(os_creds)
57 self.instance_settings = instance_settings
58 self.image_settings = image_settings
59 self.keypair_settings = keypair_settings
61 self.__floating_ip_dict = dict()
63 # Instantiated in self.create()
66 # Note: this object does not change after the VM becomes active
71 Loads the existing VMInst, Port, FloatingIps
72 :return: VMInst domain object
74 super(self.__class__, self).initialize()
76 self.__neutron = neutron_utils.neutron_client(
77 self._os_creds, self._os_session)
78 self.__keystone = keystone_utils.keystone_client(
79 self._os_creds, self._os_session)
80 self.__cinder = cinder_utils.cinder_client(
81 self._os_creds, self._os_session)
82 self.__glance = glance_utils.glance_client(
83 self._os_creds, self._os_session)
85 self.__ports = self.__query_ports(self.instance_settings.port_settings)
86 self.__lookup_existing_vm_by_name()
88 def create(self, block=False):
90 Creates a VM instance and associated objects unless they already exist
91 :param block: Thread will block until instance has either become
92 active, error, or timeout waiting.
93 Additionally, when True, floating IPs will not be applied
95 :return: VMInst domain object
99 if len(self.__ports) != len(self.instance_settings.port_settings):
100 self.__ports = self.__create_ports(
101 self.instance_settings.port_settings)
103 self.__create_vm(block)
107 def __lookup_existing_vm_by_name(self):
109 Populates the member variables 'self.vm' and 'self.floating_ips' if a
110 VM with the same name already exists
113 server = nova_utils.get_server(
114 self._nova, self.__neutron, self.__keystone,
115 vm_inst_settings=self.instance_settings)
117 if server.name == self.instance_settings.name:
120 'Found existing machine with name - %s',
121 self.instance_settings.name)
123 fips = neutron_utils.get_port_floating_ips(
124 self.__neutron, self.__ports)
125 for port_id, fip in fips:
126 settings = self.instance_settings.floating_ip_settings
127 for fip_setting in settings:
128 if port_id == fip_setting.port_id:
129 self.__floating_ip_dict[fip_setting.name] = fip
131 port = neutron_utils.get_port_by_id(
132 self.__neutron, port_id)
133 if port and port.name == fip_setting.port_name:
134 self.__floating_ip_dict[fip_setting.name] = fip
136 def __create_vm(self, block=False):
138 Responsible for creating the VM instance
139 :param block: Thread will block until instance has either become
140 active, error, or timeout waiting. Floating IPs will be
141 assigned after active when block=True
143 self.__vm = nova_utils.create_server(
144 self._nova, self.__keystone, self.__neutron, self.__glance,
145 self.instance_settings, self.image_settings,
146 self._os_creds.project_name, self.keypair_settings)
147 logger.info('Created instance with name - %s',
148 self.instance_settings.name)
151 if not self.vm_active(block=True):
152 raise VmInstanceCreationError(
153 'Fatal error, VM did not become ACTIVE within the alloted '
156 # Create server should do this but found it needed to occur here
157 for sec_grp_name in self.instance_settings.security_group_names:
158 if self.vm_active(block=True):
159 nova_utils.add_security_group(self._nova, self.__vm,
162 raise VmInstanceCreationError(
163 'Cannot applying security group with name ' +
165 ' to VM that did not activate with name - ' +
166 self.instance_settings.name)
168 if self.instance_settings.volume_names:
169 for volume_name in self.instance_settings.volume_names:
170 volume = cinder_utils.get_volume(
171 self.__cinder, self.__keystone, volume_name=volume_name,
172 project_name=self._os_creds.project_name)
174 if volume and self.vm_active(block=True):
175 vm = nova_utils.attach_volume(
176 self._nova, self.__neutron, self.__keystone, self.__vm,
177 volume, self._os_creds.project_name)
183 'Volume [%s] attachment timeout ', volume.name)
185 logger.warn('Unable to attach volume named [%s]',
188 self.__apply_floating_ips()
190 def __apply_floating_ips(self):
192 Applies the configured floating IPs to the necessary ports
195 for key, port in self.__ports:
196 port_dict[key] = port
199 for floating_ip_setting in self.instance_settings.floating_ip_settings:
200 self.add_floating_ip(floating_ip_setting)
202 def add_floating_ip(self, floating_ip_setting):
204 Adds a floating IP to a running instance
205 :param floating_ip_setting - the floating IP configuration
206 :return: the floating ip object
209 for key, port in self.__ports:
210 port_dict[key] = port
213 port = port_dict.get(floating_ip_setting.port_name)
216 raise VmInstanceCreationError(
217 'Cannot find port object with name - ' +
218 floating_ip_setting.port_name)
220 # Setup Floating IP only if there is a router with an external
222 ext_gateway = self.__ext_gateway_by_router(
223 floating_ip_setting.router_name)
224 if ext_gateway and self.vm_active(block=True):
225 floating_ip = neutron_utils.create_floating_ip(
226 self.__neutron, self.__keystone, ext_gateway, port.id)
227 self.__floating_ip_dict[floating_ip_setting.name] = floating_ip
230 'Created floating IP %s via router - %s', floating_ip.ip,
231 floating_ip_setting.router_name)
235 raise VmInstanceCreationError(
236 'Unable to add floating IP to port, cannot locate router '
237 'with an external gateway ')
239 def __ext_gateway_by_router(self, router_name):
241 Returns network name for the external network attached to a router or
243 :param router_name: The name of the router to lookup
244 :return: the external network name or None
246 router = neutron_utils.get_router(
247 self.__neutron, self.__keystone, router_name=router_name,
248 project_name=self._os_creds.project_name)
249 if router and router.external_network_id:
250 network = neutron_utils.get_network_by_id(
251 self.__neutron, router.external_network_id)
258 Destroys the VM instance
261 # Cleanup floating IPs
262 for name, floating_ip in self.__floating_ip_dict.items():
263 logger.info('Deleting Floating IP - ' + floating_ip.ip)
264 neutron_utils.delete_floating_ip(self.__neutron, floating_ip)
266 self.__floating_ip_dict = dict()
269 for name, port in self.__ports:
270 logger.info('Deleting Port with ID - %s ', port.id)
271 neutron_utils.delete_port(self.__neutron, port)
273 self.__ports = list()
277 for volume_rec in self.__vm.volume_ids:
278 volume = cinder_utils.get_volume_by_id(
279 self.__cinder, volume_rec['id'])
281 vm = nova_utils.detach_volume(
282 self._nova, self.__neutron, self.__keystone, self.__vm,
283 volume, self._os_creds.project_name)
288 'Timeout waiting to detach volume %s', volume.name)
290 logger.warn('Unable to detach volume with ID - [%s]',
295 'Deleting VM instance - ' + self.instance_settings.name)
298 nova_utils.delete_vm_instance(self._nova, self.__vm)
299 except NotFound as e:
300 logger.warn('Instance already deleted - %s', e)
302 # Block until instance cannot be found or returns the status of
304 logger.info('Checking deletion status')
306 if self.vm_deleted(block=True):
308 'VM has been properly deleted VM with name - %s',
309 self.instance_settings.name)
313 'VM not deleted within the timeout period of %s '
314 'seconds', self.instance_settings.vm_delete_timeout)
316 super(self.__class__, self).clean()
318 def __query_ports(self, port_settings):
320 Returns the previously configured ports or an empty list if none
322 :param port_settings: A list of PortSetting objects
323 :return: a list of OpenStack port tuples where the first member is the
324 port name and the second is the port object
328 for port_setting in port_settings:
329 port = neutron_utils.get_port(
330 self.__neutron, self.__keystone, port_settings=port_setting,
331 project_name=self._os_creds.project_name)
333 ports.append((port_setting.name, port))
337 def __create_ports(self, port_settings):
339 Returns the previously configured ports or creates them if they do not
341 :param port_settings: A list of PortSetting objects
342 :return: a list of OpenStack port tuples where the first member is the
343 port name and the second is the port object
347 for port_setting in port_settings:
348 port = neutron_utils.get_port(
349 self.__neutron, self.__keystone, port_settings=port_setting,
350 project_name=self._os_creds.project_name)
352 port = neutron_utils.create_port(
353 self.__neutron, self._os_creds, port_setting)
355 ports.append((port_setting.name, port))
359 def get_os_creds(self):
361 Returns the OpenStack credentials used to create these objects
362 :return: the credentials
364 return self._os_creds
366 def get_vm_inst(self):
368 Returns the latest version of this server object from OpenStack
369 :return: Server object
371 return nova_utils.get_server_object_by_id(
372 self._nova, self.__neutron, self.__keystone, self.__vm.id,
373 self._os_creds.project_name)
375 def get_console_output(self):
377 Returns the vm console object for parsing logs
378 :return: the console output object
380 return nova_utils.get_server_console_output(self._nova, self.__vm)
382 def get_port_ip(self, port_name, subnet_name=None):
384 Returns the first IP for the port corresponding with the port_name
385 parameter when subnet_name is None else returns the IP address that
386 corresponds to the subnet_name parameter
387 :param port_name: the name of the port from which to return the IP
388 :param subnet_name: the name of the subnet attached to this IP
389 :return: the IP or None if not found
391 port = self.get_port_by_name(port_name)
394 network = neutron_utils.get_network_by_id(
395 self.__neutron, port.network_id)
396 subnet = neutron_utils.get_subnet(
397 self.__neutron, network, subnet_name=subnet_name)
399 logger.warning('Cannot retrieve port IP as subnet could '
400 'not be located with name - %s',
403 for fixed_ip in port.ips:
404 if fixed_ip['subnet_id'] == subnet.id:
405 return fixed_ip['ip_address']
407 if port.ips and len(port.ips) > 0:
408 return port.ips[0]['ip_address']
411 def get_port_mac(self, port_name):
413 Returns the first IP for the port corresponding with the port_name
415 TODO - Add in the subnet as an additional parameter as a port may have
417 :param port_name: the name of the port from which to return the IP
418 :return: the IP or None if not found
420 port = self.get_port_by_name(port_name)
422 return port.mac_address
425 def get_port_by_name(self, port_name):
427 Retrieves the OpenStack port object by its given name
428 :param port_name: the name of the port
429 :return: the OpenStack port object or None if not exists
431 for key, port in self.__ports:
434 logger.warning('Cannot find port with name - ' + port_name)
437 def get_vm_info(self):
439 Returns a dictionary of a VMs info as returned by OpenStack
442 from warnings import warn
443 warn('Do not use the returned dict() structure',
446 return nova_utils.get_server_info(self._nova, self.__vm)
448 def __get_first_provisioning_floating_ip(self):
450 Returns the first floating IP tagged with the Floating IP name if
451 exists else the first one found
454 for floating_ip_setting in self.instance_settings.floating_ip_settings:
455 if floating_ip_setting.provisioning:
456 fip = self.__floating_ip_dict.get(floating_ip_setting.name)
459 elif len(self.__floating_ip_dict) > 0:
460 for key, fip in self.__floating_ip_dict.items():
463 # When cannot be found above
464 if len(self.__floating_ip_dict) > 0:
465 for key, fip in self.__floating_ip_dict.items():
468 def apply_ansible_playbook(self, pb_file_loc, variables=None,
471 Applies a playbook to a VM
472 :param pb_file_loc: the file location of the playbook to be applied
473 :param variables: a dict() of substitution values required by the
475 :param fip_name: the name of the floating IP to use for applying the
476 playbook (default - will take the first)
478 from warnings import warn
479 warn('This method will be removed in a subsequent release',
482 ansible_utils.apply_playbook(
483 pb_file_loc, [self.get_floating_ip(fip_name=fip_name).ip],
484 self.get_image_user(),
485 ssh_priv_key_file_path=self.keypair_settings.private_filepath,
486 variables=variables, proxy_setting=self._os_creds.proxy_settings)
488 def get_image_user(self):
490 Returns the instance sudo_user if it has been configured in the
491 instance_settings else it returns the image_settings.image_user value
493 if self.instance_settings.sudo_user:
494 return self.instance_settings.sudo_user
496 return self.image_settings.image_user
498 def vm_deleted(self, block=False, poll_interval=POLL_INTERVAL):
500 Returns true when the VM status returns the value of
501 expected_status_code or instance retrieval throws a NotFound exception.
502 :param block: When true, thread will block until active or timeout
503 value in seconds has been exceeded (False)
504 :param poll_interval: The polling interval in seconds
508 return self.__vm_status_check(
509 STATUS_DELETED, block,
510 self.instance_settings.vm_delete_timeout, poll_interval)
511 except NotFound as e:
513 "Instance not found when querying status for %s with message "
514 "%s", STATUS_DELETED, e)
517 def vm_active(self, block=False, poll_interval=POLL_INTERVAL):
519 Returns true when the VM status returns the value of the constant
521 :param block: When true, thread will block until active or timeout
522 value in seconds has been exceeded (False)
523 :param poll_interval: The polling interval in seconds
526 if self.__vm_status_check(
527 STATUS_ACTIVE, block, self.instance_settings.vm_boot_timeout,
529 self.__vm = nova_utils.get_server_object_by_id(
530 self._nova, self.__neutron, self.__keystone, self.__vm.id,
531 self._os_creds.project_name)
535 def __vm_status_check(self, expected_status_code, block, timeout,
538 Returns true when the VM status returns the value of
540 :param expected_status_code: instance status evaluated with this
542 :param block: When true, thread will block until active or timeout
543 value in seconds has been exceeded (False)
544 :param timeout: The timeout value
545 :param poll_interval: The polling interval in seconds
548 # sleep and wait for VM status change
552 return self.__status(expected_status_code)
554 while timeout > time.time() - start:
555 status = self.__status(expected_status_code)
557 logger.info('VM is - ' + expected_status_code)
560 logger.debug('Retry querying VM status in ' + str(
561 poll_interval) + ' seconds')
562 time.sleep(poll_interval)
563 logger.debug('VM status query timeout in ' + str(
564 timeout - (time.time() - start)))
567 'Timeout checking for VM status for ' + expected_status_code)
570 def __status(self, expected_status_code):
572 Returns True when active else False
573 :param expected_status_code: instance status evaluated with this string
578 if expected_status_code == STATUS_DELETED:
583 status = nova_utils.get_server_status(self._nova, self.__vm)
585 logger.warning('Cannot find instance with id - ' + self.__vm.id)
588 if status == 'ERROR':
589 raise VmInstanceCreationError(
590 'Instance had an error during deployment')
592 'Instance status [%s] is - %s', self.instance_settings.name,
594 return status == expected_status_code
596 def vm_ssh_active(self, user_override=None, password=None, block=False,
597 timeout=None, poll_interval=POLL_INTERVAL):
599 Returns true when the VM can be accessed via SSH
600 :param user_override: overrides the user with which to create the
602 :param password: overrides the use of a password instead of a private
603 key with which to create the connection
604 :param block: When true, thread will block until active or timeout
605 value in seconds has been exceeded (False)
606 :param timeout: the number of seconds to retry obtaining the connection
607 and overrides the ssh_connect_timeout member of the
608 self.instance_settings object
609 :param poll_interval: The polling interval
612 # sleep and wait for VM status change
613 logger.info('Checking if VM is active')
616 timeout = self.instance_settings.ssh_connect_timeout
618 if self.vm_active(block=True):
622 start = time.time() - timeout
624 while timeout > time.time() - start:
625 status = self.__ssh_active(
626 user_override=user_override, password=password)
628 logger.info('SSH is active for VM instance')
631 logger.debug('Retry SSH connection in ' + str(
632 poll_interval) + ' seconds')
633 time.sleep(poll_interval)
634 logger.debug('SSH connection timeout in ' + str(
635 timeout - (time.time() - start)))
637 logger.error('Timeout attempting to connect with VM via SSH')
640 def __ssh_active(self, user_override=None, password=None):
642 Returns True when can create a SSH session else False
645 if len(self.__floating_ip_dict) > 0:
646 ssh = self.ssh_client(
647 user_override=user_override, password=password)
653 def cloud_init_complete(self, block=False, poll_interval=POLL_INTERVAL):
655 Returns true when the VM's cloud-init routine has completed.
656 Note: this is currently done via SSH, therefore, if this instance does
657 not have a Floating IP or a running SSH server, this routine
658 will always return False or raise an Exception
659 :param block: When true, thread will block until active or timeout
660 value in seconds has been exceeded (False)
661 :param poll_interval: The polling interval
664 # sleep and wait for VM status change
665 logger.info('Checking if cloud-init has completed')
667 timeout = self.instance_settings.cloud_init_timeout
669 if self.vm_active(block=True) and self.vm_ssh_active(block=True):
673 start = time.time() - timeout
675 while timeout > time.time() - start:
676 status = self.__cloud_init_complete()
678 logger.info('cloud-init complete for VM instance')
681 logger.debug('Retry cloud-init query in ' + str(
682 poll_interval) + ' seconds')
683 time.sleep(poll_interval)
684 logger.debug('cloud-init complete timeout in ' + str(
685 timeout - (time.time() - start)))
687 logger.error('Timeout waiting for cloud-init to complete')
690 def __cloud_init_complete(self):
692 Returns True when can create a SSH session else False
695 if len(self.__floating_ip_dict) > 0:
696 ssh = self.ssh_client()
698 stdin1, stdout1, sterr1 = ssh.exec_command(
699 'ls -l /var/lib/cloud/instance/boot-finished')
700 return stdout1.channel.recv_exit_status() == 0
703 def get_floating_ip(self, fip_name=None):
705 Returns the floating IP object byt name if found, else the first known,
707 :param fip_name: the name of the floating IP to return
708 :return: the SSH client or None
710 if fip_name and self.__floating_ip_dict.get(fip_name):
711 return self.__floating_ip_dict.get(fip_name)
713 return self.__get_first_provisioning_floating_ip()
715 def ssh_client(self, fip_name=None, user_override=None, password=None):
717 Returns an SSH client using the name or the first known floating IP if
719 :param fip_name: the name of the floating IP to return
720 :param user_override: the username to use instead of the default
721 :param password: the password to use instead of the private key
722 :return: the SSH client or None
724 fip = self.get_floating_ip(fip_name)
726 ansible_user = self.get_image_user()
728 ansible_user = user_override
733 private_key = self.keypair_settings.private_filepath
736 return ansible_utils.ssh_client(
737 self.__get_first_provisioning_floating_ip().ip,
739 private_key_filepath=private_key,
741 proxy_settings=self._os_creds.proxy_settings)
743 FloatingIPAllocationError(
744 'Cannot return an SSH client. No Floating IP configured')
746 def add_security_group(self, security_group):
748 Adds a security group to this VM. Call will block until VM is active.
749 :param security_group: the SNAPS SecurityGroup domain object
750 :return True if successful else False
752 self.vm_active(block=True)
754 if not security_group:
755 logger.warning('Security group object is None, cannot add')
759 nova_utils.add_security_group(self._nova, self.get_vm_inst(),
762 except NotFound as e:
763 logger.warning('Security group not added - ' + str(e))
766 def remove_security_group(self, security_group):
768 Removes a security group to this VM. Call will block until VM is active
769 :param security_group: the OpenStack security group object
770 :return True if successful else False
772 self.vm_active(block=True)
774 if not security_group:
775 logger.warning('Security group object is None, cannot remove')
779 nova_utils.remove_security_group(self._nova, self.get_vm_inst(),
782 except NotFound as e:
783 logger.warning('Security group not removed - ' + str(e))
786 def reboot(self, reboot_type=RebootType.soft):
789 :param reboot_type: instance of
790 snaps.openstack.utils.nova_utils.RebootType
794 nova_utils.reboot_server(
795 self._nova, self.__vm, reboot_type=reboot_type)
798 def generate_creator(os_creds, vm_inst, image_config, project_name,
799 keypair_config=None):
801 Initializes an OpenStackVmInstance object
802 :param os_creds: the OpenStack credentials
803 :param vm_inst: the SNAPS-OO VmInst domain object
804 :param image_config: the associated ImageConfig object
805 :param project_name: the associated project ID
806 :param keypair_config: the associated KeypairConfig object (optional)
807 :return: an initialized OpenStackVmInstance object
809 session = keystone_utils.keystone_session(os_creds)
810 nova = nova_utils.nova_client(os_creds, session)
811 keystone = keystone_utils.keystone_client(os_creds, session)
812 neutron = neutron_utils.neutron_client(os_creds, session)
815 derived_inst_config = settings_utils.create_vm_inst_config(
816 nova, keystone, neutron, vm_inst, project_name)
818 derived_inst_creator = OpenStackVmInstance(
819 os_creds, derived_inst_config, image_config, keypair_config)
820 derived_inst_creator.initialize()
821 return derived_inst_creator
823 keystone_utils.close_session(session)
826 class VmInstanceSettings(VmInstanceConfig):
828 Deprecated, use snaps.config.vm_inst.VmInstanceConfig instead
830 def __init__(self, **kwargs):
831 from warnings import warn
832 warn('Use snaps.config.vm_inst.VmInstanceConfig instead',
834 super(self.__class__, self).__init__(**kwargs)
837 class FloatingIpSettings(FloatingIpConfig):
839 Deprecated, use snaps.config.vm_inst.FloatingIpConfig instead
841 def __init__(self, **kwargs):
842 from warnings import warn
843 warn('Use snaps.config.vm_inst.FloatingIpConfig instead',
845 super(self.__class__, self).__init__(**kwargs)
848 class VmInstanceCreationError(Exception):
850 Exception to be thrown when an VM instance cannot be created
854 class FloatingIPAllocationError(Exception):
856 Exception to be thrown when an VM instance cannot allocate a floating IP