1 # Copyright (c) 2017 Cable Television Laboratories, Inc. ("CableLabs")
2 # and others. All rights reserved.
4 # Licensed under the Apache License, Version 2.0 (the "License");
5 # you may not use this file except in compliance with the License.
6 # You may obtain a copy of the License at:
8 # http://www.apache.org/licenses/LICENSE-2.0
10 # Unless required by applicable law or agreed to in writing, software
11 # distributed under the License is distributed on an "AS IS" BASIS,
12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 # See the License for the specific language governing permissions and
14 # limitations under the License.
18 from novaclient.exceptions import NotFound, BadRequest
20 from snaps.config.vm_inst import VmInstanceConfig, FloatingIpConfig
21 from snaps.openstack.openstack_creator import OpenStackComputeObject
22 from snaps.openstack.utils import glance_utils, cinder_utils, settings_utils
23 from snaps.openstack.utils import neutron_utils
24 from snaps.openstack.utils import nova_utils
25 from snaps.openstack.utils.nova_utils import RebootType
26 from snaps.provisioning import ansible_utils
28 __author__ = 'spisarski'
30 logger = logging.getLogger('create_instance')
33 STATUS_ACTIVE = 'ACTIVE'
34 STATUS_DELETED = 'DELETED'
37 class OpenStackVmInstance(OpenStackComputeObject):
39 Class responsible for managing a VM instance in OpenStack
42 def __init__(self, os_creds, instance_settings, image_settings,
43 keypair_settings=None):
46 :param os_creds: The connection credentials to the OpenStack API
47 :param instance_settings: Contains the settings for this VM
48 :param image_settings: The OpenStack image object settings
49 :param keypair_settings: The keypair metadata (Optional)
52 super(self.__class__, self).__init__(os_creds)
56 self.instance_settings = instance_settings
57 self.image_settings = image_settings
58 self.keypair_settings = keypair_settings
60 self.__floating_ip_dict = dict()
62 # Instantiated in self.create()
65 # Note: this object does not change after the VM becomes active
70 Loads the existing VMInst, Port, FloatingIps
71 :return: VMInst domain object
73 super(self.__class__, self).initialize()
75 self.__neutron = neutron_utils.neutron_client(self._os_creds)
77 self.__ports = self.__query_ports(self.instance_settings.port_settings)
78 self.__lookup_existing_vm_by_name()
80 def create(self, block=False):
82 Creates a VM instance and associated objects unless they already exist
83 :param block: Thread will block until instance has either become
84 active, error, or timeout waiting.
85 Additionally, when True, floating IPs will not be applied
87 :return: VMInst domain object
91 if len(self.__ports) == 0:
92 self.__ports = self.__create_ports(
93 self.instance_settings.port_settings)
95 self.__create_vm(block)
99 def __lookup_existing_vm_by_name(self):
101 Populates the member variables 'self.vm' and 'self.floating_ips' if a
102 VM with the same name already exists
105 server = nova_utils.get_server(
106 self._nova, self.__neutron,
107 vm_inst_settings=self.instance_settings)
109 if server.name == self.instance_settings.name:
112 'Found existing machine with name - %s',
113 self.instance_settings.name)
115 fips = neutron_utils.get_floating_ips(self.__neutron,
117 for port_id, fip in fips:
118 settings = self.instance_settings.floating_ip_settings
119 for fip_setting in settings:
120 if port_id == fip_setting.port_id:
121 self.__floating_ip_dict[fip_setting.name] = fip
123 port = neutron_utils.get_port_by_id(
124 self.__neutron, port_id)
125 if port and port.name == fip_setting.port_name:
126 self.__floating_ip_dict[fip_setting.name] = fip
128 def __create_vm(self, block=False):
130 Responsible for creating the VM instance
131 :param block: Thread will block until instance has either become
132 active, error, or timeout waiting. Floating IPs will be
133 assigned after active when block=True
135 glance = glance_utils.glance_client(self._os_creds)
136 self.__vm = nova_utils.create_server(
137 self._nova, self.__neutron, glance, self.instance_settings,
138 self.image_settings, self.keypair_settings)
139 logger.info('Created instance with name - %s',
140 self.instance_settings.name)
143 if not self.vm_active(block=True):
144 raise VmInstanceCreationError(
145 'Fatal error, VM did not become ACTIVE within the alloted '
148 # Create server should do this but found it needed to occur here
149 for sec_grp_name in self.instance_settings.security_group_names:
150 if self.vm_active(block=True):
151 nova_utils.add_security_group(self._nova, self.__vm,
154 raise VmInstanceCreationError(
155 'Cannot applying security group with name ' +
157 ' to VM that did not activate with name - ' +
158 self.instance_settings.name)
160 if self.instance_settings.volume_names:
161 for volume_name in self.instance_settings.volume_names:
162 cinder = cinder_utils.cinder_client(self._os_creds)
163 volume = cinder_utils.get_volume(
164 cinder, volume_name=volume_name)
166 if volume and self.vm_active(block=True):
168 vm = nova_utils.attach_volume(
169 self._nova, self.__neutron, self.__vm, volume, timeout)
174 logger.warn('Volume [%s] not attached within timeout '
175 'of [%s]', volume.name, timeout)
177 logger.warn('Unable to attach volume named [%s]',
180 self.__apply_floating_ips()
182 def __apply_floating_ips(self):
184 Applies the configured floating IPs to the necessary ports
187 for key, port in self.__ports:
188 port_dict[key] = port
191 for floating_ip_setting in self.instance_settings.floating_ip_settings:
192 self.add_floating_ip(floating_ip_setting)
194 def add_floating_ip(self, floating_ip_setting):
196 Adds a floating IP to a running instance
197 :param floating_ip_setting - the floating IP configuration
198 :return: the floating ip object
201 for key, port in self.__ports:
202 port_dict[key] = port
205 port = port_dict.get(floating_ip_setting.port_name)
208 raise VmInstanceCreationError(
209 'Cannot find port object with name - ' +
210 floating_ip_setting.port_name)
212 # Setup Floating IP only if there is a router with an external
214 ext_gateway = self.__ext_gateway_by_router(
215 floating_ip_setting.router_name)
217 subnet = neutron_utils.get_subnet(
219 subnet_name=floating_ip_setting.subnet_name)
220 floating_ip = neutron_utils.create_floating_ip(
221 self.__neutron, ext_gateway)
222 self.__floating_ip_dict[floating_ip_setting.name] = floating_ip
225 'Created floating IP %s via router - %s', floating_ip.ip,
226 floating_ip_setting.router_name)
227 self.__add_floating_ip(floating_ip, port, subnet)
230 raise VmInstanceCreationError(
231 'Unable to add floating IP to port, cannot locate router '
232 'with an external gateway ')
234 def __ext_gateway_by_router(self, router_name):
236 Returns network name for the external network attached to a router or
238 :param router_name: The name of the router to lookup
239 :return: the external network name or None
241 router = neutron_utils.get_router(
242 self.__neutron, router_name=router_name)
243 if router and router.external_network_id:
244 network = neutron_utils.get_network_by_id(
245 self.__neutron, router.external_network_id)
252 Destroys the VM instance
255 # Cleanup floating IPs
256 for name, floating_ip in self.__floating_ip_dict.items():
257 logger.info('Deleting Floating IP - ' + floating_ip.ip)
258 neutron_utils.delete_floating_ip(self.__neutron, floating_ip)
260 self.__floating_ip_dict = dict()
263 for name, port in self.__ports:
264 logger.info('Deleting Port with ID - %s ', port.id)
265 neutron_utils.delete_port(self.__neutron, port)
267 self.__ports = list()
271 for volume_rec in self.__vm.volume_ids:
272 cinder = cinder_utils.cinder_client(self._os_creds)
273 volume = cinder_utils.get_volume_by_id(
274 cinder, volume_rec['id'])
276 vm = nova_utils.detach_volume(
277 self._nova, self.__neutron, self.__vm, volume, 30)
282 'Timeout waiting to detach volume %s', volume.name)
284 logger.warn('Unable to detach volume with ID - [%s]',
289 'Deleting VM instance - ' + self.instance_settings.name)
292 nova_utils.delete_vm_instance(self._nova, self.__vm)
293 except NotFound as e:
294 logger.warn('Instance already deleted - %s', e)
296 # Block until instance cannot be found or returns the status of
298 logger.info('Checking deletion status')
300 if self.vm_deleted(block=True):
302 'VM has been properly deleted VM with name - %s',
303 self.instance_settings.name)
307 'VM not deleted within the timeout period of %s '
308 'seconds', self.instance_settings.vm_delete_timeout)
310 def __query_ports(self, port_settings):
312 Returns the previously configured ports or an empty list if none
314 :param port_settings: A list of PortSetting objects
315 :return: a list of OpenStack port tuples where the first member is the
316 port name and the second is the port object
320 for port_setting in port_settings:
321 port = neutron_utils.get_port(
322 self.__neutron, port_settings=port_setting)
324 ports.append((port_setting.name, port))
328 def __create_ports(self, port_settings):
330 Returns the previously configured ports or creates them if they do not
332 :param port_settings: A list of PortSetting objects
333 :return: a list of OpenStack port tuples where the first member is the
334 port name and the second is the port object
338 for port_setting in port_settings:
339 port = neutron_utils.get_port(
340 self.__neutron, port_settings=port_setting)
342 port = neutron_utils.create_port(
343 self.__neutron, self._os_creds, port_setting)
345 ports.append((port_setting.name, port))
349 def __add_floating_ip(self, floating_ip, port, subnet, timeout=30,
350 poll_interval=POLL_INTERVAL):
352 Returns True when active else False
353 TODO - Make timeout and poll_interval configurable...
358 # Take IP of subnet if there is one configured on which to place
360 for fixed_ip in port.ips:
361 if fixed_ip['subnet_id'] == subnet.id:
362 ip = fixed_ip['ip_address']
365 # Simply take the first
366 ip = port.ips[0]['ip_address']
369 count = timeout / poll_interval
371 logger.debug('Attempting to add floating IP to instance')
373 nova_utils.add_floating_ip_to_server(
374 self._nova, self.__vm, floating_ip, ip)
376 'Added floating IP %s to port IP %s on instance %s',
377 floating_ip.ip, ip, self.instance_settings.name)
379 except BadRequest as bre:
380 logger.error('Cannot add floating IP [%s]', bre)
382 except Exception as e:
384 'Retry adding floating IP to instance. Last attempt '
385 'failed with - %s', e)
386 time.sleep(poll_interval)
390 raise VmInstanceCreationError(
391 'Unable find IP address on which to place the floating IP')
393 logger.error('Timeout attempting to add the floating IP to instance.')
394 raise VmInstanceCreationError(
395 'Timeout while attempting add floating IP to instance')
397 def get_os_creds(self):
399 Returns the OpenStack credentials used to create these objects
400 :return: the credentials
402 return self._os_creds
404 def get_vm_inst(self):
406 Returns the latest version of this server object from OpenStack
407 :return: Server object
409 return nova_utils.get_server_object_by_id(
410 self._nova, self.__neutron, self.__vm.id)
412 def get_console_output(self):
414 Returns the vm console object for parsing logs
415 :return: the console output object
417 return nova_utils.get_server_console_output(self._nova, self.__vm)
419 def get_port_ip(self, port_name, subnet_name=None):
421 Returns the first IP for the port corresponding with the port_name
422 parameter when subnet_name is None else returns the IP address that
423 corresponds to the subnet_name parameter
424 :param port_name: the name of the port from which to return the IP
425 :param subnet_name: the name of the subnet attached to this IP
426 :return: the IP or None if not found
428 port = self.get_port_by_name(port_name)
431 subnet = neutron_utils.get_subnet(
432 self.__neutron, subnet_name=subnet_name)
434 logger.warning('Cannot retrieve port IP as subnet could '
435 'not be located with name - %s',
438 for fixed_ip in port.ips:
439 if fixed_ip['subnet_id'] == subnet.id:
440 return fixed_ip['ip_address']
442 if port.ips and len(port.ips) > 0:
443 return port.ips[0]['ip_address']
446 def get_port_mac(self, port_name):
448 Returns the first IP for the port corresponding with the port_name
450 TODO - Add in the subnet as an additional parameter as a port may have
452 :param port_name: the name of the port from which to return the IP
453 :return: the IP or None if not found
455 port = self.get_port_by_name(port_name)
457 return port.mac_address
460 def get_port_by_name(self, port_name):
462 Retrieves the OpenStack port object by its given name
463 :param port_name: the name of the port
464 :return: the OpenStack port object or None if not exists
466 for key, port in self.__ports:
469 logger.warning('Cannot find port with name - ' + port_name)
472 def get_vm_info(self):
474 Returns a dictionary of a VMs info as returned by OpenStack
477 return nova_utils.get_server_info(self._nova, self.__vm)
479 def __get_first_provisioning_floating_ip(self):
481 Returns the first floating IP tagged with the Floating IP name if
482 exists else the first one found
485 for floating_ip_setting in self.instance_settings.floating_ip_settings:
486 if floating_ip_setting.provisioning:
487 fip = self.__floating_ip_dict.get(floating_ip_setting.name)
490 elif len(self.__floating_ip_dict) > 0:
491 for key, fip in self.__floating_ip_dict.items():
494 # When cannot be found above
495 if len(self.__floating_ip_dict) > 0:
496 for key, fip in self.__floating_ip_dict.items():
499 def apply_ansible_playbook(self, pb_file_loc, variables=None,
502 Applies a playbook to a VM
503 :param pb_file_loc: the file location of the playbook to be applied
504 :param variables: a dict() of substitution values required by the
506 :param fip_name: the name of the floating IP to use for applying the
507 playbook (default - will take the first)
508 :return: the return value from ansible
510 return ansible_utils.apply_playbook(
511 pb_file_loc, [self.get_floating_ip(fip_name=fip_name).ip],
512 self.get_image_user(),
513 ssh_priv_key_file_path=self.keypair_settings.private_filepath,
514 variables=variables, proxy_setting=self._os_creds.proxy_settings)
516 def get_image_user(self):
518 Returns the instance sudo_user if it has been configured in the
519 instance_settings else it returns the image_settings.image_user value
521 if self.instance_settings.sudo_user:
522 return self.instance_settings.sudo_user
524 return self.image_settings.image_user
526 def vm_deleted(self, block=False, poll_interval=POLL_INTERVAL):
528 Returns true when the VM status returns the value of
529 expected_status_code or instance retrieval throws a NotFound exception.
530 :param block: When true, thread will block until active or timeout
531 value in seconds has been exceeded (False)
532 :param poll_interval: The polling interval in seconds
536 return self.__vm_status_check(
537 STATUS_DELETED, block,
538 self.instance_settings.vm_delete_timeout, poll_interval)
539 except NotFound as e:
541 "Instance not found when querying status for %s with message "
542 "%s", STATUS_DELETED, e)
545 def vm_active(self, block=False, poll_interval=POLL_INTERVAL):
547 Returns true when the VM status returns the value of the constant
549 :param block: When true, thread will block until active or timeout
550 value in seconds has been exceeded (False)
551 :param poll_interval: The polling interval in seconds
554 if self.__vm_status_check(
555 STATUS_ACTIVE, block, self.instance_settings.vm_boot_timeout,
557 self.__vm = nova_utils.get_server_object_by_id(
558 self._nova, self.__neutron, self.__vm.id)
562 def __vm_status_check(self, expected_status_code, block, timeout,
565 Returns true when the VM status returns the value of
567 :param expected_status_code: instance status evaluated with this
569 :param block: When true, thread will block until active or timeout
570 value in seconds has been exceeded (False)
571 :param timeout: The timeout value
572 :param poll_interval: The polling interval in seconds
575 # sleep and wait for VM status change
579 return self.__status(expected_status_code)
581 while timeout > time.time() - start:
582 status = self.__status(expected_status_code)
584 logger.info('VM is - ' + expected_status_code)
587 logger.debug('Retry querying VM status in ' + str(
588 poll_interval) + ' seconds')
589 time.sleep(poll_interval)
590 logger.debug('VM status query timeout in ' + str(
591 timeout - (time.time() - start)))
594 'Timeout checking for VM status for ' + expected_status_code)
597 def __status(self, expected_status_code):
599 Returns True when active else False
600 :param expected_status_code: instance status evaluated with this string
605 if expected_status_code == STATUS_DELETED:
610 status = nova_utils.get_server_status(self._nova, self.__vm)
612 logger.warning('Cannot find instance with id - ' + self.__vm.id)
615 if status == 'ERROR':
616 raise VmInstanceCreationError(
617 'Instance had an error during deployment')
619 'Instance status [%s] is - %s', self.instance_settings.name,
621 return status == expected_status_code
623 def vm_ssh_active(self, user_override=None, password=None, block=False,
624 timeout=None, poll_interval=POLL_INTERVAL):
626 Returns true when the VM can be accessed via SSH
627 :param block: When true, thread will block until active or timeout
628 value in seconds has been exceeded (False)
629 :param poll_interval: The polling interval
632 # sleep and wait for VM status change
633 logger.info('Checking if VM is active')
636 timeout = self.instance_settings.ssh_connect_timeout
638 if self.vm_active(block=True):
642 start = time.time() - timeout
644 while timeout > time.time() - start:
645 status = self.__ssh_active(
646 user_override=user_override, password=password)
648 logger.info('SSH is active for VM instance')
651 logger.debug('Retry SSH connection in ' + str(
652 poll_interval) + ' seconds')
653 time.sleep(poll_interval)
654 logger.debug('SSH connection timeout in ' + str(
655 timeout - (time.time() - start)))
657 logger.error('Timeout attempting to connect with VM via SSH')
660 def __ssh_active(self, user_override=None, password=None):
662 Returns True when can create a SSH session else False
665 if len(self.__floating_ip_dict) > 0:
666 ssh = self.ssh_client(
667 user_override=user_override, password=password)
673 def cloud_init_complete(self, block=False, poll_interval=POLL_INTERVAL):
675 Returns true when the VM's cloud-init routine has completed.
676 Note: this is currently done via SSH, therefore, if this instance does
677 not have a Floating IP or a running SSH server, this routine
678 will always return False or raise an Exception
679 :param block: When true, thread will block until active or timeout
680 value in seconds has been exceeded (False)
681 :param poll_interval: The polling interval
684 # sleep and wait for VM status change
685 logger.info('Checking if cloud-init has completed')
687 timeout = self.instance_settings.cloud_init_timeout
689 if self.vm_active(block=True) and self.vm_ssh_active(block=True):
693 start = time.time() - timeout
695 while timeout > time.time() - start:
696 status = self.__cloud_init_complete()
698 logger.info('cloud-init complete for VM instance')
701 logger.debug('Retry cloud-init query in ' + str(
702 poll_interval) + ' seconds')
703 time.sleep(poll_interval)
704 logger.debug('cloud-init complete timeout in ' + str(
705 timeout - (time.time() - start)))
707 logger.error('Timeout waiting for cloud-init to complete')
710 def __cloud_init_complete(self):
712 Returns True when can create a SSH session else False
715 if len(self.__floating_ip_dict) > 0:
716 ssh = self.ssh_client()
718 stdin1, stdout1, sterr1 = ssh.exec_command(
719 'ls -l /var/lib/cloud/instance/boot-finished')
720 return stdout1.channel.recv_exit_status() == 0
723 def get_floating_ip(self, fip_name=None):
725 Returns the floating IP object byt name if found, else the first known,
727 :param fip_name: the name of the floating IP to return
728 :return: the SSH client or None
730 if fip_name and self.__floating_ip_dict.get(fip_name):
731 return self.__floating_ip_dict.get(fip_name)
733 return self.__get_first_provisioning_floating_ip()
735 def ssh_client(self, fip_name=None, user_override=None, password=None):
737 Returns an SSH client using the name or the first known floating IP if
739 :param fip_name: the name of the floating IP to return
740 :param user_override: the username to use instead of the default
741 :param password: the password to use instead of the private key
742 :return: the SSH client or None
744 fip = self.get_floating_ip(fip_name)
746 ansible_user = self.get_image_user()
748 ansible_user = user_override
753 private_key = self.keypair_settings.private_filepath
756 return ansible_utils.ssh_client(
757 self.__get_first_provisioning_floating_ip().ip,
759 private_key_filepath=private_key,
761 proxy_settings=self._os_creds.proxy_settings)
763 FloatingIPAllocationError(
764 'Cannot return an SSH client. No Floating IP configured')
766 def add_security_group(self, security_group):
768 Adds a security group to this VM. Call will block until VM is active.
769 :param security_group: the SNAPS SecurityGroup domain object
770 :return True if successful else False
772 self.vm_active(block=True)
774 if not security_group:
775 logger.warning('Security group object is None, cannot add')
779 nova_utils.add_security_group(self._nova, self.get_vm_inst(),
782 except NotFound as e:
783 logger.warning('Security group not added - ' + str(e))
786 def remove_security_group(self, security_group):
788 Removes a security group to this VM. Call will block until VM is active
789 :param security_group: the OpenStack security group object
790 :return True if successful else False
792 self.vm_active(block=True)
794 if not security_group:
795 logger.warning('Security group object is None, cannot remove')
799 nova_utils.remove_security_group(self._nova, self.get_vm_inst(),
802 except NotFound as e:
803 logger.warning('Security group not removed - ' + str(e))
806 def reboot(self, reboot_type=RebootType.soft):
809 :param reboot_type: instance of
810 snaps.openstack.utils.nova_utils.RebootType
814 nova_utils.reboot_server(
815 self._nova, self.__vm, reboot_type=reboot_type)
818 def generate_creator(os_creds, vm_inst, image_config, keypair_config=None):
820 Initializes an OpenStackVmInstance object
821 :param os_creds: the OpenStack credentials
822 :param vm_inst: the SNAPS-OO VmInst domain object
823 :param image_config: the associated ImageConfig object
824 :param keypair_config: the associated KeypairConfig object (optional)
825 :return: an initialized OpenStackVmInstance object
827 nova = nova_utils.nova_client(os_creds)
828 neutron = neutron_utils.neutron_client(os_creds)
829 derived_inst_config = settings_utils.create_vm_inst_config(
830 nova, neutron, vm_inst)
832 derived_inst_creator = OpenStackVmInstance(
833 os_creds, derived_inst_config, image_config, keypair_config)
834 derived_inst_creator.initialize()
835 return derived_inst_creator
838 class VmInstanceSettings(VmInstanceConfig):
840 Deprecated, use snaps.config.vm_inst.VmInstanceConfig instead
842 def __init__(self, **kwargs):
843 from warnings import warn
844 warn('Use snaps.config.vm_inst.VmInstanceConfig instead',
846 super(self.__class__, self).__init__(**kwargs)
849 class FloatingIpSettings(FloatingIpConfig):
851 Deprecated, use snaps.config.vm_inst.FloatingIpConfig instead
853 def __init__(self, **kwargs):
854 from warnings import warn
855 warn('Use snaps.config.vm_inst.FloatingIpConfig instead',
857 super(self.__class__, self).__init__(**kwargs)
860 class VmInstanceCreationError(Exception):
862 Exception to be thrown when an VM instance cannot be created
866 class FloatingIPAllocationError(Exception):
868 Exception to be thrown when an VM instance cannot allocate a floating IP