1 # Copyright (c) 2017 Cable Television Laboratories, Inc. ("CableLabs")
2 # and others. All rights reserved.
4 # Licensed under the Apache License, Version 2.0 (the "License");
5 # you may not use this file except in compliance with the License.
6 # You may obtain a copy of the License at:
8 # http://www.apache.org/licenses/LICENSE-2.0
10 # Unless required by applicable law or agreed to in writing, software
11 # distributed under the License is distributed on an "AS IS" BASIS,
12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 # See the License for the specific language governing permissions and
14 # limitations under the License.
18 from novaclient.exceptions import NotFound, BadRequest
20 from snaps.config.vm_inst import VmInstanceConfig, FloatingIpConfig
21 from snaps.openstack.openstack_creator import OpenStackComputeObject
22 from snaps.openstack.utils import glance_utils, cinder_utils, settings_utils
23 from snaps.openstack.utils import neutron_utils
24 from snaps.openstack.utils import nova_utils
25 from snaps.openstack.utils.nova_utils import RebootType
26 from snaps.provisioning import ansible_utils
28 __author__ = 'spisarski'
30 logger = logging.getLogger('create_instance')
33 STATUS_ACTIVE = 'ACTIVE'
34 STATUS_DELETED = 'DELETED'
35 CLOUD_INIT_TIMEOUT = 120
38 class OpenStackVmInstance(OpenStackComputeObject):
40 Class responsible for managing a VM instance in OpenStack
43 def __init__(self, os_creds, instance_settings, image_settings,
44 keypair_settings=None):
47 :param os_creds: The connection credentials to the OpenStack API
48 :param instance_settings: Contains the settings for this VM
49 :param image_settings: The OpenStack image object settings
50 :param keypair_settings: The keypair metadata (Optional)
53 super(self.__class__, self).__init__(os_creds)
57 self.instance_settings = instance_settings
58 self.image_settings = image_settings
59 self.keypair_settings = keypair_settings
61 self.__floating_ip_dict = dict()
63 # Instantiated in self.create()
66 # Note: this object does not change after the VM becomes active
71 Loads the existing VMInst, Port, FloatingIps
72 :return: VMInst domain object
74 super(self.__class__, self).initialize()
76 self.__neutron = neutron_utils.neutron_client(self._os_creds)
78 self.__ports = self.__query_ports(self.instance_settings.port_settings)
79 self.__lookup_existing_vm_by_name()
81 def create(self, block=False):
83 Creates a VM instance and associated objects unless they already exist
84 :param block: Thread will block until instance has either become
85 active, error, or timeout waiting.
86 Additionally, when True, floating IPs will not be applied
88 :return: VMInst domain object
92 if len(self.__ports) == 0:
93 self.__ports = self.__create_ports(
94 self.instance_settings.port_settings)
96 self.__create_vm(block)
100 def __lookup_existing_vm_by_name(self):
102 Populates the member variables 'self.vm' and 'self.floating_ips' if a
103 VM with the same name already exists
106 server = nova_utils.get_server(
107 self._nova, self.__neutron,
108 vm_inst_settings=self.instance_settings)
110 if server.name == self.instance_settings.name:
113 'Found existing machine with name - %s',
114 self.instance_settings.name)
116 fips = neutron_utils.get_floating_ips(self.__neutron,
118 for port_id, fip in fips:
119 settings = self.instance_settings.floating_ip_settings
120 for fip_setting in settings:
121 if port_id == fip_setting.port_id:
122 self.__floating_ip_dict[fip_setting.name] = fip
124 port = neutron_utils.get_port_by_id(
125 self.__neutron, port_id)
126 if port and port.name == fip_setting.port_name:
127 self.__floating_ip_dict[fip_setting.name] = fip
129 def __create_vm(self, block=False):
131 Responsible for creating the VM instance
132 :param block: Thread will block until instance has either become
133 active, error, or timeout waiting. Floating IPs will be
134 assigned after active when block=True
136 glance = glance_utils.glance_client(self._os_creds)
137 self.__vm = nova_utils.create_server(
138 self._nova, self.__neutron, glance, self.instance_settings,
139 self.image_settings, self.keypair_settings)
140 logger.info('Created instance with name - %s',
141 self.instance_settings.name)
144 if not self.vm_active(block=True):
145 raise VmInstanceCreationError(
146 'Fatal error, VM did not become ACTIVE within the alloted '
149 # Create server should do this but found it needed to occur here
150 for sec_grp_name in self.instance_settings.security_group_names:
151 if self.vm_active(block=True):
152 nova_utils.add_security_group(self._nova, self.__vm,
155 raise VmInstanceCreationError(
156 'Cannot applying security group with name ' +
158 ' to VM that did not activate with name - ' +
159 self.instance_settings.name)
161 if self.instance_settings.volume_names:
162 for volume_name in self.instance_settings.volume_names:
163 cinder = cinder_utils.cinder_client(self._os_creds)
164 volume = cinder_utils.get_volume(
165 cinder, volume_name=volume_name)
167 if volume and self.vm_active(block=True):
169 vm = nova_utils.attach_volume(
170 self._nova, self.__neutron, self.__vm, volume, timeout)
175 logger.warn('Volume [%s] not attached within timeout '
176 'of [%s]', volume.name, timeout)
178 logger.warn('Unable to attach volume named [%s]',
181 self.__apply_floating_ips()
183 def __apply_floating_ips(self):
185 Applies the configured floating IPs to the necessary ports
188 for key, port in self.__ports:
189 port_dict[key] = port
192 for floating_ip_setting in self.instance_settings.floating_ip_settings:
193 self.add_floating_ip(floating_ip_setting)
195 def add_floating_ip(self, floating_ip_setting):
197 Adds a floating IP to a running instance
198 :param floating_ip_setting - the floating IP configuration
199 :return: the floating ip object
202 for key, port in self.__ports:
203 port_dict[key] = port
206 port = port_dict.get(floating_ip_setting.port_name)
209 raise VmInstanceCreationError(
210 'Cannot find port object with name - ' +
211 floating_ip_setting.port_name)
213 # Setup Floating IP only if there is a router with an external
215 ext_gateway = self.__ext_gateway_by_router(
216 floating_ip_setting.router_name)
218 subnet = neutron_utils.get_subnet(
220 subnet_name=floating_ip_setting.subnet_name)
221 floating_ip = neutron_utils.create_floating_ip(
222 self.__neutron, ext_gateway)
223 self.__floating_ip_dict[floating_ip_setting.name] = floating_ip
226 'Created floating IP %s via router - %s', floating_ip.ip,
227 floating_ip_setting.router_name)
228 self.__add_floating_ip(floating_ip, port, subnet)
231 raise VmInstanceCreationError(
232 'Unable to add floating IP to port, cannot locate router '
233 'with an external gateway ')
235 def __ext_gateway_by_router(self, router_name):
237 Returns network name for the external network attached to a router or
239 :param router_name: The name of the router to lookup
240 :return: the external network name or None
242 router = neutron_utils.get_router(
243 self.__neutron, router_name=router_name)
244 if router and router.external_network_id:
245 network = neutron_utils.get_network_by_id(
246 self.__neutron, router.external_network_id)
253 Destroys the VM instance
256 # Cleanup floating IPs
257 for name, floating_ip in self.__floating_ip_dict.items():
258 logger.info('Deleting Floating IP - ' + floating_ip.ip)
259 neutron_utils.delete_floating_ip(self.__neutron, floating_ip)
261 self.__floating_ip_dict = dict()
264 for name, port in self.__ports:
265 logger.info('Deleting Port with ID - %s ', port.id)
266 neutron_utils.delete_port(self.__neutron, port)
268 self.__ports = list()
272 for volume_rec in self.__vm.volume_ids:
273 cinder = cinder_utils.cinder_client(self._os_creds)
274 volume = cinder_utils.get_volume_by_id(
275 cinder, volume_rec['id'])
277 vm = nova_utils.detach_volume(
278 self._nova, self.__neutron, self.__vm, volume, 30)
283 'Timeout waiting to detach volume %s', volume.name)
285 logger.warn('Unable to detach volume with ID - [%s]',
290 'Deleting VM instance - ' + self.instance_settings.name)
293 nova_utils.delete_vm_instance(self._nova, self.__vm)
294 except NotFound as e:
295 logger.warn('Instance already deleted - %s', e)
297 # Block until instance cannot be found or returns the status of
299 logger.info('Checking deletion status')
301 if self.vm_deleted(block=True):
303 'VM has been properly deleted VM with name - %s',
304 self.instance_settings.name)
308 'VM not deleted within the timeout period of %s '
309 'seconds', self.instance_settings.vm_delete_timeout)
311 def __query_ports(self, port_settings):
313 Returns the previously configured ports or an empty list if none
315 :param port_settings: A list of PortSetting objects
316 :return: a list of OpenStack port tuples where the first member is the
317 port name and the second is the port object
321 for port_setting in port_settings:
322 port = neutron_utils.get_port(
323 self.__neutron, port_settings=port_setting)
325 ports.append((port_setting.name, port))
329 def __create_ports(self, port_settings):
331 Returns the previously configured ports or creates them if they do not
333 :param port_settings: A list of PortSetting objects
334 :return: a list of OpenStack port tuples where the first member is the
335 port name and the second is the port object
339 for port_setting in port_settings:
340 port = neutron_utils.get_port(
341 self.__neutron, port_settings=port_setting)
343 port = neutron_utils.create_port(
344 self.__neutron, self._os_creds, port_setting)
346 ports.append((port_setting.name, port))
350 def __add_floating_ip(self, floating_ip, port, subnet, timeout=30,
351 poll_interval=POLL_INTERVAL):
353 Returns True when active else False
354 TODO - Make timeout and poll_interval configurable...
359 # Take IP of subnet if there is one configured on which to place
361 for fixed_ip in port.ips:
362 if fixed_ip['subnet_id'] == subnet.id:
363 ip = fixed_ip['ip_address']
366 # Simply take the first
367 ip = port.ips[0]['ip_address']
370 count = timeout / poll_interval
372 logger.debug('Attempting to add floating IP to instance')
374 nova_utils.add_floating_ip_to_server(
375 self._nova, self.__vm, floating_ip, ip)
377 'Added floating IP %s to port IP %s on instance %s',
378 floating_ip.ip, ip, self.instance_settings.name)
380 except BadRequest as bre:
381 logger.error('Cannot add floating IP [%s]', bre)
383 except Exception as e:
385 'Retry adding floating IP to instance. Last attempt '
386 'failed with - %s', e)
387 time.sleep(poll_interval)
391 raise VmInstanceCreationError(
392 'Unable find IP address on which to place the floating IP')
394 logger.error('Timeout attempting to add the floating IP to instance.')
395 raise VmInstanceCreationError(
396 'Timeout while attempting add floating IP to instance')
398 def get_os_creds(self):
400 Returns the OpenStack credentials used to create these objects
401 :return: the credentials
403 return self._os_creds
405 def get_vm_inst(self):
407 Returns the latest version of this server object from OpenStack
408 :return: Server object
410 return nova_utils.get_server_object_by_id(
411 self._nova, self.__neutron, self.__vm.id)
413 def get_console_output(self):
415 Returns the vm console object for parsing logs
416 :return: the console output object
418 return nova_utils.get_server_console_output(self._nova, self.__vm)
420 def get_port_ip(self, port_name, subnet_name=None):
422 Returns the first IP for the port corresponding with the port_name
423 parameter when subnet_name is None else returns the IP address that
424 corresponds to the subnet_name parameter
425 :param port_name: the name of the port from which to return the IP
426 :param subnet_name: the name of the subnet attached to this IP
427 :return: the IP or None if not found
429 port = self.get_port_by_name(port_name)
432 subnet = neutron_utils.get_subnet(
433 self.__neutron, subnet_name=subnet_name)
435 logger.warning('Cannot retrieve port IP as subnet could '
436 'not be located with name - %s',
439 for fixed_ip in port.ips:
440 if fixed_ip['subnet_id'] == subnet.id:
441 return fixed_ip['ip_address']
443 if port.ips and len(port.ips) > 0:
444 return port.ips[0]['ip_address']
447 def get_port_mac(self, port_name):
449 Returns the first IP for the port corresponding with the port_name
451 TODO - Add in the subnet as an additional parameter as a port may have
453 :param port_name: the name of the port from which to return the IP
454 :return: the IP or None if not found
456 port = self.get_port_by_name(port_name)
458 return port.mac_address
461 def get_port_by_name(self, port_name):
463 Retrieves the OpenStack port object by its given name
464 :param port_name: the name of the port
465 :return: the OpenStack port object or None if not exists
467 for key, port in self.__ports:
470 logger.warning('Cannot find port with name - ' + port_name)
473 def get_vm_info(self):
475 Returns a dictionary of a VMs info as returned by OpenStack
478 return nova_utils.get_server_info(self._nova, self.__vm)
480 def __get_first_provisioning_floating_ip(self):
482 Returns the first floating IP tagged with the Floating IP name if
483 exists else the first one found
486 for floating_ip_setting in self.instance_settings.floating_ip_settings:
487 if floating_ip_setting.provisioning:
488 fip = self.__floating_ip_dict.get(floating_ip_setting.name)
491 elif len(self.__floating_ip_dict) > 0:
492 for key, fip in self.__floating_ip_dict.items():
495 # When cannot be found above
496 if len(self.__floating_ip_dict) > 0:
497 for key, fip in self.__floating_ip_dict.items():
500 def apply_ansible_playbook(self, pb_file_loc, variables=None,
503 Applies a playbook to a VM
504 :param pb_file_loc: the file location of the playbook to be applied
505 :param variables: a dict() of substitution values required by the
507 :param fip_name: the name of the floating IP to use for applying the
508 playbook (default - will take the first)
509 :return: the return value from ansible
511 return ansible_utils.apply_playbook(
512 pb_file_loc, [self.get_floating_ip(fip_name=fip_name).ip],
513 self.get_image_user(), self.keypair_settings.private_filepath,
514 variables, self._os_creds.proxy_settings)
516 def get_image_user(self):
518 Returns the instance sudo_user if it has been configured in the
519 instance_settings else it returns the image_settings.image_user value
521 if self.instance_settings.sudo_user:
522 return self.instance_settings.sudo_user
524 return self.image_settings.image_user
526 def vm_deleted(self, block=False, poll_interval=POLL_INTERVAL):
528 Returns true when the VM status returns the value of
529 expected_status_code or instance retrieval throws a NotFound exception.
530 :param block: When true, thread will block until active or timeout
531 value in seconds has been exceeded (False)
532 :param poll_interval: The polling interval in seconds
536 return self.__vm_status_check(
537 STATUS_DELETED, block,
538 self.instance_settings.vm_delete_timeout, poll_interval)
539 except NotFound as e:
541 "Instance not found when querying status for %s with message "
542 "%s", STATUS_DELETED, e)
545 def vm_active(self, block=False, poll_interval=POLL_INTERVAL):
547 Returns true when the VM status returns the value of the constant
549 :param block: When true, thread will block until active or timeout
550 value in seconds has been exceeded (False)
551 :param poll_interval: The polling interval in seconds
554 if self.__vm_status_check(
555 STATUS_ACTIVE, block, self.instance_settings.vm_boot_timeout,
557 self.__vm = nova_utils.get_server_object_by_id(
558 self._nova, self.__neutron, self.__vm.id)
562 def __vm_status_check(self, expected_status_code, block, timeout,
565 Returns true when the VM status returns the value of
567 :param expected_status_code: instance status evaluated with this
569 :param block: When true, thread will block until active or timeout
570 value in seconds has been exceeded (False)
571 :param timeout: The timeout value
572 :param poll_interval: The polling interval in seconds
575 # sleep and wait for VM status change
579 return self.__status(expected_status_code)
581 while timeout > time.time() - start:
582 status = self.__status(expected_status_code)
584 logger.info('VM is - ' + expected_status_code)
587 logger.debug('Retry querying VM status in ' + str(
588 poll_interval) + ' seconds')
589 time.sleep(poll_interval)
590 logger.debug('VM status query timeout in ' + str(
591 timeout - (time.time() - start)))
594 'Timeout checking for VM status for ' + expected_status_code)
597 def __status(self, expected_status_code):
599 Returns True when active else False
600 :param expected_status_code: instance status evaluated with this string
605 if expected_status_code == STATUS_DELETED:
610 status = nova_utils.get_server_status(self._nova, self.__vm)
612 logger.warning('Cannot find instance with id - ' + self.__vm.id)
615 if status == 'ERROR':
616 raise VmInstanceCreationError(
617 'Instance had an error during deployment')
619 'Instance status [%s] is - %s', self.instance_settings.name,
621 return status == expected_status_code
623 def vm_ssh_active(self, block=False, poll_interval=POLL_INTERVAL):
625 Returns true when the VM can be accessed via SSH
626 :param block: When true, thread will block until active or timeout
627 value in seconds has been exceeded (False)
628 :param poll_interval: The polling interval
631 # sleep and wait for VM status change
632 logger.info('Checking if VM is active')
634 timeout = self.instance_settings.ssh_connect_timeout
636 if self.vm_active(block=True):
640 start = time.time() - timeout
642 while timeout > time.time() - start:
643 status = self.__ssh_active()
645 logger.info('SSH is active for VM instance')
648 logger.debug('Retry SSH connection in ' + str(
649 poll_interval) + ' seconds')
650 time.sleep(poll_interval)
651 logger.debug('SSH connection timeout in ' + str(
652 timeout - (time.time() - start)))
654 logger.error('Timeout attempting to connect with VM via SSH')
657 def __ssh_active(self):
659 Returns True when can create a SSH session else False
662 if len(self.__floating_ip_dict) > 0:
663 ssh = self.ssh_client()
669 def cloud_init_complete(self, block=False, poll_interval=POLL_INTERVAL):
671 Returns true when the VM's cloud-init routine has completed.
672 Note: this is currently done via SSH, therefore, if this instance does
673 not have a Floating IP or a running SSH server, this routine
674 will always return False or raise an Exception
675 :param block: When true, thread will block until active or timeout
676 value in seconds has been exceeded (False)
677 :param poll_interval: The polling interval
680 # sleep and wait for VM status change
681 logger.info('Checking if cloud-init has completed')
683 timeout = CLOUD_INIT_TIMEOUT
685 if self.vm_active(block=True) and self.vm_ssh_active(block=True):
689 start = time.time() - timeout
691 while timeout > time.time() - start:
692 status = self.__cloud_init_complete()
694 logger.info('cloud-init complete for VM instance')
697 logger.debug('Retry cloud-init query in ' + str(
698 poll_interval) + ' seconds')
699 time.sleep(poll_interval)
700 logger.debug('cloud-init complete timeout in ' + str(
701 timeout - (time.time() - start)))
703 logger.error('Timeout waiting for cloud-init to complete')
706 def __cloud_init_complete(self):
708 Returns True when can create a SSH session else False
711 if len(self.__floating_ip_dict) > 0:
712 ssh = self.ssh_client()
714 stdin1, stdout1, sterr1 = ssh.exec_command(
715 'ls -l /var/lib/cloud/instance/boot-finished')
716 return stdout1.channel.recv_exit_status() == 0
719 def get_floating_ip(self, fip_name=None):
721 Returns the floating IP object byt name if found, else the first known,
723 :param fip_name: the name of the floating IP to return
724 :return: the SSH client or None
726 if fip_name and self.__floating_ip_dict.get(fip_name):
727 return self.__floating_ip_dict.get(fip_name)
729 return self.__get_first_provisioning_floating_ip()
731 def ssh_client(self, fip_name=None):
733 Returns an SSH client using the name or the first known floating IP if
735 :param fip_name: the name of the floating IP to return
736 :return: the SSH client or None
738 fip = self.get_floating_ip(fip_name)
740 return ansible_utils.ssh_client(
741 self.__get_first_provisioning_floating_ip().ip,
742 self.get_image_user(),
743 self.keypair_settings.private_filepath,
744 proxy_settings=self._os_creds.proxy_settings)
746 FloatingIPAllocationError(
747 'Cannot return an SSH client. No Floating IP configured')
749 def add_security_group(self, security_group):
751 Adds a security group to this VM. Call will block until VM is active.
752 :param security_group: the SNAPS SecurityGroup domain object
753 :return True if successful else False
755 self.vm_active(block=True)
757 if not security_group:
758 logger.warning('Security group object is None, cannot add')
762 nova_utils.add_security_group(self._nova, self.get_vm_inst(),
765 except NotFound as e:
766 logger.warning('Security group not added - ' + str(e))
769 def remove_security_group(self, security_group):
771 Removes a security group to this VM. Call will block until VM is active
772 :param security_group: the OpenStack security group object
773 :return True if successful else False
775 self.vm_active(block=True)
777 if not security_group:
778 logger.warning('Security group object is None, cannot remove')
782 nova_utils.remove_security_group(self._nova, self.get_vm_inst(),
785 except NotFound as e:
786 logger.warning('Security group not removed - ' + str(e))
789 def reboot(self, reboot_type=RebootType.soft):
792 :param reboot_type: instance of
793 snaps.openstack.utils.nova_utils.RebootType
797 nova_utils.reboot_server(
798 self._nova, self.__vm, reboot_type=reboot_type)
801 def generate_creator(os_creds, vm_inst, image_config, keypair_config=None):
803 Initializes an OpenStackVmInstance object
804 :param os_creds: the OpenStack credentials
805 :param vm_inst: the SNAPS-OO VmInst domain object
806 :param image_config: the associated ImageConfig object
807 :param keypair_config: the associated KeypairConfig object (optional)
808 :return: an initialized OpenStackVmInstance object
810 nova = nova_utils.nova_client(os_creds)
811 neutron = neutron_utils.neutron_client(os_creds)
812 derived_inst_config = settings_utils.create_vm_inst_config(
813 nova, neutron, vm_inst)
815 derived_inst_creator = OpenStackVmInstance(
816 os_creds, derived_inst_config, image_config, keypair_config)
817 derived_inst_creator.initialize()
818 return derived_inst_creator
821 class VmInstanceSettings(VmInstanceConfig):
823 Deprecated, use snaps.config.vm_inst.VmInstanceConfig instead
825 def __init__(self, **kwargs):
826 from warnings import warn
827 warn('Use snaps.config.vm_inst.VmInstanceConfig instead',
829 super(self.__class__, self).__init__(**kwargs)
832 class FloatingIpSettings(FloatingIpConfig):
834 Deprecated, use snaps.config.vm_inst.FloatingIpConfig instead
836 def __init__(self, **kwargs):
837 from warnings import warn
838 warn('Use snaps.config.vm_inst.FloatingIpConfig instead',
840 super(self.__class__, self).__init__(**kwargs)
843 class VmInstanceCreationError(Exception):
845 Exception to be thrown when an VM instance cannot be created
849 class FloatingIPAllocationError(Exception):
851 Exception to be thrown when an VM instance cannot allocate a floating IP