1 # Copyright (c) 2017 Cable Television Laboratories, Inc. ("CableLabs")
2 # and others. All rights reserved.
4 # Licensed under the Apache License, Version 2.0 (the "License");
5 # you may not use this file except in compliance with the License.
6 # You may obtain a copy of the License at:
8 # http://www.apache.org/licenses/LICENSE-2.0
10 # Unless required by applicable law or agreed to in writing, software
11 # distributed under the License is distributed on an "AS IS" BASIS,
12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 # See the License for the specific language governing permissions and
14 # limitations under the License.
18 from novaclient.exceptions import NotFound
20 from snaps.config.vm_inst import VmInstanceConfig, FloatingIpConfig
21 from snaps.openstack.openstack_creator import OpenStackComputeObject
22 from snaps.openstack.utils import glance_utils, cinder_utils, settings_utils
23 from snaps.openstack.utils import neutron_utils
24 from snaps.openstack.utils import nova_utils
25 from snaps.openstack.utils.nova_utils import RebootType
26 from snaps.provisioning import ansible_utils
28 __author__ = 'spisarski'
30 logger = logging.getLogger('create_instance')
33 STATUS_ACTIVE = 'ACTIVE'
34 STATUS_DELETED = 'DELETED'
37 class OpenStackVmInstance(OpenStackComputeObject):
39 Class responsible for managing a VM instance in OpenStack
42 def __init__(self, os_creds, instance_settings, image_settings,
43 keypair_settings=None):
46 :param os_creds: The connection credentials to the OpenStack API
47 :param instance_settings: Contains the settings for this VM
48 :param image_settings: The OpenStack image object settings
49 :param keypair_settings: The keypair metadata (Optional)
52 super(self.__class__, self).__init__(os_creds)
56 self.instance_settings = instance_settings
57 self.image_settings = image_settings
58 self.keypair_settings = keypair_settings
60 self.__floating_ip_dict = dict()
62 # Instantiated in self.create()
65 # Note: this object does not change after the VM becomes active
70 Loads the existing VMInst, Port, FloatingIps
71 :return: VMInst domain object
73 super(self.__class__, self).initialize()
75 self.__neutron = neutron_utils.neutron_client(self._os_creds)
77 self.__ports = self.__query_ports(self.instance_settings.port_settings)
78 self.__lookup_existing_vm_by_name()
80 def create(self, block=False):
82 Creates a VM instance and associated objects unless they already exist
83 :param block: Thread will block until instance has either become
84 active, error, or timeout waiting.
85 Additionally, when True, floating IPs will not be applied
87 :return: VMInst domain object
91 if len(self.__ports) == 0:
92 self.__ports = self.__create_ports(
93 self.instance_settings.port_settings)
95 self.__create_vm(block)
99 def __lookup_existing_vm_by_name(self):
101 Populates the member variables 'self.vm' and 'self.floating_ips' if a
102 VM with the same name already exists
105 server = nova_utils.get_server(
106 self._nova, self.__neutron,
107 vm_inst_settings=self.instance_settings)
109 if server.name == self.instance_settings.name:
112 'Found existing machine with name - %s',
113 self.instance_settings.name)
115 fips = neutron_utils.get_floating_ips(self.__neutron,
117 for port_id, fip in fips:
118 settings = self.instance_settings.floating_ip_settings
119 for fip_setting in settings:
120 if port_id == fip_setting.port_id:
121 self.__floating_ip_dict[fip_setting.name] = fip
123 port = neutron_utils.get_port_by_id(
124 self.__neutron, port_id)
125 if port and port.name == fip_setting.port_name:
126 self.__floating_ip_dict[fip_setting.name] = fip
128 def __create_vm(self, block=False):
130 Responsible for creating the VM instance
131 :param block: Thread will block until instance has either become
132 active, error, or timeout waiting. Floating IPs will be
133 assigned after active when block=True
135 glance = glance_utils.glance_client(self._os_creds)
136 self.__vm = nova_utils.create_server(
137 self._nova, self.__neutron, glance, self.instance_settings,
138 self.image_settings, self.keypair_settings)
139 logger.info('Created instance with name - %s',
140 self.instance_settings.name)
143 if not self.vm_active(block=True):
144 raise VmInstanceCreationError(
145 'Fatal error, VM did not become ACTIVE within the alloted '
148 # Create server should do this but found it needed to occur here
149 for sec_grp_name in self.instance_settings.security_group_names:
150 if self.vm_active(block=True):
151 nova_utils.add_security_group(self._nova, self.__vm,
154 raise VmInstanceCreationError(
155 'Cannot applying security group with name ' +
157 ' to VM that did not activate with name - ' +
158 self.instance_settings.name)
160 if self.instance_settings.volume_names:
161 for volume_name in self.instance_settings.volume_names:
162 cinder = cinder_utils.cinder_client(self._os_creds)
163 volume = cinder_utils.get_volume(
164 cinder, volume_name=volume_name)
166 if volume and self.vm_active(block=True):
168 vm = nova_utils.attach_volume(
169 self._nova, self.__neutron, self.__vm, volume, timeout)
174 logger.warn('Volume [%s] not attached within timeout '
175 'of [%s]', volume.name, timeout)
177 logger.warn('Unable to attach volume named [%s]',
180 self.__apply_floating_ips()
182 def __apply_floating_ips(self):
184 Applies the configured floating IPs to the necessary ports
187 for key, port in self.__ports:
188 port_dict[key] = port
191 for floating_ip_setting in self.instance_settings.floating_ip_settings:
192 self.add_floating_ip(floating_ip_setting)
194 def add_floating_ip(self, floating_ip_setting):
196 Adds a floating IP to a running instance
197 :param floating_ip_setting - the floating IP configuration
198 :return: the floating ip object
201 for key, port in self.__ports:
202 port_dict[key] = port
205 port = port_dict.get(floating_ip_setting.port_name)
208 raise VmInstanceCreationError(
209 'Cannot find port object with name - ' +
210 floating_ip_setting.port_name)
212 # Setup Floating IP only if there is a router with an external
214 ext_gateway = self.__ext_gateway_by_router(
215 floating_ip_setting.router_name)
216 if ext_gateway and self.vm_active(block=True):
217 floating_ip = neutron_utils.create_floating_ip(
218 self.__neutron, ext_gateway, port.id)
219 self.__floating_ip_dict[floating_ip_setting.name] = floating_ip
222 'Created floating IP %s via router - %s', floating_ip.ip,
223 floating_ip_setting.router_name)
227 raise VmInstanceCreationError(
228 'Unable to add floating IP to port, cannot locate router '
229 'with an external gateway ')
231 def __ext_gateway_by_router(self, router_name):
233 Returns network name for the external network attached to a router or
235 :param router_name: The name of the router to lookup
236 :return: the external network name or None
238 router = neutron_utils.get_router(
239 self.__neutron, router_name=router_name)
240 if router and router.external_network_id:
241 network = neutron_utils.get_network_by_id(
242 self.__neutron, router.external_network_id)
249 Destroys the VM instance
252 # Cleanup floating IPs
253 for name, floating_ip in self.__floating_ip_dict.items():
254 logger.info('Deleting Floating IP - ' + floating_ip.ip)
255 neutron_utils.delete_floating_ip(self.__neutron, floating_ip)
257 self.__floating_ip_dict = dict()
260 for name, port in self.__ports:
261 logger.info('Deleting Port with ID - %s ', port.id)
262 neutron_utils.delete_port(self.__neutron, port)
264 self.__ports = list()
268 for volume_rec in self.__vm.volume_ids:
269 cinder = cinder_utils.cinder_client(self._os_creds)
270 volume = cinder_utils.get_volume_by_id(
271 cinder, volume_rec['id'])
273 vm = nova_utils.detach_volume(
274 self._nova, self.__neutron, self.__vm, volume, 30)
279 'Timeout waiting to detach volume %s', volume.name)
281 logger.warn('Unable to detach volume with ID - [%s]',
286 'Deleting VM instance - ' + self.instance_settings.name)
289 nova_utils.delete_vm_instance(self._nova, self.__vm)
290 except NotFound as e:
291 logger.warn('Instance already deleted - %s', e)
293 # Block until instance cannot be found or returns the status of
295 logger.info('Checking deletion status')
297 if self.vm_deleted(block=True):
299 'VM has been properly deleted VM with name - %s',
300 self.instance_settings.name)
304 'VM not deleted within the timeout period of %s '
305 'seconds', self.instance_settings.vm_delete_timeout)
307 def __query_ports(self, port_settings):
309 Returns the previously configured ports or an empty list if none
311 :param port_settings: A list of PortSetting objects
312 :return: a list of OpenStack port tuples where the first member is the
313 port name and the second is the port object
317 for port_setting in port_settings:
318 port = neutron_utils.get_port(
319 self.__neutron, port_settings=port_setting)
321 ports.append((port_setting.name, port))
325 def __create_ports(self, port_settings):
327 Returns the previously configured ports or creates them if they do not
329 :param port_settings: A list of PortSetting objects
330 :return: a list of OpenStack port tuples where the first member is the
331 port name and the second is the port object
335 for port_setting in port_settings:
336 port = neutron_utils.get_port(
337 self.__neutron, port_settings=port_setting)
339 port = neutron_utils.create_port(
340 self.__neutron, self._os_creds, port_setting)
342 ports.append((port_setting.name, port))
346 def get_os_creds(self):
348 Returns the OpenStack credentials used to create these objects
349 :return: the credentials
351 return self._os_creds
353 def get_vm_inst(self):
355 Returns the latest version of this server object from OpenStack
356 :return: Server object
358 return nova_utils.get_server_object_by_id(
359 self._nova, self.__neutron, self.__vm.id)
361 def get_console_output(self):
363 Returns the vm console object for parsing logs
364 :return: the console output object
366 return nova_utils.get_server_console_output(self._nova, self.__vm)
368 def get_port_ip(self, port_name, subnet_name=None):
370 Returns the first IP for the port corresponding with the port_name
371 parameter when subnet_name is None else returns the IP address that
372 corresponds to the subnet_name parameter
373 :param port_name: the name of the port from which to return the IP
374 :param subnet_name: the name of the subnet attached to this IP
375 :return: the IP or None if not found
377 port = self.get_port_by_name(port_name)
380 subnet = neutron_utils.get_subnet(
381 self.__neutron, subnet_name=subnet_name)
383 logger.warning('Cannot retrieve port IP as subnet could '
384 'not be located with name - %s',
387 for fixed_ip in port.ips:
388 if fixed_ip['subnet_id'] == subnet.id:
389 return fixed_ip['ip_address']
391 if port.ips and len(port.ips) > 0:
392 return port.ips[0]['ip_address']
395 def get_port_mac(self, port_name):
397 Returns the first IP for the port corresponding with the port_name
399 TODO - Add in the subnet as an additional parameter as a port may have
401 :param port_name: the name of the port from which to return the IP
402 :return: the IP or None if not found
404 port = self.get_port_by_name(port_name)
406 return port.mac_address
409 def get_port_by_name(self, port_name):
411 Retrieves the OpenStack port object by its given name
412 :param port_name: the name of the port
413 :return: the OpenStack port object or None if not exists
415 for key, port in self.__ports:
418 logger.warning('Cannot find port with name - ' + port_name)
421 def get_vm_info(self):
423 Returns a dictionary of a VMs info as returned by OpenStack
426 return nova_utils.get_server_info(self._nova, self.__vm)
428 def __get_first_provisioning_floating_ip(self):
430 Returns the first floating IP tagged with the Floating IP name if
431 exists else the first one found
434 for floating_ip_setting in self.instance_settings.floating_ip_settings:
435 if floating_ip_setting.provisioning:
436 fip = self.__floating_ip_dict.get(floating_ip_setting.name)
439 elif len(self.__floating_ip_dict) > 0:
440 for key, fip in self.__floating_ip_dict.items():
443 # When cannot be found above
444 if len(self.__floating_ip_dict) > 0:
445 for key, fip in self.__floating_ip_dict.items():
448 def apply_ansible_playbook(self, pb_file_loc, variables=None,
451 Applies a playbook to a VM
452 :param pb_file_loc: the file location of the playbook to be applied
453 :param variables: a dict() of substitution values required by the
455 :param fip_name: the name of the floating IP to use for applying the
456 playbook (default - will take the first)
457 :return: the return value from ansible
459 return ansible_utils.apply_playbook(
460 pb_file_loc, [self.get_floating_ip(fip_name=fip_name).ip],
461 self.get_image_user(),
462 ssh_priv_key_file_path=self.keypair_settings.private_filepath,
463 variables=variables, proxy_setting=self._os_creds.proxy_settings)
465 def get_image_user(self):
467 Returns the instance sudo_user if it has been configured in the
468 instance_settings else it returns the image_settings.image_user value
470 if self.instance_settings.sudo_user:
471 return self.instance_settings.sudo_user
473 return self.image_settings.image_user
475 def vm_deleted(self, block=False, poll_interval=POLL_INTERVAL):
477 Returns true when the VM status returns the value of
478 expected_status_code or instance retrieval throws a NotFound exception.
479 :param block: When true, thread will block until active or timeout
480 value in seconds has been exceeded (False)
481 :param poll_interval: The polling interval in seconds
485 return self.__vm_status_check(
486 STATUS_DELETED, block,
487 self.instance_settings.vm_delete_timeout, poll_interval)
488 except NotFound as e:
490 "Instance not found when querying status for %s with message "
491 "%s", STATUS_DELETED, e)
494 def vm_active(self, block=False, poll_interval=POLL_INTERVAL):
496 Returns true when the VM status returns the value of the constant
498 :param block: When true, thread will block until active or timeout
499 value in seconds has been exceeded (False)
500 :param poll_interval: The polling interval in seconds
503 if self.__vm_status_check(
504 STATUS_ACTIVE, block, self.instance_settings.vm_boot_timeout,
506 self.__vm = nova_utils.get_server_object_by_id(
507 self._nova, self.__neutron, self.__vm.id)
511 def __vm_status_check(self, expected_status_code, block, timeout,
514 Returns true when the VM status returns the value of
516 :param expected_status_code: instance status evaluated with this
518 :param block: When true, thread will block until active or timeout
519 value in seconds has been exceeded (False)
520 :param timeout: The timeout value
521 :param poll_interval: The polling interval in seconds
524 # sleep and wait for VM status change
528 return self.__status(expected_status_code)
530 while timeout > time.time() - start:
531 status = self.__status(expected_status_code)
533 logger.info('VM is - ' + expected_status_code)
536 logger.debug('Retry querying VM status in ' + str(
537 poll_interval) + ' seconds')
538 time.sleep(poll_interval)
539 logger.debug('VM status query timeout in ' + str(
540 timeout - (time.time() - start)))
543 'Timeout checking for VM status for ' + expected_status_code)
546 def __status(self, expected_status_code):
548 Returns True when active else False
549 :param expected_status_code: instance status evaluated with this string
554 if expected_status_code == STATUS_DELETED:
559 status = nova_utils.get_server_status(self._nova, self.__vm)
561 logger.warning('Cannot find instance with id - ' + self.__vm.id)
564 if status == 'ERROR':
565 raise VmInstanceCreationError(
566 'Instance had an error during deployment')
568 'Instance status [%s] is - %s', self.instance_settings.name,
570 return status == expected_status_code
572 def vm_ssh_active(self, user_override=None, password=None, block=False,
573 timeout=None, poll_interval=POLL_INTERVAL):
575 Returns true when the VM can be accessed via SSH
576 :param user_override: overrides the user with which to create the
578 :param password: overrides the use of a password instead of a private
579 key with which to create the connection
580 :param block: When true, thread will block until active or timeout
581 value in seconds has been exceeded (False)
582 :param timeout: the number of seconds to retry obtaining the connection
583 and overrides the ssh_connect_timeout member of the
584 self.instance_settings object
585 :param poll_interval: The polling interval
588 # sleep and wait for VM status change
589 logger.info('Checking if VM is active')
592 timeout = self.instance_settings.ssh_connect_timeout
594 if self.vm_active(block=True):
598 start = time.time() - timeout
600 while timeout > time.time() - start:
601 status = self.__ssh_active(
602 user_override=user_override, password=password)
604 logger.info('SSH is active for VM instance')
607 logger.debug('Retry SSH connection in ' + str(
608 poll_interval) + ' seconds')
609 time.sleep(poll_interval)
610 logger.debug('SSH connection timeout in ' + str(
611 timeout - (time.time() - start)))
613 logger.error('Timeout attempting to connect with VM via SSH')
616 def __ssh_active(self, user_override=None, password=None):
618 Returns True when can create a SSH session else False
621 if len(self.__floating_ip_dict) > 0:
622 ssh = self.ssh_client(
623 user_override=user_override, password=password)
629 def cloud_init_complete(self, block=False, poll_interval=POLL_INTERVAL):
631 Returns true when the VM's cloud-init routine has completed.
632 Note: this is currently done via SSH, therefore, if this instance does
633 not have a Floating IP or a running SSH server, this routine
634 will always return False or raise an Exception
635 :param block: When true, thread will block until active or timeout
636 value in seconds has been exceeded (False)
637 :param poll_interval: The polling interval
640 # sleep and wait for VM status change
641 logger.info('Checking if cloud-init has completed')
643 timeout = self.instance_settings.cloud_init_timeout
645 if self.vm_active(block=True) and self.vm_ssh_active(block=True):
649 start = time.time() - timeout
651 while timeout > time.time() - start:
652 status = self.__cloud_init_complete()
654 logger.info('cloud-init complete for VM instance')
657 logger.debug('Retry cloud-init query in ' + str(
658 poll_interval) + ' seconds')
659 time.sleep(poll_interval)
660 logger.debug('cloud-init complete timeout in ' + str(
661 timeout - (time.time() - start)))
663 logger.error('Timeout waiting for cloud-init to complete')
666 def __cloud_init_complete(self):
668 Returns True when can create a SSH session else False
671 if len(self.__floating_ip_dict) > 0:
672 ssh = self.ssh_client()
674 stdin1, stdout1, sterr1 = ssh.exec_command(
675 'ls -l /var/lib/cloud/instance/boot-finished')
676 return stdout1.channel.recv_exit_status() == 0
679 def get_floating_ip(self, fip_name=None):
681 Returns the floating IP object byt name if found, else the first known,
683 :param fip_name: the name of the floating IP to return
684 :return: the SSH client or None
686 if fip_name and self.__floating_ip_dict.get(fip_name):
687 return self.__floating_ip_dict.get(fip_name)
689 return self.__get_first_provisioning_floating_ip()
691 def ssh_client(self, fip_name=None, user_override=None, password=None):
693 Returns an SSH client using the name or the first known floating IP if
695 :param fip_name: the name of the floating IP to return
696 :param user_override: the username to use instead of the default
697 :param password: the password to use instead of the private key
698 :return: the SSH client or None
700 fip = self.get_floating_ip(fip_name)
702 ansible_user = self.get_image_user()
704 ansible_user = user_override
709 private_key = self.keypair_settings.private_filepath
712 return ansible_utils.ssh_client(
713 self.__get_first_provisioning_floating_ip().ip,
715 private_key_filepath=private_key,
717 proxy_settings=self._os_creds.proxy_settings)
719 FloatingIPAllocationError(
720 'Cannot return an SSH client. No Floating IP configured')
722 def add_security_group(self, security_group):
724 Adds a security group to this VM. Call will block until VM is active.
725 :param security_group: the SNAPS SecurityGroup domain object
726 :return True if successful else False
728 self.vm_active(block=True)
730 if not security_group:
731 logger.warning('Security group object is None, cannot add')
735 nova_utils.add_security_group(self._nova, self.get_vm_inst(),
738 except NotFound as e:
739 logger.warning('Security group not added - ' + str(e))
742 def remove_security_group(self, security_group):
744 Removes a security group to this VM. Call will block until VM is active
745 :param security_group: the OpenStack security group object
746 :return True if successful else False
748 self.vm_active(block=True)
750 if not security_group:
751 logger.warning('Security group object is None, cannot remove')
755 nova_utils.remove_security_group(self._nova, self.get_vm_inst(),
758 except NotFound as e:
759 logger.warning('Security group not removed - ' + str(e))
762 def reboot(self, reboot_type=RebootType.soft):
765 :param reboot_type: instance of
766 snaps.openstack.utils.nova_utils.RebootType
770 nova_utils.reboot_server(
771 self._nova, self.__vm, reboot_type=reboot_type)
774 def generate_creator(os_creds, vm_inst, image_config, keypair_config=None):
776 Initializes an OpenStackVmInstance object
777 :param os_creds: the OpenStack credentials
778 :param vm_inst: the SNAPS-OO VmInst domain object
779 :param image_config: the associated ImageConfig object
780 :param keypair_config: the associated KeypairConfig object (optional)
781 :return: an initialized OpenStackVmInstance object
783 nova = nova_utils.nova_client(os_creds)
784 neutron = neutron_utils.neutron_client(os_creds)
785 derived_inst_config = settings_utils.create_vm_inst_config(
786 nova, neutron, vm_inst)
788 derived_inst_creator = OpenStackVmInstance(
789 os_creds, derived_inst_config, image_config, keypair_config)
790 derived_inst_creator.initialize()
791 return derived_inst_creator
794 class VmInstanceSettings(VmInstanceConfig):
796 Deprecated, use snaps.config.vm_inst.VmInstanceConfig instead
798 def __init__(self, **kwargs):
799 from warnings import warn
800 warn('Use snaps.config.vm_inst.VmInstanceConfig instead',
802 super(self.__class__, self).__init__(**kwargs)
805 class FloatingIpSettings(FloatingIpConfig):
807 Deprecated, use snaps.config.vm_inst.FloatingIpConfig instead
809 def __init__(self, **kwargs):
810 from warnings import warn
811 warn('Use snaps.config.vm_inst.FloatingIpConfig instead',
813 super(self.__class__, self).__init__(**kwargs)
816 class VmInstanceCreationError(Exception):
818 Exception to be thrown when an VM instance cannot be created
822 class FloatingIPAllocationError(Exception):
824 Exception to be thrown when an VM instance cannot allocate a floating IP