1 # Copyright (c) 2017 Cable Television Laboratories, Inc. ("CableLabs")
2 # and others. All rights reserved.
4 # Licensed under the Apache License, Version 2.0 (the "License");
5 # you may not use this file except in compliance with the License.
6 # You may obtain a copy of the License at:
8 # http://www.apache.org/licenses/LICENSE-2.0
10 # Unless required by applicable law or agreed to in writing, software
11 # distributed under the License is distributed on an "AS IS" BASIS,
12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 # See the License for the specific language governing permissions and
14 # limitations under the License.
18 from neutronclient.common.exceptions import PortNotFoundClient
19 from novaclient.exceptions import NotFound, BadRequest
21 from snaps.config.vm_inst import VmInstanceConfig, FloatingIpConfig
22 from snaps.openstack.openstack_creator import OpenStackComputeObject
23 from snaps.openstack.utils import glance_utils, cinder_utils, settings_utils
24 from snaps.openstack.utils import neutron_utils
25 from snaps.openstack.utils import nova_utils
26 from snaps.openstack.utils.nova_utils import RebootType
27 from snaps.provisioning import ansible_utils
29 __author__ = 'spisarski'
31 logger = logging.getLogger('create_instance')
34 STATUS_ACTIVE = 'ACTIVE'
35 STATUS_DELETED = 'DELETED'
38 class OpenStackVmInstance(OpenStackComputeObject):
40 Class responsible for managing a VM instance in OpenStack
43 def __init__(self, os_creds, instance_settings, image_settings,
44 keypair_settings=None):
47 :param os_creds: The connection credentials to the OpenStack API
48 :param instance_settings: Contains the settings for this VM
49 :param image_settings: The OpenStack image object settings
50 :param keypair_settings: The keypair metadata (Optional)
53 super(self.__class__, self).__init__(os_creds)
57 self.instance_settings = instance_settings
58 self.image_settings = image_settings
59 self.keypair_settings = keypair_settings
61 self.__floating_ip_dict = dict()
63 # Instantiated in self.create()
66 # Note: this object does not change after the VM becomes active
71 Loads the existing VMInst, Port, FloatingIps
72 :return: VMInst domain object
74 super(self.__class__, self).initialize()
76 self.__neutron = neutron_utils.neutron_client(self._os_creds)
78 self.__ports = self.__query_ports(self.instance_settings.port_settings)
79 self.__lookup_existing_vm_by_name()
81 def create(self, block=False):
83 Creates a VM instance and associated objects unless they already exist
84 :param block: Thread will block until instance has either become
85 active, error, or timeout waiting.
86 Additionally, when True, floating IPs will not be applied
88 :return: VMInst domain object
92 if len(self.__ports) == 0:
93 self.__ports = self.__create_ports(
94 self.instance_settings.port_settings)
96 self.__create_vm(block)
100 def __lookup_existing_vm_by_name(self):
102 Populates the member variables 'self.vm' and 'self.floating_ips' if a
103 VM with the same name already exists
106 server = nova_utils.get_server(
107 self._nova, self.__neutron,
108 vm_inst_settings=self.instance_settings)
110 if server.name == self.instance_settings.name:
113 'Found existing machine with name - %s',
114 self.instance_settings.name)
116 fips = neutron_utils.get_floating_ips(self.__neutron,
118 for port_id, fip in fips:
119 settings = self.instance_settings.floating_ip_settings
120 for fip_setting in settings:
121 if port_id == fip_setting.port_id:
122 self.__floating_ip_dict[fip_setting.name] = fip
124 port = neutron_utils.get_port_by_id(
125 self.__neutron, port_id)
126 if port and port.name == fip_setting.port_name:
127 self.__floating_ip_dict[fip_setting.name] = fip
129 def __create_vm(self, block=False):
131 Responsible for creating the VM instance
132 :param block: Thread will block until instance has either become
133 active, error, or timeout waiting. Floating IPs will be
134 assigned after active when block=True
136 glance = glance_utils.glance_client(self._os_creds)
137 self.__vm = nova_utils.create_server(
138 self._nova, self.__neutron, glance, self.instance_settings,
139 self.image_settings, self.keypair_settings)
140 logger.info('Created instance with name - %s',
141 self.instance_settings.name)
144 if not self.vm_active(block=True):
145 raise VmInstanceCreationError(
146 'Fatal error, VM did not become ACTIVE within the alloted '
149 # Create server should do this but found it needed to occur here
150 for sec_grp_name in self.instance_settings.security_group_names:
151 if self.vm_active(block=True):
152 nova_utils.add_security_group(self._nova, self.__vm,
155 raise VmInstanceCreationError(
156 'Cannot applying security group with name ' +
158 ' to VM that did not activate with name - ' +
159 self.instance_settings.name)
161 if self.instance_settings.volume_names:
162 for volume_name in self.instance_settings.volume_names:
163 cinder = cinder_utils.cinder_client(self._os_creds)
164 volume = cinder_utils.get_volume(
165 cinder, volume_name=volume_name)
167 if volume and self.vm_active(block=True):
169 vm = nova_utils.attach_volume(
170 self._nova, self.__neutron, self.__vm, volume, timeout)
175 logger.warn('Volume [%s] not attached within timeout '
176 'of [%s]', volume.name, timeout)
178 logger.warn('Unable to attach volume named [%s]',
181 self.__apply_floating_ips()
183 def __apply_floating_ips(self):
185 Applies the configured floating IPs to the necessary ports
188 for key, port in self.__ports:
189 port_dict[key] = port
192 for floating_ip_setting in self.instance_settings.floating_ip_settings:
193 self.add_floating_ip(floating_ip_setting)
195 def add_floating_ip(self, floating_ip_setting):
197 Adds a floating IP to a running instance
198 :param floating_ip_setting - the floating IP configuration
201 for key, port in self.__ports:
202 port_dict[key] = port
205 port = port_dict.get(floating_ip_setting.port_name)
208 raise VmInstanceCreationError(
209 'Cannot find port object with name - ' +
210 floating_ip_setting.port_name)
212 # Setup Floating IP only if there is a router with an external
214 ext_gateway = self.__ext_gateway_by_router(
215 floating_ip_setting.router_name)
217 subnet = neutron_utils.get_subnet(
219 subnet_name=floating_ip_setting.subnet_name)
220 floating_ip = neutron_utils.create_floating_ip(
221 self.__neutron, ext_gateway)
222 self.__floating_ip_dict[floating_ip_setting.name] = floating_ip
225 'Created floating IP %s via router - %s', floating_ip.ip,
226 floating_ip_setting.router_name)
227 self.__add_floating_ip(floating_ip, port, subnet)
229 raise VmInstanceCreationError(
230 'Unable to add floating IP to port, cannot locate router '
231 'with an external gateway ')
233 def __ext_gateway_by_router(self, router_name):
235 Returns network name for the external network attached to a router or
237 :param router_name: The name of the router to lookup
238 :return: the external network name or None
240 router = neutron_utils.get_router(
241 self.__neutron, router_name=router_name)
242 if router and router.external_network_id:
243 network = neutron_utils.get_network_by_id(
244 self.__neutron, router.external_network_id)
251 Destroys the VM instance
254 # Cleanup floating IPs
255 for name, floating_ip in self.__floating_ip_dict.items():
257 logger.info('Deleting Floating IP - ' + floating_ip.ip)
258 neutron_utils.delete_floating_ip(self.__neutron, floating_ip)
259 except Exception as e:
260 logger.error('Error deleting Floating IP - ' + str(e))
261 self.__floating_ip_dict = dict()
264 for volume_rec in self.__vm.volume_ids:
265 cinder = cinder_utils.cinder_client(self._os_creds)
266 volume = cinder_utils.get_volume_by_id(cinder, volume_rec['id'])
269 vm = nova_utils.detach_volume(
270 self._nova, self.__neutron, self.__vm, volume, 30)
275 'Timeout waiting to detach volume %s', volume.name)
276 except Exception as e:
277 logger.error('Unexpected error detaching volume %s '
278 'with error %s', volume.name, e)
280 logger.warn('Unable to detach volume with ID - [%s]',
284 for name, port in self.__ports:
285 logger.info('Deleting Port with ID - %s ', port.id)
287 neutron_utils.delete_port(self.__neutron, port)
288 except PortNotFoundClient as e:
289 logger.warning('Unexpected error deleting port - %s', e)
291 self.__ports = list()
297 'Deleting VM instance - ' + self.instance_settings.name)
298 nova_utils.delete_vm_instance(self._nova, self.__vm)
299 except Exception as e:
300 logger.error('Error deleting VM - %s', e)
302 # Block until instance cannot be found or returns the status of
304 logger.info('Checking deletion status')
307 if self.vm_deleted(block=True):
309 'VM has been properly deleted VM with name - %s',
310 self.instance_settings.name)
314 'VM not deleted within the timeout period of %s '
315 'seconds', self.instance_settings.vm_delete_timeout)
316 except Exception as e:
318 'Unexpected error while checking VM instance status - %s',
321 def __query_ports(self, port_settings):
323 Returns the previously configured ports or an empty list if none
325 :param port_settings: A list of PortSetting objects
326 :return: a list of OpenStack port tuples where the first member is the
327 port name and the second is the port object
331 for port_setting in port_settings:
332 port = neutron_utils.get_port(
333 self.__neutron, port_settings=port_setting)
335 ports.append((port_setting.name, port))
339 def __create_ports(self, port_settings):
341 Returns the previously configured ports or creates them if they do not
343 :param port_settings: A list of PortSetting objects
344 :return: a list of OpenStack port tuples where the first member is the
345 port name and the second is the port object
349 for port_setting in port_settings:
350 port = neutron_utils.get_port(
351 self.__neutron, port_settings=port_setting)
353 port = neutron_utils.create_port(
354 self.__neutron, self._os_creds, port_setting)
356 ports.append((port_setting.name, port))
360 def __add_floating_ip(self, floating_ip, port, subnet, timeout=30,
361 poll_interval=POLL_INTERVAL):
363 Returns True when active else False
364 TODO - Make timeout and poll_interval configurable...
369 # Take IP of subnet if there is one configured on which to place
371 for fixed_ip in port.ips:
372 if fixed_ip['subnet_id'] == subnet.id:
373 ip = fixed_ip['ip_address']
376 # Simply take the first
377 ip = port.ips[0]['ip_address']
380 count = timeout / poll_interval
382 logger.debug('Attempting to add floating IP to instance')
384 nova_utils.add_floating_ip_to_server(
385 self._nova, self.__vm, floating_ip, ip)
387 'Added floating IP %s to port IP %s on instance %s',
388 floating_ip.ip, ip, self.instance_settings.name)
390 except BadRequest as bre:
391 logger.error('Cannot add floating IP [%s]', bre)
393 except Exception as e:
395 'Retry adding floating IP to instance. Last attempt '
396 'failed with - %s', e)
397 time.sleep(poll_interval)
401 raise VmInstanceCreationError(
402 'Unable find IP address on which to place the floating IP')
404 logger.error('Timeout attempting to add the floating IP to instance.')
405 raise VmInstanceCreationError(
406 'Timeout while attempting add floating IP to instance')
408 def get_os_creds(self):
410 Returns the OpenStack credentials used to create these objects
411 :return: the credentials
413 return self._os_creds
415 def get_vm_inst(self):
417 Returns the latest version of this server object from OpenStack
418 :return: Server object
420 return nova_utils.get_server_object_by_id(
421 self._nova, self.__neutron, self.__vm.id)
423 def get_console_output(self):
425 Returns the vm console object for parsing logs
426 :return: the console output object
428 return nova_utils.get_server_console_output(self._nova, self.__vm)
430 def get_port_ip(self, port_name, subnet_name=None):
432 Returns the first IP for the port corresponding with the port_name
433 parameter when subnet_name is None else returns the IP address that
434 corresponds to the subnet_name parameter
435 :param port_name: the name of the port from which to return the IP
436 :param subnet_name: the name of the subnet attached to this IP
437 :return: the IP or None if not found
439 port = self.get_port_by_name(port_name)
442 subnet = neutron_utils.get_subnet(
443 self.__neutron, subnet_name=subnet_name)
445 logger.warning('Cannot retrieve port IP as subnet could '
446 'not be located with name - %s',
449 for fixed_ip in port.ips:
450 if fixed_ip['subnet_id'] == subnet.id:
451 return fixed_ip['ip_address']
453 if port.ips and len(port.ips) > 0:
454 return port.ips[0]['ip_address']
457 def get_port_mac(self, port_name):
459 Returns the first IP for the port corresponding with the port_name
461 TODO - Add in the subnet as an additional parameter as a port may have
463 :param port_name: the name of the port from which to return the IP
464 :return: the IP or None if not found
466 port = self.get_port_by_name(port_name)
468 return port.mac_address
471 def get_port_by_name(self, port_name):
473 Retrieves the OpenStack port object by its given name
474 :param port_name: the name of the port
475 :return: the OpenStack port object or None if not exists
477 for key, port in self.__ports:
480 logger.warning('Cannot find port with name - ' + port_name)
483 def get_vm_info(self):
485 Returns a dictionary of a VMs info as returned by OpenStack
488 return nova_utils.get_server_info(self._nova, self.__vm)
490 def config_nics(self):
492 Responsible for configuring NICs on RPM systems where the instance has
493 more than one configured port
494 :return: the value returned by ansible_utils.apply_ansible_playbook()
496 if len(self.__ports) > 1 and len(self.__floating_ip_dict) > 0:
497 if self.vm_active(block=True) and self.vm_ssh_active(block=True):
498 for key, port in self.__ports:
499 port_index = self.__ports.index((key, port))
501 nic_name = 'eth' + repr(port_index)
502 retval = self.__config_nic(
504 self.__get_first_provisioning_floating_ip().ip)
505 logger.info('Configured NIC - %s on VM - %s',
506 nic_name, self.instance_settings.name)
509 def __get_first_provisioning_floating_ip(self):
511 Returns the first floating IP tagged with the Floating IP name if
512 exists else the first one found
515 for floating_ip_setting in self.instance_settings.floating_ip_settings:
516 if floating_ip_setting.provisioning:
517 fip = self.__floating_ip_dict.get(floating_ip_setting.name)
520 elif len(self.__floating_ip_dict) > 0:
521 for key, fip in self.__floating_ip_dict.items():
524 # When cannot be found above
525 if len(self.__floating_ip_dict) > 0:
526 for key, fip in self.__floating_ip_dict.items():
529 def __config_nic(self, nic_name, port, ip):
531 Although ports/NICs can contain multiple IPs, this code currently only
534 :param nic_name: Name of the interface
535 :param port: The port information containing the expected IP values.
536 :param ip: The IP on which to apply the playbook.
537 :return: the return value from ansible
539 port_ip = port.ips[0]['ip_address']
542 'nic_name': nic_name,
546 if self.image_settings.nic_config_pb_loc and self.keypair_settings:
547 return self.apply_ansible_playbook(
548 self.image_settings.nic_config_pb_loc, variables)
551 'VM %s cannot self configure NICs eth1++. No playbook or '
552 'keypairs found.', self.instance_settings.name)
554 def apply_ansible_playbook(self, pb_file_loc, variables=None,
557 Applies a playbook to a VM
558 :param pb_file_loc: the file location of the playbook to be applied
559 :param variables: a dict() of substitution values required by the
561 :param fip_name: the name of the floating IP to use for applying the
562 playbook (default - will take the first)
563 :return: the return value from ansible
565 return ansible_utils.apply_playbook(
566 pb_file_loc, [self.get_floating_ip(fip_name=fip_name).ip],
567 self.get_image_user(), self.keypair_settings.private_filepath,
568 variables, self._os_creds.proxy_settings)
570 def get_image_user(self):
572 Returns the instance sudo_user if it has been configured in the
573 instance_settings else it returns the image_settings.image_user value
575 if self.instance_settings.sudo_user:
576 return self.instance_settings.sudo_user
578 return self.image_settings.image_user
580 def vm_deleted(self, block=False, poll_interval=POLL_INTERVAL):
582 Returns true when the VM status returns the value of
583 expected_status_code or instance retrieval throws a NotFound exception.
584 :param block: When true, thread will block until active or timeout
585 value in seconds has been exceeded (False)
586 :param poll_interval: The polling interval in seconds
590 return self.__vm_status_check(
591 STATUS_DELETED, block,
592 self.instance_settings.vm_delete_timeout, poll_interval)
593 except NotFound as e:
595 "Instance not found when querying status for %s with message "
596 "%s", STATUS_DELETED, e)
599 def vm_active(self, block=False, poll_interval=POLL_INTERVAL):
601 Returns true when the VM status returns the value of the constant
603 :param block: When true, thread will block until active or timeout
604 value in seconds has been exceeded (False)
605 :param poll_interval: The polling interval in seconds
608 if self.__vm_status_check(
609 STATUS_ACTIVE, block, self.instance_settings.vm_boot_timeout,
611 self.__vm = nova_utils.get_server_object_by_id(
612 self._nova, self.__neutron, self.__vm.id)
616 def __vm_status_check(self, expected_status_code, block, timeout,
619 Returns true when the VM status returns the value of
621 :param expected_status_code: instance status evaluated with this
623 :param block: When true, thread will block until active or timeout
624 value in seconds has been exceeded (False)
625 :param timeout: The timeout value
626 :param poll_interval: The polling interval in seconds
629 # sleep and wait for VM status change
633 return self.__status(expected_status_code)
635 while timeout > time.time() - start:
636 status = self.__status(expected_status_code)
638 logger.info('VM is - ' + expected_status_code)
641 logger.debug('Retry querying VM status in ' + str(
642 poll_interval) + ' seconds')
643 time.sleep(poll_interval)
644 logger.debug('VM status query timeout in ' + str(
645 timeout - (time.time() - start)))
648 'Timeout checking for VM status for ' + expected_status_code)
651 def __status(self, expected_status_code):
653 Returns True when active else False
654 :param expected_status_code: instance status evaluated with this string
659 if expected_status_code == STATUS_DELETED:
664 status = nova_utils.get_server_status(self._nova, self.__vm)
666 logger.warning('Cannot find instance with id - ' + self.__vm.id)
669 if status == 'ERROR':
670 raise VmInstanceCreationError(
671 'Instance had an error during deployment')
673 'Instance status [%s] is - %s', self.instance_settings.name,
675 return status == expected_status_code
677 def vm_ssh_active(self, block=False, poll_interval=POLL_INTERVAL):
679 Returns true when the VM can be accessed via SSH
680 :param block: When true, thread will block until active or timeout
681 value in seconds has been exceeded (False)
682 :param poll_interval: The polling interval
685 # sleep and wait for VM status change
686 logger.info('Checking if VM is active')
688 timeout = self.instance_settings.ssh_connect_timeout
690 if self.vm_active(block=True):
694 start = time.time() - timeout
696 while timeout > time.time() - start:
697 status = self.__ssh_active()
699 logger.info('SSH is active for VM instance')
702 logger.debug('Retry SSH connection in ' + str(
703 poll_interval) + ' seconds')
704 time.sleep(poll_interval)
705 logger.debug('SSH connection timeout in ' + str(
706 timeout - (time.time() - start)))
708 logger.error('Timeout attempting to connect with VM via SSH')
711 def __ssh_active(self):
713 Returns True when can create a SSH session else False
716 if len(self.__floating_ip_dict) > 0:
717 ssh = self.ssh_client()
723 def get_floating_ip(self, fip_name=None):
725 Returns the floating IP object byt name if found, else the first known,
727 :param fip_name: the name of the floating IP to return
728 :return: the SSH client or None
730 if fip_name and self.__floating_ip_dict.get(fip_name):
731 return self.__floating_ip_dict.get(fip_name)
733 return self.__get_first_provisioning_floating_ip()
735 def ssh_client(self, fip_name=None):
737 Returns an SSH client using the name or the first known floating IP if
739 :param fip_name: the name of the floating IP to return
740 :return: the SSH client or None
742 fip = self.get_floating_ip(fip_name)
744 return ansible_utils.ssh_client(
745 self.__get_first_provisioning_floating_ip().ip,
746 self.get_image_user(),
747 self.keypair_settings.private_filepath,
748 proxy_settings=self._os_creds.proxy_settings)
750 FloatingIPAllocationError(
751 'Cannot return an SSH client. No Floating IP configured')
753 def add_security_group(self, security_group):
755 Adds a security group to this VM. Call will block until VM is active.
756 :param security_group: the SNAPS SecurityGroup domain object
757 :return True if successful else False
759 self.vm_active(block=True)
761 if not security_group:
762 logger.warning('Security group object is None, cannot add')
766 nova_utils.add_security_group(self._nova, self.get_vm_inst(),
769 except NotFound as e:
770 logger.warning('Security group not added - ' + str(e))
773 def remove_security_group(self, security_group):
775 Removes a security group to this VM. Call will block until VM is active
776 :param security_group: the OpenStack security group object
777 :return True if successful else False
779 self.vm_active(block=True)
781 if not security_group:
782 logger.warning('Security group object is None, cannot remove')
786 nova_utils.remove_security_group(self._nova, self.get_vm_inst(),
789 except NotFound as e:
790 logger.warning('Security group not removed - ' + str(e))
793 def reboot(self, reboot_type=RebootType.soft):
796 :param reboot_type: instance of
797 snaps.openstack.utils.nova_utils.RebootType
801 nova_utils.reboot_server(
802 self._nova, self.__vm, reboot_type=reboot_type)
805 def generate_creator(os_creds, vm_inst, image_config, keypair_config=None):
807 Initializes an OpenStackVmInstance object
808 :param os_creds: the OpenStack credentials
809 :param vm_inst: the SNAPS-OO VmInst domain object
810 :param image_config: the associated ImageConfig object
811 :param keypair_config: the associated KeypairConfig object (optional)
812 :return: an initialized OpenStackVmInstance object
814 nova = nova_utils.nova_client(os_creds)
815 neutron = neutron_utils.neutron_client(os_creds)
816 derived_inst_config = settings_utils.create_vm_inst_config(
817 nova, neutron, vm_inst)
819 derived_inst_creator = OpenStackVmInstance(
820 os_creds, derived_inst_config, image_config, keypair_config)
821 derived_inst_creator.initialize()
822 return derived_inst_creator
825 class VmInstanceSettings(VmInstanceConfig):
827 Deprecated, use snaps.config.vm_inst.VmInstanceConfig instead
829 def __init__(self, **kwargs):
830 from warnings import warn
831 warn('Use snaps.config.vm_inst.VmInstanceConfig instead',
833 super(self.__class__, self).__init__(**kwargs)
836 class FloatingIpSettings(FloatingIpConfig):
838 Deprecated, use snaps.config.vm_inst.FloatingIpConfig instead
840 def __init__(self, **kwargs):
841 from warnings import warn
842 warn('Use snaps.config.vm_inst.FloatingIpConfig instead',
844 super(self.__class__, self).__init__(**kwargs)
847 class VmInstanceCreationError(Exception):
849 Exception to be thrown when an VM instance cannot be created
853 class FloatingIPAllocationError(Exception):
855 Exception to be thrown when an VM instance cannot allocate a floating IP