1 # Copyright (c) 2017 Cable Television Laboratories, Inc. ("CableLabs")
2 # and others. All rights reserved.
4 # Licensed under the Apache License, Version 2.0 (the "License");
5 # you may not use this file except in compliance with the License.
6 # You may obtain a copy of the License at:
8 # http://www.apache.org/licenses/LICENSE-2.0
10 # Unless required by applicable law or agreed to in writing, software
11 # distributed under the License is distributed on an "AS IS" BASIS,
12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 # See the License for the specific language governing permissions and
14 # limitations under the License.
18 from neutronclient.common.exceptions import PortNotFoundClient
19 from novaclient.exceptions import NotFound, BadRequest
21 from snaps.config.vm_inst import VmInstanceConfig, FloatingIpConfig
22 from snaps.openstack.openstack_creator import OpenStackComputeObject
23 from snaps.openstack.utils import glance_utils, cinder_utils, settings_utils
24 from snaps.openstack.utils import neutron_utils
25 from snaps.openstack.utils import nova_utils
26 from snaps.openstack.utils.nova_utils import RebootType
27 from snaps.provisioning import ansible_utils
29 __author__ = 'spisarski'
31 logger = logging.getLogger('create_instance')
34 STATUS_ACTIVE = 'ACTIVE'
35 STATUS_DELETED = 'DELETED'
38 class OpenStackVmInstance(OpenStackComputeObject):
40 Class responsible for managing a VM instance in OpenStack
43 def __init__(self, os_creds, instance_settings, image_settings,
44 keypair_settings=None):
47 :param os_creds: The connection credentials to the OpenStack API
48 :param instance_settings: Contains the settings for this VM
49 :param image_settings: The OpenStack image object settings
50 :param keypair_settings: The keypair metadata (Optional)
53 super(self.__class__, self).__init__(os_creds)
57 self.instance_settings = instance_settings
58 self.image_settings = image_settings
59 self.keypair_settings = keypair_settings
61 self.__floating_ip_dict = dict()
63 # Instantiated in self.create()
66 # Note: this object does not change after the VM becomes active
71 Loads the existing VMInst, Port, FloatingIps
72 :return: VMInst domain object
74 super(self.__class__, self).initialize()
76 self.__neutron = neutron_utils.neutron_client(self._os_creds)
78 self.__ports = self.__query_ports(self.instance_settings.port_settings)
79 self.__lookup_existing_vm_by_name()
81 def create(self, block=False):
83 Creates a VM instance and associated objects unless they already exist
84 :param block: Thread will block until instance has either become
85 active, error, or timeout waiting.
86 Additionally, when True, floating IPs will not be applied
88 :return: VMInst domain object
92 if len(self.__ports) == 0:
93 self.__ports = self.__create_ports(
94 self.instance_settings.port_settings)
96 self.__create_vm(block)
100 def __lookup_existing_vm_by_name(self):
102 Populates the member variables 'self.vm' and 'self.floating_ips' if a
103 VM with the same name already exists
106 server = nova_utils.get_server(
107 self._nova, self.__neutron,
108 vm_inst_settings=self.instance_settings)
110 if server.name == self.instance_settings.name:
113 'Found existing machine with name - %s',
114 self.instance_settings.name)
116 fips = neutron_utils.get_floating_ips(self.__neutron,
118 for port_id, fip in fips:
119 settings = self.instance_settings.floating_ip_settings
120 for fip_setting in settings:
121 if port_id == fip_setting.port_id:
122 self.__floating_ip_dict[fip_setting.name] = fip
124 port = neutron_utils.get_port_by_id(
125 self.__neutron, port_id)
126 if port and port.name == fip_setting.port_name:
127 self.__floating_ip_dict[fip_setting.name] = fip
129 def __create_vm(self, block=False):
131 Responsible for creating the VM instance
132 :param block: Thread will block until instance has either become
133 active, error, or timeout waiting. Floating IPs will be
134 assigned after active when block=True
136 glance = glance_utils.glance_client(self._os_creds)
137 self.__vm = nova_utils.create_server(
138 self._nova, self.__neutron, glance, self.instance_settings,
139 self.image_settings, self.keypair_settings)
140 logger.info('Created instance with name - %s',
141 self.instance_settings.name)
144 if not self.vm_active(block=True):
145 raise VmInstanceCreationError(
146 'Fatal error, VM did not become ACTIVE within the alloted '
149 # Create server should do this but found it needed to occur here
150 for sec_grp_name in self.instance_settings.security_group_names:
151 if self.vm_active(block=True):
152 nova_utils.add_security_group(self._nova, self.__vm,
155 raise VmInstanceCreationError(
156 'Cannot applying security group with name ' +
158 ' to VM that did not activate with name - ' +
159 self.instance_settings.name)
161 if self.instance_settings.volume_names:
162 for volume_name in self.instance_settings.volume_names:
163 cinder = cinder_utils.cinder_client(self._os_creds)
164 volume = cinder_utils.get_volume(
165 cinder, volume_name=volume_name)
167 if volume and self.vm_active(block=True):
169 vm = nova_utils.attach_volume(
170 self._nova, self.__neutron, self.__vm, volume, timeout)
175 logger.warn('Volume [%s] not attached within timeout '
176 'of [%s]', volume.name, timeout)
178 logger.warn('Unable to attach volume named [%s]',
181 self.__apply_floating_ips()
183 def __apply_floating_ips(self):
185 Applies the configured floating IPs to the necessary ports
188 for key, port in self.__ports:
189 port_dict[key] = port
192 for floating_ip_setting in self.instance_settings.floating_ip_settings:
193 self.add_floating_ip(floating_ip_setting)
195 def add_floating_ip(self, floating_ip_setting):
197 Adds a floating IP to a running instance
198 :param floating_ip_setting - the floating IP configuration
199 :return: the floating ip object
202 for key, port in self.__ports:
203 port_dict[key] = port
206 port = port_dict.get(floating_ip_setting.port_name)
209 raise VmInstanceCreationError(
210 'Cannot find port object with name - ' +
211 floating_ip_setting.port_name)
213 # Setup Floating IP only if there is a router with an external
215 ext_gateway = self.__ext_gateway_by_router(
216 floating_ip_setting.router_name)
218 subnet = neutron_utils.get_subnet(
220 subnet_name=floating_ip_setting.subnet_name)
221 floating_ip = neutron_utils.create_floating_ip(
222 self.__neutron, ext_gateway)
223 self.__floating_ip_dict[floating_ip_setting.name] = floating_ip
226 'Created floating IP %s via router - %s', floating_ip.ip,
227 floating_ip_setting.router_name)
228 self.__add_floating_ip(floating_ip, port, subnet)
231 raise VmInstanceCreationError(
232 'Unable to add floating IP to port, cannot locate router '
233 'with an external gateway ')
235 def __ext_gateway_by_router(self, router_name):
237 Returns network name for the external network attached to a router or
239 :param router_name: The name of the router to lookup
240 :return: the external network name or None
242 router = neutron_utils.get_router(
243 self.__neutron, router_name=router_name)
244 if router and router.external_network_id:
245 network = neutron_utils.get_network_by_id(
246 self.__neutron, router.external_network_id)
253 Destroys the VM instance
256 # Cleanup floating IPs
257 for name, floating_ip in self.__floating_ip_dict.items():
259 logger.info('Deleting Floating IP - ' + floating_ip.ip)
260 neutron_utils.delete_floating_ip(self.__neutron, floating_ip)
261 except Exception as e:
262 logger.error('Error deleting Floating IP - ' + str(e))
263 self.__floating_ip_dict = dict()
266 for volume_rec in self.__vm.volume_ids:
267 cinder = cinder_utils.cinder_client(self._os_creds)
268 volume = cinder_utils.get_volume_by_id(cinder, volume_rec['id'])
271 vm = nova_utils.detach_volume(
272 self._nova, self.__neutron, self.__vm, volume, 30)
277 'Timeout waiting to detach volume %s', volume.name)
278 except Exception as e:
279 logger.error('Unexpected error detaching volume %s '
280 'with error %s', volume.name, e)
282 logger.warn('Unable to detach volume with ID - [%s]',
286 for name, port in self.__ports:
287 logger.info('Deleting Port with ID - %s ', port.id)
289 neutron_utils.delete_port(self.__neutron, port)
290 except PortNotFoundClient as e:
291 logger.warning('Unexpected error deleting port - %s', e)
293 self.__ports = list()
299 'Deleting VM instance - ' + self.instance_settings.name)
300 nova_utils.delete_vm_instance(self._nova, self.__vm)
301 except Exception as e:
302 logger.error('Error deleting VM - %s', e)
304 # Block until instance cannot be found or returns the status of
306 logger.info('Checking deletion status')
309 if self.vm_deleted(block=True):
311 'VM has been properly deleted VM with name - %s',
312 self.instance_settings.name)
316 'VM not deleted within the timeout period of %s '
317 'seconds', self.instance_settings.vm_delete_timeout)
318 except Exception as e:
320 'Unexpected error while checking VM instance status - %s',
323 def __query_ports(self, port_settings):
325 Returns the previously configured ports or an empty list if none
327 :param port_settings: A list of PortSetting objects
328 :return: a list of OpenStack port tuples where the first member is the
329 port name and the second is the port object
333 for port_setting in port_settings:
334 port = neutron_utils.get_port(
335 self.__neutron, port_settings=port_setting)
337 ports.append((port_setting.name, port))
341 def __create_ports(self, port_settings):
343 Returns the previously configured ports or creates them if they do not
345 :param port_settings: A list of PortSetting objects
346 :return: a list of OpenStack port tuples where the first member is the
347 port name and the second is the port object
351 for port_setting in port_settings:
352 port = neutron_utils.get_port(
353 self.__neutron, port_settings=port_setting)
355 port = neutron_utils.create_port(
356 self.__neutron, self._os_creds, port_setting)
358 ports.append((port_setting.name, port))
362 def __add_floating_ip(self, floating_ip, port, subnet, timeout=30,
363 poll_interval=POLL_INTERVAL):
365 Returns True when active else False
366 TODO - Make timeout and poll_interval configurable...
371 # Take IP of subnet if there is one configured on which to place
373 for fixed_ip in port.ips:
374 if fixed_ip['subnet_id'] == subnet.id:
375 ip = fixed_ip['ip_address']
378 # Simply take the first
379 ip = port.ips[0]['ip_address']
382 count = timeout / poll_interval
384 logger.debug('Attempting to add floating IP to instance')
386 nova_utils.add_floating_ip_to_server(
387 self._nova, self.__vm, floating_ip, ip)
389 'Added floating IP %s to port IP %s on instance %s',
390 floating_ip.ip, ip, self.instance_settings.name)
392 except BadRequest as bre:
393 logger.error('Cannot add floating IP [%s]', bre)
395 except Exception as e:
397 'Retry adding floating IP to instance. Last attempt '
398 'failed with - %s', e)
399 time.sleep(poll_interval)
403 raise VmInstanceCreationError(
404 'Unable find IP address on which to place the floating IP')
406 logger.error('Timeout attempting to add the floating IP to instance.')
407 raise VmInstanceCreationError(
408 'Timeout while attempting add floating IP to instance')
410 def get_os_creds(self):
412 Returns the OpenStack credentials used to create these objects
413 :return: the credentials
415 return self._os_creds
417 def get_vm_inst(self):
419 Returns the latest version of this server object from OpenStack
420 :return: Server object
422 return nova_utils.get_server_object_by_id(
423 self._nova, self.__neutron, self.__vm.id)
425 def get_console_output(self):
427 Returns the vm console object for parsing logs
428 :return: the console output object
430 return nova_utils.get_server_console_output(self._nova, self.__vm)
432 def get_port_ip(self, port_name, subnet_name=None):
434 Returns the first IP for the port corresponding with the port_name
435 parameter when subnet_name is None else returns the IP address that
436 corresponds to the subnet_name parameter
437 :param port_name: the name of the port from which to return the IP
438 :param subnet_name: the name of the subnet attached to this IP
439 :return: the IP or None if not found
441 port = self.get_port_by_name(port_name)
444 subnet = neutron_utils.get_subnet(
445 self.__neutron, subnet_name=subnet_name)
447 logger.warning('Cannot retrieve port IP as subnet could '
448 'not be located with name - %s',
451 for fixed_ip in port.ips:
452 if fixed_ip['subnet_id'] == subnet.id:
453 return fixed_ip['ip_address']
455 if port.ips and len(port.ips) > 0:
456 return port.ips[0]['ip_address']
459 def get_port_mac(self, port_name):
461 Returns the first IP for the port corresponding with the port_name
463 TODO - Add in the subnet as an additional parameter as a port may have
465 :param port_name: the name of the port from which to return the IP
466 :return: the IP or None if not found
468 port = self.get_port_by_name(port_name)
470 return port.mac_address
473 def get_port_by_name(self, port_name):
475 Retrieves the OpenStack port object by its given name
476 :param port_name: the name of the port
477 :return: the OpenStack port object or None if not exists
479 for key, port in self.__ports:
482 logger.warning('Cannot find port with name - ' + port_name)
485 def get_vm_info(self):
487 Returns a dictionary of a VMs info as returned by OpenStack
490 return nova_utils.get_server_info(self._nova, self.__vm)
492 def config_nics(self):
494 Responsible for configuring NICs on RPM systems where the instance has
495 more than one configured port
496 :return: the value returned by ansible_utils.apply_ansible_playbook()
498 if len(self.__ports) > 1 and len(self.__floating_ip_dict) > 0:
499 if self.vm_active(block=True) and self.vm_ssh_active(block=True):
500 for key, port in self.__ports:
501 port_index = self.__ports.index((key, port))
503 nic_name = 'eth' + repr(port_index)
504 retval = self.__config_nic(
506 self.__get_first_provisioning_floating_ip().ip)
507 logger.info('Configured NIC - %s on VM - %s',
508 nic_name, self.instance_settings.name)
511 def __get_first_provisioning_floating_ip(self):
513 Returns the first floating IP tagged with the Floating IP name if
514 exists else the first one found
517 for floating_ip_setting in self.instance_settings.floating_ip_settings:
518 if floating_ip_setting.provisioning:
519 fip = self.__floating_ip_dict.get(floating_ip_setting.name)
522 elif len(self.__floating_ip_dict) > 0:
523 for key, fip in self.__floating_ip_dict.items():
526 # When cannot be found above
527 if len(self.__floating_ip_dict) > 0:
528 for key, fip in self.__floating_ip_dict.items():
531 def __config_nic(self, nic_name, port, ip):
533 Although ports/NICs can contain multiple IPs, this code currently only
536 :param nic_name: Name of the interface
537 :param port: The port information containing the expected IP values.
538 :param ip: The IP on which to apply the playbook.
539 :return: the return value from ansible
541 port_ip = port.ips[0]['ip_address']
544 'nic_name': nic_name,
548 if self.image_settings.nic_config_pb_loc and self.keypair_settings:
549 return self.apply_ansible_playbook(
550 self.image_settings.nic_config_pb_loc, variables)
553 'VM %s cannot self configure NICs eth1++. No playbook or '
554 'keypairs found.', self.instance_settings.name)
556 def apply_ansible_playbook(self, pb_file_loc, variables=None,
559 Applies a playbook to a VM
560 :param pb_file_loc: the file location of the playbook to be applied
561 :param variables: a dict() of substitution values required by the
563 :param fip_name: the name of the floating IP to use for applying the
564 playbook (default - will take the first)
565 :return: the return value from ansible
567 return ansible_utils.apply_playbook(
568 pb_file_loc, [self.get_floating_ip(fip_name=fip_name).ip],
569 self.get_image_user(), self.keypair_settings.private_filepath,
570 variables, self._os_creds.proxy_settings)
572 def get_image_user(self):
574 Returns the instance sudo_user if it has been configured in the
575 instance_settings else it returns the image_settings.image_user value
577 if self.instance_settings.sudo_user:
578 return self.instance_settings.sudo_user
580 return self.image_settings.image_user
582 def vm_deleted(self, block=False, poll_interval=POLL_INTERVAL):
584 Returns true when the VM status returns the value of
585 expected_status_code or instance retrieval throws a NotFound exception.
586 :param block: When true, thread will block until active or timeout
587 value in seconds has been exceeded (False)
588 :param poll_interval: The polling interval in seconds
592 return self.__vm_status_check(
593 STATUS_DELETED, block,
594 self.instance_settings.vm_delete_timeout, poll_interval)
595 except NotFound as e:
597 "Instance not found when querying status for %s with message "
598 "%s", STATUS_DELETED, e)
601 def vm_active(self, block=False, poll_interval=POLL_INTERVAL):
603 Returns true when the VM status returns the value of the constant
605 :param block: When true, thread will block until active or timeout
606 value in seconds has been exceeded (False)
607 :param poll_interval: The polling interval in seconds
610 if self.__vm_status_check(
611 STATUS_ACTIVE, block, self.instance_settings.vm_boot_timeout,
613 self.__vm = nova_utils.get_server_object_by_id(
614 self._nova, self.__neutron, self.__vm.id)
618 def __vm_status_check(self, expected_status_code, block, timeout,
621 Returns true when the VM status returns the value of
623 :param expected_status_code: instance status evaluated with this
625 :param block: When true, thread will block until active or timeout
626 value in seconds has been exceeded (False)
627 :param timeout: The timeout value
628 :param poll_interval: The polling interval in seconds
631 # sleep and wait for VM status change
635 return self.__status(expected_status_code)
637 while timeout > time.time() - start:
638 status = self.__status(expected_status_code)
640 logger.info('VM is - ' + expected_status_code)
643 logger.debug('Retry querying VM status in ' + str(
644 poll_interval) + ' seconds')
645 time.sleep(poll_interval)
646 logger.debug('VM status query timeout in ' + str(
647 timeout - (time.time() - start)))
650 'Timeout checking for VM status for ' + expected_status_code)
653 def __status(self, expected_status_code):
655 Returns True when active else False
656 :param expected_status_code: instance status evaluated with this string
661 if expected_status_code == STATUS_DELETED:
666 status = nova_utils.get_server_status(self._nova, self.__vm)
668 logger.warning('Cannot find instance with id - ' + self.__vm.id)
671 if status == 'ERROR':
672 raise VmInstanceCreationError(
673 'Instance had an error during deployment')
675 'Instance status [%s] is - %s', self.instance_settings.name,
677 return status == expected_status_code
679 def vm_ssh_active(self, block=False, poll_interval=POLL_INTERVAL):
681 Returns true when the VM can be accessed via SSH
682 :param block: When true, thread will block until active or timeout
683 value in seconds has been exceeded (False)
684 :param poll_interval: The polling interval
687 # sleep and wait for VM status change
688 logger.info('Checking if VM is active')
690 timeout = self.instance_settings.ssh_connect_timeout
692 if self.vm_active(block=True):
696 start = time.time() - timeout
698 while timeout > time.time() - start:
699 status = self.__ssh_active()
701 logger.info('SSH is active for VM instance')
704 logger.debug('Retry SSH connection in ' + str(
705 poll_interval) + ' seconds')
706 time.sleep(poll_interval)
707 logger.debug('SSH connection timeout in ' + str(
708 timeout - (time.time() - start)))
710 logger.error('Timeout attempting to connect with VM via SSH')
713 def __ssh_active(self):
715 Returns True when can create a SSH session else False
718 if len(self.__floating_ip_dict) > 0:
719 ssh = self.ssh_client()
725 def get_floating_ip(self, fip_name=None):
727 Returns the floating IP object byt name if found, else the first known,
729 :param fip_name: the name of the floating IP to return
730 :return: the SSH client or None
732 if fip_name and self.__floating_ip_dict.get(fip_name):
733 return self.__floating_ip_dict.get(fip_name)
735 return self.__get_first_provisioning_floating_ip()
737 def ssh_client(self, fip_name=None):
739 Returns an SSH client using the name or the first known floating IP if
741 :param fip_name: the name of the floating IP to return
742 :return: the SSH client or None
744 fip = self.get_floating_ip(fip_name)
746 return ansible_utils.ssh_client(
747 self.__get_first_provisioning_floating_ip().ip,
748 self.get_image_user(),
749 self.keypair_settings.private_filepath,
750 proxy_settings=self._os_creds.proxy_settings)
752 FloatingIPAllocationError(
753 'Cannot return an SSH client. No Floating IP configured')
755 def add_security_group(self, security_group):
757 Adds a security group to this VM. Call will block until VM is active.
758 :param security_group: the SNAPS SecurityGroup domain object
759 :return True if successful else False
761 self.vm_active(block=True)
763 if not security_group:
764 logger.warning('Security group object is None, cannot add')
768 nova_utils.add_security_group(self._nova, self.get_vm_inst(),
771 except NotFound as e:
772 logger.warning('Security group not added - ' + str(e))
775 def remove_security_group(self, security_group):
777 Removes a security group to this VM. Call will block until VM is active
778 :param security_group: the OpenStack security group object
779 :return True if successful else False
781 self.vm_active(block=True)
783 if not security_group:
784 logger.warning('Security group object is None, cannot remove')
788 nova_utils.remove_security_group(self._nova, self.get_vm_inst(),
791 except NotFound as e:
792 logger.warning('Security group not removed - ' + str(e))
795 def reboot(self, reboot_type=RebootType.soft):
798 :param reboot_type: instance of
799 snaps.openstack.utils.nova_utils.RebootType
803 nova_utils.reboot_server(
804 self._nova, self.__vm, reboot_type=reboot_type)
807 def generate_creator(os_creds, vm_inst, image_config, keypair_config=None):
809 Initializes an OpenStackVmInstance object
810 :param os_creds: the OpenStack credentials
811 :param vm_inst: the SNAPS-OO VmInst domain object
812 :param image_config: the associated ImageConfig object
813 :param keypair_config: the associated KeypairConfig object (optional)
814 :return: an initialized OpenStackVmInstance object
816 nova = nova_utils.nova_client(os_creds)
817 neutron = neutron_utils.neutron_client(os_creds)
818 derived_inst_config = settings_utils.create_vm_inst_config(
819 nova, neutron, vm_inst)
821 derived_inst_creator = OpenStackVmInstance(
822 os_creds, derived_inst_config, image_config, keypair_config)
823 derived_inst_creator.initialize()
824 return derived_inst_creator
827 class VmInstanceSettings(VmInstanceConfig):
829 Deprecated, use snaps.config.vm_inst.VmInstanceConfig instead
831 def __init__(self, **kwargs):
832 from warnings import warn
833 warn('Use snaps.config.vm_inst.VmInstanceConfig instead',
835 super(self.__class__, self).__init__(**kwargs)
838 class FloatingIpSettings(FloatingIpConfig):
840 Deprecated, use snaps.config.vm_inst.FloatingIpConfig instead
842 def __init__(self, **kwargs):
843 from warnings import warn
844 warn('Use snaps.config.vm_inst.FloatingIpConfig instead',
846 super(self.__class__, self).__init__(**kwargs)
849 class VmInstanceCreationError(Exception):
851 Exception to be thrown when an VM instance cannot be created
855 class FloatingIPAllocationError(Exception):
857 Exception to be thrown when an VM instance cannot allocate a floating IP