1 # Copyright (c) 2017 Cable Television Laboratories, Inc. ("CableLabs")
2 # and others. All rights reserved.
4 # Licensed under the Apache License, Version 2.0 (the "License");
5 # you may not use this file except in compliance with the License.
6 # You may obtain a copy of the License at:
8 # http://www.apache.org/licenses/LICENSE-2.0
10 # Unless required by applicable law or agreed to in writing, software
11 # distributed under the License is distributed on an "AS IS" BASIS,
12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 # See the License for the specific language governing permissions and
14 # limitations under the License.
18 from novaclient.exceptions import NotFound, BadRequest
20 from snaps.config.vm_inst import VmInstanceConfig, FloatingIpConfig
21 from snaps.openstack.openstack_creator import OpenStackComputeObject
22 from snaps.openstack.utils import glance_utils, cinder_utils, settings_utils
23 from snaps.openstack.utils import neutron_utils
24 from snaps.openstack.utils import nova_utils
25 from snaps.openstack.utils.nova_utils import RebootType
26 from snaps.provisioning import ansible_utils
28 __author__ = 'spisarski'
30 logger = logging.getLogger('create_instance')
33 STATUS_ACTIVE = 'ACTIVE'
34 STATUS_DELETED = 'DELETED'
37 class OpenStackVmInstance(OpenStackComputeObject):
39 Class responsible for managing a VM instance in OpenStack
42 def __init__(self, os_creds, instance_settings, image_settings,
43 keypair_settings=None):
46 :param os_creds: The connection credentials to the OpenStack API
47 :param instance_settings: Contains the settings for this VM
48 :param image_settings: The OpenStack image object settings
49 :param keypair_settings: The keypair metadata (Optional)
52 super(self.__class__, self).__init__(os_creds)
56 self.instance_settings = instance_settings
57 self.image_settings = image_settings
58 self.keypair_settings = keypair_settings
60 self.__floating_ip_dict = dict()
62 # Instantiated in self.create()
65 # Note: this object does not change after the VM becomes active
70 Loads the existing VMInst, Port, FloatingIps
71 :return: VMInst domain object
73 super(self.__class__, self).initialize()
75 self.__neutron = neutron_utils.neutron_client(self._os_creds)
77 self.__ports = self.__query_ports(self.instance_settings.port_settings)
78 self.__lookup_existing_vm_by_name()
80 def create(self, block=False):
82 Creates a VM instance and associated objects unless they already exist
83 :param block: Thread will block until instance has either become
84 active, error, or timeout waiting.
85 Additionally, when True, floating IPs will not be applied
87 :return: VMInst domain object
91 if len(self.__ports) == 0:
92 self.__ports = self.__create_ports(
93 self.instance_settings.port_settings)
95 self.__create_vm(block)
99 def __lookup_existing_vm_by_name(self):
101 Populates the member variables 'self.vm' and 'self.floating_ips' if a
102 VM with the same name already exists
105 server = nova_utils.get_server(
106 self._nova, self.__neutron,
107 vm_inst_settings=self.instance_settings)
109 if server.name == self.instance_settings.name:
112 'Found existing machine with name - %s',
113 self.instance_settings.name)
115 fips = neutron_utils.get_floating_ips(self.__neutron,
117 for port_id, fip in fips:
118 settings = self.instance_settings.floating_ip_settings
119 for fip_setting in settings:
120 if port_id == fip_setting.port_id:
121 self.__floating_ip_dict[fip_setting.name] = fip
123 port = neutron_utils.get_port_by_id(
124 self.__neutron, port_id)
125 if port and port.name == fip_setting.port_name:
126 self.__floating_ip_dict[fip_setting.name] = fip
128 def __create_vm(self, block=False):
130 Responsible for creating the VM instance
131 :param block: Thread will block until instance has either become
132 active, error, or timeout waiting. Floating IPs will be
133 assigned after active when block=True
135 glance = glance_utils.glance_client(self._os_creds)
136 self.__vm = nova_utils.create_server(
137 self._nova, self.__neutron, glance, self.instance_settings,
138 self.image_settings, self.keypair_settings)
139 logger.info('Created instance with name - %s',
140 self.instance_settings.name)
143 if not self.vm_active(block=True):
144 raise VmInstanceCreationError(
145 'Fatal error, VM did not become ACTIVE within the alloted '
148 # Create server should do this but found it needed to occur here
149 for sec_grp_name in self.instance_settings.security_group_names:
150 if self.vm_active(block=True):
151 nova_utils.add_security_group(self._nova, self.__vm,
154 raise VmInstanceCreationError(
155 'Cannot applying security group with name ' +
157 ' to VM that did not activate with name - ' +
158 self.instance_settings.name)
160 if self.instance_settings.volume_names:
161 for volume_name in self.instance_settings.volume_names:
162 cinder = cinder_utils.cinder_client(self._os_creds)
163 volume = cinder_utils.get_volume(
164 cinder, volume_name=volume_name)
166 if volume and self.vm_active(block=True):
168 vm = nova_utils.attach_volume(
169 self._nova, self.__neutron, self.__vm, volume, timeout)
174 logger.warn('Volume [%s] not attached within timeout '
175 'of [%s]', volume.name, timeout)
177 logger.warn('Unable to attach volume named [%s]',
180 self.__apply_floating_ips()
182 def __apply_floating_ips(self):
184 Applies the configured floating IPs to the necessary ports
187 for key, port in self.__ports:
188 port_dict[key] = port
191 for floating_ip_setting in self.instance_settings.floating_ip_settings:
192 self.add_floating_ip(floating_ip_setting)
194 def add_floating_ip(self, floating_ip_setting):
196 Adds a floating IP to a running instance
197 :param floating_ip_setting - the floating IP configuration
198 :return: the floating ip object
201 for key, port in self.__ports:
202 port_dict[key] = port
205 port = port_dict.get(floating_ip_setting.port_name)
208 raise VmInstanceCreationError(
209 'Cannot find port object with name - ' +
210 floating_ip_setting.port_name)
212 # Setup Floating IP only if there is a router with an external
214 ext_gateway = self.__ext_gateway_by_router(
215 floating_ip_setting.router_name)
217 subnet = neutron_utils.get_subnet(
219 subnet_name=floating_ip_setting.subnet_name)
220 floating_ip = neutron_utils.create_floating_ip(
221 self.__neutron, ext_gateway)
222 self.__floating_ip_dict[floating_ip_setting.name] = floating_ip
225 'Created floating IP %s via router - %s', floating_ip.ip,
226 floating_ip_setting.router_name)
227 self.__add_floating_ip(floating_ip, port, subnet)
230 raise VmInstanceCreationError(
231 'Unable to add floating IP to port, cannot locate router '
232 'with an external gateway ')
234 def __ext_gateway_by_router(self, router_name):
236 Returns network name for the external network attached to a router or
238 :param router_name: The name of the router to lookup
239 :return: the external network name or None
241 router = neutron_utils.get_router(
242 self.__neutron, router_name=router_name)
243 if router and router.external_network_id:
244 network = neutron_utils.get_network_by_id(
245 self.__neutron, router.external_network_id)
252 Destroys the VM instance
255 # Cleanup floating IPs
256 for name, floating_ip in self.__floating_ip_dict.items():
257 logger.info('Deleting Floating IP - ' + floating_ip.ip)
258 neutron_utils.delete_floating_ip(self.__neutron, floating_ip)
260 self.__floating_ip_dict = dict()
263 for name, port in self.__ports:
264 logger.info('Deleting Port with ID - %s ', port.id)
265 neutron_utils.delete_port(self.__neutron, port)
267 self.__ports = list()
271 for volume_rec in self.__vm.volume_ids:
272 cinder = cinder_utils.cinder_client(self._os_creds)
273 volume = cinder_utils.get_volume_by_id(
274 cinder, volume_rec['id'])
276 vm = nova_utils.detach_volume(
277 self._nova, self.__neutron, self.__vm, volume, 30)
282 'Timeout waiting to detach volume %s', volume.name)
284 logger.warn('Unable to detach volume with ID - [%s]',
289 'Deleting VM instance - ' + self.instance_settings.name)
292 nova_utils.delete_vm_instance(self._nova, self.__vm)
293 except NotFound as e:
294 logger.warn('Instance already deleted - %s', e)
296 # Block until instance cannot be found or returns the status of
298 logger.info('Checking deletion status')
300 if self.vm_deleted(block=True):
302 'VM has been properly deleted VM with name - %s',
303 self.instance_settings.name)
307 'VM not deleted within the timeout period of %s '
308 'seconds', self.instance_settings.vm_delete_timeout)
310 def __query_ports(self, port_settings):
312 Returns the previously configured ports or an empty list if none
314 :param port_settings: A list of PortSetting objects
315 :return: a list of OpenStack port tuples where the first member is the
316 port name and the second is the port object
320 for port_setting in port_settings:
321 port = neutron_utils.get_port(
322 self.__neutron, port_settings=port_setting)
324 ports.append((port_setting.name, port))
328 def __create_ports(self, port_settings):
330 Returns the previously configured ports or creates them if they do not
332 :param port_settings: A list of PortSetting objects
333 :return: a list of OpenStack port tuples where the first member is the
334 port name and the second is the port object
338 for port_setting in port_settings:
339 port = neutron_utils.get_port(
340 self.__neutron, port_settings=port_setting)
342 port = neutron_utils.create_port(
343 self.__neutron, self._os_creds, port_setting)
345 ports.append((port_setting.name, port))
349 def __add_floating_ip(self, floating_ip, port, subnet, timeout=30,
350 poll_interval=POLL_INTERVAL):
352 Returns True when active else False
353 TODO - Make timeout and poll_interval configurable...
358 # Take IP of subnet if there is one configured on which to place
360 for fixed_ip in port.ips:
361 if fixed_ip['subnet_id'] == subnet.id:
362 ip = fixed_ip['ip_address']
365 # Simply take the first
366 ip = port.ips[0]['ip_address']
369 count = timeout / poll_interval
371 logger.debug('Attempting to add floating IP to instance')
373 nova_utils.add_floating_ip_to_server(
374 self._nova, self.__vm, floating_ip, ip)
376 'Added floating IP %s to port IP %s on instance %s',
377 floating_ip.ip, ip, self.instance_settings.name)
379 except BadRequest as bre:
380 logger.error('Cannot add floating IP [%s]', bre)
382 except Exception as e:
384 'Retry adding floating IP to instance. Last attempt '
385 'failed with - %s', e)
386 time.sleep(poll_interval)
390 raise VmInstanceCreationError(
391 'Unable find IP address on which to place the floating IP')
393 logger.error('Timeout attempting to add the floating IP to instance.')
394 raise VmInstanceCreationError(
395 'Timeout while attempting add floating IP to instance')
397 def get_os_creds(self):
399 Returns the OpenStack credentials used to create these objects
400 :return: the credentials
402 return self._os_creds
404 def get_vm_inst(self):
406 Returns the latest version of this server object from OpenStack
407 :return: Server object
409 return nova_utils.get_server_object_by_id(
410 self._nova, self.__neutron, self.__vm.id)
412 def get_console_output(self):
414 Returns the vm console object for parsing logs
415 :return: the console output object
417 return nova_utils.get_server_console_output(self._nova, self.__vm)
419 def get_port_ip(self, port_name, subnet_name=None):
421 Returns the first IP for the port corresponding with the port_name
422 parameter when subnet_name is None else returns the IP address that
423 corresponds to the subnet_name parameter
424 :param port_name: the name of the port from which to return the IP
425 :param subnet_name: the name of the subnet attached to this IP
426 :return: the IP or None if not found
428 port = self.get_port_by_name(port_name)
431 subnet = neutron_utils.get_subnet(
432 self.__neutron, subnet_name=subnet_name)
434 logger.warning('Cannot retrieve port IP as subnet could '
435 'not be located with name - %s',
438 for fixed_ip in port.ips:
439 if fixed_ip['subnet_id'] == subnet.id:
440 return fixed_ip['ip_address']
442 if port.ips and len(port.ips) > 0:
443 return port.ips[0]['ip_address']
446 def get_port_mac(self, port_name):
448 Returns the first IP for the port corresponding with the port_name
450 TODO - Add in the subnet as an additional parameter as a port may have
452 :param port_name: the name of the port from which to return the IP
453 :return: the IP or None if not found
455 port = self.get_port_by_name(port_name)
457 return port.mac_address
460 def get_port_by_name(self, port_name):
462 Retrieves the OpenStack port object by its given name
463 :param port_name: the name of the port
464 :return: the OpenStack port object or None if not exists
466 for key, port in self.__ports:
469 logger.warning('Cannot find port with name - ' + port_name)
472 def get_vm_info(self):
474 Returns a dictionary of a VMs info as returned by OpenStack
477 return nova_utils.get_server_info(self._nova, self.__vm)
479 def __get_first_provisioning_floating_ip(self):
481 Returns the first floating IP tagged with the Floating IP name if
482 exists else the first one found
485 for floating_ip_setting in self.instance_settings.floating_ip_settings:
486 if floating_ip_setting.provisioning:
487 fip = self.__floating_ip_dict.get(floating_ip_setting.name)
490 elif len(self.__floating_ip_dict) > 0:
491 for key, fip in self.__floating_ip_dict.items():
494 # When cannot be found above
495 if len(self.__floating_ip_dict) > 0:
496 for key, fip in self.__floating_ip_dict.items():
499 def apply_ansible_playbook(self, pb_file_loc, variables=None,
502 Applies a playbook to a VM
503 :param pb_file_loc: the file location of the playbook to be applied
504 :param variables: a dict() of substitution values required by the
506 :param fip_name: the name of the floating IP to use for applying the
507 playbook (default - will take the first)
508 :return: the return value from ansible
510 return ansible_utils.apply_playbook(
511 pb_file_loc, [self.get_floating_ip(fip_name=fip_name).ip],
512 self.get_image_user(), self.keypair_settings.private_filepath,
513 variables, self._os_creds.proxy_settings)
515 def get_image_user(self):
517 Returns the instance sudo_user if it has been configured in the
518 instance_settings else it returns the image_settings.image_user value
520 if self.instance_settings.sudo_user:
521 return self.instance_settings.sudo_user
523 return self.image_settings.image_user
525 def vm_deleted(self, block=False, poll_interval=POLL_INTERVAL):
527 Returns true when the VM status returns the value of
528 expected_status_code or instance retrieval throws a NotFound exception.
529 :param block: When true, thread will block until active or timeout
530 value in seconds has been exceeded (False)
531 :param poll_interval: The polling interval in seconds
535 return self.__vm_status_check(
536 STATUS_DELETED, block,
537 self.instance_settings.vm_delete_timeout, poll_interval)
538 except NotFound as e:
540 "Instance not found when querying status for %s with message "
541 "%s", STATUS_DELETED, e)
544 def vm_active(self, block=False, poll_interval=POLL_INTERVAL):
546 Returns true when the VM status returns the value of the constant
548 :param block: When true, thread will block until active or timeout
549 value in seconds has been exceeded (False)
550 :param poll_interval: The polling interval in seconds
553 if self.__vm_status_check(
554 STATUS_ACTIVE, block, self.instance_settings.vm_boot_timeout,
556 self.__vm = nova_utils.get_server_object_by_id(
557 self._nova, self.__neutron, self.__vm.id)
561 def __vm_status_check(self, expected_status_code, block, timeout,
564 Returns true when the VM status returns the value of
566 :param expected_status_code: instance status evaluated with this
568 :param block: When true, thread will block until active or timeout
569 value in seconds has been exceeded (False)
570 :param timeout: The timeout value
571 :param poll_interval: The polling interval in seconds
574 # sleep and wait for VM status change
578 return self.__status(expected_status_code)
580 while timeout > time.time() - start:
581 status = self.__status(expected_status_code)
583 logger.info('VM is - ' + expected_status_code)
586 logger.debug('Retry querying VM status in ' + str(
587 poll_interval) + ' seconds')
588 time.sleep(poll_interval)
589 logger.debug('VM status query timeout in ' + str(
590 timeout - (time.time() - start)))
593 'Timeout checking for VM status for ' + expected_status_code)
596 def __status(self, expected_status_code):
598 Returns True when active else False
599 :param expected_status_code: instance status evaluated with this string
604 if expected_status_code == STATUS_DELETED:
609 status = nova_utils.get_server_status(self._nova, self.__vm)
611 logger.warning('Cannot find instance with id - ' + self.__vm.id)
614 if status == 'ERROR':
615 raise VmInstanceCreationError(
616 'Instance had an error during deployment')
618 'Instance status [%s] is - %s', self.instance_settings.name,
620 return status == expected_status_code
622 def vm_ssh_active(self, block=False, poll_interval=POLL_INTERVAL):
624 Returns true when the VM can be accessed via SSH
625 :param block: When true, thread will block until active or timeout
626 value in seconds has been exceeded (False)
627 :param poll_interval: The polling interval
630 # sleep and wait for VM status change
631 logger.info('Checking if VM is active')
633 timeout = self.instance_settings.ssh_connect_timeout
635 if self.vm_active(block=True):
639 start = time.time() - timeout
641 while timeout > time.time() - start:
642 status = self.__ssh_active()
644 logger.info('SSH is active for VM instance')
647 logger.debug('Retry SSH connection in ' + str(
648 poll_interval) + ' seconds')
649 time.sleep(poll_interval)
650 logger.debug('SSH connection timeout in ' + str(
651 timeout - (time.time() - start)))
653 logger.error('Timeout attempting to connect with VM via SSH')
656 def __ssh_active(self):
658 Returns True when can create a SSH session else False
661 if len(self.__floating_ip_dict) > 0:
662 ssh = self.ssh_client()
668 def cloud_init_complete(self, block=False, poll_interval=POLL_INTERVAL):
670 Returns true when the VM's cloud-init routine has completed.
671 Note: this is currently done via SSH, therefore, if this instance does
672 not have a Floating IP or a running SSH server, this routine
673 will always return False or raise an Exception
674 :param block: When true, thread will block until active or timeout
675 value in seconds has been exceeded (False)
676 :param poll_interval: The polling interval
679 # sleep and wait for VM status change
680 logger.info('Checking if cloud-init has completed')
682 timeout = self.instance_settings.cloud_init_timeout
684 if self.vm_active(block=True) and self.vm_ssh_active(block=True):
688 start = time.time() - timeout
690 while timeout > time.time() - start:
691 status = self.__cloud_init_complete()
693 logger.info('cloud-init complete for VM instance')
696 logger.debug('Retry cloud-init query in ' + str(
697 poll_interval) + ' seconds')
698 time.sleep(poll_interval)
699 logger.debug('cloud-init complete timeout in ' + str(
700 timeout - (time.time() - start)))
702 logger.error('Timeout waiting for cloud-init to complete')
705 def __cloud_init_complete(self):
707 Returns True when can create a SSH session else False
710 if len(self.__floating_ip_dict) > 0:
711 ssh = self.ssh_client()
713 stdin1, stdout1, sterr1 = ssh.exec_command(
714 'ls -l /var/lib/cloud/instance/boot-finished')
715 return stdout1.channel.recv_exit_status() == 0
718 def get_floating_ip(self, fip_name=None):
720 Returns the floating IP object byt name if found, else the first known,
722 :param fip_name: the name of the floating IP to return
723 :return: the SSH client or None
725 if fip_name and self.__floating_ip_dict.get(fip_name):
726 return self.__floating_ip_dict.get(fip_name)
728 return self.__get_first_provisioning_floating_ip()
730 def ssh_client(self, fip_name=None):
732 Returns an SSH client using the name or the first known floating IP if
734 :param fip_name: the name of the floating IP to return
735 :return: the SSH client or None
737 fip = self.get_floating_ip(fip_name)
739 return ansible_utils.ssh_client(
740 self.__get_first_provisioning_floating_ip().ip,
741 self.get_image_user(),
742 self.keypair_settings.private_filepath,
743 proxy_settings=self._os_creds.proxy_settings)
745 FloatingIPAllocationError(
746 'Cannot return an SSH client. No Floating IP configured')
748 def add_security_group(self, security_group):
750 Adds a security group to this VM. Call will block until VM is active.
751 :param security_group: the SNAPS SecurityGroup domain object
752 :return True if successful else False
754 self.vm_active(block=True)
756 if not security_group:
757 logger.warning('Security group object is None, cannot add')
761 nova_utils.add_security_group(self._nova, self.get_vm_inst(),
764 except NotFound as e:
765 logger.warning('Security group not added - ' + str(e))
768 def remove_security_group(self, security_group):
770 Removes a security group to this VM. Call will block until VM is active
771 :param security_group: the OpenStack security group object
772 :return True if successful else False
774 self.vm_active(block=True)
776 if not security_group:
777 logger.warning('Security group object is None, cannot remove')
781 nova_utils.remove_security_group(self._nova, self.get_vm_inst(),
784 except NotFound as e:
785 logger.warning('Security group not removed - ' + str(e))
788 def reboot(self, reboot_type=RebootType.soft):
791 :param reboot_type: instance of
792 snaps.openstack.utils.nova_utils.RebootType
796 nova_utils.reboot_server(
797 self._nova, self.__vm, reboot_type=reboot_type)
800 def generate_creator(os_creds, vm_inst, image_config, keypair_config=None):
802 Initializes an OpenStackVmInstance object
803 :param os_creds: the OpenStack credentials
804 :param vm_inst: the SNAPS-OO VmInst domain object
805 :param image_config: the associated ImageConfig object
806 :param keypair_config: the associated KeypairConfig object (optional)
807 :return: an initialized OpenStackVmInstance object
809 nova = nova_utils.nova_client(os_creds)
810 neutron = neutron_utils.neutron_client(os_creds)
811 derived_inst_config = settings_utils.create_vm_inst_config(
812 nova, neutron, vm_inst)
814 derived_inst_creator = OpenStackVmInstance(
815 os_creds, derived_inst_config, image_config, keypair_config)
816 derived_inst_creator.initialize()
817 return derived_inst_creator
820 class VmInstanceSettings(VmInstanceConfig):
822 Deprecated, use snaps.config.vm_inst.VmInstanceConfig instead
824 def __init__(self, **kwargs):
825 from warnings import warn
826 warn('Use snaps.config.vm_inst.VmInstanceConfig instead',
828 super(self.__class__, self).__init__(**kwargs)
831 class FloatingIpSettings(FloatingIpConfig):
833 Deprecated, use snaps.config.vm_inst.FloatingIpConfig instead
835 def __init__(self, **kwargs):
836 from warnings import warn
837 warn('Use snaps.config.vm_inst.FloatingIpConfig instead',
839 super(self.__class__, self).__init__(**kwargs)
842 class VmInstanceCreationError(Exception):
844 Exception to be thrown when an VM instance cannot be created
848 class FloatingIPAllocationError(Exception):
850 Exception to be thrown when an VM instance cannot allocate a floating IP