1 # Copyright (c) 2017 Cable Television Laboratories, Inc. ("CableLabs")
2 # and others. All rights reserved.
4 # Licensed under the Apache License, Version 2.0 (the "License");
5 # you may not use this file except in compliance with the License.
6 # You may obtain a copy of the License at:
8 # http://www.apache.org/licenses/LICENSE-2.0
10 # Unless required by applicable law or agreed to in writing, software
11 # distributed under the License is distributed on an "AS IS" BASIS,
12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 # See the License for the specific language governing permissions and
14 # limitations under the License.
18 from neutronclient.common.exceptions import PortNotFoundClient
19 from novaclient.exceptions import NotFound
21 from snaps.openstack.create_network import PortSettings
22 from snaps.openstack.utils import glance_utils
23 from snaps.openstack.utils import neutron_utils
24 from snaps.openstack.utils import nova_utils
25 from snaps.provisioning import ansible_utils
27 __author__ = 'spisarski'
29 logger = logging.getLogger('create_instance')
32 STATUS_ACTIVE = 'ACTIVE'
33 STATUS_DELETED = 'DELETED'
36 class OpenStackVmInstance:
38 Class responsible for creating a VM instance in OpenStack
41 def __init__(self, os_creds, instance_settings, image_settings,
42 keypair_settings=None):
45 :param os_creds: The connection credentials to the OpenStack API
46 :param instance_settings: Contains the settings for this VM
47 :param image_settings: The OpenStack image object settings
48 :param keypair_settings: The keypair metadata (Optional)
51 self.__os_creds = os_creds
56 self.instance_settings = instance_settings
57 self.image_settings = image_settings
58 self.keypair_settings = keypair_settings
60 self.__floating_ip_dict = dict()
62 # Instantiated in self.create()
65 # Note: this object does not change after the VM becomes active
68 def create(self, cleanup=False, block=False):
71 :param cleanup: When true, only perform lookups for OpenStack objects.
72 :param block: Thread will block until instance has either become
73 active, error, or timeout waiting.
74 Additionally, when True, floating IPs will not be applied
76 :return: The VM reference object
78 self.__nova = nova_utils.nova_client(self.__os_creds)
79 self.__neutron = neutron_utils.neutron_client(self.__os_creds)
81 self.__ports = self.__setup_ports(self.instance_settings.port_settings,
83 self.__lookup_existing_vm_by_name()
84 if not self.__vm and not cleanup:
85 self.__create_vm(block)
88 def __lookup_existing_vm_by_name(self):
90 Populates the member variables 'self.vm' and 'self.floating_ips' if a
91 VM with the same name already exists
94 servers = nova_utils.get_servers_by_name(self.__nova,
95 self.instance_settings.name)
96 for server in servers:
97 if server.name == self.instance_settings.name:
100 'Found existing machine with name - %s',
101 self.instance_settings.name)
103 fips = neutron_utils.get_floating_ips(self.__neutron,
105 for port_name, fip in fips:
106 settings = self.instance_settings.floating_ip_settings
107 for fip_setting in settings:
108 if port_name == fip_setting.port_name:
109 self.__floating_ip_dict[fip_setting.name] = fip
111 def __create_vm(self, block=False):
113 Responsible for creating the VM instance
114 :param block: Thread will block until instance has either become
115 active, error, or timeout waiting. Floating IPs will be
116 assigned after active when block=True
118 glance = glance_utils.glance_client(self.__os_creds)
119 self.__vm = nova_utils.create_server(
120 self.__nova, self.__neutron, glance, self.instance_settings,
121 self.image_settings, self.keypair_settings)
122 logger.info('Created instance with name - %s',
123 self.instance_settings.name)
126 if not self.vm_active(block=True):
127 raise VmInstanceCreationError(
128 'Fatal error, VM did not become ACTIVE within the alloted '
131 # Create server should do this but found it needed to occur here
132 for sec_grp_name in self.instance_settings.security_group_names:
133 if self.vm_active(block=True):
134 nova_utils.add_security_group(self.__nova, self.__vm,
137 raise VmInstanceCreationError(
138 'Cannot applying security group with name ' +
140 ' to VM that did not activate with name - ' +
141 self.instance_settings.name)
143 self.__apply_floating_ips()
145 def __apply_floating_ips(self):
147 Applies the configured floating IPs to the necessary ports
150 for key, port in self.__ports:
151 port_dict[key] = port
154 for floating_ip_setting in self.instance_settings.floating_ip_settings:
155 port = port_dict.get(floating_ip_setting.port_name)
158 raise VmInstanceCreationError(
159 'Cannot find port object with name - ' +
160 floating_ip_setting.port_name)
162 # Setup Floating IP only if there is a router with an external
164 ext_gateway = self.__ext_gateway_by_router(
165 floating_ip_setting.router_name)
167 subnet = neutron_utils.get_subnet(
169 subnet_name=floating_ip_setting.subnet_name)
170 floating_ip = neutron_utils.create_floating_ip(
171 self.__neutron, ext_gateway)
172 self.__floating_ip_dict[floating_ip_setting.name] = floating_ip
175 'Created floating IP %s via router - %s', floating_ip.ip,
176 floating_ip_setting.router_name)
177 self.__add_floating_ip(floating_ip, port, subnet)
179 raise VmInstanceCreationError(
180 'Unable to add floating IP to port, cannot locate router '
181 'with an external gateway ')
183 def __ext_gateway_by_router(self, router_name):
185 Returns network name for the external network attached to a router or
187 :param router_name: The name of the router to lookup
188 :return: the external network name or None
190 router = neutron_utils.get_router_by_name(self.__neutron, router_name)
191 if router and router.external_gateway_info:
192 network = neutron_utils.get_network_by_id(
194 router.external_gateway_info['network_id'])
201 Destroys the VM instance
204 # Cleanup floating IPs
205 for name, floating_ip in self.__floating_ip_dict.items():
207 logger.info('Deleting Floating IP - ' + floating_ip.ip)
208 neutron_utils.delete_floating_ip(self.__neutron, floating_ip)
209 except Exception as e:
210 logger.error('Error deleting Floating IP - ' + str(e))
211 self.__floating_ip_dict = dict()
214 for name, port in self.__ports:
215 logger.info('Deleting Port - ' + name)
217 neutron_utils.delete_port(self.__neutron, port)
218 except PortNotFoundClient as e:
219 logger.warning('Unexpected error deleting port - %s', e)
221 self.__ports = list()
227 'Deleting VM instance - ' + self.instance_settings.name)
228 nova_utils.delete_vm_instance(self.__nova, self.__vm)
229 except Exception as e:
230 logger.error('Error deleting VM - %s', e)
232 # Block until instance cannot be found or returns the status of
234 logger.info('Checking deletion status')
237 if self.vm_deleted(block=True):
239 'VM has been properly deleted VM with name - %s',
240 self.instance_settings.name)
244 'VM not deleted within the timeout period of %s '
245 'seconds', self.instance_settings.vm_delete_timeout)
246 except Exception as e:
248 'Unexpected error while checking VM instance status - %s',
251 def __setup_ports(self, port_settings, cleanup):
253 Returns the previously configured ports or creates them if they do not
255 :param port_settings: A list of PortSetting objects
256 :param cleanup: When true, only perform lookups for OpenStack objects.
257 :return: a list of OpenStack port tuples where the first member is the
258 port name and the second is the port object
262 for port_setting in port_settings:
263 port = neutron_utils.get_port_by_name(self.__neutron,
266 ports.append((port_setting.name, port))
268 # Exception will be raised when port with same name already
271 (port_setting.name, neutron_utils.create_port(
272 self.__neutron, self.__os_creds, port_setting)))
276 def __add_floating_ip(self, floating_ip, port, subnet, timeout=30,
277 poll_interval=POLL_INTERVAL):
279 Returns True when active else False
280 TODO - Make timeout and poll_interval configurable...
285 # Take IP of subnet if there is one configured on which to place
287 for fixed_ip in port.ips:
288 if fixed_ip['subnet_id'] == subnet.id:
289 ip = fixed_ip['ip_address']
292 # Simply take the first
293 ip = port.ips[0]['ip_address']
296 count = timeout / poll_interval
298 logger.debug('Attempting to add floating IP to instance')
300 nova_utils.add_floating_ip_to_server(
301 self.__nova, self.__vm, floating_ip, ip)
303 'Added floating IP %s to port IP %s on instance %s',
304 floating_ip.ip, ip, self.instance_settings.name)
306 except Exception as e:
308 'Retry adding floating IP to instance. Last attempt '
309 'failed with - %s', e)
310 time.sleep(poll_interval)
314 raise VmInstanceCreationError(
315 'Unable find IP address on which to place the floating IP')
317 logger.error('Timeout attempting to add the floating IP to instance.')
318 raise VmInstanceCreationError(
319 'Timeout while attempting add floating IP to instance')
321 def get_os_creds(self):
323 Returns the OpenStack credentials used to create these objects
324 :return: the credentials
326 return self.__os_creds
328 def get_vm_inst(self):
330 Returns the latest version of this server object from OpenStack
331 :return: Server object
335 def get_console_output(self):
337 Returns the vm console object for parsing logs
338 :return: the console output object
340 return nova_utils.get_server_console_output(self.__nova, self.__vm)
342 def get_port_ip(self, port_name, subnet_name=None):
344 Returns the first IP for the port corresponding with the port_name
345 parameter when subnet_name is None else returns the IP address that
346 corresponds to the subnet_name parameter
347 :param port_name: the name of the port from which to return the IP
348 :param subnet_name: the name of the subnet attached to this IP
349 :return: the IP or None if not found
351 port = self.get_port_by_name(port_name)
354 subnet = neutron_utils.get_subnet(
355 self.__neutron, subnet_name=subnet_name)
357 logger.warning('Cannot retrieve port IP as subnet could '
358 'not be located with name - %s',
361 for fixed_ip in port.ips:
362 if fixed_ip['subnet_id'] == subnet.id:
363 return fixed_ip['ip_address']
365 if port.ips and len(port.ips) > 0:
366 return port.ips[0]['ip_address']
369 def get_port_mac(self, port_name):
371 Returns the first IP for the port corresponding with the port_name
373 TODO - Add in the subnet as an additional parameter as a port may have
375 :param port_name: the name of the port from which to return the IP
376 :return: the IP or None if not found
378 port = self.get_port_by_name(port_name)
380 return port.mac_address
383 def get_port_by_name(self, port_name):
385 Retrieves the OpenStack port object by its given name
386 :param port_name: the name of the port
387 :return: the OpenStack port object or None if not exists
389 for key, port in self.__ports:
392 logger.warning('Cannot find port with name - ' + port_name)
395 def get_vm_info(self):
397 Returns a dictionary of a VMs info as returned by OpenStack
400 return nova_utils.get_server_info(self.__nova, self.__vm)
402 def config_nics(self):
404 Responsible for configuring NICs on RPM systems where the instance has
405 more than one configured port
406 :return: the value returned by ansible_utils.apply_ansible_playbook()
408 if len(self.__ports) > 1 and len(self.__floating_ip_dict) > 0:
409 if self.vm_active(block=True) and self.vm_ssh_active(block=True):
410 for key, port in self.__ports:
411 port_index = self.__ports.index((key, port))
413 nic_name = 'eth' + repr(port_index)
414 retval = self.__config_nic(
416 self.__get_first_provisioning_floating_ip().ip)
417 logger.info('Configured NIC - %s on VM - %s',
418 nic_name, self.instance_settings.name)
421 def __get_first_provisioning_floating_ip(self):
423 Returns the first floating IP tagged with the Floating IP name if
424 exists else the first one found
427 for floating_ip_setting in self.instance_settings.floating_ip_settings:
428 if floating_ip_setting.provisioning:
429 fip = self.__floating_ip_dict.get(floating_ip_setting.name)
432 elif len(self.__floating_ip_dict) > 0:
433 for key, fip in self.__floating_ip_dict.items():
436 def __config_nic(self, nic_name, port, ip):
438 Although ports/NICs can contain multiple IPs, this code currently only
441 :param nic_name: Name of the interface
442 :param port: The port information containing the expected IP values.
443 :param ip: The IP on which to apply the playbook.
444 :return: the return value from ansible
446 port_ip = port.ips[0]['ip_address']
449 'nic_name': nic_name,
453 if self.image_settings.nic_config_pb_loc and self.keypair_settings:
454 return self.apply_ansible_playbook(
455 self.image_settings.nic_config_pb_loc, variables)
458 'VM %s cannot self configure NICs eth1++. No playbook or '
459 'keypairs found.', self.instance_settings.name)
461 def apply_ansible_playbook(self, pb_file_loc, variables=None,
464 Applies a playbook to a VM
465 :param pb_file_loc: the file location of the playbook to be applied
466 :param variables: a dict() of substitution values required by the
468 :param fip_name: the name of the floating IP to use for applying the
469 playbook (default - will take the first)
470 :return: the return value from ansible
472 return ansible_utils.apply_playbook(
473 pb_file_loc, [self.get_floating_ip(fip_name=fip_name).ip],
474 self.get_image_user(), self.keypair_settings.private_filepath,
475 variables, self.__os_creds.proxy_settings)
477 def get_image_user(self):
479 Returns the instance sudo_user if it has been configured in the
480 instance_settings else it returns the image_settings.image_user value
482 if self.instance_settings.sudo_user:
483 return self.instance_settings.sudo_user
485 return self.image_settings.image_user
487 def vm_deleted(self, block=False, poll_interval=POLL_INTERVAL):
489 Returns true when the VM status returns the value of
490 expected_status_code or instance retrieval throws a NotFound exception.
491 :param block: When true, thread will block until active or timeout
492 value in seconds has been exceeded (False)
493 :param poll_interval: The polling interval in seconds
497 return self.__vm_status_check(
498 STATUS_DELETED, block,
499 self.instance_settings.vm_delete_timeout, poll_interval)
500 except NotFound as e:
502 "Instance not found when querying status for %s with message "
503 "%s", STATUS_DELETED, e)
506 def vm_active(self, block=False, poll_interval=POLL_INTERVAL):
508 Returns true when the VM status returns the value of
510 :param block: When true, thread will block until active or timeout
511 value in seconds has been exceeded (False)
512 :param poll_interval: The polling interval in seconds
515 return self.__vm_status_check(STATUS_ACTIVE, block,
516 self.instance_settings.vm_boot_timeout,
519 def __vm_status_check(self, expected_status_code, block, timeout,
522 Returns true when the VM status returns the value of
524 :param expected_status_code: instance status evaluated with this
526 :param block: When true, thread will block until active or timeout
527 value in seconds has been exceeded (False)
528 :param timeout: The timeout value
529 :param poll_interval: The polling interval in seconds
532 # sleep and wait for VM status change
536 return self.__status(expected_status_code)
538 while timeout > time.time() - start:
539 status = self.__status(expected_status_code)
541 logger.info('VM is - ' + expected_status_code)
544 logger.debug('Retry querying VM status in ' + str(
545 poll_interval) + ' seconds')
546 time.sleep(poll_interval)
547 logger.debug('VM status query timeout in ' + str(
548 timeout - (time.time() - start)))
551 'Timeout checking for VM status for ' + expected_status_code)
554 def __status(self, expected_status_code):
556 Returns True when active else False
557 :param expected_status_code: instance status evaluated with this string
564 status = nova_utils.get_server_status(self.__nova, self.__vm)
566 logger.warning('Cannot find instance with id - ' + self.__vm.id)
569 if status == 'ERROR':
570 raise VmInstanceCreationError(
571 'Instance had an error during deployment')
573 'Instance status [%s] is - %s', self.instance_settings.name,
575 return status == expected_status_code
577 def vm_ssh_active(self, block=False, poll_interval=POLL_INTERVAL):
579 Returns true when the VM can be accessed via SSH
580 :param block: When true, thread will block until active or timeout
581 value in seconds has been exceeded (False)
582 :param poll_interval: The polling interval
585 # sleep and wait for VM status change
586 logger.info('Checking if VM is active')
588 timeout = self.instance_settings.ssh_connect_timeout
590 if self.vm_active(block=True):
594 start = time.time() - timeout
596 while timeout > time.time() - start:
597 status = self.__ssh_active()
599 logger.info('SSH is active for VM instance')
602 logger.debug('Retry SSH connection in ' + str(
603 poll_interval) + ' seconds')
604 time.sleep(poll_interval)
605 logger.debug('SSH connection timeout in ' + str(
606 timeout - (time.time() - start)))
608 logger.error('Timeout attempting to connect with VM via SSH')
611 def __ssh_active(self):
613 Returns True when can create a SSH session else False
616 if len(self.__floating_ip_dict) > 0:
617 ssh = self.ssh_client()
623 def get_floating_ip(self, fip_name=None):
625 Returns the floating IP object byt name if found, else the first known,
627 :param fip_name: the name of the floating IP to return
628 :return: the SSH client or None
631 if fip_name and self.__floating_ip_dict.get(fip_name):
632 return self.__floating_ip_dict.get(fip_name)
634 return self.__get_first_provisioning_floating_ip()
636 def ssh_client(self, fip_name=None):
638 Returns an SSH client using the name or the first known floating IP if
640 :param fip_name: the name of the floating IP to return
641 :return: the SSH client or None
643 fip = self.get_floating_ip(fip_name)
645 return ansible_utils.ssh_client(
646 self.__get_first_provisioning_floating_ip().ip,
647 self.get_image_user(),
648 self.keypair_settings.private_filepath,
649 proxy_settings=self.__os_creds.proxy_settings)
652 'Cannot return an SSH client. No Floating IP configured')
654 def add_security_group(self, security_group):
656 Adds a security group to this VM. Call will block until VM is active.
657 :param security_group: the SNAPS SecurityGroup domain object
658 :return True if successful else False
660 self.vm_active(block=True)
662 if not security_group:
663 logger.warning('Security group object is None, cannot add')
667 nova_utils.add_security_group(self.__nova, self.get_vm_inst(),
670 except NotFound as e:
671 logger.warning('Security group not added - ' + str(e))
674 def remove_security_group(self, security_group):
676 Removes a security group to this VM. Call will block until VM is active
677 :param security_group: the OpenStack security group object
678 :return True if successful else False
680 self.vm_active(block=True)
682 if not security_group:
683 logger.warning('Security group object is None, cannot remove')
687 nova_utils.remove_security_group(self.__nova, self.get_vm_inst(),
690 except NotFound as e:
691 logger.warning('Security group not removed - ' + str(e))
695 class VmInstanceSettings:
697 Class responsible for holding configuration setting for a VM Instance
700 def __init__(self, **kwargs):
703 :param name: the name of the VM
704 :param flavor: the VM's flavor
705 :param port_settings: the port configuration settings (required)
706 :param security_group_names: a set of names of the security groups to
708 :param floating_ip_settings: the floating IP configuration settings
709 :param sudo_user: the sudo user of the VM that will override the
710 instance_settings.image_user when trying to
712 :param vm_boot_timeout: the amount of time a thread will sleep waiting
713 for an instance to boot
714 :param vm_delete_timeout: the amount of time a thread will sleep
715 waiting for an instance to be deleted
716 :param ssh_connect_timeout: the amount of time a thread will sleep
717 waiting obtaining an SSH connection to a VM
718 :param availability_zone: the name of the compute server on which to
719 deploy the VM (optional)
720 :param userdata: the cloud-init script to run after the VM has been
723 self.name = kwargs.get('name')
724 self.flavor = kwargs.get('flavor')
725 self.sudo_user = kwargs.get('sudo_user')
726 self.userdata = kwargs.get('userdata')
728 self.port_settings = list()
729 port_settings = kwargs.get('ports')
730 if not port_settings:
731 port_settings = kwargs.get('port_settings')
733 for port_setting in port_settings:
734 if isinstance(port_setting, dict):
735 self.port_settings.append(PortSettings(**port_setting))
736 elif isinstance(port_setting, PortSettings):
737 self.port_settings.append(port_setting)
739 if kwargs.get('security_group_names'):
740 if isinstance(kwargs['security_group_names'], list):
741 self.security_group_names = kwargs['security_group_names']
742 elif isinstance(kwargs['security_group_names'], set):
743 self.security_group_names = kwargs['security_group_names']
744 elif isinstance(kwargs['security_group_names'], str):
745 self.security_group_names = [kwargs['security_group_names']]
747 raise VmInstanceSettingsError(
748 'Invalid data type for security_group_names attribute')
750 self.security_group_names = set()
752 self.floating_ip_settings = list()
753 floating_ip_settings = kwargs.get('floating_ips')
754 if not floating_ip_settings:
755 floating_ip_settings = kwargs.get('floating_ip_settings')
756 if floating_ip_settings:
757 for floating_ip_config in floating_ip_settings:
758 if isinstance(floating_ip_config, FloatingIpSettings):
759 self.floating_ip_settings.append(floating_ip_config)
761 self.floating_ip_settings.append(FloatingIpSettings(
762 **floating_ip_config['floating_ip']))
764 if kwargs.get('vm_boot_timeout'):
765 self.vm_boot_timeout = kwargs['vm_boot_timeout']
767 self.vm_boot_timeout = 900
769 if kwargs.get('vm_delete_timeout'):
770 self.vm_delete_timeout = kwargs['vm_delete_timeout']
772 self.vm_delete_timeout = 300
774 if kwargs.get('ssh_connect_timeout'):
775 self.ssh_connect_timeout = kwargs['ssh_connect_timeout']
777 self.ssh_connect_timeout = 180
779 if kwargs.get('availability_zone'):
780 self.availability_zone = kwargs['availability_zone']
782 self.availability_zone = None
784 if not self.name or not self.flavor:
785 raise VmInstanceSettingsError(
786 'Instance configuration requires the attributes: name, flavor')
788 if len(self.port_settings) == 0:
789 raise VmInstanceSettingsError(
790 'Instance configuration requires port settings (aka. NICS)')
793 class FloatingIpSettings:
795 Class responsible for holding configuration settings for a floating IP
798 def __init__(self, **kwargs):
801 :param name: the name of the floating IP
802 :param port_name: the name of the router to the external network
803 :param router_name: the name of the router to the external network
804 :param subnet_name: the name of the subnet on which to attach the
806 :param provisioning: when true, this floating IP can be used for
809 TODO - provisioning flag is a hack as I have only observed a single
810 Floating IPs that actually works on an instance. Multiple floating IPs
811 placed on different subnets from the same port are especially
812 troublesome as you cannot predict which one will actually connect.
813 For now, it is recommended not to setup multiple floating IPs on an
814 instance unless absolutely necessary.
816 self.name = kwargs.get('name')
817 self.port_name = kwargs.get('port_name')
818 self.router_name = kwargs.get('router_name')
819 self.subnet_name = kwargs.get('subnet_name')
820 if kwargs.get('provisioning') is not None:
821 self.provisioning = kwargs['provisioning']
823 self.provisioning = True
825 if not self.name or not self.port_name or not self.router_name:
826 raise FloatingIpSettingsError(
827 'The attributes name, port_name and router_name are required '
828 'for FloatingIPSettings')
831 class VmInstanceSettingsError(Exception):
833 Exception to be thrown when an VM instance settings are incorrect
837 class FloatingIpSettingsError(Exception):
839 Exception to be thrown when an VM instance settings are incorrect
843 class VmInstanceCreationError(Exception):
845 Exception to be thrown when an VM instance cannot be created