1 # Copyright (c) 2017 Cable Television Laboratories, Inc. ("CableLabs")
2 # and others. All rights reserved.
4 # Licensed under the Apache License, Version 2.0 (the "License");
5 # you may not use this file except in compliance with the License.
6 # You may obtain a copy of the License at:
8 # http://www.apache.org/licenses/LICENSE-2.0
10 # Unless required by applicable law or agreed to in writing, software
11 # distributed under the License is distributed on an "AS IS" BASIS,
12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 # See the License for the specific language governing permissions and
14 # limitations under the License.
18 from neutronclient.common.exceptions import PortNotFoundClient
19 from novaclient.exceptions import NotFound
21 from snaps.openstack.create_network import PortSettings
22 from snaps.openstack.utils import glance_utils
23 from snaps.openstack.utils import neutron_utils
24 from snaps.openstack.utils import nova_utils
25 from snaps.provisioning import ansible_utils
27 __author__ = 'spisarski'
29 logger = logging.getLogger('create_instance')
32 STATUS_ACTIVE = 'ACTIVE'
33 STATUS_DELETED = 'DELETED'
36 class OpenStackVmInstance:
38 Class responsible for creating a VM instance in OpenStack
41 def __init__(self, os_creds, instance_settings, image_settings,
42 keypair_settings=None):
45 :param os_creds: The connection credentials to the OpenStack API
46 :param instance_settings: Contains the settings for this VM
47 :param image_settings: The OpenStack image object settings
48 :param keypair_settings: The keypair metadata (Optional)
51 self.__os_creds = os_creds
56 self.instance_settings = instance_settings
57 self.image_settings = image_settings
58 self.keypair_settings = keypair_settings
60 # TODO - get rid of FIP list and only use the dict(). Need to fix
61 # populating this object when already exists
62 self.__floating_ips = list()
63 self.__floating_ip_dict = dict()
65 # Instantiated in self.create()
68 # Note: this object does not change after the VM becomes active
71 def create(self, cleanup=False, block=False):
74 :param cleanup: When true, only perform lookups for OpenStack objects.
75 :param block: Thread will block until instance has either become
76 active, error, or timeout waiting.
77 Additionally, when True, floating IPs will not be applied
79 :return: The VM reference object
81 self.__nova = nova_utils.nova_client(self.__os_creds)
82 self.__neutron = neutron_utils.neutron_client(self.__os_creds)
84 self.__ports = self.__setup_ports(self.instance_settings.port_settings,
86 self.__lookup_existing_vm_by_name()
87 if not self.__vm and not cleanup:
88 self.__create_vm(block)
91 def __lookup_existing_vm_by_name(self):
93 Populates the member variables 'self.vm' and 'self.floating_ips' if a
94 VM with the same name already exists
97 servers = nova_utils.get_servers_by_name(self.__nova,
98 self.instance_settings.name)
99 for server in servers:
100 if server.name == self.instance_settings.name:
103 'Found existing machine with name - %s',
104 self.instance_settings.name)
105 fips = neutron_utils.get_floating_ips(self.__neutron)
107 for subnet_name, ips in server.networks.items():
109 self.__floating_ips.append(fip)
110 # TODO - Determine a means to associate to the FIP
111 # configuration and add to FIP map
113 def __create_vm(self, block=False):
115 Responsible for creating the VM instance
116 :param block: Thread will block until instance has either become
117 active, error, or timeout waiting. Floating IPs will be
118 assigned after active when block=True
120 glance = glance_utils.glance_client(self.__os_creds)
121 self.__vm = nova_utils.create_server(
122 self.__nova, self.__neutron, glance, self.instance_settings,
123 self.image_settings, self.keypair_settings)
124 logger.info('Created instance with name - %s',
125 self.instance_settings.name)
128 if not self.vm_active(block=True):
129 raise VmInstanceCreationError(
130 'Fatal error, VM did not become ACTIVE within the alloted '
133 # Create server should do this but found it needed to occur here
134 for sec_grp_name in self.instance_settings.security_group_names:
135 if self.vm_active(block=True):
136 nova_utils.add_security_group(self.__nova, self.__vm,
139 raise VmInstanceCreationError(
140 'Cannot applying security group with name ' +
142 ' to VM that did not activate with name - ' +
143 self.instance_settings.name)
145 self.__apply_floating_ips()
147 def __apply_floating_ips(self):
149 Applies the configured floating IPs to the necessary ports
152 for key, port in self.__ports:
153 port_dict[key] = port
156 for floating_ip_setting in self.instance_settings.floating_ip_settings:
157 port = port_dict.get(floating_ip_setting.port_name)
160 raise VmInstanceCreationError(
161 'Cannot find port object with name - ' +
162 floating_ip_setting.port_name)
164 # Setup Floating IP only if there is a router with an external
166 ext_gateway = self.__ext_gateway_by_router(
167 floating_ip_setting.router_name)
169 subnet = neutron_utils.get_subnet_by_name(
170 self.__neutron, floating_ip_setting.subnet_name)
171 floating_ip = neutron_utils.create_floating_ip(
172 self.__neutron, ext_gateway)
173 self.__floating_ips.append(floating_ip)
174 self.__floating_ip_dict[floating_ip_setting.name] = floating_ip
177 'Created floating IP %s via router - %s', floating_ip.ip,
178 floating_ip_setting.router_name)
179 self.__add_floating_ip(floating_ip, port, subnet)
181 raise VmInstanceCreationError(
182 'Unable to add floating IP to port, cannot locate router '
183 'with an external gateway ')
185 def __ext_gateway_by_router(self, router_name):
187 Returns network name for the external network attached to a router or
189 :param router_name: The name of the router to lookup
190 :return: the external network name or None
192 router = neutron_utils.get_router_by_name(self.__neutron, router_name)
193 if router and router.external_gateway_info:
194 network = neutron_utils.get_network_by_id(
196 router.external_gateway_info['network_id'])
203 Destroys the VM instance
206 # Cleanup floating IPs
207 for floating_ip in self.__floating_ips:
209 logger.info('Deleting Floating IP - ' + floating_ip.ip)
210 neutron_utils.delete_floating_ip(self.__neutron, floating_ip)
211 except Exception as e:
212 logger.error('Error deleting Floating IP - ' + str(e))
213 self.__floating_ips = list()
214 self.__floating_ip_dict = dict()
217 for name, port in self.__ports:
218 logger.info('Deleting Port - ' + name)
220 neutron_utils.delete_port(self.__neutron, port)
221 except PortNotFoundClient as e:
222 logger.warning('Unexpected error deleting port - %s', e)
224 self.__ports = list()
230 'Deleting VM instance - ' + self.instance_settings.name)
231 nova_utils.delete_vm_instance(self.__nova, self.__vm)
232 except Exception as e:
233 logger.error('Error deleting VM - %s', e)
235 # Block until instance cannot be found or returns the status of
237 logger.info('Checking deletion status')
240 if self.vm_deleted(block=True):
242 'VM has been properly deleted VM with name - %s',
243 self.instance_settings.name)
247 'VM not deleted within the timeout period of %s '
248 'seconds', self.instance_settings.vm_delete_timeout)
249 except Exception as e:
251 'Unexpected error while checking VM instance status - %s',
254 def __setup_ports(self, port_settings, cleanup):
256 Returns the previously configured ports or creates them if they do not
258 :param port_settings: A list of PortSetting objects
259 :param cleanup: When true, only perform lookups for OpenStack objects.
260 :return: a list of OpenStack port tuples where the first member is the
261 port name and the second is the port object
265 for port_setting in port_settings:
266 port = neutron_utils.get_port_by_name(self.__neutron,
269 ports.append((port_setting.name, {'port': port}))
271 # Exception will be raised when port with same name already
274 (port_setting.name, neutron_utils.create_port(
275 self.__neutron, self.__os_creds, port_setting)))
279 def __add_floating_ip(self, floating_ip, port, subnet, timeout=30,
280 poll_interval=POLL_INTERVAL):
282 Returns True when active else False
283 TODO - Make timeout and poll_interval configurable...
288 # Take IP of subnet if there is one configured on which to place
290 for fixed_ip in port.fixed_ips:
291 if fixed_ip['subnet_id'] == subnet['subnet']['id']:
292 ip = fixed_ip['ip_address']
295 # Simply take the first
296 ip = port.ips[0]['ip_address']
299 count = timeout / poll_interval
301 logger.debug('Attempting to add floating IP to instance')
303 nova_utils.add_floating_ip_to_server(
304 self.__nova, self.__vm, floating_ip, ip)
306 'Added floating IP %s to port IP %s on instance %s',
307 floating_ip.ip, ip, self.instance_settings.name)
309 except Exception as e:
311 'Retry adding floating IP to instance. Last attempt '
312 'failed with - %s', e)
313 time.sleep(poll_interval)
317 raise VmInstanceCreationError(
318 'Unable find IP address on which to place the floating IP')
320 logger.error('Timeout attempting to add the floating IP to instance.')
321 raise VmInstanceCreationError(
322 'Timeout while attempting add floating IP to instance')
324 def get_os_creds(self):
326 Returns the OpenStack credentials used to create these objects
327 :return: the credentials
329 return self.__os_creds
331 def get_vm_inst(self):
333 Returns the latest version of this server object from OpenStack
334 :return: Server object
338 def get_console_output(self):
340 Returns the vm console object for parsing logs
341 :return: the console output object
343 return nova_utils.get_server_console_output(self.__nova, self.__vm)
345 def get_port_ip(self, port_name, subnet_name=None):
347 Returns the first IP for the port corresponding with the port_name
348 parameter when subnet_name is None else returns the IP address that
349 corresponds to the subnet_name parameter
350 :param port_name: the name of the port from which to return the IP
351 :param subnet_name: the name of the subnet attached to this IP
352 :return: the IP or None if not found
354 port = self.get_port_by_name(port_name)
357 subnet = neutron_utils.get_subnet_by_name(self.__neutron,
360 logger.warning('Cannot retrieve port IP as subnet could '
361 'not be located with name - %s',
364 for fixed_ip in port.ips:
365 if fixed_ip['subnet_id'] == subnet.id:
366 return fixed_ip['ip_address']
368 if port.ips and len(port.ips) > 0:
369 return port.ips[0]['ip_address']
372 def get_port_mac(self, port_name):
374 Returns the first IP for the port corresponding with the port_name
376 TODO - Add in the subnet as an additional parameter as a port may have
378 :param port_name: the name of the port from which to return the IP
379 :return: the IP or None if not found
381 port = self.get_port_by_name(port_name)
383 return port.mac_address
386 def get_port_by_name(self, port_name):
388 Retrieves the OpenStack port object by its given name
389 :param port_name: the name of the port
390 :return: the OpenStack port object or None if not exists
392 for key, port in self.__ports:
395 logger.warning('Cannot find port with name - ' + port_name)
398 def get_vm_info(self):
400 Returns a dictionary of a VMs info as returned by OpenStack
403 return nova_utils.get_server_info(self.__nova, self.__vm)
405 def config_nics(self):
407 Responsible for configuring NICs on RPM systems where the instance has
408 more than one configured port
409 :return: the value returned by ansible_utils.apply_ansible_playbook()
411 if len(self.__ports) > 1 and len(self.__floating_ips) > 0:
412 if self.vm_active(block=True) and self.vm_ssh_active(block=True):
413 for key, port in self.__ports:
414 port_index = self.__ports.index((key, port))
416 nic_name = 'eth' + repr(port_index)
417 retval = self.__config_nic(
419 self.__get_first_provisioning_floating_ip().ip)
420 logger.info('Configured NIC - %s on VM - %s',
421 nic_name, self.instance_settings.name)
424 def __get_first_provisioning_floating_ip(self):
426 Returns the first floating IP tagged with the Floating IP name if
427 exists else the first one found
430 for floating_ip_setting in self.instance_settings.floating_ip_settings:
431 if floating_ip_setting.provisioning:
432 fip = self.__floating_ip_dict.get(floating_ip_setting.name)
435 elif len(self.__floating_ips) > 0:
436 return self.__floating_ips[0]
438 def __config_nic(self, nic_name, port, ip):
440 Although ports/NICs can contain multiple IPs, this code currently only
443 :param nic_name: Name of the interface
444 :param port: The port information containing the expected IP values.
445 :param ip: The IP on which to apply the playbook.
446 :return: the return value from ansible
448 port_ip = port.ips[0]['ip_address']
451 'nic_name': nic_name,
455 if self.image_settings.nic_config_pb_loc and self.keypair_settings:
456 return self.apply_ansible_playbook(
457 self.image_settings.nic_config_pb_loc, variables)
460 'VM %s cannot self configure NICs eth1++. No playbook or '
461 'keypairs found.', self.instance_settings.name)
463 def apply_ansible_playbook(self, pb_file_loc, variables=None,
466 Applies a playbook to a VM
467 :param pb_file_loc: the file location of the playbook to be applied
468 :param variables: a dict() of substitution values required by the
470 :param fip_name: the name of the floating IP to use for applying the
471 playbook (default - will take the first)
472 :return: the return value from ansible
474 return ansible_utils.apply_playbook(
475 pb_file_loc, [self.get_floating_ip(fip_name=fip_name).ip],
476 self.get_image_user(), self.keypair_settings.private_filepath,
477 variables, self.__os_creds.proxy_settings)
479 def get_image_user(self):
481 Returns the instance sudo_user if it has been configured in the
482 instance_settings else it returns the image_settings.image_user value
484 if self.instance_settings.sudo_user:
485 return self.instance_settings.sudo_user
487 return self.image_settings.image_user
489 def vm_deleted(self, block=False, poll_interval=POLL_INTERVAL):
491 Returns true when the VM status returns the value of
492 expected_status_code or instance retrieval throws a NotFound exception.
493 :param block: When true, thread will block until active or timeout
494 value in seconds has been exceeded (False)
495 :param poll_interval: The polling interval in seconds
499 return self.__vm_status_check(
500 STATUS_DELETED, block,
501 self.instance_settings.vm_delete_timeout, poll_interval)
502 except NotFound as e:
504 "Instance not found when querying status for %s with message "
505 "%s", STATUS_DELETED, e)
508 def vm_active(self, block=False, poll_interval=POLL_INTERVAL):
510 Returns true when the VM status returns the value of
512 :param block: When true, thread will block until active or timeout
513 value in seconds has been exceeded (False)
514 :param poll_interval: The polling interval in seconds
517 return self.__vm_status_check(STATUS_ACTIVE, block,
518 self.instance_settings.vm_boot_timeout,
521 def __vm_status_check(self, expected_status_code, block, timeout,
524 Returns true when the VM status returns the value of
526 :param expected_status_code: instance status evaluated with this
528 :param block: When true, thread will block until active or timeout
529 value in seconds has been exceeded (False)
530 :param timeout: The timeout value
531 :param poll_interval: The polling interval in seconds
534 # sleep and wait for VM status change
538 return self.__status(expected_status_code)
540 while timeout > time.time() - start:
541 status = self.__status(expected_status_code)
543 logger.info('VM is - ' + expected_status_code)
546 logger.debug('Retry querying VM status in ' + str(
547 poll_interval) + ' seconds')
548 time.sleep(poll_interval)
549 logger.debug('VM status query timeout in ' + str(
550 timeout - (time.time() - start)))
553 'Timeout checking for VM status for ' + expected_status_code)
556 def __status(self, expected_status_code):
558 Returns True when active else False
559 :param expected_status_code: instance status evaluated with this string
566 status = nova_utils.get_server_status(self.__nova, self.__vm)
568 logger.warning('Cannot find instance with id - ' + self.__vm.id)
571 if status == 'ERROR':
572 raise VmInstanceCreationError(
573 'Instance had an error during deployment')
575 'Instance status [%s] is - %s', self.instance_settings.name,
577 return status == expected_status_code
579 def vm_ssh_active(self, block=False, poll_interval=POLL_INTERVAL):
581 Returns true when the VM can be accessed via SSH
582 :param block: When true, thread will block until active or timeout
583 value in seconds has been exceeded (False)
584 :param poll_interval: The polling interval
587 # sleep and wait for VM status change
588 logger.info('Checking if VM is active')
590 timeout = self.instance_settings.ssh_connect_timeout
592 if self.vm_active(block=True):
596 start = time.time() - timeout
598 while timeout > time.time() - start:
599 status = self.__ssh_active()
601 logger.info('SSH is active for VM instance')
604 logger.debug('Retry SSH connection in ' + str(
605 poll_interval) + ' seconds')
606 time.sleep(poll_interval)
607 logger.debug('SSH connection timeout in ' + str(
608 timeout - (time.time() - start)))
610 logger.error('Timeout attempting to connect with VM via SSH')
613 def __ssh_active(self):
615 Returns True when can create a SSH session else False
618 if len(self.__floating_ips) > 0:
619 ssh = self.ssh_client()
625 def get_floating_ip(self, fip_name=None):
627 Returns the floating IP object byt name if found, else the first known,
629 :param fip_name: the name of the floating IP to return
630 :return: the SSH client or None
633 if fip_name and self.__floating_ip_dict.get(fip_name):
634 return self.__floating_ip_dict.get(fip_name)
635 if not fip and len(self.__floating_ips) > 0:
636 return self.__floating_ips[0]
639 def ssh_client(self, fip_name=None):
641 Returns an SSH client using the name or the first known floating IP if
643 :param fip_name: the name of the floating IP to return
644 :return: the SSH client or None
646 fip = self.get_floating_ip(fip_name)
648 return ansible_utils.ssh_client(
649 self.__floating_ips[0].ip, self.get_image_user(),
650 self.keypair_settings.private_filepath,
651 proxy_settings=self.__os_creds.proxy_settings)
654 'Cannot return an SSH client. No Floating IP configured')
656 def add_security_group(self, security_group):
658 Adds a security group to this VM. Call will block until VM is active.
659 :param security_group: the SNAPS SecurityGroup domain object
660 :return True if successful else False
662 self.vm_active(block=True)
664 if not security_group:
665 logger.warning('Security group object is None, cannot add')
669 nova_utils.add_security_group(self.__nova, self.get_vm_inst(),
672 except NotFound as e:
673 logger.warning('Security group not added - ' + str(e))
676 def remove_security_group(self, security_group):
678 Removes a security group to this VM. Call will block until VM is active
679 :param security_group: the OpenStack security group object
680 :return True if successful else False
682 self.vm_active(block=True)
684 if not security_group:
685 logger.warning('Security group object is None, cannot remove')
689 nova_utils.remove_security_group(self.__nova, self.get_vm_inst(),
692 except NotFound as e:
693 logger.warning('Security group not removed - ' + str(e))
697 class VmInstanceSettings:
699 Class responsible for holding configuration setting for a VM Instance
702 def __init__(self, **kwargs):
705 :param name: the name of the VM
706 :param flavor: the VM's flavor
707 :param port_settings: the port configuration settings (required)
708 :param security_group_names: a set of names of the security groups to
710 :param floating_ip_settings: the floating IP configuration settings
711 :param sudo_user: the sudo user of the VM that will override the
712 instance_settings.image_user when trying to
714 :param vm_boot_timeout: the amount of time a thread will sleep waiting
715 for an instance to boot
716 :param vm_delete_timeout: the amount of time a thread will sleep
717 waiting for an instance to be deleted
718 :param ssh_connect_timeout: the amount of time a thread will sleep
719 waiting obtaining an SSH connection to a VM
720 :param availability_zone: the name of the compute server on which to
721 deploy the VM (optional)
722 :param userdata: the cloud-init script to run after the VM has been
725 self.name = kwargs.get('name')
726 self.flavor = kwargs.get('flavor')
727 self.sudo_user = kwargs.get('sudo_user')
728 self.userdata = kwargs.get('userdata')
730 self.port_settings = list()
731 port_settings = kwargs.get('ports')
732 if not port_settings:
733 port_settings = kwargs.get('port_settings')
735 for port_setting in port_settings:
736 if isinstance(port_setting, dict):
737 self.port_settings.append(PortSettings(**port_setting))
738 elif isinstance(port_setting, PortSettings):
739 self.port_settings.append(port_setting)
741 if kwargs.get('security_group_names'):
742 if isinstance(kwargs['security_group_names'], list):
743 self.security_group_names = kwargs['security_group_names']
744 elif isinstance(kwargs['security_group_names'], set):
745 self.security_group_names = kwargs['security_group_names']
746 elif isinstance(kwargs['security_group_names'], str):
747 self.security_group_names = [kwargs['security_group_names']]
749 raise VmInstanceSettingsError(
750 'Invalid data type for security_group_names attribute')
752 self.security_group_names = set()
754 self.floating_ip_settings = list()
755 floating_ip_settings = kwargs.get('floating_ips')
756 if not floating_ip_settings:
757 floating_ip_settings = kwargs.get('floating_ip_settings')
758 if floating_ip_settings:
759 for floating_ip_config in floating_ip_settings:
760 if isinstance(floating_ip_config, FloatingIpSettings):
761 self.floating_ip_settings.append(floating_ip_config)
763 self.floating_ip_settings.append(FloatingIpSettings(
764 **floating_ip_config['floating_ip']))
766 if kwargs.get('vm_boot_timeout'):
767 self.vm_boot_timeout = kwargs['vm_boot_timeout']
769 self.vm_boot_timeout = 900
771 if kwargs.get('vm_delete_timeout'):
772 self.vm_delete_timeout = kwargs['vm_delete_timeout']
774 self.vm_delete_timeout = 300
776 if kwargs.get('ssh_connect_timeout'):
777 self.ssh_connect_timeout = kwargs['ssh_connect_timeout']
779 self.ssh_connect_timeout = 180
781 if kwargs.get('availability_zone'):
782 self.availability_zone = kwargs['availability_zone']
784 self.availability_zone = None
786 if not self.name or not self.flavor:
787 raise VmInstanceSettingsError(
788 'Instance configuration requires the attributes: name, flavor')
790 if len(self.port_settings) == 0:
791 raise VmInstanceSettingsError(
792 'Instance configuration requires port settings (aka. NICS)')
795 class FloatingIpSettings:
797 Class responsible for holding configuration settings for a floating IP
800 def __init__(self, **kwargs):
803 :param name: the name of the floating IP
804 :param port_name: the name of the router to the external network
805 :param router_name: the name of the router to the external network
806 :param subnet_name: the name of the subnet on which to attach the
808 :param provisioning: when true, this floating IP can be used for
811 TODO - provisioning flag is a hack as I have only observed a single
812 Floating IPs that actually works on an instance. Multiple floating IPs
813 placed on different subnets from the same port are especially
814 troublesome as you cannot predict which one will actually connect.
815 For now, it is recommended not to setup multiple floating IPs on an
816 instance unless absolutely necessary.
818 self.name = kwargs.get('name')
819 self.port_name = kwargs.get('port_name')
820 self.router_name = kwargs.get('router_name')
821 self.subnet_name = kwargs.get('subnet_name')
822 if kwargs.get('provisioning') is not None:
823 self.provisioning = kwargs['provisioning']
825 self.provisioning = True
827 if not self.name or not self.port_name or not self.router_name:
828 raise FloatingIpSettingsError(
829 'The attributes name, port_name and router_name are required '
830 'for FloatingIPSettings')
833 class VmInstanceSettingsError(Exception):
835 Exception to be thrown when an VM instance settings are incorrect
839 class FloatingIpSettingsError(Exception):
841 Exception to be thrown when an VM instance settings are incorrect
845 class VmInstanceCreationError(Exception):
847 Exception to be thrown when an VM instance cannot be created