1 # Copyright (c) 2017 Cable Television Laboratories, Inc. ("CableLabs")
2 # and others. All rights reserved.
4 # Licensed under the Apache License, Version 2.0 (the "License");
5 # you may not use this file except in compliance with the License.
6 # You may obtain a copy of the License at:
8 # http://www.apache.org/licenses/LICENSE-2.0
10 # Unless required by applicable law or agreed to in writing, software
11 # distributed under the License is distributed on an "AS IS" BASIS,
12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 # See the License for the specific language governing permissions and
14 # limitations under the License.
18 from neutronclient.common.exceptions import PortNotFoundClient
19 from novaclient.exceptions import NotFound
21 from snaps.openstack.create_network import PortSettings
22 from snaps.openstack.utils import glance_utils
23 from snaps.openstack.utils import neutron_utils
24 from snaps.openstack.utils import nova_utils
25 from snaps.provisioning import ansible_utils
27 __author__ = 'spisarski'
29 logger = logging.getLogger('create_instance')
32 STATUS_ACTIVE = 'ACTIVE'
33 STATUS_DELETED = 'DELETED'
36 class OpenStackVmInstance:
38 Class responsible for creating a VM instance in OpenStack
41 def __init__(self, os_creds, instance_settings, image_settings,
42 keypair_settings=None):
45 :param os_creds: The connection credentials to the OpenStack API
46 :param instance_settings: Contains the settings for this VM
47 :param image_settings: The OpenStack image object settings
48 :param keypair_settings: The keypair metadata (Optional)
51 self.__os_creds = os_creds
56 self.instance_settings = instance_settings
57 self.image_settings = image_settings
58 self.keypair_settings = keypair_settings
60 # TODO - get rid of FIP list and only use the dict(). Need to fix
61 # populating this object when already exists
62 self.__floating_ips = list()
63 self.__floating_ip_dict = dict()
65 # Instantiated in self.create()
68 # Note: this object does not change after the VM becomes active
71 def create(self, cleanup=False, block=False):
74 :param cleanup: When true, only perform lookups for OpenStack objects.
75 :param block: Thread will block until instance has either become
76 active, error, or timeout waiting.
77 Additionally, when True, floating IPs will not be applied
79 :return: The VM reference object
81 self.__nova = nova_utils.nova_client(self.__os_creds)
82 self.__neutron = neutron_utils.neutron_client(self.__os_creds)
84 self.__ports = self.__setup_ports(self.instance_settings.port_settings,
86 self.__lookup_existing_vm_by_name()
87 if not self.__vm and not cleanup:
88 self.__create_vm(block)
91 def __lookup_existing_vm_by_name(self):
93 Populates the member variables 'self.vm' and 'self.floating_ips' if a
94 VM with the same name already exists
97 servers = nova_utils.get_servers_by_name(self.__nova,
98 self.instance_settings.name)
99 for server in servers:
100 if server.name == self.instance_settings.name:
103 'Found existing machine with name - %s',
104 self.instance_settings.name)
105 fips = neutron_utils.get_floating_ips(self.__nova)
107 if fip.instance_id == server.id:
108 self.__floating_ips.append(fip)
109 # TODO - Determine a means to associate to the FIP
110 # configuration and add to FIP map
112 def __create_vm(self, block=False):
114 Responsible for creating the VM instance
115 :param block: Thread will block until instance has either become
116 active, error, or timeout waiting. Floating IPs will be
117 assigned after active when block=True
119 glance = glance_utils.glance_client(self.__os_creds)
120 self.__vm = nova_utils.create_server(
121 self.__nova, self.__neutron, glance, self.instance_settings,
122 self.image_settings, self.keypair_settings)
123 logger.info('Created instance with name - %s',
124 self.instance_settings.name)
127 if not self.vm_active(block=True):
129 'Fatal error, VM did not become ACTIVE within the alloted '
132 # Create server should do this but found it needed to occur here
133 for sec_grp_name in self.instance_settings.security_group_names:
134 if self.vm_active(block=True):
135 nova_utils.add_security_group(self.__nova, self.__vm,
139 'Cannot applying security group with name ' +
141 ' to VM that did not activate with name - ' +
142 self.instance_settings.name)
144 self.__apply_floating_ips()
146 def __apply_floating_ips(self):
148 Applies the configured floating IPs to the necessary ports
151 for key, port in self.__ports:
152 port_dict[key] = port
155 for floating_ip_setting in self.instance_settings.floating_ip_settings:
156 port = port_dict.get(floating_ip_setting.port_name)
160 'Cannot find port object with name - ' +
161 floating_ip_setting.port_name)
163 # Setup Floating IP only if there is a router with an external
165 ext_gateway = self.__ext_gateway_by_router(
166 floating_ip_setting.router_name)
168 subnet = neutron_utils.get_subnet_by_name(
169 self.__neutron, floating_ip_setting.subnet_name)
170 floating_ip = neutron_utils.create_floating_ip(
171 self.__neutron, ext_gateway)
172 self.__floating_ips.append(floating_ip)
173 self.__floating_ip_dict[floating_ip_setting.name] = floating_ip
176 'Created floating IP %s via router - %s', floating_ip.ip,
177 floating_ip_setting.router_name)
178 self.__add_floating_ip(floating_ip, port, subnet)
180 raise Exception('Unable to add floating IP to port,'
181 ' cannot locate router with an external '
184 def __ext_gateway_by_router(self, router_name):
186 Returns network name for the external network attached to a router or
188 :param router_name: The name of the router to lookup
189 :return: the external network name or None
191 router = neutron_utils.get_router_by_name(self.__neutron, router_name)
192 if router and router['router'].get('external_gateway_info'):
193 network = neutron_utils.get_network_by_id(
195 router['router']['external_gateway_info']['network_id'])
197 return network['network']['name']
202 Destroys the VM instance
205 # Cleanup floating IPs
206 for floating_ip in self.__floating_ips:
208 logger.info('Deleting Floating IP - ' + floating_ip.ip)
209 neutron_utils.delete_floating_ip(self.__neutron, floating_ip)
210 except Exception as e:
211 logger.error('Error deleting Floating IP - ' + str(e))
212 self.__floating_ips = list()
213 self.__floating_ip_dict = dict()
216 for name, port in self.__ports:
217 logger.info('Deleting Port - ' + name)
219 neutron_utils.delete_port(self.__neutron, port)
220 except PortNotFoundClient as e:
221 logger.warning('Unexpected error deleting port - %s', e)
223 self.__ports = list()
229 'Deleting VM instance - ' + self.instance_settings.name)
230 nova_utils.delete_vm_instance(self.__nova, self.__vm)
231 except Exception as e:
232 logger.error('Error deleting VM - %s', e)
234 # Block until instance cannot be found or returns the status of
236 logger.info('Checking deletion status')
239 if self.vm_deleted(block=True):
241 'VM has been properly deleted VM with name - %s',
242 self.instance_settings.name)
246 'VM not deleted within the timeout period of %s '
247 'seconds', self.instance_settings.vm_delete_timeout)
248 except Exception as e:
250 'Unexpected error while checking VM instance status - %s',
253 def __setup_ports(self, port_settings, cleanup):
255 Returns the previously configured ports or creates them if they do not
257 :param port_settings: A list of PortSetting objects
258 :param cleanup: When true, only perform lookups for OpenStack objects.
259 :return: a list of OpenStack port tuples where the first member is the
260 port name and the second is the port object
264 for port_setting in port_settings:
265 # First check to see if network already has this port
266 # TODO/FIXME - this could potentially cause problems if another
267 # port with the same name exists
268 # VM has the same network/port name pair
271 # TODO/FIXME - should we not be iterating on ports for the specific
272 # network in question as unique port names
273 # seem to only be important by network
274 existing_ports = self.__neutron.list_ports()['ports']
275 for existing_port in existing_ports:
276 if existing_port['name'] == port_setting.name:
277 ports.append((port_setting.name, {'port': existing_port}))
281 if not found and not cleanup:
282 ports.append((port_setting.name,
283 neutron_utils.create_port(self.__neutron,
289 def __add_floating_ip(self, floating_ip, port, subnet, timeout=30,
290 poll_interval=POLL_INTERVAL):
292 Returns True when active else False
293 TODO - Make timeout and poll_interval configurable...
298 # Take IP of subnet if there is one configured on which to place
300 for fixed_ip in port['port']['fixed_ips']:
301 if fixed_ip['subnet_id'] == subnet['subnet']['id']:
302 ip = fixed_ip['ip_address']
305 # Simply take the first
306 ip = port['port']['fixed_ips'][0]['ip_address']
309 count = timeout / poll_interval
311 logger.debug('Attempting to add floating IP to instance')
313 nova_utils.add_floating_ip_to_server(
314 self.__nova, self.__vm, floating_ip, ip)
316 'Added floating IP %s to port IP %s on instance %s',
317 floating_ip.ip, ip, self.instance_settings.name)
319 except Exception as e:
321 'Retry adding floating IP to instance. Last attempt '
322 'failed with - %s', e)
323 time.sleep(poll_interval)
328 'Unable find IP address on which to place the floating IP')
330 logger.error('Timeout attempting to add the floating IP to instance.')
331 raise Exception('Timeout while attempting add floating IP to instance')
333 def get_os_creds(self):
335 Returns the OpenStack credentials used to create these objects
336 :return: the credentials
338 return self.__os_creds
340 def get_vm_inst(self):
342 Returns the latest version of this server object from OpenStack
343 :return: Server object
347 def get_os_vm_server_obj(self):
349 Returns the OpenStack server object
350 :return: the server object
352 return nova_utils.get_latest_server_os_object(self.__nova, self.__vm)
354 def get_port_ip(self, port_name, subnet_name=None):
356 Returns the first IP for the port corresponding with the port_name
357 parameter when subnet_name is None else returns the IP address that
358 corresponds to the subnet_name parameter
359 :param port_name: the name of the port from which to return the IP
360 :param subnet_name: the name of the subnet attached to this IP
361 :return: the IP or None if not found
363 port = self.get_port_by_name(port_name)
365 port_dict = port['port']
367 subnet = neutron_utils.get_subnet_by_name(self.__neutron,
370 logger.warning('Cannot retrieve port IP as subnet could '
371 'not be located with name - %s',
374 for fixed_ip in port_dict['fixed_ips']:
375 if fixed_ip['subnet_id'] == subnet['subnet']['id']:
376 return fixed_ip['ip_address']
378 fixed_ips = port_dict['fixed_ips']
379 if fixed_ips and len(fixed_ips) > 0:
380 return fixed_ips[0]['ip_address']
383 def get_port_mac(self, port_name):
385 Returns the first IP for the port corresponding with the port_name
387 TODO - Add in the subnet as an additional parameter as a port may have
389 :param port_name: the name of the port from which to return the IP
390 :return: the IP or None if not found
392 port = self.get_port_by_name(port_name)
394 port_dict = port['port']
395 return port_dict['mac_address']
398 def get_port_by_name(self, port_name):
400 Retrieves the OpenStack port object by its given name
401 :param port_name: the name of the port
402 :return: the OpenStack port object or None if not exists
404 for key, port in self.__ports:
407 logger.warning('Cannot find port with name - ' + port_name)
410 def config_nics(self):
412 Responsible for configuring NICs on RPM systems where the instance has
413 more than one configured port
416 if len(self.__ports) > 1 and len(self.__floating_ips) > 0:
417 if self.vm_active(block=True) and self.vm_ssh_active(block=True):
418 for key, port in self.__ports:
419 port_index = self.__ports.index((key, port))
421 nic_name = 'eth' + repr(port_index)
424 self.__get_first_provisioning_floating_ip().ip)
425 logger.info('Configured NIC - %s on VM - %s',
426 nic_name, self.instance_settings.name)
428 def __get_first_provisioning_floating_ip(self):
430 Returns the first floating IP tagged with the Floating IP name if
431 exists else the first one found
434 for floating_ip_setting in self.instance_settings.floating_ip_settings:
435 if floating_ip_setting.provisioning:
436 fip = self.__floating_ip_dict.get(floating_ip_setting.name)
439 elif len(self.__floating_ips) > 0:
440 return self.__floating_ips[0]
442 def __config_nic(self, nic_name, port, ip):
444 Although ports/NICs can contain multiple IPs, this code currently only
447 :param nic_name: Name of the interface
448 :param port: The port information containing the expected IP values.
449 :param ip: The IP on which to apply the playbook.
450 :return: the return value from ansible
452 port_ip = port['port']['fixed_ips'][0]['ip_address']
455 'nic_name': nic_name,
459 if self.image_settings.nic_config_pb_loc and self.keypair_settings:
460 return self.apply_ansible_playbook(
461 self.image_settings.nic_config_pb_loc, variables)
464 'VM %s cannot self configure NICs eth1++. No playbook or '
465 'keypairs found.', self.instance_settings.name)
467 def apply_ansible_playbook(self, pb_file_loc, variables=None,
470 Applies a playbook to a VM
471 :param pb_file_loc: the file location of the playbook to be applied
472 :param variables: a dict() of substitution values required by the
474 :param fip_name: the name of the floating IP to use for applying the
475 playbook (default - will take the first)
476 :return: the return value from ansible
478 return ansible_utils.apply_playbook(
479 pb_file_loc, [self.get_floating_ip(fip_name=fip_name).ip],
480 self.get_image_user(), self.keypair_settings.private_filepath,
481 variables, self.__os_creds.proxy_settings)
483 def get_image_user(self):
485 Returns the instance sudo_user if it has been configured in the
486 instance_settings else it returns the image_settings.image_user value
488 if self.instance_settings.sudo_user:
489 return self.instance_settings.sudo_user
491 return self.image_settings.image_user
493 def vm_deleted(self, block=False, poll_interval=POLL_INTERVAL):
495 Returns true when the VM status returns the value of
496 expected_status_code or instance retrieval throws a NotFound exception.
497 :param block: When true, thread will block until active or timeout
498 value in seconds has been exceeded (False)
499 :param poll_interval: The polling interval in seconds
503 return self.__vm_status_check(
504 STATUS_DELETED, block,
505 self.instance_settings.vm_delete_timeout, poll_interval)
506 except NotFound as e:
508 "Instance not found when querying status for %s with message "
509 "%s", STATUS_DELETED, e)
512 def vm_active(self, block=False, poll_interval=POLL_INTERVAL):
514 Returns true when the VM status returns the value of
516 :param block: When true, thread will block until active or timeout
517 value in seconds has been exceeded (False)
518 :param poll_interval: The polling interval in seconds
521 return self.__vm_status_check(STATUS_ACTIVE, block,
522 self.instance_settings.vm_boot_timeout,
525 def __vm_status_check(self, expected_status_code, block, timeout,
528 Returns true when the VM status returns the value of
530 :param expected_status_code: instance status evaluated with this
532 :param block: When true, thread will block until active or timeout
533 value in seconds has been exceeded (False)
534 :param timeout: The timeout value
535 :param poll_interval: The polling interval in seconds
538 # sleep and wait for VM status change
542 return self.__status(expected_status_code)
544 while timeout > time.time() - start:
545 status = self.__status(expected_status_code)
547 logger.info('VM is - ' + expected_status_code)
550 logger.debug('Retry querying VM status in ' + str(
551 poll_interval) + ' seconds')
552 time.sleep(poll_interval)
553 logger.debug('VM status query timeout in ' + str(
554 timeout - (time.time() - start)))
557 'Timeout checking for VM status for ' + expected_status_code)
560 def __status(self, expected_status_code):
562 Returns True when active else False
563 :param expected_status_code: instance status evaluated with this string
570 instance = nova_utils.get_latest_server_os_object(
571 self.__nova, self.__vm)
573 logger.warning('Cannot find instance with id - ' + self.__vm.id)
576 if instance.status == 'ERROR':
577 raise Exception('Instance had an error during deployment')
579 'Instance status [%s] is - %s', self.instance_settings.name,
581 return instance.status == expected_status_code
583 def vm_ssh_active(self, block=False, poll_interval=POLL_INTERVAL):
585 Returns true when the VM can be accessed via SSH
586 :param block: When true, thread will block until active or timeout
587 value in seconds has been exceeded (False)
588 :param poll_interval: The polling interval
591 # sleep and wait for VM status change
592 logger.info('Checking if VM is active')
594 timeout = self.instance_settings.ssh_connect_timeout
596 if self.vm_active(block=True):
600 start = time.time() - timeout
602 while timeout > time.time() - start:
603 status = self.__ssh_active()
605 logger.info('SSH is active for VM instance')
608 logger.debug('Retry SSH connection in ' + str(
609 poll_interval) + ' seconds')
610 time.sleep(poll_interval)
611 logger.debug('SSH connection timeout in ' + str(
612 timeout - (time.time() - start)))
614 logger.error('Timeout attempting to connect with VM via SSH')
617 def __ssh_active(self):
619 Returns True when can create a SSH session else False
622 if len(self.__floating_ips) > 0:
623 ssh = self.ssh_client()
628 def get_floating_ip(self, fip_name=None):
630 Returns the floating IP object byt name if found, else the first known,
632 :param fip_name: the name of the floating IP to return
633 :return: the SSH client or None
636 if fip_name and self.__floating_ip_dict.get(fip_name):
637 return self.__floating_ip_dict.get(fip_name)
638 if not fip and len(self.__floating_ips) > 0:
639 return self.__floating_ips[0]
642 def ssh_client(self, fip_name=None):
644 Returns an SSH client using the name or the first known floating IP if
646 :param fip_name: the name of the floating IP to return
647 :return: the SSH client or None
649 fip = self.get_floating_ip(fip_name)
651 return ansible_utils.ssh_client(
652 self.__floating_ips[0].ip, self.get_image_user(),
653 self.keypair_settings.private_filepath,
654 proxy_settings=self.__os_creds.proxy_settings)
657 'Cannot return an SSH client. No Floating IP configured')
659 def add_security_group(self, security_group):
661 Adds a security group to this VM. Call will block until VM is active.
662 :param security_group: the OpenStack security group object
663 :return True if successful else False
665 self.vm_active(block=True)
667 if not security_group:
668 logger.warning('Security group object is None, cannot add')
672 nova_utils.add_security_group(self.__nova, self.get_vm_inst(),
673 security_group['security_group'][
676 except NotFound as e:
677 logger.warning('Security group not added - ' + str(e))
680 def remove_security_group(self, security_group):
682 Removes a security group to this VM. Call will block until VM is active
683 :param security_group: the OpenStack security group object
684 :return True if successful else False
686 self.vm_active(block=True)
688 if not security_group:
689 logger.warning('Security group object is None, cannot remove')
693 nova_utils.remove_security_group(self.__nova, self.get_vm_inst(),
696 except NotFound as e:
697 logger.warning('Security group not removed - ' + str(e))
701 class VmInstanceSettings:
703 Class responsible for holding configuration setting for a VM Instance
706 def __init__(self, **kwargs):
709 :param name: the name of the VM
710 :param flavor: the VM's flavor
711 :param port_settings: the port configuration settings (required)
712 :param security_group_names: a set of names of the security groups to
714 :param floating_ip_settings: the floating IP configuration settings
715 :param sudo_user: the sudo user of the VM that will override the
716 instance_settings.image_user when trying to
718 :param vm_boot_timeout: the amount of time a thread will sleep waiting
719 for an instance to boot
720 :param vm_delete_timeout: the amount of time a thread will sleep
721 waiting for an instance to be deleted
722 :param ssh_connect_timeout: the amount of time a thread will sleep
723 waiting obtaining an SSH connection to a VM
724 :param availability_zone: the name of the compute server on which to
725 deploy the VM (optional)
726 :param userdata: the cloud-init script to run after the VM has been
729 self.name = kwargs.get('name')
730 self.flavor = kwargs.get('flavor')
731 self.sudo_user = kwargs.get('sudo_user')
732 self.userdata = kwargs.get('userdata')
734 self.port_settings = list()
735 port_settings = kwargs.get('ports')
736 if not port_settings:
737 port_settings = kwargs.get('port_settings')
739 for port_setting in port_settings:
740 if isinstance(port_setting, dict):
741 self.port_settings.append(PortSettings(**port_setting))
742 elif isinstance(port_setting, PortSettings):
743 self.port_settings.append(port_setting)
745 if kwargs.get('security_group_names'):
746 if isinstance(kwargs['security_group_names'], list):
747 self.security_group_names = kwargs['security_group_names']
748 elif isinstance(kwargs['security_group_names'], set):
749 self.security_group_names = kwargs['security_group_names']
750 elif isinstance(kwargs['security_group_names'], str):
751 self.security_group_names = [kwargs['security_group_names']]
754 'Invalid data type for security_group_names attribute')
756 self.security_group_names = set()
758 self.floating_ip_settings = list()
759 floating_ip_settings = kwargs.get('floating_ips')
760 if not floating_ip_settings:
761 floating_ip_settings = kwargs.get('floating_ip_settings')
762 if floating_ip_settings:
763 for floating_ip_config in floating_ip_settings:
764 if isinstance(floating_ip_config, FloatingIpSettings):
765 self.floating_ip_settings.append(floating_ip_config)
767 self.floating_ip_settings.append(FloatingIpSettings(
768 **floating_ip_config['floating_ip']))
770 if kwargs.get('vm_boot_timeout'):
771 self.vm_boot_timeout = kwargs['vm_boot_timeout']
773 self.vm_boot_timeout = 900
775 if kwargs.get('vm_delete_timeout'):
776 self.vm_delete_timeout = kwargs['vm_delete_timeout']
778 self.vm_delete_timeout = 300
780 if kwargs.get('ssh_connect_timeout'):
781 self.ssh_connect_timeout = kwargs['ssh_connect_timeout']
783 self.ssh_connect_timeout = 180
785 if kwargs.get('availability_zone'):
786 self.availability_zone = kwargs['availability_zone']
788 self.availability_zone = None
790 if not self.name or not self.flavor:
792 'Instance configuration requires the attributes: name, flavor')
794 if len(self.port_settings) == 0:
796 'Instance configuration requires port settings (aka. NICS)')
799 class FloatingIpSettings:
801 Class responsible for holding configuration settings for a floating IP
804 def __init__(self, **kwargs):
807 :param name: the name of the floating IP
808 :param port_name: the name of the router to the external network
809 :param router_name: the name of the router to the external network
810 :param subnet_name: the name of the subnet on which to attach the
812 :param provisioning: when true, this floating IP can be used for
815 TODO - provisioning flag is a hack as I have only observed a single
816 Floating IPs that actually works on an instance. Multiple floating IPs
817 placed on different subnets from the same port are especially
818 troublesome as you cannot predict which one will actually connect.
819 For now, it is recommended not to setup multiple floating IPs on an
820 instance unless absolutely necessary.
822 self.name = kwargs.get('name')
823 self.port_name = kwargs.get('port_name')
824 self.router_name = kwargs.get('router_name')
825 self.subnet_name = kwargs.get('subnet_name')
826 if kwargs.get('provisioning') is not None:
827 self.provisioning = kwargs['provisioning']
829 self.provisioning = True
831 if not self.name or not self.port_name or not self.router_name:
833 'The attributes name, port_name and router_name are required '
834 'for FloatingIPSettings')