1 # Copyright (c) 2017 Cable Television Laboratories, Inc. ("CableLabs")
2 # and others. All rights reserved.
4 # Licensed under the Apache License, Version 2.0 (the "License");
5 # you may not use this file except in compliance with the License.
6 # You may obtain a copy of the License at:
8 # http://www.apache.org/licenses/LICENSE-2.0
10 # Unless required by applicable law or agreed to in writing, software
11 # distributed under the License is distributed on an "AS IS" BASIS,
12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 # See the License for the specific language governing permissions and
14 # limitations under the License.
18 from neutronclient.common.exceptions import PortNotFoundClient
19 from novaclient.exceptions import NotFound
21 from snaps.openstack.create_network import PortSettings
22 from snaps.openstack.utils import glance_utils
23 from snaps.openstack.utils import neutron_utils
24 from snaps.openstack.utils import nova_utils
25 from snaps.provisioning import ansible_utils
27 __author__ = 'spisarski'
29 logger = logging.getLogger('create_instance')
32 STATUS_ACTIVE = 'ACTIVE'
33 STATUS_DELETED = 'DELETED'
36 class OpenStackVmInstance:
38 Class responsible for creating a VM instance in OpenStack
41 def __init__(self, os_creds, instance_settings, image_settings,
42 keypair_settings=None):
45 :param os_creds: The connection credentials to the OpenStack API
46 :param instance_settings: Contains the settings for this VM
47 :param image_settings: The OpenStack image object settings
48 :param keypair_settings: The keypair metadata (Optional)
51 self.__os_creds = os_creds
56 self.instance_settings = instance_settings
57 self.image_settings = image_settings
58 self.keypair_settings = keypair_settings
60 self.__floating_ip_dict = dict()
62 # Instantiated in self.create()
65 # Note: this object does not change after the VM becomes active
68 def create(self, cleanup=False, block=False):
71 :param cleanup: When true, only perform lookups for OpenStack objects.
72 :param block: Thread will block until instance has either become
73 active, error, or timeout waiting.
74 Additionally, when True, floating IPs will not be applied
76 :return: The VM reference object
78 self.__nova = nova_utils.nova_client(self.__os_creds)
79 self.__neutron = neutron_utils.neutron_client(self.__os_creds)
81 self.__ports = self.__setup_ports(self.instance_settings.port_settings,
83 self.__lookup_existing_vm_by_name()
84 if not self.__vm and not cleanup:
85 self.__create_vm(block)
88 def __lookup_existing_vm_by_name(self):
90 Populates the member variables 'self.vm' and 'self.floating_ips' if a
91 VM with the same name already exists
94 server = nova_utils.get_server(
95 self.__nova, vm_inst_settings=self.instance_settings)
97 if server.name == self.instance_settings.name:
100 'Found existing machine with name - %s',
101 self.instance_settings.name)
103 fips = neutron_utils.get_floating_ips(self.__neutron,
105 for port_name, fip in fips:
106 settings = self.instance_settings.floating_ip_settings
107 for fip_setting in settings:
108 if port_name == fip_setting.port_name:
109 self.__floating_ip_dict[fip_setting.name] = fip
111 def __create_vm(self, block=False):
113 Responsible for creating the VM instance
114 :param block: Thread will block until instance has either become
115 active, error, or timeout waiting. Floating IPs will be
116 assigned after active when block=True
118 glance = glance_utils.glance_client(self.__os_creds)
119 self.__vm = nova_utils.create_server(
120 self.__nova, self.__neutron, glance, self.instance_settings,
121 self.image_settings, self.keypair_settings)
122 logger.info('Created instance with name - %s',
123 self.instance_settings.name)
126 if not self.vm_active(block=True):
127 raise VmInstanceCreationError(
128 'Fatal error, VM did not become ACTIVE within the alloted '
131 # Create server should do this but found it needed to occur here
132 for sec_grp_name in self.instance_settings.security_group_names:
133 if self.vm_active(block=True):
134 nova_utils.add_security_group(self.__nova, self.__vm,
137 raise VmInstanceCreationError(
138 'Cannot applying security group with name ' +
140 ' to VM that did not activate with name - ' +
141 self.instance_settings.name)
143 self.__apply_floating_ips()
145 def __apply_floating_ips(self):
147 Applies the configured floating IPs to the necessary ports
150 for key, port in self.__ports:
151 port_dict[key] = port
154 for floating_ip_setting in self.instance_settings.floating_ip_settings:
155 port = port_dict.get(floating_ip_setting.port_name)
158 raise VmInstanceCreationError(
159 'Cannot find port object with name - ' +
160 floating_ip_setting.port_name)
162 # Setup Floating IP only if there is a router with an external
164 ext_gateway = self.__ext_gateway_by_router(
165 floating_ip_setting.router_name)
167 subnet = neutron_utils.get_subnet(
169 subnet_name=floating_ip_setting.subnet_name)
170 floating_ip = neutron_utils.create_floating_ip(
171 self.__neutron, ext_gateway)
172 self.__floating_ip_dict[floating_ip_setting.name] = floating_ip
175 'Created floating IP %s via router - %s', floating_ip.ip,
176 floating_ip_setting.router_name)
177 self.__add_floating_ip(floating_ip, port, subnet)
179 raise VmInstanceCreationError(
180 'Unable to add floating IP to port, cannot locate router '
181 'with an external gateway ')
183 def __ext_gateway_by_router(self, router_name):
185 Returns network name for the external network attached to a router or
187 :param router_name: The name of the router to lookup
188 :return: the external network name or None
190 router = neutron_utils.get_router(
191 self.__neutron, router_name=router_name)
192 if router and router.external_gateway_info:
193 network = neutron_utils.get_network_by_id(
195 router.external_gateway_info['network_id'])
202 Destroys the VM instance
205 # Cleanup floating IPs
206 for name, floating_ip in self.__floating_ip_dict.items():
208 logger.info('Deleting Floating IP - ' + floating_ip.ip)
209 neutron_utils.delete_floating_ip(self.__neutron, floating_ip)
210 except Exception as e:
211 logger.error('Error deleting Floating IP - ' + str(e))
212 self.__floating_ip_dict = dict()
215 for name, port in self.__ports:
216 logger.info('Deleting Port - ' + name)
218 neutron_utils.delete_port(self.__neutron, port)
219 except PortNotFoundClient as e:
220 logger.warning('Unexpected error deleting port - %s', e)
222 self.__ports = list()
228 'Deleting VM instance - ' + self.instance_settings.name)
229 nova_utils.delete_vm_instance(self.__nova, self.__vm)
230 except Exception as e:
231 logger.error('Error deleting VM - %s', e)
233 # Block until instance cannot be found or returns the status of
235 logger.info('Checking deletion status')
238 if self.vm_deleted(block=True):
240 'VM has been properly deleted VM with name - %s',
241 self.instance_settings.name)
245 'VM not deleted within the timeout period of %s '
246 'seconds', self.instance_settings.vm_delete_timeout)
247 except Exception as e:
249 'Unexpected error while checking VM instance status - %s',
252 def __setup_ports(self, port_settings, cleanup):
254 Returns the previously configured ports or creates them if they do not
256 :param port_settings: A list of PortSetting objects
257 :param cleanup: When true, only perform lookups for OpenStack objects.
258 :return: a list of OpenStack port tuples where the first member is the
259 port name and the second is the port object
263 for port_setting in port_settings:
264 port = neutron_utils.get_port(
265 self.__neutron, port_settings=port_setting)
267 ports.append((port_setting.name, port))
269 # Exception will be raised when port with same name already
272 (port_setting.name, neutron_utils.create_port(
273 self.__neutron, self.__os_creds, port_setting)))
277 def __add_floating_ip(self, floating_ip, port, subnet, timeout=30,
278 poll_interval=POLL_INTERVAL):
280 Returns True when active else False
281 TODO - Make timeout and poll_interval configurable...
286 # Take IP of subnet if there is one configured on which to place
288 for fixed_ip in port.ips:
289 if fixed_ip['subnet_id'] == subnet.id:
290 ip = fixed_ip['ip_address']
293 # Simply take the first
294 ip = port.ips[0]['ip_address']
297 count = timeout / poll_interval
299 logger.debug('Attempting to add floating IP to instance')
301 nova_utils.add_floating_ip_to_server(
302 self.__nova, self.__vm, floating_ip, ip)
304 'Added floating IP %s to port IP %s on instance %s',
305 floating_ip.ip, ip, self.instance_settings.name)
307 except Exception as e:
309 'Retry adding floating IP to instance. Last attempt '
310 'failed with - %s', e)
311 time.sleep(poll_interval)
315 raise VmInstanceCreationError(
316 'Unable find IP address on which to place the floating IP')
318 logger.error('Timeout attempting to add the floating IP to instance.')
319 raise VmInstanceCreationError(
320 'Timeout while attempting add floating IP to instance')
322 def get_os_creds(self):
324 Returns the OpenStack credentials used to create these objects
325 :return: the credentials
327 return self.__os_creds
329 def get_vm_inst(self):
331 Returns the latest version of this server object from OpenStack
332 :return: Server object
336 def get_console_output(self):
338 Returns the vm console object for parsing logs
339 :return: the console output object
341 return nova_utils.get_server_console_output(self.__nova, self.__vm)
343 def get_port_ip(self, port_name, subnet_name=None):
345 Returns the first IP for the port corresponding with the port_name
346 parameter when subnet_name is None else returns the IP address that
347 corresponds to the subnet_name parameter
348 :param port_name: the name of the port from which to return the IP
349 :param subnet_name: the name of the subnet attached to this IP
350 :return: the IP or None if not found
352 port = self.get_port_by_name(port_name)
355 subnet = neutron_utils.get_subnet(
356 self.__neutron, subnet_name=subnet_name)
358 logger.warning('Cannot retrieve port IP as subnet could '
359 'not be located with name - %s',
362 for fixed_ip in port.ips:
363 if fixed_ip['subnet_id'] == subnet.id:
364 return fixed_ip['ip_address']
366 if port.ips and len(port.ips) > 0:
367 return port.ips[0]['ip_address']
370 def get_port_mac(self, port_name):
372 Returns the first IP for the port corresponding with the port_name
374 TODO - Add in the subnet as an additional parameter as a port may have
376 :param port_name: the name of the port from which to return the IP
377 :return: the IP or None if not found
379 port = self.get_port_by_name(port_name)
381 return port.mac_address
384 def get_port_by_name(self, port_name):
386 Retrieves the OpenStack port object by its given name
387 :param port_name: the name of the port
388 :return: the OpenStack port object or None if not exists
390 for key, port in self.__ports:
393 logger.warning('Cannot find port with name - ' + port_name)
396 def get_vm_info(self):
398 Returns a dictionary of a VMs info as returned by OpenStack
401 return nova_utils.get_server_info(self.__nova, self.__vm)
403 def config_nics(self):
405 Responsible for configuring NICs on RPM systems where the instance has
406 more than one configured port
407 :return: the value returned by ansible_utils.apply_ansible_playbook()
409 if len(self.__ports) > 1 and len(self.__floating_ip_dict) > 0:
410 if self.vm_active(block=True) and self.vm_ssh_active(block=True):
411 for key, port in self.__ports:
412 port_index = self.__ports.index((key, port))
414 nic_name = 'eth' + repr(port_index)
415 retval = self.__config_nic(
417 self.__get_first_provisioning_floating_ip().ip)
418 logger.info('Configured NIC - %s on VM - %s',
419 nic_name, self.instance_settings.name)
422 def __get_first_provisioning_floating_ip(self):
424 Returns the first floating IP tagged with the Floating IP name if
425 exists else the first one found
428 for floating_ip_setting in self.instance_settings.floating_ip_settings:
429 if floating_ip_setting.provisioning:
430 fip = self.__floating_ip_dict.get(floating_ip_setting.name)
433 elif len(self.__floating_ip_dict) > 0:
434 for key, fip in self.__floating_ip_dict.items():
437 def __config_nic(self, nic_name, port, ip):
439 Although ports/NICs can contain multiple IPs, this code currently only
442 :param nic_name: Name of the interface
443 :param port: The port information containing the expected IP values.
444 :param ip: The IP on which to apply the playbook.
445 :return: the return value from ansible
447 port_ip = port.ips[0]['ip_address']
450 'nic_name': nic_name,
454 if self.image_settings.nic_config_pb_loc and self.keypair_settings:
455 return self.apply_ansible_playbook(
456 self.image_settings.nic_config_pb_loc, variables)
459 'VM %s cannot self configure NICs eth1++. No playbook or '
460 'keypairs found.', self.instance_settings.name)
462 def apply_ansible_playbook(self, pb_file_loc, variables=None,
465 Applies a playbook to a VM
466 :param pb_file_loc: the file location of the playbook to be applied
467 :param variables: a dict() of substitution values required by the
469 :param fip_name: the name of the floating IP to use for applying the
470 playbook (default - will take the first)
471 :return: the return value from ansible
473 return ansible_utils.apply_playbook(
474 pb_file_loc, [self.get_floating_ip(fip_name=fip_name).ip],
475 self.get_image_user(), self.keypair_settings.private_filepath,
476 variables, self.__os_creds.proxy_settings)
478 def get_image_user(self):
480 Returns the instance sudo_user if it has been configured in the
481 instance_settings else it returns the image_settings.image_user value
483 if self.instance_settings.sudo_user:
484 return self.instance_settings.sudo_user
486 return self.image_settings.image_user
488 def vm_deleted(self, block=False, poll_interval=POLL_INTERVAL):
490 Returns true when the VM status returns the value of
491 expected_status_code or instance retrieval throws a NotFound exception.
492 :param block: When true, thread will block until active or timeout
493 value in seconds has been exceeded (False)
494 :param poll_interval: The polling interval in seconds
498 return self.__vm_status_check(
499 STATUS_DELETED, block,
500 self.instance_settings.vm_delete_timeout, poll_interval)
501 except NotFound as e:
503 "Instance not found when querying status for %s with message "
504 "%s", STATUS_DELETED, e)
507 def vm_active(self, block=False, poll_interval=POLL_INTERVAL):
509 Returns true when the VM status returns the value of
511 :param block: When true, thread will block until active or timeout
512 value in seconds has been exceeded (False)
513 :param poll_interval: The polling interval in seconds
516 return self.__vm_status_check(STATUS_ACTIVE, block,
517 self.instance_settings.vm_boot_timeout,
520 def __vm_status_check(self, expected_status_code, block, timeout,
523 Returns true when the VM status returns the value of
525 :param expected_status_code: instance status evaluated with this
527 :param block: When true, thread will block until active or timeout
528 value in seconds has been exceeded (False)
529 :param timeout: The timeout value
530 :param poll_interval: The polling interval in seconds
533 # sleep and wait for VM status change
537 return self.__status(expected_status_code)
539 while timeout > time.time() - start:
540 status = self.__status(expected_status_code)
542 logger.info('VM is - ' + expected_status_code)
545 logger.debug('Retry querying VM status in ' + str(
546 poll_interval) + ' seconds')
547 time.sleep(poll_interval)
548 logger.debug('VM status query timeout in ' + str(
549 timeout - (time.time() - start)))
552 'Timeout checking for VM status for ' + expected_status_code)
555 def __status(self, expected_status_code):
557 Returns True when active else False
558 :param expected_status_code: instance status evaluated with this string
565 status = nova_utils.get_server_status(self.__nova, self.__vm)
567 logger.warning('Cannot find instance with id - ' + self.__vm.id)
570 if status == 'ERROR':
571 raise VmInstanceCreationError(
572 'Instance had an error during deployment')
574 'Instance status [%s] is - %s', self.instance_settings.name,
576 return status == expected_status_code
578 def vm_ssh_active(self, block=False, poll_interval=POLL_INTERVAL):
580 Returns true when the VM can be accessed via SSH
581 :param block: When true, thread will block until active or timeout
582 value in seconds has been exceeded (False)
583 :param poll_interval: The polling interval
586 # sleep and wait for VM status change
587 logger.info('Checking if VM is active')
589 timeout = self.instance_settings.ssh_connect_timeout
591 if self.vm_active(block=True):
595 start = time.time() - timeout
597 while timeout > time.time() - start:
598 status = self.__ssh_active()
600 logger.info('SSH is active for VM instance')
603 logger.debug('Retry SSH connection in ' + str(
604 poll_interval) + ' seconds')
605 time.sleep(poll_interval)
606 logger.debug('SSH connection timeout in ' + str(
607 timeout - (time.time() - start)))
609 logger.error('Timeout attempting to connect with VM via SSH')
612 def __ssh_active(self):
614 Returns True when can create a SSH session else False
617 if len(self.__floating_ip_dict) > 0:
618 ssh = self.ssh_client()
624 def get_floating_ip(self, fip_name=None):
626 Returns the floating IP object byt name if found, else the first known,
628 :param fip_name: the name of the floating IP to return
629 :return: the SSH client or None
632 if fip_name and self.__floating_ip_dict.get(fip_name):
633 return self.__floating_ip_dict.get(fip_name)
635 return self.__get_first_provisioning_floating_ip()
637 def ssh_client(self, fip_name=None):
639 Returns an SSH client using the name or the first known floating IP if
641 :param fip_name: the name of the floating IP to return
642 :return: the SSH client or None
644 fip = self.get_floating_ip(fip_name)
646 return ansible_utils.ssh_client(
647 self.__get_first_provisioning_floating_ip().ip,
648 self.get_image_user(),
649 self.keypair_settings.private_filepath,
650 proxy_settings=self.__os_creds.proxy_settings)
653 'Cannot return an SSH client. No Floating IP configured')
655 def add_security_group(self, security_group):
657 Adds a security group to this VM. Call will block until VM is active.
658 :param security_group: the SNAPS SecurityGroup domain object
659 :return True if successful else False
661 self.vm_active(block=True)
663 if not security_group:
664 logger.warning('Security group object is None, cannot add')
668 nova_utils.add_security_group(self.__nova, self.get_vm_inst(),
671 except NotFound as e:
672 logger.warning('Security group not added - ' + str(e))
675 def remove_security_group(self, security_group):
677 Removes a security group to this VM. Call will block until VM is active
678 :param security_group: the OpenStack security group object
679 :return True if successful else False
681 self.vm_active(block=True)
683 if not security_group:
684 logger.warning('Security group object is None, cannot remove')
688 nova_utils.remove_security_group(self.__nova, self.get_vm_inst(),
691 except NotFound as e:
692 logger.warning('Security group not removed - ' + str(e))
696 class VmInstanceSettings:
698 Class responsible for holding configuration setting for a VM Instance
701 def __init__(self, **kwargs):
704 :param name: the name of the VM
705 :param flavor: the VM's flavor
706 :param port_settings: the port configuration settings (required)
707 :param security_group_names: a set of names of the security groups to
709 :param floating_ip_settings: the floating IP configuration settings
710 :param sudo_user: the sudo user of the VM that will override the
711 instance_settings.image_user when trying to
713 :param vm_boot_timeout: the amount of time a thread will sleep waiting
714 for an instance to boot
715 :param vm_delete_timeout: the amount of time a thread will sleep
716 waiting for an instance to be deleted
717 :param ssh_connect_timeout: the amount of time a thread will sleep
718 waiting obtaining an SSH connection to a VM
719 :param availability_zone: the name of the compute server on which to
720 deploy the VM (optional)
721 :param userdata: the cloud-init script to run after the VM has been
724 self.name = kwargs.get('name')
725 self.flavor = kwargs.get('flavor')
726 self.sudo_user = kwargs.get('sudo_user')
727 self.userdata = kwargs.get('userdata')
729 self.port_settings = list()
730 port_settings = kwargs.get('ports')
731 if not port_settings:
732 port_settings = kwargs.get('port_settings')
734 for port_setting in port_settings:
735 if isinstance(port_setting, dict):
736 self.port_settings.append(PortSettings(**port_setting))
737 elif isinstance(port_setting, PortSettings):
738 self.port_settings.append(port_setting)
740 if kwargs.get('security_group_names'):
741 if isinstance(kwargs['security_group_names'], list):
742 self.security_group_names = kwargs['security_group_names']
743 elif isinstance(kwargs['security_group_names'], set):
744 self.security_group_names = kwargs['security_group_names']
745 elif isinstance(kwargs['security_group_names'], str):
746 self.security_group_names = [kwargs['security_group_names']]
748 raise VmInstanceSettingsError(
749 'Invalid data type for security_group_names attribute')
751 self.security_group_names = set()
753 self.floating_ip_settings = list()
754 floating_ip_settings = kwargs.get('floating_ips')
755 if not floating_ip_settings:
756 floating_ip_settings = kwargs.get('floating_ip_settings')
757 if floating_ip_settings:
758 for floating_ip_config in floating_ip_settings:
759 if isinstance(floating_ip_config, FloatingIpSettings):
760 self.floating_ip_settings.append(floating_ip_config)
762 self.floating_ip_settings.append(FloatingIpSettings(
763 **floating_ip_config['floating_ip']))
765 if kwargs.get('vm_boot_timeout'):
766 self.vm_boot_timeout = kwargs['vm_boot_timeout']
768 self.vm_boot_timeout = 900
770 if kwargs.get('vm_delete_timeout'):
771 self.vm_delete_timeout = kwargs['vm_delete_timeout']
773 self.vm_delete_timeout = 300
775 if kwargs.get('ssh_connect_timeout'):
776 self.ssh_connect_timeout = kwargs['ssh_connect_timeout']
778 self.ssh_connect_timeout = 180
780 if kwargs.get('availability_zone'):
781 self.availability_zone = kwargs['availability_zone']
783 self.availability_zone = None
785 if not self.name or not self.flavor:
786 raise VmInstanceSettingsError(
787 'Instance configuration requires the attributes: name, flavor')
789 if len(self.port_settings) == 0:
790 raise VmInstanceSettingsError(
791 'Instance configuration requires port settings (aka. NICS)')
794 class FloatingIpSettings:
796 Class responsible for holding configuration settings for a floating IP
799 def __init__(self, **kwargs):
802 :param name: the name of the floating IP
803 :param port_name: the name of the router to the external network
804 :param router_name: the name of the router to the external network
805 :param subnet_name: the name of the subnet on which to attach the
807 :param provisioning: when true, this floating IP can be used for
810 TODO - provisioning flag is a hack as I have only observed a single
811 Floating IPs that actually works on an instance. Multiple floating IPs
812 placed on different subnets from the same port are especially
813 troublesome as you cannot predict which one will actually connect.
814 For now, it is recommended not to setup multiple floating IPs on an
815 instance unless absolutely necessary.
817 self.name = kwargs.get('name')
818 self.port_name = kwargs.get('port_name')
819 self.router_name = kwargs.get('router_name')
820 self.subnet_name = kwargs.get('subnet_name')
821 if kwargs.get('provisioning') is not None:
822 self.provisioning = kwargs['provisioning']
824 self.provisioning = True
826 if not self.name or not self.port_name or not self.router_name:
827 raise FloatingIpSettingsError(
828 'The attributes name, port_name and router_name are required '
829 'for FloatingIPSettings')
832 class VmInstanceSettingsError(Exception):
834 Exception to be thrown when an VM instance settings are incorrect
838 class FloatingIpSettingsError(Exception):
840 Exception to be thrown when an VM instance settings are incorrect
844 class VmInstanceCreationError(Exception):
846 Exception to be thrown when an VM instance cannot be created