1 # Copyright (c) 2017 Cable Television Laboratories, Inc. ("CableLabs")
2 # and others. All rights reserved.
4 # Licensed under the Apache License, Version 2.0 (the "License");
5 # you may not use this file except in compliance with the License.
6 # You may obtain a copy of the License at:
8 # http://www.apache.org/licenses/LICENSE-2.0
10 # Unless required by applicable law or agreed to in writing, software
11 # distributed under the License is distributed on an "AS IS" BASIS,
12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 # See the License for the specific language governing permissions and
14 # limitations under the License.
18 from neutronclient.common.exceptions import PortNotFoundClient
19 from novaclient.exceptions import NotFound
21 from snaps.openstack.create_network import PortSettings
22 from snaps.openstack.utils import glance_utils
23 from snaps.openstack.utils import neutron_utils
24 from snaps.openstack.utils import nova_utils
25 from snaps.provisioning import ansible_utils
27 __author__ = 'spisarski'
29 logger = logging.getLogger('create_instance')
32 STATUS_ACTIVE = 'ACTIVE'
33 STATUS_DELETED = 'DELETED'
36 class OpenStackVmInstance:
38 Class responsible for creating a VM instance in OpenStack
41 def __init__(self, os_creds, instance_settings, image_settings,
42 keypair_settings=None):
45 :param os_creds: The connection credentials to the OpenStack API
46 :param instance_settings: Contains the settings for this VM
47 :param image_settings: The OpenStack image object settings
48 :param keypair_settings: The keypair metadata (Optional)
51 self.__os_creds = os_creds
56 self.instance_settings = instance_settings
57 self.image_settings = image_settings
58 self.keypair_settings = keypair_settings
60 self.__floating_ip_dict = dict()
62 # Instantiated in self.create()
65 # Note: this object does not change after the VM becomes active
68 def create(self, cleanup=False, block=False):
71 :param cleanup: When true, this object is initialized only via queries,
72 else objects will be created when the queries return
73 None. The name of this parameter should be changed to
74 something like 'readonly' as the same goes with all of
75 the other creator classes.
76 :param block: Thread will block until instance has either become
77 active, error, or timeout waiting.
78 Additionally, when True, floating IPs will not be applied
80 :return: The VM reference object
82 self.__nova = nova_utils.nova_client(self.__os_creds)
83 self.__neutron = neutron_utils.neutron_client(self.__os_creds)
85 self.__ports = self.__setup_ports(self.instance_settings.port_settings,
87 self.__lookup_existing_vm_by_name()
88 if not self.__vm and not cleanup:
89 self.__create_vm(block)
92 def __lookup_existing_vm_by_name(self):
94 Populates the member variables 'self.vm' and 'self.floating_ips' if a
95 VM with the same name already exists
98 server = nova_utils.get_server(
99 self.__nova, vm_inst_settings=self.instance_settings)
101 if server.name == self.instance_settings.name:
104 'Found existing machine with name - %s',
105 self.instance_settings.name)
107 fips = neutron_utils.get_floating_ips(self.__neutron,
109 for port_id, fip in fips:
110 settings = self.instance_settings.floating_ip_settings
111 for fip_setting in settings:
112 if port_id == fip_setting.port_id:
113 self.__floating_ip_dict[fip_setting.name] = fip
115 port = neutron_utils.get_port_by_id(
116 self.__neutron, port_id)
117 if port and port.name == fip_setting.port_name:
118 self.__floating_ip_dict[fip_setting.name] = fip
120 def __create_vm(self, block=False):
122 Responsible for creating the VM instance
123 :param block: Thread will block until instance has either become
124 active, error, or timeout waiting. Floating IPs will be
125 assigned after active when block=True
127 glance = glance_utils.glance_client(self.__os_creds)
128 self.__vm = nova_utils.create_server(
129 self.__nova, self.__neutron, glance, self.instance_settings,
130 self.image_settings, self.keypair_settings)
131 logger.info('Created instance with name - %s',
132 self.instance_settings.name)
135 if not self.vm_active(block=True):
136 raise VmInstanceCreationError(
137 'Fatal error, VM did not become ACTIVE within the alloted '
140 # Create server should do this but found it needed to occur here
141 for sec_grp_name in self.instance_settings.security_group_names:
142 if self.vm_active(block=True):
143 nova_utils.add_security_group(self.__nova, self.__vm,
146 raise VmInstanceCreationError(
147 'Cannot applying security group with name ' +
149 ' to VM that did not activate with name - ' +
150 self.instance_settings.name)
152 self.__apply_floating_ips()
154 def __apply_floating_ips(self):
156 Applies the configured floating IPs to the necessary ports
159 for key, port in self.__ports:
160 port_dict[key] = port
163 for floating_ip_setting in self.instance_settings.floating_ip_settings:
164 port = port_dict.get(floating_ip_setting.port_name)
167 raise VmInstanceCreationError(
168 'Cannot find port object with name - ' +
169 floating_ip_setting.port_name)
171 # Setup Floating IP only if there is a router with an external
173 ext_gateway = self.__ext_gateway_by_router(
174 floating_ip_setting.router_name)
176 subnet = neutron_utils.get_subnet(
178 subnet_name=floating_ip_setting.subnet_name)
179 floating_ip = neutron_utils.create_floating_ip(
180 self.__neutron, ext_gateway)
181 self.__floating_ip_dict[floating_ip_setting.name] = floating_ip
184 'Created floating IP %s via router - %s', floating_ip.ip,
185 floating_ip_setting.router_name)
186 self.__add_floating_ip(floating_ip, port, subnet)
188 raise VmInstanceCreationError(
189 'Unable to add floating IP to port, cannot locate router '
190 'with an external gateway ')
192 def __ext_gateway_by_router(self, router_name):
194 Returns network name for the external network attached to a router or
196 :param router_name: The name of the router to lookup
197 :return: the external network name or None
199 router = neutron_utils.get_router(
200 self.__neutron, router_name=router_name)
201 if router and router.external_gateway_info:
202 network = neutron_utils.get_network_by_id(
204 router.external_gateway_info['network_id'])
211 Destroys the VM instance
214 # Cleanup floating IPs
215 for name, floating_ip in self.__floating_ip_dict.items():
217 logger.info('Deleting Floating IP - ' + floating_ip.ip)
218 neutron_utils.delete_floating_ip(self.__neutron, floating_ip)
219 except Exception as e:
220 logger.error('Error deleting Floating IP - ' + str(e))
221 self.__floating_ip_dict = dict()
224 for name, port in self.__ports:
225 logger.info('Deleting Port with ID - %S ' + port.id)
227 neutron_utils.delete_port(self.__neutron, port)
228 except PortNotFoundClient as e:
229 logger.warning('Unexpected error deleting port - %s', e)
231 self.__ports = list()
237 'Deleting VM instance - ' + self.instance_settings.name)
238 nova_utils.delete_vm_instance(self.__nova, self.__vm)
239 except Exception as e:
240 logger.error('Error deleting VM - %s', e)
242 # Block until instance cannot be found or returns the status of
244 logger.info('Checking deletion status')
247 if self.vm_deleted(block=True):
249 'VM has been properly deleted VM with name - %s',
250 self.instance_settings.name)
254 'VM not deleted within the timeout period of %s '
255 'seconds', self.instance_settings.vm_delete_timeout)
256 except Exception as e:
258 'Unexpected error while checking VM instance status - %s',
261 def __setup_ports(self, port_settings, cleanup):
263 Returns the previously configured ports or creates them if they do not
265 :param port_settings: A list of PortSetting objects
266 :param cleanup: When true, only perform lookups for OpenStack objects.
267 :return: a list of OpenStack port tuples where the first member is the
268 port name and the second is the port object
272 for port_setting in port_settings:
273 port = neutron_utils.get_port(
274 self.__neutron, port_settings=port_setting)
276 network = neutron_utils.get_network(
277 self.__neutron, network_name=port_setting.network_name)
278 net_ports = neutron_utils.get_ports(self.__neutron, network)
279 for net_port in net_ports:
280 if port_setting.mac_address == net_port.mac_address:
284 ports.append((port_setting.name, port))
286 # Exception will be raised when port with same name already
289 (port_setting.name, neutron_utils.create_port(
290 self.__neutron, self.__os_creds, port_setting)))
294 def __add_floating_ip(self, floating_ip, port, subnet, timeout=30,
295 poll_interval=POLL_INTERVAL):
297 Returns True when active else False
298 TODO - Make timeout and poll_interval configurable...
303 # Take IP of subnet if there is one configured on which to place
305 for fixed_ip in port.ips:
306 if fixed_ip['subnet_id'] == subnet.id:
307 ip = fixed_ip['ip_address']
310 # Simply take the first
311 ip = port.ips[0]['ip_address']
314 count = timeout / poll_interval
316 logger.debug('Attempting to add floating IP to instance')
318 nova_utils.add_floating_ip_to_server(
319 self.__nova, self.__vm, floating_ip, ip)
321 'Added floating IP %s to port IP %s on instance %s',
322 floating_ip.ip, ip, self.instance_settings.name)
324 except Exception as e:
326 'Retry adding floating IP to instance. Last attempt '
327 'failed with - %s', e)
328 time.sleep(poll_interval)
332 raise VmInstanceCreationError(
333 'Unable find IP address on which to place the floating IP')
335 logger.error('Timeout attempting to add the floating IP to instance.')
336 raise VmInstanceCreationError(
337 'Timeout while attempting add floating IP to instance')
339 def get_os_creds(self):
341 Returns the OpenStack credentials used to create these objects
342 :return: the credentials
344 return self.__os_creds
346 def get_vm_inst(self):
348 Returns the latest version of this server object from OpenStack
349 :return: Server object
351 return nova_utils.get_server_object_by_id(self.__nova, self.__vm.id)
353 def get_console_output(self):
355 Returns the vm console object for parsing logs
356 :return: the console output object
358 return nova_utils.get_server_console_output(self.__nova, self.__vm)
360 def get_port_ip(self, port_name, subnet_name=None):
362 Returns the first IP for the port corresponding with the port_name
363 parameter when subnet_name is None else returns the IP address that
364 corresponds to the subnet_name parameter
365 :param port_name: the name of the port from which to return the IP
366 :param subnet_name: the name of the subnet attached to this IP
367 :return: the IP or None if not found
369 port = self.get_port_by_name(port_name)
372 subnet = neutron_utils.get_subnet(
373 self.__neutron, subnet_name=subnet_name)
375 logger.warning('Cannot retrieve port IP as subnet could '
376 'not be located with name - %s',
379 for fixed_ip in port.ips:
380 if fixed_ip['subnet_id'] == subnet.id:
381 return fixed_ip['ip_address']
383 if port.ips and len(port.ips) > 0:
384 return port.ips[0]['ip_address']
387 def get_port_mac(self, port_name):
389 Returns the first IP for the port corresponding with the port_name
391 TODO - Add in the subnet as an additional parameter as a port may have
393 :param port_name: the name of the port from which to return the IP
394 :return: the IP or None if not found
396 port = self.get_port_by_name(port_name)
398 return port.mac_address
401 def get_port_by_name(self, port_name):
403 Retrieves the OpenStack port object by its given name
404 :param port_name: the name of the port
405 :return: the OpenStack port object or None if not exists
407 for key, port in self.__ports:
410 logger.warning('Cannot find port with name - ' + port_name)
413 def get_vm_info(self):
415 Returns a dictionary of a VMs info as returned by OpenStack
418 return nova_utils.get_server_info(self.__nova, self.__vm)
420 def config_nics(self):
422 Responsible for configuring NICs on RPM systems where the instance has
423 more than one configured port
424 :return: the value returned by ansible_utils.apply_ansible_playbook()
426 if len(self.__ports) > 1 and len(self.__floating_ip_dict) > 0:
427 if self.vm_active(block=True) and self.vm_ssh_active(block=True):
428 for key, port in self.__ports:
429 port_index = self.__ports.index((key, port))
431 nic_name = 'eth' + repr(port_index)
432 retval = self.__config_nic(
434 self.__get_first_provisioning_floating_ip().ip)
435 logger.info('Configured NIC - %s on VM - %s',
436 nic_name, self.instance_settings.name)
439 def __get_first_provisioning_floating_ip(self):
441 Returns the first floating IP tagged with the Floating IP name if
442 exists else the first one found
445 for floating_ip_setting in self.instance_settings.floating_ip_settings:
446 if floating_ip_setting.provisioning:
447 fip = self.__floating_ip_dict.get(floating_ip_setting.name)
450 elif len(self.__floating_ip_dict) > 0:
451 for key, fip in self.__floating_ip_dict.items():
454 def __config_nic(self, nic_name, port, ip):
456 Although ports/NICs can contain multiple IPs, this code currently only
459 :param nic_name: Name of the interface
460 :param port: The port information containing the expected IP values.
461 :param ip: The IP on which to apply the playbook.
462 :return: the return value from ansible
464 port_ip = port.ips[0]['ip_address']
467 'nic_name': nic_name,
471 if self.image_settings.nic_config_pb_loc and self.keypair_settings:
472 return self.apply_ansible_playbook(
473 self.image_settings.nic_config_pb_loc, variables)
476 'VM %s cannot self configure NICs eth1++. No playbook or '
477 'keypairs found.', self.instance_settings.name)
479 def apply_ansible_playbook(self, pb_file_loc, variables=None,
482 Applies a playbook to a VM
483 :param pb_file_loc: the file location of the playbook to be applied
484 :param variables: a dict() of substitution values required by the
486 :param fip_name: the name of the floating IP to use for applying the
487 playbook (default - will take the first)
488 :return: the return value from ansible
490 return ansible_utils.apply_playbook(
491 pb_file_loc, [self.get_floating_ip(fip_name=fip_name).ip],
492 self.get_image_user(), self.keypair_settings.private_filepath,
493 variables, self.__os_creds.proxy_settings)
495 def get_image_user(self):
497 Returns the instance sudo_user if it has been configured in the
498 instance_settings else it returns the image_settings.image_user value
500 if self.instance_settings.sudo_user:
501 return self.instance_settings.sudo_user
503 return self.image_settings.image_user
505 def vm_deleted(self, block=False, poll_interval=POLL_INTERVAL):
507 Returns true when the VM status returns the value of
508 expected_status_code or instance retrieval throws a NotFound exception.
509 :param block: When true, thread will block until active or timeout
510 value in seconds has been exceeded (False)
511 :param poll_interval: The polling interval in seconds
515 return self.__vm_status_check(
516 STATUS_DELETED, block,
517 self.instance_settings.vm_delete_timeout, poll_interval)
518 except NotFound as e:
520 "Instance not found when querying status for %s with message "
521 "%s", STATUS_DELETED, e)
524 def vm_active(self, block=False, poll_interval=POLL_INTERVAL):
526 Returns true when the VM status returns the value of the constant
528 :param block: When true, thread will block until active or timeout
529 value in seconds has been exceeded (False)
530 :param poll_interval: The polling interval in seconds
533 return self.__vm_status_check(STATUS_ACTIVE, block,
534 self.instance_settings.vm_boot_timeout,
537 def __vm_status_check(self, expected_status_code, block, timeout,
540 Returns true when the VM status returns the value of
542 :param expected_status_code: instance status evaluated with this
544 :param block: When true, thread will block until active or timeout
545 value in seconds has been exceeded (False)
546 :param timeout: The timeout value
547 :param poll_interval: The polling interval in seconds
550 # sleep and wait for VM status change
554 return self.__status(expected_status_code)
556 while timeout > time.time() - start:
557 status = self.__status(expected_status_code)
559 logger.info('VM is - ' + expected_status_code)
562 logger.debug('Retry querying VM status in ' + str(
563 poll_interval) + ' seconds')
564 time.sleep(poll_interval)
565 logger.debug('VM status query timeout in ' + str(
566 timeout - (time.time() - start)))
569 'Timeout checking for VM status for ' + expected_status_code)
572 def __status(self, expected_status_code):
574 Returns True when active else False
575 :param expected_status_code: instance status evaluated with this string
580 if expected_status_code == STATUS_DELETED:
585 status = nova_utils.get_server_status(self.__nova, self.__vm)
587 logger.warning('Cannot find instance with id - ' + self.__vm.id)
590 if status == 'ERROR':
591 raise VmInstanceCreationError(
592 'Instance had an error during deployment')
594 'Instance status [%s] is - %s', self.instance_settings.name,
596 return status == expected_status_code
598 def vm_ssh_active(self, block=False, poll_interval=POLL_INTERVAL):
600 Returns true when the VM can be accessed via SSH
601 :param block: When true, thread will block until active or timeout
602 value in seconds has been exceeded (False)
603 :param poll_interval: The polling interval
606 # sleep and wait for VM status change
607 logger.info('Checking if VM is active')
609 timeout = self.instance_settings.ssh_connect_timeout
611 if self.vm_active(block=True):
615 start = time.time() - timeout
617 while timeout > time.time() - start:
618 status = self.__ssh_active()
620 logger.info('SSH is active for VM instance')
623 logger.debug('Retry SSH connection in ' + str(
624 poll_interval) + ' seconds')
625 time.sleep(poll_interval)
626 logger.debug('SSH connection timeout in ' + str(
627 timeout - (time.time() - start)))
629 logger.error('Timeout attempting to connect with VM via SSH')
632 def __ssh_active(self):
634 Returns True when can create a SSH session else False
637 if len(self.__floating_ip_dict) > 0:
638 ssh = self.ssh_client()
644 def get_floating_ip(self, fip_name=None):
646 Returns the floating IP object byt name if found, else the first known,
648 :param fip_name: the name of the floating IP to return
649 :return: the SSH client or None
652 if fip_name and self.__floating_ip_dict.get(fip_name):
653 return self.__floating_ip_dict.get(fip_name)
655 return self.__get_first_provisioning_floating_ip()
657 def ssh_client(self, fip_name=None):
659 Returns an SSH client using the name or the first known floating IP if
661 :param fip_name: the name of the floating IP to return
662 :return: the SSH client or None
664 fip = self.get_floating_ip(fip_name)
666 return ansible_utils.ssh_client(
667 self.__get_first_provisioning_floating_ip().ip,
668 self.get_image_user(),
669 self.keypair_settings.private_filepath,
670 proxy_settings=self.__os_creds.proxy_settings)
673 'Cannot return an SSH client. No Floating IP configured')
675 def add_security_group(self, security_group):
677 Adds a security group to this VM. Call will block until VM is active.
678 :param security_group: the SNAPS SecurityGroup domain object
679 :return True if successful else False
681 self.vm_active(block=True)
683 if not security_group:
684 logger.warning('Security group object is None, cannot add')
688 nova_utils.add_security_group(self.__nova, self.get_vm_inst(),
691 except NotFound as e:
692 logger.warning('Security group not added - ' + str(e))
695 def remove_security_group(self, security_group):
697 Removes a security group to this VM. Call will block until VM is active
698 :param security_group: the OpenStack security group object
699 :return True if successful else False
701 self.vm_active(block=True)
703 if not security_group:
704 logger.warning('Security group object is None, cannot remove')
708 nova_utils.remove_security_group(self.__nova, self.get_vm_inst(),
711 except NotFound as e:
712 logger.warning('Security group not removed - ' + str(e))
716 class VmInstanceSettings:
718 Class responsible for holding configuration setting for a VM Instance
721 def __init__(self, **kwargs):
724 :param name: the name of the VM
725 :param flavor: the VM's flavor name
726 :param port_settings: the port configuration settings (required)
727 :param security_group_names: a set of names of the security groups to
729 :param floating_ip_settings: the floating IP configuration settings
730 :param sudo_user: the sudo user of the VM that will override the
731 instance_settings.image_user when trying to
733 :param vm_boot_timeout: the amount of time a thread will sleep waiting
734 for an instance to boot
735 :param vm_delete_timeout: the amount of time a thread will sleep
736 waiting for an instance to be deleted
737 :param ssh_connect_timeout: the amount of time a thread will sleep
738 waiting obtaining an SSH connection to a VM
739 :param availability_zone: the name of the compute server on which to
740 deploy the VM (optional)
741 :param userdata: the cloud-init script to run after the VM has been
744 self.name = kwargs.get('name')
745 self.flavor = kwargs.get('flavor')
746 self.sudo_user = kwargs.get('sudo_user')
747 self.userdata = kwargs.get('userdata')
749 self.port_settings = list()
750 port_settings = kwargs.get('ports')
751 if not port_settings:
752 port_settings = kwargs.get('port_settings')
754 for port_setting in port_settings:
755 if isinstance(port_setting, dict):
756 self.port_settings.append(PortSettings(**port_setting))
757 elif isinstance(port_setting, PortSettings):
758 self.port_settings.append(port_setting)
760 if kwargs.get('security_group_names'):
761 if isinstance(kwargs['security_group_names'], list):
762 self.security_group_names = kwargs['security_group_names']
763 elif isinstance(kwargs['security_group_names'], set):
764 self.security_group_names = kwargs['security_group_names']
765 elif isinstance(kwargs['security_group_names'], str):
766 self.security_group_names = [kwargs['security_group_names']]
768 raise VmInstanceSettingsError(
769 'Invalid data type for security_group_names attribute')
771 self.security_group_names = set()
773 self.floating_ip_settings = list()
774 floating_ip_settings = kwargs.get('floating_ips')
775 if not floating_ip_settings:
776 floating_ip_settings = kwargs.get('floating_ip_settings')
777 if floating_ip_settings:
778 for floating_ip_config in floating_ip_settings:
779 if isinstance(floating_ip_config, FloatingIpSettings):
780 self.floating_ip_settings.append(floating_ip_config)
782 self.floating_ip_settings.append(FloatingIpSettings(
783 **floating_ip_config['floating_ip']))
785 if kwargs.get('vm_boot_timeout'):
786 self.vm_boot_timeout = kwargs['vm_boot_timeout']
788 self.vm_boot_timeout = 900
790 if kwargs.get('vm_delete_timeout'):
791 self.vm_delete_timeout = kwargs['vm_delete_timeout']
793 self.vm_delete_timeout = 300
795 if kwargs.get('ssh_connect_timeout'):
796 self.ssh_connect_timeout = kwargs['ssh_connect_timeout']
798 self.ssh_connect_timeout = 180
800 if kwargs.get('availability_zone'):
801 self.availability_zone = kwargs['availability_zone']
803 self.availability_zone = None
805 if not self.name or not self.flavor:
806 raise VmInstanceSettingsError(
807 'Instance configuration requires the attributes: name, flavor')
809 if len(self.port_settings) == 0:
810 raise VmInstanceSettingsError(
811 'Instance configuration requires port settings (aka. NICS)')
814 class FloatingIpSettings:
816 Class responsible for holding configuration settings for a floating IP
819 def __init__(self, **kwargs):
822 :param name: the name of the floating IP
823 :param port_name: the name of the router to the external network
824 :param router_name: the name of the router to the external network
825 :param subnet_name: the name of the subnet on which to attach the
827 :param provisioning: when true, this floating IP can be used for
830 TODO - provisioning flag is a hack as I have only observed a single
831 Floating IPs that actually works on an instance. Multiple floating IPs
832 placed on different subnets from the same port are especially
833 troublesome as you cannot predict which one will actually connect.
834 For now, it is recommended not to setup multiple floating IPs on an
835 instance unless absolutely necessary.
837 self.name = kwargs.get('name')
838 self.port_name = kwargs.get('port_name')
839 self.port_id = kwargs.get('port_id')
840 self.router_name = kwargs.get('router_name')
841 self.subnet_name = kwargs.get('subnet_name')
842 if kwargs.get('provisioning') is not None:
843 self.provisioning = kwargs['provisioning']
845 self.provisioning = True
847 # if not self.name or not self.port_name or not self.router_name:
848 if not self.name or not self.router_name:
849 raise FloatingIpSettingsError(
850 'The attributes name, port_name and router_name are required')
852 if not self.port_name and not self.port_id:
853 raise FloatingIpSettingsError(
854 'The attributes port_name or port_id are required')
857 class VmInstanceSettingsError(Exception):
859 Exception to be thrown when an VM instance settings are incorrect
863 class FloatingIpSettingsError(Exception):
865 Exception to be thrown when an VM instance settings are incorrect
869 class VmInstanceCreationError(Exception):
871 Exception to be thrown when an VM instance cannot be created