1 # Copyright (c) 2017 Cable Television Laboratories, Inc. ("CableLabs")
2 # and others. All rights reserved.
4 # Licensed under the Apache License, Version 2.0 (the "License");
5 # you may not use this file except in compliance with the License.
6 # You may obtain a copy of the License at:
8 # http://www.apache.org/licenses/LICENSE-2.0
10 # Unless required by applicable law or agreed to in writing, software
11 # distributed under the License is distributed on an "AS IS" BASIS,
12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 # See the License for the specific language governing permissions and
14 # limitations under the License.
18 from neutronclient.common.exceptions import PortNotFoundClient
19 from novaclient.exceptions import NotFound
21 from snaps.openstack.create_network import PortSettings
22 from snaps.openstack.utils import glance_utils
23 from snaps.openstack.utils import neutron_utils
24 from snaps.openstack.utils import nova_utils
25 from snaps.provisioning import ansible_utils
27 __author__ = 'spisarski'
29 logger = logging.getLogger('create_instance')
32 STATUS_ACTIVE = 'ACTIVE'
33 STATUS_DELETED = 'DELETED'
36 class OpenStackVmInstance:
38 Class responsible for creating a VM instance in OpenStack
41 def __init__(self, os_creds, instance_settings, image_settings,
42 keypair_settings=None):
45 :param os_creds: The connection credentials to the OpenStack API
46 :param instance_settings: Contains the settings for this VM
47 :param image_settings: The OpenStack image object settings
48 :param keypair_settings: The keypair metadata (Optional)
51 self.__os_creds = os_creds
56 self.instance_settings = instance_settings
57 self.image_settings = image_settings
58 self.keypair_settings = keypair_settings
60 # TODO - get rid of FIP list and only use the dict(). Need to fix
61 # populating this object when already exists
62 self.__floating_ips = list()
63 self.__floating_ip_dict = dict()
65 # Instantiated in self.create()
68 # Note: this object does not change after the VM becomes active
71 def create(self, cleanup=False, block=False):
74 :param cleanup: When true, only perform lookups for OpenStack objects.
75 :param block: Thread will block until instance has either become
76 active, error, or timeout waiting.
77 Additionally, when True, floating IPs will not be applied
79 :return: The VM reference object
81 self.__nova = nova_utils.nova_client(self.__os_creds)
82 self.__neutron = neutron_utils.neutron_client(self.__os_creds)
84 self.__ports = self.__setup_ports(self.instance_settings.port_settings,
86 self.__lookup_existing_vm_by_name()
87 if not self.__vm and not cleanup:
88 self.__create_vm(block)
91 def __lookup_existing_vm_by_name(self):
93 Populates the member variables 'self.vm' and 'self.floating_ips' if a
94 VM with the same name already exists
97 servers = nova_utils.get_servers_by_name(self.__nova,
98 self.instance_settings.name)
99 for server in servers:
100 if server.name == self.instance_settings.name:
103 'Found existing machine with name - %s',
104 self.instance_settings.name)
105 fips = self.__nova.floating_ips.list()
107 if fip.instance_id == server.id:
108 self.__floating_ips.append(fip)
109 # TODO - Determine a means to associate to the FIP
110 # configuration and add to FIP map
112 def __create_vm(self, block=False):
114 Responsible for creating the VM instance
115 :param block: Thread will block until instance has either become
116 active, error, or timeout waiting. Floating IPs will be
117 assigned after active when block=True
120 for key, port in self.__ports:
122 kv['port-id'] = port['port']['id']
125 logger.info('Creating VM with name - ' + self.instance_settings.name)
127 if self.keypair_settings:
128 keypair_name = self.keypair_settings.name
130 flavor = nova_utils.get_flavor_by_name(self.__nova,
131 self.instance_settings.flavor)
134 'Flavor not found with name - %s',
135 self.instance_settings.flavor)
137 image = glance_utils.get_image(
138 glance_utils.glance_client(self.__os_creds),
139 self.image_settings.name)
141 self.__vm = self.__nova.servers.create(
142 name=self.instance_settings.name,
146 key_name=keypair_name,
147 security_groups=self.instance_settings.security_group_names,
148 userdata=self.instance_settings.userdata,
149 availability_zone=self.instance_settings.availability_zone)
153 'Cannot create instance, image cannot be located with name %s',
154 self.image_settings.name)
157 'Created instance with name - %s', self.instance_settings.name)
160 if not self.vm_active(block=True):
162 'Fatal error, VM did not become ACTIVE within the alloted '
165 # TODO - the call above should add security groups. The return object
166 # shows they exist but the association had never been made by
167 # OpenStack. This call is here to ensure they have been added
168 for sec_grp_name in self.instance_settings.security_group_names:
169 if self.vm_active(block=True):
170 nova_utils.add_security_group(self.__nova, self.__vm,
174 'Cannot applying security group with name ' +
176 ' to VM that did not activate with name - ' +
177 self.instance_settings.name)
179 self.__apply_floating_ips()
181 def __apply_floating_ips(self):
183 Applies the configured floating IPs to the necessary ports
186 for key, port in self.__ports:
187 port_dict[key] = port
190 for floating_ip_setting in self.instance_settings.floating_ip_settings:
191 port = port_dict.get(floating_ip_setting.port_name)
195 'Cannot find port object with name - ' +
196 floating_ip_setting.port_name)
198 # Setup Floating IP only if there is a router with an external
200 ext_gateway = self.__ext_gateway_by_router(
201 floating_ip_setting.router_name)
203 subnet = neutron_utils.get_subnet_by_name(
204 self.__neutron, floating_ip_setting.subnet_name)
205 floating_ip = nova_utils.create_floating_ip(self.__nova,
207 self.__floating_ips.append(floating_ip)
208 self.__floating_ip_dict[floating_ip_setting.name] = floating_ip
211 'Created floating IP %s via router - %s', floating_ip.ip,
212 floating_ip_setting.router_name)
213 self.__add_floating_ip(floating_ip, port, subnet)
215 raise Exception('Unable to add floating IP to port,'
216 ' cannot locate router with an external '
219 def __ext_gateway_by_router(self, router_name):
221 Returns network name for the external network attached to a router or
223 :param router_name: The name of the router to lookup
224 :return: the external network name or None
226 router = neutron_utils.get_router_by_name(self.__neutron, router_name)
227 if router and router['router'].get('external_gateway_info'):
228 network = neutron_utils.get_network_by_id(
230 router['router']['external_gateway_info']['network_id'])
232 return network['network']['name']
237 Destroys the VM instance
240 # Cleanup floating IPs
241 for floating_ip in self.__floating_ips:
243 logger.info('Deleting Floating IP - ' + floating_ip.ip)
244 nova_utils.delete_floating_ip(self.__nova, floating_ip)
245 except Exception as e:
246 logger.error('Error deleting Floating IP - ' + str(e))
247 self.__floating_ips = list()
248 self.__floating_ip_dict = dict()
251 for name, port in self.__ports:
252 logger.info('Deleting Port - ' + name)
254 neutron_utils.delete_port(self.__neutron, port)
255 except PortNotFoundClient as e:
256 logger.warning('Unexpected error deleting port - %s', e)
258 self.__ports = list()
264 'Deleting VM instance - ' + self.instance_settings.name)
265 nova_utils.delete_vm_instance(self.__nova, self.__vm)
266 except Exception as e:
267 logger.error('Error deleting VM - %s', e)
269 # Block until instance cannot be found or returns the status of
271 logger.info('Checking deletion status')
274 if self.vm_deleted(block=True):
276 'VM has been properly deleted VM with name - %s',
277 self.instance_settings.name)
281 'VM not deleted within the timeout period of %s '
282 'seconds', self.instance_settings.vm_delete_timeout)
283 except Exception as e:
285 'Unexpected error while checking VM instance status - %s',
288 def __setup_ports(self, port_settings, cleanup):
290 Returns the previously configured ports or creates them if they do not
292 :param port_settings: A list of PortSetting objects
293 :param cleanup: When true, only perform lookups for OpenStack objects.
294 :return: a list of OpenStack port tuples where the first member is the
295 port name and the second is the port object
299 for port_setting in port_settings:
300 # First check to see if network already has this port
301 # TODO/FIXME - this could potentially cause problems if another
302 # port with the same name exists
303 # VM has the same network/port name pair
306 # TODO/FIXME - should we not be iterating on ports for the specific
307 # network in question as unique port names
308 # seem to only be important by network
309 existing_ports = self.__neutron.list_ports()['ports']
310 for existing_port in existing_ports:
311 if existing_port['name'] == port_setting.name:
312 ports.append((port_setting.name, {'port': existing_port}))
316 if not found and not cleanup:
317 ports.append((port_setting.name,
318 neutron_utils.create_port(self.__neutron,
324 def __add_floating_ip(self, floating_ip, port, subnet, timeout=30,
325 poll_interval=POLL_INTERVAL):
327 Returns True when active else False
328 TODO - Make timeout and poll_interval configurable...
333 # Take IP of subnet if there is one configured on which to place
335 for fixed_ip in port['port']['fixed_ips']:
336 if fixed_ip['subnet_id'] == subnet['subnet']['id']:
337 ip = fixed_ip['ip_address']
340 # Simply take the first
341 ip = port['port']['fixed_ips'][0]['ip_address']
344 count = timeout / poll_interval
346 logger.debug('Attempting to add floating IP to instance')
348 self.__vm.add_floating_ip(floating_ip, ip)
350 'Added floating IP %s to port IP %s on instance %s',
351 floating_ip.ip, ip, self.instance_settings.name)
353 except Exception as e:
355 'Retry adding floating IP to instance. Last attempt '
356 'failed with - %s', e)
357 time.sleep(poll_interval)
362 'Unable find IP address on which to place the floating IP')
364 logger.error('Timeout attempting to add the floating IP to instance.')
365 raise Exception('Timeout while attempting add floating IP to instance')
367 def get_os_creds(self):
369 Returns the OpenStack credentials used to create these objects
370 :return: the credentials
372 return self.__os_creds
374 def get_vm_inst(self):
376 Returns the latest version of this server object from OpenStack
377 :return: Server object
379 return nova_utils.get_latest_server_object(self.__nova, self.__vm)
381 def get_port_ip(self, port_name, subnet_name=None):
383 Returns the first IP for the port corresponding with the port_name
384 parameter when subnet_name is None else returns the IP address that
385 corresponds to the subnet_name parameter
386 :param port_name: the name of the port from which to return the IP
387 :param subnet_name: the name of the subnet attached to this IP
388 :return: the IP or None if not found
390 port = self.get_port_by_name(port_name)
392 port_dict = port['port']
394 subnet = neutron_utils.get_subnet_by_name(self.__neutron,
397 logger.warning('Cannot retrieve port IP as subnet could '
398 'not be located with name - %s',
401 for fixed_ip in port_dict['fixed_ips']:
402 if fixed_ip['subnet_id'] == subnet['subnet']['id']:
403 return fixed_ip['ip_address']
405 fixed_ips = port_dict['fixed_ips']
406 if fixed_ips and len(fixed_ips) > 0:
407 return fixed_ips[0]['ip_address']
410 def get_port_mac(self, port_name):
412 Returns the first IP for the port corresponding with the port_name
414 TODO - Add in the subnet as an additional parameter as a port may have
416 :param port_name: the name of the port from which to return the IP
417 :return: the IP or None if not found
419 port = self.get_port_by_name(port_name)
421 port_dict = port['port']
422 return port_dict['mac_address']
425 def get_port_by_name(self, port_name):
427 Retrieves the OpenStack port object by its given name
428 :param port_name: the name of the port
429 :return: the OpenStack port object or None if not exists
431 for key, port in self.__ports:
434 logger.warning('Cannot find port with name - ' + port_name)
437 def config_nics(self):
439 Responsible for configuring NICs on RPM systems where the instance has
440 more than one configured port
443 if len(self.__ports) > 1 and len(self.__floating_ips) > 0:
444 if self.vm_active(block=True) and self.vm_ssh_active(block=True):
445 for key, port in self.__ports:
446 port_index = self.__ports.index((key, port))
448 nic_name = 'eth' + repr(port_index)
451 self.__get_first_provisioning_floating_ip().ip)
452 logger.info('Configured NIC - %s on VM - %s',
453 nic_name, self.instance_settings.name)
455 def __get_first_provisioning_floating_ip(self):
457 Returns the first floating IP tagged with the Floating IP name if
458 exists else the first one found
461 for floating_ip_setting in self.instance_settings.floating_ip_settings:
462 if floating_ip_setting.provisioning:
463 fip = self.__floating_ip_dict.get(floating_ip_setting.name)
466 elif len(self.__floating_ips) > 0:
467 return self.__floating_ips[0]
469 def __config_nic(self, nic_name, port, floating_ip):
471 Although ports/NICs can contain multiple IPs, this code currently only
474 :param nic_name: Name of the interface
475 :param port: The port information containing the expected IP values.
476 :param floating_ip: The floating IP on which to apply the playbook.
477 :return: the return value from ansible
479 ip = port['port']['fixed_ips'][0]['ip_address']
481 'floating_ip': floating_ip,
482 'nic_name': nic_name,
486 if self.image_settings.nic_config_pb_loc and self.keypair_settings:
487 return self.apply_ansible_playbook(
488 self.image_settings.nic_config_pb_loc, variables)
491 'VM %s cannot self configure NICs eth1++. No playbook or '
492 'keypairs found.', self.instance_settings.name)
494 def apply_ansible_playbook(self, pb_file_loc, variables=None,
497 Applies a playbook to a VM
498 :param pb_file_loc: the file location of the playbook to be applied
499 :param variables: a dict() of substitution values required by the
501 :param fip_name: the name of the floating IP to use for applying the
502 playbook (default - will take the first)
503 :return: the return value from ansible
505 return ansible_utils.apply_playbook(
506 pb_file_loc, [self.get_floating_ip(fip_name=fip_name).ip],
507 self.get_image_user(), self.keypair_settings.private_filepath,
508 variables, self.__os_creds.proxy_settings)
510 def get_image_user(self):
512 Returns the instance sudo_user if it has been configured in the
513 instance_settings else it returns the image_settings.image_user value
515 if self.instance_settings.sudo_user:
516 return self.instance_settings.sudo_user
518 return self.image_settings.image_user
520 def vm_deleted(self, block=False, poll_interval=POLL_INTERVAL):
522 Returns true when the VM status returns the value of
523 expected_status_code or instance retrieval throws a NotFound exception.
524 :param block: When true, thread will block until active or timeout
525 value in seconds has been exceeded (False)
526 :param poll_interval: The polling interval in seconds
530 return self.__vm_status_check(
531 STATUS_DELETED, block,
532 self.instance_settings.vm_delete_timeout, poll_interval)
533 except NotFound as e:
535 "Instance not found when querying status for %s with message "
536 "%s", STATUS_DELETED, e)
539 def vm_active(self, block=False, poll_interval=POLL_INTERVAL):
541 Returns true when the VM status returns the value of
543 :param block: When true, thread will block until active or timeout
544 value in seconds has been exceeded (False)
545 :param poll_interval: The polling interval in seconds
548 return self.__vm_status_check(STATUS_ACTIVE, block,
549 self.instance_settings.vm_boot_timeout,
552 def __vm_status_check(self, expected_status_code, block, timeout,
555 Returns true when the VM status returns the value of
557 :param expected_status_code: instance status evaluated with this
559 :param block: When true, thread will block until active or timeout
560 value in seconds has been exceeded (False)
561 :param timeout: The timeout value
562 :param poll_interval: The polling interval in seconds
565 # sleep and wait for VM status change
569 return self.__status(expected_status_code)
571 while timeout > time.time() - start:
572 status = self.__status(expected_status_code)
574 logger.info('VM is - ' + expected_status_code)
577 logger.debug('Retry querying VM status in ' + str(
578 poll_interval) + ' seconds')
579 time.sleep(poll_interval)
580 logger.debug('VM status query timeout in ' + str(
581 timeout - (time.time() - start)))
584 'Timeout checking for VM status for ' + expected_status_code)
587 def __status(self, expected_status_code):
589 Returns True when active else False
590 :param expected_status_code: instance status evaluated with this string
597 instance = self.__nova.servers.get(self.__vm.id)
599 logger.warning('Cannot find instance with id - ' + self.__vm.id)
602 if instance.status == 'ERROR':
603 raise Exception('Instance had an error during deployment')
605 'Instance status [%s] is - %s', self.instance_settings.name,
607 return instance.status == expected_status_code
609 def vm_ssh_active(self, block=False, poll_interval=POLL_INTERVAL):
611 Returns true when the VM can be accessed via SSH
612 :param block: When true, thread will block until active or timeout
613 value in seconds has been exceeded (False)
614 :param poll_interval: The polling interval
617 # sleep and wait for VM status change
618 logger.info('Checking if VM is active')
620 timeout = self.instance_settings.ssh_connect_timeout
622 if self.vm_active(block=True):
626 start = time.time() - timeout
628 while timeout > time.time() - start:
629 status = self.__ssh_active()
631 logger.info('SSH is active for VM instance')
634 logger.debug('Retry SSH connection in ' + str(
635 poll_interval) + ' seconds')
636 time.sleep(poll_interval)
637 logger.debug('SSH connection timeout in ' + str(
638 timeout - (time.time() - start)))
640 logger.error('Timeout attempting to connect with VM via SSH')
643 def __ssh_active(self):
645 Returns True when can create a SSH session else False
648 if len(self.__floating_ips) > 0:
649 ssh = self.ssh_client()
654 def get_floating_ip(self, fip_name=None):
656 Returns the floating IP object byt name if found, else the first known,
658 :param fip_name: the name of the floating IP to return
659 :return: the SSH client or None
662 if fip_name and self.__floating_ip_dict.get(fip_name):
663 return self.__floating_ip_dict.get(fip_name)
664 if not fip and len(self.__floating_ips) > 0:
665 return self.__floating_ips[0]
668 def ssh_client(self, fip_name=None):
670 Returns an SSH client using the name or the first known floating IP if
672 :param fip_name: the name of the floating IP to return
673 :return: the SSH client or None
675 fip = self.get_floating_ip(fip_name)
677 return ansible_utils.ssh_client(
678 self.__floating_ips[0].ip, self.get_image_user(),
679 self.keypair_settings.private_filepath,
680 proxy_settings=self.__os_creds.proxy_settings)
683 'Cannot return an SSH client. No Floating IP configured')
685 def add_security_group(self, security_group):
687 Adds a security group to this VM. Call will block until VM is active.
688 :param security_group: the OpenStack security group object
689 :return True if successful else False
691 self.vm_active(block=True)
693 if not security_group:
694 logger.warning('Security group object is None, cannot add')
698 nova_utils.add_security_group(self.__nova, self.get_vm_inst(),
699 security_group['security_group'][
702 except NotFound as e:
703 logger.warning('Security group not added - ' + str(e))
706 def remove_security_group(self, security_group):
708 Removes a security group to this VM. Call will block until VM is active
709 :param security_group: the OpenStack security group object
710 :return True if successful else False
712 self.vm_active(block=True)
714 if not security_group:
715 logger.warning('Security group object is None, cannot remove')
719 nova_utils.remove_security_group(self.__nova, self.get_vm_inst(),
722 except NotFound as e:
723 logger.warning('Security group not removed - ' + str(e))
727 class VmInstanceSettings:
729 Class responsible for holding configuration setting for a VM Instance
732 def __init__(self, **kwargs):
735 :param name: the name of the VM
736 :param flavor: the VM's flavor
737 :param port_settings: the port configuration settings (required)
738 :param security_group_names: a set of names of the security groups to
740 :param floating_ip_settings: the floating IP configuration settings
741 :param sudo_user: the sudo user of the VM that will override the
742 instance_settings.image_user when trying to
744 :param vm_boot_timeout: the amount of time a thread will sleep waiting
745 for an instance to boot
746 :param vm_delete_timeout: the amount of time a thread will sleep
747 waiting for an instance to be deleted
748 :param ssh_connect_timeout: the amount of time a thread will sleep
749 waiting obtaining an SSH connection to a VM
750 :param availability_zone: the name of the compute server on which to
751 deploy the VM (optional)
752 :param userdata: the cloud-init script to run after the VM has been
755 self.name = kwargs.get('name')
756 self.flavor = kwargs.get('flavor')
757 self.sudo_user = kwargs.get('sudo_user')
758 self.userdata = kwargs.get('userdata')
760 self.port_settings = list()
761 port_settings = kwargs.get('ports')
762 if not port_settings:
763 port_settings = kwargs.get('port_settings')
765 for port_setting in port_settings:
766 if isinstance(port_setting, dict):
767 self.port_settings.append(PortSettings(**port_setting))
768 elif isinstance(port_setting, PortSettings):
769 self.port_settings.append(port_setting)
771 if kwargs.get('security_group_names'):
772 if isinstance(kwargs['security_group_names'], list):
773 self.security_group_names = kwargs['security_group_names']
774 elif isinstance(kwargs['security_group_names'], set):
775 self.security_group_names = kwargs['security_group_names']
776 elif isinstance(kwargs['security_group_names'], str):
777 self.security_group_names = [kwargs['security_group_names']]
780 'Invalid data type for security_group_names attribute')
782 self.security_group_names = set()
784 self.floating_ip_settings = list()
785 floating_ip_settings = kwargs.get('floating_ips')
786 if not floating_ip_settings:
787 floating_ip_settings = kwargs.get('floating_ip_settings')
788 if floating_ip_settings:
789 for floating_ip_config in floating_ip_settings:
790 if isinstance(floating_ip_config, FloatingIpSettings):
791 self.floating_ip_settings.append(floating_ip_config)
793 self.floating_ip_settings.append(FloatingIpSettings(
794 **floating_ip_config['floating_ip']))
796 if kwargs.get('vm_boot_timeout'):
797 self.vm_boot_timeout = kwargs['vm_boot_timeout']
799 self.vm_boot_timeout = 900
801 if kwargs.get('vm_delete_timeout'):
802 self.vm_delete_timeout = kwargs['vm_delete_timeout']
804 self.vm_delete_timeout = 300
806 if kwargs.get('ssh_connect_timeout'):
807 self.ssh_connect_timeout = kwargs['ssh_connect_timeout']
809 self.ssh_connect_timeout = 180
811 if kwargs.get('availability_zone'):
812 self.availability_zone = kwargs['availability_zone']
814 self.availability_zone = None
816 if not self.name or not self.flavor:
818 'Instance configuration requires the attributes: name, flavor')
820 if len(self.port_settings) == 0:
822 'Instance configuration requires port settings (aka. NICS)')
825 class FloatingIpSettings:
827 Class responsible for holding configuration settings for a floating IP
830 def __init__(self, **kwargs):
833 :param name: the name of the floating IP
834 :param port_name: the name of the router to the external network
835 :param router_name: the name of the router to the external network
836 :param subnet_name: the name of the subnet on which to attach the
838 :param provisioning: when true, this floating IP can be used for
841 TODO - provisioning flag is a hack as I have only observed a single
842 Floating IPs that actually works on an instance. Multiple floating IPs
843 placed on different subnets from the same port are especially
844 troublesome as you cannot predict which one will actually connect.
845 For now, it is recommended not to setup multiple floating IPs on an
846 instance unless absolutely necessary.
848 self.name = kwargs.get('name')
849 self.port_name = kwargs.get('port_name')
850 self.router_name = kwargs.get('router_name')
851 self.subnet_name = kwargs.get('subnet_name')
852 if kwargs.get('provisioning') is not None:
853 self.provisioning = kwargs['provisioning']
855 self.provisioning = True
857 if not self.name or not self.port_name or not self.router_name:
859 'The attributes name, port_name and router_name are required '
860 'for FloatingIPSettings')