1 # Copyright (c) 2016 Cable Television Laboratories, Inc. ("CableLabs")
2 # and others. All rights reserved.
4 # Licensed under the Apache License, Version 2.0 (the "License");
5 # you may not use this file except in compliance with the License.
6 # You may obtain a copy of the License at:
8 # http://www.apache.org/licenses/LICENSE-2.0
10 # Unless required by applicable law or agreed to in writing, software
11 # distributed under the License is distributed on an "AS IS" BASIS,
12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 # See the License for the specific language governing permissions and
14 # limitations under the License.
18 from neutronclient.common.exceptions import PortNotFoundClient
19 from novaclient.exceptions import NotFound
21 from snaps.openstack.utils import glance_utils
22 from snaps.openstack.utils import neutron_utils
23 from snaps.openstack.create_network import PortSettings
24 from snaps.provisioning import ansible_utils
25 from snaps.openstack.utils import nova_utils
27 __author__ = 'spisarski'
29 logger = logging.getLogger('create_instance')
32 STATUS_ACTIVE = 'ACTIVE'
33 STATUS_DELETED = 'DELETED'
36 class OpenStackVmInstance:
38 Class responsible for creating a VM instance in OpenStack
41 def __init__(self, os_creds, instance_settings, image_settings, keypair_settings=None):
44 :param os_creds: The connection credentials to the OpenStack API
45 :param instance_settings: Contains the settings for this VM
46 :param image_settings: The OpenStack image object settings
47 :param keypair_settings: The keypair metadata (Optional)
50 self.__os_creds = os_creds
52 self.__nova = nova_utils.nova_client(self.__os_creds)
53 self.__neutron = neutron_utils.neutron_client(self.__os_creds)
55 self.instance_settings = instance_settings
56 self.image_settings = image_settings
57 self.keypair_settings = keypair_settings
59 # TODO - get rid of FIP list and only use the dict(). Need to fix populating this object when already exists
60 self.__floating_ips = list()
61 self.__floating_ip_dict = dict()
63 # Instantiated in self.create()
66 # Note: this object does not change after the VM becomes active
69 def create(self, cleanup=False, block=False):
72 :param cleanup: When true, only perform lookups for OpenStack objects.
73 :param block: Thread will block until instance has either become active, error, or timeout waiting.
74 Additionally, when True, floating IPs will not be applied until VM is active.
75 :return: The VM reference object
78 self.__ports = self.__setup_ports(self.instance_settings.port_settings, cleanup)
79 self.__lookup_existing_vm_by_name()
80 if not self.__vm and not cleanup:
81 self.__create_vm(block)
83 except Exception as e:
84 logger.exception('Error occurred while setting up instance')
88 def __lookup_existing_vm_by_name(self):
90 Populates the member variables 'self.vm' and 'self.floating_ips' if a VM with the same name already exists
93 servers = nova_utils.get_servers_by_name(self.__nova, self.instance_settings.name)
94 for server in servers:
95 if server.name == self.instance_settings.name:
97 logger.info('Found existing machine with name - ' + self.instance_settings.name)
98 fips = self.__nova.floating_ips.list()
100 if fip.instance_id == server.id:
101 self.__floating_ips.append(fip)
102 # TODO - Determine a means to associate to the FIP configuration and add to FIP map
104 def __create_vm(self, block=False):
106 Responsible for creating the VM instance
107 :param block: Thread will block until instance has either become active, error, or timeout waiting.
108 Floating IPs will be assigned after active when block=True
111 for key, port in self.__ports:
113 kv['port-id'] = port['port']['id']
116 logger.info('Creating VM with name - ' + self.instance_settings.name)
118 if self.keypair_settings:
119 keypair_name = self.keypair_settings.name
121 flavor = nova_utils.get_flavor_by_name(self.__nova, self.instance_settings.flavor)
123 raise Exception('Flavor not found with name - ' + self.instance_settings.flavor)
125 image = glance_utils.get_image(self.__nova, glance_utils.glance_client(self.__os_creds),
126 self.image_settings.name)
128 self.__vm = self.__nova.servers.create(
129 name=self.instance_settings.name,
133 key_name=keypair_name,
134 security_groups=self.instance_settings.security_group_names,
135 userdata=self.instance_settings.userdata,
136 availability_zone=self.instance_settings.availability_zone)
139 raise Exception('Cannot create instance, image cannot be located with name ' + self.image_settings.name)
141 logger.info('Created instance with name - ' + self.instance_settings.name)
144 self.vm_active(block=True)
146 # TODO - the call above should add security groups. The return object shows they exist but the association
147 # had never been made by OpenStack. This call is here to ensure they have been added
148 for sec_grp_name in self.instance_settings.security_group_names:
149 if self.vm_active(block=True):
150 nova_utils.add_security_group(self.__nova, self.__vm, sec_grp_name)
152 raise Exception('Cannot applying security group with name ' + sec_grp_name +
153 ' to VM that did not activate with name - ' + self.instance_settings.name)
155 self.__apply_floating_ips()
157 def __apply_floating_ips(self):
159 Applies the configured floating IPs to the necessary ports
162 for key, port in self.__ports:
163 port_dict[key] = port
166 for floating_ip_setting in self.instance_settings.floating_ip_settings:
167 port = port_dict.get(floating_ip_setting.port_name)
170 raise Exception('Cannot find port object with name - ' + floating_ip_setting.port_name)
172 # Setup Floating IP only if there is a router with an external gateway
173 ext_gateway = self.__ext_gateway_by_router(floating_ip_setting.router_name)
175 subnet = neutron_utils.get_subnet_by_name(self.__neutron, floating_ip_setting.subnet_name)
176 floating_ip = nova_utils.create_floating_ip(self.__nova, ext_gateway)
177 self.__floating_ips.append(floating_ip)
178 self.__floating_ip_dict[floating_ip_setting.name] = floating_ip
180 logger.info('Created floating IP ' + floating_ip.ip + ' via router - ' +
181 floating_ip_setting.router_name)
182 self.__add_floating_ip(floating_ip, port, subnet)
184 raise Exception('Unable to add floating IP to port,' +
185 ' cannot locate router with an external gateway ')
187 def __ext_gateway_by_router(self, router_name):
189 Returns network name for the external network attached to a router or None if not found
190 :param router_name: The name of the router to lookup
191 :return: the external network name or None
193 router = neutron_utils.get_router_by_name(self.__neutron, router_name)
194 if router and router['router'].get('external_gateway_info'):
195 network = neutron_utils.get_network_by_id(self.__neutron,
196 router['router']['external_gateway_info']['network_id'])
198 return network['network']['name']
203 Destroys the VM instance
206 # Cleanup floating IPs
207 for floating_ip in self.__floating_ips:
209 logger.info('Deleting Floating IP - ' + floating_ip.ip)
210 nova_utils.delete_floating_ip(self.__nova, floating_ip)
211 except Exception as e:
212 logger.error('Error deleting Floating IP - ' + e.message)
213 self.__floating_ips = list()
214 self.__floating_ip_dict = dict()
217 for name, port in self.__ports:
218 logger.info('Deleting Port - ' + name)
220 neutron_utils.delete_port(self.__neutron, port)
221 except PortNotFoundClient as e:
222 logger.warn('Unexpected error deleting port - ' + e.message)
224 self.__ports = list()
229 logger.info('Deleting VM instance - ' + self.instance_settings.name)
230 nova_utils.delete_vm_instance(self.__nova, self.__vm)
231 except Exception as e:
232 logger.error('Error deleting VM - ' + str(e))
234 # Block until instance cannot be found or returns the status of DELETED
235 logger.info('Checking deletion status')
238 if self.vm_deleted(block=True):
239 logger.info('VM has been properly deleted VM with name - ' + self.instance_settings.name)
242 logger.error('VM not deleted within the timeout period of ' +
243 str(self.instance_settings.vm_delete_timeout) + ' seconds')
244 except Exception as e:
245 logger.error('Unexpected error while checking VM instance status - ' + e.message)
247 def __setup_ports(self, port_settings, cleanup):
249 Returns the previously configured ports or creates them if they do not exist
250 :param port_settings: A list of PortSetting objects
251 :param cleanup: When true, only perform lookups for OpenStack objects.
252 :return: a list of OpenStack port tuples where the first member is the port name and the second is the port
257 for port_setting in port_settings:
258 # First check to see if network already has this port
259 # TODO/FIXME - this could potentially cause problems if another port with the same name exists
260 # VM has the same network/port name pair
263 # TODO/FIXME - should we not be iterating on ports for the specific network in question as unique port names
264 # seem to only be important by network
265 existing_ports = self.__neutron.list_ports()['ports']
266 for existing_port in existing_ports:
267 if existing_port['name'] == port_setting.name:
268 ports.append((port_setting.name, {'port': existing_port}))
272 if not found and not cleanup:
273 ports.append((port_setting.name, neutron_utils.create_port(self.__neutron, self.__os_creds,
278 def __add_floating_ip(self, floating_ip, port, subnet, timeout=30, poll_interval=POLL_INTERVAL):
280 Returns True when active else False
281 TODO - Make timeout and poll_interval configurable...
286 # Take IP of subnet if there is one configured on which to place the floating IP
287 for fixed_ip in port['port']['fixed_ips']:
288 if fixed_ip['subnet_id'] == subnet['subnet']['id']:
289 ip = fixed_ip['ip_address']
292 # Simply take the first
293 ip = port['port']['fixed_ips'][0]['ip_address']
296 count = timeout / poll_interval
298 logger.debug('Attempting to add floating IP to instance')
300 self.__vm.add_floating_ip(floating_ip, ip)
301 logger.info('Added floating IP ' + floating_ip.ip + ' to port IP - ' + ip +
302 ' on instance - ' + self.instance_settings.name)
304 except Exception as e:
305 logger.debug('Retry adding floating IP to instance. Last attempt failed with - ' + e.message)
306 time.sleep(poll_interval)
310 raise Exception('Unable find IP address on which to place the floating IP')
312 logger.error('Timeout attempting to add the floating IP to instance.')
313 raise Exception('Timeout while attempting add floating IP to instance')
315 def get_os_creds(self):
317 Returns the OpenStack credentials used to create these objects
318 :return: the credentials
320 return self.__os_creds
322 def get_vm_inst(self):
324 Returns the latest version of this server object from OpenStack
325 :return: Server object
327 return nova_utils.get_latest_server_object(self.__nova, self.__vm)
329 def get_port_ip(self, port_name, subnet_name=None):
331 Returns the first IP for the port corresponding with the port_name parameter when subnet_name is None
332 else returns the IP address that corresponds to the subnet_name parameter
333 :param port_name: the name of the port from which to return the IP
334 :param subnet_name: the name of the subnet attached to this IP
335 :return: the IP or None if not found
337 port = self.get_port_by_name(port_name)
339 port_dict = port['port']
341 subnet = neutron_utils.get_subnet_by_name(self.__neutron, subnet_name)
343 logger.warn('Cannot retrieve port IP as subnet could not be located with name - ' + subnet_name)
345 for fixed_ip in port_dict['fixed_ips']:
346 if fixed_ip['subnet_id'] == subnet['subnet']['id']:
347 return fixed_ip['ip_address']
349 fixed_ips = port_dict['fixed_ips']
350 if fixed_ips and len(fixed_ips) > 0:
351 return fixed_ips[0]['ip_address']
354 def get_port_mac(self, port_name):
356 Returns the first IP for the port corresponding with the port_name parameter
357 TODO - Add in the subnet as an additional parameter as a port may have multiple fixed_ips
358 :param port_name: the name of the port from which to return the IP
359 :return: the IP or None if not found
361 port = self.get_port_by_name(port_name)
363 port_dict = port['port']
364 return port_dict['mac_address']
367 def get_port_by_name(self, port_name):
369 Retrieves the OpenStack port object by its given name
370 :param port_name: the name of the port
371 :return: the OpenStack port object or None if not exists
373 for key, port in self.__ports:
376 logger.warn('Cannot find port with name - ' + port_name)
379 def config_nics(self):
381 Responsible for configuring NICs on RPM systems where the instance has more than one configured port
384 if len(self.__ports) > 1 and len(self.__floating_ips) > 0:
385 if self.vm_active(block=True) and self.vm_ssh_active(block=True):
386 for key, port in self.__ports:
387 port_index = self.__ports.index((key, port))
389 nic_name = 'eth' + repr(port_index)
390 self.__config_nic(nic_name, port, self.__get_first_provisioning_floating_ip().ip)
391 logger.info('Configured NIC - ' + nic_name + ' on VM - ' + self.instance_settings.name)
393 def __get_first_provisioning_floating_ip(self):
395 Returns the first floating IP tagged with the Floating IP name if exists else the first one found
398 for floating_ip_setting in self.instance_settings.floating_ip_settings:
399 if floating_ip_setting.provisioning:
400 fip = self.__floating_ip_dict.get(floating_ip_setting.name)
403 elif len(self.__floating_ips) > 0:
404 return self.__floating_ips[0]
406 def __config_nic(self, nic_name, port, floating_ip):
408 Although ports/NICs can contain multiple IPs, this code currently only supports the first.
410 Your CWD at this point must be the <repo dir>/python directory.
411 TODO - fix this restriction.
413 :param nic_name: Name of the interface
414 :param port: The port information containing the expected IP values.
415 :param floating_ip: The floating IP on which to apply the playbook.
417 ip = port['port']['fixed_ips'][0]['ip_address']
419 'floating_ip': floating_ip,
420 'nic_name': nic_name,
424 if self.image_settings.nic_config_pb_loc and self.keypair_settings:
425 ansible_utils.apply_playbook(self.image_settings.nic_config_pb_loc,
426 [floating_ip], self.get_image_user(), self.keypair_settings.private_filepath,
427 variables, self.__os_creds.proxy_settings)
429 logger.warn('VM ' + self.instance_settings.name + ' cannot self configure NICs eth1++. ' +
430 'No playbook or keypairs found.')
432 def get_image_user(self):
434 Returns the instance sudo_user if it has been configured in the instance_settings else it returns the
435 image_settings.image_user value
437 if self.instance_settings.sudo_user:
438 return self.instance_settings.sudo_user
440 return self.image_settings.image_user
442 def vm_deleted(self, block=False, poll_interval=POLL_INTERVAL):
444 Returns true when the VM status returns the value of expected_status_code or instance retrieval throws
445 a NotFound exception.
446 :param block: When true, thread will block until active or timeout value in seconds has been exceeded (False)
447 :param poll_interval: The polling interval in seconds
451 return self.__vm_status_check(STATUS_DELETED, block, self.instance_settings.vm_delete_timeout,
453 except NotFound as e:
454 logger.debug("Instance not found when querying status for " + STATUS_DELETED + ' with message ' + e.message)
457 def vm_active(self, block=False, poll_interval=POLL_INTERVAL):
459 Returns true when the VM status returns the value of expected_status_code
460 :param block: When true, thread will block until active or timeout value in seconds has been exceeded (False)
461 :param poll_interval: The polling interval in seconds
464 return self.__vm_status_check(STATUS_ACTIVE, block, self.instance_settings.vm_boot_timeout, poll_interval)
466 def __vm_status_check(self, expected_status_code, block, timeout, poll_interval):
468 Returns true when the VM status returns the value of expected_status_code
469 :param expected_status_code: instance status evaluated with this string value
470 :param block: When true, thread will block until active or timeout value in seconds has been exceeded (False)
471 :param timeout: The timeout value
472 :param poll_interval: The polling interval in seconds
475 # sleep and wait for VM status change
479 start = time.time() - timeout
481 while timeout > time.time() - start:
482 status = self.__status(expected_status_code)
484 logger.info('VM is - ' + expected_status_code)
487 logger.debug('Retry querying VM status in ' + str(poll_interval) + ' seconds')
488 time.sleep(poll_interval)
489 logger.debug('VM status query timeout in ' + str(timeout - (time.time() - start)))
491 logger.error('Timeout checking for VM status for ' + expected_status_code)
494 def __status(self, expected_status_code):
496 Returns True when active else False
497 :param expected_status_code: instance status evaluated with this string value
500 instance = self.__nova.servers.get(self.__vm.id)
502 logger.warn('Cannot find instance with id - ' + self.__vm.id)
505 if instance.status == 'ERROR':
506 raise Exception('Instance had an error during deployment')
507 logger.debug('Instance status [' + self.instance_settings.name + '] is - ' + instance.status)
508 return instance.status == expected_status_code
510 def vm_ssh_active(self, block=False, poll_interval=POLL_INTERVAL):
512 Returns true when the VM can be accessed via SSH
513 :param block: When true, thread will block until active or timeout value in seconds has been exceeded (False)
514 :param poll_interval: The polling interval
517 # sleep and wait for VM status change
518 logger.info('Checking if VM is active')
520 timeout = self.instance_settings.ssh_connect_timeout
522 if self.vm_active(block=True):
526 start = time.time() - timeout
528 while timeout > time.time() - start:
529 status = self.__ssh_active()
531 logger.info('SSH is active for VM instance')
534 logger.debug('Retry SSH connection in ' + str(poll_interval) + ' seconds')
535 time.sleep(poll_interval)
536 logger.debug('SSH connection timeout in ' + str(timeout - (time.time() - start)))
538 logger.error('Timeout attempting to connect with VM via SSH')
541 def __ssh_active(self):
543 Returns True when can create a SSH session else False
546 if len(self.__floating_ips) > 0:
547 ssh = self.ssh_client()
552 def get_floating_ip(self, fip_name=None):
554 Returns the floating IP object byt name if found, else the first known, else None
555 :param fip_name: the name of the floating IP to return
556 :return: the SSH client or None
559 if fip_name and self.__floating_ip_dict.get(fip_name):
560 return self.__floating_ip_dict.get(fip_name)
561 if not fip and len(self.__floating_ips) > 0:
562 return self.__floating_ips[0]
565 def ssh_client(self, fip_name=None):
567 Returns an SSH client using the name or the first known floating IP if exists, else None
568 :param fip_name: the name of the floating IP to return
569 :return: the SSH client or None
571 fip = self.get_floating_ip(fip_name)
573 return ansible_utils.ssh_client(self.__floating_ips[0].ip, self.get_image_user(),
574 self.keypair_settings.private_filepath,
575 proxy_settings=self.__os_creds.proxy_settings)
577 logger.warn('Cannot return an SSH client. No Floating IP configured')
579 def add_security_group(self, security_group):
581 Adds a security group to this VM. Call will block until VM is active.
582 :param security_group: the OpenStack security group object
583 :return True if successful else False
585 self.vm_active(block=True)
587 if not security_group:
588 logger.warn('Security group object is None, cannot add')
592 nova_utils.add_security_group(self.__nova, self.get_vm_inst(), security_group['security_group']['name'])
594 except NotFound as e:
595 logger.warn('Security group not added - ' + e.message)
598 def remove_security_group(self, security_group):
600 Removes a security group to this VM. Call will block until VM is active.
601 :param security_group: the OpenStack security group object
602 :return True if successful else False
604 self.vm_active(block=True)
606 if not security_group:
607 logger.warn('Security group object is None, cannot remove')
611 nova_utils.remove_security_group(self.__nova, self.get_vm_inst(), security_group)
613 except NotFound as e:
614 logger.warn('Security group not removed - ' + e.message)
618 class VmInstanceSettings:
620 Class responsible for holding configuration setting for a VM Instance
622 def __init__(self, config=None, name=None, flavor=None, port_settings=list(), security_group_names=set(),
623 floating_ip_settings=list(), sudo_user=None, vm_boot_timeout=900,
624 vm_delete_timeout=300, ssh_connect_timeout=180, availability_zone=None, userdata=None):
627 :param config: dict() object containing the configuration settings using the attribute names below as each
628 member's the key and overrides any of the other parameters.
629 :param name: the name of the VM
630 :param flavor: the VM's flavor
631 :param port_settings: the port configuration settings (required)
632 :param security_group_names: a set of names of the security groups to add to the VM
633 :param floating_ip_settings: the floating IP configuration settings
634 :param sudo_user: the sudo user of the VM that will override the instance_settings.image_user when trying to
636 :param vm_boot_timeout: the amount of time a thread will sleep waiting for an instance to boot
637 :param vm_delete_timeout: the amount of time a thread will sleep waiting for an instance to be deleted
638 :param ssh_connect_timeout: the amount of time a thread will sleep waiting obtaining an SSH connection to a VM
639 :param availability_zone: the name of the compute server on which to deploy the VM (optional)
640 :param userdata: the cloud-init script to run after the VM has been started
643 self.name = config.get('name')
644 self.flavor = config.get('flavor')
645 self.sudo_user = config.get('sudo_user')
646 self.userdata = config.get('userdata')
648 self.port_settings = list()
649 if config.get('ports'):
650 for port_config in config['ports']:
651 if isinstance(port_config, PortSettings):
652 self.port_settings.append(port_config)
654 self.port_settings.append(PortSettings(config=port_config['port']))
656 if config.get('security_group_names'):
657 if isinstance(config['security_group_names'], list):
658 self.security_group_names = set(config['security_group_names'])
659 elif isinstance(config['security_group_names'], set):
660 self.security_group_names = config['security_group_names']
661 elif isinstance(config['security_group_names'], basestring):
662 self.security_group_names = [config['security_group_names']]
664 raise Exception('Invalid data type for security_group_names attribute')
666 self.security_group_names = set()
668 self.floating_ip_settings = list()
669 if config.get('floating_ips'):
670 for floating_ip_config in config['floating_ips']:
671 if isinstance(floating_ip_config, FloatingIpSettings):
672 self.floating_ip_settings.append(floating_ip_config)
674 self.floating_ip_settings.append(FloatingIpSettings(config=floating_ip_config['floating_ip']))
676 if config.get('vm_boot_timeout'):
677 self.vm_boot_timeout = config['vm_boot_timeout']
679 self.vm_boot_timeout = vm_boot_timeout
681 if config.get('vm_delete_timeout'):
682 self.vm_delete_timeout = config['vm_delete_timeout']
684 self.vm_delete_timeout = vm_delete_timeout
686 if config.get('ssh_connect_timeout'):
687 self.ssh_connect_timeout = config['ssh_connect_timeout']
689 self.ssh_connect_timeout = ssh_connect_timeout
691 if config.get('availability_zone'):
692 self.availability_zone = config['availability_zone']
694 self.availability_zone = None
698 self.port_settings = port_settings
699 self.security_group_names = security_group_names
700 self.floating_ip_settings = floating_ip_settings
701 self.sudo_user = sudo_user
702 self.vm_boot_timeout = vm_boot_timeout
703 self.vm_delete_timeout = vm_delete_timeout
704 self.ssh_connect_timeout = ssh_connect_timeout
705 self.availability_zone = availability_zone
706 self.userdata = userdata
708 if not self.name or not self.flavor:
709 raise Exception('Instance configuration requires the attributes: name, flavor')
711 if len(self.port_settings) == 0:
712 raise Exception('Instance configuration requires port settings (aka. NICS)')
715 class FloatingIpSettings:
717 Class responsible for holding configuration settings for a floating IP
719 def __init__(self, config=None, name=None, port_name=None, router_name=None, subnet_name=None, provisioning=True):
722 :param config: dict() object containing the configuration settings using the attribute names below as each
723 member's the key and overrides any of the other parameters.
724 :param name: the name of the floating IP
725 :param port_name: the name of the router to the external network
726 :param router_name: the name of the router to the external network
727 :param subnet_name: the name of the subnet on which to attach the floating IP
728 :param provisioning: when true, this floating IP can be used for provisioning
730 TODO - provisioning flag is a hack as I have only observed a single Floating IPs that actually works on
731 an instance. Multiple floating IPs placed on different subnets from the same port are especially troublesome
732 as you cannot predict which one will actually connect. For now, it is recommended not to setup multiple
733 floating IPs on an instance unless absolutely necessary.
736 self.name = config.get('name')
737 self.port_name = config.get('port_name')
738 self.router_name = config.get('router_name')
739 self.subnet_name = config.get('subnet_name')
740 if config.get('provisioning') is not None:
741 self.provisioning = config['provisioning']
743 self.provisioning = provisioning
746 self.port_name = port_name
747 self.router_name = router_name
748 self.subnet_name = subnet_name
749 self.provisioning = provisioning
751 if not self.name or not self.port_name or not self.router_name:
752 raise Exception('The attributes name, port_name and router_name are required for FloatingIPSettings')