1 # Copyright (c) 2016 Cable Television Laboratories, Inc. ("CableLabs")
2 # and others. All rights reserved.
4 # Licensed under the Apache License, Version 2.0 (the "License");
5 # you may not use this file except in compliance with the License.
6 # You may obtain a copy of the License at:
8 # http://www.apache.org/licenses/LICENSE-2.0
10 # Unless required by applicable law or agreed to in writing, software
11 # distributed under the License is distributed on an "AS IS" BASIS,
12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 # See the License for the specific language governing permissions and
14 # limitations under the License.
18 from neutronclient.common.exceptions import PortNotFoundClient
19 from novaclient.exceptions import NotFound
21 from snaps.openstack.utils import glance_utils
22 from snaps.openstack.utils import neutron_utils
23 from snaps.openstack.create_network import PortSettings
24 from snaps.provisioning import ansible_utils
25 from snaps.openstack.utils import nova_utils
27 __author__ = 'spisarski'
29 logger = logging.getLogger('create_instance')
32 STATUS_ACTIVE = 'ACTIVE'
33 STATUS_DELETED = 'DELETED'
36 class OpenStackVmInstance:
38 Class responsible for creating a VM instance in OpenStack
41 def __init__(self, os_creds, instance_settings, image_settings, keypair_settings=None):
44 :param os_creds: The connection credentials to the OpenStack API
45 :param instance_settings: Contains the settings for this VM
46 :param image_settings: The OpenStack image object settings
47 :param keypair_settings: The keypair metadata (Optional)
50 self.__os_creds = os_creds
52 self.__nova = nova_utils.nova_client(self.__os_creds)
53 self.__neutron = neutron_utils.neutron_client(self.__os_creds)
55 self.instance_settings = instance_settings
56 self.image_settings = image_settings
57 self.keypair_settings = keypair_settings
59 # TODO - get rid of FIP list and only use the dict(). Need to fix populating this object when already exists
60 self.__floating_ips = list()
61 self.__floating_ip_dict = dict()
63 # Instantiated in self.create()
66 # Note: this object does not change after the VM becomes active
69 def create(self, cleanup=False, block=False):
72 :param cleanup: When true, only perform lookups for OpenStack objects.
73 :param block: Thread will block until instance has either become active, error, or timeout waiting.
74 Additionally, when True, floating IPs will not be applied until VM is active.
75 :return: The VM reference object
78 self.__ports = self.__setup_ports(self.instance_settings.port_settings, cleanup)
79 self.__lookup_existing_vm_by_name()
80 if not self.__vm and not cleanup:
81 self.__create_vm(block)
83 except Exception as e:
84 logger.exception('Error occurred while setting up instance')
88 def __lookup_existing_vm_by_name(self):
90 Populates the member variables 'self.vm' and 'self.floating_ips' if a VM with the same name already exists
93 servers = nova_utils.get_servers_by_name(self.__nova, self.instance_settings.name)
94 for server in servers:
95 if server.name == self.instance_settings.name:
97 logger.info('Found existing machine with name - ' + self.instance_settings.name)
98 fips = self.__nova.floating_ips.list()
100 if fip.instance_id == server.id:
101 self.__floating_ips.append(fip)
102 # TODO - Determine a means to associate to the FIP configuration and add to FIP map
104 def __create_vm(self, block=False):
106 Responsible for creating the VM instance
107 :param block: Thread will block until instance has either become active, error, or timeout waiting.
108 Floating IPs will be assigned after active when block=True
111 for key, port in self.__ports:
113 kv['port-id'] = port['port']['id']
116 logger.info('Creating VM with name - ' + self.instance_settings.name)
118 if self.keypair_settings:
119 keypair_name = self.keypair_settings.name
121 flavor = nova_utils.get_flavor_by_name(self.__nova, self.instance_settings.flavor)
123 raise Exception('Flavor not found with name - ' + self.instance_settings.flavor)
125 image = glance_utils.get_image(self.__nova, glance_utils.glance_client(self.__os_creds),
126 self.image_settings.name)
128 self.__vm = self.__nova.servers.create(
129 name=self.instance_settings.name,
133 key_name=keypair_name,
134 security_groups=self.instance_settings.security_group_names,
135 userdata=self.instance_settings.userdata,
136 availability_zone=self.instance_settings.availability_zone)
139 raise Exception('Cannot create instance, image cannot be located with name ' + self.image_settings.name)
141 logger.info('Created instance with name - ' + self.instance_settings.name)
144 self.vm_active(block=True)
146 # TODO - the call above should add security groups. The return object shows they exist but the association
147 # had never been made by OpenStack. This call is here to ensure they have been added
148 for sec_grp_name in self.instance_settings.security_group_names:
149 nova_utils.add_security_group(self.__nova, self.__vm, sec_grp_name)
151 self.__apply_floating_ips()
153 def __apply_floating_ips(self):
155 Applies the configured floating IPs to the necessary ports
158 for key, port in self.__ports:
159 port_dict[key] = port
162 for floating_ip_setting in self.instance_settings.floating_ip_settings:
163 port = port_dict.get(floating_ip_setting.port_name)
166 raise Exception('Cannot find port object with name - ' + floating_ip_setting.port_name)
168 # Setup Floating IP only if there is a router with an external gateway
169 ext_gateway = self.__ext_gateway_by_router(floating_ip_setting.router_name)
171 subnet = neutron_utils.get_subnet_by_name(self.__neutron, floating_ip_setting.subnet_name)
172 floating_ip = nova_utils.create_floating_ip(self.__nova, ext_gateway)
173 self.__floating_ips.append(floating_ip)
174 self.__floating_ip_dict[floating_ip_setting.name] = floating_ip
176 logger.info('Created floating IP ' + floating_ip.ip + ' via router - ' +
177 floating_ip_setting.router_name)
178 self.__add_floating_ip(floating_ip, port, subnet)
180 raise Exception('Unable to add floating IP to port,' +
181 ' cannot locate router with an external gateway ')
183 def __ext_gateway_by_router(self, router_name):
185 Returns network name for the external network attached to a router or None if not found
186 :param router_name: The name of the router to lookup
187 :return: the external network name or None
189 router = neutron_utils.get_router_by_name(self.__neutron, router_name)
190 if router and router['router'].get('external_gateway_info'):
191 network = neutron_utils.get_network_by_id(self.__neutron,
192 router['router']['external_gateway_info']['network_id'])
194 return network['network']['name']
199 Destroys the VM instance
202 # Cleanup floating IPs
203 for floating_ip in self.__floating_ips:
205 logger.info('Deleting Floating IP - ' + floating_ip.ip)
206 nova_utils.delete_floating_ip(self.__nova, floating_ip)
207 except Exception as e:
208 logger.error('Error deleting Floating IP - ' + e.message)
209 self.__floating_ips = list()
210 self.__floating_ip_dict = dict()
213 for name, port in self.__ports:
214 logger.info('Deleting Port - ' + name)
216 neutron_utils.delete_port(self.__neutron, port)
217 except PortNotFoundClient as e:
218 logger.warn('Unexpected error deleting port - ' + e.message)
220 self.__ports = list()
225 logger.info('Deleting VM instance - ' + self.instance_settings.name)
226 nova_utils.delete_vm_instance(self.__nova, self.__vm)
227 except Exception as e:
228 logger.error('Error deleting VM - ' + str(e))
230 # Block until instance cannot be found or returns the status of DELETED
231 logger.info('Checking deletion status')
234 if self.vm_deleted(block=True):
235 logger.info('VM has been properly deleted VM with name - ' + self.instance_settings.name)
238 logger.error('VM not deleted within the timeout period of ' +
239 str(self.instance_settings.vm_delete_timeout) + ' seconds')
240 except Exception as e:
241 logger.error('Unexpected error while checking VM instance status - ' + e.message)
243 def __setup_ports(self, port_settings, cleanup):
245 Returns the previously configured ports or creates them if they do not exist
246 :param port_settings: A list of PortSetting objects
247 :param cleanup: When true, only perform lookups for OpenStack objects.
248 :return: a list of OpenStack port tuples where the first member is the port name and the second is the port
253 for port_setting in port_settings:
254 # First check to see if network already has this port
255 # TODO/FIXME - this could potentially cause problems if another port with the same name exists
256 # VM has the same network/port name pair
259 # TODO/FIXME - should we not be iterating on ports for the specific network in question as unique port names
260 # seem to only be important by network
261 existing_ports = self.__neutron.list_ports()['ports']
262 for existing_port in existing_ports:
263 if existing_port['name'] == port_setting.name:
264 ports.append((port_setting.name, {'port': existing_port}))
268 if not found and not cleanup:
269 ports.append((port_setting.name, neutron_utils.create_port(self.__neutron, self.__os_creds,
274 def __add_floating_ip(self, floating_ip, port, subnet, timeout=30, poll_interval=POLL_INTERVAL):
276 Returns True when active else False
277 TODO - Make timeout and poll_interval configurable...
282 # Take IP of subnet if there is one configured on which to place the floating IP
283 for fixed_ip in port['port']['fixed_ips']:
284 if fixed_ip['subnet_id'] == subnet['subnet']['id']:
285 ip = fixed_ip['ip_address']
288 # Simply take the first
289 ip = port['port']['fixed_ips'][0]['ip_address']
292 count = timeout / poll_interval
294 logger.debug('Attempting to add floating IP to instance')
296 self.__vm.add_floating_ip(floating_ip, ip)
297 logger.info('Added floating IP ' + floating_ip.ip + ' to port IP - ' + ip +
298 ' on instance - ' + self.instance_settings.name)
300 except Exception as e:
301 logger.debug('Retry adding floating IP to instance. Last attempt failed with - ' + e.message)
302 time.sleep(poll_interval)
306 raise Exception('Unable find IP address on which to place the floating IP')
308 logger.error('Timeout attempting to add the floating IP to instance.')
309 raise Exception('Timeout while attempting add floating IP to instance')
311 def get_os_creds(self):
313 Returns the OpenStack credentials used to create these objects
314 :return: the credentials
316 return self.__os_creds
318 def get_vm_inst(self):
320 Returns the latest version of this server object from OpenStack
321 :return: Server object
323 return nova_utils.get_latest_server_object(self.__nova, self.__vm)
325 def get_port_ip(self, port_name, subnet_name=None):
327 Returns the first IP for the port corresponding with the port_name parameter when subnet_name is None
328 else returns the IP address that corresponds to the subnet_name parameter
329 :param port_name: the name of the port from which to return the IP
330 :param subnet_name: the name of the subnet attached to this IP
331 :return: the IP or None if not found
333 port = self.get_port_by_name(port_name)
335 port_dict = port['port']
337 subnet = neutron_utils.get_subnet_by_name(self.__neutron, subnet_name)
339 logger.warn('Cannot retrieve port IP as subnet could not be located with name - ' + subnet_name)
341 for fixed_ip in port_dict['fixed_ips']:
342 if fixed_ip['subnet_id'] == subnet['subnet']['id']:
343 return fixed_ip['ip_address']
345 fixed_ips = port_dict['fixed_ips']
346 if fixed_ips and len(fixed_ips) > 0:
347 return fixed_ips[0]['ip_address']
350 def get_port_mac(self, port_name):
352 Returns the first IP for the port corresponding with the port_name parameter
353 TODO - Add in the subnet as an additional parameter as a port may have multiple fixed_ips
354 :param port_name: the name of the port from which to return the IP
355 :return: the IP or None if not found
357 port = self.get_port_by_name(port_name)
359 port_dict = port['port']
360 return port_dict['mac_address']
363 def get_port_by_name(self, port_name):
365 Retrieves the OpenStack port object by its given name
366 :param port_name: the name of the port
367 :return: the OpenStack port object or None if not exists
369 for key, port in self.__ports:
372 logger.warn('Cannot find port with name - ' + port_name)
375 def config_nics(self):
377 Responsible for configuring NICs on RPM systems where the instance has more than one configured port
380 if len(self.__ports) > 1 and len(self.__floating_ips) > 0:
381 if self.vm_active(block=True) and self.vm_ssh_active(block=True):
382 for key, port in self.__ports:
383 port_index = self.__ports.index((key, port))
385 nic_name = 'eth' + repr(port_index)
386 self.__config_nic(nic_name, port, self.__get_first_provisioning_floating_ip().ip)
387 logger.info('Configured NIC - ' + nic_name + ' on VM - ' + self.instance_settings.name)
389 def __get_first_provisioning_floating_ip(self):
391 Returns the first floating IP tagged with the Floating IP name if exists else the first one found
394 for floating_ip_setting in self.instance_settings.floating_ip_settings:
395 if floating_ip_setting.provisioning:
396 fip = self.__floating_ip_dict.get(floating_ip_setting.name)
399 elif len(self.__floating_ips) > 0:
400 return self.__floating_ips[0]
402 def __config_nic(self, nic_name, port, floating_ip):
404 Although ports/NICs can contain multiple IPs, this code currently only supports the first.
406 Your CWD at this point must be the <repo dir>/python directory.
407 TODO - fix this restriction.
409 :param nic_name: Name of the interface
410 :param port: The port information containing the expected IP values.
411 :param floating_ip: The floating IP on which to apply the playbook.
413 ip = port['port']['fixed_ips'][0]['ip_address']
415 'floating_ip': floating_ip,
416 'nic_name': nic_name,
420 if self.image_settings.nic_config_pb_loc and self.keypair_settings:
421 ansible_utils.apply_playbook(self.image_settings.nic_config_pb_loc,
422 [floating_ip], self.get_image_user(), self.keypair_settings.private_filepath,
423 variables, self.__os_creds.proxy_settings)
425 logger.warn('VM ' + self.instance_settings.name + ' cannot self configure NICs eth1++. ' +
426 'No playbook or keypairs found.')
428 def get_image_user(self):
430 Returns the instance sudo_user if it has been configured in the instance_settings else it returns the
431 image_settings.image_user value
433 if self.instance_settings.sudo_user:
434 return self.instance_settings.sudo_user
436 return self.image_settings.image_user
438 def vm_deleted(self, block=False, poll_interval=POLL_INTERVAL):
440 Returns true when the VM status returns the value of expected_status_code or instance retrieval throws
441 a NotFound exception.
442 :param block: When true, thread will block until active or timeout value in seconds has been exceeded (False)
443 :param poll_interval: The polling interval in seconds
447 return self.__vm_status_check(STATUS_DELETED, block, self.instance_settings.vm_delete_timeout,
449 except NotFound as e:
450 logger.debug("Instance not found when querying status for " + STATUS_DELETED + ' with message ' + e.message)
453 def vm_active(self, block=False, poll_interval=POLL_INTERVAL):
455 Returns true when the VM status returns the value of expected_status_code
456 :param block: When true, thread will block until active or timeout value in seconds has been exceeded (False)
457 :param poll_interval: The polling interval in seconds
460 return self.__vm_status_check(STATUS_ACTIVE, block, self.instance_settings.vm_boot_timeout, poll_interval)
462 def __vm_status_check(self, expected_status_code, block, timeout, poll_interval):
464 Returns true when the VM status returns the value of expected_status_code
465 :param expected_status_code: instance status evaluated with this string value
466 :param block: When true, thread will block until active or timeout value in seconds has been exceeded (False)
467 :param timeout: The timeout value
468 :param poll_interval: The polling interval in seconds
471 # sleep and wait for VM status change
475 start = time.time() - timeout
477 while timeout > time.time() - start:
478 status = self.__status(expected_status_code)
480 logger.info('VM is - ' + expected_status_code)
483 logger.debug('Retry querying VM status in ' + str(poll_interval) + ' seconds')
484 time.sleep(poll_interval)
485 logger.debug('VM status query timeout in ' + str(timeout - (time.time() - start)))
487 logger.error('Timeout checking for VM status for ' + expected_status_code)
490 def __status(self, expected_status_code):
492 Returns True when active else False
493 :param expected_status_code: instance status evaluated with this string value
496 instance = self.__nova.servers.get(self.__vm.id)
498 logger.warn('Cannot find instance with id - ' + self.__vm.id)
501 if instance.status == 'ERROR':
502 raise Exception('Instance had an error during deployment')
503 logger.debug('Instance status [' + self.instance_settings.name + '] is - ' + instance.status)
504 return instance.status == expected_status_code
506 def vm_ssh_active(self, block=False, poll_interval=POLL_INTERVAL):
508 Returns true when the VM can be accessed via SSH
509 :param block: When true, thread will block until active or timeout value in seconds has been exceeded (False)
510 :param poll_interval: The polling interval
513 # sleep and wait for VM status change
514 logger.info('Checking if VM is active')
516 timeout = self.instance_settings.ssh_connect_timeout
518 if self.vm_active(block=True):
522 start = time.time() - timeout
524 while timeout > time.time() - start:
525 status = self.__ssh_active()
527 logger.info('SSH is active for VM instance')
530 logger.debug('Retry SSH connection in ' + str(poll_interval) + ' seconds')
531 time.sleep(poll_interval)
532 logger.debug('SSH connection timeout in ' + str(timeout - (time.time() - start)))
534 logger.error('Timeout attempting to connect with VM via SSH')
537 def __ssh_active(self):
539 Returns True when can create a SSH session else False
542 if len(self.__floating_ips) > 0:
543 ssh = self.ssh_client()
548 def get_floating_ip(self, fip_name=None):
550 Returns the floating IP object byt name if found, else the first known, else None
551 :param fip_name: the name of the floating IP to return
552 :return: the SSH client or None
555 if fip_name and self.__floating_ip_dict.get(fip_name):
556 return self.__floating_ip_dict.get(fip_name)
557 if not fip and len(self.__floating_ips) > 0:
558 return self.__floating_ips[0]
561 def ssh_client(self, fip_name=None):
563 Returns an SSH client using the name or the first known floating IP if exists, else None
564 :param fip_name: the name of the floating IP to return
565 :return: the SSH client or None
567 fip = self.get_floating_ip(fip_name)
569 return ansible_utils.ssh_client(self.__floating_ips[0].ip, self.get_image_user(),
570 self.keypair_settings.private_filepath,
571 proxy_settings=self.__os_creds.proxy_settings)
573 logger.warn('Cannot return an SSH client. No Floating IP configured')
575 def add_security_group(self, security_group):
577 Adds a security group to this VM. Call will block until VM is active.
578 :param security_group: the OpenStack security group object
579 :return True if successful else False
581 self.vm_active(block=True)
583 if not security_group:
584 logger.warn('Security group object is None, cannot add')
588 nova_utils.add_security_group(self.__nova, self.get_vm_inst(), security_group['security_group']['name'])
590 except NotFound as e:
591 logger.warn('Security group not added - ' + e.message)
594 def remove_security_group(self, security_group):
596 Removes a security group to this VM. Call will block until VM is active.
597 :param security_group: the OpenStack security group object
598 :return True if successful else False
600 self.vm_active(block=True)
602 if not security_group:
603 logger.warn('Security group object is None, cannot remove')
607 nova_utils.remove_security_group(self.__nova, self.get_vm_inst(), security_group)
609 except NotFound as e:
610 logger.warn('Security group not removed - ' + e.message)
614 class VmInstanceSettings:
616 Class responsible for holding configuration setting for a VM Instance
618 def __init__(self, config=None, name=None, flavor=None, port_settings=list(), security_group_names=set(),
619 floating_ip_settings=list(), sudo_user=None, vm_boot_timeout=900,
620 vm_delete_timeout=300, ssh_connect_timeout=180, availability_zone=None, userdata=None):
623 :param config: dict() object containing the configuration settings using the attribute names below as each
624 member's the key and overrides any of the other parameters.
625 :param name: the name of the VM
626 :param flavor: the VM's flavor
627 :param port_settings: the port configuration settings (required)
628 :param security_group_names: a set of names of the security groups to add to the VM
629 :param floating_ip_settings: the floating IP configuration settings
630 :param sudo_user: the sudo user of the VM that will override the instance_settings.image_user when trying to
632 :param vm_boot_timeout: the amount of time a thread will sleep waiting for an instance to boot
633 :param vm_delete_timeout: the amount of time a thread will sleep waiting for an instance to be deleted
634 :param ssh_connect_timeout: the amount of time a thread will sleep waiting obtaining an SSH connection to a VM
635 :param availability_zone: the name of the compute server on which to deploy the VM (optional)
636 :param userdata: the cloud-init script to run after the VM has been started
639 self.name = config.get('name')
640 self.flavor = config.get('flavor')
641 self.sudo_user = config.get('sudo_user')
642 self.userdata = config.get('userdata')
644 self.port_settings = list()
645 if config.get('ports'):
646 for port_config in config['ports']:
647 if isinstance(port_config, PortSettings):
648 self.port_settings.append(port_config)
650 self.port_settings.append(PortSettings(config=port_config['port']))
652 if config.get('security_group_names'):
653 if isinstance(config['security_group_names'], list):
654 self.security_group_names = set(config['security_group_names'])
655 elif isinstance(config['security_group_names'], set):
656 self.security_group_names = config['security_group_names']
657 elif isinstance(config['security_group_names'], basestring):
658 self.security_group_names = [config['security_group_names']]
660 raise Exception('Invalid data type for security_group_names attribute')
662 self.security_group_names = set()
664 self.floating_ip_settings = list()
665 if config.get('floating_ips'):
666 for floating_ip_config in config['floating_ips']:
667 if isinstance(floating_ip_config, FloatingIpSettings):
668 self.floating_ip_settings.append(floating_ip_config)
670 self.floating_ip_settings.append(FloatingIpSettings(config=floating_ip_config['floating_ip']))
672 if config.get('vm_boot_timeout'):
673 self.vm_boot_timeout = config['vm_boot_timeout']
675 self.vm_boot_timeout = vm_boot_timeout
677 if config.get('vm_delete_timeout'):
678 self.vm_delete_timeout = config['vm_delete_timeout']
680 self.vm_delete_timeout = vm_delete_timeout
682 if config.get('ssh_connect_timeout'):
683 self.ssh_connect_timeout = config['ssh_connect_timeout']
685 self.ssh_connect_timeout = ssh_connect_timeout
687 if config.get('availability_zone'):
688 self.availability_zone = config['availability_zone']
690 self.availability_zone = None
694 self.port_settings = port_settings
695 self.security_group_names = security_group_names
696 self.floating_ip_settings = floating_ip_settings
697 self.sudo_user = sudo_user
698 self.vm_boot_timeout = vm_boot_timeout
699 self.vm_delete_timeout = vm_delete_timeout
700 self.ssh_connect_timeout = ssh_connect_timeout
701 self.availability_zone = availability_zone
702 self.userdata = userdata
704 if not self.name or not self.flavor:
705 raise Exception('Instance configuration requires the attributes: name, flavor')
707 if len(self.port_settings) == 0:
708 raise Exception('Instance configuration requires port settings (aka. NICS)')
711 class FloatingIpSettings:
713 Class responsible for holding configuration settings for a floating IP
715 def __init__(self, config=None, name=None, port_name=None, router_name=None, subnet_name=None, provisioning=True):
718 :param config: dict() object containing the configuration settings using the attribute names below as each
719 member's the key and overrides any of the other parameters.
720 :param name: the name of the floating IP
721 :param port_name: the name of the router to the external network
722 :param router_name: the name of the router to the external network
723 :param subnet_name: the name of the subnet on which to attach the floating IP
724 :param provisioning: when true, this floating IP can be used for provisioning
726 TODO - provisioning flag is a hack as I have only observed a single Floating IPs that actually works on
727 an instance. Multiple floating IPs placed on different subnets from the same port are especially troublesome
728 as you cannot predict which one will actually connect. For now, it is recommended not to setup multiple
729 floating IPs on an instance unless absolutely necessary.
732 self.name = config.get('name')
733 self.port_name = config.get('port_name')
734 self.router_name = config.get('router_name')
735 self.subnet_name = config.get('subnet_name')
736 if config.get('provisioning') is not None:
737 self.provisioning = config['provisioning']
739 self.provisioning = provisioning
742 self.port_name = port_name
743 self.router_name = router_name
744 self.subnet_name = subnet_name
745 self.provisioning = provisioning
747 if not self.name or not self.port_name or not self.router_name:
748 raise Exception('The attributes name, port_name and router_name are required for FloatingIPSettings')