1 # Copyright (c) 2016 Cable Television Laboratories, Inc. ("CableLabs")
2 # and others. All rights reserved.
4 # Licensed under the Apache License, Version 2.0 (the "License");
5 # you may not use this file except in compliance with the License.
6 # You may obtain a copy of the License at:
8 # http://www.apache.org/licenses/LICENSE-2.0
10 # Unless required by applicable law or agreed to in writing, software
11 # distributed under the License is distributed on an "AS IS" BASIS,
12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 # See the License for the specific language governing permissions and
14 # limitations under the License.
18 from neutronclient.common.exceptions import PortNotFoundClient
19 from novaclient.exceptions import NotFound
21 from snaps.openstack.utils import glance_utils
22 from snaps.openstack.utils import neutron_utils
23 from snaps.openstack.create_network import PortSettings
24 from snaps.provisioning import ansible_utils
25 from snaps.openstack.utils import nova_utils
27 __author__ = 'spisarski'
29 logger = logging.getLogger('create_instance')
32 STATUS_ACTIVE = 'ACTIVE'
33 STATUS_DELETED = 'DELETED'
36 class OpenStackVmInstance:
38 Class responsible for creating a VM instance in OpenStack
41 def __init__(self, os_creds, instance_settings, image_settings, keypair_settings=None):
44 :param os_creds: The connection credentials to the OpenStack API
45 :param instance_settings: Contains the settings for this VM
46 :param image_settings: The OpenStack image object settings
47 :param keypair_settings: The keypair metadata (Optional)
50 self.__os_creds = os_creds
52 self.__nova = nova_utils.nova_client(self.__os_creds)
53 self.__neutron = neutron_utils.neutron_client(self.__os_creds)
55 self.instance_settings = instance_settings
56 self.image_settings = image_settings
57 self.keypair_settings = keypair_settings
59 # TODO - get rid of FIP list and only use the dict(). Need to fix populating this object when already exists
60 self.__floating_ips = list()
61 self.__floating_ip_dict = dict()
63 # Instantiated in self.create()
66 # Note: this object does not change after the VM becomes active
69 def create(self, cleanup=False, block=False):
72 :param cleanup: When true, only perform lookups for OpenStack objects.
73 :param block: Thread will block until instance has either become active, error, or timeout waiting.
74 Additionally, when True, floating IPs will not be applied until VM is active.
75 :return: The VM reference object
78 self.__ports = self.__setup_ports(self.instance_settings.port_settings, cleanup)
79 self.__lookup_existing_vm_by_name()
80 if not self.__vm and not cleanup:
81 self.__create_vm(block)
83 except Exception as e:
84 logger.exception('Error occurred while setting up instance')
88 def __lookup_existing_vm_by_name(self):
90 Populates the member variables 'self.vm' and 'self.floating_ips' if a VM with the same name already exists
93 servers = nova_utils.get_servers_by_name(self.__nova, self.instance_settings.name)
94 for server in servers:
95 if server.name == self.instance_settings.name:
97 logger.info('Found existing machine with name - ' + self.instance_settings.name)
98 fips = self.__nova.floating_ips.list()
100 if fip.instance_id == server.id:
101 self.__floating_ips.append(fip)
102 # TODO - Determine a means to associate to the FIP configuration and add to FIP map
104 def __create_vm(self, block=False):
106 Responsible for creating the VM instance
107 :param block: Thread will block until instance has either become active, error, or timeout waiting.
108 Floating IPs will be assigned after active when block=True
111 for key, port in self.__ports:
113 kv['port-id'] = port['port']['id']
116 logger.info('Creating VM with name - ' + self.instance_settings.name)
118 if self.keypair_settings:
119 keypair_name = self.keypair_settings.name
121 flavor = nova_utils.get_flavor_by_name(self.__nova, self.instance_settings.flavor)
123 raise Exception('Flavor not found with name - ' + self.instance_settings.flavor)
125 image = glance_utils.get_image(self.__nova, glance_utils.glance_client(self.__os_creds),
126 self.image_settings.name)
128 self.__vm = self.__nova.servers.create(
129 name=self.instance_settings.name,
133 key_name=keypair_name,
134 security_groups=self.instance_settings.security_group_names,
135 userdata=self.instance_settings.userdata,
136 availability_zone=self.instance_settings.availability_zone)
139 raise Exception('Cannot create instance, image cannot be located with name ' + self.image_settings.name)
141 logger.info('Created instance with name - ' + self.instance_settings.name)
144 if not self.vm_active(block=True):
145 raise Exception('Fatal error, VM did not become ACTIVE within the alloted time')
147 # TODO - the call above should add security groups. The return object shows they exist but the association
148 # had never been made by OpenStack. This call is here to ensure they have been added
149 for sec_grp_name in self.instance_settings.security_group_names:
150 if self.vm_active(block=True):
151 nova_utils.add_security_group(self.__nova, self.__vm, sec_grp_name)
153 raise Exception('Cannot applying security group with name ' + sec_grp_name +
154 ' to VM that did not activate with name - ' + self.instance_settings.name)
156 self.__apply_floating_ips()
158 def __apply_floating_ips(self):
160 Applies the configured floating IPs to the necessary ports
163 for key, port in self.__ports:
164 port_dict[key] = port
167 for floating_ip_setting in self.instance_settings.floating_ip_settings:
168 port = port_dict.get(floating_ip_setting.port_name)
171 raise Exception('Cannot find port object with name - ' + floating_ip_setting.port_name)
173 # Setup Floating IP only if there is a router with an external gateway
174 ext_gateway = self.__ext_gateway_by_router(floating_ip_setting.router_name)
176 subnet = neutron_utils.get_subnet_by_name(self.__neutron, floating_ip_setting.subnet_name)
177 floating_ip = nova_utils.create_floating_ip(self.__nova, ext_gateway)
178 self.__floating_ips.append(floating_ip)
179 self.__floating_ip_dict[floating_ip_setting.name] = floating_ip
181 logger.info('Created floating IP ' + floating_ip.ip + ' via router - ' +
182 floating_ip_setting.router_name)
183 self.__add_floating_ip(floating_ip, port, subnet)
185 raise Exception('Unable to add floating IP to port,' +
186 ' cannot locate router with an external gateway ')
188 def __ext_gateway_by_router(self, router_name):
190 Returns network name for the external network attached to a router or None if not found
191 :param router_name: The name of the router to lookup
192 :return: the external network name or None
194 router = neutron_utils.get_router_by_name(self.__neutron, router_name)
195 if router and router['router'].get('external_gateway_info'):
196 network = neutron_utils.get_network_by_id(self.__neutron,
197 router['router']['external_gateway_info']['network_id'])
199 return network['network']['name']
204 Destroys the VM instance
207 # Cleanup floating IPs
208 for floating_ip in self.__floating_ips:
210 logger.info('Deleting Floating IP - ' + floating_ip.ip)
211 nova_utils.delete_floating_ip(self.__nova, floating_ip)
212 except Exception as e:
213 logger.error('Error deleting Floating IP - ' + e.message)
214 self.__floating_ips = list()
215 self.__floating_ip_dict = dict()
218 for name, port in self.__ports:
219 logger.info('Deleting Port - ' + name)
221 neutron_utils.delete_port(self.__neutron, port)
222 except PortNotFoundClient as e:
223 logger.warn('Unexpected error deleting port - ' + e.message)
225 self.__ports = list()
230 logger.info('Deleting VM instance - ' + self.instance_settings.name)
231 nova_utils.delete_vm_instance(self.__nova, self.__vm)
232 except Exception as e:
233 logger.error('Error deleting VM - ' + str(e))
235 # Block until instance cannot be found or returns the status of DELETED
236 logger.info('Checking deletion status')
239 if self.vm_deleted(block=True):
240 logger.info('VM has been properly deleted VM with name - ' + self.instance_settings.name)
243 logger.error('VM not deleted within the timeout period of ' +
244 str(self.instance_settings.vm_delete_timeout) + ' seconds')
245 except Exception as e:
246 logger.error('Unexpected error while checking VM instance status - ' + e.message)
248 def __setup_ports(self, port_settings, cleanup):
250 Returns the previously configured ports or creates them if they do not exist
251 :param port_settings: A list of PortSetting objects
252 :param cleanup: When true, only perform lookups for OpenStack objects.
253 :return: a list of OpenStack port tuples where the first member is the port name and the second is the port
258 for port_setting in port_settings:
259 # First check to see if network already has this port
260 # TODO/FIXME - this could potentially cause problems if another port with the same name exists
261 # VM has the same network/port name pair
264 # TODO/FIXME - should we not be iterating on ports for the specific network in question as unique port names
265 # seem to only be important by network
266 existing_ports = self.__neutron.list_ports()['ports']
267 for existing_port in existing_ports:
268 if existing_port['name'] == port_setting.name:
269 ports.append((port_setting.name, {'port': existing_port}))
273 if not found and not cleanup:
274 ports.append((port_setting.name, neutron_utils.create_port(self.__neutron, self.__os_creds,
279 def __add_floating_ip(self, floating_ip, port, subnet, timeout=30, poll_interval=POLL_INTERVAL):
281 Returns True when active else False
282 TODO - Make timeout and poll_interval configurable...
287 # Take IP of subnet if there is one configured on which to place the floating IP
288 for fixed_ip in port['port']['fixed_ips']:
289 if fixed_ip['subnet_id'] == subnet['subnet']['id']:
290 ip = fixed_ip['ip_address']
293 # Simply take the first
294 ip = port['port']['fixed_ips'][0]['ip_address']
297 count = timeout / poll_interval
299 logger.debug('Attempting to add floating IP to instance')
301 self.__vm.add_floating_ip(floating_ip, ip)
302 logger.info('Added floating IP ' + floating_ip.ip + ' to port IP - ' + ip +
303 ' on instance - ' + self.instance_settings.name)
305 except Exception as e:
306 logger.debug('Retry adding floating IP to instance. Last attempt failed with - ' + e.message)
307 time.sleep(poll_interval)
311 raise Exception('Unable find IP address on which to place the floating IP')
313 logger.error('Timeout attempting to add the floating IP to instance.')
314 raise Exception('Timeout while attempting add floating IP to instance')
316 def get_os_creds(self):
318 Returns the OpenStack credentials used to create these objects
319 :return: the credentials
321 return self.__os_creds
323 def get_vm_inst(self):
325 Returns the latest version of this server object from OpenStack
326 :return: Server object
328 return nova_utils.get_latest_server_object(self.__nova, self.__vm)
330 def get_port_ip(self, port_name, subnet_name=None):
332 Returns the first IP for the port corresponding with the port_name parameter when subnet_name is None
333 else returns the IP address that corresponds to the subnet_name parameter
334 :param port_name: the name of the port from which to return the IP
335 :param subnet_name: the name of the subnet attached to this IP
336 :return: the IP or None if not found
338 port = self.get_port_by_name(port_name)
340 port_dict = port['port']
342 subnet = neutron_utils.get_subnet_by_name(self.__neutron, subnet_name)
344 logger.warn('Cannot retrieve port IP as subnet could not be located with name - ' + subnet_name)
346 for fixed_ip in port_dict['fixed_ips']:
347 if fixed_ip['subnet_id'] == subnet['subnet']['id']:
348 return fixed_ip['ip_address']
350 fixed_ips = port_dict['fixed_ips']
351 if fixed_ips and len(fixed_ips) > 0:
352 return fixed_ips[0]['ip_address']
355 def get_port_mac(self, port_name):
357 Returns the first IP for the port corresponding with the port_name parameter
358 TODO - Add in the subnet as an additional parameter as a port may have multiple fixed_ips
359 :param port_name: the name of the port from which to return the IP
360 :return: the IP or None if not found
362 port = self.get_port_by_name(port_name)
364 port_dict = port['port']
365 return port_dict['mac_address']
368 def get_port_by_name(self, port_name):
370 Retrieves the OpenStack port object by its given name
371 :param port_name: the name of the port
372 :return: the OpenStack port object or None if not exists
374 for key, port in self.__ports:
377 logger.warn('Cannot find port with name - ' + port_name)
380 def config_nics(self):
382 Responsible for configuring NICs on RPM systems where the instance has more than one configured port
385 if len(self.__ports) > 1 and len(self.__floating_ips) > 0:
386 if self.vm_active(block=True) and self.vm_ssh_active(block=True):
387 for key, port in self.__ports:
388 port_index = self.__ports.index((key, port))
390 nic_name = 'eth' + repr(port_index)
391 self.__config_nic(nic_name, port, self.__get_first_provisioning_floating_ip().ip)
392 logger.info('Configured NIC - ' + nic_name + ' on VM - ' + self.instance_settings.name)
394 def __get_first_provisioning_floating_ip(self):
396 Returns the first floating IP tagged with the Floating IP name if exists else the first one found
399 for floating_ip_setting in self.instance_settings.floating_ip_settings:
400 if floating_ip_setting.provisioning:
401 fip = self.__floating_ip_dict.get(floating_ip_setting.name)
404 elif len(self.__floating_ips) > 0:
405 return self.__floating_ips[0]
407 def __config_nic(self, nic_name, port, floating_ip):
409 Although ports/NICs can contain multiple IPs, this code currently only supports the first.
411 Your CWD at this point must be the <repo dir>/python directory.
412 TODO - fix this restriction.
414 :param nic_name: Name of the interface
415 :param port: The port information containing the expected IP values.
416 :param floating_ip: The floating IP on which to apply the playbook.
418 ip = port['port']['fixed_ips'][0]['ip_address']
420 'floating_ip': floating_ip,
421 'nic_name': nic_name,
425 if self.image_settings.nic_config_pb_loc and self.keypair_settings:
426 ansible_utils.apply_playbook(self.image_settings.nic_config_pb_loc,
427 [floating_ip], self.get_image_user(), self.keypair_settings.private_filepath,
428 variables, self.__os_creds.proxy_settings)
430 logger.warn('VM ' + self.instance_settings.name + ' cannot self configure NICs eth1++. ' +
431 'No playbook or keypairs found.')
433 def get_image_user(self):
435 Returns the instance sudo_user if it has been configured in the instance_settings else it returns the
436 image_settings.image_user value
438 if self.instance_settings.sudo_user:
439 return self.instance_settings.sudo_user
441 return self.image_settings.image_user
443 def vm_deleted(self, block=False, poll_interval=POLL_INTERVAL):
445 Returns true when the VM status returns the value of expected_status_code or instance retrieval throws
446 a NotFound exception.
447 :param block: When true, thread will block until active or timeout value in seconds has been exceeded (False)
448 :param poll_interval: The polling interval in seconds
452 return self.__vm_status_check(STATUS_DELETED, block, self.instance_settings.vm_delete_timeout,
454 except NotFound as e:
455 logger.debug("Instance not found when querying status for " + STATUS_DELETED + ' with message ' + e.message)
458 def vm_active(self, block=False, poll_interval=POLL_INTERVAL):
460 Returns true when the VM status returns the value of expected_status_code
461 :param block: When true, thread will block until active or timeout value in seconds has been exceeded (False)
462 :param poll_interval: The polling interval in seconds
465 return self.__vm_status_check(STATUS_ACTIVE, block, self.instance_settings.vm_boot_timeout, poll_interval)
467 def __vm_status_check(self, expected_status_code, block, timeout, poll_interval):
469 Returns true when the VM status returns the value of expected_status_code
470 :param expected_status_code: instance status evaluated with this string value
471 :param block: When true, thread will block until active or timeout value in seconds has been exceeded (False)
472 :param timeout: The timeout value
473 :param poll_interval: The polling interval in seconds
476 # sleep and wait for VM status change
480 start = time.time() - timeout
482 while timeout > time.time() - start:
483 status = self.__status(expected_status_code)
485 logger.info('VM is - ' + expected_status_code)
488 logger.debug('Retry querying VM status in ' + str(poll_interval) + ' seconds')
489 time.sleep(poll_interval)
490 logger.debug('VM status query timeout in ' + str(timeout - (time.time() - start)))
492 logger.error('Timeout checking for VM status for ' + expected_status_code)
495 def __status(self, expected_status_code):
497 Returns True when active else False
498 :param expected_status_code: instance status evaluated with this string value
501 instance = self.__nova.servers.get(self.__vm.id)
503 logger.warn('Cannot find instance with id - ' + self.__vm.id)
506 if instance.status == 'ERROR':
507 raise Exception('Instance had an error during deployment')
508 logger.debug('Instance status [' + self.instance_settings.name + '] is - ' + instance.status)
509 return instance.status == expected_status_code
511 def vm_ssh_active(self, block=False, poll_interval=POLL_INTERVAL):
513 Returns true when the VM can be accessed via SSH
514 :param block: When true, thread will block until active or timeout value in seconds has been exceeded (False)
515 :param poll_interval: The polling interval
518 # sleep and wait for VM status change
519 logger.info('Checking if VM is active')
521 timeout = self.instance_settings.ssh_connect_timeout
523 if self.vm_active(block=True):
527 start = time.time() - timeout
529 while timeout > time.time() - start:
530 status = self.__ssh_active()
532 logger.info('SSH is active for VM instance')
535 logger.debug('Retry SSH connection in ' + str(poll_interval) + ' seconds')
536 time.sleep(poll_interval)
537 logger.debug('SSH connection timeout in ' + str(timeout - (time.time() - start)))
539 logger.error('Timeout attempting to connect with VM via SSH')
542 def __ssh_active(self):
544 Returns True when can create a SSH session else False
547 if len(self.__floating_ips) > 0:
548 ssh = self.ssh_client()
553 def get_floating_ip(self, fip_name=None):
555 Returns the floating IP object byt name if found, else the first known, else None
556 :param fip_name: the name of the floating IP to return
557 :return: the SSH client or None
560 if fip_name and self.__floating_ip_dict.get(fip_name):
561 return self.__floating_ip_dict.get(fip_name)
562 if not fip and len(self.__floating_ips) > 0:
563 return self.__floating_ips[0]
566 def ssh_client(self, fip_name=None):
568 Returns an SSH client using the name or the first known floating IP if exists, else None
569 :param fip_name: the name of the floating IP to return
570 :return: the SSH client or None
572 fip = self.get_floating_ip(fip_name)
574 return ansible_utils.ssh_client(self.__floating_ips[0].ip, self.get_image_user(),
575 self.keypair_settings.private_filepath,
576 proxy_settings=self.__os_creds.proxy_settings)
578 logger.warn('Cannot return an SSH client. No Floating IP configured')
580 def add_security_group(self, security_group):
582 Adds a security group to this VM. Call will block until VM is active.
583 :param security_group: the OpenStack security group object
584 :return True if successful else False
586 self.vm_active(block=True)
588 if not security_group:
589 logger.warn('Security group object is None, cannot add')
593 nova_utils.add_security_group(self.__nova, self.get_vm_inst(), security_group['security_group']['name'])
595 except NotFound as e:
596 logger.warn('Security group not added - ' + e.message)
599 def remove_security_group(self, security_group):
601 Removes a security group to this VM. Call will block until VM is active.
602 :param security_group: the OpenStack security group object
603 :return True if successful else False
605 self.vm_active(block=True)
607 if not security_group:
608 logger.warn('Security group object is None, cannot remove')
612 nova_utils.remove_security_group(self.__nova, self.get_vm_inst(), security_group)
614 except NotFound as e:
615 logger.warn('Security group not removed - ' + e.message)
619 class VmInstanceSettings:
621 Class responsible for holding configuration setting for a VM Instance
623 def __init__(self, config=None, name=None, flavor=None, port_settings=list(), security_group_names=set(),
624 floating_ip_settings=list(), sudo_user=None, vm_boot_timeout=900,
625 vm_delete_timeout=300, ssh_connect_timeout=180, availability_zone=None, userdata=None):
628 :param config: dict() object containing the configuration settings using the attribute names below as each
629 member's the key and overrides any of the other parameters.
630 :param name: the name of the VM
631 :param flavor: the VM's flavor
632 :param port_settings: the port configuration settings (required)
633 :param security_group_names: a set of names of the security groups to add to the VM
634 :param floating_ip_settings: the floating IP configuration settings
635 :param sudo_user: the sudo user of the VM that will override the instance_settings.image_user when trying to
637 :param vm_boot_timeout: the amount of time a thread will sleep waiting for an instance to boot
638 :param vm_delete_timeout: the amount of time a thread will sleep waiting for an instance to be deleted
639 :param ssh_connect_timeout: the amount of time a thread will sleep waiting obtaining an SSH connection to a VM
640 :param availability_zone: the name of the compute server on which to deploy the VM (optional)
641 :param userdata: the cloud-init script to run after the VM has been started
644 self.name = config.get('name')
645 self.flavor = config.get('flavor')
646 self.sudo_user = config.get('sudo_user')
647 self.userdata = config.get('userdata')
649 self.port_settings = list()
650 if config.get('ports'):
651 for port_config in config['ports']:
652 if isinstance(port_config, PortSettings):
653 self.port_settings.append(port_config)
655 self.port_settings.append(PortSettings(config=port_config['port']))
657 if config.get('security_group_names'):
658 if isinstance(config['security_group_names'], list):
659 self.security_group_names = set(config['security_group_names'])
660 elif isinstance(config['security_group_names'], set):
661 self.security_group_names = config['security_group_names']
662 elif isinstance(config['security_group_names'], basestring):
663 self.security_group_names = [config['security_group_names']]
665 raise Exception('Invalid data type for security_group_names attribute')
667 self.security_group_names = set()
669 self.floating_ip_settings = list()
670 if config.get('floating_ips'):
671 for floating_ip_config in config['floating_ips']:
672 if isinstance(floating_ip_config, FloatingIpSettings):
673 self.floating_ip_settings.append(floating_ip_config)
675 self.floating_ip_settings.append(FloatingIpSettings(config=floating_ip_config['floating_ip']))
677 if config.get('vm_boot_timeout'):
678 self.vm_boot_timeout = config['vm_boot_timeout']
680 self.vm_boot_timeout = vm_boot_timeout
682 if config.get('vm_delete_timeout'):
683 self.vm_delete_timeout = config['vm_delete_timeout']
685 self.vm_delete_timeout = vm_delete_timeout
687 if config.get('ssh_connect_timeout'):
688 self.ssh_connect_timeout = config['ssh_connect_timeout']
690 self.ssh_connect_timeout = ssh_connect_timeout
692 if config.get('availability_zone'):
693 self.availability_zone = config['availability_zone']
695 self.availability_zone = None
699 self.port_settings = port_settings
700 self.security_group_names = security_group_names
701 self.floating_ip_settings = floating_ip_settings
702 self.sudo_user = sudo_user
703 self.vm_boot_timeout = vm_boot_timeout
704 self.vm_delete_timeout = vm_delete_timeout
705 self.ssh_connect_timeout = ssh_connect_timeout
706 self.availability_zone = availability_zone
707 self.userdata = userdata
709 if not self.name or not self.flavor:
710 raise Exception('Instance configuration requires the attributes: name, flavor')
712 if len(self.port_settings) == 0:
713 raise Exception('Instance configuration requires port settings (aka. NICS)')
716 class FloatingIpSettings:
718 Class responsible for holding configuration settings for a floating IP
720 def __init__(self, config=None, name=None, port_name=None, router_name=None, subnet_name=None, provisioning=True):
723 :param config: dict() object containing the configuration settings using the attribute names below as each
724 member's the key and overrides any of the other parameters.
725 :param name: the name of the floating IP
726 :param port_name: the name of the router to the external network
727 :param router_name: the name of the router to the external network
728 :param subnet_name: the name of the subnet on which to attach the floating IP
729 :param provisioning: when true, this floating IP can be used for provisioning
731 TODO - provisioning flag is a hack as I have only observed a single Floating IPs that actually works on
732 an instance. Multiple floating IPs placed on different subnets from the same port are especially troublesome
733 as you cannot predict which one will actually connect. For now, it is recommended not to setup multiple
734 floating IPs on an instance unless absolutely necessary.
737 self.name = config.get('name')
738 self.port_name = config.get('port_name')
739 self.router_name = config.get('router_name')
740 self.subnet_name = config.get('subnet_name')
741 if config.get('provisioning') is not None:
742 self.provisioning = config['provisioning']
744 self.provisioning = provisioning
747 self.port_name = port_name
748 self.router_name = router_name
749 self.subnet_name = subnet_name
750 self.provisioning = provisioning
752 if not self.name or not self.port_name or not self.router_name:
753 raise Exception('The attributes name, port_name and router_name are required for FloatingIPSettings')