1 # Copyright (c) 2016 Cable Television Laboratories, Inc. ("CableLabs")
2 # and others. All rights reserved.
4 # Licensed under the Apache License, Version 2.0 (the "License");
5 # you may not use this file except in compliance with the License.
6 # You may obtain a copy of the License at:
8 # http://www.apache.org/licenses/LICENSE-2.0
10 # Unless required by applicable law or agreed to in writing, software
11 # distributed under the License is distributed on an "AS IS" BASIS,
12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 # See the License for the specific language governing permissions and
14 # limitations under the License.
18 from neutronclient.common.exceptions import PortNotFoundClient
19 from novaclient.exceptions import NotFound
21 from snaps.openstack.utils import glance_utils
22 from snaps.openstack.utils import neutron_utils
23 from snaps.openstack.create_network import PortSettings
24 from snaps.provisioning import ansible_utils
25 from snaps.openstack.utils import nova_utils
27 __author__ = 'spisarski'
29 logger = logging.getLogger('create_instance')
32 STATUS_ACTIVE = 'ACTIVE'
33 STATUS_DELETED = 'DELETED'
36 class OpenStackVmInstance:
38 Class responsible for creating a VM instance in OpenStack
41 def __init__(self, os_creds, instance_settings, image_settings, keypair_settings=None):
44 :param os_creds: The connection credentials to the OpenStack API
45 :param instance_settings: Contains the settings for this VM
46 :param image_settings: The OpenStack image object settings
47 :param keypair_settings: The keypair metadata (Optional)
50 self.__os_creds = os_creds
52 self.__nova = nova_utils.nova_client(self.__os_creds)
53 self.__neutron = neutron_utils.neutron_client(self.__os_creds)
55 self.instance_settings = instance_settings
56 self.image_settings = image_settings
57 self.keypair_settings = keypair_settings
59 # TODO - get rid of FIP list and only use the dict(). Need to fix populating this object when already exists
60 self.__floating_ips = list()
61 self.__floating_ip_dict = dict()
63 # Instantiated in self.create()
66 # Note: this object does not change after the VM becomes active
69 def create(self, cleanup=False, block=False):
72 :param cleanup: When true, only perform lookups for OpenStack objects.
73 :param block: Thread will block until instance has either become active, error, or timeout waiting.
74 Additionally, when True, floating IPs will not be applied until VM is active.
75 :return: The VM reference object
78 self.__ports = self.__setup_ports(self.instance_settings.port_settings, cleanup)
79 self.__lookup_existing_vm_by_name()
80 if not self.__vm and not cleanup:
81 self.__create_vm(block)
83 except Exception as e:
84 logger.exception('Error occurred while setting up instance')
88 def __lookup_existing_vm_by_name(self):
90 Populates the member variables 'self.vm' and 'self.floating_ips' if a VM with the same name already exists
93 servers = nova_utils.get_servers_by_name(self.__nova, self.instance_settings.name)
94 for server in servers:
95 if server.name == self.instance_settings.name:
97 logger.info('Found existing machine with name - ' + self.instance_settings.name)
98 fips = self.__nova.floating_ips.list()
100 if fip.instance_id == server.id:
101 self.__floating_ips.append(fip)
102 # TODO - Determine a means to associate to the FIP configuration and add to FIP map
104 def __create_vm(self, block=False):
106 Responsible for creating the VM instance
107 :param block: Thread will block until instance has either become active, error, or timeout waiting.
108 Floating IPs will be assigned after active when block=True
111 for key, port in self.__ports:
113 kv['port-id'] = port['port']['id']
116 logger.info('Creating VM with name - ' + self.instance_settings.name)
118 if self.keypair_settings:
119 keypair_name = self.keypair_settings.name
121 flavor = nova_utils.get_flavor_by_name(self.__nova, self.instance_settings.flavor)
123 raise Exception('Flavor not found with name - ' + self.instance_settings.flavor)
125 image = glance_utils.get_image(self.__nova, glance_utils.glance_client(self.__os_creds),
126 self.image_settings.name)
128 self.__vm = self.__nova.servers.create(
129 name=self.instance_settings.name,
133 key_name=keypair_name,
134 security_groups=list(self.instance_settings.security_group_names),
135 userdata=self.instance_settings.userdata,
136 availability_zone=self.instance_settings.availability_zone)
138 raise Exception('Cannot create instance, image cannot be located with name ' + self.image_settings.name)
140 logger.info('Created instance with name - ' + self.instance_settings.name)
143 self.vm_active(block=True)
145 self.__apply_floating_ips()
147 def __apply_floating_ips(self):
149 Applies the configured floating IPs to the necessary ports
152 for key, port in self.__ports:
153 port_dict[key] = port
156 for floating_ip_setting in self.instance_settings.floating_ip_settings:
157 port = port_dict.get(floating_ip_setting.port_name)
160 raise Exception('Cannot find port object with name - ' + floating_ip_setting.port_name)
162 # Setup Floating IP only if there is a router with an external gateway
163 ext_gateway = self.__ext_gateway_by_router(floating_ip_setting.router_name)
165 subnet = neutron_utils.get_subnet_by_name(self.__neutron, floating_ip_setting.subnet_name)
166 floating_ip = nova_utils.create_floating_ip(self.__nova, ext_gateway)
167 self.__floating_ips.append(floating_ip)
168 self.__floating_ip_dict[floating_ip_setting.name] = floating_ip
170 logger.info('Created floating IP ' + floating_ip.ip + ' via router - ' +
171 floating_ip_setting.router_name)
172 self.__add_floating_ip(floating_ip, port, subnet)
174 raise Exception('Unable to add floating IP to port,' +
175 ' cannot locate router with an external gateway ')
177 def __ext_gateway_by_router(self, router_name):
179 Returns network name for the external network attached to a router or None if not found
180 :param router_name: The name of the router to lookup
181 :return: the external network name or None
183 router = neutron_utils.get_router_by_name(self.__neutron, router_name)
184 if router and router['router'].get('external_gateway_info'):
185 network = neutron_utils.get_network_by_id(self.__neutron,
186 router['router']['external_gateway_info']['network_id'])
188 return network['network']['name']
193 Destroys the VM instance
196 # Cleanup floating IPs
197 for floating_ip in self.__floating_ips:
199 logger.info('Deleting Floating IP - ' + floating_ip.ip)
200 nova_utils.delete_floating_ip(self.__nova, floating_ip)
201 except Exception as e:
202 logger.error('Error deleting Floating IP - ' + e.message)
203 self.__floating_ips = list()
204 self.__floating_ip_dict = dict()
207 for name, port in self.__ports:
208 logger.info('Deleting Port - ' + name)
210 neutron_utils.delete_port(self.__neutron, port)
211 except PortNotFoundClient as e:
212 logger.warn('Unexpected error deleting port - ' + e.message)
214 self.__ports = list()
219 logger.info('Deleting VM instance - ' + self.instance_settings.name)
220 nova_utils.delete_vm_instance(self.__nova, self.__vm)
221 except Exception as e:
222 logger.error('Error deleting VM - ' + str(e))
224 # Block until instance cannot be found or returns the status of DELETED
225 logger.info('Checking deletion status')
228 if self.vm_deleted(block=True):
229 logger.info('VM has been properly deleted VM with name - ' + self.instance_settings.name)
232 logger.error('VM not deleted within the timeout period of ' +
233 str(self.instance_settings.vm_delete_timeout) + ' seconds')
234 except Exception as e:
235 logger.error('Unexpected error while checking VM instance status - ' + e.message)
237 def __setup_ports(self, port_settings, cleanup):
239 Returns the previously configured ports or creates them if they do not exist
240 :param port_settings: A list of PortSetting objects
241 :param cleanup: When true, only perform lookups for OpenStack objects.
242 :return: a list of OpenStack port tuples where the first member is the port name and the second is the port
247 for port_setting in port_settings:
248 # First check to see if network already has this port
249 # TODO/FIXME - this could potentially cause problems if another port with the same name exists
250 # VM has the same network/port name pair
253 # TODO/FIXME - should we not be iterating on ports for the specific network in question as unique port names
254 # seem to only be important by network
255 existing_ports = self.__neutron.list_ports()['ports']
256 for existing_port in existing_ports:
257 if existing_port['name'] == port_setting.name:
258 ports.append((port_setting.name, {'port': existing_port}))
262 if not found and not cleanup:
263 ports.append((port_setting.name, neutron_utils.create_port(self.__neutron, self.__os_creds,
268 def __add_floating_ip(self, floating_ip, port, subnet, timeout=30, poll_interval=POLL_INTERVAL):
270 Returns True when active else False
271 TODO - Make timeout and poll_interval configurable...
276 # Take IP of subnet if there is one configured on which to place the floating IP
277 for fixed_ip in port['port']['fixed_ips']:
278 if fixed_ip['subnet_id'] == subnet['subnet']['id']:
279 ip = fixed_ip['ip_address']
282 # Simply take the first
283 ip = port['port']['fixed_ips'][0]['ip_address']
286 count = timeout / poll_interval
288 logger.debug('Attempting to add floating IP to instance')
290 self.__vm.add_floating_ip(floating_ip, ip)
291 logger.info('Added floating IP ' + floating_ip.ip + ' to port IP - ' + ip +
292 ' on instance - ' + self.instance_settings.name)
294 except Exception as e:
295 logger.debug('Retry adding floating IP to instance. Last attempt failed with - ' + e.message)
296 time.sleep(poll_interval)
300 raise Exception('Unable find IP address on which to place the floating IP')
302 logger.error('Timeout attempting to add the floating IP to instance.')
303 raise Exception('Timeout while attempting add floating IP to instance')
305 def get_os_creds(self):
307 Returns the OpenStack credentials used to create these objects
308 :return: the credentials
310 return self.__os_creds
312 def get_vm_inst(self):
314 Returns the latest version of this server object from OpenStack
315 :return: Server object
317 return nova_utils.get_latest_server_object(self.__nova, self.__vm)
319 def get_port_ip(self, port_name, subnet_name=None):
321 Returns the first IP for the port corresponding with the port_name parameter when subnet_name is None
322 else returns the IP address that corresponds to the subnet_name parameter
323 :param port_name: the name of the port from which to return the IP
324 :param subnet_name: the name of the subnet attached to this IP
325 :return: the IP or None if not found
327 port = self.get_port_by_name(port_name)
329 port_dict = port['port']
331 subnet = neutron_utils.get_subnet_by_name(self.__neutron, subnet_name)
333 logger.warn('Cannot retrieve port IP as subnet could not be located with name - ' + subnet_name)
335 for fixed_ip in port_dict['fixed_ips']:
336 if fixed_ip['subnet_id'] == subnet['subnet']['id']:
337 return fixed_ip['ip_address']
339 fixed_ips = port_dict['fixed_ips']
340 if fixed_ips and len(fixed_ips) > 0:
341 return fixed_ips[0]['ip_address']
344 def get_port_mac(self, port_name):
346 Returns the first IP for the port corresponding with the port_name parameter
347 TODO - Add in the subnet as an additional parameter as a port may have multiple fixed_ips
348 :param port_name: the name of the port from which to return the IP
349 :return: the IP or None if not found
351 port = self.get_port_by_name(port_name)
353 port_dict = port['port']
354 return port_dict['mac_address']
357 def get_port_by_name(self, port_name):
359 Retrieves the OpenStack port object by its given name
360 :param port_name: the name of the port
361 :return: the OpenStack port object or None if not exists
363 for key, port in self.__ports:
366 logger.warn('Cannot find port with name - ' + port_name)
369 def config_nics(self):
371 Responsible for configuring NICs on RPM systems where the instance has more than one configured port
374 if len(self.__ports) > 1 and len(self.__floating_ips) > 0:
375 if self.vm_active(block=True) and self.vm_ssh_active(block=True):
376 for key, port in self.__ports:
377 port_index = self.__ports.index((key, port))
379 nic_name = 'eth' + repr(port_index)
380 self.__config_nic(nic_name, port, self.__get_first_provisioning_floating_ip().ip)
381 logger.info('Configured NIC - ' + nic_name + ' on VM - ' + self.instance_settings.name)
383 def __get_first_provisioning_floating_ip(self):
385 Returns the first floating IP tagged with the Floating IP name if exists else the first one found
388 for floating_ip_setting in self.instance_settings.floating_ip_settings:
389 if floating_ip_setting.provisioning:
390 fip = self.__floating_ip_dict.get(floating_ip_setting.name)
393 elif len(self.__floating_ips) > 0:
394 return self.__floating_ips[0]
396 def __config_nic(self, nic_name, port, floating_ip):
398 Although ports/NICs can contain multiple IPs, this code currently only supports the first.
400 Your CWD at this point must be the <repo dir>/python directory.
401 TODO - fix this restriction.
403 :param nic_name: Name of the interface
404 :param port: The port information containing the expected IP values.
405 :param floating_ip: The floating IP on which to apply the playbook.
407 ip = port['port']['fixed_ips'][0]['ip_address']
409 'floating_ip': floating_ip,
410 'nic_name': nic_name,
414 if self.image_settings.nic_config_pb_loc and self.keypair_settings:
415 ansible_utils.apply_playbook(self.image_settings.nic_config_pb_loc,
416 [floating_ip], self.get_image_user(), self.keypair_settings.private_filepath,
417 variables, self.__os_creds.proxy_settings)
419 logger.warn('VM ' + self.instance_settings.name + ' cannot self configure NICs eth1++. ' +
420 'No playbook or keypairs found.')
422 def get_image_user(self):
424 Returns the instance sudo_user if it has been configured in the instance_settings else it returns the
425 image_settings.image_user value
427 if self.instance_settings.sudo_user:
428 return self.instance_settings.sudo_user
430 return self.image_settings.image_user
432 def vm_deleted(self, block=False, poll_interval=POLL_INTERVAL):
434 Returns true when the VM status returns the value of expected_status_code or instance retrieval throws
435 a NotFound exception.
436 :param block: When true, thread will block until active or timeout value in seconds has been exceeded (False)
437 :param poll_interval: The polling interval in seconds
441 return self.__vm_status_check(STATUS_DELETED, block, self.instance_settings.vm_delete_timeout,
443 except NotFound as e:
444 logger.debug("Instance not found when querying status for " + STATUS_DELETED + ' with message ' + e.message)
447 def vm_active(self, block=False, poll_interval=POLL_INTERVAL):
449 Returns true when the VM status returns the value of expected_status_code
450 :param block: When true, thread will block until active or timeout value in seconds has been exceeded (False)
451 :param poll_interval: The polling interval in seconds
454 return self.__vm_status_check(STATUS_ACTIVE, block, self.instance_settings.vm_boot_timeout, poll_interval)
456 def __vm_status_check(self, expected_status_code, block, timeout, poll_interval):
458 Returns true when the VM status returns the value of expected_status_code
459 :param expected_status_code: instance status evaluated with this string value
460 :param block: When true, thread will block until active or timeout value in seconds has been exceeded (False)
461 :param timeout: The timeout value
462 :param poll_interval: The polling interval in seconds
465 # sleep and wait for VM status change
469 start = time.time() - timeout
471 while timeout > time.time() - start:
472 status = self.__status(expected_status_code)
474 logger.info('VM is - ' + expected_status_code)
477 logger.debug('Retry querying VM status in ' + str(poll_interval) + ' seconds')
478 time.sleep(poll_interval)
479 logger.debug('VM status query timeout in ' + str(timeout - (time.time() - start)))
481 logger.error('Timeout checking for VM status for ' + expected_status_code)
484 def __status(self, expected_status_code):
486 Returns True when active else False
487 :param expected_status_code: instance status evaluated with this string value
490 instance = self.__nova.servers.get(self.__vm.id)
492 logger.warn('Cannot find instance with id - ' + self.__vm.id)
495 if instance.status == 'ERROR':
496 raise Exception('Instance had an error during deployment')
497 logger.debug('Instance status [' + self.instance_settings.name + '] is - ' + instance.status)
498 return instance.status == expected_status_code
500 def vm_ssh_active(self, block=False, poll_interval=POLL_INTERVAL):
502 Returns true when the VM can be accessed via SSH
503 :param block: When true, thread will block until active or timeout value in seconds has been exceeded (False)
504 :param poll_interval: The polling interval
507 # sleep and wait for VM status change
508 logger.info('Checking if VM is active')
510 timeout = self.instance_settings.ssh_connect_timeout
512 if self.vm_active(block=True):
516 start = time.time() - timeout
518 while timeout > time.time() - start:
519 status = self.__ssh_active()
521 logger.info('SSH is active for VM instance')
524 logger.debug('Retry SSH connection in ' + str(poll_interval) + ' seconds')
525 time.sleep(poll_interval)
526 logger.debug('SSH connection timeout in ' + str(timeout - (time.time() - start)))
528 logger.error('Timeout attempting to connect with VM via SSH')
531 def __ssh_active(self):
533 Returns True when can create a SSH session else False
536 if len(self.__floating_ips) > 0:
537 ssh = self.ssh_client()
542 def get_floating_ip(self, fip_name=None):
544 Returns the floating IP object byt name if found, else the first known, else None
545 :param fip_name: the name of the floating IP to return
546 :return: the SSH client or None
549 if fip_name and self.__floating_ip_dict.get(fip_name):
550 return self.__floating_ip_dict.get(fip_name)
551 if not fip and len(self.__floating_ips) > 0:
552 return self.__floating_ips[0]
555 def ssh_client(self, fip_name=None):
557 Returns an SSH client using the name or the first known floating IP if exists, else None
558 :param fip_name: the name of the floating IP to return
559 :return: the SSH client or None
561 fip = self.get_floating_ip(fip_name)
563 return ansible_utils.ssh_client(self.__floating_ips[0].ip, self.get_image_user(),
564 self.keypair_settings.private_filepath,
565 proxy_settings=self.__os_creds.proxy_settings)
567 logger.warn('Cannot return an SSH client. No Floating IP configured')
569 def add_security_group(self, security_group):
571 Adds a security group to this VM. Call will block until VM is active.
572 :param security_group: the OpenStack security group object
573 :return True if successful else False
575 self.vm_active(block=True)
577 if not security_group:
578 logger.warn('Security group object is None, cannot add')
582 nova_utils.add_security_group(self.__nova, self.get_vm_inst(), security_group['security_group']['name'])
584 except NotFound as e:
585 logger.warn('Security group not added - ' + e.message)
588 def remove_security_group(self, security_group):
590 Removes a security group to this VM. Call will block until VM is active.
591 :param security_group: the OpenStack security group object
592 :return True if successful else False
594 self.vm_active(block=True)
596 if not security_group:
597 logger.warn('Security group object is None, cannot add')
601 nova_utils.remove_security_group(self.__nova, self.get_vm_inst(), security_group['security_group']['name'])
603 except NotFound as e:
604 logger.warn('Security group not added - ' + e.message)
608 class VmInstanceSettings:
610 Class responsible for holding configuration setting for a VM Instance
612 def __init__(self, config=None, name=None, flavor=None, port_settings=list(), security_group_names=set(),
613 floating_ip_settings=list(), sudo_user=None, vm_boot_timeout=900,
614 vm_delete_timeout=300, ssh_connect_timeout=180, availability_zone=None, userdata=None):
617 :param config: dict() object containing the configuration settings using the attribute names below as each
618 member's the key and overrides any of the other parameters.
619 :param name: the name of the VM
620 :param flavor: the VM's flavor
621 :param port_settings: the port configuration settings
622 :param security_group_names: a set of names of the security groups to add to the VM
623 :param floating_ip_settings: the floating IP configuration settings
624 :param sudo_user: the sudo user of the VM that will override the instance_settings.image_user when trying to
626 :param vm_boot_timeout: the amount of time a thread will sleep waiting for an instance to boot
627 :param vm_delete_timeout: the amount of time a thread will sleep waiting for an instance to be deleted
628 :param ssh_connect_timeout: the amount of time a thread will sleep waiting obtaining an SSH connection to a VM
629 :param availability_zone: the name of the compute server on which to deploy the VM (optional)
630 :param userdata: the cloud-init script to run after the VM has been started
633 self.name = config.get('name')
634 self.flavor = config.get('flavor')
635 self.sudo_user = config.get('sudo_user')
636 self.userdata = config.get('userdata')
638 self.port_settings = list()
639 if config.get('ports'):
640 for port_config in config['ports']:
641 if isinstance(port_config, PortSettings):
642 self.port_settings.append(port_config)
644 self.port_settings.append(PortSettings(config=port_config['port']))
646 if config.get('security_group_names'):
647 if isinstance(config['security_group_names'], list):
648 self.security_group_names = set(config['security_group_names'])
649 elif isinstance(config['security_group_names'], set):
650 self.security_group_names = config['security_group_names']
651 elif isinstance(config['security_group_names'], basestring):
652 self.security_group_names = [config['security_group_names']]
654 raise Exception('Invalid data type for security_group_names attribute')
656 self.security_group_names = set()
658 self.floating_ip_settings = list()
659 if config.get('floating_ips'):
660 for floating_ip_config in config['floating_ips']:
661 if isinstance(floating_ip_config, FloatingIpSettings):
662 self.floating_ip_settings.append(floating_ip_config)
664 self.floating_ip_settings.append(FloatingIpSettings(config=floating_ip_config['floating_ip']))
666 if config.get('vm_boot_timeout'):
667 self.vm_boot_timeout = config['vm_boot_timeout']
669 self.vm_boot_timeout = vm_boot_timeout
671 if config.get('vm_delete_timeout'):
672 self.vm_delete_timeout = config['vm_delete_timeout']
674 self.vm_delete_timeout = vm_delete_timeout
676 if config.get('ssh_connect_timeout'):
677 self.ssh_connect_timeout = config['ssh_connect_timeout']
679 self.ssh_connect_timeout = ssh_connect_timeout
681 if config.get('availability_zone'):
682 self.availability_zone = config['availability_zone']
684 self.availability_zone = None
688 self.port_settings = port_settings
689 self.security_group_names = security_group_names
690 self.floating_ip_settings = floating_ip_settings
691 self.sudo_user = sudo_user
692 self.vm_boot_timeout = vm_boot_timeout
693 self.vm_delete_timeout = vm_delete_timeout
694 self.ssh_connect_timeout = ssh_connect_timeout
695 self.availability_zone = availability_zone
696 self.userdata = userdata
698 if not self.name or not self.flavor:
699 raise Exception('Instance configuration requires the attributes: name, flavor')
702 class FloatingIpSettings:
704 Class responsible for holding configuration settings for a floating IP
706 def __init__(self, config=None, name=None, port_name=None, router_name=None, subnet_name=None, provisioning=True):
709 :param config: dict() object containing the configuration settings using the attribute names below as each
710 member's the key and overrides any of the other parameters.
711 :param name: the name of the floating IP
712 :param port_name: the name of the router to the external network
713 :param router_name: the name of the router to the external network
714 :param subnet_name: the name of the subnet on which to attach the floating IP
715 :param provisioning: when true, this floating IP can be used for provisioning
717 TODO - provisioning flag is a hack as I have only observed a single Floating IPs that actually works on
718 an instance. Multiple floating IPs placed on different subnets from the same port are especially troublesome
719 as you cannot predict which one will actually connect. For now, it is recommended not to setup multiple
720 floating IPs on an instance unless absolutely necessary.
723 self.name = config.get('name')
724 self.port_name = config.get('port_name')
725 self.router_name = config.get('router_name')
726 self.subnet_name = config.get('subnet_name')
727 if config.get('provisioning') is not None:
728 self.provisioning = config['provisioning']
730 self.provisioning = provisioning
733 self.port_name = port_name
734 self.router_name = router_name
735 self.subnet_name = subnet_name
736 self.provisioning = provisioning
738 if not self.name or not self.port_name or not self.router_name:
739 raise Exception('The attributes name, port_name and router_name are required for FloatingIPSettings')