1 # Copyright (c) 2017 Cable Television Laboratories, Inc. ("CableLabs")
2 # and others. All rights reserved.
4 # Licensed under the Apache License, Version 2.0 (the "License");
5 # you may not use this file except in compliance with the License.
6 # You may obtain a copy of the License at:
8 # http://www.apache.org/licenses/LICENSE-2.0
10 # Unless required by applicable law or agreed to in writing, software
11 # distributed under the License is distributed on an "AS IS" BASIS,
12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 # See the License for the specific language governing permissions and
14 # limitations under the License.
18 from neutronclient.common.exceptions import PortNotFoundClient
19 from novaclient.exceptions import NotFound
21 from snaps.openstack.utils import glance_utils
22 from snaps.openstack.utils import neutron_utils
23 from snaps.openstack.create_network import PortSettings
24 from snaps.provisioning import ansible_utils
25 from snaps.openstack.utils import nova_utils
27 __author__ = 'spisarski'
29 logger = logging.getLogger('create_instance')
32 STATUS_ACTIVE = 'ACTIVE'
33 STATUS_DELETED = 'DELETED'
36 class OpenStackVmInstance:
38 Class responsible for creating a VM instance in OpenStack
41 def __init__(self, os_creds, instance_settings, image_settings, keypair_settings=None):
44 :param os_creds: The connection credentials to the OpenStack API
45 :param instance_settings: Contains the settings for this VM
46 :param image_settings: The OpenStack image object settings
47 :param keypair_settings: The keypair metadata (Optional)
50 self.__os_creds = os_creds
55 self.instance_settings = instance_settings
56 self.image_settings = image_settings
57 self.keypair_settings = keypair_settings
59 # TODO - get rid of FIP list and only use the dict(). Need to fix populating this object when already exists
60 self.__floating_ips = list()
61 self.__floating_ip_dict = dict()
63 # Instantiated in self.create()
66 # Note: this object does not change after the VM becomes active
69 def create(self, cleanup=False, block=False):
72 :param cleanup: When true, only perform lookups for OpenStack objects.
73 :param block: Thread will block until instance has either become active, error, or timeout waiting.
74 Additionally, when True, floating IPs will not be applied until VM is active.
75 :return: The VM reference object
77 self.__nova = nova_utils.nova_client(self.__os_creds)
78 self.__neutron = neutron_utils.neutron_client(self.__os_creds)
80 self.__ports = self.__setup_ports(self.instance_settings.port_settings, cleanup)
81 self.__lookup_existing_vm_by_name()
82 if not self.__vm and not cleanup:
83 self.__create_vm(block)
86 def __lookup_existing_vm_by_name(self):
88 Populates the member variables 'self.vm' and 'self.floating_ips' if a VM with the same name already exists
91 servers = nova_utils.get_servers_by_name(self.__nova, self.instance_settings.name)
92 for server in servers:
93 if server.name == self.instance_settings.name:
95 logger.info('Found existing machine with name - ' + self.instance_settings.name)
96 fips = self.__nova.floating_ips.list()
98 if fip.instance_id == server.id:
99 self.__floating_ips.append(fip)
100 # TODO - Determine a means to associate to the FIP configuration and add to FIP map
102 def __create_vm(self, block=False):
104 Responsible for creating the VM instance
105 :param block: Thread will block until instance has either become active, error, or timeout waiting.
106 Floating IPs will be assigned after active when block=True
109 for key, port in self.__ports:
111 kv['port-id'] = port['port']['id']
114 logger.info('Creating VM with name - ' + self.instance_settings.name)
116 if self.keypair_settings:
117 keypair_name = self.keypair_settings.name
119 flavor = nova_utils.get_flavor_by_name(self.__nova, self.instance_settings.flavor)
121 raise Exception('Flavor not found with name - ' + self.instance_settings.flavor)
123 image = glance_utils.get_image(glance_utils.glance_client(self.__os_creds),
124 self.image_settings.name)
126 self.__vm = self.__nova.servers.create(
127 name=self.instance_settings.name,
131 key_name=keypair_name,
132 security_groups=self.instance_settings.security_group_names,
133 userdata=self.instance_settings.userdata,
134 availability_zone=self.instance_settings.availability_zone)
137 raise Exception('Cannot create instance, image cannot be located with name ' + self.image_settings.name)
139 logger.info('Created instance with name - ' + self.instance_settings.name)
142 if not self.vm_active(block=True):
143 raise Exception('Fatal error, VM did not become ACTIVE within the alloted time')
145 # TODO - the call above should add security groups. The return object shows they exist but the association
146 # had never been made by OpenStack. This call is here to ensure they have been added
147 for sec_grp_name in self.instance_settings.security_group_names:
148 if self.vm_active(block=True):
149 nova_utils.add_security_group(self.__nova, self.__vm, sec_grp_name)
151 raise Exception('Cannot applying security group with name ' + sec_grp_name +
152 ' to VM that did not activate with name - ' + self.instance_settings.name)
154 self.__apply_floating_ips()
156 def __apply_floating_ips(self):
158 Applies the configured floating IPs to the necessary ports
161 for key, port in self.__ports:
162 port_dict[key] = port
165 for floating_ip_setting in self.instance_settings.floating_ip_settings:
166 port = port_dict.get(floating_ip_setting.port_name)
169 raise Exception('Cannot find port object with name - ' + floating_ip_setting.port_name)
171 # Setup Floating IP only if there is a router with an external gateway
172 ext_gateway = self.__ext_gateway_by_router(floating_ip_setting.router_name)
174 subnet = neutron_utils.get_subnet_by_name(self.__neutron, floating_ip_setting.subnet_name)
175 floating_ip = nova_utils.create_floating_ip(self.__nova, ext_gateway)
176 self.__floating_ips.append(floating_ip)
177 self.__floating_ip_dict[floating_ip_setting.name] = floating_ip
179 logger.info('Created floating IP ' + floating_ip.ip + ' via router - ' +
180 floating_ip_setting.router_name)
181 self.__add_floating_ip(floating_ip, port, subnet)
183 raise Exception('Unable to add floating IP to port,' +
184 ' cannot locate router with an external gateway ')
186 def __ext_gateway_by_router(self, router_name):
188 Returns network name for the external network attached to a router or None if not found
189 :param router_name: The name of the router to lookup
190 :return: the external network name or None
192 router = neutron_utils.get_router_by_name(self.__neutron, router_name)
193 if router and router['router'].get('external_gateway_info'):
194 network = neutron_utils.get_network_by_id(self.__neutron,
195 router['router']['external_gateway_info']['network_id'])
197 return network['network']['name']
202 Destroys the VM instance
205 # Cleanup floating IPs
206 for floating_ip in self.__floating_ips:
208 logger.info('Deleting Floating IP - ' + floating_ip.ip)
209 nova_utils.delete_floating_ip(self.__nova, floating_ip)
210 except Exception as e:
211 logger.error('Error deleting Floating IP - ' + str(e))
212 self.__floating_ips = list()
213 self.__floating_ip_dict = dict()
216 for name, port in self.__ports:
217 logger.info('Deleting Port - ' + name)
219 neutron_utils.delete_port(self.__neutron, port)
220 except PortNotFoundClient as e:
221 logger.warning('Unexpected error deleting port - ' + str(e))
223 self.__ports = list()
228 logger.info('Deleting VM instance - ' + self.instance_settings.name)
229 nova_utils.delete_vm_instance(self.__nova, self.__vm)
230 except Exception as e:
231 logger.error('Error deleting VM - ' + str(e))
233 # Block until instance cannot be found or returns the status of DELETED
234 logger.info('Checking deletion status')
237 if self.vm_deleted(block=True):
238 logger.info('VM has been properly deleted VM with name - ' + self.instance_settings.name)
241 logger.error('VM not deleted within the timeout period of ' +
242 str(self.instance_settings.vm_delete_timeout) + ' seconds')
243 except Exception as e:
244 logger.error('Unexpected error while checking VM instance status - ' + str(e))
246 def __setup_ports(self, port_settings, cleanup):
248 Returns the previously configured ports or creates them if they do not exist
249 :param port_settings: A list of PortSetting objects
250 :param cleanup: When true, only perform lookups for OpenStack objects.
251 :return: a list of OpenStack port tuples where the first member is the port name and the second is the port
256 for port_setting in port_settings:
257 # First check to see if network already has this port
258 # TODO/FIXME - this could potentially cause problems if another port with the same name exists
259 # VM has the same network/port name pair
262 # TODO/FIXME - should we not be iterating on ports for the specific network in question as unique port names
263 # seem to only be important by network
264 existing_ports = self.__neutron.list_ports()['ports']
265 for existing_port in existing_ports:
266 if existing_port['name'] == port_setting.name:
267 ports.append((port_setting.name, {'port': existing_port}))
271 if not found and not cleanup:
272 ports.append((port_setting.name, neutron_utils.create_port(self.__neutron, self.__os_creds,
277 def __add_floating_ip(self, floating_ip, port, subnet, timeout=30, poll_interval=POLL_INTERVAL):
279 Returns True when active else False
280 TODO - Make timeout and poll_interval configurable...
285 # Take IP of subnet if there is one configured on which to place the floating IP
286 for fixed_ip in port['port']['fixed_ips']:
287 if fixed_ip['subnet_id'] == subnet['subnet']['id']:
288 ip = fixed_ip['ip_address']
291 # Simply take the first
292 ip = port['port']['fixed_ips'][0]['ip_address']
295 count = timeout / poll_interval
297 logger.debug('Attempting to add floating IP to instance')
299 self.__vm.add_floating_ip(floating_ip, ip)
300 logger.info('Added floating IP ' + floating_ip.ip + ' to port IP - ' + ip +
301 ' on instance - ' + self.instance_settings.name)
303 except Exception as e:
304 logger.debug('Retry adding floating IP to instance. Last attempt failed with - ' + str(e))
305 time.sleep(poll_interval)
309 raise Exception('Unable find IP address on which to place the floating IP')
311 logger.error('Timeout attempting to add the floating IP to instance.')
312 raise Exception('Timeout while attempting add floating IP to instance')
314 def get_os_creds(self):
316 Returns the OpenStack credentials used to create these objects
317 :return: the credentials
319 return self.__os_creds
321 def get_vm_inst(self):
323 Returns the latest version of this server object from OpenStack
324 :return: Server object
326 return nova_utils.get_latest_server_object(self.__nova, self.__vm)
328 def get_port_ip(self, port_name, subnet_name=None):
330 Returns the first IP for the port corresponding with the port_name parameter when subnet_name is None
331 else returns the IP address that corresponds to the subnet_name parameter
332 :param port_name: the name of the port from which to return the IP
333 :param subnet_name: the name of the subnet attached to this IP
334 :return: the IP or None if not found
336 port = self.get_port_by_name(port_name)
338 port_dict = port['port']
340 subnet = neutron_utils.get_subnet_by_name(self.__neutron, subnet_name)
342 logger.warning('Cannot retrieve port IP as subnet could not be located with name - ' + subnet_name)
344 for fixed_ip in port_dict['fixed_ips']:
345 if fixed_ip['subnet_id'] == subnet['subnet']['id']:
346 return fixed_ip['ip_address']
348 fixed_ips = port_dict['fixed_ips']
349 if fixed_ips and len(fixed_ips) > 0:
350 return fixed_ips[0]['ip_address']
353 def get_port_mac(self, port_name):
355 Returns the first IP for the port corresponding with the port_name parameter
356 TODO - Add in the subnet as an additional parameter as a port may have multiple fixed_ips
357 :param port_name: the name of the port from which to return the IP
358 :return: the IP or None if not found
360 port = self.get_port_by_name(port_name)
362 port_dict = port['port']
363 return port_dict['mac_address']
366 def get_port_by_name(self, port_name):
368 Retrieves the OpenStack port object by its given name
369 :param port_name: the name of the port
370 :return: the OpenStack port object or None if not exists
372 for key, port in self.__ports:
375 logger.warning('Cannot find port with name - ' + port_name)
378 def config_nics(self):
380 Responsible for configuring NICs on RPM systems where the instance has more than one configured port
383 if len(self.__ports) > 1 and len(self.__floating_ips) > 0:
384 if self.vm_active(block=True) and self.vm_ssh_active(block=True):
385 for key, port in self.__ports:
386 port_index = self.__ports.index((key, port))
388 nic_name = 'eth' + repr(port_index)
389 self.__config_nic(nic_name, port, self.__get_first_provisioning_floating_ip().ip)
390 logger.info('Configured NIC - ' + nic_name + ' on VM - ' + self.instance_settings.name)
392 def __get_first_provisioning_floating_ip(self):
394 Returns the first floating IP tagged with the Floating IP name if exists else the first one found
397 for floating_ip_setting in self.instance_settings.floating_ip_settings:
398 if floating_ip_setting.provisioning:
399 fip = self.__floating_ip_dict.get(floating_ip_setting.name)
402 elif len(self.__floating_ips) > 0:
403 return self.__floating_ips[0]
405 def __config_nic(self, nic_name, port, floating_ip):
407 Although ports/NICs can contain multiple IPs, this code currently only supports the first.
409 Your CWD at this point must be the <repo dir>/python directory.
410 TODO - fix this restriction.
412 :param nic_name: Name of the interface
413 :param port: The port information containing the expected IP values.
414 :param floating_ip: The floating IP on which to apply the playbook.
416 ip = port['port']['fixed_ips'][0]['ip_address']
418 'floating_ip': floating_ip,
419 'nic_name': nic_name,
423 if self.image_settings.nic_config_pb_loc and self.keypair_settings:
424 ansible_utils.apply_playbook(self.image_settings.nic_config_pb_loc,
425 [floating_ip], self.get_image_user(), self.keypair_settings.private_filepath,
426 variables, self.__os_creds.proxy_settings)
428 logger.warning('VM ' + self.instance_settings.name + ' cannot self configure NICs eth1++. ' +
429 'No playbook or keypairs found.')
431 def get_image_user(self):
433 Returns the instance sudo_user if it has been configured in the instance_settings else it returns the
434 image_settings.image_user value
436 if self.instance_settings.sudo_user:
437 return self.instance_settings.sudo_user
439 return self.image_settings.image_user
441 def vm_deleted(self, block=False, poll_interval=POLL_INTERVAL):
443 Returns true when the VM status returns the value of expected_status_code or instance retrieval throws
444 a NotFound exception.
445 :param block: When true, thread will block until active or timeout value in seconds has been exceeded (False)
446 :param poll_interval: The polling interval in seconds
450 return self.__vm_status_check(STATUS_DELETED, block, self.instance_settings.vm_delete_timeout,
452 except NotFound as e:
453 logger.debug("Instance not found when querying status for " + STATUS_DELETED + ' with message ' + str(e))
456 def vm_active(self, block=False, poll_interval=POLL_INTERVAL):
458 Returns true when the VM status returns the value of expected_status_code
459 :param block: When true, thread will block until active or timeout value in seconds has been exceeded (False)
460 :param poll_interval: The polling interval in seconds
463 return self.__vm_status_check(STATUS_ACTIVE, block, self.instance_settings.vm_boot_timeout, poll_interval)
465 def __vm_status_check(self, expected_status_code, block, timeout, poll_interval):
467 Returns true when the VM status returns the value of expected_status_code
468 :param expected_status_code: instance status evaluated with this string value
469 :param block: When true, thread will block until active or timeout value in seconds has been exceeded (False)
470 :param timeout: The timeout value
471 :param poll_interval: The polling interval in seconds
474 # sleep and wait for VM status change
478 start = time.time() - timeout
480 while timeout > time.time() - start:
481 status = self.__status(expected_status_code)
483 logger.info('VM is - ' + expected_status_code)
486 logger.debug('Retry querying VM status in ' + str(poll_interval) + ' seconds')
487 time.sleep(poll_interval)
488 logger.debug('VM status query timeout in ' + str(timeout - (time.time() - start)))
490 logger.error('Timeout checking for VM status for ' + expected_status_code)
493 def __status(self, expected_status_code):
495 Returns True when active else False
496 :param expected_status_code: instance status evaluated with this string value
502 instance = self.__nova.servers.get(self.__vm.id)
504 logger.warning('Cannot find instance with id - ' + self.__vm.id)
507 if instance.status == 'ERROR':
508 raise Exception('Instance had an error during deployment')
509 logger.debug('Instance status [' + self.instance_settings.name + '] is - ' + instance.status)
510 return instance.status == expected_status_code
512 def vm_ssh_active(self, block=False, poll_interval=POLL_INTERVAL):
514 Returns true when the VM can be accessed via SSH
515 :param block: When true, thread will block until active or timeout value in seconds has been exceeded (False)
516 :param poll_interval: The polling interval
519 # sleep and wait for VM status change
520 logger.info('Checking if VM is active')
522 timeout = self.instance_settings.ssh_connect_timeout
524 if self.vm_active(block=True):
528 start = time.time() - timeout
530 while timeout > time.time() - start:
531 status = self.__ssh_active()
533 logger.info('SSH is active for VM instance')
536 logger.debug('Retry SSH connection in ' + str(poll_interval) + ' seconds')
537 time.sleep(poll_interval)
538 logger.debug('SSH connection timeout in ' + str(timeout - (time.time() - start)))
540 logger.error('Timeout attempting to connect with VM via SSH')
543 def __ssh_active(self):
545 Returns True when can create a SSH session else False
548 if len(self.__floating_ips) > 0:
549 ssh = self.ssh_client()
554 def get_floating_ip(self, fip_name=None):
556 Returns the floating IP object byt name if found, else the first known, else None
557 :param fip_name: the name of the floating IP to return
558 :return: the SSH client or None
561 if fip_name and self.__floating_ip_dict.get(fip_name):
562 return self.__floating_ip_dict.get(fip_name)
563 if not fip and len(self.__floating_ips) > 0:
564 return self.__floating_ips[0]
567 def ssh_client(self, fip_name=None):
569 Returns an SSH client using the name or the first known floating IP if exists, else None
570 :param fip_name: the name of the floating IP to return
571 :return: the SSH client or None
573 fip = self.get_floating_ip(fip_name)
575 return ansible_utils.ssh_client(self.__floating_ips[0].ip, self.get_image_user(),
576 self.keypair_settings.private_filepath,
577 proxy_settings=self.__os_creds.proxy_settings)
579 logger.warning('Cannot return an SSH client. No Floating IP configured')
581 def add_security_group(self, security_group):
583 Adds a security group to this VM. Call will block until VM is active.
584 :param security_group: the OpenStack security group object
585 :return True if successful else False
587 self.vm_active(block=True)
589 if not security_group:
590 logger.warning('Security group object is None, cannot add')
594 nova_utils.add_security_group(self.__nova, self.get_vm_inst(), security_group['security_group']['name'])
596 except NotFound as e:
597 logger.warning('Security group not added - ' + str(e))
600 def remove_security_group(self, security_group):
602 Removes a security group to this VM. Call will block until VM is active.
603 :param security_group: the OpenStack security group object
604 :return True if successful else False
606 self.vm_active(block=True)
608 if not security_group:
609 logger.warning('Security group object is None, cannot remove')
613 nova_utils.remove_security_group(self.__nova, self.get_vm_inst(), security_group)
615 except NotFound as e:
616 logger.warning('Security group not removed - ' + str(e))
620 class VmInstanceSettings:
622 Class responsible for holding configuration setting for a VM Instance
624 def __init__(self, config=None, name=None, flavor=None, port_settings=list(), security_group_names=set(),
625 floating_ip_settings=list(), sudo_user=None, vm_boot_timeout=900,
626 vm_delete_timeout=300, ssh_connect_timeout=180, availability_zone=None, userdata=None):
629 :param config: dict() object containing the configuration settings using the attribute names below as each
630 member's the key and overrides any of the other parameters.
631 :param name: the name of the VM
632 :param flavor: the VM's flavor
633 :param port_settings: the port configuration settings (required)
634 :param security_group_names: a set of names of the security groups to add to the VM
635 :param floating_ip_settings: the floating IP configuration settings
636 :param sudo_user: the sudo user of the VM that will override the instance_settings.image_user when trying to
638 :param vm_boot_timeout: the amount of time a thread will sleep waiting for an instance to boot
639 :param vm_delete_timeout: the amount of time a thread will sleep waiting for an instance to be deleted
640 :param ssh_connect_timeout: the amount of time a thread will sleep waiting obtaining an SSH connection to a VM
641 :param availability_zone: the name of the compute server on which to deploy the VM (optional)
642 :param userdata: the cloud-init script to run after the VM has been started
645 self.name = config.get('name')
646 self.flavor = config.get('flavor')
647 self.sudo_user = config.get('sudo_user')
648 self.userdata = config.get('userdata')
650 self.port_settings = list()
651 if config.get('ports'):
652 for port_config in config['ports']:
653 if isinstance(port_config, PortSettings):
654 self.port_settings.append(port_config)
656 self.port_settings.append(PortSettings(config=port_config['port']))
658 if config.get('security_group_names'):
659 if isinstance(config['security_group_names'], list):
660 self.security_group_names = set(config['security_group_names'])
661 elif isinstance(config['security_group_names'], set):
662 self.security_group_names = config['security_group_names']
663 elif isinstance(config['security_group_names'], str):
664 self.security_group_names = [config['security_group_names']]
666 raise Exception('Invalid data type for security_group_names attribute')
668 self.security_group_names = set()
670 self.floating_ip_settings = list()
671 if config.get('floating_ips'):
672 for floating_ip_config in config['floating_ips']:
673 if isinstance(floating_ip_config, FloatingIpSettings):
674 self.floating_ip_settings.append(floating_ip_config)
676 self.floating_ip_settings.append(FloatingIpSettings(config=floating_ip_config['floating_ip']))
678 if config.get('vm_boot_timeout'):
679 self.vm_boot_timeout = config['vm_boot_timeout']
681 self.vm_boot_timeout = vm_boot_timeout
683 if config.get('vm_delete_timeout'):
684 self.vm_delete_timeout = config['vm_delete_timeout']
686 self.vm_delete_timeout = vm_delete_timeout
688 if config.get('ssh_connect_timeout'):
689 self.ssh_connect_timeout = config['ssh_connect_timeout']
691 self.ssh_connect_timeout = ssh_connect_timeout
693 if config.get('availability_zone'):
694 self.availability_zone = config['availability_zone']
696 self.availability_zone = None
700 self.port_settings = port_settings
701 self.security_group_names = security_group_names
702 self.floating_ip_settings = floating_ip_settings
703 self.sudo_user = sudo_user
704 self.vm_boot_timeout = vm_boot_timeout
705 self.vm_delete_timeout = vm_delete_timeout
706 self.ssh_connect_timeout = ssh_connect_timeout
707 self.availability_zone = availability_zone
708 self.userdata = userdata
710 if not self.name or not self.flavor:
711 raise Exception('Instance configuration requires the attributes: name, flavor')
713 if len(self.port_settings) == 0:
714 raise Exception('Instance configuration requires port settings (aka. NICS)')
717 class FloatingIpSettings:
719 Class responsible for holding configuration settings for a floating IP
721 def __init__(self, config=None, name=None, port_name=None, router_name=None, subnet_name=None, provisioning=True):
724 :param config: dict() object containing the configuration settings using the attribute names below as each
725 member's the key and overrides any of the other parameters.
726 :param name: the name of the floating IP
727 :param port_name: the name of the router to the external network
728 :param router_name: the name of the router to the external network
729 :param subnet_name: the name of the subnet on which to attach the floating IP
730 :param provisioning: when true, this floating IP can be used for provisioning
732 TODO - provisioning flag is a hack as I have only observed a single Floating IPs that actually works on
733 an instance. Multiple floating IPs placed on different subnets from the same port are especially troublesome
734 as you cannot predict which one will actually connect. For now, it is recommended not to setup multiple
735 floating IPs on an instance unless absolutely necessary.
738 self.name = config.get('name')
739 self.port_name = config.get('port_name')
740 self.router_name = config.get('router_name')
741 self.subnet_name = config.get('subnet_name')
742 if config.get('provisioning') is not None:
743 self.provisioning = config['provisioning']
745 self.provisioning = provisioning
748 self.port_name = port_name
749 self.router_name = router_name
750 self.subnet_name = subnet_name
751 self.provisioning = provisioning
753 if not self.name or not self.port_name or not self.router_name:
754 raise Exception('The attributes name, port_name and router_name are required for FloatingIPSettings')