import logging
import time
-from neutronclient.common.exceptions import PortNotFoundClient
-from novaclient.exceptions import NotFound, BadRequest
+from novaclient.exceptions import NotFound
from snaps.config.vm_inst import VmInstanceConfig, FloatingIpConfig
from snaps.openstack.openstack_creator import OpenStackComputeObject
logger = logging.getLogger('create_instance')
POLL_INTERVAL = 3
+VOL_DETACH_TIMEOUT = 120
STATUS_ACTIVE = 'ACTIVE'
STATUS_DELETED = 'DELETED'
'Found existing machine with name - %s',
self.instance_settings.name)
- fips = neutron_utils.get_floating_ips(self.__neutron,
- self.__ports)
+ fips = neutron_utils.get_port_floating_ips(
+ self.__neutron, self.__ports)
for port_id, fip in fips:
settings = self.instance_settings.floating_ip_settings
for fip_setting in settings:
glance = glance_utils.glance_client(self._os_creds)
self.__vm = nova_utils.create_server(
self._nova, self.__neutron, glance, self.instance_settings,
- self.image_settings, self.keypair_settings)
+ self.image_settings, self.project_id, self.keypair_settings)
logger.info('Created instance with name - %s',
self.instance_settings.name)
cinder, volume_name=volume_name)
if volume and self.vm_active(block=True):
- timeout = 30
vm = nova_utils.attach_volume(
- self._nova, self.__neutron, self.__vm, volume, timeout)
+ self._nova, self.__neutron, self.__vm, volume,
+ self.project_id, timeout=VOL_DETACH_TIMEOUT)
if vm:
self.__vm = vm
else:
logger.warn('Volume [%s] not attached within timeout '
- 'of [%s]', volume.name, timeout)
+ 'of [%s]', volume.name, VOL_DETACH_TIMEOUT)
else:
logger.warn('Unable to attach volume named [%s]',
volume_name)
"""
Adds a floating IP to a running instance
:param floating_ip_setting - the floating IP configuration
+ :return: the floating ip object
"""
port_dict = dict()
for key, port in self.__ports:
# gateway
ext_gateway = self.__ext_gateway_by_router(
floating_ip_setting.router_name)
- if ext_gateway:
- subnet = neutron_utils.get_subnet(
- self.__neutron,
- subnet_name=floating_ip_setting.subnet_name)
+ if ext_gateway and self.vm_active(block=True):
floating_ip = neutron_utils.create_floating_ip(
- self.__neutron, ext_gateway)
+ self.__neutron, ext_gateway, port.id)
self.__floating_ip_dict[floating_ip_setting.name] = floating_ip
logger.info(
'Created floating IP %s via router - %s', floating_ip.ip,
floating_ip_setting.router_name)
- self.__add_floating_ip(floating_ip, port, subnet)
+
+ return floating_ip
else:
raise VmInstanceCreationError(
'Unable to add floating IP to port, cannot locate router '
# Cleanup floating IPs
for name, floating_ip in self.__floating_ip_dict.items():
- try:
- logger.info('Deleting Floating IP - ' + floating_ip.ip)
- neutron_utils.delete_floating_ip(self.__neutron, floating_ip)
- except Exception as e:
- logger.error('Error deleting Floating IP - ' + str(e))
+ logger.info('Deleting Floating IP - ' + floating_ip.ip)
+ neutron_utils.delete_floating_ip(self.__neutron, floating_ip)
+
self.__floating_ip_dict = dict()
- # Detach Volume
- for volume_rec in self.__vm.volume_ids:
- cinder = cinder_utils.cinder_client(self._os_creds)
- volume = cinder_utils.get_volume_by_id(cinder, volume_rec['id'])
- if volume:
- try:
+ # Cleanup ports
+ for name, port in self.__ports:
+ logger.info('Deleting Port with ID - %s ', port.id)
+ neutron_utils.delete_port(self.__neutron, port)
+
+ self.__ports = list()
+
+ if self.__vm:
+ # Detach Volume
+ for volume_rec in self.__vm.volume_ids:
+ cinder = cinder_utils.cinder_client(self._os_creds)
+ volume = cinder_utils.get_volume_by_id(
+ cinder, volume_rec['id'])
+ if volume:
vm = nova_utils.detach_volume(
- self._nova, self.__neutron, self.__vm, volume, 30)
+ self._nova, self.__neutron, self.__vm, volume,
+ self.project_id, timeout=VOL_DETACH_TIMEOUT)
if vm:
self.__vm = vm
else:
logger.warn(
'Timeout waiting to detach volume %s', volume.name)
- except Exception as e:
- logger.error('Unexpected error detaching volume %s '
- 'with error %s', volume.name, e)
- else:
- logger.warn('Unable to detach volume with ID - [%s]',
- volume_rec['id'])
+ else:
+ logger.warn('Unable to detach volume with ID - [%s]',
+ volume_rec['id'])
- # Cleanup ports
- for name, port in self.__ports:
- logger.info('Deleting Port with ID - %s ', port.id)
- try:
- neutron_utils.delete_port(self.__neutron, port)
- except PortNotFoundClient as e:
- logger.warning('Unexpected error deleting port - %s', e)
- pass
- self.__ports = list()
+ # Cleanup VM
+ logger.info(
+ 'Deleting VM instance - ' + self.instance_settings.name)
- # Cleanup VM
- if self.__vm:
try:
- logger.info(
- 'Deleting VM instance - ' + self.instance_settings.name)
nova_utils.delete_vm_instance(self._nova, self.__vm)
- except Exception as e:
- logger.error('Error deleting VM - %s', e)
+ except NotFound as e:
+ logger.warn('Instance already deleted - %s', e)
# Block until instance cannot be found or returns the status of
# DELETED
logger.info('Checking deletion status')
- try:
- if self.vm_deleted(block=True):
- logger.info(
- 'VM has been properly deleted VM with name - %s',
- self.instance_settings.name)
- self.__vm = None
- else:
- logger.error(
- 'VM not deleted within the timeout period of %s '
- 'seconds', self.instance_settings.vm_delete_timeout)
- except Exception as e:
+ if self.vm_deleted(block=True):
+ logger.info(
+ 'VM has been properly deleted VM with name - %s',
+ self.instance_settings.name)
+ self.__vm = None
+ else:
logger.error(
- 'Unexpected error while checking VM instance status - %s',
- e)
+ 'VM not deleted within the timeout period of %s '
+ 'seconds', self.instance_settings.vm_delete_timeout)
def __query_ports(self, port_settings):
"""
for port_setting in port_settings:
port = neutron_utils.get_port(
- self.__neutron, port_settings=port_setting)
+ self.__neutron, port_settings=port_setting,
+ project_id=self.project_id)
if port:
ports.append((port_setting.name, port))
return ports
- def __add_floating_ip(self, floating_ip, port, subnet, timeout=30,
- poll_interval=POLL_INTERVAL):
- """
- Returns True when active else False
- TODO - Make timeout and poll_interval configurable...
- """
- ip = None
-
- if subnet:
- # Take IP of subnet if there is one configured on which to place
- # the floating IP
- for fixed_ip in port.ips:
- if fixed_ip['subnet_id'] == subnet.id:
- ip = fixed_ip['ip_address']
- break
- else:
- # Simply take the first
- ip = port.ips[0]['ip_address']
-
- if ip:
- count = timeout / poll_interval
- while count > 0:
- logger.debug('Attempting to add floating IP to instance')
- try:
- nova_utils.add_floating_ip_to_server(
- self._nova, self.__vm, floating_ip, ip)
- logger.info(
- 'Added floating IP %s to port IP %s on instance %s',
- floating_ip.ip, ip, self.instance_settings.name)
- return
- except BadRequest as bre:
- logger.error('Cannot add floating IP [%s]', bre)
- raise
- except Exception as e:
- logger.debug(
- 'Retry adding floating IP to instance. Last attempt '
- 'failed with - %s', e)
- time.sleep(poll_interval)
- count -= 1
- pass
- else:
- raise VmInstanceCreationError(
- 'Unable find IP address on which to place the floating IP')
-
- logger.error('Timeout attempting to add the floating IP to instance.')
- raise VmInstanceCreationError(
- 'Timeout while attempting add floating IP to instance')
-
def get_os_creds(self):
"""
Returns the OpenStack credentials used to create these objects
:return: Server object
"""
return nova_utils.get_server_object_by_id(
- self._nova, self.__neutron, self.__vm.id)
+ self._nova, self.__neutron, self.__vm.id, self.project_id)
def get_console_output(self):
"""
"""
return nova_utils.get_server_info(self._nova, self.__vm)
- def config_nics(self):
- """
- Responsible for configuring NICs on RPM systems where the instance has
- more than one configured port
- :return: the value returned by ansible_utils.apply_ansible_playbook()
- """
- if len(self.__ports) > 1 and len(self.__floating_ip_dict) > 0:
- if self.vm_active(block=True) and self.vm_ssh_active(block=True):
- for key, port in self.__ports:
- port_index = self.__ports.index((key, port))
- if port_index > 0:
- nic_name = 'eth' + repr(port_index)
- retval = self.__config_nic(
- nic_name, port,
- self.__get_first_provisioning_floating_ip().ip)
- logger.info('Configured NIC - %s on VM - %s',
- nic_name, self.instance_settings.name)
- return retval
-
def __get_first_provisioning_floating_ip(self):
"""
Returns the first floating IP tagged with the Floating IP name if
for key, fip in self.__floating_ip_dict.items():
return fip
- def __config_nic(self, nic_name, port, ip):
- """
- Although ports/NICs can contain multiple IPs, this code currently only
- supports the first.
-
- :param nic_name: Name of the interface
- :param port: The port information containing the expected IP values.
- :param ip: The IP on which to apply the playbook.
- :return: the return value from ansible
- """
- port_ip = port.ips[0]['ip_address']
- variables = {
- 'floating_ip': ip,
- 'nic_name': nic_name,
- 'nic_ip': port_ip
- }
-
- if self.image_settings.nic_config_pb_loc and self.keypair_settings:
- return self.apply_ansible_playbook(
- self.image_settings.nic_config_pb_loc, variables)
- else:
- logger.warning(
- 'VM %s cannot self configure NICs eth1++. No playbook or '
- 'keypairs found.', self.instance_settings.name)
-
def apply_ansible_playbook(self, pb_file_loc, variables=None,
fip_name=None):
"""
"""
return ansible_utils.apply_playbook(
pb_file_loc, [self.get_floating_ip(fip_name=fip_name).ip],
- self.get_image_user(), self.keypair_settings.private_filepath,
- variables, self._os_creds.proxy_settings)
+ self.get_image_user(),
+ ssh_priv_key_file_path=self.keypair_settings.private_filepath,
+ variables=variables, proxy_setting=self._os_creds.proxy_settings)
def get_image_user(self):
"""
STATUS_ACTIVE, block, self.instance_settings.vm_boot_timeout,
poll_interval):
self.__vm = nova_utils.get_server_object_by_id(
- self._nova, self.__neutron, self.__vm.id)
+ self._nova, self.__neutron, self.__vm.id, self.project_id)
return True
return False
status)
return status == expected_status_code
- def vm_ssh_active(self, block=False, poll_interval=POLL_INTERVAL):
+ def vm_ssh_active(self, user_override=None, password=None, block=False,
+ timeout=None, poll_interval=POLL_INTERVAL):
"""
Returns true when the VM can be accessed via SSH
+ :param user_override: overrides the user with which to create the
+ connection
+ :param password: overrides the use of a password instead of a private
+ key with which to create the connection
:param block: When true, thread will block until active or timeout
value in seconds has been exceeded (False)
+ :param timeout: the number of seconds to retry obtaining the connection
+ and overrides the ssh_connect_timeout member of the
+ self.instance_settings object
:param poll_interval: The polling interval
:return: T/F
"""
# sleep and wait for VM status change
logger.info('Checking if VM is active')
- timeout = self.instance_settings.ssh_connect_timeout
+ if not timeout:
+ timeout = self.instance_settings.ssh_connect_timeout
if self.vm_active(block=True):
if block:
start = time.time() - timeout
while timeout > time.time() - start:
- status = self.__ssh_active()
+ status = self.__ssh_active(
+ user_override=user_override, password=password)
if status:
logger.info('SSH is active for VM instance')
return True
logger.error('Timeout attempting to connect with VM via SSH')
return False
- def __ssh_active(self):
+ def __ssh_active(self, user_override=None, password=None):
"""
Returns True when can create a SSH session else False
:return: T/F
"""
if len(self.__floating_ip_dict) > 0:
- ssh = self.ssh_client()
+ ssh = self.ssh_client(
+ user_override=user_override, password=password)
if ssh:
ssh.close()
return True
return False
+ def cloud_init_complete(self, block=False, poll_interval=POLL_INTERVAL):
+ """
+ Returns true when the VM's cloud-init routine has completed.
+ Note: this is currently done via SSH, therefore, if this instance does
+ not have a Floating IP or a running SSH server, this routine
+ will always return False or raise an Exception
+ :param block: When true, thread will block until active or timeout
+ value in seconds has been exceeded (False)
+ :param poll_interval: The polling interval
+ :return: T/F
+ """
+ # sleep and wait for VM status change
+ logger.info('Checking if cloud-init has completed')
+
+ timeout = self.instance_settings.cloud_init_timeout
+
+ if self.vm_active(block=True) and self.vm_ssh_active(block=True):
+ if block:
+ start = time.time()
+ else:
+ start = time.time() - timeout
+
+ while timeout > time.time() - start:
+ status = self.__cloud_init_complete()
+ if status:
+ logger.info('cloud-init complete for VM instance')
+ return True
+
+ logger.debug('Retry cloud-init query in ' + str(
+ poll_interval) + ' seconds')
+ time.sleep(poll_interval)
+ logger.debug('cloud-init complete timeout in ' + str(
+ timeout - (time.time() - start)))
+
+ logger.error('Timeout waiting for cloud-init to complete')
+ return False
+
+ def __cloud_init_complete(self):
+ """
+ Returns True when can create a SSH session else False
+ :return: T/F
+ """
+ if len(self.__floating_ip_dict) > 0:
+ ssh = self.ssh_client()
+ if ssh:
+ stdin1, stdout1, sterr1 = ssh.exec_command(
+ 'ls -l /var/lib/cloud/instance/boot-finished')
+ return stdout1.channel.recv_exit_status() == 0
+ return False
+
def get_floating_ip(self, fip_name=None):
"""
Returns the floating IP object byt name if found, else the first known,
else:
return self.__get_first_provisioning_floating_ip()
- def ssh_client(self, fip_name=None):
+ def ssh_client(self, fip_name=None, user_override=None, password=None):
"""
Returns an SSH client using the name or the first known floating IP if
exists, else None
:param fip_name: the name of the floating IP to return
+ :param user_override: the username to use instead of the default
+ :param password: the password to use instead of the private key
:return: the SSH client or None
"""
fip = self.get_floating_ip(fip_name)
+
+ ansible_user = self.get_image_user()
+ if user_override:
+ ansible_user = user_override
+
+ if password:
+ private_key = None
+ else:
+ private_key = self.keypair_settings.private_filepath
+
if fip:
return ansible_utils.ssh_client(
self.__get_first_provisioning_floating_ip().ip,
- self.get_image_user(),
- self.keypair_settings.private_filepath,
+ ansible_user,
+ private_key_filepath=private_key,
+ password=password,
proxy_settings=self._os_creds.proxy_settings)
else:
FloatingIPAllocationError(
self._nova, self.__vm, reboot_type=reboot_type)
-def generate_creator(os_creds, vm_inst, image_config, keypair_config=None):
+def generate_creator(os_creds, vm_inst, image_config, project_id,
+ keypair_config=None):
"""
Initializes an OpenStackVmInstance object
:param os_creds: the OpenStack credentials
:param vm_inst: the SNAPS-OO VmInst domain object
:param image_config: the associated ImageConfig object
+ :param project_id: the associated project ID
:param keypair_config: the associated KeypairConfig object (optional)
:return: an initialized OpenStackVmInstance object
"""
nova = nova_utils.nova_client(os_creds)
neutron = neutron_utils.neutron_client(os_creds)
derived_inst_config = settings_utils.create_vm_inst_config(
- nova, neutron, vm_inst)
+ nova, neutron, vm_inst, project_id)
derived_inst_creator = OpenStackVmInstance(
os_creds, derived_inst_config, image_config, keypair_config)