snaps/openstack/create_instance.py

   1 # Copyright (c) 2017 Cable Television Laboratories, Inc. ("CableLabs")
   2 #                    and others.  All rights reserved.
   3 #
   4 # Licensed under the Apache License, Version 2.0 (the "License");
   5 # you may not use this file except in compliance with the License.
   6 # You may obtain a copy of the License at:
   7 #
   8 #     http://www.apache.org/licenses/LICENSE-2.0
   9 #
  10 # Unless required by applicable law or agreed to in writing, software
  11 # distributed under the License is distributed on an "AS IS" BASIS,
  12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 # See the License for the specific language governing permissions and
  14 # limitations under the License.
  15 import logging
  16 import time
  17
  18 from novaclient.exceptions import NotFound, BadRequest
  19
  20 from snaps.config.vm_inst import VmInstanceConfig, FloatingIpConfig
  21 from snaps.openstack.openstack_creator import OpenStackComputeObject
  22 from snaps.openstack.utils import glance_utils, cinder_utils, settings_utils
  23 from snaps.openstack.utils import neutron_utils
  24 from snaps.openstack.utils import nova_utils
  25 from snaps.openstack.utils.nova_utils import RebootType
  26 from snaps.provisioning import ansible_utils
  27
  28 __author__ = 'spisarski'
  29
  30 logger = logging.getLogger('create_instance')
  31
  32 POLL_INTERVAL = 3
  33 STATUS_ACTIVE = 'ACTIVE'
  34 STATUS_DELETED = 'DELETED'
  35
  36
  37 class OpenStackVmInstance(OpenStackComputeObject):
  38     """
  39     Class responsible for managing a VM instance in OpenStack
  40     """
  41
  42     def __init__(self, os_creds, instance_settings, image_settings,
  43                  keypair_settings=None):
  44         """
  45         Constructor
  46         :param os_creds: The connection credentials to the OpenStack API
  47         :param instance_settings: Contains the settings for this VM
  48         :param image_settings: The OpenStack image object settings
  49         :param keypair_settings: The keypair metadata (Optional)
  50         :raises Exception
  51         """
  52         super(self.__class__, self).__init__(os_creds)
  53
  54         self.__neutron = None
  55
  56         self.instance_settings = instance_settings
  57         self.image_settings = image_settings
  58         self.keypair_settings = keypair_settings
  59
  60         self.__floating_ip_dict = dict()
  61
  62         # Instantiated in self.create()
  63         self.__ports = list()
  64
  65         # Note: this object does not change after the VM becomes active
  66         self.__vm = None
  67
  68     def initialize(self):
  69         """
  70         Loads the existing VMInst, Port, FloatingIps
  71         :return: VMInst domain object
  72         """
  73         super(self.__class__, self).initialize()
  74
  75         self.__neutron = neutron_utils.neutron_client(self._os_creds)
  76
  77         self.__ports = self.__query_ports(self.instance_settings.port_settings)
  78         self.__lookup_existing_vm_by_name()
  79
  80     def create(self, block=False):
  81         """
  82         Creates a VM instance and associated objects unless they already exist
  83         :param block: Thread will block until instance has either become
  84                       active, error, or timeout waiting.
  85                       Additionally, when True, floating IPs will not be applied
  86                       until VM is active.
  87         :return: VMInst domain object
  88         """
  89         self.initialize()
  90
  91         if len(self.__ports) == 0:
  92             self.__ports = self.__create_ports(
  93                 self.instance_settings.port_settings)
  94         if not self.__vm:
  95             self.__create_vm(block)
  96
  97         return self.__vm
  98
  99     def __lookup_existing_vm_by_name(self):
 100         """
 101         Populates the member variables 'self.vm' and 'self.floating_ips' if a
 102         VM with the same name already exists
 103         within the project
 104         """
 105         server = nova_utils.get_server(
 106             self._nova, self.__neutron,
 107             vm_inst_settings=self.instance_settings)
 108         if server:
 109             if server.name == self.instance_settings.name:
 110                 self.__vm = server
 111                 logger.info(
 112                     'Found existing machine with name - %s',
 113                     self.instance_settings.name)
 114
 115                 fips = neutron_utils.get_floating_ips(self.__neutron,
 116                                                       self.__ports)
 117                 for port_id, fip in fips:
 118                     settings = self.instance_settings.floating_ip_settings
 119                     for fip_setting in settings:
 120                         if port_id == fip_setting.port_id:
 121                             self.__floating_ip_dict[fip_setting.name] = fip
 122                         else:
 123                             port = neutron_utils.get_port_by_id(
 124                                 self.__neutron, port_id)
 125                             if port and port.name == fip_setting.port_name:
 126                                 self.__floating_ip_dict[fip_setting.name] = fip
 127
 128     def __create_vm(self, block=False):
 129         """
 130         Responsible for creating the VM instance
 131         :param block: Thread will block until instance has either become
 132                       active, error, or timeout waiting. Floating IPs will be
 133                       assigned after active when block=True
 134         """
 135         glance = glance_utils.glance_client(self._os_creds)
 136         self.__vm = nova_utils.create_server(
 137             self._nova, self.__neutron, glance, self.instance_settings,
 138             self.image_settings, self.keypair_settings)
 139         logger.info('Created instance with name - %s',
 140                     self.instance_settings.name)
 141
 142         if block:
 143             if not self.vm_active(block=True):
 144                 raise VmInstanceCreationError(
 145                     'Fatal error, VM did not become ACTIVE within the alloted '
 146                     'time')
 147
 148         # Create server should do this but found it needed to occur here
 149         for sec_grp_name in self.instance_settings.security_group_names:
 150             if self.vm_active(block=True):
 151                 nova_utils.add_security_group(self._nova, self.__vm,
 152                                               sec_grp_name)
 153             else:
 154                 raise VmInstanceCreationError(
 155                     'Cannot applying security group with name ' +
 156                     sec_grp_name +
 157                     ' to VM that did not activate with name - ' +
 158                     self.instance_settings.name)
 159
 160         if self.instance_settings.volume_names:
 161             for volume_name in self.instance_settings.volume_names:
 162                 cinder = cinder_utils.cinder_client(self._os_creds)
 163                 volume = cinder_utils.get_volume(
 164                     cinder, volume_name=volume_name)
 165
 166                 if volume and self.vm_active(block=True):
 167                     timeout = 30
 168                     vm = nova_utils.attach_volume(
 169                         self._nova, self.__neutron, self.__vm, volume, timeout)
 170
 171                     if vm:
 172                         self.__vm = vm
 173                     else:
 174                         logger.warn('Volume [%s] not attached within timeout '
 175                                     'of [%s]', volume.name, timeout)
 176                 else:
 177                     logger.warn('Unable to attach volume named [%s]',
 178                                 volume_name)
 179
 180         self.__apply_floating_ips()
 181
 182     def __apply_floating_ips(self):
 183         """
 184         Applies the configured floating IPs to the necessary ports
 185         """
 186         port_dict = dict()
 187         for key, port in self.__ports:
 188             port_dict[key] = port
 189
 190         # Apply floating IPs
 191         for floating_ip_setting in self.instance_settings.floating_ip_settings:
 192             self.add_floating_ip(floating_ip_setting)
 193
 194     def add_floating_ip(self, floating_ip_setting):
 195         """
 196         Adds a floating IP to a running instance
 197         :param floating_ip_setting - the floating IP configuration
 198         :return: the floating ip object
 199         """
 200         port_dict = dict()
 201         for key, port in self.__ports:
 202             port_dict[key] = port
 203
 204         # Apply floating IP
 205         port = port_dict.get(floating_ip_setting.port_name)
 206
 207         if not port:
 208             raise VmInstanceCreationError(
 209                 'Cannot find port object with name - ' +
 210                 floating_ip_setting.port_name)
 211
 212         # Setup Floating IP only if there is a router with an external
 213         # gateway
 214         ext_gateway = self.__ext_gateway_by_router(
 215             floating_ip_setting.router_name)
 216         if ext_gateway:
 217             subnet = neutron_utils.get_subnet(
 218                 self.__neutron,
 219                 subnet_name=floating_ip_setting.subnet_name)
 220             floating_ip = neutron_utils.create_floating_ip(
 221                 self.__neutron, ext_gateway)
 222             self.__floating_ip_dict[floating_ip_setting.name] = floating_ip
 223
 224             logger.info(
 225                 'Created floating IP %s via router - %s', floating_ip.ip,
 226                 floating_ip_setting.router_name)
 227             self.__add_floating_ip(floating_ip, port, subnet)
 228             return floating_ip
 229         else:
 230             raise VmInstanceCreationError(
 231                 'Unable to add floating IP to port, cannot locate router '
 232                 'with an external gateway ')
 233
 234     def __ext_gateway_by_router(self, router_name):
 235         """
 236         Returns network name for the external network attached to a router or
 237         None if not found
 238         :param router_name: The name of the router to lookup
 239         :return: the external network name or None
 240         """
 241         router = neutron_utils.get_router(
 242             self.__neutron, router_name=router_name)
 243         if router and router.external_network_id:
 244             network = neutron_utils.get_network_by_id(
 245                 self.__neutron, router.external_network_id)
 246             if network:
 247                 return network.name
 248         return None
 249
 250     def clean(self):
 251         """
 252         Destroys the VM instance
 253         """
 254
 255         # Cleanup floating IPs
 256         for name, floating_ip in self.__floating_ip_dict.items():
 257             logger.info('Deleting Floating IP - ' + floating_ip.ip)
 258             neutron_utils.delete_floating_ip(self.__neutron, floating_ip)
 259
 260         self.__floating_ip_dict = dict()
 261
 262         # Cleanup ports
 263         for name, port in self.__ports:
 264             logger.info('Deleting Port with ID - %s ', port.id)
 265             neutron_utils.delete_port(self.__neutron, port)
 266
 267         self.__ports = list()
 268
 269         if self.__vm:
 270             # Detach Volume
 271             for volume_rec in self.__vm.volume_ids:
 272                 cinder = cinder_utils.cinder_client(self._os_creds)
 273                 volume = cinder_utils.get_volume_by_id(
 274                     cinder, volume_rec['id'])
 275                 if volume:
 276                     vm = nova_utils.detach_volume(
 277                         self._nova, self.__neutron, self.__vm, volume, 30)
 278                     if vm:
 279                         self.__vm = vm
 280                     else:
 281                         logger.warn(
 282                             'Timeout waiting to detach volume %s', volume.name)
 283                 else:
 284                     logger.warn('Unable to detach volume with ID - [%s]',
 285                                 volume_rec['id'])
 286
 287             # Cleanup VM
 288             logger.info(
 289                 'Deleting VM instance - ' + self.instance_settings.name)
 290
 291             try:
 292                 nova_utils.delete_vm_instance(self._nova, self.__vm)
 293             except NotFound as e:
 294                 logger.warn('Instance already deleted - %s', e)
 295
 296             # Block until instance cannot be found or returns the status of
 297             # DELETED
 298             logger.info('Checking deletion status')
 299
 300             if self.vm_deleted(block=True):
 301                 logger.info(
 302                     'VM has been properly deleted VM with name - %s',
 303                     self.instance_settings.name)
 304                 self.__vm = None
 305             else:
 306                 logger.error(
 307                     'VM not deleted within the timeout period of %s '
 308                     'seconds', self.instance_settings.vm_delete_timeout)
 309
 310     def __query_ports(self, port_settings):
 311         """
 312         Returns the previously configured ports or an empty list if none
 313         exist
 314         :param port_settings: A list of PortSetting objects
 315         :return: a list of OpenStack port tuples where the first member is the
 316                  port name and the second is the port object
 317         """
 318         ports = list()
 319
 320         for port_setting in port_settings:
 321             port = neutron_utils.get_port(
 322                 self.__neutron, port_settings=port_setting)
 323             if port:
 324                 ports.append((port_setting.name, port))
 325
 326         return ports
 327
 328     def __create_ports(self, port_settings):
 329         """
 330         Returns the previously configured ports or creates them if they do not
 331         exist
 332         :param port_settings: A list of PortSetting objects
 333         :return: a list of OpenStack port tuples where the first member is the
 334                  port name and the second is the port object
 335         """
 336         ports = list()
 337
 338         for port_setting in port_settings:
 339             port = neutron_utils.get_port(
 340                 self.__neutron, port_settings=port_setting)
 341             if not port:
 342                 port = neutron_utils.create_port(
 343                     self.__neutron, self._os_creds, port_setting)
 344                 if port:
 345                     ports.append((port_setting.name, port))
 346
 347         return ports
 348
 349     def __add_floating_ip(self, floating_ip, port, subnet, timeout=30,
 350                           poll_interval=POLL_INTERVAL):
 351         """
 352         Returns True when active else False
 353         TODO - Make timeout and poll_interval configurable...
 354         """
 355         ip = None
 356
 357         if subnet:
 358             # Take IP of subnet if there is one configured on which to place
 359             # the floating IP
 360             for fixed_ip in port.ips:
 361                 if fixed_ip['subnet_id'] == subnet.id:
 362                     ip = fixed_ip['ip_address']
 363                     break
 364         else:
 365             # Simply take the first
 366             ip = port.ips[0]['ip_address']
 367
 368         if ip:
 369             count = timeout / poll_interval
 370             while count > 0:
 371                 logger.debug('Attempting to add floating IP to instance')
 372                 try:
 373                     nova_utils.add_floating_ip_to_server(
 374                         self._nova, self.__vm, floating_ip, ip)
 375                     logger.info(
 376                         'Added floating IP %s to port IP %s on instance %s',
 377                         floating_ip.ip, ip, self.instance_settings.name)
 378                     return
 379                 except BadRequest as bre:
 380                     logger.error('Cannot add floating IP [%s]', bre)
 381                     raise
 382                 except Exception as e:
 383                     logger.warn(
 384                         'Retry adding floating IP to instance. Last attempt '
 385                         'failed with - %s', e)
 386                     time.sleep(poll_interval)
 387                     count -= 1
 388                     pass
 389         else:
 390             raise VmInstanceCreationError(
 391                 'Unable find IP address on which to place the floating IP')
 392
 393         logger.error('Timeout attempting to add the floating IP to instance.')
 394         raise VmInstanceCreationError(
 395             'Timeout while attempting add floating IP to instance')
 396
 397     def get_os_creds(self):
 398         """
 399         Returns the OpenStack credentials used to create these objects
 400         :return: the credentials
 401         """
 402         return self._os_creds
 403
 404     def get_vm_inst(self):
 405         """
 406         Returns the latest version of this server object from OpenStack
 407         :return: Server object
 408         """
 409         return nova_utils.get_server_object_by_id(
 410             self._nova, self.__neutron, self.__vm.id)
 411
 412     def get_console_output(self):
 413         """
 414         Returns the vm console object for parsing logs
 415         :return: the console output object
 416         """
 417         return nova_utils.get_server_console_output(self._nova, self.__vm)
 418
 419     def get_port_ip(self, port_name, subnet_name=None):
 420         """
 421         Returns the first IP for the port corresponding with the port_name
 422         parameter when subnet_name is None else returns the IP address that
 423         corresponds to the subnet_name parameter
 424         :param port_name: the name of the port from which to return the IP
 425         :param subnet_name: the name of the subnet attached to this IP
 426         :return: the IP or None if not found
 427         """
 428         port = self.get_port_by_name(port_name)
 429         if port:
 430             if subnet_name:
 431                 subnet = neutron_utils.get_subnet(
 432                     self.__neutron, subnet_name=subnet_name)
 433                 if not subnet:
 434                     logger.warning('Cannot retrieve port IP as subnet could '
 435                                    'not be located with name - %s',
 436                                    subnet_name)
 437                     return None
 438                 for fixed_ip in port.ips:
 439                     if fixed_ip['subnet_id'] == subnet.id:
 440                         return fixed_ip['ip_address']
 441             else:
 442                 if port.ips and len(port.ips) > 0:
 443                     return port.ips[0]['ip_address']
 444         return None
 445
 446     def get_port_mac(self, port_name):
 447         """
 448         Returns the first IP for the port corresponding with the port_name
 449         parameter
 450         TODO - Add in the subnet as an additional parameter as a port may have
 451         multiple fixed_ips
 452         :param port_name: the name of the port from which to return the IP
 453         :return: the IP or None if not found
 454         """
 455         port = self.get_port_by_name(port_name)
 456         if port:
 457             return port.mac_address
 458         return None
 459
 460     def get_port_by_name(self, port_name):
 461         """
 462         Retrieves the OpenStack port object by its given name
 463         :param port_name: the name of the port
 464         :return: the OpenStack port object or None if not exists
 465         """
 466         for key, port in self.__ports:
 467             if key == port_name:
 468                 return port
 469         logger.warning('Cannot find port with name - ' + port_name)
 470         return None
 471
 472     def get_vm_info(self):
 473         """
 474         Returns a dictionary of a VMs info as returned by OpenStack
 475         :return: a dict()
 476         """
 477         return nova_utils.get_server_info(self._nova, self.__vm)
 478
 479     def __get_first_provisioning_floating_ip(self):
 480         """
 481         Returns the first floating IP tagged with the Floating IP name if
 482         exists else the first one found
 483         :return:
 484         """
 485         for floating_ip_setting in self.instance_settings.floating_ip_settings:
 486             if floating_ip_setting.provisioning:
 487                 fip = self.__floating_ip_dict.get(floating_ip_setting.name)
 488                 if fip:
 489                     return fip
 490                 elif len(self.__floating_ip_dict) > 0:
 491                     for key, fip in self.__floating_ip_dict.items():
 492                         return fip
 493
 494         # When cannot be found above
 495         if len(self.__floating_ip_dict) > 0:
 496             for key, fip in self.__floating_ip_dict.items():
 497                 return fip
 498
 499     def apply_ansible_playbook(self, pb_file_loc, variables=None,
 500                                fip_name=None):
 501         """
 502         Applies a playbook to a VM
 503         :param pb_file_loc: the file location of the playbook to be applied
 504         :param variables: a dict() of substitution values required by the
 505                           playbook
 506         :param fip_name: the name of the floating IP to use for applying the
 507                          playbook (default - will take the first)
 508         :return: the return value from ansible
 509         """
 510         return ansible_utils.apply_playbook(
 511             pb_file_loc, [self.get_floating_ip(fip_name=fip_name).ip],
 512             self.get_image_user(),
 513             ssh_priv_key_file_path=self.keypair_settings.private_filepath,
 514             variables=variables, proxy_setting=self._os_creds.proxy_settings)
 515
 516     def get_image_user(self):
 517         """
 518         Returns the instance sudo_user if it has been configured in the
 519         instance_settings else it returns the image_settings.image_user value
 520         """
 521         if self.instance_settings.sudo_user:
 522             return self.instance_settings.sudo_user
 523         else:
 524             return self.image_settings.image_user
 525
 526     def vm_deleted(self, block=False, poll_interval=POLL_INTERVAL):
 527         """
 528         Returns true when the VM status returns the value of
 529         expected_status_code or instance retrieval throws a NotFound exception.
 530         :param block: When true, thread will block until active or timeout
 531                       value in seconds has been exceeded (False)
 532         :param poll_interval: The polling interval in seconds
 533         :return: T/F
 534         """
 535         try:
 536             return self.__vm_status_check(
 537                 STATUS_DELETED, block,
 538                 self.instance_settings.vm_delete_timeout, poll_interval)
 539         except NotFound as e:
 540             logger.debug(
 541                 "Instance not found when querying status for %s with message "
 542                 "%s", STATUS_DELETED, e)
 543             return True
 544
 545     def vm_active(self, block=False, poll_interval=POLL_INTERVAL):
 546         """
 547         Returns true when the VM status returns the value of the constant
 548         STATUS_ACTIVE
 549         :param block: When true, thread will block until active or timeout
 550                       value in seconds has been exceeded (False)
 551         :param poll_interval: The polling interval in seconds
 552         :return: T/F
 553         """
 554         if self.__vm_status_check(
 555                 STATUS_ACTIVE, block, self.instance_settings.vm_boot_timeout,
 556                 poll_interval):
 557             self.__vm = nova_utils.get_server_object_by_id(
 558                 self._nova, self.__neutron, self.__vm.id)
 559             return True
 560         return False
 561
 562     def __vm_status_check(self, expected_status_code, block, timeout,
 563                           poll_interval):
 564         """
 565         Returns true when the VM status returns the value of
 566         expected_status_code
 567         :param expected_status_code: instance status evaluated with this
 568                                      string value
 569         :param block: When true, thread will block until active or timeout
 570                       value in seconds has been exceeded (False)
 571         :param timeout: The timeout value
 572         :param poll_interval: The polling interval in seconds
 573         :return: T/F
 574         """
 575         # sleep and wait for VM status change
 576         if block:
 577             start = time.time()
 578         else:
 579             return self.__status(expected_status_code)
 580
 581         while timeout > time.time() - start:
 582             status = self.__status(expected_status_code)
 583             if status:
 584                 logger.info('VM is - ' + expected_status_code)
 585                 return True
 586
 587             logger.debug('Retry querying VM status in ' + str(
 588                 poll_interval) + ' seconds')
 589             time.sleep(poll_interval)
 590             logger.debug('VM status query timeout in ' + str(
 591                 timeout - (time.time() - start)))
 592
 593         logger.error(
 594             'Timeout checking for VM status for ' + expected_status_code)
 595         return False
 596
 597     def __status(self, expected_status_code):
 598         """
 599         Returns True when active else False
 600         :param expected_status_code: instance status evaluated with this string
 601                                      value
 602         :return: T/F
 603         """
 604         if not self.__vm:
 605             if expected_status_code == STATUS_DELETED:
 606                 return True
 607             else:
 608                 return False
 609
 610         status = nova_utils.get_server_status(self._nova, self.__vm)
 611         if not status:
 612             logger.warning('Cannot find instance with id - ' + self.__vm.id)
 613             return False
 614
 615         if status == 'ERROR':
 616             raise VmInstanceCreationError(
 617                 'Instance had an error during deployment')
 618         logger.debug(
 619             'Instance status [%s] is - %s', self.instance_settings.name,
 620             status)
 621         return status == expected_status_code
 622
 623     def vm_ssh_active(self, user_override=None, password=None, block=False,
 624                       timeout=None, poll_interval=POLL_INTERVAL):
 625         """
 626         Returns true when the VM can be accessed via SSH
 627         :param block: When true, thread will block until active or timeout
 628                       value in seconds has been exceeded (False)
 629         :param poll_interval: The polling interval
 630         :return: T/F
 631         """
 632         # sleep and wait for VM status change
 633         logger.info('Checking if VM is active')
 634
 635         if not timeout:
 636             timeout = self.instance_settings.ssh_connect_timeout
 637
 638         if self.vm_active(block=True):
 639             if block:
 640                 start = time.time()
 641             else:
 642                 start = time.time() - timeout
 643
 644             while timeout > time.time() - start:
 645                 status = self.__ssh_active(
 646                     user_override=user_override, password=password)
 647                 if status:
 648                     logger.info('SSH is active for VM instance')
 649                     return True
 650
 651                 logger.debug('Retry SSH connection in ' + str(
 652                     poll_interval) + ' seconds')
 653                 time.sleep(poll_interval)
 654                 logger.debug('SSH connection timeout in ' + str(
 655                     timeout - (time.time() - start)))
 656
 657         logger.error('Timeout attempting to connect with VM via SSH')
 658         return False
 659
 660     def __ssh_active(self, user_override=None, password=None):
 661         """
 662         Returns True when can create a SSH session else False
 663         :return: T/F
 664         """
 665         if len(self.__floating_ip_dict) > 0:
 666             ssh = self.ssh_client(
 667                 user_override=user_override, password=password)
 668             if ssh:
 669                 ssh.close()
 670                 return True
 671         return False
 672
 673     def cloud_init_complete(self, block=False, poll_interval=POLL_INTERVAL):
 674         """
 675         Returns true when the VM's cloud-init routine has completed.
 676         Note: this is currently done via SSH, therefore, if this instance does
 677               not have a Floating IP or a running SSH server, this routine
 678               will always return False or raise an Exception
 679         :param block: When true, thread will block until active or timeout
 680                       value in seconds has been exceeded (False)
 681         :param poll_interval: The polling interval
 682         :return: T/F
 683         """
 684         # sleep and wait for VM status change
 685         logger.info('Checking if cloud-init has completed')
 686
 687         timeout = self.instance_settings.cloud_init_timeout
 688
 689         if self.vm_active(block=True) and self.vm_ssh_active(block=True):
 690             if block:
 691                 start = time.time()
 692             else:
 693                 start = time.time() - timeout
 694
 695             while timeout > time.time() - start:
 696                 status = self.__cloud_init_complete()
 697                 if status:
 698                     logger.info('cloud-init complete for VM instance')
 699                     return True
 700
 701                 logger.debug('Retry cloud-init query in ' + str(
 702                     poll_interval) + ' seconds')
 703                 time.sleep(poll_interval)
 704                 logger.debug('cloud-init complete timeout in ' + str(
 705                     timeout - (time.time() - start)))
 706
 707         logger.error('Timeout waiting for cloud-init to complete')
 708         return False
 709
 710     def __cloud_init_complete(self):
 711         """
 712         Returns True when can create a SSH session else False
 713         :return: T/F
 714         """
 715         if len(self.__floating_ip_dict) > 0:
 716             ssh = self.ssh_client()
 717             if ssh:
 718                 stdin1, stdout1, sterr1 = ssh.exec_command(
 719                     'ls -l /var/lib/cloud/instance/boot-finished')
 720                 return stdout1.channel.recv_exit_status() == 0
 721         return False
 722
 723     def get_floating_ip(self, fip_name=None):
 724         """
 725         Returns the floating IP object byt name if found, else the first known,
 726         else None
 727         :param fip_name: the name of the floating IP to return
 728         :return: the SSH client or None
 729         """
 730         if fip_name and self.__floating_ip_dict.get(fip_name):
 731             return self.__floating_ip_dict.get(fip_name)
 732         else:
 733             return self.__get_first_provisioning_floating_ip()
 734
 735     def ssh_client(self, fip_name=None, user_override=None, password=None):
 736         """
 737         Returns an SSH client using the name or the first known floating IP if
 738         exists, else None
 739         :param fip_name: the name of the floating IP to return
 740         :param user_override: the username to use instead of the default
 741         :param password: the password to use instead of the private key
 742         :return: the SSH client or None
 743         """
 744         fip = self.get_floating_ip(fip_name)
 745
 746         ansible_user = self.get_image_user()
 747         if user_override:
 748             ansible_user = user_override
 749
 750         if password:
 751             private_key = None
 752         else:
 753             private_key = self.keypair_settings.private_filepath
 754
 755         if fip:
 756             return ansible_utils.ssh_client(
 757                 self.__get_first_provisioning_floating_ip().ip,
 758                 ansible_user,
 759                 private_key_filepath=private_key,
 760                 password=password,
 761                 proxy_settings=self._os_creds.proxy_settings)
 762         else:
 763             FloatingIPAllocationError(
 764                 'Cannot return an SSH client. No Floating IP configured')
 765
 766     def add_security_group(self, security_group):
 767         """
 768         Adds a security group to this VM. Call will block until VM is active.
 769         :param security_group: the SNAPS SecurityGroup domain object
 770         :return True if successful else False
 771         """
 772         self.vm_active(block=True)
 773
 774         if not security_group:
 775             logger.warning('Security group object is None, cannot add')
 776             return False
 777
 778         try:
 779             nova_utils.add_security_group(self._nova, self.get_vm_inst(),
 780                                           security_group.name)
 781             return True
 782         except NotFound as e:
 783             logger.warning('Security group not added - ' + str(e))
 784             return False
 785
 786     def remove_security_group(self, security_group):
 787         """
 788         Removes a security group to this VM. Call will block until VM is active
 789         :param security_group: the OpenStack security group object
 790         :return True if successful else False
 791         """
 792         self.vm_active(block=True)
 793
 794         if not security_group:
 795             logger.warning('Security group object is None, cannot remove')
 796             return False
 797
 798         try:
 799             nova_utils.remove_security_group(self._nova, self.get_vm_inst(),
 800                                              security_group)
 801             return True
 802         except NotFound as e:
 803             logger.warning('Security group not removed - ' + str(e))
 804             return False
 805
 806     def reboot(self, reboot_type=RebootType.soft):
 807         """
 808         Issues a reboot call
 809         :param reboot_type: instance of
 810                             snaps.openstack.utils.nova_utils.RebootType
 811                             enumeration
 812         :return:
 813         """
 814         nova_utils.reboot_server(
 815             self._nova, self.__vm, reboot_type=reboot_type)
 816
 817
 818 def generate_creator(os_creds, vm_inst, image_config, keypair_config=None):
 819     """
 820     Initializes an OpenStackVmInstance object
 821     :param os_creds: the OpenStack credentials
 822     :param vm_inst: the SNAPS-OO VmInst domain object
 823     :param image_config: the associated ImageConfig object
 824     :param keypair_config: the associated KeypairConfig object (optional)
 825     :return: an initialized OpenStackVmInstance object
 826     """
 827     nova = nova_utils.nova_client(os_creds)
 828     neutron = neutron_utils.neutron_client(os_creds)
 829     derived_inst_config = settings_utils.create_vm_inst_config(
 830         nova, neutron, vm_inst)
 831
 832     derived_inst_creator = OpenStackVmInstance(
 833         os_creds, derived_inst_config, image_config, keypair_config)
 834     derived_inst_creator.initialize()
 835     return derived_inst_creator
 836
 837
 838 class VmInstanceSettings(VmInstanceConfig):
 839     """
 840     Deprecated, use snaps.config.vm_inst.VmInstanceConfig instead
 841     """
 842     def __init__(self, **kwargs):
 843         from warnings import warn
 844         warn('Use snaps.config.vm_inst.VmInstanceConfig instead',
 845              DeprecationWarning)
 846         super(self.__class__, self).__init__(**kwargs)
 847
 848
 849 class FloatingIpSettings(FloatingIpConfig):
 850     """
 851     Deprecated, use snaps.config.vm_inst.FloatingIpConfig instead
 852     """
 853     def __init__(self, **kwargs):
 854         from warnings import warn
 855         warn('Use snaps.config.vm_inst.FloatingIpConfig instead',
 856              DeprecationWarning)
 857         super(self.__class__, self).__init__(**kwargs)
 858
 859
 860 class VmInstanceCreationError(Exception):
 861     """
 862     Exception to be thrown when an VM instance cannot be created
 863     """
 864
 865
 866 class FloatingIPAllocationError(Exception):
 867     """
 868     Exception to be thrown when an VM instance cannot allocate a floating IP
 869     """