nsb_installation: updates
[yardstick.git] / yardstick / common / openstack_utils.py
index 2df8fa5..d1223ed 100644 (file)
@@ -11,10 +11,12 @@ from __future__ import absolute_import
 
 import os
 import time
 
 import os
 import time
+import sys
 import logging
 
 from keystoneauth1 import loading
 from keystoneauth1 import session
 import logging
 
 from keystoneauth1 import loading
 from keystoneauth1 import session
+from cinderclient import client as cinderclient
 from novaclient import client as novaclient
 from glanceclient import client as glanceclient
 from neutronclient.neutron import client as neutronclient
 from novaclient import client as novaclient
 from glanceclient import client as glanceclient
 from neutronclient.neutron import client as neutronclient
@@ -79,14 +81,20 @@ def get_session():
     except KeyError:
         return session.Session(auth=auth)
     else:
     except KeyError:
         return session.Session(auth=auth)
     else:
+        insecure = os.getenv('OS_INSECURE', '').lower() == 'true'
+        cacert = False if insecure else cacert
         return session.Session(auth=auth, verify=cacert)
 
 
 def get_endpoint(service_type, endpoint_type='publicURL'):
     auth = get_session_auth()
         return session.Session(auth=auth, verify=cacert)
 
 
 def get_endpoint(service_type, endpoint_type='publicURL'):
     auth = get_session_auth()
+    # for multi-region, we need to specify region
+    # when finding the endpoint
     return get_session().get_endpoint(auth=auth,
                                       service_type=service_type,
     return get_session().get_endpoint(auth=auth,
                                       service_type=service_type,
-                                      endpoint_type=endpoint_type)
+                                      endpoint_type=endpoint_type,
+                                      region_name=os.environ.get(
+                                          "OS_REGION_NAME"))
 
 
 # *********************************************
 
 
 # *********************************************
@@ -102,6 +110,21 @@ def get_heat_api_version():     # pragma: no cover
         return api_version
 
 
         return api_version
 
 
+def get_cinder_client_version():      # pragma: no cover
+    try:
+        api_version = os.environ['OS_VOLUME_API_VERSION']
+    except KeyError:
+        return DEFAULT_API_VERSION
+    else:
+        log.info("OS_VOLUME_API_VERSION is set in env as '%s'", api_version)
+        return api_version
+
+
+def get_cinder_client():      # pragma: no cover
+    sess = get_session()
+    return cinderclient.Client(get_cinder_client_version(), session=sess)
+
+
 def get_nova_client_version():      # pragma: no cover
     try:
         api_version = os.environ['OS_COMPUTE_API_VERSION']
 def get_nova_client_version():      # pragma: no cover
     try:
         api_version = os.environ['OS_COMPUTE_API_VERSION']
@@ -242,84 +265,52 @@ def create_aggregate_with_host(nova_client, aggregate_name, av_zone,
         return True
 
 
         return True
 
 
-def create_instance(flavor_name,
-                    image_id,
-                    network_id,
-                    instance_name="instance-vm",
-                    confdrive=True,
-                    userdata=None,
-                    av_zone='',
-                    fixed_ip=None,
-                    files=None):    # pragma: no cover
-    nova_client = get_nova_client()
+def create_keypair(nova_client, name, key_path=None):    # pragma: no cover
     try:
     try:
-        flavor = nova_client.flavors.find(name=flavor_name)
-    except:
-        flavors = nova_client.flavors.list()
-        log.exception("Error: Flavor '%s' not found. Available flavors are: "
-                      "\n%s", flavor_name, flavors)
+        with open(key_path) as fpubkey:
+            keypair = get_nova_client().keypairs.create(name=name, public_key=fpubkey.read())
+            return keypair
+    except Exception:
+        log.exception("Error [create_keypair(nova_client)]")
+
+
+def create_instance(json_body):    # pragma: no cover
+    try:
+        return get_nova_client().servers.create(**json_body)
+    except Exception:
+        log.exception("Error create instance failed")
         return None
         return None
-    if fixed_ip is not None:
-        nics = {"net-id": network_id, "v4-fixed-ip": fixed_ip}
-    else:
-        nics = {"net-id": network_id}
-    if userdata is None:
-        instance = nova_client.servers.create(
-            name=instance_name,
-            flavor=flavor,
-            image=image_id,
-            nics=[nics],
-            availability_zone=av_zone,
-            files=files
-        )
-    else:
-        instance = nova_client.servers.create(
-            name=instance_name,
-            flavor=flavor,
-            image=image_id,
-            nics=[nics],
-            config_drive=confdrive,
-            userdata=userdata,
-            availability_zone=av_zone,
-            files=files
-        )
-    return instance
-
-
-def create_instance_and_wait_for_active(flavor_name,
-                                        image_id,
-                                        network_id,
-                                        instance_name="instance-vm",
-                                        config_drive=False,
-                                        userdata="",
-                                        av_zone='',
-                                        fixed_ip=None,
-                                        files=None):    # pragma: no cover
+
+
+def create_instance_and_wait_for_active(json_body):    # pragma: no cover
     SLEEP = 3
     VM_BOOT_TIMEOUT = 180
     nova_client = get_nova_client()
     SLEEP = 3
     VM_BOOT_TIMEOUT = 180
     nova_client = get_nova_client()
-    instance = create_instance(flavor_name,
-                               image_id,
-                               network_id,
-                               instance_name,
-                               config_drive,
-                               userdata,
-                               av_zone=av_zone,
-                               fixed_ip=fixed_ip,
-                               files=files)
+    instance = create_instance(json_body)
     count = VM_BOOT_TIMEOUT / SLEEP
     for n in range(count, -1, -1):
         status = get_instance_status(nova_client, instance)
         if status.lower() == "active":
             return instance
         elif status.lower() == "error":
     count = VM_BOOT_TIMEOUT / SLEEP
     for n in range(count, -1, -1):
         status = get_instance_status(nova_client, instance)
         if status.lower() == "active":
             return instance
         elif status.lower() == "error":
-            log.error("The instance %s went to ERROR status.", instance_name)
+            log.error("The instance went to ERROR status.")
             return None
         time.sleep(SLEEP)
             return None
         time.sleep(SLEEP)
-    log.error("Timeout booting the instance %s.", instance_name)
+    log.error("Timeout booting the instance.")
     return None
 
 
     return None
 
 
+def attach_server_volume(server_id, volume_id, device=None):    # pragma: no cover
+    try:
+        get_nova_client().volumes.create_server_volume(server_id, volume_id, device)
+    except Exception:
+        log.exception("Error [attach_server_volume(nova_client, '%s', '%s')]",
+                      server_id, volume_id)
+        return False
+    else:
+        return True
+
+
 def delete_instance(nova_client, instance_id):      # pragma: no cover
     try:
         nova_client.servers.force_delete(instance_id)
 def delete_instance(nova_client, instance_id):      # pragma: no cover
     try:
         nova_client.servers.force_delete(instance_id)
@@ -373,6 +364,15 @@ def get_server_by_name(name):   # pragma: no cover
         raise
 
 
         raise
 
 
+def create_flavor(name, ram, vcpus, disk, **kwargs):   # pragma: no cover
+    try:
+        return get_nova_client().flavors.create(name, ram, vcpus, disk, **kwargs)
+    except Exception:
+        log.exception("Error [create_flavor(nova_client, %s, %s, %s, %s, %s)]",
+                      name, ram, disk, vcpus, kwargs['is_public'])
+        return None
+
+
 def get_image_by_name(name):    # pragma: no cover
     images = get_nova_client().images.list()
     try:
 def get_image_by_name(name):    # pragma: no cover
     images = get_nova_client().images.list()
     try:
@@ -381,6 +381,16 @@ def get_image_by_name(name):    # pragma: no cover
         log.exception('No image matched')
 
 
         log.exception('No image matched')
 
 
+def get_flavor_id(nova_client, flavor_name):    # pragma: no cover
+    flavors = nova_client.flavors.list(detailed=True)
+    flavor_id = ''
+    for f in flavors:
+        if f.name == flavor_name:
+            flavor_id = f.id
+            break
+    return flavor_id
+
+
 def get_flavor_by_name(name):   # pragma: no cover
     flavors = get_nova_client().flavors.list()
     try:
 def get_flavor_by_name(name):   # pragma: no cover
     flavors = get_nova_client().flavors.list()
     try:
@@ -404,6 +414,25 @@ def check_status(status, name, iterations, interval):   # pragma: no cover
     return False
 
 
     return False
 
 
+def delete_flavor(flavor_id):    # pragma: no cover
+    try:
+        get_nova_client().flavors.delete(flavor_id)
+    except Exception:
+        log.exception("Error [delete_flavor(nova_client, %s)]", flavor_id)
+        return False
+    else:
+        return True
+
+
+def delete_keypair(nova_client, key):     # pragma: no cover
+    try:
+        nova_client.keypairs.delete(key=key)
+        return True
+    except Exception:
+        log.exception("Error [delete_keypair(nova_client)]")
+        return False
+
+
 # *********************************************
 #   NEUTRON
 # *********************************************
 # *********************************************
 #   NEUTRON
 # *********************************************
@@ -418,9 +447,312 @@ def get_port_id_by_ip(neutron_client, ip_address):      # pragma: no cover
         'fixed_ips') if j['ip_address'] == ip_address), None)
 
 
         'fixed_ips') if j['ip_address'] == ip_address), None)
 
 
+def create_neutron_net(neutron_client, json_body):      # pragma: no cover
+    try:
+        network = neutron_client.create_network(body=json_body)
+        return network['network']['id']
+    except Exception:
+        log.error("Error [create_neutron_net(neutron_client)]")
+        raise Exception("operation error")
+        return None
+
+
+def delete_neutron_net(neutron_client, network_id):      # pragma: no cover
+    try:
+        neutron_client.delete_network(network_id)
+        return True
+    except Exception:
+        log.error("Error [delete_neutron_net(neutron_client, '%s')]" % network_id)
+        return False
+
+
+def create_neutron_subnet(neutron_client, json_body):      # pragma: no cover
+    try:
+        subnet = neutron_client.create_subnet(body=json_body)
+        return subnet['subnets'][0]['id']
+    except Exception:
+        log.error("Error [create_neutron_subnet")
+        raise Exception("operation error")
+        return None
+
+
+def create_neutron_router(neutron_client, json_body):      # pragma: no cover
+    try:
+        router = neutron_client.create_router(json_body)
+        return router['router']['id']
+    except Exception:
+        log.error("Error [create_neutron_router(neutron_client)]")
+        raise Exception("operation error")
+        return None
+
+
+def delete_neutron_router(neutron_client, router_id):      # pragma: no cover
+    try:
+        neutron_client.delete_router(router=router_id)
+        return True
+    except Exception:
+        log.error("Error [delete_neutron_router(neutron_client, '%s')]" % router_id)
+        return False
+
+
+def remove_gateway_router(neutron_client, router_id):      # pragma: no cover
+    try:
+        neutron_client.remove_gateway_router(router_id)
+        return True
+    except Exception:
+        log.error("Error [remove_gateway_router(neutron_client, '%s')]" % router_id)
+        return False
+
+
+def remove_interface_router(neutron_client, router_id, subnet_id,
+                            **json_body):      # pragma: no cover
+    json_body.update({"subnet_id": subnet_id})
+    try:
+        neutron_client.remove_interface_router(router=router_id,
+                                               body=json_body)
+        return True
+    except Exception:
+        log.error("Error [remove_interface_router(neutron_client, '%s', "
+                  "'%s')]" % (router_id, subnet_id))
+        return False
+
+
+def create_floating_ip(neutron_client, extnet_id):      # pragma: no cover
+    props = {'floating_network_id': extnet_id}
+    try:
+        ip_json = neutron_client.create_floatingip({'floatingip': props})
+        fip_addr = ip_json['floatingip']['floating_ip_address']
+        fip_id = ip_json['floatingip']['id']
+    except Exception:
+        log.error("Error [create_floating_ip(neutron_client)]")
+        return None
+    return {'fip_addr': fip_addr, 'fip_id': fip_id}
+
+
+def delete_floating_ip(nova_client, floatingip_id):      # pragma: no cover
+    try:
+        nova_client.floating_ips.delete(floatingip_id)
+        return True
+    except Exception:
+        log.error("Error [delete_floating_ip(nova_client, '%s')]" % floatingip_id)
+        return False
+
+
+def get_security_groups(neutron_client):      # pragma: no cover
+    try:
+        security_groups = neutron_client.list_security_groups()[
+            'security_groups']
+        return security_groups
+    except Exception:
+        log.error("Error [get_security_groups(neutron_client)]")
+        return None
+
+
+def get_security_group_id(neutron_client, sg_name):      # pragma: no cover
+    security_groups = get_security_groups(neutron_client)
+    id = ''
+    for sg in security_groups:
+        if sg['name'] == sg_name:
+            id = sg['id']
+            break
+    return id
+
+
+def create_security_group(neutron_client, sg_name, sg_description):      # pragma: no cover
+    json_body = {'security_group': {'name': sg_name,
+                                    'description': sg_description}}
+    try:
+        secgroup = neutron_client.create_security_group(json_body)
+        return secgroup['security_group']
+    except Exception:
+        log.error("Error [create_security_group(neutron_client, '%s', "
+                  "'%s')]" % (sg_name, sg_description))
+        return None
+
+
+def create_secgroup_rule(neutron_client, sg_id, direction, protocol,
+                         port_range_min=None, port_range_max=None,
+                         **json_body):      # pragma: no cover
+    # We create a security group in 2 steps
+    # 1 - we check the format and set the json body accordingly
+    # 2 - we call neturon client to create the security group
+
+    # Format check
+    json_body.update({'security_group_rule': {'direction': direction,
+                     'security_group_id': sg_id, 'protocol': protocol}})
+    # parameters may be
+    # - both None => we do nothing
+    # - both Not None => we add them to the json description
+    # but one cannot be None is the other is not None
+    if (port_range_min is not None and port_range_max is not None):
+        # add port_range in json description
+        json_body['security_group_rule']['port_range_min'] = port_range_min
+        json_body['security_group_rule']['port_range_max'] = port_range_max
+        log.debug("Security_group format set (port range included)")
+    else:
+        # either both port range are set to None => do nothing
+        # or one is set but not the other => log it and return False
+        if port_range_min is None and port_range_max is None:
+            log.debug("Security_group format set (no port range mentioned)")
+        else:
+            log.error("Bad security group format."
+                      "One of the port range is not properly set:"
+                      "range min: {},"
+                      "range max: {}".format(port_range_min,
+                                             port_range_max))
+            return False
+
+    # Create security group using neutron client
+    try:
+        neutron_client.create_security_group_rule(json_body)
+        return True
+    except Exception:
+        log.exception("Impossible to create_security_group_rule,"
+                      "security group rule probably already exists")
+        return False
+
+
+def create_security_group_full(neutron_client,
+                               sg_name, sg_description):      # pragma: no cover
+    sg_id = get_security_group_id(neutron_client, sg_name)
+    if sg_id != '':
+        log.info("Using existing security group '%s'..." % sg_name)
+    else:
+        log.info("Creating security group  '%s'..." % sg_name)
+        SECGROUP = create_security_group(neutron_client,
+                                         sg_name,
+                                         sg_description)
+        if not SECGROUP:
+            log.error("Failed to create the security group...")
+            return None
+
+        sg_id = SECGROUP['id']
+
+        log.debug("Security group '%s' with ID=%s created successfully."
+                  % (SECGROUP['name'], sg_id))
+
+        log.debug("Adding ICMP rules in security group '%s'..."
+                  % sg_name)
+        if not create_secgroup_rule(neutron_client, sg_id,
+                                    'ingress', 'icmp'):
+            log.error("Failed to create the security group rule...")
+            return None
+
+        log.debug("Adding SSH rules in security group '%s'..."
+                  % sg_name)
+        if not create_secgroup_rule(
+                neutron_client, sg_id, 'ingress', 'tcp', '22', '22'):
+            log.error("Failed to create the security group rule...")
+            return None
+
+        if not create_secgroup_rule(
+                neutron_client, sg_id, 'egress', 'tcp', '22', '22'):
+            log.error("Failed to create the security group rule...")
+            return None
+    return sg_id
+
+
 # *********************************************
 #   GLANCE
 # *********************************************
 def get_image_id(glance_client, image_name):    # pragma: no cover
     images = glance_client.images.list()
     return next((i.id for i in images if i.name == image_name), None)
 # *********************************************
 #   GLANCE
 # *********************************************
 def get_image_id(glance_client, image_name):    # pragma: no cover
     images = glance_client.images.list()
     return next((i.id for i in images if i.name == image_name), None)
+
+
+def create_image(glance_client, image_name, file_path, disk_format,
+                 container_format, min_disk, min_ram, protected, tag,
+                 public, **kwargs):    # pragma: no cover
+    if not os.path.isfile(file_path):
+        log.error("Error: file %s does not exist." % file_path)
+        return None
+    try:
+        image_id = get_image_id(glance_client, image_name)
+        if image_id is not None:
+            log.info("Image %s already exists." % image_name)
+        else:
+            log.info("Creating image '%s' from '%s'...", image_name, file_path)
+
+            image = glance_client.images.create(name=image_name,
+                                                visibility=public,
+                                                disk_format=disk_format,
+                                                container_format=container_format,
+                                                min_disk=min_disk,
+                                                min_ram=min_ram,
+                                                tags=tag,
+                                                protected=protected,
+                                                **kwargs)
+            image_id = image.id
+            with open(file_path) as image_data:
+                glance_client.images.upload(image_id, image_data)
+        return image_id
+    except Exception:
+        log.error("Error [create_glance_image(glance_client, '%s', '%s', '%s')]",
+                  image_name, file_path, public)
+        return None
+
+
+def delete_image(glance_client, image_id):    # pragma: no cover
+    try:
+        glance_client.images.delete(image_id)
+
+    except Exception:
+        log.exception("Error [delete_flavor(glance_client, %s)]", image_id)
+        return False
+    else:
+        return True
+
+
+# *********************************************
+#   CINDER
+# *********************************************
+def get_volume_id(volume_name):    # pragma: no cover
+    volumes = get_cinder_client().volumes.list()
+    return next((v.id for v in volumes if v.name == volume_name), None)
+
+
+def create_volume(cinder_client, volume_name, volume_size,
+                  volume_image=False):    # pragma: no cover
+    try:
+        if volume_image:
+            volume = cinder_client.volumes.create(name=volume_name,
+                                                  size=volume_size,
+                                                  imageRef=volume_image)
+        else:
+            volume = cinder_client.volumes.create(name=volume_name,
+                                                  size=volume_size)
+        return volume
+    except Exception:
+        log.exception("Error [create_volume(cinder_client, %s)]",
+                      (volume_name, volume_size))
+        return None
+
+
+def delete_volume(cinder_client, volume_id, forced=False):      # pragma: no cover
+    try:
+        if forced:
+            try:
+                cinder_client.volumes.detach(volume_id)
+            except:
+                log.error(sys.exc_info()[0])
+            cinder_client.volumes.force_delete(volume_id)
+        else:
+            while True:
+                volume = get_cinder_client().volumes.get(volume_id)
+                if volume.status.lower() == 'available':
+                    break
+            cinder_client.volumes.delete(volume_id)
+        return True
+    except Exception:
+        log.exception("Error [delete_volume(cinder_client, '%s')]" % volume_id)
+        return False
+
+
+def detach_volume(server_id, volume_id):      # pragma: no cover
+    try:
+        get_nova_client().volumes.delete_server_volume(server_id, volume_id)
+        return True
+    except Exception:
+        log.exception("Error [detach_server_volume(nova_client, '%s', '%s')]",
+                      server_id, volume_id)
+        return False