yardstick/common/openstack_utils.py

   1 ##############################################################################
   2 # Copyright (c) 2016 Huawei Technologies Co.,Ltd and others.
   3 #
   4 # All rights reserved. This program and the accompanying materials
   5 # are made available under the terms of the Apache License, Version 2.0
   6 # which accompanies this distribution, and is available at
   7 # http://www.apache.org/licenses/LICENSE-2.0
   8 ##############################################################################
   9
  10 from __future__ import absolute_import
  11
  12 import os
  13 import time
  14 import logging
  15
  16 from keystoneauth1 import loading
  17 from keystoneauth1 import session
  18 from cinderclient import client as cinderclient
  19 from novaclient import client as novaclient
  20 from glanceclient import client as glanceclient
  21 from neutronclient.neutron import client as neutronclient
  22
  23 log = logging.getLogger(__name__)
  24
  25 DEFAULT_HEAT_API_VERSION = '1'
  26 DEFAULT_API_VERSION = '2'
  27
  28
  29 # *********************************************
  30 #   CREDENTIALS
  31 # *********************************************
  32 def get_credentials():
  33     """Returns a creds dictionary filled with parsed from env"""
  34     creds = {}
  35
  36     keystone_api_version = os.getenv('OS_IDENTITY_API_VERSION')
  37
  38     if keystone_api_version is None or keystone_api_version == '2':
  39         keystone_v3 = False
  40         tenant_env = 'OS_TENANT_NAME'
  41         tenant = 'tenant_name'
  42     else:
  43         keystone_v3 = True
  44         tenant_env = 'OS_PROJECT_NAME'
  45         tenant = 'project_name'
  46
  47     # The most common way to pass these info to the script is to do it
  48     # through environment variables.
  49     creds.update({
  50         "username": os.environ.get("OS_USERNAME"),
  51         "password": os.environ.get("OS_PASSWORD"),
  52         "auth_url": os.environ.get("OS_AUTH_URL"),
  53         tenant: os.environ.get(tenant_env)
  54     })
  55
  56     if keystone_v3:
  57         if os.getenv('OS_USER_DOMAIN_NAME') is not None:
  58             creds.update({
  59                 "user_domain_name": os.getenv('OS_USER_DOMAIN_NAME')
  60             })
  61         if os.getenv('OS_PROJECT_DOMAIN_NAME') is not None:
  62             creds.update({
  63                 "project_domain_name": os.getenv('OS_PROJECT_DOMAIN_NAME')
  64             })
  65
  66     return creds
  67
  68
  69 def get_session_auth():
  70     loader = loading.get_plugin_loader('password')
  71     creds = get_credentials()
  72     auth = loader.load_from_options(**creds)
  73     return auth
  74
  75
  76 def get_session():
  77     auth = get_session_auth()
  78     try:
  79         cacert = os.environ['OS_CACERT']
  80     except KeyError:
  81         return session.Session(auth=auth)
  82     else:
  83         insecure = os.getenv('OS_INSECURE', '').lower() == 'true'
  84         cacert = False if insecure else cacert
  85         return session.Session(auth=auth, verify=cacert)
  86
  87
  88 def get_endpoint(service_type, endpoint_type='publicURL'):
  89     auth = get_session_auth()
  90     # for multi-region, we need to specify region
  91     # when finding the endpoint
  92     return get_session().get_endpoint(auth=auth,
  93                                       service_type=service_type,
  94                                       endpoint_type=endpoint_type,
  95                                       region_name=os.environ.get(
  96                                           "OS_REGION_NAME"))
  97
  98
  99 # *********************************************
 100 #   CLIENTS
 101 # *********************************************
 102 def get_heat_api_version():     # pragma: no cover
 103     try:
 104         api_version = os.environ['HEAT_API_VERSION']
 105     except KeyError:
 106         return DEFAULT_HEAT_API_VERSION
 107     else:
 108         log.info("HEAT_API_VERSION is set in env as '%s'", api_version)
 109         return api_version
 110
 111
 112 def get_cinder_client_version():      # pragma: no cover
 113     try:
 114         api_version = os.environ['OS_VOLUME_API_VERSION']
 115     except KeyError:
 116         return DEFAULT_API_VERSION
 117     else:
 118         log.info("OS_VOLUME_API_VERSION is set in env as '%s'", api_version)
 119         return api_version
 120
 121
 122 def get_cinder_client():      # pragma: no cover
 123     sess = get_session()
 124     return cinderclient.Client(get_cinder_client_version(), session=sess)
 125
 126
 127 def get_nova_client_version():      # pragma: no cover
 128     try:
 129         api_version = os.environ['OS_COMPUTE_API_VERSION']
 130     except KeyError:
 131         return DEFAULT_API_VERSION
 132     else:
 133         log.info("OS_COMPUTE_API_VERSION is set in env as '%s'", api_version)
 134         return api_version
 135
 136
 137 def get_nova_client():      # pragma: no cover
 138     sess = get_session()
 139     return novaclient.Client(get_nova_client_version(), session=sess)
 140
 141
 142 def get_neutron_client_version():   # pragma: no cover
 143     try:
 144         api_version = os.environ['OS_NETWORK_API_VERSION']
 145     except KeyError:
 146         return DEFAULT_API_VERSION
 147     else:
 148         log.info("OS_NETWORK_API_VERSION is set in env as '%s'", api_version)
 149         return api_version
 150
 151
 152 def get_neutron_client():   # pragma: no cover
 153     sess = get_session()
 154     return neutronclient.Client(get_neutron_client_version(), session=sess)
 155
 156
 157 def get_glance_client_version():    # pragma: no cover
 158     try:
 159         api_version = os.environ['OS_IMAGE_API_VERSION']
 160     except KeyError:
 161         return DEFAULT_API_VERSION
 162     else:
 163         log.info("OS_IMAGE_API_VERSION is set in env as '%s'", api_version)
 164         return api_version
 165
 166
 167 def get_glance_client():    # pragma: no cover
 168     sess = get_session()
 169     return glanceclient.Client(get_glance_client_version(), session=sess)
 170
 171
 172 # *********************************************
 173 #   NOVA
 174 # *********************************************
 175 def get_instances(nova_client):     # pragma: no cover
 176     try:
 177         return nova_client.servers.list(search_opts={'all_tenants': 1})
 178     except Exception:
 179         log.exception("Error [get_instances(nova_client)]")
 180
 181
 182 def get_instance_status(nova_client, instance):     # pragma: no cover
 183     try:
 184         return nova_client.servers.get(instance.id).status
 185     except Exception:
 186         log.exception("Error [get_instance_status(nova_client)]")
 187
 188
 189 def get_instance_by_name(nova_client, instance_name):   # pragma: no cover
 190     try:
 191         return nova_client.servers.find(name=instance_name)
 192     except Exception:
 193         log.exception("Error [get_instance_by_name(nova_client, '%s')]",
 194                       instance_name)
 195
 196
 197 def get_aggregates(nova_client):    # pragma: no cover
 198     try:
 199         return nova_client.aggregates.list()
 200     except Exception:
 201         log.exception("Error [get_aggregates(nova_client)]")
 202
 203
 204 def get_availability_zones(nova_client):    # pragma: no cover
 205     try:
 206         return nova_client.availability_zones.list()
 207     except Exception:
 208         log.exception("Error [get_availability_zones(nova_client)]")
 209
 210
 211 def get_availability_zone_names(nova_client):   # pragma: no cover
 212     try:
 213         return [az.zoneName for az in get_availability_zones(nova_client)]
 214     except Exception:
 215         log.exception("Error [get_availability_zone_names(nova_client)]")
 216
 217
 218 def create_aggregate(nova_client, aggregate_name, av_zone):  # pragma: no cover
 219     try:
 220         nova_client.aggregates.create(aggregate_name, av_zone)
 221     except Exception:
 222         log.exception("Error [create_aggregate(nova_client, %s, %s)]",
 223                       aggregate_name, av_zone)
 224         return False
 225     else:
 226         return True
 227
 228
 229 def get_aggregate_id(nova_client, aggregate_name):      # pragma: no cover
 230     try:
 231         aggregates = get_aggregates(nova_client)
 232         _id = next((ag.id for ag in aggregates if ag.name == aggregate_name))
 233     except Exception:
 234         log.exception("Error [get_aggregate_id(nova_client, %s)]",
 235                       aggregate_name)
 236     else:
 237         return _id
 238
 239
 240 def add_host_to_aggregate(nova_client, aggregate_name,
 241                           compute_host):    # pragma: no cover
 242     try:
 243         aggregate_id = get_aggregate_id(nova_client, aggregate_name)
 244         nova_client.aggregates.add_host(aggregate_id, compute_host)
 245     except Exception:
 246         log.exception("Error [add_host_to_aggregate(nova_client, %s, %s)]",
 247                       aggregate_name, compute_host)
 248         return False
 249     else:
 250         return True
 251
 252
 253 def create_aggregate_with_host(nova_client, aggregate_name, av_zone,
 254                                compute_host):    # pragma: no cover
 255     try:
 256         create_aggregate(nova_client, aggregate_name, av_zone)
 257         add_host_to_aggregate(nova_client, aggregate_name, compute_host)
 258     except Exception:
 259         log.exception("Error [create_aggregate_with_host("
 260                       "nova_client, %s, %s, %s)]",
 261                       aggregate_name, av_zone, compute_host)
 262         return False
 263     else:
 264         return True
 265
 266
 267 def create_keypair(nova_client, name, key_path=None):    # pragma: no cover
 268     try:
 269         with open(key_path) as fpubkey:
 270             keypair = get_nova_client().keypairs.create(name=name, public_key=fpubkey.read())
 271             return keypair
 272     except Exception:
 273         log.exception("Error [create_keypair(nova_client)]")
 274
 275
 276 def create_instance(json_body):    # pragma: no cover
 277     try:
 278         return get_nova_client().servers.create(**json_body)
 279     except Exception:
 280         log.exception("Error create instance failed")
 281         return None
 282
 283
 284 def create_instance_and_wait_for_active(json_body):    # pragma: no cover
 285     SLEEP = 3
 286     VM_BOOT_TIMEOUT = 180
 287     nova_client = get_nova_client()
 288     instance = create_instance(json_body)
 289     count = VM_BOOT_TIMEOUT / SLEEP
 290     for n in range(count, -1, -1):
 291         status = get_instance_status(nova_client, instance)
 292         if status.lower() == "active":
 293             return instance
 294         elif status.lower() == "error":
 295             log.error("The instance went to ERROR status.")
 296             return None
 297         time.sleep(SLEEP)
 298     log.error("Timeout booting the instance.")
 299     return None
 300
 301
 302 def attach_server_volume(server_id, volume_id, device=None):    # pragma: no cover
 303     try:
 304         get_nova_client().volumes.create_server_volume(server_id, volume_id, device)
 305     except Exception:
 306         log.exception("Error [attach_server_volume(nova_client, '%s', '%s')]",
 307                       server_id, volume_id)
 308         return False
 309     else:
 310         return True
 311
 312
 313 def delete_instance(nova_client, instance_id):      # pragma: no cover
 314     try:
 315         nova_client.servers.force_delete(instance_id)
 316     except Exception:
 317         log.exception("Error [delete_instance(nova_client, '%s')]",
 318                       instance_id)
 319         return False
 320     else:
 321         return True
 322
 323
 324 def remove_host_from_aggregate(nova_client, aggregate_name,
 325                                compute_host):  # pragma: no cover
 326     try:
 327         aggregate_id = get_aggregate_id(nova_client, aggregate_name)
 328         nova_client.aggregates.remove_host(aggregate_id, compute_host)
 329     except Exception:
 330         log.exception("Error remove_host_from_aggregate(nova_client, %s, %s)",
 331                       aggregate_name, compute_host)
 332         return False
 333     else:
 334         return True
 335
 336
 337 def remove_hosts_from_aggregate(nova_client,
 338                                 aggregate_name):   # pragma: no cover
 339     aggregate_id = get_aggregate_id(nova_client, aggregate_name)
 340     hosts = nova_client.aggregates.get(aggregate_id).hosts
 341     assert(
 342         all(remove_host_from_aggregate(nova_client, aggregate_name, host)
 343             for host in hosts))
 344
 345
 346 def delete_aggregate(nova_client, aggregate_name):  # pragma: no cover
 347     try:
 348         remove_hosts_from_aggregate(nova_client, aggregate_name)
 349         nova_client.aggregates.delete(aggregate_name)
 350     except Exception:
 351         log.exception("Error [delete_aggregate(nova_client, %s)]",
 352                       aggregate_name)
 353         return False
 354     else:
 355         return True
 356
 357
 358 def get_server_by_name(name):   # pragma: no cover
 359     try:
 360         return get_nova_client().servers.list(search_opts={'name': name})[0]
 361     except IndexError:
 362         log.exception('Failed to get nova client')
 363         raise
 364
 365
 366 def create_flavor(name, ram, vcpus, disk, **kwargs):   # pragma: no cover
 367     try:
 368         return get_nova_client().flavors.create(name, ram, vcpus, disk, **kwargs)
 369     except Exception:
 370         log.exception("Error [create_flavor(nova_client, %s, %s, %s, %s, %s)]",
 371                       name, ram, disk, vcpus, kwargs['is_public'])
 372         return None
 373
 374
 375 def get_image_by_name(name):    # pragma: no cover
 376     images = get_nova_client().images.list()
 377     try:
 378         return next((a for a in images if a.name == name))
 379     except StopIteration:
 380         log.exception('No image matched')
 381
 382
 383 def get_flavor_id(nova_client, flavor_name):    # pragma: no cover
 384     flavors = nova_client.flavors.list(detailed=True)
 385     flavor_id = ''
 386     for f in flavors:
 387         if f.name == flavor_name:
 388             flavor_id = f.id
 389             break
 390     return flavor_id
 391
 392
 393 def get_flavor_by_name(name):   # pragma: no cover
 394     flavors = get_nova_client().flavors.list()
 395     try:
 396         return next((a for a in flavors if a.name == name))
 397     except StopIteration:
 398         log.exception('No flavor matched')
 399
 400
 401 def check_status(status, name, iterations, interval):   # pragma: no cover
 402     for i in range(iterations):
 403         try:
 404             server = get_server_by_name(name)
 405         except IndexError:
 406             log.error('Cannot found %s server', name)
 407             raise
 408
 409         if server.status == status:
 410             return True
 411
 412         time.sleep(interval)
 413     return False
 414
 415
 416 def delete_flavor(flavor_id):    # pragma: no cover
 417     try:
 418         get_nova_client().flavors.delete(flavor_id)
 419     except Exception:
 420         log.exception("Error [delete_flavor(nova_client, %s)]", flavor_id)
 421         return False
 422     else:
 423         return True
 424
 425
 426 # *********************************************
 427 #   NEUTRON
 428 # *********************************************
 429 def get_network_id(neutron_client, network_name):       # pragma: no cover
 430     networks = neutron_client.list_networks()['networks']
 431     return next((n['id'] for n in networks if n['name'] == network_name), None)
 432
 433
 434 def get_port_id_by_ip(neutron_client, ip_address):      # pragma: no cover
 435     ports = neutron_client.list_ports()['ports']
 436     return next((i['id'] for i in ports for j in i.get(
 437         'fixed_ips') if j['ip_address'] == ip_address), None)
 438
 439
 440 def create_neutron_net(neutron_client, json_body):      # pragma: no cover
 441     try:
 442         network = neutron_client.create_network(body=json_body)
 443         return network['network']['id']
 444     except Exception:
 445         log.error("Error [create_neutron_net(neutron_client)]")
 446         raise Exception("operation error")
 447         return None
 448
 449
 450 def create_neutron_subnet(neutron_client, json_body):      # pragma: no cover
 451     try:
 452         subnet = neutron_client.create_subnet(body=json_body)
 453         return subnet['subnets'][0]['id']
 454     except Exception:
 455         log.error("Error [create_neutron_subnet")
 456         raise Exception("operation error")
 457         return None
 458
 459
 460 def create_neutron_router(neutron_client, json_body):      # pragma: no cover
 461     try:
 462         router = neutron_client.create_router(json_body)
 463         return router['router']['id']
 464     except Exception:
 465         log.error("Error [create_neutron_router(neutron_client)]")
 466         raise Exception("operation error")
 467         return None
 468
 469
 470 def create_floating_ip(neutron_client, extnet_id):      # pragma: no cover
 471     props = {'floating_network_id': extnet_id}
 472     try:
 473         ip_json = neutron_client.create_floatingip({'floatingip': props})
 474         fip_addr = ip_json['floatingip']['floating_ip_address']
 475         fip_id = ip_json['floatingip']['id']
 476     except Exception:
 477         log.error("Error [create_floating_ip(neutron_client)]")
 478         return None
 479     return {'fip_addr': fip_addr, 'fip_id': fip_id}
 480
 481
 482 def get_security_groups(neutron_client):      # pragma: no cover
 483     try:
 484         security_groups = neutron_client.list_security_groups()[
 485             'security_groups']
 486         return security_groups
 487     except Exception:
 488         log.error("Error [get_security_groups(neutron_client)]")
 489         return None
 490
 491
 492 def get_security_group_id(neutron_client, sg_name):      # pragma: no cover
 493     security_groups = get_security_groups(neutron_client)
 494     id = ''
 495     for sg in security_groups:
 496         if sg['name'] == sg_name:
 497             id = sg['id']
 498             break
 499     return id
 500
 501
 502 def create_security_group(neutron_client, sg_name, sg_description):      # pragma: no cover
 503     json_body = {'security_group': {'name': sg_name,
 504                                     'description': sg_description}}
 505     try:
 506         secgroup = neutron_client.create_security_group(json_body)
 507         return secgroup['security_group']
 508     except Exception:
 509         log.error("Error [create_security_group(neutron_client, '%s', "
 510                   "'%s')]" % (sg_name, sg_description))
 511         return None
 512
 513
 514 def create_secgroup_rule(neutron_client, sg_id, direction, protocol,
 515                          port_range_min=None, port_range_max=None,
 516                          **json_body):      # pragma: no cover
 517     # We create a security group in 2 steps
 518     # 1 - we check the format and set the json body accordingly
 519     # 2 - we call neturon client to create the security group
 520
 521     # Format check
 522     json_body.update({'security_group_rule': {'direction': direction,
 523                      'security_group_id': sg_id, 'protocol': protocol}})
 524     # parameters may be
 525     # - both None => we do nothing
 526     # - both Not None => we add them to the json description
 527     # but one cannot be None is the other is not None
 528     if (port_range_min is not None and port_range_max is not None):
 529         # add port_range in json description
 530         json_body['security_group_rule']['port_range_min'] = port_range_min
 531         json_body['security_group_rule']['port_range_max'] = port_range_max
 532         log.debug("Security_group format set (port range included)")
 533     else:
 534         # either both port range are set to None => do nothing
 535         # or one is set but not the other => log it and return False
 536         if port_range_min is None and port_range_max is None:
 537             log.debug("Security_group format set (no port range mentioned)")
 538         else:
 539             log.error("Bad security group format."
 540                       "One of the port range is not properly set:"
 541                       "range min: {},"
 542                       "range max: {}".format(port_range_min,
 543                                              port_range_max))
 544             return False
 545
 546     # Create security group using neutron client
 547     try:
 548         neutron_client.create_security_group_rule(json_body)
 549         return True
 550     except Exception:
 551         log.exception("Impossible to create_security_group_rule,"
 552                       "security group rule probably already exists")
 553         return False
 554
 555
 556 def create_security_group_full(neutron_client,
 557                                sg_name, sg_description):      # pragma: no cover
 558     sg_id = get_security_group_id(neutron_client, sg_name)
 559     if sg_id != '':
 560         log.info("Using existing security group '%s'..." % sg_name)
 561     else:
 562         log.info("Creating security group  '%s'..." % sg_name)
 563         SECGROUP = create_security_group(neutron_client,
 564                                          sg_name,
 565                                          sg_description)
 566         if not SECGROUP:
 567             log.error("Failed to create the security group...")
 568             return None
 569
 570         sg_id = SECGROUP['id']
 571
 572         log.debug("Security group '%s' with ID=%s created successfully."
 573                   % (SECGROUP['name'], sg_id))
 574
 575         log.debug("Adding ICMP rules in security group '%s'..."
 576                   % sg_name)
 577         if not create_secgroup_rule(neutron_client, sg_id,
 578                                     'ingress', 'icmp'):
 579             log.error("Failed to create the security group rule...")
 580             return None
 581
 582         log.debug("Adding SSH rules in security group '%s'..."
 583                   % sg_name)
 584         if not create_secgroup_rule(
 585                 neutron_client, sg_id, 'ingress', 'tcp', '22', '22'):
 586             log.error("Failed to create the security group rule...")
 587             return None
 588
 589         if not create_secgroup_rule(
 590                 neutron_client, sg_id, 'egress', 'tcp', '22', '22'):
 591             log.error("Failed to create the security group rule...")
 592             return None
 593     return sg_id
 594
 595
 596 # *********************************************
 597 #   GLANCE
 598 # *********************************************
 599 def get_image_id(glance_client, image_name):    # pragma: no cover
 600     images = glance_client.images.list()
 601     return next((i.id for i in images if i.name == image_name), None)
 602
 603
 604 def create_image(glance_client, image_name, file_path, disk_format,
 605                  container_format, min_disk, min_ram, protected, tag,
 606                  public, **kwargs):    # pragma: no cover
 607     if not os.path.isfile(file_path):
 608         log.error("Error: file %s does not exist." % file_path)
 609         return None
 610     try:
 611         image_id = get_image_id(glance_client, image_name)
 612         if image_id is not None:
 613             log.info("Image %s already exists." % image_name)
 614         else:
 615             log.info("Creating image '%s' from '%s'...", image_name, file_path)
 616
 617             image = glance_client.images.create(name=image_name,
 618                                                 visibility=public,
 619                                                 disk_format=disk_format,
 620                                                 container_format=container_format,
 621                                                 min_disk=min_disk,
 622                                                 min_ram=min_ram,
 623                                                 tags=tag,
 624                                                 protected=protected,
 625                                                 **kwargs)
 626             image_id = image.id
 627             with open(file_path) as image_data:
 628                 glance_client.images.upload(image_id, image_data)
 629         return image_id
 630     except Exception:
 631         log.error("Error [create_glance_image(glance_client, '%s', '%s', '%s')]",
 632                   image_name, file_path, public)
 633         return None
 634
 635
 636 def delete_image(glance_client, image_id):    # pragma: no cover
 637     try:
 638         glance_client.images.delete(image_id)
 639
 640     except Exception:
 641         log.exception("Error [delete_flavor(glance_client, %s)]", image_id)
 642         return False
 643     else:
 644         return True
 645
 646
 647 # *********************************************
 648 #   CINDER
 649 # *********************************************
 650 def get_volume_id(volume_name):    # pragma: no cover
 651     volumes = get_cinder_client().volumes.list()
 652     return next((v.id for v in volumes if v.name == volume_name), None)
 653
 654
 655 def create_volume(cinder_client, volume_name, volume_size,
 656                   volume_image=False):    # pragma: no cover
 657     try:
 658         if volume_image:
 659             volume = cinder_client.volumes.create(name=volume_name,
 660                                                   size=volume_size,
 661                                                   imageRef=volume_image)
 662         else:
 663             volume = cinder_client.volumes.create(name=volume_name,
 664                                                   size=volume_size)
 665         return volume
 666     except Exception:
 667         log.exception("Error [create_volume(cinder_client, %s)]",
 668                       (volume_name, volume_size))
 669         return None