functest/utils/openstack_utils.py

   1 #!/usr/bin/env python
   2 #
   3 # jose.lausuch@ericsson.com
   4 # valentin.boucher@orange.com
   5 # All rights reserved. This program and the accompanying materials
   6 # are made available under the terms of the Apache License, Version 2.0
   7 # which accompanies this distribution, and is available at
   8 # http://www.apache.org/licenses/LICENSE-2.0
   9 #
  10
  11 import os
  12 import os.path
  13 import re
  14 import sys
  15 import time
  16
  17 from keystoneauth1 import loading
  18 from keystoneauth1 import session
  19 from cinderclient import client as cinderclient
  20 from glanceclient import client as glanceclient
  21 from heatclient import client as heatclient
  22 from novaclient import client as novaclient
  23 from keystoneclient import client as keystoneclient
  24 from neutronclient.neutron import client as neutronclient
  25
  26 import functest.utils.functest_logger as ft_logger
  27 import functest.utils.functest_utils as ft_utils
  28
  29 logger = ft_logger.Logger("openstack_utils").getLogger()
  30
  31 DEFAULT_API_VERSION = '2'
  32 DEFAULT_HEAT_API_VERSION = '1'
  33
  34
  35 # *********************************************
  36 #   CREDENTIALS
  37 # *********************************************
  38 class MissingEnvVar(Exception):
  39
  40     def __init__(self, var):
  41         self.var = var
  42
  43     def __str__(self):
  44         return str.format("Please set the mandatory env var: {}", self.var)
  45
  46
  47 def is_keystone_v3():
  48     keystone_api_version = os.getenv('OS_IDENTITY_API_VERSION')
  49     if (keystone_api_version is None or
  50             keystone_api_version == '2'):
  51         return False
  52     else:
  53         return True
  54
  55
  56 def get_rc_env_vars():
  57     env_vars = ['OS_AUTH_URL', 'OS_USERNAME', 'OS_PASSWORD']
  58     if is_keystone_v3():
  59         env_vars.extend(['OS_PROJECT_NAME',
  60                          'OS_USER_DOMAIN_NAME',
  61                          'OS_PROJECT_DOMAIN_NAME'])
  62     else:
  63         env_vars.extend(['OS_TENANT_NAME'])
  64     return env_vars
  65
  66
  67 def check_credentials():
  68     """
  69     Check if the OpenStack credentials (openrc) are sourced
  70     """
  71     env_vars = get_rc_env_vars()
  72     return all(map(lambda v: v in os.environ and os.environ[v], env_vars))
  73
  74
  75 def get_env_cred_dict():
  76     env_cred_dict = {
  77         'OS_USERNAME': 'username',
  78         'OS_PASSWORD': 'password',
  79         'OS_AUTH_URL': 'auth_url',
  80         'OS_TENANT_NAME': 'tenant_name',
  81         'OS_USER_DOMAIN_NAME': 'user_domain_name',
  82         'OS_PROJECT_DOMAIN_NAME': 'project_domain_name',
  83         'OS_PROJECT_NAME': 'project_name',
  84         'OS_ENDPOINT_TYPE': 'endpoint_type',
  85         'OS_REGION_NAME': 'region_name',
  86         'OS_CACERT': 'https_cacert'
  87     }
  88     return env_cred_dict
  89
  90
  91 def get_credentials(other_creds={}):
  92     """Returns a creds dictionary filled with parsed from env
  93     """
  94     creds = {}
  95     env_vars = get_rc_env_vars()
  96     env_cred_dict = get_env_cred_dict()
  97
  98     for envvar in env_vars:
  99         if os.getenv(envvar) is None:
 100             raise MissingEnvVar(envvar)
 101         else:
 102             creds_key = env_cred_dict.get(envvar)
 103             creds.update({creds_key: os.getenv(envvar)})
 104
 105     if 'tenant' in other_creds.keys():
 106         if is_keystone_v3():
 107             tenant = 'project_name'
 108         else:
 109             tenant = 'tenant_name'
 110         other_creds[tenant] = other_creds.pop('tenant')
 111
 112     creds.update(other_creds)
 113
 114     return creds
 115
 116
 117 def source_credentials(rc_file):
 118     with open(rc_file, "r") as f:
 119         for line in f:
 120             var = line.rstrip('"\n').replace('export ', '').split("=")
 121             # The two next lines should be modified as soon as rc_file
 122             # conforms with common rules. Be aware that it could induce
 123             # issues if value starts with '
 124             key = re.sub(r'^["\' ]*|[ \'"]*$', '', var[0])
 125             value = re.sub(r'^["\' ]*|[ \'"]*$', '', "".join(var[1:]))
 126             os.environ[key] = value
 127
 128
 129 def get_credentials_for_rally():
 130     creds = get_credentials()
 131     env_cred_dict = get_env_cred_dict()
 132     rally_conf = {"type": "ExistingCloud", "admin": {}}
 133     for key in creds:
 134         if key == 'auth_url':
 135             rally_conf[key] = creds[key]
 136         else:
 137             rally_conf['admin'][key] = creds[key]
 138
 139     endpoint_types = [('internalURL', 'internal'),
 140                       ('publicURL', 'public'), ('adminURL', 'admin')]
 141
 142     endpoint_type = os.getenv('OS_ENDPOINT_TYPE')
 143     if endpoint_type is not None:
 144         cred_key = env_cred_dict.get('OS_ENDPOINT_TYPE')
 145         for k, v in endpoint_types:
 146             if endpoint_type == k:
 147                 rally_conf[cred_key] = v
 148
 149     region_name = os.getenv('OS_REGION_NAME')
 150     if region_name is not None:
 151         cred_key = env_cred_dict.get('OS_REGION_NAME')
 152         rally_conf[cred_key] = region_name
 153
 154     cacert = os.getenv('OS_CACERT')
 155     if cacert is not None:
 156         cred_key = env_cred_dict.get('OS_CACERT')
 157         rally_conf[cred_key] = cacert
 158     return rally_conf
 159
 160
 161 def get_session_auth(other_creds={}):
 162     loader = loading.get_plugin_loader('password')
 163     creds = get_credentials(other_creds)
 164     auth = loader.load_from_options(**creds)
 165     return auth
 166
 167
 168 def get_endpoint(service_type, endpoint_type='publicURL'):
 169     auth = get_session_auth()
 170     return get_session().get_endpoint(auth=auth,
 171                                       service_type=service_type,
 172                                       endpoint_type=endpoint_type)
 173
 174
 175 def get_session(other_creds={}):
 176     auth = get_session_auth(other_creds)
 177     cacert = os.getenv('OS_CACERT')
 178     if cacert is not None:
 179         if not os.path.isfile(cacert):
 180             raise Exception("The 'OS_CACERT' environment"
 181                             "variable is set to %s but the file"
 182                             "does not exist.", cacert)
 183
 184     return session.Session(auth=auth, verify=cacert)
 185
 186
 187 # *********************************************
 188 #   CLIENTS
 189 # *********************************************
 190 def get_keystone_client_version():
 191     api_version = os.getenv('OS_IDENTITY_API_VERSION')
 192     if api_version is not None:
 193         logger.info("OS_IDENTITY_API_VERSION is set in env as '%s'",
 194                     api_version)
 195         return api_version
 196     return DEFAULT_API_VERSION
 197
 198
 199 def get_keystone_client(other_creds={}):
 200     sess = get_session(other_creds)
 201     return keystoneclient.Client(get_keystone_client_version(), session=sess)
 202
 203
 204 def get_nova_client_version():
 205     api_version = os.getenv('OS_COMPUTE_API_VERSION')
 206     if api_version is not None:
 207         logger.info("OS_COMPUTE_API_VERSION is set in env as '%s'",
 208                     api_version)
 209         return api_version
 210     return DEFAULT_API_VERSION
 211
 212
 213 def get_nova_client(other_creds={}):
 214     sess = get_session(other_creds)
 215     return novaclient.Client(get_nova_client_version(), session=sess)
 216
 217
 218 def get_cinder_client_version():
 219     api_version = os.getenv('OS_VOLUME_API_VERSION')
 220     if api_version is not None:
 221         logger.info("OS_VOLUME_API_VERSION is set in env as '%s'",
 222                     api_version)
 223         return api_version
 224     return DEFAULT_API_VERSION
 225
 226
 227 def get_cinder_client(other_creds={}):
 228     sess = get_session(other_creds)
 229     return cinderclient.Client(get_cinder_client_version(), session=sess)
 230
 231
 232 def get_neutron_client_version():
 233     api_version = os.getenv('OS_NETWORK_API_VERSION')
 234     if api_version is not None:
 235         logger.info("OS_NETWORK_API_VERSION is set in env as '%s'",
 236                     api_version)
 237         return api_version
 238     return DEFAULT_API_VERSION
 239
 240
 241 def get_neutron_client(other_creds={}):
 242     sess = get_session(other_creds)
 243     return neutronclient.Client(get_neutron_client_version(), session=sess)
 244
 245
 246 def get_glance_client_version():
 247     api_version = os.getenv('OS_IMAGE_API_VERSION')
 248     if api_version is not None:
 249         logger.info("OS_IMAGE_API_VERSION is set in env as '%s'", api_version)
 250         return api_version
 251     return DEFAULT_API_VERSION
 252
 253
 254 def get_glance_client(other_creds={}):
 255     sess = get_session(other_creds)
 256     return glanceclient.Client(get_glance_client_version(), session=sess)
 257
 258
 259 def get_heat_client_version():
 260     api_version = os.getenv('OS_ORCHESTRATION_API_VERSION')
 261     if api_version is not None:
 262         logger.info("OS_ORCHESTRATION_API_VERSION is set in env as '%s'",
 263                     api_version)
 264         return api_version
 265     return DEFAULT_HEAT_API_VERSION
 266
 267
 268 def get_heat_client(other_creds={}):
 269     sess = get_session(other_creds)
 270     return heatclient.Client(get_heat_client_version(), session=sess)
 271
 272
 273 # *********************************************
 274 #   NOVA
 275 # *********************************************
 276 def get_instances(nova_client):
 277     try:
 278         instances = nova_client.servers.list(search_opts={'all_tenants': 1})
 279         return instances
 280     except Exception, e:
 281         logger.error("Error [get_instances(nova_client)]: %s" % e)
 282         return None
 283
 284
 285 def get_instance_status(nova_client, instance):
 286     try:
 287         instance = nova_client.servers.get(instance.id)
 288         return instance.status
 289     except Exception, e:
 290         logger.error("Error [get_instance_status(nova_client)]: %s" % e)
 291         return None
 292
 293
 294 def get_instance_by_name(nova_client, instance_name):
 295     try:
 296         instance = nova_client.servers.find(name=instance_name)
 297         return instance
 298     except Exception, e:
 299         logger.error("Error [get_instance_by_name(nova_client, '%s')]: %s"
 300                      % (instance_name, e))
 301         return None
 302
 303
 304 def get_flavor_id(nova_client, flavor_name):
 305     flavors = nova_client.flavors.list(detailed=True)
 306     id = ''
 307     for f in flavors:
 308         if f.name == flavor_name:
 309             id = f.id
 310             break
 311     return id
 312
 313
 314 def get_flavor_id_by_ram_range(nova_client, min_ram, max_ram):
 315     flavors = nova_client.flavors.list(detailed=True)
 316     id = ''
 317     for f in flavors:
 318         if min_ram <= f.ram and f.ram <= max_ram:
 319             id = f.id
 320             break
 321     return id
 322
 323
 324 def get_aggregates(nova_client):
 325     try:
 326         aggregates = nova_client.aggregates.list()
 327         return aggregates
 328     except Exception, e:
 329         logger.error("Error [get_aggregates(nova_client)]: %s" % e)
 330         return None
 331
 332
 333 def get_aggregate_id(nova_client, aggregate_name):
 334     try:
 335         aggregates = get_aggregates(nova_client)
 336         _id = [ag.id for ag in aggregates if ag.name == aggregate_name][0]
 337         return _id
 338     except Exception, e:
 339         logger.error("Error [get_aggregate_id(nova_client, %s)]:"
 340                      " %s" % (aggregate_name, e))
 341         return None
 342
 343
 344 def get_availability_zones(nova_client):
 345     try:
 346         availability_zones = nova_client.availability_zones.list()
 347         return availability_zones
 348     except Exception, e:
 349         logger.error("Error [get_availability_zones(nova_client)]: %s" % e)
 350         return None
 351
 352
 353 def get_availability_zone_names(nova_client):
 354     try:
 355         az_names = [az.zoneName for az in get_availability_zones(nova_client)]
 356         return az_names
 357     except Exception, e:
 358         logger.error("Error [get_availability_zone_names(nova_client)]:"
 359                      " %s" % e)
 360         return None
 361
 362
 363 def create_flavor(nova_client, flavor_name, ram, disk, vcpus, public=True):
 364     try:
 365         flavor = nova_client.flavors.create(
 366             flavor_name, ram, vcpus, disk, is_public=public)
 367         try:
 368             extra_specs = ft_utils.get_functest_config(
 369                 'general.flavor_extra_specs')
 370             flavor.set_keys(extra_specs)
 371         except ValueError:
 372             # flavor extra specs are not configured, therefore skip the update
 373             pass
 374
 375     except Exception, e:
 376         logger.error("Error [create_flavor(nova_client, '%s', '%s', '%s', "
 377                      "'%s')]: %s" % (flavor_name, ram, disk, vcpus, e))
 378         return None
 379     return flavor.id
 380
 381
 382 def get_or_create_flavor(flavor_name, ram, disk, vcpus, public=True):
 383     flavor_exists = False
 384     nova_client = get_nova_client()
 385
 386     flavor_id = get_flavor_id(nova_client, flavor_name)
 387     if flavor_id != '':
 388         logger.info("Using existing flavor '%s'..." % flavor_name)
 389         flavor_exists = True
 390     else:
 391         logger.info("Creating flavor '%s' with '%s' RAM, '%s' disk size, "
 392                     "'%s' vcpus..." % (flavor_name, ram, disk, vcpus))
 393         flavor_id = create_flavor(
 394             nova_client, flavor_name, ram, disk, vcpus, public=public)
 395         if not flavor_id:
 396             logger.error("Failed to create flavor '%s'..." % (flavor_name))
 397         else:
 398             logger.debug("Flavor '%s' with ID=%s created successfully."
 399                          % (flavor_name, flavor_id))
 400
 401     return flavor_exists, flavor_id
 402
 403
 404 def get_floating_ips(nova_client):
 405     try:
 406         floating_ips = nova_client.floating_ips.list()
 407         return floating_ips
 408     except Exception, e:
 409         logger.error("Error [get_floating_ips(nova_client)]: %s" % e)
 410         return None
 411
 412
 413 def get_hypervisors(nova_client):
 414     try:
 415         nodes = []
 416         hypervisors = nova_client.hypervisors.list()
 417         for hypervisor in hypervisors:
 418             if hypervisor.state == "up":
 419                 nodes.append(hypervisor.hypervisor_hostname)
 420         return nodes
 421     except Exception, e:
 422         logger.error("Error [get_hypervisors(nova_client)]: %s" % e)
 423         return None
 424
 425
 426 def create_aggregate(nova_client, aggregate_name, av_zone):
 427     try:
 428         nova_client.aggregates.create(aggregate_name, av_zone)
 429         return True
 430     except Exception, e:
 431         logger.error("Error [create_aggregate(nova_client, %s, %s)]: %s"
 432                      % (aggregate_name, av_zone, e))
 433         return None
 434
 435
 436 def add_host_to_aggregate(nova_client, aggregate_name, compute_host):
 437     try:
 438         aggregate_id = get_aggregate_id(nova_client, aggregate_name)
 439         nova_client.aggregates.add_host(aggregate_id, compute_host)
 440         return True
 441     except Exception, e:
 442         logger.error("Error [add_host_to_aggregate(nova_client, %s, %s)]: %s"
 443                      % (aggregate_name, compute_host, e))
 444         return None
 445
 446
 447 def create_aggregate_with_host(
 448         nova_client, aggregate_name, av_zone, compute_host):
 449     try:
 450         create_aggregate(nova_client, aggregate_name, av_zone)
 451         add_host_to_aggregate(nova_client, aggregate_name, compute_host)
 452         return True
 453     except Exception, e:
 454         logger.error("Error [create_aggregate_with_host("
 455                      "nova_client, %s, %s, %s)]: %s"
 456                      % (aggregate_name, av_zone, compute_host, e))
 457         return None
 458
 459
 460 def create_instance(flavor_name,
 461                     image_id,
 462                     network_id,
 463                     instance_name="functest-vm",
 464                     confdrive=True,
 465                     userdata=None,
 466                     av_zone='',
 467                     fixed_ip=None,
 468                     files=None):
 469     nova_client = get_nova_client()
 470     try:
 471         flavor = nova_client.flavors.find(name=flavor_name)
 472     except:
 473         flavors = nova_client.flavors.list()
 474         logger.error("Error: Flavor '%s' not found. Available flavors are: "
 475                      "\n%s" % (flavor_name, flavors))
 476         return None
 477     if fixed_ip is not None:
 478         nics = {"net-id": network_id, "v4-fixed-ip": fixed_ip}
 479     else:
 480         nics = {"net-id": network_id}
 481     if userdata is None:
 482         instance = nova_client.servers.create(
 483             name=instance_name,
 484             flavor=flavor,
 485             image=image_id,
 486             nics=[nics],
 487             availability_zone=av_zone,
 488             files=files
 489         )
 490     else:
 491         instance = nova_client.servers.create(
 492             name=instance_name,
 493             flavor=flavor,
 494             image=image_id,
 495             nics=[nics],
 496             config_drive=confdrive,
 497             userdata=userdata,
 498             availability_zone=av_zone,
 499             files=files
 500         )
 501     return instance
 502
 503
 504 def create_instance_and_wait_for_active(flavor_name,
 505                                         image_id,
 506                                         network_id,
 507                                         instance_name="",
 508                                         config_drive=False,
 509                                         userdata="",
 510                                         av_zone='',
 511                                         fixed_ip=None,
 512                                         files=None):
 513     SLEEP = 3
 514     VM_BOOT_TIMEOUT = 180
 515     nova_client = get_nova_client()
 516     instance = create_instance(flavor_name,
 517                                image_id,
 518                                network_id,
 519                                instance_name,
 520                                config_drive,
 521                                userdata,
 522                                av_zone=av_zone,
 523                                fixed_ip=fixed_ip,
 524                                files=files)
 525     count = VM_BOOT_TIMEOUT / SLEEP
 526     for n in range(count, -1, -1):
 527         status = get_instance_status(nova_client, instance)
 528         if status.lower() == "active":
 529             return instance
 530         elif status.lower() == "error":
 531             logger.error("The instance %s went to ERROR status."
 532                          % instance_name)
 533             return None
 534         time.sleep(SLEEP)
 535     logger.error("Timeout booting the instance %s." % instance_name)
 536     return None
 537
 538
 539 def create_floating_ip(neutron_client):
 540     extnet_id = get_external_net_id(neutron_client)
 541     props = {'floating_network_id': extnet_id}
 542     try:
 543         ip_json = neutron_client.create_floatingip({'floatingip': props})
 544         fip_addr = ip_json['floatingip']['floating_ip_address']
 545         fip_id = ip_json['floatingip']['id']
 546     except Exception, e:
 547         logger.error("Error [create_floating_ip(neutron_client)]: %s" % e)
 548         return None
 549     return {'fip_addr': fip_addr, 'fip_id': fip_id}
 550
 551
 552 def add_floating_ip(nova_client, server_id, floatingip_addr):
 553     try:
 554         nova_client.servers.add_floating_ip(server_id, floatingip_addr)
 555         return True
 556     except Exception, e:
 557         logger.error("Error [add_floating_ip(nova_client, '%s', '%s')]: %s"
 558                      % (server_id, floatingip_addr, e))
 559         return False
 560
 561
 562 def delete_instance(nova_client, instance_id):
 563     try:
 564         nova_client.servers.force_delete(instance_id)
 565         return True
 566     except Exception, e:
 567         logger.error("Error [delete_instance(nova_client, '%s')]: %s"
 568                      % (instance_id, e))
 569         return False
 570
 571
 572 def delete_floating_ip(nova_client, floatingip_id):
 573     try:
 574         nova_client.floating_ips.delete(floatingip_id)
 575         return True
 576     except Exception, e:
 577         logger.error("Error [delete_floating_ip(nova_client, '%s')]: %s"
 578                      % (floatingip_id, e))
 579         return False
 580
 581
 582 def remove_host_from_aggregate(nova_client, aggregate_name, compute_host):
 583     try:
 584         aggregate_id = get_aggregate_id(nova_client, aggregate_name)
 585         nova_client.aggregates.remove_host(aggregate_id, compute_host)
 586         return True
 587     except Exception, e:
 588         logger.error("Error [remove_host_from_aggregate(nova_client, %s, %s)]:"
 589                      " %s" % (aggregate_name, compute_host, e))
 590         return False
 591
 592
 593 def remove_hosts_from_aggregate(nova_client, aggregate_name):
 594     aggregate_id = get_aggregate_id(nova_client, aggregate_name)
 595     hosts = nova_client.aggregates.get(aggregate_id).hosts
 596     assert(
 597         all(remove_host_from_aggregate(nova_client, aggregate_name, host)
 598             for host in hosts))
 599
 600
 601 def delete_aggregate(nova_client, aggregate_name):
 602     try:
 603         remove_hosts_from_aggregate(nova_client, aggregate_name)
 604         nova_client.aggregates.delete(aggregate_name)
 605         return True
 606     except Exception, e:
 607         logger.error("Error [delete_aggregate(nova_client, %s)]: %s"
 608                      % (aggregate_name, e))
 609         return False
 610
 611
 612 # *********************************************
 613 #   NEUTRON
 614 # *********************************************
 615 def get_network_list(neutron_client):
 616     network_list = neutron_client.list_networks()['networks']
 617     if len(network_list) == 0:
 618         return None
 619     else:
 620         return network_list
 621
 622
 623 def get_router_list(neutron_client):
 624     router_list = neutron_client.list_routers()['routers']
 625     if len(router_list) == 0:
 626         return None
 627     else:
 628         return router_list
 629
 630
 631 def get_port_list(neutron_client):
 632     port_list = neutron_client.list_ports()['ports']
 633     if len(port_list) == 0:
 634         return None
 635     else:
 636         return port_list
 637
 638
 639 def get_network_id(neutron_client, network_name):
 640     networks = neutron_client.list_networks()['networks']
 641     id = ''
 642     for n in networks:
 643         if n['name'] == network_name:
 644             id = n['id']
 645             break
 646     return id
 647
 648
 649 def get_subnet_id(neutron_client, subnet_name):
 650     subnets = neutron_client.list_subnets()['subnets']
 651     id = ''
 652     for s in subnets:
 653         if s['name'] == subnet_name:
 654             id = s['id']
 655             break
 656     return id
 657
 658
 659 def get_router_id(neutron_client, router_name):
 660     routers = neutron_client.list_routers()['routers']
 661     id = ''
 662     for r in routers:
 663         if r['name'] == router_name:
 664             id = r['id']
 665             break
 666     return id
 667
 668
 669 def get_private_net(neutron_client):
 670     # Checks if there is an existing shared private network
 671     networks = neutron_client.list_networks()['networks']
 672     if len(networks) == 0:
 673         return None
 674     for net in networks:
 675         if (net['router:external'] is False) and (net['shared'] is True):
 676             return net
 677     return None
 678
 679
 680 def get_external_net(neutron_client):
 681     for network in neutron_client.list_networks()['networks']:
 682         if network['router:external']:
 683             return network['name']
 684     return None
 685
 686
 687 def get_external_net_id(neutron_client):
 688     for network in neutron_client.list_networks()['networks']:
 689         if network['router:external']:
 690             return network['id']
 691     return None
 692
 693
 694 def check_neutron_net(neutron_client, net_name):
 695     for network in neutron_client.list_networks()['networks']:
 696         if network['name'] == net_name:
 697             for subnet in network['subnets']:
 698                 return True
 699     return False
 700
 701
 702 def create_neutron_net(neutron_client, name):
 703     json_body = {'network': {'name': name,
 704                              'admin_state_up': True}}
 705     try:
 706         network = neutron_client.create_network(body=json_body)
 707         network_dict = network['network']
 708         return network_dict['id']
 709     except Exception, e:
 710         logger.error("Error [create_neutron_net(neutron_client, '%s')]: %s"
 711                      % (name, e))
 712         return None
 713
 714
 715 def create_neutron_subnet(neutron_client, name, cidr, net_id):
 716     json_body = {'subnets': [{'name': name, 'cidr': cidr,
 717                               'ip_version': 4, 'network_id': net_id}]}
 718     try:
 719         subnet = neutron_client.create_subnet(body=json_body)
 720         return subnet['subnets'][0]['id']
 721     except Exception, e:
 722         logger.error("Error [create_neutron_subnet(neutron_client, '%s', "
 723                      "'%s', '%s')]: %s" % (name, cidr, net_id, e))
 724         return None
 725
 726
 727 def create_neutron_router(neutron_client, name):
 728     json_body = {'router': {'name': name, 'admin_state_up': True}}
 729     try:
 730         router = neutron_client.create_router(json_body)
 731         return router['router']['id']
 732     except Exception, e:
 733         logger.error("Error [create_neutron_router(neutron_client, '%s')]: %s"
 734                      % (name, e))
 735         return None
 736
 737
 738 def create_neutron_port(neutron_client, name, network_id, ip):
 739     json_body = {'port': {
 740                  'admin_state_up': True,
 741                  'name': name,
 742                  'network_id': network_id,
 743                  'fixed_ips': [{"ip_address": ip}]
 744                  }}
 745     try:
 746         port = neutron_client.create_port(body=json_body)
 747         return port['port']['id']
 748     except Exception, e:
 749         logger.error("Error [create_neutron_port(neutron_client, '%s', '%s', "
 750                      "'%s')]: %s" % (name, network_id, ip, e))
 751         return None
 752
 753
 754 def update_neutron_net(neutron_client, network_id, shared=False):
 755     json_body = {'network': {'shared': shared}}
 756     try:
 757         neutron_client.update_network(network_id, body=json_body)
 758         return True
 759     except Exception, e:
 760         logger.error("Error [update_neutron_net(neutron_client, '%s', '%s')]: "
 761                      "%s" % (network_id, str(shared), e))
 762         return False
 763
 764
 765 def update_neutron_port(neutron_client, port_id, device_owner):
 766     json_body = {'port': {
 767                  'device_owner': device_owner,
 768                  }}
 769     try:
 770         port = neutron_client.update_port(port=port_id,
 771                                           body=json_body)
 772         return port['port']['id']
 773     except Exception, e:
 774         logger.error("Error [update_neutron_port(neutron_client, '%s', '%s')]:"
 775                      " %s" % (port_id, device_owner, e))
 776         return None
 777
 778
 779 def add_interface_router(neutron_client, router_id, subnet_id):
 780     json_body = {"subnet_id": subnet_id}
 781     try:
 782         neutron_client.add_interface_router(router=router_id, body=json_body)
 783         return True
 784     except Exception, e:
 785         logger.error("Error [add_interface_router(neutron_client, '%s', "
 786                      "'%s')]: %s" % (router_id, subnet_id, e))
 787         return False
 788
 789
 790 def add_gateway_router(neutron_client, router_id):
 791     ext_net_id = get_external_net_id(neutron_client)
 792     router_dict = {'network_id': ext_net_id}
 793     try:
 794         neutron_client.add_gateway_router(router_id, router_dict)
 795         return True
 796     except Exception, e:
 797         logger.error("Error [add_gateway_router(neutron_client, '%s')]: %s"
 798                      % (router_id, e))
 799         return False
 800
 801
 802 def delete_neutron_net(neutron_client, network_id):
 803     try:
 804         neutron_client.delete_network(network_id)
 805         return True
 806     except Exception, e:
 807         logger.error("Error [delete_neutron_net(neutron_client, '%s')]: %s"
 808                      % (network_id, e))
 809         return False
 810
 811
 812 def delete_neutron_subnet(neutron_client, subnet_id):
 813     try:
 814         neutron_client.delete_subnet(subnet_id)
 815         return True
 816     except Exception, e:
 817         logger.error("Error [delete_neutron_subnet(neutron_client, '%s')]: %s"
 818                      % (subnet_id, e))
 819         return False
 820
 821
 822 def delete_neutron_router(neutron_client, router_id):
 823     try:
 824         neutron_client.delete_router(router=router_id)
 825         return True
 826     except Exception, e:
 827         logger.error("Error [delete_neutron_router(neutron_client, '%s')]: %s"
 828                      % (router_id, e))
 829         return False
 830
 831
 832 def delete_neutron_port(neutron_client, port_id):
 833     try:
 834         neutron_client.delete_port(port_id)
 835         return True
 836     except Exception, e:
 837         logger.error("Error [delete_neutron_port(neutron_client, '%s')]: %s"
 838                      % (port_id, e))
 839         return False
 840
 841
 842 def remove_interface_router(neutron_client, router_id, subnet_id):
 843     json_body = {"subnet_id": subnet_id}
 844     try:
 845         neutron_client.remove_interface_router(router=router_id,
 846                                                body=json_body)
 847         return True
 848     except Exception, e:
 849         logger.error("Error [remove_interface_router(neutron_client, '%s', "
 850                      "'%s')]: %s" % (router_id, subnet_id, e))
 851         return False
 852
 853
 854 def remove_gateway_router(neutron_client, router_id):
 855     try:
 856         neutron_client.remove_gateway_router(router_id)
 857         return True
 858     except Exception, e:
 859         logger.error("Error [remove_gateway_router(neutron_client, '%s')]: %s"
 860                      % (router_id, e))
 861         return False
 862
 863
 864 def create_network_full(neutron_client,
 865                         net_name,
 866                         subnet_name,
 867                         router_name,
 868                         cidr):
 869
 870     # Check if the network already exists
 871     network_id = get_network_id(neutron_client, net_name)
 872     subnet_id = get_subnet_id(neutron_client, subnet_name)
 873     router_id = get_router_id(neutron_client, router_name)
 874
 875     if network_id != '' and subnet_id != '' and router_id != '':
 876         logger.info("A network with name '%s' already exists..." % net_name)
 877     else:
 878         neutron_client.format = 'json'
 879         logger.info('Creating neutron network %s...' % net_name)
 880         network_id = create_neutron_net(neutron_client, net_name)
 881
 882         if not network_id:
 883             return False
 884
 885         logger.debug("Network '%s' created successfully" % network_id)
 886         logger.debug('Creating Subnet....')
 887         subnet_id = create_neutron_subnet(neutron_client, subnet_name,
 888                                           cidr, network_id)
 889         if not subnet_id:
 890             return None
 891
 892         logger.debug("Subnet '%s' created successfully" % subnet_id)
 893         logger.debug('Creating Router...')
 894         router_id = create_neutron_router(neutron_client, router_name)
 895
 896         if not router_id:
 897             return None
 898
 899         logger.debug("Router '%s' created successfully" % router_id)
 900         logger.debug('Adding router to subnet...')
 901
 902         if not add_interface_router(neutron_client, router_id, subnet_id):
 903             return None
 904
 905         logger.debug("Interface added successfully.")
 906
 907         logger.debug('Adding gateway to router...')
 908         if not add_gateway_router(neutron_client, router_id):
 909             return None
 910
 911         logger.debug("Gateway added successfully.")
 912
 913     network_dic = {'net_id': network_id,
 914                    'subnet_id': subnet_id,
 915                    'router_id': router_id}
 916     return network_dic
 917
 918
 919 def create_shared_network_full(net_name, subnt_name, router_name, subnet_cidr):
 920     neutron_client = get_neutron_client()
 921
 922     network_dic = create_network_full(neutron_client,
 923                                       net_name,
 924                                       subnt_name,
 925                                       router_name,
 926                                       subnet_cidr)
 927     if network_dic:
 928         if not update_neutron_net(neutron_client,
 929                                   network_dic['net_id'],
 930                                   shared=True):
 931             logger.error("Failed to update network %s..." % net_name)
 932             return None
 933         else:
 934             logger.debug("Network '%s' is available..." % net_name)
 935     else:
 936         logger.error("Network %s creation failed" % net_name)
 937         return None
 938     return network_dic
 939
 940
 941 def create_bgpvpn(neutron_client, **kwargs):
 942     # route_distinguishers
 943     # route_targets
 944     json_body = {"bgpvpn": kwargs}
 945     return neutron_client.create_bgpvpn(json_body)
 946
 947
 948 def create_network_association(neutron_client, bgpvpn_id, neutron_network_id):
 949     json_body = {"network_association": {"network_id": neutron_network_id}}
 950     return neutron_client.create_network_association(bgpvpn_id, json_body)
 951
 952
 953 def create_router_association(neutron_client, bgpvpn_id, router_id):
 954     json_body = {"router_association": {"router_id": router_id}}
 955     return neutron_client.create_router_association(bgpvpn_id, json_body)
 956
 957
 958 def update_bgpvpn(neutron_client, bgpvpn_id, **kwargs):
 959     json_body = {"bgpvpn": kwargs}
 960     return neutron_client.update_bgpvpn(bgpvpn_id, json_body)
 961
 962
 963 def delete_bgpvpn(neutron_client, bgpvpn_id):
 964     return neutron_client.delete_bgpvpn(bgpvpn_id)
 965
 966
 967 def get_bgpvpn(neutron_client, bgpvpn_id):
 968     return neutron_client.show_bgpvpn(bgpvpn_id)
 969
 970
 971 def get_bgpvpn_routers(neutron_client, bgpvpn_id):
 972     return get_bgpvpn(neutron_client, bgpvpn_id)['bgpvpn']['routers']
 973
 974
 975 def get_bgpvpn_networks(neutron_client, bgpvpn_id):
 976     return get_bgpvpn(neutron_client, bgpvpn_id)['bgpvpn']['networks']
 977
 978 # *********************************************
 979 #   SEC GROUPS
 980 # *********************************************
 981
 982
 983 def get_security_groups(neutron_client):
 984     try:
 985         security_groups = neutron_client.list_security_groups()[
 986             'security_groups']
 987         return security_groups
 988     except Exception, e:
 989         logger.error("Error [get_security_groups(neutron_client)]: %s" % e)
 990         return None
 991
 992
 993 def get_security_group_id(neutron_client, sg_name):
 994     security_groups = get_security_groups(neutron_client)
 995     id = ''
 996     for sg in security_groups:
 997         if sg['name'] == sg_name:
 998             id = sg['id']
 999             break
1000     return id
1001
1002
1003 def create_security_group(neutron_client, sg_name, sg_description):
1004     json_body = {'security_group': {'name': sg_name,
1005                                     'description': sg_description}}
1006     try:
1007         secgroup = neutron_client.create_security_group(json_body)
1008         return secgroup['security_group']
1009     except Exception, e:
1010         logger.error("Error [create_security_group(neutron_client, '%s', "
1011                      "'%s')]: %s" % (sg_name, sg_description, e))
1012         return None
1013
1014
1015 def create_secgroup_rule(neutron_client, sg_id, direction, protocol,
1016                          port_range_min=None, port_range_max=None):
1017     # We create a security group in 2 steps
1018     # 1 - we check the format and set the json body accordingly
1019     # 2 - we call neturon client to create the security group
1020
1021     # Format check
1022     json_body = {'security_group_rule': {'direction': direction,
1023                                          'security_group_id': sg_id,
1024                                          'protocol': protocol}}
1025     # parameters may be
1026     # - both None => we do nothing
1027     # - both Not None => we add them to the json description
1028     # but one cannot be None is the other is not None
1029     if (port_range_min is not None and port_range_max is not None):
1030         # add port_range in json description
1031         json_body['security_group_rule']['port_range_min'] = port_range_min
1032         json_body['security_group_rule']['port_range_max'] = port_range_max
1033         logger.debug("Security_group format set (port range included)")
1034     else:
1035         # either both port range are set to None => do nothing
1036         # or one is set but not the other => log it and return False
1037         if port_range_min is None and port_range_max is None:
1038             logger.debug("Security_group format set (no port range mentioned)")
1039         else:
1040             logger.error("Bad security group format."
1041                          "One of the port range is not properly set:"
1042                          "range min: {},"
1043                          "range max: {}".format(port_range_min,
1044                                                 port_range_max))
1045             return False
1046
1047     # Create security group using neutron client
1048     try:
1049         neutron_client.create_security_group_rule(json_body)
1050         return True
1051     except:
1052         logger.exception("Impossible to create_security_group_rule,"
1053                          "security group rule probably already exists")
1054         return False
1055
1056
1057 def create_security_group_full(neutron_client,
1058                                sg_name, sg_description):
1059     sg_id = get_security_group_id(neutron_client, sg_name)
1060     if sg_id != '':
1061         logger.info("Using existing security group '%s'..." % sg_name)
1062     else:
1063         logger.info("Creating security group  '%s'..." % sg_name)
1064         SECGROUP = create_security_group(neutron_client,
1065                                          sg_name,
1066                                          sg_description)
1067         if not SECGROUP:
1068             logger.error("Failed to create the security group...")
1069             return None
1070
1071         sg_id = SECGROUP['id']
1072
1073         logger.debug("Security group '%s' with ID=%s created successfully."
1074                      % (SECGROUP['name'], sg_id))
1075
1076         logger.debug("Adding ICMP rules in security group '%s'..."
1077                      % sg_name)
1078         if not create_secgroup_rule(neutron_client, sg_id,
1079                                     'ingress', 'icmp'):
1080             logger.error("Failed to create the security group rule...")
1081             return None
1082
1083         logger.debug("Adding SSH rules in security group '%s'..."
1084                      % sg_name)
1085         if not create_secgroup_rule(
1086                 neutron_client, sg_id, 'ingress', 'tcp', '22', '22'):
1087             logger.error("Failed to create the security group rule...")
1088             return None
1089
1090         if not create_secgroup_rule(
1091                 neutron_client, sg_id, 'egress', 'tcp', '22', '22'):
1092             logger.error("Failed to create the security group rule...")
1093             return None
1094     return sg_id
1095
1096
1097 def add_secgroup_to_instance(nova_client, instance_id, secgroup_id):
1098     try:
1099         nova_client.servers.add_security_group(instance_id, secgroup_id)
1100         return True
1101     except Exception, e:
1102         logger.error("Error [add_secgroup_to_instance(nova_client, '%s', "
1103                      "'%s')]: %s" % (instance_id, secgroup_id, e))
1104         return False
1105
1106
1107 def update_sg_quota(neutron_client, tenant_id, sg_quota, sg_rule_quota):
1108     json_body = {"quota": {
1109         "security_group": sg_quota,
1110         "security_group_rule": sg_rule_quota
1111     }}
1112
1113     try:
1114         neutron_client.update_quota(tenant_id=tenant_id,
1115                                     body=json_body)
1116         return True
1117     except Exception, e:
1118         logger.error("Error [update_sg_quota(neutron_client, '%s', '%s', "
1119                      "'%s')]: %s" % (tenant_id, sg_quota, sg_rule_quota, e))
1120         return False
1121
1122
1123 def delete_security_group(neutron_client, secgroup_id):
1124     try:
1125         neutron_client.delete_security_group(secgroup_id)
1126         return True
1127     except Exception, e:
1128         logger.error("Error [delete_security_group(neutron_client, '%s')]: %s"
1129                      % (secgroup_id, e))
1130         return False
1131
1132
1133 # *********************************************
1134 #   GLANCE
1135 # *********************************************
1136 def get_images(nova_client):
1137     try:
1138         images = nova_client.images.list()
1139         return images
1140     except Exception, e:
1141         logger.error("Error [get_images]: %s" % e)
1142         return None
1143
1144
1145 def get_image_id(glance_client, image_name):
1146     images = glance_client.images.list()
1147     id = ''
1148     for i in images:
1149         if i.name == image_name:
1150             id = i.id
1151             break
1152     return id
1153
1154
1155 def create_glance_image(glance_client, image_name, file_path, disk="qcow2",
1156                         container="bare", public="public"):
1157     if not os.path.isfile(file_path):
1158         logger.error("Error: file %s does not exist." % file_path)
1159         return None
1160     try:
1161         image_id = get_image_id(glance_client, image_name)
1162         if image_id != '':
1163             logger.info("Image %s already exists." % image_name)
1164         else:
1165             logger.info("Creating image '%s' from '%s'..." % (image_name,
1166                                                               file_path))
1167
1168             image = glance_client.images.create(name=image_name,
1169                                                 visibility=public,
1170                                                 disk_format=disk,
1171                                                 container_format=container)
1172             image_id = image.id
1173             with open(file_path) as image_data:
1174                 glance_client.images.upload(image_id, image_data)
1175         return image_id
1176     except Exception, e:
1177         logger.error("Error [create_glance_image(glance_client, '%s', '%s', "
1178                      "'%s')]: %s" % (image_name, file_path, public, e))
1179         return None
1180
1181
1182 def get_or_create_image(name, path, format):
1183     image_exists = False
1184     glance_client = get_glance_client()
1185
1186     image_id = get_image_id(glance_client, name)
1187     if image_id != '':
1188         logger.info("Using existing image '%s'..." % name)
1189         image_exists = True
1190     else:
1191         logger.info("Creating image '%s' from '%s'..." % (name, path))
1192         image_id = create_glance_image(glance_client, name, path, format)
1193         if not image_id:
1194             logger.error("Failed to create a Glance image...")
1195         else:
1196             logger.debug("Image '%s' with ID=%s created successfully."
1197                          % (name, image_id))
1198
1199     return image_exists, image_id
1200
1201
1202 def delete_glance_image(nova_client, image_id):
1203     try:
1204         nova_client.images.delete(image_id)
1205         return True
1206     except Exception, e:
1207         logger.error("Error [delete_glance_image(nova_client, '%s')]: %s"
1208                      % (image_id, e))
1209         return False
1210
1211
1212 # *********************************************
1213 #   CINDER
1214 # *********************************************
1215 def get_volumes(cinder_client):
1216     try:
1217         volumes = cinder_client.volumes.list(search_opts={'all_tenants': 1})
1218         return volumes
1219     except Exception, e:
1220         logger.error("Error [get_volumes(cinder_client)]: %s" % e)
1221         return None
1222
1223
1224 def list_volume_types(cinder_client, public=True, private=True):
1225     try:
1226         volume_types = cinder_client.volume_types.list()
1227         if not public:
1228             volume_types = [vt for vt in volume_types if not vt.is_public]
1229         if not private:
1230             volume_types = [vt for vt in volume_types if vt.is_public]
1231         return volume_types
1232     except Exception, e:
1233         logger.error("Error [list_volume_types(cinder_client)]: %s" % e)
1234         return None
1235
1236
1237 def create_volume_type(cinder_client, name):
1238     try:
1239         volume_type = cinder_client.volume_types.create(name)
1240         return volume_type
1241     except Exception, e:
1242         logger.error("Error [create_volume_type(cinder_client, '%s')]: %s"
1243                      % (name, e))
1244         return None
1245
1246
1247 def update_cinder_quota(cinder_client, tenant_id, vols_quota,
1248                         snapshots_quota, gigabytes_quota):
1249     quotas_values = {"volumes": vols_quota,
1250                      "snapshots": snapshots_quota,
1251                      "gigabytes": gigabytes_quota}
1252
1253     try:
1254         cinder_client.quotas.update(tenant_id, **quotas_values)
1255         return True
1256     except Exception, e:
1257         logger.error("Error [update_cinder_quota(cinder_client, '%s', '%s', "
1258                      "'%s' '%s')]: %s" % (tenant_id, vols_quota,
1259                                           snapshots_quota, gigabytes_quota, e))
1260         return False
1261
1262
1263 def delete_volume(cinder_client, volume_id, forced=False):
1264     try:
1265         if forced:
1266             try:
1267                 cinder_client.volumes.detach(volume_id)
1268             except:
1269                 logger.error(sys.exc_info()[0])
1270             cinder_client.volumes.force_delete(volume_id)
1271         else:
1272             cinder_client.volumes.delete(volume_id)
1273         return True
1274     except Exception, e:
1275         logger.error("Error [delete_volume(cinder_client, '%s', '%s')]: %s"
1276                      % (volume_id, str(forced), e))
1277         return False
1278
1279
1280 def delete_volume_type(cinder_client, volume_type):
1281     try:
1282         cinder_client.volume_types.delete(volume_type)
1283         return True
1284     except Exception, e:
1285         logger.error("Error [delete_volume_type(cinder_client, '%s')]: %s"
1286                      % (volume_type, e))
1287         return False
1288
1289
1290 # *********************************************
1291 #   KEYSTONE
1292 # *********************************************
1293 def get_tenants(keystone_client):
1294     try:
1295         if is_keystone_v3():
1296             tenants = keystone_client.projects.list()
1297         else:
1298             tenants = keystone_client.tenants.list()
1299         return tenants
1300     except Exception, e:
1301         logger.error("Error [get_tenants(keystone_client)]: %s" % e)
1302         return None
1303
1304
1305 def get_users(keystone_client):
1306     try:
1307         users = keystone_client.users.list()
1308         return users
1309     except Exception, e:
1310         logger.error("Error [get_users(keystone_client)]: %s" % e)
1311         return None
1312
1313
1314 def get_tenant_id(keystone_client, tenant_name):
1315     tenants = get_tenants(keystone_client)
1316     id = ''
1317     for t in tenants:
1318         if t.name == tenant_name:
1319             id = t.id
1320             break
1321     return id
1322
1323
1324 def get_user_id(keystone_client, user_name):
1325     users = get_users(keystone_client)
1326     id = ''
1327     for u in users:
1328         if u.name == user_name:
1329             id = u.id
1330             break
1331     return id
1332
1333
1334 def get_role_id(keystone_client, role_name):
1335     roles = keystone_client.roles.list()
1336     id = ''
1337     for r in roles:
1338         if r.name == role_name:
1339             id = r.id
1340             break
1341     return id
1342
1343
1344 def create_tenant(keystone_client, tenant_name, tenant_description):
1345     try:
1346         if is_keystone_v3():
1347             tenant = keystone_client.projects.create(
1348                 name=tenant_name,
1349                 description=tenant_description,
1350                 domain="default",
1351                 enabled=True)
1352         else:
1353             tenant = keystone_client.tenants.create(tenant_name,
1354                                                     tenant_description,
1355                                                     enabled=True)
1356         return tenant.id
1357     except Exception, e:
1358         logger.error("Error [create_tenant(keystone_client, '%s', '%s')]: %s"
1359                      % (tenant_name, tenant_description, e))
1360         return None
1361
1362
1363 def create_user(keystone_client, user_name, user_password,
1364                 user_email, tenant_id):
1365     try:
1366         if is_keystone_v3():
1367             user = keystone_client.users.create(name=user_name,
1368                                                 password=user_password,
1369                                                 email=user_email,
1370                                                 project_id=tenant_id,
1371                                                 enabled=True)
1372         else:
1373             user = keystone_client.users.create(user_name,
1374                                                 user_password,
1375                                                 user_email,
1376                                                 tenant_id,
1377                                                 enabled=True)
1378         return user.id
1379     except Exception, e:
1380         logger.error("Error [create_user(keystone_client, '%s', '%s', '%s'"
1381                      "'%s')]: %s" % (user_name, user_password,
1382                                      user_email, tenant_id, e))
1383         return None
1384
1385
1386 def add_role_user(keystone_client, user_id, role_id, tenant_id):
1387     try:
1388         if is_keystone_v3():
1389             keystone_client.roles.grant(role=role_id,
1390                                         user=user_id,
1391                                         project=tenant_id)
1392         else:
1393             keystone_client.roles.add_user_role(user_id, role_id, tenant_id)
1394         return True
1395     except Exception, e:
1396         logger.error("Error [add_role_user(keystone_client, '%s', '%s'"
1397                      "'%s')]: %s " % (user_id, role_id, tenant_id, e))
1398         return False
1399
1400
1401 def delete_tenant(keystone_client, tenant_id):
1402     try:
1403         if is_keystone_v3():
1404             keystone_client.projects.delete(tenant_id)
1405         else:
1406             keystone_client.tenants.delete(tenant_id)
1407         return True
1408     except Exception, e:
1409         logger.error("Error [delete_tenant(keystone_client, '%s')]: %s"
1410                      % (tenant_id, e))
1411         return False
1412
1413
1414 def delete_user(keystone_client, user_id):
1415     try:
1416         keystone_client.users.delete(user_id)
1417         return True
1418     except Exception, e:
1419         logger.error("Error [delete_user(keystone_client, '%s')]: %s"
1420                      % (user_id, e))
1421         return False
1422
1423
1424 # *********************************************
1425 #   HEAT
1426 # *********************************************
1427 def get_resource(heat_client, stack_id, resource):
1428     try:
1429         resources = heat_client.resources.get(stack_id, resource)
1430         return resources
1431     except Exception, e:
1432         logger.error("Error [get_resource]: %s" % e)
1433         return None