sdnvpn/lib/openstack_utils.py

   1 #!/usr/bin/env python
   2 #
   3 # jose.lausuch@ericsson.com
   4 # valentin.boucher@orange.com
   5 # All rights reserved. This program and the accompanying materials
   6 # are made available under the terms of the Apache License, Version 2.0
   7 # which accompanies this distribution, and is available at
   8 # http://www.apache.org/licenses/LICENSE-2.0
   9 #
  10
  11 import base64
  12 import logging
  13 import os.path
  14 import shutil
  15 import sys
  16 import time
  17 import urllib
  18
  19 from keystoneauth1 import loading
  20 from keystoneauth1 import session
  21 from cinderclient import client as cinderclient
  22 from heatclient import client as heatclient
  23 from keystoneclient import client as keystoneclient
  24 from neutronclient.neutron import client as neutronclient
  25 from openstack import connection
  26 from openstack import cloud as os_cloud
  27
  28 from functest.utils import env
  29
  30 logger = logging.getLogger(__name__)
  31
  32 DEFAULT_API_VERSION = '2'
  33 DEFAULT_HEAT_API_VERSION = '1'
  34
  35
  36 # *********************************************
  37 #   CREDENTIALS
  38 # *********************************************
  39 class MissingEnvVar(Exception):
  40
  41     def __init__(self, var):
  42         self.var = var
  43
  44     def __str__(self):
  45         return str.format("Please set the mandatory env var: {}", self.var)
  46
  47
  48 def get_os_connection():
  49     return connection.from_config()
  50
  51
  52 def get_os_cloud():
  53     return os_cloud.openstack_cloud()
  54
  55
  56 def is_keystone_v3():
  57     keystone_api_version = os.getenv('OS_IDENTITY_API_VERSION')
  58     if (keystone_api_version is None or
  59             keystone_api_version == '2'):
  60         return False
  61     else:
  62         return True
  63
  64
  65 def get_rc_env_vars():
  66     env_vars = ['OS_AUTH_URL', 'OS_USERNAME', 'OS_PASSWORD']
  67     if is_keystone_v3():
  68         env_vars.extend(['OS_PROJECT_NAME',
  69                          'OS_USER_DOMAIN_NAME',
  70                          'OS_PROJECT_DOMAIN_NAME'])
  71     else:
  72         env_vars.extend(['OS_TENANT_NAME'])
  73     return env_vars
  74
  75
  76 def check_credentials():
  77     """
  78     Check if the OpenStack credentials (openrc) are sourced
  79     """
  80     env_vars = get_rc_env_vars()
  81     return all(map(lambda v: v in os.environ and os.environ[v], env_vars))
  82
  83
  84 def get_env_cred_dict():
  85     env_cred_dict = {
  86         'OS_USERNAME': 'username',
  87         'OS_PASSWORD': 'password',
  88         'OS_AUTH_URL': 'auth_url',
  89         'OS_TENANT_NAME': 'tenant_name',
  90         'OS_USER_DOMAIN_NAME': 'user_domain_name',
  91         'OS_PROJECT_DOMAIN_NAME': 'project_domain_name',
  92         'OS_PROJECT_NAME': 'project_name',
  93         'OS_ENDPOINT_TYPE': 'endpoint_type',
  94         'OS_REGION_NAME': 'region_name',
  95         'OS_CACERT': 'https_cacert',
  96         'OS_INSECURE': 'https_insecure'
  97     }
  98     return env_cred_dict
  99
 100
 101 def get_credentials(other_creds={}):
 102     """Returns a creds dictionary filled with parsed from env
 103     """
 104     creds = {}
 105     env_vars = get_rc_env_vars()
 106     env_cred_dict = get_env_cred_dict()
 107
 108     for envvar in env_vars:
 109         if os.getenv(envvar) is None:
 110             raise MissingEnvVar(envvar)
 111         else:
 112             creds_key = env_cred_dict.get(envvar)
 113             creds.update({creds_key: os.getenv(envvar)})
 114
 115     if 'tenant' in other_creds.keys():
 116         if is_keystone_v3():
 117             tenant = 'project_name'
 118         else:
 119             tenant = 'tenant_name'
 120         other_creds[tenant] = other_creds.pop('tenant')
 121
 122     creds.update(other_creds)
 123
 124     return creds
 125
 126
 127 def get_session_auth(other_creds={}):
 128     loader = loading.get_plugin_loader('password')
 129     creds = get_credentials(other_creds)
 130     auth = loader.load_from_options(**creds)
 131     return auth
 132
 133
 134 def get_endpoint(service_type, interface='public'):
 135     auth = get_session_auth()
 136     return get_session().get_endpoint(auth=auth,
 137                                       service_type=service_type,
 138                                       interface=interface)
 139
 140
 141 def get_session(other_creds={}):
 142     auth = get_session_auth(other_creds)
 143     https_cacert = os.getenv('OS_CACERT', '')
 144     https_insecure = os.getenv('OS_INSECURE', '').lower() == 'true'
 145     return session.Session(auth=auth,
 146                            verify=(https_cacert or not https_insecure))
 147
 148
 149 # *********************************************
 150 #   CLIENTS
 151 # *********************************************
 152 def get_keystone_client_version():
 153     api_version = os.getenv('OS_IDENTITY_API_VERSION')
 154     if api_version is not None:
 155         logger.info("OS_IDENTITY_API_VERSION is set in env as '%s'",
 156                     api_version)
 157         return api_version
 158     return DEFAULT_API_VERSION
 159
 160
 161 def get_keystone_client(other_creds={}):
 162     sess = get_session(other_creds)
 163     return keystoneclient.Client(get_keystone_client_version(),
 164                                  session=sess,
 165                                  interface=os.getenv('OS_INTERFACE', 'admin'))
 166
 167
 168 def get_cinder_client_version():
 169     api_version = os.getenv('OS_VOLUME_API_VERSION')
 170     if api_version is not None:
 171         logger.info("OS_VOLUME_API_VERSION is set in env as '%s'",
 172                     api_version)
 173         return api_version
 174     return DEFAULT_API_VERSION
 175
 176
 177 def get_cinder_client(other_creds={}):
 178     sess = get_session(other_creds)
 179     return cinderclient.Client(get_cinder_client_version(), session=sess)
 180
 181
 182 def get_neutron_client_version():
 183     api_version = os.getenv('OS_NETWORK_API_VERSION')
 184     if api_version is not None:
 185         logger.info("OS_NETWORK_API_VERSION is set in env as '%s'",
 186                     api_version)
 187         return api_version
 188     return DEFAULT_API_VERSION
 189
 190
 191 def get_neutron_client(other_creds={}):
 192     sess = get_session(other_creds)
 193     return neutronclient.Client(get_neutron_client_version(), session=sess)
 194
 195
 196 def get_heat_client_version():
 197     api_version = os.getenv('OS_ORCHESTRATION_API_VERSION')
 198     if api_version is not None:
 199         logger.info("OS_ORCHESTRATION_API_VERSION is set in env as '%s'",
 200                     api_version)
 201         return api_version
 202     return DEFAULT_HEAT_API_VERSION
 203
 204
 205 def get_heat_client(other_creds={}):
 206     sess = get_session(other_creds)
 207     return heatclient.Client(get_heat_client_version(), session=sess)
 208
 209
 210 def download_url(url, dest_path):
 211     """
 212     Download a file to a destination path given a URL
 213     """
 214     name = url.rsplit('/')[-1]
 215     dest = dest_path + "/" + name
 216     try:
 217         response = urllib.urlopen(url)
 218     except Exception:
 219         return False
 220
 221     with open(dest, 'wb') as lfile:
 222         shutil.copyfileobj(response, lfile)
 223     return True
 224
 225
 226 def download_and_add_image_on_glance(conn, image_name, image_url, data_dir):
 227     try:
 228         dest_path = data_dir
 229         if not os.path.exists(dest_path):
 230             os.makedirs(dest_path)
 231         file_name = image_url.rsplit('/')[-1]
 232         if not download_url(image_url, dest_path):
 233             return False
 234     except Exception:
 235         raise Exception("Impossible to download image from {}".format(
 236                         image_url))
 237
 238     try:
 239         image = create_glance_image(
 240             conn, image_name, dest_path + file_name)
 241         if not image:
 242             return False
 243         else:
 244             return image
 245     except Exception:
 246         raise Exception("Impossible to put image {} in glance".format(
 247                         image_name))
 248
 249
 250 # *********************************************
 251 #   NOVA
 252 # *********************************************
 253 def get_instances(conn):
 254     try:
 255         instances = conn.compute.servers(all_tenants=1)
 256         return instances
 257     except Exception as e:
 258         logger.error("Error [get_instances(compute)]: %s" % e)
 259         return None
 260
 261
 262 def get_instance_status(conn, instance):
 263     try:
 264         instance = conn.compute.get_server(instance.id)
 265         return instance.status
 266     except Exception as e:
 267         logger.error("Error [get_instance_status(compute)]: %s" % e)
 268         return None
 269
 270
 271 def get_instance_by_name(conn, instance_name):
 272     try:
 273         instance = conn.compute.find_server(instance_name,
 274                                             ignore_missing=False)
 275         return instance
 276     except Exception as e:
 277         logger.error("Error [get_instance_by_name(compute, '%s')]: %s"
 278                      % (instance_name, e))
 279         return None
 280
 281
 282 def get_flavor_id(conn, flavor_name):
 283     flavors = conn.compute.flavors()
 284     id = ''
 285     for f in flavors:
 286         if f.name == flavor_name:
 287             id = f.id
 288             break
 289     return id
 290
 291
 292 def get_flavor_id_by_ram_range(conn, min_ram, max_ram):
 293     flavors = conn.compute.flavors()
 294     id = ''
 295     for f in flavors:
 296         if min_ram <= f.ram and f.ram <= max_ram:
 297             id = f.id
 298             break
 299     return id
 300
 301
 302 def get_aggregates(cloud):
 303     try:
 304         aggregates = cloud.list_aggregates()
 305         return aggregates
 306     except Exception as e:
 307         logger.error("Error [get_aggregates(compute)]: %s" % e)
 308         return None
 309
 310
 311 def get_aggregate_id(cloud, aggregate_name):
 312     try:
 313         aggregates = get_aggregates(cloud)
 314         _id = [ag.id for ag in aggregates if ag.name == aggregate_name][0]
 315         return _id
 316     except Exception as e:
 317         logger.error("Error [get_aggregate_id(compute, %s)]:"
 318                      " %s" % (aggregate_name, e))
 319         return None
 320
 321
 322 def get_availability_zones(conn):
 323     try:
 324         availability_zones = conn.compute.availability_zones()
 325         return availability_zones
 326     except Exception as e:
 327         logger.error("Error [get_availability_zones(compute)]: %s" % e)
 328         return None
 329
 330
 331 def get_availability_zone_names(conn):
 332     try:
 333         az_names = [az.zoneName for az in get_availability_zones(conn)]
 334         return az_names
 335     except Exception as e:
 336         logger.error("Error [get_availability_zone_names(compute)]:"
 337                      " %s" % e)
 338         return None
 339
 340
 341 def create_flavor(conn, flavor_name, ram, disk, vcpus, public=True):
 342     try:
 343         flavor = conn.compute.create_flavor(
 344             name=flavor_name, ram=ram, disk=disk, vcpus=vcpus,
 345             is_public=public)
 346     except Exception as e:
 347         logger.error("Error [create_flavor(compute, '%s', '%s', '%s', "
 348                      "'%s')]: %s" % (flavor_name, ram, disk, vcpus, e))
 349         return None
 350     return flavor.id
 351
 352
 353 def get_or_create_flavor(flavor_name, ram, disk, vcpus, public=True):
 354     flavor_exists = False
 355     conn = get_os_connection()
 356
 357     flavor_id = get_flavor_id(conn, flavor_name)
 358     if flavor_id != '':
 359         logger.info("Using existing flavor '%s'..." % flavor_name)
 360         flavor_exists = True
 361     else:
 362         logger.info("Creating flavor '%s' with '%s' RAM, '%s' disk size, "
 363                     "'%s' vcpus..." % (flavor_name, ram, disk, vcpus))
 364         flavor_id = create_flavor(
 365             conn, flavor_name, ram, disk, vcpus, public=public)
 366         if not flavor_id:
 367             raise Exception("Failed to create flavor '%s'..." % (flavor_name))
 368         else:
 369             logger.debug("Flavor '%s' with ID=%s created successfully."
 370                          % (flavor_name, flavor_id))
 371
 372     return flavor_exists, flavor_id
 373
 374
 375 def get_floating_ips(neutron_client):
 376     try:
 377         floating_ips = neutron_client.list_floatingips()
 378         return floating_ips['floatingips']
 379     except Exception as e:
 380         logger.error("Error [get_floating_ips(neutron_client)]: %s" % e)
 381         return None
 382
 383
 384 def get_hypervisors(conn):
 385     try:
 386         nodes = []
 387         hypervisors = conn.compute.hypervisors()
 388         for hypervisor in hypervisors:
 389             if hypervisor.state == "up":
 390                 nodes.append(hypervisor.name)
 391         return nodes
 392     except Exception as e:
 393         logger.error("Error [get_hypervisors(compute)]: %s" % e)
 394         return None
 395
 396
 397 def create_aggregate(cloud, aggregate_name, av_zone):
 398     try:
 399         cloud.create_aggregate(aggregate_name, av_zone)
 400         return True
 401     except Exception as e:
 402         logger.error("Error [create_aggregate(compute, %s, %s)]: %s"
 403                      % (aggregate_name, av_zone, e))
 404         return None
 405
 406
 407 def add_host_to_aggregate(cloud, aggregate_name, compute_host):
 408     try:
 409         aggregate_id = get_aggregate_id(cloud, aggregate_name)
 410         cloud.add_host_to_aggregate(aggregate_id, compute_host)
 411         return True
 412     except Exception as e:
 413         logger.error("Error [add_host_to_aggregate(compute, %s, %s)]: %s"
 414                      % (aggregate_name, compute_host, e))
 415         return None
 416
 417
 418 def create_aggregate_with_host(
 419         cloud, aggregate_name, av_zone, compute_host):
 420     try:
 421         create_aggregate(cloud, aggregate_name, av_zone)
 422         add_host_to_aggregate(cloud, aggregate_name, compute_host)
 423         return True
 424     except Exception as e:
 425         logger.error("Error [create_aggregate_with_host("
 426                      "compute, %s, %s, %s)]: %s"
 427                      % (aggregate_name, av_zone, compute_host, e))
 428         return None
 429
 430
 431 def create_instance(flavor_name,
 432                     image_id,
 433                     network_id,
 434                     instance_name="functest-vm",
 435                     confdrive=True,
 436                     userdata=None,
 437                     av_zone=None,
 438                     fixed_ip=None,
 439                     files=[]):
 440     conn = get_os_connection()
 441     try:
 442         flavor = conn.compute.find_flavor(flavor_name, ignore_missing=False)
 443     except:
 444         flavors = [flavor.name for flavor in conn.compute.flavors()]
 445         logger.error("Error: Flavor '%s' not found. Available flavors are: "
 446                      "\n%s" % (flavor_name, flavors))
 447         return None
 448     if fixed_ip is not None:
 449         networks = {"uuid": network_id, "fixed_ip": fixed_ip}
 450     else:
 451         networks = {"uuid": network_id}
 452
 453     server_attrs = {
 454         'name': instance_name,
 455         'flavor_id': flavor.id,
 456         'image_id': image_id,
 457         'networks': [networks],
 458         'personality': files
 459     }
 460     if userdata is not None:
 461         server_attrs['config_drive'] = confdrive
 462         server_attrs['user_data'] = base64.b64encode(userdata.encode())
 463     if av_zone is not None:
 464         server_attrs['availability_zone'] = av_zone
 465
 466     instance = conn.compute.create_server(**server_attrs)
 467     return instance
 468
 469
 470 def create_instance_and_wait_for_active(flavor_name,
 471                                         image_id,
 472                                         network_id,
 473                                         instance_name="",
 474                                         config_drive=False,
 475                                         userdata="",
 476                                         av_zone=None,
 477                                         fixed_ip=None,
 478                                         files=[]):
 479     SLEEP = 3
 480     VM_BOOT_TIMEOUT = 180
 481     conn = get_os_connection()
 482     instance = create_instance(flavor_name,
 483                                image_id,
 484                                network_id,
 485                                instance_name,
 486                                config_drive,
 487                                userdata,
 488                                av_zone=av_zone,
 489                                fixed_ip=fixed_ip,
 490                                files=files)
 491     count = VM_BOOT_TIMEOUT / SLEEP
 492     for n in range(count, -1, -1):
 493         status = get_instance_status(conn, instance)
 494         if status is None:
 495             time.sleep(SLEEP)
 496             continue
 497         elif status.lower() == "active":
 498             return instance
 499         elif status.lower() == "error":
 500             logger.error("The instance %s went to ERROR status."
 501                          % instance_name)
 502             return None
 503         time.sleep(SLEEP)
 504     logger.error("Timeout booting the instance %s." % instance_name)
 505     return None
 506
 507
 508 def create_floating_ip(neutron_client):
 509     extnet_id = get_external_net_id(neutron_client)
 510     props = {'floating_network_id': extnet_id}
 511     try:
 512         ip_json = neutron_client.create_floatingip({'floatingip': props})
 513         fip_addr = ip_json['floatingip']['floating_ip_address']
 514         fip_id = ip_json['floatingip']['id']
 515     except Exception as e:
 516         logger.error("Error [create_floating_ip(neutron_client)]: %s" % e)
 517         return None
 518     return {'fip_addr': fip_addr, 'fip_id': fip_id}
 519
 520
 521 def attach_floating_ip(neutron_client, port_id):
 522     extnet_id = get_external_net_id(neutron_client)
 523     props = {'floating_network_id': extnet_id,
 524              'port_id': port_id}
 525     try:
 526         return neutron_client.create_floatingip({'floatingip': props})
 527     except Exception as e:
 528         logger.error("Error [Attach_floating_ip(neutron_client), %s]: %s"
 529                      % (port_id, e))
 530         return None
 531
 532
 533 def add_floating_ip(conn, server_id, floatingip_addr):
 534     try:
 535         conn.compute.add_floating_ip_to_server(server_id, floatingip_addr)
 536         return True
 537     except Exception as e:
 538         logger.error("Error [add_floating_ip(compute, '%s', '%s')]: %s"
 539                      % (server_id, floatingip_addr, e))
 540         return False
 541
 542
 543 def delete_instance(conn, instance_id):
 544     try:
 545         conn.compute.delete_server(instance_id, force=True)
 546         return True
 547     except Exception as e:
 548         logger.error("Error [delete_instance(compute, '%s')]: %s"
 549                      % (instance_id, e))
 550         return False
 551
 552
 553 def delete_floating_ip(neutron_client, floatingip_id):
 554     try:
 555         neutron_client.delete_floatingip(floatingip_id)
 556         return True
 557     except Exception as e:
 558         logger.error("Error [delete_floating_ip(neutron_client, '%s')]: %s"
 559                      % (floatingip_id, e))
 560         return False
 561
 562
 563 def remove_host_from_aggregate(cloud, aggregate_name, compute_host):
 564     try:
 565         aggregate_id = get_aggregate_id(cloud, aggregate_name)
 566         cloud.remove_host_from_aggregate(aggregate_id, compute_host)
 567         return True
 568     except Exception as e:
 569         logger.error("Error [remove_host_from_aggregate(compute, %s, %s)]:"
 570                      " %s" % (aggregate_name, compute_host, e))
 571         return False
 572
 573
 574 def remove_hosts_from_aggregate(cloud, aggregate_name):
 575     aggregate_id = get_aggregate_id(cloud, aggregate_name)
 576     hosts = cloud.get_aggregate(aggregate_id).hosts
 577     assert(
 578         all(remove_host_from_aggregate(cloud, aggregate_name, host)
 579             for host in hosts))
 580
 581
 582 def delete_aggregate(cloud, aggregate_name):
 583     try:
 584         remove_hosts_from_aggregate(cloud, aggregate_name)
 585         cloud.delete_aggregate(aggregate_name)
 586         return True
 587     except Exception as e:
 588         logger.error("Error [delete_aggregate(compute, %s)]: %s"
 589                      % (aggregate_name, e))
 590         return False
 591
 592
 593 # *********************************************
 594 #   NEUTRON
 595 # *********************************************
 596 def get_network_list(neutron_client):
 597     network_list = neutron_client.list_networks()['networks']
 598     if len(network_list) == 0:
 599         return None
 600     else:
 601         return network_list
 602
 603
 604 def get_router_list(neutron_client):
 605     router_list = neutron_client.list_routers()['routers']
 606     if len(router_list) == 0:
 607         return None
 608     else:
 609         return router_list
 610
 611
 612 def get_port_list(neutron_client):
 613     port_list = neutron_client.list_ports()['ports']
 614     if len(port_list) == 0:
 615         return None
 616     else:
 617         return port_list
 618
 619
 620 def get_network_id(neutron_client, network_name):
 621     networks = neutron_client.list_networks()['networks']
 622     id = ''
 623     for n in networks:
 624         if n['name'] == network_name:
 625             id = n['id']
 626             break
 627     return id
 628
 629
 630 def get_subnet_id(neutron_client, subnet_name):
 631     subnets = neutron_client.list_subnets()['subnets']
 632     id = ''
 633     for s in subnets:
 634         if s['name'] == subnet_name:
 635             id = s['id']
 636             break
 637     return id
 638
 639
 640 def get_router_id(neutron_client, router_name):
 641     routers = neutron_client.list_routers()['routers']
 642     id = ''
 643     for r in routers:
 644         if r['name'] == router_name:
 645             id = r['id']
 646             break
 647     return id
 648
 649
 650 def get_private_net(neutron_client):
 651     # Checks if there is an existing shared private network
 652     networks = neutron_client.list_networks()['networks']
 653     if len(networks) == 0:
 654         return None
 655     for net in networks:
 656         if (net['router:external'] is False) and (net['shared'] is True):
 657             return net
 658     return None
 659
 660
 661 def get_external_net(neutron_client):
 662     if (env.get('EXTERNAL_NETWORK')):
 663         return env.get('EXTERNAL_NETWORK')
 664     for network in neutron_client.list_networks()['networks']:
 665         if network['router:external']:
 666             return network['name']
 667     return None
 668
 669
 670 def get_external_net_id(neutron_client):
 671     if (env.get('EXTERNAL_NETWORK')):
 672         networks = neutron_client.list_networks(
 673             name=env.get('EXTERNAL_NETWORK'))
 674         net_id = networks['networks'][0]['id']
 675         return net_id
 676     for network in neutron_client.list_networks()['networks']:
 677         if network['router:external']:
 678             return network['id']
 679     return None
 680
 681
 682 def check_neutron_net(neutron_client, net_name):
 683     for network in neutron_client.list_networks()['networks']:
 684         if network['name'] == net_name:
 685             for subnet in network['subnets']:
 686                 return True
 687     return False
 688
 689
 690 def create_neutron_net(neutron_client, name):
 691     json_body = {'network': {'name': name,
 692                              'admin_state_up': True}}
 693     try:
 694         network = neutron_client.create_network(body=json_body)
 695         network_dict = network['network']
 696         return network_dict['id']
 697     except Exception as e:
 698         logger.error("Error [create_neutron_net(neutron_client, '%s')]: %s"
 699                      % (name, e))
 700         return None
 701
 702
 703 def create_neutron_subnet(neutron_client, name, cidr, net_id,
 704                           dns=['8.8.8.8', '8.8.4.4']):
 705     json_body = {'subnets': [{'name': name, 'cidr': cidr,
 706                               'ip_version': 4, 'network_id': net_id,
 707                               'dns_nameservers': dns}]}
 708
 709     try:
 710         subnet = neutron_client.create_subnet(body=json_body)
 711         return subnet['subnets'][0]['id']
 712     except Exception as e:
 713         logger.error("Error [create_neutron_subnet(neutron_client, '%s', "
 714                      "'%s', '%s')]: %s" % (name, cidr, net_id, e))
 715         return None
 716
 717
 718 def create_neutron_router(neutron_client, name):
 719     json_body = {'router': {'name': name, 'admin_state_up': True}}
 720     try:
 721         router = neutron_client.create_router(json_body)
 722         return router['router']['id']
 723     except Exception as e:
 724         logger.error("Error [create_neutron_router(neutron_client, '%s')]: %s"
 725                      % (name, e))
 726         return None
 727
 728
 729 def create_neutron_port(neutron_client, name, network_id, ip):
 730     json_body = {'port': {
 731                  'admin_state_up': True,
 732                  'name': name,
 733                  'network_id': network_id,
 734                  'fixed_ips': [{"ip_address": ip}]
 735                  }}
 736     try:
 737         port = neutron_client.create_port(body=json_body)
 738         return port['port']['id']
 739     except Exception as e:
 740         logger.error("Error [create_neutron_port(neutron_client, '%s', '%s', "
 741                      "'%s')]: %s" % (name, network_id, ip, e))
 742         return None
 743
 744
 745 def update_neutron_net(neutron_client, network_id, shared=False):
 746     json_body = {'network': {'shared': shared}}
 747     try:
 748         neutron_client.update_network(network_id, body=json_body)
 749         return True
 750     except Exception as e:
 751         logger.error("Error [update_neutron_net(neutron_client, '%s', '%s')]: "
 752                      "%s" % (network_id, str(shared), e))
 753         return False
 754
 755
 756 def update_neutron_port(neutron_client, port_id, device_owner):
 757     json_body = {'port': {
 758                  'device_owner': device_owner,
 759                  }}
 760     try:
 761         port = neutron_client.update_port(port=port_id,
 762                                           body=json_body)
 763         return port['port']['id']
 764     except Exception as e:
 765         logger.error("Error [update_neutron_port(neutron_client, '%s', '%s')]:"
 766                      " %s" % (port_id, device_owner, e))
 767         return None
 768
 769
 770 def add_interface_router(neutron_client, router_id, subnet_id):
 771     json_body = {"subnet_id": subnet_id}
 772     try:
 773         neutron_client.add_interface_router(router=router_id, body=json_body)
 774         return True
 775     except Exception as e:
 776         logger.error("Error [add_interface_router(neutron_client, '%s', "
 777                      "'%s')]: %s" % (router_id, subnet_id, e))
 778         return False
 779
 780
 781 def add_gateway_router(neutron_client, router_id):
 782     ext_net_id = get_external_net_id(neutron_client)
 783     router_dict = {'network_id': ext_net_id}
 784     try:
 785         neutron_client.add_gateway_router(router_id, router_dict)
 786         return True
 787     except Exception as e:
 788         logger.error("Error [add_gateway_router(neutron_client, '%s')]: %s"
 789                      % (router_id, e))
 790         return False
 791
 792
 793 def delete_neutron_net(neutron_client, network_id):
 794     try:
 795         neutron_client.delete_network(network_id)
 796         return True
 797     except Exception as e:
 798         logger.error("Error [delete_neutron_net(neutron_client, '%s')]: %s"
 799                      % (network_id, e))
 800         return False
 801
 802
 803 def delete_neutron_subnet(neutron_client, subnet_id):
 804     try:
 805         neutron_client.delete_subnet(subnet_id)
 806         return True
 807     except Exception as e:
 808         logger.error("Error [delete_neutron_subnet(neutron_client, '%s')]: %s"
 809                      % (subnet_id, e))
 810         return False
 811
 812
 813 def delete_neutron_router(neutron_client, router_id):
 814     try:
 815         neutron_client.delete_router(router=router_id)
 816         return True
 817     except Exception as e:
 818         logger.error("Error [delete_neutron_router(neutron_client, '%s')]: %s"
 819                      % (router_id, e))
 820         return False
 821
 822
 823 def delete_neutron_port(neutron_client, port_id):
 824     try:
 825         neutron_client.delete_port(port_id)
 826         return True
 827     except Exception as e:
 828         logger.error("Error [delete_neutron_port(neutron_client, '%s')]: %s"
 829                      % (port_id, e))
 830         return False
 831
 832
 833 def remove_interface_router(neutron_client, router_id, subnet_id):
 834     json_body = {"subnet_id": subnet_id}
 835     try:
 836         neutron_client.remove_interface_router(router=router_id,
 837                                                body=json_body)
 838         return True
 839     except Exception as e:
 840         logger.error("Error [remove_interface_router(neutron_client, '%s', "
 841                      "'%s')]: %s" % (router_id, subnet_id, e))
 842         return False
 843
 844
 845 def remove_gateway_router(neutron_client, router_id):
 846     try:
 847         neutron_client.remove_gateway_router(router_id)
 848         return True
 849     except Exception as e:
 850         logger.error("Error [remove_gateway_router(neutron_client, '%s')]: %s"
 851                      % (router_id, e))
 852         return False
 853
 854
 855 def create_network_full(neutron_client,
 856                         net_name,
 857                         subnet_name,
 858                         router_name,
 859                         cidr,
 860                         dns=['8.8.8.8', '8.8.4.4']):
 861
 862     # Check if the network already exists
 863     network_id = get_network_id(neutron_client, net_name)
 864     subnet_id = get_subnet_id(neutron_client, subnet_name)
 865     router_id = get_router_id(neutron_client, router_name)
 866
 867     if network_id != '' and subnet_id != '' and router_id != '':
 868         logger.info("A network with name '%s' already exists..." % net_name)
 869     else:
 870         neutron_client.format = 'json'
 871
 872         logger.info('Creating neutron network %s...' % net_name)
 873         if network_id == '':
 874             network_id = create_neutron_net(neutron_client, net_name)
 875         if not network_id:
 876             return False
 877         logger.debug("Network '%s' created successfully" % network_id)
 878
 879         logger.debug('Creating Subnet....')
 880         if subnet_id == '':
 881             subnet_id = create_neutron_subnet(neutron_client, subnet_name,
 882                                               cidr, network_id, dns)
 883         if not subnet_id:
 884             return None
 885         logger.debug("Subnet '%s' created successfully" % subnet_id)
 886
 887         logger.debug('Creating Router...')
 888         if router_id == '':
 889             router_id = create_neutron_router(neutron_client, router_name)
 890         if not router_id:
 891             return None
 892         logger.debug("Router '%s' created successfully" % router_id)
 893
 894         logger.debug('Adding router to subnet...')
 895
 896         if not add_interface_router(neutron_client, router_id, subnet_id):
 897             return None
 898         logger.debug("Interface added successfully.")
 899
 900         logger.debug('Adding gateway to router...')
 901         if not add_gateway_router(neutron_client, router_id):
 902             return None
 903         logger.debug("Gateway added successfully.")
 904
 905     network_dic = {'net_id': network_id,
 906                    'subnet_id': subnet_id,
 907                    'router_id': router_id}
 908     return network_dic
 909
 910
 911 def create_shared_network_full(net_name, subnt_name, router_name, subnet_cidr):
 912     neutron_client = get_neutron_client()
 913
 914     network_dic = create_network_full(neutron_client,
 915                                       net_name,
 916                                       subnt_name,
 917                                       router_name,
 918                                       subnet_cidr)
 919     if network_dic:
 920         if not update_neutron_net(neutron_client,
 921                                   network_dic['net_id'],
 922                                   shared=True):
 923             logger.error("Failed to update network %s..." % net_name)
 924             return None
 925         else:
 926             logger.debug("Network '%s' is available..." % net_name)
 927     else:
 928         logger.error("Network %s creation failed" % net_name)
 929         return None
 930     return network_dic
 931
 932
 933 # *********************************************
 934 #   SEC GROUPS
 935 # *********************************************
 936
 937
 938 def get_security_groups(neutron_client):
 939     try:
 940         security_groups = neutron_client.list_security_groups()[
 941             'security_groups']
 942         return security_groups
 943     except Exception as e:
 944         logger.error("Error [get_security_groups(neutron_client)]: %s" % e)
 945         return None
 946
 947
 948 def get_security_group_id(neutron_client, sg_name):
 949     security_groups = get_security_groups(neutron_client)
 950     id = ''
 951     for sg in security_groups:
 952         if sg['name'] == sg_name:
 953             id = sg['id']
 954             break
 955     return id
 956
 957
 958 def create_security_group(neutron_client, sg_name, sg_description):
 959     json_body = {'security_group': {'name': sg_name,
 960                                     'description': sg_description}}
 961     try:
 962         secgroup = neutron_client.create_security_group(json_body)
 963         return secgroup['security_group']
 964     except Exception as e:
 965         logger.error("Error [create_security_group(neutron_client, '%s', "
 966                      "'%s')]: %s" % (sg_name, sg_description, e))
 967         return None
 968
 969
 970 def create_secgroup_rule(neutron_client, sg_id, direction, protocol,
 971                          port_range_min=None, port_range_max=None):
 972     # We create a security group in 2 steps
 973     # 1 - we check the format and set the json body accordingly
 974     # 2 - we call neturon client to create the security group
 975
 976     # Format check
 977     json_body = {'security_group_rule': {'direction': direction,
 978                                          'security_group_id': sg_id,
 979                                          'protocol': protocol}}
 980     # parameters may be
 981     # - both None => we do nothing
 982     # - both Not None => we add them to the json description
 983     # but one cannot be None is the other is not None
 984     if (port_range_min is not None and port_range_max is not None):
 985         # add port_range in json description
 986         json_body['security_group_rule']['port_range_min'] = port_range_min
 987         json_body['security_group_rule']['port_range_max'] = port_range_max
 988         logger.debug("Security_group format set (port range included)")
 989     else:
 990         # either both port range are set to None => do nothing
 991         # or one is set but not the other => log it and return False
 992         if port_range_min is None and port_range_max is None:
 993             logger.debug("Security_group format set (no port range mentioned)")
 994         else:
 995             logger.error("Bad security group format."
 996                          "One of the port range is not properly set:"
 997                          "range min: {},"
 998                          "range max: {}".format(port_range_min,
 999                                                 port_range_max))
1000             return False
1001
1002     # Create security group using neutron client
1003     try:
1004         neutron_client.create_security_group_rule(json_body)
1005         return True
1006     except:
1007         logger.exception("Impossible to create_security_group_rule,"
1008                          "security group rule probably already exists")
1009         return False
1010
1011
1012 def get_security_group_rules(neutron_client, sg_id):
1013     try:
1014         security_rules = neutron_client.list_security_group_rules()[
1015             'security_group_rules']
1016         security_rules = [rule for rule in security_rules
1017                           if rule["security_group_id"] == sg_id]
1018         return security_rules
1019     except Exception as e:
1020         logger.error("Error [get_security_group_rules(neutron_client, sg_id)]:"
1021                      " %s" % e)
1022         return None
1023
1024
1025 def check_security_group_rules(neutron_client, sg_id, direction, protocol,
1026                                port_min=None, port_max=None):
1027     try:
1028         security_rules = get_security_group_rules(neutron_client, sg_id)
1029         security_rules = [rule for rule in security_rules
1030                           if (rule["direction"].lower() == direction and
1031                               rule["protocol"].lower() == protocol and
1032                               rule["port_range_min"] == port_min and
1033                               rule["port_range_max"] == port_max)]
1034         if len(security_rules) == 0:
1035             return True
1036         else:
1037             return False
1038     except Exception as e:
1039         logger.error("Error [check_security_group_rules("
1040                      " neutron_client, sg_id, direction,"
1041                      " protocol, port_min=None, port_max=None)]: "
1042                      "%s" % e)
1043         return None
1044
1045
1046 def create_security_group_full(neutron_client,
1047                                sg_name, sg_description):
1048     sg_id = get_security_group_id(neutron_client, sg_name)
1049     if sg_id != '':
1050         logger.info("Using existing security group '%s'..." % sg_name)
1051     else:
1052         logger.info("Creating security group  '%s'..." % sg_name)
1053         SECGROUP = create_security_group(neutron_client,
1054                                          sg_name,
1055                                          sg_description)
1056         if not SECGROUP:
1057             logger.error("Failed to create the security group...")
1058             return None
1059
1060         sg_id = SECGROUP['id']
1061
1062         logger.debug("Security group '%s' with ID=%s created successfully."
1063                      % (SECGROUP['name'], sg_id))
1064
1065         logger.debug("Adding ICMP rules in security group '%s'..."
1066                      % sg_name)
1067         if not create_secgroup_rule(neutron_client, sg_id,
1068                                     'ingress', 'icmp'):
1069             logger.error("Failed to create the security group rule...")
1070             return None
1071
1072         logger.debug("Adding SSH rules in security group '%s'..."
1073                      % sg_name)
1074         if not create_secgroup_rule(
1075                 neutron_client, sg_id, 'ingress', 'tcp', '22', '22'):
1076             logger.error("Failed to create the security group rule...")
1077             return None
1078
1079         if not create_secgroup_rule(
1080                 neutron_client, sg_id, 'egress', 'tcp', '22', '22'):
1081             logger.error("Failed to create the security group rule...")
1082             return None
1083     return sg_id
1084
1085
1086 def add_secgroup_to_instance(conn, instance_id, secgroup_id):
1087     try:
1088         conn.compute.add_security_group_to_server(instance_id, secgroup_id)
1089         return True
1090     except Exception as e:
1091         logger.error("Error [add_secgroup_to_instance(compute, '%s', "
1092                      "'%s')]: %s" % (instance_id, secgroup_id, e))
1093         return False
1094
1095
1096 def update_sg_quota(neutron_client, tenant_id, sg_quota, sg_rule_quota):
1097     json_body = {"quota": {
1098         "security_group": sg_quota,
1099         "security_group_rule": sg_rule_quota
1100     }}
1101
1102     try:
1103         neutron_client.update_quota(tenant_id=tenant_id,
1104                                     body=json_body)
1105         return True
1106     except Exception as e:
1107         logger.error("Error [update_sg_quota(neutron_client, '%s', '%s', "
1108                      "'%s')]: %s" % (tenant_id, sg_quota, sg_rule_quota, e))
1109         return False
1110
1111
1112 def delete_security_group(neutron_client, secgroup_id):
1113     try:
1114         neutron_client.delete_security_group(secgroup_id)
1115         return True
1116     except Exception as e:
1117         logger.error("Error [delete_security_group(neutron_client, '%s')]: %s"
1118                      % (secgroup_id, e))
1119         return False
1120
1121
1122 # *********************************************
1123 #   GLANCE
1124 # *********************************************
1125 def get_images(conn):
1126     try:
1127         images = conn.image.images()
1128         return images
1129     except Exception as e:
1130         logger.error("Error [get_images]: %s" % e)
1131         return None
1132
1133
1134 def get_image_id(conn, image_name):
1135     images = conn.image.images()
1136     id = ''
1137     for i in images:
1138         if i.name == image_name:
1139             id = i.id
1140             break
1141     return id
1142
1143
1144 def create_glance_image(conn,
1145                         image_name,
1146                         file_path,
1147                         disk="qcow2",
1148                         extra_properties={},
1149                         container="bare",
1150                         public="public"):
1151     if not os.path.isfile(file_path):
1152         logger.error("Error: file %s does not exist." % file_path)
1153         return None
1154     try:
1155         image_id = get_image_id(conn, image_name)
1156         if image_id != '':
1157             logger.info("Image %s already exists." % image_name)
1158         else:
1159             logger.info("Creating image '%s' from '%s'..." % (image_name,
1160                                                               file_path))
1161             with open(file_path) as image_data:
1162                 image = conn.image.upload_image(name=image_name,
1163                                                 is_public=public,
1164                                                 disk_format=disk,
1165                                                 container_format=container,
1166                                                 data=image_data,
1167                                                 **extra_properties)
1168             image_id = image.id
1169         return image_id
1170     except Exception as e:
1171         logger.error("Error [create_glance_image(image, '%s', '%s', "
1172                      "'%s')]: %s" % (image_name, file_path, public, e))
1173         return None
1174
1175
1176 def get_or_create_image(name, path, format, extra_properties):
1177     image_exists = False
1178     conn = get_os_connection()
1179
1180     image_id = get_image_id(conn, name)
1181     if image_id != '':
1182         logger.info("Using existing image '%s'..." % name)
1183         image_exists = True
1184     else:
1185         logger.info("Creating image '%s' from '%s'..." % (name, path))
1186         image_id = create_glance_image(conn,
1187                                        name,
1188                                        path,
1189                                        format,
1190                                        extra_properties)
1191         if not image_id:
1192             logger.error("Failed to create a Glance image...")
1193         else:
1194             logger.debug("Image '%s' with ID=%s created successfully."
1195                          % (name, image_id))
1196
1197     return image_exists, image_id
1198
1199
1200 def delete_glance_image(conn, image_id):
1201     try:
1202         conn.image.delete_image(image_id)
1203         return True
1204     except Exception as e:
1205         logger.error("Error [delete_glance_image(image, '%s')]: %s"
1206                      % (image_id, e))
1207         return False
1208
1209
1210 # *********************************************
1211 #   CINDER
1212 # *********************************************
1213 def get_volumes(cinder_client):
1214     try:
1215         volumes = cinder_client.volumes.list(search_opts={'all_tenants': 1})
1216         return volumes
1217     except Exception as e:
1218         logger.error("Error [get_volumes(cinder_client)]: %s" % e)
1219         return None
1220
1221
1222 def update_cinder_quota(cinder_client, tenant_id, vols_quota,
1223                         snapshots_quota, gigabytes_quota):
1224     quotas_values = {"volumes": vols_quota,
1225                      "snapshots": snapshots_quota,
1226                      "gigabytes": gigabytes_quota}
1227
1228     try:
1229         cinder_client.quotas.update(tenant_id, **quotas_values)
1230         return True
1231     except Exception as e:
1232         logger.error("Error [update_cinder_quota(cinder_client, '%s', '%s', "
1233                      "'%s' '%s')]: %s" % (tenant_id, vols_quota,
1234                                           snapshots_quota, gigabytes_quota, e))
1235         return False
1236
1237
1238 def delete_volume(cinder_client, volume_id, forced=False):
1239     try:
1240         if forced:
1241             try:
1242                 cinder_client.volumes.detach(volume_id)
1243             except:
1244                 logger.error(sys.exc_info()[0])
1245             cinder_client.volumes.force_delete(volume_id)
1246         else:
1247             cinder_client.volumes.delete(volume_id)
1248         return True
1249     except Exception as e:
1250         logger.error("Error [delete_volume(cinder_client, '%s', '%s')]: %s"
1251                      % (volume_id, str(forced), e))
1252         return False
1253
1254
1255 # *********************************************
1256 #   KEYSTONE
1257 # *********************************************
1258 def get_tenants(keystone_client):
1259     try:
1260         if is_keystone_v3():
1261             tenants = keystone_client.projects.list()
1262         else:
1263             tenants = keystone_client.tenants.list()
1264         return tenants
1265     except Exception as e:
1266         logger.error("Error [get_tenants(keystone_client)]: %s" % e)
1267         return None
1268
1269
1270 def get_users(keystone_client):
1271     try:
1272         users = keystone_client.users.list()
1273         return users
1274     except Exception as e:
1275         logger.error("Error [get_users(keystone_client)]: %s" % e)
1276         return None
1277
1278
1279 def get_tenant_id(keystone_client, tenant_name):
1280     tenants = get_tenants(keystone_client)
1281     id = ''
1282     for t in tenants:
1283         if t.name == tenant_name:
1284             id = t.id
1285             break
1286     return id
1287
1288
1289 def get_user_id(keystone_client, user_name):
1290     users = get_users(keystone_client)
1291     id = ''
1292     for u in users:
1293         if u.name == user_name:
1294             id = u.id
1295             break
1296     return id
1297
1298
1299 def get_role_id(keystone_client, role_name):
1300     roles = keystone_client.roles.list()
1301     id = ''
1302     for r in roles:
1303         if r.name == role_name:
1304             id = r.id
1305             break
1306     return id
1307
1308
1309 def get_domain_id(keystone_client, domain_name):
1310     domains = keystone_client.domains.list()
1311     id = ''
1312     for d in domains:
1313         if d.name == domain_name:
1314             id = d.id
1315             break
1316     return id
1317
1318
1319 def create_tenant(keystone_client, tenant_name, tenant_description):
1320     try:
1321         if is_keystone_v3():
1322             domain_name = os.environ['OS_PROJECT_DOMAIN_NAME']
1323             domain_id = get_domain_id(keystone_client, domain_name)
1324             tenant = keystone_client.projects.create(
1325                 name=tenant_name,
1326                 description=tenant_description,
1327                 domain=domain_id,
1328                 enabled=True)
1329         else:
1330             tenant = keystone_client.tenants.create(tenant_name,
1331                                                     tenant_description,
1332                                                     enabled=True)
1333         return tenant.id
1334     except Exception as e:
1335         logger.error("Error [create_tenant(keystone_client, '%s', '%s')]: %s"
1336                      % (tenant_name, tenant_description, e))
1337         return None
1338
1339
1340 def get_or_create_tenant(keystone_client, tenant_name, tenant_description):
1341     tenant_id = get_tenant_id(keystone_client, tenant_name)
1342     if not tenant_id:
1343         tenant_id = create_tenant(keystone_client, tenant_name,
1344                                   tenant_description)
1345
1346     return tenant_id
1347
1348
1349 def get_or_create_tenant_for_vnf(keystone_client, tenant_name,
1350                                  tenant_description):
1351     """Get or Create a Tenant
1352
1353         Args:
1354             keystone_client: keystone client reference
1355             tenant_name: the name of the tenant
1356             tenant_description: the description of the tenant
1357
1358         return False if tenant retrieved though get
1359         return True if tenant created
1360         raise Exception if error during processing
1361     """
1362     try:
1363         tenant_id = get_tenant_id(keystone_client, tenant_name)
1364         if not tenant_id:
1365             tenant_id = create_tenant(keystone_client, tenant_name,
1366                                       tenant_description)
1367             return True
1368         else:
1369             return False
1370     except:
1371         raise Exception("Impossible to create a Tenant for the VNF {}".format(
1372                             tenant_name))
1373
1374
1375 def create_user(keystone_client, user_name, user_password,
1376                 user_email, tenant_id):
1377     try:
1378         if is_keystone_v3():
1379             user = keystone_client.users.create(name=user_name,
1380                                                 password=user_password,
1381                                                 email=user_email,
1382                                                 project_id=tenant_id,
1383                                                 enabled=True)
1384         else:
1385             user = keystone_client.users.create(user_name,
1386                                                 user_password,
1387                                                 user_email,
1388                                                 tenant_id,
1389                                                 enabled=True)
1390         return user.id
1391     except Exception as e:
1392         logger.error("Error [create_user(keystone_client, '%s', '%s', '%s'"
1393                      "'%s')]: %s" % (user_name, user_password,
1394                                      user_email, tenant_id, e))
1395         return None
1396
1397
1398 def get_or_create_user(keystone_client, user_name, user_password,
1399                        tenant_id, user_email=None):
1400     user_id = get_user_id(keystone_client, user_name)
1401     if not user_id:
1402         user_id = create_user(keystone_client, user_name, user_password,
1403                               user_email, tenant_id)
1404     return user_id
1405
1406
1407 def get_or_create_user_for_vnf(keystone_client, vnf_ref):
1408     """Get or Create user for VNF
1409
1410         Args:
1411             keystone_client: keystone client reference
1412             vnf_ref: VNF reference used as user name & password, tenant name
1413
1414         return False if user retrieved through get
1415         return True if user created
1416         raise Exception if error during processing
1417     """
1418     try:
1419         user_id = get_user_id(keystone_client, vnf_ref)
1420         tenant_id = get_tenant_id(keystone_client, vnf_ref)
1421         created = False
1422         if not user_id:
1423             user_id = create_user(keystone_client, vnf_ref, vnf_ref,
1424                                   "", tenant_id)
1425             created = True
1426         try:
1427             role_id = get_role_id(keystone_client, 'admin')
1428             tenant_id = get_tenant_id(keystone_client, vnf_ref)
1429             add_role_user(keystone_client, user_id, role_id, tenant_id)
1430         except:
1431             logger.warn("Cannot associate user to role admin on tenant")
1432         return created
1433     except:
1434         raise Exception("Impossible to create a user for the VNF {}".format(
1435             vnf_ref))
1436
1437
1438 def add_role_user(keystone_client, user_id, role_id, tenant_id):
1439     try:
1440         if is_keystone_v3():
1441             keystone_client.roles.grant(role=role_id,
1442                                         user=user_id,
1443                                         project=tenant_id)
1444         else:
1445             keystone_client.roles.add_user_role(user_id, role_id, tenant_id)
1446         return True
1447     except Exception as e:
1448         logger.error("Error [add_role_user(keystone_client, '%s', '%s'"
1449                      "'%s')]: %s " % (user_id, role_id, tenant_id, e))
1450         return False
1451
1452
1453 def delete_tenant(keystone_client, tenant_id):
1454     try:
1455         if is_keystone_v3():
1456             keystone_client.projects.delete(tenant_id)
1457         else:
1458             keystone_client.tenants.delete(tenant_id)
1459         return True
1460     except Exception as e:
1461         logger.error("Error [delete_tenant(keystone_client, '%s')]: %s"
1462                      % (tenant_id, e))
1463         return False
1464
1465
1466 def delete_user(keystone_client, user_id):
1467     try:
1468         keystone_client.users.delete(user_id)
1469         return True
1470     except Exception as e:
1471         logger.error("Error [delete_user(keystone_client, '%s')]: %s"
1472                      % (user_id, e))
1473         return False
1474
1475
1476 # *********************************************
1477 #   HEAT
1478 # *********************************************
1479 def get_resource(heat_client, stack_id, resource):
1480     try:
1481         resources = heat_client.resources.get(stack_id, resource)
1482         return resources
1483     except Exception as e:
1484         logger.error("Error [get_resource]: %s" % e)
1485         return None