Merge "Replace glance client calls with openstack sdk"
[sdnvpn.git] / sdnvpn / lib / openstack_utils.py
1 #!/usr/bin/env python
2 #
3 # jose.lausuch@ericsson.com
4 # valentin.boucher@orange.com
5 # All rights reserved. This program and the accompanying materials
6 # are made available under the terms of the Apache License, Version 2.0
7 # which accompanies this distribution, and is available at
8 # http://www.apache.org/licenses/LICENSE-2.0
9 #
10
11 import logging
12 import os.path
13 import shutil
14 import sys
15 import time
16 import urllib
17
18 from keystoneauth1 import loading
19 from keystoneauth1 import session
20 from cinderclient import client as cinderclient
21 from heatclient import client as heatclient
22 from novaclient import client as novaclient
23 from keystoneclient import client as keystoneclient
24 from neutronclient.neutron import client as neutronclient
25 from openstack import connection
26
27 from functest.utils import env
28
29 logger = logging.getLogger(__name__)
30
31 DEFAULT_API_VERSION = '2'
32 DEFAULT_HEAT_API_VERSION = '1'
33
34
35 # *********************************************
36 #   CREDENTIALS
37 # *********************************************
38 class MissingEnvVar(Exception):
39
40     def __init__(self, var):
41         self.var = var
42
43     def __str__(self):
44         return str.format("Please set the mandatory env var: {}", self.var)
45
46
47 def get_os_connection():
48     return connection.from_config()
49
50
51 def is_keystone_v3():
52     keystone_api_version = os.getenv('OS_IDENTITY_API_VERSION')
53     if (keystone_api_version is None or
54             keystone_api_version == '2'):
55         return False
56     else:
57         return True
58
59
60 def get_rc_env_vars():
61     env_vars = ['OS_AUTH_URL', 'OS_USERNAME', 'OS_PASSWORD']
62     if is_keystone_v3():
63         env_vars.extend(['OS_PROJECT_NAME',
64                          'OS_USER_DOMAIN_NAME',
65                          'OS_PROJECT_DOMAIN_NAME'])
66     else:
67         env_vars.extend(['OS_TENANT_NAME'])
68     return env_vars
69
70
71 def check_credentials():
72     """
73     Check if the OpenStack credentials (openrc) are sourced
74     """
75     env_vars = get_rc_env_vars()
76     return all(map(lambda v: v in os.environ and os.environ[v], env_vars))
77
78
79 def get_env_cred_dict():
80     env_cred_dict = {
81         'OS_USERNAME': 'username',
82         'OS_PASSWORD': 'password',
83         'OS_AUTH_URL': 'auth_url',
84         'OS_TENANT_NAME': 'tenant_name',
85         'OS_USER_DOMAIN_NAME': 'user_domain_name',
86         'OS_PROJECT_DOMAIN_NAME': 'project_domain_name',
87         'OS_PROJECT_NAME': 'project_name',
88         'OS_ENDPOINT_TYPE': 'endpoint_type',
89         'OS_REGION_NAME': 'region_name',
90         'OS_CACERT': 'https_cacert',
91         'OS_INSECURE': 'https_insecure'
92     }
93     return env_cred_dict
94
95
96 def get_credentials(other_creds={}):
97     """Returns a creds dictionary filled with parsed from env
98     """
99     creds = {}
100     env_vars = get_rc_env_vars()
101     env_cred_dict = get_env_cred_dict()
102
103     for envvar in env_vars:
104         if os.getenv(envvar) is None:
105             raise MissingEnvVar(envvar)
106         else:
107             creds_key = env_cred_dict.get(envvar)
108             creds.update({creds_key: os.getenv(envvar)})
109
110     if 'tenant' in other_creds.keys():
111         if is_keystone_v3():
112             tenant = 'project_name'
113         else:
114             tenant = 'tenant_name'
115         other_creds[tenant] = other_creds.pop('tenant')
116
117     creds.update(other_creds)
118
119     return creds
120
121
122 def get_session_auth(other_creds={}):
123     loader = loading.get_plugin_loader('password')
124     creds = get_credentials(other_creds)
125     auth = loader.load_from_options(**creds)
126     return auth
127
128
129 def get_endpoint(service_type, interface='public'):
130     auth = get_session_auth()
131     return get_session().get_endpoint(auth=auth,
132                                       service_type=service_type,
133                                       interface=interface)
134
135
136 def get_session(other_creds={}):
137     auth = get_session_auth(other_creds)
138     https_cacert = os.getenv('OS_CACERT', '')
139     https_insecure = os.getenv('OS_INSECURE', '').lower() == 'true'
140     return session.Session(auth=auth,
141                            verify=(https_cacert or not https_insecure))
142
143
144 # *********************************************
145 #   CLIENTS
146 # *********************************************
147 def get_keystone_client_version():
148     api_version = os.getenv('OS_IDENTITY_API_VERSION')
149     if api_version is not None:
150         logger.info("OS_IDENTITY_API_VERSION is set in env as '%s'",
151                     api_version)
152         return api_version
153     return DEFAULT_API_VERSION
154
155
156 def get_keystone_client(other_creds={}):
157     sess = get_session(other_creds)
158     return keystoneclient.Client(get_keystone_client_version(),
159                                  session=sess,
160                                  interface=os.getenv('OS_INTERFACE', 'admin'))
161
162
163 def get_nova_client_version():
164     api_version = os.getenv('OS_COMPUTE_API_VERSION')
165     if api_version is not None:
166         logger.info("OS_COMPUTE_API_VERSION is set in env as '%s'",
167                     api_version)
168         return api_version
169     return DEFAULT_API_VERSION
170
171
172 def get_nova_client(other_creds={}):
173     sess = get_session(other_creds)
174     return novaclient.Client(get_nova_client_version(), session=sess)
175
176
177 def get_cinder_client_version():
178     api_version = os.getenv('OS_VOLUME_API_VERSION')
179     if api_version is not None:
180         logger.info("OS_VOLUME_API_VERSION is set in env as '%s'",
181                     api_version)
182         return api_version
183     return DEFAULT_API_VERSION
184
185
186 def get_cinder_client(other_creds={}):
187     sess = get_session(other_creds)
188     return cinderclient.Client(get_cinder_client_version(), session=sess)
189
190
191 def get_neutron_client_version():
192     api_version = os.getenv('OS_NETWORK_API_VERSION')
193     if api_version is not None:
194         logger.info("OS_NETWORK_API_VERSION is set in env as '%s'",
195                     api_version)
196         return api_version
197     return DEFAULT_API_VERSION
198
199
200 def get_neutron_client(other_creds={}):
201     sess = get_session(other_creds)
202     return neutronclient.Client(get_neutron_client_version(), session=sess)
203
204
205 def get_heat_client_version():
206     api_version = os.getenv('OS_ORCHESTRATION_API_VERSION')
207     if api_version is not None:
208         logger.info("OS_ORCHESTRATION_API_VERSION is set in env as '%s'",
209                     api_version)
210         return api_version
211     return DEFAULT_HEAT_API_VERSION
212
213
214 def get_heat_client(other_creds={}):
215     sess = get_session(other_creds)
216     return heatclient.Client(get_heat_client_version(), session=sess)
217
218
219 def download_url(url, dest_path):
220     """
221     Download a file to a destination path given a URL
222     """
223     name = url.rsplit('/')[-1]
224     dest = dest_path + "/" + name
225     try:
226         response = urllib.urlopen(url)
227     except Exception:
228         return False
229
230     with open(dest, 'wb') as lfile:
231         shutil.copyfileobj(response, lfile)
232     return True
233
234
235 def download_and_add_image_on_glance(conn, image_name, image_url, data_dir):
236     try:
237         dest_path = data_dir
238         if not os.path.exists(dest_path):
239             os.makedirs(dest_path)
240         file_name = image_url.rsplit('/')[-1]
241         if not download_url(image_url, dest_path):
242             return False
243     except Exception:
244         raise Exception("Impossible to download image from {}".format(
245                         image_url))
246
247     try:
248         image = create_glance_image(
249             conn, image_name, dest_path + file_name)
250         if not image:
251             return False
252         else:
253             return image
254     except Exception:
255         raise Exception("Impossible to put image {} in glance".format(
256                         image_name))
257
258
259 # *********************************************
260 #   NOVA
261 # *********************************************
262 def get_instances(nova_client):
263     try:
264         instances = nova_client.servers.list(search_opts={'all_tenants': 1})
265         return instances
266     except Exception as e:
267         logger.error("Error [get_instances(nova_client)]: %s" % e)
268         return None
269
270
271 def get_instance_status(nova_client, instance):
272     try:
273         instance = nova_client.servers.get(instance.id)
274         return instance.status
275     except Exception as e:
276         logger.error("Error [get_instance_status(nova_client)]: %s" % e)
277         return None
278
279
280 def get_instance_by_name(nova_client, instance_name):
281     try:
282         instance = nova_client.servers.find(name=instance_name)
283         return instance
284     except Exception as e:
285         logger.error("Error [get_instance_by_name(nova_client, '%s')]: %s"
286                      % (instance_name, e))
287         return None
288
289
290 def get_flavor_id(nova_client, flavor_name):
291     flavors = nova_client.flavors.list(detailed=True)
292     id = ''
293     for f in flavors:
294         if f.name == flavor_name:
295             id = f.id
296             break
297     return id
298
299
300 def get_flavor_id_by_ram_range(nova_client, min_ram, max_ram):
301     flavors = nova_client.flavors.list(detailed=True)
302     id = ''
303     for f in flavors:
304         if min_ram <= f.ram and f.ram <= max_ram:
305             id = f.id
306             break
307     return id
308
309
310 def get_aggregates(nova_client):
311     try:
312         aggregates = nova_client.aggregates.list()
313         return aggregates
314     except Exception as e:
315         logger.error("Error [get_aggregates(nova_client)]: %s" % e)
316         return None
317
318
319 def get_aggregate_id(nova_client, aggregate_name):
320     try:
321         aggregates = get_aggregates(nova_client)
322         _id = [ag.id for ag in aggregates if ag.name == aggregate_name][0]
323         return _id
324     except Exception as e:
325         logger.error("Error [get_aggregate_id(nova_client, %s)]:"
326                      " %s" % (aggregate_name, e))
327         return None
328
329
330 def get_availability_zones(nova_client):
331     try:
332         availability_zones = nova_client.availability_zones.list()
333         return availability_zones
334     except Exception as e:
335         logger.error("Error [get_availability_zones(nova_client)]: %s" % e)
336         return None
337
338
339 def get_availability_zone_names(nova_client):
340     try:
341         az_names = [az.zoneName for az in get_availability_zones(nova_client)]
342         return az_names
343     except Exception as e:
344         logger.error("Error [get_availability_zone_names(nova_client)]:"
345                      " %s" % e)
346         return None
347
348
349 def create_flavor(nova_client, flavor_name, ram, disk, vcpus, public=True):
350     try:
351         flavor = nova_client.flavors.create(
352             flavor_name, ram, vcpus, disk, is_public=public)
353     except Exception as e:
354         logger.error("Error [create_flavor(nova_client, '%s', '%s', '%s', "
355                      "'%s')]: %s" % (flavor_name, ram, disk, vcpus, e))
356         return None
357     return flavor.id
358
359
360 def get_or_create_flavor(flavor_name, ram, disk, vcpus, public=True):
361     flavor_exists = False
362     nova_client = get_nova_client()
363
364     flavor_id = get_flavor_id(nova_client, flavor_name)
365     if flavor_id != '':
366         logger.info("Using existing flavor '%s'..." % flavor_name)
367         flavor_exists = True
368     else:
369         logger.info("Creating flavor '%s' with '%s' RAM, '%s' disk size, "
370                     "'%s' vcpus..." % (flavor_name, ram, disk, vcpus))
371         flavor_id = create_flavor(
372             nova_client, flavor_name, ram, disk, vcpus, public=public)
373         if not flavor_id:
374             raise Exception("Failed to create flavor '%s'..." % (flavor_name))
375         else:
376             logger.debug("Flavor '%s' with ID=%s created successfully."
377                          % (flavor_name, flavor_id))
378
379     return flavor_exists, flavor_id
380
381
382 def get_floating_ips(neutron_client):
383     try:
384         floating_ips = neutron_client.list_floatingips()
385         return floating_ips['floatingips']
386     except Exception as e:
387         logger.error("Error [get_floating_ips(neutron_client)]: %s" % e)
388         return None
389
390
391 def get_hypervisors(nova_client):
392     try:
393         nodes = []
394         hypervisors = nova_client.hypervisors.list()
395         for hypervisor in hypervisors:
396             if hypervisor.state == "up":
397                 nodes.append(hypervisor.hypervisor_hostname)
398         return nodes
399     except Exception as e:
400         logger.error("Error [get_hypervisors(nova_client)]: %s" % e)
401         return None
402
403
404 def create_aggregate(nova_client, aggregate_name, av_zone):
405     try:
406         nova_client.aggregates.create(aggregate_name, av_zone)
407         return True
408     except Exception as e:
409         logger.error("Error [create_aggregate(nova_client, %s, %s)]: %s"
410                      % (aggregate_name, av_zone, e))
411         return None
412
413
414 def add_host_to_aggregate(nova_client, aggregate_name, compute_host):
415     try:
416         aggregate_id = get_aggregate_id(nova_client, aggregate_name)
417         nova_client.aggregates.add_host(aggregate_id, compute_host)
418         return True
419     except Exception as e:
420         logger.error("Error [add_host_to_aggregate(nova_client, %s, %s)]: %s"
421                      % (aggregate_name, compute_host, e))
422         return None
423
424
425 def create_aggregate_with_host(
426         nova_client, aggregate_name, av_zone, compute_host):
427     try:
428         create_aggregate(nova_client, aggregate_name, av_zone)
429         add_host_to_aggregate(nova_client, aggregate_name, compute_host)
430         return True
431     except Exception as e:
432         logger.error("Error [create_aggregate_with_host("
433                      "nova_client, %s, %s, %s)]: %s"
434                      % (aggregate_name, av_zone, compute_host, e))
435         return None
436
437
438 def create_instance(flavor_name,
439                     image_id,
440                     network_id,
441                     instance_name="functest-vm",
442                     confdrive=True,
443                     userdata=None,
444                     av_zone='',
445                     fixed_ip=None,
446                     files=None):
447     nova_client = get_nova_client()
448     try:
449         flavor = nova_client.flavors.find(name=flavor_name)
450     except:
451         flavors = nova_client.flavors.list()
452         logger.error("Error: Flavor '%s' not found. Available flavors are: "
453                      "\n%s" % (flavor_name, flavors))
454         return None
455     if fixed_ip is not None:
456         nics = {"net-id": network_id, "v4-fixed-ip": fixed_ip}
457     else:
458         nics = {"net-id": network_id}
459     if userdata is None:
460         instance = nova_client.servers.create(
461             name=instance_name,
462             flavor=flavor,
463             image=image_id,
464             nics=[nics],
465             availability_zone=av_zone,
466             files=files
467         )
468     else:
469         instance = nova_client.servers.create(
470             name=instance_name,
471             flavor=flavor,
472             image=image_id,
473             nics=[nics],
474             config_drive=confdrive,
475             userdata=userdata,
476             availability_zone=av_zone,
477             files=files
478         )
479     return instance
480
481
482 def create_instance_and_wait_for_active(flavor_name,
483                                         image_id,
484                                         network_id,
485                                         instance_name="",
486                                         config_drive=False,
487                                         userdata="",
488                                         av_zone='',
489                                         fixed_ip=None,
490                                         files=None):
491     SLEEP = 3
492     VM_BOOT_TIMEOUT = 180
493     nova_client = get_nova_client()
494     instance = create_instance(flavor_name,
495                                image_id,
496                                network_id,
497                                instance_name,
498                                config_drive,
499                                userdata,
500                                av_zone=av_zone,
501                                fixed_ip=fixed_ip,
502                                files=files)
503     count = VM_BOOT_TIMEOUT / SLEEP
504     for n in range(count, -1, -1):
505         status = get_instance_status(nova_client, instance)
506         if status is None:
507             time.sleep(SLEEP)
508             continue
509         elif status.lower() == "active":
510             return instance
511         elif status.lower() == "error":
512             logger.error("The instance %s went to ERROR status."
513                          % instance_name)
514             return None
515         time.sleep(SLEEP)
516     logger.error("Timeout booting the instance %s." % instance_name)
517     return None
518
519
520 def create_floating_ip(neutron_client):
521     extnet_id = get_external_net_id(neutron_client)
522     props = {'floating_network_id': extnet_id}
523     try:
524         ip_json = neutron_client.create_floatingip({'floatingip': props})
525         fip_addr = ip_json['floatingip']['floating_ip_address']
526         fip_id = ip_json['floatingip']['id']
527     except Exception as e:
528         logger.error("Error [create_floating_ip(neutron_client)]: %s" % e)
529         return None
530     return {'fip_addr': fip_addr, 'fip_id': fip_id}
531
532
533 def attach_floating_ip(neutron_client, port_id):
534     extnet_id = get_external_net_id(neutron_client)
535     props = {'floating_network_id': extnet_id,
536              'port_id': port_id}
537     try:
538         return neutron_client.create_floatingip({'floatingip': props})
539     except Exception as e:
540         logger.error("Error [Attach_floating_ip(neutron_client), %s]: %s"
541                      % (port_id, e))
542         return None
543
544
545 def add_floating_ip(nova_client, server_id, floatingip_addr):
546     try:
547         nova_client.servers.add_floating_ip(server_id, floatingip_addr)
548         return True
549     except Exception as e:
550         logger.error("Error [add_floating_ip(nova_client, '%s', '%s')]: %s"
551                      % (server_id, floatingip_addr, e))
552         return False
553
554
555 def delete_instance(nova_client, instance_id):
556     try:
557         nova_client.servers.force_delete(instance_id)
558         return True
559     except Exception as e:
560         logger.error("Error [delete_instance(nova_client, '%s')]: %s"
561                      % (instance_id, e))
562         return False
563
564
565 def delete_floating_ip(neutron_client, floatingip_id):
566     try:
567         neutron_client.delete_floatingip(floatingip_id)
568         return True
569     except Exception as e:
570         logger.error("Error [delete_floating_ip(neutron_client, '%s')]: %s"
571                      % (floatingip_id, e))
572         return False
573
574
575 def remove_host_from_aggregate(nova_client, aggregate_name, compute_host):
576     try:
577         aggregate_id = get_aggregate_id(nova_client, aggregate_name)
578         nova_client.aggregates.remove_host(aggregate_id, compute_host)
579         return True
580     except Exception as e:
581         logger.error("Error [remove_host_from_aggregate(nova_client, %s, %s)]:"
582                      " %s" % (aggregate_name, compute_host, e))
583         return False
584
585
586 def remove_hosts_from_aggregate(nova_client, aggregate_name):
587     aggregate_id = get_aggregate_id(nova_client, aggregate_name)
588     hosts = nova_client.aggregates.get(aggregate_id).hosts
589     assert(
590         all(remove_host_from_aggregate(nova_client, aggregate_name, host)
591             for host in hosts))
592
593
594 def delete_aggregate(nova_client, aggregate_name):
595     try:
596         remove_hosts_from_aggregate(nova_client, aggregate_name)
597         nova_client.aggregates.delete(aggregate_name)
598         return True
599     except Exception as e:
600         logger.error("Error [delete_aggregate(nova_client, %s)]: %s"
601                      % (aggregate_name, e))
602         return False
603
604
605 # *********************************************
606 #   NEUTRON
607 # *********************************************
608 def get_network_list(neutron_client):
609     network_list = neutron_client.list_networks()['networks']
610     if len(network_list) == 0:
611         return None
612     else:
613         return network_list
614
615
616 def get_router_list(neutron_client):
617     router_list = neutron_client.list_routers()['routers']
618     if len(router_list) == 0:
619         return None
620     else:
621         return router_list
622
623
624 def get_port_list(neutron_client):
625     port_list = neutron_client.list_ports()['ports']
626     if len(port_list) == 0:
627         return None
628     else:
629         return port_list
630
631
632 def get_network_id(neutron_client, network_name):
633     networks = neutron_client.list_networks()['networks']
634     id = ''
635     for n in networks:
636         if n['name'] == network_name:
637             id = n['id']
638             break
639     return id
640
641
642 def get_subnet_id(neutron_client, subnet_name):
643     subnets = neutron_client.list_subnets()['subnets']
644     id = ''
645     for s in subnets:
646         if s['name'] == subnet_name:
647             id = s['id']
648             break
649     return id
650
651
652 def get_router_id(neutron_client, router_name):
653     routers = neutron_client.list_routers()['routers']
654     id = ''
655     for r in routers:
656         if r['name'] == router_name:
657             id = r['id']
658             break
659     return id
660
661
662 def get_private_net(neutron_client):
663     # Checks if there is an existing shared private network
664     networks = neutron_client.list_networks()['networks']
665     if len(networks) == 0:
666         return None
667     for net in networks:
668         if (net['router:external'] is False) and (net['shared'] is True):
669             return net
670     return None
671
672
673 def get_external_net(neutron_client):
674     if (env.get('EXTERNAL_NETWORK')):
675         return env.get('EXTERNAL_NETWORK')
676     for network in neutron_client.list_networks()['networks']:
677         if network['router:external']:
678             return network['name']
679     return None
680
681
682 def get_external_net_id(neutron_client):
683     if (env.get('EXTERNAL_NETWORK')):
684         networks = neutron_client.list_networks(
685             name=env.get('EXTERNAL_NETWORK'))
686         net_id = networks['networks'][0]['id']
687         return net_id
688     for network in neutron_client.list_networks()['networks']:
689         if network['router:external']:
690             return network['id']
691     return None
692
693
694 def check_neutron_net(neutron_client, net_name):
695     for network in neutron_client.list_networks()['networks']:
696         if network['name'] == net_name:
697             for subnet in network['subnets']:
698                 return True
699     return False
700
701
702 def create_neutron_net(neutron_client, name):
703     json_body = {'network': {'name': name,
704                              'admin_state_up': True}}
705     try:
706         network = neutron_client.create_network(body=json_body)
707         network_dict = network['network']
708         return network_dict['id']
709     except Exception as e:
710         logger.error("Error [create_neutron_net(neutron_client, '%s')]: %s"
711                      % (name, e))
712         return None
713
714
715 def create_neutron_subnet(neutron_client, name, cidr, net_id,
716                           dns=['8.8.8.8', '8.8.4.4']):
717     json_body = {'subnets': [{'name': name, 'cidr': cidr,
718                               'ip_version': 4, 'network_id': net_id,
719                               'dns_nameservers': dns}]}
720
721     try:
722         subnet = neutron_client.create_subnet(body=json_body)
723         return subnet['subnets'][0]['id']
724     except Exception as e:
725         logger.error("Error [create_neutron_subnet(neutron_client, '%s', "
726                      "'%s', '%s')]: %s" % (name, cidr, net_id, e))
727         return None
728
729
730 def create_neutron_router(neutron_client, name):
731     json_body = {'router': {'name': name, 'admin_state_up': True}}
732     try:
733         router = neutron_client.create_router(json_body)
734         return router['router']['id']
735     except Exception as e:
736         logger.error("Error [create_neutron_router(neutron_client, '%s')]: %s"
737                      % (name, e))
738         return None
739
740
741 def create_neutron_port(neutron_client, name, network_id, ip):
742     json_body = {'port': {
743                  'admin_state_up': True,
744                  'name': name,
745                  'network_id': network_id,
746                  'fixed_ips': [{"ip_address": ip}]
747                  }}
748     try:
749         port = neutron_client.create_port(body=json_body)
750         return port['port']['id']
751     except Exception as e:
752         logger.error("Error [create_neutron_port(neutron_client, '%s', '%s', "
753                      "'%s')]: %s" % (name, network_id, ip, e))
754         return None
755
756
757 def update_neutron_net(neutron_client, network_id, shared=False):
758     json_body = {'network': {'shared': shared}}
759     try:
760         neutron_client.update_network(network_id, body=json_body)
761         return True
762     except Exception as e:
763         logger.error("Error [update_neutron_net(neutron_client, '%s', '%s')]: "
764                      "%s" % (network_id, str(shared), e))
765         return False
766
767
768 def update_neutron_port(neutron_client, port_id, device_owner):
769     json_body = {'port': {
770                  'device_owner': device_owner,
771                  }}
772     try:
773         port = neutron_client.update_port(port=port_id,
774                                           body=json_body)
775         return port['port']['id']
776     except Exception as e:
777         logger.error("Error [update_neutron_port(neutron_client, '%s', '%s')]:"
778                      " %s" % (port_id, device_owner, e))
779         return None
780
781
782 def add_interface_router(neutron_client, router_id, subnet_id):
783     json_body = {"subnet_id": subnet_id}
784     try:
785         neutron_client.add_interface_router(router=router_id, body=json_body)
786         return True
787     except Exception as e:
788         logger.error("Error [add_interface_router(neutron_client, '%s', "
789                      "'%s')]: %s" % (router_id, subnet_id, e))
790         return False
791
792
793 def add_gateway_router(neutron_client, router_id):
794     ext_net_id = get_external_net_id(neutron_client)
795     router_dict = {'network_id': ext_net_id}
796     try:
797         neutron_client.add_gateway_router(router_id, router_dict)
798         return True
799     except Exception as e:
800         logger.error("Error [add_gateway_router(neutron_client, '%s')]: %s"
801                      % (router_id, e))
802         return False
803
804
805 def delete_neutron_net(neutron_client, network_id):
806     try:
807         neutron_client.delete_network(network_id)
808         return True
809     except Exception as e:
810         logger.error("Error [delete_neutron_net(neutron_client, '%s')]: %s"
811                      % (network_id, e))
812         return False
813
814
815 def delete_neutron_subnet(neutron_client, subnet_id):
816     try:
817         neutron_client.delete_subnet(subnet_id)
818         return True
819     except Exception as e:
820         logger.error("Error [delete_neutron_subnet(neutron_client, '%s')]: %s"
821                      % (subnet_id, e))
822         return False
823
824
825 def delete_neutron_router(neutron_client, router_id):
826     try:
827         neutron_client.delete_router(router=router_id)
828         return True
829     except Exception as e:
830         logger.error("Error [delete_neutron_router(neutron_client, '%s')]: %s"
831                      % (router_id, e))
832         return False
833
834
835 def delete_neutron_port(neutron_client, port_id):
836     try:
837         neutron_client.delete_port(port_id)
838         return True
839     except Exception as e:
840         logger.error("Error [delete_neutron_port(neutron_client, '%s')]: %s"
841                      % (port_id, e))
842         return False
843
844
845 def remove_interface_router(neutron_client, router_id, subnet_id):
846     json_body = {"subnet_id": subnet_id}
847     try:
848         neutron_client.remove_interface_router(router=router_id,
849                                                body=json_body)
850         return True
851     except Exception as e:
852         logger.error("Error [remove_interface_router(neutron_client, '%s', "
853                      "'%s')]: %s" % (router_id, subnet_id, e))
854         return False
855
856
857 def remove_gateway_router(neutron_client, router_id):
858     try:
859         neutron_client.remove_gateway_router(router_id)
860         return True
861     except Exception as e:
862         logger.error("Error [remove_gateway_router(neutron_client, '%s')]: %s"
863                      % (router_id, e))
864         return False
865
866
867 def create_network_full(neutron_client,
868                         net_name,
869                         subnet_name,
870                         router_name,
871                         cidr,
872                         dns=['8.8.8.8', '8.8.4.4']):
873
874     # Check if the network already exists
875     network_id = get_network_id(neutron_client, net_name)
876     subnet_id = get_subnet_id(neutron_client, subnet_name)
877     router_id = get_router_id(neutron_client, router_name)
878
879     if network_id != '' and subnet_id != '' and router_id != '':
880         logger.info("A network with name '%s' already exists..." % net_name)
881     else:
882         neutron_client.format = 'json'
883
884         logger.info('Creating neutron network %s...' % net_name)
885         if network_id == '':
886             network_id = create_neutron_net(neutron_client, net_name)
887         if not network_id:
888             return False
889         logger.debug("Network '%s' created successfully" % network_id)
890
891         logger.debug('Creating Subnet....')
892         if subnet_id == '':
893             subnet_id = create_neutron_subnet(neutron_client, subnet_name,
894                                               cidr, network_id, dns)
895         if not subnet_id:
896             return None
897         logger.debug("Subnet '%s' created successfully" % subnet_id)
898
899         logger.debug('Creating Router...')
900         if router_id == '':
901             router_id = create_neutron_router(neutron_client, router_name)
902         if not router_id:
903             return None
904         logger.debug("Router '%s' created successfully" % router_id)
905
906         logger.debug('Adding router to subnet...')
907
908         if not add_interface_router(neutron_client, router_id, subnet_id):
909             return None
910         logger.debug("Interface added successfully.")
911
912         logger.debug('Adding gateway to router...')
913         if not add_gateway_router(neutron_client, router_id):
914             return None
915         logger.debug("Gateway added successfully.")
916
917     network_dic = {'net_id': network_id,
918                    'subnet_id': subnet_id,
919                    'router_id': router_id}
920     return network_dic
921
922
923 def create_shared_network_full(net_name, subnt_name, router_name, subnet_cidr):
924     neutron_client = get_neutron_client()
925
926     network_dic = create_network_full(neutron_client,
927                                       net_name,
928                                       subnt_name,
929                                       router_name,
930                                       subnet_cidr)
931     if network_dic:
932         if not update_neutron_net(neutron_client,
933                                   network_dic['net_id'],
934                                   shared=True):
935             logger.error("Failed to update network %s..." % net_name)
936             return None
937         else:
938             logger.debug("Network '%s' is available..." % net_name)
939     else:
940         logger.error("Network %s creation failed" % net_name)
941         return None
942     return network_dic
943
944
945 # *********************************************
946 #   SEC GROUPS
947 # *********************************************
948
949
950 def get_security_groups(neutron_client):
951     try:
952         security_groups = neutron_client.list_security_groups()[
953             'security_groups']
954         return security_groups
955     except Exception as e:
956         logger.error("Error [get_security_groups(neutron_client)]: %s" % e)
957         return None
958
959
960 def get_security_group_id(neutron_client, sg_name):
961     security_groups = get_security_groups(neutron_client)
962     id = ''
963     for sg in security_groups:
964         if sg['name'] == sg_name:
965             id = sg['id']
966             break
967     return id
968
969
970 def create_security_group(neutron_client, sg_name, sg_description):
971     json_body = {'security_group': {'name': sg_name,
972                                     'description': sg_description}}
973     try:
974         secgroup = neutron_client.create_security_group(json_body)
975         return secgroup['security_group']
976     except Exception as e:
977         logger.error("Error [create_security_group(neutron_client, '%s', "
978                      "'%s')]: %s" % (sg_name, sg_description, e))
979         return None
980
981
982 def create_secgroup_rule(neutron_client, sg_id, direction, protocol,
983                          port_range_min=None, port_range_max=None):
984     # We create a security group in 2 steps
985     # 1 - we check the format and set the json body accordingly
986     # 2 - we call neturon client to create the security group
987
988     # Format check
989     json_body = {'security_group_rule': {'direction': direction,
990                                          'security_group_id': sg_id,
991                                          'protocol': protocol}}
992     # parameters may be
993     # - both None => we do nothing
994     # - both Not None => we add them to the json description
995     # but one cannot be None is the other is not None
996     if (port_range_min is not None and port_range_max is not None):
997         # add port_range in json description
998         json_body['security_group_rule']['port_range_min'] = port_range_min
999         json_body['security_group_rule']['port_range_max'] = port_range_max
1000         logger.debug("Security_group format set (port range included)")
1001     else:
1002         # either both port range are set to None => do nothing
1003         # or one is set but not the other => log it and return False
1004         if port_range_min is None and port_range_max is None:
1005             logger.debug("Security_group format set (no port range mentioned)")
1006         else:
1007             logger.error("Bad security group format."
1008                          "One of the port range is not properly set:"
1009                          "range min: {},"
1010                          "range max: {}".format(port_range_min,
1011                                                 port_range_max))
1012             return False
1013
1014     # Create security group using neutron client
1015     try:
1016         neutron_client.create_security_group_rule(json_body)
1017         return True
1018     except:
1019         logger.exception("Impossible to create_security_group_rule,"
1020                          "security group rule probably already exists")
1021         return False
1022
1023
1024 def get_security_group_rules(neutron_client, sg_id):
1025     try:
1026         security_rules = neutron_client.list_security_group_rules()[
1027             'security_group_rules']
1028         security_rules = [rule for rule in security_rules
1029                           if rule["security_group_id"] == sg_id]
1030         return security_rules
1031     except Exception as e:
1032         logger.error("Error [get_security_group_rules(neutron_client, sg_id)]:"
1033                      " %s" % e)
1034         return None
1035
1036
1037 def check_security_group_rules(neutron_client, sg_id, direction, protocol,
1038                                port_min=None, port_max=None):
1039     try:
1040         security_rules = get_security_group_rules(neutron_client, sg_id)
1041         security_rules = [rule for rule in security_rules
1042                           if (rule["direction"].lower() == direction and
1043                               rule["protocol"].lower() == protocol and
1044                               rule["port_range_min"] == port_min and
1045                               rule["port_range_max"] == port_max)]
1046         if len(security_rules) == 0:
1047             return True
1048         else:
1049             return False
1050     except Exception as e:
1051         logger.error("Error [check_security_group_rules("
1052                      " neutron_client, sg_id, direction,"
1053                      " protocol, port_min=None, port_max=None)]: "
1054                      "%s" % e)
1055         return None
1056
1057
1058 def create_security_group_full(neutron_client,
1059                                sg_name, sg_description):
1060     sg_id = get_security_group_id(neutron_client, sg_name)
1061     if sg_id != '':
1062         logger.info("Using existing security group '%s'..." % sg_name)
1063     else:
1064         logger.info("Creating security group  '%s'..." % sg_name)
1065         SECGROUP = create_security_group(neutron_client,
1066                                          sg_name,
1067                                          sg_description)
1068         if not SECGROUP:
1069             logger.error("Failed to create the security group...")
1070             return None
1071
1072         sg_id = SECGROUP['id']
1073
1074         logger.debug("Security group '%s' with ID=%s created successfully."
1075                      % (SECGROUP['name'], sg_id))
1076
1077         logger.debug("Adding ICMP rules in security group '%s'..."
1078                      % sg_name)
1079         if not create_secgroup_rule(neutron_client, sg_id,
1080                                     'ingress', 'icmp'):
1081             logger.error("Failed to create the security group rule...")
1082             return None
1083
1084         logger.debug("Adding SSH rules in security group '%s'..."
1085                      % sg_name)
1086         if not create_secgroup_rule(
1087                 neutron_client, sg_id, 'ingress', 'tcp', '22', '22'):
1088             logger.error("Failed to create the security group rule...")
1089             return None
1090
1091         if not create_secgroup_rule(
1092                 neutron_client, sg_id, 'egress', 'tcp', '22', '22'):
1093             logger.error("Failed to create the security group rule...")
1094             return None
1095     return sg_id
1096
1097
1098 def add_secgroup_to_instance(nova_client, instance_id, secgroup_id):
1099     try:
1100         nova_client.servers.add_security_group(instance_id, secgroup_id)
1101         return True
1102     except Exception as e:
1103         logger.error("Error [add_secgroup_to_instance(nova_client, '%s', "
1104                      "'%s')]: %s" % (instance_id, secgroup_id, e))
1105         return False
1106
1107
1108 def update_sg_quota(neutron_client, tenant_id, sg_quota, sg_rule_quota):
1109     json_body = {"quota": {
1110         "security_group": sg_quota,
1111         "security_group_rule": sg_rule_quota
1112     }}
1113
1114     try:
1115         neutron_client.update_quota(tenant_id=tenant_id,
1116                                     body=json_body)
1117         return True
1118     except Exception as e:
1119         logger.error("Error [update_sg_quota(neutron_client, '%s', '%s', "
1120                      "'%s')]: %s" % (tenant_id, sg_quota, sg_rule_quota, e))
1121         return False
1122
1123
1124 def delete_security_group(neutron_client, secgroup_id):
1125     try:
1126         neutron_client.delete_security_group(secgroup_id)
1127         return True
1128     except Exception as e:
1129         logger.error("Error [delete_security_group(neutron_client, '%s')]: %s"
1130                      % (secgroup_id, e))
1131         return False
1132
1133
1134 # *********************************************
1135 #   GLANCE
1136 # *********************************************
1137 def get_images(conn):
1138     try:
1139         images = conn.image.images()
1140         return images
1141     except Exception as e:
1142         logger.error("Error [get_images]: %s" % e)
1143         return None
1144
1145
1146 def get_image_id(conn, image_name):
1147     images = conn.image.images()
1148     id = ''
1149     for i in images:
1150         if i.name == image_name:
1151             id = i.id
1152             break
1153     return id
1154
1155
1156 def create_glance_image(conn,
1157                         image_name,
1158                         file_path,
1159                         disk="qcow2",
1160                         extra_properties={},
1161                         container="bare",
1162                         public="public"):
1163     if not os.path.isfile(file_path):
1164         logger.error("Error: file %s does not exist." % file_path)
1165         return None
1166     try:
1167         image_id = get_image_id(conn, image_name)
1168         if image_id != '':
1169             logger.info("Image %s already exists." % image_name)
1170         else:
1171             logger.info("Creating image '%s' from '%s'..." % (image_name,
1172                                                               file_path))
1173             with open(file_path) as image_data:
1174                 image = conn.image.upload_image(name=image_name,
1175                                                 is_public=public,
1176                                                 disk_format=disk,
1177                                                 container_format=container,
1178                                                 data=image_data,
1179                                                 **extra_properties)
1180             image_id = image.id
1181         return image_id
1182     except Exception as e:
1183         logger.error("Error [create_glance_image(image, '%s', '%s', "
1184                      "'%s')]: %s" % (image_name, file_path, public, e))
1185         return None
1186
1187
1188 def get_or_create_image(name, path, format, extra_properties):
1189     image_exists = False
1190     conn = get_os_connection()
1191
1192     image_id = get_image_id(conn, name)
1193     if image_id != '':
1194         logger.info("Using existing image '%s'..." % name)
1195         image_exists = True
1196     else:
1197         logger.info("Creating image '%s' from '%s'..." % (name, path))
1198         image_id = create_glance_image(conn,
1199                                        name,
1200                                        path,
1201                                        format,
1202                                        extra_properties)
1203         if not image_id:
1204             logger.error("Failed to create a Glance image...")
1205         else:
1206             logger.debug("Image '%s' with ID=%s created successfully."
1207                          % (name, image_id))
1208
1209     return image_exists, image_id
1210
1211
1212 def delete_glance_image(conn, image_id):
1213     try:
1214         conn.image.delete_image(image_id)
1215         return True
1216     except Exception as e:
1217         logger.error("Error [delete_glance_image(image, '%s')]: %s"
1218                      % (image_id, e))
1219         return False
1220
1221
1222 # *********************************************
1223 #   CINDER
1224 # *********************************************
1225 def get_volumes(cinder_client):
1226     try:
1227         volumes = cinder_client.volumes.list(search_opts={'all_tenants': 1})
1228         return volumes
1229     except Exception as e:
1230         logger.error("Error [get_volumes(cinder_client)]: %s" % e)
1231         return None
1232
1233
1234 def update_cinder_quota(cinder_client, tenant_id, vols_quota,
1235                         snapshots_quota, gigabytes_quota):
1236     quotas_values = {"volumes": vols_quota,
1237                      "snapshots": snapshots_quota,
1238                      "gigabytes": gigabytes_quota}
1239
1240     try:
1241         cinder_client.quotas.update(tenant_id, **quotas_values)
1242         return True
1243     except Exception as e:
1244         logger.error("Error [update_cinder_quota(cinder_client, '%s', '%s', "
1245                      "'%s' '%s')]: %s" % (tenant_id, vols_quota,
1246                                           snapshots_quota, gigabytes_quota, e))
1247         return False
1248
1249
1250 def delete_volume(cinder_client, volume_id, forced=False):
1251     try:
1252         if forced:
1253             try:
1254                 cinder_client.volumes.detach(volume_id)
1255             except:
1256                 logger.error(sys.exc_info()[0])
1257             cinder_client.volumes.force_delete(volume_id)
1258         else:
1259             cinder_client.volumes.delete(volume_id)
1260         return True
1261     except Exception as e:
1262         logger.error("Error [delete_volume(cinder_client, '%s', '%s')]: %s"
1263                      % (volume_id, str(forced), e))
1264         return False
1265
1266
1267 # *********************************************
1268 #   KEYSTONE
1269 # *********************************************
1270 def get_tenants(keystone_client):
1271     try:
1272         if is_keystone_v3():
1273             tenants = keystone_client.projects.list()
1274         else:
1275             tenants = keystone_client.tenants.list()
1276         return tenants
1277     except Exception as e:
1278         logger.error("Error [get_tenants(keystone_client)]: %s" % e)
1279         return None
1280
1281
1282 def get_users(keystone_client):
1283     try:
1284         users = keystone_client.users.list()
1285         return users
1286     except Exception as e:
1287         logger.error("Error [get_users(keystone_client)]: %s" % e)
1288         return None
1289
1290
1291 def get_tenant_id(keystone_client, tenant_name):
1292     tenants = get_tenants(keystone_client)
1293     id = ''
1294     for t in tenants:
1295         if t.name == tenant_name:
1296             id = t.id
1297             break
1298     return id
1299
1300
1301 def get_user_id(keystone_client, user_name):
1302     users = get_users(keystone_client)
1303     id = ''
1304     for u in users:
1305         if u.name == user_name:
1306             id = u.id
1307             break
1308     return id
1309
1310
1311 def get_role_id(keystone_client, role_name):
1312     roles = keystone_client.roles.list()
1313     id = ''
1314     for r in roles:
1315         if r.name == role_name:
1316             id = r.id
1317             break
1318     return id
1319
1320
1321 def get_domain_id(keystone_client, domain_name):
1322     domains = keystone_client.domains.list()
1323     id = ''
1324     for d in domains:
1325         if d.name == domain_name:
1326             id = d.id
1327             break
1328     return id
1329
1330
1331 def create_tenant(keystone_client, tenant_name, tenant_description):
1332     try:
1333         if is_keystone_v3():
1334             domain_name = os.environ['OS_PROJECT_DOMAIN_NAME']
1335             domain_id = get_domain_id(keystone_client, domain_name)
1336             tenant = keystone_client.projects.create(
1337                 name=tenant_name,
1338                 description=tenant_description,
1339                 domain=domain_id,
1340                 enabled=True)
1341         else:
1342             tenant = keystone_client.tenants.create(tenant_name,
1343                                                     tenant_description,
1344                                                     enabled=True)
1345         return tenant.id
1346     except Exception as e:
1347         logger.error("Error [create_tenant(keystone_client, '%s', '%s')]: %s"
1348                      % (tenant_name, tenant_description, e))
1349         return None
1350
1351
1352 def get_or_create_tenant(keystone_client, tenant_name, tenant_description):
1353     tenant_id = get_tenant_id(keystone_client, tenant_name)
1354     if not tenant_id:
1355         tenant_id = create_tenant(keystone_client, tenant_name,
1356                                   tenant_description)
1357
1358     return tenant_id
1359
1360
1361 def get_or_create_tenant_for_vnf(keystone_client, tenant_name,
1362                                  tenant_description):
1363     """Get or Create a Tenant
1364
1365         Args:
1366             keystone_client: keystone client reference
1367             tenant_name: the name of the tenant
1368             tenant_description: the description of the tenant
1369
1370         return False if tenant retrieved though get
1371         return True if tenant created
1372         raise Exception if error during processing
1373     """
1374     try:
1375         tenant_id = get_tenant_id(keystone_client, tenant_name)
1376         if not tenant_id:
1377             tenant_id = create_tenant(keystone_client, tenant_name,
1378                                       tenant_description)
1379             return True
1380         else:
1381             return False
1382     except:
1383         raise Exception("Impossible to create a Tenant for the VNF {}".format(
1384                             tenant_name))
1385
1386
1387 def create_user(keystone_client, user_name, user_password,
1388                 user_email, tenant_id):
1389     try:
1390         if is_keystone_v3():
1391             user = keystone_client.users.create(name=user_name,
1392                                                 password=user_password,
1393                                                 email=user_email,
1394                                                 project_id=tenant_id,
1395                                                 enabled=True)
1396         else:
1397             user = keystone_client.users.create(user_name,
1398                                                 user_password,
1399                                                 user_email,
1400                                                 tenant_id,
1401                                                 enabled=True)
1402         return user.id
1403     except Exception as e:
1404         logger.error("Error [create_user(keystone_client, '%s', '%s', '%s'"
1405                      "'%s')]: %s" % (user_name, user_password,
1406                                      user_email, tenant_id, e))
1407         return None
1408
1409
1410 def get_or_create_user(keystone_client, user_name, user_password,
1411                        tenant_id, user_email=None):
1412     user_id = get_user_id(keystone_client, user_name)
1413     if not user_id:
1414         user_id = create_user(keystone_client, user_name, user_password,
1415                               user_email, tenant_id)
1416     return user_id
1417
1418
1419 def get_or_create_user_for_vnf(keystone_client, vnf_ref):
1420     """Get or Create user for VNF
1421
1422         Args:
1423             keystone_client: keystone client reference
1424             vnf_ref: VNF reference used as user name & password, tenant name
1425
1426         return False if user retrieved through get
1427         return True if user created
1428         raise Exception if error during processing
1429     """
1430     try:
1431         user_id = get_user_id(keystone_client, vnf_ref)
1432         tenant_id = get_tenant_id(keystone_client, vnf_ref)
1433         created = False
1434         if not user_id:
1435             user_id = create_user(keystone_client, vnf_ref, vnf_ref,
1436                                   "", tenant_id)
1437             created = True
1438         try:
1439             role_id = get_role_id(keystone_client, 'admin')
1440             tenant_id = get_tenant_id(keystone_client, vnf_ref)
1441             add_role_user(keystone_client, user_id, role_id, tenant_id)
1442         except:
1443             logger.warn("Cannot associate user to role admin on tenant")
1444         return created
1445     except:
1446         raise Exception("Impossible to create a user for the VNF {}".format(
1447             vnf_ref))
1448
1449
1450 def add_role_user(keystone_client, user_id, role_id, tenant_id):
1451     try:
1452         if is_keystone_v3():
1453             keystone_client.roles.grant(role=role_id,
1454                                         user=user_id,
1455                                         project=tenant_id)
1456         else:
1457             keystone_client.roles.add_user_role(user_id, role_id, tenant_id)
1458         return True
1459     except Exception as e:
1460         logger.error("Error [add_role_user(keystone_client, '%s', '%s'"
1461                      "'%s')]: %s " % (user_id, role_id, tenant_id, e))
1462         return False
1463
1464
1465 def delete_tenant(keystone_client, tenant_id):
1466     try:
1467         if is_keystone_v3():
1468             keystone_client.projects.delete(tenant_id)
1469         else:
1470             keystone_client.tenants.delete(tenant_id)
1471         return True
1472     except Exception as e:
1473         logger.error("Error [delete_tenant(keystone_client, '%s')]: %s"
1474                      % (tenant_id, e))
1475         return False
1476
1477
1478 def delete_user(keystone_client, user_id):
1479     try:
1480         keystone_client.users.delete(user_id)
1481         return True
1482     except Exception as e:
1483         logger.error("Error [delete_user(keystone_client, '%s')]: %s"
1484                      % (user_id, e))
1485         return False
1486
1487
1488 # *********************************************
1489 #   HEAT
1490 # *********************************************
1491 def get_resource(heat_client, stack_id, resource):
1492     try:
1493         resources = heat_client.resources.get(stack_id, resource)
1494         return resources
1495     except Exception as e:
1496         logger.error("Error [get_resource]: %s" % e)
1497         return None