remove extra specs for the flavor
[sdnvpn.git] / sdnvpn / lib / openstack_utils.py
1 #!/usr/bin/env python
2 #
3 # jose.lausuch@ericsson.com
4 # valentin.boucher@orange.com
5 # All rights reserved. This program and the accompanying materials
6 # are made available under the terms of the Apache License, Version 2.0
7 # which accompanies this distribution, and is available at
8 # http://www.apache.org/licenses/LICENSE-2.0
9 #
10
11 import logging
12 import os.path
13 import shutil
14 import sys
15 import time
16 import urllib
17
18 from keystoneauth1 import loading
19 from keystoneauth1 import session
20 from cinderclient import client as cinderclient
21 from glanceclient import client as glanceclient
22 from heatclient import client as heatclient
23 from novaclient import client as novaclient
24 from keystoneclient import client as keystoneclient
25 from neutronclient.neutron import client as neutronclient
26
27 from functest.utils import config
28 from functest.utils import env
29
30 logger = logging.getLogger(__name__)
31
32 DEFAULT_API_VERSION = '2'
33 DEFAULT_HEAT_API_VERSION = '1'
34
35
36 # *********************************************
37 #   CREDENTIALS
38 # *********************************************
39 class MissingEnvVar(Exception):
40
41     def __init__(self, var):
42         self.var = var
43
44     def __str__(self):
45         return str.format("Please set the mandatory env var: {}", self.var)
46
47
48 def is_keystone_v3():
49     keystone_api_version = os.getenv('OS_IDENTITY_API_VERSION')
50     if (keystone_api_version is None or
51             keystone_api_version == '2'):
52         return False
53     else:
54         return True
55
56
57 def get_rc_env_vars():
58     env_vars = ['OS_AUTH_URL', 'OS_USERNAME', 'OS_PASSWORD']
59     if is_keystone_v3():
60         env_vars.extend(['OS_PROJECT_NAME',
61                          'OS_USER_DOMAIN_NAME',
62                          'OS_PROJECT_DOMAIN_NAME'])
63     else:
64         env_vars.extend(['OS_TENANT_NAME'])
65     return env_vars
66
67
68 def check_credentials():
69     """
70     Check if the OpenStack credentials (openrc) are sourced
71     """
72     env_vars = get_rc_env_vars()
73     return all(map(lambda v: v in os.environ and os.environ[v], env_vars))
74
75
76 def get_env_cred_dict():
77     env_cred_dict = {
78         'OS_USERNAME': 'username',
79         'OS_PASSWORD': 'password',
80         'OS_AUTH_URL': 'auth_url',
81         'OS_TENANT_NAME': 'tenant_name',
82         'OS_USER_DOMAIN_NAME': 'user_domain_name',
83         'OS_PROJECT_DOMAIN_NAME': 'project_domain_name',
84         'OS_PROJECT_NAME': 'project_name',
85         'OS_ENDPOINT_TYPE': 'endpoint_type',
86         'OS_REGION_NAME': 'region_name',
87         'OS_CACERT': 'https_cacert',
88         'OS_INSECURE': 'https_insecure'
89     }
90     return env_cred_dict
91
92
93 def get_credentials(other_creds={}):
94     """Returns a creds dictionary filled with parsed from env
95     """
96     creds = {}
97     env_vars = get_rc_env_vars()
98     env_cred_dict = get_env_cred_dict()
99
100     for envvar in env_vars:
101         if os.getenv(envvar) is None:
102             raise MissingEnvVar(envvar)
103         else:
104             creds_key = env_cred_dict.get(envvar)
105             creds.update({creds_key: os.getenv(envvar)})
106
107     if 'tenant' in other_creds.keys():
108         if is_keystone_v3():
109             tenant = 'project_name'
110         else:
111             tenant = 'tenant_name'
112         other_creds[tenant] = other_creds.pop('tenant')
113
114     creds.update(other_creds)
115
116     return creds
117
118
119 def get_session_auth(other_creds={}):
120     loader = loading.get_plugin_loader('password')
121     creds = get_credentials(other_creds)
122     auth = loader.load_from_options(**creds)
123     return auth
124
125
126 def get_endpoint(service_type, interface='public'):
127     auth = get_session_auth()
128     return get_session().get_endpoint(auth=auth,
129                                       service_type=service_type,
130                                       interface=interface)
131
132
133 def get_session(other_creds={}):
134     auth = get_session_auth(other_creds)
135     https_cacert = os.getenv('OS_CACERT', '')
136     https_insecure = os.getenv('OS_INSECURE', '').lower() == 'true'
137     return session.Session(auth=auth,
138                            verify=(https_cacert or not https_insecure))
139
140
141 # *********************************************
142 #   CLIENTS
143 # *********************************************
144 def get_keystone_client_version():
145     api_version = os.getenv('OS_IDENTITY_API_VERSION')
146     if api_version is not None:
147         logger.info("OS_IDENTITY_API_VERSION is set in env as '%s'",
148                     api_version)
149         return api_version
150     return DEFAULT_API_VERSION
151
152
153 def get_keystone_client(other_creds={}):
154     sess = get_session(other_creds)
155     return keystoneclient.Client(get_keystone_client_version(),
156                                  session=sess,
157                                  interface=os.getenv('OS_INTERFACE', 'admin'))
158
159
160 def get_nova_client_version():
161     api_version = os.getenv('OS_COMPUTE_API_VERSION')
162     if api_version is not None:
163         logger.info("OS_COMPUTE_API_VERSION is set in env as '%s'",
164                     api_version)
165         return api_version
166     return DEFAULT_API_VERSION
167
168
169 def get_nova_client(other_creds={}):
170     sess = get_session(other_creds)
171     return novaclient.Client(get_nova_client_version(), session=sess)
172
173
174 def get_cinder_client_version():
175     api_version = os.getenv('OS_VOLUME_API_VERSION')
176     if api_version is not None:
177         logger.info("OS_VOLUME_API_VERSION is set in env as '%s'",
178                     api_version)
179         return api_version
180     return DEFAULT_API_VERSION
181
182
183 def get_cinder_client(other_creds={}):
184     sess = get_session(other_creds)
185     return cinderclient.Client(get_cinder_client_version(), session=sess)
186
187
188 def get_neutron_client_version():
189     api_version = os.getenv('OS_NETWORK_API_VERSION')
190     if api_version is not None:
191         logger.info("OS_NETWORK_API_VERSION is set in env as '%s'",
192                     api_version)
193         return api_version
194     return DEFAULT_API_VERSION
195
196
197 def get_neutron_client(other_creds={}):
198     sess = get_session(other_creds)
199     return neutronclient.Client(get_neutron_client_version(), session=sess)
200
201
202 def get_glance_client_version():
203     api_version = os.getenv('OS_IMAGE_API_VERSION')
204     if api_version is not None:
205         logger.info("OS_IMAGE_API_VERSION is set in env as '%s'", api_version)
206         return api_version
207     return DEFAULT_API_VERSION
208
209
210 def get_glance_client(other_creds={}):
211     sess = get_session(other_creds)
212     return glanceclient.Client(get_glance_client_version(), session=sess)
213
214
215 def get_heat_client_version():
216     api_version = os.getenv('OS_ORCHESTRATION_API_VERSION')
217     if api_version is not None:
218         logger.info("OS_ORCHESTRATION_API_VERSION is set in env as '%s'",
219                     api_version)
220         return api_version
221     return DEFAULT_HEAT_API_VERSION
222
223
224 def get_heat_client(other_creds={}):
225     sess = get_session(other_creds)
226     return heatclient.Client(get_heat_client_version(), session=sess)
227
228
229 def download_url(url, dest_path):
230     """
231     Download a file to a destination path given a URL
232     """
233     name = url.rsplit('/')[-1]
234     dest = dest_path + "/" + name
235     try:
236         response = urllib.urlopen(url)
237     except Exception:
238         return False
239
240     with open(dest, 'wb') as lfile:
241         shutil.copyfileobj(response, lfile)
242     return True
243
244
245 def download_and_add_image_on_glance(glance, image_name, image_url, data_dir):
246     try:
247         dest_path = data_dir
248         if not os.path.exists(dest_path):
249             os.makedirs(dest_path)
250         file_name = image_url.rsplit('/')[-1]
251         if not download_url(image_url, dest_path):
252             return False
253     except Exception:
254         raise Exception("Impossible to download image from {}".format(
255                         image_url))
256
257     try:
258         image = create_glance_image(
259             glance, image_name, dest_path + file_name)
260         if not image:
261             return False
262         else:
263             return image
264     except Exception:
265         raise Exception("Impossible to put image {} in glance".format(
266                         image_name))
267
268
269 # *********************************************
270 #   NOVA
271 # *********************************************
272 def get_instances(nova_client):
273     try:
274         instances = nova_client.servers.list(search_opts={'all_tenants': 1})
275         return instances
276     except Exception as e:
277         logger.error("Error [get_instances(nova_client)]: %s" % e)
278         return None
279
280
281 def get_instance_status(nova_client, instance):
282     try:
283         instance = nova_client.servers.get(instance.id)
284         return instance.status
285     except Exception as e:
286         logger.error("Error [get_instance_status(nova_client)]: %s" % e)
287         return None
288
289
290 def get_instance_by_name(nova_client, instance_name):
291     try:
292         instance = nova_client.servers.find(name=instance_name)
293         return instance
294     except Exception as e:
295         logger.error("Error [get_instance_by_name(nova_client, '%s')]: %s"
296                      % (instance_name, e))
297         return None
298
299
300 def get_flavor_id(nova_client, flavor_name):
301     flavors = nova_client.flavors.list(detailed=True)
302     id = ''
303     for f in flavors:
304         if f.name == flavor_name:
305             id = f.id
306             break
307     return id
308
309
310 def get_flavor_id_by_ram_range(nova_client, min_ram, max_ram):
311     flavors = nova_client.flavors.list(detailed=True)
312     id = ''
313     for f in flavors:
314         if min_ram <= f.ram and f.ram <= max_ram:
315             id = f.id
316             break
317     return id
318
319
320 def get_aggregates(nova_client):
321     try:
322         aggregates = nova_client.aggregates.list()
323         return aggregates
324     except Exception as e:
325         logger.error("Error [get_aggregates(nova_client)]: %s" % e)
326         return None
327
328
329 def get_aggregate_id(nova_client, aggregate_name):
330     try:
331         aggregates = get_aggregates(nova_client)
332         _id = [ag.id for ag in aggregates if ag.name == aggregate_name][0]
333         return _id
334     except Exception as e:
335         logger.error("Error [get_aggregate_id(nova_client, %s)]:"
336                      " %s" % (aggregate_name, e))
337         return None
338
339
340 def get_availability_zones(nova_client):
341     try:
342         availability_zones = nova_client.availability_zones.list()
343         return availability_zones
344     except Exception as e:
345         logger.error("Error [get_availability_zones(nova_client)]: %s" % e)
346         return None
347
348
349 def get_availability_zone_names(nova_client):
350     try:
351         az_names = [az.zoneName for az in get_availability_zones(nova_client)]
352         return az_names
353     except Exception as e:
354         logger.error("Error [get_availability_zone_names(nova_client)]:"
355                      " %s" % e)
356         return None
357
358
359 def create_flavor(nova_client, flavor_name, ram, disk, vcpus, public=True):
360     try:
361         flavor = nova_client.flavors.create(
362             flavor_name, ram, vcpus, disk, is_public=public)
363     except Exception as e:
364         logger.error("Error [create_flavor(nova_client, '%s', '%s', '%s', "
365                      "'%s')]: %s" % (flavor_name, ram, disk, vcpus, e))
366         return None
367     return flavor.id
368
369
370 def get_or_create_flavor(flavor_name, ram, disk, vcpus, public=True):
371     flavor_exists = False
372     nova_client = get_nova_client()
373
374     flavor_id = get_flavor_id(nova_client, flavor_name)
375     if flavor_id != '':
376         logger.info("Using existing flavor '%s'..." % flavor_name)
377         flavor_exists = True
378     else:
379         logger.info("Creating flavor '%s' with '%s' RAM, '%s' disk size, "
380                     "'%s' vcpus..." % (flavor_name, ram, disk, vcpus))
381         flavor_id = create_flavor(
382             nova_client, flavor_name, ram, disk, vcpus, public=public)
383         if not flavor_id:
384             raise Exception("Failed to create flavor '%s'..." % (flavor_name))
385         else:
386             logger.debug("Flavor '%s' with ID=%s created successfully."
387                          % (flavor_name, flavor_id))
388
389     return flavor_exists, flavor_id
390
391
392 def get_floating_ips(neutron_client):
393     try:
394         floating_ips = neutron_client.list_floatingips()
395         return floating_ips['floatingips']
396     except Exception as e:
397         logger.error("Error [get_floating_ips(neutron_client)]: %s" % e)
398         return None
399
400
401 def get_hypervisors(nova_client):
402     try:
403         nodes = []
404         hypervisors = nova_client.hypervisors.list()
405         for hypervisor in hypervisors:
406             if hypervisor.state == "up":
407                 nodes.append(hypervisor.hypervisor_hostname)
408         return nodes
409     except Exception as e:
410         logger.error("Error [get_hypervisors(nova_client)]: %s" % e)
411         return None
412
413
414 def create_aggregate(nova_client, aggregate_name, av_zone):
415     try:
416         nova_client.aggregates.create(aggregate_name, av_zone)
417         return True
418     except Exception as e:
419         logger.error("Error [create_aggregate(nova_client, %s, %s)]: %s"
420                      % (aggregate_name, av_zone, e))
421         return None
422
423
424 def add_host_to_aggregate(nova_client, aggregate_name, compute_host):
425     try:
426         aggregate_id = get_aggregate_id(nova_client, aggregate_name)
427         nova_client.aggregates.add_host(aggregate_id, compute_host)
428         return True
429     except Exception as e:
430         logger.error("Error [add_host_to_aggregate(nova_client, %s, %s)]: %s"
431                      % (aggregate_name, compute_host, e))
432         return None
433
434
435 def create_aggregate_with_host(
436         nova_client, aggregate_name, av_zone, compute_host):
437     try:
438         create_aggregate(nova_client, aggregate_name, av_zone)
439         add_host_to_aggregate(nova_client, aggregate_name, compute_host)
440         return True
441     except Exception as e:
442         logger.error("Error [create_aggregate_with_host("
443                      "nova_client, %s, %s, %s)]: %s"
444                      % (aggregate_name, av_zone, compute_host, e))
445         return None
446
447
448 def create_instance(flavor_name,
449                     image_id,
450                     network_id,
451                     instance_name="functest-vm",
452                     confdrive=True,
453                     userdata=None,
454                     av_zone='',
455                     fixed_ip=None,
456                     files=None):
457     nova_client = get_nova_client()
458     try:
459         flavor = nova_client.flavors.find(name=flavor_name)
460     except:
461         flavors = nova_client.flavors.list()
462         logger.error("Error: Flavor '%s' not found. Available flavors are: "
463                      "\n%s" % (flavor_name, flavors))
464         return None
465     if fixed_ip is not None:
466         nics = {"net-id": network_id, "v4-fixed-ip": fixed_ip}
467     else:
468         nics = {"net-id": network_id}
469     if userdata is None:
470         instance = nova_client.servers.create(
471             name=instance_name,
472             flavor=flavor,
473             image=image_id,
474             nics=[nics],
475             availability_zone=av_zone,
476             files=files
477         )
478     else:
479         instance = nova_client.servers.create(
480             name=instance_name,
481             flavor=flavor,
482             image=image_id,
483             nics=[nics],
484             config_drive=confdrive,
485             userdata=userdata,
486             availability_zone=av_zone,
487             files=files
488         )
489     return instance
490
491
492 def create_instance_and_wait_for_active(flavor_name,
493                                         image_id,
494                                         network_id,
495                                         instance_name="",
496                                         config_drive=False,
497                                         userdata="",
498                                         av_zone='',
499                                         fixed_ip=None,
500                                         files=None):
501     SLEEP = 3
502     VM_BOOT_TIMEOUT = 180
503     nova_client = get_nova_client()
504     instance = create_instance(flavor_name,
505                                image_id,
506                                network_id,
507                                instance_name,
508                                config_drive,
509                                userdata,
510                                av_zone=av_zone,
511                                fixed_ip=fixed_ip,
512                                files=files)
513     count = VM_BOOT_TIMEOUT / SLEEP
514     for n in range(count, -1, -1):
515         status = get_instance_status(nova_client, instance)
516         if status is None:
517             time.sleep(SLEEP)
518             continue
519         elif status.lower() == "active":
520             return instance
521         elif status.lower() == "error":
522             logger.error("The instance %s went to ERROR status."
523                          % instance_name)
524             return None
525         time.sleep(SLEEP)
526     logger.error("Timeout booting the instance %s." % instance_name)
527     return None
528
529
530 def create_floating_ip(neutron_client):
531     extnet_id = get_external_net_id(neutron_client)
532     props = {'floating_network_id': extnet_id}
533     try:
534         ip_json = neutron_client.create_floatingip({'floatingip': props})
535         fip_addr = ip_json['floatingip']['floating_ip_address']
536         fip_id = ip_json['floatingip']['id']
537     except Exception as e:
538         logger.error("Error [create_floating_ip(neutron_client)]: %s" % e)
539         return None
540     return {'fip_addr': fip_addr, 'fip_id': fip_id}
541
542
543 def add_floating_ip(nova_client, server_id, floatingip_addr):
544     try:
545         nova_client.servers.add_floating_ip(server_id, floatingip_addr)
546         return True
547     except Exception as e:
548         logger.error("Error [add_floating_ip(nova_client, '%s', '%s')]: %s"
549                      % (server_id, floatingip_addr, e))
550         return False
551
552
553 def delete_instance(nova_client, instance_id):
554     try:
555         nova_client.servers.force_delete(instance_id)
556         return True
557     except Exception as e:
558         logger.error("Error [delete_instance(nova_client, '%s')]: %s"
559                      % (instance_id, e))
560         return False
561
562
563 def delete_floating_ip(neutron_client, floatingip_id):
564     try:
565         neutron_client.delete_floatingip(floatingip_id)
566         return True
567     except Exception as e:
568         logger.error("Error [delete_floating_ip(neutron_client, '%s')]: %s"
569                      % (floatingip_id, e))
570         return False
571
572
573 def remove_host_from_aggregate(nova_client, aggregate_name, compute_host):
574     try:
575         aggregate_id = get_aggregate_id(nova_client, aggregate_name)
576         nova_client.aggregates.remove_host(aggregate_id, compute_host)
577         return True
578     except Exception as e:
579         logger.error("Error [remove_host_from_aggregate(nova_client, %s, %s)]:"
580                      " %s" % (aggregate_name, compute_host, e))
581         return False
582
583
584 def remove_hosts_from_aggregate(nova_client, aggregate_name):
585     aggregate_id = get_aggregate_id(nova_client, aggregate_name)
586     hosts = nova_client.aggregates.get(aggregate_id).hosts
587     assert(
588         all(remove_host_from_aggregate(nova_client, aggregate_name, host)
589             for host in hosts))
590
591
592 def delete_aggregate(nova_client, aggregate_name):
593     try:
594         remove_hosts_from_aggregate(nova_client, aggregate_name)
595         nova_client.aggregates.delete(aggregate_name)
596         return True
597     except Exception as e:
598         logger.error("Error [delete_aggregate(nova_client, %s)]: %s"
599                      % (aggregate_name, e))
600         return False
601
602
603 # *********************************************
604 #   NEUTRON
605 # *********************************************
606 def get_network_list(neutron_client):
607     network_list = neutron_client.list_networks()['networks']
608     if len(network_list) == 0:
609         return None
610     else:
611         return network_list
612
613
614 def get_router_list(neutron_client):
615     router_list = neutron_client.list_routers()['routers']
616     if len(router_list) == 0:
617         return None
618     else:
619         return router_list
620
621
622 def get_port_list(neutron_client):
623     port_list = neutron_client.list_ports()['ports']
624     if len(port_list) == 0:
625         return None
626     else:
627         return port_list
628
629
630 def get_network_id(neutron_client, network_name):
631     networks = neutron_client.list_networks()['networks']
632     id = ''
633     for n in networks:
634         if n['name'] == network_name:
635             id = n['id']
636             break
637     return id
638
639
640 def get_subnet_id(neutron_client, subnet_name):
641     subnets = neutron_client.list_subnets()['subnets']
642     id = ''
643     for s in subnets:
644         if s['name'] == subnet_name:
645             id = s['id']
646             break
647     return id
648
649
650 def get_router_id(neutron_client, router_name):
651     routers = neutron_client.list_routers()['routers']
652     id = ''
653     for r in routers:
654         if r['name'] == router_name:
655             id = r['id']
656             break
657     return id
658
659
660 def get_private_net(neutron_client):
661     # Checks if there is an existing shared private network
662     networks = neutron_client.list_networks()['networks']
663     if len(networks) == 0:
664         return None
665     for net in networks:
666         if (net['router:external'] is False) and (net['shared'] is True):
667             return net
668     return None
669
670
671 def get_external_net(neutron_client):
672     if (env.get('EXTERNAL_NETWORK')):
673         return env.get('EXTERNAL_NETWORK')
674     for network in neutron_client.list_networks()['networks']:
675         if network['router:external']:
676             return network['name']
677     return None
678
679
680 def get_external_net_id(neutron_client):
681     if (env.get('EXTERNAL_NETWORK')):
682         networks = neutron_client.list_networks(
683             name=env.get('EXTERNAL_NETWORK'))
684         net_id = networks['networks'][0]['id']
685         return net_id
686     for network in neutron_client.list_networks()['networks']:
687         if network['router:external']:
688             return network['id']
689     return None
690
691
692 def check_neutron_net(neutron_client, net_name):
693     for network in neutron_client.list_networks()['networks']:
694         if network['name'] == net_name:
695             for subnet in network['subnets']:
696                 return True
697     return False
698
699
700 def create_neutron_net(neutron_client, name):
701     json_body = {'network': {'name': name,
702                              'admin_state_up': True}}
703     try:
704         network = neutron_client.create_network(body=json_body)
705         network_dict = network['network']
706         return network_dict['id']
707     except Exception as e:
708         logger.error("Error [create_neutron_net(neutron_client, '%s')]: %s"
709                      % (name, e))
710         return None
711
712
713 def create_neutron_subnet(neutron_client, name, cidr, net_id,
714                           dns=['8.8.8.8', '8.8.4.4']):
715     json_body = {'subnets': [{'name': name, 'cidr': cidr,
716                               'ip_version': 4, 'network_id': net_id,
717                               'dns_nameservers': dns}]}
718
719     try:
720         subnet = neutron_client.create_subnet(body=json_body)
721         return subnet['subnets'][0]['id']
722     except Exception as e:
723         logger.error("Error [create_neutron_subnet(neutron_client, '%s', "
724                      "'%s', '%s')]: %s" % (name, cidr, net_id, e))
725         return None
726
727
728 def create_neutron_router(neutron_client, name):
729     json_body = {'router': {'name': name, 'admin_state_up': True}}
730     try:
731         router = neutron_client.create_router(json_body)
732         return router['router']['id']
733     except Exception as e:
734         logger.error("Error [create_neutron_router(neutron_client, '%s')]: %s"
735                      % (name, e))
736         return None
737
738
739 def create_neutron_port(neutron_client, name, network_id, ip):
740     json_body = {'port': {
741                  'admin_state_up': True,
742                  'name': name,
743                  'network_id': network_id,
744                  'fixed_ips': [{"ip_address": ip}]
745                  }}
746     try:
747         port = neutron_client.create_port(body=json_body)
748         return port['port']['id']
749     except Exception as e:
750         logger.error("Error [create_neutron_port(neutron_client, '%s', '%s', "
751                      "'%s')]: %s" % (name, network_id, ip, e))
752         return None
753
754
755 def update_neutron_net(neutron_client, network_id, shared=False):
756     json_body = {'network': {'shared': shared}}
757     try:
758         neutron_client.update_network(network_id, body=json_body)
759         return True
760     except Exception as e:
761         logger.error("Error [update_neutron_net(neutron_client, '%s', '%s')]: "
762                      "%s" % (network_id, str(shared), e))
763         return False
764
765
766 def update_neutron_port(neutron_client, port_id, device_owner):
767     json_body = {'port': {
768                  'device_owner': device_owner,
769                  }}
770     try:
771         port = neutron_client.update_port(port=port_id,
772                                           body=json_body)
773         return port['port']['id']
774     except Exception as e:
775         logger.error("Error [update_neutron_port(neutron_client, '%s', '%s')]:"
776                      " %s" % (port_id, device_owner, e))
777         return None
778
779
780 def add_interface_router(neutron_client, router_id, subnet_id):
781     json_body = {"subnet_id": subnet_id}
782     try:
783         neutron_client.add_interface_router(router=router_id, body=json_body)
784         return True
785     except Exception as e:
786         logger.error("Error [add_interface_router(neutron_client, '%s', "
787                      "'%s')]: %s" % (router_id, subnet_id, e))
788         return False
789
790
791 def add_gateway_router(neutron_client, router_id):
792     ext_net_id = get_external_net_id(neutron_client)
793     router_dict = {'network_id': ext_net_id}
794     try:
795         neutron_client.add_gateway_router(router_id, router_dict)
796         return True
797     except Exception as e:
798         logger.error("Error [add_gateway_router(neutron_client, '%s')]: %s"
799                      % (router_id, e))
800         return False
801
802
803 def delete_neutron_net(neutron_client, network_id):
804     try:
805         neutron_client.delete_network(network_id)
806         return True
807     except Exception as e:
808         logger.error("Error [delete_neutron_net(neutron_client, '%s')]: %s"
809                      % (network_id, e))
810         return False
811
812
813 def delete_neutron_subnet(neutron_client, subnet_id):
814     try:
815         neutron_client.delete_subnet(subnet_id)
816         return True
817     except Exception as e:
818         logger.error("Error [delete_neutron_subnet(neutron_client, '%s')]: %s"
819                      % (subnet_id, e))
820         return False
821
822
823 def delete_neutron_router(neutron_client, router_id):
824     try:
825         neutron_client.delete_router(router=router_id)
826         return True
827     except Exception as e:
828         logger.error("Error [delete_neutron_router(neutron_client, '%s')]: %s"
829                      % (router_id, e))
830         return False
831
832
833 def delete_neutron_port(neutron_client, port_id):
834     try:
835         neutron_client.delete_port(port_id)
836         return True
837     except Exception as e:
838         logger.error("Error [delete_neutron_port(neutron_client, '%s')]: %s"
839                      % (port_id, e))
840         return False
841
842
843 def remove_interface_router(neutron_client, router_id, subnet_id):
844     json_body = {"subnet_id": subnet_id}
845     try:
846         neutron_client.remove_interface_router(router=router_id,
847                                                body=json_body)
848         return True
849     except Exception as e:
850         logger.error("Error [remove_interface_router(neutron_client, '%s', "
851                      "'%s')]: %s" % (router_id, subnet_id, e))
852         return False
853
854
855 def remove_gateway_router(neutron_client, router_id):
856     try:
857         neutron_client.remove_gateway_router(router_id)
858         return True
859     except Exception as e:
860         logger.error("Error [remove_gateway_router(neutron_client, '%s')]: %s"
861                      % (router_id, e))
862         return False
863
864
865 def create_network_full(neutron_client,
866                         net_name,
867                         subnet_name,
868                         router_name,
869                         cidr,
870                         dns=['8.8.8.8', '8.8.4.4']):
871
872     # Check if the network already exists
873     network_id = get_network_id(neutron_client, net_name)
874     subnet_id = get_subnet_id(neutron_client, subnet_name)
875     router_id = get_router_id(neutron_client, router_name)
876
877     if network_id != '' and subnet_id != '' and router_id != '':
878         logger.info("A network with name '%s' already exists..." % net_name)
879     else:
880         neutron_client.format = 'json'
881         logger.info('Creating neutron network %s...' % net_name)
882         network_id = create_neutron_net(neutron_client, net_name)
883
884         if not network_id:
885             return False
886
887         logger.debug("Network '%s' created successfully" % network_id)
888         logger.debug('Creating Subnet....')
889         subnet_id = create_neutron_subnet(neutron_client, subnet_name,
890                                           cidr, network_id, dns)
891         if not subnet_id:
892             return None
893
894         logger.debug("Subnet '%s' created successfully" % subnet_id)
895         logger.debug('Creating Router...')
896         router_id = create_neutron_router(neutron_client, router_name)
897
898         if not router_id:
899             return None
900
901         logger.debug("Router '%s' created successfully" % router_id)
902         logger.debug('Adding router to subnet...')
903
904         if not add_interface_router(neutron_client, router_id, subnet_id):
905             return None
906
907         logger.debug("Interface added successfully.")
908
909         logger.debug('Adding gateway to router...')
910         if not add_gateway_router(neutron_client, router_id):
911             return None
912
913         logger.debug("Gateway added successfully.")
914
915     network_dic = {'net_id': network_id,
916                    'subnet_id': subnet_id,
917                    'router_id': router_id}
918     return network_dic
919
920
921 def create_shared_network_full(net_name, subnt_name, router_name, subnet_cidr):
922     neutron_client = get_neutron_client()
923
924     network_dic = create_network_full(neutron_client,
925                                       net_name,
926                                       subnt_name,
927                                       router_name,
928                                       subnet_cidr)
929     if network_dic:
930         if not update_neutron_net(neutron_client,
931                                   network_dic['net_id'],
932                                   shared=True):
933             logger.error("Failed to update network %s..." % net_name)
934             return None
935         else:
936             logger.debug("Network '%s' is available..." % net_name)
937     else:
938         logger.error("Network %s creation failed" % net_name)
939         return None
940     return network_dic
941
942
943 # *********************************************
944 #   SEC GROUPS
945 # *********************************************
946
947
948 def get_security_groups(neutron_client):
949     try:
950         security_groups = neutron_client.list_security_groups()[
951             'security_groups']
952         return security_groups
953     except Exception as e:
954         logger.error("Error [get_security_groups(neutron_client)]: %s" % e)
955         return None
956
957
958 def get_security_group_id(neutron_client, sg_name):
959     security_groups = get_security_groups(neutron_client)
960     id = ''
961     for sg in security_groups:
962         if sg['name'] == sg_name:
963             id = sg['id']
964             break
965     return id
966
967
968 def create_security_group(neutron_client, sg_name, sg_description):
969     json_body = {'security_group': {'name': sg_name,
970                                     'description': sg_description}}
971     try:
972         secgroup = neutron_client.create_security_group(json_body)
973         return secgroup['security_group']
974     except Exception as e:
975         logger.error("Error [create_security_group(neutron_client, '%s', "
976                      "'%s')]: %s" % (sg_name, sg_description, e))
977         return None
978
979
980 def create_secgroup_rule(neutron_client, sg_id, direction, protocol,
981                          port_range_min=None, port_range_max=None):
982     # We create a security group in 2 steps
983     # 1 - we check the format and set the json body accordingly
984     # 2 - we call neturon client to create the security group
985
986     # Format check
987     json_body = {'security_group_rule': {'direction': direction,
988                                          'security_group_id': sg_id,
989                                          'protocol': protocol}}
990     # parameters may be
991     # - both None => we do nothing
992     # - both Not None => we add them to the json description
993     # but one cannot be None is the other is not None
994     if (port_range_min is not None and port_range_max is not None):
995         # add port_range in json description
996         json_body['security_group_rule']['port_range_min'] = port_range_min
997         json_body['security_group_rule']['port_range_max'] = port_range_max
998         logger.debug("Security_group format set (port range included)")
999     else:
1000         # either both port range are set to None => do nothing
1001         # or one is set but not the other => log it and return False
1002         if port_range_min is None and port_range_max is None:
1003             logger.debug("Security_group format set (no port range mentioned)")
1004         else:
1005             logger.error("Bad security group format."
1006                          "One of the port range is not properly set:"
1007                          "range min: {},"
1008                          "range max: {}".format(port_range_min,
1009                                                 port_range_max))
1010             return False
1011
1012     # Create security group using neutron client
1013     try:
1014         neutron_client.create_security_group_rule(json_body)
1015         return True
1016     except:
1017         logger.exception("Impossible to create_security_group_rule,"
1018                          "security group rule probably already exists")
1019         return False
1020
1021
1022 def get_security_group_rules(neutron_client, sg_id):
1023     try:
1024         security_rules = neutron_client.list_security_group_rules()[
1025             'security_group_rules']
1026         security_rules = [rule for rule in security_rules
1027                           if rule["security_group_id"] == sg_id]
1028         return security_rules
1029     except Exception as e:
1030         logger.error("Error [get_security_group_rules(neutron_client, sg_id)]:"
1031                      " %s" % e)
1032         return None
1033
1034
1035 def check_security_group_rules(neutron_client, sg_id, direction, protocol,
1036                                port_min=None, port_max=None):
1037     try:
1038         security_rules = get_security_group_rules(neutron_client, sg_id)
1039         security_rules = [rule for rule in security_rules
1040                           if (rule["direction"].lower() == direction and
1041                               rule["protocol"].lower() == protocol and
1042                               rule["port_range_min"] == port_min and
1043                               rule["port_range_max"] == port_max)]
1044         if len(security_rules) == 0:
1045             return True
1046         else:
1047             return False
1048     except Exception as e:
1049         logger.error("Error [check_security_group_rules("
1050                      " neutron_client, sg_id, direction,"
1051                      " protocol, port_min=None, port_max=None)]: "
1052                      "%s" % e)
1053         return None
1054
1055
1056 def create_security_group_full(neutron_client,
1057                                sg_name, sg_description):
1058     sg_id = get_security_group_id(neutron_client, sg_name)
1059     if sg_id != '':
1060         logger.info("Using existing security group '%s'..." % sg_name)
1061     else:
1062         logger.info("Creating security group  '%s'..." % sg_name)
1063         SECGROUP = create_security_group(neutron_client,
1064                                          sg_name,
1065                                          sg_description)
1066         if not SECGROUP:
1067             logger.error("Failed to create the security group...")
1068             return None
1069
1070         sg_id = SECGROUP['id']
1071
1072         logger.debug("Security group '%s' with ID=%s created successfully."
1073                      % (SECGROUP['name'], sg_id))
1074
1075         logger.debug("Adding ICMP rules in security group '%s'..."
1076                      % sg_name)
1077         if not create_secgroup_rule(neutron_client, sg_id,
1078                                     'ingress', 'icmp'):
1079             logger.error("Failed to create the security group rule...")
1080             return None
1081
1082         logger.debug("Adding SSH rules in security group '%s'..."
1083                      % sg_name)
1084         if not create_secgroup_rule(
1085                 neutron_client, sg_id, 'ingress', 'tcp', '22', '22'):
1086             logger.error("Failed to create the security group rule...")
1087             return None
1088
1089         if not create_secgroup_rule(
1090                 neutron_client, sg_id, 'egress', 'tcp', '22', '22'):
1091             logger.error("Failed to create the security group rule...")
1092             return None
1093     return sg_id
1094
1095
1096 def add_secgroup_to_instance(nova_client, instance_id, secgroup_id):
1097     try:
1098         nova_client.servers.add_security_group(instance_id, secgroup_id)
1099         return True
1100     except Exception as e:
1101         logger.error("Error [add_secgroup_to_instance(nova_client, '%s', "
1102                      "'%s')]: %s" % (instance_id, secgroup_id, e))
1103         return False
1104
1105
1106 def update_sg_quota(neutron_client, tenant_id, sg_quota, sg_rule_quota):
1107     json_body = {"quota": {
1108         "security_group": sg_quota,
1109         "security_group_rule": sg_rule_quota
1110     }}
1111
1112     try:
1113         neutron_client.update_quota(tenant_id=tenant_id,
1114                                     body=json_body)
1115         return True
1116     except Exception as e:
1117         logger.error("Error [update_sg_quota(neutron_client, '%s', '%s', "
1118                      "'%s')]: %s" % (tenant_id, sg_quota, sg_rule_quota, e))
1119         return False
1120
1121
1122 def delete_security_group(neutron_client, secgroup_id):
1123     try:
1124         neutron_client.delete_security_group(secgroup_id)
1125         return True
1126     except Exception as e:
1127         logger.error("Error [delete_security_group(neutron_client, '%s')]: %s"
1128                      % (secgroup_id, e))
1129         return False
1130
1131
1132 # *********************************************
1133 #   GLANCE
1134 # *********************************************
1135 def get_images(glance_client):
1136     try:
1137         images = glance_client.images.list()
1138         return images
1139     except Exception as e:
1140         logger.error("Error [get_images]: %s" % e)
1141         return None
1142
1143
1144 def get_image_id(glance_client, image_name):
1145     images = glance_client.images.list()
1146     id = ''
1147     for i in images:
1148         if i.name == image_name:
1149             id = i.id
1150             break
1151     return id
1152
1153
1154 def create_glance_image(glance_client,
1155                         image_name,
1156                         file_path,
1157                         disk="qcow2",
1158                         extra_properties={},
1159                         container="bare",
1160                         public="public"):
1161     if not os.path.isfile(file_path):
1162         logger.error("Error: file %s does not exist." % file_path)
1163         return None
1164     try:
1165         image_id = get_image_id(glance_client, image_name)
1166         if image_id != '':
1167             logger.info("Image %s already exists." % image_name)
1168         else:
1169             logger.info("Creating image '%s' from '%s'..." % (image_name,
1170                                                               file_path))
1171
1172             image = glance_client.images.create(name=image_name,
1173                                                 visibility=public,
1174                                                 disk_format=disk,
1175                                                 container_format=container,
1176                                                 **extra_properties)
1177             image_id = image.id
1178             with open(file_path) as image_data:
1179                 glance_client.images.upload(image_id, image_data)
1180         return image_id
1181     except Exception as e:
1182         logger.error("Error [create_glance_image(glance_client, '%s', '%s', "
1183                      "'%s')]: %s" % (image_name, file_path, public, e))
1184         return None
1185
1186
1187 def get_or_create_image(name, path, format, extra_properties):
1188     image_exists = False
1189     glance_client = get_glance_client()
1190
1191     image_id = get_image_id(glance_client, name)
1192     if image_id != '':
1193         logger.info("Using existing image '%s'..." % name)
1194         image_exists = True
1195     else:
1196         logger.info("Creating image '%s' from '%s'..." % (name, path))
1197         image_id = create_glance_image(glance_client,
1198                                        name,
1199                                        path,
1200                                        format,
1201                                        extra_properties)
1202         if not image_id:
1203             logger.error("Failed to create a Glance image...")
1204         else:
1205             logger.debug("Image '%s' with ID=%s created successfully."
1206                          % (name, image_id))
1207
1208     return image_exists, image_id
1209
1210
1211 def delete_glance_image(glance_client, image_id):
1212     try:
1213         glance_client.images.delete(image_id)
1214         return True
1215     except Exception as e:
1216         logger.error("Error [delete_glance_image(glance_client, '%s')]: %s"
1217                      % (image_id, e))
1218         return False
1219
1220
1221 # *********************************************
1222 #   CINDER
1223 # *********************************************
1224 def get_volumes(cinder_client):
1225     try:
1226         volumes = cinder_client.volumes.list(search_opts={'all_tenants': 1})
1227         return volumes
1228     except Exception as e:
1229         logger.error("Error [get_volumes(cinder_client)]: %s" % e)
1230         return None
1231
1232
1233 def update_cinder_quota(cinder_client, tenant_id, vols_quota,
1234                         snapshots_quota, gigabytes_quota):
1235     quotas_values = {"volumes": vols_quota,
1236                      "snapshots": snapshots_quota,
1237                      "gigabytes": gigabytes_quota}
1238
1239     try:
1240         cinder_client.quotas.update(tenant_id, **quotas_values)
1241         return True
1242     except Exception as e:
1243         logger.error("Error [update_cinder_quota(cinder_client, '%s', '%s', "
1244                      "'%s' '%s')]: %s" % (tenant_id, vols_quota,
1245                                           snapshots_quota, gigabytes_quota, e))
1246         return False
1247
1248
1249 def delete_volume(cinder_client, volume_id, forced=False):
1250     try:
1251         if forced:
1252             try:
1253                 cinder_client.volumes.detach(volume_id)
1254             except:
1255                 logger.error(sys.exc_info()[0])
1256             cinder_client.volumes.force_delete(volume_id)
1257         else:
1258             cinder_client.volumes.delete(volume_id)
1259         return True
1260     except Exception as e:
1261         logger.error("Error [delete_volume(cinder_client, '%s', '%s')]: %s"
1262                      % (volume_id, str(forced), e))
1263         return False
1264
1265
1266 # *********************************************
1267 #   KEYSTONE
1268 # *********************************************
1269 def get_tenants(keystone_client):
1270     try:
1271         if is_keystone_v3():
1272             tenants = keystone_client.projects.list()
1273         else:
1274             tenants = keystone_client.tenants.list()
1275         return tenants
1276     except Exception as e:
1277         logger.error("Error [get_tenants(keystone_client)]: %s" % e)
1278         return None
1279
1280
1281 def get_users(keystone_client):
1282     try:
1283         users = keystone_client.users.list()
1284         return users
1285     except Exception as e:
1286         logger.error("Error [get_users(keystone_client)]: %s" % e)
1287         return None
1288
1289
1290 def get_tenant_id(keystone_client, tenant_name):
1291     tenants = get_tenants(keystone_client)
1292     id = ''
1293     for t in tenants:
1294         if t.name == tenant_name:
1295             id = t.id
1296             break
1297     return id
1298
1299
1300 def get_user_id(keystone_client, user_name):
1301     users = get_users(keystone_client)
1302     id = ''
1303     for u in users:
1304         if u.name == user_name:
1305             id = u.id
1306             break
1307     return id
1308
1309
1310 def get_role_id(keystone_client, role_name):
1311     roles = keystone_client.roles.list()
1312     id = ''
1313     for r in roles:
1314         if r.name == role_name:
1315             id = r.id
1316             break
1317     return id
1318
1319
1320 def get_domain_id(keystone_client, domain_name):
1321     domains = keystone_client.domains.list()
1322     id = ''
1323     for d in domains:
1324         if d.name == domain_name:
1325             id = d.id
1326             break
1327     return id
1328
1329
1330 def create_tenant(keystone_client, tenant_name, tenant_description):
1331     try:
1332         if is_keystone_v3():
1333             domain_name = os.environ['OS_PROJECT_DOMAIN_NAME']
1334             domain_id = get_domain_id(keystone_client, domain_name)
1335             tenant = keystone_client.projects.create(
1336                 name=tenant_name,
1337                 description=tenant_description,
1338                 domain=domain_id,
1339                 enabled=True)
1340         else:
1341             tenant = keystone_client.tenants.create(tenant_name,
1342                                                     tenant_description,
1343                                                     enabled=True)
1344         return tenant.id
1345     except Exception as e:
1346         logger.error("Error [create_tenant(keystone_client, '%s', '%s')]: %s"
1347                      % (tenant_name, tenant_description, e))
1348         return None
1349
1350
1351 def get_or_create_tenant(keystone_client, tenant_name, tenant_description):
1352     tenant_id = get_tenant_id(keystone_client, tenant_name)
1353     if not tenant_id:
1354         tenant_id = create_tenant(keystone_client, tenant_name,
1355                                   tenant_description)
1356
1357     return tenant_id
1358
1359
1360 def get_or_create_tenant_for_vnf(keystone_client, tenant_name,
1361                                  tenant_description):
1362     """Get or Create a Tenant
1363
1364         Args:
1365             keystone_client: keystone client reference
1366             tenant_name: the name of the tenant
1367             tenant_description: the description of the tenant
1368
1369         return False if tenant retrieved though get
1370         return True if tenant created
1371         raise Exception if error during processing
1372     """
1373     try:
1374         tenant_id = get_tenant_id(keystone_client, tenant_name)
1375         if not tenant_id:
1376             tenant_id = create_tenant(keystone_client, tenant_name,
1377                                       tenant_description)
1378             return True
1379         else:
1380             return False
1381     except:
1382         raise Exception("Impossible to create a Tenant for the VNF {}".format(
1383                             tenant_name))
1384
1385
1386 def create_user(keystone_client, user_name, user_password,
1387                 user_email, tenant_id):
1388     try:
1389         if is_keystone_v3():
1390             user = keystone_client.users.create(name=user_name,
1391                                                 password=user_password,
1392                                                 email=user_email,
1393                                                 project_id=tenant_id,
1394                                                 enabled=True)
1395         else:
1396             user = keystone_client.users.create(user_name,
1397                                                 user_password,
1398                                                 user_email,
1399                                                 tenant_id,
1400                                                 enabled=True)
1401         return user.id
1402     except Exception as e:
1403         logger.error("Error [create_user(keystone_client, '%s', '%s', '%s'"
1404                      "'%s')]: %s" % (user_name, user_password,
1405                                      user_email, tenant_id, e))
1406         return None
1407
1408
1409 def get_or_create_user(keystone_client, user_name, user_password,
1410                        tenant_id, user_email=None):
1411     user_id = get_user_id(keystone_client, user_name)
1412     if not user_id:
1413         user_id = create_user(keystone_client, user_name, user_password,
1414                               user_email, tenant_id)
1415     return user_id
1416
1417
1418 def get_or_create_user_for_vnf(keystone_client, vnf_ref):
1419     """Get or Create user for VNF
1420
1421         Args:
1422             keystone_client: keystone client reference
1423             vnf_ref: VNF reference used as user name & password, tenant name
1424
1425         return False if user retrieved through get
1426         return True if user created
1427         raise Exception if error during processing
1428     """
1429     try:
1430         user_id = get_user_id(keystone_client, vnf_ref)
1431         tenant_id = get_tenant_id(keystone_client, vnf_ref)
1432         created = False
1433         if not user_id:
1434             user_id = create_user(keystone_client, vnf_ref, vnf_ref,
1435                                   "", tenant_id)
1436             created = True
1437         try:
1438             role_id = get_role_id(keystone_client, 'admin')
1439             tenant_id = get_tenant_id(keystone_client, vnf_ref)
1440             add_role_user(keystone_client, user_id, role_id, tenant_id)
1441         except:
1442             logger.warn("Cannot associate user to role admin on tenant")
1443         return created
1444     except:
1445         raise Exception("Impossible to create a user for the VNF {}".format(
1446             vnf_ref))
1447
1448
1449 def add_role_user(keystone_client, user_id, role_id, tenant_id):
1450     try:
1451         if is_keystone_v3():
1452             keystone_client.roles.grant(role=role_id,
1453                                         user=user_id,
1454                                         project=tenant_id)
1455         else:
1456             keystone_client.roles.add_user_role(user_id, role_id, tenant_id)
1457         return True
1458     except Exception as e:
1459         logger.error("Error [add_role_user(keystone_client, '%s', '%s'"
1460                      "'%s')]: %s " % (user_id, role_id, tenant_id, e))
1461         return False
1462
1463
1464 def delete_tenant(keystone_client, tenant_id):
1465     try:
1466         if is_keystone_v3():
1467             keystone_client.projects.delete(tenant_id)
1468         else:
1469             keystone_client.tenants.delete(tenant_id)
1470         return True
1471     except Exception as e:
1472         logger.error("Error [delete_tenant(keystone_client, '%s')]: %s"
1473                      % (tenant_id, e))
1474         return False
1475
1476
1477 def delete_user(keystone_client, user_id):
1478     try:
1479         keystone_client.users.delete(user_id)
1480         return True
1481     except Exception as e:
1482         logger.error("Error [delete_user(keystone_client, '%s')]: %s"
1483                      % (user_id, e))
1484         return False
1485
1486
1487 # *********************************************
1488 #   HEAT
1489 # *********************************************
1490 def get_resource(heat_client, stack_id, resource):
1491     try:
1492         resources = heat_client.resources.get(stack_id, resource)
1493         return resources
1494     except Exception as e:
1495         logger.error("Error [get_resource]: %s" % e)
1496         return None