Merge "SNAPS tests now support offline testing."
[functest.git] / functest / utils / openstack_utils.py
1 #!/usr/bin/env python
2 #
3 # jose.lausuch@ericsson.com
4 # valentin.boucher@orange.com
5 # All rights reserved. This program and the accompanying materials
6 # are made available under the terms of the Apache License, Version 2.0
7 # which accompanies this distribution, and is available at
8 # http://www.apache.org/licenses/LICENSE-2.0
9 #
10
11 import logging
12 import os.path
13 import re
14 import sys
15 import time
16
17 from keystoneauth1 import loading
18 from keystoneauth1 import session
19 from cinderclient import client as cinderclient
20 from glanceclient import client as glanceclient
21 from heatclient import client as heatclient
22 from novaclient import client as novaclient
23 from keystoneclient import client as keystoneclient
24 from neutronclient.neutron import client as neutronclient
25
26 import functest.utils.functest_utils as ft_utils
27
28 logger = logging.getLogger(__name__)
29
30 DEFAULT_API_VERSION = '2'
31 DEFAULT_HEAT_API_VERSION = '1'
32
33
34 # *********************************************
35 #   CREDENTIALS
36 # *********************************************
37 class MissingEnvVar(Exception):
38
39     def __init__(self, var):
40         self.var = var
41
42     def __str__(self):
43         return str.format("Please set the mandatory env var: {}", self.var)
44
45
46 def is_keystone_v3():
47     keystone_api_version = os.getenv('OS_IDENTITY_API_VERSION')
48     if (keystone_api_version is None or
49             keystone_api_version == '2'):
50         return False
51     else:
52         return True
53
54
55 def get_rc_env_vars():
56     env_vars = ['OS_AUTH_URL', 'OS_USERNAME', 'OS_PASSWORD']
57     if is_keystone_v3():
58         env_vars.extend(['OS_PROJECT_NAME',
59                          'OS_USER_DOMAIN_NAME',
60                          'OS_PROJECT_DOMAIN_NAME'])
61     else:
62         env_vars.extend(['OS_TENANT_NAME'])
63     return env_vars
64
65
66 def check_credentials():
67     """
68     Check if the OpenStack credentials (openrc) are sourced
69     """
70     env_vars = get_rc_env_vars()
71     return all(map(lambda v: v in os.environ and os.environ[v], env_vars))
72
73
74 def get_env_cred_dict():
75     env_cred_dict = {
76         'OS_USERNAME': 'username',
77         'OS_PASSWORD': 'password',
78         'OS_AUTH_URL': 'auth_url',
79         'OS_TENANT_NAME': 'tenant_name',
80         'OS_USER_DOMAIN_NAME': 'user_domain_name',
81         'OS_PROJECT_DOMAIN_NAME': 'project_domain_name',
82         'OS_PROJECT_NAME': 'project_name',
83         'OS_ENDPOINT_TYPE': 'endpoint_type',
84         'OS_REGION_NAME': 'region_name',
85         'OS_CACERT': 'https_cacert',
86         'OS_INSECURE': 'https_insecure'
87     }
88     return env_cred_dict
89
90
91 def get_credentials(other_creds={}):
92     """Returns a creds dictionary filled with parsed from env
93     """
94     creds = {}
95     env_vars = get_rc_env_vars()
96     env_cred_dict = get_env_cred_dict()
97
98     for envvar in env_vars:
99         if os.getenv(envvar) is None:
100             raise MissingEnvVar(envvar)
101         else:
102             creds_key = env_cred_dict.get(envvar)
103             creds.update({creds_key: os.getenv(envvar)})
104
105     if 'tenant' in other_creds.keys():
106         if is_keystone_v3():
107             tenant = 'project_name'
108         else:
109             tenant = 'tenant_name'
110         other_creds[tenant] = other_creds.pop('tenant')
111
112     creds.update(other_creds)
113
114     return creds
115
116
117 def source_credentials(rc_file):
118     with open(rc_file, "r") as f:
119         for line in f:
120             var = line.rstrip('"\n').replace('export ', '').split("=")
121             # The two next lines should be modified as soon as rc_file
122             # conforms with common rules. Be aware that it could induce
123             # issues if value starts with '
124             key = re.sub(r'^["\' ]*|[ \'"]*$', '', var[0])
125             value = re.sub(r'^["\' ]*|[ \'"]*$', '', "".join(var[1:]))
126             os.environ[key] = value
127
128
129 def get_credentials_for_rally():
130     creds = get_credentials()
131     env_cred_dict = get_env_cred_dict()
132     rally_conf = {"type": "ExistingCloud", "admin": {}}
133     for key in creds:
134         if key == 'auth_url':
135             rally_conf[key] = creds[key]
136         else:
137             rally_conf['admin'][key] = creds[key]
138
139     endpoint_types = [('internalURL', 'internal'),
140                       ('publicURL', 'public'), ('adminURL', 'admin')]
141
142     endpoint_type = get_endpoint_type_from_env()
143     if endpoint_type is not None:
144         cred_key = env_cred_dict.get('OS_ENDPOINT_TYPE')
145         for k, v in endpoint_types:
146             if endpoint_type == v:
147                 rally_conf[cred_key] = v
148
149     region_name = os.getenv('OS_REGION_NAME')
150     if region_name is not None:
151         cred_key = env_cred_dict.get('OS_REGION_NAME')
152         rally_conf[cred_key] = region_name
153
154     cred_key = env_cred_dict.get('OS_CACERT')
155     rally_conf[cred_key] = os.getenv('OS_CACERT', '')
156
157     insecure_key = env_cred_dict.get('OS_INSECURE')
158     rally_conf[insecure_key] = os.getenv('OS_INSECURE', '').lower() == 'true'
159
160     return rally_conf
161
162
163 def get_endpoint_type_from_env():
164     endpoint_type = os.environ.get("OS_ENDPOINT_TYPE",
165                                    os.environ.get("OS_INTERFACE"))
166     if endpoint_type and "URL" in endpoint_type:
167         endpoint_type = endpoint_type.replace("URL", "")
168     return endpoint_type
169
170
171 def get_session_auth(other_creds={}):
172     loader = loading.get_plugin_loader('password')
173     creds = get_credentials(other_creds)
174     auth = loader.load_from_options(**creds)
175     return auth
176
177
178 def get_endpoint(service_type, endpoint_type='publicURL'):
179     auth = get_session_auth()
180     return get_session().get_endpoint(auth=auth,
181                                       service_type=service_type,
182                                       endpoint_type=endpoint_type)
183
184
185 def get_session(other_creds={}):
186     auth = get_session_auth(other_creds)
187     https_cacert = os.getenv('OS_CACERT', '')
188     https_insecure = os.getenv('OS_INSECURE', '').lower() == 'true'
189     return session.Session(auth=auth,
190                            verify=(https_cacert or not https_insecure))
191
192
193 # *********************************************
194 #   CLIENTS
195 # *********************************************
196 def get_keystone_client_version():
197     api_version = os.getenv('OS_IDENTITY_API_VERSION')
198     if api_version is not None:
199         logger.info("OS_IDENTITY_API_VERSION is set in env as '%s'",
200                     api_version)
201         return api_version
202     return DEFAULT_API_VERSION
203
204
205 def get_keystone_client(other_creds={}):
206     sess = get_session(other_creds)
207     return keystoneclient.Client(get_keystone_client_version(),
208                                  session=sess,
209                                  interface=os.getenv('OS_INTERFACE', 'admin'))
210
211
212 def get_nova_client_version():
213     api_version = os.getenv('OS_COMPUTE_API_VERSION')
214     if api_version is not None:
215         logger.info("OS_COMPUTE_API_VERSION is set in env as '%s'",
216                     api_version)
217         return api_version
218     return DEFAULT_API_VERSION
219
220
221 def get_nova_client(other_creds={}):
222     sess = get_session(other_creds)
223     return novaclient.Client(get_nova_client_version(), session=sess)
224
225
226 def get_cinder_client_version():
227     api_version = os.getenv('OS_VOLUME_API_VERSION')
228     if api_version is not None:
229         logger.info("OS_VOLUME_API_VERSION is set in env as '%s'",
230                     api_version)
231         return api_version
232     return DEFAULT_API_VERSION
233
234
235 def get_cinder_client(other_creds={}):
236     sess = get_session(other_creds)
237     return cinderclient.Client(get_cinder_client_version(), session=sess)
238
239
240 def get_neutron_client_version():
241     api_version = os.getenv('OS_NETWORK_API_VERSION')
242     if api_version is not None:
243         logger.info("OS_NETWORK_API_VERSION is set in env as '%s'",
244                     api_version)
245         return api_version
246     return DEFAULT_API_VERSION
247
248
249 def get_neutron_client(other_creds={}):
250     sess = get_session(other_creds)
251     return neutronclient.Client(get_neutron_client_version(), session=sess)
252
253
254 def get_glance_client_version():
255     api_version = os.getenv('OS_IMAGE_API_VERSION')
256     if api_version is not None:
257         logger.info("OS_IMAGE_API_VERSION is set in env as '%s'", api_version)
258         return api_version
259     return DEFAULT_API_VERSION
260
261
262 def get_glance_client(other_creds={}):
263     sess = get_session(other_creds)
264     return glanceclient.Client(get_glance_client_version(), session=sess)
265
266
267 def get_heat_client_version():
268     api_version = os.getenv('OS_ORCHESTRATION_API_VERSION')
269     if api_version is not None:
270         logger.info("OS_ORCHESTRATION_API_VERSION is set in env as '%s'",
271                     api_version)
272         return api_version
273     return DEFAULT_HEAT_API_VERSION
274
275
276 def get_heat_client(other_creds={}):
277     sess = get_session(other_creds)
278     return heatclient.Client(get_heat_client_version(), session=sess)
279
280
281 def download_and_add_image_on_glance(glance, image_name, image_url, data_dir):
282     dest_path = data_dir
283     if not os.path.exists(dest_path):
284         os.makedirs(dest_path)
285     file_name = image_url.rsplit('/')[-1]
286     if not ft_utils.download_url(image_url, dest_path):
287         return False
288
289     image = create_glance_image(
290         glance, image_name, dest_path + file_name)
291     if not image:
292         return False
293
294     return image
295
296
297 # *********************************************
298 #   NOVA
299 # *********************************************
300 def get_instances(nova_client):
301     try:
302         instances = nova_client.servers.list(search_opts={'all_tenants': 1})
303         return instances
304     except Exception as e:
305         logger.error("Error [get_instances(nova_client)]: %s" % e)
306         return None
307
308
309 def get_instance_status(nova_client, instance):
310     try:
311         instance = nova_client.servers.get(instance.id)
312         return instance.status
313     except Exception as e:
314         logger.error("Error [get_instance_status(nova_client)]: %s" % e)
315         return None
316
317
318 def get_instance_by_name(nova_client, instance_name):
319     try:
320         instance = nova_client.servers.find(name=instance_name)
321         return instance
322     except Exception as e:
323         logger.error("Error [get_instance_by_name(nova_client, '%s')]: %s"
324                      % (instance_name, e))
325         return None
326
327
328 def get_flavor_id(nova_client, flavor_name):
329     flavors = nova_client.flavors.list(detailed=True)
330     id = ''
331     for f in flavors:
332         if f.name == flavor_name:
333             id = f.id
334             break
335     return id
336
337
338 def get_flavor_id_by_ram_range(nova_client, min_ram, max_ram):
339     flavors = nova_client.flavors.list(detailed=True)
340     id = ''
341     for f in flavors:
342         if min_ram <= f.ram and f.ram <= max_ram:
343             id = f.id
344             break
345     return id
346
347
348 def get_aggregates(nova_client):
349     try:
350         aggregates = nova_client.aggregates.list()
351         return aggregates
352     except Exception as e:
353         logger.error("Error [get_aggregates(nova_client)]: %s" % e)
354         return None
355
356
357 def get_aggregate_id(nova_client, aggregate_name):
358     try:
359         aggregates = get_aggregates(nova_client)
360         _id = [ag.id for ag in aggregates if ag.name == aggregate_name][0]
361         return _id
362     except Exception as e:
363         logger.error("Error [get_aggregate_id(nova_client, %s)]:"
364                      " %s" % (aggregate_name, e))
365         return None
366
367
368 def get_availability_zones(nova_client):
369     try:
370         availability_zones = nova_client.availability_zones.list()
371         return availability_zones
372     except Exception as e:
373         logger.error("Error [get_availability_zones(nova_client)]: %s" % e)
374         return None
375
376
377 def get_availability_zone_names(nova_client):
378     try:
379         az_names = [az.zoneName for az in get_availability_zones(nova_client)]
380         return az_names
381     except Exception as e:
382         logger.error("Error [get_availability_zone_names(nova_client)]:"
383                      " %s" % e)
384         return None
385
386
387 def create_flavor(nova_client, flavor_name, ram, disk, vcpus, public=True):
388     try:
389         flavor = nova_client.flavors.create(
390             flavor_name, ram, vcpus, disk, is_public=public)
391         try:
392             extra_specs = ft_utils.get_functest_config(
393                 'general.flavor_extra_specs')
394             flavor.set_keys(extra_specs)
395         except ValueError:
396             # flavor extra specs are not configured, therefore skip the update
397             pass
398
399     except Exception as e:
400         logger.error("Error [create_flavor(nova_client, '%s', '%s', '%s', "
401                      "'%s')]: %s" % (flavor_name, ram, disk, vcpus, e))
402         return None
403     return flavor.id
404
405
406 def get_or_create_flavor(flavor_name, ram, disk, vcpus, public=True):
407     flavor_exists = False
408     nova_client = get_nova_client()
409
410     flavor_id = get_flavor_id(nova_client, flavor_name)
411     if flavor_id != '':
412         logger.info("Using existing flavor '%s'..." % flavor_name)
413         flavor_exists = True
414     else:
415         logger.info("Creating flavor '%s' with '%s' RAM, '%s' disk size, "
416                     "'%s' vcpus..." % (flavor_name, ram, disk, vcpus))
417         flavor_id = create_flavor(
418             nova_client, flavor_name, ram, disk, vcpus, public=public)
419         if not flavor_id:
420             logger.error("Failed to create flavor '%s'..." % (flavor_name))
421         else:
422             logger.debug("Flavor '%s' with ID=%s created successfully."
423                          % (flavor_name, flavor_id))
424
425     return flavor_exists, flavor_id
426
427
428 def get_floating_ips(neutron_client):
429     try:
430         floating_ips = neutron_client.list_floatingips()
431         return floating_ips['floatingips']
432     except Exception as e:
433         logger.error("Error [get_floating_ips(neutron_client)]: %s" % e)
434         return None
435
436
437 def get_hypervisors(nova_client):
438     try:
439         nodes = []
440         hypervisors = nova_client.hypervisors.list()
441         for hypervisor in hypervisors:
442             if hypervisor.state == "up":
443                 nodes.append(hypervisor.hypervisor_hostname)
444         return nodes
445     except Exception as e:
446         logger.error("Error [get_hypervisors(nova_client)]: %s" % e)
447         return None
448
449
450 def create_aggregate(nova_client, aggregate_name, av_zone):
451     try:
452         nova_client.aggregates.create(aggregate_name, av_zone)
453         return True
454     except Exception as e:
455         logger.error("Error [create_aggregate(nova_client, %s, %s)]: %s"
456                      % (aggregate_name, av_zone, e))
457         return None
458
459
460 def add_host_to_aggregate(nova_client, aggregate_name, compute_host):
461     try:
462         aggregate_id = get_aggregate_id(nova_client, aggregate_name)
463         nova_client.aggregates.add_host(aggregate_id, compute_host)
464         return True
465     except Exception as e:
466         logger.error("Error [add_host_to_aggregate(nova_client, %s, %s)]: %s"
467                      % (aggregate_name, compute_host, e))
468         return None
469
470
471 def create_aggregate_with_host(
472         nova_client, aggregate_name, av_zone, compute_host):
473     try:
474         create_aggregate(nova_client, aggregate_name, av_zone)
475         add_host_to_aggregate(nova_client, aggregate_name, compute_host)
476         return True
477     except Exception as e:
478         logger.error("Error [create_aggregate_with_host("
479                      "nova_client, %s, %s, %s)]: %s"
480                      % (aggregate_name, av_zone, compute_host, e))
481         return None
482
483
484 def create_instance(flavor_name,
485                     image_id,
486                     network_id,
487                     instance_name="functest-vm",
488                     confdrive=True,
489                     userdata=None,
490                     av_zone='',
491                     fixed_ip=None,
492                     files=None):
493     nova_client = get_nova_client()
494     try:
495         flavor = nova_client.flavors.find(name=flavor_name)
496     except:
497         flavors = nova_client.flavors.list()
498         logger.error("Error: Flavor '%s' not found. Available flavors are: "
499                      "\n%s" % (flavor_name, flavors))
500         return None
501     if fixed_ip is not None:
502         nics = {"net-id": network_id, "v4-fixed-ip": fixed_ip}
503     else:
504         nics = {"net-id": network_id}
505     if userdata is None:
506         instance = nova_client.servers.create(
507             name=instance_name,
508             flavor=flavor,
509             image=image_id,
510             nics=[nics],
511             availability_zone=av_zone,
512             files=files
513         )
514     else:
515         instance = nova_client.servers.create(
516             name=instance_name,
517             flavor=flavor,
518             image=image_id,
519             nics=[nics],
520             config_drive=confdrive,
521             userdata=userdata,
522             availability_zone=av_zone,
523             files=files
524         )
525     return instance
526
527
528 def create_instance_and_wait_for_active(flavor_name,
529                                         image_id,
530                                         network_id,
531                                         instance_name="",
532                                         config_drive=False,
533                                         userdata="",
534                                         av_zone='',
535                                         fixed_ip=None,
536                                         files=None):
537     SLEEP = 3
538     VM_BOOT_TIMEOUT = 180
539     nova_client = get_nova_client()
540     instance = create_instance(flavor_name,
541                                image_id,
542                                network_id,
543                                instance_name,
544                                config_drive,
545                                userdata,
546                                av_zone=av_zone,
547                                fixed_ip=fixed_ip,
548                                files=files)
549     count = VM_BOOT_TIMEOUT / SLEEP
550     for n in range(count, -1, -1):
551         status = get_instance_status(nova_client, instance)
552         if status.lower() == "active":
553             return instance
554         elif status.lower() == "error":
555             logger.error("The instance %s went to ERROR status."
556                          % instance_name)
557             return None
558         time.sleep(SLEEP)
559     logger.error("Timeout booting the instance %s." % instance_name)
560     return None
561
562
563 def create_floating_ip(neutron_client):
564     extnet_id = get_external_net_id(neutron_client)
565     props = {'floating_network_id': extnet_id}
566     try:
567         ip_json = neutron_client.create_floatingip({'floatingip': props})
568         fip_addr = ip_json['floatingip']['floating_ip_address']
569         fip_id = ip_json['floatingip']['id']
570     except Exception as e:
571         logger.error("Error [create_floating_ip(neutron_client)]: %s" % e)
572         return None
573     return {'fip_addr': fip_addr, 'fip_id': fip_id}
574
575
576 def add_floating_ip(nova_client, server_id, floatingip_addr):
577     try:
578         nova_client.servers.add_floating_ip(server_id, floatingip_addr)
579         return True
580     except Exception as e:
581         logger.error("Error [add_floating_ip(nova_client, '%s', '%s')]: %s"
582                      % (server_id, floatingip_addr, e))
583         return False
584
585
586 def delete_instance(nova_client, instance_id):
587     try:
588         nova_client.servers.force_delete(instance_id)
589         return True
590     except Exception as e:
591         logger.error("Error [delete_instance(nova_client, '%s')]: %s"
592                      % (instance_id, e))
593         return False
594
595
596 def delete_floating_ip(neutron_client, floatingip_id):
597     try:
598         neutron_client.delete_floatingip(floatingip_id)
599         return True
600     except Exception as e:
601         logger.error("Error [delete_floating_ip(neutron_client, '%s')]: %s"
602                      % (floatingip_id, e))
603         return False
604
605
606 def remove_host_from_aggregate(nova_client, aggregate_name, compute_host):
607     try:
608         aggregate_id = get_aggregate_id(nova_client, aggregate_name)
609         nova_client.aggregates.remove_host(aggregate_id, compute_host)
610         return True
611     except Exception as e:
612         logger.error("Error [remove_host_from_aggregate(nova_client, %s, %s)]:"
613                      " %s" % (aggregate_name, compute_host, e))
614         return False
615
616
617 def remove_hosts_from_aggregate(nova_client, aggregate_name):
618     aggregate_id = get_aggregate_id(nova_client, aggregate_name)
619     hosts = nova_client.aggregates.get(aggregate_id).hosts
620     assert(
621         all(remove_host_from_aggregate(nova_client, aggregate_name, host)
622             for host in hosts))
623
624
625 def delete_aggregate(nova_client, aggregate_name):
626     try:
627         remove_hosts_from_aggregate(nova_client, aggregate_name)
628         nova_client.aggregates.delete(aggregate_name)
629         return True
630     except Exception as e:
631         logger.error("Error [delete_aggregate(nova_client, %s)]: %s"
632                      % (aggregate_name, e))
633         return False
634
635
636 # *********************************************
637 #   NEUTRON
638 # *********************************************
639 def get_network_list(neutron_client):
640     network_list = neutron_client.list_networks()['networks']
641     if len(network_list) == 0:
642         return None
643     else:
644         return network_list
645
646
647 def get_router_list(neutron_client):
648     router_list = neutron_client.list_routers()['routers']
649     if len(router_list) == 0:
650         return None
651     else:
652         return router_list
653
654
655 def get_port_list(neutron_client):
656     port_list = neutron_client.list_ports()['ports']
657     if len(port_list) == 0:
658         return None
659     else:
660         return port_list
661
662
663 def get_network_id(neutron_client, network_name):
664     networks = neutron_client.list_networks()['networks']
665     id = ''
666     for n in networks:
667         if n['name'] == network_name:
668             id = n['id']
669             break
670     return id
671
672
673 def get_subnet_id(neutron_client, subnet_name):
674     subnets = neutron_client.list_subnets()['subnets']
675     id = ''
676     for s in subnets:
677         if s['name'] == subnet_name:
678             id = s['id']
679             break
680     return id
681
682
683 def get_router_id(neutron_client, router_name):
684     routers = neutron_client.list_routers()['routers']
685     id = ''
686     for r in routers:
687         if r['name'] == router_name:
688             id = r['id']
689             break
690     return id
691
692
693 def get_private_net(neutron_client):
694     # Checks if there is an existing shared private network
695     networks = neutron_client.list_networks()['networks']
696     if len(networks) == 0:
697         return None
698     for net in networks:
699         if (net['router:external'] is False) and (net['shared'] is True):
700             return net
701     return None
702
703
704 def get_external_net(neutron_client):
705     for network in neutron_client.list_networks()['networks']:
706         if network['router:external']:
707             return network['name']
708     return None
709
710
711 def get_external_net_id(neutron_client):
712     for network in neutron_client.list_networks()['networks']:
713         if network['router:external']:
714             return network['id']
715     return None
716
717
718 def check_neutron_net(neutron_client, net_name):
719     for network in neutron_client.list_networks()['networks']:
720         if network['name'] == net_name:
721             for subnet in network['subnets']:
722                 return True
723     return False
724
725
726 def create_neutron_net(neutron_client, name):
727     json_body = {'network': {'name': name,
728                              'admin_state_up': True}}
729     try:
730         network = neutron_client.create_network(body=json_body)
731         network_dict = network['network']
732         return network_dict['id']
733     except Exception as e:
734         logger.error("Error [create_neutron_net(neutron_client, '%s')]: %s"
735                      % (name, e))
736         return None
737
738
739 def create_neutron_subnet(neutron_client, name, cidr, net_id):
740     json_body = {'subnets': [{'name': name, 'cidr': cidr,
741                               'ip_version': 4, 'network_id': net_id}]}
742     try:
743         subnet = neutron_client.create_subnet(body=json_body)
744         return subnet['subnets'][0]['id']
745     except Exception as e:
746         logger.error("Error [create_neutron_subnet(neutron_client, '%s', "
747                      "'%s', '%s')]: %s" % (name, cidr, net_id, e))
748         return None
749
750
751 def create_neutron_router(neutron_client, name):
752     json_body = {'router': {'name': name, 'admin_state_up': True}}
753     try:
754         router = neutron_client.create_router(json_body)
755         return router['router']['id']
756     except Exception as e:
757         logger.error("Error [create_neutron_router(neutron_client, '%s')]: %s"
758                      % (name, e))
759         return None
760
761
762 def create_neutron_port(neutron_client, name, network_id, ip):
763     json_body = {'port': {
764                  'admin_state_up': True,
765                  'name': name,
766                  'network_id': network_id,
767                  'fixed_ips': [{"ip_address": ip}]
768                  }}
769     try:
770         port = neutron_client.create_port(body=json_body)
771         return port['port']['id']
772     except Exception as e:
773         logger.error("Error [create_neutron_port(neutron_client, '%s', '%s', "
774                      "'%s')]: %s" % (name, network_id, ip, e))
775         return None
776
777
778 def update_neutron_net(neutron_client, network_id, shared=False):
779     json_body = {'network': {'shared': shared}}
780     try:
781         neutron_client.update_network(network_id, body=json_body)
782         return True
783     except Exception as e:
784         logger.error("Error [update_neutron_net(neutron_client, '%s', '%s')]: "
785                      "%s" % (network_id, str(shared), e))
786         return False
787
788
789 def update_neutron_port(neutron_client, port_id, device_owner):
790     json_body = {'port': {
791                  'device_owner': device_owner,
792                  }}
793     try:
794         port = neutron_client.update_port(port=port_id,
795                                           body=json_body)
796         return port['port']['id']
797     except Exception as e:
798         logger.error("Error [update_neutron_port(neutron_client, '%s', '%s')]:"
799                      " %s" % (port_id, device_owner, e))
800         return None
801
802
803 def add_interface_router(neutron_client, router_id, subnet_id):
804     json_body = {"subnet_id": subnet_id}
805     try:
806         neutron_client.add_interface_router(router=router_id, body=json_body)
807         return True
808     except Exception as e:
809         logger.error("Error [add_interface_router(neutron_client, '%s', "
810                      "'%s')]: %s" % (router_id, subnet_id, e))
811         return False
812
813
814 def add_gateway_router(neutron_client, router_id):
815     ext_net_id = get_external_net_id(neutron_client)
816     router_dict = {'network_id': ext_net_id}
817     try:
818         neutron_client.add_gateway_router(router_id, router_dict)
819         return True
820     except Exception as e:
821         logger.error("Error [add_gateway_router(neutron_client, '%s')]: %s"
822                      % (router_id, e))
823         return False
824
825
826 def delete_neutron_net(neutron_client, network_id):
827     try:
828         neutron_client.delete_network(network_id)
829         return True
830     except Exception as e:
831         logger.error("Error [delete_neutron_net(neutron_client, '%s')]: %s"
832                      % (network_id, e))
833         return False
834
835
836 def delete_neutron_subnet(neutron_client, subnet_id):
837     try:
838         neutron_client.delete_subnet(subnet_id)
839         return True
840     except Exception as e:
841         logger.error("Error [delete_neutron_subnet(neutron_client, '%s')]: %s"
842                      % (subnet_id, e))
843         return False
844
845
846 def delete_neutron_router(neutron_client, router_id):
847     try:
848         neutron_client.delete_router(router=router_id)
849         return True
850     except Exception as e:
851         logger.error("Error [delete_neutron_router(neutron_client, '%s')]: %s"
852                      % (router_id, e))
853         return False
854
855
856 def delete_neutron_port(neutron_client, port_id):
857     try:
858         neutron_client.delete_port(port_id)
859         return True
860     except Exception as e:
861         logger.error("Error [delete_neutron_port(neutron_client, '%s')]: %s"
862                      % (port_id, e))
863         return False
864
865
866 def remove_interface_router(neutron_client, router_id, subnet_id):
867     json_body = {"subnet_id": subnet_id}
868     try:
869         neutron_client.remove_interface_router(router=router_id,
870                                                body=json_body)
871         return True
872     except Exception as e:
873         logger.error("Error [remove_interface_router(neutron_client, '%s', "
874                      "'%s')]: %s" % (router_id, subnet_id, e))
875         return False
876
877
878 def remove_gateway_router(neutron_client, router_id):
879     try:
880         neutron_client.remove_gateway_router(router_id)
881         return True
882     except Exception as e:
883         logger.error("Error [remove_gateway_router(neutron_client, '%s')]: %s"
884                      % (router_id, e))
885         return False
886
887
888 def create_network_full(neutron_client,
889                         net_name,
890                         subnet_name,
891                         router_name,
892                         cidr):
893
894     # Check if the network already exists
895     network_id = get_network_id(neutron_client, net_name)
896     subnet_id = get_subnet_id(neutron_client, subnet_name)
897     router_id = get_router_id(neutron_client, router_name)
898
899     if network_id != '' and subnet_id != '' and router_id != '':
900         logger.info("A network with name '%s' already exists..." % net_name)
901     else:
902         neutron_client.format = 'json'
903         logger.info('Creating neutron network %s...' % net_name)
904         network_id = create_neutron_net(neutron_client, net_name)
905
906         if not network_id:
907             return False
908
909         logger.debug("Network '%s' created successfully" % network_id)
910         logger.debug('Creating Subnet....')
911         subnet_id = create_neutron_subnet(neutron_client, subnet_name,
912                                           cidr, network_id)
913         if not subnet_id:
914             return None
915
916         logger.debug("Subnet '%s' created successfully" % subnet_id)
917         logger.debug('Creating Router...')
918         router_id = create_neutron_router(neutron_client, router_name)
919
920         if not router_id:
921             return None
922
923         logger.debug("Router '%s' created successfully" % router_id)
924         logger.debug('Adding router to subnet...')
925
926         if not add_interface_router(neutron_client, router_id, subnet_id):
927             return None
928
929         logger.debug("Interface added successfully.")
930
931         logger.debug('Adding gateway to router...')
932         if not add_gateway_router(neutron_client, router_id):
933             return None
934
935         logger.debug("Gateway added successfully.")
936
937     network_dic = {'net_id': network_id,
938                    'subnet_id': subnet_id,
939                    'router_id': router_id}
940     return network_dic
941
942
943 def create_shared_network_full(net_name, subnt_name, router_name, subnet_cidr):
944     neutron_client = get_neutron_client()
945
946     network_dic = create_network_full(neutron_client,
947                                       net_name,
948                                       subnt_name,
949                                       router_name,
950                                       subnet_cidr)
951     if network_dic:
952         if not update_neutron_net(neutron_client,
953                                   network_dic['net_id'],
954                                   shared=True):
955             logger.error("Failed to update network %s..." % net_name)
956             return None
957         else:
958             logger.debug("Network '%s' is available..." % net_name)
959     else:
960         logger.error("Network %s creation failed" % net_name)
961         return None
962     return network_dic
963
964
965 # *********************************************
966 #   SEC GROUPS
967 # *********************************************
968
969
970 def get_security_groups(neutron_client):
971     try:
972         security_groups = neutron_client.list_security_groups()[
973             'security_groups']
974         return security_groups
975     except Exception as e:
976         logger.error("Error [get_security_groups(neutron_client)]: %s" % e)
977         return None
978
979
980 def get_security_group_id(neutron_client, sg_name):
981     security_groups = get_security_groups(neutron_client)
982     id = ''
983     for sg in security_groups:
984         if sg['name'] == sg_name:
985             id = sg['id']
986             break
987     return id
988
989
990 def create_security_group(neutron_client, sg_name, sg_description):
991     json_body = {'security_group': {'name': sg_name,
992                                     'description': sg_description}}
993     try:
994         secgroup = neutron_client.create_security_group(json_body)
995         return secgroup['security_group']
996     except Exception as e:
997         logger.error("Error [create_security_group(neutron_client, '%s', "
998                      "'%s')]: %s" % (sg_name, sg_description, e))
999         return None
1000
1001
1002 def create_secgroup_rule(neutron_client, sg_id, direction, protocol,
1003                          port_range_min=None, port_range_max=None):
1004     # We create a security group in 2 steps
1005     # 1 - we check the format and set the json body accordingly
1006     # 2 - we call neturon client to create the security group
1007
1008     # Format check
1009     json_body = {'security_group_rule': {'direction': direction,
1010                                          'security_group_id': sg_id,
1011                                          'protocol': protocol}}
1012     # parameters may be
1013     # - both None => we do nothing
1014     # - both Not None => we add them to the json description
1015     # but one cannot be None is the other is not None
1016     if (port_range_min is not None and port_range_max is not None):
1017         # add port_range in json description
1018         json_body['security_group_rule']['port_range_min'] = port_range_min
1019         json_body['security_group_rule']['port_range_max'] = port_range_max
1020         logger.debug("Security_group format set (port range included)")
1021     else:
1022         # either both port range are set to None => do nothing
1023         # or one is set but not the other => log it and return False
1024         if port_range_min is None and port_range_max is None:
1025             logger.debug("Security_group format set (no port range mentioned)")
1026         else:
1027             logger.error("Bad security group format."
1028                          "One of the port range is not properly set:"
1029                          "range min: {},"
1030                          "range max: {}".format(port_range_min,
1031                                                 port_range_max))
1032             return False
1033
1034     # Create security group using neutron client
1035     try:
1036         neutron_client.create_security_group_rule(json_body)
1037         return True
1038     except:
1039         logger.exception("Impossible to create_security_group_rule,"
1040                          "security group rule probably already exists")
1041         return False
1042
1043
1044 def get_security_group_rules(neutron_client, sg_id):
1045     try:
1046         security_rules = neutron_client.list_security_group_rules()[
1047             'security_group_rules']
1048         security_rules = [rule for rule in security_rules
1049                           if rule["security_group_id"] == sg_id]
1050         return security_rules
1051     except Exception as e:
1052         logger.error("Error [get_security_group_rules(neutron_client, sg_id)]:"
1053                      " %s" % e)
1054         return None
1055
1056
1057 def check_security_group_rules(neutron_client, sg_id, direction, protocol,
1058                                port_min=None, port_max=None):
1059     try:
1060         security_rules = get_security_group_rules(neutron_client, sg_id)
1061         security_rules = [rule for rule in security_rules
1062                           if (rule["direction"].lower() == direction and
1063                               rule["protocol"].lower() == protocol and
1064                               rule["port_range_min"] == port_min and
1065                               rule["port_range_max"] == port_max)]
1066         if len(security_rules) == 0:
1067             return True
1068         else:
1069             return False
1070     except Exception as e:
1071         logger.error("Error [check_security_group_rules("
1072                      " neutron_client, sg_id, direction,"
1073                      " protocol, port_min=None, port_max=None)]: "
1074                      "%s" % e)
1075         return None
1076
1077
1078 def create_security_group_full(neutron_client,
1079                                sg_name, sg_description):
1080     sg_id = get_security_group_id(neutron_client, sg_name)
1081     if sg_id != '':
1082         logger.info("Using existing security group '%s'..." % sg_name)
1083     else:
1084         logger.info("Creating security group  '%s'..." % sg_name)
1085         SECGROUP = create_security_group(neutron_client,
1086                                          sg_name,
1087                                          sg_description)
1088         if not SECGROUP:
1089             logger.error("Failed to create the security group...")
1090             return None
1091
1092         sg_id = SECGROUP['id']
1093
1094         logger.debug("Security group '%s' with ID=%s created successfully."
1095                      % (SECGROUP['name'], sg_id))
1096
1097         logger.debug("Adding ICMP rules in security group '%s'..."
1098                      % sg_name)
1099         if not create_secgroup_rule(neutron_client, sg_id,
1100                                     'ingress', 'icmp'):
1101             logger.error("Failed to create the security group rule...")
1102             return None
1103
1104         logger.debug("Adding SSH rules in security group '%s'..."
1105                      % sg_name)
1106         if not create_secgroup_rule(
1107                 neutron_client, sg_id, 'ingress', 'tcp', '22', '22'):
1108             logger.error("Failed to create the security group rule...")
1109             return None
1110
1111         if not create_secgroup_rule(
1112                 neutron_client, sg_id, 'egress', 'tcp', '22', '22'):
1113             logger.error("Failed to create the security group rule...")
1114             return None
1115     return sg_id
1116
1117
1118 def add_secgroup_to_instance(nova_client, instance_id, secgroup_id):
1119     try:
1120         nova_client.servers.add_security_group(instance_id, secgroup_id)
1121         return True
1122     except Exception as e:
1123         logger.error("Error [add_secgroup_to_instance(nova_client, '%s', "
1124                      "'%s')]: %s" % (instance_id, secgroup_id, e))
1125         return False
1126
1127
1128 def update_sg_quota(neutron_client, tenant_id, sg_quota, sg_rule_quota):
1129     json_body = {"quota": {
1130         "security_group": sg_quota,
1131         "security_group_rule": sg_rule_quota
1132     }}
1133
1134     try:
1135         neutron_client.update_quota(tenant_id=tenant_id,
1136                                     body=json_body)
1137         return True
1138     except Exception as e:
1139         logger.error("Error [update_sg_quota(neutron_client, '%s', '%s', "
1140                      "'%s')]: %s" % (tenant_id, sg_quota, sg_rule_quota, e))
1141         return False
1142
1143
1144 def delete_security_group(neutron_client, secgroup_id):
1145     try:
1146         neutron_client.delete_security_group(secgroup_id)
1147         return True
1148     except Exception as e:
1149         logger.error("Error [delete_security_group(neutron_client, '%s')]: %s"
1150                      % (secgroup_id, e))
1151         return False
1152
1153
1154 # *********************************************
1155 #   GLANCE
1156 # *********************************************
1157 def get_images(glance_client):
1158     try:
1159         images = glance_client.images.list()
1160         return images
1161     except Exception as e:
1162         logger.error("Error [get_images]: %s" % e)
1163         return None
1164
1165
1166 def get_image_id(glance_client, image_name):
1167     images = glance_client.images.list()
1168     id = ''
1169     for i in images:
1170         if i.name == image_name:
1171             id = i.id
1172             break
1173     return id
1174
1175
1176 def create_glance_image(glance_client, image_name, file_path, disk="qcow2",
1177                         container="bare", public="public"):
1178     if not os.path.isfile(file_path):
1179         logger.error("Error: file %s does not exist." % file_path)
1180         return None
1181     try:
1182         image_id = get_image_id(glance_client, image_name)
1183         if image_id != '':
1184             logger.info("Image %s already exists." % image_name)
1185         else:
1186             logger.info("Creating image '%s' from '%s'..." % (image_name,
1187                                                               file_path))
1188
1189             image = glance_client.images.create(name=image_name,
1190                                                 visibility=public,
1191                                                 disk_format=disk,
1192                                                 container_format=container)
1193             image_id = image.id
1194             with open(file_path) as image_data:
1195                 glance_client.images.upload(image_id, image_data)
1196         return image_id
1197     except Exception as e:
1198         logger.error("Error [create_glance_image(glance_client, '%s', '%s', "
1199                      "'%s')]: %s" % (image_name, file_path, public, e))
1200         return None
1201
1202
1203 def get_or_create_image(name, path, format):
1204     image_exists = False
1205     glance_client = get_glance_client()
1206
1207     image_id = get_image_id(glance_client, name)
1208     if image_id != '':
1209         logger.info("Using existing image '%s'..." % name)
1210         image_exists = True
1211     else:
1212         logger.info("Creating image '%s' from '%s'..." % (name, path))
1213         image_id = create_glance_image(glance_client, name, path, format)
1214         if not image_id:
1215             logger.error("Failed to create a Glance image...")
1216         else:
1217             logger.debug("Image '%s' with ID=%s created successfully."
1218                          % (name, image_id))
1219
1220     return image_exists, image_id
1221
1222
1223 def delete_glance_image(glance_client, image_id):
1224     try:
1225         glance_client.images.delete(image_id)
1226         return True
1227     except Exception as e:
1228         logger.error("Error [delete_glance_image(glance_client, '%s')]: %s"
1229                      % (image_id, e))
1230         return False
1231
1232
1233 # *********************************************
1234 #   CINDER
1235 # *********************************************
1236 def get_volumes(cinder_client):
1237     try:
1238         volumes = cinder_client.volumes.list(search_opts={'all_tenants': 1})
1239         return volumes
1240     except Exception as e:
1241         logger.error("Error [get_volumes(cinder_client)]: %s" % e)
1242         return None
1243
1244
1245 def list_volume_types(cinder_client, public=True, private=True):
1246     try:
1247         volume_types = cinder_client.volume_types.list()
1248         if not public:
1249             volume_types = [vt for vt in volume_types if not vt.is_public]
1250         if not private:
1251             volume_types = [vt for vt in volume_types if vt.is_public]
1252         return volume_types
1253     except Exception as e:
1254         logger.error("Error [list_volume_types(cinder_client)]: %s" % e)
1255         return None
1256
1257
1258 def create_volume_type(cinder_client, name):
1259     try:
1260         volume_type = cinder_client.volume_types.create(name)
1261         return volume_type
1262     except Exception as e:
1263         logger.error("Error [create_volume_type(cinder_client, '%s')]: %s"
1264                      % (name, e))
1265         return None
1266
1267
1268 def update_cinder_quota(cinder_client, tenant_id, vols_quota,
1269                         snapshots_quota, gigabytes_quota):
1270     quotas_values = {"volumes": vols_quota,
1271                      "snapshots": snapshots_quota,
1272                      "gigabytes": gigabytes_quota}
1273
1274     try:
1275         cinder_client.quotas.update(tenant_id, **quotas_values)
1276         return True
1277     except Exception as e:
1278         logger.error("Error [update_cinder_quota(cinder_client, '%s', '%s', "
1279                      "'%s' '%s')]: %s" % (tenant_id, vols_quota,
1280                                           snapshots_quota, gigabytes_quota, e))
1281         return False
1282
1283
1284 def delete_volume(cinder_client, volume_id, forced=False):
1285     try:
1286         if forced:
1287             try:
1288                 cinder_client.volumes.detach(volume_id)
1289             except:
1290                 logger.error(sys.exc_info()[0])
1291             cinder_client.volumes.force_delete(volume_id)
1292         else:
1293             cinder_client.volumes.delete(volume_id)
1294         return True
1295     except Exception as e:
1296         logger.error("Error [delete_volume(cinder_client, '%s', '%s')]: %s"
1297                      % (volume_id, str(forced), e))
1298         return False
1299
1300
1301 def delete_volume_type(cinder_client, volume_type):
1302     try:
1303         cinder_client.volume_types.delete(volume_type)
1304         return True
1305     except Exception as e:
1306         logger.error("Error [delete_volume_type(cinder_client, '%s')]: %s"
1307                      % (volume_type, e))
1308         return False
1309
1310
1311 # *********************************************
1312 #   KEYSTONE
1313 # *********************************************
1314 def get_tenants(keystone_client):
1315     try:
1316         if is_keystone_v3():
1317             tenants = keystone_client.projects.list()
1318         else:
1319             tenants = keystone_client.tenants.list()
1320         return tenants
1321     except Exception as e:
1322         logger.error("Error [get_tenants(keystone_client)]: %s" % e)
1323         return None
1324
1325
1326 def get_users(keystone_client):
1327     try:
1328         users = keystone_client.users.list()
1329         return users
1330     except Exception as e:
1331         logger.error("Error [get_users(keystone_client)]: %s" % e)
1332         return None
1333
1334
1335 def get_tenant_id(keystone_client, tenant_name):
1336     tenants = get_tenants(keystone_client)
1337     id = ''
1338     for t in tenants:
1339         if t.name == tenant_name:
1340             id = t.id
1341             break
1342     return id
1343
1344
1345 def get_user_id(keystone_client, user_name):
1346     users = get_users(keystone_client)
1347     id = ''
1348     for u in users:
1349         if u.name == user_name:
1350             id = u.id
1351             break
1352     return id
1353
1354
1355 def get_role_id(keystone_client, role_name):
1356     roles = keystone_client.roles.list()
1357     id = ''
1358     for r in roles:
1359         if r.name == role_name:
1360             id = r.id
1361             break
1362     return id
1363
1364
1365 def create_tenant(keystone_client, tenant_name, tenant_description):
1366     try:
1367         if is_keystone_v3():
1368             tenant = keystone_client.projects.create(
1369                 name=tenant_name,
1370                 description=tenant_description,
1371                 domain="default",
1372                 enabled=True)
1373         else:
1374             tenant = keystone_client.tenants.create(tenant_name,
1375                                                     tenant_description,
1376                                                     enabled=True)
1377         return tenant.id
1378     except Exception as e:
1379         logger.error("Error [create_tenant(keystone_client, '%s', '%s')]: %s"
1380                      % (tenant_name, tenant_description, e))
1381         return None
1382
1383
1384 def get_or_create_tenant(keystone_client, tenant_name, tenant_description):
1385     tenant_id = get_tenant_id(keystone_client, tenant_name)
1386     if not tenant_id:
1387         tenant_id = create_tenant(keystone_client, tenant_name,
1388                                   tenant_description)
1389
1390     return tenant_id
1391
1392
1393 def get_or_create_tenant_for_vnf(keystone_client, tenant_name,
1394                                  tenant_description):
1395     """Get or Create a Tenant
1396
1397         Args:
1398             keystone_client: keystone client reference
1399             tenant_name: the name of the tenant
1400             tenant_description: the description of the tenant
1401
1402         return False if tenant retrieved though get
1403         return True if tenant created
1404         raise Exception if error during processing
1405     """
1406     try:
1407         tenant_id = get_tenant_id(keystone_client, tenant_name)
1408         if not tenant_id:
1409             tenant_id = create_tenant(keystone_client, tenant_name,
1410                                       tenant_description)
1411             return True
1412         else:
1413             return False
1414     except:
1415         raise Exception("Impossible to create a Tenant for the VNF {}".format(
1416                             tenant_name))
1417
1418
1419 def create_user(keystone_client, user_name, user_password,
1420                 user_email, tenant_id):
1421     try:
1422         if is_keystone_v3():
1423             user = keystone_client.users.create(name=user_name,
1424                                                 password=user_password,
1425                                                 email=user_email,
1426                                                 project_id=tenant_id,
1427                                                 enabled=True)
1428         else:
1429             user = keystone_client.users.create(user_name,
1430                                                 user_password,
1431                                                 user_email,
1432                                                 tenant_id,
1433                                                 enabled=True)
1434         return user.id
1435     except Exception as e:
1436         logger.error("Error [create_user(keystone_client, '%s', '%s', '%s'"
1437                      "'%s')]: %s" % (user_name, user_password,
1438                                      user_email, tenant_id, e))
1439         return None
1440
1441
1442 def get_or_create_user(keystone_client, user_name, user_password,
1443                        tenant_id, user_email=None):
1444     user_id = get_user_id(keystone_client, user_name)
1445     if not user_id:
1446         user_id = create_user(keystone_client, user_name, user_password,
1447                               user_email, tenant_id)
1448     return user_id
1449
1450
1451 def get_or_create_user_for_vnf(keystone_client, vnf_ref):
1452     """Get or Create user for VNF
1453
1454         Args:
1455             keystone_client: keystone client reference
1456             vnf_ref: VNF reference used as user name & password, tenant name
1457
1458         return False if user retrieved through get
1459         return True if user created
1460         raise Exception if error during processing
1461     """
1462     try:
1463         user_id = get_user_id(keystone_client, vnf_ref)
1464         tenant_id = get_tenant_id(keystone_client, vnf_ref)
1465         if not user_id:
1466             user_id = create_user(keystone_client, vnf_ref, vnf_ref,
1467                                   "", tenant_id)
1468             return True
1469         else:
1470             return False
1471         add_role_user(keystone_client, user_id, 'admin', vnf_ref)
1472     except:
1473         raise Exception("Impossible to create a user for the VNF {}".format(
1474             vnf_ref))
1475
1476
1477 def add_role_user(keystone_client, user_id, role_id, tenant_id):
1478     try:
1479         if is_keystone_v3():
1480             keystone_client.roles.grant(role=role_id,
1481                                         user=user_id,
1482                                         project=tenant_id)
1483         else:
1484             keystone_client.roles.add_user_role(user_id, role_id, tenant_id)
1485         return True
1486     except Exception as e:
1487         logger.error("Error [add_role_user(keystone_client, '%s', '%s'"
1488                      "'%s')]: %s " % (user_id, role_id, tenant_id, e))
1489         return False
1490
1491
1492 def delete_tenant(keystone_client, tenant_id):
1493     try:
1494         if is_keystone_v3():
1495             keystone_client.projects.delete(tenant_id)
1496         else:
1497             keystone_client.tenants.delete(tenant_id)
1498         return True
1499     except Exception as e:
1500         logger.error("Error [delete_tenant(keystone_client, '%s')]: %s"
1501                      % (tenant_id, e))
1502         return False
1503
1504
1505 def delete_user(keystone_client, user_id):
1506     try:
1507         keystone_client.users.delete(user_id)
1508         return True
1509     except Exception as e:
1510         logger.error("Error [delete_user(keystone_client, '%s')]: %s"
1511                      % (user_id, e))
1512         return False
1513
1514
1515 # *********************************************
1516 #   HEAT
1517 # *********************************************
1518 def get_resource(heat_client, stack_id, resource):
1519     try:
1520         resources = heat_client.resources.get(stack_id, resource)
1521         return resources
1522     except Exception as e:
1523         logger.error("Error [get_resource]: %s" % e)
1524         return None