1d7cdafb9c44db7c8cfd094d33fbfea927a91785
[functest.git] / functest / utils / openstack_utils.py
1 #!/usr/bin/env python
2 #
3 # jose.lausuch@ericsson.com
4 # valentin.boucher@orange.com
5 # All rights reserved. This program and the accompanying materials
6 # are made available under the terms of the Apache License, Version 2.0
7 # which accompanies this distribution, and is available at
8 # http://www.apache.org/licenses/LICENSE-2.0
9 #
10
11 import logging
12 import os.path
13 import re
14 import sys
15 import time
16
17 from keystoneauth1 import loading
18 from keystoneauth1 import session
19 from cinderclient import client as cinderclient
20 from glanceclient import client as glanceclient
21 from heatclient import client as heatclient
22 from novaclient import client as novaclient
23 from keystoneclient import client as keystoneclient
24 from neutronclient.neutron import client as neutronclient
25
26 from functest.utils.constants import CONST
27 import functest.utils.functest_utils as ft_utils
28
29 logger = logging.getLogger(__name__)
30
31 DEFAULT_API_VERSION = '2'
32 DEFAULT_HEAT_API_VERSION = '1'
33
34
35 # *********************************************
36 #   CREDENTIALS
37 # *********************************************
38 class MissingEnvVar(Exception):
39
40     def __init__(self, var):
41         self.var = var
42
43     def __str__(self):
44         return str.format("Please set the mandatory env var: {}", self.var)
45
46
47 def is_keystone_v3():
48     keystone_api_version = os.getenv('OS_IDENTITY_API_VERSION')
49     if (keystone_api_version is None or
50             keystone_api_version == '2'):
51         return False
52     else:
53         return True
54
55
56 def get_rc_env_vars():
57     env_vars = ['OS_AUTH_URL', 'OS_USERNAME', 'OS_PASSWORD']
58     if is_keystone_v3():
59         env_vars.extend(['OS_PROJECT_NAME',
60                          'OS_USER_DOMAIN_NAME',
61                          'OS_PROJECT_DOMAIN_NAME'])
62     else:
63         env_vars.extend(['OS_TENANT_NAME'])
64     return env_vars
65
66
67 def check_credentials():
68     """
69     Check if the OpenStack credentials (openrc) are sourced
70     """
71     env_vars = get_rc_env_vars()
72     return all(map(lambda v: v in os.environ and os.environ[v], env_vars))
73
74
75 def get_env_cred_dict():
76     env_cred_dict = {
77         'OS_USERNAME': 'username',
78         'OS_PASSWORD': 'password',
79         'OS_AUTH_URL': 'auth_url',
80         'OS_TENANT_NAME': 'tenant_name',
81         'OS_USER_DOMAIN_NAME': 'user_domain_name',
82         'OS_PROJECT_DOMAIN_NAME': 'project_domain_name',
83         'OS_PROJECT_NAME': 'project_name',
84         'OS_ENDPOINT_TYPE': 'endpoint_type',
85         'OS_REGION_NAME': 'region_name',
86         'OS_CACERT': 'https_cacert',
87         'OS_INSECURE': 'https_insecure'
88     }
89     return env_cred_dict
90
91
92 def get_credentials(other_creds={}):
93     """Returns a creds dictionary filled with parsed from env
94     """
95     creds = {}
96     env_vars = get_rc_env_vars()
97     env_cred_dict = get_env_cred_dict()
98
99     for envvar in env_vars:
100         if os.getenv(envvar) is None:
101             raise MissingEnvVar(envvar)
102         else:
103             creds_key = env_cred_dict.get(envvar)
104             creds.update({creds_key: os.getenv(envvar)})
105
106     if 'tenant' in other_creds.keys():
107         if is_keystone_v3():
108             tenant = 'project_name'
109         else:
110             tenant = 'tenant_name'
111         other_creds[tenant] = other_creds.pop('tenant')
112
113     creds.update(other_creds)
114
115     return creds
116
117
118 def source_credentials(rc_file):
119     with open(rc_file, "r") as f:
120         for line in f:
121             var = (line.rstrip('"\n').replace('export ', '').split("=")
122                    if re.search(r'(.*)=(.*)', line) else None)
123             # The two next lines should be modified as soon as rc_file
124             # conforms with common rules. Be aware that it could induce
125             # issues if value starts with '
126             if var:
127                 key = re.sub(r'^["\' ]*|[ \'"]*$', '', var[0])
128                 value = re.sub(r'^["\' ]*|[ \'"]*$', '', "".join(var[1:]))
129                 os.environ[key] = value
130                 CONST.__setattr__(key, value)
131
132
133 def get_session_auth(other_creds={}):
134     loader = loading.get_plugin_loader('password')
135     creds = get_credentials(other_creds)
136     auth = loader.load_from_options(**creds)
137     return auth
138
139
140 def get_endpoint(service_type, interface='public'):
141     auth = get_session_auth()
142     return get_session().get_endpoint(auth=auth,
143                                       service_type=service_type,
144                                       interface=interface)
145
146
147 def get_session(other_creds={}):
148     auth = get_session_auth(other_creds)
149     https_cacert = os.getenv('OS_CACERT', '')
150     https_insecure = os.getenv('OS_INSECURE', '').lower() == 'true'
151     return session.Session(auth=auth,
152                            verify=(https_cacert or not https_insecure))
153
154
155 # *********************************************
156 #   CLIENTS
157 # *********************************************
158 def get_keystone_client_version():
159     api_version = os.getenv('OS_IDENTITY_API_VERSION')
160     if api_version is not None:
161         logger.info("OS_IDENTITY_API_VERSION is set in env as '%s'",
162                     api_version)
163         return api_version
164     return DEFAULT_API_VERSION
165
166
167 def get_keystone_client(other_creds={}):
168     sess = get_session(other_creds)
169     return keystoneclient.Client(get_keystone_client_version(),
170                                  session=sess,
171                                  interface=os.getenv('OS_INTERFACE', 'admin'))
172
173
174 def get_nova_client_version():
175     api_version = os.getenv('OS_COMPUTE_API_VERSION')
176     if api_version is not None:
177         logger.info("OS_COMPUTE_API_VERSION is set in env as '%s'",
178                     api_version)
179         return api_version
180     return DEFAULT_API_VERSION
181
182
183 def get_nova_client(other_creds={}):
184     sess = get_session(other_creds)
185     return novaclient.Client(get_nova_client_version(), session=sess)
186
187
188 def get_cinder_client_version():
189     api_version = os.getenv('OS_VOLUME_API_VERSION')
190     if api_version is not None:
191         logger.info("OS_VOLUME_API_VERSION is set in env as '%s'",
192                     api_version)
193         return api_version
194     return DEFAULT_API_VERSION
195
196
197 def get_cinder_client(other_creds={}):
198     sess = get_session(other_creds)
199     return cinderclient.Client(get_cinder_client_version(), session=sess)
200
201
202 def get_neutron_client_version():
203     api_version = os.getenv('OS_NETWORK_API_VERSION')
204     if api_version is not None:
205         logger.info("OS_NETWORK_API_VERSION is set in env as '%s'",
206                     api_version)
207         return api_version
208     return DEFAULT_API_VERSION
209
210
211 def get_neutron_client(other_creds={}):
212     sess = get_session(other_creds)
213     return neutronclient.Client(get_neutron_client_version(), session=sess)
214
215
216 def get_glance_client_version():
217     api_version = os.getenv('OS_IMAGE_API_VERSION')
218     if api_version is not None:
219         logger.info("OS_IMAGE_API_VERSION is set in env as '%s'", api_version)
220         return api_version
221     return DEFAULT_API_VERSION
222
223
224 def get_glance_client(other_creds={}):
225     sess = get_session(other_creds)
226     return glanceclient.Client(get_glance_client_version(), session=sess)
227
228
229 def get_heat_client_version():
230     api_version = os.getenv('OS_ORCHESTRATION_API_VERSION')
231     if api_version is not None:
232         logger.info("OS_ORCHESTRATION_API_VERSION is set in env as '%s'",
233                     api_version)
234         return api_version
235     return DEFAULT_HEAT_API_VERSION
236
237
238 def get_heat_client(other_creds={}):
239     sess = get_session(other_creds)
240     return heatclient.Client(get_heat_client_version(), session=sess)
241
242
243 def download_and_add_image_on_glance(glance, image_name, image_url, data_dir):
244     try:
245         dest_path = data_dir
246         if not os.path.exists(dest_path):
247             os.makedirs(dest_path)
248         file_name = image_url.rsplit('/')[-1]
249         if not ft_utils.download_url(image_url, dest_path):
250             return False
251     except Exception:
252         raise Exception("Impossible to download image from {}".format(
253                         image_url))
254
255     try:
256         image = create_glance_image(
257             glance, image_name, dest_path + file_name)
258         if not image:
259             return False
260         else:
261             return image
262     except Exception:
263         raise Exception("Impossible to put image {} in glance".format(
264                         image_name))
265
266
267 # *********************************************
268 #   NOVA
269 # *********************************************
270 def get_instances(nova_client):
271     try:
272         instances = nova_client.servers.list(search_opts={'all_tenants': 1})
273         return instances
274     except Exception as e:
275         logger.error("Error [get_instances(nova_client)]: %s" % e)
276         return None
277
278
279 def get_instance_status(nova_client, instance):
280     try:
281         instance = nova_client.servers.get(instance.id)
282         return instance.status
283     except Exception as e:
284         logger.error("Error [get_instance_status(nova_client)]: %s" % e)
285         return None
286
287
288 def get_instance_by_name(nova_client, instance_name):
289     try:
290         instance = nova_client.servers.find(name=instance_name)
291         return instance
292     except Exception as e:
293         logger.error("Error [get_instance_by_name(nova_client, '%s')]: %s"
294                      % (instance_name, e))
295         return None
296
297
298 def get_flavor_id(nova_client, flavor_name):
299     flavors = nova_client.flavors.list(detailed=True)
300     id = ''
301     for f in flavors:
302         if f.name == flavor_name:
303             id = f.id
304             break
305     return id
306
307
308 def get_flavor_id_by_ram_range(nova_client, min_ram, max_ram):
309     flavors = nova_client.flavors.list(detailed=True)
310     id = ''
311     for f in flavors:
312         if min_ram <= f.ram and f.ram <= max_ram:
313             id = f.id
314             break
315     return id
316
317
318 def get_aggregates(nova_client):
319     try:
320         aggregates = nova_client.aggregates.list()
321         return aggregates
322     except Exception as e:
323         logger.error("Error [get_aggregates(nova_client)]: %s" % e)
324         return None
325
326
327 def get_aggregate_id(nova_client, aggregate_name):
328     try:
329         aggregates = get_aggregates(nova_client)
330         _id = [ag.id for ag in aggregates if ag.name == aggregate_name][0]
331         return _id
332     except Exception as e:
333         logger.error("Error [get_aggregate_id(nova_client, %s)]:"
334                      " %s" % (aggregate_name, e))
335         return None
336
337
338 def get_availability_zones(nova_client):
339     try:
340         availability_zones = nova_client.availability_zones.list()
341         return availability_zones
342     except Exception as e:
343         logger.error("Error [get_availability_zones(nova_client)]: %s" % e)
344         return None
345
346
347 def get_availability_zone_names(nova_client):
348     try:
349         az_names = [az.zoneName for az in get_availability_zones(nova_client)]
350         return az_names
351     except Exception as e:
352         logger.error("Error [get_availability_zone_names(nova_client)]:"
353                      " %s" % e)
354         return None
355
356
357 def create_flavor(nova_client, flavor_name, ram, disk, vcpus, public=True):
358     try:
359         flavor = nova_client.flavors.create(
360             flavor_name, ram, vcpus, disk, is_public=public)
361         try:
362             extra_specs = ft_utils.get_functest_config(
363                 'general.flavor_extra_specs')
364             flavor.set_keys(extra_specs)
365         except ValueError:
366             # flavor extra specs are not configured, therefore skip the update
367             pass
368
369     except Exception as e:
370         logger.error("Error [create_flavor(nova_client, '%s', '%s', '%s', "
371                      "'%s')]: %s" % (flavor_name, ram, disk, vcpus, e))
372         return None
373     return flavor.id
374
375
376 def get_or_create_flavor(flavor_name, ram, disk, vcpus, public=True):
377     flavor_exists = False
378     nova_client = get_nova_client()
379
380     flavor_id = get_flavor_id(nova_client, flavor_name)
381     if flavor_id != '':
382         logger.info("Using existing flavor '%s'..." % flavor_name)
383         flavor_exists = True
384     else:
385         logger.info("Creating flavor '%s' with '%s' RAM, '%s' disk size, "
386                     "'%s' vcpus..." % (flavor_name, ram, disk, vcpus))
387         flavor_id = create_flavor(
388             nova_client, flavor_name, ram, disk, vcpus, public=public)
389         if not flavor_id:
390             raise Exception("Failed to create flavor '%s'..." % (flavor_name))
391         else:
392             logger.debug("Flavor '%s' with ID=%s created successfully."
393                          % (flavor_name, flavor_id))
394
395     return flavor_exists, flavor_id
396
397
398 def get_floating_ips(neutron_client):
399     try:
400         floating_ips = neutron_client.list_floatingips()
401         return floating_ips['floatingips']
402     except Exception as e:
403         logger.error("Error [get_floating_ips(neutron_client)]: %s" % e)
404         return None
405
406
407 def get_hypervisors(nova_client):
408     try:
409         nodes = []
410         hypervisors = nova_client.hypervisors.list()
411         for hypervisor in hypervisors:
412             if hypervisor.state == "up":
413                 nodes.append(hypervisor.hypervisor_hostname)
414         return nodes
415     except Exception as e:
416         logger.error("Error [get_hypervisors(nova_client)]: %s" % e)
417         return None
418
419
420 def create_aggregate(nova_client, aggregate_name, av_zone):
421     try:
422         nova_client.aggregates.create(aggregate_name, av_zone)
423         return True
424     except Exception as e:
425         logger.error("Error [create_aggregate(nova_client, %s, %s)]: %s"
426                      % (aggregate_name, av_zone, e))
427         return None
428
429
430 def add_host_to_aggregate(nova_client, aggregate_name, compute_host):
431     try:
432         aggregate_id = get_aggregate_id(nova_client, aggregate_name)
433         nova_client.aggregates.add_host(aggregate_id, compute_host)
434         return True
435     except Exception as e:
436         logger.error("Error [add_host_to_aggregate(nova_client, %s, %s)]: %s"
437                      % (aggregate_name, compute_host, e))
438         return None
439
440
441 def create_aggregate_with_host(
442         nova_client, aggregate_name, av_zone, compute_host):
443     try:
444         create_aggregate(nova_client, aggregate_name, av_zone)
445         add_host_to_aggregate(nova_client, aggregate_name, compute_host)
446         return True
447     except Exception as e:
448         logger.error("Error [create_aggregate_with_host("
449                      "nova_client, %s, %s, %s)]: %s"
450                      % (aggregate_name, av_zone, compute_host, e))
451         return None
452
453
454 def create_instance(flavor_name,
455                     image_id,
456                     network_id,
457                     instance_name="functest-vm",
458                     confdrive=True,
459                     userdata=None,
460                     av_zone='',
461                     fixed_ip=None,
462                     files=None):
463     nova_client = get_nova_client()
464     try:
465         flavor = nova_client.flavors.find(name=flavor_name)
466     except:
467         flavors = nova_client.flavors.list()
468         logger.error("Error: Flavor '%s' not found. Available flavors are: "
469                      "\n%s" % (flavor_name, flavors))
470         return None
471     if fixed_ip is not None:
472         nics = {"net-id": network_id, "v4-fixed-ip": fixed_ip}
473     else:
474         nics = {"net-id": network_id}
475     if userdata is None:
476         instance = nova_client.servers.create(
477             name=instance_name,
478             flavor=flavor,
479             image=image_id,
480             nics=[nics],
481             availability_zone=av_zone,
482             files=files
483         )
484     else:
485         instance = nova_client.servers.create(
486             name=instance_name,
487             flavor=flavor,
488             image=image_id,
489             nics=[nics],
490             config_drive=confdrive,
491             userdata=userdata,
492             availability_zone=av_zone,
493             files=files
494         )
495     return instance
496
497
498 def create_instance_and_wait_for_active(flavor_name,
499                                         image_id,
500                                         network_id,
501                                         instance_name="",
502                                         config_drive=False,
503                                         userdata="",
504                                         av_zone='',
505                                         fixed_ip=None,
506                                         files=None):
507     SLEEP = 3
508     VM_BOOT_TIMEOUT = 180
509     nova_client = get_nova_client()
510     instance = create_instance(flavor_name,
511                                image_id,
512                                network_id,
513                                instance_name,
514                                config_drive,
515                                userdata,
516                                av_zone=av_zone,
517                                fixed_ip=fixed_ip,
518                                files=files)
519     count = VM_BOOT_TIMEOUT / SLEEP
520     for n in range(count, -1, -1):
521         status = get_instance_status(nova_client, instance)
522         if status.lower() == "active":
523             return instance
524         elif status.lower() == "error":
525             logger.error("The instance %s went to ERROR status."
526                          % instance_name)
527             return None
528         time.sleep(SLEEP)
529     logger.error("Timeout booting the instance %s." % instance_name)
530     return None
531
532
533 def create_floating_ip(neutron_client):
534     extnet_id = get_external_net_id(neutron_client)
535     props = {'floating_network_id': extnet_id}
536     try:
537         ip_json = neutron_client.create_floatingip({'floatingip': props})
538         fip_addr = ip_json['floatingip']['floating_ip_address']
539         fip_id = ip_json['floatingip']['id']
540     except Exception as e:
541         logger.error("Error [create_floating_ip(neutron_client)]: %s" % e)
542         return None
543     return {'fip_addr': fip_addr, 'fip_id': fip_id}
544
545
546 def add_floating_ip(nova_client, server_id, floatingip_addr):
547     try:
548         nova_client.servers.add_floating_ip(server_id, floatingip_addr)
549         return True
550     except Exception as e:
551         logger.error("Error [add_floating_ip(nova_client, '%s', '%s')]: %s"
552                      % (server_id, floatingip_addr, e))
553         return False
554
555
556 def delete_instance(nova_client, instance_id):
557     try:
558         nova_client.servers.force_delete(instance_id)
559         return True
560     except Exception as e:
561         logger.error("Error [delete_instance(nova_client, '%s')]: %s"
562                      % (instance_id, e))
563         return False
564
565
566 def delete_floating_ip(neutron_client, floatingip_id):
567     try:
568         neutron_client.delete_floatingip(floatingip_id)
569         return True
570     except Exception as e:
571         logger.error("Error [delete_floating_ip(neutron_client, '%s')]: %s"
572                      % (floatingip_id, e))
573         return False
574
575
576 def remove_host_from_aggregate(nova_client, aggregate_name, compute_host):
577     try:
578         aggregate_id = get_aggregate_id(nova_client, aggregate_name)
579         nova_client.aggregates.remove_host(aggregate_id, compute_host)
580         return True
581     except Exception as e:
582         logger.error("Error [remove_host_from_aggregate(nova_client, %s, %s)]:"
583                      " %s" % (aggregate_name, compute_host, e))
584         return False
585
586
587 def remove_hosts_from_aggregate(nova_client, aggregate_name):
588     aggregate_id = get_aggregate_id(nova_client, aggregate_name)
589     hosts = nova_client.aggregates.get(aggregate_id).hosts
590     assert(
591         all(remove_host_from_aggregate(nova_client, aggregate_name, host)
592             for host in hosts))
593
594
595 def delete_aggregate(nova_client, aggregate_name):
596     try:
597         remove_hosts_from_aggregate(nova_client, aggregate_name)
598         nova_client.aggregates.delete(aggregate_name)
599         return True
600     except Exception as e:
601         logger.error("Error [delete_aggregate(nova_client, %s)]: %s"
602                      % (aggregate_name, e))
603         return False
604
605
606 # *********************************************
607 #   NEUTRON
608 # *********************************************
609 def get_network_list(neutron_client):
610     network_list = neutron_client.list_networks()['networks']
611     if len(network_list) == 0:
612         return None
613     else:
614         return network_list
615
616
617 def get_router_list(neutron_client):
618     router_list = neutron_client.list_routers()['routers']
619     if len(router_list) == 0:
620         return None
621     else:
622         return router_list
623
624
625 def get_port_list(neutron_client):
626     port_list = neutron_client.list_ports()['ports']
627     if len(port_list) == 0:
628         return None
629     else:
630         return port_list
631
632
633 def get_network_id(neutron_client, network_name):
634     networks = neutron_client.list_networks()['networks']
635     id = ''
636     for n in networks:
637         if n['name'] == network_name:
638             id = n['id']
639             break
640     return id
641
642
643 def get_subnet_id(neutron_client, subnet_name):
644     subnets = neutron_client.list_subnets()['subnets']
645     id = ''
646     for s in subnets:
647         if s['name'] == subnet_name:
648             id = s['id']
649             break
650     return id
651
652
653 def get_router_id(neutron_client, router_name):
654     routers = neutron_client.list_routers()['routers']
655     id = ''
656     for r in routers:
657         if r['name'] == router_name:
658             id = r['id']
659             break
660     return id
661
662
663 def get_private_net(neutron_client):
664     # Checks if there is an existing shared private network
665     networks = neutron_client.list_networks()['networks']
666     if len(networks) == 0:
667         return None
668     for net in networks:
669         if (net['router:external'] is False) and (net['shared'] is True):
670             return net
671     return None
672
673
674 def get_external_net(neutron_client):
675     if (hasattr(CONST, 'EXTERNAL_NETWORK')):
676         return CONST.__getattribute__('EXTERNAL_NETWORK')
677     for network in neutron_client.list_networks()['networks']:
678         if network['router:external']:
679             return network['name']
680     return None
681
682
683 def get_external_net_id(neutron_client):
684     if (hasattr(CONST, 'EXTERNAL_NETWORK')):
685         networks = neutron_client.list_networks(
686             name=CONST.__getattribute__('EXTERNAL_NETWORK'))
687         net_id = networks['networks'][0]['id']
688         return net_id
689     for network in neutron_client.list_networks()['networks']:
690         if network['router:external']:
691             return network['id']
692     return None
693
694
695 def check_neutron_net(neutron_client, net_name):
696     for network in neutron_client.list_networks()['networks']:
697         if network['name'] == net_name:
698             for subnet in network['subnets']:
699                 return True
700     return False
701
702
703 def create_neutron_net(neutron_client, name):
704     json_body = {'network': {'name': name,
705                              'admin_state_up': True}}
706     try:
707         network = neutron_client.create_network(body=json_body)
708         network_dict = network['network']
709         return network_dict['id']
710     except Exception as e:
711         logger.error("Error [create_neutron_net(neutron_client, '%s')]: %s"
712                      % (name, e))
713         return None
714
715
716 def create_neutron_subnet(neutron_client, name, cidr, net_id,
717                           dns=['8.8.8.8', '8.8.4.4']):
718     json_body = {'subnets': [{'name': name, 'cidr': cidr,
719                               'ip_version': 4, 'network_id': net_id,
720                               'dns_nameservers': dns}]}
721
722     try:
723         subnet = neutron_client.create_subnet(body=json_body)
724         return subnet['subnets'][0]['id']
725     except Exception as e:
726         logger.error("Error [create_neutron_subnet(neutron_client, '%s', "
727                      "'%s', '%s')]: %s" % (name, cidr, net_id, e))
728         return None
729
730
731 def create_neutron_router(neutron_client, name):
732     json_body = {'router': {'name': name, 'admin_state_up': True}}
733     try:
734         router = neutron_client.create_router(json_body)
735         return router['router']['id']
736     except Exception as e:
737         logger.error("Error [create_neutron_router(neutron_client, '%s')]: %s"
738                      % (name, e))
739         return None
740
741
742 def create_neutron_port(neutron_client, name, network_id, ip):
743     json_body = {'port': {
744                  'admin_state_up': True,
745                  'name': name,
746                  'network_id': network_id,
747                  'fixed_ips': [{"ip_address": ip}]
748                  }}
749     try:
750         port = neutron_client.create_port(body=json_body)
751         return port['port']['id']
752     except Exception as e:
753         logger.error("Error [create_neutron_port(neutron_client, '%s', '%s', "
754                      "'%s')]: %s" % (name, network_id, ip, e))
755         return None
756
757
758 def update_neutron_net(neutron_client, network_id, shared=False):
759     json_body = {'network': {'shared': shared}}
760     try:
761         neutron_client.update_network(network_id, body=json_body)
762         return True
763     except Exception as e:
764         logger.error("Error [update_neutron_net(neutron_client, '%s', '%s')]: "
765                      "%s" % (network_id, str(shared), e))
766         return False
767
768
769 def update_neutron_port(neutron_client, port_id, device_owner):
770     json_body = {'port': {
771                  'device_owner': device_owner,
772                  }}
773     try:
774         port = neutron_client.update_port(port=port_id,
775                                           body=json_body)
776         return port['port']['id']
777     except Exception as e:
778         logger.error("Error [update_neutron_port(neutron_client, '%s', '%s')]:"
779                      " %s" % (port_id, device_owner, e))
780         return None
781
782
783 def add_interface_router(neutron_client, router_id, subnet_id):
784     json_body = {"subnet_id": subnet_id}
785     try:
786         neutron_client.add_interface_router(router=router_id, body=json_body)
787         return True
788     except Exception as e:
789         logger.error("Error [add_interface_router(neutron_client, '%s', "
790                      "'%s')]: %s" % (router_id, subnet_id, e))
791         return False
792
793
794 def add_gateway_router(neutron_client, router_id):
795     ext_net_id = get_external_net_id(neutron_client)
796     router_dict = {'network_id': ext_net_id}
797     try:
798         neutron_client.add_gateway_router(router_id, router_dict)
799         return True
800     except Exception as e:
801         logger.error("Error [add_gateway_router(neutron_client, '%s')]: %s"
802                      % (router_id, e))
803         return False
804
805
806 def delete_neutron_net(neutron_client, network_id):
807     try:
808         neutron_client.delete_network(network_id)
809         return True
810     except Exception as e:
811         logger.error("Error [delete_neutron_net(neutron_client, '%s')]: %s"
812                      % (network_id, e))
813         return False
814
815
816 def delete_neutron_subnet(neutron_client, subnet_id):
817     try:
818         neutron_client.delete_subnet(subnet_id)
819         return True
820     except Exception as e:
821         logger.error("Error [delete_neutron_subnet(neutron_client, '%s')]: %s"
822                      % (subnet_id, e))
823         return False
824
825
826 def delete_neutron_router(neutron_client, router_id):
827     try:
828         neutron_client.delete_router(router=router_id)
829         return True
830     except Exception as e:
831         logger.error("Error [delete_neutron_router(neutron_client, '%s')]: %s"
832                      % (router_id, e))
833         return False
834
835
836 def delete_neutron_port(neutron_client, port_id):
837     try:
838         neutron_client.delete_port(port_id)
839         return True
840     except Exception as e:
841         logger.error("Error [delete_neutron_port(neutron_client, '%s')]: %s"
842                      % (port_id, e))
843         return False
844
845
846 def remove_interface_router(neutron_client, router_id, subnet_id):
847     json_body = {"subnet_id": subnet_id}
848     try:
849         neutron_client.remove_interface_router(router=router_id,
850                                                body=json_body)
851         return True
852     except Exception as e:
853         logger.error("Error [remove_interface_router(neutron_client, '%s', "
854                      "'%s')]: %s" % (router_id, subnet_id, e))
855         return False
856
857
858 def remove_gateway_router(neutron_client, router_id):
859     try:
860         neutron_client.remove_gateway_router(router_id)
861         return True
862     except Exception as e:
863         logger.error("Error [remove_gateway_router(neutron_client, '%s')]: %s"
864                      % (router_id, e))
865         return False
866
867
868 def create_network_full(neutron_client,
869                         net_name,
870                         subnet_name,
871                         router_name,
872                         cidr,
873                         dns=['8.8.8.8', '8.8.4.4']):
874
875     # Check if the network already exists
876     network_id = get_network_id(neutron_client, net_name)
877     subnet_id = get_subnet_id(neutron_client, subnet_name)
878     router_id = get_router_id(neutron_client, router_name)
879
880     if network_id != '' and subnet_id != '' and router_id != '':
881         logger.info("A network with name '%s' already exists..." % net_name)
882     else:
883         neutron_client.format = 'json'
884         logger.info('Creating neutron network %s...' % net_name)
885         network_id = create_neutron_net(neutron_client, net_name)
886
887         if not network_id:
888             return False
889
890         logger.debug("Network '%s' created successfully" % network_id)
891         logger.debug('Creating Subnet....')
892         subnet_id = create_neutron_subnet(neutron_client, subnet_name,
893                                           cidr, network_id, dns)
894         if not subnet_id:
895             return None
896
897         logger.debug("Subnet '%s' created successfully" % subnet_id)
898         logger.debug('Creating Router...')
899         router_id = create_neutron_router(neutron_client, router_name)
900
901         if not router_id:
902             return None
903
904         logger.debug("Router '%s' created successfully" % router_id)
905         logger.debug('Adding router to subnet...')
906
907         if not add_interface_router(neutron_client, router_id, subnet_id):
908             return None
909
910         logger.debug("Interface added successfully.")
911
912         logger.debug('Adding gateway to router...')
913         if not add_gateway_router(neutron_client, router_id):
914             return None
915
916         logger.debug("Gateway added successfully.")
917
918     network_dic = {'net_id': network_id,
919                    'subnet_id': subnet_id,
920                    'router_id': router_id}
921     return network_dic
922
923
924 def create_shared_network_full(net_name, subnt_name, router_name, subnet_cidr):
925     neutron_client = get_neutron_client()
926
927     network_dic = create_network_full(neutron_client,
928                                       net_name,
929                                       subnt_name,
930                                       router_name,
931                                       subnet_cidr)
932     if network_dic:
933         if not update_neutron_net(neutron_client,
934                                   network_dic['net_id'],
935                                   shared=True):
936             logger.error("Failed to update network %s..." % net_name)
937             return None
938         else:
939             logger.debug("Network '%s' is available..." % net_name)
940     else:
941         logger.error("Network %s creation failed" % net_name)
942         return None
943     return network_dic
944
945
946 # *********************************************
947 #   SEC GROUPS
948 # *********************************************
949
950
951 def get_security_groups(neutron_client):
952     try:
953         security_groups = neutron_client.list_security_groups()[
954             'security_groups']
955         return security_groups
956     except Exception as e:
957         logger.error("Error [get_security_groups(neutron_client)]: %s" % e)
958         return None
959
960
961 def get_security_group_id(neutron_client, sg_name):
962     security_groups = get_security_groups(neutron_client)
963     id = ''
964     for sg in security_groups:
965         if sg['name'] == sg_name:
966             id = sg['id']
967             break
968     return id
969
970
971 def create_security_group(neutron_client, sg_name, sg_description):
972     json_body = {'security_group': {'name': sg_name,
973                                     'description': sg_description}}
974     try:
975         secgroup = neutron_client.create_security_group(json_body)
976         return secgroup['security_group']
977     except Exception as e:
978         logger.error("Error [create_security_group(neutron_client, '%s', "
979                      "'%s')]: %s" % (sg_name, sg_description, e))
980         return None
981
982
983 def create_secgroup_rule(neutron_client, sg_id, direction, protocol,
984                          port_range_min=None, port_range_max=None):
985     # We create a security group in 2 steps
986     # 1 - we check the format and set the json body accordingly
987     # 2 - we call neturon client to create the security group
988
989     # Format check
990     json_body = {'security_group_rule': {'direction': direction,
991                                          'security_group_id': sg_id,
992                                          'protocol': protocol}}
993     # parameters may be
994     # - both None => we do nothing
995     # - both Not None => we add them to the json description
996     # but one cannot be None is the other is not None
997     if (port_range_min is not None and port_range_max is not None):
998         # add port_range in json description
999         json_body['security_group_rule']['port_range_min'] = port_range_min
1000         json_body['security_group_rule']['port_range_max'] = port_range_max
1001         logger.debug("Security_group format set (port range included)")
1002     else:
1003         # either both port range are set to None => do nothing
1004         # or one is set but not the other => log it and return False
1005         if port_range_min is None and port_range_max is None:
1006             logger.debug("Security_group format set (no port range mentioned)")
1007         else:
1008             logger.error("Bad security group format."
1009                          "One of the port range is not properly set:"
1010                          "range min: {},"
1011                          "range max: {}".format(port_range_min,
1012                                                 port_range_max))
1013             return False
1014
1015     # Create security group using neutron client
1016     try:
1017         neutron_client.create_security_group_rule(json_body)
1018         return True
1019     except:
1020         logger.exception("Impossible to create_security_group_rule,"
1021                          "security group rule probably already exists")
1022         return False
1023
1024
1025 def get_security_group_rules(neutron_client, sg_id):
1026     try:
1027         security_rules = neutron_client.list_security_group_rules()[
1028             'security_group_rules']
1029         security_rules = [rule for rule in security_rules
1030                           if rule["security_group_id"] == sg_id]
1031         return security_rules
1032     except Exception as e:
1033         logger.error("Error [get_security_group_rules(neutron_client, sg_id)]:"
1034                      " %s" % e)
1035         return None
1036
1037
1038 def check_security_group_rules(neutron_client, sg_id, direction, protocol,
1039                                port_min=None, port_max=None):
1040     try:
1041         security_rules = get_security_group_rules(neutron_client, sg_id)
1042         security_rules = [rule for rule in security_rules
1043                           if (rule["direction"].lower() == direction and
1044                               rule["protocol"].lower() == protocol and
1045                               rule["port_range_min"] == port_min and
1046                               rule["port_range_max"] == port_max)]
1047         if len(security_rules) == 0:
1048             return True
1049         else:
1050             return False
1051     except Exception as e:
1052         logger.error("Error [check_security_group_rules("
1053                      " neutron_client, sg_id, direction,"
1054                      " protocol, port_min=None, port_max=None)]: "
1055                      "%s" % e)
1056         return None
1057
1058
1059 def create_security_group_full(neutron_client,
1060                                sg_name, sg_description):
1061     sg_id = get_security_group_id(neutron_client, sg_name)
1062     if sg_id != '':
1063         logger.info("Using existing security group '%s'..." % sg_name)
1064     else:
1065         logger.info("Creating security group  '%s'..." % sg_name)
1066         SECGROUP = create_security_group(neutron_client,
1067                                          sg_name,
1068                                          sg_description)
1069         if not SECGROUP:
1070             logger.error("Failed to create the security group...")
1071             return None
1072
1073         sg_id = SECGROUP['id']
1074
1075         logger.debug("Security group '%s' with ID=%s created successfully."
1076                      % (SECGROUP['name'], sg_id))
1077
1078         logger.debug("Adding ICMP rules in security group '%s'..."
1079                      % sg_name)
1080         if not create_secgroup_rule(neutron_client, sg_id,
1081                                     'ingress', 'icmp'):
1082             logger.error("Failed to create the security group rule...")
1083             return None
1084
1085         logger.debug("Adding SSH rules in security group '%s'..."
1086                      % sg_name)
1087         if not create_secgroup_rule(
1088                 neutron_client, sg_id, 'ingress', 'tcp', '22', '22'):
1089             logger.error("Failed to create the security group rule...")
1090             return None
1091
1092         if not create_secgroup_rule(
1093                 neutron_client, sg_id, 'egress', 'tcp', '22', '22'):
1094             logger.error("Failed to create the security group rule...")
1095             return None
1096     return sg_id
1097
1098
1099 def add_secgroup_to_instance(nova_client, instance_id, secgroup_id):
1100     try:
1101         nova_client.servers.add_security_group(instance_id, secgroup_id)
1102         return True
1103     except Exception as e:
1104         logger.error("Error [add_secgroup_to_instance(nova_client, '%s', "
1105                      "'%s')]: %s" % (instance_id, secgroup_id, e))
1106         return False
1107
1108
1109 def update_sg_quota(neutron_client, tenant_id, sg_quota, sg_rule_quota):
1110     json_body = {"quota": {
1111         "security_group": sg_quota,
1112         "security_group_rule": sg_rule_quota
1113     }}
1114
1115     try:
1116         neutron_client.update_quota(tenant_id=tenant_id,
1117                                     body=json_body)
1118         return True
1119     except Exception as e:
1120         logger.error("Error [update_sg_quota(neutron_client, '%s', '%s', "
1121                      "'%s')]: %s" % (tenant_id, sg_quota, sg_rule_quota, e))
1122         return False
1123
1124
1125 def delete_security_group(neutron_client, secgroup_id):
1126     try:
1127         neutron_client.delete_security_group(secgroup_id)
1128         return True
1129     except Exception as e:
1130         logger.error("Error [delete_security_group(neutron_client, '%s')]: %s"
1131                      % (secgroup_id, e))
1132         return False
1133
1134
1135 # *********************************************
1136 #   GLANCE
1137 # *********************************************
1138 def get_images(glance_client):
1139     try:
1140         images = glance_client.images.list()
1141         return images
1142     except Exception as e:
1143         logger.error("Error [get_images]: %s" % e)
1144         return None
1145
1146
1147 def get_image_id(glance_client, image_name):
1148     images = glance_client.images.list()
1149     id = ''
1150     for i in images:
1151         if i.name == image_name:
1152             id = i.id
1153             break
1154     return id
1155
1156
1157 def create_glance_image(glance_client,
1158                         image_name,
1159                         file_path,
1160                         disk="qcow2",
1161                         extra_properties={},
1162                         container="bare",
1163                         public="public"):
1164     if not os.path.isfile(file_path):
1165         logger.error("Error: file %s does not exist." % file_path)
1166         return None
1167     try:
1168         image_id = get_image_id(glance_client, image_name)
1169         if image_id != '':
1170             logger.info("Image %s already exists." % image_name)
1171         else:
1172             logger.info("Creating image '%s' from '%s'..." % (image_name,
1173                                                               file_path))
1174
1175             image = glance_client.images.create(name=image_name,
1176                                                 visibility=public,
1177                                                 disk_format=disk,
1178                                                 container_format=container,
1179                                                 **extra_properties)
1180             image_id = image.id
1181             with open(file_path) as image_data:
1182                 glance_client.images.upload(image_id, image_data)
1183         return image_id
1184     except Exception as e:
1185         logger.error("Error [create_glance_image(glance_client, '%s', '%s', "
1186                      "'%s')]: %s" % (image_name, file_path, public, e))
1187         return None
1188
1189
1190 def get_or_create_image(name, path, format, extra_properties):
1191     image_exists = False
1192     glance_client = get_glance_client()
1193
1194     image_id = get_image_id(glance_client, name)
1195     if image_id != '':
1196         logger.info("Using existing image '%s'..." % name)
1197         image_exists = True
1198     else:
1199         logger.info("Creating image '%s' from '%s'..." % (name, path))
1200         image_id = create_glance_image(glance_client,
1201                                        name,
1202                                        path,
1203                                        format,
1204                                        extra_properties)
1205         if not image_id:
1206             logger.error("Failed to create a Glance image...")
1207         else:
1208             logger.debug("Image '%s' with ID=%s created successfully."
1209                          % (name, image_id))
1210
1211     return image_exists, image_id
1212
1213
1214 def delete_glance_image(glance_client, image_id):
1215     try:
1216         glance_client.images.delete(image_id)
1217         return True
1218     except Exception as e:
1219         logger.error("Error [delete_glance_image(glance_client, '%s')]: %s"
1220                      % (image_id, e))
1221         return False
1222
1223
1224 # *********************************************
1225 #   CINDER
1226 # *********************************************
1227 def get_volumes(cinder_client):
1228     try:
1229         volumes = cinder_client.volumes.list(search_opts={'all_tenants': 1})
1230         return volumes
1231     except Exception as e:
1232         logger.error("Error [get_volumes(cinder_client)]: %s" % e)
1233         return None
1234
1235
1236 def update_cinder_quota(cinder_client, tenant_id, vols_quota,
1237                         snapshots_quota, gigabytes_quota):
1238     quotas_values = {"volumes": vols_quota,
1239                      "snapshots": snapshots_quota,
1240                      "gigabytes": gigabytes_quota}
1241
1242     try:
1243         cinder_client.quotas.update(tenant_id, **quotas_values)
1244         return True
1245     except Exception as e:
1246         logger.error("Error [update_cinder_quota(cinder_client, '%s', '%s', "
1247                      "'%s' '%s')]: %s" % (tenant_id, vols_quota,
1248                                           snapshots_quota, gigabytes_quota, e))
1249         return False
1250
1251
1252 def delete_volume(cinder_client, volume_id, forced=False):
1253     try:
1254         if forced:
1255             try:
1256                 cinder_client.volumes.detach(volume_id)
1257             except:
1258                 logger.error(sys.exc_info()[0])
1259             cinder_client.volumes.force_delete(volume_id)
1260         else:
1261             cinder_client.volumes.delete(volume_id)
1262         return True
1263     except Exception as e:
1264         logger.error("Error [delete_volume(cinder_client, '%s', '%s')]: %s"
1265                      % (volume_id, str(forced), e))
1266         return False
1267
1268
1269 # *********************************************
1270 #   KEYSTONE
1271 # *********************************************
1272 def get_tenants(keystone_client):
1273     try:
1274         if is_keystone_v3():
1275             tenants = keystone_client.projects.list()
1276         else:
1277             tenants = keystone_client.tenants.list()
1278         return tenants
1279     except Exception as e:
1280         logger.error("Error [get_tenants(keystone_client)]: %s" % e)
1281         return None
1282
1283
1284 def get_users(keystone_client):
1285     try:
1286         users = keystone_client.users.list()
1287         return users
1288     except Exception as e:
1289         logger.error("Error [get_users(keystone_client)]: %s" % e)
1290         return None
1291
1292
1293 def get_tenant_id(keystone_client, tenant_name):
1294     tenants = get_tenants(keystone_client)
1295     id = ''
1296     for t in tenants:
1297         if t.name == tenant_name:
1298             id = t.id
1299             break
1300     return id
1301
1302
1303 def get_user_id(keystone_client, user_name):
1304     users = get_users(keystone_client)
1305     id = ''
1306     for u in users:
1307         if u.name == user_name:
1308             id = u.id
1309             break
1310     return id
1311
1312
1313 def get_role_id(keystone_client, role_name):
1314     roles = keystone_client.roles.list()
1315     id = ''
1316     for r in roles:
1317         if r.name == role_name:
1318             id = r.id
1319             break
1320     return id
1321
1322
1323 def get_domain_id(keystone_client, domain_name):
1324     domains = keystone_client.domains.list()
1325     id = ''
1326     for d in domains:
1327         if d.name == domain_name:
1328             id = d.id
1329             break
1330     return id
1331
1332
1333 def create_tenant(keystone_client, tenant_name, tenant_description):
1334     try:
1335         if is_keystone_v3():
1336             domain_name = CONST.__getattribute__('OS_PROJECT_DOMAIN_NAME')
1337             domain_id = get_domain_id(keystone_client, domain_name)
1338             tenant = keystone_client.projects.create(
1339                 name=tenant_name,
1340                 description=tenant_description,
1341                 domain=domain_id,
1342                 enabled=True)
1343         else:
1344             tenant = keystone_client.tenants.create(tenant_name,
1345                                                     tenant_description,
1346                                                     enabled=True)
1347         return tenant.id
1348     except Exception as e:
1349         logger.error("Error [create_tenant(keystone_client, '%s', '%s')]: %s"
1350                      % (tenant_name, tenant_description, e))
1351         return None
1352
1353
1354 def get_or_create_tenant(keystone_client, tenant_name, tenant_description):
1355     tenant_id = get_tenant_id(keystone_client, tenant_name)
1356     if not tenant_id:
1357         tenant_id = create_tenant(keystone_client, tenant_name,
1358                                   tenant_description)
1359
1360     return tenant_id
1361
1362
1363 def get_or_create_tenant_for_vnf(keystone_client, tenant_name,
1364                                  tenant_description):
1365     """Get or Create a Tenant
1366
1367         Args:
1368             keystone_client: keystone client reference
1369             tenant_name: the name of the tenant
1370             tenant_description: the description of the tenant
1371
1372         return False if tenant retrieved though get
1373         return True if tenant created
1374         raise Exception if error during processing
1375     """
1376     try:
1377         tenant_id = get_tenant_id(keystone_client, tenant_name)
1378         if not tenant_id:
1379             tenant_id = create_tenant(keystone_client, tenant_name,
1380                                       tenant_description)
1381             return True
1382         else:
1383             return False
1384     except:
1385         raise Exception("Impossible to create a Tenant for the VNF {}".format(
1386                             tenant_name))
1387
1388
1389 def create_user(keystone_client, user_name, user_password,
1390                 user_email, tenant_id):
1391     try:
1392         if is_keystone_v3():
1393             user = keystone_client.users.create(name=user_name,
1394                                                 password=user_password,
1395                                                 email=user_email,
1396                                                 project_id=tenant_id,
1397                                                 enabled=True)
1398         else:
1399             user = keystone_client.users.create(user_name,
1400                                                 user_password,
1401                                                 user_email,
1402                                                 tenant_id,
1403                                                 enabled=True)
1404         return user.id
1405     except Exception as e:
1406         logger.error("Error [create_user(keystone_client, '%s', '%s', '%s'"
1407                      "'%s')]: %s" % (user_name, user_password,
1408                                      user_email, tenant_id, e))
1409         return None
1410
1411
1412 def get_or_create_user(keystone_client, user_name, user_password,
1413                        tenant_id, user_email=None):
1414     user_id = get_user_id(keystone_client, user_name)
1415     if not user_id:
1416         user_id = create_user(keystone_client, user_name, user_password,
1417                               user_email, tenant_id)
1418     return user_id
1419
1420
1421 def get_or_create_user_for_vnf(keystone_client, vnf_ref):
1422     """Get or Create user for VNF
1423
1424         Args:
1425             keystone_client: keystone client reference
1426             vnf_ref: VNF reference used as user name & password, tenant name
1427
1428         return False if user retrieved through get
1429         return True if user created
1430         raise Exception if error during processing
1431     """
1432     try:
1433         user_id = get_user_id(keystone_client, vnf_ref)
1434         tenant_id = get_tenant_id(keystone_client, vnf_ref)
1435         created = False
1436         if not user_id:
1437             user_id = create_user(keystone_client, vnf_ref, vnf_ref,
1438                                   "", tenant_id)
1439             created = True
1440         try:
1441             role_id = get_role_id(keystone_client, 'admin')
1442             tenant_id = get_tenant_id(keystone_client, vnf_ref)
1443             add_role_user(keystone_client, user_id, role_id, tenant_id)
1444         except:
1445             logger.warn("Cannot associate user to role admin on tenant")
1446         return created
1447     except:
1448         raise Exception("Impossible to create a user for the VNF {}".format(
1449             vnf_ref))
1450
1451
1452 def add_role_user(keystone_client, user_id, role_id, tenant_id):
1453     try:
1454         if is_keystone_v3():
1455             keystone_client.roles.grant(role=role_id,
1456                                         user=user_id,
1457                                         project=tenant_id)
1458         else:
1459             keystone_client.roles.add_user_role(user_id, role_id, tenant_id)
1460         return True
1461     except Exception as e:
1462         logger.error("Error [add_role_user(keystone_client, '%s', '%s'"
1463                      "'%s')]: %s " % (user_id, role_id, tenant_id, e))
1464         return False
1465
1466
1467 def delete_tenant(keystone_client, tenant_id):
1468     try:
1469         if is_keystone_v3():
1470             keystone_client.projects.delete(tenant_id)
1471         else:
1472             keystone_client.tenants.delete(tenant_id)
1473         return True
1474     except Exception as e:
1475         logger.error("Error [delete_tenant(keystone_client, '%s')]: %s"
1476                      % (tenant_id, e))
1477         return False
1478
1479
1480 def delete_user(keystone_client, user_id):
1481     try:
1482         keystone_client.users.delete(user_id)
1483         return True
1484     except Exception as e:
1485         logger.error("Error [delete_user(keystone_client, '%s')]: %s"
1486                      % (user_id, e))
1487         return False
1488
1489
1490 # *********************************************
1491 #   HEAT
1492 # *********************************************
1493 def get_resource(heat_client, stack_id, resource):
1494     try:
1495         resources = heat_client.resources.get(stack_id, resource)
1496         return resources
1497     except Exception as e:
1498         logger.error("Error [get_resource]: %s" % e)
1499         return None