Merge "Remove openstack utils calls in rally and tempest"
[functest.git] / functest / utils / openstack_utils.py
1 #!/usr/bin/env python
2 #
3 # jose.lausuch@ericsson.com
4 # valentin.boucher@orange.com
5 # All rights reserved. This program and the accompanying materials
6 # are made available under the terms of the Apache License, Version 2.0
7 # which accompanies this distribution, and is available at
8 # http://www.apache.org/licenses/LICENSE-2.0
9 #
10
11 import logging
12 import os.path
13 import re
14 import sys
15 import time
16
17 from keystoneauth1 import loading
18 from keystoneauth1 import session
19 from cinderclient import client as cinderclient
20 from glanceclient import client as glanceclient
21 from heatclient import client as heatclient
22 from novaclient import client as novaclient
23 from keystoneclient import client as keystoneclient
24 from neutronclient.neutron import client as neutronclient
25
26 from functest.utils.constants import CONST
27 import functest.utils.functest_utils as ft_utils
28
29 logger = logging.getLogger(__name__)
30
31 DEFAULT_API_VERSION = '2'
32 DEFAULT_HEAT_API_VERSION = '1'
33
34
35 # *********************************************
36 #   CREDENTIALS
37 # *********************************************
38 class MissingEnvVar(Exception):
39
40     def __init__(self, var):
41         self.var = var
42
43     def __str__(self):
44         return str.format("Please set the mandatory env var: {}", self.var)
45
46
47 def is_keystone_v3():
48     keystone_api_version = os.getenv('OS_IDENTITY_API_VERSION')
49     if (keystone_api_version is None or
50             keystone_api_version == '2'):
51         return False
52     else:
53         return True
54
55
56 def get_rc_env_vars():
57     env_vars = ['OS_AUTH_URL', 'OS_USERNAME', 'OS_PASSWORD']
58     if is_keystone_v3():
59         env_vars.extend(['OS_PROJECT_NAME',
60                          'OS_USER_DOMAIN_NAME',
61                          'OS_PROJECT_DOMAIN_NAME'])
62     else:
63         env_vars.extend(['OS_TENANT_NAME'])
64     return env_vars
65
66
67 def check_credentials():
68     """
69     Check if the OpenStack credentials (openrc) are sourced
70     """
71     env_vars = get_rc_env_vars()
72     return all(map(lambda v: v in os.environ and os.environ[v], env_vars))
73
74
75 def get_env_cred_dict():
76     env_cred_dict = {
77         'OS_USERNAME': 'username',
78         'OS_PASSWORD': 'password',
79         'OS_AUTH_URL': 'auth_url',
80         'OS_TENANT_NAME': 'tenant_name',
81         'OS_USER_DOMAIN_NAME': 'user_domain_name',
82         'OS_PROJECT_DOMAIN_NAME': 'project_domain_name',
83         'OS_PROJECT_NAME': 'project_name',
84         'OS_ENDPOINT_TYPE': 'endpoint_type',
85         'OS_REGION_NAME': 'region_name',
86         'OS_CACERT': 'https_cacert',
87         'OS_INSECURE': 'https_insecure'
88     }
89     return env_cred_dict
90
91
92 def get_credentials(other_creds={}):
93     """Returns a creds dictionary filled with parsed from env
94     """
95     creds = {}
96     env_vars = get_rc_env_vars()
97     env_cred_dict = get_env_cred_dict()
98
99     for envvar in env_vars:
100         if os.getenv(envvar) is None:
101             raise MissingEnvVar(envvar)
102         else:
103             creds_key = env_cred_dict.get(envvar)
104             creds.update({creds_key: os.getenv(envvar)})
105
106     if 'tenant' in other_creds.keys():
107         if is_keystone_v3():
108             tenant = 'project_name'
109         else:
110             tenant = 'tenant_name'
111         other_creds[tenant] = other_creds.pop('tenant')
112
113     creds.update(other_creds)
114
115     return creds
116
117
118 def source_credentials(rc_file):
119     with open(rc_file, "r") as f:
120         for line in f:
121             var = (line.rstrip('"\n').replace('export ', '').split("=")
122                    if re.search(r'(.*)=(.*)', line) else None)
123             # The two next lines should be modified as soon as rc_file
124             # conforms with common rules. Be aware that it could induce
125             # issues if value starts with '
126             if var:
127                 key = re.sub(r'^["\' ]*|[ \'"]*$', '', var[0])
128                 value = re.sub(r'^["\' ]*|[ \'"]*$', '', "".join(var[1:]))
129                 os.environ[key] = value
130
131
132 def get_session_auth(other_creds={}):
133     loader = loading.get_plugin_loader('password')
134     creds = get_credentials(other_creds)
135     auth = loader.load_from_options(**creds)
136     return auth
137
138
139 def get_endpoint(service_type, interface='public'):
140     auth = get_session_auth()
141     return get_session().get_endpoint(auth=auth,
142                                       service_type=service_type,
143                                       interface=interface)
144
145
146 def get_session(other_creds={}):
147     auth = get_session_auth(other_creds)
148     https_cacert = os.getenv('OS_CACERT', '')
149     https_insecure = os.getenv('OS_INSECURE', '').lower() == 'true'
150     return session.Session(auth=auth,
151                            verify=(https_cacert or not https_insecure))
152
153
154 # *********************************************
155 #   CLIENTS
156 # *********************************************
157 def get_keystone_client_version():
158     api_version = os.getenv('OS_IDENTITY_API_VERSION')
159     if api_version is not None:
160         logger.info("OS_IDENTITY_API_VERSION is set in env as '%s'",
161                     api_version)
162         return api_version
163     return DEFAULT_API_VERSION
164
165
166 def get_keystone_client(other_creds={}):
167     sess = get_session(other_creds)
168     return keystoneclient.Client(get_keystone_client_version(),
169                                  session=sess,
170                                  interface=os.getenv('OS_INTERFACE', 'admin'))
171
172
173 def get_nova_client_version():
174     api_version = os.getenv('OS_COMPUTE_API_VERSION')
175     if api_version is not None:
176         logger.info("OS_COMPUTE_API_VERSION is set in env as '%s'",
177                     api_version)
178         return api_version
179     return DEFAULT_API_VERSION
180
181
182 def get_nova_client(other_creds={}):
183     sess = get_session(other_creds)
184     return novaclient.Client(get_nova_client_version(), session=sess)
185
186
187 def get_cinder_client_version():
188     api_version = os.getenv('OS_VOLUME_API_VERSION')
189     if api_version is not None:
190         logger.info("OS_VOLUME_API_VERSION is set in env as '%s'",
191                     api_version)
192         return api_version
193     return DEFAULT_API_VERSION
194
195
196 def get_cinder_client(other_creds={}):
197     sess = get_session(other_creds)
198     return cinderclient.Client(get_cinder_client_version(), session=sess)
199
200
201 def get_neutron_client_version():
202     api_version = os.getenv('OS_NETWORK_API_VERSION')
203     if api_version is not None:
204         logger.info("OS_NETWORK_API_VERSION is set in env as '%s'",
205                     api_version)
206         return api_version
207     return DEFAULT_API_VERSION
208
209
210 def get_neutron_client(other_creds={}):
211     sess = get_session(other_creds)
212     return neutronclient.Client(get_neutron_client_version(), session=sess)
213
214
215 def get_glance_client_version():
216     api_version = os.getenv('OS_IMAGE_API_VERSION')
217     if api_version is not None:
218         logger.info("OS_IMAGE_API_VERSION is set in env as '%s'", api_version)
219         return api_version
220     return DEFAULT_API_VERSION
221
222
223 def get_glance_client(other_creds={}):
224     sess = get_session(other_creds)
225     return glanceclient.Client(get_glance_client_version(), session=sess)
226
227
228 def get_heat_client_version():
229     api_version = os.getenv('OS_ORCHESTRATION_API_VERSION')
230     if api_version is not None:
231         logger.info("OS_ORCHESTRATION_API_VERSION is set in env as '%s'",
232                     api_version)
233         return api_version
234     return DEFAULT_HEAT_API_VERSION
235
236
237 def get_heat_client(other_creds={}):
238     sess = get_session(other_creds)
239     return heatclient.Client(get_heat_client_version(), session=sess)
240
241
242 def download_and_add_image_on_glance(glance, image_name, image_url, data_dir):
243     try:
244         dest_path = data_dir
245         if not os.path.exists(dest_path):
246             os.makedirs(dest_path)
247         file_name = image_url.rsplit('/')[-1]
248         if not ft_utils.download_url(image_url, dest_path):
249             return False
250     except Exception:
251         raise Exception("Impossible to download image from {}".format(
252                         image_url))
253
254     try:
255         image = create_glance_image(
256             glance, image_name, dest_path + file_name)
257         if not image:
258             return False
259         else:
260             return image
261     except Exception:
262         raise Exception("Impossible to put image {} in glance".format(
263                         image_name))
264
265
266 # *********************************************
267 #   NOVA
268 # *********************************************
269 def get_instances(nova_client):
270     try:
271         instances = nova_client.servers.list(search_opts={'all_tenants': 1})
272         return instances
273     except Exception as e:
274         logger.error("Error [get_instances(nova_client)]: %s" % e)
275         return None
276
277
278 def get_instance_status(nova_client, instance):
279     try:
280         instance = nova_client.servers.get(instance.id)
281         return instance.status
282     except Exception as e:
283         logger.error("Error [get_instance_status(nova_client)]: %s" % e)
284         return None
285
286
287 def get_instance_by_name(nova_client, instance_name):
288     try:
289         instance = nova_client.servers.find(name=instance_name)
290         return instance
291     except Exception as e:
292         logger.error("Error [get_instance_by_name(nova_client, '%s')]: %s"
293                      % (instance_name, e))
294         return None
295
296
297 def get_flavor_id(nova_client, flavor_name):
298     flavors = nova_client.flavors.list(detailed=True)
299     id = ''
300     for f in flavors:
301         if f.name == flavor_name:
302             id = f.id
303             break
304     return id
305
306
307 def get_flavor_id_by_ram_range(nova_client, min_ram, max_ram):
308     flavors = nova_client.flavors.list(detailed=True)
309     id = ''
310     for f in flavors:
311         if min_ram <= f.ram and f.ram <= max_ram:
312             id = f.id
313             break
314     return id
315
316
317 def get_aggregates(nova_client):
318     try:
319         aggregates = nova_client.aggregates.list()
320         return aggregates
321     except Exception as e:
322         logger.error("Error [get_aggregates(nova_client)]: %s" % e)
323         return None
324
325
326 def get_aggregate_id(nova_client, aggregate_name):
327     try:
328         aggregates = get_aggregates(nova_client)
329         _id = [ag.id for ag in aggregates if ag.name == aggregate_name][0]
330         return _id
331     except Exception as e:
332         logger.error("Error [get_aggregate_id(nova_client, %s)]:"
333                      " %s" % (aggregate_name, e))
334         return None
335
336
337 def get_availability_zones(nova_client):
338     try:
339         availability_zones = nova_client.availability_zones.list()
340         return availability_zones
341     except Exception as e:
342         logger.error("Error [get_availability_zones(nova_client)]: %s" % e)
343         return None
344
345
346 def get_availability_zone_names(nova_client):
347     try:
348         az_names = [az.zoneName for az in get_availability_zones(nova_client)]
349         return az_names
350     except Exception as e:
351         logger.error("Error [get_availability_zone_names(nova_client)]:"
352                      " %s" % e)
353         return None
354
355
356 def create_flavor(nova_client, flavor_name, ram, disk, vcpus, public=True):
357     try:
358         flavor = nova_client.flavors.create(
359             flavor_name, ram, vcpus, disk, is_public=public)
360         try:
361             extra_specs = ft_utils.get_functest_config(
362                 'general.flavor_extra_specs')
363             flavor.set_keys(extra_specs)
364         except ValueError:
365             # flavor extra specs are not configured, therefore skip the update
366             pass
367
368     except Exception as e:
369         logger.error("Error [create_flavor(nova_client, '%s', '%s', '%s', "
370                      "'%s')]: %s" % (flavor_name, ram, disk, vcpus, e))
371         return None
372     return flavor.id
373
374
375 def get_or_create_flavor(flavor_name, ram, disk, vcpus, public=True):
376     flavor_exists = False
377     nova_client = get_nova_client()
378
379     flavor_id = get_flavor_id(nova_client, flavor_name)
380     if flavor_id != '':
381         logger.info("Using existing flavor '%s'..." % flavor_name)
382         flavor_exists = True
383     else:
384         logger.info("Creating flavor '%s' with '%s' RAM, '%s' disk size, "
385                     "'%s' vcpus..." % (flavor_name, ram, disk, vcpus))
386         flavor_id = create_flavor(
387             nova_client, flavor_name, ram, disk, vcpus, public=public)
388         if not flavor_id:
389             raise Exception("Failed to create flavor '%s'..." % (flavor_name))
390         else:
391             logger.debug("Flavor '%s' with ID=%s created successfully."
392                          % (flavor_name, flavor_id))
393
394     return flavor_exists, flavor_id
395
396
397 def get_floating_ips(neutron_client):
398     try:
399         floating_ips = neutron_client.list_floatingips()
400         return floating_ips['floatingips']
401     except Exception as e:
402         logger.error("Error [get_floating_ips(neutron_client)]: %s" % e)
403         return None
404
405
406 def get_hypervisors(nova_client):
407     try:
408         nodes = []
409         hypervisors = nova_client.hypervisors.list()
410         for hypervisor in hypervisors:
411             if hypervisor.state == "up":
412                 nodes.append(hypervisor.hypervisor_hostname)
413         return nodes
414     except Exception as e:
415         logger.error("Error [get_hypervisors(nova_client)]: %s" % e)
416         return None
417
418
419 def create_aggregate(nova_client, aggregate_name, av_zone):
420     try:
421         nova_client.aggregates.create(aggregate_name, av_zone)
422         return True
423     except Exception as e:
424         logger.error("Error [create_aggregate(nova_client, %s, %s)]: %s"
425                      % (aggregate_name, av_zone, e))
426         return None
427
428
429 def add_host_to_aggregate(nova_client, aggregate_name, compute_host):
430     try:
431         aggregate_id = get_aggregate_id(nova_client, aggregate_name)
432         nova_client.aggregates.add_host(aggregate_id, compute_host)
433         return True
434     except Exception as e:
435         logger.error("Error [add_host_to_aggregate(nova_client, %s, %s)]: %s"
436                      % (aggregate_name, compute_host, e))
437         return None
438
439
440 def create_aggregate_with_host(
441         nova_client, aggregate_name, av_zone, compute_host):
442     try:
443         create_aggregate(nova_client, aggregate_name, av_zone)
444         add_host_to_aggregate(nova_client, aggregate_name, compute_host)
445         return True
446     except Exception as e:
447         logger.error("Error [create_aggregate_with_host("
448                      "nova_client, %s, %s, %s)]: %s"
449                      % (aggregate_name, av_zone, compute_host, e))
450         return None
451
452
453 def create_instance(flavor_name,
454                     image_id,
455                     network_id,
456                     instance_name="functest-vm",
457                     confdrive=True,
458                     userdata=None,
459                     av_zone='',
460                     fixed_ip=None,
461                     files=None):
462     nova_client = get_nova_client()
463     try:
464         flavor = nova_client.flavors.find(name=flavor_name)
465     except:
466         flavors = nova_client.flavors.list()
467         logger.error("Error: Flavor '%s' not found. Available flavors are: "
468                      "\n%s" % (flavor_name, flavors))
469         return None
470     if fixed_ip is not None:
471         nics = {"net-id": network_id, "v4-fixed-ip": fixed_ip}
472     else:
473         nics = {"net-id": network_id}
474     if userdata is None:
475         instance = nova_client.servers.create(
476             name=instance_name,
477             flavor=flavor,
478             image=image_id,
479             nics=[nics],
480             availability_zone=av_zone,
481             files=files
482         )
483     else:
484         instance = nova_client.servers.create(
485             name=instance_name,
486             flavor=flavor,
487             image=image_id,
488             nics=[nics],
489             config_drive=confdrive,
490             userdata=userdata,
491             availability_zone=av_zone,
492             files=files
493         )
494     return instance
495
496
497 def create_instance_and_wait_for_active(flavor_name,
498                                         image_id,
499                                         network_id,
500                                         instance_name="",
501                                         config_drive=False,
502                                         userdata="",
503                                         av_zone='',
504                                         fixed_ip=None,
505                                         files=None):
506     SLEEP = 3
507     VM_BOOT_TIMEOUT = 180
508     nova_client = get_nova_client()
509     instance = create_instance(flavor_name,
510                                image_id,
511                                network_id,
512                                instance_name,
513                                config_drive,
514                                userdata,
515                                av_zone=av_zone,
516                                fixed_ip=fixed_ip,
517                                files=files)
518     count = VM_BOOT_TIMEOUT / SLEEP
519     for n in range(count, -1, -1):
520         status = get_instance_status(nova_client, instance)
521         if status.lower() == "active":
522             return instance
523         elif status.lower() == "error":
524             logger.error("The instance %s went to ERROR status."
525                          % instance_name)
526             return None
527         time.sleep(SLEEP)
528     logger.error("Timeout booting the instance %s." % instance_name)
529     return None
530
531
532 def create_floating_ip(neutron_client):
533     extnet_id = get_external_net_id(neutron_client)
534     props = {'floating_network_id': extnet_id}
535     try:
536         ip_json = neutron_client.create_floatingip({'floatingip': props})
537         fip_addr = ip_json['floatingip']['floating_ip_address']
538         fip_id = ip_json['floatingip']['id']
539     except Exception as e:
540         logger.error("Error [create_floating_ip(neutron_client)]: %s" % e)
541         return None
542     return {'fip_addr': fip_addr, 'fip_id': fip_id}
543
544
545 def add_floating_ip(nova_client, server_id, floatingip_addr):
546     try:
547         nova_client.servers.add_floating_ip(server_id, floatingip_addr)
548         return True
549     except Exception as e:
550         logger.error("Error [add_floating_ip(nova_client, '%s', '%s')]: %s"
551                      % (server_id, floatingip_addr, e))
552         return False
553
554
555 def delete_instance(nova_client, instance_id):
556     try:
557         nova_client.servers.force_delete(instance_id)
558         return True
559     except Exception as e:
560         logger.error("Error [delete_instance(nova_client, '%s')]: %s"
561                      % (instance_id, e))
562         return False
563
564
565 def delete_floating_ip(neutron_client, floatingip_id):
566     try:
567         neutron_client.delete_floatingip(floatingip_id)
568         return True
569     except Exception as e:
570         logger.error("Error [delete_floating_ip(neutron_client, '%s')]: %s"
571                      % (floatingip_id, e))
572         return False
573
574
575 def remove_host_from_aggregate(nova_client, aggregate_name, compute_host):
576     try:
577         aggregate_id = get_aggregate_id(nova_client, aggregate_name)
578         nova_client.aggregates.remove_host(aggregate_id, compute_host)
579         return True
580     except Exception as e:
581         logger.error("Error [remove_host_from_aggregate(nova_client, %s, %s)]:"
582                      " %s" % (aggregate_name, compute_host, e))
583         return False
584
585
586 def remove_hosts_from_aggregate(nova_client, aggregate_name):
587     aggregate_id = get_aggregate_id(nova_client, aggregate_name)
588     hosts = nova_client.aggregates.get(aggregate_id).hosts
589     assert(
590         all(remove_host_from_aggregate(nova_client, aggregate_name, host)
591             for host in hosts))
592
593
594 def delete_aggregate(nova_client, aggregate_name):
595     try:
596         remove_hosts_from_aggregate(nova_client, aggregate_name)
597         nova_client.aggregates.delete(aggregate_name)
598         return True
599     except Exception as e:
600         logger.error("Error [delete_aggregate(nova_client, %s)]: %s"
601                      % (aggregate_name, e))
602         return False
603
604
605 # *********************************************
606 #   NEUTRON
607 # *********************************************
608 def get_network_list(neutron_client):
609     network_list = neutron_client.list_networks()['networks']
610     if len(network_list) == 0:
611         return None
612     else:
613         return network_list
614
615
616 def get_router_list(neutron_client):
617     router_list = neutron_client.list_routers()['routers']
618     if len(router_list) == 0:
619         return None
620     else:
621         return router_list
622
623
624 def get_port_list(neutron_client):
625     port_list = neutron_client.list_ports()['ports']
626     if len(port_list) == 0:
627         return None
628     else:
629         return port_list
630
631
632 def get_network_id(neutron_client, network_name):
633     networks = neutron_client.list_networks()['networks']
634     id = ''
635     for n in networks:
636         if n['name'] == network_name:
637             id = n['id']
638             break
639     return id
640
641
642 def get_subnet_id(neutron_client, subnet_name):
643     subnets = neutron_client.list_subnets()['subnets']
644     id = ''
645     for s in subnets:
646         if s['name'] == subnet_name:
647             id = s['id']
648             break
649     return id
650
651
652 def get_router_id(neutron_client, router_name):
653     routers = neutron_client.list_routers()['routers']
654     id = ''
655     for r in routers:
656         if r['name'] == router_name:
657             id = r['id']
658             break
659     return id
660
661
662 def get_private_net(neutron_client):
663     # Checks if there is an existing shared private network
664     networks = neutron_client.list_networks()['networks']
665     if len(networks) == 0:
666         return None
667     for net in networks:
668         if (net['router:external'] is False) and (net['shared'] is True):
669             return net
670     return None
671
672
673 def get_external_net(neutron_client):
674     if (hasattr(CONST, 'EXTERNAL_NETWORK')):
675         return CONST.__getattribute__('EXTERNAL_NETWORK')
676     for network in neutron_client.list_networks()['networks']:
677         if network['router:external']:
678             return network['name']
679     return None
680
681
682 def get_external_net_id(neutron_client):
683     if (hasattr(CONST, 'EXTERNAL_NETWORK')):
684         networks = neutron_client.list_networks(
685             name=CONST.__getattribute__('EXTERNAL_NETWORK'))
686         net_id = networks['networks'][0]['id']
687         return net_id
688     for network in neutron_client.list_networks()['networks']:
689         if network['router:external']:
690             return network['id']
691     return None
692
693
694 def check_neutron_net(neutron_client, net_name):
695     for network in neutron_client.list_networks()['networks']:
696         if network['name'] == net_name:
697             for subnet in network['subnets']:
698                 return True
699     return False
700
701
702 def create_neutron_net(neutron_client, name):
703     json_body = {'network': {'name': name,
704                              'admin_state_up': True}}
705     try:
706         network = neutron_client.create_network(body=json_body)
707         network_dict = network['network']
708         return network_dict['id']
709     except Exception as e:
710         logger.error("Error [create_neutron_net(neutron_client, '%s')]: %s"
711                      % (name, e))
712         return None
713
714
715 def create_neutron_subnet(neutron_client, name, cidr, net_id,
716                           dns=['8.8.8.8', '8.8.4.4']):
717     json_body = {'subnets': [{'name': name, 'cidr': cidr,
718                               'ip_version': 4, 'network_id': net_id,
719                               'dns_nameservers': dns}]}
720
721     try:
722         subnet = neutron_client.create_subnet(body=json_body)
723         return subnet['subnets'][0]['id']
724     except Exception as e:
725         logger.error("Error [create_neutron_subnet(neutron_client, '%s', "
726                      "'%s', '%s')]: %s" % (name, cidr, net_id, e))
727         return None
728
729
730 def create_neutron_router(neutron_client, name):
731     json_body = {'router': {'name': name, 'admin_state_up': True}}
732     try:
733         router = neutron_client.create_router(json_body)
734         return router['router']['id']
735     except Exception as e:
736         logger.error("Error [create_neutron_router(neutron_client, '%s')]: %s"
737                      % (name, e))
738         return None
739
740
741 def create_neutron_port(neutron_client, name, network_id, ip):
742     json_body = {'port': {
743                  'admin_state_up': True,
744                  'name': name,
745                  'network_id': network_id,
746                  'fixed_ips': [{"ip_address": ip}]
747                  }}
748     try:
749         port = neutron_client.create_port(body=json_body)
750         return port['port']['id']
751     except Exception as e:
752         logger.error("Error [create_neutron_port(neutron_client, '%s', '%s', "
753                      "'%s')]: %s" % (name, network_id, ip, e))
754         return None
755
756
757 def update_neutron_net(neutron_client, network_id, shared=False):
758     json_body = {'network': {'shared': shared}}
759     try:
760         neutron_client.update_network(network_id, body=json_body)
761         return True
762     except Exception as e:
763         logger.error("Error [update_neutron_net(neutron_client, '%s', '%s')]: "
764                      "%s" % (network_id, str(shared), e))
765         return False
766
767
768 def update_neutron_port(neutron_client, port_id, device_owner):
769     json_body = {'port': {
770                  'device_owner': device_owner,
771                  }}
772     try:
773         port = neutron_client.update_port(port=port_id,
774                                           body=json_body)
775         return port['port']['id']
776     except Exception as e:
777         logger.error("Error [update_neutron_port(neutron_client, '%s', '%s')]:"
778                      " %s" % (port_id, device_owner, e))
779         return None
780
781
782 def add_interface_router(neutron_client, router_id, subnet_id):
783     json_body = {"subnet_id": subnet_id}
784     try:
785         neutron_client.add_interface_router(router=router_id, body=json_body)
786         return True
787     except Exception as e:
788         logger.error("Error [add_interface_router(neutron_client, '%s', "
789                      "'%s')]: %s" % (router_id, subnet_id, e))
790         return False
791
792
793 def add_gateway_router(neutron_client, router_id):
794     ext_net_id = get_external_net_id(neutron_client)
795     router_dict = {'network_id': ext_net_id}
796     try:
797         neutron_client.add_gateway_router(router_id, router_dict)
798         return True
799     except Exception as e:
800         logger.error("Error [add_gateway_router(neutron_client, '%s')]: %s"
801                      % (router_id, e))
802         return False
803
804
805 def delete_neutron_net(neutron_client, network_id):
806     try:
807         neutron_client.delete_network(network_id)
808         return True
809     except Exception as e:
810         logger.error("Error [delete_neutron_net(neutron_client, '%s')]: %s"
811                      % (network_id, e))
812         return False
813
814
815 def delete_neutron_subnet(neutron_client, subnet_id):
816     try:
817         neutron_client.delete_subnet(subnet_id)
818         return True
819     except Exception as e:
820         logger.error("Error [delete_neutron_subnet(neutron_client, '%s')]: %s"
821                      % (subnet_id, e))
822         return False
823
824
825 def delete_neutron_router(neutron_client, router_id):
826     try:
827         neutron_client.delete_router(router=router_id)
828         return True
829     except Exception as e:
830         logger.error("Error [delete_neutron_router(neutron_client, '%s')]: %s"
831                      % (router_id, e))
832         return False
833
834
835 def delete_neutron_port(neutron_client, port_id):
836     try:
837         neutron_client.delete_port(port_id)
838         return True
839     except Exception as e:
840         logger.error("Error [delete_neutron_port(neutron_client, '%s')]: %s"
841                      % (port_id, e))
842         return False
843
844
845 def remove_interface_router(neutron_client, router_id, subnet_id):
846     json_body = {"subnet_id": subnet_id}
847     try:
848         neutron_client.remove_interface_router(router=router_id,
849                                                body=json_body)
850         return True
851     except Exception as e:
852         logger.error("Error [remove_interface_router(neutron_client, '%s', "
853                      "'%s')]: %s" % (router_id, subnet_id, e))
854         return False
855
856
857 def remove_gateway_router(neutron_client, router_id):
858     try:
859         neutron_client.remove_gateway_router(router_id)
860         return True
861     except Exception as e:
862         logger.error("Error [remove_gateway_router(neutron_client, '%s')]: %s"
863                      % (router_id, e))
864         return False
865
866
867 def create_network_full(neutron_client,
868                         net_name,
869                         subnet_name,
870                         router_name,
871                         cidr,
872                         dns=['8.8.8.8', '8.8.4.4']):
873
874     # Check if the network already exists
875     network_id = get_network_id(neutron_client, net_name)
876     subnet_id = get_subnet_id(neutron_client, subnet_name)
877     router_id = get_router_id(neutron_client, router_name)
878
879     if network_id != '' and subnet_id != '' and router_id != '':
880         logger.info("A network with name '%s' already exists..." % net_name)
881     else:
882         neutron_client.format = 'json'
883         logger.info('Creating neutron network %s...' % net_name)
884         network_id = create_neutron_net(neutron_client, net_name)
885
886         if not network_id:
887             return False
888
889         logger.debug("Network '%s' created successfully" % network_id)
890         logger.debug('Creating Subnet....')
891         subnet_id = create_neutron_subnet(neutron_client, subnet_name,
892                                           cidr, network_id, dns)
893         if not subnet_id:
894             return None
895
896         logger.debug("Subnet '%s' created successfully" % subnet_id)
897         logger.debug('Creating Router...')
898         router_id = create_neutron_router(neutron_client, router_name)
899
900         if not router_id:
901             return None
902
903         logger.debug("Router '%s' created successfully" % router_id)
904         logger.debug('Adding router to subnet...')
905
906         if not add_interface_router(neutron_client, router_id, subnet_id):
907             return None
908
909         logger.debug("Interface added successfully.")
910
911         logger.debug('Adding gateway to router...')
912         if not add_gateway_router(neutron_client, router_id):
913             return None
914
915         logger.debug("Gateway added successfully.")
916
917     network_dic = {'net_id': network_id,
918                    'subnet_id': subnet_id,
919                    'router_id': router_id}
920     return network_dic
921
922
923 def create_shared_network_full(net_name, subnt_name, router_name, subnet_cidr):
924     neutron_client = get_neutron_client()
925
926     network_dic = create_network_full(neutron_client,
927                                       net_name,
928                                       subnt_name,
929                                       router_name,
930                                       subnet_cidr)
931     if network_dic:
932         if not update_neutron_net(neutron_client,
933                                   network_dic['net_id'],
934                                   shared=True):
935             logger.error("Failed to update network %s..." % net_name)
936             return None
937         else:
938             logger.debug("Network '%s' is available..." % net_name)
939     else:
940         logger.error("Network %s creation failed" % net_name)
941         return None
942     return network_dic
943
944
945 # *********************************************
946 #   SEC GROUPS
947 # *********************************************
948
949
950 def get_security_groups(neutron_client):
951     try:
952         security_groups = neutron_client.list_security_groups()[
953             'security_groups']
954         return security_groups
955     except Exception as e:
956         logger.error("Error [get_security_groups(neutron_client)]: %s" % e)
957         return None
958
959
960 def get_security_group_id(neutron_client, sg_name):
961     security_groups = get_security_groups(neutron_client)
962     id = ''
963     for sg in security_groups:
964         if sg['name'] == sg_name:
965             id = sg['id']
966             break
967     return id
968
969
970 def create_security_group(neutron_client, sg_name, sg_description):
971     json_body = {'security_group': {'name': sg_name,
972                                     'description': sg_description}}
973     try:
974         secgroup = neutron_client.create_security_group(json_body)
975         return secgroup['security_group']
976     except Exception as e:
977         logger.error("Error [create_security_group(neutron_client, '%s', "
978                      "'%s')]: %s" % (sg_name, sg_description, e))
979         return None
980
981
982 def create_secgroup_rule(neutron_client, sg_id, direction, protocol,
983                          port_range_min=None, port_range_max=None):
984     # We create a security group in 2 steps
985     # 1 - we check the format and set the json body accordingly
986     # 2 - we call neturon client to create the security group
987
988     # Format check
989     json_body = {'security_group_rule': {'direction': direction,
990                                          'security_group_id': sg_id,
991                                          'protocol': protocol}}
992     # parameters may be
993     # - both None => we do nothing
994     # - both Not None => we add them to the json description
995     # but one cannot be None is the other is not None
996     if (port_range_min is not None and port_range_max is not None):
997         # add port_range in json description
998         json_body['security_group_rule']['port_range_min'] = port_range_min
999         json_body['security_group_rule']['port_range_max'] = port_range_max
1000         logger.debug("Security_group format set (port range included)")
1001     else:
1002         # either both port range are set to None => do nothing
1003         # or one is set but not the other => log it and return False
1004         if port_range_min is None and port_range_max is None:
1005             logger.debug("Security_group format set (no port range mentioned)")
1006         else:
1007             logger.error("Bad security group format."
1008                          "One of the port range is not properly set:"
1009                          "range min: {},"
1010                          "range max: {}".format(port_range_min,
1011                                                 port_range_max))
1012             return False
1013
1014     # Create security group using neutron client
1015     try:
1016         neutron_client.create_security_group_rule(json_body)
1017         return True
1018     except:
1019         logger.exception("Impossible to create_security_group_rule,"
1020                          "security group rule probably already exists")
1021         return False
1022
1023
1024 def get_security_group_rules(neutron_client, sg_id):
1025     try:
1026         security_rules = neutron_client.list_security_group_rules()[
1027             'security_group_rules']
1028         security_rules = [rule for rule in security_rules
1029                           if rule["security_group_id"] == sg_id]
1030         return security_rules
1031     except Exception as e:
1032         logger.error("Error [get_security_group_rules(neutron_client, sg_id)]:"
1033                      " %s" % e)
1034         return None
1035
1036
1037 def check_security_group_rules(neutron_client, sg_id, direction, protocol,
1038                                port_min=None, port_max=None):
1039     try:
1040         security_rules = get_security_group_rules(neutron_client, sg_id)
1041         security_rules = [rule for rule in security_rules
1042                           if (rule["direction"].lower() == direction and
1043                               rule["protocol"].lower() == protocol and
1044                               rule["port_range_min"] == port_min and
1045                               rule["port_range_max"] == port_max)]
1046         if len(security_rules) == 0:
1047             return True
1048         else:
1049             return False
1050     except Exception as e:
1051         logger.error("Error [check_security_group_rules("
1052                      " neutron_client, sg_id, direction,"
1053                      " protocol, port_min=None, port_max=None)]: "
1054                      "%s" % e)
1055         return None
1056
1057
1058 def create_security_group_full(neutron_client,
1059                                sg_name, sg_description):
1060     sg_id = get_security_group_id(neutron_client, sg_name)
1061     if sg_id != '':
1062         logger.info("Using existing security group '%s'..." % sg_name)
1063     else:
1064         logger.info("Creating security group  '%s'..." % sg_name)
1065         SECGROUP = create_security_group(neutron_client,
1066                                          sg_name,
1067                                          sg_description)
1068         if not SECGROUP:
1069             logger.error("Failed to create the security group...")
1070             return None
1071
1072         sg_id = SECGROUP['id']
1073
1074         logger.debug("Security group '%s' with ID=%s created successfully."
1075                      % (SECGROUP['name'], sg_id))
1076
1077         logger.debug("Adding ICMP rules in security group '%s'..."
1078                      % sg_name)
1079         if not create_secgroup_rule(neutron_client, sg_id,
1080                                     'ingress', 'icmp'):
1081             logger.error("Failed to create the security group rule...")
1082             return None
1083
1084         logger.debug("Adding SSH rules in security group '%s'..."
1085                      % sg_name)
1086         if not create_secgroup_rule(
1087                 neutron_client, sg_id, 'ingress', 'tcp', '22', '22'):
1088             logger.error("Failed to create the security group rule...")
1089             return None
1090
1091         if not create_secgroup_rule(
1092                 neutron_client, sg_id, 'egress', 'tcp', '22', '22'):
1093             logger.error("Failed to create the security group rule...")
1094             return None
1095     return sg_id
1096
1097
1098 def add_secgroup_to_instance(nova_client, instance_id, secgroup_id):
1099     try:
1100         nova_client.servers.add_security_group(instance_id, secgroup_id)
1101         return True
1102     except Exception as e:
1103         logger.error("Error [add_secgroup_to_instance(nova_client, '%s', "
1104                      "'%s')]: %s" % (instance_id, secgroup_id, e))
1105         return False
1106
1107
1108 def update_sg_quota(neutron_client, tenant_id, sg_quota, sg_rule_quota):
1109     json_body = {"quota": {
1110         "security_group": sg_quota,
1111         "security_group_rule": sg_rule_quota
1112     }}
1113
1114     try:
1115         neutron_client.update_quota(tenant_id=tenant_id,
1116                                     body=json_body)
1117         return True
1118     except Exception as e:
1119         logger.error("Error [update_sg_quota(neutron_client, '%s', '%s', "
1120                      "'%s')]: %s" % (tenant_id, sg_quota, sg_rule_quota, e))
1121         return False
1122
1123
1124 def delete_security_group(neutron_client, secgroup_id):
1125     try:
1126         neutron_client.delete_security_group(secgroup_id)
1127         return True
1128     except Exception as e:
1129         logger.error("Error [delete_security_group(neutron_client, '%s')]: %s"
1130                      % (secgroup_id, e))
1131         return False
1132
1133
1134 # *********************************************
1135 #   GLANCE
1136 # *********************************************
1137 def get_images(glance_client):
1138     try:
1139         images = glance_client.images.list()
1140         return images
1141     except Exception as e:
1142         logger.error("Error [get_images]: %s" % e)
1143         return None
1144
1145
1146 def get_image_id(glance_client, image_name):
1147     images = glance_client.images.list()
1148     id = ''
1149     for i in images:
1150         if i.name == image_name:
1151             id = i.id
1152             break
1153     return id
1154
1155
1156 def create_glance_image(glance_client,
1157                         image_name,
1158                         file_path,
1159                         disk="qcow2",
1160                         extra_properties={},
1161                         container="bare",
1162                         public="public"):
1163     if not os.path.isfile(file_path):
1164         logger.error("Error: file %s does not exist." % file_path)
1165         return None
1166     try:
1167         image_id = get_image_id(glance_client, image_name)
1168         if image_id != '':
1169             logger.info("Image %s already exists." % image_name)
1170         else:
1171             logger.info("Creating image '%s' from '%s'..." % (image_name,
1172                                                               file_path))
1173
1174             image = glance_client.images.create(name=image_name,
1175                                                 visibility=public,
1176                                                 disk_format=disk,
1177                                                 container_format=container,
1178                                                 **extra_properties)
1179             image_id = image.id
1180             with open(file_path) as image_data:
1181                 glance_client.images.upload(image_id, image_data)
1182         return image_id
1183     except Exception as e:
1184         logger.error("Error [create_glance_image(glance_client, '%s', '%s', "
1185                      "'%s')]: %s" % (image_name, file_path, public, e))
1186         return None
1187
1188
1189 def get_or_create_image(name, path, format, extra_properties):
1190     image_exists = False
1191     glance_client = get_glance_client()
1192
1193     image_id = get_image_id(glance_client, name)
1194     if image_id != '':
1195         logger.info("Using existing image '%s'..." % name)
1196         image_exists = True
1197     else:
1198         logger.info("Creating image '%s' from '%s'..." % (name, path))
1199         image_id = create_glance_image(glance_client,
1200                                        name,
1201                                        path,
1202                                        format,
1203                                        extra_properties)
1204         if not image_id:
1205             logger.error("Failed to create a Glance image...")
1206         else:
1207             logger.debug("Image '%s' with ID=%s created successfully."
1208                          % (name, image_id))
1209
1210     return image_exists, image_id
1211
1212
1213 def delete_glance_image(glance_client, image_id):
1214     try:
1215         glance_client.images.delete(image_id)
1216         return True
1217     except Exception as e:
1218         logger.error("Error [delete_glance_image(glance_client, '%s')]: %s"
1219                      % (image_id, e))
1220         return False
1221
1222
1223 # *********************************************
1224 #   CINDER
1225 # *********************************************
1226 def get_volumes(cinder_client):
1227     try:
1228         volumes = cinder_client.volumes.list(search_opts={'all_tenants': 1})
1229         return volumes
1230     except Exception as e:
1231         logger.error("Error [get_volumes(cinder_client)]: %s" % e)
1232         return None
1233
1234
1235 def update_cinder_quota(cinder_client, tenant_id, vols_quota,
1236                         snapshots_quota, gigabytes_quota):
1237     quotas_values = {"volumes": vols_quota,
1238                      "snapshots": snapshots_quota,
1239                      "gigabytes": gigabytes_quota}
1240
1241     try:
1242         cinder_client.quotas.update(tenant_id, **quotas_values)
1243         return True
1244     except Exception as e:
1245         logger.error("Error [update_cinder_quota(cinder_client, '%s', '%s', "
1246                      "'%s' '%s')]: %s" % (tenant_id, vols_quota,
1247                                           snapshots_quota, gigabytes_quota, e))
1248         return False
1249
1250
1251 def delete_volume(cinder_client, volume_id, forced=False):
1252     try:
1253         if forced:
1254             try:
1255                 cinder_client.volumes.detach(volume_id)
1256             except:
1257                 logger.error(sys.exc_info()[0])
1258             cinder_client.volumes.force_delete(volume_id)
1259         else:
1260             cinder_client.volumes.delete(volume_id)
1261         return True
1262     except Exception as e:
1263         logger.error("Error [delete_volume(cinder_client, '%s', '%s')]: %s"
1264                      % (volume_id, str(forced), e))
1265         return False
1266
1267
1268 # *********************************************
1269 #   KEYSTONE
1270 # *********************************************
1271 def get_tenants(keystone_client):
1272     try:
1273         if is_keystone_v3():
1274             tenants = keystone_client.projects.list()
1275         else:
1276             tenants = keystone_client.tenants.list()
1277         return tenants
1278     except Exception as e:
1279         logger.error("Error [get_tenants(keystone_client)]: %s" % e)
1280         return None
1281
1282
1283 def get_users(keystone_client):
1284     try:
1285         users = keystone_client.users.list()
1286         return users
1287     except Exception as e:
1288         logger.error("Error [get_users(keystone_client)]: %s" % e)
1289         return None
1290
1291
1292 def get_tenant_id(keystone_client, tenant_name):
1293     tenants = get_tenants(keystone_client)
1294     id = ''
1295     for t in tenants:
1296         if t.name == tenant_name:
1297             id = t.id
1298             break
1299     return id
1300
1301
1302 def get_user_id(keystone_client, user_name):
1303     users = get_users(keystone_client)
1304     id = ''
1305     for u in users:
1306         if u.name == user_name:
1307             id = u.id
1308             break
1309     return id
1310
1311
1312 def get_role_id(keystone_client, role_name):
1313     roles = keystone_client.roles.list()
1314     id = ''
1315     for r in roles:
1316         if r.name == role_name:
1317             id = r.id
1318             break
1319     return id
1320
1321
1322 def get_domain_id(keystone_client, domain_name):
1323     domains = keystone_client.domains.list()
1324     id = ''
1325     for d in domains:
1326         if d.name == domain_name:
1327             id = d.id
1328             break
1329     return id
1330
1331
1332 def create_tenant(keystone_client, tenant_name, tenant_description):
1333     try:
1334         if is_keystone_v3():
1335             domain_name = CONST.__getattribute__('OS_PROJECT_DOMAIN_NAME')
1336             domain_id = get_domain_id(keystone_client, domain_name)
1337             tenant = keystone_client.projects.create(
1338                 name=tenant_name,
1339                 description=tenant_description,
1340                 domain=domain_id,
1341                 enabled=True)
1342         else:
1343             tenant = keystone_client.tenants.create(tenant_name,
1344                                                     tenant_description,
1345                                                     enabled=True)
1346         return tenant.id
1347     except Exception as e:
1348         logger.error("Error [create_tenant(keystone_client, '%s', '%s')]: %s"
1349                      % (tenant_name, tenant_description, e))
1350         return None
1351
1352
1353 def get_or_create_tenant(keystone_client, tenant_name, tenant_description):
1354     tenant_id = get_tenant_id(keystone_client, tenant_name)
1355     if not tenant_id:
1356         tenant_id = create_tenant(keystone_client, tenant_name,
1357                                   tenant_description)
1358
1359     return tenant_id
1360
1361
1362 def get_or_create_tenant_for_vnf(keystone_client, tenant_name,
1363                                  tenant_description):
1364     """Get or Create a Tenant
1365
1366         Args:
1367             keystone_client: keystone client reference
1368             tenant_name: the name of the tenant
1369             tenant_description: the description of the tenant
1370
1371         return False if tenant retrieved though get
1372         return True if tenant created
1373         raise Exception if error during processing
1374     """
1375     try:
1376         tenant_id = get_tenant_id(keystone_client, tenant_name)
1377         if not tenant_id:
1378             tenant_id = create_tenant(keystone_client, tenant_name,
1379                                       tenant_description)
1380             return True
1381         else:
1382             return False
1383     except:
1384         raise Exception("Impossible to create a Tenant for the VNF {}".format(
1385                             tenant_name))
1386
1387
1388 def create_user(keystone_client, user_name, user_password,
1389                 user_email, tenant_id):
1390     try:
1391         if is_keystone_v3():
1392             user = keystone_client.users.create(name=user_name,
1393                                                 password=user_password,
1394                                                 email=user_email,
1395                                                 project_id=tenant_id,
1396                                                 enabled=True)
1397         else:
1398             user = keystone_client.users.create(user_name,
1399                                                 user_password,
1400                                                 user_email,
1401                                                 tenant_id,
1402                                                 enabled=True)
1403         return user.id
1404     except Exception as e:
1405         logger.error("Error [create_user(keystone_client, '%s', '%s', '%s'"
1406                      "'%s')]: %s" % (user_name, user_password,
1407                                      user_email, tenant_id, e))
1408         return None
1409
1410
1411 def get_or_create_user(keystone_client, user_name, user_password,
1412                        tenant_id, user_email=None):
1413     user_id = get_user_id(keystone_client, user_name)
1414     if not user_id:
1415         user_id = create_user(keystone_client, user_name, user_password,
1416                               user_email, tenant_id)
1417     return user_id
1418
1419
1420 def get_or_create_user_for_vnf(keystone_client, vnf_ref):
1421     """Get or Create user for VNF
1422
1423         Args:
1424             keystone_client: keystone client reference
1425             vnf_ref: VNF reference used as user name & password, tenant name
1426
1427         return False if user retrieved through get
1428         return True if user created
1429         raise Exception if error during processing
1430     """
1431     try:
1432         user_id = get_user_id(keystone_client, vnf_ref)
1433         tenant_id = get_tenant_id(keystone_client, vnf_ref)
1434         created = False
1435         if not user_id:
1436             user_id = create_user(keystone_client, vnf_ref, vnf_ref,
1437                                   "", tenant_id)
1438             created = True
1439         try:
1440             role_id = get_role_id(keystone_client, 'admin')
1441             tenant_id = get_tenant_id(keystone_client, vnf_ref)
1442             add_role_user(keystone_client, user_id, role_id, tenant_id)
1443         except:
1444             logger.warn("Cannot associate user to role admin on tenant")
1445         return created
1446     except:
1447         raise Exception("Impossible to create a user for the VNF {}".format(
1448             vnf_ref))
1449
1450
1451 def add_role_user(keystone_client, user_id, role_id, tenant_id):
1452     try:
1453         if is_keystone_v3():
1454             keystone_client.roles.grant(role=role_id,
1455                                         user=user_id,
1456                                         project=tenant_id)
1457         else:
1458             keystone_client.roles.add_user_role(user_id, role_id, tenant_id)
1459         return True
1460     except Exception as e:
1461         logger.error("Error [add_role_user(keystone_client, '%s', '%s'"
1462                      "'%s')]: %s " % (user_id, role_id, tenant_id, e))
1463         return False
1464
1465
1466 def delete_tenant(keystone_client, tenant_id):
1467     try:
1468         if is_keystone_v3():
1469             keystone_client.projects.delete(tenant_id)
1470         else:
1471             keystone_client.tenants.delete(tenant_id)
1472         return True
1473     except Exception as e:
1474         logger.error("Error [delete_tenant(keystone_client, '%s')]: %s"
1475                      % (tenant_id, e))
1476         return False
1477
1478
1479 def delete_user(keystone_client, user_id):
1480     try:
1481         keystone_client.users.delete(user_id)
1482         return True
1483     except Exception as e:
1484         logger.error("Error [delete_user(keystone_client, '%s')]: %s"
1485                      % (user_id, e))
1486         return False
1487
1488
1489 # *********************************************
1490 #   HEAT
1491 # *********************************************
1492 def get_resource(heat_client, stack_id, resource):
1493     try:
1494         resources = heat_client.resources.get(stack_id, resource)
1495         return resources
1496     except Exception as e:
1497         logger.error("Error [get_resource]: %s" % e)
1498         return None