Merge "Fix pylint errors/warnings in rally"
[functest.git] / functest / utils / openstack_utils.py
1 #!/usr/bin/env python
2 #
3 # jose.lausuch@ericsson.com
4 # valentin.boucher@orange.com
5 # All rights reserved. This program and the accompanying materials
6 # are made available under the terms of the Apache License, Version 2.0
7 # which accompanies this distribution, and is available at
8 # http://www.apache.org/licenses/LICENSE-2.0
9 #
10
11 import logging
12 import os.path
13 import re
14 import sys
15 import time
16
17 from keystoneauth1 import loading
18 from keystoneauth1 import session
19 from cinderclient import client as cinderclient
20 from glanceclient import client as glanceclient
21 from heatclient import client as heatclient
22 from novaclient import client as novaclient
23 from keystoneclient import client as keystoneclient
24 from neutronclient.neutron import client as neutronclient
25
26 from functest.utils.constants import CONST
27 import functest.utils.functest_utils as ft_utils
28
29 logger = logging.getLogger(__name__)
30
31 DEFAULT_API_VERSION = '2'
32 DEFAULT_HEAT_API_VERSION = '1'
33
34
35 # *********************************************
36 #   CREDENTIALS
37 # *********************************************
38 class MissingEnvVar(Exception):
39
40     def __init__(self, var):
41         self.var = var
42
43     def __str__(self):
44         return str.format("Please set the mandatory env var: {}", self.var)
45
46
47 def is_keystone_v3():
48     keystone_api_version = os.getenv('OS_IDENTITY_API_VERSION')
49     if (keystone_api_version is None or
50             keystone_api_version == '2'):
51         return False
52     else:
53         return True
54
55
56 def get_rc_env_vars():
57     env_vars = ['OS_AUTH_URL', 'OS_USERNAME', 'OS_PASSWORD']
58     if is_keystone_v3():
59         env_vars.extend(['OS_PROJECT_NAME',
60                          'OS_USER_DOMAIN_NAME',
61                          'OS_PROJECT_DOMAIN_NAME'])
62     else:
63         env_vars.extend(['OS_TENANT_NAME'])
64     return env_vars
65
66
67 def check_credentials():
68     """
69     Check if the OpenStack credentials (openrc) are sourced
70     """
71     env_vars = get_rc_env_vars()
72     return all(map(lambda v: v in os.environ and os.environ[v], env_vars))
73
74
75 def get_env_cred_dict():
76     env_cred_dict = {
77         'OS_USERNAME': 'username',
78         'OS_PASSWORD': 'password',
79         'OS_AUTH_URL': 'auth_url',
80         'OS_TENANT_NAME': 'tenant_name',
81         'OS_USER_DOMAIN_NAME': 'user_domain_name',
82         'OS_PROJECT_DOMAIN_NAME': 'project_domain_name',
83         'OS_PROJECT_NAME': 'project_name',
84         'OS_ENDPOINT_TYPE': 'endpoint_type',
85         'OS_REGION_NAME': 'region_name',
86         'OS_CACERT': 'https_cacert',
87         'OS_INSECURE': 'https_insecure'
88     }
89     return env_cred_dict
90
91
92 def get_credentials(other_creds={}):
93     """Returns a creds dictionary filled with parsed from env
94     """
95     creds = {}
96     env_vars = get_rc_env_vars()
97     env_cred_dict = get_env_cred_dict()
98
99     for envvar in env_vars:
100         if os.getenv(envvar) is None:
101             raise MissingEnvVar(envvar)
102         else:
103             creds_key = env_cred_dict.get(envvar)
104             creds.update({creds_key: os.getenv(envvar)})
105
106     if 'tenant' in other_creds.keys():
107         if is_keystone_v3():
108             tenant = 'project_name'
109         else:
110             tenant = 'tenant_name'
111         other_creds[tenant] = other_creds.pop('tenant')
112
113     creds.update(other_creds)
114
115     return creds
116
117
118 def source_credentials(rc_file):
119     with open(rc_file, "r") as f:
120         for line in f:
121             var = (line.rstrip('"\n').replace('export ', '').split("=")
122                    if re.search(r'(.*)=(.*)', line) else None)
123             # The two next lines should be modified as soon as rc_file
124             # conforms with common rules. Be aware that it could induce
125             # issues if value starts with '
126             if var:
127                 key = re.sub(r'^["\' ]*|[ \'"]*$', '', var[0])
128                 value = re.sub(r'^["\' ]*|[ \'"]*$', '', "".join(var[1:]))
129                 os.environ[key] = value
130                 CONST.__setattr__(key, value)
131
132
133 def get_session_auth(other_creds={}):
134     loader = loading.get_plugin_loader('password')
135     creds = get_credentials(other_creds)
136     auth = loader.load_from_options(**creds)
137     return auth
138
139
140 def get_endpoint(service_type, interface='public'):
141     auth = get_session_auth()
142     return get_session().get_endpoint(auth=auth,
143                                       service_type=service_type,
144                                       interface=interface)
145
146
147 def get_session(other_creds={}):
148     auth = get_session_auth(other_creds)
149     https_cacert = os.getenv('OS_CACERT', '')
150     https_insecure = os.getenv('OS_INSECURE', '').lower() == 'true'
151     return session.Session(auth=auth,
152                            verify=(https_cacert or not https_insecure))
153
154
155 # *********************************************
156 #   CLIENTS
157 # *********************************************
158 def get_keystone_client_version():
159     api_version = os.getenv('OS_IDENTITY_API_VERSION')
160     if api_version is not None:
161         logger.info("OS_IDENTITY_API_VERSION is set in env as '%s'",
162                     api_version)
163         return api_version
164     return DEFAULT_API_VERSION
165
166
167 def get_keystone_client(other_creds={}):
168     sess = get_session(other_creds)
169     return keystoneclient.Client(get_keystone_client_version(),
170                                  session=sess,
171                                  interface=os.getenv('OS_INTERFACE', 'admin'))
172
173
174 def get_nova_client_version():
175     api_version = os.getenv('OS_COMPUTE_API_VERSION')
176     if api_version is not None:
177         logger.info("OS_COMPUTE_API_VERSION is set in env as '%s'",
178                     api_version)
179         return api_version
180     return DEFAULT_API_VERSION
181
182
183 def get_nova_client(other_creds={}):
184     sess = get_session(other_creds)
185     return novaclient.Client(get_nova_client_version(), session=sess)
186
187
188 def get_cinder_client_version():
189     api_version = os.getenv('OS_VOLUME_API_VERSION')
190     if api_version is not None:
191         logger.info("OS_VOLUME_API_VERSION is set in env as '%s'",
192                     api_version)
193         return api_version
194     return DEFAULT_API_VERSION
195
196
197 def get_cinder_client(other_creds={}):
198     sess = get_session(other_creds)
199     return cinderclient.Client(get_cinder_client_version(), session=sess)
200
201
202 def get_neutron_client_version():
203     api_version = os.getenv('OS_NETWORK_API_VERSION')
204     if api_version is not None:
205         logger.info("OS_NETWORK_API_VERSION is set in env as '%s'",
206                     api_version)
207         return api_version
208     return DEFAULT_API_VERSION
209
210
211 def get_neutron_client(other_creds={}):
212     sess = get_session(other_creds)
213     return neutronclient.Client(get_neutron_client_version(), session=sess)
214
215
216 def get_glance_client_version():
217     api_version = os.getenv('OS_IMAGE_API_VERSION')
218     if api_version is not None:
219         logger.info("OS_IMAGE_API_VERSION is set in env as '%s'", api_version)
220         return api_version
221     return DEFAULT_API_VERSION
222
223
224 def get_glance_client(other_creds={}):
225     sess = get_session(other_creds)
226     return glanceclient.Client(get_glance_client_version(), session=sess)
227
228
229 def get_heat_client_version():
230     api_version = os.getenv('OS_ORCHESTRATION_API_VERSION')
231     if api_version is not None:
232         logger.info("OS_ORCHESTRATION_API_VERSION is set in env as '%s'",
233                     api_version)
234         return api_version
235     return DEFAULT_HEAT_API_VERSION
236
237
238 def get_heat_client(other_creds={}):
239     sess = get_session(other_creds)
240     return heatclient.Client(get_heat_client_version(), session=sess)
241
242
243 def download_and_add_image_on_glance(glance, image_name, image_url, data_dir):
244     try:
245         dest_path = data_dir
246         if not os.path.exists(dest_path):
247             os.makedirs(dest_path)
248         file_name = image_url.rsplit('/')[-1]
249         if not ft_utils.download_url(image_url, dest_path):
250             return False
251     except Exception:
252         raise Exception("Impossible to download image from {}".format(
253                         image_url))
254
255     try:
256         image = create_glance_image(
257             glance, image_name, dest_path + file_name)
258         if not image:
259             return False
260         else:
261             return image
262     except Exception:
263         raise Exception("Impossible to put image {} in glance".format(
264                         image_name))
265
266
267 # *********************************************
268 #   NOVA
269 # *********************************************
270 def get_instances(nova_client):
271     try:
272         instances = nova_client.servers.list(search_opts={'all_tenants': 1})
273         return instances
274     except Exception as e:
275         logger.error("Error [get_instances(nova_client)]: %s" % e)
276         return None
277
278
279 def get_instance_status(nova_client, instance):
280     try:
281         instance = nova_client.servers.get(instance.id)
282         return instance.status
283     except Exception as e:
284         logger.error("Error [get_instance_status(nova_client)]: %s" % e)
285         return None
286
287
288 def get_instance_by_name(nova_client, instance_name):
289     try:
290         instance = nova_client.servers.find(name=instance_name)
291         return instance
292     except Exception as e:
293         logger.error("Error [get_instance_by_name(nova_client, '%s')]: %s"
294                      % (instance_name, e))
295         return None
296
297
298 def get_flavor_id(nova_client, flavor_name):
299     flavors = nova_client.flavors.list(detailed=True)
300     id = ''
301     for f in flavors:
302         if f.name == flavor_name:
303             id = f.id
304             break
305     return id
306
307
308 def get_flavor_id_by_ram_range(nova_client, min_ram, max_ram):
309     flavors = nova_client.flavors.list(detailed=True)
310     id = ''
311     for f in flavors:
312         if min_ram <= f.ram and f.ram <= max_ram:
313             id = f.id
314             break
315     return id
316
317
318 def get_aggregates(nova_client):
319     try:
320         aggregates = nova_client.aggregates.list()
321         return aggregates
322     except Exception as e:
323         logger.error("Error [get_aggregates(nova_client)]: %s" % e)
324         return None
325
326
327 def get_aggregate_id(nova_client, aggregate_name):
328     try:
329         aggregates = get_aggregates(nova_client)
330         _id = [ag.id for ag in aggregates if ag.name == aggregate_name][0]
331         return _id
332     except Exception as e:
333         logger.error("Error [get_aggregate_id(nova_client, %s)]:"
334                      " %s" % (aggregate_name, e))
335         return None
336
337
338 def get_availability_zones(nova_client):
339     try:
340         availability_zones = nova_client.availability_zones.list()
341         return availability_zones
342     except Exception as e:
343         logger.error("Error [get_availability_zones(nova_client)]: %s" % e)
344         return None
345
346
347 def get_availability_zone_names(nova_client):
348     try:
349         az_names = [az.zoneName for az in get_availability_zones(nova_client)]
350         return az_names
351     except Exception as e:
352         logger.error("Error [get_availability_zone_names(nova_client)]:"
353                      " %s" % e)
354         return None
355
356
357 def create_flavor(nova_client, flavor_name, ram, disk, vcpus, public=True):
358     try:
359         flavor = nova_client.flavors.create(
360             flavor_name, ram, vcpus, disk, is_public=public)
361         try:
362             extra_specs = ft_utils.get_functest_config(
363                 'general.flavor_extra_specs')
364             flavor.set_keys(extra_specs)
365         except ValueError:
366             # flavor extra specs are not configured, therefore skip the update
367             pass
368
369     except Exception as e:
370         logger.error("Error [create_flavor(nova_client, '%s', '%s', '%s', "
371                      "'%s')]: %s" % (flavor_name, ram, disk, vcpus, e))
372         return None
373     return flavor.id
374
375
376 def get_or_create_flavor(flavor_name, ram, disk, vcpus, public=True):
377     flavor_exists = False
378     nova_client = get_nova_client()
379
380     flavor_id = get_flavor_id(nova_client, flavor_name)
381     if flavor_id != '':
382         logger.info("Using existing flavor '%s'..." % flavor_name)
383         flavor_exists = True
384     else:
385         logger.info("Creating flavor '%s' with '%s' RAM, '%s' disk size, "
386                     "'%s' vcpus..." % (flavor_name, ram, disk, vcpus))
387         flavor_id = create_flavor(
388             nova_client, flavor_name, ram, disk, vcpus, public=public)
389         if not flavor_id:
390             raise Exception("Failed to create flavor '%s'..." % (flavor_name))
391         else:
392             logger.debug("Flavor '%s' with ID=%s created successfully."
393                          % (flavor_name, flavor_id))
394
395     return flavor_exists, flavor_id
396
397
398 def get_floating_ips(neutron_client):
399     try:
400         floating_ips = neutron_client.list_floatingips()
401         return floating_ips['floatingips']
402     except Exception as e:
403         logger.error("Error [get_floating_ips(neutron_client)]: %s" % e)
404         return None
405
406
407 def get_hypervisors(nova_client):
408     try:
409         nodes = []
410         hypervisors = nova_client.hypervisors.list()
411         for hypervisor in hypervisors:
412             if hypervisor.state == "up":
413                 nodes.append(hypervisor.hypervisor_hostname)
414         return nodes
415     except Exception as e:
416         logger.error("Error [get_hypervisors(nova_client)]: %s" % e)
417         return None
418
419
420 def create_aggregate(nova_client, aggregate_name, av_zone):
421     try:
422         nova_client.aggregates.create(aggregate_name, av_zone)
423         return True
424     except Exception as e:
425         logger.error("Error [create_aggregate(nova_client, %s, %s)]: %s"
426                      % (aggregate_name, av_zone, e))
427         return None
428
429
430 def add_host_to_aggregate(nova_client, aggregate_name, compute_host):
431     try:
432         aggregate_id = get_aggregate_id(nova_client, aggregate_name)
433         nova_client.aggregates.add_host(aggregate_id, compute_host)
434         return True
435     except Exception as e:
436         logger.error("Error [add_host_to_aggregate(nova_client, %s, %s)]: %s"
437                      % (aggregate_name, compute_host, e))
438         return None
439
440
441 def create_aggregate_with_host(
442         nova_client, aggregate_name, av_zone, compute_host):
443     try:
444         create_aggregate(nova_client, aggregate_name, av_zone)
445         add_host_to_aggregate(nova_client, aggregate_name, compute_host)
446         return True
447     except Exception as e:
448         logger.error("Error [create_aggregate_with_host("
449                      "nova_client, %s, %s, %s)]: %s"
450                      % (aggregate_name, av_zone, compute_host, e))
451         return None
452
453
454 def create_instance(flavor_name,
455                     image_id,
456                     network_id,
457                     instance_name="functest-vm",
458                     confdrive=True,
459                     userdata=None,
460                     av_zone='',
461                     fixed_ip=None,
462                     files=None):
463     nova_client = get_nova_client()
464     try:
465         flavor = nova_client.flavors.find(name=flavor_name)
466     except:
467         flavors = nova_client.flavors.list()
468         logger.error("Error: Flavor '%s' not found. Available flavors are: "
469                      "\n%s" % (flavor_name, flavors))
470         return None
471     if fixed_ip is not None:
472         nics = {"net-id": network_id, "v4-fixed-ip": fixed_ip}
473     else:
474         nics = {"net-id": network_id}
475     if userdata is None:
476         instance = nova_client.servers.create(
477             name=instance_name,
478             flavor=flavor,
479             image=image_id,
480             nics=[nics],
481             availability_zone=av_zone,
482             files=files
483         )
484     else:
485         instance = nova_client.servers.create(
486             name=instance_name,
487             flavor=flavor,
488             image=image_id,
489             nics=[nics],
490             config_drive=confdrive,
491             userdata=userdata,
492             availability_zone=av_zone,
493             files=files
494         )
495     return instance
496
497
498 def create_instance_and_wait_for_active(flavor_name,
499                                         image_id,
500                                         network_id,
501                                         instance_name="",
502                                         config_drive=False,
503                                         userdata="",
504                                         av_zone='',
505                                         fixed_ip=None,
506                                         files=None):
507     SLEEP = 3
508     VM_BOOT_TIMEOUT = 180
509     nova_client = get_nova_client()
510     instance = create_instance(flavor_name,
511                                image_id,
512                                network_id,
513                                instance_name,
514                                config_drive,
515                                userdata,
516                                av_zone=av_zone,
517                                fixed_ip=fixed_ip,
518                                files=files)
519     count = VM_BOOT_TIMEOUT / SLEEP
520     for n in range(count, -1, -1):
521         status = get_instance_status(nova_client, instance)
522         if status is None:
523             time.sleep(SLEEP)
524             continue
525         elif status.lower() == "active":
526             return instance
527         elif status.lower() == "error":
528             logger.error("The instance %s went to ERROR status."
529                          % instance_name)
530             return None
531         time.sleep(SLEEP)
532     logger.error("Timeout booting the instance %s." % instance_name)
533     return None
534
535
536 def create_floating_ip(neutron_client):
537     extnet_id = get_external_net_id(neutron_client)
538     props = {'floating_network_id': extnet_id}
539     try:
540         ip_json = neutron_client.create_floatingip({'floatingip': props})
541         fip_addr = ip_json['floatingip']['floating_ip_address']
542         fip_id = ip_json['floatingip']['id']
543     except Exception as e:
544         logger.error("Error [create_floating_ip(neutron_client)]: %s" % e)
545         return None
546     return {'fip_addr': fip_addr, 'fip_id': fip_id}
547
548
549 def add_floating_ip(nova_client, server_id, floatingip_addr):
550     try:
551         nova_client.servers.add_floating_ip(server_id, floatingip_addr)
552         return True
553     except Exception as e:
554         logger.error("Error [add_floating_ip(nova_client, '%s', '%s')]: %s"
555                      % (server_id, floatingip_addr, e))
556         return False
557
558
559 def delete_instance(nova_client, instance_id):
560     try:
561         nova_client.servers.force_delete(instance_id)
562         return True
563     except Exception as e:
564         logger.error("Error [delete_instance(nova_client, '%s')]: %s"
565                      % (instance_id, e))
566         return False
567
568
569 def delete_floating_ip(neutron_client, floatingip_id):
570     try:
571         neutron_client.delete_floatingip(floatingip_id)
572         return True
573     except Exception as e:
574         logger.error("Error [delete_floating_ip(neutron_client, '%s')]: %s"
575                      % (floatingip_id, e))
576         return False
577
578
579 def remove_host_from_aggregate(nova_client, aggregate_name, compute_host):
580     try:
581         aggregate_id = get_aggregate_id(nova_client, aggregate_name)
582         nova_client.aggregates.remove_host(aggregate_id, compute_host)
583         return True
584     except Exception as e:
585         logger.error("Error [remove_host_from_aggregate(nova_client, %s, %s)]:"
586                      " %s" % (aggregate_name, compute_host, e))
587         return False
588
589
590 def remove_hosts_from_aggregate(nova_client, aggregate_name):
591     aggregate_id = get_aggregate_id(nova_client, aggregate_name)
592     hosts = nova_client.aggregates.get(aggregate_id).hosts
593     assert(
594         all(remove_host_from_aggregate(nova_client, aggregate_name, host)
595             for host in hosts))
596
597
598 def delete_aggregate(nova_client, aggregate_name):
599     try:
600         remove_hosts_from_aggregate(nova_client, aggregate_name)
601         nova_client.aggregates.delete(aggregate_name)
602         return True
603     except Exception as e:
604         logger.error("Error [delete_aggregate(nova_client, %s)]: %s"
605                      % (aggregate_name, e))
606         return False
607
608
609 # *********************************************
610 #   NEUTRON
611 # *********************************************
612 def get_network_list(neutron_client):
613     network_list = neutron_client.list_networks()['networks']
614     if len(network_list) == 0:
615         return None
616     else:
617         return network_list
618
619
620 def get_router_list(neutron_client):
621     router_list = neutron_client.list_routers()['routers']
622     if len(router_list) == 0:
623         return None
624     else:
625         return router_list
626
627
628 def get_port_list(neutron_client):
629     port_list = neutron_client.list_ports()['ports']
630     if len(port_list) == 0:
631         return None
632     else:
633         return port_list
634
635
636 def get_network_id(neutron_client, network_name):
637     networks = neutron_client.list_networks()['networks']
638     id = ''
639     for n in networks:
640         if n['name'] == network_name:
641             id = n['id']
642             break
643     return id
644
645
646 def get_subnet_id(neutron_client, subnet_name):
647     subnets = neutron_client.list_subnets()['subnets']
648     id = ''
649     for s in subnets:
650         if s['name'] == subnet_name:
651             id = s['id']
652             break
653     return id
654
655
656 def get_router_id(neutron_client, router_name):
657     routers = neutron_client.list_routers()['routers']
658     id = ''
659     for r in routers:
660         if r['name'] == router_name:
661             id = r['id']
662             break
663     return id
664
665
666 def get_private_net(neutron_client):
667     # Checks if there is an existing shared private network
668     networks = neutron_client.list_networks()['networks']
669     if len(networks) == 0:
670         return None
671     for net in networks:
672         if (net['router:external'] is False) and (net['shared'] is True):
673             return net
674     return None
675
676
677 def get_external_net(neutron_client):
678     if (hasattr(CONST, 'EXTERNAL_NETWORK')):
679         return CONST.__getattribute__('EXTERNAL_NETWORK')
680     for network in neutron_client.list_networks()['networks']:
681         if network['router:external']:
682             return network['name']
683     return None
684
685
686 def get_external_net_id(neutron_client):
687     if (hasattr(CONST, 'EXTERNAL_NETWORK')):
688         networks = neutron_client.list_networks(
689             name=CONST.__getattribute__('EXTERNAL_NETWORK'))
690         net_id = networks['networks'][0]['id']
691         return net_id
692     for network in neutron_client.list_networks()['networks']:
693         if network['router:external']:
694             return network['id']
695     return None
696
697
698 def check_neutron_net(neutron_client, net_name):
699     for network in neutron_client.list_networks()['networks']:
700         if network['name'] == net_name:
701             for subnet in network['subnets']:
702                 return True
703     return False
704
705
706 def create_neutron_net(neutron_client, name):
707     json_body = {'network': {'name': name,
708                              'admin_state_up': True}}
709     try:
710         network = neutron_client.create_network(body=json_body)
711         network_dict = network['network']
712         return network_dict['id']
713     except Exception as e:
714         logger.error("Error [create_neutron_net(neutron_client, '%s')]: %s"
715                      % (name, e))
716         return None
717
718
719 def create_neutron_subnet(neutron_client, name, cidr, net_id,
720                           dns=['8.8.8.8', '8.8.4.4']):
721     json_body = {'subnets': [{'name': name, 'cidr': cidr,
722                               'ip_version': 4, 'network_id': net_id,
723                               'dns_nameservers': dns}]}
724
725     try:
726         subnet = neutron_client.create_subnet(body=json_body)
727         return subnet['subnets'][0]['id']
728     except Exception as e:
729         logger.error("Error [create_neutron_subnet(neutron_client, '%s', "
730                      "'%s', '%s')]: %s" % (name, cidr, net_id, e))
731         return None
732
733
734 def create_neutron_router(neutron_client, name):
735     json_body = {'router': {'name': name, 'admin_state_up': True}}
736     try:
737         router = neutron_client.create_router(json_body)
738         return router['router']['id']
739     except Exception as e:
740         logger.error("Error [create_neutron_router(neutron_client, '%s')]: %s"
741                      % (name, e))
742         return None
743
744
745 def create_neutron_port(neutron_client, name, network_id, ip):
746     json_body = {'port': {
747                  'admin_state_up': True,
748                  'name': name,
749                  'network_id': network_id,
750                  'fixed_ips': [{"ip_address": ip}]
751                  }}
752     try:
753         port = neutron_client.create_port(body=json_body)
754         return port['port']['id']
755     except Exception as e:
756         logger.error("Error [create_neutron_port(neutron_client, '%s', '%s', "
757                      "'%s')]: %s" % (name, network_id, ip, e))
758         return None
759
760
761 def update_neutron_net(neutron_client, network_id, shared=False):
762     json_body = {'network': {'shared': shared}}
763     try:
764         neutron_client.update_network(network_id, body=json_body)
765         return True
766     except Exception as e:
767         logger.error("Error [update_neutron_net(neutron_client, '%s', '%s')]: "
768                      "%s" % (network_id, str(shared), e))
769         return False
770
771
772 def update_neutron_port(neutron_client, port_id, device_owner):
773     json_body = {'port': {
774                  'device_owner': device_owner,
775                  }}
776     try:
777         port = neutron_client.update_port(port=port_id,
778                                           body=json_body)
779         return port['port']['id']
780     except Exception as e:
781         logger.error("Error [update_neutron_port(neutron_client, '%s', '%s')]:"
782                      " %s" % (port_id, device_owner, e))
783         return None
784
785
786 def add_interface_router(neutron_client, router_id, subnet_id):
787     json_body = {"subnet_id": subnet_id}
788     try:
789         neutron_client.add_interface_router(router=router_id, body=json_body)
790         return True
791     except Exception as e:
792         logger.error("Error [add_interface_router(neutron_client, '%s', "
793                      "'%s')]: %s" % (router_id, subnet_id, e))
794         return False
795
796
797 def add_gateway_router(neutron_client, router_id):
798     ext_net_id = get_external_net_id(neutron_client)
799     router_dict = {'network_id': ext_net_id}
800     try:
801         neutron_client.add_gateway_router(router_id, router_dict)
802         return True
803     except Exception as e:
804         logger.error("Error [add_gateway_router(neutron_client, '%s')]: %s"
805                      % (router_id, e))
806         return False
807
808
809 def delete_neutron_net(neutron_client, network_id):
810     try:
811         neutron_client.delete_network(network_id)
812         return True
813     except Exception as e:
814         logger.error("Error [delete_neutron_net(neutron_client, '%s')]: %s"
815                      % (network_id, e))
816         return False
817
818
819 def delete_neutron_subnet(neutron_client, subnet_id):
820     try:
821         neutron_client.delete_subnet(subnet_id)
822         return True
823     except Exception as e:
824         logger.error("Error [delete_neutron_subnet(neutron_client, '%s')]: %s"
825                      % (subnet_id, e))
826         return False
827
828
829 def delete_neutron_router(neutron_client, router_id):
830     try:
831         neutron_client.delete_router(router=router_id)
832         return True
833     except Exception as e:
834         logger.error("Error [delete_neutron_router(neutron_client, '%s')]: %s"
835                      % (router_id, e))
836         return False
837
838
839 def delete_neutron_port(neutron_client, port_id):
840     try:
841         neutron_client.delete_port(port_id)
842         return True
843     except Exception as e:
844         logger.error("Error [delete_neutron_port(neutron_client, '%s')]: %s"
845                      % (port_id, e))
846         return False
847
848
849 def remove_interface_router(neutron_client, router_id, subnet_id):
850     json_body = {"subnet_id": subnet_id}
851     try:
852         neutron_client.remove_interface_router(router=router_id,
853                                                body=json_body)
854         return True
855     except Exception as e:
856         logger.error("Error [remove_interface_router(neutron_client, '%s', "
857                      "'%s')]: %s" % (router_id, subnet_id, e))
858         return False
859
860
861 def remove_gateway_router(neutron_client, router_id):
862     try:
863         neutron_client.remove_gateway_router(router_id)
864         return True
865     except Exception as e:
866         logger.error("Error [remove_gateway_router(neutron_client, '%s')]: %s"
867                      % (router_id, e))
868         return False
869
870
871 def create_network_full(neutron_client,
872                         net_name,
873                         subnet_name,
874                         router_name,
875                         cidr,
876                         dns=['8.8.8.8', '8.8.4.4']):
877
878     # Check if the network already exists
879     network_id = get_network_id(neutron_client, net_name)
880     subnet_id = get_subnet_id(neutron_client, subnet_name)
881     router_id = get_router_id(neutron_client, router_name)
882
883     if network_id != '' and subnet_id != '' and router_id != '':
884         logger.info("A network with name '%s' already exists..." % net_name)
885     else:
886         neutron_client.format = 'json'
887         logger.info('Creating neutron network %s...' % net_name)
888         network_id = create_neutron_net(neutron_client, net_name)
889
890         if not network_id:
891             return False
892
893         logger.debug("Network '%s' created successfully" % network_id)
894         logger.debug('Creating Subnet....')
895         subnet_id = create_neutron_subnet(neutron_client, subnet_name,
896                                           cidr, network_id, dns)
897         if not subnet_id:
898             return None
899
900         logger.debug("Subnet '%s' created successfully" % subnet_id)
901         logger.debug('Creating Router...')
902         router_id = create_neutron_router(neutron_client, router_name)
903
904         if not router_id:
905             return None
906
907         logger.debug("Router '%s' created successfully" % router_id)
908         logger.debug('Adding router to subnet...')
909
910         if not add_interface_router(neutron_client, router_id, subnet_id):
911             return None
912
913         logger.debug("Interface added successfully.")
914
915         logger.debug('Adding gateway to router...')
916         if not add_gateway_router(neutron_client, router_id):
917             return None
918
919         logger.debug("Gateway added successfully.")
920
921     network_dic = {'net_id': network_id,
922                    'subnet_id': subnet_id,
923                    'router_id': router_id}
924     return network_dic
925
926
927 def create_shared_network_full(net_name, subnt_name, router_name, subnet_cidr):
928     neutron_client = get_neutron_client()
929
930     network_dic = create_network_full(neutron_client,
931                                       net_name,
932                                       subnt_name,
933                                       router_name,
934                                       subnet_cidr)
935     if network_dic:
936         if not update_neutron_net(neutron_client,
937                                   network_dic['net_id'],
938                                   shared=True):
939             logger.error("Failed to update network %s..." % net_name)
940             return None
941         else:
942             logger.debug("Network '%s' is available..." % net_name)
943     else:
944         logger.error("Network %s creation failed" % net_name)
945         return None
946     return network_dic
947
948
949 # *********************************************
950 #   SEC GROUPS
951 # *********************************************
952
953
954 def get_security_groups(neutron_client):
955     try:
956         security_groups = neutron_client.list_security_groups()[
957             'security_groups']
958         return security_groups
959     except Exception as e:
960         logger.error("Error [get_security_groups(neutron_client)]: %s" % e)
961         return None
962
963
964 def get_security_group_id(neutron_client, sg_name):
965     security_groups = get_security_groups(neutron_client)
966     id = ''
967     for sg in security_groups:
968         if sg['name'] == sg_name:
969             id = sg['id']
970             break
971     return id
972
973
974 def create_security_group(neutron_client, sg_name, sg_description):
975     json_body = {'security_group': {'name': sg_name,
976                                     'description': sg_description}}
977     try:
978         secgroup = neutron_client.create_security_group(json_body)
979         return secgroup['security_group']
980     except Exception as e:
981         logger.error("Error [create_security_group(neutron_client, '%s', "
982                      "'%s')]: %s" % (sg_name, sg_description, e))
983         return None
984
985
986 def create_secgroup_rule(neutron_client, sg_id, direction, protocol,
987                          port_range_min=None, port_range_max=None):
988     # We create a security group in 2 steps
989     # 1 - we check the format and set the json body accordingly
990     # 2 - we call neturon client to create the security group
991
992     # Format check
993     json_body = {'security_group_rule': {'direction': direction,
994                                          'security_group_id': sg_id,
995                                          'protocol': protocol}}
996     # parameters may be
997     # - both None => we do nothing
998     # - both Not None => we add them to the json description
999     # but one cannot be None is the other is not None
1000     if (port_range_min is not None and port_range_max is not None):
1001         # add port_range in json description
1002         json_body['security_group_rule']['port_range_min'] = port_range_min
1003         json_body['security_group_rule']['port_range_max'] = port_range_max
1004         logger.debug("Security_group format set (port range included)")
1005     else:
1006         # either both port range are set to None => do nothing
1007         # or one is set but not the other => log it and return False
1008         if port_range_min is None and port_range_max is None:
1009             logger.debug("Security_group format set (no port range mentioned)")
1010         else:
1011             logger.error("Bad security group format."
1012                          "One of the port range is not properly set:"
1013                          "range min: {},"
1014                          "range max: {}".format(port_range_min,
1015                                                 port_range_max))
1016             return False
1017
1018     # Create security group using neutron client
1019     try:
1020         neutron_client.create_security_group_rule(json_body)
1021         return True
1022     except:
1023         logger.exception("Impossible to create_security_group_rule,"
1024                          "security group rule probably already exists")
1025         return False
1026
1027
1028 def get_security_group_rules(neutron_client, sg_id):
1029     try:
1030         security_rules = neutron_client.list_security_group_rules()[
1031             'security_group_rules']
1032         security_rules = [rule for rule in security_rules
1033                           if rule["security_group_id"] == sg_id]
1034         return security_rules
1035     except Exception as e:
1036         logger.error("Error [get_security_group_rules(neutron_client, sg_id)]:"
1037                      " %s" % e)
1038         return None
1039
1040
1041 def check_security_group_rules(neutron_client, sg_id, direction, protocol,
1042                                port_min=None, port_max=None):
1043     try:
1044         security_rules = get_security_group_rules(neutron_client, sg_id)
1045         security_rules = [rule for rule in security_rules
1046                           if (rule["direction"].lower() == direction and
1047                               rule["protocol"].lower() == protocol and
1048                               rule["port_range_min"] == port_min and
1049                               rule["port_range_max"] == port_max)]
1050         if len(security_rules) == 0:
1051             return True
1052         else:
1053             return False
1054     except Exception as e:
1055         logger.error("Error [check_security_group_rules("
1056                      " neutron_client, sg_id, direction,"
1057                      " protocol, port_min=None, port_max=None)]: "
1058                      "%s" % e)
1059         return None
1060
1061
1062 def create_security_group_full(neutron_client,
1063                                sg_name, sg_description):
1064     sg_id = get_security_group_id(neutron_client, sg_name)
1065     if sg_id != '':
1066         logger.info("Using existing security group '%s'..." % sg_name)
1067     else:
1068         logger.info("Creating security group  '%s'..." % sg_name)
1069         SECGROUP = create_security_group(neutron_client,
1070                                          sg_name,
1071                                          sg_description)
1072         if not SECGROUP:
1073             logger.error("Failed to create the security group...")
1074             return None
1075
1076         sg_id = SECGROUP['id']
1077
1078         logger.debug("Security group '%s' with ID=%s created successfully."
1079                      % (SECGROUP['name'], sg_id))
1080
1081         logger.debug("Adding ICMP rules in security group '%s'..."
1082                      % sg_name)
1083         if not create_secgroup_rule(neutron_client, sg_id,
1084                                     'ingress', 'icmp'):
1085             logger.error("Failed to create the security group rule...")
1086             return None
1087
1088         logger.debug("Adding SSH rules in security group '%s'..."
1089                      % sg_name)
1090         if not create_secgroup_rule(
1091                 neutron_client, sg_id, 'ingress', 'tcp', '22', '22'):
1092             logger.error("Failed to create the security group rule...")
1093             return None
1094
1095         if not create_secgroup_rule(
1096                 neutron_client, sg_id, 'egress', 'tcp', '22', '22'):
1097             logger.error("Failed to create the security group rule...")
1098             return None
1099     return sg_id
1100
1101
1102 def add_secgroup_to_instance(nova_client, instance_id, secgroup_id):
1103     try:
1104         nova_client.servers.add_security_group(instance_id, secgroup_id)
1105         return True
1106     except Exception as e:
1107         logger.error("Error [add_secgroup_to_instance(nova_client, '%s', "
1108                      "'%s')]: %s" % (instance_id, secgroup_id, e))
1109         return False
1110
1111
1112 def update_sg_quota(neutron_client, tenant_id, sg_quota, sg_rule_quota):
1113     json_body = {"quota": {
1114         "security_group": sg_quota,
1115         "security_group_rule": sg_rule_quota
1116     }}
1117
1118     try:
1119         neutron_client.update_quota(tenant_id=tenant_id,
1120                                     body=json_body)
1121         return True
1122     except Exception as e:
1123         logger.error("Error [update_sg_quota(neutron_client, '%s', '%s', "
1124                      "'%s')]: %s" % (tenant_id, sg_quota, sg_rule_quota, e))
1125         return False
1126
1127
1128 def delete_security_group(neutron_client, secgroup_id):
1129     try:
1130         neutron_client.delete_security_group(secgroup_id)
1131         return True
1132     except Exception as e:
1133         logger.error("Error [delete_security_group(neutron_client, '%s')]: %s"
1134                      % (secgroup_id, e))
1135         return False
1136
1137
1138 # *********************************************
1139 #   GLANCE
1140 # *********************************************
1141 def get_images(glance_client):
1142     try:
1143         images = glance_client.images.list()
1144         return images
1145     except Exception as e:
1146         logger.error("Error [get_images]: %s" % e)
1147         return None
1148
1149
1150 def get_image_id(glance_client, image_name):
1151     images = glance_client.images.list()
1152     id = ''
1153     for i in images:
1154         if i.name == image_name:
1155             id = i.id
1156             break
1157     return id
1158
1159
1160 def create_glance_image(glance_client,
1161                         image_name,
1162                         file_path,
1163                         disk="qcow2",
1164                         extra_properties={},
1165                         container="bare",
1166                         public="public"):
1167     if not os.path.isfile(file_path):
1168         logger.error("Error: file %s does not exist." % file_path)
1169         return None
1170     try:
1171         image_id = get_image_id(glance_client, image_name)
1172         if image_id != '':
1173             logger.info("Image %s already exists." % image_name)
1174         else:
1175             logger.info("Creating image '%s' from '%s'..." % (image_name,
1176                                                               file_path))
1177
1178             image = glance_client.images.create(name=image_name,
1179                                                 visibility=public,
1180                                                 disk_format=disk,
1181                                                 container_format=container,
1182                                                 **extra_properties)
1183             image_id = image.id
1184             with open(file_path) as image_data:
1185                 glance_client.images.upload(image_id, image_data)
1186         return image_id
1187     except Exception as e:
1188         logger.error("Error [create_glance_image(glance_client, '%s', '%s', "
1189                      "'%s')]: %s" % (image_name, file_path, public, e))
1190         return None
1191
1192
1193 def get_or_create_image(name, path, format, extra_properties):
1194     image_exists = False
1195     glance_client = get_glance_client()
1196
1197     image_id = get_image_id(glance_client, name)
1198     if image_id != '':
1199         logger.info("Using existing image '%s'..." % name)
1200         image_exists = True
1201     else:
1202         logger.info("Creating image '%s' from '%s'..." % (name, path))
1203         image_id = create_glance_image(glance_client,
1204                                        name,
1205                                        path,
1206                                        format,
1207                                        extra_properties)
1208         if not image_id:
1209             logger.error("Failed to create a Glance image...")
1210         else:
1211             logger.debug("Image '%s' with ID=%s created successfully."
1212                          % (name, image_id))
1213
1214     return image_exists, image_id
1215
1216
1217 def delete_glance_image(glance_client, image_id):
1218     try:
1219         glance_client.images.delete(image_id)
1220         return True
1221     except Exception as e:
1222         logger.error("Error [delete_glance_image(glance_client, '%s')]: %s"
1223                      % (image_id, e))
1224         return False
1225
1226
1227 # *********************************************
1228 #   CINDER
1229 # *********************************************
1230 def get_volumes(cinder_client):
1231     try:
1232         volumes = cinder_client.volumes.list(search_opts={'all_tenants': 1})
1233         return volumes
1234     except Exception as e:
1235         logger.error("Error [get_volumes(cinder_client)]: %s" % e)
1236         return None
1237
1238
1239 def update_cinder_quota(cinder_client, tenant_id, vols_quota,
1240                         snapshots_quota, gigabytes_quota):
1241     quotas_values = {"volumes": vols_quota,
1242                      "snapshots": snapshots_quota,
1243                      "gigabytes": gigabytes_quota}
1244
1245     try:
1246         cinder_client.quotas.update(tenant_id, **quotas_values)
1247         return True
1248     except Exception as e:
1249         logger.error("Error [update_cinder_quota(cinder_client, '%s', '%s', "
1250                      "'%s' '%s')]: %s" % (tenant_id, vols_quota,
1251                                           snapshots_quota, gigabytes_quota, e))
1252         return False
1253
1254
1255 def delete_volume(cinder_client, volume_id, forced=False):
1256     try:
1257         if forced:
1258             try:
1259                 cinder_client.volumes.detach(volume_id)
1260             except:
1261                 logger.error(sys.exc_info()[0])
1262             cinder_client.volumes.force_delete(volume_id)
1263         else:
1264             cinder_client.volumes.delete(volume_id)
1265         return True
1266     except Exception as e:
1267         logger.error("Error [delete_volume(cinder_client, '%s', '%s')]: %s"
1268                      % (volume_id, str(forced), e))
1269         return False
1270
1271
1272 # *********************************************
1273 #   KEYSTONE
1274 # *********************************************
1275 def get_tenants(keystone_client):
1276     try:
1277         if is_keystone_v3():
1278             tenants = keystone_client.projects.list()
1279         else:
1280             tenants = keystone_client.tenants.list()
1281         return tenants
1282     except Exception as e:
1283         logger.error("Error [get_tenants(keystone_client)]: %s" % e)
1284         return None
1285
1286
1287 def get_users(keystone_client):
1288     try:
1289         users = keystone_client.users.list()
1290         return users
1291     except Exception as e:
1292         logger.error("Error [get_users(keystone_client)]: %s" % e)
1293         return None
1294
1295
1296 def get_tenant_id(keystone_client, tenant_name):
1297     tenants = get_tenants(keystone_client)
1298     id = ''
1299     for t in tenants:
1300         if t.name == tenant_name:
1301             id = t.id
1302             break
1303     return id
1304
1305
1306 def get_user_id(keystone_client, user_name):
1307     users = get_users(keystone_client)
1308     id = ''
1309     for u in users:
1310         if u.name == user_name:
1311             id = u.id
1312             break
1313     return id
1314
1315
1316 def get_role_id(keystone_client, role_name):
1317     roles = keystone_client.roles.list()
1318     id = ''
1319     for r in roles:
1320         if r.name == role_name:
1321             id = r.id
1322             break
1323     return id
1324
1325
1326 def get_domain_id(keystone_client, domain_name):
1327     domains = keystone_client.domains.list()
1328     id = ''
1329     for d in domains:
1330         if d.name == domain_name:
1331             id = d.id
1332             break
1333     return id
1334
1335
1336 def create_tenant(keystone_client, tenant_name, tenant_description):
1337     try:
1338         if is_keystone_v3():
1339             domain_name = CONST.__getattribute__('OS_PROJECT_DOMAIN_NAME')
1340             domain_id = get_domain_id(keystone_client, domain_name)
1341             tenant = keystone_client.projects.create(
1342                 name=tenant_name,
1343                 description=tenant_description,
1344                 domain=domain_id,
1345                 enabled=True)
1346         else:
1347             tenant = keystone_client.tenants.create(tenant_name,
1348                                                     tenant_description,
1349                                                     enabled=True)
1350         return tenant.id
1351     except Exception as e:
1352         logger.error("Error [create_tenant(keystone_client, '%s', '%s')]: %s"
1353                      % (tenant_name, tenant_description, e))
1354         return None
1355
1356
1357 def get_or_create_tenant(keystone_client, tenant_name, tenant_description):
1358     tenant_id = get_tenant_id(keystone_client, tenant_name)
1359     if not tenant_id:
1360         tenant_id = create_tenant(keystone_client, tenant_name,
1361                                   tenant_description)
1362
1363     return tenant_id
1364
1365
1366 def get_or_create_tenant_for_vnf(keystone_client, tenant_name,
1367                                  tenant_description):
1368     """Get or Create a Tenant
1369
1370         Args:
1371             keystone_client: keystone client reference
1372             tenant_name: the name of the tenant
1373             tenant_description: the description of the tenant
1374
1375         return False if tenant retrieved though get
1376         return True if tenant created
1377         raise Exception if error during processing
1378     """
1379     try:
1380         tenant_id = get_tenant_id(keystone_client, tenant_name)
1381         if not tenant_id:
1382             tenant_id = create_tenant(keystone_client, tenant_name,
1383                                       tenant_description)
1384             return True
1385         else:
1386             return False
1387     except:
1388         raise Exception("Impossible to create a Tenant for the VNF {}".format(
1389                             tenant_name))
1390
1391
1392 def create_user(keystone_client, user_name, user_password,
1393                 user_email, tenant_id):
1394     try:
1395         if is_keystone_v3():
1396             user = keystone_client.users.create(name=user_name,
1397                                                 password=user_password,
1398                                                 email=user_email,
1399                                                 project_id=tenant_id,
1400                                                 enabled=True)
1401         else:
1402             user = keystone_client.users.create(user_name,
1403                                                 user_password,
1404                                                 user_email,
1405                                                 tenant_id,
1406                                                 enabled=True)
1407         return user.id
1408     except Exception as e:
1409         logger.error("Error [create_user(keystone_client, '%s', '%s', '%s'"
1410                      "'%s')]: %s" % (user_name, user_password,
1411                                      user_email, tenant_id, e))
1412         return None
1413
1414
1415 def get_or_create_user(keystone_client, user_name, user_password,
1416                        tenant_id, user_email=None):
1417     user_id = get_user_id(keystone_client, user_name)
1418     if not user_id:
1419         user_id = create_user(keystone_client, user_name, user_password,
1420                               user_email, tenant_id)
1421     return user_id
1422
1423
1424 def get_or_create_user_for_vnf(keystone_client, vnf_ref):
1425     """Get or Create user for VNF
1426
1427         Args:
1428             keystone_client: keystone client reference
1429             vnf_ref: VNF reference used as user name & password, tenant name
1430
1431         return False if user retrieved through get
1432         return True if user created
1433         raise Exception if error during processing
1434     """
1435     try:
1436         user_id = get_user_id(keystone_client, vnf_ref)
1437         tenant_id = get_tenant_id(keystone_client, vnf_ref)
1438         created = False
1439         if not user_id:
1440             user_id = create_user(keystone_client, vnf_ref, vnf_ref,
1441                                   "", tenant_id)
1442             created = True
1443         try:
1444             role_id = get_role_id(keystone_client, 'admin')
1445             tenant_id = get_tenant_id(keystone_client, vnf_ref)
1446             add_role_user(keystone_client, user_id, role_id, tenant_id)
1447         except:
1448             logger.warn("Cannot associate user to role admin on tenant")
1449         return created
1450     except:
1451         raise Exception("Impossible to create a user for the VNF {}".format(
1452             vnf_ref))
1453
1454
1455 def add_role_user(keystone_client, user_id, role_id, tenant_id):
1456     try:
1457         if is_keystone_v3():
1458             keystone_client.roles.grant(role=role_id,
1459                                         user=user_id,
1460                                         project=tenant_id)
1461         else:
1462             keystone_client.roles.add_user_role(user_id, role_id, tenant_id)
1463         return True
1464     except Exception as e:
1465         logger.error("Error [add_role_user(keystone_client, '%s', '%s'"
1466                      "'%s')]: %s " % (user_id, role_id, tenant_id, e))
1467         return False
1468
1469
1470 def delete_tenant(keystone_client, tenant_id):
1471     try:
1472         if is_keystone_v3():
1473             keystone_client.projects.delete(tenant_id)
1474         else:
1475             keystone_client.tenants.delete(tenant_id)
1476         return True
1477     except Exception as e:
1478         logger.error("Error [delete_tenant(keystone_client, '%s')]: %s"
1479                      % (tenant_id, e))
1480         return False
1481
1482
1483 def delete_user(keystone_client, user_id):
1484     try:
1485         keystone_client.users.delete(user_id)
1486         return True
1487     except Exception as e:
1488         logger.error("Error [delete_user(keystone_client, '%s')]: %s"
1489                      % (user_id, e))
1490         return False
1491
1492
1493 # *********************************************
1494 #   HEAT
1495 # *********************************************
1496 def get_resource(heat_client, stack_id, resource):
1497     try:
1498         resources = heat_client.resources.get(stack_id, resource)
1499         return resources
1500     except Exception as e:
1501         logger.error("Error [get_resource]: %s" % e)
1502         return None