Enable https insecure for functest
[functest.git] / functest / utils / openstack_utils.py
1 #!/usr/bin/env python
2 #
3 # jose.lausuch@ericsson.com
4 # valentin.boucher@orange.com
5 # All rights reserved. This program and the accompanying materials
6 # are made available under the terms of the Apache License, Version 2.0
7 # which accompanies this distribution, and is available at
8 # http://www.apache.org/licenses/LICENSE-2.0
9 #
10
11 import logging
12 import os.path
13 import re
14 import sys
15 import time
16
17 from keystoneauth1 import loading
18 from keystoneauth1 import session
19 from cinderclient import client as cinderclient
20 from glanceclient import client as glanceclient
21 from heatclient import client as heatclient
22 from novaclient import client as novaclient
23 from keystoneclient import client as keystoneclient
24 from neutronclient.neutron import client as neutronclient
25
26 import functest.utils.functest_utils as ft_utils
27
28 logger = logging.getLogger(__name__)
29
30 DEFAULT_API_VERSION = '2'
31 DEFAULT_HEAT_API_VERSION = '1'
32
33
34 # *********************************************
35 #   CREDENTIALS
36 # *********************************************
37 class MissingEnvVar(Exception):
38
39     def __init__(self, var):
40         self.var = var
41
42     def __str__(self):
43         return str.format("Please set the mandatory env var: {}", self.var)
44
45
46 def is_keystone_v3():
47     keystone_api_version = os.getenv('OS_IDENTITY_API_VERSION')
48     if (keystone_api_version is None or
49             keystone_api_version == '2'):
50         return False
51     else:
52         return True
53
54
55 def get_rc_env_vars():
56     env_vars = ['OS_AUTH_URL', 'OS_USERNAME', 'OS_PASSWORD']
57     if is_keystone_v3():
58         env_vars.extend(['OS_PROJECT_NAME',
59                          'OS_USER_DOMAIN_NAME',
60                          'OS_PROJECT_DOMAIN_NAME'])
61     else:
62         env_vars.extend(['OS_TENANT_NAME'])
63     return env_vars
64
65
66 def check_credentials():
67     """
68     Check if the OpenStack credentials (openrc) are sourced
69     """
70     env_vars = get_rc_env_vars()
71     return all(map(lambda v: v in os.environ and os.environ[v], env_vars))
72
73
74 def get_env_cred_dict():
75     env_cred_dict = {
76         'OS_USERNAME': 'username',
77         'OS_PASSWORD': 'password',
78         'OS_AUTH_URL': 'auth_url',
79         'OS_TENANT_NAME': 'tenant_name',
80         'OS_USER_DOMAIN_NAME': 'user_domain_name',
81         'OS_PROJECT_DOMAIN_NAME': 'project_domain_name',
82         'OS_PROJECT_NAME': 'project_name',
83         'OS_ENDPOINT_TYPE': 'endpoint_type',
84         'OS_REGION_NAME': 'region_name',
85         'OS_CACERT': 'https_cacert'
86     }
87     return env_cred_dict
88
89
90 def get_credentials(other_creds={}):
91     """Returns a creds dictionary filled with parsed from env
92     """
93     creds = {}
94     env_vars = get_rc_env_vars()
95     env_cred_dict = get_env_cred_dict()
96
97     for envvar in env_vars:
98         if os.getenv(envvar) is None:
99             raise MissingEnvVar(envvar)
100         else:
101             creds_key = env_cred_dict.get(envvar)
102             creds.update({creds_key: os.getenv(envvar)})
103
104     if 'tenant' in other_creds.keys():
105         if is_keystone_v3():
106             tenant = 'project_name'
107         else:
108             tenant = 'tenant_name'
109         other_creds[tenant] = other_creds.pop('tenant')
110
111     creds.update(other_creds)
112
113     return creds
114
115
116 def source_credentials(rc_file):
117     with open(rc_file, "r") as f:
118         for line in f:
119             var = line.rstrip('"\n').replace('export ', '').split("=")
120             # The two next lines should be modified as soon as rc_file
121             # conforms with common rules. Be aware that it could induce
122             # issues if value starts with '
123             key = re.sub(r'^["\' ]*|[ \'"]*$', '', var[0])
124             value = re.sub(r'^["\' ]*|[ \'"]*$', '', "".join(var[1:]))
125             os.environ[key] = value
126
127
128 def get_credentials_for_rally():
129     creds = get_credentials()
130     env_cred_dict = get_env_cred_dict()
131     rally_conf = {"type": "ExistingCloud", "admin": {}}
132     for key in creds:
133         if key == 'auth_url':
134             rally_conf[key] = creds[key]
135         else:
136             rally_conf['admin'][key] = creds[key]
137
138     endpoint_types = [('internalURL', 'internal'),
139                       ('publicURL', 'public'), ('adminURL', 'admin')]
140
141     endpoint_type = get_endpoint_type_from_env()
142     if endpoint_type is not None:
143         cred_key = env_cred_dict.get('OS_ENDPOINT_TYPE')
144         for k, v in endpoint_types:
145             if endpoint_type == v:
146                 rally_conf[cred_key] = v
147
148     region_name = os.getenv('OS_REGION_NAME')
149     if region_name is not None:
150         cred_key = env_cred_dict.get('OS_REGION_NAME')
151         rally_conf[cred_key] = region_name
152
153     cacert = os.getenv('OS_CACERT')
154     if cacert is not None:
155         cred_key = env_cred_dict.get('OS_CACERT')
156         rally_conf[cred_key] = cacert
157     return rally_conf
158
159
160 def get_endpoint_type_from_env():
161     endpoint_type = os.environ.get("OS_ENDPOINT_TYPE",
162                                    os.environ.get("OS_INTERFACE"))
163     if endpoint_type and "URL" in endpoint_type:
164         endpoint_type = endpoint_type.replace("URL", "")
165     return endpoint_type
166
167
168 def get_session_auth(other_creds={}):
169     loader = loading.get_plugin_loader('password')
170     creds = get_credentials(other_creds)
171     auth = loader.load_from_options(**creds)
172     return auth
173
174
175 def get_endpoint(service_type, endpoint_type='publicURL'):
176     auth = get_session_auth()
177     return get_session().get_endpoint(auth=auth,
178                                       service_type=service_type,
179                                       endpoint_type=endpoint_type)
180
181
182 def get_session(other_creds={}):
183     auth = get_session_auth(other_creds)
184     https_cacert = os.getenv('OS_CACERT', '')
185     https_insecure = os.getenv('OS_INSECURE', '').lower() == 'true'
186     return session.Session(auth=auth,
187                            verify=(https_cacert or not https_insecure))
188
189
190 # *********************************************
191 #   CLIENTS
192 # *********************************************
193 def get_keystone_client_version():
194     api_version = os.getenv('OS_IDENTITY_API_VERSION')
195     if api_version is not None:
196         logger.info("OS_IDENTITY_API_VERSION is set in env as '%s'",
197                     api_version)
198         return api_version
199     return DEFAULT_API_VERSION
200
201
202 def get_keystone_client(other_creds={}):
203     sess = get_session(other_creds)
204     return keystoneclient.Client(get_keystone_client_version(),
205                                  session=sess,
206                                  interface=os.getenv('OS_INTERFACE', 'admin'))
207
208
209 def get_nova_client_version():
210     api_version = os.getenv('OS_COMPUTE_API_VERSION')
211     if api_version is not None:
212         logger.info("OS_COMPUTE_API_VERSION is set in env as '%s'",
213                     api_version)
214         return api_version
215     return DEFAULT_API_VERSION
216
217
218 def get_nova_client(other_creds={}):
219     sess = get_session(other_creds)
220     return novaclient.Client(get_nova_client_version(), session=sess)
221
222
223 def get_cinder_client_version():
224     api_version = os.getenv('OS_VOLUME_API_VERSION')
225     if api_version is not None:
226         logger.info("OS_VOLUME_API_VERSION is set in env as '%s'",
227                     api_version)
228         return api_version
229     return DEFAULT_API_VERSION
230
231
232 def get_cinder_client(other_creds={}):
233     sess = get_session(other_creds)
234     return cinderclient.Client(get_cinder_client_version(), session=sess)
235
236
237 def get_neutron_client_version():
238     api_version = os.getenv('OS_NETWORK_API_VERSION')
239     if api_version is not None:
240         logger.info("OS_NETWORK_API_VERSION is set in env as '%s'",
241                     api_version)
242         return api_version
243     return DEFAULT_API_VERSION
244
245
246 def get_neutron_client(other_creds={}):
247     sess = get_session(other_creds)
248     return neutronclient.Client(get_neutron_client_version(), session=sess)
249
250
251 def get_glance_client_version():
252     api_version = os.getenv('OS_IMAGE_API_VERSION')
253     if api_version is not None:
254         logger.info("OS_IMAGE_API_VERSION is set in env as '%s'", api_version)
255         return api_version
256     return DEFAULT_API_VERSION
257
258
259 def get_glance_client(other_creds={}):
260     sess = get_session(other_creds)
261     return glanceclient.Client(get_glance_client_version(), session=sess)
262
263
264 def get_heat_client_version():
265     api_version = os.getenv('OS_ORCHESTRATION_API_VERSION')
266     if api_version is not None:
267         logger.info("OS_ORCHESTRATION_API_VERSION is set in env as '%s'",
268                     api_version)
269         return api_version
270     return DEFAULT_HEAT_API_VERSION
271
272
273 def get_heat_client(other_creds={}):
274     sess = get_session(other_creds)
275     return heatclient.Client(get_heat_client_version(), session=sess)
276
277
278 # *********************************************
279 #   NOVA
280 # *********************************************
281 def get_instances(nova_client):
282     try:
283         instances = nova_client.servers.list(search_opts={'all_tenants': 1})
284         return instances
285     except Exception as e:
286         logger.error("Error [get_instances(nova_client)]: %s" % e)
287         return None
288
289
290 def get_instance_status(nova_client, instance):
291     try:
292         instance = nova_client.servers.get(instance.id)
293         return instance.status
294     except Exception as e:
295         logger.error("Error [get_instance_status(nova_client)]: %s" % e)
296         return None
297
298
299 def get_instance_by_name(nova_client, instance_name):
300     try:
301         instance = nova_client.servers.find(name=instance_name)
302         return instance
303     except Exception as e:
304         logger.error("Error [get_instance_by_name(nova_client, '%s')]: %s"
305                      % (instance_name, e))
306         return None
307
308
309 def get_flavor_id(nova_client, flavor_name):
310     flavors = nova_client.flavors.list(detailed=True)
311     id = ''
312     for f in flavors:
313         if f.name == flavor_name:
314             id = f.id
315             break
316     return id
317
318
319 def get_flavor_id_by_ram_range(nova_client, min_ram, max_ram):
320     flavors = nova_client.flavors.list(detailed=True)
321     id = ''
322     for f in flavors:
323         if min_ram <= f.ram and f.ram <= max_ram:
324             id = f.id
325             break
326     return id
327
328
329 def get_aggregates(nova_client):
330     try:
331         aggregates = nova_client.aggregates.list()
332         return aggregates
333     except Exception as e:
334         logger.error("Error [get_aggregates(nova_client)]: %s" % e)
335         return None
336
337
338 def get_aggregate_id(nova_client, aggregate_name):
339     try:
340         aggregates = get_aggregates(nova_client)
341         _id = [ag.id for ag in aggregates if ag.name == aggregate_name][0]
342         return _id
343     except Exception as e:
344         logger.error("Error [get_aggregate_id(nova_client, %s)]:"
345                      " %s" % (aggregate_name, e))
346         return None
347
348
349 def get_availability_zones(nova_client):
350     try:
351         availability_zones = nova_client.availability_zones.list()
352         return availability_zones
353     except Exception as e:
354         logger.error("Error [get_availability_zones(nova_client)]: %s" % e)
355         return None
356
357
358 def get_availability_zone_names(nova_client):
359     try:
360         az_names = [az.zoneName for az in get_availability_zones(nova_client)]
361         return az_names
362     except Exception as e:
363         logger.error("Error [get_availability_zone_names(nova_client)]:"
364                      " %s" % e)
365         return None
366
367
368 def create_flavor(nova_client, flavor_name, ram, disk, vcpus, public=True):
369     try:
370         flavor = nova_client.flavors.create(
371             flavor_name, ram, vcpus, disk, is_public=public)
372         try:
373             extra_specs = ft_utils.get_functest_config(
374                 'general.flavor_extra_specs')
375             flavor.set_keys(extra_specs)
376         except ValueError:
377             # flavor extra specs are not configured, therefore skip the update
378             pass
379
380     except Exception as e:
381         logger.error("Error [create_flavor(nova_client, '%s', '%s', '%s', "
382                      "'%s')]: %s" % (flavor_name, ram, disk, vcpus, e))
383         return None
384     return flavor.id
385
386
387 def get_or_create_flavor(flavor_name, ram, disk, vcpus, public=True):
388     flavor_exists = False
389     nova_client = get_nova_client()
390
391     flavor_id = get_flavor_id(nova_client, flavor_name)
392     if flavor_id != '':
393         logger.info("Using existing flavor '%s'..." % flavor_name)
394         flavor_exists = True
395     else:
396         logger.info("Creating flavor '%s' with '%s' RAM, '%s' disk size, "
397                     "'%s' vcpus..." % (flavor_name, ram, disk, vcpus))
398         flavor_id = create_flavor(
399             nova_client, flavor_name, ram, disk, vcpus, public=public)
400         if not flavor_id:
401             logger.error("Failed to create flavor '%s'..." % (flavor_name))
402         else:
403             logger.debug("Flavor '%s' with ID=%s created successfully."
404                          % (flavor_name, flavor_id))
405
406     return flavor_exists, flavor_id
407
408
409 def get_floating_ips(nova_client):
410     try:
411         floating_ips = nova_client.floating_ips.list()
412         return floating_ips
413     except Exception as e:
414         logger.error("Error [get_floating_ips(nova_client)]: %s" % e)
415         return None
416
417
418 def get_hypervisors(nova_client):
419     try:
420         nodes = []
421         hypervisors = nova_client.hypervisors.list()
422         for hypervisor in hypervisors:
423             if hypervisor.state == "up":
424                 nodes.append(hypervisor.hypervisor_hostname)
425         return nodes
426     except Exception as e:
427         logger.error("Error [get_hypervisors(nova_client)]: %s" % e)
428         return None
429
430
431 def create_aggregate(nova_client, aggregate_name, av_zone):
432     try:
433         nova_client.aggregates.create(aggregate_name, av_zone)
434         return True
435     except Exception as e:
436         logger.error("Error [create_aggregate(nova_client, %s, %s)]: %s"
437                      % (aggregate_name, av_zone, e))
438         return None
439
440
441 def add_host_to_aggregate(nova_client, aggregate_name, compute_host):
442     try:
443         aggregate_id = get_aggregate_id(nova_client, aggregate_name)
444         nova_client.aggregates.add_host(aggregate_id, compute_host)
445         return True
446     except Exception as e:
447         logger.error("Error [add_host_to_aggregate(nova_client, %s, %s)]: %s"
448                      % (aggregate_name, compute_host, e))
449         return None
450
451
452 def create_aggregate_with_host(
453         nova_client, aggregate_name, av_zone, compute_host):
454     try:
455         create_aggregate(nova_client, aggregate_name, av_zone)
456         add_host_to_aggregate(nova_client, aggregate_name, compute_host)
457         return True
458     except Exception as e:
459         logger.error("Error [create_aggregate_with_host("
460                      "nova_client, %s, %s, %s)]: %s"
461                      % (aggregate_name, av_zone, compute_host, e))
462         return None
463
464
465 def create_instance(flavor_name,
466                     image_id,
467                     network_id,
468                     instance_name="functest-vm",
469                     confdrive=True,
470                     userdata=None,
471                     av_zone='',
472                     fixed_ip=None,
473                     files=None):
474     nova_client = get_nova_client()
475     try:
476         flavor = nova_client.flavors.find(name=flavor_name)
477     except:
478         flavors = nova_client.flavors.list()
479         logger.error("Error: Flavor '%s' not found. Available flavors are: "
480                      "\n%s" % (flavor_name, flavors))
481         return None
482     if fixed_ip is not None:
483         nics = {"net-id": network_id, "v4-fixed-ip": fixed_ip}
484     else:
485         nics = {"net-id": network_id}
486     if userdata is None:
487         instance = nova_client.servers.create(
488             name=instance_name,
489             flavor=flavor,
490             image=image_id,
491             nics=[nics],
492             availability_zone=av_zone,
493             files=files
494         )
495     else:
496         instance = nova_client.servers.create(
497             name=instance_name,
498             flavor=flavor,
499             image=image_id,
500             nics=[nics],
501             config_drive=confdrive,
502             userdata=userdata,
503             availability_zone=av_zone,
504             files=files
505         )
506     return instance
507
508
509 def create_instance_and_wait_for_active(flavor_name,
510                                         image_id,
511                                         network_id,
512                                         instance_name="",
513                                         config_drive=False,
514                                         userdata="",
515                                         av_zone='',
516                                         fixed_ip=None,
517                                         files=None):
518     SLEEP = 3
519     VM_BOOT_TIMEOUT = 180
520     nova_client = get_nova_client()
521     instance = create_instance(flavor_name,
522                                image_id,
523                                network_id,
524                                instance_name,
525                                config_drive,
526                                userdata,
527                                av_zone=av_zone,
528                                fixed_ip=fixed_ip,
529                                files=files)
530     count = VM_BOOT_TIMEOUT / SLEEP
531     for n in range(count, -1, -1):
532         status = get_instance_status(nova_client, instance)
533         if status.lower() == "active":
534             return instance
535         elif status.lower() == "error":
536             logger.error("The instance %s went to ERROR status."
537                          % instance_name)
538             return None
539         time.sleep(SLEEP)
540     logger.error("Timeout booting the instance %s." % instance_name)
541     return None
542
543
544 def create_floating_ip(neutron_client):
545     extnet_id = get_external_net_id(neutron_client)
546     props = {'floating_network_id': extnet_id}
547     try:
548         ip_json = neutron_client.create_floatingip({'floatingip': props})
549         fip_addr = ip_json['floatingip']['floating_ip_address']
550         fip_id = ip_json['floatingip']['id']
551     except Exception as e:
552         logger.error("Error [create_floating_ip(neutron_client)]: %s" % e)
553         return None
554     return {'fip_addr': fip_addr, 'fip_id': fip_id}
555
556
557 def add_floating_ip(nova_client, server_id, floatingip_addr):
558     try:
559         nova_client.servers.add_floating_ip(server_id, floatingip_addr)
560         return True
561     except Exception as e:
562         logger.error("Error [add_floating_ip(nova_client, '%s', '%s')]: %s"
563                      % (server_id, floatingip_addr, e))
564         return False
565
566
567 def delete_instance(nova_client, instance_id):
568     try:
569         nova_client.servers.force_delete(instance_id)
570         return True
571     except Exception as e:
572         logger.error("Error [delete_instance(nova_client, '%s')]: %s"
573                      % (instance_id, e))
574         return False
575
576
577 def delete_floating_ip(nova_client, floatingip_id):
578     try:
579         nova_client.floating_ips.delete(floatingip_id)
580         return True
581     except Exception as e:
582         logger.error("Error [delete_floating_ip(nova_client, '%s')]: %s"
583                      % (floatingip_id, e))
584         return False
585
586
587 def remove_host_from_aggregate(nova_client, aggregate_name, compute_host):
588     try:
589         aggregate_id = get_aggregate_id(nova_client, aggregate_name)
590         nova_client.aggregates.remove_host(aggregate_id, compute_host)
591         return True
592     except Exception as e:
593         logger.error("Error [remove_host_from_aggregate(nova_client, %s, %s)]:"
594                      " %s" % (aggregate_name, compute_host, e))
595         return False
596
597
598 def remove_hosts_from_aggregate(nova_client, aggregate_name):
599     aggregate_id = get_aggregate_id(nova_client, aggregate_name)
600     hosts = nova_client.aggregates.get(aggregate_id).hosts
601     assert(
602         all(remove_host_from_aggregate(nova_client, aggregate_name, host)
603             for host in hosts))
604
605
606 def delete_aggregate(nova_client, aggregate_name):
607     try:
608         remove_hosts_from_aggregate(nova_client, aggregate_name)
609         nova_client.aggregates.delete(aggregate_name)
610         return True
611     except Exception as e:
612         logger.error("Error [delete_aggregate(nova_client, %s)]: %s"
613                      % (aggregate_name, e))
614         return False
615
616
617 # *********************************************
618 #   NEUTRON
619 # *********************************************
620 def get_network_list(neutron_client):
621     network_list = neutron_client.list_networks()['networks']
622     if len(network_list) == 0:
623         return None
624     else:
625         return network_list
626
627
628 def get_router_list(neutron_client):
629     router_list = neutron_client.list_routers()['routers']
630     if len(router_list) == 0:
631         return None
632     else:
633         return router_list
634
635
636 def get_port_list(neutron_client):
637     port_list = neutron_client.list_ports()['ports']
638     if len(port_list) == 0:
639         return None
640     else:
641         return port_list
642
643
644 def get_network_id(neutron_client, network_name):
645     networks = neutron_client.list_networks()['networks']
646     id = ''
647     for n in networks:
648         if n['name'] == network_name:
649             id = n['id']
650             break
651     return id
652
653
654 def get_subnet_id(neutron_client, subnet_name):
655     subnets = neutron_client.list_subnets()['subnets']
656     id = ''
657     for s in subnets:
658         if s['name'] == subnet_name:
659             id = s['id']
660             break
661     return id
662
663
664 def get_router_id(neutron_client, router_name):
665     routers = neutron_client.list_routers()['routers']
666     id = ''
667     for r in routers:
668         if r['name'] == router_name:
669             id = r['id']
670             break
671     return id
672
673
674 def get_private_net(neutron_client):
675     # Checks if there is an existing shared private network
676     networks = neutron_client.list_networks()['networks']
677     if len(networks) == 0:
678         return None
679     for net in networks:
680         if (net['router:external'] is False) and (net['shared'] is True):
681             return net
682     return None
683
684
685 def get_external_net(neutron_client):
686     for network in neutron_client.list_networks()['networks']:
687         if network['router:external']:
688             return network['name']
689     return None
690
691
692 def get_external_net_id(neutron_client):
693     for network in neutron_client.list_networks()['networks']:
694         if network['router:external']:
695             return network['id']
696     return None
697
698
699 def check_neutron_net(neutron_client, net_name):
700     for network in neutron_client.list_networks()['networks']:
701         if network['name'] == net_name:
702             for subnet in network['subnets']:
703                 return True
704     return False
705
706
707 def create_neutron_net(neutron_client, name):
708     json_body = {'network': {'name': name,
709                              'admin_state_up': True}}
710     try:
711         network = neutron_client.create_network(body=json_body)
712         network_dict = network['network']
713         return network_dict['id']
714     except Exception as e:
715         logger.error("Error [create_neutron_net(neutron_client, '%s')]: %s"
716                      % (name, e))
717         return None
718
719
720 def create_neutron_subnet(neutron_client, name, cidr, net_id):
721     json_body = {'subnets': [{'name': name, 'cidr': cidr,
722                               'ip_version': 4, 'network_id': net_id}]}
723     try:
724         subnet = neutron_client.create_subnet(body=json_body)
725         return subnet['subnets'][0]['id']
726     except Exception as e:
727         logger.error("Error [create_neutron_subnet(neutron_client, '%s', "
728                      "'%s', '%s')]: %s" % (name, cidr, net_id, e))
729         return None
730
731
732 def create_neutron_router(neutron_client, name):
733     json_body = {'router': {'name': name, 'admin_state_up': True}}
734     try:
735         router = neutron_client.create_router(json_body)
736         return router['router']['id']
737     except Exception as e:
738         logger.error("Error [create_neutron_router(neutron_client, '%s')]: %s"
739                      % (name, e))
740         return None
741
742
743 def create_neutron_port(neutron_client, name, network_id, ip):
744     json_body = {'port': {
745                  'admin_state_up': True,
746                  'name': name,
747                  'network_id': network_id,
748                  'fixed_ips': [{"ip_address": ip}]
749                  }}
750     try:
751         port = neutron_client.create_port(body=json_body)
752         return port['port']['id']
753     except Exception as e:
754         logger.error("Error [create_neutron_port(neutron_client, '%s', '%s', "
755                      "'%s')]: %s" % (name, network_id, ip, e))
756         return None
757
758
759 def update_neutron_net(neutron_client, network_id, shared=False):
760     json_body = {'network': {'shared': shared}}
761     try:
762         neutron_client.update_network(network_id, body=json_body)
763         return True
764     except Exception as e:
765         logger.error("Error [update_neutron_net(neutron_client, '%s', '%s')]: "
766                      "%s" % (network_id, str(shared), e))
767         return False
768
769
770 def update_neutron_port(neutron_client, port_id, device_owner):
771     json_body = {'port': {
772                  'device_owner': device_owner,
773                  }}
774     try:
775         port = neutron_client.update_port(port=port_id,
776                                           body=json_body)
777         return port['port']['id']
778     except Exception as e:
779         logger.error("Error [update_neutron_port(neutron_client, '%s', '%s')]:"
780                      " %s" % (port_id, device_owner, e))
781         return None
782
783
784 def add_interface_router(neutron_client, router_id, subnet_id):
785     json_body = {"subnet_id": subnet_id}
786     try:
787         neutron_client.add_interface_router(router=router_id, body=json_body)
788         return True
789     except Exception as e:
790         logger.error("Error [add_interface_router(neutron_client, '%s', "
791                      "'%s')]: %s" % (router_id, subnet_id, e))
792         return False
793
794
795 def add_gateway_router(neutron_client, router_id):
796     ext_net_id = get_external_net_id(neutron_client)
797     router_dict = {'network_id': ext_net_id}
798     try:
799         neutron_client.add_gateway_router(router_id, router_dict)
800         return True
801     except Exception as e:
802         logger.error("Error [add_gateway_router(neutron_client, '%s')]: %s"
803                      % (router_id, e))
804         return False
805
806
807 def delete_neutron_net(neutron_client, network_id):
808     try:
809         neutron_client.delete_network(network_id)
810         return True
811     except Exception as e:
812         logger.error("Error [delete_neutron_net(neutron_client, '%s')]: %s"
813                      % (network_id, e))
814         return False
815
816
817 def delete_neutron_subnet(neutron_client, subnet_id):
818     try:
819         neutron_client.delete_subnet(subnet_id)
820         return True
821     except Exception as e:
822         logger.error("Error [delete_neutron_subnet(neutron_client, '%s')]: %s"
823                      % (subnet_id, e))
824         return False
825
826
827 def delete_neutron_router(neutron_client, router_id):
828     try:
829         neutron_client.delete_router(router=router_id)
830         return True
831     except Exception as e:
832         logger.error("Error [delete_neutron_router(neutron_client, '%s')]: %s"
833                      % (router_id, e))
834         return False
835
836
837 def delete_neutron_port(neutron_client, port_id):
838     try:
839         neutron_client.delete_port(port_id)
840         return True
841     except Exception as e:
842         logger.error("Error [delete_neutron_port(neutron_client, '%s')]: %s"
843                      % (port_id, e))
844         return False
845
846
847 def remove_interface_router(neutron_client, router_id, subnet_id):
848     json_body = {"subnet_id": subnet_id}
849     try:
850         neutron_client.remove_interface_router(router=router_id,
851                                                body=json_body)
852         return True
853     except Exception as e:
854         logger.error("Error [remove_interface_router(neutron_client, '%s', "
855                      "'%s')]: %s" % (router_id, subnet_id, e))
856         return False
857
858
859 def remove_gateway_router(neutron_client, router_id):
860     try:
861         neutron_client.remove_gateway_router(router_id)
862         return True
863     except Exception as e:
864         logger.error("Error [remove_gateway_router(neutron_client, '%s')]: %s"
865                      % (router_id, e))
866         return False
867
868
869 def create_network_full(neutron_client,
870                         net_name,
871                         subnet_name,
872                         router_name,
873                         cidr):
874
875     # Check if the network already exists
876     network_id = get_network_id(neutron_client, net_name)
877     subnet_id = get_subnet_id(neutron_client, subnet_name)
878     router_id = get_router_id(neutron_client, router_name)
879
880     if network_id != '' and subnet_id != '' and router_id != '':
881         logger.info("A network with name '%s' already exists..." % net_name)
882     else:
883         neutron_client.format = 'json'
884         logger.info('Creating neutron network %s...' % net_name)
885         network_id = create_neutron_net(neutron_client, net_name)
886
887         if not network_id:
888             return False
889
890         logger.debug("Network '%s' created successfully" % network_id)
891         logger.debug('Creating Subnet....')
892         subnet_id = create_neutron_subnet(neutron_client, subnet_name,
893                                           cidr, network_id)
894         if not subnet_id:
895             return None
896
897         logger.debug("Subnet '%s' created successfully" % subnet_id)
898         logger.debug('Creating Router...')
899         router_id = create_neutron_router(neutron_client, router_name)
900
901         if not router_id:
902             return None
903
904         logger.debug("Router '%s' created successfully" % router_id)
905         logger.debug('Adding router to subnet...')
906
907         if not add_interface_router(neutron_client, router_id, subnet_id):
908             return None
909
910         logger.debug("Interface added successfully.")
911
912         logger.debug('Adding gateway to router...')
913         if not add_gateway_router(neutron_client, router_id):
914             return None
915
916         logger.debug("Gateway added successfully.")
917
918     network_dic = {'net_id': network_id,
919                    'subnet_id': subnet_id,
920                    'router_id': router_id}
921     return network_dic
922
923
924 def create_shared_network_full(net_name, subnt_name, router_name, subnet_cidr):
925     neutron_client = get_neutron_client()
926
927     network_dic = create_network_full(neutron_client,
928                                       net_name,
929                                       subnt_name,
930                                       router_name,
931                                       subnet_cidr)
932     if network_dic:
933         if not update_neutron_net(neutron_client,
934                                   network_dic['net_id'],
935                                   shared=True):
936             logger.error("Failed to update network %s..." % net_name)
937             return None
938         else:
939             logger.debug("Network '%s' is available..." % net_name)
940     else:
941         logger.error("Network %s creation failed" % net_name)
942         return None
943     return network_dic
944
945
946 def create_bgpvpn(neutron_client, **kwargs):
947     # route_distinguishers
948     # route_targets
949     json_body = {"bgpvpn": kwargs}
950     return neutron_client.create_bgpvpn(json_body)
951
952
953 def create_network_association(neutron_client, bgpvpn_id, neutron_network_id):
954     json_body = {"network_association": {"network_id": neutron_network_id}}
955     return neutron_client.create_network_association(bgpvpn_id, json_body)
956
957
958 def create_router_association(neutron_client, bgpvpn_id, router_id):
959     json_body = {"router_association": {"router_id": router_id}}
960     return neutron_client.create_router_association(bgpvpn_id, json_body)
961
962
963 def update_bgpvpn(neutron_client, bgpvpn_id, **kwargs):
964     json_body = {"bgpvpn": kwargs}
965     return neutron_client.update_bgpvpn(bgpvpn_id, json_body)
966
967
968 def delete_bgpvpn(neutron_client, bgpvpn_id):
969     return neutron_client.delete_bgpvpn(bgpvpn_id)
970
971
972 def get_bgpvpn(neutron_client, bgpvpn_id):
973     return neutron_client.show_bgpvpn(bgpvpn_id)
974
975
976 def get_bgpvpn_routers(neutron_client, bgpvpn_id):
977     return get_bgpvpn(neutron_client, bgpvpn_id)['bgpvpn']['routers']
978
979
980 def get_bgpvpn_networks(neutron_client, bgpvpn_id):
981     return get_bgpvpn(neutron_client, bgpvpn_id)['bgpvpn']['networks']
982
983 # *********************************************
984 #   SEC GROUPS
985 # *********************************************
986
987
988 def get_security_groups(neutron_client):
989     try:
990         security_groups = neutron_client.list_security_groups()[
991             'security_groups']
992         return security_groups
993     except Exception as e:
994         logger.error("Error [get_security_groups(neutron_client)]: %s" % e)
995         return None
996
997
998 def get_security_group_id(neutron_client, sg_name):
999     security_groups = get_security_groups(neutron_client)
1000     id = ''
1001     for sg in security_groups:
1002         if sg['name'] == sg_name:
1003             id = sg['id']
1004             break
1005     return id
1006
1007
1008 def create_security_group(neutron_client, sg_name, sg_description):
1009     json_body = {'security_group': {'name': sg_name,
1010                                     'description': sg_description}}
1011     try:
1012         secgroup = neutron_client.create_security_group(json_body)
1013         return secgroup['security_group']
1014     except Exception as e:
1015         logger.error("Error [create_security_group(neutron_client, '%s', "
1016                      "'%s')]: %s" % (sg_name, sg_description, e))
1017         return None
1018
1019
1020 def create_secgroup_rule(neutron_client, sg_id, direction, protocol,
1021                          port_range_min=None, port_range_max=None):
1022     # We create a security group in 2 steps
1023     # 1 - we check the format and set the json body accordingly
1024     # 2 - we call neturon client to create the security group
1025
1026     # Format check
1027     json_body = {'security_group_rule': {'direction': direction,
1028                                          'security_group_id': sg_id,
1029                                          'protocol': protocol}}
1030     # parameters may be
1031     # - both None => we do nothing
1032     # - both Not None => we add them to the json description
1033     # but one cannot be None is the other is not None
1034     if (port_range_min is not None and port_range_max is not None):
1035         # add port_range in json description
1036         json_body['security_group_rule']['port_range_min'] = port_range_min
1037         json_body['security_group_rule']['port_range_max'] = port_range_max
1038         logger.debug("Security_group format set (port range included)")
1039     else:
1040         # either both port range are set to None => do nothing
1041         # or one is set but not the other => log it and return False
1042         if port_range_min is None and port_range_max is None:
1043             logger.debug("Security_group format set (no port range mentioned)")
1044         else:
1045             logger.error("Bad security group format."
1046                          "One of the port range is not properly set:"
1047                          "range min: {},"
1048                          "range max: {}".format(port_range_min,
1049                                                 port_range_max))
1050             return False
1051
1052     # Create security group using neutron client
1053     try:
1054         neutron_client.create_security_group_rule(json_body)
1055         return True
1056     except:
1057         logger.exception("Impossible to create_security_group_rule,"
1058                          "security group rule probably already exists")
1059         return False
1060
1061
1062 def get_security_group_rules(neutron_client, sg_id):
1063     try:
1064         security_rules = neutron_client.list_security_group_rules()[
1065             'security_group_rules']
1066         security_rules = [rule for rule in security_rules
1067                           if rule["security_group_id"] == sg_id]
1068         return security_rules
1069     except Exception as e:
1070         logger.error("Error [get_security_group_rules(neutron_client, sg_id)]:"
1071                      " %s" % e)
1072         return None
1073
1074
1075 def check_security_group_rules(neutron_client, sg_id, direction, protocol,
1076                                port_min=None, port_max=None):
1077     try:
1078         security_rules = get_security_group_rules(neutron_client, sg_id)
1079         security_rules = [rule for rule in security_rules
1080                           if (rule["direction"].lower() == direction and
1081                               rule["protocol"].lower() == protocol and
1082                               rule["port_range_min"] == port_min and
1083                               rule["port_range_max"] == port_max)]
1084         if len(security_rules) == 0:
1085             return True
1086         else:
1087             return False
1088     except Exception as e:
1089         logger.error("Error [check_security_group_rules("
1090                      " neutron_client, sg_id, direction,"
1091                      " protocol, port_min=None, port_max=None)]: "
1092                      "%s" % e)
1093         return None
1094
1095
1096 def create_security_group_full(neutron_client,
1097                                sg_name, sg_description):
1098     sg_id = get_security_group_id(neutron_client, sg_name)
1099     if sg_id != '':
1100         logger.info("Using existing security group '%s'..." % sg_name)
1101     else:
1102         logger.info("Creating security group  '%s'..." % sg_name)
1103         SECGROUP = create_security_group(neutron_client,
1104                                          sg_name,
1105                                          sg_description)
1106         if not SECGROUP:
1107             logger.error("Failed to create the security group...")
1108             return None
1109
1110         sg_id = SECGROUP['id']
1111
1112         logger.debug("Security group '%s' with ID=%s created successfully."
1113                      % (SECGROUP['name'], sg_id))
1114
1115         logger.debug("Adding ICMP rules in security group '%s'..."
1116                      % sg_name)
1117         if not create_secgroup_rule(neutron_client, sg_id,
1118                                     'ingress', 'icmp'):
1119             logger.error("Failed to create the security group rule...")
1120             return None
1121
1122         logger.debug("Adding SSH rules in security group '%s'..."
1123                      % sg_name)
1124         if not create_secgroup_rule(
1125                 neutron_client, sg_id, 'ingress', 'tcp', '22', '22'):
1126             logger.error("Failed to create the security group rule...")
1127             return None
1128
1129         if not create_secgroup_rule(
1130                 neutron_client, sg_id, 'egress', 'tcp', '22', '22'):
1131             logger.error("Failed to create the security group rule...")
1132             return None
1133     return sg_id
1134
1135
1136 def add_secgroup_to_instance(nova_client, instance_id, secgroup_id):
1137     try:
1138         nova_client.servers.add_security_group(instance_id, secgroup_id)
1139         return True
1140     except Exception as e:
1141         logger.error("Error [add_secgroup_to_instance(nova_client, '%s', "
1142                      "'%s')]: %s" % (instance_id, secgroup_id, e))
1143         return False
1144
1145
1146 def update_sg_quota(neutron_client, tenant_id, sg_quota, sg_rule_quota):
1147     json_body = {"quota": {
1148         "security_group": sg_quota,
1149         "security_group_rule": sg_rule_quota
1150     }}
1151
1152     try:
1153         neutron_client.update_quota(tenant_id=tenant_id,
1154                                     body=json_body)
1155         return True
1156     except Exception as e:
1157         logger.error("Error [update_sg_quota(neutron_client, '%s', '%s', "
1158                      "'%s')]: %s" % (tenant_id, sg_quota, sg_rule_quota, e))
1159         return False
1160
1161
1162 def delete_security_group(neutron_client, secgroup_id):
1163     try:
1164         neutron_client.delete_security_group(secgroup_id)
1165         return True
1166     except Exception as e:
1167         logger.error("Error [delete_security_group(neutron_client, '%s')]: %s"
1168                      % (secgroup_id, e))
1169         return False
1170
1171
1172 # *********************************************
1173 #   GLANCE
1174 # *********************************************
1175 def get_images(nova_client):
1176     try:
1177         images = nova_client.images.list()
1178         return images
1179     except Exception as e:
1180         logger.error("Error [get_images]: %s" % e)
1181         return None
1182
1183
1184 def get_image_id(glance_client, image_name):
1185     images = glance_client.images.list()
1186     id = ''
1187     for i in images:
1188         if i.name == image_name:
1189             id = i.id
1190             break
1191     return id
1192
1193
1194 def create_glance_image(glance_client, image_name, file_path, disk="qcow2",
1195                         container="bare", public="public"):
1196     if not os.path.isfile(file_path):
1197         logger.error("Error: file %s does not exist." % file_path)
1198         return None
1199     try:
1200         image_id = get_image_id(glance_client, image_name)
1201         if image_id != '':
1202             logger.info("Image %s already exists." % image_name)
1203         else:
1204             logger.info("Creating image '%s' from '%s'..." % (image_name,
1205                                                               file_path))
1206
1207             image = glance_client.images.create(name=image_name,
1208                                                 visibility=public,
1209                                                 disk_format=disk,
1210                                                 container_format=container)
1211             image_id = image.id
1212             with open(file_path) as image_data:
1213                 glance_client.images.upload(image_id, image_data)
1214         return image_id
1215     except Exception as e:
1216         logger.error("Error [create_glance_image(glance_client, '%s', '%s', "
1217                      "'%s')]: %s" % (image_name, file_path, public, e))
1218         return None
1219
1220
1221 def get_or_create_image(name, path, format):
1222     image_exists = False
1223     glance_client = get_glance_client()
1224
1225     image_id = get_image_id(glance_client, name)
1226     if image_id != '':
1227         logger.info("Using existing image '%s'..." % name)
1228         image_exists = True
1229     else:
1230         logger.info("Creating image '%s' from '%s'..." % (name, path))
1231         image_id = create_glance_image(glance_client, name, path, format)
1232         if not image_id:
1233             logger.error("Failed to create a Glance image...")
1234         else:
1235             logger.debug("Image '%s' with ID=%s created successfully."
1236                          % (name, image_id))
1237
1238     return image_exists, image_id
1239
1240
1241 def delete_glance_image(nova_client, image_id):
1242     try:
1243         nova_client.images.delete(image_id)
1244         return True
1245     except Exception as e:
1246         logger.error("Error [delete_glance_image(nova_client, '%s')]: %s"
1247                      % (image_id, e))
1248         return False
1249
1250
1251 # *********************************************
1252 #   CINDER
1253 # *********************************************
1254 def get_volumes(cinder_client):
1255     try:
1256         volumes = cinder_client.volumes.list(search_opts={'all_tenants': 1})
1257         return volumes
1258     except Exception as e:
1259         logger.error("Error [get_volumes(cinder_client)]: %s" % e)
1260         return None
1261
1262
1263 def list_volume_types(cinder_client, public=True, private=True):
1264     try:
1265         volume_types = cinder_client.volume_types.list()
1266         if not public:
1267             volume_types = [vt for vt in volume_types if not vt.is_public]
1268         if not private:
1269             volume_types = [vt for vt in volume_types if vt.is_public]
1270         return volume_types
1271     except Exception as e:
1272         logger.error("Error [list_volume_types(cinder_client)]: %s" % e)
1273         return None
1274
1275
1276 def create_volume_type(cinder_client, name):
1277     try:
1278         volume_type = cinder_client.volume_types.create(name)
1279         return volume_type
1280     except Exception as e:
1281         logger.error("Error [create_volume_type(cinder_client, '%s')]: %s"
1282                      % (name, e))
1283         return None
1284
1285
1286 def update_cinder_quota(cinder_client, tenant_id, vols_quota,
1287                         snapshots_quota, gigabytes_quota):
1288     quotas_values = {"volumes": vols_quota,
1289                      "snapshots": snapshots_quota,
1290                      "gigabytes": gigabytes_quota}
1291
1292     try:
1293         cinder_client.quotas.update(tenant_id, **quotas_values)
1294         return True
1295     except Exception as e:
1296         logger.error("Error [update_cinder_quota(cinder_client, '%s', '%s', "
1297                      "'%s' '%s')]: %s" % (tenant_id, vols_quota,
1298                                           snapshots_quota, gigabytes_quota, e))
1299         return False
1300
1301
1302 def delete_volume(cinder_client, volume_id, forced=False):
1303     try:
1304         if forced:
1305             try:
1306                 cinder_client.volumes.detach(volume_id)
1307             except:
1308                 logger.error(sys.exc_info()[0])
1309             cinder_client.volumes.force_delete(volume_id)
1310         else:
1311             cinder_client.volumes.delete(volume_id)
1312         return True
1313     except Exception as e:
1314         logger.error("Error [delete_volume(cinder_client, '%s', '%s')]: %s"
1315                      % (volume_id, str(forced), e))
1316         return False
1317
1318
1319 def delete_volume_type(cinder_client, volume_type):
1320     try:
1321         cinder_client.volume_types.delete(volume_type)
1322         return True
1323     except Exception as e:
1324         logger.error("Error [delete_volume_type(cinder_client, '%s')]: %s"
1325                      % (volume_type, e))
1326         return False
1327
1328
1329 # *********************************************
1330 #   KEYSTONE
1331 # *********************************************
1332 def get_tenants(keystone_client):
1333     try:
1334         if is_keystone_v3():
1335             tenants = keystone_client.projects.list()
1336         else:
1337             tenants = keystone_client.tenants.list()
1338         return tenants
1339     except Exception as e:
1340         logger.error("Error [get_tenants(keystone_client)]: %s" % e)
1341         return None
1342
1343
1344 def get_users(keystone_client):
1345     try:
1346         users = keystone_client.users.list()
1347         return users
1348     except Exception as e:
1349         logger.error("Error [get_users(keystone_client)]: %s" % e)
1350         return None
1351
1352
1353 def get_tenant_id(keystone_client, tenant_name):
1354     tenants = get_tenants(keystone_client)
1355     id = ''
1356     for t in tenants:
1357         if t.name == tenant_name:
1358             id = t.id
1359             break
1360     return id
1361
1362
1363 def get_user_id(keystone_client, user_name):
1364     users = get_users(keystone_client)
1365     id = ''
1366     for u in users:
1367         if u.name == user_name:
1368             id = u.id
1369             break
1370     return id
1371
1372
1373 def get_role_id(keystone_client, role_name):
1374     roles = keystone_client.roles.list()
1375     id = ''
1376     for r in roles:
1377         if r.name == role_name:
1378             id = r.id
1379             break
1380     return id
1381
1382
1383 def create_tenant(keystone_client, tenant_name, tenant_description):
1384     try:
1385         if is_keystone_v3():
1386             tenant = keystone_client.projects.create(
1387                 name=tenant_name,
1388                 description=tenant_description,
1389                 domain="default",
1390                 enabled=True)
1391         else:
1392             tenant = keystone_client.tenants.create(tenant_name,
1393                                                     tenant_description,
1394                                                     enabled=True)
1395         return tenant.id
1396     except Exception as e:
1397         logger.error("Error [create_tenant(keystone_client, '%s', '%s')]: %s"
1398                      % (tenant_name, tenant_description, e))
1399         return None
1400
1401
1402 def get_or_create_tenant(keystone_client, tenant_name, tenant_description):
1403     tenant_id = get_tenant_id(keystone_client, tenant_name)
1404     if not tenant_id:
1405         tenant_id = create_tenant(keystone_client, tenant_name,
1406                                   tenant_description)
1407
1408     return tenant_id
1409
1410
1411 def create_user(keystone_client, user_name, user_password,
1412                 user_email, tenant_id):
1413     try:
1414         if is_keystone_v3():
1415             user = keystone_client.users.create(name=user_name,
1416                                                 password=user_password,
1417                                                 email=user_email,
1418                                                 project_id=tenant_id,
1419                                                 enabled=True)
1420         else:
1421             user = keystone_client.users.create(user_name,
1422                                                 user_password,
1423                                                 user_email,
1424                                                 tenant_id,
1425                                                 enabled=True)
1426         return user.id
1427     except Exception as e:
1428         logger.error("Error [create_user(keystone_client, '%s', '%s', '%s'"
1429                      "'%s')]: %s" % (user_name, user_password,
1430                                      user_email, tenant_id, e))
1431         return None
1432
1433
1434 def get_or_create_user(keystone_client, user_name, user_password,
1435                        tenant_id, user_email=None):
1436     user_id = get_user_id(keystone_client, user_name)
1437     if not user_id:
1438         user_id = create_user(keystone_client, user_name, user_password,
1439                               user_email, tenant_id)
1440     return user_id
1441
1442
1443 def add_role_user(keystone_client, user_id, role_id, tenant_id):
1444     try:
1445         if is_keystone_v3():
1446             keystone_client.roles.grant(role=role_id,
1447                                         user=user_id,
1448                                         project=tenant_id)
1449         else:
1450             keystone_client.roles.add_user_role(user_id, role_id, tenant_id)
1451         return True
1452     except Exception as e:
1453         logger.error("Error [add_role_user(keystone_client, '%s', '%s'"
1454                      "'%s')]: %s " % (user_id, role_id, tenant_id, e))
1455         return False
1456
1457
1458 def delete_tenant(keystone_client, tenant_id):
1459     try:
1460         if is_keystone_v3():
1461             keystone_client.projects.delete(tenant_id)
1462         else:
1463             keystone_client.tenants.delete(tenant_id)
1464         return True
1465     except Exception as e:
1466         logger.error("Error [delete_tenant(keystone_client, '%s')]: %s"
1467                      % (tenant_id, e))
1468         return False
1469
1470
1471 def delete_user(keystone_client, user_id):
1472     try:
1473         keystone_client.users.delete(user_id)
1474         return True
1475     except Exception as e:
1476         logger.error("Error [delete_user(keystone_client, '%s')]: %s"
1477                      % (user_id, e))
1478         return False
1479
1480
1481 # *********************************************
1482 #   HEAT
1483 # *********************************************
1484 def get_resource(heat_client, stack_id, resource):
1485     try:
1486         resources = heat_client.resources.get(stack_id, resource)
1487         return resources
1488     except Exception as e:
1489         logger.error("Error [get_resource]: %s" % e)
1490         return None