Re-Enable Promise testcases
[functest.git] / functest / utils / openstack_utils.py
1 #!/usr/bin/env python
2 #
3 # jose.lausuch@ericsson.com
4 # valentin.boucher@orange.com
5 # All rights reserved. This program and the accompanying materials
6 # are made available under the terms of the Apache License, Version 2.0
7 # which accompanies this distribution, and is available at
8 # http://www.apache.org/licenses/LICENSE-2.0
9 #
10
11 import logging
12 import os.path
13 import re
14 import sys
15 import time
16
17 from keystoneauth1 import loading
18 from keystoneauth1 import session
19 from cinderclient import client as cinderclient
20 from glanceclient import client as glanceclient
21 from heatclient import client as heatclient
22 from novaclient import client as novaclient
23 from keystoneclient import client as keystoneclient
24 from neutronclient.neutron import client as neutronclient
25
26 import functest.utils.functest_utils as ft_utils
27
28 logger = logging.getLogger(__name__)
29
30 DEFAULT_API_VERSION = '2'
31 DEFAULT_HEAT_API_VERSION = '1'
32
33
34 # *********************************************
35 #   CREDENTIALS
36 # *********************************************
37 class MissingEnvVar(Exception):
38
39     def __init__(self, var):
40         self.var = var
41
42     def __str__(self):
43         return str.format("Please set the mandatory env var: {}", self.var)
44
45
46 def is_keystone_v3():
47     keystone_api_version = os.getenv('OS_IDENTITY_API_VERSION')
48     if (keystone_api_version is None or
49             keystone_api_version == '2'):
50         return False
51     else:
52         return True
53
54
55 def get_rc_env_vars():
56     env_vars = ['OS_AUTH_URL', 'OS_USERNAME', 'OS_PASSWORD']
57     if is_keystone_v3():
58         env_vars.extend(['OS_PROJECT_NAME',
59                          'OS_USER_DOMAIN_NAME',
60                          'OS_PROJECT_DOMAIN_NAME'])
61     else:
62         env_vars.extend(['OS_TENANT_NAME'])
63     return env_vars
64
65
66 def check_credentials():
67     """
68     Check if the OpenStack credentials (openrc) are sourced
69     """
70     env_vars = get_rc_env_vars()
71     return all(map(lambda v: v in os.environ and os.environ[v], env_vars))
72
73
74 def get_env_cred_dict():
75     env_cred_dict = {
76         'OS_USERNAME': 'username',
77         'OS_PASSWORD': 'password',
78         'OS_AUTH_URL': 'auth_url',
79         'OS_TENANT_NAME': 'tenant_name',
80         'OS_USER_DOMAIN_NAME': 'user_domain_name',
81         'OS_PROJECT_DOMAIN_NAME': 'project_domain_name',
82         'OS_PROJECT_NAME': 'project_name',
83         'OS_ENDPOINT_TYPE': 'endpoint_type',
84         'OS_REGION_NAME': 'region_name',
85         'OS_CACERT': 'https_cacert'
86     }
87     return env_cred_dict
88
89
90 def get_credentials(other_creds={}):
91     """Returns a creds dictionary filled with parsed from env
92     """
93     creds = {}
94     env_vars = get_rc_env_vars()
95     env_cred_dict = get_env_cred_dict()
96
97     for envvar in env_vars:
98         if os.getenv(envvar) is None:
99             raise MissingEnvVar(envvar)
100         else:
101             creds_key = env_cred_dict.get(envvar)
102             creds.update({creds_key: os.getenv(envvar)})
103
104     if 'tenant' in other_creds.keys():
105         if is_keystone_v3():
106             tenant = 'project_name'
107         else:
108             tenant = 'tenant_name'
109         other_creds[tenant] = other_creds.pop('tenant')
110
111     creds.update(other_creds)
112
113     return creds
114
115
116 def source_credentials(rc_file):
117     with open(rc_file, "r") as f:
118         for line in f:
119             var = line.rstrip('"\n').replace('export ', '').split("=")
120             # The two next lines should be modified as soon as rc_file
121             # conforms with common rules. Be aware that it could induce
122             # issues if value starts with '
123             key = re.sub(r'^["\' ]*|[ \'"]*$', '', var[0])
124             value = re.sub(r'^["\' ]*|[ \'"]*$', '', "".join(var[1:]))
125             os.environ[key] = value
126
127
128 def get_credentials_for_rally():
129     creds = get_credentials()
130     env_cred_dict = get_env_cred_dict()
131     rally_conf = {"type": "ExistingCloud", "admin": {}}
132     for key in creds:
133         if key == 'auth_url':
134             rally_conf[key] = creds[key]
135         else:
136             rally_conf['admin'][key] = creds[key]
137
138     endpoint_types = [('internalURL', 'internal'),
139                       ('publicURL', 'public'), ('adminURL', 'admin')]
140
141     endpoint_type = get_endpoint_type_from_env()
142     if endpoint_type is not None:
143         cred_key = env_cred_dict.get('OS_ENDPOINT_TYPE')
144         for k, v in endpoint_types:
145             if endpoint_type == v:
146                 rally_conf[cred_key] = v
147
148     region_name = os.getenv('OS_REGION_NAME')
149     if region_name is not None:
150         cred_key = env_cred_dict.get('OS_REGION_NAME')
151         rally_conf[cred_key] = region_name
152
153     cacert = os.getenv('OS_CACERT')
154     if cacert is not None:
155         cred_key = env_cred_dict.get('OS_CACERT')
156         rally_conf[cred_key] = cacert
157     return rally_conf
158
159
160 def get_endpoint_type_from_env():
161     endpoint_type = os.environ.get("OS_ENDPOINT_TYPE",
162                                    os.environ.get("OS_INTERFACE"))
163     if endpoint_type and "URL" in endpoint_type:
164         endpoint_type = endpoint_type.replace("URL", "")
165     return endpoint_type
166
167
168 def get_session_auth(other_creds={}):
169     loader = loading.get_plugin_loader('password')
170     creds = get_credentials(other_creds)
171     auth = loader.load_from_options(**creds)
172     return auth
173
174
175 def get_endpoint(service_type, endpoint_type='publicURL'):
176     auth = get_session_auth()
177     return get_session().get_endpoint(auth=auth,
178                                       service_type=service_type,
179                                       endpoint_type=endpoint_type)
180
181
182 def get_session(other_creds={}):
183     auth = get_session_auth(other_creds)
184     cacert = os.getenv('OS_CACERT')
185     if cacert is not None:
186         if not os.path.isfile(cacert):
187             raise Exception("The 'OS_CACERT' environment"
188                             "variable is set to %s but the file"
189                             "does not exist.", cacert)
190
191     return session.Session(auth=auth, verify=cacert)
192
193
194 # *********************************************
195 #   CLIENTS
196 # *********************************************
197 def get_keystone_client_version():
198     api_version = os.getenv('OS_IDENTITY_API_VERSION')
199     if api_version is not None:
200         logger.info("OS_IDENTITY_API_VERSION is set in env as '%s'",
201                     api_version)
202         return api_version
203     return DEFAULT_API_VERSION
204
205
206 def get_keystone_client(other_creds={}):
207     sess = get_session(other_creds)
208     return keystoneclient.Client(get_keystone_client_version(),
209                                  session=sess,
210                                  interface=os.getenv('OS_INTERFACE', 'admin'))
211
212
213 def get_nova_client_version():
214     api_version = os.getenv('OS_COMPUTE_API_VERSION')
215     if api_version is not None:
216         logger.info("OS_COMPUTE_API_VERSION is set in env as '%s'",
217                     api_version)
218         return api_version
219     return DEFAULT_API_VERSION
220
221
222 def get_nova_client(other_creds={}):
223     sess = get_session(other_creds)
224     return novaclient.Client(get_nova_client_version(), session=sess)
225
226
227 def get_cinder_client_version():
228     api_version = os.getenv('OS_VOLUME_API_VERSION')
229     if api_version is not None:
230         logger.info("OS_VOLUME_API_VERSION is set in env as '%s'",
231                     api_version)
232         return api_version
233     return DEFAULT_API_VERSION
234
235
236 def get_cinder_client(other_creds={}):
237     sess = get_session(other_creds)
238     return cinderclient.Client(get_cinder_client_version(), session=sess)
239
240
241 def get_neutron_client_version():
242     api_version = os.getenv('OS_NETWORK_API_VERSION')
243     if api_version is not None:
244         logger.info("OS_NETWORK_API_VERSION is set in env as '%s'",
245                     api_version)
246         return api_version
247     return DEFAULT_API_VERSION
248
249
250 def get_neutron_client(other_creds={}):
251     sess = get_session(other_creds)
252     return neutronclient.Client(get_neutron_client_version(), session=sess)
253
254
255 def get_glance_client_version():
256     api_version = os.getenv('OS_IMAGE_API_VERSION')
257     if api_version is not None:
258         logger.info("OS_IMAGE_API_VERSION is set in env as '%s'", api_version)
259         return api_version
260     return DEFAULT_API_VERSION
261
262
263 def get_glance_client(other_creds={}):
264     sess = get_session(other_creds)
265     return glanceclient.Client(get_glance_client_version(), session=sess)
266
267
268 def get_heat_client_version():
269     api_version = os.getenv('OS_ORCHESTRATION_API_VERSION')
270     if api_version is not None:
271         logger.info("OS_ORCHESTRATION_API_VERSION is set in env as '%s'",
272                     api_version)
273         return api_version
274     return DEFAULT_HEAT_API_VERSION
275
276
277 def get_heat_client(other_creds={}):
278     sess = get_session(other_creds)
279     return heatclient.Client(get_heat_client_version(), session=sess)
280
281
282 # *********************************************
283 #   NOVA
284 # *********************************************
285 def get_instances(nova_client):
286     try:
287         instances = nova_client.servers.list(search_opts={'all_tenants': 1})
288         return instances
289     except Exception as e:
290         logger.error("Error [get_instances(nova_client)]: %s" % e)
291         return None
292
293
294 def get_instance_status(nova_client, instance):
295     try:
296         instance = nova_client.servers.get(instance.id)
297         return instance.status
298     except Exception as e:
299         logger.error("Error [get_instance_status(nova_client)]: %s" % e)
300         return None
301
302
303 def get_instance_by_name(nova_client, instance_name):
304     try:
305         instance = nova_client.servers.find(name=instance_name)
306         return instance
307     except Exception as e:
308         logger.error("Error [get_instance_by_name(nova_client, '%s')]: %s"
309                      % (instance_name, e))
310         return None
311
312
313 def get_flavor_id(nova_client, flavor_name):
314     flavors = nova_client.flavors.list(detailed=True)
315     id = ''
316     for f in flavors:
317         if f.name == flavor_name:
318             id = f.id
319             break
320     return id
321
322
323 def get_flavor_id_by_ram_range(nova_client, min_ram, max_ram):
324     flavors = nova_client.flavors.list(detailed=True)
325     id = ''
326     for f in flavors:
327         if min_ram <= f.ram and f.ram <= max_ram:
328             id = f.id
329             break
330     return id
331
332
333 def get_aggregates(nova_client):
334     try:
335         aggregates = nova_client.aggregates.list()
336         return aggregates
337     except Exception as e:
338         logger.error("Error [get_aggregates(nova_client)]: %s" % e)
339         return None
340
341
342 def get_aggregate_id(nova_client, aggregate_name):
343     try:
344         aggregates = get_aggregates(nova_client)
345         _id = [ag.id for ag in aggregates if ag.name == aggregate_name][0]
346         return _id
347     except Exception as e:
348         logger.error("Error [get_aggregate_id(nova_client, %s)]:"
349                      " %s" % (aggregate_name, e))
350         return None
351
352
353 def get_availability_zones(nova_client):
354     try:
355         availability_zones = nova_client.availability_zones.list()
356         return availability_zones
357     except Exception as e:
358         logger.error("Error [get_availability_zones(nova_client)]: %s" % e)
359         return None
360
361
362 def get_availability_zone_names(nova_client):
363     try:
364         az_names = [az.zoneName for az in get_availability_zones(nova_client)]
365         return az_names
366     except Exception as e:
367         logger.error("Error [get_availability_zone_names(nova_client)]:"
368                      " %s" % e)
369         return None
370
371
372 def create_flavor(nova_client, flavor_name, ram, disk, vcpus, public=True):
373     try:
374         flavor = nova_client.flavors.create(
375             flavor_name, ram, vcpus, disk, is_public=public)
376         try:
377             extra_specs = ft_utils.get_functest_config(
378                 'general.flavor_extra_specs')
379             flavor.set_keys(extra_specs)
380         except ValueError:
381             # flavor extra specs are not configured, therefore skip the update
382             pass
383
384     except Exception as e:
385         logger.error("Error [create_flavor(nova_client, '%s', '%s', '%s', "
386                      "'%s')]: %s" % (flavor_name, ram, disk, vcpus, e))
387         return None
388     return flavor.id
389
390
391 def get_or_create_flavor(flavor_name, ram, disk, vcpus, public=True):
392     flavor_exists = False
393     nova_client = get_nova_client()
394
395     flavor_id = get_flavor_id(nova_client, flavor_name)
396     if flavor_id != '':
397         logger.info("Using existing flavor '%s'..." % flavor_name)
398         flavor_exists = True
399     else:
400         logger.info("Creating flavor '%s' with '%s' RAM, '%s' disk size, "
401                     "'%s' vcpus..." % (flavor_name, ram, disk, vcpus))
402         flavor_id = create_flavor(
403             nova_client, flavor_name, ram, disk, vcpus, public=public)
404         if not flavor_id:
405             logger.error("Failed to create flavor '%s'..." % (flavor_name))
406         else:
407             logger.debug("Flavor '%s' with ID=%s created successfully."
408                          % (flavor_name, flavor_id))
409
410     return flavor_exists, flavor_id
411
412
413 def get_floating_ips(nova_client):
414     try:
415         floating_ips = nova_client.floating_ips.list()
416         return floating_ips
417     except Exception as e:
418         logger.error("Error [get_floating_ips(nova_client)]: %s" % e)
419         return None
420
421
422 def get_hypervisors(nova_client):
423     try:
424         nodes = []
425         hypervisors = nova_client.hypervisors.list()
426         for hypervisor in hypervisors:
427             if hypervisor.state == "up":
428                 nodes.append(hypervisor.hypervisor_hostname)
429         return nodes
430     except Exception as e:
431         logger.error("Error [get_hypervisors(nova_client)]: %s" % e)
432         return None
433
434
435 def create_aggregate(nova_client, aggregate_name, av_zone):
436     try:
437         nova_client.aggregates.create(aggregate_name, av_zone)
438         return True
439     except Exception as e:
440         logger.error("Error [create_aggregate(nova_client, %s, %s)]: %s"
441                      % (aggregate_name, av_zone, e))
442         return None
443
444
445 def add_host_to_aggregate(nova_client, aggregate_name, compute_host):
446     try:
447         aggregate_id = get_aggregate_id(nova_client, aggregate_name)
448         nova_client.aggregates.add_host(aggregate_id, compute_host)
449         return True
450     except Exception as e:
451         logger.error("Error [add_host_to_aggregate(nova_client, %s, %s)]: %s"
452                      % (aggregate_name, compute_host, e))
453         return None
454
455
456 def create_aggregate_with_host(
457         nova_client, aggregate_name, av_zone, compute_host):
458     try:
459         create_aggregate(nova_client, aggregate_name, av_zone)
460         add_host_to_aggregate(nova_client, aggregate_name, compute_host)
461         return True
462     except Exception as e:
463         logger.error("Error [create_aggregate_with_host("
464                      "nova_client, %s, %s, %s)]: %s"
465                      % (aggregate_name, av_zone, compute_host, e))
466         return None
467
468
469 def create_instance(flavor_name,
470                     image_id,
471                     network_id,
472                     instance_name="functest-vm",
473                     confdrive=True,
474                     userdata=None,
475                     av_zone='',
476                     fixed_ip=None,
477                     files=None):
478     nova_client = get_nova_client()
479     try:
480         flavor = nova_client.flavors.find(name=flavor_name)
481     except:
482         flavors = nova_client.flavors.list()
483         logger.error("Error: Flavor '%s' not found. Available flavors are: "
484                      "\n%s" % (flavor_name, flavors))
485         return None
486     if fixed_ip is not None:
487         nics = {"net-id": network_id, "v4-fixed-ip": fixed_ip}
488     else:
489         nics = {"net-id": network_id}
490     if userdata is None:
491         instance = nova_client.servers.create(
492             name=instance_name,
493             flavor=flavor,
494             image=image_id,
495             nics=[nics],
496             availability_zone=av_zone,
497             files=files
498         )
499     else:
500         instance = nova_client.servers.create(
501             name=instance_name,
502             flavor=flavor,
503             image=image_id,
504             nics=[nics],
505             config_drive=confdrive,
506             userdata=userdata,
507             availability_zone=av_zone,
508             files=files
509         )
510     return instance
511
512
513 def create_instance_and_wait_for_active(flavor_name,
514                                         image_id,
515                                         network_id,
516                                         instance_name="",
517                                         config_drive=False,
518                                         userdata="",
519                                         av_zone='',
520                                         fixed_ip=None,
521                                         files=None):
522     SLEEP = 3
523     VM_BOOT_TIMEOUT = 180
524     nova_client = get_nova_client()
525     instance = create_instance(flavor_name,
526                                image_id,
527                                network_id,
528                                instance_name,
529                                config_drive,
530                                userdata,
531                                av_zone=av_zone,
532                                fixed_ip=fixed_ip,
533                                files=files)
534     count = VM_BOOT_TIMEOUT / SLEEP
535     for n in range(count, -1, -1):
536         status = get_instance_status(nova_client, instance)
537         if status.lower() == "active":
538             return instance
539         elif status.lower() == "error":
540             logger.error("The instance %s went to ERROR status."
541                          % instance_name)
542             return None
543         time.sleep(SLEEP)
544     logger.error("Timeout booting the instance %s." % instance_name)
545     return None
546
547
548 def create_floating_ip(neutron_client):
549     extnet_id = get_external_net_id(neutron_client)
550     props = {'floating_network_id': extnet_id}
551     try:
552         ip_json = neutron_client.create_floatingip({'floatingip': props})
553         fip_addr = ip_json['floatingip']['floating_ip_address']
554         fip_id = ip_json['floatingip']['id']
555     except Exception as e:
556         logger.error("Error [create_floating_ip(neutron_client)]: %s" % e)
557         return None
558     return {'fip_addr': fip_addr, 'fip_id': fip_id}
559
560
561 def add_floating_ip(nova_client, server_id, floatingip_addr):
562     try:
563         nova_client.servers.add_floating_ip(server_id, floatingip_addr)
564         return True
565     except Exception as e:
566         logger.error("Error [add_floating_ip(nova_client, '%s', '%s')]: %s"
567                      % (server_id, floatingip_addr, e))
568         return False
569
570
571 def delete_instance(nova_client, instance_id):
572     try:
573         nova_client.servers.force_delete(instance_id)
574         return True
575     except Exception as e:
576         logger.error("Error [delete_instance(nova_client, '%s')]: %s"
577                      % (instance_id, e))
578         return False
579
580
581 def delete_floating_ip(nova_client, floatingip_id):
582     try:
583         nova_client.floating_ips.delete(floatingip_id)
584         return True
585     except Exception as e:
586         logger.error("Error [delete_floating_ip(nova_client, '%s')]: %s"
587                      % (floatingip_id, e))
588         return False
589
590
591 def remove_host_from_aggregate(nova_client, aggregate_name, compute_host):
592     try:
593         aggregate_id = get_aggregate_id(nova_client, aggregate_name)
594         nova_client.aggregates.remove_host(aggregate_id, compute_host)
595         return True
596     except Exception as e:
597         logger.error("Error [remove_host_from_aggregate(nova_client, %s, %s)]:"
598                      " %s" % (aggregate_name, compute_host, e))
599         return False
600
601
602 def remove_hosts_from_aggregate(nova_client, aggregate_name):
603     aggregate_id = get_aggregate_id(nova_client, aggregate_name)
604     hosts = nova_client.aggregates.get(aggregate_id).hosts
605     assert(
606         all(remove_host_from_aggregate(nova_client, aggregate_name, host)
607             for host in hosts))
608
609
610 def delete_aggregate(nova_client, aggregate_name):
611     try:
612         remove_hosts_from_aggregate(nova_client, aggregate_name)
613         nova_client.aggregates.delete(aggregate_name)
614         return True
615     except Exception as e:
616         logger.error("Error [delete_aggregate(nova_client, %s)]: %s"
617                      % (aggregate_name, e))
618         return False
619
620
621 # *********************************************
622 #   NEUTRON
623 # *********************************************
624 def get_network_list(neutron_client):
625     network_list = neutron_client.list_networks()['networks']
626     if len(network_list) == 0:
627         return None
628     else:
629         return network_list
630
631
632 def get_router_list(neutron_client):
633     router_list = neutron_client.list_routers()['routers']
634     if len(router_list) == 0:
635         return None
636     else:
637         return router_list
638
639
640 def get_port_list(neutron_client):
641     port_list = neutron_client.list_ports()['ports']
642     if len(port_list) == 0:
643         return None
644     else:
645         return port_list
646
647
648 def get_network_id(neutron_client, network_name):
649     networks = neutron_client.list_networks()['networks']
650     id = ''
651     for n in networks:
652         if n['name'] == network_name:
653             id = n['id']
654             break
655     return id
656
657
658 def get_subnet_id(neutron_client, subnet_name):
659     subnets = neutron_client.list_subnets()['subnets']
660     id = ''
661     for s in subnets:
662         if s['name'] == subnet_name:
663             id = s['id']
664             break
665     return id
666
667
668 def get_router_id(neutron_client, router_name):
669     routers = neutron_client.list_routers()['routers']
670     id = ''
671     for r in routers:
672         if r['name'] == router_name:
673             id = r['id']
674             break
675     return id
676
677
678 def get_private_net(neutron_client):
679     # Checks if there is an existing shared private network
680     networks = neutron_client.list_networks()['networks']
681     if len(networks) == 0:
682         return None
683     for net in networks:
684         if (net['router:external'] is False) and (net['shared'] is True):
685             return net
686     return None
687
688
689 def get_external_net(neutron_client):
690     for network in neutron_client.list_networks()['networks']:
691         if network['router:external']:
692             return network['name']
693     return None
694
695
696 def get_external_net_id(neutron_client):
697     for network in neutron_client.list_networks()['networks']:
698         if network['router:external']:
699             return network['id']
700     return None
701
702
703 def check_neutron_net(neutron_client, net_name):
704     for network in neutron_client.list_networks()['networks']:
705         if network['name'] == net_name:
706             for subnet in network['subnets']:
707                 return True
708     return False
709
710
711 def create_neutron_net(neutron_client, name):
712     json_body = {'network': {'name': name,
713                              'admin_state_up': True}}
714     try:
715         network = neutron_client.create_network(body=json_body)
716         network_dict = network['network']
717         return network_dict['id']
718     except Exception as e:
719         logger.error("Error [create_neutron_net(neutron_client, '%s')]: %s"
720                      % (name, e))
721         return None
722
723
724 def create_neutron_subnet(neutron_client, name, cidr, net_id):
725     json_body = {'subnets': [{'name': name, 'cidr': cidr,
726                               'ip_version': 4, 'network_id': net_id}]}
727     try:
728         subnet = neutron_client.create_subnet(body=json_body)
729         return subnet['subnets'][0]['id']
730     except Exception as e:
731         logger.error("Error [create_neutron_subnet(neutron_client, '%s', "
732                      "'%s', '%s')]: %s" % (name, cidr, net_id, e))
733         return None
734
735
736 def create_neutron_router(neutron_client, name):
737     json_body = {'router': {'name': name, 'admin_state_up': True}}
738     try:
739         router = neutron_client.create_router(json_body)
740         return router['router']['id']
741     except Exception as e:
742         logger.error("Error [create_neutron_router(neutron_client, '%s')]: %s"
743                      % (name, e))
744         return None
745
746
747 def create_neutron_port(neutron_client, name, network_id, ip):
748     json_body = {'port': {
749                  'admin_state_up': True,
750                  'name': name,
751                  'network_id': network_id,
752                  'fixed_ips': [{"ip_address": ip}]
753                  }}
754     try:
755         port = neutron_client.create_port(body=json_body)
756         return port['port']['id']
757     except Exception as e:
758         logger.error("Error [create_neutron_port(neutron_client, '%s', '%s', "
759                      "'%s')]: %s" % (name, network_id, ip, e))
760         return None
761
762
763 def update_neutron_net(neutron_client, network_id, shared=False):
764     json_body = {'network': {'shared': shared}}
765     try:
766         neutron_client.update_network(network_id, body=json_body)
767         return True
768     except Exception as e:
769         logger.error("Error [update_neutron_net(neutron_client, '%s', '%s')]: "
770                      "%s" % (network_id, str(shared), e))
771         return False
772
773
774 def update_neutron_port(neutron_client, port_id, device_owner):
775     json_body = {'port': {
776                  'device_owner': device_owner,
777                  }}
778     try:
779         port = neutron_client.update_port(port=port_id,
780                                           body=json_body)
781         return port['port']['id']
782     except Exception as e:
783         logger.error("Error [update_neutron_port(neutron_client, '%s', '%s')]:"
784                      " %s" % (port_id, device_owner, e))
785         return None
786
787
788 def add_interface_router(neutron_client, router_id, subnet_id):
789     json_body = {"subnet_id": subnet_id}
790     try:
791         neutron_client.add_interface_router(router=router_id, body=json_body)
792         return True
793     except Exception as e:
794         logger.error("Error [add_interface_router(neutron_client, '%s', "
795                      "'%s')]: %s" % (router_id, subnet_id, e))
796         return False
797
798
799 def add_gateway_router(neutron_client, router_id):
800     ext_net_id = get_external_net_id(neutron_client)
801     router_dict = {'network_id': ext_net_id}
802     try:
803         neutron_client.add_gateway_router(router_id, router_dict)
804         return True
805     except Exception as e:
806         logger.error("Error [add_gateway_router(neutron_client, '%s')]: %s"
807                      % (router_id, e))
808         return False
809
810
811 def delete_neutron_net(neutron_client, network_id):
812     try:
813         neutron_client.delete_network(network_id)
814         return True
815     except Exception as e:
816         logger.error("Error [delete_neutron_net(neutron_client, '%s')]: %s"
817                      % (network_id, e))
818         return False
819
820
821 def delete_neutron_subnet(neutron_client, subnet_id):
822     try:
823         neutron_client.delete_subnet(subnet_id)
824         return True
825     except Exception as e:
826         logger.error("Error [delete_neutron_subnet(neutron_client, '%s')]: %s"
827                      % (subnet_id, e))
828         return False
829
830
831 def delete_neutron_router(neutron_client, router_id):
832     try:
833         neutron_client.delete_router(router=router_id)
834         return True
835     except Exception as e:
836         logger.error("Error [delete_neutron_router(neutron_client, '%s')]: %s"
837                      % (router_id, e))
838         return False
839
840
841 def delete_neutron_port(neutron_client, port_id):
842     try:
843         neutron_client.delete_port(port_id)
844         return True
845     except Exception as e:
846         logger.error("Error [delete_neutron_port(neutron_client, '%s')]: %s"
847                      % (port_id, e))
848         return False
849
850
851 def remove_interface_router(neutron_client, router_id, subnet_id):
852     json_body = {"subnet_id": subnet_id}
853     try:
854         neutron_client.remove_interface_router(router=router_id,
855                                                body=json_body)
856         return True
857     except Exception as e:
858         logger.error("Error [remove_interface_router(neutron_client, '%s', "
859                      "'%s')]: %s" % (router_id, subnet_id, e))
860         return False
861
862
863 def remove_gateway_router(neutron_client, router_id):
864     try:
865         neutron_client.remove_gateway_router(router_id)
866         return True
867     except Exception as e:
868         logger.error("Error [remove_gateway_router(neutron_client, '%s')]: %s"
869                      % (router_id, e))
870         return False
871
872
873 def create_network_full(neutron_client,
874                         net_name,
875                         subnet_name,
876                         router_name,
877                         cidr):
878
879     # Check if the network already exists
880     network_id = get_network_id(neutron_client, net_name)
881     subnet_id = get_subnet_id(neutron_client, subnet_name)
882     router_id = get_router_id(neutron_client, router_name)
883
884     if network_id != '' and subnet_id != '' and router_id != '':
885         logger.info("A network with name '%s' already exists..." % net_name)
886     else:
887         neutron_client.format = 'json'
888         logger.info('Creating neutron network %s...' % net_name)
889         network_id = create_neutron_net(neutron_client, net_name)
890
891         if not network_id:
892             return False
893
894         logger.debug("Network '%s' created successfully" % network_id)
895         logger.debug('Creating Subnet....')
896         subnet_id = create_neutron_subnet(neutron_client, subnet_name,
897                                           cidr, network_id)
898         if not subnet_id:
899             return None
900
901         logger.debug("Subnet '%s' created successfully" % subnet_id)
902         logger.debug('Creating Router...')
903         router_id = create_neutron_router(neutron_client, router_name)
904
905         if not router_id:
906             return None
907
908         logger.debug("Router '%s' created successfully" % router_id)
909         logger.debug('Adding router to subnet...')
910
911         if not add_interface_router(neutron_client, router_id, subnet_id):
912             return None
913
914         logger.debug("Interface added successfully.")
915
916         logger.debug('Adding gateway to router...')
917         if not add_gateway_router(neutron_client, router_id):
918             return None
919
920         logger.debug("Gateway added successfully.")
921
922     network_dic = {'net_id': network_id,
923                    'subnet_id': subnet_id,
924                    'router_id': router_id}
925     return network_dic
926
927
928 def create_shared_network_full(net_name, subnt_name, router_name, subnet_cidr):
929     neutron_client = get_neutron_client()
930
931     network_dic = create_network_full(neutron_client,
932                                       net_name,
933                                       subnt_name,
934                                       router_name,
935                                       subnet_cidr)
936     if network_dic:
937         if not update_neutron_net(neutron_client,
938                                   network_dic['net_id'],
939                                   shared=True):
940             logger.error("Failed to update network %s..." % net_name)
941             return None
942         else:
943             logger.debug("Network '%s' is available..." % net_name)
944     else:
945         logger.error("Network %s creation failed" % net_name)
946         return None
947     return network_dic
948
949
950 def create_bgpvpn(neutron_client, **kwargs):
951     # route_distinguishers
952     # route_targets
953     json_body = {"bgpvpn": kwargs}
954     return neutron_client.create_bgpvpn(json_body)
955
956
957 def create_network_association(neutron_client, bgpvpn_id, neutron_network_id):
958     json_body = {"network_association": {"network_id": neutron_network_id}}
959     return neutron_client.create_network_association(bgpvpn_id, json_body)
960
961
962 def create_router_association(neutron_client, bgpvpn_id, router_id):
963     json_body = {"router_association": {"router_id": router_id}}
964     return neutron_client.create_router_association(bgpvpn_id, json_body)
965
966
967 def update_bgpvpn(neutron_client, bgpvpn_id, **kwargs):
968     json_body = {"bgpvpn": kwargs}
969     return neutron_client.update_bgpvpn(bgpvpn_id, json_body)
970
971
972 def delete_bgpvpn(neutron_client, bgpvpn_id):
973     return neutron_client.delete_bgpvpn(bgpvpn_id)
974
975
976 def get_bgpvpn(neutron_client, bgpvpn_id):
977     return neutron_client.show_bgpvpn(bgpvpn_id)
978
979
980 def get_bgpvpn_routers(neutron_client, bgpvpn_id):
981     return get_bgpvpn(neutron_client, bgpvpn_id)['bgpvpn']['routers']
982
983
984 def get_bgpvpn_networks(neutron_client, bgpvpn_id):
985     return get_bgpvpn(neutron_client, bgpvpn_id)['bgpvpn']['networks']
986
987 # *********************************************
988 #   SEC GROUPS
989 # *********************************************
990
991
992 def get_security_groups(neutron_client):
993     try:
994         security_groups = neutron_client.list_security_groups()[
995             'security_groups']
996         return security_groups
997     except Exception as e:
998         logger.error("Error [get_security_groups(neutron_client)]: %s" % e)
999         return None
1000
1001
1002 def get_security_group_id(neutron_client, sg_name):
1003     security_groups = get_security_groups(neutron_client)
1004     id = ''
1005     for sg in security_groups:
1006         if sg['name'] == sg_name:
1007             id = sg['id']
1008             break
1009     return id
1010
1011
1012 def create_security_group(neutron_client, sg_name, sg_description):
1013     json_body = {'security_group': {'name': sg_name,
1014                                     'description': sg_description}}
1015     try:
1016         secgroup = neutron_client.create_security_group(json_body)
1017         return secgroup['security_group']
1018     except Exception as e:
1019         logger.error("Error [create_security_group(neutron_client, '%s', "
1020                      "'%s')]: %s" % (sg_name, sg_description, e))
1021         return None
1022
1023
1024 def create_secgroup_rule(neutron_client, sg_id, direction, protocol,
1025                          port_range_min=None, port_range_max=None):
1026     # We create a security group in 2 steps
1027     # 1 - we check the format and set the json body accordingly
1028     # 2 - we call neturon client to create the security group
1029
1030     # Format check
1031     json_body = {'security_group_rule': {'direction': direction,
1032                                          'security_group_id': sg_id,
1033                                          'protocol': protocol}}
1034     # parameters may be
1035     # - both None => we do nothing
1036     # - both Not None => we add them to the json description
1037     # but one cannot be None is the other is not None
1038     if (port_range_min is not None and port_range_max is not None):
1039         # add port_range in json description
1040         json_body['security_group_rule']['port_range_min'] = port_range_min
1041         json_body['security_group_rule']['port_range_max'] = port_range_max
1042         logger.debug("Security_group format set (port range included)")
1043     else:
1044         # either both port range are set to None => do nothing
1045         # or one is set but not the other => log it and return False
1046         if port_range_min is None and port_range_max is None:
1047             logger.debug("Security_group format set (no port range mentioned)")
1048         else:
1049             logger.error("Bad security group format."
1050                          "One of the port range is not properly set:"
1051                          "range min: {},"
1052                          "range max: {}".format(port_range_min,
1053                                                 port_range_max))
1054             return False
1055
1056     # Create security group using neutron client
1057     try:
1058         neutron_client.create_security_group_rule(json_body)
1059         return True
1060     except:
1061         logger.exception("Impossible to create_security_group_rule,"
1062                          "security group rule probably already exists")
1063         return False
1064
1065
1066 def get_security_group_rules(neutron_client, sg_id):
1067     try:
1068         security_rules = neutron_client.list_security_group_rules()[
1069             'security_group_rules']
1070         security_rules = [rule for rule in security_rules
1071                           if rule["security_group_id"] == sg_id]
1072         return security_rules
1073     except Exception as e:
1074         logger.error("Error [get_security_group_rules(neutron_client, sg_id)]:"
1075                      " %s" % e)
1076         return None
1077
1078
1079 def check_security_group_rules(neutron_client, sg_id, direction, protocol,
1080                                port_min=None, port_max=None):
1081     try:
1082         security_rules = get_security_group_rules(neutron_client, sg_id)
1083         security_rules = [rule for rule in security_rules
1084                           if (rule["direction"].lower() == direction
1085                               and rule["protocol"].lower() == protocol
1086                               and rule["port_range_min"] == port_min
1087                               and rule["port_range_max"] == port_max)]
1088         if len(security_rules) == 0:
1089             return True
1090         else:
1091             return False
1092     except Exception as e:
1093         logger.error("Error [check_security_group_rules("
1094                      " neutron_client, sg_id, direction,"
1095                      " protocol, port_min=None, port_max=None)]: "
1096                      "%s" % e)
1097         return None
1098
1099
1100 def create_security_group_full(neutron_client,
1101                                sg_name, sg_description):
1102     sg_id = get_security_group_id(neutron_client, sg_name)
1103     if sg_id != '':
1104         logger.info("Using existing security group '%s'..." % sg_name)
1105     else:
1106         logger.info("Creating security group  '%s'..." % sg_name)
1107         SECGROUP = create_security_group(neutron_client,
1108                                          sg_name,
1109                                          sg_description)
1110         if not SECGROUP:
1111             logger.error("Failed to create the security group...")
1112             return None
1113
1114         sg_id = SECGROUP['id']
1115
1116         logger.debug("Security group '%s' with ID=%s created successfully."
1117                      % (SECGROUP['name'], sg_id))
1118
1119         logger.debug("Adding ICMP rules in security group '%s'..."
1120                      % sg_name)
1121         if not create_secgroup_rule(neutron_client, sg_id,
1122                                     'ingress', 'icmp'):
1123             logger.error("Failed to create the security group rule...")
1124             return None
1125
1126         logger.debug("Adding SSH rules in security group '%s'..."
1127                      % sg_name)
1128         if not create_secgroup_rule(
1129                 neutron_client, sg_id, 'ingress', 'tcp', '22', '22'):
1130             logger.error("Failed to create the security group rule...")
1131             return None
1132
1133         if not create_secgroup_rule(
1134                 neutron_client, sg_id, 'egress', 'tcp', '22', '22'):
1135             logger.error("Failed to create the security group rule...")
1136             return None
1137     return sg_id
1138
1139
1140 def add_secgroup_to_instance(nova_client, instance_id, secgroup_id):
1141     try:
1142         nova_client.servers.add_security_group(instance_id, secgroup_id)
1143         return True
1144     except Exception as e:
1145         logger.error("Error [add_secgroup_to_instance(nova_client, '%s', "
1146                      "'%s')]: %s" % (instance_id, secgroup_id, e))
1147         return False
1148
1149
1150 def update_sg_quota(neutron_client, tenant_id, sg_quota, sg_rule_quota):
1151     json_body = {"quota": {
1152         "security_group": sg_quota,
1153         "security_group_rule": sg_rule_quota
1154     }}
1155
1156     try:
1157         neutron_client.update_quota(tenant_id=tenant_id,
1158                                     body=json_body)
1159         return True
1160     except Exception as e:
1161         logger.error("Error [update_sg_quota(neutron_client, '%s', '%s', "
1162                      "'%s')]: %s" % (tenant_id, sg_quota, sg_rule_quota, e))
1163         return False
1164
1165
1166 def delete_security_group(neutron_client, secgroup_id):
1167     try:
1168         neutron_client.delete_security_group(secgroup_id)
1169         return True
1170     except Exception as e:
1171         logger.error("Error [delete_security_group(neutron_client, '%s')]: %s"
1172                      % (secgroup_id, e))
1173         return False
1174
1175
1176 # *********************************************
1177 #   GLANCE
1178 # *********************************************
1179 def get_images(nova_client):
1180     try:
1181         images = nova_client.images.list()
1182         return images
1183     except Exception as e:
1184         logger.error("Error [get_images]: %s" % e)
1185         return None
1186
1187
1188 def get_image_id(glance_client, image_name):
1189     images = glance_client.images.list()
1190     id = ''
1191     for i in images:
1192         if i.name == image_name:
1193             id = i.id
1194             break
1195     return id
1196
1197
1198 def create_glance_image(glance_client, image_name, file_path, disk="qcow2",
1199                         container="bare", public="public"):
1200     if not os.path.isfile(file_path):
1201         logger.error("Error: file %s does not exist." % file_path)
1202         return None
1203     try:
1204         image_id = get_image_id(glance_client, image_name)
1205         if image_id != '':
1206             logger.info("Image %s already exists." % image_name)
1207         else:
1208             logger.info("Creating image '%s' from '%s'..." % (image_name,
1209                                                               file_path))
1210
1211             image = glance_client.images.create(name=image_name,
1212                                                 visibility=public,
1213                                                 disk_format=disk,
1214                                                 container_format=container)
1215             image_id = image.id
1216             with open(file_path) as image_data:
1217                 glance_client.images.upload(image_id, image_data)
1218         return image_id
1219     except Exception as e:
1220         logger.error("Error [create_glance_image(glance_client, '%s', '%s', "
1221                      "'%s')]: %s" % (image_name, file_path, public, e))
1222         return None
1223
1224
1225 def get_or_create_image(name, path, format):
1226     image_exists = False
1227     glance_client = get_glance_client()
1228
1229     image_id = get_image_id(glance_client, name)
1230     if image_id != '':
1231         logger.info("Using existing image '%s'..." % name)
1232         image_exists = True
1233     else:
1234         logger.info("Creating image '%s' from '%s'..." % (name, path))
1235         image_id = create_glance_image(glance_client, name, path, format)
1236         if not image_id:
1237             logger.error("Failed to create a Glance image...")
1238         else:
1239             logger.debug("Image '%s' with ID=%s created successfully."
1240                          % (name, image_id))
1241
1242     return image_exists, image_id
1243
1244
1245 def delete_glance_image(nova_client, image_id):
1246     try:
1247         nova_client.images.delete(image_id)
1248         return True
1249     except Exception as e:
1250         logger.error("Error [delete_glance_image(nova_client, '%s')]: %s"
1251                      % (image_id, e))
1252         return False
1253
1254
1255 # *********************************************
1256 #   CINDER
1257 # *********************************************
1258 def get_volumes(cinder_client):
1259     try:
1260         volumes = cinder_client.volumes.list(search_opts={'all_tenants': 1})
1261         return volumes
1262     except Exception as e:
1263         logger.error("Error [get_volumes(cinder_client)]: %s" % e)
1264         return None
1265
1266
1267 def list_volume_types(cinder_client, public=True, private=True):
1268     try:
1269         volume_types = cinder_client.volume_types.list()
1270         if not public:
1271             volume_types = [vt for vt in volume_types if not vt.is_public]
1272         if not private:
1273             volume_types = [vt for vt in volume_types if vt.is_public]
1274         return volume_types
1275     except Exception as e:
1276         logger.error("Error [list_volume_types(cinder_client)]: %s" % e)
1277         return None
1278
1279
1280 def create_volume_type(cinder_client, name):
1281     try:
1282         volume_type = cinder_client.volume_types.create(name)
1283         return volume_type
1284     except Exception as e:
1285         logger.error("Error [create_volume_type(cinder_client, '%s')]: %s"
1286                      % (name, e))
1287         return None
1288
1289
1290 def update_cinder_quota(cinder_client, tenant_id, vols_quota,
1291                         snapshots_quota, gigabytes_quota):
1292     quotas_values = {"volumes": vols_quota,
1293                      "snapshots": snapshots_quota,
1294                      "gigabytes": gigabytes_quota}
1295
1296     try:
1297         cinder_client.quotas.update(tenant_id, **quotas_values)
1298         return True
1299     except Exception as e:
1300         logger.error("Error [update_cinder_quota(cinder_client, '%s', '%s', "
1301                      "'%s' '%s')]: %s" % (tenant_id, vols_quota,
1302                                           snapshots_quota, gigabytes_quota, e))
1303         return False
1304
1305
1306 def delete_volume(cinder_client, volume_id, forced=False):
1307     try:
1308         if forced:
1309             try:
1310                 cinder_client.volumes.detach(volume_id)
1311             except:
1312                 logger.error(sys.exc_info()[0])
1313             cinder_client.volumes.force_delete(volume_id)
1314         else:
1315             cinder_client.volumes.delete(volume_id)
1316         return True
1317     except Exception as e:
1318         logger.error("Error [delete_volume(cinder_client, '%s', '%s')]: %s"
1319                      % (volume_id, str(forced), e))
1320         return False
1321
1322
1323 def delete_volume_type(cinder_client, volume_type):
1324     try:
1325         cinder_client.volume_types.delete(volume_type)
1326         return True
1327     except Exception as e:
1328         logger.error("Error [delete_volume_type(cinder_client, '%s')]: %s"
1329                      % (volume_type, e))
1330         return False
1331
1332
1333 # *********************************************
1334 #   KEYSTONE
1335 # *********************************************
1336 def get_tenants(keystone_client):
1337     try:
1338         if is_keystone_v3():
1339             tenants = keystone_client.projects.list()
1340         else:
1341             tenants = keystone_client.tenants.list()
1342         return tenants
1343     except Exception as e:
1344         logger.error("Error [get_tenants(keystone_client)]: %s" % e)
1345         return None
1346
1347
1348 def get_users(keystone_client):
1349     try:
1350         users = keystone_client.users.list()
1351         return users
1352     except Exception as e:
1353         logger.error("Error [get_users(keystone_client)]: %s" % e)
1354         return None
1355
1356
1357 def get_tenant_id(keystone_client, tenant_name):
1358     tenants = get_tenants(keystone_client)
1359     id = ''
1360     for t in tenants:
1361         if t.name == tenant_name:
1362             id = t.id
1363             break
1364     return id
1365
1366
1367 def get_user_id(keystone_client, user_name):
1368     users = get_users(keystone_client)
1369     id = ''
1370     for u in users:
1371         if u.name == user_name:
1372             id = u.id
1373             break
1374     return id
1375
1376
1377 def get_role_id(keystone_client, role_name):
1378     roles = keystone_client.roles.list()
1379     id = ''
1380     for r in roles:
1381         if r.name == role_name:
1382             id = r.id
1383             break
1384     return id
1385
1386
1387 def create_tenant(keystone_client, tenant_name, tenant_description):
1388     try:
1389         if is_keystone_v3():
1390             tenant = keystone_client.projects.create(
1391                 name=tenant_name,
1392                 description=tenant_description,
1393                 domain="default",
1394                 enabled=True)
1395         else:
1396             tenant = keystone_client.tenants.create(tenant_name,
1397                                                     tenant_description,
1398                                                     enabled=True)
1399         return tenant.id
1400     except Exception as e:
1401         logger.error("Error [create_tenant(keystone_client, '%s', '%s')]: %s"
1402                      % (tenant_name, tenant_description, e))
1403         return None
1404
1405
1406 def get_or_create_tenant(keystone_client, tenant_name, tenant_description):
1407     tenant_id = get_tenant_id(keystone_client, tenant_name)
1408     if not tenant_id:
1409         tenant_id = create_tenant(keystone_client, tenant_name,
1410                                   tenant_description)
1411
1412     return tenant_id
1413
1414
1415 def create_user(keystone_client, user_name, user_password,
1416                 user_email, tenant_id):
1417     try:
1418         if is_keystone_v3():
1419             user = keystone_client.users.create(name=user_name,
1420                                                 password=user_password,
1421                                                 email=user_email,
1422                                                 project_id=tenant_id,
1423                                                 enabled=True)
1424         else:
1425             user = keystone_client.users.create(user_name,
1426                                                 user_password,
1427                                                 user_email,
1428                                                 tenant_id,
1429                                                 enabled=True)
1430         return user.id
1431     except Exception as e:
1432         logger.error("Error [create_user(keystone_client, '%s', '%s', '%s'"
1433                      "'%s')]: %s" % (user_name, user_password,
1434                                      user_email, tenant_id, e))
1435         return None
1436
1437
1438 def get_or_create_user(keystone_client, user_name, user_password,
1439                        tenant_id, user_email=None):
1440     user_id = get_user_id(keystone_client, user_name)
1441     if not user_id:
1442         user_id = create_user(keystone_client, user_name, user_password,
1443                               user_email, tenant_id)
1444     return user_id
1445
1446
1447 def add_role_user(keystone_client, user_id, role_id, tenant_id):
1448     try:
1449         if is_keystone_v3():
1450             keystone_client.roles.grant(role=role_id,
1451                                         user=user_id,
1452                                         project=tenant_id)
1453         else:
1454             keystone_client.roles.add_user_role(user_id, role_id, tenant_id)
1455         return True
1456     except Exception as e:
1457         logger.error("Error [add_role_user(keystone_client, '%s', '%s'"
1458                      "'%s')]: %s " % (user_id, role_id, tenant_id, e))
1459         return False
1460
1461
1462 def delete_tenant(keystone_client, tenant_id):
1463     try:
1464         if is_keystone_v3():
1465             keystone_client.projects.delete(tenant_id)
1466         else:
1467             keystone_client.tenants.delete(tenant_id)
1468         return True
1469     except Exception as e:
1470         logger.error("Error [delete_tenant(keystone_client, '%s')]: %s"
1471                      % (tenant_id, e))
1472         return False
1473
1474
1475 def delete_user(keystone_client, user_id):
1476     try:
1477         keystone_client.users.delete(user_id)
1478         return True
1479     except Exception as e:
1480         logger.error("Error [delete_user(keystone_client, '%s')]: %s"
1481                      % (user_id, e))
1482         return False
1483
1484
1485 # *********************************************
1486 #   HEAT
1487 # *********************************************
1488 def get_resource(heat_client, stack_id, resource):
1489     try:
1490         resources = heat_client.resources.get(stack_id, resource)
1491         return resources
1492     except Exception as e:
1493         logger.error("Error [get_resource]: %s" % e)
1494         return None