49b9c840af2f0ec9f37bb92a5d284725200697ac
[functest.git] / functest / utils / openstack_utils.py
1 #!/usr/bin/env python
2 #
3 # jose.lausuch@ericsson.com
4 # valentin.boucher@orange.com
5 # All rights reserved. This program and the accompanying materials
6 # are made available under the terms of the Apache License, Version 2.0
7 # which accompanies this distribution, and is available at
8 # http://www.apache.org/licenses/LICENSE-2.0
9 #
10
11 import os
12 import os.path
13 import re
14 import sys
15 import time
16
17 from keystoneauth1 import loading
18 from keystoneauth1 import session
19 from cinderclient import client as cinderclient
20 from glanceclient import client as glanceclient
21 from heatclient import client as heatclient
22 from novaclient import client as novaclient
23 from keystoneclient import client as keystoneclient
24 from neutronclient.neutron import client as neutronclient
25
26 import functest.utils.functest_logger as ft_logger
27 import functest.utils.functest_utils as ft_utils
28
29 logger = ft_logger.Logger("openstack_utils").getLogger()
30
31 DEFAULT_API_VERSION = '2'
32 DEFAULT_HEAT_API_VERSION = '1'
33
34
35 # *********************************************
36 #   CREDENTIALS
37 # *********************************************
38 class MissingEnvVar(Exception):
39
40     def __init__(self, var):
41         self.var = var
42
43     def __str__(self):
44         return str.format("Please set the mandatory env var: {}", self.var)
45
46
47 def is_keystone_v3():
48     keystone_api_version = os.getenv('OS_IDENTITY_API_VERSION')
49     if (keystone_api_version is None or
50             keystone_api_version == '2'):
51         return False
52     else:
53         return True
54
55
56 def get_rc_env_vars():
57     env_vars = ['OS_AUTH_URL', 'OS_USERNAME', 'OS_PASSWORD']
58     if is_keystone_v3():
59         env_vars.extend(['OS_PROJECT_NAME',
60                          'OS_USER_DOMAIN_NAME',
61                          'OS_PROJECT_DOMAIN_NAME'])
62     else:
63         env_vars.extend(['OS_TENANT_NAME'])
64     return env_vars
65
66
67 def check_credentials():
68     """
69     Check if the OpenStack credentials (openrc) are sourced
70     """
71     env_vars = get_rc_env_vars()
72     return all(map(lambda v: v in os.environ and os.environ[v], env_vars))
73
74
75 def get_env_cred_dict():
76     env_cred_dict = {
77         'OS_USERNAME': 'username',
78         'OS_PASSWORD': 'password',
79         'OS_AUTH_URL': 'auth_url',
80         'OS_TENANT_NAME': 'tenant_name',
81         'OS_USER_DOMAIN_NAME': 'user_domain_name',
82         'OS_PROJECT_DOMAIN_NAME': 'project_domain_name',
83         'OS_PROJECT_NAME': 'project_name',
84         'OS_ENDPOINT_TYPE': 'endpoint_type',
85         'OS_REGION_NAME': 'region_name',
86         'OS_CACERT': 'https_cacert'
87     }
88     return env_cred_dict
89
90
91 def get_credentials(other_creds={}):
92     """Returns a creds dictionary filled with parsed from env
93     """
94     creds = {}
95     env_vars = get_rc_env_vars()
96     env_cred_dict = get_env_cred_dict()
97
98     for envvar in env_vars:
99         if os.getenv(envvar) is None:
100             raise MissingEnvVar(envvar)
101         else:
102             creds_key = env_cred_dict.get(envvar)
103             creds.update({creds_key: os.getenv(envvar)})
104
105     if 'tenant' in other_creds.keys():
106         if is_keystone_v3():
107             tenant = 'project_name'
108         else:
109             tenant = 'tenant_name'
110         other_creds[tenant] = other_creds.pop('tenant')
111
112     creds.update(other_creds)
113
114     return creds
115
116
117 def source_credentials(rc_file):
118     with open(rc_file, "r") as f:
119         for line in f:
120             var = line.rstrip('"\n').replace('export ', '').split("=")
121             # The two next lines should be modified as soon as rc_file
122             # conforms with common rules. Be aware that it could induce
123             # issues if value starts with '
124             key = re.sub(r'^["\' ]*|[ \'"]*$', '', var[0])
125             value = re.sub(r'^["\' ]*|[ \'"]*$', '', "".join(var[1:]))
126             os.environ[key] = value
127
128
129 def get_credentials_for_rally():
130     creds = get_credentials()
131     env_cred_dict = get_env_cred_dict()
132     rally_conf = {"type": "ExistingCloud", "admin": {}}
133     for key in creds:
134         if key == 'auth_url':
135             rally_conf[key] = creds[key]
136         else:
137             rally_conf['admin'][key] = creds[key]
138
139     endpoint_types = [('internalURL', 'internal'),
140                       ('publicURL', 'public'), ('adminURL', 'admin')]
141
142     endpoint_type = os.getenv('OS_ENDPOINT_TYPE')
143     if endpoint_type is not None:
144         cred_key = env_cred_dict.get('OS_ENDPOINT_TYPE')
145         for k, v in endpoint_types:
146             if endpoint_type == k:
147                 rally_conf[cred_key] = v
148
149     region_name = os.getenv('OS_REGION_NAME')
150     if region_name is not None:
151         cred_key = env_cred_dict.get('OS_REGION_NAME')
152         rally_conf[cred_key] = region_name
153
154     cacert = os.getenv('OS_CACERT')
155     if cacert is not None:
156         cred_key = env_cred_dict.get('OS_CACERT')
157         rally_conf[cred_key] = cacert
158     return rally_conf
159
160
161 def get_session_auth(other_creds={}):
162     loader = loading.get_plugin_loader('password')
163     creds = get_credentials(other_creds)
164     auth = loader.load_from_options(**creds)
165     return auth
166
167
168 def get_endpoint(service_type, endpoint_type='publicURL'):
169     auth = get_session_auth()
170     return get_session().get_endpoint(auth=auth,
171                                       service_type=service_type,
172                                       endpoint_type=endpoint_type)
173
174
175 def get_session(other_creds={}):
176     auth = get_session_auth(other_creds)
177     cacert = os.getenv('OS_CACERT')
178     if cacert is not None:
179         if not os.path.isfile(cacert):
180             raise Exception("The 'OS_CACERT' environment"
181                             "variable is set to %s but the file"
182                             "does not exist.", cacert)
183
184     return session.Session(auth=auth, verify=cacert)
185
186
187 # *********************************************
188 #   CLIENTS
189 # *********************************************
190 def get_keystone_client_version():
191     api_version = os.getenv('OS_IDENTITY_API_VERSION')
192     if api_version is not None:
193         logger.info("OS_IDENTITY_API_VERSION is set in env as '%s'",
194                     api_version)
195         return api_version
196     return DEFAULT_API_VERSION
197
198
199 def get_keystone_client(other_creds={}):
200     sess = get_session(other_creds)
201     return keystoneclient.Client(get_keystone_client_version(),
202                                  session=sess,
203                                  interface=os.getenv('OS_INTERFACE', 'admin'))
204
205
206 def get_nova_client_version():
207     api_version = os.getenv('OS_COMPUTE_API_VERSION')
208     if api_version is not None:
209         logger.info("OS_COMPUTE_API_VERSION is set in env as '%s'",
210                     api_version)
211         return api_version
212     return DEFAULT_API_VERSION
213
214
215 def get_nova_client(other_creds={}):
216     sess = get_session(other_creds)
217     return novaclient.Client(get_nova_client_version(), session=sess)
218
219
220 def get_cinder_client_version():
221     api_version = os.getenv('OS_VOLUME_API_VERSION')
222     if api_version is not None:
223         logger.info("OS_VOLUME_API_VERSION is set in env as '%s'",
224                     api_version)
225         return api_version
226     return DEFAULT_API_VERSION
227
228
229 def get_cinder_client(other_creds={}):
230     sess = get_session(other_creds)
231     return cinderclient.Client(get_cinder_client_version(), session=sess)
232
233
234 def get_neutron_client_version():
235     api_version = os.getenv('OS_NETWORK_API_VERSION')
236     if api_version is not None:
237         logger.info("OS_NETWORK_API_VERSION is set in env as '%s'",
238                     api_version)
239         return api_version
240     return DEFAULT_API_VERSION
241
242
243 def get_neutron_client(other_creds={}):
244     sess = get_session(other_creds)
245     return neutronclient.Client(get_neutron_client_version(), session=sess)
246
247
248 def get_glance_client_version():
249     api_version = os.getenv('OS_IMAGE_API_VERSION')
250     if api_version is not None:
251         logger.info("OS_IMAGE_API_VERSION is set in env as '%s'", api_version)
252         return api_version
253     return DEFAULT_API_VERSION
254
255
256 def get_glance_client(other_creds={}):
257     sess = get_session(other_creds)
258     return glanceclient.Client(get_glance_client_version(), session=sess)
259
260
261 def get_heat_client_version():
262     api_version = os.getenv('OS_ORCHESTRATION_API_VERSION')
263     if api_version is not None:
264         logger.info("OS_ORCHESTRATION_API_VERSION is set in env as '%s'",
265                     api_version)
266         return api_version
267     return DEFAULT_HEAT_API_VERSION
268
269
270 def get_heat_client(other_creds={}):
271     sess = get_session(other_creds)
272     return heatclient.Client(get_heat_client_version(), session=sess)
273
274
275 # *********************************************
276 #   NOVA
277 # *********************************************
278 def get_instances(nova_client):
279     try:
280         instances = nova_client.servers.list(search_opts={'all_tenants': 1})
281         return instances
282     except Exception, e:
283         logger.error("Error [get_instances(nova_client)]: %s" % e)
284         return None
285
286
287 def get_instance_status(nova_client, instance):
288     try:
289         instance = nova_client.servers.get(instance.id)
290         return instance.status
291     except Exception, e:
292         logger.error("Error [get_instance_status(nova_client)]: %s" % e)
293         return None
294
295
296 def get_instance_by_name(nova_client, instance_name):
297     try:
298         instance = nova_client.servers.find(name=instance_name)
299         return instance
300     except Exception, e:
301         logger.error("Error [get_instance_by_name(nova_client, '%s')]: %s"
302                      % (instance_name, e))
303         return None
304
305
306 def get_flavor_id(nova_client, flavor_name):
307     flavors = nova_client.flavors.list(detailed=True)
308     id = ''
309     for f in flavors:
310         if f.name == flavor_name:
311             id = f.id
312             break
313     return id
314
315
316 def get_flavor_id_by_ram_range(nova_client, min_ram, max_ram):
317     flavors = nova_client.flavors.list(detailed=True)
318     id = ''
319     for f in flavors:
320         if min_ram <= f.ram and f.ram <= max_ram:
321             id = f.id
322             break
323     return id
324
325
326 def get_aggregates(nova_client):
327     try:
328         aggregates = nova_client.aggregates.list()
329         return aggregates
330     except Exception, e:
331         logger.error("Error [get_aggregates(nova_client)]: %s" % e)
332         return None
333
334
335 def get_aggregate_id(nova_client, aggregate_name):
336     try:
337         aggregates = get_aggregates(nova_client)
338         _id = [ag.id for ag in aggregates if ag.name == aggregate_name][0]
339         return _id
340     except Exception, e:
341         logger.error("Error [get_aggregate_id(nova_client, %s)]:"
342                      " %s" % (aggregate_name, e))
343         return None
344
345
346 def get_availability_zones(nova_client):
347     try:
348         availability_zones = nova_client.availability_zones.list()
349         return availability_zones
350     except Exception, e:
351         logger.error("Error [get_availability_zones(nova_client)]: %s" % e)
352         return None
353
354
355 def get_availability_zone_names(nova_client):
356     try:
357         az_names = [az.zoneName for az in get_availability_zones(nova_client)]
358         return az_names
359     except Exception, e:
360         logger.error("Error [get_availability_zone_names(nova_client)]:"
361                      " %s" % e)
362         return None
363
364
365 def create_flavor(nova_client, flavor_name, ram, disk, vcpus, public=True):
366     try:
367         flavor = nova_client.flavors.create(
368             flavor_name, ram, vcpus, disk, is_public=public)
369         try:
370             extra_specs = ft_utils.get_functest_config(
371                 'general.flavor_extra_specs')
372             flavor.set_keys(extra_specs)
373         except ValueError:
374             # flavor extra specs are not configured, therefore skip the update
375             pass
376
377     except Exception, e:
378         logger.error("Error [create_flavor(nova_client, '%s', '%s', '%s', "
379                      "'%s')]: %s" % (flavor_name, ram, disk, vcpus, e))
380         return None
381     return flavor.id
382
383
384 def get_or_create_flavor(flavor_name, ram, disk, vcpus, public=True):
385     flavor_exists = False
386     nova_client = get_nova_client()
387
388     flavor_id = get_flavor_id(nova_client, flavor_name)
389     if flavor_id != '':
390         logger.info("Using existing flavor '%s'..." % flavor_name)
391         flavor_exists = True
392     else:
393         logger.info("Creating flavor '%s' with '%s' RAM, '%s' disk size, "
394                     "'%s' vcpus..." % (flavor_name, ram, disk, vcpus))
395         flavor_id = create_flavor(
396             nova_client, flavor_name, ram, disk, vcpus, public=public)
397         if not flavor_id:
398             logger.error("Failed to create flavor '%s'..." % (flavor_name))
399         else:
400             logger.debug("Flavor '%s' with ID=%s created successfully."
401                          % (flavor_name, flavor_id))
402
403     return flavor_exists, flavor_id
404
405
406 def get_floating_ips(nova_client):
407     try:
408         floating_ips = nova_client.floating_ips.list()
409         return floating_ips
410     except Exception, e:
411         logger.error("Error [get_floating_ips(nova_client)]: %s" % e)
412         return None
413
414
415 def get_hypervisors(nova_client):
416     try:
417         nodes = []
418         hypervisors = nova_client.hypervisors.list()
419         for hypervisor in hypervisors:
420             if hypervisor.state == "up":
421                 nodes.append(hypervisor.hypervisor_hostname)
422         return nodes
423     except Exception, e:
424         logger.error("Error [get_hypervisors(nova_client)]: %s" % e)
425         return None
426
427
428 def create_aggregate(nova_client, aggregate_name, av_zone):
429     try:
430         nova_client.aggregates.create(aggregate_name, av_zone)
431         return True
432     except Exception, e:
433         logger.error("Error [create_aggregate(nova_client, %s, %s)]: %s"
434                      % (aggregate_name, av_zone, e))
435         return None
436
437
438 def add_host_to_aggregate(nova_client, aggregate_name, compute_host):
439     try:
440         aggregate_id = get_aggregate_id(nova_client, aggregate_name)
441         nova_client.aggregates.add_host(aggregate_id, compute_host)
442         return True
443     except Exception, e:
444         logger.error("Error [add_host_to_aggregate(nova_client, %s, %s)]: %s"
445                      % (aggregate_name, compute_host, e))
446         return None
447
448
449 def create_aggregate_with_host(
450         nova_client, aggregate_name, av_zone, compute_host):
451     try:
452         create_aggregate(nova_client, aggregate_name, av_zone)
453         add_host_to_aggregate(nova_client, aggregate_name, compute_host)
454         return True
455     except Exception, e:
456         logger.error("Error [create_aggregate_with_host("
457                      "nova_client, %s, %s, %s)]: %s"
458                      % (aggregate_name, av_zone, compute_host, e))
459         return None
460
461
462 def create_instance(flavor_name,
463                     image_id,
464                     network_id,
465                     instance_name="functest-vm",
466                     confdrive=True,
467                     userdata=None,
468                     av_zone='',
469                     fixed_ip=None,
470                     files=None):
471     nova_client = get_nova_client()
472     try:
473         flavor = nova_client.flavors.find(name=flavor_name)
474     except:
475         flavors = nova_client.flavors.list()
476         logger.error("Error: Flavor '%s' not found. Available flavors are: "
477                      "\n%s" % (flavor_name, flavors))
478         return None
479     if fixed_ip is not None:
480         nics = {"net-id": network_id, "v4-fixed-ip": fixed_ip}
481     else:
482         nics = {"net-id": network_id}
483     if userdata is None:
484         instance = nova_client.servers.create(
485             name=instance_name,
486             flavor=flavor,
487             image=image_id,
488             nics=[nics],
489             availability_zone=av_zone,
490             files=files
491         )
492     else:
493         instance = nova_client.servers.create(
494             name=instance_name,
495             flavor=flavor,
496             image=image_id,
497             nics=[nics],
498             config_drive=confdrive,
499             userdata=userdata,
500             availability_zone=av_zone,
501             files=files
502         )
503     return instance
504
505
506 def create_instance_and_wait_for_active(flavor_name,
507                                         image_id,
508                                         network_id,
509                                         instance_name="",
510                                         config_drive=False,
511                                         userdata="",
512                                         av_zone='',
513                                         fixed_ip=None,
514                                         files=None):
515     SLEEP = 3
516     VM_BOOT_TIMEOUT = 180
517     nova_client = get_nova_client()
518     instance = create_instance(flavor_name,
519                                image_id,
520                                network_id,
521                                instance_name,
522                                config_drive,
523                                userdata,
524                                av_zone=av_zone,
525                                fixed_ip=fixed_ip,
526                                files=files)
527     count = VM_BOOT_TIMEOUT / SLEEP
528     for n in range(count, -1, -1):
529         status = get_instance_status(nova_client, instance)
530         if status.lower() == "active":
531             return instance
532         elif status.lower() == "error":
533             logger.error("The instance %s went to ERROR status."
534                          % instance_name)
535             return None
536         time.sleep(SLEEP)
537     logger.error("Timeout booting the instance %s." % instance_name)
538     return None
539
540
541 def create_floating_ip(neutron_client):
542     extnet_id = get_external_net_id(neutron_client)
543     props = {'floating_network_id': extnet_id}
544     try:
545         ip_json = neutron_client.create_floatingip({'floatingip': props})
546         fip_addr = ip_json['floatingip']['floating_ip_address']
547         fip_id = ip_json['floatingip']['id']
548     except Exception, e:
549         logger.error("Error [create_floating_ip(neutron_client)]: %s" % e)
550         return None
551     return {'fip_addr': fip_addr, 'fip_id': fip_id}
552
553
554 def add_floating_ip(nova_client, server_id, floatingip_addr):
555     try:
556         nova_client.servers.add_floating_ip(server_id, floatingip_addr)
557         return True
558     except Exception, e:
559         logger.error("Error [add_floating_ip(nova_client, '%s', '%s')]: %s"
560                      % (server_id, floatingip_addr, e))
561         return False
562
563
564 def delete_instance(nova_client, instance_id):
565     try:
566         nova_client.servers.force_delete(instance_id)
567         return True
568     except Exception, e:
569         logger.error("Error [delete_instance(nova_client, '%s')]: %s"
570                      % (instance_id, e))
571         return False
572
573
574 def delete_floating_ip(nova_client, floatingip_id):
575     try:
576         nova_client.floating_ips.delete(floatingip_id)
577         return True
578     except Exception, e:
579         logger.error("Error [delete_floating_ip(nova_client, '%s')]: %s"
580                      % (floatingip_id, e))
581         return False
582
583
584 def remove_host_from_aggregate(nova_client, aggregate_name, compute_host):
585     try:
586         aggregate_id = get_aggregate_id(nova_client, aggregate_name)
587         nova_client.aggregates.remove_host(aggregate_id, compute_host)
588         return True
589     except Exception, e:
590         logger.error("Error [remove_host_from_aggregate(nova_client, %s, %s)]:"
591                      " %s" % (aggregate_name, compute_host, e))
592         return False
593
594
595 def remove_hosts_from_aggregate(nova_client, aggregate_name):
596     aggregate_id = get_aggregate_id(nova_client, aggregate_name)
597     hosts = nova_client.aggregates.get(aggregate_id).hosts
598     assert(
599         all(remove_host_from_aggregate(nova_client, aggregate_name, host)
600             for host in hosts))
601
602
603 def delete_aggregate(nova_client, aggregate_name):
604     try:
605         remove_hosts_from_aggregate(nova_client, aggregate_name)
606         nova_client.aggregates.delete(aggregate_name)
607         return True
608     except Exception, e:
609         logger.error("Error [delete_aggregate(nova_client, %s)]: %s"
610                      % (aggregate_name, e))
611         return False
612
613
614 # *********************************************
615 #   NEUTRON
616 # *********************************************
617 def get_network_list(neutron_client):
618     network_list = neutron_client.list_networks()['networks']
619     if len(network_list) == 0:
620         return None
621     else:
622         return network_list
623
624
625 def get_router_list(neutron_client):
626     router_list = neutron_client.list_routers()['routers']
627     if len(router_list) == 0:
628         return None
629     else:
630         return router_list
631
632
633 def get_port_list(neutron_client):
634     port_list = neutron_client.list_ports()['ports']
635     if len(port_list) == 0:
636         return None
637     else:
638         return port_list
639
640
641 def get_network_id(neutron_client, network_name):
642     networks = neutron_client.list_networks()['networks']
643     id = ''
644     for n in networks:
645         if n['name'] == network_name:
646             id = n['id']
647             break
648     return id
649
650
651 def get_subnet_id(neutron_client, subnet_name):
652     subnets = neutron_client.list_subnets()['subnets']
653     id = ''
654     for s in subnets:
655         if s['name'] == subnet_name:
656             id = s['id']
657             break
658     return id
659
660
661 def get_router_id(neutron_client, router_name):
662     routers = neutron_client.list_routers()['routers']
663     id = ''
664     for r in routers:
665         if r['name'] == router_name:
666             id = r['id']
667             break
668     return id
669
670
671 def get_private_net(neutron_client):
672     # Checks if there is an existing shared private network
673     networks = neutron_client.list_networks()['networks']
674     if len(networks) == 0:
675         return None
676     for net in networks:
677         if (net['router:external'] is False) and (net['shared'] is True):
678             return net
679     return None
680
681
682 def get_external_net(neutron_client):
683     for network in neutron_client.list_networks()['networks']:
684         if network['router:external']:
685             return network['name']
686     return None
687
688
689 def get_external_net_id(neutron_client):
690     for network in neutron_client.list_networks()['networks']:
691         if network['router:external']:
692             return network['id']
693     return None
694
695
696 def check_neutron_net(neutron_client, net_name):
697     for network in neutron_client.list_networks()['networks']:
698         if network['name'] == net_name:
699             for subnet in network['subnets']:
700                 return True
701     return False
702
703
704 def create_neutron_net(neutron_client, name):
705     json_body = {'network': {'name': name,
706                              'admin_state_up': True}}
707     try:
708         network = neutron_client.create_network(body=json_body)
709         network_dict = network['network']
710         return network_dict['id']
711     except Exception, e:
712         logger.error("Error [create_neutron_net(neutron_client, '%s')]: %s"
713                      % (name, e))
714         return None
715
716
717 def create_neutron_subnet(neutron_client, name, cidr, net_id):
718     json_body = {'subnets': [{'name': name, 'cidr': cidr,
719                               'ip_version': 4, 'network_id': net_id}]}
720     try:
721         subnet = neutron_client.create_subnet(body=json_body)
722         return subnet['subnets'][0]['id']
723     except Exception, e:
724         logger.error("Error [create_neutron_subnet(neutron_client, '%s', "
725                      "'%s', '%s')]: %s" % (name, cidr, net_id, e))
726         return None
727
728
729 def create_neutron_router(neutron_client, name):
730     json_body = {'router': {'name': name, 'admin_state_up': True}}
731     try:
732         router = neutron_client.create_router(json_body)
733         return router['router']['id']
734     except Exception, e:
735         logger.error("Error [create_neutron_router(neutron_client, '%s')]: %s"
736                      % (name, e))
737         return None
738
739
740 def create_neutron_port(neutron_client, name, network_id, ip):
741     json_body = {'port': {
742                  'admin_state_up': True,
743                  'name': name,
744                  'network_id': network_id,
745                  'fixed_ips': [{"ip_address": ip}]
746                  }}
747     try:
748         port = neutron_client.create_port(body=json_body)
749         return port['port']['id']
750     except Exception, e:
751         logger.error("Error [create_neutron_port(neutron_client, '%s', '%s', "
752                      "'%s')]: %s" % (name, network_id, ip, e))
753         return None
754
755
756 def update_neutron_net(neutron_client, network_id, shared=False):
757     json_body = {'network': {'shared': shared}}
758     try:
759         neutron_client.update_network(network_id, body=json_body)
760         return True
761     except Exception, e:
762         logger.error("Error [update_neutron_net(neutron_client, '%s', '%s')]: "
763                      "%s" % (network_id, str(shared), e))
764         return False
765
766
767 def update_neutron_port(neutron_client, port_id, device_owner):
768     json_body = {'port': {
769                  'device_owner': device_owner,
770                  }}
771     try:
772         port = neutron_client.update_port(port=port_id,
773                                           body=json_body)
774         return port['port']['id']
775     except Exception, e:
776         logger.error("Error [update_neutron_port(neutron_client, '%s', '%s')]:"
777                      " %s" % (port_id, device_owner, e))
778         return None
779
780
781 def add_interface_router(neutron_client, router_id, subnet_id):
782     json_body = {"subnet_id": subnet_id}
783     try:
784         neutron_client.add_interface_router(router=router_id, body=json_body)
785         return True
786     except Exception, e:
787         logger.error("Error [add_interface_router(neutron_client, '%s', "
788                      "'%s')]: %s" % (router_id, subnet_id, e))
789         return False
790
791
792 def add_gateway_router(neutron_client, router_id):
793     ext_net_id = get_external_net_id(neutron_client)
794     router_dict = {'network_id': ext_net_id}
795     try:
796         neutron_client.add_gateway_router(router_id, router_dict)
797         return True
798     except Exception, e:
799         logger.error("Error [add_gateway_router(neutron_client, '%s')]: %s"
800                      % (router_id, e))
801         return False
802
803
804 def delete_neutron_net(neutron_client, network_id):
805     try:
806         neutron_client.delete_network(network_id)
807         return True
808     except Exception, e:
809         logger.error("Error [delete_neutron_net(neutron_client, '%s')]: %s"
810                      % (network_id, e))
811         return False
812
813
814 def delete_neutron_subnet(neutron_client, subnet_id):
815     try:
816         neutron_client.delete_subnet(subnet_id)
817         return True
818     except Exception, e:
819         logger.error("Error [delete_neutron_subnet(neutron_client, '%s')]: %s"
820                      % (subnet_id, e))
821         return False
822
823
824 def delete_neutron_router(neutron_client, router_id):
825     try:
826         neutron_client.delete_router(router=router_id)
827         return True
828     except Exception, e:
829         logger.error("Error [delete_neutron_router(neutron_client, '%s')]: %s"
830                      % (router_id, e))
831         return False
832
833
834 def delete_neutron_port(neutron_client, port_id):
835     try:
836         neutron_client.delete_port(port_id)
837         return True
838     except Exception, e:
839         logger.error("Error [delete_neutron_port(neutron_client, '%s')]: %s"
840                      % (port_id, e))
841         return False
842
843
844 def remove_interface_router(neutron_client, router_id, subnet_id):
845     json_body = {"subnet_id": subnet_id}
846     try:
847         neutron_client.remove_interface_router(router=router_id,
848                                                body=json_body)
849         return True
850     except Exception, e:
851         logger.error("Error [remove_interface_router(neutron_client, '%s', "
852                      "'%s')]: %s" % (router_id, subnet_id, e))
853         return False
854
855
856 def remove_gateway_router(neutron_client, router_id):
857     try:
858         neutron_client.remove_gateway_router(router_id)
859         return True
860     except Exception, e:
861         logger.error("Error [remove_gateway_router(neutron_client, '%s')]: %s"
862                      % (router_id, e))
863         return False
864
865
866 def create_network_full(neutron_client,
867                         net_name,
868                         subnet_name,
869                         router_name,
870                         cidr):
871
872     # Check if the network already exists
873     network_id = get_network_id(neutron_client, net_name)
874     subnet_id = get_subnet_id(neutron_client, subnet_name)
875     router_id = get_router_id(neutron_client, router_name)
876
877     if network_id != '' and subnet_id != '' and router_id != '':
878         logger.info("A network with name '%s' already exists..." % net_name)
879     else:
880         neutron_client.format = 'json'
881         logger.info('Creating neutron network %s...' % net_name)
882         network_id = create_neutron_net(neutron_client, net_name)
883
884         if not network_id:
885             return False
886
887         logger.debug("Network '%s' created successfully" % network_id)
888         logger.debug('Creating Subnet....')
889         subnet_id = create_neutron_subnet(neutron_client, subnet_name,
890                                           cidr, network_id)
891         if not subnet_id:
892             return None
893
894         logger.debug("Subnet '%s' created successfully" % subnet_id)
895         logger.debug('Creating Router...')
896         router_id = create_neutron_router(neutron_client, router_name)
897
898         if not router_id:
899             return None
900
901         logger.debug("Router '%s' created successfully" % router_id)
902         logger.debug('Adding router to subnet...')
903
904         if not add_interface_router(neutron_client, router_id, subnet_id):
905             return None
906
907         logger.debug("Interface added successfully.")
908
909         logger.debug('Adding gateway to router...')
910         if not add_gateway_router(neutron_client, router_id):
911             return None
912
913         logger.debug("Gateway added successfully.")
914
915     network_dic = {'net_id': network_id,
916                    'subnet_id': subnet_id,
917                    'router_id': router_id}
918     return network_dic
919
920
921 def create_shared_network_full(net_name, subnt_name, router_name, subnet_cidr):
922     neutron_client = get_neutron_client()
923
924     network_dic = create_network_full(neutron_client,
925                                       net_name,
926                                       subnt_name,
927                                       router_name,
928                                       subnet_cidr)
929     if network_dic:
930         if not update_neutron_net(neutron_client,
931                                   network_dic['net_id'],
932                                   shared=True):
933             logger.error("Failed to update network %s..." % net_name)
934             return None
935         else:
936             logger.debug("Network '%s' is available..." % net_name)
937     else:
938         logger.error("Network %s creation failed" % net_name)
939         return None
940     return network_dic
941
942
943 def create_bgpvpn(neutron_client, **kwargs):
944     # route_distinguishers
945     # route_targets
946     json_body = {"bgpvpn": kwargs}
947     return neutron_client.create_bgpvpn(json_body)
948
949
950 def create_network_association(neutron_client, bgpvpn_id, neutron_network_id):
951     json_body = {"network_association": {"network_id": neutron_network_id}}
952     return neutron_client.create_network_association(bgpvpn_id, json_body)
953
954
955 def create_router_association(neutron_client, bgpvpn_id, router_id):
956     json_body = {"router_association": {"router_id": router_id}}
957     return neutron_client.create_router_association(bgpvpn_id, json_body)
958
959
960 def update_bgpvpn(neutron_client, bgpvpn_id, **kwargs):
961     json_body = {"bgpvpn": kwargs}
962     return neutron_client.update_bgpvpn(bgpvpn_id, json_body)
963
964
965 def delete_bgpvpn(neutron_client, bgpvpn_id):
966     return neutron_client.delete_bgpvpn(bgpvpn_id)
967
968
969 def get_bgpvpn(neutron_client, bgpvpn_id):
970     return neutron_client.show_bgpvpn(bgpvpn_id)
971
972
973 def get_bgpvpn_routers(neutron_client, bgpvpn_id):
974     return get_bgpvpn(neutron_client, bgpvpn_id)['bgpvpn']['routers']
975
976
977 def get_bgpvpn_networks(neutron_client, bgpvpn_id):
978     return get_bgpvpn(neutron_client, bgpvpn_id)['bgpvpn']['networks']
979
980 # *********************************************
981 #   SEC GROUPS
982 # *********************************************
983
984
985 def get_security_groups(neutron_client):
986     try:
987         security_groups = neutron_client.list_security_groups()[
988             'security_groups']
989         return security_groups
990     except Exception, e:
991         logger.error("Error [get_security_groups(neutron_client)]: %s" % e)
992         return None
993
994
995 def get_security_group_id(neutron_client, sg_name):
996     security_groups = get_security_groups(neutron_client)
997     id = ''
998     for sg in security_groups:
999         if sg['name'] == sg_name:
1000             id = sg['id']
1001             break
1002     return id
1003
1004
1005 def create_security_group(neutron_client, sg_name, sg_description):
1006     json_body = {'security_group': {'name': sg_name,
1007                                     'description': sg_description}}
1008     try:
1009         secgroup = neutron_client.create_security_group(json_body)
1010         return secgroup['security_group']
1011     except Exception, e:
1012         logger.error("Error [create_security_group(neutron_client, '%s', "
1013                      "'%s')]: %s" % (sg_name, sg_description, e))
1014         return None
1015
1016
1017 def create_secgroup_rule(neutron_client, sg_id, direction, protocol,
1018                          port_range_min=None, port_range_max=None):
1019     # We create a security group in 2 steps
1020     # 1 - we check the format and set the json body accordingly
1021     # 2 - we call neturon client to create the security group
1022
1023     # Format check
1024     json_body = {'security_group_rule': {'direction': direction,
1025                                          'security_group_id': sg_id,
1026                                          'protocol': protocol}}
1027     # parameters may be
1028     # - both None => we do nothing
1029     # - both Not None => we add them to the json description
1030     # but one cannot be None is the other is not None
1031     if (port_range_min is not None and port_range_max is not None):
1032         # add port_range in json description
1033         json_body['security_group_rule']['port_range_min'] = port_range_min
1034         json_body['security_group_rule']['port_range_max'] = port_range_max
1035         logger.debug("Security_group format set (port range included)")
1036     else:
1037         # either both port range are set to None => do nothing
1038         # or one is set but not the other => log it and return False
1039         if port_range_min is None and port_range_max is None:
1040             logger.debug("Security_group format set (no port range mentioned)")
1041         else:
1042             logger.error("Bad security group format."
1043                          "One of the port range is not properly set:"
1044                          "range min: {},"
1045                          "range max: {}".format(port_range_min,
1046                                                 port_range_max))
1047             return False
1048
1049     # Create security group using neutron client
1050     try:
1051         neutron_client.create_security_group_rule(json_body)
1052         return True
1053     except:
1054         logger.exception("Impossible to create_security_group_rule,"
1055                          "security group rule probably already exists")
1056         return False
1057
1058
1059 def get_security_group_rules(neutron_client, sg_id):
1060     try:
1061         security_rules = neutron_client.list_security_group_rules()[
1062             'security_group_rules']
1063         security_rules = [rule for rule in security_rules
1064                           if rule["security_group_id"] == sg_id]
1065         return security_rules
1066     except Exception, e:
1067         logger.error("Error [get_security_group_rules(neutron_client, sg_id)]:"
1068                      " %s" % e)
1069         return None
1070
1071
1072 def check_security_group_rules(neutron_client, sg_id, direction, protocol,
1073                                port_min=None, port_max=None):
1074     try:
1075         security_rules = get_security_group_rules(neutron_client, sg_id)
1076         security_rules = [rule for rule in security_rules
1077                           if (rule["direction"].lower() == direction
1078                               and rule["protocol"].lower() == protocol
1079                               and rule["port_range_min"] == port_min
1080                               and rule["port_range_max"] == port_max)]
1081         if len(security_rules) == 0:
1082             return True
1083         else:
1084             return False
1085     except Exception, e:
1086         logger.error("Error [check_security_group_rules("
1087                      " neutron_client, sg_id, direction,"
1088                      " protocol, port_min=None, port_max=None)]: "
1089                      "%s" % e)
1090         return None
1091
1092
1093 def create_security_group_full(neutron_client,
1094                                sg_name, sg_description):
1095     sg_id = get_security_group_id(neutron_client, sg_name)
1096     if sg_id != '':
1097         logger.info("Using existing security group '%s'..." % sg_name)
1098     else:
1099         logger.info("Creating security group  '%s'..." % sg_name)
1100         SECGROUP = create_security_group(neutron_client,
1101                                          sg_name,
1102                                          sg_description)
1103         if not SECGROUP:
1104             logger.error("Failed to create the security group...")
1105             return None
1106
1107         sg_id = SECGROUP['id']
1108
1109         logger.debug("Security group '%s' with ID=%s created successfully."
1110                      % (SECGROUP['name'], sg_id))
1111
1112         logger.debug("Adding ICMP rules in security group '%s'..."
1113                      % sg_name)
1114         if not create_secgroup_rule(neutron_client, sg_id,
1115                                     'ingress', 'icmp'):
1116             logger.error("Failed to create the security group rule...")
1117             return None
1118
1119         logger.debug("Adding SSH rules in security group '%s'..."
1120                      % sg_name)
1121         if not create_secgroup_rule(
1122                 neutron_client, sg_id, 'ingress', 'tcp', '22', '22'):
1123             logger.error("Failed to create the security group rule...")
1124             return None
1125
1126         if not create_secgroup_rule(
1127                 neutron_client, sg_id, 'egress', 'tcp', '22', '22'):
1128             logger.error("Failed to create the security group rule...")
1129             return None
1130     return sg_id
1131
1132
1133 def add_secgroup_to_instance(nova_client, instance_id, secgroup_id):
1134     try:
1135         nova_client.servers.add_security_group(instance_id, secgroup_id)
1136         return True
1137     except Exception, e:
1138         logger.error("Error [add_secgroup_to_instance(nova_client, '%s', "
1139                      "'%s')]: %s" % (instance_id, secgroup_id, e))
1140         return False
1141
1142
1143 def update_sg_quota(neutron_client, tenant_id, sg_quota, sg_rule_quota):
1144     json_body = {"quota": {
1145         "security_group": sg_quota,
1146         "security_group_rule": sg_rule_quota
1147     }}
1148
1149     try:
1150         neutron_client.update_quota(tenant_id=tenant_id,
1151                                     body=json_body)
1152         return True
1153     except Exception, e:
1154         logger.error("Error [update_sg_quota(neutron_client, '%s', '%s', "
1155                      "'%s')]: %s" % (tenant_id, sg_quota, sg_rule_quota, e))
1156         return False
1157
1158
1159 def delete_security_group(neutron_client, secgroup_id):
1160     try:
1161         neutron_client.delete_security_group(secgroup_id)
1162         return True
1163     except Exception, e:
1164         logger.error("Error [delete_security_group(neutron_client, '%s')]: %s"
1165                      % (secgroup_id, e))
1166         return False
1167
1168
1169 # *********************************************
1170 #   GLANCE
1171 # *********************************************
1172 def get_images(nova_client):
1173     try:
1174         images = nova_client.images.list()
1175         return images
1176     except Exception, e:
1177         logger.error("Error [get_images]: %s" % e)
1178         return None
1179
1180
1181 def get_image_id(glance_client, image_name):
1182     images = glance_client.images.list()
1183     id = ''
1184     for i in images:
1185         if i.name == image_name:
1186             id = i.id
1187             break
1188     return id
1189
1190
1191 def create_glance_image(glance_client, image_name, file_path, disk="qcow2",
1192                         container="bare", public="public"):
1193     if not os.path.isfile(file_path):
1194         logger.error("Error: file %s does not exist." % file_path)
1195         return None
1196     try:
1197         image_id = get_image_id(glance_client, image_name)
1198         if image_id != '':
1199             logger.info("Image %s already exists." % image_name)
1200         else:
1201             logger.info("Creating image '%s' from '%s'..." % (image_name,
1202                                                               file_path))
1203
1204             image = glance_client.images.create(name=image_name,
1205                                                 visibility=public,
1206                                                 disk_format=disk,
1207                                                 container_format=container)
1208             image_id = image.id
1209             with open(file_path) as image_data:
1210                 glance_client.images.upload(image_id, image_data)
1211         return image_id
1212     except Exception, e:
1213         logger.error("Error [create_glance_image(glance_client, '%s', '%s', "
1214                      "'%s')]: %s" % (image_name, file_path, public, e))
1215         return None
1216
1217
1218 def get_or_create_image(name, path, format):
1219     image_exists = False
1220     glance_client = get_glance_client()
1221
1222     image_id = get_image_id(glance_client, name)
1223     if image_id != '':
1224         logger.info("Using existing image '%s'..." % name)
1225         image_exists = True
1226     else:
1227         logger.info("Creating image '%s' from '%s'..." % (name, path))
1228         image_id = create_glance_image(glance_client, name, path, format)
1229         if not image_id:
1230             logger.error("Failed to create a Glance image...")
1231         else:
1232             logger.debug("Image '%s' with ID=%s created successfully."
1233                          % (name, image_id))
1234
1235     return image_exists, image_id
1236
1237
1238 def delete_glance_image(nova_client, image_id):
1239     try:
1240         nova_client.images.delete(image_id)
1241         return True
1242     except Exception, e:
1243         logger.error("Error [delete_glance_image(nova_client, '%s')]: %s"
1244                      % (image_id, e))
1245         return False
1246
1247
1248 # *********************************************
1249 #   CINDER
1250 # *********************************************
1251 def get_volumes(cinder_client):
1252     try:
1253         volumes = cinder_client.volumes.list(search_opts={'all_tenants': 1})
1254         return volumes
1255     except Exception, e:
1256         logger.error("Error [get_volumes(cinder_client)]: %s" % e)
1257         return None
1258
1259
1260 def list_volume_types(cinder_client, public=True, private=True):
1261     try:
1262         volume_types = cinder_client.volume_types.list()
1263         if not public:
1264             volume_types = [vt for vt in volume_types if not vt.is_public]
1265         if not private:
1266             volume_types = [vt for vt in volume_types if vt.is_public]
1267         return volume_types
1268     except Exception, e:
1269         logger.error("Error [list_volume_types(cinder_client)]: %s" % e)
1270         return None
1271
1272
1273 def create_volume_type(cinder_client, name):
1274     try:
1275         volume_type = cinder_client.volume_types.create(name)
1276         return volume_type
1277     except Exception, e:
1278         logger.error("Error [create_volume_type(cinder_client, '%s')]: %s"
1279                      % (name, e))
1280         return None
1281
1282
1283 def update_cinder_quota(cinder_client, tenant_id, vols_quota,
1284                         snapshots_quota, gigabytes_quota):
1285     quotas_values = {"volumes": vols_quota,
1286                      "snapshots": snapshots_quota,
1287                      "gigabytes": gigabytes_quota}
1288
1289     try:
1290         cinder_client.quotas.update(tenant_id, **quotas_values)
1291         return True
1292     except Exception, e:
1293         logger.error("Error [update_cinder_quota(cinder_client, '%s', '%s', "
1294                      "'%s' '%s')]: %s" % (tenant_id, vols_quota,
1295                                           snapshots_quota, gigabytes_quota, e))
1296         return False
1297
1298
1299 def delete_volume(cinder_client, volume_id, forced=False):
1300     try:
1301         if forced:
1302             try:
1303                 cinder_client.volumes.detach(volume_id)
1304             except:
1305                 logger.error(sys.exc_info()[0])
1306             cinder_client.volumes.force_delete(volume_id)
1307         else:
1308             cinder_client.volumes.delete(volume_id)
1309         return True
1310     except Exception, e:
1311         logger.error("Error [delete_volume(cinder_client, '%s', '%s')]: %s"
1312                      % (volume_id, str(forced), e))
1313         return False
1314
1315
1316 def delete_volume_type(cinder_client, volume_type):
1317     try:
1318         cinder_client.volume_types.delete(volume_type)
1319         return True
1320     except Exception, e:
1321         logger.error("Error [delete_volume_type(cinder_client, '%s')]: %s"
1322                      % (volume_type, e))
1323         return False
1324
1325
1326 # *********************************************
1327 #   KEYSTONE
1328 # *********************************************
1329 def get_tenants(keystone_client):
1330     try:
1331         if is_keystone_v3():
1332             tenants = keystone_client.projects.list()
1333         else:
1334             tenants = keystone_client.tenants.list()
1335         return tenants
1336     except Exception, e:
1337         logger.error("Error [get_tenants(keystone_client)]: %s" % e)
1338         return None
1339
1340
1341 def get_users(keystone_client):
1342     try:
1343         users = keystone_client.users.list()
1344         return users
1345     except Exception, e:
1346         logger.error("Error [get_users(keystone_client)]: %s" % e)
1347         return None
1348
1349
1350 def get_tenant_id(keystone_client, tenant_name):
1351     tenants = get_tenants(keystone_client)
1352     id = ''
1353     for t in tenants:
1354         if t.name == tenant_name:
1355             id = t.id
1356             break
1357     return id
1358
1359
1360 def get_user_id(keystone_client, user_name):
1361     users = get_users(keystone_client)
1362     id = ''
1363     for u in users:
1364         if u.name == user_name:
1365             id = u.id
1366             break
1367     return id
1368
1369
1370 def get_role_id(keystone_client, role_name):
1371     roles = keystone_client.roles.list()
1372     id = ''
1373     for r in roles:
1374         if r.name == role_name:
1375             id = r.id
1376             break
1377     return id
1378
1379
1380 def create_tenant(keystone_client, tenant_name, tenant_description):
1381     try:
1382         if is_keystone_v3():
1383             tenant = keystone_client.projects.create(
1384                 name=tenant_name,
1385                 description=tenant_description,
1386                 domain="default",
1387                 enabled=True)
1388         else:
1389             tenant = keystone_client.tenants.create(tenant_name,
1390                                                     tenant_description,
1391                                                     enabled=True)
1392         return tenant.id
1393     except Exception, e:
1394         logger.error("Error [create_tenant(keystone_client, '%s', '%s')]: %s"
1395                      % (tenant_name, tenant_description, e))
1396         return None
1397
1398
1399 def get_or_create_tenant(keystone_client, tenant_name, tenant_description):
1400     tenant_id = get_tenant_id(keystone_client, tenant_name)
1401     if not tenant_id:
1402         tenant_id = create_tenant(keystone_client, tenant_name,
1403                                   tenant_description)
1404
1405     return tenant_id
1406
1407
1408 def create_user(keystone_client, user_name, user_password,
1409                 user_email, tenant_id):
1410     try:
1411         if is_keystone_v3():
1412             user = keystone_client.users.create(name=user_name,
1413                                                 password=user_password,
1414                                                 email=user_email,
1415                                                 project_id=tenant_id,
1416                                                 enabled=True)
1417         else:
1418             user = keystone_client.users.create(user_name,
1419                                                 user_password,
1420                                                 user_email,
1421                                                 tenant_id,
1422                                                 enabled=True)
1423         return user.id
1424     except Exception, e:
1425         logger.error("Error [create_user(keystone_client, '%s', '%s', '%s'"
1426                      "'%s')]: %s" % (user_name, user_password,
1427                                      user_email, tenant_id, e))
1428         return None
1429
1430
1431 def get_or_create_user(keystone_client, user_name, user_password,
1432                        tenant_id, user_email=None):
1433     user_id = get_user_id(keystone_client, user_name)
1434     if not user_id:
1435         user_id = create_user(keystone_client, user_name, user_password,
1436                               user_email, tenant_id)
1437     return user_id
1438
1439
1440 def add_role_user(keystone_client, user_id, role_id, tenant_id):
1441     try:
1442         if is_keystone_v3():
1443             keystone_client.roles.grant(role=role_id,
1444                                         user=user_id,
1445                                         project=tenant_id)
1446         else:
1447             keystone_client.roles.add_user_role(user_id, role_id, tenant_id)
1448         return True
1449     except Exception, e:
1450         logger.error("Error [add_role_user(keystone_client, '%s', '%s'"
1451                      "'%s')]: %s " % (user_id, role_id, tenant_id, e))
1452         return False
1453
1454
1455 def delete_tenant(keystone_client, tenant_id):
1456     try:
1457         if is_keystone_v3():
1458             keystone_client.projects.delete(tenant_id)
1459         else:
1460             keystone_client.tenants.delete(tenant_id)
1461         return True
1462     except Exception, e:
1463         logger.error("Error [delete_tenant(keystone_client, '%s')]: %s"
1464                      % (tenant_id, e))
1465         return False
1466
1467
1468 def delete_user(keystone_client, user_id):
1469     try:
1470         keystone_client.users.delete(user_id)
1471         return True
1472     except Exception, e:
1473         logger.error("Error [delete_user(keystone_client, '%s')]: %s"
1474                      % (user_id, e))
1475         return False
1476
1477
1478 # *********************************************
1479 #   HEAT
1480 # *********************************************
1481 def get_resource(heat_client, stack_id, resource):
1482     try:
1483         resources = heat_client.resources.get(stack_id, resource)
1484         return resources
1485     except Exception, e:
1486         logger.error("Error [get_resource]: %s" % e)
1487         return None