Merge "Update rally_conf.json for creating deployment"
[functest.git] / functest / utils / openstack_utils.py
1 #!/usr/bin/env python
2 #
3 # jose.lausuch@ericsson.com
4 # valentin.boucher@orange.com
5 # All rights reserved. This program and the accompanying materials
6 # are made available under the terms of the Apache License, Version 2.0
7 # which accompanies this distribution, and is available at
8 # http://www.apache.org/licenses/LICENSE-2.0
9 #
10
11 import os
12 import os.path
13 import re
14 import sys
15 import time
16
17 from keystoneauth1 import loading
18 from keystoneauth1 import session
19 from cinderclient import client as cinderclient
20 from glanceclient import client as glanceclient
21 from heatclient import client as heatclient
22 from novaclient import client as novaclient
23 from keystoneclient import client as keystoneclient
24 from neutronclient.neutron import client as neutronclient
25
26 import functest.utils.functest_logger as ft_logger
27 import functest.utils.functest_utils as ft_utils
28
29 logger = ft_logger.Logger("openstack_utils").getLogger()
30
31 DEFAULT_API_VERSION = '2'
32 DEFAULT_HEAT_API_VERSION = '1'
33
34
35 # *********************************************
36 #   CREDENTIALS
37 # *********************************************
38 class MissingEnvVar(Exception):
39
40     def __init__(self, var):
41         self.var = var
42
43     def __str__(self):
44         return str.format("Please set the mandatory env var: {}", self.var)
45
46
47 def is_keystone_v3():
48     keystone_api_version = os.getenv('OS_IDENTITY_API_VERSION')
49     if (keystone_api_version is None or
50             keystone_api_version == '2'):
51         return False
52     else:
53         return True
54
55
56 def get_rc_env_vars():
57     env_vars = ['OS_AUTH_URL', 'OS_USERNAME', 'OS_PASSWORD']
58     if is_keystone_v3():
59         env_vars.extend(['OS_PROJECT_NAME',
60                          'OS_USER_DOMAIN_NAME',
61                          'OS_PROJECT_DOMAIN_NAME'])
62     else:
63         env_vars.extend(['OS_TENANT_NAME'])
64     return env_vars
65
66
67 def check_credentials():
68     """
69     Check if the OpenStack credentials (openrc) are sourced
70     """
71     env_vars = get_rc_env_vars()
72     return all(map(lambda v: v in os.environ and os.environ[v], env_vars))
73
74
75 def get_env_cred_dict():
76     env_cred_dict = {
77         'OS_USERNAME': 'username',
78         'OS_PASSWORD': 'password',
79         'OS_AUTH_URL': 'auth_url',
80         'OS_TENANT_NAME': 'tenant_name',
81         'OS_USER_DOMAIN_NAME': 'user_domain_name',
82         'OS_PROJECT_DOMAIN_NAME': 'project_domain_name',
83         'OS_PROJECT_NAME': 'project_name',
84         'OS_ENDPOINT_TYPE': 'endpoint_type',
85         'OS_REGION_NAME': 'region_name',
86         'OS_CACERT': 'https_cacert'
87     }
88     return env_cred_dict
89
90
91 def get_credentials(other_creds={}):
92     """Returns a creds dictionary filled with parsed from env
93     """
94     creds = {}
95     env_vars = get_rc_env_vars()
96     env_cred_dict = get_env_cred_dict()
97
98     for envvar in env_vars:
99         if os.getenv(envvar) is None:
100             raise MissingEnvVar(envvar)
101         else:
102             creds_key = env_cred_dict.get(envvar)
103             creds.update({creds_key: os.getenv(envvar)})
104
105     if 'tenant' in other_creds.keys():
106         if is_keystone_v3():
107             tenant = 'project_name'
108         else:
109             tenant = 'tenant_name'
110         other_creds[tenant] = other_creds.pop('tenant')
111
112     creds.update(other_creds)
113
114     return creds
115
116
117 def source_credentials(rc_file):
118     with open(rc_file, "r") as f:
119         for line in f:
120             var = line.rstrip('"\n').replace('export ', '').split("=")
121             # The two next lines should be modified as soon as rc_file
122             # conforms with common rules. Be aware that it could induce
123             # issues if value starts with '
124             key = re.sub(r'^["\' ]*|[ \'"]*$', '', var[0])
125             value = re.sub(r'^["\' ]*|[ \'"]*$', '', "".join(var[1:]))
126             os.environ[key] = value
127
128
129 def get_credentials_for_rally():
130     creds = get_credentials()
131     env_cred_dict = get_env_cred_dict()
132     rally_conf = {"type": "ExistingCloud", "admin": {}}
133     for key in creds:
134         if key == 'auth_url':
135             rally_conf[key] = creds[key]
136         else:
137             rally_conf['admin'][key] = creds[key]
138
139     endpoint_types = [('internalURL', 'internal'),
140                       ('publicURL', 'public'), ('adminURL', 'admin')]
141
142     endpoint_type = get_endpoint_type_from_env()
143     if endpoint_type is not None:
144         cred_key = env_cred_dict.get('OS_ENDPOINT_TYPE')
145         for k, v in endpoint_types:
146             if endpoint_type == v:
147                 rally_conf[cred_key] = v
148
149     region_name = os.getenv('OS_REGION_NAME')
150     if region_name is not None:
151         cred_key = env_cred_dict.get('OS_REGION_NAME')
152         rally_conf[cred_key] = region_name
153
154     cacert = os.getenv('OS_CACERT')
155     if cacert is not None:
156         cred_key = env_cred_dict.get('OS_CACERT')
157         rally_conf[cred_key] = cacert
158     return rally_conf
159
160
161 def get_endpoint_type_from_env():
162     endpoint_type = os.environ.get("OS_ENDPOINT_TYPE",
163                                    os.environ.get("OS_INTERFACE"))
164     if endpoint_type and "URL" in endpoint_type:
165         endpoint_type = endpoint_type.replace("URL", "")
166     return endpoint_type
167
168
169 def get_session_auth(other_creds={}):
170     loader = loading.get_plugin_loader('password')
171     creds = get_credentials(other_creds)
172     auth = loader.load_from_options(**creds)
173     return auth
174
175
176 def get_endpoint(service_type, endpoint_type='publicURL'):
177     auth = get_session_auth()
178     return get_session().get_endpoint(auth=auth,
179                                       service_type=service_type,
180                                       endpoint_type=endpoint_type)
181
182
183 def get_session(other_creds={}):
184     auth = get_session_auth(other_creds)
185     cacert = os.getenv('OS_CACERT')
186     if cacert is not None:
187         if not os.path.isfile(cacert):
188             raise Exception("The 'OS_CACERT' environment"
189                             "variable is set to %s but the file"
190                             "does not exist.", cacert)
191
192     return session.Session(auth=auth, verify=cacert)
193
194
195 # *********************************************
196 #   CLIENTS
197 # *********************************************
198 def get_keystone_client_version():
199     api_version = os.getenv('OS_IDENTITY_API_VERSION')
200     if api_version is not None:
201         logger.info("OS_IDENTITY_API_VERSION is set in env as '%s'",
202                     api_version)
203         return api_version
204     return DEFAULT_API_VERSION
205
206
207 def get_keystone_client(other_creds={}):
208     sess = get_session(other_creds)
209     return keystoneclient.Client(get_keystone_client_version(),
210                                  session=sess,
211                                  interface=os.getenv('OS_INTERFACE', 'admin'))
212
213
214 def get_nova_client_version():
215     api_version = os.getenv('OS_COMPUTE_API_VERSION')
216     if api_version is not None:
217         logger.info("OS_COMPUTE_API_VERSION is set in env as '%s'",
218                     api_version)
219         return api_version
220     return DEFAULT_API_VERSION
221
222
223 def get_nova_client(other_creds={}):
224     sess = get_session(other_creds)
225     return novaclient.Client(get_nova_client_version(), session=sess)
226
227
228 def get_cinder_client_version():
229     api_version = os.getenv('OS_VOLUME_API_VERSION')
230     if api_version is not None:
231         logger.info("OS_VOLUME_API_VERSION is set in env as '%s'",
232                     api_version)
233         return api_version
234     return DEFAULT_API_VERSION
235
236
237 def get_cinder_client(other_creds={}):
238     sess = get_session(other_creds)
239     return cinderclient.Client(get_cinder_client_version(), session=sess)
240
241
242 def get_neutron_client_version():
243     api_version = os.getenv('OS_NETWORK_API_VERSION')
244     if api_version is not None:
245         logger.info("OS_NETWORK_API_VERSION is set in env as '%s'",
246                     api_version)
247         return api_version
248     return DEFAULT_API_VERSION
249
250
251 def get_neutron_client(other_creds={}):
252     sess = get_session(other_creds)
253     return neutronclient.Client(get_neutron_client_version(), session=sess)
254
255
256 def get_glance_client_version():
257     api_version = os.getenv('OS_IMAGE_API_VERSION')
258     if api_version is not None:
259         logger.info("OS_IMAGE_API_VERSION is set in env as '%s'", api_version)
260         return api_version
261     return DEFAULT_API_VERSION
262
263
264 def get_glance_client(other_creds={}):
265     sess = get_session(other_creds)
266     return glanceclient.Client(get_glance_client_version(), session=sess)
267
268
269 def get_heat_client_version():
270     api_version = os.getenv('OS_ORCHESTRATION_API_VERSION')
271     if api_version is not None:
272         logger.info("OS_ORCHESTRATION_API_VERSION is set in env as '%s'",
273                     api_version)
274         return api_version
275     return DEFAULT_HEAT_API_VERSION
276
277
278 def get_heat_client(other_creds={}):
279     sess = get_session(other_creds)
280     return heatclient.Client(get_heat_client_version(), session=sess)
281
282
283 # *********************************************
284 #   NOVA
285 # *********************************************
286 def get_instances(nova_client):
287     try:
288         instances = nova_client.servers.list(search_opts={'all_tenants': 1})
289         return instances
290     except Exception, e:
291         logger.error("Error [get_instances(nova_client)]: %s" % e)
292         return None
293
294
295 def get_instance_status(nova_client, instance):
296     try:
297         instance = nova_client.servers.get(instance.id)
298         return instance.status
299     except Exception, e:
300         logger.error("Error [get_instance_status(nova_client)]: %s" % e)
301         return None
302
303
304 def get_instance_by_name(nova_client, instance_name):
305     try:
306         instance = nova_client.servers.find(name=instance_name)
307         return instance
308     except Exception, e:
309         logger.error("Error [get_instance_by_name(nova_client, '%s')]: %s"
310                      % (instance_name, e))
311         return None
312
313
314 def get_flavor_id(nova_client, flavor_name):
315     flavors = nova_client.flavors.list(detailed=True)
316     id = ''
317     for f in flavors:
318         if f.name == flavor_name:
319             id = f.id
320             break
321     return id
322
323
324 def get_flavor_id_by_ram_range(nova_client, min_ram, max_ram):
325     flavors = nova_client.flavors.list(detailed=True)
326     id = ''
327     for f in flavors:
328         if min_ram <= f.ram and f.ram <= max_ram:
329             id = f.id
330             break
331     return id
332
333
334 def get_aggregates(nova_client):
335     try:
336         aggregates = nova_client.aggregates.list()
337         return aggregates
338     except Exception, e:
339         logger.error("Error [get_aggregates(nova_client)]: %s" % e)
340         return None
341
342
343 def get_aggregate_id(nova_client, aggregate_name):
344     try:
345         aggregates = get_aggregates(nova_client)
346         _id = [ag.id for ag in aggregates if ag.name == aggregate_name][0]
347         return _id
348     except Exception, e:
349         logger.error("Error [get_aggregate_id(nova_client, %s)]:"
350                      " %s" % (aggregate_name, e))
351         return None
352
353
354 def get_availability_zones(nova_client):
355     try:
356         availability_zones = nova_client.availability_zones.list()
357         return availability_zones
358     except Exception, e:
359         logger.error("Error [get_availability_zones(nova_client)]: %s" % e)
360         return None
361
362
363 def get_availability_zone_names(nova_client):
364     try:
365         az_names = [az.zoneName for az in get_availability_zones(nova_client)]
366         return az_names
367     except Exception, e:
368         logger.error("Error [get_availability_zone_names(nova_client)]:"
369                      " %s" % e)
370         return None
371
372
373 def create_flavor(nova_client, flavor_name, ram, disk, vcpus, public=True):
374     try:
375         flavor = nova_client.flavors.create(
376             flavor_name, ram, vcpus, disk, is_public=public)
377         try:
378             extra_specs = ft_utils.get_functest_config(
379                 'general.flavor_extra_specs')
380             flavor.set_keys(extra_specs)
381         except ValueError:
382             # flavor extra specs are not configured, therefore skip the update
383             pass
384
385     except Exception, e:
386         logger.error("Error [create_flavor(nova_client, '%s', '%s', '%s', "
387                      "'%s')]: %s" % (flavor_name, ram, disk, vcpus, e))
388         return None
389     return flavor.id
390
391
392 def get_or_create_flavor(flavor_name, ram, disk, vcpus, public=True):
393     flavor_exists = False
394     nova_client = get_nova_client()
395
396     flavor_id = get_flavor_id(nova_client, flavor_name)
397     if flavor_id != '':
398         logger.info("Using existing flavor '%s'..." % flavor_name)
399         flavor_exists = True
400     else:
401         logger.info("Creating flavor '%s' with '%s' RAM, '%s' disk size, "
402                     "'%s' vcpus..." % (flavor_name, ram, disk, vcpus))
403         flavor_id = create_flavor(
404             nova_client, flavor_name, ram, disk, vcpus, public=public)
405         if not flavor_id:
406             logger.error("Failed to create flavor '%s'..." % (flavor_name))
407         else:
408             logger.debug("Flavor '%s' with ID=%s created successfully."
409                          % (flavor_name, flavor_id))
410
411     return flavor_exists, flavor_id
412
413
414 def get_floating_ips(nova_client):
415     try:
416         floating_ips = nova_client.floating_ips.list()
417         return floating_ips
418     except Exception, e:
419         logger.error("Error [get_floating_ips(nova_client)]: %s" % e)
420         return None
421
422
423 def get_hypervisors(nova_client):
424     try:
425         nodes = []
426         hypervisors = nova_client.hypervisors.list()
427         for hypervisor in hypervisors:
428             if hypervisor.state == "up":
429                 nodes.append(hypervisor.hypervisor_hostname)
430         return nodes
431     except Exception, e:
432         logger.error("Error [get_hypervisors(nova_client)]: %s" % e)
433         return None
434
435
436 def create_aggregate(nova_client, aggregate_name, av_zone):
437     try:
438         nova_client.aggregates.create(aggregate_name, av_zone)
439         return True
440     except Exception, e:
441         logger.error("Error [create_aggregate(nova_client, %s, %s)]: %s"
442                      % (aggregate_name, av_zone, e))
443         return None
444
445
446 def add_host_to_aggregate(nova_client, aggregate_name, compute_host):
447     try:
448         aggregate_id = get_aggregate_id(nova_client, aggregate_name)
449         nova_client.aggregates.add_host(aggregate_id, compute_host)
450         return True
451     except Exception, e:
452         logger.error("Error [add_host_to_aggregate(nova_client, %s, %s)]: %s"
453                      % (aggregate_name, compute_host, e))
454         return None
455
456
457 def create_aggregate_with_host(
458         nova_client, aggregate_name, av_zone, compute_host):
459     try:
460         create_aggregate(nova_client, aggregate_name, av_zone)
461         add_host_to_aggregate(nova_client, aggregate_name, compute_host)
462         return True
463     except Exception, e:
464         logger.error("Error [create_aggregate_with_host("
465                      "nova_client, %s, %s, %s)]: %s"
466                      % (aggregate_name, av_zone, compute_host, e))
467         return None
468
469
470 def create_instance(flavor_name,
471                     image_id,
472                     network_id,
473                     instance_name="functest-vm",
474                     confdrive=True,
475                     userdata=None,
476                     av_zone='',
477                     fixed_ip=None,
478                     files=None):
479     nova_client = get_nova_client()
480     try:
481         flavor = nova_client.flavors.find(name=flavor_name)
482     except:
483         flavors = nova_client.flavors.list()
484         logger.error("Error: Flavor '%s' not found. Available flavors are: "
485                      "\n%s" % (flavor_name, flavors))
486         return None
487     if fixed_ip is not None:
488         nics = {"net-id": network_id, "v4-fixed-ip": fixed_ip}
489     else:
490         nics = {"net-id": network_id}
491     if userdata is None:
492         instance = nova_client.servers.create(
493             name=instance_name,
494             flavor=flavor,
495             image=image_id,
496             nics=[nics],
497             availability_zone=av_zone,
498             files=files
499         )
500     else:
501         instance = nova_client.servers.create(
502             name=instance_name,
503             flavor=flavor,
504             image=image_id,
505             nics=[nics],
506             config_drive=confdrive,
507             userdata=userdata,
508             availability_zone=av_zone,
509             files=files
510         )
511     return instance
512
513
514 def create_instance_and_wait_for_active(flavor_name,
515                                         image_id,
516                                         network_id,
517                                         instance_name="",
518                                         config_drive=False,
519                                         userdata="",
520                                         av_zone='',
521                                         fixed_ip=None,
522                                         files=None):
523     SLEEP = 3
524     VM_BOOT_TIMEOUT = 180
525     nova_client = get_nova_client()
526     instance = create_instance(flavor_name,
527                                image_id,
528                                network_id,
529                                instance_name,
530                                config_drive,
531                                userdata,
532                                av_zone=av_zone,
533                                fixed_ip=fixed_ip,
534                                files=files)
535     count = VM_BOOT_TIMEOUT / SLEEP
536     for n in range(count, -1, -1):
537         status = get_instance_status(nova_client, instance)
538         if status.lower() == "active":
539             return instance
540         elif status.lower() == "error":
541             logger.error("The instance %s went to ERROR status."
542                          % instance_name)
543             return None
544         time.sleep(SLEEP)
545     logger.error("Timeout booting the instance %s." % instance_name)
546     return None
547
548
549 def create_floating_ip(neutron_client):
550     extnet_id = get_external_net_id(neutron_client)
551     props = {'floating_network_id': extnet_id}
552     try:
553         ip_json = neutron_client.create_floatingip({'floatingip': props})
554         fip_addr = ip_json['floatingip']['floating_ip_address']
555         fip_id = ip_json['floatingip']['id']
556     except Exception, e:
557         logger.error("Error [create_floating_ip(neutron_client)]: %s" % e)
558         return None
559     return {'fip_addr': fip_addr, 'fip_id': fip_id}
560
561
562 def add_floating_ip(nova_client, server_id, floatingip_addr):
563     try:
564         nova_client.servers.add_floating_ip(server_id, floatingip_addr)
565         return True
566     except Exception, e:
567         logger.error("Error [add_floating_ip(nova_client, '%s', '%s')]: %s"
568                      % (server_id, floatingip_addr, e))
569         return False
570
571
572 def delete_instance(nova_client, instance_id):
573     try:
574         nova_client.servers.force_delete(instance_id)
575         return True
576     except Exception, e:
577         logger.error("Error [delete_instance(nova_client, '%s')]: %s"
578                      % (instance_id, e))
579         return False
580
581
582 def delete_floating_ip(nova_client, floatingip_id):
583     try:
584         nova_client.floating_ips.delete(floatingip_id)
585         return True
586     except Exception, e:
587         logger.error("Error [delete_floating_ip(nova_client, '%s')]: %s"
588                      % (floatingip_id, e))
589         return False
590
591
592 def remove_host_from_aggregate(nova_client, aggregate_name, compute_host):
593     try:
594         aggregate_id = get_aggregate_id(nova_client, aggregate_name)
595         nova_client.aggregates.remove_host(aggregate_id, compute_host)
596         return True
597     except Exception, e:
598         logger.error("Error [remove_host_from_aggregate(nova_client, %s, %s)]:"
599                      " %s" % (aggregate_name, compute_host, e))
600         return False
601
602
603 def remove_hosts_from_aggregate(nova_client, aggregate_name):
604     aggregate_id = get_aggregate_id(nova_client, aggregate_name)
605     hosts = nova_client.aggregates.get(aggregate_id).hosts
606     assert(
607         all(remove_host_from_aggregate(nova_client, aggregate_name, host)
608             for host in hosts))
609
610
611 def delete_aggregate(nova_client, aggregate_name):
612     try:
613         remove_hosts_from_aggregate(nova_client, aggregate_name)
614         nova_client.aggregates.delete(aggregate_name)
615         return True
616     except Exception, e:
617         logger.error("Error [delete_aggregate(nova_client, %s)]: %s"
618                      % (aggregate_name, e))
619         return False
620
621
622 # *********************************************
623 #   NEUTRON
624 # *********************************************
625 def get_network_list(neutron_client):
626     network_list = neutron_client.list_networks()['networks']
627     if len(network_list) == 0:
628         return None
629     else:
630         return network_list
631
632
633 def get_router_list(neutron_client):
634     router_list = neutron_client.list_routers()['routers']
635     if len(router_list) == 0:
636         return None
637     else:
638         return router_list
639
640
641 def get_port_list(neutron_client):
642     port_list = neutron_client.list_ports()['ports']
643     if len(port_list) == 0:
644         return None
645     else:
646         return port_list
647
648
649 def get_network_id(neutron_client, network_name):
650     networks = neutron_client.list_networks()['networks']
651     id = ''
652     for n in networks:
653         if n['name'] == network_name:
654             id = n['id']
655             break
656     return id
657
658
659 def get_subnet_id(neutron_client, subnet_name):
660     subnets = neutron_client.list_subnets()['subnets']
661     id = ''
662     for s in subnets:
663         if s['name'] == subnet_name:
664             id = s['id']
665             break
666     return id
667
668
669 def get_router_id(neutron_client, router_name):
670     routers = neutron_client.list_routers()['routers']
671     id = ''
672     for r in routers:
673         if r['name'] == router_name:
674             id = r['id']
675             break
676     return id
677
678
679 def get_private_net(neutron_client):
680     # Checks if there is an existing shared private network
681     networks = neutron_client.list_networks()['networks']
682     if len(networks) == 0:
683         return None
684     for net in networks:
685         if (net['router:external'] is False) and (net['shared'] is True):
686             return net
687     return None
688
689
690 def get_external_net(neutron_client):
691     for network in neutron_client.list_networks()['networks']:
692         if network['router:external']:
693             return network['name']
694     return None
695
696
697 def get_external_net_id(neutron_client):
698     for network in neutron_client.list_networks()['networks']:
699         if network['router:external']:
700             return network['id']
701     return None
702
703
704 def check_neutron_net(neutron_client, net_name):
705     for network in neutron_client.list_networks()['networks']:
706         if network['name'] == net_name:
707             for subnet in network['subnets']:
708                 return True
709     return False
710
711
712 def create_neutron_net(neutron_client, name):
713     json_body = {'network': {'name': name,
714                              'admin_state_up': True}}
715     try:
716         network = neutron_client.create_network(body=json_body)
717         network_dict = network['network']
718         return network_dict['id']
719     except Exception, e:
720         logger.error("Error [create_neutron_net(neutron_client, '%s')]: %s"
721                      % (name, e))
722         return None
723
724
725 def create_neutron_subnet(neutron_client, name, cidr, net_id):
726     json_body = {'subnets': [{'name': name, 'cidr': cidr,
727                               'ip_version': 4, 'network_id': net_id}]}
728     try:
729         subnet = neutron_client.create_subnet(body=json_body)
730         return subnet['subnets'][0]['id']
731     except Exception, e:
732         logger.error("Error [create_neutron_subnet(neutron_client, '%s', "
733                      "'%s', '%s')]: %s" % (name, cidr, net_id, e))
734         return None
735
736
737 def create_neutron_router(neutron_client, name):
738     json_body = {'router': {'name': name, 'admin_state_up': True}}
739     try:
740         router = neutron_client.create_router(json_body)
741         return router['router']['id']
742     except Exception, e:
743         logger.error("Error [create_neutron_router(neutron_client, '%s')]: %s"
744                      % (name, e))
745         return None
746
747
748 def create_neutron_port(neutron_client, name, network_id, ip):
749     json_body = {'port': {
750                  'admin_state_up': True,
751                  'name': name,
752                  'network_id': network_id,
753                  'fixed_ips': [{"ip_address": ip}]
754                  }}
755     try:
756         port = neutron_client.create_port(body=json_body)
757         return port['port']['id']
758     except Exception, e:
759         logger.error("Error [create_neutron_port(neutron_client, '%s', '%s', "
760                      "'%s')]: %s" % (name, network_id, ip, e))
761         return None
762
763
764 def update_neutron_net(neutron_client, network_id, shared=False):
765     json_body = {'network': {'shared': shared}}
766     try:
767         neutron_client.update_network(network_id, body=json_body)
768         return True
769     except Exception, e:
770         logger.error("Error [update_neutron_net(neutron_client, '%s', '%s')]: "
771                      "%s" % (network_id, str(shared), e))
772         return False
773
774
775 def update_neutron_port(neutron_client, port_id, device_owner):
776     json_body = {'port': {
777                  'device_owner': device_owner,
778                  }}
779     try:
780         port = neutron_client.update_port(port=port_id,
781                                           body=json_body)
782         return port['port']['id']
783     except Exception, e:
784         logger.error("Error [update_neutron_port(neutron_client, '%s', '%s')]:"
785                      " %s" % (port_id, device_owner, e))
786         return None
787
788
789 def add_interface_router(neutron_client, router_id, subnet_id):
790     json_body = {"subnet_id": subnet_id}
791     try:
792         neutron_client.add_interface_router(router=router_id, body=json_body)
793         return True
794     except Exception, e:
795         logger.error("Error [add_interface_router(neutron_client, '%s', "
796                      "'%s')]: %s" % (router_id, subnet_id, e))
797         return False
798
799
800 def add_gateway_router(neutron_client, router_id):
801     ext_net_id = get_external_net_id(neutron_client)
802     router_dict = {'network_id': ext_net_id}
803     try:
804         neutron_client.add_gateway_router(router_id, router_dict)
805         return True
806     except Exception, e:
807         logger.error("Error [add_gateway_router(neutron_client, '%s')]: %s"
808                      % (router_id, e))
809         return False
810
811
812 def delete_neutron_net(neutron_client, network_id):
813     try:
814         neutron_client.delete_network(network_id)
815         return True
816     except Exception, e:
817         logger.error("Error [delete_neutron_net(neutron_client, '%s')]: %s"
818                      % (network_id, e))
819         return False
820
821
822 def delete_neutron_subnet(neutron_client, subnet_id):
823     try:
824         neutron_client.delete_subnet(subnet_id)
825         return True
826     except Exception, e:
827         logger.error("Error [delete_neutron_subnet(neutron_client, '%s')]: %s"
828                      % (subnet_id, e))
829         return False
830
831
832 def delete_neutron_router(neutron_client, router_id):
833     try:
834         neutron_client.delete_router(router=router_id)
835         return True
836     except Exception, e:
837         logger.error("Error [delete_neutron_router(neutron_client, '%s')]: %s"
838                      % (router_id, e))
839         return False
840
841
842 def delete_neutron_port(neutron_client, port_id):
843     try:
844         neutron_client.delete_port(port_id)
845         return True
846     except Exception, e:
847         logger.error("Error [delete_neutron_port(neutron_client, '%s')]: %s"
848                      % (port_id, e))
849         return False
850
851
852 def remove_interface_router(neutron_client, router_id, subnet_id):
853     json_body = {"subnet_id": subnet_id}
854     try:
855         neutron_client.remove_interface_router(router=router_id,
856                                                body=json_body)
857         return True
858     except Exception, e:
859         logger.error("Error [remove_interface_router(neutron_client, '%s', "
860                      "'%s')]: %s" % (router_id, subnet_id, e))
861         return False
862
863
864 def remove_gateway_router(neutron_client, router_id):
865     try:
866         neutron_client.remove_gateway_router(router_id)
867         return True
868     except Exception, e:
869         logger.error("Error [remove_gateway_router(neutron_client, '%s')]: %s"
870                      % (router_id, e))
871         return False
872
873
874 def create_network_full(neutron_client,
875                         net_name,
876                         subnet_name,
877                         router_name,
878                         cidr):
879
880     # Check if the network already exists
881     network_id = get_network_id(neutron_client, net_name)
882     subnet_id = get_subnet_id(neutron_client, subnet_name)
883     router_id = get_router_id(neutron_client, router_name)
884
885     if network_id != '' and subnet_id != '' and router_id != '':
886         logger.info("A network with name '%s' already exists..." % net_name)
887     else:
888         neutron_client.format = 'json'
889         logger.info('Creating neutron network %s...' % net_name)
890         network_id = create_neutron_net(neutron_client, net_name)
891
892         if not network_id:
893             return False
894
895         logger.debug("Network '%s' created successfully" % network_id)
896         logger.debug('Creating Subnet....')
897         subnet_id = create_neutron_subnet(neutron_client, subnet_name,
898                                           cidr, network_id)
899         if not subnet_id:
900             return None
901
902         logger.debug("Subnet '%s' created successfully" % subnet_id)
903         logger.debug('Creating Router...')
904         router_id = create_neutron_router(neutron_client, router_name)
905
906         if not router_id:
907             return None
908
909         logger.debug("Router '%s' created successfully" % router_id)
910         logger.debug('Adding router to subnet...')
911
912         if not add_interface_router(neutron_client, router_id, subnet_id):
913             return None
914
915         logger.debug("Interface added successfully.")
916
917         logger.debug('Adding gateway to router...')
918         if not add_gateway_router(neutron_client, router_id):
919             return None
920
921         logger.debug("Gateway added successfully.")
922
923     network_dic = {'net_id': network_id,
924                    'subnet_id': subnet_id,
925                    'router_id': router_id}
926     return network_dic
927
928
929 def create_shared_network_full(net_name, subnt_name, router_name, subnet_cidr):
930     neutron_client = get_neutron_client()
931
932     network_dic = create_network_full(neutron_client,
933                                       net_name,
934                                       subnt_name,
935                                       router_name,
936                                       subnet_cidr)
937     if network_dic:
938         if not update_neutron_net(neutron_client,
939                                   network_dic['net_id'],
940                                   shared=True):
941             logger.error("Failed to update network %s..." % net_name)
942             return None
943         else:
944             logger.debug("Network '%s' is available..." % net_name)
945     else:
946         logger.error("Network %s creation failed" % net_name)
947         return None
948     return network_dic
949
950
951 def create_bgpvpn(neutron_client, **kwargs):
952     # route_distinguishers
953     # route_targets
954     json_body = {"bgpvpn": kwargs}
955     return neutron_client.create_bgpvpn(json_body)
956
957
958 def create_network_association(neutron_client, bgpvpn_id, neutron_network_id):
959     json_body = {"network_association": {"network_id": neutron_network_id}}
960     return neutron_client.create_network_association(bgpvpn_id, json_body)
961
962
963 def create_router_association(neutron_client, bgpvpn_id, router_id):
964     json_body = {"router_association": {"router_id": router_id}}
965     return neutron_client.create_router_association(bgpvpn_id, json_body)
966
967
968 def update_bgpvpn(neutron_client, bgpvpn_id, **kwargs):
969     json_body = {"bgpvpn": kwargs}
970     return neutron_client.update_bgpvpn(bgpvpn_id, json_body)
971
972
973 def delete_bgpvpn(neutron_client, bgpvpn_id):
974     return neutron_client.delete_bgpvpn(bgpvpn_id)
975
976
977 def get_bgpvpn(neutron_client, bgpvpn_id):
978     return neutron_client.show_bgpvpn(bgpvpn_id)
979
980
981 def get_bgpvpn_routers(neutron_client, bgpvpn_id):
982     return get_bgpvpn(neutron_client, bgpvpn_id)['bgpvpn']['routers']
983
984
985 def get_bgpvpn_networks(neutron_client, bgpvpn_id):
986     return get_bgpvpn(neutron_client, bgpvpn_id)['bgpvpn']['networks']
987
988 # *********************************************
989 #   SEC GROUPS
990 # *********************************************
991
992
993 def get_security_groups(neutron_client):
994     try:
995         security_groups = neutron_client.list_security_groups()[
996             'security_groups']
997         return security_groups
998     except Exception, e:
999         logger.error("Error [get_security_groups(neutron_client)]: %s" % e)
1000         return None
1001
1002
1003 def get_security_group_id(neutron_client, sg_name):
1004     security_groups = get_security_groups(neutron_client)
1005     id = ''
1006     for sg in security_groups:
1007         if sg['name'] == sg_name:
1008             id = sg['id']
1009             break
1010     return id
1011
1012
1013 def create_security_group(neutron_client, sg_name, sg_description):
1014     json_body = {'security_group': {'name': sg_name,
1015                                     'description': sg_description}}
1016     try:
1017         secgroup = neutron_client.create_security_group(json_body)
1018         return secgroup['security_group']
1019     except Exception, e:
1020         logger.error("Error [create_security_group(neutron_client, '%s', "
1021                      "'%s')]: %s" % (sg_name, sg_description, e))
1022         return None
1023
1024
1025 def create_secgroup_rule(neutron_client, sg_id, direction, protocol,
1026                          port_range_min=None, port_range_max=None):
1027     # We create a security group in 2 steps
1028     # 1 - we check the format and set the json body accordingly
1029     # 2 - we call neturon client to create the security group
1030
1031     # Format check
1032     json_body = {'security_group_rule': {'direction': direction,
1033                                          'security_group_id': sg_id,
1034                                          'protocol': protocol}}
1035     # parameters may be
1036     # - both None => we do nothing
1037     # - both Not None => we add them to the json description
1038     # but one cannot be None is the other is not None
1039     if (port_range_min is not None and port_range_max is not None):
1040         # add port_range in json description
1041         json_body['security_group_rule']['port_range_min'] = port_range_min
1042         json_body['security_group_rule']['port_range_max'] = port_range_max
1043         logger.debug("Security_group format set (port range included)")
1044     else:
1045         # either both port range are set to None => do nothing
1046         # or one is set but not the other => log it and return False
1047         if port_range_min is None and port_range_max is None:
1048             logger.debug("Security_group format set (no port range mentioned)")
1049         else:
1050             logger.error("Bad security group format."
1051                          "One of the port range is not properly set:"
1052                          "range min: {},"
1053                          "range max: {}".format(port_range_min,
1054                                                 port_range_max))
1055             return False
1056
1057     # Create security group using neutron client
1058     try:
1059         neutron_client.create_security_group_rule(json_body)
1060         return True
1061     except:
1062         logger.exception("Impossible to create_security_group_rule,"
1063                          "security group rule probably already exists")
1064         return False
1065
1066
1067 def get_security_group_rules(neutron_client, sg_id):
1068     try:
1069         security_rules = neutron_client.list_security_group_rules()[
1070             'security_group_rules']
1071         security_rules = [rule for rule in security_rules
1072                           if rule["security_group_id"] == sg_id]
1073         return security_rules
1074     except Exception, e:
1075         logger.error("Error [get_security_group_rules(neutron_client, sg_id)]:"
1076                      " %s" % e)
1077         return None
1078
1079
1080 def check_security_group_rules(neutron_client, sg_id, direction, protocol,
1081                                port_min=None, port_max=None):
1082     try:
1083         security_rules = get_security_group_rules(neutron_client, sg_id)
1084         security_rules = [rule for rule in security_rules
1085                           if (rule["direction"].lower() == direction
1086                               and rule["protocol"].lower() == protocol
1087                               and rule["port_range_min"] == port_min
1088                               and rule["port_range_max"] == port_max)]
1089         if len(security_rules) == 0:
1090             return True
1091         else:
1092             return False
1093     except Exception, e:
1094         logger.error("Error [check_security_group_rules("
1095                      " neutron_client, sg_id, direction,"
1096                      " protocol, port_min=None, port_max=None)]: "
1097                      "%s" % e)
1098         return None
1099
1100
1101 def create_security_group_full(neutron_client,
1102                                sg_name, sg_description):
1103     sg_id = get_security_group_id(neutron_client, sg_name)
1104     if sg_id != '':
1105         logger.info("Using existing security group '%s'..." % sg_name)
1106     else:
1107         logger.info("Creating security group  '%s'..." % sg_name)
1108         SECGROUP = create_security_group(neutron_client,
1109                                          sg_name,
1110                                          sg_description)
1111         if not SECGROUP:
1112             logger.error("Failed to create the security group...")
1113             return None
1114
1115         sg_id = SECGROUP['id']
1116
1117         logger.debug("Security group '%s' with ID=%s created successfully."
1118                      % (SECGROUP['name'], sg_id))
1119
1120         logger.debug("Adding ICMP rules in security group '%s'..."
1121                      % sg_name)
1122         if not create_secgroup_rule(neutron_client, sg_id,
1123                                     'ingress', 'icmp'):
1124             logger.error("Failed to create the security group rule...")
1125             return None
1126
1127         logger.debug("Adding SSH rules in security group '%s'..."
1128                      % sg_name)
1129         if not create_secgroup_rule(
1130                 neutron_client, sg_id, 'ingress', 'tcp', '22', '22'):
1131             logger.error("Failed to create the security group rule...")
1132             return None
1133
1134         if not create_secgroup_rule(
1135                 neutron_client, sg_id, 'egress', 'tcp', '22', '22'):
1136             logger.error("Failed to create the security group rule...")
1137             return None
1138     return sg_id
1139
1140
1141 def add_secgroup_to_instance(nova_client, instance_id, secgroup_id):
1142     try:
1143         nova_client.servers.add_security_group(instance_id, secgroup_id)
1144         return True
1145     except Exception, e:
1146         logger.error("Error [add_secgroup_to_instance(nova_client, '%s', "
1147                      "'%s')]: %s" % (instance_id, secgroup_id, e))
1148         return False
1149
1150
1151 def update_sg_quota(neutron_client, tenant_id, sg_quota, sg_rule_quota):
1152     json_body = {"quota": {
1153         "security_group": sg_quota,
1154         "security_group_rule": sg_rule_quota
1155     }}
1156
1157     try:
1158         neutron_client.update_quota(tenant_id=tenant_id,
1159                                     body=json_body)
1160         return True
1161     except Exception, e:
1162         logger.error("Error [update_sg_quota(neutron_client, '%s', '%s', "
1163                      "'%s')]: %s" % (tenant_id, sg_quota, sg_rule_quota, e))
1164         return False
1165
1166
1167 def delete_security_group(neutron_client, secgroup_id):
1168     try:
1169         neutron_client.delete_security_group(secgroup_id)
1170         return True
1171     except Exception, e:
1172         logger.error("Error [delete_security_group(neutron_client, '%s')]: %s"
1173                      % (secgroup_id, e))
1174         return False
1175
1176
1177 # *********************************************
1178 #   GLANCE
1179 # *********************************************
1180 def get_images(nova_client):
1181     try:
1182         images = nova_client.images.list()
1183         return images
1184     except Exception, e:
1185         logger.error("Error [get_images]: %s" % e)
1186         return None
1187
1188
1189 def get_image_id(glance_client, image_name):
1190     images = glance_client.images.list()
1191     id = ''
1192     for i in images:
1193         if i.name == image_name:
1194             id = i.id
1195             break
1196     return id
1197
1198
1199 def create_glance_image(glance_client, image_name, file_path, disk="qcow2",
1200                         container="bare", public="public"):
1201     if not os.path.isfile(file_path):
1202         logger.error("Error: file %s does not exist." % file_path)
1203         return None
1204     try:
1205         image_id = get_image_id(glance_client, image_name)
1206         if image_id != '':
1207             logger.info("Image %s already exists." % image_name)
1208         else:
1209             logger.info("Creating image '%s' from '%s'..." % (image_name,
1210                                                               file_path))
1211
1212             image = glance_client.images.create(name=image_name,
1213                                                 visibility=public,
1214                                                 disk_format=disk,
1215                                                 container_format=container)
1216             image_id = image.id
1217             with open(file_path) as image_data:
1218                 glance_client.images.upload(image_id, image_data)
1219         return image_id
1220     except Exception, e:
1221         logger.error("Error [create_glance_image(glance_client, '%s', '%s', "
1222                      "'%s')]: %s" % (image_name, file_path, public, e))
1223         return None
1224
1225
1226 def get_or_create_image(name, path, format):
1227     image_exists = False
1228     glance_client = get_glance_client()
1229
1230     image_id = get_image_id(glance_client, name)
1231     if image_id != '':
1232         logger.info("Using existing image '%s'..." % name)
1233         image_exists = True
1234     else:
1235         logger.info("Creating image '%s' from '%s'..." % (name, path))
1236         image_id = create_glance_image(glance_client, name, path, format)
1237         if not image_id:
1238             logger.error("Failed to create a Glance image...")
1239         else:
1240             logger.debug("Image '%s' with ID=%s created successfully."
1241                          % (name, image_id))
1242
1243     return image_exists, image_id
1244
1245
1246 def delete_glance_image(nova_client, image_id):
1247     try:
1248         nova_client.images.delete(image_id)
1249         return True
1250     except Exception, e:
1251         logger.error("Error [delete_glance_image(nova_client, '%s')]: %s"
1252                      % (image_id, e))
1253         return False
1254
1255
1256 # *********************************************
1257 #   CINDER
1258 # *********************************************
1259 def get_volumes(cinder_client):
1260     try:
1261         volumes = cinder_client.volumes.list(search_opts={'all_tenants': 1})
1262         return volumes
1263     except Exception, e:
1264         logger.error("Error [get_volumes(cinder_client)]: %s" % e)
1265         return None
1266
1267
1268 def list_volume_types(cinder_client, public=True, private=True):
1269     try:
1270         volume_types = cinder_client.volume_types.list()
1271         if not public:
1272             volume_types = [vt for vt in volume_types if not vt.is_public]
1273         if not private:
1274             volume_types = [vt for vt in volume_types if vt.is_public]
1275         return volume_types
1276     except Exception, e:
1277         logger.error("Error [list_volume_types(cinder_client)]: %s" % e)
1278         return None
1279
1280
1281 def create_volume_type(cinder_client, name):
1282     try:
1283         volume_type = cinder_client.volume_types.create(name)
1284         return volume_type
1285     except Exception, e:
1286         logger.error("Error [create_volume_type(cinder_client, '%s')]: %s"
1287                      % (name, e))
1288         return None
1289
1290
1291 def update_cinder_quota(cinder_client, tenant_id, vols_quota,
1292                         snapshots_quota, gigabytes_quota):
1293     quotas_values = {"volumes": vols_quota,
1294                      "snapshots": snapshots_quota,
1295                      "gigabytes": gigabytes_quota}
1296
1297     try:
1298         cinder_client.quotas.update(tenant_id, **quotas_values)
1299         return True
1300     except Exception, e:
1301         logger.error("Error [update_cinder_quota(cinder_client, '%s', '%s', "
1302                      "'%s' '%s')]: %s" % (tenant_id, vols_quota,
1303                                           snapshots_quota, gigabytes_quota, e))
1304         return False
1305
1306
1307 def delete_volume(cinder_client, volume_id, forced=False):
1308     try:
1309         if forced:
1310             try:
1311                 cinder_client.volumes.detach(volume_id)
1312             except:
1313                 logger.error(sys.exc_info()[0])
1314             cinder_client.volumes.force_delete(volume_id)
1315         else:
1316             cinder_client.volumes.delete(volume_id)
1317         return True
1318     except Exception, e:
1319         logger.error("Error [delete_volume(cinder_client, '%s', '%s')]: %s"
1320                      % (volume_id, str(forced), e))
1321         return False
1322
1323
1324 def delete_volume_type(cinder_client, volume_type):
1325     try:
1326         cinder_client.volume_types.delete(volume_type)
1327         return True
1328     except Exception, e:
1329         logger.error("Error [delete_volume_type(cinder_client, '%s')]: %s"
1330                      % (volume_type, e))
1331         return False
1332
1333
1334 # *********************************************
1335 #   KEYSTONE
1336 # *********************************************
1337 def get_tenants(keystone_client):
1338     try:
1339         if is_keystone_v3():
1340             tenants = keystone_client.projects.list()
1341         else:
1342             tenants = keystone_client.tenants.list()
1343         return tenants
1344     except Exception, e:
1345         logger.error("Error [get_tenants(keystone_client)]: %s" % e)
1346         return None
1347
1348
1349 def get_users(keystone_client):
1350     try:
1351         users = keystone_client.users.list()
1352         return users
1353     except Exception, e:
1354         logger.error("Error [get_users(keystone_client)]: %s" % e)
1355         return None
1356
1357
1358 def get_tenant_id(keystone_client, tenant_name):
1359     tenants = get_tenants(keystone_client)
1360     id = ''
1361     for t in tenants:
1362         if t.name == tenant_name:
1363             id = t.id
1364             break
1365     return id
1366
1367
1368 def get_user_id(keystone_client, user_name):
1369     users = get_users(keystone_client)
1370     id = ''
1371     for u in users:
1372         if u.name == user_name:
1373             id = u.id
1374             break
1375     return id
1376
1377
1378 def get_role_id(keystone_client, role_name):
1379     roles = keystone_client.roles.list()
1380     id = ''
1381     for r in roles:
1382         if r.name == role_name:
1383             id = r.id
1384             break
1385     return id
1386
1387
1388 def create_tenant(keystone_client, tenant_name, tenant_description):
1389     try:
1390         if is_keystone_v3():
1391             tenant = keystone_client.projects.create(
1392                 name=tenant_name,
1393                 description=tenant_description,
1394                 domain="default",
1395                 enabled=True)
1396         else:
1397             tenant = keystone_client.tenants.create(tenant_name,
1398                                                     tenant_description,
1399                                                     enabled=True)
1400         return tenant.id
1401     except Exception, e:
1402         logger.error("Error [create_tenant(keystone_client, '%s', '%s')]: %s"
1403                      % (tenant_name, tenant_description, e))
1404         return None
1405
1406
1407 def get_or_create_tenant(keystone_client, tenant_name, tenant_description):
1408     tenant_id = get_tenant_id(keystone_client, tenant_name)
1409     if not tenant_id:
1410         tenant_id = create_tenant(keystone_client, tenant_name,
1411                                   tenant_description)
1412
1413     return tenant_id
1414
1415
1416 def create_user(keystone_client, user_name, user_password,
1417                 user_email, tenant_id):
1418     try:
1419         if is_keystone_v3():
1420             user = keystone_client.users.create(name=user_name,
1421                                                 password=user_password,
1422                                                 email=user_email,
1423                                                 project_id=tenant_id,
1424                                                 enabled=True)
1425         else:
1426             user = keystone_client.users.create(user_name,
1427                                                 user_password,
1428                                                 user_email,
1429                                                 tenant_id,
1430                                                 enabled=True)
1431         return user.id
1432     except Exception, e:
1433         logger.error("Error [create_user(keystone_client, '%s', '%s', '%s'"
1434                      "'%s')]: %s" % (user_name, user_password,
1435                                      user_email, tenant_id, e))
1436         return None
1437
1438
1439 def get_or_create_user(keystone_client, user_name, user_password,
1440                        tenant_id, user_email=None):
1441     user_id = get_user_id(keystone_client, user_name)
1442     if not user_id:
1443         user_id = create_user(keystone_client, user_name, user_password,
1444                               user_email, tenant_id)
1445     return user_id
1446
1447
1448 def add_role_user(keystone_client, user_id, role_id, tenant_id):
1449     try:
1450         if is_keystone_v3():
1451             keystone_client.roles.grant(role=role_id,
1452                                         user=user_id,
1453                                         project=tenant_id)
1454         else:
1455             keystone_client.roles.add_user_role(user_id, role_id, tenant_id)
1456         return True
1457     except Exception, e:
1458         logger.error("Error [add_role_user(keystone_client, '%s', '%s'"
1459                      "'%s')]: %s " % (user_id, role_id, tenant_id, e))
1460         return False
1461
1462
1463 def delete_tenant(keystone_client, tenant_id):
1464     try:
1465         if is_keystone_v3():
1466             keystone_client.projects.delete(tenant_id)
1467         else:
1468             keystone_client.tenants.delete(tenant_id)
1469         return True
1470     except Exception, e:
1471         logger.error("Error [delete_tenant(keystone_client, '%s')]: %s"
1472                      % (tenant_id, e))
1473         return False
1474
1475
1476 def delete_user(keystone_client, user_id):
1477     try:
1478         keystone_client.users.delete(user_id)
1479         return True
1480     except Exception, e:
1481         logger.error("Error [delete_user(keystone_client, '%s')]: %s"
1482                      % (user_id, e))
1483         return False
1484
1485
1486 # *********************************************
1487 #   HEAT
1488 # *********************************************
1489 def get_resource(heat_client, stack_id, resource):
1490     try:
1491         resources = heat_client.resources.get(stack_id, resource)
1492         return resources
1493     except Exception, e:
1494         logger.error("Error [get_resource]: %s" % e)
1495         return None