702acc0254c29ae0be40429e2d8d84feded46216
[functest.git] / functest / utils / openstack_utils.py
1 #!/usr/bin/env python
2 #
3 # jose.lausuch@ericsson.com
4 # valentin.boucher@orange.com
5 # All rights reserved. This program and the accompanying materials
6 # are made available under the terms of the Apache License, Version 2.0
7 # which accompanies this distribution, and is available at
8 # http://www.apache.org/licenses/LICENSE-2.0
9 #
10
11 import os
12 import os.path
13 import re
14 import sys
15 import time
16
17 from keystoneauth1 import loading
18 from keystoneauth1 import session
19 from cinderclient import client as cinderclient
20 from glanceclient import client as glanceclient
21 from heatclient import client as heatclient
22 from novaclient import client as novaclient
23 from keystoneclient import client as keystoneclient
24 from neutronclient.neutron import client as neutronclient
25
26 import functest.utils.functest_logger as ft_logger
27 import functest.utils.functest_utils as ft_utils
28
29 logger = ft_logger.Logger("openstack_utils").getLogger()
30
31 DEFAULT_API_VERSION = '2'
32 DEFAULT_HEAT_API_VERSION = '1'
33
34
35 # *********************************************
36 #   CREDENTIALS
37 # *********************************************
38 class MissingEnvVar(Exception):
39
40     def __init__(self, var):
41         self.var = var
42
43     def __str__(self):
44         return str.format("Please set the mandatory env var: {}", self.var)
45
46
47 def is_keystone_v3():
48     keystone_api_version = os.getenv('OS_IDENTITY_API_VERSION')
49     if (keystone_api_version is None or
50             keystone_api_version == '2'):
51         return False
52     else:
53         return True
54
55
56 def get_rc_env_vars():
57     env_vars = ['OS_AUTH_URL', 'OS_USERNAME', 'OS_PASSWORD']
58     if is_keystone_v3():
59         env_vars.extend(['OS_PROJECT_NAME',
60                          'OS_USER_DOMAIN_NAME',
61                          'OS_PROJECT_DOMAIN_NAME'])
62     else:
63         env_vars.extend(['OS_TENANT_NAME'])
64     return env_vars
65
66
67 def check_credentials():
68     """
69     Check if the OpenStack credentials (openrc) are sourced
70     """
71     env_vars = get_rc_env_vars()
72     return all(map(lambda v: v in os.environ and os.environ[v], env_vars))
73
74
75 def get_env_cred_dict():
76     env_cred_dict = {
77         'OS_USERNAME': 'username',
78         'OS_PASSWORD': 'password',
79         'OS_AUTH_URL': 'auth_url',
80         'OS_TENANT_NAME': 'tenant_name',
81         'OS_USER_DOMAIN_NAME': 'user_domain_name',
82         'OS_PROJECT_DOMAIN_NAME': 'project_domain_name',
83         'OS_PROJECT_NAME': 'project_name',
84         'OS_ENDPOINT_TYPE': 'endpoint_type',
85         'OS_REGION_NAME': 'region_name',
86         'OS_CACERT': 'https_cacert'
87     }
88     return env_cred_dict
89
90
91 def get_credentials(other_creds={}):
92     """Returns a creds dictionary filled with parsed from env
93     """
94     creds = {}
95     env_vars = get_rc_env_vars()
96     env_cred_dict = get_env_cred_dict()
97
98     for envvar in env_vars:
99         if os.getenv(envvar) is None:
100             raise MissingEnvVar(envvar)
101         else:
102             creds_key = env_cred_dict.get(envvar)
103             creds.update({creds_key: os.getenv(envvar)})
104
105     if 'tenant' in other_creds.keys():
106         if is_keystone_v3():
107             tenant = 'project_name'
108         else:
109             tenant = 'tenant_name'
110         other_creds[tenant] = other_creds.pop('tenant')
111
112     creds.update(other_creds)
113
114     return creds
115
116
117 def source_credentials(rc_file):
118     with open(rc_file, "r") as f:
119         for line in f:
120             var = line.rstrip('"\n').replace('export ', '').split("=")
121             # The two next lines should be modified as soon as rc_file
122             # conforms with common rules. Be aware that it could induce
123             # issues if value starts with '
124             key = re.sub(r'^["\' ]*|[ \'"]*$', '', var[0])
125             value = re.sub(r'^["\' ]*|[ \'"]*$', '', "".join(var[1:]))
126             os.environ[key] = value
127
128
129 def get_credentials_for_rally():
130     creds = get_credentials()
131     env_cred_dict = get_env_cred_dict()
132     rally_conf = {"type": "ExistingCloud", "admin": {}}
133     for key in creds:
134         if key == 'auth_url':
135             rally_conf[key] = creds[key]
136         else:
137             rally_conf['admin'][key] = creds[key]
138
139     endpoint_types = [('internalURL', 'internal'),
140                       ('publicURL', 'public'), ('adminURL', 'admin')]
141
142     endpoint_type = get_endpoint_type_from_env()
143     if endpoint_type is not None:
144         cred_key = env_cred_dict.get('OS_ENDPOINT_TYPE')
145         for k, v in endpoint_types:
146             if endpoint_type == v:
147                 rally_conf[cred_key] = v
148
149     region_name = os.getenv('OS_REGION_NAME')
150     if region_name is not None:
151         cred_key = env_cred_dict.get('OS_REGION_NAME')
152         rally_conf[cred_key] = region_name
153
154     cacert = os.getenv('OS_CACERT')
155     if cacert is not None:
156         cred_key = env_cred_dict.get('OS_CACERT')
157         rally_conf[cred_key] = cacert
158     return rally_conf
159
160
161 def get_endpoint_type_from_env():
162     endpoint_type = os.environ.get("OS_ENDPOINT_TYPE",
163                                    os.environ.get("OS_INTERFACE"))
164     if endpoint_type and "URL" in endpoint_type:
165         endpoint_type = endpoint_type.replace("URL", "")
166     return endpoint_type
167
168
169 def get_session_auth(other_creds={}):
170     loader = loading.get_plugin_loader('password')
171     creds = get_credentials(other_creds)
172     auth = loader.load_from_options(**creds)
173     return auth
174
175
176 def get_endpoint(service_type, endpoint_type='publicURL'):
177     auth = get_session_auth()
178     return get_session().get_endpoint(auth=auth,
179                                       service_type=service_type,
180                                       endpoint_type=endpoint_type)
181
182
183 def get_session(other_creds={}):
184     auth = get_session_auth(other_creds)
185     cacert = os.getenv('OS_CACERT')
186     if cacert is not None:
187         if not os.path.isfile(cacert):
188             raise Exception("The 'OS_CACERT' environment"
189                             "variable is set to %s but the file"
190                             "does not exist.", cacert)
191
192     return session.Session(auth=auth, verify=cacert)
193
194
195 # *********************************************
196 #   CLIENTS
197 # *********************************************
198 def get_keystone_client_version():
199     api_version = os.getenv('OS_IDENTITY_API_VERSION')
200     if api_version is not None:
201         logger.info("OS_IDENTITY_API_VERSION is set in env as '%s'",
202                     api_version)
203         return api_version
204     return DEFAULT_API_VERSION
205
206
207 def get_keystone_client(other_creds={}):
208     sess = get_session(other_creds)
209     return keystoneclient.Client(get_keystone_client_version(), session=sess)
210
211
212 def get_nova_client_version():
213     api_version = os.getenv('OS_COMPUTE_API_VERSION')
214     if api_version is not None:
215         logger.info("OS_COMPUTE_API_VERSION is set in env as '%s'",
216                     api_version)
217         return api_version
218     return DEFAULT_API_VERSION
219
220
221 def get_nova_client(other_creds={}):
222     sess = get_session(other_creds)
223     return novaclient.Client(get_nova_client_version(), session=sess)
224
225
226 def get_cinder_client_version():
227     api_version = os.getenv('OS_VOLUME_API_VERSION')
228     if api_version is not None:
229         logger.info("OS_VOLUME_API_VERSION is set in env as '%s'",
230                     api_version)
231         return api_version
232     return DEFAULT_API_VERSION
233
234
235 def get_cinder_client(other_creds={}):
236     sess = get_session(other_creds)
237     return cinderclient.Client(get_cinder_client_version(), session=sess)
238
239
240 def get_neutron_client_version():
241     api_version = os.getenv('OS_NETWORK_API_VERSION')
242     if api_version is not None:
243         logger.info("OS_NETWORK_API_VERSION is set in env as '%s'",
244                     api_version)
245         return api_version
246     return DEFAULT_API_VERSION
247
248
249 def get_neutron_client(other_creds={}):
250     sess = get_session(other_creds)
251     return neutronclient.Client(get_neutron_client_version(), session=sess)
252
253
254 def get_glance_client_version():
255     api_version = os.getenv('OS_IMAGE_API_VERSION')
256     if api_version is not None:
257         logger.info("OS_IMAGE_API_VERSION is set in env as '%s'", api_version)
258         return api_version
259     return DEFAULT_API_VERSION
260
261
262 def get_glance_client(other_creds={}):
263     sess = get_session(other_creds)
264     return glanceclient.Client(get_glance_client_version(), session=sess)
265
266
267 def get_heat_client_version():
268     api_version = os.getenv('OS_ORCHESTRATION_API_VERSION')
269     if api_version is not None:
270         logger.info("OS_ORCHESTRATION_API_VERSION is set in env as '%s'",
271                     api_version)
272         return api_version
273     return DEFAULT_HEAT_API_VERSION
274
275
276 def get_heat_client(other_creds={}):
277     sess = get_session(other_creds)
278     return heatclient.Client(get_heat_client_version(), session=sess)
279
280
281 # *********************************************
282 #   NOVA
283 # *********************************************
284 def get_instances(nova_client):
285     try:
286         instances = nova_client.servers.list(search_opts={'all_tenants': 1})
287         return instances
288     except Exception, e:
289         logger.error("Error [get_instances(nova_client)]: %s" % e)
290         return None
291
292
293 def get_instance_status(nova_client, instance):
294     try:
295         instance = nova_client.servers.get(instance.id)
296         return instance.status
297     except Exception, e:
298         logger.error("Error [get_instance_status(nova_client)]: %s" % e)
299         return None
300
301
302 def get_instance_by_name(nova_client, instance_name):
303     try:
304         instance = nova_client.servers.find(name=instance_name)
305         return instance
306     except Exception, e:
307         logger.error("Error [get_instance_by_name(nova_client, '%s')]: %s"
308                      % (instance_name, e))
309         return None
310
311
312 def get_flavor_id(nova_client, flavor_name):
313     flavors = nova_client.flavors.list(detailed=True)
314     id = ''
315     for f in flavors:
316         if f.name == flavor_name:
317             id = f.id
318             break
319     return id
320
321
322 def get_flavor_id_by_ram_range(nova_client, min_ram, max_ram):
323     flavors = nova_client.flavors.list(detailed=True)
324     id = ''
325     for f in flavors:
326         if min_ram <= f.ram and f.ram <= max_ram:
327             id = f.id
328             break
329     return id
330
331
332 def get_aggregates(nova_client):
333     try:
334         aggregates = nova_client.aggregates.list()
335         return aggregates
336     except Exception, e:
337         logger.error("Error [get_aggregates(nova_client)]: %s" % e)
338         return None
339
340
341 def get_aggregate_id(nova_client, aggregate_name):
342     try:
343         aggregates = get_aggregates(nova_client)
344         _id = [ag.id for ag in aggregates if ag.name == aggregate_name][0]
345         return _id
346     except Exception, e:
347         logger.error("Error [get_aggregate_id(nova_client, %s)]:"
348                      " %s" % (aggregate_name, e))
349         return None
350
351
352 def get_availability_zones(nova_client):
353     try:
354         availability_zones = nova_client.availability_zones.list()
355         return availability_zones
356     except Exception, e:
357         logger.error("Error [get_availability_zones(nova_client)]: %s" % e)
358         return None
359
360
361 def get_availability_zone_names(nova_client):
362     try:
363         az_names = [az.zoneName for az in get_availability_zones(nova_client)]
364         return az_names
365     except Exception, e:
366         logger.error("Error [get_availability_zone_names(nova_client)]:"
367                      " %s" % e)
368         return None
369
370
371 def create_flavor(nova_client, flavor_name, ram, disk, vcpus, public=True):
372     try:
373         flavor = nova_client.flavors.create(
374             flavor_name, ram, vcpus, disk, is_public=public)
375         try:
376             extra_specs = ft_utils.get_functest_config(
377                 'general.flavor_extra_specs')
378             flavor.set_keys(extra_specs)
379         except ValueError:
380             # flavor extra specs are not configured, therefore skip the update
381             pass
382
383     except Exception, e:
384         logger.error("Error [create_flavor(nova_client, '%s', '%s', '%s', "
385                      "'%s')]: %s" % (flavor_name, ram, disk, vcpus, e))
386         return None
387     return flavor.id
388
389
390 def get_or_create_flavor(flavor_name, ram, disk, vcpus, public=True):
391     flavor_exists = False
392     nova_client = get_nova_client()
393
394     flavor_id = get_flavor_id(nova_client, flavor_name)
395     if flavor_id != '':
396         logger.info("Using existing flavor '%s'..." % flavor_name)
397         flavor_exists = True
398     else:
399         logger.info("Creating flavor '%s' with '%s' RAM, '%s' disk size, "
400                     "'%s' vcpus..." % (flavor_name, ram, disk, vcpus))
401         flavor_id = create_flavor(
402             nova_client, flavor_name, ram, disk, vcpus, public=public)
403         if not flavor_id:
404             logger.error("Failed to create flavor '%s'..." % (flavor_name))
405         else:
406             logger.debug("Flavor '%s' with ID=%s created successfully."
407                          % (flavor_name, flavor_id))
408
409     return flavor_exists, flavor_id
410
411
412 def get_floating_ips(nova_client):
413     try:
414         floating_ips = nova_client.floating_ips.list()
415         return floating_ips
416     except Exception, e:
417         logger.error("Error [get_floating_ips(nova_client)]: %s" % e)
418         return None
419
420
421 def get_hypervisors(nova_client):
422     try:
423         nodes = []
424         hypervisors = nova_client.hypervisors.list()
425         for hypervisor in hypervisors:
426             if hypervisor.state == "up":
427                 nodes.append(hypervisor.hypervisor_hostname)
428         return nodes
429     except Exception, e:
430         logger.error("Error [get_hypervisors(nova_client)]: %s" % e)
431         return None
432
433
434 def create_aggregate(nova_client, aggregate_name, av_zone):
435     try:
436         nova_client.aggregates.create(aggregate_name, av_zone)
437         return True
438     except Exception, e:
439         logger.error("Error [create_aggregate(nova_client, %s, %s)]: %s"
440                      % (aggregate_name, av_zone, e))
441         return None
442
443
444 def add_host_to_aggregate(nova_client, aggregate_name, compute_host):
445     try:
446         aggregate_id = get_aggregate_id(nova_client, aggregate_name)
447         nova_client.aggregates.add_host(aggregate_id, compute_host)
448         return True
449     except Exception, e:
450         logger.error("Error [add_host_to_aggregate(nova_client, %s, %s)]: %s"
451                      % (aggregate_name, compute_host, e))
452         return None
453
454
455 def create_aggregate_with_host(
456         nova_client, aggregate_name, av_zone, compute_host):
457     try:
458         create_aggregate(nova_client, aggregate_name, av_zone)
459         add_host_to_aggregate(nova_client, aggregate_name, compute_host)
460         return True
461     except Exception, e:
462         logger.error("Error [create_aggregate_with_host("
463                      "nova_client, %s, %s, %s)]: %s"
464                      % (aggregate_name, av_zone, compute_host, e))
465         return None
466
467
468 def create_instance(flavor_name,
469                     image_id,
470                     network_id,
471                     instance_name="functest-vm",
472                     confdrive=True,
473                     userdata=None,
474                     av_zone='',
475                     fixed_ip=None,
476                     files=None):
477     nova_client = get_nova_client()
478     try:
479         flavor = nova_client.flavors.find(name=flavor_name)
480     except:
481         flavors = nova_client.flavors.list()
482         logger.error("Error: Flavor '%s' not found. Available flavors are: "
483                      "\n%s" % (flavor_name, flavors))
484         return None
485     if fixed_ip is not None:
486         nics = {"net-id": network_id, "v4-fixed-ip": fixed_ip}
487     else:
488         nics = {"net-id": network_id}
489     if userdata is None:
490         instance = nova_client.servers.create(
491             name=instance_name,
492             flavor=flavor,
493             image=image_id,
494             nics=[nics],
495             availability_zone=av_zone,
496             files=files
497         )
498     else:
499         instance = nova_client.servers.create(
500             name=instance_name,
501             flavor=flavor,
502             image=image_id,
503             nics=[nics],
504             config_drive=confdrive,
505             userdata=userdata,
506             availability_zone=av_zone,
507             files=files
508         )
509     return instance
510
511
512 def create_instance_and_wait_for_active(flavor_name,
513                                         image_id,
514                                         network_id,
515                                         instance_name="",
516                                         config_drive=False,
517                                         userdata="",
518                                         av_zone='',
519                                         fixed_ip=None,
520                                         files=None):
521     SLEEP = 3
522     VM_BOOT_TIMEOUT = 180
523     nova_client = get_nova_client()
524     instance = create_instance(flavor_name,
525                                image_id,
526                                network_id,
527                                instance_name,
528                                config_drive,
529                                userdata,
530                                av_zone=av_zone,
531                                fixed_ip=fixed_ip,
532                                files=files)
533     count = VM_BOOT_TIMEOUT / SLEEP
534     for n in range(count, -1, -1):
535         status = get_instance_status(nova_client, instance)
536         if status.lower() == "active":
537             return instance
538         elif status.lower() == "error":
539             logger.error("The instance %s went to ERROR status."
540                          % instance_name)
541             return None
542         time.sleep(SLEEP)
543     logger.error("Timeout booting the instance %s." % instance_name)
544     return None
545
546
547 def create_floating_ip(neutron_client):
548     extnet_id = get_external_net_id(neutron_client)
549     props = {'floating_network_id': extnet_id}
550     try:
551         ip_json = neutron_client.create_floatingip({'floatingip': props})
552         fip_addr = ip_json['floatingip']['floating_ip_address']
553         fip_id = ip_json['floatingip']['id']
554     except Exception, e:
555         logger.error("Error [create_floating_ip(neutron_client)]: %s" % e)
556         return None
557     return {'fip_addr': fip_addr, 'fip_id': fip_id}
558
559
560 def add_floating_ip(nova_client, server_id, floatingip_addr):
561     try:
562         nova_client.servers.add_floating_ip(server_id, floatingip_addr)
563         return True
564     except Exception, e:
565         logger.error("Error [add_floating_ip(nova_client, '%s', '%s')]: %s"
566                      % (server_id, floatingip_addr, e))
567         return False
568
569
570 def delete_instance(nova_client, instance_id):
571     try:
572         nova_client.servers.force_delete(instance_id)
573         return True
574     except Exception, e:
575         logger.error("Error [delete_instance(nova_client, '%s')]: %s"
576                      % (instance_id, e))
577         return False
578
579
580 def delete_floating_ip(nova_client, floatingip_id):
581     try:
582         nova_client.floating_ips.delete(floatingip_id)
583         return True
584     except Exception, e:
585         logger.error("Error [delete_floating_ip(nova_client, '%s')]: %s"
586                      % (floatingip_id, e))
587         return False
588
589
590 def remove_host_from_aggregate(nova_client, aggregate_name, compute_host):
591     try:
592         aggregate_id = get_aggregate_id(nova_client, aggregate_name)
593         nova_client.aggregates.remove_host(aggregate_id, compute_host)
594         return True
595     except Exception, e:
596         logger.error("Error [remove_host_from_aggregate(nova_client, %s, %s)]:"
597                      " %s" % (aggregate_name, compute_host, e))
598         return False
599
600
601 def remove_hosts_from_aggregate(nova_client, aggregate_name):
602     aggregate_id = get_aggregate_id(nova_client, aggregate_name)
603     hosts = nova_client.aggregates.get(aggregate_id).hosts
604     assert(
605         all(remove_host_from_aggregate(nova_client, aggregate_name, host)
606             for host in hosts))
607
608
609 def delete_aggregate(nova_client, aggregate_name):
610     try:
611         remove_hosts_from_aggregate(nova_client, aggregate_name)
612         nova_client.aggregates.delete(aggregate_name)
613         return True
614     except Exception, e:
615         logger.error("Error [delete_aggregate(nova_client, %s)]: %s"
616                      % (aggregate_name, e))
617         return False
618
619
620 # *********************************************
621 #   NEUTRON
622 # *********************************************
623 def get_network_list(neutron_client):
624     network_list = neutron_client.list_networks()['networks']
625     if len(network_list) == 0:
626         return None
627     else:
628         return network_list
629
630
631 def get_router_list(neutron_client):
632     router_list = neutron_client.list_routers()['routers']
633     if len(router_list) == 0:
634         return None
635     else:
636         return router_list
637
638
639 def get_port_list(neutron_client):
640     port_list = neutron_client.list_ports()['ports']
641     if len(port_list) == 0:
642         return None
643     else:
644         return port_list
645
646
647 def get_network_id(neutron_client, network_name):
648     networks = neutron_client.list_networks()['networks']
649     id = ''
650     for n in networks:
651         if n['name'] == network_name:
652             id = n['id']
653             break
654     return id
655
656
657 def get_subnet_id(neutron_client, subnet_name):
658     subnets = neutron_client.list_subnets()['subnets']
659     id = ''
660     for s in subnets:
661         if s['name'] == subnet_name:
662             id = s['id']
663             break
664     return id
665
666
667 def get_router_id(neutron_client, router_name):
668     routers = neutron_client.list_routers()['routers']
669     id = ''
670     for r in routers:
671         if r['name'] == router_name:
672             id = r['id']
673             break
674     return id
675
676
677 def get_private_net(neutron_client):
678     # Checks if there is an existing shared private network
679     networks = neutron_client.list_networks()['networks']
680     if len(networks) == 0:
681         return None
682     for net in networks:
683         if (net['router:external'] is False) and (net['shared'] is True):
684             return net
685     return None
686
687
688 def get_external_net(neutron_client):
689     for network in neutron_client.list_networks()['networks']:
690         if network['router:external']:
691             return network['name']
692     return None
693
694
695 def get_external_net_id(neutron_client):
696     for network in neutron_client.list_networks()['networks']:
697         if network['router:external']:
698             return network['id']
699     return None
700
701
702 def check_neutron_net(neutron_client, net_name):
703     for network in neutron_client.list_networks()['networks']:
704         if network['name'] == net_name:
705             for subnet in network['subnets']:
706                 return True
707     return False
708
709
710 def create_neutron_net(neutron_client, name):
711     json_body = {'network': {'name': name,
712                              'admin_state_up': True}}
713     try:
714         network = neutron_client.create_network(body=json_body)
715         network_dict = network['network']
716         return network_dict['id']
717     except Exception, e:
718         logger.error("Error [create_neutron_net(neutron_client, '%s')]: %s"
719                      % (name, e))
720         return None
721
722
723 def create_neutron_subnet(neutron_client, name, cidr, net_id):
724     json_body = {'subnets': [{'name': name, 'cidr': cidr,
725                               'ip_version': 4, 'network_id': net_id}]}
726     try:
727         subnet = neutron_client.create_subnet(body=json_body)
728         return subnet['subnets'][0]['id']
729     except Exception, e:
730         logger.error("Error [create_neutron_subnet(neutron_client, '%s', "
731                      "'%s', '%s')]: %s" % (name, cidr, net_id, e))
732         return None
733
734
735 def create_neutron_router(neutron_client, name):
736     json_body = {'router': {'name': name, 'admin_state_up': True}}
737     try:
738         router = neutron_client.create_router(json_body)
739         return router['router']['id']
740     except Exception, e:
741         logger.error("Error [create_neutron_router(neutron_client, '%s')]: %s"
742                      % (name, e))
743         return None
744
745
746 def create_neutron_port(neutron_client, name, network_id, ip):
747     json_body = {'port': {
748                  'admin_state_up': True,
749                  'name': name,
750                  'network_id': network_id,
751                  'fixed_ips': [{"ip_address": ip}]
752                  }}
753     try:
754         port = neutron_client.create_port(body=json_body)
755         return port['port']['id']
756     except Exception, e:
757         logger.error("Error [create_neutron_port(neutron_client, '%s', '%s', "
758                      "'%s')]: %s" % (name, network_id, ip, e))
759         return None
760
761
762 def update_neutron_net(neutron_client, network_id, shared=False):
763     json_body = {'network': {'shared': shared}}
764     try:
765         neutron_client.update_network(network_id, body=json_body)
766         return True
767     except Exception, e:
768         logger.error("Error [update_neutron_net(neutron_client, '%s', '%s')]: "
769                      "%s" % (network_id, str(shared), e))
770         return False
771
772
773 def update_neutron_port(neutron_client, port_id, device_owner):
774     json_body = {'port': {
775                  'device_owner': device_owner,
776                  }}
777     try:
778         port = neutron_client.update_port(port=port_id,
779                                           body=json_body)
780         return port['port']['id']
781     except Exception, e:
782         logger.error("Error [update_neutron_port(neutron_client, '%s', '%s')]:"
783                      " %s" % (port_id, device_owner, e))
784         return None
785
786
787 def add_interface_router(neutron_client, router_id, subnet_id):
788     json_body = {"subnet_id": subnet_id}
789     try:
790         neutron_client.add_interface_router(router=router_id, body=json_body)
791         return True
792     except Exception, e:
793         logger.error("Error [add_interface_router(neutron_client, '%s', "
794                      "'%s')]: %s" % (router_id, subnet_id, e))
795         return False
796
797
798 def add_gateway_router(neutron_client, router_id):
799     ext_net_id = get_external_net_id(neutron_client)
800     router_dict = {'network_id': ext_net_id}
801     try:
802         neutron_client.add_gateway_router(router_id, router_dict)
803         return True
804     except Exception, e:
805         logger.error("Error [add_gateway_router(neutron_client, '%s')]: %s"
806                      % (router_id, e))
807         return False
808
809
810 def delete_neutron_net(neutron_client, network_id):
811     try:
812         neutron_client.delete_network(network_id)
813         return True
814     except Exception, e:
815         logger.error("Error [delete_neutron_net(neutron_client, '%s')]: %s"
816                      % (network_id, e))
817         return False
818
819
820 def delete_neutron_subnet(neutron_client, subnet_id):
821     try:
822         neutron_client.delete_subnet(subnet_id)
823         return True
824     except Exception, e:
825         logger.error("Error [delete_neutron_subnet(neutron_client, '%s')]: %s"
826                      % (subnet_id, e))
827         return False
828
829
830 def delete_neutron_router(neutron_client, router_id):
831     try:
832         neutron_client.delete_router(router=router_id)
833         return True
834     except Exception, e:
835         logger.error("Error [delete_neutron_router(neutron_client, '%s')]: %s"
836                      % (router_id, e))
837         return False
838
839
840 def delete_neutron_port(neutron_client, port_id):
841     try:
842         neutron_client.delete_port(port_id)
843         return True
844     except Exception, e:
845         logger.error("Error [delete_neutron_port(neutron_client, '%s')]: %s"
846                      % (port_id, e))
847         return False
848
849
850 def remove_interface_router(neutron_client, router_id, subnet_id):
851     json_body = {"subnet_id": subnet_id}
852     try:
853         neutron_client.remove_interface_router(router=router_id,
854                                                body=json_body)
855         return True
856     except Exception, e:
857         logger.error("Error [remove_interface_router(neutron_client, '%s', "
858                      "'%s')]: %s" % (router_id, subnet_id, e))
859         return False
860
861
862 def remove_gateway_router(neutron_client, router_id):
863     try:
864         neutron_client.remove_gateway_router(router_id)
865         return True
866     except Exception, e:
867         logger.error("Error [remove_gateway_router(neutron_client, '%s')]: %s"
868                      % (router_id, e))
869         return False
870
871
872 def create_network_full(neutron_client,
873                         net_name,
874                         subnet_name,
875                         router_name,
876                         cidr):
877
878     # Check if the network already exists
879     network_id = get_network_id(neutron_client, net_name)
880     subnet_id = get_subnet_id(neutron_client, subnet_name)
881     router_id = get_router_id(neutron_client, router_name)
882
883     if network_id != '' and subnet_id != '' and router_id != '':
884         logger.info("A network with name '%s' already exists..." % net_name)
885     else:
886         neutron_client.format = 'json'
887         logger.info('Creating neutron network %s...' % net_name)
888         network_id = create_neutron_net(neutron_client, net_name)
889
890         if not network_id:
891             return False
892
893         logger.debug("Network '%s' created successfully" % network_id)
894         logger.debug('Creating Subnet....')
895         subnet_id = create_neutron_subnet(neutron_client, subnet_name,
896                                           cidr, network_id)
897         if not subnet_id:
898             return None
899
900         logger.debug("Subnet '%s' created successfully" % subnet_id)
901         logger.debug('Creating Router...')
902         router_id = create_neutron_router(neutron_client, router_name)
903
904         if not router_id:
905             return None
906
907         logger.debug("Router '%s' created successfully" % router_id)
908         logger.debug('Adding router to subnet...')
909
910         if not add_interface_router(neutron_client, router_id, subnet_id):
911             return None
912
913         logger.debug("Interface added successfully.")
914
915         logger.debug('Adding gateway to router...')
916         if not add_gateway_router(neutron_client, router_id):
917             return None
918
919         logger.debug("Gateway added successfully.")
920
921     network_dic = {'net_id': network_id,
922                    'subnet_id': subnet_id,
923                    'router_id': router_id}
924     return network_dic
925
926
927 def create_shared_network_full(net_name, subnt_name, router_name, subnet_cidr):
928     neutron_client = get_neutron_client()
929
930     network_dic = create_network_full(neutron_client,
931                                       net_name,
932                                       subnt_name,
933                                       router_name,
934                                       subnet_cidr)
935     if network_dic:
936         if not update_neutron_net(neutron_client,
937                                   network_dic['net_id'],
938                                   shared=True):
939             logger.error("Failed to update network %s..." % net_name)
940             return None
941         else:
942             logger.debug("Network '%s' is available..." % net_name)
943     else:
944         logger.error("Network %s creation failed" % net_name)
945         return None
946     return network_dic
947
948
949 def create_bgpvpn(neutron_client, **kwargs):
950     # route_distinguishers
951     # route_targets
952     json_body = {"bgpvpn": kwargs}
953     return neutron_client.create_bgpvpn(json_body)
954
955
956 def create_network_association(neutron_client, bgpvpn_id, neutron_network_id):
957     json_body = {"network_association": {"network_id": neutron_network_id}}
958     return neutron_client.create_network_association(bgpvpn_id, json_body)
959
960
961 def create_router_association(neutron_client, bgpvpn_id, router_id):
962     json_body = {"router_association": {"router_id": router_id}}
963     return neutron_client.create_router_association(bgpvpn_id, json_body)
964
965
966 def update_bgpvpn(neutron_client, bgpvpn_id, **kwargs):
967     json_body = {"bgpvpn": kwargs}
968     return neutron_client.update_bgpvpn(bgpvpn_id, json_body)
969
970
971 def delete_bgpvpn(neutron_client, bgpvpn_id):
972     return neutron_client.delete_bgpvpn(bgpvpn_id)
973
974
975 def get_bgpvpn(neutron_client, bgpvpn_id):
976     return neutron_client.show_bgpvpn(bgpvpn_id)
977
978
979 def get_bgpvpn_routers(neutron_client, bgpvpn_id):
980     return get_bgpvpn(neutron_client, bgpvpn_id)['bgpvpn']['routers']
981
982
983 def get_bgpvpn_networks(neutron_client, bgpvpn_id):
984     return get_bgpvpn(neutron_client, bgpvpn_id)['bgpvpn']['networks']
985
986 # *********************************************
987 #   SEC GROUPS
988 # *********************************************
989
990
991 def get_security_groups(neutron_client):
992     try:
993         security_groups = neutron_client.list_security_groups()[
994             'security_groups']
995         return security_groups
996     except Exception, e:
997         logger.error("Error [get_security_groups(neutron_client)]: %s" % e)
998         return None
999
1000
1001 def get_security_group_id(neutron_client, sg_name):
1002     security_groups = get_security_groups(neutron_client)
1003     id = ''
1004     for sg in security_groups:
1005         if sg['name'] == sg_name:
1006             id = sg['id']
1007             break
1008     return id
1009
1010
1011 def create_security_group(neutron_client, sg_name, sg_description):
1012     json_body = {'security_group': {'name': sg_name,
1013                                     'description': sg_description}}
1014     try:
1015         secgroup = neutron_client.create_security_group(json_body)
1016         return secgroup['security_group']
1017     except Exception, e:
1018         logger.error("Error [create_security_group(neutron_client, '%s', "
1019                      "'%s')]: %s" % (sg_name, sg_description, e))
1020         return None
1021
1022
1023 def create_secgroup_rule(neutron_client, sg_id, direction, protocol,
1024                          port_range_min=None, port_range_max=None):
1025     # We create a security group in 2 steps
1026     # 1 - we check the format and set the json body accordingly
1027     # 2 - we call neturon client to create the security group
1028
1029     # Format check
1030     json_body = {'security_group_rule': {'direction': direction,
1031                                          'security_group_id': sg_id,
1032                                          'protocol': protocol}}
1033     # parameters may be
1034     # - both None => we do nothing
1035     # - both Not None => we add them to the json description
1036     # but one cannot be None is the other is not None
1037     if (port_range_min is not None and port_range_max is not None):
1038         # add port_range in json description
1039         json_body['security_group_rule']['port_range_min'] = port_range_min
1040         json_body['security_group_rule']['port_range_max'] = port_range_max
1041         logger.debug("Security_group format set (port range included)")
1042     else:
1043         # either both port range are set to None => do nothing
1044         # or one is set but not the other => log it and return False
1045         if port_range_min is None and port_range_max is None:
1046             logger.debug("Security_group format set (no port range mentioned)")
1047         else:
1048             logger.error("Bad security group format."
1049                          "One of the port range is not properly set:"
1050                          "range min: {},"
1051                          "range max: {}".format(port_range_min,
1052                                                 port_range_max))
1053             return False
1054
1055     # Create security group using neutron client
1056     try:
1057         neutron_client.create_security_group_rule(json_body)
1058         return True
1059     except:
1060         logger.exception("Impossible to create_security_group_rule,"
1061                          "security group rule probably already exists")
1062         return False
1063
1064
1065 def get_security_group_rules(neutron_client, sg_id):
1066     try:
1067         security_rules = neutron_client.list_security_group_rules()[
1068             'security_group_rules']
1069         security_rules = [rule for rule in security_rules
1070                           if rule["security_group_id"] == sg_id]
1071         return security_rules
1072     except Exception, e:
1073         logger.error("Error [get_security_group_rules(neutron_client, sg_id)]:"
1074                      " %s" % e)
1075         return None
1076
1077
1078 def check_security_group_rules(neutron_client, sg_id, direction, protocol,
1079                                port_min=None, port_max=None):
1080     try:
1081         security_rules = get_security_group_rules(neutron_client, sg_id)
1082         security_rules = [rule for rule in security_rules
1083                           if (rule["direction"].lower() == direction
1084                               and rule["protocol"].lower() == protocol
1085                               and rule["port_range_min"] == port_min
1086                               and rule["port_range_max"] == port_max)]
1087         if len(security_rules) == 0:
1088             return True
1089         else:
1090             return False
1091     except Exception, e:
1092         logger.error("Error [check_security_group_rules("
1093                      " neutron_client, sg_id, direction,"
1094                      " protocol, port_min=None, port_max=None)]: "
1095                      "%s" % e)
1096         return None
1097
1098
1099 def create_security_group_full(neutron_client,
1100                                sg_name, sg_description):
1101     sg_id = get_security_group_id(neutron_client, sg_name)
1102     if sg_id != '':
1103         logger.info("Using existing security group '%s'..." % sg_name)
1104     else:
1105         logger.info("Creating security group  '%s'..." % sg_name)
1106         SECGROUP = create_security_group(neutron_client,
1107                                          sg_name,
1108                                          sg_description)
1109         if not SECGROUP:
1110             logger.error("Failed to create the security group...")
1111             return None
1112
1113         sg_id = SECGROUP['id']
1114
1115         logger.debug("Security group '%s' with ID=%s created successfully."
1116                      % (SECGROUP['name'], sg_id))
1117
1118         logger.debug("Adding ICMP rules in security group '%s'..."
1119                      % sg_name)
1120         if not create_secgroup_rule(neutron_client, sg_id,
1121                                     'ingress', 'icmp'):
1122             logger.error("Failed to create the security group rule...")
1123             return None
1124
1125         logger.debug("Adding SSH rules in security group '%s'..."
1126                      % sg_name)
1127         if not create_secgroup_rule(
1128                 neutron_client, sg_id, 'ingress', 'tcp', '22', '22'):
1129             logger.error("Failed to create the security group rule...")
1130             return None
1131
1132         if not create_secgroup_rule(
1133                 neutron_client, sg_id, 'egress', 'tcp', '22', '22'):
1134             logger.error("Failed to create the security group rule...")
1135             return None
1136     return sg_id
1137
1138
1139 def add_secgroup_to_instance(nova_client, instance_id, secgroup_id):
1140     try:
1141         nova_client.servers.add_security_group(instance_id, secgroup_id)
1142         return True
1143     except Exception, e:
1144         logger.error("Error [add_secgroup_to_instance(nova_client, '%s', "
1145                      "'%s')]: %s" % (instance_id, secgroup_id, e))
1146         return False
1147
1148
1149 def update_sg_quota(neutron_client, tenant_id, sg_quota, sg_rule_quota):
1150     json_body = {"quota": {
1151         "security_group": sg_quota,
1152         "security_group_rule": sg_rule_quota
1153     }}
1154
1155     try:
1156         neutron_client.update_quota(tenant_id=tenant_id,
1157                                     body=json_body)
1158         return True
1159     except Exception, e:
1160         logger.error("Error [update_sg_quota(neutron_client, '%s', '%s', "
1161                      "'%s')]: %s" % (tenant_id, sg_quota, sg_rule_quota, e))
1162         return False
1163
1164
1165 def delete_security_group(neutron_client, secgroup_id):
1166     try:
1167         neutron_client.delete_security_group(secgroup_id)
1168         return True
1169     except Exception, e:
1170         logger.error("Error [delete_security_group(neutron_client, '%s')]: %s"
1171                      % (secgroup_id, e))
1172         return False
1173
1174
1175 # *********************************************
1176 #   GLANCE
1177 # *********************************************
1178 def get_images(nova_client):
1179     try:
1180         images = nova_client.images.list()
1181         return images
1182     except Exception, e:
1183         logger.error("Error [get_images]: %s" % e)
1184         return None
1185
1186
1187 def get_image_id(glance_client, image_name):
1188     images = glance_client.images.list()
1189     id = ''
1190     for i in images:
1191         if i.name == image_name:
1192             id = i.id
1193             break
1194     return id
1195
1196
1197 def create_glance_image(glance_client, image_name, file_path, disk="qcow2",
1198                         container="bare", public="public"):
1199     if not os.path.isfile(file_path):
1200         logger.error("Error: file %s does not exist." % file_path)
1201         return None
1202     try:
1203         image_id = get_image_id(glance_client, image_name)
1204         if image_id != '':
1205             logger.info("Image %s already exists." % image_name)
1206         else:
1207             logger.info("Creating image '%s' from '%s'..." % (image_name,
1208                                                               file_path))
1209
1210             image = glance_client.images.create(name=image_name,
1211                                                 visibility=public,
1212                                                 disk_format=disk,
1213                                                 container_format=container)
1214             image_id = image.id
1215             with open(file_path) as image_data:
1216                 glance_client.images.upload(image_id, image_data)
1217         return image_id
1218     except Exception, e:
1219         logger.error("Error [create_glance_image(glance_client, '%s', '%s', "
1220                      "'%s')]: %s" % (image_name, file_path, public, e))
1221         return None
1222
1223
1224 def get_or_create_image(name, path, format):
1225     image_exists = False
1226     glance_client = get_glance_client()
1227
1228     image_id = get_image_id(glance_client, name)
1229     if image_id != '':
1230         logger.info("Using existing image '%s'..." % name)
1231         image_exists = True
1232     else:
1233         logger.info("Creating image '%s' from '%s'..." % (name, path))
1234         image_id = create_glance_image(glance_client, name, path, format)
1235         if not image_id:
1236             logger.error("Failed to create a Glance image...")
1237         else:
1238             logger.debug("Image '%s' with ID=%s created successfully."
1239                          % (name, image_id))
1240
1241     return image_exists, image_id
1242
1243
1244 def delete_glance_image(nova_client, image_id):
1245     try:
1246         nova_client.images.delete(image_id)
1247         return True
1248     except Exception, e:
1249         logger.error("Error [delete_glance_image(nova_client, '%s')]: %s"
1250                      % (image_id, e))
1251         return False
1252
1253
1254 # *********************************************
1255 #   CINDER
1256 # *********************************************
1257 def get_volumes(cinder_client):
1258     try:
1259         volumes = cinder_client.volumes.list(search_opts={'all_tenants': 1})
1260         return volumes
1261     except Exception, e:
1262         logger.error("Error [get_volumes(cinder_client)]: %s" % e)
1263         return None
1264
1265
1266 def list_volume_types(cinder_client, public=True, private=True):
1267     try:
1268         volume_types = cinder_client.volume_types.list()
1269         if not public:
1270             volume_types = [vt for vt in volume_types if not vt.is_public]
1271         if not private:
1272             volume_types = [vt for vt in volume_types if vt.is_public]
1273         return volume_types
1274     except Exception, e:
1275         logger.error("Error [list_volume_types(cinder_client)]: %s" % e)
1276         return None
1277
1278
1279 def create_volume_type(cinder_client, name):
1280     try:
1281         volume_type = cinder_client.volume_types.create(name)
1282         return volume_type
1283     except Exception, e:
1284         logger.error("Error [create_volume_type(cinder_client, '%s')]: %s"
1285                      % (name, e))
1286         return None
1287
1288
1289 def update_cinder_quota(cinder_client, tenant_id, vols_quota,
1290                         snapshots_quota, gigabytes_quota):
1291     quotas_values = {"volumes": vols_quota,
1292                      "snapshots": snapshots_quota,
1293                      "gigabytes": gigabytes_quota}
1294
1295     try:
1296         cinder_client.quotas.update(tenant_id, **quotas_values)
1297         return True
1298     except Exception, e:
1299         logger.error("Error [update_cinder_quota(cinder_client, '%s', '%s', "
1300                      "'%s' '%s')]: %s" % (tenant_id, vols_quota,
1301                                           snapshots_quota, gigabytes_quota, e))
1302         return False
1303
1304
1305 def delete_volume(cinder_client, volume_id, forced=False):
1306     try:
1307         if forced:
1308             try:
1309                 cinder_client.volumes.detach(volume_id)
1310             except:
1311                 logger.error(sys.exc_info()[0])
1312             cinder_client.volumes.force_delete(volume_id)
1313         else:
1314             cinder_client.volumes.delete(volume_id)
1315         return True
1316     except Exception, e:
1317         logger.error("Error [delete_volume(cinder_client, '%s', '%s')]: %s"
1318                      % (volume_id, str(forced), e))
1319         return False
1320
1321
1322 def delete_volume_type(cinder_client, volume_type):
1323     try:
1324         cinder_client.volume_types.delete(volume_type)
1325         return True
1326     except Exception, e:
1327         logger.error("Error [delete_volume_type(cinder_client, '%s')]: %s"
1328                      % (volume_type, e))
1329         return False
1330
1331
1332 # *********************************************
1333 #   KEYSTONE
1334 # *********************************************
1335 def get_tenants(keystone_client):
1336     try:
1337         if is_keystone_v3():
1338             tenants = keystone_client.projects.list()
1339         else:
1340             tenants = keystone_client.tenants.list()
1341         return tenants
1342     except Exception, e:
1343         logger.error("Error [get_tenants(keystone_client)]: %s" % e)
1344         return None
1345
1346
1347 def get_users(keystone_client):
1348     try:
1349         users = keystone_client.users.list()
1350         return users
1351     except Exception, e:
1352         logger.error("Error [get_users(keystone_client)]: %s" % e)
1353         return None
1354
1355
1356 def get_tenant_id(keystone_client, tenant_name):
1357     tenants = get_tenants(keystone_client)
1358     id = ''
1359     for t in tenants:
1360         if t.name == tenant_name:
1361             id = t.id
1362             break
1363     return id
1364
1365
1366 def get_user_id(keystone_client, user_name):
1367     users = get_users(keystone_client)
1368     id = ''
1369     for u in users:
1370         if u.name == user_name:
1371             id = u.id
1372             break
1373     return id
1374
1375
1376 def get_role_id(keystone_client, role_name):
1377     roles = keystone_client.roles.list()
1378     id = ''
1379     for r in roles:
1380         if r.name == role_name:
1381             id = r.id
1382             break
1383     return id
1384
1385
1386 def create_tenant(keystone_client, tenant_name, tenant_description):
1387     try:
1388         if is_keystone_v3():
1389             tenant = keystone_client.projects.create(
1390                 name=tenant_name,
1391                 description=tenant_description,
1392                 domain="default",
1393                 enabled=True)
1394         else:
1395             tenant = keystone_client.tenants.create(tenant_name,
1396                                                     tenant_description,
1397                                                     enabled=True)
1398         return tenant.id
1399     except Exception, e:
1400         logger.error("Error [create_tenant(keystone_client, '%s', '%s')]: %s"
1401                      % (tenant_name, tenant_description, e))
1402         return None
1403
1404
1405 def get_or_create_tenant(keystone_client, tenant_name, tenant_description):
1406     tenant_id = get_tenant_id(keystone_client, tenant_name)
1407     if not tenant_id:
1408         tenant_id = create_tenant(keystone_client, tenant_name,
1409                                   tenant_description)
1410
1411     return tenant_id
1412
1413
1414 def create_user(keystone_client, user_name, user_password,
1415                 user_email, tenant_id):
1416     try:
1417         if is_keystone_v3():
1418             user = keystone_client.users.create(name=user_name,
1419                                                 password=user_password,
1420                                                 email=user_email,
1421                                                 project_id=tenant_id,
1422                                                 enabled=True)
1423         else:
1424             user = keystone_client.users.create(user_name,
1425                                                 user_password,
1426                                                 user_email,
1427                                                 tenant_id,
1428                                                 enabled=True)
1429         return user.id
1430     except Exception, e:
1431         logger.error("Error [create_user(keystone_client, '%s', '%s', '%s'"
1432                      "'%s')]: %s" % (user_name, user_password,
1433                                      user_email, tenant_id, e))
1434         return None
1435
1436
1437 def get_or_create_user(keystone_client, user_name, user_password,
1438                        tenant_id, user_email=None):
1439     user_id = get_user_id(keystone_client, user_name)
1440     if not user_id:
1441         user_id = create_user(keystone_client, user_name, user_password,
1442                               user_email, tenant_id)
1443     return user_id
1444
1445
1446 def add_role_user(keystone_client, user_id, role_id, tenant_id):
1447     try:
1448         if is_keystone_v3():
1449             keystone_client.roles.grant(role=role_id,
1450                                         user=user_id,
1451                                         project=tenant_id)
1452         else:
1453             keystone_client.roles.add_user_role(user_id, role_id, tenant_id)
1454         return True
1455     except Exception, e:
1456         logger.error("Error [add_role_user(keystone_client, '%s', '%s'"
1457                      "'%s')]: %s " % (user_id, role_id, tenant_id, e))
1458         return False
1459
1460
1461 def delete_tenant(keystone_client, tenant_id):
1462     try:
1463         if is_keystone_v3():
1464             keystone_client.projects.delete(tenant_id)
1465         else:
1466             keystone_client.tenants.delete(tenant_id)
1467         return True
1468     except Exception, e:
1469         logger.error("Error [delete_tenant(keystone_client, '%s')]: %s"
1470                      % (tenant_id, e))
1471         return False
1472
1473
1474 def delete_user(keystone_client, user_id):
1475     try:
1476         keystone_client.users.delete(user_id)
1477         return True
1478     except Exception, e:
1479         logger.error("Error [delete_user(keystone_client, '%s')]: %s"
1480                      % (user_id, e))
1481         return False
1482
1483
1484 # *********************************************
1485 #   HEAT
1486 # *********************************************
1487 def get_resource(heat_client, stack_id, resource):
1488     try:
1489         resources = heat_client.resources.get(stack_id, resource)
1490         return resources
1491     except Exception, e:
1492         logger.error("Error [get_resource]: %s" % e)
1493         return None