Merge "Add rpm,image directories and SLA options to Livemigration"
[yardstick.git] / yardstick / common / openstack_utils.py
1 ##############################################################################
2 # Copyright (c) 2016 Huawei Technologies Co.,Ltd and others.
3 #
4 # All rights reserved. This program and the accompanying materials
5 # are made available under the terms of the Apache License, Version 2.0
6 # which accompanies this distribution, and is available at
7 # http://www.apache.org/licenses/LICENSE-2.0
8 ##############################################################################
9
10 from __future__ import absolute_import
11
12 import os
13 import time
14 import sys
15 import logging
16
17 from keystoneauth1 import loading
18 from keystoneauth1 import session
19 from cinderclient import client as cinderclient
20 from novaclient import client as novaclient
21 from glanceclient import client as glanceclient
22 from neutronclient.neutron import client as neutronclient
23
24 log = logging.getLogger(__name__)
25
26 DEFAULT_HEAT_API_VERSION = '1'
27 DEFAULT_API_VERSION = '2'
28
29
30 # *********************************************
31 #   CREDENTIALS
32 # *********************************************
33 def get_credentials():
34     """Returns a creds dictionary filled with parsed from env"""
35     creds = {}
36
37     keystone_api_version = os.getenv('OS_IDENTITY_API_VERSION')
38
39     if keystone_api_version is None or keystone_api_version == '2':
40         keystone_v3 = False
41         tenant_env = 'OS_TENANT_NAME'
42         tenant = 'tenant_name'
43     else:
44         keystone_v3 = True
45         tenant_env = 'OS_PROJECT_NAME'
46         tenant = 'project_name'
47
48     # The most common way to pass these info to the script is to do it
49     # through environment variables.
50     creds.update({
51         "username": os.environ.get("OS_USERNAME"),
52         "password": os.environ.get("OS_PASSWORD"),
53         "auth_url": os.environ.get("OS_AUTH_URL"),
54         tenant: os.environ.get(tenant_env)
55     })
56
57     if keystone_v3:
58         if os.getenv('OS_USER_DOMAIN_NAME') is not None:
59             creds.update({
60                 "user_domain_name": os.getenv('OS_USER_DOMAIN_NAME')
61             })
62         if os.getenv('OS_PROJECT_DOMAIN_NAME') is not None:
63             creds.update({
64                 "project_domain_name": os.getenv('OS_PROJECT_DOMAIN_NAME')
65             })
66
67     return creds
68
69
70 def get_session_auth():
71     loader = loading.get_plugin_loader('password')
72     creds = get_credentials()
73     auth = loader.load_from_options(**creds)
74     return auth
75
76
77 def get_session():
78     auth = get_session_auth()
79     try:
80         cacert = os.environ['OS_CACERT']
81     except KeyError:
82         return session.Session(auth=auth)
83     else:
84         insecure = os.getenv('OS_INSECURE', '').lower() == 'true'
85         cacert = False if insecure else cacert
86         return session.Session(auth=auth, verify=cacert)
87
88
89 def get_endpoint(service_type, endpoint_type='publicURL'):
90     auth = get_session_auth()
91     # for multi-region, we need to specify region
92     # when finding the endpoint
93     return get_session().get_endpoint(auth=auth,
94                                       service_type=service_type,
95                                       endpoint_type=endpoint_type,
96                                       region_name=os.environ.get(
97                                           "OS_REGION_NAME"))
98
99
100 # *********************************************
101 #   CLIENTS
102 # *********************************************
103 def get_heat_api_version():     # pragma: no cover
104     try:
105         api_version = os.environ['HEAT_API_VERSION']
106     except KeyError:
107         return DEFAULT_HEAT_API_VERSION
108     else:
109         log.info("HEAT_API_VERSION is set in env as '%s'", api_version)
110         return api_version
111
112
113 def get_cinder_client_version():      # pragma: no cover
114     try:
115         api_version = os.environ['OS_VOLUME_API_VERSION']
116     except KeyError:
117         return DEFAULT_API_VERSION
118     else:
119         log.info("OS_VOLUME_API_VERSION is set in env as '%s'", api_version)
120         return api_version
121
122
123 def get_cinder_client():      # pragma: no cover
124     sess = get_session()
125     return cinderclient.Client(get_cinder_client_version(), session=sess)
126
127
128 def get_nova_client_version():      # pragma: no cover
129     try:
130         api_version = os.environ['OS_COMPUTE_API_VERSION']
131     except KeyError:
132         return DEFAULT_API_VERSION
133     else:
134         log.info("OS_COMPUTE_API_VERSION is set in env as '%s'", api_version)
135         return api_version
136
137
138 def get_nova_client():      # pragma: no cover
139     sess = get_session()
140     return novaclient.Client(get_nova_client_version(), session=sess)
141
142
143 def get_neutron_client_version():   # pragma: no cover
144     try:
145         api_version = os.environ['OS_NETWORK_API_VERSION']
146     except KeyError:
147         return DEFAULT_API_VERSION
148     else:
149         log.info("OS_NETWORK_API_VERSION is set in env as '%s'", api_version)
150         return api_version
151
152
153 def get_neutron_client():   # pragma: no cover
154     sess = get_session()
155     return neutronclient.Client(get_neutron_client_version(), session=sess)
156
157
158 def get_glance_client_version():    # pragma: no cover
159     try:
160         api_version = os.environ['OS_IMAGE_API_VERSION']
161     except KeyError:
162         return DEFAULT_API_VERSION
163     else:
164         log.info("OS_IMAGE_API_VERSION is set in env as '%s'", api_version)
165         return api_version
166
167
168 def get_glance_client():    # pragma: no cover
169     sess = get_session()
170     return glanceclient.Client(get_glance_client_version(), session=sess)
171
172
173 # *********************************************
174 #   NOVA
175 # *********************************************
176 def get_instances(nova_client):     # pragma: no cover
177     try:
178         return nova_client.servers.list(search_opts={'all_tenants': 1})
179     except Exception:
180         log.exception("Error [get_instances(nova_client)]")
181
182
183 def get_instance_status(nova_client, instance):     # pragma: no cover
184     try:
185         return nova_client.servers.get(instance.id).status
186     except Exception:
187         log.exception("Error [get_instance_status(nova_client)]")
188
189
190 def get_instance_by_name(nova_client, instance_name):   # pragma: no cover
191     try:
192         return nova_client.servers.find(name=instance_name)
193     except Exception:
194         log.exception("Error [get_instance_by_name(nova_client, '%s')]",
195                       instance_name)
196
197
198 def get_aggregates(nova_client):    # pragma: no cover
199     try:
200         return nova_client.aggregates.list()
201     except Exception:
202         log.exception("Error [get_aggregates(nova_client)]")
203
204
205 def get_availability_zones(nova_client):    # pragma: no cover
206     try:
207         return nova_client.availability_zones.list()
208     except Exception:
209         log.exception("Error [get_availability_zones(nova_client)]")
210
211
212 def get_availability_zone_names(nova_client):   # pragma: no cover
213     try:
214         return [az.zoneName for az in get_availability_zones(nova_client)]
215     except Exception:
216         log.exception("Error [get_availability_zone_names(nova_client)]")
217
218
219 def create_aggregate(nova_client, aggregate_name, av_zone):  # pragma: no cover
220     try:
221         nova_client.aggregates.create(aggregate_name, av_zone)
222     except Exception:
223         log.exception("Error [create_aggregate(nova_client, %s, %s)]",
224                       aggregate_name, av_zone)
225         return False
226     else:
227         return True
228
229
230 def get_aggregate_id(nova_client, aggregate_name):      # pragma: no cover
231     try:
232         aggregates = get_aggregates(nova_client)
233         _id = next((ag.id for ag in aggregates if ag.name == aggregate_name))
234     except Exception:
235         log.exception("Error [get_aggregate_id(nova_client, %s)]",
236                       aggregate_name)
237     else:
238         return _id
239
240
241 def add_host_to_aggregate(nova_client, aggregate_name,
242                           compute_host):    # pragma: no cover
243     try:
244         aggregate_id = get_aggregate_id(nova_client, aggregate_name)
245         nova_client.aggregates.add_host(aggregate_id, compute_host)
246     except Exception:
247         log.exception("Error [add_host_to_aggregate(nova_client, %s, %s)]",
248                       aggregate_name, compute_host)
249         return False
250     else:
251         return True
252
253
254 def create_aggregate_with_host(nova_client, aggregate_name, av_zone,
255                                compute_host):    # pragma: no cover
256     try:
257         create_aggregate(nova_client, aggregate_name, av_zone)
258         add_host_to_aggregate(nova_client, aggregate_name, compute_host)
259     except Exception:
260         log.exception("Error [create_aggregate_with_host("
261                       "nova_client, %s, %s, %s)]",
262                       aggregate_name, av_zone, compute_host)
263         return False
264     else:
265         return True
266
267
268 def create_keypair(nova_client, name, key_path=None):    # pragma: no cover
269     try:
270         with open(key_path) as fpubkey:
271             keypair = get_nova_client().keypairs.create(name=name, public_key=fpubkey.read())
272             return keypair
273     except Exception:
274         log.exception("Error [create_keypair(nova_client)]")
275
276
277 def create_instance(json_body):    # pragma: no cover
278     try:
279         return get_nova_client().servers.create(**json_body)
280     except Exception:
281         log.exception("Error create instance failed")
282         return None
283
284
285 def create_instance_and_wait_for_active(json_body):    # pragma: no cover
286     SLEEP = 3
287     VM_BOOT_TIMEOUT = 180
288     nova_client = get_nova_client()
289     instance = create_instance(json_body)
290     count = VM_BOOT_TIMEOUT / SLEEP
291     for n in range(count, -1, -1):
292         status = get_instance_status(nova_client, instance)
293         if status.lower() == "active":
294             return instance
295         elif status.lower() == "error":
296             log.error("The instance went to ERROR status.")
297             return None
298         time.sleep(SLEEP)
299     log.error("Timeout booting the instance.")
300     return None
301
302
303 def attach_server_volume(server_id, volume_id, device=None):    # pragma: no cover
304     try:
305         get_nova_client().volumes.create_server_volume(server_id, volume_id, device)
306     except Exception:
307         log.exception("Error [attach_server_volume(nova_client, '%s', '%s')]",
308                       server_id, volume_id)
309         return False
310     else:
311         return True
312
313
314 def delete_instance(nova_client, instance_id):      # pragma: no cover
315     try:
316         nova_client.servers.force_delete(instance_id)
317     except Exception:
318         log.exception("Error [delete_instance(nova_client, '%s')]",
319                       instance_id)
320         return False
321     else:
322         return True
323
324
325 def remove_host_from_aggregate(nova_client, aggregate_name,
326                                compute_host):  # pragma: no cover
327     try:
328         aggregate_id = get_aggregate_id(nova_client, aggregate_name)
329         nova_client.aggregates.remove_host(aggregate_id, compute_host)
330     except Exception:
331         log.exception("Error remove_host_from_aggregate(nova_client, %s, %s)",
332                       aggregate_name, compute_host)
333         return False
334     else:
335         return True
336
337
338 def remove_hosts_from_aggregate(nova_client,
339                                 aggregate_name):   # pragma: no cover
340     aggregate_id = get_aggregate_id(nova_client, aggregate_name)
341     hosts = nova_client.aggregates.get(aggregate_id).hosts
342     assert(
343         all(remove_host_from_aggregate(nova_client, aggregate_name, host)
344             for host in hosts))
345
346
347 def delete_aggregate(nova_client, aggregate_name):  # pragma: no cover
348     try:
349         remove_hosts_from_aggregate(nova_client, aggregate_name)
350         nova_client.aggregates.delete(aggregate_name)
351     except Exception:
352         log.exception("Error [delete_aggregate(nova_client, %s)]",
353                       aggregate_name)
354         return False
355     else:
356         return True
357
358
359 def get_server_by_name(name):   # pragma: no cover
360     try:
361         return get_nova_client().servers.list(search_opts={'name': name})[0]
362     except IndexError:
363         log.exception('Failed to get nova client')
364         raise
365
366
367 def create_flavor(name, ram, vcpus, disk, **kwargs):   # pragma: no cover
368     try:
369         return get_nova_client().flavors.create(name, ram, vcpus, disk, **kwargs)
370     except Exception:
371         log.exception("Error [create_flavor(nova_client, %s, %s, %s, %s, %s)]",
372                       name, ram, disk, vcpus, kwargs['is_public'])
373         return None
374
375
376 def get_image_by_name(name):    # pragma: no cover
377     images = get_nova_client().images.list()
378     try:
379         return next((a for a in images if a.name == name))
380     except StopIteration:
381         log.exception('No image matched')
382
383
384 def get_flavor_id(nova_client, flavor_name):    # pragma: no cover
385     flavors = nova_client.flavors.list(detailed=True)
386     flavor_id = ''
387     for f in flavors:
388         if f.name == flavor_name:
389             flavor_id = f.id
390             break
391     return flavor_id
392
393
394 def get_flavor_by_name(name):   # pragma: no cover
395     flavors = get_nova_client().flavors.list()
396     try:
397         return next((a for a in flavors if a.name == name))
398     except StopIteration:
399         log.exception('No flavor matched')
400
401
402 def check_status(status, name, iterations, interval):   # pragma: no cover
403     for i in range(iterations):
404         try:
405             server = get_server_by_name(name)
406         except IndexError:
407             log.error('Cannot found %s server', name)
408             raise
409
410         if server.status == status:
411             return True
412
413         time.sleep(interval)
414     return False
415
416
417 def delete_flavor(flavor_id):    # pragma: no cover
418     try:
419         get_nova_client().flavors.delete(flavor_id)
420     except Exception:
421         log.exception("Error [delete_flavor(nova_client, %s)]", flavor_id)
422         return False
423     else:
424         return True
425
426
427 def delete_keypair(nova_client, key):     # pragma: no cover
428     try:
429         nova_client.keypairs.delete(key=key)
430         return True
431     except Exception:
432         log.exception("Error [delete_keypair(nova_client)]")
433         return False
434
435
436 # *********************************************
437 #   NEUTRON
438 # *********************************************
439 def get_network_id(neutron_client, network_name):       # pragma: no cover
440     networks = neutron_client.list_networks()['networks']
441     return next((n['id'] for n in networks if n['name'] == network_name), None)
442
443
444 def get_port_id_by_ip(neutron_client, ip_address):      # pragma: no cover
445     ports = neutron_client.list_ports()['ports']
446     return next((i['id'] for i in ports for j in i.get(
447         'fixed_ips') if j['ip_address'] == ip_address), None)
448
449
450 def create_neutron_net(neutron_client, json_body):      # pragma: no cover
451     try:
452         network = neutron_client.create_network(body=json_body)
453         return network['network']['id']
454     except Exception:
455         log.error("Error [create_neutron_net(neutron_client)]")
456         raise Exception("operation error")
457         return None
458
459
460 def create_neutron_subnet(neutron_client, json_body):      # pragma: no cover
461     try:
462         subnet = neutron_client.create_subnet(body=json_body)
463         return subnet['subnets'][0]['id']
464     except Exception:
465         log.error("Error [create_neutron_subnet")
466         raise Exception("operation error")
467         return None
468
469
470 def create_neutron_router(neutron_client, json_body):      # pragma: no cover
471     try:
472         router = neutron_client.create_router(json_body)
473         return router['router']['id']
474     except Exception:
475         log.error("Error [create_neutron_router(neutron_client)]")
476         raise Exception("operation error")
477         return None
478
479
480 def create_floating_ip(neutron_client, extnet_id):      # pragma: no cover
481     props = {'floating_network_id': extnet_id}
482     try:
483         ip_json = neutron_client.create_floatingip({'floatingip': props})
484         fip_addr = ip_json['floatingip']['floating_ip_address']
485         fip_id = ip_json['floatingip']['id']
486     except Exception:
487         log.error("Error [create_floating_ip(neutron_client)]")
488         return None
489     return {'fip_addr': fip_addr, 'fip_id': fip_id}
490
491
492 def delete_floating_ip(nova_client, floatingip_id):      # pragma: no cover
493     try:
494         nova_client.floating_ips.delete(floatingip_id)
495         return True
496     except Exception:
497         log.error("Error [delete_floating_ip(nova_client, '%s')]" % floatingip_id)
498         return False
499
500
501 def get_security_groups(neutron_client):      # pragma: no cover
502     try:
503         security_groups = neutron_client.list_security_groups()[
504             'security_groups']
505         return security_groups
506     except Exception:
507         log.error("Error [get_security_groups(neutron_client)]")
508         return None
509
510
511 def get_security_group_id(neutron_client, sg_name):      # pragma: no cover
512     security_groups = get_security_groups(neutron_client)
513     id = ''
514     for sg in security_groups:
515         if sg['name'] == sg_name:
516             id = sg['id']
517             break
518     return id
519
520
521 def create_security_group(neutron_client, sg_name, sg_description):      # pragma: no cover
522     json_body = {'security_group': {'name': sg_name,
523                                     'description': sg_description}}
524     try:
525         secgroup = neutron_client.create_security_group(json_body)
526         return secgroup['security_group']
527     except Exception:
528         log.error("Error [create_security_group(neutron_client, '%s', "
529                   "'%s')]" % (sg_name, sg_description))
530         return None
531
532
533 def create_secgroup_rule(neutron_client, sg_id, direction, protocol,
534                          port_range_min=None, port_range_max=None,
535                          **json_body):      # pragma: no cover
536     # We create a security group in 2 steps
537     # 1 - we check the format and set the json body accordingly
538     # 2 - we call neturon client to create the security group
539
540     # Format check
541     json_body.update({'security_group_rule': {'direction': direction,
542                      'security_group_id': sg_id, 'protocol': protocol}})
543     # parameters may be
544     # - both None => we do nothing
545     # - both Not None => we add them to the json description
546     # but one cannot be None is the other is not None
547     if (port_range_min is not None and port_range_max is not None):
548         # add port_range in json description
549         json_body['security_group_rule']['port_range_min'] = port_range_min
550         json_body['security_group_rule']['port_range_max'] = port_range_max
551         log.debug("Security_group format set (port range included)")
552     else:
553         # either both port range are set to None => do nothing
554         # or one is set but not the other => log it and return False
555         if port_range_min is None and port_range_max is None:
556             log.debug("Security_group format set (no port range mentioned)")
557         else:
558             log.error("Bad security group format."
559                       "One of the port range is not properly set:"
560                       "range min: {},"
561                       "range max: {}".format(port_range_min,
562                                              port_range_max))
563             return False
564
565     # Create security group using neutron client
566     try:
567         neutron_client.create_security_group_rule(json_body)
568         return True
569     except Exception:
570         log.exception("Impossible to create_security_group_rule,"
571                       "security group rule probably already exists")
572         return False
573
574
575 def create_security_group_full(neutron_client,
576                                sg_name, sg_description):      # pragma: no cover
577     sg_id = get_security_group_id(neutron_client, sg_name)
578     if sg_id != '':
579         log.info("Using existing security group '%s'..." % sg_name)
580     else:
581         log.info("Creating security group  '%s'..." % sg_name)
582         SECGROUP = create_security_group(neutron_client,
583                                          sg_name,
584                                          sg_description)
585         if not SECGROUP:
586             log.error("Failed to create the security group...")
587             return None
588
589         sg_id = SECGROUP['id']
590
591         log.debug("Security group '%s' with ID=%s created successfully."
592                   % (SECGROUP['name'], sg_id))
593
594         log.debug("Adding ICMP rules in security group '%s'..."
595                   % sg_name)
596         if not create_secgroup_rule(neutron_client, sg_id,
597                                     'ingress', 'icmp'):
598             log.error("Failed to create the security group rule...")
599             return None
600
601         log.debug("Adding SSH rules in security group '%s'..."
602                   % sg_name)
603         if not create_secgroup_rule(
604                 neutron_client, sg_id, 'ingress', 'tcp', '22', '22'):
605             log.error("Failed to create the security group rule...")
606             return None
607
608         if not create_secgroup_rule(
609                 neutron_client, sg_id, 'egress', 'tcp', '22', '22'):
610             log.error("Failed to create the security group rule...")
611             return None
612     return sg_id
613
614
615 # *********************************************
616 #   GLANCE
617 # *********************************************
618 def get_image_id(glance_client, image_name):    # pragma: no cover
619     images = glance_client.images.list()
620     return next((i.id for i in images if i.name == image_name), None)
621
622
623 def create_image(glance_client, image_name, file_path, disk_format,
624                  container_format, min_disk, min_ram, protected, tag,
625                  public, **kwargs):    # pragma: no cover
626     if not os.path.isfile(file_path):
627         log.error("Error: file %s does not exist." % file_path)
628         return None
629     try:
630         image_id = get_image_id(glance_client, image_name)
631         if image_id is not None:
632             log.info("Image %s already exists." % image_name)
633         else:
634             log.info("Creating image '%s' from '%s'...", image_name, file_path)
635
636             image = glance_client.images.create(name=image_name,
637                                                 visibility=public,
638                                                 disk_format=disk_format,
639                                                 container_format=container_format,
640                                                 min_disk=min_disk,
641                                                 min_ram=min_ram,
642                                                 tags=tag,
643                                                 protected=protected,
644                                                 **kwargs)
645             image_id = image.id
646             with open(file_path) as image_data:
647                 glance_client.images.upload(image_id, image_data)
648         return image_id
649     except Exception:
650         log.error("Error [create_glance_image(glance_client, '%s', '%s', '%s')]",
651                   image_name, file_path, public)
652         return None
653
654
655 def delete_image(glance_client, image_id):    # pragma: no cover
656     try:
657         glance_client.images.delete(image_id)
658
659     except Exception:
660         log.exception("Error [delete_flavor(glance_client, %s)]", image_id)
661         return False
662     else:
663         return True
664
665
666 # *********************************************
667 #   CINDER
668 # *********************************************
669 def get_volume_id(volume_name):    # pragma: no cover
670     volumes = get_cinder_client().volumes.list()
671     return next((v.id for v in volumes if v.name == volume_name), None)
672
673
674 def create_volume(cinder_client, volume_name, volume_size,
675                   volume_image=False):    # pragma: no cover
676     try:
677         if volume_image:
678             volume = cinder_client.volumes.create(name=volume_name,
679                                                   size=volume_size,
680                                                   imageRef=volume_image)
681         else:
682             volume = cinder_client.volumes.create(name=volume_name,
683                                                   size=volume_size)
684         return volume
685     except Exception:
686         log.exception("Error [create_volume(cinder_client, %s)]",
687                       (volume_name, volume_size))
688         return None
689
690
691 def delete_volume(cinder_client, volume_id, forced=False):      # pragma: no cover
692     try:
693         if forced:
694             try:
695                 cinder_client.volumes.detach(volume_id)
696             except:
697                 log.error(sys.exc_info()[0])
698             cinder_client.volumes.force_delete(volume_id)
699         else:
700             while True:
701                 volume = get_cinder_client().volumes.get(volume_id)
702                 if volume.status.lower() == 'available':
703                     break
704             cinder_client.volumes.delete(volume_id)
705         return True
706     except Exception:
707         log.exception("Error [delete_volume(cinder_client, '%s')]" % volume_id)
708         return False
709
710
711 def detach_volume(server_id, volume_id):      # pragma: no cover
712     try:
713         get_nova_client().volumes.delete_server_volume(server_id, volume_id)
714         return True
715     except Exception:
716         log.exception("Error [detach_server_volume(nova_client, '%s', '%s')]",
717                       server_id, volume_id)
718         return False