9da37e64b07c66b6034d6f11327b4dc590eea2fa
[yardstick.git] / yardstick / common / openstack_utils.py
1 ##############################################################################
2 # Copyright (c) 2016 Huawei Technologies Co.,Ltd and others.
3 #
4 # All rights reserved. This program and the accompanying materials
5 # are made available under the terms of the Apache License, Version 2.0
6 # which accompanies this distribution, and is available at
7 # http://www.apache.org/licenses/LICENSE-2.0
8 ##############################################################################
9
10 import os
11 import time
12 import sys
13 import logging
14
15 from keystoneauth1 import loading
16 from keystoneauth1 import session
17 import shade
18 from shade import exc
19
20 from cinderclient import client as cinderclient
21 from novaclient import client as novaclient
22 from glanceclient import client as glanceclient
23 from neutronclient.neutron import client as neutronclient
24
25
26 log = logging.getLogger(__name__)
27
28 DEFAULT_HEAT_API_VERSION = '1'
29 DEFAULT_API_VERSION = '2'
30
31
32 # *********************************************
33 #   CREDENTIALS
34 # *********************************************
35 def get_credentials():
36     """Returns a creds dictionary filled with parsed from env
37
38     Keystone API version used is 3; v2 was deprecated in 2014 (Icehouse). Along
39     with this deprecation, environment variable 'OS_TENANT_NAME' is replaced by
40     'OS_PROJECT_NAME'.
41     """
42     creds = {'username': os.environ.get('OS_USERNAME'),
43              'password': os.environ.get('OS_PASSWORD'),
44              'auth_url': os.environ.get('OS_AUTH_URL'),
45              'project_name': os.environ.get('OS_PROJECT_NAME')
46              }
47
48     if os.getenv('OS_USER_DOMAIN_NAME'):
49         creds['user_domain_name'] = os.getenv('OS_USER_DOMAIN_NAME')
50     if os.getenv('OS_PROJECT_DOMAIN_NAME'):
51         creds['project_domain_name'] = os.getenv('OS_PROJECT_DOMAIN_NAME')
52
53     return creds
54
55
56 def get_session_auth():
57     loader = loading.get_plugin_loader('password')
58     creds = get_credentials()
59     auth = loader.load_from_options(**creds)
60     return auth
61
62
63 def get_session():
64     auth = get_session_auth()
65     try:
66         cacert = os.environ['OS_CACERT']
67     except KeyError:
68         return session.Session(auth=auth)
69     else:
70         insecure = os.getenv('OS_INSECURE', '').lower() == 'true'
71         cacert = False if insecure else cacert
72         return session.Session(auth=auth, verify=cacert)
73
74
75 def get_endpoint(service_type, endpoint_type='publicURL'):
76     auth = get_session_auth()
77     # for multi-region, we need to specify region
78     # when finding the endpoint
79     return get_session().get_endpoint(auth=auth,
80                                       service_type=service_type,
81                                       endpoint_type=endpoint_type,
82                                       region_name=os.environ.get(
83                                           "OS_REGION_NAME"))
84
85
86 # *********************************************
87 #   CLIENTS
88 # *********************************************
89 def get_heat_api_version():     # pragma: no cover
90     try:
91         api_version = os.environ['HEAT_API_VERSION']
92     except KeyError:
93         return DEFAULT_HEAT_API_VERSION
94     else:
95         log.info("HEAT_API_VERSION is set in env as '%s'", api_version)
96         return api_version
97
98
99 def get_cinder_client_version():      # pragma: no cover
100     try:
101         api_version = os.environ['OS_VOLUME_API_VERSION']
102     except KeyError:
103         return DEFAULT_API_VERSION
104     else:
105         log.info("OS_VOLUME_API_VERSION is set in env as '%s'", api_version)
106         return api_version
107
108
109 def get_cinder_client():      # pragma: no cover
110     sess = get_session()
111     return cinderclient.Client(get_cinder_client_version(), session=sess)
112
113
114 def get_nova_client_version():      # pragma: no cover
115     try:
116         api_version = os.environ['OS_COMPUTE_API_VERSION']
117     except KeyError:
118         return DEFAULT_API_VERSION
119     else:
120         log.info("OS_COMPUTE_API_VERSION is set in env as '%s'", api_version)
121         return api_version
122
123
124 def get_nova_client():      # pragma: no cover
125     sess = get_session()
126     return novaclient.Client(get_nova_client_version(), session=sess)
127
128
129 def get_neutron_client_version():   # pragma: no cover
130     try:
131         api_version = os.environ['OS_NETWORK_API_VERSION']
132     except KeyError:
133         return DEFAULT_API_VERSION
134     else:
135         log.info("OS_NETWORK_API_VERSION is set in env as '%s'", api_version)
136         return api_version
137
138
139 def get_neutron_client():   # pragma: no cover
140     sess = get_session()
141     return neutronclient.Client(get_neutron_client_version(), session=sess)
142
143
144 def get_glance_client_version():    # pragma: no cover
145     try:
146         api_version = os.environ['OS_IMAGE_API_VERSION']
147     except KeyError:
148         return DEFAULT_API_VERSION
149     else:
150         log.info("OS_IMAGE_API_VERSION is set in env as '%s'", api_version)
151         return api_version
152
153
154 def get_glance_client():    # pragma: no cover
155     sess = get_session()
156     return glanceclient.Client(get_glance_client_version(), session=sess)
157
158
159 def get_shade_client():
160     return shade.openstack_cloud()
161
162
163 # *********************************************
164 #   NOVA
165 # *********************************************
166 def get_instances(nova_client):
167     try:
168         return nova_client.servers.list(search_opts={'all_tenants': 1})
169     except Exception:  # pylint: disable=broad-except
170         log.exception("Error [get_instances(nova_client)]")
171
172
173 def get_instance_status(nova_client, instance):     # pragma: no cover
174     try:
175         return nova_client.servers.get(instance.id).status
176     except Exception:  # pylint: disable=broad-except
177         log.exception("Error [get_instance_status(nova_client)]")
178
179
180 def get_instance_by_name(nova_client, instance_name):   # pragma: no cover
181     try:
182         return nova_client.servers.find(name=instance_name)
183     except Exception:  # pylint: disable=broad-except
184         log.exception("Error [get_instance_by_name(nova_client, '%s')]",
185                       instance_name)
186
187
188 def get_aggregates(nova_client):    # pragma: no cover
189     try:
190         return nova_client.aggregates.list()
191     except Exception:  # pylint: disable=broad-except
192         log.exception("Error [get_aggregates(nova_client)]")
193
194
195 def get_availability_zones(nova_client):    # pragma: no cover
196     try:
197         return nova_client.availability_zones.list()
198     except Exception:  # pylint: disable=broad-except
199         log.exception("Error [get_availability_zones(nova_client)]")
200
201
202 def get_availability_zone_names(nova_client):   # pragma: no cover
203     try:
204         return [az.zoneName for az in get_availability_zones(nova_client)]
205     except Exception:  # pylint: disable=broad-except
206         log.exception("Error [get_availability_zone_names(nova_client)]")
207
208
209 def create_aggregate(nova_client, aggregate_name, av_zone):  # pragma: no cover
210     try:
211         nova_client.aggregates.create(aggregate_name, av_zone)
212     except Exception:  # pylint: disable=broad-except
213         log.exception("Error [create_aggregate(nova_client, %s, %s)]",
214                       aggregate_name, av_zone)
215         return False
216     else:
217         return True
218
219
220 def get_aggregate_id(nova_client, aggregate_name):      # pragma: no cover
221     try:
222         aggregates = get_aggregates(nova_client)
223         _id = next((ag.id for ag in aggregates if ag.name == aggregate_name))
224     except Exception:  # pylint: disable=broad-except
225         log.exception("Error [get_aggregate_id(nova_client, %s)]",
226                       aggregate_name)
227     else:
228         return _id
229
230
231 def add_host_to_aggregate(nova_client, aggregate_name,
232                           compute_host):    # pragma: no cover
233     try:
234         aggregate_id = get_aggregate_id(nova_client, aggregate_name)
235         nova_client.aggregates.add_host(aggregate_id, compute_host)
236     except Exception:  # pylint: disable=broad-except
237         log.exception("Error [add_host_to_aggregate(nova_client, %s, %s)]",
238                       aggregate_name, compute_host)
239         return False
240     else:
241         return True
242
243
244 def create_aggregate_with_host(nova_client, aggregate_name, av_zone,
245                                compute_host):    # pragma: no cover
246     try:
247         create_aggregate(nova_client, aggregate_name, av_zone)
248         add_host_to_aggregate(nova_client, aggregate_name, compute_host)
249     except Exception:  # pylint: disable=broad-except
250         log.exception("Error [create_aggregate_with_host("
251                       "nova_client, %s, %s, %s)]",
252                       aggregate_name, av_zone, compute_host)
253         return False
254     else:
255         return True
256
257
258 def create_keypair(name, key_path=None):    # pragma: no cover
259     try:
260         with open(key_path) as fpubkey:
261             keypair = get_nova_client().keypairs.create(
262                 name=name, public_key=fpubkey.read())
263             return keypair
264     except Exception:  # pylint: disable=broad-except
265         log.exception("Error [create_keypair(nova_client)]")
266
267
268 def create_instance(json_body):    # pragma: no cover
269     try:
270         return get_nova_client().servers.create(**json_body)
271     except Exception:  # pylint: disable=broad-except
272         log.exception("Error create instance failed")
273         return None
274
275
276 def create_instance_and_wait_for_active(json_body):    # pragma: no cover
277     SLEEP = 3
278     VM_BOOT_TIMEOUT = 180
279     nova_client = get_nova_client()
280     instance = create_instance(json_body)
281     for _ in range(int(VM_BOOT_TIMEOUT / SLEEP)):
282         status = get_instance_status(nova_client, instance)
283         if status.lower() == "active":
284             return instance
285         elif status.lower() == "error":
286             log.error("The instance went to ERROR status.")
287             return None
288         time.sleep(SLEEP)
289     log.error("Timeout booting the instance.")
290     return None
291
292
293 def attach_server_volume(server_id, volume_id,
294                          device=None):    # pragma: no cover
295     try:
296         get_nova_client().volumes.create_server_volume(server_id,
297                                                        volume_id, device)
298     except Exception:  # pylint: disable=broad-except
299         log.exception("Error [attach_server_volume(nova_client, '%s', '%s')]",
300                       server_id, volume_id)
301         return False
302     else:
303         return True
304
305
306 def delete_instance(nova_client, instance_id):      # pragma: no cover
307     try:
308         nova_client.servers.force_delete(instance_id)
309     except Exception:  # pylint: disable=broad-except
310         log.exception("Error [delete_instance(nova_client, '%s')]",
311                       instance_id)
312         return False
313     else:
314         return True
315
316
317 def remove_host_from_aggregate(nova_client, aggregate_name,
318                                compute_host):  # pragma: no cover
319     try:
320         aggregate_id = get_aggregate_id(nova_client, aggregate_name)
321         nova_client.aggregates.remove_host(aggregate_id, compute_host)
322     except Exception:  # pylint: disable=broad-except
323         log.exception("Error remove_host_from_aggregate(nova_client, %s, %s)",
324                       aggregate_name, compute_host)
325         return False
326     else:
327         return True
328
329
330 def remove_hosts_from_aggregate(nova_client,
331                                 aggregate_name):   # pragma: no cover
332     aggregate_id = get_aggregate_id(nova_client, aggregate_name)
333     hosts = nova_client.aggregates.get(aggregate_id).hosts
334     assert(
335         all(remove_host_from_aggregate(nova_client, aggregate_name, host)
336             for host in hosts))
337
338
339 def delete_aggregate(nova_client, aggregate_name):  # pragma: no cover
340     try:
341         remove_hosts_from_aggregate(nova_client, aggregate_name)
342         nova_client.aggregates.delete(aggregate_name)
343     except Exception:  # pylint: disable=broad-except
344         log.exception("Error [delete_aggregate(nova_client, %s)]",
345                       aggregate_name)
346         return False
347     else:
348         return True
349
350
351 def get_server_by_name(name):   # pragma: no cover
352     try:
353         return get_nova_client().servers.list(search_opts={'name': name})[0]
354     except IndexError:
355         log.exception('Failed to get nova client')
356         raise
357
358
359 def create_flavor(name, ram, vcpus, disk, **kwargs):   # pragma: no cover
360     try:
361         return get_nova_client().flavors.create(name, ram, vcpus,
362                                                 disk, **kwargs)
363     except Exception:  # pylint: disable=broad-except
364         log.exception("Error [create_flavor(nova_client, %s, %s, %s, %s, %s)]",
365                       name, ram, disk, vcpus, kwargs['is_public'])
366         return None
367
368
369 def get_image_by_name(name):    # pragma: no cover
370     images = get_nova_client().images.list()
371     try:
372         return next((a for a in images if a.name == name))
373     except StopIteration:
374         log.exception('No image matched')
375
376
377 def get_flavor_id(nova_client, flavor_name):    # pragma: no cover
378     flavors = nova_client.flavors.list(detailed=True)
379     flavor_id = ''
380     for f in flavors:
381         if f.name == flavor_name:
382             flavor_id = f.id
383             break
384     return flavor_id
385
386
387 def get_flavor_by_name(name):   # pragma: no cover
388     flavors = get_nova_client().flavors.list()
389     try:
390         return next((a for a in flavors if a.name == name))
391     except StopIteration:
392         log.exception('No flavor matched')
393
394
395 def check_status(status, name, iterations, interval):   # pragma: no cover
396     for _ in range(iterations):
397         try:
398             server = get_server_by_name(name)
399         except IndexError:
400             log.error('Cannot found %s server', name)
401             raise
402
403         if server.status == status:
404             return True
405
406         time.sleep(interval)
407     return False
408
409
410 def delete_flavor(flavor_id):    # pragma: no cover
411     try:
412         get_nova_client().flavors.delete(flavor_id)
413     except Exception:  # pylint: disable=broad-except
414         log.exception("Error [delete_flavor(nova_client, %s)]", flavor_id)
415         return False
416     else:
417         return True
418
419
420 def delete_keypair(nova_client, key):     # pragma: no cover
421     try:
422         nova_client.keypairs.delete(key=key)
423         return True
424     except Exception:  # pylint: disable=broad-except
425         log.exception("Error [delete_keypair(nova_client)]")
426         return False
427
428
429 # *********************************************
430 #   NEUTRON
431 # *********************************************
432 def get_network_id(shade_client, network_name):
433     networks = shade_client.list_networks({'name': network_name})
434     if networks:
435         return networks[0]['id']
436
437
438 def create_neutron_net(shade_client, network_name, shared=False,
439                        admin_state_up=True, external=False, provider=None,
440                        project_id=None):
441     """Create a neutron network.
442
443     :param network_name:(string) name of the network being created.
444     :param shared:(bool) whether the network is shared.
445     :param admin_state_up:(bool) set the network administrative state.
446     :param external:(bool) whether this network is externally accessible.
447     :param provider:(dict) a dict of network provider options.
448     :param project_id:(string) specify the project ID this network
449                       will be created on (admin-only).
450     :returns:(string) the network id.
451     """
452     try:
453         networks = shade_client.create_network(
454             name=network_name, shared=shared, admin_state_up=admin_state_up,
455             external=external, provider=provider, project_id=project_id)
456         return networks['id']
457     except exc.OpenStackCloudException as o_exc:
458         log.error("Error [create_neutron_net(shade_client)]."
459                   "Exception message, '%s'", o_exc.orig_message)
460         return None
461
462
463 def delete_neutron_net(shade_client, network_id):
464     try:
465         return shade_client.delete_network(network_id)
466     except exc.OpenStackCloudException:
467         log.error("Error [delete_neutron_net(shade_client, '%s')]", network_id)
468         return False
469
470
471 def create_neutron_subnet(neutron_client, json_body):      # pragma: no cover
472     try:
473         subnet = neutron_client.create_subnet(body=json_body)
474         return subnet['subnets'][0]['id']
475     except Exception:  # pylint: disable=broad-except
476         log.error("Error [create_neutron_subnet")
477         raise Exception("operation error")
478
479
480 def create_neutron_router(neutron_client, json_body):      # pragma: no cover
481     try:
482         router = neutron_client.create_router(json_body)
483         return router['router']['id']
484     except Exception:  # pylint: disable=broad-except
485         log.error("Error [create_neutron_router(neutron_client)]")
486         raise Exception("operation error")
487
488
489 def delete_neutron_router(neutron_client, router_id):      # pragma: no cover
490     try:
491         neutron_client.delete_router(router=router_id)
492         return True
493     except Exception:  # pylint: disable=broad-except
494         log.error("Error [delete_neutron_router(neutron_client, '%s')]",
495                   router_id)
496         return False
497
498
499 def remove_gateway_router(neutron_client, router_id):      # pragma: no cover
500     try:
501         neutron_client.remove_gateway_router(router_id)
502         return True
503     except Exception:  # pylint: disable=broad-except
504         log.error("Error [remove_gateway_router(neutron_client, '%s')]",
505                   router_id)
506         return False
507
508
509 def remove_interface_router(neutron_client, router_id, subnet_id,
510                             **json_body):      # pragma: no cover
511     json_body.update({"subnet_id": subnet_id})
512     try:
513         neutron_client.remove_interface_router(router=router_id,
514                                                body=json_body)
515         return True
516     except Exception:  # pylint: disable=broad-except
517         log.error("Error [remove_interface_router(neutron_client, '%s', "
518                   "'%s')]", router_id, subnet_id)
519         return False
520
521
522 def create_floating_ip(neutron_client, extnet_id):      # pragma: no cover
523     props = {'floating_network_id': extnet_id}
524     try:
525         ip_json = neutron_client.create_floatingip({'floatingip': props})
526         fip_addr = ip_json['floatingip']['floating_ip_address']
527         fip_id = ip_json['floatingip']['id']
528     except Exception:  # pylint: disable=broad-except
529         log.error("Error [create_floating_ip(neutron_client)]")
530         return None
531     return {'fip_addr': fip_addr, 'fip_id': fip_id}
532
533
534 def delete_floating_ip(nova_client, floatingip_id):      # pragma: no cover
535     try:
536         nova_client.floating_ips.delete(floatingip_id)
537         return True
538     except Exception:  # pylint: disable=broad-except
539         log.error("Error [delete_floating_ip(nova_client, '%s')]",
540                   floatingip_id)
541         return False
542
543
544 def get_security_groups(neutron_client):      # pragma: no cover
545     try:
546         security_groups = neutron_client.list_security_groups()[
547             'security_groups']
548         return security_groups
549     except Exception:  # pylint: disable=broad-except
550         log.error("Error [get_security_groups(neutron_client)]")
551         return None
552
553
554 def get_security_group_id(neutron_client, sg_name):      # pragma: no cover
555     security_groups = get_security_groups(neutron_client)
556     id = ''
557     for sg in security_groups:
558         if sg['name'] == sg_name:
559             id = sg['id']
560             break
561     return id
562
563
564 def create_security_group(neutron_client, sg_name,
565                           sg_description):      # pragma: no cover
566     json_body = {'security_group': {'name': sg_name,
567                                     'description': sg_description}}
568     try:
569         secgroup = neutron_client.create_security_group(json_body)
570         return secgroup['security_group']
571     except Exception:  # pylint: disable=broad-except
572         log.error("Error [create_security_group(neutron_client, '%s', "
573                   "'%s')]", sg_name, sg_description)
574         return None
575
576
577 def create_secgroup_rule(neutron_client, sg_id, direction, protocol,
578                          port_range_min=None, port_range_max=None,
579                          **json_body):      # pragma: no cover
580     # We create a security group in 2 steps
581     # 1 - we check the format and set the json body accordingly
582     # 2 - we call neturon client to create the security group
583
584     # Format check
585     json_body.update({'security_group_rule': {'direction': direction,
586                      'security_group_id': sg_id, 'protocol': protocol}})
587     # parameters may be
588     # - both None => we do nothing
589     # - both Not None => we add them to the json description
590     # but one cannot be None is the other is not None
591     if (port_range_min is not None and port_range_max is not None):
592         # add port_range in json description
593         json_body['security_group_rule']['port_range_min'] = port_range_min
594         json_body['security_group_rule']['port_range_max'] = port_range_max
595         log.debug("Security_group format set (port range included)")
596     else:
597         # either both port range are set to None => do nothing
598         # or one is set but not the other => log it and return False
599         if port_range_min is None and port_range_max is None:
600             log.debug("Security_group format set (no port range mentioned)")
601         else:
602             log.error("Bad security group format."
603                       "One of the port range is not properly set:"
604                       "range min: %s, range max: %s", port_range_min,
605                       port_range_max)
606             return False
607
608     # Create security group using neutron client
609     try:
610         neutron_client.create_security_group_rule(json_body)
611         return True
612     except Exception:  # pylint: disable=broad-except
613         log.exception("Impossible to create_security_group_rule,"
614                       "security group rule probably already exists")
615         return False
616
617
618 def create_security_group_full(neutron_client, sg_name,
619                                sg_description):      # pragma: no cover
620     sg_id = get_security_group_id(neutron_client, sg_name)
621     if sg_id != '':
622         log.info("Using existing security group '%s'...", sg_name)
623     else:
624         log.info("Creating security group  '%s'...", sg_name)
625         SECGROUP = create_security_group(neutron_client,
626                                          sg_name,
627                                          sg_description)
628         if not SECGROUP:
629             log.error("Failed to create the security group...")
630             return None
631
632         sg_id = SECGROUP['id']
633
634         log.debug("Security group '%s' with ID=%s created successfully.",
635                   SECGROUP['name'], sg_id)
636
637         log.debug("Adding ICMP rules in security group '%s'...", sg_name)
638         if not create_secgroup_rule(neutron_client, sg_id,
639                                     'ingress', 'icmp'):
640             log.error("Failed to create the security group rule...")
641             return None
642
643         log.debug("Adding SSH rules in security group '%s'...", sg_name)
644         if not create_secgroup_rule(
645                 neutron_client, sg_id, 'ingress', 'tcp', '22', '22'):
646             log.error("Failed to create the security group rule...")
647             return None
648
649         if not create_secgroup_rule(
650                 neutron_client, sg_id, 'egress', 'tcp', '22', '22'):
651             log.error("Failed to create the security group rule...")
652             return None
653     return sg_id
654
655
656 # *********************************************
657 #   GLANCE
658 # *********************************************
659 def get_image_id(glance_client, image_name):    # pragma: no cover
660     images = glance_client.images.list()
661     return next((i.id for i in images if i.name == image_name), None)
662
663
664 def create_image(glance_client, image_name, file_path, disk_format,
665                  container_format, min_disk, min_ram, protected, tag,
666                  public, **kwargs):    # pragma: no cover
667     if not os.path.isfile(file_path):
668         log.error("Error: file %s does not exist.", file_path)
669         return None
670     try:
671         image_id = get_image_id(glance_client, image_name)
672         if image_id is not None:
673             log.info("Image %s already exists.", image_name)
674         else:
675             log.info("Creating image '%s' from '%s'...", image_name, file_path)
676
677             image = glance_client.images.create(
678                 name=image_name, visibility=public, disk_format=disk_format,
679                 container_format=container_format, min_disk=min_disk,
680                 min_ram=min_ram, tags=tag, protected=protected, **kwargs)
681             image_id = image.id
682             with open(file_path) as image_data:
683                 glance_client.images.upload(image_id, image_data)
684         return image_id
685     except Exception:  # pylint: disable=broad-except
686         log.error(
687             "Error [create_glance_image(glance_client, '%s', '%s', '%s')]",
688             image_name, file_path, public)
689         return None
690
691
692 def delete_image(glance_client, image_id):    # pragma: no cover
693     try:
694         glance_client.images.delete(image_id)
695
696     except Exception:  # pylint: disable=broad-except
697         log.exception("Error [delete_flavor(glance_client, %s)]", image_id)
698         return False
699     else:
700         return True
701
702
703 # *********************************************
704 #   CINDER
705 # *********************************************
706 def get_volume_id(volume_name):    # pragma: no cover
707     volumes = get_cinder_client().volumes.list()
708     return next((v.id for v in volumes if v.name == volume_name), None)
709
710
711 def create_volume(cinder_client, volume_name, volume_size,
712                   volume_image=False):    # pragma: no cover
713     try:
714         if volume_image:
715             volume = cinder_client.volumes.create(name=volume_name,
716                                                   size=volume_size,
717                                                   imageRef=volume_image)
718         else:
719             volume = cinder_client.volumes.create(name=volume_name,
720                                                   size=volume_size)
721         return volume
722     except Exception:  # pylint: disable=broad-except
723         log.exception("Error [create_volume(cinder_client, %s)]",
724                       (volume_name, volume_size))
725         return None
726
727
728 def delete_volume(cinder_client, volume_id,
729                   forced=False):      # pragma: no cover
730     try:
731         if forced:
732             try:
733                 cinder_client.volumes.detach(volume_id)
734             except Exception:  # pylint: disable=broad-except
735                 log.error(sys.exc_info()[0])
736             cinder_client.volumes.force_delete(volume_id)
737         else:
738             while True:
739                 volume = get_cinder_client().volumes.get(volume_id)
740                 if volume.status.lower() == 'available':
741                     break
742             cinder_client.volumes.delete(volume_id)
743         return True
744     except Exception:  # pylint: disable=broad-except
745         log.exception("Error [delete_volume(cinder_client, '%s')]", volume_id)
746         return False
747
748
749 def detach_volume(server_id, volume_id):      # pragma: no cover
750     try:
751         get_nova_client().volumes.delete_server_volume(server_id, volume_id)
752         return True
753     except Exception:  # pylint: disable=broad-except
754         log.exception("Error [detach_server_volume(nova_client, '%s', '%s')]",
755                       server_id, volume_id)
756         return False