cleanup/nfvbench_cleanup.py

   1 #!/usr/bin/env python
   2 # Copyright 2017 Cisco Systems, Inc.  All rights reserved.
   3 #
   4 #    Licensed under the Apache License, Version 2.0 (the "License"); you may
   5 #    not use this file except in compliance with the License. You may obtain
   6 #    a copy of the License at
   7 #
   8 #         http://www.apache.org/licenses/LICENSE-2.0
   9 #
  10 #    Unless required by applicable law or agreed to in writing, software
  11 #    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  12 #    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  13 #    License for the specific language governing permissions and limitations
  14 #    under the License.
  15 #
  16
  17 ###############################################################################
  18 #                                                                             #
  19 # This is a helper script which will delete all resources created by          #
  20 # NFVbench.                                                                   #
  21 #                                                                             #
  22 # Normally, NFVbench will clean up automatically when it is done. However,    #
  23 # sometimes errors or timeouts happen during the resource creation stage,     #
  24 # which will cause NFVbench out of sync with the real environment. If that    #
  25 # happens, a force cleanup may be needed.                                     #
  26 #                                                                             #
  27 # It is safe to use the script with the resource list generated by            #
  28 # NFVbench, usage:                                                            #
  29 #     $ python nfvbench_cleanup.py -r /path/to/openrc                         #
  30 #                                                                             #
  31 # Note: If running under single-tenant or tenant/user reusing mode, you have  #
  32 #       to cleanup the server resources first, then client resources.         #
  33 #                                                                             #
  34 # When there is no resource list provided, the script will simply grep the    #
  35 # resource name with "nfvbench" and delete them. If running on a production   #
  36 # network, please double and triple check all resources names are *NOT*       #
  37 # starting with "nfvbench", otherwise they will be deleted by the script.     #
  38 #                                                                             #
  39 ###############################################################################
  40
  41 # ======================================================
  42 #                        WARNING
  43 # ======================================================
  44 # IMPORTANT FOR RUNNING NFVbench ON PRODUCTION CLOUDS
  45 #
  46 # DOUBLE CHECK THE NAMES OF ALL RESOURCES THAT DO NOT
  47 # BELONG TO NFVbench ARE *NOT* STARTING WITH "nfvbench".
  48 # ======================================================
  49
  50 from abc import ABCMeta
  51 from abc import abstractmethod
  52 import argparse
  53 import re
  54 import sys
  55 import time
  56 import traceback
  57
  58 # openstack python clients
  59 import cinderclient
  60 from keystoneclient import client as keystoneclient
  61 import neutronclient
  62 from novaclient.exceptions import NotFound
  63 from tabulate import tabulate
  64
  65 from nfvbench import credentials
  66
  67 resource_name_re = None
  68
  69 def prompt_to_run():
  70     print "Warning: You didn't specify a resource list file as the input. "\
  71           "The script will delete all resources shown above."
  72     answer = raw_input("Are you sure? (y/n) ")
  73     if answer.lower() != 'y':
  74         sys.exit(0)
  75
  76 def fetch_resources(fetcher, options=None):
  77     try:
  78         if options:
  79             res_list = fetcher(search_opts=options)
  80         else:
  81             res_list = fetcher()
  82     except Exception as e:
  83         res_list = []
  84         traceback.print_exc()
  85         print "Warning exception while listing resources:" + str(e)
  86     resources = {}
  87     for res in res_list:
  88         # some objects provide direct access some
  89         # require access by key
  90         try:
  91             resid = res.id
  92             resname = res.name
  93         except AttributeError:
  94             resid = res['id']
  95             resname = res['name']
  96         if resname and resource_name_re.match(resname):
  97             resources[resid] = resname
  98     return resources
  99
 100 class AbstractCleaner(object):
 101     __metaclass__ = ABCMeta
 102
 103     def __init__(self, res_category, res_desc, resources, dryrun):
 104         self.dryrun = dryrun
 105         self.category = res_category
 106         self.resources = {}
 107         if not resources:
 108             print 'Discovering %s resources...' % (res_category)
 109         for rtype, fetch_args in res_desc.iteritems():
 110             if resources:
 111                 if rtype in resources:
 112                     self.resources[rtype] = resources[rtype]
 113             else:
 114                 self.resources[rtype] = fetch_resources(*fetch_args)
 115
 116     def report_deletion(self, rtype, name):
 117         if self.dryrun:
 118             print '    + ' + rtype + ' ' + name + ' should be deleted (but is not deleted: dry run)'
 119         else:
 120             print '    + ' + rtype + ' ' + name + ' is successfully deleted'
 121
 122     def report_not_found(self, rtype, name):
 123         print '    ? ' + rtype + ' ' + name + ' not found (already deleted?)'
 124
 125     def report_error(self, rtype, name, reason):
 126         print '    - ' + rtype + ' ' + name + ' ERROR:' + reason
 127
 128     def get_resource_list(self):
 129         result = []
 130         for rtype, rdict in self.resources.iteritems():
 131             for resid, resname in rdict.iteritems():
 132                 result.append([rtype, resname, resid])
 133         return result
 134
 135     @abstractmethod
 136     def clean(self):
 137         pass
 138
 139 class StorageCleaner(AbstractCleaner):
 140     def __init__(self, sess, resources, dryrun):
 141         from cinderclient import client as cclient
 142         from novaclient import client as nclient
 143
 144         self.nova = nclient.Client('2', endpoint_type='publicURL', session=sess)
 145         self.cinder = cclient.Client('2', endpoint_type='publicURL', session=sess)
 146
 147         res_desc = {'volumes': [self.cinder.volumes.list, {"all_tenants": 1}]}
 148         super(StorageCleaner, self).__init__('Storage', res_desc, resources, dryrun)
 149
 150     def clean(self):
 151         print '*** STORAGE cleanup'
 152         try:
 153             volumes = []
 154             detaching_volumes = []
 155             for id, name in self.resources['volumes'].iteritems():
 156                 try:
 157                     vol = self.cinder.volumes.get(id)
 158                     if vol.attachments:
 159                         # detach the volume
 160                         try:
 161                             if not self.dryrun:
 162                                 ins_id = vol.attachments[0]['server_id']
 163                                 self.nova.volumes.delete_server_volume(ins_id, id)
 164                                 print '    . VOLUME ' + vol.name + ' detaching...'
 165                             else:
 166                                 print '    . VOLUME ' + vol.name + ' to be detached...'
 167                             detaching_volumes.append(vol)
 168                         except NotFound:
 169                             print 'WARNING: Volume %s attached to an instance that no longer '\
 170                                   'exists (will require manual cleanup of the database)' % (id)
 171                         except Exception as e:
 172                             print str(e)
 173                     else:
 174                         # no attachments
 175                         volumes.append(vol)
 176                 except cinderclient.exceptions.NotFound:
 177                     self.report_not_found('VOLUME', name)
 178
 179             # check that the volumes are no longer attached
 180             if detaching_volumes:
 181                 if not self.dryrun:
 182                     print '    . Waiting for %d volumes to be fully detached...' % \
 183                         (len(detaching_volumes))
 184                 retry_count = 5 + len(detaching_volumes)
 185                 while True:
 186                     retry_count -= 1
 187                     for vol in list(detaching_volumes):
 188                         if not self.dryrun:
 189                             latest_vol = self.cinder.volumes.get(detaching_volumes[0].id)
 190                         if self.dryrun or not latest_vol.attachments:
 191                             if not self.dryrun:
 192                                 print '    + VOLUME ' + vol.name + ' detach complete'
 193                             detaching_volumes.remove(vol)
 194                             volumes.append(vol)
 195                     if detaching_volumes and not self.dryrun:
 196                         if retry_count:
 197                             print '    . VOLUME %d left to be detached, retries left=%d...' % \
 198                                 (len(detaching_volumes), retry_count)
 199                             time.sleep(2)
 200                         else:
 201                             print '    - VOLUME detach timeout, %d volumes left:' % \
 202                                 (len(detaching_volumes))
 203                             for vol in detaching_volumes:
 204                                 print '         ', vol.name, vol.status, vol.id, vol.attachments
 205                             break
 206                     else:
 207                         break
 208
 209             # finally delete the volumes
 210             for vol in volumes:
 211                 if not self.dryrun:
 212                     try:
 213                         vol.force_delete()
 214                     except cinderclient.exceptions.BadRequest as exc:
 215                         print str(exc)
 216                 self.report_deletion('VOLUME', vol.name)
 217         except KeyError:
 218             pass
 219
 220 class ComputeCleaner(AbstractCleaner):
 221     def __init__(self, sess, resources, dryrun):
 222         from neutronclient.neutron import client as nclient
 223         from novaclient import client as novaclient
 224         self.neutron_client = nclient.Client('2.0', endpoint_type='publicURL', session=sess)
 225         self.nova_client = novaclient.Client('2', endpoint_type='publicURL', session=sess)
 226         res_desc = {
 227             'instances': [self.nova_client.servers.list, {"all_tenants": 1}],
 228             'flavors': [self.nova_client.flavors.list],
 229             'keypairs': [self.nova_client.keypairs.list]
 230         }
 231         super(ComputeCleaner, self).__init__('Compute', res_desc, resources, dryrun)
 232
 233     def clean(self):
 234         print '*** COMPUTE cleanup'
 235         try:
 236             # Get a list of floating IPs
 237             fip_lst = self.neutron_client.list_floatingips()['floatingips']
 238             deleting_instances = self.resources['instances']
 239             for id, name in self.resources['instances'].iteritems():
 240                 try:
 241                     if self.nova_client.servers.get(id).addresses.values():
 242                         ins_addr = self.nova_client.servers.get(id).addresses.values()[0]
 243                         fips = [x['addr'] for x in ins_addr if x['OS-EXT-IPS:type'] == 'floating']
 244                     else:
 245                         fips = []
 246                     if self.dryrun:
 247                         self.nova_client.servers.get(id)
 248                         for fip in fips:
 249                             self.report_deletion('FLOATING IP', fip)
 250                         self.report_deletion('INSTANCE', name)
 251                     else:
 252                         for fip in fips:
 253                             fip_id = [x['id'] for x in fip_lst if x['floating_ip_address'] == fip]
 254                             self.neutron_client.delete_floatingip(fip_id[0])
 255                             self.report_deletion('FLOATING IP', fip)
 256                         self.nova_client.servers.delete(id)
 257                 except NotFound:
 258                     deleting_instances.remove(id)
 259                     self.report_not_found('INSTANCE', name)
 260
 261             if not self.dryrun and len(deleting_instances):
 262                 print '    . Waiting for %d instances to be fully deleted...' % \
 263                     (len(deleting_instances))
 264                 retry_count = 5 + len(deleting_instances)
 265                 while True:
 266                     retry_count -= 1
 267                     for ins_id in deleting_instances.keys():
 268                         try:
 269                             self.nova_client.servers.get(ins_id)
 270                         except NotFound:
 271                             self.report_deletion('INSTANCE', deleting_instances[ins_id])
 272                             deleting_instances.pop(ins_id)
 273
 274                     if not len(deleting_instances):
 275                         break
 276
 277                     if retry_count:
 278                         print '    . INSTANCE %d left to be deleted, retries left=%d...' % \
 279                             (len(deleting_instances), retry_count)
 280                         time.sleep(2)
 281                     else:
 282                         print '    - INSTANCE deletion timeout, %d instances left:' % \
 283                             (len(deleting_instances))
 284                         for ins_id in deleting_instances.keys():
 285                             try:
 286                                 ins = self.nova_client.servers.get(ins_id)
 287                                 print '         ', ins.name, ins.status, ins.id
 288                             except NotFound:
 289                                 print('         ', deleting_instances[ins_id],
 290                                       '(just deleted)', ins_id)
 291                         break
 292         except KeyError:
 293             pass
 294
 295         try:
 296             for id, name in self.resources['flavors'].iteritems():
 297                 try:
 298                     flavor = self.nova_client.flavors.find(name=name)
 299                     if not self.dryrun:
 300                         flavor.delete()
 301                     self.report_deletion('FLAVOR', name)
 302                 except NotFound:
 303                     self.report_not_found('FLAVOR', name)
 304         except KeyError:
 305             pass
 306
 307         try:
 308             for id, name in self.resources['keypairs'].iteritems():
 309                 try:
 310                     if self.dryrun:
 311                         self.nova_client.keypairs.get(name)
 312                     else:
 313                         self.nova_client.keypairs.delete(name)
 314                     self.report_deletion('KEY PAIR', name)
 315                 except NotFound:
 316                     self.report_not_found('KEY PAIR', name)
 317         except KeyError:
 318             pass
 319
 320 class NetworkCleaner(AbstractCleaner):
 321
 322     def __init__(self, sess, resources, dryrun):
 323         from neutronclient.neutron import client as nclient
 324         self.neutron = nclient.Client('2.0', endpoint_type='publicURL', session=sess)
 325
 326         # because the response has an extra level of indirection
 327         # we need to extract it to present the list of network or router objects
 328         def networks_fetcher():
 329             return self.neutron.list_networks()['networks']
 330
 331         def routers_fetcher():
 332             return self.neutron.list_routers()['routers']
 333
 334         def secgroup_fetcher():
 335             return self.neutron.list_security_groups()['security_groups']
 336
 337         res_desc = {
 338             'sec_groups': [secgroup_fetcher],
 339             'networks': [networks_fetcher],
 340             'routers': [routers_fetcher]
 341         }
 342         super(NetworkCleaner, self).__init__('Network', res_desc, resources, dryrun)
 343
 344     def remove_router_interface(self, router_id, port):
 345         """
 346         Remove the network interface from router
 347         """
 348         body = {
 349             # 'port_id': port['id']
 350             'subnet_id': port['fixed_ips'][0]['subnet_id']
 351         }
 352         try:
 353             self.neutron.remove_interface_router(router_id, body)
 354             self.report_deletion('Router Interface', port['fixed_ips'][0]['ip_address'])
 355         except neutronclient.common.exceptions.NotFound:
 356             pass
 357
 358     def remove_network_ports(self, net):
 359         """
 360         Remove ports belonging to network
 361         """
 362         for port in filter(lambda p: p['network_id'] == net, self.neutron.list_ports()['ports']):
 363             try:
 364                 self.neutron.delete_port(port['id'])
 365                 self.report_deletion('Network port', port['id'])
 366             except neutronclient.common.exceptions.NotFound:
 367                 pass
 368
 369     def clean(self):
 370         print '*** NETWORK cleanup'
 371
 372         try:
 373             for id, name in self.resources['sec_groups'].iteritems():
 374                 try:
 375                     if self.dryrun:
 376                         self.neutron.show_security_group(id)
 377                     else:
 378                         self.neutron.delete_security_group(id)
 379                     self.report_deletion('SECURITY GROUP', name)
 380                 except NotFound:
 381                     self.report_not_found('SECURITY GROUP', name)
 382         except KeyError:
 383             pass
 384
 385         try:
 386             for id, name in self.resources['floating_ips'].iteritems():
 387                 try:
 388                     if self.dryrun:
 389                         self.neutron.show_floatingip(id)
 390                     else:
 391                         self.neutron.delete_floatingip(id)
 392                     self.report_deletion('FLOATING IP', name)
 393                 except neutronclient.common.exceptions.NotFound:
 394                     self.report_not_found('FLOATING IP', name)
 395         except KeyError:
 396             pass
 397
 398         try:
 399             for id, name in self.resources['routers'].iteritems():
 400                 try:
 401                     if self.dryrun:
 402                         self.neutron.show_router(id)
 403                         self.report_deletion('Router Gateway', name)
 404                         port_list = self.neutron.list_ports(id)['ports']
 405                         for port in port_list:
 406                             if 'fixed_ips' in port:
 407                                 self.report_deletion('Router Interface',
 408                                                      port['fixed_ips'][0]['ip_address'])
 409                     else:
 410                         self.neutron.remove_gateway_router(id)
 411                         self.report_deletion('Router Gateway', name)
 412                         # need to delete each interface before deleting the router
 413                         port_list = self.neutron.list_ports(id)['ports']
 414                         for port in port_list:
 415                             self.remove_router_interface(id, port)
 416                         self.neutron.delete_router(id)
 417                     self.report_deletion('ROUTER', name)
 418                 except neutronclient.common.exceptions.NotFound:
 419                     self.report_not_found('ROUTER', name)
 420                 except neutronclient.common.exceptions.Conflict as exc:
 421                     self.report_error('ROUTER', name, str(exc))
 422         except KeyError:
 423             pass
 424         try:
 425             for id, name in self.resources['networks'].iteritems():
 426                 try:
 427                     if self.dryrun:
 428                         self.neutron.show_network(id)
 429                     else:
 430                         self.remove_network_ports(id)
 431                         self.neutron.delete_network(id)
 432                     self.report_deletion('NETWORK', name)
 433                 except neutronclient.common.exceptions.NetworkNotFoundClient:
 434                     self.report_not_found('NETWORK', name)
 435                 except neutronclient.common.exceptions.NetworkInUseClient as exc:
 436                     self.report_error('NETWORK', name, str(exc))
 437         except KeyError:
 438             pass
 439
 440 class KeystoneCleaner(AbstractCleaner):
 441
 442     def __init__(self, sess, resources, dryrun):
 443         self.keystone = keystoneclient.Client(endpoint_type='publicURL', session=sess)
 444         self.tenant_api = self.keystone.tenants \
 445             if self.keystone.version == 'v2.0' else self.keystone.projects
 446         res_desc = {
 447             'users': [self.keystone.users.list],
 448             'tenants': [self.tenant_api.list]
 449         }
 450         super(KeystoneCleaner, self).__init__('Keystone', res_desc, resources, dryrun)
 451
 452     def clean(self):
 453         print '*** KEYSTONE cleanup'
 454         try:
 455             for id, name in self.resources['users'].iteritems():
 456                 try:
 457                     if self.dryrun:
 458                         self.keystone.users.get(id)
 459                     else:
 460                         self.keystone.users.delete(id)
 461                     self.report_deletion('USER', name)
 462                 except keystoneclient.auth.exceptions.http.NotFound:
 463                     self.report_not_found('USER', name)
 464         except KeyError:
 465             pass
 466
 467         try:
 468             for id, name in self.resources['tenants'].iteritems():
 469                 try:
 470                     if self.dryrun:
 471                         self.tenant_api.get(id)
 472                     else:
 473                         self.tenant_api.delete(id)
 474                     self.report_deletion('TENANT', name)
 475                 except keystoneclient.auth.exceptions.http.NotFound:
 476                     self.report_not_found('TENANT', name)
 477         except KeyError:
 478             pass
 479
 480 class Cleaners(object):
 481
 482     def __init__(self, creds_obj, resources, dryrun):
 483         self.cleaners = []
 484         sess = creds_obj.get_session()
 485         for cleaner_type in [StorageCleaner, ComputeCleaner, NetworkCleaner, KeystoneCleaner]:
 486             self.cleaners.append(cleaner_type(sess, resources, dryrun))
 487
 488     def show_resources(self):
 489         table = [["Type", "Name", "UUID"]]
 490         for cleaner in self.cleaners:
 491             table.extend(cleaner.get_resource_list())
 492         count = len(table) - 1
 493         print
 494         if count:
 495             print 'SELECTED RESOURCES:'
 496             print tabulate(table, headers="firstrow", tablefmt="psql")
 497         else:
 498             print 'There are no resources to delete.'
 499         print
 500         return count
 501
 502     def clean(self):
 503         for cleaner in self.cleaners:
 504             cleaner.clean()
 505
 506 # A dictionary of resources to cleanup
 507 # First level keys are:
 508 # flavors, keypairs, users, routers, floating_ips, instances, volumes, sec_groups, tenants, networks
 509 # second level keys are the resource IDs
 510 # values are the resource name  (e.g. 'nfvbench-net0')
 511 def get_resources_from_cleanup_log(logfile):
 512     '''Load triplets separated by '|' into a 2 level dictionary
 513     '''
 514     resources = {}
 515     with open(logfile) as ff:
 516         content = ff.readlines()
 517         for line in content:
 518             tokens = line.strip().split('|')
 519             restype = tokens[0]
 520             resname = tokens[1]
 521             resid = tokens[2]
 522             if not resid:
 523                 # normally only the keypairs have no ID
 524                 if restype != "keypairs":
 525                     print 'Error: resource type %s has no ID - ignored!!!' % (restype)
 526                 else:
 527                     resid = '0'
 528             if restype not in resources:
 529                 resources[restype] = {}
 530             tres = resources[restype]
 531             tres[resid] = resname
 532     return resources
 533
 534
 535 def main():
 536     parser = argparse.ArgumentParser(description='NFVbench Force Cleanup')
 537
 538     parser.add_argument('-r', '--rc', dest='rc',
 539                         action='store', required=False,
 540                         help='openrc file',
 541                         metavar='<file>')
 542     parser.add_argument('-f', '--file', dest='file',
 543                         action='store', required=False,
 544                         help='get resources to delete from cleanup log file '
 545                              '(default:discover from OpenStack)',
 546                         metavar='<file>')
 547     parser.add_argument('-d', '--dryrun', dest='dryrun',
 548                         action='store_true',
 549                         default=False,
 550                         help='check resources only - do not delete anything')
 551     parser.add_argument('--filter', dest='filter',
 552                         action='store', required=False,
 553                         help='resource name regular expression filter (default:"nfvbench")'
 554                              ' - OpenStack discovery only',
 555                         metavar='<any-python-regex>')
 556     opts = parser.parse_args()
 557
 558     cred = credentials.Credentials(openrc_file=opts.rc)
 559
 560     if opts.file:
 561         resources = get_resources_from_cleanup_log(opts.file)
 562     else:
 563         # None means try to find the resources from openstack directly by name
 564         resources = None
 565     global resource_name_re
 566     if opts.filter:
 567         try:
 568             resource_name_re = re.compile(opts.filter)
 569         except Exception as exc:
 570             print 'Provided filter is not a valid python regular expression: ' + opts.filter
 571             print str(exc)
 572             sys.exit(1)
 573     else:
 574         resource_name_re = re.compile('nfvbench')
 575
 576     cleaners = Cleaners(cred, resources, opts.dryrun)
 577
 578     if opts.dryrun:
 579         print
 580         print('!!! DRY RUN - RESOURCES WILL BE CHECKED BUT WILL NOT BE DELETED !!!')
 581         print
 582
 583     # Display resources to be deleted
 584     count = cleaners.show_resources()
 585     if not count:
 586         sys.exit(0)
 587
 588     if not opts.file and not opts.dryrun:
 589         prompt_to_run()
 590
 591     cleaners.clean()
 592
 593 if __name__ == '__main__':
 594     main()