+++ /dev/null
-"""Scrub testing"""
-from cStringIO import StringIO
-
-import contextlib
-import json
-import logging
-import os
-import time
-import tempfile
-
-import ceph_manager
-from teuthology import misc as teuthology
-
-log = logging.getLogger(__name__)
-
-
-def wait_for_victim_pg(manager):
- """Return a PG with some data and its acting set"""
- # wait for some PG to have data that we can mess with
- victim = None
- while victim is None:
- stats = manager.get_pg_stats()
- for pg in stats:
- size = pg['stat_sum']['num_bytes']
- if size > 0:
- victim = pg['pgid']
- acting = pg['acting']
- return victim, acting
- time.sleep(3)
-
-
-def find_victim_object(ctx, pg, osd):
- """Return a file to be fuzzed"""
- (osd_remote,) = ctx.cluster.only('osd.%d' % osd).remotes.iterkeys()
- data_path = os.path.join(
- '/var/lib/ceph/osd',
- 'ceph-{id}'.format(id=osd),
- 'fuse',
- '{pg}_head'.format(pg=pg),
- 'all',
- )
-
- # fuzz time
- with contextlib.closing(StringIO()) as ls_fp:
- osd_remote.run(
- args=['sudo', 'ls', data_path],
- stdout=ls_fp,
- )
- ls_out = ls_fp.getvalue()
-
- # find an object file we can mess with (and not the pg info object)
- osdfilename = next(line for line in ls_out.split('\n')
- if not line.endswith('::::head#'))
- assert osdfilename is not None
-
- # Get actual object name from osd stored filename
- objname = osdfilename.split(':')[4]
- return osd_remote, os.path.join(data_path, osdfilename), objname
-
-
-def corrupt_file(osd_remote, path):
- # put a single \0 at the beginning of the file
- osd_remote.run(
- args=['sudo', 'dd',
- 'if=/dev/zero',
- 'of=%s/data' % path,
- 'bs=1', 'count=1', 'conv=notrunc']
- )
-
-
-def get_pgnum(pgid):
- pos = pgid.find('.')
- assert pos != -1
- return pgid[pos+1:]
-
-
-def deep_scrub(manager, victim, pool):
- # scrub, verify inconsistent
- pgnum = get_pgnum(victim)
- manager.do_pg_scrub(pool, pgnum, 'deep-scrub')
-
- stats = manager.get_single_pg_stats(victim)
- inconsistent = stats['state'].find('+inconsistent') != -1
- assert inconsistent
-
-
-def repair(manager, victim, pool):
- # repair, verify no longer inconsistent
- pgnum = get_pgnum(victim)
- manager.do_pg_scrub(pool, pgnum, 'repair')
-
- stats = manager.get_single_pg_stats(victim)
- inconsistent = stats['state'].find('+inconsistent') != -1
- assert not inconsistent
-
-
-def test_repair_corrupted_obj(ctx, manager, pg, osd_remote, obj_path, pool):
- corrupt_file(osd_remote, obj_path)
- deep_scrub(manager, pg, pool)
- repair(manager, pg, pool)
-
-
-def test_repair_bad_omap(ctx, manager, pg, osd, objname):
- # Test deep-scrub with various omap modifications
- # Modify omap on specific osd
- log.info('fuzzing omap of %s' % objname)
- manager.osd_admin_socket(osd, ['rmomapkey', 'rbd', objname, 'key'])
- manager.osd_admin_socket(osd, ['setomapval', 'rbd', objname,
- 'badkey', 'badval'])
- manager.osd_admin_socket(osd, ['setomapheader', 'rbd', objname, 'badhdr'])
-
- deep_scrub(manager, pg, 'rbd')
- # please note, the repair here is errnomous, it rewrites the correct omap
- # digest and data digest on the replicas with the corresponding digests
- # from the primary osd which is hosting the victim object, see
- # find_victim_object().
- # so we need to either put this test and the end of this task or
- # undo the mess-up manually before the "repair()" that just ensures
- # the cleanup is sane, otherwise the succeeding tests will fail. if they
- # try set "badkey" in hope to get an "inconsistent" pg with a deep-scrub.
- manager.osd_admin_socket(osd, ['setomapheader', 'rbd', objname, 'hdr'])
- manager.osd_admin_socket(osd, ['rmomapkey', 'rbd', objname, 'badkey'])
- manager.osd_admin_socket(osd, ['setomapval', 'rbd', objname,
- 'key', 'val'])
- repair(manager, pg, 'rbd')
-
-
-class MessUp:
- def __init__(self, manager, osd_remote, pool, osd_id,
- obj_name, obj_path, omap_key, omap_val):
- self.manager = manager
- self.osd = osd_remote
- self.pool = pool
- self.osd_id = osd_id
- self.obj = obj_name
- self.path = obj_path
- self.omap_key = omap_key
- self.omap_val = omap_val
-
- @contextlib.contextmanager
- def _test_with_file(self, messup_cmd, *checks):
- temp = tempfile.mktemp()
- backup_cmd = ['sudo', 'cp', os.path.join(self.path, 'data'), temp]
- self.osd.run(args=backup_cmd)
- self.osd.run(args=messup_cmd.split())
- yield checks
- create_cmd = ['sudo', 'mkdir', self.path]
- self.osd.run(args=create_cmd, check_status=False)
- restore_cmd = ['sudo', 'cp', temp, os.path.join(self.path, 'data')]
- self.osd.run(args=restore_cmd)
-
- def remove(self):
- cmd = 'sudo rmdir {path}'.format(path=self.path)
- return self._test_with_file(cmd, 'missing')
-
- def append(self):
- cmd = 'sudo dd if=/dev/zero of={path}/data bs=1 count=1 ' \
- 'conv=notrunc oflag=append'.format(path=self.path)
- return self._test_with_file(cmd,
- 'data_digest_mismatch',
- 'size_mismatch')
-
- def truncate(self):
- cmd = 'sudo dd if=/dev/null of={path}/data'.format(path=self.path)
- return self._test_with_file(cmd,
- 'data_digest_mismatch',
- 'size_mismatch')
-
- def change_obj(self):
- cmd = 'sudo dd if=/dev/zero of={path}/data bs=1 count=1 ' \
- 'conv=notrunc'.format(path=self.path)
- return self._test_with_file(cmd,
- 'data_digest_mismatch')
-
- @contextlib.contextmanager
- def rm_omap(self):
- cmd = ['rmomapkey', self.pool, self.obj, self.omap_key]
- self.manager.osd_admin_socket(self.osd_id, cmd)
- yield ('omap_digest_mismatch',)
- cmd = ['setomapval', self.pool, self.obj,
- self.omap_key, self.omap_val]
- self.manager.osd_admin_socket(self.osd_id, cmd)
-
- @contextlib.contextmanager
- def add_omap(self):
- cmd = ['setomapval', self.pool, self.obj, 'badkey', 'badval']
- self.manager.osd_admin_socket(self.osd_id, cmd)
- yield ('omap_digest_mismatch',)
- cmd = ['rmomapkey', self.pool, self.obj, 'badkey']
- self.manager.osd_admin_socket(self.osd_id, cmd)
-
- @contextlib.contextmanager
- def change_omap(self):
- cmd = ['setomapval', self.pool, self.obj, self.omap_key, 'badval']
- self.manager.osd_admin_socket(self.osd_id, cmd)
- yield ('omap_digest_mismatch',)
- cmd = ['setomapval', self.pool, self.obj, self.omap_key, self.omap_val]
- self.manager.osd_admin_socket(self.osd_id, cmd)
-
-
-class InconsistentObjChecker:
- """Check the returned inconsistents/inconsistent info"""
-
- def __init__(self, osd, acting, obj_name):
- self.osd = osd
- self.acting = acting
- self.obj = obj_name
- assert self.osd in self.acting
-
- def basic_checks(self, inc):
- assert inc['object']['name'] == self.obj
- assert inc['object']['snap'] == "head"
- assert len(inc['shards']) == len(self.acting), \
- "the number of returned shard does not match with the acting set"
-
- def run(self, check, inc):
- func = getattr(self, check)
- func(inc)
-
- def _check_errors(self, inc, err_name):
- bad_found = False
- good_found = False
- for shard in inc['shards']:
- log.info('shard = %r' % shard)
- log.info('err = %s' % err_name)
- assert 'osd' in shard
- osd = shard['osd']
- err = err_name in shard['errors']
- if osd == self.osd:
- assert bad_found is False, \
- "multiple entries found for the given OSD"
- assert err is True, \
- "Didn't find '{err}' in errors".format(err=err_name)
- bad_found = True
- else:
- assert osd in self.acting, "shard not in acting set"
- assert err is False, \
- "Expected '{err}' in errors".format(err=err_name)
- good_found = True
- assert bad_found is True, \
- "Shard for osd.{osd} not found".format(osd=self.osd)
- assert good_found is True, \
- "No other acting shards found"
-
- def _check_attrs(self, inc, attr_name):
- bad_attr = None
- good_attr = None
- for shard in inc['shards']:
- log.info('shard = %r' % shard)
- log.info('attr = %s' % attr_name)
- assert 'osd' in shard
- osd = shard['osd']
- attr = shard.get(attr_name, False)
- if osd == self.osd:
- assert bad_attr is None, \
- "multiple entries found for the given OSD"
- bad_attr = attr
- else:
- assert osd in self.acting, "shard not in acting set"
- assert good_attr is None or good_attr == attr, \
- "multiple good attrs found"
- good_attr = attr
- assert bad_attr is not None, \
- "bad {attr} not found".format(attr=attr_name)
- assert good_attr is not None, \
- "good {attr} not found".format(attr=attr_name)
- assert good_attr != bad_attr, \
- "bad attr is identical to the good ones: " \
- "{0} == {1}".format(good_attr, bad_attr)
-
- def data_digest_mismatch(self, inc):
- assert 'data_digest_mismatch' in inc['errors']
- self._check_attrs(inc, 'data_digest')
-
- def missing(self, inc):
- assert 'missing' in inc['union_shard_errors']
- self._check_errors(inc, 'missing')
-
- def size_mismatch(self, inc):
- assert 'size_mismatch' in inc['errors']
- self._check_attrs(inc, 'size')
-
- def omap_digest_mismatch(self, inc):
- assert 'omap_digest_mismatch' in inc['errors']
- self._check_attrs(inc, 'omap_digest')
-
-
-def test_list_inconsistent_obj(ctx, manager, osd_remote, pg, acting, osd_id,
- obj_name, obj_path):
- mon = manager.controller
- pool = 'rbd'
- omap_key = 'key'
- omap_val = 'val'
- manager.do_rados(mon, ['-p', pool, 'setomapval', obj_name,
- omap_key, omap_val])
- # Update missing digests, requires "osd deep scrub update digest min age: 0"
- pgnum = get_pgnum(pg)
- manager.do_pg_scrub(pool, pgnum, 'deep-scrub')
-
- messup = MessUp(manager, osd_remote, pool, osd_id, obj_name, obj_path,
- omap_key, omap_val)
- for test in [messup.rm_omap, messup.add_omap, messup.change_omap,
- messup.append, messup.truncate, messup.change_obj,
- messup.remove]:
- with test() as checks:
- deep_scrub(manager, pg, pool)
- cmd = 'rados list-inconsistent-pg {pool} ' \
- '--format=json'.format(pool=pool)
- with contextlib.closing(StringIO()) as out:
- mon.run(args=cmd.split(), stdout=out)
- pgs = json.loads(out.getvalue())
- assert pgs == [pg]
-
- cmd = 'rados list-inconsistent-obj {pg} ' \
- '--format=json'.format(pg=pg)
- with contextlib.closing(StringIO()) as out:
- mon.run(args=cmd.split(), stdout=out)
- objs = json.loads(out.getvalue())
- assert len(objs['inconsistents']) == 1
-
- checker = InconsistentObjChecker(osd_id, acting, obj_name)
- inc_obj = objs['inconsistents'][0]
- log.info('inc = %r', inc_obj)
- checker.basic_checks(inc_obj)
- for check in checks:
- checker.run(check, inc_obj)
-
-
-def task(ctx, config):
- """
- Test [deep] scrub
-
- tasks:
- - chef:
- - install:
- - ceph:
- log-whitelist:
- - '!= data_digest'
- - '!= omap_digest'
- - '!= size'
- - deep-scrub 0 missing, 1 inconsistent objects
- - deep-scrub [0-9]+ errors
- - repair 0 missing, 1 inconsistent objects
- - repair [0-9]+ errors, [0-9]+ fixed
- - shard [0-9]+ missing
- - deep-scrub 1 missing, 1 inconsistent objects
- - does not match object info size
- - attr name mistmatch
- - deep-scrub 1 missing, 0 inconsistent objects
- - failed to pick suitable auth object
- conf:
- osd:
- osd deep scrub update digest min age: 0
- - scrub_test:
- """
- if config is None:
- config = {}
- assert isinstance(config, dict), \
- 'scrub_test task only accepts a dict for configuration'
- first_mon = teuthology.get_first_mon(ctx, config)
- (mon,) = ctx.cluster.only(first_mon).remotes.iterkeys()
-
- num_osds = teuthology.num_instances_of_type(ctx.cluster, 'osd')
- log.info('num_osds is %s' % num_osds)
-
- manager = ceph_manager.CephManager(
- mon,
- ctx=ctx,
- logger=log.getChild('ceph_manager'),
- )
-
- while len(manager.get_osd_status()['up']) < num_osds:
- time.sleep(10)
-
- for i in range(num_osds):
- manager.raw_cluster_cmd('tell', 'osd.%d' % i, 'injectargs',
- '--', '--osd-objectstore-fuse')
- manager.flush_pg_stats(range(num_osds))
- manager.wait_for_clean()
-
- # write some data
- p = manager.do_rados(mon, ['-p', 'rbd', 'bench', '--no-cleanup', '1',
- 'write', '-b', '4096'])
- log.info('err is %d' % p.exitstatus)
-
- # wait for some PG to have data that we can mess with
- pg, acting = wait_for_victim_pg(manager)
- osd = acting[0]
-
- osd_remote, obj_path, obj_name = find_victim_object(ctx, pg, osd)
- manager.do_rados(mon, ['-p', 'rbd', 'setomapval', obj_name, 'key', 'val'])
- log.info('err is %d' % p.exitstatus)
- manager.do_rados(mon, ['-p', 'rbd', 'setomapheader', obj_name, 'hdr'])
- log.info('err is %d' % p.exitstatus)
-
- # Update missing digests, requires "osd deep scrub update digest min age: 0"
- pgnum = get_pgnum(pg)
- manager.do_pg_scrub('rbd', pgnum, 'deep-scrub')
-
- log.info('messing with PG %s on osd %d' % (pg, osd))
- test_repair_corrupted_obj(ctx, manager, pg, osd_remote, obj_path, 'rbd')
- test_repair_bad_omap(ctx, manager, pg, osd, obj_name)
- test_list_inconsistent_obj(ctx, manager, osd_remote, pg, acting, osd,
- obj_name, obj_path)
- log.info('test successful!')
-
- # shut down fuse mount
- for i in range(num_osds):
- manager.raw_cluster_cmd('tell', 'osd.%d' % i, 'injectargs',
- '--', '--no-osd-objectstore-fuse')
- time.sleep(5)
- log.info('done')
Code Review / stor4nfv.git / blobdiff