+++ /dev/null
-
-"""
-Test our tools for recovering metadata from the data pool into an alternate pool
-"""
-import json
-
-import logging
-import os
-from textwrap import dedent
-import traceback
-from collections import namedtuple, defaultdict
-
-from teuthology.orchestra.run import CommandFailedError
-from tasks.cephfs.cephfs_test_case import CephFSTestCase, for_teuthology
-
-log = logging.getLogger(__name__)
-
-
-ValidationError = namedtuple("ValidationError", ["exception", "backtrace"])
-
-
-class OverlayWorkload(object):
- def __init__(self, orig_fs, recovery_fs, orig_mount, recovery_mount):
- self._orig_fs = orig_fs
- self._recovery_fs = recovery_fs
- self._orig_mount = orig_mount
- self._recovery_mount = recovery_mount
- self._initial_state = None
-
- # Accumulate backtraces for every failed validation, and return them. Backtraces
- # are rather verbose, but we only see them when something breaks, and they
- # let us see which check failed without having to decorate each check with
- # a string
- self._errors = []
-
- def assert_equal(self, a, b):
- try:
- if a != b:
- raise AssertionError("{0} != {1}".format(a, b))
- except AssertionError as e:
- self._errors.append(
- ValidationError(e, traceback.format_exc(3))
- )
-
- def write(self):
- """
- Write the workload files to the mount
- """
- raise NotImplementedError()
-
- def validate(self):
- """
- Read from the mount and validate that the workload files are present (i.e. have
- survived or been reconstructed from the test scenario)
- """
- raise NotImplementedError()
-
- def damage(self):
- """
- Damage the filesystem pools in ways that will be interesting to recover from. By
- default just wipe everything in the metadata pool
- """
- # Delete every object in the metadata pool
- objects = self._orig_fs.rados(["ls"]).split("\n")
- for o in objects:
- self._orig_fs.rados(["rm", o])
-
- def flush(self):
- """
- Called after client unmount, after write: flush whatever you want
- """
- self._orig_fs.mds_asok(["flush", "journal"])
- self._recovery_fs.mds_asok(["flush", "journal"])
-
-
-class SimpleOverlayWorkload(OverlayWorkload):
- """
- Single file, single directory, check that it gets recovered and so does its size
- """
- def write(self):
- self._orig_mount.run_shell(["mkdir", "subdir"])
- self._orig_mount.write_n_mb("subdir/sixmegs", 6)
- self._initial_state = self._orig_mount.stat("subdir/sixmegs")
-
- def validate(self):
- self._recovery_mount.run_shell(["ls", "subdir"])
- st = self._recovery_mount.stat("subdir/sixmegs")
- self.assert_equal(st['st_size'], self._initial_state['st_size'])
- return self._errors
-
-class TestRecoveryPool(CephFSTestCase):
- MDSS_REQUIRED = 2
- CLIENTS_REQUIRED = 2
- REQUIRE_RECOVERY_FILESYSTEM = True
-
- def is_marked_damaged(self, rank):
- mds_map = self.fs.get_mds_map()
- return rank in mds_map['damaged']
-
- def _rebuild_metadata(self, workload, other_pool=None, workers=1):
- """
- That when all objects in metadata pool are removed, we can rebuild a metadata pool
- based on the contents of a data pool, and a client can see and read our files.
- """
-
- # First, inject some files
-
- workload.write()
-
- # Unmount the client and flush the journal: the tool should also cope with
- # situations where there is dirty metadata, but we'll test that separately
- self.mount_a.umount_wait()
- self.mount_b.umount_wait()
- workload.flush()
-
- # Create the alternate pool if requested
- recovery_fs = self.recovery_fs.name
- recovery_pool = self.recovery_fs.get_metadata_pool_name()
- self.recovery_fs.data_scan(['init', '--force-init',
- '--filesystem', recovery_fs,
- '--alternate-pool', recovery_pool])
- self.recovery_fs.mon_manager.raw_cluster_cmd('-s')
- self.recovery_fs.table_tool([recovery_fs + ":0", "reset", "session"])
- self.recovery_fs.table_tool([recovery_fs + ":0", "reset", "snap"])
- self.recovery_fs.table_tool([recovery_fs + ":0", "reset", "inode"])
-
- # Stop the MDS
- self.fs.mds_stop()
- self.fs.mds_fail()
-
- # After recovery, we need the MDS to not be strict about stats (in production these options
- # are off by default, but in QA we need to explicitly disable them)
- self.fs.set_ceph_conf('mds', 'mds verify scatter', False)
- self.fs.set_ceph_conf('mds', 'mds debug scatterstat', False)
-
- # Apply any data damage the workload wants
- workload.damage()
-
- # Reset the MDS map in case multiple ranks were in play: recovery procedure
- # only understands how to rebuild metadata under rank 0
- self.fs.mon_manager.raw_cluster_cmd('fs', 'reset', self.fs.name,
- '--yes-i-really-mean-it')
-
- def get_state(mds_id):
- info = self.mds_cluster.get_mds_info(mds_id)
- return info['state'] if info is not None else None
-
- self.fs.table_tool([self.fs.name + ":0", "reset", "session"])
- self.fs.table_tool([self.fs.name + ":0", "reset", "snap"])
- self.fs.table_tool([self.fs.name + ":0", "reset", "inode"])
-
- # Run the recovery procedure
- if False:
- with self.assertRaises(CommandFailedError):
- # Normal reset should fail when no objects are present, we'll use --force instead
- self.fs.journal_tool(["journal", "reset"])
-
- self.fs.mds_stop()
- self.fs.data_scan(['scan_extents', '--alternate-pool',
- recovery_pool, '--filesystem', self.fs.name,
- self.fs.get_data_pool_name()])
- self.fs.data_scan(['scan_inodes', '--alternate-pool',
- recovery_pool, '--filesystem', self.fs.name,
- '--force-corrupt', '--force-init',
- self.fs.get_data_pool_name()])
- self.fs.journal_tool(['--rank=' + self.fs.name + ":0", 'event',
- 'recover_dentries', 'list',
- '--alternate-pool', recovery_pool])
-
- self.fs.data_scan(['init', '--force-init', '--filesystem',
- self.fs.name])
- self.fs.data_scan(['scan_inodes', '--filesystem', self.fs.name,
- '--force-corrupt', '--force-init',
- self.fs.get_data_pool_name()])
- self.fs.journal_tool(['--rank=' + self.fs.name + ":0", 'event',
- 'recover_dentries', 'list'])
-
- self.fs.journal_tool(['--rank=' + recovery_fs + ":0", 'journal',
- 'reset', '--force'])
- self.fs.journal_tool(['--rank=' + self.fs.name + ":0", 'journal',
- 'reset', '--force'])
- self.fs.mon_manager.raw_cluster_cmd('mds', 'repaired',
- recovery_fs + ":0")
-
- # Mark the MDS repaired
- self.fs.mon_manager.raw_cluster_cmd('mds', 'repaired', '0')
-
- # Start the MDS
- self.fs.mds_restart()
- self.recovery_fs.mds_restart()
- self.fs.wait_for_daemons()
- self.recovery_fs.wait_for_daemons()
- for mds_id in self.recovery_fs.mds_ids:
- self.fs.mon_manager.raw_cluster_cmd('tell', "mds." + mds_id,
- 'injectargs', '--debug-mds=20')
- self.fs.mon_manager.raw_cluster_cmd('daemon', "mds." + mds_id,
- 'scrub_path', '/',
- 'recursive', 'repair')
- log.info(str(self.mds_cluster.status()))
-
- # Mount a client
- self.mount_a.mount()
- self.mount_b.mount(mount_fs_name=recovery_fs)
- self.mount_a.wait_until_mounted()
- self.mount_b.wait_until_mounted()
-
- # See that the files are present and correct
- errors = workload.validate()
- if errors:
- log.error("Validation errors found: {0}".format(len(errors)))
- for e in errors:
- log.error(e.exception)
- log.error(e.backtrace)
- raise AssertionError("Validation failed, first error: {0}\n{1}".format(
- errors[0].exception, errors[0].backtrace
- ))
-
- def test_rebuild_simple(self):
- self._rebuild_metadata(SimpleOverlayWorkload(self.fs, self.recovery_fs,
- self.mount_a, self.mount_b))
Code Review / stor4nfv.git / blobdiff