7 from teuthology import misc as teuthology
10 log = logging.getLogger(__name__)
13 def rados_start(ctx, remote, cmd):
15 Run a remote rados command (currently used to only write data)
17 log.info("rados %s" % ' '.join(cmd))
18 testdir = teuthology.get_testdir(ctx)
22 '{tdir}/archive/coverage'.format(tdir=testdir),
32 def task(ctx, config):
38 assert isinstance(config, dict), \
39 'thrashosds task only accepts a dict for configuration'
40 first_mon = teuthology.get_first_mon(ctx, config)
41 (mon,) = ctx.cluster.only(first_mon).remotes.iterkeys()
43 num_osds = teuthology.num_instances_of_type(ctx.cluster, 'osd')
44 log.info('num_osds is %s' % num_osds)
47 manager = ceph_manager.CephManager(
50 logger=log.getChild('ceph_manager'),
53 while len(manager.get_osd_status()['up']) < 3:
55 manager.flush_pg_stats([0, 1, 2])
56 manager.wait_for_clean()
59 p = rados_start(ctx, mon, ['-p', 'rbd', 'bench', '15', 'write', '-b', '4096',
62 log.info('err is %d' % err)
64 # mark osd.0 out to trigger a rebalance/backfill
65 manager.mark_out_osd(0)
67 # also mark it down to it won't be included in pg_temps
69 manager.mark_down_osd(0)
71 # wait for everything to peer and be happy...
72 manager.flush_pg_stats([1, 2])
73 manager.wait_for_recovery()
76 p = rados_start(ctx, mon, ['-p', 'rbd', 'bench', '30', 'write', '-b', '4096',
81 # blackhole + restart osd.1
82 # this triggers a divergent backfill target
83 manager.blackhole_kill_osd(1)
87 # wait for our writes to complete + succeed
89 log.info('err is %d' % err)
91 # wait for osd.1 and osd.2 to be up
92 manager.wait_till_osd_is_up(1)
93 manager.wait_till_osd_is_up(2)
95 # cluster must recover
96 manager.flush_pg_stats([1, 2])
97 manager.wait_for_recovery()
100 manager.revive_osd(0)
101 manager.flush_pg_stats([1, 2])
102 manager.wait_for_clean()