3 from __future__ import print_function
11 if sys.version_info[0] == 2:
12 from cStringIO import StringIO
16 elif sys.version_info[0] == 3:
17 from io import StringIO
22 import cephfs as cephfs
26 def get_name(b, i, j):
27 c = '{pre}.{pid}.{i}.{j}'.format(pre=prefix, pid=os.getpid(), i=i, j=j)
31 print("mkdir {d}".format(d=d), file=sys.stderr)
33 return ceph.stat(d)['st_ino']
36 print("creating {f}".format(f=f), file=sys.stderr)
37 fd = ceph.open(f, os.O_CREAT | os.O_RDWR, 0o644)
39 return ceph.stat(f)['st_ino']
41 def set_mds_config_param(ceph, param):
42 with open('/dev/null', 'rb') as devnull:
45 confarg = '-c {c}'.format(c=conf)
46 r = subprocess.call("ceph {ca} mds tell a injectargs '{p}'".format(ca=confarg, p=param), shell=True, stdout=devnull)
53 class _TrimIndentFile(object):
54 def __init__(self, fp):
58 line = self.fp.readline()
59 return line.lstrip(' \t')
62 s = s.replace('_', ' ')
63 s = '_'.join(s.split())
66 def conf_set_kill_mds(location, killnum):
67 print('setting mds kill config option for {l}.{k}'.format(l=location, k=killnum), file=sys.stderr)
68 print("restart mds a mds_kill_{l}_at {k}".format(l=location, k=killnum))
70 for l in sys.stdin.readline():
74 def flush(ceph, testnum):
75 print('flushing {t}'.format(t=testnum), file=sys.stderr)
76 set_mds_config_param(ceph, '--mds_log_max_segments 1')
78 for i in range(1, 500):
79 f = '{p}.{pid}.{t}.{i}'.format(p=prefix, pid=os.getpid(), t=testnum, i=i)
80 print('flushing with create {f}'.format(f=f), file=sys.stderr)
81 fd = ceph.open(f, os.O_CREAT | os.O_RDWR, 0o644)
85 print('flush doing shutdown', file=sys.stderr)
87 print('flush reinitializing ceph', file=sys.stderr)
88 ceph = cephfs.LibCephFS(conffile=conf)
89 print('flush doing mount', file=sys.stderr)
93 def kill_mds(ceph, location, killnum):
94 print('killing mds: {l}.{k}'.format(l=location, k=killnum), file=sys.stderr)
95 set_mds_config_param(ceph, '--mds_kill_{l}_at {k}'.format(l=location, k=killnum))
97 def wait_for_mds(ceph):
102 confarg = '-c {c}'.format(c=conf)
103 r = subprocess.check_output("ceph {ca} mds stat".format(ca=confarg), shell=True).decode()
104 if r.find('a=up:active'):
110 tmpfile = '/tmp/{p}.{pid}'.format(p=prefix, pid=os.getpid())
111 with open(tmpfile, 'w+') as f:
114 p = subprocess.Popen(
124 stdin=subprocess.PIPE,
125 stdout=subprocess.PIPE,
127 (stdout, _) = p.communicate(input=value)
129 if p.returncode != 0:
132 return json.loads(stdout)
134 class VerifyFailure(Exception):
137 def verify(rados_ioctx, ino, values, pool):
138 print('getting parent attr for ino: %lx.00000000' % ino, file=sys.stderr)
140 for i in range(1, 20):
143 binbt = rados_ioctx.get_xattr('%lx.00000000' % ino, 'parent')
144 except rados.ObjectNotFound as e:
145 # wait for a bit to let segments get flushed out
154 raise VerifyFailure('inode mismatch: {bi} != {ino}\n\tbacktrace:\n\t\t{bt}\n\tfailed verify against:\n\t\t{i}, {v}'.format(
155 bi=bt['ancestors'][ind]['dname'], ino=ino, bt=bt, i=ino, v=values))
157 for (n, i) in values:
158 if bt['ancestors'][ind]['dirino'] != i:
159 raise VerifyFailure('ancestor dirino mismatch: {b} != {ind}\n\tbacktrace:\n\t\t{bt}\n\tfailed verify against:\n\t\t{i}, {v}'.format(
160 b=bt['ancestors'][ind]['dirino'], ind=i, bt=bt, i=ino, v=values))
161 if bt['ancestors'][ind]['dname'] != n:
162 raise VerifyFailure('ancestor dname mismatch: {b} != {n}\n\tbacktrace:\n\t\t{bt}\n\tfailed verify against:\n\t\t{i}, {v}'.format(
163 b=bt['ancestors'][ind]['dname'], n=n, bt=bt, i=ino, v=values))
166 if bt['pool'] != pool:
167 raise VerifyFailure('pool mismatch: {btp} != {p}\n\tbacktrace:\n\t\t{bt}\n\tfailed verify against:\n\t\t{i}, {v}'.format(
168 btp=bt['pool'], p=pool, bt=bt, i=ino, v=values))
170 def make_abc(ceph, rooti, i):
172 c, d = get_name("/", i, 0)
173 expected_bt = [(c, rooti)] + expected_bt
175 c, d = get_name(d, i, 1)
176 expected_bt = [(c, di)] + expected_bt
178 c, f = get_name(d, i, 2)
180 expected_bt = [(c, di)] + expected_bt
181 return fi, expected_bt
184 if len(sys.argv) > 1:
185 test = int(sys.argv[1])
188 if len(sys.argv) > 2:
191 radosobj = rados.Rados(conffile=conf)
193 ioctx = radosobj.open_ioctx('data')
195 ceph = cephfs.LibCephFS(conffile=conf)
198 rooti = ceph.stat('/')['st_ino']
201 if len(sys.argv) > 1:
202 test = int(sys.argv[1])
204 conf = '/etc/ceph/ceph.conf'
205 if len(sys.argv) > 2:
213 if test < 0 or test == i:
214 print('Running test %d: basic verify' % i, file=sys.stderr)
215 ino, expected_bt = make_abc(ceph, rooti, i)
216 ceph = flush(ceph, i)
217 verify(ioctx, ino, expected_bt, 0)
221 # kill-mds-at-openc-1
227 if test < 0 or test == i:
228 print('Running test %d: kill openc' % i, file=sys.stderr)
229 print("restart mds a")
231 kill_mds(ceph, 'openc', 1)
232 ino, expected_bt = make_abc(ceph, rooti, i)
233 ceph = flush(ceph, i)
234 verify(ioctx, ino, expected_bt, 0)
238 # kill-mds-at-openc-1
240 # restart-mds with kill-mds-at-replay-1
244 if test < 0 or test == i:
245 print('Running test %d: kill openc/replay' % i, file=sys.stderr)
246 # these are reversed because we want to prepare the config
247 conf_set_kill_mds('journal_replay', 1)
248 kill_mds(ceph, 'openc', 1)
249 print("restart mds a")
251 ino, expected_bt = make_abc(ceph, rooti, i)
252 ceph = flush(ceph, i)
253 verify(ioctx, ino, expected_bt, 0)