3 # Copyright (C) 2015 Red Hat <contact@redhat.com>
6 # Author: Kefu Chai <kchai@redhat.com>
8 # This program is free software; you can redistribute it and/or modify
9 # it under the terms of the GNU Library Public License as published by
10 # the Free Software Foundation; either version 2, or (at your option)
13 # This program is distributed in the hope that it will be useful,
14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 # GNU Library Public License for more details.
19 source $CEPH_ROOT/qa/standalone/ceph-helpers.sh
25 export CEPH_MON="127.0.0.1:7112" # git grep '\<7112\>' : there must be only one
27 CEPH_ARGS+="--fsid=$(uuidgen) --auth-supported=none "
28 CEPH_ARGS+="--mon-host=$CEPH_MON "
30 local funcs=${@:-$(set | sed -n -e 's/^\(TEST_[0-9a-z_]*\) .*/\1/p')}
31 for func in $funcs ; do
32 setup $dir || return 1
33 run_mon $dir a || return 1
34 run_mgr $dir x || return 1
35 create_rbd_pool || return 1
37 # check that erasure code plugins are preloaded
38 CEPH_ARGS='' ceph --admin-daemon $(get_asok_path mon.a) log flush || return 1
39 grep 'load: jerasure.*lrc' $dir/mon.a.log || return 1
40 $func $dir || return 1
41 teardown $dir || return 1
45 function setup_osds() {
46 for id in $(seq 0 3) ; do
47 run_osd $dir $id || return 1
49 wait_for_clean || return 1
51 # check that erasure code plugins are preloaded
52 CEPH_ARGS='' ceph --admin-daemon $(get_asok_path osd.0) log flush || return 1
53 grep 'load: jerasure.*lrc' $dir/osd.0.log || return 1
56 function create_erasure_coded_pool() {
59 ceph osd erasure-code-profile set myprofile \
62 crush-failure-domain=osd || return 1
63 create_pool $poolname 1 1 erasure myprofile \
65 wait_for_clean || return 1
68 function delete_pool() {
71 ceph osd pool delete $poolname $poolname --yes-i-really-really-mean-it
72 ceph osd erasure-code-profile rm myprofile
75 function rados_put() {
78 local objname=${3:-SOMETHING}
80 for marker in AAA BBB CCCC DDDD ; do
81 printf "%*s" 1024 $marker
84 # get and put an object, compare they are equal
86 rados --pool $poolname put $objname $dir/ORIGINAL || return 1
89 function rados_get() {
92 local objname=${3:-SOMETHING}
96 # Expect a failure to get object
98 if [ $expect = "fail" ];
100 ! rados --pool $poolname get $objname $dir/COPY
104 # get an object, compare with $dir/ORIGINAL
106 rados --pool $poolname get $objname $dir/COPY || return 1
107 diff $dir/ORIGINAL $dir/COPY || return 1
111 function rados_put_get() {
114 local objname=${3:-SOMETHING}
118 # get and put an object, compare they are equal
120 rados_put $dir $poolname $objname || return 1
121 # We can read even though caller injected read error on one of the shards
122 rados_get $dir $poolname $objname || return 1
124 if [ -n "$recovery" ];
127 # take out the last OSD used to store the object,
128 # bring it back, and check for clean PGs which means
129 # recovery didn't crash the primary.
131 local -a initial_osds=($(get_osds $poolname $objname))
132 local last=$((${#initial_osds[@]} - 1))
134 kill_daemons $dir TERM osd.${initial_osds[$last]} >&2 < /dev/null || return 1
135 ceph osd out ${initial_osds[$last]} || return 1
136 ! get_osds $poolname $objname | grep '\<'${initial_osds[$last]}'\>' || return 1
137 ceph osd in ${initial_osds[$last]} || return 1
138 run_osd $dir ${initial_osds[$last]} || return 1
139 wait_for_clean || return 1
145 function rados_get_data_eio() {
153 # inject eio to speificied shard
155 local poolname=pool-jerasure
156 local objname=obj-eio-$$-$shard_id
157 inject_eio ec data $poolname $objname $dir $shard_id || return 1
158 rados_put_get $dir $poolname $objname $recovery || return 1
160 shard_id=$(expr $shard_id + 1)
161 inject_eio ec data $poolname $objname $dir $shard_id || return 1
162 # Now 2 out of 3 shards get EIO, so should fail
163 rados_get $dir $poolname $objname fail || return 1
166 # Change the size of speificied shard
168 function set_size() {
179 local poolname=pool-jerasure
180 local -a initial_osds=($(get_osds $poolname $objname))
181 local osd_id=${initial_osds[$shard_id]}
183 if [ "$mode" = "add" ];
185 objectstore_tool $dir $osd_id $objname get-bytes $dir/CORRUPT || return 1
186 dd if=/dev/urandom bs=$bytes count=1 >> $dir/CORRUPT
187 elif [ "$bytes" = "0" ];
191 dd if=/dev/urandom bs=$bytes count=1 of=$dir/CORRUPT
193 objectstore_tool $dir $osd_id $objname set-bytes $dir/CORRUPT || return 1
198 function rados_get_data_bad_size() {
207 local poolname=pool-jerasure
208 local objname=obj-size-$$-$shard_id-$bytes
209 rados_put $dir $poolname $objname || return 1
211 # Change the size of speificied shard
213 set_size $objname $dir $shard_id $bytes $mode || return 1
215 rados_get $dir $poolname $objname || return 1
217 # Leave objname and modify another shard
218 shard_id=$(expr $shard_id + 1)
219 set_size $objname $dir $shard_id $bytes $mode || return 1
220 rados_get $dir $poolname $objname fail || return 1
224 # These two test cases try to validate the following behavior:
225 # For object on EC pool, if there is one shard having read error (
226 # either primary or replica), client can still read object.
228 # If 2 shards have read errors the client will get an error.
230 function TEST_rados_get_subread_eio_shard_0() {
232 setup_osds || return 1
234 local poolname=pool-jerasure
235 create_erasure_coded_pool $poolname || return 1
236 # inject eio on primary OSD (0) and replica OSD (1)
238 rados_get_data_eio $dir $shard_id || return 1
239 delete_pool $poolname
242 function TEST_rados_get_subread_eio_shard_1() {
244 setup_osds || return 1
246 local poolname=pool-jerasure
247 create_erasure_coded_pool $poolname || return 1
248 # inject eio into replicas OSD (1) and OSD (2)
250 rados_get_data_eio $dir $shard_id || return 1
251 delete_pool $poolname
255 # These two test cases try to validate that following behavior:
256 # For object on EC pool, if there is one shard which an incorrect
257 # size this will cause an internal read error, client can still read object.
259 # If 2 shards have incorrect size the client will get an error.
261 function TEST_rados_get_bad_size_shard_0() {
263 setup_osds || return 1
265 local poolname=pool-jerasure
266 create_erasure_coded_pool $poolname || return 1
267 # Set incorrect size into primary OSD (0) and replica OSD (1)
269 rados_get_data_bad_size $dir $shard_id 10 || return 1
270 rados_get_data_bad_size $dir $shard_id 0 || return 1
271 rados_get_data_bad_size $dir $shard_id 256 add || return 1
272 delete_pool $poolname
275 function TEST_rados_get_bad_size_shard_1() {
277 setup_osds || return 1
279 local poolname=pool-jerasure
280 create_erasure_coded_pool $poolname || return 1
281 # Set incorrect size into replicas OSD (1) and OSD (2)
283 rados_get_data_bad_size $dir $shard_id 10 || return 1
284 rados_get_data_bad_size $dir $shard_id 0 || return 1
285 rados_get_data_bad_size $dir $shard_id 256 add || return 1
286 delete_pool $poolname
289 function TEST_rados_get_with_subreadall_eio_shard_0() {
293 setup_osds || return 1
295 local poolname=pool-jerasure
296 create_erasure_coded_pool $poolname || return 1
297 # inject eio on primary OSD (0)
299 rados_get_data_eio $dir $shard_id recovery || return 1
301 delete_pool $poolname
304 function TEST_rados_get_with_subreadall_eio_shard_1() {
308 setup_osds || return 1
310 local poolname=pool-jerasure
311 create_erasure_coded_pool $poolname || return 1
312 # inject eio on replica OSD (1)
314 rados_get_data_eio $dir $shard_id recovery || return 1
316 delete_pool $poolname
319 main test-erasure-eio "$@"
322 # compile-command: "cd ../.. ; make -j4 && test/erasure-code/test-erasure-eio.sh"