--- /dev/null
+#!/bin/bash -x
+
+#
+# Test the lost object logic
+#
+
+# Includes
+source "`dirname $0`/test_common.sh"
+
+TEST_POOL=rbd
+
+# Functions
+setup() {
+ export CEPH_NUM_OSD=$1
+ vstart_config=$2
+
+ # Start ceph
+ ./stop.sh
+
+ # set recovery start to a really long time to ensure that we don't start recovery
+ ./vstart.sh -d -n -o "$vstart_config" || die "vstart failed"
+
+ # for exiting pools set size not greater than number of OSDs,
+ # so recovery from degraded ps is possible
+ local changed=0
+ for pool in `./ceph osd pool ls`; do
+ local size=`./ceph osd pool get ${pool} size | awk '{print $2}'`
+ if [ "${size}" -gt "${CEPH_NUM_OSD}" ]; then
+ ./ceph osd pool set ${pool} size ${CEPH_NUM_OSD}
+ changed=1
+ fi
+ done
+ if [ ${changed} -eq 1 ]; then
+ # XXX: When a pool has degraded pgs due to size greater than number
+ # of OSDs, after decreasing the size the recovery still could stuck
+ # and requires an additional kick.
+ ./ceph osd out 0
+ ./ceph osd in 0
+ fi
+
+ poll_cmd "./ceph health" HEALTH_OK 1 30
+}
+
+recovery1_impl() {
+ # Write lots and lots of objects
+ write_objects 1 1 200 4000 $TEST_POOL
+
+ # Take down osd1
+ stop_osd 1
+
+ # Continue writing a lot of objects
+ write_objects 2 2 200 4000 $TEST_POOL
+
+ # Bring up osd1
+ restart_osd 1
+
+ # Finish peering.
+ sleep 15
+
+ # Stop osd0.
+ # At this point we have peered, but *NOT* recovered.
+ # Objects should be lost.
+ stop_osd 0
+
+ poll_cmd "./ceph pg debug degraded_pgs_exist" TRUE 3 120
+ [ $? -eq 1 ] || die "Failed to see degraded PGs."
+ poll_cmd "./ceph pg debug unfound_objects_exist" TRUE 3 120
+ [ $? -eq 1 ] || die "Failed to see unfound objects."
+ echo "Got unfound objects."
+
+ restart_osd 0
+ sleep 20
+ start_recovery 2
+
+ # Turn on recovery and wait for it to complete.
+ poll_cmd "./ceph pg debug unfound_objects_exist" FALSE 3 120
+ [ $? -eq 1 ] || die "Failed to recover unfound objects."
+ poll_cmd "./ceph pg debug degraded_pgs_exist" FALSE 3 120
+ [ $? -eq 1 ] || die "Recovery never finished."
+}
+
+recovery1() {
+ setup 2 'osd recovery delay start = 10000'
+ recovery1_impl
+}
+
+lost1_impl() {
+ local flags="$@"
+ local lost_action=delete
+ local pgs_unfound pg
+
+ if is_set revert_lost $flags; then
+ lost_action=revert
+ fi
+
+ # Write lots and lots of objects
+ write_objects 1 1 20 8000 $TEST_POOL
+
+ # Take down osd1
+ stop_osd 1
+
+ # Continue writing a lot of objects
+ write_objects 2 2 20 8000 $TEST_POOL
+
+ # Bring up osd1
+ restart_osd 1
+
+ # Finish peering.
+ sleep 15
+
+ # Stop osd0.
+ # At this point we have peered, but *NOT* recovered.
+ # Objects should be lost.
+ stop_osd 0
+
+ # Since recovery can't proceed, stuff should be unfound.
+ poll_cmd "./ceph pg debug unfound_objects_exist" TRUE 3 120
+ [ $? -eq 1 ] || die "Failed to see unfound objects."
+
+ pgs_unfound=`./ceph health detail |awk '$1 = "pg" && /[0-9] unfound$/ {print $2}'`
+
+ [ -n "$pgs_unfound" ] || die "no pg with unfound objects"
+
+ for pg in $pgs_unfound; do
+ ./ceph pg $pg mark_unfound_lost revert &&
+ die "mark_unfound_lost unexpectedly succeeded for pg $pg"
+ done
+
+ if ! is_set mark_osd_lost $flags && ! is_set rm_osd $flags; then
+ return
+ fi
+
+ if is_set try_to_fetch_unfound $flags; then
+ # Ask for an object while it's still unfound, and
+ # verify we get woken to an error when it's declared lost.
+ echo "trying to get one of the unfound objects"
+ (
+ ./rados -c ./ceph.conf -p $TEST_POOL get obj02 $TEMPDIR/obj02 &&\
+ die "expected radostool error"
+ ) &
+ fi
+
+ if is_set mark_osd_lost $flags; then
+ ./ceph osd lost 0 --yes-i-really-mean-it
+ fi
+
+ if is_set rm_osd $flags; then
+ ./ceph osd rm 0
+ fi
+
+ if ! is_set auto_mark_unfound_lost $flags; then
+ for pg in $pgs_unfound; do
+ ./ceph pg $pg mark_unfound_lost ${lost_action} ||
+ die "mark_unfound_lost failed for pg $pg"
+ done
+ fi
+
+ start_recovery 2
+
+ # Unfound objects go away and are turned into lost objects.
+ poll_cmd "./ceph pg debug unfound_objects_exist" FALSE 3 120
+ [ $? -eq 1 ] || die "Unfound objects didn't go away."
+
+ for pg in `ceph pg ls | awk '/^[0-9]/ {print $1}'`; do
+ ./ceph pg $pg mark_unfound_lost revert 2>&1 |
+ grep 'pg has no unfound objects' ||
+ die "pg $pg has unfound objects"
+ done
+
+ # Reading from a lost object gives back an error code.
+ # TODO: check error code
+ ./rados -c ./ceph.conf -p $TEST_POOL get obj01 $TEMPDIR/obj01
+ if [ lost_action = delete -a $? -eq 0 ]; then
+ die "expected radostool error"
+ elif [ lost_action = revert -a $? -ne 0 ]; then
+ die "unexpected radostool error"
+ fi
+
+ if is_set try_to_fetch_unfound $flags; then
+ echo "waiting for the try_to_fetch_unfound \
+radostool instance to finish"
+ wait
+ fi
+}
+
+lost1() {
+ setup 2 'osd recovery delay start = 10000'
+ lost1_impl mark_osd_lost revert_lost
+}
+
+lost2() {
+ setup 2 'osd recovery delay start = 10000'
+ lost1_impl mark_osd_lost try_to_fetch_unfound
+}
+
+lost3() {
+ setup 2 'osd recovery delay start = 10000'
+ lost1_impl rm_osd
+}
+
+lost4() {
+ setup 2 'osd recovery delay start = 10000'
+ lost1_impl mark_osd_lost rm_osd
+}
+
+lost5() {
+ setup 2 'osd recovery delay start = 10000'
+ lost1_impl mark_osd_lost auto_mark_unfound_lost
+}
+
+all_osds_die_impl() {
+ poll_cmd "./ceph osd stat" '3 up, 3 in' 20 240
+ [ $? -eq 1 ] || die "didn't start 3 osds"
+
+ stop_osd 0
+ stop_osd 1
+ stop_osd 2
+
+ # wait for the MOSDPGStat timeout
+ poll_cmd "./ceph osd stat" '0 up' 20 240
+ [ $? -eq 1 ] || die "all osds weren't marked as down"
+}
+
+all_osds_die() {
+ setup 3 'osd mon report interval max = 60
+ osd mon report interval min = 3
+ mon osd report timeout = 60'
+
+ all_osds_die_impl
+}
+
+run() {
+ recovery1 || die "test failed"
+
+ lost1 || die "test failed"
+
+ # XXX: try_to_fetch_unfound test currently hangs on "waiting for the
+ # try_to_fetch_unfound radostool instance to finish"
+ #lost2 || die "test failed"
+
+ lost3 || die "test failed"
+
+ lost4 || die "test failed"
+
+ # XXX: automatically marking lost is not implemented
+ #lost5 || die "test failed"
+
+ all_osds_die || die "test failed"
+}
+
+if [ -z "$@" ]; then
+ run
+ echo OK
+ exit 0
+fi
+
+$@
Code Review / stor4nfv.git / blobdiff