3 # Copyright (C) 2014 Cloudwatt <libre.licensing@cloudwatt.com>
4 # Copyright (C) 2014, 2015, 2016, 2017 Red Hat <contact@redhat.com>
6 # Author: Loic Dachary <loic@dachary.org>
8 # This program is free software; you can redistribute it and/or modify
9 # it under the terms of the GNU Library Public License as published by
10 # the Free Software Foundation; either version 2, or (at your option)
13 # This program is distributed in the hope that it will be useful,
14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 # GNU Library Public License for more details.
19 # ceph-disk.sh is launched by tox which expects tox.ini in current
20 # directory. so we cannot run ceph-disk.sh in build directory directly,
21 # and hence not able to use detect-build-env-vars.sh to set the build
23 if [ -z "$CEPH_ROOT" ] || [ -z "$CEPH_BIN" ] || [ -z "$CEPH_LIB" ]; then
24 CEPH_ROOT=`readlink -f $(dirname $0)/../../..`
26 CEPH_LIB=$CEPH_ROOT/.libs
28 source $CEPH_ROOT/qa/standalone/ceph-helpers.sh
32 PS4='${BASH_SOURCE[0]}:$LINENO: ${FUNCNAME[0]}: '
34 export PATH=$CEPH_BIN:.:$PATH # make sure program from sources are preferred
35 export LD_LIBRARY_PATH=$CEPH_LIB
36 : ${CEPH_DISK:=ceph-disk}
38 CEPH_DISK_ARGS+=" --verbose"
39 CEPH_DISK_ARGS+=" --prepend-to-path="
40 : ${CEPH_DISK_TIMEOUT:=360}
41 if [ `uname` != FreeBSD ]; then
44 PROCDIR="/compat/linux"
51 uuidgen=$(which uuidgen)
52 if [ `uname` = FreeBSD ]; then
53 # for unknown reasons FreeBSD timeout does not return sometimes
56 timeout="$(which timeout) $CEPH_DISK_TIMEOUT"
63 mkdir -p $(get_asok_dir)
64 touch $dir/ceph.conf # so ceph-disk think ceph is the cluster
69 if ! test -e $dir ; then
73 if [ `uname` != FreeBSD ] && \
74 [ $(stat -f -c '%T' .) == "btrfs" ]; then
78 grep " $(pwd)/$dir/" < ${PROCDIR}/proc/mounts | while read mounted rest ; do
82 rm -rf $(get_asok_dir)
85 function command_fixture() {
90 local fpath=`readlink -f $(which $command)`
91 [ "$fpath" = `readlink -f $CEPH_BIN/$command` ] || [ "$fpath" = `readlink -f $(pwd)/$command` ] || return 1
93 cat > $dir/$command <<EOF
95 touch $dir/used-$command
96 exec $CEPH_BIN/$command "\$@"
98 chmod +x $dir/$command
101 function tweak_path() {
109 command_fixture $dir ceph-conf || return 1
110 command_fixture $dir ceph-osd || return 1
112 test_activate_dir $dir || return 1
114 [ ! -f $dir/used-ceph-conf ] || return 1
115 [ ! -f $dir/used-ceph-osd ] || return 1
121 command_fixture $dir ceph-conf || return 1
122 command_fixture $dir ceph-osd || return 1
124 $tweaker $dir test_activate_dir || return 1
126 [ -f $dir/used-ceph-osd ] || return 1
131 function use_prepend_to_path() {
136 ceph_disk_args+=" --statedir=$dir"
137 ceph_disk_args+=" --sysconfdir=$dir"
138 ceph_disk_args+=" --prepend-to-path=$dir"
139 ceph_disk_args+=" --verbose"
140 CEPH_DISK_ARGS="$ceph_disk_args" \
141 "$@" $dir || return 1
144 function test_prepend_to_path() {
147 tweak_path $dir use_prepend_to_path || return 1
150 function use_path() {
154 "$@" $dir || return 1
157 function test_path() {
160 tweak_path $dir use_path || return 1
163 function test_no_path() {
166 ( export PATH=../ceph-detect-init/virtualenv/bin:virtualenv/bin:$CEPH_BIN:/usr/bin:/bin:/usr/local/bin ; test_activate_dir $dir) || return 1
169 function test_mark_init() {
176 local osd_data=$dir/dir
179 local osd_uuid=$($uuidgen)
183 ${CEPH_DISK} $CEPH_DISK_ARGS \
184 prepare --filestore --osd-uuid $osd_uuid $osd_data || return 1
186 ${CEPH_DISK} $CEPH_DISK_ARGS \
191 $osd_data || return 1
193 test -f $osd_data/$(ceph-detect-init) || return 1
195 if test systemd = $(ceph-detect-init) ; then
200 $timeout ${CEPH_DISK} $CEPH_DISK_ARGS \
203 --mark-init=$expected \
205 $osd_data || return 1
207 ! test -f $osd_data/$(ceph-detect-init) || return 1
208 test -f $osd_data/$expected || return 1
211 function test_zap() {
213 local osd_data=$dir/dir
216 ${CEPH_DISK} $CEPH_DISK_ARGS zap $osd_data 2>&1 | grep -q 'not full block device' || return 1
221 # ceph-disk prepare returns immediately on success if the magic file
222 # exists in the --osd-data directory.
223 function test_activate_dir_magic() {
225 local uuid=$($uuidgen)
226 local osd_data=$dir/osd
228 echo a failure to create the fsid file implies the magic file is not created
230 mkdir -p $osd_data/fsid
231 CEPH_ARGS="--fsid $uuid" \
232 ${CEPH_DISK} $CEPH_DISK_ARGS prepare --filestore $osd_data > $dir/out 2>&1
233 grep --quiet 'Is a directory' $dir/out || return 1
234 ! [ -f $osd_data/magic ] || return 1
237 echo successfully prepare the OSD
239 CEPH_ARGS="--fsid $uuid" \
240 ${CEPH_DISK} $CEPH_DISK_ARGS prepare --filestore $osd_data 2>&1 | tee $dir/out
241 grep --quiet 'Preparing osd data dir' $dir/out || return 1
242 grep --quiet $uuid $osd_data/ceph_fsid || return 1
243 [ -f $osd_data/magic ] || return 1
245 echo will not override an existing OSD
247 CEPH_ARGS="--fsid $($uuidgen)" \
248 ${CEPH_DISK} $CEPH_DISK_ARGS prepare --filestore $osd_data 2>&1 | tee $dir/out
249 grep --quiet 'Data dir .* already exists' $dir/out || return 1
250 grep --quiet $uuid $osd_data/ceph_fsid || return 1
253 function read_write() {
255 local file=${2:-$(uuidgen)}
258 echo FOO > $dir/$file
259 $timeout rados --pool $pool put $file $dir/$file || return 1
260 $timeout rados --pool $pool get $file $dir/$file.copy || return 1
261 $diff $dir/$file $dir/$file.copy || return 1
264 function test_pool_read_write() {
268 $timeout ceph osd pool set $pool size 1 || return 1
269 read_write $dir || return 1
272 function test_activate() {
280 ${CEPH_DISK} $CEPH_DISK_ARGS \
281 prepare --filestore "$@" $osd_data || return 1
283 $timeout ${CEPH_DISK} $CEPH_DISK_ARGS \
286 $osd_data || return 1
288 test_pool_read_write $dir || return 1
291 function test_reuse_osd_id() {
294 run_mon $dir a || return 1
295 run_mgr $dir x || return 1
298 test_activate $dir $dir/dir1 --osd-uuid $(uuidgen) || return 1
301 # add a new OSD with a given OSD id (13)
303 local osd_uuid=$($uuidgen)
305 test_activate $dir $dir/dir2 --osd-id $osd_id --osd-uuid $osd_uuid || return 1
306 test $osd_id = $(ceph osd new $osd_uuid) || return 1
309 # make sure the OSD is in use by the PGs
311 wait_osd_id_used_by_pgs $osd_id $PG_NUM || return 1
312 read_write $dir SOMETHING || return 1
315 # set the OSD out and verify it is no longer used by the PGs
317 ceph osd out osd.$osd_id || return 1
318 wait_osd_id_used_by_pgs $osd_id 0 || return 1
321 # kill the OSD and destroy it (do not purge, retain its place in the crushmap)
323 kill_daemons $dir TERM osd.$osd_id || return 1
324 ceph osd destroy osd.$osd_id --yes-i-really-mean-it || return 1
327 # add a new OSD with the same id as the destroyed OSD
330 test_activate $dir $dir/dir3 --osd-id $osd_id --osd-uuid $osd_uuid || return 1
331 test $osd_id = $(ceph osd new $osd_uuid) || return 1
334 function test_activate_dir() {
338 run_mon $dir a || return 1
339 run_mgr $dir x || return 1
343 test_activate $dir $dir/dir || return 1
346 function test_activate_dir_bluestore() {
349 run_mon $dir a || return 1
350 run_mgr $dir x || return 1
353 local osd_data=$dir/dir
355 local to_prepare=$osd_data
356 local to_activate=$osd_data
357 local osd_uuid=$($uuidgen)
359 CEPH_ARGS=" --bluestore-block-size=10737418240 $CEPH_ARGS" \
360 ${CEPH_DISK} $CEPH_DISK_ARGS \
361 prepare --bluestore --block-file --osd-uuid $osd_uuid $to_prepare || return 1
363 CEPH_ARGS=" --osd-objectstore=bluestore --bluestore-fsck-on-mount=true --bluestore-block-db-size=67108864 --bluestore-block-wal-size=134217728 --bluestore-block-size=10737418240 $CEPH_ARGS" \
364 $timeout ${CEPH_DISK} $CEPH_DISK_ARGS \
367 $to_activate || return 1
369 test_pool_read_write $dir || return 1
372 function test_find_cluster_by_uuid() {
374 test_activate_dir $dir 2>&1 | tee $dir/test_find
375 ! grep "No cluster conf found in $dir" $dir/test_find || return 1
379 test_activate_dir $dir "rm $dir/ceph.conf" > $dir/test_find 2>&1
380 cp $dir/test_find /tmp
381 grep --quiet "No cluster conf found in $dir" $dir/test_find || return 1
384 # http://tracker.ceph.com/issues/9653
385 function test_keyring_path() {
387 test_activate_dir $dir 2>&1 | tee $dir/test_keyring
388 grep --quiet "keyring $dir/bootstrap-osd/ceph.keyring" $dir/test_keyring || return 1
391 function test_crush_device_class() {
398 local osd_data=$dir/dir
401 local osd_uuid=$($uuidgen)
405 ${CEPH_DISK} $CEPH_DISK_ARGS \
406 prepare --filestore --osd-uuid $osd_uuid \
407 --crush-device-class CRUSH_CLASS \
408 $osd_data || return 1
409 test -f $osd_data/crush_device_class || return 1
410 test $(cat $osd_data/crush_device_class) = CRUSH_CLASS || return 1
412 CEPH_ARGS="--crush-location=root=default $CEPH_ARGS" \
413 ${CEPH_DISK} $CEPH_DISK_ARGS \
417 $osd_data || return 1
420 for delay in 2 4 8 16 32 64 128 256 ; do
421 if ceph osd crush dump | grep --quiet 'CRUSH_CLASS' ; then
426 ceph osd crush dump # for debugging purposes
434 CEPH_DISK_ARGS+=" --statedir=$dir"
435 CEPH_DISK_ARGS+=" --sysconfdir=$dir"
437 export CEPH_MON="127.0.0.1:7451" # git grep '\<7451\>' : there must be only one
439 CEPH_ARGS+=" --fsid=$(uuidgen)"
440 CEPH_ARGS+=" --auth-supported=none"
441 CEPH_ARGS+=" --mon-host=$CEPH_MON"
442 CEPH_ARGS+=" --chdir="
443 CEPH_ARGS+=" --journal-dio=false"
444 CEPH_ARGS+=" --erasure-code-dir=$CEPH_LIB"
445 CEPH_ARGS+=" --plugin-dir=$CEPH_LIB"
446 CEPH_ARGS+=" --log-file=$dir/\$name.log"
447 CEPH_ARGS+=" --pid-file=$dir/\$name.pidfile"
448 CEPH_ARGS+=" --osd-class-dir=$CEPH_LIB"
449 CEPH_ARGS+=" --run-dir=$dir"
450 CEPH_ARGS+=" --osd-failsafe-full-ratio=.99"
451 CEPH_ARGS+=" --osd-journal-size=100"
452 CEPH_ARGS+=" --debug-osd=20"
453 CEPH_ARGS+=" --debug-bdev=20"
454 CEPH_ARGS+=" --debug-bluestore=20"
455 CEPH_ARGS+=" --osd-max-object-name-len=460"
456 CEPH_ARGS+=" --osd-max-object-namespace-len=64 "
457 local default_actions
458 default_actions+="test_path "
459 default_actions+="test_no_path "
460 default_actions+="test_find_cluster_by_uuid "
461 default_actions+="test_prepend_to_path "
462 default_actions+="test_activate_dir_magic "
463 default_actions+="test_activate_dir "
464 default_actions+="test_keyring_path "
465 [ `uname` != FreeBSD ] && \
466 default_actions+="test_mark_init "
467 default_actions+="test_zap "
468 [ `uname` != FreeBSD ] && \
469 default_actions+="test_activate_dir_bluestore "
470 default_actions+="test_crush_device_class "
471 default_actions+="test_reuse_osd_id "
472 local actions=${@:-$default_actions}
473 for action in $actions ; do
474 setup $dir || return 1
476 $action $dir || return 1
478 teardown $dir || return 1
482 main test-ceph-disk "$@"
485 # compile-command: "cd .. ; test/ceph-disk.sh # test_activate_dir"