4 # Copyright (C) 2015 Ubuntu Kylin
6 # Author: Min Chen <minchen@ubuntukylin.com>
8 # This program is free software; you can redistribute it and/or modify
9 # it under the terms of the GNU Library Public License as published by
10 # the Free Software Foundation; either version 2, or (at your option)
13 # This program is distributed in the hope that it will be useful,
14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 # GNU Library Public License for more details.
19 my_dir=$(dirname "$0")
32 #init osd_data and get all objects path
33 function gen_database()
35 local func="gen_database"
43 trap 'echo $func failed; exit;' INT HUP
47 host=`echo $line|awk '{print $1}'`
48 data_path=`echo $line|awk '{print $2}'`
49 if [ "$host"x = ""x ] || [ "$data_path"x = ""x ];then
52 local cmds="find $data_path/current -type f"
53 ssh $ssh_option $host $cmds > $database/$host
60 # collect hobjects from database
61 # and choose the object whose epoch is latest
62 # then, sort the objects by their offsets in image
63 function gather_hobject_common()
65 func="gather_hobject_common"
67 trap 'echo $func failed; exit;' INT HUP
69 echo "$func: parameters: <pool_id> <image_prefix> [<snap_id>]"
76 local hex_pool_id=`printf "%x" $pool_id`
78 local snap_id=`printf "%u" -2`
79 local hex_snap_id="head"
84 hex_snap_id=`printf "%x" $snap_id`
88 local underline_image_prefix=`convert_underline $image_prefix`
89 local dump_image_prefix=`dump_backslash $underline_image_prefix`
90 local ddump_image_prefix=`dump_dump_backslash $underline_image_prefix`
91 local images_raw_dir=$rbd_image/raw
92 local image_hobjects_dir=$images/pool_$pool_id/$image_prefix
93 # $images/raw/$image_prefix"_head"
94 local image_hobjects_raw=$images_raw_dir/$image_prefix"$fsuffix"
95 # $images/$image_prefix/$image_prefix"_head"
96 local image_hobjects_stable=$image_hobjects_dir/$image_prefix"$fsuffix"
98 if [ ! -e $images_raw_dir ];then
99 mkdir -p $images_raw_dir
101 if [ ! -e $image_hobjects_dir ];then
102 local image_metadata=$images_meta/$image_name_in
103 mkdir -p $image_hobjects_dir
106 pushd $database >/dev/null
107 local pattern="\.[0-9a-f]+__"$hex_snap_id"_[0-9A-F]{8}__"$hex_pool_id
109 grep -r -E $dump_image_prefix""$pattern * >$image_hobjects_raw
110 if [ ! -s $image_hobjects_raw ];then
111 echo "$func: image snap [ $image_prefix"$psuffix" ] is empty"
112 return 1 #no data available
116 local offset_dir_temp=$images_raw_dir/$image_prefix"$fsuffix""_dir_temp"
117 rm -rf $offset_dir_temp
118 mkdir -p $offset_dir_temp
120 echo "gather hobjects from database: snapid=$snap_id ..."
122 # format: ceph2:/var/lib/ceph/osd/ceph-1/current/2.d3_head/rb.0.1293.6b8b4567.000000000002__head_FB425CD3__2
123 local tmp_image=$offset_dir_temp/tmpimage.$$$$
125 cat $image_hobjects_raw |
129 tmp_image="'$tmp_image'"
130 osd_host_mapping="'$osd_host_mapping'"
133 # $2 = /var/lib/ceph/osd/ceph-1/current/2.d3_head/rb.0.1293.6b8b4567.000000000002__head_FB425CD3__2
135 split($2, arr1, "/current/"); # {/var/lib/ceph/osd/ceph-1/, 2.d3_head/rb.0.1293.6b8b4567.000000000002__head_FB425CD3__2}
136 split(arr1[2], arr2, "/"); # {2.d3_head, rb.0.1293.6b8b4567.000000000002__head_FB425CD3__2}
137 split(arr2[1], arr3, "_head"); # {2.d3,}
141 gsub(/\\u/, "\\\\\\\\u", hobject); # dump backslash to delay escape (\ -> \\)
142 "awk \"\\$1 == \\\""$1"\\\" {print \\$2}\" "osd_host_mapping" | head -n 1" | getline node
146 offset_hobject=arr2[len] # rb.0.1293.6b8b4567.000000000002__head_FB425CD3__2
147 split(offset_hobject, offarr1, "."); # {rb, 0, 1293, 6b8b4567, 000000000002__head_FB425CD3__2}
149 offset_p=offarr1[len1] # 000000000002__head_FB425CD3__2
150 split(offset_p, offarr2, "__"); # {000000000002, head_FB425CD3, 2}
151 offset=offarr2[1]; # 000000000002
153 system("echo -n \""node" "pgid" "hobject" "offset" "snapid" \" >>"tmp_image);
154 #system("echo -n \""node" "pgid" "hobject" "offset" "snapid" \"");
155 #print node" "pgid" "hobject" "offset" "snapid
157 # find pg_epoch from pg_coll database
158 system("awk \"\\$1 == \\\""node"\\\" && \\$2 == \\\""pgid"\\\" && \\$4 == \\\""data_path"\\\" {print \\$3}\" "pg_coll" >>"tmp_image);
159 #system("awk \"\\$1 == \\\""node"\\\" && \\$2 == \\\""pgid"\\\" && \\$4 == \\\""data_path"\\\" {print \\$3}\" "pg_coll);
162 local sort_image=$offset_dir_temp/sortimage.$$$$
164 sort -t ' ' -k 4.1,4 -k 6.1nr -k 1.1,1 $tmp_image >$sort_image
165 sort -t ' ' -k 4.1,4 -u $sort_image > $image_hobjects_stable
167 #rm -rf $offset_dir_temp
171 function gather_hobject_nosnap()
173 gather_hobject_common $1 $2
176 function gather_hobject_snap()
178 gather_hobject_common $1 $2 $3
181 # select the max pg_epoch item of the same $field
182 # if no same $field, choose the first
183 # format : "node $field pg_epoch"
184 function choose_epoch()
186 cat $1|sort -t ' ' -k 3.1,3nr -k 2.1,2n |head -n 1;
189 # lookup image info , after scatter_node_jobs & gather_node_infos
190 function lookup_image()
192 local func="lookup_image"
194 echo "$func: parameters error <pool_id> <image_name> [<snap_name>]"
200 echo -e "$func: pool_id = $pool_id\timage_name = $image_name\tsnap_name = $snap_name"
201 if [ $pool_id -lt 0 ];then
202 echo "$func: pool_id must great than zero"
205 local hex_pool_id=`printf "%x" $pool_id`
206 input_image $image_name
208 local item=/tmp/item.$$$$
209 local img_name=`dump_backslash $image_name`
212 local image_id_hobject=
213 local image_header_hobject=
214 local result=/tmp/tmp_result.$$$$
215 local res1=/tmp/tmp_res1.$$$$
216 local res2=/tmp/tmp_res2.$$$$
221 cat $image_coll_v1|grep -E "/$img_name\.rbd__head_[0-9A-F]{8}__$hex_pool_id" >$res1
223 echo -n "$func: rbd_header_hobject = "
224 choose_epoch $res1| tee $item
225 #choose_epoch $res1 > $item
228 node=`cat $item|awk '{print $1}'`
229 image_header_hobject=`cat $item|awk '{print $2}'`
230 if [ "$node"x = ""x ];then
231 echo "$func: v1 node is NULL"
234 if [ "$image_header_hobject"x = ""x ];then
235 echo "$func: v1 image_header_hobject is NULL"
242 echo -e "image_name:\t$image_name_in"
243 echo -e "image_format:\t$image_format"
244 data_path=`echo $image_header_hobject|awk -F "/current" '{print $1}'`
247 cmds="bash $job_path/osd_job do_image_metadata_v1 $data_path `dump_backslash $image_header_hobject` $snap_name"
248 ssh $ssh_option $node $cmds | tee $result
254 cat $image_coll_v2|grep -E "/rbd\\\\uid\."$img_name"__head_[0-9A-F]{8}__$hex_pool_id" >$res2
256 echo -n "$func: rbd_id_hobject = "
257 choose_epoch $res2 | tee $item
258 #choose_epoch $res2 > $item
261 node=`cat $item|awk '{print $1}'`
262 image_id_hobject=`cat $item|awk '{print $2}'`
263 if [ "$node"x = ""x ];then
264 echo "$func: v2 node is NULL(to get image_id_hobject)"
267 if [ "$image_id_hobject"x = ""x ];then
268 echo "$func: v2 image_id_hobject is NULL"
274 check_osd_process $node
277 local tid=/tmp/image_id.$$$$
278 data_path=`echo $image_id_hobject|awk -F "/current" '{print $1}'`
280 cmds="bash $job_path/osd_job do_image_id $data_path `dump_backslash $image_id_hobject`"
281 ssh $ssh_option $node $cmds > $tid
283 local image_id=`cat $tid`
286 #get image_header_hobject
287 pushd $database >/dev/null
288 local pattern="header\."$image_id"__head_[0-9A-F]{8}__$hex_pool_id"
289 local tcoll=/tmp/tmp_image_head_coll.$$$$
291 # hostname(by command hostname) in $pg_coll maybe different from hostname in tcoll(input by user)
292 # t_host: hostname read from config file ($tcoll)
293 # t_host_remote: $(hostname) on osd node ($pg_coll)
294 grep -r -E $pattern * >$tcoll
297 local t_host=(`cat $tcoll|awk -F ":" '{print $1}'`)
298 local t_pgid=(`cat $tcoll|awk -F ":" '{print $2}'|sed -n 's/.*\/\([0-9a-fA-F]\+\.[0-9a-fA-F]\+\)_head\/.*/\1/p'`)
299 local t_hobject=(`cat $tcoll|awk -F ":" '{print $2}'`)
300 local t_data_path=(`cat $tcoll|awk -F ":" '{split($2, arr, "/current/"); print arr[1];}'`)
302 declare -a t_host_remote
304 #if there is no failed pg migration, number of t_host is replica num
305 #replica num : 3, 4, 5 ...
306 local t_hostname=/tmp/t_hostname.$$$$
307 for ((i=0; i<${#t_host[*]}; i++))
309 ssh $ssh_option ${t_host[$i]} "hostname" >$t_hostname
311 echo "$func: ${t_host[$i]} get host_remote failed"
314 t_host_remote[$i]=`cat $t_hostname`
318 local t_item=/tmp/tmp_item.$$$$
319 local tmp_item=/tmp/tmp_tmp_item.$$$$
322 for ((i=0; i<${#t_host_remote[*]}; i++ ))
324 local node=${t_host_remote[$i]}
325 local pgid=${t_pgid[$i]}
326 awk '$1 == "'"$node"'" && $2 == "'"$pgid"'" {print}' $pg_coll >>$tmp_item
329 # t_item: <remote_hostname> <pgid> <epoch> <data_path>
330 sort -u $tmp_item >$t_item
333 local entry=`choose_epoch $t_item` #t_host_remote
336 node=`echo $entry|awk '{print $1}'`
337 data_path=`echo $entry|awk '{print $4}'`
338 if [ "$node"x = ""x ];then
339 echo "$func: v2 node is NULL (to get image_header_hobject)"
343 for ((i=0; i<${#t_host_remote[*]}; i++))
345 if [ "${t_host_remote[$i]}"x = "$node"x ] && [ "${t_data_path[$i]}"x = "$data_path"x ];then
346 image_header_hobject=${t_hobject[$i]}
351 if [ "$image_id_hobject"x = ""x ];then
352 echo "$func: v2 image_header_hobject is NULL"
356 check_osd_process $node
358 echo "$func: rbd_header_hobject = $node $image_header_hobject"
359 echo -e "image_name:\t$image_name_in"
360 echo -e "image_format:\t$image_format"
362 #data_path=`echo $image_header_hobject|awk -F "/current" '{print $1}'`
364 cmds="bash $job_path/osd_job do_image_metadata_v2 $data_path $image_id `dump_backslash $image_header_hobject` $snap_name"
365 ssh $ssh_option $node $cmds | tee $result
369 if [ ! -s $result ];then
370 echo "$func: $image_name_in not exists"
374 # to assign value to global variable
375 db_image_prefix=`cat $result|awk '/^(object_prefix|block_name):/{print $2}'`
376 if [ "$db_image_prefix"x = ""x ];then
377 echo "$func: image_prefix is NULL"
381 db_image_size=`cat $result|awk '/^image_size:/{print $2}'`
382 db_order=`cat $result|awk '/^order:/{print $2}'`
383 if [ "$snap_name"x != ""x ];then
384 db_snap_id=`cat $result|awk '/^snapshot:/{print $2}'`
385 if [ "$db_snap_id"x = ""x ];then
386 echo "$func: $image_name_in@$snap_name NOT EXISTS"
389 db_snap_image_size=`cat $result|awk '/^snapshot:/{print $4}'`
392 local image_snaplist=$images/pool_$pool_id/$image_name_in/@snaplist
393 local image_dir=$images/pool_$pool_id/$image_name_in
394 if [ ! -e $image_dir ];then
397 cat $result|awk '/^snapshot:/{print $2" "$3" "$4}' >$image_snaplist
403 function list_images()
405 echo "=============== format =============="
406 echo "format: <pool_id>/<image_name>"
407 echo "================ v1: ================"
408 #sed -n 's/\(.*\)\/\(.*\)\.rbd__\(.*\)/\2/p' $image_coll_v1|sort -u|sed -e 's/\\u/_/g'
409 sed -n 's/.*\/\(.*\)\.rbd__head_[0-9A-F]\{8\}__\([0-9a-f]\+\).*/\2 \1/p' $image_coll_v1|sort -u|awk '{print strtonum("0x"$1)"/"$2;}'|sed -e 's/\\u/_/g'
410 echo "================ v2: ================"
411 #sed -n 's/\(.*\)\/rbd\\uid.\(.*\)__\(head.*\)/\2/p' $image_coll_v2|sort -u|sed 's/\\u/_/g'
412 sed -n 's/.*\/rbd\\uid.\(.*\)__head_[0-9A-F]\{8\}__\([0-9a-f]\+\).*/\2 \1/p' $image_coll_v2|sort -u|awk '{print strtonum("0x"$1)"/"$2}'|sed 's/\\u/_/g'
415 # lookup image metadata
417 # collect hobjects of image with the latest pg epoch
418 function discover_image_nosnap()
420 local func="discover_image_nosnap"
424 pool_id=$(($pool_id))
425 lookup_image $pool_id $image_name # assign $image_prefix
426 gather_hobject_nosnap $pool_id $db_image_prefix
430 local image_hobjects_stable_nosnap=$images/pool_$pool_id/$db_image_prefix/$db_image_prefix"_head"
431 local image_hobjects_dir=$images/pool_$pool_id/$image_name_in
432 if [ ! -e $image_hobjects_dir ];then
433 mkdir -p $image_hobjects_dir
435 # mv image_prefix to image_name
436 mv $image_hobjects_stable_nosnap $image_hobjects_dir/$image_name_in
437 rm -rf $images/pool_$pool_id/$db_image_prefix
440 # get the offset snapid object
441 # if there is no object, choose the smallest snapid which is greater than current snapid
442 function get_object_clone()
444 local func="get_object_clone"
449 local object_offset_string=$1
451 local snaplist_path=$3
452 local snapset_output_dir=$4
455 local snap_coll_arr=(`
456 cat $snaplist_path|awk '{ if ($1 >= '"$snapid"') print "'"$snapset_output_dir"'/@"$1}'`)
458 local hex_snapid=`printf "%x" $snapid`
459 pushd $snapset_output_dir >/dev/null
460 # get object with the smallest snapid greater than current snapid
461 awk '$4 == "'"$object_offset_string"'" && $5 >= '$snapid' {print}' `echo ${snap_coll_arr[@]}` |tail -n 1
465 # gather hobject for each snapid
466 function gen_snapset_hobject()
468 local func="gen_image_snapset"
471 echo "$func: parameters: <pool_id> <image_prefix> <snaplist_path> <snapset_output_dir>"
475 local image_prefix=$2
476 local snaplist_path=$3
477 local snapset_output_dir=$4
478 pool_id=$(($pool_id))
481 local snaparr=(`cat $snaplist_path`)
482 # gather hobject for each snapshot
483 trap 'echo $func failed; exit;' INT HUP
484 for line in ${snaparr[@]}
488 local field=(`echo $line`)
489 local snapid=${field[0]}
490 local image_hobjects_stable_snap=$images/pool_$pool_id/$image_prefix/$image_prefix"_"$snapid
491 local image_snap=$snapset_output_dir/@$snapid
492 gather_hobject_snap $pool_id $image_prefix $snapid
494 if [ $res -ne 0 ];then
497 mv $image_hobjects_stable_snap $image_snap
504 # lookup image metadata and get snapid hobjects
505 function discover_image_snap()
507 local func="discover_image_snap"
510 echo "$func: paramters: <pool_id> <image_name> [<snap_name>]"
516 pool_id=$(($pool_id))
517 #mkdir -p $images/$image_prefix
518 lookup_image $pool_id $image_name $snap_name # input image_name and snap_name to lookup metadata and snap_id
519 if [ "$db_snap_id"x = ""x ];then
520 echo "$func: lookup image failed to gen snapid"
523 local image_hobjects_dir_prefix=$images/pool_$pool_id/$db_image_prefix
524 local image_nosnap=$images/pool_$pool_id/$image_name_in
525 #check if image nosnap recovered
526 if [ ! -s $image_nosnap ];then
527 echo "$func: please recover image nosnap before recover with snap"
528 rm -rf $image_hobjects_dir_prefix
531 local image_hobject_dir=$images/pool_$pool_id/$image_name_in
532 local image_snap_hobject=$image_hobject_dir/$image_name_in@$db_snap_id
533 local image_snap_hobject_head=$image_hobject_dir/$image_name_in@$db_snap_id@head
534 local image_snaplist=$image_hobject_dir/@snaplist
535 local image_snapset_dir=$image_hobject_dir/@snapset_dir
536 local image_head=$image_hobject_dir/$image_name_in
537 if [ ! -e $image_hobject_dir ];then
538 mkdir -p $image_hobject_dir
540 # only gen snapset one time
541 if [ ! -e $image_snapset_dir ];then
542 mkdir -p $image_snapset_dir
543 gen_snapset_hobject $pool_id $db_image_prefix $image_snaplist $image_snapset_dir
547 echo "$func: will get object clone ..."
549 >$image_snap_hobject_head
551 trap 'echo $func failed; exit;' INT HUP
552 # get each offset 's snapid hobject
558 local field=(`echo $line`)
559 local offset_string=${field[3]}
561 local entry=`get_object_clone $offset_string $db_snap_id $image_snaplist $image_snapset_dir`
562 if [ "$entry"x != ""x ];then
563 echo $entry >> $image_snap_hobject
564 echo `dump_backslash $line` >> $image_snap_hobject_head
567 rm -rf $image_hobjects_dir_prefix
570 # after discover_image_nosnap
571 # collect objects from osds one by one in sequence
572 function copy_image_nosnap_single_thread()
574 local func="copy_image_nosnap_single_thread"
577 echo "$func: parameters: <pool_id> <image_hobjects> <backup_dir>"
581 local image_hobjects=$2
583 pool_id=$(($pool_id))
585 # make sure lookup_image first
586 if [ $found = 0 ];then
587 echo "$func: image not found, maybe forget to discover_image"
590 if [ ! -e $backup_dir ];then
594 local image_dir=$backup_dir/pool_$pool_id/$image_name_in
595 local image_file=$image_dir/$image_name_in
596 local CURRENT=$image_dir/@CURRENT
597 local LOCK=$image_dir/@LOCK
598 if [ ! -e $image_dir ];then
602 echo "$func: $LOCK is locked by other process"
609 truncate -s $db_image_size $image_file
612 local count=$(($db_image_size >> $db_order))
613 local start=`cat $image_hobjects|head -n 1|awk '{print $4}'`
614 local end=`cat $image_hobjects|tail -n 1|awk '{print $4}'`
615 local entry_count=`cat $image_hobjects|wc -l`
617 local char_bits=$((`echo $start|wc -c` -1 ))
618 local format="%0"$char_bits"x"
620 local expect_start=`printf $format 0`
621 local expect_end=`printf $format $(($count -1 ))`
623 echo -e "object_count\t$entry_count"
624 echo -e "expect\t\t[$expect_start ~ $expect_end] count:$count"
625 echo -e "range\t\t[$start ~ $end] count:$entry_count"
632 trap 'echo $func failed; exit;' INT HUP
633 local unit=$((1<<$db_order))
637 icount=$(($icount+1))
638 node=`echo $line|awk '{print $1}'`
639 hobject=`echo $line|awk '{print $3}'`
640 offset=`echo $line|awk '{print $4}'`
642 if [ $icount = 1 ];then
645 hobject=`dump_backslash $hobject`
647 sshcmd="cat $hobject"
648 ssh $ssh_option $node $sshcmd < /dev/null | dd of=$image_file bs=$unit seek=$off conv=notrunc 2>/dev/null
649 percent=`echo "scale=3; 100*$icount/$entry_count"|bc`
650 tput sc #record current cursor
651 echo -n -e "complete\t[$istart ~ $iend] $icount/$entry_count ==> "$percent"%"
652 if [ $icount != $entry_count ];then
653 tput rc # backport most recent cursor
656 done < $image_hobjects
660 ls -lh $image_file|awk '{print $5"\t"$9}'
668 # ssh copy snap_object & head_object from osd to admin node
669 # copy all snapshot objects
671 # all head objects which have the same offset as snapshot objects
672 function collect_image_snap_objects()
674 local func="collect_image_snap_objects"
675 #$1=backup_dir, $2=snap_name, $3=snap_hobjects, $4=head_hobjects
677 echo "$func: parameters: <pool_id> <image_name> <snap_id> <snap_hobjects> <head_hobjects> <backup_dir>"
684 local snap_hobjects=$4 #snap hobjects info
685 local head_hobjects=$5 #head hobjects info
687 pool_id=$(($pool_id))
689 local head_dir=$backup_dir/pool_$pool_id/$image_name/@head
690 local snap_dir=$backup_dir/pool_$pool_id/$image_name/@$snap_id
691 local CURRENT=$backup_dir/pool_$pool_id/$image_name/@CURRENT
693 if [ ! -e $head_dir ];then
696 if [ ! -e $snap_dir ];then
700 local snap_node= #osd node
701 local snap_hobject= #hobject path with snapid on osd
710 # ignore if there is no object in snapshot(empty )
711 if [ ! -s $snap_hobjects ];then
712 echo "$func: $snap_hobjects is empty"
715 local start=`head -n 1 $snap_hobjects|awk '{print $4}'`
716 local end=`tail -n 1 $snap_hobjects|awk '{print $4}'`
717 local entry_count=`cat $snap_hobjects|wc -l`
718 if [ $((16#$first_offset)) -gt $((16#$last_offset)) ];then
719 echo "$func: $snap_hobjects not sorted"
723 # just assert if ignored empty snapshot
724 if [ "$start"x = ""x ] || [ "$end"x = ""x ];then
728 # speed up copy snapshot
729 # lookup the coresponding head hobject of snap hobject
730 # use command: grep <offset> <head hobjects>
733 # head hobjects: (32 objects, snapid = uint64(-2) = 18446744073709551614)
734 # ceph1 29.4d /var/lib/ceph/osd/ceph-0/current/29.4d_head/rb.0.1c414.6b8b4567.000000000000__head_EC2C1C4D__1d 000000000000 18446744073709551614 869
735 # ceph1 29.8c /var/lib/ceph/osd/ceph-0/current/29.8c_head/rb.0.1c414.6b8b4567.000000000001__head_0F439A8C__1d 000000000001 18446744073709551614 867
736 # ceph1 29.6a /var/lib/ceph/osd/ceph-0/current/29.6a_head/rb.0.1c414.6b8b4567.000000000002__head_FC55706A__1d 000000000002 18446744073709551614 869
737 # ceph1 29.8b /var/lib/ceph/osd/ceph-0/current/29.8b_head/rb.0.1c414.6b8b4567.000000000003__head_20A6328B__1d 000000000003 18446744073709551614 869
738 # ceph2 29.75 /var/lib/ceph/osd/ceph-1/current/29.75_head/rb.0.1c414.6b8b4567.000000000004__head_AC5ADB75__1d 000000000004 18446744073709551614 867
739 # ceph2 29.23 /var/lib/ceph/osd/ceph-1/current/29.23_head/rb.0.1c414.6b8b4567.000000000005__head_1FDEA823__1d 000000000005 18446744073709551614 867
741 # ceph1 29.34 /var/lib/ceph/osd/ceph-0/current/29.34_head/rb.0.1c414.6b8b4567.00000000001f__head_52373734__1d 00000000001f 18446744073709551614 869
743 # snap hobjects: (3 objects, snapid >= 29)
744 # ceph1 29.8c /var/lib/ceph/osd/ceph-0/current/29.8c_head/rb.0.1c414.6b8b4567.000000000001__1f_0F439A8C__1d 000000000001 31 867
745 # ceph1 29.6a /var/lib/ceph/osd/ceph-0/current/29.6a_head/rb.0.1c414.6b8b4567.000000000002__1e_FC55706A__1d 000000000002 30 869
746 # ceph1 29.8b /var/lib/ceph/osd/ceph-0/current/29.8b_head/rb.0.1c414.6b8b4567.000000000003__1d_20A6328B__1d 000000000003 29 869
748 # so find out offset in head hobjects line number:
749 # snap hobjects: 000000000001 ---> head hobjects: 2 (n1)
750 # snap hobjects: 000000000003 ---> head hobjects: 4 (n2)
752 # finally , grep range from the whole file [1 ~ N] shranked to part of file [n1 ~ n2]
753 # the worst case : [n1 ~ n2] = [1 ~ N], means no shranking
755 # get the line number of the start offset in head hobjects
756 local n1=`grep -n $start $head_hobjects|head -n 1|cut -d ":" -f 1`
757 # get the line number of the end offset in head hobjects
758 local n2=`grep -n $end $head_hobjects|head -n 1|cut -d ":" -f 1`
768 #assume file:snap_hobjects is not very large, and can be loaded into memory
769 local snap_arr=(`cat $snap_hobjects`)
770 local snap_tmp=/tmp/snaptmp.$$$$
773 # consists of snap hobject or head hobject
774 # select lineno range: [n1 ~ n2]
775 head -n $n2 $head_hobjects|tail -n $(($n2-$n1+1)) >$snap_tmp
777 echo "copy image snap/head objects from osd ..."
778 echo -e "object_count\t$entry_count"
779 echo -e "range\t\t[$start ~ $end] count:$entry_count"
781 trap 'echo $func failed; exit;' INT HUP
782 for line in ${snap_arr[*]}
784 icount=$(($icount+1))
789 local arr=(`echo $line`)
791 snap_hobject=${arr[2]}
792 snap_offset=${arr[3]}
793 snap_filename=$snap_dir/$snap_offset
795 if [ $icount = 1 ];then
800 #lookup corresponding head hobject of snap hobject
801 local res=`grep $snap_offset $snap_tmp|head -n 1`
802 if [ "$res"x = ""x ];then
803 echo "$func: image object[ $snap_offset ] missing"
807 local arr2=(`echo $res`)
809 head_hobject=${arr2[2]}
810 head_offset=${arr2[3]}
811 head_filename=$head_dir/$head_offset
813 # just copy object(snap/head) if it does not exist
814 if [ ! -e $snap_filename ];then
815 ssh $ssh_option $snap_node "cat $snap_hobject" > $snap_filename
817 if [ ! -e $head_filename ];then
818 ssh $ssh_option $head_node "cat $head_hobject" > $head_filename
822 percent=`echo "scale=3; 100*$icount/$entry_count"|bc`
823 tput sc #record current cursor
824 echo -n -e "complete\t[$istart ~ $iend] $icount/$entry_count ==> "$percent"%"
825 if [ $icount != $entry_count ];then
826 tput rc # backport most recent cursor
835 # copy all snap objects and corresponding head objects from osds
837 function copy_image_snap_single_thread()
839 local func="copy_image_snap_single_thread"
841 echo "$func: parameters: <pool_id> <image_name> <snap_id> <snap_hobjects> <head_hobjects> <backup_dir>"
847 local snap_hobjects=$4
848 local head_hobjects=$5
850 pool_id=$(($pool_id))
852 local CURRENT=$backup_dir/pool_$pool_id/$image_name/@CURRENT
853 local LOCK=$backup_dir/pool_$pool_id/$image_name/@LOCK
856 echo "$func: $LOCK is locked by other process"
861 collect_image_snap_objects $pool_id $image_name $snap_id $snap_hobjects $head_hobjects $backup_dir
866 # after all snap objects and necessary head objects are copied,
867 # just pick appropriate head objects and snap objects and write them to image
868 # in order to rollback image to snapshot
870 # init: image is created by copy_image_nosnap_single_thread firstly
872 # all output include 3 parts:
873 # <image> <head objects> <snap objects>
875 # head objects1 --- snap1 objects
876 # head objects2 --- snap2 objects
877 # image head objects3 --- snap3 objects
879 # head objectsN --- snapN objects
882 # firstly rollback to head, secondly write <snapX objects>
883 # head = <image> + <head objects>
884 # snap1 = <image> + <head objects> + <snap1 objects>
885 # snap2 = <image> + <head objects> + <snap2 objects>
886 # snap3 = <image> + <head objects> + <snap3 objects>
888 # snapN = <image> + <head objects> + <snapN objects>
891 # there is intersection of head objects and snapX objects, if snapX objects are not empty
892 # and need to deduplicate the intersection.
894 # - get difference set of head objects and snapX objects
895 # - write the difference set objects to image
896 # - write the snapX objects to image
897 function rollback_image_snap()
899 local func="rollback_image_snap"
903 trap 'echo $func failed; exit;' INT HUP
905 echo "$func: parameters <pool_id> <image_name> <snap_id> <snap_object_dir> <backup_dir> <image_unit>"
911 local snap_object_dir=$4
915 local need_diff_set=0
917 local image_path=$backup_dir/pool_$pool_id/$image_name/$image_name
918 local head_object_dir=$backup_dir/pool_$pool_id/$image_name/@head
919 local CURRENT=$backup_dir/pool_$pool_id/$image_name/@CURRENT
920 local LOCK=$backup_dir/pool_$pool_id/$image_name/@LOCK
922 echo "$func: $LOCK is locked by other process"
927 if [ $snap_id -ne -2 ];then
928 echo $snap_id > $CURRENT
930 echo "head" > $CURRENT
933 if [ ! -e $snap_object_dir ];then
937 if [ "$snap_object_dir"x != "$head_object_dir"x ];then
938 echo "$func: need to compute diff_set of head"
941 echo "$func: NO diff_set"
958 snap_objects=(`ls $snap_object_dir`)
960 # if need to compute difference set of head_objects and snap_objects
961 if [ $need_diff_set -ne 0 ];then
962 head_objects=(`ls $head_object_dir`)
964 #get the difference set: ( head_objects - snap_objects )
966 sort -m <(echo ${head_objects[@]}|xargs -n 1 echo) <(echo ${snap_objects[@]}|xargs -n 1 echo) \
967 <(echo ${snap_objects[@]}|xargs -n 1 echo) |uniq -u`)
969 # copy diff_set of head object to image
970 pushd $head_object_dir >/dev/null
972 echo "$func: copy diff_set head objects ..."
973 entry_count=${#diff_set[@]}
976 if [ $entry_count -gt 0 ];then
977 end=${diff_set[$(($entry_count - 1))]}
985 echo -e "object_count\t$entry_count"
986 echo -e "range\t\t[$start ~ $end] count:$entry_count"
988 for object in ${diff_set[@]}
990 icount=$(($icount+1))
991 if [ $icount = 1 ];then
996 local offset=$((16#$object))
997 dd if=$object of=$image_path bs=$image_unit seek=$offset conv=notrunc 2>/dev/null
999 percent=`echo "scale=3; 100*$icount/$entry_count"|bc`
1000 tput sc #record current cursor
1001 echo -n -e "complete\t[$istart ~ $iend] $icount/$entry_count ==> "$percent"%"
1002 if [ $icount != $entry_count ];then
1003 tput rc # backport most recent cursor
1006 if [ $entry_count -gt 0 ];then
1011 if [ $snap_id -ne -2 ];then
1012 echo -e "$image_name already rollback diff_set: (head - snap)"
1016 # copy snap object to image
1017 pushd $snap_object_dir >/dev/null
1019 if [ $need_diff_set -ne 0 ];then
1020 echo "$func: copy snap objects ..."
1022 echo "$func: copy head objects ..."
1024 entry_count=${#snap_objects[@]}
1025 start=${snap_objects[0]}
1027 if [ $entry_count -gt 0 ];then
1028 end=${snap_objects[$(($entry_count - 1))]}
1036 echo -e "object_count\t$entry_count"
1037 echo -e "range\t\t[$start ~ $end] count:$entry_count"
1039 for object in ${snap_objects[@]}
1041 icount=$(($icount+1))
1042 if [ $icount = 1 ];then
1047 local offset=$((16#$object))
1048 dd if=$object of=$image_path bs=$image_unit seek=$offset conv=notrunc 2>/dev/null
1050 percent=`echo "scale=3; 100*$icount/$entry_count"|bc`
1051 tput sc #record current cursor
1052 echo -n -e "complete\t[$istart ~ $iend] $icount/$entry_count ==> "$percent"%"
1053 if [ $icount != $entry_count ];then
1054 tput rc # backport most recent cursor
1057 if [ $entry_count -gt 0 ];then
1063 if [ $snap_id -ne -2 ];then
1064 echo "$image_name rollback to snapid: $snap_id"
1066 echo "$image_name rollback to head"
1070 function recover_image()
1072 local func="recover_image"
1075 if [ $# -lt 3 ];then
1076 echo "$func: paramters: <pool_id> <image_name> <snap_name> [<backup_dir>]"
1084 pool_id=$(($pool_id))
1085 if [ "$snap_name"x = "@"x ];then
1088 if [ "$backup_dir"x = ""x ];then
1089 backup_dir=$default_backup_dir
1092 #recover image with nosnap
1093 if [ "$snap_name"x = ""x ];then
1094 discover_image_nosnap $pool_id $img_name #input image_name
1095 local image_hobjects=$images/pool_$pool_id/$image_name_in/$image_name_in
1096 copy_image_nosnap_single_thread $pool_id $image_hobjects $backup_dir
1098 #recover image with snap
1101 # check if recovered head already
1102 local img_hobjects_path=$images/pool_$pool_id/$img_name/$img_name
1103 local img_file_path=$backup_dir/pool_$pool_id/$img_name/$img_name
1104 if [ ! -e $img_hobjects_path ] || [ ! -e $img_file_path ];then
1105 echo "$func: $img_name@$snap_name : can not rollback to snapshot, please recover image head first"
1110 if [ "$snap_name"x = "@@"x ];then
1111 local head_dir=$backup_dir/pool_$pool_id/$img_name/@head
1112 if [ -e $head_dir ];then
1113 local unit=`pushd $head_dir >/dev/null; ls|head -n 1|xargs -n 1 stat|awk '/Size:/{print $2}'`
1115 rollback_image_snap $pool_id $img_name -2 $backup_dir/$img_name/@head $backup_dir $unit
1116 echo "$image_name_in head : $backup_dir/$img_name/$img_name"
1118 echo "$func: no need to rollback to head"
1124 discover_image_snap $pool_id $img_name $snap_name # get image meta & get snapid object
1125 local snap_hobjects=$images/pool_$pool_id/$image_name_in/$image_name_in@$db_snap_id
1126 local head_hobjects=$images/pool_$pool_id/$image_name_in/$image_name_in@$db_snap_id@head
1127 local snap_object_dir=$backup_dir/pool_$pool_id/$image_name_in/@$db_snap_id
1128 local image_path=$backup_dir/pool_$pool_id/$image_name_in/$image_name_in
1129 local image_unit=$((1<<$db_order))
1130 copy_image_snap_single_thread $pool_id $image_name_in $db_snap_id $snap_hobjects $head_hobjects $backup_dir
1131 rollback_image_snap $pool_id $image_name_in $db_snap_id $snap_object_dir $backup_dir $image_unit
1132 echo "$image_name_in@$snap_name : $image_path"