Fix some bugs when testing opensds ansible
[stor4nfv.git] / src / ceph / qa / workunits / rbd / image_read.sh
1 #!/bin/bash -e
2
3 # Copyright (C) 2013 Inktank Storage, Inc.
4 #
5 # This is free software; see the source for copying conditions.
6 # There is NO warranty; not even for MERCHANTABILITY or FITNESS FOR
7 # A PARTICULAR PURPOSE.
8 #
9 # This is free software; you can redistribute it and/or modify it
10 # under the terms of the GNU General Public License as
11 # published by the Free Software Foundation version 2.
12
13 # Alex Elder <elder@inktank.com>
14 # April 10, 2013
15
16 ################################################################
17
18 # The purpose of this test is to validate that data read from a
19 # mapped rbd image is what it's expected to be.
20 #
21 # By default it creates an image and fills it with some data.  It
22 # then reads back the data at a series of offsets known to cover
23 # various situations (such as reading the beginning, end, or the
24 # entirety of an object, or doing a read that spans multiple
25 # objects), and stashes the results in a set of local files.
26 #
27 # It also creates and maps a snapshot of the original image after
28 # it's been filled, and reads back the same ranges of data from the
29 # snapshot.  It then compares the data read back with what was read
30 # back from the original image, verifying they match.
31 #
32 # Clone functionality is tested as well, in which case a clone is
33 # made of the snapshot, and the same ranges of data are again read
34 # and compared with the original.  In addition, a snapshot of that
35 # clone is created, and a clone of *that* snapshot is put through
36 # the same set of tests.  (Clone testing can be optionally skipped.)
37
38 ################################################################
39
40 # Default parameter values.  Environment variables, if set, will
41 # supercede these defaults.  Such variables have names that begin
42 # with "IMAGE_READ_", for e.g. use IMAGE_READ_PAGE_SIZE=65536
43 # to use 65536 as the page size.
44
45 DEFAULT_VERBOSE=true
46 DEFAULT_TEST_CLONES=true
47 DEFAULT_LOCAL_FILES=false
48 DEFAULT_FORMAT=2
49 DEFAULT_DOUBLE_ORDER=true
50 DEFAULT_HALF_ORDER=false
51 DEFAULT_PAGE_SIZE=4096
52 DEFAULT_OBJECT_ORDER=22
53 MIN_OBJECT_ORDER=12     # technically 9, but the rbd CLI enforces 12
54 MAX_OBJECT_ORDER=32
55
56 PROGNAME=$(basename $0)
57
58 ORIGINAL=original-$$
59 SNAP1=snap1-$$
60 CLONE1=clone1-$$
61 SNAP2=snap2-$$
62 CLONE2=clone2-$$
63
64 function err() {
65         if [ $# -gt 0 ]; then
66                 echo "${PROGNAME}: $@" >&2
67         fi
68         exit 2
69 }
70
71 function usage() {
72         if [ $# -gt 0 ]; then
73                 echo "" >&2
74                 echo "${PROGNAME}: $@" >&2
75         fi
76         echo "" >&2
77         echo "Usage: ${PROGNAME} [<options>]" >&2
78         echo "" >&2
79         echo "options are:" >&2
80         echo "    -o object_order" >&2
81         echo "        must be ${MIN_OBJECT_ORDER}..${MAX_OBJECT_ORDER}" >&2
82         echo "    -p page_size    (in bytes)" >&2
83         echo "        note: there must be at least 4 pages per object" >&2
84         echo "    -1" >&2
85         echo "        test using format 1 rbd images (default)" >&2
86         echo "    -2" >&2
87         echo "        test using format 2 rbd images" >&2
88         echo "    -c" >&2
89         echo "        also test rbd clone images (implies format 2)" >&2
90         echo "    -d" >&2
91         echo "        clone object order double its parent's (format 2)" >&2
92         echo "    -h" >&2
93         echo "        clone object order half of its parent's (format 2)" >&2
94         echo "    -l" >&2
95         echo "        use local files rather than rbd images" >&2
96         echo "    -v" >&2
97         echo "        disable reporting of what's going on" >&2
98         echo "" >&2
99         exit 1
100 }
101
102 function verbose() {
103         [ "${VERBOSE}" = true ] && echo "$@"
104         true    # Don't let the verbose test spoil our return value
105 }
106
107 function quiet() {
108         "$@" 2> /dev/null
109 }
110
111 function boolean_toggle() {
112         [ $# -eq 1 ] || exit 99
113         test "$1" = "true" && echo false || echo true
114 }
115
116 function parseargs() {
117         local opts="o:p:12clv"
118         local lopts="order:,page_size:,local,clone,verbose"
119         local parsed
120         local clone_order_msg
121
122         # use values from environment if available
123         VERBOSE="${IMAGE_READ_VERBOSE:-${DEFAULT_VERBOSE}}"
124         TEST_CLONES="${IMAGE_READ_TEST_CLONES:-${DEFAULT_TEST_CLONES}}"
125         LOCAL_FILES="${IMAGE_READ_LOCAL_FILES:-${DEFAULT_LOCAL_FILES}}"
126         DOUBLE_ORDER="${IMAGE_READ_DOUBLE_ORDER:-${DEFAULT_DOUBLE_ORDER}}"
127         HALF_ORDER="${IMAGE_READ_HALF_ORDER:-${DEFAULT_HALF_ORDER}}"
128         FORMAT="${IMAGE_READ_FORMAT:-${DEFAULT_FORMAT}}"
129         PAGE_SIZE="${IMAGE_READ_PAGE_SIZE:-${DEFAULT_PAGE_SIZE}}"
130         OBJECT_ORDER="${IMAGE_READ_OBJECT_ORDER:-${DEFAULT_OBJECT_ORDER}}"
131
132         parsed=$(getopt -o "${opts}" -l "${lopts}" -n "${PROGNAME}" -- "$@") ||
133                 usage
134         eval set -- "${parsed}"
135         while true; do
136                 case "$1" in
137                 -v|--verbose)
138                         VERBOSE=$(boolean_toggle "${VERBOSE}");;
139                 -c|--clone)
140                         TEST_CLONES=$(boolean_toggle "${TEST_CLONES}");;
141                 -d|--double)
142                         DOUBLE_ORDER=$(boolean_toggle "${DOUBLE_ORDER}");;
143                 -h|--half)
144                         HALF_ORDER=$(boolean_toggle "${HALF_ORDER}");;
145                 -l|--local)
146                         LOCAL_FILES=$(boolean_toggle "${LOCAL_FILES}");;
147                 -1|-2)
148                         FORMAT="${1:1}";;
149                 -p|--page_size)
150                         PAGE_SIZE="$2"; shift;;
151                 -o|--order)
152                         OBJECT_ORDER="$2"; shift;;
153                 --)
154                         shift; break;;
155                 *)
156                         err "getopt internal error"
157                 esac
158                 shift
159         done
160         [ $# -gt 0 ] && usage "excess arguments ($*)"
161
162         if [ "${TEST_CLONES}" = true ]; then
163                 # If we're using different object orders for clones,
164                 # make sure the limits are updated accordingly.  If
165                 # both "half" and "double" are specified, just
166                 # ignore them both.
167                 if [ "${DOUBLE_ORDER}" = true ]; then
168                         if [ "${HALF_ORDER}" = true ]; then
169                                 DOUBLE_ORDER=false
170                                 HALF_ORDER=false
171                         else
172                                 ((MAX_OBJECT_ORDER -= 2))
173                         fi
174                 elif [ "${HALF_ORDER}" = true ]; then
175                         ((MIN_OBJECT_ORDER += 2))
176                 fi
177         fi
178
179         [ "${OBJECT_ORDER}" -lt "${MIN_OBJECT_ORDER}" ] &&
180                 usage "object order (${OBJECT_ORDER}) must be" \
181                         "at least ${MIN_OBJECT_ORDER}"
182         [ "${OBJECT_ORDER}" -gt "${MAX_OBJECT_ORDER}" ] &&
183                 usage "object order (${OBJECT_ORDER}) must be" \
184                         "at most ${MAX_OBJECT_ORDER}"
185
186         if [ "${TEST_CLONES}" = true ]; then
187                 if [ "${DOUBLE_ORDER}" = true ]; then
188                         ((CLONE1_ORDER = OBJECT_ORDER + 1))
189                         ((CLONE2_ORDER = OBJECT_ORDER + 2))
190                         clone_order_msg="double"
191                 elif [ "${HALF_ORDER}" = true ]; then
192                         ((CLONE1_ORDER = OBJECT_ORDER - 1))
193                         ((CLONE2_ORDER = OBJECT_ORDER - 2))
194                         clone_order_msg="half of"
195                 else
196                         CLONE1_ORDER="${OBJECT_ORDER}"
197                         CLONE2_ORDER="${OBJECT_ORDER}"
198                         clone_order_msg="the same as"
199                 fi
200         fi
201
202         [ "${TEST_CLONES}" != true ] || FORMAT=2
203
204         OBJECT_SIZE=$(echo "2 ^ ${OBJECT_ORDER}" | bc)
205         OBJECT_PAGES=$(echo "${OBJECT_SIZE} / ${PAGE_SIZE}" | bc)
206         IMAGE_SIZE=$((2 * 16 * OBJECT_SIZE / (1024 * 1024)))
207         [ "${IMAGE_SIZE}" -lt 1 ] && IMAGE_SIZE=1
208         IMAGE_OBJECTS=$((IMAGE_SIZE * (1024 * 1024) / OBJECT_SIZE))
209
210         [ "${OBJECT_PAGES}" -lt 4 ] &&
211                 usage "object size (${OBJECT_SIZE}) must be" \
212                         "at least 4 * page size (${PAGE_SIZE})"
213
214         echo "parameters for this run:"
215         echo "    format ${FORMAT} images will be tested"
216         echo "    object order is ${OBJECT_ORDER}, so" \
217                 "objects are ${OBJECT_SIZE} bytes"
218         echo "    page size is ${PAGE_SIZE} bytes, so" \
219                 "there are are ${OBJECT_PAGES} pages in an object"
220         echo "    derived image size is ${IMAGE_SIZE} MB, so" \
221                 "there are ${IMAGE_OBJECTS} objects in an image"
222         if [ "${TEST_CLONES}" = true ]; then
223                 echo "    clone functionality will be tested"
224                 echo "    object size for a clone will be ${clone_order_msg}"
225                 echo "        the object size of its parent image"
226         fi
227
228         true    # Don't let the clones test spoil our return value
229 }
230
231 function image_dev_path() {
232         [ $# -eq 1 ] || exit 99
233         local image_name="$1"
234
235         if [ "${LOCAL_FILES}" = true ]; then
236                 echo "${TEMP}/${image_name}"
237                 return
238         fi
239
240         echo "/dev/rbd/rbd/${image_name}"
241 }
242
243 function out_data_dir() {
244         [ $# -lt 2 ] || exit 99
245         local out_data="${TEMP}/data"
246         local image_name
247
248         if [ $# -eq 1 ]; then
249                 image_name="$1"
250                 echo "${out_data}/${image_name}"
251         else
252                 echo "${out_data}"
253         fi
254 }
255
256 function setup() {
257         verbose "===== setting up ====="
258         TEMP=$(mktemp -d /tmp/rbd_image_read.XXXXX)
259         mkdir -p $(out_data_dir)
260
261         # create and fill the original image with some data
262         create_image "${ORIGINAL}"
263         map_image "${ORIGINAL}"
264         fill_original
265
266         # create a snapshot of the original
267         create_image_snap "${ORIGINAL}" "${SNAP1}"
268         map_image_snap "${ORIGINAL}" "${SNAP1}"
269
270         if [ "${TEST_CLONES}" = true ]; then
271                 # create a clone of the original snapshot
272                 create_snap_clone "${ORIGINAL}" "${SNAP1}" \
273                         "${CLONE1}" "${CLONE1_ORDER}"
274                 map_image "${CLONE1}"
275
276                 # create a snapshot of that clone
277                 create_image_snap "${CLONE1}" "${SNAP2}"
278                 map_image_snap "${CLONE1}" "${SNAP2}"
279
280                 # create a clone of that clone's snapshot
281                 create_snap_clone "${CLONE1}" "${SNAP2}" \
282                         "${CLONE2}" "${CLONE2_ORDER}"
283                 map_image "${CLONE2}"
284         fi
285 }
286
287 function teardown() {
288         verbose "===== cleaning up ====="
289         if [ "${TEST_CLONES}" = true ]; then
290                 unmap_image "${CLONE2}"                                 || true
291                 destroy_snap_clone "${CLONE1}" "${SNAP2}" "${CLONE2}"   || true
292
293                 unmap_image_snap "${CLONE1}" "${SNAP2}"                 || true
294                 destroy_image_snap "${CLONE1}" "${SNAP2}"               || true
295
296                 unmap_image "${CLONE1}"                                 || true
297                 destroy_snap_clone "${ORIGINAL}" "${SNAP1}" "${CLONE1}" || true
298         fi
299         unmap_image_snap "${ORIGINAL}" "${SNAP1}"                       || true
300         destroy_image_snap "${ORIGINAL}" "${SNAP1}"                     || true
301         unmap_image "${ORIGINAL}"                                       || true
302         destroy_image "${ORIGINAL}"                                     || true
303
304         rm -rf $(out_data_dir)
305         rmdir "${TEMP}"
306 }
307
308 function create_image() {
309         [ $# -eq 1 ] || exit 99
310         local image_name="$1"
311         local image_path
312         local bytes
313
314         verbose "creating image \"${image_name}\""
315         if [ "${LOCAL_FILES}" = true ]; then
316                 image_path=$(image_dev_path "${image_name}")
317                 bytes=$(echo "${IMAGE_SIZE} * 1024 * 1024 - 1" | bc)
318                 quiet dd if=/dev/zero bs=1 count=1 seek="${bytes}" \
319                         of="${image_path}"
320                 return
321         fi
322
323         rbd create "${image_name}" --image-format "${FORMAT}" \
324                 --size "${IMAGE_SIZE}" --order "${OBJECT_ORDER}" \
325                 --image-shared
326 }
327
328 function destroy_image() {
329         [ $# -eq 1 ] || exit 99
330         local image_name="$1"
331         local image_path
332
333         verbose "destroying image \"${image_name}\""
334         if [ "${LOCAL_FILES}" = true ]; then
335                 image_path=$(image_dev_path "${image_name}")
336                 rm -f "${image_path}"
337                 return
338         fi
339
340         rbd rm "${image_name}"
341 }
342
343 function map_image() {
344         [ $# -eq 1 ] || exit 99
345         local image_name="$1"           # can be image@snap too
346
347         if [ "${LOCAL_FILES}" = true ]; then
348                 return
349         fi
350
351         sudo rbd map "${image_name}"
352 }
353
354 function unmap_image() {
355         [ $# -eq 1 ] || exit 99
356         local image_name="$1"           # can be image@snap too
357         local image_path
358
359         if [ "${LOCAL_FILES}" = true ]; then
360                 return
361         fi
362         image_path=$(image_dev_path "${image_name}")
363
364         if [ -e "${image_path}" ]; then
365                 sudo rbd unmap "${image_path}"
366         fi
367 }
368
369 function map_image_snap() {
370         [ $# -eq 2 ] || exit 99
371         local image_name="$1"
372         local snap_name="$2"
373         local image_snap
374
375         if [ "${LOCAL_FILES}" = true ]; then
376                 return
377         fi
378
379         image_snap="${image_name}@${snap_name}"
380         map_image "${image_snap}"
381 }
382
383 function unmap_image_snap() {
384         [ $# -eq 2 ] || exit 99
385         local image_name="$1"
386         local snap_name="$2"
387         local image_snap
388
389         if [ "${LOCAL_FILES}" = true ]; then
390                 return
391         fi
392
393         image_snap="${image_name}@${snap_name}"
394         unmap_image "${image_snap}"
395 }
396
397 function create_image_snap() {
398         [ $# -eq 2 ] || exit 99
399         local image_name="$1"
400         local snap_name="$2"
401         local image_snap="${image_name}@${snap_name}"
402         local image_path
403         local snap_path
404
405         verbose "creating snapshot \"${snap_name}\"" \
406                 "of image \"${image_name}\""
407         if [ "${LOCAL_FILES}" = true ]; then
408                 image_path=$(image_dev_path "${image_name}")
409                 snap_path=$(image_dev_path "${image_snap}")
410
411                 cp "${image_path}" "${snap_path}"
412                 return
413         fi
414
415         rbd snap create "${image_snap}"
416 }
417
418 function destroy_image_snap() {
419         [ $# -eq 2 ] || exit 99
420         local image_name="$1"
421         local snap_name="$2"
422         local image_snap="${image_name}@${snap_name}"
423         local snap_path
424
425         verbose "destroying snapshot \"${snap_name}\"" \
426                 "of image \"${image_name}\""
427         if [ "${LOCAL_FILES}" = true ]; then
428                 snap_path=$(image_dev_path "${image_snap}")
429                 rm -rf "${snap_path}"
430                 return
431         fi
432
433         rbd snap rm "${image_snap}"
434 }
435
436 function create_snap_clone() {
437         [ $# -eq 4 ] || exit 99
438         local image_name="$1"
439         local snap_name="$2"
440         local clone_name="$3"
441         local clone_order="$4"
442         local image_snap="${image_name}@${snap_name}"
443         local snap_path
444         local clone_path
445
446         verbose "creating clone image \"${clone_name}\"" \
447                 "of image snapshot \"${image_name}@${snap_name}\""
448         if [ "${LOCAL_FILES}" = true ]; then
449                 snap_path=$(image_dev_path "${image_name}@${snap_name}")
450                 clone_path=$(image_dev_path "${clone_name}")
451
452                 cp "${snap_path}" "${clone_path}"
453                 return
454         fi
455
456         rbd snap protect "${image_snap}"
457         rbd clone --order "${clone_order}" --image-shared \
458                 "${image_snap}" "${clone_name}"
459 }
460
461 function destroy_snap_clone() {
462         [ $# -eq 3 ] || exit 99
463         local image_name="$1"
464         local snap_name="$2"
465         local clone_name="$3"
466         local image_snap="${image_name}@${snap_name}"
467         local clone_path
468
469         verbose "destroying clone image \"${clone_name}\""
470         if [ "${LOCAL_FILES}" = true ]; then
471                 clone_path=$(image_dev_path "${clone_name}")
472
473                 rm -rf "${clone_path}"
474                 return
475         fi
476
477         rbd rm "${clone_name}"
478         rbd snap unprotect "${image_snap}"
479 }
480
481 # function that produces "random" data with which to fill the image
482 function source_data() {
483         while quiet dd if=/bin/bash skip=$(($$ % 199)) bs="${PAGE_SIZE}"; do
484                 :       # Just do the dd
485         done
486 }
487
488 function fill_original() {
489         local image_path=$(image_dev_path "${ORIGINAL}")
490
491         verbose "filling original image"
492         # Fill 16 objects worth of "random" data
493         source_data |
494         quiet dd bs="${PAGE_SIZE}" count=$((16 * OBJECT_PAGES)) \
495                 of="${image_path}"
496 }
497
498 function do_read() {
499         [ $# -eq 3 -o $# -eq 4 ] || exit 99
500         local image_name="$1"
501         local offset="$2"
502         local length="$3"
503         [ "${length}" -gt 0 ] || err "do_read: length must be non-zero"
504         local image_path=$(image_dev_path "${image_name}")
505         local out_data=$(out_data_dir "${image_name}")
506         local range=$(printf "%06u~%04u" "${offset}" "${length}")
507         local out_file
508
509         [ $# -eq 4 ] && offset=$((offset + 16 * OBJECT_PAGES))
510
511         verbose "reading \"${image_name}\" pages ${range}"
512
513         out_file="${out_data}/pages_${range}"
514
515         quiet dd bs="${PAGE_SIZE}" skip="${offset}" count="${length}" \
516                 if="${image_path}" of="${out_file}"
517 }
518
519 function one_pass() {
520         [ $# -eq 1 -o $# -eq 2 ] || exit 99
521         local image_name="$1"
522         local extended
523         [ $# -eq 2 ] && extended="true"
524         local offset
525         local length
526
527         offset=0
528
529         # +-----------+-----------+---
530         # |X:X:X...X:X| : : ... : | :
531         # +-----------+-----------+---
532         length="${OBJECT_PAGES}"
533         do_read "${image_name}" "${offset}" "${length}" ${extended}
534         offset=$((offset + length))
535
536         # ---+-----------+---
537         #  : |X: : ... : | :
538         # ---+-----------+---
539         length=1
540         do_read "${image_name}" "${offset}" "${length}" ${extended}
541         offset=$((offset + length))
542
543         # ---+-----------+---
544         #  : | :X: ... : | :
545         # ---+-----------+---
546         length=1
547         do_read "${image_name}" "${offset}" "${length}" ${extended}
548         offset=$((offset + length))
549
550         # ---+-----------+---
551         #  : | : :X...X: | :
552         # ---+-----------+---
553         length=$((OBJECT_PAGES - 3))
554         do_read "${image_name}" "${offset}" "${length}" ${extended}
555         offset=$((offset + length))
556
557         # ---+-----------+---
558         #  : | : : ... :X| :
559         # ---+-----------+---
560         length=1
561         do_read "${image_name}" "${offset}" "${length}" ${extended}
562         offset=$((offset + length))
563
564         # ---+-----------+---
565         #  : |X:X:X...X:X| :
566         # ---+-----------+---
567         length="${OBJECT_PAGES}"
568         do_read "${image_name}" "${offset}" "${length}" ${extended}
569         offset=$((offset + length))
570
571         offset=$((offset + 1))          # skip 1
572
573         # ---+-----------+---
574         #  : | :X:X...X:X| :
575         # ---+-----------+---
576         length=$((OBJECT_PAGES - 1))
577         do_read "${image_name}" "${offset}" "${length}" ${extended}
578         offset=$((offset + length))
579
580         # ---+-----------+-----------+---
581         #  : |X:X:X...X:X|X: : ... : | :
582         # ---+-----------+-----------+---
583         length=$((OBJECT_PAGES + 1))
584         do_read "${image_name}" "${offset}" "${length}" ${extended}
585         offset=$((offset + length))
586
587         # ---+-----------+-----------+---
588         #  : | :X:X...X:X|X: : ... : | :
589         # ---+-----------+-----------+---
590         length="${OBJECT_PAGES}"
591         do_read "${image_name}" "${offset}" "${length}" ${extended}
592         offset=$((offset + length))
593
594         # ---+-----------+-----------+---
595         #  : | :X:X...X:X|X:X: ... : | :
596         # ---+-----------+-----------+---
597         length=$((OBJECT_PAGES + 1))
598         do_read "${image_name}" "${offset}" "${length}" ${extended}
599         offset=$((offset + length))
600
601         # ---+-----------+-----------+---
602         #  : | : :X...X:X|X:X:X...X:X| :
603         # ---+-----------+-----------+---
604         length=$((2 * OBJECT_PAGES + 2))
605         do_read "${image_name}" "${offset}" "${length}" ${extended}
606         offset=$((offset + length))
607
608         offset=$((offset + 1))          # skip 1
609
610         # ---+-----------+-----------+-----
611         #  : | :X:X...X:X|X:X:X...X:X|X: :
612         # ---+-----------+-----------+-----
613         length=$((2 * OBJECT_PAGES))
614         do_read "${image_name}" "${offset}" "${length}" ${extended}
615         offset=$((offset + length))
616
617         # --+-----------+-----------+--------
618         #  : | :X:X...X:X|X:X:X...X:X|X:X: :
619         # --+-----------+-----------+--------
620         length=2049
621         length=$((2 * OBJECT_PAGES + 1))
622         do_read "${image_name}" "${offset}" "${length}" ${extended}
623         # offset=$((offset + length))
624 }
625
626 function run_using() {
627         [ $# -eq 1 ] || exit 99
628         local image_name="$1"
629         local out_data=$(out_data_dir "${image_name}")
630
631         verbose "===== running using \"${image_name}\" ====="
632         mkdir -p "${out_data}"
633         one_pass "${image_name}"
634         one_pass "${image_name}" extended
635 }
636
637 function compare() {
638         [ $# -eq 1 ] || exit 99
639         local image_name="$1"
640         local out_data=$(out_data_dir "${image_name}")
641         local original=$(out_data_dir "${ORIGINAL}")
642
643         verbose "===== comparing \"${image_name}\" ====="
644         for i in $(ls "${original}"); do
645                 verbose compare "\"${image_name}\" \"${i}\""
646                 cmp "${original}/${i}" "${out_data}/${i}"
647         done
648         [ "${image_name}" = "${ORIGINAL}" ] || rm -rf "${out_data}"
649 }
650
651 function doit() {
652         [ $# -eq 1 ] || exit 99
653         local image_name="$1"
654
655         run_using "${image_name}"
656         compare "${image_name}"
657 }
658
659 ########## Start
660
661 parseargs "$@"
662
663 trap teardown EXIT HUP INT
664 setup
665
666 run_using "${ORIGINAL}"
667 doit "${ORIGINAL}@${SNAP1}"
668 if [ "${TEST_CLONES}" = true ]; then
669         doit "${CLONE1}"
670         doit "${CLONE1}@${SNAP2}"
671         doit "${CLONE2}"
672 fi
673 rm -rf $(out_data_dir "${ORIGINAL}")
674
675 echo "Success!"
676
677 exit 0