3 # Copyright (C) 2013 Inktank Storage, Inc.
5 # This is free software; see the source for copying conditions.
6 # There is NO warranty; not even for MERCHANTABILITY or FITNESS FOR
7 # A PARTICULAR PURPOSE.
9 # This is free software; you can redistribute it and/or modify it
10 # under the terms of the GNU General Public License as
11 # published by the Free Software Foundation version 2.
13 # Alex Elder <elder@inktank.com>
16 ################################################################
18 # The purpose of this test is to exercise paths through the rbd
19 # code, making sure no bad pointer references or invalid reference
20 # count operations occur in the face of concurrent activity.
22 # Each pass of the test creates an rbd image, maps it, and writes
23 # some data into the image. It also reads some data from all of the
24 # other images that exist at the time the pass executes. Finally,
25 # the image is unmapped and removed. The image removal completes in
28 # An iteration of the test consists of performing some number of
29 # passes, initating each pass as a background job, and finally
30 # sleeping for a variable delay. The delay is initially a specified
31 # value, but each iteration shortens that proportionally, such that
32 # the last iteration will not delay at all.
34 # The result exercises concurrent creates and deletes of rbd images,
35 # writes to new images, reads from both written and unwritten image
36 # data (including reads concurrent with writes), and attempts to
37 # unmap images being read.
39 # Usage: concurrent [-i <iter>] [-c <count>] [-d <delay>]
44 # 2: other runtime error
45 # 99: argument count error (programming error)
46 # 100: getopt error (internal error)
48 ################################################################
52 # Default flag values; RBD_CONCURRENT_ITER names are intended
53 # to be used in yaml scripts to pass in alternate values, e.g.:
55 # RBD_CONCURRENT_ITER: 20
56 # RBD_CONCURRENT_COUNT: 5
57 # RBD_CONCURRENT_DELAY: 3
58 ITER_DEFAULT=${RBD_CONCURRENT_ITER:-100}
59 COUNT_DEFAULT=${RBD_CONCURRENT_COUNT:-5}
60 DELAY_DEFAULT=${RBD_CONCURRENT_DELAY:-5} # seconds
62 CEPH_SECRET_FILE=${CEPH_SECRET_FILE:-}
63 CEPH_ID=${CEPH_ID:-admin}
65 if [ "${CEPH_SECRET_FILE}" ]; then
66 SECRET_ARGS="--secret $CEPH_SECRET_FILE"
69 ################################################################
72 ID_MAX_DIR=$(mktemp -d /tmp/image_max_id.XXXXX)
73 ID_COUNT_DIR=$(mktemp -d /tmp/image_ids.XXXXXX)
74 NAMES_DIR=$(mktemp -d /tmp/image_names.XXXXXX)
75 SOURCE_DATA=$(mktemp /tmp/source_data.XXXXXX)
77 # Use urandom to generate SOURCE_DATA
78 dd if=/dev/urandom of=${SOURCE_DATA} bs=2048 count=66 \
81 # List of rbd id's *not* created by this script
82 export INITIAL_RBD_IDS=$(ls /sys/bus/rbd/devices)
84 # Set up some environment for normal teuthology test setup.
85 # This really should not be necessary but I found it was.
87 export CEPH_ARGS=" --name client.0"
91 [ ! "${ID_MAX_DIR}" ] && return
95 # Unmap mapped devices
96 for id in $(rbd_ids); do
97 image=$(cat "/sys/bus/rbd/devices/${id}/name")
98 rbd_unmap_image "${id}"
99 rbd_destroy_image "${image}"
101 # Get any leftover images
102 for image in $(rbd ls 2>/dev/null); do
103 rbd_destroy_image "${image}"
107 rm -f "${SOURCE_DATA}"
108 [ -d "${NAMES_DIR}" ] && rmdir "${NAMES_DIR}"
109 echo "Max concurrent rbd image count was $(get_max "${ID_COUNT_DIR}")"
110 rm -rf "${ID_COUNT_DIR}"
111 echo "Max rbd image id was $(get_max "${ID_MAX_DIR}")"
112 rm -rf "${ID_MAX_DIR}"
116 [ $# -eq 1 ] || exit 99
119 ls -U "${dir}" | sort -n | tail -1
122 trap cleanup HUP INT QUIT
124 # print a usage message and quit
126 # if a message is supplied, print that first, and then exit
127 # with non-zero status
129 if [ $# -gt 0 ]; then
135 echo "Usage: ${PROGNAME} <options> <tests>" >&2
138 echo " -h or --help" >&2
139 echo " show this message" >&2
140 echo " -i or --iterations" >&2
141 echo " iteration count (1 or more)" >&2
142 echo " -c or --count" >&2
143 echo " images created per iteration (1 or more)" >&2
144 echo " -d or --delay" >&2
145 echo " maximum delay between iterations" >&2
147 echo " defaults:" >&2
148 echo " iterations: ${ITER_DEFAULT}"
149 echo " count: ${COUNT_DEFAULT}"
150 echo " delay: ${DELAY_DEFAULT} (seconds)"
153 [ $# -gt 0 ] && exit 1
155 exit 0 # This is used for a --help
158 # parse command line arguments
159 function parseargs() {
160 ITER="${ITER_DEFAULT}"
161 COUNT="${COUNT_DEFAULT}"
162 DELAY="${DELAY_DEFAULT}"
166 SHORT_OPTS="${SHORT_OPTS},h"
167 SHORT_OPTS="${SHORT_OPTS},i:"
168 SHORT_OPTS="${SHORT_OPTS},c:"
169 SHORT_OPTS="${SHORT_OPTS},d:"
173 LONG_OPTS="${LONG_OPTS},help"
174 LONG_OPTS="${LONG_OPTS},iterations:"
175 LONG_OPTS="${LONG_OPTS},count:"
176 LONG_OPTS="${LONG_OPTS},delay:"
178 TEMP=$(getopt --name "${PROGNAME}" \
179 --options "${SHORT_OPTS}" \
180 --longoptions "${LONG_OPTS}" \
184 while [ "$1" != "--" ]; do
191 [ "${ITER}" -lt 1 ] &&
192 usage "bad iterations value"
197 [ "${COUNT}" -lt 1 ] &&
198 usage "bad count value"
206 exit 100 # Internal error
215 [ $# -eq 0 ] || exit 99
219 [ -d /sys/bus/rbd ] || return
220 ids=" $(echo $(ls /sys/bus/rbd/devices)) "
221 for i in ${INITIAL_RBD_IDS}; do
227 function update_maxes() {
230 # These aren't 100% safe against concurrent updates but it
231 # should be pretty close
232 count=$(echo ${ids} | wc -w)
233 touch "${ID_COUNT_DIR}/${count}"
235 last_id=${last_id##* }
236 touch "${ID_MAX_DIR}/${last_id}"
239 function rbd_create_image() {
240 [ $# -eq 0 ] || exit 99
241 local image=$(basename $(mktemp "${NAMES_DIR}/image.XXXXXX"))
243 rbd create "${image}" --size=1024
247 function rbd_image_id() {
248 [ $# -eq 1 ] || exit 99
251 grep -l "${image}" /sys/bus/rbd/devices/*/name 2>/dev/null |
255 function rbd_map_image() {
256 [ $# -eq 1 ] || exit 99
260 sudo rbd map "${image}" --user "${CEPH_ID}" ${SECRET_ARGS} \
263 id=$(rbd_image_id "${image}")
267 function rbd_write_image() {
268 [ $# -eq 1 ] || exit 99
271 # Offset and size here are meant to ensure beginning and end
272 # cross both (4K or 64K) page and (4MB) rbd object boundaries.
273 # It assumes the SOURCE_DATA file has size 66 * 2048 bytes
274 dd if="${SOURCE_DATA}" of="/dev/rbd${id}" bs=2048 seek=2015 \
278 # All starting and ending offsets here are selected so they are not
279 # aligned on a (4 KB or 64 KB) page boundary
280 function rbd_read_image() {
281 [ $# -eq 1 ] || exit 99
284 # First read starting and ending at an offset before any
285 # written data. The osd zero-fills data read from an
286 # existing rbd object, but before any previously-written
288 dd if="/dev/rbd${id}" of=/dev/null bs=2048 count=34 skip=3 \
290 # Next read starting at an offset before any written data,
291 # but ending at an offset that includes data that's been
292 # written. The osd zero-fills unwritten data at the
293 # beginning of a read.
294 dd if="/dev/rbd${id}" of=/dev/null bs=2048 count=34 skip=1983 \
296 # Read the data at offset 2015 * 2048 bytes (where it was
297 # written) and make sure it matches the original data.
298 cmp --quiet "${SOURCE_DATA}" "/dev/rbd${id}" 0 4126720 ||
300 # Now read starting within the pre-written data, but ending
301 # beyond it. The rbd client zero-fills the unwritten
302 # portion at the end of a read.
303 dd if="/dev/rbd${id}" of=/dev/null bs=2048 count=34 skip=2079 \
305 # Now read starting from an unwritten range within a written
306 # rbd object. The rbd client zero-fills this.
307 dd if="/dev/rbd${id}" of=/dev/null bs=2048 count=34 skip=2115 \
309 # Finally read from an unwritten region which would reside
310 # in a different (non-existent) osd object. The osd client
311 # zero-fills unwritten data when the target object doesn't
313 dd if="/dev/rbd${id}" of=/dev/null bs=2048 count=34 skip=4098 \
317 function rbd_unmap_image() {
318 [ $# -eq 1 ] || exit 99
321 sudo rbd unmap "/dev/rbd${id}"
324 function rbd_destroy_image() {
325 [ $# -eq 1 ] || exit 99
328 # Don't wait for it to complete, to increase concurrency
329 rbd rm "${image}" >/dev/null 2>&1 &
330 rm -f "${NAMES_DIR}/${image}"
333 function one_pass() {
334 [ $# -eq 0 ] || exit 99
340 image=$(rbd_create_image)
341 id=$(rbd_map_image "${image}")
343 update_maxes "${ids}"
344 for i in ${rbd_ids}; do
345 if [ "${i}" -eq "${id}" ]; then
346 rbd_write_image "${i}"
348 rbd_read_image "${i}"
351 rbd_unmap_image "${id}"
352 rbd_destroy_image "${image}"
355 ################################################################
361 for iter in $(seq 1 "${ITER}"); do
362 for count in $(seq 1 "${COUNT}"); do
365 # Sleep longer at first, overlap iterations more later.
366 # Use awk to get sub-second granularity (see sleep(1)).
367 sleep $(echo "${DELAY}" "${iter}" "${ITER}" |
368 awk '{ printf("%.2f\n", $1 - $1 * $2 / $3);}')