3 # rbd_mirror_stress.sh - stress test rbd-mirror daemon
5 # The following additional environment variables affect the test:
7 # RBD_MIRROR_REDUCE_WRITES - if not empty, don't run the stress bench write
8 # tool during the many image test
14 . $(dirname $0)/rbd_mirror_helpers.sh
23 rbd --cluster ${cluster} -p ${pool} snap create ${image}@${snap_name} \
24 --debug-rbd=20 --debug-journaler=20 2> ${TEMPDIR}/rbd-snap-create.log
33 local rmt_export=${TEMPDIR}/${CLUSTER2}-${pool}-${image}.export
34 local loc_export=${TEMPDIR}/${CLUSTER1}-${pool}-${image}.export
36 rm -f ${rmt_export} ${loc_export}
37 rbd --cluster ${CLUSTER2} -p ${pool} export ${image}@${snap_name} ${rmt_export}
38 rbd --cluster ${CLUSTER1} -p ${pool} export ${image}@${snap_name} ${loc_export}
39 cmp ${rmt_export} ${loc_export}
40 rm -f ${rmt_export} ${loc_export}
43 wait_for_pool_images()
53 for s in `seq 1 40`; do
54 test $s -ne 1 && sleep 30
55 count=$(rbd --cluster ${cluster} -p ${pool} mirror pool status | grep 'images: ' | cut -d' ' -f 2)
56 test "${count}" = "${image_count}" && return 0
58 # reset timeout if making forward progress
59 test $count -ne $last_count && break
62 test $count -eq $last_count && break
65 rbd --cluster ${cluster} -p ${pool} mirror pool status --verbose >&2
69 wait_for_pool_healthy()
76 for s in `seq 1 40`; do
77 test $s -ne 1 && sleep 30
78 state=$(rbd --cluster ${cluster} -p ${pool} mirror pool status | grep 'health:' | cut -d' ' -f 2)
79 test "${state}" = "ERROR" && break
80 test "${state}" = "OK" && return 0
82 rbd --cluster ${cluster} -p ${pool} mirror pool status --verbose >&2
86 start_mirror ${CLUSTER1}
87 start_mirror ${CLUSTER2}
89 testlog "TEST: add image and test replay after client crashes"
91 create_image ${CLUSTER2} ${POOL} ${image} '512M'
92 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
96 stress_write_image ${CLUSTER2} ${POOL} ${image}
98 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' 'master_position'
101 create_snap ${CLUSTER2} ${POOL} ${image} ${snap_name}
102 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
103 wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image}
104 wait_for_snap_present ${CLUSTER1} ${POOL} ${image} ${snap_name}
105 compare_image_snaps ${POOL} ${image} ${snap_name}
111 remove_snapshot ${CLUSTER2} ${POOL} ${image} ${snap_name}
114 remove_image_retry ${CLUSTER2} ${POOL} ${image}
115 wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'deleted'
117 testlog "TEST: create many images"
119 for i in `seq 1 ${IMAGE_COUNT}`
122 create_image ${CLUSTER2} ${POOL} ${image} '128M'
123 if [ -n "${RBD_MIRROR_REDUCE_WRITES}" ]; then
124 write_image ${CLUSTER2} ${POOL} ${image} 100
126 stress_write_image ${CLUSTER2} ${POOL} ${image}
130 wait_for_pool_images ${CLUSTER2} ${POOL} ${IMAGE_COUNT}
131 wait_for_pool_healthy ${CLUSTER2} ${POOL}
133 wait_for_pool_images ${CLUSTER1} ${POOL} ${IMAGE_COUNT}
134 wait_for_pool_healthy ${CLUSTER1} ${POOL}
136 testlog "TEST: compare many images"
137 for i in `seq 1 ${IMAGE_COUNT}`
140 create_snap ${CLUSTER2} ${POOL} ${image} ${snap_name}
141 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
142 wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image}
143 wait_for_snap_present ${CLUSTER1} ${POOL} ${image} ${snap_name}
144 compare_image_snaps ${POOL} ${image} ${snap_name}
147 testlog "TEST: delete many images"
148 for i in `seq 1 ${IMAGE_COUNT}`
151 remove_snapshot ${CLUSTER2} ${POOL} ${image} ${snap_name}
152 remove_image_retry ${CLUSTER2} ${POOL} ${image}
155 testlog "TEST: image deletions should propagate"
156 wait_for_pool_images ${CLUSTER1} ${POOL} 0
157 wait_for_pool_healthy ${CLUSTER1} ${POOL} 0
158 for i in `seq 1 ${IMAGE_COUNT}`
161 wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'deleted'
164 testlog "TEST: delete images during bootstrap"
165 set_pool_mirror_mode ${CLUSTER1} ${POOL} 'image'
166 set_pool_mirror_mode ${CLUSTER2} ${POOL} 'image'
168 start_mirror ${CLUSTER1}
174 create_image ${CLUSTER2} ${POOL} ${image} '512M'
175 enable_mirror ${CLUSTER2} ${POOL} ${image}
177 stress_write_image ${CLUSTER2} ${POOL} ${image}
178 wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'present'
180 disable_mirror ${CLUSTER2} ${POOL} ${image}
181 wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'deleted'
182 purge_snapshots ${CLUSTER2} ${POOL} ${image}
183 remove_image_retry ${CLUSTER2} ${POOL} ${image}