src/ceph/qa/workunits/rbd/rbd_mirror.sh

   1 #!/bin/sh
   2 #
   3 # rbd_mirror.sh - test rbd-mirror daemon
   4 #
   5 # The scripts starts two ("local" and "remote") clusters using mstart.sh script,
   6 # creates a temporary directory, used for cluster configs, daemon logs, admin
   7 # socket, temporary files, and launches rbd-mirror daemon.
   8 #
   9
  10 . $(dirname $0)/rbd_mirror_helpers.sh
  11
  12 testlog "TEST: add image and test replay"
  13 start_mirror ${CLUSTER1}
  14 image=test
  15 create_image ${CLUSTER2} ${POOL} ${image}
  16 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
  17 write_image ${CLUSTER2} ${POOL} ${image} 100
  18 wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image}
  19 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' 'master_position'
  20 if [ -z "${RBD_MIRROR_USE_RBD_MIRROR}" ]; then
  21   wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'down+unknown'
  22 fi
  23 compare_images ${POOL} ${image}
  24
  25 testlog "TEST: stop mirror, add image, start mirror and test replay"
  26 stop_mirror ${CLUSTER1}
  27 image1=test1
  28 create_image ${CLUSTER2} ${POOL} ${image1}
  29 write_image ${CLUSTER2} ${POOL} ${image1} 100
  30 start_mirror ${CLUSTER1}
  31 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image1}
  32 wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image1}
  33 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image1} 'up+replaying' 'master_position'
  34 if [ -z "${RBD_MIRROR_USE_RBD_MIRROR}" ]; then
  35   wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image1} 'down+unknown'
  36 fi
  37 compare_images ${POOL} ${image1}
  38
  39 testlog "TEST: test the first image is replaying after restart"
  40 write_image ${CLUSTER2} ${POOL} ${image} 100
  41 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
  42 wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image}
  43 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' 'master_position'
  44 compare_images ${POOL} ${image}
  45
  46 testlog "TEST: stop/start/restart mirror via admin socket"
  47 admin_daemon ${CLUSTER1} rbd mirror stop
  48 wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image}
  49 wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image1}
  50 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+stopped'
  51 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image1} 'up+stopped'
  52
  53 admin_daemon ${CLUSTER1} rbd mirror start
  54 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
  55 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image1}
  56 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying'
  57 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image1} 'up+replaying'
  58
  59 admin_daemon ${CLUSTER1} rbd mirror restart
  60 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
  61 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image1}
  62 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying'
  63 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image1} 'up+replaying'
  64
  65 admin_daemon ${CLUSTER1} rbd mirror stop
  66 wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image}
  67 wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image1}
  68 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+stopped'
  69 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image1} 'up+stopped'
  70
  71 admin_daemon ${CLUSTER1} rbd mirror restart
  72 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
  73 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image1}
  74 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying'
  75 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image1} 'up+replaying'
  76
  77 admin_daemon ${CLUSTER1} rbd mirror stop ${POOL} ${CLUSTER2}
  78 wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image}
  79 wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image1}
  80 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+stopped'
  81 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image1} 'up+stopped'
  82
  83 admin_daemon ${CLUSTER1} rbd mirror start ${POOL}/${image}
  84 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
  85 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying'
  86
  87 admin_daemon ${CLUSTER1} rbd mirror start ${POOL} ${CLUSTER2}
  88 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image1}
  89 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image1} 'up+replaying'
  90
  91 admin_daemon ${CLUSTER1} rbd mirror restart ${POOL}/${image}
  92 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
  93 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying'
  94
  95 admin_daemon ${CLUSTER1} rbd mirror restart ${POOL} ${CLUSTER2}
  96 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
  97 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image1}
  98
  99 admin_daemon ${CLUSTER1} rbd mirror stop ${POOL} ${CLUSTER2}
 100 wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image}
 101 wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image1}
 102 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+stopped'
 103 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image1} 'up+stopped'
 104
 105 admin_daemon ${CLUSTER1} rbd mirror restart ${POOL} ${CLUSTER2}
 106 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
 107 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image1}
 108 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying'
 109 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image1} 'up+replaying'
 110
 111 admin_daemon ${CLUSTER1} rbd mirror flush
 112 admin_daemon ${CLUSTER1} rbd mirror status
 113
 114 testlog "TEST: test image rename"
 115 new_name="${image}_RENAMED"
 116 rename_image ${CLUSTER2} ${POOL} ${image} ${new_name}
 117 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${new_name}
 118 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${new_name} 'up+replaying'
 119 admin_daemon ${CLUSTER1} rbd mirror status ${POOL}/${new_name}
 120 admin_daemon ${CLUSTER1} rbd mirror restart ${POOL}/${new_name}
 121 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${new_name}
 122 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${new_name} 'up+replaying'
 123 rename_image ${CLUSTER2} ${POOL} ${new_name} ${image}
 124 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
 125
 126 testlog "TEST: failover and failback"
 127 start_mirror ${CLUSTER2}
 128
 129 # demote and promote same cluster
 130 demote_image ${CLUSTER2} ${POOL} ${image}
 131 wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image}
 132 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+unknown'
 133 wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+unknown'
 134 promote_image ${CLUSTER2} ${POOL} ${image}
 135 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
 136 write_image ${CLUSTER2} ${POOL} ${image} 100
 137 wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image}
 138 wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+stopped'
 139 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' 'master_position'
 140 compare_images ${POOL} ${image}
 141
 142 # failover (unmodified)
 143 demote_image ${CLUSTER2} ${POOL} ${image}
 144 wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image}
 145 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+unknown'
 146 wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+unknown'
 147 promote_image ${CLUSTER1} ${POOL} ${image}
 148 wait_for_image_replay_started ${CLUSTER2} ${POOL} ${image}
 149
 150 # failback (unmodified)
 151 demote_image ${CLUSTER1} ${POOL} ${image}
 152 wait_for_image_replay_stopped ${CLUSTER2} ${POOL} ${image}
 153 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+unknown'
 154 wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+unknown'
 155 promote_image ${CLUSTER2} ${POOL} ${image}
 156 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
 157 wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image}
 158 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' 'master_position'
 159 wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+stopped'
 160 compare_images ${POOL} ${image}
 161
 162 # failover
 163 demote_image ${CLUSTER2} ${POOL} ${image}
 164 wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image}
 165 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+unknown'
 166 wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+unknown'
 167 promote_image ${CLUSTER1} ${POOL} ${image}
 168 wait_for_image_replay_started ${CLUSTER2} ${POOL} ${image}
 169 write_image ${CLUSTER1} ${POOL} ${image} 100
 170 wait_for_replay_complete ${CLUSTER2} ${CLUSTER1} ${POOL} ${image}
 171 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+stopped'
 172 wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+replaying' 'master_position'
 173 compare_images ${POOL} ${image}
 174
 175 # failback
 176 demote_image ${CLUSTER1} ${POOL} ${image}
 177 wait_for_image_replay_stopped ${CLUSTER2} ${POOL} ${image}
 178 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+unknown'
 179 wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+unknown'
 180 promote_image ${CLUSTER2} ${POOL} ${image}
 181 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
 182 write_image ${CLUSTER2} ${POOL} ${image} 100
 183 wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image}
 184 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' 'master_position'
 185 wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} 'up+stopped'
 186 compare_images ${POOL} ${image}
 187
 188 # force promote
 189 force_promote_image=test_force_promote
 190 create_image ${CLUSTER2} ${POOL} ${force_promote_image}
 191 write_image ${CLUSTER2} ${POOL} ${force_promote_image} 100
 192 wait_for_image_replay_stopped ${CLUSTER2} ${POOL} ${force_promote_image}
 193 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${force_promote_image}
 194 wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${force_promote_image}
 195 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${force_promote_image} 'up+replaying' 'master_position'
 196 wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${force_promote_image} 'up+stopped'
 197 promote_image ${CLUSTER1} ${POOL} ${force_promote_image} '--force'
 198 wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${force_promote_image}
 199 wait_for_image_replay_stopped ${CLUSTER2} ${POOL} ${force_promote_image}
 200 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${force_promote_image} 'up+stopped'
 201 wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${force_promote_image} 'up+stopped'
 202 write_image ${CLUSTER1} ${POOL} ${force_promote_image} 100
 203 write_image ${CLUSTER2} ${POOL} ${force_promote_image} 100
 204
 205 testlog "TEST: cloned images"
 206 parent_image=test_parent
 207 parent_snap=snap
 208 create_image ${CLUSTER2} ${PARENT_POOL} ${parent_image}
 209 write_image ${CLUSTER2} ${PARENT_POOL} ${parent_image} 100
 210 create_snapshot ${CLUSTER2} ${PARENT_POOL} ${parent_image} ${parent_snap}
 211 protect_snapshot ${CLUSTER2} ${PARENT_POOL} ${parent_image} ${parent_snap}
 212
 213 clone_image=test_clone
 214 clone_image ${CLUSTER2} ${PARENT_POOL} ${parent_image} ${parent_snap} ${POOL} ${clone_image}
 215 write_image ${CLUSTER2} ${POOL} ${clone_image} 100
 216
 217 enable_mirror ${CLUSTER2} ${PARENT_POOL} ${parent_image}
 218 wait_for_image_replay_started ${CLUSTER1} ${PARENT_POOL} ${parent_image}
 219 wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${PARENT_POOL} ${parent_image}
 220 wait_for_status_in_pool_dir ${CLUSTER1} ${PARENT_POOL} ${parent_image} 'up+replaying' 'master_position'
 221 compare_images ${PARENT_POOL} ${parent_image}
 222
 223 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${clone_image}
 224 wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${clone_image}
 225 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${clone_image} 'up+replaying' 'master_position'
 226 compare_images ${POOL} ${clone_image}
 227
 228 expect_failure "is non-primary" clone_image ${CLUSTER1} ${PARENT_POOL} \
 229     ${parent_image} ${parent_snap} ${POOL} ${clone_image}1
 230
 231 testlog "TEST: data pool"
 232 dp_image=test_data_pool
 233 create_image ${CLUSTER2} ${POOL} ${dp_image} 128 --data-pool ${PARENT_POOL}
 234 data_pool=$(get_image_data_pool ${CLUSTER2} ${POOL} ${dp_image})
 235 test "${data_pool}" = "${PARENT_POOL}"
 236 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${dp_image}
 237 data_pool=$(get_image_data_pool ${CLUSTER1} ${POOL} ${dp_image})
 238 test "${data_pool}" = "${PARENT_POOL}"
 239 create_snapshot ${CLUSTER2} ${POOL} ${dp_image} 'snap1'
 240 write_image ${CLUSTER2} ${POOL} ${dp_image} 100
 241 create_snapshot ${CLUSTER2} ${POOL} ${dp_image} 'snap2'
 242 write_image ${CLUSTER2} ${POOL} ${dp_image} 100
 243 wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${dp_image}
 244 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${dp_image} 'up+replaying' 'master_position'
 245 compare_images ${POOL} ${dp_image}@snap1
 246 compare_images ${POOL} ${dp_image}@snap2
 247 compare_images ${POOL} ${dp_image}
 248
 249 testlog "TEST: disable mirroring / delete non-primary image"
 250 image2=test2
 251 image3=test3
 252 image4=test4
 253 image5=test5
 254 for i in ${image2} ${image3} ${image4} ${image5}; do
 255   create_image ${CLUSTER2} ${POOL} ${i}
 256   write_image ${CLUSTER2} ${POOL} ${i} 100
 257   create_snapshot ${CLUSTER2} ${POOL} ${i} 'snap1'
 258   create_snapshot ${CLUSTER2} ${POOL} ${i} 'snap2'
 259   if [ "${i}" = "${image4}" ] || [ "${i}" = "${image5}" ]; then
 260     protect_snapshot ${CLUSTER2} ${POOL} ${i} 'snap1'
 261     protect_snapshot ${CLUSTER2} ${POOL} ${i} 'snap2'
 262   fi
 263   write_image ${CLUSTER2} ${POOL} ${i} 100
 264   wait_for_image_present ${CLUSTER1} ${POOL} ${i} 'present'
 265   wait_for_snap_present ${CLUSTER1} ${POOL} ${i} 'snap2'
 266 done
 267
 268 set_pool_mirror_mode ${CLUSTER2} ${POOL} 'image'
 269 for i in ${image2} ${image4}; do
 270   disable_mirror ${CLUSTER2} ${POOL} ${i}
 271 done
 272
 273 unprotect_snapshot ${CLUSTER2} ${POOL} ${image5} 'snap1'
 274 unprotect_snapshot ${CLUSTER2} ${POOL} ${image5} 'snap2'
 275 for i in ${image3} ${image5}; do
 276   remove_snapshot ${CLUSTER2} ${POOL} ${i} 'snap1'
 277   remove_snapshot ${CLUSTER2} ${POOL} ${i} 'snap2'
 278   remove_image_retry ${CLUSTER2} ${POOL} ${i}
 279 done
 280
 281 for i in ${image2} ${image3} ${image4} ${image5}; do
 282   wait_for_image_present ${CLUSTER1} ${POOL} ${i} 'deleted'
 283 done
 284
 285 set_pool_mirror_mode ${CLUSTER2} ${POOL} 'pool'
 286 for i in ${image2} ${image4}; do
 287   wait_for_image_present ${CLUSTER1} ${POOL} ${i} 'present'
 288   wait_for_snap_present ${CLUSTER1} ${POOL} ${i} 'snap2'
 289   wait_for_image_replay_started ${CLUSTER1} ${POOL} ${i}
 290   wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${i}
 291   compare_images ${POOL} ${i}
 292 done
 293
 294 testlog "TEST: snapshot rename"
 295 snap_name='snap_rename'
 296 create_snapshot ${CLUSTER2} ${POOL} ${image2} "${snap_name}_0"
 297 for i in `seq 1 20`; do
 298   rename_snapshot ${CLUSTER2} ${POOL} ${image2} "${snap_name}_$(expr ${i} - 1)" "${snap_name}_${i}"
 299 done
 300 wait_for_snap_present ${CLUSTER1} ${POOL} ${image2} "${snap_name}_${i}"
 301
 302 testlog "TEST: disable mirror while daemon is stopped"
 303 stop_mirror ${CLUSTER1}
 304 stop_mirror ${CLUSTER2}
 305 set_pool_mirror_mode ${CLUSTER2} ${POOL} 'image'
 306 disable_mirror ${CLUSTER2} ${POOL} ${image}
 307 if [ -z "${RBD_MIRROR_USE_RBD_MIRROR}" ]; then
 308   test_image_present ${CLUSTER1} ${POOL} ${image} 'present'
 309 fi
 310 start_mirror ${CLUSTER1}
 311 wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'deleted'
 312 set_pool_mirror_mode ${CLUSTER2} ${POOL} 'pool'
 313 wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'present'
 314 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
 315
 316 testlog "TEST: simple image resync"
 317 request_resync_image ${CLUSTER1} ${POOL} ${image} image_id
 318 wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'deleted' ${image_id}
 319 wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'present'
 320 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
 321 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' 'master_position'
 322 compare_images ${POOL} ${image}
 323
 324 testlog "TEST: image resync while replayer is stopped"
 325 admin_daemon ${CLUSTER1} rbd mirror stop ${POOL}/${image}
 326 wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image}
 327 request_resync_image ${CLUSTER1} ${POOL} ${image} image_id
 328 admin_daemon ${CLUSTER1} rbd mirror start ${POOL}/${image}
 329 wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'deleted' ${image_id}
 330 admin_daemon ${CLUSTER1} rbd mirror start ${POOL}/${image}
 331 wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'present'
 332 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
 333 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' 'master_position'
 334 compare_images ${POOL} ${image}
 335
 336 testlog "TEST: request image resync while daemon is offline"
 337 stop_mirror ${CLUSTER1}
 338 request_resync_image ${CLUSTER1} ${POOL} ${image} image_id
 339 start_mirror ${CLUSTER1}
 340 wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'deleted' ${image_id}
 341 wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'present'
 342 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
 343 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' 'master_position'
 344 compare_images ${POOL} ${image}
 345
 346 testlog "TEST: client disconnect"
 347 image=laggy
 348 create_image ${CLUSTER2} ${POOL} ${image} 128 --journal-object-size 64K
 349 write_image ${CLUSTER2} ${POOL} ${image} 10
 350
 351 testlog " - replay stopped after disconnect"
 352 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
 353 wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image}
 354 test -n "$(get_mirror_position ${CLUSTER2} ${POOL} ${image})"
 355 disconnect_image ${CLUSTER2} ${POOL} ${image}
 356 test -z "$(get_mirror_position ${CLUSTER2} ${POOL} ${image})"
 357 wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image}
 358 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+error' 'disconnected'
 359
 360 testlog " - replay started after resync requested"
 361 request_resync_image ${CLUSTER1} ${POOL} ${image} image_id
 362 wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'deleted' ${image_id}
 363 wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'present'
 364 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
 365 wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image}
 366 test -n "$(get_mirror_position ${CLUSTER2} ${POOL} ${image})"
 367 compare_images ${POOL} ${image}
 368
 369 testlog " - disconnected after max_concurrent_object_sets reached"
 370 admin_daemon ${CLUSTER1} rbd mirror stop ${POOL}/${image}
 371 wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image}
 372 test -n "$(get_mirror_position ${CLUSTER2} ${POOL} ${image})"
 373 set_image_meta ${CLUSTER2} ${POOL} ${image} \
 374                conf_rbd_journal_max_concurrent_object_sets 1
 375 write_image ${CLUSTER2} ${POOL} ${image} 20 16384
 376 write_image ${CLUSTER2} ${POOL} ${image} 20 16384
 377 test -z "$(get_mirror_position ${CLUSTER2} ${POOL} ${image})"
 378 set_image_meta ${CLUSTER2} ${POOL} ${image} \
 379                conf_rbd_journal_max_concurrent_object_sets 0
 380
 381 testlog " - replay is still stopped (disconnected) after restart"
 382 admin_daemon ${CLUSTER1} rbd mirror start ${POOL}/${image}
 383 wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image}
 384 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+error' 'disconnected'
 385
 386 testlog " - replay started after resync requested"
 387 request_resync_image ${CLUSTER1} ${POOL} ${image} image_id
 388 wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'deleted' ${image_id}
 389 wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'present'
 390 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
 391 wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image}
 392 test -n "$(get_mirror_position ${CLUSTER2} ${POOL} ${image})"
 393 compare_images ${POOL} ${image}
 394
 395 testlog " - rbd_mirroring_resync_after_disconnect config option"
 396 set_image_meta ${CLUSTER2} ${POOL} ${image} \
 397                conf_rbd_mirroring_resync_after_disconnect true
 398 wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image}
 399 image_id=$(get_image_id ${CLUSTER1} ${pool} ${image})
 400 disconnect_image ${CLUSTER2} ${POOL} ${image}
 401 wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'deleted' ${image_id}
 402 wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'present'
 403 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
 404 wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image}
 405 test -n "$(get_mirror_position ${CLUSTER2} ${POOL} ${image})"
 406 compare_images ${POOL} ${image}
 407 set_image_meta ${CLUSTER2} ${POOL} ${image} \
 408                conf_rbd_mirroring_resync_after_disconnect false
 409 wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image}
 410 disconnect_image ${CLUSTER2} ${POOL} ${image}
 411 test -z "$(get_mirror_position ${CLUSTER2} ${POOL} ${image})"
 412 wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image}
 413 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+error' 'disconnected'
 414
 415 testlog "TEST: split-brain"
 416 image=split-brain
 417 create_image ${CLUSTER2} ${POOL} ${image}
 418 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' 'master_position'
 419 demote_image ${CLUSTER2} ${POOL} ${image}
 420 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+unknown'
 421 promote_image ${CLUSTER1} ${POOL} ${image}
 422 write_image ${CLUSTER1} ${POOL} ${image} 10
 423 demote_image ${CLUSTER1} ${POOL} ${image}
 424 promote_image ${CLUSTER2} ${POOL} ${image}
 425 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+error' 'split-brain'
 426 request_resync_image ${CLUSTER1} ${POOL} ${image} image_id
 427 wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' 'master_position'
 428
 429 testlog "TEST: no blacklists"
 430 CEPH_ARGS='--id admin' ceph --cluster ${CLUSTER1} osd blacklist ls 2>&1 | grep -q "listed 0 entries"
 431 CEPH_ARGS='--id admin' ceph --cluster ${CLUSTER2} osd blacklist ls 2>&1 | grep -q "listed 0 entries"
 432
 433 echo OK