src/ceph/qa/workunits/rbd/rbd_mirror_stress.sh

   1 #!/bin/sh
   2 #
   3 # rbd_mirror_stress.sh - stress test rbd-mirror daemon
   4 #
   5 # The following additional environment variables affect the test:
   6 #
   7 #  RBD_MIRROR_REDUCE_WRITES - if not empty, don't run the stress bench write
   8 #                             tool during the many image test
   9 #
  10
  11 IMAGE_COUNT=50
  12 export LOCKDEP=0
  13
  14 . $(dirname $0)/rbd_mirror_helpers.sh
  15
  16 create_snap()
  17 {
  18     local cluster=$1
  19     local pool=$2
  20     local image=$3
  21     local snap_name=$4
  22
  23     rbd --cluster ${cluster} -p ${pool} snap create ${image}@${snap_name} \
  24         --debug-rbd=20 --debug-journaler=20 2> ${TEMPDIR}/rbd-snap-create.log
  25 }
  26
  27 compare_image_snaps()
  28 {
  29     local pool=$1
  30     local image=$2
  31     local snap_name=$3
  32
  33     local rmt_export=${TEMPDIR}/${CLUSTER2}-${pool}-${image}.export
  34     local loc_export=${TEMPDIR}/${CLUSTER1}-${pool}-${image}.export
  35
  36     rm -f ${rmt_export} ${loc_export}
  37     rbd --cluster ${CLUSTER2} -p ${pool} export ${image}@${snap_name} ${rmt_export}
  38     rbd --cluster ${CLUSTER1} -p ${pool} export ${image}@${snap_name} ${loc_export}
  39     cmp ${rmt_export} ${loc_export}
  40     rm -f ${rmt_export} ${loc_export}
  41 }
  42
  43 wait_for_pool_images()
  44 {
  45     local cluster=$1
  46     local pool=$2
  47     local image_count=$3
  48     local s
  49     local count
  50     local last_count=0
  51
  52     while true; do
  53         for s in `seq 1 40`; do
  54             test $s -ne 1 && sleep 30
  55             count=$(rbd --cluster ${cluster} -p ${pool} mirror pool status | grep 'images: ' | cut -d' ' -f 2)
  56             test "${count}" = "${image_count}" && return 0
  57
  58             # reset timeout if making forward progress
  59             test $count -ne $last_count && break
  60         done
  61
  62         test $count -eq $last_count && break
  63         last_count=$count
  64     done
  65     rbd --cluster ${cluster} -p ${pool} mirror pool status --verbose >&2
  66     return 1
  67 }
  68
  69 wait_for_pool_healthy()
  70 {
  71     local cluster=$1
  72     local pool=$2
  73     local s
  74     local state
  75
  76     for s in `seq 1 40`; do
  77         test $s -ne 1 && sleep 30
  78         state=$(rbd --cluster ${cluster} -p ${pool} mirror pool status | grep 'health:' | cut -d' ' -f 2)
  79         test "${state}" = "ERROR" && break
  80         test "${state}" = "OK" && return 0
  81     done
  82     rbd --cluster ${cluster} -p ${pool} mirror pool status --verbose >&2
  83     return 1
  84 }
  85
  86 start_mirror ${CLUSTER1}
  87 start_mirror ${CLUSTER2}
  88
  89 testlog "TEST: add image and test replay after client crashes"
  90 image=test
  91 create_image ${CLUSTER2} ${POOL} ${image} '512M'
  92 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
  93
  94 for i in `seq 1 10`
  95 do
  96   stress_write_image ${CLUSTER2} ${POOL} ${image}
  97
  98   wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' 'master_position'
  99
 100   snap_name="snap${i}"
 101   create_snap ${CLUSTER2} ${POOL} ${image} ${snap_name}
 102   wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
 103   wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image}
 104   wait_for_snap_present ${CLUSTER1} ${POOL} ${image} ${snap_name}
 105   compare_image_snaps ${POOL} ${image} ${snap_name}
 106 done
 107
 108 for i in `seq 1 10`
 109 do
 110   snap_name="snap${i}"
 111   remove_snapshot ${CLUSTER2} ${POOL} ${image} ${snap_name}
 112 done
 113
 114 remove_image_retry ${CLUSTER2} ${POOL} ${image}
 115 wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'deleted'
 116
 117 testlog "TEST: create many images"
 118 snap_name="snap"
 119 for i in `seq 1 ${IMAGE_COUNT}`
 120 do
 121   image="image_${i}"
 122   create_image ${CLUSTER2} ${POOL} ${image} '128M'
 123   if [ -n "${RBD_MIRROR_REDUCE_WRITES}" ]; then
 124     write_image ${CLUSTER2} ${POOL} ${image} 100
 125   else
 126     stress_write_image ${CLUSTER2} ${POOL} ${image}
 127   fi
 128 done
 129
 130 wait_for_pool_images ${CLUSTER2} ${POOL} ${IMAGE_COUNT}
 131 wait_for_pool_healthy ${CLUSTER2} ${POOL}
 132
 133 wait_for_pool_images ${CLUSTER1} ${POOL} ${IMAGE_COUNT}
 134 wait_for_pool_healthy ${CLUSTER1} ${POOL}
 135
 136 testlog "TEST: compare many images"
 137 for i in `seq 1 ${IMAGE_COUNT}`
 138 do
 139   image="image_${i}"
 140   create_snap ${CLUSTER2} ${POOL} ${image} ${snap_name}
 141   wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
 142   wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image}
 143   wait_for_snap_present ${CLUSTER1} ${POOL} ${image} ${snap_name}
 144   compare_image_snaps ${POOL} ${image} ${snap_name}
 145 done
 146
 147 testlog "TEST: delete many images"
 148 for i in `seq 1 ${IMAGE_COUNT}`
 149 do
 150   image="image_${i}"
 151   remove_snapshot ${CLUSTER2} ${POOL} ${image} ${snap_name}
 152   remove_image_retry ${CLUSTER2} ${POOL} ${image}
 153 done
 154
 155 testlog "TEST: image deletions should propagate"
 156 wait_for_pool_images ${CLUSTER1} ${POOL} 0
 157 wait_for_pool_healthy ${CLUSTER1} ${POOL} 0
 158 for i in `seq 1 ${IMAGE_COUNT}`
 159 do
 160   image="image_${i}"
 161   wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'deleted'
 162 done
 163
 164 testlog "TEST: delete images during bootstrap"
 165 set_pool_mirror_mode ${CLUSTER1} ${POOL} 'image'
 166 set_pool_mirror_mode ${CLUSTER2} ${POOL} 'image'
 167
 168 start_mirror ${CLUSTER1}
 169 image=test
 170
 171 for i in `seq 1 10`
 172 do
 173   image="image_${i}"
 174   create_image ${CLUSTER2} ${POOL} ${image} '512M'
 175   enable_mirror ${CLUSTER2} ${POOL} ${image}
 176
 177   stress_write_image ${CLUSTER2} ${POOL} ${image}
 178   wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'present'
 179
 180   disable_mirror ${CLUSTER2} ${POOL} ${image}
 181   wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'deleted'
 182   purge_snapshots ${CLUSTER2} ${POOL} ${image}
 183   remove_image_retry ${CLUSTER2} ${POOL} ${image}
 184 done
 185
 186 echo OK