src/ceph/src/test/rbd_mirror/test_PoolWatcher.cc

   1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
   2 // vim: ts=8 sw=2 smarttab
   3
   4 #include "include/rados/librados.hpp"
   5 #include "include/rbd/librbd.hpp"
   6 #include "include/stringify.h"
   7 #include "test/rbd_mirror/test_fixture.h"
   8 #include "cls/rbd/cls_rbd_types.h"
   9 #include "cls/rbd/cls_rbd_client.h"
  10 #include "include/rbd_types.h"
  11 #include "librbd/internal.h"
  12 #include "librbd/ImageCtx.h"
  13 #include "librbd/ImageState.h"
  14 #include "librbd/Operations.h"
  15 #include "librbd/Utils.h"
  16 #include "librbd/api/Mirror.h"
  17 #include "common/Cond.h"
  18 #include "common/errno.h"
  19 #include "common/Mutex.h"
  20 #include "tools/rbd_mirror/PoolWatcher.h"
  21 #include "tools/rbd_mirror/Threads.h"
  22 #include "tools/rbd_mirror/types.h"
  23 #include "test/librados/test.h"
  24 #include "gtest/gtest.h"
  25 #include <boost/scope_exit.hpp>
  26 #include <iostream>
  27 #include <map>
  28 #include <memory>
  29 #include <set>
  30 #include <vector>
  31
  32 using rbd::mirror::ImageId;
  33 using rbd::mirror::ImageIds;
  34 using rbd::mirror::PoolWatcher;
  35 using rbd::mirror::peer_t;
  36 using rbd::mirror::RadosRef;
  37 using std::map;
  38 using std::set;
  39 using std::string;
  40
  41 void register_test_pool_watcher() {
  42 }
  43
  44 class TestPoolWatcher : public ::rbd::mirror::TestFixture {
  45 public:
  46
  47   TestPoolWatcher()
  48     : m_lock("TestPoolWatcherLock"), m_pool_watcher_listener(this),
  49       m_image_number(0), m_snap_number(0)
  50   {
  51     m_cluster = std::make_shared<librados::Rados>();
  52     EXPECT_EQ("", connect_cluster_pp(*m_cluster));
  53   }
  54
  55   void TearDown() override {
  56     if (m_pool_watcher) {
  57       C_SaferCond ctx;
  58       m_pool_watcher->shut_down(&ctx);
  59       EXPECT_EQ(0, ctx.wait());
  60     }
  61
  62     m_cluster->wait_for_latest_osdmap();
  63     for (auto& pool : m_pools) {
  64       EXPECT_EQ(0, m_cluster->pool_delete(pool.c_str()));
  65     }
  66
  67     TestFixture::TearDown();
  68   }
  69
  70   struct PoolWatcherListener : public PoolWatcher<>::Listener {
  71     TestPoolWatcher *test;
  72     Cond cond;
  73     ImageIds image_ids;
  74
  75     PoolWatcherListener(TestPoolWatcher *test) : test(test) {
  76     }
  77
  78     void handle_update(const std::string &mirror_uuid,
  79                        ImageIds &&added_image_ids,
  80                        ImageIds &&removed_image_ids) override {
  81       Mutex::Locker locker(test->m_lock);
  82       for (auto &image_id : removed_image_ids) {
  83         image_ids.erase(image_id);
  84       }
  85       image_ids.insert(added_image_ids.begin(), added_image_ids.end());
  86       cond.Signal();
  87     }
  88   };
  89
  90   void create_pool(bool enable_mirroring, const peer_t &peer, string *name=nullptr) {
  91     string pool_name = get_temp_pool_name("test-rbd-mirror-");
  92     ASSERT_EQ(0, m_cluster->pool_create(pool_name.c_str()));
  93
  94     int64_t pool_id = m_cluster->pool_lookup(pool_name.c_str());
  95     ASSERT_GE(pool_id, 0);
  96     m_pools.insert(pool_name);
  97
  98     librados::IoCtx ioctx;
  99     ASSERT_EQ(0, m_cluster->ioctx_create2(pool_id, ioctx));
 100     ioctx.application_enable("rbd", true);
 101
 102     m_pool_watcher.reset(new PoolWatcher<>(m_threads, ioctx,
 103                                            m_pool_watcher_listener));
 104
 105     if (enable_mirroring) {
 106       ASSERT_EQ(0, librbd::api::Mirror<>::mode_set(ioctx,
 107                                                    RBD_MIRROR_MODE_POOL));
 108       std::string uuid;
 109       ASSERT_EQ(0, librbd::api::Mirror<>::peer_add(ioctx, &uuid,
 110                                                    peer.cluster_name,
 111                                                    peer.client_name));
 112     }
 113     if (name != nullptr) {
 114       *name = pool_name;
 115     }
 116
 117     m_pool_watcher->init();
 118   }
 119
 120   string get_image_id(librados::IoCtx *ioctx, const string &image_name) {
 121     string obj = librbd::util::id_obj_name(image_name);
 122     string id;
 123     EXPECT_EQ(0, librbd::cls_client::get_id(ioctx, obj, &id));
 124     return id;
 125   }
 126
 127   void create_image(const string &pool_name, bool mirrored=true,
 128                     string *image_name=nullptr) {
 129     uint64_t features = librbd::util::get_rbd_default_features(g_ceph_context);
 130     string name = "image" + stringify(++m_image_number);
 131     if (mirrored) {
 132       features |= RBD_FEATURE_EXCLUSIVE_LOCK | RBD_FEATURE_JOURNALING;
 133     }
 134
 135     librados::IoCtx ioctx;
 136     ASSERT_EQ(0, m_cluster->ioctx_create(pool_name.c_str(), ioctx));
 137     int order = 0;
 138     ASSERT_EQ(0, librbd::create(ioctx, name.c_str(), 1 << 22, false,
 139                                 features, &order, 0, 0));
 140     if (mirrored) {
 141       librbd::Image image;
 142       librbd::RBD rbd;
 143       rbd.open(ioctx, image, name.c_str());
 144       image.mirror_image_enable();
 145
 146       librbd::mirror_image_info_t mirror_image_info;
 147       ASSERT_EQ(0, image.mirror_image_get_info(&mirror_image_info,
 148                                                sizeof(mirror_image_info)));
 149       image.close();
 150
 151       m_mirrored_images.insert(ImageId(
 152         mirror_image_info.global_id, get_image_id(&ioctx, name)));
 153     }
 154     if (image_name != nullptr)
 155       *image_name = name;
 156   }
 157
 158   void clone_image(const string &parent_pool_name,
 159                    const string &parent_image_name,
 160                    const string &clone_pool_name,
 161                    bool mirrored=true,
 162                    string *image_name=nullptr) {
 163     librados::IoCtx pioctx, cioctx;
 164     ASSERT_EQ(0, m_cluster->ioctx_create(parent_pool_name.c_str(), pioctx));
 165     ASSERT_EQ(0, m_cluster->ioctx_create(clone_pool_name.c_str(), cioctx));
 166
 167     string snap_name = "snap" + stringify(++m_snap_number);
 168     {
 169       librbd::ImageCtx *ictx = new librbd::ImageCtx(parent_image_name.c_str(),
 170                                                     "", "", pioctx, false);
 171       ictx->state->open(false);
 172       EXPECT_EQ(0, ictx->operations->snap_create(cls::rbd::UserSnapshotNamespace(),
 173                                                  snap_name.c_str()));
 174       EXPECT_EQ(0, ictx->operations->snap_protect(cls::rbd::UserSnapshotNamespace(),
 175                                                   snap_name.c_str()));
 176       ictx->state->close();
 177     }
 178
 179     uint64_t features = librbd::util::get_rbd_default_features(g_ceph_context);
 180     string name = "clone" + stringify(++m_image_number);
 181     if (mirrored) {
 182       features |= RBD_FEATURE_EXCLUSIVE_LOCK | RBD_FEATURE_JOURNALING;
 183     }
 184     int order = 0;
 185     librbd::clone(pioctx, parent_image_name.c_str(), snap_name.c_str(),
 186                   cioctx, name.c_str(), features, &order, 0, 0);
 187     if (mirrored) {
 188       librbd::Image image;
 189       librbd::RBD rbd;
 190       rbd.open(cioctx, image, name.c_str());
 191       image.mirror_image_enable();
 192
 193       librbd::mirror_image_info_t mirror_image_info;
 194       ASSERT_EQ(0, image.mirror_image_get_info(&mirror_image_info,
 195                                                sizeof(mirror_image_info)));
 196       image.close();
 197
 198       m_mirrored_images.insert(ImageId(
 199         mirror_image_info.global_id, get_image_id(&cioctx, name)));
 200     }
 201     if (image_name != nullptr)
 202       *image_name = name;
 203   }
 204
 205   void check_images() {
 206     Mutex::Locker l(m_lock);
 207     while (m_mirrored_images != m_pool_watcher_listener.image_ids) {
 208       if (m_pool_watcher_listener.cond.WaitInterval(
 209             m_lock, utime_t(10, 0)) != 0) {
 210         break;
 211       }
 212     }
 213
 214     ASSERT_EQ(m_mirrored_images, m_pool_watcher_listener.image_ids);
 215   }
 216
 217   Mutex m_lock;
 218   RadosRef m_cluster;
 219   PoolWatcherListener m_pool_watcher_listener;
 220   unique_ptr<PoolWatcher<> > m_pool_watcher;
 221
 222   set<string> m_pools;
 223   ImageIds m_mirrored_images;
 224
 225   uint64_t m_image_number;
 226   uint64_t m_snap_number;
 227 };
 228
 229 TEST_F(TestPoolWatcher, EmptyPool) {
 230   string uuid1 = "00000000-0000-0000-0000-000000000001";
 231   peer_t site1(uuid1, "site1", "mirror1");
 232   create_pool(true, site1);
 233   check_images();
 234 }
 235
 236 TEST_F(TestPoolWatcher, ReplicatedPools) {
 237   string uuid1 = "00000000-0000-0000-0000-000000000001";
 238   peer_t site1(uuid1, "site1", "mirror1");
 239   string first_pool, local_pool, last_pool;
 240   create_pool(true, site1, &first_pool);
 241   check_images();
 242   create_image(first_pool);
 243   check_images();
 244   string parent_image, parent_image2;
 245   create_image(first_pool, true, &parent_image);
 246   check_images();
 247   clone_image(first_pool, parent_image, first_pool);
 248   check_images();
 249   clone_image(first_pool, parent_image, first_pool, true, &parent_image2);
 250   check_images();
 251   create_image(first_pool, false);
 252   check_images();
 253 }