src/ceph/src/test/rbd_mirror/test_mock_PoolWatcher.cc

   1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
   2 // vim: ts=8 sw=2 smarttab
   3
   4 #include "test/rbd_mirror/test_mock_fixture.h"
   5 #include "test/librados_test_stub/MockTestMemIoCtxImpl.h"
   6 #include "test/librados_test_stub/MockTestMemRadosClient.h"
   7 #include "test/librbd/mock/MockImageCtx.h"
   8 #include "test/rbd_mirror/mock/MockContextWQ.h"
   9 #include "test/rbd_mirror/mock/MockSafeTimer.h"
  10 #include "librbd/MirroringWatcher.h"
  11 #include "tools/rbd_mirror/Threads.h"
  12 #include "tools/rbd_mirror/PoolWatcher.h"
  13 #include "tools/rbd_mirror/pool_watcher/RefreshImagesRequest.h"
  14 #include "include/stringify.h"
  15
  16 namespace librbd {
  17 namespace {
  18
  19 struct MockTestImageCtx : public librbd::MockImageCtx {
  20   MockTestImageCtx(librbd::ImageCtx &image_ctx)
  21     : librbd::MockImageCtx(image_ctx) {
  22   }
  23 };
  24
  25 } // anonymous namespace
  26
  27 struct MockMirroringWatcher {
  28   static MockMirroringWatcher *s_instance;
  29   static MockMirroringWatcher &get_instance() {
  30     assert(s_instance != nullptr);
  31     return *s_instance;
  32   }
  33
  34   MockMirroringWatcher() {
  35     s_instance = this;
  36   }
  37
  38   MOCK_CONST_METHOD0(is_unregistered, bool());
  39   MOCK_METHOD1(register_watch, void(Context*));
  40   MOCK_METHOD1(unregister_watch, void(Context*));
  41
  42   MOCK_CONST_METHOD0(get_oid, std::string());
  43 };
  44
  45 template <>
  46 struct MirroringWatcher<MockTestImageCtx> {
  47   static MirroringWatcher *s_instance;
  48
  49   MirroringWatcher(librados::IoCtx &io_ctx, ::MockContextWQ *work_queue) {
  50     s_instance = this;
  51   }
  52   virtual ~MirroringWatcher() {
  53   }
  54
  55   static MirroringWatcher<MockTestImageCtx> &get_instance() {
  56     assert(s_instance != nullptr);
  57     return *s_instance;
  58   }
  59
  60   virtual void handle_rewatch_complete(int r) = 0;
  61
  62   virtual void handle_mode_updated(cls::rbd::MirrorMode mirror_mode) = 0;
  63   virtual void handle_image_updated(cls::rbd::MirrorImageState state,
  64                                     const std::string &remote_image_id,
  65                                     const std::string &global_image_id) = 0;
  66
  67   bool is_unregistered() const {
  68     return MockMirroringWatcher::get_instance().is_unregistered();
  69   }
  70   void register_watch(Context *ctx) {
  71     MockMirroringWatcher::get_instance().register_watch(ctx);
  72   }
  73   void unregister_watch(Context *ctx) {
  74     MockMirroringWatcher::get_instance().unregister_watch(ctx);
  75   }
  76   std::string get_oid() const {
  77     return MockMirroringWatcher::get_instance().get_oid();
  78   }
  79 };
  80
  81 MockMirroringWatcher *MockMirroringWatcher::s_instance = nullptr;
  82 MirroringWatcher<MockTestImageCtx> *MirroringWatcher<MockTestImageCtx>::s_instance = nullptr;
  83
  84 } // namespace librbd
  85
  86 namespace rbd {
  87 namespace mirror {
  88
  89 template <>
  90 struct Threads<librbd::MockTestImageCtx> {
  91   MockSafeTimer *timer;
  92   Mutex &timer_lock;
  93
  94   MockContextWQ *work_queue;
  95
  96   Threads(Threads<librbd::ImageCtx> *threads)
  97     : timer(new MockSafeTimer()),
  98       timer_lock(threads->timer_lock),
  99       work_queue(new MockContextWQ()) {
 100   }
 101   ~Threads() {
 102     delete timer;
 103     delete work_queue;
 104   }
 105 };
 106
 107 namespace pool_watcher {
 108
 109 template <>
 110 struct RefreshImagesRequest<librbd::MockTestImageCtx> {
 111   ImageIds *image_ids = nullptr;
 112   Context *on_finish = nullptr;
 113   static RefreshImagesRequest *s_instance;
 114   static RefreshImagesRequest *create(librados::IoCtx &io_ctx,
 115                                       ImageIds *image_ids,
 116                                       Context *on_finish) {
 117     assert(s_instance != nullptr);
 118     s_instance->image_ids = image_ids;
 119     s_instance->on_finish = on_finish;
 120     return s_instance;
 121   }
 122
 123   MOCK_METHOD0(send, void());
 124
 125   RefreshImagesRequest() {
 126     s_instance = this;
 127   }
 128 };
 129
 130 RefreshImagesRequest<librbd::MockTestImageCtx> *RefreshImagesRequest<librbd::MockTestImageCtx>::s_instance = nullptr;
 131
 132 } // namespace pool_watcher
 133
 134 } // namespace mirror
 135 } // namespace rbd
 136
 137 // template definitions
 138 #include "tools/rbd_mirror/PoolWatcher.cc"
 139
 140 namespace rbd {
 141 namespace mirror {
 142
 143 using ::testing::_;
 144 using ::testing::DoAll;
 145 using ::testing::InSequence;
 146 using ::testing::Invoke;
 147 using ::testing::Return;
 148 using ::testing::ReturnArg;
 149 using ::testing::StrEq;
 150 using ::testing::WithArg;
 151 using ::testing::WithoutArgs;
 152
 153 class TestMockPoolWatcher : public TestMockFixture {
 154 public:
 155   typedef PoolWatcher<librbd::MockTestImageCtx> MockPoolWatcher;
 156   typedef Threads<librbd::MockTestImageCtx> MockThreads;
 157   typedef pool_watcher::RefreshImagesRequest<librbd::MockTestImageCtx> MockRefreshImagesRequest;
 158   typedef librbd::MockMirroringWatcher MockMirroringWatcher;
 159   typedef librbd::MirroringWatcher<librbd::MockTestImageCtx> MirroringWatcher;
 160
 161   struct MockListener : MockPoolWatcher::Listener {
 162     TestMockPoolWatcher *test;
 163
 164     MockListener(TestMockPoolWatcher *test) : test(test) {
 165     }
 166
 167     MOCK_METHOD3(mock_handle_update, void(const std::string &, const ImageIds &,
 168                                           const ImageIds &));
 169     void handle_update(const std::string &mirror_uuid,
 170                        ImageIds &&added_image_ids,
 171                        ImageIds &&removed_image_ids) override {
 172       mock_handle_update(mirror_uuid, added_image_ids, removed_image_ids);
 173     }
 174   };
 175
 176   TestMockPoolWatcher() : m_lock("TestMockPoolWatcher::m_lock") {
 177   }
 178
 179   void expect_work_queue(MockThreads &mock_threads) {
 180     EXPECT_CALL(*mock_threads.work_queue, queue(_, _))
 181       .WillRepeatedly(Invoke([this](Context *ctx, int r) {
 182           m_threads->work_queue->queue(ctx, r);
 183         }));
 184   }
 185
 186   void expect_mirroring_watcher_is_unregistered(MockMirroringWatcher &mock_mirroring_watcher,
 187                                                 bool unregistered) {
 188     EXPECT_CALL(mock_mirroring_watcher, is_unregistered())
 189       .WillOnce(Return(unregistered));
 190   }
 191
 192   void expect_mirroring_watcher_register(MockMirroringWatcher &mock_mirroring_watcher,
 193                                          int r) {
 194     EXPECT_CALL(mock_mirroring_watcher, register_watch(_))
 195       .WillOnce(CompleteContext(r));
 196   }
 197
 198   void expect_mirroring_watcher_unregister(MockMirroringWatcher &mock_mirroring_watcher,
 199                                          int r) {
 200     EXPECT_CALL(mock_mirroring_watcher, unregister_watch(_))
 201       .WillOnce(CompleteContext(r));
 202   }
 203
 204   void expect_refresh_images(MockRefreshImagesRequest &request,
 205                              const ImageIds &image_ids, int r) {
 206     EXPECT_CALL(request, send())
 207       .WillOnce(Invoke([&request, image_ids, r]() {
 208           *request.image_ids = image_ids;
 209           request.on_finish->complete(r);
 210         }));
 211   }
 212
 213   void expect_listener_handle_update(MockListener &mock_listener,
 214                                      const std::string &mirror_uuid,
 215                                      const ImageIds &added_image_ids,
 216                                      const ImageIds &removed_image_ids) {
 217     EXPECT_CALL(mock_listener, mock_handle_update(mirror_uuid, added_image_ids,
 218                                                   removed_image_ids))
 219       .WillOnce(WithoutArgs(Invoke([this]() {
 220           Mutex::Locker locker(m_lock);
 221           ++m_update_count;
 222           m_cond.Signal();
 223         })));
 224   }
 225
 226   void expect_mirror_uuid_get(librados::IoCtx &io_ctx,
 227                               const std::string &uuid, int r) {
 228     bufferlist out_bl;
 229     ::encode(uuid, out_bl);
 230
 231     EXPECT_CALL(get_mock_io_ctx(io_ctx),
 232                 exec(RBD_MIRRORING, _, StrEq("rbd"), StrEq("mirror_uuid_get"),
 233                      _, _, _))
 234       .WillOnce(DoAll(WithArg<5>(Invoke([this, out_bl](bufferlist *bl) {
 235                           *bl = out_bl;
 236                         })),
 237                       Return(r)));
 238   }
 239
 240   void expect_timer_add_event(MockThreads &mock_threads) {
 241     EXPECT_CALL(*mock_threads.timer, add_event_after(_, _))
 242       .WillOnce(DoAll(WithArg<1>(Invoke([this](Context *ctx) {
 243                         auto wrapped_ctx =
 244                           new FunctionContext([this, ctx](int r) {
 245                               Mutex::Locker timer_locker(m_threads->timer_lock);
 246                               ctx->complete(r);
 247                             });
 248                         m_threads->work_queue->queue(wrapped_ctx, 0);
 249                       })),
 250                       ReturnArg<1>()));
 251   }
 252
 253   int when_shut_down(MockPoolWatcher &mock_pool_watcher) {
 254     C_SaferCond ctx;
 255     mock_pool_watcher.shut_down(&ctx);
 256     return ctx.wait();
 257   }
 258
 259   bool wait_for_update(uint32_t count) {
 260     Mutex::Locker locker(m_lock);
 261     while (m_update_count < count) {
 262       if (m_cond.WaitInterval(m_lock, utime_t(10, 0)) != 0) {
 263         break;
 264       }
 265     }
 266     if (m_update_count < count) {
 267       return false;
 268     }
 269
 270     m_update_count -= count;
 271     return true;
 272   }
 273
 274   Mutex m_lock;
 275   Cond m_cond;
 276   uint32_t m_update_count = 0;
 277 };
 278
 279 TEST_F(TestMockPoolWatcher, EmptyPool) {
 280   MockThreads mock_threads(m_threads);
 281   expect_work_queue(mock_threads);
 282
 283   InSequence seq;
 284   MockMirroringWatcher mock_mirroring_watcher;
 285   expect_mirroring_watcher_is_unregistered(mock_mirroring_watcher, true);
 286   expect_mirroring_watcher_register(mock_mirroring_watcher, 0);
 287
 288   MockRefreshImagesRequest mock_refresh_images_request;
 289   expect_refresh_images(mock_refresh_images_request, {}, 0);
 290   expect_mirror_uuid_get(m_remote_io_ctx, "remote uuid", 0);
 291
 292   MockListener mock_listener(this);
 293   expect_listener_handle_update(mock_listener, "remote uuid", {}, {});
 294
 295   MockPoolWatcher mock_pool_watcher(&mock_threads, m_remote_io_ctx,
 296                                     mock_listener);
 297   C_SaferCond ctx;
 298   mock_pool_watcher.init(&ctx);
 299   ASSERT_EQ(0, ctx.wait());
 300
 301   ASSERT_TRUE(wait_for_update(1));
 302   expect_mirroring_watcher_unregister(mock_mirroring_watcher, 0);
 303   ASSERT_EQ(0, when_shut_down(mock_pool_watcher));
 304 }
 305
 306 TEST_F(TestMockPoolWatcher, NonEmptyPool) {
 307   MockThreads mock_threads(m_threads);
 308   expect_work_queue(mock_threads);
 309
 310   InSequence seq;
 311   MockMirroringWatcher mock_mirroring_watcher;
 312   expect_mirroring_watcher_is_unregistered(mock_mirroring_watcher, true);
 313   expect_mirroring_watcher_register(mock_mirroring_watcher, 0);
 314
 315   ImageIds image_ids{
 316     {"global id 1", "remote id 1"},
 317     {"global id 2", "remote id 2"}};
 318   MockRefreshImagesRequest mock_refresh_images_request;
 319   expect_refresh_images(mock_refresh_images_request, image_ids, 0);
 320   expect_mirror_uuid_get(m_remote_io_ctx, "remote uuid", 0);
 321
 322   MockListener mock_listener(this);
 323   expect_listener_handle_update(mock_listener, "remote uuid", image_ids, {});
 324
 325   MockPoolWatcher mock_pool_watcher(&mock_threads, m_remote_io_ctx,
 326                                     mock_listener);
 327   C_SaferCond ctx;
 328   mock_pool_watcher.init(&ctx);
 329   ASSERT_EQ(0, ctx.wait());
 330
 331   ASSERT_TRUE(wait_for_update(1));
 332   expect_mirroring_watcher_unregister(mock_mirroring_watcher, 0);
 333   ASSERT_EQ(0, when_shut_down(mock_pool_watcher));
 334 }
 335
 336 TEST_F(TestMockPoolWatcher, NotifyDuringRefresh) {
 337   MockThreads mock_threads(m_threads);
 338   expect_work_queue(mock_threads);
 339
 340   InSequence seq;
 341   MockMirroringWatcher mock_mirroring_watcher;
 342   expect_mirroring_watcher_is_unregistered(mock_mirroring_watcher, true);
 343   expect_mirroring_watcher_register(mock_mirroring_watcher, 0);
 344
 345   ImageIds image_ids{
 346     {"global id 1", "remote id 1"},
 347     {"global id 2", "remote id 2"}};
 348   MockRefreshImagesRequest mock_refresh_images_request;
 349   bool refresh_sent = false;
 350   EXPECT_CALL(mock_refresh_images_request, send())
 351     .WillOnce(Invoke([this, &mock_refresh_images_request, &image_ids,
 352                       &refresh_sent]() {
 353        *mock_refresh_images_request.image_ids = image_ids;
 354
 355         Mutex::Locker locker(m_lock);
 356         refresh_sent = true;
 357         m_cond.Signal();
 358       }));
 359
 360   expect_mirror_uuid_get(m_remote_io_ctx, "remote uuid", 0);
 361
 362   MockListener mock_listener(this);
 363   image_ids = {
 364     {"global id 1", "remote id 1a"},
 365     {"global id 3", "remote id 3"}};
 366   expect_listener_handle_update(mock_listener, "remote uuid", image_ids, {});
 367
 368   MockPoolWatcher mock_pool_watcher(&mock_threads, m_remote_io_ctx,
 369                                     mock_listener);
 370   mock_pool_watcher.init(nullptr);
 371
 372   {
 373     Mutex::Locker locker(m_lock);
 374     while (!refresh_sent) {
 375       m_cond.Wait(m_lock);
 376     }
 377   }
 378
 379   MirroringWatcher::get_instance().handle_image_updated(
 380     cls::rbd::MIRROR_IMAGE_STATE_DISABLING, "remote id 2", "global id 2");
 381   MirroringWatcher::get_instance().handle_image_updated(
 382     cls::rbd::MIRROR_IMAGE_STATE_ENABLED, "remote id 1a", "global id 1");
 383   MirroringWatcher::get_instance().handle_image_updated(
 384     cls::rbd::MIRROR_IMAGE_STATE_ENABLED, "remote id 3", "global id 3");
 385
 386   mock_refresh_images_request.on_finish->complete(0);
 387   ASSERT_TRUE(wait_for_update(1));
 388
 389   expect_mirroring_watcher_unregister(mock_mirroring_watcher, 0);
 390   ASSERT_EQ(0, when_shut_down(mock_pool_watcher));
 391 }
 392
 393 TEST_F(TestMockPoolWatcher, Notify) {
 394   MockThreads mock_threads(m_threads);
 395
 396   InSequence seq;
 397   MockMirroringWatcher mock_mirroring_watcher;
 398   expect_mirroring_watcher_is_unregistered(mock_mirroring_watcher, true);
 399   expect_mirroring_watcher_register(mock_mirroring_watcher, 0);
 400
 401   ImageIds image_ids{
 402     {"global id 1", "remote id 1"},
 403     {"global id 2", "remote id 2"}};
 404   MockRefreshImagesRequest mock_refresh_images_request;
 405   expect_refresh_images(mock_refresh_images_request, image_ids, 0);
 406   expect_mirror_uuid_get(m_remote_io_ctx, "remote uuid", 0);
 407
 408   EXPECT_CALL(*mock_threads.work_queue, queue(_, _))
 409     .WillOnce(Invoke([this](Context *ctx, int r) {
 410         m_threads->work_queue->queue(ctx, r);
 411       }));
 412
 413   MockListener mock_listener(this);
 414   expect_listener_handle_update(mock_listener, "remote uuid", image_ids, {});
 415
 416   Context *notify_ctx = nullptr;
 417   EXPECT_CALL(*mock_threads.work_queue, queue(_, _))
 418     .WillOnce(Invoke([this, &notify_ctx](Context *ctx, int r) {
 419         Mutex::Locker locker(m_lock);
 420         ASSERT_EQ(nullptr, notify_ctx);
 421         notify_ctx = ctx;
 422         m_cond.Signal();
 423       }));
 424   expect_listener_handle_update(
 425     mock_listener, "remote uuid",
 426     {{"global id 1", "remote id 1a"}, {"global id 3", "remote id 3"}},
 427     {{"global id 1", "remote id 1"}, {"global id 2", "remote id 2"}});
 428
 429   MockPoolWatcher mock_pool_watcher(&mock_threads, m_remote_io_ctx,
 430                                     mock_listener);
 431   C_SaferCond ctx;
 432   mock_pool_watcher.init(&ctx);
 433   ASSERT_EQ(0, ctx.wait());
 434   ASSERT_TRUE(wait_for_update(1));
 435
 436   C_SaferCond flush_ctx;
 437   m_threads->work_queue->queue(&flush_ctx, 0);
 438   ASSERT_EQ(0, flush_ctx.wait());
 439
 440   MirroringWatcher::get_instance().handle_image_updated(
 441     cls::rbd::MIRROR_IMAGE_STATE_DISABLING, "remote id 2", "global id 2");
 442   MirroringWatcher::get_instance().handle_image_updated(
 443     cls::rbd::MIRROR_IMAGE_STATE_DISABLED, "remote id 2", "global id 2");
 444   MirroringWatcher::get_instance().handle_image_updated(
 445     cls::rbd::MIRROR_IMAGE_STATE_ENABLED, "remote id 1a", "global id 1");
 446   MirroringWatcher::get_instance().handle_image_updated(
 447     cls::rbd::MIRROR_IMAGE_STATE_ENABLED, "remote id 3", "global id 3");
 448   notify_ctx->complete(0);
 449
 450   ASSERT_TRUE(wait_for_update(1));
 451
 452   expect_mirroring_watcher_unregister(mock_mirroring_watcher, 0);
 453   ASSERT_EQ(0, when_shut_down(mock_pool_watcher));
 454 }
 455
 456 TEST_F(TestMockPoolWatcher, RegisterWatcherBlacklist) {
 457   MockThreads mock_threads(m_threads);
 458   expect_work_queue(mock_threads);
 459
 460   InSequence seq;
 461   MockMirroringWatcher mock_mirroring_watcher;
 462   expect_mirroring_watcher_is_unregistered(mock_mirroring_watcher, true);
 463   expect_mirroring_watcher_register(mock_mirroring_watcher, -EBLACKLISTED);
 464
 465   MockListener mock_listener(this);
 466   MockPoolWatcher mock_pool_watcher(&mock_threads, m_remote_io_ctx,
 467                                     mock_listener);
 468   C_SaferCond ctx;
 469   mock_pool_watcher.init(&ctx);
 470   ASSERT_EQ(-EBLACKLISTED, ctx.wait());
 471   ASSERT_TRUE(mock_pool_watcher.is_blacklisted());
 472
 473   expect_mirroring_watcher_unregister(mock_mirroring_watcher, 0);
 474   ASSERT_EQ(0, when_shut_down(mock_pool_watcher));
 475 }
 476
 477 TEST_F(TestMockPoolWatcher, RegisterWatcherMissing) {
 478   MockThreads mock_threads(m_threads);
 479   expect_work_queue(mock_threads);
 480
 481   InSequence seq;
 482   MockMirroringWatcher mock_mirroring_watcher;
 483   expect_mirroring_watcher_is_unregistered(mock_mirroring_watcher, true);
 484   expect_mirroring_watcher_register(mock_mirroring_watcher, -ENOENT);
 485   expect_timer_add_event(mock_threads);
 486
 487   expect_mirroring_watcher_is_unregistered(mock_mirroring_watcher, true);
 488   expect_mirroring_watcher_register(mock_mirroring_watcher, 0);
 489
 490   MockRefreshImagesRequest mock_refresh_images_request;
 491   expect_refresh_images(mock_refresh_images_request, {}, 0);
 492   expect_mirror_uuid_get(m_remote_io_ctx, "remote uuid", 0);
 493
 494   MockListener mock_listener(this);
 495   expect_listener_handle_update(mock_listener, "remote uuid", {}, {});
 496
 497   MockPoolWatcher mock_pool_watcher(&mock_threads, m_remote_io_ctx,
 498                                     mock_listener);
 499   C_SaferCond ctx;
 500   mock_pool_watcher.init(&ctx);
 501   ASSERT_EQ(0, ctx.wait());
 502
 503   ASSERT_TRUE(wait_for_update(1));
 504   expect_mirroring_watcher_unregister(mock_mirroring_watcher, 0);
 505   ASSERT_EQ(0, when_shut_down(mock_pool_watcher));
 506 }
 507
 508 TEST_F(TestMockPoolWatcher, RegisterWatcherError) {
 509   MockThreads mock_threads(m_threads);
 510   expect_work_queue(mock_threads);
 511
 512   InSequence seq;
 513   MockMirroringWatcher mock_mirroring_watcher;
 514   expect_mirroring_watcher_is_unregistered(mock_mirroring_watcher, true);
 515   expect_mirroring_watcher_register(mock_mirroring_watcher, -EINVAL);
 516   expect_timer_add_event(mock_threads);
 517
 518   expect_mirroring_watcher_is_unregistered(mock_mirroring_watcher, true);
 519   expect_mirroring_watcher_register(mock_mirroring_watcher, 0);
 520
 521   MockRefreshImagesRequest mock_refresh_images_request;
 522   expect_refresh_images(mock_refresh_images_request, {}, 0);
 523   expect_mirror_uuid_get(m_remote_io_ctx, "remote uuid", 0);
 524
 525   MockListener mock_listener(this);
 526   expect_listener_handle_update(mock_listener, "remote uuid", {}, {});
 527
 528   MockPoolWatcher mock_pool_watcher(&mock_threads, m_remote_io_ctx,
 529                                     mock_listener);
 530   C_SaferCond ctx;
 531   mock_pool_watcher.init(&ctx);
 532   ASSERT_EQ(0, ctx.wait());
 533
 534   ASSERT_TRUE(wait_for_update(1));
 535   expect_mirroring_watcher_unregister(mock_mirroring_watcher, 0);
 536   ASSERT_EQ(0, when_shut_down(mock_pool_watcher));
 537 }
 538
 539 TEST_F(TestMockPoolWatcher, RefreshBlacklist) {
 540   MockThreads mock_threads(m_threads);
 541   expect_work_queue(mock_threads);
 542
 543   InSequence seq;
 544   MockMirroringWatcher mock_mirroring_watcher;
 545   expect_mirroring_watcher_is_unregistered(mock_mirroring_watcher, true);
 546   expect_mirroring_watcher_register(mock_mirroring_watcher, 0);
 547
 548   MockRefreshImagesRequest mock_refresh_images_request;
 549   expect_refresh_images(mock_refresh_images_request, {}, -EBLACKLISTED);
 550
 551   MockListener mock_listener(this);
 552   MockPoolWatcher mock_pool_watcher(&mock_threads, m_remote_io_ctx,
 553                                     mock_listener);
 554   C_SaferCond ctx;
 555   mock_pool_watcher.init(&ctx);
 556   ASSERT_EQ(-EBLACKLISTED, ctx.wait());
 557   ASSERT_TRUE(mock_pool_watcher.is_blacklisted());
 558
 559   expect_mirroring_watcher_unregister(mock_mirroring_watcher, 0);
 560   ASSERT_EQ(0, when_shut_down(mock_pool_watcher));
 561 }
 562
 563 TEST_F(TestMockPoolWatcher, RefreshMissing) {
 564   MockThreads mock_threads(m_threads);
 565   expect_work_queue(mock_threads);
 566
 567   InSequence seq;
 568   MockMirroringWatcher mock_mirroring_watcher;
 569   expect_mirroring_watcher_is_unregistered(mock_mirroring_watcher, true);
 570   expect_mirroring_watcher_register(mock_mirroring_watcher, 0);
 571
 572   MockRefreshImagesRequest mock_refresh_images_request;
 573   expect_refresh_images(mock_refresh_images_request, {}, -ENOENT);
 574   expect_mirror_uuid_get(m_remote_io_ctx, "remote uuid", 0);
 575
 576   MockListener mock_listener(this);
 577   expect_listener_handle_update(mock_listener, "remote uuid", {}, {});
 578
 579   MockPoolWatcher mock_pool_watcher(&mock_threads, m_remote_io_ctx,
 580                                     mock_listener);
 581   C_SaferCond ctx;
 582   mock_pool_watcher.init(&ctx);
 583   ASSERT_EQ(0, ctx.wait());
 584
 585   ASSERT_TRUE(wait_for_update(1));
 586   expect_mirroring_watcher_unregister(mock_mirroring_watcher, 0);
 587   ASSERT_EQ(0, when_shut_down(mock_pool_watcher));
 588 }
 589
 590 TEST_F(TestMockPoolWatcher, RefreshError) {
 591   MockThreads mock_threads(m_threads);
 592   expect_work_queue(mock_threads);
 593
 594   InSequence seq;
 595   MockMirroringWatcher mock_mirroring_watcher;
 596   expect_mirroring_watcher_is_unregistered(mock_mirroring_watcher, true);
 597   expect_mirroring_watcher_register(mock_mirroring_watcher, 0);
 598
 599   MockRefreshImagesRequest mock_refresh_images_request;
 600   expect_refresh_images(mock_refresh_images_request, {}, -EINVAL);
 601   expect_timer_add_event(mock_threads);
 602
 603   expect_mirroring_watcher_is_unregistered(mock_mirroring_watcher, false);
 604   expect_refresh_images(mock_refresh_images_request, {}, 0);
 605   expect_mirror_uuid_get(m_remote_io_ctx, "remote uuid", 0);
 606
 607   MockListener mock_listener(this);
 608   expect_listener_handle_update(mock_listener, "remote uuid", {}, {});
 609
 610   MockPoolWatcher mock_pool_watcher(&mock_threads, m_remote_io_ctx,
 611                                     mock_listener);
 612   C_SaferCond ctx;
 613   mock_pool_watcher.init(&ctx);
 614   ASSERT_EQ(0, ctx.wait());
 615
 616   ASSERT_TRUE(wait_for_update(1));
 617   expect_mirroring_watcher_unregister(mock_mirroring_watcher, 0);
 618   ASSERT_EQ(0, when_shut_down(mock_pool_watcher));
 619 }
 620
 621 TEST_F(TestMockPoolWatcher, GetMirrorUuidBlacklist) {
 622   MockThreads mock_threads(m_threads);
 623   expect_work_queue(mock_threads);
 624
 625   InSequence seq;
 626   MockMirroringWatcher mock_mirroring_watcher;
 627   expect_mirroring_watcher_is_unregistered(mock_mirroring_watcher, true);
 628   expect_mirroring_watcher_register(mock_mirroring_watcher, 0);
 629
 630   MockRefreshImagesRequest mock_refresh_images_request;
 631   expect_refresh_images(mock_refresh_images_request, {}, 0);
 632   expect_mirror_uuid_get(m_remote_io_ctx, "remote uuid", -EBLACKLISTED);
 633
 634   MockListener mock_listener(this);
 635   MockPoolWatcher mock_pool_watcher(&mock_threads, m_remote_io_ctx,
 636                                     mock_listener);
 637   C_SaferCond ctx;
 638   mock_pool_watcher.init(&ctx);
 639   ASSERT_EQ(-EBLACKLISTED, ctx.wait());
 640   ASSERT_TRUE(mock_pool_watcher.is_blacklisted());
 641
 642   expect_mirroring_watcher_unregister(mock_mirroring_watcher, 0);
 643   ASSERT_EQ(0, when_shut_down(mock_pool_watcher));
 644 }
 645
 646 TEST_F(TestMockPoolWatcher, GetMirrorUuidMissing) {
 647   MockThreads mock_threads(m_threads);
 648   expect_work_queue(mock_threads);
 649
 650   InSequence seq;
 651   MockMirroringWatcher mock_mirroring_watcher;
 652   expect_mirroring_watcher_is_unregistered(mock_mirroring_watcher, true);
 653   expect_mirroring_watcher_register(mock_mirroring_watcher, 0);
 654
 655   MockRefreshImagesRequest mock_refresh_images_request;
 656   expect_refresh_images(mock_refresh_images_request, {}, 0);
 657   expect_mirror_uuid_get(m_remote_io_ctx, "", -ENOENT);
 658   expect_timer_add_event(mock_threads);
 659
 660   expect_mirroring_watcher_is_unregistered(mock_mirroring_watcher, false);
 661   expect_refresh_images(mock_refresh_images_request, {}, 0);
 662   expect_mirror_uuid_get(m_remote_io_ctx, "remote uuid", 0);
 663
 664   MockListener mock_listener(this);
 665   expect_listener_handle_update(mock_listener, "remote uuid", {}, {});
 666
 667   MockPoolWatcher mock_pool_watcher(&mock_threads, m_remote_io_ctx,
 668                                     mock_listener);
 669   C_SaferCond ctx;
 670   mock_pool_watcher.init(&ctx);
 671   ASSERT_EQ(-ENOENT, ctx.wait());
 672
 673   ASSERT_TRUE(wait_for_update(1));
 674   expect_mirroring_watcher_unregister(mock_mirroring_watcher, 0);
 675   ASSERT_EQ(0, when_shut_down(mock_pool_watcher));
 676 }
 677
 678 TEST_F(TestMockPoolWatcher, GetMirrorUuidError) {
 679   MockThreads mock_threads(m_threads);
 680   expect_work_queue(mock_threads);
 681
 682   InSequence seq;
 683   MockMirroringWatcher mock_mirroring_watcher;
 684   expect_mirroring_watcher_is_unregistered(mock_mirroring_watcher, true);
 685   expect_mirroring_watcher_register(mock_mirroring_watcher, 0);
 686
 687   MockRefreshImagesRequest mock_refresh_images_request;
 688   expect_refresh_images(mock_refresh_images_request, {}, 0);
 689   expect_mirror_uuid_get(m_remote_io_ctx, "remote uuid", -EINVAL);
 690   expect_timer_add_event(mock_threads);
 691
 692   expect_mirroring_watcher_is_unregistered(mock_mirroring_watcher, false);
 693   expect_refresh_images(mock_refresh_images_request, {}, 0);
 694   expect_mirror_uuid_get(m_remote_io_ctx, "remote uuid", 0);
 695
 696   MockListener mock_listener(this);
 697   expect_listener_handle_update(mock_listener, "remote uuid", {}, {});
 698
 699   MockPoolWatcher mock_pool_watcher(&mock_threads, m_remote_io_ctx,
 700                                     mock_listener);
 701   C_SaferCond ctx;
 702   mock_pool_watcher.init(&ctx);
 703   ASSERT_EQ(0, ctx.wait());
 704
 705   ASSERT_TRUE(wait_for_update(1));
 706   expect_mirroring_watcher_unregister(mock_mirroring_watcher, 0);
 707   ASSERT_EQ(0, when_shut_down(mock_pool_watcher));
 708 }
 709
 710 TEST_F(TestMockPoolWatcher, Rewatch) {
 711   MockThreads mock_threads(m_threads);
 712   expect_work_queue(mock_threads);
 713
 714   InSequence seq;
 715   MockMirroringWatcher mock_mirroring_watcher;
 716   expect_mirroring_watcher_is_unregistered(mock_mirroring_watcher, true);
 717   expect_mirroring_watcher_register(mock_mirroring_watcher, 0);
 718
 719   MockRefreshImagesRequest mock_refresh_images_request;
 720   expect_refresh_images(mock_refresh_images_request, {}, 0);
 721   expect_mirror_uuid_get(m_remote_io_ctx, "remote uuid", 0);
 722
 723   MockListener mock_listener(this);
 724   expect_listener_handle_update(mock_listener, "remote uuid", {}, {});
 725
 726   expect_timer_add_event(mock_threads);
 727   expect_mirroring_watcher_is_unregistered(mock_mirroring_watcher, false);
 728   expect_refresh_images(mock_refresh_images_request, {{"global id", "image id"}}, 0);
 729   expect_mirror_uuid_get(m_remote_io_ctx, "remote uuid", 0);
 730   expect_listener_handle_update(mock_listener, "remote uuid",
 731                                 {{"global id", "image id"}}, {});
 732
 733   MockPoolWatcher mock_pool_watcher(&mock_threads, m_remote_io_ctx,
 734                                     mock_listener);
 735   C_SaferCond ctx;
 736   mock_pool_watcher.init(&ctx);
 737   ASSERT_EQ(0, ctx.wait());
 738   ASSERT_TRUE(wait_for_update(1));
 739
 740   MirroringWatcher::get_instance().handle_rewatch_complete(0);
 741   ASSERT_TRUE(wait_for_update(1));
 742
 743   expect_mirroring_watcher_unregister(mock_mirroring_watcher, 0);
 744   ASSERT_EQ(0, when_shut_down(mock_pool_watcher));
 745 }
 746
 747 TEST_F(TestMockPoolWatcher, RewatchBlacklist) {
 748   MockThreads mock_threads(m_threads);
 749   expect_work_queue(mock_threads);
 750
 751   InSequence seq;
 752   MockMirroringWatcher mock_mirroring_watcher;
 753   expect_mirroring_watcher_is_unregistered(mock_mirroring_watcher, true);
 754   expect_mirroring_watcher_register(mock_mirroring_watcher, 0);
 755
 756   MockRefreshImagesRequest mock_refresh_images_request;
 757   expect_refresh_images(mock_refresh_images_request, {}, 0);
 758   expect_mirror_uuid_get(m_remote_io_ctx, "remote uuid", 0);
 759
 760   MockListener mock_listener(this);
 761   expect_listener_handle_update(mock_listener, "remote uuid", {}, {});
 762
 763   MockPoolWatcher mock_pool_watcher(&mock_threads, m_remote_io_ctx,
 764                                     mock_listener);
 765   C_SaferCond ctx;
 766   mock_pool_watcher.init(&ctx);
 767   ASSERT_EQ(0, ctx.wait());
 768   ASSERT_TRUE(wait_for_update(1));
 769
 770   MirroringWatcher::get_instance().handle_rewatch_complete(-EBLACKLISTED);
 771   ASSERT_TRUE(mock_pool_watcher.is_blacklisted());
 772
 773   expect_mirroring_watcher_unregister(mock_mirroring_watcher, 0);
 774   ASSERT_EQ(0, when_shut_down(mock_pool_watcher));
 775 }
 776
 777 TEST_F(TestMockPoolWatcher, RewatchError) {
 778   MockThreads mock_threads(m_threads);
 779   expect_work_queue(mock_threads);
 780
 781   InSequence seq;
 782   MockMirroringWatcher mock_mirroring_watcher;
 783   expect_mirroring_watcher_is_unregistered(mock_mirroring_watcher, true);
 784   expect_mirroring_watcher_register(mock_mirroring_watcher, 0);
 785
 786   MockRefreshImagesRequest mock_refresh_images_request;
 787   expect_refresh_images(mock_refresh_images_request, {}, 0);
 788   expect_mirror_uuid_get(m_remote_io_ctx, "remote uuid", 0);
 789
 790   MockListener mock_listener(this);
 791   expect_listener_handle_update(mock_listener, "remote uuid", {}, {});
 792
 793   expect_timer_add_event(mock_threads);
 794   expect_mirroring_watcher_is_unregistered(mock_mirroring_watcher, false);
 795   expect_refresh_images(mock_refresh_images_request, {{"global id", "image id"}}, 0);
 796   expect_mirror_uuid_get(m_remote_io_ctx, "remote uuid", 0);
 797   expect_listener_handle_update(mock_listener, "remote uuid",
 798                                 {{"global id", "image id"}}, {});
 799
 800   MockPoolWatcher mock_pool_watcher(&mock_threads, m_remote_io_ctx,
 801                                     mock_listener);
 802   C_SaferCond ctx;
 803   mock_pool_watcher.init(&ctx);
 804   ASSERT_EQ(0, ctx.wait());
 805   ASSERT_TRUE(wait_for_update(1));
 806
 807   MirroringWatcher::get_instance().handle_rewatch_complete(-EINVAL);
 808   ASSERT_TRUE(wait_for_update(1));
 809
 810   expect_mirroring_watcher_unregister(mock_mirroring_watcher, 0);
 811   ASSERT_EQ(0, when_shut_down(mock_pool_watcher));
 812 }
 813
 814 TEST_F(TestMockPoolWatcher, DeferredRefresh) {
 815   MockThreads mock_threads(m_threads);
 816   expect_work_queue(mock_threads);
 817
 818   InSequence seq;
 819   MockMirroringWatcher mock_mirroring_watcher;
 820   expect_mirroring_watcher_is_unregistered(mock_mirroring_watcher, true);
 821   expect_mirroring_watcher_register(mock_mirroring_watcher, 0);
 822
 823   MockRefreshImagesRequest mock_refresh_images_request;
 824
 825   EXPECT_CALL(mock_refresh_images_request, send())
 826     .WillOnce(Invoke([&mock_refresh_images_request]() {
 827         *mock_refresh_images_request.image_ids = {};
 828         MirroringWatcher::get_instance().handle_rewatch_complete(0);
 829         mock_refresh_images_request.on_finish->complete(0);
 830         }));
 831   expect_mirror_uuid_get(m_remote_io_ctx, "remote uuid", 0);
 832   expect_timer_add_event(mock_threads);
 833
 834   expect_mirroring_watcher_is_unregistered(mock_mirroring_watcher, false);
 835   expect_refresh_images(mock_refresh_images_request, {}, 0);
 836   expect_mirror_uuid_get(m_remote_io_ctx, "remote uuid", 0);
 837
 838   MockListener mock_listener(this);
 839   expect_listener_handle_update(mock_listener, "remote uuid", {}, {});
 840
 841   MockPoolWatcher mock_pool_watcher(&mock_threads, m_remote_io_ctx,
 842                                     mock_listener);
 843   C_SaferCond ctx;
 844   mock_pool_watcher.init(&ctx);
 845   ASSERT_EQ(0, ctx.wait());
 846   ASSERT_TRUE(wait_for_update(1));
 847
 848   expect_mirroring_watcher_unregister(mock_mirroring_watcher, 0);
 849   ASSERT_EQ(0, when_shut_down(mock_pool_watcher));
 850 }
 851
 852 TEST_F(TestMockPoolWatcher, MirrorUuidUpdated) {
 853   MockThreads mock_threads(m_threads);
 854   expect_work_queue(mock_threads);
 855
 856   InSequence seq;
 857   MockMirroringWatcher mock_mirroring_watcher;
 858   expect_mirroring_watcher_is_unregistered(mock_mirroring_watcher, true);
 859   expect_mirroring_watcher_register(mock_mirroring_watcher, 0);
 860
 861   ImageIds image_ids{
 862     {"global id 1", "remote id 1"},
 863     {"global id 2", "remote id 2"}};
 864   MockRefreshImagesRequest mock_refresh_images_request;
 865   expect_refresh_images(mock_refresh_images_request, image_ids, 0);
 866   expect_mirror_uuid_get(m_remote_io_ctx, "remote uuid", 0);
 867
 868   MockListener mock_listener(this);
 869   expect_listener_handle_update(mock_listener, "remote uuid", image_ids, {});
 870
 871   MockPoolWatcher mock_pool_watcher(&mock_threads, m_remote_io_ctx,
 872                                     mock_listener);
 873   C_SaferCond ctx;
 874   mock_pool_watcher.init(&ctx);
 875   ASSERT_EQ(0, ctx.wait());
 876
 877   ASSERT_TRUE(wait_for_update(1));
 878
 879   expect_timer_add_event(mock_threads);
 880   ImageIds new_image_ids{
 881     {"global id 1", "remote id 1"}};
 882   expect_mirroring_watcher_is_unregistered(mock_mirroring_watcher, false);
 883   expect_refresh_images(mock_refresh_images_request, new_image_ids, 0);
 884   expect_mirror_uuid_get(m_remote_io_ctx, "updated uuid", 0);
 885   expect_listener_handle_update(mock_listener, "remote uuid", {}, image_ids);
 886   expect_listener_handle_update(mock_listener, "updated uuid", new_image_ids,
 887                                 {});
 888
 889   MirroringWatcher::get_instance().handle_rewatch_complete(0);
 890   ASSERT_TRUE(wait_for_update(2));
 891
 892   expect_mirroring_watcher_unregister(mock_mirroring_watcher, 0);
 893   ASSERT_EQ(0, when_shut_down(mock_pool_watcher));
 894 }
 895
 896 } // namespace mirror
 897 } // namespace rbd