src/ceph/src/test/rbd_mirror/test_mock_LeaderWatcher.cc

   1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
   2 // vim: ts=8 sw=2 smarttab
   3
   4 #include "librbd/Utils.h"
   5 #include "test/librbd/mock/MockImageCtx.h"
   6 #include "test/rbd_mirror/test_mock_fixture.h"
   7 #include "tools/rbd_mirror/LeaderWatcher.h"
   8 #include "tools/rbd_mirror/Threads.h"
   9
  10 using librbd::util::create_async_context_callback;
  11
  12 namespace librbd {
  13
  14 namespace {
  15
  16 struct MockTestImageCtx : public MockImageCtx {
  17   MockTestImageCtx(librbd::ImageCtx &image_ctx)
  18     : librbd::MockImageCtx(image_ctx) {
  19   }
  20 };
  21
  22 } // anonymous namespace
  23
  24 struct MockManagedLock {
  25   static MockManagedLock *s_instance;
  26   static MockManagedLock &get_instance() {
  27     assert(s_instance != nullptr);
  28     return *s_instance;
  29   }
  30
  31   MockManagedLock() {
  32     s_instance = this;
  33   }
  34
  35   bool m_release_lock_on_shutdown = false;
  36
  37   MOCK_METHOD0(construct, void());
  38   MOCK_METHOD0(destroy, void());
  39
  40   MOCK_CONST_METHOD0(is_lock_owner, bool());
  41
  42   MOCK_METHOD1(shut_down, void(Context *));
  43   MOCK_METHOD1(try_acquire_lock, void(Context *));
  44   MOCK_METHOD1(release_lock, void(Context *));
  45   MOCK_METHOD3(break_lock, void(const managed_lock::Locker &, bool, Context *));
  46   MOCK_METHOD2(get_locker, void(managed_lock::Locker *, Context *));
  47
  48   MOCK_METHOD0(set_state_post_acquiring, void());
  49
  50   MOCK_CONST_METHOD0(is_shutdown, bool());
  51
  52   MOCK_CONST_METHOD0(is_state_post_acquiring, bool());
  53   MOCK_CONST_METHOD0(is_state_pre_releasing, bool());
  54   MOCK_CONST_METHOD0(is_state_locked, bool());
  55 };
  56
  57 MockManagedLock *MockManagedLock::s_instance = nullptr;
  58
  59 template <>
  60 struct ManagedLock<MockTestImageCtx> {
  61   ManagedLock(librados::IoCtx& ioctx, ContextWQ *work_queue,
  62               const std::string& oid, librbd::Watcher *watcher,
  63               managed_lock::Mode  mode, bool blacklist_on_break_lock,
  64               uint32_t blacklist_expire_seconds)
  65     : m_work_queue(work_queue), m_lock("ManagedLock::m_lock") {
  66     MockManagedLock::get_instance().construct();
  67   }
  68
  69   virtual ~ManagedLock() {
  70     MockManagedLock::get_instance().destroy();
  71   }
  72
  73   ContextWQ *m_work_queue;
  74
  75   mutable Mutex m_lock;
  76
  77   bool is_lock_owner() const {
  78     return MockManagedLock::get_instance().is_lock_owner();
  79   }
  80
  81   void shut_down(Context *on_shutdown) {
  82     if (MockManagedLock::get_instance().m_release_lock_on_shutdown) {
  83       on_shutdown = new FunctionContext(
  84         [this, on_shutdown](int r) {
  85           MockManagedLock::get_instance().m_release_lock_on_shutdown = false;
  86           shut_down(on_shutdown);
  87         });
  88       release_lock(on_shutdown);
  89       return;
  90     }
  91
  92     MockManagedLock::get_instance().shut_down(on_shutdown);
  93   }
  94
  95   void try_acquire_lock(Context *on_acquired) {
  96     Context *post_acquire_ctx = create_async_context_callback(
  97       m_work_queue, new FunctionContext(
  98         [this, on_acquired](int r) {
  99           post_acquire_lock_handler(r, on_acquired);
 100         }));
 101     MockManagedLock::get_instance().try_acquire_lock(post_acquire_ctx);
 102   }
 103
 104   void release_lock(Context *on_released) {
 105     Context *post_release_ctx = new FunctionContext(
 106       [this, on_released](int r) {
 107         post_release_lock_handler(false, r, on_released);
 108       });
 109
 110     Context *release_ctx = new FunctionContext(
 111       [this, on_released, post_release_ctx](int r) {
 112         if (r < 0) {
 113           on_released->complete(r);
 114         } else {
 115           MockManagedLock::get_instance().release_lock(post_release_ctx);
 116         }
 117       });
 118
 119     Context *pre_release_ctx = new FunctionContext(
 120       [this, release_ctx](int r) {
 121         bool shutting_down =
 122           MockManagedLock::get_instance().m_release_lock_on_shutdown;
 123         pre_release_lock_handler(shutting_down, release_ctx);
 124       });
 125
 126     m_work_queue->queue(pre_release_ctx, 0);
 127   }
 128
 129   void get_locker(managed_lock::Locker *locker, Context *on_finish) {
 130     MockManagedLock::get_instance().get_locker(locker, on_finish);
 131   }
 132
 133   void break_lock(const managed_lock::Locker &locker, bool force_break_lock,
 134                   Context *on_finish) {
 135     MockManagedLock::get_instance().break_lock(locker, force_break_lock,
 136                                                on_finish);
 137   }
 138
 139   void set_state_post_acquiring() {
 140     MockManagedLock::get_instance().set_state_post_acquiring();
 141   }
 142
 143   bool is_shutdown() const {
 144     return MockManagedLock::get_instance().is_shutdown();
 145   }
 146
 147   bool is_state_post_acquiring() const {
 148     return MockManagedLock::get_instance().is_state_post_acquiring();
 149   }
 150
 151   bool is_state_pre_releasing() const {
 152     return MockManagedLock::get_instance().is_state_pre_releasing();
 153   }
 154
 155   bool is_state_locked() const {
 156     return MockManagedLock::get_instance().is_state_locked();
 157   }
 158
 159   virtual void post_acquire_lock_handler(int r, Context *on_finish) = 0;
 160   virtual void pre_release_lock_handler(bool shutting_down,
 161                                         Context *on_finish) = 0;
 162   virtual void post_release_lock_handler(bool shutting_down, int r,
 163                                          Context *on_finish) = 0;
 164 };
 165
 166 } // namespace librbd
 167
 168 namespace rbd {
 169 namespace mirror {
 170
 171 template <>
 172 struct Threads<librbd::MockTestImageCtx> {
 173   Mutex &timer_lock;
 174   SafeTimer *timer;
 175   ContextWQ *work_queue;
 176
 177   Threads(Threads<librbd::ImageCtx> *threads)
 178     : timer_lock(threads->timer_lock), timer(threads->timer),
 179       work_queue(threads->work_queue) {
 180   }
 181 };
 182
 183 template <>
 184 struct MirrorStatusWatcher<librbd::MockTestImageCtx> {
 185   static MirrorStatusWatcher* s_instance;
 186
 187   static MirrorStatusWatcher *create(librados::IoCtx &io_ctx,
 188                                      ContextWQ *work_queue) {
 189     assert(s_instance != nullptr);
 190     return s_instance;
 191   }
 192
 193   MirrorStatusWatcher() {
 194     assert(s_instance == nullptr);
 195     s_instance = this;
 196   }
 197
 198   ~MirrorStatusWatcher() {
 199     assert(s_instance == this);
 200     s_instance = nullptr;
 201   }
 202
 203   MOCK_METHOD0(destroy, void());
 204   MOCK_METHOD1(init, void(Context *));
 205   MOCK_METHOD1(shut_down, void(Context *));
 206 };
 207
 208 MirrorStatusWatcher<librbd::MockTestImageCtx> *MirrorStatusWatcher<librbd::MockTestImageCtx>::s_instance = nullptr;
 209
 210 template <>
 211 struct Instances<librbd::MockTestImageCtx> {
 212   static Instances* s_instance;
 213
 214   static Instances *create(Threads<librbd::MockTestImageCtx> *threads,
 215                            librados::IoCtx &ioctx) {
 216     assert(s_instance != nullptr);
 217     return s_instance;
 218   }
 219
 220   Instances() {
 221     assert(s_instance == nullptr);
 222     s_instance = this;
 223   }
 224
 225   ~Instances() {
 226     assert(s_instance == this);
 227     s_instance = nullptr;
 228   }
 229
 230   MOCK_METHOD0(destroy, void());
 231   MOCK_METHOD1(init, void(Context *));
 232   MOCK_METHOD1(shut_down, void(Context *));
 233   MOCK_METHOD1(notify, void(const std::string &));
 234 };
 235
 236 Instances<librbd::MockTestImageCtx> *Instances<librbd::MockTestImageCtx>::s_instance = nullptr;
 237
 238 } // namespace mirror
 239 } // namespace rbd
 240
 241
 242 // template definitions
 243 #include "tools/rbd_mirror/LeaderWatcher.cc"
 244
 245 namespace rbd {
 246 namespace mirror {
 247
 248 using ::testing::_;
 249 using ::testing::AtLeast;
 250 using ::testing::DoAll;
 251 using ::testing::InSequence;
 252 using ::testing::Invoke;
 253 using ::testing::Return;
 254
 255 using librbd::MockManagedLock;
 256
 257 struct MockListener : public LeaderWatcher<librbd::MockTestImageCtx>::Listener {
 258   static MockListener* s_instance;
 259
 260   MockListener() {
 261     assert(s_instance == nullptr);
 262     s_instance = this;
 263   }
 264
 265   ~MockListener() override {
 266     assert(s_instance == this);
 267     s_instance = nullptr;
 268   }
 269
 270   MOCK_METHOD1(post_acquire_handler, void(Context *));
 271   MOCK_METHOD1(pre_release_handler, void(Context *));
 272
 273   MOCK_METHOD1(update_leader_handler, void(const std::string &));
 274 };
 275
 276 MockListener *MockListener::s_instance = nullptr;
 277
 278 class TestMockLeaderWatcher : public TestMockFixture {
 279 public:
 280   typedef MirrorStatusWatcher<librbd::MockTestImageCtx> MockMirrorStatusWatcher;
 281   typedef Instances<librbd::MockTestImageCtx> MockInstances;
 282   typedef LeaderWatcher<librbd::MockTestImageCtx> MockLeaderWatcher;
 283   typedef Threads<librbd::MockTestImageCtx> MockThreads;
 284
 285   void SetUp() override {
 286     TestMockFixture::SetUp();
 287     m_mock_threads = new MockThreads(m_threads);
 288   }
 289
 290   void TearDown() override {
 291     delete m_mock_threads;
 292     TestMockFixture::TearDown();
 293   }
 294
 295   void expect_construct(MockManagedLock &mock_managed_lock) {
 296     EXPECT_CALL(mock_managed_lock, construct());
 297   }
 298
 299   void expect_destroy(MockManagedLock &mock_managed_lock) {
 300     EXPECT_CALL(mock_managed_lock, destroy());
 301   }
 302
 303   void expect_is_lock_owner(MockManagedLock &mock_managed_lock, bool owner) {
 304     EXPECT_CALL(mock_managed_lock, is_lock_owner())
 305       .WillOnce(Return(owner));
 306   }
 307
 308   void expect_shut_down(MockManagedLock &mock_managed_lock,
 309                         bool release_lock_on_shutdown, int r) {
 310     mock_managed_lock.m_release_lock_on_shutdown = release_lock_on_shutdown;
 311     EXPECT_CALL(mock_managed_lock, shut_down(_))
 312       .WillOnce(CompleteContext(r));
 313   }
 314
 315   void expect_try_acquire_lock(MockManagedLock &mock_managed_lock, int r) {
 316     EXPECT_CALL(mock_managed_lock, try_acquire_lock(_))
 317       .WillOnce(CompleteContext(r));
 318     if (r == 0) {
 319       expect_set_state_post_acquiring(mock_managed_lock);
 320     }
 321   }
 322
 323   void expect_release_lock(MockManagedLock &mock_managed_lock, int r,
 324                            Context *on_finish = nullptr) {
 325     EXPECT_CALL(mock_managed_lock, release_lock(_))
 326       .WillOnce(Invoke([on_finish, r](Context *ctx) {
 327                          ctx->complete(r);
 328                          if (on_finish != nullptr) {
 329                            on_finish->complete(0);
 330                          }
 331                        }));
 332   }
 333
 334   void expect_get_locker(MockManagedLock &mock_managed_lock,
 335                          const librbd::managed_lock::Locker &locker, int r) {
 336     EXPECT_CALL(mock_managed_lock, get_locker(_, _))
 337       .WillOnce(Invoke([r, locker](librbd::managed_lock::Locker *out,
 338                                    Context *ctx) {
 339                          if (r == 0) {
 340                            *out = locker;
 341                          }
 342                          ctx->complete(r);
 343                        }));
 344   }
 345
 346   void expect_break_lock(MockManagedLock &mock_managed_lock,
 347                          const librbd::managed_lock::Locker &locker, int r,
 348                          Context *on_finish) {
 349     EXPECT_CALL(mock_managed_lock, break_lock(locker, true, _))
 350       .WillOnce(Invoke([on_finish, r](const librbd::managed_lock::Locker &,
 351                                       bool, Context *ctx) {
 352                          ctx->complete(r);
 353                          on_finish->complete(0);
 354                        }));
 355   }
 356
 357   void expect_set_state_post_acquiring(MockManagedLock &mock_managed_lock) {
 358     EXPECT_CALL(mock_managed_lock, set_state_post_acquiring());
 359   }
 360
 361   void expect_is_shutdown(MockManagedLock &mock_managed_lock) {
 362     EXPECT_CALL(mock_managed_lock, is_shutdown())
 363       .Times(AtLeast(0)).WillRepeatedly(Return(false));
 364   }
 365
 366   void expect_is_leader(MockManagedLock &mock_managed_lock, bool post_acquiring,
 367                         bool locked) {
 368     EXPECT_CALL(mock_managed_lock, is_state_post_acquiring())
 369       .WillOnce(Return(post_acquiring));
 370     if (!post_acquiring) {
 371       EXPECT_CALL(mock_managed_lock, is_state_locked())
 372         .WillOnce(Return(locked));
 373     }
 374   }
 375
 376   void expect_is_leader(MockManagedLock &mock_managed_lock) {
 377     EXPECT_CALL(mock_managed_lock, is_state_post_acquiring())
 378       .Times(AtLeast(0)).WillRepeatedly(Return(false));
 379     EXPECT_CALL(mock_managed_lock, is_state_locked())
 380       .Times(AtLeast(0)).WillRepeatedly(Return(false));
 381     EXPECT_CALL(mock_managed_lock, is_state_pre_releasing())
 382       .Times(AtLeast(0)).WillRepeatedly(Return(false));
 383   }
 384
 385   void expect_notify_heartbeat(MockManagedLock &mock_managed_lock,
 386                                Context *on_finish) {
 387     // is_leader in notify_heartbeat
 388     EXPECT_CALL(mock_managed_lock, is_state_post_acquiring())
 389       .WillOnce(Return(false));
 390     EXPECT_CALL(mock_managed_lock, is_state_locked())
 391       .WillOnce(Return(true));
 392
 393     // is_leader in handle_notify_heartbeat
 394     EXPECT_CALL(mock_managed_lock, is_state_post_acquiring())
 395       .WillOnce(Return(false));
 396     EXPECT_CALL(mock_managed_lock, is_state_locked())
 397       .WillOnce(DoAll(Invoke([on_finish]() {
 398                         on_finish->complete(0);
 399                       }),
 400                       Return(true)));
 401   }
 402
 403   void expect_destroy(MockMirrorStatusWatcher &mock_mirror_status_watcher) {
 404     EXPECT_CALL(mock_mirror_status_watcher, destroy());
 405   }
 406
 407   void expect_init(MockMirrorStatusWatcher &mock_mirror_status_watcher, int r) {
 408     EXPECT_CALL(mock_mirror_status_watcher, init(_))
 409       .WillOnce(CompleteContext(m_mock_threads->work_queue, r));
 410   }
 411
 412   void expect_shut_down(MockMirrorStatusWatcher &mock_mirror_status_watcher, int r) {
 413     EXPECT_CALL(mock_mirror_status_watcher, shut_down(_))
 414       .WillOnce(CompleteContext(m_mock_threads->work_queue, r));
 415     expect_destroy(mock_mirror_status_watcher);
 416   }
 417
 418   void expect_destroy(MockInstances &mock_instances) {
 419     EXPECT_CALL(mock_instances, destroy());
 420   }
 421
 422   void expect_init(MockInstances &mock_instances, int r) {
 423     EXPECT_CALL(mock_instances, init(_))
 424       .WillOnce(CompleteContext(m_mock_threads->work_queue, r));
 425   }
 426
 427   void expect_shut_down(MockInstances &mock_instances, int r) {
 428     EXPECT_CALL(mock_instances, shut_down(_))
 429       .WillOnce(CompleteContext(m_mock_threads->work_queue, r));
 430     expect_destroy(mock_instances);
 431   }
 432
 433   void expect_acquire_notify(MockManagedLock &mock_managed_lock,
 434                              MockListener &mock_listener, int r) {
 435     expect_is_leader(mock_managed_lock, true, false);
 436     EXPECT_CALL(mock_listener, post_acquire_handler(_))
 437       .WillOnce(CompleteContext(r));
 438     expect_is_leader(mock_managed_lock, true, false);
 439   }
 440
 441   void expect_release_notify(MockManagedLock &mock_managed_lock,
 442                              MockListener &mock_listener, int r) {
 443     expect_is_leader(mock_managed_lock, false, false);
 444     EXPECT_CALL(mock_listener, pre_release_handler(_))
 445       .WillOnce(CompleteContext(r));
 446     expect_is_leader(mock_managed_lock, false, false);
 447   }
 448
 449   MockThreads *m_mock_threads;
 450 };
 451
 452 TEST_F(TestMockLeaderWatcher, InitShutdown) {
 453   MockManagedLock mock_managed_lock;
 454   MockMirrorStatusWatcher mock_mirror_status_watcher;
 455   MockInstances mock_instances;
 456   MockListener listener;
 457
 458   expect_is_shutdown(mock_managed_lock);
 459   expect_destroy(mock_managed_lock);
 460
 461   InSequence seq;
 462
 463   expect_construct(mock_managed_lock);
 464   MockLeaderWatcher leader_watcher(m_mock_threads, m_local_io_ctx, &listener);
 465
 466   // Inint
 467   C_SaferCond on_heartbeat_finish;
 468   expect_is_leader(mock_managed_lock, false, false);
 469   expect_try_acquire_lock(mock_managed_lock, 0);
 470   expect_init(mock_mirror_status_watcher, 0);
 471   expect_init(mock_instances, 0);
 472   expect_acquire_notify(mock_managed_lock, listener, 0);
 473   expect_notify_heartbeat(mock_managed_lock, &on_heartbeat_finish);
 474
 475   ASSERT_EQ(0, leader_watcher.init());
 476   ASSERT_EQ(0, on_heartbeat_finish.wait());
 477
 478   // Shutdown
 479   expect_release_notify(mock_managed_lock, listener, 0);
 480   expect_shut_down(mock_instances, 0);
 481   expect_shut_down(mock_mirror_status_watcher, 0);
 482   expect_is_leader(mock_managed_lock, false, false);
 483   expect_release_lock(mock_managed_lock, 0);
 484   expect_shut_down(mock_managed_lock, true, 0);
 485   expect_is_leader(mock_managed_lock, false, false);
 486
 487   leader_watcher.shut_down();
 488 }
 489
 490 TEST_F(TestMockLeaderWatcher, InitReleaseShutdown) {
 491   MockManagedLock mock_managed_lock;
 492   MockMirrorStatusWatcher mock_mirror_status_watcher;
 493   MockInstances mock_instances;
 494   MockListener listener;
 495
 496   expect_is_shutdown(mock_managed_lock);
 497   expect_destroy(mock_managed_lock);
 498
 499   InSequence seq;
 500
 501   expect_construct(mock_managed_lock);
 502   MockLeaderWatcher leader_watcher(m_mock_threads, m_local_io_ctx, &listener);
 503
 504   // Inint
 505   C_SaferCond on_heartbeat_finish;
 506   expect_is_leader(mock_managed_lock, false, false);
 507   expect_try_acquire_lock(mock_managed_lock, 0);
 508   expect_init(mock_mirror_status_watcher, 0);
 509   expect_init(mock_instances, 0);
 510   expect_acquire_notify(mock_managed_lock, listener, 0);
 511   expect_notify_heartbeat(mock_managed_lock, &on_heartbeat_finish);
 512
 513   ASSERT_EQ(0, leader_watcher.init());
 514   ASSERT_EQ(0, on_heartbeat_finish.wait());
 515
 516   // Release
 517   expect_is_leader(mock_managed_lock, false, true);
 518   expect_release_notify(mock_managed_lock, listener, 0);
 519   expect_shut_down(mock_instances, 0);
 520   expect_shut_down(mock_mirror_status_watcher, 0);
 521   expect_is_leader(mock_managed_lock, false, false);
 522   C_SaferCond on_release;
 523   expect_release_lock(mock_managed_lock, 0, &on_release);
 524
 525   leader_watcher.release_leader();
 526   ASSERT_EQ(0, on_release.wait());
 527
 528   // Shutdown
 529   expect_shut_down(mock_managed_lock, false, 0);
 530   expect_is_leader(mock_managed_lock, false, false);
 531
 532   leader_watcher.shut_down();
 533 }
 534
 535 TEST_F(TestMockLeaderWatcher, AcquireError) {
 536   MockManagedLock mock_managed_lock;
 537   MockMirrorStatusWatcher mock_mirror_status_watcher;
 538   MockInstances mock_instances;
 539   MockListener listener;
 540
 541   expect_is_shutdown(mock_managed_lock);
 542   expect_is_leader(mock_managed_lock);
 543   expect_destroy(mock_managed_lock);
 544
 545   InSequence seq;
 546
 547   expect_construct(mock_managed_lock);
 548   MockLeaderWatcher leader_watcher(m_mock_threads, m_local_io_ctx, &listener);
 549
 550   // Inint
 551   C_SaferCond on_heartbeat_finish;
 552   expect_is_leader(mock_managed_lock, false, false);
 553   expect_try_acquire_lock(mock_managed_lock, -EAGAIN);
 554   expect_get_locker(mock_managed_lock, librbd::managed_lock::Locker(), -ENOENT);
 555   expect_try_acquire_lock(mock_managed_lock, 0);
 556   expect_init(mock_mirror_status_watcher, 0);
 557   expect_init(mock_instances, 0);
 558   expect_acquire_notify(mock_managed_lock, listener, 0);
 559   expect_notify_heartbeat(mock_managed_lock, &on_heartbeat_finish);
 560
 561   ASSERT_EQ(0, leader_watcher.init());
 562   ASSERT_EQ(0, on_heartbeat_finish.wait());
 563
 564   // Shutdown
 565   expect_release_notify(mock_managed_lock, listener, 0);
 566   expect_shut_down(mock_instances, 0);
 567   expect_shut_down(mock_mirror_status_watcher, 0);
 568   expect_is_leader(mock_managed_lock, false, false);
 569   expect_release_lock(mock_managed_lock, 0);
 570   expect_shut_down(mock_managed_lock, true, 0);
 571   expect_is_leader(mock_managed_lock, false, false);
 572
 573   leader_watcher.shut_down();
 574 }
 575
 576 TEST_F(TestMockLeaderWatcher, ReleaseError) {
 577   MockManagedLock mock_managed_lock;
 578   MockMirrorStatusWatcher mock_mirror_status_watcher;
 579   MockInstances mock_instances;
 580   MockListener listener;
 581
 582   expect_is_shutdown(mock_managed_lock);
 583   expect_destroy(mock_managed_lock);
 584
 585   InSequence seq;
 586
 587   expect_construct(mock_managed_lock);
 588   MockLeaderWatcher leader_watcher(m_mock_threads, m_local_io_ctx, &listener);
 589
 590   // Inint
 591   C_SaferCond on_heartbeat_finish;
 592   expect_is_leader(mock_managed_lock, false, false);
 593   expect_try_acquire_lock(mock_managed_lock, 0);
 594   expect_init(mock_mirror_status_watcher, 0);
 595   expect_init(mock_instances, 0);
 596   expect_acquire_notify(mock_managed_lock, listener, 0);
 597   expect_notify_heartbeat(mock_managed_lock, &on_heartbeat_finish);
 598
 599   ASSERT_EQ(0, leader_watcher.init());
 600   ASSERT_EQ(0, on_heartbeat_finish.wait());
 601
 602   // Release
 603   expect_is_leader(mock_managed_lock, false, true);
 604   expect_release_notify(mock_managed_lock, listener, -EINVAL);
 605   expect_shut_down(mock_instances, 0);
 606   expect_shut_down(mock_mirror_status_watcher, -EINVAL);
 607   expect_is_leader(mock_managed_lock, false, false);
 608   C_SaferCond on_release;
 609   expect_release_lock(mock_managed_lock, -EINVAL, &on_release);
 610
 611   leader_watcher.release_leader();
 612   ASSERT_EQ(0, on_release.wait());
 613
 614   // Shutdown
 615   expect_shut_down(mock_managed_lock, false, 0);
 616   expect_is_leader(mock_managed_lock, false, false);
 617
 618   leader_watcher.shut_down();
 619 }
 620
 621 TEST_F(TestMockLeaderWatcher, Break) {
 622   EXPECT_EQ(0, _rados->conf_set("rbd_mirror_leader_heartbeat_interval", "1"));
 623   EXPECT_EQ(0, _rados->conf_set("rbd_mirror_leader_max_missed_heartbeats",
 624                                 "1"));
 625   CephContext *cct = reinterpret_cast<CephContext *>(m_local_io_ctx.cct());
 626   int max_acquire_attempts = cct->_conf->get_val<int64_t>(
 627     "rbd_mirror_leader_max_acquire_attempts_before_break");
 628
 629   MockManagedLock mock_managed_lock;
 630   MockMirrorStatusWatcher mock_mirror_status_watcher;
 631   MockInstances mock_instances;
 632   MockListener listener;
 633   librbd::managed_lock::Locker
 634     locker{entity_name_t::CLIENT(1), "auto 123", "1.2.3.4:0/0", 123};
 635
 636   expect_is_shutdown(mock_managed_lock);
 637   expect_is_leader(mock_managed_lock);
 638   expect_destroy(mock_managed_lock);
 639   EXPECT_CALL(listener, update_leader_handler(_));
 640
 641   InSequence seq;
 642
 643   expect_construct(mock_managed_lock);
 644   MockLeaderWatcher leader_watcher(m_mock_threads, m_local_io_ctx, &listener);
 645
 646   // Init
 647   expect_is_leader(mock_managed_lock, false, false);
 648   for (int i = 0; i < max_acquire_attempts; i++) {
 649     expect_try_acquire_lock(mock_managed_lock, -EAGAIN);
 650     expect_get_locker(mock_managed_lock, locker, 0);
 651   }
 652   C_SaferCond on_break;
 653   expect_break_lock(mock_managed_lock, locker, 0, &on_break);
 654   C_SaferCond on_heartbeat_finish;
 655   expect_try_acquire_lock(mock_managed_lock, 0);
 656   expect_init(mock_mirror_status_watcher, 0);
 657   expect_init(mock_instances, 0);
 658   expect_acquire_notify(mock_managed_lock, listener, 0);
 659   expect_notify_heartbeat(mock_managed_lock, &on_heartbeat_finish);
 660
 661   ASSERT_EQ(0, leader_watcher.init());
 662   ASSERT_EQ(0, on_heartbeat_finish.wait());
 663
 664   // Shutdown
 665   expect_release_notify(mock_managed_lock, listener, 0);
 666   expect_shut_down(mock_instances, 0);
 667   expect_shut_down(mock_mirror_status_watcher, 0);
 668   expect_is_leader(mock_managed_lock, false, false);
 669   expect_release_lock(mock_managed_lock, 0);
 670   expect_shut_down(mock_managed_lock, true, 0);
 671   expect_is_leader(mock_managed_lock, false, false);
 672
 673   leader_watcher.shut_down();
 674 }
 675
 676 } // namespace mirror
 677 } // namespace rbd