1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
2 // vim: ts=8 sw=2 smarttab
4 #include "librbd/Utils.h"
5 #include "test/librbd/mock/MockImageCtx.h"
6 #include "test/rbd_mirror/test_mock_fixture.h"
7 #include "tools/rbd_mirror/LeaderWatcher.h"
8 #include "tools/rbd_mirror/Threads.h"
10 using librbd::util::create_async_context_callback;
16 struct MockTestImageCtx : public MockImageCtx {
17 MockTestImageCtx(librbd::ImageCtx &image_ctx)
18 : librbd::MockImageCtx(image_ctx) {
22 } // anonymous namespace
24 struct MockManagedLock {
25 static MockManagedLock *s_instance;
26 static MockManagedLock &get_instance() {
27 assert(s_instance != nullptr);
35 bool m_release_lock_on_shutdown = false;
37 MOCK_METHOD0(construct, void());
38 MOCK_METHOD0(destroy, void());
40 MOCK_CONST_METHOD0(is_lock_owner, bool());
42 MOCK_METHOD1(shut_down, void(Context *));
43 MOCK_METHOD1(try_acquire_lock, void(Context *));
44 MOCK_METHOD1(release_lock, void(Context *));
45 MOCK_METHOD3(break_lock, void(const managed_lock::Locker &, bool, Context *));
46 MOCK_METHOD2(get_locker, void(managed_lock::Locker *, Context *));
48 MOCK_METHOD0(set_state_post_acquiring, void());
50 MOCK_CONST_METHOD0(is_shutdown, bool());
52 MOCK_CONST_METHOD0(is_state_post_acquiring, bool());
53 MOCK_CONST_METHOD0(is_state_pre_releasing, bool());
54 MOCK_CONST_METHOD0(is_state_locked, bool());
57 MockManagedLock *MockManagedLock::s_instance = nullptr;
60 struct ManagedLock<MockTestImageCtx> {
61 ManagedLock(librados::IoCtx& ioctx, ContextWQ *work_queue,
62 const std::string& oid, librbd::Watcher *watcher,
63 managed_lock::Mode mode, bool blacklist_on_break_lock,
64 uint32_t blacklist_expire_seconds)
65 : m_work_queue(work_queue), m_lock("ManagedLock::m_lock") {
66 MockManagedLock::get_instance().construct();
69 virtual ~ManagedLock() {
70 MockManagedLock::get_instance().destroy();
73 ContextWQ *m_work_queue;
77 bool is_lock_owner() const {
78 return MockManagedLock::get_instance().is_lock_owner();
81 void shut_down(Context *on_shutdown) {
82 if (MockManagedLock::get_instance().m_release_lock_on_shutdown) {
83 on_shutdown = new FunctionContext(
84 [this, on_shutdown](int r) {
85 MockManagedLock::get_instance().m_release_lock_on_shutdown = false;
86 shut_down(on_shutdown);
88 release_lock(on_shutdown);
92 MockManagedLock::get_instance().shut_down(on_shutdown);
95 void try_acquire_lock(Context *on_acquired) {
96 Context *post_acquire_ctx = create_async_context_callback(
97 m_work_queue, new FunctionContext(
98 [this, on_acquired](int r) {
99 post_acquire_lock_handler(r, on_acquired);
101 MockManagedLock::get_instance().try_acquire_lock(post_acquire_ctx);
104 void release_lock(Context *on_released) {
105 Context *post_release_ctx = new FunctionContext(
106 [this, on_released](int r) {
107 post_release_lock_handler(false, r, on_released);
110 Context *release_ctx = new FunctionContext(
111 [this, on_released, post_release_ctx](int r) {
113 on_released->complete(r);
115 MockManagedLock::get_instance().release_lock(post_release_ctx);
119 Context *pre_release_ctx = new FunctionContext(
120 [this, release_ctx](int r) {
122 MockManagedLock::get_instance().m_release_lock_on_shutdown;
123 pre_release_lock_handler(shutting_down, release_ctx);
126 m_work_queue->queue(pre_release_ctx, 0);
129 void get_locker(managed_lock::Locker *locker, Context *on_finish) {
130 MockManagedLock::get_instance().get_locker(locker, on_finish);
133 void break_lock(const managed_lock::Locker &locker, bool force_break_lock,
134 Context *on_finish) {
135 MockManagedLock::get_instance().break_lock(locker, force_break_lock,
139 void set_state_post_acquiring() {
140 MockManagedLock::get_instance().set_state_post_acquiring();
143 bool is_shutdown() const {
144 return MockManagedLock::get_instance().is_shutdown();
147 bool is_state_post_acquiring() const {
148 return MockManagedLock::get_instance().is_state_post_acquiring();
151 bool is_state_pre_releasing() const {
152 return MockManagedLock::get_instance().is_state_pre_releasing();
155 bool is_state_locked() const {
156 return MockManagedLock::get_instance().is_state_locked();
159 virtual void post_acquire_lock_handler(int r, Context *on_finish) = 0;
160 virtual void pre_release_lock_handler(bool shutting_down,
161 Context *on_finish) = 0;
162 virtual void post_release_lock_handler(bool shutting_down, int r,
163 Context *on_finish) = 0;
166 } // namespace librbd
172 struct Threads<librbd::MockTestImageCtx> {
175 ContextWQ *work_queue;
177 Threads(Threads<librbd::ImageCtx> *threads)
178 : timer_lock(threads->timer_lock), timer(threads->timer),
179 work_queue(threads->work_queue) {
184 struct MirrorStatusWatcher<librbd::MockTestImageCtx> {
185 static MirrorStatusWatcher* s_instance;
187 static MirrorStatusWatcher *create(librados::IoCtx &io_ctx,
188 ContextWQ *work_queue) {
189 assert(s_instance != nullptr);
193 MirrorStatusWatcher() {
194 assert(s_instance == nullptr);
198 ~MirrorStatusWatcher() {
199 assert(s_instance == this);
200 s_instance = nullptr;
203 MOCK_METHOD0(destroy, void());
204 MOCK_METHOD1(init, void(Context *));
205 MOCK_METHOD1(shut_down, void(Context *));
208 MirrorStatusWatcher<librbd::MockTestImageCtx> *MirrorStatusWatcher<librbd::MockTestImageCtx>::s_instance = nullptr;
211 struct Instances<librbd::MockTestImageCtx> {
212 static Instances* s_instance;
214 static Instances *create(Threads<librbd::MockTestImageCtx> *threads,
215 librados::IoCtx &ioctx) {
216 assert(s_instance != nullptr);
221 assert(s_instance == nullptr);
226 assert(s_instance == this);
227 s_instance = nullptr;
230 MOCK_METHOD0(destroy, void());
231 MOCK_METHOD1(init, void(Context *));
232 MOCK_METHOD1(shut_down, void(Context *));
233 MOCK_METHOD1(notify, void(const std::string &));
236 Instances<librbd::MockTestImageCtx> *Instances<librbd::MockTestImageCtx>::s_instance = nullptr;
238 } // namespace mirror
242 // template definitions
243 #include "tools/rbd_mirror/LeaderWatcher.cc"
249 using ::testing::AtLeast;
250 using ::testing::DoAll;
251 using ::testing::InSequence;
252 using ::testing::Invoke;
253 using ::testing::Return;
255 using librbd::MockManagedLock;
257 struct MockListener : public LeaderWatcher<librbd::MockTestImageCtx>::Listener {
258 static MockListener* s_instance;
261 assert(s_instance == nullptr);
265 ~MockListener() override {
266 assert(s_instance == this);
267 s_instance = nullptr;
270 MOCK_METHOD1(post_acquire_handler, void(Context *));
271 MOCK_METHOD1(pre_release_handler, void(Context *));
273 MOCK_METHOD1(update_leader_handler, void(const std::string &));
276 MockListener *MockListener::s_instance = nullptr;
278 class TestMockLeaderWatcher : public TestMockFixture {
280 typedef MirrorStatusWatcher<librbd::MockTestImageCtx> MockMirrorStatusWatcher;
281 typedef Instances<librbd::MockTestImageCtx> MockInstances;
282 typedef LeaderWatcher<librbd::MockTestImageCtx> MockLeaderWatcher;
283 typedef Threads<librbd::MockTestImageCtx> MockThreads;
285 void SetUp() override {
286 TestMockFixture::SetUp();
287 m_mock_threads = new MockThreads(m_threads);
290 void TearDown() override {
291 delete m_mock_threads;
292 TestMockFixture::TearDown();
295 void expect_construct(MockManagedLock &mock_managed_lock) {
296 EXPECT_CALL(mock_managed_lock, construct());
299 void expect_destroy(MockManagedLock &mock_managed_lock) {
300 EXPECT_CALL(mock_managed_lock, destroy());
303 void expect_is_lock_owner(MockManagedLock &mock_managed_lock, bool owner) {
304 EXPECT_CALL(mock_managed_lock, is_lock_owner())
305 .WillOnce(Return(owner));
308 void expect_shut_down(MockManagedLock &mock_managed_lock,
309 bool release_lock_on_shutdown, int r) {
310 mock_managed_lock.m_release_lock_on_shutdown = release_lock_on_shutdown;
311 EXPECT_CALL(mock_managed_lock, shut_down(_))
312 .WillOnce(CompleteContext(r));
315 void expect_try_acquire_lock(MockManagedLock &mock_managed_lock, int r) {
316 EXPECT_CALL(mock_managed_lock, try_acquire_lock(_))
317 .WillOnce(CompleteContext(r));
319 expect_set_state_post_acquiring(mock_managed_lock);
323 void expect_release_lock(MockManagedLock &mock_managed_lock, int r,
324 Context *on_finish = nullptr) {
325 EXPECT_CALL(mock_managed_lock, release_lock(_))
326 .WillOnce(Invoke([on_finish, r](Context *ctx) {
328 if (on_finish != nullptr) {
329 on_finish->complete(0);
334 void expect_get_locker(MockManagedLock &mock_managed_lock,
335 const librbd::managed_lock::Locker &locker, int r) {
336 EXPECT_CALL(mock_managed_lock, get_locker(_, _))
337 .WillOnce(Invoke([r, locker](librbd::managed_lock::Locker *out,
346 void expect_break_lock(MockManagedLock &mock_managed_lock,
347 const librbd::managed_lock::Locker &locker, int r,
348 Context *on_finish) {
349 EXPECT_CALL(mock_managed_lock, break_lock(locker, true, _))
350 .WillOnce(Invoke([on_finish, r](const librbd::managed_lock::Locker &,
351 bool, Context *ctx) {
353 on_finish->complete(0);
357 void expect_set_state_post_acquiring(MockManagedLock &mock_managed_lock) {
358 EXPECT_CALL(mock_managed_lock, set_state_post_acquiring());
361 void expect_is_shutdown(MockManagedLock &mock_managed_lock) {
362 EXPECT_CALL(mock_managed_lock, is_shutdown())
363 .Times(AtLeast(0)).WillRepeatedly(Return(false));
366 void expect_is_leader(MockManagedLock &mock_managed_lock, bool post_acquiring,
368 EXPECT_CALL(mock_managed_lock, is_state_post_acquiring())
369 .WillOnce(Return(post_acquiring));
370 if (!post_acquiring) {
371 EXPECT_CALL(mock_managed_lock, is_state_locked())
372 .WillOnce(Return(locked));
376 void expect_is_leader(MockManagedLock &mock_managed_lock) {
377 EXPECT_CALL(mock_managed_lock, is_state_post_acquiring())
378 .Times(AtLeast(0)).WillRepeatedly(Return(false));
379 EXPECT_CALL(mock_managed_lock, is_state_locked())
380 .Times(AtLeast(0)).WillRepeatedly(Return(false));
381 EXPECT_CALL(mock_managed_lock, is_state_pre_releasing())
382 .Times(AtLeast(0)).WillRepeatedly(Return(false));
385 void expect_notify_heartbeat(MockManagedLock &mock_managed_lock,
386 Context *on_finish) {
387 // is_leader in notify_heartbeat
388 EXPECT_CALL(mock_managed_lock, is_state_post_acquiring())
389 .WillOnce(Return(false));
390 EXPECT_CALL(mock_managed_lock, is_state_locked())
391 .WillOnce(Return(true));
393 // is_leader in handle_notify_heartbeat
394 EXPECT_CALL(mock_managed_lock, is_state_post_acquiring())
395 .WillOnce(Return(false));
396 EXPECT_CALL(mock_managed_lock, is_state_locked())
397 .WillOnce(DoAll(Invoke([on_finish]() {
398 on_finish->complete(0);
403 void expect_destroy(MockMirrorStatusWatcher &mock_mirror_status_watcher) {
404 EXPECT_CALL(mock_mirror_status_watcher, destroy());
407 void expect_init(MockMirrorStatusWatcher &mock_mirror_status_watcher, int r) {
408 EXPECT_CALL(mock_mirror_status_watcher, init(_))
409 .WillOnce(CompleteContext(m_mock_threads->work_queue, r));
412 void expect_shut_down(MockMirrorStatusWatcher &mock_mirror_status_watcher, int r) {
413 EXPECT_CALL(mock_mirror_status_watcher, shut_down(_))
414 .WillOnce(CompleteContext(m_mock_threads->work_queue, r));
415 expect_destroy(mock_mirror_status_watcher);
418 void expect_destroy(MockInstances &mock_instances) {
419 EXPECT_CALL(mock_instances, destroy());
422 void expect_init(MockInstances &mock_instances, int r) {
423 EXPECT_CALL(mock_instances, init(_))
424 .WillOnce(CompleteContext(m_mock_threads->work_queue, r));
427 void expect_shut_down(MockInstances &mock_instances, int r) {
428 EXPECT_CALL(mock_instances, shut_down(_))
429 .WillOnce(CompleteContext(m_mock_threads->work_queue, r));
430 expect_destroy(mock_instances);
433 void expect_acquire_notify(MockManagedLock &mock_managed_lock,
434 MockListener &mock_listener, int r) {
435 expect_is_leader(mock_managed_lock, true, false);
436 EXPECT_CALL(mock_listener, post_acquire_handler(_))
437 .WillOnce(CompleteContext(r));
438 expect_is_leader(mock_managed_lock, true, false);
441 void expect_release_notify(MockManagedLock &mock_managed_lock,
442 MockListener &mock_listener, int r) {
443 expect_is_leader(mock_managed_lock, false, false);
444 EXPECT_CALL(mock_listener, pre_release_handler(_))
445 .WillOnce(CompleteContext(r));
446 expect_is_leader(mock_managed_lock, false, false);
449 MockThreads *m_mock_threads;
452 TEST_F(TestMockLeaderWatcher, InitShutdown) {
453 MockManagedLock mock_managed_lock;
454 MockMirrorStatusWatcher mock_mirror_status_watcher;
455 MockInstances mock_instances;
456 MockListener listener;
458 expect_is_shutdown(mock_managed_lock);
459 expect_destroy(mock_managed_lock);
463 expect_construct(mock_managed_lock);
464 MockLeaderWatcher leader_watcher(m_mock_threads, m_local_io_ctx, &listener);
467 C_SaferCond on_heartbeat_finish;
468 expect_is_leader(mock_managed_lock, false, false);
469 expect_try_acquire_lock(mock_managed_lock, 0);
470 expect_init(mock_mirror_status_watcher, 0);
471 expect_init(mock_instances, 0);
472 expect_acquire_notify(mock_managed_lock, listener, 0);
473 expect_notify_heartbeat(mock_managed_lock, &on_heartbeat_finish);
475 ASSERT_EQ(0, leader_watcher.init());
476 ASSERT_EQ(0, on_heartbeat_finish.wait());
479 expect_release_notify(mock_managed_lock, listener, 0);
480 expect_shut_down(mock_instances, 0);
481 expect_shut_down(mock_mirror_status_watcher, 0);
482 expect_is_leader(mock_managed_lock, false, false);
483 expect_release_lock(mock_managed_lock, 0);
484 expect_shut_down(mock_managed_lock, true, 0);
485 expect_is_leader(mock_managed_lock, false, false);
487 leader_watcher.shut_down();
490 TEST_F(TestMockLeaderWatcher, InitReleaseShutdown) {
491 MockManagedLock mock_managed_lock;
492 MockMirrorStatusWatcher mock_mirror_status_watcher;
493 MockInstances mock_instances;
494 MockListener listener;
496 expect_is_shutdown(mock_managed_lock);
497 expect_destroy(mock_managed_lock);
501 expect_construct(mock_managed_lock);
502 MockLeaderWatcher leader_watcher(m_mock_threads, m_local_io_ctx, &listener);
505 C_SaferCond on_heartbeat_finish;
506 expect_is_leader(mock_managed_lock, false, false);
507 expect_try_acquire_lock(mock_managed_lock, 0);
508 expect_init(mock_mirror_status_watcher, 0);
509 expect_init(mock_instances, 0);
510 expect_acquire_notify(mock_managed_lock, listener, 0);
511 expect_notify_heartbeat(mock_managed_lock, &on_heartbeat_finish);
513 ASSERT_EQ(0, leader_watcher.init());
514 ASSERT_EQ(0, on_heartbeat_finish.wait());
517 expect_is_leader(mock_managed_lock, false, true);
518 expect_release_notify(mock_managed_lock, listener, 0);
519 expect_shut_down(mock_instances, 0);
520 expect_shut_down(mock_mirror_status_watcher, 0);
521 expect_is_leader(mock_managed_lock, false, false);
522 C_SaferCond on_release;
523 expect_release_lock(mock_managed_lock, 0, &on_release);
525 leader_watcher.release_leader();
526 ASSERT_EQ(0, on_release.wait());
529 expect_shut_down(mock_managed_lock, false, 0);
530 expect_is_leader(mock_managed_lock, false, false);
532 leader_watcher.shut_down();
535 TEST_F(TestMockLeaderWatcher, AcquireError) {
536 MockManagedLock mock_managed_lock;
537 MockMirrorStatusWatcher mock_mirror_status_watcher;
538 MockInstances mock_instances;
539 MockListener listener;
541 expect_is_shutdown(mock_managed_lock);
542 expect_is_leader(mock_managed_lock);
543 expect_destroy(mock_managed_lock);
547 expect_construct(mock_managed_lock);
548 MockLeaderWatcher leader_watcher(m_mock_threads, m_local_io_ctx, &listener);
551 C_SaferCond on_heartbeat_finish;
552 expect_is_leader(mock_managed_lock, false, false);
553 expect_try_acquire_lock(mock_managed_lock, -EAGAIN);
554 expect_get_locker(mock_managed_lock, librbd::managed_lock::Locker(), -ENOENT);
555 expect_try_acquire_lock(mock_managed_lock, 0);
556 expect_init(mock_mirror_status_watcher, 0);
557 expect_init(mock_instances, 0);
558 expect_acquire_notify(mock_managed_lock, listener, 0);
559 expect_notify_heartbeat(mock_managed_lock, &on_heartbeat_finish);
561 ASSERT_EQ(0, leader_watcher.init());
562 ASSERT_EQ(0, on_heartbeat_finish.wait());
565 expect_release_notify(mock_managed_lock, listener, 0);
566 expect_shut_down(mock_instances, 0);
567 expect_shut_down(mock_mirror_status_watcher, 0);
568 expect_is_leader(mock_managed_lock, false, false);
569 expect_release_lock(mock_managed_lock, 0);
570 expect_shut_down(mock_managed_lock, true, 0);
571 expect_is_leader(mock_managed_lock, false, false);
573 leader_watcher.shut_down();
576 TEST_F(TestMockLeaderWatcher, ReleaseError) {
577 MockManagedLock mock_managed_lock;
578 MockMirrorStatusWatcher mock_mirror_status_watcher;
579 MockInstances mock_instances;
580 MockListener listener;
582 expect_is_shutdown(mock_managed_lock);
583 expect_destroy(mock_managed_lock);
587 expect_construct(mock_managed_lock);
588 MockLeaderWatcher leader_watcher(m_mock_threads, m_local_io_ctx, &listener);
591 C_SaferCond on_heartbeat_finish;
592 expect_is_leader(mock_managed_lock, false, false);
593 expect_try_acquire_lock(mock_managed_lock, 0);
594 expect_init(mock_mirror_status_watcher, 0);
595 expect_init(mock_instances, 0);
596 expect_acquire_notify(mock_managed_lock, listener, 0);
597 expect_notify_heartbeat(mock_managed_lock, &on_heartbeat_finish);
599 ASSERT_EQ(0, leader_watcher.init());
600 ASSERT_EQ(0, on_heartbeat_finish.wait());
603 expect_is_leader(mock_managed_lock, false, true);
604 expect_release_notify(mock_managed_lock, listener, -EINVAL);
605 expect_shut_down(mock_instances, 0);
606 expect_shut_down(mock_mirror_status_watcher, -EINVAL);
607 expect_is_leader(mock_managed_lock, false, false);
608 C_SaferCond on_release;
609 expect_release_lock(mock_managed_lock, -EINVAL, &on_release);
611 leader_watcher.release_leader();
612 ASSERT_EQ(0, on_release.wait());
615 expect_shut_down(mock_managed_lock, false, 0);
616 expect_is_leader(mock_managed_lock, false, false);
618 leader_watcher.shut_down();
621 TEST_F(TestMockLeaderWatcher, Break) {
622 EXPECT_EQ(0, _rados->conf_set("rbd_mirror_leader_heartbeat_interval", "1"));
623 EXPECT_EQ(0, _rados->conf_set("rbd_mirror_leader_max_missed_heartbeats",
625 CephContext *cct = reinterpret_cast<CephContext *>(m_local_io_ctx.cct());
626 int max_acquire_attempts = cct->_conf->get_val<int64_t>(
627 "rbd_mirror_leader_max_acquire_attempts_before_break");
629 MockManagedLock mock_managed_lock;
630 MockMirrorStatusWatcher mock_mirror_status_watcher;
631 MockInstances mock_instances;
632 MockListener listener;
633 librbd::managed_lock::Locker
634 locker{entity_name_t::CLIENT(1), "auto 123", "1.2.3.4:0/0", 123};
636 expect_is_shutdown(mock_managed_lock);
637 expect_is_leader(mock_managed_lock);
638 expect_destroy(mock_managed_lock);
639 EXPECT_CALL(listener, update_leader_handler(_));
643 expect_construct(mock_managed_lock);
644 MockLeaderWatcher leader_watcher(m_mock_threads, m_local_io_ctx, &listener);
647 expect_is_leader(mock_managed_lock, false, false);
648 for (int i = 0; i < max_acquire_attempts; i++) {
649 expect_try_acquire_lock(mock_managed_lock, -EAGAIN);
650 expect_get_locker(mock_managed_lock, locker, 0);
652 C_SaferCond on_break;
653 expect_break_lock(mock_managed_lock, locker, 0, &on_break);
654 C_SaferCond on_heartbeat_finish;
655 expect_try_acquire_lock(mock_managed_lock, 0);
656 expect_init(mock_mirror_status_watcher, 0);
657 expect_init(mock_instances, 0);
658 expect_acquire_notify(mock_managed_lock, listener, 0);
659 expect_notify_heartbeat(mock_managed_lock, &on_heartbeat_finish);
661 ASSERT_EQ(0, leader_watcher.init());
662 ASSERT_EQ(0, on_heartbeat_finish.wait());
665 expect_release_notify(mock_managed_lock, listener, 0);
666 expect_shut_down(mock_instances, 0);
667 expect_shut_down(mock_mirror_status_watcher, 0);
668 expect_is_leader(mock_managed_lock, false, false);
669 expect_release_lock(mock_managed_lock, 0);
670 expect_shut_down(mock_managed_lock, true, 0);
671 expect_is_leader(mock_managed_lock, false, false);
673 leader_watcher.shut_down();
676 } // namespace mirror