1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
2 // vim: ts=8 sw=2 smarttab
4 #include "librbd/ManagedLock.h"
5 #include "librbd/managed_lock/AcquireRequest.h"
6 #include "librbd/managed_lock/BreakRequest.h"
7 #include "librbd/managed_lock/GetLockerRequest.h"
8 #include "librbd/managed_lock/ReleaseRequest.h"
9 #include "librbd/managed_lock/ReacquireRequest.h"
10 #include "librbd/managed_lock/Types.h"
11 #include "librbd/managed_lock/Utils.h"
12 #include "librbd/Watcher.h"
13 #include "librbd/ImageCtx.h"
14 #include "cls/lock/cls_lock_client.h"
15 #include "common/dout.h"
16 #include "common/errno.h"
17 #include "common/WorkQueue.h"
18 #include "librbd/Utils.h"
20 #define dout_subsys ceph_subsys_rbd
22 #define dout_prefix *_dout << "librbd::ManagedLock: " << this << " " \
28 using namespace managed_lock;
33 struct C_SendLockRequest : public Context {
35 explicit C_SendLockRequest(R* request) : request(request) {
37 void finish(int r) override {
42 struct C_Tracked : public Context {
43 AsyncOpTracker &tracker;
45 C_Tracked(AsyncOpTracker &tracker, Context *ctx)
46 : tracker(tracker), ctx(ctx) {
49 ~C_Tracked() override {
52 void finish(int r) override {
57 } // anonymous namespace
59 using librbd::util::create_context_callback;
60 using librbd::util::unique_lock_name;
61 using managed_lock::util::decode_lock_cookie;
62 using managed_lock::util::encode_lock_cookie;
65 ManagedLock<I>::ManagedLock(librados::IoCtx &ioctx, ContextWQ *work_queue,
66 const string& oid, Watcher *watcher, Mode mode,
67 bool blacklist_on_break_lock,
68 uint32_t blacklist_expire_seconds)
69 : m_lock(unique_lock_name("librbd::ManagedLock<I>::m_lock", this)),
70 m_ioctx(ioctx), m_cct(reinterpret_cast<CephContext *>(ioctx.cct())),
71 m_work_queue(work_queue),
75 m_blacklist_on_break_lock(blacklist_on_break_lock),
76 m_blacklist_expire_seconds(blacklist_expire_seconds),
77 m_state(STATE_UNLOCKED) {
81 ManagedLock<I>::~ManagedLock() {
82 Mutex::Locker locker(m_lock);
83 assert(m_state == STATE_SHUTDOWN || m_state == STATE_UNLOCKED ||
84 m_state == STATE_UNINITIALIZED);
85 if (m_state == STATE_UNINITIALIZED) {
86 // never initialized -- ensure any in-flight ops are complete
87 // since we wouldn't expect shut_down to be invoked
89 m_async_op_tracker.wait_for_ops(&ctx);
92 assert(m_async_op_tracker.empty());
96 bool ManagedLock<I>::is_lock_owner() const {
97 Mutex::Locker locker(m_lock);
99 return is_lock_owner(m_lock);
102 template <typename I>
103 bool ManagedLock<I>::is_lock_owner(Mutex &lock) const {
105 assert(m_lock.is_locked());
111 case STATE_REACQUIRING:
112 case STATE_PRE_SHUTTING_DOWN:
113 case STATE_POST_ACQUIRING:
114 case STATE_PRE_RELEASING:
122 ldout(m_cct, 20) << "=" << lock_owner << dendl;
126 template <typename I>
127 void ManagedLock<I>::shut_down(Context *on_shut_down) {
128 ldout(m_cct, 10) << dendl;
130 Mutex::Locker locker(m_lock);
131 assert(!is_state_shutdown());
132 execute_action(ACTION_SHUT_DOWN, on_shut_down);
135 template <typename I>
136 void ManagedLock<I>::acquire_lock(Context *on_acquired) {
139 Mutex::Locker locker(m_lock);
140 if (is_state_shutdown()) {
142 } else if (m_state != STATE_LOCKED || !m_actions_contexts.empty()) {
143 ldout(m_cct, 10) << dendl;
144 execute_action(ACTION_ACQUIRE_LOCK, on_acquired);
149 if (on_acquired != nullptr) {
150 on_acquired->complete(r);
154 template <typename I>
155 void ManagedLock<I>::try_acquire_lock(Context *on_acquired) {
158 Mutex::Locker locker(m_lock);
159 if (is_state_shutdown()) {
161 } else if (m_state != STATE_LOCKED || !m_actions_contexts.empty()) {
162 ldout(m_cct, 10) << dendl;
163 execute_action(ACTION_TRY_LOCK, on_acquired);
168 if (on_acquired != nullptr) {
169 on_acquired->complete(r);
173 template <typename I>
174 void ManagedLock<I>::release_lock(Context *on_released) {
177 Mutex::Locker locker(m_lock);
178 if (is_state_shutdown()) {
180 } else if (m_state != STATE_UNLOCKED || !m_actions_contexts.empty()) {
181 ldout(m_cct, 10) << dendl;
182 execute_action(ACTION_RELEASE_LOCK, on_released);
187 if (on_released != nullptr) {
188 on_released->complete(r);
192 template <typename I>
193 void ManagedLock<I>::reacquire_lock(Context *on_reacquired) {
195 Mutex::Locker locker(m_lock);
197 if (m_state == STATE_WAITING_FOR_REGISTER) {
198 // restart the acquire lock process now that watch is valid
199 ldout(m_cct, 10) << ": " << "woke up waiting acquire" << dendl;
200 Action active_action = get_active_action();
201 assert(active_action == ACTION_TRY_LOCK ||
202 active_action == ACTION_ACQUIRE_LOCK);
203 execute_next_action();
204 } else if (!is_state_shutdown() &&
205 (m_state == STATE_LOCKED ||
206 m_state == STATE_ACQUIRING ||
207 m_state == STATE_POST_ACQUIRING ||
208 m_state == STATE_WAITING_FOR_LOCK)) {
209 // interlock the lock operation with other state ops
210 ldout(m_cct, 10) << dendl;
211 execute_action(ACTION_REACQUIRE_LOCK, on_reacquired);
216 // ignore request if shutdown or not in a locked-related state
217 if (on_reacquired != nullptr) {
218 on_reacquired->complete(0);
222 template <typename I>
223 void ManagedLock<I>::get_locker(managed_lock::Locker *locker,
224 Context *on_finish) {
225 ldout(m_cct, 10) << dendl;
229 Mutex::Locker l(m_lock);
230 if (is_state_shutdown()) {
233 on_finish = new C_Tracked(m_async_op_tracker, on_finish);
234 auto req = managed_lock::GetLockerRequest<I>::create(
235 m_ioctx, m_oid, m_mode == EXCLUSIVE, locker, on_finish);
241 on_finish->complete(r);
244 template <typename I>
245 void ManagedLock<I>::break_lock(const managed_lock::Locker &locker,
246 bool force_break_lock, Context *on_finish) {
247 ldout(m_cct, 10) << dendl;
251 Mutex::Locker l(m_lock);
252 if (is_state_shutdown()) {
254 } else if (is_lock_owner(m_lock)) {
257 on_finish = new C_Tracked(m_async_op_tracker, on_finish);
258 auto req = managed_lock::BreakRequest<I>::create(
259 m_ioctx, m_work_queue, m_oid, locker, m_mode == EXCLUSIVE,
260 m_blacklist_on_break_lock, m_blacklist_expire_seconds, force_break_lock,
267 on_finish->complete(r);
270 template <typename I>
271 int ManagedLock<I>::assert_header_locked() {
272 ldout(m_cct, 10) << dendl;
274 librados::ObjectReadOperation op;
276 Mutex::Locker locker(m_lock);
277 rados::cls::lock::assert_locked(&op, RBD_LOCK_NAME,
278 (m_mode == EXCLUSIVE ? LOCK_EXCLUSIVE :
281 managed_lock::util::get_watcher_lock_tag());
284 int r = m_ioctx.operate(m_oid, &op, nullptr);
286 if (r == -EBLACKLISTED) {
287 ldout(m_cct, 5) << "client is not lock owner -- client blacklisted"
289 } else if (r == -ENOENT) {
290 ldout(m_cct, 5) << "client is not lock owner -- no lock detected"
292 } else if (r == -EBUSY) {
293 ldout(m_cct, 5) << "client is not lock owner -- owned by different client"
296 lderr(m_cct) << "failed to verify lock ownership: " << cpp_strerror(r)
306 template <typename I>
307 void ManagedLock<I>::shutdown_handler(int r, Context *on_finish) {
308 on_finish->complete(r);
311 template <typename I>
312 void ManagedLock<I>::pre_acquire_lock_handler(Context *on_finish) {
313 on_finish->complete(0);
316 template <typename I>
317 void ManagedLock<I>::post_acquire_lock_handler(int r, Context *on_finish) {
318 on_finish->complete(r);
321 template <typename I>
322 void ManagedLock<I>::pre_release_lock_handler(bool shutting_down,
323 Context *on_finish) {
324 on_finish->complete(0);
327 template <typename I>
328 void ManagedLock<I>::post_release_lock_handler(bool shutting_down, int r,
329 Context *on_finish) {
330 on_finish->complete(r);
333 template <typename I>
334 void ManagedLock<I>::post_reacquire_lock_handler(int r, Context *on_finish) {
335 on_finish->complete(r);
338 template <typename I>
339 bool ManagedLock<I>::is_transition_state() const {
341 case STATE_ACQUIRING:
342 case STATE_WAITING_FOR_REGISTER:
343 case STATE_REACQUIRING:
344 case STATE_RELEASING:
345 case STATE_PRE_SHUTTING_DOWN:
346 case STATE_SHUTTING_DOWN:
347 case STATE_INITIALIZING:
348 case STATE_WAITING_FOR_LOCK:
349 case STATE_POST_ACQUIRING:
350 case STATE_PRE_RELEASING:
355 case STATE_UNINITIALIZED:
361 template <typename I>
362 void ManagedLock<I>::append_context(Action action, Context *ctx) {
363 assert(m_lock.is_locked());
365 for (auto &action_ctxs : m_actions_contexts) {
366 if (action == action_ctxs.first) {
367 if (ctx != nullptr) {
368 action_ctxs.second.push_back(ctx);
375 if (ctx != nullptr) {
376 contexts.push_back(ctx);
378 m_actions_contexts.push_back({action, std::move(contexts)});
381 template <typename I>
382 void ManagedLock<I>::execute_action(Action action, Context *ctx) {
383 assert(m_lock.is_locked());
385 append_context(action, ctx);
386 if (!is_transition_state()) {
387 execute_next_action();
391 template <typename I>
392 void ManagedLock<I>::execute_next_action() {
393 assert(m_lock.is_locked());
394 assert(!m_actions_contexts.empty());
395 switch (get_active_action()) {
396 case ACTION_ACQUIRE_LOCK:
397 case ACTION_TRY_LOCK:
400 case ACTION_REACQUIRE_LOCK:
401 send_reacquire_lock();
403 case ACTION_RELEASE_LOCK:
406 case ACTION_SHUT_DOWN:
415 template <typename I>
416 typename ManagedLock<I>::Action ManagedLock<I>::get_active_action() const {
417 assert(m_lock.is_locked());
418 assert(!m_actions_contexts.empty());
419 return m_actions_contexts.front().first;
422 template <typename I>
423 void ManagedLock<I>::complete_active_action(State next_state, int r) {
424 assert(m_lock.is_locked());
425 assert(!m_actions_contexts.empty());
427 ActionContexts action_contexts(std::move(m_actions_contexts.front()));
428 m_actions_contexts.pop_front();
429 m_state = next_state;
432 for (auto ctx : action_contexts.second) {
437 if (!is_transition_state() && !m_actions_contexts.empty()) {
438 execute_next_action();
442 template <typename I>
443 bool ManagedLock<I>::is_state_shutdown() const {
444 assert(m_lock.is_locked());
446 return ((m_state == STATE_SHUTDOWN) ||
447 (!m_actions_contexts.empty() &&
448 m_actions_contexts.back().first == ACTION_SHUT_DOWN));
451 template <typename I>
452 void ManagedLock<I>::send_acquire_lock() {
453 assert(m_lock.is_locked());
454 if (m_state == STATE_LOCKED) {
455 complete_active_action(STATE_LOCKED, 0);
459 ldout(m_cct, 10) << dendl;
460 m_state = STATE_ACQUIRING;
462 uint64_t watch_handle = m_watcher->get_watch_handle();
463 if (watch_handle == 0) {
464 lderr(m_cct) << "watcher not registered - delaying request" << dendl;
465 m_state = STATE_WAITING_FOR_REGISTER;
468 m_cookie = encode_lock_cookie(watch_handle);
470 m_work_queue->queue(new FunctionContext([this](int r) {
471 pre_acquire_lock_handler(create_context_callback<
472 ManagedLock<I>, &ManagedLock<I>::handle_pre_acquire_lock>(this));
476 template <typename I>
477 void ManagedLock<I>::handle_pre_acquire_lock(int r) {
478 ldout(m_cct, 10) << ": r=" << r << dendl;
481 handle_acquire_lock(r);
485 using managed_lock::AcquireRequest;
486 AcquireRequest<I>* req = AcquireRequest<I>::create(
487 m_ioctx, m_watcher, m_work_queue, m_oid, m_cookie, m_mode == EXCLUSIVE,
488 m_blacklist_on_break_lock, m_blacklist_expire_seconds,
489 create_context_callback<
490 ManagedLock<I>, &ManagedLock<I>::handle_acquire_lock>(this));
491 m_work_queue->queue(new C_SendLockRequest<AcquireRequest<I>>(req), 0);
494 template <typename I>
495 void ManagedLock<I>::handle_acquire_lock(int r) {
496 ldout(m_cct, 10) << ": r=" << r << dendl;
498 if (r == -EBUSY || r == -EAGAIN) {
499 ldout(m_cct, 5) << ": unable to acquire exclusive lock" << dendl;
501 lderr(m_cct) << ": failed to acquire exclusive lock:" << cpp_strerror(r)
504 ldout(m_cct, 5) << ": successfully acquired exclusive lock" << dendl;
507 m_post_next_state = (r < 0 ? STATE_UNLOCKED : STATE_LOCKED);
509 m_work_queue->queue(new FunctionContext([this, r](int ret) {
510 post_acquire_lock_handler(r, create_context_callback<
511 ManagedLock<I>, &ManagedLock<I>::handle_post_acquire_lock>(this));
515 template <typename I>
516 void ManagedLock<I>::handle_post_acquire_lock(int r) {
517 ldout(m_cct, 10) << ": r=" << r << dendl;
519 Mutex::Locker locker(m_lock);
521 if (r < 0 && m_post_next_state == STATE_LOCKED) {
522 // release_lock without calling pre and post handlers
523 revert_to_unlock_state(r);
524 } else if (r != -ECANCELED) {
525 // fail the lock request
526 complete_active_action(m_post_next_state, r);
530 template <typename I>
531 void ManagedLock<I>::revert_to_unlock_state(int r) {
532 ldout(m_cct, 10) << ": r=" << r << dendl;
534 using managed_lock::ReleaseRequest;
535 ReleaseRequest<I>* req = ReleaseRequest<I>::create(m_ioctx, m_watcher,
536 m_work_queue, m_oid, m_cookie,
537 new FunctionContext([this, r](int ret) {
538 Mutex::Locker locker(m_lock);
540 complete_active_action(STATE_UNLOCKED, r);
542 m_work_queue->queue(new C_SendLockRequest<ReleaseRequest<I>>(req));
545 template <typename I>
546 void ManagedLock<I>::send_reacquire_lock() {
547 assert(m_lock.is_locked());
549 if (m_state != STATE_LOCKED) {
550 complete_active_action(m_state, 0);
554 uint64_t watch_handle = m_watcher->get_watch_handle();
555 if (watch_handle == 0) {
556 // watch (re)failed while recovering
557 lderr(m_cct) << ": aborting reacquire due to invalid watch handle"
559 complete_active_action(STATE_LOCKED, 0);
563 m_new_cookie = encode_lock_cookie(watch_handle);
564 if (m_cookie == m_new_cookie) {
565 ldout(m_cct, 10) << ": skipping reacquire since cookie still valid"
567 complete_active_action(STATE_LOCKED, 0);
571 ldout(m_cct, 10) << dendl;
572 m_state = STATE_REACQUIRING;
574 auto ctx = create_context_callback<
575 ManagedLock, &ManagedLock<I>::handle_reacquire_lock>(this);
576 ctx = new FunctionContext([this, ctx](int r) {
577 post_reacquire_lock_handler(r, ctx);
580 using managed_lock::ReacquireRequest;
581 ReacquireRequest<I>* req = ReacquireRequest<I>::create(m_ioctx, m_oid,
582 m_cookie, m_new_cookie, m_mode == EXCLUSIVE, ctx);
583 m_work_queue->queue(new C_SendLockRequest<ReacquireRequest<I>>(req));
586 template <typename I>
587 void ManagedLock<I>::handle_reacquire_lock(int r) {
588 ldout(m_cct, 10) << ": r=" << r << dendl;
590 Mutex::Locker locker(m_lock);
591 assert(m_state == STATE_REACQUIRING);
594 if (r == -EOPNOTSUPP) {
595 ldout(m_cct, 10) << ": updating lock is not supported" << dendl;
597 lderr(m_cct) << ": failed to update lock cookie: " << cpp_strerror(r)
601 if (!is_state_shutdown()) {
602 // queue a release and re-acquire of the lock since cookie cannot
603 // be updated on older OSDs
604 execute_action(ACTION_RELEASE_LOCK, nullptr);
606 assert(!m_actions_contexts.empty());
607 ActionContexts &action_contexts(m_actions_contexts.front());
609 // reacquire completes when the request lock completes
611 std::swap(contexts, action_contexts.second);
612 if (contexts.empty()) {
613 execute_action(ACTION_ACQUIRE_LOCK, nullptr);
615 for (auto ctx : contexts) {
616 ctx = new FunctionContext([ctx, r](int acquire_ret_val) {
617 if (acquire_ret_val >= 0) {
620 ctx->complete(acquire_ret_val);
622 execute_action(ACTION_ACQUIRE_LOCK, ctx);
627 m_cookie = m_new_cookie;
630 complete_active_action(STATE_LOCKED, r);
633 template <typename I>
634 void ManagedLock<I>::send_release_lock() {
635 assert(m_lock.is_locked());
636 if (m_state == STATE_UNLOCKED) {
637 complete_active_action(STATE_UNLOCKED, 0);
641 ldout(m_cct, 10) << dendl;
642 m_state = STATE_PRE_RELEASING;
644 m_work_queue->queue(new FunctionContext([this](int r) {
645 pre_release_lock_handler(false, create_context_callback<
646 ManagedLock<I>, &ManagedLock<I>::handle_pre_release_lock>(this));
650 template <typename I>
651 void ManagedLock<I>::handle_pre_release_lock(int r) {
652 ldout(m_cct, 10) << ": r=" << r << dendl;
655 Mutex::Locker locker(m_lock);
656 assert(m_state == STATE_PRE_RELEASING);
657 m_state = STATE_RELEASING;
661 handle_release_lock(r);
665 using managed_lock::ReleaseRequest;
666 ReleaseRequest<I>* req = ReleaseRequest<I>::create(m_ioctx, m_watcher,
667 m_work_queue, m_oid, m_cookie,
668 create_context_callback<
669 ManagedLock<I>, &ManagedLock<I>::handle_release_lock>(this));
670 m_work_queue->queue(new C_SendLockRequest<ReleaseRequest<I>>(req), 0);
673 template <typename I>
674 void ManagedLock<I>::handle_release_lock(int r) {
675 ldout(m_cct, 10) << ": r=" << r << dendl;
677 Mutex::Locker locker(m_lock);
678 assert(m_state == STATE_RELEASING);
684 m_post_next_state = r < 0 ? STATE_LOCKED : STATE_UNLOCKED;
686 m_work_queue->queue(new FunctionContext([this, r](int ret) {
687 post_release_lock_handler(false, r, create_context_callback<
688 ManagedLock<I>, &ManagedLock<I>::handle_post_release_lock>(this));
692 template <typename I>
693 void ManagedLock<I>::handle_post_release_lock(int r) {
694 ldout(m_cct, 10) << ": r=" << r << dendl;
696 Mutex::Locker locker(m_lock);
697 complete_active_action(m_post_next_state, r);
700 template <typename I>
701 void ManagedLock<I>::send_shutdown() {
702 ldout(m_cct, 10) << dendl;
703 assert(m_lock.is_locked());
704 if (m_state == STATE_UNLOCKED) {
705 m_state = STATE_SHUTTING_DOWN;
706 m_work_queue->queue(new FunctionContext([this](int r) {
707 shutdown_handler(r, create_context_callback<
708 ManagedLock<I>, &ManagedLock<I>::handle_shutdown>(this));
713 assert(m_state == STATE_LOCKED);
714 m_state = STATE_PRE_SHUTTING_DOWN;
717 m_work_queue->queue(new C_ShutDownRelease(this), 0);
721 template <typename I>
722 void ManagedLock<I>::handle_shutdown(int r) {
723 ldout(m_cct, 10) << ": r=" << r << dendl;
725 wait_for_tracked_ops(r);
728 template <typename I>
729 void ManagedLock<I>::send_shutdown_release() {
730 ldout(m_cct, 10) << dendl;
732 Mutex::Locker locker(m_lock);
734 m_work_queue->queue(new FunctionContext([this](int r) {
735 pre_release_lock_handler(true, create_context_callback<
736 ManagedLock<I>, &ManagedLock<I>::handle_shutdown_pre_release>(this));
740 template <typename I>
741 void ManagedLock<I>::handle_shutdown_pre_release(int r) {
742 ldout(m_cct, 10) << ": r=" << r << dendl;
746 Mutex::Locker locker(m_lock);
749 assert(m_state == STATE_PRE_SHUTTING_DOWN);
750 m_state = STATE_SHUTTING_DOWN;
753 using managed_lock::ReleaseRequest;
754 ReleaseRequest<I>* req = ReleaseRequest<I>::create(m_ioctx, m_watcher,
755 m_work_queue, m_oid, cookie,
756 new FunctionContext([this](int r) {
757 post_release_lock_handler(true, r, create_context_callback<
758 ManagedLock<I>, &ManagedLock<I>::handle_shutdown_post_release>(this));
764 template <typename I>
765 void ManagedLock<I>::handle_shutdown_post_release(int r) {
766 ldout(m_cct, 10) << ": r=" << r << dendl;
768 wait_for_tracked_ops(r);
771 template <typename I>
772 void ManagedLock<I>::wait_for_tracked_ops(int r) {
773 ldout(m_cct, 10) << ": r=" << r << dendl;
775 Context *ctx = new FunctionContext([this, r](int ret) {
776 complete_shutdown(r);
779 m_async_op_tracker.wait_for_ops(ctx);
782 template <typename I>
783 void ManagedLock<I>::complete_shutdown(int r) {
784 ldout(m_cct, 10) << ": r=" << r << dendl;
787 lderr(m_cct) << "failed to shut down lock: " << cpp_strerror(r)
791 ActionContexts action_contexts;
793 Mutex::Locker locker(m_lock);
794 assert(m_lock.is_locked());
795 assert(m_actions_contexts.size() == 1);
797 action_contexts = std::move(m_actions_contexts.front());
798 m_actions_contexts.pop_front();
799 m_state = STATE_SHUTDOWN;
802 // expect to be destroyed after firing callback
803 for (auto ctx : action_contexts.second) {
808 } // namespace librbd
810 template class librbd::ManagedLock<librbd::ImageCtx>;