1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
2 // vim: ts=8 sw=2 smarttab
4 #include "librbd/ImageState.h"
5 #include "include/rbd/librbd.hpp"
6 #include "common/dout.h"
7 #include "common/errno.h"
8 #include "common/Cond.h"
9 #include "common/WorkQueue.h"
10 #include "librbd/ImageCtx.h"
11 #include "librbd/Utils.h"
12 #include "librbd/image/CloseRequest.h"
13 #include "librbd/image/OpenRequest.h"
14 #include "librbd/image/RefreshRequest.h"
15 #include "librbd/image/SetSnapRequest.h"
17 #define dout_subsys ceph_subsys_rbd
19 #define dout_prefix *_dout << "librbd::ImageState: " << this << " "
23 using util::create_async_context_callback;
24 using util::create_context_callback;
26 class ImageUpdateWatchers {
29 ImageUpdateWatchers(CephContext *cct) : m_cct(cct),
30 m_lock(util::unique_lock_name("librbd::ImageUpdateWatchers::m_lock", this)) {
33 ~ImageUpdateWatchers() {
34 assert(m_watchers.empty());
35 assert(m_in_flight.empty());
36 assert(m_pending_unregister.empty());
37 assert(m_on_shut_down_finish == nullptr);
42 void flush(Context *on_finish) {
43 ldout(m_cct, 20) << "ImageUpdateWatchers::" << __func__ << dendl;
45 Mutex::Locker locker(m_lock);
46 if (!m_in_flight.empty()) {
47 Context *ctx = new FunctionContext(
48 [this, on_finish](int r) {
49 ldout(m_cct, 20) << "ImageUpdateWatchers::" << __func__
50 << ": completing flush" << dendl;
51 on_finish->complete(r);
53 m_work_queue->queue(ctx, 0);
57 ldout(m_cct, 20) << "ImageUpdateWatchers::" << __func__
58 << ": completing flush" << dendl;
59 on_finish->complete(0);
62 void shut_down(Context *on_finish) {
63 ldout(m_cct, 20) << "ImageUpdateWatchers::" << __func__ << dendl;
65 Mutex::Locker locker(m_lock);
66 assert(m_on_shut_down_finish == nullptr);
68 if (!m_in_flight.empty()) {
69 m_on_shut_down_finish = on_finish;
73 ldout(m_cct, 20) << "ImageUpdateWatchers::" << __func__
74 << ": completing shut down" << dendl;
75 on_finish->complete(0);
78 void register_watcher(UpdateWatchCtx *watcher, uint64_t *handle) {
79 ldout(m_cct, 20) << __func__ << ": watcher=" << watcher << dendl;
81 Mutex::Locker locker(m_lock);
82 assert(m_on_shut_down_finish == nullptr);
86 *handle = m_next_handle++;
87 m_watchers.insert(std::make_pair(*handle, watcher));
90 void unregister_watcher(uint64_t handle, Context *on_finish) {
91 ldout(m_cct, 20) << "ImageUpdateWatchers::" << __func__ << ": handle="
95 Mutex::Locker locker(m_lock);
96 auto it = m_watchers.find(handle);
97 if (it == m_watchers.end()) {
100 if (m_in_flight.find(handle) != m_in_flight.end()) {
101 assert(m_pending_unregister.find(handle) == m_pending_unregister.end());
102 m_pending_unregister[handle] = on_finish;
105 m_watchers.erase(it);
110 ldout(m_cct, 20) << "ImageUpdateWatchers::" << __func__
111 << ": completing unregister" << dendl;
112 on_finish->complete(r);
117 ldout(m_cct, 20) << "ImageUpdateWatchers::" << __func__ << dendl;
119 Mutex::Locker locker(m_lock);
120 for (auto it : m_watchers) {
121 send_notify(it.first, it.second);
125 void send_notify(uint64_t handle, UpdateWatchCtx *watcher) {
126 assert(m_lock.is_locked());
128 ldout(m_cct, 20) << "ImageUpdateWatchers::" << __func__ << ": handle="
129 << handle << ", watcher=" << watcher << dendl;
131 m_in_flight.insert(handle);
133 Context *ctx = new FunctionContext(
134 [this, handle, watcher](int r) {
135 handle_notify(handle, watcher);
138 m_work_queue->queue(ctx, 0);
141 void handle_notify(uint64_t handle, UpdateWatchCtx *watcher) {
143 ldout(m_cct, 20) << "ImageUpdateWatchers::" << __func__ << ": handle="
144 << handle << ", watcher=" << watcher << dendl;
146 watcher->handle_notify();
148 Context *on_unregister_finish = nullptr;
149 Context *on_shut_down_finish = nullptr;
152 Mutex::Locker locker(m_lock);
154 auto in_flight_it = m_in_flight.find(handle);
155 assert(in_flight_it != m_in_flight.end());
156 m_in_flight.erase(in_flight_it);
158 // If there is no more in flight notifications for this watcher
159 // and it is pending unregister, complete it now.
160 if (m_in_flight.find(handle) == m_in_flight.end()) {
161 auto it = m_pending_unregister.find(handle);
162 if (it != m_pending_unregister.end()) {
163 on_unregister_finish = it->second;
164 m_pending_unregister.erase(it);
168 if (m_in_flight.empty()) {
169 assert(m_pending_unregister.empty());
170 if (m_on_shut_down_finish != nullptr) {
171 std::swap(m_on_shut_down_finish, on_shut_down_finish);
176 if (on_unregister_finish != nullptr) {
177 ldout(m_cct, 20) << "ImageUpdateWatchers::" << __func__
178 << ": completing unregister" << dendl;
179 on_unregister_finish->complete(0);
182 if (on_shut_down_finish != nullptr) {
183 ldout(m_cct, 20) << "ImageUpdateWatchers::" << __func__
184 << ": completing shut down" << dendl;
185 on_shut_down_finish->complete(0);
190 class ThreadPoolSingleton : public ThreadPool {
192 explicit ThreadPoolSingleton(CephContext *cct)
193 : ThreadPool(cct, "librbd::ImageUpdateWatchers::thread_pool", "tp_librbd",
197 ~ThreadPoolSingleton() override {
204 ContextWQ *m_work_queue = nullptr;
205 std::map<uint64_t, UpdateWatchCtx*> m_watchers;
206 uint64_t m_next_handle = 0;
207 std::multiset<uint64_t> m_in_flight;
208 std::map<uint64_t, Context*> m_pending_unregister;
209 Context *m_on_shut_down_finish = nullptr;
211 void create_work_queue() {
212 if (m_work_queue != nullptr) {
215 ThreadPoolSingleton *thread_pool_singleton;
216 m_cct->lookup_or_create_singleton_object<ThreadPoolSingleton>(
217 thread_pool_singleton, "librbd::ImageUpdateWatchers::thread_pool");
218 m_work_queue = new ContextWQ("librbd::ImageUpdateWatchers::op_work_queue",
219 m_cct->_conf->get_val<int64_t>("rbd_op_thread_timeout"),
220 thread_pool_singleton);
223 void destroy_work_queue() {
224 if (m_work_queue == nullptr) {
227 m_work_queue->drain();
232 template <typename I>
233 ImageState<I>::ImageState(I *image_ctx)
234 : m_image_ctx(image_ctx), m_state(STATE_UNINITIALIZED),
235 m_lock(util::unique_lock_name("librbd::ImageState::m_lock", this)),
236 m_last_refresh(0), m_refresh_seq(0),
237 m_update_watchers(new ImageUpdateWatchers(image_ctx->cct)),
238 m_skip_open_parent_image(false) {
241 template <typename I>
242 ImageState<I>::~ImageState() {
243 assert(m_state == STATE_UNINITIALIZED || m_state == STATE_CLOSED);
244 delete m_update_watchers;
247 template <typename I>
248 int ImageState<I>::open(bool skip_open_parent) {
250 open(skip_open_parent, &ctx);
259 template <typename I>
260 void ImageState<I>::open(bool skip_open_parent, Context *on_finish) {
261 CephContext *cct = m_image_ctx->cct;
262 ldout(cct, 20) << __func__ << dendl;
265 assert(m_state == STATE_UNINITIALIZED);
266 m_skip_open_parent_image = skip_open_parent;
268 Action action(ACTION_TYPE_OPEN);
269 action.refresh_seq = m_refresh_seq;
271 execute_action_unlock(action, on_finish);
274 template <typename I>
275 int ImageState<I>::close() {
284 template <typename I>
285 void ImageState<I>::close(Context *on_finish) {
286 CephContext *cct = m_image_ctx->cct;
287 ldout(cct, 20) << __func__ << dendl;
290 assert(!is_closed());
292 Action action(ACTION_TYPE_CLOSE);
293 action.refresh_seq = m_refresh_seq;
294 execute_action_unlock(action, on_finish);
297 template <typename I>
298 void ImageState<I>::handle_update_notification() {
299 Mutex::Locker locker(m_lock);
302 CephContext *cct = m_image_ctx->cct;
303 ldout(cct, 20) << __func__ << ": refresh_seq = " << m_refresh_seq << ", "
304 << "last_refresh = " << m_last_refresh << dendl;
306 if (m_state == STATE_OPEN) {
307 m_update_watchers->notify();
311 template <typename I>
312 bool ImageState<I>::is_refresh_required() const {
313 Mutex::Locker locker(m_lock);
314 return (m_last_refresh != m_refresh_seq || find_pending_refresh() != nullptr);
317 template <typename I>
318 int ImageState<I>::refresh() {
319 C_SaferCond refresh_ctx;
320 refresh(&refresh_ctx);
321 return refresh_ctx.wait();
324 template <typename I>
325 void ImageState<I>::refresh(Context *on_finish) {
326 CephContext *cct = m_image_ctx->cct;
327 ldout(cct, 20) << __func__ << dendl;
332 on_finish->complete(-ESHUTDOWN);
336 Action action(ACTION_TYPE_REFRESH);
337 action.refresh_seq = m_refresh_seq;
338 execute_action_unlock(action, on_finish);
341 template <typename I>
342 int ImageState<I>::refresh_if_required() {
346 Action action(ACTION_TYPE_REFRESH);
347 action.refresh_seq = m_refresh_seq;
349 auto refresh_action = find_pending_refresh();
350 if (refresh_action != nullptr) {
351 // if a refresh is in-flight, delay until it is finished
352 action = *refresh_action;
353 } else if (m_last_refresh == m_refresh_seq) {
356 } else if (is_closed()) {
361 execute_action_unlock(action, &ctx);
367 template <typename I>
368 const typename ImageState<I>::Action *
369 ImageState<I>::find_pending_refresh() const {
370 assert(m_lock.is_locked());
372 auto it = std::find_if(m_actions_contexts.rbegin(),
373 m_actions_contexts.rend(),
374 [](const ActionContexts& action_contexts) {
375 return (action_contexts.first == ACTION_TYPE_REFRESH);
377 if (it != m_actions_contexts.rend()) {
383 template <typename I>
384 void ImageState<I>::snap_set(const cls::rbd::SnapshotNamespace &snap_namespace,
385 const std::string &snap_name,
386 Context *on_finish) {
387 CephContext *cct = m_image_ctx->cct;
388 ldout(cct, 20) << __func__ << ": snap_name=" << snap_name << dendl;
390 Action action(ACTION_TYPE_SET_SNAP);
391 action.snap_namespace = snap_namespace;
392 action.snap_name = snap_name;
395 execute_action_unlock(action, on_finish);
398 template <typename I>
399 void ImageState<I>::prepare_lock(Context *on_ready) {
400 CephContext *cct = m_image_ctx->cct;
401 ldout(cct, 10) << __func__ << dendl;
406 on_ready->complete(-ESHUTDOWN);
410 Action action(ACTION_TYPE_LOCK);
411 action.on_ready = on_ready;
412 execute_action_unlock(action, nullptr);
415 template <typename I>
416 void ImageState<I>::handle_prepare_lock_complete() {
417 CephContext *cct = m_image_ctx->cct;
418 ldout(cct, 10) << __func__ << dendl;
421 if (m_state != STATE_PREPARING_LOCK) {
426 complete_action_unlock(STATE_OPEN, 0);
429 template <typename I>
430 int ImageState<I>::register_update_watcher(UpdateWatchCtx *watcher,
432 CephContext *cct = m_image_ctx->cct;
433 ldout(cct, 20) << __func__ << dendl;
435 m_update_watchers->register_watcher(watcher, handle);
437 ldout(cct, 20) << __func__ << ": handle=" << *handle << dendl;
441 template <typename I>
442 int ImageState<I>::unregister_update_watcher(uint64_t handle) {
443 CephContext *cct = m_image_ctx->cct;
444 ldout(cct, 20) << __func__ << ": handle=" << handle << dendl;
447 m_update_watchers->unregister_watcher(handle, &ctx);
451 template <typename I>
452 void ImageState<I>::flush_update_watchers(Context *on_finish) {
453 CephContext *cct = m_image_ctx->cct;
454 ldout(cct, 20) << __func__ << dendl;
456 m_update_watchers->flush(on_finish);
459 template <typename I>
460 void ImageState<I>::shut_down_update_watchers(Context *on_finish) {
461 CephContext *cct = m_image_ctx->cct;
462 ldout(cct, 20) << __func__ << dendl;
464 m_update_watchers->shut_down(on_finish);
467 template <typename I>
468 bool ImageState<I>::is_transition_state() const {
470 case STATE_UNINITIALIZED:
476 case STATE_REFRESHING:
477 case STATE_SETTING_SNAP:
478 case STATE_PREPARING_LOCK:
484 template <typename I>
485 bool ImageState<I>::is_closed() const {
486 assert(m_lock.is_locked());
488 return ((m_state == STATE_CLOSED) ||
489 (!m_actions_contexts.empty() &&
490 m_actions_contexts.back().first.action_type == ACTION_TYPE_CLOSE));
493 template <typename I>
494 void ImageState<I>::append_context(const Action &action, Context *context) {
495 assert(m_lock.is_locked());
497 ActionContexts *action_contexts = nullptr;
498 for (auto &action_ctxs : m_actions_contexts) {
499 if (action == action_ctxs.first) {
500 action_contexts = &action_ctxs;
505 if (action_contexts == nullptr) {
506 m_actions_contexts.push_back({action, {}});
507 action_contexts = &m_actions_contexts.back();
510 if (context != nullptr) {
511 action_contexts->second.push_back(context);
515 template <typename I>
516 void ImageState<I>::execute_next_action_unlock() {
517 assert(m_lock.is_locked());
518 assert(!m_actions_contexts.empty());
519 switch (m_actions_contexts.front().first.action_type) {
520 case ACTION_TYPE_OPEN:
523 case ACTION_TYPE_CLOSE:
526 case ACTION_TYPE_REFRESH:
527 send_refresh_unlock();
529 case ACTION_TYPE_SET_SNAP:
530 send_set_snap_unlock();
532 case ACTION_TYPE_LOCK:
533 send_prepare_lock_unlock();
539 template <typename I>
540 void ImageState<I>::execute_action_unlock(const Action &action,
541 Context *on_finish) {
542 assert(m_lock.is_locked());
544 append_context(action, on_finish);
545 if (!is_transition_state()) {
546 execute_next_action_unlock();
552 template <typename I>
553 void ImageState<I>::complete_action_unlock(State next_state, int r) {
554 assert(m_lock.is_locked());
555 assert(!m_actions_contexts.empty());
557 ActionContexts action_contexts(std::move(m_actions_contexts.front()));
558 m_actions_contexts.pop_front();
560 m_state = next_state;
563 for (auto ctx : action_contexts.second) {
567 if (next_state != STATE_UNINITIALIZED && next_state != STATE_CLOSED) {
569 if (!is_transition_state() && !m_actions_contexts.empty()) {
570 execute_next_action_unlock();
577 template <typename I>
578 void ImageState<I>::send_open_unlock() {
579 assert(m_lock.is_locked());
580 CephContext *cct = m_image_ctx->cct;
581 ldout(cct, 10) << this << " " << __func__ << dendl;
583 m_state = STATE_OPENING;
585 Context *ctx = create_async_context_callback(
586 *m_image_ctx, create_context_callback<
587 ImageState<I>, &ImageState<I>::handle_open>(this));
588 image::OpenRequest<I> *req = image::OpenRequest<I>::create(
589 m_image_ctx, m_skip_open_parent_image, ctx);
595 template <typename I>
596 void ImageState<I>::handle_open(int r) {
597 CephContext *cct = m_image_ctx->cct;
598 ldout(cct, 10) << this << " " << __func__ << ": r=" << r << dendl;
600 if (r < 0 && r != -ENOENT) {
601 lderr(cct) << "failed to open image: " << cpp_strerror(r) << dendl;
605 complete_action_unlock(r < 0 ? STATE_UNINITIALIZED : STATE_OPEN, r);
608 template <typename I>
609 void ImageState<I>::send_close_unlock() {
610 assert(m_lock.is_locked());
611 CephContext *cct = m_image_ctx->cct;
612 ldout(cct, 10) << this << " " << __func__ << dendl;
614 m_state = STATE_CLOSING;
616 Context *ctx = create_context_callback<
617 ImageState<I>, &ImageState<I>::handle_close>(this);
618 image::CloseRequest<I> *req = image::CloseRequest<I>::create(
625 template <typename I>
626 void ImageState<I>::handle_close(int r) {
627 CephContext *cct = m_image_ctx->cct;
628 ldout(cct, 10) << this << " " << __func__ << ": r=" << r << dendl;
631 lderr(cct) << "error occurred while closing image: " << cpp_strerror(r)
636 complete_action_unlock(STATE_CLOSED, r);
639 template <typename I>
640 void ImageState<I>::send_refresh_unlock() {
641 assert(m_lock.is_locked());
642 CephContext *cct = m_image_ctx->cct;
643 ldout(cct, 10) << this << " " << __func__ << dendl;
645 m_state = STATE_REFRESHING;
646 assert(!m_actions_contexts.empty());
647 auto &action_context = m_actions_contexts.front().first;
648 assert(action_context.action_type == ACTION_TYPE_REFRESH);
650 Context *ctx = create_async_context_callback(
651 *m_image_ctx, create_context_callback<
652 ImageState<I>, &ImageState<I>::handle_refresh>(this));
653 image::RefreshRequest<I> *req = image::RefreshRequest<I>::create(
654 *m_image_ctx, false, false, ctx);
660 template <typename I>
661 void ImageState<I>::handle_refresh(int r) {
662 CephContext *cct = m_image_ctx->cct;
663 ldout(cct, 10) << this << " " << __func__ << ": r=" << r << dendl;
666 assert(!m_actions_contexts.empty());
668 ActionContexts &action_contexts(m_actions_contexts.front());
669 assert(action_contexts.first.action_type == ACTION_TYPE_REFRESH);
670 assert(m_last_refresh <= action_contexts.first.refresh_seq);
672 if (r == -ERESTART) {
673 ldout(cct, 5) << "incomplete refresh: not updating sequence" << dendl;
676 m_last_refresh = action_contexts.first.refresh_seq;
679 complete_action_unlock(STATE_OPEN, r);
682 template <typename I>
683 void ImageState<I>::send_set_snap_unlock() {
684 assert(m_lock.is_locked());
686 m_state = STATE_SETTING_SNAP;
688 assert(!m_actions_contexts.empty());
689 ActionContexts &action_contexts(m_actions_contexts.front());
690 assert(action_contexts.first.action_type == ACTION_TYPE_SET_SNAP);
692 CephContext *cct = m_image_ctx->cct;
693 ldout(cct, 10) << this << " " << __func__ << ": "
694 << "snap_name=" << action_contexts.first.snap_name << dendl;
696 Context *ctx = create_async_context_callback(
697 *m_image_ctx, create_context_callback<
698 ImageState<I>, &ImageState<I>::handle_set_snap>(this));
699 image::SetSnapRequest<I> *req = image::SetSnapRequest<I>::create(
700 *m_image_ctx, action_contexts.first.snap_namespace, action_contexts.first.snap_name, ctx);
706 template <typename I>
707 void ImageState<I>::handle_set_snap(int r) {
708 CephContext *cct = m_image_ctx->cct;
709 ldout(cct, 10) << this << " " << __func__ << " r=" << r << dendl;
711 if (r < 0 && r != -ENOENT) {
712 lderr(cct) << "failed to set snapshot: " << cpp_strerror(r) << dendl;
716 complete_action_unlock(STATE_OPEN, r);
719 template <typename I>
720 void ImageState<I>::send_prepare_lock_unlock() {
721 CephContext *cct = m_image_ctx->cct;
722 ldout(cct, 10) << this << " " << __func__ << dendl;
724 assert(m_lock.is_locked());
725 m_state = STATE_PREPARING_LOCK;
727 assert(!m_actions_contexts.empty());
728 ActionContexts &action_contexts(m_actions_contexts.front());
729 assert(action_contexts.first.action_type == ACTION_TYPE_LOCK);
731 Context *on_ready = action_contexts.first.on_ready;
734 if (on_ready == nullptr) {
735 complete_action_unlock(STATE_OPEN, 0);
739 // wake up the lock handler now that its safe to proceed
740 on_ready->complete(0);
743 } // namespace librbd
745 template class librbd::ImageState<librbd::ImageCtx>;