1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
2 // vim: ts=8 sw=2 smarttab
4 #include "include/compat.h"
5 #include "BootstrapRequest.h"
6 #include "CloseImageRequest.h"
7 #include "CreateImageRequest.h"
8 #include "IsPrimaryRequest.h"
9 #include "OpenImageRequest.h"
10 #include "OpenLocalImageRequest.h"
11 #include "common/debug.h"
12 #include "common/dout.h"
13 #include "common/errno.h"
14 #include "common/WorkQueue.h"
15 #include "cls/rbd/cls_rbd_client.h"
16 #include "journal/Journaler.h"
17 #include "librbd/ImageCtx.h"
18 #include "librbd/ImageState.h"
19 #include "librbd/internal.h"
20 #include "librbd/Journal.h"
21 #include "librbd/Utils.h"
22 #include "librbd/journal/Types.h"
23 #include "tools/rbd_mirror/ProgressContext.h"
24 #include "tools/rbd_mirror/ImageSync.h"
26 #define dout_context g_ceph_context
27 #define dout_subsys ceph_subsys_rbd_mirror
29 #define dout_prefix *_dout << "rbd::mirror::image_replayer::BootstrapRequest: " \
30 << this << " " << __func__
34 namespace image_replayer {
36 using librbd::util::create_context_callback;
37 using librbd::util::create_rados_callback;
38 using librbd::util::unique_lock_name;
41 BootstrapRequest<I>::BootstrapRequest(
42 librados::IoCtx &local_io_ctx,
43 librados::IoCtx &remote_io_ctx,
44 InstanceWatcher<I> *instance_watcher,
46 const std::string &local_image_id,
47 const std::string &remote_image_id,
48 const std::string &global_image_id,
49 ContextWQ *work_queue, SafeTimer *timer,
51 const std::string &local_mirror_uuid,
52 const std::string &remote_mirror_uuid,
54 MirrorPeerClientMeta *client_meta,
57 rbd::mirror::ProgressContext *progress_ctx)
58 : BaseRequest("rbd::mirror::image_replayer::BootstrapRequest",
59 reinterpret_cast<CephContext*>(local_io_ctx.cct()), on_finish),
60 m_local_io_ctx(local_io_ctx), m_remote_io_ctx(remote_io_ctx),
61 m_instance_watcher(instance_watcher), m_local_image_ctx(local_image_ctx),
62 m_local_image_id(local_image_id), m_remote_image_id(remote_image_id),
63 m_global_image_id(global_image_id), m_work_queue(work_queue),
64 m_timer(timer), m_timer_lock(timer_lock),
65 m_local_mirror_uuid(local_mirror_uuid),
66 m_remote_mirror_uuid(remote_mirror_uuid), m_journaler(journaler),
67 m_client_meta(client_meta), m_progress_ctx(progress_ctx),
68 m_do_resync(do_resync),
69 m_lock(unique_lock_name("BootstrapRequest::m_lock", this)) {
73 BootstrapRequest<I>::~BootstrapRequest() {
74 assert(m_remote_image_ctx == nullptr);
78 bool BootstrapRequest<I>::is_syncing() const {
79 Mutex::Locker locker(m_lock);
80 return (m_image_sync != nullptr);
84 void BootstrapRequest<I>::send() {
87 get_remote_tag_class();
91 void BootstrapRequest<I>::cancel() {
94 Mutex::Locker locker(m_lock);
97 if (m_image_sync != nullptr) {
98 m_image_sync->cancel();
102 template <typename I>
103 void BootstrapRequest<I>::get_remote_tag_class() {
106 update_progress("GET_REMOTE_TAG_CLASS");
108 Context *ctx = create_context_callback<
109 BootstrapRequest<I>, &BootstrapRequest<I>::handle_get_remote_tag_class>(
111 m_journaler->get_client(librbd::Journal<>::IMAGE_CLIENT_ID, &m_client, ctx);
114 template <typename I>
115 void BootstrapRequest<I>::handle_get_remote_tag_class(int r) {
116 dout(20) << ": r=" << r << dendl;
119 derr << ": failed to retrieve remote client: " << cpp_strerror(r) << dendl;
124 librbd::journal::ClientData client_data;
125 bufferlist::iterator it = m_client.data.begin();
127 ::decode(client_data, it);
128 } catch (const buffer::error &err) {
129 derr << ": failed to decode remote client meta data: " << err.what()
135 librbd::journal::ImageClientMeta *client_meta =
136 boost::get<librbd::journal::ImageClientMeta>(&client_data.client_meta);
137 if (client_meta == nullptr) {
138 derr << ": unknown remote client registration" << dendl;
143 m_remote_tag_class = client_meta->tag_class;
144 dout(10) << ": remote tag class=" << m_remote_tag_class << dendl;
149 template <typename I>
150 void BootstrapRequest<I>::open_remote_image() {
153 update_progress("OPEN_REMOTE_IMAGE");
155 Context *ctx = create_context_callback<
156 BootstrapRequest<I>, &BootstrapRequest<I>::handle_open_remote_image>(
158 OpenImageRequest<I> *request = OpenImageRequest<I>::create(
159 m_remote_io_ctx, &m_remote_image_ctx, m_remote_image_id, false,
164 template <typename I>
165 void BootstrapRequest<I>::handle_open_remote_image(int r) {
166 dout(20) << ": r=" << r << dendl;
169 derr << ": failed to open remote image: " << cpp_strerror(r) << dendl;
170 assert(m_remote_image_ctx == nullptr);
178 template <typename I>
179 void BootstrapRequest<I>::get_client() {
182 update_progress("GET_CLIENT");
184 Context *ctx = create_context_callback<
185 BootstrapRequest<I>, &BootstrapRequest<I>::handle_get_client>(
187 m_journaler->get_client(m_local_mirror_uuid, &m_client, ctx);
190 template <typename I>
191 void BootstrapRequest<I>::handle_get_client(int r) {
192 dout(20) << ": r=" << r << dendl;
195 dout(10) << ": client not registered" << dendl;
197 derr << ": failed to retrieve client: " << cpp_strerror(r) << dendl;
199 close_remote_image();
201 } else if (decode_client_meta()) {
202 // skip registration if it already exists
210 template <typename I>
211 void BootstrapRequest<I>::register_client() {
214 update_progress("REGISTER_CLIENT");
216 librbd::journal::MirrorPeerClientMeta mirror_peer_client_meta{
218 mirror_peer_client_meta.state = librbd::journal::MIRROR_PEER_STATE_REPLAYING;
220 librbd::journal::ClientData client_data{mirror_peer_client_meta};
221 bufferlist client_data_bl;
222 ::encode(client_data, client_data_bl);
224 Context *ctx = create_context_callback<
225 BootstrapRequest<I>, &BootstrapRequest<I>::handle_register_client>(
227 m_journaler->register_client(client_data_bl, ctx);
230 template <typename I>
231 void BootstrapRequest<I>::handle_register_client(int r) {
232 dout(20) << ": r=" << r << dendl;
235 derr << ": failed to register with remote journal: " << cpp_strerror(r)
238 close_remote_image();
243 *m_client_meta = librbd::journal::MirrorPeerClientMeta(m_local_image_id);
244 m_client_meta->state = librbd::journal::MIRROR_PEER_STATE_REPLAYING;
249 template <typename I>
250 void BootstrapRequest<I>::is_primary() {
253 update_progress("OPEN_REMOTE_IMAGE");
255 Context *ctx = create_context_callback<
256 BootstrapRequest<I>, &BootstrapRequest<I>::handle_is_primary>(
258 IsPrimaryRequest<I> *request = IsPrimaryRequest<I>::create(m_remote_image_ctx,
263 template <typename I>
264 void BootstrapRequest<I>::handle_is_primary(int r) {
265 dout(20) << ": r=" << r << dendl;
268 derr << ": error querying remote image primary status: " << cpp_strerror(r)
271 close_remote_image();
276 dout(5) << ": remote image is not primary -- skipping image replay"
278 m_ret_val = -EREMOTEIO;
279 update_client_state();
283 if (m_local_image_id.empty()) {
284 update_client_image();
291 template <typename I>
292 void BootstrapRequest<I>::update_client_state() {
293 if (m_client_meta->state == librbd::journal::MIRROR_PEER_STATE_REPLAYING) {
294 // state already set for replaying upon failover
295 close_remote_image();
300 update_progress("UPDATE_CLIENT_STATE");
302 librbd::journal::MirrorPeerClientMeta client_meta(*m_client_meta);
303 client_meta.state = librbd::journal::MIRROR_PEER_STATE_REPLAYING;
305 librbd::journal::ClientData client_data(client_meta);
307 ::encode(client_data, data_bl);
309 Context *ctx = create_context_callback<
310 BootstrapRequest<I>, &BootstrapRequest<I>::handle_update_client_state>(
312 m_journaler->update_client(data_bl, ctx);
315 template <typename I>
316 void BootstrapRequest<I>::handle_update_client_state(int r) {
317 dout(20) << ": r=" << r << dendl;
319 derr << ": failed to update client: " << cpp_strerror(r) << dendl;
321 m_client_meta->state = librbd::journal::MIRROR_PEER_STATE_REPLAYING;
324 close_remote_image();
327 template <typename I>
328 void BootstrapRequest<I>::open_local_image() {
331 update_progress("OPEN_LOCAL_IMAGE");
333 Context *ctx = create_context_callback<
334 BootstrapRequest<I>, &BootstrapRequest<I>::handle_open_local_image>(
336 OpenLocalImageRequest<I> *request = OpenLocalImageRequest<I>::create(
337 m_local_io_ctx, m_local_image_ctx, m_local_image_id, m_work_queue,
342 template <typename I>
343 void BootstrapRequest<I>::handle_open_local_image(int r) {
344 dout(20) << ": r=" << r << dendl;
347 assert(*m_local_image_ctx == nullptr);
348 dout(10) << ": local image missing" << dendl;
351 } else if (r == -EREMOTEIO) {
352 assert(*m_local_image_ctx == nullptr);
353 dout(10) << "local image is primary -- skipping image replay" << dendl;
355 close_remote_image();
358 assert(*m_local_image_ctx == nullptr);
359 derr << ": failed to open local image: " << cpp_strerror(r) << dendl;
361 close_remote_image();
365 I *local_image_ctx = (*m_local_image_ctx);
367 RWLock::RLocker snap_locker(local_image_ctx->snap_lock);
368 if (local_image_ctx->journal == nullptr) {
369 derr << ": local image does not support journaling" << dendl;
375 r = (*m_local_image_ctx)->journal->is_resync_requested(m_do_resync);
377 derr << ": failed to check if a resync was requested" << dendl;
385 close_remote_image();
389 if (m_client.state == cls::journal::CLIENT_STATE_DISCONNECTED) {
390 dout(10) << ": client flagged disconnected -- skipping bootstrap" << dendl;
391 // The caller is expected to detect disconnect initializing remote journal.
393 close_remote_image();
400 template <typename I>
401 void BootstrapRequest<I>::unregister_client() {
403 update_progress("UNREGISTER_CLIENT");
405 m_local_image_id = "";
406 Context *ctx = create_context_callback<
407 BootstrapRequest<I>, &BootstrapRequest<I>::handle_unregister_client>(
409 m_journaler->unregister_client(ctx);
412 template <typename I>
413 void BootstrapRequest<I>::handle_unregister_client(int r) {
414 dout(20) << ": r=" << r << dendl;
416 derr << ": failed to unregister with remote journal: " << cpp_strerror(r)
419 close_remote_image();
423 *m_client_meta = librbd::journal::MirrorPeerClientMeta("");
427 template <typename I>
428 void BootstrapRequest<I>::update_client_image() {
430 update_progress("UPDATE_CLIENT_IMAGE");
432 assert(m_local_image_id.empty());
433 m_local_image_id = librbd::util::generate_image_id<I>(m_local_io_ctx);
435 librbd::journal::MirrorPeerClientMeta client_meta{m_local_image_id};
436 client_meta.state = librbd::journal::MIRROR_PEER_STATE_SYNCING;
438 librbd::journal::ClientData client_data(client_meta);
440 ::encode(client_data, data_bl);
442 Context *ctx = create_context_callback<
443 BootstrapRequest<I>, &BootstrapRequest<I>::handle_update_client_image>(
445 m_journaler->update_client(data_bl, ctx);
448 template <typename I>
449 void BootstrapRequest<I>::handle_update_client_image(int r) {
450 dout(20) << ": r=" << r << dendl;
453 derr << ": failed to update client: " << cpp_strerror(r) << dendl;
455 close_remote_image();
460 dout(10) << ": request canceled" << dendl;
461 m_ret_val = -ECANCELED;
462 close_remote_image();
466 *m_client_meta = {m_local_image_id};
467 m_client_meta->state = librbd::journal::MIRROR_PEER_STATE_SYNCING;
468 create_local_image();
471 template <typename I>
472 void BootstrapRequest<I>::create_local_image() {
474 update_progress("CREATE_LOCAL_IMAGE");
476 m_remote_image_ctx->snap_lock.get_read();
477 std::string image_name = m_remote_image_ctx->name;
478 m_remote_image_ctx->snap_lock.put_read();
480 Context *ctx = create_context_callback<
481 BootstrapRequest<I>, &BootstrapRequest<I>::handle_create_local_image>(
483 CreateImageRequest<I> *request = CreateImageRequest<I>::create(
484 m_local_io_ctx, m_work_queue, m_global_image_id, m_remote_mirror_uuid,
485 image_name, m_local_image_id, m_remote_image_ctx, ctx);
489 template <typename I>
490 void BootstrapRequest<I>::handle_create_local_image(int r) {
491 dout(20) << ": r=" << r << dendl;
494 derr << ": failed to create local image: " << cpp_strerror(r) << dendl;
496 close_remote_image();
503 template <typename I>
504 void BootstrapRequest<I>::get_remote_tags() {
505 if (m_client_meta->state == librbd::journal::MIRROR_PEER_STATE_SYNCING) {
506 // optimization -- no need to compare remote tags if we just created
507 // the image locally or sync was interrupted
513 update_progress("GET_REMOTE_TAGS");
515 Context *ctx = create_context_callback<
516 BootstrapRequest<I>, &BootstrapRequest<I>::handle_get_remote_tags>(this);
517 m_journaler->get_tags(m_remote_tag_class, &m_remote_tags, ctx);
520 template <typename I>
521 void BootstrapRequest<I>::handle_get_remote_tags(int r) {
522 dout(20) << ": r=" << r << dendl;
525 derr << ": failed to retrieve remote tags: " << cpp_strerror(r) << dendl;
532 dout(10) << ": request canceled" << dendl;
533 m_ret_val = -ECANCELED;
538 // At this point, the local image was existing, non-primary, and replaying;
539 // and the remote image is primary. Attempt to link the local image's most
540 // recent tag to the remote image's tag chain.
541 uint64_t local_tag_tid;
542 librbd::journal::TagData local_tag_data;
543 I *local_image_ctx = (*m_local_image_ctx);
545 RWLock::RLocker snap_locker(local_image_ctx->snap_lock);
546 if (local_image_ctx->journal == nullptr) {
547 derr << ": local image does not support journaling" << dendl;
553 local_tag_tid = local_image_ctx->journal->get_tag_tid();
554 local_tag_data = local_image_ctx->journal->get_tag_data();
555 dout(20) << ": local tag " << local_tag_tid << ": "
556 << local_tag_data << dendl;
559 bool remote_tag_data_valid = false;
560 librbd::journal::TagData remote_tag_data;
561 boost::optional<uint64_t> remote_orphan_tag_tid =
562 boost::make_optional<uint64_t>(false, 0U);
563 bool reconnect_orphan = false;
565 // decode the remote tags
566 for (auto &remote_tag : m_remote_tags) {
567 if (local_tag_data.predecessor.commit_valid &&
568 local_tag_data.predecessor.mirror_uuid == m_remote_mirror_uuid &&
569 local_tag_data.predecessor.tag_tid > remote_tag.tid) {
570 dout(20) << ": skipping processed predecessor remote tag "
571 << remote_tag.tid << dendl;
576 bufferlist::iterator it = remote_tag.data.begin();
577 ::decode(remote_tag_data, it);
578 remote_tag_data_valid = true;
579 } catch (const buffer::error &err) {
580 derr << ": failed to decode remote tag " << remote_tag.tid << ": "
581 << err.what() << dendl;
582 m_ret_val = -EBADMSG;
587 dout(10) << ": decoded remote tag " << remote_tag.tid << ": "
588 << remote_tag_data << dendl;
590 if (!local_tag_data.predecessor.commit_valid) {
591 // newly synced local image (no predecessor) replays from the first tag
592 if (remote_tag_data.mirror_uuid != librbd::Journal<>::LOCAL_MIRROR_UUID) {
593 dout(20) << ": skipping non-primary remote tag" << dendl;
597 dout(20) << ": using initial primary remote tag" << dendl;
601 if (local_tag_data.mirror_uuid == librbd::Journal<>::ORPHAN_MIRROR_UUID) {
602 // demotion last available local epoch
604 if (remote_tag_data.mirror_uuid == local_tag_data.mirror_uuid &&
605 remote_tag_data.predecessor.commit_valid &&
606 remote_tag_data.predecessor.tag_tid ==
607 local_tag_data.predecessor.tag_tid) {
608 // demotion matches remote epoch
610 if (remote_tag_data.predecessor.mirror_uuid == m_local_mirror_uuid &&
611 local_tag_data.predecessor.mirror_uuid ==
612 librbd::Journal<>::LOCAL_MIRROR_UUID) {
613 // local demoted and remote has matching event
614 dout(20) << ": found matching local demotion tag" << dendl;
615 remote_orphan_tag_tid = remote_tag.tid;
619 if (local_tag_data.predecessor.mirror_uuid == m_remote_mirror_uuid &&
620 remote_tag_data.predecessor.mirror_uuid ==
621 librbd::Journal<>::LOCAL_MIRROR_UUID) {
622 // remote demoted and local has matching event
623 dout(20) << ": found matching remote demotion tag" << dendl;
624 remote_orphan_tag_tid = remote_tag.tid;
629 if (remote_tag_data.mirror_uuid == librbd::Journal<>::LOCAL_MIRROR_UUID &&
630 remote_tag_data.predecessor.mirror_uuid == librbd::Journal<>::ORPHAN_MIRROR_UUID &&
631 remote_tag_data.predecessor.commit_valid && remote_orphan_tag_tid &&
632 remote_tag_data.predecessor.tag_tid == *remote_orphan_tag_tid) {
633 // remote promotion tag chained to remote/local demotion tag
634 dout(20) << ": found chained remote promotion tag" << dendl;
635 reconnect_orphan = true;
639 // promotion must follow demotion
640 remote_orphan_tag_tid = boost::none;
644 if (remote_tag_data_valid &&
645 local_tag_data.mirror_uuid == m_remote_mirror_uuid) {
646 dout(20) << ": local image is in clean replay state" << dendl;
647 } else if (reconnect_orphan) {
648 dout(20) << ": remote image was demoted/promoted" << dendl;
650 derr << ": split-brain detected -- skipping image replay" << dendl;
659 template <typename I>
660 void BootstrapRequest<I>::image_sync() {
661 if (m_client_meta->state == librbd::journal::MIRROR_PEER_STATE_REPLAYING) {
662 // clean replay state -- no image sync required
663 close_remote_image();
669 Mutex::Locker locker(m_lock);
671 m_ret_val = -ECANCELED;
673 assert(m_image_sync == nullptr);
675 Context *ctx = create_context_callback<
676 BootstrapRequest<I>, &BootstrapRequest<I>::handle_image_sync>(this);
677 m_image_sync = ImageSync<I>::create(
678 *m_local_image_ctx, m_remote_image_ctx, m_timer, m_timer_lock,
679 m_local_mirror_uuid, m_journaler, m_client_meta, m_work_queue,
680 m_instance_watcher, ctx, m_progress_ctx);
685 update_progress("IMAGE_SYNC");
688 m_image_sync->send();
693 dout(10) << ": request canceled" << dendl;
694 close_remote_image();
697 template <typename I>
698 void BootstrapRequest<I>::handle_image_sync(int r) {
699 dout(20) << ": r=" << r << dendl;
702 Mutex::Locker locker(m_lock);
704 m_image_sync = nullptr;
707 dout(10) << ": request canceled" << dendl;
708 m_ret_val = -ECANCELED;
712 derr << ": failed to sync remote image: " << cpp_strerror(r) << dendl;
717 close_remote_image();
720 template <typename I>
721 void BootstrapRequest<I>::close_local_image() {
724 update_progress("CLOSE_LOCAL_IMAGE");
726 Context *ctx = create_context_callback<
727 BootstrapRequest<I>, &BootstrapRequest<I>::handle_close_local_image>(
729 CloseImageRequest<I> *request = CloseImageRequest<I>::create(
730 m_local_image_ctx, ctx);
734 template <typename I>
735 void BootstrapRequest<I>::handle_close_local_image(int r) {
736 dout(20) << ": r=" << r << dendl;
739 derr << ": error encountered closing local image: " << cpp_strerror(r)
743 close_remote_image();
746 template <typename I>
747 void BootstrapRequest<I>::close_remote_image() {
750 update_progress("CLOSE_REMOTE_IMAGE");
752 Context *ctx = create_context_callback<
753 BootstrapRequest<I>, &BootstrapRequest<I>::handle_close_remote_image>(
755 CloseImageRequest<I> *request = CloseImageRequest<I>::create(
756 &m_remote_image_ctx, ctx);
760 template <typename I>
761 void BootstrapRequest<I>::handle_close_remote_image(int r) {
762 dout(20) << ": r=" << r << dendl;
765 derr << ": error encountered closing remote image: " << cpp_strerror(r)
772 template <typename I>
773 bool BootstrapRequest<I>::decode_client_meta() {
776 librbd::journal::ClientData client_data;
777 bufferlist::iterator it = m_client.data.begin();
779 ::decode(client_data, it);
780 } catch (const buffer::error &err) {
781 derr << ": failed to decode client meta data: " << err.what() << dendl;
785 librbd::journal::MirrorPeerClientMeta *client_meta =
786 boost::get<librbd::journal::MirrorPeerClientMeta>(&client_data.client_meta);
787 if (client_meta == nullptr) {
788 derr << ": unknown peer registration" << dendl;
790 } else if (!client_meta->image_id.empty()) {
791 // have an image id -- use that to open the image
792 m_local_image_id = client_meta->image_id;
795 *m_client_meta = *client_meta;
797 dout(20) << ": client found: image_id=" << m_local_image_id
798 << ", client_meta=" << *m_client_meta << dendl;
802 template <typename I>
803 void BootstrapRequest<I>::update_progress(const std::string &description) {
804 dout(20) << ": " << description << dendl;
806 if (m_progress_ctx) {
807 m_progress_ctx->update_progress(description);
811 } // namespace image_replayer
812 } // namespace mirror
815 template class rbd::mirror::image_replayer::BootstrapRequest<librbd::ImageCtx>;