1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
2 // vim: ts=8 sw=2 smarttab
4 * Ceph distributed storage system
6 * Copyright (C) 2016 Mirantis Inc
8 * Author: Mykola Golub <mgolub@mirantis.com>
10 * This library is free software; you can redistribute it and/or
11 * modify it under the terms of the GNU Lesser General Public
12 * License as published by the Free Software Foundation; either
13 * version 2.1 of the License, or (at your option) any later version.
17 #include "include/rados/librados.hpp"
18 #include "include/rbd/librbd.hpp"
19 #include "include/stringify.h"
20 #include "test/rbd_mirror/test_fixture.h"
21 #include "cls/journal/cls_journal_types.h"
22 #include "cls/journal/cls_journal_client.h"
23 #include "cls/rbd/cls_rbd_types.h"
24 #include "cls/rbd/cls_rbd_client.h"
25 #include "journal/Journaler.h"
26 #include "librbd/ExclusiveLock.h"
27 #include "librbd/ImageCtx.h"
28 #include "librbd/ImageState.h"
29 #include "librbd/Journal.h"
30 #include "librbd/Operations.h"
31 #include "librbd/Utils.h"
32 #include "librbd/internal.h"
33 #include "librbd/api/Mirror.h"
34 #include "librbd/io/AioCompletion.h"
35 #include "librbd/io/ImageRequestWQ.h"
36 #include "librbd/io/ReadResult.h"
37 #include "tools/rbd_mirror/types.h"
38 #include "tools/rbd_mirror/ImageDeleter.h"
39 #include "tools/rbd_mirror/ImageReplayer.h"
40 #include "tools/rbd_mirror/InstanceWatcher.h"
41 #include "tools/rbd_mirror/ServiceDaemon.h"
42 #include "tools/rbd_mirror/Threads.h"
44 #include "test/librados/test.h"
45 #include "gtest/gtest.h"
47 using rbd::mirror::RadosRef;
49 void register_test_rbd_mirror() {
52 #define TEST_IO_SIZE 512
53 #define TEST_IO_COUNT 11
55 class TestImageReplayer : public ::rbd::mirror::TestFixture {
57 struct C_WatchCtx : public librados::WatchCtx2 {
58 TestImageReplayer *test;
64 C_WatchCtx(TestImageReplayer *test, const std::string &oid)
65 : test(test), oid(oid), lock("C_WatchCtx::lock"), notified(false) {
68 void handle_notify(uint64_t notify_id, uint64_t cookie,
69 uint64_t notifier_id, bufferlist& bl_) override {
71 test->m_remote_ioctx.notify_ack(oid, notify_id, cookie, bl);
73 Mutex::Locker locker(lock);
78 void handle_error(uint64_t cookie, int err) override {
84 : m_local_cluster(new librados::Rados()), m_watch_handle(0)
86 EXPECT_EQ("", connect_cluster_pp(*m_local_cluster.get()));
87 EXPECT_EQ(0, m_local_cluster->conf_set("rbd_cache", "false"));
88 EXPECT_EQ(0, m_local_cluster->conf_set("rbd_mirror_journal_poll_age", "1"));
90 m_local_pool_name = get_temp_pool_name();
91 EXPECT_EQ(0, m_local_cluster->pool_create(m_local_pool_name.c_str()));
92 EXPECT_EQ(0, m_local_cluster->ioctx_create(m_local_pool_name.c_str(),
94 m_local_ioctx.application_enable("rbd", true);
96 EXPECT_EQ("", connect_cluster_pp(m_remote_cluster));
97 EXPECT_EQ(0, m_remote_cluster.conf_set("rbd_cache", "false"));
99 m_remote_pool_name = get_temp_pool_name();
100 EXPECT_EQ(0, m_remote_cluster.pool_create(m_remote_pool_name.c_str()));
101 m_remote_pool_id = m_remote_cluster.pool_lookup(m_remote_pool_name.c_str());
102 EXPECT_GE(m_remote_pool_id, 0);
104 EXPECT_EQ(0, m_remote_cluster.ioctx_create(m_remote_pool_name.c_str(),
106 m_remote_ioctx.application_enable("rbd", true);
108 EXPECT_EQ(0, librbd::api::Mirror<>::mode_set(m_remote_ioctx,
109 RBD_MIRROR_MODE_POOL));
111 m_image_name = get_temp_image_name();
112 uint64_t features = librbd::util::get_rbd_default_features(g_ceph_context);
113 features |= RBD_FEATURE_EXCLUSIVE_LOCK | RBD_FEATURE_JOURNALING;
115 EXPECT_EQ(0, librbd::create(m_remote_ioctx, m_image_name.c_str(), 1 << 22,
116 false, features, &order, 0, 0));
117 m_remote_image_id = get_image_id(m_remote_ioctx, m_image_name);
118 m_global_image_id = get_global_image_id(m_remote_ioctx, m_remote_image_id);
120 m_threads.reset(new rbd::mirror::Threads<>(reinterpret_cast<CephContext*>(
121 m_local_ioctx.cct())));
123 m_service_daemon.reset(new rbd::mirror::ServiceDaemon<>(g_ceph_context,
126 m_image_deleter.reset(new rbd::mirror::ImageDeleter<>(
127 m_threads->work_queue, m_threads->timer, &m_threads->timer_lock,
128 m_service_daemon.get()));
129 m_instance_watcher = rbd::mirror::InstanceWatcher<>::create(
130 m_local_ioctx, m_threads->work_queue, nullptr);
131 m_instance_watcher->handle_acquire_leader();
134 ~TestImageReplayer() override
138 m_instance_watcher->handle_release_leader();
141 delete m_instance_watcher;
143 EXPECT_EQ(0, m_remote_cluster.pool_delete(m_remote_pool_name.c_str()));
144 EXPECT_EQ(0, m_local_cluster->pool_delete(m_local_pool_name.c_str()));
147 template <typename ImageReplayerT = rbd::mirror::ImageReplayer<> >
148 void create_replayer() {
149 m_replayer = new ImageReplayerT(
150 m_threads.get(), m_image_deleter.get(), m_instance_watcher,
151 rbd::mirror::RadosRef(new librados::Rados(m_local_ioctx)),
152 m_local_mirror_uuid, m_local_ioctx.get_id(), m_global_image_id);
153 m_replayer->add_peer("peer uuid", m_remote_ioctx);
159 m_replayer->start(&cond);
160 ASSERT_EQ(0, cond.wait());
162 ASSERT_EQ(0U, m_watch_handle);
163 std::string oid = ::journal::Journaler::header_oid(m_remote_image_id);
164 m_watch_ctx = new C_WatchCtx(this, oid);
165 ASSERT_EQ(0, m_remote_ioctx.watch2(oid, &m_watch_handle, m_watch_ctx));
169 if (m_watch_handle != 0) {
170 m_remote_ioctx.unwatch2(m_watch_handle);
172 m_watch_ctx = nullptr;
182 m_replayer->stop(&cond);
183 ASSERT_EQ(0, cond.wait());
191 wait_for_replay_complete();
195 std::string get_temp_image_name()
197 return "image" + stringify(++_image_number);
200 std::string get_image_id(librados::IoCtx &ioctx, const string &image_name)
202 std::string obj = librbd::util::id_obj_name(image_name);
204 EXPECT_EQ(0, librbd::cls_client::get_id(&ioctx, obj, &id));
208 std::string get_global_image_id(librados::IoCtx& io_ctx,
209 const std::string& image_id) {
210 cls::rbd::MirrorImage mirror_image;
211 EXPECT_EQ(0, librbd::cls_client::mirror_image_get(&io_ctx, image_id,
213 return mirror_image.global_image_id;
216 void open_image(librados::IoCtx &ioctx, const std::string &image_name,
217 bool readonly, librbd::ImageCtx **ictxp)
219 librbd::ImageCtx *ictx = new librbd::ImageCtx(image_name.c_str(),
220 "", "", ioctx, readonly);
221 EXPECT_EQ(0, ictx->state->open(false));
225 void open_local_image(librbd::ImageCtx **ictxp)
227 open_image(m_local_ioctx, m_image_name, true, ictxp);
230 void open_remote_image(librbd::ImageCtx **ictxp)
232 open_image(m_remote_ioctx, m_image_name, false, ictxp);
235 void close_image(librbd::ImageCtx *ictx)
237 ictx->state->close();
240 void get_commit_positions(cls::journal::ObjectPosition *master_position,
241 cls::journal::ObjectPosition *mirror_position)
243 std::string master_client_id = "";
244 std::string mirror_client_id = m_local_mirror_uuid;
247 uint64_t minimum_set;
249 std::set<cls::journal::Client> registered_clients;
250 std::string oid = ::journal::Journaler::header_oid(m_remote_image_id);
251 cls::journal::client::get_mutable_metadata(m_remote_ioctx, oid,
252 &minimum_set, &active_set,
253 ®istered_clients, &cond);
254 ASSERT_EQ(0, cond.wait());
256 *master_position = cls::journal::ObjectPosition();
257 *mirror_position = cls::journal::ObjectPosition();
259 std::set<cls::journal::Client>::const_iterator c;
260 for (c = registered_clients.begin(); c != registered_clients.end(); ++c) {
261 std::cout << __func__ << ": client: " << *c << std::endl;
262 if (c->state != cls::journal::CLIENT_STATE_CONNECTED) {
265 cls::journal::ObjectPositions object_positions =
266 c->commit_position.object_positions;
267 cls::journal::ObjectPositions::const_iterator p =
268 object_positions.begin();
269 if (p != object_positions.end()) {
270 if (c->id == master_client_id) {
271 ASSERT_EQ(cls::journal::ObjectPosition(), *master_position);
272 *master_position = *p;
273 } else if (c->id == mirror_client_id) {
274 ASSERT_EQ(cls::journal::ObjectPosition(), *mirror_position);
275 *mirror_position = *p;
281 bool wait_for_watcher_notify(int seconds)
283 if (m_watch_handle == 0) {
287 Mutex::Locker locker(m_watch_ctx->lock);
288 while (!m_watch_ctx->notified) {
289 if (m_watch_ctx->cond.WaitInterval(m_watch_ctx->lock,
290 utime_t(seconds, 0)) != 0) {
294 m_watch_ctx->notified = false;
298 void wait_for_replay_complete()
300 cls::journal::ObjectPosition master_position;
301 cls::journal::ObjectPosition mirror_position;
303 for (int i = 0; i < 100; i++) {
304 printf("m_replayer->flush()\n");
306 m_replayer->flush(&cond);
307 ASSERT_EQ(0, cond.wait());
308 get_commit_positions(&master_position, &mirror_position);
309 if (master_position == mirror_position) {
312 wait_for_watcher_notify(1);
315 ASSERT_EQ(master_position, mirror_position);
318 void wait_for_stopped() {
319 for (int i = 0; i < 100; i++) {
320 if (m_replayer->is_stopped()) {
323 wait_for_watcher_notify(1);
325 ASSERT_TRUE(m_replayer->is_stopped());
328 void write_test_data(librbd::ImageCtx *ictx, const char *test_data, off_t off,
333 bl.append(std::string(test_data, len));
334 written = ictx->io_work_queue->write(off, len, std::move(bl), 0);
335 printf("wrote: %d\n", (int)written);
336 ASSERT_EQ(len, written);
339 void read_test_data(librbd::ImageCtx *ictx, const char *expected, off_t off,
343 char *result = (char *)malloc(len + 1);
345 ASSERT_NE(static_cast<char *>(NULL), result);
346 read = ictx->io_work_queue->read(
347 off, len, librbd::io::ReadResult{result, len}, 0);
348 printf("read: %d\n", (int)read);
349 ASSERT_EQ(len, static_cast<size_t>(read));
351 if (memcmp(result, expected, len)) {
352 printf("read: %s\nexpected: %s\n", result, expected);
353 ASSERT_EQ(0, memcmp(result, expected, len));
358 void generate_test_data() {
359 for (int i = 0; i < TEST_IO_SIZE; ++i) {
360 m_test_data[i] = (char) (rand() % (126 - 33) + 33);
362 m_test_data[TEST_IO_SIZE] = '\0';
365 void flush(librbd::ImageCtx *ictx)
367 C_SaferCond aio_flush_ctx;
368 auto c = librbd::io::AioCompletion::create(&aio_flush_ctx);
370 ictx->io_work_queue->aio_flush(c);
371 ASSERT_EQ(0, c->wait_for_complete());
374 C_SaferCond journal_flush_ctx;
375 ictx->journal->flush_commit_position(&journal_flush_ctx);
376 ASSERT_EQ(0, journal_flush_ctx.wait());
381 static int _image_number;
383 std::shared_ptr<librados::Rados> m_local_cluster;
384 std::unique_ptr<rbd::mirror::Threads<>> m_threads;
385 std::unique_ptr<rbd::mirror::ServiceDaemon<>> m_service_daemon;
386 std::unique_ptr<rbd::mirror::ImageDeleter<>> m_image_deleter;
387 librados::Rados m_remote_cluster;
388 rbd::mirror::InstanceWatcher<> *m_instance_watcher;
389 std::string m_local_mirror_uuid = "local mirror uuid";
390 std::string m_remote_mirror_uuid = "remote mirror uuid";
391 std::string m_local_pool_name, m_remote_pool_name;
392 librados::IoCtx m_local_ioctx, m_remote_ioctx;
393 std::string m_image_name;
394 int64_t m_remote_pool_id;
395 std::string m_remote_image_id;
396 std::string m_global_image_id;
397 rbd::mirror::ImageReplayer<> *m_replayer;
398 C_WatchCtx *m_watch_ctx;
399 uint64_t m_watch_handle;
400 char m_test_data[TEST_IO_SIZE + 1];
403 int TestImageReplayer::_image_number;
405 TEST_F(TestImageReplayer, Bootstrap)
410 TEST_F(TestImageReplayer, BootstrapErrorLocalImageExists)
413 EXPECT_EQ(0, librbd::create(m_local_ioctx, m_image_name.c_str(), 1 << 22,
414 false, 0, &order, 0, 0));
418 m_replayer->start(&cond);
419 ASSERT_EQ(-EEXIST, cond.wait());
422 TEST_F(TestImageReplayer, BootstrapErrorNoJournal)
424 ASSERT_EQ(0, librbd::Journal<>::remove(m_remote_ioctx, m_remote_image_id));
428 m_replayer->start(&cond);
429 ASSERT_EQ(-ENOENT, cond.wait());
432 TEST_F(TestImageReplayer, BootstrapErrorMirrorDisabled)
434 // disable remote image mirroring
435 ASSERT_EQ(0, librbd::api::Mirror<>::mode_set(m_remote_ioctx,
436 RBD_MIRROR_MODE_IMAGE));
437 librbd::ImageCtx *ictx;
438 open_remote_image(&ictx);
439 ASSERT_EQ(0, librbd::api::Mirror<>::image_disable(ictx, true));
444 m_replayer->start(&cond);
445 ASSERT_EQ(-ENOENT, cond.wait());
448 TEST_F(TestImageReplayer, BootstrapMirrorDisabling)
450 // set remote image mirroring state to DISABLING
451 ASSERT_EQ(0, librbd::api::Mirror<>::mode_set(m_remote_ioctx,
452 RBD_MIRROR_MODE_IMAGE));
453 librbd::ImageCtx *ictx;
454 open_remote_image(&ictx);
455 ASSERT_EQ(0, librbd::api::Mirror<>::image_enable(ictx, false));
456 cls::rbd::MirrorImage mirror_image;
457 ASSERT_EQ(0, librbd::cls_client::mirror_image_get(&m_remote_ioctx, ictx->id,
459 mirror_image.state = cls::rbd::MirrorImageState::MIRROR_IMAGE_STATE_DISABLING;
460 ASSERT_EQ(0, librbd::cls_client::mirror_image_set(&m_remote_ioctx, ictx->id,
466 m_replayer->start(&cond);
467 ASSERT_EQ(-EREMOTEIO, cond.wait());
468 ASSERT_TRUE(m_replayer->is_stopped());
471 TEST_F(TestImageReplayer, BootstrapDemoted)
473 // demote remote image
474 librbd::ImageCtx *ictx;
475 open_remote_image(&ictx);
476 ASSERT_EQ(0, librbd::api::Mirror<>::image_demote(ictx));
481 m_replayer->start(&cond);
482 ASSERT_EQ(-EREMOTEIO, cond.wait());
483 ASSERT_TRUE(m_replayer->is_stopped());
486 TEST_F(TestImageReplayer, StartInterrupted)
489 C_SaferCond start_cond, stop_cond;
490 m_replayer->start(&start_cond);
491 m_replayer->stop(&stop_cond);
492 int r = start_cond.wait();
493 printf("start returned %d\n", r);
494 // TODO: improve the test to avoid this race
495 ASSERT_TRUE(r == -ECANCELED || r == 0);
496 ASSERT_EQ(0, stop_cond.wait());
499 TEST_F(TestImageReplayer, JournalReset)
504 ASSERT_EQ(0, librbd::Journal<>::reset(m_remote_ioctx, m_remote_image_id));
510 TEST_F(TestImageReplayer, ErrorNoJournal)
514 // disable remote journal journaling
515 // (reset before disabling, so it does not fail with EBUSY)
516 ASSERT_EQ(0, librbd::Journal<>::reset(m_remote_ioctx, m_remote_image_id));
517 librbd::ImageCtx *ictx;
518 open_remote_image(&ictx);
520 ASSERT_EQ(0, librbd::get_features(ictx, &features));
521 ASSERT_EQ(0, ictx->operations->update_features(RBD_FEATURE_JOURNALING,
526 m_replayer->start(&cond);
527 ASSERT_EQ(0, cond.wait());
530 TEST_F(TestImageReplayer, StartStop)
535 wait_for_replay_complete();
539 TEST_F(TestImageReplayer, WriteAndStartReplay)
543 // Write to remote image and start replay
545 librbd::ImageCtx *ictx;
547 generate_test_data();
548 open_remote_image(&ictx);
549 for (int i = 0; i < TEST_IO_COUNT; ++i) {
550 write_test_data(ictx, m_test_data, TEST_IO_SIZE * i, TEST_IO_SIZE);
556 wait_for_replay_complete();
559 open_local_image(&ictx);
560 for (int i = 0; i < TEST_IO_COUNT; ++i) {
561 read_test_data(ictx, m_test_data, TEST_IO_SIZE * i, TEST_IO_SIZE);
566 TEST_F(TestImageReplayer, StartReplayAndWrite)
570 // Start replay and write to remote image
572 librbd::ImageCtx *ictx;
576 generate_test_data();
577 open_remote_image(&ictx);
578 for (int i = 0; i < TEST_IO_COUNT; ++i) {
579 write_test_data(ictx, m_test_data, TEST_IO_SIZE * i, TEST_IO_SIZE);
583 wait_for_replay_complete();
585 for (int i = TEST_IO_COUNT; i < 2 * TEST_IO_COUNT; ++i) {
586 write_test_data(ictx, m_test_data, TEST_IO_SIZE * i, TEST_IO_SIZE);
591 wait_for_replay_complete();
593 open_local_image(&ictx);
594 for (int i = 0; i < 2 * TEST_IO_COUNT; ++i) {
595 read_test_data(ictx, m_test_data, TEST_IO_SIZE * i, TEST_IO_SIZE);
602 TEST_F(TestImageReplayer, NextTag)
606 // write, reopen, and write again to test switch to the next tag
608 librbd::ImageCtx *ictx;
612 generate_test_data();
616 for (int j = 0; j < N; j++) {
617 open_remote_image(&ictx);
618 for (int i = j * TEST_IO_COUNT; i < (j + 1) * TEST_IO_COUNT; ++i) {
619 write_test_data(ictx, m_test_data, TEST_IO_SIZE * i, TEST_IO_SIZE);
624 wait_for_replay_complete();
626 open_local_image(&ictx);
627 for (int i = 0; i < N * TEST_IO_COUNT; ++i) {
628 read_test_data(ictx, m_test_data, TEST_IO_SIZE * i, TEST_IO_SIZE);
635 TEST_F(TestImageReplayer, Resync)
639 librbd::ImageCtx *ictx;
643 generate_test_data();
645 open_remote_image(&ictx);
646 for (int i = 0; i < TEST_IO_COUNT; ++i) {
647 write_test_data(ictx, m_test_data, TEST_IO_SIZE * i, TEST_IO_SIZE);
651 wait_for_replay_complete();
653 for (int i = TEST_IO_COUNT; i < 2 * TEST_IO_COUNT; ++i) {
654 write_test_data(ictx, m_test_data, TEST_IO_SIZE * i, TEST_IO_SIZE);
660 m_replayer->resync_image(&ctx);
661 ASSERT_EQ(0, ctx.wait());
663 C_SaferCond delete_ctx;
664 m_image_deleter->wait_for_scheduled_deletion(
665 m_local_ioctx.get_id(), m_replayer->get_global_image_id(), &delete_ctx);
666 EXPECT_EQ(0, delete_ctx.wait());
669 m_replayer->start(&cond);
670 ASSERT_EQ(0, cond.wait());
672 ASSERT_TRUE(m_replayer->is_replaying());
674 wait_for_replay_complete();
676 open_local_image(&ictx);
677 for (int i = 0; i < 2 * TEST_IO_COUNT; ++i) {
678 read_test_data(ictx, m_test_data, TEST_IO_SIZE * i, TEST_IO_SIZE);
685 TEST_F(TestImageReplayer, Resync_While_Stop)
692 generate_test_data();
694 librbd::ImageCtx *ictx;
695 open_remote_image(&ictx);
696 for (int i = 0; i < TEST_IO_COUNT; ++i) {
697 write_test_data(ictx, m_test_data, TEST_IO_SIZE * i, TEST_IO_SIZE);
701 wait_for_replay_complete();
703 for (int i = TEST_IO_COUNT; i < 2 * TEST_IO_COUNT; ++i) {
704 write_test_data(ictx, m_test_data, TEST_IO_SIZE * i, TEST_IO_SIZE);
709 wait_for_replay_complete();
712 m_replayer->stop(&cond);
713 ASSERT_EQ(0, cond.wait());
715 open_local_image(&ictx);
716 librbd::Journal<>::request_resync(ictx);
720 m_replayer->start(&cond2);
721 ASSERT_EQ(0, cond2.wait());
723 ASSERT_TRUE(m_replayer->is_stopped());
725 C_SaferCond delete_ctx;
726 m_image_deleter->wait_for_scheduled_deletion(
727 m_local_ioctx.get_id(), m_replayer->get_global_image_id(), &delete_ctx);
728 EXPECT_EQ(0, delete_ctx.wait());
731 m_replayer->start(&cond3);
732 ASSERT_EQ(0, cond3.wait());
734 ASSERT_TRUE(m_replayer->is_replaying());
736 wait_for_replay_complete();
738 open_local_image(&ictx);
739 for (int i = 0; i < 2 * TEST_IO_COUNT; ++i) {
740 read_test_data(ictx, m_test_data, TEST_IO_SIZE * i, TEST_IO_SIZE);
747 TEST_F(TestImageReplayer, Resync_StartInterrupted)
752 librbd::ImageCtx *ictx;
753 open_local_image(&ictx);
754 librbd::Journal<>::request_resync(ictx);
758 m_replayer->start(&cond);
759 ASSERT_EQ(0, cond.wait());
761 ASSERT_TRUE(m_replayer->is_stopped());
763 C_SaferCond delete_ctx;
764 m_image_deleter->wait_for_scheduled_deletion(
765 m_local_ioctx.get_id(), m_replayer->get_global_image_id(), &delete_ctx);
766 EXPECT_EQ(0, delete_ctx.wait());
769 m_replayer->start(&cond2);
770 ASSERT_EQ(0, cond2.wait());
772 ASSERT_EQ(0U, m_watch_handle);
773 std::string oid = ::journal::Journaler::header_oid(m_remote_image_id);
774 m_watch_ctx = new C_WatchCtx(this, oid);
775 ASSERT_EQ(0, m_remote_ioctx.watch2(oid, &m_watch_handle, m_watch_ctx));
777 ASSERT_TRUE(m_replayer->is_replaying());
779 generate_test_data();
780 open_remote_image(&ictx);
781 for (int i = 0; i < TEST_IO_COUNT; ++i) {
782 write_test_data(ictx, m_test_data, TEST_IO_SIZE * i, TEST_IO_SIZE);
786 wait_for_replay_complete();
788 for (int i = TEST_IO_COUNT; i < 2 * TEST_IO_COUNT; ++i) {
789 write_test_data(ictx, m_test_data, TEST_IO_SIZE * i, TEST_IO_SIZE);
794 wait_for_replay_complete();
796 open_local_image(&ictx);
797 for (int i = 0; i < 2 * TEST_IO_COUNT; ++i) {
798 read_test_data(ictx, m_test_data, TEST_IO_SIZE * i, TEST_IO_SIZE);
805 TEST_F(TestImageReplayer, MultipleReplayFailures_SingleEpoch) {
808 // inject a snapshot that cannot be unprotected
809 librbd::ImageCtx *ictx;
810 open_image(m_local_ioctx, m_image_name, false, &ictx);
811 ictx->features &= ~RBD_FEATURE_JOURNALING;
812 ASSERT_EQ(0, ictx->operations->snap_create(cls::rbd::UserSnapshotNamespace(),
814 ASSERT_EQ(0, ictx->operations->snap_protect(cls::rbd::UserSnapshotNamespace(),
816 ASSERT_EQ(0, librbd::cls_client::add_child(&ictx->md_ctx, RBD_CHILDREN,
817 {ictx->md_ctx.get_id(),
819 ictx->snap_ids[{cls::rbd::UserSnapshotNamespace(), "foo"}]},
823 // race failed op shut down with new ops
824 open_remote_image(&ictx);
825 for (uint64_t i = 0; i < 10; ++i) {
826 RWLock::RLocker owner_locker(ictx->owner_lock);
827 C_SaferCond request_lock;
828 ictx->exclusive_lock->acquire_lock(&request_lock);
829 ASSERT_EQ(0, request_lock.wait());
831 C_SaferCond append_ctx;
832 ictx->journal->append_op_event(
834 librbd::journal::EventEntry{
835 librbd::journal::SnapUnprotectEvent{i,
836 cls::rbd::UserSnapshotNamespace(),
839 ASSERT_EQ(0, append_ctx.wait());
841 C_SaferCond commit_ctx;
842 ictx->journal->commit_op_event(i, 0, &commit_ctx);
843 ASSERT_EQ(0, commit_ctx.wait());
845 C_SaferCond release_ctx;
846 ictx->exclusive_lock->release_lock(&release_ctx);
847 ASSERT_EQ(0, release_ctx.wait());
850 for (uint64_t i = 0; i < 5; ++i) {
858 TEST_F(TestImageReplayer, MultipleReplayFailures_MultiEpoch) {
861 // inject a snapshot that cannot be unprotected
862 librbd::ImageCtx *ictx;
863 open_image(m_local_ioctx, m_image_name, false, &ictx);
864 ictx->features &= ~RBD_FEATURE_JOURNALING;
865 ASSERT_EQ(0, ictx->operations->snap_create(cls::rbd::UserSnapshotNamespace(),
867 ASSERT_EQ(0, ictx->operations->snap_protect(cls::rbd::UserSnapshotNamespace(),
869 ASSERT_EQ(0, librbd::cls_client::add_child(&ictx->md_ctx, RBD_CHILDREN,
870 {ictx->md_ctx.get_id(),
872 ictx->snap_ids[{cls::rbd::UserSnapshotNamespace(),
877 // race failed op shut down with new tag flush
878 open_remote_image(&ictx);
880 RWLock::RLocker owner_locker(ictx->owner_lock);
881 C_SaferCond request_lock;
882 ictx->exclusive_lock->acquire_lock(&request_lock);
883 ASSERT_EQ(0, request_lock.wait());
885 C_SaferCond append_ctx;
886 ictx->journal->append_op_event(
888 librbd::journal::EventEntry{
889 librbd::journal::SnapUnprotectEvent{1U,
890 cls::rbd::UserSnapshotNamespace(),
893 ASSERT_EQ(0, append_ctx.wait());
895 C_SaferCond commit_ctx;
896 ictx->journal->commit_op_event(1U, 0, &commit_ctx);
897 ASSERT_EQ(0, commit_ctx.wait());
899 C_SaferCond release_ctx;
900 ictx->exclusive_lock->release_lock(&release_ctx);
901 ASSERT_EQ(0, release_ctx.wait());
904 generate_test_data();
905 write_test_data(ictx, m_test_data, 0, TEST_IO_SIZE);
907 for (uint64_t i = 0; i < 5; ++i) {
915 TEST_F(TestImageReplayer, Disconnect)
919 // Make sure rbd_mirroring_resync_after_disconnect is not set
920 EXPECT_EQ(0, m_local_cluster->conf_set("rbd_mirroring_resync_after_disconnect", "false"));
922 // Test start fails if disconnected
924 librbd::ImageCtx *ictx;
926 generate_test_data();
927 open_remote_image(&ictx);
928 for (int i = 0; i < TEST_IO_COUNT; ++i) {
929 write_test_data(ictx, m_test_data, TEST_IO_SIZE * i, TEST_IO_SIZE);
934 std::string oid = ::journal::Journaler::header_oid(m_remote_image_id);
935 ASSERT_EQ(0, cls::journal::client::client_update_state(m_remote_ioctx, oid,
936 m_local_mirror_uuid, cls::journal::CLIENT_STATE_DISCONNECTED));
939 m_replayer->start(&cond1);
940 ASSERT_EQ(-ENOTCONN, cond1.wait());
942 // Test start succeeds after resync
944 open_local_image(&ictx);
945 librbd::Journal<>::request_resync(ictx);
948 m_replayer->start(&cond2);
949 ASSERT_EQ(0, cond2.wait());
950 C_SaferCond delete_cond;
951 m_image_deleter->wait_for_scheduled_deletion(
952 m_local_ioctx.get_id(), m_replayer->get_global_image_id(), &delete_cond);
953 EXPECT_EQ(0, delete_cond.wait());
956 wait_for_replay_complete();
958 // Test replay stopped after disconnect
960 open_remote_image(&ictx);
961 for (int i = TEST_IO_COUNT; i < 2 * TEST_IO_COUNT; ++i) {
962 write_test_data(ictx, m_test_data, TEST_IO_SIZE * i, TEST_IO_SIZE);
967 ASSERT_EQ(0, cls::journal::client::client_update_state(m_remote_ioctx, oid,
968 m_local_mirror_uuid, cls::journal::CLIENT_STATE_DISCONNECTED));
970 ASSERT_EQ(0, m_remote_ioctx.notify2(oid, bl, 5000, NULL));
974 // Test start fails after disconnect
977 m_replayer->start(&cond3);
978 ASSERT_EQ(-ENOTCONN, cond3.wait());
980 m_replayer->start(&cond4);
981 ASSERT_EQ(-ENOTCONN, cond4.wait());
983 // Test automatic resync if rbd_mirroring_resync_after_disconnect is set
985 EXPECT_EQ(0, m_local_cluster->conf_set("rbd_mirroring_resync_after_disconnect", "true"));
987 // Resync is flagged on first start attempt
989 m_replayer->start(&cond5);
990 ASSERT_EQ(-ENOTCONN, cond5.wait());
991 C_SaferCond delete_cond1;
992 m_image_deleter->wait_for_scheduled_deletion(
993 m_local_ioctx.get_id(), m_replayer->get_global_image_id(), &delete_cond1);
994 EXPECT_EQ(0, delete_cond1.wait());
997 m_replayer->start(&cond6);
998 ASSERT_EQ(0, cond6.wait());
999 wait_for_replay_complete();
1004 TEST_F(TestImageReplayer, UpdateFeatures)
1006 const uint64_t FEATURES_TO_UPDATE =
1007 RBD_FEATURE_OBJECT_MAP | RBD_FEATURE_FAST_DIFF;
1010 librbd::ImageCtx *ictx;
1012 // Make sure the features we will update are disabled initially
1014 open_remote_image(&ictx);
1015 ASSERT_EQ(0, librbd::get_features(ictx, &features));
1016 features &= FEATURES_TO_UPDATE;
1018 ASSERT_EQ(0, ictx->operations->update_features(FEATURES_TO_UPDATE,
1021 ASSERT_EQ(0, librbd::get_features(ictx, &features));
1022 ASSERT_EQ(0U, features & FEATURES_TO_UPDATE);
1027 open_remote_image(&ictx);
1028 ASSERT_EQ(0, librbd::get_features(ictx, &features));
1029 ASSERT_EQ(0U, features & FEATURES_TO_UPDATE);
1032 open_local_image(&ictx);
1033 ASSERT_EQ(0, librbd::get_features(ictx, &features));
1034 ASSERT_EQ(0U, features & FEATURES_TO_UPDATE);
1037 // Start replay and update features
1041 open_remote_image(&ictx);
1042 ASSERT_EQ(0, ictx->operations->update_features(FEATURES_TO_UPDATE,
1044 ASSERT_EQ(0, librbd::get_features(ictx, &features));
1045 ASSERT_EQ(FEATURES_TO_UPDATE, features & FEATURES_TO_UPDATE);
1048 wait_for_replay_complete();
1050 open_local_image(&ictx);
1051 ASSERT_EQ(0, librbd::get_features(ictx, &features));
1052 ASSERT_EQ(FEATURES_TO_UPDATE, features & FEATURES_TO_UPDATE);
1055 open_remote_image(&ictx);
1056 ASSERT_EQ(0, ictx->operations->update_features(FEATURES_TO_UPDATE,
1058 ASSERT_EQ(0, librbd::get_features(ictx, &features));
1059 ASSERT_EQ(0U, features & FEATURES_TO_UPDATE);
1062 wait_for_replay_complete();
1064 open_local_image(&ictx);
1065 ASSERT_EQ(0, librbd::get_features(ictx, &features));
1066 ASSERT_EQ(0U, features & FEATURES_TO_UPDATE);
1069 // Test update_features error does not stop replication
1071 open_remote_image(&ictx);
1072 ASSERT_EQ(0, librbd::get_features(ictx, &features));
1073 ASSERT_NE(0U, features & RBD_FEATURE_EXCLUSIVE_LOCK);
1074 ASSERT_EQ(-EINVAL, ictx->operations->update_features(RBD_FEATURE_EXCLUSIVE_LOCK,
1076 generate_test_data();
1077 for (int i = 0; i < TEST_IO_COUNT; ++i) {
1078 write_test_data(ictx, m_test_data, TEST_IO_SIZE * i, TEST_IO_SIZE);
1083 wait_for_replay_complete();
1085 open_local_image(&ictx);
1086 for (int i = 0; i < TEST_IO_COUNT; ++i) {
1087 read_test_data(ictx, m_test_data, TEST_IO_SIZE * i, TEST_IO_SIZE);
1094 TEST_F(TestImageReplayer, MetadataSetRemove)
1096 const std::string KEY = "test_key";
1097 const std::string VALUE = "test_value";
1099 librbd::ImageCtx *ictx;
1106 // Test metadata_set replication
1108 open_remote_image(&ictx);
1109 ASSERT_EQ(0, ictx->operations->metadata_set(KEY, VALUE));
1111 ASSERT_EQ(0, librbd::metadata_get(ictx, KEY, &value));
1112 ASSERT_EQ(VALUE, value);
1115 wait_for_replay_complete();
1117 open_local_image(&ictx);
1119 ASSERT_EQ(0, librbd::metadata_get(ictx, KEY, &value));
1120 ASSERT_EQ(VALUE, value);
1123 // Test metadata_remove replication
1125 open_remote_image(&ictx);
1126 ASSERT_EQ(0, ictx->operations->metadata_remove(KEY));
1127 ASSERT_EQ(-ENOENT, librbd::metadata_get(ictx, KEY, &value));
1130 wait_for_replay_complete();
1132 open_local_image(&ictx);
1133 ASSERT_EQ(-ENOENT, librbd::metadata_get(ictx, KEY, &value));
1139 TEST_F(TestImageReplayer, MirroringDelay)
1141 const double DELAY = 10; // set less than wait_for_replay_complete timeout
1143 librbd::ImageCtx *ictx;
1149 ASSERT_EQ(0, m_local_cluster->conf_set("rbd_mirroring_replay_delay",
1150 stringify(DELAY).c_str()));
1151 open_local_image(&ictx);
1152 ASSERT_EQ(DELAY, ictx->mirroring_replay_delay);
1159 generate_test_data();
1160 open_remote_image(&ictx);
1161 start_time = ceph_clock_now();
1162 for (int i = 0; i < TEST_IO_COUNT; ++i) {
1163 write_test_data(ictx, m_test_data, TEST_IO_SIZE * i, TEST_IO_SIZE);
1168 wait_for_replay_complete();
1169 delay = ceph_clock_now() - start_time;
1170 ASSERT_GE(delay, DELAY);
1172 // Test stop when delaying replay
1174 open_remote_image(&ictx);
1175 start_time = ceph_clock_now();
1176 for (int i = 0; i < TEST_IO_COUNT; ++i) {
1177 write_test_data(ictx, m_test_data, TEST_IO_SIZE * i, TEST_IO_SIZE);
1185 wait_for_replay_complete();
1186 delay = ceph_clock_now() - start_time;
1187 ASSERT_GE(delay, DELAY);