src/ceph/src/test/librbd/test_mirroring.cc

   1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
   2 // vim: ts=8 sw=2 smarttab
   3 /*
   4  * Ceph - scalable distributed file system
   5  *
   6  * Copyright (C) 2016 SUSE LINUX GmbH
   7  *
   8  * This is free software; you can redistribute it and/or
   9  * modify it under the terms of the GNU Lesser General Public
  10  * License version 2.1, as published by the Free Software
  11  * Foundation.  See file COPYING.
  12  *
  13  */
  14 #include "test/librbd/test_fixture.h"
  15 #include "test/librbd/test_support.h"
  16 #include "librbd/ExclusiveLock.h"
  17 #include "librbd/ImageState.h"
  18 #include "librbd/ImageWatcher.h"
  19 #include "librbd/internal.h"
  20 #include "librbd/ObjectMap.h"
  21 #include "librbd/Operations.h"
  22 #include "librbd/io/AioCompletion.h"
  23 #include "librbd/io/ImageRequest.h"
  24 #include "librbd/io/ImageRequestWQ.h"
  25 #include "librbd/journal/Types.h"
  26 #include "journal/Journaler.h"
  27 #include "journal/Settings.h"
  28 #include <boost/scope_exit.hpp>
  29 #include <boost/assign/list_of.hpp>
  30 #include <utility>
  31 #include <vector>
  32
  33 void register_test_mirroring() {
  34 }
  35
  36 class TestMirroring : public TestFixture {
  37 public:
  38
  39   TestMirroring() {}
  40
  41
  42   void TearDown() override {
  43     unlock_image();
  44
  45     TestFixture::TearDown();
  46   }
  47
  48   void SetUp() override {
  49     ASSERT_EQ(0, _rados.ioctx_create(_pool_name.c_str(), m_ioctx));
  50   }
  51
  52   std::string image_name = "mirrorimg1";
  53
  54   void check_mirror_image_enable(rbd_mirror_mode_t mirror_mode,
  55                                  uint64_t features,
  56                                  int expected_r,
  57                                  rbd_mirror_image_state_t mirror_state) {
  58
  59     ASSERT_EQ(0, m_rbd.mirror_mode_set(m_ioctx, RBD_MIRROR_MODE_DISABLED));
  60
  61     int order = 20;
  62     ASSERT_EQ(0, m_rbd.create2(m_ioctx, image_name.c_str(), 4096, features, &order));
  63     librbd::Image image;
  64     ASSERT_EQ(0, m_rbd.open(m_ioctx, image, image_name.c_str()));
  65
  66     ASSERT_EQ(0, m_rbd.mirror_mode_set(m_ioctx, mirror_mode));
  67
  68     ASSERT_EQ(expected_r, image.mirror_image_enable());
  69
  70     librbd::mirror_image_info_t mirror_image;
  71     ASSERT_EQ(0, image.mirror_image_get_info(&mirror_image, sizeof(mirror_image)));
  72     ASSERT_EQ(mirror_state, mirror_image.state);
  73
  74     librbd::mirror_image_status_t status;
  75     ASSERT_EQ(0, image.mirror_image_get_status(&status, sizeof(status)));
  76     ASSERT_EQ(MIRROR_IMAGE_STATUS_STATE_UNKNOWN, status.state);
  77
  78     ASSERT_EQ(0, image.close());
  79     ASSERT_EQ(0, m_rbd.remove(m_ioctx, image_name.c_str()));
  80     ASSERT_EQ(0, m_rbd.mirror_mode_set(m_ioctx, RBD_MIRROR_MODE_DISABLED));
  81   }
  82
  83   void check_mirror_image_disable(rbd_mirror_mode_t mirror_mode,
  84                                   uint64_t features,
  85                                   int expected_r,
  86                                   rbd_mirror_image_state_t mirror_state) {
  87
  88     ASSERT_EQ(0, m_rbd.mirror_mode_set(m_ioctx, RBD_MIRROR_MODE_POOL));
  89
  90     int order = 20;
  91     ASSERT_EQ(0, m_rbd.create2(m_ioctx, image_name.c_str(), 4096, features, &order));
  92     librbd::Image image;
  93     ASSERT_EQ(0, m_rbd.open(m_ioctx, image, image_name.c_str()));
  94
  95     ASSERT_EQ(0, m_rbd.mirror_mode_set(m_ioctx, mirror_mode));
  96
  97     ASSERT_EQ(expected_r, image.mirror_image_disable(false));
  98
  99     librbd::mirror_image_info_t mirror_image;
 100     ASSERT_EQ(0, image.mirror_image_get_info(&mirror_image, sizeof(mirror_image)));
 101     ASSERT_EQ(mirror_state, mirror_image.state);
 102
 103     librbd::mirror_image_status_t status;
 104     ASSERT_EQ(0, image.mirror_image_get_status(&status, sizeof(status)));
 105     ASSERT_EQ(MIRROR_IMAGE_STATUS_STATE_UNKNOWN, status.state);
 106
 107     ASSERT_EQ(0, image.close());
 108     ASSERT_EQ(0, m_rbd.remove(m_ioctx, image_name.c_str()));
 109     ASSERT_EQ(0, m_rbd.mirror_mode_set(m_ioctx, RBD_MIRROR_MODE_DISABLED));
 110   }
 111
 112   void check_mirroring_status(size_t *images_count) {
 113     std::map<std::string, librbd::mirror_image_status_t> images;
 114     ASSERT_EQ(0, m_rbd.mirror_image_status_list(m_ioctx, "", 4096, &images));
 115
 116     std::map<librbd::mirror_image_status_state_t, int> states;
 117     ASSERT_EQ(0, m_rbd.mirror_image_status_summary(m_ioctx, &states));
 118     size_t states_count = 0;
 119     for (auto &s : states) {
 120       states_count += s.second;
 121     }
 122     ASSERT_EQ(images.size(), states_count);
 123
 124     *images_count = images.size();
 125   }
 126
 127   void check_mirroring_on_create(uint64_t features,
 128                                  rbd_mirror_mode_t mirror_mode,
 129                                  rbd_mirror_image_state_t mirror_state) {
 130
 131     ASSERT_EQ(0, m_rbd.mirror_mode_set(m_ioctx, mirror_mode));
 132
 133     size_t mirror_images_count = 0;
 134     check_mirroring_status(&mirror_images_count);
 135
 136     int order = 20;
 137     ASSERT_EQ(0, m_rbd.create2(m_ioctx, image_name.c_str(), 4096, features, &order));
 138     librbd::Image image;
 139     ASSERT_EQ(0, m_rbd.open(m_ioctx, image, image_name.c_str()));
 140
 141     librbd::mirror_image_info_t mirror_image;
 142     ASSERT_EQ(0, image.mirror_image_get_info(&mirror_image, sizeof(mirror_image)));
 143     ASSERT_EQ(mirror_state, mirror_image.state);
 144
 145     librbd::mirror_image_status_t status;
 146     ASSERT_EQ(0, image.mirror_image_get_status(&status, sizeof(status)));
 147     ASSERT_EQ(MIRROR_IMAGE_STATUS_STATE_UNKNOWN, status.state);
 148
 149     size_t mirror_images_new_count = 0;
 150     check_mirroring_status(&mirror_images_new_count);
 151     if (mirror_mode == RBD_MIRROR_MODE_POOL &&
 152         mirror_state == RBD_MIRROR_IMAGE_ENABLED) {
 153       ASSERT_EQ(mirror_images_new_count, mirror_images_count + 1);
 154     } else {
 155       ASSERT_EQ(mirror_images_new_count, mirror_images_count);
 156     }
 157
 158     ASSERT_EQ(0, image.close());
 159     ASSERT_EQ(0, m_rbd.remove(m_ioctx, image_name.c_str()));
 160     ASSERT_EQ(0, m_rbd.mirror_mode_set(m_ioctx, RBD_MIRROR_MODE_DISABLED));
 161
 162     check_mirroring_status(&mirror_images_new_count);
 163     ASSERT_EQ(mirror_images_new_count, mirror_images_count);
 164   }
 165
 166   void check_mirroring_on_update_features(uint64_t init_features,
 167                                  bool enable, bool enable_mirroring,
 168                                  uint64_t features, int expected_r,
 169                                  rbd_mirror_mode_t mirror_mode,
 170                                  rbd_mirror_image_state_t mirror_state) {
 171
 172     ASSERT_EQ(0, m_rbd.mirror_mode_set(m_ioctx, mirror_mode));
 173
 174     int order = 20;
 175     ASSERT_EQ(0, m_rbd.create2(m_ioctx, image_name.c_str(), 4096, init_features, &order));
 176     librbd::Image image;
 177     ASSERT_EQ(0, m_rbd.open(m_ioctx, image, image_name.c_str()));
 178
 179     if (enable_mirroring) {
 180       ASSERT_EQ(0, image.mirror_image_enable());
 181     }
 182
 183     ASSERT_EQ(expected_r, image.update_features(features, enable));
 184
 185     librbd::mirror_image_info_t mirror_image;
 186     ASSERT_EQ(0, image.mirror_image_get_info(&mirror_image, sizeof(mirror_image)));
 187     ASSERT_EQ(mirror_state, mirror_image.state);
 188
 189     librbd::mirror_image_status_t status;
 190     ASSERT_EQ(0, image.mirror_image_get_status(&status, sizeof(status)));
 191     ASSERT_EQ(MIRROR_IMAGE_STATUS_STATE_UNKNOWN, status.state);
 192
 193     ASSERT_EQ(0, image.close());
 194     ASSERT_EQ(0, m_rbd.remove(m_ioctx, image_name.c_str()));
 195     ASSERT_EQ(0, m_rbd.mirror_mode_set(m_ioctx, RBD_MIRROR_MODE_DISABLED));
 196   }
 197
 198   void setup_images_with_mirror_mode(rbd_mirror_mode_t mirror_mode,
 199                                         std::vector<uint64_t>& features_vec) {
 200
 201     ASSERT_EQ(0, m_rbd.mirror_mode_set(m_ioctx, mirror_mode));
 202
 203     int id = 1;
 204     int order = 20;
 205     for (const auto& features : features_vec) {
 206       std::stringstream img_name("img_");
 207       img_name << id++;
 208       std::string img_name_str = img_name.str();
 209       ASSERT_EQ(0, m_rbd.create2(m_ioctx, img_name_str.c_str(), 2048, features, &order));
 210     }
 211   }
 212
 213   void check_mirroring_on_mirror_mode_set(rbd_mirror_mode_t mirror_mode,
 214                             std::vector<rbd_mirror_image_state_t>& states_vec) {
 215
 216     ASSERT_EQ(0, m_rbd.mirror_mode_set(m_ioctx, mirror_mode));
 217
 218     std::vector< std::tuple<std::string, rbd_mirror_image_state_t> > images;
 219     int id = 1;
 220     for (const auto& mirror_state : states_vec) {
 221       std::stringstream img_name("img_");
 222       img_name << id++;
 223       std::string img_name_str = img_name.str();
 224       librbd::Image image;
 225       ASSERT_EQ(0, m_rbd.open(m_ioctx, image, img_name_str.c_str()));
 226       images.push_back(std::make_tuple(img_name_str, mirror_state));
 227
 228       librbd::mirror_image_info_t mirror_image;
 229       ASSERT_EQ(0, image.mirror_image_get_info(&mirror_image, sizeof(mirror_image)));
 230       ASSERT_EQ(mirror_state, mirror_image.state);
 231
 232       librbd::mirror_image_status_t status;
 233       ASSERT_EQ(0, image.mirror_image_get_status(&status, sizeof(status)));
 234       ASSERT_EQ(MIRROR_IMAGE_STATUS_STATE_UNKNOWN, status.state);
 235
 236       ASSERT_EQ(0, image.close());
 237       ASSERT_EQ(0, m_rbd.remove(m_ioctx, img_name_str.c_str()));
 238     }
 239   }
 240
 241   void check_remove_image(rbd_mirror_mode_t mirror_mode, uint64_t features,
 242                           bool enable_mirroring, bool demote = false) {
 243
 244     ASSERT_EQ(0, m_rbd.mirror_mode_set(m_ioctx, mirror_mode));
 245
 246     int order = 20;
 247     ASSERT_EQ(0, m_rbd.create2(m_ioctx, image_name.c_str(), 4096, features,
 248               &order));
 249     librbd::Image image;
 250     ASSERT_EQ(0, m_rbd.open(m_ioctx, image, image_name.c_str()));
 251
 252     if (enable_mirroring) {
 253       ASSERT_EQ(0, image.mirror_image_enable());
 254     }
 255
 256     if (demote) {
 257       ASSERT_EQ(0, image.mirror_image_demote());
 258       ASSERT_EQ(0, image.mirror_image_disable(true));
 259     }
 260
 261     image.close();
 262     ASSERT_EQ(0, m_rbd.remove(m_ioctx, image_name.c_str()));
 263     ASSERT_EQ(0, m_rbd.mirror_mode_set(m_ioctx, RBD_MIRROR_MODE_DISABLED));
 264   }
 265
 266   void setup_mirror_peer(librados::IoCtx &io_ctx, librbd::Image &image) {
 267     ASSERT_EQ(0, image.snap_create("sync-point-snap"));
 268
 269     std::string image_id;
 270     ASSERT_EQ(0, get_image_id(image, &image_id));
 271
 272     librbd::journal::MirrorPeerClientMeta peer_client_meta(
 273       "remote-image-id", {{{}, "sync-point-snap", boost::none}}, {});
 274     librbd::journal::ClientData client_data(peer_client_meta);
 275
 276     journal::Journaler journaler(io_ctx, image_id, "peer-client", {});
 277     C_SaferCond init_ctx;
 278     journaler.init(&init_ctx);
 279     ASSERT_EQ(-ENOENT, init_ctx.wait());
 280
 281     bufferlist client_data_bl;
 282     ::encode(client_data, client_data_bl);
 283     ASSERT_EQ(0, journaler.register_client(client_data_bl));
 284
 285     C_SaferCond shut_down_ctx;
 286     journaler.shut_down(&shut_down_ctx);
 287     ASSERT_EQ(0, shut_down_ctx.wait());
 288   }
 289
 290 };
 291
 292 TEST_F(TestMirroring, EnableImageMirror_In_MirrorModeImage) {
 293   uint64_t features = 0;
 294   features |= RBD_FEATURE_OBJECT_MAP;
 295   features |= RBD_FEATURE_EXCLUSIVE_LOCK;
 296   features |= RBD_FEATURE_JOURNALING;
 297   check_mirror_image_enable(RBD_MIRROR_MODE_IMAGE, features, 0,
 298       RBD_MIRROR_IMAGE_ENABLED);
 299 }
 300
 301 TEST_F(TestMirroring, EnableImageMirror_In_MirrorModePool) {
 302   uint64_t features = 0;
 303   features |= RBD_FEATURE_OBJECT_MAP;
 304   features |= RBD_FEATURE_EXCLUSIVE_LOCK;
 305   features |= RBD_FEATURE_JOURNALING;
 306   check_mirror_image_enable(RBD_MIRROR_MODE_POOL, features, -EINVAL,
 307       RBD_MIRROR_IMAGE_ENABLED);
 308 }
 309
 310 TEST_F(TestMirroring, EnableImageMirror_In_MirrorModeDisabled) {
 311   uint64_t features = 0;
 312   features |= RBD_FEATURE_OBJECT_MAP;
 313   features |= RBD_FEATURE_EXCLUSIVE_LOCK;
 314   features |= RBD_FEATURE_JOURNALING;
 315   check_mirror_image_enable(RBD_MIRROR_MODE_DISABLED, features, -EINVAL,
 316       RBD_MIRROR_IMAGE_DISABLED);
 317 }
 318
 319 TEST_F(TestMirroring, DisableImageMirror_In_MirrorModeImage) {
 320   uint64_t features = 0;
 321   features |= RBD_FEATURE_OBJECT_MAP;
 322   features |= RBD_FEATURE_EXCLUSIVE_LOCK;
 323   features |= RBD_FEATURE_JOURNALING;
 324   check_mirror_image_disable(RBD_MIRROR_MODE_IMAGE, features, 0,
 325       RBD_MIRROR_IMAGE_DISABLED);
 326 }
 327
 328 TEST_F(TestMirroring, DisableImageMirror_In_MirrorModePool) {
 329   uint64_t features = 0;
 330   features |= RBD_FEATURE_OBJECT_MAP;
 331   features |= RBD_FEATURE_EXCLUSIVE_LOCK;
 332   features |= RBD_FEATURE_JOURNALING;
 333   check_mirror_image_disable(RBD_MIRROR_MODE_POOL, features, -EINVAL,
 334       RBD_MIRROR_IMAGE_ENABLED);
 335 }
 336
 337 TEST_F(TestMirroring, DisableImageMirror_In_MirrorModeDisabled) {
 338   uint64_t features = 0;
 339   features |= RBD_FEATURE_OBJECT_MAP;
 340   features |= RBD_FEATURE_EXCLUSIVE_LOCK;
 341   features |= RBD_FEATURE_JOURNALING;
 342   check_mirror_image_disable(RBD_MIRROR_MODE_DISABLED, features, -EINVAL,
 343       RBD_MIRROR_IMAGE_DISABLED);
 344 }
 345
 346 TEST_F(TestMirroring, DisableImageMirrorWithPeer) {
 347   REQUIRE_FEATURE(RBD_FEATURE_JOURNALING);
 348
 349   ASSERT_EQ(0, m_rbd.mirror_mode_set(m_ioctx, RBD_MIRROR_MODE_IMAGE));
 350
 351   uint64_t features = RBD_FEATURE_EXCLUSIVE_LOCK | RBD_FEATURE_JOURNALING;
 352   int order = 20;
 353   ASSERT_EQ(0, m_rbd.create2(m_ioctx, image_name.c_str(), 4096, features,
 354                              &order));
 355
 356   librbd::Image image;
 357   ASSERT_EQ(0, m_rbd.open(m_ioctx, image, image_name.c_str()));
 358   ASSERT_EQ(0, image.mirror_image_enable());
 359
 360   setup_mirror_peer(m_ioctx, image);
 361
 362   ASSERT_EQ(0, image.mirror_image_disable(false));
 363
 364   std::vector<librbd::snap_info_t> snaps;
 365   ASSERT_EQ(0, image.snap_list(snaps));
 366   ASSERT_TRUE(snaps.empty());
 367
 368   librbd::mirror_image_info_t mirror_image;
 369   ASSERT_EQ(0, image.mirror_image_get_info(&mirror_image,
 370                                            sizeof(mirror_image)));
 371   ASSERT_EQ(RBD_MIRROR_IMAGE_DISABLED, mirror_image.state);
 372
 373   librbd::mirror_image_status_t status;
 374   ASSERT_EQ(0, image.mirror_image_get_status(&status, sizeof(status)));
 375   ASSERT_EQ(MIRROR_IMAGE_STATUS_STATE_UNKNOWN, status.state);
 376
 377   ASSERT_EQ(0, image.close());
 378   ASSERT_EQ(0, m_rbd.remove(m_ioctx, image_name.c_str()));
 379   ASSERT_EQ(0, m_rbd.mirror_mode_set(m_ioctx, RBD_MIRROR_MODE_DISABLED));
 380 }
 381
 382 TEST_F(TestMirroring, DisableJournalingWithPeer) {
 383   REQUIRE_FEATURE(RBD_FEATURE_JOURNALING);
 384
 385   ASSERT_EQ(0, m_rbd.mirror_mode_set(m_ioctx, RBD_MIRROR_MODE_POOL));
 386
 387   uint64_t features = RBD_FEATURE_EXCLUSIVE_LOCK | RBD_FEATURE_JOURNALING;
 388   int order = 20;
 389   ASSERT_EQ(0, m_rbd.create2(m_ioctx, image_name.c_str(), 4096, features,
 390                              &order));
 391
 392   librbd::Image image;
 393   ASSERT_EQ(0, m_rbd.open(m_ioctx, image, image_name.c_str()));
 394
 395   setup_mirror_peer(m_ioctx, image);
 396
 397   ASSERT_EQ(0, image.update_features(RBD_FEATURE_JOURNALING, false));
 398
 399   std::vector<librbd::snap_info_t> snaps;
 400   ASSERT_EQ(0, image.snap_list(snaps));
 401   ASSERT_TRUE(snaps.empty());
 402
 403   librbd::mirror_image_info_t mirror_image;
 404   ASSERT_EQ(0, image.mirror_image_get_info(&mirror_image,
 405                                            sizeof(mirror_image)));
 406   ASSERT_EQ(RBD_MIRROR_IMAGE_DISABLED, mirror_image.state);
 407
 408   librbd::mirror_image_status_t status;
 409   ASSERT_EQ(0, image.mirror_image_get_status(&status, sizeof(status)));
 410   ASSERT_EQ(MIRROR_IMAGE_STATUS_STATE_UNKNOWN, status.state);
 411
 412   ASSERT_EQ(0, image.close());
 413   ASSERT_EQ(0, m_rbd.remove(m_ioctx, image_name.c_str()));
 414   ASSERT_EQ(0, m_rbd.mirror_mode_set(m_ioctx, RBD_MIRROR_MODE_DISABLED));
 415 }
 416
 417 TEST_F(TestMirroring, EnableImageMirror_WithoutJournaling) {
 418   uint64_t features = 0;
 419   features |= RBD_FEATURE_OBJECT_MAP;
 420   features |= RBD_FEATURE_EXCLUSIVE_LOCK;
 421   check_mirror_image_enable(RBD_MIRROR_MODE_DISABLED, features, -EINVAL,
 422       RBD_MIRROR_IMAGE_DISABLED);
 423 }
 424
 425 TEST_F(TestMirroring, CreateImage_In_MirrorModeDisabled) {
 426   uint64_t features = 0;
 427   features |= RBD_FEATURE_OBJECT_MAP;
 428   features |= RBD_FEATURE_EXCLUSIVE_LOCK;
 429   features |= RBD_FEATURE_JOURNALING;
 430   check_mirroring_on_create(features, RBD_MIRROR_MODE_DISABLED,
 431                             RBD_MIRROR_IMAGE_DISABLED);
 432 }
 433
 434 TEST_F(TestMirroring, CreateImage_In_MirrorModeImage) {
 435   uint64_t features = 0;
 436   features |= RBD_FEATURE_OBJECT_MAP;
 437   features |= RBD_FEATURE_EXCLUSIVE_LOCK;
 438   features |= RBD_FEATURE_JOURNALING;
 439   check_mirroring_on_create(features, RBD_MIRROR_MODE_IMAGE,
 440                             RBD_MIRROR_IMAGE_DISABLED);
 441 }
 442
 443 TEST_F(TestMirroring, CreateImage_In_MirrorModePool) {
 444   uint64_t features = 0;
 445   features |= RBD_FEATURE_OBJECT_MAP;
 446   features |= RBD_FEATURE_EXCLUSIVE_LOCK;
 447   features |= RBD_FEATURE_JOURNALING;
 448   check_mirroring_on_create(features, RBD_MIRROR_MODE_POOL,
 449                             RBD_MIRROR_IMAGE_ENABLED);
 450 }
 451
 452 TEST_F(TestMirroring, CreateImage_In_MirrorModePool_WithoutJournaling) {
 453   uint64_t features = 0;
 454   features |= RBD_FEATURE_OBJECT_MAP;
 455   features |= RBD_FEATURE_EXCLUSIVE_LOCK;
 456   check_mirroring_on_create(features, RBD_MIRROR_MODE_POOL,
 457                             RBD_MIRROR_IMAGE_DISABLED);
 458 }
 459
 460 TEST_F(TestMirroring, CreateImage_In_MirrorModeImage_WithoutJournaling) {
 461   uint64_t features = 0;
 462   features |= RBD_FEATURE_OBJECT_MAP;
 463   features |= RBD_FEATURE_EXCLUSIVE_LOCK;
 464   check_mirroring_on_create(features, RBD_MIRROR_MODE_IMAGE,
 465                             RBD_MIRROR_IMAGE_DISABLED);
 466 }
 467
 468 TEST_F(TestMirroring, EnableJournaling_In_MirrorModeDisabled) {
 469   uint64_t init_features = 0;
 470   init_features |= RBD_FEATURE_OBJECT_MAP;
 471   init_features |= RBD_FEATURE_EXCLUSIVE_LOCK;
 472   uint64_t features = RBD_FEATURE_JOURNALING;
 473   check_mirroring_on_update_features(init_features, true, false, features, 0,
 474                       RBD_MIRROR_MODE_DISABLED, RBD_MIRROR_IMAGE_DISABLED);
 475 }
 476
 477 TEST_F(TestMirroring, EnableJournaling_In_MirrorModeImage) {
 478   uint64_t init_features = 0;
 479   init_features |= RBD_FEATURE_OBJECT_MAP;
 480   init_features |= RBD_FEATURE_EXCLUSIVE_LOCK;
 481   uint64_t features = RBD_FEATURE_JOURNALING;
 482   check_mirroring_on_update_features(init_features, true, false, features, 0,
 483                       RBD_MIRROR_MODE_IMAGE, RBD_MIRROR_IMAGE_DISABLED);
 484 }
 485
 486 TEST_F(TestMirroring, EnableJournaling_In_MirrorModePool) {
 487   uint64_t init_features = 0;
 488   init_features |= RBD_FEATURE_OBJECT_MAP;
 489   init_features |= RBD_FEATURE_EXCLUSIVE_LOCK;
 490   uint64_t features = RBD_FEATURE_JOURNALING;
 491   check_mirroring_on_update_features(init_features, true, false, features, 0,
 492                       RBD_MIRROR_MODE_POOL, RBD_MIRROR_IMAGE_ENABLED);
 493 }
 494
 495 TEST_F(TestMirroring, DisableJournaling_In_MirrorModePool) {
 496   uint64_t init_features = 0;
 497   init_features |= RBD_FEATURE_OBJECT_MAP;
 498   init_features |= RBD_FEATURE_EXCLUSIVE_LOCK;
 499   init_features |= RBD_FEATURE_JOURNALING;
 500   uint64_t features = RBD_FEATURE_JOURNALING;
 501   check_mirroring_on_update_features(init_features, false, false, features, 0,
 502                       RBD_MIRROR_MODE_POOL, RBD_MIRROR_IMAGE_DISABLED);
 503 }
 504
 505 TEST_F(TestMirroring, DisableJournaling_In_MirrorModeImage) {
 506   uint64_t init_features = 0;
 507   init_features |= RBD_FEATURE_OBJECT_MAP;
 508   init_features |= RBD_FEATURE_EXCLUSIVE_LOCK;
 509   init_features |= RBD_FEATURE_JOURNALING;
 510   uint64_t features = RBD_FEATURE_JOURNALING;
 511   check_mirroring_on_update_features(init_features, false, true, features, -EINVAL,
 512                       RBD_MIRROR_MODE_IMAGE, RBD_MIRROR_IMAGE_ENABLED);
 513 }
 514
 515 TEST_F(TestMirroring, MirrorModeSet_DisabledMode_To_PoolMode) {
 516   std::vector<uint64_t> features_vec;
 517   features_vec.push_back(RBD_FEATURE_EXCLUSIVE_LOCK);
 518   features_vec.push_back(RBD_FEATURE_EXCLUSIVE_LOCK | RBD_FEATURE_JOURNALING);
 519
 520   setup_images_with_mirror_mode(RBD_MIRROR_MODE_DISABLED, features_vec);
 521
 522   std::vector<rbd_mirror_image_state_t> states_vec;
 523   states_vec.push_back(RBD_MIRROR_IMAGE_DISABLED);
 524   states_vec.push_back(RBD_MIRROR_IMAGE_ENABLED);
 525   check_mirroring_on_mirror_mode_set(RBD_MIRROR_MODE_POOL, states_vec);
 526 }
 527
 528 TEST_F(TestMirroring, MirrorModeSet_PoolMode_To_DisabledMode) {
 529   std::vector<uint64_t> features_vec;
 530   features_vec.push_back(RBD_FEATURE_EXCLUSIVE_LOCK);
 531   features_vec.push_back(RBD_FEATURE_EXCLUSIVE_LOCK | RBD_FEATURE_JOURNALING);
 532
 533   setup_images_with_mirror_mode(RBD_MIRROR_MODE_POOL, features_vec);
 534
 535   std::vector<rbd_mirror_image_state_t> states_vec;
 536   states_vec.push_back(RBD_MIRROR_IMAGE_DISABLED);
 537   states_vec.push_back(RBD_MIRROR_IMAGE_DISABLED);
 538   check_mirroring_on_mirror_mode_set(RBD_MIRROR_MODE_DISABLED, states_vec);
 539 }
 540
 541 TEST_F(TestMirroring, MirrorModeSet_DisabledMode_To_ImageMode) {
 542   std::vector<uint64_t> features_vec;
 543   features_vec.push_back(RBD_FEATURE_EXCLUSIVE_LOCK);
 544   features_vec.push_back(RBD_FEATURE_EXCLUSIVE_LOCK | RBD_FEATURE_JOURNALING);
 545
 546   setup_images_with_mirror_mode(RBD_MIRROR_MODE_DISABLED, features_vec);
 547
 548   std::vector<rbd_mirror_image_state_t> states_vec;
 549   states_vec.push_back(RBD_MIRROR_IMAGE_DISABLED);
 550   states_vec.push_back(RBD_MIRROR_IMAGE_DISABLED);
 551   check_mirroring_on_mirror_mode_set(RBD_MIRROR_MODE_IMAGE, states_vec);
 552 }
 553
 554
 555 TEST_F(TestMirroring, MirrorModeSet_PoolMode_To_ImageMode) {
 556   std::vector<uint64_t> features_vec;
 557   features_vec.push_back(RBD_FEATURE_EXCLUSIVE_LOCK);
 558   features_vec.push_back(RBD_FEATURE_EXCLUSIVE_LOCK | RBD_FEATURE_JOURNALING);
 559
 560   setup_images_with_mirror_mode(RBD_MIRROR_MODE_POOL, features_vec);
 561
 562   std::vector<rbd_mirror_image_state_t> states_vec;
 563   states_vec.push_back(RBD_MIRROR_IMAGE_DISABLED);
 564   states_vec.push_back(RBD_MIRROR_IMAGE_ENABLED);
 565   check_mirroring_on_mirror_mode_set(RBD_MIRROR_MODE_IMAGE, states_vec);
 566 }
 567
 568 TEST_F(TestMirroring, MirrorModeSet_ImageMode_To_PoolMode) {
 569   std::vector<uint64_t> features_vec;
 570   features_vec.push_back(RBD_FEATURE_EXCLUSIVE_LOCK);
 571   features_vec.push_back(RBD_FEATURE_EXCLUSIVE_LOCK | RBD_FEATURE_JOURNALING);
 572
 573   setup_images_with_mirror_mode(RBD_MIRROR_MODE_IMAGE, features_vec);
 574
 575   std::vector<rbd_mirror_image_state_t> states_vec;
 576   states_vec.push_back(RBD_MIRROR_IMAGE_DISABLED);
 577   states_vec.push_back(RBD_MIRROR_IMAGE_ENABLED);
 578   check_mirroring_on_mirror_mode_set(RBD_MIRROR_MODE_POOL, states_vec);
 579 }
 580
 581 TEST_F(TestMirroring, MirrorModeSet_ImageMode_To_DisabledMode) {
 582   std::vector<uint64_t> features_vec;
 583   features_vec.push_back(RBD_FEATURE_EXCLUSIVE_LOCK);
 584   features_vec.push_back(RBD_FEATURE_EXCLUSIVE_LOCK | RBD_FEATURE_JOURNALING);
 585
 586   setup_images_with_mirror_mode(RBD_MIRROR_MODE_POOL, features_vec);
 587
 588   ASSERT_EQ(0, m_rbd.mirror_mode_set(m_ioctx, RBD_MIRROR_MODE_IMAGE));
 589   ASSERT_EQ(-EINVAL, m_rbd.mirror_mode_set(m_ioctx, RBD_MIRROR_MODE_DISABLED));
 590   ASSERT_EQ(0, m_rbd.mirror_mode_set(m_ioctx, RBD_MIRROR_MODE_POOL));
 591
 592   std::vector<rbd_mirror_image_state_t> states_vec;
 593   states_vec.push_back(RBD_MIRROR_IMAGE_DISABLED);
 594   states_vec.push_back(RBD_MIRROR_IMAGE_DISABLED);
 595   check_mirroring_on_mirror_mode_set(RBD_MIRROR_MODE_DISABLED, states_vec);
 596 }
 597
 598 TEST_F(TestMirroring, RemoveImage_With_MirrorImageEnabled) {
 599   check_remove_image(RBD_MIRROR_MODE_IMAGE,
 600                      RBD_FEATURE_EXCLUSIVE_LOCK | RBD_FEATURE_JOURNALING,
 601                      true);
 602 }
 603
 604 TEST_F(TestMirroring, RemoveImage_With_MirrorImageDisabled) {
 605   check_remove_image(RBD_MIRROR_MODE_IMAGE,
 606                      RBD_FEATURE_EXCLUSIVE_LOCK | RBD_FEATURE_JOURNALING,
 607                      false);
 608 }
 609
 610 TEST_F(TestMirroring, RemoveImage_With_ImageWithoutJournal) {
 611   check_remove_image(RBD_MIRROR_MODE_IMAGE,
 612                      RBD_FEATURE_EXCLUSIVE_LOCK,
 613                      false);
 614 }
 615
 616 TEST_F(TestMirroring, RemoveImage_With_MirrorImageDemoted) {
 617   check_remove_image(RBD_MIRROR_MODE_IMAGE,
 618                      RBD_FEATURE_EXCLUSIVE_LOCK | RBD_FEATURE_JOURNALING,
 619                      true, true);
 620 }
 621
 622 TEST_F(TestMirroring, MirrorStatusList) {
 623   std::vector<uint64_t>
 624       features_vec(5, RBD_FEATURE_EXCLUSIVE_LOCK | RBD_FEATURE_JOURNALING);
 625   setup_images_with_mirror_mode(RBD_MIRROR_MODE_POOL, features_vec);
 626
 627   std::string last_read = "";
 628   std::map<std::string, librbd::mirror_image_status_t> images;
 629   ASSERT_EQ(0, m_rbd.mirror_image_status_list(m_ioctx, last_read, 2, &images));
 630   ASSERT_EQ(2U, images.size());
 631
 632   last_read = images.rbegin()->first;
 633   images.clear();
 634   ASSERT_EQ(0, m_rbd.mirror_image_status_list(m_ioctx, last_read, 2, &images));
 635   ASSERT_EQ(2U, images.size());
 636
 637   last_read = images.rbegin()->first;
 638   images.clear();
 639   ASSERT_EQ(0, m_rbd.mirror_image_status_list(m_ioctx, last_read, 4096, &images));
 640   ASSERT_EQ(1U, images.size());
 641
 642   last_read = images.rbegin()->first;
 643   images.clear();
 644   ASSERT_EQ(0, m_rbd.mirror_image_status_list(m_ioctx, last_read, 4096, &images));
 645   ASSERT_EQ(0U, images.size());
 646 }
 647
 648 TEST_F(TestMirroring, RemoveBootstrapped)
 649 {
 650   ASSERT_EQ(0, m_rbd.mirror_mode_set(m_ioctx, RBD_MIRROR_MODE_POOL));
 651
 652   uint64_t features = RBD_FEATURE_EXCLUSIVE_LOCK | RBD_FEATURE_JOURNALING;
 653   int order = 20;
 654   ASSERT_EQ(0, m_rbd.create2(m_ioctx, image_name.c_str(), 4096, features,
 655                              &order));
 656   librbd::Image image;
 657   ASSERT_EQ(0, m_rbd.open(m_ioctx, image, image_name.c_str()));
 658   ASSERT_EQ(-EBUSY, m_rbd.remove(m_ioctx, image_name.c_str()));
 659
 660   // simulate the image is open by rbd-mirror bootstrap
 661   uint64_t handle;
 662   struct MirrorWatcher : public librados::WatchCtx2 {
 663     MirrorWatcher(librados::IoCtx &ioctx) : m_ioctx(ioctx) {
 664     }
 665     void handle_notify(uint64_t notify_id, uint64_t cookie,
 666                                uint64_t notifier_id, bufferlist& bl) override {
 667       // received IMAGE_UPDATED notification from remove
 668       m_notified = true;
 669       m_ioctx.notify_ack(RBD_MIRRORING, notify_id, cookie, bl);
 670     }
 671     void handle_error(uint64_t cookie, int err) override {
 672     }
 673     librados::IoCtx &m_ioctx;
 674     bool m_notified = false;
 675   } watcher(m_ioctx);
 676   ASSERT_EQ(0, m_ioctx.create(RBD_MIRRORING, false));
 677   ASSERT_EQ(0, m_ioctx.watch2(RBD_MIRRORING, &handle, &watcher));
 678   // now remove should succeed
 679   ASSERT_EQ(0, m_rbd.remove(m_ioctx, image_name.c_str()));
 680   ASSERT_EQ(0, m_ioctx.unwatch2(handle));
 681   ASSERT_TRUE(watcher.m_notified);
 682   ASSERT_EQ(0, image.close());
 683 }
 684
 685 TEST_F(TestMirroring, AioPromoteDemote) {
 686   std::list<std::string> image_names;
 687   for (size_t idx = 0; idx < 10; ++idx) {
 688     image_names.push_back(get_temp_image_name());
 689   }
 690
 691   ASSERT_EQ(0, m_rbd.mirror_mode_set(m_ioctx, RBD_MIRROR_MODE_IMAGE));
 692
 693   // create mirror images
 694   int order = 20;
 695   std::list<librbd::Image> images;
 696   for (auto &image_name : image_names) {
 697     ASSERT_EQ(0, m_rbd.create2(m_ioctx, image_name.c_str(), 2048,
 698                                RBD_FEATURE_EXCLUSIVE_LOCK |
 699                                  RBD_FEATURE_JOURNALING,
 700                                &order));
 701
 702     images.emplace_back();
 703     ASSERT_EQ(0, m_rbd.open(m_ioctx, images.back(), image_name.c_str()));
 704     ASSERT_EQ(0, images.back().mirror_image_enable());
 705   }
 706
 707   // demote all images
 708   std::list<librbd::RBD::AioCompletion *> aio_comps;
 709   for (auto &image : images) {
 710     aio_comps.push_back(new librbd::RBD::AioCompletion(nullptr, nullptr));
 711     ASSERT_EQ(0, image.aio_mirror_image_demote(aio_comps.back()));
 712   }
 713   for (auto aio_comp : aio_comps) {
 714     ASSERT_EQ(0, aio_comp->wait_for_complete());
 715     ASSERT_EQ(1, aio_comp->is_complete());
 716     ASSERT_EQ(0, aio_comp->get_return_value());
 717     aio_comp->release();
 718   }
 719   aio_comps.clear();
 720
 721   // verify demotions
 722   for (auto &image : images) {
 723     librbd::mirror_image_info_t mirror_image;
 724     ASSERT_EQ(0, image.mirror_image_get_info(&mirror_image,
 725                                              sizeof(mirror_image)));
 726     ASSERT_FALSE(mirror_image.primary);
 727   }
 728
 729   // promote all images
 730   for (auto &image : images) {
 731     aio_comps.push_back(new librbd::RBD::AioCompletion(nullptr, nullptr));
 732     ASSERT_EQ(0, image.aio_mirror_image_promote(false, aio_comps.back()));
 733   }
 734   for (auto aio_comp : aio_comps) {
 735     ASSERT_EQ(0, aio_comp->wait_for_complete());
 736     ASSERT_EQ(1, aio_comp->is_complete());
 737     ASSERT_EQ(0, aio_comp->get_return_value());
 738     aio_comp->release();
 739   }
 740
 741   // verify promotions
 742   for (auto &image : images) {
 743     librbd::mirror_image_info_t mirror_image;
 744     ASSERT_EQ(0, image.mirror_image_get_info(&mirror_image,
 745                                              sizeof(mirror_image)));
 746     ASSERT_TRUE(mirror_image.primary);
 747   }
 748 }
 749
 750 TEST_F(TestMirroring, AioGetInfo) {
 751   std::list<std::string> image_names;
 752   for (size_t idx = 0; idx < 10; ++idx) {
 753     image_names.push_back(get_temp_image_name());
 754   }
 755
 756   ASSERT_EQ(0, m_rbd.mirror_mode_set(m_ioctx, RBD_MIRROR_MODE_POOL));
 757
 758   // create mirror images
 759   int order = 20;
 760   std::list<librbd::Image> images;
 761   for (auto &image_name : image_names) {
 762     ASSERT_EQ(0, m_rbd.create2(m_ioctx, image_name.c_str(), 2048,
 763                                RBD_FEATURE_EXCLUSIVE_LOCK |
 764                                  RBD_FEATURE_JOURNALING,
 765                                &order));
 766
 767     images.emplace_back();
 768     ASSERT_EQ(0, m_rbd.open(m_ioctx, images.back(), image_name.c_str()));
 769   }
 770
 771   std::list<librbd::RBD::AioCompletion *> aio_comps;
 772   std::list<librbd::mirror_image_info_t> infos;
 773   for (auto &image : images) {
 774     aio_comps.push_back(new librbd::RBD::AioCompletion(nullptr, nullptr));
 775     infos.emplace_back();
 776     ASSERT_EQ(0, image.aio_mirror_image_get_info(&infos.back(),
 777                                                  sizeof(infos.back()),
 778                                                  aio_comps.back()));
 779   }
 780   for (auto aio_comp : aio_comps) {
 781     ASSERT_EQ(0, aio_comp->wait_for_complete());
 782     ASSERT_EQ(1, aio_comp->is_complete());
 783     ASSERT_EQ(0, aio_comp->get_return_value());
 784     aio_comp->release();
 785   }
 786   aio_comps.clear();
 787
 788   for (auto &info : infos) {
 789     ASSERT_NE("", info.global_id);
 790     ASSERT_EQ(RBD_MIRROR_IMAGE_ENABLED, info.state);
 791     ASSERT_TRUE(info.primary);
 792   }
 793 }
 794
 795 TEST_F(TestMirroring, AioGetStatus) {
 796   std::list<std::string> image_names;
 797   for (size_t idx = 0; idx < 10; ++idx) {
 798     image_names.push_back(get_temp_image_name());
 799   }
 800
 801   ASSERT_EQ(0, m_rbd.mirror_mode_set(m_ioctx, RBD_MIRROR_MODE_POOL));
 802
 803   // create mirror images
 804   int order = 20;
 805   std::list<librbd::Image> images;
 806   for (auto &image_name : image_names) {
 807     ASSERT_EQ(0, m_rbd.create2(m_ioctx, image_name.c_str(), 2048,
 808                                RBD_FEATURE_EXCLUSIVE_LOCK |
 809                                  RBD_FEATURE_JOURNALING,
 810                                &order));
 811
 812     images.emplace_back();
 813     ASSERT_EQ(0, m_rbd.open(m_ioctx, images.back(), image_name.c_str()));
 814   }
 815
 816   std::list<librbd::RBD::AioCompletion *> aio_comps;
 817   std::list<librbd::mirror_image_status_t> statuses;
 818   for (auto &image : images) {
 819     aio_comps.push_back(new librbd::RBD::AioCompletion(nullptr, nullptr));
 820     statuses.emplace_back();
 821     ASSERT_EQ(0, image.aio_mirror_image_get_status(&statuses.back(),
 822                                                    sizeof(statuses.back()),
 823                                                    aio_comps.back()));
 824   }
 825   for (auto aio_comp : aio_comps) {
 826     ASSERT_EQ(0, aio_comp->wait_for_complete());
 827     ASSERT_EQ(1, aio_comp->is_complete());
 828     ASSERT_EQ(0, aio_comp->get_return_value());
 829     aio_comp->release();
 830   }
 831   aio_comps.clear();
 832
 833   for (auto &status : statuses) {
 834     ASSERT_NE("", status.name);
 835     ASSERT_NE("", status.info.global_id);
 836     ASSERT_EQ(RBD_MIRROR_IMAGE_ENABLED, status.info.state);
 837     ASSERT_TRUE(status.info.primary);
 838     ASSERT_EQ(MIRROR_IMAGE_STATUS_STATE_UNKNOWN, status.state);
 839     ASSERT_EQ("status not found", status.description);
 840     ASSERT_FALSE(status.up);
 841     ASSERT_EQ(0, status.last_update);
 842   }
 843 }