1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
2 // vim: ts=8 sw=2 smarttab
3 #include "gtest/gtest.h"
5 #include "mds/mdstypes.h"
6 #include "include/buffer.h"
7 #include "include/rbd_types.h"
8 #include "include/rados/librados.h"
9 #include "include/rados/librados.hpp"
10 #include "include/stringify.h"
11 #include "include/types.h"
12 #include "global/global_context.h"
13 #include "common/Cond.h"
14 #include "test/librados/test.h"
15 #include "test/librados/TestCase.h"
16 #include "json_spirit/json_spirit.h"
18 #include "osd/HitSet.h"
25 using namespace librados;
27 using std::ostringstream;
30 typedef RadosTestPP LibRadosTierPP;
31 typedef RadosTestECPP LibRadosTierECPP;
33 void flush_evict_all(librados::Rados& cluster, librados::IoCtx& cache_ioctx)
36 cache_ioctx.set_namespace(all_nspaces);
37 for (NObjectIterator it = cache_ioctx.nobjects_begin();
38 it != cache_ioctx.nobjects_end(); ++it) {
39 cache_ioctx.locator_set_key(it->get_locator());
40 cache_ioctx.set_namespace(it->get_nspace());
42 ObjectReadOperation op;
44 librados::AioCompletion *completion = cluster.aio_create_completion();
45 cache_ioctx.aio_operate(
46 it->get_oid(), completion, &op,
47 librados::OPERATION_IGNORE_OVERLAY, NULL);
48 completion->wait_for_safe();
49 completion->get_return_value();
50 completion->release();
53 ObjectReadOperation op;
55 librados::AioCompletion *completion = cluster.aio_create_completion();
56 cache_ioctx.aio_operate(
57 it->get_oid(), completion, &op,
58 librados::OPERATION_IGNORE_OVERLAY, NULL);
59 completion->wait_for_safe();
60 completion->get_return_value();
61 completion->release();
66 class LibRadosTwoPoolsPP : public RadosTestPP
69 LibRadosTwoPoolsPP() {};
70 ~LibRadosTwoPoolsPP() override {};
72 static void SetUpTestCase() {
73 pool_name = get_temp_pool_name();
74 ASSERT_EQ("", create_one_pool_pp(pool_name, s_cluster));
76 static void TearDownTestCase() {
77 ASSERT_EQ(0, destroy_one_pool_pp(pool_name, s_cluster));
79 static std::string cache_pool_name;
81 void SetUp() override {
82 cache_pool_name = get_temp_pool_name();
83 ASSERT_EQ(0, s_cluster.pool_create(cache_pool_name.c_str()));
86 ASSERT_EQ(0, cluster.ioctx_create(cache_pool_name.c_str(), cache_ioctx));
87 cache_ioctx.application_enable("rados", true);
88 cache_ioctx.set_namespace(nspace);
90 void TearDown() override {
91 // flush + evict cache
92 flush_evict_all(cluster, cache_ioctx);
96 ASSERT_EQ(0, cluster.mon_command(
97 "{\"prefix\": \"osd tier remove-overlay\", \"pool\": \"" + pool_name +
100 ASSERT_EQ(0, cluster.mon_command(
101 "{\"prefix\": \"osd tier remove\", \"pool\": \"" + pool_name +
102 "\", \"tierpool\": \"" + cache_pool_name + "\"}",
105 // wait for maps to settle before next test
106 cluster.wait_for_latest_osdmap();
108 RadosTestPP::TearDown();
110 cleanup_default_namespace(cache_ioctx);
111 cleanup_namespace(cache_ioctx, nspace);
114 ASSERT_EQ(0, s_cluster.pool_delete(cache_pool_name.c_str()));
116 librados::IoCtx cache_ioctx;
122 Completions() = default;
123 librados::AioCompletion* getCompletion() {
124 librados::AioCompletion* comp = librados::Rados::aio_create_completion();
125 m_completions.push_back(comp);
130 for (auto& comp : m_completions) {
136 vector<librados::AioCompletion *> m_completions;
139 Completions completions;
141 std::string LibRadosTwoPoolsPP::cache_pool_name;
143 TEST_F(LibRadosTierPP, Dirty) {
145 ObjectWriteOperation op;
147 ASSERT_EQ(0, ioctx.operate("foo", &op)); // still get 0 if it dne
150 ObjectWriteOperation op;
152 ASSERT_EQ(0, ioctx.operate("foo", &op));
157 ObjectReadOperation op;
158 op.is_dirty(&dirty, &r);
159 ASSERT_EQ(0, ioctx.operate("foo", &op, NULL));
164 ObjectWriteOperation op;
166 ASSERT_EQ(0, ioctx.operate("foo", &op));
169 ObjectWriteOperation op;
171 ASSERT_EQ(0, ioctx.operate("foo", &op)); // still 0 if already clean
176 ObjectReadOperation op;
177 op.is_dirty(&dirty, &r);
178 ASSERT_EQ(0, ioctx.operate("foo", &op, NULL));
183 ObjectWriteOperation op;
184 op.truncate(0); // still a write even tho it is a no-op
185 ASSERT_EQ(0, ioctx.operate("foo", &op));
190 ObjectReadOperation op;
191 op.is_dirty(&dirty, &r);
192 ASSERT_EQ(0, ioctx.operate("foo", &op, NULL));
198 TEST_F(LibRadosTwoPoolsPP, Overlay) {
203 ObjectWriteOperation op;
205 ASSERT_EQ(0, ioctx.operate("foo", &op));
210 ObjectWriteOperation op;
212 ASSERT_EQ(0, cache_ioctx.operate("foo", &op));
217 ASSERT_EQ(0, cluster.mon_command(
218 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
219 "\", \"tierpool\": \"" + cache_pool_name +
220 "\", \"force_nonempty\": \"--force-nonempty\" }",
222 ASSERT_EQ(0, cluster.mon_command(
223 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
224 "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
227 // wait for maps to settle
228 cluster.wait_for_latest_osdmap();
230 // by default, the overlay sends us to cache pool
233 ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
234 ASSERT_EQ('c', bl[0]);
238 ASSERT_EQ(1, cache_ioctx.read("foo", bl, 1, 0));
239 ASSERT_EQ('c', bl[0]);
242 // unless we say otherwise
245 ObjectReadOperation op;
246 op.read(0, 1, &bl, NULL);
247 librados::AioCompletion *completion = cluster.aio_create_completion();
248 ASSERT_EQ(0, ioctx.aio_operate(
249 "foo", completion, &op,
250 librados::OPERATION_IGNORE_OVERLAY, NULL));
251 completion->wait_for_safe();
252 ASSERT_EQ(0, completion->get_return_value());
253 completion->release();
254 ASSERT_EQ('b', bl[0]);
258 TEST_F(LibRadosTwoPoolsPP, Promote) {
262 bl.append("hi there");
263 ObjectWriteOperation op;
265 ASSERT_EQ(0, ioctx.operate("foo", &op));
270 ASSERT_EQ(0, cluster.mon_command(
271 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
272 "\", \"tierpool\": \"" + cache_pool_name +
273 "\", \"force_nonempty\": \"--force-nonempty\" }",
275 ASSERT_EQ(0, cluster.mon_command(
276 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
277 "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
279 ASSERT_EQ(0, cluster.mon_command(
280 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
281 "\", \"mode\": \"writeback\"}",
284 // wait for maps to settle
285 cluster.wait_for_latest_osdmap();
287 // read, trigger a promote
290 ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
293 // read, trigger a whiteout
296 ASSERT_EQ(-ENOENT, ioctx.read("bar", bl, 1, 0));
297 ASSERT_EQ(-ENOENT, ioctx.read("bar", bl, 1, 0));
300 // verify the object is present in the cache tier
302 NObjectIterator it = cache_ioctx.nobjects_begin();
303 ASSERT_TRUE(it != cache_ioctx.nobjects_end());
304 ASSERT_TRUE(it->get_oid() == string("foo") || it->get_oid() == string("bar"));
306 ASSERT_TRUE(it->get_oid() == string("foo") || it->get_oid() == string("bar"));
308 ASSERT_TRUE(it == cache_ioctx.nobjects_end());
312 TEST_F(LibRadosTwoPoolsPP, PromoteSnap) {
316 bl.append("hi there");
317 ObjectWriteOperation op;
319 ASSERT_EQ(0, ioctx.operate("foo", &op));
323 bl.append("hi there");
324 ObjectWriteOperation op;
326 ASSERT_EQ(0, ioctx.operate("bar", &op));
330 bl.append("hi there");
331 ObjectWriteOperation op;
333 ASSERT_EQ(0, ioctx.operate("baz", &op));
337 bl.append("hi there");
338 ObjectWriteOperation op;
340 ASSERT_EQ(0, ioctx.operate("bam", &op));
343 // create a snapshot, clone
344 vector<uint64_t> my_snaps(1);
345 ASSERT_EQ(0, ioctx.selfmanaged_snap_create(&my_snaps[0]));
346 ASSERT_EQ(0, ioctx.selfmanaged_snap_set_write_ctx(my_snaps[0],
351 ObjectWriteOperation op;
353 ASSERT_EQ(0, ioctx.operate("foo", &op));
358 ObjectWriteOperation op;
360 ASSERT_EQ(0, ioctx.operate("bar", &op));
363 ObjectWriteOperation op;
365 ASSERT_EQ(0, ioctx.operate("baz", &op));
370 ObjectWriteOperation op;
372 ASSERT_EQ(0, ioctx.operate("bam", &op));
377 ASSERT_EQ(0, cluster.mon_command(
378 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
379 "\", \"tierpool\": \"" + cache_pool_name +
380 "\", \"force_nonempty\": \"--force-nonempty\" }",
382 ASSERT_EQ(0, cluster.mon_command(
383 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
384 "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
386 ASSERT_EQ(0, cluster.mon_command(
387 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
388 "\", \"mode\": \"writeback\"}",
391 // wait for maps to settle
392 cluster.wait_for_latest_osdmap();
394 // read, trigger a promote on the head
397 ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
398 ASSERT_EQ('c', bl[0]);
402 ASSERT_EQ(1, ioctx.read("bam", bl, 1, 0));
403 ASSERT_EQ('c', bl[0]);
406 ioctx.snap_set_read(my_snaps[0]);
411 ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
412 ASSERT_EQ('h', bl[0]);
418 ASSERT_EQ(1, ioctx.read("bar", bl, 1, 0));
419 ASSERT_EQ('h', bl[0]);
425 ASSERT_EQ(1, ioctx.read("baz", bl, 1, 0));
426 ASSERT_EQ('h', bl[0]);
429 ioctx.snap_set_read(librados::SNAP_HEAD);
434 ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
435 ASSERT_EQ('c', bl[0]);
441 ASSERT_EQ(1, ioctx.read("bar", bl, 1, 0));
442 ASSERT_EQ('c', bl[0]);
448 ASSERT_EQ(-ENOENT, ioctx.read("baz", bl, 1, 0));
452 ioctx.selfmanaged_snap_remove(my_snaps[0]);
455 TEST_F(LibRadosTwoPoolsPP, PromoteSnapScrub) {
459 for (int i=0; i<num; ++i) {
461 bl.append("hi there");
462 ObjectWriteOperation op;
464 ASSERT_EQ(0, ioctx.operate(string("foo") + stringify(i), &op));
467 vector<uint64_t> my_snaps;
468 for (int snap=0; snap<4; ++snap) {
469 // create a snapshot, clone
470 vector<uint64_t> ns(1);
471 ns.insert(ns.end(), my_snaps.begin(), my_snaps.end());
473 ASSERT_EQ(0, ioctx.selfmanaged_snap_create(&my_snaps[0]));
474 cout << "my_snaps " << my_snaps << std::endl;
475 ASSERT_EQ(0, ioctx.selfmanaged_snap_set_write_ctx(my_snaps[0],
477 for (int i=0; i<num; ++i) {
479 bl.append(string("ciao! snap") + stringify(snap));
480 ObjectWriteOperation op;
482 ASSERT_EQ(0, ioctx.operate(string("foo") + stringify(i), &op));
488 ASSERT_EQ(0, cluster.mon_command(
489 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
490 "\", \"tierpool\": \"" + cache_pool_name +
491 "\", \"force_nonempty\": \"--force-nonempty\" }",
493 ASSERT_EQ(0, cluster.mon_command(
494 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
495 "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
497 ASSERT_EQ(0, cluster.mon_command(
498 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
499 "\", \"mode\": \"writeback\"}",
502 // wait for maps to settle
503 cluster.wait_for_latest_osdmap();
505 // read, trigger a promote on _some_ heads to make sure we handle cases
506 // where snaps are present and where they are not.
507 cout << "promoting some heads" << std::endl;
508 for (int i=0; i<num; ++i) {
509 if (i % 5 == 0 || i > num - 3) {
511 ASSERT_EQ(1, ioctx.read(string("foo") + stringify(i), bl, 1, 0));
512 ASSERT_EQ('c', bl[0]);
516 for (unsigned snap = 0; snap < my_snaps.size(); ++snap) {
517 cout << "promoting from clones for snap " << my_snaps[snap] << std::endl;
518 ioctx.snap_set_read(my_snaps[snap]);
520 // read some snaps, semi-randomly
521 for (int i=0; i<50; ++i) {
523 string o = string("foo") + stringify((snap * i * 137) % 80);
524 //cout << o << std::endl;
525 ASSERT_EQ(1, ioctx.read(o, bl, 1, 0));
529 // ok, stop and scrub this pool (to make sure scrub can handle
530 // missing clones in the cache tier).
533 ASSERT_EQ(0, cluster.ioctx_create(cache_pool_name.c_str(), cache_ioctx));
534 for (int i=0; i<10; ++i) {
537 ss << "{\"prefix\": \"pg scrub\", \"pgid\": \""
538 << cache_ioctx.get_id() << "." << i
540 int r = cluster.mon_command(ss.str(), inbl, NULL, NULL);
541 if (r == -ENOENT || // in case mgr osdmap is stale
549 // give it a few seconds to go. this is sloppy but is usually enough time
550 cout << "waiting for scrubs..." << std::endl;
552 cout << "done waiting" << std::endl;
555 ioctx.snap_set_read(librados::SNAP_HEAD);
558 for (unsigned snap = 0; snap < my_snaps.size(); ++snap) {
559 ioctx.selfmanaged_snap_remove(my_snaps[snap]);
563 TEST_F(LibRadosTwoPoolsPP, PromoteSnapTrimRace) {
567 bl.append("hi there");
568 ObjectWriteOperation op;
570 ASSERT_EQ(0, ioctx.operate("foo", &op));
573 // create a snapshot, clone
574 vector<uint64_t> my_snaps(1);
575 ASSERT_EQ(0, ioctx.selfmanaged_snap_create(&my_snaps[0]));
576 ASSERT_EQ(0, ioctx.selfmanaged_snap_set_write_ctx(my_snaps[0],
581 ObjectWriteOperation op;
583 ASSERT_EQ(0, ioctx.operate("foo", &op));
588 ASSERT_EQ(0, cluster.mon_command(
589 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
590 "\", \"tierpool\": \"" + cache_pool_name +
591 "\", \"force_nonempty\": \"--force-nonempty\" }",
593 ASSERT_EQ(0, cluster.mon_command(
594 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
595 "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
597 ASSERT_EQ(0, cluster.mon_command(
598 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
599 "\", \"mode\": \"writeback\"}",
602 // wait for maps to settle
603 cluster.wait_for_latest_osdmap();
606 ASSERT_EQ(0, ioctx.selfmanaged_snap_remove(my_snaps[0]));
608 ioctx.snap_set_read(my_snaps[0]);
613 ASSERT_EQ(-ENOENT, ioctx.read("foo", bl, 1, 0));
617 ioctx.selfmanaged_snap_remove(my_snaps[0]);
620 TEST_F(LibRadosTwoPoolsPP, Whiteout) {
624 bl.append("hi there");
625 ObjectWriteOperation op;
627 ASSERT_EQ(0, ioctx.operate("foo", &op));
632 ASSERT_EQ(0, cluster.mon_command(
633 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
634 "\", \"tierpool\": \"" + cache_pool_name +
635 "\", \"force_nonempty\": \"--force-nonempty\" }",
637 ASSERT_EQ(0, cluster.mon_command(
638 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
639 "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
641 ASSERT_EQ(0, cluster.mon_command(
642 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
643 "\", \"mode\": \"writeback\"}",
646 // wait for maps to settle
647 cluster.wait_for_latest_osdmap();
649 // create some whiteouts, verify they behave
651 ObjectWriteOperation op;
654 ASSERT_EQ(0, ioctx.operate("foo", &op));
658 ObjectWriteOperation op;
661 ASSERT_EQ(-ENOENT, ioctx.operate("bar", &op));
664 ObjectWriteOperation op;
667 ASSERT_EQ(-ENOENT, ioctx.operate("bar", &op));
670 // verify the whiteouts are there in the cache tier
672 NObjectIterator it = cache_ioctx.nobjects_begin();
673 ASSERT_TRUE(it != cache_ioctx.nobjects_end());
674 ASSERT_TRUE(it->get_oid() == string("foo") || it->get_oid() == string("bar"));
676 ASSERT_TRUE(it->get_oid() == string("foo") || it->get_oid() == string("bar"));
678 ASSERT_TRUE(it == cache_ioctx.nobjects_end());
681 // delete a whiteout and verify it goes away
682 ASSERT_EQ(-ENOENT, ioctx.remove("foo"));
684 ObjectWriteOperation op;
686 librados::AioCompletion *completion = cluster.aio_create_completion();
687 ASSERT_EQ(0, ioctx.aio_operate("bar", completion, &op,
688 librados::OPERATION_IGNORE_CACHE));
689 completion->wait_for_safe();
690 ASSERT_EQ(0, completion->get_return_value());
691 completion->release();
693 NObjectIterator it = cache_ioctx.nobjects_begin();
694 ASSERT_TRUE(it != cache_ioctx.nobjects_end());
695 ASSERT_TRUE(it->get_oid() == string("foo"));
697 ASSERT_TRUE(it == cache_ioctx.nobjects_end());
700 // recreate an object and verify we can read it
703 bl.append("hi there");
704 ObjectWriteOperation op;
706 ASSERT_EQ(0, ioctx.operate("foo", &op));
710 ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
711 ASSERT_EQ('h', bl[0]);
715 TEST_F(LibRadosTwoPoolsPP, WhiteoutDeleteCreate) {
718 ASSERT_EQ(0, cluster.mon_command(
719 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
720 "\", \"tierpool\": \"" + cache_pool_name +
721 "\", \"force_nonempty\": \"--force-nonempty\" }",
723 ASSERT_EQ(0, cluster.mon_command(
724 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
725 "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
727 ASSERT_EQ(0, cluster.mon_command(
728 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
729 "\", \"mode\": \"writeback\"}",
732 // wait for maps to settle
733 cluster.wait_for_latest_osdmap();
739 ASSERT_EQ(0, ioctx.write_full("foo", bl));
742 // do delete + create operation
744 ObjectWriteOperation op;
749 ASSERT_EQ(0, ioctx.operate("foo", &op));
752 // verify it still "exists" (w/ new content)
755 ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
756 ASSERT_EQ('b', bl[0]);
760 TEST_F(LibRadosTwoPoolsPP, Evict) {
764 bl.append("hi there");
765 ObjectWriteOperation op;
767 ASSERT_EQ(0, ioctx.operate("foo", &op));
772 ASSERT_EQ(0, cluster.mon_command(
773 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
774 "\", \"tierpool\": \"" + cache_pool_name +
775 "\", \"force_nonempty\": \"--force-nonempty\" }",
777 ASSERT_EQ(0, cluster.mon_command(
778 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
779 "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
781 ASSERT_EQ(0, cluster.mon_command(
782 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
783 "\", \"mode\": \"writeback\"}",
786 // wait for maps to settle
787 cluster.wait_for_latest_osdmap();
789 // read, trigger a promote
792 ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
795 // read, trigger a whiteout, and a dirty object
798 ASSERT_EQ(-ENOENT, ioctx.read("bar", bl, 1, 0));
799 ASSERT_EQ(-ENOENT, ioctx.read("bar", bl, 1, 0));
800 ASSERT_EQ(0, ioctx.write("bar", bl, bl.length(), 0));
803 // verify the object is present in the cache tier
805 NObjectIterator it = cache_ioctx.nobjects_begin();
806 ASSERT_TRUE(it != cache_ioctx.nobjects_end());
807 ASSERT_TRUE(it->get_oid() == string("foo") || it->get_oid() == string("bar"));
809 ASSERT_TRUE(it->get_oid() == string("foo") || it->get_oid() == string("bar"));
811 ASSERT_TRUE(it == cache_ioctx.nobjects_end());
816 ObjectWriteOperation op;
818 librados::AioCompletion *completion = cluster.aio_create_completion();
819 ASSERT_EQ(0, cache_ioctx.aio_operate("foo", completion, &op));
820 completion->wait_for_safe();
821 ASSERT_EQ(0, completion->get_return_value());
822 completion->release();
825 // evict the pinned object with -EPERM
827 ObjectReadOperation op;
829 librados::AioCompletion *completion = cluster.aio_create_completion();
830 ASSERT_EQ(0, cache_ioctx.aio_operate("foo", completion, &op,
831 librados::OPERATION_IGNORE_CACHE,
833 completion->wait_for_safe();
834 ASSERT_EQ(-EPERM, completion->get_return_value());
835 completion->release();
840 ObjectWriteOperation op;
842 librados::AioCompletion *completion = cluster.aio_create_completion();
843 ASSERT_EQ(0, cache_ioctx.aio_operate("foo", completion, &op));
844 completion->wait_for_safe();
845 ASSERT_EQ(0, completion->get_return_value());
846 completion->release();
851 ObjectReadOperation op;
853 librados::AioCompletion *completion = cluster.aio_create_completion();
854 ASSERT_EQ(0, cache_ioctx.aio_operate(
855 "foo", completion, &op,
856 librados::OPERATION_IGNORE_OVERLAY, NULL));
857 completion->wait_for_safe();
858 ASSERT_EQ(0, completion->get_return_value());
859 completion->release();
866 ObjectReadOperation op;
867 op.is_dirty(&dirty, &r);
868 ASSERT_EQ(0, cache_ioctx.operate("foo", &op, NULL));
875 ObjectReadOperation op;
877 librados::AioCompletion *completion = cluster.aio_create_completion();
878 ASSERT_EQ(0, cache_ioctx.aio_operate("foo", completion, &op,
879 librados::OPERATION_IGNORE_CACHE,
881 completion->wait_for_safe();
882 ASSERT_EQ(0, completion->get_return_value());
883 completion->release();
886 ObjectReadOperation op;
888 librados::AioCompletion *completion = cluster.aio_create_completion();
889 ASSERT_EQ(0, cache_ioctx.aio_operate(
890 "foo", completion, &op,
891 librados::OPERATION_IGNORE_CACHE, NULL));
892 completion->wait_for_safe();
893 ASSERT_EQ(0, completion->get_return_value());
894 completion->release();
897 ObjectReadOperation op;
899 librados::AioCompletion *completion = cluster.aio_create_completion();
900 ASSERT_EQ(0, cache_ioctx.aio_operate(
901 "bar", completion, &op,
902 librados::OPERATION_IGNORE_CACHE, NULL));
903 completion->wait_for_safe();
904 ASSERT_EQ(-EBUSY, completion->get_return_value());
905 completion->release();
909 TEST_F(LibRadosTwoPoolsPP, EvictSnap) {
913 bl.append("hi there");
914 ObjectWriteOperation op;
916 ASSERT_EQ(0, ioctx.operate("foo", &op));
920 bl.append("hi there");
921 ObjectWriteOperation op;
923 ASSERT_EQ(0, ioctx.operate("bar", &op));
927 bl.append("hi there");
928 ObjectWriteOperation op;
930 ASSERT_EQ(0, ioctx.operate("baz", &op));
934 bl.append("hi there");
935 ObjectWriteOperation op;
937 ASSERT_EQ(0, ioctx.operate("bam", &op));
940 // create a snapshot, clone
941 vector<uint64_t> my_snaps(1);
942 ASSERT_EQ(0, ioctx.selfmanaged_snap_create(&my_snaps[0]));
943 ASSERT_EQ(0, ioctx.selfmanaged_snap_set_write_ctx(my_snaps[0],
948 ObjectWriteOperation op;
950 ASSERT_EQ(0, ioctx.operate("foo", &op));
955 ObjectWriteOperation op;
957 ASSERT_EQ(0, ioctx.operate("bar", &op));
960 ObjectWriteOperation op;
962 ASSERT_EQ(0, ioctx.operate("baz", &op));
967 ObjectWriteOperation op;
969 ASSERT_EQ(0, ioctx.operate("bam", &op));
974 ASSERT_EQ(0, cluster.mon_command(
975 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
976 "\", \"tierpool\": \"" + cache_pool_name +
977 "\", \"force_nonempty\": \"--force-nonempty\" }",
979 ASSERT_EQ(0, cluster.mon_command(
980 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
981 "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
983 ASSERT_EQ(0, cluster.mon_command(
984 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
985 "\", \"mode\": \"writeback\"}",
988 // wait for maps to settle
989 cluster.wait_for_latest_osdmap();
991 // read, trigger a promote on the head
994 ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
995 ASSERT_EQ('c', bl[0]);
999 ASSERT_EQ(1, ioctx.read("bam", bl, 1, 0));
1000 ASSERT_EQ('c', bl[0]);
1005 ObjectReadOperation op;
1007 librados::AioCompletion *completion = cluster.aio_create_completion();
1008 ASSERT_EQ(0, cache_ioctx.aio_operate(
1009 "bam", completion, &op,
1010 librados::OPERATION_IGNORE_CACHE, NULL));
1011 completion->wait_for_safe();
1012 ASSERT_EQ(0, completion->get_return_value());
1013 completion->release();
1017 ObjectReadOperation op;
1018 op.read(1, 0, &bl, NULL);
1019 librados::AioCompletion *completion = cluster.aio_create_completion();
1020 ASSERT_EQ(0, cache_ioctx.aio_operate(
1021 "bam", completion, &op,
1022 librados::OPERATION_IGNORE_CACHE, NULL));
1023 completion->wait_for_safe();
1024 ASSERT_EQ(-ENOENT, completion->get_return_value());
1025 completion->release();
1029 ioctx.snap_set_read(my_snaps[0]);
1032 ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
1033 ASSERT_EQ('h', bl[0]);
1038 ObjectReadOperation op;
1040 librados::AioCompletion *completion = cluster.aio_create_completion();
1041 ASSERT_EQ(0, ioctx.aio_operate(
1042 "foo", completion, &op,
1043 librados::OPERATION_IGNORE_CACHE, NULL));
1044 completion->wait_for_safe();
1045 ASSERT_EQ(0, completion->get_return_value());
1046 completion->release();
1051 ObjectReadOperation op;
1052 op.read(1, 0, &bl, NULL);
1053 librados::AioCompletion *completion = cluster.aio_create_completion();
1054 ASSERT_EQ(0, ioctx.aio_operate(
1055 "foo", completion, &op,
1056 librados::OPERATION_IGNORE_CACHE, NULL));
1057 completion->wait_for_safe();
1058 ASSERT_EQ(-ENOENT, completion->get_return_value());
1059 completion->release();
1061 // head is still there...
1062 ioctx.snap_set_read(librados::SNAP_HEAD);
1065 ObjectReadOperation op;
1066 op.read(1, 0, &bl, NULL);
1067 librados::AioCompletion *completion = cluster.aio_create_completion();
1068 ASSERT_EQ(0, ioctx.aio_operate(
1069 "foo", completion, &op,
1070 librados::OPERATION_IGNORE_CACHE, NULL));
1071 completion->wait_for_safe();
1072 ASSERT_EQ(0, completion->get_return_value());
1073 completion->release();
1076 // promote head + snap of bar
1077 ioctx.snap_set_read(librados::SNAP_HEAD);
1080 ASSERT_EQ(1, ioctx.read("bar", bl, 1, 0));
1081 ASSERT_EQ('c', bl[0]);
1083 ioctx.snap_set_read(my_snaps[0]);
1086 ASSERT_EQ(1, ioctx.read("bar", bl, 1, 0));
1087 ASSERT_EQ('h', bl[0]);
1090 // evict bar head (fail)
1091 ioctx.snap_set_read(librados::SNAP_HEAD);
1093 ObjectReadOperation op;
1095 librados::AioCompletion *completion = cluster.aio_create_completion();
1096 ASSERT_EQ(0, ioctx.aio_operate(
1097 "bar", completion, &op,
1098 librados::OPERATION_IGNORE_CACHE, NULL));
1099 completion->wait_for_safe();
1100 ASSERT_EQ(-EBUSY, completion->get_return_value());
1101 completion->release();
1105 ioctx.snap_set_read(my_snaps[0]);
1107 ObjectReadOperation op;
1109 librados::AioCompletion *completion = cluster.aio_create_completion();
1110 ASSERT_EQ(0, ioctx.aio_operate(
1111 "bar", completion, &op,
1112 librados::OPERATION_IGNORE_CACHE, NULL));
1113 completion->wait_for_safe();
1114 ASSERT_EQ(0, completion->get_return_value());
1115 completion->release();
1118 ioctx.snap_set_read(librados::SNAP_HEAD);
1121 ObjectReadOperation op;
1122 op.read(1, 0, &bl, NULL);
1123 librados::AioCompletion *completion = cluster.aio_create_completion();
1124 ASSERT_EQ(0, ioctx.aio_operate(
1125 "bar", completion, &op,
1126 librados::OPERATION_IGNORE_CACHE, NULL));
1127 completion->wait_for_safe();
1128 ASSERT_EQ(0, completion->get_return_value());
1129 completion->release();
1132 ObjectReadOperation op;
1134 librados::AioCompletion *completion = cluster.aio_create_completion();
1135 ASSERT_EQ(0, ioctx.aio_operate(
1136 "bar", completion, &op,
1137 librados::OPERATION_IGNORE_CACHE, NULL));
1138 completion->wait_for_safe();
1139 ASSERT_EQ(0, completion->get_return_value());
1140 completion->release();
1144 ioctx.selfmanaged_snap_remove(my_snaps[0]);
1147 // this test case reproduces http://tracker.ceph.com/issues/8629
1148 TEST_F(LibRadosTwoPoolsPP, EvictSnap2) {
1152 bl.append("hi there");
1153 ObjectWriteOperation op;
1155 ASSERT_EQ(0, ioctx.operate("foo", &op));
1157 // create a snapshot, clone
1158 vector<uint64_t> my_snaps(1);
1159 ASSERT_EQ(0, ioctx.selfmanaged_snap_create(&my_snaps[0]));
1160 ASSERT_EQ(0, ioctx.selfmanaged_snap_set_write_ctx(my_snaps[0],
1165 ObjectWriteOperation op;
1167 ASSERT_EQ(0, ioctx.operate("foo", &op));
1171 ASSERT_EQ(0, cluster.mon_command(
1172 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
1173 "\", \"tierpool\": \"" + cache_pool_name +
1174 "\", \"force_nonempty\": \"--force-nonempty\" }",
1176 ASSERT_EQ(0, cluster.mon_command(
1177 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
1178 "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
1180 ASSERT_EQ(0, cluster.mon_command(
1181 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
1182 "\", \"mode\": \"writeback\"}",
1185 // wait for maps to settle
1186 cluster.wait_for_latest_osdmap();
1188 // read, trigger a promote on the head
1191 ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
1192 ASSERT_EQ('c', bl[0]);
1197 ObjectReadOperation op;
1199 librados::AioCompletion *completion = cluster.aio_create_completion();
1200 ASSERT_EQ(0, cache_ioctx.aio_operate(
1201 "foo", completion, &op,
1202 librados::OPERATION_IGNORE_CACHE, NULL));
1203 completion->wait_for_safe();
1204 ASSERT_EQ(0, completion->get_return_value());
1205 completion->release();
1208 // verify the snapdir is not present in the cache pool
1210 ObjectReadOperation op;
1211 librados::snap_set_t snapset;
1212 op.list_snaps(&snapset, NULL);
1213 ioctx.snap_set_read(librados::SNAP_DIR);
1214 librados::AioCompletion *completion = cluster.aio_create_completion();
1215 ASSERT_EQ(0, ioctx.aio_operate("foo", completion, &op,
1216 librados::OPERATION_IGNORE_CACHE, NULL));
1217 completion->wait_for_safe();
1218 ASSERT_EQ(-ENOENT, completion->get_return_value());
1219 completion->release();
1223 TEST_F(LibRadosTwoPoolsPP, TryFlush) {
1226 ASSERT_EQ(0, cluster.mon_command(
1227 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
1228 "\", \"tierpool\": \"" + cache_pool_name +
1229 "\", \"force_nonempty\": \"--force-nonempty\" }",
1231 ASSERT_EQ(0, cluster.mon_command(
1232 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
1233 "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
1235 ASSERT_EQ(0, cluster.mon_command(
1236 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
1237 "\", \"mode\": \"writeback\"}",
1240 // wait for maps to settle
1241 cluster.wait_for_latest_osdmap();
1246 bl.append("hi there");
1247 ObjectWriteOperation op;
1249 ASSERT_EQ(0, ioctx.operate("foo", &op));
1252 // verify the object is present in the cache tier
1254 NObjectIterator it = cache_ioctx.nobjects_begin();
1255 ASSERT_TRUE(it != cache_ioctx.nobjects_end());
1256 ASSERT_TRUE(it->get_oid() == string("foo"));
1258 ASSERT_TRUE(it == cache_ioctx.nobjects_end());
1261 // verify the object is NOT present in the base tier
1263 NObjectIterator it = ioctx.nobjects_begin();
1264 ASSERT_TRUE(it == ioctx.nobjects_end());
1271 ObjectReadOperation op;
1272 op.is_dirty(&dirty, &r);
1273 ASSERT_EQ(0, cache_ioctx.operate("foo", &op, NULL));
1280 ObjectWriteOperation op;
1282 librados::AioCompletion *completion = cluster.aio_create_completion();
1283 ASSERT_EQ(0, cache_ioctx.aio_operate("foo", completion, &op));
1284 completion->wait_for_safe();
1285 ASSERT_EQ(0, completion->get_return_value());
1286 completion->release();
1289 // flush the pinned object with -EPERM
1291 ObjectReadOperation op;
1292 op.cache_try_flush();
1293 librados::AioCompletion *completion = cluster.aio_create_completion();
1294 ASSERT_EQ(0, cache_ioctx.aio_operate(
1295 "foo", completion, &op,
1296 librados::OPERATION_IGNORE_OVERLAY |
1297 librados::OPERATION_SKIPRWLOCKS, NULL));
1298 completion->wait_for_safe();
1299 ASSERT_EQ(-EPERM, completion->get_return_value());
1300 completion->release();
1305 ObjectWriteOperation op;
1307 librados::AioCompletion *completion = cluster.aio_create_completion();
1308 ASSERT_EQ(0, cache_ioctx.aio_operate("foo", completion, &op));
1309 completion->wait_for_safe();
1310 ASSERT_EQ(0, completion->get_return_value());
1311 completion->release();
1316 ObjectReadOperation op;
1317 op.cache_try_flush();
1318 librados::AioCompletion *completion = cluster.aio_create_completion();
1319 ASSERT_EQ(0, cache_ioctx.aio_operate(
1320 "foo", completion, &op,
1321 librados::OPERATION_IGNORE_OVERLAY |
1322 librados::OPERATION_SKIPRWLOCKS, NULL));
1323 completion->wait_for_safe();
1324 ASSERT_EQ(0, completion->get_return_value());
1325 completion->release();
1332 ObjectReadOperation op;
1333 op.is_dirty(&dirty, &r);
1334 ASSERT_EQ(0, cache_ioctx.operate("foo", &op, NULL));
1335 ASSERT_FALSE(dirty);
1339 // verify in base tier
1341 NObjectIterator it = ioctx.nobjects_begin();
1342 ASSERT_TRUE(it != ioctx.nobjects_end());
1343 ASSERT_TRUE(it->get_oid() == string("foo"));
1345 ASSERT_TRUE(it == ioctx.nobjects_end());
1350 ObjectReadOperation op;
1352 librados::AioCompletion *completion = cluster.aio_create_completion();
1353 ASSERT_EQ(0, cache_ioctx.aio_operate(
1354 "foo", completion, &op, librados::OPERATION_IGNORE_CACHE, NULL));
1355 completion->wait_for_safe();
1356 ASSERT_EQ(0, completion->get_return_value());
1357 completion->release();
1360 // verify no longer in cache tier
1362 NObjectIterator it = cache_ioctx.nobjects_begin();
1363 ASSERT_TRUE(it == cache_ioctx.nobjects_end());
1367 TEST_F(LibRadosTwoPoolsPP, Flush) {
1370 ASSERT_EQ(0, cluster.mon_command(
1371 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
1372 "\", \"tierpool\": \"" + cache_pool_name +
1373 "\", \"force_nonempty\": \"--force-nonempty\" }",
1375 ASSERT_EQ(0, cluster.mon_command(
1376 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
1377 "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
1379 ASSERT_EQ(0, cluster.mon_command(
1380 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
1381 "\", \"mode\": \"writeback\"}",
1384 // wait for maps to settle
1385 cluster.wait_for_latest_osdmap();
1387 uint64_t user_version = 0;
1392 bl.append("hi there");
1393 ObjectWriteOperation op;
1395 ASSERT_EQ(0, ioctx.operate("foo", &op));
1398 // verify the object is present in the cache tier
1400 NObjectIterator it = cache_ioctx.nobjects_begin();
1401 ASSERT_TRUE(it != cache_ioctx.nobjects_end());
1402 ASSERT_TRUE(it->get_oid() == string("foo"));
1404 ASSERT_TRUE(it == cache_ioctx.nobjects_end());
1407 // verify the object is NOT present in the base tier
1409 NObjectIterator it = ioctx.nobjects_begin();
1410 ASSERT_TRUE(it == ioctx.nobjects_end());
1417 ObjectReadOperation op;
1418 op.is_dirty(&dirty, &r);
1419 ASSERT_EQ(0, cache_ioctx.operate("foo", &op, NULL));
1422 user_version = cache_ioctx.get_last_version();
1427 ObjectWriteOperation op;
1429 librados::AioCompletion *completion = cluster.aio_create_completion();
1430 ASSERT_EQ(0, cache_ioctx.aio_operate("foo", completion, &op));
1431 completion->wait_for_safe();
1432 ASSERT_EQ(0, completion->get_return_value());
1433 completion->release();
1436 // flush the pinned object with -EPERM
1438 ObjectReadOperation op;
1439 op.cache_try_flush();
1440 librados::AioCompletion *completion = cluster.aio_create_completion();
1441 ASSERT_EQ(0, cache_ioctx.aio_operate(
1442 "foo", completion, &op,
1443 librados::OPERATION_IGNORE_OVERLAY |
1444 librados::OPERATION_SKIPRWLOCKS, NULL));
1445 completion->wait_for_safe();
1446 ASSERT_EQ(-EPERM, completion->get_return_value());
1447 completion->release();
1452 ObjectWriteOperation op;
1454 librados::AioCompletion *completion = cluster.aio_create_completion();
1455 ASSERT_EQ(0, cache_ioctx.aio_operate("foo", completion, &op));
1456 completion->wait_for_safe();
1457 ASSERT_EQ(0, completion->get_return_value());
1458 completion->release();
1463 ObjectReadOperation op;
1465 librados::AioCompletion *completion = cluster.aio_create_completion();
1466 ASSERT_EQ(0, cache_ioctx.aio_operate(
1467 "foo", completion, &op,
1468 librados::OPERATION_IGNORE_OVERLAY, NULL));
1469 completion->wait_for_safe();
1470 ASSERT_EQ(0, completion->get_return_value());
1471 completion->release();
1478 ObjectReadOperation op;
1479 op.is_dirty(&dirty, &r);
1480 ASSERT_EQ(0, cache_ioctx.operate("foo", &op, NULL));
1481 ASSERT_FALSE(dirty);
1485 // verify in base tier
1487 NObjectIterator it = ioctx.nobjects_begin();
1488 ASSERT_TRUE(it != ioctx.nobjects_end());
1489 ASSERT_TRUE(it->get_oid() == string("foo"));
1491 ASSERT_TRUE(it == ioctx.nobjects_end());
1496 ObjectReadOperation op;
1498 librados::AioCompletion *completion = cluster.aio_create_completion();
1499 ASSERT_EQ(0, cache_ioctx.aio_operate(
1500 "foo", completion, &op, librados::OPERATION_IGNORE_CACHE, NULL));
1501 completion->wait_for_safe();
1502 ASSERT_EQ(0, completion->get_return_value());
1503 completion->release();
1506 // verify no longer in cache tier
1508 NObjectIterator it = cache_ioctx.nobjects_begin();
1509 ASSERT_TRUE(it == cache_ioctx.nobjects_end());
1512 // read it again and verify the version is consistent
1515 ASSERT_EQ(1, cache_ioctx.read("foo", bl, 1, 0));
1516 ASSERT_EQ(user_version, cache_ioctx.get_last_version());
1521 ObjectWriteOperation op;
1523 ASSERT_EQ(0, ioctx.operate("foo", &op));
1528 ObjectReadOperation op;
1530 librados::AioCompletion *completion = cluster.aio_create_completion();
1531 ASSERT_EQ(0, cache_ioctx.aio_operate(
1532 "foo", completion, &op,
1533 librados::OPERATION_IGNORE_OVERLAY, NULL));
1534 completion->wait_for_safe();
1535 ASSERT_EQ(0, completion->get_return_value());
1536 completion->release();
1541 ObjectReadOperation op;
1543 librados::AioCompletion *completion = cluster.aio_create_completion();
1544 ASSERT_EQ(0, cache_ioctx.aio_operate(
1545 "foo", completion, &op, librados::OPERATION_IGNORE_CACHE, NULL));
1546 completion->wait_for_safe();
1547 ASSERT_EQ(0, completion->get_return_value());
1548 completion->release();
1551 // verify no longer in cache tier
1553 NObjectIterator it = cache_ioctx.nobjects_begin();
1554 ASSERT_TRUE(it == cache_ioctx.nobjects_end());
1558 NObjectIterator it = ioctx.nobjects_begin();
1559 ASSERT_TRUE(it == ioctx.nobjects_end());
1563 TEST_F(LibRadosTwoPoolsPP, FlushSnap) {
1566 ASSERT_EQ(0, cluster.mon_command(
1567 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
1568 "\", \"tierpool\": \"" + cache_pool_name +
1569 "\", \"force_nonempty\": \"--force-nonempty\" }",
1571 ASSERT_EQ(0, cluster.mon_command(
1572 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
1573 "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
1575 ASSERT_EQ(0, cluster.mon_command(
1576 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
1577 "\", \"mode\": \"writeback\"}",
1580 // wait for maps to settle
1581 cluster.wait_for_latest_osdmap();
1587 ObjectWriteOperation op;
1589 ASSERT_EQ(0, ioctx.operate("foo", &op));
1592 // create a snapshot, clone
1593 vector<uint64_t> my_snaps(1);
1594 ASSERT_EQ(0, ioctx.selfmanaged_snap_create(&my_snaps[0]));
1595 ASSERT_EQ(0, ioctx.selfmanaged_snap_set_write_ctx(my_snaps[0],
1600 ObjectWriteOperation op;
1602 ASSERT_EQ(0, ioctx.operate("foo", &op));
1607 my_snaps[1] = my_snaps[0];
1608 ASSERT_EQ(0, ioctx.selfmanaged_snap_create(&my_snaps[0]));
1609 ASSERT_EQ(0, ioctx.selfmanaged_snap_set_write_ctx(my_snaps[0],
1614 ObjectWriteOperation op;
1616 ASSERT_EQ(0, ioctx.operate("foo", &op));
1619 // verify the object is present in the cache tier
1621 NObjectIterator it = cache_ioctx.nobjects_begin();
1622 ASSERT_TRUE(it != cache_ioctx.nobjects_end());
1623 ASSERT_TRUE(it->get_oid() == string("foo"));
1625 ASSERT_TRUE(it == cache_ioctx.nobjects_end());
1628 // verify the object is NOT present in the base tier
1630 NObjectIterator it = ioctx.nobjects_begin();
1631 ASSERT_TRUE(it == ioctx.nobjects_end());
1634 // flush on head (should fail)
1635 ioctx.snap_set_read(librados::SNAP_HEAD);
1637 ObjectReadOperation op;
1639 librados::AioCompletion *completion = cluster.aio_create_completion();
1640 ASSERT_EQ(0, ioctx.aio_operate(
1641 "foo", completion, &op,
1642 librados::OPERATION_IGNORE_CACHE, NULL));
1643 completion->wait_for_safe();
1644 ASSERT_EQ(-EBUSY, completion->get_return_value());
1645 completion->release();
1647 // flush on recent snap (should fail)
1648 ioctx.snap_set_read(my_snaps[0]);
1650 ObjectReadOperation op;
1652 librados::AioCompletion *completion = cluster.aio_create_completion();
1653 ASSERT_EQ(0, ioctx.aio_operate(
1654 "foo", completion, &op,
1655 librados::OPERATION_IGNORE_CACHE, NULL));
1656 completion->wait_for_safe();
1657 ASSERT_EQ(-EBUSY, completion->get_return_value());
1658 completion->release();
1660 // flush on oldest snap
1661 ioctx.snap_set_read(my_snaps[1]);
1663 ObjectReadOperation op;
1665 librados::AioCompletion *completion = cluster.aio_create_completion();
1666 ASSERT_EQ(0, ioctx.aio_operate(
1667 "foo", completion, &op,
1668 librados::OPERATION_IGNORE_CACHE, NULL));
1669 completion->wait_for_safe();
1670 ASSERT_EQ(0, completion->get_return_value());
1671 completion->release();
1673 // flush on next oldest snap
1674 ioctx.snap_set_read(my_snaps[0]);
1676 ObjectReadOperation op;
1678 librados::AioCompletion *completion = cluster.aio_create_completion();
1679 ASSERT_EQ(0, ioctx.aio_operate(
1680 "foo", completion, &op,
1681 librados::OPERATION_IGNORE_CACHE, NULL));
1682 completion->wait_for_safe();
1683 ASSERT_EQ(0, completion->get_return_value());
1684 completion->release();
1687 ioctx.snap_set_read(librados::SNAP_HEAD);
1689 ObjectReadOperation op;
1691 librados::AioCompletion *completion = cluster.aio_create_completion();
1692 ASSERT_EQ(0, ioctx.aio_operate(
1693 "foo", completion, &op,
1694 librados::OPERATION_IGNORE_CACHE, NULL));
1695 completion->wait_for_safe();
1696 ASSERT_EQ(0, completion->get_return_value());
1697 completion->release();
1700 // verify i can read the snaps from the cache pool
1701 ioctx.snap_set_read(librados::SNAP_HEAD);
1704 ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
1705 ASSERT_EQ('c', bl[0]);
1707 ioctx.snap_set_read(my_snaps[0]);
1710 ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
1711 ASSERT_EQ('b', bl[0]);
1713 ioctx.snap_set_read(my_snaps[1]);
1716 ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
1717 ASSERT_EQ('a', bl[0]);
1721 ASSERT_EQ(0, cluster.mon_command(
1722 "{\"prefix\": \"osd tier remove-overlay\", \"pool\": \"" + pool_name +
1726 // wait for maps to settle
1727 cluster.wait_for_latest_osdmap();
1729 // verify i can read the snaps from the base pool
1730 ioctx.snap_set_read(librados::SNAP_HEAD);
1733 ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
1734 ASSERT_EQ('c', bl[0]);
1736 ioctx.snap_set_read(my_snaps[0]);
1739 ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
1740 ASSERT_EQ('b', bl[0]);
1742 ioctx.snap_set_read(my_snaps[1]);
1745 ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
1746 ASSERT_EQ('a', bl[0]);
1749 ASSERT_EQ(0, cluster.mon_command(
1750 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
1751 "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
1755 ioctx.selfmanaged_snap_remove(my_snaps[0]);
1758 TEST_F(LibRadosTierPP, FlushWriteRaces) {
1760 std::string pool_name = get_temp_pool_name();
1761 std::string cache_pool_name = pool_name + "-cache";
1762 ASSERT_EQ("", create_one_pool_pp(pool_name, cluster));
1763 ASSERT_EQ(0, cluster.pool_create(cache_pool_name.c_str()));
1765 ASSERT_EQ(0, cluster.ioctx_create(cache_pool_name.c_str(), cache_ioctx));
1766 cache_ioctx.application_enable("rados", true);
1768 ASSERT_EQ(0, cluster.ioctx_create(pool_name.c_str(), ioctx));
1772 ASSERT_EQ(0, cluster.mon_command(
1773 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
1774 "\", \"tierpool\": \"" + cache_pool_name + "\"}",
1776 ASSERT_EQ(0, cluster.mon_command(
1777 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
1778 "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
1780 ASSERT_EQ(0, cluster.mon_command(
1781 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
1782 "\", \"mode\": \"writeback\"}",
1785 // wait for maps to settle
1786 cluster.wait_for_latest_osdmap();
1788 // create/dirty object
1790 bl.append("hi there");
1792 ObjectWriteOperation op;
1794 ASSERT_EQ(0, ioctx.operate("foo", &op));
1799 ObjectReadOperation op;
1801 librados::AioCompletion *completion = cluster.aio_create_completion();
1802 ASSERT_EQ(0, cache_ioctx.aio_operate(
1803 "foo", completion, &op,
1804 librados::OPERATION_IGNORE_OVERLAY, NULL));
1806 ObjectWriteOperation op2;
1808 librados::AioCompletion *completion2 = cluster.aio_create_completion();
1809 ASSERT_EQ(0, ioctx.aio_operate(
1810 "foo", completion2, &op2, 0));
1812 completion->wait_for_safe();
1813 completion2->wait_for_safe();
1814 ASSERT_EQ(0, completion->get_return_value());
1815 ASSERT_EQ(0, completion2->get_return_value());
1816 completion->release();
1817 completion2->release();
1822 // create/dirty object
1825 bl.append("hi there");
1826 ObjectWriteOperation op;
1828 ASSERT_EQ(0, ioctx.operate("foo", &op));
1831 // try-flush + write
1833 ObjectReadOperation op;
1834 op.cache_try_flush();
1835 librados::AioCompletion *completion = cluster.aio_create_completion();
1836 ASSERT_EQ(0, cache_ioctx.aio_operate(
1837 "foo", completion, &op,
1838 librados::OPERATION_IGNORE_OVERLAY |
1839 librados::OPERATION_SKIPRWLOCKS, NULL));
1841 ObjectWriteOperation op2;
1843 librados::AioCompletion *completion2 = cluster.aio_create_completion();
1844 ASSERT_EQ(0, ioctx.aio_operate("foo", completion2, &op2, 0));
1846 completion->wait_for_safe();
1847 completion2->wait_for_safe();
1848 int r = completion->get_return_value();
1849 ASSERT_TRUE(r == -EBUSY || r == 0);
1850 ASSERT_EQ(0, completion2->get_return_value());
1851 completion->release();
1852 completion2->release();
1855 cout << "didn't get EBUSY, trying again" << std::endl;
1857 ASSERT_TRUE(--tries);
1861 ASSERT_EQ(0, cluster.mon_command(
1862 "{\"prefix\": \"osd tier remove-overlay\", \"pool\": \"" + pool_name +
1865 ASSERT_EQ(0, cluster.mon_command(
1866 "{\"prefix\": \"osd tier remove\", \"pool\": \"" + pool_name +
1867 "\", \"tierpool\": \"" + cache_pool_name + "\"}",
1870 // wait for maps to settle before next test
1871 cluster.wait_for_latest_osdmap();
1873 ASSERT_EQ(0, cluster.pool_delete(cache_pool_name.c_str()));
1874 ASSERT_EQ(0, destroy_one_pool_pp(pool_name, cluster));
1877 TEST_F(LibRadosTwoPoolsPP, FlushTryFlushRaces) {
1880 ASSERT_EQ(0, cluster.mon_command(
1881 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
1882 "\", \"tierpool\": \"" + cache_pool_name +
1883 "\", \"force_nonempty\": \"--force-nonempty\" }",
1885 ASSERT_EQ(0, cluster.mon_command(
1886 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
1887 "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
1889 ASSERT_EQ(0, cluster.mon_command(
1890 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
1891 "\", \"mode\": \"writeback\"}",
1894 // wait for maps to settle
1895 cluster.wait_for_latest_osdmap();
1897 // create/dirty object
1900 bl.append("hi there");
1901 ObjectWriteOperation op;
1903 ASSERT_EQ(0, ioctx.operate("foo", &op));
1908 ObjectReadOperation op;
1910 librados::AioCompletion *completion = cluster.aio_create_completion();
1911 ASSERT_EQ(0, cache_ioctx.aio_operate(
1912 "foo", completion, &op,
1913 librados::OPERATION_IGNORE_OVERLAY, NULL));
1915 ObjectReadOperation op2;
1917 librados::AioCompletion *completion2 = cluster.aio_create_completion();
1918 ASSERT_EQ(0, cache_ioctx.aio_operate(
1919 "foo", completion2, &op2,
1920 librados::OPERATION_IGNORE_OVERLAY, NULL));
1922 completion->wait_for_safe();
1923 completion2->wait_for_safe();
1924 ASSERT_EQ(0, completion->get_return_value());
1925 ASSERT_EQ(0, completion2->get_return_value());
1926 completion->release();
1927 completion2->release();
1930 // create/dirty object
1933 bl.append("hi there");
1934 ObjectWriteOperation op;
1936 ASSERT_EQ(0, ioctx.operate("foo", &op));
1939 // flush + try-flush
1941 ObjectReadOperation op;
1943 librados::AioCompletion *completion = cluster.aio_create_completion();
1944 ASSERT_EQ(0, cache_ioctx.aio_operate(
1945 "foo", completion, &op,
1946 librados::OPERATION_IGNORE_OVERLAY, NULL));
1948 ObjectReadOperation op2;
1949 op2.cache_try_flush();
1950 librados::AioCompletion *completion2 = cluster.aio_create_completion();
1951 ASSERT_EQ(0, cache_ioctx.aio_operate(
1952 "foo", completion2, &op2,
1953 librados::OPERATION_IGNORE_OVERLAY |
1954 librados::OPERATION_SKIPRWLOCKS, NULL));
1956 completion->wait_for_safe();
1957 completion2->wait_for_safe();
1958 ASSERT_EQ(0, completion->get_return_value());
1959 ASSERT_EQ(0, completion2->get_return_value());
1960 completion->release();
1961 completion2->release();
1964 // create/dirty object
1969 bl.append("hi there");
1970 ObjectWriteOperation op;
1972 ASSERT_EQ(0, ioctx.operate("foo", &op));
1975 // try-flush + flush
1976 // (flush will not piggyback on try-flush)
1978 ObjectReadOperation op;
1979 op.cache_try_flush();
1980 librados::AioCompletion *completion = cluster.aio_create_completion();
1981 ASSERT_EQ(0, cache_ioctx.aio_operate(
1982 "foo", completion, &op,
1983 librados::OPERATION_IGNORE_OVERLAY |
1984 librados::OPERATION_SKIPRWLOCKS, NULL));
1986 ObjectReadOperation op2;
1988 librados::AioCompletion *completion2 = cluster.aio_create_completion();
1989 ASSERT_EQ(0, cache_ioctx.aio_operate(
1990 "foo", completion2, &op2,
1991 librados::OPERATION_IGNORE_OVERLAY, NULL));
1993 completion->wait_for_safe();
1994 completion2->wait_for_safe();
1995 int r = completion->get_return_value();
1996 ASSERT_TRUE(r == -EBUSY || r == 0);
1997 ASSERT_EQ(0, completion2->get_return_value());
1998 completion->release();
1999 completion2->release();
2002 cout << "didn't get EBUSY, trying again" << std::endl;
2004 ASSERT_TRUE(--tries);
2007 // create/dirty object
2010 bl.append("hi there");
2011 ObjectWriteOperation op;
2013 ASSERT_EQ(0, ioctx.operate("foo", &op));
2016 // try-flush + try-flush
2018 ObjectReadOperation op;
2019 op.cache_try_flush();
2020 librados::AioCompletion *completion = cluster.aio_create_completion();
2021 ASSERT_EQ(0, cache_ioctx.aio_operate(
2022 "foo", completion, &op,
2023 librados::OPERATION_IGNORE_OVERLAY |
2024 librados::OPERATION_SKIPRWLOCKS, NULL));
2026 ObjectReadOperation op2;
2027 op2.cache_try_flush();
2028 librados::AioCompletion *completion2 = cluster.aio_create_completion();
2029 ASSERT_EQ(0, cache_ioctx.aio_operate(
2030 "foo", completion2, &op2,
2031 librados::OPERATION_IGNORE_OVERLAY |
2032 librados::OPERATION_SKIPRWLOCKS, NULL));
2034 completion->wait_for_safe();
2035 completion2->wait_for_safe();
2036 ASSERT_EQ(0, completion->get_return_value());
2037 ASSERT_EQ(0, completion2->get_return_value());
2038 completion->release();
2039 completion2->release();
2044 IoCtx *read_ioctx = 0;
2045 Mutex test_lock("FlushReadRaces::lock");
2047 int max_reads = 100;
2048 int num_reads = 0; // in progress
2050 void flush_read_race_cb(completion_t cb, void *arg);
2052 void start_flush_read()
2054 //cout << " starting read" << std::endl;
2055 ObjectReadOperation op;
2056 op.stat(NULL, NULL, NULL);
2057 librados::AioCompletion *completion = completions.getCompletion();
2058 completion->set_complete_callback(0, flush_read_race_cb);
2059 read_ioctx->aio_operate("foo", completion, &op, NULL);
2062 void flush_read_race_cb(completion_t cb, void *arg)
2064 //cout << " finished read" << std::endl;
2066 if (num_reads > max_reads) {
2075 TEST_F(LibRadosTwoPoolsPP, TryFlushReadRace) {
2078 ASSERT_EQ(0, cluster.mon_command(
2079 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
2080 "\", \"tierpool\": \"" + cache_pool_name +
2081 "\", \"force_nonempty\": \"--force-nonempty\" }",
2083 ASSERT_EQ(0, cluster.mon_command(
2084 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
2085 "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
2087 ASSERT_EQ(0, cluster.mon_command(
2088 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
2089 "\", \"mode\": \"writeback\"}",
2092 // wait for maps to settle
2093 cluster.wait_for_latest_osdmap();
2095 // create/dirty object
2098 bl.append("hi there");
2099 bufferptr bp(4000000); // make it big!
2102 ObjectWriteOperation op;
2104 ASSERT_EQ(0, ioctx.operate("foo", &op));
2107 // start a continuous stream of reads
2108 read_ioctx = &ioctx;
2110 for (int i = 0; i < max_reads; ++i) {
2117 ObjectReadOperation op;
2118 op.cache_try_flush();
2119 librados::AioCompletion *completion = cluster.aio_create_completion();
2120 ASSERT_EQ(0, cache_ioctx.aio_operate(
2121 "foo", completion, &op,
2122 librados::OPERATION_IGNORE_OVERLAY |
2123 librados::OPERATION_SKIPRWLOCKS, NULL));
2125 completion->wait_for_safe();
2126 ASSERT_EQ(0, completion->get_return_value());
2127 completion->release();
2132 while (num_reads > 0)
2133 cond.Wait(test_lock);
2137 TEST_F(LibRadosTierPP, HitSetNone) {
2139 list< pair<time_t,time_t> > ls;
2140 AioCompletion *c = librados::Rados::aio_create_completion();
2141 ASSERT_EQ(0, ioctx.hit_set_list(123, c, &ls));
2142 c->wait_for_complete();
2143 ASSERT_EQ(0, c->get_return_value());
2144 ASSERT_TRUE(ls.empty());
2149 AioCompletion *c = librados::Rados::aio_create_completion();
2150 ASSERT_EQ(0, ioctx.hit_set_get(123, c, 12345, &bl));
2151 c->wait_for_complete();
2152 ASSERT_EQ(-ENOENT, c->get_return_value());
2157 string set_pool_str(string pool, string var, string val)
2159 return string("{\"prefix\": \"osd pool set\",\"pool\":\"") + pool
2160 + string("\",\"var\": \"") + var + string("\",\"val\": \"")
2161 + val + string("\"}");
2164 string set_pool_str(string pool, string var, int val)
2166 return string("{\"prefix\": \"osd pool set\",\"pool\":\"") + pool
2167 + string("\",\"var\": \"") + var + string("\",\"val\": \"")
2168 + stringify(val) + string("\"}");
2171 TEST_F(LibRadosTwoPoolsPP, HitSetRead) {
2174 ASSERT_EQ(0, cluster.mon_command(
2175 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
2176 "\", \"tierpool\": \"" + cache_pool_name +
2177 "\", \"force_nonempty\": \"--force-nonempty\" }",
2180 // enable hitset tracking for this pool
2181 ASSERT_EQ(0, cluster.mon_command(set_pool_str(cache_pool_name, "hit_set_count", 2),
2183 ASSERT_EQ(0, cluster.mon_command(set_pool_str(cache_pool_name, "hit_set_period", 600),
2185 ASSERT_EQ(0, cluster.mon_command(set_pool_str(cache_pool_name, "hit_set_type",
2189 // wait for maps to settle
2190 cluster.wait_for_latest_osdmap();
2192 cache_ioctx.set_namespace("");
2194 // keep reading until we see our object appear in the HitSet
2195 utime_t start = ceph_clock_now();
2196 utime_t hard_stop = start + utime_t(600, 0);
2199 utime_t now = ceph_clock_now();
2200 ASSERT_TRUE(now < hard_stop);
2202 string name = "foo";
2204 ASSERT_EQ(0, cache_ioctx.get_object_hash_position2(name, &hash));
2205 hobject_t oid(sobject_t(name, CEPH_NOSNAP), "", hash,
2206 cluster.pool_lookup(cache_pool_name.c_str()), "");
2209 ASSERT_EQ(-ENOENT, cache_ioctx.read("foo", bl, 1, 0));
2212 AioCompletion *c = librados::Rados::aio_create_completion();
2213 ASSERT_EQ(0, cache_ioctx.hit_set_get(hash, c, now.sec(), &hbl));
2214 c->wait_for_complete();
2218 bufferlist::iterator p = hbl.begin();
2221 if (hs.contains(oid)) {
2222 cout << "ok, hit_set contains " << oid << std::endl;
2225 cout << "hmm, not in HitSet yet" << std::endl;
2227 cout << "hmm, no HitSet yet" << std::endl;
2234 static int _get_pg_num(Rados& cluster, string pool_name)
2237 string cmd = string("{\"prefix\": \"osd pool get\",\"pool\":\"")
2239 + string("\",\"var\": \"pg_num\",\"format\": \"json\"}");
2241 int r = cluster.mon_command(cmd, inbl, &outbl, NULL);
2243 string outstr(outbl.c_str(), outbl.length());
2244 json_spirit::Value v;
2245 if (!json_spirit::read(outstr, v)) {
2246 cerr <<" unable to parse json " << outstr << std::endl;
2250 json_spirit::Object& o = v.get_obj();
2251 for (json_spirit::Object::size_type i=0; i<o.size(); i++) {
2252 json_spirit::Pair& p = o[i];
2253 if (p.name_ == "pg_num") {
2254 cout << "pg_num = " << p.value_.get_int() << std::endl;
2255 return p.value_.get_int();
2258 cerr << "didn't find pg_num in " << outstr << std::endl;
2263 TEST_F(LibRadosTwoPoolsPP, HitSetWrite) {
2264 int num_pg = _get_pg_num(cluster, pool_name);
2269 ASSERT_EQ(0, cluster.mon_command(
2270 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
2271 "\", \"tierpool\": \"" + cache_pool_name +
2272 "\", \"force_nonempty\": \"--force-nonempty\" }",
2275 // enable hitset tracking for this pool
2276 ASSERT_EQ(0, cluster.mon_command(set_pool_str(cache_pool_name, "hit_set_count", 8),
2278 ASSERT_EQ(0, cluster.mon_command(set_pool_str(cache_pool_name, "hit_set_period", 600),
2280 ASSERT_EQ(0, cluster.mon_command(set_pool_str(cache_pool_name, "hit_set_type",
2284 // wait for maps to settle
2285 cluster.wait_for_latest_osdmap();
2287 cache_ioctx.set_namespace("");
2291 // do a bunch of writes
2292 for (int i=0; i<num; ++i) {
2295 ASSERT_EQ(0, cache_ioctx.write(stringify(i), bl, 1, 0));
2299 std::map<int,HitSet> hitsets;
2300 for (int i=0; i<num_pg; ++i) {
2301 list< pair<time_t,time_t> > ls;
2302 AioCompletion *c = librados::Rados::aio_create_completion();
2303 ASSERT_EQ(0, cache_ioctx.hit_set_list(i, c, &ls));
2304 c->wait_for_complete();
2306 std::cout << "pg " << i << " ls " << ls << std::endl;
2307 ASSERT_FALSE(ls.empty());
2310 c = librados::Rados::aio_create_completion();
2312 ASSERT_EQ(0, cache_ioctx.hit_set_get(i, c, ls.back().first, &bl));
2313 c->wait_for_complete();
2317 bufferlist::iterator p = bl.begin();
2318 ::decode(hitsets[i], p);
2320 catch (buffer::error& e) {
2321 std::cout << "failed to decode hit set; bl len is " << bl.length() << "\n";
2322 bl.hexdump(std::cout);
2323 std::cout << std::endl;
2327 // cope with racing splits by refreshing pg_num
2328 if (i == num_pg - 1)
2329 num_pg = _get_pg_num(cluster, cache_pool_name);
2332 for (int i=0; i<num; ++i) {
2333 string n = stringify(i);
2335 ASSERT_EQ(0, cache_ioctx.get_object_hash_position2(n, &hash));
2336 hobject_t oid(sobject_t(n, CEPH_NOSNAP), "", hash,
2337 cluster.pool_lookup(cache_pool_name.c_str()), "");
2338 std::cout << "checking for " << oid << std::endl;
2340 for (int p=0; p<num_pg; ++p) {
2341 if (hitsets[p].contains(oid)) {
2350 TEST_F(LibRadosTwoPoolsPP, HitSetTrim) {
2352 unsigned period = 3;
2356 ASSERT_EQ(0, cluster.mon_command(
2357 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
2358 "\", \"tierpool\": \"" + cache_pool_name +
2359 "\", \"force_nonempty\": \"--force-nonempty\" }",
2362 // enable hitset tracking for this pool
2363 ASSERT_EQ(0, cluster.mon_command(set_pool_str(cache_pool_name, "hit_set_count", count),
2365 ASSERT_EQ(0, cluster.mon_command(set_pool_str(cache_pool_name, "hit_set_period", period),
2367 ASSERT_EQ(0, cluster.mon_command(set_pool_str(cache_pool_name, "hit_set_type", "bloom"),
2369 ASSERT_EQ(0, cluster.mon_command(set_pool_str(cache_pool_name, "hit_set_fpp", ".01"),
2372 // wait for maps to settle
2373 cluster.wait_for_latest_osdmap();
2375 cache_ioctx.set_namespace("");
2377 // do a bunch of writes and make sure the hitsets rotate
2378 utime_t start = ceph_clock_now();
2379 utime_t hard_stop = start + utime_t(count * period * 50, 0);
2383 string name = "foo";
2385 ASSERT_EQ(0, cache_ioctx.get_object_hash_position2(name, &hash));
2386 hobject_t oid(sobject_t(name, CEPH_NOSNAP), "", hash, -1, "");
2390 ASSERT_EQ(0, cache_ioctx.write("foo", bl, 1, 0));
2392 list<pair<time_t, time_t> > ls;
2393 AioCompletion *c = librados::Rados::aio_create_completion();
2394 ASSERT_EQ(0, cache_ioctx.hit_set_list(hash, c, &ls));
2395 c->wait_for_complete();
2398 cout << " got ls " << ls << std::endl;
2401 first = ls.front().first;
2402 cout << "first is " << first << std::endl;
2404 if (ls.front().first != first) {
2405 cout << "first now " << ls.front().first << ", trimmed" << std::endl;
2411 utime_t now = ceph_clock_now();
2412 ASSERT_TRUE(now < hard_stop);
2418 TEST_F(LibRadosTwoPoolsPP, PromoteOn2ndRead) {
2420 for (int i=0; i<20; ++i) {
2422 bl.append("hi there");
2423 ObjectWriteOperation op;
2425 ASSERT_EQ(0, ioctx.operate("foo" + stringify(i), &op));
2430 ASSERT_EQ(0, cluster.mon_command(
2431 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
2432 "\", \"tierpool\": \"" + cache_pool_name +
2433 "\", \"force_nonempty\": \"--force-nonempty\" }",
2435 ASSERT_EQ(0, cluster.mon_command(
2436 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
2437 "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
2439 ASSERT_EQ(0, cluster.mon_command(
2440 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
2441 "\", \"mode\": \"writeback\"}",
2444 // enable hitset tracking for this pool
2445 ASSERT_EQ(0, cluster.mon_command(
2446 set_pool_str(cache_pool_name, "hit_set_count", 2),
2448 ASSERT_EQ(0, cluster.mon_command(
2449 set_pool_str(cache_pool_name, "hit_set_period", 600),
2451 ASSERT_EQ(0, cluster.mon_command(
2452 set_pool_str(cache_pool_name, "hit_set_type", "bloom"),
2454 ASSERT_EQ(0, cluster.mon_command(
2455 set_pool_str(cache_pool_name, "min_read_recency_for_promote", 1),
2457 ASSERT_EQ(0, cluster.mon_command(
2458 set_pool_str(cache_pool_name, "hit_set_grade_decay_rate", 20),
2460 ASSERT_EQ(0, cluster.mon_command(
2461 set_pool_str(cache_pool_name, "hit_set_search_last_n", 1),
2464 // wait for maps to settle
2465 cluster.wait_for_latest_osdmap();
2467 int fake = 0; // set this to non-zero to test spurious promotion,
2468 // e.g. from thrashing
2472 // 1st read, don't trigger a promote
2473 obj = "foo" + stringify(attempt);
2474 cout << obj << std::endl;
2477 ASSERT_EQ(1, ioctx.read(obj.c_str(), bl, 1, 0));
2480 ASSERT_EQ(1, ioctx.read(obj.c_str(), bl, 1, 0));
2485 // verify the object is NOT present in the cache tier
2488 NObjectIterator it = cache_ioctx.nobjects_begin();
2489 while (it != cache_ioctx.nobjects_end()) {
2490 cout << " see " << it->get_oid() << std::endl;
2491 if (it->get_oid() == string(obj.c_str())) {
2502 ASSERT_LE(attempt, 20);
2503 cout << "hrm, object is present in cache on attempt " << attempt
2504 << ", retrying" << std::endl;
2507 // Read until the object is present in the cache tier
2508 cout << "verifying " << obj << " is eventually promoted" << std::endl;
2511 ASSERT_EQ(1, ioctx.read(obj.c_str(), bl, 1, 0));
2514 NObjectIterator it = cache_ioctx.nobjects_begin();
2515 while (it != cache_ioctx.nobjects_end()) {
2516 if (it->get_oid() == string(obj.c_str())) {
2529 ASSERT_EQ(0, cluster.mon_command(
2530 "{\"prefix\": \"osd tier remove-overlay\", \"pool\": \"" + pool_name +
2533 ASSERT_EQ(0, cluster.mon_command(
2534 "{\"prefix\": \"osd tier remove\", \"pool\": \"" + pool_name +
2535 "\", \"tierpool\": \"" + cache_pool_name + "\"}",
2538 // wait for maps to settle before next test
2539 cluster.wait_for_latest_osdmap();
2542 TEST_F(LibRadosTwoPoolsPP, ProxyRead) {
2546 bl.append("hi there");
2547 ObjectWriteOperation op;
2549 ASSERT_EQ(0, ioctx.operate("foo", &op));
2554 ASSERT_EQ(0, cluster.mon_command(
2555 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
2556 "\", \"tierpool\": \"" + cache_pool_name +
2557 "\", \"force_nonempty\": \"--force-nonempty\" }",
2559 ASSERT_EQ(0, cluster.mon_command(
2560 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
2561 "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
2563 ASSERT_EQ(0, cluster.mon_command(
2564 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
2565 "\", \"mode\": \"readproxy\"}",
2568 // wait for maps to settle
2569 cluster.wait_for_latest_osdmap();
2571 // read and verify the object
2574 ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
2575 ASSERT_EQ('h', bl[0]);
2578 // Verify 10 times the object is NOT present in the cache tier
2581 NObjectIterator it = cache_ioctx.nobjects_begin();
2582 ASSERT_TRUE(it == cache_ioctx.nobjects_end());
2587 ASSERT_EQ(0, cluster.mon_command(
2588 "{\"prefix\": \"osd tier remove-overlay\", \"pool\": \"" + pool_name +
2591 ASSERT_EQ(0, cluster.mon_command(
2592 "{\"prefix\": \"osd tier remove\", \"pool\": \"" + pool_name +
2593 "\", \"tierpool\": \"" + cache_pool_name + "\"}",
2596 // wait for maps to settle before next test
2597 cluster.wait_for_latest_osdmap();
2600 TEST_F(LibRadosTwoPoolsPP, CachePin) {
2604 bl.append("hi there");
2605 ObjectWriteOperation op;
2607 ASSERT_EQ(0, ioctx.operate("foo", &op));
2611 bl.append("hi there");
2612 ObjectWriteOperation op;
2614 ASSERT_EQ(0, ioctx.operate("bar", &op));
2618 bl.append("hi there");
2619 ObjectWriteOperation op;
2621 ASSERT_EQ(0, ioctx.operate("baz", &op));
2625 bl.append("hi there");
2626 ObjectWriteOperation op;
2628 ASSERT_EQ(0, ioctx.operate("bam", &op));
2633 ASSERT_EQ(0, cluster.mon_command(
2634 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
2635 "\", \"tierpool\": \"" + cache_pool_name +
2636 "\", \"force_nonempty\": \"--force-nonempty\" }",
2638 ASSERT_EQ(0, cluster.mon_command(
2639 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
2640 "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
2642 ASSERT_EQ(0, cluster.mon_command(
2643 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
2644 "\", \"mode\": \"writeback\"}",
2647 // wait for maps to settle
2648 cluster.wait_for_latest_osdmap();
2650 // read, trigger promote
2653 ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
2654 ASSERT_EQ(1, ioctx.read("bar", bl, 1, 0));
2655 ASSERT_EQ(1, ioctx.read("baz", bl, 1, 0));
2656 ASSERT_EQ(1, ioctx.read("bam", bl, 1, 0));
2659 // verify the objects are present in the cache tier
2661 NObjectIterator it = cache_ioctx.nobjects_begin();
2662 ASSERT_TRUE(it != cache_ioctx.nobjects_end());
2663 for (uint32_t i = 0; i < 4; i++) {
2664 ASSERT_TRUE(it->get_oid() == string("foo") ||
2665 it->get_oid() == string("bar") ||
2666 it->get_oid() == string("baz") ||
2667 it->get_oid() == string("bam"));
2670 ASSERT_TRUE(it == cache_ioctx.nobjects_end());
2675 ObjectWriteOperation op;
2677 librados::AioCompletion *completion = cluster.aio_create_completion();
2678 ASSERT_EQ(0, cache_ioctx.aio_operate("foo", completion, &op));
2679 completion->wait_for_safe();
2680 ASSERT_EQ(0, completion->get_return_value());
2681 completion->release();
2684 ObjectWriteOperation op;
2686 librados::AioCompletion *completion = cluster.aio_create_completion();
2687 ASSERT_EQ(0, cache_ioctx.aio_operate("baz", completion, &op));
2688 completion->wait_for_safe();
2689 ASSERT_EQ(0, completion->get_return_value());
2690 completion->release();
2694 ASSERT_EQ(0, cluster.mon_command(
2695 set_pool_str(cache_pool_name, "hit_set_count", 2),
2697 ASSERT_EQ(0, cluster.mon_command(
2698 set_pool_str(cache_pool_name, "hit_set_period", 600),
2700 ASSERT_EQ(0, cluster.mon_command(
2701 set_pool_str(cache_pool_name, "hit_set_type", "bloom"),
2703 ASSERT_EQ(0, cluster.mon_command(
2704 set_pool_str(cache_pool_name, "min_read_recency_for_promote", 1),
2706 ASSERT_EQ(0, cluster.mon_command(
2707 set_pool_str(cache_pool_name, "target_max_objects", 1),
2712 // Verify the pinned object 'foo' is not flushed/evicted
2716 ASSERT_EQ(1, ioctx.read("baz", bl, 1, 0));
2719 NObjectIterator it = cache_ioctx.nobjects_begin();
2720 while (it != cache_ioctx.nobjects_end()) {
2721 ASSERT_TRUE(it->get_oid() == string("foo") ||
2722 it->get_oid() == string("bar") ||
2723 it->get_oid() == string("baz") ||
2724 it->get_oid() == string("bam"));
2729 ASSERT_TRUE(it->get_oid() == string("foo") ||
2730 it->get_oid() == string("baz"));
2738 ASSERT_EQ(0, cluster.mon_command(
2739 "{\"prefix\": \"osd tier remove-overlay\", \"pool\": \"" + pool_name +
2742 ASSERT_EQ(0, cluster.mon_command(
2743 "{\"prefix\": \"osd tier remove\", \"pool\": \"" + pool_name +
2744 "\", \"tierpool\": \"" + cache_pool_name + "\"}",
2747 // wait for maps to settle before next test
2748 cluster.wait_for_latest_osdmap();
2751 TEST_F(LibRadosTwoPoolsPP, SetRedirectRead) {
2752 // skip test if not yet luminous
2754 bufferlist inbl, outbl;
2755 ASSERT_EQ(0, cluster.mon_command(
2756 "{\"prefix\": \"osd dump\"}",
2757 inbl, &outbl, NULL));
2758 string s(outbl.c_str(), outbl.length());
2759 if (s.find("luminous") == std::string::npos) {
2760 cout << "cluster is not yet luminous, skipping test" << std::endl;
2768 bl.append("hi there");
2769 ObjectWriteOperation op;
2771 ASSERT_EQ(0, ioctx.operate("foo", &op));
2776 ObjectWriteOperation op;
2778 ASSERT_EQ(0, cache_ioctx.operate("bar", &op));
2783 ASSERT_EQ(0, cluster.mon_command(
2784 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
2785 "\", \"tierpool\": \"" + cache_pool_name +
2786 "\", \"force_nonempty\": \"--force-nonempty\" }",
2789 // wait for maps to settle
2790 cluster.wait_for_latest_osdmap();
2793 ObjectWriteOperation op;
2794 op.set_redirect("bar", cache_ioctx, 0);
2795 librados::AioCompletion *completion = cluster.aio_create_completion();
2796 ASSERT_EQ(0, ioctx.aio_operate("foo", completion, &op));
2797 completion->wait_for_safe();
2798 ASSERT_EQ(0, completion->get_return_value());
2799 completion->release();
2801 // read and verify the object
2804 ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
2805 ASSERT_EQ('t', bl[0]);
2808 ASSERT_EQ(0, cluster.mon_command(
2809 "{\"prefix\": \"osd tier remove\", \"pool\": \"" + pool_name +
2810 "\", \"tierpool\": \"" + cache_pool_name + "\"}",
2813 // wait for maps to settle before next test
2814 cluster.wait_for_latest_osdmap();
2817 class LibRadosTwoPoolsECPP : public RadosTestECPP
2820 LibRadosTwoPoolsECPP() {};
2821 ~LibRadosTwoPoolsECPP() override {};
2823 static void SetUpTestCase() {
2824 pool_name = get_temp_pool_name();
2825 ASSERT_EQ("", create_one_ec_pool_pp(pool_name, s_cluster));
2827 static void TearDownTestCase() {
2828 ASSERT_EQ(0, destroy_one_ec_pool_pp(pool_name, s_cluster));
2830 static std::string cache_pool_name;
2832 void SetUp() override {
2833 cache_pool_name = get_temp_pool_name();
2834 ASSERT_EQ(0, s_cluster.pool_create(cache_pool_name.c_str()));
2835 RadosTestECPP::SetUp();
2837 ASSERT_EQ(0, cluster.ioctx_create(cache_pool_name.c_str(), cache_ioctx));
2838 cache_ioctx.application_enable("rados", true);
2839 cache_ioctx.set_namespace(nspace);
2841 void TearDown() override {
2842 // flush + evict cache
2843 flush_evict_all(cluster, cache_ioctx);
2847 ASSERT_EQ(0, cluster.mon_command(
2848 "{\"prefix\": \"osd tier remove-overlay\", \"pool\": \"" + pool_name +
2851 ASSERT_EQ(0, cluster.mon_command(
2852 "{\"prefix\": \"osd tier remove\", \"pool\": \"" + pool_name +
2853 "\", \"tierpool\": \"" + cache_pool_name + "\"}",
2856 // wait for maps to settle before next test
2857 cluster.wait_for_latest_osdmap();
2859 RadosTestECPP::TearDown();
2861 cleanup_default_namespace(cache_ioctx);
2862 cleanup_namespace(cache_ioctx, nspace);
2864 cache_ioctx.close();
2865 ASSERT_EQ(0, s_cluster.pool_delete(cache_pool_name.c_str()));
2868 librados::IoCtx cache_ioctx;
2871 std::string LibRadosTwoPoolsECPP::cache_pool_name;
2873 TEST_F(LibRadosTierECPP, Dirty) {
2875 ObjectWriteOperation op;
2877 ASSERT_EQ(0, ioctx.operate("foo", &op)); // still get 0 if it dne
2880 ObjectWriteOperation op;
2882 ASSERT_EQ(0, ioctx.operate("foo", &op));
2887 ObjectReadOperation op;
2888 op.is_dirty(&dirty, &r);
2889 ASSERT_EQ(0, ioctx.operate("foo", &op, NULL));
2894 ObjectWriteOperation op;
2896 ASSERT_EQ(0, ioctx.operate("foo", &op));
2899 ObjectWriteOperation op;
2901 ASSERT_EQ(0, ioctx.operate("foo", &op)); // still 0 if already clean
2906 ObjectReadOperation op;
2907 op.is_dirty(&dirty, &r);
2908 ASSERT_EQ(0, ioctx.operate("foo", &op, NULL));
2909 ASSERT_FALSE(dirty);
2913 // ObjectWriteOperation op;
2914 // op.truncate(0); // still a write even tho it is a no-op
2915 // ASSERT_EQ(0, ioctx.operate("foo", &op));
2918 // bool dirty = false;
2920 // ObjectReadOperation op;
2921 // op.is_dirty(&dirty, &r);
2922 // ASSERT_EQ(0, ioctx.operate("foo", &op, NULL));
2923 // ASSERT_TRUE(dirty);
2928 TEST_F(LibRadosTwoPoolsECPP, Overlay) {
2933 ObjectWriteOperation op;
2935 ASSERT_EQ(0, ioctx.operate("foo", &op));
2940 ObjectWriteOperation op;
2942 ASSERT_EQ(0, cache_ioctx.operate("foo", &op));
2947 ASSERT_EQ(0, cluster.mon_command(
2948 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
2949 "\", \"tierpool\": \"" + cache_pool_name +
2950 "\", \"force_nonempty\": \"--force-nonempty\" }",
2952 ASSERT_EQ(0, cluster.mon_command(
2953 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
2954 "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
2957 // wait for maps to settle
2958 cluster.wait_for_latest_osdmap();
2960 // by default, the overlay sends us to cache pool
2963 ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
2964 ASSERT_EQ('c', bl[0]);
2968 ASSERT_EQ(1, cache_ioctx.read("foo", bl, 1, 0));
2969 ASSERT_EQ('c', bl[0]);
2972 // unless we say otherwise
2975 ObjectReadOperation op;
2976 op.read(0, 1, &bl, NULL);
2977 librados::AioCompletion *completion = cluster.aio_create_completion();
2978 ASSERT_EQ(0, ioctx.aio_operate(
2979 "foo", completion, &op,
2980 librados::OPERATION_IGNORE_OVERLAY, NULL));
2981 completion->wait_for_safe();
2982 ASSERT_EQ(0, completion->get_return_value());
2983 completion->release();
2984 ASSERT_EQ('b', bl[0]);
2988 TEST_F(LibRadosTwoPoolsECPP, Promote) {
2992 bl.append("hi there");
2993 ObjectWriteOperation op;
2995 ASSERT_EQ(0, ioctx.operate("foo", &op));
3000 ASSERT_EQ(0, cluster.mon_command(
3001 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
3002 "\", \"tierpool\": \"" + cache_pool_name +
3003 "\", \"force_nonempty\": \"--force-nonempty\" }",
3005 ASSERT_EQ(0, cluster.mon_command(
3006 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
3007 "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
3009 ASSERT_EQ(0, cluster.mon_command(
3010 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
3011 "\", \"mode\": \"writeback\"}",
3014 // wait for maps to settle
3015 cluster.wait_for_latest_osdmap();
3017 // read, trigger a promote
3020 ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
3023 // read, trigger a whiteout
3026 ASSERT_EQ(-ENOENT, ioctx.read("bar", bl, 1, 0));
3027 ASSERT_EQ(-ENOENT, ioctx.read("bar", bl, 1, 0));
3030 // verify the object is present in the cache tier
3032 NObjectIterator it = cache_ioctx.nobjects_begin();
3033 ASSERT_TRUE(it != cache_ioctx.nobjects_end());
3034 ASSERT_TRUE(it->get_oid() == string("foo") || it->get_oid() == string("bar"));
3036 ASSERT_TRUE(it->get_oid() == string("foo") || it->get_oid() == string("bar"));
3038 ASSERT_TRUE(it == cache_ioctx.nobjects_end());
3042 TEST_F(LibRadosTwoPoolsECPP, PromoteSnap) {
3046 bl.append("hi there");
3047 ObjectWriteOperation op;
3049 ASSERT_EQ(0, ioctx.operate("foo", &op));
3053 bl.append("hi there");
3054 ObjectWriteOperation op;
3056 ASSERT_EQ(0, ioctx.operate("bar", &op));
3060 bl.append("hi there");
3061 ObjectWriteOperation op;
3063 ASSERT_EQ(0, ioctx.operate("baz", &op));
3067 bl.append("hi there");
3068 ObjectWriteOperation op;
3070 ASSERT_EQ(0, ioctx.operate("bam", &op));
3073 // create a snapshot, clone
3074 vector<uint64_t> my_snaps(1);
3075 ASSERT_EQ(0, ioctx.selfmanaged_snap_create(&my_snaps[0]));
3076 ASSERT_EQ(0, ioctx.selfmanaged_snap_set_write_ctx(my_snaps[0],
3081 ObjectWriteOperation op;
3083 ASSERT_EQ(0, ioctx.operate("foo", &op));
3088 ObjectWriteOperation op;
3090 ASSERT_EQ(0, ioctx.operate("bar", &op));
3093 ObjectWriteOperation op;
3095 ASSERT_EQ(0, ioctx.operate("baz", &op));
3100 ObjectWriteOperation op;
3102 ASSERT_EQ(0, ioctx.operate("bam", &op));
3107 ASSERT_EQ(0, cluster.mon_command(
3108 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
3109 "\", \"tierpool\": \"" + cache_pool_name +
3110 "\", \"force_nonempty\": \"--force-nonempty\" }",
3112 ASSERT_EQ(0, cluster.mon_command(
3113 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
3114 "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
3116 ASSERT_EQ(0, cluster.mon_command(
3117 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
3118 "\", \"mode\": \"writeback\"}",
3121 // wait for maps to settle
3122 cluster.wait_for_latest_osdmap();
3124 // read, trigger a promote on the head
3127 ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
3128 ASSERT_EQ('c', bl[0]);
3132 ASSERT_EQ(1, ioctx.read("bam", bl, 1, 0));
3133 ASSERT_EQ('c', bl[0]);
3136 ioctx.snap_set_read(my_snaps[0]);
3138 // stop and scrub this pg (to make sure scrub can handle missing
3139 // clones in the cache tier)
3140 // This test requires cache tier and base tier to have the same pg_num/pgp_num
3142 for (int tries = 0; tries < 5; ++tries) {
3144 ASSERT_EQ(0, cluster.ioctx_create(cache_pool_name.c_str(), cache_ioctx));
3146 ASSERT_EQ(0, ioctx.get_object_pg_hash_position2("foo", &hash));
3148 ss << "{\"prefix\": \"pg scrub\", \"pgid\": \""
3149 << cache_ioctx.get_id() << "."
3152 int r = cluster.mon_command(ss.str(), inbl, NULL, NULL);
3154 r == -ENOENT) { // in case mgr osdmap is a bit stale
3161 // give it a few seconds to go. this is sloppy but is usually enough time
3162 cout << "waiting for scrub..." << std::endl;
3164 cout << "done waiting" << std::endl;
3170 ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
3171 ASSERT_EQ('h', bl[0]);
3177 ASSERT_EQ(1, ioctx.read("bar", bl, 1, 0));
3178 ASSERT_EQ('h', bl[0]);
3184 ASSERT_EQ(1, ioctx.read("baz", bl, 1, 0));
3185 ASSERT_EQ('h', bl[0]);
3188 ioctx.snap_set_read(librados::SNAP_HEAD);
3193 ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
3194 ASSERT_EQ('c', bl[0]);
3200 ASSERT_EQ(1, ioctx.read("bar", bl, 1, 0));
3201 ASSERT_EQ('c', bl[0]);
3207 ASSERT_EQ(-ENOENT, ioctx.read("baz", bl, 1, 0));
3211 ioctx.selfmanaged_snap_remove(my_snaps[0]);
3214 TEST_F(LibRadosTwoPoolsECPP, PromoteSnapTrimRace) {
3218 bl.append("hi there");
3219 ObjectWriteOperation op;
3221 ASSERT_EQ(0, ioctx.operate("foo", &op));
3224 // create a snapshot, clone
3225 vector<uint64_t> my_snaps(1);
3226 ASSERT_EQ(0, ioctx.selfmanaged_snap_create(&my_snaps[0]));
3227 ASSERT_EQ(0, ioctx.selfmanaged_snap_set_write_ctx(my_snaps[0],
3232 ObjectWriteOperation op;
3234 ASSERT_EQ(0, ioctx.operate("foo", &op));
3239 ASSERT_EQ(0, cluster.mon_command(
3240 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
3241 "\", \"tierpool\": \"" + cache_pool_name +
3242 "\", \"force_nonempty\": \"--force-nonempty\" }",
3244 ASSERT_EQ(0, cluster.mon_command(
3245 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
3246 "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
3248 ASSERT_EQ(0, cluster.mon_command(
3249 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
3250 "\", \"mode\": \"writeback\"}",
3253 // wait for maps to settle
3254 cluster.wait_for_latest_osdmap();
3257 ASSERT_EQ(0, ioctx.selfmanaged_snap_remove(my_snaps[0]));
3259 ioctx.snap_set_read(my_snaps[0]);
3264 ASSERT_EQ(-ENOENT, ioctx.read("foo", bl, 1, 0));
3268 ioctx.selfmanaged_snap_remove(my_snaps[0]);
3271 TEST_F(LibRadosTwoPoolsECPP, Whiteout) {
3275 bl.append("hi there");
3276 ObjectWriteOperation op;
3278 ASSERT_EQ(0, ioctx.operate("foo", &op));
3283 ASSERT_EQ(0, cluster.mon_command(
3284 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
3285 "\", \"tierpool\": \"" + cache_pool_name +
3286 "\", \"force_nonempty\": \"--force-nonempty\" }",
3288 ASSERT_EQ(0, cluster.mon_command(
3289 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
3290 "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
3292 ASSERT_EQ(0, cluster.mon_command(
3293 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
3294 "\", \"mode\": \"writeback\"}",
3297 // wait for maps to settle
3298 cluster.wait_for_latest_osdmap();
3300 // create some whiteouts, verify they behave
3302 ObjectWriteOperation op;
3305 ASSERT_EQ(0, ioctx.operate("foo", &op));
3309 ObjectWriteOperation op;
3312 ASSERT_EQ(-ENOENT, ioctx.operate("bar", &op));
3315 ObjectWriteOperation op;
3318 ASSERT_EQ(-ENOENT, ioctx.operate("bar", &op));
3321 // verify the whiteouts are there in the cache tier
3323 NObjectIterator it = cache_ioctx.nobjects_begin();
3324 ASSERT_TRUE(it != cache_ioctx.nobjects_end());
3325 ASSERT_TRUE(it->get_oid() == string("foo") || it->get_oid() == string("bar"));
3327 ASSERT_TRUE(it->get_oid() == string("foo") || it->get_oid() == string("bar"));
3329 ASSERT_TRUE(it == cache_ioctx.nobjects_end());
3332 // delete a whiteout and verify it goes away
3333 ASSERT_EQ(-ENOENT, ioctx.remove("foo"));
3335 ObjectWriteOperation op;
3337 librados::AioCompletion *completion = cluster.aio_create_completion();
3338 ASSERT_EQ(0, ioctx.aio_operate("bar", completion, &op,
3339 librados::OPERATION_IGNORE_CACHE));
3340 completion->wait_for_safe();
3341 ASSERT_EQ(0, completion->get_return_value());
3342 completion->release();
3344 NObjectIterator it = cache_ioctx.nobjects_begin();
3345 ASSERT_TRUE(it != cache_ioctx.nobjects_end());
3346 ASSERT_TRUE(it->get_oid() == string("foo"));
3348 ASSERT_TRUE(it == cache_ioctx.nobjects_end());
3351 // recreate an object and verify we can read it
3354 bl.append("hi there");
3355 ObjectWriteOperation op;
3357 ASSERT_EQ(0, ioctx.operate("foo", &op));
3361 ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
3362 ASSERT_EQ('h', bl[0]);
3366 TEST_F(LibRadosTwoPoolsECPP, Evict) {
3370 bl.append("hi there");
3371 ObjectWriteOperation op;
3373 ASSERT_EQ(0, ioctx.operate("foo", &op));
3378 ASSERT_EQ(0, cluster.mon_command(
3379 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
3380 "\", \"tierpool\": \"" + cache_pool_name +
3381 "\", \"force_nonempty\": \"--force-nonempty\" }",
3383 ASSERT_EQ(0, cluster.mon_command(
3384 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
3385 "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
3387 ASSERT_EQ(0, cluster.mon_command(
3388 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
3389 "\", \"mode\": \"writeback\"}",
3392 // wait for maps to settle
3393 cluster.wait_for_latest_osdmap();
3395 // read, trigger a promote
3398 ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
3401 // read, trigger a whiteout, and a dirty object
3404 ASSERT_EQ(-ENOENT, ioctx.read("bar", bl, 1, 0));
3405 ASSERT_EQ(-ENOENT, ioctx.read("bar", bl, 1, 0));
3406 ASSERT_EQ(0, ioctx.write("bar", bl, bl.length(), 0));
3409 // verify the object is present in the cache tier
3411 NObjectIterator it = cache_ioctx.nobjects_begin();
3412 ASSERT_TRUE(it != cache_ioctx.nobjects_end());
3413 ASSERT_TRUE(it->get_oid() == string("foo") || it->get_oid() == string("bar"));
3415 ASSERT_TRUE(it->get_oid() == string("foo") || it->get_oid() == string("bar"));
3417 ASSERT_TRUE(it == cache_ioctx.nobjects_end());
3422 ObjectWriteOperation op;
3424 librados::AioCompletion *completion = cluster.aio_create_completion();
3425 ASSERT_EQ(0, cache_ioctx.aio_operate("foo", completion, &op));
3426 completion->wait_for_safe();
3427 ASSERT_EQ(0, completion->get_return_value());
3428 completion->release();
3431 // evict the pinned object with -EPERM
3433 ObjectReadOperation op;
3435 librados::AioCompletion *completion = cluster.aio_create_completion();
3436 ASSERT_EQ(0, cache_ioctx.aio_operate("foo", completion, &op,
3437 librados::OPERATION_IGNORE_CACHE,
3439 completion->wait_for_safe();
3440 ASSERT_EQ(-EPERM, completion->get_return_value());
3441 completion->release();
3446 ObjectWriteOperation op;
3448 librados::AioCompletion *completion = cluster.aio_create_completion();
3449 ASSERT_EQ(0, cache_ioctx.aio_operate("foo", completion, &op));
3450 completion->wait_for_safe();
3451 ASSERT_EQ(0, completion->get_return_value());
3452 completion->release();
3457 ObjectReadOperation op;
3459 librados::AioCompletion *completion = cluster.aio_create_completion();
3460 ASSERT_EQ(0, cache_ioctx.aio_operate(
3461 "foo", completion, &op,
3462 librados::OPERATION_IGNORE_OVERLAY, NULL));
3463 completion->wait_for_safe();
3464 ASSERT_EQ(0, completion->get_return_value());
3465 completion->release();
3472 ObjectReadOperation op;
3473 op.is_dirty(&dirty, &r);
3474 ASSERT_EQ(0, cache_ioctx.operate("foo", &op, NULL));
3475 ASSERT_FALSE(dirty);
3481 ObjectReadOperation op;
3483 librados::AioCompletion *completion = cluster.aio_create_completion();
3484 ASSERT_EQ(0, cache_ioctx.aio_operate("foo", completion, &op,
3485 librados::OPERATION_IGNORE_CACHE,
3487 completion->wait_for_safe();
3488 ASSERT_EQ(0, completion->get_return_value());
3489 completion->release();
3492 ObjectReadOperation op;
3494 librados::AioCompletion *completion = cluster.aio_create_completion();
3495 ASSERT_EQ(0, cache_ioctx.aio_operate(
3496 "foo", completion, &op,
3497 librados::OPERATION_IGNORE_CACHE, NULL));
3498 completion->wait_for_safe();
3499 ASSERT_EQ(0, completion->get_return_value());
3500 completion->release();
3503 ObjectReadOperation op;
3505 librados::AioCompletion *completion = cluster.aio_create_completion();
3506 ASSERT_EQ(0, cache_ioctx.aio_operate(
3507 "bar", completion, &op,
3508 librados::OPERATION_IGNORE_CACHE, NULL));
3509 completion->wait_for_safe();
3510 ASSERT_EQ(-EBUSY, completion->get_return_value());
3511 completion->release();
3515 TEST_F(LibRadosTwoPoolsECPP, EvictSnap) {
3519 bl.append("hi there");
3520 ObjectWriteOperation op;
3522 ASSERT_EQ(0, ioctx.operate("foo", &op));
3526 bl.append("hi there");
3527 ObjectWriteOperation op;
3529 ASSERT_EQ(0, ioctx.operate("bar", &op));
3533 bl.append("hi there");
3534 ObjectWriteOperation op;
3536 ASSERT_EQ(0, ioctx.operate("baz", &op));
3540 bl.append("hi there");
3541 ObjectWriteOperation op;
3543 ASSERT_EQ(0, ioctx.operate("bam", &op));
3546 // create a snapshot, clone
3547 vector<uint64_t> my_snaps(1);
3548 ASSERT_EQ(0, ioctx.selfmanaged_snap_create(&my_snaps[0]));
3549 ASSERT_EQ(0, ioctx.selfmanaged_snap_set_write_ctx(my_snaps[0],
3554 ObjectWriteOperation op;
3556 ASSERT_EQ(0, ioctx.operate("foo", &op));
3561 ObjectWriteOperation op;
3563 ASSERT_EQ(0, ioctx.operate("bar", &op));
3566 ObjectWriteOperation op;
3568 ASSERT_EQ(0, ioctx.operate("baz", &op));
3573 ObjectWriteOperation op;
3575 ASSERT_EQ(0, ioctx.operate("bam", &op));
3580 ASSERT_EQ(0, cluster.mon_command(
3581 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
3582 "\", \"tierpool\": \"" + cache_pool_name +
3583 "\", \"force_nonempty\": \"--force-nonempty\" }",
3585 ASSERT_EQ(0, cluster.mon_command(
3586 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
3587 "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
3589 ASSERT_EQ(0, cluster.mon_command(
3590 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
3591 "\", \"mode\": \"writeback\"}",
3594 // wait for maps to settle
3595 cluster.wait_for_latest_osdmap();
3597 // read, trigger a promote on the head
3600 ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
3601 ASSERT_EQ('c', bl[0]);
3605 ASSERT_EQ(1, ioctx.read("bam", bl, 1, 0));
3606 ASSERT_EQ('c', bl[0]);
3611 ObjectReadOperation op;
3613 librados::AioCompletion *completion = cluster.aio_create_completion();
3614 ASSERT_EQ(0, cache_ioctx.aio_operate(
3615 "bam", completion, &op,
3616 librados::OPERATION_IGNORE_CACHE, NULL));
3617 completion->wait_for_safe();
3618 ASSERT_EQ(0, completion->get_return_value());
3619 completion->release();
3623 ObjectReadOperation op;
3624 op.read(1, 0, &bl, NULL);
3625 librados::AioCompletion *completion = cluster.aio_create_completion();
3626 ASSERT_EQ(0, cache_ioctx.aio_operate(
3627 "bam", completion, &op,
3628 librados::OPERATION_IGNORE_CACHE, NULL));
3629 completion->wait_for_safe();
3630 ASSERT_EQ(-ENOENT, completion->get_return_value());
3631 completion->release();
3635 ioctx.snap_set_read(my_snaps[0]);
3638 ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
3639 ASSERT_EQ('h', bl[0]);
3644 ObjectReadOperation op;
3646 librados::AioCompletion *completion = cluster.aio_create_completion();
3647 ASSERT_EQ(0, ioctx.aio_operate(
3648 "foo", completion, &op,
3649 librados::OPERATION_IGNORE_CACHE, NULL));
3650 completion->wait_for_safe();
3651 ASSERT_EQ(0, completion->get_return_value());
3652 completion->release();
3657 ObjectReadOperation op;
3658 op.read(1, 0, &bl, NULL);
3659 librados::AioCompletion *completion = cluster.aio_create_completion();
3660 ASSERT_EQ(0, ioctx.aio_operate(
3661 "foo", completion, &op,
3662 librados::OPERATION_IGNORE_CACHE, NULL));
3663 completion->wait_for_safe();
3664 ASSERT_EQ(-ENOENT, completion->get_return_value());
3665 completion->release();
3667 // head is still there...
3668 ioctx.snap_set_read(librados::SNAP_HEAD);
3671 ObjectReadOperation op;
3672 op.read(1, 0, &bl, NULL);
3673 librados::AioCompletion *completion = cluster.aio_create_completion();
3674 ASSERT_EQ(0, ioctx.aio_operate(
3675 "foo", completion, &op,
3676 librados::OPERATION_IGNORE_CACHE, NULL));
3677 completion->wait_for_safe();
3678 ASSERT_EQ(0, completion->get_return_value());
3679 completion->release();
3682 // promote head + snap of bar
3683 ioctx.snap_set_read(librados::SNAP_HEAD);
3686 ASSERT_EQ(1, ioctx.read("bar", bl, 1, 0));
3687 ASSERT_EQ('c', bl[0]);
3689 ioctx.snap_set_read(my_snaps[0]);
3692 ASSERT_EQ(1, ioctx.read("bar", bl, 1, 0));
3693 ASSERT_EQ('h', bl[0]);
3696 // evict bar head (fail)
3697 ioctx.snap_set_read(librados::SNAP_HEAD);
3699 ObjectReadOperation op;
3701 librados::AioCompletion *completion = cluster.aio_create_completion();
3702 ASSERT_EQ(0, ioctx.aio_operate(
3703 "bar", completion, &op,
3704 librados::OPERATION_IGNORE_CACHE, NULL));
3705 completion->wait_for_safe();
3706 ASSERT_EQ(-EBUSY, completion->get_return_value());
3707 completion->release();
3711 ioctx.snap_set_read(my_snaps[0]);
3713 ObjectReadOperation op;
3715 librados::AioCompletion *completion = cluster.aio_create_completion();
3716 ASSERT_EQ(0, ioctx.aio_operate(
3717 "bar", completion, &op,
3718 librados::OPERATION_IGNORE_CACHE, NULL));
3719 completion->wait_for_safe();
3720 ASSERT_EQ(0, completion->get_return_value());
3721 completion->release();
3724 ioctx.snap_set_read(librados::SNAP_HEAD);
3727 ObjectReadOperation op;
3728 op.read(1, 0, &bl, NULL);
3729 librados::AioCompletion *completion = cluster.aio_create_completion();
3730 ASSERT_EQ(0, ioctx.aio_operate(
3731 "bar", completion, &op,
3732 librados::OPERATION_IGNORE_CACHE, NULL));
3733 completion->wait_for_safe();
3734 ASSERT_EQ(0, completion->get_return_value());
3735 completion->release();
3738 ObjectReadOperation op;
3740 librados::AioCompletion *completion = cluster.aio_create_completion();
3741 ASSERT_EQ(0, ioctx.aio_operate(
3742 "bar", completion, &op,
3743 librados::OPERATION_IGNORE_CACHE, NULL));
3744 completion->wait_for_safe();
3745 ASSERT_EQ(0, completion->get_return_value());
3746 completion->release();
3750 ioctx.selfmanaged_snap_remove(my_snaps[0]);
3753 TEST_F(LibRadosTwoPoolsECPP, TryFlush) {
3756 ASSERT_EQ(0, cluster.mon_command(
3757 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
3758 "\", \"tierpool\": \"" + cache_pool_name +
3759 "\", \"force_nonempty\": \"--force-nonempty\" }",
3761 ASSERT_EQ(0, cluster.mon_command(
3762 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
3763 "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
3765 ASSERT_EQ(0, cluster.mon_command(
3766 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
3767 "\", \"mode\": \"writeback\"}",
3770 // wait for maps to settle
3771 cluster.wait_for_latest_osdmap();
3776 bl.append("hi there");
3777 ObjectWriteOperation op;
3779 ASSERT_EQ(0, ioctx.operate("foo", &op));
3782 // verify the object is present in the cache tier
3784 NObjectIterator it = cache_ioctx.nobjects_begin();
3785 ASSERT_TRUE(it != cache_ioctx.nobjects_end());
3786 ASSERT_TRUE(it->get_oid() == string("foo"));
3788 ASSERT_TRUE(it == cache_ioctx.nobjects_end());
3791 // verify the object is NOT present in the base tier
3793 NObjectIterator it = ioctx.nobjects_begin();
3794 ASSERT_TRUE(it == ioctx.nobjects_end());
3801 ObjectReadOperation op;
3802 op.is_dirty(&dirty, &r);
3803 ASSERT_EQ(0, cache_ioctx.operate("foo", &op, NULL));
3810 ObjectWriteOperation op;
3812 librados::AioCompletion *completion = cluster.aio_create_completion();
3813 ASSERT_EQ(0, cache_ioctx.aio_operate("foo", completion, &op));
3814 completion->wait_for_safe();
3815 ASSERT_EQ(0, completion->get_return_value());
3816 completion->release();
3819 // flush the pinned object with -EPERM
3821 ObjectReadOperation op;
3822 op.cache_try_flush();
3823 librados::AioCompletion *completion = cluster.aio_create_completion();
3824 ASSERT_EQ(0, cache_ioctx.aio_operate(
3825 "foo", completion, &op,
3826 librados::OPERATION_IGNORE_OVERLAY |
3827 librados::OPERATION_SKIPRWLOCKS, NULL));
3828 completion->wait_for_safe();
3829 ASSERT_EQ(-EPERM, completion->get_return_value());
3830 completion->release();
3835 ObjectWriteOperation op;
3837 librados::AioCompletion *completion = cluster.aio_create_completion();
3838 ASSERT_EQ(0, cache_ioctx.aio_operate("foo", completion, &op));
3839 completion->wait_for_safe();
3840 ASSERT_EQ(0, completion->get_return_value());
3841 completion->release();
3846 ObjectReadOperation op;
3847 op.cache_try_flush();
3848 librados::AioCompletion *completion = cluster.aio_create_completion();
3849 ASSERT_EQ(0, cache_ioctx.aio_operate(
3850 "foo", completion, &op,
3851 librados::OPERATION_IGNORE_OVERLAY |
3852 librados::OPERATION_SKIPRWLOCKS, NULL));
3853 completion->wait_for_safe();
3854 ASSERT_EQ(0, completion->get_return_value());
3855 completion->release();
3862 ObjectReadOperation op;
3863 op.is_dirty(&dirty, &r);
3864 ASSERT_EQ(0, cache_ioctx.operate("foo", &op, NULL));
3865 ASSERT_FALSE(dirty);
3869 // verify in base tier
3871 NObjectIterator it = ioctx.nobjects_begin();
3872 ASSERT_TRUE(it != ioctx.nobjects_end());
3873 ASSERT_TRUE(it->get_oid() == string("foo"));
3875 ASSERT_TRUE(it == ioctx.nobjects_end());
3880 ObjectReadOperation op;
3882 librados::AioCompletion *completion = cluster.aio_create_completion();
3883 ASSERT_EQ(0, cache_ioctx.aio_operate(
3884 "foo", completion, &op, librados::OPERATION_IGNORE_CACHE, NULL));
3885 completion->wait_for_safe();
3886 ASSERT_EQ(0, completion->get_return_value());
3887 completion->release();
3890 // verify no longer in cache tier
3892 NObjectIterator it = cache_ioctx.nobjects_begin();
3893 ASSERT_TRUE(it == cache_ioctx.nobjects_end());
3897 TEST_F(LibRadosTwoPoolsECPP, FailedFlush) {
3900 ASSERT_EQ(0, cluster.mon_command(
3901 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
3902 "\", \"tierpool\": \"" + cache_pool_name +
3903 "\", \"force_nonempty\": \"--force-nonempty\" }",
3905 ASSERT_EQ(0, cluster.mon_command(
3906 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
3907 "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
3909 ASSERT_EQ(0, cluster.mon_command(
3910 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
3911 "\", \"mode\": \"writeback\"}",
3914 // wait for maps to settle
3915 cluster.wait_for_latest_osdmap();
3920 bl.append("hi there");
3921 ObjectWriteOperation op;
3923 ASSERT_EQ(0, ioctx.operate("foo", &op));
3926 // verify the object is present in the cache tier
3928 NObjectIterator it = cache_ioctx.nobjects_begin();
3929 ASSERT_TRUE(it != cache_ioctx.nobjects_end());
3930 ASSERT_TRUE(it->get_oid() == string("foo"));
3932 ASSERT_TRUE(it == cache_ioctx.nobjects_end());
3935 // verify the object is NOT present in the base tier
3937 NObjectIterator it = ioctx.nobjects_begin();
3938 ASSERT_TRUE(it == ioctx.nobjects_end());
3943 ObjectWriteOperation op;
3944 std::map<std::string, bufferlist> omap;
3945 omap["somekey"] = bufferlist();
3947 librados::AioCompletion *completion = cluster.aio_create_completion();
3948 ASSERT_EQ(0, cache_ioctx.aio_operate("foo", completion, &op));
3949 completion->wait_for_safe();
3950 ASSERT_EQ(0, completion->get_return_value());
3951 completion->release();
3956 ObjectReadOperation op;
3958 librados::AioCompletion *completion = cluster.aio_create_completion();
3959 ASSERT_EQ(0, cache_ioctx.aio_operate(
3960 "foo", completion, &op,
3961 librados::OPERATION_IGNORE_OVERLAY, NULL));
3962 completion->wait_for_safe();
3963 ASSERT_NE(0, completion->get_return_value());
3964 completion->release();
3969 ObjectReadOperation op;
3972 std::set<std::string> keys;
3973 keys.insert("somekey");
3974 std::map<std::string, bufferlist> map;
3976 op.omap_get_vals_by_keys(keys, &map, &prval);
3977 librados::AioCompletion *completion = cluster.aio_create_completion();
3978 ASSERT_EQ(0, cache_ioctx.aio_operate("foo", completion, &op, &bl));
3980 bool completed = completion->is_complete();
3982 cache_ioctx.aio_cancel(completion);
3983 std::cerr << "Most probably test case will hang here, please reset manually" << std::endl;
3984 ASSERT_TRUE(completed); //in fact we are locked forever at test case shutdown unless fix for http://tracker.ceph.com/issues/14511 is applied. Seems there is no workaround for that
3986 completion->release();
3988 // verify still not in base tier
3990 ASSERT_TRUE(ioctx.nobjects_begin() == ioctx.nobjects_end());
3994 ObjectWriteOperation op;
3996 ASSERT_EQ(0, ioctx.operate("foo", &op));
4000 ObjectReadOperation op;
4002 librados::AioCompletion *completion = cluster.aio_create_completion();
4003 ASSERT_EQ(0, cache_ioctx.aio_operate(
4004 "foo", completion, &op,
4005 librados::OPERATION_IGNORE_OVERLAY, NULL));
4006 completion->wait_for_safe();
4007 ASSERT_EQ(0, completion->get_return_value());
4008 completion->release();
4012 ObjectReadOperation op;
4014 librados::AioCompletion *completion = cluster.aio_create_completion();
4015 ASSERT_EQ(0, cache_ioctx.aio_operate(
4016 "foo", completion, &op, librados::OPERATION_IGNORE_CACHE, NULL));
4017 completion->wait_for_safe();
4018 ASSERT_EQ(0, completion->get_return_value());
4019 completion->release();
4022 // verify no longer in cache tier
4024 NObjectIterator it = cache_ioctx.nobjects_begin();
4025 ASSERT_TRUE(it == cache_ioctx.nobjects_end());
4029 NObjectIterator it = ioctx.nobjects_begin();
4030 ASSERT_TRUE(it == ioctx.nobjects_end());
4034 TEST_F(LibRadosTwoPoolsECPP, Flush) {
4037 ASSERT_EQ(0, cluster.mon_command(
4038 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
4039 "\", \"tierpool\": \"" + cache_pool_name +
4040 "\", \"force_nonempty\": \"--force-nonempty\" }",
4042 ASSERT_EQ(0, cluster.mon_command(
4043 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
4044 "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
4046 ASSERT_EQ(0, cluster.mon_command(
4047 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
4048 "\", \"mode\": \"writeback\"}",
4051 // wait for maps to settle
4052 cluster.wait_for_latest_osdmap();
4054 uint64_t user_version = 0;
4059 bl.append("hi there");
4060 ObjectWriteOperation op;
4062 ASSERT_EQ(0, ioctx.operate("foo", &op));
4065 // verify the object is present in the cache tier
4067 NObjectIterator it = cache_ioctx.nobjects_begin();
4068 ASSERT_TRUE(it != cache_ioctx.nobjects_end());
4069 ASSERT_TRUE(it->get_oid() == string("foo"));
4071 ASSERT_TRUE(it == cache_ioctx.nobjects_end());
4074 // verify the object is NOT present in the base tier
4076 NObjectIterator it = ioctx.nobjects_begin();
4077 ASSERT_TRUE(it == ioctx.nobjects_end());
4084 ObjectReadOperation op;
4085 op.is_dirty(&dirty, &r);
4086 ASSERT_EQ(0, cache_ioctx.operate("foo", &op, NULL));
4089 user_version = cache_ioctx.get_last_version();
4094 ObjectWriteOperation op;
4096 librados::AioCompletion *completion = cluster.aio_create_completion();
4097 ASSERT_EQ(0, cache_ioctx.aio_operate("foo", completion, &op));
4098 completion->wait_for_safe();
4099 ASSERT_EQ(0, completion->get_return_value());
4100 completion->release();
4103 // flush the pinned object with -EPERM
4105 ObjectReadOperation op;
4106 op.cache_try_flush();
4107 librados::AioCompletion *completion = cluster.aio_create_completion();
4108 ASSERT_EQ(0, cache_ioctx.aio_operate(
4109 "foo", completion, &op,
4110 librados::OPERATION_IGNORE_OVERLAY |
4111 librados::OPERATION_SKIPRWLOCKS, NULL));
4112 completion->wait_for_safe();
4113 ASSERT_EQ(-EPERM, completion->get_return_value());
4114 completion->release();
4119 ObjectWriteOperation op;
4121 librados::AioCompletion *completion = cluster.aio_create_completion();
4122 ASSERT_EQ(0, cache_ioctx.aio_operate("foo", completion, &op));
4123 completion->wait_for_safe();
4124 ASSERT_EQ(0, completion->get_return_value());
4125 completion->release();
4130 ObjectReadOperation op;
4132 librados::AioCompletion *completion = cluster.aio_create_completion();
4133 ASSERT_EQ(0, cache_ioctx.aio_operate(
4134 "foo", completion, &op,
4135 librados::OPERATION_IGNORE_OVERLAY, NULL));
4136 completion->wait_for_safe();
4137 ASSERT_EQ(0, completion->get_return_value());
4138 completion->release();
4145 ObjectReadOperation op;
4146 op.is_dirty(&dirty, &r);
4147 ASSERT_EQ(0, cache_ioctx.operate("foo", &op, NULL));
4148 ASSERT_FALSE(dirty);
4152 // verify in base tier
4154 NObjectIterator it = ioctx.nobjects_begin();
4155 ASSERT_TRUE(it != ioctx.nobjects_end());
4156 ASSERT_TRUE(it->get_oid() == string("foo"));
4158 ASSERT_TRUE(it == ioctx.nobjects_end());
4163 ObjectReadOperation op;
4165 librados::AioCompletion *completion = cluster.aio_create_completion();
4166 ASSERT_EQ(0, cache_ioctx.aio_operate(
4167 "foo", completion, &op, librados::OPERATION_IGNORE_CACHE, NULL));
4168 completion->wait_for_safe();
4169 ASSERT_EQ(0, completion->get_return_value());
4170 completion->release();
4173 // verify no longer in cache tier
4175 NObjectIterator it = cache_ioctx.nobjects_begin();
4176 ASSERT_TRUE(it == cache_ioctx.nobjects_end());
4179 // read it again and verify the version is consistent
4182 ASSERT_EQ(1, cache_ioctx.read("foo", bl, 1, 0));
4183 ASSERT_EQ(user_version, cache_ioctx.get_last_version());
4188 ObjectWriteOperation op;
4190 ASSERT_EQ(0, ioctx.operate("foo", &op));
4195 ObjectReadOperation op;
4197 librados::AioCompletion *completion = cluster.aio_create_completion();
4198 ASSERT_EQ(0, cache_ioctx.aio_operate(
4199 "foo", completion, &op,
4200 librados::OPERATION_IGNORE_OVERLAY, NULL));
4201 completion->wait_for_safe();
4202 ASSERT_EQ(0, completion->get_return_value());
4203 completion->release();
4208 ObjectReadOperation op;
4210 librados::AioCompletion *completion = cluster.aio_create_completion();
4211 ASSERT_EQ(0, cache_ioctx.aio_operate(
4212 "foo", completion, &op, librados::OPERATION_IGNORE_CACHE, NULL));
4213 completion->wait_for_safe();
4214 ASSERT_EQ(0, completion->get_return_value());
4215 completion->release();
4218 // verify no longer in cache tier
4220 NObjectIterator it = cache_ioctx.nobjects_begin();
4221 ASSERT_TRUE(it == cache_ioctx.nobjects_end());
4225 NObjectIterator it = ioctx.nobjects_begin();
4226 ASSERT_TRUE(it == ioctx.nobjects_end());
4230 TEST_F(LibRadosTwoPoolsECPP, FlushSnap) {
4233 ASSERT_EQ(0, cluster.mon_command(
4234 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
4235 "\", \"tierpool\": \"" + cache_pool_name +
4236 "\", \"force_nonempty\": \"--force-nonempty\" }",
4238 ASSERT_EQ(0, cluster.mon_command(
4239 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
4240 "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
4242 ASSERT_EQ(0, cluster.mon_command(
4243 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
4244 "\", \"mode\": \"writeback\"}",
4247 // wait for maps to settle
4248 cluster.wait_for_latest_osdmap();
4254 ObjectWriteOperation op;
4256 ASSERT_EQ(0, ioctx.operate("foo", &op));
4259 // create a snapshot, clone
4260 vector<uint64_t> my_snaps(1);
4261 ASSERT_EQ(0, ioctx.selfmanaged_snap_create(&my_snaps[0]));
4262 ASSERT_EQ(0, ioctx.selfmanaged_snap_set_write_ctx(my_snaps[0],
4267 ObjectWriteOperation op;
4269 ASSERT_EQ(0, ioctx.operate("foo", &op));
4274 my_snaps[1] = my_snaps[0];
4275 ASSERT_EQ(0, ioctx.selfmanaged_snap_create(&my_snaps[0]));
4276 ASSERT_EQ(0, ioctx.selfmanaged_snap_set_write_ctx(my_snaps[0],
4281 ObjectWriteOperation op;
4283 ASSERT_EQ(0, ioctx.operate("foo", &op));
4286 // verify the object is present in the cache tier
4288 NObjectIterator it = cache_ioctx.nobjects_begin();
4289 ASSERT_TRUE(it != cache_ioctx.nobjects_end());
4290 ASSERT_TRUE(it->get_oid() == string("foo"));
4292 ASSERT_TRUE(it == cache_ioctx.nobjects_end());
4295 // verify the object is NOT present in the base tier
4297 NObjectIterator it = ioctx.nobjects_begin();
4298 ASSERT_TRUE(it == ioctx.nobjects_end());
4301 // flush on head (should fail)
4302 ioctx.snap_set_read(librados::SNAP_HEAD);
4304 ObjectReadOperation op;
4306 librados::AioCompletion *completion = cluster.aio_create_completion();
4307 ASSERT_EQ(0, ioctx.aio_operate(
4308 "foo", completion, &op,
4309 librados::OPERATION_IGNORE_CACHE, NULL));
4310 completion->wait_for_safe();
4311 ASSERT_EQ(-EBUSY, completion->get_return_value());
4312 completion->release();
4314 // flush on recent snap (should fail)
4315 ioctx.snap_set_read(my_snaps[0]);
4317 ObjectReadOperation op;
4319 librados::AioCompletion *completion = cluster.aio_create_completion();
4320 ASSERT_EQ(0, ioctx.aio_operate(
4321 "foo", completion, &op,
4322 librados::OPERATION_IGNORE_CACHE, NULL));
4323 completion->wait_for_safe();
4324 ASSERT_EQ(-EBUSY, completion->get_return_value());
4325 completion->release();
4327 // flush on oldest snap
4328 ioctx.snap_set_read(my_snaps[1]);
4330 ObjectReadOperation op;
4332 librados::AioCompletion *completion = cluster.aio_create_completion();
4333 ASSERT_EQ(0, ioctx.aio_operate(
4334 "foo", completion, &op,
4335 librados::OPERATION_IGNORE_CACHE, NULL));
4336 completion->wait_for_safe();
4337 ASSERT_EQ(0, completion->get_return_value());
4338 completion->release();
4340 // flush on next oldest snap
4341 ioctx.snap_set_read(my_snaps[0]);
4343 ObjectReadOperation op;
4345 librados::AioCompletion *completion = cluster.aio_create_completion();
4346 ASSERT_EQ(0, ioctx.aio_operate(
4347 "foo", completion, &op,
4348 librados::OPERATION_IGNORE_CACHE, NULL));
4349 completion->wait_for_safe();
4350 ASSERT_EQ(0, completion->get_return_value());
4351 completion->release();
4354 ioctx.snap_set_read(librados::SNAP_HEAD);
4356 ObjectReadOperation op;
4358 librados::AioCompletion *completion = cluster.aio_create_completion();
4359 ASSERT_EQ(0, ioctx.aio_operate(
4360 "foo", completion, &op,
4361 librados::OPERATION_IGNORE_CACHE, NULL));
4362 completion->wait_for_safe();
4363 ASSERT_EQ(0, completion->get_return_value());
4364 completion->release();
4367 // verify i can read the snaps from the cache pool
4368 ioctx.snap_set_read(librados::SNAP_HEAD);
4371 ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
4372 ASSERT_EQ('c', bl[0]);
4374 ioctx.snap_set_read(my_snaps[0]);
4377 ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
4378 ASSERT_EQ('b', bl[0]);
4380 ioctx.snap_set_read(my_snaps[1]);
4383 ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
4384 ASSERT_EQ('a', bl[0]);
4388 ASSERT_EQ(0, cluster.mon_command(
4389 "{\"prefix\": \"osd tier remove-overlay\", \"pool\": \"" + pool_name +
4393 // wait for maps to settle
4394 cluster.wait_for_latest_osdmap();
4396 // verify i can read the snaps from the base pool
4397 ioctx.snap_set_read(librados::SNAP_HEAD);
4400 ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
4401 ASSERT_EQ('c', bl[0]);
4403 ioctx.snap_set_read(my_snaps[0]);
4406 ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
4407 ASSERT_EQ('b', bl[0]);
4409 ioctx.snap_set_read(my_snaps[1]);
4412 ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
4413 ASSERT_EQ('a', bl[0]);
4416 ASSERT_EQ(0, cluster.mon_command(
4417 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
4418 "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
4420 cluster.wait_for_latest_osdmap();
4423 ioctx.selfmanaged_snap_remove(my_snaps[0]);
4426 TEST_F(LibRadosTierECPP, FlushWriteRaces) {
4428 std::string pool_name = get_temp_pool_name();
4429 std::string cache_pool_name = pool_name + "-cache";
4430 ASSERT_EQ("", create_one_pool_pp(pool_name, cluster));
4431 ASSERT_EQ(0, cluster.pool_create(cache_pool_name.c_str()));
4433 ASSERT_EQ(0, cluster.ioctx_create(cache_pool_name.c_str(), cache_ioctx));
4434 cache_ioctx.application_enable("rados", true);
4436 ASSERT_EQ(0, cluster.ioctx_create(pool_name.c_str(), ioctx));
4440 ASSERT_EQ(0, cluster.mon_command(
4441 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
4442 "\", \"tierpool\": \"" + cache_pool_name + "\"}",
4444 ASSERT_EQ(0, cluster.mon_command(
4445 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
4446 "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
4448 ASSERT_EQ(0, cluster.mon_command(
4449 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
4450 "\", \"mode\": \"writeback\"}",
4453 // wait for maps to settle
4454 cluster.wait_for_latest_osdmap();
4456 // create/dirty object
4458 bl.append("hi there");
4460 ObjectWriteOperation op;
4462 ASSERT_EQ(0, ioctx.operate("foo", &op));
4467 ObjectReadOperation op;
4469 librados::AioCompletion *completion = cluster.aio_create_completion();
4470 ASSERT_EQ(0, cache_ioctx.aio_operate(
4471 "foo", completion, &op,
4472 librados::OPERATION_IGNORE_OVERLAY, NULL));
4474 ObjectWriteOperation op2;
4476 librados::AioCompletion *completion2 = cluster.aio_create_completion();
4477 ASSERT_EQ(0, ioctx.aio_operate(
4478 "foo", completion2, &op2, 0));
4480 completion->wait_for_safe();
4481 completion2->wait_for_safe();
4482 ASSERT_EQ(0, completion->get_return_value());
4483 ASSERT_EQ(0, completion2->get_return_value());
4484 completion->release();
4485 completion2->release();
4490 // create/dirty object
4493 bl.append("hi there");
4494 ObjectWriteOperation op;
4496 ASSERT_EQ(0, ioctx.operate("foo", &op));
4499 // try-flush + write
4501 ObjectReadOperation op;
4502 op.cache_try_flush();
4503 librados::AioCompletion *completion = cluster.aio_create_completion();
4504 ASSERT_EQ(0, cache_ioctx.aio_operate(
4505 "foo", completion, &op,
4506 librados::OPERATION_IGNORE_OVERLAY |
4507 librados::OPERATION_SKIPRWLOCKS, NULL));
4509 ObjectWriteOperation op2;
4511 librados::AioCompletion *completion2 = cluster.aio_create_completion();
4512 ASSERT_EQ(0, ioctx.aio_operate("foo", completion2, &op2, 0));
4514 completion->wait_for_safe();
4515 completion2->wait_for_safe();
4516 int r = completion->get_return_value();
4517 ASSERT_TRUE(r == -EBUSY || r == 0);
4518 ASSERT_EQ(0, completion2->get_return_value());
4519 completion->release();
4520 completion2->release();
4523 cout << "didn't get EBUSY, trying again" << std::endl;
4525 ASSERT_TRUE(--tries);
4529 ASSERT_EQ(0, cluster.mon_command(
4530 "{\"prefix\": \"osd tier remove-overlay\", \"pool\": \"" + pool_name +
4533 ASSERT_EQ(0, cluster.mon_command(
4534 "{\"prefix\": \"osd tier remove\", \"pool\": \"" + pool_name +
4535 "\", \"tierpool\": \"" + cache_pool_name + "\"}",
4538 // wait for maps to settle before next test
4539 cluster.wait_for_latest_osdmap();
4541 ASSERT_EQ(0, cluster.pool_delete(cache_pool_name.c_str()));
4542 ASSERT_EQ(0, destroy_one_pool_pp(pool_name, cluster));
4545 TEST_F(LibRadosTwoPoolsECPP, FlushTryFlushRaces) {
4548 ASSERT_EQ(0, cluster.mon_command(
4549 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
4550 "\", \"tierpool\": \"" + cache_pool_name +
4551 "\", \"force_nonempty\": \"--force-nonempty\" }",
4553 ASSERT_EQ(0, cluster.mon_command(
4554 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
4555 "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
4557 ASSERT_EQ(0, cluster.mon_command(
4558 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
4559 "\", \"mode\": \"writeback\"}",
4562 // wait for maps to settle
4563 cluster.wait_for_latest_osdmap();
4565 // create/dirty object
4568 bl.append("hi there");
4569 ObjectWriteOperation op;
4571 ASSERT_EQ(0, ioctx.operate("foo", &op));
4576 ObjectReadOperation op;
4578 librados::AioCompletion *completion = cluster.aio_create_completion();
4579 ASSERT_EQ(0, cache_ioctx.aio_operate(
4580 "foo", completion, &op,
4581 librados::OPERATION_IGNORE_OVERLAY, NULL));
4583 ObjectReadOperation op2;
4585 librados::AioCompletion *completion2 = cluster.aio_create_completion();
4586 ASSERT_EQ(0, cache_ioctx.aio_operate(
4587 "foo", completion2, &op2,
4588 librados::OPERATION_IGNORE_OVERLAY, NULL));
4590 completion->wait_for_safe();
4591 completion2->wait_for_safe();
4592 ASSERT_EQ(0, completion->get_return_value());
4593 ASSERT_EQ(0, completion2->get_return_value());
4594 completion->release();
4595 completion2->release();
4598 // create/dirty object
4601 bl.append("hi there");
4602 ObjectWriteOperation op;
4604 ASSERT_EQ(0, ioctx.operate("foo", &op));
4607 // flush + try-flush
4609 ObjectReadOperation op;
4611 librados::AioCompletion *completion = cluster.aio_create_completion();
4612 ASSERT_EQ(0, cache_ioctx.aio_operate(
4613 "foo", completion, &op,
4614 librados::OPERATION_IGNORE_OVERLAY, NULL));
4616 ObjectReadOperation op2;
4617 op2.cache_try_flush();
4618 librados::AioCompletion *completion2 = cluster.aio_create_completion();
4619 ASSERT_EQ(0, cache_ioctx.aio_operate(
4620 "foo", completion2, &op2,
4621 librados::OPERATION_IGNORE_OVERLAY |
4622 librados::OPERATION_SKIPRWLOCKS, NULL));
4624 completion->wait_for_safe();
4625 completion2->wait_for_safe();
4626 ASSERT_EQ(0, completion->get_return_value());
4627 ASSERT_EQ(0, completion2->get_return_value());
4628 completion->release();
4629 completion2->release();
4632 // create/dirty object
4637 bl.append("hi there");
4638 ObjectWriteOperation op;
4640 ASSERT_EQ(0, ioctx.operate("foo", &op));
4643 // try-flush + flush
4644 // (flush will not piggyback on try-flush)
4646 ObjectReadOperation op;
4647 op.cache_try_flush();
4648 librados::AioCompletion *completion = cluster.aio_create_completion();
4649 ASSERT_EQ(0, cache_ioctx.aio_operate(
4650 "foo", completion, &op,
4651 librados::OPERATION_IGNORE_OVERLAY |
4652 librados::OPERATION_SKIPRWLOCKS, NULL));
4654 ObjectReadOperation op2;
4656 librados::AioCompletion *completion2 = cluster.aio_create_completion();
4657 ASSERT_EQ(0, cache_ioctx.aio_operate(
4658 "foo", completion2, &op2,
4659 librados::OPERATION_IGNORE_OVERLAY, NULL));
4661 completion->wait_for_safe();
4662 completion2->wait_for_safe();
4663 int r = completion->get_return_value();
4664 ASSERT_TRUE(r == -EBUSY || r == 0);
4665 ASSERT_EQ(0, completion2->get_return_value());
4666 completion->release();
4667 completion2->release();
4670 cout << "didn't get EBUSY, trying again" << std::endl;
4672 ASSERT_TRUE(--tries);
4675 // create/dirty object
4678 bl.append("hi there");
4679 ObjectWriteOperation op;
4681 ASSERT_EQ(0, ioctx.operate("foo", &op));
4684 // try-flush + try-flush
4686 ObjectReadOperation op;
4687 op.cache_try_flush();
4688 librados::AioCompletion *completion = cluster.aio_create_completion();
4689 ASSERT_EQ(0, cache_ioctx.aio_operate(
4690 "foo", completion, &op,
4691 librados::OPERATION_IGNORE_OVERLAY |
4692 librados::OPERATION_SKIPRWLOCKS, NULL));
4694 ObjectReadOperation op2;
4695 op2.cache_try_flush();
4696 librados::AioCompletion *completion2 = cluster.aio_create_completion();
4697 ASSERT_EQ(0, cache_ioctx.aio_operate(
4698 "foo", completion2, &op2,
4699 librados::OPERATION_IGNORE_OVERLAY |
4700 librados::OPERATION_SKIPRWLOCKS, NULL));
4702 completion->wait_for_safe();
4703 completion2->wait_for_safe();
4704 ASSERT_EQ(0, completion->get_return_value());
4705 ASSERT_EQ(0, completion2->get_return_value());
4706 completion->release();
4707 completion2->release();
4711 TEST_F(LibRadosTwoPoolsECPP, TryFlushReadRace) {
4714 ASSERT_EQ(0, cluster.mon_command(
4715 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
4716 "\", \"tierpool\": \"" + cache_pool_name +
4717 "\", \"force_nonempty\": \"--force-nonempty\" }",
4719 ASSERT_EQ(0, cluster.mon_command(
4720 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
4721 "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
4723 ASSERT_EQ(0, cluster.mon_command(
4724 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
4725 "\", \"mode\": \"writeback\"}",
4728 // wait for maps to settle
4729 cluster.wait_for_latest_osdmap();
4731 // create/dirty object
4734 bl.append("hi there");
4735 bufferptr bp(4000000); // make it big!
4738 ObjectWriteOperation op;
4740 ASSERT_EQ(0, ioctx.operate("foo", &op));
4743 // start a continuous stream of reads
4744 read_ioctx = &ioctx;
4746 for (int i = 0; i < max_reads; ++i) {
4753 ObjectReadOperation op;
4754 op.cache_try_flush();
4755 librados::AioCompletion *completion = cluster.aio_create_completion();
4756 ASSERT_EQ(0, cache_ioctx.aio_operate(
4757 "foo", completion, &op,
4758 librados::OPERATION_IGNORE_OVERLAY |
4759 librados::OPERATION_SKIPRWLOCKS, NULL));
4761 completion->wait_for_safe();
4762 ASSERT_EQ(0, completion->get_return_value());
4763 completion->release();
4768 while (num_reads > 0)
4769 cond.Wait(test_lock);
4773 TEST_F(LibRadosTierECPP, CallForcesPromote) {
4775 std::string pool_name = get_temp_pool_name();
4776 std::string cache_pool_name = pool_name + "-cache";
4777 ASSERT_EQ("", create_one_ec_pool_pp(pool_name, cluster));
4778 ASSERT_EQ(0, cluster.pool_create(cache_pool_name.c_str()));
4780 ASSERT_EQ(0, cluster.ioctx_create(cache_pool_name.c_str(), cache_ioctx));
4781 cache_ioctx.application_enable("rados", true);
4783 ASSERT_EQ(0, cluster.ioctx_create(pool_name.c_str(), ioctx));
4787 ASSERT_EQ(0, cluster.mon_command(
4788 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
4789 "\", \"tierpool\": \"" + cache_pool_name + "\"}",
4791 ASSERT_EQ(0, cluster.mon_command(
4792 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
4793 "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
4795 ASSERT_EQ(0, cluster.mon_command(
4796 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
4797 "\", \"mode\": \"writeback\"}",
4800 // set things up such that the op would normally be proxied
4801 ASSERT_EQ(0, cluster.mon_command(
4802 set_pool_str(cache_pool_name, "hit_set_count", 2),
4804 ASSERT_EQ(0, cluster.mon_command(
4805 set_pool_str(cache_pool_name, "hit_set_period", 600),
4807 ASSERT_EQ(0, cluster.mon_command(
4808 set_pool_str(cache_pool_name, "hit_set_type",
4811 ASSERT_EQ(0, cluster.mon_command(
4812 set_pool_str(cache_pool_name, "min_read_recency_for_promote",
4816 // wait for maps to settle
4817 cluster.wait_for_latest_osdmap();
4819 // create/dirty object
4821 bl.append("hi there");
4823 ObjectWriteOperation op;
4825 ASSERT_EQ(0, ioctx.operate("foo", &op));
4830 ObjectReadOperation op;
4832 librados::AioCompletion *completion = cluster.aio_create_completion();
4833 ASSERT_EQ(0, cache_ioctx.aio_operate(
4834 "foo", completion, &op,
4835 librados::OPERATION_IGNORE_OVERLAY, NULL));
4836 completion->wait_for_safe();
4837 ASSERT_EQ(0, completion->get_return_value());
4838 completion->release();
4843 ObjectReadOperation op;
4845 librados::AioCompletion *completion = cluster.aio_create_completion();
4846 ASSERT_EQ(0, cache_ioctx.aio_operate("foo", completion, &op,
4847 librados::OPERATION_IGNORE_CACHE,
4849 completion->wait_for_safe();
4850 ASSERT_EQ(0, completion->get_return_value());
4851 completion->release();
4856 ObjectReadOperation op;
4858 op.exec("rbd", "get_id", bl);
4860 // should get EIO (not an rbd object), not -EOPNOTSUPP (we didn't promote)
4861 ASSERT_EQ(-5, ioctx.operate("foo", &op, &out));
4864 // make sure foo is back in the cache tier
4866 NObjectIterator it = cache_ioctx.nobjects_begin();
4867 ASSERT_TRUE(it != cache_ioctx.nobjects_end());
4868 ASSERT_TRUE(it->get_oid() == string("foo"));
4870 ASSERT_TRUE(it == cache_ioctx.nobjects_end());
4874 ASSERT_EQ(0, cluster.mon_command(
4875 "{\"prefix\": \"osd tier remove-overlay\", \"pool\": \"" + pool_name +
4878 ASSERT_EQ(0, cluster.mon_command(
4879 "{\"prefix\": \"osd tier remove\", \"pool\": \"" + pool_name +
4880 "\", \"tierpool\": \"" + cache_pool_name + "\"}",
4883 // wait for maps to settle before next test
4884 cluster.wait_for_latest_osdmap();
4886 ASSERT_EQ(0, cluster.pool_delete(cache_pool_name.c_str()));
4887 ASSERT_EQ(0, destroy_one_pool_pp(pool_name, cluster));
4890 TEST_F(LibRadosTierECPP, HitSetNone) {
4892 list< pair<time_t,time_t> > ls;
4893 AioCompletion *c = librados::Rados::aio_create_completion();
4894 ASSERT_EQ(0, ioctx.hit_set_list(123, c, &ls));
4895 c->wait_for_complete();
4896 ASSERT_EQ(0, c->get_return_value());
4897 ASSERT_TRUE(ls.empty());
4902 AioCompletion *c = librados::Rados::aio_create_completion();
4903 ASSERT_EQ(0, ioctx.hit_set_get(123, c, 12345, &bl));
4904 c->wait_for_complete();
4905 ASSERT_EQ(-ENOENT, c->get_return_value());
4910 TEST_F(LibRadosTwoPoolsECPP, HitSetRead) {
4913 ASSERT_EQ(0, cluster.mon_command(
4914 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
4915 "\", \"tierpool\": \"" + cache_pool_name +
4916 "\", \"force_nonempty\": \"--force-nonempty\" }",
4919 // enable hitset tracking for this pool
4920 ASSERT_EQ(0, cluster.mon_command(set_pool_str(cache_pool_name, "hit_set_count", 2),
4922 ASSERT_EQ(0, cluster.mon_command(set_pool_str(cache_pool_name, "hit_set_period", 600),
4924 ASSERT_EQ(0, cluster.mon_command(set_pool_str(cache_pool_name, "hit_set_type",
4928 // wait for maps to settle
4929 cluster.wait_for_latest_osdmap();
4931 cache_ioctx.set_namespace("");
4933 // keep reading until we see our object appear in the HitSet
4934 utime_t start = ceph_clock_now();
4935 utime_t hard_stop = start + utime_t(600, 0);
4938 utime_t now = ceph_clock_now();
4939 ASSERT_TRUE(now < hard_stop);
4941 string name = "foo";
4943 ASSERT_EQ(0, cache_ioctx.get_object_hash_position2(name, &hash));
4944 hobject_t oid(sobject_t(name, CEPH_NOSNAP), "", hash,
4945 cluster.pool_lookup(cache_pool_name.c_str()), "");
4948 ASSERT_EQ(-ENOENT, cache_ioctx.read("foo", bl, 1, 0));
4951 AioCompletion *c = librados::Rados::aio_create_completion();
4952 ASSERT_EQ(0, cache_ioctx.hit_set_get(hash, c, now.sec(), &hbl));
4953 c->wait_for_complete();
4957 bufferlist::iterator p = hbl.begin();
4960 if (hs.contains(oid)) {
4961 cout << "ok, hit_set contains " << oid << std::endl;
4964 cout << "hmm, not in HitSet yet" << std::endl;
4966 cout << "hmm, no HitSet yet" << std::endl;
4973 // disable this test until hitset-get reliably works on EC pools
4975 TEST_F(LibRadosTierECPP, HitSetWrite) {
4976 int num_pg = _get_pg_num(cluster, pool_name);
4979 // enable hitset tracking for this pool
4981 ASSERT_EQ(0, cluster.mon_command(set_pool_str(pool_name, "hit_set_count", 8),
4983 ASSERT_EQ(0, cluster.mon_command(set_pool_str(pool_name, "hit_set_period", 600),
4985 ASSERT_EQ(0, cluster.mon_command(set_pool_str(pool_name, "hit_set_type",
4989 // wait for maps to settle
4990 cluster.wait_for_latest_osdmap();
4992 ioctx.set_namespace("");
4994 // do a bunch of writes
4995 for (int i=0; i<1000; ++i) {
4998 ASSERT_EQ(0, ioctx.write(stringify(i), bl, 1, 0));
5002 std::map<int,HitSet> hitsets;
5003 for (int i=0; i<num_pg; ++i) {
5004 list< pair<time_t,time_t> > ls;
5005 AioCompletion *c = librados::Rados::aio_create_completion();
5006 ASSERT_EQ(0, ioctx.hit_set_list(i, c, &ls));
5007 c->wait_for_complete();
5009 std::cout << "pg " << i << " ls " << ls << std::endl;
5010 ASSERT_FALSE(ls.empty());
5013 c = librados::Rados::aio_create_completion();
5015 ASSERT_EQ(0, ioctx.hit_set_get(i, c, ls.back().first, &bl));
5016 c->wait_for_complete();
5019 //std::cout << "bl len is " << bl.length() << "\n";
5020 //bl.hexdump(std::cout);
5021 //std::cout << std::endl;
5023 bufferlist::iterator p = bl.begin();
5024 ::decode(hitsets[i], p);
5026 // cope with racing splits by refreshing pg_num
5027 if (i == num_pg - 1)
5028 num_pg = _get_pg_num(cluster, pool_name);
5031 for (int i=0; i<1000; ++i) {
5032 string n = stringify(i);
5033 uint32_t hash = ioctx.get_object_hash_position(n);
5034 hobject_t oid(sobject_t(n, CEPH_NOSNAP), "", hash,
5035 cluster.pool_lookup(pool_name.c_str()), "");
5036 std::cout << "checking for " << oid << std::endl;
5038 for (int p=0; p<num_pg; ++p) {
5039 if (hitsets[p].contains(oid)) {
5049 TEST_F(LibRadosTwoPoolsECPP, HitSetTrim) {
5051 unsigned period = 3;
5055 ASSERT_EQ(0, cluster.mon_command(
5056 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
5057 "\", \"tierpool\": \"" + cache_pool_name +
5058 "\", \"force_nonempty\": \"--force-nonempty\" }",
5061 // enable hitset tracking for this pool
5062 ASSERT_EQ(0, cluster.mon_command(set_pool_str(cache_pool_name, "hit_set_count", count),
5064 ASSERT_EQ(0, cluster.mon_command(set_pool_str(cache_pool_name, "hit_set_period", period),
5066 ASSERT_EQ(0, cluster.mon_command(set_pool_str(cache_pool_name, "hit_set_type", "bloom"),
5068 ASSERT_EQ(0, cluster.mon_command(set_pool_str(cache_pool_name, "hit_set_fpp", ".01"),
5071 // wait for maps to settle
5072 cluster.wait_for_latest_osdmap();
5074 cache_ioctx.set_namespace("");
5076 // do a bunch of writes and make sure the hitsets rotate
5077 utime_t start = ceph_clock_now();
5078 utime_t hard_stop = start + utime_t(count * period * 50, 0);
5081 int bsize = alignment;
5082 char *buf = (char *)new char[bsize];
5083 memset(buf, 'f', bsize);
5086 string name = "foo";
5088 ASSERT_EQ(0, cache_ioctx.get_object_hash_position2(name, &hash));
5089 hobject_t oid(sobject_t(name, CEPH_NOSNAP), "", hash, -1, "");
5092 bl.append(buf, bsize);
5093 ASSERT_EQ(0, cache_ioctx.append("foo", bl, bsize));
5095 list<pair<time_t, time_t> > ls;
5096 AioCompletion *c = librados::Rados::aio_create_completion();
5097 ASSERT_EQ(0, cache_ioctx.hit_set_list(hash, c, &ls));
5098 c->wait_for_complete();
5101 cout << " got ls " << ls << std::endl;
5104 first = ls.front().first;
5105 cout << "first is " << first << std::endl;
5107 if (ls.front().first != first) {
5108 cout << "first now " << ls.front().first << ", trimmed" << std::endl;
5114 utime_t now = ceph_clock_now();
5115 ASSERT_TRUE(now < hard_stop);
5122 TEST_F(LibRadosTwoPoolsECPP, PromoteOn2ndRead) {
5124 for (int i=0; i<20; ++i) {
5126 bl.append("hi there");
5127 ObjectWriteOperation op;
5129 ASSERT_EQ(0, ioctx.operate("foo" + stringify(i), &op));
5134 ASSERT_EQ(0, cluster.mon_command(
5135 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
5136 "\", \"tierpool\": \"" + cache_pool_name +
5137 "\", \"force_nonempty\": \"--force-nonempty\" }",
5139 ASSERT_EQ(0, cluster.mon_command(
5140 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
5141 "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
5143 ASSERT_EQ(0, cluster.mon_command(
5144 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
5145 "\", \"mode\": \"writeback\"}",
5148 // enable hitset tracking for this pool
5149 ASSERT_EQ(0, cluster.mon_command(
5150 set_pool_str(cache_pool_name, "hit_set_count", 2),
5152 ASSERT_EQ(0, cluster.mon_command(
5153 set_pool_str(cache_pool_name, "hit_set_period", 600),
5155 ASSERT_EQ(0, cluster.mon_command(
5156 set_pool_str(cache_pool_name, "hit_set_type", "bloom"),
5158 ASSERT_EQ(0, cluster.mon_command(
5159 set_pool_str(cache_pool_name, "min_read_recency_for_promote", 1),
5161 ASSERT_EQ(0, cluster.mon_command(
5162 set_pool_str(cache_pool_name, "hit_set_grade_decay_rate", 20),
5164 ASSERT_EQ(0, cluster.mon_command(
5165 set_pool_str(cache_pool_name, "hit_set_search_last_n", 1),
5168 // wait for maps to settle
5169 cluster.wait_for_latest_osdmap();
5171 int fake = 0; // set this to non-zero to test spurious promotion,
5172 // e.g. from thrashing
5176 // 1st read, don't trigger a promote
5177 obj = "foo" + stringify(attempt);
5178 cout << obj << std::endl;
5181 ASSERT_EQ(1, ioctx.read(obj.c_str(), bl, 1, 0));
5184 ASSERT_EQ(1, ioctx.read(obj.c_str(), bl, 1, 0));
5189 // verify the object is NOT present in the cache tier
5192 NObjectIterator it = cache_ioctx.nobjects_begin();
5193 while (it != cache_ioctx.nobjects_end()) {
5194 cout << " see " << it->get_oid() << std::endl;
5195 if (it->get_oid() == string(obj.c_str())) {
5206 ASSERT_LE(attempt, 20);
5207 cout << "hrm, object is present in cache on attempt " << attempt
5208 << ", retrying" << std::endl;
5211 // Read until the object is present in the cache tier
5212 cout << "verifying " << obj << " is eventually promoted" << std::endl;
5215 ASSERT_EQ(1, ioctx.read(obj.c_str(), bl, 1, 0));
5218 NObjectIterator it = cache_ioctx.nobjects_begin();
5219 while (it != cache_ioctx.nobjects_end()) {
5220 if (it->get_oid() == string(obj.c_str())) {
5233 ASSERT_EQ(0, cluster.mon_command(
5234 "{\"prefix\": \"osd tier remove-overlay\", \"pool\": \"" + pool_name +
5237 ASSERT_EQ(0, cluster.mon_command(
5238 "{\"prefix\": \"osd tier remove\", \"pool\": \"" + pool_name +
5239 "\", \"tierpool\": \"" + cache_pool_name + "\"}",
5242 // wait for maps to settle before next test
5243 cluster.wait_for_latest_osdmap();
5246 TEST_F(LibRadosTwoPoolsECPP, ProxyRead) {
5250 bl.append("hi there");
5251 ObjectWriteOperation op;
5253 ASSERT_EQ(0, ioctx.operate("foo", &op));
5258 ASSERT_EQ(0, cluster.mon_command(
5259 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
5260 "\", \"tierpool\": \"" + cache_pool_name +
5261 "\", \"force_nonempty\": \"--force-nonempty\" }",
5263 ASSERT_EQ(0, cluster.mon_command(
5264 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
5265 "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
5267 ASSERT_EQ(0, cluster.mon_command(
5268 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
5269 "\", \"mode\": \"readproxy\"}",
5272 // wait for maps to settle
5273 cluster.wait_for_latest_osdmap();
5275 // read and verify the object
5278 ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
5279 ASSERT_EQ('h', bl[0]);
5282 // Verify 10 times the object is NOT present in the cache tier
5285 NObjectIterator it = cache_ioctx.nobjects_begin();
5286 ASSERT_TRUE(it == cache_ioctx.nobjects_end());
5291 ASSERT_EQ(0, cluster.mon_command(
5292 "{\"prefix\": \"osd tier remove-overlay\", \"pool\": \"" + pool_name +
5295 ASSERT_EQ(0, cluster.mon_command(
5296 "{\"prefix\": \"osd tier remove\", \"pool\": \"" + pool_name +
5297 "\", \"tierpool\": \"" + cache_pool_name + "\"}",
5300 // wait for maps to settle before next test
5301 cluster.wait_for_latest_osdmap();
5304 TEST_F(LibRadosTwoPoolsECPP, CachePin) {
5308 bl.append("hi there");
5309 ObjectWriteOperation op;
5311 ASSERT_EQ(0, ioctx.operate("foo", &op));
5315 bl.append("hi there");
5316 ObjectWriteOperation op;
5318 ASSERT_EQ(0, ioctx.operate("bar", &op));
5322 bl.append("hi there");
5323 ObjectWriteOperation op;
5325 ASSERT_EQ(0, ioctx.operate("baz", &op));
5329 bl.append("hi there");
5330 ObjectWriteOperation op;
5332 ASSERT_EQ(0, ioctx.operate("bam", &op));
5337 ASSERT_EQ(0, cluster.mon_command(
5338 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
5339 "\", \"tierpool\": \"" + cache_pool_name +
5340 "\", \"force_nonempty\": \"--force-nonempty\" }",
5342 ASSERT_EQ(0, cluster.mon_command(
5343 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
5344 "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
5346 ASSERT_EQ(0, cluster.mon_command(
5347 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
5348 "\", \"mode\": \"writeback\"}",
5351 // wait for maps to settle
5352 cluster.wait_for_latest_osdmap();
5354 // read, trigger promote
5357 ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
5358 ASSERT_EQ(1, ioctx.read("bar", bl, 1, 0));
5359 ASSERT_EQ(1, ioctx.read("baz", bl, 1, 0));
5360 ASSERT_EQ(1, ioctx.read("bam", bl, 1, 0));
5363 // verify the objects are present in the cache tier
5365 NObjectIterator it = cache_ioctx.nobjects_begin();
5366 ASSERT_TRUE(it != cache_ioctx.nobjects_end());
5367 for (uint32_t i = 0; i < 4; i++) {
5368 ASSERT_TRUE(it->get_oid() == string("foo") ||
5369 it->get_oid() == string("bar") ||
5370 it->get_oid() == string("baz") ||
5371 it->get_oid() == string("bam"));
5374 ASSERT_TRUE(it == cache_ioctx.nobjects_end());
5379 ObjectWriteOperation op;
5381 librados::AioCompletion *completion = cluster.aio_create_completion();
5382 ASSERT_EQ(0, cache_ioctx.aio_operate("foo", completion, &op));
5383 completion->wait_for_safe();
5384 ASSERT_EQ(0, completion->get_return_value());
5385 completion->release();
5388 ObjectWriteOperation op;
5390 librados::AioCompletion *completion = cluster.aio_create_completion();
5391 ASSERT_EQ(0, cache_ioctx.aio_operate("baz", completion, &op));
5392 completion->wait_for_safe();
5393 ASSERT_EQ(0, completion->get_return_value());
5394 completion->release();
5398 ASSERT_EQ(0, cluster.mon_command(
5399 set_pool_str(cache_pool_name, "hit_set_count", 2),
5401 ASSERT_EQ(0, cluster.mon_command(
5402 set_pool_str(cache_pool_name, "hit_set_period", 600),
5404 ASSERT_EQ(0, cluster.mon_command(
5405 set_pool_str(cache_pool_name, "hit_set_type", "bloom"),
5407 ASSERT_EQ(0, cluster.mon_command(
5408 set_pool_str(cache_pool_name, "min_read_recency_for_promote", 1),
5410 ASSERT_EQ(0, cluster.mon_command(
5411 set_pool_str(cache_pool_name, "target_max_objects", 1),
5416 // Verify the pinned object 'foo' is not flushed/evicted
5420 ASSERT_EQ(1, ioctx.read("baz", bl, 1, 0));
5423 NObjectIterator it = cache_ioctx.nobjects_begin();
5424 while (it != cache_ioctx.nobjects_end()) {
5425 ASSERT_TRUE(it->get_oid() == string("foo") ||
5426 it->get_oid() == string("bar") ||
5427 it->get_oid() == string("baz") ||
5428 it->get_oid() == string("bam"));
5433 ASSERT_TRUE(it->get_oid() == string("foo") ||
5434 it->get_oid() == string("baz"));
5442 ASSERT_EQ(0, cluster.mon_command(
5443 "{\"prefix\": \"osd tier remove-overlay\", \"pool\": \"" + pool_name +
5446 ASSERT_EQ(0, cluster.mon_command(
5447 "{\"prefix\": \"osd tier remove\", \"pool\": \"" + pool_name +
5448 "\", \"tierpool\": \"" + cache_pool_name + "\"}",
5451 // wait for maps to settle before next test
5452 cluster.wait_for_latest_osdmap();
5454 TEST_F(LibRadosTwoPoolsECPP, SetRedirectRead) {
5455 // skip test if not yet luminous
5457 bufferlist inbl, outbl;
5458 ASSERT_EQ(0, cluster.mon_command(
5459 "{\"prefix\": \"osd dump\"}",
5460 inbl, &outbl, NULL));
5461 string s(outbl.c_str(), outbl.length());
5462 if (s.find("luminous") == std::string::npos) {
5463 cout << "cluster is not yet luminous, skipping test" << std::endl;
5471 bl.append("hi there");
5472 ObjectWriteOperation op;
5474 ASSERT_EQ(0, ioctx.operate("foo", &op));
5479 ObjectWriteOperation op;
5481 ASSERT_EQ(0, cache_ioctx.operate("bar", &op));
5486 ASSERT_EQ(0, cluster.mon_command(
5487 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
5488 "\", \"tierpool\": \"" + cache_pool_name +
5489 "\", \"force_nonempty\": \"--force-nonempty\" }",
5492 // wait for maps to settle
5493 cluster.wait_for_latest_osdmap();
5496 ObjectWriteOperation op;
5497 op.set_redirect("bar", cache_ioctx, 0);
5498 librados::AioCompletion *completion = cluster.aio_create_completion();
5499 ASSERT_EQ(0, ioctx.aio_operate("foo", completion, &op));
5500 completion->wait_for_safe();
5501 ASSERT_EQ(0, completion->get_return_value());
5502 completion->release();
5504 // read and verify the object
5507 ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
5508 ASSERT_EQ('t', bl[0]);
5511 ASSERT_EQ(0, cluster.mon_command(
5512 "{\"prefix\": \"osd tier remove\", \"pool\": \"" + pool_name +
5513 "\", \"tierpool\": \"" + cache_pool_name + "\"}",
5516 // wait for maps to settle before next test
5517 cluster.wait_for_latest_osdmap();