1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
2 // vim: ts=8 sw=2 smarttab
4 * Ceph - scalable distributed file system
6 * Copyright (C) 2004-2006 Sage Weil <sage@newdream.net>
8 * This is free software; you can redistribute it and/or
9 * modify it under the terms of the GNU Lesser General Public
10 * License version 2.1, as published by the Free Software
11 * Foundation. See file COPYING.
20 #include <sys/mount.h>
21 #include <boost/scoped_ptr.hpp>
22 #include <boost/random/mersenne_twister.hpp>
23 #include <boost/random/uniform_int.hpp>
24 #include <boost/random/binomial_distribution.hpp>
25 #include <gtest/gtest.h>
27 #include "os/ObjectStore.h"
28 #include "os/filestore/FileStore.h"
29 #if defined(HAVE_LIBAIO)
30 #include "os/bluestore/BlueStore.h"
32 #include "include/Context.h"
33 #include "common/ceph_argparse.h"
34 #include "global/global_init.h"
35 #include "common/Mutex.h"
36 #include "common/Cond.h"
37 #include "common/errno.h"
38 #include "include/stringify.h"
39 #include "include/coredumpctl.h"
41 #include "include/unordered_map.h"
42 #include "store_test_fixture.h"
45 typedef boost::mt11213b gen_type;
47 #define dout_context g_ceph_context
49 #if GTEST_HAS_PARAM_TEST
51 static bool bl_eq(bufferlist& expected, bufferlist& actual)
53 if (expected.contents_equal(actual))
57 if(expected.length() != actual.length()) {
58 cout << "--- buffer lengths mismatch " << std::hex
59 << "expected 0x" << expected.length() << " != actual 0x"
60 << actual.length() << std::dec << std::endl;
61 derr << "--- buffer lengths mismatch " << std::hex
62 << "expected 0x" << expected.length() << " != actual 0x"
63 << actual.length() << std::dec << dendl;
65 auto len = MIN(expected.length(), actual.length());
66 while ( first<len && expected[first] == actual[first])
69 while (last > 0 && expected[last-1] == actual[last-1])
72 cout << "--- buffer mismatch between offset 0x" << std::hex << first
73 << " and 0x" << last << ", total 0x" << len << std::dec
75 derr << "--- buffer mismatch between offset 0x" << std::hex << first
76 << " and 0x" << last << ", total 0x" << len << std::dec
78 cout << "--- expected:\n";
79 expected.hexdump(cout);
80 cout << "--- actual:\n";
89 int apply_transaction(
91 ObjectStore::Sequencer *osr,
92 ObjectStore::Transaction &&t) {
94 ObjectStore::Transaction t2;
96 return store->apply_transaction(osr, std::move(t2));
98 return store->apply_transaction(osr, std::move(t));
103 bool sorted(const vector<ghobject_t> &in) {
105 for (vector<ghobject_t>::const_iterator i = in.begin();
109 cout << start << " should follow " << *i << std::endl;
117 class StoreTest : public StoreTestFixture,
118 public ::testing::WithParamInterface<const char*> {
121 : StoreTestFixture(GetParam())
125 class StoreTestDeferredSetup : public StoreTest {
126 void SetUp() override {
131 void DeferredSetup() {
138 class StoreTestSpecificAUSize : public StoreTestDeferredSetup {
143 boost::scoped_ptr<ObjectStore>& store,
147 uint64_t align)> MatrixTest;
149 void StartDeferred(size_t min_alloc_size) {
150 g_conf->set_val("bluestore_min_alloc_size", stringify(min_alloc_size));
154 void TearDown() override {
155 g_conf->set_val("bluestore_min_alloc_size", "0");
156 StoreTestDeferredSetup::TearDown();
160 // bluestore matrix testing
161 uint64_t max_write = 40 * 1024;
162 uint64_t max_size = 400 * 1024;
163 uint64_t alignment = 0;
164 uint64_t num_ops = 10000;
167 string matrix_get(const char *k) {
168 if (string(k) == "max_write") {
169 return stringify(max_write);
170 } else if (string(k) == "max_size") {
171 return stringify(max_size);
172 } else if (string(k) == "alignment") {
173 return stringify(alignment);
174 } else if (string(k) == "num_ops") {
175 return stringify(num_ops);
178 g_conf->get_val(k, &buf, -1);
185 void matrix_set(const char *k, const char *v) {
186 if (string(k) == "max_write") {
187 max_write = atoll(v);
188 } else if (string(k) == "max_size") {
190 } else if (string(k) == "alignment") {
191 alignment = atoll(v);
192 } else if (string(k) == "num_ops") {
195 g_conf->set_val(k, v);
199 void do_matrix_choose(const char *matrix[][10],
200 int i, int pos, int num,
201 boost::scoped_ptr<ObjectStore>& store,
205 for (count = 0; matrix[i][count+1]; ++count) ;
206 for (int j = 1; matrix[i][j]; ++j) {
207 matrix_set(matrix[i][0], matrix[i][j]);
208 do_matrix_choose(matrix,
216 cout << "---------------------- " << (pos + 1) << " / " << num
217 << " ----------------------" << std::endl;
218 for (unsigned k=0; matrix[k][0]; ++k) {
219 cout << " " << matrix[k][0] << " = " << matrix_get(matrix[k][0])
222 g_ceph_context->_conf->apply_changes(NULL);
223 fn(store, num_ops, max_size, max_write, alignment);
227 void do_matrix(const char *matrix[][10],
228 boost::scoped_ptr<ObjectStore>& store,
230 map<string,string> old;
231 for (unsigned i=0; matrix[i][0]; ++i) {
232 old[matrix[i][0]] = matrix_get(matrix[i][0]);
234 cout << "saved config options " << old << std::endl;
236 if (strcmp(matrix[0][0], "bluestore_min_alloc_size") == 0) {
238 for (count = 0; matrix[0][count+1]; ++count) ;
239 for (size_t j = 1; matrix[0][j]; ++j) {
243 StartDeferred(strtoll(matrix[0][j], NULL, 10));
244 do_matrix_choose(matrix, 1, j - 1, count, store, fn);
248 do_matrix_choose(matrix, 0, 0, 1, store, fn);
251 cout << "restoring config options " << old << std::endl;
253 cout << " " << p.first << " = " << p.second << std::endl;
254 matrix_set(p.first.c_str(), p.second.c_str());
256 g_ceph_context->_conf->apply_changes(NULL);
261 TEST_P(StoreTest, collect_metadata) {
262 map<string,string> pm;
263 store->collect_metadata(&pm);
264 if (GetParam() == string("filestore")) {
265 ASSERT_NE(pm.count("filestore_backend"), 0u);
266 ASSERT_NE(pm.count("filestore_f_type"), 0u);
267 ASSERT_NE(pm.count("backend_filestore_partition_path"), 0u);
268 ASSERT_NE(pm.count("backend_filestore_dev_node"), 0u);
272 TEST_P(StoreTest, Trivial) {
275 TEST_P(StoreTest, TrivialRemount) {
276 int r = store->umount();
282 TEST_P(StoreTest, SimpleRemount) {
283 ObjectStore::Sequencer osr("test");
285 ghobject_t hoid(hobject_t(sobject_t("Object 1", CEPH_NOSNAP)));
286 ghobject_t hoid2(hobject_t(sobject_t("Object 2", CEPH_NOSNAP)));
288 bl.append("1234512345");
291 cerr << "create collection + write" << std::endl;
292 ObjectStore::Transaction t;
293 t.create_collection(cid, 0);
294 t.write(cid, hoid, 0, bl.length(), bl);
295 r = apply_transaction(store, &osr, std::move(t));
303 ObjectStore::Transaction t;
304 t.write(cid, hoid2, 0, bl.length(), bl);
305 r = apply_transaction(store, &osr, std::move(t));
309 ObjectStore::Transaction t;
311 t.remove(cid, hoid2);
312 t.remove_collection(cid);
313 cerr << "remove collection" << std::endl;
314 r = apply_transaction(store, &osr, std::move(t));
322 ObjectStore::Transaction t;
323 t.create_collection(cid, 0);
324 r = apply_transaction(store, &osr, std::move(t));
326 bool exists = store->exists(cid, hoid);
327 ASSERT_TRUE(!exists);
330 ObjectStore::Transaction t;
331 t.remove_collection(cid);
332 cerr << "remove collection" << std::endl;
333 r = apply_transaction(store, &osr, std::move(t));
338 TEST_P(StoreTest, IORemount) {
339 ObjectStore::Sequencer osr("test");
342 bl.append("1234512345");
345 cerr << "create collection + objects" << std::endl;
346 ObjectStore::Transaction t;
347 t.create_collection(cid, 0);
348 for (int n=1; n<=100; ++n) {
349 ghobject_t hoid(hobject_t(sobject_t("Object " + stringify(n), CEPH_NOSNAP)));
350 t.write(cid, hoid, 0, bl.length(), bl);
352 r = apply_transaction(store, &osr, std::move(t));
357 cout << "overwrites" << std::endl;
358 for (int n=1; n<=100; ++n) {
359 ObjectStore::Transaction t;
360 ghobject_t hoid(hobject_t(sobject_t("Object " + stringify(n), CEPH_NOSNAP)));
361 t.write(cid, hoid, 1, bl.length(), bl);
362 r = apply_transaction(store, &osr, std::move(t));
371 ObjectStore::Transaction t;
372 for (int n=1; n<=100; ++n) {
373 ghobject_t hoid(hobject_t(sobject_t("Object " + stringify(n), CEPH_NOSNAP)));
376 t.remove_collection(cid);
377 r = apply_transaction(store, &osr, std::move(t));
382 TEST_P(StoreTest, UnprintableCharsName) {
383 ObjectStore::Sequencer osr("test");
385 string name = "funnychars_";
386 for (unsigned i = 0; i < 256; ++i) {
389 ghobject_t oid(hobject_t(sobject_t(name, CEPH_NOSNAP)));
392 cerr << "create collection + object" << std::endl;
393 ObjectStore::Transaction t;
394 t.create_collection(cid, 0);
396 r = apply_transaction(store, &osr, std::move(t));
404 cout << "removing" << std::endl;
405 ObjectStore::Transaction t;
407 t.remove_collection(cid);
408 r = apply_transaction(store, &osr, std::move(t));
413 TEST_P(StoreTest, FiemapEmpty) {
414 ObjectStore::Sequencer osr("test");
417 ghobject_t oid(hobject_t(sobject_t("fiemap_object", CEPH_NOSNAP)));
419 ObjectStore::Transaction t;
420 t.create_collection(cid, 0);
422 t.truncate(cid, oid, 100000);
423 r = apply_transaction(store, &osr, std::move(t));
428 store->fiemap(cid, oid, 0, 100000, bl);
429 map<uint64_t,uint64_t> m, e;
430 bufferlist::iterator p = bl.begin();
432 cout << " got " << m << std::endl;
434 EXPECT_TRUE(m == e || m.empty());
437 ObjectStore::Transaction t;
439 t.remove_collection(cid);
440 cerr << "remove collection" << std::endl;
441 r = apply_transaction(store, &osr, std::move(t));
446 TEST_P(StoreTest, FiemapHoles) {
447 ObjectStore::Sequencer osr("test");
448 const uint64_t MAX_EXTENTS = 4000;
449 const uint64_t SKIP_STEP = 65536;
452 ghobject_t oid(hobject_t(sobject_t("fiemap_object", CEPH_NOSNAP)));
456 ObjectStore::Transaction t;
457 t.create_collection(cid, 0);
459 for (uint64_t i = 0; i < MAX_EXTENTS; i++)
460 t.write(cid, oid, SKIP_STEP * i, 3, bl);
461 r = apply_transaction(store, &osr, std::move(t));
465 //fiemap test from 0 to SKIP_STEP * (MAX_EXTENTS - 1) + 3
467 store->fiemap(cid, oid, 0, SKIP_STEP * (MAX_EXTENTS - 1) + 3, bl);
468 map<uint64_t,uint64_t> m, e;
469 bufferlist::iterator p = bl.begin();
471 cout << " got " << m << std::endl;
472 ASSERT_TRUE(!m.empty());
474 bool extents_exist = true;
475 if (m.size() == MAX_EXTENTS) {
476 for (uint64_t i = 0; i < MAX_EXTENTS; i++)
477 extents_exist = extents_exist && m.count(SKIP_STEP*i);
479 ASSERT_TRUE((m.size() == 1 &&
480 m[0] > SKIP_STEP * (MAX_EXTENTS - 1)) ||
481 (m.size() == MAX_EXTENTS && extents_exist));
483 // fiemap test from SKIP_STEP to SKIP_STEP * (MAX_EXTENTS - 2) + 3
484 // reset bufferlist and map
488 store->fiemap(cid, oid, SKIP_STEP, SKIP_STEP * (MAX_EXTENTS - 2) + 3, bl);
491 cout << " got " << m << std::endl;
492 ASSERT_TRUE(!m.empty());
493 ASSERT_GE(m[SKIP_STEP], 3u);
494 extents_exist = true;
495 if (m.size() == (MAX_EXTENTS - 2)) {
496 for (uint64_t i = 1; i < MAX_EXTENTS - 1; i++)
497 extents_exist = extents_exist && m.count(SKIP_STEP*i);
499 ASSERT_TRUE((m.size() == 1 &&
500 m[SKIP_STEP] > SKIP_STEP * (MAX_EXTENTS - 2)) ||
501 (m.size() == (MAX_EXTENTS - 1) && extents_exist));
504 ObjectStore::Transaction t;
506 t.remove_collection(cid);
507 cerr << "remove collection" << std::endl;
508 r = apply_transaction(store, &osr, std::move(t));
513 TEST_P(StoreTest, SimpleMetaColTest) {
514 ObjectStore::Sequencer osr("test");
518 ObjectStore::Transaction t;
519 t.create_collection(cid, 0);
520 cerr << "create collection" << std::endl;
521 r = apply_transaction(store, &osr, std::move(t));
525 ObjectStore::Transaction t;
526 t.remove_collection(cid);
527 cerr << "remove collection" << std::endl;
528 r = apply_transaction(store, &osr, std::move(t));
532 ObjectStore::Transaction t;
533 t.create_collection(cid, 0);
534 cerr << "add collection" << std::endl;
535 r = apply_transaction(store, &osr, std::move(t));
539 ObjectStore::Transaction t;
540 t.remove_collection(cid);
541 cerr << "remove collection" << std::endl;
542 r = apply_transaction(store, &osr, std::move(t));
547 TEST_P(StoreTest, SimplePGColTest) {
548 ObjectStore::Sequencer osr("test");
549 coll_t cid(spg_t(pg_t(1,2), shard_id_t::NO_SHARD));
552 ObjectStore::Transaction t;
553 t.create_collection(cid, 4);
554 cerr << "create collection" << std::endl;
555 r = apply_transaction(store, &osr, std::move(t));
559 ObjectStore::Transaction t;
560 t.remove_collection(cid);
561 cerr << "remove collection" << std::endl;
562 r = apply_transaction(store, &osr, std::move(t));
566 ObjectStore::Transaction t;
567 t.create_collection(cid, 4);
568 cerr << "add collection" << std::endl;
569 r = apply_transaction(store, &osr, std::move(t));
573 ObjectStore::Transaction t;
574 t.remove_collection(cid);
575 cerr << "remove collection" << std::endl;
576 r = apply_transaction(store, &osr, std::move(t));
581 TEST_P(StoreTest, SimpleColPreHashTest) {
582 ObjectStore::Sequencer osr("test");
583 // Firstly we will need to revert the value making sure
584 // collection hint actually works
585 int merge_threshold = g_ceph_context->_conf->filestore_merge_threshold;
586 std::ostringstream oss;
587 if (merge_threshold > 0) {
588 oss << "-" << merge_threshold;
589 g_ceph_context->_conf->set_val("filestore_merge_threshold", oss.str().c_str());
592 uint32_t pg_num = 128;
594 boost::uniform_int<> pg_id_range(0, pg_num);
595 gen_type rng(time(NULL));
596 int pg_id = pg_id_range(rng);
598 int objs_per_folder = abs(merge_threshold) * 16 * g_ceph_context->_conf->filestore_split_multiple;
599 boost::uniform_int<> folders_range(5, 256);
600 uint64_t expected_num_objs = (uint64_t)objs_per_folder * (uint64_t)folders_range(rng);
602 coll_t cid(spg_t(pg_t(pg_id, 15), shard_id_t::NO_SHARD));
605 // Create a collection along with a hint
606 ObjectStore::Transaction t;
607 t.create_collection(cid, 5);
608 cerr << "create collection" << std::endl;
610 ::encode(pg_num, hint);
611 ::encode(expected_num_objs, hint);
612 t.collection_hint(cid, ObjectStore::Transaction::COLL_HINT_EXPECTED_NUM_OBJECTS, hint);
613 cerr << "collection hint" << std::endl;
614 r = apply_transaction(store, &osr, std::move(t));
618 // Remove the collection
619 ObjectStore::Transaction t;
620 t.remove_collection(cid);
621 cerr << "remove collection" << std::endl;
622 r = apply_transaction(store, &osr, std::move(t));
625 // Revert the config change so that it does not affect the split/merge tests
626 if (merge_threshold > 0) {
628 oss << merge_threshold;
629 g_ceph_context->_conf->set_val("filestore_merge_threshold", oss.str().c_str());
633 TEST_P(StoreTest, SmallBlockWrites) {
634 ObjectStore::Sequencer osr("test");
637 ghobject_t hoid(hobject_t(sobject_t("foo", CEPH_NOSNAP)));
639 ObjectStore::Transaction t;
640 t.create_collection(cid, 0);
641 cerr << "Creating collection " << cid << std::endl;
642 r = apply_transaction(store, &osr, std::move(t));
646 bufferptr ap(0x1000);
647 memset(ap.c_str(), 'a', 0x1000);
650 bufferptr bp(0x1000);
651 memset(bp.c_str(), 'b', 0x1000);
654 bufferptr cp(0x1000);
655 memset(cp.c_str(), 'c', 0x1000);
657 bufferptr zp(0x1000);
662 ObjectStore::Transaction t;
663 t.write(cid, hoid, 0, 0x1000, a);
664 r = apply_transaction(store, &osr, std::move(t));
668 r = store->read(cid, hoid, 0, 0x4000, in);
669 ASSERT_EQ(0x1000, r);
671 ASSERT_TRUE(bl_eq(exp, in));
674 ObjectStore::Transaction t;
675 t.write(cid, hoid, 0x1000, 0x1000, b);
676 r = apply_transaction(store, &osr, std::move(t));
680 r = store->read(cid, hoid, 0, 0x4000, in);
681 ASSERT_EQ(0x2000, r);
684 ASSERT_TRUE(bl_eq(exp, in));
687 ObjectStore::Transaction t;
688 t.write(cid, hoid, 0x3000, 0x1000, c);
689 r = apply_transaction(store, &osr, std::move(t));
693 r = store->read(cid, hoid, 0, 0x4000, in);
694 ASSERT_EQ(0x4000, r);
699 ASSERT_TRUE(bl_eq(exp, in));
702 ObjectStore::Transaction t;
703 t.write(cid, hoid, 0x2000, 0x1000, a);
704 r = apply_transaction(store, &osr, std::move(t));
708 r = store->read(cid, hoid, 0, 0x4000, in);
709 ASSERT_EQ(0x4000, r);
714 ASSERT_TRUE(bl_eq(exp, in));
717 ObjectStore::Transaction t;
718 t.write(cid, hoid, 0, 0x1000, c);
719 r = apply_transaction(store, &osr, std::move(t));
724 r = store->read(cid, hoid, 0, 0x4000, in);
725 ASSERT_EQ(0x4000, r);
730 ASSERT_TRUE(bl_eq(exp, in));
733 ObjectStore::Transaction t;
735 t.remove_collection(cid);
736 cerr << "Cleaning" << std::endl;
737 r = apply_transaction(store, &osr, std::move(t));
742 TEST_P(StoreTest, BufferCacheReadTest) {
743 ObjectStore::Sequencer osr("test");
746 ghobject_t hoid(hobject_t(sobject_t("Object 1", CEPH_NOSNAP)));
749 r = store->read(cid, hoid, 0, 5, in);
750 ASSERT_EQ(-ENOENT, r);
753 ObjectStore::Transaction t;
754 t.create_collection(cid, 0);
755 cerr << "Creating collection " << cid << std::endl;
756 r = apply_transaction(store, &osr, std::move(t));
760 bool exists = store->exists(cid, hoid);
761 ASSERT_TRUE(!exists);
763 ObjectStore::Transaction t;
765 cerr << "Creating object " << hoid << std::endl;
766 r = apply_transaction(store, &osr, std::move(t));
769 exists = store->exists(cid, hoid);
770 ASSERT_EQ(true, exists);
773 ObjectStore::Transaction t;
774 bufferlist bl, newdata;
776 t.write(cid, hoid, 0, 5, bl);
777 t.write(cid, hoid, 10, 5, bl);
778 cerr << "TwinWrite" << std::endl;
779 r = apply_transaction(store, &osr, std::move(t));
782 r = store->read(cid, hoid, 0, 15, newdata);
787 expected.append_zero(5);
789 ASSERT_TRUE(bl_eq(expected, newdata));
792 //overwrite over the same extents
794 ObjectStore::Transaction t;
795 bufferlist bl, newdata;
797 t.write(cid, hoid, 0, 5, bl);
798 t.write(cid, hoid, 10, 5, bl);
799 cerr << "TwinWrite" << std::endl;
800 r = apply_transaction(store, &osr, std::move(t));
803 r = store->read(cid, hoid, 0, 15, newdata);
808 expected.append_zero(5);
810 ASSERT_TRUE(bl_eq(expected, newdata));
813 //additional write to an unused region of some blob
815 ObjectStore::Transaction t;
816 bufferlist bl2, newdata;
817 bl2.append("1234567890");
819 t.write(cid, hoid, 20, bl2.length(), bl2);
820 cerr << "Append" << std::endl;
821 r = apply_transaction(store, &osr, std::move(t));
824 r = store->read(cid, hoid, 0, 30, newdata);
828 expected.append("edcba");
829 expected.append_zero(5);
830 expected.append("edcba");
831 expected.append_zero(5);
832 expected.append(bl2);
834 ASSERT_TRUE(bl_eq(expected, newdata));
837 //additional write to an unused region of some blob and partial owerite over existing extents
839 ObjectStore::Transaction t;
840 bufferlist bl, bl2, bl3, newdata;
842 bl2.append("1234567890");
845 t.write(cid, hoid, 30, bl2.length(), bl2);
846 t.write(cid, hoid, 1, bl.length(), bl);
847 t.write(cid, hoid, 13, bl3.length(), bl3);
848 cerr << "TripleWrite" << std::endl;
849 r = apply_transaction(store, &osr, std::move(t));
852 r = store->read(cid, hoid, 0, 40, newdata);
856 expected.append("eDCBa");
857 expected.append_zero(5);
858 expected.append("edcBA");
859 expected.append_zero(5);
860 expected.append(bl2);
861 expected.append(bl2);
863 ASSERT_TRUE(bl_eq(expected, newdata));
868 void doCompressionTest( boost::scoped_ptr<ObjectStore>& store)
870 ObjectStore::Sequencer osr("test");
873 ghobject_t hoid(hobject_t(sobject_t("Object 1", CEPH_NOSNAP)));
876 r = store->read(cid, hoid, 0, 5, in);
877 ASSERT_EQ(-ENOENT, r);
880 ObjectStore::Transaction t;
881 t.create_collection(cid, 0);
882 cerr << "Creating collection " << cid << std::endl;
883 r = apply_transaction(store, &osr, std::move(t));
887 bool exists = store->exists(cid, hoid);
888 ASSERT_TRUE(!exists);
890 ObjectStore::Transaction t;
892 cerr << "Creating object " << hoid << std::endl;
893 r = apply_transaction(store, &osr, std::move(t));
896 exists = store->exists(cid, hoid);
897 ASSERT_EQ(true, exists);
900 data.resize(0x10000 * 4);
901 for(size_t i = 0;i < data.size(); i++)
904 ObjectStore::Transaction t;
905 bufferlist bl, newdata;
907 t.write(cid, hoid, 0, bl.length(), bl);
908 cerr << "CompressibleData (4xAU) Write" << std::endl;
909 r = apply_transaction(store, &osr, std::move(t));
912 r = store->read(cid, hoid, 0, data.size() , newdata);
914 ASSERT_EQ(r, (int)data.size());
917 expected.append(data);
918 ASSERT_TRUE(bl_eq(expected, newdata));
921 r = store->read(cid, hoid, 0, 711 , newdata);
925 expected.append(data.substr(0,711));
926 ASSERT_TRUE(bl_eq(expected, newdata));
929 r = store->read(cid, hoid, 0xf00f, data.size(), newdata);
930 ASSERT_EQ(r, int(data.size() - 0xf00f) );
933 expected.append(data.substr(0xf00f));
934 ASSERT_TRUE(bl_eq(expected, newdata));
937 struct store_statfs_t statfs;
938 int r = store->statfs(&statfs);
940 ASSERT_EQ(statfs.stored, (unsigned)data.size());
941 ASSERT_LE(statfs.compressed, (unsigned)data.size());
942 ASSERT_EQ(statfs.compressed_original, (unsigned)data.size());
943 ASSERT_LE(statfs.compressed_allocated, (unsigned)data.size());
947 data2.resize(0x10000 * 4 - 0x9000);
948 for(size_t i = 0;i < data2.size(); i++)
949 data2[i] = (i+1) / 256;
951 ObjectStore::Transaction t;
952 bufferlist bl, newdata;
954 t.write(cid, hoid, 0x8000, bl.length(), bl);
955 cerr << "CompressibleData partial overwrite" << std::endl;
956 r = apply_transaction(store, &osr, std::move(t));
959 r = store->read(cid, hoid, 0, 0x10000, newdata);
960 ASSERT_EQ(r, (int)0x10000);
963 expected.append(data.substr(0, 0x8000));
964 expected.append(data2.substr(0, 0x8000));
965 ASSERT_TRUE(bl_eq(expected, newdata));
968 r = store->read(cid, hoid, 0x9000, 711 , newdata);
972 expected.append(data2.substr(0x1000,711));
973 ASSERT_TRUE(bl_eq(expected, newdata));
976 r = store->read(cid, hoid, 0x0, 0x40000, newdata);
977 ASSERT_EQ(r, int(0x40000) );
980 expected.append(data.substr(0, 0x8000));
981 expected.append(data2.substr(0, 0x37000));
982 expected.append(data.substr(0x3f000, 0x1000));
983 ASSERT_TRUE(bl_eq(expected, newdata));
986 data2.resize(0x3f000);
987 for(size_t i = 0;i < data2.size(); i++)
988 data2[i] = (i+2) / 256;
990 ObjectStore::Transaction t;
991 bufferlist bl, newdata;
993 t.write(cid, hoid, 0, bl.length(), bl);
994 cerr << "CompressibleData partial overwrite, two extents overlapped, single one to be removed" << std::endl;
995 r = apply_transaction(store, &osr, std::move(t));
998 r = store->read(cid, hoid, 0, 0x3e000 - 1, newdata);
999 ASSERT_EQ(r, (int)0x3e000 - 1);
1001 bufferlist expected;
1002 expected.append(data2.substr(0, 0x3e000 - 1));
1003 ASSERT_TRUE(bl_eq(expected, newdata));
1006 r = store->read(cid, hoid, 0x3e000-1, 0x2001, newdata);
1007 ASSERT_EQ(r, 0x2001);
1009 bufferlist expected;
1010 expected.append(data2.substr(0x3e000-1, 0x1001));
1011 expected.append(data.substr(0x3f000, 0x1000));
1012 ASSERT_TRUE(bl_eq(expected, newdata));
1015 r = store->read(cid, hoid, 0x0, 0x40000, newdata);
1016 ASSERT_EQ(r, int(0x40000) );
1018 bufferlist expected;
1019 expected.append(data2.substr(0, 0x3f000));
1020 expected.append(data.substr(0x3f000, 0x1000));
1021 ASSERT_TRUE(bl_eq(expected, newdata));
1024 data.resize(0x1001);
1025 for(size_t i = 0;i < data.size(); i++)
1026 data[i] = (i+3) / 256;
1028 ObjectStore::Transaction t;
1029 bufferlist bl, newdata;
1031 t.write(cid, hoid, 0x3f000-1, bl.length(), bl);
1032 cerr << "Small chunk partial overwrite, two extents overlapped, single one to be removed" << std::endl;
1033 r = apply_transaction(store, &osr, std::move(t));
1036 r = store->read(cid, hoid, 0x3e000, 0x2000, newdata);
1037 ASSERT_EQ(r, (int)0x2000);
1039 bufferlist expected;
1040 expected.append(data2.substr(0x3e000, 0x1000 - 1));
1041 expected.append(data.substr(0, 0x1001));
1042 ASSERT_TRUE(bl_eq(expected, newdata));
1046 ObjectStore::Transaction t;
1047 t.remove(cid, hoid);
1048 cerr << "Cleaning object" << std::endl;
1049 r = apply_transaction(store, &osr, std::move(t));
1053 EXPECT_EQ(store->umount(), 0);
1054 EXPECT_EQ(store->mount(), 0);
1055 auto orig_min_blob_size = g_conf->bluestore_compression_min_blob_size;
1057 g_conf->set_val("bluestore_compression_min_blob_size", "262144");
1058 g_ceph_context->_conf->apply_changes(NULL);
1059 data.resize(0x10000*6);
1061 for(size_t i = 0;i < data.size(); i++)
1063 ObjectStore::Transaction t;
1064 bufferlist bl, newdata;
1066 t.write(cid, hoid, 0, bl.length(), bl);
1067 cerr << "CompressibleData large blob" << std::endl;
1068 r = apply_transaction(store, &osr, std::move(t));
1072 EXPECT_EQ(store->umount(), 0);
1073 EXPECT_EQ(store->mount(), 0);
1076 ObjectStore::Transaction t;
1077 t.remove(cid, hoid);
1078 t.remove_collection(cid);
1079 cerr << "Cleaning" << std::endl;
1080 r = apply_transaction(store, &osr, std::move(t));
1083 g_conf->set_val("bluestore_compression_min_blob_size", stringify(orig_min_blob_size));
1084 g_ceph_context->_conf->apply_changes(NULL);
1087 TEST_P(StoreTest, CompressionTest) {
1088 if (string(GetParam()) != "bluestore")
1091 g_conf->set_val("bluestore_compression_algorithm", "snappy");
1092 g_conf->set_val("bluestore_compression_mode", "force");
1094 g_ceph_context->_conf->apply_changes(NULL);
1096 doCompressionTest(store);
1098 g_conf->set_val("bluestore_compression_algorithm", "zlib");
1099 g_conf->set_val("bluestore_compression_mode", "force");
1100 g_ceph_context->_conf->apply_changes(NULL);
1102 doCompressionTest(store);
1104 g_conf->set_val("bluestore_compression_algorithm", "snappy");
1105 g_conf->set_val("bluestore_compression_mode", "none");
1106 g_ceph_context->_conf->apply_changes(NULL);
1109 TEST_P(StoreTest, SimpleObjectTest) {
1110 ObjectStore::Sequencer osr("test");
1113 ghobject_t hoid(hobject_t(sobject_t("Object 1", CEPH_NOSNAP)));
1116 r = store->read(cid, hoid, 0, 5, in);
1117 ASSERT_EQ(-ENOENT, r);
1120 ObjectStore::Transaction t;
1121 t.create_collection(cid, 0);
1122 cerr << "Creating collection " << cid << std::endl;
1123 r = apply_transaction(store, &osr, std::move(t));
1127 bool exists = store->exists(cid, hoid);
1128 ASSERT_TRUE(!exists);
1130 ObjectStore::Transaction t;
1132 cerr << "Creating object " << hoid << std::endl;
1133 r = apply_transaction(store, &osr, std::move(t));
1136 exists = store->exists(cid, hoid);
1137 ASSERT_EQ(true, exists);
1140 ObjectStore::Transaction t;
1141 t.remove(cid, hoid);
1143 cerr << "Remove then create" << std::endl;
1144 r = apply_transaction(store, &osr, std::move(t));
1148 ObjectStore::Transaction t;
1149 bufferlist bl, orig;
1152 t.remove(cid, hoid);
1153 t.write(cid, hoid, 0, 5, bl);
1154 cerr << "Remove then create" << std::endl;
1155 r = apply_transaction(store, &osr, std::move(t));
1159 r = store->read(cid, hoid, 0, 5, in);
1161 ASSERT_TRUE(bl_eq(orig, in));
1164 ObjectStore::Transaction t;
1169 t.write(cid, hoid, 5, 5, bl);
1170 cerr << "Append" << std::endl;
1171 r = apply_transaction(store, &osr, std::move(t));
1175 r = store->read(cid, hoid, 0, 10, in);
1177 ASSERT_TRUE(bl_eq(exp, in));
1180 ObjectStore::Transaction t;
1182 bl.append("abcdeabcde");
1184 t.write(cid, hoid, 0, 10, bl);
1185 cerr << "Full overwrite" << std::endl;
1186 r = apply_transaction(store, &osr, std::move(t));
1190 r = store->read(cid, hoid, 0, 10, in);
1192 ASSERT_TRUE(bl_eq(exp, in));
1195 ObjectStore::Transaction t;
1198 t.write(cid, hoid, 3, 5, bl);
1199 cerr << "Partial overwrite" << std::endl;
1200 r = apply_transaction(store, &osr, std::move(t));
1204 exp.append("abcabcdede");
1205 r = store->read(cid, hoid, 0, 10, in);
1208 ASSERT_TRUE(bl_eq(exp, in));
1212 ObjectStore::Transaction t;
1215 t.truncate(cid, hoid, 0);
1216 t.write(cid, hoid, 5, 5, bl);
1217 cerr << "Truncate + hole" << std::endl;
1218 r = apply_transaction(store, &osr, std::move(t));
1222 ObjectStore::Transaction t;
1225 t.write(cid, hoid, 0, 5, bl);
1226 cerr << "Reverse fill-in" << std::endl;
1227 r = apply_transaction(store, &osr, std::move(t));
1232 exp.append("abcdefghij");
1233 r = store->read(cid, hoid, 0, 10, in);
1236 ASSERT_TRUE(bl_eq(exp, in));
1239 ObjectStore::Transaction t;
1241 bl.append("abcde01234012340123401234abcde01234012340123401234abcde01234012340123401234abcde01234012340123401234");
1242 t.write(cid, hoid, 0, bl.length(), bl);
1243 cerr << "larger overwrite" << std::endl;
1244 r = apply_transaction(store, &osr, std::move(t));
1248 r = store->read(cid, hoid, 0, bl.length(), in);
1249 ASSERT_EQ((int)bl.length(), r);
1251 ASSERT_TRUE(bl_eq(bl, in));
1255 bl.append("abcde01234012340123401234abcde01234012340123401234abcde01234012340123401234abcde01234012340123401234");
1257 //test: offset=len=0 mean read all data
1259 r = store->read(cid, hoid, 0, 0, in);
1260 ASSERT_EQ((int)bl.length(), r);
1262 ASSERT_TRUE(bl_eq(bl, in));
1265 //verifying unaligned csums
1266 std::string s1("1"), s2(0x1000, '2'), s3("00");
1268 ObjectStore::Transaction t;
1272 t.truncate(cid, hoid, 0);
1273 t.write(cid, hoid, 0x1000-1, bl.length(), bl);
1274 cerr << "Write unaligned csum, stage 1" << std::endl;
1275 r = apply_transaction(store, &osr, std::move(t));
1279 bufferlist in, exp1, exp2, exp3;
1283 r = store->read(cid, hoid, 0x1000-1, 1, in);
1285 ASSERT_TRUE(bl_eq(exp1, in));
1287 r = store->read(cid, hoid, 0x1000, 0x1000, in);
1288 ASSERT_EQ(0x1000, r);
1289 ASSERT_TRUE(bl_eq(exp2, in));
1292 ObjectStore::Transaction t;
1295 t.write(cid, hoid, 1, bl.length(), bl);
1296 cerr << "Write unaligned csum, stage 2" << std::endl;
1297 r = apply_transaction(store, &osr, std::move(t));
1301 r = store->read(cid, hoid, 1, 2, in);
1303 ASSERT_TRUE(bl_eq(exp3, in));
1305 r = store->read(cid, hoid, 0x1000-1, 1, in);
1307 ASSERT_TRUE(bl_eq(exp1, in));
1309 r = store->read(cid, hoid, 0x1000, 0x1000, in);
1310 ASSERT_EQ(0x1000, r);
1311 ASSERT_TRUE(bl_eq(exp2, in));
1316 ObjectStore::Transaction t;
1317 t.remove(cid, hoid);
1318 t.remove_collection(cid);
1319 cerr << "Cleaning" << std::endl;
1320 r = apply_transaction(store, &osr, std::move(t));
1325 #if defined(HAVE_LIBAIO)
1326 TEST_P(StoreTestSpecificAUSize, BluestoreStatFSTest) {
1327 if(string(GetParam()) != "bluestore")
1329 StartDeferred(65536);
1330 g_conf->set_val("bluestore_compression_mode", "force");
1332 // just a big number to disble gc
1333 g_conf->set_val("bluestore_gc_enable_total_threshold", "100000");
1334 g_conf->apply_changes(NULL);
1337 ObjectStore::Sequencer osr("test");
1339 ghobject_t hoid(hobject_t(sobject_t("Object 1", CEPH_NOSNAP)));
1340 ghobject_t hoid2 = hoid;
1341 hoid2.hobj.snap = 1;
1344 r = store->read(cid, hoid, 0, 5, in);
1345 ASSERT_EQ(-ENOENT, r);
1348 ObjectStore::Transaction t;
1349 t.create_collection(cid, 0);
1350 cerr << "Creating collection " << cid << std::endl;
1351 r = apply_transaction(store, &osr, std::move(t));
1355 bool exists = store->exists(cid, hoid);
1356 ASSERT_TRUE(!exists);
1358 ObjectStore::Transaction t;
1360 cerr << "Creating object " << hoid << std::endl;
1361 r = apply_transaction(store, &osr, std::move(t));
1364 exists = store->exists(cid, hoid);
1365 ASSERT_EQ(true, exists);
1368 struct store_statfs_t statfs;
1369 int r = store->statfs(&statfs);
1371 ASSERT_EQ( 0u, statfs.allocated);
1372 ASSERT_EQ( 0u, statfs.stored);
1373 ASSERT_EQ(g_conf->bluestore_block_size, statfs.total);
1374 ASSERT_TRUE(statfs.available > 0u && statfs.available < g_conf->bluestore_block_size);
1376 EXPECT_EQ(store->umount(), 0);
1377 EXPECT_EQ(store->mount(), 0);
1380 ObjectStore::Transaction t;
1383 t.write(cid, hoid, 0, 5, bl);
1384 cerr << "Append 5 bytes" << std::endl;
1385 r = apply_transaction(store, &osr, std::move(t));
1388 struct store_statfs_t statfs;
1389 int r = store->statfs(&statfs);
1391 ASSERT_EQ(5, statfs.stored);
1392 ASSERT_EQ(0x10000, statfs.allocated);
1393 ASSERT_EQ(0, statfs.compressed);
1394 ASSERT_EQ(0, statfs.compressed_original);
1395 ASSERT_EQ(0, statfs.compressed_allocated);
1397 EXPECT_EQ(store->umount(), 0);
1398 EXPECT_EQ(store->mount(), 0);
1401 ObjectStore::Transaction t;
1402 std::string s(0x30000, 'a');
1405 t.write(cid, hoid, 0x10000, bl.length(), bl);
1406 cerr << "Append 0x30000 compressible bytes" << std::endl;
1407 r = apply_transaction(store, &osr, std::move(t));
1410 struct store_statfs_t statfs;
1411 int r = store->statfs(&statfs);
1413 ASSERT_EQ(0x30005, statfs.stored);
1414 ASSERT_EQ(0x30000, statfs.allocated);
1415 ASSERT_LE(statfs.compressed, 0x10000);
1416 ASSERT_EQ(0x20000, statfs.compressed_original);
1417 ASSERT_EQ(statfs.compressed_allocated, 0x10000);
1419 EXPECT_EQ(store->umount(), 0);
1420 EXPECT_EQ(store->mount(), 0);
1423 ObjectStore::Transaction t;
1424 t.zero(cid, hoid, 1, 3);
1425 t.zero(cid, hoid, 0x20000, 9);
1426 cerr << "Punch hole at 1~3, 0x20000~9" << std::endl;
1427 r = apply_transaction(store, &osr, std::move(t));
1430 struct store_statfs_t statfs;
1431 int r = store->statfs(&statfs);
1433 ASSERT_EQ(0x30005 - 3 - 9, statfs.stored);
1434 ASSERT_EQ(0x30000, statfs.allocated);
1435 ASSERT_LE(statfs.compressed, 0x10000);
1436 ASSERT_EQ(0x20000 - 9, statfs.compressed_original);
1437 ASSERT_EQ(statfs.compressed_allocated, 0x10000);
1439 EXPECT_EQ(store->umount(), 0);
1440 EXPECT_EQ(store->mount(), 0);
1443 ObjectStore::Transaction t;
1444 std::string s(0x1000, 'b');
1447 t.write(cid, hoid, 1, bl.length(), bl);
1448 t.write(cid, hoid, 0x10001, bl.length(), bl);
1449 cerr << "Overwrite first and second(compressible) extents" << std::endl;
1450 r = apply_transaction(store, &osr, std::move(t));
1453 struct store_statfs_t statfs;
1454 int r = store->statfs(&statfs);
1456 ASSERT_EQ(0x30001 - 9 + 0x1000, statfs.stored);
1457 ASSERT_EQ(0x40000, statfs.allocated);
1458 ASSERT_LE(statfs.compressed, 0x10000);
1459 ASSERT_EQ(0x20000 - 9 - 0x1000, statfs.compressed_original);
1460 ASSERT_EQ(statfs.compressed_allocated, 0x10000);
1462 EXPECT_EQ(store->umount(), 0);
1463 EXPECT_EQ(store->mount(), 0);
1466 ObjectStore::Transaction t;
1467 std::string s(0x10000, 'c');
1470 t.write(cid, hoid, 0x10000, bl.length(), bl);
1471 t.write(cid, hoid, 0x20000, bl.length(), bl);
1472 t.write(cid, hoid, 0x30000, bl.length(), bl);
1473 cerr << "Overwrite compressed extent with 3 uncompressible ones" << std::endl;
1474 r = apply_transaction(store, &osr, std::move(t));
1477 struct store_statfs_t statfs;
1478 int r = store->statfs(&statfs);
1480 ASSERT_EQ(0x30000 + 0x1001, statfs.stored);
1481 ASSERT_EQ(0x40000, statfs.allocated);
1482 ASSERT_LE(statfs.compressed, 0);
1483 ASSERT_EQ(0, statfs.compressed_original);
1484 ASSERT_EQ(0, statfs.compressed_allocated);
1486 EXPECT_EQ(store->umount(), 0);
1487 EXPECT_EQ(store->mount(), 0);
1490 ObjectStore::Transaction t;
1491 t.zero(cid, hoid, 0, 0x40000);
1492 cerr << "Zero object" << std::endl;
1493 r = apply_transaction(store, &osr, std::move(t));
1495 struct store_statfs_t statfs;
1496 int r = store->statfs(&statfs);
1498 ASSERT_EQ(0u, statfs.allocated);
1499 ASSERT_EQ(0u, statfs.stored);
1500 ASSERT_EQ(0u, statfs.compressed_original);
1501 ASSERT_EQ(0u, statfs.compressed);
1502 ASSERT_EQ(0u, statfs.compressed_allocated);
1504 EXPECT_EQ(store->umount(), 0);
1505 EXPECT_EQ(store->mount(), 0);
1508 ObjectStore::Transaction t;
1509 std::string s(0x10000, 'c');
1514 bl.append(s.substr(0, 0x10000-2));
1515 t.write(cid, hoid, 0, bl.length(), bl);
1516 cerr << "Yet another compressible write" << std::endl;
1517 r = apply_transaction(store, &osr, std::move(t));
1519 struct store_statfs_t statfs;
1520 r = store->statfs(&statfs);
1522 ASSERT_EQ(0x40000 - 2, statfs.stored);
1523 ASSERT_EQ(0x30000, statfs.allocated);
1524 ASSERT_LE(statfs.compressed, 0x10000);
1525 ASSERT_EQ(0x20000, statfs.compressed_original);
1526 ASSERT_EQ(0x10000, statfs.compressed_allocated);
1528 EXPECT_EQ(store->umount(), 0);
1529 EXPECT_EQ(store->mount(), 0);
1532 struct store_statfs_t statfs;
1533 r = store->statfs(&statfs);
1536 ObjectStore::Transaction t;
1537 t.clone(cid, hoid, hoid2);
1538 cerr << "Clone compressed objecte" << std::endl;
1539 r = apply_transaction(store, &osr, std::move(t));
1541 struct store_statfs_t statfs2;
1542 r = store->statfs(&statfs2);
1544 ASSERT_GT(statfs2.stored, statfs.stored);
1545 ASSERT_EQ(statfs2.allocated, statfs.allocated);
1546 ASSERT_GT(statfs2.compressed, statfs.compressed);
1547 ASSERT_GT(statfs2.compressed_original, statfs.compressed_original);
1548 ASSERT_EQ(statfs2.compressed_allocated, statfs.compressed_allocated);
1552 ObjectStore::Transaction t;
1553 t.remove(cid, hoid);
1554 t.remove(cid, hoid2);
1555 t.remove_collection(cid);
1556 cerr << "Cleaning" << std::endl;
1557 r = apply_transaction(store, &osr, std::move(t));
1560 struct store_statfs_t statfs;
1561 r = store->statfs(&statfs);
1563 ASSERT_EQ( 0u, statfs.allocated);
1564 ASSERT_EQ( 0u, statfs.stored);
1565 ASSERT_EQ( 0u, statfs.compressed_original);
1566 ASSERT_EQ( 0u, statfs.compressed);
1567 ASSERT_EQ( 0u, statfs.compressed_allocated);
1569 g_conf->set_val("bluestore_gc_enable_total_threshold", "0");
1570 g_conf->set_val("bluestore_compression_mode", "none");
1571 g_ceph_context->_conf->apply_changes(NULL);
1574 TEST_P(StoreTestSpecificAUSize, BluestoreFragmentedBlobTest) {
1575 if(string(GetParam()) != "bluestore")
1577 StartDeferred(0x10000);
1579 ObjectStore::Sequencer osr("test");
1582 ghobject_t hoid(hobject_t(sobject_t("Object 1", CEPH_NOSNAP)));
1584 ObjectStore::Transaction t;
1585 t.create_collection(cid, 0);
1586 cerr << "Creating collection " << cid << std::endl;
1587 r = apply_transaction(store, &osr, std::move(t));
1591 bool exists = store->exists(cid, hoid);
1592 ASSERT_TRUE(!exists);
1594 ObjectStore::Transaction t;
1596 cerr << "Creating object " << hoid << std::endl;
1597 r = apply_transaction(store, &osr, std::move(t));
1600 exists = store->exists(cid, hoid);
1601 ASSERT_EQ(true, exists);
1604 struct store_statfs_t statfs;
1605 int r = store->statfs(&statfs);
1607 ASSERT_EQ(g_conf->bluestore_block_size, statfs.total);
1608 ASSERT_EQ(0u, statfs.allocated);
1609 ASSERT_EQ(0u, statfs.stored);
1610 ASSERT_TRUE(statfs.available > 0u && statfs.available < g_conf->bluestore_block_size);
1613 data.resize(0x10000 * 3);
1615 ObjectStore::Transaction t;
1616 for(size_t i = 0;i < data.size(); i++)
1617 data[i] = i / 256 + 1;
1618 bufferlist bl, newdata;
1620 t.write(cid, hoid, 0, bl.length(), bl);
1621 t.zero(cid, hoid, 0x10000, 0x10000);
1622 cerr << "Append 3*0x10000 bytes and punch a hole 0x10000~10000" << std::endl;
1623 r = apply_transaction(store, &osr, std::move(t));
1626 struct store_statfs_t statfs;
1627 int r = store->statfs(&statfs);
1629 ASSERT_EQ(0x20000, statfs.stored);
1630 ASSERT_EQ(0x20000, statfs.allocated);
1632 r = store->read(cid, hoid, 0, data.size(), newdata);
1633 ASSERT_EQ(r, (int)data.size());
1635 bufferlist expected;
1636 expected.append(data.substr(0, 0x10000));
1637 expected.append(string(0x10000, 0));
1638 expected.append(data.substr(0x20000, 0x10000));
1639 ASSERT_TRUE(bl_eq(expected, newdata));
1643 r = store->read(cid, hoid, 1, data.size()-2, newdata);
1644 ASSERT_EQ(r, (int)data.size()-2);
1646 bufferlist expected;
1647 expected.append(data.substr(1, 0x10000-1));
1648 expected.append(string(0x10000, 0));
1649 expected.append(data.substr(0x20000, 0x10000 - 1));
1650 ASSERT_TRUE(bl_eq(expected, newdata));
1655 EXPECT_EQ(store->umount(), 0);
1656 EXPECT_EQ(store->mount(), 0);
1659 ObjectStore::Transaction t;
1660 std::string data2(3, 'b');
1661 bufferlist bl, newdata;
1663 t.write(cid, hoid, 0x20000, bl.length(), bl);
1664 cerr << "Write 3 bytes after the hole" << std::endl;
1665 r = apply_transaction(store, &osr, std::move(t));
1668 struct store_statfs_t statfs;
1669 int r = store->statfs(&statfs);
1671 ASSERT_EQ(0x20000, statfs.allocated);
1672 ASSERT_EQ(0x20000, statfs.stored);
1674 r = store->read(cid, hoid, 0x20000-1, 21, newdata);
1675 ASSERT_EQ(r, (int)21);
1677 bufferlist expected;
1678 expected.append(string(0x1, 0));
1679 expected.append(string(data2));
1680 expected.append(data.substr(0x20003, 21-4));
1681 ASSERT_TRUE(bl_eq(expected, newdata));
1686 EXPECT_EQ(store->umount(), 0);
1687 EXPECT_EQ(store->mount(), 0);
1690 ObjectStore::Transaction t;
1691 std::string data2(3, 'a');
1692 bufferlist bl, newdata;
1694 t.write(cid, hoid, 0x10000+1, bl.length(), bl);
1695 cerr << "Write 3 bytes to the hole" << std::endl;
1696 r = apply_transaction(store, &osr, std::move(t));
1699 struct store_statfs_t statfs;
1700 int r = store->statfs(&statfs);
1702 ASSERT_EQ(0x30000, statfs.allocated);
1703 ASSERT_EQ(0x20003, statfs.stored);
1705 r = store->read(cid, hoid, 0x10000-1, 0x10000+22, newdata);
1706 ASSERT_EQ(r, (int)0x10000+22);
1708 bufferlist expected;
1709 expected.append(data.substr(0x10000-1, 1));
1710 expected.append(string(0x1, 0));
1711 expected.append(data2);
1712 expected.append(string(0x10000-4, 0));
1713 expected.append(string(0x3, 'b'));
1714 expected.append(data.substr(0x20004, 21-3));
1715 ASSERT_TRUE(bl_eq(expected, newdata));
1720 ObjectStore::Transaction t;
1721 bufferlist bl, newdata;
1722 bl.append(string(0x30000, 'c'));
1723 t.write(cid, hoid, 0, 0x30000, bl);
1724 t.zero(cid, hoid, 0, 0x10000);
1725 t.zero(cid, hoid, 0x20000, 0x10000);
1726 cerr << "Rewrite an object and create two holes at the begining and the end" << std::endl;
1727 r = apply_transaction(store, &osr, std::move(t));
1730 struct store_statfs_t statfs;
1731 int r = store->statfs(&statfs);
1733 ASSERT_EQ(0x10000, statfs.allocated);
1734 ASSERT_EQ(0x10000, statfs.stored);
1736 r = store->read(cid, hoid, 0, 0x30000, newdata);
1737 ASSERT_EQ(r, (int)0x30000);
1739 bufferlist expected;
1740 expected.append(string(0x10000, 0));
1741 expected.append(string(0x10000, 'c'));
1742 expected.append(string(0x10000, 0));
1743 ASSERT_TRUE(bl_eq(expected, newdata));
1749 EXPECT_EQ(store->umount(), 0);
1750 EXPECT_EQ(store->mount(), 0);
1753 ObjectStore::Transaction t;
1754 t.remove(cid, hoid);
1755 t.remove_collection(cid);
1756 cerr << "Cleaning" << std::endl;
1757 r = apply_transaction(store, &osr, std::move(t));
1760 struct store_statfs_t statfs;
1761 r = store->statfs(&statfs);
1763 ASSERT_EQ( 0u, statfs.allocated);
1764 ASSERT_EQ( 0u, statfs.stored);
1765 ASSERT_EQ( 0u, statfs.compressed_original);
1766 ASSERT_EQ( 0u, statfs.compressed);
1767 ASSERT_EQ( 0u, statfs.compressed_allocated);
1772 TEST_P(StoreTest, ManySmallWrite) {
1773 ObjectStore::Sequencer osr("test");
1776 ghobject_t a(hobject_t(sobject_t("Object 1", CEPH_NOSNAP)));
1777 ghobject_t b(hobject_t(sobject_t("Object 2", CEPH_NOSNAP)));
1779 ObjectStore::Transaction t;
1780 t.create_collection(cid, 0);
1781 cerr << "Creating collection " << cid << std::endl;
1782 r = apply_transaction(store, &osr, std::move(t));
1789 for (int i=0; i<100; ++i) {
1790 ObjectStore::Transaction t;
1791 t.write(cid, a, i*4096, 4096, bl, 0);
1792 r = apply_transaction(store, &osr, std::move(t));
1795 for (int i=0; i<100; ++i) {
1796 ObjectStore::Transaction t;
1797 t.write(cid, b, (rand() % 1024)*4096, 4096, bl, 0);
1798 r = apply_transaction(store, &osr, std::move(t));
1802 ObjectStore::Transaction t;
1805 t.remove_collection(cid);
1806 cerr << "Cleaning" << std::endl;
1807 r = apply_transaction(store, &osr, std::move(t));
1812 TEST_P(StoreTest, MultiSmallWriteSameBlock) {
1813 ObjectStore::Sequencer osr("test");
1816 ghobject_t a(hobject_t(sobject_t("Object 1", CEPH_NOSNAP)));
1818 ObjectStore::Transaction t;
1819 t.create_collection(cid, 0);
1820 cerr << "Creating collection " << cid << std::endl;
1821 r = apply_transaction(store, &osr, std::move(t));
1827 // touch same block in both same transaction, tls, and pipelined txns
1829 ObjectStore::Transaction t, u;
1830 t.write(cid, a, 0, 5, bl, 0);
1831 t.write(cid, a, 5, 5, bl, 0);
1832 t.write(cid, a, 4094, 5, bl, 0);
1833 t.write(cid, a, 9000, 5, bl, 0);
1834 u.write(cid, a, 10, 5, bl, 0);
1835 u.write(cid, a, 7000, 5, bl, 0);
1836 vector<ObjectStore::Transaction> v = {t, u};
1837 store->queue_transactions(&osr, v, nullptr, &c);
1840 ObjectStore::Transaction t, u;
1841 t.write(cid, a, 40, 5, bl, 0);
1842 t.write(cid, a, 45, 5, bl, 0);
1843 t.write(cid, a, 4094, 5, bl, 0);
1844 t.write(cid, a, 6000, 5, bl, 0);
1845 u.write(cid, a, 610, 5, bl, 0);
1846 u.write(cid, a, 11000, 5, bl, 0);
1847 vector<ObjectStore::Transaction> v = {t, u};
1848 store->queue_transactions(&osr, v, nullptr, &d);
1854 r = store->read(cid, a, 0, 16000, bl2);
1858 ObjectStore::Transaction t;
1860 t.remove_collection(cid);
1861 cerr << "Cleaning" << std::endl;
1862 r = apply_transaction(store, &osr, std::move(t));
1867 TEST_P(StoreTest, SmallSkipFront) {
1868 ObjectStore::Sequencer osr("test");
1871 ghobject_t a(hobject_t(sobject_t("Object 1", CEPH_NOSNAP)));
1873 ObjectStore::Transaction t;
1874 t.create_collection(cid, 0);
1875 cerr << "Creating collection " << cid << std::endl;
1876 r = apply_transaction(store, &osr, std::move(t));
1880 ObjectStore::Transaction t;
1882 t.truncate(cid, a, 3000);
1883 r = apply_transaction(store, &osr, std::move(t));
1889 memset(bp.c_str(), 1, 4096);
1891 ObjectStore::Transaction t;
1892 t.write(cid, a, 4096, 4096, bl);
1893 r = apply_transaction(store, &osr, std::move(t));
1898 ASSERT_EQ(8192, store->read(cid, a, 0, 8192, bl));
1899 for (unsigned i=0; i<4096; ++i)
1900 ASSERT_EQ(0, bl[i]);
1901 for (unsigned i=4096; i<8192; ++i)
1902 ASSERT_EQ(1, bl[i]);
1905 ObjectStore::Transaction t;
1907 t.remove_collection(cid);
1908 cerr << "Cleaning" << std::endl;
1909 r = apply_transaction(store, &osr, std::move(t));
1914 TEST_P(StoreTest, AppendDeferredVsTailCache) {
1915 ObjectStore::Sequencer osr("test");
1918 ghobject_t a(hobject_t(sobject_t("fooo", CEPH_NOSNAP)));
1920 ObjectStore::Transaction t;
1921 t.create_collection(cid, 0);
1922 cerr << "Creating collection " << cid << std::endl;
1923 r = store->apply_transaction(&osr, std::move(t));
1926 unsigned min_alloc = g_conf->bluestore_min_alloc_size;
1927 g_conf->set_val("bluestore_inject_deferred_apply_delay", "1.0");
1928 g_ceph_context->_conf->apply_changes(NULL);
1929 unsigned size = min_alloc / 3;
1930 bufferptr bpa(size);
1931 memset(bpa.c_str(), 1, bpa.length());
1935 ObjectStore::Transaction t;
1936 t.write(cid, a, 0, bla.length(), bla, 0);
1937 r = store->apply_transaction(&osr, std::move(t));
1941 // force cached tail to clear ...
1943 int r = store->umount();
1949 bufferptr bpb(size);
1950 memset(bpb.c_str(), 2, bpb.length());
1954 ObjectStore::Transaction t;
1955 t.write(cid, a, bla.length(), blb.length(), blb, 0);
1956 r = store->apply_transaction(&osr, std::move(t));
1959 bufferptr bpc(size);
1960 memset(bpc.c_str(), 3, bpc.length());
1964 ObjectStore::Transaction t;
1965 t.write(cid, a, bla.length() + blb.length(), blc.length(), blc, 0);
1966 r = store->apply_transaction(&osr, std::move(t));
1975 ASSERT_EQ((int)final.length(),
1976 store->read(cid, a, 0, final.length(), actual));
1977 ASSERT_TRUE(bl_eq(final, actual));
1980 ObjectStore::Transaction t;
1982 t.remove_collection(cid);
1983 cerr << "Cleaning" << std::endl;
1984 r = store->apply_transaction(&osr, std::move(t));
1987 g_conf->set_val("bluestore_inject_deferred_apply_delay", "0");
1988 g_ceph_context->_conf->apply_changes(NULL);
1991 TEST_P(StoreTest, AppendZeroTrailingSharedBlock) {
1992 ObjectStore::Sequencer osr("test");
1995 ghobject_t a(hobject_t(sobject_t("fooo", CEPH_NOSNAP)));
1999 ObjectStore::Transaction t;
2000 t.create_collection(cid, 0);
2001 cerr << "Creating collection " << cid << std::endl;
2002 r = store->apply_transaction(&osr, std::move(t));
2005 unsigned min_alloc = g_conf->bluestore_min_alloc_size;
2006 unsigned size = min_alloc / 3;
2007 bufferptr bpa(size);
2008 memset(bpa.c_str(), 1, bpa.length());
2011 // make sure there is some trailing gunk in the last block
2015 bt.append("BADBADBADBAD");
2016 ObjectStore::Transaction t;
2017 t.write(cid, a, 0, bt.length(), bt, 0);
2018 r = store->apply_transaction(&osr, std::move(t));
2022 ObjectStore::Transaction t;
2023 t.truncate(cid, a, size);
2024 r = store->apply_transaction(&osr, std::move(t));
2030 ObjectStore::Transaction t;
2032 r = store->apply_transaction(&osr, std::move(t));
2036 // append with implicit zeroing
2037 bufferptr bpb(size);
2038 memset(bpb.c_str(), 2, bpb.length());
2042 ObjectStore::Transaction t;
2043 t.write(cid, a, min_alloc * 3, blb.length(), blb, 0);
2044 r = store->apply_transaction(&osr, std::move(t));
2050 zeros.append_zero(min_alloc * 3 - size);
2051 final.append(zeros);
2055 ASSERT_EQ((int)final.length(),
2056 store->read(cid, a, 0, final.length(), actual));
2057 final.hexdump(cout);
2058 actual.hexdump(cout);
2059 ASSERT_TRUE(bl_eq(final, actual));
2062 ObjectStore::Transaction t;
2065 t.remove_collection(cid);
2066 cerr << "Cleaning" << std::endl;
2067 r = store->apply_transaction(&osr, std::move(t));
2072 TEST_P(StoreTest, SmallSequentialUnaligned) {
2073 ObjectStore::Sequencer osr("test");
2076 ghobject_t a(hobject_t(sobject_t("Object 1", CEPH_NOSNAP)));
2078 ObjectStore::Transaction t;
2079 t.create_collection(cid, 0);
2080 cerr << "Creating collection " << cid << std::endl;
2081 r = apply_transaction(store, &osr, std::move(t));
2089 for (int i=0; i<1000; ++i) {
2090 ObjectStore::Transaction t;
2091 t.write(cid, a, i*len, len, bl, 0);
2092 r = apply_transaction(store, &osr, std::move(t));
2096 ObjectStore::Transaction t;
2098 t.remove_collection(cid);
2099 cerr << "Cleaning" << std::endl;
2100 r = apply_transaction(store, &osr, std::move(t));
2105 TEST_P(StoreTest, ManyBigWrite) {
2106 ObjectStore::Sequencer osr("test");
2109 ghobject_t a(hobject_t(sobject_t("Object 1", CEPH_NOSNAP)));
2110 ghobject_t b(hobject_t(sobject_t("Object 2", CEPH_NOSNAP)));
2112 ObjectStore::Transaction t;
2113 t.create_collection(cid, 0);
2114 cerr << "Creating collection " << cid << std::endl;
2115 r = apply_transaction(store, &osr, std::move(t));
2119 bufferptr bp(4 * 1048576);
2122 for (int i=0; i<10; ++i) {
2123 ObjectStore::Transaction t;
2124 t.write(cid, a, i*4*1048586, 4*1048576, bl, 0);
2125 r = apply_transaction(store, &osr, std::move(t));
2129 for (int i=0; i<10; ++i) {
2130 ObjectStore::Transaction t;
2131 t.write(cid, b, (rand() % 256)*4*1048576, 4*1048576, bl, 0);
2132 r = apply_transaction(store, &osr, std::move(t));
2136 for (int i=0; i<10; ++i) {
2137 ObjectStore::Transaction t;
2138 t.write(cid, b, (rand() % (256*4096))*1024, 4*1048576, bl, 0);
2139 r = apply_transaction(store, &osr, std::move(t));
2143 for (int i=0; i<10; ++i) {
2144 ObjectStore::Transaction t;
2145 t.zero(cid, b, (rand() % (256*4096))*1024, 16*1048576);
2146 r = apply_transaction(store, &osr, std::move(t));
2150 ObjectStore::Transaction t;
2153 t.remove_collection(cid);
2154 cerr << "Cleaning" << std::endl;
2155 r = apply_transaction(store, &osr, std::move(t));
2160 TEST_P(StoreTest, BigWriteBigZero) {
2161 ObjectStore::Sequencer osr("test");
2164 ghobject_t a(hobject_t(sobject_t("foo", CEPH_NOSNAP)));
2166 ObjectStore::Transaction t;
2167 t.create_collection(cid, 0);
2168 r = apply_transaction(store, &osr, std::move(t));
2172 bufferptr bp(1048576);
2173 memset(bp.c_str(), 'b', bp.length());
2177 memset(sp.c_str(), 's', sp.length());
2180 ObjectStore::Transaction t;
2181 t.write(cid, a, 0, bl.length(), bl);
2182 r = apply_transaction(store, &osr, std::move(t));
2186 ObjectStore::Transaction t;
2187 t.zero(cid, a, bl.length() / 4, bl.length() / 2);
2188 r = apply_transaction(store, &osr, std::move(t));
2192 ObjectStore::Transaction t;
2193 t.write(cid, a, bl.length() / 2, s.length(), s);
2194 r = apply_transaction(store, &osr, std::move(t));
2198 ObjectStore::Transaction t;
2200 t.remove_collection(cid);
2201 r = apply_transaction(store, &osr, std::move(t));
2206 TEST_P(StoreTest, MiscFragmentTests) {
2207 ObjectStore::Sequencer osr("test");
2210 ghobject_t a(hobject_t(sobject_t("Object 1", CEPH_NOSNAP)));
2212 ObjectStore::Transaction t;
2213 t.create_collection(cid, 0);
2214 cerr << "Creating collection " << cid << std::endl;
2215 r = apply_transaction(store, &osr, std::move(t));
2219 bufferptr bp(524288);
2223 ObjectStore::Transaction t;
2224 t.write(cid, a, 0, 524288, bl, 0);
2225 r = apply_transaction(store, &osr, std::move(t));
2229 ObjectStore::Transaction t;
2230 t.write(cid, a, 1048576, 524288, bl, 0);
2231 r = apply_transaction(store, &osr, std::move(t));
2236 int r = store->read(cid, a, 524288 + 131072, 1024, inbl);
2238 ASSERT_EQ(inbl.length(), 1024u);
2239 ASSERT_TRUE(inbl.is_zero());
2242 ObjectStore::Transaction t;
2243 t.write(cid, a, 1048576 - 4096, 524288, bl, 0);
2244 r = apply_transaction(store, &osr, std::move(t));
2248 ObjectStore::Transaction t;
2250 t.remove_collection(cid);
2251 cerr << "Cleaning" << std::endl;
2252 r = apply_transaction(store, &osr, std::move(t));
2258 TEST_P(StoreTest, ZeroLengthWrite) {
2259 ObjectStore::Sequencer osr("test");
2262 ghobject_t hoid(hobject_t(sobject_t("foo", CEPH_NOSNAP)));
2264 ObjectStore::Transaction t;
2265 t.create_collection(cid, 0);
2267 r = apply_transaction(store, &osr, std::move(t));
2271 ObjectStore::Transaction t;
2273 t.write(cid, hoid, 1048576, 0, empty);
2274 r = apply_transaction(store, &osr, std::move(t));
2278 r = store->stat(cid, hoid, &stat);
2280 ASSERT_EQ(0, stat.st_size);
2283 r = store->read(cid, hoid, 0, 1048576, newdata);
2287 TEST_P(StoreTest, SimpleAttrTest) {
2288 ObjectStore::Sequencer osr("test");
2291 ghobject_t hoid(hobject_t(sobject_t("attr object 1", CEPH_NOSNAP)));
2292 bufferlist val, val2;
2293 val.append("value");
2294 val.append("value2");
2297 map<string,bufferptr> aset;
2298 r = store->getattr(cid, hoid, "nofoo", bp);
2299 ASSERT_EQ(-ENOENT, r);
2300 r = store->getattrs(cid, hoid, aset);
2301 ASSERT_EQ(-ENOENT, r);
2304 ObjectStore::Transaction t;
2305 t.create_collection(cid, 0);
2306 r = apply_transaction(store, &osr, std::move(t));
2311 int r = store->collection_empty(cid, &empty);
2317 r = store->getattr(cid, hoid, "nofoo", bp);
2318 ASSERT_EQ(-ENOENT, r);
2321 ObjectStore::Transaction t;
2323 t.setattr(cid, hoid, "foo", val);
2324 t.setattr(cid, hoid, "bar", val2);
2325 r = apply_transaction(store, &osr, std::move(t));
2330 int r = store->collection_empty(cid, &empty);
2332 ASSERT_TRUE(!empty);
2336 r = store->getattr(cid, hoid, "nofoo", bp);
2337 ASSERT_EQ(-ENODATA, r);
2339 r = store->getattr(cid, hoid, "foo", bp);
2343 ASSERT_TRUE(bl_eq(val, bl));
2345 map<string,bufferptr> bm;
2346 r = store->getattrs(cid, hoid, bm);
2351 ObjectStore::Transaction t;
2352 t.remove(cid, hoid);
2353 t.remove_collection(cid);
2354 r = apply_transaction(store, &osr, std::move(t));
2359 TEST_P(StoreTest, SimpleListTest) {
2360 ObjectStore::Sequencer osr("test");
2362 coll_t cid(spg_t(pg_t(0, 1), shard_id_t(1)));
2364 ObjectStore::Transaction t;
2365 t.create_collection(cid, 0);
2366 cerr << "Creating collection " << cid << std::endl;
2367 r = apply_transaction(store, &osr, std::move(t));
2370 set<ghobject_t> all;
2372 ObjectStore::Transaction t;
2373 for (int i=0; i<200; ++i) {
2374 string name("object_");
2375 name += stringify(i);
2376 ghobject_t hoid(hobject_t(sobject_t(name, CEPH_NOSNAP)),
2377 ghobject_t::NO_GEN, shard_id_t(1));
2381 cerr << "Creating object " << hoid << std::endl;
2383 r = apply_transaction(store, &osr, std::move(t));
2387 set<ghobject_t> saw;
2388 vector<ghobject_t> objects;
2389 ghobject_t next, current;
2390 while (!next.is_max()) {
2391 int r = store->collection_list(cid, current, ghobject_t::get_max(),
2395 ASSERT_TRUE(sorted(objects));
2396 cout << " got " << objects.size() << " next " << next << std::endl;
2397 for (vector<ghobject_t>::iterator p = objects.begin(); p != objects.end();
2399 if (saw.count(*p)) {
2400 cout << "got DUP " << *p << std::endl;
2402 //cout << "got new " << *p << std::endl;
2409 ASSERT_EQ(saw.size(), all.size());
2410 ASSERT_EQ(saw, all);
2413 ObjectStore::Transaction t;
2414 for (set<ghobject_t>::iterator p = all.begin(); p != all.end(); ++p)
2416 t.remove_collection(cid);
2417 cerr << "Cleaning" << std::endl;
2418 r = apply_transaction(store, &osr, std::move(t));
2423 TEST_P(StoreTest, ListEndTest) {
2424 ObjectStore::Sequencer osr("test");
2426 coll_t cid(spg_t(pg_t(0, 1), shard_id_t(1)));
2428 ObjectStore::Transaction t;
2429 t.create_collection(cid, 0);
2430 cerr << "Creating collection " << cid << std::endl;
2431 r = apply_transaction(store, &osr, std::move(t));
2434 set<ghobject_t> all;
2436 ObjectStore::Transaction t;
2437 for (int i=0; i<200; ++i) {
2438 string name("object_");
2439 name += stringify(i);
2440 ghobject_t hoid(hobject_t(sobject_t(name, CEPH_NOSNAP)),
2441 ghobject_t::NO_GEN, shard_id_t(1));
2445 cerr << "Creating object " << hoid << std::endl;
2447 r = apply_transaction(store, &osr, std::move(t));
2451 ghobject_t end(hobject_t(sobject_t("object_100", CEPH_NOSNAP)),
2452 ghobject_t::NO_GEN, shard_id_t(1));
2454 vector<ghobject_t> objects;
2456 int r = store->collection_list(cid, ghobject_t(), end, 500,
2459 for (auto &p : objects) {
2464 ObjectStore::Transaction t;
2465 for (set<ghobject_t>::iterator p = all.begin(); p != all.end(); ++p)
2467 t.remove_collection(cid);
2468 cerr << "Cleaning" << std::endl;
2469 r = apply_transaction(store, &osr, std::move(t));
2474 TEST_P(StoreTest, Sort) {
2476 hobject_t a(sobject_t("a", CEPH_NOSNAP));
2489 ghobject_t a(hobject_t(sobject_t("a", CEPH_NOSNAP)));
2490 ghobject_t b(hobject_t(sobject_t("b", CEPH_NOSNAP)));
2502 TEST_P(StoreTest, MultipoolListTest) {
2503 ObjectStore::Sequencer osr("test");
2506 coll_t cid = coll_t(spg_t(pg_t(0, poolid), shard_id_t::NO_SHARD));
2508 ObjectStore::Transaction t;
2509 t.create_collection(cid, 0);
2510 cerr << "Creating collection " << cid << std::endl;
2511 r = apply_transaction(store, &osr, std::move(t));
2514 set<ghobject_t> all, saw;
2516 ObjectStore::Transaction t;
2517 for (int i=0; i<200; ++i) {
2518 string name("object_");
2519 name += stringify(i);
2520 ghobject_t hoid(hobject_t(sobject_t(name, CEPH_NOSNAP)));
2522 hoid.hobj.pool = -2 - poolid;
2524 hoid.hobj.pool = poolid;
2527 cerr << "Creating object " << hoid << std::endl;
2529 r = apply_transaction(store, &osr, std::move(t));
2533 vector<ghobject_t> objects;
2534 ghobject_t next, current;
2535 while (!next.is_max()) {
2536 int r = store->collection_list(cid, current, ghobject_t::get_max(), 50,
2539 cout << " got " << objects.size() << " next " << next << std::endl;
2540 for (vector<ghobject_t>::iterator p = objects.begin(); p != objects.end();
2547 ASSERT_EQ(saw, all);
2550 ObjectStore::Transaction t;
2551 for (set<ghobject_t>::iterator p = all.begin(); p != all.end(); ++p)
2553 t.remove_collection(cid);
2554 cerr << "Cleaning" << std::endl;
2555 r = apply_transaction(store, &osr, std::move(t));
2560 TEST_P(StoreTest, SimpleCloneTest) {
2561 ObjectStore::Sequencer osr("test");
2565 ObjectStore::Transaction t;
2566 t.create_collection(cid, 0);
2567 cerr << "Creating collection " << cid << std::endl;
2568 r = apply_transaction(store, &osr, std::move(t));
2571 ghobject_t hoid(hobject_t(sobject_t("Object 1", CEPH_NOSNAP),
2572 "key", 123, -1, ""));
2573 bufferlist small, large, xlarge, newdata, attr;
2574 small.append("small");
2575 large.append("large");
2576 xlarge.append("xlarge");
2578 ObjectStore::Transaction t;
2580 t.setattr(cid, hoid, "attr1", small);
2581 t.setattr(cid, hoid, "attr2", large);
2582 t.setattr(cid, hoid, "attr3", xlarge);
2583 t.write(cid, hoid, 0, small.length(), small);
2584 t.write(cid, hoid, 10, small.length(), small);
2585 cerr << "Creating object and set attr " << hoid << std::endl;
2586 r = apply_transaction(store, &osr, std::move(t));
2590 ghobject_t hoid2(hobject_t(sobject_t("Object 2", CEPH_NOSNAP),
2591 "key", 123, -1, ""));
2592 ghobject_t hoid3(hobject_t(sobject_t("Object 3", CEPH_NOSNAP)));
2594 ObjectStore::Transaction t;
2595 t.clone(cid, hoid, hoid2);
2596 t.setattr(cid, hoid2, "attr2", small);
2597 t.rmattr(cid, hoid2, "attr1");
2598 t.write(cid, hoid, 10, large.length(), large);
2599 t.setattr(cid, hoid, "attr1", large);
2600 t.setattr(cid, hoid, "attr2", small);
2601 cerr << "Clone object and rm attr" << std::endl;
2602 r = apply_transaction(store, &osr, std::move(t));
2605 r = store->read(cid, hoid, 10, 5, newdata);
2607 ASSERT_TRUE(bl_eq(large, newdata));
2610 r = store->read(cid, hoid, 0, 5, newdata);
2612 ASSERT_TRUE(bl_eq(small, newdata));
2615 r = store->read(cid, hoid2, 10, 5, newdata);
2617 ASSERT_TRUE(bl_eq(small, newdata));
2619 r = store->getattr(cid, hoid2, "attr2", attr);
2621 ASSERT_TRUE(bl_eq(small, attr));
2624 r = store->getattr(cid, hoid2, "attr3", attr);
2626 ASSERT_TRUE(bl_eq(xlarge, attr));
2629 r = store->getattr(cid, hoid, "attr1", attr);
2631 ASSERT_TRUE(bl_eq(large, attr));
2634 ObjectStore::Transaction t;
2635 t.remove(cid, hoid);
2636 t.remove(cid, hoid2);
2637 ASSERT_EQ(0, apply_transaction(store, &osr, std::move(t)));
2642 memset(p.c_str(), 1, p.length());
2646 ObjectStore::Transaction t;
2647 t.write(cid, hoid, 0, pl.length(), pl);
2648 t.clone(cid, hoid, hoid2);
2650 memset(a.c_str(), 2, a.length());
2654 t.write(cid, hoid, pl.length(), a.length(), al);
2655 r = apply_transaction(store, &osr, std::move(t));
2658 ASSERT_EQ((int)final.length(),
2659 store->read(cid, hoid, 0, final.length(), rl));
2660 ASSERT_TRUE(bl_eq(rl, final));
2663 ObjectStore::Transaction t;
2664 t.remove(cid, hoid);
2665 t.remove(cid, hoid2);
2666 ASSERT_EQ(0, apply_transaction(store, &osr, std::move(t)));
2671 memset(p.c_str(), 111, p.length());
2675 ObjectStore::Transaction t;
2676 t.write(cid, hoid, 0, pl.length(), pl);
2677 t.clone(cid, hoid, hoid2);
2682 memset(a.c_str(), 112, a.length());
2686 t.write(cid, hoid, pl.length() + z.length(), a.length(), al);
2687 r = apply_transaction(store, &osr, std::move(t));
2690 ASSERT_EQ((int)final.length(),
2691 store->read(cid, hoid, 0, final.length(), rl));
2692 ASSERT_TRUE(bl_eq(rl, final));
2695 ObjectStore::Transaction t;
2696 t.remove(cid, hoid);
2697 t.remove(cid, hoid2);
2698 ASSERT_EQ(0, apply_transaction(store, &osr, std::move(t)));
2703 memset(p.c_str(), 5, p.length());
2707 ObjectStore::Transaction t;
2708 t.write(cid, hoid, 0, pl.length(), pl);
2709 t.clone(cid, hoid, hoid2);
2714 memset(a.c_str(), 6, a.length());
2718 t.write(cid, hoid, 17000, a.length(), al);
2719 ASSERT_EQ(0, apply_transaction(store, &osr, std::move(t)));
2721 ASSERT_EQ((int)final.length(),
2722 store->read(cid, hoid, 0, final.length(), rl));
2723 /*cout << "expected:\n";
2724 final.hexdump(cout);
2727 ASSERT_TRUE(bl_eq(rl, final));
2730 ObjectStore::Transaction t;
2731 t.remove(cid, hoid);
2732 t.remove(cid, hoid2);
2733 ASSERT_EQ(0, apply_transaction(store, &osr, std::move(t)));
2736 bufferptr p(1048576);
2737 memset(p.c_str(), 3, p.length());
2740 ObjectStore::Transaction t;
2741 t.write(cid, hoid, 0, pl.length(), pl);
2742 t.clone(cid, hoid, hoid2);
2744 memset(a.c_str(), 4, a.length());
2747 t.write(cid, hoid, a.length(), a.length(), al);
2748 ASSERT_EQ(0, apply_transaction(store, &osr, std::move(t)));
2751 final.substr_of(pl, 0, al.length());
2754 end.substr_of(pl, al.length()*2, pl.length() - al.length()*2);
2756 ASSERT_EQ((int)final.length(),
2757 store->read(cid, hoid, 0, final.length(), rl));
2758 /*cout << "expected:\n";
2759 final.hexdump(cout);
2762 ASSERT_TRUE(bl_eq(rl, final));
2765 ObjectStore::Transaction t;
2766 t.remove(cid, hoid);
2767 t.remove(cid, hoid2);
2768 ASSERT_EQ(0, apply_transaction(store, &osr, std::move(t)));
2772 memset(p.c_str(), 7, p.length());
2775 ObjectStore::Transaction t;
2776 t.write(cid, hoid, 0, pl.length(), pl);
2777 t.clone(cid, hoid, hoid2);
2779 memset(a.c_str(), 8, a.length());
2782 t.write(cid, hoid, 32768, a.length(), al);
2783 ASSERT_EQ(0, apply_transaction(store, &osr, std::move(t)));
2786 final.substr_of(pl, 0, 32768);
2789 end.substr_of(pl, final.length(), pl.length() - final.length());
2791 ASSERT_EQ((int)final.length(),
2792 store->read(cid, hoid, 0, final.length(), rl));
2793 /*cout << "expected:\n";
2794 final.hexdump(cout);
2797 ASSERT_TRUE(bl_eq(rl, final));
2800 ObjectStore::Transaction t;
2801 t.remove(cid, hoid);
2802 t.remove(cid, hoid2);
2803 ASSERT_EQ(0, apply_transaction(store, &osr, std::move(t)));
2807 memset(p.c_str(), 9, p.length());
2810 ObjectStore::Transaction t;
2811 t.write(cid, hoid, 0, pl.length(), pl);
2812 t.clone(cid, hoid, hoid2);
2814 memset(a.c_str(), 10, a.length());
2817 t.write(cid, hoid, 33768, a.length(), al);
2818 ASSERT_EQ(0, apply_transaction(store, &osr, std::move(t)));
2821 final.substr_of(pl, 0, 33768);
2824 end.substr_of(pl, final.length(), pl.length() - final.length());
2826 ASSERT_EQ((int)final.length(),
2827 store->read(cid, hoid, 0, final.length(), rl));
2828 /*cout << "expected:\n";
2829 final.hexdump(cout);
2832 ASSERT_TRUE(bl_eq(rl, final));
2835 //Unfortunately we need a workaround for filestore since EXPECT_DEATH
2836 // macro has potential issues when using /in multithread environments.
2837 //It works well for all stores but filestore for now.
2838 //A fix setting gtest_death_test_style = "threadsafe" doesn't help as well -
2839 // test app clone asserts on store folder presence.
2841 if (string(GetParam()) != "filestore") {
2842 //verify if non-empty collection is properly handled after store reload
2843 r = store->umount();
2848 ObjectStore::Transaction t;
2849 t.remove_collection(cid);
2850 cerr << "Invalid rm coll" << std::endl;
2851 PrCtl unset_dumpable;
2852 EXPECT_DEATH(apply_transaction(store, &osr, std::move(t)), "");
2855 ObjectStore::Transaction t;
2856 t.touch(cid, hoid3); //new record in db
2857 r = apply_transaction(store, &osr, std::move(t));
2860 //See comment above for "filestore" check explanation.
2861 if (string(GetParam()) != "filestore") {
2862 ObjectStore::Transaction t;
2863 //verify if non-empty collection is properly handled when there are some pending removes and live records in db
2864 cerr << "Invalid rm coll again" << std::endl;
2865 r = store->umount();
2870 t.remove(cid, hoid);
2871 t.remove(cid, hoid2);
2872 t.remove_collection(cid);
2873 PrCtl unset_dumpable;
2874 EXPECT_DEATH(apply_transaction(store, &osr, std::move(t)), "");
2877 ObjectStore::Transaction t;
2878 t.remove(cid, hoid);
2879 t.remove(cid, hoid2);
2880 t.remove(cid, hoid3);
2881 t.remove_collection(cid);
2882 cerr << "Cleaning" << std::endl;
2883 r = apply_transaction(store, &osr, std::move(t));
2888 TEST_P(StoreTest, OmapSimple) {
2889 ObjectStore::Sequencer osr("test");
2893 ObjectStore::Transaction t;
2894 t.create_collection(cid, 0);
2895 cerr << "Creating collection " << cid << std::endl;
2896 r = apply_transaction(store, &osr, std::move(t));
2899 ghobject_t hoid(hobject_t(sobject_t("omap_obj", CEPH_NOSNAP),
2900 "key", 123, -1, ""));
2902 small.append("small");
2903 map<string,bufferlist> km;
2905 km["bar"].append("asdfjkasdkjdfsjkafskjsfdj");
2907 header.append("this is a header");
2909 ObjectStore::Transaction t;
2911 t.omap_setkeys(cid, hoid, km);
2912 t.omap_setheader(cid, hoid, header);
2913 cerr << "Creating object and set omap " << hoid << std::endl;
2914 r = apply_transaction(store, &osr, std::move(t));
2920 map<string,bufferlist> r;
2921 store->omap_get(cid, hoid, &h, &r);
2922 ASSERT_TRUE(bl_eq(header, h));
2923 ASSERT_EQ(r.size(), km.size());
2924 cout << "r: " << r << std::endl;
2926 // test iterator with seek_to_first
2928 map<string,bufferlist> r;
2929 ObjectMap::ObjectMapIterator iter = store->get_omap_iterator(cid, hoid);
2930 for (iter->seek_to_first(); iter->valid(); iter->next(false)) {
2931 r[iter->key()] = iter->value();
2933 cout << "r: " << r << std::endl;
2934 ASSERT_EQ(r.size(), km.size());
2936 // test iterator with initial lower_bound
2938 map<string,bufferlist> r;
2939 ObjectMap::ObjectMapIterator iter = store->get_omap_iterator(cid, hoid);
2940 for (iter->lower_bound(string()); iter->valid(); iter->next(false)) {
2941 r[iter->key()] = iter->value();
2943 cout << "r: " << r << std::endl;
2944 ASSERT_EQ(r.size(), km.size());
2947 ObjectStore::Transaction t;
2948 t.remove(cid, hoid);
2949 t.remove_collection(cid);
2950 cerr << "Cleaning" << std::endl;
2951 r = apply_transaction(store, &osr, std::move(t));
2956 TEST_P(StoreTest, OmapCloneTest) {
2957 ObjectStore::Sequencer osr("test");
2961 ObjectStore::Transaction t;
2962 t.create_collection(cid, 0);
2963 cerr << "Creating collection " << cid << std::endl;
2964 r = apply_transaction(store, &osr, std::move(t));
2967 ghobject_t hoid(hobject_t(sobject_t("Object 1", CEPH_NOSNAP),
2968 "key", 123, -1, ""));
2970 small.append("small");
2971 map<string,bufferlist> km;
2973 km["bar"].append("asdfjkasdkjdfsjkafskjsfdj");
2975 header.append("this is a header");
2977 ObjectStore::Transaction t;
2979 t.omap_setkeys(cid, hoid, km);
2980 t.omap_setheader(cid, hoid, header);
2981 cerr << "Creating object and set omap " << hoid << std::endl;
2982 r = apply_transaction(store, &osr, std::move(t));
2985 ghobject_t hoid2(hobject_t(sobject_t("Object 2", CEPH_NOSNAP),
2986 "key", 123, -1, ""));
2988 ObjectStore::Transaction t;
2989 t.clone(cid, hoid, hoid2);
2990 cerr << "Clone object" << std::endl;
2991 r = apply_transaction(store, &osr, std::move(t));
2995 map<string,bufferlist> r;
2997 store->omap_get(cid, hoid2, &h, &r);
2998 ASSERT_TRUE(bl_eq(header, h));
2999 ASSERT_EQ(r.size(), km.size());
3002 ObjectStore::Transaction t;
3003 t.remove(cid, hoid);
3004 t.remove(cid, hoid2);
3005 t.remove_collection(cid);
3006 cerr << "Cleaning" << std::endl;
3007 r = apply_transaction(store, &osr, std::move(t));
3012 TEST_P(StoreTest, SimpleCloneRangeTest) {
3013 ObjectStore::Sequencer osr("test");
3017 ObjectStore::Transaction t;
3018 t.create_collection(cid, 0);
3019 cerr << "Creating collection " << cid << std::endl;
3020 r = apply_transaction(store, &osr, std::move(t));
3023 ghobject_t hoid(hobject_t(sobject_t("Object 1", CEPH_NOSNAP)));
3024 hoid.hobj.pool = -1;
3025 bufferlist small, newdata;
3026 small.append("small");
3028 ObjectStore::Transaction t;
3029 t.write(cid, hoid, 10, 5, small);
3030 cerr << "Creating object and write bl " << hoid << std::endl;
3031 r = apply_transaction(store, &osr, std::move(t));
3034 ghobject_t hoid2(hobject_t(sobject_t("Object 2", CEPH_NOSNAP)));
3035 hoid2.hobj.pool = -1;
3037 ObjectStore::Transaction t;
3038 t.clone_range(cid, hoid, hoid2, 10, 5, 10);
3039 cerr << "Clone range object" << std::endl;
3040 r = apply_transaction(store, &osr, std::move(t));
3042 r = store->read(cid, hoid2, 10, 5, newdata);
3044 ASSERT_TRUE(bl_eq(small, newdata));
3047 ObjectStore::Transaction t;
3048 t.truncate(cid, hoid, 1024*1024);
3049 t.clone_range(cid, hoid, hoid2, 0, 1024*1024, 0);
3050 cerr << "Clone range object" << std::endl;
3051 r = apply_transaction(store, &osr, std::move(t));
3053 struct stat stat, stat2;
3054 r = store->stat(cid, hoid, &stat);
3055 r = store->stat(cid, hoid2, &stat2);
3056 ASSERT_EQ(stat.st_size, stat2.st_size);
3057 ASSERT_EQ(1024*1024, stat2.st_size);
3060 ObjectStore::Transaction t;
3061 t.remove(cid, hoid);
3062 t.remove(cid, hoid2);
3063 t.remove_collection(cid);
3064 cerr << "Cleaning" << std::endl;
3065 r = apply_transaction(store, &osr, std::move(t));
3071 TEST_P(StoreTest, SimpleObjectLongnameTest) {
3072 ObjectStore::Sequencer osr("test");
3076 ObjectStore::Transaction t;
3077 t.create_collection(cid, 0);
3078 cerr << "Creating collection " << cid << std::endl;
3079 r = apply_transaction(store, &osr, std::move(t));
3082 ghobject_t hoid(hobject_t(sobject_t("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaObjectaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa 1", CEPH_NOSNAP)));
3084 ObjectStore::Transaction t;
3086 cerr << "Creating object " << hoid << std::endl;
3087 r = apply_transaction(store, &osr, std::move(t));
3091 ObjectStore::Transaction t;
3092 t.remove(cid, hoid);
3093 t.remove_collection(cid);
3094 cerr << "Cleaning" << std::endl;
3095 r = apply_transaction(store, &osr, std::move(t));
3100 ghobject_t generate_long_name(unsigned i)
3103 name << "object id " << i << " ";
3104 for (unsigned j = 0; j < 500; ++j) name << 'a';
3105 ghobject_t hoid(hobject_t(sobject_t(name.str(), CEPH_NOSNAP)));
3106 hoid.hobj.set_hash(i % 2);
3110 TEST_P(StoreTest, LongnameSplitTest) {
3111 ObjectStore::Sequencer osr("test");
3115 ObjectStore::Transaction t;
3116 t.create_collection(cid, 0);
3117 cerr << "Creating collection " << cid << std::endl;
3118 r = apply_transaction(store, &osr, std::move(t));
3121 for (unsigned i = 0; i < 320; ++i) {
3122 ObjectStore::Transaction t;
3123 ghobject_t hoid = generate_long_name(i);
3125 cerr << "Creating object " << hoid << std::endl;
3126 r = apply_transaction(store, &osr, std::move(t));
3130 ghobject_t test_obj = generate_long_name(319);
3131 ghobject_t test_obj_2 = test_obj;
3132 test_obj_2.generation = 0;
3134 ObjectStore::Transaction t;
3135 // should cause a split
3136 t.collection_move_rename(
3139 r = apply_transaction(store, &osr, std::move(t));
3143 for (unsigned i = 0; i < 319; ++i) {
3144 ObjectStore::Transaction t;
3145 ghobject_t hoid = generate_long_name(i);
3146 t.remove(cid, hoid);
3147 cerr << "Removing object " << hoid << std::endl;
3148 r = apply_transaction(store, &osr, std::move(t));
3152 ObjectStore::Transaction t;
3153 t.remove(cid, test_obj_2);
3154 t.remove_collection(cid);
3155 cerr << "Cleaning" << std::endl;
3156 r = apply_transaction(store, &osr, std::move(t));
3162 TEST_P(StoreTest, ManyObjectTest) {
3163 ObjectStore::Sequencer osr("test");
3164 int NUM_OBJS = 2000;
3168 for (int i = 0; i < 100; ++i) base.append("aaaaa");
3169 set<ghobject_t> created;
3171 ObjectStore::Transaction t;
3172 t.create_collection(cid, 0);
3173 r = apply_transaction(store, &osr, std::move(t));
3176 for (int i = 0; i < NUM_OBJS; ++i) {
3178 cerr << "Object " << i << std::endl;
3180 ObjectStore::Transaction t;
3182 snprintf(buf, sizeof(buf), "%d", i);
3183 ghobject_t hoid(hobject_t(sobject_t(string(buf) + base, CEPH_NOSNAP)));
3185 created.insert(hoid);
3186 r = apply_transaction(store, &osr, std::move(t));
3190 for (set<ghobject_t>::iterator i = created.begin();
3194 ASSERT_TRUE(!store->stat(cid, *i, &buf));
3197 set<ghobject_t> listed, listed2;
3198 vector<ghobject_t> objects;
3199 r = store->collection_list(cid, ghobject_t(), ghobject_t::get_max(), INT_MAX, &objects, 0);
3202 cerr << "objects.size() is " << objects.size() << std::endl;
3203 for (vector<ghobject_t> ::iterator i = objects.begin();
3207 ASSERT_TRUE(created.count(*i));
3209 ASSERT_TRUE(listed.size() == created.size());
3211 ghobject_t start, next;
3213 r = store->collection_list(
3215 ghobject_t::get_max(),
3216 ghobject_t::get_max(),
3222 ASSERT_TRUE(objects.empty());
3226 ghobject_t start2, next2;
3228 r = store->collection_list(cid, start, ghobject_t::get_max(),
3232 ASSERT_TRUE(sorted(objects));
3234 listed.insert(objects.begin(), objects.end());
3235 if (objects.size() < 50) {
3236 ASSERT_TRUE(next.is_max());
3243 cerr << "listed.size() is " << listed.size() << std::endl;
3244 ASSERT_TRUE(listed.size() == created.size());
3245 if (listed2.size()) {
3246 ASSERT_EQ(listed.size(), listed2.size());
3248 for (set<ghobject_t>::iterator i = listed.begin();
3251 ASSERT_TRUE(created.count(*i));
3254 for (set<ghobject_t>::iterator i = created.begin();
3257 ObjectStore::Transaction t;
3259 r = apply_transaction(store, &osr, std::move(t));
3262 cerr << "cleaning up" << std::endl;
3264 ObjectStore::Transaction t;
3265 t.remove_collection(cid);
3266 r = apply_transaction(store, &osr, std::move(t));
3272 class ObjectGenerator {
3274 virtual ghobject_t create_object(gen_type *gen) = 0;
3275 virtual ~ObjectGenerator() {}
3278 class MixedGenerator : public ObjectGenerator {
3282 explicit MixedGenerator(int64_t p) : seq(0), poolid(p) {}
3283 ghobject_t create_object(gen_type *gen) override {
3285 snprintf(buf, sizeof(buf), "OBJ_%u", seq);
3288 for (unsigned i = 0; i < 300; ++i) {
3289 name.push_back('a');
3295 name, string(), rand() & 2 ? CEPH_NOSNAP : rand(),
3296 (((seq / 1024) % 2) * 0xF00 ) +
3302 class SyntheticWorkloadState {
3305 map<string, bufferlist> attrs;
3308 static const unsigned max_in_flight = 16;
3309 static const unsigned max_objects = 3000;
3310 static const unsigned max_attr_size = 5;
3311 static const unsigned max_attr_name_len = 100;
3312 static const unsigned max_attr_value_len = 1024 * 64;
3314 unsigned write_alignment;
3315 unsigned max_object_len, max_write_len;
3317 map<ghobject_t, Object> contents;
3318 set<ghobject_t> available_objects;
3319 set<ghobject_t> in_flight_objects;
3320 ObjectGenerator *object_gen;
3323 ObjectStore::Sequencer *osr;
3330 explicit EnterExit(const char *m) : msg(m) {
3331 //cout << pthread_self() << " enter " << msg << std::endl;
3334 //cout << pthread_self() << " exit " << msg << std::endl;
3338 class C_SyntheticOnReadable : public Context {
3340 SyntheticWorkloadState *state;
3342 C_SyntheticOnReadable(SyntheticWorkloadState *state, ghobject_t hoid)
3343 : state(state), hoid(hoid) {}
3345 void finish(int r) override {
3346 Mutex::Locker locker(state->lock);
3347 EnterExit ee("onreadable finish");
3348 ASSERT_TRUE(state->in_flight_objects.count(hoid));
3350 state->in_flight_objects.erase(hoid);
3351 if (state->contents.count(hoid))
3352 state->available_objects.insert(hoid);
3353 --(state->in_flight);
3354 state->cond.Signal();
3357 r = state->store->read(state->cid, hoid, 0, state->contents[hoid].data.length(), r2);
3358 assert(bl_eq(state->contents[hoid].data, r2));
3359 state->cond.Signal();
3363 class C_SyntheticOnStash : public Context {
3365 SyntheticWorkloadState *state;
3366 ghobject_t oid, noid;
3368 C_SyntheticOnStash(SyntheticWorkloadState *state,
3369 ghobject_t oid, ghobject_t noid)
3370 : state(state), oid(oid), noid(noid) {}
3372 void finish(int r) override {
3373 Mutex::Locker locker(state->lock);
3374 EnterExit ee("stash finish");
3375 ASSERT_TRUE(state->in_flight_objects.count(oid));
3377 state->in_flight_objects.erase(oid);
3378 if (state->contents.count(noid))
3379 state->available_objects.insert(noid);
3380 --(state->in_flight);
3382 r = state->store->read(
3383 state->cid, noid, 0,
3384 state->contents[noid].data.length(), r2);
3385 assert(bl_eq(state->contents[noid].data, r2));
3386 state->cond.Signal();
3390 class C_SyntheticOnClone : public Context {
3392 SyntheticWorkloadState *state;
3393 ghobject_t oid, noid;
3395 C_SyntheticOnClone(SyntheticWorkloadState *state,
3396 ghobject_t oid, ghobject_t noid)
3397 : state(state), oid(oid), noid(noid) {}
3399 void finish(int r) override {
3400 Mutex::Locker locker(state->lock);
3401 EnterExit ee("clone finish");
3402 ASSERT_TRUE(state->in_flight_objects.count(oid));
3404 state->in_flight_objects.erase(oid);
3405 if (state->contents.count(oid))
3406 state->available_objects.insert(oid);
3407 if (state->contents.count(noid))
3408 state->available_objects.insert(noid);
3409 --(state->in_flight);
3411 r = state->store->read(state->cid, noid, 0, state->contents[noid].data.length(), r2);
3412 assert(bl_eq(state->contents[noid].data, r2));
3413 state->cond.Signal();
3417 static void filled_byte_array(bufferlist& bl, size_t size)
3419 static const char alphanum[] = "0123456789"
3420 "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
3421 "abcdefghijklmnopqrstuvwxyz";
3426 for (unsigned int i = 0; i < size - 1; i++) {
3427 // severely limit entropy so we can compress...
3428 bp[i] = alphanum[rand() % 10]; //(sizeof(alphanum) - 1)];
3430 bp[size - 1] = '\0';
3435 SyntheticWorkloadState(ObjectStore *store,
3436 ObjectGenerator *gen,
3438 ObjectStore::Sequencer *osr,
3443 : cid(cid), write_alignment(alignment), max_object_len(max_size),
3444 max_write_len(max_write), in_flight(0), object_gen(gen),
3445 rng(rng), store(store), osr(osr), lock("State lock") {}
3448 ObjectStore::Transaction t;
3449 t.create_collection(cid, 0);
3450 return apply_transaction(store, osr, std::move(t));
3454 vector<ghobject_t> objects;
3455 int r = store->collection_list(cid, ghobject_t(), ghobject_t::get_max(),
3458 if (objects.empty())
3460 ObjectStore::Transaction t;
3461 for (vector<ghobject_t>::iterator p = objects.begin();
3462 p != objects.end(); ++p) {
3465 apply_transaction(store, osr, std::move(t));
3467 ObjectStore::Transaction t;
3468 t.remove_collection(cid);
3469 apply_transaction(store, osr, std::move(t));
3471 void statfs(store_statfs_t& stat) {
3472 store->statfs(&stat);
3475 ghobject_t get_uniform_random_object() {
3476 while (in_flight >= max_in_flight || available_objects.empty())
3478 boost::uniform_int<> choose(0, available_objects.size() - 1);
3479 int index = choose(*rng);
3480 set<ghobject_t>::iterator i = available_objects.begin();
3481 for ( ; index > 0; --index, ++i) ;
3482 ghobject_t ret = *i;
3486 void wait_for_ready() {
3487 while (in_flight >= max_in_flight)
3491 void wait_for_done() {
3493 Mutex::Locker locker(lock);
3499 return (available_objects.size() + in_flight_objects.size()) < max_objects;
3503 return (available_objects.size() + in_flight_objects.size()) > 0;
3506 unsigned get_random_alloc_hints() {
3509 boost::uniform_int<> u(0, 3);
3512 f |= CEPH_OSD_ALLOC_HINT_FLAG_SEQUENTIAL_WRITE;
3515 f |= CEPH_OSD_ALLOC_HINT_FLAG_RANDOM_WRITE;
3520 boost::uniform_int<> u(0, 3);
3523 f |= CEPH_OSD_ALLOC_HINT_FLAG_SEQUENTIAL_READ;
3526 f |= CEPH_OSD_ALLOC_HINT_FLAG_RANDOM_READ;
3531 // append_only, immutable
3532 boost::uniform_int<> u(0, 4);
3536 boost::uniform_int<> u(0, 3);
3539 f |= CEPH_OSD_ALLOC_HINT_FLAG_SHORTLIVED;
3542 f |= CEPH_OSD_ALLOC_HINT_FLAG_LONGLIVED;
3547 boost::uniform_int<> u(0, 3);
3550 f |= CEPH_OSD_ALLOC_HINT_FLAG_COMPRESSIBLE;
3553 f |= CEPH_OSD_ALLOC_HINT_FLAG_INCOMPRESSIBLE;
3561 Mutex::Locker locker(lock);
3562 EnterExit ee("touch");
3566 ghobject_t new_obj = object_gen->create_object(rng);
3567 available_objects.erase(new_obj);
3568 ObjectStore::Transaction t;
3569 t.touch(cid, new_obj);
3570 boost::uniform_int<> u(17, 22);
3571 boost::uniform_int<> v(12, 17);
3572 t.set_alloc_hint(cid, new_obj,
3575 get_random_alloc_hints());
3577 in_flight_objects.insert(new_obj);
3578 if (!contents.count(new_obj))
3579 contents[new_obj] = Object();
3580 int status = store->queue_transaction(osr, std::move(t), new C_SyntheticOnReadable(this, new_obj));
3585 Mutex::Locker locker(lock);
3586 EnterExit ee("stash");
3596 old_obj = get_uniform_random_object();
3597 } while (--max && !contents[old_obj].data.length());
3598 available_objects.erase(old_obj);
3599 ghobject_t new_obj = old_obj;
3600 new_obj.generation++;
3601 available_objects.erase(new_obj);
3603 ObjectStore::Transaction t;
3604 t.collection_move_rename(cid, old_obj, cid, new_obj);
3606 in_flight_objects.insert(old_obj);
3608 contents[new_obj].attrs = contents[old_obj].attrs;
3609 contents[new_obj].data = contents[old_obj].data;
3610 contents.erase(old_obj);
3611 int status = store->queue_transaction(
3613 new C_SyntheticOnStash(this, old_obj, new_obj));
3618 Mutex::Locker locker(lock);
3619 EnterExit ee("clone");
3629 old_obj = get_uniform_random_object();
3630 } while (--max && !contents[old_obj].data.length());
3631 available_objects.erase(old_obj);
3632 ghobject_t new_obj = object_gen->create_object(rng);
3633 // make the hash match
3634 new_obj.hobj.set_hash(old_obj.hobj.get_hash());
3635 available_objects.erase(new_obj);
3637 ObjectStore::Transaction t;
3638 t.clone(cid, old_obj, new_obj);
3640 in_flight_objects.insert(old_obj);
3642 contents[new_obj].attrs = contents[old_obj].attrs;
3643 contents[new_obj].data = contents[old_obj].data;
3645 int status = store->queue_transaction(
3647 new C_SyntheticOnClone(this, old_obj, new_obj));
3652 Mutex::Locker locker(lock);
3653 EnterExit ee("clone_range");
3663 old_obj = get_uniform_random_object();
3664 } while (--max && !contents[old_obj].data.length());
3665 bufferlist &srcdata = contents[old_obj].data;
3666 if (srcdata.length() == 0) {
3669 available_objects.erase(old_obj);
3670 ghobject_t new_obj = get_uniform_random_object();
3671 available_objects.erase(new_obj);
3673 boost::uniform_int<> u1(0, max_object_len - max_write_len);
3674 boost::uniform_int<> u2(0, max_write_len);
3675 uint64_t srcoff = u1(*rng);
3676 // make src and dst offsets match, since that's what the osd does
3677 uint64_t dstoff = srcoff; //u1(*rng);
3678 uint64_t len = u2(*rng);
3679 if (write_alignment) {
3680 srcoff = ROUND_UP_TO(srcoff, write_alignment);
3681 dstoff = ROUND_UP_TO(dstoff, write_alignment);
3682 len = ROUND_UP_TO(len, write_alignment);
3685 if (srcoff > srcdata.length() - 1) {
3686 srcoff = srcdata.length() - 1;
3688 if (srcoff + len > srcdata.length()) {
3689 len = srcdata.length() - srcoff;
3692 cout << __func__ << " from " << srcoff << "~" << len
3693 << " (size " << srcdata.length() << ") to "
3694 << dstoff << "~" << len << std::endl;
3696 ObjectStore::Transaction t;
3697 t.clone_range(cid, old_obj, new_obj, srcoff, len, dstoff);
3699 in_flight_objects.insert(old_obj);
3702 if (srcoff < srcdata.length()) {
3703 if (srcoff + len > srcdata.length()) {
3704 bl.substr_of(srcdata, srcoff, srcdata.length() - srcoff);
3706 bl.substr_of(srcdata, srcoff, len);
3710 bufferlist& dstdata = contents[new_obj].data;
3711 if (dstdata.length() <= dstoff) {
3712 if (bl.length() > 0) {
3713 dstdata.append_zero(dstoff - dstdata.length());
3718 assert(dstdata.length() > dstoff);
3719 dstdata.copy(0, dstoff, value);
3721 if (value.length() < dstdata.length())
3722 dstdata.copy(value.length(),
3723 dstdata.length() - value.length(), value);
3724 value.swap(dstdata);
3727 int status = store->queue_transaction(
3728 osr, std::move(t), new C_SyntheticOnClone(this, old_obj, new_obj));
3734 Mutex::Locker locker(lock);
3735 EnterExit ee("write");
3740 ghobject_t new_obj = get_uniform_random_object();
3741 available_objects.erase(new_obj);
3742 ObjectStore::Transaction t;
3744 boost::uniform_int<> u1(0, max_object_len - max_write_len);
3745 boost::uniform_int<> u2(0, max_write_len);
3746 uint64_t offset = u1(*rng);
3747 uint64_t len = u2(*rng);
3749 if (write_alignment) {
3750 offset = ROUND_UP_TO(offset, write_alignment);
3751 len = ROUND_UP_TO(len, write_alignment);
3754 filled_byte_array(bl, len);
3756 bufferlist& data = contents[new_obj].data;
3757 if (data.length() <= offset) {
3759 data.append_zero(offset-data.length());
3764 assert(data.length() > offset);
3765 data.copy(0, offset, value);
3767 if (value.length() < data.length())
3768 data.copy(value.length(),
3769 data.length()-value.length(), value);
3773 t.write(cid, new_obj, offset, len, bl);
3775 in_flight_objects.insert(new_obj);
3776 int status = store->queue_transaction(
3777 osr, std::move(t), new C_SyntheticOnReadable(this, new_obj));
3782 Mutex::Locker locker(lock);
3783 EnterExit ee("truncate");
3788 ghobject_t obj = get_uniform_random_object();
3789 available_objects.erase(obj);
3790 ObjectStore::Transaction t;
3792 boost::uniform_int<> choose(0, max_object_len);
3793 size_t len = choose(*rng);
3794 if (write_alignment) {
3795 len = ROUND_UP_TO(len, write_alignment);
3798 t.truncate(cid, obj, len);
3800 in_flight_objects.insert(obj);
3801 bufferlist& data = contents[obj].data;
3802 if (data.length() <= len) {
3803 data.append_zero(len - data.length());
3806 data.copy(0, len, bl);
3810 int status = store->queue_transaction(
3811 osr, std::move(t), new C_SyntheticOnReadable(this, obj));
3816 Mutex::Locker locker(lock);
3817 EnterExit ee("zero");
3822 ghobject_t new_obj = get_uniform_random_object();
3823 available_objects.erase(new_obj);
3824 ObjectStore::Transaction t;
3826 boost::uniform_int<> u1(0, max_object_len - max_write_len);
3827 boost::uniform_int<> u2(0, max_write_len);
3828 uint64_t offset = u1(*rng);
3829 uint64_t len = u2(*rng);
3830 if (write_alignment) {
3831 offset = ROUND_UP_TO(offset, write_alignment);
3832 len = ROUND_UP_TO(len, write_alignment);
3835 auto& data = contents[new_obj].data;
3836 if (data.length() < offset + len) {
3837 data.append_zero(offset+len-data.length());
3840 n.substr_of(data, 0, offset);
3842 if (data.length() > offset + len)
3843 data.copy(offset + len, data.length() - offset - len, n);
3846 t.zero(cid, new_obj, offset, len);
3848 in_flight_objects.insert(new_obj);
3849 int status = store->queue_transaction(
3850 osr, std::move(t), new C_SyntheticOnReadable(this, new_obj));
3855 EnterExit ee("read");
3856 boost::uniform_int<> u1(0, max_object_len/2);
3857 boost::uniform_int<> u2(0, max_object_len);
3858 uint64_t offset = u1(*rng);
3859 uint64_t len = u2(*rng);
3864 bufferlist expected;
3867 Mutex::Locker locker(lock);
3868 EnterExit ee("read locked");
3873 obj = get_uniform_random_object();
3874 expected = contents[obj].data;
3876 bufferlist bl, result;
3877 if (0) cout << " obj " << obj
3878 << " size " << expected.length()
3879 << " offset " << offset
3880 << " len " << len << std::endl;
3881 r = store->read(cid, obj, offset, len, result);
3882 if (offset >= expected.length()) {
3885 size_t max_len = expected.length() - offset;
3888 assert(len == result.length());
3889 ASSERT_EQ(len, result.length());
3890 expected.copy(offset, len, bl);
3891 ASSERT_EQ(r, (int)len);
3892 ASSERT_TRUE(bl_eq(bl, result));
3897 Mutex::Locker locker(lock);
3898 EnterExit ee("setattrs");
3903 ghobject_t obj = get_uniform_random_object();
3904 available_objects.erase(obj);
3905 ObjectStore::Transaction t;
3907 boost::uniform_int<> u0(1, max_attr_size);
3908 boost::uniform_int<> u1(4, max_attr_name_len);
3909 boost::uniform_int<> u2(4, max_attr_value_len);
3910 boost::uniform_int<> u3(0, 100);
3911 uint64_t size = u0(*rng);
3913 map<string, bufferlist> attrs;
3915 for (map<string, bufferlist>::iterator it = contents[obj].attrs.begin();
3916 it != contents[obj].attrs.end(); ++it)
3917 keys.insert(it->first);
3920 bufferlist name, value;
3921 uint64_t get_exist = u3(*rng);
3922 uint64_t value_len = u2(*rng);
3923 filled_byte_array(value, value_len);
3924 if (get_exist < 50 && keys.size()) {
3925 set<string>::iterator k = keys.begin();
3927 contents[obj].attrs[*k] = value;
3930 name_len = u1(*rng);
3931 filled_byte_array(name, name_len);
3932 attrs[name.c_str()] = value;
3933 contents[obj].attrs[name.c_str()] = value;
3936 t.setattrs(cid, obj, attrs);
3938 in_flight_objects.insert(obj);
3939 int status = store->queue_transaction(
3940 osr, std::move(t), new C_SyntheticOnReadable(this, obj));
3945 EnterExit ee("getattrs");
3947 map<string, bufferlist> expected;
3949 Mutex::Locker locker(lock);
3950 EnterExit ee("getattrs locked");
3957 obj = get_uniform_random_object();
3960 } while (contents[obj].attrs.empty());
3961 expected = contents[obj].attrs;
3963 map<string, bufferlist> attrs;
3964 int r = store->getattrs(cid, obj, attrs);
3965 ASSERT_TRUE(r == 0);
3966 ASSERT_TRUE(attrs.size() == expected.size());
3967 for (map<string, bufferlist>::iterator it = expected.begin();
3968 it != expected.end(); ++it) {
3969 ASSERT_TRUE(bl_eq(attrs[it->first], it->second));
3974 EnterExit ee("getattr");
3978 map<string, bufferlist> expected;
3980 Mutex::Locker locker(lock);
3981 EnterExit ee("getattr locked");
3988 obj = get_uniform_random_object();
3991 } while (contents[obj].attrs.empty());
3992 expected = contents[obj].attrs;
3994 boost::uniform_int<> u(0, expected.size()-1);
3996 map<string, bufferlist>::iterator it = expected.begin();
4003 r = store->getattr(cid, obj, it->first, bl);
4005 ASSERT_TRUE(bl_eq(it->second, bl));
4009 Mutex::Locker locker(lock);
4010 EnterExit ee("rmattr");
4018 obj = get_uniform_random_object();
4021 } while (contents[obj].attrs.empty());
4023 boost::uniform_int<> u(0, contents[obj].attrs.size()-1);
4025 map<string, bufferlist>::iterator it = contents[obj].attrs.begin();
4031 available_objects.erase(obj);
4032 ObjectStore::Transaction t;
4033 t.rmattr(cid, obj, it->first);
4035 contents[obj].attrs.erase(it->first);
4037 in_flight_objects.insert(obj);
4038 int status = store->queue_transaction(
4039 osr, std::move(t), new C_SyntheticOnReadable(this, obj));
4043 void fsck(bool deep) {
4044 Mutex::Locker locker(lock);
4045 EnterExit ee("fsck");
4049 int r = store->fsck(deep);
4050 assert(r == 0 || r == -EOPNOTSUPP);
4055 Mutex::Locker locker(lock);
4056 EnterExit ee("scan");
4059 vector<ghobject_t> objects;
4060 set<ghobject_t> objects_set, objects_set2;
4061 ghobject_t next, current;
4063 //cerr << "scanning..." << std::endl;
4064 int r = store->collection_list(cid, current, ghobject_t::get_max(), 100,
4067 ASSERT_TRUE(sorted(objects));
4068 objects_set.insert(objects.begin(), objects.end());
4070 if (next.is_max()) break;
4073 if (objects_set.size() != available_objects.size()) {
4074 for (set<ghobject_t>::iterator p = objects_set.begin();
4075 p != objects_set.end();
4077 if (available_objects.count(*p) == 0) {
4078 cerr << "+ " << *p << std::endl;
4081 for (set<ghobject_t>::iterator p = available_objects.begin();
4082 p != available_objects.end();
4084 if (objects_set.count(*p) == 0)
4085 cerr << "- " << *p << std::endl;
4086 //cerr << " objects_set: " << objects_set << std::endl;
4087 //cerr << " available_set: " << available_objects << std::endl;
4088 assert(0 == "badness");
4091 ASSERT_EQ(objects_set.size(), available_objects.size());
4092 for (set<ghobject_t>::iterator i = objects_set.begin();
4093 i != objects_set.end();
4095 ASSERT_GT(available_objects.count(*i), (unsigned)0);
4098 int r = store->collection_list(cid, ghobject_t(), ghobject_t::get_max(),
4099 INT_MAX, &objects, 0);
4101 objects_set2.insert(objects.begin(), objects.end());
4102 ASSERT_EQ(objects_set2.size(), available_objects.size());
4103 for (set<ghobject_t>::iterator i = objects_set2.begin();
4104 i != objects_set2.end();
4106 ASSERT_GT(available_objects.count(*i), (unsigned)0);
4107 if (available_objects.count(*i) == 0) {
4108 cerr << "+ " << *i << std::endl;
4114 EnterExit ee("stat");
4118 Mutex::Locker locker(lock);
4119 EnterExit ee("stat lock1");
4122 hoid = get_uniform_random_object();
4123 in_flight_objects.insert(hoid);
4124 available_objects.erase(hoid);
4126 expected = contents[hoid].data.length();
4129 int r = store->stat(cid, hoid, &buf);
4131 assert((uint64_t)buf.st_size == expected);
4132 ASSERT_TRUE((uint64_t)buf.st_size == expected);
4134 Mutex::Locker locker(lock);
4135 EnterExit ee("stat lock2");
4138 in_flight_objects.erase(hoid);
4139 available_objects.insert(hoid);
4144 Mutex::Locker locker(lock);
4145 EnterExit ee("unlink");
4148 ghobject_t to_remove = get_uniform_random_object();
4149 ObjectStore::Transaction t;
4150 t.remove(cid, to_remove);
4152 available_objects.erase(to_remove);
4153 in_flight_objects.insert(to_remove);
4154 contents.erase(to_remove);
4155 int status = store->queue_transaction(osr, std::move(t), new C_SyntheticOnReadable(this, to_remove));
4159 void print_internal_state() {
4160 Mutex::Locker locker(lock);
4161 cerr << "available_objects: " << available_objects.size()
4162 << " in_flight_objects: " << in_flight_objects.size()
4163 << " total objects: " << in_flight_objects.size() + available_objects.size()
4164 << " in_flight " << in_flight << std::endl;
4169 void doSyntheticTest(boost::scoped_ptr<ObjectStore>& store,
4171 uint64_t max_obj, uint64_t max_wr, uint64_t align)
4173 ObjectStore::Sequencer osr("test");
4174 MixedGenerator gen(555);
4175 gen_type rng(time(NULL));
4176 coll_t cid(spg_t(pg_t(0,555), shard_id_t::NO_SHARD));
4178 g_ceph_context->_conf->set_val("bluestore_fsck_on_mount", "false");
4179 g_ceph_context->_conf->set_val("bluestore_fsck_on_umount", "false");
4180 g_ceph_context->_conf->apply_changes(NULL);
4182 SyntheticWorkloadState test_obj(store.get(), &gen, &rng, &osr, cid,
4183 max_obj, max_wr, align);
4185 for (int i = 0; i < num_ops/10; ++i) {
4186 if (!(i % 500)) cerr << "seeding object " << i << std::endl;
4189 for (int i = 0; i < num_ops; ++i) {
4191 cerr << "Op " << i << std::endl;
4192 test_obj.print_internal_state();
4194 boost::uniform_int<> true_false(0, 999);
4195 int val = true_false(rng);
4197 test_obj.fsck(true);
4198 } else if (val > 997) {
4199 test_obj.fsck(false);
4200 } else if (val > 970) {
4202 } else if (val > 950) {
4204 } else if (val > 850) {
4206 } else if (val > 800) {
4208 } else if (val > 550) {
4210 } else if (val > 500) {
4212 } else if (val > 450) {
4213 test_obj.clone_range();
4214 } else if (val > 300) {
4216 } else if (val > 100) {
4219 test_obj.truncate();
4222 test_obj.wait_for_done();
4223 test_obj.shutdown();
4225 g_ceph_context->_conf->set_val("bluestore_fsck_on_mount", "true");
4226 g_ceph_context->_conf->set_val("bluestore_fsck_on_umount", "true");
4227 g_ceph_context->_conf->apply_changes(NULL);
4230 TEST_P(StoreTest, Synthetic) {
4231 doSyntheticTest(store, 10000, 400*1024, 40*1024, 0);
4235 TEST_P(StoreTestSpecificAUSize, SyntheticMatrixSharding) {
4236 if (string(GetParam()) != "bluestore")
4239 const char *m[][10] = {
4240 { "bluestore_min_alloc_size", "4096", 0 }, // must be the first!
4241 { "num_ops", "50000", 0 },
4242 { "max_write", "65536", 0 },
4243 { "max_size", "262144", 0 },
4244 { "alignment", "4096", 0 },
4245 { "bluestore_max_blob_size", "65536", 0 },
4246 { "bluestore_extent_map_shard_min_size", "60", 0 },
4247 { "bluestore_extent_map_shard_max_size", "300", 0 },
4248 { "bluestore_extent_map_shard_target_size", "150", 0 },
4249 { "bluestore_default_buffered_read", "true", 0 },
4250 { "bluestore_default_buffered_write", "true", 0 },
4253 do_matrix(m, store, doSyntheticTest);
4256 TEST_P(StoreTestSpecificAUSize, ZipperPatternSharded) {
4257 if(string(GetParam()) != "bluestore")
4259 StartDeferred(4096);
4262 ObjectStore::Sequencer osr("test");
4264 ghobject_t a(hobject_t(sobject_t("Object 1", CEPH_NOSNAP)));
4266 ObjectStore::Transaction t;
4267 t.create_collection(cid, 0);
4268 cerr << "Creating collection " << cid << std::endl;
4269 r = apply_transaction(store, &osr, std::move(t));
4277 for (int i=0; i<1000; ++i) {
4278 ObjectStore::Transaction t;
4279 t.write(cid, a, i*2*len, len, bl, 0);
4280 r = apply_transaction(store, &osr, std::move(t));
4283 for (int i=0; i<1000; ++i) {
4284 ObjectStore::Transaction t;
4285 t.write(cid, a, i*2*len + 1, len, bl, 0);
4286 r = apply_transaction(store, &osr, std::move(t));
4290 ObjectStore::Transaction t;
4292 t.remove_collection(cid);
4293 cerr << "Cleaning" << std::endl;
4294 r = apply_transaction(store, &osr, std::move(t));
4299 TEST_P(StoreTestSpecificAUSize, SyntheticMatrixCsumAlgorithm) {
4300 if (string(GetParam()) != "bluestore")
4303 const char *m[][10] = {
4304 { "bluestore_min_alloc_size", "65536", 0 }, // must be the first!
4305 { "max_write", "65536", 0 },
4306 { "max_size", "1048576", 0 },
4307 { "alignment", "16", 0 },
4308 { "bluestore_csum_type", "crc32c", "crc32c_16", "crc32c_8", "xxhash32",
4309 "xxhash64", "none", 0 },
4310 { "bluestore_default_buffered_write", "false", 0 },
4313 do_matrix(m, store, doSyntheticTest);
4316 TEST_P(StoreTestSpecificAUSize, SyntheticMatrixCsumVsCompression) {
4317 if (string(GetParam()) != "bluestore")
4320 const char *m[][10] = {
4321 { "bluestore_min_alloc_size", "4096", "16384", 0 }, //to be the first!
4322 { "max_write", "131072", 0 },
4323 { "max_size", "262144", 0 },
4324 { "alignment", "512", 0 },
4325 { "bluestore_compression_mode", "force", 0},
4326 { "bluestore_compression_algorithm", "snappy", "zlib", 0 },
4327 { "bluestore_csum_type", "crc32c", 0 },
4328 { "bluestore_default_buffered_read", "true", "false", 0 },
4329 { "bluestore_default_buffered_write", "true", "false", 0 },
4330 { "bluestore_sync_submit_transaction", "false", 0 },
4333 do_matrix(m, store, doSyntheticTest);
4336 TEST_P(StoreTestSpecificAUSize, SyntheticMatrixCompression) {
4337 if (string(GetParam()) != "bluestore")
4340 const char *m[][10] = {
4341 { "bluestore_min_alloc_size", "4096", "65536", 0 }, // to be the first!
4342 { "max_write", "1048576", 0 },
4343 { "max_size", "4194304", 0 },
4344 { "alignment", "65536", 0 },
4345 { "bluestore_compression_mode", "force", "aggressive", "passive", "none", 0},
4346 { "bluestore_default_buffered_write", "false", 0 },
4347 { "bluestore_sync_submit_transaction", "true", 0 },
4350 do_matrix(m, store, doSyntheticTest);
4353 TEST_P(StoreTestSpecificAUSize, SyntheticMatrixCompressionAlgorithm) {
4354 if (string(GetParam()) != "bluestore")
4357 const char *m[][10] = {
4358 { "bluestore_min_alloc_size", "4096", "65536", 0 }, // to be the first!
4359 { "max_write", "1048576", 0 },
4360 { "max_size", "4194304", 0 },
4361 { "alignment", "65536", 0 },
4362 { "bluestore_compression_algorithm", "zlib", "snappy", 0 },
4363 { "bluestore_compression_mode", "force", 0 },
4364 { "bluestore_default_buffered_write", "false", 0 },
4367 do_matrix(m, store, doSyntheticTest);
4370 TEST_P(StoreTestSpecificAUSize, SyntheticMatrixNoCsum) {
4371 if (string(GetParam()) != "bluestore")
4374 const char *m[][10] = {
4375 { "bluestore_min_alloc_size", "4096", "65536", 0 }, // to be the first!
4376 { "max_write", "65536", 0 },
4377 { "max_size", "1048576", 0 },
4378 { "alignment", "512", 0 },
4379 { "bluestore_max_blob_size", "262144", 0 },
4380 { "bluestore_compression_mode", "force", "none", 0},
4381 { "bluestore_csum_type", "none", 0},
4382 { "bluestore_default_buffered_read", "true", "false", 0 },
4383 { "bluestore_default_buffered_write", "true", 0 },
4384 { "bluestore_sync_submit_transaction", "true", "false", 0 },
4387 do_matrix(m, store, doSyntheticTest);
4390 TEST_P(StoreTestSpecificAUSize, SyntheticMatrixPreferDeferred) {
4391 if (string(GetParam()) != "bluestore")
4394 const char *m[][10] = {
4395 { "bluestore_min_alloc_size", "4096", "65536", 0 }, // to be the first!
4396 { "max_write", "65536", 0 },
4397 { "max_size", "1048576", 0 },
4398 { "alignment", "512", 0 },
4399 { "bluestore_max_blob_size", "262144", 0 },
4400 { "bluestore_compression_mode", "force", "none", 0},
4401 { "bluestore_prefer_deferred_size", "32768", "0", 0},
4404 do_matrix(m, store, doSyntheticTest);
4407 TEST_P(StoreTest, AttrSynthetic) {
4408 ObjectStore::Sequencer osr("test");
4409 MixedGenerator gen(447);
4410 gen_type rng(time(NULL));
4411 coll_t cid(spg_t(pg_t(0,447),shard_id_t::NO_SHARD));
4413 SyntheticWorkloadState test_obj(store.get(), &gen, &rng, &osr, cid, 40*1024, 4*1024, 0);
4415 for (int i = 0; i < 500; ++i) {
4416 if (!(i % 10)) cerr << "seeding object " << i << std::endl;
4419 for (int i = 0; i < 1000; ++i) {
4421 cerr << "Op " << i << std::endl;
4422 test_obj.print_internal_state();
4424 boost::uniform_int<> true_false(0, 99);
4425 int val = true_false(rng);
4428 } else if (val > 93) {
4430 } else if (val > 75) {
4432 } else if (val > 47) {
4433 test_obj.setattrs();
4434 } else if (val > 45) {
4436 } else if (val > 37) {
4438 } else if (val > 30) {
4439 test_obj.getattrs();
4444 test_obj.wait_for_done();
4445 test_obj.shutdown();
4448 TEST_P(StoreTest, HashCollisionTest) {
4449 ObjectStore::Sequencer osr("test");
4450 int64_t poolid = 11;
4451 coll_t cid(spg_t(pg_t(0,poolid),shard_id_t::NO_SHARD));
4454 ObjectStore::Transaction t;
4455 t.create_collection(cid, 0);
4456 r = apply_transaction(store, &osr, std::move(t));
4460 for (int i = 0; i < 100; ++i) base.append("aaaaa");
4461 set<ghobject_t> created;
4462 for (int n = 0; n < 10; ++n) {
4464 sprintf(nbuf, "n%d", n);
4465 for (int i = 0; i < 1000; ++i) {
4467 sprintf(buf, "%d", i);
4469 cerr << "Object n" << n << " "<< i << std::endl;
4471 ghobject_t hoid(hobject_t(string(buf) + base, string(), CEPH_NOSNAP, 0, poolid, string(nbuf)));
4473 ObjectStore::Transaction t;
4475 r = apply_transaction(store, &osr, std::move(t));
4478 created.insert(hoid);
4481 vector<ghobject_t> objects;
4482 r = store->collection_list(cid, ghobject_t(), ghobject_t::get_max(), INT_MAX, &objects, 0);
4484 set<ghobject_t> listed(objects.begin(), objects.end());
4485 cerr << "listed.size() is " << listed.size() << " and created.size() is " << created.size() << std::endl;
4486 ASSERT_TRUE(listed.size() == created.size());
4489 ghobject_t current, next;
4491 r = store->collection_list(cid, current, ghobject_t::get_max(), 60,
4494 ASSERT_TRUE(sorted(objects));
4495 for (vector<ghobject_t>::iterator i = objects.begin();
4498 if (listed.count(*i))
4499 cerr << *i << " repeated" << std::endl;
4502 if (objects.size() < 50) {
4503 ASSERT_TRUE(next.is_max());
4509 cerr << "listed.size() is " << listed.size() << std::endl;
4510 ASSERT_TRUE(listed.size() == created.size());
4511 for (set<ghobject_t>::iterator i = listed.begin();
4514 ASSERT_TRUE(created.count(*i));
4517 for (set<ghobject_t>::iterator i = created.begin();
4520 ObjectStore::Transaction t;
4522 r = apply_transaction(store, &osr, std::move(t));
4525 ObjectStore::Transaction t;
4526 t.remove_collection(cid);
4527 r = apply_transaction(store, &osr, std::move(t));
4531 TEST_P(StoreTest, ScrubTest) {
4532 ObjectStore::Sequencer osr("test");
4533 int64_t poolid = 111;
4534 coll_t cid(spg_t(pg_t(0, poolid),shard_id_t(1)));
4537 ObjectStore::Transaction t;
4538 t.create_collection(cid, 0);
4539 r = apply_transaction(store, &osr, std::move(t));
4542 string base = "aaaaa";
4543 set<ghobject_t> created;
4544 for (int i = 0; i < 1000; ++i) {
4546 sprintf(buf, "%d", i);
4548 cerr << "Object " << i << std::endl;
4550 ghobject_t hoid(hobject_t(string(buf) + base, string(), CEPH_NOSNAP, i,
4552 ghobject_t::NO_GEN, shard_id_t(1));
4554 ObjectStore::Transaction t;
4556 r = apply_transaction(store, &osr, std::move(t));
4559 created.insert(hoid);
4562 // Add same hobject_t but different generation
4564 ghobject_t hoid1(hobject_t("same-object", string(), CEPH_NOSNAP, 0, poolid, ""),
4565 ghobject_t::NO_GEN, shard_id_t(1));
4566 ghobject_t hoid2(hobject_t("same-object", string(), CEPH_NOSNAP, 0, poolid, ""), (gen_t)1, shard_id_t(1));
4567 ghobject_t hoid3(hobject_t("same-object", string(), CEPH_NOSNAP, 0, poolid, ""), (gen_t)2, shard_id_t(1));
4568 ObjectStore::Transaction t;
4569 t.touch(cid, hoid1);
4570 t.touch(cid, hoid2);
4571 t.touch(cid, hoid3);
4572 r = apply_transaction(store, &osr, std::move(t));
4573 created.insert(hoid1);
4574 created.insert(hoid2);
4575 created.insert(hoid3);
4579 vector<ghobject_t> objects;
4580 r = store->collection_list(cid, ghobject_t(), ghobject_t::get_max(),
4581 INT_MAX, &objects, 0);
4583 set<ghobject_t> listed(objects.begin(), objects.end());
4584 cerr << "listed.size() is " << listed.size() << " and created.size() is " << created.size() << std::endl;
4585 ASSERT_TRUE(listed.size() == created.size());
4588 ghobject_t current, next;
4590 r = store->collection_list(cid, current, ghobject_t::get_max(), 60,
4593 ASSERT_TRUE(sorted(objects));
4594 for (vector<ghobject_t>::iterator i = objects.begin();
4595 i != objects.end(); ++i) {
4596 if (listed.count(*i))
4597 cerr << *i << " repeated" << std::endl;
4600 if (objects.size() < 50) {
4601 ASSERT_TRUE(next.is_max());
4605 current = next.get_boundary();
4607 cerr << "listed.size() is " << listed.size() << std::endl;
4608 ASSERT_TRUE(listed.size() == created.size());
4609 for (set<ghobject_t>::iterator i = listed.begin();
4612 ASSERT_TRUE(created.count(*i));
4615 for (set<ghobject_t>::iterator i = created.begin();
4618 ObjectStore::Transaction t;
4620 r = apply_transaction(store, &osr, std::move(t));
4623 ObjectStore::Transaction t;
4624 t.remove_collection(cid);
4625 r = apply_transaction(store, &osr, std::move(t));
4630 TEST_P(StoreTest, OMapTest) {
4631 ObjectStore::Sequencer osr("test");
4633 ghobject_t hoid(hobject_t("tesomap", "", CEPH_NOSNAP, 0, 0, ""));
4636 ObjectStore::Transaction t;
4637 t.create_collection(cid, 0);
4638 r = apply_transaction(store, &osr, std::move(t));
4642 map<string, bufferlist> attrs;
4644 ObjectStore::Transaction t;
4646 t.omap_clear(cid, hoid);
4647 map<string, bufferlist> start_set;
4648 t.omap_setkeys(cid, hoid, start_set);
4649 r = apply_transaction(store, &osr, std::move(t));
4653 for (int i = 0; i < 100; i++) {
4655 std::cout << "On iteration " << i << std::endl;
4657 ObjectStore::Transaction t;
4659 map<string, bufferlist> cur_attrs;
4660 r = store->omap_get(cid, hoid, &bl, &cur_attrs);
4662 for (map<string, bufferlist>::iterator j = attrs.begin();
4665 bool correct = cur_attrs.count(j->first) && string(cur_attrs[j->first].c_str()) == string(j->second.c_str());
4667 std::cout << j->first << " is present in cur_attrs " << cur_attrs.count(j->first) << " times " << std::endl;
4668 if (cur_attrs.count(j->first) > 0) {
4669 std::cout << j->second.c_str() << " : " << cur_attrs[j->first].c_str() << std::endl;
4672 ASSERT_EQ(correct, true);
4674 ASSERT_EQ(attrs.size(), cur_attrs.size());
4677 snprintf(buf, sizeof(buf), "%d", i);
4679 bufferptr bp(buf, strlen(buf) + 1);
4681 map<string, bufferlist> to_add;
4682 to_add.insert(pair<string, bufferlist>("key-" + string(buf), bl));
4683 attrs.insert(pair<string, bufferlist>("key-" + string(buf), bl));
4684 t.omap_setkeys(cid, hoid, to_add);
4685 r = apply_transaction(store, &osr, std::move(t));
4690 while (attrs.size()) {
4692 std::cout << "removal: On iteration " << i << std::endl;
4694 ObjectStore::Transaction t;
4696 map<string, bufferlist> cur_attrs;
4697 r = store->omap_get(cid, hoid, &bl, &cur_attrs);
4699 for (map<string, bufferlist>::iterator j = attrs.begin();
4702 bool correct = cur_attrs.count(j->first) && string(cur_attrs[j->first].c_str()) == string(j->second.c_str());
4704 std::cout << j->first << " is present in cur_attrs " << cur_attrs.count(j->first) << " times " << std::endl;
4705 if (cur_attrs.count(j->first) > 0) {
4706 std::cout << j->second.c_str() << " : " << cur_attrs[j->first].c_str() << std::endl;
4709 ASSERT_EQ(correct, true);
4712 string to_remove = attrs.begin()->first;
4713 set<string> keys_to_remove;
4714 keys_to_remove.insert(to_remove);
4715 t.omap_rmkeys(cid, hoid, keys_to_remove);
4716 r = apply_transaction(store, &osr, std::move(t));
4719 attrs.erase(to_remove);
4726 bl1.append("omap_header");
4727 ObjectStore::Transaction t;
4728 t.omap_setheader(cid, hoid, bl1);
4729 r = apply_transaction(store, &osr, std::move(t));
4731 t = ObjectStore::Transaction();
4734 bl2.append("value");
4735 map<string, bufferlist> to_add;
4736 to_add.insert(pair<string, bufferlist>("key", bl2));
4737 t.omap_setkeys(cid, hoid, to_add);
4738 r = apply_transaction(store, &osr, std::move(t));
4742 map<string, bufferlist> cur_attrs;
4743 r = store->omap_get(cid, hoid, &bl3, &cur_attrs);
4745 ASSERT_EQ(cur_attrs.size(), size_t(1));
4746 ASSERT_TRUE(bl_eq(bl1, bl3));
4749 r = store->omap_get_keys(cid, hoid, &keys);
4751 ASSERT_EQ(keys.size(), size_t(1));
4754 // test omap_clear, omap_rmkey_range
4757 map<string,bufferlist> to_set;
4758 for (int n=0; n<10; ++n) {
4759 to_set[stringify(n)].append("foo");
4763 ObjectStore::Transaction t;
4764 t.remove(cid, hoid);
4766 t.omap_setheader(cid, hoid, h);
4767 t.omap_setkeys(cid, hoid, to_set);
4768 r = apply_transaction(store, &osr, std::move(t));
4772 ObjectStore::Transaction t;
4773 t.omap_rmkeyrange(cid, hoid, "3", "7");
4774 r = apply_transaction(store, &osr, std::move(t));
4779 map<string,bufferlist> m;
4780 store->omap_get(cid, hoid, &hdr, &m);
4781 ASSERT_EQ(6u, hdr.length());
4782 ASSERT_TRUE(m.count("2"));
4783 ASSERT_TRUE(!m.count("3"));
4784 ASSERT_TRUE(!m.count("6"));
4785 ASSERT_TRUE(m.count("7"));
4786 ASSERT_TRUE(m.count("8"));
4787 //cout << m << std::endl;
4788 ASSERT_EQ(6u, m.size());
4791 ObjectStore::Transaction t;
4792 t.omap_clear(cid, hoid);
4793 r = apply_transaction(store, &osr, std::move(t));
4798 map<string,bufferlist> m;
4799 store->omap_get(cid, hoid, &hdr, &m);
4800 ASSERT_EQ(0u, hdr.length());
4801 ASSERT_EQ(0u, m.size());
4805 ObjectStore::Transaction t;
4806 t.remove(cid, hoid);
4807 t.remove_collection(cid);
4808 r = apply_transaction(store, &osr, std::move(t));
4812 TEST_P(StoreTest, OMapIterator) {
4813 ObjectStore::Sequencer osr("test");
4815 ghobject_t hoid(hobject_t("tesomap", "", CEPH_NOSNAP, 0, 0, ""));
4819 ObjectStore::Transaction t;
4820 t.create_collection(cid, 0);
4821 r = apply_transaction(store, &osr, std::move(t));
4825 map<string, bufferlist> attrs;
4827 ObjectStore::Transaction t;
4829 t.omap_clear(cid, hoid);
4830 map<string, bufferlist> start_set;
4831 t.omap_setkeys(cid, hoid, start_set);
4832 r = apply_transaction(store, &osr, std::move(t));
4835 ObjectMap::ObjectMapIterator iter;
4838 for (int i = 0; i < 100; i++) {
4840 std::cout << "On iteration " << i << std::endl;
4844 // FileStore may deadlock two active iterators over the same data
4845 iter = ObjectMap::ObjectMapIterator();
4847 iter = store->get_omap_iterator(cid, hoid);
4848 for (iter->seek_to_first(), count=0; iter->valid(); iter->next(), count++) {
4849 string key = iter->key();
4850 bufferlist value = iter->value();
4851 correct = attrs.count(key) && (string(value.c_str()) == string(attrs[key].c_str()));
4853 if (attrs.count(key) > 0) {
4854 std::cout << "key " << key << "in omap , " << value.c_str() << " : " << attrs[key].c_str() << std::endl;
4857 std::cout << "key " << key << "should not exists in omap" << std::endl;
4859 ASSERT_EQ(correct, true);
4861 ASSERT_EQ((int)attrs.size(), count);
4863 // FileStore may deadlock an active iterator vs apply_transaction
4864 iter = ObjectMap::ObjectMapIterator();
4867 snprintf(buf, sizeof(buf), "%d", i);
4869 bufferptr bp(buf, strlen(buf) + 1);
4871 map<string, bufferlist> to_add;
4872 to_add.insert(pair<string, bufferlist>("key-" + string(buf), bl));
4873 attrs.insert(pair<string, bufferlist>("key-" + string(buf), bl));
4874 ObjectStore::Transaction t;
4875 t.omap_setkeys(cid, hoid, to_add);
4876 r = apply_transaction(store, &osr, std::move(t));
4880 iter = store->get_omap_iterator(cid, hoid);
4882 string bound_key = "key-5";
4883 iter->lower_bound(bound_key);
4884 correct = bound_key <= iter->key();
4886 std::cout << "lower bound, bound key is " << bound_key << " < iter key is " << iter->key() << std::endl;
4888 ASSERT_EQ(correct, true);
4890 iter->upper_bound(bound_key);
4891 correct = iter->key() > bound_key;
4893 std::cout << "upper bound, bound key is " << bound_key << " >= iter key is " << iter->key() << std::endl;
4895 ASSERT_EQ(correct, true);
4897 // FileStore may deadlock an active iterator vs apply_transaction
4898 iter = ObjectMap::ObjectMapIterator();
4900 ObjectStore::Transaction t;
4901 t.remove(cid, hoid);
4902 t.remove_collection(cid);
4903 r = apply_transaction(store, &osr, std::move(t));
4908 TEST_P(StoreTest, XattrTest) {
4909 ObjectStore::Sequencer osr("test");
4911 ghobject_t hoid(hobject_t("tesomap", "", CEPH_NOSNAP, 0, 0, ""));
4913 for (unsigned i = 0; i < 10000; ++i) {
4917 for (unsigned i = 0; i < 10; ++i) {
4922 ObjectStore::Transaction t;
4923 t.create_collection(cid, 0);
4925 r = apply_transaction(store, &osr, std::move(t));
4929 map<string, bufferlist> attrs;
4931 ObjectStore::Transaction t;
4932 t.setattr(cid, hoid, "attr1", small);
4933 attrs["attr1"] = small;
4934 t.setattr(cid, hoid, "attr2", big);
4935 attrs["attr2"] = big;
4936 t.setattr(cid, hoid, "attr3", small);
4937 attrs["attr3"] = small;
4938 t.setattr(cid, hoid, "attr1", small);
4939 attrs["attr1"] = small;
4940 t.setattr(cid, hoid, "attr4", big);
4941 attrs["attr4"] = big;
4942 t.setattr(cid, hoid, "attr3", big);
4943 attrs["attr3"] = big;
4944 r = apply_transaction(store, &osr, std::move(t));
4948 map<string, bufferptr> aset;
4949 store->getattrs(cid, hoid, aset);
4950 ASSERT_EQ(aset.size(), attrs.size());
4951 for (map<string, bufferptr>::iterator i = aset.begin();
4955 bl.push_back(i->second);
4956 ASSERT_TRUE(attrs[i->first] == bl);
4960 ObjectStore::Transaction t;
4961 t.rmattr(cid, hoid, "attr2");
4962 attrs.erase("attr2");
4963 r = apply_transaction(store, &osr, std::move(t));
4968 store->getattrs(cid, hoid, aset);
4969 ASSERT_EQ(aset.size(), attrs.size());
4970 for (map<string, bufferptr>::iterator i = aset.begin();
4974 bl.push_back(i->second);
4975 ASSERT_TRUE(attrs[i->first] == bl);
4979 r = store->getattr(cid, hoid, "attr2", bp);
4980 ASSERT_EQ(r, -ENODATA);
4982 r = store->getattr(cid, hoid, "attr3", bp);
4986 ASSERT_TRUE(bl2 == attrs["attr3"]);
4988 ObjectStore::Transaction t;
4989 t.remove(cid, hoid);
4990 t.remove_collection(cid);
4991 r = apply_transaction(store, &osr, std::move(t));
4997 unsigned num_objects,
4998 unsigned common_suffix_size,
5001 ObjectStore::Sequencer osr("test");
5002 coll_t cid(spg_t(pg_t(0,52),shard_id_t::NO_SHARD));
5003 coll_t tid(spg_t(pg_t(1<<common_suffix_size,52),shard_id_t::NO_SHARD));
5006 ObjectStore::Transaction t;
5007 t.create_collection(cid, common_suffix_size);
5008 r = apply_transaction(store, &osr, std::move(t));
5012 small.append("small");
5014 ObjectStore::Transaction t;
5015 for (uint32_t i = 0; i < (2 - (int)clones)*num_objects; ++i) {
5016 stringstream objname;
5017 objname << "obj" << i;
5018 ghobject_t a(hobject_t(
5022 i<<common_suffix_size,
5024 t.write(cid, a, 0, small.length(), small,
5025 CEPH_OSD_OP_FLAG_FADVISE_WILLNEED);
5027 objname << "-clone";
5028 ghobject_t b(hobject_t(
5032 i<<common_suffix_size,
5037 r = apply_transaction(store, &osr, std::move(t));
5039 t = ObjectStore::Transaction();
5042 r = apply_transaction(store, &osr, std::move(t));
5046 ObjectStore::Transaction t;
5047 t.create_collection(tid, common_suffix_size + 1);
5048 t.split_collection(cid, common_suffix_size+1, 1<<common_suffix_size, tid);
5049 r = apply_transaction(store, &osr, std::move(t));
5053 ObjectStore::Transaction t;
5054 vector<ghobject_t> objects;
5055 r = store->collection_list(cid, ghobject_t(), ghobject_t::get_max(),
5056 INT_MAX, &objects, 0);
5058 ASSERT_EQ(objects.size(), num_objects);
5060 for (vector<ghobject_t>::iterator i = objects.begin();
5063 ASSERT_EQ(!!(i->hobj.get_hash() & (1<<common_suffix_size)), 0u);
5067 r = apply_transaction(store, &osr, std::move(t));
5069 t = ObjectStore::Transaction();
5074 r = store->collection_list(tid, ghobject_t(), ghobject_t::get_max(),
5075 INT_MAX, &objects, 0);
5077 ASSERT_EQ(objects.size(), num_objects);
5078 for (vector<ghobject_t>::iterator i = objects.begin();
5081 ASSERT_EQ(!(i->hobj.get_hash() & (1<<common_suffix_size)), 0u);
5085 r = apply_transaction(store, &osr, std::move(t));
5087 t = ObjectStore::Transaction();
5091 t.remove_collection(cid);
5092 t.remove_collection(tid);
5093 r = apply_transaction(store, &osr, std::move(t));
5097 TEST_P(StoreTest, ColSplitTest1) {
5098 colsplittest(store.get(), 10000, 11, false);
5100 TEST_P(StoreTest, ColSplitTest1Clones) {
5101 colsplittest(store.get(), 10000, 11, true);
5103 TEST_P(StoreTest, ColSplitTest2) {
5104 colsplittest(store.get(), 100, 7, false);
5106 TEST_P(StoreTest, ColSplitTest2Clones) {
5107 colsplittest(store.get(), 100, 7, true);
5111 TEST_P(StoreTest, ColSplitTest3) {
5112 colsplittest(store.get(), 100000, 25);
5117 * This test tests adding two different groups
5118 * of objects, each with 1 common prefix and 1
5119 * different prefix. We then remove half
5120 * in order to verify that the merging correctly
5121 * stops at the common prefix subdir. See bug
5123 TEST_P(StoreTest, TwoHash) {
5124 ObjectStore::Sequencer osr("test");
5128 ObjectStore::Transaction t;
5129 t.create_collection(cid, 0);
5130 r = apply_transaction(store, &osr, std::move(t));
5133 std::cout << "Making objects" << std::endl;
5134 for (int i = 0; i < 360; ++i) {
5135 ObjectStore::Transaction t;
5139 o.hobj.set_hash((i << 16) | 0xA1);
5142 o.hobj.set_hash((i << 16) | 0xB1);
5144 r = apply_transaction(store, &osr, std::move(t));
5147 std::cout << "Removing half" << std::endl;
5148 for (int i = 1; i < 8; ++i) {
5149 ObjectStore::Transaction t;
5152 o.hobj.set_hash((i << 16) | 0xA1);
5154 r = apply_transaction(store, &osr, std::move(t));
5157 std::cout << "Checking" << std::endl;
5158 for (int i = 1; i < 8; ++i) {
5159 ObjectStore::Transaction t;
5161 o.hobj.set_hash((i << 16) | 0xA1);
5163 bool exists = store->exists(cid, o);
5164 ASSERT_EQ(exists, false);
5168 o.hobj.set_hash(0xA1);
5170 bool exists = store->exists(cid, o);
5171 ASSERT_EQ(exists, true);
5173 std::cout << "Cleanup" << std::endl;
5174 for (int i = 0; i < 360; ++i) {
5175 ObjectStore::Transaction t;
5177 o.hobj.set_hash((i << 16) | 0xA1);
5180 o.hobj.set_hash((i << 16) | 0xB1);
5182 r = apply_transaction(store, &osr, std::move(t));
5185 ObjectStore::Transaction t;
5186 t.remove_collection(cid);
5187 r = apply_transaction(store, &osr, std::move(t));
5191 TEST_P(StoreTest, Rename) {
5192 ObjectStore::Sequencer osr("test");
5193 coll_t cid(spg_t(pg_t(0, 2122),shard_id_t::NO_SHARD));
5194 ghobject_t srcoid(hobject_t("src_oid", "", CEPH_NOSNAP, 0, 0, ""));
5195 ghobject_t dstoid(hobject_t("dest_oid", "", CEPH_NOSNAP, 0, 0, ""));
5201 ObjectStore::Transaction t;
5202 t.create_collection(cid, 0);
5203 t.write(cid, srcoid, 0, a.length(), a);
5204 r = apply_transaction(store, &osr, std::move(t));
5207 ASSERT_TRUE(store->exists(cid, srcoid));
5209 ObjectStore::Transaction t;
5210 t.collection_move_rename(cid, srcoid, cid, dstoid);
5211 t.write(cid, srcoid, 0, b.length(), b);
5212 t.setattr(cid, srcoid, "attr", b);
5213 r = apply_transaction(store, &osr, std::move(t));
5216 ASSERT_TRUE(store->exists(cid, srcoid));
5217 ASSERT_TRUE(store->exists(cid, dstoid));
5220 store->read(cid, srcoid, 0, 3, bl);
5221 ASSERT_TRUE(bl_eq(b, bl));
5222 store->read(cid, dstoid, 0, 3, bl);
5223 ASSERT_TRUE(bl_eq(a, bl));
5226 ObjectStore::Transaction t;
5227 t.remove(cid, dstoid);
5228 t.collection_move_rename(cid, srcoid, cid, dstoid);
5229 r = apply_transaction(store, &osr, std::move(t));
5232 ASSERT_TRUE(store->exists(cid, dstoid));
5233 ASSERT_FALSE(store->exists(cid, srcoid));
5236 store->read(cid, dstoid, 0, 3, bl);
5237 ASSERT_TRUE(bl_eq(b, bl));
5240 ObjectStore::Transaction t;
5241 t.remove(cid, dstoid);
5242 t.remove_collection(cid);
5243 r = apply_transaction(store, &osr, std::move(t));
5248 TEST_P(StoreTest, MoveRename) {
5249 ObjectStore::Sequencer osr("test");
5250 coll_t cid(spg_t(pg_t(0, 212),shard_id_t::NO_SHARD));
5251 ghobject_t temp_oid(hobject_t("tmp_oid", "", CEPH_NOSNAP, 0, 0, ""));
5252 ghobject_t oid(hobject_t("dest_oid", "", CEPH_NOSNAP, 0, 0, ""));
5255 ObjectStore::Transaction t;
5256 t.create_collection(cid, 0);
5258 r = apply_transaction(store, &osr, std::move(t));
5261 ASSERT_TRUE(store->exists(cid, oid));
5262 bufferlist data, attr;
5263 map<string, bufferlist> omap;
5264 data.append("data payload");
5265 attr.append("attr value");
5266 omap["omap_key"].append("omap value");
5268 ObjectStore::Transaction t;
5269 t.touch(cid, temp_oid);
5270 t.write(cid, temp_oid, 0, data.length(), data);
5271 t.setattr(cid, temp_oid, "attr", attr);
5272 t.omap_setkeys(cid, temp_oid, omap);
5273 r = apply_transaction(store, &osr, std::move(t));
5276 ASSERT_TRUE(store->exists(cid, temp_oid));
5278 ObjectStore::Transaction t;
5280 t.collection_move_rename(cid, temp_oid, cid, oid);
5281 r = apply_transaction(store, &osr, std::move(t));
5284 ASSERT_TRUE(store->exists(cid, oid));
5285 ASSERT_FALSE(store->exists(cid, temp_oid));
5288 r = store->read(cid, oid, 0, 1000, newdata);
5290 ASSERT_TRUE(bl_eq(data, newdata));
5292 r = store->getattr(cid, oid, "attr", newattr);
5294 ASSERT_TRUE(bl_eq(attr, newattr));
5296 keys.insert("omap_key");
5297 map<string, bufferlist> newomap;
5298 r = store->omap_get_values(cid, oid, keys, &newomap);
5300 ASSERT_EQ(1u, newomap.size());
5301 ASSERT_TRUE(newomap.count("omap_key"));
5302 ASSERT_TRUE(bl_eq(omap["omap_key"], newomap["omap_key"]));
5305 ObjectStore::Transaction t;
5307 t.remove_collection(cid);
5308 r = apply_transaction(store, &osr, std::move(t));
5313 TEST_P(StoreTest, BigRGWObjectName) {
5314 ObjectStore::Sequencer osr("test");
5315 coll_t cid(spg_t(pg_t(0,12),shard_id_t::NO_SHARD));
5318 "default.4106.50_aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
5325 shard_id_t::NO_SHARD);
5326 ghobject_t oid2(oid);
5327 oid2.generation = 17;
5328 ghobject_t oidhead(oid);
5329 oidhead.generation = ghobject_t::NO_GEN;
5333 ObjectStore::Transaction t;
5334 t.create_collection(cid, 0);
5335 t.touch(cid, oidhead);
5336 t.collection_move_rename(cid, oidhead, cid, oid);
5337 t.touch(cid, oidhead);
5338 t.collection_move_rename(cid, oidhead, cid, oid2);
5339 r = apply_transaction(store, &osr, std::move(t));
5344 ObjectStore::Transaction t;
5346 r = apply_transaction(store, &osr, std::move(t));
5351 vector<ghobject_t> objects;
5352 r = store->collection_list(cid, ghobject_t(), ghobject_t::get_max(),
5353 INT_MAX, &objects, 0);
5355 ASSERT_EQ(objects.size(), 1u);
5356 ASSERT_EQ(objects[0], oid2);
5359 ASSERT_FALSE(store->exists(cid, oid));
5362 ObjectStore::Transaction t;
5363 t.remove(cid, oid2);
5364 t.remove_collection(cid);
5365 r = apply_transaction(store, &osr, std::move(t));
5371 TEST_P(StoreTest, SetAllocHint) {
5372 ObjectStore::Sequencer osr("test");
5374 ghobject_t hoid(hobject_t("test_hint", "", CEPH_NOSNAP, 0, 0, ""));
5377 ObjectStore::Transaction t;
5378 t.create_collection(cid, 0);
5380 r = apply_transaction(store, &osr, std::move(t));
5384 ObjectStore::Transaction t;
5385 t.set_alloc_hint(cid, hoid, 4*1024*1024, 1024*4, 0);
5386 r = apply_transaction(store, &osr, std::move(t));
5390 ObjectStore::Transaction t;
5391 t.remove(cid, hoid);
5392 r = apply_transaction(store, &osr, std::move(t));
5396 ObjectStore::Transaction t;
5397 t.set_alloc_hint(cid, hoid, 4*1024*1024, 1024*4, 0);
5398 r = apply_transaction(store, &osr, std::move(t));
5402 ObjectStore::Transaction t;
5403 t.remove_collection(cid);
5404 r = apply_transaction(store, &osr, std::move(t));
5409 TEST_P(StoreTest, TryMoveRename) {
5410 ObjectStore::Sequencer osr("test");
5412 ghobject_t hoid(hobject_t("test_hint", "", CEPH_NOSNAP, 0, -1, ""));
5413 ghobject_t hoid2(hobject_t("test_hint2", "", CEPH_NOSNAP, 0, -1, ""));
5416 ObjectStore::Transaction t;
5417 t.create_collection(cid, 0);
5418 r = apply_transaction(store, &osr, std::move(t));
5422 ObjectStore::Transaction t;
5423 t.try_rename(cid, hoid, hoid2);
5424 r = apply_transaction(store, &osr, std::move(t));
5428 ObjectStore::Transaction t;
5430 r = apply_transaction(store, &osr, std::move(t));
5434 ObjectStore::Transaction t;
5435 t.try_rename(cid, hoid, hoid2);
5436 r = apply_transaction(store, &osr, std::move(t));
5440 ASSERT_EQ(store->stat(cid, hoid, &st), -ENOENT);
5441 ASSERT_EQ(store->stat(cid, hoid2, &st), 0);
5444 #if defined(HAVE_LIBAIO)
5445 TEST_P(StoreTest, BluestoreOnOffCSumTest) {
5446 if (string(GetParam()) != "bluestore")
5448 g_conf->set_val("bluestore_csum_type", "crc32c");
5449 g_conf->apply_changes(NULL);
5451 ObjectStore::Sequencer osr("test");
5454 ghobject_t hoid(hobject_t(sobject_t("Object 1", CEPH_NOSNAP)));
5457 r = store->read(cid, hoid, 0, 5, in);
5458 ASSERT_EQ(-ENOENT, r);
5461 ObjectStore::Transaction t;
5462 t.create_collection(cid, 0);
5463 cerr << "Creating collection " << cid << std::endl;
5464 r = apply_transaction(store, &osr, std::move(t));
5468 //write with csum enabled followed by read with csum disabled
5469 size_t block_size = 64*1024;
5470 ObjectStore::Transaction t;
5471 bufferlist bl, orig;
5472 bl.append(std::string(block_size, 'a'));
5474 t.remove(cid, hoid);
5475 t.set_alloc_hint(cid, hoid, 4*1024*1024, 1024*8, 0);
5476 t.write(cid, hoid, 0, bl.length(), bl);
5477 cerr << "Remove then create" << std::endl;
5478 r = apply_transaction(store, &osr, std::move(t));
5481 g_conf->set_val("bluestore_csum_type", "none");
5482 g_conf->apply_changes(NULL);
5485 r = store->read(cid, hoid, 0, block_size, in);
5486 ASSERT_EQ((int)block_size, r);
5487 ASSERT_TRUE(bl_eq(orig, in));
5491 //write with csum disabled followed by read with csum enabled
5493 size_t block_size = 64*1024;
5494 ObjectStore::Transaction t;
5495 bufferlist bl, orig;
5496 bl.append(std::string(block_size, 'a'));
5498 t.remove(cid, hoid);
5499 t.set_alloc_hint(cid, hoid, 4*1024*1024, 1024*8, 0);
5500 t.write(cid, hoid, 0, bl.length(), bl);
5501 cerr << "Remove then create" << std::endl;
5502 r = apply_transaction(store, &osr, std::move(t));
5505 g_conf->set_val("bluestore_csum_type", "crc32c");
5506 g_conf->apply_changes(NULL);
5509 r = store->read(cid, hoid, 0, block_size, in);
5510 ASSERT_EQ((int)block_size, r);
5511 ASSERT_TRUE(bl_eq(orig, in));
5514 //'mixed' non-overlapping writes to the same blob
5516 ObjectStore::Transaction t;
5517 bufferlist bl, orig;
5518 size_t block_size = 8000;
5519 bl.append(std::string(block_size, 'a'));
5521 t.remove(cid, hoid);
5522 t.write(cid, hoid, 0, bl.length(), bl);
5523 cerr << "Remove then create" << std::endl;
5524 r = apply_transaction(store, &osr, std::move(t));
5527 g_conf->set_val("bluestore_csum_type", "none");
5528 g_conf->apply_changes(NULL);
5530 ObjectStore::Transaction t2;
5531 t2.write(cid, hoid, block_size*2, bl.length(), bl);
5532 cerr << "Append 'unprotected'" << std::endl;
5533 r = apply_transaction(store, &osr, std::move(t2));
5537 r = store->read(cid, hoid, 0, block_size, in);
5538 ASSERT_EQ((int)block_size, r);
5539 ASSERT_TRUE(bl_eq(orig, in));
5541 r = store->read(cid, hoid, block_size*2, block_size, in);
5542 ASSERT_EQ((int)block_size, r);
5543 ASSERT_TRUE(bl_eq(orig, in));
5545 g_conf->set_val("bluestore_csum_type", "crc32c");
5546 g_conf->apply_changes(NULL);
5548 r = store->read(cid, hoid, 0, block_size, in);
5549 ASSERT_EQ((int)block_size, r);
5550 ASSERT_TRUE(bl_eq(orig, in));
5552 r = store->read(cid, hoid, block_size*2, block_size, in);
5553 ASSERT_EQ((int)block_size, r);
5554 ASSERT_TRUE(bl_eq(orig, in));
5557 //partially blob overwrite under a different csum enablement mode
5559 ObjectStore::Transaction t;
5560 bufferlist bl, orig, orig2;
5561 size_t block_size0 = 0x10000;
5562 size_t block_size = 9000;
5563 size_t block_size2 = 5000;
5564 bl.append(std::string(block_size0, 'a'));
5565 t.remove(cid, hoid);
5566 t.set_alloc_hint(cid, hoid, 4*1024*1024, 1024*8, 0);
5567 t.write(cid, hoid, 0, bl.length(), bl);
5568 cerr << "Remove then create" << std::endl;
5569 r = apply_transaction(store, &osr, std::move(t));
5572 g_conf->set_val("bluestore_csum_type", "none");
5573 g_conf->apply_changes(NULL);
5575 ObjectStore::Transaction t2;
5577 bl.append(std::string(block_size, 'b'));
5578 t2.write(cid, hoid, 0, bl.length(), bl);
5579 t2.write(cid, hoid, block_size0, bl.length(), bl);
5580 cerr << "Overwrite with unprotected data" << std::endl;
5581 r = apply_transaction(store, &osr, std::move(t2));
5586 orig.append( std::string(block_size0 - block_size, 'a'));
5589 r = store->read(cid, hoid, 0, block_size0, in);
5590 ASSERT_EQ((int)block_size0, r);
5591 ASSERT_TRUE(bl_eq(orig, in));
5593 r = store->read(cid, hoid, block_size0, block_size, in);
5594 ASSERT_EQ((int)block_size, r);
5595 ASSERT_TRUE(bl_eq(orig2, in));
5597 g_conf->set_val("bluestore_csum_type", "crc32c");
5598 g_conf->apply_changes(NULL);
5600 ObjectStore::Transaction t3;
5602 bl.append(std::string(block_size2, 'c'));
5603 t3.write(cid, hoid, block_size0, bl.length(), bl);
5604 cerr << "Overwrite with protected data" << std::endl;
5605 r = apply_transaction(store, &osr, std::move(t3));
5610 orig.append( std::string(block_size - block_size2, 'b'));
5611 r = store->read(cid, hoid, block_size0, block_size, in);
5612 ASSERT_EQ((int)block_size, r);
5613 ASSERT_TRUE(bl_eq(orig, in));
5617 ObjectStore::Transaction t;
5618 t.remove(cid, hoid);
5619 t.remove_collection(cid);
5620 cerr << "Cleaning" << std::endl;
5621 r = apply_transaction(store, &osr, std::move(t));
5627 INSTANTIATE_TEST_CASE_P(
5633 #if defined(HAVE_LIBAIO)
5638 // Note: instantiate all stores to preserve store numbering order only
5639 INSTANTIATE_TEST_CASE_P(
5641 StoreTestSpecificAUSize,
5645 #if defined(HAVE_LIBAIO)
5652 // Google Test may not support value-parameterized tests with some
5653 // compilers. If we use conditional compilation to compile out all
5654 // code referring to the gtest_main library, MSVC linker will not link
5655 // that library at all and consequently complain about missing entry
5656 // point defined in that library (fatal error LNK1561: entry point
5657 // must be defined). This dummy test keeps gtest_main linked in.
5658 TEST(DummyTest, ValueParameterizedTestsAreNotSupportedOnThisPlatform) {}
5662 void doMany4KWritesTest(boost::scoped_ptr<ObjectStore>& store,
5663 unsigned max_objects,
5665 unsigned max_object_size,
5666 unsigned max_write_size,
5667 unsigned write_alignment,
5668 store_statfs_t* res_stat)
5670 ObjectStore::Sequencer osr("test");
5671 MixedGenerator gen(555);
5672 gen_type rng(time(NULL));
5673 coll_t cid(spg_t(pg_t(0,555), shard_id_t::NO_SHARD));
5675 SyntheticWorkloadState test_obj(store.get(),
5684 for (unsigned i = 0; i < max_objects; ++i) {
5685 if (!(i % 500)) cerr << "seeding object " << i << std::endl;
5688 for (unsigned i = 0; i < max_ops; ++i) {
5690 cerr << "Op " << i << std::endl;
5691 test_obj.print_internal_state();
5695 test_obj.wait_for_done();
5697 test_obj.statfs(*res_stat);
5699 test_obj.shutdown();
5702 TEST_P(StoreTestSpecificAUSize, Many4KWritesTest) {
5703 if (string(GetParam()) != "bluestore")
5706 StartDeferred(0x10000);
5708 store_statfs_t res_stat;
5709 unsigned max_object = 4*1024*1024;
5711 doMany4KWritesTest(store, 1, 1000, 4*1024*1024, 4*1024, 0, &res_stat);
5713 ASSERT_LE(res_stat.stored, max_object);
5714 ASSERT_EQ(res_stat.allocated, max_object);
5717 TEST_P(StoreTestSpecificAUSize, Many4KWritesNoCSumTest) {
5718 if (string(GetParam()) != "bluestore")
5720 StartDeferred(0x10000);
5721 g_conf->set_val("bluestore_csum_type", "none");
5722 g_ceph_context->_conf->apply_changes(NULL);
5723 store_statfs_t res_stat;
5724 unsigned max_object = 4*1024*1024;
5726 doMany4KWritesTest(store, 1, 1000, max_object, 4*1024, 0, &res_stat );
5728 ASSERT_LE(res_stat.stored, max_object);
5729 ASSERT_EQ(res_stat.allocated, max_object);
5730 g_conf->set_val("bluestore_csum_type", "crc32c");
5733 TEST_P(StoreTestSpecificAUSize, TooManyBlobsTest) {
5734 if (string(GetParam()) != "bluestore")
5736 StartDeferred(0x10000);
5737 store_statfs_t res_stat;
5738 unsigned max_object = 4*1024*1024;
5739 doMany4KWritesTest(store, 1, 1000, max_object, 4*1024, 0, &res_stat);
5740 ASSERT_LE(res_stat.stored, max_object);
5741 ASSERT_EQ(res_stat.allocated, max_object);
5744 #if defined(HAVE_LIBAIO)
5745 void get_mempool_stats(uint64_t* total_bytes, uint64_t* total_items)
5747 uint64_t onode_allocated = mempool::bluestore_cache_onode::allocated_bytes();
5748 uint64_t other_allocated = mempool::bluestore_cache_other::allocated_bytes();
5750 uint64_t onode_items = mempool::bluestore_cache_onode::allocated_items();
5751 uint64_t other_items = mempool::bluestore_cache_other::allocated_items();
5752 cout << "onode(" << onode_allocated << "/" << onode_items
5753 << ") other(" << other_allocated << "/" << other_items
5754 << ")" << std::endl;
5755 *total_bytes = onode_allocated + other_allocated;
5756 *total_items = onode_items;
5759 TEST_P(StoreTestSpecificAUSize, OnodeSizeTracking) {
5761 if (string(GetParam()) != "bluestore")
5764 size_t block_size = 4096;
5765 StartDeferred(block_size);
5766 g_conf->set_val("bluestore_compression_mode", "none");
5767 g_conf->set_val("bluestore_csum_type", "none");
5768 g_conf->set_val("bluestore_cache_size_hdd", "400000000");
5769 g_conf->set_val("bluestore_cache_size_ssd", "400000000");
5770 g_conf->apply_changes(NULL);
5772 ObjectStore::Sequencer osr("test");
5775 ghobject_t hoid(hobject_t("test_hint", "", CEPH_NOSNAP, 0, -1, ""));
5776 size_t obj_size = 4 * 1024 * 1024;
5777 uint64_t total_bytes, total_bytes2;
5778 uint64_t total_onodes;
5779 get_mempool_stats(&total_bytes, &total_onodes);
5780 ASSERT_EQ(total_onodes, 0u);
5783 ObjectStore::Transaction t;
5784 t.create_collection(cid, 0);
5785 r = apply_transaction(store, &osr, std::move(t));
5789 ObjectStore::Transaction t;
5790 bufferlist bl, orig, orig2;
5792 bl.append(std::string(obj_size, 'a'));
5793 t.write(cid, hoid, 0, bl.length(), bl);
5794 r = apply_transaction(store, &osr, std::move(t));
5797 get_mempool_stats(&total_bytes, &total_onodes);
5798 ASSERT_NE(total_bytes, 0u);
5799 ASSERT_EQ(total_onodes, 1u);
5802 ObjectStore::Transaction t;
5803 t.truncate(cid, hoid, 0);
5804 r = apply_transaction(store, &osr, std::move(t));
5808 for(size_t i = 0; i < 1; ++i) {
5810 bl.append(std::string(block_size * (i+1), 'a'));
5811 for( size_t j = 0; j < obj_size; j+= bl.length()) {
5812 ObjectStore::Transaction t;
5813 t.write(cid, hoid, j, bl.length(), bl);
5814 r = apply_transaction(store, &osr, std::move(t));
5817 get_mempool_stats(&total_bytes2, &total_onodes);
5818 ASSERT_NE(total_bytes2, 0u);
5819 ASSERT_EQ(total_onodes, 1u);
5822 cout <<" mempool dump:\n";
5823 JSONFormatter f(true);
5824 f.open_object_section("transaction");
5832 for (size_t i = 0; i < obj_size; i += 0x1000) {
5833 store->read(cid, hoid, i, 0x1000, bl);
5836 get_mempool_stats(&total_bytes, &total_onodes);
5837 ASSERT_NE(total_bytes, 0u);
5838 ASSERT_EQ(total_onodes, 1u);
5841 cout <<" mempool dump:\n";
5842 JSONFormatter f(true);
5843 f.open_object_section("transaction");
5850 ObjectStore::Transaction t;
5851 t.remove(cid, hoid);
5852 t.remove_collection(cid);
5853 cerr << "Cleaning" << std::endl;
5854 r = apply_transaction(store, &osr, std::move(t));
5857 g_ceph_context->_conf->set_val("bluestore_cache_size_hdd", "4000000");
5858 g_ceph_context->_conf->set_val("bluestore_cache_size_ssd", "4000000");
5859 g_conf->set_val("bluestore_compression_mode", "none");
5860 g_conf->set_val("bluestore_csum_type", "crc32c");
5864 TEST_P(StoreTestSpecificAUSize, BlobReuseOnOverwrite) {
5866 if (string(GetParam()) != "bluestore")
5869 size_t block_size = 4096;
5870 StartDeferred(block_size);
5871 g_conf->set_val("bluestore_max_blob_size", "65536");
5872 g_conf->apply_changes(NULL);
5874 ObjectStore::Sequencer osr("test");
5877 ghobject_t hoid(hobject_t("test_hint", "", CEPH_NOSNAP, 0, -1, ""));
5879 const PerfCounters* logger = store->get_perf_counters();
5882 ObjectStore::Transaction t;
5883 t.create_collection(cid, 0);
5884 r = apply_transaction(store, &osr, std::move(t));
5888 ObjectStore::Transaction t;
5891 bl.append(std::string(block_size * 2, 'a'));
5892 t.write(cid, hoid, 0, bl.length(), bl, CEPH_OSD_OP_FLAG_FADVISE_WILLNEED);
5893 r = apply_transaction(store, &osr, std::move(t));
5897 // overwrite at the beginning
5898 ObjectStore::Transaction t;
5901 bl.append(std::string(block_size, 'b'));
5902 t.write(cid, hoid, 0, bl.length(), bl, CEPH_OSD_OP_FLAG_FADVISE_WILLNEED);
5903 r = apply_transaction(store, &osr, std::move(t));
5908 ObjectStore::Transaction t;
5911 bl.append(std::string(block_size * 2, 'c'));
5912 t.write(cid, hoid, block_size * 2, bl.length(), bl, CEPH_OSD_OP_FLAG_FADVISE_WILLNEED);
5913 r = apply_transaction(store, &osr, std::move(t));
5917 // append with a gap
5918 ObjectStore::Transaction t;
5921 bl.append(std::string(block_size * 2, 'd'));
5922 t.write(cid, hoid, block_size * 5, bl.length(), bl, CEPH_OSD_OP_FLAG_FADVISE_WILLNEED);
5923 r = apply_transaction(store, &osr, std::move(t));
5927 // We need to issue a read to trigger cache stat update that refresh
5928 // perf counters. additionally we need to wait some time for mempool
5929 // thread to update stats.
5931 bufferlist bl, expected;
5932 r = store->read(cid, hoid, 0, block_size, bl);
5933 ASSERT_EQ(r, (int)block_size);
5934 expected.append(string(block_size, 'b'));
5935 ASSERT_TRUE(bl_eq(expected, bl));
5936 ASSERT_EQ(logger->get(l_bluestore_blobs), 1u);
5937 ASSERT_EQ(logger->get(l_bluestore_extents), 2u);
5941 ObjectStore::Transaction t;
5944 bl.append(std::string(block_size * 2, 'e'));
5946 // Currently we are unable to reuse blob when overwriting in a single step
5947 t.write(cid, hoid, block_size * 6, bl.length(), bl, CEPH_OSD_OP_FLAG_FADVISE_WILLNEED);
5948 r = apply_transaction(store, &osr, std::move(t));
5952 // We need to issue a read to trigger cache stat update that refresh
5953 // perf counters. additionally we need to wait some time for mempool
5954 // thread to update stats.
5956 bufferlist bl, expected;
5957 r = store->read(cid, hoid, 0, block_size, bl);
5958 ASSERT_EQ(r, (int)block_size);
5959 expected.append(string(block_size, 'b'));
5960 ASSERT_TRUE(bl_eq(expected, bl));
5961 ASSERT_EQ(logger->get(l_bluestore_blobs), 1u);
5962 ASSERT_EQ(logger->get(l_bluestore_extents), 2u);
5966 ObjectStore::Transaction t;
5969 bl.append(std::string(block_size, 'f'));
5971 t.write(cid, hoid, block_size * 4, bl.length(), bl, CEPH_OSD_OP_FLAG_FADVISE_WILLNEED);
5972 r = apply_transaction(store, &osr, std::move(t));
5976 // we need to wait some time for mempool
5977 // thread to update stats to be able to check blob/extent numbers from
5981 bufferlist bl, expected;
5982 r = store->read(cid, hoid, 0, block_size, bl);
5983 ASSERT_EQ(r, (int)block_size);
5984 expected.append(string(block_size, 'b'));
5985 ASSERT_TRUE(bl_eq(expected, bl));
5989 r = store->read(cid, hoid, block_size, block_size, bl);
5990 ASSERT_EQ(r, (int)block_size);
5991 expected.append(string(block_size, 'a'));
5992 ASSERT_TRUE(bl_eq(expected, bl));
5996 r = store->read(cid, hoid, block_size * 2, block_size * 2, bl);
5997 ASSERT_EQ(r, (int)block_size * 2);
5998 expected.append(string(block_size * 2, 'c'));
5999 ASSERT_TRUE(bl_eq(expected, bl));
6003 r = store->read(cid, hoid, block_size * 4, block_size, bl);
6004 ASSERT_EQ(r, (int)block_size);
6005 expected.append(string(block_size, 'f'));
6006 ASSERT_TRUE(bl_eq(expected, bl));
6010 r = store->read(cid, hoid, block_size * 5, block_size, bl);
6011 ASSERT_EQ(r, (int)block_size);
6012 expected.append(string(block_size, 'd'));
6013 ASSERT_TRUE(bl_eq(expected, bl));
6017 r = store->read(cid, hoid, block_size * 5, block_size * 3, bl);
6018 ASSERT_EQ(r, (int)block_size * 3);
6019 expected.append(string(block_size, 'd'));
6020 expected.append(string(block_size * 2, 'e'));
6021 ASSERT_TRUE(bl_eq(expected, bl));
6023 ASSERT_EQ(logger->get(l_bluestore_blobs), 1u);
6024 ASSERT_EQ(logger->get(l_bluestore_extents), 1u);
6028 ObjectStore::Transaction t;
6029 t.remove(cid, hoid);
6030 t.remove_collection(cid);
6031 cerr << "Cleaning" << std::endl;
6032 r = apply_transaction(store, &osr, std::move(t));
6035 g_conf->set_val("bluestore_max_blob_size", "0");
6039 TEST_P(StoreTestSpecificAUSize, BlobReuseOnOverwriteReverse) {
6041 if (string(GetParam()) != "bluestore")
6044 size_t block_size = 4096;
6045 StartDeferred(block_size);
6046 g_conf->set_val("bluestore_max_blob_size", "65536");
6048 g_conf->apply_changes(NULL);
6050 ObjectStore::Sequencer osr("test");
6053 ghobject_t hoid(hobject_t("test_hint", "", CEPH_NOSNAP, 0, -1, ""));
6055 const PerfCounters* logger = store->get_perf_counters();
6057 ObjectStore::Transaction t;
6058 t.create_collection(cid, 0);
6059 r = apply_transaction(store, &osr, std::move(t));
6063 ObjectStore::Transaction t;
6066 bl.append(std::string(block_size * 2, 'a'));
6067 t.write(cid, hoid, block_size * 10, bl.length(), bl,
6068 CEPH_OSD_OP_FLAG_FADVISE_WILLNEED);
6069 r = apply_transaction(store, &osr, std::move(t));
6074 ObjectStore::Transaction t;
6077 bl.append(std::string(block_size, 'b'));
6078 t.write(cid, hoid, block_size * 9, bl.length(), bl,
6079 CEPH_OSD_OP_FLAG_FADVISE_WILLNEED);
6080 r = apply_transaction(store, &osr, std::move(t));
6084 // We need to issue a read to trigger cache stat update that refresh
6085 // perf counters. additionally we need to wait some time for mempool
6086 // thread to update stats.
6088 bufferlist bl, expected;
6089 r = store->read(cid, hoid, block_size * 9, block_size * 2, bl);
6090 ASSERT_EQ(r, (int)block_size * 2);
6091 expected.append(string(block_size, 'b'));
6092 expected.append(string(block_size, 'a'));
6093 ASSERT_TRUE(bl_eq(expected, bl));
6094 ASSERT_EQ(logger->get(l_bluestore_blobs), 1u);
6095 ASSERT_EQ(logger->get(l_bluestore_extents), 1u);
6100 // prepend existing with a gap
6101 ObjectStore::Transaction t;
6104 bl.append(std::string(block_size, 'c'));
6105 t.write(cid, hoid, block_size * 7, bl.length(), bl,
6106 CEPH_OSD_OP_FLAG_FADVISE_WILLNEED);
6107 r = apply_transaction(store, &osr, std::move(t));
6111 // We need to issue a read to trigger cache stat update that refresh
6112 // perf counters. additionally we need to wait some time for mempool
6113 // thread to update stats.
6115 bufferlist bl, expected;
6116 r = store->read(cid, hoid, block_size * 7, block_size * 3, bl);
6117 ASSERT_EQ(r, (int)block_size * 3);
6118 expected.append(string(block_size, 'c'));
6119 expected.append(string(block_size, 0));
6120 expected.append(string(block_size, 'b'));
6121 ASSERT_TRUE(bl_eq(expected, bl));
6122 ASSERT_EQ(logger->get(l_bluestore_blobs), 1u);
6123 ASSERT_EQ(logger->get(l_bluestore_extents), 2u);
6127 // append after existing with a gap
6128 ObjectStore::Transaction t;
6131 bl.append(std::string(block_size, 'd'));
6132 t.write(cid, hoid, block_size * 13, bl.length(), bl,
6133 CEPH_OSD_OP_FLAG_FADVISE_WILLNEED);
6134 r = apply_transaction(store, &osr, std::move(t));
6138 // We need to issue a read to trigger cache stat update that refresh
6139 // perf counters. additionally we need to wait some time for mempool
6140 // thread to update stats.
6142 bufferlist bl, expected;
6143 r = store->read(cid, hoid, block_size * 11, block_size * 3, bl);
6144 ASSERT_EQ(r, (int)block_size * 3);
6145 expected.append(string(block_size, 'a'));
6146 expected.append(string(block_size, 0));
6147 expected.append(string(block_size, 'd'));
6148 ASSERT_TRUE(bl_eq(expected, bl));
6149 ASSERT_EQ(logger->get(l_bluestore_blobs), 1u);
6150 ASSERT_EQ(logger->get(l_bluestore_extents), 3u);
6154 // append twice to the next max_blob slot
6155 ObjectStore::Transaction t;
6158 bl.append(std::string(block_size, 'e'));
6159 t.write(cid, hoid, block_size * 17, bl.length(), bl,
6160 CEPH_OSD_OP_FLAG_FADVISE_WILLNEED);
6161 t.write(cid, hoid, block_size * 19, bl.length(), bl,
6162 CEPH_OSD_OP_FLAG_FADVISE_WILLNEED);
6163 r = apply_transaction(store, &osr, std::move(t));
6167 // We need to issue a read to trigger cache stat update that refresh
6168 // perf counters. additionally we need to wait some time for mempool
6169 // thread to update stats.
6171 bufferlist bl, expected;
6172 r = store->read(cid, hoid, block_size * 17, block_size * 3, bl);
6173 ASSERT_EQ(r, (int)block_size * 3);
6174 expected.append(string(block_size, 'e'));
6175 expected.append(string(block_size, 0));
6176 expected.append(string(block_size, 'e'));
6177 ASSERT_TRUE(bl_eq(expected, bl));
6178 ASSERT_EQ(logger->get(l_bluestore_blobs), 2u);
6179 ASSERT_EQ(logger->get(l_bluestore_extents), 5u);
6182 // fill gaps at the second slot
6183 ObjectStore::Transaction t;
6186 bl.append(std::string(block_size, 'f'));
6187 t.write(cid, hoid, block_size * 16, bl.length(), bl,
6188 CEPH_OSD_OP_FLAG_FADVISE_WILLNEED);
6189 t.write(cid, hoid, block_size * 18, bl.length(), bl,
6190 CEPH_OSD_OP_FLAG_FADVISE_WILLNEED);
6191 r = apply_transaction(store, &osr, std::move(t));
6195 // We need to issue a read to trigger cache stat update that refresh
6196 // perf counters. additionally we need to wait some time for mempool
6197 // thread to update stats.
6199 bufferlist bl, expected;
6200 r = store->read(cid, hoid, block_size * 16, block_size * 4, bl);
6201 ASSERT_EQ(r, (int)block_size * 4);
6202 expected.append(string(block_size, 'f'));
6203 expected.append(string(block_size, 'e'));
6204 expected.append(string(block_size, 'f'));
6205 expected.append(string(block_size, 'e'));
6206 ASSERT_TRUE(bl_eq(expected, bl));
6207 ASSERT_EQ(logger->get(l_bluestore_blobs), 2u);
6208 ASSERT_EQ(logger->get(l_bluestore_extents), 4u);
6211 ObjectStore::Transaction t;
6212 t.remove(cid, hoid);
6213 t.remove_collection(cid);
6214 cerr << "Cleaning" << std::endl;
6215 r = apply_transaction(store, &osr, std::move(t));
6218 g_conf->set_val("bluestore_max_blob_size", "0");
6221 TEST_P(StoreTestSpecificAUSize, BlobReuseOnSmallOverwrite) {
6223 if (string(GetParam()) != "bluestore")
6226 size_t block_size = 4096;
6227 StartDeferred(block_size);
6228 g_conf->set_val("bluestore_max_blob_size", "65536");
6229 g_conf->apply_changes(NULL);
6231 ObjectStore::Sequencer osr("test");
6234 ghobject_t hoid(hobject_t("test_hint", "", CEPH_NOSNAP, 0, -1, ""));
6236 const PerfCounters* logger = store->get_perf_counters();
6239 ObjectStore::Transaction t;
6240 t.create_collection(cid, 0);
6241 r = apply_transaction(store, &osr, std::move(t));
6245 ObjectStore::Transaction t;
6248 bl.append(std::string(block_size, 'a'));
6249 t.write(cid, hoid, 0, bl.length(), bl, CEPH_OSD_OP_FLAG_FADVISE_WILLNEED);
6250 t.write(cid, hoid, block_size * 2, bl.length(), bl,
6251 CEPH_OSD_OP_FLAG_FADVISE_WILLNEED);
6252 r = apply_transaction(store, &osr, std::move(t));
6256 // write small into the gap
6257 ObjectStore::Transaction t;
6260 bl.append(std::string(3, 'b'));
6261 t.write(cid, hoid, block_size + 1, bl.length(), bl,
6262 CEPH_OSD_OP_FLAG_FADVISE_WILLNEED);
6263 r = apply_transaction(store, &osr, std::move(t));
6267 // We need to issue a read to trigger cache stat update that refresh
6268 // perf counters. additionally we need to wait some time for mempool
6269 // thread to update stats.
6271 bufferlist bl, expected;
6272 r = store->read(cid, hoid, 0, block_size * 3, bl);
6273 ASSERT_EQ(r, (int)block_size * 3);
6274 expected.append(string(block_size, 'a'));
6275 expected.append(string(1, 0));
6276 expected.append(string(3, 'b'));
6277 expected.append(string(block_size - 4, 0));
6278 expected.append(string(block_size, 'a'));
6279 ASSERT_TRUE(bl_eq(expected, bl));
6281 ASSERT_EQ(logger->get(l_bluestore_blobs), 1u);
6282 ASSERT_EQ(logger->get(l_bluestore_extents), 3u);
6285 ObjectStore::Transaction t;
6286 t.remove(cid, hoid);
6287 t.remove_collection(cid);
6288 cerr << "Cleaning" << std::endl;
6289 r = apply_transaction(store, &osr, std::move(t));
6292 g_conf->set_val("bluestore_max_blob_size", "0");
6295 // The test case to reproduce an issue when write happens
6296 // to a zero space between the extents sharing the same spanning blob
6297 // with unloaded shard map.
6298 // Second extent might be filled with zeros this way due to wrong result
6299 // returned by has_any_extents() call in do_write_small. The latter is caused
6300 // by incompletly loaded extent map.
6301 TEST_P(StoreTestSpecificAUSize, SmallWriteOnShardedExtents) {
6302 if (string(GetParam()) != "bluestore")
6305 size_t block_size = 0x10000;
6306 StartDeferred(block_size);
6308 g_conf->set_val("bluestore_csum_type", "xxhash64");
6309 g_conf->set_val("bluestore_max_target_blob", "524288"); // for sure
6311 g_conf->apply_changes(NULL);
6313 ObjectStore::Sequencer osr("test");
6316 ghobject_t hoid1(hobject_t(sobject_t("Object 1", CEPH_NOSNAP)));
6319 ObjectStore::Transaction t;
6320 t.create_collection(cid, 0);
6321 r = apply_transaction(store, &osr, std::move(t));
6325 //doing some tricks to have sharded extents/spanning objects
6326 ObjectStore::Transaction t;
6329 bl.append(std::string(0x80000, 'a'));
6330 t.write(cid, hoid1, 0, bl.length(), bl, 0);
6331 t.zero(cid, hoid1, 0x719e0, 0x75b0 );
6332 r = apply_transaction(store, &osr, std::move(t));
6335 bl2.append(std::string(0x70000, 'b'));
6336 t.write(cid, hoid1, 0, bl2.length(), bl2, 0);
6337 t.zero(cid, hoid1, 0, 0x50000);
6338 r = apply_transaction(store, &osr, std::move(t));
6346 // do a write to zero space in between some extents sharing the same blob
6347 ObjectStore::Transaction t;
6350 bl.append(std::string(0x6520, 'c'));
6351 t.write(cid, hoid1, 0x71c00, bl.length(), bl, 0);
6353 r = apply_transaction(store, &osr, std::move(t));
6358 ObjectStore::Transaction t;
6359 bufferlist bl, expected;
6361 r = store->read(cid, hoid1, 0x70000, 0x9c00, bl);
6362 ASSERT_EQ(r, (int)0x9c00);
6363 expected.append(string(0x19e0, 'a'));
6364 expected.append(string(0x220, 0));
6365 expected.append(string(0x6520, 'c'));
6366 expected.append(string(0xe70, 0));
6367 expected.append(string(0xc70, 'a'));
6368 ASSERT_TRUE(bl_eq(expected, bl));
6374 ObjectStore::Transaction t;
6375 t.remove(cid, hoid1);
6376 t.remove_collection(cid);
6377 cerr << "Cleaning" << std::endl;
6378 r = apply_transaction(store, &osr, std::move(t));
6381 g_conf->set_val("bluestore_max_target_blob", "524288");
6382 g_conf->set_val("bluestore_csum_type", "crc32c");
6385 #endif //#if defined(HAVE_LIBAIO)
6387 TEST_P(StoreTest, KVDBHistogramTest) {
6388 if (string(GetParam()) != "bluestore")
6391 ObjectStore::Sequencer osr("test");
6395 string base("testobj.");
6397 bufferptr ap(0x1000);
6398 memset(ap.c_str(), 'a', 0x1000);
6401 ObjectStore::Transaction t;
6402 t.create_collection(cid, 0);
6403 r = apply_transaction(store, &osr, std::move(t));
6406 for (int i = 0; i < NUM_OBJS; ++i) {
6407 ObjectStore::Transaction t;
6409 snprintf(buf, sizeof(buf), "%d", i);
6410 ghobject_t hoid(hobject_t(sobject_t(base + string(buf), CEPH_NOSNAP)));
6411 t.write(cid, hoid, 0, 0x1000, a);
6412 r = apply_transaction(store, &osr, std::move(t));
6416 Formatter *f = Formatter::create("store_test", "json-pretty", "json-pretty");
6417 store->generate_db_histogram(f);
6422 TEST_P(StoreTest, KVDBStatsTest) {
6423 if (string(GetParam()) != "bluestore")
6426 g_conf->set_val("rocksdb_perf", "true");
6427 g_conf->set_val("rocksdb_collect_compaction_stats", "true");
6428 g_conf->set_val("rocksdb_collect_extended_stats","true");
6429 g_conf->set_val("rocksdb_collect_memory_stats","true");
6430 g_ceph_context->_conf->apply_changes(NULL);
6431 int r = store->umount();
6433 r = store->mount(); //to force rocksdb stats
6436 ObjectStore::Sequencer osr("test");
6439 string base("testobj.");
6441 bufferptr ap(0x1000);
6442 memset(ap.c_str(), 'a', 0x1000);
6445 ObjectStore::Transaction t;
6446 t.create_collection(cid, 0);
6447 r = apply_transaction(store, &osr, std::move(t));
6450 for (int i = 0; i < NUM_OBJS; ++i) {
6451 ObjectStore::Transaction t;
6453 snprintf(buf, sizeof(buf), "%d", i);
6454 ghobject_t hoid(hobject_t(sobject_t(base + string(buf), CEPH_NOSNAP)));
6455 t.write(cid, hoid, 0, 0x1000, a);
6456 r = apply_transaction(store, &osr, std::move(t));
6460 Formatter *f = Formatter::create("store_test", "json-pretty", "json-pretty");
6461 store->get_db_statistics(f);
6464 g_conf->set_val("rocksdb_perf", "false");
6465 g_conf->set_val("rocksdb_collect_compaction_stats", "false");
6466 g_conf->set_val("rocksdb_collect_extended_stats","false");
6467 g_conf->set_val("rocksdb_collect_memory_stats","false");
6470 #if defined(HAVE_LIBAIO)
6471 TEST_P(StoreTestSpecificAUSize, garbageCollection) {
6472 ObjectStore::Sequencer osr("test");
6475 int buf_len = 256 * 1024;
6476 int overlap_offset = 64 * 1024;
6477 int write_offset = buf_len;
6478 if (string(GetParam()) != "bluestore")
6481 #define WRITE_AT(offset, _length) {\
6482 ObjectStore::Transaction t;\
6483 if ((uint64_t)_length != bl.length()) { \
6484 buffer::ptr p(bl.c_str(), _length);\
6486 bl_tmp.push_back(p);\
6487 t.write(cid, hoid, offset, bl_tmp.length(), bl_tmp);\
6489 t.write(cid, hoid, offset, bl.length(), bl);\
6491 r = apply_transaction(store, &osr, std::move(t));\
6495 StartDeferred(65536);
6497 g_conf->set_val("bluestore_compression_max_blob_size", "524288");
6498 g_conf->set_val("bluestore_compression_min_blob_size", "262144");
6499 g_conf->set_val("bluestore_compression_mode", "force");
6500 g_conf->apply_changes(NULL);
6502 ghobject_t hoid(hobject_t(sobject_t("Object 1", CEPH_NOSNAP)));
6505 r = store->read(cid, hoid, 0, 5, in);
6506 ASSERT_EQ(-ENOENT, r);
6509 ObjectStore::Transaction t;
6510 t.create_collection(cid, 0);
6511 cerr << "Creating collection " << cid << std::endl;
6512 r = apply_transaction(store, &osr, std::move(t));
6517 data.resize(buf_len);
6521 bool exists = store->exists(cid, hoid);
6522 ASSERT_TRUE(!exists);
6524 ObjectStore::Transaction t;
6526 cerr << "Creating object " << hoid << std::endl;
6527 r = apply_transaction(store, &osr, std::move(t));
6530 exists = store->exists(cid, hoid);
6531 ASSERT_EQ(true, exists);
6535 for(size_t i = 0; i < data.size(); i++)
6541 struct store_statfs_t statfs;
6542 WRITE_AT(0, buf_len);
6543 int r = store->statfs(&statfs);
6545 ASSERT_EQ(statfs.compressed_allocated, 0x10000);
6548 struct store_statfs_t statfs;
6549 WRITE_AT(write_offset - 2 * overlap_offset, buf_len);
6550 int r = store->statfs(&statfs);
6552 ASSERT_EQ(statfs.compressed_allocated, 0x20000);
6553 const PerfCounters* counters = store->get_perf_counters();
6554 ASSERT_EQ(counters->get(l_bluestore_gc_merged), 0u);
6558 struct store_statfs_t statfs;
6559 WRITE_AT(write_offset - overlap_offset, buf_len);
6560 int r = store->statfs(&statfs);
6562 ASSERT_EQ(statfs.compressed_allocated, 0x20000);
6563 const PerfCounters* counters = store->get_perf_counters();
6564 ASSERT_EQ(counters->get(l_bluestore_gc_merged), 0x10000u);
6567 struct store_statfs_t statfs;
6568 WRITE_AT(write_offset - 3 * overlap_offset, buf_len);
6569 int r = store->statfs(&statfs);
6571 ASSERT_EQ(statfs.compressed_allocated, 0x20000);
6572 const PerfCounters* counters = store->get_perf_counters();
6573 ASSERT_EQ(counters->get(l_bluestore_gc_merged), 0x20000u);
6576 struct store_statfs_t statfs;
6577 WRITE_AT(write_offset + 1, overlap_offset-1);
6578 int r = store->statfs(&statfs);
6580 ASSERT_EQ(statfs.compressed_allocated, 0x20000);
6581 const PerfCounters* counters = store->get_perf_counters();
6582 ASSERT_EQ(counters->get(l_bluestore_gc_merged), 0x20000u);
6585 struct store_statfs_t statfs;
6586 WRITE_AT(write_offset + 1, overlap_offset);
6587 int r = store->statfs(&statfs);
6589 ASSERT_EQ(statfs.compressed_allocated, 0x10000);
6590 const PerfCounters* counters = store->get_perf_counters();
6591 ASSERT_EQ(counters->get(l_bluestore_gc_merged), 0x3ffffu);
6594 struct store_statfs_t statfs;
6595 WRITE_AT(0, buf_len-1);
6596 int r = store->statfs(&statfs);
6598 ASSERT_EQ(statfs.compressed_allocated, 0x10000);
6599 const PerfCounters* counters = store->get_perf_counters();
6600 ASSERT_EQ(counters->get(l_bluestore_gc_merged), 0x40001u);
6602 g_conf->set_val("bluestore_gc_enable_total_threshold", "1"); //forbid GC when saving = 0
6604 struct store_statfs_t statfs;
6605 WRITE_AT(1, overlap_offset-2);
6606 WRITE_AT(overlap_offset * 2 + 1, overlap_offset-2);
6607 int r = store->statfs(&statfs);
6609 ASSERT_EQ(statfs.compressed_allocated, 0x10000);
6610 const PerfCounters* counters = store->get_perf_counters();
6611 ASSERT_EQ(counters->get(l_bluestore_gc_merged), 0x40001u);
6614 struct store_statfs_t statfs;
6615 WRITE_AT(overlap_offset + 1, overlap_offset-2);
6616 int r = store->statfs(&statfs);
6618 ASSERT_EQ(statfs.compressed_allocated, 0x0);
6619 const PerfCounters* counters = store->get_perf_counters();
6620 ASSERT_EQ(counters->get(l_bluestore_gc_merged), 0x40007u);
6623 ObjectStore::Transaction t;
6624 t.remove(cid, hoid);
6625 cerr << "Cleaning" << std::endl;
6626 r = apply_transaction(store, &osr, std::move(t));
6630 g_conf->set_val("bluestore_gc_enable_total_threshold", "0");
6631 g_conf->set_val("bluestore_compression_min_blob_size", "0");
6632 g_conf->set_val("bluestore_compression_max_blob_size", "0");
6633 g_conf->set_val("bluestore_compression_mode", "none");
6634 g_conf->apply_changes(NULL);
6638 int main(int argc, char **argv) {
6639 vector<const char*> args;
6640 argv_to_vec(argc, (const char **)argv, args);
6643 auto cct = global_init(NULL, args, CEPH_ENTITY_TYPE_CLIENT,
6644 CODE_ENVIRONMENT_UTILITY, 0);
6645 common_init_finish(g_ceph_context);
6647 g_ceph_context->_conf->set_val("osd_journal_size", "400");
6648 g_ceph_context->_conf->set_val("filestore_index_retry_probability", "0.5");
6649 g_ceph_context->_conf->set_val("filestore_op_thread_timeout", "1000");
6650 g_ceph_context->_conf->set_val("filestore_op_thread_suicide_timeout", "10000");
6651 //g_ceph_context->_conf->set_val("filestore_fiemap", "true");
6652 g_ceph_context->_conf->set_val("bluestore_fsck_on_mount", "true");
6653 g_ceph_context->_conf->set_val("bluestore_fsck_on_umount", "true");
6654 g_ceph_context->_conf->set_val("bluestore_debug_misc", "true");
6655 g_ceph_context->_conf->set_val("bluestore_debug_small_allocations", "4");
6656 g_ceph_context->_conf->set_val("bluestore_debug_freelist", "true");
6657 g_ceph_context->_conf->set_val("bluestore_clone_cow", "true");
6658 g_ceph_context->_conf->set_val("bluestore_max_alloc_size", "196608");
6660 // set small cache sizes so we see trimming during Synthetic tests
6661 g_ceph_context->_conf->set_val("bluestore_cache_size_hdd", "4000000");
6662 g_ceph_context->_conf->set_val("bluestore_cache_size_ssd", "4000000");
6664 // very short *_max prealloc so that we fall back to async submits
6665 g_ceph_context->_conf->set_val("bluestore_blobid_prealloc", "10");
6666 g_ceph_context->_conf->set_val("bluestore_nid_prealloc", "10");
6667 g_ceph_context->_conf->set_val("bluestore_debug_randomize_serial_transaction",
6670 g_ceph_context->_conf->set_val("bdev_debug_aio", "true");
6672 // specify device size
6673 g_ceph_context->_conf->set_val("bluestore_block_size", "10240000000");
6675 g_ceph_context->_conf->set_val(
6676 "enable_experimental_unrecoverable_data_corrupting_features", "*");
6677 g_ceph_context->_conf->apply_changes(NULL);
6679 ::testing::InitGoogleTest(&argc, argv);
6680 return RUN_ALL_TESTS();
6685 * compile-command: "cd ../.. ; make ceph_test_objectstore &&
6686 * ./ceph_test_objectstore \
6687 * --gtest_filter=*.collect_metadata* --log-to-stderr=true --debug-filestore=20