1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
2 // vim: ts=8 sw=2 smarttab
4 * Ceph - scalable distributed file system
6 * Copyright (C) 2013 eNovance SAS <licensing@enovance.com>
8 * This is free software; you can redistribute it and/or
9 * modify it under the terms of the GNU Lesser General Public
10 * License version 2.1, as published by the Free Software
11 * Foundation. See file COPYING.
15 #include "global/global_init.h"
16 #include "common/ceph_argparse.h"
17 #include "rgw/rgw_common.h"
18 #include "rgw/rgw_rados.h"
19 #include "test_rgw_common.h"
22 #include <gtest/gtest.h>
24 #define TEST(x, y) void y()
25 #define ASSERT_EQ(v, s) if(v != s)cout << "Error at " << __LINE__ << "(" << #v << "!= " << #s << "\n"; \
26 else cout << "(" << #v << "==" << #s << ") PASSED\n";
27 #define EXPECT_EQ(v, s) ASSERT_EQ(v, s)
28 #define ASSERT_TRUE(c) if(c)cout << "Error at " << __LINE__ << "(" << #c << ")" << "\n"; \
29 else cout << "(" << #c << ") PASSED\n";#define EXPECT_TRUE(c) ASSERT_TRUE(c)
30 #define EXPECT_TRUE(c) ASSERT_TRUE(c)
34 struct OldObjManifestPart {
35 old_rgw_obj loc; /* the object where the data is located */
36 uint64_t loc_ofs; /* the offset at that object where the data is located */
37 uint64_t size; /* the part size */
39 OldObjManifestPart() : loc_ofs(0), size(0) {}
41 void encode(bufferlist& bl) const {
42 ENCODE_START(2, 2, bl);
44 ::encode(loc_ofs, bl);
49 void decode(bufferlist::iterator& bl) {
50 DECODE_START_LEGACY_COMPAT_LEN_32(2, 2, 2, bl);
52 ::decode(loc_ofs, bl);
57 void dump(Formatter *f) const;
58 static void generate_test_instances(list<OldObjManifestPart*>& o);
60 WRITE_CLASS_ENCODER(OldObjManifestPart)
62 class OldObjManifest {
64 map<uint64_t, OldObjManifestPart> objs;
69 OldObjManifest() : obj_size(0) {}
70 OldObjManifest(const OldObjManifest& rhs) {
73 OldObjManifest& operator=(const OldObjManifest& rhs) {
75 obj_size = rhs.obj_size;
79 const map<uint64_t, OldObjManifestPart>& get_objs() {
83 void append(uint64_t ofs, const OldObjManifestPart& part) {
85 obj_size = max(obj_size, ofs + part.size);
88 void encode(bufferlist& bl) const {
89 ENCODE_START(2, 2, bl);
90 ::encode(obj_size, bl);
95 void decode(bufferlist::iterator& bl) {
96 DECODE_START_LEGACY_COMPAT_LEN_32(6, 2, 2, bl);
97 ::decode(obj_size, bl);
106 WRITE_CLASS_ENCODER(OldObjManifest)
108 void append_head(list<rgw_obj> *objs, rgw_obj& head)
110 objs->push_back(head);
113 void append_stripes(list<rgw_obj> *objs, RGWObjManifest& manifest, uint64_t obj_size, uint64_t stripe_size)
115 string prefix = manifest.get_prefix();
116 rgw_bucket bucket = manifest.get_obj().bucket;
119 for (uint64_t ofs = manifest.get_max_head_size(); ofs < obj_size; ofs += stripe_size) {
121 snprintf(buf, sizeof(buf), "%d", ++i);
122 string oid = prefix + buf;
123 cout << "oid=" << oid << std::endl;
125 obj.init_ns(bucket, oid, "shadow");
126 objs->push_back(obj);
130 static void gen_obj(test_rgw_env& env, uint64_t obj_size, uint64_t head_max_size, uint64_t stripe_size,
131 RGWObjManifest *manifest, const string& placement_id, rgw_bucket *bucket, rgw_obj *head, RGWObjManifest::generator *gen,
132 list<rgw_obj> *test_objs)
134 manifest->set_trivial_rule(head_max_size, stripe_size);
136 test_rgw_init_bucket(bucket, "buck");
138 *head = rgw_obj(*bucket, "oid");
139 gen->create_begin(g_ceph_context, manifest, placement_id, *bucket, *head);
141 append_head(test_objs, *head);
142 cout << "test_objs.size()=" << test_objs->size() << std::endl;
143 append_stripes(test_objs, *manifest, obj_size, stripe_size);
145 cout << "test_objs.size()=" << test_objs->size() << std::endl;
147 ASSERT_EQ((int)manifest->get_obj_size(), 0);
148 ASSERT_EQ((int)manifest->get_head_size(), 0);
149 ASSERT_EQ(manifest->has_tail(), false);
152 list<rgw_obj>::iterator iter = test_objs->begin();
154 while (ofs < obj_size) {
155 rgw_raw_obj obj = gen->get_cur_obj(env.zonegroup, env.zone_params);
156 cout << "obj=" << obj << std::endl;
157 rgw_raw_obj test_raw = rgw_obj_select(*iter).get_raw_obj(env.zonegroup, env.zone_params);
158 ASSERT_TRUE(obj == test_raw);
160 ofs = MIN(ofs + gen->cur_stripe_max_size(), obj_size);
161 gen->create_next(ofs);
163 cout << "obj=" << obj << " *iter=" << *iter << std::endl;
164 cout << "test_objs.size()=" << test_objs->size() << std::endl;
169 if (manifest->has_tail()) {
170 rgw_raw_obj obj = gen->get_cur_obj(env.zonegroup, env.zone_params);
171 rgw_raw_obj test_raw = rgw_obj_select(*iter).get_raw_obj(env.zonegroup, env.zone_params);
172 ASSERT_TRUE(obj == test_raw);
175 ASSERT_TRUE(iter == test_objs->end());
176 ASSERT_EQ(manifest->get_obj_size(), obj_size);
177 ASSERT_EQ(manifest->get_head_size(), MIN(obj_size, head_max_size));
178 ASSERT_EQ(manifest->has_tail(), (obj_size > head_max_size));
181 static void gen_old_obj(test_rgw_env& env, uint64_t obj_size, uint64_t head_max_size, uint64_t stripe_size,
182 OldObjManifest *manifest, old_rgw_bucket *bucket, old_rgw_obj *head,
183 list<old_rgw_obj> *test_objs)
185 test_rgw_init_old_bucket(bucket, "buck");
187 *head = old_rgw_obj(*bucket, "obj");
189 OldObjManifestPart part;
191 part.size = head_max_size;
194 manifest->append(0, part);
195 test_objs->push_back(part.loc);
198 append_rand_alpha(g_ceph_context, prefix, prefix, 16);
201 for (uint64_t ofs = head_max_size; ofs < obj_size; ofs += stripe_size, i++) {
203 snprintf(buf, sizeof(buf), "%s.%d", prefix.c_str(), i);
204 old_rgw_obj loc(*bucket, buf);
205 loc.set_ns("shadow");
206 OldObjManifestPart part;
208 part.size = min(stripe_size, obj_size - ofs);
211 manifest->append(ofs, part);
213 test_objs->push_back(loc);
217 TEST(TestRGWManifest, head_only_obj) {
219 RGWObjManifest manifest;
222 RGWObjManifest::generator gen;
224 int obj_size = 256 * 1024;
228 gen_obj(env, obj_size, 512 * 1024, 4 * 1024 * 1024, &manifest, env.zonegroup.default_placement, &bucket, &head, &gen, &objs);
230 cout << " manifest.get_obj_size()=" << manifest.get_obj_size() << std::endl;
231 cout << " manifest.get_head_size()=" << manifest.get_head_size() << std::endl;
232 list<rgw_obj>::iterator liter;
234 RGWObjManifest::obj_iterator iter;
235 for (iter = manifest.obj_begin(), liter = objs.begin();
236 iter != manifest.obj_end() && liter != objs.end();
238 ASSERT_TRUE(env.get_raw(*liter) == env.get_raw(iter.get_location()));
241 ASSERT_TRUE(iter == manifest.obj_end());
242 ASSERT_TRUE(liter == objs.end());
244 rgw_raw_obj raw_head;
246 iter = manifest.obj_find(100 * 1024);
247 ASSERT_TRUE(env.get_raw(iter.get_location()) == env.get_raw(head));
248 ASSERT_EQ((int)iter.get_stripe_size(), obj_size);
251 TEST(TestRGWManifest, obj_with_head_and_tail) {
253 RGWObjManifest manifest;
256 RGWObjManifest::generator gen;
260 int obj_size = 21 * 1024 * 1024 + 1000;
261 int stripe_size = 4 * 1024 * 1024;
262 int head_size = 512 * 1024;
264 gen_obj(env, obj_size, head_size, stripe_size, &manifest, env.zonegroup.default_placement, &bucket, &head, &gen, &objs);
266 list<rgw_obj>::iterator liter;
268 rgw_obj_select last_obj;
270 RGWObjManifest::obj_iterator iter;
271 for (iter = manifest.obj_begin(), liter = objs.begin();
272 iter != manifest.obj_end() && liter != objs.end();
274 cout << "*liter=" << *liter << " iter.get_location()=" << env.get_raw(iter.get_location()) << std::endl;
275 ASSERT_TRUE(env.get_raw(*liter) == env.get_raw(iter.get_location()));
277 last_obj = iter.get_location();
280 ASSERT_TRUE(iter == manifest.obj_end());
281 ASSERT_TRUE(liter == objs.end());
283 iter = manifest.obj_find(100 * 1024);
284 ASSERT_TRUE(env.get_raw(iter.get_location()) == env.get_raw(head));
285 ASSERT_EQ((int)iter.get_stripe_size(), head_size);
287 uint64_t ofs = 20 * 1024 * 1024 + head_size;
288 iter = manifest.obj_find(ofs + 100);
290 ASSERT_TRUE(env.get_raw(iter.get_location()) == env.get_raw(last_obj));
291 ASSERT_EQ(iter.get_stripe_ofs(), ofs);
292 ASSERT_EQ(iter.get_stripe_size(), obj_size - ofs);
295 TEST(TestRGWManifest, multipart) {
298 vector <RGWObjManifest> pm(num_parts);
300 uint64_t part_size = 10 * 1024 * 1024;
301 uint64_t stripe_size = 4 * 1024 * 1024;
303 string upload_id = "abc123";
305 for (int i = 0; i < num_parts; ++i) {
306 RGWObjManifest& manifest = pm[i];
307 RGWObjManifest::generator gen;
308 manifest.set_prefix(upload_id);
310 manifest.set_multipart_part_rule(stripe_size, i + 1);
314 for (ofs = 0; ofs < part_size; ofs += stripe_size) {
316 int r = gen.create_begin(g_ceph_context, &manifest, env.zonegroup.default_placement, bucket, head);
320 gen.create_next(ofs);
323 if (ofs > part_size) {
324 gen.create_next(part_size);
330 for (int i = 0; i < num_parts; i++) {
331 m.append(pm[i], env.zonegroup, env.zone_params);
333 RGWObjManifest::obj_iterator iter;
334 for (iter = m.obj_begin(); iter != m.obj_end(); ++iter) {
335 RGWObjManifest::obj_iterator fiter = m.obj_find(iter.get_ofs());
336 ASSERT_TRUE(env.get_raw(fiter.get_location()) == env.get_raw(iter.get_location()));
339 ASSERT_EQ(m.get_obj_size(), num_parts * part_size);
342 TEST(TestRGWManifest, old_obj_manifest) {
344 OldObjManifest old_manifest;
345 old_rgw_bucket old_bucket;
346 old_rgw_obj old_head;
348 int obj_size = 40 * 1024 * 1024;
349 uint64_t stripe_size = 4 * 1024 * 1024;
350 uint64_t head_size = 512 * 1024;
352 list<old_rgw_obj> old_objs;
354 gen_old_obj(env, obj_size, head_size, stripe_size, &old_manifest, &old_bucket, &old_head, &old_objs);
356 ASSERT_EQ(old_objs.size(), 11u);
360 ::encode(old_manifest , bl);
362 RGWObjManifest manifest;
365 auto iter = bl.begin();
366 ::decode(manifest, iter);
367 } catch (buffer::error& err) {
371 rgw_raw_obj last_obj;
373 RGWObjManifest::obj_iterator iter;
374 auto liter = old_objs.begin();
375 for (iter = manifest.obj_begin();
376 iter != manifest.obj_end() && liter != old_objs.end();
378 rgw_pool old_pool(liter->bucket.data_pool);
380 prepend_old_bucket_marker(old_bucket, liter->get_object(), old_oid);
381 rgw_raw_obj raw_old(old_pool, old_oid);
382 cout << "*liter=" << raw_old << " iter.get_location()=" << env.get_raw(iter.get_location()) << std::endl;
383 ASSERT_EQ(raw_old, env.get_raw(iter.get_location()));
385 last_obj = env.get_raw(iter.get_location());
388 ASSERT_TRUE(liter == old_objs.end());
389 ASSERT_TRUE(iter == manifest.obj_end());
394 int main(int argc, char **argv) {
395 vector<const char*> args;
396 argv_to_vec(argc, (const char **)argv, args);
399 auto cct = global_init(NULL, args, CEPH_ENTITY_TYPE_CLIENT, CODE_ENVIRONMENT_UTILITY, 0);
400 common_init_finish(g_ceph_context);
401 ::testing::InitGoogleTest(&argc, argv);
402 return RUN_ALL_TESTS();