1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
2 // vim: ts=8 sw=2 smarttab
4 * Ceph - scalable distributed file system
6 * Copyright (C) 2012 Inktank, Inc.
8 * This is free software; you can redistribute it and/or
9 * modify it under the terms of the GNU Lesser General Public
10 * License version 2.1, as published by the Free Software
11 * Foundation. See file COPYING.
17 #include <boost/scoped_ptr.hpp>
19 #include "common/ceph_argparse.h"
20 #include "common/config.h"
21 #include "common/errno.h"
22 #include "common/strtol.h"
23 #include "global/global_context.h"
24 #include "global/global_init.h"
25 #include "include/stringify.h"
26 #include "include/utime.h"
27 #include "common/Clock.h"
28 #include "kv/KeyValueDB.h"
29 #include "common/url_escape.h"
32 #include "os/bluestore/BlueStore.h"
39 boost::scoped_ptr<BlueStore> bluestore;
41 // TODO: make KeyValueDB enable_shared_from_this
42 // bluestore will hold *db* also, use unique_ptr/shared_ptr will
49 StoreTool(string type, const string &path) : store_path(path) {
51 if (type == "bluestore-kv") {
53 // note: we'll leak this! the only user is ceph-kvstore-tool and
55 bluestore.reset(new BlueStore(g_ceph_context, path));
56 int r = bluestore->start_kv_only(&db_ptr);
61 cerr << "bluestore not compiled in" << std::endl;
65 db_ptr = KeyValueDB::create(g_ceph_context, type, path);
66 int r = db_ptr->open(std::cerr);
68 cerr << "failed to open type " << type << " path " << path << ": "
69 << cpp_strerror(r) << std::endl;
87 uint32_t traverse(const string &prefix,
90 KeyValueDB::WholeSpaceIterator iter = db->get_iterator();
93 iter->seek_to_first();
95 iter->seek_to_first(prefix);
99 while (iter->valid()) {
100 pair<string,string> rk = iter->raw_key();
101 if (!prefix.empty() && (rk.first != prefix))
105 *out << url_escape(rk.first) << "\t" << url_escape(rk.second);
109 bl.append(rk.second);
110 bl.append(iter->value());
112 crc = bl.crc32c(crc);
114 *out << "\t" << bl.crc32c(0);
125 void list(const string &prefix, const bool do_crc) {
126 traverse(prefix, do_crc, &std::cout);
129 bool exists(const string &prefix) {
130 assert(!prefix.empty());
131 KeyValueDB::WholeSpaceIterator iter = db->get_iterator();
132 iter->seek_to_first(prefix);
133 return (iter->valid() && (iter->raw_key().first == prefix));
136 bool exists(const string &prefix, const string &key) {
137 assert(!prefix.empty());
140 return exists(prefix);
144 get(prefix, key, exists);
148 bufferlist get(const string &prefix, const string &key, bool &exists) {
149 assert(!prefix.empty() && !key.empty());
151 map<string,bufferlist> result;
152 std::set<std::string> keys;
154 db->get(prefix, keys, &result);
156 if (result.count(key) > 0) {
164 uint64_t get_size() {
165 map<string,uint64_t> extras;
166 uint64_t s = db->get_estimated_size(extras);
167 for (map<string,uint64_t>::iterator p = extras.begin();
168 p != extras.end(); ++p) {
169 std::cout << p->first << " - " << p->second << std::endl;
171 std::cout << "total: " << s << std::endl;
175 bool set(const string &prefix, const string &key, bufferlist &val) {
176 assert(!prefix.empty());
177 assert(!key.empty());
178 assert(val.length() > 0);
180 KeyValueDB::Transaction tx = db->get_transaction();
181 tx->set(prefix, key, val);
182 int ret = db->submit_transaction_sync(tx);
187 bool rm(const string& prefix, const string& key) {
188 assert(!prefix.empty());
189 assert(!key.empty());
191 KeyValueDB::Transaction tx = db->get_transaction();
192 tx->rmkey(prefix, key);
193 int ret = db->submit_transaction_sync(tx);
198 bool rm_prefix(const string& prefix) {
199 assert(!prefix.empty());
201 KeyValueDB::Transaction tx = db->get_transaction();
202 tx->rmkeys_by_prefix(prefix);
203 int ret = db->submit_transaction_sync(tx);
208 int copy_store_to(string type, const string &other_path,
209 const int num_keys_per_tx) {
211 if (num_keys_per_tx <= 0) {
212 std::cerr << "must specify a number of keys/tx > 0" << std::endl;
216 // open or create a leveldb store at @p other_path
217 KeyValueDB *other = KeyValueDB::create(g_ceph_context, type, other_path);
218 int err = other->create_and_open(std::cerr);
222 KeyValueDB::WholeSpaceIterator it = db->get_iterator();
224 uint64_t total_keys = 0;
225 uint64_t total_size = 0;
226 uint64_t total_txs = 0;
228 utime_t started_at = ceph_clock_now();
233 KeyValueDB::Transaction tx = other->get_transaction();
236 while (it->valid() && num_keys < num_keys_per_tx) {
237 pair<string,string> k = it->raw_key();
238 bufferlist v = it->value();
239 tx->set(k.first, k.second, v);
242 total_size += v.length();
248 total_keys += num_keys;
251 other->submit_transaction_sync(tx);
253 utime_t cur_duration = ceph_clock_now() - started_at;
254 std::cout << "ts = " << cur_duration << "s, copied " << total_keys
255 << " keys so far (" << stringify(si_t(total_size)) << ")"
258 } while (it->valid());
260 utime_t time_taken = ceph_clock_now() - started_at;
262 std::cout << "summary:" << std::endl;
263 std::cout << " copied " << total_keys << " keys" << std::endl;
264 std::cout << " used " << total_txs << " transactions" << std::endl;
265 std::cout << " total size " << stringify(si_t(total_size)) << std::endl;
266 std::cout << " from '" << store_path << "' to '" << other_path << "'"
268 std::cout << " duration " << time_taken << " seconds" << std::endl;
276 void compact_prefix(string prefix) {
277 db->compact_prefix(prefix);
279 void compact_range(string prefix, string start, string end) {
280 db->compact_range(prefix, start, end);
284 void usage(const char *pname)
286 std::cerr << "Usage: " << pname << " <leveldb|rocksdb|bluestore-kv> <store path> command [args...]\n"
289 << " list [prefix]\n"
290 << " list-crc [prefix]\n"
291 << " exists <prefix> [key]\n"
292 << " get <prefix> <key> [out <file>]\n"
293 << " crc <prefix> <key>\n"
294 << " get-size [<prefix> <key>]\n"
295 << " set <prefix> <key> [ver <N>|in <file>]\n"
296 << " rm <prefix> <key>\n"
297 << " rm-prefix <prefix>\n"
298 << " store-copy <path> [num-keys-per-tx]\n"
299 << " store-crc <path>\n"
301 << " compact-prefix <prefix>\n"
302 << " compact-range <prefix> <start> <end>\n"
306 int main(int argc, const char *argv[])
308 vector<const char*> args;
309 argv_to_vec(argc, argv, args);
312 auto cct = global_init(
314 CEPH_ENTITY_TYPE_CLIENT, CODE_ENVIRONMENT_UTILITY, 0);
315 common_init_finish(g_ceph_context);
318 if (args.size() < 3) {
323 string type(args[0]);
324 string path(args[1]);
327 StoreTool st(type, path);
329 if (cmd == "list" || cmd == "list-crc") {
332 prefix = url_unescape(argv[4]);
334 bool do_crc = (cmd == "list-crc");
336 st.list(prefix, do_crc);
338 } else if (cmd == "exists") {
344 string prefix(url_unescape(argv[4]));
346 key = url_unescape(argv[5]);
348 bool ret = st.exists(prefix, key);
349 std::cout << "(" << url_escape(prefix) << ", " << url_escape(key) << ") "
350 << (ret ? "exists" : "does not exist")
352 return (ret ? 0 : 1);
354 } else if (cmd == "get") {
359 string prefix(url_unescape(argv[4]));
360 string key(url_unescape(argv[5]));
363 bufferlist bl = st.get(prefix, key, exists);
364 std::cout << "(" << url_escape(prefix) << ", " << url_escape(key) << ")";
366 std::cout << " does not exist" << std::endl;
369 std::cout << std::endl;
372 string subcmd(argv[6]);
373 if (subcmd != "out") {
374 std::cerr << "unrecognized subcmd '" << subcmd << "'"
379 std::cerr << "output path not specified" << std::endl;
385 std::cerr << "unspecified out file" << std::endl;
389 int err = bl.write_file(argv[7], 0644);
391 std::cerr << "error writing value to '" << out << "': "
392 << cpp_strerror(err) << std::endl;
398 std::cout << os.str() << std::endl;
401 } else if (cmd == "crc") {
406 string prefix(url_unescape(argv[4]));
407 string key(url_unescape(argv[5]));
410 bufferlist bl = st.get(prefix, key, exists);
411 std::cout << "(" << url_escape(prefix) << ", " << url_escape(key) << ") ";
413 std::cout << " does not exist" << std::endl;
416 std::cout << " crc " << bl.crc32c(0) << std::endl;
418 } else if (cmd == "get-size") {
419 std::cout << "estimated store size: " << st.get_size() << std::endl;
428 string prefix(url_unescape(argv[4]));
429 string key(url_unescape(argv[5]));
432 bufferlist bl = st.get(prefix, key, exists);
434 std::cerr << "(" << url_escape(prefix) << "," << url_escape(key)
435 << ") does not exist" << std::endl;
438 std::cout << "(" << url_escape(prefix) << "," << url_escape(key)
439 << ") size " << si_t(bl.length()) << std::endl;
441 } else if (cmd == "set") {
446 string prefix(url_unescape(argv[4]));
447 string key(url_unescape(argv[5]));
448 string subcmd(argv[6]);
452 if (subcmd == "ver") {
453 version_t v = (version_t) strict_strtoll(argv[7], 10, &errstr);
454 if (!errstr.empty()) {
455 std::cerr << "error reading version: " << errstr << std::endl;
459 } else if (subcmd == "in") {
460 int ret = val.read_file(argv[7], &errstr);
461 if (ret < 0 || !errstr.empty()) {
462 std::cerr << "error reading file: " << errstr << std::endl;
466 std::cerr << "unrecognized subcommand '" << subcmd << "'" << std::endl;
471 bool ret = st.set(prefix, key, val);
473 std::cerr << "error setting ("
474 << url_escape(prefix) << "," << url_escape(key) << ")" << std::endl;
477 } else if (cmd == "rm") {
482 string prefix(url_unescape(argv[4]));
483 string key(url_unescape(argv[5]));
485 bool ret = st.rm(prefix, key);
487 std::cerr << "error removing ("
488 << url_escape(prefix) << "," << url_escape(key) << ")"
492 } else if (cmd == "rm-prefix") {
497 string prefix(url_unescape(argv[4]));
499 bool ret = st.rm_prefix(prefix);
501 std::cerr << "error removing prefix ("
502 << url_escape(prefix) << ")"
506 } else if (cmd == "store-copy") {
507 int num_keys_per_tx = 128; // magic number that just feels right.
511 } else if (argc > 5) {
513 num_keys_per_tx = strict_strtol(argv[5], 10, &err);
515 std::cerr << "invalid num_keys_per_tx: " << err << std::endl;
520 int ret = st.copy_store_to(argv[1], argv[4], num_keys_per_tx);
522 std::cerr << "error copying store to path '" << argv[4]
523 << "': " << cpp_strerror(ret) << std::endl;
527 } else if (cmd == "store-crc") {
528 uint32_t crc = st.traverse(string(), true, NULL);
529 std::cout << "store at '" << path << "' crc " << crc << std::endl;
531 } else if (cmd == "compact") {
533 } else if (cmd == "compact-prefix") {
538 string prefix(url_unescape(argv[4]));
539 st.compact_prefix(prefix);
540 } else if (cmd == "compact-range") {
545 string prefix(url_unescape(argv[4]));
546 string start(url_unescape(argv[5]));
547 string end(url_unescape(argv[6]));
548 st.compact_range(prefix, start, end);
550 std::cerr << "Unrecognized command: " << cmd << std::endl;