1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
2 // vim: ts=8 sw=2 smarttab
4 * Ceph - scalable distributed file system
6 * Copyright (C) 2015 Red Hat
8 * This is free software; you can redistribute it and/or
9 * modify it under the terms of the GNU Lesser General Public
10 * License version 2.1, as published by the Free Software
11 * Foundation. See file COPYING.
15 #ifndef CEPH_RGW_ORPHAN_H
16 #define CEPH_RGW_ORPHAN_H
18 #include "common/config.h"
19 #include "common/Formatter.h"
20 #include "common/errno.h"
22 #include "rgw_rados.h"
24 #define dout_subsys ceph_subsys_rgw
26 #define RGW_ORPHAN_INDEX_OID "orphan.index"
27 #define RGW_ORPHAN_INDEX_PREFIX "orphan.scan"
30 enum RGWOrphanSearchStageId {
31 ORPHAN_SEARCH_STAGE_UNKNOWN = 0,
32 ORPHAN_SEARCH_STAGE_INIT = 1,
33 ORPHAN_SEARCH_STAGE_LSPOOL = 2,
34 ORPHAN_SEARCH_STAGE_LSBUCKETS = 3,
35 ORPHAN_SEARCH_STAGE_ITERATE_BI = 4,
36 ORPHAN_SEARCH_STAGE_COMPARE = 5,
40 struct RGWOrphanSearchStage {
41 RGWOrphanSearchStageId stage;
45 RGWOrphanSearchStage() : stage(ORPHAN_SEARCH_STAGE_UNKNOWN), shard(0) {}
46 explicit RGWOrphanSearchStage(RGWOrphanSearchStageId _stage) : stage(_stage), shard(0) {}
47 RGWOrphanSearchStage(RGWOrphanSearchStageId _stage, int _shard, const string& _marker) : stage(_stage), shard(_shard), marker(_marker) {}
49 void encode(bufferlist& bl) const {
50 ENCODE_START(1, 1, bl);
51 ::encode((int)stage, bl);
57 void decode(bufferlist::iterator& bl) {
61 stage = (RGWOrphanSearchStageId)s;
67 void dump(Formatter *f) const;
69 WRITE_CLASS_ENCODER(RGWOrphanSearchStage)
71 struct RGWOrphanSearchInfo {
77 void encode(bufferlist& bl) const {
78 ENCODE_START(2, 1, bl);
79 ::encode(job_name, bl);
80 ::encode(pool.to_str(), bl);
81 ::encode(num_shards, bl);
82 ::encode(start_time, bl);
86 void decode(bufferlist::iterator& bl) {
88 ::decode(job_name, bl);
92 ::decode(num_shards, bl);
93 ::decode(start_time, bl);
97 void dump(Formatter *f) const;
99 WRITE_CLASS_ENCODER(RGWOrphanSearchInfo)
101 struct RGWOrphanSearchState {
102 RGWOrphanSearchInfo info;
103 RGWOrphanSearchStage stage;
105 RGWOrphanSearchState() : stage(ORPHAN_SEARCH_STAGE_UNKNOWN) {}
107 void encode(bufferlist& bl) const {
108 ENCODE_START(1, 1, bl);
114 void decode(bufferlist::iterator& bl) {
121 void dump(Formatter *f) const;
123 WRITE_CLASS_ENCODER(RGWOrphanSearchState)
125 class RGWOrphanStore {
127 librados::IoCtx ioctx;
132 explicit RGWOrphanStore(RGWRados *_store) : store(_store), oid(RGW_ORPHAN_INDEX_OID) {}
134 librados::IoCtx& get_ioctx() { return ioctx; }
138 int read_job(const string& job_name, RGWOrphanSearchState& state);
139 int write_job(const string& job_name, const RGWOrphanSearchState& state);
140 int remove_job(const string& job_name);
141 int list_jobs(map<string,RGWOrphanSearchState> &job_list);
144 int store_entries(const string& oid, const map<string, bufferlist>& entries);
145 int read_entries(const string& oid, const string& marker, map<string, bufferlist> *entries, bool *truncated);
149 class RGWOrphanSearch {
152 RGWOrphanStore orphan_store;
154 RGWOrphanSearchInfo search_info;
155 RGWOrphanSearchStage search_stage;
157 map<int, string> all_objs_index;
158 map<int, string> buckets_instance_index;
159 map<int, string> linked_objs_index;
161 string index_objs_prefix;
163 uint16_t max_concurrent_ios;
166 struct log_iter_info {
168 list<string>::iterator cur;
169 list<string>::iterator end;
172 int log_oids(map<int, string>& log_shards, map<int, list<string> >& oids);
174 #define RGW_ORPHANSEARCH_HASH_PRIME 7877
175 int orphan_shard(const string& str) {
176 return ceph_str_hash_linux(str.c_str(), str.size()) % RGW_ORPHANSEARCH_HASH_PRIME % search_info.num_shards;
179 int handle_stat_result(map<int, list<string> >& oids, RGWRados::Object::Stat::Result& result);
180 int pop_and_handle_stat_op(map<int, list<string> >& oids, std::deque<RGWRados::Object::Stat>& ops);
183 int remove_index(map<int, string>& index);
185 RGWOrphanSearch(RGWRados *_store, int _max_ios, uint64_t _stale_secs) : store(_store), orphan_store(store), max_concurrent_ios(_max_ios), stale_secs(_stale_secs) {}
188 RGWOrphanSearchState state;
189 state.info = search_info;
190 state.stage = search_stage;
191 return orphan_store.write_job(search_info.job_name, state);
194 int init(const string& job_name, RGWOrphanSearchInfo *info);
196 int create(const string& job_name, int num_shards);
198 int build_all_oids_index();
199 int build_buckets_instance_index();
200 int build_linked_oids_for_bucket(const string& bucket_instance_id, map<int, list<string> >& oids);
201 int build_linked_oids_index();
202 int compare_oid_indexes();