+++ /dev/null
-#include "rgw_sync_module_es.h"
-#include "rgw_sync_module_es_rest.h"
-#include "rgw_es_query.h"
-#include "rgw_op.h"
-#include "rgw_rest.h"
-#include "rgw_rest_s3.h"
-
-#define dout_context g_ceph_context
-#define dout_subsys ceph_subsys_rgw
-
-struct es_index_obj_response {
- string bucket;
- rgw_obj_key key;
- uint64_t versioned_epoch{0};
- ACLOwner owner;
- set<string> read_permissions;
-
- struct {
- uint64_t size{0};
- ceph::real_time mtime;
- string etag;
- string content_type;
- map<string, string> custom_str;
- map<string, int64_t> custom_int;
- map<string, string> custom_date;
-
- template <class T>
- struct _custom_entry {
- string name;
- T value;
- void decode_json(JSONObj *obj) {
- JSONDecoder::decode_json("name", name, obj);
- JSONDecoder::decode_json("value", value, obj);
- }
- };
-
- void decode_json(JSONObj *obj) {
- JSONDecoder::decode_json("size", size, obj);
- string mtime_str;
- JSONDecoder::decode_json("mtime", mtime_str, obj);
- parse_time(mtime_str.c_str(), &mtime);
- JSONDecoder::decode_json("etag", etag, obj);
- JSONDecoder::decode_json("content_type", content_type, obj);
- list<_custom_entry<string> > str_entries;
- JSONDecoder::decode_json("custom-string", str_entries, obj);
- for (auto& e : str_entries) {
- custom_str[e.name] = e.value;
- }
- list<_custom_entry<int64_t> > int_entries;
- JSONDecoder::decode_json("custom-int", int_entries, obj);
- for (auto& e : int_entries) {
- custom_int[e.name] = e.value;
- }
- list<_custom_entry<string> > date_entries;
- JSONDecoder::decode_json("custom-date", date_entries, obj);
- for (auto& e : date_entries) {
- custom_date[e.name] = e.value;
- }
- }
- } meta;
-
- void decode_json(JSONObj *obj) {
- JSONDecoder::decode_json("bucket", bucket, obj);
- JSONDecoder::decode_json("name", key.name, obj);
- JSONDecoder::decode_json("instance", key.instance, obj);
- JSONDecoder::decode_json("versioned_epoch", versioned_epoch, obj);
- JSONDecoder::decode_json("permissions", read_permissions, obj);
- JSONDecoder::decode_json("owner", owner, obj);
- JSONDecoder::decode_json("meta", meta, obj);
- }
-};
-
-struct es_search_response {
- uint32_t took;
- bool timed_out;
- struct {
- uint32_t total;
- uint32_t successful;
- uint32_t failed;
- void decode_json(JSONObj *obj) {
- JSONDecoder::decode_json("total", total, obj);
- JSONDecoder::decode_json("successful", successful, obj);
- JSONDecoder::decode_json("failed", failed, obj);
- }
- } shards;
- struct obj_hit {
- string index;
- string type;
- string id;
- // double score
- es_index_obj_response source;
- void decode_json(JSONObj *obj) {
- JSONDecoder::decode_json("_index", index, obj);
- JSONDecoder::decode_json("_type", type, obj);
- JSONDecoder::decode_json("_id", id, obj);
- JSONDecoder::decode_json("_source", source, obj);
- }
- };
- struct {
- uint32_t total;
- // double max_score;
- list<obj_hit> hits;
- void decode_json(JSONObj *obj) {
- JSONDecoder::decode_json("total", total, obj);
- // JSONDecoder::decode_json("max_score", max_score, obj);
- JSONDecoder::decode_json("hits", hits, obj);
- }
- } hits;
- void decode_json(JSONObj *obj) {
- JSONDecoder::decode_json("took", took, obj);
- JSONDecoder::decode_json("timed_out", timed_out, obj);
- JSONDecoder::decode_json("_shards", shards, obj);
- JSONDecoder::decode_json("hits", hits, obj);
- }
-};
-
-class RGWMetadataSearchOp : public RGWOp {
- RGWSyncModuleInstanceRef sync_module_ref;
- RGWElasticSyncModuleInstance *es_module;
-protected:
- string expression;
- string custom_prefix;
-#define MAX_KEYS_DEFAULT 100
- uint64_t max_keys{MAX_KEYS_DEFAULT};
- string marker_str;
- uint64_t marker{0};
- string next_marker;
- bool is_truncated{false};
- string err;
-
- es_search_response response;
-
-public:
- RGWMetadataSearchOp(const RGWSyncModuleInstanceRef& sync_module) : sync_module_ref(sync_module) {
- es_module = static_cast<RGWElasticSyncModuleInstance *>(sync_module_ref.get());
- }
-
- int verify_permission() {
- return 0;
- }
- virtual int get_params() = 0;
- void pre_exec();
- void execute();
-
- virtual void send_response() = 0;
- virtual const string name() { return "metadata_search"; }
- virtual RGWOpType get_type() { return RGW_OP_METADATA_SEARCH; }
- virtual uint32_t op_mask() { return RGW_OP_TYPE_READ; }
-};
-
-void RGWMetadataSearchOp::pre_exec()
-{
- rgw_bucket_object_pre_exec(s);
-}
-
-void RGWMetadataSearchOp::execute()
-{
- op_ret = get_params();
- if (op_ret < 0)
- return;
-
- list<pair<string, string> > conds;
-
- if (!s->user->system) {
- conds.push_back(make_pair("permissions", s->user->user_id.to_str()));
- }
-
- if (!s->bucket_name.empty()) {
- conds.push_back(make_pair("bucket", s->bucket_name));
- }
-
- ESQueryCompiler es_query(expression, &conds, custom_prefix);
-
- static map<string, string, ltstr_nocase> aliases = {
- { "bucket", "bucket" }, /* forces lowercase */
- { "name", "name" },
- { "key", "name" },
- { "instance", "instance" },
- { "etag", "meta.etag" },
- { "size", "meta.size" },
- { "mtime", "meta.mtime" },
- { "lastmodified", "meta.mtime" },
- { "contenttype", "meta.contenttype" },
- };
- es_query.set_field_aliases(&aliases);
-
- static map<string, ESEntityTypeMap::EntityType> generic_map = { {"bucket", ESEntityTypeMap::ES_ENTITY_STR},
- {"name", ESEntityTypeMap::ES_ENTITY_STR},
- {"instance", ESEntityTypeMap::ES_ENTITY_STR},
- {"permissions", ESEntityTypeMap::ES_ENTITY_STR},
- {"meta.etag", ESEntityTypeMap::ES_ENTITY_STR},
- {"meta.contenttype", ESEntityTypeMap::ES_ENTITY_STR},
- {"meta.mtime", ESEntityTypeMap::ES_ENTITY_DATE},
- {"meta.size", ESEntityTypeMap::ES_ENTITY_INT} };
- ESEntityTypeMap gm(generic_map);
- es_query.set_generic_type_map(&gm);
-
- static set<string> restricted_fields = { {"permissions"} };
- es_query.set_restricted_fields(&restricted_fields);
-
- map<string, ESEntityTypeMap::EntityType> custom_map;
- for (auto& i : s->bucket_info.mdsearch_config) {
- custom_map[i.first] = (ESEntityTypeMap::EntityType)i.second;
- }
-
- ESEntityTypeMap em(custom_map);
- es_query.set_custom_type_map(&em);
-
- bool valid = es_query.compile(&err);
- if (!valid) {
- ldout(s->cct, 10) << "invalid query, failed generating request json" << dendl;
- op_ret = -EINVAL;
- return;
- }
-
- JSONFormatter f;
- encode_json("root", es_query, &f);
-
- RGWRESTConn *conn = es_module->get_rest_conn();
-
- bufferlist in;
- bufferlist out;
-
- stringstream ss;
-
- f.flush(ss);
- in.append(ss.str());
-
- string resource = es_module->get_index_path() + "/_search";
- param_vec_t params;
- static constexpr int BUFSIZE = 32;
- char buf[BUFSIZE];
- snprintf(buf, sizeof(buf), "%lld", (long long)max_keys);
- params.push_back(param_pair_t("size", buf));
- if (marker > 0) {
- params.push_back(param_pair_t("from", marker_str.c_str()));
- }
- ldout(s->cct, 20) << "sending request to elasticsearch, payload=" << string(in.c_str(), in.length()) << dendl;
- op_ret = conn->get_resource(resource, ¶ms, nullptr, out, &in);
- if (op_ret < 0) {
- ldout(s->cct, 0) << "ERROR: failed to fetch resource (r=" << resource << ", ret=" << op_ret << ")" << dendl;
- return;
- }
-
- ldout(s->cct, 20) << "response: " << string(out.c_str(), out.length()) << dendl;
-
- JSONParser jparser;
- if (!jparser.parse(out.c_str(), out.length())) {
- ldout(s->cct, 0) << "ERROR: failed to parse elasticsearch response" << dendl;
- op_ret = -EINVAL;
- return;
- }
-
- try {
- decode_json_obj(response, &jparser);
- } catch (JSONDecoder::err& e) {
- ldout(s->cct, 0) << "ERROR: failed to decode JSON input: " << e.message << dendl;
- op_ret = -EINVAL;
- return;
- }
-
-}
-
-class RGWMetadataSearch_ObjStore_S3 : public RGWMetadataSearchOp {
-public:
- RGWMetadataSearch_ObjStore_S3(const RGWSyncModuleInstanceRef& _sync_module) : RGWMetadataSearchOp(_sync_module) {
- custom_prefix = "x-amz-meta-";
- }
-
- int get_params() override {
- expression = s->info.args.get("query");
- bool exists;
- string max_keys_str = s->info.args.get("max-keys", &exists);
-#define MAX_KEYS_MAX 10000
- if (exists) {
- string err;
- max_keys = strict_strtoll(max_keys_str.c_str(), 10, &err);
- if (!err.empty()) {
- return -EINVAL;
- }
- if (max_keys > MAX_KEYS_MAX) {
- max_keys = MAX_KEYS_MAX;
- }
- }
- marker_str = s->info.args.get("marker", &exists);
- if (exists) {
- string err;
- marker = strict_strtoll(marker_str.c_str(), 10, &err);
- if (!err.empty()) {
- return -EINVAL;
- }
- }
- uint64_t nm = marker + max_keys;
- static constexpr int BUFSIZE = 32;
- char buf[BUFSIZE];
- snprintf(buf, sizeof(buf), "%lld", (long long)nm);
- next_marker = buf;
- return 0;
- }
- void send_response() override {
- if (op_ret) {
- s->err.message = err;
- set_req_state_err(s, op_ret);
- }
- dump_errno(s);
- end_header(s, this, "application/xml");
-
- if (op_ret < 0) {
- return;
- }
-
- is_truncated = (response.hits.hits.size() >= max_keys);
-
- s->formatter->open_object_section("SearchMetadataResponse");
- s->formatter->dump_string("Marker", marker_str);
- s->formatter->dump_string("IsTruncated", (is_truncated ? "true" : "false"));
- if (is_truncated) {
- s->formatter->dump_string("NextMarker", next_marker);
- }
- if (s->format == RGW_FORMAT_JSON) {
- s->formatter->open_array_section("Objects");
- }
- for (auto& i : response.hits.hits) {
- s->formatter->open_object_section("Contents");
- es_index_obj_response& e = i.source;
- s->formatter->dump_string("Bucket", e.bucket);
- s->formatter->dump_string("Key", e.key.name);
- string instance = (!e.key.instance.empty() ? e.key.instance : "null");
- s->formatter->dump_string("Instance", instance.c_str());
- s->formatter->dump_int("VersionedEpoch", e.versioned_epoch);
- dump_time(s, "LastModified", &e.meta.mtime);
- s->formatter->dump_int("Size", e.meta.size);
- s->formatter->dump_format("ETag", "\"%s\"", e.meta.etag.c_str());
- s->formatter->dump_string("ContentType", e.meta.content_type.c_str());
- dump_owner(s, e.owner.get_id(), e.owner.get_display_name());
- s->formatter->open_array_section("CustomMetadata");
- for (auto& m : e.meta.custom_str) {
- s->formatter->open_object_section("Entry");
- s->formatter->dump_string("Name", m.first.c_str());
- s->formatter->dump_string("Value", m.second);
- s->formatter->close_section();
- }
- for (auto& m : e.meta.custom_int) {
- s->formatter->open_object_section("Entry");
- s->formatter->dump_string("Name", m.first.c_str());
- s->formatter->dump_int("Value", m.second);
- s->formatter->close_section();
- }
- for (auto& m : e.meta.custom_date) {
- s->formatter->open_object_section("Entry");
- s->formatter->dump_string("Name", m.first.c_str());
- s->formatter->dump_string("Value", m.second);
- s->formatter->close_section();
- }
- s->formatter->close_section();
- rgw_flush_formatter(s, s->formatter);
- s->formatter->close_section();
- };
- if (s->format == RGW_FORMAT_JSON) {
- s->formatter->close_section();
- }
- s->formatter->close_section();
- rgw_flush_formatter_and_reset(s, s->formatter);
- }
-};
-
-class RGWHandler_REST_MDSearch_S3 : public RGWHandler_REST_S3 {
-protected:
- RGWOp *op_get() {
- if (s->info.args.exists("query")) {
- return new RGWMetadataSearch_ObjStore_S3(store->get_sync_module());
- }
- if (!s->init_state.url_bucket.empty() &&
- s->info.args.exists("mdsearch")) {
- return new RGWGetBucketMetaSearch_ObjStore_S3;
- }
- return nullptr;
- }
- RGWOp *op_head() {
- return nullptr;
- }
- RGWOp *op_post() {
- return nullptr;
- }
-public:
- RGWHandler_REST_MDSearch_S3(const rgw::auth::StrategyRegistry& auth_registry) : RGWHandler_REST_S3(auth_registry) {}
- virtual ~RGWHandler_REST_MDSearch_S3() {}
-};
-
-
-RGWHandler_REST* RGWRESTMgr_MDSearch_S3::get_handler(struct req_state* const s,
- const rgw::auth::StrategyRegistry& auth_registry,
- const std::string& frontend_prefix)
-{
- int ret =
- RGWHandler_REST_S3::init_from_header(s,
- RGW_FORMAT_XML, true);
- if (ret < 0) {
- return nullptr;
- }
-
- if (!s->object.empty()) {
- return nullptr;
- }
-
- RGWHandler_REST *handler = new RGWHandler_REST_MDSearch_S3(auth_registry);
-
- ldout(s->cct, 20) << __func__ << " handler=" << typeid(*handler).name()
- << dendl;
- return handler;
-}
-
Code Review / stor4nfv.git / blobdiff