1 From b233d6540160c8bc5cc25b870c2140fa48776fa6 Mon Sep 17 00:00:00 2001
2 From: Dehao Shang <dehao.shang@intel.com>
3 Date: Mon, 6 Aug 2018 22:42:38 +0800
4 Subject: [PATCH 06/10] librbd: LRU policy based eviction
6 Signed-off-by: Dehao Shang <dehao.shang@intel.com>
7 Signed-off-by: Yuan Zhou <yuan.zhou@intel.com>
9 src/tools/rbd_cache/ObjectCacheStore.cc | 73 ++++++++-----
10 src/tools/rbd_cache/ObjectCacheStore.h | 14 +--
11 src/tools/rbd_cache/Policy.hpp | 22 ++++
12 src/tools/rbd_cache/SimplePolicy.hpp | 180 ++++++++++++++++++++++++++++++++
13 4 files changed, 254 insertions(+), 35 deletions(-)
14 create mode 100644 src/tools/rbd_cache/Policy.hpp
15 create mode 100644 src/tools/rbd_cache/SimplePolicy.hpp
17 diff --git a/src/tools/rbd_cache/ObjectCacheStore.cc b/src/tools/rbd_cache/ObjectCacheStore.cc
18 index 2a87469..b39fe66 100644
19 --- a/src/tools/rbd_cache/ObjectCacheStore.cc
20 +++ b/src/tools/rbd_cache/ObjectCacheStore.cc
21 @@ -14,12 +14,12 @@ namespace cache {
23 ObjectCacheStore::ObjectCacheStore(CephContext *cct, ContextWQ* work_queue)
24 : m_cct(cct), m_work_queue(work_queue),
25 - m_cache_table_lock("rbd::cache::ObjectCacheStore"),
26 m_rados(new librados::Rados()) {
27 + m_policy = new SimplePolicy(4096, 0.9); // TODO
30 ObjectCacheStore::~ObjectCacheStore() {
35 int ObjectCacheStore::init(bool reset) {
36 @@ -43,6 +43,7 @@ int ObjectCacheStore::do_promote(std::string pool_name, std::string object_name)
38 std::string cache_file_name = pool_name + object_name;
40 + //TODO(): lock on ioctx map
41 if (m_ioctxs.find(pool_name) == m_ioctxs.end()) {
42 librados::IoCtx* io_ctx = new librados::IoCtx();
43 ret = m_rados->ioctx_create(pool_name.c_str(), *io_ctx);
44 @@ -58,10 +59,8 @@ int ObjectCacheStore::do_promote(std::string pool_name, std::string object_name)
45 librados::IoCtx* ioctx = m_ioctxs[pool_name];
47 //promoting: update metadata
49 - Mutex::Locker locker(m_cache_table_lock);
50 - m_cache_table.emplace(cache_file_name, PROMOTING);
52 + m_policy->update_status(cache_file_name, PROMOTING);
53 + assert(PROMOTING == m_policy->get_status(cache_file_name));
55 librados::bufferlist* read_buf = new librados::bufferlist();
56 int object_size = 4096*1024; //TODO(): read config from image metadata
57 @@ -83,42 +82,60 @@ int ObjectCacheStore::do_promote(std::string pool_name, std::string object_name)
59 ret = cache_file.write_object_to_file(*read_buf, object_size);
61 - assert(m_cache_table.find(cache_file_name) != m_cache_table.end());
65 - Mutex::Locker locker(m_cache_table_lock);
66 - m_cache_table.emplace(cache_file_name, PROMOTED);
68 + assert(PROMOTING == m_policy->get_status(cache_file_name));
69 + m_policy->update_status(cache_file_name, PROMOTED);
70 + assert(PROMOTED == m_policy->get_status(cache_file_name));
76 +// return -1, client need to read data from cluster.
77 +// return 0, client directly read data from cache.
78 int ObjectCacheStore::lookup_object(std::string pool_name, std::string object_name) {
80 std::string cache_file_name = pool_name + object_name;
82 - Mutex::Locker locker(m_cache_table_lock);
84 - auto it = m_cache_table.find(cache_file_name);
85 - if (it != m_cache_table.end()) {
87 - if (it->second == PROMOTING) {
89 - } else if (it->second == PROMOTED) {
96 + // TODO lookup and return status;
99 + ret = m_policy->lookup_object(cache_file_name);
103 + return do_promote(pool_name, object_name);
113 - int ret = do_promote(pool_name, object_name);
114 +void ObjectCacheStore::evict_thread_body() {
116 + while(m_evict_go) {
117 + std::string temp_cache_file;
120 + ret = m_policy->evict_object(temp_cache_file);
126 + // delete temp_cache_file file.
128 + assert(EVICTING == m_policy->get_status(temp_cache_file));
130 + m_policy->update_status(temp_cache_file, EVICTED);
132 + assert(NONE == m_policy->get_status(temp_cache_file));
137 int ObjectCacheStore::shutdown() {
140 diff --git a/src/tools/rbd_cache/ObjectCacheStore.h b/src/tools/rbd_cache/ObjectCacheStore.h
141 index db09efa..5118a73 100644
142 --- a/src/tools/rbd_cache/ObjectCacheStore.h
143 +++ b/src/tools/rbd_cache/ObjectCacheStore.h
145 #include "librbd/ImageCtx.h"
146 #include "librbd/ImageState.h"
147 #include "librbd/cache/SharedPersistentObjectCacherFile.h"
148 +#include "SimplePolicy.hpp"
150 using librados::Rados;
151 using librados::IoCtx;
152 @@ -39,6 +40,8 @@ class ObjectCacheStore
154 int lock_cache(std::string vol_name);
156 + void evict_thread_body();
161 @@ -48,21 +51,18 @@ class ObjectCacheStore
162 librados::bufferlist* read_buf,
171 ContextWQ* m_work_queue;
172 - Mutex m_cache_table_lock;
175 - std::map<std::string, uint8_t> m_cache_table;
177 std::map<std::string, librados::IoCtx*> m_ioctxs;
179 librbd::cache::SyncFile *m_cache_file;
187 diff --git a/src/tools/rbd_cache/Policy.hpp b/src/tools/rbd_cache/Policy.hpp
189 index 0000000..575c294
191 +++ b/src/tools/rbd_cache/Policy.hpp
193 +#ifndef RBD_CACHE_POLICY_HPP
194 +#define RBD_CACHE_POLICY_HPP
208 + virtual ~Policy(){};
209 + virtual CACHESTATUS lookup_object(std::string) = 0;
210 + virtual int evict_object(std::string&) = 0;
211 + virtual void update_status(std::string, CACHESTATUS) = 0;
212 + virtual CACHESTATUS get_status(std::string) = 0;
215 diff --git a/src/tools/rbd_cache/SimplePolicy.hpp b/src/tools/rbd_cache/SimplePolicy.hpp
217 index 0000000..a0d8de7
219 +++ b/src/tools/rbd_cache/SimplePolicy.hpp
221 +#ifndef RBD_CACHE_SIMPLE_POLICY_HPP
222 +#define RBD_CACHE_SIMPLE_POLICY_HPP
224 +#include "Policy.hpp"
225 +#include "include/lru.h"
226 +#include "common/Mutex.h"
229 +#include <unordered_map>
232 +class SimplePolicy : public Policy {
234 + SimplePolicy(uint64_t block_num, float level)
236 + m_lock("SimplePolicy"),
237 + m_entry_count(block_num)
240 + Entry m_entries[m_entry_count];
242 + for(auto &entry : m_entries) {
243 + m_free_lru.lru_insert_bot(&entry);
249 + CACHESTATUS lookup_object(std::string cache_file_name) {
250 + Mutex::Locker locker(m_lock);
252 + auto entry_it = m_oid_to_entry.find(cache_file_name);
253 + if(entry_it == m_oid_to_entry.end()) {
257 + Entry* entry = entry_it->second;
260 + if(entry->status == PROMOTED) {
261 + lru = &m_promoted_lru;
263 + lru = &m_handing_lru;
267 + lru->lru_remove(entry);
268 + lru->lru_insert_top(entry);
270 + return entry->status;
273 + int evict_object(std::string& out_cache_file_name) {
274 + Mutex::Locker locker(m_lock);
276 + // still have enough free space, don't need to evict lru.
277 + uint64_t temp_current_size = m_oid_to_entry.size();
278 + float temp_current_evict_level = temp_current_size / m_entry_count;
279 + if(temp_current_evict_level < m_level) {
283 + // when all entries are USING, PROMOTING or EVICTING, just busy waiting.
284 + if(m_promoted_lru.lru_get_size() == 0) {
288 + assert(m_promoted_lru.lru_get_size() != 0);
290 + // evict one item from promoted lru
291 + Entry *entry = reinterpret_cast<Entry*>(m_promoted_lru.lru_get_next_expire());
292 + assert(entry != nullptr);
294 + assert(entry->status == PROMOTED);
296 + out_cache_file_name = entry->cache_file_name;
297 + entry->status = EVICTING;
299 + m_promoted_lru.lru_remove(entry);
300 + m_handing_lru.lru_insert_top(entry);
305 + // TODO(): simplify the logic
306 + void update_status(std::string _file_name, CACHESTATUS _status) {
307 + Mutex::Locker locker(m_lock);
310 + auto entry_it = m_oid_to_entry.find(_file_name);
313 + if(_status == PROMOTING) {
314 + assert(m_oid_to_entry.find(_file_name) == m_oid_to_entry.end());
317 + // miss this object.
318 + if(entry_it == m_oid_to_entry.end() && _status == PROMOTING) {
319 + entry = reinterpret_cast<Entry*>(m_free_lru.lru_get_next_expire());
320 + if(entry == nullptr) {
321 + assert(0); // namely evict thread have some problems.
324 + entry->status = PROMOTING;
326 + m_oid_to_entry[_file_name] = entry;
327 + m_free_lru.lru_remove(entry);
328 + m_handing_lru.lru_insert_top(entry);
333 + assert(entry_it != m_oid_to_entry.end());
335 + entry = entry_it->second;
337 + // promoting action have been finished, so update it.
338 + if(entry->status == PROMOTING && _status== PROMOTED) {
339 + m_handing_lru.lru_remove(entry);
340 + m_promoted_lru.lru_insert_top(entry);
341 + entry->status = PROMOTED;
345 + // will delete this cache file
346 + if(entry->status == PROMOTED && _status == EVICTING) {
347 + m_promoted_lru.lru_remove(entry);
348 + m_handing_lru.lru_insert_top(entry);
349 + entry->status = EVICTING;
354 + if(_status == EVICTED) {
355 + m_oid_to_entry.erase(entry_it);
356 + m_handing_lru.lru_remove(entry);
357 + m_free_lru.lru_insert_bot(entry);
364 + // get entry status
365 + CACHESTATUS get_status(std::string _file_name) {
366 + Mutex::Locker locker(m_lock);
367 + auto entry_it = m_oid_to_entry.find(_file_name);
368 + if(entry_it == m_oid_to_entry.end()) {
372 + return entry_it->second->status;
378 + class Entry : public LRUObject {
380 + CACHESTATUS status;
381 + Entry() : status(NONE){}
382 + std::string cache_file_name;
383 + void encode(bufferlist &bl){}
384 + void decode(bufferlist::iterator &it){}
387 + std::unordered_map<std::string, Entry*> m_oid_to_entry;
390 + LRU m_handing_lru; // include promoting status or evicting status
391 + LRU m_promoted_lru; // include promoted, using status.
393 + mutable Mutex m_lock;
396 + uint64_t m_entry_count;