--- /dev/null
+From b233d6540160c8bc5cc25b870c2140fa48776fa6 Mon Sep 17 00:00:00 2001
+From: Dehao Shang <dehao.shang@intel.com>
+Date: Mon, 6 Aug 2018 22:42:38 +0800
+Subject: [PATCH 06/10] librbd: LRU policy based eviction
+
+Signed-off-by: Dehao Shang <dehao.shang@intel.com>
+Signed-off-by: Yuan Zhou <yuan.zhou@intel.com>
+---
+ src/tools/rbd_cache/ObjectCacheStore.cc | 73 ++++++++-----
+ src/tools/rbd_cache/ObjectCacheStore.h | 14 +--
+ src/tools/rbd_cache/Policy.hpp | 22 ++++
+ src/tools/rbd_cache/SimplePolicy.hpp | 180 ++++++++++++++++++++++++++++++++
+ 4 files changed, 254 insertions(+), 35 deletions(-)
+ create mode 100644 src/tools/rbd_cache/Policy.hpp
+ create mode 100644 src/tools/rbd_cache/SimplePolicy.hpp
+
+diff --git a/src/tools/rbd_cache/ObjectCacheStore.cc b/src/tools/rbd_cache/ObjectCacheStore.cc
+index 2a87469..b39fe66 100644
+--- a/src/tools/rbd_cache/ObjectCacheStore.cc
++++ b/src/tools/rbd_cache/ObjectCacheStore.cc
+@@ -14,12 +14,12 @@ namespace cache {
+
+ ObjectCacheStore::ObjectCacheStore(CephContext *cct, ContextWQ* work_queue)
+ : m_cct(cct), m_work_queue(work_queue),
+- m_cache_table_lock("rbd::cache::ObjectCacheStore"),
+ m_rados(new librados::Rados()) {
++ m_policy = new SimplePolicy(4096, 0.9); // TODO
+ }
+
+ ObjectCacheStore::~ObjectCacheStore() {
+-
++ delete m_policy;
+ }
+
+ int ObjectCacheStore::init(bool reset) {
+@@ -43,6 +43,7 @@ int ObjectCacheStore::do_promote(std::string pool_name, std::string object_name)
+ int ret = 0;
+ std::string cache_file_name = pool_name + object_name;
+
++ //TODO(): lock on ioctx map
+ if (m_ioctxs.find(pool_name) == m_ioctxs.end()) {
+ librados::IoCtx* io_ctx = new librados::IoCtx();
+ ret = m_rados->ioctx_create(pool_name.c_str(), *io_ctx);
+@@ -58,10 +59,8 @@ int ObjectCacheStore::do_promote(std::string pool_name, std::string object_name)
+ librados::IoCtx* ioctx = m_ioctxs[pool_name];
+
+ //promoting: update metadata
+- {
+- Mutex::Locker locker(m_cache_table_lock);
+- m_cache_table.emplace(cache_file_name, PROMOTING);
+- }
++ m_policy->update_status(cache_file_name, PROMOTING);
++ assert(PROMOTING == m_policy->get_status(cache_file_name));
+
+ librados::bufferlist* read_buf = new librados::bufferlist();
+ int object_size = 4096*1024; //TODO(): read config from image metadata
+@@ -83,42 +82,60 @@ int ObjectCacheStore::do_promote(std::string pool_name, std::string object_name)
+ cache_file.open();
+ ret = cache_file.write_object_to_file(*read_buf, object_size);
+
+- assert(m_cache_table.find(cache_file_name) != m_cache_table.end());
+-
+ // update metadata
+- {
+- Mutex::Locker locker(m_cache_table_lock);
+- m_cache_table.emplace(cache_file_name, PROMOTED);
+- }
++ assert(PROMOTING == m_policy->get_status(cache_file_name));
++ m_policy->update_status(cache_file_name, PROMOTED);
++ assert(PROMOTED == m_policy->get_status(cache_file_name));
+
+ return ret;
+
+ }
+
++// return -1, client need to read data from cluster.
++// return 0, client directly read data from cache.
+ int ObjectCacheStore::lookup_object(std::string pool_name, std::string object_name) {
+
+ std::string cache_file_name = pool_name + object_name;
+- {
+- Mutex::Locker locker(m_cache_table_lock);
+-
+- auto it = m_cache_table.find(cache_file_name);
+- if (it != m_cache_table.end()) {
+-
+- if (it->second == PROMOTING) {
+- return -1;
+- } else if (it->second == PROMOTED) {
+- return 0;
+- } else {
+- assert(0);
+- }
+- }
++
++ // TODO lookup and return status;
++
++ CACHESTATUS ret;
++ ret = m_policy->lookup_object(cache_file_name);
++
++ switch(ret) {
++ case NONE:
++ return do_promote(pool_name, object_name);
++ case PROMOTING:
++ return -1;
++ case PROMOTED:
++ return 0;
++ default:
++ return -1;
+ }
++}
+
+- int ret = do_promote(pool_name, object_name);
++void ObjectCacheStore::evict_thread_body() {
++ int ret;
++ while(m_evict_go) {
++ std::string temp_cache_file;
+
+- return ret;
++ ret = m_policy->evict_object(temp_cache_file);
++ if(ret == 0) {
++ continue;
++ }
++
++ // TODO
++ // delete temp_cache_file file.
++
++ assert(EVICTING == m_policy->get_status(temp_cache_file));
++
++ m_policy->update_status(temp_cache_file, EVICTED);
++
++ assert(NONE == m_policy->get_status(temp_cache_file));
++ }
+ }
+
++
+ int ObjectCacheStore::shutdown() {
+ m_rados->shutdown();
+ return 0;
+diff --git a/src/tools/rbd_cache/ObjectCacheStore.h b/src/tools/rbd_cache/ObjectCacheStore.h
+index db09efa..5118a73 100644
+--- a/src/tools/rbd_cache/ObjectCacheStore.h
++++ b/src/tools/rbd_cache/ObjectCacheStore.h
+@@ -13,6 +13,7 @@
+ #include "librbd/ImageCtx.h"
+ #include "librbd/ImageState.h"
+ #include "librbd/cache/SharedPersistentObjectCacherFile.h"
++#include "SimplePolicy.hpp"
+
+ using librados::Rados;
+ using librados::IoCtx;
+@@ -39,6 +40,8 @@ class ObjectCacheStore
+
+ int lock_cache(std::string vol_name);
+
++ void evict_thread_body();
++
+ private:
+ int _evict_object();
+
+@@ -48,21 +51,18 @@ class ObjectCacheStore
+ librados::bufferlist* read_buf,
+ uint64_t length);
+
+- enum {
+- PROMOTING = 0,
+- PROMOTED,
+- };
+-
+ CephContext *m_cct;
+ ContextWQ* m_work_queue;
+- Mutex m_cache_table_lock;
+ RadosRef m_rados;
+
+- std::map<std::string, uint8_t> m_cache_table;
+
+ std::map<std::string, librados::IoCtx*> m_ioctxs;
+
+ librbd::cache::SyncFile *m_cache_file;
++
++ Policy* m_policy;
++
++ bool m_evict_go;
+ };
+
+ } // namespace rbd
+diff --git a/src/tools/rbd_cache/Policy.hpp b/src/tools/rbd_cache/Policy.hpp
+new file mode 100644
+index 0000000..575c294
+--- /dev/null
++++ b/src/tools/rbd_cache/Policy.hpp
+@@ -0,0 +1,22 @@
++#ifndef RBD_CACHE_POLICY_HPP
++#define RBD_CACHE_POLICY_HPP
++
++enum CACHESTATUS {
++ NONE = 0,
++ PROMOTING,
++ PROMOTED,
++ EVICTING,
++ EVICTED,
++};
++
++
++class Policy {
++public:
++ Policy(){}
++ virtual ~Policy(){};
++ virtual CACHESTATUS lookup_object(std::string) = 0;
++ virtual int evict_object(std::string&) = 0;
++ virtual void update_status(std::string, CACHESTATUS) = 0;
++ virtual CACHESTATUS get_status(std::string) = 0;
++};
++#endif
+diff --git a/src/tools/rbd_cache/SimplePolicy.hpp b/src/tools/rbd_cache/SimplePolicy.hpp
+new file mode 100644
+index 0000000..a0d8de7
+--- /dev/null
++++ b/src/tools/rbd_cache/SimplePolicy.hpp
+@@ -0,0 +1,180 @@
++#ifndef RBD_CACHE_SIMPLE_POLICY_HPP
++#define RBD_CACHE_SIMPLE_POLICY_HPP
++
++#include "Policy.hpp"
++#include "include/lru.h"
++#include "common/Mutex.h"
++
++#include <vector>
++#include <unordered_map>
++#include <string>
++
++class SimplePolicy : public Policy {
++public:
++ SimplePolicy(uint64_t block_num, float level)
++ : m_level(level),
++ m_lock("SimplePolicy"),
++ m_entry_count(block_num)
++ {
++
++ Entry m_entries[m_entry_count];
++
++ for(auto &entry : m_entries) {
++ m_free_lru.lru_insert_bot(&entry);
++ }
++ }
++
++ ~SimplePolicy() {}
++
++ CACHESTATUS lookup_object(std::string cache_file_name) {
++ Mutex::Locker locker(m_lock);
++
++ auto entry_it = m_oid_to_entry.find(cache_file_name);
++ if(entry_it == m_oid_to_entry.end()) {
++ return NONE;
++ }
++
++ Entry* entry = entry_it->second;
++
++ LRU* lru;
++ if(entry->status == PROMOTED) {
++ lru = &m_promoted_lru;
++ } else {
++ lru = &m_handing_lru;
++ }
++
++ // touch it
++ lru->lru_remove(entry);
++ lru->lru_insert_top(entry);
++
++ return entry->status;
++ }
++
++ int evict_object(std::string& out_cache_file_name) {
++ Mutex::Locker locker(m_lock);
++
++ // still have enough free space, don't need to evict lru.
++ uint64_t temp_current_size = m_oid_to_entry.size();
++ float temp_current_evict_level = temp_current_size / m_entry_count;
++ if(temp_current_evict_level < m_level) {
++ return 0;
++ }
++
++ // when all entries are USING, PROMOTING or EVICTING, just busy waiting.
++ if(m_promoted_lru.lru_get_size() == 0) {
++ return 0;
++ }
++
++ assert(m_promoted_lru.lru_get_size() != 0);
++
++ // evict one item from promoted lru
++ Entry *entry = reinterpret_cast<Entry*>(m_promoted_lru.lru_get_next_expire());
++ assert(entry != nullptr);
++
++ assert(entry->status == PROMOTED);
++
++ out_cache_file_name = entry->cache_file_name;
++ entry->status = EVICTING;
++
++ m_promoted_lru.lru_remove(entry);
++ m_handing_lru.lru_insert_top(entry);
++
++ return 1;
++ }
++
++ // TODO(): simplify the logic
++ void update_status(std::string _file_name, CACHESTATUS _status) {
++ Mutex::Locker locker(m_lock);
++
++ Entry* entry;
++ auto entry_it = m_oid_to_entry.find(_file_name);
++
++ // just check.
++ if(_status == PROMOTING) {
++ assert(m_oid_to_entry.find(_file_name) == m_oid_to_entry.end());
++ }
++
++ // miss this object.
++ if(entry_it == m_oid_to_entry.end() && _status == PROMOTING) {
++ entry = reinterpret_cast<Entry*>(m_free_lru.lru_get_next_expire());
++ if(entry == nullptr) {
++ assert(0); // namely evict thread have some problems.
++ }
++
++ entry->status = PROMOTING;
++
++ m_oid_to_entry[_file_name] = entry;
++ m_free_lru.lru_remove(entry);
++ m_handing_lru.lru_insert_top(entry);
++
++ return;
++ }
++
++ assert(entry_it != m_oid_to_entry.end());
++
++ entry = entry_it->second;
++
++ // promoting action have been finished, so update it.
++ if(entry->status == PROMOTING && _status== PROMOTED) {
++ m_handing_lru.lru_remove(entry);
++ m_promoted_lru.lru_insert_top(entry);
++ entry->status = PROMOTED;
++ return;
++ }
++
++ // will delete this cache file
++ if(entry->status == PROMOTED && _status == EVICTING) {
++ m_promoted_lru.lru_remove(entry);
++ m_handing_lru.lru_insert_top(entry);
++ entry->status = EVICTING;
++ return;
++ }
++
++
++ if(_status == EVICTED) {
++ m_oid_to_entry.erase(entry_it);
++ m_handing_lru.lru_remove(entry);
++ m_free_lru.lru_insert_bot(entry);
++ return;
++ }
++
++ assert(0);
++ }
++
++ // get entry status
++ CACHESTATUS get_status(std::string _file_name) {
++ Mutex::Locker locker(m_lock);
++ auto entry_it = m_oid_to_entry.find(_file_name);
++ if(entry_it == m_oid_to_entry.end()) {
++ return NONE;
++ }
++
++ return entry_it->second->status;
++ }
++
++
++private:
++
++ class Entry : public LRUObject {
++ public:
++ CACHESTATUS status;
++ Entry() : status(NONE){}
++ std::string cache_file_name;
++ void encode(bufferlist &bl){}
++ void decode(bufferlist::iterator &it){}
++ };
++
++ std::unordered_map<std::string, Entry*> m_oid_to_entry;
++
++ LRU m_free_lru;
++ LRU m_handing_lru; // include promoting status or evicting status
++ LRU m_promoted_lru; // include promoted, using status.
++
++ mutable Mutex m_lock;
++
++ float m_level;
++ uint64_t m_entry_count;
++
++};
++
++#endif
+--
+2.7.4
+
Code Review / stor4nfv.git / blobdiff