X-Git-Url: https://gerrit.opnfv.org/gerrit/gitweb?p=stor4nfv.git;a=blobdiff_plain;f=src%2Fceph%2F0007-librbd-cleanup-policy-based-promotion-eviction.patch;fp=src%2Fceph%2F0007-librbd-cleanup-policy-based-promotion-eviction.patch;h=0000000000000000000000000000000000000000;hp=010407b1cce8f7d0538988c7891ea838e6b62186;hb=fe427fc3f69ecc5e104a6fdb2c9f5d82f0254fdb;hpb=79b3874aa314884557bc59983cbdb99f62e77699

diff --git a/src/ceph/0007-librbd-cleanup-policy-based-promotion-eviction.patch b/src/ceph/0007-librbd-cleanup-policy-based-promotion-eviction.patch
deleted file mode 100644
index 010407b..0000000
--- a/src/ceph/0007-librbd-cleanup-policy-based-promotion-eviction.patch
+++ /dev/null
@@ -1,512 +0,0 @@
-From dd4804fb05ad8aca51516b0112975cc91ef85a6b Mon Sep 17 00:00:00 2001
-From: Yuan Zhou <yuan.zhou@intel.com>
-Date: Wed, 8 Aug 2018 15:31:47 +0800
-Subject: [PATCH 07/10] librbd: cleanup policy based promotion/eviction
-
-Signed-off-by: Yuan Zhou <yuan.zhou@intel.com>
----
- src/common/options.cc                              |   4 +
- .../rbd_cache/CacheControllerSocketClient.hpp      |   3 +-
- src/tools/rbd_cache/ObjectCacheStore.cc            |  63 +++----
- src/tools/rbd_cache/ObjectCacheStore.h             |  10 +-
- src/tools/rbd_cache/Policy.hpp                     |  18 +-
- src/tools/rbd_cache/SimplePolicy.hpp               | 188 +++++++++------------
- 6 files changed, 141 insertions(+), 145 deletions(-)
-
-diff --git a/src/common/options.cc b/src/common/options.cc
-index 7839a31..b334c1e 100644
---- a/src/common/options.cc
-+++ b/src/common/options.cc
-@@ -6365,6 +6365,10 @@ static std::vector<Option> get_rbd_options() {
-     .set_default("/tmp")
-     .set_description("shared ssd caching data dir"),
- 
-+    Option("rbd_shared_cache_entries", Option::TYPE_INT, Option::LEVEL_ADVANCED)
-+    .set_default(4096)
-+    .set_description("shared ssd caching data entries"),
-+
-     Option("rbd_non_blocking_aio", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
-     .set_default(true)
-     .set_description("process AIO ops from a dispatch thread to prevent blocking"),
-diff --git a/src/tools/rbd_cache/CacheControllerSocketClient.hpp b/src/tools/rbd_cache/CacheControllerSocketClient.hpp
-index 4e1f36c..56b79ce 100644
---- a/src/tools/rbd_cache/CacheControllerSocketClient.hpp
-+++ b/src/tools/rbd_cache/CacheControllerSocketClient.hpp
-@@ -90,7 +90,8 @@ public:
-         }
-     });
-     std::unique_lock<std::mutex> lk(m);
--    cv.wait(lk);
-+    //cv.wait(lk);
-+    cv.wait_for(lk, std::chrono::milliseconds(100));
-     return 0;
-   }
- 
-diff --git a/src/tools/rbd_cache/ObjectCacheStore.cc b/src/tools/rbd_cache/ObjectCacheStore.cc
-index b39fe66..99f90d6 100644
---- a/src/tools/rbd_cache/ObjectCacheStore.cc
-+++ b/src/tools/rbd_cache/ObjectCacheStore.cc
-@@ -15,7 +15,12 @@ namespace cache {
- ObjectCacheStore::ObjectCacheStore(CephContext *cct, ContextWQ* work_queue)
-       : m_cct(cct), m_work_queue(work_queue),
-         m_rados(new librados::Rados()) {
--  m_policy = new SimplePolicy(4096, 0.9); // TODO
-+
-+  uint64_t object_cache_entries =
-+    cct->_conf.get_val<int64_t>("rbd_shared_cache_entries");
-+
-+  //TODO(): allow to set level
-+  m_policy = new SimplePolicy(object_cache_entries, 0.5);
- }
- 
- ObjectCacheStore::~ObjectCacheStore() {
-@@ -35,7 +40,16 @@ int ObjectCacheStore::init(bool reset) {
-     lderr(m_cct) << "fail to conect to cluster" << dendl;
-     return ret;
-   }
--  //TODO(): check existing cache objects
-+
-+  std::string cache_path = m_cct->_conf.get_val<std::string>("rbd_shared_cache_path");
-+  //TODO(): check and reuse existing cache objects
-+  if(reset) {
-+    std::string cmd = "exec rm -rf " + cache_path + "/rbd_cache*; exec mkdir -p " + cache_path;
-+    //TODO(): to use std::filesystem
-+    int r = system(cmd.c_str());
-+  }
-+
-+  evict_thd = new std::thread([this]{this->evict_thread_body();});
-   return ret;
- }
- 
-@@ -58,10 +72,6 @@ int ObjectCacheStore::do_promote(std::string pool_name, std::string object_name)
-   
-   librados::IoCtx* ioctx = m_ioctxs[pool_name]; 
- 
--  //promoting: update metadata 
--  m_policy->update_status(cache_file_name, PROMOTING);
--  assert(PROMOTING == m_policy->get_status(cache_file_name));
--
-   librados::bufferlist* read_buf = new librados::bufferlist();
-   int object_size = 4096*1024; //TODO(): read config from image metadata
- 
-@@ -83,9 +93,9 @@ int ObjectCacheStore::do_promote(std::string pool_name, std::string object_name)
-   ret = cache_file.write_object_to_file(*read_buf, object_size);
-   
-   // update metadata
--  assert(PROMOTING == m_policy->get_status(cache_file_name));
--  m_policy->update_status(cache_file_name, PROMOTED);
--  assert(PROMOTED == m_policy->get_status(cache_file_name));
-+  assert(OBJ_CACHE_PROMOTING == m_policy->get_status(cache_file_name));
-+  m_policy->update_status(cache_file_name, OBJ_CACHE_PROMOTED);
-+  assert(OBJ_CACHE_PROMOTED == m_policy->get_status(cache_file_name));
- 
-   return ret;
- 
-@@ -97,18 +107,15 @@ int ObjectCacheStore::lookup_object(std::string pool_name, std::string object_na
- 
-   std::string cache_file_name =  pool_name + object_name;
- 
--  // TODO lookup and return status;
--
-   CACHESTATUS ret;
-   ret = m_policy->lookup_object(cache_file_name);
- 
-   switch(ret) {
--    case NONE:
-+    case OBJ_CACHE_NONE:
-       return do_promote(pool_name, object_name);
--    case PROMOTING:
--      return -1;
--    case PROMOTED:
-+    case OBJ_CACHE_PROMOTED:
-       return 0;
-+    case OBJ_CACHE_PROMOTING:
-     default:
-       return -1;
-   }
-@@ -117,26 +124,14 @@ int ObjectCacheStore::lookup_object(std::string pool_name, std::string object_na
- void ObjectCacheStore::evict_thread_body() {
-   int ret;
-   while(m_evict_go) {
--    std::string temp_cache_file;
--
--    ret = m_policy->evict_object(temp_cache_file);
--    if(ret == 0) {
--      continue;
--    }
--
--    // TODO
--    // delete temp_cache_file file.
--
--    assert(EVICTING == m_policy->get_status(temp_cache_file));
--
--    m_policy->update_status(temp_cache_file, EVICTED);
--
--    assert(NONE == m_policy->get_status(temp_cache_file));
-+    ret = evict_objects();
-   }
- }
- 
- 
- int ObjectCacheStore::shutdown() {
-+  m_evict_go = false;
-+  evict_thd->join();
-   m_rados->shutdown();
-   return 0;
- }
-@@ -165,5 +160,13 @@ int ObjectCacheStore::promote_object(librados::IoCtx* ioctx, std::string object_
-   
- }
- 
-+int ObjectCacheStore::evict_objects() {
-+  std::list<std::string> obj_list;
-+  m_policy->get_evict_list(&obj_list);
-+  for (auto& obj: obj_list) {
-+    //do_evict(obj);
-+  }
-+}
-+
- } // namespace cache
- } // namespace rbd
-diff --git a/src/tools/rbd_cache/ObjectCacheStore.h b/src/tools/rbd_cache/ObjectCacheStore.h
-index 5118a73..ba0e1f1 100644
---- a/src/tools/rbd_cache/ObjectCacheStore.h
-+++ b/src/tools/rbd_cache/ObjectCacheStore.h
-@@ -15,6 +15,7 @@
- #include "librbd/cache/SharedPersistentObjectCacherFile.h"
- #include "SimplePolicy.hpp"
- 
-+
- using librados::Rados;
- using librados::IoCtx;
- 
-@@ -40,10 +41,9 @@ class ObjectCacheStore
- 
-     int lock_cache(std::string vol_name);
- 
--    void evict_thread_body();
--
-   private:
--    int _evict_object();
-+    void evict_thread_body();
-+    int evict_objects();
- 
-     int do_promote(std::string pool_name, std::string object_name);
- 
-@@ -61,8 +61,8 @@ class ObjectCacheStore
-     librbd::cache::SyncFile *m_cache_file;
- 
-     Policy* m_policy;
--
--    bool m_evict_go;
-+    std::thread* evict_thd;
-+    bool m_evict_go = false;
- };
- 
- } // namespace rbd
-diff --git a/src/tools/rbd_cache/Policy.hpp b/src/tools/rbd_cache/Policy.hpp
-index 575c294..711e3bd 100644
---- a/src/tools/rbd_cache/Policy.hpp
-+++ b/src/tools/rbd_cache/Policy.hpp
-@@ -1,12 +1,16 @@
- #ifndef RBD_CACHE_POLICY_HPP
- #define RBD_CACHE_POLICY_HPP
- 
-+#include <list>
-+#include <string>
-+
-+namespace rbd {
-+namespace cache {
-+
- enum CACHESTATUS {
--  NONE = 0,
--  PROMOTING,
--  PROMOTED,
--  EVICTING,
--  EVICTED,
-+  OBJ_CACHE_NONE = 0,
-+  OBJ_CACHE_PROMOTING,
-+  OBJ_CACHE_PROMOTED,
- };
- 
- 
-@@ -18,5 +22,9 @@ public:
-   virtual int evict_object(std::string&) = 0;
-   virtual void update_status(std::string, CACHESTATUS) = 0;
-   virtual CACHESTATUS get_status(std::string) = 0;
-+  virtual void get_evict_list(std::list<std::string>* obj_list) = 0;
- };
-+
-+} // namespace cache
-+} // namespace rbd
- #endif
-diff --git a/src/tools/rbd_cache/SimplePolicy.hpp b/src/tools/rbd_cache/SimplePolicy.hpp
-index a0d8de7..e785de1 100644
---- a/src/tools/rbd_cache/SimplePolicy.hpp
-+++ b/src/tools/rbd_cache/SimplePolicy.hpp
-@@ -3,138 +3,93 @@
- 
- #include "Policy.hpp"
- #include "include/lru.h"
-+#include "common/RWLock.h"
- #include "common/Mutex.h"
- 
- #include <vector>
- #include <unordered_map>
- #include <string>
- 
-+namespace rbd {
-+namespace cache {
-+
-+
- class SimplePolicy : public Policy {
- public:
--  SimplePolicy(uint64_t block_num, float level)
--    : m_level(level),
--      m_lock("SimplePolicy"),
--      m_entry_count(block_num)
-+  SimplePolicy(uint64_t block_num, float watermark)
-+    : m_watermark(watermark), m_entry_count(block_num),
-+      m_cache_map_lock("rbd::cache::SimplePolicy::m_cache_map_lock"),
-+      m_free_list_lock("rbd::cache::SimplePolicy::m_free_list_lock")
-   {
- 
--    Entry m_entries[m_entry_count];
--
--    for(auto &entry : m_entries) {
--      m_free_lru.lru_insert_bot(&entry);
-+    for(uint64_t i = 0; i < m_entry_count; i++) {
-+      m_free_list.push_back(new Entry());
-     }
-+
-   }
- 
--  ~SimplePolicy() {}
-+  ~SimplePolicy() {
-+    for(uint64_t i = 0; i < m_entry_count; i++) {
-+      Entry* entry = reinterpret_cast<Entry*>(m_free_list.front());
-+      delete entry;
-+      m_free_list.pop_front();
-+    }
-+  }
- 
-   CACHESTATUS lookup_object(std::string cache_file_name) {
--    Mutex::Locker locker(m_lock);
- 
--    auto entry_it = m_oid_to_entry.find(cache_file_name);
--    if(entry_it == m_oid_to_entry.end()) {
--      return NONE;
-+    //TODO(): check race condition
-+    RWLock::WLocker wlocker(m_cache_map_lock);
-+
-+    auto entry_it = m_cache_map.find(cache_file_name);
-+    if(entry_it == m_cache_map.end()) {
-+      Mutex::Locker locker(m_free_list_lock);
-+      Entry* entry = reinterpret_cast<Entry*>(m_free_list.front());
-+      assert(entry != nullptr);
-+      m_free_list.pop_front();
-+      entry->status = OBJ_CACHE_PROMOTING;
-+
-+      m_cache_map[cache_file_name] = entry;
-+
-+      return OBJ_CACHE_NONE;
-     }
- 
-     Entry* entry = entry_it->second;
- 
--    LRU* lru;
--    if(entry->status == PROMOTED) {
--      lru = &m_promoted_lru;
--    } else {
--      lru = &m_handing_lru;
-+    if(entry->status == OBJ_CACHE_PROMOTED) {
-+      // touch it
-+      m_promoted_lru.lru_touch(entry);
-     }
- 
--    // touch it
--    lru->lru_remove(entry);
--    lru->lru_insert_top(entry);
--
-     return entry->status;
-   }
- 
-   int evict_object(std::string& out_cache_file_name) {
--    Mutex::Locker locker(m_lock);
--
--    // still have enough free space, don't need to evict lru.
--    uint64_t temp_current_size = m_oid_to_entry.size();
--    float temp_current_evict_level = temp_current_size / m_entry_count;
--    if(temp_current_evict_level < m_level) {
--      return 0;
--    }
--
--    // when all entries are USING, PROMOTING or EVICTING, just busy waiting.
--    if(m_promoted_lru.lru_get_size() == 0) {
--      return 0;
--    }
--
--    assert(m_promoted_lru.lru_get_size() != 0);
--
--    // evict one item from promoted lru
--    Entry *entry = reinterpret_cast<Entry*>(m_promoted_lru.lru_get_next_expire());
--    assert(entry != nullptr);
--
--    assert(entry->status == PROMOTED);
--
--    out_cache_file_name = entry->cache_file_name;
--    entry->status = EVICTING;
--
--    m_promoted_lru.lru_remove(entry);
--    m_handing_lru.lru_insert_top(entry);
-+    RWLock::WLocker locker(m_cache_map_lock);
- 
-     return 1;
-   }
- 
-   // TODO(): simplify the logic
--  void update_status(std::string _file_name, CACHESTATUS _status) {
--    Mutex::Locker locker(m_lock);
-+  void update_status(std::string file_name, CACHESTATUS new_status) {
-+    RWLock::WLocker locker(m_cache_map_lock);
- 
-     Entry* entry;
--    auto entry_it = m_oid_to_entry.find(_file_name);
-+    auto entry_it = m_cache_map.find(file_name);
- 
-     // just check.
--    if(_status == PROMOTING) {
--      assert(m_oid_to_entry.find(_file_name) == m_oid_to_entry.end());
-+    if(new_status == OBJ_CACHE_PROMOTING) {
-+      assert(entry_it == m_cache_map.end());
-     }
- 
--    // miss this object.
--    if(entry_it == m_oid_to_entry.end() && _status == PROMOTING) {
--      entry = reinterpret_cast<Entry*>(m_free_lru.lru_get_next_expire());
--      if(entry == nullptr) {
--        assert(0); // namely evict thread have some problems.
--      }
--
--      entry->status = PROMOTING;
--
--      m_oid_to_entry[_file_name] = entry;
--      m_free_lru.lru_remove(entry);
--      m_handing_lru.lru_insert_top(entry);
--
--      return;
--    }
--
--    assert(entry_it != m_oid_to_entry.end());
-+    assert(entry_it != m_cache_map.end());
- 
-     entry = entry_it->second;
- 
--    // promoting action have been finished, so update it.
--    if(entry->status == PROMOTING && _status== PROMOTED) {
--      m_handing_lru.lru_remove(entry);
-+    // promoting is done, so update it.
-+    if(entry->status == OBJ_CACHE_PROMOTING && new_status== OBJ_CACHE_PROMOTED) {
-       m_promoted_lru.lru_insert_top(entry);
--      entry->status = PROMOTED;
--      return;
--    }
--
--    // will delete this cache file
--    if(entry->status == PROMOTED && _status == EVICTING) {
--      m_promoted_lru.lru_remove(entry);
--      m_handing_lru.lru_insert_top(entry);
--      entry->status = EVICTING;
--      return;
--    }
--
--
--    if(_status == EVICTED) {
--      m_oid_to_entry.erase(entry_it);
--      m_handing_lru.lru_remove(entry);
--      m_free_lru.lru_insert_bot(entry);
-+      entry->status = new_status;
-       return;
-     }
- 
-@@ -142,39 +97,64 @@ public:
-   }
- 
-   // get entry status
--  CACHESTATUS get_status(std::string _file_name) {
--    Mutex::Locker locker(m_lock);
--    auto entry_it = m_oid_to_entry.find(_file_name);
--    if(entry_it == m_oid_to_entry.end()) {
--      return NONE;
-+  CACHESTATUS get_status(std::string file_name) {
-+    RWLock::RLocker locker(m_cache_map_lock);
-+    auto entry_it = m_cache_map.find(file_name);
-+    if(entry_it == m_cache_map.end()) {
-+      return OBJ_CACHE_NONE;
-     }
- 
-     return entry_it->second->status;
-   }
- 
-+  void get_evict_list(std::list<std::string>* obj_list) {
-+    RWLock::WLocker locker(m_cache_map_lock);
-+    // check free ratio, pop entries from LRU
-+    if (m_free_list.size() / m_entry_count < m_watermark) {
-+      int evict_num = 10; //TODO(): make this configurable
-+      for(int i = 0; i < evict_num; i++) {
-+        Entry* entry = reinterpret_cast<Entry*>(m_promoted_lru.lru_expire());
-+        if (entry == nullptr) {
-+	  continue;
-+        }
-+        std::string file_name = entry->cache_file_name;
-+        obj_list->push_back(file_name);
-+
-+        auto entry_it = m_cache_map.find(file_name);
-+        m_cache_map.erase(entry_it);
-+
-+        //mark this entry as free
-+	entry->status = OBJ_CACHE_NONE;
-+        Mutex::Locker locker(m_free_list_lock);
-+        m_free_list.push_back(entry);
-+      }
-+   }
-+  }
- 
- private:
- 
-   class Entry : public LRUObject {
-     public:
-       CACHESTATUS status;
--      Entry() : status(NONE){}
-+      Entry() : status(OBJ_CACHE_NONE){}
-       std::string cache_file_name;
-       void encode(bufferlist &bl){}
-       void decode(bufferlist::iterator &it){}
-   };
- 
--  std::unordered_map<std::string, Entry*> m_oid_to_entry;
-+  float m_watermark;
-+  uint64_t m_entry_count;
- 
--  LRU m_free_lru;
--  LRU m_handing_lru; // include promoting status or evicting status
--  LRU m_promoted_lru; // include promoted, using status.
-+  std::unordered_map<std::string, Entry*> m_cache_map;
-+  RWLock m_cache_map_lock;
- 
--  mutable Mutex m_lock;
-+  std::deque<Entry*> m_free_list;
-+  Mutex m_free_list_lock;
- 
--  float m_level;
--  uint64_t m_entry_count;
-+  LRU m_promoted_lru; // include promoted, using status.
- 
- };
- 
-+} // namespace cache
-+} // namespace rbd
- #endif
--- 
-2.7.4
-