1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
2 // vim: ts=8 sw=2 smarttab
4 * Ceph - scalable distributed file system
6 * Copyright (C) 2016 Allen Samuels <allen.samuels@sandisk.com>
8 * This is free software; you can redistribute it and/or
9 * modify it under the terms of the GNU Lesser General Public
10 * License version 2.1, as published by the Free Software
11 * Foundation. See file COPYING.
15 #ifndef _CEPH_INCLUDE_MEMPOOL_H
16 #define _CEPH_INCLUDE_MEMPOOL_H
20 #include <unordered_map>
28 #include <common/Formatter.h>
29 #include "include/assert.h"
37 A memory pool is a method for accounting the consumption of memory of
40 Memory pools are statically declared (see pool_index_t).
42 Each memory pool tracks the number of bytes and items it contains.
44 Allocators can be declared and associated with a type so that they are
45 tracked independently of the pool total. This additional accounting
46 is optional and only incurs an overhead if the debugging is enabled at
47 runtime. This allows developers to see what types are consuming the
54 Using memory pools is very easy.
56 To create a new memory pool, simply add a new name into the list of
57 memory pools that's defined in "DEFINE_MEMORY_POOLS_HELPER". That's
60 For each memory pool that's created a C++ namespace is also
61 automatically created (name is same as in DEFINE_MEMORY_POOLS_HELPER).
62 That namespace contains a set of common STL containers that are predefined
63 with the appropriate allocators.
65 Thus for mempool "osd" we have automatically available to us:
68 mempool::osd::multimap
70 mempool::osd::multiset
73 mempool::osd::unordered_map
76 Putting objects in a mempool
77 ----------------------------
79 In order to use a memory pool with a particular type, a few additional
80 declarations are needed.
85 MEMPOOL_CLASS_HELPERS();
89 Then, in an appropriate .cc file,
91 MEMPOOL_DEFINE_OBJECT_FACTORY(Foo, foo, osd);
93 The second argument can generally be identical to the first, except
94 when the type contains a nested scope. For example, for
95 BlueStore::Onode, we need to do
97 MEMPOOL_DEFINE_OBJECT_FACTORY(BlueStore::Onode, bluestore_onode,
100 (This is just because we need to name some static variables and we
101 can't use :: in a variable name.)
103 XXX Note: the new operator hard-codes the allocation size to the size of the
104 object given in MEMPOOL_DEFINE_OBJECT_FACTORY. For this reason, you cannot
105 incorporate mempools into a base class without also defining a helper/factory
106 for the child class as well (as the base class is usually smaller than the
109 In order to use the STL containers, simply use the namespaced variant
110 of the container type. For example,
112 mempool::osd::map<int> myvec;
117 The simplest way to interrogate the process is with
122 This will dump information about *all* memory pools. When debug mode
123 is enabled, the runtime complexity of dump is O(num_shards *
124 num_types). When debug name is disabled it is O(num_shards).
126 You can also interrogate a specific pool programmatically with
128 size_t bytes = mempool::unittest_2::allocated_bytes();
129 size_t items = mempool::unittest_2::allocated_items();
131 The runtime complexity is O(num_shards).
133 Note that you cannot easily query per-type, primarily because debug
134 mode is optional and you should not rely on that information being
141 // --------------------------------------------------------------
142 // define memory pools
144 #define DEFINE_MEMORY_POOLS_HELPER(f) \
147 f(bluestore_cache_data) \
148 f(bluestore_cache_onode) \
149 f(bluestore_cache_other) \
152 f(bluestore_writing_deferred) \
153 f(bluestore_writing) \
168 // give them integer ids
169 #define P(x) mempool_##x,
171 DEFINE_MEMORY_POOLS_HELPER(P)
172 num_pools // Must be last.
176 extern bool debug_mode;
177 extern void set_debug_mode(bool d);
179 // --------------------------------------------------------------
182 // we shard pool stats across many shard_t's to reduce the amount
183 // of cacheline ping pong.
188 num_shards = 1 << num_shard_bits
191 // align shard to a cacheline
193 std::atomic<size_t> bytes = {0};
194 std::atomic<size_t> items = {0};
195 char __padding[128 - sizeof(std::atomic<size_t>)*2];
196 } __attribute__ ((aligned (128)));
198 static_assert(sizeof(shard_t) == 128, "shard_t should be cacheline-sized");
203 void dump(ceph::Formatter *f) const {
204 f->dump_int("items", items);
205 f->dump_int("bytes", bytes);
208 stats_t& operator+=(const stats_t& o) {
215 pool_t& get_pool(pool_index_t ix);
216 const char *get_pool_name(pool_index_t ix);
219 const char *type_name;
221 std::atomic<ssize_t> items = {0}; // signed
224 struct type_info_hash {
225 std::size_t operator()(const std::type_info& k) const {
226 return k.hash_code();
231 shard_t shard[num_shards];
233 mutable std::mutex lock; // only used for types list
234 std::unordered_map<const char *, type_t> type_map;
238 // How much this pool consumes. O(<num_shards>)
240 size_t allocated_bytes() const;
241 size_t allocated_items() const;
243 void adjust_count(ssize_t items, ssize_t bytes);
245 shard_t* pick_a_shard() {
247 // http://fossies.org/dox/glibc-2.24/pthread__self_8c_source.html
248 size_t me = (size_t)pthread_self();
249 size_t i = (me >> 3) & ((1 << num_shard_bits) - 1);
253 type_t *get_type(const std::type_info& ti, size_t size) {
254 std::lock_guard<std::mutex> l(lock);
255 auto p = type_map.find(ti.name());
256 if (p != type_map.end()) {
259 type_t &t = type_map[ti.name()];
260 t.type_name = ti.name();
265 // get pool stats. by_type is not populated if !debug
266 void get_stats(stats_t *total,
267 std::map<std::string, stats_t> *by_type) const;
269 void dump(ceph::Formatter *f, stats_t *ptotal=0) const;
272 void dump(ceph::Formatter *f);
275 // STL allocator for use with containers. All actual state
276 // is stored in the static pool_allocator_base_t, which saves us from
277 // passing the allocator to container constructors.
279 template<pool_index_t pool_ix, typename T>
280 class pool_allocator {
282 type_t *type = nullptr;
285 typedef pool_allocator<pool_ix, T> allocator_type;
286 typedef T value_type;
287 typedef value_type *pointer;
288 typedef const value_type * const_pointer;
289 typedef value_type& reference;
290 typedef const value_type& const_reference;
291 typedef std::size_t size_type;
292 typedef std::ptrdiff_t difference_type;
294 template<typename U> struct rebind {
295 typedef pool_allocator<pool_ix,U> other;
298 void init(bool force_register) {
299 pool = &get_pool(pool_ix);
300 if (debug_mode || force_register) {
301 type = pool->get_type(typeid(T), sizeof(T));
305 pool_allocator(bool force_register=false) {
306 init(force_register);
309 pool_allocator(const pool_allocator<pool_ix,U>&) {
313 T* allocate(size_t n, void *p = nullptr) {
314 size_t total = sizeof(T) * n;
315 shard_t *shard = pool->pick_a_shard();
316 shard->bytes += total;
321 T* r = reinterpret_cast<T*>(new char[total]);
325 void deallocate(T* p, size_t n) {
326 size_t total = sizeof(T) * n;
327 shard_t *shard = pool->pick_a_shard();
328 shard->bytes -= total;
333 delete[] reinterpret_cast<char*>(p);
336 T* allocate_aligned(size_t n, size_t align, void *p = nullptr) {
337 size_t total = sizeof(T) * n;
338 shard_t *shard = pool->pick_a_shard();
339 shard->bytes += total;
345 int rc = ::posix_memalign((void**)(void*)&ptr, align, total);
347 throw std::bad_alloc();
348 T* r = reinterpret_cast<T*>(ptr);
352 void deallocate_aligned(T* p, size_t n) {
353 size_t total = sizeof(T) * n;
354 shard_t *shard = pool->pick_a_shard();
355 shard->bytes -= total;
372 void construct(T* p, const T& val) {
373 ::new ((void *)p) T(val);
376 template<class U, class... Args> void construct(U* p,Args&&... args) {
377 ::new((void *)p) U(std::forward<Args>(args)...);
380 bool operator==(const pool_allocator&) const { return true; }
381 bool operator!=(const pool_allocator&) const { return false; }
389 static const mempool::pool_index_t id = mempool::mempool_##x; \
390 template<typename v> \
391 using pool_allocator = mempool::pool_allocator<id,v>; \
393 using string = std::basic_string<char,std::char_traits<char>, \
394 pool_allocator<char>>; \
396 template<typename k,typename v, typename cmp = std::less<k> > \
397 using map = std::map<k, v, cmp, \
398 pool_allocator<std::pair<const k,v>>>; \
400 template<typename k,typename v, typename cmp = std::less<k> > \
401 using multimap = std::multimap<k,v,cmp, \
402 pool_allocator<std::pair<const k, \
405 template<typename k, typename cmp = std::less<k> > \
406 using set = std::set<k,cmp,pool_allocator<k>>; \
408 template<typename v> \
409 using list = std::list<v,pool_allocator<v>>; \
411 template<typename v> \
412 using vector = std::vector<v,pool_allocator<v>>; \
414 template<typename k, typename v, \
415 typename h=std::hash<k>, \
416 typename eq = std::equal_to<k>> \
417 using unordered_map = \
418 std::unordered_map<k,v,h,eq,pool_allocator<std::pair<const k,v>>>;\
420 inline size_t allocated_bytes() { \
421 return mempool::get_pool(id).allocated_bytes(); \
423 inline size_t allocated_items() { \
424 return mempool::get_pool(id).allocated_items(); \
428 DEFINE_MEMORY_POOLS_HELPER(P)
436 // Use this for any type that is contained by a container (unless it
437 // is a class you defined; see below).
438 #define MEMPOOL_DECLARE_FACTORY(obj, factoryname, pool) \
439 namespace mempool { \
441 extern pool_allocator<obj> alloc_##factoryname; \
445 #define MEMPOOL_DEFINE_FACTORY(obj, factoryname, pool) \
446 namespace mempool { \
448 pool_allocator<obj> alloc_##factoryname = {true}; \
452 // Use this for each class that belongs to a mempool. For example,
455 // MEMPOOL_CLASS_HELPERS();
459 #define MEMPOOL_CLASS_HELPERS() \
460 void *operator new(size_t size); \
461 void *operator new[](size_t size) noexcept { \
462 assert(0 == "no array new"); \
464 void operator delete(void *); \
465 void operator delete[](void *) { assert(0 == "no array delete"); }
468 // Use this in some particular .cc file to match each class with a
469 // MEMPOOL_CLASS_HELPERS().
470 #define MEMPOOL_DEFINE_OBJECT_FACTORY(obj,factoryname,pool) \
471 MEMPOOL_DEFINE_FACTORY(obj, factoryname, pool) \
472 void *obj::operator new(size_t size) { \
473 return mempool::pool::alloc_##factoryname.allocate(1); \
475 void obj::operator delete(void *p) { \
476 return mempool::pool::alloc_##factoryname.deallocate((obj*)p, 1); \