--- /dev/null
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil <sage@newdream.net>
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation. See file COPYING.
+ *
+ */
+
+#include <atomic>
+#include <errno.h>
+#include <limits.h>
+
+#include <sys/uio.h>
+
+#include "include/compat.h"
+#include "include/mempool.h"
+#include "armor.h"
+#include "common/environment.h"
+#include "common/errno.h"
+#include "common/safe_io.h"
+#include "common/simple_spin.h"
+#include "common/strtol.h"
+#include "common/likely.h"
+#include "common/valgrind.h"
+#include "common/deleter.h"
+#include "common/RWLock.h"
+#include "include/types.h"
+#include "include/scope_guard.h"
+
+#if defined(HAVE_XIO)
+#include "msg/xio/XioMsg.h"
+#endif
+
+using namespace ceph;
+
+#define CEPH_BUFFER_ALLOC_UNIT (MIN(CEPH_PAGE_SIZE, 4096))
+#define CEPH_BUFFER_APPEND_SIZE (CEPH_BUFFER_ALLOC_UNIT - sizeof(raw_combined))
+
+#ifdef BUFFER_DEBUG
+static std::atomic_flag buffer_debug_lock = ATOMIC_FLAG_INIT;
+# define bdout { simple_spin_lock(&buffer_debug_lock); std::cout
+# define bendl std::endl; simple_spin_unlock(&buffer_debug_lock); }
+#else
+# define bdout if (0) { std::cout
+# define bendl std::endl; }
+#endif
+
+ static std::atomic<uint64_t> buffer_total_alloc { 0 };
+ static std::atomic<uint64_t> buffer_history_alloc_bytes { 0 };
+ static std::atomic<uint64_t> buffer_history_alloc_num { 0 };
+
+ const bool buffer_track_alloc = get_env_bool("CEPH_BUFFER_TRACK");
+
+ namespace {
+ void inc_total_alloc(unsigned len) {
+ if (buffer_track_alloc)
+ buffer_total_alloc += len;
+ }
+
+ void dec_total_alloc(unsigned len) {
+ if (buffer_track_alloc)
+ buffer_total_alloc -= len;
+ }
+
+ void inc_history_alloc(uint64_t len) {
+ if (buffer_track_alloc) {
+ buffer_history_alloc_bytes += len;
+ buffer_history_alloc_num++;
+ }
+ }
+ } // namespace
+
+ int buffer::get_total_alloc() {
+ return buffer_total_alloc;
+ }
+ uint64_t buffer::get_history_alloc_bytes() {
+ return buffer_history_alloc_bytes;
+ }
+ uint64_t buffer::get_history_alloc_num() {
+ return buffer_history_alloc_num;
+ }
+
+ static std::atomic<unsigned> buffer_cached_crc { 0 };
+ static std::atomic<unsigned> buffer_cached_crc_adjusted { 0 };
+ static std::atomic<unsigned> buffer_missed_crc { 0 };
+
+ static bool buffer_track_crc = get_env_bool("CEPH_BUFFER_TRACK");
+
+ void buffer::track_cached_crc(bool b) {
+ buffer_track_crc = b;
+ }
+ int buffer::get_cached_crc() {
+ return buffer_cached_crc;
+ }
+ int buffer::get_cached_crc_adjusted() {
+ return buffer_cached_crc_adjusted;
+ }
+
+ int buffer::get_missed_crc() {
+ return buffer_missed_crc;
+ }
+
+ static std::atomic<unsigned> buffer_c_str_accesses { 0 };
+
+ static bool buffer_track_c_str = get_env_bool("CEPH_BUFFER_TRACK");
+
+ void buffer::track_c_str(bool b) {
+ buffer_track_c_str = b;
+ }
+ int buffer::get_c_str_accesses() {
+ return buffer_c_str_accesses;
+ }
+
+#ifdef CEPH_HAVE_SETPIPE_SZ
+ static std::atomic<unsigned> buffer_max_pipe_size { 0 };
+ int update_max_pipe_size() {
+ char buf[32];
+ int r;
+ std::string err;
+ struct stat stat_result;
+ if (::stat(PROCPREFIX "/proc/sys/fs/pipe-max-size", &stat_result) == -1)
+ return -errno;
+ r = safe_read_file(PROCPREFIX "/proc/sys/fs/", "pipe-max-size",
+ buf, sizeof(buf) - 1);
+ if (r < 0)
+ return r;
+ buf[r] = '\0';
+ size_t size = strict_strtol(buf, 10, &err);
+ if (!err.empty())
+ return -EIO;
+ buffer_max_pipe_size = size;
+ return 0;
+ }
+
+ size_t get_max_pipe_size() {
+ size_t size = buffer_max_pipe_size;
+ if (size)
+ return size;
+ if (update_max_pipe_size() == 0)
+ return buffer_max_pipe_size;
+ // this is the max size hardcoded in linux before 2.6.35
+ return 65536;
+ }
+#else
+ size_t get_max_pipe_size() { return 65536; }
+#endif
+
+
+ const char * buffer::error::what() const throw () {
+ return "buffer::exception";
+ }
+ const char * buffer::bad_alloc::what() const throw () {
+ return "buffer::bad_alloc";
+ }
+ const char * buffer::end_of_buffer::what() const throw () {
+ return "buffer::end_of_buffer";
+ }
+ const char * buffer::malformed_input::what() const throw () {
+ return buf;
+ }
+ buffer::error_code::error_code(int error) :
+ buffer::malformed_input(cpp_strerror(error).c_str()), code(error) {}
+
+ class buffer::raw {
+ public:
+ char *data;
+ unsigned len;
+ std::atomic<unsigned> nref { 0 };
+ int mempool;
+
+ mutable std::atomic_flag crc_spinlock = ATOMIC_FLAG_INIT;
+ map<pair<size_t, size_t>, pair<uint32_t, uint32_t> > crc_map;
+
+ explicit raw(unsigned l, int mempool=mempool::mempool_buffer_anon)
+ : data(NULL), len(l), nref(0), mempool(mempool) {
+ mempool::get_pool(mempool::pool_index_t(mempool)).adjust_count(1, len);
+ }
+ raw(char *c, unsigned l, int mempool=mempool::mempool_buffer_anon)
+ : data(c), len(l), nref(0), mempool(mempool) {
+ mempool::get_pool(mempool::pool_index_t(mempool)).adjust_count(1, len);
+ }
+ virtual ~raw() {
+ mempool::get_pool(mempool::pool_index_t(mempool)).adjust_count(
+ -1, -(int)len);
+ }
+
+ void _set_len(unsigned l) {
+ mempool::get_pool(mempool::pool_index_t(mempool)).adjust_count(
+ -1, -(int)len);
+ len = l;
+ mempool::get_pool(mempool::pool_index_t(mempool)).adjust_count(1, len);
+ }
+
+ void reassign_to_mempool(int pool) {
+ if (pool == mempool) {
+ return;
+ }
+ mempool::get_pool(mempool::pool_index_t(mempool)).adjust_count(
+ -1, -(int)len);
+ mempool = pool;
+ mempool::get_pool(mempool::pool_index_t(pool)).adjust_count(1, len);
+ }
+
+ void try_assign_to_mempool(int pool) {
+ if (mempool == mempool::mempool_buffer_anon) {
+ reassign_to_mempool(pool);
+ }
+ }
+
+ // no copying.
+ // cppcheck-suppress noExplicitConstructor
+ raw(const raw &other);
+ const raw& operator=(const raw &other);
+
+ virtual char *get_data() {
+ return data;
+ }
+ virtual raw* clone_empty() = 0;
+ raw *clone() {
+ raw *c = clone_empty();
+ memcpy(c->data, data, len);
+ return c;
+ }
+ virtual bool can_zero_copy() const {
+ return false;
+ }
+ virtual int zero_copy_to_fd(int fd, loff_t *offset) {
+ return -ENOTSUP;
+ }
+ virtual bool is_page_aligned() {
+ return ((long)data & ~CEPH_PAGE_MASK) == 0;
+ }
+ bool is_n_page_sized() {
+ return (len & ~CEPH_PAGE_MASK) == 0;
+ }
+ virtual bool is_shareable() {
+ // true if safe to reference/share the existing buffer copy
+ // false if it is not safe to share the buffer, e.g., due to special
+ // and/or registered memory that is scarce
+ return true;
+ }
+ bool get_crc(const pair<size_t, size_t> &fromto,
+ pair<uint32_t, uint32_t> *crc) const {
+ simple_spin_lock(&crc_spinlock);
+ map<pair<size_t, size_t>, pair<uint32_t, uint32_t> >::const_iterator i =
+ crc_map.find(fromto);
+ if (i == crc_map.end()) {
+ simple_spin_unlock(&crc_spinlock);
+ return false;
+ }
+ *crc = i->second;
+ simple_spin_unlock(&crc_spinlock);
+ return true;
+ }
+ void set_crc(const pair<size_t, size_t> &fromto,
+ const pair<uint32_t, uint32_t> &crc) {
+ simple_spin_lock(&crc_spinlock);
+ crc_map[fromto] = crc;
+ simple_spin_unlock(&crc_spinlock);
+ }
+ void invalidate_crc() {
+ simple_spin_lock(&crc_spinlock);
+ if (crc_map.size() != 0) {
+ crc_map.clear();
+ }
+ simple_spin_unlock(&crc_spinlock);
+ }
+ };
+
+ /*
+ * raw_combined is always placed within a single allocation along
+ * with the data buffer. the data goes at the beginning, and
+ * raw_combined at the end.
+ */
+ class buffer::raw_combined : public buffer::raw {
+ size_t alignment;
+ public:
+ raw_combined(char *dataptr, unsigned l, unsigned align,
+ int mempool)
+ : raw(dataptr, l, mempool),
+ alignment(align) {
+ inc_total_alloc(len);
+ inc_history_alloc(len);
+ }
+ ~raw_combined() override {
+ dec_total_alloc(len);
+ }
+ raw* clone_empty() override {
+ return create(len, alignment);
+ }
+
+ static raw_combined *create(unsigned len,
+ unsigned align,
+ int mempool = mempool::mempool_buffer_anon) {
+ if (!align)
+ align = sizeof(size_t);
+ size_t rawlen = ROUND_UP_TO(sizeof(buffer::raw_combined),
+ alignof(buffer::raw_combined));
+ size_t datalen = ROUND_UP_TO(len, alignof(buffer::raw_combined));
+
+#ifdef DARWIN
+ char *ptr = (char *) valloc(rawlen + datalen);
+#else
+ char *ptr = 0;
+ int r = ::posix_memalign((void**)(void*)&ptr, align, rawlen + datalen);
+ if (r)
+ throw bad_alloc();
+#endif /* DARWIN */
+ if (!ptr)
+ throw bad_alloc();
+
+ // actual data first, since it has presumably larger alignment restriction
+ // then put the raw_combined at the end
+ return new (ptr + datalen) raw_combined(ptr, len, align, mempool);
+ }
+
+ static void operator delete(void *ptr) {
+ raw_combined *raw = (raw_combined *)ptr;
+ ::free((void *)raw->data);
+ }
+ };
+
+ class buffer::raw_malloc : public buffer::raw {
+ public:
+ MEMPOOL_CLASS_HELPERS();
+
+ explicit raw_malloc(unsigned l) : raw(l) {
+ if (len) {
+ data = (char *)malloc(len);
+ if (!data)
+ throw bad_alloc();
+ } else {
+ data = 0;
+ }
+ inc_total_alloc(len);
+ inc_history_alloc(len);
+ bdout << "raw_malloc " << this << " alloc " << (void *)data << " " << l << " " << buffer::get_total_alloc() << bendl;
+ }
+ raw_malloc(unsigned l, char *b) : raw(b, l) {
+ inc_total_alloc(len);
+ bdout << "raw_malloc " << this << " alloc " << (void *)data << " " << l << " " << buffer::get_total_alloc() << bendl;
+ }
+ ~raw_malloc() override {
+ free(data);
+ dec_total_alloc(len);
+ bdout << "raw_malloc " << this << " free " << (void *)data << " " << buffer::get_total_alloc() << bendl;
+ }
+ raw* clone_empty() override {
+ return new raw_malloc(len);
+ }
+ };
+
+#ifndef __CYGWIN__
+ class buffer::raw_mmap_pages : public buffer::raw {
+ public:
+ MEMPOOL_CLASS_HELPERS();
+
+ explicit raw_mmap_pages(unsigned l) : raw(l) {
+ data = (char*)::mmap(NULL, len, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANON, -1, 0);
+ if (!data)
+ throw bad_alloc();
+ inc_total_alloc(len);
+ inc_history_alloc(len);
+ bdout << "raw_mmap " << this << " alloc " << (void *)data << " " << l << " " << buffer::get_total_alloc() << bendl;
+ }
+ ~raw_mmap_pages() override {
+ ::munmap(data, len);
+ dec_total_alloc(len);
+ bdout << "raw_mmap " << this << " free " << (void *)data << " " << buffer::get_total_alloc() << bendl;
+ }
+ raw* clone_empty() override {
+ return new raw_mmap_pages(len);
+ }
+ };
+
+ class buffer::raw_posix_aligned : public buffer::raw {
+ unsigned align;
+ public:
+ MEMPOOL_CLASS_HELPERS();
+
+ raw_posix_aligned(unsigned l, unsigned _align) : raw(l) {
+ align = _align;
+ assert((align >= sizeof(void *)) && (align & (align - 1)) == 0);
+#ifdef DARWIN
+ data = (char *) valloc(len);
+#else
+ int r = ::posix_memalign((void**)(void*)&data, align, len);
+ if (r)
+ throw bad_alloc();
+#endif /* DARWIN */
+ if (!data)
+ throw bad_alloc();
+ inc_total_alloc(len);
+ inc_history_alloc(len);
+ bdout << "raw_posix_aligned " << this << " alloc " << (void *)data << " l=" << l << ", align=" << align << " total_alloc=" << buffer::get_total_alloc() << bendl;
+ }
+ ~raw_posix_aligned() override {
+ ::free(data);
+ dec_total_alloc(len);
+ bdout << "raw_posix_aligned " << this << " free " << (void *)data << " " << buffer::get_total_alloc() << bendl;
+ }
+ raw* clone_empty() override {
+ return new raw_posix_aligned(len, align);
+ }
+ };
+#endif
+
+#ifdef __CYGWIN__
+ class buffer::raw_hack_aligned : public buffer::raw {
+ unsigned align;
+ char *realdata;
+ public:
+ raw_hack_aligned(unsigned l, unsigned _align) : raw(l) {
+ align = _align;
+ realdata = new char[len+align-1];
+ unsigned off = ((unsigned)realdata) & (align-1);
+ if (off)
+ data = realdata + align - off;
+ else
+ data = realdata;
+ inc_total_alloc(len+align-1);
+ inc_history_alloc(len+align-1);
+ //cout << "hack aligned " << (unsigned)data
+ //<< " in raw " << (unsigned)realdata
+ //<< " off " << off << std::endl;
+ assert(((unsigned)data & (align-1)) == 0);
+ }
+ ~raw_hack_aligned() {
+ delete[] realdata;
+ dec_total_alloc(len+align-1);
+ }
+ raw* clone_empty() {
+ return new raw_hack_aligned(len, align);
+ }
+ };
+#endif
+
+#ifdef CEPH_HAVE_SPLICE
+ class buffer::raw_pipe : public buffer::raw {
+ public:
+ MEMPOOL_CLASS_HELPERS();
+
+ explicit raw_pipe(unsigned len) : raw(len), source_consumed(false) {
+ size_t max = get_max_pipe_size();
+ if (len > max) {
+ bdout << "raw_pipe: requested length " << len
+ << " > max length " << max << bendl;
+ throw malformed_input("length larger than max pipe size");
+ }
+ pipefds[0] = -1;
+ pipefds[1] = -1;
+
+ int r;
+ if (::pipe(pipefds) == -1) {
+ r = -errno;
+ bdout << "raw_pipe: error creating pipe: " << cpp_strerror(r) << bendl;
+ throw error_code(r);
+ }
+
+ r = set_nonblocking(pipefds);
+ if (r < 0) {
+ bdout << "raw_pipe: error setting nonblocking flag on temp pipe: "
+ << cpp_strerror(r) << bendl;
+ throw error_code(r);
+ }
+
+ r = set_pipe_size(pipefds, len);
+ if (r < 0) {
+ bdout << "raw_pipe: could not set pipe size" << bendl;
+ // continue, since the pipe should become large enough as needed
+ }
+
+ inc_total_alloc(len);
+ inc_history_alloc(len);
+ bdout << "raw_pipe " << this << " alloc " << len << " "
+ << buffer::get_total_alloc() << bendl;
+ }
+
+ ~raw_pipe() override {
+ if (data)
+ free(data);
+ close_pipe(pipefds);
+ dec_total_alloc(len);
+ bdout << "raw_pipe " << this << " free " << (void *)data << " "
+ << buffer::get_total_alloc() << bendl;
+ }
+
+ bool can_zero_copy() const override {
+ return true;
+ }
+
+ int set_source(int fd, loff_t *off) {
+ int flags = SPLICE_F_NONBLOCK;
+ ssize_t r = safe_splice(fd, off, pipefds[1], NULL, len, flags);
+ if (r < 0) {
+ bdout << "raw_pipe: error splicing into pipe: " << cpp_strerror(r)
+ << bendl;
+ return r;
+ }
+ // update length with actual amount read
+ _set_len(r);
+ return 0;
+ }
+
+ int zero_copy_to_fd(int fd, loff_t *offset) override {
+ assert(!source_consumed);
+ int flags = SPLICE_F_NONBLOCK;
+ ssize_t r = safe_splice_exact(pipefds[0], NULL, fd, offset, len, flags);
+ if (r < 0) {
+ bdout << "raw_pipe: error splicing from pipe to fd: "
+ << cpp_strerror(r) << bendl;
+ return r;
+ }
+ source_consumed = true;
+ return 0;
+ }
+
+ buffer::raw* clone_empty() override {
+ // cloning doesn't make sense for pipe-based buffers,
+ // and is only used by unit tests for other types of buffers
+ return NULL;
+ }
+
+ char *get_data() override {
+ if (data)
+ return data;
+ return copy_pipe(pipefds);
+ }
+
+ private:
+ int set_pipe_size(int *fds, long length) {
+#ifdef CEPH_HAVE_SETPIPE_SZ
+ if (::fcntl(fds[1], F_SETPIPE_SZ, length) == -1) {
+ int r = -errno;
+ if (r == -EPERM) {
+ // pipe limit must have changed - EPERM means we requested
+ // more than the maximum size as an unprivileged user
+ update_max_pipe_size();
+ throw malformed_input("length larger than new max pipe size");
+ }
+ return r;
+ }
+#endif
+ return 0;
+ }
+
+ int set_nonblocking(int *fds) {
+ if (::fcntl(fds[0], F_SETFL, O_NONBLOCK) == -1)
+ return -errno;
+ if (::fcntl(fds[1], F_SETFL, O_NONBLOCK) == -1)
+ return -errno;
+ return 0;
+ }
+
+ static void close_pipe(const int *fds) {
+ if (fds[0] >= 0)
+ VOID_TEMP_FAILURE_RETRY(::close(fds[0]));
+ if (fds[1] >= 0)
+ VOID_TEMP_FAILURE_RETRY(::close(fds[1]));
+ }
+ char *copy_pipe(int *fds) {
+ /* preserve original pipe contents by copying into a temporary
+ * pipe before reading.
+ */
+ int tmpfd[2];
+ int r;
+
+ assert(!source_consumed);
+ assert(fds[0] >= 0);
+
+ if (::pipe(tmpfd) == -1) {
+ r = -errno;
+ bdout << "raw_pipe: error creating temp pipe: " << cpp_strerror(r)
+ << bendl;
+ throw error_code(r);
+ }
+ auto sg = make_scope_guard([=] { close_pipe(tmpfd); });
+ r = set_nonblocking(tmpfd);
+ if (r < 0) {
+ bdout << "raw_pipe: error setting nonblocking flag on temp pipe: "
+ << cpp_strerror(r) << bendl;
+ throw error_code(r);
+ }
+ r = set_pipe_size(tmpfd, len);
+ if (r < 0) {
+ bdout << "raw_pipe: error setting pipe size on temp pipe: "
+ << cpp_strerror(r) << bendl;
+ }
+ int flags = SPLICE_F_NONBLOCK;
+ if (::tee(fds[0], tmpfd[1], len, flags) == -1) {
+ r = errno;
+ bdout << "raw_pipe: error tee'ing into temp pipe: " << cpp_strerror(r)
+ << bendl;
+ throw error_code(r);
+ }
+ data = (char *)malloc(len);
+ if (!data) {
+ throw bad_alloc();
+ }
+ r = safe_read(tmpfd[0], data, len);
+ if (r < (ssize_t)len) {
+ bdout << "raw_pipe: error reading from temp pipe:" << cpp_strerror(r)
+ << bendl;
+ free(data);
+ data = NULL;
+ throw error_code(r);
+ }
+ return data;
+ }
+ bool source_consumed;
+ int pipefds[2];
+ };
+#endif // CEPH_HAVE_SPLICE
+
+ /*
+ * primitive buffer types
+ */
+ class buffer::raw_char : public buffer::raw {
+ public:
+ MEMPOOL_CLASS_HELPERS();
+
+ explicit raw_char(unsigned l) : raw(l) {
+ if (len)
+ data = new char[len];
+ else
+ data = 0;
+ inc_total_alloc(len);
+ inc_history_alloc(len);
+ bdout << "raw_char " << this << " alloc " << (void *)data << " " << l << " " << buffer::get_total_alloc() << bendl;
+ }
+ raw_char(unsigned l, char *b) : raw(b, l) {
+ inc_total_alloc(len);
+ bdout << "raw_char " << this << " alloc " << (void *)data << " " << l << " " << buffer::get_total_alloc() << bendl;
+ }
+ ~raw_char() override {
+ delete[] data;
+ dec_total_alloc(len);
+ bdout << "raw_char " << this << " free " << (void *)data << " " << buffer::get_total_alloc() << bendl;
+ }
+ raw* clone_empty() override {
+ return new raw_char(len);
+ }
+ };
+
+ class buffer::raw_claimed_char : public buffer::raw {
+ public:
+ MEMPOOL_CLASS_HELPERS();
+
+ explicit raw_claimed_char(unsigned l, char *b) : raw(b, l) {
+ inc_total_alloc(len);
+ bdout << "raw_claimed_char " << this << " alloc " << (void *)data
+ << " " << l << " " << buffer::get_total_alloc() << bendl;
+ }
+ ~raw_claimed_char() override {
+ dec_total_alloc(len);
+ bdout << "raw_claimed_char " << this << " free " << (void *)data
+ << " " << buffer::get_total_alloc() << bendl;
+ }
+ raw* clone_empty() override {
+ return new raw_char(len);
+ }
+ };
+
+ class buffer::raw_unshareable : public buffer::raw {
+ public:
+ MEMPOOL_CLASS_HELPERS();
+
+ explicit raw_unshareable(unsigned l) : raw(l) {
+ if (len)
+ data = new char[len];
+ else
+ data = 0;
+ }
+ raw_unshareable(unsigned l, char *b) : raw(b, l) {
+ }
+ raw* clone_empty() override {
+ return new raw_char(len);
+ }
+ bool is_shareable() override {
+ return false; // !shareable, will force make_shareable()
+ }
+ ~raw_unshareable() override {
+ delete[] data;
+ }
+ };
+
+ class buffer::raw_static : public buffer::raw {
+ public:
+ MEMPOOL_CLASS_HELPERS();
+
+ raw_static(const char *d, unsigned l) : raw((char*)d, l) { }
+ ~raw_static() override {}
+ raw* clone_empty() override {
+ return new buffer::raw_char(len);
+ }
+ };
+
+ class buffer::raw_claim_buffer : public buffer::raw {
+ deleter del;
+ public:
+ raw_claim_buffer(const char *b, unsigned l, deleter d)
+ : raw((char*)b, l), del(std::move(d)) { }
+ ~raw_claim_buffer() override {}
+ raw* clone_empty() override {
+ return new buffer::raw_char(len);
+ }
+ };
+
+#if defined(HAVE_XIO)
+ class buffer::xio_msg_buffer : public buffer::raw {
+ private:
+ XioDispatchHook* m_hook;
+ public:
+ xio_msg_buffer(XioDispatchHook* _m_hook, const char *d,
+ unsigned l) :
+ raw((char*)d, l), m_hook(_m_hook->get()) {}
+
+ bool is_shareable() { return false; }
+ static void operator delete(void *p)
+ {
+ xio_msg_buffer *buf = static_cast<xio_msg_buffer*>(p);
+ // return hook ref (counts against pool); it appears illegal
+ // to do this in our dtor, because this fires after that
+ buf->m_hook->put();
+ }
+ raw* clone_empty() {
+ return new buffer::raw_char(len);
+ }
+ };
+
+ class buffer::xio_mempool : public buffer::raw {
+ public:
+ struct xio_reg_mem *mp;
+ xio_mempool(struct xio_reg_mem *_mp, unsigned l) :
+ raw((char*)_mp->addr, l), mp(_mp)
+ { }
+ ~xio_mempool() {}
+ raw* clone_empty() {
+ return new buffer::raw_char(len);
+ }
+ };
+
+ struct xio_reg_mem* get_xio_mp(const buffer::ptr& bp)
+ {
+ buffer::xio_mempool *mb = dynamic_cast<buffer::xio_mempool*>(bp.get_raw());
+ if (mb) {
+ return mb->mp;
+ }
+ return NULL;
+ }
+
+ buffer::raw* buffer::create_msg(
+ unsigned len, char *buf, XioDispatchHook* m_hook) {
+ XioPool& pool = m_hook->get_pool();
+ buffer::raw* bp =
+ static_cast<buffer::raw*>(pool.alloc(sizeof(xio_msg_buffer)));
+ new (bp) xio_msg_buffer(m_hook, buf, len);
+ return bp;
+ }
+#endif /* HAVE_XIO */
+
+ buffer::raw* buffer::copy(const char *c, unsigned len) {
+ raw* r = buffer::create_aligned(len, sizeof(size_t));
+ memcpy(r->data, c, len);
+ return r;
+ }
+
+ buffer::raw* buffer::create(unsigned len) {
+ return buffer::create_aligned(len, sizeof(size_t));
+ }
+ buffer::raw* buffer::create_in_mempool(unsigned len, int mempool) {
+ return buffer::create_aligned_in_mempool(len, sizeof(size_t), mempool);
+ }
+ buffer::raw* buffer::claim_char(unsigned len, char *buf) {
+ return new raw_claimed_char(len, buf);
+ }
+ buffer::raw* buffer::create_malloc(unsigned len) {
+ return new raw_malloc(len);
+ }
+ buffer::raw* buffer::claim_malloc(unsigned len, char *buf) {
+ return new raw_malloc(len, buf);
+ }
+ buffer::raw* buffer::create_static(unsigned len, char *buf) {
+ return new raw_static(buf, len);
+ }
+ buffer::raw* buffer::claim_buffer(unsigned len, char *buf, deleter del) {
+ return new raw_claim_buffer(buf, len, std::move(del));
+ }
+
+ buffer::raw* buffer::create_aligned_in_mempool(
+ unsigned len, unsigned align, int mempool) {
+ // If alignment is a page multiple, use a separate buffer::raw to
+ // avoid fragmenting the heap.
+ //
+ // Somewhat unexpectedly, I see consistently better performance
+ // from raw_combined than from raw even when the allocation size is
+ // a page multiple (but alignment is not).
+ //
+ // I also see better performance from a separate buffer::raw once the
+ // size passes 8KB.
+ if ((align & ~CEPH_PAGE_MASK) == 0 ||
+ len >= CEPH_PAGE_SIZE * 2) {
+#ifndef __CYGWIN__
+ return new raw_posix_aligned(len, align);
+#else
+ return new raw_hack_aligned(len, align);
+#endif
+ }
+ return raw_combined::create(len, align, mempool);
+ }
+ buffer::raw* buffer::create_aligned(
+ unsigned len, unsigned align) {
+ return create_aligned_in_mempool(len, align,
+ mempool::mempool_buffer_anon);
+ }
+
+ buffer::raw* buffer::create_page_aligned(unsigned len) {
+ return create_aligned(len, CEPH_PAGE_SIZE);
+ }
+
+ buffer::raw* buffer::create_zero_copy(unsigned len, int fd, int64_t *offset) {
+#ifdef CEPH_HAVE_SPLICE
+ buffer::raw_pipe* buf = new raw_pipe(len);
+ int r = buf->set_source(fd, (loff_t*)offset);
+ if (r < 0) {
+ delete buf;
+ throw error_code(r);
+ }
+ return buf;
+#else
+ throw error_code(-ENOTSUP);
+#endif
+ }
+
+ buffer::raw* buffer::create_unshareable(unsigned len) {
+ return new raw_unshareable(len);
+ }
+
+ buffer::ptr::ptr(raw *r) : _raw(r), _off(0), _len(r->len) // no lock needed; this is an unref raw.
+ {
+ r->nref++;
+ bdout << "ptr " << this << " get " << _raw << bendl;
+ }
+ buffer::ptr::ptr(unsigned l) : _off(0), _len(l)
+ {
+ _raw = create(l);
+ _raw->nref++;
+ bdout << "ptr " << this << " get " << _raw << bendl;
+ }
+ buffer::ptr::ptr(const char *d, unsigned l) : _off(0), _len(l) // ditto.
+ {
+ _raw = copy(d, l);
+ _raw->nref++;
+ bdout << "ptr " << this << " get " << _raw << bendl;
+ }
+ buffer::ptr::ptr(const ptr& p) : _raw(p._raw), _off(p._off), _len(p._len)
+ {
+ if (_raw) {
+ _raw->nref++;
+ bdout << "ptr " << this << " get " << _raw << bendl;
+ }
+ }
+ buffer::ptr::ptr(ptr&& p) noexcept : _raw(p._raw), _off(p._off), _len(p._len)
+ {
+ p._raw = nullptr;
+ p._off = p._len = 0;
+ }
+ buffer::ptr::ptr(const ptr& p, unsigned o, unsigned l)
+ : _raw(p._raw), _off(p._off + o), _len(l)
+ {
+ assert(o+l <= p._len);
+ assert(_raw);
+ _raw->nref++;
+ bdout << "ptr " << this << " get " << _raw << bendl;
+ }
+ buffer::ptr& buffer::ptr::operator= (const ptr& p)
+ {
+ if (p._raw) {
+ p._raw->nref++;
+ bdout << "ptr " << this << " get " << _raw << bendl;
+ }
+ buffer::raw *raw = p._raw;
+ release();
+ if (raw) {
+ _raw = raw;
+ _off = p._off;
+ _len = p._len;
+ } else {
+ _off = _len = 0;
+ }
+ return *this;
+ }
+ buffer::ptr& buffer::ptr::operator= (ptr&& p) noexcept
+ {
+ release();
+ buffer::raw *raw = p._raw;
+ if (raw) {
+ _raw = raw;
+ _off = p._off;
+ _len = p._len;
+ p._raw = nullptr;
+ p._off = p._len = 0;
+ } else {
+ _off = _len = 0;
+ }
+ return *this;
+ }
+
+ buffer::raw *buffer::ptr::clone()
+ {
+ return _raw->clone();
+ }
+
+ buffer::ptr& buffer::ptr::make_shareable() {
+ if (_raw && !_raw->is_shareable()) {
+ buffer::raw *tr = _raw;
+ _raw = tr->clone();
+ _raw->nref = 1;
+ if (unlikely(--tr->nref == 0)) {
+ ANNOTATE_HAPPENS_AFTER(&tr->nref);
+ ANNOTATE_HAPPENS_BEFORE_FORGET_ALL(&tr->nref);
+ delete tr;
+ } else {
+ ANNOTATE_HAPPENS_BEFORE(&tr->nref);
+ }
+ }
+ return *this;
+ }
+
+ void buffer::ptr::swap(ptr& other)
+ {
+ raw *r = _raw;
+ unsigned o = _off;
+ unsigned l = _len;
+ _raw = other._raw;
+ _off = other._off;
+ _len = other._len;
+ other._raw = r;
+ other._off = o;
+ other._len = l;
+ }
+
+ void buffer::ptr::release()
+ {
+ if (_raw) {
+ bdout << "ptr " << this << " release " << _raw << bendl;
+ if (--_raw->nref == 0) {
+ //cout << "hosing raw " << (void*)_raw << " len " << _raw->len << std::endl;
+ ANNOTATE_HAPPENS_AFTER(&_raw->nref);
+ ANNOTATE_HAPPENS_BEFORE_FORGET_ALL(&_raw->nref);
+ delete _raw; // dealloc old (if any)
+ } else {
+ ANNOTATE_HAPPENS_BEFORE(&_raw->nref);
+ }
+ _raw = 0;
+ }
+ }
+
+ bool buffer::ptr::at_buffer_tail() const { return _off + _len == _raw->len; }
+
+ int buffer::ptr::get_mempool() const {
+ if (_raw) {
+ return _raw->mempool;
+ }
+ return mempool::mempool_buffer_anon;
+ }
+
+ void buffer::ptr::reassign_to_mempool(int pool) {
+ if (_raw) {
+ _raw->reassign_to_mempool(pool);
+ }
+ }
+ void buffer::ptr::try_assign_to_mempool(int pool) {
+ if (_raw) {
+ _raw->try_assign_to_mempool(pool);
+ }
+ }
+
+ const char *buffer::ptr::c_str() const {
+ assert(_raw);
+ if (buffer_track_c_str)
+ buffer_c_str_accesses++;
+ return _raw->get_data() + _off;
+ }
+ char *buffer::ptr::c_str() {
+ assert(_raw);
+ if (buffer_track_c_str)
+ buffer_c_str_accesses++;
+ return _raw->get_data() + _off;
+ }
+ const char *buffer::ptr::end_c_str() const {
+ assert(_raw);
+ if (buffer_track_c_str)
+ buffer_c_str_accesses++;
+ return _raw->get_data() + _off + _len;
+ }
+ char *buffer::ptr::end_c_str() {
+ assert(_raw);
+ if (buffer_track_c_str)
+ buffer_c_str_accesses++;
+ return _raw->get_data() + _off + _len;
+ }
+
+ unsigned buffer::ptr::unused_tail_length() const
+ {
+ if (_raw)
+ return _raw->len - (_off+_len);
+ else
+ return 0;
+ }
+ const char& buffer::ptr::operator[](unsigned n) const
+ {
+ assert(_raw);
+ assert(n < _len);
+ return _raw->get_data()[_off + n];
+ }
+ char& buffer::ptr::operator[](unsigned n)
+ {
+ assert(_raw);
+ assert(n < _len);
+ return _raw->get_data()[_off + n];
+ }
+
+ const char *buffer::ptr::raw_c_str() const { assert(_raw); return _raw->data; }
+ unsigned buffer::ptr::raw_length() const { assert(_raw); return _raw->len; }
+ int buffer::ptr::raw_nref() const { assert(_raw); return _raw->nref; }
+
+ void buffer::ptr::copy_out(unsigned o, unsigned l, char *dest) const {
+ assert(_raw);
+ if (o+l > _len)
+ throw end_of_buffer();
+ char* src = _raw->data + _off + o;
+ maybe_inline_memcpy(dest, src, l, 8);
+ }
+
+ unsigned buffer::ptr::wasted() const
+ {
+ return _raw->len - _len;
+ }
+
+ int buffer::ptr::cmp(const ptr& o) const
+ {
+ int l = _len < o._len ? _len : o._len;
+ if (l) {
+ int r = memcmp(c_str(), o.c_str(), l);
+ if (r)
+ return r;
+ }
+ if (_len < o._len)
+ return -1;
+ if (_len > o._len)
+ return 1;
+ return 0;
+ }
+
+ bool buffer::ptr::is_zero() const
+ {
+ return mem_is_zero(c_str(), _len);
+ }
+
+ unsigned buffer::ptr::append(char c)
+ {
+ assert(_raw);
+ assert(1 <= unused_tail_length());
+ char* ptr = _raw->data + _off + _len;
+ *ptr = c;
+ _len++;
+ return _len + _off;
+ }
+
+ unsigned buffer::ptr::append(const char *p, unsigned l)
+ {
+ assert(_raw);
+ assert(l <= unused_tail_length());
+ char* c = _raw->data + _off + _len;
+ maybe_inline_memcpy(c, p, l, 32);
+ _len += l;
+ return _len + _off;
+ }
+
+ void buffer::ptr::copy_in(unsigned o, unsigned l, const char *src)
+ {
+ copy_in(o, l, src, true);
+ }
+
+ void buffer::ptr::copy_in(unsigned o, unsigned l, const char *src, bool crc_reset)
+ {
+ assert(_raw);
+ assert(o <= _len);
+ assert(o+l <= _len);
+ char* dest = _raw->data + _off + o;
+ if (crc_reset)
+ _raw->invalidate_crc();
+ maybe_inline_memcpy(dest, src, l, 64);
+ }
+
+ void buffer::ptr::zero()
+ {
+ zero(true);
+ }
+
+ void buffer::ptr::zero(bool crc_reset)
+ {
+ if (crc_reset)
+ _raw->invalidate_crc();
+ memset(c_str(), 0, _len);
+ }
+
+ void buffer::ptr::zero(unsigned o, unsigned l)
+ {
+ zero(o, l, true);
+ }
+
+ void buffer::ptr::zero(unsigned o, unsigned l, bool crc_reset)
+ {
+ assert(o+l <= _len);
+ if (crc_reset)
+ _raw->invalidate_crc();
+ memset(c_str()+o, 0, l);
+ }
+ bool buffer::ptr::can_zero_copy() const
+ {
+ return _raw->can_zero_copy();
+ }
+
+ int buffer::ptr::zero_copy_to_fd(int fd, int64_t *offset) const
+ {
+ return _raw->zero_copy_to_fd(fd, (loff_t*)offset);
+ }
+
+ // -- buffer::list::iterator --
+ /*
+ buffer::list::iterator operator=(const buffer::list::iterator& other)
+ {
+ if (this != &other) {
+ bl = other.bl;
+ ls = other.ls;
+ off = other.off;
+ p = other.p;
+ p_off = other.p_off;
+ }
+ return *this;
+ }*/
+
+ template<bool is_const>
+ buffer::list::iterator_impl<is_const>::iterator_impl(bl_t *l, unsigned o)
+ : bl(l), ls(&bl->_buffers), off(0), p(ls->begin()), p_off(0)
+ {
+ advance(o);
+ }
+
+ template<bool is_const>
+ buffer::list::iterator_impl<is_const>::iterator_impl(const buffer::list::iterator& i)
+ : iterator_impl<is_const>(i.bl, i.off, i.p, i.p_off) {}
+
+ template<bool is_const>
+ void buffer::list::iterator_impl<is_const>::advance(int o)
+ {
+ //cout << this << " advance " << o << " from " << off << " (p_off " << p_off << " in " << p->length() << ")" << std::endl;
+ if (o > 0) {
+ p_off += o;
+ while (p_off > 0) {
+ if (p == ls->end())
+ throw end_of_buffer();
+ if (p_off >= p->length()) {
+ // skip this buffer
+ p_off -= p->length();
+ p++;
+ } else {
+ // somewhere in this buffer!
+ break;
+ }
+ }
+ off += o;
+ return;
+ }
+ while (o < 0) {
+ if (p_off) {
+ unsigned d = -o;
+ if (d > p_off)
+ d = p_off;
+ p_off -= d;
+ off -= d;
+ o += d;
+ } else if (off > 0) {
+ assert(p != ls->begin());
+ p--;
+ p_off = p->length();
+ } else {
+ throw end_of_buffer();
+ }
+ }
+ }
+
+ template<bool is_const>
+ void buffer::list::iterator_impl<is_const>::seek(unsigned o)
+ {
+ p = ls->begin();
+ off = p_off = 0;
+ advance(o);
+ }
+
+ template<bool is_const>
+ char buffer::list::iterator_impl<is_const>::operator*() const
+ {
+ if (p == ls->end())
+ throw end_of_buffer();
+ return (*p)[p_off];
+ }
+
+ template<bool is_const>
+ buffer::list::iterator_impl<is_const>&
+ buffer::list::iterator_impl<is_const>::operator++()
+ {
+ if (p == ls->end())
+ throw end_of_buffer();
+ advance(1);
+ return *this;
+ }
+
+ template<bool is_const>
+ buffer::ptr buffer::list::iterator_impl<is_const>::get_current_ptr() const
+ {
+ if (p == ls->end())
+ throw end_of_buffer();
+ return ptr(*p, p_off, p->length() - p_off);
+ }
+
+ // copy data out.
+ // note that these all _append_ to dest!
+ template<bool is_const>
+ void buffer::list::iterator_impl<is_const>::copy(unsigned len, char *dest)
+ {
+ if (p == ls->end()) seek(off);
+ while (len > 0) {
+ if (p == ls->end())
+ throw end_of_buffer();
+ assert(p->length() > 0);
+
+ unsigned howmuch = p->length() - p_off;
+ if (len < howmuch) howmuch = len;
+ p->copy_out(p_off, howmuch, dest);
+ dest += howmuch;
+
+ len -= howmuch;
+ advance(howmuch);
+ }
+ }
+
+ template<bool is_const>
+ void buffer::list::iterator_impl<is_const>::copy(unsigned len, ptr &dest)
+ {
+ copy_deep(len, dest);
+ }
+
+ template<bool is_const>
+ void buffer::list::iterator_impl<is_const>::copy_deep(unsigned len, ptr &dest)
+ {
+ if (!len) {
+ return;
+ }
+ if (p == ls->end())
+ throw end_of_buffer();
+ assert(p->length() > 0);
+ dest = create(len);
+ copy(len, dest.c_str());
+ }
+ template<bool is_const>
+ void buffer::list::iterator_impl<is_const>::copy_shallow(unsigned len,
+ ptr &dest)
+ {
+ if (!len) {
+ return;
+ }
+ if (p == ls->end())
+ throw end_of_buffer();
+ assert(p->length() > 0);
+ unsigned howmuch = p->length() - p_off;
+ if (howmuch < len) {
+ dest = create(len);
+ copy(len, dest.c_str());
+ } else {
+ dest = ptr(*p, p_off, len);
+ advance(len);
+ }
+ }
+
+ template<bool is_const>
+ void buffer::list::iterator_impl<is_const>::copy(unsigned len, list &dest)
+ {
+ if (p == ls->end())
+ seek(off);
+ while (len > 0) {
+ if (p == ls->end())
+ throw end_of_buffer();
+
+ unsigned howmuch = p->length() - p_off;
+ if (len < howmuch)
+ howmuch = len;
+ dest.append(*p, p_off, howmuch);
+
+ len -= howmuch;
+ advance(howmuch);
+ }
+ }
+
+ template<bool is_const>
+ void buffer::list::iterator_impl<is_const>::copy(unsigned len, std::string &dest)
+ {
+ if (p == ls->end())
+ seek(off);
+ while (len > 0) {
+ if (p == ls->end())
+ throw end_of_buffer();
+
+ unsigned howmuch = p->length() - p_off;
+ const char *c_str = p->c_str();
+ if (len < howmuch)
+ howmuch = len;
+ dest.append(c_str + p_off, howmuch);
+
+ len -= howmuch;
+ advance(howmuch);
+ }
+ }
+
+ template<bool is_const>
+ void buffer::list::iterator_impl<is_const>::copy_all(list &dest)
+ {
+ if (p == ls->end())
+ seek(off);
+ while (1) {
+ if (p == ls->end())
+ return;
+ assert(p->length() > 0);
+
+ unsigned howmuch = p->length() - p_off;
+ const char *c_str = p->c_str();
+ dest.append(c_str + p_off, howmuch);
+
+ advance(howmuch);
+ }
+ }
+
+ template<bool is_const>
+ size_t buffer::list::iterator_impl<is_const>::get_ptr_and_advance(
+ size_t want, const char **data)
+ {
+ if (p == ls->end()) {
+ seek(off);
+ if (p == ls->end()) {
+ return 0;
+ }
+ }
+ *data = p->c_str() + p_off;
+ size_t l = MIN(p->length() - p_off, want);
+ p_off += l;
+ if (p_off == p->length()) {
+ ++p;
+ p_off = 0;
+ }
+ off += l;
+ return l;
+ }
+
+ template<bool is_const>
+ uint32_t buffer::list::iterator_impl<is_const>::crc32c(
+ size_t length, uint32_t crc)
+ {
+ length = MIN( length, get_remaining());
+ while (length > 0) {
+ const char *p;
+ size_t l = get_ptr_and_advance(length, &p);
+ crc = ceph_crc32c(crc, (unsigned char*)p, l);
+ length -= l;
+ }
+ return crc;
+ }
+
+ // explicitly instantiate only the iterator types we need, so we can hide the
+ // details in this compilation unit without introducing unnecessary link time
+ // dependencies.
+ template class buffer::list::iterator_impl<true>;
+ template class buffer::list::iterator_impl<false>;
+
+ buffer::list::iterator::iterator(bl_t *l, unsigned o)
+ : iterator_impl(l, o)
+ {}
+
+ buffer::list::iterator::iterator(bl_t *l, unsigned o, list_iter_t ip, unsigned po)
+ : iterator_impl(l, o, ip, po)
+ {}
+
+ void buffer::list::iterator::advance(int o)
+ {
+ buffer::list::iterator_impl<false>::advance(o);
+ }
+
+ void buffer::list::iterator::seek(unsigned o)
+ {
+ buffer::list::iterator_impl<false>::seek(o);
+ }
+
+ char buffer::list::iterator::operator*()
+ {
+ if (p == ls->end()) {
+ throw end_of_buffer();
+ }
+ return (*p)[p_off];
+ }
+
+ buffer::list::iterator& buffer::list::iterator::operator++()
+ {
+ buffer::list::iterator_impl<false>::operator++();
+ return *this;
+ }
+
+ buffer::ptr buffer::list::iterator::get_current_ptr()
+ {
+ if (p == ls->end()) {
+ throw end_of_buffer();
+ }
+ return ptr(*p, p_off, p->length() - p_off);
+ }
+
+ void buffer::list::iterator::copy(unsigned len, char *dest)
+ {
+ return buffer::list::iterator_impl<false>::copy(len, dest);
+ }
+
+ void buffer::list::iterator::copy(unsigned len, ptr &dest)
+ {
+ return buffer::list::iterator_impl<false>::copy_deep(len, dest);
+ }
+
+ void buffer::list::iterator::copy_deep(unsigned len, ptr &dest)
+ {
+ buffer::list::iterator_impl<false>::copy_deep(len, dest);
+ }
+
+ void buffer::list::iterator::copy_shallow(unsigned len, ptr &dest)
+ {
+ buffer::list::iterator_impl<false>::copy_shallow(len, dest);
+ }
+
+ void buffer::list::iterator::copy(unsigned len, list &dest)
+ {
+ buffer::list::iterator_impl<false>::copy(len, dest);
+ }
+
+ void buffer::list::iterator::copy(unsigned len, std::string &dest)
+ {
+ buffer::list::iterator_impl<false>::copy(len, dest);
+ }
+
+ void buffer::list::iterator::copy_all(list &dest)
+ {
+ buffer::list::iterator_impl<false>::copy_all(dest);
+ }
+
+ void buffer::list::iterator::copy_in(unsigned len, const char *src)
+ {
+ copy_in(len, src, true);
+ }
+
+ // copy data in
+ void buffer::list::iterator::copy_in(unsigned len, const char *src, bool crc_reset)
+ {
+ // copy
+ if (p == ls->end())
+ seek(off);
+ while (len > 0) {
+ if (p == ls->end())
+ throw end_of_buffer();
+
+ unsigned howmuch = p->length() - p_off;
+ if (len < howmuch)
+ howmuch = len;
+ p->copy_in(p_off, howmuch, src, crc_reset);
+
+ src += howmuch;
+ len -= howmuch;
+ advance(howmuch);
+ }
+ }
+
+ void buffer::list::iterator::copy_in(unsigned len, const list& otherl)
+ {
+ if (p == ls->end())
+ seek(off);
+ unsigned left = len;
+ for (std::list<ptr>::const_iterator i = otherl._buffers.begin();
+ i != otherl._buffers.end();
+ ++i) {
+ unsigned l = (*i).length();
+ if (left < l)
+ l = left;
+ copy_in(l, i->c_str());
+ left -= l;
+ if (left == 0)
+ break;
+ }
+ }
+
+ // -- buffer::list --
+
+ buffer::list::list(list&& other)
+ : _buffers(std::move(other._buffers)),
+ _len(other._len),
+ _memcopy_count(other._memcopy_count),
+ last_p(this) {
+ append_buffer.swap(other.append_buffer);
+ other.clear();
+ }
+
+ void buffer::list::swap(list& other)
+ {
+ std::swap(_len, other._len);
+ std::swap(_memcopy_count, other._memcopy_count);
+ _buffers.swap(other._buffers);
+ append_buffer.swap(other.append_buffer);
+ //last_p.swap(other.last_p);
+ last_p = begin();
+ other.last_p = other.begin();
+ }
+
+ bool buffer::list::contents_equal(buffer::list& other)
+ {
+ return static_cast<const buffer::list*>(this)->contents_equal(other);
+ }
+
+ bool buffer::list::contents_equal(const ceph::buffer::list& other) const
+ {
+ if (length() != other.length())
+ return false;
+
+ // buffer-wise comparison
+ if (true) {
+ std::list<ptr>::const_iterator a = _buffers.begin();
+ std::list<ptr>::const_iterator b = other._buffers.begin();
+ unsigned aoff = 0, boff = 0;
+ while (a != _buffers.end()) {
+ unsigned len = a->length() - aoff;
+ if (len > b->length() - boff)
+ len = b->length() - boff;
+ if (memcmp(a->c_str() + aoff, b->c_str() + boff, len) != 0)
+ return false;
+ aoff += len;
+ if (aoff == a->length()) {
+ aoff = 0;
+ ++a;
+ }
+ boff += len;
+ if (boff == b->length()) {
+ boff = 0;
+ ++b;
+ }
+ }
+ assert(b == other._buffers.end());
+ return true;
+ }
+
+ // byte-wise comparison
+ if (false) {
+ bufferlist::const_iterator me = begin();
+ bufferlist::const_iterator him = other.begin();
+ while (!me.end()) {
+ if (*me != *him)
+ return false;
+ ++me;
+ ++him;
+ }
+ return true;
+ }
+ }
+
+ bool buffer::list::can_zero_copy() const
+ {
+ for (std::list<ptr>::const_iterator it = _buffers.begin();
+ it != _buffers.end();
+ ++it)
+ if (!it->can_zero_copy())
+ return false;
+ return true;
+ }
+
+ bool buffer::list::is_provided_buffer(const char *dst) const
+ {
+ if (_buffers.empty())
+ return false;
+ return (is_contiguous() && (_buffers.front().c_str() == dst));
+ }
+
+ bool buffer::list::is_aligned(unsigned align) const
+ {
+ for (std::list<ptr>::const_iterator it = _buffers.begin();
+ it != _buffers.end();
+ ++it)
+ if (!it->is_aligned(align))
+ return false;
+ return true;
+ }
+
+ bool buffer::list::is_n_align_sized(unsigned align) const
+ {
+ for (std::list<ptr>::const_iterator it = _buffers.begin();
+ it != _buffers.end();
+ ++it)
+ if (!it->is_n_align_sized(align))
+ return false;
+ return true;
+ }
+
+ bool buffer::list::is_aligned_size_and_memory(unsigned align_size,
+ unsigned align_memory) const
+ {
+ for (std::list<ptr>::const_iterator it = _buffers.begin();
+ it != _buffers.end();
+ ++it) {
+ if (!it->is_aligned(align_memory) || !it->is_n_align_sized(align_size))
+ return false;
+ }
+ return true;
+ }
+
+ bool buffer::list::is_zero() const {
+ for (std::list<ptr>::const_iterator it = _buffers.begin();
+ it != _buffers.end();
+ ++it) {
+ if (!it->is_zero()) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ void buffer::list::zero()
+ {
+ for (std::list<ptr>::iterator it = _buffers.begin();
+ it != _buffers.end();
+ ++it)
+ it->zero();
+ }
+
+ void buffer::list::zero(unsigned o, unsigned l)
+ {
+ assert(o+l <= _len);
+ unsigned p = 0;
+ for (std::list<ptr>::iterator it = _buffers.begin();
+ it != _buffers.end();
+ ++it) {
+ if (p + it->length() > o) {
+ if (p >= o && p+it->length() <= o+l) {
+ // 'o'------------- l -----------|
+ // 'p'-- it->length() --|
+ it->zero();
+ } else if (p >= o) {
+ // 'o'------------- l -----------|
+ // 'p'------- it->length() -------|
+ it->zero(0, o+l-p);
+ } else if (p + it->length() <= o+l) {
+ // 'o'------------- l -----------|
+ // 'p'------- it->length() -------|
+ it->zero(o-p, it->length()-(o-p));
+ } else {
+ // 'o'----------- l -----------|
+ // 'p'---------- it->length() ----------|
+ it->zero(o-p, l);
+ }
+ }
+ p += it->length();
+ if (o+l <= p)
+ break; // done
+ }
+ }
+
+ bool buffer::list::is_contiguous() const
+ {
+ return &(*_buffers.begin()) == &(*_buffers.rbegin());
+ }
+
+ bool buffer::list::is_n_page_sized() const
+ {
+ return is_n_align_sized(CEPH_PAGE_SIZE);
+ }
+
+ bool buffer::list::is_page_aligned() const
+ {
+ return is_aligned(CEPH_PAGE_SIZE);
+ }
+
+ int buffer::list::get_mempool() const
+ {
+ if (_buffers.empty()) {
+ return mempool::mempool_buffer_anon;
+ }
+ return _buffers.back().get_mempool();
+ }
+
+ void buffer::list::reassign_to_mempool(int pool)
+ {
+ if (append_buffer.get_raw()) {
+ append_buffer.get_raw()->reassign_to_mempool(pool);
+ }
+ for (auto& p : _buffers) {
+ p.get_raw()->reassign_to_mempool(pool);
+ }
+ }
+
+ void buffer::list::try_assign_to_mempool(int pool)
+ {
+ if (append_buffer.get_raw()) {
+ append_buffer.get_raw()->try_assign_to_mempool(pool);
+ }
+ for (auto& p : _buffers) {
+ p.get_raw()->try_assign_to_mempool(pool);
+ }
+ }
+
+ void buffer::list::rebuild()
+ {
+ if (_len == 0) {
+ _buffers.clear();
+ return;
+ }
+ ptr nb;
+ if ((_len & ~CEPH_PAGE_MASK) == 0)
+ nb = buffer::create_page_aligned(_len);
+ else
+ nb = buffer::create(_len);
+ rebuild(nb);
+ }
+
+ void buffer::list::rebuild(ptr& nb)
+ {
+ unsigned pos = 0;
+ for (std::list<ptr>::iterator it = _buffers.begin();
+ it != _buffers.end();
+ ++it) {
+ nb.copy_in(pos, it->length(), it->c_str(), false);
+ pos += it->length();
+ }
+ _memcopy_count += pos;
+ _buffers.clear();
+ if (nb.length())
+ _buffers.push_back(nb);
+ invalidate_crc();
+ last_p = begin();
+ }
+
+ bool buffer::list::rebuild_aligned(unsigned align)
+ {
+ return rebuild_aligned_size_and_memory(align, align);
+ }
+
+ bool buffer::list::rebuild_aligned_size_and_memory(unsigned align_size,
+ unsigned align_memory)
+ {
+ unsigned old_memcopy_count = _memcopy_count;
+ std::list<ptr>::iterator p = _buffers.begin();
+ while (p != _buffers.end()) {
+ // keep anything that's already align and sized aligned
+ if (p->is_aligned(align_memory) && p->is_n_align_sized(align_size)) {
+ /*cout << " segment " << (void*)p->c_str()
+ << " offset " << ((unsigned long)p->c_str() & (align - 1))
+ << " length " << p->length()
+ << " " << (p->length() & (align - 1)) << " ok" << std::endl;
+ */
+ ++p;
+ continue;
+ }
+
+ // consolidate unaligned items, until we get something that is sized+aligned
+ list unaligned;
+ unsigned offset = 0;
+ do {
+ /*cout << " segment " << (void*)p->c_str()
+ << " offset " << ((unsigned long)p->c_str() & (align - 1))
+ << " length " << p->length() << " " << (p->length() & (align - 1))
+ << " overall offset " << offset << " " << (offset & (align - 1))
+ << " not ok" << std::endl;
+ */
+ offset += p->length();
+ unaligned.push_back(*p);
+ _buffers.erase(p++);
+ } while (p != _buffers.end() &&
+ (!p->is_aligned(align_memory) ||
+ !p->is_n_align_sized(align_size) ||
+ (offset % align_size)));
+ if (!(unaligned.is_contiguous() && unaligned._buffers.front().is_aligned(align_memory))) {
+ ptr nb(buffer::create_aligned(unaligned._len, align_memory));
+ unaligned.rebuild(nb);
+ _memcopy_count += unaligned._len;
+ }
+ _buffers.insert(p, unaligned._buffers.front());
+ }
+ last_p = begin();
+
+ return (old_memcopy_count != _memcopy_count);
+ }
+
+ bool buffer::list::rebuild_page_aligned()
+ {
+ return rebuild_aligned(CEPH_PAGE_SIZE);
+ }
+
+ void buffer::list::reserve(size_t prealloc)
+ {
+ if (append_buffer.unused_tail_length() < prealloc) {
+ append_buffer = buffer::create_in_mempool(prealloc, get_mempool());
+ append_buffer.set_length(0); // unused, so far.
+ }
+ }
+
+ // sort-of-like-assignment-op
+ void buffer::list::claim(list& bl, unsigned int flags)
+ {
+ // free my buffers
+ clear();
+ claim_append(bl, flags);
+ }
+
+ void buffer::list::claim_append(list& bl, unsigned int flags)
+ {
+ // steal the other guy's buffers
+ _len += bl._len;
+ if (!(flags & CLAIM_ALLOW_NONSHAREABLE))
+ bl.make_shareable();
+ _buffers.splice(_buffers.end(), bl._buffers );
+ bl._len = 0;
+ bl.last_p = bl.begin();
+ }
+
+ void buffer::list::claim_prepend(list& bl, unsigned int flags)
+ {
+ // steal the other guy's buffers
+ _len += bl._len;
+ if (!(flags & CLAIM_ALLOW_NONSHAREABLE))
+ bl.make_shareable();
+ _buffers.splice(_buffers.begin(), bl._buffers );
+ bl._len = 0;
+ bl.last_p = bl.begin();
+ }
+
+ void buffer::list::claim_append_piecewise(list& bl)
+ {
+ // steal the other guy's buffers
+ for (std::list<buffer::ptr>::const_iterator i = bl.buffers().begin();
+ i != bl.buffers().end(); i++) {
+ append(*i, 0, i->length());
+ }
+ bl.clear();
+ }
+
+ void buffer::list::copy(unsigned off, unsigned len, char *dest) const
+ {
+ if (off + len > length())
+ throw end_of_buffer();
+ if (last_p.get_off() != off)
+ last_p.seek(off);
+ last_p.copy(len, dest);
+ }
+
+ void buffer::list::copy(unsigned off, unsigned len, list &dest) const
+ {
+ if (off + len > length())
+ throw end_of_buffer();
+ if (last_p.get_off() != off)
+ last_p.seek(off);
+ last_p.copy(len, dest);
+ }
+
+ void buffer::list::copy(unsigned off, unsigned len, std::string& dest) const
+ {
+ if (last_p.get_off() != off)
+ last_p.seek(off);
+ return last_p.copy(len, dest);
+ }
+
+ void buffer::list::copy_in(unsigned off, unsigned len, const char *src)
+ {
+ copy_in(off, len, src, true);
+ }
+
+ void buffer::list::copy_in(unsigned off, unsigned len, const char *src, bool crc_reset)
+ {
+ if (off + len > length())
+ throw end_of_buffer();
+
+ if (last_p.get_off() != off)
+ last_p.seek(off);
+ last_p.copy_in(len, src, crc_reset);
+ }
+
+ void buffer::list::copy_in(unsigned off, unsigned len, const list& src)
+ {
+ if (last_p.get_off() != off)
+ last_p.seek(off);
+ last_p.copy_in(len, src);
+ }
+
+ void buffer::list::append(char c)
+ {
+ // put what we can into the existing append_buffer.
+ unsigned gap = append_buffer.unused_tail_length();
+ if (!gap) {
+ // make a new append_buffer!
+ append_buffer = raw_combined::create(CEPH_BUFFER_APPEND_SIZE, 0,
+ get_mempool());
+ append_buffer.set_length(0); // unused, so far.
+ }
+ append(append_buffer, append_buffer.append(c) - 1, 1); // add segment to the list
+ }
+
+ void buffer::list::append(const char *data, unsigned len)
+ {
+ while (len > 0) {
+ // put what we can into the existing append_buffer.
+ unsigned gap = append_buffer.unused_tail_length();
+ if (gap > 0) {
+ if (gap > len) gap = len;
+ //cout << "append first char is " << data[0] << ", last char is " << data[len-1] << std::endl;
+ append_buffer.append(data, gap);
+ append(append_buffer, append_buffer.length() - gap, gap); // add segment to the list
+ len -= gap;
+ data += gap;
+ }
+ if (len == 0)
+ break; // done!
+
+ // make a new append_buffer. fill out a complete page, factoring in the
+ // raw_combined overhead.
+ size_t need = ROUND_UP_TO(len, sizeof(size_t)) + sizeof(raw_combined);
+ size_t alen = ROUND_UP_TO(need, CEPH_BUFFER_ALLOC_UNIT) -
+ sizeof(raw_combined);
+ append_buffer = raw_combined::create(alen, 0, get_mempool());
+ append_buffer.set_length(0); // unused, so far.
+ }
+ }
+
+ void buffer::list::append(const ptr& bp)
+ {
+ if (bp.length())
+ push_back(bp);
+ }
+
+ void buffer::list::append(ptr&& bp)
+ {
+ if (bp.length())
+ push_back(std::move(bp));
+ }
+
+ void buffer::list::append(const ptr& bp, unsigned off, unsigned len)
+ {
+ assert(len+off <= bp.length());
+ if (!_buffers.empty()) {
+ ptr &l = _buffers.back();
+ if (l.get_raw() == bp.get_raw() &&
+ l.end() == bp.start() + off) {
+ // yay contiguous with tail bp!
+ l.set_length(l.length()+len);
+ _len += len;
+ return;
+ }
+ }
+ // add new item to list
+ push_back(ptr(bp, off, len));
+ }
+
+ void buffer::list::append(const list& bl)
+ {
+ _len += bl._len;
+ for (std::list<ptr>::const_iterator p = bl._buffers.begin();
+ p != bl._buffers.end();
+ ++p)
+ _buffers.push_back(*p);
+ }
+
+ void buffer::list::append(std::istream& in)
+ {
+ while (!in.eof()) {
+ std::string s;
+ getline(in, s);
+ append(s.c_str(), s.length());
+ if (s.length())
+ append("\n", 1);
+ }
+ }
+
+ void buffer::list::prepend_zero(unsigned len)
+ {
+ ptr bp(len);
+ bp.zero(false);
+ _len += len;
+ _buffers.emplace_front(std::move(bp));
+ }
+
+ void buffer::list::append_zero(unsigned len)
+ {
+ ptr bp(len);
+ bp.zero(false);
+ append(std::move(bp));
+ }
+
+
+ /*
+ * get a char
+ */
+ const char& buffer::list::operator[](unsigned n) const
+ {
+ if (n >= _len)
+ throw end_of_buffer();
+
+ for (std::list<ptr>::const_iterator p = _buffers.begin();
+ p != _buffers.end();
+ ++p) {
+ if (n >= p->length()) {
+ n -= p->length();
+ continue;
+ }
+ return (*p)[n];
+ }
+ ceph_abort();
+ }
+
+ /*
+ * return a contiguous ptr to whole bufferlist contents.
+ */
+ char *buffer::list::c_str()
+ {
+ if (_buffers.empty())
+ return 0; // no buffers
+
+ std::list<ptr>::const_iterator iter = _buffers.begin();
+ ++iter;
+
+ if (iter != _buffers.end())
+ rebuild();
+ return _buffers.front().c_str(); // good, we're already contiguous.
+ }
+
+ string buffer::list::to_str() const {
+ string s;
+ s.reserve(length());
+ for (std::list<ptr>::const_iterator p = _buffers.begin();
+ p != _buffers.end();
+ ++p) {
+ if (p->length()) {
+ s.append(p->c_str(), p->length());
+ }
+ }
+ return s;
+ }
+
+ char *buffer::list::get_contiguous(unsigned orig_off, unsigned len)
+ {
+ if (orig_off + len > length())
+ throw end_of_buffer();
+
+ if (len == 0) {
+ return 0;
+ }
+
+ unsigned off = orig_off;
+ std::list<ptr>::iterator curbuf = _buffers.begin();
+ while (off > 0 && off >= curbuf->length()) {
+ off -= curbuf->length();
+ ++curbuf;
+ }
+
+ if (off + len > curbuf->length()) {
+ bufferlist tmp;
+ unsigned l = off + len;
+
+ do {
+ if (l >= curbuf->length())
+ l -= curbuf->length();
+ else
+ l = 0;
+ tmp.append(*curbuf);
+ curbuf = _buffers.erase(curbuf);
+
+ } while (curbuf != _buffers.end() && l > 0);
+
+ assert(l == 0);
+
+ tmp.rebuild();
+ _buffers.insert(curbuf, tmp._buffers.front());
+ return tmp.c_str() + off;
+ }
+
+ last_p = begin(); // we modified _buffers
+
+ return curbuf->c_str() + off;
+ }
+
+ void buffer::list::substr_of(const list& other, unsigned off, unsigned len)
+ {
+ if (off + len > other.length())
+ throw end_of_buffer();
+
+ clear();
+
+ // skip off
+ std::list<ptr>::const_iterator curbuf = other._buffers.begin();
+ while (off > 0 &&
+ off >= curbuf->length()) {
+ // skip this buffer
+ //cout << "skipping over " << *curbuf << std::endl;
+ off -= (*curbuf).length();
+ ++curbuf;
+ }
+ assert(len == 0 || curbuf != other._buffers.end());
+
+ while (len > 0) {
+ // partial?
+ if (off + len < curbuf->length()) {
+ //cout << "copying partial of " << *curbuf << std::endl;
+ _buffers.push_back( ptr( *curbuf, off, len ) );
+ _len += len;
+ break;
+ }
+
+ // through end
+ //cout << "copying end (all?) of " << *curbuf << std::endl;
+ unsigned howmuch = curbuf->length() - off;
+ _buffers.push_back( ptr( *curbuf, off, howmuch ) );
+ _len += howmuch;
+ len -= howmuch;
+ off = 0;
+ ++curbuf;
+ }
+ }
+
+ // funky modifer
+ void buffer::list::splice(unsigned off, unsigned len, list *claim_by /*, bufferlist& replace_with */)
+ { // fixme?
+ if (len == 0)
+ return;
+
+ if (off >= length())
+ throw end_of_buffer();
+
+ assert(len > 0);
+ //cout << "splice off " << off << " len " << len << " ... mylen = " << length() << std::endl;
+
+ // skip off
+ std::list<ptr>::iterator curbuf = _buffers.begin();
+ while (off > 0) {
+ assert(curbuf != _buffers.end());
+ if (off >= (*curbuf).length()) {
+ // skip this buffer
+ //cout << "off = " << off << " skipping over " << *curbuf << std::endl;
+ off -= (*curbuf).length();
+ ++curbuf;
+ } else {
+ // somewhere in this buffer!
+ //cout << "off = " << off << " somewhere in " << *curbuf << std::endl;
+ break;
+ }
+ }
+
+ if (off) {
+ // add a reference to the front bit
+ // insert it before curbuf (which we'll hose)
+ //cout << "keeping front " << off << " of " << *curbuf << std::endl;
+ _buffers.insert( curbuf, ptr( *curbuf, 0, off ) );
+ _len += off;
+ }
+
+ while (len > 0) {
+ // partial?
+ if (off + len < (*curbuf).length()) {
+ //cout << "keeping end of " << *curbuf << ", losing first " << off+len << std::endl;
+ if (claim_by)
+ claim_by->append( *curbuf, off, len );
+ (*curbuf).set_offset( off+len + (*curbuf).offset() ); // ignore beginning big
+ (*curbuf).set_length( (*curbuf).length() - (len+off) );
+ _len -= off+len;
+ //cout << " now " << *curbuf << std::endl;
+ break;
+ }
+
+ // hose though the end
+ unsigned howmuch = (*curbuf).length() - off;
+ //cout << "discarding " << howmuch << " of " << *curbuf << std::endl;
+ if (claim_by)
+ claim_by->append( *curbuf, off, howmuch );
+ _len -= (*curbuf).length();
+ _buffers.erase( curbuf++ );
+ len -= howmuch;
+ off = 0;
+ }
+
+ // splice in *replace (implement me later?)
+
+ last_p = begin(); // just in case we were in the removed region.
+ }
+
+ void buffer::list::write(int off, int len, std::ostream& out) const
+ {
+ list s;
+ s.substr_of(*this, off, len);
+ for (std::list<ptr>::const_iterator it = s._buffers.begin();
+ it != s._buffers.end();
+ ++it)
+ if (it->length())
+ out.write(it->c_str(), it->length());
+ /*iterator p(this, off);
+ while (len > 0 && !p.end()) {
+ int l = p.left_in_this_buf();
+ if (l > len)
+ l = len;
+ out.write(p.c_str(), l);
+ len -= l;
+ }*/
+ }
+
+void buffer::list::encode_base64(buffer::list& o)
+{
+ bufferptr bp(length() * 4 / 3 + 3);
+ int l = ceph_armor(bp.c_str(), bp.c_str() + bp.length(), c_str(), c_str() + length());
+ bp.set_length(l);
+ o.push_back(std::move(bp));
+}
+
+void buffer::list::decode_base64(buffer::list& e)
+{
+ bufferptr bp(4 + ((e.length() * 3) / 4));
+ int l = ceph_unarmor(bp.c_str(), bp.c_str() + bp.length(), e.c_str(), e.c_str() + e.length());
+ if (l < 0) {
+ std::ostringstream oss;
+ oss << "decode_base64: decoding failed:\n";
+ hexdump(oss);
+ throw buffer::malformed_input(oss.str().c_str());
+ }
+ assert(l <= (int)bp.length());
+ bp.set_length(l);
+ push_back(std::move(bp));
+}
+
+
+
+int buffer::list::read_file(const char *fn, std::string *error)
+{
+ int fd = TEMP_FAILURE_RETRY(::open(fn, O_RDONLY));
+ if (fd < 0) {
+ int err = errno;
+ std::ostringstream oss;
+ oss << "can't open " << fn << ": " << cpp_strerror(err);
+ *error = oss.str();
+ return -err;
+ }
+
+ struct stat st;
+ memset(&st, 0, sizeof(st));
+ if (::fstat(fd, &st) < 0) {
+ int err = errno;
+ std::ostringstream oss;
+ oss << "bufferlist::read_file(" << fn << "): stat error: "
+ << cpp_strerror(err);
+ *error = oss.str();
+ VOID_TEMP_FAILURE_RETRY(::close(fd));
+ return -err;
+ }
+
+ ssize_t ret = read_fd(fd, st.st_size);
+ if (ret < 0) {
+ std::ostringstream oss;
+ oss << "bufferlist::read_file(" << fn << "): read error:"
+ << cpp_strerror(ret);
+ *error = oss.str();
+ VOID_TEMP_FAILURE_RETRY(::close(fd));
+ return ret;
+ }
+ else if (ret != st.st_size) {
+ // Premature EOF.
+ // Perhaps the file changed between stat() and read()?
+ std::ostringstream oss;
+ oss << "bufferlist::read_file(" << fn << "): warning: got premature EOF.";
+ *error = oss.str();
+ // not actually an error, but weird
+ }
+ VOID_TEMP_FAILURE_RETRY(::close(fd));
+ return 0;
+}
+
+ssize_t buffer::list::read_fd(int fd, size_t len)
+{
+ // try zero copy first
+ if (false && read_fd_zero_copy(fd, len) == 0) {
+ // TODO fix callers to not require correct read size, which is not
+ // available for raw_pipe until we actually inspect the data
+ return 0;
+ }
+ bufferptr bp = buffer::create(len);
+ ssize_t ret = safe_read(fd, (void*)bp.c_str(), len);
+ if (ret >= 0) {
+ bp.set_length(ret);
+ append(std::move(bp));
+ }
+ return ret;
+}
+
+int buffer::list::read_fd_zero_copy(int fd, size_t len)
+{
+#ifdef CEPH_HAVE_SPLICE
+ try {
+ append(buffer::create_zero_copy(len, fd, NULL));
+ } catch (buffer::error_code &e) {
+ return e.code;
+ } catch (buffer::malformed_input &e) {
+ return -EIO;
+ }
+ return 0;
+#else
+ return -ENOTSUP;
+#endif
+}
+
+int buffer::list::write_file(const char *fn, int mode)
+{
+ int fd = TEMP_FAILURE_RETRY(::open(fn, O_WRONLY|O_CREAT|O_TRUNC, mode));
+ if (fd < 0) {
+ int err = errno;
+ cerr << "bufferlist::write_file(" << fn << "): failed to open file: "
+ << cpp_strerror(err) << std::endl;
+ return -err;
+ }
+ int ret = write_fd(fd);
+ if (ret) {
+ cerr << "bufferlist::write_fd(" << fn << "): write_fd error: "
+ << cpp_strerror(ret) << std::endl;
+ VOID_TEMP_FAILURE_RETRY(::close(fd));
+ return ret;
+ }
+ if (TEMP_FAILURE_RETRY(::close(fd))) {
+ int err = errno;
+ cerr << "bufferlist::write_file(" << fn << "): close error: "
+ << cpp_strerror(err) << std::endl;
+ return -err;
+ }
+ return 0;
+}
+
+static int do_writev(int fd, struct iovec *vec, uint64_t offset, unsigned veclen, unsigned bytes)
+{
+ ssize_t r = 0;
+ while (bytes > 0) {
+#ifdef HAVE_PWRITEV
+ r = ::pwritev(fd, vec, veclen, offset);
+#else
+ r = ::lseek64(fd, offset, SEEK_SET);
+ if (r != offset) {
+ r = -errno;
+ return r;
+ }
+ r = ::writev(fd, vec, veclen);
+#endif
+ if (r < 0) {
+ if (errno == EINTR)
+ continue;
+ return -errno;
+ }
+
+ bytes -= r;
+ offset += r;
+ if (bytes == 0) break;
+
+ while (r > 0) {
+ if (vec[0].iov_len <= (size_t)r) {
+ // drain this whole item
+ r -= vec[0].iov_len;
+ ++vec;
+ --veclen;
+ } else {
+ vec[0].iov_base = (char *)vec[0].iov_base + r;
+ vec[0].iov_len -= r;
+ break;
+ }
+ }
+ }
+ return 0;
+}
+
+int buffer::list::write_fd(int fd) const
+{
+ if (can_zero_copy())
+ return write_fd_zero_copy(fd);
+
+ // use writev!
+ iovec iov[IOV_MAX];
+ int iovlen = 0;
+ ssize_t bytes = 0;
+
+ std::list<ptr>::const_iterator p = _buffers.begin();
+ while (p != _buffers.end()) {
+ if (p->length() > 0) {
+ iov[iovlen].iov_base = (void *)p->c_str();
+ iov[iovlen].iov_len = p->length();
+ bytes += p->length();
+ iovlen++;
+ }
+ ++p;
+
+ if (iovlen == IOV_MAX ||
+ p == _buffers.end()) {
+ iovec *start = iov;
+ int num = iovlen;
+ ssize_t wrote;
+ retry:
+ wrote = ::writev(fd, start, num);
+ if (wrote < 0) {
+ int err = errno;
+ if (err == EINTR)
+ goto retry;
+ return -err;
+ }
+ if (wrote < bytes) {
+ // partial write, recover!
+ while ((size_t)wrote >= start[0].iov_len) {
+ wrote -= start[0].iov_len;
+ bytes -= start[0].iov_len;
+ start++;
+ num--;
+ }
+ if (wrote > 0) {
+ start[0].iov_len -= wrote;
+ start[0].iov_base = (char *)start[0].iov_base + wrote;
+ bytes -= wrote;
+ }
+ goto retry;
+ }
+ iovlen = 0;
+ bytes = 0;
+ }
+ }
+ return 0;
+}
+
+int buffer::list::write_fd(int fd, uint64_t offset) const
+{
+ iovec iov[IOV_MAX];
+
+ std::list<ptr>::const_iterator p = _buffers.begin();
+ uint64_t left_pbrs = _buffers.size();
+ while (left_pbrs) {
+ ssize_t bytes = 0;
+ unsigned iovlen = 0;
+ uint64_t size = MIN(left_pbrs, IOV_MAX);
+ left_pbrs -= size;
+ while (size > 0) {
+ iov[iovlen].iov_base = (void *)p->c_str();
+ iov[iovlen].iov_len = p->length();
+ iovlen++;
+ bytes += p->length();
+ ++p;
+ size--;
+ }
+
+ int r = do_writev(fd, iov, offset, iovlen, bytes);
+ if (r < 0)
+ return r;
+ offset += bytes;
+ }
+ return 0;
+}
+
+int buffer::list::write_fd_zero_copy(int fd) const
+{
+ if (!can_zero_copy())
+ return -ENOTSUP;
+ /* pass offset to each call to avoid races updating the fd seek
+ * position, since the I/O may be non-blocking
+ */
+ int64_t offset = ::lseek(fd, 0, SEEK_CUR);
+ int64_t *off_p = &offset;
+ if (offset < 0 && errno != ESPIPE)
+ return -errno;
+ if (errno == ESPIPE)
+ off_p = NULL;
+ for (std::list<ptr>::const_iterator it = _buffers.begin();
+ it != _buffers.end(); ++it) {
+ int r = it->zero_copy_to_fd(fd, off_p);
+ if (r < 0)
+ return r;
+ if (off_p)
+ offset += it->length();
+ }
+ return 0;
+}
+
+__u32 buffer::list::crc32c(__u32 crc) const
+{
+ for (std::list<ptr>::const_iterator it = _buffers.begin();
+ it != _buffers.end();
+ ++it) {
+ if (it->length()) {
+ raw *r = it->get_raw();
+ pair<size_t, size_t> ofs(it->offset(), it->offset() + it->length());
+ pair<uint32_t, uint32_t> ccrc;
+ if (r->get_crc(ofs, &ccrc)) {
+ if (ccrc.first == crc) {
+ // got it already
+ crc = ccrc.second;
+ if (buffer_track_crc)
+ buffer_cached_crc++;
+ } else {
+ /* If we have cached crc32c(buf, v) for initial value v,
+ * we can convert this to a different initial value v' by:
+ * crc32c(buf, v') = crc32c(buf, v) ^ adjustment
+ * where adjustment = crc32c(0*len(buf), v ^ v')
+ *
+ * http://crcutil.googlecode.com/files/crc-doc.1.0.pdf
+ * note, u for our crc32c implementation is 0
+ */
+ crc = ccrc.second ^ ceph_crc32c(ccrc.first ^ crc, NULL, it->length());
+ if (buffer_track_crc)
+ buffer_cached_crc_adjusted++;
+ }
+ } else {
+ if (buffer_track_crc)
+ buffer_missed_crc++;
+ uint32_t base = crc;
+ crc = ceph_crc32c(crc, (unsigned char*)it->c_str(), it->length());
+ r->set_crc(ofs, make_pair(base, crc));
+ }
+ }
+ }
+ return crc;
+}
+
+void buffer::list::invalidate_crc()
+{
+ for (std::list<ptr>::const_iterator p = _buffers.begin(); p != _buffers.end(); ++p) {
+ raw *r = p->get_raw();
+ if (r) {
+ r->invalidate_crc();
+ }
+ }
+}
+
+/**
+ * Binary write all contents to a C++ stream
+ */
+void buffer::list::write_stream(std::ostream &out) const
+{
+ for (std::list<ptr>::const_iterator p = _buffers.begin(); p != _buffers.end(); ++p) {
+ if (p->length() > 0) {
+ out.write(p->c_str(), p->length());
+ }
+ }
+}
+
+
+void buffer::list::hexdump(std::ostream &out, bool trailing_newline) const
+{
+ if (!length())
+ return;
+
+ std::ios_base::fmtflags original_flags = out.flags();
+
+ // do our best to match the output of hexdump -C, for better
+ // diff'ing!
+
+ out.setf(std::ios::right);
+ out.fill('0');
+
+ unsigned per = 16;
+ bool was_zeros = false, did_star = false;
+ for (unsigned o=0; o<length(); o += per) {
+ bool row_is_zeros = false;
+ if (o + per < length()) {
+ row_is_zeros = true;
+ for (unsigned i=0; i<per && o+i<length(); i++) {
+ if ((*this)[o+i]) {
+ row_is_zeros = false;
+ }
+ }
+ if (row_is_zeros) {
+ if (was_zeros) {
+ if (!did_star) {
+ out << "\n*";
+ did_star = true;
+ }
+ continue;
+ }
+ was_zeros = true;
+ } else {
+ was_zeros = false;
+ did_star = false;
+ }
+ }
+ if (o)
+ out << "\n";
+ out << std::hex << std::setw(8) << o << " ";
+
+ unsigned i;
+ for (i=0; i<per && o+i<length(); i++) {
+ if (i == 8)
+ out << ' ';
+ out << " " << std::setw(2) << ((unsigned)(*this)[o+i] & 0xff);
+ }
+ for (; i<per; i++) {
+ if (i == 8)
+ out << ' ';
+ out << " ";
+ }
+
+ out << " |";
+ for (i=0; i<per && o+i<length(); i++) {
+ char c = (*this)[o+i];
+ if (isupper(c) || islower(c) || isdigit(c) || c == ' ' || ispunct(c))
+ out << c;
+ else
+ out << '.';
+ }
+ out << '|' << std::dec;
+ }
+ if (trailing_newline) {
+ out << "\n" << std::hex << std::setw(8) << length();
+ out << "\n";
+ }
+
+ out.flags(original_flags);
+}
+
+
+buffer::list buffer::list::static_from_mem(char* c, size_t l) {
+ list bl;
+ bl.push_back(ptr(create_static(l, c)));
+ return bl;
+}
+
+buffer::list buffer::list::static_from_cstring(char* c) {
+ return static_from_mem(c, std::strlen(c));
+}
+
+buffer::list buffer::list::static_from_string(string& s) {
+ // C++14 just has string::data return a char* from a non-const
+ // string.
+ return static_from_mem(const_cast<char*>(s.data()), s.length());
+ // But the way buffer::list mostly doesn't work in a sane way with
+ // const makes me generally sad.
+}
+
+std::ostream& buffer::operator<<(std::ostream& out, const buffer::raw &r) {
+ return out << "buffer::raw(" << (void*)r.data << " len " << r.len << " nref " << r.nref.load() << ")";
+}
+
+std::ostream& buffer::operator<<(std::ostream& out, const buffer::ptr& bp) {
+ if (bp.have_raw())
+ out << "buffer::ptr(" << bp.offset() << "~" << bp.length()
+ << " " << (void*)bp.c_str()
+ << " in raw " << (void*)bp.raw_c_str()
+ << " len " << bp.raw_length()
+ << " nref " << bp.raw_nref() << ")";
+ else
+ out << "buffer:ptr(" << bp.offset() << "~" << bp.length() << " no raw)";
+ return out;
+}
+
+std::ostream& buffer::operator<<(std::ostream& out, const buffer::list& bl) {
+ out << "buffer::list(len=" << bl.length() << "," << std::endl;
+
+ std::list<buffer::ptr>::const_iterator it = bl.buffers().begin();
+ while (it != bl.buffers().end()) {
+ out << "\t" << *it;
+ if (++it == bl.buffers().end()) break;
+ out << "," << std::endl;
+ }
+ out << std::endl << ")";
+ return out;
+}
+
+std::ostream& buffer::operator<<(std::ostream& out, const buffer::error& e)
+{
+ return out << e.what();
+}
+
+MEMPOOL_DEFINE_OBJECT_FACTORY(buffer::raw_malloc, buffer_raw_malloc,
+ buffer_meta);
+MEMPOOL_DEFINE_OBJECT_FACTORY(buffer::raw_mmap_pages, buffer_raw_mmap_pagse,
+ buffer_meta);
+MEMPOOL_DEFINE_OBJECT_FACTORY(buffer::raw_posix_aligned,
+ buffer_raw_posix_aligned, buffer_meta);
+#ifdef CEPH_HAVE_SPLICE
+MEMPOOL_DEFINE_OBJECT_FACTORY(buffer::raw_pipe, buffer_raw_pipe, buffer_meta);
+#endif
+MEMPOOL_DEFINE_OBJECT_FACTORY(buffer::raw_char, buffer_raw_char, buffer_meta);
+MEMPOOL_DEFINE_OBJECT_FACTORY(buffer::raw_claimed_char, buffer_raw_claimed_char,
+ buffer_meta);
+MEMPOOL_DEFINE_OBJECT_FACTORY(buffer::raw_unshareable, buffer_raw_unshareable,
+ buffer_meta);
+MEMPOOL_DEFINE_OBJECT_FACTORY(buffer::raw_static, buffer_raw_static,
+ buffer_meta);
+
Code Review / stor4nfv.git / blobdiff