1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
2 // vim: ts=8 sw=2 smarttab
4 * Ceph - scalable distributed file system
6 * Copyright (C) 2014 Sebastien Ponce <sebastien.ponce@cern.ch>
8 * This is free software; you can redistribute it and/or
9 * modify it under the terms of the GNU Lesser General Public
10 * License version 2.1, as published by the Free Software
11 * Foundation. See file COPYING.
15 #include <boost/algorithm/string/replace.hpp>
17 #include "libradosstriper/RadosStriperImpl.h"
25 #include "include/types.h"
26 #include "include/uuid.h"
27 #include "include/ceph_fs.h"
28 #include "common/dout.h"
29 #include "common/strtol.h"
30 #include "osdc/Striper.h"
31 #include "librados/AioCompletionImpl.h"
32 #include <cls/lock/cls_lock_client.h>
35 * This file contents the actual implementation of the rados striped objects interface.
37 * Striped objects are stored in rados in a set of regular rados objects, after their
38 * content has been striped using the osdc/Striper interface.
40 * The external attributes of the striped object are mapped to the attributes of the
41 * first underlying object. This first object has a set of extra external attributes
42 * storing the layout of the striped object for future read back. These attributes are :
43 * - striper.layout.object_size : the size of rados objects used.
44 * Must be a multiple of striper.layout.stripe_unit
45 * - striper.layout.stripe_unit : the size of a stripe unit
46 * - striper.layout.stripe_count : the number of stripes used
47 * - striper.size : total striped object size
49 * In general operations on striped objects are not atomic.
50 * However, a certain number of safety guards have been put to make the interface closer
52 * - each data operation takes a shared lock on the first rados object for the
53 * whole time of the operation
54 * - the remove and trunc operations take an exclusive lock on the first rados object
55 * for the whole time of the operation
56 * This makes sure that no removal/truncation of a striped object occurs while
57 * data operations are happening and vice versa. It thus makes sure that the layout
58 * of a striped object does not change during data operation, which is essential for
61 * Still the writing to a striped object is not atomic. This means in particular that
62 * the size of an object may not be in sync with its content at all times.
63 * As the size is always garanteed to be updated first and in an atomic way, and as
64 * sparse striped objects are supported (see below), what will typically happen is
65 * that a reader that comes too soon after a write will read 0s instead of the actual
68 * Note that remove handles the pieces of the striped object in reverse order,
69 * so that the head object is removed last, making the completion of the deletion atomic.
71 * Striped objects can be sparse, typically in case data was written at the end of the
72 * striped object only. In such a case, some rados objects constituing the striped object
73 * may be missing. Other can be partial (only the beginning will have data)
74 * When dealing with such sparse striped files, missing objects are detected and
75 * considered as full of 0s. They are however not created until real data is written
78 * There are a number of missing features/improvements that could be implemented.
79 * Here are some ideas :
80 * - implementation of missing entry points (compared to rados)
81 * In particular : clone_range, sparse_read, exec, aio_flush_async, tmaps, omaps, ...
85 #define dout_subsys ceph_subsys_rados
87 #define dout_prefix *_dout << "libradosstriper: "
89 /// size of xattr buffer
90 #define XATTR_BUFFER_SIZE 32
92 /// names of the different xattr entries
93 #define XATTR_LAYOUT_STRIPE_UNIT "striper.layout.stripe_unit"
94 #define XATTR_LAYOUT_STRIPE_COUNT "striper.layout.stripe_count"
95 #define XATTR_LAYOUT_OBJECT_SIZE "striper.layout.object_size"
96 #define XATTR_SIZE "striper.size"
97 #define LOCK_PREFIX "lock."
99 /// name of the lock used on objects to ensure layout stability during IO
100 #define RADOS_LOCK_NAME "striper.lock"
102 /// format of the extension of rados objects created for a given striped object
103 #define RADOS_OBJECT_EXTENSION_FORMAT ".%016llx"
105 /// default object layout
106 struct ceph_file_layout default_file_layout = {
107 init_le32(1<<22), // fl_stripe_unit
108 init_le32(1), // fl_stripe_count
109 init_le32(1<<22), // fl_object_size
110 init_le32(0), // fl_cas_hash
111 init_le32(0), // fl_object_stripe_unit
112 init_le32(-1), // fl_unused
113 init_le32(-1), // fl_pg_pool
116 using libradosstriper::MultiAioCompletionImplPtr;
120 ///////////////////////// CompletionData /////////////////////////////
123 * struct handling the data needed to pass to the call back
124 * function in asynchronous operations
126 struct CompletionData : RefCountedObject {
128 CompletionData(libradosstriper::RadosStriperImpl * striper,
129 const std::string& soid,
130 const std::string& lockCookie,
131 librados::AioCompletionImpl *userCompletion = 0,
134 ~CompletionData() override;
136 void complete(int r);
137 /// striper to be used to handle the write completion
138 libradosstriper::RadosStriperImpl *m_striper;
139 /// striped object concerned by the write operation
141 /// shared lock to be released at completion
142 std::string m_lockCookie;
143 /// completion handler
144 librados::IoCtxImpl::C_aio_Complete *m_ack;
147 CompletionData::CompletionData
148 (libradosstriper::RadosStriperImpl* striper,
149 const std::string& soid,
150 const std::string& lockCookie,
151 librados::AioCompletionImpl *userCompletion,
153 RefCountedObject(striper->cct(), n),
154 m_striper(striper), m_soid(soid), m_lockCookie(lockCookie), m_ack(0) {
156 if (userCompletion) {
157 m_ack = new librados::IoCtxImpl::C_aio_Complete(userCompletion);
158 userCompletion->io = striper->m_ioCtxImpl;
162 CompletionData::~CompletionData() {
163 if (m_ack) delete m_ack;
167 void CompletionData::complete(int r) {
168 if (m_ack) m_ack->finish(r);
172 * struct handling the data needed to pass to the call back
173 * function in asynchronous read operations
175 struct ReadCompletionData : CompletionData {
176 /// bufferlist containing final result
178 /// extents that will be read
179 std::vector<ObjectExtent>* m_extents;
180 /// intermediate results
181 std::vector<bufferlist>* m_resultbl;
182 /// return code of read completion, to be remembered until unlocking happened
184 /// completion object for the unlocking of the striped object at the end of the read
185 librados::AioCompletion *m_unlockCompletion;
187 ReadCompletionData(libradosstriper::RadosStriperImpl * striper,
188 const std::string& soid,
189 const std::string& lockCookie,
190 librados::AioCompletionImpl *userCompletion,
192 std::vector<ObjectExtent>* extents,
193 std::vector<bufferlist>* resultbl,
196 ~ReadCompletionData() override;
197 /// complete method for when reading is over
198 void complete_read(int r);
199 /// complete method for when object is unlocked
200 void complete_unlock(int r);
203 ReadCompletionData::ReadCompletionData
204 (libradosstriper::RadosStriperImpl* striper,
205 const std::string& soid,
206 const std::string& lockCookie,
207 librados::AioCompletionImpl *userCompletion,
209 std::vector<ObjectExtent>* extents,
210 std::vector<bufferlist>* resultbl,
212 CompletionData(striper, soid, lockCookie, userCompletion, n),
213 m_bl(bl), m_extents(extents), m_resultbl(resultbl), m_readRc(0),
214 m_unlockCompletion(0) {}
216 ReadCompletionData::~ReadCompletionData() {
217 m_unlockCompletion->release();
222 void ReadCompletionData::complete_read(int r) {
223 // gather data into final buffer
224 Striper::StripedReadResult readResult;
225 vector<bufferlist>::iterator bit = m_resultbl->begin();
226 for (vector<ObjectExtent>::iterator eit = m_extents->begin();
227 eit != m_extents->end();
229 readResult.add_partial_result(m_striper->cct(), *bit, eit->buffer_extents);
232 readResult.assemble_result(m_striper->cct(), *m_bl, true);
233 // Remember return code
237 void ReadCompletionData::complete_unlock(int r) {
238 // call parent's completion method
239 // Note that we ignore the return code of the unlock as we cannot do much about it
240 CompletionData::complete(m_readRc?m_readRc:m_bl->length());
244 * struct handling the data needed to pass to the call back
245 * function in asynchronous write operations
247 struct WriteCompletionData : CompletionData {
248 /// safe completion handler
249 librados::IoCtxImpl::C_aio_Complete *m_safe;
250 /// return code of write completion, to be remembered until unlocking happened
252 /// completion object for the unlocking of the striped object at the end of the write
253 librados::AioCompletion *m_unlockCompletion;
255 WriteCompletionData(libradosstriper::RadosStriperImpl * striper,
256 const std::string& soid,
257 const std::string& lockCookie,
258 librados::AioCompletionImpl *userCompletion,
261 ~WriteCompletionData() override;
262 /// complete method for when writing is over
263 void complete_write(int r);
264 /// complete method for when object is unlocked
265 void complete_unlock(int r);
270 WriteCompletionData::WriteCompletionData
271 (libradosstriper::RadosStriperImpl* striper,
272 const std::string& soid,
273 const std::string& lockCookie,
274 librados::AioCompletionImpl *userCompletion,
276 CompletionData(striper, soid, lockCookie, userCompletion, n), m_safe(0),
277 m_unlockCompletion(0), m_writeRc(0) {
278 if (userCompletion) {
279 m_safe = new librados::IoCtxImpl::C_aio_Complete(userCompletion);
283 WriteCompletionData::~WriteCompletionData() {
284 m_unlockCompletion->release();
285 if (m_safe) delete m_safe;
288 void WriteCompletionData::complete_unlock(int r) {
289 // call parent's completion method
290 // Note that we ignore the return code of the unlock as we cannot do much about it
291 CompletionData::complete(m_writeRc);
294 void WriteCompletionData::complete_write(int r) {
295 // Remember return code
299 void WriteCompletionData::safe(int r) {
300 if (m_safe) m_safe->finish(r);
303 struct RemoveCompletionData : CompletionData {
308 * note that the constructed object will take ownership of the lock
310 RemoveCompletionData(libradosstriper::RadosStriperImpl * striper,
311 const std::string& soid,
312 const std::string& lockCookie,
313 librados::AioCompletionImpl *userCompletion,
315 CompletionData(striper, soid, lockCookie, userCompletion), flags(flags) {}
319 * struct handling the data needed to pass to the call back
320 * function in asynchronous truncate operations
322 struct TruncateCompletionData : RefCountedObject {
324 TruncateCompletionData(libradosstriper::RadosStriperImpl* striper,
325 const std::string& soid,
327 RefCountedObject(striper->cct()),
328 m_striper(striper), m_soid(soid), m_size(size) {
332 ~TruncateCompletionData() override {
335 /// striper to be used
336 libradosstriper::RadosStriperImpl *m_striper;
337 /// striped object concerned by the truncate operation
339 /// the final size of the truncated object
344 * struct handling the data needed to pass to the call back
345 * function in asynchronous read operations of a Rados File
347 struct RadosReadCompletionData : RefCountedObject {
349 RadosReadCompletionData(MultiAioCompletionImplPtr multiAioCompl,
350 uint64_t expectedBytes,
352 CephContext *context,
354 RefCountedObject(context, n),
355 m_multiAioCompl(multiAioCompl), m_expectedBytes(expectedBytes), m_bl(bl) {}
356 /// the multi asynch io completion object to be used
357 MultiAioCompletionImplPtr m_multiAioCompl;
358 /// the expected number of bytes
359 uint64_t m_expectedBytes;
360 /// the bufferlist object where data have been written
365 * struct handling (most of) the data needed to pass to the call back
366 * function in asynchronous stat operations.
367 * Inherited by the actual type for adding time information in different
368 * versions (time_t or struct timespec)
370 struct BasicStatCompletionData : CompletionData {
372 BasicStatCompletionData(libradosstriper::RadosStriperImpl* striper,
373 const std::string& soid,
374 librados::AioCompletionImpl *userCompletion,
375 libradosstriper::MultiAioCompletionImpl *multiCompletion,
378 CompletionData(striper, soid, "", userCompletion, n),
379 m_multiCompletion(multiCompletion), m_psize(psize),
380 m_statRC(0), m_getxattrRC(0) {};
381 // MultiAioCompletionImpl used to handle the double aysnc
382 // call in the back (stat + getxattr)
383 libradosstriper::MultiAioCompletionImpl *m_multiCompletion;
384 // where to store the size of first objct
385 // this will be ignored but we need a place to store it when
386 // async stat is called
387 uint64_t m_objectSize;
388 // where to store the file size
390 /// the bufferlist object used for the getxattr call
392 /// return code of the stat
394 /// return code of the getxattr
399 * struct handling the data needed to pass to the call back
400 * function in asynchronous stat operations.
401 * Simple templated extension of BasicStatCompletionData.
402 * The template parameter is the type of the time information
403 * (used with time_t for stat and struct timespec for stat2)
405 template<class TimeType>
406 struct StatCompletionData : BasicStatCompletionData {
408 StatCompletionData(libradosstriper::RadosStriperImpl* striper,
409 const std::string& soid,
410 librados::AioCompletionImpl *userCompletion,
411 libradosstriper::MultiAioCompletionImpl *multiCompletion,
415 BasicStatCompletionData(striper, soid, userCompletion, multiCompletion, psize, n),
417 // where to store the file time
422 * struct handling the data needed to pass to the call back
423 * function in asynchronous remove operations of a Rados File
425 struct RadosRemoveCompletionData : RefCountedObject {
427 RadosRemoveCompletionData(MultiAioCompletionImplPtr multiAioCompl,
428 CephContext *context) :
429 RefCountedObject(context, 2),
430 m_multiAioCompl(multiAioCompl) {};
431 /// the multi asynch io completion object to be used
432 MultiAioCompletionImplPtr m_multiAioCompl;
438 ///////////////////////// constructor /////////////////////////////
440 libradosstriper::RadosStriperImpl::RadosStriperImpl(librados::IoCtx& ioctx, librados::IoCtxImpl *ioctx_impl) :
441 m_refCnt(0),lock("RadosStriper Refcont", false, false), m_radosCluster(ioctx), m_ioCtx(ioctx), m_ioCtxImpl(ioctx_impl),
442 m_layout(default_file_layout) {}
444 ///////////////////////// layout /////////////////////////////
446 int libradosstriper::RadosStriperImpl::setObjectLayoutStripeUnit
447 (unsigned int stripe_unit)
449 /* stripe unit must be non-zero, 64k increment */
450 if (!stripe_unit || (stripe_unit & (CEPH_MIN_STRIPE_UNIT-1)))
452 m_layout.fl_stripe_unit = stripe_unit;
456 int libradosstriper::RadosStriperImpl::setObjectLayoutStripeCount
457 (unsigned int stripe_count)
459 /* stripe count must be non-zero */
462 m_layout.fl_stripe_count = stripe_count;
466 int libradosstriper::RadosStriperImpl::setObjectLayoutObjectSize
467 (unsigned int object_size)
469 /* object size must be non-zero, 64k increment */
470 if (!object_size || (object_size & (CEPH_MIN_STRIPE_UNIT-1)))
472 /* object size must be a multiple of stripe unit */
473 if (object_size < m_layout.fl_stripe_unit ||
474 object_size % m_layout.fl_stripe_unit)
476 m_layout.fl_object_size = object_size;
480 ///////////////////////// xattrs /////////////////////////////
482 int libradosstriper::RadosStriperImpl::getxattr(const object_t& soid,
486 std::string firstObjOid = getObjectId(soid, 0);
487 return m_ioCtx.getxattr(firstObjOid, name, bl);
490 int libradosstriper::RadosStriperImpl::setxattr(const object_t& soid,
494 std::string firstObjOid = getObjectId(soid, 0);
495 return m_ioCtx.setxattr(firstObjOid, name, bl);
498 int libradosstriper::RadosStriperImpl::getxattrs(const object_t& soid,
499 map<string, bufferlist>& attrset)
501 std::string firstObjOid = getObjectId(soid, 0);
502 int rc = m_ioCtx.getxattrs(firstObjOid, attrset);
504 // cleanup internal attributes dedicated to striping and locking
505 attrset.erase(XATTR_LAYOUT_STRIPE_UNIT);
506 attrset.erase(XATTR_LAYOUT_STRIPE_COUNT);
507 attrset.erase(XATTR_LAYOUT_OBJECT_SIZE);
508 attrset.erase(XATTR_SIZE);
509 attrset.erase(std::string(LOCK_PREFIX) + RADOS_LOCK_NAME);
513 int libradosstriper::RadosStriperImpl::rmxattr(const object_t& soid,
516 std::string firstObjOid = getObjectId(soid, 0);
517 return m_ioCtx.rmxattr(firstObjOid, name);
520 ///////////////////////// io /////////////////////////////
522 int libradosstriper::RadosStriperImpl::write(const std::string& soid,
523 const bufferlist& bl,
527 // open the object. This will create it if needed, retrieve its layout
528 // and size and take a shared lock on it
529 ceph_file_layout layout;
530 std::string lockCookie;
531 int rc = createAndOpenStripedObject(soid, &layout, len+off, &lockCookie, true);
533 return write_in_open_object(soid, layout, lockCookie, bl, len, off);
536 int libradosstriper::RadosStriperImpl::append(const std::string& soid,
537 const bufferlist& bl,
540 // open the object. This will create it if needed, retrieve its layout
541 // and size and take a shared lock on it
542 ceph_file_layout layout;
544 std::string lockCookie;
545 int rc = openStripedObjectForWrite(soid, &layout, &size, &lockCookie, false);
547 return write_in_open_object(soid, layout, lockCookie, bl, len, size);
550 int libradosstriper::RadosStriperImpl::write_full(const std::string& soid,
551 const bufferlist& bl)
553 int rc = trunc(soid, 0);
554 if (rc && rc != -ENOENT) return rc; // ENOENT is obviously ok
555 return write(soid, bl, bl.length(), 0);
558 int libradosstriper::RadosStriperImpl::read(const std::string& soid,
563 // create a completion object
564 librados::AioCompletionImpl c;
565 // call asynchronous method
566 int rc = aio_read(soid, &c, bl, len, off);
567 // and wait for completion
569 // wait for completion
570 c.wait_for_complete_and_cb();
572 rc = c.get_return_value();
577 ///////////////////////// asynchronous io /////////////////////////////
579 int libradosstriper::RadosStriperImpl::aio_write(const std::string& soid,
580 librados::AioCompletionImpl *c,
581 const bufferlist& bl,
585 ceph_file_layout layout;
586 std::string lockCookie;
587 int rc = createAndOpenStripedObject(soid, &layout, len+off, &lockCookie, true);
589 return aio_write_in_open_object(soid, c, layout, lockCookie, bl, len, off);
592 int libradosstriper::RadosStriperImpl::aio_append(const std::string& soid,
593 librados::AioCompletionImpl *c,
594 const bufferlist& bl,
597 ceph_file_layout layout;
599 std::string lockCookie;
600 int rc = openStripedObjectForWrite(soid, &layout, &size, &lockCookie, false);
602 // create a completion object
603 return aio_write_in_open_object(soid, c, layout, lockCookie, bl, len, size);
606 int libradosstriper::RadosStriperImpl::aio_write_full(const std::string& soid,
607 librados::AioCompletionImpl *c,
608 const bufferlist& bl)
610 int rc = trunc(soid, 0);
612 return aio_write(soid, c, bl, bl.length(), 0);
615 static void rados_read_aio_unlock_complete(rados_striper_multi_completion_t c, void *arg)
617 auto cdata = reinterpret_cast<ReadCompletionData*>(arg);
618 libradosstriper::MultiAioCompletionImpl *comp =
619 reinterpret_cast<libradosstriper::MultiAioCompletionImpl*>(c);
620 cdata->complete_unlock(comp->rval);
624 static void striper_read_aio_req_complete(rados_striper_multi_completion_t c, void *arg)
626 auto cdata = reinterpret_cast<ReadCompletionData*>(arg);
627 // launch the async unlocking of the object
628 cdata->m_striper->aio_unlockObject(cdata->m_soid, cdata->m_lockCookie, cdata->m_unlockCompletion);
629 // complete the read part in parallel
630 libradosstriper::MultiAioCompletionImpl *comp =
631 reinterpret_cast<libradosstriper::MultiAioCompletionImpl*>(c);
632 cdata->complete_read(comp->rval);
635 static void rados_req_read_safe(rados_completion_t c, void *arg)
637 auto data = reinterpret_cast<RadosReadCompletionData*>(arg);
638 int rc = rados_aio_get_return_value(c);
639 // ENOENT means that we are dealing with a sparse file. This is fine,
640 // data (0s) will be created on the fly by the rados_req_read_complete method
641 if (rc == -ENOENT) rc = 0;
642 auto multiAioComp = data->m_multiAioCompl;
643 multiAioComp->safe_request(rc);
647 static void rados_req_read_complete(rados_completion_t c, void *arg)
649 auto data = reinterpret_cast<RadosReadCompletionData*>(arg);
650 int rc = rados_aio_get_return_value(c);
651 // We need to handle the case of sparse files here
653 // the object did not exist at all. This can happen for sparse files.
654 // we consider we've read 0 bytes and it will fall into next case
657 if (rc >= 0 && (((uint64_t)rc) < data->m_expectedBytes)) {
658 // only partial data were present in the object (or the object did not
659 // even exist if we've gone through previous case).
660 // This is typical of sparse file and we need to complete with 0s.
661 unsigned int lenOfZeros = data->m_expectedBytes-rc;
662 unsigned int existingDataToZero = min(data->m_bl->length()-rc, lenOfZeros);
663 if (existingDataToZero > 0) {
664 data->m_bl->zero(rc, existingDataToZero);
666 if (lenOfZeros > existingDataToZero) {
667 ceph::bufferptr zeros(ceph::buffer::create(lenOfZeros-existingDataToZero));
669 data->m_bl->push_back(zeros);
671 rc = data->m_expectedBytes;
673 auto multiAioComp = data->m_multiAioCompl;
674 multiAioComp->complete_request(rc);
678 int libradosstriper::RadosStriperImpl::aio_read(const std::string& soid,
679 librados::AioCompletionImpl *c,
684 // open the object. This will retrieve its layout and size
685 // and take a shared lock on it
686 ceph_file_layout layout;
688 std::string lockCookie;
689 int rc = openStripedObjectForRead(soid, &layout, &size, &lockCookie);
691 // find out the actual number of bytes we can read
694 // nothing to read ! We are done.
697 read_len = min(len, (size_t)(size-off));
699 // get list of extents to be read from
700 vector<ObjectExtent> *extents = new vector<ObjectExtent>();
702 std::string format = soid;
703 boost::replace_all(format, "%", "%%");
704 format += RADOS_OBJECT_EXTENSION_FORMAT;
706 l.from_legacy(layout);
707 Striper::file_to_extents(cct(), format.c_str(), &l, off, read_len,
711 // create a completion object and transfer ownership of extents and resultbl
712 vector<bufferlist> *resultbl = new vector<bufferlist>(extents->size());
713 ReadCompletionData *cdata = new ReadCompletionData(this, soid, lockCookie, c,
714 bl, extents, resultbl, 1);
717 // create a completion for the unlocking of the striped object at the end of the read
718 librados::AioCompletion *unlock_completion =
719 librados::Rados::aio_create_completion(cdata, rados_read_aio_unlock_complete, 0);
720 cdata->m_unlockCompletion = unlock_completion;
721 // create the multiCompletion object handling the reads
722 MultiAioCompletionImplPtr nc{new libradosstriper::MultiAioCompletionImpl,
724 nc->set_complete_callback(cdata, striper_read_aio_req_complete);
725 // go through the extents
727 for (vector<ObjectExtent>::iterator p = extents->begin(); p != extents->end(); ++p) {
728 // create a buffer list describing where to place data read from current extend
729 bufferlist *oid_bl = &((*resultbl)[i++]);
730 for (vector<pair<uint64_t,uint64_t> >::iterator q = p->buffer_extents.begin();
731 q != p->buffer_extents.end();
733 bufferlist buffer_bl;
734 buffer_bl.substr_of(*bl, q->first, q->second);
735 oid_bl->append(buffer_bl);
737 // read all extends of a given object in one go
739 // we need 2 references on data as both rados_req_read_safe and rados_req_read_complete
741 RadosReadCompletionData *data = new RadosReadCompletionData(nc, p->length, oid_bl, cct(), 2);
742 librados::AioCompletion *rados_completion =
743 librados::Rados::aio_create_completion(data, rados_req_read_complete, rados_req_read_safe);
744 r = m_ioCtx.aio_read(p->oid.name, rados_completion, oid_bl, p->length, p->offset);
745 rados_completion->release();
749 nc->finish_adding_requests();
753 int libradosstriper::RadosStriperImpl::aio_read(const std::string& soid,
754 librados::AioCompletionImpl *c,
759 // create a buffer list and store it inside the completion object
761 c->bl.push_back(buffer::create_static(len, buf));
762 // call the bufferlist version of this method
763 return aio_read(soid, c, &c->bl, len, off);
766 int libradosstriper::RadosStriperImpl::aio_flush()
769 // pass to the rados level
770 ret = m_ioCtx.aio_flush();
773 //wait all CompletionData are released
781 ///////////////////////// stat and deletion /////////////////////////////
783 int libradosstriper::RadosStriperImpl::stat(const std::string& soid, uint64_t *psize, time_t *pmtime)
785 // create a completion object
786 librados::AioCompletionImpl c;
787 // call asynchronous version of stat
788 int rc = aio_stat(soid, &c, psize, pmtime);
790 // wait for completion of the remove
791 c.wait_for_complete();
793 rc = c.get_return_value();
798 static void striper_stat_aio_stat_complete(rados_completion_t c, void *arg) {
799 auto data = reinterpret_cast<BasicStatCompletionData*>(arg);
800 int rc = rados_aio_get_return_value(c);
802 // remember this has failed
805 data->m_multiCompletion->complete_request(rc);
809 static void striper_stat_aio_getxattr_complete(rados_completion_t c, void *arg) {
810 auto data = reinterpret_cast<BasicStatCompletionData*>(arg);
811 int rc = rados_aio_get_return_value(c);
812 // We need to handle the case of sparse files here
814 // remember this has failed
815 data->m_getxattrRC = rc;
817 // this intermediate string allows to add a null terminator before calling strtol
819 std::string strsize(data->m_bl.c_str(), data->m_bl.length());
820 *data->m_psize = strict_strtoll(strsize.c_str(), 10, &err);
822 lderr(data->m_striper->cct()) << XATTR_SIZE << " : " << err << dendl;
823 data->m_getxattrRC = -EINVAL;
827 data->m_multiCompletion->complete_request(rc);
831 static void striper_stat_aio_req_complete(rados_striper_multi_completion_t c,
833 auto data = reinterpret_cast<BasicStatCompletionData*>(arg);
834 if (data->m_statRC) {
835 data->complete(data->m_statRC);
837 if (data->m_getxattrRC < 0) {
838 data->complete(data->m_getxattrRC);
846 template<class TimeType>
847 int libradosstriper::RadosStriperImpl::aio_generic_stat
848 (const std::string& soid,
849 librados::AioCompletionImpl *c,
852 typename libradosstriper::RadosStriperImpl::StatFunction<TimeType>::Type statFunction)
854 // use a MultiAioCompletion object for dealing with the fact
855 // that we'll do 2 asynchronous calls in parallel
856 MultiAioCompletionImplPtr multi_completion{
857 new libradosstriper::MultiAioCompletionImpl, false};
858 // Data object used for passing context to asynchronous calls
859 std::string firstObjOid = getObjectId(soid, 0);
860 StatCompletionData<TimeType> *cdata =
861 new StatCompletionData<TimeType>(this, firstObjOid, c,
862 multi_completion.get(), psize, pmtime, 4);
863 multi_completion->set_complete_callback(cdata, striper_stat_aio_req_complete);
864 // use a regular AioCompletion for the stat async call
865 librados::AioCompletion *stat_completion =
866 librados::Rados::aio_create_completion(cdata, striper_stat_aio_stat_complete, 0);
867 multi_completion->add_safe_request();
868 object_t obj(firstObjOid);
869 int rc = (m_ioCtxImpl->*statFunction)(obj, stat_completion->pc,
870 &cdata->m_objectSize, cdata->m_pmtime);
871 stat_completion->release();
873 // nothing is really started so cancel everything
877 // use a regular AioCompletion for the getxattr async call
878 librados::AioCompletion *getxattr_completion =
879 librados::Rados::aio_create_completion(cdata, striper_stat_aio_getxattr_complete, 0);
880 multi_completion->add_safe_request();
881 // in parallel, get the pmsize from the first object asynchronously
882 rc = m_ioCtxImpl->aio_getxattr(obj, getxattr_completion->pc,
883 XATTR_SIZE, cdata->m_bl);
884 getxattr_completion->release();
885 multi_completion->finish_adding_requests();
887 // the async stat is ongoing, so we need to go on
888 // we mark the getxattr as failed in the data object
889 cdata->m_getxattrRC = rc;
890 multi_completion->complete_request(rc);
897 int libradosstriper::RadosStriperImpl::aio_stat(const std::string& soid,
898 librados::AioCompletionImpl *c,
902 return aio_generic_stat<time_t>(soid, c, psize, pmtime, &librados::IoCtxImpl::aio_stat);
905 int libradosstriper::RadosStriperImpl::stat2(const std::string& soid, uint64_t *psize, struct timespec *pts)
907 // create a completion object
908 librados::AioCompletionImpl c;
909 // call asynchronous version of stat
910 int rc = aio_stat2(soid, &c, psize, pts);
912 // wait for completion of the remove
913 c.wait_for_complete_and_cb();
915 rc = c.get_return_value();
920 int libradosstriper::RadosStriperImpl::aio_stat2(const std::string& soid,
921 librados::AioCompletionImpl *c,
923 struct timespec *pts)
925 return aio_generic_stat<struct timespec>(soid, c, psize, pts, &librados::IoCtxImpl::aio_stat2);
928 static void rados_req_remove_complete(rados_completion_t c, void *arg)
930 auto cdata = reinterpret_cast<RadosRemoveCompletionData*>(arg);
931 int rc = rados_aio_get_return_value(c);
932 // in case the object did not exist, it means we had a sparse file, all is fine
936 cdata->m_multiAioCompl->complete_request(rc);
940 static void rados_req_remove_safe(rados_completion_t c, void *arg)
942 auto cdata = reinterpret_cast<RadosRemoveCompletionData*>(arg);
943 int rc = rados_aio_get_return_value(c);
944 // in case the object did not exist, it means we had a sparse file, all is fine
948 cdata->m_multiAioCompl->safe_request(rc);
952 static void striper_remove_aio_req_complete(rados_striper_multi_completion_t c, void *arg)
954 auto cdata = reinterpret_cast<RemoveCompletionData*>(arg);
955 libradosstriper::MultiAioCompletionImpl *comp =
956 reinterpret_cast<libradosstriper::MultiAioCompletionImpl*>(c);
957 ldout(cdata->m_striper->cct(), 10)
958 << "RadosStriperImpl : striper_remove_aio_req_complete called for "
959 << cdata->m_soid << dendl;
962 // All went fine, synchronously remove first object
963 rc = cdata->m_striper->m_ioCtx.remove(cdata->m_striper->getObjectId(cdata->m_soid, 0),
966 lderr(cdata->m_striper->cct())
967 << "RadosStriperImpl : deletion/truncation incomplete for " << cdata->m_soid
968 << ", as errors were encountered. The file is left present but it's content "
969 << " has been partially removed"
976 int libradosstriper::RadosStriperImpl::remove(const std::string& soid, int flags)
978 // create a completion object
979 librados::AioCompletionImpl c;
980 // call asynchronous version of remove
981 int rc = aio_remove(soid, &c, flags);
983 // wait for completion of the remove
984 c.wait_for_complete_and_cb();
986 rc = c.get_return_value();
991 int libradosstriper::RadosStriperImpl::aio_remove(const std::string& soid,
992 librados::AioCompletionImpl *c,
995 // the RemoveCompletionData object will lock the given soid for the duration
997 std::string lockCookie = getUUID();
998 int rc = m_ioCtx.lock_exclusive(getObjectId(soid, 0), RADOS_LOCK_NAME, lockCookie, "", 0, 0);
1000 // create CompletionData for the async remove call
1001 RemoveCompletionData *cdata = new RemoveCompletionData(this, soid, lockCookie, c, flags);
1002 MultiAioCompletionImplPtr multi_completion{
1003 new libradosstriper::MultiAioCompletionImpl, false};
1004 multi_completion->set_complete_callback(cdata, striper_remove_aio_req_complete);
1005 // call asynchronous internal version of remove
1007 << "RadosStriperImpl : Aio_remove starting for "
1009 rc = internal_aio_remove(soid, multi_completion);
1013 int libradosstriper::RadosStriperImpl::internal_aio_remove(
1014 const std::string& soid,
1015 MultiAioCompletionImplPtr multi_completion,
1018 std::string firstObjOid = getObjectId(soid, 0);
1020 // check size and get number of rados objects to delete
1021 uint64_t nb_objects = 0;
1023 int rc = getxattr(soid, XATTR_SIZE, bl2);
1025 // no object size (or not able to get it)
1026 // try to find the number of object "by hand"
1029 while (!m_ioCtx.stat(getObjectId(soid, nb_objects), &psize, &pmtime)) {
1033 // count total number of rados objects in the striped object
1035 // this intermediate string allows to add a null terminator before calling strtol
1036 std::string strsize(bl2.c_str(), bl2.length());
1037 uint64_t size = strict_strtoll(strsize.c_str(), 10, &err);
1039 lderr(cct()) << XATTR_SIZE << " : " << err << dendl;
1043 uint64_t object_size = m_layout.fl_object_size;
1044 uint64_t su = m_layout.fl_stripe_unit;
1045 uint64_t stripe_count = m_layout.fl_stripe_count;
1046 uint64_t nb_complete_sets = size / (object_size*stripe_count);
1047 uint64_t remaining_data = size % (object_size*stripe_count);
1048 uint64_t remaining_stripe_units = (remaining_data + su -1) / su;
1049 uint64_t remaining_objects = std::min(remaining_stripe_units, stripe_count);
1050 nb_objects = nb_complete_sets * stripe_count + remaining_objects;
1052 // delete rados objects in reverse order
1053 // Note that we do not drop the first object. This one will only be dropped
1054 // if all other removals have been successful, and this is done in the
1055 // callback of the multi_completion object
1057 for (int i = nb_objects-1; i >= 1; i--) {
1058 multi_completion->add_request();
1059 RadosRemoveCompletionData *data =
1060 new RadosRemoveCompletionData(multi_completion, cct());
1061 librados::AioCompletion *rados_completion =
1062 librados::Rados::aio_create_completion(data,
1063 rados_req_remove_complete,
1064 rados_req_remove_safe);
1066 rcr = m_ioCtx.aio_remove(getObjectId(soid, i), rados_completion);
1068 rcr = m_ioCtx.aio_remove(getObjectId(soid, i), rados_completion, flags);
1070 rados_completion->release();
1071 if (rcr < 0 and -ENOENT != rcr) {
1072 lderr(cct()) << "RadosStriperImpl::remove : deletion incomplete for " << soid
1073 << ", as " << getObjectId(soid, i) << " could not be deleted (rc=" << rc << ")"
1078 // we are over adding requests to the multi_completion object
1079 multi_completion->finish_adding_requests();
1082 } catch (ErrorCode &e) {
1083 // errror caught when trying to take the exclusive lock
1089 int libradosstriper::RadosStriperImpl::trunc(const std::string& soid, uint64_t size)
1091 // lock the object in exclusive mode
1092 std::string firstObjOid = getObjectId(soid, 0);
1093 librados::ObjectWriteOperation op;
1095 std::string lockCookie = RadosStriperImpl::getUUID();
1096 utime_t dur = utime_t();
1097 rados::cls::lock::lock(&op, RADOS_LOCK_NAME, LOCK_EXCLUSIVE, lockCookie, "", "", dur, 0);
1098 int rc = m_ioCtx.operate(firstObjOid, &op);
1100 // load layout and size
1101 ceph_file_layout layout;
1102 uint64_t original_size;
1103 rc = internal_get_layout_and_size(firstObjOid, &layout, &original_size);
1105 if (size < original_size) {
1106 rc = truncate(soid, original_size, size, layout);
1107 } else if (size > original_size) {
1108 rc = grow(soid, original_size, size, layout);
1111 // unlock object, ignore return code as we cannot do much
1112 m_ioCtx.unlock(firstObjOid, RADOS_LOCK_NAME, lockCookie);
1118 ///////////////////////// private helpers /////////////////////////////
1120 std::string libradosstriper::RadosStriperImpl::getObjectId(const object_t& soid,
1121 long long unsigned objectno)
1123 std::ostringstream s;
1124 s << soid << '.' << std::setfill ('0') << std::setw(16) << std::hex << objectno;
1128 void libradosstriper::RadosStriperImpl::unlockObject(const std::string& soid,
1129 const std::string& lockCookie)
1131 // unlock the shared lock on the first rados object
1132 std::string firstObjOid = getObjectId(soid, 0);
1133 m_ioCtx.unlock(firstObjOid, RADOS_LOCK_NAME, lockCookie);
1136 void libradosstriper::RadosStriperImpl::aio_unlockObject(const std::string& soid,
1137 const std::string& lockCookie,
1138 librados::AioCompletion *c)
1140 // unlock the shared lock on the first rados object
1141 std::string firstObjOid = getObjectId(soid, 0);
1142 m_ioCtx.aio_unlock(firstObjOid, RADOS_LOCK_NAME, lockCookie, c);
1145 static void rados_write_aio_unlock_complete(rados_striper_multi_completion_t c, void *arg)
1147 auto cdata = reinterpret_cast<WriteCompletionData*>(arg);
1148 libradosstriper::MultiAioCompletionImpl *comp =
1149 reinterpret_cast<libradosstriper::MultiAioCompletionImpl*>(c);
1150 cdata->complete_unlock(comp->rval);
1154 static void striper_write_aio_req_complete(rados_striper_multi_completion_t c, void *arg)
1156 auto cdata = reinterpret_cast<WriteCompletionData*>(arg);
1157 // launch the async unlocking of the object
1158 cdata->m_striper->aio_unlockObject(cdata->m_soid, cdata->m_lockCookie, cdata->m_unlockCompletion);
1159 // complete the write part in parallel
1160 libradosstriper::MultiAioCompletionImpl *comp =
1161 reinterpret_cast<libradosstriper::MultiAioCompletionImpl*>(c);
1162 cdata->complete_write(comp->rval);
1166 static void striper_write_aio_req_safe(rados_striper_multi_completion_t c, void *arg)
1168 auto cdata = reinterpret_cast<WriteCompletionData*>(arg);
1169 libradosstriper::MultiAioCompletionImpl *comp =
1170 reinterpret_cast<libradosstriper::MultiAioCompletionImpl*>(c);
1171 cdata->safe(comp->rval);
1175 int libradosstriper::RadosStriperImpl::write_in_open_object(const std::string& soid,
1176 const ceph_file_layout& layout,
1177 const std::string& lockCookie,
1178 const bufferlist& bl,
1181 // create a completion object to be passed to the callbacks of the multicompletion
1182 // we need 3 references as striper_write_aio_req_complete will release two and
1183 // striper_write_aio_req_safe will release one
1184 WriteCompletionData *cdata = new WriteCompletionData(this, soid, lockCookie, 0, 3);
1185 cdata->get(); // local ref
1186 // create a completion object for the unlocking of the striped object at the end of the write
1187 librados::AioCompletion *unlock_completion =
1188 librados::Rados::aio_create_completion(cdata, rados_write_aio_unlock_complete, 0);
1189 cdata->m_unlockCompletion = unlock_completion;
1190 // create the multicompletion that will handle the write completion
1191 MultiAioCompletionImplPtr c{new libradosstriper::MultiAioCompletionImpl,
1193 c->set_complete_callback(cdata, striper_write_aio_req_complete);
1194 c->set_safe_callback(cdata, striper_write_aio_req_safe);
1195 // call the asynchronous API
1196 int rc = internal_aio_write(soid, c, bl, len, off, layout);
1198 // wait for completion and safety of data
1199 c->wait_for_complete_and_cb();
1200 c->wait_for_safe_and_cb();
1201 // wait for the unlocking
1202 unlock_completion->wait_for_complete();
1204 rc = c->get_return_value();
1210 int libradosstriper::RadosStriperImpl::aio_write_in_open_object(const std::string& soid,
1211 librados::AioCompletionImpl *c,
1212 const ceph_file_layout& layout,
1213 const std::string& lockCookie,
1214 const bufferlist& bl,
1217 // create a completion object to be passed to the callbacks of the multicompletion
1218 // we need 3 references as striper_write_aio_req_complete will release two and
1219 // striper_write_aio_req_safe will release one
1220 WriteCompletionData *cdata = new WriteCompletionData(this, soid, lockCookie, c, 3);
1221 cdata->get(); // local ref
1223 c->io = m_ioCtxImpl;
1224 // create a completion object for the unlocking of the striped object at the end of the write
1225 librados::AioCompletion *unlock_completion =
1226 librados::Rados::aio_create_completion(cdata, rados_write_aio_unlock_complete, 0);
1227 cdata->m_unlockCompletion = unlock_completion;
1228 // create the multicompletion that will handle the write completion
1229 libradosstriper::MultiAioCompletionImplPtr nc{
1230 new libradosstriper::MultiAioCompletionImpl, false};
1231 nc->set_complete_callback(cdata, striper_write_aio_req_complete);
1232 nc->set_safe_callback(cdata, striper_write_aio_req_safe);
1233 // internal asynchronous API
1234 int rc = internal_aio_write(soid, nc, bl, len, off, layout);
1239 static void rados_req_write_safe(rados_completion_t c, void *arg)
1241 libradosstriper::MultiAioCompletionImpl *comp =
1242 reinterpret_cast<libradosstriper::MultiAioCompletionImpl*>(arg);
1243 comp->safe_request(rados_aio_get_return_value(c));
1246 static void rados_req_write_complete(rados_completion_t c, void *arg)
1248 libradosstriper::MultiAioCompletionImpl *comp =
1249 reinterpret_cast<libradosstriper::MultiAioCompletionImpl*>(arg);
1250 comp->complete_request(rados_aio_get_return_value(c));
1254 libradosstriper::RadosStriperImpl::internal_aio_write(const std::string& soid,
1255 libradosstriper::MultiAioCompletionImplPtr c,
1256 const bufferlist& bl,
1259 const ceph_file_layout& layout)
1262 // Do not try anything if we are called with empty buffer,
1263 // file_to_extents would raise an exception
1265 // get list of extents to be written to
1266 vector<ObjectExtent> extents;
1267 std::string format = soid;
1268 boost::replace_all(format, "%", "%%");
1269 format += RADOS_OBJECT_EXTENSION_FORMAT;
1271 l.from_legacy(layout);
1272 Striper::file_to_extents(cct(), format.c_str(), &l, off, len, 0, extents);
1273 // go through the extents
1274 for (vector<ObjectExtent>::iterator p = extents.begin(); p != extents.end(); ++p) {
1275 // assemble pieces of a given object into a single buffer list
1277 for (vector<pair<uint64_t,uint64_t> >::iterator q = p->buffer_extents.begin();
1278 q != p->buffer_extents.end();
1280 bufferlist buffer_bl;
1281 buffer_bl.substr_of(bl, q->first, q->second);
1282 oid_bl.append(buffer_bl);
1284 // and write the object
1286 librados::AioCompletion *rados_completion =
1287 librados::Rados::aio_create_completion(c.get(),
1288 rados_req_write_complete,
1289 rados_req_write_safe);
1290 r = m_ioCtx.aio_write(p->oid.name, rados_completion, oid_bl,
1291 p->length, p->offset);
1292 rados_completion->release();
1297 c->finish_adding_requests();
1301 int libradosstriper::RadosStriperImpl::extract_uint32_attr
1302 (std::map<std::string, bufferlist> &attrs,
1303 const std::string& key,
1306 std::map<std::string, bufferlist>::iterator attrsIt = attrs.find(key);
1307 if (attrsIt != attrs.end()) {
1308 // this intermediate string allows to add a null terminator before calling strtol
1309 std::string strvalue(attrsIt->second.c_str(), attrsIt->second.length());
1311 *value = strict_strtol(strvalue.c_str(), 10, &err);
1313 lderr(cct()) << key << " : " << err << dendl;
1322 int libradosstriper::RadosStriperImpl::extract_sizet_attr
1323 (std::map<std::string, bufferlist> &attrs,
1324 const std::string& key,
1327 std::map<std::string, bufferlist>::iterator attrsIt = attrs.find(key);
1328 if (attrsIt != attrs.end()) {
1329 // this intermediate string allows to add a null terminator before calling strtol
1330 std::string strvalue(attrsIt->second.c_str(), attrsIt->second.length());
1332 *value = strict_strtoll(strvalue.c_str(), 10, &err);
1334 lderr(cct()) << key << " : " << err << dendl;
1343 int libradosstriper::RadosStriperImpl::internal_get_layout_and_size(
1344 const std::string& oid,
1345 ceph_file_layout *layout,
1348 // get external attributes of the first rados object
1349 std::map<std::string, bufferlist> attrs;
1350 int rc = m_ioCtx.getxattrs(oid, attrs);
1352 // deal with stripe_unit
1353 rc = extract_uint32_attr(attrs, XATTR_LAYOUT_STRIPE_UNIT, &layout->fl_stripe_unit);
1355 // deal with stripe_count
1356 rc = extract_uint32_attr(attrs, XATTR_LAYOUT_STRIPE_COUNT, &layout->fl_stripe_count);
1358 // deal with object_size
1359 rc = extract_uint32_attr(attrs, XATTR_LAYOUT_OBJECT_SIZE, &layout->fl_object_size);
1363 rc = extract_sizet_attr(attrs, XATTR_SIZE, &ssize);
1368 // make valgrind happy by setting unused fl_pg_pool
1369 layout->fl_pg_pool = 0;
1373 int libradosstriper::RadosStriperImpl::openStripedObjectForRead(
1374 const std::string& soid,
1375 ceph_file_layout *layout,
1377 std::string *lockCookie)
1379 // take a lock the first rados object, if it exists and gets its size
1380 // check, lock and size reading must be atomic and are thus done within a single operation
1381 librados::ObjectWriteOperation op;
1383 *lockCookie = getUUID();
1384 utime_t dur = utime_t();
1385 rados::cls::lock::lock(&op, RADOS_LOCK_NAME, LOCK_SHARED, *lockCookie, "Tag", "", dur, 0);
1386 std::string firstObjOid = getObjectId(soid, 0);
1387 int rc = m_ioCtx.operate(firstObjOid, &op);
1389 // error case (including -ENOENT)
1392 rc = internal_get_layout_and_size(firstObjOid, layout, size);
1394 unlockObject(soid, *lockCookie);
1395 lderr(cct()) << "RadosStriperImpl::openStripedObjectForRead : "
1396 << "could not load layout and size for "
1397 << soid << " : rc = " << rc << dendl;
1402 int libradosstriper::RadosStriperImpl::openStripedObjectForWrite(const std::string& soid,
1403 ceph_file_layout *layout,
1405 std::string *lockCookie,
1406 bool isFileSizeAbsolute)
1408 // take a lock the first rados object, if it exists
1409 // check and lock must be atomic and are thus done within a single operation
1410 librados::ObjectWriteOperation op;
1412 *lockCookie = getUUID();
1413 utime_t dur = utime_t();
1414 rados::cls::lock::lock(&op, RADOS_LOCK_NAME, LOCK_SHARED, *lockCookie, "Tag", "", dur, 0);
1415 std::string firstObjOid = getObjectId(soid, 0);
1416 int rc = m_ioCtx.operate(firstObjOid, &op);
1418 if (rc == -ENOENT) {
1419 // object does not exist, delegate to createEmptyStripedObject
1420 int rc = createAndOpenStripedObject(soid, layout, *size, lockCookie, isFileSizeAbsolute);
1421 // return original size
1430 rc = internal_get_layout_and_size(firstObjOid, layout, &curSize);
1432 unlockObject(soid, *lockCookie);
1433 lderr(cct()) << "RadosStriperImpl::openStripedObjectForWrite : "
1434 << "could not load layout and size for "
1435 << soid << " : rc = " << rc << dendl;
1438 // atomically update object size, only if smaller than current one
1439 if (!isFileSizeAbsolute)
1441 librados::ObjectWriteOperation writeOp;
1442 writeOp.cmpxattr(XATTR_SIZE, LIBRADOS_CMPXATTR_OP_GT, *size);
1443 std::ostringstream oss;
1446 bl.append(oss.str());
1447 writeOp.setxattr(XATTR_SIZE, bl);
1448 rc = m_ioCtx.operate(firstObjOid, &writeOp);
1449 // return current size
1451 // handle case where objectsize is already bigger than size
1452 if (-ECANCELED == rc)
1455 unlockObject(soid, *lockCookie);
1456 lderr(cct()) << "RadosStriperImpl::openStripedObjectForWrite : "
1457 << "could not set new size for "
1458 << soid << " : rc = " << rc << dendl;
1463 int libradosstriper::RadosStriperImpl::createAndOpenStripedObject(const std::string& soid,
1464 ceph_file_layout *layout,
1466 std::string *lockCookie,
1467 bool isFileSizeAbsolute)
1469 // build atomic write operation
1470 librados::ObjectWriteOperation writeOp;
1471 writeOp.create(true);
1473 std::ostringstream oss_object_size;
1474 oss_object_size << m_layout.fl_object_size;
1475 bufferlist bl_object_size;
1476 bl_object_size.append(oss_object_size.str());
1477 writeOp.setxattr(XATTR_LAYOUT_OBJECT_SIZE, bl_object_size);
1479 std::ostringstream oss_stripe_unit;
1480 oss_stripe_unit << m_layout.fl_stripe_unit;
1481 bufferlist bl_stripe_unit;
1482 bl_stripe_unit.append(oss_stripe_unit.str());
1483 writeOp.setxattr(XATTR_LAYOUT_STRIPE_UNIT, bl_stripe_unit);
1485 std::ostringstream oss_stripe_count;
1486 oss_stripe_count << m_layout.fl_stripe_count;
1487 bufferlist bl_stripe_count;
1488 bl_stripe_count.append(oss_stripe_count.str());
1489 writeOp.setxattr(XATTR_LAYOUT_STRIPE_COUNT, bl_stripe_count);
1491 std::ostringstream oss_size;
1492 oss_size << (isFileSizeAbsolute?size:0);
1494 bl_size.append(oss_size.str());
1495 writeOp.setxattr(XATTR_SIZE, bl_size);
1496 // effectively change attributes
1497 std::string firstObjOid = getObjectId(soid, 0);
1498 int rc = m_ioCtx.operate(firstObjOid, &writeOp);
1499 // in case of error (but no EEXIST which would mean the object existed), return
1500 if (rc && -EEXIST != rc) return rc;
1501 // Otherwise open the object
1502 uint64_t fileSize = size;
1503 return openStripedObjectForWrite(soid, layout, &fileSize, lockCookie, isFileSizeAbsolute);
1506 static void striper_truncate_aio_req_complete(rados_striper_multi_completion_t c, void *arg)
1508 auto cdata = reinterpret_cast<TruncateCompletionData*>(arg);
1509 libradosstriper::MultiAioCompletionImpl *comp =
1510 reinterpret_cast<libradosstriper::MultiAioCompletionImpl*>(c);
1511 if (0 == comp->rval) {
1512 // all went fine, change size in the external attributes
1513 std::ostringstream oss;
1514 oss << cdata->m_size;
1516 bl.append(oss.str());
1517 cdata->m_striper->setxattr(cdata->m_soid, XATTR_SIZE, bl);
1522 int libradosstriper::RadosStriperImpl::truncate(const std::string& soid,
1523 uint64_t original_size,
1525 ceph_file_layout &layout)
1527 TruncateCompletionData *cdata = new TruncateCompletionData(this, soid, size);
1528 libradosstriper::MultiAioCompletionImplPtr multi_completion{
1529 new libradosstriper::MultiAioCompletionImpl, false};
1530 multi_completion->set_complete_callback(cdata, striper_truncate_aio_req_complete);
1531 // call asynchrous version of truncate
1532 int rc = aio_truncate(soid, multi_completion, original_size, size, layout);
1533 // wait for completion of the truncation
1534 multi_completion->finish_adding_requests();
1535 multi_completion->wait_for_complete_and_cb();
1538 rc = multi_completion->get_return_value();
1543 int libradosstriper::RadosStriperImpl::aio_truncate
1544 (const std::string& soid,
1545 libradosstriper::MultiAioCompletionImplPtr multi_completion,
1546 uint64_t original_size,
1548 ceph_file_layout &layout)
1550 // handle the underlying rados objects. 3 cases here :
1551 // -- the objects belonging to object sets entirely located
1552 // before the truncation are unchanged
1553 // -- the objects belonging to the object set where the
1554 // truncation took place are truncated or removed
1555 // -- the objects belonging to object sets entirely located
1556 // after the truncation are removed
1557 // Note that we do it backward and that we change the size in
1558 // the external attributes only at the end. This make sure that
1559 // no rados object stays behind if we remove the striped object
1560 // after a truncation has failed
1561 uint64_t trunc_objectsetno = size / layout.fl_object_size / layout.fl_stripe_count;
1562 uint64_t last_objectsetno = original_size / layout.fl_object_size / layout.fl_stripe_count;
1563 bool exists = false;
1564 for (int64_t objectno = (last_objectsetno+1) * layout.fl_stripe_count-1;
1565 objectno >= (int64_t)((trunc_objectsetno + 1) * layout.fl_stripe_count);
1567 // if no object existed so far, check object existence
1569 uint64_t nb_full_object_set = objectno / layout.fl_stripe_count;
1570 uint64_t object_index_in_set = objectno % layout.fl_stripe_count;
1571 uint64_t set_start_off = nb_full_object_set * layout.fl_object_size * layout.fl_stripe_count;
1572 uint64_t object_start_off = set_start_off + object_index_in_set * layout.fl_stripe_unit;
1573 exists = (original_size > object_start_off);
1576 // remove asynchronously
1577 multi_completion->add_request();
1578 RadosRemoveCompletionData *data =
1579 new RadosRemoveCompletionData(multi_completion, cct());
1580 librados::AioCompletion *rados_completion =
1581 librados::Rados::aio_create_completion(data,
1582 rados_req_remove_complete,
1583 rados_req_remove_safe);
1584 int rc = m_ioCtx.aio_remove(getObjectId(soid, objectno), rados_completion);
1585 rados_completion->release();
1586 // in case the object did not exist, it means we had a sparse file, all is fine
1587 if (rc && rc != -ENOENT) return rc;
1590 for (int64_t objectno = ((trunc_objectsetno + 1) * layout.fl_stripe_count) -1;
1591 objectno >= (int64_t)(trunc_objectsetno * layout.fl_stripe_count);
1593 // if no object existed so far, check object existence
1595 uint64_t object_start_off = ((objectno / layout.fl_stripe_count) * layout.fl_object_size) +
1596 ((objectno % layout.fl_stripe_count) * layout.fl_stripe_unit);
1597 exists = (original_size > object_start_off);
1602 l.from_legacy(layout);
1603 uint64_t new_object_size = Striper::object_truncate_size(cct(), &l, objectno, size);
1605 if (new_object_size > 0 or 0 == objectno) {
1606 // trunc is synchronous as there is no async version
1607 // but note that only a single object will be truncated
1608 // reducing the overload to a fixed amount
1609 rc = m_ioCtx.trunc(getObjectId(soid, objectno), new_object_size);
1611 // removes are asynchronous in order to speed up truncations of big files
1612 multi_completion->add_request();
1613 RadosRemoveCompletionData *data =
1614 new RadosRemoveCompletionData(multi_completion, cct());
1615 librados::AioCompletion *rados_completion =
1616 librados::Rados::aio_create_completion(data,
1617 rados_req_remove_complete,
1618 rados_req_remove_safe);
1619 rc = m_ioCtx.aio_remove(getObjectId(soid, objectno), rados_completion);
1620 rados_completion->release();
1622 // in case the object did not exist, it means we had a sparse file, all is fine
1623 if (rc && rc != -ENOENT) return rc;
1629 int libradosstriper::RadosStriperImpl::grow(const std::string& soid,
1630 uint64_t original_size,
1632 ceph_file_layout &layout)
1634 // handle the underlying rados objects. As we support sparse objects,
1635 // we only have to change the size in the external attributes
1636 std::ostringstream oss;
1639 bl.append(oss.str());
1640 int rc = m_ioCtx.setxattr(getObjectId(soid, 0), XATTR_SIZE, bl);
1644 std::string libradosstriper::RadosStriperImpl::getUUID()
1647 uuid.generate_random();
1650 return std::string(suuid);