1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
2 // vim: ts=8 sw=2 smarttab
4 #include "boost/algorithm/string.hpp"
7 #include "common/debug.h"
8 #include "common/errno.h"
9 #include "common/perf_counters.h"
10 #include "BlockDevice.h"
11 #include "Allocator.h"
12 #include "include/assert.h"
14 #define dout_context cct
15 #define dout_subsys ceph_subsys_bluefs
17 #define dout_prefix *_dout << "bluefs "
19 MEMPOOL_DEFINE_OBJECT_FACTORY(BlueFS::File, bluefs_file, bluefs);
20 MEMPOOL_DEFINE_OBJECT_FACTORY(BlueFS::Dir, bluefs_dir, bluefs);
21 MEMPOOL_DEFINE_OBJECT_FACTORY(BlueFS::FileWriter, bluefs_file_writer, bluefs);
22 MEMPOOL_DEFINE_OBJECT_FACTORY(BlueFS::FileReaderBuffer,
23 bluefs_file_reader_buffer, bluefs);
24 MEMPOOL_DEFINE_OBJECT_FACTORY(BlueFS::FileReader, bluefs_file_reader, bluefs);
25 MEMPOOL_DEFINE_OBJECT_FACTORY(BlueFS::FileLock, bluefs_file_lock, bluefs);
28 BlueFS::BlueFS(CephContext* cct)
33 block_total(MAX_BDEV, 0)
54 void BlueFS::_init_logger()
56 PerfCountersBuilder b(cct, "bluefs",
57 l_bluefs_first, l_bluefs_last);
58 b.add_u64_counter(l_bluefs_gift_bytes, "gift_bytes",
59 "Bytes gifted from BlueStore");
60 b.add_u64_counter(l_bluefs_reclaim_bytes, "reclaim_bytes",
61 "Bytes reclaimed by BlueStore");
62 b.add_u64(l_bluefs_db_total_bytes, "db_total_bytes",
63 "Total bytes (main db device)",
64 "b", PerfCountersBuilder::PRIO_USEFUL);
65 b.add_u64(l_bluefs_db_used_bytes, "db_used_bytes",
66 "Used bytes (main db device)",
67 "u", PerfCountersBuilder::PRIO_USEFUL);
68 b.add_u64(l_bluefs_wal_total_bytes, "wal_total_bytes",
69 "Total bytes (wal device)",
70 "walb", PerfCountersBuilder::PRIO_USEFUL);
71 b.add_u64(l_bluefs_wal_used_bytes, "wal_used_bytes",
72 "Used bytes (wal device)",
73 "walu", PerfCountersBuilder::PRIO_USEFUL);
74 b.add_u64(l_bluefs_slow_total_bytes, "slow_total_bytes",
75 "Total bytes (slow device)",
76 "slob", PerfCountersBuilder::PRIO_USEFUL);
77 b.add_u64(l_bluefs_slow_used_bytes, "slow_used_bytes",
78 "Used bytes (slow device)",
79 "slou", PerfCountersBuilder::PRIO_USEFUL);
80 b.add_u64(l_bluefs_num_files, "num_files", "File count",
81 "f", PerfCountersBuilder::PRIO_USEFUL);
82 b.add_u64(l_bluefs_log_bytes, "log_bytes", "Size of the metadata log",
83 "jlen", PerfCountersBuilder::PRIO_INTERESTING);
84 b.add_u64_counter(l_bluefs_log_compactions, "log_compactions",
85 "Compactions of the metadata log");
86 b.add_u64_counter(l_bluefs_logged_bytes, "logged_bytes",
87 "Bytes written to the metadata log", "j",
88 PerfCountersBuilder::PRIO_CRITICAL);
89 b.add_u64_counter(l_bluefs_files_written_wal, "files_written_wal",
90 "Files written to WAL");
91 b.add_u64_counter(l_bluefs_files_written_sst, "files_written_sst",
92 "Files written to SSTs");
93 b.add_u64_counter(l_bluefs_bytes_written_wal, "bytes_written_wal",
94 "Bytes written to WAL", "wal",
95 PerfCountersBuilder::PRIO_CRITICAL);
96 b.add_u64_counter(l_bluefs_bytes_written_sst, "bytes_written_sst",
97 "Bytes written to SSTs", "sst",
98 PerfCountersBuilder::PRIO_CRITICAL);
99 logger = b.create_perf_counters();
100 cct->get_perfcounters_collection()->add(logger);
103 void BlueFS::_shutdown_logger()
105 cct->get_perfcounters_collection()->remove(logger);
109 void BlueFS::_update_logger_stats()
111 // we must be holding the lock
112 logger->set(l_bluefs_num_files, file_map.size());
113 logger->set(l_bluefs_log_bytes, log_writer->file->fnode.size);
115 if (alloc[BDEV_WAL]) {
116 logger->set(l_bluefs_wal_total_bytes, block_total[BDEV_WAL]);
117 logger->set(l_bluefs_wal_used_bytes,
118 block_total[BDEV_WAL] - alloc[BDEV_WAL]->get_free());
120 if (alloc[BDEV_DB]) {
121 logger->set(l_bluefs_db_total_bytes, block_total[BDEV_DB]);
122 logger->set(l_bluefs_db_used_bytes,
123 block_total[BDEV_DB] - alloc[BDEV_DB]->get_free());
125 if (alloc[BDEV_SLOW]) {
126 logger->set(l_bluefs_slow_total_bytes, block_total[BDEV_SLOW]);
127 logger->set(l_bluefs_slow_used_bytes,
128 block_total[BDEV_SLOW] - alloc[BDEV_SLOW]->get_free());
132 int BlueFS::add_block_device(unsigned id, const string& path)
134 dout(10) << __func__ << " bdev " << id << " path " << path << dendl;
135 assert(id < bdev.size());
136 assert(bdev[id] == NULL);
137 BlockDevice *b = BlockDevice::create(cct, path, NULL, NULL);
138 int r = b->open(path);
143 dout(1) << __func__ << " bdev " << id << " path " << path
144 << " size " << pretty_si_t(b->get_size()) << "B" << dendl;
146 ioc[id] = new IOContext(cct, NULL);
150 bool BlueFS::bdev_support_label(unsigned id)
152 assert(id < bdev.size());
154 return bdev[id]->supported_bdev_label();
157 uint64_t BlueFS::get_block_device_size(unsigned id)
159 if (id < bdev.size() && bdev[id])
160 return bdev[id]->get_size();
164 void BlueFS::add_block_extent(unsigned id, uint64_t offset, uint64_t length)
166 std::unique_lock<std::mutex> l(lock);
167 dout(1) << __func__ << " bdev " << id
168 << " 0x" << std::hex << offset << "~" << length << std::dec
170 assert(id < bdev.size());
172 assert(bdev[id]->get_size() >= offset + length);
173 block_all[id].insert(offset, length);
174 block_total[id] += length;
176 if (id < alloc.size() && alloc[id]) {
177 log_t.op_alloc_add(id, offset, length);
178 int r = _flush_and_sync_log(l);
180 alloc[id]->init_add_free(offset, length);
184 logger->inc(l_bluefs_gift_bytes, length);
185 dout(10) << __func__ << " done" << dendl;
188 int BlueFS::reclaim_blocks(unsigned id, uint64_t want,
189 AllocExtentVector *extents)
191 std::unique_lock<std::mutex> l(lock);
192 dout(1) << __func__ << " bdev " << id
193 << " want 0x" << std::hex << want << std::dec << dendl;
194 assert(id < alloc.size());
196 int r = alloc[id]->reserve(want);
197 assert(r == 0); // caller shouldn't ask for more than they can get
198 int64_t got = alloc[id]->allocate(want, cct->_conf->bluefs_alloc_size, 0,
200 if (got < (int64_t)want) {
201 alloc[id]->unreserve(want - MAX(0, got));
204 derr << __func__ << " failed to allocate space to return to bluestore"
210 for (auto& p : *extents) {
211 block_all[id].erase(p.offset, p.length);
212 block_total[id] -= p.length;
213 log_t.op_alloc_rm(id, p.offset, p.length);
217 r = _flush_and_sync_log(l);
221 logger->inc(l_bluefs_reclaim_bytes, got);
222 dout(1) << __func__ << " bdev " << id << " want 0x" << std::hex << want
223 << " got " << *extents << dendl;
227 uint64_t BlueFS::get_fs_usage()
229 std::lock_guard<std::mutex> l(lock);
230 uint64_t total_bytes = 0;
231 for (auto& p : file_map) {
232 total_bytes += p.second->fnode.get_allocated();
237 uint64_t BlueFS::get_total(unsigned id)
239 std::lock_guard<std::mutex> l(lock);
240 assert(id < block_all.size());
241 return block_total[id];
244 uint64_t BlueFS::get_free(unsigned id)
246 std::lock_guard<std::mutex> l(lock);
247 assert(id < alloc.size());
248 return alloc[id]->get_free();
251 void BlueFS::dump_perf_counters(Formatter *f)
253 f->open_object_section("bluefs_perf_counters");
254 logger->dump_formatted(f,0);
258 void BlueFS::dump_block_extents(ostream& out)
260 for (unsigned i = 0; i < MAX_BDEV; ++i) {
264 out << i << " : size 0x" << std::hex << bdev[i]->get_size()
265 << " : own 0x" << block_all[i] << std::dec << "\n";
269 void BlueFS::get_usage(vector<pair<uint64_t,uint64_t>> *usage)
271 std::lock_guard<std::mutex> l(lock);
272 usage->resize(bdev.size());
273 for (unsigned id = 0; id < bdev.size(); ++id) {
275 (*usage)[id] = make_pair(0, 0);
278 (*usage)[id].first = alloc[id]->get_free();
279 (*usage)[id].second = block_total[id];
281 (block_total[id] - (*usage)[id].first) * 100 / block_total[id];
282 dout(10) << __func__ << " bdev " << id
283 << " free " << (*usage)[id].first
284 << " (" << pretty_si_t((*usage)[id].first) << "B)"
285 << " / " << (*usage)[id].second
286 << " (" << pretty_si_t((*usage)[id].second) << "B)"
287 << ", used " << used << "%"
292 int BlueFS::get_block_extents(unsigned id, interval_set<uint64_t> *extents)
294 std::lock_guard<std::mutex> l(lock);
295 dout(10) << __func__ << " bdev " << id << dendl;
296 if (id >= block_all.size())
298 *extents = block_all[id];
302 int BlueFS::mkfs(uuid_d osd_uuid)
304 std::unique_lock<std::mutex> l(lock);
306 << " osd_uuid " << osd_uuid
313 super.block_size = bdev[BDEV_DB]->get_block_size();
314 super.osd_uuid = osd_uuid;
315 super.uuid.generate_random();
316 dout(1) << __func__ << " uuid " << super.uuid << dendl;
319 FileRef log_file = new File;
320 log_file->fnode.ino = 1;
321 log_file->fnode.prefer_bdev = BDEV_WAL;
323 log_file->fnode.prefer_bdev,
324 cct->_conf->bluefs_max_log_runway,
325 &log_file->fnode.extents);
326 log_file->fnode.recalc_allocated();
328 log_writer = _create_writer(log_file);
332 for (unsigned bdev = 0; bdev < MAX_BDEV; ++bdev) {
333 interval_set<uint64_t>& p = block_all[bdev];
336 for (interval_set<uint64_t>::iterator q = p.begin(); q != p.end(); ++q) {
337 dout(20) << __func__ << " op_alloc_add " << bdev << " 0x"
338 << std::hex << q.get_start() << "~" << q.get_len() << std::dec
340 log_t.op_alloc_add(bdev, q.get_start(), q.get_len());
343 _flush_and_sync_log(l);
346 super.log_fnode = log_file->fnode;
351 super = bluefs_super_t();
352 _close_writer(log_writer);
359 dout(10) << __func__ << " success" << dendl;
363 void BlueFS::_init_alloc()
365 dout(20) << __func__ << dendl;
366 alloc.resize(MAX_BDEV);
367 pending_release.resize(MAX_BDEV);
368 for (unsigned id = 0; id < bdev.size(); ++id) {
372 assert(bdev[id]->get_size());
373 alloc[id] = Allocator::create(cct, cct->_conf->bluefs_allocator,
374 bdev[id]->get_size(),
375 cct->_conf->bluefs_alloc_size);
376 interval_set<uint64_t>& p = block_all[id];
377 for (interval_set<uint64_t>::iterator q = p.begin(); q != p.end(); ++q) {
378 alloc[id]->init_add_free(q.get_start(), q.get_len());
383 void BlueFS::_stop_alloc()
385 dout(20) << __func__ << dendl;
386 for (auto p : alloc) {
397 dout(1) << __func__ << dendl;
399 int r = _open_super();
401 derr << __func__ << " failed to open super: " << cpp_strerror(r) << dendl;
406 block_all.resize(MAX_BDEV);
408 block_total.resize(MAX_BDEV, 0);
413 derr << __func__ << " failed to replay log: " << cpp_strerror(r) << dendl;
419 for (auto& p : file_map) {
420 dout(30) << __func__ << " noting alloc for " << p.second->fnode << dendl;
421 for (auto& q : p.second->fnode.extents) {
422 alloc[q.bdev]->init_rm_free(q.offset, q.length);
426 // set up the log for future writes
427 log_writer = _create_writer(_get_file(1));
428 assert(log_writer->file->fnode.ino == 1);
429 log_writer->pos = log_writer->file->fnode.size;
430 dout(10) << __func__ << " log write pos set to 0x"
431 << std::hex << log_writer->pos << std::dec
438 super = bluefs_super_t();
442 void BlueFS::umount()
444 dout(1) << __func__ << dendl;
448 _close_writer(log_writer);
454 super = bluefs_super_t();
459 void BlueFS::collect_metadata(map<string,string> *pm)
462 bdev[BDEV_DB]->collect_metadata("bluefs_db_", pm);
464 bdev[BDEV_WAL]->collect_metadata("bluefs_wal_", pm);
466 bdev[BDEV_SLOW]->collect_metadata("bluefs_slow_", pm);
471 std::lock_guard<std::mutex> l(lock);
472 dout(1) << __func__ << dendl;
473 // hrm, i think we check everything on mount...
477 int BlueFS::_write_super()
482 uint32_t crc = bl.crc32c(-1);
484 dout(10) << __func__ << " super block length(encoded): " << bl.length() << dendl;
485 dout(10) << __func__ << " superblock " << super.version << dendl;
486 dout(10) << __func__ << " log_fnode " << super.log_fnode << dendl;
487 assert(bl.length() <= get_super_length());
488 bl.append_zero(get_super_length() - bl.length());
490 bdev[BDEV_DB]->write(get_super_offset(), bl, false);
491 dout(20) << __func__ << " v " << super.version
492 << " crc 0x" << std::hex << crc
493 << " offset 0x" << get_super_offset() << std::dec
498 int BlueFS::_open_super()
500 dout(10) << __func__ << dendl;
503 uint32_t expected_crc, crc;
506 // always the second block
507 r = bdev[BDEV_DB]->read(get_super_offset(), get_super_length(),
508 &bl, ioc[BDEV_DB], false);
512 bufferlist::iterator p = bl.begin();
516 t.substr_of(bl, 0, p.get_off());
519 ::decode(expected_crc, p);
520 if (crc != expected_crc) {
521 derr << __func__ << " bad crc on superblock, expected 0x"
522 << std::hex << expected_crc << " != actual 0x" << crc << std::dec
526 dout(10) << __func__ << " superblock " << super.version << dendl;
527 dout(10) << __func__ << " log_fnode " << super.log_fnode << dendl;
531 int BlueFS::_replay(bool noop)
533 dout(10) << __func__ << (noop ? " NO-OP" : "") << dendl;
534 ino_last = 1; // by the log
541 log_file = _get_file(1);
543 log_file->fnode = super.log_fnode;
544 dout(10) << __func__ << " log_fnode " << super.log_fnode << dendl;
546 FileReader *log_reader = new FileReader(
547 log_file, cct->_conf->bluefs_max_prefetch,
551 assert((log_reader->buf.pos & ~super.block_mask()) == 0);
552 uint64_t pos = log_reader->buf.pos;
553 uint64_t read_pos = pos;
556 int r = _read(log_reader, &log_reader->buf, read_pos, super.block_size,
558 assert(r == (int)super.block_size);
565 bufferlist::iterator p = bl.begin();
573 if (len + 6 > bl.length()) {
574 more = ROUND_UP_TO(len + 6 - bl.length(), super.block_size);
577 if (uuid != super.uuid) {
578 dout(10) << __func__ << " 0x" << std::hex << pos << std::dec
579 << ": stop: uuid " << uuid << " != super.uuid " << super.uuid
583 if (seq != log_seq + 1) {
584 dout(10) << __func__ << " 0x" << std::hex << pos << std::dec
585 << ": stop: seq " << seq << " != expected " << log_seq + 1
590 dout(20) << __func__ << " need 0x" << std::hex << more << std::dec
591 << " more bytes" << dendl;
593 int r = _read(log_reader, &log_reader->buf, read_pos, more, &t, NULL);
595 dout(10) << __func__ << " 0x" << std::hex << pos
596 << ": stop: len is 0x" << bl.length() + more << std::dec
597 << ", which is past eof" << dendl;
600 assert(r == (int)more);
604 bluefs_transaction_t t;
606 bufferlist::iterator p = bl.begin();
609 catch (buffer::error& e) {
610 dout(10) << __func__ << " 0x" << std::hex << pos << std::dec
611 << ": stop: failed to decode: " << e.what()
616 assert(seq == t.seq);
617 dout(10) << __func__ << " 0x" << std::hex << pos << std::dec
618 << ": " << t << dendl;
620 bufferlist::iterator p = t.op_bl.begin();
626 case bluefs_transaction_t::OP_INIT:
627 dout(20) << __func__ << " 0x" << std::hex << pos << std::dec
628 << ": op_init" << dendl;
632 case bluefs_transaction_t::OP_JUMP:
636 ::decode(next_seq, p);
638 dout(20) << __func__ << " 0x" << std::hex << pos << std::dec
639 << ": op_jump seq " << next_seq
640 << " offset 0x" << std::hex << offset << std::dec << dendl;
641 assert(next_seq >= log_seq);
642 log_seq = next_seq - 1; // we will increment it below
643 uint64_t skip = offset - read_pos;
646 int r = _read(log_reader, &log_reader->buf, read_pos, skip, &junk,
648 if (r != (int)skip) {
649 dout(10) << __func__ << " 0x" << std::hex << read_pos
650 << ": stop: failed to skip to " << offset
651 << std::dec << dendl;
652 assert(0 == "problem with op_jump");
658 case bluefs_transaction_t::OP_JUMP_SEQ:
661 ::decode(next_seq, p);
662 dout(20) << __func__ << " 0x" << std::hex << pos << std::dec
663 << ": op_jump_seq " << next_seq << dendl;
664 assert(next_seq >= log_seq);
665 log_seq = next_seq - 1; // we will increment it below
669 case bluefs_transaction_t::OP_ALLOC_ADD:
672 uint64_t offset, length;
676 dout(20) << __func__ << " 0x" << std::hex << pos << std::dec
677 << ": op_alloc_add " << " " << (int)id
678 << ":0x" << std::hex << offset << "~" << length << std::dec
681 block_all[id].insert(offset, length);
682 block_total[id] += length;
683 alloc[id]->init_add_free(offset, length);
688 case bluefs_transaction_t::OP_ALLOC_RM:
691 uint64_t offset, length;
695 dout(20) << __func__ << " 0x" << std::hex << pos << std::dec
696 << ": op_alloc_rm " << " " << (int)id
697 << ":0x" << std::hex << offset << "~" << length << std::dec
700 block_all[id].erase(offset, length);
701 block_total[id] -= length;
702 alloc[id]->init_rm_free(offset, length);
707 case bluefs_transaction_t::OP_DIR_LINK:
709 string dirname, filename;
711 ::decode(dirname, p);
712 ::decode(filename, p);
714 dout(20) << __func__ << " 0x" << std::hex << pos << std::dec
715 << ": op_dir_link " << " " << dirname << "/" << filename
719 FileRef file = _get_file(ino);
720 assert(file->fnode.ino);
721 map<string,DirRef>::iterator q = dir_map.find(dirname);
722 assert(q != dir_map.end());
723 map<string,FileRef>::iterator r = q->second->file_map.find(filename);
724 assert(r == q->second->file_map.end());
725 q->second->file_map[filename] = file;
731 case bluefs_transaction_t::OP_DIR_UNLINK:
733 string dirname, filename;
734 ::decode(dirname, p);
735 ::decode(filename, p);
736 dout(20) << __func__ << " 0x" << std::hex << pos << std::dec
737 << ": op_dir_unlink " << " " << dirname << "/" << filename
740 map<string,DirRef>::iterator q = dir_map.find(dirname);
741 assert(q != dir_map.end());
742 map<string,FileRef>::iterator r = q->second->file_map.find(filename);
743 assert(r != q->second->file_map.end());
744 assert(r->second->refs > 0);
746 q->second->file_map.erase(r);
751 case bluefs_transaction_t::OP_DIR_CREATE:
754 ::decode(dirname, p);
755 dout(20) << __func__ << " 0x" << std::hex << pos << std::dec
756 << ": op_dir_create " << dirname << dendl;
758 map<string,DirRef>::iterator q = dir_map.find(dirname);
759 assert(q == dir_map.end());
760 dir_map[dirname] = new Dir;
765 case bluefs_transaction_t::OP_DIR_REMOVE:
768 ::decode(dirname, p);
769 dout(20) << __func__ << " 0x" << std::hex << pos << std::dec
770 << ": op_dir_remove " << dirname << dendl;
772 map<string,DirRef>::iterator q = dir_map.find(dirname);
773 assert(q != dir_map.end());
774 assert(q->second->file_map.empty());
780 case bluefs_transaction_t::OP_FILE_UPDATE:
782 bluefs_fnode_t fnode;
784 dout(20) << __func__ << " 0x" << std::hex << pos << std::dec
785 << ": op_file_update " << " " << fnode << dendl;
787 FileRef f = _get_file(fnode.ino);
789 if (fnode.ino > ino_last) {
790 ino_last = fnode.ino;
796 case bluefs_transaction_t::OP_FILE_REMOVE:
800 dout(20) << __func__ << " 0x" << std::hex << pos << std::dec
801 << ": op_file_remove " << ino << dendl;
803 auto p = file_map.find(ino);
804 assert(p != file_map.end());
811 derr << __func__ << " 0x" << std::hex << pos << std::dec
812 << ": stop: unrecognized op " << (int)op << dendl;
819 // we successfully replayed the transaction; bump the seq and log size
821 log_file->fnode.size = log_reader->buf.pos;
824 dout(10) << __func__ << " log file size was 0x"
825 << std::hex << log_file->fnode.size << std::dec << dendl;
829 // verify file link counts are all >0
830 for (auto& p : file_map) {
831 if (p.second->refs == 0 &&
832 p.second->fnode.ino > 1) {
833 derr << __func__ << " file with link count 0: " << p.second->fnode
840 dout(10) << __func__ << " done" << dendl;
844 BlueFS::FileRef BlueFS::_get_file(uint64_t ino)
846 auto p = file_map.find(ino);
847 if (p == file_map.end()) {
848 FileRef f = new File;
850 dout(30) << __func__ << " ino " << ino << " = " << f
851 << " (new)" << dendl;
854 dout(30) << __func__ << " ino " << ino << " = " << p->second << dendl;
859 void BlueFS::_drop_link(FileRef file)
861 dout(20) << __func__ << " had refs " << file->refs
862 << " on " << file->fnode << dendl;
863 assert(file->refs > 0);
865 if (file->refs == 0) {
866 dout(20) << __func__ << " destroying " << file->fnode << dendl;
867 assert(file->num_reading.load() == 0);
868 log_t.op_file_remove(file->fnode.ino);
869 for (auto& r : file->fnode.extents) {
870 pending_release[r.bdev].insert(r.offset, r.length);
872 file_map.erase(file->fnode.ino);
873 file->deleted = true;
874 file->fnode.recalc_allocated();
875 if (file->dirty_seq) {
876 assert(file->dirty_seq > log_seq_stable);
877 assert(dirty_files.count(file->dirty_seq));
878 auto it = dirty_files[file->dirty_seq].iterator_to(*file);
879 dirty_files[file->dirty_seq].erase(it);
885 int BlueFS::_read_random(
886 FileReader *h, ///< [in] read from here
887 uint64_t off, ///< [in] offset
888 size_t len, ///< [in] this many bytes
889 char *out) ///< [out] optional: or copy it here
891 dout(10) << __func__ << " h " << h
892 << " 0x" << std::hex << off << "~" << len << std::dec
893 << " from " << h->file->fnode << dendl;
895 ++h->file->num_reading;
897 if (!h->ignore_eof &&
898 off + len > h->file->fnode.size) {
899 if (off > h->file->fnode.size)
902 len = h->file->fnode.size - off;
903 dout(20) << __func__ << " reaching (or past) eof, len clipped to 0x"
904 << std::hex << len << std::dec << dendl;
910 auto p = h->file->fnode.seek(off, &x_off);
911 uint64_t l = MIN(p->length - x_off, len);
912 dout(20) << __func__ << " read buffered 0x"
913 << std::hex << x_off << "~" << l << std::dec
914 << " of " << *p << dendl;
915 int r = bdev[p->bdev]->read_random(p->offset + x_off, l, out,
916 cct->_conf->bluefs_buffered_io);
924 dout(20) << __func__ << " got " << ret << dendl;
925 --h->file->num_reading;
930 FileReader *h, ///< [in] read from here
931 FileReaderBuffer *buf, ///< [in] reader state
932 uint64_t off, ///< [in] offset
933 size_t len, ///< [in] this many bytes
934 bufferlist *outbl, ///< [out] optional: reference the result here
935 char *out) ///< [out] optional: or copy it here
937 dout(10) << __func__ << " h " << h
938 << " 0x" << std::hex << off << "~" << len << std::dec
939 << " from " << h->file->fnode << dendl;
941 ++h->file->num_reading;
943 if (!h->ignore_eof &&
944 off + len > h->file->fnode.size) {
945 if (off > h->file->fnode.size)
948 len = h->file->fnode.size - off;
949 dout(20) << __func__ << " reaching (or past) eof, len clipped to 0x"
950 << std::hex << len << std::dec << dendl;
958 if (off < buf->bl_off || off >= buf->get_buf_end()) {
960 buf->bl_off = off & super.block_mask();
962 auto p = h->file->fnode.seek(buf->bl_off, &x_off);
963 uint64_t want = ROUND_UP_TO(len + (off & ~super.block_mask()),
965 want = MAX(want, buf->max_prefetch);
966 uint64_t l = MIN(p->length - x_off, want);
967 uint64_t eof_offset = ROUND_UP_TO(h->file->fnode.size, super.block_size);
968 if (!h->ignore_eof &&
969 buf->bl_off + l > eof_offset) {
970 l = eof_offset - buf->bl_off;
972 dout(20) << __func__ << " fetching 0x"
973 << std::hex << x_off << "~" << l << std::dec
974 << " of " << *p << dendl;
975 int r = bdev[p->bdev]->read(p->offset + x_off, l, &buf->bl, ioc[p->bdev],
976 cct->_conf->bluefs_buffered_io);
979 left = buf->get_buf_remaining(off);
980 dout(20) << __func__ << " left 0x" << std::hex << left
981 << " len 0x" << len << std::dec << dendl;
983 int r = MIN(len, left);
986 t.substr_of(buf->bl, off - buf->bl_off, r);
987 outbl->claim_append(t);
990 // NOTE: h->bl is normally a contiguous buffer so c_str() is free.
991 memcpy(out, buf->bl.c_str() + off - buf->bl_off, r);
995 dout(30) << __func__ << " result chunk (0x"
996 << std::hex << r << std::dec << " bytes):\n";
998 t.substr_of(buf->bl, off - buf->bl_off, r);
1008 dout(20) << __func__ << " got " << ret << dendl;
1009 assert(!outbl || (int)outbl->length() == ret);
1010 --h->file->num_reading;
1014 void BlueFS::_invalidate_cache(FileRef f, uint64_t offset, uint64_t length)
1016 dout(10) << __func__ << " file " << f->fnode
1017 << " 0x" << std::hex << offset << "~" << length << std::dec
1019 if (offset & ~super.block_mask()) {
1020 offset &= super.block_mask();
1021 length = ROUND_UP_TO(length, super.block_size);
1024 auto p = f->fnode.seek(offset, &x_off);
1025 while (length > 0 && p != f->fnode.extents.end()) {
1026 uint64_t x_len = MIN(p->length - x_off, length);
1027 bdev[p->bdev]->invalidate_cache(p->offset + x_off, x_len);
1028 dout(20) << __func__ << " 0x" << std::hex << x_off << "~" << x_len
1029 << std:: dec << " of " << *p << dendl;
1035 uint64_t BlueFS::_estimate_log_size()
1037 int avg_dir_size = 40; // fixme
1038 int avg_file_size = 12;
1039 uint64_t size = 4096 * 2;
1040 size += file_map.size() * (1 + sizeof(bluefs_fnode_t));
1041 for (auto& p : block_all)
1042 size += p.num_intervals() * (1 + 1 + sizeof(uint64_t) * 2);
1043 size += dir_map.size() + (1 + avg_dir_size);
1044 size += file_map.size() * (1 + avg_dir_size + avg_file_size);
1045 return ROUND_UP_TO(size, super.block_size);
1048 void BlueFS::compact_log()
1050 std::unique_lock<std::mutex> l(lock);
1051 if (cct->_conf->bluefs_compact_log_sync) {
1052 _compact_log_sync();
1054 _compact_log_async(l);
1058 bool BlueFS::_should_compact_log()
1060 uint64_t current = log_writer->file->fnode.size;
1061 uint64_t expected = _estimate_log_size();
1062 float ratio = (float)current / (float)expected;
1063 dout(10) << __func__ << " current 0x" << std::hex << current
1064 << " expected " << expected << std::dec
1065 << " ratio " << ratio
1066 << (new_log ? " (async compaction in progress)" : "")
1069 current < cct->_conf->bluefs_log_compact_min_size ||
1070 ratio < cct->_conf->bluefs_log_compact_min_ratio) {
1076 void BlueFS::_compact_log_dump_metadata(bluefs_transaction_t *t)
1079 t->uuid = super.uuid;
1080 dout(20) << __func__ << " op_init" << dendl;
1083 for (unsigned bdev = 0; bdev < MAX_BDEV; ++bdev) {
1084 interval_set<uint64_t>& p = block_all[bdev];
1085 for (interval_set<uint64_t>::iterator q = p.begin(); q != p.end(); ++q) {
1086 dout(20) << __func__ << " op_alloc_add " << bdev << " 0x"
1087 << std::hex << q.get_start() << "~" << q.get_len() << std::dec
1089 t->op_alloc_add(bdev, q.get_start(), q.get_len());
1092 for (auto& p : file_map) {
1095 dout(20) << __func__ << " op_file_update " << p.second->fnode << dendl;
1096 assert(p.first > 1);
1097 t->op_file_update(p.second->fnode);
1099 for (auto& p : dir_map) {
1100 dout(20) << __func__ << " op_dir_create " << p.first << dendl;
1101 t->op_dir_create(p.first);
1102 for (auto& q : p.second->file_map) {
1103 dout(20) << __func__ << " op_dir_link " << p.first << "/" << q.first
1104 << " to " << q.second->fnode.ino << dendl;
1105 t->op_dir_link(p.first, q.first, q.second->fnode.ino);
1110 void BlueFS::_compact_log_sync()
1112 dout(10) << __func__ << dendl;
1113 File *log_file = log_writer->file.get();
1115 // clear out log (be careful who calls us!!!)
1118 bluefs_transaction_t t;
1119 _compact_log_dump_metadata(&t);
1121 dout(20) << __func__ << " op_jump_seq " << log_seq << dendl;
1122 t.op_jump_seq(log_seq);
1128 uint64_t need = bl.length() + cct->_conf->bluefs_max_log_runway;
1129 dout(20) << __func__ << " need " << need << dendl;
1131 mempool::bluefs::vector<bluefs_extent_t> old_extents;
1132 old_extents.swap(log_file->fnode.extents);
1133 log_file->fnode.recalc_allocated();
1134 while (log_file->fnode.get_allocated() < need) {
1135 int r = _allocate(log_file->fnode.prefer_bdev,
1136 need - log_file->fnode.get_allocated(),
1137 &log_file->fnode.extents);
1138 log_file->fnode.recalc_allocated();
1142 _close_writer(log_writer);
1144 log_file->fnode.size = bl.length();
1145 log_writer = _create_writer(log_file);
1146 log_writer->append(bl);
1147 int r = _flush(log_writer, true);
1149 wait_for_aio(log_writer);
1151 list<aio_t> completed_ios;
1152 _claim_completed_aios(log_writer, &completed_ios);
1154 completed_ios.clear();
1156 dout(10) << __func__ << " writing super" << dendl;
1157 super.log_fnode = log_file->fnode;
1162 dout(10) << __func__ << " release old log extents " << old_extents << dendl;
1163 for (auto& r : old_extents) {
1164 pending_release[r.bdev].insert(r.offset, r.length);
1167 logger->inc(l_bluefs_log_compactions);
1171 * 1. Allocate a new extent to continue the log, and then log an event
1172 * that jumps the log write position to the new extent. At this point, the
1173 * old extent(s) won't be written to, and reflect everything to compact.
1174 * New events will be written to the new region that we'll keep.
1176 * 2. While still holding the lock, encode a bufferlist that dumps all of the
1177 * in-memory fnodes and names. This will become the new beginning of the
1178 * log. The last event will jump to the log continuation extent from #1.
1180 * 3. Queue a write to a new extent for the new beginnging of the log.
1182 * 4. Drop lock and wait
1184 * 5. Retake the lock.
1186 * 6. Update the log_fnode to splice in the new beginning.
1188 * 7. Write the new superblock.
1190 * 8. Release the old log space. Clean up.
1192 void BlueFS::_compact_log_async(std::unique_lock<std::mutex>& l)
1194 dout(10) << __func__ << dendl;
1195 File *log_file = log_writer->file.get();
1197 assert(!new_log_writer);
1199 // create a new log [writer] so that we know compaction is in progress
1200 // (see _should_compact_log)
1202 new_log->fnode.ino = 0; // so that _flush_range won't try to log the fnode
1204 // 0. wait for any racing flushes to complete. (We do not want to block
1205 // in _flush_sync_log with jump_to set or else a racing thread might flush
1206 // our entries and our jump_to update won't be correct.)
1207 while (log_flushing) {
1208 dout(10) << __func__ << " log is currently flushing, waiting" << dendl;
1212 // 1. allocate new log space and jump to it.
1213 old_log_jump_to = log_file->fnode.get_allocated();
1214 uint64_t need = old_log_jump_to + cct->_conf->bluefs_max_log_runway;
1215 dout(10) << __func__ << " old_log_jump_to 0x" << std::hex << old_log_jump_to
1216 << " need 0x" << need << std::dec << dendl;
1217 while (log_file->fnode.get_allocated() < need) {
1218 int r = _allocate(log_file->fnode.prefer_bdev,
1219 cct->_conf->bluefs_max_log_runway,
1220 &log_file->fnode.extents);
1222 log_file->fnode.recalc_allocated();
1224 dout(10) << __func__ << " log extents " << log_file->fnode.extents << dendl;
1226 // update the log file change and log a jump to the offset where we want to
1227 // write the new entries
1228 log_t.op_file_update(log_file->fnode);
1229 log_t.op_jump(log_seq, old_log_jump_to);
1231 flush_bdev(); // FIXME?
1233 _flush_and_sync_log(l, 0, old_log_jump_to);
1235 // 2. prepare compacted log
1236 bluefs_transaction_t t;
1237 //avoid record two times in log_t and _compact_log_dump_metadata.
1239 _compact_log_dump_metadata(&t);
1241 // conservative estimate for final encoded size
1242 new_log_jump_to = ROUND_UP_TO(t.op_bl.length() + super.block_size * 2,
1243 cct->_conf->bluefs_alloc_size);
1244 t.op_jump(log_seq, new_log_jump_to);
1250 dout(10) << __func__ << " new_log_jump_to 0x" << std::hex << new_log_jump_to
1251 << std::dec << dendl;
1254 int r = _allocate(BlueFS::BDEV_DB, new_log_jump_to,
1255 &new_log->fnode.extents);
1257 new_log->fnode.recalc_allocated();
1258 new_log_writer = _create_writer(new_log);
1259 new_log_writer->append(bl);
1262 r = _flush(new_log_writer, true);
1267 dout(10) << __func__ << " waiting for compacted log to sync" << dendl;
1268 wait_for_aio(new_log_writer);
1270 list<aio_t> completed_ios;
1271 _claim_completed_aios(new_log_writer, &completed_ios);
1273 completed_ios.clear();
1278 // 6. update our log fnode
1279 // discard first old_log_jump_to extents
1280 dout(10) << __func__ << " remove 0x" << std::hex << old_log_jump_to << std::dec
1281 << " of " << log_file->fnode.extents << dendl;
1282 uint64_t discarded = 0;
1283 mempool::bluefs::vector<bluefs_extent_t> old_extents;
1284 while (discarded < old_log_jump_to) {
1285 assert(!log_file->fnode.extents.empty());
1286 bluefs_extent_t& e = log_file->fnode.extents.front();
1287 bluefs_extent_t temp = e;
1288 if (discarded + e.length <= old_log_jump_to) {
1289 dout(10) << __func__ << " remove old log extent " << e << dendl;
1290 discarded += e.length;
1291 log_file->fnode.extents.erase(log_file->fnode.extents.begin());
1293 dout(10) << __func__ << " remove front of old log extent " << e << dendl;
1294 uint64_t drop = old_log_jump_to - discarded;
1299 dout(10) << __func__ << " kept " << e << " removed " << temp << dendl;
1301 old_extents.push_back(temp);
1303 new_log->fnode.extents.insert(new_log->fnode.extents.end(),
1304 log_file->fnode.extents.begin(),
1305 log_file->fnode.extents.end());
1307 // clear the extents from old log file, they are added to new log
1308 log_file->fnode.extents.clear();
1310 // swap the log files. New log file is the log file now.
1311 log_file->fnode.extents.swap(new_log->fnode.extents);
1312 log_file->fnode.recalc_allocated();
1313 new_log->fnode.recalc_allocated();
1314 log_writer->pos = log_writer->file->fnode.size =
1315 log_writer->pos - old_log_jump_to + new_log_jump_to;
1317 // 7. write the super block to reflect the changes
1318 dout(10) << __func__ << " writing super" << dendl;
1319 super.log_fnode = log_file->fnode;
1327 // 8. release old space
1328 dout(10) << __func__ << " release old log extents " << old_extents << dendl;
1329 for (auto& r : old_extents) {
1330 pending_release[r.bdev].insert(r.offset, r.length);
1333 // delete the new log, remove from the dirty files list
1334 _close_writer(new_log_writer);
1335 if (new_log->dirty_seq) {
1336 assert(dirty_files.count(new_log->dirty_seq));
1337 auto it = dirty_files[new_log->dirty_seq].iterator_to(*new_log);
1338 dirty_files[new_log->dirty_seq].erase(it);
1340 new_log_writer = nullptr;
1342 log_cond.notify_all();
1344 dout(10) << __func__ << " log extents " << log_file->fnode.extents << dendl;
1345 logger->inc(l_bluefs_log_compactions);
1348 void BlueFS::_pad_bl(bufferlist& bl)
1350 uint64_t partial = bl.length() % super.block_size;
1352 dout(10) << __func__ << " padding with 0x" << std::hex
1353 << super.block_size - partial << " zeros" << std::dec << dendl;
1354 bl.append_zero(super.block_size - partial);
1358 void BlueFS::flush_log()
1360 std::unique_lock<std::mutex> l(lock);
1362 _flush_and_sync_log(l);
1365 int BlueFS::_flush_and_sync_log(std::unique_lock<std::mutex>& l,
1369 while (log_flushing) {
1370 dout(10) << __func__ << " want_seq " << want_seq
1371 << " log is currently flushing, waiting" << dendl;
1375 if (want_seq && want_seq <= log_seq_stable) {
1376 dout(10) << __func__ << " want_seq " << want_seq << " <= log_seq_stable "
1377 << log_seq_stable << ", done" << dendl;
1381 if (log_t.empty() && dirty_files.empty()) {
1382 dout(10) << __func__ << " want_seq " << want_seq
1383 << " " << log_t << " not dirty, dirty_files empty, no-op" << dendl;
1388 uint64_t seq = log_t.seq = ++log_seq;
1389 assert(want_seq == 0 || want_seq <= seq);
1390 log_t.uuid = super.uuid;
1393 auto lsi = dirty_files.find(seq);
1394 if (lsi != dirty_files.end()) {
1395 dout(20) << __func__ << " " << lsi->second.size() << " dirty_files" << dendl;
1396 for (auto &f : lsi->second) {
1397 dout(20) << __func__ << " op_file_update " << f.fnode << dendl;
1398 log_t.op_file_update(f.fnode);
1402 dout(10) << __func__ << " " << log_t << dendl;
1403 assert(!log_t.empty());
1405 // allocate some more space (before we run out)?
1406 int64_t runway = log_writer->file->fnode.get_allocated() -
1407 log_writer->get_effective_write_pos();
1408 if (runway < (int64_t)cct->_conf->bluefs_min_log_runway) {
1409 dout(10) << __func__ << " allocating more log runway (0x"
1410 << std::hex << runway << std::dec << " remaining)" << dendl;
1411 while (new_log_writer) {
1412 dout(10) << __func__ << " waiting for async compaction" << dendl;
1415 int r = _allocate(log_writer->file->fnode.prefer_bdev,
1416 cct->_conf->bluefs_max_log_runway,
1417 &log_writer->file->fnode.extents);
1419 log_writer->file->fnode.recalc_allocated();
1420 log_t.op_file_update(log_writer->file->fnode);
1424 ::encode(log_t, bl);
1426 // pad to block boundary
1428 logger->inc(l_bluefs_logged_bytes, bl.length());
1430 log_writer->append(bl);
1433 log_t.seq = 0; // just so debug output is less confusing
1434 log_flushing = true;
1436 int r = _flush(log_writer, true);
1440 dout(10) << __func__ << " jumping log offset from 0x" << std::hex
1441 << log_writer->pos << " -> 0x" << jump_to << std::dec << dendl;
1442 log_writer->pos = jump_to;
1443 log_writer->file->fnode.size = jump_to;
1446 _flush_bdev_safely(log_writer);
1448 log_flushing = false;
1449 log_cond.notify_all();
1451 // clean dirty files
1452 if (seq > log_seq_stable) {
1453 log_seq_stable = seq;
1454 dout(20) << __func__ << " log_seq_stable " << log_seq_stable << dendl;
1456 auto p = dirty_files.begin();
1457 while (p != dirty_files.end()) {
1458 if (p->first > log_seq_stable) {
1459 dout(20) << __func__ << " done cleaning up dirty files" << dendl;
1463 auto l = p->second.begin();
1464 while (l != p->second.end()) {
1466 assert(file->dirty_seq > 0);
1467 assert(file->dirty_seq <= log_seq_stable);
1468 dout(20) << __func__ << " cleaned file " << file->fnode << dendl;
1469 file->dirty_seq = 0;
1470 p->second.erase(l++);
1473 assert(p->second.empty());
1474 dirty_files.erase(p++);
1477 dout(20) << __func__ << " log_seq_stable " << log_seq_stable
1478 << " already >= out seq " << seq
1479 << ", we lost a race against another log flush, done" << dendl;
1481 _update_logger_stats();
1486 int BlueFS::_flush_range(FileWriter *h, uint64_t offset, uint64_t length)
1488 dout(10) << __func__ << " " << h << " pos 0x" << std::hex << h->pos
1489 << " 0x" << offset << "~" << length << std::dec
1490 << " to " << h->file->fnode << dendl;
1491 assert(!h->file->deleted);
1492 assert(h->file->num_readers.load() == 0);
1494 h->buffer_appender.flush();
1497 if (h->file->fnode.ino == 1)
1500 buffered = cct->_conf->bluefs_buffered_io;
1502 if (offset + length <= h->pos)
1504 if (offset < h->pos) {
1505 length -= h->pos - offset;
1507 dout(10) << " still need 0x"
1508 << std::hex << offset << "~" << length << std::dec
1511 assert(offset <= h->file->fnode.size);
1513 uint64_t allocated = h->file->fnode.get_allocated();
1515 // do not bother to dirty the file if we are overwriting
1516 // previously allocated extents.
1517 bool must_dirty = false;
1518 if (allocated < offset + length) {
1519 // we should never run out of log space here; see the min runway check
1520 // in _flush_and_sync_log.
1521 assert(h->file->fnode.ino != 1);
1522 int r = _allocate(h->file->fnode.prefer_bdev,
1523 offset + length - allocated,
1524 &h->file->fnode.extents);
1526 derr << __func__ << " allocated: 0x" << std::hex << allocated
1527 << " offset: 0x" << offset << " length: 0x" << length << std::dec
1529 assert(0 == "bluefs enospc");
1532 h->file->fnode.recalc_allocated();
1533 if (cct->_conf->bluefs_preextend_wal_files &&
1534 h->writer_type == WRITER_WAL) {
1535 // NOTE: this *requires* that rocksdb also has log recycling
1536 // enabled and is therefore doing robust CRCs on the log
1537 // records. otherwise, we will fail to reply the rocksdb log
1538 // properly due to garbage on the device.
1539 h->file->fnode.size = h->file->fnode.get_allocated();
1540 dout(10) << __func__ << " extending WAL size to 0x" << std::hex
1541 << h->file->fnode.size << std::dec << " to include allocated"
1546 if (h->file->fnode.size < offset + length) {
1547 h->file->fnode.size = offset + length;
1548 if (h->file->fnode.ino > 1) {
1549 // we do not need to dirty the log file (or it's compacting
1550 // replacement) when the file size changes because replay is
1551 // smart enough to discover it on its own.
1556 h->file->fnode.mtime = ceph_clock_now();
1557 assert(h->file->fnode.ino >= 1);
1558 if (h->file->dirty_seq == 0) {
1559 h->file->dirty_seq = log_seq + 1;
1560 dirty_files[h->file->dirty_seq].push_back(*h->file);
1561 dout(20) << __func__ << " dirty_seq = " << log_seq + 1
1562 << " (was clean)" << dendl;
1564 if (h->file->dirty_seq != log_seq + 1) {
1565 // need re-dirty, erase from list first
1566 assert(dirty_files.count(h->file->dirty_seq));
1567 auto it = dirty_files[h->file->dirty_seq].iterator_to(*h->file);
1568 dirty_files[h->file->dirty_seq].erase(it);
1569 h->file->dirty_seq = log_seq + 1;
1570 dirty_files[h->file->dirty_seq].push_back(*h->file);
1571 dout(20) << __func__ << " dirty_seq = " << log_seq + 1
1572 << " (was " << h->file->dirty_seq << ")" << dendl;
1574 dout(20) << __func__ << " dirty_seq = " << log_seq + 1
1575 << " (unchanged, do nothing) " << dendl;
1579 dout(20) << __func__ << " file now " << h->file->fnode << dendl;
1582 auto p = h->file->fnode.seek(offset, &x_off);
1583 assert(p != h->file->fnode.extents.end());
1584 dout(20) << __func__ << " in " << *p << " x_off 0x"
1585 << std::hex << x_off << std::dec << dendl;
1587 unsigned partial = x_off & ~super.block_mask();
1590 dout(20) << __func__ << " using partial tail 0x"
1591 << std::hex << partial << std::dec << dendl;
1592 assert(h->tail_block.length() == partial);
1593 bl.claim_append_piecewise(h->tail_block);
1597 dout(20) << __func__ << " waiting for previous aio to complete" << dendl;
1598 for (auto p : h->iocv) {
1604 if (length == partial + h->buffer.length()) {
1605 bl.claim_append_piecewise(h->buffer);
1608 h->buffer.splice(0, length, &t);
1609 bl.claim_append_piecewise(t);
1610 t.substr_of(h->buffer, length, h->buffer.length() - length);
1612 dout(20) << " leaving 0x" << std::hex << h->buffer.length() << std::dec
1613 << " unflushed" << dendl;
1615 assert(bl.length() == length);
1617 switch (h->writer_type) {
1619 logger->inc(l_bluefs_bytes_written_wal, length);
1622 logger->inc(l_bluefs_bytes_written_sst, length);
1626 dout(30) << "dump:\n";
1630 h->pos = offset + length;
1631 h->tail_block.clear();
1634 while (length > 0) {
1635 uint64_t x_len = MIN(p->length - x_off, length);
1637 t.substr_of(bl, bloff, x_len);
1638 unsigned tail = x_len & ~super.block_mask();
1640 size_t zlen = super.block_size - tail;
1641 dout(20) << __func__ << " caching tail of 0x"
1643 << " and padding block with 0x" << zlen
1644 << std::dec << dendl;
1645 h->tail_block.substr_of(bl, bl.length() - tail, tail);
1646 if (h->file->fnode.ino > 1) {
1647 // we are using the page_aligned_appender, and can safely use
1648 // the tail of the raw buffer.
1649 const bufferptr &last = t.back();
1650 if (last.unused_tail_length() < zlen) {
1651 derr << " wtf, last is " << last << " from " << t << dendl;
1652 assert(last.unused_tail_length() >= zlen);
1655 z.set_offset(last.offset() + last.length());
1658 t.append(z, 0, zlen);
1660 t.append_zero(zlen);
1663 if (cct->_conf->bluefs_sync_write) {
1664 bdev[p->bdev]->write(p->offset + x_off, t, buffered);
1666 bdev[p->bdev]->aio_write(p->offset + x_off, t, h->iocv[p->bdev], buffered);
1673 for (unsigned i = 0; i < MAX_BDEV; ++i) {
1676 if (h->iocv[i]->has_pending_aios()) {
1677 bdev[i]->aio_submit(h->iocv[i]);
1681 dout(20) << __func__ << " h " << h << " pos now 0x"
1682 << std::hex << h->pos << std::dec << dendl;
1686 // we need to retire old completed aios so they don't stick around in
1687 // memory indefinitely (along with their bufferlist refs).
1688 void BlueFS::_claim_completed_aios(FileWriter *h, list<aio_t> *ls)
1690 for (auto p : h->iocv) {
1692 ls->splice(ls->end(), p->running_aios);
1695 dout(10) << __func__ << " got " << ls->size() << " aios" << dendl;
1698 void BlueFS::wait_for_aio(FileWriter *h)
1700 // NOTE: this is safe to call without a lock, as long as our reference is
1702 dout(10) << __func__ << " " << h << dendl;
1703 utime_t start = ceph_clock_now();
1704 for (auto p : h->iocv) {
1709 utime_t end = ceph_clock_now();
1710 utime_t dur = end - start;
1711 dout(10) << __func__ << " " << h << " done in " << dur << dendl;
1714 int BlueFS::_flush(FileWriter *h, bool force)
1716 h->buffer_appender.flush();
1717 uint64_t length = h->buffer.length();
1718 uint64_t offset = h->pos;
1720 length < cct->_conf->bluefs_min_flush_size) {
1721 dout(10) << __func__ << " " << h << " ignoring, length " << length
1722 << " < min_flush_size " << cct->_conf->bluefs_min_flush_size
1727 dout(10) << __func__ << " " << h << " no dirty data on "
1728 << h->file->fnode << dendl;
1731 dout(10) << __func__ << " " << h << " 0x"
1732 << std::hex << offset << "~" << length << std::dec
1733 << " to " << h->file->fnode << dendl;
1734 assert(h->pos <= h->file->fnode.size);
1735 return _flush_range(h, offset, length);
1738 int BlueFS::_truncate(FileWriter *h, uint64_t offset)
1740 dout(10) << __func__ << " 0x" << std::hex << offset << std::dec
1741 << " file " << h->file->fnode << dendl;
1742 if (h->file->deleted) {
1743 dout(10) << __func__ << " deleted, no-op" << dendl;
1747 // we never truncate internal log files
1748 assert(h->file->fnode.ino > 1);
1750 h->buffer_appender.flush();
1752 // truncate off unflushed data?
1753 if (h->pos < offset &&
1754 h->pos + h->buffer.length() > offset) {
1756 dout(20) << __func__ << " tossing out last " << offset - h->pos
1757 << " unflushed bytes" << dendl;
1758 t.substr_of(h->buffer, 0, offset - h->pos);
1760 assert(0 == "actually this shouldn't happen");
1762 if (h->buffer.length()) {
1763 int r = _flush(h, true);
1767 if (offset == h->file->fnode.size) {
1770 if (offset > h->file->fnode.size) {
1771 assert(0 == "truncate up not supported");
1773 assert(h->file->fnode.size >= offset);
1774 h->file->fnode.size = offset;
1775 log_t.op_file_update(h->file->fnode);
1779 int BlueFS::_fsync(FileWriter *h, std::unique_lock<std::mutex>& l)
1781 dout(10) << __func__ << " " << h << " " << h->file->fnode << dendl;
1782 int r = _flush(h, true);
1785 uint64_t old_dirty_seq = h->file->dirty_seq;
1787 _flush_bdev_safely(h);
1789 if (old_dirty_seq) {
1790 uint64_t s = log_seq;
1791 dout(20) << __func__ << " file metadata was dirty (" << old_dirty_seq
1792 << ") on " << h->file->fnode << ", flushing log" << dendl;
1793 _flush_and_sync_log(l, old_dirty_seq);
1794 assert(h->file->dirty_seq == 0 || // cleaned
1795 h->file->dirty_seq > s); // or redirtied by someone else
1800 void BlueFS::_flush_bdev_safely(FileWriter *h)
1802 if (!cct->_conf->bluefs_sync_write) {
1803 list<aio_t> completed_ios;
1804 _claim_completed_aios(h, &completed_ios);
1807 completed_ios.clear();
1817 void BlueFS::flush_bdev()
1819 // NOTE: this is safe to call without a lock.
1820 dout(20) << __func__ << dendl;
1821 for (auto p : bdev) {
1827 int BlueFS::_allocate(uint8_t id, uint64_t len,
1828 mempool::bluefs::vector<bluefs_extent_t> *ev)
1830 dout(10) << __func__ << " len 0x" << std::hex << len << std::dec
1831 << " from " << (int)id << dendl;
1832 assert(id < alloc.size());
1833 uint64_t min_alloc_size = cct->_conf->bluefs_alloc_size;
1835 uint64_t left = ROUND_UP_TO(len, min_alloc_size);
1838 r = alloc[id]->reserve(left);
1841 if (id != BDEV_SLOW) {
1843 dout(1) << __func__ << " failed to allocate 0x" << std::hex << left
1844 << " on bdev " << (int)id
1845 << ", free 0x" << alloc[id]->get_free()
1846 << "; fallback to bdev " << (int)id + 1
1847 << std::dec << dendl;
1849 return _allocate(id + 1, len, ev);
1852 derr << __func__ << " failed to allocate 0x" << std::hex << left
1853 << " on bdev " << (int)id
1854 << ", free 0x" << alloc[id]->get_free() << std::dec << dendl;
1856 derr << __func__ << " failed to allocate 0x" << std::hex << left
1857 << " on bdev " << (int)id << ", dne" << std::dec << dendl;
1863 hint = ev->back().end();
1866 AllocExtentVector extents;
1867 extents.reserve(4); // 4 should be (more than) enough for most allocations
1868 int64_t alloc_len = alloc[id]->allocate(left, min_alloc_size, hint,
1870 if (alloc_len < (int64_t)left) {
1871 derr << __func__ << " allocate failed on 0x" << std::hex << left
1872 << " min_alloc_size 0x" << min_alloc_size
1873 << " hint 0x" << hint << std::dec << dendl;
1875 assert(0 == "allocate failed... wtf");
1879 for (auto& p : extents) {
1880 bluefs_extent_t e = bluefs_extent_t(id, p.offset, p.length);
1882 ev->back().bdev == e.bdev &&
1883 ev->back().end() == (uint64_t) e.offset) {
1884 ev->back().length += e.length;
1893 int BlueFS::_preallocate(FileRef f, uint64_t off, uint64_t len)
1895 dout(10) << __func__ << " file " << f->fnode << " 0x"
1896 << std::hex << off << "~" << len << std::dec << dendl;
1898 dout(10) << __func__ << " deleted, no-op" << dendl;
1901 assert(f->fnode.ino > 1);
1902 uint64_t allocated = f->fnode.get_allocated();
1903 if (off + len > allocated) {
1904 uint64_t want = off + len - allocated;
1905 int r = _allocate(f->fnode.prefer_bdev, want, &f->fnode.extents);
1908 f->fnode.recalc_allocated();
1909 log_t.op_file_update(f->fnode);
1914 void BlueFS::sync_metadata()
1916 std::unique_lock<std::mutex> l(lock);
1917 if (log_t.empty()) {
1918 dout(10) << __func__ << " - no pending log events" << dendl;
1921 dout(10) << __func__ << dendl;
1922 utime_t start = ceph_clock_now();
1923 vector<interval_set<uint64_t>> to_release(pending_release.size());
1924 to_release.swap(pending_release);
1925 flush_bdev(); // FIXME?
1926 _flush_and_sync_log(l);
1927 for (unsigned i = 0; i < to_release.size(); ++i) {
1928 for (auto p = to_release[i].begin(); p != to_release[i].end(); ++p) {
1929 alloc[i]->release(p.get_start(), p.get_len());
1933 if (_should_compact_log()) {
1934 if (cct->_conf->bluefs_compact_log_sync) {
1935 _compact_log_sync();
1937 _compact_log_async(l);
1941 utime_t end = ceph_clock_now();
1942 utime_t dur = end - start;
1943 dout(10) << __func__ << " done in " << dur << dendl;
1946 int BlueFS::open_for_write(
1947 const string& dirname,
1948 const string& filename,
1952 std::lock_guard<std::mutex> l(lock);
1953 dout(10) << __func__ << " " << dirname << "/" << filename << dendl;
1954 map<string,DirRef>::iterator p = dir_map.find(dirname);
1956 if (p == dir_map.end()) {
1957 // implicitly create the dir
1958 dout(20) << __func__ << " dir " << dirname
1959 << " does not exist" << dendl;
1966 bool create = false;
1967 map<string,FileRef>::iterator q = dir->file_map.find(filename);
1968 if (q == dir->file_map.end()) {
1970 dout(20) << __func__ << " dir " << dirname << " (" << dir
1971 << ") file " << filename
1972 << " does not exist" << dendl;
1976 file->fnode.ino = ++ino_last;
1977 file_map[ino_last] = file;
1978 dir->file_map[filename] = file;
1982 // overwrite existing file?
1985 dout(20) << __func__ << " dir " << dirname << " (" << dir
1986 << ") file " << filename
1987 << " already exists, overwrite in place" << dendl;
1989 dout(20) << __func__ << " dir " << dirname << " (" << dir
1990 << ") file " << filename
1991 << " already exists, truncate + overwrite" << dendl;
1992 file->fnode.size = 0;
1993 for (auto& p : file->fnode.extents) {
1994 pending_release[p.bdev].insert(p.offset, p.length);
1996 file->fnode.extents.clear();
1997 file->fnode.recalc_allocated();
2000 assert(file->fnode.ino > 1);
2002 file->fnode.mtime = ceph_clock_now();
2003 file->fnode.prefer_bdev = BlueFS::BDEV_DB;
2004 if (dirname.length() > 5) {
2005 // the "db.slow" and "db.wal" directory names are hard-coded at
2006 // match up with bluestore. the slow device is always the second
2007 // one (when a dedicated block.db device is present and used at
2008 // bdev 0). the wal device is always last.
2009 if (boost::algorithm::ends_with(dirname, ".slow")) {
2010 file->fnode.prefer_bdev = BlueFS::BDEV_SLOW;
2011 } else if (boost::algorithm::ends_with(dirname, ".wal")) {
2012 file->fnode.prefer_bdev = BlueFS::BDEV_WAL;
2015 dout(20) << __func__ << " mapping " << dirname << "/" << filename
2016 << " to bdev " << (int)file->fnode.prefer_bdev << dendl;
2018 log_t.op_file_update(file->fnode);
2020 log_t.op_dir_link(dirname, filename, file->fnode.ino);
2022 *h = _create_writer(file);
2024 if (boost::algorithm::ends_with(filename, ".log")) {
2025 (*h)->writer_type = BlueFS::WRITER_WAL;
2026 if (logger && !overwrite) {
2027 logger->inc(l_bluefs_files_written_wal);
2029 } else if (boost::algorithm::ends_with(filename, ".sst")) {
2030 (*h)->writer_type = BlueFS::WRITER_SST;
2032 logger->inc(l_bluefs_files_written_sst);
2036 dout(10) << __func__ << " h " << *h << " on " << file->fnode << dendl;
2040 BlueFS::FileWriter *BlueFS::_create_writer(FileRef f)
2042 FileWriter *w = new FileWriter(f);
2043 for (unsigned i = 0; i < MAX_BDEV; ++i) {
2045 w->iocv[i] = new IOContext(cct, NULL);
2053 void BlueFS::_close_writer(FileWriter *h)
2055 dout(10) << __func__ << " " << h << " type " << h->writer_type << dendl;
2056 for (unsigned i=0; i<MAX_BDEV; ++i) {
2059 h->iocv[i]->aio_wait();
2060 bdev[i]->queue_reap_ioc(h->iocv[i]);
2066 int BlueFS::open_for_read(
2067 const string& dirname,
2068 const string& filename,
2072 std::lock_guard<std::mutex> l(lock);
2073 dout(10) << __func__ << " " << dirname << "/" << filename
2074 << (random ? " (random)":" (sequential)") << dendl;
2075 map<string,DirRef>::iterator p = dir_map.find(dirname);
2076 if (p == dir_map.end()) {
2077 dout(20) << __func__ << " dir " << dirname << " not found" << dendl;
2080 DirRef dir = p->second;
2082 map<string,FileRef>::iterator q = dir->file_map.find(filename);
2083 if (q == dir->file_map.end()) {
2084 dout(20) << __func__ << " dir " << dirname << " (" << dir
2085 << ") file " << filename
2086 << " not found" << dendl;
2089 File *file = q->second.get();
2091 *h = new FileReader(file, random ? 4096 : cct->_conf->bluefs_max_prefetch,
2093 dout(10) << __func__ << " h " << *h << " on " << file->fnode << dendl;
2098 const string& old_dirname, const string& old_filename,
2099 const string& new_dirname, const string& new_filename)
2101 std::lock_guard<std::mutex> l(lock);
2102 dout(10) << __func__ << " " << old_dirname << "/" << old_filename
2103 << " -> " << new_dirname << "/" << new_filename << dendl;
2104 map<string,DirRef>::iterator p = dir_map.find(old_dirname);
2105 if (p == dir_map.end()) {
2106 dout(20) << __func__ << " dir " << old_dirname << " not found" << dendl;
2109 DirRef old_dir = p->second;
2110 map<string,FileRef>::iterator q = old_dir->file_map.find(old_filename);
2111 if (q == old_dir->file_map.end()) {
2112 dout(20) << __func__ << " dir " << old_dirname << " (" << old_dir
2113 << ") file " << old_filename
2114 << " not found" << dendl;
2117 FileRef file = q->second;
2119 p = dir_map.find(new_dirname);
2120 if (p == dir_map.end()) {
2121 dout(20) << __func__ << " dir " << new_dirname << " not found" << dendl;
2124 DirRef new_dir = p->second;
2125 q = new_dir->file_map.find(new_filename);
2126 if (q != new_dir->file_map.end()) {
2127 dout(20) << __func__ << " dir " << new_dirname << " (" << old_dir
2128 << ") file " << new_filename
2129 << " already exists, unlinking" << dendl;
2130 assert(q->second != file);
2131 log_t.op_dir_unlink(new_dirname, new_filename);
2132 _drop_link(q->second);
2135 dout(10) << __func__ << " " << new_dirname << "/" << new_filename << " "
2136 << " " << file->fnode << dendl;
2138 new_dir->file_map[new_filename] = file;
2139 old_dir->file_map.erase(old_filename);
2141 log_t.op_dir_link(new_dirname, new_filename, file->fnode.ino);
2142 log_t.op_dir_unlink(old_dirname, old_filename);
2146 int BlueFS::mkdir(const string& dirname)
2148 std::lock_guard<std::mutex> l(lock);
2149 dout(10) << __func__ << " " << dirname << dendl;
2150 map<string,DirRef>::iterator p = dir_map.find(dirname);
2151 if (p != dir_map.end()) {
2152 dout(20) << __func__ << " dir " << dirname << " exists" << dendl;
2155 dir_map[dirname] = new Dir;
2156 log_t.op_dir_create(dirname);
2160 int BlueFS::rmdir(const string& dirname)
2162 std::lock_guard<std::mutex> l(lock);
2163 dout(10) << __func__ << " " << dirname << dendl;
2164 map<string,DirRef>::iterator p = dir_map.find(dirname);
2165 if (p == dir_map.end()) {
2166 dout(20) << __func__ << " dir " << dirname << " does not exist" << dendl;
2169 DirRef dir = p->second;
2170 if (!dir->file_map.empty()) {
2171 dout(20) << __func__ << " dir " << dirname << " not empty" << dendl;
2174 dir_map.erase(dirname);
2175 log_t.op_dir_remove(dirname);
2179 bool BlueFS::dir_exists(const string& dirname)
2181 std::lock_guard<std::mutex> l(lock);
2182 map<string,DirRef>::iterator p = dir_map.find(dirname);
2183 bool exists = p != dir_map.end();
2184 dout(10) << __func__ << " " << dirname << " = " << (int)exists << dendl;
2188 int BlueFS::stat(const string& dirname, const string& filename,
2189 uint64_t *size, utime_t *mtime)
2191 std::lock_guard<std::mutex> l(lock);
2192 dout(10) << __func__ << " " << dirname << "/" << filename << dendl;
2193 map<string,DirRef>::iterator p = dir_map.find(dirname);
2194 if (p == dir_map.end()) {
2195 dout(20) << __func__ << " dir " << dirname << " not found" << dendl;
2198 DirRef dir = p->second;
2199 map<string,FileRef>::iterator q = dir->file_map.find(filename);
2200 if (q == dir->file_map.end()) {
2201 dout(20) << __func__ << " dir " << dirname << " (" << dir
2202 << ") file " << filename
2203 << " not found" << dendl;
2206 File *file = q->second.get();
2207 dout(10) << __func__ << " " << dirname << "/" << filename
2208 << " " << file->fnode << dendl;
2210 *size = file->fnode.size;
2212 *mtime = file->fnode.mtime;
2216 int BlueFS::lock_file(const string& dirname, const string& filename,
2219 std::lock_guard<std::mutex> l(lock);
2220 dout(10) << __func__ << " " << dirname << "/" << filename << dendl;
2221 map<string,DirRef>::iterator p = dir_map.find(dirname);
2222 if (p == dir_map.end()) {
2223 dout(20) << __func__ << " dir " << dirname << " not found" << dendl;
2226 DirRef dir = p->second;
2227 map<string,FileRef>::iterator q = dir->file_map.find(filename);
2229 if (q == dir->file_map.end()) {
2230 dout(20) << __func__ << " dir " << dirname << " (" << dir
2231 << ") file " << filename
2232 << " not found, creating" << dendl;
2234 file->fnode.ino = ++ino_last;
2235 file->fnode.mtime = ceph_clock_now();
2236 file_map[ino_last] = file;
2237 dir->file_map[filename] = file;
2239 log_t.op_file_update(file->fnode);
2240 log_t.op_dir_link(dirname, filename, file->fnode.ino);
2242 file = q->second.get();
2244 dout(10) << __func__ << " already locked" << dendl;
2248 file->locked = true;
2249 *plock = new FileLock(file);
2250 dout(10) << __func__ << " locked " << file->fnode
2251 << " with " << *plock << dendl;
2255 int BlueFS::unlock_file(FileLock *fl)
2257 std::lock_guard<std::mutex> l(lock);
2258 dout(10) << __func__ << " " << fl << " on " << fl->file->fnode << dendl;
2259 assert(fl->file->locked);
2260 fl->file->locked = false;
2265 int BlueFS::readdir(const string& dirname, vector<string> *ls)
2267 std::lock_guard<std::mutex> l(lock);
2268 dout(10) << __func__ << " " << dirname << dendl;
2269 if (dirname.empty()) {
2271 ls->reserve(dir_map.size() + 2);
2272 for (auto& q : dir_map) {
2273 ls->push_back(q.first);
2276 // list files in dir
2277 map<string,DirRef>::iterator p = dir_map.find(dirname);
2278 if (p == dir_map.end()) {
2279 dout(20) << __func__ << " dir " << dirname << " not found" << dendl;
2282 DirRef dir = p->second;
2283 ls->reserve(dir->file_map.size() + 2);
2284 for (auto& q : dir->file_map) {
2285 ls->push_back(q.first);
2289 ls->push_back("..");
2293 int BlueFS::unlink(const string& dirname, const string& filename)
2295 std::lock_guard<std::mutex> l(lock);
2296 dout(10) << __func__ << " " << dirname << "/" << filename << dendl;
2297 map<string,DirRef>::iterator p = dir_map.find(dirname);
2298 if (p == dir_map.end()) {
2299 dout(20) << __func__ << " dir " << dirname << " not found" << dendl;
2302 DirRef dir = p->second;
2303 map<string,FileRef>::iterator q = dir->file_map.find(filename);
2304 if (q == dir->file_map.end()) {
2305 dout(20) << __func__ << " file " << dirname << "/" << filename
2306 << " not found" << dendl;
2309 FileRef file = q->second;
2311 dout(20) << __func__ << " file " << dirname << "/" << filename
2312 << " is locked" << dendl;
2315 dir->file_map.erase(filename);
2316 log_t.op_dir_unlink(dirname, filename);
2321 bool BlueFS::wal_is_rotational()
2323 if (!bdev[BDEV_WAL] || bdev[BDEV_WAL]->is_rotational())