1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
2 // vim: ts=8 sw=2 smarttab
4 * Ceph - scalable distributed file system
6 * Copyright (C) 2016 SUSE LINUX GmbH
8 * This is free software; you can redistribute it and/or
9 * modify it under the terms of the GNU Lesser General Public
10 * License version 2.1, as published by the Free Software
11 * Foundation. See file COPYING.
15 #include <boost/bind.hpp>
20 #include "include/rados/librados.hpp"
21 #include "common/Formatter.h"
22 #include "common/admin_socket.h"
23 #include "common/debug.h"
24 #include "common/errno.h"
25 #include "common/WorkQueue.h"
26 #include "global/global_context.h"
27 #include "librbd/internal.h"
28 #include "librbd/ImageCtx.h"
29 #include "librbd/ImageState.h"
30 #include "librbd/Journal.h"
31 #include "librbd/Operations.h"
32 #include "librbd/journal/Policy.h"
33 #include "cls/rbd/cls_rbd_client.h"
34 #include "cls/rbd/cls_rbd_types.h"
35 #include "librbd/Utils.h"
36 #include "ImageDeleter.h"
38 #define dout_context g_ceph_context
39 #define dout_subsys ceph_subsys_rbd_mirror
41 #define dout_prefix *_dout << "rbd::mirror::ImageDeleter: " << this << " " \
46 using std::stringstream;
51 using librados::IoCtx;
52 using namespace librbd;
59 class ImageDeleterAdminSocketCommand {
61 virtual ~ImageDeleterAdminSocketCommand() {}
62 virtual bool call(Formatter *f, stringstream *ss) = 0;
66 class StatusCommand : public ImageDeleterAdminSocketCommand {
68 explicit StatusCommand(ImageDeleter<I> *image_del) : image_del(image_del) {}
70 bool call(Formatter *f, stringstream *ss) override {
71 image_del->print_status(f, ss);
76 ImageDeleter<I> *image_del;
79 struct DeleteJournalPolicy : public librbd::journal::Policy {
80 bool append_disabled() const override {
83 bool journal_disabled() const override {
87 void allocate_tag_on_lock(Context *on_finish) override {
88 on_finish->complete(0);
92 } // anonymous namespace
95 class ImageDeleterAdminSocketHook : public AdminSocketHook {
97 ImageDeleterAdminSocketHook(CephContext *cct, ImageDeleter<I> *image_del) :
98 admin_socket(cct->get_admin_socket()) {
103 command = "rbd mirror deletion status";
104 r = admin_socket->register_command(command, command, this,
105 "get status for image deleter");
107 commands[command] = new StatusCommand<I>(image_del);
112 ~ImageDeleterAdminSocketHook() override {
113 for (Commands::const_iterator i = commands.begin(); i != commands.end();
115 (void)admin_socket->unregister_command(i->first);
120 bool call(std::string command, cmdmap_t& cmdmap, std::string format,
121 bufferlist& out) override {
122 Commands::const_iterator i = commands.find(command);
123 assert(i != commands.end());
124 Formatter *f = Formatter::create(format);
126 bool r = i->second->call(f, &ss);
133 typedef std::map<std::string, ImageDeleterAdminSocketCommand*> Commands;
134 AdminSocket *admin_socket;
138 template <typename I>
139 ImageDeleter<I>::ImageDeleter(ContextWQ *work_queue, SafeTimer *timer,
141 ServiceDaemon<librbd::ImageCtx>* service_daemon)
142 : m_work_queue(work_queue),
143 m_service_daemon(service_daemon),
144 m_delete_lock("rbd::mirror::ImageDeleter::Delete"),
145 m_image_deleter_thread(this),
146 m_failed_timer(timer),
147 m_failed_timer_lock(timer_lock),
148 m_asok_hook(new ImageDeleterAdminSocketHook<I>(g_ceph_context, this))
150 set_failed_timer_interval(g_ceph_context->_conf->get_val<double>(
151 "rbd_mirror_delete_retry_interval"));
152 m_image_deleter_thread.create("image_deleter");
155 template <typename I>
156 ImageDeleter<I>::~ImageDeleter() {
157 dout(20) << "enter" << dendl;
161 Mutex::Locker l (m_delete_lock);
162 m_delete_queue_cond.Signal();
164 if (m_image_deleter_thread.is_started()) {
165 m_image_deleter_thread.join();
169 dout(20) << "return" << dendl;
172 template <typename I>
173 void ImageDeleter<I>::run() {
174 dout(20) << "enter" << dendl;
176 m_delete_lock.Lock();
177 while (m_delete_queue.empty()) {
178 dout(20) << "waiting for delete requests" << dendl;
179 m_delete_queue_cond.Wait(m_delete_lock);
182 m_delete_lock.Unlock();
183 dout(20) << "return" << dendl;
188 m_active_delete = std::move(m_delete_queue.back());
189 m_delete_queue.pop_back();
190 m_delete_lock.Unlock();
192 bool move_to_next = process_image_delete();
195 dout(20) << "return" << dendl;
199 Mutex::Locker l(m_delete_lock);
200 if (m_delete_queue.size() == 1) {
201 m_delete_queue_cond.Wait(m_delete_lock);
207 template <typename I>
208 void ImageDeleter<I>::schedule_image_delete(RadosRef local_rados,
209 int64_t local_pool_id,
210 const std::string& global_image_id,
211 bool ignore_orphaned) {
212 dout(20) << "enter" << dendl;
214 Mutex::Locker locker(m_delete_lock);
216 auto del_info = find_delete_info(local_pool_id, global_image_id);
217 if (del_info != nullptr) {
218 dout(20) << "image " << global_image_id << " "
219 << "was already scheduled for deletion" << dendl;
220 if (ignore_orphaned) {
221 (*del_info)->ignore_orphaned = true;
226 m_delete_queue.push_front(
227 unique_ptr<DeleteInfo>(new DeleteInfo(local_rados, local_pool_id,
228 global_image_id, ignore_orphaned)));
229 m_delete_queue_cond.Signal();
232 template <typename I>
233 void ImageDeleter<I>::wait_for_scheduled_deletion(int64_t local_pool_id,
234 const std::string &global_image_id,
236 bool notify_on_failed_retry) {
238 ctx = new FunctionContext([this, ctx](int r) {
239 m_work_queue->queue(ctx, r);
242 Mutex::Locker locker(m_delete_lock);
243 auto del_info = find_delete_info(local_pool_id, global_image_id);
245 // image not scheduled for deletion
250 dout(20) << "local_pool_id=" << local_pool_id << ", "
251 << "global_image_id=" << global_image_id << dendl;
253 if ((*del_info)->on_delete != nullptr) {
254 (*del_info)->on_delete->complete(-ESTALE);
256 (*del_info)->on_delete = ctx;
257 (*del_info)->notify_on_failed_retry = notify_on_failed_retry;
260 template <typename I>
261 void ImageDeleter<I>::cancel_waiter(int64_t local_pool_id,
262 const std::string &global_image_id) {
263 Mutex::Locker locker(m_delete_lock);
264 auto del_info = find_delete_info(local_pool_id, global_image_id);
269 if ((*del_info)->on_delete != nullptr) {
270 (*del_info)->on_delete->complete(-ECANCELED);
271 (*del_info)->on_delete = nullptr;
275 template <typename I>
276 bool ImageDeleter<I>::process_image_delete() {
278 m_active_delete->to_string(ss);
279 std::string del_info_str = ss.str();
280 dout(10) << "start processing delete request: " << del_info_str << dendl;
282 cls::rbd::MirrorImage mirror_image;
284 // remote image was disabled, now we need to delete local image
286 r = m_active_delete->local_rados->ioctx_create2(
287 m_active_delete->local_pool_id, ioctx);
289 derr << "error accessing local pool " << m_active_delete->local_pool_id
290 << ": " << cpp_strerror(r) << dendl;
291 enqueue_failed_delete(r);
295 dout(20) << "connected to local pool: " << ioctx.get_pool_name() << dendl;
297 auto &global_image_id = m_active_delete->global_image_id;
298 std::string local_image_id;
299 r = librbd::cls_client::mirror_image_get_image_id(
300 &ioctx, global_image_id, &local_image_id);
302 dout(10) << "image " << global_image_id << " is not mirrored" << dendl;
303 complete_active_delete(r);
306 derr << "error retrieving local id for image " << global_image_id
307 << ": " << cpp_strerror(r) << dendl;
308 enqueue_failed_delete(r);
312 std::string mirror_uuid;
313 C_SaferCond tag_owner_ctx;
314 Journal<>::get_tag_owner(ioctx, local_image_id, &mirror_uuid, m_work_queue,
316 r = tag_owner_ctx.wait();
317 if (r < 0 && r != -ENOENT) {
318 derr << "error retrieving image primary info for image " << global_image_id
319 << ": " << cpp_strerror(r) << dendl;
320 enqueue_failed_delete(r);
322 } else if (r != -ENOENT) {
323 if (mirror_uuid == Journal<>::LOCAL_MIRROR_UUID) {
324 dout(10) << "image " << global_image_id << " is local primary" << dendl;
325 complete_active_delete(-EISPRM);
327 } else if (mirror_uuid == Journal<>::ORPHAN_MIRROR_UUID &&
328 !m_active_delete->ignore_orphaned) {
329 dout(10) << "image " << global_image_id << " is orphaned" << dendl;
330 complete_active_delete(-EISPRM);
335 dout(20) << "local image is not the primary" << dendl;
337 r = image_has_snapshots_and_children(&ioctx, local_image_id, &has_snapshots);
339 enqueue_failed_delete(r);
343 mirror_image.global_image_id = global_image_id;
344 mirror_image.state = cls::rbd::MIRROR_IMAGE_STATE_DISABLING;
345 r = cls_client::mirror_image_set(&ioctx, local_image_id, mirror_image);
347 dout(10) << "local image is not mirrored, aborting deletion..." << dendl;
348 complete_active_delete(r);
350 } else if (r == -EEXIST || r == -EINVAL) {
351 derr << "cannot disable mirroring for image " << global_image_id
352 << ": global_image_id has changed/reused: "
353 << cpp_strerror(r) << dendl;
354 complete_active_delete(r);
357 derr << "cannot disable mirroring for image " << global_image_id
358 << ": " << cpp_strerror(r) << dendl;
359 enqueue_failed_delete(r);
363 dout(20) << "set local image mirroring to disable" << dendl;
366 dout(20) << "local image has snapshots" << dendl;
368 ImageCtx *imgctx = new ImageCtx("", local_image_id, nullptr, ioctx, false);
369 r = imgctx->state->open(false);
371 derr << "error opening image " << global_image_id << " ("
372 << local_image_id << "): " << cpp_strerror(r) << dendl;
373 enqueue_failed_delete(r);
378 RWLock::WLocker snap_locker(imgctx->snap_lock);
379 imgctx->set_journal_policy(new DeleteJournalPolicy());
382 std::vector<librbd::snap_info_t> snaps;
383 r = librbd::snap_list(imgctx, snaps);
385 derr << "error listing snapshot of image " << imgctx->name
386 << cpp_strerror(r) << dendl;
387 imgctx->state->close();
388 enqueue_failed_delete(r);
392 for (const auto& snap : snaps) {
393 dout(20) << "processing deletion of snapshot " << imgctx->name << "@"
394 << snap.name << dendl;
397 r = librbd::snap_is_protected(imgctx, snap.name.c_str(), &is_protected);
399 derr << "error checking snapshot protection of snapshot "
400 << imgctx->name << "@" << snap.name << ": " << cpp_strerror(r)
402 imgctx->state->close();
403 enqueue_failed_delete(r);
407 dout(20) << "snapshot " << imgctx->name << "@" << snap.name
408 << " is protected, issuing unprotect command" << dendl;
410 r = imgctx->operations->snap_unprotect(
411 cls::rbd::UserSnapshotNamespace(), snap.name.c_str());
413 // there are still clones of snapshots of this image, therefore send
414 // the delete request to the end of the queue
415 dout(10) << "local image id " << local_image_id << " has "
416 << "snapshots with cloned children, postponing deletion..."
418 imgctx->state->close();
419 Mutex::Locker l(m_delete_lock);
420 m_active_delete->notify(r);
421 m_delete_queue.push_front(std::move(m_active_delete));
424 derr << "error unprotecting snapshot " << imgctx->name << "@"
425 << snap.name << ": " << cpp_strerror(r) << dendl;
426 imgctx->state->close();
427 enqueue_failed_delete(r);
432 r = imgctx->operations->snap_remove(cls::rbd::UserSnapshotNamespace(),
435 derr << "error removing snapshot " << imgctx->name << "@"
436 << snap.name << ": " << cpp_strerror(r) << dendl;
437 imgctx->state->close();
438 enqueue_failed_delete(r);
442 dout(10) << "snapshot " << imgctx->name << "@" << snap.name
443 << " was deleted" << dendl;
446 imgctx->state->close();
449 librbd::NoOpProgressContext ctx;
450 r = librbd::remove(ioctx, "", local_image_id, ctx, true);
451 if (r < 0 && r != -ENOENT) {
452 derr << "error removing image " << global_image_id << " "
453 << "(" << local_image_id << ") from local pool: "
454 << cpp_strerror(r) << dendl;
455 enqueue_failed_delete(r);
459 // image was already deleted from rbd_directory, now we will make sure
460 // that will be also removed from rbd_mirroring
462 dout(20) << "local image does not exist, removing image from rbd_mirroring"
466 r = cls_client::mirror_image_remove(&ioctx, local_image_id);
467 if (r < 0 && r != -ENOENT) {
468 derr << "error removing image from mirroring directory: "
469 << cpp_strerror(r) << dendl;
470 enqueue_failed_delete(r);
474 dout(10) << "Successfully deleted image "
475 << global_image_id << " " << "(" << local_image_id << ")" << dendl;
477 complete_active_delete(0);
481 template <typename I>
482 int ImageDeleter<I>::image_has_snapshots_and_children(IoCtx *ioctx,
484 bool *has_snapshots) {
485 string header_oid = librbd::util::header_name(image_id);
487 int r = cls_client::get_snapcontext(ioctx, header_oid, &snapc);
488 if (r < 0 && r != -ENOENT) {
489 derr << "error retrieving snapshot context for image id " << image_id
490 << ": " << cpp_strerror(r) << dendl;
494 *has_snapshots = !snapc.snaps.empty();
499 template <typename I>
500 void ImageDeleter<I>::complete_active_delete(int r) {
503 Mutex::Locker delete_locker(m_delete_lock);
504 m_active_delete->notify(r);
505 m_active_delete.reset();
508 template <typename I>
509 void ImageDeleter<I>::enqueue_failed_delete(int error_code) {
510 dout(20) << "enter" << dendl;
512 if (error_code == -EBLACKLISTED) {
513 derr << "blacklisted while deleting local image" << dendl;
514 complete_active_delete(error_code);
518 m_delete_lock.Lock();
519 if (m_active_delete->notify_on_failed_retry) {
520 m_active_delete->notify(error_code);
522 m_active_delete->error_code = error_code;
523 bool was_empty = m_failed_queue.empty();
524 m_failed_queue.push_front(std::move(m_active_delete));
525 m_delete_lock.Unlock();
527 FunctionContext *ctx = new FunctionContext(
528 boost::bind(&ImageDeleter<I>::retry_failed_deletions, this));
529 Mutex::Locker l(*m_failed_timer_lock);
530 m_failed_timer->add_event_after(m_failed_interval, ctx);
534 template <typename I>
535 void ImageDeleter<I>::retry_failed_deletions() {
536 dout(20) << "enter" << dendl;
538 Mutex::Locker l(m_delete_lock);
540 bool empty = m_failed_queue.empty();
541 while (!m_failed_queue.empty()) {
542 m_delete_queue.push_back(std::move(m_failed_queue.back()));
543 m_delete_queue.back()->retries++;
544 m_failed_queue.pop_back();
547 m_delete_queue_cond.Signal();
551 template <typename I>
552 unique_ptr<typename ImageDeleter<I>::DeleteInfo> const*
553 ImageDeleter<I>::find_delete_info(int64_t local_pool_id,
554 const std::string &global_image_id) {
555 assert(m_delete_lock.is_locked());
557 if (m_active_delete && m_active_delete->match(local_pool_id,
559 return &m_active_delete;
562 for (const auto& del_info : m_delete_queue) {
563 if (del_info->match(local_pool_id, global_image_id)) {
568 for (const auto& del_info : m_failed_queue) {
569 if (del_info->match(local_pool_id, global_image_id)) {
577 template <typename I>
578 void ImageDeleter<I>::print_status(Formatter *f, stringstream *ss) {
579 dout(20) << "enter" << dendl;
582 f->open_object_section("image_deleter_status");
583 f->open_array_section("delete_images_queue");
586 Mutex::Locker l(m_delete_lock);
587 for (const auto& image : m_delete_queue) {
588 image->print_status(f, ss);
593 f->open_array_section("failed_deletes_queue");
596 for (const auto& image : m_failed_queue) {
597 image->print_status(f, ss, true);
607 template <typename I>
608 void ImageDeleter<I>::DeleteInfo::notify(int r) {
610 dout(20) << "executing image deletion handler r=" << r << dendl;
612 Context *ctx = on_delete;
618 template <typename I>
619 void ImageDeleter<I>::DeleteInfo::to_string(stringstream& ss) {
620 ss << "[" << "local_pool_id=" << local_pool_id << ", ";
621 ss << "global_image_id=" << global_image_id << "]";
624 template <typename I>
625 void ImageDeleter<I>::DeleteInfo::print_status(Formatter *f, stringstream *ss,
626 bool print_failure_info) {
628 f->open_object_section("delete_info");
629 f->dump_int("local_pool_id", local_pool_id);
630 f->dump_string("global_image_id", global_image_id);
631 if (print_failure_info) {
632 f->dump_string("error_code", cpp_strerror(error_code));
633 f->dump_int("retries", retries);
638 this->to_string(*ss);
642 template <typename I>
643 vector<string> ImageDeleter<I>::get_delete_queue_items() {
644 vector<string> items;
646 Mutex::Locker l(m_delete_lock);
647 for (const auto& del_info : m_delete_queue) {
648 items.push_back(del_info->global_image_id);
654 template <typename I>
655 vector<pair<string, int> > ImageDeleter<I>::get_failed_queue_items() {
656 vector<pair<string, int> > items;
658 Mutex::Locker l(m_delete_lock);
659 for (const auto& del_info : m_failed_queue) {
660 items.push_back(make_pair(del_info->global_image_id,
661 del_info->error_code));
667 template <typename I>
668 void ImageDeleter<I>::set_failed_timer_interval(double interval) {
669 this->m_failed_interval = interval;
672 } // namespace mirror
675 template class rbd::mirror::ImageDeleter<librbd::ImageCtx>;