src/ceph/src/tools/rbd_mirror/LeaderWatcher.h

   1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
   2 // vim: ts=8 sw=2 smarttab
   3
   4 #ifndef CEPH_RBD_MIRROR_LEADER_WATCHER_H
   5 #define CEPH_RBD_MIRROR_LEADER_WATCHER_H
   6
   7 #include <list>
   8 #include <memory>
   9 #include <string>
  10
  11 #include "common/AsyncOpTracker.h"
  12 #include "librbd/ManagedLock.h"
  13 #include "librbd/Watcher.h"
  14 #include "librbd/managed_lock/Types.h"
  15 #include "librbd/watcher/Types.h"
  16 #include "Instances.h"
  17 #include "MirrorStatusWatcher.h"
  18 #include "tools/rbd_mirror/leader_watcher/Types.h"
  19
  20 namespace librbd { class ImageCtx; }
  21
  22 namespace rbd {
  23 namespace mirror {
  24
  25 template <typename> struct Threads;
  26
  27 template <typename ImageCtxT = librbd::ImageCtx>
  28 class LeaderWatcher : protected librbd::Watcher {
  29 public:
  30   struct Listener {
  31     virtual ~Listener() {
  32     }
  33
  34     virtual void post_acquire_handler(Context *on_finish) = 0;
  35     virtual void pre_release_handler(Context *on_finish) = 0;
  36
  37     virtual void update_leader_handler(
  38       const std::string &leader_instance_id) = 0;
  39   };
  40
  41   LeaderWatcher(Threads<ImageCtxT> *threads, librados::IoCtx &io_ctx,
  42                 Listener *listener);
  43   ~LeaderWatcher() override;
  44
  45   int init();
  46   void shut_down();
  47
  48   void init(Context *on_finish);
  49   void shut_down(Context *on_finish);
  50
  51   bool is_leader() const;
  52   bool is_releasing_leader() const;
  53   bool get_leader_instance_id(std::string *instance_id) const;
  54   void release_leader();
  55   void list_instances(std::vector<std::string> *instance_ids);
  56
  57   std::string get_instance_id();
  58
  59 private:
  60   /**
  61    * @verbatim
  62    *
  63    *  <uninitialized> <------------------------------ WAIT_FOR_TASKS
  64    *     | (init)      ^                                      ^
  65    *     v             *                                      |
  66    *  CREATE_OBJECT  * *  (error)                     UNREGISTER_WATCH
  67    *     |             *                                      ^
  68    *     v             *                                      |
  69    *  REGISTER_WATCH * *                              SHUT_DOWN_LEADER_LOCK
  70    *     |                                                    ^
  71    *     |           (no leader heartbeat and acquire failed) |
  72    *     | BREAK_LOCK <-------------------------------------\ |
  73    *     |    |                 (no leader heartbeat)       | | (shut down)
  74    *     |    |  /----------------------------------------\ | |
  75    *     |    |  |              (lock_released received)    | |
  76    *     |    |  |  /-------------------------------------\ | |
  77    *     |    |  |  |                   (lock_acquired or | | |
  78    *     |    |  |  |                 heartbeat received) | | |
  79    *     |    |  |  |       (ENOENT)        /-----------\ | | |
  80    *     |    |  |  |  * * * * * * * * * *  |           | | | |
  81    *     v    v  v  v  v  (error)        *  v           | | | |
  82    *  ACQUIRE_LEADER_LOCK  * * * * *> GET_LOCKER ---> <secondary>
  83    *     |                   *                           ^
  84    * ....|...................*....................  .....|.....................
  85    * .   v                   *                   .  .    |       post_release .
  86    * .INIT_STATUS_WATCHER  * * (error)           .  .NOTIFY_LOCK_RELEASED     .
  87    * .   |                   ^                   .  .....^.....................
  88    * .   v         (error)   |                   .       |
  89    * .INIT_INSTANCES *> SHUT_DOWN_STATUS_WATCHER .   RELEASE_LEADER_LOCK
  90    * .   |                                       .       ^
  91    * .   v                                       .  .....|.....................
  92    * .NOTIFY_LISTENER                            .  .SHUT_DOWN_STATUS_WATCHER .
  93    * .   |                                       .  .    ^                    .
  94    * .   v                                       .  .    |                    .
  95    * .NOTIFY_LOCK_ACQUIRED          post_acquire .  .SHUT_DOWN_INSTANCES      .
  96    * ....|........................................  .    ^                    .
  97    *     v                                          .    |                    .
  98    *  <leader> -----------------------------------> .NOTIFY_LISTENER          .
  99    *            (shut_down, release_leader,         .             pre_release .
 100    *             notify error)                      ...........................
 101    * @endverbatim
 102    */
 103
 104   class LeaderLock : public librbd::ManagedLock<ImageCtxT> {
 105   public:
 106     typedef librbd::ManagedLock<ImageCtxT> Parent;
 107
 108     LeaderLock(librados::IoCtx& ioctx, ContextWQ *work_queue,
 109                const std::string& oid, LeaderWatcher *watcher,
 110                bool blacklist_on_break_lock,
 111                uint32_t blacklist_expire_seconds)
 112       : Parent(ioctx, work_queue, oid, watcher, librbd::managed_lock::EXCLUSIVE,
 113                blacklist_on_break_lock, blacklist_expire_seconds),
 114         watcher(watcher) {
 115     }
 116
 117     bool is_leader() const {
 118       Mutex::Locker locker(Parent::m_lock);
 119       return Parent::is_state_post_acquiring() || Parent::is_state_locked();
 120     }
 121
 122     bool is_releasing_leader() const {
 123       Mutex::Locker locker(Parent::m_lock);
 124       return Parent::is_state_pre_releasing();
 125     }
 126
 127   protected:
 128     void post_acquire_lock_handler(int r, Context *on_finish) {
 129       if (r == 0) {
 130         // lock is owned at this point
 131         Mutex::Locker locker(Parent::m_lock);
 132         Parent::set_state_post_acquiring();
 133       }
 134       watcher->handle_post_acquire_leader_lock(r, on_finish);
 135     }
 136     void pre_release_lock_handler(bool shutting_down,
 137                                   Context *on_finish) {
 138       watcher->handle_pre_release_leader_lock(on_finish);
 139     }
 140     void post_release_lock_handler(bool shutting_down, int r,
 141                                    Context *on_finish) {
 142       watcher->handle_post_release_leader_lock(r, on_finish);
 143     }
 144   private:
 145     LeaderWatcher *watcher;
 146   };
 147
 148   struct HandlePayloadVisitor : public boost::static_visitor<void> {
 149     LeaderWatcher *leader_watcher;
 150     Context *on_notify_ack;
 151
 152     HandlePayloadVisitor(LeaderWatcher *leader_watcher, Context *on_notify_ack)
 153       : leader_watcher(leader_watcher), on_notify_ack(on_notify_ack) {
 154     }
 155
 156     template <typename Payload>
 157     inline void operator()(const Payload &payload) const {
 158       leader_watcher->handle_payload(payload, on_notify_ack);
 159     }
 160   };
 161
 162   struct C_GetLocker : public Context {
 163     LeaderWatcher *leader_watcher;
 164     librbd::managed_lock::Locker locker;
 165
 166     C_GetLocker(LeaderWatcher *leader_watcher)
 167       : leader_watcher(leader_watcher) {
 168     }
 169
 170     void finish(int r) override {
 171       leader_watcher->handle_get_locker(r, locker);
 172     }
 173   };
 174
 175   typedef void (LeaderWatcher<ImageCtxT>::*TimerCallback)();
 176
 177   struct C_TimerGate : public Context {
 178     LeaderWatcher *leader_watcher;
 179
 180     bool leader = false;
 181     TimerCallback timer_callback = nullptr;
 182
 183     C_TimerGate(LeaderWatcher *leader_watcher)
 184       : leader_watcher(leader_watcher) {
 185     }
 186
 187     void finish(int r) override {
 188       leader_watcher->m_timer_gate = nullptr;
 189       leader_watcher->execute_timer_task(leader, timer_callback);
 190     }
 191   };
 192
 193   Threads<ImageCtxT> *m_threads;
 194   Listener *m_listener;
 195
 196   mutable Mutex m_lock;
 197   uint64_t m_notifier_id;
 198   LeaderLock *m_leader_lock;
 199   Context *m_on_finish = nullptr;
 200   Context *m_on_shut_down_finish = nullptr;
 201   int m_acquire_attempts = 0;
 202   int m_ret_val = 0;
 203   MirrorStatusWatcher<ImageCtxT> *m_status_watcher = nullptr;
 204   Instances<ImageCtxT> *m_instances = nullptr;
 205   librbd::managed_lock::Locker m_locker;
 206
 207   AsyncOpTracker m_timer_op_tracker;
 208   Context *m_timer_task = nullptr;
 209   C_TimerGate *m_timer_gate = nullptr;
 210
 211   librbd::watcher::NotifyResponse m_heartbeat_response;
 212
 213   bool is_leader(Mutex &m_lock) const;
 214   bool is_releasing_leader(Mutex &m_lock) const;
 215
 216   void cancel_timer_task();
 217   void schedule_timer_task(const std::string &name,
 218                            int delay_factor, bool leader,
 219                            TimerCallback callback, bool shutting_down);
 220   void execute_timer_task(bool leader, TimerCallback timer_callback);
 221
 222   void create_leader_object();
 223   void handle_create_leader_object(int r);
 224
 225   void register_watch();
 226   void handle_register_watch(int r);
 227
 228   void shut_down_leader_lock();
 229   void handle_shut_down_leader_lock(int r);
 230
 231   void unregister_watch();
 232   void handle_unregister_watch(int r);
 233
 234   void wait_for_tasks();
 235   void handle_wait_for_tasks();
 236
 237   void break_leader_lock();
 238   void handle_break_leader_lock(int r);
 239
 240   void schedule_get_locker(bool reset_leader, uint32_t delay_factor);
 241   void get_locker();
 242   void handle_get_locker(int r, librbd::managed_lock::Locker& locker);
 243
 244   void schedule_acquire_leader_lock(uint32_t delay_factor);
 245   void acquire_leader_lock();
 246   void handle_acquire_leader_lock(int r);
 247
 248   void release_leader_lock();
 249   void handle_release_leader_lock(int r);
 250
 251   void init_status_watcher();
 252   void handle_init_status_watcher(int r);
 253
 254   void shut_down_status_watcher();
 255   void handle_shut_down_status_watcher(int r);
 256
 257   void init_instances();
 258   void handle_init_instances(int r);
 259
 260   void shut_down_instances();
 261   void handle_shut_down_instances(int r);
 262
 263   void notify_listener();
 264   void handle_notify_listener(int r);
 265
 266   void notify_lock_acquired();
 267   void handle_notify_lock_acquired(int r);
 268
 269   void notify_lock_released();
 270   void handle_notify_lock_released(int r);
 271
 272   void notify_heartbeat();
 273   void handle_notify_heartbeat(int r);
 274
 275   void get_instances();
 276   void handle_get_instances(int r);
 277
 278   void handle_post_acquire_leader_lock(int r, Context *on_finish);
 279   void handle_pre_release_leader_lock(Context *on_finish);
 280   void handle_post_release_leader_lock(int r, Context *on_finish);
 281
 282   void handle_notify(uint64_t notify_id, uint64_t handle,
 283                      uint64_t notifier_id, bufferlist &bl) override;
 284
 285   void handle_heartbeat(Context *on_ack);
 286   void handle_lock_acquired(Context *on_ack);
 287   void handle_lock_released(Context *on_ack);
 288
 289   void handle_payload(const leader_watcher::HeartbeatPayload &payload,
 290                       Context *on_notify_ack);
 291   void handle_payload(const leader_watcher::LockAcquiredPayload &payload,
 292                       Context *on_notify_ack);
 293   void handle_payload(const leader_watcher::LockReleasedPayload &payload,
 294                       Context *on_notify_ack);
 295   void handle_payload(const leader_watcher::UnknownPayload &payload,
 296                       Context *on_notify_ack);
 297 };
 298
 299 } // namespace mirror
 300 } // namespace rbd
 301
 302 #endif // CEPH_RBD_MIRROR_LEADER_WATCHER_H