1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
2 // vim: ts=8 sw=2 smarttab
4 * Ceph - scalable distributed file system
6 * Copyright (C) 2015 Red Hat
8 * This is free software; you can redistribute it and/or
9 * modify it under the terms of the GNU Lesser General Public
10 * License version 2.1, as published by the Free Software
11 * Foundation. See file COPYING.
18 #include "common/DecayCounter.h"
19 #include "common/LogClient.h"
20 #include "common/Timer.h"
21 #include "common/TrackedOp.h"
23 #include "messages/MCommand.h"
26 #include "DamageTable.h"
28 #include "SessionMap.h"
32 #include "PurgeQueue.h"
33 #include "osdc/Journaler.h"
35 // Full .h import instead of forward declaration for PerfCounter, for the
36 // benefit of those including this header and using MDSRank::logger
37 #include "common/perf_counters.h"
53 l_mds_inodes_pin_tail,
56 l_mds_inodes_with_caps,
61 l_mds_traverse_forward,
62 l_mds_traverse_discover,
63 l_mds_traverse_dir_fetch,
64 l_mds_traverse_remote_ino,
67 l_mds_dispatch_queue_len,
69 l_mds_exported_inodes,
71 l_mds_imported_inodes,
97 struct heartbeat_handle_d;
108 class MDSTableServer;
109 class MDSTableClient;
118 * The public part of this class's interface is what's exposed to all
119 * the various subsystems (server, mdcache, etc), such as pointers
120 * to the other subsystems, and message-sending calls.
124 const mds_rank_t whoami;
126 // Incarnation as seen in MDSMap at the point where a rank is
131 mds_rank_t get_nodeid() const { return whoami; }
132 int64_t get_metadata_pool();
134 // Reference to global MDS::mds_lock, so that users of MDSRank don't
135 // carry around references to the outer MDS, and we can substitute
136 // a separate lock here in future potentially.
139 bool is_daemon_stopping() const;
141 // Reference to global cluster log client, just to avoid initialising
142 // a separate one here.
145 // Reference to global timer utility, because MDSRank and MDSDaemon
146 // currently both use the same mds_lock, so it makes sense for them
159 MDBalancer *balancer;
160 ScrubStack *scrubstack;
161 DamageTable damage_table;
166 SnapServer *snapserver;
167 SnapClient *snapclient;
169 MDSTableClient *get_table_client(int t);
170 MDSTableServer *get_table_server(int t);
172 SessionMap sessionmap;
173 Session *get_session(client_t client) {
174 return sessionmap.get_session(entity_name_t::CLIENT(client.v));
177 PerfCounters *logger, *mlogger;
178 OpTracker op_tracker;
180 // The last different state I held before current
181 MDSMap::DaemonState last_state;
182 // The state assigned to me by the MDSMap
183 MDSMap::DaemonState state;
185 bool cluster_degraded;
187 MDSMap::DaemonState get_state() const { return state; }
188 MDSMap::DaemonState get_want_state() const { return beacon.get_want_state(); }
190 bool is_creating() const { return state == MDSMap::STATE_CREATING; }
191 bool is_starting() const { return state == MDSMap::STATE_STARTING; }
192 bool is_standby() const { return state == MDSMap::STATE_STANDBY; }
193 bool is_replay() const { return state == MDSMap::STATE_REPLAY; }
194 bool is_standby_replay() const { return state == MDSMap::STATE_STANDBY_REPLAY; }
195 bool is_resolve() const { return state == MDSMap::STATE_RESOLVE; }
196 bool is_reconnect() const { return state == MDSMap::STATE_RECONNECT; }
197 bool is_rejoin() const { return state == MDSMap::STATE_REJOIN; }
198 bool is_clientreplay() const { return state == MDSMap::STATE_CLIENTREPLAY; }
199 bool is_active() const { return state == MDSMap::STATE_ACTIVE; }
200 bool is_stopping() const { return state == MDSMap::STATE_STOPPING; }
201 bool is_any_replay() const { return (is_replay() || is_standby_replay()); }
202 bool is_stopped() const { return mdsmap->is_stopped(whoami); }
203 bool is_cluster_degraded() const { return cluster_degraded; }
205 void handle_write_error(int err);
207 void handle_conf_change(const struct md_config_t *conf,
208 const std::set <std::string> &changed)
210 purge_queue.handle_conf_change(conf, changed, *mdsmap);
213 void update_mlogger();
215 // Flag to indicate we entered shutdown: anyone seeing this to be true
216 // after taking mds_lock must drop out.
219 // PurgeQueue is only used by StrayManager, but it is owned by MDSRank
220 // because its init/shutdown happens at the top level.
221 PurgeQueue purge_queue;
223 class ProgressThread : public Thread {
227 explicit ProgressThread(MDSRank *mds_) : mds(mds_) {}
228 void * entry() override;
230 void signal() {cond.Signal();}
233 list<Message*> waiting_for_nolaggy;
234 list<MDSInternalContextBase*> finished_queue;
235 // Dispatch, retry, queues
237 void inc_dispatch_depth() { ++dispatch_depth; }
238 void dec_dispatch_depth() { --dispatch_depth; }
239 void retry_dispatch(Message *m);
240 bool handle_deferrable_message(Message *m);
241 void _advance_queues();
242 bool _dispatch(Message *m, bool new_msg);
244 ceph::heartbeat_handle_d *hb; // Heartbeat for threads using mds_lock
246 bool is_stale_message(Message *m) const;
248 map<mds_rank_t, version_t> peer_mdsmap_epoch;
250 ceph_tid_t last_tid; // for mds-initiated requests (e.g. stray rename)
252 list<MDSInternalContextBase*> waiting_for_active, waiting_for_replay, waiting_for_reconnect, waiting_for_resolve;
253 list<MDSInternalContextBase*> replay_queue;
254 map<mds_rank_t, list<MDSInternalContextBase*> > waiting_for_active_peer;
255 map<epoch_t, list<MDSInternalContextBase*> > waiting_for_mdsmap;
257 epoch_t osd_epoch_barrier;
259 // Const reference to the beacon so that we can behave differently
264 * Emit clog warnings for any ops reported as warnings by optracker
266 void check_ops_in_flight();
268 int mds_slow_req_count;
271 * Share MDSMap with clients
273 void bcast_mds_map(); // to mounted clients
274 epoch_t last_client_mdsmap_bcast;
276 map<mds_rank_t,DecayCounter> export_targets; /* targets this MDS is exporting to or wants/tries to */
278 void create_logger();
281 void queue_waiter(MDSInternalContextBase *c) {
282 finished_queue.push_back(c);
283 progress_thread.signal();
285 void queue_waiters(list<MDSInternalContextBase*>& ls) {
286 finished_queue.splice( finished_queue.end(), ls );
287 progress_thread.signal();
293 LogChannelRef &clog_,
299 Context *respawn_hook_,
300 Context *suicide_hook_);
307 // Daemon lifetime functions: these guys break the abstraction
308 // and call up into the parent MDSDaemon instance. It's kind
309 // of unavoidable: if we want any depth into our calls
310 // to be able to e.g. tear down the whole process, we have to
311 // have a reference going all the way down.
318 * Call this periodically if inside a potentially long running piece
319 * of code while holding the mds_lock
321 void heartbeat_reset();
324 * Report state DAMAGED to the mon, and then pass on to respawn(). Call
325 * this when an unrecoverable error is encountered while attempting
326 * to load an MDS rank's data structures. This is *not* for use with
327 * errors affecting normal dirfrag/inode objects -- they should be handled
328 * through cleaner scrub/repair mechanisms.
330 * Callers must already hold mds_lock.
335 * Wrapper around `damaged` for users who are not
336 * already holding mds_lock.
338 * Callers must not already hold mds_lock.
340 void damaged_unlocked();
342 utime_t get_laggy_until() const;
344 void send_message_mds(Message *m, mds_rank_t mds);
345 void forward_message_mds(Message *req, mds_rank_t mds);
347 void send_message_client_counted(Message *m, client_t client);
348 void send_message_client_counted(Message *m, Session *session);
349 void send_message_client_counted(Message *m, Connection *connection);
350 void send_message_client_counted(Message *m, const ConnectionRef& con) {
351 send_message_client_counted(m, con.get());
353 void send_message_client(Message *m, Session *session);
354 void send_message(Message *m, Connection *c);
355 void send_message(Message *m, const ConnectionRef& c) {
356 send_message(m, c.get());
359 void wait_for_active_peer(mds_rank_t who, MDSInternalContextBase *c) {
360 waiting_for_active_peer[who].push_back(c);
362 void wait_for_cluster_recovered(MDSInternalContextBase *c) {
363 assert(cluster_degraded);
364 waiting_for_active_peer[MDS_RANK_NONE].push_back(c);
367 void wait_for_active(MDSInternalContextBase *c) {
368 waiting_for_active.push_back(c);
370 void wait_for_replay(MDSInternalContextBase *c) {
371 waiting_for_replay.push_back(c);
373 void wait_for_reconnect(MDSInternalContextBase *c) {
374 waiting_for_reconnect.push_back(c);
376 void wait_for_resolve(MDSInternalContextBase *c) {
377 waiting_for_resolve.push_back(c);
379 void wait_for_mdsmap(epoch_t e, MDSInternalContextBase *c) {
380 waiting_for_mdsmap[e].push_back(c);
382 void enqueue_replay(MDSInternalContextBase *c) {
383 replay_queue.push_back(c);
386 bool queue_one_replay();
388 void set_osd_epoch_barrier(epoch_t e);
389 epoch_t get_osd_epoch_barrier() const {return osd_epoch_barrier;}
390 epoch_t get_osd_epoch() const;
392 ceph_tid_t issue_tid() { return ++last_tid; }
396 MDSMap *get_mds_map() { return mdsmap; }
398 int get_req_rate() const { return logger->get(l_mds_request); }
400 int get_mds_slow_req_count() const { return mds_slow_req_count; }
402 void dump_status(Formatter *f) const;
404 void hit_export_target(utime_t now, mds_rank_t rank, double amount=-1.0);
405 bool is_export_target(mds_rank_t rank) {
406 const set<mds_rank_t>& map_targets = mdsmap->get_mds_info(get_nodeid()).export_targets;
407 return map_targets.count(rank);
410 bool evict_client(int64_t session_id, bool wait, bool blacklist,
411 std::stringstream& ss, Context *on_killed=nullptr);
414 void dump_clientreplay_status(Formatter *f) const;
415 void command_scrub_path(Formatter *f, const string& path, vector<string>& scrubop_vec);
416 void command_tag_path(Formatter *f, const string& path,
418 void command_flush_path(Formatter *f, const string& path);
419 void command_flush_journal(Formatter *f);
420 void command_get_subtrees(Formatter *f);
421 void command_export_dir(Formatter *f,
422 const std::string &path, mds_rank_t dest);
423 bool command_dirfrag_split(
426 bool command_dirfrag_merge(
429 bool command_dirfrag_ls(
433 int _command_export_dir(const std::string &path, mds_rank_t dest);
434 int _command_flush_journal(std::stringstream *ss);
435 CDir *_command_dirfrag_get(
436 const cmdmap_t &cmdmap,
440 Messenger *messenger;
443 Context *respawn_hook;
444 Context *suicide_hook;
446 // Friended to access retry_dispatch
447 friend class C_MDS_RetryMessage;
449 // FIXME the state machine logic should be separable from the dispatch
450 // logic that calls it.
452 void calc_recovery_set();
453 void request_state(MDSMap::DaemonState s);
455 bool standby_replaying; // true if current replay pass is in standby-replay mode
458 // The MDSMap is available, configure default layouts and structures
459 MDS_BOOT_INITIAL = 0,
460 // We are ready to open some inodes
462 // We are ready to do a replay if needed
463 MDS_BOOT_PREPARE_LOG,
464 // Replay is complete
467 friend class C_MDS_BootStart;
468 friend class C_MDS_InternalBootStart;
469 void boot_create(); // i am new mds.
470 void boot_start(BootStep step=MDS_BOOT_INITIAL, int r=0); // starting|replay
473 void creating_done();
474 void starting_done();
476 void standby_replay_restart();
477 void _standby_replay_restart_finish(int r, uint64_t old_read_pos);
478 class C_MDS_StandbyReplayRestart;
479 class C_MDS_StandbyReplayRestartFinish;
483 void resolve_start();
485 void reconnect_start();
486 void reconnect_done();
487 void rejoin_joint_start();
490 void recovery_done(int oldstate);
491 void clientreplay_start();
492 void clientreplay_done();
494 void stopping_start();
495 void stopping_done();
497 void validate_sessions();
501 void handle_mds_recovery(mds_rank_t who);
502 void handle_mds_failure(mds_rank_t who);
505 /* Update MDSMap export_targets for this rank. Called on ::tick(). */
506 void update_targets(utime_t now);
509 /* This expects to be given a reference which it is responsible for.
510 * The finish function calls functions which
511 * will put the Message exactly once.*/
512 class C_MDS_RetryMessage : public MDSInternalContext {
516 C_MDS_RetryMessage(MDSRank *mds, Message *m)
517 : MDSInternalContext(mds)
522 void finish(int r) override {
523 mds->retry_dispatch(m);
528 * The aspect of MDSRank exposed to MDSDaemon but not subsystems: i.e.
529 * the service/dispatcher stuff like init/shutdown that subsystems should
532 class MDSRankDispatcher : public MDSRank
538 bool handle_asok_command(std::string command, cmdmap_t& cmdmap,
539 Formatter *f, std::ostream& ss);
540 void handle_mds_map(MMDSMap *m, MDSMap *oldmap);
541 void handle_osd_map();
542 void update_log_config();
545 const cmdmap_t &cmdmap,
548 std::stringstream *ds,
549 std::stringstream *ss,
552 void dump_sessions(const SessionFilter &filter, Formatter *f) const;
553 void evict_clients(const SessionFilter &filter, MCommand *m);
555 // Call into me from MDS::ms_dispatch
556 bool ms_dispatch(Message *m);
561 LogChannelRef &clog_,
567 Context *respawn_hook_,
568 Context *suicide_hook_);
571 // This utility for MDS and MDSRank dispatchers.
572 #define ALLOW_MESSAGES_FROM(peers) \
574 if (m->get_connection() && (m->get_connection()->get_peer_type() & (peers)) == 0) { \
575 dout(0) << __FILE__ << "." << __LINE__ << ": filtered out request, peer=" << m->get_connection()->get_peer_type() \
576 << " allowing=" << #peers << " message=" << *m << dendl; \
582 #endif // MDS_RANK_H_