--- /dev/null
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2015 Red Hat
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation. See file COPYING.
+ *
+ */
+
+#ifndef MDS_RANK_H_
+#define MDS_RANK_H_
+
+#include "common/DecayCounter.h"
+#include "common/LogClient.h"
+#include "common/Timer.h"
+#include "common/TrackedOp.h"
+
+#include "messages/MCommand.h"
+
+#include "Beacon.h"
+#include "DamageTable.h"
+#include "MDSMap.h"
+#include "SessionMap.h"
+#include "MDCache.h"
+#include "Migrator.h"
+#include "MDLog.h"
+#include "PurgeQueue.h"
+#include "osdc/Journaler.h"
+
+// Full .h import instead of forward declaration for PerfCounter, for the
+// benefit of those including this header and using MDSRank::logger
+#include "common/perf_counters.h"
+
+enum {
+ l_mds_first = 2000,
+ l_mds_request,
+ l_mds_reply,
+ l_mds_reply_latency,
+ l_mds_forward,
+ l_mds_dir_fetch,
+ l_mds_dir_commit,
+ l_mds_dir_split,
+ l_mds_dir_merge,
+ l_mds_inode_max,
+ l_mds_inodes,
+ l_mds_inodes_top,
+ l_mds_inodes_bottom,
+ l_mds_inodes_pin_tail,
+ l_mds_inodes_pinned,
+ l_mds_inodes_expired,
+ l_mds_inodes_with_caps,
+ l_mds_caps,
+ l_mds_subtrees,
+ l_mds_traverse,
+ l_mds_traverse_hit,
+ l_mds_traverse_forward,
+ l_mds_traverse_discover,
+ l_mds_traverse_dir_fetch,
+ l_mds_traverse_remote_ino,
+ l_mds_traverse_lock,
+ l_mds_load_cent,
+ l_mds_dispatch_queue_len,
+ l_mds_exported,
+ l_mds_exported_inodes,
+ l_mds_imported,
+ l_mds_imported_inodes,
+ l_mds_last,
+};
+
+// memory utilization
+enum {
+ l_mdm_first = 2500,
+ l_mdm_ino,
+ l_mdm_inoa,
+ l_mdm_inos,
+ l_mdm_dir,
+ l_mdm_dira,
+ l_mdm_dirs,
+ l_mdm_dn,
+ l_mdm_dna,
+ l_mdm_dns,
+ l_mdm_cap,
+ l_mdm_capa,
+ l_mdm_caps,
+ l_mdm_rss,
+ l_mdm_heap,
+ l_mdm_buf,
+ l_mdm_last,
+};
+
+namespace ceph {
+ struct heartbeat_handle_d;
+}
+
+class Server;
+class Locker;
+class MDCache;
+class MDLog;
+class MDBalancer;
+class InoTable;
+class SnapServer;
+class SnapClient;
+class MDSTableServer;
+class MDSTableClient;
+class Messenger;
+class Objecter;
+class MonClient;
+class Finisher;
+class MMDSMap;
+class ScrubStack;
+
+/**
+ * The public part of this class's interface is what's exposed to all
+ * the various subsystems (server, mdcache, etc), such as pointers
+ * to the other subsystems, and message-sending calls.
+ */
+class MDSRank {
+ protected:
+ const mds_rank_t whoami;
+
+ // Incarnation as seen in MDSMap at the point where a rank is
+ // assigned.
+ int incarnation;
+
+ public:
+ mds_rank_t get_nodeid() const { return whoami; }
+ int64_t get_metadata_pool();
+
+ // Reference to global MDS::mds_lock, so that users of MDSRank don't
+ // carry around references to the outer MDS, and we can substitute
+ // a separate lock here in future potentially.
+ Mutex &mds_lock;
+
+ bool is_daemon_stopping() const;
+
+ // Reference to global cluster log client, just to avoid initialising
+ // a separate one here.
+ LogChannelRef &clog;
+
+ // Reference to global timer utility, because MDSRank and MDSDaemon
+ // currently both use the same mds_lock, so it makes sense for them
+ // to share a timer.
+ SafeTimer &timer;
+
+ MDSMap *&mdsmap;
+
+ Objecter *objecter;
+
+ // sub systems
+ Server *server;
+ MDCache *mdcache;
+ Locker *locker;
+ MDLog *mdlog;
+ MDBalancer *balancer;
+ ScrubStack *scrubstack;
+ DamageTable damage_table;
+
+
+ InoTable *inotable;
+
+ SnapServer *snapserver;
+ SnapClient *snapclient;
+
+ MDSTableClient *get_table_client(int t);
+ MDSTableServer *get_table_server(int t);
+
+ SessionMap sessionmap;
+ Session *get_session(client_t client) {
+ return sessionmap.get_session(entity_name_t::CLIENT(client.v));
+ }
+
+ PerfCounters *logger, *mlogger;
+ OpTracker op_tracker;
+
+ // The last different state I held before current
+ MDSMap::DaemonState last_state;
+ // The state assigned to me by the MDSMap
+ MDSMap::DaemonState state;
+
+ bool cluster_degraded;
+
+ MDSMap::DaemonState get_state() const { return state; }
+ MDSMap::DaemonState get_want_state() const { return beacon.get_want_state(); }
+
+ bool is_creating() const { return state == MDSMap::STATE_CREATING; }
+ bool is_starting() const { return state == MDSMap::STATE_STARTING; }
+ bool is_standby() const { return state == MDSMap::STATE_STANDBY; }
+ bool is_replay() const { return state == MDSMap::STATE_REPLAY; }
+ bool is_standby_replay() const { return state == MDSMap::STATE_STANDBY_REPLAY; }
+ bool is_resolve() const { return state == MDSMap::STATE_RESOLVE; }
+ bool is_reconnect() const { return state == MDSMap::STATE_RECONNECT; }
+ bool is_rejoin() const { return state == MDSMap::STATE_REJOIN; }
+ bool is_clientreplay() const { return state == MDSMap::STATE_CLIENTREPLAY; }
+ bool is_active() const { return state == MDSMap::STATE_ACTIVE; }
+ bool is_stopping() const { return state == MDSMap::STATE_STOPPING; }
+ bool is_any_replay() const { return (is_replay() || is_standby_replay()); }
+ bool is_stopped() const { return mdsmap->is_stopped(whoami); }
+ bool is_cluster_degraded() const { return cluster_degraded; }
+
+ void handle_write_error(int err);
+
+ void handle_conf_change(const struct md_config_t *conf,
+ const std::set <std::string> &changed)
+ {
+ purge_queue.handle_conf_change(conf, changed, *mdsmap);
+ }
+
+ void update_mlogger();
+ protected:
+ // Flag to indicate we entered shutdown: anyone seeing this to be true
+ // after taking mds_lock must drop out.
+ bool stopping;
+
+ // PurgeQueue is only used by StrayManager, but it is owned by MDSRank
+ // because its init/shutdown happens at the top level.
+ PurgeQueue purge_queue;
+
+ class ProgressThread : public Thread {
+ MDSRank *mds;
+ Cond cond;
+ public:
+ explicit ProgressThread(MDSRank *mds_) : mds(mds_) {}
+ void * entry() override;
+ void shutdown();
+ void signal() {cond.Signal();}
+ } progress_thread;
+
+ list<Message*> waiting_for_nolaggy;
+ list<MDSInternalContextBase*> finished_queue;
+ // Dispatch, retry, queues
+ int dispatch_depth;
+ void inc_dispatch_depth() { ++dispatch_depth; }
+ void dec_dispatch_depth() { --dispatch_depth; }
+ void retry_dispatch(Message *m);
+ bool handle_deferrable_message(Message *m);
+ void _advance_queues();
+ bool _dispatch(Message *m, bool new_msg);
+
+ ceph::heartbeat_handle_d *hb; // Heartbeat for threads using mds_lock
+
+ bool is_stale_message(Message *m) const;
+
+ map<mds_rank_t, version_t> peer_mdsmap_epoch;
+
+ ceph_tid_t last_tid; // for mds-initiated requests (e.g. stray rename)
+
+ list<MDSInternalContextBase*> waiting_for_active, waiting_for_replay, waiting_for_reconnect, waiting_for_resolve;
+ list<MDSInternalContextBase*> replay_queue;
+ map<mds_rank_t, list<MDSInternalContextBase*> > waiting_for_active_peer;
+ map<epoch_t, list<MDSInternalContextBase*> > waiting_for_mdsmap;
+
+ epoch_t osd_epoch_barrier;
+
+ // Const reference to the beacon so that we can behave differently
+ // when it's laggy.
+ Beacon &beacon;
+
+ /**
+ * Emit clog warnings for any ops reported as warnings by optracker
+ */
+ void check_ops_in_flight();
+
+ int mds_slow_req_count;
+
+ /**
+ * Share MDSMap with clients
+ */
+ void bcast_mds_map(); // to mounted clients
+ epoch_t last_client_mdsmap_bcast;
+
+ map<mds_rank_t,DecayCounter> export_targets; /* targets this MDS is exporting to or wants/tries to */
+
+ void create_logger();
+ public:
+
+ void queue_waiter(MDSInternalContextBase *c) {
+ finished_queue.push_back(c);
+ progress_thread.signal();
+ }
+ void queue_waiters(list<MDSInternalContextBase*>& ls) {
+ finished_queue.splice( finished_queue.end(), ls );
+ progress_thread.signal();
+ }
+
+ MDSRank(
+ mds_rank_t whoami_,
+ Mutex &mds_lock_,
+ LogChannelRef &clog_,
+ SafeTimer &timer_,
+ Beacon &beacon_,
+ MDSMap *& mdsmap_,
+ Messenger *msgr,
+ MonClient *monc_,
+ Context *respawn_hook_,
+ Context *suicide_hook_);
+
+ protected:
+ ~MDSRank();
+
+ public:
+
+ // Daemon lifetime functions: these guys break the abstraction
+ // and call up into the parent MDSDaemon instance. It's kind
+ // of unavoidable: if we want any depth into our calls
+ // to be able to e.g. tear down the whole process, we have to
+ // have a reference going all the way down.
+ // >>>
+ void suicide();
+ void respawn();
+ // <<<
+
+ /**
+ * Call this periodically if inside a potentially long running piece
+ * of code while holding the mds_lock
+ */
+ void heartbeat_reset();
+
+ /**
+ * Report state DAMAGED to the mon, and then pass on to respawn(). Call
+ * this when an unrecoverable error is encountered while attempting
+ * to load an MDS rank's data structures. This is *not* for use with
+ * errors affecting normal dirfrag/inode objects -- they should be handled
+ * through cleaner scrub/repair mechanisms.
+ *
+ * Callers must already hold mds_lock.
+ */
+ void damaged();
+
+ /**
+ * Wrapper around `damaged` for users who are not
+ * already holding mds_lock.
+ *
+ * Callers must not already hold mds_lock.
+ */
+ void damaged_unlocked();
+
+ utime_t get_laggy_until() const;
+
+ void send_message_mds(Message *m, mds_rank_t mds);
+ void forward_message_mds(Message *req, mds_rank_t mds);
+
+ void send_message_client_counted(Message *m, client_t client);
+ void send_message_client_counted(Message *m, Session *session);
+ void send_message_client_counted(Message *m, Connection *connection);
+ void send_message_client_counted(Message *m, const ConnectionRef& con) {
+ send_message_client_counted(m, con.get());
+ }
+ void send_message_client(Message *m, Session *session);
+ void send_message(Message *m, Connection *c);
+ void send_message(Message *m, const ConnectionRef& c) {
+ send_message(m, c.get());
+ }
+
+ void wait_for_active_peer(mds_rank_t who, MDSInternalContextBase *c) {
+ waiting_for_active_peer[who].push_back(c);
+ }
+ void wait_for_cluster_recovered(MDSInternalContextBase *c) {
+ assert(cluster_degraded);
+ waiting_for_active_peer[MDS_RANK_NONE].push_back(c);
+ }
+
+ void wait_for_active(MDSInternalContextBase *c) {
+ waiting_for_active.push_back(c);
+ }
+ void wait_for_replay(MDSInternalContextBase *c) {
+ waiting_for_replay.push_back(c);
+ }
+ void wait_for_reconnect(MDSInternalContextBase *c) {
+ waiting_for_reconnect.push_back(c);
+ }
+ void wait_for_resolve(MDSInternalContextBase *c) {
+ waiting_for_resolve.push_back(c);
+ }
+ void wait_for_mdsmap(epoch_t e, MDSInternalContextBase *c) {
+ waiting_for_mdsmap[e].push_back(c);
+ }
+ void enqueue_replay(MDSInternalContextBase *c) {
+ replay_queue.push_back(c);
+ }
+
+ bool queue_one_replay();
+
+ void set_osd_epoch_barrier(epoch_t e);
+ epoch_t get_osd_epoch_barrier() const {return osd_epoch_barrier;}
+ epoch_t get_osd_epoch() const;
+
+ ceph_tid_t issue_tid() { return ++last_tid; }
+
+ Finisher *finisher;
+
+ MDSMap *get_mds_map() { return mdsmap; }
+
+ int get_req_rate() const { return logger->get(l_mds_request); }
+
+ int get_mds_slow_req_count() const { return mds_slow_req_count; }
+
+ void dump_status(Formatter *f) const;
+
+ void hit_export_target(utime_t now, mds_rank_t rank, double amount=-1.0);
+ bool is_export_target(mds_rank_t rank) {
+ const set<mds_rank_t>& map_targets = mdsmap->get_mds_info(get_nodeid()).export_targets;
+ return map_targets.count(rank);
+ }
+
+ bool evict_client(int64_t session_id, bool wait, bool blacklist,
+ std::stringstream& ss, Context *on_killed=nullptr);
+
+ protected:
+ void dump_clientreplay_status(Formatter *f) const;
+ void command_scrub_path(Formatter *f, const string& path, vector<string>& scrubop_vec);
+ void command_tag_path(Formatter *f, const string& path,
+ const string &tag);
+ void command_flush_path(Formatter *f, const string& path);
+ void command_flush_journal(Formatter *f);
+ void command_get_subtrees(Formatter *f);
+ void command_export_dir(Formatter *f,
+ const std::string &path, mds_rank_t dest);
+ bool command_dirfrag_split(
+ cmdmap_t cmdmap,
+ std::ostream &ss);
+ bool command_dirfrag_merge(
+ cmdmap_t cmdmap,
+ std::ostream &ss);
+ bool command_dirfrag_ls(
+ cmdmap_t cmdmap,
+ std::ostream &ss,
+ Formatter *f);
+ int _command_export_dir(const std::string &path, mds_rank_t dest);
+ int _command_flush_journal(std::stringstream *ss);
+ CDir *_command_dirfrag_get(
+ const cmdmap_t &cmdmap,
+ std::ostream &ss);
+
+ protected:
+ Messenger *messenger;
+ MonClient *monc;
+
+ Context *respawn_hook;
+ Context *suicide_hook;
+
+ // Friended to access retry_dispatch
+ friend class C_MDS_RetryMessage;
+
+ // FIXME the state machine logic should be separable from the dispatch
+ // logic that calls it.
+ // >>>
+ void calc_recovery_set();
+ void request_state(MDSMap::DaemonState s);
+
+ bool standby_replaying; // true if current replay pass is in standby-replay mode
+
+ typedef enum {
+ // The MDSMap is available, configure default layouts and structures
+ MDS_BOOT_INITIAL = 0,
+ // We are ready to open some inodes
+ MDS_BOOT_OPEN_ROOT,
+ // We are ready to do a replay if needed
+ MDS_BOOT_PREPARE_LOG,
+ // Replay is complete
+ MDS_BOOT_REPLAY_DONE
+ } BootStep;
+ friend class C_MDS_BootStart;
+ friend class C_MDS_InternalBootStart;
+ void boot_create(); // i am new mds.
+ void boot_start(BootStep step=MDS_BOOT_INITIAL, int r=0); // starting|replay
+
+ void replay_start();
+ void creating_done();
+ void starting_done();
+ void replay_done();
+ void standby_replay_restart();
+ void _standby_replay_restart_finish(int r, uint64_t old_read_pos);
+ class C_MDS_StandbyReplayRestart;
+ class C_MDS_StandbyReplayRestartFinish;
+
+ void reopen_log();
+
+ void resolve_start();
+ void resolve_done();
+ void reconnect_start();
+ void reconnect_done();
+ void rejoin_joint_start();
+ void rejoin_start();
+ void rejoin_done();
+ void recovery_done(int oldstate);
+ void clientreplay_start();
+ void clientreplay_done();
+ void active_start();
+ void stopping_start();
+ void stopping_done();
+
+ void validate_sessions();
+ // <<<
+
+ // >>>
+ void handle_mds_recovery(mds_rank_t who);
+ void handle_mds_failure(mds_rank_t who);
+ // <<<
+
+ /* Update MDSMap export_targets for this rank. Called on ::tick(). */
+ void update_targets(utime_t now);
+};
+
+/* This expects to be given a reference which it is responsible for.
+ * The finish function calls functions which
+ * will put the Message exactly once.*/
+class C_MDS_RetryMessage : public MDSInternalContext {
+protected:
+ Message *m;
+public:
+ C_MDS_RetryMessage(MDSRank *mds, Message *m)
+ : MDSInternalContext(mds)
+ {
+ assert(m);
+ this->m = m;
+ }
+ void finish(int r) override {
+ mds->retry_dispatch(m);
+ }
+};
+
+/**
+ * The aspect of MDSRank exposed to MDSDaemon but not subsystems: i.e.
+ * the service/dispatcher stuff like init/shutdown that subsystems should
+ * never touch.
+ */
+class MDSRankDispatcher : public MDSRank
+{
+public:
+ void init();
+ void tick();
+ void shutdown();
+ bool handle_asok_command(std::string command, cmdmap_t& cmdmap,
+ Formatter *f, std::ostream& ss);
+ void handle_mds_map(MMDSMap *m, MDSMap *oldmap);
+ void handle_osd_map();
+ void update_log_config();
+
+ bool handle_command(
+ const cmdmap_t &cmdmap,
+ MCommand *m,
+ int *r,
+ std::stringstream *ds,
+ std::stringstream *ss,
+ bool *need_reply);
+
+ void dump_sessions(const SessionFilter &filter, Formatter *f) const;
+ void evict_clients(const SessionFilter &filter, MCommand *m);
+
+ // Call into me from MDS::ms_dispatch
+ bool ms_dispatch(Message *m);
+
+ MDSRankDispatcher(
+ mds_rank_t whoami_,
+ Mutex &mds_lock_,
+ LogChannelRef &clog_,
+ SafeTimer &timer_,
+ Beacon &beacon_,
+ MDSMap *& mdsmap_,
+ Messenger *msgr,
+ MonClient *monc_,
+ Context *respawn_hook_,
+ Context *suicide_hook_);
+};
+
+// This utility for MDS and MDSRank dispatchers.
+#define ALLOW_MESSAGES_FROM(peers) \
+do { \
+ if (m->get_connection() && (m->get_connection()->get_peer_type() & (peers)) == 0) { \
+ dout(0) << __FILE__ << "." << __LINE__ << ": filtered out request, peer=" << m->get_connection()->get_peer_type() \
+ << " allowing=" << #peers << " message=" << *m << dendl; \
+ m->put(); \
+ return true; \
+ } \
+} while (0)
+
+#endif // MDS_RANK_H_
+
Code Review / stor4nfv.git / blobdiff