1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
2 // vim: ts=8 sw=2 smarttab
4 * Ceph - scalable distributed file system
6 * Copyright (C) 2016 John Spray <john.spray@redhat.com>
8 * This is free software; you can redistribute it and/or
9 * modify it under the terms of the GNU Lesser General Public
10 * License version 2.1, as published by the Free Software
11 * Foundation. See file COPYING.
14 #include <boost/tokenizer.hpp>
16 #include "messages/MMgrBeacon.h"
17 #include "messages/MMgrMap.h"
18 #include "messages/MMgrDigest.h"
20 #include "PGStatService.h"
21 #include "include/stringify.h"
22 #include "mgr/MgrContext.h"
23 #include "mgr/mgr_commands.h"
24 #include "OSDMonitor.h"
26 #include "MgrMonitor.h"
28 #define MGR_METADATA_PREFIX "mgr_metadata"
30 #define dout_subsys ceph_subsys_mon
32 #define dout_prefix _prefix(_dout, mon, map)
33 static ostream& _prefix(std::ostream *_dout, Monitor *mon,
34 const MgrMap& mgrmap) {
35 return *_dout << "mon." << mon->name << "@" << mon->rank
36 << "(" << mon->get_state_name()
37 << ").mgr e" << mgrmap.get_epoch() << " ";
40 // Prefix for mon store of active mgr's command descriptions
41 const static std::string command_descs_prefix = "mgr_command_descs";
44 void MgrMonitor::create_initial()
46 // Take a local copy of initial_modules for tokenizer to iterate over.
47 auto initial_modules = g_conf->get_val<std::string>("mgr_initial_modules");
48 boost::tokenizer<> tok(initial_modules);
50 pending_map.modules.insert(m);
52 pending_command_descs = mgr_commands;
53 dout(10) << __func__ << " initial modules " << pending_map.modules
54 << ", " << pending_command_descs.size() << " commands"
58 void MgrMonitor::get_store_prefixes(std::set<string>& s)
60 s.insert(service_name);
61 s.insert(command_descs_prefix);
62 s.insert(MGR_METADATA_PREFIX);
65 void MgrMonitor::update_from_paxos(bool *need_bootstrap)
67 version_t version = get_last_committed();
68 if (version != map.epoch) {
69 dout(4) << "loading version " << version << dendl;
72 int err = get_version(version, bl);
75 bool old_available = map.get_available();
76 uint64_t old_gid = map.get_active_gid();
78 bufferlist::iterator p = bl.begin();
81 dout(4) << "active server: " << map.active_addr
82 << "(" << map.active_gid << ")" << dendl;
84 ever_had_active_mgr = get_value("ever_had_active_mgr");
89 first_seen_inactive = utime_t();
91 first_seen_inactive = ceph_clock_now();
97 || command_descs.empty()
98 || (map.get_available()
99 && (!old_available || old_gid != map.get_active_gid()))) {
100 dout(4) << "mkfs or daemon transitioned to available, loading commands"
102 bufferlist loaded_commands;
103 int r = mon->store->get(command_descs_prefix, "", loaded_commands);
105 derr << "Failed to load mgr commands: " << cpp_strerror(r) << dendl;
107 auto p = loaded_commands.begin();
108 ::decode(command_descs, p);
113 // feed our pet MgrClient
114 mon->mgr_client.ms_dispatch(new MMgrMap(map));
117 void MgrMonitor::create_pending()
122 if (map.get_epoch() == 1 &&
123 command_descs.empty() &&
124 pending_command_descs.empty()) {
125 // we've been through the initial map and we haven't populated the
126 // command_descs vector. This likely means we came from kraken, where
127 // we wouldn't populate the vector, nor would we write it to disk, on
133 health_status_t MgrMonitor::should_warn_about_mgr_down()
135 utime_t now = ceph_clock_now();
137 // - we've ever had an active mgr, or
138 // - we have osds AND we've exceeded the grace period
139 // which means a new mon cluster and be HEALTH_OK indefinitely as long as
140 // no OSDs are ever created.
141 if (ever_had_active_mgr ||
142 (mon->osdmon()->osdmap.get_num_osds() > 0 &&
143 now > mon->monmap->created + g_conf->get_val<int64_t>("mon_mgr_mkfs_grace"))) {
144 health_status_t level = HEALTH_WARN;
145 if (first_seen_inactive != utime_t() &&
146 now - first_seen_inactive > g_conf->get_val<int64_t>("mon_mgr_inactive_grace")) {
154 void MgrMonitor::encode_pending(MonitorDBStore::TransactionRef t)
156 dout(10) << __func__ << " " << pending_map << dendl;
158 pending_map.encode(bl, mon->get_quorum_con_features());
159 put_version(t, pending_map.epoch, bl);
160 put_last_committed(t, pending_map.epoch);
162 for (auto& p : pending_metadata) {
163 dout(10) << __func__ << " set metadata for " << p.first << dendl;
164 t->put(MGR_METADATA_PREFIX, p.first, p.second);
166 for (auto& name : pending_metadata_rm) {
167 dout(10) << __func__ << " rm metadata for " << name << dendl;
168 t->erase(MGR_METADATA_PREFIX, name);
170 pending_metadata.clear();
171 pending_metadata_rm.clear();
173 health_check_map_t next;
174 if (pending_map.active_gid == 0) {
175 auto level = should_warn_about_mgr_down();
176 if (level != HEALTH_OK) {
177 next.add("MGR_DOWN", level, "no active mgr");
179 dout(10) << __func__ << " no health warning (never active and new cluster)"
183 put_value(t, "ever_had_active_mgr", 1);
185 encode_health(next, t);
187 if (pending_command_descs.size()) {
188 dout(4) << __func__ << " encoding " << pending_command_descs.size()
189 << " command_descs" << dendl;
190 for (auto& p : pending_command_descs) {
191 p.set_flag(MonCommand::FLAG_MGR);
194 ::encode(pending_command_descs, bl);
195 t->put(command_descs_prefix, "", bl);
196 pending_command_descs.clear();
200 bool MgrMonitor::check_caps(MonOpRequestRef op, const uuid_d& fsid)
203 MonSession *session = op->get_session();
206 if (!session->is_capable("mgr", MON_CAP_X)) {
207 dout(1) << __func__ << " insufficient caps " << session->caps << dendl;
210 if (fsid != mon->monmap->fsid) {
211 dout(1) << __func__ << " op fsid " << fsid
212 << " != " << mon->monmap->fsid << dendl;
218 bool MgrMonitor::preprocess_query(MonOpRequestRef op)
220 PaxosServiceMessage *m = static_cast<PaxosServiceMessage*>(op->get_req());
221 switch (m->get_type()) {
223 return preprocess_beacon(op);
224 case MSG_MON_COMMAND:
225 return preprocess_command(op);
228 derr << "Unhandled message type " << m->get_type() << dendl;
233 bool MgrMonitor::prepare_update(MonOpRequestRef op)
235 PaxosServiceMessage *m = static_cast<PaxosServiceMessage*>(op->get_req());
236 switch (m->get_type()) {
238 return prepare_beacon(op);
240 case MSG_MON_COMMAND:
241 return prepare_command(op);
245 derr << "Unhandled message type " << m->get_type() << dendl;
252 class C_Updated : public Context {
256 C_Updated(MgrMonitor *a, MonOpRequestRef c) :
258 void finish(int r) override {
261 } else if (r == -ECANCELED) {
262 mm->mon->no_reply(op);
264 mm->dispatch(op); // try again
269 bool MgrMonitor::preprocess_beacon(MonOpRequestRef op)
271 MMgrBeacon *m = static_cast<MMgrBeacon*>(op->get_req());
272 dout(4) << "beacon from " << m->get_gid() << dendl;
274 if (!check_caps(op, m->get_fsid())) {
275 // drop it on the floor
279 // always send this to the leader's prepare_beacon()
283 bool MgrMonitor::prepare_beacon(MonOpRequestRef op)
285 MMgrBeacon *m = static_cast<MMgrBeacon*>(op->get_req());
286 dout(4) << "beacon from " << m->get_gid() << dendl;
288 // See if we are seeing same name, new GID for the active daemon
289 if (m->get_name() == pending_map.active_name
290 && m->get_gid() != pending_map.active_gid)
292 dout(4) << "Active daemon restart (mgr." << m->get_name() << ")" << dendl;
293 mon->clog->info() << "Active manager daemon " << m->get_name()
298 // See if we are seeing same name, new GID for any standbys
299 for (const auto &i : pending_map.standbys) {
300 const StandbyInfo &s = i.second;
301 if (s.name == m->get_name() && s.gid != m->get_gid()) {
302 dout(4) << "Standby daemon restart (mgr." << m->get_name() << ")" << dendl;
303 mon->clog->debug() << "Standby manager daemon " << m->get_name()
305 drop_standby(i.first);
310 last_beacon[m->get_gid()] = ceph::coarse_mono_clock::now();
312 // Track whether we modified pending_map
313 bool updated = false;
315 if (pending_map.active_gid == m->get_gid()) {
316 if (pending_map.services != m->get_services()) {
317 dout(4) << "updated services from mgr." << m->get_name()
318 << ": " << m->get_services() << dendl;
319 pending_map.services = m->get_services();
323 // A beacon from the currently active daemon
324 if (pending_map.active_addr != m->get_server_addr()) {
325 dout(4) << "learned address " << m->get_server_addr()
326 << " (was " << pending_map.active_addr << ")" << dendl;
327 pending_map.active_addr = m->get_server_addr();
331 if (pending_map.get_available() != m->get_available()) {
332 dout(4) << "available " << m->get_gid() << dendl;
333 mon->clog->info() << "Manager daemon " << pending_map.active_name
334 << " is now available";
336 // This beacon should include command descriptions
337 pending_command_descs = m->get_command_descs();
338 if (pending_command_descs.empty()) {
339 // This should not happen, but it also isn't fatal: we just
340 // won't successfully update our list of commands.
341 dout(4) << "First available beacon from " << pending_map.active_name
342 << "(" << m->get_gid() << ") does not include command descs"
345 dout(4) << "First available beacon from " << pending_map.active_name
346 << "(" << m->get_gid() << ") includes "
347 << pending_command_descs.size() << " command descs" << dendl;
350 pending_map.available = m->get_available();
353 if (pending_map.available_modules != m->get_available_modules()) {
354 dout(4) << "available_modules " << m->get_available_modules()
355 << " (was " << pending_map.available_modules << ")" << dendl;
356 pending_map.available_modules = m->get_available_modules();
359 } else if (pending_map.active_gid == 0) {
360 // There is no currently active daemon, select this one.
361 if (pending_map.standbys.count(m->get_gid())) {
362 drop_standby(m->get_gid(), false);
364 dout(4) << "selecting new active " << m->get_gid()
365 << " " << m->get_name()
366 << " (was " << pending_map.active_gid << " "
367 << pending_map.active_name << ")" << dendl;
368 pending_map.active_gid = m->get_gid();
369 pending_map.active_name = m->get_name();
370 pending_map.available_modules = m->get_available_modules();
371 ::encode(m->get_metadata(), pending_metadata[m->get_name()]);
372 pending_metadata_rm.erase(m->get_name());
374 mon->clog->info() << "Activating manager daemon "
375 << pending_map.active_name;
379 if (pending_map.standbys.count(m->get_gid()) > 0) {
380 dout(10) << "from existing standby " << m->get_gid() << dendl;
381 if (pending_map.standbys[m->get_gid()].available_modules !=
382 m->get_available_modules()) {
383 dout(10) << "existing standby " << m->get_gid() << " available_modules "
384 << m->get_available_modules() << " (was "
385 << pending_map.standbys[m->get_gid()].available_modules << ")"
387 pending_map.standbys[m->get_gid()].available_modules =
388 m->get_available_modules();
392 dout(10) << "new standby " << m->get_gid() << dendl;
393 mon->clog->debug() << "Standby manager daemon " << m->get_name()
395 pending_map.standbys[m->get_gid()] = {m->get_gid(), m->get_name(),
396 m->get_available_modules()};
397 ::encode(m->get_metadata(), pending_metadata[m->get_name()]);
398 pending_metadata_rm.erase(m->get_name());
404 dout(4) << "updating map" << dendl;
405 wait_for_finished_proposal(op, new C_Updated(this, op));
407 dout(10) << "no change" << dendl;
413 void MgrMonitor::check_subs()
415 const std::string type = "mgrmap";
416 if (mon->session_map.subs.count(type) == 0)
418 for (auto sub : *(mon->session_map.subs[type])) {
423 void MgrMonitor::check_sub(Subscription *sub)
425 if (sub->type == "mgrmap") {
426 if (sub->next <= map.get_epoch()) {
427 dout(20) << "Sending map to subscriber " << sub->session->con
428 << " " << sub->session->con->get_peer_addr() << dendl;
429 sub->session->con->send_message(new MMgrMap(map));
431 mon->session_map.remove_sub(sub);
433 sub->next = map.get_epoch() + 1;
437 assert(sub->type == "mgrdigest");
438 if (sub->next == 0) {
439 // new registration; cancel previous timer
442 if (digest_event == nullptr) {
449 * Handle digest subscriptions separately (outside of check_sub) because
450 * they are going to be periodic rather than version-driven.
452 void MgrMonitor::send_digests()
459 dout(10) << __func__ << dendl;
461 const std::string type = "mgrdigest";
462 if (mon->session_map.subs.count(type) == 0)
465 for (auto sub : *(mon->session_map.subs[type])) {
466 dout(10) << __func__ << " sending digest to subscriber " << sub->session->con
467 << " " << sub->session->con->get_peer_addr() << dendl;
468 MMgrDigest *mdigest = new MMgrDigest;
471 mon->get_health_status(true, &f, nullptr, nullptr, nullptr);
472 f.flush(mdigest->health_json);
475 std::ostringstream ss;
476 mon->get_mon_status(&f, ss);
477 f.flush(mdigest->mon_status_json);
480 sub->session->con->send_message(mdigest);
483 digest_event = mon->timer.add_event_after(
484 g_conf->get_val<int64_t>("mon_mgr_digest_period"),
485 new C_MonContext(mon, [this](int) {
490 void MgrMonitor::cancel_timer()
493 mon->timer.cancel_event(digest_event);
494 digest_event = nullptr;
498 void MgrMonitor::on_active()
500 if (mon->is_leader()) {
501 mon->clog->debug() << "mgrmap e" << map.epoch << ": " << map;
505 void MgrMonitor::get_health(
506 list<pair<health_status_t,string> >& summary,
507 list<pair<health_status_t,string> > *detail,
508 CephContext *cct) const
510 // start mgr warnings as soon as the mons and osds are all upgraded,
511 // but before the require_luminous osdmap flag is set. this way the
512 // user gets some warning before the osd flag is set and mgr is
513 // actually *required*.
514 if (!mon->monmap->get_required_features().contains_all(
515 ceph::features::mon::FEATURE_LUMINOUS) ||
516 !HAVE_FEATURE(mon->osdmon()->osdmap.get_up_osd_features(),
521 if (map.active_gid == 0) {
522 auto level = HEALTH_WARN;
523 // do not escalate to ERR if they are still upgrading to jewel.
524 if (mon->osdmon()->osdmap.require_osd_release >= CEPH_RELEASE_LUMINOUS) {
525 utime_t now = ceph_clock_now();
526 if (first_seen_inactive != utime_t() &&
527 now - first_seen_inactive > g_conf->get_val<int64_t>("mon_mgr_inactive_grace")) {
531 summary.push_back(make_pair(level, "no active mgr"));
535 void MgrMonitor::tick()
537 if (!is_active() || !mon->is_leader())
540 const auto now = ceph::coarse_mono_clock::now();
542 const auto mgr_beacon_grace = std::chrono::seconds(
543 g_conf->get_val<int64_t>("mon_mgr_beacon_grace"));
545 // Note that this is the mgr daemon's tick period, not ours (the
546 // beacon is sent with this period).
547 const auto mgr_tick_period = std::chrono::seconds(
548 g_conf->get_val<int64_t>("mgr_tick_period"));
550 if (last_tick != ceph::coarse_mono_clock::time_point::min()
551 && (now - last_tick > (mgr_beacon_grace - mgr_tick_period))) {
552 // This case handles either local slowness (calls being delayed
553 // for whatever reason) or cluster election slowness (a long gap
554 // between calls while an election happened)
555 dout(4) << __func__ << ": resetting beacon timeouts due to mon delay "
556 "(slow election?) of " << now - last_tick << " seconds" << dendl;
557 for (auto &i : last_beacon) {
564 // Populate any missing beacons (i.e. no beacon since MgrMonitor
565 // instantiation) with the current time, so that they will
566 // eventually look laggy if they fail to give us a beacon.
567 if (pending_map.active_gid != 0
568 && last_beacon.count(pending_map.active_gid) == 0) {
569 last_beacon[pending_map.active_gid] = now;
571 for (auto s : pending_map.standbys) {
572 if (last_beacon.count(s.first) == 0) {
573 last_beacon[s.first] = now;
577 // Cull standbys first so that any remaining standbys
578 // will be eligible to take over from the active if we cull him.
579 std::list<uint64_t> dead_standbys;
580 const auto cutoff = now - mgr_beacon_grace;
581 for (const auto &i : pending_map.standbys) {
582 auto last_beacon_time = last_beacon.at(i.first);
583 if (last_beacon_time < cutoff) {
584 dead_standbys.push_back(i.first);
588 bool propose = false;
590 for (auto i : dead_standbys) {
591 dout(4) << "Dropping laggy standby " << i << dendl;
596 if (pending_map.active_gid != 0
597 && last_beacon.at(pending_map.active_gid) < cutoff) {
598 const std::string old_active_name = pending_map.active_name;
601 dout(4) << "Dropping active" << pending_map.active_gid << dendl;
602 if (promote_standby()) {
603 dout(4) << "Promoted standby " << pending_map.active_gid << dendl;
604 mon->clog->info() << "Manager daemon " << old_active_name
605 << " is unresponsive, replacing it with standby"
606 << " daemon " << pending_map.active_name;
608 dout(4) << "Active is laggy but have no standbys to replace it" << dendl;
609 mon->clog->warn() << "Manager daemon " << old_active_name
610 << " is unresponsive. No standby daemons available.";
612 } else if (pending_map.active_gid == 0) {
613 if (promote_standby()) {
614 dout(4) << "Promoted standby " << pending_map.active_gid << dendl;
615 mon->clog->info() << "Activating manager daemon "
616 << pending_map.active_name;
621 if (!pending_map.available &&
622 !ever_had_active_mgr &&
623 should_warn_about_mgr_down() != HEALTH_OK) {
624 dout(10) << " exceeded mon_mgr_mkfs_grace "
625 << g_conf->get_val<int64_t>("mon_mgr_mkfs_grace")
626 << " seconds" << dendl;
635 void MgrMonitor::on_restart()
637 // Clear out the leader-specific state.
639 last_tick = ceph::coarse_mono_clock::now();
643 bool MgrMonitor::promote_standby()
645 assert(pending_map.active_gid == 0);
646 if (pending_map.standbys.size()) {
647 // Promote a replacement (arbitrary choice of standby)
648 auto replacement_gid = pending_map.standbys.begin()->first;
649 pending_map.active_gid = replacement_gid;
650 pending_map.active_name = pending_map.standbys.at(replacement_gid).name;
651 pending_map.available = false;
652 pending_map.active_addr = entity_addr_t();
654 drop_standby(replacement_gid, false);
662 void MgrMonitor::drop_active()
664 if (last_beacon.count(pending_map.active_gid) > 0) {
665 last_beacon.erase(pending_map.active_gid);
668 pending_metadata_rm.insert(pending_map.active_name);
669 pending_metadata.erase(pending_map.active_name);
670 pending_map.active_name = "";
671 pending_map.active_gid = 0;
672 pending_map.available = false;
673 pending_map.active_addr = entity_addr_t();
674 pending_map.services.clear();
676 // So that when new active mgr subscribes to mgrdigest, it will
677 // get an immediate response instead of waiting for next timer
681 void MgrMonitor::drop_standby(uint64_t gid, bool drop_meta)
684 pending_metadata_rm.insert(pending_map.standbys[gid].name);
685 pending_metadata.erase(pending_map.standbys[gid].name);
687 pending_map.standbys.erase(gid);
688 if (last_beacon.count(gid) > 0) {
689 last_beacon.erase(gid);
693 bool MgrMonitor::preprocess_command(MonOpRequestRef op)
695 MMonCommand *m = static_cast<MMonCommand*>(op->get_req());
696 std::stringstream ss;
699 std::map<std::string, cmd_vartype> cmdmap;
700 if (!cmdmap_from_json(m->cmd, &cmdmap, ss)) {
701 string rs = ss.str();
702 mon->reply_command(op, -EINVAL, rs, rdata, get_last_committed());
706 MonSession *session = m->get_session();
708 mon->reply_command(op, -EACCES, "access denied", rdata,
709 get_last_committed());
714 cmd_getval(g_ceph_context, cmdmap, "format", format, string("json-pretty"));
715 boost::scoped_ptr<Formatter> f(Formatter::create(format));
718 cmd_getval(g_ceph_context, cmdmap, "prefix", prefix);
721 if (prefix == "mgr dump") {
723 cmd_getval(g_ceph_context, cmdmap, "epoch", epoch, (int64_t)map.get_epoch());
724 if (epoch == (int64_t)map.get_epoch()) {
725 f->dump_object("mgrmap", map);
728 int err = get_version(epoch, bl);
729 if (err == -ENOENT) {
731 ss << "there is no map for epoch " << epoch;
737 f->dump_object("mgrmap", m);
740 } else if (prefix == "mgr module ls") {
741 f->open_object_section("modules");
743 f->open_array_section("enabled_modules");
744 for (auto& p : map.modules) {
745 f->dump_string("module", p);
748 f->open_array_section("disabled_modules");
749 for (auto& p : map.available_modules) {
750 if (map.modules.count(p) == 0) {
751 f->dump_string("module", p);
758 } else if (prefix == "mgr services") {
759 f->open_object_section("services");
760 for (const auto &i : map.services) {
761 f->dump_string(i.first.c_str(), i.second);
765 } else if (prefix == "mgr metadata") {
767 cmd_getval(g_ceph_context, cmdmap, "id", name);
768 if (name.size() > 0 && !map.have_name(name)) {
769 ss << "mgr." << name << " does not exist";
774 cmd_getval(g_ceph_context, cmdmap, "format", format);
775 boost::scoped_ptr<Formatter> f(Formatter::create(format, "json-pretty", "json-pretty"));
777 f->open_object_section("mgr_metadata");
778 f->dump_string("id", name);
779 r = dump_metadata(name, f.get(), &ss);
785 f->open_array_section("mgr_metadata");
786 for (auto& i : map.get_all_names()) {
787 f->open_object_section("mgr");
788 f->dump_string("id", i);
789 r = dump_metadata(i, f.get(), NULL);
790 if (r == -EINVAL || r == -ENOENT) {
791 // Drop error, continue to get other daemons' metadata
792 dout(4) << "No metadata for mgr." << i << dendl;
803 } else if (prefix == "mgr versions") {
805 f.reset(Formatter::create("json-pretty"));
806 count_metadata("ceph_version", f.get());
809 } else if (prefix == "mgr count-metadata") {
811 f.reset(Formatter::create("json-pretty"));
813 cmd_getval(g_ceph_context, cmdmap, "property", field);
814 count_metadata(field, f.get());
824 mon->reply_command(op, r, rs, rdata, get_last_committed());
828 bool MgrMonitor::prepare_command(MonOpRequestRef op)
830 MMonCommand *m = static_cast<MMonCommand*>(op->get_req());
832 std::stringstream ss;
835 std::map<std::string, cmd_vartype> cmdmap;
836 if (!cmdmap_from_json(m->cmd, &cmdmap, ss)) {
837 string rs = ss.str();
838 mon->reply_command(op, -EINVAL, rs, rdata, get_last_committed());
842 MonSession *session = m->get_session();
844 mon->reply_command(op, -EACCES, "access denied", rdata, get_last_committed());
849 cmd_getval(g_ceph_context, cmdmap, "format", format, string("plain"));
850 boost::scoped_ptr<Formatter> f(Formatter::create(format));
853 cmd_getval(g_ceph_context, cmdmap, "prefix", prefix);
857 if (prefix == "mgr fail") {
859 cmd_getval(g_ceph_context, cmdmap, "who", who);
862 uint64_t gid = strict_strtol(who.c_str(), 10, &err);
863 bool changed = false;
865 // Does not parse as a gid, treat it as a name
866 if (pending_map.active_name == who) {
871 for (const auto &i : pending_map.standbys) {
872 if (i.second.name == who) {
881 ss << "Daemon not found '" << who << "', already failed?";
885 if (pending_map.active_gid == gid) {
888 } else if (pending_map.standbys.count(gid) > 0) {
892 ss << "Daemon not found '" << gid << "', already failed?";
896 if (changed && pending_map.active_gid == 0) {
899 } else if (prefix == "mgr module enable") {
901 cmd_getval(g_ceph_context, cmdmap, "module", module);
902 if (module.empty()) {
907 cmd_getval(g_ceph_context, cmdmap, "force", force);
908 if (!pending_map.all_support_module(module) &&
909 force != "--force") {
910 ss << "all mgr daemons do not support module '" << module << "', pass "
911 << "--force to force enablement";
915 pending_map.modules.insert(module);
916 } else if (prefix == "mgr module disable") {
918 cmd_getval(g_ceph_context, cmdmap, "module", module);
919 if (module.empty()) {
923 pending_map.modules.erase(module);
925 ss << "Command '" << prefix << "' not implemented!";
930 dout(4) << __func__ << " done, r=" << r << dendl;
931 /* Compose response */
936 // success.. delay reply
937 wait_for_finished_proposal(op, new Monitor::C_Command(mon, op, r, rs,
938 get_last_committed() + 1));
942 mon->reply_command(op, r, rs, rdata, get_last_committed());
947 void MgrMonitor::init()
949 if (digest_event == nullptr) {
950 send_digests(); // To get it to schedule its own event
954 void MgrMonitor::on_shutdown()
959 int MgrMonitor::load_metadata(const string& name, std::map<string, string>& m,
963 int r = mon->store->get(MGR_METADATA_PREFIX, name, bl);
967 bufferlist::iterator p = bl.begin();
970 catch (buffer::error& e) {
972 *err << "mgr." << name << " metadata is corrupt";
978 void MgrMonitor::count_metadata(const string& field, std::map<string,int> *out)
980 std::set<string> ls = map.get_all_names();
981 for (auto& name : ls) {
982 std::map<string,string> meta;
983 load_metadata(name, meta, nullptr);
984 auto p = meta.find(field);
985 if (p == meta.end()) {
993 void MgrMonitor::count_metadata(const string& field, Formatter *f)
995 std::map<string,int> by_val;
996 count_metadata(field, &by_val);
997 f->open_object_section(field.c_str());
998 for (auto& p : by_val) {
999 f->dump_int(p.first.c_str(), p.second);
1004 int MgrMonitor::dump_metadata(const string& name, Formatter *f, ostream *err)
1006 std::map<string,string> m;
1007 if (int r = load_metadata(name, m, err))
1010 f->dump_string(p.first.c_str(), p.second);
1015 const std::vector<MonCommand> &MgrMonitor::get_command_descs() const
1017 if (command_descs.empty()) {
1018 // must have just upgraded; fallback to static commands
1019 return mgr_commands;
1021 return command_descs;