--- /dev/null
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil <sage@newdream.net>
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation. See file COPYING.
+ *
+ */
+
+#include "Elector.h"
+#include "Monitor.h"
+
+#include "common/Timer.h"
+#include "MonitorDBStore.h"
+#include "messages/MMonElection.h"
+
+#include "common/config.h"
+#include "include/assert.h"
+
+#define dout_subsys ceph_subsys_mon
+#undef dout_prefix
+#define dout_prefix _prefix(_dout, mon, epoch)
+static ostream& _prefix(std::ostream *_dout, Monitor *mon, epoch_t epoch) {
+ return *_dout << "mon." << mon->name << "@" << mon->rank
+ << "(" << mon->get_state_name()
+ << ").elector(" << epoch << ") ";
+}
+
+
+void Elector::init()
+{
+ epoch = mon->store->get(Monitor::MONITOR_NAME, "election_epoch");
+ if (!epoch) {
+ dout(1) << "init, first boot, initializing epoch at 1 " << dendl;
+ epoch = 1;
+ } else if (epoch % 2) {
+ dout(1) << "init, last seen epoch " << epoch
+ << ", mid-election, bumping" << dendl;
+ ++epoch;
+ auto t(std::make_shared<MonitorDBStore::Transaction>());
+ t->put(Monitor::MONITOR_NAME, "election_epoch", epoch);
+ mon->store->apply_transaction(t);
+ } else {
+ dout(1) << "init, last seen epoch " << epoch << dendl;
+ }
+}
+
+void Elector::shutdown()
+{
+ cancel_timer();
+}
+
+void Elector::bump_epoch(epoch_t e)
+{
+ dout(10) << "bump_epoch " << epoch << " to " << e << dendl;
+ assert(epoch <= e);
+ epoch = e;
+ auto t(std::make_shared<MonitorDBStore::Transaction>());
+ t->put(Monitor::MONITOR_NAME, "election_epoch", epoch);
+ mon->store->apply_transaction(t);
+
+ mon->join_election();
+
+ // clear up some state
+ electing_me = false;
+ acked_me.clear();
+}
+
+
+void Elector::start()
+{
+ if (!participating) {
+ dout(0) << "not starting new election -- not participating" << dendl;
+ return;
+ }
+ dout(5) << "start -- can i be leader?" << dendl;
+
+ acked_me.clear();
+ init();
+
+ // start by trying to elect me
+ if (epoch % 2 == 0) {
+ bump_epoch(epoch+1); // odd == election cycle
+ } else {
+ // do a trivial db write just to ensure it is writeable.
+ auto t(std::make_shared<MonitorDBStore::Transaction>());
+ t->put(Monitor::MONITOR_NAME, "election_writeable_test", rand());
+ int r = mon->store->apply_transaction(t);
+ assert(r >= 0);
+ }
+ start_stamp = ceph_clock_now();
+ electing_me = true;
+ acked_me[mon->rank].cluster_features = CEPH_FEATURES_ALL;
+ acked_me[mon->rank].mon_features = ceph::features::mon::get_supported();
+ mon->collect_metadata(&acked_me[mon->rank].metadata);
+ leader_acked = -1;
+
+ // bcast to everyone else
+ for (unsigned i=0; i<mon->monmap->size(); ++i) {
+ if ((int)i == mon->rank) continue;
+ MMonElection *m =
+ new MMonElection(MMonElection::OP_PROPOSE, epoch, mon->monmap);
+ m->mon_features = ceph::features::mon::get_supported();
+ mon->messenger->send_message(m, mon->monmap->get_inst(i));
+ }
+
+ reset_timer();
+}
+
+void Elector::defer(int who)
+{
+ dout(5) << "defer to " << who << dendl;
+
+ if (electing_me) {
+ // drop out
+ acked_me.clear();
+ electing_me = false;
+ }
+
+ // ack them
+ leader_acked = who;
+ ack_stamp = ceph_clock_now();
+ MMonElection *m = new MMonElection(MMonElection::OP_ACK, epoch, mon->monmap);
+ m->mon_features = ceph::features::mon::get_supported();
+ mon->collect_metadata(&m->metadata);
+
+ // This field is unused completely in luminous, but jewel uses it to
+ // determine whether we are a dumpling mon due to some crufty old
+ // code. It only needs to see this buffer non-empty, so put
+ // something useless there.
+ m->sharing_bl = mon->get_local_commands_bl(mon->get_required_mon_features());
+
+ mon->messenger->send_message(m, mon->monmap->get_inst(who));
+
+ // set a timer
+ reset_timer(1.0); // give the leader some extra time to declare victory
+}
+
+
+void Elector::reset_timer(double plus)
+{
+ // set the timer
+ cancel_timer();
+ /**
+ * This class is used as the callback when the expire_event timer fires up.
+ *
+ * If the expire_event is fired, then it means that we had an election going,
+ * either started by us or by some other participant, but it took too long,
+ * thus expiring.
+ *
+ * When the election expires, we will check if we were the ones who won, and
+ * if so we will declare victory. If that is not the case, then we assume
+ * that the one we defered to didn't declare victory quickly enough (in fact,
+ * as far as we know, we may even be dead); so, just propose ourselves as the
+ * Leader.
+ */
+ expire_event = mon->timer.add_event_after(
+ g_conf->mon_election_timeout + plus,
+ new C_MonContext(mon, [this](int) {
+ expire();
+ }));
+}
+
+
+void Elector::cancel_timer()
+{
+ if (expire_event) {
+ mon->timer.cancel_event(expire_event);
+ expire_event = 0;
+ }
+}
+
+void Elector::expire()
+{
+ dout(5) << "election timer expired" << dendl;
+
+ // did i win?
+ if (electing_me &&
+ acked_me.size() > (unsigned)(mon->monmap->size() / 2)) {
+ // i win
+ victory();
+ } else {
+ // whoever i deferred to didn't declare victory quickly enough.
+ if (mon->has_ever_joined)
+ start();
+ else
+ mon->bootstrap();
+ }
+}
+
+
+void Elector::victory()
+{
+ leader_acked = -1;
+ electing_me = false;
+
+ uint64_t cluster_features = CEPH_FEATURES_ALL;
+ mon_feature_t mon_features = ceph::features::mon::get_supported();
+ set<int> quorum;
+ map<int,Metadata> metadata;
+ for (map<int, elector_info_t>::iterator p = acked_me.begin();
+ p != acked_me.end();
+ ++p) {
+ quorum.insert(p->first);
+ cluster_features &= p->second.cluster_features;
+ mon_features &= p->second.mon_features;
+ metadata[p->first] = p->second.metadata;
+ }
+
+ cancel_timer();
+
+ assert(epoch % 2 == 1); // election
+ bump_epoch(epoch+1); // is over!
+
+ // tell everyone!
+ for (set<int>::iterator p = quorum.begin();
+ p != quorum.end();
+ ++p) {
+ if (*p == mon->rank) continue;
+ MMonElection *m = new MMonElection(MMonElection::OP_VICTORY, epoch,
+ mon->monmap);
+ m->quorum = quorum;
+ m->quorum_features = cluster_features;
+ m->mon_features = mon_features;
+ m->sharing_bl = mon->get_local_commands_bl(mon_features);
+ mon->messenger->send_message(m, mon->monmap->get_inst(*p));
+ }
+
+ // tell monitor
+ mon->win_election(epoch, quorum,
+ cluster_features, mon_features, metadata);
+}
+
+
+void Elector::handle_propose(MonOpRequestRef op)
+{
+ op->mark_event("elector:handle_propose");
+ MMonElection *m = static_cast<MMonElection*>(op->get_req());
+ dout(5) << "handle_propose from " << m->get_source() << dendl;
+ int from = m->get_source().num();
+
+ assert(m->epoch % 2 == 1); // election
+ uint64_t required_features = mon->get_required_features();
+ mon_feature_t required_mon_features = mon->get_required_mon_features();
+
+ dout(10) << __func__ << " required features " << required_features
+ << " " << required_mon_features
+ << ", peer features " << m->get_connection()->get_features()
+ << " " << m->mon_features
+ << dendl;
+
+ if ((required_features ^ m->get_connection()->get_features()) &
+ required_features) {
+ dout(5) << " ignoring propose from mon" << from
+ << " without required features" << dendl;
+ nak_old_peer(op);
+ return;
+ } else if (!m->mon_features.contains_all(required_mon_features)) {
+ // all the features in 'required_mon_features' not in 'm->mon_features'
+ mon_feature_t missing = required_mon_features.diff(m->mon_features);
+ dout(5) << " ignoring propose from mon." << from
+ << " without required mon_features " << missing
+ << dendl;
+ nak_old_peer(op);
+ } else if (m->epoch > epoch) {
+ bump_epoch(m->epoch);
+ } else if (m->epoch < epoch) {
+ // got an "old" propose,
+ if (epoch % 2 == 0 && // in a non-election cycle
+ mon->quorum.count(from) == 0) { // from someone outside the quorum
+ // a mon just started up, call a new election so they can rejoin!
+ dout(5) << " got propose from old epoch, quorum is " << mon->quorum
+ << ", " << m->get_source() << " must have just started" << dendl;
+ // we may be active; make sure we reset things in the monitor appropriately.
+ mon->start_election();
+ } else {
+ dout(5) << " ignoring old propose" << dendl;
+ return;
+ }
+ }
+
+ if (mon->rank < from) {
+ // i would win over them.
+ if (leader_acked >= 0) { // we already acked someone
+ assert(leader_acked < from); // and they still win, of course
+ dout(5) << "no, we already acked " << leader_acked << dendl;
+ } else {
+ // wait, i should win!
+ if (!electing_me) {
+ mon->start_election();
+ }
+ }
+ } else {
+ // they would win over me
+ if (leader_acked < 0 || // haven't acked anyone yet, or
+ leader_acked > from || // they would win over who you did ack, or
+ leader_acked == from) { // this is the guy we're already deferring to
+ defer(from);
+ } else {
+ // ignore them!
+ dout(5) << "no, we already acked " << leader_acked << dendl;
+ }
+ }
+}
+
+void Elector::handle_ack(MonOpRequestRef op)
+{
+ op->mark_event("elector:handle_ack");
+ MMonElection *m = static_cast<MMonElection*>(op->get_req());
+ dout(5) << "handle_ack from " << m->get_source() << dendl;
+ int from = m->get_source().num();
+
+ assert(m->epoch % 2 == 1); // election
+ if (m->epoch > epoch) {
+ dout(5) << "woah, that's a newer epoch, i must have rebooted. bumping and re-starting!" << dendl;
+ bump_epoch(m->epoch);
+ start();
+ return;
+ }
+ assert(m->epoch == epoch);
+ uint64_t required_features = mon->get_required_features();
+ if ((required_features ^ m->get_connection()->get_features()) &
+ required_features) {
+ dout(5) << " ignoring ack from mon" << from
+ << " without required features" << dendl;
+ return;
+ }
+
+ mon_feature_t required_mon_features = mon->get_required_mon_features();
+ if (!m->mon_features.contains_all(required_mon_features)) {
+ mon_feature_t missing = required_mon_features.diff(m->mon_features);
+ dout(5) << " ignoring ack from mon." << from
+ << " without required mon_features " << missing
+ << dendl;
+ return;
+ }
+
+ if (electing_me) {
+ // thanks
+ acked_me[from].cluster_features = m->get_connection()->get_features();
+ acked_me[from].mon_features = m->mon_features;
+ acked_me[from].metadata = m->metadata;
+ dout(5) << " so far i have {";
+ for (map<int, elector_info_t>::const_iterator p = acked_me.begin();
+ p != acked_me.end();
+ ++p) {
+ if (p != acked_me.begin())
+ *_dout << ",";
+ *_dout << " mon." << p->first << ":"
+ << " features " << p->second.cluster_features
+ << " " << p->second.mon_features;
+ }
+ *_dout << " }" << dendl;
+
+ // is that _everyone_?
+ if (acked_me.size() == mon->monmap->size()) {
+ // if yes, shortcut to election finish
+ victory();
+ }
+ } else {
+ // ignore, i'm deferring already.
+ assert(leader_acked >= 0);
+ }
+}
+
+
+void Elector::handle_victory(MonOpRequestRef op)
+{
+ op->mark_event("elector:handle_victory");
+ MMonElection *m = static_cast<MMonElection*>(op->get_req());
+ dout(5) << "handle_victory from " << m->get_source()
+ << " quorum_features " << m->quorum_features
+ << " " << m->mon_features
+ << dendl;
+ int from = m->get_source().num();
+
+ assert(from < mon->rank);
+ assert(m->epoch % 2 == 0);
+
+ leader_acked = -1;
+
+ // i should have seen this election if i'm getting the victory.
+ if (m->epoch != epoch + 1) {
+ dout(5) << "woah, that's a funny epoch, i must have rebooted. bumping and re-starting!" << dendl;
+ bump_epoch(m->epoch);
+ start();
+ return;
+ }
+
+ bump_epoch(m->epoch);
+
+ // they win
+ mon->lose_election(epoch, m->quorum, from,
+ m->quorum_features, m->mon_features);
+
+ // cancel my timer
+ cancel_timer();
+
+ // stash leader's commands
+ assert(m->sharing_bl.length());
+ vector<MonCommand> new_cmds;
+ bufferlist::iterator bi = m->sharing_bl.begin();
+ MonCommand::decode_vector(new_cmds, bi);
+ mon->set_leader_commands(new_cmds);
+}
+
+void Elector::nak_old_peer(MonOpRequestRef op)
+{
+ op->mark_event("elector:nak_old_peer");
+ MMonElection *m = static_cast<MMonElection*>(op->get_req());
+ uint64_t supported_features = m->get_connection()->get_features();
+ uint64_t required_features = mon->get_required_features();
+ mon_feature_t required_mon_features = mon->get_required_mon_features();
+ dout(10) << "sending nak to peer " << m->get_source()
+ << " that only supports " << supported_features
+ << " " << m->mon_features
+ << " of the required " << required_features
+ << " " << required_mon_features
+ << dendl;
+
+ MMonElection *reply = new MMonElection(MMonElection::OP_NAK, m->epoch,
+ mon->monmap);
+ reply->quorum_features = required_features;
+ reply->mon_features = required_mon_features;
+ mon->features.encode(reply->sharing_bl);
+ m->get_connection()->send_message(reply);
+}
+
+void Elector::handle_nak(MonOpRequestRef op)
+{
+ op->mark_event("elector:handle_nak");
+ MMonElection *m = static_cast<MMonElection*>(op->get_req());
+ dout(1) << "handle_nak from " << m->get_source()
+ << " quorum_features " << m->quorum_features
+ << " " << m->mon_features
+ << dendl;
+
+ CompatSet other;
+ bufferlist::iterator bi = m->sharing_bl.begin();
+ other.decode(bi);
+ CompatSet diff = Monitor::get_supported_features().unsupported(other);
+
+ mon_feature_t mon_supported = ceph::features::mon::get_supported();
+ // all features in 'm->mon_features' not in 'mon_supported'
+ mon_feature_t mon_diff = m->mon_features.diff(mon_supported);
+
+ derr << "Shutting down because I do not support required monitor features: { "
+ << diff << " } " << mon_diff << dendl;
+
+ exit(0);
+ // the end!
+}
+
+void Elector::dispatch(MonOpRequestRef op)
+{
+ op->mark_event("elector:dispatch");
+ assert(op->is_type_election());
+
+ switch (op->get_req()->get_type()) {
+
+ case MSG_MON_ELECTION:
+ {
+ if (!participating) {
+ return;
+ }
+ if (op->get_req()->get_source().num() >= mon->monmap->size()) {
+ dout(5) << " ignoring bogus election message with bad mon rank "
+ << op->get_req()->get_source() << dendl;
+ return;
+ }
+
+ MMonElection *em = static_cast<MMonElection*>(op->get_req());
+
+ // assume an old message encoding would have matched
+ if (em->fsid != mon->monmap->fsid) {
+ dout(0) << " ignoring election msg fsid "
+ << em->fsid << " != " << mon->monmap->fsid << dendl;
+ return;
+ }
+
+ if (!mon->monmap->contains(em->get_source_addr())) {
+ dout(1) << "discarding election message: " << em->get_source_addr()
+ << " not in my monmap " << *mon->monmap << dendl;
+ return;
+ }
+
+ MonMap peermap;
+ peermap.decode(em->monmap_bl);
+ if (peermap.epoch > mon->monmap->epoch) {
+ dout(0) << em->get_source_inst() << " has newer monmap epoch " << peermap.epoch
+ << " > my epoch " << mon->monmap->epoch
+ << ", taking it"
+ << dendl;
+ mon->monmap->decode(em->monmap_bl);
+ auto t(std::make_shared<MonitorDBStore::Transaction>());
+ t->put("monmap", mon->monmap->epoch, em->monmap_bl);
+ t->put("monmap", "last_committed", mon->monmap->epoch);
+ mon->store->apply_transaction(t);
+ //mon->monmon()->paxos->stash_latest(mon->monmap->epoch, em->monmap_bl);
+ cancel_timer();
+ mon->bootstrap();
+ return;
+ }
+ if (peermap.epoch < mon->monmap->epoch) {
+ dout(0) << em->get_source_inst() << " has older monmap epoch " << peermap.epoch
+ << " < my epoch " << mon->monmap->epoch
+ << dendl;
+ }
+
+ switch (em->op) {
+ case MMonElection::OP_PROPOSE:
+ handle_propose(op);
+ return;
+ }
+
+ if (em->epoch < epoch) {
+ dout(5) << "old epoch, dropping" << dendl;
+ break;
+ }
+
+ switch (em->op) {
+ case MMonElection::OP_ACK:
+ handle_ack(op);
+ return;
+ case MMonElection::OP_VICTORY:
+ handle_victory(op);
+ return;
+ case MMonElection::OP_NAK:
+ handle_nak(op);
+ return;
+ default:
+ ceph_abort();
+ }
+ }
+ break;
+
+ default:
+ ceph_abort();
+ }
+}
+
+void Elector::start_participating()
+{
+ if (!participating) {
+ participating = true;
+ }
+}
Code Review / stor4nfv.git / blobdiff