1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
2 // vim: ts=8 sw=2 smarttab
4 * Ceph - scalable distributed file system
6 * Copyright (C) 2016 John Spray <john.spray@redhat.com>
8 * This is free software; you can redistribute it and/or
9 * modify it under the terms of the GNU Lesser General Public
10 * License version 2.1, as published by the Free Software
11 * Foundation. See file COPYING.
16 #include "common/errno.h"
17 #include "common/signal.h"
18 #include "include/compat.h"
20 #include "include/stringify.h"
21 #include "global/global_context.h"
22 #include "global/signal_handler.h"
24 #include "mgr/MgrContext.h"
26 #include "messages/MMgrBeacon.h"
27 #include "messages/MMgrMap.h"
30 #include "MgrStandby.h"
32 #define dout_context g_ceph_context
33 #define dout_subsys ceph_subsys_mgr
35 #define dout_prefix *_dout << "mgr " << __func__ << " "
38 MgrStandby::MgrStandby(int argc, const char **argv) :
39 Dispatcher(g_ceph_context),
41 client_messenger(Messenger::create_client_messenger(g_ceph_context, "mgr")),
42 objecter{g_ceph_context, client_messenger.get(), &monc, NULL, 0, 0},
43 client{client_messenger.get(), &monc, &objecter},
44 log_client(g_ceph_context, client_messenger.get(), &monc.monmap, LogClient::NO_FLAGS),
45 clog(log_client.create_channel(CLOG_CHANNEL_CLUSTER)),
46 audit_clog(log_client.create_channel(CLOG_CHANNEL_AUDIT)),
47 lock("MgrStandby::lock"),
48 timer(g_ceph_context, lock),
49 py_module_registry(clog),
53 available_in_map(false)
57 MgrStandby::~MgrStandby() = default;
59 const char** MgrStandby::get_tracked_conf_keys() const
61 static const char* KEYS[] = {
65 "clog_to_syslog_facility",
66 "clog_to_syslog_level",
67 "osd_objectstore_fuse",
69 "clog_to_graylog_host",
70 "clog_to_graylog_port",
78 void MgrStandby::handle_conf_change(
79 const struct md_config_t *conf,
80 const std::set <std::string> &changed)
82 if (changed.count("clog_to_monitors") ||
83 changed.count("clog_to_syslog") ||
84 changed.count("clog_to_syslog_level") ||
85 changed.count("clog_to_syslog_facility") ||
86 changed.count("clog_to_graylog") ||
87 changed.count("clog_to_graylog_host") ||
88 changed.count("clog_to_graylog_port") ||
89 changed.count("host") ||
90 changed.count("fsid")) {
95 int MgrStandby::init()
97 Mutex::Locker l(lock);
99 // Initialize Messenger
100 client_messenger->add_dispatcher_tail(this);
101 client_messenger->add_dispatcher_head(&objecter);
102 client_messenger->add_dispatcher_tail(&client);
103 client_messenger->start();
105 // Initialize MonClient
106 if (monc.build_initial_monmap() < 0) {
107 client_messenger->shutdown();
108 client_messenger->wait();
112 monc.sub_want("mgrmap", 0, 0);
114 monc.set_want_keys(CEPH_ENTITY_TYPE_MON|CEPH_ENTITY_TYPE_OSD
115 |CEPH_ENTITY_TYPE_MDS|CEPH_ENTITY_TYPE_MGR);
116 monc.set_messenger(client_messenger.get());
120 client_messenger->shutdown();
121 client_messenger->wait();
124 r = monc.authenticate();
126 derr << "Authentication failed, did you specify a mgr ID with a valid keyring?" << dendl;
128 client_messenger->shutdown();
129 client_messenger->wait();
133 client_t whoami = monc.get_global_id();
134 client_messenger->set_myname(entity_name_t::CLIENT(whoami.v));
135 monc.set_log_client(&log_client);
136 _update_log_config();
137 objecter.set_client_incarnation(0);
145 dout(4) << "Complete." << dendl;
149 void MgrStandby::send_beacon()
151 assert(lock.is_locked_by_me());
152 dout(1) << state_str() << dendl;
155 PyModuleRegistry::list_modules(&modules);
157 // Whether I think I am available (request MgrMonitor to set me
158 // as available in the map)
159 bool available = active_mgr != nullptr && active_mgr->is_initialized();
161 auto addr = available ? active_mgr->get_server_addr() : entity_addr_t();
162 dout(10) << "sending beacon as gid " << monc.get_global_id() << dendl;
164 map<string,string> metadata;
165 collect_sys_info(&metadata, g_ceph_context);
167 MMgrBeacon *m = new MMgrBeacon(monc.get_fsid(),
168 monc.get_global_id(),
169 g_conf->name.get_id(),
173 std::move(metadata));
176 if (!available_in_map) {
177 // We are informing the mon that we are done initializing: inform
178 // it of our command set. This has to happen after init() because
179 // it needs the python modules to have loaded.
180 m->set_command_descs(active_mgr->get_command_set());
181 dout(4) << "going active, including " << m->get_command_descs().size()
182 << " commands in beacon" << dendl;
185 m->set_services(active_mgr->get_services());
188 monc.send_mon_message(m);
191 void MgrStandby::tick()
193 dout(10) << __func__ << dendl;
196 if (active_mgr && active_mgr->is_initialized()) {
200 timer.add_event_after(g_conf->get_val<int64_t>("mgr_tick_period"),
201 new FunctionContext([this](int r){
207 void MgrStandby::handle_signal(int signum)
209 Mutex::Locker l(lock);
210 assert(signum == SIGINT || signum == SIGTERM);
211 derr << "*** Got signal " << sig_str(signum) << " ***" << dendl;
215 void MgrStandby::shutdown()
217 // Expect already to be locked as we're called from signal handler
218 assert(lock.is_locked_by_me());
220 dout(4) << "Shutting down" << dendl;
222 // stop sending beacon first, i use monc to talk with monitors
224 // client uses monc and objecter
226 // stop monc, so mon won't be able to instruct me to shutdown/activate after
227 // the active_mgr is stopped
230 active_mgr->shutdown();
233 py_module_registry.shutdown();
235 // objecter is used by monc and active_mgr
237 // client_messenger is used by all of them, so stop it in the end
238 client_messenger->shutdown();
241 void MgrStandby::respawn()
243 char *new_argv[orig_argc+1];
244 dout(1) << " e: '" << orig_argv[0] << "'" << dendl;
245 for (int i=0; i<orig_argc; i++) {
246 new_argv[i] = (char *)orig_argv[i];
247 dout(1) << " " << i << ": '" << orig_argv[i] << "'" << dendl;
249 new_argv[orig_argc] = NULL;
251 /* Determine the path to our executable, test if Linux /proc/self/exe exists.
252 * This allows us to exec the same executable even if it has since been
255 char exe_path[PATH_MAX] = "";
256 if (readlink(PROCPREFIX "/proc/self/exe", exe_path, PATH_MAX-1) == -1) {
257 /* Print CWD for the user's interest */
259 char *cwd = getcwd(buf, sizeof(buf));
261 dout(1) << " cwd " << cwd << dendl;
263 /* Fall back to a best-effort: just running in our CWD */
264 strncpy(exe_path, orig_argv[0], PATH_MAX-1);
266 dout(1) << "respawning with exe " << exe_path << dendl;
267 strcpy(exe_path, PROCPREFIX "/proc/self/exe");
270 dout(1) << " exe_path " << exe_path << dendl;
272 unblock_all_signals(NULL);
273 execv(exe_path, new_argv);
275 derr << "respawn execv " << orig_argv[0]
276 << " failed with " << cpp_strerror(errno) << dendl;
280 void MgrStandby::_update_log_config()
282 map<string,string> log_to_monitors;
283 map<string,string> log_to_syslog;
284 map<string,string> log_channel;
285 map<string,string> log_prio;
286 map<string,string> log_to_graylog;
287 map<string,string> log_to_graylog_host;
288 map<string,string> log_to_graylog_port;
292 if (parse_log_client_options(cct, log_to_monitors, log_to_syslog,
293 log_channel, log_prio, log_to_graylog,
294 log_to_graylog_host, log_to_graylog_port,
296 clog->update_config(log_to_monitors, log_to_syslog,
297 log_channel, log_prio, log_to_graylog,
298 log_to_graylog_host, log_to_graylog_port,
300 audit_clog->update_config(log_to_monitors, log_to_syslog,
301 log_channel, log_prio, log_to_graylog,
302 log_to_graylog_host, log_to_graylog_port,
307 void MgrStandby::handle_mgr_map(MMgrMap* mmap)
309 auto &map = mmap->get_map();
310 dout(4) << "received map epoch " << map.get_epoch() << dendl;
311 const bool active_in_map = map.active_gid == monc.get_global_id();
312 dout(4) << "active in map: " << active_in_map
313 << " active is " << map.active_gid << dendl;
315 if (!py_module_registry.is_initialized()) {
316 int r = py_module_registry.init(map);
318 // FIXME: error handling
321 bool need_respawn = py_module_registry.handle_mgr_map(map);
329 dout(1) << "Activating!" << dendl;
330 active_mgr.reset(new Mgr(&monc, map, &py_module_registry,
331 client_messenger.get(), &objecter,
332 &client, clog, audit_clog));
333 active_mgr->background_init(new FunctionContext(
335 // Advertise our active-ness ASAP instead of waiting for
337 Mutex::Locker l(lock);
340 dout(1) << "I am now activating" << dendl;
342 dout(10) << "I was already active" << dendl;
343 bool need_respawn = active_mgr->got_mgr_map(map);
349 if (!available_in_map && map.get_available()) {
350 dout(4) << "Map now says I am available" << dendl;
351 available_in_map = true;
353 } else if (active_mgr != nullptr) {
354 derr << "I was active but no longer am" << dendl;
357 if (map.active_gid != 0 && map.active_name != g_conf->name.get_id()) {
358 // I am the standby and someone else is active, start modules
359 // in standby mode to do redirects if needed
360 if (!py_module_registry.is_standby_running()) {
361 py_module_registry.standby_start(&monc);
369 bool MgrStandby::ms_dispatch(Message *m)
371 Mutex::Locker l(lock);
372 dout(4) << state_str() << " " << *m << dendl;
374 if (m->get_type() == MSG_MGR_MAP) {
375 handle_mgr_map(static_cast<MMgrMap*>(m));
377 } else if (active_mgr) {
378 auto am = active_mgr;
380 bool handled = am->ms_dispatch(m);
389 bool MgrStandby::ms_get_authorizer(int dest_type, AuthAuthorizer **authorizer,
392 if (dest_type == CEPH_ENTITY_TYPE_MON)
396 if (monc.wait_auth_rotating(10) < 0)
400 *authorizer = monc.build_authorizer(dest_type);
401 return *authorizer != NULL;
404 bool MgrStandby::ms_handle_refused(Connection *con)
406 // do nothing for now
410 // A reference for use by the signal handler
411 static MgrStandby *signal_mgr = nullptr;
413 static void handle_mgr_signal(int signum)
416 signal_mgr->handle_signal(signum);
420 int MgrStandby::main(vector<const char *> args)
422 // Enable signal handlers
424 init_async_signal_handler();
425 register_async_signal_handler(SIGHUP, sighup_handler);
426 register_async_signal_handler_oneshot(SIGINT, handle_mgr_signal);
427 register_async_signal_handler_oneshot(SIGTERM, handle_mgr_signal);
429 client_messenger->wait();
431 // Disable signal handlers
432 unregister_async_signal_handler(SIGHUP, sighup_handler);
433 unregister_async_signal_handler(SIGINT, handle_mgr_signal);
434 unregister_async_signal_handler(SIGTERM, handle_mgr_signal);
435 shutdown_async_signal_handler();
436 signal_mgr = nullptr;
442 std::string MgrStandby::state_str()
444 if (active_mgr == nullptr) {
446 } else if (active_mgr->is_initialized()) {
449 return "active (starting)";