1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
2 // vim: ts=8 sw=2 smarttab
4 * Ceph - scalable distributed file system
6 * Copyright (C) 2017 John Spray <john.spray@redhat.com>
8 * This is free software; you can redistribute it and/or
9 * modify it under the terms of the GNU Lesser General Public
10 * License version 2.1, as published by the Free Software
11 * Foundation. See file COPYING.
15 #include "include/stringify.h"
16 #include "common/errno.h"
17 #include "common/backport14.h"
19 #include "BaseMgrModule.h"
21 #include "BaseMgrStandbyModule.h"
24 #include "ActivePyModules.h"
26 #include "PyModuleRegistry.h"
28 // definition for non-const static member
29 std::string PyModuleRegistry::config_prefix;
33 #define dout_context g_ceph_context
34 #define dout_subsys ceph_subsys_mgr
37 #define dout_prefix *_dout << "mgr[py] "
40 PyObject* log_write(PyObject*, PyObject* args) {
42 if (PyArg_ParseTuple(args, "s", &m)) {
44 if (len && m[len-1] == '\n') {
47 dout(4) << m << dendl;
52 PyObject* log_flush(PyObject*, PyObject*){
56 static PyMethodDef log_methods[] = {
57 {"write", log_write, METH_VARARGS, "write stdout and stderr"},
58 {"flush", log_flush, METH_VARARGS, "flush"},
59 {nullptr, nullptr, 0, nullptr}
64 #define dout_prefix *_dout << "mgr " << __func__ << " "
68 std::string PyModule::get_site_packages()
70 std::stringstream site_packages;
72 // CPython doesn't auto-add site-packages dirs to sys.path for us,
73 // but it does provide a module that we can ask for them.
74 auto site_module = PyImport_ImportModule("site");
77 auto site_packages_fn = PyObject_GetAttrString(site_module, "getsitepackages");
78 if (site_packages_fn != nullptr) {
79 auto site_packages_list = PyObject_CallObject(site_packages_fn, nullptr);
80 assert(site_packages_list);
82 auto n = PyList_Size(site_packages_list);
83 for (Py_ssize_t i = 0; i < n; ++i) {
87 site_packages << PyString_AsString(PyList_GetItem(site_packages_list, i));
90 Py_DECREF(site_packages_list);
91 Py_DECREF(site_packages_fn);
93 // Fall back to generating our own site-packages paths by imitating
94 // what the standard site.py does. This is annoying but it lets us
95 // run inside virtualenvs :-/
97 auto site_packages_fn = PyObject_GetAttrString(site_module, "addsitepackages");
98 assert(site_packages_fn);
100 auto known_paths = PySet_New(nullptr);
101 auto pArgs = PyTuple_Pack(1, known_paths);
102 PyObject_CallObject(site_packages_fn, pArgs);
104 Py_DECREF(known_paths);
105 Py_DECREF(site_packages_fn);
107 auto sys_module = PyImport_ImportModule("sys");
109 auto sys_path = PyObject_GetAttrString(sys_module, "path");
112 dout(1) << "sys.path:" << dendl;
113 auto n = PyList_Size(sys_path);
115 for (Py_ssize_t i = 0; i < n; ++i) {
116 dout(1) << " " << PyString_AsString(PyList_GetItem(sys_path, i)) << dendl;
120 site_packages << ":";
122 site_packages << PyString_AsString(PyList_GetItem(sys_path, i));
126 Py_DECREF(sys_module);
129 Py_DECREF(site_module);
131 return site_packages.str();
134 int PyModuleRegistry::init(const MgrMap &map)
136 Mutex::Locker locker(lock);
138 // Don't try and init me if you don't really have a map
139 assert(map.epoch > 0);
143 // namespace in config-key prefixed by "mgr/"
144 config_prefix = std::string(g_conf->name.get_type_str()) + "/";
146 // Set up global python interpreter
147 Py_SetProgramName(const_cast<char*>(PYTHON_EXECUTABLE));
150 // Let CPython know that we will be calling it back from other
151 // threads in future.
152 if (! PyEval_ThreadsInitialized()) {
153 PyEval_InitThreads();
156 // Drop the GIL and remember the main thread state (current
157 // thread state becomes NULL)
158 pMainThreadState = PyEval_SaveThread();
159 assert(pMainThreadState != nullptr);
161 std::list<std::string> failed_modules;
164 for (const auto& module_name : mgr_map.modules) {
165 dout(1) << "Loading python module '" << module_name << "'" << dendl;
166 auto mod = ceph::make_unique<PyModule>(module_name);
167 int r = mod->load(pMainThreadState);
169 // Don't use handle_pyerror() here; we don't have the GIL
170 // or the right thread state (this is deliberate).
171 derr << "Error loading module '" << module_name << "': "
172 << cpp_strerror(r) << dendl;
173 failed_modules.push_back(module_name);
174 // Don't drop out here, load the other modules
177 modules[module_name] = std::move(mod);
181 if (!failed_modules.empty()) {
182 clog->error() << "Failed to load ceph-mgr modules: " << joinify(
183 failed_modules.begin(), failed_modules.end(), std::string(", "));
190 int PyModule::load(PyThreadState *pMainThreadState)
192 assert(pMainThreadState != nullptr);
194 // Configure sub-interpreter and construct C++-generated python classes
196 SafeThreadState sts(pMainThreadState);
199 auto thread_state = Py_NewInterpreter();
200 if (thread_state == nullptr) {
201 derr << "Failed to create python sub-interpreter for '" << module_name << '"' << dendl;
204 pMyThreadState.set(thread_state);
205 // Some python modules do not cope with an unpopulated argv, so lets
206 // fake one. This step also picks up site-packages into sys.path.
207 const char *argv[] = {"ceph-mgr"};
208 PySys_SetArgv(1, (char**)argv);
210 if (g_conf->daemonize) {
211 auto py_logger = Py_InitModule("ceph_logger", log_methods);
212 #if PY_MAJOR_VERSION >= 3
213 PySys_SetObject("stderr", py_logger);
214 PySys_SetObject("stdout", py_logger);
216 PySys_SetObject(const_cast<char*>("stderr"), py_logger);
217 PySys_SetObject(const_cast<char*>("stdout"), py_logger);
221 // Configure sys.path to include mgr_module_path
222 std::string sys_path = std::string(Py_GetPath()) + ":" + get_site_packages()
223 + ":" + g_conf->get_val<std::string>("mgr_module_path");
224 dout(10) << "Computed sys.path '" << sys_path << "'" << dendl;
226 PySys_SetPath(const_cast<char*>(sys_path.c_str()));
229 PyMethodDef ModuleMethods[] = {
234 PyObject *ceph_module = Py_InitModule("ceph_module", ModuleMethods);
235 assert(ceph_module != nullptr);
237 auto load_class = [ceph_module](const char *name, PyTypeObject *type)
239 type->tp_new = PyType_GenericNew;
240 if (PyType_Ready(type) < 0) {
245 PyModule_AddObject(ceph_module, name, (PyObject *)type);
248 load_class("BaseMgrModule", &BaseMgrModuleType);
249 load_class("BaseMgrStandbyModule", &BaseMgrStandbyModuleType);
250 load_class("BasePyOSDMap", &BasePyOSDMapType);
251 load_class("BasePyOSDMapIncremental", &BasePyOSDMapIncrementalType);
252 load_class("BasePyCRUSH", &BasePyCRUSHType);
255 // Environment is all good, import the external module
257 Gil gil(pMyThreadState);
260 PyObject *pName = PyString_FromString(module_name.c_str());
261 auto pModule = PyImport_Import(pName);
263 if (pModule == nullptr) {
264 derr << "Module not found: '" << module_name << "'" << dendl;
265 derr << handle_pyerror() << dendl;
270 // TODO: let them call it what they want instead of just 'Module'
271 pClass = PyObject_GetAttrString(pModule, (const char*)"Module");
272 if (pClass == nullptr) {
273 derr << "Class not found in module '" << module_name << "'" << dendl;
274 derr << handle_pyerror() << dendl;
278 pStandbyClass = PyObject_GetAttrString(pModule,
279 (const char*)"StandbyModule");
281 dout(4) << "Standby mode available in module '" << module_name
284 dout(4) << "Standby mode not provided by module '" << module_name
295 PyModule::~PyModule()
297 if (pMyThreadState.ts != nullptr) {
298 Gil gil(pMyThreadState, true);
300 Py_XDECREF(pStandbyClass);
304 void PyModuleRegistry::standby_start(MonClient *monc)
306 Mutex::Locker l(lock);
307 assert(active_modules == nullptr);
308 assert(standby_modules == nullptr);
309 assert(is_initialized());
311 dout(4) << "Starting modules in standby mode" << dendl;
313 standby_modules.reset(new StandbyPyModules(monc, mgr_map));
315 std::set<std::string> failed_modules;
316 for (const auto &i : modules) {
317 if (i.second->pStandbyClass) {
318 dout(4) << "starting module " << i.second->get_name() << dendl;
319 int r = standby_modules->start_one(i.first,
320 i.second->pStandbyClass,
321 i.second->pMyThreadState);
323 derr << "failed to start module '" << i.second->get_name()
325 failed_modules.insert(i.second->get_name());
326 // Continue trying to load any other modules
329 dout(4) << "skipping module '" << i.second->get_name() << "' because "
330 "it does not implement a standby mode" << dendl;
334 if (!failed_modules.empty()) {
335 clog->error() << "Failed to execute ceph-mgr module(s) in standby mode: "
336 << joinify(failed_modules.begin(), failed_modules.end(),
341 void PyModuleRegistry::active_start(
342 PyModuleConfig &config_,
343 DaemonStateIndex &ds, ClusterState &cs, MonClient &mc,
344 LogChannelRef clog_, Objecter &objecter_, Client &client_,
347 Mutex::Locker locker(lock);
349 dout(4) << "Starting modules in active mode" << dendl;
351 assert(active_modules == nullptr);
352 assert(is_initialized());
354 if (standby_modules != nullptr) {
355 standby_modules->shutdown();
356 standby_modules.reset();
359 active_modules.reset(new ActivePyModules(
360 config_, ds, cs, mc, clog_, objecter_, client_, f));
362 for (const auto &i : modules) {
363 dout(4) << "Starting " << i.first << dendl;
364 int r = active_modules->start_one(i.first,
366 i.second->pMyThreadState);
368 derr << "Failed to run module in active mode ('" << i.first << "')"
374 void PyModuleRegistry::active_shutdown()
376 Mutex::Locker locker(lock);
378 if (active_modules != nullptr) {
379 active_modules->shutdown();
380 active_modules.reset();
384 void PyModuleRegistry::shutdown()
386 Mutex::Locker locker(lock);
388 if (standby_modules != nullptr) {
389 standby_modules->shutdown();
390 standby_modules.reset();
393 // Ideally, now, we'd be able to do this for all modules:
395 // Py_EndInterpreter(pMyThreadState);
396 // PyThreadState_Swap(pMainThreadState);
398 // Unfortunately, if the module has any other *python* threads active
399 // at this point, Py_EndInterpreter() will abort with:
401 // Fatal Python error: Py_EndInterpreter: not the last thread
403 // This can happen when using CherryPy in a module, becuase CherryPy
404 // runs an extra thread as a timeout monitor, which spends most of its
405 // life inside a time.sleep(60). Unless you are very, very lucky with
406 // the timing calling this destructor, that thread will still be stuck
407 // in a sleep, and Py_EndInterpreter() will abort.
409 // This could of course also happen with a poorly written module which
410 // made no attempt to clean up any additional threads it created.
412 // The safest thing to do is just not call Py_EndInterpreter(), and
413 // let Py_Finalize() kill everything after all modules are shut down.
417 PyEval_RestoreThread(pMainThreadState);
421 static void _list_modules(
422 const std::string path,
423 std::set<std::string> *modules)
425 DIR *dir = opendir(path.c_str());
429 struct dirent *entry = NULL;
430 while ((entry = readdir(dir)) != NULL) {
431 string n(entry->d_name);
432 string fn = path + "/" + n;
434 int r = ::stat(fn.c_str(), &st);
435 if (r == 0 && S_ISDIR(st.st_mode)) {
436 string initfn = fn + "/module.py";
437 r = ::stat(initfn.c_str(), &st);
446 void PyModuleRegistry::list_modules(std::set<std::string> *modules)
448 _list_modules(g_conf->get_val<std::string>("mgr_module_path"), modules);