--- /dev/null
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2017 John Spray <john.spray@redhat.com>
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation. See file COPYING.
+ */
+
+
+#include "include/stringify.h"
+#include "common/errno.h"
+#include "common/backport14.h"
+
+#include "BaseMgrModule.h"
+#include "PyOSDMap.h"
+#include "BaseMgrStandbyModule.h"
+#include "Gil.h"
+
+#include "ActivePyModules.h"
+
+#include "PyModuleRegistry.h"
+
+// definition for non-const static member
+std::string PyModuleRegistry::config_prefix;
+
+
+
+#define dout_context g_ceph_context
+#define dout_subsys ceph_subsys_mgr
+
+#undef dout_prefix
+#define dout_prefix *_dout << "mgr[py] "
+
+namespace {
+ PyObject* log_write(PyObject*, PyObject* args) {
+ char* m = nullptr;
+ if (PyArg_ParseTuple(args, "s", &m)) {
+ auto len = strlen(m);
+ if (len && m[len-1] == '\n') {
+ m[len-1] = '\0';
+ }
+ dout(4) << m << dendl;
+ }
+ Py_RETURN_NONE;
+ }
+
+ PyObject* log_flush(PyObject*, PyObject*){
+ Py_RETURN_NONE;
+ }
+
+ static PyMethodDef log_methods[] = {
+ {"write", log_write, METH_VARARGS, "write stdout and stderr"},
+ {"flush", log_flush, METH_VARARGS, "flush"},
+ {nullptr, nullptr, 0, nullptr}
+ };
+}
+
+#undef dout_prefix
+#define dout_prefix *_dout << "mgr " << __func__ << " "
+
+
+
+std::string PyModule::get_site_packages()
+{
+ std::stringstream site_packages;
+
+ // CPython doesn't auto-add site-packages dirs to sys.path for us,
+ // but it does provide a module that we can ask for them.
+ auto site_module = PyImport_ImportModule("site");
+ assert(site_module);
+
+ auto site_packages_fn = PyObject_GetAttrString(site_module, "getsitepackages");
+ if (site_packages_fn != nullptr) {
+ auto site_packages_list = PyObject_CallObject(site_packages_fn, nullptr);
+ assert(site_packages_list);
+
+ auto n = PyList_Size(site_packages_list);
+ for (Py_ssize_t i = 0; i < n; ++i) {
+ if (i != 0) {
+ site_packages << ":";
+ }
+ site_packages << PyString_AsString(PyList_GetItem(site_packages_list, i));
+ }
+
+ Py_DECREF(site_packages_list);
+ Py_DECREF(site_packages_fn);
+ } else {
+ // Fall back to generating our own site-packages paths by imitating
+ // what the standard site.py does. This is annoying but it lets us
+ // run inside virtualenvs :-/
+
+ auto site_packages_fn = PyObject_GetAttrString(site_module, "addsitepackages");
+ assert(site_packages_fn);
+
+ auto known_paths = PySet_New(nullptr);
+ auto pArgs = PyTuple_Pack(1, known_paths);
+ PyObject_CallObject(site_packages_fn, pArgs);
+ Py_DECREF(pArgs);
+ Py_DECREF(known_paths);
+ Py_DECREF(site_packages_fn);
+
+ auto sys_module = PyImport_ImportModule("sys");
+ assert(sys_module);
+ auto sys_path = PyObject_GetAttrString(sys_module, "path");
+ assert(sys_path);
+
+ dout(1) << "sys.path:" << dendl;
+ auto n = PyList_Size(sys_path);
+ bool first = true;
+ for (Py_ssize_t i = 0; i < n; ++i) {
+ dout(1) << " " << PyString_AsString(PyList_GetItem(sys_path, i)) << dendl;
+ if (first) {
+ first = false;
+ } else {
+ site_packages << ":";
+ }
+ site_packages << PyString_AsString(PyList_GetItem(sys_path, i));
+ }
+
+ Py_DECREF(sys_path);
+ Py_DECREF(sys_module);
+ }
+
+ Py_DECREF(site_module);
+
+ return site_packages.str();
+}
+
+int PyModuleRegistry::init(const MgrMap &map)
+{
+ Mutex::Locker locker(lock);
+
+ // Don't try and init me if you don't really have a map
+ assert(map.epoch > 0);
+
+ mgr_map = map;
+
+ // namespace in config-key prefixed by "mgr/"
+ config_prefix = std::string(g_conf->name.get_type_str()) + "/";
+
+ // Set up global python interpreter
+ Py_SetProgramName(const_cast<char*>(PYTHON_EXECUTABLE));
+ Py_InitializeEx(0);
+
+ // Let CPython know that we will be calling it back from other
+ // threads in future.
+ if (! PyEval_ThreadsInitialized()) {
+ PyEval_InitThreads();
+ }
+
+ // Drop the GIL and remember the main thread state (current
+ // thread state becomes NULL)
+ pMainThreadState = PyEval_SaveThread();
+ assert(pMainThreadState != nullptr);
+
+ std::list<std::string> failed_modules;
+
+ // Load python code
+ for (const auto& module_name : mgr_map.modules) {
+ dout(1) << "Loading python module '" << module_name << "'" << dendl;
+ auto mod = ceph::make_unique<PyModule>(module_name);
+ int r = mod->load(pMainThreadState);
+ if (r != 0) {
+ // Don't use handle_pyerror() here; we don't have the GIL
+ // or the right thread state (this is deliberate).
+ derr << "Error loading module '" << module_name << "': "
+ << cpp_strerror(r) << dendl;
+ failed_modules.push_back(module_name);
+ // Don't drop out here, load the other modules
+ } else {
+ // Success!
+ modules[module_name] = std::move(mod);
+ }
+ }
+
+ if (!failed_modules.empty()) {
+ clog->error() << "Failed to load ceph-mgr modules: " << joinify(
+ failed_modules.begin(), failed_modules.end(), std::string(", "));
+ }
+
+ return 0;
+}
+
+
+int PyModule::load(PyThreadState *pMainThreadState)
+{
+ assert(pMainThreadState != nullptr);
+
+ // Configure sub-interpreter and construct C++-generated python classes
+ {
+ SafeThreadState sts(pMainThreadState);
+ Gil gil(sts);
+
+ auto thread_state = Py_NewInterpreter();
+ if (thread_state == nullptr) {
+ derr << "Failed to create python sub-interpreter for '" << module_name << '"' << dendl;
+ return -EINVAL;
+ } else {
+ pMyThreadState.set(thread_state);
+ // Some python modules do not cope with an unpopulated argv, so lets
+ // fake one. This step also picks up site-packages into sys.path.
+ const char *argv[] = {"ceph-mgr"};
+ PySys_SetArgv(1, (char**)argv);
+
+ if (g_conf->daemonize) {
+ auto py_logger = Py_InitModule("ceph_logger", log_methods);
+#if PY_MAJOR_VERSION >= 3
+ PySys_SetObject("stderr", py_logger);
+ PySys_SetObject("stdout", py_logger);
+#else
+ PySys_SetObject(const_cast<char*>("stderr"), py_logger);
+ PySys_SetObject(const_cast<char*>("stdout"), py_logger);
+#endif
+ }
+
+ // Configure sys.path to include mgr_module_path
+ std::string sys_path = std::string(Py_GetPath()) + ":" + get_site_packages()
+ + ":" + g_conf->get_val<std::string>("mgr_module_path");
+ dout(10) << "Computed sys.path '" << sys_path << "'" << dendl;
+
+ PySys_SetPath(const_cast<char*>(sys_path.c_str()));
+ }
+
+ PyMethodDef ModuleMethods[] = {
+ {nullptr}
+ };
+
+ // Initialize module
+ PyObject *ceph_module = Py_InitModule("ceph_module", ModuleMethods);
+ assert(ceph_module != nullptr);
+
+ auto load_class = [ceph_module](const char *name, PyTypeObject *type)
+ {
+ type->tp_new = PyType_GenericNew;
+ if (PyType_Ready(type) < 0) {
+ assert(0);
+ }
+ Py_INCREF(type);
+
+ PyModule_AddObject(ceph_module, name, (PyObject *)type);
+ };
+
+ load_class("BaseMgrModule", &BaseMgrModuleType);
+ load_class("BaseMgrStandbyModule", &BaseMgrStandbyModuleType);
+ load_class("BasePyOSDMap", &BasePyOSDMapType);
+ load_class("BasePyOSDMapIncremental", &BasePyOSDMapIncrementalType);
+ load_class("BasePyCRUSH", &BasePyCRUSHType);
+ }
+
+ // Environment is all good, import the external module
+ {
+ Gil gil(pMyThreadState);
+
+ // Load the module
+ PyObject *pName = PyString_FromString(module_name.c_str());
+ auto pModule = PyImport_Import(pName);
+ Py_DECREF(pName);
+ if (pModule == nullptr) {
+ derr << "Module not found: '" << module_name << "'" << dendl;
+ derr << handle_pyerror() << dendl;
+ return -ENOENT;
+ }
+
+ // Find the class
+ // TODO: let them call it what they want instead of just 'Module'
+ pClass = PyObject_GetAttrString(pModule, (const char*)"Module");
+ if (pClass == nullptr) {
+ derr << "Class not found in module '" << module_name << "'" << dendl;
+ derr << handle_pyerror() << dendl;
+ return -EINVAL;
+ }
+
+ pStandbyClass = PyObject_GetAttrString(pModule,
+ (const char*)"StandbyModule");
+ if (pStandbyClass) {
+ dout(4) << "Standby mode available in module '" << module_name
+ << "'" << dendl;
+ } else {
+ dout(4) << "Standby mode not provided by module '" << module_name
+ << "'" << dendl;
+ PyErr_Clear();
+ }
+
+ Py_DECREF(pModule);
+ }
+
+ return 0;
+}
+
+PyModule::~PyModule()
+{
+ if (pMyThreadState.ts != nullptr) {
+ Gil gil(pMyThreadState, true);
+ Py_XDECREF(pClass);
+ Py_XDECREF(pStandbyClass);
+ }
+}
+
+void PyModuleRegistry::standby_start(MonClient *monc)
+{
+ Mutex::Locker l(lock);
+ assert(active_modules == nullptr);
+ assert(standby_modules == nullptr);
+ assert(is_initialized());
+
+ dout(4) << "Starting modules in standby mode" << dendl;
+
+ standby_modules.reset(new StandbyPyModules(monc, mgr_map));
+
+ std::set<std::string> failed_modules;
+ for (const auto &i : modules) {
+ if (i.second->pStandbyClass) {
+ dout(4) << "starting module " << i.second->get_name() << dendl;
+ int r = standby_modules->start_one(i.first,
+ i.second->pStandbyClass,
+ i.second->pMyThreadState);
+ if (r != 0) {
+ derr << "failed to start module '" << i.second->get_name()
+ << "'" << dendl;;
+ failed_modules.insert(i.second->get_name());
+ // Continue trying to load any other modules
+ }
+ } else {
+ dout(4) << "skipping module '" << i.second->get_name() << "' because "
+ "it does not implement a standby mode" << dendl;
+ }
+ }
+
+ if (!failed_modules.empty()) {
+ clog->error() << "Failed to execute ceph-mgr module(s) in standby mode: "
+ << joinify(failed_modules.begin(), failed_modules.end(),
+ std::string(", "));
+ }
+}
+
+void PyModuleRegistry::active_start(
+ PyModuleConfig &config_,
+ DaemonStateIndex &ds, ClusterState &cs, MonClient &mc,
+ LogChannelRef clog_, Objecter &objecter_, Client &client_,
+ Finisher &f)
+{
+ Mutex::Locker locker(lock);
+
+ dout(4) << "Starting modules in active mode" << dendl;
+
+ assert(active_modules == nullptr);
+ assert(is_initialized());
+
+ if (standby_modules != nullptr) {
+ standby_modules->shutdown();
+ standby_modules.reset();
+ }
+
+ active_modules.reset(new ActivePyModules(
+ config_, ds, cs, mc, clog_, objecter_, client_, f));
+
+ for (const auto &i : modules) {
+ dout(4) << "Starting " << i.first << dendl;
+ int r = active_modules->start_one(i.first,
+ i.second->pClass,
+ i.second->pMyThreadState);
+ if (r != 0) {
+ derr << "Failed to run module in active mode ('" << i.first << "')"
+ << dendl;
+ }
+ }
+}
+
+void PyModuleRegistry::active_shutdown()
+{
+ Mutex::Locker locker(lock);
+
+ if (active_modules != nullptr) {
+ active_modules->shutdown();
+ active_modules.reset();
+ }
+}
+
+void PyModuleRegistry::shutdown()
+{
+ Mutex::Locker locker(lock);
+
+ if (standby_modules != nullptr) {
+ standby_modules->shutdown();
+ standby_modules.reset();
+ }
+
+ // Ideally, now, we'd be able to do this for all modules:
+ //
+ // Py_EndInterpreter(pMyThreadState);
+ // PyThreadState_Swap(pMainThreadState);
+ //
+ // Unfortunately, if the module has any other *python* threads active
+ // at this point, Py_EndInterpreter() will abort with:
+ //
+ // Fatal Python error: Py_EndInterpreter: not the last thread
+ //
+ // This can happen when using CherryPy in a module, becuase CherryPy
+ // runs an extra thread as a timeout monitor, which spends most of its
+ // life inside a time.sleep(60). Unless you are very, very lucky with
+ // the timing calling this destructor, that thread will still be stuck
+ // in a sleep, and Py_EndInterpreter() will abort.
+ //
+ // This could of course also happen with a poorly written module which
+ // made no attempt to clean up any additional threads it created.
+ //
+ // The safest thing to do is just not call Py_EndInterpreter(), and
+ // let Py_Finalize() kill everything after all modules are shut down.
+
+ modules.clear();
+
+ PyEval_RestoreThread(pMainThreadState);
+ Py_Finalize();
+}
+
+static void _list_modules(
+ const std::string path,
+ std::set<std::string> *modules)
+{
+ DIR *dir = opendir(path.c_str());
+ if (!dir) {
+ return;
+ }
+ struct dirent *entry = NULL;
+ while ((entry = readdir(dir)) != NULL) {
+ string n(entry->d_name);
+ string fn = path + "/" + n;
+ struct stat st;
+ int r = ::stat(fn.c_str(), &st);
+ if (r == 0 && S_ISDIR(st.st_mode)) {
+ string initfn = fn + "/module.py";
+ r = ::stat(initfn.c_str(), &st);
+ if (r == 0) {
+ modules->insert(n);
+ }
+ }
+ }
+ closedir(dir);
+}
+
+void PyModuleRegistry::list_modules(std::set<std::string> *modules)
+{
+ _list_modules(g_conf->get_val<std::string>("mgr_module_path"), modules);
+}
+
Code Review / stor4nfv.git / blobdiff