1 """Classes used by client.py"""
2 # -*- coding: utf-8 -*-
4 #Licensed under the Apache License, Version 2.0 (the "License"); you may
5 # not use this file except in compliance with the License. You may obtain
6 # a copy of the License at
8 # http://www.apache.org/licenses/LICENSE-2.0
10 # Unless required by applicable law or agreed to in writing, software
11 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
12 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
13 # License for the specific language governing permissions and limitations
21 ID_RSA_PATH = '/home/opnfv/.ssh/id_rsa'
22 SSH_KEYS_SCRIPT = '/home/opnfv/barometer/baro_utils/get_ssh_keys.sh'
23 DEF_PLUGIN_INTERVAL = 10
24 COLLECTD_CONF = '/etc/collectd/collectd.conf'
25 COLLECTD_CONF_DIR = '/etc/collectd/collectd.conf.d'
29 """Node configuration class"""
30 def __init__(self, attrs):
31 self.__id = int(attrs[0])
32 self.__status = attrs[1]
33 self.__name = attrs[2]
34 self.__cluster = int(attrs[3]) if attrs[3] else None
37 self.__roles = [x.strip(' ') for x in attrs[6].split(',')]
38 self.__pending_roles = attrs[7]
39 self.__online = int(attrs[8]) if attrs[3] and attrs[8]else None
40 self.__group_id = int(attrs[9]) if attrs[3] else None
51 """Get node IP address"""
59 class ConfigServer(object):
60 """Class to get env configuration"""
61 def __init__(self, host, user, logger, passwd=None):
64 self.__passwd = passwd
65 self.__priv_key = None
67 self.__logger = logger
69 self.__private_key_file = ID_RSA_PATH
70 if not os.path.isfile(self.__private_key_file):
72 "Private key file '{}'".format(self.__private_key_file)
73 + " not found. Please try to run {} script.".format(SSH_KEYS_SCRIPT))
74 raise IOError("Private key file '{}' not found.".format(self.__private_key_file))
76 # get list of available nodes
77 ssh, sftp = self.__open_sftp_session(self.__host, self.__user, self.__passwd)
79 fuel_node_passed = False
81 while (attempt <= 10) and not fuel_node_passed:
82 stdin, stdout, stderr = ssh.exec_command("fuel node")
83 stderr_lines = stderr.readlines()
85 self.__logger.warning("'fuel node' command failed (try {}):".format(attempt))
86 for line in stderr_lines:
87 self.__logger.debug(line.strip())
89 fuel_node_passed = True
91 self.__logger.info("'fuel node' command passed (try {})".format(attempt))
93 if not fuel_node_passed:
94 self.__logger.error("'fuel node' command failed. This was the last try.")
95 raise OSError("'fuel node' command failed. This was the last try.")
96 node_table = stdout.readlines()\
98 # skip table title and parse table values
99 for entry in node_table[2:]:
100 self.__nodes.append(Node([str(x.strip(' \n')) for x in entry.split('|')]))
102 def get_controllers(self):
103 """Get list of controllers"""
104 return [node for node in self.__nodes if 'controller' in node.get_roles()]
106 def get_computes(self):
107 """Get list of computes"""
108 return [node for node in self.__nodes if 'compute' in node.get_roles()]
111 """Get list of nodes"""
114 def __open_sftp_session(self, host, user, passwd=None):
115 """Connect to given host.
118 host -- host to connect
120 passwd -- password to use
122 Return tuple of SSH and SFTP client instances.
125 ssh = paramiko.SSHClient()
126 ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
128 # try a direct access using password or private key
129 if not passwd and not self.__priv_key:
131 self.__priv_key = paramiko.RSAKey.from_private_key_file(self.__private_key_file)
133 # connect to the server
134 ssh.connect(host, username=user, password=passwd, pkey=self.__priv_key)
135 sftp = ssh.open_sftp()
137 # return SFTP client instance
140 def get_plugin_interval(self, compute, plugin):
141 """Find the plugin interval in collectd configuration.
144 compute -- compute node instance
145 plugin -- plug-in name
147 If found, return interval value, otherwise the default value"""
148 ssh, sftp = self.__open_sftp_session(compute.get_ip(), 'root')
151 default_interval = DEF_PLUGIN_INTERVAL
152 config_files = [COLLECTD_CONF] \
153 + [COLLECTD_CONF_DIR + '/' + conf_file for conf_file in sftp.listdir(COLLECTD_CONF_DIR)]
154 for config_file in config_files:
156 with sftp.open(config_file) as config:
157 for line in config.readlines():
159 if len(words) > 1 and words[0] == '<LoadPlugin':
161 plugin_name = words[1].strip('">')
162 if words and words[0] == '</LoadPlugin>':
164 if words and words[0] == 'Interval':
165 if in_plugin and plugin_name == plugin:
168 default_interval = int(words[1])
170 self.__logger.error("Could not open collectd.conf file.")
171 return default_interval
173 def get_plugin_config_values(self, compute, plugin, parameter):
174 """Get parameter values from collectd config file.
177 compute -- compute node instance
178 plugin -- plug-in name
179 parameter -- plug-in parameter
181 Return list of found values."""
182 ssh, sftp = self.__open_sftp_session(compute.get_ip(), 'root')
183 # find the plugin value
187 config_files = [COLLECTD_CONF] \
188 + [COLLECTD_CONF_DIR + '/' + conf_file for conf_file in sftp.listdir(COLLECTD_CONF_DIR)]
189 for config_file in config_files:
191 with sftp.open(config_file) as config:
192 for line in config.readlines():
194 if len(words) > 1 and words[0] == '<Plugin':
196 plugin_name = words[1].strip('">')
197 if len(words) > 0 and words[0] == '</Plugin>':
199 if len(words) > 0 and words[0] == parameter:
200 if in_plugin and plugin_name == plugin:
201 return [word.strip('"') for word in words[1:]]
203 self.__logger.error("Could not open collectd.conf file.")
204 return default_values
206 def execute_command(self, command, host_ip=None, ssh=None):
207 """Execute command on node and return list of lines of standard output.
211 host_ip -- IP of the node
212 ssh -- existing open SSH session to use
214 One of host_ip or ssh must not be None. If both are not None, existing ssh session is used.
216 if host_ip is None and ssh is None:
217 raise ValueError('One of host_ip or ssh must not be None.')
219 ssh, sftp = self.__open_sftp_session(host_ip, 'root')
220 stdin, stdout, stderr = ssh.exec_command(command)
221 return stdout.readlines()
223 def get_ovs_interfaces(self, compute):
224 """Get list of configured OVS interfaces
227 compute -- compute node instance
229 stdout = self.execute_command("ovs-vsctl list-br", compute.get_ip())
230 return [interface.strip() for interface in stdout]
232 def is_ceilometer_running(self, controller):
233 """Check whether Ceilometer is running on controller.
236 controller -- controller node instance
238 Return boolean value whether Ceilometer is running.
240 lines = self.execute_command('service --status-all | grep ceilometer', controller.get_ip())
244 if '[ + ] ceilometer-agent-notification' in line:
246 if '[ + ] ceilometer-collector' in line:
248 return agent and collector
250 def is_installed(self, compute, package):
251 """Check whether package exists on compute node.
254 compute -- compute node instance
255 package -- Linux package to search for
257 Return boolean value whether package is installed.
259 stdout = self.execute_command('dpkg -l | grep {}'.format(package), compute.get_ip())
260 return len(stdout) > 0
262 def check_ceil_plugin_included(self, compute):
263 """Check if ceilometer plugin is included in collectd.conf file If not,
267 compute -- compute node instance
269 Return boolean value whether ceilometer plugin is included or it's enabling was successful.
271 ssh, sftp = self.__open_sftp_session(compute.get_ip(), 'root')
273 config = sftp.open(COLLECTD_CONF, mode='r')
276 'Cannot open {} on node {}'.format(COLLECTD_CONF, compute.get_id()))
278 in_lines = config.readlines()
279 out_lines = in_lines[:]
280 include_section_indexes = [
281 (start, end) for start in range(len(in_lines)) for end in range(len(in_lines))
283 and '<Include' in in_lines[start]
284 and COLLECTD_CONF_DIR in in_lines[start]
285 and '#' not in in_lines[start]
286 and '</Include>' in in_lines[end]
287 and '#' not in in_lines[end]
288 and len([i for i in in_lines[start + 1: end]
289 if 'Filter' in i and '*.conf' in i and '#' not in i]) > 0]
290 if len(include_section_indexes) == 0:
291 out_lines.append('<Include "{}">\n'.format(COLLECTD_CONF_DIR))
292 out_lines.append(' Filter "*.conf"\n')
293 out_lines.append('</Include>\n')
295 config = sftp.open(COLLECTD_CONF, mode='w')
296 config.writelines(out_lines)
298 self.__logger.info('Creating backup of collectd.conf...')
299 config = sftp.open(COLLECTD_CONF + '.backup', mode='w')
300 config.writelines(in_lines)
304 def enable_plugins(self, compute, plugins, error_plugins, create_backup=True):
305 """Enable plugins on compute node
308 compute -- compute node instance
309 plugins -- list of plugins to be enabled
310 error_plugins -- list of tuples with found errors, new entries may be added there
311 (plugin, error_description, is_critical):
312 plugin -- plug-in name
313 error_decription -- description of the error
314 is_critical -- boolean value indicating whether error is critical
315 create_backup -- boolean value indicating whether backup shall be created
317 Return boolean value indicating whether function was successful.
319 plugins = sorted(plugins)
320 ssh, sftp = self.__open_sftp_session(compute.get_ip(), 'root')
321 plugins_to_enable = plugins[:]
322 for plugin in plugins:
323 plugin_file = '/usr/lib/collectd/{}.so'.format(plugin)
325 sftp.stat(plugin_file)
328 'Plugin file {0} not found on node {1}, plugin {2} will not be enabled'.format(
329 plugin_file, compute.get_id(), plugin))
330 error_plugins.append((plugin, 'plugin file {} not found'.format(plugin_file), True))
331 plugins_to_enable.remove(plugin)
332 self.__logger.debug('Following plugins will be enabled on node {}: {}'.format(
333 compute.get_id(), ', '.join(plugins_to_enable)))
335 config = sftp.open(COLLECTD_CONF, mode='r')
337 self.__logger.warning(
338 'Cannot open {} on node {}'.format(COLLECTD_CONF, compute.get_id()))
340 in_lines = config.readlines()
343 enabled_sections = []
345 comment_section = False
346 uncomment_section = False
347 for line in in_lines:
348 if 'LoadPlugin' in line:
349 for plugin in plugins_to_enable:
351 commented = '#' in line
352 #list of uncommented lines which contain LoadPlugin for this plugin
354 ll for ll in in_lines if 'LoadPlugin' in ll
355 and plugin in ll and '#' not in ll]
356 if len(loadlines) == 0:
357 if plugin not in enabled_plugins:
358 line = line.lstrip(string.whitespace + '#')
359 enabled_plugins.append(plugin)
360 error_plugins.append((
361 plugin, 'plugin not enabled in '
362 + '{}, trying to enable it'.format(COLLECTD_CONF), False))
364 if plugin not in enabled_plugins:
365 enabled_plugins.append(plugin)
368 error_plugins.append((
369 plugin, 'plugin enabled more than once '
370 + '(additional occurrence of LoadPlugin found in '
371 + '{}), trying to comment it out.'.format(
372 COLLECTD_CONF), False))
373 elif line.lstrip(string.whitespace + '#').find('<Plugin') == 0:
375 for plugin in plugins_to_enable:
377 commented = '#' in line
378 #list of uncommented lines which contain Plugin for this plugin
380 pl for pl in in_lines if '<Plugin' in pl
381 and plugin in pl and '#' not in pl]
382 if len(pluginlines) == 0:
383 if plugin not in enabled_sections:
384 line = line[line.rfind('#') + 1:]
385 uncomment_section = True
386 enabled_sections.append(plugin)
387 error_plugins.append((
388 plugin, 'plugin section found in '
389 + '{}, but commented out, trying to uncomment it.'.format(
390 COLLECTD_CONF), False))
392 if plugin not in enabled_sections:
393 enabled_sections.append(plugin)
396 comment_section = True
397 error_plugins.append((
399 'additional occurrence of plugin section found in '
400 + '{}, trying to comment it out.'.format(COLLECTD_CONF),
403 if comment_section and '#' not in line:
405 if uncomment_section and '#' in line:
406 line = line[line.rfind('#') + 1:]
407 if '</Plugin>' in line:
410 comment_section = False
411 uncomment_section = False
412 elif '</Plugin>' in line:
414 'Unexpected closure os plugin section on line'
415 + ' {} in collectd.conf, matching section start not found.'.format(
418 out_lines.append(line)
421 'Unexpected end of file collectd.conf, '
422 + 'closure of last plugin section not found.')
425 'LoadPlugin {}\n'.format(plugin) for plugin in plugins_to_enable
426 if plugin not in enabled_plugins] + out_lines
427 for plugin in plugins_to_enable:
428 if plugin not in enabled_plugins:
429 error_plugins.append((
431 'plugin not enabled in {}, trying to enable it.'.format(COLLECTD_CONF),
433 unenabled_sections = [
434 plugin for plugin in plugins_to_enable if plugin not in enabled_sections]
435 if unenabled_sections:
436 self.__logger.error('Plugin sections for following plugins not found: {}'.format(
437 ', '.join(unenabled_sections)))
442 self.__logger.info('Creating backup of collectd.conf...')
443 config = sftp.open(COLLECTD_CONF + '.backup', mode='w')
444 config.writelines(in_lines)
446 self.__logger.info('Updating collectd.conf...')
447 config = sftp.open(COLLECTD_CONF, mode='w')
448 config.writelines(out_lines)
450 diff_command = "diff {} {}.backup".format(COLLECTD_CONF, COLLECTD_CONF)
451 stdin, stdout, stderr = ssh.exec_command(diff_command)
452 self.__logger.debug(diff_command)
453 for line in stdout.readlines():
454 self.__logger.debug(line.strip())
457 def restore_config(self, compute):
458 """Restore collectd config file from backup on compute node.
461 compute -- compute node instance
463 ssh, sftp = self.__open_sftp_session(compute.get_ip(), 'root')
465 self.__logger.info('Restoring config file from backup...')
466 ssh.exec_command("cp {0} {0}.used".format(COLLECTD_CONF))
467 ssh.exec_command("cp {0}.backup {0}".format(COLLECTD_CONF))
469 def restart_collectd(self, compute):
470 """Restart collectd on compute node.
473 compute -- compute node instance
475 Retrun tuple with boolean indicating success and list of warnings received
476 during collectd start.
479 def get_collectd_processes(ssh_session):
480 """Get number of running collectd processes.
483 ssh_session -- instance of SSH session in which to check for processes
485 stdin, stdout, stderr = ssh_session.exec_command("pgrep collectd")
486 return len(stdout.readlines())
488 ssh, sftp = self.__open_sftp_session(compute.get_ip(), 'root')
490 self.__logger.info('Stopping collectd service...')
491 stdout = self.execute_command("service collectd stop", ssh=ssh)
493 if get_collectd_processes(ssh):
494 self.__logger.error('Collectd is still running...')
496 self.__logger.info('Starting collectd service...')
497 stdout = self.execute_command("service collectd start", ssh=ssh)
499 warning = [output.strip() for output in stdout if 'WARN: ' in output]
500 if get_collectd_processes(ssh) == 0:
501 self.__logger.error('Collectd is still not running...')
502 return False, warning