1 # -*- coding: utf-8 -*-
3 # Licensed under the Apache License, Version 2.0 (the "License"); you may
4 # not use this file except in compliance with the License. You may obtain
5 # a copy of the License at
7 # http://www.apache.org/licenses/LICENSE-2.0
9 # Unless required by applicable law or agreed to in writing, software
10 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
11 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
12 # License for the specific language governing permissions and limitations
15 """Classes used by collectd.py"""
23 ID_RSA_PATH = '/home/opnfv/.ssh/id_rsa'
24 SSH_KEYS_SCRIPT = '/home/opnfv/barometer/baro_utils/get_ssh_keys.sh'
25 DEF_PLUGIN_INTERVAL = 10
26 COLLECTD_CONF = '/etc/collectd.conf'
27 COLLECTD_CONF_DIR = '/etc/collectd/collectd.conf.d'
28 NOTIFICATION_FILE = '/var/log/python-notifications.dump'
29 COLLECTD_NOTIFICATION = '/etc/collectd_notification_dump.py'
33 """Node configuration class"""
34 def __init__(self, attrs):
35 self.__null = attrs[0]
37 self.__name = attrs[2]
38 self.__status = attrs[3] if attrs[3] else None
39 self.__taskState = attrs[4]
40 self.__pwrState = attrs[5]
41 self.__ip = re.sub('^[a-z]+=', '', attrs[6])
52 """Get node IP address"""
60 class ConfigServer(object):
61 """Class to get env configuration"""
62 def __init__(self, host, user, logger, priv_key=None):
66 self.__priv_key = priv_key
68 self.__logger = logger
70 self.__private_key_file = ID_RSA_PATH
71 if not os.path.isfile(self.__private_key_file):
73 "Private key file '{}'".format(self.__private_key_file)
75 raise IOError("Private key file '{}' not found.".format(
76 self.__private_key_file))
78 # get list of available nodes
79 ssh, sftp = self.__open_sftp_session(
80 self.__host, self.__user, self.__passwd)
82 fuel_node_passed = False
84 while (attempt <= 10) and not fuel_node_passed:
85 stdin, stdout, stderr = ssh.exec_command(
86 "source stackrc; nova list")
87 stderr_lines = stderr.readlines()
89 self.__logger.warning(
90 "'fuel node' command failed (try {}):".format(attempt))
91 for line in stderr_lines:
92 self.__logger.debug(line.strip())
94 fuel_node_passed = True
97 "'fuel node' command passed (try {})".format(attempt))
99 if not fuel_node_passed:
101 "'fuel node' command failed. This was the last try.")
103 "'fuel node' command failed. This was the last try.")
104 node_table = stdout.readlines()\
106 # skip table title and parse table values
108 for entry in node_table[3:]:
109 if entry[0] == '+' or entry[0] == '\n':
114 Node([str(x.strip(' \n')) for x in entry.split('|')]))
116 def get_controllers(self):
117 # Get list of controllers
118 print self.__nodes[0]._Node__ip
120 [node for node in self.__nodes if 'controller' in node.get_name()])
122 def get_computes(self):
123 # Get list of computes
125 [node for node in self.__nodes if 'compute' in node.get_name()])
131 def __open_sftp_session(self, host, user, passwd=None):
132 # Connect to given host.
133 """Keyword arguments:
134 host -- host to connect
136 passwd -- password to use
138 Return tuple of SSH and SFTP client instances.
141 ssh = paramiko.SSHClient()
142 ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
144 # try a direct access using password or private key
145 if not passwd and not self.__priv_key:
147 self.__priv_key = paramiko.RSAKey.from_private_key_file(
148 self.__private_key_file)
150 # connect to the server
152 host, username=user, password=passwd, pkey=self.__priv_key)
153 sftp = ssh.open_sftp()
155 # return SFTP client instance
158 def get_plugin_interval(self, compute, plugin):
159 """Find the plugin interval in collectd configuration.
162 compute -- compute node instance
163 plugin -- plug-in name
165 If found, return interval value, otherwise the default value"""
166 ssh, sftp = self.__open_sftp_session(
167 compute.get_ip(), 'root', 'opnfvapex')
170 default_interval = DEF_PLUGIN_INTERVAL
171 config_files = [COLLECTD_CONF] + [
172 COLLECTD_CONF_DIR + '/'
173 + conf_file for conf_file in sftp.listdir(COLLECTD_CONF_DIR)]
174 for config_file in config_files:
176 with sftp.open(config_file) as config:
177 for line in config.readlines():
179 if len(words) > 1 and words[0] == '<LoadPlugin':
181 plugin_name = words[1].strip('">')
182 if words and words[0] == '</LoadPlugin>':
184 if words and words[0] == 'Interval':
185 if in_plugin and plugin_name == plugin:
188 default_interval = int(words[1])
190 self.__logger.error("Could not open collectd.conf file.")
191 return default_interval
193 def get_plugin_config_values(self, compute, plugin, parameter):
194 """Get parameter values from collectd config file.
197 compute -- compute node instance
198 plugin -- plug-in name
199 parameter -- plug-in parameter
201 Return list of found values."""
202 ssh, sftp = self.__open_sftp_session(
203 compute.get_ip(), 'root', 'opnfvapex')
204 # find the plugin value
208 config_files = [COLLECTD_CONF] + [
209 COLLECTD_CONF_DIR + '/'
210 + conf_file for conf_file in sftp.listdir(COLLECTD_CONF_DIR)]
211 for config_file in config_files:
213 with sftp.open(config_file) as config:
214 for line in config.readlines():
216 if len(words) > 1 and words[0] == '<Plugin':
218 plugin_name = words[1].strip('">')
219 if len(words) > 0 and words[0] == '</Plugin>':
221 if len(words) > 0 and words[0] == parameter:
222 if in_plugin and plugin_name == plugin:
223 return [word.strip('"') for word in words[1:]]
225 self.__logger.error("Could not open collectd.conf file.")
226 return default_values
228 def execute_command(self, command, host_ip=None, ssh=None):
229 """Execute command on node and return list of lines of standard output.
233 host_ip -- IP of the node
234 ssh -- existing open SSH session to use
236 One of host_ip or ssh must not be None. If both are not None,
237 existing ssh session is used.
239 if host_ip is None and ssh is None:
240 raise ValueError('One of host_ip or ssh must not be None.')
242 ssh, sftp = self.__open_sftp_session(host_ip, 'root', 'opnfvapex')
243 stdin, stdout, stderr = ssh.exec_command(command)
244 return stdout.readlines()
246 def get_ovs_interfaces(self, compute):
247 """Get list of configured OVS interfaces
250 compute -- compute node instance
252 stdout = self.execute_command("ovs-vsctl list-br", compute.get_ip())
253 return [interface.strip() for interface in stdout]
255 def is_gnocchi_running(self, controller):
256 """Check whether Gnocchi is running on controller.
259 controller -- controller node instance
261 Return boolean value whether Gnocchi is running.
263 gnocchi_present = False
264 lines = self.execute_command(
265 'source overcloudrc.v3;systemctl status openstack-gnocchi-api | '
266 + 'grep running', controller.get_ip())
268 if '(running)' in line:
269 gnocchi_present = True
270 return gnocchi_present
272 def is_aodh_running(self, controller):
273 """Check whether aodh service is running on controller
276 lines = self.execute_command(
277 'source overcloudrc.v3;systemctl openstack-aodh-api | grep running',
280 self.__logger.info("Line = {}" .format(line))
281 if '(running)' in line:
285 def is_installed(self, compute, package):
286 """Check whether package exists on compute node.
289 compute -- compute node instance
290 package -- Linux package to search for
292 Return boolean value whether package is installed.
294 stdout = self.execute_command(
295 'yum list installed | grep {}'.format(package),
297 return len(stdout) > 0
299 def is_libpqos_on_node(self, compute):
300 """Check whether libpqos is present on compute node"""
301 ssh, sftp = self.__open_sftp_session(
302 compute.get_ip(), 'root', 'opnfvapex')
303 stdin, stdout, stderr = \
304 ssh.exec_command("ls /usr/local/lib/ | grep libpqos")
305 output = stdout.readlines()
311 def check_gnocchi_plugin_included(self, compute):
312 """Check if gnocchi plugin is included in collectd.conf file.
313 If not, try to enable it.
316 compute -- compute node instance
318 Return boolean value whether gnocchi plugin is included
319 or it's enabling was successful.
321 ssh, sftp = self.__open_sftp_session(
322 compute.get_ip(), 'root', 'opnfvapex')
324 config = sftp.open(COLLECTD_CONF, mode='r')
327 'Cannot open {} on node {}'.format(
328 COLLECTD_CONF, compute.get_name()))
330 in_lines = config.readlines()
331 out_lines = in_lines[:]
332 include_section_indexes = [
333 (start, end) for start in range(len(in_lines))
334 for end in range(len(in_lines))
336 and '<Include' in in_lines[start]
337 and COLLECTD_CONF_DIR in in_lines[start]
338 and '#' not in in_lines[start]
339 and '</Include>' in in_lines[end]
340 and '#' not in in_lines[end]
342 i for i in in_lines[start + 1: end]
343 if 'Filter' in i and '*.conf' in i and '#' not in i]) > 0]
344 if len(include_section_indexes) == 0:
345 out_lines.append('<Include "{}">\n'.format(COLLECTD_CONF_DIR))
346 out_lines.append(' Filter "*.conf"\n')
347 out_lines.append('</Include>\n')
349 config = sftp.open(COLLECTD_CONF, mode='w')
350 config.writelines(out_lines)
352 self.__logger.info('Creating backup of collectd.conf...')
353 config = sftp.open(COLLECTD_CONF + '.backup', mode='w')
354 config.writelines(in_lines)
358 def check_ceil_plugin_included(self, compute):
359 """Check if ceilometer plugin is included in collectd.conf file.
360 If not, try to enable it.
363 compute -- compute node instance
365 Return boolean value whether ceilometer plugin is included
366 or it's enabling was successful.
368 ssh, sftp = self.__open_sftp_session(compute.get_ip(), 'root')
370 config = sftp.open(COLLECTD_CONF, mode='r')
373 'Cannot open {} on node {}'.format(
374 COLLECTD_CONF, compute.get_id()))
376 in_lines = config.readlines()
377 out_lines = in_lines[:]
378 include_section_indexes = [
379 (start, end) for start in range(len(in_lines))
380 for end in range(len(in_lines))
382 and '<Include' in in_lines[start]
383 and COLLECTD_CONF_DIR in in_lines[start]
384 and '#' not in in_lines[start]
385 and '</Include>' in in_lines[end]
386 and '#' not in in_lines[end]
388 i for i in in_lines[start + 1: end]
389 if 'Filter' in i and '*.conf' in i and '#' not in i]) > 0]
390 if len(include_section_indexes) == 0:
391 out_lines.append('<Include "{}">\n'.format(COLLECTD_CONF_DIR))
392 out_lines.append(' Filter "*.conf"\n')
393 out_lines.append('</Include>\n')
395 config = sftp.open(COLLECTD_CONF, mode='w')
396 config.writelines(out_lines)
398 self.__logger.info('Creating backup of collectd.conf...')
399 config = sftp.open(COLLECTD_CONF + '.backup', mode='w')
400 config.writelines(in_lines)
405 self, compute, plugins, error_plugins, create_backup=True):
406 """Enable plugins on compute node
409 compute -- compute node instance
410 plugins -- list of plugins to be enabled
411 error_plugins -- list of tuples with found errors, new entries
412 may be added there (plugin, error_description, is_critical):
413 plugin -- plug-in name
414 error_decription -- description of the error
415 is_critical -- boolean value indicating whether error
417 create_backup -- boolean value indicating whether backup
420 Return boolean value indicating whether function was successful.
422 plugins = sorted(plugins)
423 ssh, sftp = self.__open_sftp_session(
424 compute.get_ip(), 'root', 'opnfvapex')
425 plugins_to_enable = plugins[:]
426 for plugin in plugins:
427 plugin_file = '/usr/lib64/collectd/{}.so'.format(plugin)
429 sftp.stat(plugin_file)
432 'Plugin file {} not found on node'.format(plugin_file)
433 + ' {0}, plugin {1} will not be enabled'.format(
434 compute.get_name(), plugin))
435 error_plugins.append((
436 plugin, 'plugin file {} not found'.format(plugin_file),
438 plugins_to_enable.remove(plugin)
440 'Following plugins will be enabled on node {}: {}'.format(
441 compute.get_name(), ', '.join(plugins_to_enable)))
443 config = sftp.open(COLLECTD_CONF, mode='r')
445 self.__logger.warning(
446 'Cannot open {} on node {}'.format(
447 COLLECTD_CONF, compute.get_name()))
449 in_lines = config.readlines()
452 enabled_sections = []
454 comment_section = False
455 uncomment_section = False
456 for line in in_lines:
457 if 'LoadPlugin' in line:
458 for plugin in plugins_to_enable:
460 commented = '#' in line
461 # list of uncommented lines which contain LoadPlugin
464 ll for ll in in_lines if 'LoadPlugin' in ll
465 and plugin in ll and '#' not in ll]
466 if len(loadlines) == 0:
467 if plugin not in enabled_plugins:
468 line = line.lstrip(string.whitespace + '#')
469 enabled_plugins.append(plugin)
470 error_plugins.append((
471 plugin, 'plugin not enabled in '
472 + '{}, trying to enable it'.format(
473 COLLECTD_CONF), False))
475 if plugin not in enabled_plugins:
476 enabled_plugins.append(plugin)
479 error_plugins.append((
480 plugin, 'plugin enabled more than once '
481 + '(additional occurrence of LoadPlugin '
482 + 'found in {}), '.format(COLLECTD_CONF)
483 + 'trying to comment it out.', False))
484 elif line.lstrip(string.whitespace + '#').find('<Plugin') == 0:
486 for plugin in plugins_to_enable:
488 commented = '#' in line
489 # list of uncommented lines which contain Plugin for
492 pl for pl in in_lines if '<Plugin' in pl
493 and plugin in pl and '#' not in pl]
494 if len(pluginlines) == 0:
495 if plugin not in enabled_sections:
496 line = line[line.rfind('#') + 1:]
497 uncomment_section = True
498 enabled_sections.append(plugin)
499 error_plugins.append((
500 plugin, 'plugin section found in '
501 + '{}, but commented'.format(COLLECTD_CONF)
502 + ' out, trying to uncomment it.', False))
504 if plugin not in enabled_sections:
505 enabled_sections.append(plugin)
508 comment_section = True
509 error_plugins.append((
510 plugin, 'additional occurrence of plugin '
511 + 'section found in {}'.format(
513 + ', trying to comment it out.', False))
515 if comment_section and '#' not in line:
517 if uncomment_section and '#' in line:
518 line = line[line.rfind('#') + 1:]
519 if '</Plugin>' in line:
522 comment_section = False
523 uncomment_section = False
524 elif '</Plugin>' in line:
526 'Unexpected closure os plugin section on line'
527 + ' {} in collectd.conf'.format(len(out_lines) + 1)
528 + ', matching section start not found.')
530 out_lines.append(line)
533 'Unexpected end of file collectd.conf, '
534 + 'closure of last plugin section not found.')
537 'LoadPlugin {}\n'.format(plugin) for plugin in plugins_to_enable
538 if plugin not in enabled_plugins] + out_lines
539 for plugin in plugins_to_enable:
540 if plugin not in enabled_plugins:
541 error_plugins.append((
542 plugin, 'plugin not enabled in {},'.format(COLLECTD_CONF)
543 + ' trying to enable it.', False))
544 unenabled_sections = [plugin for plugin in plugins_to_enable
545 if plugin not in enabled_sections]
546 if unenabled_sections:
548 'Plugin sections for following plugins not found: {}'.format(
549 ', '.join(unenabled_sections)))
554 self.__logger.info('Creating backup of collectd.conf...')
555 config = sftp.open(COLLECTD_CONF + '.backup', mode='w')
556 config.writelines(in_lines)
558 self.__logger.info('Updating collectd.conf...')
559 config = sftp.open(COLLECTD_CONF, mode='w')
560 config.writelines(out_lines)
563 "diff {} {}.backup".format(COLLECTD_CONF, COLLECTD_CONF)
564 stdin, stdout, stderr = ssh.exec_command(diff_command)
565 self.__logger.debug(diff_command)
566 for line in stdout.readlines():
567 self.__logger.debug(line.strip())
570 def restore_config(self, compute):
571 """Restore collectd config file from backup on compute node.
574 compute -- compute node instance
576 ssh, sftp = self.__open_sftp_session(
577 compute.get_ip(), 'root', 'opnfvapex')
579 self.__logger.info('Restoring config file from backup...')
580 ssh.exec_command("cp {0} {0}.used".format(COLLECTD_CONF))
581 ssh.exec_command("cp {0}.backup {0}".format(COLLECTD_CONF))
583 def restart_collectd(self, compute):
584 """Restart collectd on compute node.
587 compute -- compute node instance
589 Retrun tuple with boolean indicating success and list of warnings
590 received during collectd start.
593 def get_collectd_processes(ssh_session):
594 """Get number of running collectd processes.
597 ssh_session -- instance of SSH session in which to check
600 stdin, stdout, stderr = ssh_session.exec_command(
602 return len(stdout.readlines())
604 ssh, sftp = self.__open_sftp_session(
605 compute.get_ip(), 'root', 'opnfvapex')
607 self.__logger.info('Stopping collectd service...')
608 stdout = self.execute_command("service collectd stop", ssh=ssh)
610 if get_collectd_processes(ssh):
611 self.__logger.error('Collectd is still running...')
613 self.__logger.info('Starting collectd service...')
614 stdout = self.execute_command("service collectd start", ssh=ssh)
616 warning = [output.strip() for output in stdout if 'WARN: ' in output]
617 if get_collectd_processes(ssh) == 0:
618 self.__logger.error('Collectd is still not running...')
619 return False, warning
622 def test_gnocchi_is_sending_data(self, controller):
623 """ Checking if Gnocchi is sending metrics to controller"""
627 ssh, sftp = self.__open_sftp_session(
628 controller.get_ip(), 'root', 'opnfvapex')
630 self.__logger.info('Getting gnocchi metric list on{}'.format(
631 controller.get_name()))
632 stdout = self.execute_command(
633 "source overcloudrc.v3;gnocchi metric list | grep if_packets",
636 metric_ids = [r.split('|')[1] for r in stdout]
637 self.__logger.info("Metric ids = {}" .format(metric_ids))
638 for metric_id in metric_ids:
639 metric_id = metric_id.replace("u", "")
640 stdout = self.execute_command(
641 "source overcloudrc.v3;gnocchi measures show {}" .format(
643 self.__logger.info("stdout measures ={}" .format(stdout))
648 self.__logger.info("Line = {}" .format(line))
649 timestamps1 = [line.split('|')[1]]
650 self.__logger.info("Last line timetamp1 = {}" .format(timestamps1))
652 stdout = self.execute_command(
653 "source overcloudrc.v3;gnocchi measures show {}" .format(
659 timestamps2 = [line.split('|')[1]]
660 self.__logger.info("Last line timetamp2 = {}" .format(timestamps2))
661 if timestamps1 == timestamps2:
662 self.__logger.info("False")
666 self.__logger.info("True")
669 def test_plugins_with_aodh(self, controller):
670 """Checking if AODH is sending metrics to controller"""
674 ssh, sftp = self.__open_sftp_session(
675 controller.get_ip(), 'root', 'opnfvapex')
676 self.__logger.info('Getting AODH alarm list on{}'.format(
677 controller.get_name()))
678 stdout = self.execute_command(
679 "source overcloudrc.v3;aodh alarm list | grep mcelog",
682 metric_ids = [r.split('|')[1] for r in stdout]
683 self.__logger.info("Metric ids = {}" .format(metric_ids))
684 for metric_id in metric_ids:
685 metric_id = metric_id.replace("u", "")
686 stdout = self.execute_command(
687 "source overcloudrc.v3;aodh alarm show {}" .format(
689 self.__logger.info("stdout alarms ={}" .format(stdout))
694 self.__logger.info("Line = {}" .format(line))
695 timestamps1 = [line.split('|')[1]]
696 self.__logger.info("Last line timetamp1 = {}" .format(timestamps1))
698 stdout = self.execute_command(
699 "source overcloudrc.v3;aodh alarm show {}" .format(
705 timestamps2 = [line.split('|')[1]]
706 self.__logger.info("Last line timetamp2 = {}" .format(timestamps2))
707 if timestamps1 == timestamps2:
708 self.__logger.info("False")
712 self.__logger.info("True")
715 def test_plugins_with_gnocchi(
716 self, controller, compute_node, plugin_interval, logger,
722 ssh, sftp = self.__open_sftp_session(
723 controller.get_ip(), 'root', 'opnfvapex')
724 self.__logger.info('Getting gnocchi metric list on{}'.format(
725 controller.get_name()))
726 stdout = self.execute_command(
727 "source overcloudrc.v3;gnocchi metric list | grep {0} | grep {1}"
728 .format(compute_node.get_name(), criteria_list), ssh=ssh)
730 metric_ids = [r.split('|')[1] for r in stdout]
731 self.__logger.info("Metric ids = {}" .format(metric_ids))
732 for metric_id in metric_ids:
733 metric_id = metric_id.replace("u", "")
734 stdout = self.execute_command(
735 "source overcloudrc.v3;gnocchi measures show {}" .format(
737 self.__logger.info("stdout measures ={}" .format(stdout))
742 self.__logger.info("Line = {}" .format(line))
743 timestamps1 = [line.split('|')[1]]
744 self.__logger.info("Last line timetamp1 = {}" .format(timestamps1))
746 stdout = self.execute_command(
747 "source overcloudrc.v3;gnocchi measures show {}" .format(
753 timestamps2 = [line.split('|')[1]]
754 self.__logger.info("Last line timetamp2 = {}" .format(timestamps2))
755 if timestamps1 == timestamps2:
756 self.__logger.info("False")
759 self.__logger.info("True")