-#!/usr/bin/python
+"""Executing test of plugins"""
+# -*- coding: utf-8 -*-
-import sys
+#Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+import requests
+from keystoneclient.v3 import client
+import os
+import time
+import logging
+from config_server import *
+from tests import *
-def main(logger):
- logger.info("Running Baromtercollectd test suite...")
- #
- # TODO: implementation
- #
- logger.info("Test suite successfully completed.")
+CEILOMETER_NAME = 'ceilometer'
+
+
+class KeystoneException(Exception):
+ """Keystone exception class"""
+ def __init__(self, message, exc=None, response=None):
+ """
+ Keyword arguments:
+ message -- error message
+ exc -- exception
+ response -- response
+ """
+ if exc:
+ message += "\nReason: %s" % exc
+ super(KeystoneException, self).__init__(message)
+
+ self.response = response
+ self.exception = exc
+
+
+class InvalidResponse(KeystoneException):
+ """Invalid Keystone exception class"""
+ def __init__(self, exc, response):
+ """
+ Keyword arguments:
+ exc -- exception
+ response -- response
+ """
+ super(InvalidResponse, self).__init__(
+ "Invalid response", exc, response)
+
+
+class CeilometerClient(object):
+ """Ceilometer Client to authenticate and request meters"""
+ def __init__(self, bc_logger):
+ """
+ Keyword arguments:
+ bc_logger - logger instance
+ """
+ self._auth_token = None
+ self._ceilometer_url = None
+ self._meter_list = None
+ self._logger = bc_logger
+
+ def auth_token(self):
+ """Get auth token"""
+ self._auth_server()
+ return self._auth_token
+
+ def get_ceilometer_url(self):
+ """Get Ceilometer URL"""
+ return self._ceilometer_url
+
+ def get_ceil_metrics(self, criteria=None):
+ """Get Ceilometer metrics for given criteria
+
+ Keyword arguments:
+ criteria -- criteria for ceilometer meter list
+ """
+ self._request_meters(criteria)
+ return self._meter_list
+
+ def _auth_server(self):
+ """Request token in authentication server"""
+ self._logger.debug('Connecting to the auth server {}'.format(os.environ['OS_AUTH_URL']))
+ keystone = client.Client(username=os.environ['OS_USERNAME'],
+ password=os.environ['OS_PASSWORD'],
+ tenant_name=os.environ['OS_TENANT_NAME'],
+ auth_url=os.environ['OS_AUTH_URL'])
+ self._auth_token = keystone.auth_token
+ for service in keystone.service_catalog.get_data():
+ if service['name'] == CEILOMETER_NAME:
+ for service_type in service['endpoints']:
+ if service_type['interface'] == 'internal':
+ self._ceilometer_url = service_type['url']
+ break
+
+ if self._ceilometer_url is None:
+ self._logger.warning('Ceilometer is not registered in service catalog')
+
+ def _request_meters(self, criteria):
+ """Request meter list values from ceilometer
+
+ Keyword arguments:
+ criteria -- criteria for ceilometer meter list
+ """
+ if criteria is None:
+ url = self._ceilometer_url + ('/v2/samples?limit=400')
+ else:
+ url = self._ceilometer_url + ('/v2/meters/%s?q.field=resource_id&limit=400' % criteria)
+ headers = {'X-Auth-Token': self._auth_token}
+ resp = requests.get(url, headers=headers)
+ try:
+ resp.raise_for_status()
+ self._meter_list = resp.json()
+ except (KeyError, ValueError, requests.exceptions.HTTPError) as err:
+ raise InvalidResponse(err, resp)
+
+
+class CSVClient(object):
+ """Client to request CSV meters"""
+ def __init__(self, bc_logger, conf):
+ """
+ Keyword arguments:
+ bc_logger - logger instance
+ conf -- ConfigServer instance
+ """
+ self._logger = bc_logger
+ self.conf = conf
+
+ def get_csv_metrics(self, compute_node, plugin_subdirectories, meter_categories):
+ """Get CSV metrics.
+
+ Keyword arguments:
+ compute_node -- compute node instance
+ plugin_subdirectories -- list of subdirectories of plug-in
+ meter_categories -- categories which will be tested
+
+ Return list of metrics.
+ """
+ stdout = self.conf.execute_command("date '+%Y-%m-%d'", compute_node.get_ip())
+ date = stdout[0].strip()
+ metrics = []
+ for plugin_subdir in plugin_subdirectories:
+ for meter_category in meter_categories:
+ stdout = self.conf.execute_command(
+ "tail -2 /var/lib/collectd/csv/node-"
+ + "{0}.domain.tld/{1}/{2}-{3}".format(
+ compute_node.get_id(), plugin_subdir, meter_category, date),
+ compute_node.get_ip())
+ #Storing last two values
+ values = stdout
+ if len(values) < 2:
+ self._logger.error(
+ 'Getting last two CSV entries of meter category '
+ + '{0} in {1} subdir failed'.format(meter_category, plugin_subdir))
+ else:
+ old_value = int(values[0][0:values[0].index('.')])
+ new_value = int(values[1][0:values[1].index('.')])
+ metrics.append((plugin_subdir, meter_category, old_value, new_value))
+ return metrics
+
+
+def _check_logger():
+ """Check whether there is global logger available and if not, define one."""
+ if 'logger' not in globals():
+ global logger
+ logger = logger.Logger("barometercollectd").getLogger()
+
+
+def _process_result(compute_node, test, result, results_list):
+ """Print test result and append it to results list.
+
+ Keyword arguments:
+ test -- testcase name
+ result -- boolean test result
+ results_list -- results list
+ """
+ if result:
+ logger.info('Compute node {0} test case {1} PASSED.'.format(compute_node, test))
+ else:
+ logger.error('Compute node {0} test case {1} FAILED.'.format(compute_node, test))
+ results_list.append((compute_node, test, result))
+
+
+def _print_label(label):
+ """Print label on the screen
+
+ Keyword arguments:
+ label -- label string
+ """
+ label = label.strip()
+ length = 70
+ if label != '':
+ label = ' ' + label + ' '
+ length_label = len(label)
+ length1 = (length - length_label) / 2
+ length2 = length - length_label - length1
+ length1 = max(3, length1)
+ length2 = max(3, length2)
+ logger.info(('=' * length1) + label + ('=' * length2))
+
+
+def _print_plugin_label(plugin, node_id):
+ """Print plug-in label.
+
+ Keyword arguments:
+ plugin -- plug-in name
+ node_id -- node ID
+ """
+ _print_label('Node {0}: Plug-in {1} Test case execution'.format(node_id, plugin))
+
+
+def _print_final_result_of_plugin(plugin, compute_ids, results, out_plugins, out_plugin):
+ """Print final results of plug-in.
+
+ Keyword arguments:
+ plugin -- plug-in name
+ compute_ids -- list of compute node IDs
+ results -- results list
+ out_plugins -- list of out plug-ins
+ out_plugin -- used out plug-in
+ """
+ print_line = ''
+ for id in compute_ids:
+ if out_plugins[id] == out_plugin:
+ if (id, plugin, True) in results:
+ print_line += ' PASS |'
+ elif (id, plugin, False) in results and out_plugins[id] == out_plugin:
+ print_line += ' FAIL |'
+ else:
+ print_line += ' NOT EX |'
+ elif out_plugin == 'Ceilometer':
+ print_line += ' NOT EX |'
+ else:
+ print_line += ' SKIP |'
+ return print_line
+
+
+def print_overall_summary(compute_ids, tested_plugins, results, out_plugins):
+ """Print overall summary table.
+
+ Keyword arguments:
+ compute_ids -- list of compute IDs
+ tested_plugins -- list of plug-ins
+ results -- results list
+ out_plugins -- list of used out plug-ins
+ """
+ compute_node_names = ['Node-{}'.format(id) for id in compute_ids]
+ all_computes_in_line = ''
+ for compute in compute_node_names:
+ all_computes_in_line = all_computes_in_line + '| ' + compute + (' ' * (7 - len(compute)))
+ line_of_nodes = '| Test ' + all_computes_in_line + '|'
+ logger.info('=' * 70)
+ logger.info('+' + ('-' * ((9 * len(compute_node_names))+16)) + '+')
+ logger.info(
+ '|' + ' ' * ((9*len(compute_node_names))/2) + ' OVERALL SUMMARY'
+ + ' ' * (9*len(compute_node_names) - (9*len(compute_node_names))/2) + '|')
+ logger.info('+' + ('-' * 16) + '+' + (('-' * 8) + '+') * len(compute_node_names))
+ logger.info(line_of_nodes)
+ logger.info('+' + ('-' * 16) + '+' + (('-' * 8) + '+') * len(compute_node_names))
+ out_plugins_print = ['Ceilometer']
+ if 'CSV' in out_plugins.values():
+ out_plugins_print.append('CSV')
+ for out_plugin in out_plugins_print:
+ output_plugins_line = ''
+ for id in compute_ids:
+ out_plugin_result = '----'
+ if out_plugin == 'Ceilometer':
+ out_plugin_result = 'PASS' if out_plugins[id] == out_plugin else 'FAIL'
+ if out_plugin == 'CSV':
+ if out_plugins[id] == out_plugin:
+ out_plugin_result = \
+ 'PASS' if [
+ plugin for comp_id, plugin,
+ res in results if comp_id == id and res] else 'FAIL'
+ else:
+ out_plugin_result = 'SKIP'
+ output_plugins_line += '| ' + out_plugin_result + ' '
+ logger.info(
+ '| OUT:{}'.format(out_plugin) + (' ' * (11 - len(out_plugin)))
+ + output_plugins_line + '|')
+ for plugin in sorted(tested_plugins.values()):
+ line_plugin = _print_final_result_of_plugin(
+ plugin, compute_ids, results, out_plugins, out_plugin)
+ logger.info('| IN:{}'.format(plugin) + (' ' * (11-len(plugin))) + '|' + line_plugin)
+ logger.info('+' + ('-' * 16) + '+' + (('-' * 8) + '+') * len(compute_node_names))
+ logger.info('=' * 70)
+
+
+def _exec_testcase(
+ test_labels, name, ceilometer_running, compute_node,
+ conf, results, error_plugins):
+ """Execute the testcase.
+
+ Keyword arguments:
+ test_labels -- dictionary of plug-in IDs and their display names
+ name -- plug-in ID, key of test_labels dictionary
+ ceilometer_running -- boolean indicating whether Ceilometer is running
+ compute_node -- compute node ID
+ conf -- ConfigServer instance
+ results -- results list
+ error_plugins -- list of tuples with plug-in errors (plugin, error_description, is_critical):
+ plugin -- plug-in ID, key of test_labels dictionary
+ error_decription -- description of the error
+ is_critical -- boolean value indicating whether error is critical
+ """
+ ovs_interfaces = conf.get_ovs_interfaces(compute_node)
+ ovs_configured_interfaces = conf.get_plugin_config_values(
+ compute_node, 'ovs_events', 'Interfaces')
+ ovs_existing_configured_int = [
+ interface for interface in ovs_interfaces
+ if interface in ovs_configured_interfaces]
+ plugin_prerequisites = {
+ 'mcelog': [(conf.is_installed(compute_node, 'mcelog'), 'mcelog must be installed.')],
+ 'ovs_events': [(
+ len(ovs_existing_configured_int) > 0 or len(ovs_interfaces) > 0,
+ 'Interfaces must be configured.')]}
+ ceilometer_criteria_lists = {
+ 'hugepages': ['hugepages.vmpage_number'],
+ 'mcelog': ['mcelog.errors'],
+ 'ovs_events': ['ovs_events.gauge']}
+ ceilometer_substr_lists = {
+ 'ovs_events': ovs_existing_configured_int if len(ovs_existing_configured_int) > 0 else ovs_interfaces}
+ csv_subdirs = {
+ 'hugepages': [
+ 'hugepages-mm-2048Kb', 'hugepages-node0-2048Kb', 'hugepages-node1-2048Kb',
+ 'hugepages-mm-1048576Kb', 'hugepages-node0-1048576Kb', 'hugepages-node1-1048576Kb'],
+ 'mcelog': ['mcelog-SOCKET_0_CHANNEL_0_DIMM_any', 'mcelog-SOCKET_0_CHANNEL_any_DIMM_any'],
+ 'ovs_events': [
+ 'ovs_events-{}'.format(interface)
+ for interface in (ovs_existing_configured_int if len(ovs_existing_configured_int) > 0 else ovs_interfaces)]}
+ csv_meter_categories = {
+ 'hugepages': ['vmpage_number-free', 'vmpage_number-used'],
+ 'mcelog': [
+ 'errors-corrected_memory_errors', 'errors-uncorrected_memory_errors',
+ 'errors-corrected_memory_errors_in_24h', 'errors-uncorrected_memory_errors_in_24h'],
+ 'ovs_events': ['gauge-link_status']}
+
+ _print_plugin_label(test_labels[name] if name in test_labels else name, compute_node.get_id())
+ plugin_critical_errors = [
+ error for plugin, error, critical in error_plugins if plugin == name and critical]
+ if plugin_critical_errors:
+ logger.error('Following critical errors occurred:'.format(name))
+ for error in plugin_critical_errors:
+ logger.error(' * ' + error)
+ _process_result(compute_node.get_id(), test_labels[name], False, results)
+ else:
+ plugin_errors = [
+ error for plugin, error, critical in error_plugins if plugin == name and not critical]
+ if plugin_errors:
+ logger.warning('Following non-critical errors occured:')
+ for error in plugin_errors:
+ logger.warning(' * ' + error)
+ failed_prerequisites = []
+ if name in plugin_prerequisites:
+ failed_prerequisites = [
+ prerequisite_name for prerequisite_passed,
+ prerequisite_name in plugin_prerequisites[name] if not prerequisite_passed]
+ if failed_prerequisites:
+ logger.error(
+ '{} test will not be executed, '.format(name)
+ + 'following prerequisites failed:')
+ for prerequisite in failed_prerequisites:
+ logger.error(' * {}'.format(prerequisite))
+ else:
+ if ceilometer_running:
+ res = test_ceilometer_node_sends_data(
+ compute_node.get_id(), conf.get_plugin_interval(compute_node, name),
+ logger=logger, client=CeilometerClient(logger),
+ criteria_list=ceilometer_criteria_lists[name],
+ resource_id_substrings = (ceilometer_substr_lists[name]
+ if name in ceilometer_substr_lists else ['']))
+ else:
+ res = test_csv_handles_plugin_data(
+ compute_node, conf.get_plugin_interval(compute_node, name), name,
+ csv_subdirs[name], csv_meter_categories[name], logger,
+ CSVClient(logger, conf))
+ if res and plugin_errors:
+ logger.info(
+ 'Test works, but will be reported as failure,'
+ + 'because of non-critical errors.')
+ res = False
+ _process_result(compute_node.get_id(), test_labels[name], res, results)
+
+
+def main(bt_logger=None):
+ """Check each compute node sends ceilometer metrics.
+
+ Keyword arguments:
+ bt_logger -- logger instance
+ """
+ logging.getLogger("paramiko").setLevel(logging.WARNING)
+ logging.getLogger("stevedore").setLevel(logging.WARNING)
+ if bt_logger is None:
+ _check_logger()
+ else:
+ global logger
+ logger = bt_logger
+ conf = ConfigServer('10.20.0.2', 'root', logger)
+ controllers = conf.get_controllers()
+ if len(controllers) == 0:
+ logger.error('No controller nodes found!')
+ return 1
+ computes = conf.get_computes()
+ if len(computes) == 0:
+ logger.error('No compute nodes found!')
+ return 1
+
+ _print_label('Display of Control and Compute nodes available in the set up')
+ logger.info('controllers: {}'.format([('{0}: {1} ({2})'.format(
+ node.get_id(), node.get_name(), node.get_ip())) for node in controllers]))
+ logger.info('computes: {}'.format([('{0}: {1} ({2})'.format(
+ node.get_id(), node.get_name(), node.get_ip())) for node in computes]))
+
+ ceilometer_running_on_con = False
+ _print_label('Test Ceilometer on control nodes')
+ for controller in controllers:
+ ceil_client = CeilometerClient(logger)
+ ceil_client.auth_token()
+ ceilometer_running_on_con = (
+ ceilometer_running_on_con or conf.is_ceilometer_running(controller))
+ if ceilometer_running_on_con:
+ logger.info("Ceilometer is running on control node.")
+ else:
+ logger.error("Ceilometer is not running on control node.")
+ logger.info("CSV will be enabled on compute nodes.")
+ compute_ids = []
+ results = []
+ plugin_labels = {
+ 'hugepages': 'Hugepages',
+ 'mcelog': 'Mcelog',
+ 'ovs_events': 'OVS events'}
+ out_plugins = {}
+ for compute_node in computes:
+ node_id = compute_node.get_id()
+ out_plugins[node_id] = 'CSV'
+ compute_ids.append(node_id)
+ #plugins_to_enable = plugin_labels.keys()
+ plugins_to_enable = []
+ _print_label('NODE {}: Test Ceilometer Plug-in'.format(node_id))
+ logger.info('Checking if ceilometer plug-in is included.')
+ if not conf.check_ceil_plugin_included(compute_node):
+ logger.error('Ceilometer plug-in is not included.')
+ logger.info('Testcases on node {} will not be executed'.format(node_id))
+ else:
+ collectd_restarted, collectd_warnings = conf.restart_collectd(compute_node)
+ sleep_time = 30
+ logger.info('Sleeping for {} seconds after collectd restart...'.format(sleep_time))
+ time.sleep(sleep_time)
+ if not collectd_restarted:
+ for warning in collectd_warnings:
+ logger.warning(warning)
+ logger.error('Restart of collectd on node {} failed'.format(node_id))
+ logger.info('Testcases on node {} will not be executed'.format(node_id))
+ else:
+ for warning in collectd_warnings:
+ logger.warning(warning)
+ ceilometer_running = (
+ ceilometer_running_on_con and test_ceilometer_node_sends_data(
+ node_id, 10, logger=logger, client=CeilometerClient(logger)))
+ if ceilometer_running:
+ out_plugins[node_id] = 'Ceilometer'
+ logger.info("Ceilometer is running.")
+ else:
+ plugins_to_enable.append('csv')
+ out_plugins[node_id] = 'CSV'
+ logger.error("Ceilometer is not running.")
+ logger.info("CSV will be enabled for verification of test plugins.")
+ if plugins_to_enable:
+ _print_label(
+ 'NODE {}: Enabling Test Plug-in '.format(node_id)
+ + 'and Test case execution')
+ error_plugins = []
+ if plugins_to_enable and not conf.enable_plugins(
+ compute_node, plugins_to_enable, error_plugins, create_backup=False):
+ logger.error('Failed to test plugins on node {}.'.format(node_id))
+ logger.info('Testcases on node {} will not be executed'.format(node_id))
+ else:
+ if plugins_to_enable:
+ collectd_restarted, collectd_warnings = conf.restart_collectd(compute_node)
+ sleep_time = 30
+ logger.info(
+ 'Sleeping for {} seconds after collectd restart...'.format(sleep_time))
+ time.sleep(sleep_time)
+ if plugins_to_enable and not collectd_restarted:
+ for warning in collectd_warnings:
+ logger.warning(warning)
+ logger.error('Restart of collectd on node {} failed'.format(node_id))
+ logger.info('Testcases on node {} will not be executed'.format(node_id))
+ else:
+ if collectd_warnings:
+ for warning in collectd_warnings:
+ logger.warning(warning)
+
+ for plugin_name in sorted(plugin_labels.keys()):
+ _exec_testcase(
+ plugin_labels, plugin_name, ceilometer_running,
+ compute_node, conf, results, error_plugins)
+
+ _print_label('NODE {}: Restoring config file'.format(node_id))
+ conf.restore_config(compute_node)
+
+ print_overall_summary(compute_ids, plugin_labels, results, out_plugins)
+
+ if ((len([res for res in results if not res[2]]) > 0)
+ or (len(results) < len(computes) * len(plugin_labels))):
+ logger.error('Some tests have failed or have not been executed')
+ return 1
return 0
if __name__ == '__main__':
- sys.exit(main())
+ sys.exit(main())
Code Review / barometer.git / blobdiff