From ec47f67b94f64a1afcff23b3211c951d6b5973ec Mon Sep 17 00:00:00 2001 From: Sharada Shiddibhavi Date: Wed, 16 Aug 2017 11:32:59 +0000 Subject: [PATCH] Added Functest testcases for Barometer project Added different method to get user credentials of the installer nodes instead of reading from installer_params.yaml Change-Id: I97419c942e1cd9f943a62c36dbce424872a10cb1 Signed-off-by: Sharada Shiddibhavi --- baro_tests/collectd.py | 551 +++++++++++++++++++++++++++----------------- baro_tests/config_server.py | 360 ++++++++++++++++++++--------- baro_tests/mce-inject_ea | Bin 0 -> 75144 bytes baro_tests/tests.py | 105 ++++++--- 4 files changed, 670 insertions(+), 346 deletions(-) create mode 100755 baro_tests/mce-inject_ea diff --git a/baro_tests/collectd.py b/baro_tests/collectd.py index 5ac3c8fe..9e9b3f6b 100644 --- a/baro_tests/collectd.py +++ b/baro_tests/collectd.py @@ -1,4 +1,3 @@ -"""Executing test of plugins""" # -*- coding: utf-8 -*- # Licensed under the Apache License, Version 2.0 (the "License"); you may @@ -13,26 +12,27 @@ # License for the specific language governing permissions and limitations # under the License. +"""Executing test of plugins""" + import requests from keystoneclient.v3 import client import os -import pkg_resources +import sys import time import logging -from config_server import * -from tests import * +import config_server +import tests +import subprocess from opnfv.deployment import factory -from functest.utils import functest_utils -CEILOMETER_NAME = 'ceilometer' +GNOCCHI_NAME = 'gnocchi' ID_RSA_SRC = '/root/.ssh/id_rsa' ID_RSA_DST_DIR = '/home/opnfv/.ssh' ID_RSA_DST = ID_RSA_DST_DIR + '/id_rsa' -INSTALLER_PARAMS_YAML = pkg_resources.resource_filename( - 'functest', 'ci/installer_params.yaml') -FUEL_IP = functest_utils.get_parameter_from_yaml('fuel.ip', INSTALLER_PARAMS_YAML) -FUEL_USER = functest_utils.get_parameter_from_yaml('fuel.user', INSTALLER_PARAMS_YAML) -FUEL_PW = functest_utils.get_parameter_from_yaml('fuel.password', INSTALLER_PARAMS_YAML) +APEX_IP = subprocess.check_output("echo $INSTALLER_IP", shell=True) +APEX_USER = 'root' +APEX_USER_STACK = 'stack' +APEX_PKEY = '/root/.ssh/id_rsa' class KeystoneException(Exception): @@ -64,53 +64,44 @@ class InvalidResponse(KeystoneException): "Invalid response", exc, response) -class CeilometerClient(object): - """Ceilometer Client to authenticate and request meters""" - def __init__(self, bc_logger): - """ - Keyword arguments: - bc_logger - logger instance - """ +class GnocchiClient(object): + # Gnocchi Client to authenticate and request meters + def __init__(self): self._auth_token = None - self._ceilometer_url = None + self._gnocchi_url = None self._meter_list = None - self._logger = bc_logger def auth_token(self): - """Get auth token""" + # Get auth token self._auth_server() return self._auth_token - def get_ceilometer_url(self): - """Get Ceilometer URL""" - return self._ceilometer_url + def get_gnocchi_url(self): + # Get Gnocchi URL + return self._gnocchi_url - def get_ceil_metrics(self, criteria=None): - """Get Ceilometer metrics for given criteria - - Keyword arguments: - criteria -- criteria for ceilometer meter list - """ + def get_gnocchi_metrics(self, criteria=None): + # Subject to change if metric gathering is different for gnocchi self._request_meters(criteria) return self._meter_list def _auth_server(self): - """Request token in authentication server""" - self._logger.debug('Connecting to the auth server {}'.format(os.environ['OS_AUTH_URL'])) + # Request token in authentication server + logger.debug('Connecting to the auth server {}'.format( + os.environ['OS_AUTH_URL'])) keystone = client.Client(username=os.environ['OS_USERNAME'], password=os.environ['OS_PASSWORD'], - tenant_name=os.environ['OS_TENANT_NAME'], + tenant_name=os.environ['OS_USERNAME'], auth_url=os.environ['OS_AUTH_URL']) self._auth_token = keystone.auth_token for service in keystone.service_catalog.get_data(): - if service['name'] == CEILOMETER_NAME: + if service['name'] == GNOCCHI_NAME: for service_type in service['endpoints']: if service_type['interface'] == 'internal': - self._ceilometer_url = service_type['url'] - break + self._gnocchi_url = service_type['url'] - if self._ceilometer_url is None: - self._logger.warning('Ceilometer is not registered in service catalog') + if self._gnocchi_url is None: + logger.warning('Gnocchi is not registered in service catalog') def _request_meters(self, criteria): """Request meter list values from ceilometer @@ -119,9 +110,10 @@ class CeilometerClient(object): criteria -- criteria for ceilometer meter list """ if criteria is None: - url = self._ceilometer_url + ('/v2/samples?limit=400') + url = self._gnocchi_url + ('/v3/resource?limit=400') else: - url = self._ceilometer_url + ('/v2/meters/%s?q.field=resource_id&limit=400' % criteria) + url = self._gnocchi_url \ + + ('/v3/resource/%s?q.field=resource_id&limit=400' % criteria) headers = {'X-Auth-Token': self._auth_token} resp = requests.get(url, headers=headers) try: @@ -133,16 +125,15 @@ class CeilometerClient(object): class CSVClient(object): """Client to request CSV meters""" - def __init__(self, bc_logger, conf): + def __init__(self, conf): """ Keyword arguments: - bc_logger - logger instance conf -- ConfigServer instance """ - self._logger = bc_logger self.conf = conf - def get_csv_metrics(self, compute_node, plugin_subdirectories, meter_categories): + def get_csv_metrics( + self, compute_node, plugin_subdirectories, meter_categories): """Get CSV metrics. Keyword arguments: @@ -152,34 +143,48 @@ class CSVClient(object): Return list of metrics. """ - stdout = self.conf.execute_command("date '+%Y-%m-%d'", compute_node.get_ip()) + stdout = self.conf.execute_command( + "date '+%Y-%m-%d'", compute_node.get_ip()) date = stdout[0].strip() metrics = [] for plugin_subdir in plugin_subdirectories: for meter_category in meter_categories: stdout = self.conf.execute_command( - "tail -2 /var/lib/collectd/csv/node-" - + "{0}.domain.tld/{1}/{2}-{3}".format( - compute_node.get_id(), plugin_subdir, meter_category, date), + "tail -2 /var/lib/collectd/csv/" + + "{0}.jf.intel.com/{1}/{2}-{3}".format( + compute_node.get_name(), plugin_subdir, meter_category, + date), compute_node.get_ip()) # Storing last two values values = stdout if len(values) < 2: - self._logger.error( + logger.error( 'Getting last two CSV entries of meter category ' - + '{0} in {1} subdir failed'.format(meter_category, plugin_subdir)) + + '{0} in {1} subdir failed'.format( + meter_category, plugin_subdir)) else: old_value = int(values[0][0:values[0].index('.')]) new_value = int(values[1][0:values[1].index('.')]) - metrics.append((plugin_subdir, meter_category, old_value, new_value)) + metrics.append(( + plugin_subdir, meter_category, old_value, new_value)) return metrics -def _check_logger(): - """Check whether there is global logger available and if not, define one.""" - if 'logger' not in globals(): - global logger - logger = logger.Logger("barometercollectd").getLogger() +def get_csv_categories_for_ipmi(conf, compute_node): + """Get CSV metrics. + + Keyword arguments: + compute_node -- compute node instance + + Return list of categories. + """ + stdout = conf.execute_command( + "date '+%Y-%m-%d'", compute_node.get_ip()) + date = stdout[0].strip() + categories = conf.execute_command( + "ls /var/lib/collectd/csv/{0}.jf.intel.com/ipmi | grep {1}".format( + compute_node.get_name(), date), compute_node.get_ip()) + return [category.strip()[:-11] for category in categories] def _process_result(compute_node, test, result, results_list): @@ -191,9 +196,13 @@ def _process_result(compute_node, test, result, results_list): results_list -- results list """ if result: - logger.info('Compute node {0} test case {1} PASSED.'.format(compute_node, test)) + logger.info( + 'Compute node {0} test case {1} PASSED.'.format( + compute_node, test)) else: - logger.error('Compute node {0} test case {1} FAILED.'.format(compute_node, test)) + logger.error( + 'Compute node {0} test case {1} FAILED.'.format( + compute_node, test)) results_list.append((compute_node, test, result)) @@ -215,17 +224,19 @@ def _print_label(label): logger.info(('=' * length1) + label + ('=' * length2)) -def _print_plugin_label(plugin, node_id): +def _print_plugin_label(plugin, node_name): """Print plug-in label. Keyword arguments: plugin -- plug-in name node_id -- node ID """ - _print_label('Node {0}: Plug-in {1} Test case execution'.format(node_id, plugin)) + _print_label( + 'Node {0}: Plug-in {1} Test case execution'.format(node_name, plugin)) -def _print_final_result_of_plugin(plugin, compute_ids, results, out_plugins, out_plugin): +def _print_final_result_of_plugin( + plugin, compute_ids, results, out_plugins, out_plugin): """Print final results of plug-in. Keyword arguments: @@ -240,11 +251,12 @@ def _print_final_result_of_plugin(plugin, compute_ids, results, out_plugins, out if out_plugins[id] == out_plugin: if (id, plugin, True) in results: print_line += ' PASS |' - elif (id, plugin, False) in results and out_plugins[id] == out_plugin: + elif (id, plugin, False) in results \ + and out_plugins[id] == out_plugin: print_line += ' FAIL |' else: print_line += ' NOT EX |' - elif out_plugin == 'Ceilometer': + elif out_plugin == 'Gnocchi': print_line += ' NOT EX |' else: print_line += ' SKIP |' @@ -260,34 +272,50 @@ def print_overall_summary(compute_ids, tested_plugins, results, out_plugins): results -- results list out_plugins -- list of used out plug-ins """ - compute_node_names = ['Node-{}'.format(id) for id in compute_ids] + compute_node_names = ['Node-{}'.format(i) for i in range( + len((compute_ids)))] + # compute_node_names = ['Node-{}'.format(id) for id in compute_ids] all_computes_in_line = '' for compute in compute_node_names: - all_computes_in_line = all_computes_in_line + '| ' + compute + (' ' * (7 - len(compute))) + all_computes_in_line += '| ' + compute + (' ' * (7 - len(compute))) line_of_nodes = '| Test ' + all_computes_in_line + '|' logger.info('=' * 70) logger.info('+' + ('-' * ((9 * len(compute_node_names))+16)) + '+') logger.info( - '|' + ' ' * ((9*len(compute_node_names))/2) + ' OVERALL SUMMARY' - + ' ' * (9*len(compute_node_names) - (9*len(compute_node_names))/2) + '|') - logger.info('+' + ('-' * 16) + '+' + (('-' * 8) + '+') * len(compute_node_names)) + '|' + ' ' * ((9*len(compute_node_names))/2) + + ' OVERALL SUMMARY' + + ' ' * ( + 9*len(compute_node_names) - (9*len(compute_node_names))/2) + + '|') + logger.info( + '+' + ('-' * 16) + '+' + (('-' * 8) + '+') * len(compute_node_names)) logger.info(line_of_nodes) - logger.info('+' + ('-' * 16) + '+' + (('-' * 8) + '+') * len(compute_node_names)) - out_plugins_print = ['Ceilometer'] + logger.info( + '+' + ('-' * 16) + '+' + (('-' * 8) + '+') * len(compute_node_names)) + out_plugins_print = ['Gnocchi'] + if 'SNMP' in out_plugins.values(): + out_plugins_print.append('SNMP') if 'CSV' in out_plugins.values(): out_plugins_print.append('CSV') for out_plugin in out_plugins_print: output_plugins_line = '' for id in compute_ids: out_plugin_result = '----' - if out_plugin == 'Ceilometer': - out_plugin_result = 'PASS' if out_plugins[id] == out_plugin else 'FAIL' + if out_plugin == 'Gnocchi': + out_plugin_result = \ + 'PASS' if out_plugins[id] == out_plugin else 'FAIL' + if out_plugin == 'SNMP': + if out_plugins[id] == out_plugin: + out_plugin_result = \ + 'PASS' if out_plugins[id] == out_plugin else 'FAIL' + else: + out_plugin_result = 'SKIP' if out_plugin == 'CSV': if out_plugins[id] == out_plugin: out_plugin_result = \ 'PASS' if [ - plugin for comp_id, plugin, - res in results if comp_id == id and res] else 'FAIL' + plugin for comp_id, plugin, res in results + if comp_id == id and res] else 'FAIL' else: out_plugin_result = 'SKIP' output_plugins_line += '| ' + out_plugin_result + ' ' @@ -297,13 +325,17 @@ def print_overall_summary(compute_ids, tested_plugins, results, out_plugins): for plugin in sorted(tested_plugins.values()): line_plugin = _print_final_result_of_plugin( plugin, compute_ids, results, out_plugins, out_plugin) - logger.info('| IN:{}'.format(plugin) + (' ' * (11-len(plugin))) + '|' + line_plugin) - logger.info('+' + ('-' * 16) + '+' + (('-' * 8) + '+') * len(compute_node_names)) + logger.info( + '| IN:{}'.format(plugin) + (' ' * (11-len(plugin))) + + '|' + line_plugin) + logger.info( + '+' + ('-' * 16) + '+' + + (('-' * 8) + '+') * len(compute_node_names)) logger.info('=' * 70) def _exec_testcase( - test_labels, name, ceilometer_running, compute_node, + test_labels, name, gnocchi_running, compute_node, conf, results, error_plugins): """Execute the testcase. @@ -314,7 +346,8 @@ def _exec_testcase( compute_node -- compute node ID conf -- ConfigServer instance results -- results list - error_plugins -- list of tuples with plug-in errors (plugin, error_description, is_critical): + error_plugins -- list of tuples with plug-in errors + (plugin, error_description, is_critical): plugin -- plug-in ID, key of test_labels dictionary error_decription -- description of the error is_critical -- boolean value indicating whether error is critical @@ -322,46 +355,94 @@ def _exec_testcase( ovs_interfaces = conf.get_ovs_interfaces(compute_node) ovs_configured_interfaces = conf.get_plugin_config_values( compute_node, 'ovs_events', 'Interfaces') + ovs_configured_bridges = conf.get_plugin_config_values( + compute_node, 'ovs_stats', 'Bridges') ovs_existing_configured_int = [ interface for interface in ovs_interfaces if interface in ovs_configured_interfaces] + ovs_existing_configured_bridges = [ + bridge for bridge in ovs_interfaces + if bridge in ovs_configured_bridges] plugin_prerequisites = { - 'mcelog': [(conf.is_installed(compute_node, 'mcelog'), 'mcelog must be installed.')], + 'intel_rdt': [( + conf.is_libpqos_on_node(compute_node), + 'libpqos must be installed.')], + 'mcelog': [( + conf.is_installed(compute_node, 'mcelog'), + 'mcelog must be installed.')], 'ovs_events': [( len(ovs_existing_configured_int) > 0 or len(ovs_interfaces) > 0, - 'Interfaces must be configured.')]} + 'Interfaces must be configured.')], + 'ovs_stats': [( + len(ovs_existing_configured_bridges) > 0, + 'Bridges must be configured.')]} ceilometer_criteria_lists = { + 'intel_rdt': [ + 'intel_rdt.ipc', 'intel_rdt.bytes', + 'intel_rdt.memory_bandwidth'], 'hugepages': ['hugepages.vmpage_number'], + 'ipmi': ['ipmi.temperature', 'ipmi.voltage'], 'mcelog': ['mcelog.errors'], + 'ovs_stats': ['interface.if_packets'], 'ovs_events': ['ovs_events.gauge']} ceilometer_substr_lists = { - 'ovs_events': ovs_existing_configured_int if len(ovs_existing_configured_int) > 0 else ovs_interfaces} + 'ovs_events': ovs_existing_configured_int if len( + ovs_existing_configured_int) > 0 else ovs_interfaces} csv_subdirs = { + 'intel_rdt': [ + 'intel_rdt-{}'.format(core) + for core in conf.get_plugin_config_values( + compute_node, 'intel_rdt', 'Cores')], 'hugepages': [ - 'hugepages-mm-2048Kb', 'hugepages-node0-2048Kb', 'hugepages-node1-2048Kb', - 'hugepages-mm-1048576Kb', 'hugepages-node0-1048576Kb', 'hugepages-node1-1048576Kb'], - 'mcelog': ['mcelog-SOCKET_0_CHANNEL_0_DIMM_any', 'mcelog-SOCKET_0_CHANNEL_any_DIMM_any'], + 'hugepages-mm-2048Kb', 'hugepages-node0-2048Kb', + 'hugepages-node1-2048Kb', 'hugepages-mm-1048576Kb', + 'hugepages-node0-1048576Kb', 'hugepages-node1-1048576Kb'], + 'ipmi': ['ipmi'], + 'mcelog': [ + 'mcelog-SOCKET_0_CHANNEL_0_DIMM_any', + 'mcelog-SOCKET_0_CHANNEL_any_DIMM_any'], + 'ovs_stats': [ + 'ovs_stats-{0}.{0}'.format(interface) + for interface in ovs_existing_configured_bridges], 'ovs_events': [ 'ovs_events-{}'.format(interface) - for interface in (ovs_existing_configured_int if len(ovs_existing_configured_int) > 0 else ovs_interfaces)]} + for interface in ( + ovs_existing_configured_int + if len(ovs_existing_configured_int) > 0 else ovs_interfaces)]} + csv_meter_categories_ipmi = get_csv_categories_for_ipmi(conf, compute_node) csv_meter_categories = { + 'intel_rdt': [ + 'bytes-llc', 'ipc', 'memory_bandwidth-local', + 'memory_bandwidth-remote'], 'hugepages': ['vmpage_number-free', 'vmpage_number-used'], + 'ipmi': csv_meter_categories_ipmi, 'mcelog': [ - 'errors-corrected_memory_errors', 'errors-uncorrected_memory_errors', - 'errors-corrected_memory_errors_in_24h', 'errors-uncorrected_memory_errors_in_24h'], + 'errors-corrected_memory_errors', + 'errors-uncorrected_memory_errors', + 'errors-corrected_memory_errors_in_24h', + 'errors-uncorrected_memory_errors_in_24h'], + 'ovs_stats': [ + 'if_collisions', 'if_dropped', 'if_errors', 'if_packets', + 'if_rx_errors-crc', 'if_rx_errors-frame', 'if_rx_errors-over', + 'if_rx_octets', 'if_tx_octets'], 'ovs_events': ['gauge-link_status']} - _print_plugin_label(test_labels[name] if name in test_labels else name, compute_node.get_id()) + _print_plugin_label( + test_labels[name] if name in test_labels else name, + compute_node.get_name()) plugin_critical_errors = [ - error for plugin, error, critical in error_plugins if plugin == name and critical] + error for plugin, error, critical in error_plugins + if plugin == name and critical] if plugin_critical_errors: logger.error('Following critical errors occurred:'.format(name)) for error in plugin_critical_errors: logger.error(' * ' + error) - _process_result(compute_node.get_id(), test_labels[name], False, results) + _process_result( + compute_node.get_id(), test_labels[name], False, results) else: plugin_errors = [ - error for plugin, error, critical in error_plugins if plugin == name and not critical] + error for plugin, error, critical in error_plugins + if plugin == name and not critical] if plugin_errors: logger.warning('Following non-critical errors occured:') for error in plugin_errors: @@ -370,7 +451,8 @@ def _exec_testcase( if name in plugin_prerequisites: failed_prerequisites = [ prerequisite_name for prerequisite_passed, - prerequisite_name in plugin_prerequisites[name] if not prerequisite_passed] + prerequisite_name in plugin_prerequisites[name] + if not prerequisite_passed] if failed_prerequisites: logger.error( '{} test will not be executed, '.format(name) @@ -378,86 +460,102 @@ def _exec_testcase( for prerequisite in failed_prerequisites: logger.error(' * {}'.format(prerequisite)) else: - if ceilometer_running: - res = test_ceilometer_node_sends_data( - compute_node.get_id(), conf.get_plugin_interval(compute_node, name), - logger=logger, client=CeilometerClient(logger), + if gnocchi_running: + res = conf.test_plugins_with_gnocchi( + compute_node.get_id(), + conf.get_plugin_interval(compute_node, name), + logger, client=GnocchiClient(), criteria_list=ceilometer_criteria_lists[name], - resource_id_substrings=(ceilometer_substr_lists[name] - if name in ceilometer_substr_lists else [''])) + resource_id_substrings=( + ceilometer_substr_lists[name] + if name in ceilometer_substr_lists else [''])) else: - res = test_csv_handles_plugin_data( - compute_node, conf.get_plugin_interval(compute_node, name), name, - csv_subdirs[name], csv_meter_categories[name], logger, - CSVClient(logger, conf)) + res = tests.test_csv_handles_plugin_data( + compute_node, conf.get_plugin_interval(compute_node, name), + name, csv_subdirs[name], csv_meter_categories[name], + logger, CSVClient(conf)) if res and plugin_errors: logger.info( 'Test works, but will be reported as failure,' + 'because of non-critical errors.') res = False - _process_result(compute_node.get_id(), test_labels[name], res, results) - + _process_result( + compute_node.get_id(), test_labels[name], res, results) -def mcelog_install(logger): - """Install mcelog on compute nodes. - Keyword arguments: - logger - logger instance +def get_results_for_ovs_events( + plugin_labels, plugin_name, gnocchi_running, + compute_node, conf, results, error_plugins): + """ Testing OVS Events with python plugin """ + plugin_label = 'OVS events' + res = conf.enable_ovs_events( + compute_node, plugin_label, error_plugins, create_backup=False) + _process_result( + compute_node.get_id(), plugin_label, res, results) + logger.info("Results for OVS Events = {}" .format(results)) + + +def mcelog_install(): + """Install mcelog on compute nodes.""" _print_label('Enabling mcelog on compute nodes') - handler = factory.Factory.get_handler('fuel', FUEL_IP, FUEL_USER, installer_pwd='') + handler = factory.Factory.get_handler('apex', + APEX_IP, + APEX_USER_STACK, + APEX_PKEY) nodes = handler.get_nodes() - openstack_version = handler.get_openstack_version() - if openstack_version.find('14.') != 0: - logger.info('Mcelog will not be installed,' - + ' unsupported Openstack version found ({}).'.format(openstack_version)) - else: - for node in nodes: - if node.is_compute(): - ubuntu_release = node.run_cmd('lsb_release -r') - if '16.04' not in ubuntu_release: - logger.info('Mcelog will not be enabled' - + 'on node-{0}, unsupported Ubuntu release found ({1}).'.format( - node.get_dict()['id'], ubuntu_release)) - else: - logger.info('Checking if mcelog is enabled on node-{}...'.format( + for node in nodes: + if node.is_compute(): + centos_release = node.run_cmd('uname -r') + if '3.10.0-514.26.2.el7.x86_64' not in centos_release: + logger.info( + 'Mcelog will not be enabled ' + + 'on node-{0}, '.format(node.get_dict()['id']) + + 'unsupported CentOS release found ({1}).'.format( + centos_release)) + else: + logger.info( + 'Checking if mcelog is enabled' + + ' on node-{}...'.format(node.get_dict()['id'])) + res = node.run_cmd('ls') + if 'mce-inject_ea' and 'corrected' in res: + logger.info( + 'Mcelog seems to be already installed ' + + 'on node-{}.'.format(node.get_dict()['id'])) + node.run_cmd('modprobe mce-inject_ea') + node.run_cmd('mce-inject_ea < corrected') + else: + logger.info( + 'Mcelog will be enabled on node-{}...'.format( node.get_dict()['id'])) - res = node.run_cmd('ls /root/') - if 'mce-inject_df' and 'corrected' in res: - logger.info('Mcelog seems to be already installed on node-{}.'.format( - node.get_dict()['id'])) - res = node.run_cmd('modprobe mce-inject') - res = node.run_cmd('/root/mce-inject_df < /root/corrected') - else: - logger.info('Mcelog will be enabled on node-{}...'.format( - node.get_dict()['id'])) - res = node.put_file('/home/opnfv/repos/barometer/baro_utils/mce-inject_df', - '/root/mce-inject_df') - res = node.run_cmd('chmod a+x /root/mce-inject_df') - res = node.run_cmd('echo "CPU 0 BANK 0" > /root/corrected') - res = node.run_cmd('echo "STATUS 0xcc00008000010090" >> /root/corrected') - res = node.run_cmd('echo "ADDR 0x0010FFFFFFF" >> /root/corrected') - res = node.run_cmd('modprobe mce-inject') - res = node.run_cmd('/root/mce-inject_df < /root/corrected') - logger.info('Mcelog is installed on all compute nodes') - - -def mcelog_delete(logger): - """Uninstall mcelog from compute nodes. - - Keyword arguments: - logger - logger instance - """ - handler = factory.Factory.get_handler('fuel', FUEL_IP, FUEL_USER, installer_pwd='') + node.put_file( + '/usr/local/lib/python2.7/dist-packages/baro_tests/' + + 'mce-inject_ea', 'mce-inject_ea') + node.run_cmd('chmod a+x mce-inject_ea') + node.run_cmd('echo "CPU 0 BANK 0" > corrected') + node.run_cmd( + 'echo "STATUS 0xcc00008000010090" >>' + + ' corrected') + node.run_cmd( + 'echo "ADDR 0x0010FFFFFFF" >> corrected') + node.run_cmd('modprobe mce-inject') + node.run_cmd('mce-inject_ea < corrected') + logger.info('Mcelog is installed on all compute nodes') + + +def mcelog_delete(): + """Uninstall mcelog from compute nodes.""" + handler = factory.Factory.get_handler( + 'apex', APEX_IP, APEX_USER, APEX_PKEY) nodes = handler.get_nodes() for node in nodes: if node.is_compute(): - output = node.run_cmd('ls /root/') - if 'mce-inject_df' in output: - res = node.run_cmd('rm /root/mce-inject_df') + output = node.run_cmd('ls') + if 'mce-inject_ea' in output: + node.run_cmd('rm mce-inject_ea') if 'corrected' in output: - res = node.run_cmd('rm /root/corrected') - res = node.run_cmd('systemctl restart mcelog') + node.run_cmd('rm corrected') + node.run_cmd('systemctl restart mcelog') logger.info('Mcelog is deleted from all compute nodes') @@ -465,16 +563,26 @@ def get_ssh_keys(): if not os.path.isdir(ID_RSA_DST_DIR): os.makedirs(ID_RSA_DST_DIR) if not os.path.isfile(ID_RSA_DST): - logger.info("RSA key file {} doesn't exist, it will be downloaded from installer node.".format(ID_RSA_DST)) - handler = factory.Factory.get_handler('fuel', FUEL_IP, FUEL_USER, installer_pwd=FUEL_PW) - fuel = handler.get_installer_node() - fuel.get_file(ID_RSA_SRC, ID_RSA_DST) + logger.info( + "RSA key file {} doesn't exist".format(ID_RSA_DST) + + ", it will be downloaded from installer node.") + handler = factory.Factory.get_handler( + 'apex', APEX_IP, APEX_USER, APEX_PKEY) + apex = handler.get_installer_node() + apex.get_file(ID_RSA_SRC, ID_RSA_DST) else: logger.info("RSA key file {} exists.".format(ID_RSA_DST)) +def _check_logger(): + """Check whether there is global logger available and if not, define one.""" + if 'logger' not in globals(): + global logger + logger = logger.Logger("barometercollectd").getLogger() + + def main(bt_logger=None): - """Check each compute node sends ceilometer metrics. + """Check each compute node sends gnocchi metrics. Keyword arguments: bt_logger -- logger instance @@ -487,8 +595,9 @@ def main(bt_logger=None): else: global logger logger = bt_logger + _print_label("Starting barometer tests suite") get_ssh_keys() - conf = ConfigServer(FUEL_IP, FUEL_USER, logger) + conf = config_server.ConfigServer(APEX_IP, APEX_USER, logger) controllers = conf.get_controllers() if len(controllers) == 0: logger.error('No controller nodes found!') @@ -498,89 +607,120 @@ def main(bt_logger=None): logger.error('No compute nodes found!') return 1 - _print_label('Display of Control and Compute nodes available in the set up') + _print_label( + 'Display of Control and Compute nodes available in the set up') logger.info('controllers: {}'.format([('{0}: {1} ({2})'.format( - node.get_id(), node.get_name(), node.get_ip())) for node in controllers])) + node.get_id(), node.get_name(), + node.get_ip())) for node in controllers])) logger.info('computes: {}'.format([('{0}: {1} ({2})'.format( - node.get_id(), node.get_name(), node.get_ip())) for node in computes])) + node.get_id(), node.get_name(), node.get_ip())) + for node in computes])) - mcelog_install(logger) # installation of mcelog + mcelog_install() + gnocchi_running_on_con = False + _print_label('Test Gnocchi on controller nodes') - ceilometer_running_on_con = False - _print_label('Test Ceilometer on control nodes') for controller in controllers: - ceil_client = CeilometerClient(logger) - ceil_client.auth_token() - ceilometer_running_on_con = ( - ceilometer_running_on_con or conf.is_ceilometer_running(controller)) - if ceilometer_running_on_con: - logger.info("Ceilometer is running on control node.") + logger.info("Controller = {}" .format(controller)) + gnocchi_client = GnocchiClient() + gnocchi_client.auth_token() + gnocchi_running_on_con = ( + gnocchi_running_on_con or conf.is_gnocchi_running( + controller)) + if gnocchi_running_on_con: + logger.info("Gnocchi is running on controller.") else: - logger.error("Ceilometer is not running on control node.") + logger.error("Gnocchi is not running on controller.") logger.info("CSV will be enabled on compute nodes.") + compute_ids = [] + compute_node_names = [] results = [] plugin_labels = { + 'intel_rdt': 'Intel RDT', 'hugepages': 'Hugepages', + # 'ipmi': 'IPMI', 'mcelog': 'Mcelog', + 'ovs_stats': 'OVS stats', 'ovs_events': 'OVS events'} out_plugins = {} for compute_node in computes: node_id = compute_node.get_id() + node_name = compute_node.get_name() out_plugins[node_id] = 'CSV' compute_ids.append(node_id) - # plugins_to_enable = plugin_labels.keys() + compute_node_names.append(node_name) plugins_to_enable = [] - _print_label('NODE {}: Test Ceilometer Plug-in'.format(node_id)) - logger.info('Checking if ceilometer plug-in is included.') - if not conf.check_ceil_plugin_included(compute_node): - logger.error('Ceilometer plug-in is not included.') - logger.info('Testcases on node {} will not be executed'.format(node_id)) + _print_label('NODE {}: Test Gnocchi Plug-in'.format(node_name)) + logger.info('Checking if gnocchi plug-in is included in compute nodes.') + if not conf.check_gnocchi_plugin_included(compute_node): + logger.error('Gnocchi plug-in is not included.') + logger.info( + 'Testcases on node {} will not be executed'.format(node_name)) else: - collectd_restarted, collectd_warnings = conf.restart_collectd(compute_node) - sleep_time = 30 - logger.info('Sleeping for {} seconds after collectd restart...'.format(sleep_time)) + collectd_restarted, collectd_warnings = \ + conf.restart_collectd(compute_node) + sleep_time = 5 + logger.info( + 'Sleeping for {} seconds after collectd restart...'.format( + sleep_time)) time.sleep(sleep_time) if not collectd_restarted: for warning in collectd_warnings: logger.warning(warning) - logger.error('Restart of collectd on node {} failed'.format(node_id)) - logger.info('Testcases on node {} will not be executed'.format(node_id)) + logger.error( + 'Restart of collectd on node {} failed'.format(node_name)) + logger.info( + 'Testcases on node {} will not be executed'.format( + node_name)) else: for warning in collectd_warnings: logger.warning(warning) - ceilometer_running = ( - ceilometer_running_on_con and test_ceilometer_node_sends_data( - node_id, 10, logger=logger, client=CeilometerClient(logger))) - if ceilometer_running: - out_plugins[node_id] = 'Ceilometer' - logger.info("Ceilometer is running.") + gnocchi_running = ( + gnocchi_running_on_con + and conf.test_gnocchi_is_sending_data( + controller)) + if gnocchi_running: + out_plugins[node_id] = 'Gnocchi' + logger.info("Gnocchi is active and collecting data") else: plugins_to_enable.append('csv') out_plugins[node_id] = 'CSV' - logger.error("Ceilometer is not running.") - logger.info("CSV will be enabled for verification of test plugins.") + logger.error("Gnocchi is inactive and not collecting data") + logger.info( + "CSV will be enabled for verification " + + "of test plugins.") if plugins_to_enable: _print_label( - 'NODE {}: Enabling Test Plug-in '.format(node_id) + 'NODE {}: Enabling Test Plug-in '.format(node_name) + 'and Test case execution') error_plugins = [] if plugins_to_enable and not conf.enable_plugins( - compute_node, plugins_to_enable, error_plugins, create_backup=False): - logger.error('Failed to test plugins on node {}.'.format(node_id)) - logger.info('Testcases on node {} will not be executed'.format(node_id)) + compute_node, plugins_to_enable, error_plugins, + create_backup=False): + logger.error( + 'Failed to test plugins on node {}.'.format(node_id)) + logger.info( + 'Testcases on node {} will not be executed'.format( + node_id)) else: if plugins_to_enable: - collectd_restarted, collectd_warnings = conf.restart_collectd(compute_node) + collectd_restarted, collectd_warnings = \ + conf.restart_collectd(compute_node) sleep_time = 30 logger.info( - 'Sleeping for {} seconds after collectd restart...'.format(sleep_time)) + 'Sleeping for {} seconds'.format(sleep_time) + + ' after collectd restart...') time.sleep(sleep_time) if plugins_to_enable and not collectd_restarted: for warning in collectd_warnings: logger.warning(warning) - logger.error('Restart of collectd on node {} failed'.format(node_id)) - logger.info('Testcases on node {} will not be executed'.format(node_id)) + logger.error( + 'Restart of collectd on node {} failed'.format( + node_id)) + logger.info( + 'Testcases on node {}'.format(node_id) + + ' will not be executed.') else: if collectd_warnings: for warning in collectd_warnings: @@ -588,14 +728,13 @@ def main(bt_logger=None): for plugin_name in sorted(plugin_labels.keys()): _exec_testcase( - plugin_labels, plugin_name, ceilometer_running, + plugin_labels, plugin_name, + gnocchi_running, compute_node, conf, results, error_plugins) - _print_label('NODE {}: Restoring config file'.format(node_id)) + _print_label('NODE {}: Restoring config file'.format(node_name)) conf.restore_config(compute_node) - - mcelog_delete(logger) # uninstalling mcelog from compute nodes - + mcelog_delete() print_overall_summary(compute_ids, plugin_labels, results, out_plugins) if ((len([res for res in results if not res[2]]) > 0) diff --git a/baro_tests/config_server.py b/baro_tests/config_server.py index 358a8ffe..efe2691a 100644 --- a/baro_tests/config_server.py +++ b/baro_tests/config_server.py @@ -1,7 +1,6 @@ -"""Classes used by client.py""" # -*- coding: utf-8 -*- -#Licensed under the Apache License, Version 2.0 (the "License"); you may +# Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at # @@ -13,31 +12,33 @@ # License for the specific language governing permissions and limitations # under the License. +"""Classes used by collectd.py""" + import paramiko import time import string import os.path - +import os +import re ID_RSA_PATH = '/home/opnfv/.ssh/id_rsa' SSH_KEYS_SCRIPT = '/home/opnfv/barometer/baro_utils/get_ssh_keys.sh' DEF_PLUGIN_INTERVAL = 10 -COLLECTD_CONF = '/etc/collectd/collectd.conf' +COLLECTD_CONF = '/etc/collectd.conf' COLLECTD_CONF_DIR = '/etc/collectd/collectd.conf.d' +NOTIFICATION_FILE = '/var/log/python-notifications.dump' +COLLECTD_NOTIFICATION = '/etc/collectd_notification_dump.py' class Node(object): """Node configuration class""" def __init__(self, attrs): - self.__id = int(attrs[0]) - self.__status = attrs[1] + self.__null = attrs[0] + self.__id = attrs[1] self.__name = attrs[2] - self.__cluster = int(attrs[3]) if attrs[3] else None - self.__ip = attrs[4] - self.__mac = attrs[5] - self.__roles = [x.strip(' ') for x in attrs[6].split(',')] - self.__pending_roles = attrs[7] - self.__online = int(attrs[8]) if attrs[3] and attrs[8]else None - self.__group_id = int(attrs[9]) if attrs[3] else None + self.__status = attrs[3] if attrs[3] else None + self.__taskState = attrs[4] + self.__pwrState = attrs[5] + self.__ip = re.sub('^[a-z]+=', '', attrs[6]) def get_name(self): """Get node name""" @@ -52,68 +53,84 @@ class Node(object): return self.__ip def get_roles(self): - """Get node roles""" + """Get node role""" return self.__roles class ConfigServer(object): """Class to get env configuration""" - def __init__(self, host, user, logger, passwd=None): + def __init__(self, host, user, logger, priv_key=None): self.__host = host self.__user = user - self.__passwd = passwd - self.__priv_key = None + self.__passwd = None + self.__priv_key = priv_key self.__nodes = list() self.__logger = logger self.__private_key_file = ID_RSA_PATH if not os.path.isfile(self.__private_key_file): self.__logger.error( - "Private key file '{}' not found.".format(self.__private_key_file)) - raise IOError("Private key file '{}' not found.".format(self.__private_key_file)) + "Private key file '{}'".format(self.__private_key_file) + + " not found.") + raise IOError("Private key file '{}' not found.".format( + self.__private_key_file)) # get list of available nodes - ssh, sftp = self.__open_sftp_session(self.__host, self.__user, self.__passwd) + ssh, sftp = self.__open_sftp_session( + self.__host, self.__user, self.__passwd) attempt = 1 fuel_node_passed = False while (attempt <= 10) and not fuel_node_passed: - stdin, stdout, stderr = ssh.exec_command("fuel node") + stdin, stdout, stderr = ssh.exec_command( + "source stackrc; nova list") stderr_lines = stderr.readlines() if stderr_lines: - self.__logger.warning("'fuel node' command failed (try {}):".format(attempt)) + self.__logger.warning( + "'fuel node' command failed (try {}):".format(attempt)) for line in stderr_lines: self.__logger.debug(line.strip()) else: fuel_node_passed = True if attempt > 1: - self.__logger.info("'fuel node' command passed (try {})".format(attempt)) + self.__logger.info( + "'fuel node' command passed (try {})".format(attempt)) attempt += 1 if not fuel_node_passed: - self.__logger.error("'fuel node' command failed. This was the last try.") - raise OSError("'fuel node' command failed. This was the last try.") + self.__logger.error( + "'fuel node' command failed. This was the last try.") + raise OSError( + "'fuel node' command failed. This was the last try.") node_table = stdout.readlines()\ # skip table title and parse table values - for entry in node_table[2:]: - self.__nodes.append(Node([str(x.strip(' \n')) for x in entry.split('|')])) + + for entry in node_table[3:]: + if entry[0] == '+' or entry[0] == '\n': + print entry + pass + else: + self.__nodes.append( + Node([str(x.strip(' \n')) for x in entry.split('|')])) def get_controllers(self): - """Get list of controllers""" - return [node for node in self.__nodes if 'controller' in node.get_roles()] + # Get list of controllers + print self.__nodes[0]._Node__ip + return ( + [node for node in self.__nodes if 'controller' in node.get_name()]) def get_computes(self): - """Get list of computes""" - return [node for node in self.__nodes if 'compute' in node.get_roles()] + # Get list of computes + return ( + [node for node in self.__nodes if 'compute' in node.get_name()]) def get_nodes(self): - """Get list of nodes""" + # Get list of nodes return self.__nodes def __open_sftp_session(self, host, user, passwd=None): - """Connect to given host. - - Keyword arguments: + # Connect to given host. + """Keyword arguments: host -- host to connect user -- user to use passwd -- password to use @@ -127,10 +144,12 @@ class ConfigServer(object): # try a direct access using password or private key if not passwd and not self.__priv_key: # get private key - self.__priv_key = paramiko.RSAKey.from_private_key_file(self.__private_key_file) + self.__priv_key = paramiko.RSAKey.from_private_key_file( + self.__private_key_file) # connect to the server - ssh.connect(host, username=user, password=passwd, pkey=self.__priv_key) + ssh.connect( + host, username=user, password=passwd, pkey=self.__priv_key) sftp = ssh.open_sftp() # return SFTP client instance @@ -144,12 +163,14 @@ class ConfigServer(object): plugin -- plug-in name If found, return interval value, otherwise the default value""" - ssh, sftp = self.__open_sftp_session(compute.get_ip(), 'root') + ssh, sftp = self.__open_sftp_session( + compute.get_ip(), 'root', 'opnfvapex') in_plugin = False plugin_name = '' default_interval = DEF_PLUGIN_INTERVAL - config_files = [COLLECTD_CONF] \ - + [COLLECTD_CONF_DIR + '/' + conf_file for conf_file in sftp.listdir(COLLECTD_CONF_DIR)] + config_files = [COLLECTD_CONF] + [ + COLLECTD_CONF_DIR + '/' + + conf_file for conf_file in sftp.listdir(COLLECTD_CONF_DIR)] for config_file in config_files: try: with sftp.open(config_file) as config: @@ -178,13 +199,15 @@ class ConfigServer(object): parameter -- plug-in parameter Return list of found values.""" - ssh, sftp = self.__open_sftp_session(compute.get_ip(), 'root') + ssh, sftp = self.__open_sftp_session( + compute.get_ip(), 'root', 'opnfvapex') # find the plugin value in_plugin = False plugin_name = '' default_values = [] - config_files = [COLLECTD_CONF] \ - + [COLLECTD_CONF_DIR + '/' + conf_file for conf_file in sftp.listdir(COLLECTD_CONF_DIR)] + config_files = [COLLECTD_CONF] + [ + COLLECTD_CONF_DIR + '/' + + conf_file for conf_file in sftp.listdir(COLLECTD_CONF_DIR)] for config_file in config_files: try: with sftp.open(config_file) as config: @@ -210,12 +233,13 @@ class ConfigServer(object): host_ip -- IP of the node ssh -- existing open SSH session to use - One of host_ip or ssh must not be None. If both are not None, existing ssh session is used. + One of host_ip or ssh must not be None. If both are not None, + existing ssh session is used. """ if host_ip is None and ssh is None: raise ValueError('One of host_ip or ssh must not be None.') if ssh is None: - ssh, sftp = self.__open_sftp_session(host_ip, 'root') + ssh, sftp = self.__open_sftp_session(host_ip, 'root', 'opnfvapex') stdin, stdout, stderr = ssh.exec_command(command) return stdout.readlines() @@ -228,23 +252,22 @@ class ConfigServer(object): stdout = self.execute_command("ovs-vsctl list-br", compute.get_ip()) return [interface.strip() for interface in stdout] - def is_ceilometer_running(self, controller): - """Check whether Ceilometer is running on controller. + def is_gnocchi_running(self, controller): + """Check whether Gnocchi is running on controller. Keyword arguments: controller -- controller node instance - Return boolean value whether Ceilometer is running. + Return boolean value whether Gnocchi is running. """ - lines = self.execute_command('service --status-all | grep ceilometer', controller.get_ip()) - agent = False - collector = False + gnocchi_present = False + lines = self.execute_command( + 'source overcloudrc.v3;openstack service list | grep gnocchi', + controller.get_ip()) for line in lines: - if '[ + ] ceilometer-agent-notification' in line: - agent = True - if '[ + ] ceilometer-collector' in line: - collector = True - return agent and collector + if 'gnocchi' in line: + gnocchi_present = True + return not gnocchi_present def is_installed(self, compute, package): """Check whether package exists on compute node. @@ -255,36 +278,101 @@ class ConfigServer(object): Return boolean value whether package is installed. """ - stdout = self.execute_command('dpkg -l | grep {}'.format(package), compute.get_ip()) + stdout = self.execute_command( + 'yum list installed | grep {}'.format(package), + compute.get_ip()) return len(stdout) > 0 + def is_libpqos_on_node(self, compute): + """Check whether libpqos is present on compute node""" + ssh, sftp = self.__open_sftp_session( + compute.get_ip(), 'root', 'opnfvapex') + stdin, stdout, stderr = \ + ssh.exec_command("ls /usr/local/lib/ | grep libpqos") + output = stdout.readlines() + for lib in output: + if 'libpqos' in lib: + return True + return False + + def check_gnocchi_plugin_included(self, compute): + """Check if gnocchi plugin is included in collectd.conf file. + If not, try to enable it. + + Keyword arguments: + compute -- compute node instance + + Return boolean value whether gnocchi plugin is included + or it's enabling was successful. + """ + ssh, sftp = self.__open_sftp_session( + compute.get_ip(), 'root', 'opnfvapex') + try: + config = sftp.open(COLLECTD_CONF, mode='r') + except IOError: + self.__logger.error( + 'Cannot open {} on node {}'.format( + COLLECTD_CONF, compute.get_name())) + return False + in_lines = config.readlines() + out_lines = in_lines[:] + include_section_indexes = [ + (start, end) for start in range(len(in_lines)) + for end in range(len(in_lines)) + if (start < end) + and '' in in_lines[end] + and '#' not in in_lines[end] + and len([ + i for i in in_lines[start + 1: end] + if 'Filter' in i and '*.conf' in i and '#' not in i]) > 0] + if len(include_section_indexes) == 0: + out_lines.append('\n'.format(COLLECTD_CONF_DIR)) + out_lines.append(' Filter "*.conf"\n') + out_lines.append('\n') + config.close() + config = sftp.open(COLLECTD_CONF, mode='w') + config.writelines(out_lines) + config.close() + self.__logger.info('Creating backup of collectd.conf...') + config = sftp.open(COLLECTD_CONF + '.backup', mode='w') + config.writelines(in_lines) + config.close() + return True + def check_ceil_plugin_included(self, compute): - """Check if ceilometer plugin is included in collectd.conf file If not, - try to enable it. + """Check if ceilometer plugin is included in collectd.conf file. + If not, try to enable it. Keyword arguments: compute -- compute node instance - Return boolean value whether ceilometer plugin is included or it's enabling was successful. + Return boolean value whether ceilometer plugin is included + or it's enabling was successful. """ ssh, sftp = self.__open_sftp_session(compute.get_ip(), 'root') try: config = sftp.open(COLLECTD_CONF, mode='r') except IOError: self.__logger.error( - 'Cannot open {} on node {}'.format(COLLECTD_CONF, compute.get_id())) + 'Cannot open {} on node {}'.format( + COLLECTD_CONF, compute.get_id())) return False in_lines = config.readlines() out_lines = in_lines[:] include_section_indexes = [ - (start, end) for start in range(len(in_lines)) for end in range(len(in_lines)) + (start, end) for start in range(len(in_lines)) + for end in range(len(in_lines)) if (start < end) and '' in in_lines[end] and '#' not in in_lines[end] - and len([i for i in in_lines[start + 1: end] + and len([ + i for i in in_lines[start + 1: end] if 'Filter' in i and '*.conf' in i and '#' not in i]) > 0] if len(include_section_indexes) == 0: out_lines.append('\n'.format(COLLECTD_CONF_DIR)) @@ -300,41 +388,50 @@ class ConfigServer(object): config.close() return True - def enable_plugins(self, compute, plugins, error_plugins, create_backup=True): + def enable_plugins( + self, compute, plugins, error_plugins, create_backup=True): """Enable plugins on compute node Keyword arguments: compute -- compute node instance plugins -- list of plugins to be enabled - error_plugins -- list of tuples with found errors, new entries may be added there - (plugin, error_description, is_critical): + error_plugins -- list of tuples with found errors, new entries + may be added there (plugin, error_description, is_critical): plugin -- plug-in name error_decription -- description of the error - is_critical -- boolean value indicating whether error is critical - create_backup -- boolean value indicating whether backup shall be created + is_critical -- boolean value indicating whether error + is critical + create_backup -- boolean value indicating whether backup + shall be created Return boolean value indicating whether function was successful. """ plugins = sorted(plugins) - ssh, sftp = self.__open_sftp_session(compute.get_ip(), 'root') + ssh, sftp = self.__open_sftp_session( + compute.get_ip(), 'root', 'opnfvapex') plugins_to_enable = plugins[:] for plugin in plugins: - plugin_file = '/usr/lib/collectd/{}.so'.format(plugin) + plugin_file = '/usr/lib64/collectd/{}.so'.format(plugin) try: sftp.stat(plugin_file) except IOError: self.__logger.debug( - 'Plugin file {0} not found on node {1}, plugin {2} will not be enabled'.format( - plugin_file, compute.get_id(), plugin)) - error_plugins.append((plugin, 'plugin file {} not found'.format(plugin_file), True)) + 'Plugin file {} not found on node'.format(plugin_file) + + ' {0}, plugin {1} will not be enabled'.format( + compute.get_name(), plugin)) + error_plugins.append(( + plugin, 'plugin file {} not found'.format(plugin_file), + True)) plugins_to_enable.remove(plugin) - self.__logger.debug('Following plugins will be enabled on node {}: {}'.format( - compute.get_id(), ', '.join(plugins_to_enable))) + self.__logger.debug( + 'Following plugins will be enabled on node {}: {}'.format( + compute.get_name(), ', '.join(plugins_to_enable))) try: config = sftp.open(COLLECTD_CONF, mode='r') except IOError: self.__logger.warning( - 'Cannot open {} on node {}'.format(COLLECTD_CONF, compute.get_id())) + 'Cannot open {} on node {}'.format( + COLLECTD_CONF, compute.get_name())) return False in_lines = config.readlines() out_lines = [] @@ -348,7 +445,8 @@ class ConfigServer(object): for plugin in plugins_to_enable: if plugin in line: commented = '#' in line - #list of uncommented lines which contain LoadPlugin for this plugin + # list of uncommented lines which contain LoadPlugin + # for this plugin loadlines = [ ll for ll in in_lines if 'LoadPlugin' in ll and plugin in ll and '#' not in ll] @@ -358,7 +456,8 @@ class ConfigServer(object): enabled_plugins.append(plugin) error_plugins.append(( plugin, 'plugin not enabled in ' - + '{}, trying to enable it'.format(COLLECTD_CONF), False)) + + '{}, trying to enable it'.format( + COLLECTD_CONF), False)) elif not commented: if plugin not in enabled_plugins: enabled_plugins.append(plugin) @@ -366,15 +465,16 @@ class ConfigServer(object): line = '#' + line error_plugins.append(( plugin, 'plugin enabled more than once ' - + '(additional occurrence of LoadPlugin found in ' - + '{}), trying to comment it out.'.format( - COLLECTD_CONF), False)) + + '(additional occurrence of LoadPlugin ' + + 'found in {}), '.format(COLLECTD_CONF) + + 'trying to comment it out.', False)) elif line.lstrip(string.whitespace + '#').find(' 0: if comment_section and '#' not in line: line = '#' + line @@ -411,8 +511,8 @@ class ConfigServer(object): elif '' in line: self.__logger.error( 'Unexpected closure os plugin section on line' - + ' {} in collectd.conf, matching section start not found.'.format( - len(out_lines) + 1)) + + ' {} in collectd.conf'.format(len(out_lines) + 1) + + ', matching section start not found.') return False out_lines.append(line) if in_section > 0: @@ -426,14 +526,14 @@ class ConfigServer(object): for plugin in plugins_to_enable: if plugin not in enabled_plugins: error_plugins.append(( - plugin, - 'plugin not enabled in {}, trying to enable it.'.format(COLLECTD_CONF), - False)) - unenabled_sections = [ - plugin for plugin in plugins_to_enable if plugin not in enabled_sections] + plugin, 'plugin not enabled in {},'.format(COLLECTD_CONF) + + ' trying to enable it.', False)) + unenabled_sections = [plugin for plugin in plugins_to_enable + if plugin not in enabled_sections] if unenabled_sections: - self.__logger.error('Plugin sections for following plugins not found: {}'.format( - ', '.join(unenabled_sections))) + self.__logger.error( + 'Plugin sections for following plugins not found: {}'.format( + ', '.join(unenabled_sections))) return False config.close() @@ -446,7 +546,8 @@ class ConfigServer(object): config = sftp.open(COLLECTD_CONF, mode='w') config.writelines(out_lines) config.close() - diff_command = "diff {} {}.backup".format(COLLECTD_CONF, COLLECTD_CONF) + diff_command = \ + "diff {} {}.backup".format(COLLECTD_CONF, COLLECTD_CONF) stdin, stdout, stderr = ssh.exec_command(diff_command) self.__logger.debug(diff_command) for line in stdout.readlines(): @@ -459,7 +560,8 @@ class ConfigServer(object): Keyword arguments: compute -- compute node instance """ - ssh, sftp = self.__open_sftp_session(compute.get_ip(), 'root') + ssh, sftp = self.__open_sftp_session( + compute.get_ip(), 'root', 'opnfvapex') self.__logger.info('Restoring config file from backup...') ssh.exec_command("cp {0} {0}.used".format(COLLECTD_CONF)) @@ -471,20 +573,23 @@ class ConfigServer(object): Keyword arguments: compute -- compute node instance - Retrun tuple with boolean indicating success and list of warnings received - during collectd start. + Retrun tuple with boolean indicating success and list of warnings + received during collectd start. """ def get_collectd_processes(ssh_session): """Get number of running collectd processes. Keyword arguments: - ssh_session -- instance of SSH session in which to check for processes + ssh_session -- instance of SSH session in which to check + for processes """ - stdin, stdout, stderr = ssh_session.exec_command("pgrep collectd") + stdin, stdout, stderr = ssh_session.exec_command( + "pgrep collectd") return len(stdout.readlines()) - ssh, sftp = self.__open_sftp_session(compute.get_ip(), 'root') + ssh, sftp = self.__open_sftp_session( + compute.get_ip(), 'root', 'opnfvapex') self.__logger.info('Stopping collectd service...') stdout = self.execute_command("service collectd stop", ssh=ssh) @@ -500,3 +605,50 @@ class ConfigServer(object): self.__logger.error('Collectd is still not running...') return False, warning return True, warning + + def test_gnocchi_is_sending_data(self, controller): + """ Checking if Gnocchi is sending metrics to controller""" + metric_ids = [] + timestamps1 = {} + timestamps2 = {} + ssh, sftp = self.__open_sftp_session( + controller.get_ip(), 'root', 'opnfvapex') + + self.__logger.info('Getting gnocchi metric list on{}'.format( + controller.get_name())) + stdout = self.execute_command( + "source overcloudrc.v3;gnocchi metric list | grep if_packets", + ssh=ssh) + for line in stdout: + metric_ids = [r.split('|')[1] for r in stdout] + self.__logger.info("Metric ids = {}" .format(metric_ids)) + for metric_id in metric_ids: + metric_id = metric_id.replace("u", "") + stdout = self.execute_command( + "source overcloudrc.v3;gnocchi measures show {}" .format( + metric_id), ssh=ssh) + self.__logger.info("stdout measures ={}" .format(stdout)) + for line in stdout: + if line[0] == '+': + pass + else: + self.__logger.info("Line = {}" .format(line)) + timestamps1 = [line.split('|')[1]] + self.__logger.info("Last line timetamp1 = {}" .format(timestamps1)) + time.sleep(10) + stdout = self.execute_command( + "source overcloudrc.v3;gnocchi measures show {}" .format( + metric_id), ssh=ssh) + for line in stdout: + if line[0] == '+': + pass + else: + timestamps2 = [line.split('|')[1]] + self.__logger.info("Last line timetamp2 = {}" .format(timestamps2)) + if timestamps1 == timestamps2: + self.__logger.info("False") + # return False + return True + else: + self.__logger.info("True") + return True diff --git a/baro_tests/mce-inject_ea b/baro_tests/mce-inject_ea new file mode 100755 index 0000000000000000000000000000000000000000..12fa1df219779e51fca69197fb4f9aa28d08cca6 GIT binary patch literal 75144 zcmeFa33wD$+BSTu(%q@-4V6Fw0UAgk0kX0sK%hes=s0i|i8 zxNpqJh`Ww5j^j2CiGqZ2bOs%F9cA3;7*SM2R3!g>pE{LvqI~c7{{Qz~@AY5T6j$mw z&w8HooVCuWE;f~vP0=+?*q1IQ3Sy7+@Jf<(7s8j^DdQD+B0_jYsu(0(kaFVBmh-BV z4NodH8yb-16^1_t{_=EUgkFLUm@F|NY-s0B$ox7luS#0*wJ8>ALm@npKt1{@W=a%r zv6WEp+;9uM)pk(n(LL%VD!oJ{Z^OehjLU|$I?Kktd8&N#bT2UCUII}p@12mcyvjeR zUCQ**0*L~8Wx|Fpsr+nc>&=FqET5(JU*6qHVvfr1OmPv)e;caoMb?VtOU8`KTCp@^ z#qw2k8!|TJjma1@Dzj#FX09wZ?NfH@%=t1WJB}V;^f?}XYy;$P+P>=8u*B4}Zk_Q^ z#HUYudf@Z;v-Xjffj`=i<nV zoIyVB4Dw6PKukWATo1utis#fb z$e(=%dLit1x@QV}?HS}h0`Hgd{TB2dB1tTszZfYY>~Vagv=bxBGQ5&*kDGr1M)_sk zP0J7S60+6U%g7Gd=^zAx7w->^w+DGuc8~3 zbYP6A+OWJ<(4VEt*NW=ZYpS3KL#x+{$`z|?sz?QQ1+0pi+NDrkUQTqDeA4Rs}S`&zhRbidEHAwJfZyU0t_gg{WS>qH5J@QN4ccaulhWev6B0RA#}S>lTjW*N5DF+g(39F4yq7~L+DW<^kpIR=n(pv5PD1qeM1O6 zHiTXuLXQifZw;Z_bxih-5IWZ$_H}CrJ#?Jg6+$;{VZi%C=vR^tLE;=5S`SJ5F5(;#TAL+)J8=#Lt$QSXJ#h{Jt-ByLziv;%5_Q7jMm$_#EQw z+N~amPb1DQ-RhS3Wa8|~twQ4Ch_ef~9y<*}YA$g%@xu}yMx0%?^^n8|5${R7S>k<& zo5c4>yeDyX&DLEKk0j16*}6mGI&pTz)_RHm;04Yu*t$mIUlUIxzF6X)5oeccEtmL* z#M#way%K+iIJ;PDw!~j0&aTz!k@$xr{#wC>wJcB*~@-`_U#Vd2?1YVqv2~TGJJmLv0^s%_S(UF z)5my4@FEm%OLJXF8HmsN0Yy(6O3P_rWoS5dPwn=-?&b2arHlbDy&>=V+<_mGgc64}A?EA1j|% z+R}Uo>r!9KbBiTtIp#*Lx1kir;|6Soa~Q8Lz(RDsFN5>Dui<2@&;QAK{k(lok;jgF z{JbWz+7HJKs3+|yVz&HqC&M7`^FQx39YaBbH=wyT{P?T&z5MU^0tF9D_K3YPnO<>DPIK#f zkUeML+#HDi_%!lChWqP%{v(LlHxC7-*ZL}-^A)rssOwGNb3qsUe@fQkKM(u-j^lgl zp^?)b90Dr=rx$J`M0(qCU!cYv*tjNmA)G=>sn2OXXP^IcAZ7uwh6hsta$e1U(SPW& zC|~>Y`L(y4s{5*~M|o2Tyvk_~-ryAC?1qEw`{uHl1)SlKZ$YIZGh`6AWgB4ALdXVJ z{)h}@S?i_-kNp5I@`CBIRTey7h`jvIJwv&@S6ha^fcR7G&~BUNYusS^8t0`%=QQuV zo2DkC4DAPN<9z-Pf-aU~4}8_5?k!(li&jp(=dvpMx42UvWwz6TyKplXrdo7YhCR>mr*Tq8CB%e3joe zIVkZM5x+CB&?9_}m!`OVjkPI>KL6X#aQF?*ww&|@z`1;!U6%c{vJTF*AFO-PSLyr+ z-MKjb@MU`&%e6pRO0+-W6;OPE1dc=g=quaq32yx!rA3k4h+jj&A3#TY^aV!OBh~a8 z?F|73#q^wWUg5d>%6*jujbQCXpA*4Y%f}TUoDEi>U0lAMoAa!%xDoScL?^qNQzF+zLccyP;q<^kA*VUE6!x z6z1keZd<41`vOJodSAZQHt@uh{II&d#s2ofy)0BPhm~t()B0U|d)o+K<3=OrM6rM4 znlk@jSgvI|c%vsO=WX!eWMtEitg8DEVPE?TrT+I?Kf`cZvgJf=!U^Xdl*XtHFRY)D z?{0GvI9iVCL_m?l-XR3(qS&zi`n$ zSzrDi1DlFv&yRcJ!a~^^^Iy1pV9*E8ri|zjFqJ?XrU^`b}UNX556O5qyGq6lN>j1`xebhHb>S+;jc<^+Bpst&MSQO zAdFxry5jR77Q`*inh72dn7Rllgs9!u2=}4uYdjZ@`ByND)gOd`Q{V<(_$=>baZ?VLUU(|f z*U%CkydCag&?)sFh}--G1ZTnU^u}{D5R4X|{|ldgM9Q?r5l87)X`?qB!@xIp`x*|P z^aX0^P3w%t5iW+zUFQE19Jbs1mfDyyzau61+i%dZ1CEr^#&i3hp59pII$hehBI6|8 zZCHT3%2=W@|EQG0hITz}^GuLR;`TmzIL9jr8-6+xw|OgaZ78TlEKk=(1$Q8xe&<QfGvOB6x-q~Mv7NXb6fls-;Y4Jo?@R|A0GF_71<+UN_zp;O4m=EQF*j5?uo{1 zn9qN%E~_N&iNDk%U4ni#5`E8rofhYbaVqo;FQ9lOm~Qv!t=mu;P`$c5*NYWFU3lvz zLGB+i2l;#-|-iI#M-ONo!}#0!b1D%|g! zMwG`e)b=|k5e@H@9!dN=RD;Uwn>^r0I`NNygLSMe4tKizszo@s)fuGdvz?;%lk*gC zUXHc|Wzio>HhdAzHPqv>ns&E3ZzEs&b^}svs~TeZz$dM(sv)Ka5=Ps(4KW51?zXuN zG3^l1+oo-EE=594&3aq(Hs=Bx)3!RN(MnoF%xi#-`qXg%-?b$+#O#-N%rF3>%_s}3 z6lGB>Pflxe{6v3eXzgX`Kl;*Nh-(_@^9Oy6E79PW8E7U~9Yb?z2#)mK8n<`rA+$s7 ztA_7O*Y`!kiF@K{*+{>uiz!B$jkL8E6Gd6v-ly`Q)p`MtcIQ-N+4LSpuQq2>D*)!c zgIGDSL;V>wA=>nucV&lU_hheh2BVLmpF#xZ*=dITFK_-T4VCI30pk*Ud)=!)kO`@3QvI-p^qzhF|OoTS>+*! z{$pxf)8?Lg^3d+JgQb;k#Wn5hyJsvq-@e+!BC^(i)o?QY@4HBd8sMCL=%gcw2NEU_G{ztD1$1A)=kW!+7#^K7C9@||$Hz#uZwUe*B>Wkv zZD{3ffUq;h7nsyK;7bI<{{vT%4v_cm!h!}d%tMTtID!QGf7RYTf9@RcpQWKK?R5p- zCyn3*3?iQbocDl?rSy5DyjVSRyl#P-z@poEEjX+LB)22UtE~)Ap&Mzav8dgb@7#do z+B}Rec%U8eFG^s%4UX$D?gPJeHzt;}=BM4@@}a5Uc&O~RI`+03Jd5&Vo?DLACi(nd27mQMd%M3Sm<^M79K5qR?ulk} zt!97Gh!V_>s0gl9u}UrUABO`!HTj@DH&_TQ*-G}IgBNVTFJaq4Z9vyI?1h6i65d zuvd5m(giP!1gUYvq2MBE55s=I`4wrZ=-)COm~aTiL3Zy4pF)kJJgqlyQc=~sHu$}q zViKN`im%pAMT>b>8g{%WH$~92MmK=q-Bz%jVCzI*AkXJtj~Xkep(w#z+IHqzJr z9QyO6lI*M_@FzbX9E2=doS8sb{Bx`&f1qCp`CHzto#6{)lp>9#ahts~&JWhG+*^*? ztK(YOe%a^G@K7n>Od@EQuus&+1@~cz-{Sm7t}@TZN?*fiZQZ4X^U4~lPqWtM)?YgI zw5VGN&@Sq#WhDkL`_$V|fzU6f+vG4rHOdvOTG1 z6<@<&k@XK>KY|g&_uVVL#*5nJu57^hdA1xDoj2xKaI=EZ2pNan}!4eGpv_B;4Kmv!Nos<$5slL z#>|#mDL#L}>ERwc-$F{(x$|ns)J3Bae@gmaFlYK2oEL*=D=n;*_3k^%{X0POC6fQpEHa4}=<4-5VoPpIfhZ-Bndc^LPt zB_iN#qL{xe5c3EVcvc07dtzkqHpibh!TTzYRX1P)RP*~k<4O*eRUX6&<|HQT$0Inu zYp2U9-@t5tV^7ZSh56506MTd{!T$rc{vs!Mtb;a>vF7)E$}kOh>aOv}WJx~~JOsCL z9l$Kno_k?m$&{}3bI>)#@%i6DZ$Dcq^$R0N{|qzzSx9X&4}^)W38yo$m^&FAclA}+ z;lmQ#3+J{ugP?8=!(99%5&`GaxVK%~?rhD#^6>*SNL08JHgaCY*2JkJ44m^QWS;7a zI&7_kCDg^oAjIv(^4UUP2P$`y3ZLS>_0x%{?Ps`ez1P~##vWYos~}9F`b=M-+#T>@ z^hmfFYOOtB;wiqq*BUO3jou1s({S`)tVK>@HPAXuIu&ya4q~Le80=@*oZ#nqNC&sV zaO)7t_}}skcb0;NO6&vBeh?3|SjL`!CQI!6mL{xaQ^N+Ys2d;`PyU#frRkWP3H&%n z(p$WkhM5JHR|SuPTo%aKC56^twGutCH2-C;CqRk2dK(2R&&4LV_BX6hE(S69FrO+b z524-klQ*(g$yOPa+!bhWl(IpH2C+A=+XhT zC=MiCCCxfl17N=o`J^HxHz4U~ng0i0*ajo`7PPDZcnsKeqv7XI!{Fx*(9wTV3mYan zu>JTDYuCjoFHrC=@C#db%82uok1?lQkMJj2xoHw$SsF!Ws3mp~PHE31^gDCZV?`Q<#c7xkg5e+9J8GAqNz4vj02~db&KZ0S&Ef2Qv#mYt@(%J} zB)L2o3vp{Jt{5qvcgb^t!Ud)N6FeDsoy)=0tsW1K*cvkk=a!{^f`{nmehR}8ixIr( zFmpOu+uuIFEAxL==EpX9dtrP0dw7)7eEDs4pTT=0#QPo9nYR2l`>J=)5+(MbLim1A z=vBJv*Hx9Z%U50CSy@@-S-r}WwX|wo7SbzLUmyT7mP>(gp1I5La+MU4uUvUjmyYvX zShaRl)e6tj)m1f~Rja|RsI6Q!DN7|ERvuL%;Dpq-F>ej45P^y+PZnevn*GThesY}x%yEqpT(W3JCP>&Ro3ujK3 ziFtF2JlPvk%Qi%cd8IQ-JgH@Hcur|KiP^au#wp&6(oVU;;^H~B&Wxg|bLa86!X}i@ znN?IWckZk?C^?lkL`Dk4jkSf>+E&$8Z17kwyREERxq9uzo~jMYD(X-hOI4$Iv#(dC zK2=a8TX=5;2P*hH&G>{~y(yPDaM0lXnFEqN$(h;75jp*mbCWasS{C(; z%&g+1GKaq~>w6j<5#3M(NjvVRR8csq&H19`} z<-PM3lu1v&BZH3gaHsanADSND=g4r&a=Yg2o19uWV7$u520BI@mvHcX3?V#y8LfUY z-G4xs;S`ZQV-k8rM+wF!LS$xTXAB#jmOerje5gnrG&p6T5T@uYRd|aRcq$6L@WQ1Y zrbTOd$K{C7b%&-dfgiKr=PdGW(?WjUBE-m1qekNzGiK~BUU_->xWTBUg=Ro z(DGHDi!ZKPRa3XNYSEIqYP^igmu)LAOiLI43kCHyv(or4Wcd#9e@F7-+Nv6uTvD;r zV|nC}PA;)Tir{Qo|N!pgs9?;XsBol85v9K@K$w3#;VmBYb#bQ zUAH&M56fe6z}mMd{2%kRYJCD4$hUCQ23-m5cc^ zi)PK4Q&Kdqq*zSB#%YCWGG;A&QPzU%pGFEXB;f6*96op^6N{tW**?K>dng0z9( z_(h0F5hee{!Ll3w;zcZe-w>nz^~7%}dSbjbVLujs@n_7T(|$Z=2-}MUq*-$+@O~rq5dmU(1C1U155_|-P`T$D*&H)r@j4dz@_iDx9)$TIRRJXQG0x%u07%&@wScsJloz-IwJ0(=iJ2zVUu zC}1?={WV|;U;*~DCjjoo5&gq}O@R9WUjcj%FbH@Y@DyM)2EY~AC!Yrx&kDt*T{1^# z8{Aso=x}3)7VaiJ8Gq-$+TK2l0_yr@c_C83r~IM6x3}*?eTrE3l-R^+aZ&4ydNHZb z_+cYc2GIm%it+a??B<&$z9qTTFgmux6 z?f9Dpxltrl#=2*KOZ#`@uNw4+NSF4DV?Fxav2J8A0;K);`wINe$e$eR-mXuNO}r|s zFxJ!HD30~&|BUq%#wJ2>a;(98j^pnkv}^f@3-VdWhxBN)^G87MgEaG*2YMNjDWLBG z{bfMfzgAxmn|Mvw^E9$9!Q0;X%Lwln+O%; zOrt`uY_+hHf&P-D>}}WKp7K|}A<}lWFiNkDO)RpM8Ha}HkaM z-wM7-zPd5t#QQW6`dg-%#Y(pJ^K5-NQ)(GKYBxGx$Xwv^)GBZ zlCLh7i{s50&^bri7wNBoo(}pz5{qNq1S$CAxReX}P7+YAs~GnNwiEqk8FaKKS$>Q| z_d;%+l7m0Bwm;;L0{;~FQ^<$?27M#idx>oC%zqyEgWku+Ecxmh1<3L59MJ1YL);p5 zG>+}e|0=Z8v<7E!Y_`54HVp(gX0;l&Z;qF{p?8+j+pZV0y_B+j>*$pzN=N%*IbVbP zOORhr`Ob2()BHNLoTnhy_k;HK%P1$yp>MFu$$XPhzZ<|`LO$H-(oWCrn(qXNltKPM z$Zw~-kMd|2SA`*8_<2#RvG2f3MND56wp}mU2mJ$Z{3R<#=C>XCC!k+VeVLyur=`EJ zYkso+AwL2e(z3r&2bZjW&@(~Du-b8v{yFIVK)1&nGD&w}9*Fq`dNSxg!JqE(qd-6U z3v|vm{{p>xzMOwv`UUw*L4Woa=r@A?@GsbT81y^3(b*pNgT4*)n^+*(U-cX8dSv-N z2EQGAfR2mwQy_f)3v|whZ~p>)1n37qpH3^~v2J#XG9)=~eh+#z3yylbmhD=Nk8(T+ z9Us?(%?Lm9@F>Th8)5I{i|y@unU5^bIcU1OS`I%)WpMZ4E)8ByJlLXMq z;h!u&=tn^h9p_~ELH`_dX+YU2rY)3%^M4iX$nwbXt)yeT_i=plalD@q>s|4SrJ;SU z1KgO;G3@;3`cDM@6M_Fk;6D-gPXzuGf&WGX*7WpB>2d{aRrNR`_uojf@vTZATEVnf zuS~@#{Iv`VPg1y;xmc#uai`p0B*9)&8j3I18lc;2!EaBuucq9sx|cxivk)!|_Y%nc zOu|FDmq6}A6277wwVzq+{oe#KLoO`Mt*d;og!UTZEydvaf|tG4v`-|IitX6QHf>$Ycvx4_3_=JK76nsO$PZd0& zpv$fDS8#}eV-%dC-~t7!6qO>Meo^-e!ilc-RM<{9^Z|=LeZVw=ocxvs~dft zqVtzrq1PpfZgit>QuK&!bibnG1@$i14T>Jsjee)1a}O%?dRWn8y3wCjbpHA}^xCWF zvEAq|C_2CI7J9v`=n38EuPQo!;1+tlt?0eG(s#V6=!sqFdp=Thx!2Wc=&@Es@7Gmc ze5L69yV1W>^a0)I8s2E;G zmp4h#`Fp^S%ag3=L%P!40~CE|SGs<+Dwq6~LZ=}uOUb8qk*^nn6@6G2I=)k-=)=3x zwG2fc(S=?wUpbjtR5Z?$hA&*Edq!pEWsdgbX6KB~9+NxLlQsvh5cn!;EqX>yPP!NY z=d2)FQ^{}CgzUpZ^emi#A)UMbcPx(gAa$PFZ23{SD;m-V;g?~JvC_~{T{ za#KL2y56k(7JF^|a5d4-?<&@TErw@SFeCi# z7y!*!2;ne2{BTbINB9Gh=sH^xoyBfgcR2Bm1r(g0V`E<5fm_&Cn$X{(?DpkM9zgNJ z8f7wq)N7Vlc?<$@8u~@V{zSIRc!FuoRe>DzaNHc(2SEuBL-H->iBcUU--WdEJDL#2 zX9%?O)E*{{KcaxnpO|(Vvrr$-(@ZBC{F#&Ma)4_oGyu!N8ArH&Txq=jHpCtoe|m%~5R3a8$%Qi62(oK3Tajfm9Inco#tNM(t`csUqh;U8CTKX1Aaw2} zkaii?JqHRsdO8wH--S{0d5$IK1l%-}wV<2NvJaRq!-&KDh-EiV;_fnE0yo@jguG#z zh+u>{0j(_3?2GmoWiH0zHQM|=Bx6i|VJg-f33{A499nMkO33#xhoHIjG@pcJ)4UVh zc=IB-kYN5Ap7t_Nz;bW12hxe=jgUz)Z--VNvp@XpYrctwl59ST#@EmEqLTWXRVdg1 zvk^IZ%oE@aG#4YUL8cSw!REhUKgFzvZmOAwe218SM%@iHqi|0%d%*K_a{>GuW=6u! zaPxlH9AWa5FT=bN=}hxg$YhzBkk2;X1trJ)J^aZvUAT`ld0scl+zvlSo6WE=#^l#u z$C}xo(dul39(u-YiTHGJ8&# z944O^W*<1SPnbgh_6u_X%)cPa3*glOVcvwAZV~2k*gh!CxybIX!n_~E7lrv(DEv*B zAHkECg!vnU;ALSRf!^PR`7(0&hcFv~9TH|DL|zf*6o6NS`8urpQR7R?+9k*hS*Kp?Ny%r_C9t(y5O z*z`jLV4G%!!TEq@-j0H8*USabYt+m~;LF(=tlXlR^~m8?&HMqOyG=9Sg~IJnKs@iz%wZ7OshQUz zhu>&s3Dke9naRNJ)Xa3~?b6J?2*~d=vlsO4(#-P^y4{-jcZB?I&8&ql_h{zF$nRdw zycO#AX{HIi`!#bS!t(%13$Gv4%#E<}kY?^itvsxmmB``un%NV)M>I1W*rS?x2qJql zb3BawK{NAE_Qw!UDEv_~A3(N`Yvw%&V1CFYZG0vSD}ncqP2Y0caM-ZQ8G z*!(ky@cLQJ44|}s(aa$7+pC%RkZjV-1E`K>&76P`Jg31h`k!V#i9qht%txWHAI9Lz z3!3>S@D3nKC|HYT?t(9Eni&a^&!GUDM-dlDexaHB;LDeq`2=hp)66j_(N~&zF%-Vm z%nMKf-=L1**>TPM9N=5c+zek%AUw$7JJdUjoy5z*(_2HNQ2;0VHm<+FH z5+g51@hC)F&H-^ke*lpPzv5F+6TU}giJH-CG1x{kifAOQp^otsyf^wVBg43a^Z`?l zb{m6HV#6~D>7K?8rUz2hv?iK0b|&qjS8?ea;jZ5ZO7Gn$yUw2mN5oT6{|L8;-n6G* zf>K8GnZaZ)HA{16KxN8W+b2|L>4kV*_erNMNVeA$e4vniY%gPvGEQ) zE@s+itVaPOOPDS*BI(W)x>Jt5C-~rqjyV9g%tYKY^Ca%NxdIKsVV=h*n{!adZV`SD z+Q9~GTOvF)o&#G8Fe#GoDQBT=|0{Acl2!_>u|KG8<2|snYaT`}iAmKE`vgS7o&7q1{vD9Fp#{mVuR4Kh$k-*TDD9qtrA*}uR3^>{}HNExVI;pt;~B|8E0)lLYYL|Qhy(v5nNmaCZZJD-CKSPQ>qEXFDND5(nS zP96Qa|8hi8^0}TEuv};>DEuUl^yNxp60$)S5n&~nUZp%N1#YiW{D(R$W}p=f9|cxK zSpEFPl01A8Q2n|UBe+=BbgX*jAh8IEA$KVKOr`%U{K=qr24yoS*UebkE~PRXE~VY4 zkaK~gKd+FE3c6Z3iwceipJ40%er9Y@d zSQ6P}`ME(}vJ_=#N7+i^Wk`Hu6(b^yA4RmJ(8`ZwS%~n=QnEqFM!{hg&zLJbOKmzj z#xRQ>5ysCW%FA<+VxNZE42!Lef;YxGVR%vu&k*6cR*58HA9|g#d|f%4^K}d1sy1$* zHiwZ+drJwHLeO(qA?EdW&8z+%!d#^ZBbZ)MYcEZwVhwU zN_v7K(_grtVIs`j$_iZCUvcc+eQoCuH=G)-$aE?@F9Z>p*JAq7?zL(14KU=d?YyUl z8olmib$$VNIC|a7(Q6!3*a`0^=PDpWZ(`;W{RC*L9EBg?02_8)@baM!4m}TYFe$=> zFyvm9zrC}%2ssUqt>P)Rink%d(dTI@@Tc>hW~Fcqy#F_fBWrboNOSk{LS6_N>JzemY)yp{FDFCto3&fAJ-{kjT4 zkh9ZKdi^xLZdLrF$m?dA7jhA4#}%1QlA8N7)NuKn8#Zp-ikj`H$+qS)*zK8vldqb_G)k>*kxd$RNNSI9VIC-PRY_WEfy@j=TV zde8ImnB-A5@KCm;D4QooX_iCt5!(rV)lpua!Vvb$Hv57O_UsV$=Qf*PeYCVILfBEr zTSj4H2fHqWJGC5M3?8)bd)P+#dhOm1@gxxxSv6M{x02yd$sc6U%VLNOh zPZ6~AD_|ocY^x%=4A%Yl0<4Mg3t8<3 zkF>pbft6vwx@7Dht0(+M0`ziE?dS_!*+) zzL_p^%D0FGUt{KXxHT4qlY6HXQBVq6%!T@LC~!hlzPyNP>fzcABByMtj51yYf5ceg z*s0$GRsmV^R>YKyEVDo$_Q*RmiXXuPSWpDHDu8DU-48>!evoGjVO`D`jPOCoQJpcU z;{>O1HfXLzWT?Y{e=z5?i$Dp#0yq8{e6B`s+lywO=#+;hk0z?T+N}g>jnc#7G`r80}1Z zqIrB_2#rWuG>Un#UK$Xz5#X^c{3`tacjt>=!XSzA)~HkjEFs z?^wu49$y%DF)1Q>d|~Wnag9hGUl{kXv~JWZ*WgZrl^D({rrWC7ISp2)q{g!GvoV?& zSI3MJQ94ih+;WG*Eq6HF)((f-Cwdhoe!fq{!$7r9M6*5A zKG7nyqeyF?s0uk?QwHutj%M13wDyUZc1K$KL`*04l>0L`g=%$gRQ)@oxRbb5|62M#SiFxN!)ngC$6ol59dy8<0&Qax*}^k{|v>xD{C$Qllf0a zJ5BCau|{Z@wEzvf?*7aB-<9N*;yqyEB8Vq(ABP{XmRJ8L6niAt+>HbqkDW={Y$}WX zEsA{>*!M~Hd~KXad_|FKL58nfpAQ(!vOh1&ZnNd@U67Bqa~nLvQ!^x9loD3}A2^Kh zeM#bC&`U(#a8M9?LaRMJXzoqN?Z7k9vpw&T|CYqr{s;9U|6Ru}DPLdX8AapoIT-HW zfbSVY`F}}#JiPRj692%F2X}(N2VZ>gDxv>dW>m#}Yq}_=n^l zaj+f_5g)+(j!XV@^!}6*%11gA`gmnvQ#A6O&MW&Wd^hz+hNtyc_+{h|4==QFuJ!bk zPCTFddCsqFd&|hb*%`3yeM0{2uD-VZDdM}Fzg7Iy^C^GQm3lkrAG=c463_3jKZIU} zBY2*h$#6Rv819h|%|^oW8xnJ!jQ$`Dzn+c6M>_WauLnM44TPS@V#~9i=Eq9yjhJ-A zkVClfcX}2lPjcGH>0hmiauvdV5)x=Qu1m$Z{)59s+VhIH6FdweJya1r?@#ZOndBb+tdcKPo&R);r%dxlxJ(z@W zRzHK#L9GY1AD$cruK}mC&xUhEeFuW+9Ln)7$)Lv=!9QxpK#K_DN5*-%{t8seIyrk$ zcC;AuBx5>Q=Y)pc*jz6qh3i|04&s7c&(z0A7QY_e!ID;{+g9d?LBnZfyj~$i%Wcu| zK*N6x=hoSxh8VPhqNVzFDay}9ORLqkC^y-qZ?L+DqOQw@;Uh8JnCrj$i3X&Xf1C{rST$@rDM+7M0CsukfXE9g;zwyv+kY84`SVEc+Zv z)JTcBT_m!ILh1_KvnNxkKxbh%XlF4YjftF0?D#G&Pcs*3l7`>mnza%lN0c3bbAo%1N}A@!5!-eNDi0qL%QZ zK>L6Fnh+P!j_p-2As5k(MkZlx5P;Kg8P^dFc?-yyy&DwkEg;v46jZc)3&{1|`5;?w z0l7{En3Qh;xqf2WdJD*PnrZ7TAXhun)>}X>oFW0S-U4!I60zO_a_JJW-U4!kNyK^! z$mNuX^%juJB@yc_AXm6Vthaz%5!$(M(RvHW6{Rf!V!Z|AiqYt?^%jt;heWKmfLx}w z0vzitAXjf~G7#%6AXlPR4#av3$d#n=ziG4H0&?}!HUP2S0&)$})}ZXvD)RxN4*8)Qf~pde#b(} zw}4!CF=@R8RT5kcl?qg~B7SM3etf$Z@)-BzRz7h#iS)G!4jZHO044;@>6Es%4 z%h&}*EcILl4XT%S)N@wMBj}sqpXe_@$YuNu_c#iDfP}l_2F@oS^w2*;tJX&T3Ie*b z50axOtH)zqr`={fS;wGjg&zmOr~TzYq>ase!EE{ylEUD=gZ3O7j$w>PkI|lI+HItu zpJ^==Pc$NE>?Ni>#tPD3Wgcn9<~jPIEKkm)?*kCZvW z^aLZ1=~kw_LO+KY|CW{IGH%5^CY#(@+IXhn7I_Q>H>*)thmmcf1IOQl z0$r&m@M}R0dICQfv{{dT4E}D>;}1anDm{VU2D)00e-bL0${G`$Cdi)rO+@Z(cg|P0_;}^oE-{|qz zLFBi3{C5C%>hXslvP+Nu5+c9T@>zNamLLek4v0D!>8AQJz^s(HiWZU?u|np&J+ zTLeq;4NJ1`9!lQf)Em3NSLW}(hJ8e5T9N@(St`i9Fv z6=FL8Yw;;mKkkSf;FU5yc?W4{N;PTS5Pcw9DrU`J*HF~6nDCoPYQ6&GRR zumsMudcXnhFgBs4f4XpXF zo^_qlD^UE$kb~rI05_Wr$(DecytMnAH$y0AK6rZ|lzW>pF^%OM@uVT%apr6SojYVXHArM3G2(=hS}Y4X4`&{{Mos4! z_PV3L$18D*$_iWnbE8u%?kU7{4vgduRbq@#?iC7QBNLOS9a6D1nU(&lsxM6c1Kb+AyaSJsYB9 z-wF{$>}1WtrCaFI4CgzFJKN4_C*SO&OEaWP_rk=`VFSIAEen%&B{g%L!xgc znIkp7hUVx-cGMgvr;UldutxrY_$b+Nw*k*BP(~VIbJ%o+Feo>|7QE#_@-V-m-wGif zrIL=i$lamnkAN=n7O^^Xne{`8aaMOIipKweviyY%V9~E&U_C%ABA>E{c&uf zk1HeP45-VQ4-4bjJbfdY6o+k-z8F5$s^3!#NXbi!b8dv1Y}F-jZ^Xa5YDQkFPSpqN zDCR4Q$wq!V)L=RyjQ^R2yhfZ-Y}Stz8FX6NQb*mvzQ5n$7-Y$=w=~eavM}b1zM3)L z?_eEp!#y`$iLspyD^m#TE&|f985SwLVpxr0FuQ>kgOxFKqoP|4o16aHubnpWGGSfX z&JE02_K;u8kaAMW^($T3Lq6|fsFX$6>O854ODsDm!XIELci>>JB(UG*PFINZAn!Sr z?j@PW*^05fL-_*ikBnXjB}fE79gjiTs`i)UO!9l+toFZmR{O=u$PNu!Z#YN7f~>#6 zaBxI}VusfIu$vT-^)?6MvflU~Xvk~Cn~KdL(yBK!Rxg^4c3S58sp3~#@+k5oJQs|< zi~%|A^-hBgg^4q!d*$oQI?V_BTgv+yOgMQF)Sy=E2UWLEnrWHaRza(%r{LU3^hh zPB^c@v)rRfK+a8G=K6!8Ot-B)?#g{r(PWqYNFf|GL<_>elV3+| z>UFYBHAA*YF2L*9dy298UQb#v7wGy1t^%_Zs^z=nNphKM5t4ro!`fm8 zZ6?bFZ-rd&Cd<`rrCi;L5+7U9Q(-dW$6+D)Yid0c7NC}c%s*S{#VD@!tQ5@0SYxG< zkI;d?Na_^6e3Sef#Xb$=@_ouRAaK+@8piP}xhLD;mr}N0@>phbT*_ggpI<4op3>8N zjA}g{>Bybye<_vWpfwCS+|1GL(&_8(RPd_9$pLvMcP|IY;@_?FIpAH?+#p%(dvq3N z13KcMbC|1zA0fVlF5N3FX7@v5bP#_{pIQKAoZ$I(5`Lpk9_nQ#BJ1KIR${3{@i`bQ z)vc$6H+4QOoQ4rm%G$R-w%V+rUUUBextv`f-_~D5n6Uql`@>{-JU? z1}E9fK$A^||J@2NEZ21-eYiG9^9|P)MugeV8(@_kgUM`0z|Zk!qzv*8uwLdxANQCZWN&P)RHj~*mak?;&eA_s{^Ciog`D-%JmD8-c+<6M&h?jedLMWK~LWr2iO&j8s z4#~yjbcJv+xd5$>!zmY&x%MZCxnvu*d8nn*fO2iJq_k{AK+6`x9IEwkeEqHDVGYdx zA4py!(iHn~u&I%Qw^W};-_)GdrZRa73KB*OzioXr9kdNQDlD-ev zkhS35PTk5O=%^eszH)nIp2rYPeP=lYNA&1nk=~ZM!?DgDh-!?KhP zAADBBi0rxPV=O~rSVai?8k?PMvvW5oc077-?q-GbC33SuWc}QskU`|!t&j{Lm=8F{ z&AbVwwV%?F#JklC76XF#I$Y9DTf}mk_zH)E_5{Q-BFv|VE`#H==Xs9lJ+$kf8xiJJ zJdfPx>#1=ljtI-MsaJA_=p}c9aTa1zpJtA|<*r;r*a(}-o}eWvYMMegy6NPj&AUPxm@gW2`(S`YlE2K(;NKC_JoFZI?Dhite#RJwQDVKwi5~tB|ZU zp)6@-oo!_}-jJnlk7(PasM{8;rj<9C!IRp3lEwcuidU>;^`};|#-TeR%wq@cYbrda zLZ4^{j?03eonxAqxd+balire6?PfePhF!_eP7x92R+_U0L&kYOx3g#UM4dPvkVzgk zIUi)Q{2wf*san7erB;3c)X*WgMvn+HY-%+pQkyDlDvztQU#YrYIDfG$V|jKTP+wLB zskdeMj+(Y#$(~fQvw6y;Z)01>ag~}EXK|!9F5sr*Y`$-(Uu`v}t;*DFzImu`wTRml zan8S3`gl3@b92nirIx&jIXuyFE|pnxN6gK#N!DC3k4I7A{dF!!-Lxp?@F*($6?wG4 z@z%|bCg!}1u!Mggsa!U?{UBPbc|4^G|4`#BA<1GsUo(u!y$2r5xvVGhK3mS;-0qNh zi#ZR#TI4`UfNw~~>Rd^vGX2C%zt-snXRgk(0_l`>lmBzl zoZAq1*B@o+xsG=8hVg~R6>~mf8<{BmU~g95C9%1@ptInd!#c}qnPvFIJigiJ92dsP zz*76wak}&d_v<+eSl+WuTBTdaH%g+yIRmKJZRf0OxeXtv70TtoBei86fo^JDUo${=s3ZHD0z11 z_-+zpt+PYNscV^(XNQiTn6}Oi9jBSL&JG>zOj~D%4*aMLh;?>||Mi53b#~~`C1RZ& zI>IDkogF%y60yz>9WIGjXNQh(iCAZcjtH$1E?Q@Yjwo$85bNyF5u?##>+I0cLn7AM zp~KYHfMcB7$j@>M-b#~~ukEP|=VJ>Jc<3V6itX{eueH9X{oW(C+8AUrYz2%hvLas)^`-tdZ{qGu?wPFJ>*LOxP@sfqq4~sejMVy; zDH`TEd1*U`B7ajJr1WK+U!dJNO6oH5@TcsG9_K1XNZ5K=b0#adH{XzU$v31~zZ6QO z%|tpZlGJ3vuq2W%`uq(M5C5lZYs@NbIcGV&;)yTF&Nir5HTAqMS7rlh@K9uGg~ta`6S1n zF3k{Xg7nR4`-Zpa!VJ4~y-jFL!2FR2A*0h0_BU-Idfdm+#hbpF+N2~aL_?EG@tFmd zAbC`W3U(Yap}-%#>BrCz4g}L+Uoc~wMAy(9B>050;@3>7tSqQbZ_bTREwcr4(MTEaz{BD8ri|KbiQl*E!M z=^#a|itV&FB|#&?Os958K#BRmhyM@Vo!!!9_0Dj++ZuMajp#Z6M0WO9_8{bcD5g@9x{#XxSRMKotE{lf*E|cGKUsNgoGvSdtagC3BBzajJz5 z>_n3h$w8ectqxY1DB+Yy)hqbod#a7d0bqV_W;`@Xn-2#=yS=PPJU$HTw9GeF(BthP z#Oev?`JDvjjS;}^f|HU+=rR^qZ8gj47};IRYw>bIi54mM=N({VhZKU|y(Q1@-9!7I zpSVL+oXoe|%$VOz5%tS!r)}qt826G-L=^#b_7PYA**>O zg*NX#onswdq3_-kIBju`kZ$*jLWAiaYSFTXhH43M964h%Mcogy2lUVeik3DwQ{@se z4&zPo&W_j3Y--n@Gz3nDOq)ZyOd@HW&UJAuy|dixjl)8F-tdtAJi=z6E&puB%d{uD ztj@_ZJ7hYNV@-}d9pze%aYh^&DugL4G^kzN#9U@qfSl2&HwWDla~-3Oc%4*C7Zee<6OMW+!#lAXhw?()z(7OJX_N zZF1{UM9gurh;rJqT2zs;E?1Amoz;u8sLtxNmsV4-Ohg%`c8+8|JIN8QG^DU{c`(fy z-lp5Dl``VAFhdmrj<5(bk#ETQXjUg`jdkUUfLK{W+w4#Yj&*ZFtJR*;=YoL^9(#?5 zz*@uF{Ek@5KyVaWAf>u26@JyBsHEul!cd}R?W|C1-A6=*|9^f&R7?5LGInVfadyvR z5#1qOvWLyU0zb6m-I_f1C_{&Zc+^hFD3Q?F8GD7+8)Jnj-(F&cE}rmnNDk6T9o&w( z>zwpuKe4Ky^Ep7uq1HdcGnp=KK%>;OA7f*1mtWH)o>f<~HfzQ5C0Q3#R%UI;8?$K4 zsEoB$OP5vDX5hb_uG^3$zq9%O6!$IQaaGs3dvuH#115lNFwg|~0pSr#et4%$yk!a=)+NJ+|hoz0O)|uf6u#kF(D?Yd4Kf42%^A zGfI8j^;8G1p^~`x83JLosx8@wZZ$D!c-dlv_bD;FOtUgCUN zrFKp`dk+72s<!$x@t8-xg`hzEe zBhH)ZaA|m?v*XU>4F{X^rDM**^-eYwoWH!OsWci~()&K*>pd83PCtD~)4Cmt*EJtI zb?Q{G`P8W&o;r5Pp81D*cTl{#Kwzu(b_&D%|I^zWXQ)ca1q*%s)jkgo!tUsB2&h@< z5anK8vo*}(pJ@JuU@tscYp6$-UtN}nPdT&zL>bP-c#<&Xk4btR7sK0mKQ|0q2ynv~j*OH|6ZzdROxuoyD7@OnJiT9y-x`{Rmx~ zDKziZ=v4iQ;7xf_|Y{!QU6@}xi55VkC>G9Jd*3w-$B3Yun%8q{;jL-N$Pd0bf(a*uABhSIq*QOQ{BVGcSgO}X#vx$CgEIJ43 z3eFw-4m)oI*5eEvJDj;&o$2)*#UlO}r+S@H9MX86KuLrg|pWAof5DCCL z{d4NlPyXrA^e3mCRBFNC2Iqtdp0{xCg1hkVL-^;MQJDXM`5#y?J>OYeAeN=XBFfU3 zEWZ!yB~EfZk)g4q8;<$7k`;g&Hv|R+l%RS#inBi zn?MLJxo^&pbH=kLoxO#JnvRs7FFj8=SYx*Aoc?HV$5FCK;OBKq$>$Dk2!6a2$xYo^ zciH*BJXjot;Lq%B9PSOCa*nH)i^03o>2$}sr<}jupXv=ZrGw8QlQ&LFJ%k)a?{v-# zu0L9G&Ny6m#rdx|AFfM1PyF}qDS+rFr#A*4`^25jYqzGHyJ6E)w+`Lu9D`vWs4J8{ z2^U)MY;XE&Uvw@F7B=LLIJ@hX26l8kKh=5gMdzPRJK#L9H`RT@DIT71&OKcBo4`V6 zLGTGDyZ*?H`R2Xf8GhV({It5u0?s$~ZgoC;+K}_i-c-lYPdH!Rx7GPlU~BQv^UhQI zUUn8AxPAJ@n^!qIQ%9UL@0seEdT!?l=iK0V!7ohTSls0t+<(}48FlyI{@{T+1T6^6 z!5*u1@+ygQe<0x8)PT;{xoN$#@MY(wtbe64iB}{$&lO(2&YP#U=Vl z>%zJ5Hn%g>6Y5;;wqZ~6kQ)#8h7@7Lk%ZgTno@b}P*3BsMs~HvVxe|-Q@Aq}*9dHb z-r26{h$qq;>W&(GXNQ|;>(FpKym?zPlnfa%H%}7Xg1(g4(j97Tce|q-lko($T@Oc* z@0L(^IH51MhkC+oAvX+}xRURi%DCA0USHBmU(HIl#<5AfMsJO7blaklM0d1PWeX#j z+_-Yx;>tDpV%wJ1Zmq<0ZnThbF-XuPWoWDqlSwmea7_X+AM{l!Q=+?-R1mG);#w^$ z6^(@=F7|eh)4uPdo)OyLmp7)`97=5NcDuDft$||Uuxs|SFHZR;&- z*GBk4#3s3Nbgjg?qlr+PsTC>&yd%^d2_@9{XnsH|ZmFYfp?KVl!`7j8 zZOBM#mpDRk^(tI1=G=koSaIACLAE9)?R3WNq0U6BR%vu%wBVX*(j;A>u4s1}R9Z7# z3#cw#E!uBk(M$osHgzQ$KQlVUZ3)TwLf!6m_!~~0(Y6jYdjeOw4BWNNlbP8wYW#Ln zcvDnSm5fddVyZ$}ta95tcObv1c5Q1!6mGmPcT*;(IXAYs?cpx1)L@oN=((Q})0#-Y zl@hIO9j)!{-I}fES~uJoTfNGSZrVhaM4m}$v__%W#yAzKHDOw`ZZa#~cGJ*1!;z3_ z?9hf9G&K}Fz0u~9>Qeab(+ziT+t?j#h0)^)HCP; zHq+e!tt7bgu_hHD9PP2P*je$lEOj6HvMGbx-kNCL*c#`OmbHAheRFqfmnPL^rq&aV zY}SR2bcMY_v#aNgRMVlshPpYMCNz{)17n%K32EIUeR((6H!+BJ9!Sw7j0K;AzB(y= zF4_v&vvPUGb#>p}m*?Iz7LIVc)~<#n4K1UxDcCWuyLCHv&!+8jf1z2}MDf;5A$LGr|fWInhjcTB9uz# zCa%Xq(|pRid~`$9JDR#!g}y=&MS^~9(auhHdnmklOTx=TMlR#>G|ub()Sc`bTnU|VT4heF&FN}$ zx<*U%b)!^~dHU3kax+s)Gt5?$G#TH5UR%l-j`f)N>y{`c$rXf{I-oV8+qJ}OB$_*6 zR+C0HTSDzgbQ7D^%#1M(W~c>a7I&39p2gBODYMFrZ#M-)LR=fOSI@1rmvqL^ifPD~ z#=OdXTQnI-P&T?`X4cXkf_BNaM6}zb8CI*;XPTm;k~KN}z%~@Nw#H-f*P5m# zW~S{%O+BJ@CnLJ=zzbu9MK9x@ij;K^xFOi2$*fsrwm+z{lkiR&72L^DjP#m}CZA&b zjji3?Va)RN1>Lz+6k33U|nQi zU7+{It$&ZAX8GUKGQxDN7t) z62pX#XM@N)qG4)geFHZYCrwLp^CQ_@fn1*2Tz6|YjxILZZYC7QdAiW2V-97+hPvDc z1_-bDB`Rhr?AW+V#a$A{(uvF-R2r2QMl;oid8vP5Y=$hY>k!qr&MMJc(Uy!ZXfACX zu_$JGMx9tImaNE9?4u|_gjJ=+*VYQlQnB+hd3x$$1ncQDTD~<-JPikr^4yL4A-R)< z_t1SQ+J~`ud`qaaQ_I-|SJ~W^Q05VkMw$M-tvekv`tgK;r_B`(h}i^Gb>qnxYA?Yf zh-r6vWHJ4YRG+_4c7lFS4uN`xDkRM_;3dURI(G>7Kwq?c}s(nvxTw$0o@>Jm=jNyJ#D zd`O3nxrnSH%MxIyG?Tn4jV>k!Cz0UB2e(hULQz%cJ?~EcKiX zjZvlmm<4ycxW)9wF_{M2wmUS6@*C}qUz=uX>}~pLd$UD@3^{Ly8qUD&#zx1r3K27> zvlXN7Ny~ag=a z>{KQ)hRvuC8Q zm@i}oO&JQfH_nggwLdATTj-;5<@0^053d&UT1%=7h+c)8(qXfR$_CEoG2y0l6)KMf z^JK6oNHXQaNk1qnd!CC6*_*Uq2s>t!Qpb4dA(vNOm=NJ+N}&(z&tM9k#YA(8jA$kW zGc(gUxFwRKd2*Ue4z)UJ2uK~8s1Jb;LTQmCJE@XwkycdiKz1D3xI+^d>o>MWIw%8@ zPWRh-5rJ+1NWj3zO>{djfXVIv76~+lq!KjNmoK=+?0DG$H!a#n+zDE&gzo9IyD_)zY{X(VB*B9e0tHa@kQUa*+!4B4Wfw+!)v6S7Gj1f= z>1t;|%@^P*V=9Du2@>U*7y0M2#hh*h$XR5Q6fz%7M^73i6d;$P6hKGwq?U?H>YLRu zq(~>4jd?|dS6JH^a}mvCE;BWnE#|rO;2aKaORyH2uuajQ3~96#R&833W}u)mE;7>yQcZvhIw;cpvIb zq3@ZZORSIAjd$9rWVfclT5>_%oJW_e?A?>vgayD(sv1H; zW`QQ$)Filza_QdRH3i|uD7}K)KdIu~tx@8l4nwb_0zs;gCumWkCASzo|0dOk#=M3? z=>h33-j8)K!!PZ|xSe!BWtE^ibGYDG27byJ&`_d6skTr|&q~p0LKCkoWvKfh6xW!q zuP~WJl3g2(fidG9q{^c;S}2LjFdhM96c`%)3nfiwqlR-B;k4Z-L)UbUbbWLJqo_+` z4epXQQ@4|hD=ctY$)u5PLS#2_CaiARyts6Th3ISG=s#&AIAk(`ibKI;$N~`)v+y!K zMA0J{dX0?RBbqzOhpVAS1=k>Mi^Z)?_m-V-Gg;k{0`GF?IhMuJve4;n4ZKuTxD-~6 z$nVU9FF5j$o=hp1{ogfn47YDQTZ9_fA=4Z)m|2sr!60p|F*Rvgg?=MLKO?2*qzW&@ z>m#6&0$#_cIoNi<1NvH!VAz(RUqKZ+zBE#3G`Ut3)U#JM^up~(VAF!ynfDbaN zwx0TRuAXYZ~dCLn!gPY3GE0N(F0rq-X2Y7(h0OGDp*((zR* zB`M%AUiPFTpj-h*7*)FpMAm;=SqT9LtgM{^mdmPc6Te^?B7mw>;Gh-OBjA~ga@16y zMBiG{LE*XS2&h?$Eh5Xse+{ph8n#`}F!&;$>(t7Uz!@uDf6{Mw%&*@N< z^hFH{*lmUH5U|IhlG_!?F7?lV;0ZxhpB zXZd0RhAk>CAcfbV1AZ;wghj;#%vn@&4};&asJMVHSX5H1E)`zG+e6h>F{*gK;&VA` zrKEtTEGjObgO?QPFy1aHe9fQ_XdvKbi%JN1dj+rI@P4=ORoc0N4#*O)z$i!M1EncW zOnY>O!TZH)?D|-0Nn!%-wWuBeZ?mYRC&EDgzy_dKGwG<|LEmGNA6i?*1pIf4N(!i~ zpsMC4*16k~Bn4bnFWJWhyoyn^dmXvD)-omqysm;#Qq3NrfEy|ZjRM}UT;1$33b^HM z8Sy2OK0eN8jI;v&%%ZA#?RxIDSMZs74}s~+emc-6(y^S#Jih(e$fP;}FDzn^w7Rt4I<%Aoi#Py+sUu6T^(!IC7T zmjRuaQhFJTGXi?n?U53Qx?r45S+SI(g2!z;8??Ia5U^lTF#&I}6;(CZF5qH#4R)u1 z4_eoDA7jvK!!ZGM+!^SMmM-A`vWzhSrRV{*OTgc8UAlxk*f_8^Q-Bo zxo=k8WPG1R{HqnUQ^5aV)akfIS-osZEfK^?YoDD0ew$I2RuG~D8p&*>eVX&3k_mj0 zarJ6H_O83ABH}khF9L+Sg^@Aog{+Cf-QK_gBy^Zqr|}ZF{G< zybLEbs#Ti53|};a2rT1Q*--X83nwyxU$hxk&BJK-KRO<@BryR!;Yk6dOFu*HIeqn| ztC#J=uq`;rWTAKM7Wny6O}L~am;aG7S_J*-%|%ElrxXeqEG|;F;NOy(w9@m+fbn*; zrl8)h6zX$c(FGJ`9h9X?^LnFs$**Tn^ep+c@~(Kf8%T^NWj!4-$Yxa{>;=DsPrj!` zqyC-GnDq(x8l%ouGiuc`8$ED|;NS9C{8A-w>yAr|bVwCYwsojIKNiNcWlRWI&RBg5 zE?37aV^Y9ri`pijZ2M4KiC386q6X2vYT7HiK0M5%bgwO2JA9)I=Qm6>-`W~{YpdG6 zYz-kQ)v0P4*&sq_YD^=9s^&W|U4_j1Rz(*KWTd!(a36b^ZR#uOI<+7?9;3j7H zE6cdkueu2Mf+a~>aEVx^C&1twIoDjj7Fl1kvJwJ1%!oAs0n24otNtR(c*)923Mc~3 zM8_IzSQc!!Y-wFdkRtUgXqd29mG2O$%dJ{T0gDzD7w``mRlCCE>RpyGDd4XxDlXvb zZ!1gonAo?pMbQCo7Vt%j>JjiCEh;JCKU-9ffU*NbZS5t;1C}u%pk(8yTk1-Zkc==r z5%7o1s_7(Ao@;gO$~G7x6}L`Xs`^(E@hhuZQoxR7VxJuX%7zfNiz%{xYh@(_3|m<} z0+!3FR$lh1SZ5hyvQ33B*0^zzeKEFJuDF1bE$&$BN|F#!_2kHKaP1FeKWDL=MONN6 z0WW5hqiznA-pz>>9$lFf4N6wqElE{}E!#5u%#u{ynaQ3C9hbIfsqkF^I~i5GZscmr zV-!&KNO-4=iCT=+H7H_}UXB9Ft_!tu9KFnxZH&*TX9><<_g*P>YypFRz~{Nj#!nt* z)Tl=-G3~Fq!U|~NAm|}JBO8f}{!I_1tz7ogTcuo&=y~*VksX(8F1Lc>lHF%Llw|jx z9;z%S4GwU2^R3=J7c#iPa`lMpH$9YJkXUY>XA`kp%0r384tl7MF!+FndWgYCJk)Cp zzU86jvN_&s%@SMA;0g~l!z%Bys>j6o-}g}d(%ivj`5>RqRt1`=H(b}EDVGTTqhdal zS9)!woJcO7WTJpd3HU85w^(rpAo$lk7Qbp@oNurt;Fl8k+E4lnlYGxJ+P^UPGY{qG zYG>nZ^^7Rsl~=ZCYApLbuwS-45OZqe57?%!^I0SPIY?u9!OKBFaXvhd6;OPAHR@pH zGDiZaFS)NSxMEpw#brwaXT62vB3zTO==NGwhkH6Zw+KI5-K-3I`xUhQB6!Jk!~a93 z0TimLWz}{$4^*s4F=k^1um3sp> z$R1l{x4Ou#-<eF7w%@91_maVbR@N>7MHUJz;8PxzUwVximJMUR z$__>y#mY;u>Q@rXalUenvRix98w$Ddv~|uZ^NDyNsZ;xdeiKW4jbEPAqAy`eD&S-1 z8dU9ho-hWP3U@mKo@-HY0WV|}y5C?)5{;HFCg8Q@bduQOA@`IdA=;VAf3T|FM86U< zQ_mytC%udX6mfOx(3sywf53>ve1?q#Y-Chz8&xwVYB5&NzL&7rt89+3_c16r*60o8 z(pt-v6fnf7+Iq^HZyPORtc^iotdX5u+G@Fy0!lNjo!!=e;gIEDSvZcL#*%y|c@`i` z&%gdX`TvCJkMQ|Ze3o_Zr#Sr;KEKK5cdFm-lKXcoPnNw!f62#JTqSFG-)d;lmt+A? zR_pxqkMjbMtV%t>Y1t1C2E)7-u|6zr0UzezVE*)64YEPk$DVdvbdw(^Egc==T%uYX6PT7xOgycj+rL z0+~m=Q~GCEzi;zd<_doO{*|vU;=#v1DEsMUNRz?pA}-G|KL0NLWL)`Mrk9~eb~p3q z_Zp8aGO+xb(=uM1#oe(C7JhyS+$OP*q-&Qx+fR>;=w`<8kr2*!S6-h-1kzutOsd9- zY$4Owx-q<*KD{w1e0$VLpUDp{$9H^um&wm>6mCZQb2nQ*B*#hRiiD7(Q#mUhL1niJY{r&iy zLsIlsyTJ6uHxtn(Mhvo~DuY)e92HP&>sqQVI0(j-sEKz}p(cKgy1yp=GU~D=#`qCSiX3yDJf#a@2cf!Cx{9zF`*p;wpRq zm^tdTxBM5*0HRLKQ_|192i){gPANVJOav}qI3%jS3~#vfq(oXwAaw=n%tO@E=<(z9D*q+CxiUN%}3Jbm0i_LNN$1^*)B zTUZX&6CFRsALT1g_s`T=>`cjcd3bMY8jM6QkLe#^`tyOOd}BU6uVlPzt|;}jiRE0y z^R?SpPLk@r#sfFZjG;1PudU%MWv!zPi2d2cF9H5|7iY4%TE1 z{+#n#1f|b^!T6XDPxEZD!yazeNb2Khjj!&1PO+RxpPaWeeRV(fp1RuQx(ayGUxHQ3 z?anoff0pGBvOXIafBpOxjiGi$M+A7%|9;kA?BB!qdl)bF-^KXbeDuSNzspBY+oIK~ zml@zme~EuAuRQN#IY(HIw3pj79^=Kz7Oi(&|0?xC#;;gy@XNUZ?iD=KZ)N(AG5$6m z{*#Q4tuk^%{-cbSm&DevoX;@+Ri+pG=NT{WMbBmWFEV~D(+mDM@U`qSn?FCJ<(#Wd zZZdkZctvk322{15)Ad^Ne++sA&aWCDf1~A8kAseGXVw0BKJWomJr2GP`14S|@(#-s zWru^__appC9{05xz|%J$grBpd1u2CWv&_+}@ti0-jm?6;i{&^ATQs$dp!dx}|0wY1 z!v9CO9~1w7d=~m|vHT)G5tVj9-m3Q z!==E(`c=nuv*0^t!SB}id3F~`nMn-JLO%_>why;Q(esaHp??tgbD;kYrWb`D()9Dx z8;e>r>N|YlQO1AsLWBP((sVqIGvyofjW1u&^oFMV_v^FZ>1F!!YRo@>r1A6ALtKxi zv#_57Px;CdGAS>;0ZsKQPpxHH=QIFBf0xgH<^f-;z7}izJS8v9q&eSJv(SgM{6$JO zu!S1>h-mzUN;az%JiT6BE8ncf&r^5${B|$P8T09X2h+>reI)fkTXNTu|Hv%(udp0> z_AYXcGyYF}_W#Z-a(=}0r(MvZ^|+AR*D1!!3oUXMv0%F0RCKrr2dib{_9yyFWlbwXGD*3$(F|AZ@<*^^VF@&jC|2QfQ+cUyocNS zU$Lk&fIk=Zl>L;2e!j+g_k$vT3CsDzB}V?OEN6|Tzfev3{I*%sGf~-T+bsAYmM`x| zZepHk#_wkPWElU!S>$|d7W`*e&V>tFw1l6qoG&r{^(6-1&-ic6BIgz0&w>5#=khYS z`YG^KFZcNT?LQd5+h?EOX!-M$f1N3SiqQ4Nc~!g0J_~$9ncf2aWEMFJ6Gf`uJ-+%q9eAp*abLSTN8?4R zcV03JehtaTE>$uQ5srE?|d_2VdA+c#qu%JaLIXKOaY6(e z?8D-^TiXy39r58^S0Tz@W*9MYDUj2+i$KZ~SPZU#a$SThG0|q+!9q5Nm{P?(*bHiX zk^-;{Hm+HH_0^DEg9xEdCPvL7H6r{Yb`j!_(6TlHJdYz55-)P=l!ToxfI(|!t_xZp;v5hG}x{2>NVG{HV8yQfH!5oarrQZHl`32VgRktC`W45j0po2Yt~?5On0IAc0Wv!BL` za#R+!Ovgp{Q<54wd(;Vs;76c39VUvmmDjT}0EZC9` z99T(K<_P4*I6UQ|4vy=zHLJY!jgh(=~QhOw0vgwg~KI@Jkw*6SfhO4U-<^^hJP71|F6|cj% z+i-*wA5QzHOF`0+OLbg14r{1cvXz-oaHkrTp-o3?(#5)aA7c3QdG|ue_sr(Lk93C>z!-sT98tBo~%z*PxOek{8)+v&5?y)o;N2L z(NIndSni(%y%In*M~q7C!5flIICnj?o8hXDW^e zQuM4z_)pfgAf@oUTc6bZoMAnZRCeKCWG<^7oUxQ@`Q>@|2;Vq-(`PVWQoo6ccfPIu z^4$D-%$q&M(jZ zW6Xa<3dRED{!GsI`S@iW;8iAgSr(#w{G3#O=Hr+53BJADP_P+=yvqNNFu$lN?;X6v z{I|c$vT8u|7VM{d{PKRnV6&m1=Y8b|yR$`o9uSd8{_-BfiW>~cLvjPm98!OrR9^-_ zrlvNa4|z`^Vv>G(fBI>VQ4xh--irvnL;mN%OZp4HygzZyJLLa9c**yKU*5A=#QZO@ z{lq^-f2oI8eEG}!7_Uch0f*3=l$RF#PjR7KfBW7>%y8`H{Ac6;xsPAggLm^n(;HGS z79e_yyx#yIU4%^5TXrN3#adtfetvqvf=E^va{DX%?8X)^;^Z6zAh0C@@FC9sWH}PB zpZ0=T_|w!e&-Im@q~f9Wjzp^q7k$UNaRXFvbSS@`e$l%c}ae*1m) zuMN}x^z}c&FXt`HUzUXi-_M47^SzZKc>r?uka{d)>n$+bB%y}731Nf76BjX44ueI_|(az2|{>|3``}6<5f3olR literal 0 HcmV?d00001 diff --git a/baro_tests/tests.py b/baro_tests/tests.py index 80335ad9..7d19d3f4 100644 --- a/baro_tests/tests.py +++ b/baro_tests/tests.py @@ -1,7 +1,6 @@ -"""Function for testing collectd plug-ins with different oup plug-ins""" # -*- coding: utf-8 -*- -#Licensed under the Apache License, Version 2.0 (the "License"); you may +# Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at # @@ -13,9 +12,18 @@ # License for the specific language governing permissions and limitations # under the License. +"""Function for testing collectd plug-ins with different oup plug-ins""" + import time +def test_gnocchi_node_sends_data( + node_id, interval, logger, client, criteria_list=[], + resource_id_substrings=['']): + logger.info("Gnocchi test cases will be coming soon!!") + return False + + def test_ceilometer_node_sends_data( node_id, interval, logger, client, criteria_list=[], resource_id_substrings=['']): @@ -43,7 +51,8 @@ def test_ceilometer_node_sends_data( Return latest entry from meter list which contains given node string and (if defined) subsrting. """ - res = [entry for entry in meterlist if node_str in entry['resource_id'] + res = [ + entry for entry in meterlist if node_str in entry['resource_id'] and substr in entry['resource_id']] if res: return res[0] @@ -54,24 +63,30 @@ def test_ceilometer_node_sends_data( timestamps = {} node_str = 'node-{}'.format(node_id) if node_id else '' - logger.info('Searching for timestamps of latest entries{0}{1}{2}...'.format( - '' if node_str == '' else ' for {}'.format(node_str), - '' if len(criteria_list) == 0 else (' for criteria ' + ', '.join(criteria_list)), - '' if resource_id_substrings == [''] else ' and resource ID substrings "{}"'.format( - '", "'.join(resource_id_substrings)))) + logger.info( + 'Searching for timestamps of latest entries{0}{1}{2}...'.format( + '' if node_str == '' else ' for {}'.format(node_str), + '' if len(criteria_list) == 0 else ( + ' for criteria ' + ', '.join(criteria_list)), + '' if resource_id_substrings == [''] else + ' and resource ID substrings "{}"'.format( + '", "'.join(resource_id_substrings)))) for criterion in criteria_list if len(criteria_list) > 0 else [None]: - meter_list = client.get_ceil_metrics(criterion) + meter_list = client.get_gnocchi_metrics(criterion) for resource_id_substring in resource_id_substrings: - last_entry = _search_meterlist_latest_entry(meter_list, node_str, resource_id_substring) + last_entry = _search_meterlist_latest_entry( + meter_list, node_str, resource_id_substring) if len(last_entry) == 0: logger.error('Entry{0}{1}{2} not found'.format( '' if node_str == '' else ' for {}'.format(node_str), - '' if criterion is None else 'for criterion {}'.format(criterion), - '' if resource_id_substring == '' - else 'and resource ID substring "{}"'.format(resource_id_substring))) + '' if criterion is None else 'for criterion {}'.format( + criterion), + '' if resource_id_substring == '' else 'and resource ' + + 'ID substring "{}"'.format(resource_id_substring))) return False timestamp = last_entry['timestamp'] - logger.debug('Last entry found: {0} {1}'.format(timestamp, last_entry['resource_id'])) + logger.debug('Last entry found: {0} {1}'.format( + timestamp, last_entry['resource_id'])) timestamps[(criterion, resource_id_substring)] = timestamp attempt = 1 @@ -87,11 +102,14 @@ def test_ceilometer_node_sends_data( + '(interval is {} sec)...'.format(interval)) time.sleep(sleep_time) - logger.info('Searching for timestamps of latest entries{}{}{}...'.format( - '' if node_str == '' else ' for {}'.format(node_str), - '' if len(criteria_list) == 0 else (' for criteria ' + ', '.join(criteria_list)), - '' if resource_id_substrings == [''] - else ' and resource ID substrings "{}"'.format('", "'.join(resource_id_substrings)))) + logger.info( + 'Searching for timestamps of latest entries{}{}{}...' .format( + '' if node_str == '' else ' for {}'.format(node_str), + '' if len(criteria_list) == 0 else ( + ' for criteria ' + ', ' .join(criteria_list)), + '' if resource_id_substrings == [''] + else ' and resource ID substrings "{}"' .format( + '", "'.join(resource_id_substrings)))) for criterion in criteria_list if len(criteria_list) > 0 else [None]: meter_list = client.get_ceil_metrics(criterion) for resource_id_substring in resource_id_substrings: @@ -100,19 +118,25 @@ def test_ceilometer_node_sends_data( if len(last_entry) == 0: logger.error('Entry{0}{1}{2} not found'.format( '' if node_str == '' else ' for {}'.format(node_str), - '' if criterion is None else 'for criterion {}'.format(criterion), - '' if resource_id_substring == '' - else ' and resource ID substring "{}"'.format(resource_id_substring))) + '' if criterion is None else 'for criterion {}'.format( + criterion), + '' if resource_id_substring == '' else ' and resource' + + 'ID substring "{}"'.format(resource_id_substring))) return False timestamp = last_entry['timestamp'] - logger.debug('Last entry found: {} {}'.format(timestamp, last_entry['resource_id'])) + logger.debug('Last entry found: {} {}'.format( + timestamp, last_entry['resource_id'])) if timestamp == timestamps[(criterion, resource_id_substring)]: logger.warning( - 'Last entry{0}{1}{2} has the same timestamp as before the sleep'.format( - '' if node_str == '' else ' for {}'.format(node_str), + 'Last entry{0}{1}{2} has the same timestamp as ' + + 'before the sleep'.format( + '' if node_str == '' else ' for {}'.format( + node_str), '' if resource_id_substring == '' - else ', substring "{}"'.format(resource_id_substring), - '' if criterion is None else ' for criterion {}'.format(criterion))) + else ', substring "{}"'.format( + resource_id_substring), + '' if criterion is None else + ' for criterion {}'.format(criterion))) is_passed = False attempt += 1 if not is_passed: @@ -140,22 +164,28 @@ def test_csv_handles_plugin_data( Return boolean value indicating success or failure. """ - logger.info('Getting CSV metrics of plugin {} on compute node {}...'.format( - plugin, compute.get_id())) + logger.info( + 'Getting CSV metrics of plugin {} on compute node {}...' .format( + plugin, compute.get_id())) logger.debug('Interval: {}'.format(interval)) logger.debug('Plugin subdirs: {}'.format(plugin_subdirs)) logger.debug('Plugin meter categories: {}'.format(meter_categories)) - plugin_metrics = client.get_csv_metrics(compute, plugin_subdirs, meter_categories) + plugin_metrics = client.get_csv_metrics( + compute, plugin_subdirs, meter_categories) if len(plugin_metrics) < len(plugin_subdirs) * len(meter_categories): logger.error('Some plugin metrics not found') return False - logger.info('Checking that last two entries in metrics are corresponding to interval...') + logger.info( + 'Checking that last two entries in metrics are corresponding' + + 'to interval...') for metric in plugin_metrics: logger.debug('{0} {1} {2} ... '.format(metric[0], metric[1], metric[2])) if metric[3] - metric[2] != interval: - logger.error('Time of last two entries differ by {}, but interval is {}'.format( - metric[3] - metric[2], interval)) + logger.error( + 'Time of last two entries differ by ' + + '{}, but interval is {}'.format( + metric[3] - metric[2], interval)) return False else: logger.debug('OK') @@ -168,8 +198,10 @@ def test_csv_handles_plugin_data( + '(interval is {} sec)...'.format(interval)) time.sleep(sleep_time) - logger.info('Getting new metrics of compute node {}...'.format(compute.get_id())) - plugin_metrics2 = client.get_csv_metrics(compute, plugin_subdirs, meter_categories) + logger.info('Getting new metrics of compute node {}...'.format( + compute.get_name())) + plugin_metrics2 = client.get_csv_metrics( + compute, plugin_subdirs, meter_categories) if len(plugin_metrics2) < len(plugin_subdirs) * len(meter_categories): logger.error('Some plugin metrics not found') return False @@ -182,7 +214,8 @@ def test_csv_handles_plugin_data( return False for i in range(len(plugin_metrics2)): logger.debug('{0} {1} {2} - {3} {4} {5} ... '.format( - plugin_metrics[i][0], plugin_metrics[i][1], plugin_metrics[i][2], plugin_metrics2[i][0], + plugin_metrics[i][0], plugin_metrics[i][1], + plugin_metrics[i][2], plugin_metrics2[i][0], plugin_metrics2[i][1], plugin_metrics2[i][2])) if plugin_metrics[i] == plugin_metrics2[i]: logger.error('FAIL') -- 2.16.6