-"""Classes used by client.py"""
# -*- coding: utf-8 -*-
-
-#Licensed under the Apache License, Version 2.0 (the "License"); you may
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# License for the specific language governing permissions and limitations
# under the License.
-import paramiko
+"""Classes used by collectd.py"""
+
import time
-import string
import os.path
+import os
+import re
+import yaml
+
+from opnfv.deployment import factory
+import paramiko
+from functest.utils import constants
-ID_RSA_PATH = '/home/opnfv/.ssh/id_rsa'
+ID_RSA_PATH = '/root/.ssh/id_rsa'
SSH_KEYS_SCRIPT = '/home/opnfv/barometer/baro_utils/get_ssh_keys.sh'
DEF_PLUGIN_INTERVAL = 10
-COLLECTD_CONF = '/etc/collectd/collectd.conf'
+COLLECTD_CONF = '/etc/collectd.conf'
COLLECTD_CONF_DIR = '/etc/collectd/collectd.conf.d'
+NOTIFICATION_FILE = '/var/log/python-notifications.dump'
+COLLECTD_NOTIFICATION = '/etc/collectd_notification_dump.py'
+APEX_IP = os.getenv("INSTALLER_IP").rstrip('\n')
+APEX_USER = 'root'
+APEX_USER_STACK = 'stack'
+APEX_PKEY = '/root/.ssh/id_rsa'
class Node(object):
"""Node configuration class"""
def __init__(self, attrs):
- self.__id = int(attrs[0])
- self.__status = attrs[1]
+ self.__null = attrs[0]
+ self.__id = attrs[1]
self.__name = attrs[2]
- self.__cluster = int(attrs[3]) if attrs[3] else None
- self.__ip = attrs[4]
- self.__mac = attrs[5]
- self.__roles = [x.strip(' ') for x in attrs[6].split(',')]
- self.__pending_roles = attrs[7]
- self.__online = int(attrs[8]) if attrs[3] and attrs[8]else None
- self.__group_id = int(attrs[9]) if attrs[3] else None
+ self.__status = attrs[3] if attrs[3] else None
+ self.__taskState = attrs[4]
+ self.__pwrState = attrs[5]
+ self.__ip = re.sub('^[a-z]+=', '', attrs[6])
def get_name(self):
"""Get node name"""
return self.__ip
def get_roles(self):
- """Get node roles"""
+ """Get node role"""
return self.__roles
+def get_apex_nodes():
+ handler = factory.Factory.get_handler('apex',
+ APEX_IP,
+ APEX_USER_STACK,
+ APEX_PKEY)
+ nodes = handler.get_nodes()
+ return nodes
+
+
class ConfigServer(object):
"""Class to get env configuration"""
- def __init__(self, host, user, logger, passwd=None):
+ def __init__(self, host, user, logger, priv_key=None):
self.__host = host
self.__user = user
- self.__passwd = passwd
- self.__priv_key = None
+ self.__passwd = None
+ self.__priv_key = priv_key
self.__nodes = list()
self.__logger = logger
self.__private_key_file = ID_RSA_PATH
if not os.path.isfile(self.__private_key_file):
self.__logger.error(
- "Private key file '{}' not found.".format(self.__private_key_file))
- raise IOError("Private key file '{}' not found.".format(self.__private_key_file))
+ "Private key file '{}'".format(self.__private_key_file)
+ + " not found.")
+ raise IOError("Private key file '{}' not found.".format(
+ self.__private_key_file))
# get list of available nodes
- ssh, sftp = self.__open_sftp_session(self.__host, self.__user, self.__passwd)
+ ssh, sftp = self.__open_sftp_session(
+ self.__host, self.__user, self.__passwd)
attempt = 1
fuel_node_passed = False
while (attempt <= 10) and not fuel_node_passed:
- stdin, stdout, stderr = ssh.exec_command("fuel node")
+ stdin, stdout, stderr = ssh.exec_command(
+ "source stackrc; nova list")
stderr_lines = stderr.readlines()
if stderr_lines:
- self.__logger.warning("'fuel node' command failed (try {}):".format(attempt))
+ self.__logger.warning(
+ "'Apex node' command failed (try {}):".format(attempt))
for line in stderr_lines:
self.__logger.debug(line.strip())
else:
fuel_node_passed = True
if attempt > 1:
- self.__logger.info("'fuel node' command passed (try {})".format(attempt))
+ self.__logger.info(
+ "'Apex node' command passed (try {})".format(attempt))
attempt += 1
if not fuel_node_passed:
- self.__logger.error("'fuel node' command failed. This was the last try.")
- raise OSError("'fuel node' command failed. This was the last try.")
+ self.__logger.error(
+ "'Apex node' command failed. This was the last try.")
+ raise OSError(
+ "'Apex node' command failed. This was the last try.")
node_table = stdout.readlines()\
# skip table title and parse table values
- for entry in node_table[2:]:
- self.__nodes.append(Node([str(x.strip(' \n')) for x in entry.split('|')]))
+
+ for entry in node_table[3:]:
+ if entry[0] == '+' or entry[0] == '\n':
+ print entry
+ pass
+ else:
+ self.__nodes.append(
+ Node([str(x.strip(' \n')) for x in entry.split('|')]))
def get_controllers(self):
- """Get list of controllers"""
- return [node for node in self.__nodes if 'controller' in node.get_roles()]
+ # Get list of controllers
+ print self.__nodes[0]._Node__ip
+ return (
+ [node for node in self.__nodes if 'controller' in node.get_name()])
def get_computes(self):
- """Get list of computes"""
- return [node for node in self.__nodes if 'compute' in node.get_roles()]
+ # Get list of computes
+ return (
+ [node for node in self.__nodes if 'compute' in node.get_name()])
def get_nodes(self):
- """Get list of nodes"""
+ # Get list of nodes
return self.__nodes
def __open_sftp_session(self, host, user, passwd=None):
- """Connect to given host.
-
- Keyword arguments:
+ # Connect to given host.
+ """Keyword arguments:
host -- host to connect
user -- user to use
passwd -- password to use
# try a direct access using password or private key
if not passwd and not self.__priv_key:
# get private key
- self.__priv_key = paramiko.RSAKey.from_private_key_file(self.__private_key_file)
+ self.__priv_key = paramiko.RSAKey.from_private_key_file(
+ self.__private_key_file)
# connect to the server
- ssh.connect(host, username=user, password=passwd, pkey=self.__priv_key)
+ ssh.connect(
+ host, username=user, password=passwd, pkey=self.__priv_key)
sftp = ssh.open_sftp()
# return SFTP client instance
plugin -- plug-in name
If found, return interval value, otherwise the default value"""
- ssh, sftp = self.__open_sftp_session(compute.get_ip(), 'root')
- in_plugin = False
- plugin_name = ''
default_interval = DEF_PLUGIN_INTERVAL
- config_files = [COLLECTD_CONF] \
- + [COLLECTD_CONF_DIR + '/' + conf_file for conf_file in sftp.listdir(COLLECTD_CONF_DIR)]
- for config_file in config_files:
- try:
- with sftp.open(config_file) as config:
- for line in config.readlines():
- words = line.split()
- if len(words) > 1 and words[0] == '<LoadPlugin':
- in_plugin = True
- plugin_name = words[1].strip('">')
- if words and words[0] == '</LoadPlugin>':
- in_plugin = False
- if words and words[0] == 'Interval':
- if in_plugin and plugin_name == plugin:
- return int(words[1])
- if not in_plugin:
- default_interval = int(words[1])
- except IOError:
- self.__logger.error("Could not open collectd.conf file.")
+ compute_name = compute.get_name()
+ nodes = get_apex_nodes()
+ for node in nodes:
+ if compute_name == node.get_dict()['name']:
+ stdout = node.run_cmd(
+ 'cat /etc/collectd/collectd.conf.d/{}.conf'.format(plugin))
+ if stdout is None:
+ return default_interval
+ for line in stdout.split('\n'):
+ if 'Interval' in line:
+ return 1
return default_interval
def get_plugin_config_values(self, compute, plugin, parameter):
parameter -- plug-in parameter
Return list of found values."""
- ssh, sftp = self.__open_sftp_session(compute.get_ip(), 'root')
- # find the plugin value
- in_plugin = False
- plugin_name = ''
default_values = []
- config_files = [COLLECTD_CONF] \
- + [COLLECTD_CONF_DIR + '/' + conf_file for conf_file in sftp.listdir(COLLECTD_CONF_DIR)]
- for config_file in config_files:
- try:
- with sftp.open(config_file) as config:
- for line in config.readlines():
- words = line.split()
- if len(words) > 1 and words[0] == '<Plugin':
- in_plugin = True
- plugin_name = words[1].strip('">')
- if len(words) > 0 and words[0] == '</Plugin>':
- in_plugin = False
- if len(words) > 0 and words[0] == parameter:
- if in_plugin and plugin_name == plugin:
- return [word.strip('"') for word in words[1:]]
- except IOError:
- self.__logger.error("Could not open collectd.conf file.")
+ compute_name = compute.get_name()
+ nodes = get_apex_nodes()
+ for node in nodes:
+ if compute_name == node.get_dict()['name']:
+ stdout = node.run_cmd(
+ 'cat /etc/collectd/collectd.conf.d/{}.conf' .format(plugin))
+ if stdout is None:
+ return default_values
+ for line in stdout.split('\n'):
+ if 'Interfaces' in line:
+ return line.split(' ', 1)[1]
+ elif 'Bridges' in line:
+ return line.split(' ', 1)[1]
+ elif 'Cores' in line:
+ return line.split(' ', 1)[1]
+ else:
+ pass
return default_values
def execute_command(self, command, host_ip=None, ssh=None):
host_ip -- IP of the node
ssh -- existing open SSH session to use
- One of host_ip or ssh must not be None. If both are not None, existing ssh session is used.
+ One of host_ip or ssh must not be None. If both are not None,
+ existing ssh session is used.
"""
if host_ip is None and ssh is None:
raise ValueError('One of host_ip or ssh must not be None.')
if ssh is None:
- ssh, sftp = self.__open_sftp_session(host_ip, 'root')
+ ssh, sftp = self.__open_sftp_session(host_ip, 'root', 'opnfvapex')
stdin, stdout, stderr = ssh.exec_command(command)
return stdout.readlines()
Keyword arguments:
compute -- compute node instance
"""
- stdout = self.execute_command("ovs-vsctl list-br", compute.get_ip())
- return [interface.strip() for interface in stdout]
+ compute_name = compute.get_name()
+ nodes = get_apex_nodes()
+ for node in nodes:
+ if compute_name == node.get_dict()['name']:
+ stdout = node.run_cmd('sudo ovs-vsctl list-br')
+ return stdout
- def is_ceilometer_running(self, controller):
- """Check whether Ceilometer is running on controller.
+ def is_gnocchi_running(self, controller):
+ """Check whether Gnocchi is running on controller.
Keyword arguments:
controller -- controller node instance
- Return boolean value whether Ceilometer is running.
+ Return boolean value whether Gnocchi is running.
+ """
+ gnocchi_present = False
+ controller_name = controller.get_name()
+ nodes = get_apex_nodes()
+ for node in nodes:
+ if controller_name == node.get_dict()['name']:
+ node.put_file(constants.ENV_FILE, 'overcloudrc.v3')
+ stdout = node.run_cmd(
+ "source overcloudrc.v3;"
+ + "openstack catalog list | grep gnocchi")
+ if stdout is None:
+ return False
+ elif 'gnocchi' in stdout:
+ gnocchi_present = True
+ return gnocchi_present
+ else:
+ return False
+ return gnocchi_present
+
+ def is_aodh_running(self, controller):
+ """Check whether aodh service is running on controller
"""
- lines = self.execute_command('service --status-all | grep ceilometer', controller.get_ip())
- agent = False
- collector = False
- for line in lines:
- if '[ + ] ceilometer-agent-notification' in line:
- agent = True
- if '[ + ] ceilometer-collector' in line:
- collector = True
- return agent and collector
-
- def is_installed(self, compute, package):
+ aodh_present = False
+ controller_name = controller.get_name()
+ nodes = get_apex_nodes()
+ for node in nodes:
+ if controller_name == node.get_dict()['name']:
+ node.put_file(constants.ENV_FILE, 'overcloudrc.v3')
+ stdout = node.run_cmd(
+ "source overcloudrc.v3;"
+ + "openstack catalog list | grep aodh")
+ if stdout is None:
+ return False
+ elif 'aodh' in stdout:
+ aodh_present = True
+ return aodh_present
+ else:
+ return False
+ return aodh_present
+
+ def is_redis_running(self, compute):
+ """Check whether redis service is running on compute"""
+ compute_name = compute.get_name()
+ nodes = get_apex_nodes()
+ for node in nodes:
+ if compute_name == node.get_dict()['name']:
+ stdout = node.run_cmd('sudo systemctl status docker'
+ '&& sudo docker ps'
+ '| grep barometer-redis')
+ if stdout and 'barometer-redis' in stdout:
+ self.__logger.info(
+ 'Redis is running in node {}'.format(
+ compute_name))
+ return True
+ self.__logger.info(
+ 'Redis is *not* running in node {}'.format(
+ compute_name))
+ return False
+
+ def is_localagent_server_running(self, compute):
+ """Check whether LocalAgent server is running on compute"""
+ compute_name = compute.get_name()
+ nodes = get_apex_nodes()
+ for node in nodes:
+ if compute_name == node.get_dict()['name']:
+ stdout = node.run_cmd('sudo systemctl status docker'
+ '&& sudo docker ps'
+ '| grep opnfv/barometer-localagent')
+ if stdout and '/server' in stdout:
+ self.__logger.info(
+ 'LocalAgent Server is running in node {}'.format(
+ compute_name))
+ return True
+ self.__logger.info(
+ 'LocalAgent Server is *not* running in node {}'.format(
+ compute_name))
+ return False
+
+ def is_localagent_infofetch_running(self, compute):
+ """Check whether LocalAgent infofetch is running on compute"""
+ compute_name = compute.get_name()
+ nodes = get_apex_nodes()
+ for node in nodes:
+ if compute_name == node.get_dict()['name']:
+ stdout = node.run_cmd('sudo systemctl status docker'
+ '&& sudo docker ps'
+ '| grep opnfv/barometer-localagent')
+ if stdout and '/infofetch' in stdout:
+ self.__logger.info(
+ 'LocalAgent InfoFetch is running in node {}'.format(
+ compute_name))
+ return True
+ self.__logger.info(
+ 'LocalAgent InfoFetch is *not* running in node {}'.format(
+ compute_name))
+ return False
+
+ def get_localagent_config(self, compute):
+ """Get config values of LocalAgent"""
+ compute_name = compute.get_name()
+ nodes = get_apex_nodes()
+ for node in nodes:
+ if compute_name == node.get_dict()['name']:
+ # We use following after functest accept python-toml
+ # stdout = node.run_cmd(
+ # 'cat /etc/barometer-localagent/config.toml')
+ # try:
+ # agent_conf = toml.loads(stdout)
+ # except (TypeError, TomlDecodeError) as e:
+ # self.__logger.error(
+ # 'LocalAgent config error: {}'.format(e))
+ # agent_conf = None
+ # finally:
+ # return agent_conf
+ readcmd = (
+ 'egrep "listen_port|amqp_"'
+ ' /etc/barometer-localagent/config.toml'
+ '| sed -e "s/#.*$//" | sed -e "s/=/:/"'
+ )
+ stdout = node.run_cmd(readcmd)
+ agent_conf = {"server": yaml.load(stdout)}
+
+ pingcmd = (
+ 'ping -n -c1 ' + agent_conf["server"]["amqp_host"] +
+ '| sed -ne "s/^.*bytes from //p" | sed -e "s/:.*//"'
+ )
+ agent_conf["server"]["amqp_host"] = node.run_cmd(pingcmd)
+
+ return agent_conf
+ return None
+
+ def is_mcelog_installed(self, compute, package):
"""Check whether package exists on compute node.
Keyword arguments:
Return boolean value whether package is installed.
"""
- stdout = self.execute_command('dpkg -l | grep {}'.format(package), compute.get_ip())
- return len(stdout) > 0
+ compute_name = compute.get_name()
+ nodes = get_apex_nodes()
+ for node in nodes:
+ if compute_name == node.get_dict()['name']:
+ stdout = node.run_cmd(
+ 'rpm -qa | grep mcelog')
+ if stdout is None:
+ return 0
+ elif 'mcelog' in stdout:
+ return 1
+ else:
+ return 0
+
+ def is_rdt_available(self, compute):
+ """Check whether the compute node is a virtual machine."""
+ compute_name = compute.get_name()
+ nodes = get_apex_nodes()
+ for node in nodes:
+ if compute_name == node.get_dict()['name']:
+ stdout = node.run_cmd('cat /proc/cpuinfo | grep hypervisor')
+ if 'hypervisor' in stdout:
+ return False
+ return True
+
+ def is_libpqos_on_node(self, compute):
+ """Check whether libpqos is present on compute node"""
+
+ compute_name = compute.get_name()
+ nodes = get_apex_nodes()
+ for node in nodes:
+ if compute_name == node.get_dict()['name']:
+ stdout = node.run_cmd('ls /usr/local/lib/ | grep libpqos')
+ if 'libpqos' in stdout:
+ return True
+ return False
- def check_ceil_plugin_included(self, compute):
- """Check if ceilometer plugin is included in collectd.conf file If not,
- try to enable it.
+ def check_aodh_plugin_included(self, compute):
+ """Check if aodh plugin is included in collectd.conf file.
+ If not, try to enable it.
Keyword arguments:
compute -- compute node instance
- Return boolean value whether ceilometer plugin is included or it's enabling was successful.
+ Return boolean value whether AODH plugin is included
+ or it's enabling was successful.
"""
- ssh, sftp = self.__open_sftp_session(compute.get_ip(), 'root')
- try:
- config = sftp.open(COLLECTD_CONF, mode='r')
- except IOError:
- self.__logger.error(
- 'Cannot open {} on node {}'.format(COLLECTD_CONF, compute.get_id()))
- return False
- in_lines = config.readlines()
- out_lines = in_lines[:]
- include_section_indexes = [
- (start, end) for start in range(len(in_lines)) for end in range(len(in_lines))
- if (start < end)
- and '<Include' in in_lines[start]
- and COLLECTD_CONF_DIR in in_lines[start]
- and '#' not in in_lines[start]
- and '</Include>' in in_lines[end]
- and '#' not in in_lines[end]
- and len([i for i in in_lines[start + 1: end]
- if 'Filter' in i and '*.conf' in i and '#' not in i]) > 0]
- if len(include_section_indexes) == 0:
- out_lines.append('<Include "{}">\n'.format(COLLECTD_CONF_DIR))
- out_lines.append(' Filter "*.conf"\n')
- out_lines.append('</Include>\n')
- config.close()
- config = sftp.open(COLLECTD_CONF, mode='w')
- config.writelines(out_lines)
- config.close()
- self.__logger.info('Creating backup of collectd.conf...')
- config = sftp.open(COLLECTD_CONF + '.backup', mode='w')
- config.writelines(in_lines)
- config.close()
+ compute_name = compute.get_name()
+ nodes = get_apex_nodes()
+ for node in nodes:
+ if compute_name == node.get_dict()['name']:
+ aodh_conf = node.run_cmd('ls /etc/collectd/collectd.conf.d')
+ if 'aodh.conf' not in aodh_conf:
+ self.__logger.info(
+ "AODH Plugin not included in {}".format(compute_name))
+ return False
+ else:
+ self.__logger.info(
+ "AODH plugin present in compute node {}" .format(
+ compute_name))
+ return True
return True
- def enable_plugins(self, compute, plugins, error_plugins, create_backup=True):
- """Enable plugins on compute node
+ def check_gnocchi_plugin_included(self, compute):
+ """Check if gnocchi plugin is included in collectd.conf file.
+ If not, try to enable it.
Keyword arguments:
compute -- compute node instance
- plugins -- list of plugins to be enabled
- error_plugins -- list of tuples with found errors, new entries may be added there
- (plugin, error_description, is_critical):
- plugin -- plug-in name
- error_decription -- description of the error
- is_critical -- boolean value indicating whether error is critical
- create_backup -- boolean value indicating whether backup shall be created
- Return boolean value indicating whether function was successful.
+ Return boolean value whether gnocchi plugin is included
+ or it's enabling was successful.
"""
- plugins = sorted(plugins)
- ssh, sftp = self.__open_sftp_session(compute.get_ip(), 'root')
- plugins_to_enable = plugins[:]
- for plugin in plugins:
- plugin_file = '/usr/lib/collectd/{}.so'.format(plugin)
- try:
- sftp.stat(plugin_file)
- except IOError:
- self.__logger.debug(
- 'Plugin file {0} not found on node {1}, plugin {2} will not be enabled'.format(
- plugin_file, compute.get_id(), plugin))
- error_plugins.append((plugin, 'plugin file {} not found'.format(plugin_file), True))
- plugins_to_enable.remove(plugin)
- self.__logger.debug('Following plugins will be enabled on node {}: {}'.format(
- compute.get_id(), ', '.join(plugins_to_enable)))
- try:
- config = sftp.open(COLLECTD_CONF, mode='r')
- except IOError:
- self.__logger.warning(
- 'Cannot open {} on node {}'.format(COLLECTD_CONF, compute.get_id()))
- return False
- in_lines = config.readlines()
- out_lines = []
- enabled_plugins = []
- enabled_sections = []
- in_section = 0
- comment_section = False
- uncomment_section = False
- for line in in_lines:
- if 'LoadPlugin' in line:
- for plugin in plugins_to_enable:
- if plugin in line:
- commented = '#' in line
- #list of uncommented lines which contain LoadPlugin for this plugin
- loadlines = [
- ll for ll in in_lines if 'LoadPlugin' in ll
- and plugin in ll and '#' not in ll]
- if len(loadlines) == 0:
- if plugin not in enabled_plugins:
- line = line.lstrip(string.whitespace + '#')
- enabled_plugins.append(plugin)
- error_plugins.append((
- plugin, 'plugin not enabled in '
- + '{}, trying to enable it'.format(COLLECTD_CONF), False))
- elif not commented:
- if plugin not in enabled_plugins:
- enabled_plugins.append(plugin)
- else:
- line = '#' + line
- error_plugins.append((
- plugin, 'plugin enabled more than once '
- + '(additional occurrence of LoadPlugin found in '
- + '{}), trying to comment it out.'.format(
- COLLECTD_CONF), False))
- elif line.lstrip(string.whitespace + '#').find('<Plugin') == 0:
- in_section += 1
- for plugin in plugins_to_enable:
- if plugin in line:
- commented = '#' in line
- #list of uncommented lines which contain Plugin for this plugin
- pluginlines = [
- pl for pl in in_lines if '<Plugin' in pl
- and plugin in pl and '#' not in pl]
- if len(pluginlines) == 0:
- if plugin not in enabled_sections:
- line = line[line.rfind('#') + 1:]
- uncomment_section = True
- enabled_sections.append(plugin)
- error_plugins.append((
- plugin, 'plugin section found in '
- + '{}, but commented out, trying to uncomment it.'.format(
- COLLECTD_CONF), False))
- elif not commented:
- if plugin not in enabled_sections:
- enabled_sections.append(plugin)
- else:
- line = '#' + line
- comment_section = True
- error_plugins.append((
- plugin,
- 'additional occurrence of plugin section found in '
- + '{}, trying to comment it out.'.format(COLLECTD_CONF),
- False))
- elif in_section > 0:
- if comment_section and '#' not in line:
- line = '#' + line
- if uncomment_section and '#' in line:
- line = line[line.rfind('#') + 1:]
- if '</Plugin>' in line:
- in_section -= 1
- if in_section == 0:
- comment_section = False
- uncomment_section = False
- elif '</Plugin>' in line:
- self.__logger.error(
- 'Unexpected closure os plugin section on line'
- + ' {} in collectd.conf, matching section start not found.'.format(
- len(out_lines) + 1))
- return False
- out_lines.append(line)
- if in_section > 0:
- self.__logger.error(
- 'Unexpected end of file collectd.conf, '
- + 'closure of last plugin section not found.')
- return False
- out_lines = [
- 'LoadPlugin {}\n'.format(plugin) for plugin in plugins_to_enable
- if plugin not in enabled_plugins] + out_lines
- for plugin in plugins_to_enable:
- if plugin not in enabled_plugins:
- error_plugins.append((
- plugin,
- 'plugin not enabled in {}, trying to enable it.'.format(COLLECTD_CONF),
- False))
- unenabled_sections = [
- plugin for plugin in plugins_to_enable if plugin not in enabled_sections]
- if unenabled_sections:
- self.__logger.error('Plugin sections for following plugins not found: {}'.format(
- ', '.join(unenabled_sections)))
- return False
-
- config.close()
- if create_backup:
- self.__logger.info('Creating backup of collectd.conf...')
- config = sftp.open(COLLECTD_CONF + '.backup', mode='w')
- config.writelines(in_lines)
- config.close()
- self.__logger.info('Updating collectd.conf...')
- config = sftp.open(COLLECTD_CONF, mode='w')
- config.writelines(out_lines)
- config.close()
- diff_command = "diff {} {}.backup".format(COLLECTD_CONF, COLLECTD_CONF)
- stdin, stdout, stderr = ssh.exec_command(diff_command)
- self.__logger.debug(diff_command)
- for line in stdout.readlines():
- self.__logger.debug(line.strip())
+ compute_name = compute.get_name()
+ nodes = get_apex_nodes()
+ for node in nodes:
+ if compute_name == node.get_dict()['name']:
+ gnocchi_conf = node.run_cmd('ls /etc/collectd/collectd.conf.d')
+ if 'collectd-ceilometer-plugin.conf' not in gnocchi_conf:
+ self.__logger.info(
+ "Gnocchi Plugin not included in node {}".format(
+ compute_name))
+ return False
+ else:
+ self.__logger.info(
+ "Gnocchi plugin available in compute node {}" .format(
+ compute_name))
+ return True
return True
- def restore_config(self, compute):
- """Restore collectd config file from backup on compute node.
+ def check_snmp_plugin_included(self, compute):
+ """Check if SNMP plugin is active in compute node.
+ """
+ snmp_mib = '/usr/share/snmp/mibs/Intel-Rdt.txt'
+ snmp_string = 'INTEL-RDT-MIB::intelRdt'
+ compute_name = compute.get_name()
+ nodes = get_apex_nodes()
+ for node in nodes:
+ if compute_name == node.get_dict()['name']:
+ stdout = node.run_cmd(
+ 'snmpwalk -v2c -m {0} -c public localhost {1}' .format(
+ snmp_mib, snmp_string))
+ self.__logger.info("snmp output = {}" .format(stdout))
+ if 'OID' in stdout:
+ return False
+ else:
+ return True
+
+ def enable_plugins(
+ self, compute, plugins, error_plugins, create_backup=True):
+ """Enable plugins on compute node
Keyword arguments:
compute -- compute node instance
- """
- ssh, sftp = self.__open_sftp_session(compute.get_ip(), 'root')
+ plugins -- list of plugins to be enabled
- self.__logger.info('Restoring config file from backup...')
- ssh.exec_command("cp {0} {0}.used".format(COLLECTD_CONF))
- ssh.exec_command("cp {0}.backup {0}".format(COLLECTD_CONF))
+ Return boolean value indicating whether function was successful.
+ """
+ csv_file = os.path.dirname(os.path.realpath(__file__)) + '/csv.conf'
+ plugins = sorted(plugins)
+ compute_name = compute.get_name()
+ nodes = get_apex_nodes()
+ for node in nodes:
+ if compute_name == node.get_dict()['name']:
+ node.put_file(csv_file, 'csv.conf')
+ node.run_cmd(
+ 'sudo cp csv.conf '
+ + '/etc/collectd/collectd.conf.d/csv.conf')
+ return True
def restart_collectd(self, compute):
"""Restart collectd on compute node.
Keyword arguments:
compute -- compute node instance
- Retrun tuple with boolean indicating success and list of warnings received
- during collectd start.
+ Retrun tuple with boolean indicating success and list of warnings
+ received during collectd start.
"""
+ compute_name = compute.get_name()
+ nodes = get_apex_nodes()
- def get_collectd_processes(ssh_session):
+ def get_collectd_processes(compute_node):
"""Get number of running collectd processes.
Keyword arguments:
- ssh_session -- instance of SSH session in which to check for processes
+ ssh_session -- instance of SSH session in which to check
+ for processes
"""
- stdin, stdout, stderr = ssh_session.exec_command("pgrep collectd")
- return len(stdout.readlines())
-
- ssh, sftp = self.__open_sftp_session(compute.get_ip(), 'root')
-
- self.__logger.info('Stopping collectd service...')
- stdout = self.execute_command("service collectd stop", ssh=ssh)
- time.sleep(10)
- if get_collectd_processes(ssh):
- self.__logger.error('Collectd is still running...')
- return False, []
- self.__logger.info('Starting collectd service...')
- stdout = self.execute_command("service collectd start", ssh=ssh)
- time.sleep(10)
- warning = [output.strip() for output in stdout if 'WARN: ' in output]
- if get_collectd_processes(ssh) == 0:
- self.__logger.error('Collectd is still not running...')
- return False, warning
+ stdout = compute_node.run_cmd("pgrep collectd")
+ return len(stdout)
+
+ for node in nodes:
+ if compute_name == node.get_dict()['name']:
+ # node.run_cmd('su; "opnfvapex"')
+ self.__logger.info('Stopping collectd service...')
+ node.run_cmd('sudo systemctl stop collectd')
+ time.sleep(10)
+ if get_collectd_processes(node):
+ self.__logger.error('Collectd is still running...')
+ return False, []
+ self.__logger.info('Starting collectd service...')
+ stdout = node.run_cmd('sudo systemctl start collectd')
+ time.sleep(10)
+ warning = [
+ output.strip() for output in stdout if 'WARN: ' in output]
+ if get_collectd_processes(node) == 0:
+ self.__logger.error('Collectd is still not running...')
+ return False, warning
return True, warning
+
+ def trigger_alarm_update(self, alarm, compute_node):
+ # TODO: move these actions to main, with criteria lists so that we can reference that
+ # i.e. test_plugin_with_aodh(self, compute, plugin.., logger, criteria_list, alarm_action)
+ if alarm == 'mcelog':
+ compute_node.run_cmd('sudo modprobe mce-inject')
+ compute_node.run_cmd('sudo ./mce-inject_ea < corrected')
+ if alarm == 'ovs_events':
+ compute_node.run_cmd('sudo ifconfig -a | grep br0')
+ compute_node.run_cmd('sudo ifconfig br0 down; sudo ifconfig br0 up')
+
+ def test_plugins_with_aodh(
+ self, compute, plugin_interval, logger,
+ criteria_list=[]):
+
+ metric_id = {}
+ timestamps1 = {}
+ timestamps2 = {}
+ nodes = get_apex_nodes()
+ compute_node = [node for node in nodes if node.get_dict()['name'] == compute][0]
+ for node in nodes:
+ if node.is_controller():
+ self.__logger.info('Getting AODH Alarm list on {}' .format(
+ (node.get_dict()['name'])))
+ node.put_file(constants.ENV_FILE, 'overcloudrc.v3')
+ self.trigger_alarm_update(criteria_list, compute_node)
+ stdout = node.run_cmd(
+ "source overcloudrc.v3;"
+ + "aodh alarm list | grep {0} | grep {1}"
+ .format(criteria_list, compute))
+ if stdout is None:
+ self.__logger.info("aodh alarm list was empty")
+ return False
+ for line in stdout.splitlines():
+ line = line.replace('|', "")
+ metric_id = line.split()[0]
+ stdout = node.run_cmd(
+ 'source overcloudrc.v3; aodh alarm show {}' .format(
+ metric_id))
+ if stdout is None:
+ self.__logger.info("aodh alarm list was empty")
+ return False
+ for line in stdout.splitlines()[3: -1]:
+ line = line.replace('|', "")
+ if line.split()[0] == 'state_timestamp':
+ timestamps1 = line.split()[1]
+ self.trigger_alarm_update(criteria_list, compute_node)
+ time.sleep(12)
+ stdout = node.run_cmd(
+ "source overcloudrc.v3; aodh alarm show {}" .format(
+ metric_id))
+ if stdout is None:
+ self.__logger.info("aodh alarm list was empty")
+ return False
+ for line in stdout.splitlines()[3:-1]:
+ line = line.replace('|', "")
+ if line.split()[0] == 'state_timestamp':
+ timestamps2 = line.split()[1]
+ if timestamps1 == timestamps2:
+ self.__logger.info(
+ "Data not updated after interval of 12 seconds")
+ return False
+ else:
+ self.__logger.info("PASS")
+ return True
+
+ def test_plugins_with_gnocchi(
+ self, compute, plugin_interval, logger,
+ criteria_list=[]):
+
+ metric_id = {}
+ timestamps1 = {}
+ timestamps2 = {}
+ nodes = get_apex_nodes()
+ if plugin_interval > 15:
+ sleep_time = plugin_interval*2
+ else:
+ sleep_time = 30
+
+ for node in nodes:
+ if node.is_controller():
+ self.__logger.info('Getting gnocchi metric list on {}' .format(
+ (node.get_dict()['name'])))
+ node.put_file(constants.ENV_FILE, 'overcloudrc.v3')
+ stdout = node.run_cmd(
+ "source overcloudrc.v3;"
+ + "gnocchi metric list | grep {0} | grep {1}"
+ .format(criteria_list, compute))
+ if stdout is None:
+ self.__logger.info("gnocchi list was empty")
+ return False
+ for line in stdout.splitlines():
+ line = line.replace('|', "")
+ metric_id = line.split()[0]
+ stdout = node.run_cmd(
+ 'source overcloudrc.v3;gnocchi measures show {}'.format(
+ metric_id))
+ if stdout is None:
+ self.__logger.info("gnocchi list was empty")
+ return False
+ for line in stdout.splitlines()[3: -1]:
+ if line[0] == '+':
+ pass
+ else:
+ timestamps1 = line.replace('|', "")
+ timestamps1 = timestamps1.split()[0]
+ time.sleep(sleep_time)
+ stdout = node.run_cmd(
+ "source overcloudrc.v3;gnocchi measures show {}".format(
+ metric_id))
+ if stdout is None:
+ self.__logger.info("gnocchi measures was empty")
+ return False
+ for line in stdout.splitlines()[3:-1]:
+ if line[0] == '+':
+ pass
+ else:
+ timestamps2 = line.replace('|', "")
+ timestamps2 = timestamps2.split()[0]
+ if timestamps1 == timestamps2:
+ self.__logger.info(
+ "Plugin Interval is {}" .format(plugin_interval))
+ self.__logger.info(
+ "Data not updated after {} seconds".format(
+ sleep_time))
+ return False
+ else:
+ self.__logger.info("PASS")
+ return True
+ return False
+
+ def test_plugins_with_snmp(
+ self, compute, plugin_interval, logger, plugin, snmp_mib_files=[],
+ snmp_mib_strings=[], snmp_in_commands=[]):
+
+ if plugin in ('hugepages', 'intel_rdt', 'mcelog'):
+ nodes = get_apex_nodes()
+ for node in nodes:
+ if compute == node.get_dict()['name']:
+ stdout = node.run_cmd(
+ 'snmpwalk -v2c -m {0} -c public localhost {1}' .format(
+ snmp_mib_files, snmp_mib_strings))
+ self.__logger.info("{}" .format(stdout))
+ if stdout is None:
+ self.__logger.info("No output from snmpwalk")
+ return False
+ elif 'OID' in stdout:
+ self.__logger.info("SNMP query failed")
+ return False
+ else:
+ counter1 = stdout.split()[3]
+ time.sleep(10)
+ stdout = node.run_cmd(
+ 'snmpwalk -v2c -m {0} -c public localhost {1}' .format(
+ snmp_mib_files, snmp_mib_strings))
+ self.__logger.info("{}" .format(stdout))
+ if stdout is None:
+ self.__logger.info("No output from snmpwalk")
+ elif 'OID' in stdout:
+ self.__logger.info(
+ "SNMP query failed during second check")
+ self.__logger.info("waiting for 10 sec")
+ time.sleep(10)
+ stdout = node.run_cmd(
+ 'snmpwalk -v2c -m {0} -c public localhost {1}' .format(
+ snmp_mib_files, snmp_mib_strings))
+ self.__logger.info("{}" .format(stdout))
+ if stdout is None:
+ self.__logger.info("No output from snmpwalk")
+ elif 'OID' in stdout:
+ self.__logger.info("SNMP query failed again")
+ self.__logger.info("Failing this test case")
+ return False
+ else:
+ counter2 = stdout.split()[3]
+
+ if counter1 == counter2:
+ return False
+ else:
+ return True
+ else:
+ return False
+
+ def check_localagent_dummy_included(self, compute, name):
+ """Check if dummy collectd config by LocalAgent
+ is included in collectd.conf file.
+
+ Keyword arguments:
+ compute -- compute node instance
+ name -- config file name
+ """
+ compute_name = compute.get_name()
+ nodes = get_apex_nodes()
+ for node in nodes:
+ if compute_name == node.get_dict()['name']:
+ dummy_conf = node.run_cmd('ls /etc/collectd/collectd.conf.d')
+ if name + '.conf' not in dummy_conf:
+ self.__logger.error('check conf FAIL')
+ return False
+ else:
+ self.__logger.info('check conf PASS')
+ fullpath = '/etc/collectd/collectd.conf.d/{}'.format(
+ name + '.conf')
+ self.__logger.info('Delete file {}'.format(fullpath))
+ node.run_cmd('sudo rm -f ' + fullpath)
+ return True
+ self.__logger.error('Some panic, compute not found')
+ return False
+
+ def create_testvm(self, compute_node, test_name):
+ nodes = get_apex_nodes()
+ compute_name = compute_node.get_name()
+
+ controller_node = None
+ for node in nodes:
+ if node.is_controller():
+ controller_node = node
+ break
+
+ self.__logger.debug('Creating Test VM on {}' .format(compute_name))
+ self.__logger.debug('Create command is executed in {}' .format(
+ (controller_node.get_dict()['name'])))
+
+ image_filename = 'cirros-0.4.0-x86_64-disk.img'
+ controller_node.run_cmd(
+ 'curl -sO '
+ 'http://download.cirros-cloud.net/0.4.0/'
+ + image_filename)
+
+ node.put_file(constants.ENV_FILE, 'overcloudrc.v3')
+ image = controller_node.run_cmd(
+ 'source overcloudrc.v3;'
+ 'openstack image create -f value -c id'
+ ' --disk-format qcow2 --file {0} {1}'
+ .format(image_filename, test_name))
+ flavor = controller_node.run_cmd(
+ 'source overcloudrc.v3;'
+ 'openstack flavor create -f value -c id {}'
+ .format(test_name))
+ host = controller_node.run_cmd(
+ 'source overcloudrc.v3;'
+ 'openstack hypervisor list -f value -c "Hypervisor Hostname"'
+ ' | grep "^{}\\."'
+ .format(compute_name))
+ server = controller_node.run_cmd(
+ 'source overcloudrc.v3;'
+ 'openstack server create -f value -c id'
+ ' --image {0} --flavor {1} --availability-zone {2} {3}'
+ .format(image, flavor, 'nova:' + host, test_name))
+
+ resources = {"image": image, "flavor": flavor, "server": server}
+
+ if server:
+ self.__logger.debug('VM created')
+ self.__logger.debug('VM info: {}'.format(resources))
+
+ return resources
+
+ def delete_testvm(self, resources):
+ nodes = get_apex_nodes()
+
+ controller_node = None
+ for node in nodes:
+ if node.is_controller():
+ controller_node = node
+ break
+
+ self.__logger.debug('Deleteing Test VM')
+ self.__logger.debug('VM to be deleted info: {}'.format(resources))
+ self.__logger.debug('Delete command is executed in {}' .format(
+ (controller_node.get_dict()['name'])))
+
+ server = resources.get('server', None)
+ flavor = resources.get('flavor', None)
+ image = resources.get('image', None)
+ if server:
+ controller_node.run_cmd(
+ 'source overcloudrc.v3;'
+ 'openstack server delete {}'.format(server))
+ if flavor:
+ controller_node.run_cmd(
+ 'source overcloudrc.v3;'
+ 'openstack flavor delete {}'.format(flavor))
+ if image:
+ controller_node.run_cmd(
+ 'source overcloudrc.v3;'
+ 'openstack image delete {}'.format(image))
+
+ self.__logger.debug('VM and other OpenStack resources deleted')
+
+ def test_localagent_infofetch_get_data(self, compute, test_name):
+ compute_name = compute.get_name()
+ nodes = get_apex_nodes()
+ for node in nodes:
+ if compute_name == node.get_dict()['name']:
+ stdout = node.run_cmd(
+ 'redis-cli keys "barometer-localagent/vm/*/vminfo"'
+ ' | while read k; do redis-cli get $k; done'
+ ' | grep {}'.format(test_name))
+ self.__logger.debug('InfoFetch data: {}'.format(stdout))
+ if stdout and test_name in stdout:
+ self.__logger.info('PASS')
+ return True
+ else:
+ self.__logger.info('No test vm info')
+
+ self.__logger.info('FAIL')
+ return False