import os.path
import os
import re
-import subprocess
from opnfv.deployment import factory
ID_RSA_PATH = '/root/.ssh/id_rsa'
SSH_KEYS_SCRIPT = '/home/opnfv/barometer/baro_utils/get_ssh_keys.sh'
COLLECTD_CONF_DIR = '/etc/collectd/collectd.conf.d'
NOTIFICATION_FILE = '/var/log/python-notifications.dump'
COLLECTD_NOTIFICATION = '/etc/collectd_notification_dump.py'
-APEX_IP = subprocess.check_output("echo $INSTALLER_IP", shell=True)
+APEX_IP = os.getenv("INSTALLER_IP").rstrip('\n')
APEX_USER = 'root'
APEX_USER_STACK = 'stack'
APEX_PKEY = '/root/.ssh/id_rsa'
stderr_lines = stderr.readlines()
if stderr_lines:
self.__logger.warning(
- "'fuel node' command failed (try {}):".format(attempt))
+ "'Apex node' command failed (try {}):".format(attempt))
for line in stderr_lines:
self.__logger.debug(line.strip())
else:
fuel_node_passed = True
if attempt > 1:
self.__logger.info(
- "'fuel node' command passed (try {})".format(attempt))
+ "'Apex node' command passed (try {})".format(attempt))
attempt += 1
if not fuel_node_passed:
self.__logger.error(
- "'fuel node' command failed. This was the last try.")
+ "'Apex node' command failed. This was the last try.")
raise OSError(
- "'fuel node' command failed. This was the last try.")
+ "'Apex node' command failed. This was the last try.")
node_table = stdout.readlines()\
# skip table title and parse table values
if compute_name == node.get_dict()['name']:
stdout = node.run_cmd(
'cat /etc/collectd/collectd.conf.d/{}.conf'.format(plugin))
+ if stdout is None:
+ return default_interval
for line in stdout.split('\n'):
if 'Interval' in line:
- # line = line.strip('Interval')
return 1
return default_interval
if compute_name == node.get_dict()['name']:
stdout = node.run_cmd(
'cat /etc/collectd/collectd.conf.d/{}.conf' .format(plugin))
+ if stdout is None:
+ return default_values
for line in stdout.split('\n'):
if 'Interfaces' in line:
return line.split(' ', 1)[1]
Return boolean value whether Gnocchi is running.
"""
gnocchi_present = False
- lines = self.execute_command(
- 'source overcloudrc.v3;systemctl status openstack-gnocchi-api | '
- + 'grep running', controller.get_ip())
- for line in lines:
- if '(running)' in line:
- gnocchi_present = True
+ controller_name = controller.get_name()
+ nodes = get_apex_nodes()
+ for node in nodes:
+ if controller_name == node.get_dict()['name']:
+ node.put_file(
+ '/home/opnfv/functest/conf/openstack.creds',
+ 'overcloudrc.v3')
+ stdout = node.run_cmd(
+ "source overcloudrc.v3;"
+ + "openstack catalog list | grep gnocchi")
+ if stdout is None:
+ return False
+ elif 'gnocchi' in stdout:
+ gnocchi_present = True
+ return gnocchi_present
+ else:
+ return False
return gnocchi_present
def is_aodh_running(self, controller):
"""Check whether aodh service is running on controller
"""
aodh_present = False
- lines = self.execute_command(
- 'source overcloudrc.v3;systemctl openstack-aodh-api | grep running',
- controller.get_ip())
- for line in lines:
- self.__logger.info("Line = {}" .format(line))
- if '(running)' in line:
- aodh_present = True
+ controller_name = controller.get_name()
+ nodes = get_apex_nodes()
+ for node in nodes:
+ if controller_name == node.get_dict()['name']:
+ node.put_file(
+ '/home/opnfv/functest/conf/openstack.creds',
+ 'overcloudrc.v3')
+ stdout = node.run_cmd(
+ "source overcloudrc.v3;"
+ + "openstack catalog list | grep aodh")
+ if stdout is None:
+ return False
+ elif 'aodh' in stdout:
+ aodh_present = True
+ return aodh_present
+ else:
+ return False
return aodh_present
- def is_installed(self, compute, package):
+ def is_mcelog_installed(self, compute, package):
"""Check whether package exists on compute node.
Keyword arguments:
for node in nodes:
if compute_name == node.get_dict()['name']:
stdout = node.run_cmd(
- 'yum list installed | grep mcelog')
- if 'mcelog' in stdout:
+ 'rpm -qa | grep mcelog')
+ if stdout is None:
+ return 0
+ elif 'mcelog' in stdout:
return 1
else:
return 0
return True
return False
+ def check_aodh_plugin_included(self, compute):
+ """Check if aodh plugin is included in collectd.conf file.
+ If not, try to enable it.
+
+ Keyword arguments:
+ compute -- compute node instance
+
+ Return boolean value whether AODH plugin is included
+ or it's enabling was successful.
+ """
+ compute_name = compute.get_name()
+ nodes = get_apex_nodes()
+ for node in nodes:
+ if compute_name == node.get_dict()['name']:
+ aodh_conf = node.run_cmd('ls /etc/collectd/collectd.conf.d')
+ if 'aodh.conf' not in aodh_conf:
+ self.__logger.info(
+ "AODH Plugin not included in {}".format(compute_name))
+ return False
+ else:
+ self.__logger.info(
+ "AODH plugin present in compute node {}" .format(
+ compute_name))
+ return True
+ return True
+
def check_gnocchi_plugin_included(self, compute):
"""Check if gnocchi plugin is included in collectd.conf file.
If not, try to enable it.
nodes = get_apex_nodes()
for node in nodes:
if compute_name == node.get_dict()['name']:
- # node.run_cmd('su; "opnfvapex"')
gnocchi_conf = node.run_cmd('ls /etc/collectd/collectd.conf.d')
if 'collectd-ceilometer-plugin.conf' not in gnocchi_conf:
- self.__logger.info("Gnocchi Plugin not included")
- return True
+ self.__logger.info(
+ "Gnocchi Plugin not included in node {}".format(
+ compute_name))
+ return False
else:
- self.__logger.info("Gnochi plugin present")
+ self.__logger.info(
+ "Gnocchi plugin available in compute node {}" .format(
+ compute_name))
return True
return True
+ def check_snmp_plugin_included(self, compute):
+ """Check if SNMP plugin is active in compute node.
+ """
+ snmp_mib = '/usr/share/snmp/mibs/Intel-Rdt.txt'
+ snmp_string = 'INTEL-RDT-MIB::intelRdt'
+ compute_name = compute.get_name()
+ nodes = get_apex_nodes()
+ for node in nodes:
+ if compute_name == node.get_dict()['name']:
+ stdout = node.run_cmd(
+ 'snmpwalk -v2c -m {0} -c public localhost {1}' .format(
+ snmp_mib, snmp_string))
+ self.__logger.info("snmp output = {}" .format(stdout))
+ if 'OID' in stdout:
+ return False
+ else:
+ return True
+
def enable_plugins(
self, compute, plugins, error_plugins, create_backup=True):
"""Enable plugins on compute node
Keyword arguments:
compute -- compute node instance
plugins -- list of plugins to be enabled
- error_plugins -- list of tuples with found errors, new entries
- may be added there (plugin, error_description, is_critical):
- plugin -- plug-in name
- error_decription -- description of the error
- is_critical -- boolean value indicating whether error
- is critical
- create_backup -- boolean value indicating whether backup
- shall be created
Return boolean value indicating whether function was successful.
"""
+ csv_file = os.path.dirname(os.path.realpath(__file__)) + '/csv.conf'
plugins = sorted(plugins)
compute_name = compute.get_name()
nodes = get_apex_nodes()
for node in nodes:
if compute_name == node.get_dict()['name']:
- node.put_file(
- '/usr/local/lib/python2.7/dist-packages/baro_tests/'
- + 'csv.conf', 'csv.conf')
+ node.put_file(csv_file, 'csv.conf')
node.run_cmd(
'sudo cp csv.conf '
+ '/etc/collectd/collectd.conf.d/csv.conf')
return True
- def restore_config(self, compute):
- """Restore collectd config file from backup on compute node.
-
- Keyword arguments:
- compute -- compute node instance
- """
- ssh, sftp = self.__open_sftp_session(
- compute.get_ip(), 'root', 'opnfvapex')
-
- self.__logger.info('Restoring config file from backup...')
- ssh.exec_command("cp {0} {0}.used".format(COLLECTD_CONF))
- ssh.exec_command("cp {0}.backup {0}".format(COLLECTD_CONF))
-
def restart_collectd(self, compute):
"""Restart collectd on compute node.
return False, warning
return True, warning
- def test_gnocchi_is_sending_data(self, controller):
- """ Checking if Gnocchi is sending metrics to controller"""
- metric_ids = []
- timestamps1 = {}
- timestamps2 = {}
- ssh, sftp = self.__open_sftp_session(
- controller.get_ip(), 'root', 'opnfvapex')
-
- self.__logger.info('Getting gnocchi metric list on{}'.format(
- controller.get_name()))
- stdout = self.execute_command(
- "source overcloudrc.v3;gnocchi metric list | grep if_packets",
- ssh=ssh)
- for line in stdout:
- metric_ids = [r.split('|')[1] for r in stdout]
- self.__logger.info("Metric ids = {}" .format(metric_ids))
- for metric_id in metric_ids:
- metric_id = metric_id.replace("u", "")
- stdout = self.execute_command(
- "source overcloudrc.v3;gnocchi measures show {}" .format(
- metric_id), ssh=ssh)
- self.__logger.info("stdout measures ={}" .format(stdout))
- for line in stdout:
- if line[0] == '+':
- pass
- else:
- self.__logger.info("Line = {}" .format(line))
- timestamps1 = [line.split('|')[1]]
- self.__logger.info("Last line timetamp1 = {}" .format(timestamps1))
- time.sleep(10)
- stdout = self.execute_command(
- "source overcloudrc.v3;gnocchi measures show {}" .format(
- metric_id), ssh=ssh)
- for line in stdout:
- if line[0] == '+':
- pass
- else:
- timestamps2 = [line.split('|')[1]]
- self.__logger.info("Last line timetamp2 = {}" .format(timestamps2))
- if timestamps1 == timestamps2:
- self.__logger.info("False")
- # return False
- return True
- else:
- self.__logger.info("True")
- return True
+ def test_plugins_with_aodh(
+ self, compute, plugin_interval, logger,
+ criteria_list=[]):
- def test_plugins_with_aodh(self, controller):
- """Checking if AODH is sending metrics to controller"""
- metric_ids = []
+ metric_id = {}
timestamps1 = {}
timestamps2 = {}
- ssh, sftp = self.__open_sftp_session(
- controller.get_ip(), 'root', 'opnfvapex')
- self.__logger.info('Getting AODH alarm list on{}'.format(
- controller.get_name()))
- stdout = self.execute_command(
- "source overcloudrc.v3;aodh alarm list | grep mcelog",
- ssh=ssh)
- for line in stdout:
- metric_ids = [r.split('|')[1] for r in stdout]
- self.__logger.info("Metric ids = {}" .format(metric_ids))
- for metric_id in metric_ids:
- metric_id = metric_id.replace("u", "")
- stdout = self.execute_command(
- "source overcloudrc.v3;aodh alarm show {}" .format(
- metric_id), ssh=ssh)
- self.__logger.info("stdout alarms ={}" .format(stdout))
- for line in stdout:
- if line[0] == '+':
- pass
- else:
- self.__logger.info("Line = {}" .format(line))
- timestamps1 = [line.split('|')[1]]
- self.__logger.info("Last line timetamp1 = {}" .format(timestamps1))
- time.sleep(10)
- stdout = self.execute_command(
- "source overcloudrc.v3;aodh alarm show {}" .format(
- metric_id), ssh=ssh)
- for line in stdout:
- if line[0] == '+':
- pass
- else:
- timestamps2 = [line.split('|')[1]]
- self.__logger.info("Last line timetamp2 = {}" .format(timestamps2))
- if timestamps1 == timestamps2:
- self.__logger.info("False")
- # return False
- return True
- else:
- self.__logger.info("True")
- return True
+ nodes = get_apex_nodes()
+ for node in nodes:
+ if node.is_controller():
+ self.__logger.info('Getting AODH Alarm list on {}' .format(
+ (node.get_dict()['name'])))
+ node.put_file(
+ '/home/opnfv/functest/conf/openstack.creds',
+ 'overcloudrc.v3')
+ stdout = node.run_cmd(
+ "source overcloudrc.v3;"
+ + "aodh alarm list | grep {0} | grep {1}"
+ .format(criteria_list, compute))
+ if stdout is None:
+ self.__logger.info("aodh alarm list was empty")
+ return False
+ for line in stdout.splitlines():
+ line = line.replace('|', "")
+ metric_id = line.split()[0]
+ stdout = node.run_cmd(
+ 'source overcloudrc.v3; aodh alarm show {}' .format(
+ metric_id))
+ if stdout is None:
+ self.__logger.info("aodh alarm list was empty")
+ return False
+ for line in stdout.splitlines()[3: -1]:
+ line = line.replace('|', "")
+ if line.split()[0] == 'timestamp':
+ timestamps1 = line.split()[1]
+ else:
+ pass
+ time.sleep(12)
+ stdout = node.run_cmd(
+ "source overcloudrc.v3; aodh alarm show {}" .format(
+ metric_id))
+ if stdout is None:
+ self.__logger.info("aodh alarm list was empty")
+ return False
+ for line in stdout.splitlines()[3:-1]:
+ line = line.replace('|', "")
+ if line.split()[0] == 'timestamp':
+ timestamps2 = line.split()[1]
+ else:
+ pass
+ if timestamps1 == timestamps2:
+ self.__logger.info(
+ "Data not updated after interval of 12 seconds")
+ return False
+ else:
+ self.__logger.info("PASS")
+ return True
def test_plugins_with_gnocchi(
- self, controller, compute_node, plugin_interval, logger,
+ self, compute, plugin_interval, logger,
criteria_list=[]):
- metric_ids = []
+ metric_id = {}
timestamps1 = {}
timestamps2 = {}
- ssh, sftp = self.__open_sftp_session(
- controller.get_ip(), 'root', 'opnfvapex')
- self.__logger.info('Getting gnocchi metric list on{}'.format(
- controller.get_name()))
- stdout = self.execute_command(
- "source overcloudrc.v3;gnocchi metric list | grep {0} | grep {1}"
- .format(compute_node.get_name(), criteria_list), ssh=ssh)
- for line in stdout:
- metric_ids = [r.split('|')[1] for r in stdout]
- self.__logger.info("Metric ids = {}" .format(metric_ids))
- for metric_id in metric_ids:
- metric_id = metric_id.replace("u", "")
- stdout = self.execute_command(
- "source overcloudrc.v3;gnocchi measures show {}" .format(
- metric_id), ssh=ssh)
- self.__logger.info("stdout measures ={}" .format(stdout))
- for line in stdout:
- if line[0] == '+':
- pass
- else:
- self.__logger.info("Line = {}" .format(line))
- timestamps1 = [line.split('|')[1]]
- self.__logger.info("Last line timetamp1 = {}" .format(timestamps1))
- time.sleep(10)
- stdout = self.execute_command(
- "source overcloudrc.v3;gnocchi measures show {}" .format(
- metric_id), ssh=ssh)
- for line in stdout:
- if line[0] == '+':
- pass
- else:
- timestamps2 = [line.split('|')[1]]
- self.__logger.info("Last line timetamp2 = {}" .format(timestamps2))
- if timestamps1 == timestamps2:
- self.__logger.info("False")
- return False
- else:
- self.__logger.info("True")
- return True
+ nodes = get_apex_nodes()
+ sleep_time = plugin_interval + 2
+ for node in nodes:
+ if node.is_controller():
+ self.__logger.info('Getting gnocchi metric list on {}' .format(
+ (node.get_dict()['name'])))
+ node.put_file(
+ '/home/opnfv/functest/conf/openstack.creds',
+ 'overcloudrc.v3')
+ stdout = node.run_cmd(
+ "source overcloudrc.v3;"
+ + "gnocchi metric list | grep {0} | grep {1}"
+ .format(criteria_list, compute))
+ if stdout is None:
+ self.__logger.info("gnocchi list was empty")
+ return False
+ for line in stdout.splitlines():
+ line = line.replace('|', "")
+ metric_id = line.split()[0]
+ stdout = node.run_cmd(
+ 'source overcloudrc.v3;gnocchi measures show {}'.format(
+ metric_id))
+ if stdout is None:
+ self.__logger.info("gnocchi list was empty")
+ return False
+ for line in stdout.splitlines()[3: -1]:
+ if line[0] == '+':
+ pass
+ else:
+ timestamps1 = line.replace('|', "")
+ timestamps1 = timestamps1.split()[0]
+ time.sleep(sleep_time)
+ stdout = node.run_cmd(
+ "source overcloudrc.v3;gnocchi measures show {}".format(
+ metric_id))
+ if stdout is None:
+ self.__logger.info("gnocchi measures was empty")
+ return False
+ for line in stdout.splitlines()[3:-1]:
+ if line[0] == '+':
+ pass
+ else:
+ timestamps2 = line.replace('|', "")
+ timestamps2 = timestamps2.split()[0]
+ if timestamps1 == timestamps2:
+ self.__logger.info(
+ "Plugin Interval is {}" .format(plugin_interval))
+ self.__logger.info(
+ "Data not updated after {} seconds".format(
+ sleep_time))
+ return False
+ else:
+ self.__logger.info("PASS")
+ return True
+ return False
+
+ def test_plugins_with_snmp(
+ self, compute, plugin_interval, logger, plugin, snmp_mib_files=[],
+ snmp_mib_strings=[], snmp_in_commands=[]):
+
+ if plugin == 'hugepages' or 'intel_rdt' or 'mcelog':
+ nodes = get_apex_nodes()
+ for node in nodes:
+ if compute == node.get_dict()['name']:
+ stdout = node.run_cmd(
+ 'snmpwalk -v2c -m {0} -c public localhost {1}' .format(
+ snmp_mib_files, snmp_mib_strings))
+ self.__logger.info("{}" .format(stdout))
+ if stdout is None:
+ self.__logger.info("No output from snmpwalk")
+ return False
+ elif 'OID' in stdout:
+ self.__logger.info("SNMP query failed")
+ return False
+ else:
+ counter1 = stdout.split()[3]
+ time.sleep(10)
+ stdout = node.run_cmd(
+ 'snmpwalk -v2c -m {0} -c public localhost {1}' .format(
+ snmp_mib_files, snmp_mib_strings))
+ self.__logger.info("{}" .format(stdout))
+ if stdout is None:
+ self.__logger.info("No output from snmpwalk")
+ elif 'OID' in stdout:
+ self.__logger.info(
+ "SNMP query failed during second check")
+ self.__logger.info("waiting for 10 sec")
+ time.sleep(10)
+ stdout = node.run_cmd(
+ 'snmpwalk -v2c -m {0} -c public localhost {1}' .format(
+ snmp_mib_files, snmp_mib_strings))
+ self.__logger.info("{}" .format(stdout))
+ if stdout is None:
+ self.__logger.info("No output from snmpwalk")
+ elif 'OID' in stdout:
+ self.__logger.info("SNMP query failed again")
+ self.__logger.info("Failing this test case")
+ return False
+ else:
+ counter2 = stdout.split()[3]
+
+ if counter1 == counter2:
+ return False
+ else:
+ return True
+ else:
+ return False
Code Review / barometer.git / blobdiff