X-Git-Url: https://gerrit.opnfv.org/gerrit/gitweb?a=blobdiff_plain;f=baro_tests%2Fconfig_server.py;h=a6849f0529fcab6df95709d6ebbc66485a1d49af;hb=HEAD;hp=102a237720b980d0eba0745375b57b397f32796a;hpb=ac54d72c06e3af986bbd1f939003744568c3ac84;p=barometer.git diff --git a/baro_tests/config_server.py b/baro_tests/config_server.py index 102a2377..a6849f05 100644 --- a/baro_tests/config_server.py +++ b/baro_tests/config_server.py @@ -1,26 +1,32 @@ # -*- coding: utf-8 -*- # +# Copyright(c) 2017-2019 Intel Corporation and OPNFV. All rights reserved. +# # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. +# """Classes used by collectd.py""" -import paramiko import time import os.path import os import re -import subprocess +import yaml + from opnfv.deployment import factory +import paramiko +from functest.utils import constants + ID_RSA_PATH = '/root/.ssh/id_rsa' SSH_KEYS_SCRIPT = '/home/opnfv/barometer/baro_utils/get_ssh_keys.sh' DEF_PLUGIN_INTERVAL = 10 @@ -28,10 +34,12 @@ COLLECTD_CONF = '/etc/collectd.conf' COLLECTD_CONF_DIR = '/etc/collectd/collectd.conf.d' NOTIFICATION_FILE = '/var/log/python-notifications.dump' COLLECTD_NOTIFICATION = '/etc/collectd_notification_dump.py' -APEX_IP = subprocess.check_output("echo $INSTALLER_IP", shell=True) +APEX_IP = os.getenv("INSTALLER_IP").rstrip('\n') APEX_USER = 'root' APEX_USER_STACK = 'stack' APEX_PKEY = '/root/.ssh/id_rsa' +TEST_VM_IMAGE = 'cirros-0.4.0-x86_64-disk.img' +TEST_VM_IMAGE_PATH = '/home/opnfv/functest/images/' + TEST_VM_IMAGE class Node(object): @@ -101,20 +109,20 @@ class ConfigServer(object): stderr_lines = stderr.readlines() if stderr_lines: self.__logger.warning( - "'fuel node' command failed (try {}):".format(attempt)) + "'Apex node' command failed (try {}):".format(attempt)) for line in stderr_lines: self.__logger.debug(line.strip()) else: fuel_node_passed = True if attempt > 1: self.__logger.info( - "'fuel node' command passed (try {})".format(attempt)) + "'Apex node' command passed (try {})".format(attempt)) attempt += 1 if not fuel_node_passed: self.__logger.error( - "'fuel node' command failed. This was the last try.") + "'Apex node' command failed. This was the last try.") raise OSError( - "'fuel node' command failed. This was the last try.") + "'Apex node' command failed. This was the last try.") node_table = stdout.readlines()\ # skip table title and parse table values @@ -184,9 +192,10 @@ class ConfigServer(object): if compute_name == node.get_dict()['name']: stdout = node.run_cmd( 'cat /etc/collectd/collectd.conf.d/{}.conf'.format(plugin)) + if stdout is None: + return default_interval for line in stdout.split('\n'): if 'Interval' in line: - # line = line.strip('Interval') return 1 return default_interval @@ -206,6 +215,8 @@ class ConfigServer(object): if compute_name == node.get_dict()['name']: stdout = node.run_cmd( 'cat /etc/collectd/collectd.conf.d/{}.conf' .format(plugin)) + if stdout is None: + return default_values for line in stdout.split('\n'): if 'Interfaces' in line: return line.split(' ', 1)[1] @@ -261,14 +272,17 @@ class ConfigServer(object): nodes = get_apex_nodes() for node in nodes: if controller_name == node.get_dict()['name']: - node.put_file( - '/home/opnfv/functest/conf/openstack.creds', - 'overcloudrc.v3') + node.put_file(constants.ENV_FILE, 'overcloudrc.v3') stdout = node.run_cmd( "source overcloudrc.v3;" + "openstack catalog list | grep gnocchi") - if 'gnocchi' in stdout: + if stdout is None: + return False + elif 'gnocchi' in stdout: gnocchi_present = True + return gnocchi_present + else: + return False return gnocchi_present def is_aodh_running(self, controller): @@ -279,16 +293,110 @@ class ConfigServer(object): nodes = get_apex_nodes() for node in nodes: if controller_name == node.get_dict()['name']: - node.put_file( - '/home/opnfv/functest/conf/openstack.creds', - 'overcloudrc.v3') + node.put_file(constants.ENV_FILE, 'overcloudrc.v3') stdout = node.run_cmd( "source overcloudrc.v3;" + "openstack catalog list | grep aodh") - if 'aodh' in stdout: + if stdout is None: + return False + elif 'aodh' in stdout: aodh_present = True + return aodh_present + else: + return False return aodh_present + def is_redis_running(self, compute): + """Check whether redis service is running on compute""" + compute_name = compute.get_name() + nodes = get_apex_nodes() + for node in nodes: + if compute_name == node.get_dict()['name']: + stdout = node.run_cmd('sudo systemctl status docker' + '&& sudo docker ps' + '| grep barometer-redis') + if stdout and 'barometer-redis' in stdout: + self.__logger.info( + 'Redis is running in node {}'.format( + compute_name)) + return True + self.__logger.info( + 'Redis is *not* running in node {}'.format( + compute_name)) + return False + + def is_dma_server_running(self, compute): + """Check whether DMA server is running on compute""" + compute_name = compute.get_name() + nodes = get_apex_nodes() + for node in nodes: + if compute_name == node.get_dict()['name']: + stdout = node.run_cmd('sudo systemctl status docker' + '&& sudo docker ps' + '| grep opnfv/barometer-dma') + if stdout and '/server' in stdout: + self.__logger.info( + 'DMA Server is running in node {}'.format( + compute_name)) + return True + self.__logger.info( + 'DMA Server is *not* running in node {}'.format( + compute_name)) + return False + + def is_dma_infofetch_running(self, compute): + """Check whether DMA infofetch is running on compute""" + compute_name = compute.get_name() + nodes = get_apex_nodes() + for node in nodes: + if compute_name == node.get_dict()['name']: + stdout = node.run_cmd('sudo systemctl status docker' + '&& sudo docker ps' + '| grep opnfv/barometer-dma') + if stdout and '/infofetch' in stdout: + self.__logger.info( + 'DMA InfoFetch is running in node {}'.format( + compute_name)) + return True + self.__logger.info( + 'DMA InfoFetch is *not* running in node {}'.format( + compute_name)) + return False + + def get_dma_config(self, compute): + """Get config values of DMA""" + compute_name = compute.get_name() + nodes = get_apex_nodes() + for node in nodes: + if compute_name == node.get_dict()['name']: + # We use following after functest accept python-toml + # stdout = node.run_cmd( + # 'cat /etc/barometer-dma/config.toml') + # try: + # agent_conf = toml.loads(stdout) + # except (TypeError, TomlDecodeError) as e: + # self.__logger.error( + # 'DMA config error: {}'.format(e)) + # agent_conf = None + # finally: + # return agent_conf + readcmd = ( + 'egrep "listen_port|amqp_"' + ' /etc/barometer-dma/config.toml' + '| sed -e "s/#.*$//" | sed -e "s/=/:/"' + ) + stdout = node.run_cmd(readcmd) + agent_conf = {"server": yaml.safe_load(stdout)} + + pingcmd = ( + 'ping -n -c1 ' + agent_conf["server"]["amqp_host"] + + '| sed -ne "s/^.*bytes from //p" | sed -e "s/:.*//"' + ) + agent_conf["server"]["amqp_host"] = node.run_cmd(pingcmd) + + return agent_conf + return None + def is_mcelog_installed(self, compute, package): """Check whether package exists on compute node. @@ -303,12 +411,25 @@ class ConfigServer(object): for node in nodes: if compute_name == node.get_dict()['name']: stdout = node.run_cmd( - 'yum list installed | grep mcelog') - if 'mcelog' in stdout: + 'rpm -qa | grep mcelog') + if stdout is None: + return 0 + elif 'mcelog' in stdout: return 1 else: return 0 + def is_rdt_available(self, compute): + """Check whether the compute node is a virtual machine.""" + compute_name = compute.get_name() + nodes = get_apex_nodes() + for node in nodes: + if compute_name == node.get_dict()['name']: + stdout = node.run_cmd('cat /proc/cpuinfo | grep hypervisor') + if 'hypervisor' in stdout: + return False + return True + def is_libpqos_on_node(self, compute): """Check whether libpqos is present on compute node""" @@ -338,7 +459,7 @@ class ConfigServer(object): aodh_conf = node.run_cmd('ls /etc/collectd/collectd.conf.d') if 'aodh.conf' not in aodh_conf: self.__logger.info( - "AODH Plugin not included in compute node") + "AODH Plugin not included in {}".format(compute_name)) return False else: self.__logger.info( @@ -363,11 +484,13 @@ class ConfigServer(object): if compute_name == node.get_dict()['name']: gnocchi_conf = node.run_cmd('ls /etc/collectd/collectd.conf.d') if 'collectd-ceilometer-plugin.conf' not in gnocchi_conf: - self.__logger.info("Gnocchi Plugin not included") + self.__logger.info( + "Gnocchi Plugin not included in node {}".format( + compute_name)) return False else: self.__logger.info( - "Gnochi plugin available in compute node {}" .format( + "Gnocchi plugin available in compute node {}" .format( compute_name)) return True return True @@ -400,14 +523,13 @@ class ConfigServer(object): Return boolean value indicating whether function was successful. """ + csv_file = os.path.dirname(os.path.realpath(__file__)) + '/csv.conf' plugins = sorted(plugins) compute_name = compute.get_name() nodes = get_apex_nodes() for node in nodes: if compute_name == node.get_dict()['name']: - node.put_file( - '/usr/local/lib/python2.7/dist-packages/baro_tests/' - + 'csv.conf', 'csv.conf') + node.put_file(csv_file, 'csv.conf') node.run_cmd( 'sudo cp csv.conf ' + '/etc/collectd/collectd.conf.d/csv.conf') @@ -454,6 +576,16 @@ class ConfigServer(object): return False, warning return True, warning + def trigger_alarm_update(self, alarm, compute_node): + # TODO: move these actions to main, with criteria lists so that we can reference that + # i.e. test_plugin_with_aodh(self, compute, plugin.., logger, criteria_list, alarm_action) + if alarm == 'mcelog': + compute_node.run_cmd('sudo modprobe mce-inject') + compute_node.run_cmd('sudo ./mce-inject_ea < corrected') + if alarm == 'ovs_events': + compute_node.run_cmd('sudo ifconfig -a | grep br0') + compute_node.run_cmd('sudo ifconfig br0 down; sudo ifconfig br0 up') + def test_plugins_with_aodh( self, compute, plugin_interval, logger, criteria_list=[]): @@ -462,39 +594,45 @@ class ConfigServer(object): timestamps1 = {} timestamps2 = {} nodes = get_apex_nodes() + compute_node = [node for node in nodes if node.get_dict()['name'] == compute][0] for node in nodes: if node.is_controller(): self.__logger.info('Getting AODH Alarm list on {}' .format( (node.get_dict()['name']))) - node.put_file( - '/home/opnfv/functest/conf/openstack.creds', - 'overcloudrc.v3') + node.put_file(constants.ENV_FILE, 'overcloudrc.v3') + self.trigger_alarm_update(criteria_list, compute_node) stdout = node.run_cmd( "source overcloudrc.v3;" + "aodh alarm list | grep {0} | grep {1}" .format(criteria_list, compute)) + if stdout is None: + self.__logger.info("aodh alarm list was empty") + return False for line in stdout.splitlines(): line = line.replace('|', "") metric_id = line.split()[0] stdout = node.run_cmd( 'source overcloudrc.v3; aodh alarm show {}' .format( metric_id)) + if stdout is None: + self.__logger.info("aodh alarm list was empty") + return False for line in stdout.splitlines()[3: -1]: line = line.replace('|', "") - if line.split()[0] == 'timestamp': + if line.split()[0] == 'state_timestamp': timestamps1 = line.split()[1] - else: - pass + self.trigger_alarm_update(criteria_list, compute_node) time.sleep(12) stdout = node.run_cmd( "source overcloudrc.v3; aodh alarm show {}" .format( metric_id)) + if stdout is None: + self.__logger.info("aodh alarm list was empty") + return False for line in stdout.splitlines()[3:-1]: line = line.replace('|', "") - if line.split()[0] == 'timestamp': + if line.split()[0] == 'state_timestamp': timestamps2 = line.split()[1] - else: - pass if timestamps1 == timestamps2: self.__logger.info( "Data not updated after interval of 12 seconds") @@ -511,33 +649,45 @@ class ConfigServer(object): timestamps1 = {} timestamps2 = {} nodes = get_apex_nodes() + if plugin_interval > 15: + sleep_time = plugin_interval*2 + else: + sleep_time = 30 + for node in nodes: if node.is_controller(): self.__logger.info('Getting gnocchi metric list on {}' .format( (node.get_dict()['name']))) - node.put_file( - '/home/opnfv/functest/conf/openstack.creds', - 'overcloudrc.v3') + node.put_file(constants.ENV_FILE, 'overcloudrc.v3') stdout = node.run_cmd( "source overcloudrc.v3;" + "gnocchi metric list | grep {0} | grep {1}" .format(criteria_list, compute)) + if stdout is None: + self.__logger.info("gnocchi list was empty") + return False for line in stdout.splitlines(): line = line.replace('|', "") metric_id = line.split()[0] stdout = node.run_cmd( 'source overcloudrc.v3;gnocchi measures show {}'.format( metric_id)) + if stdout is None: + self.__logger.info("gnocchi list was empty") + return False for line in stdout.splitlines()[3: -1]: if line[0] == '+': pass else: timestamps1 = line.replace('|', "") timestamps1 = timestamps1.split()[0] - time.sleep(10) + time.sleep(sleep_time) stdout = node.run_cmd( "source overcloudrc.v3;gnocchi measures show {}".format( metric_id)) + if stdout is None: + self.__logger.info("gnocchi measures was empty") + return False for line in stdout.splitlines()[3:-1]: if line[0] == '+': pass @@ -545,17 +695,22 @@ class ConfigServer(object): timestamps2 = line.replace('|', "") timestamps2 = timestamps2.split()[0] if timestamps1 == timestamps2: - self.__logger.info("Data not updated after 12 seconds") + self.__logger.info( + "Plugin Interval is {}" .format(plugin_interval)) + self.__logger.info( + "Data not updated after {} seconds".format( + sleep_time)) return False else: self.__logger.info("PASS") return True + return False def test_plugins_with_snmp( self, compute, plugin_interval, logger, plugin, snmp_mib_files=[], snmp_mib_strings=[], snmp_in_commands=[]): - if plugin == 'intel_rdt': + if plugin in ('hugepages', 'intel_rdt', 'mcelog'): nodes = get_apex_nodes() for node in nodes: if compute == node.get_dict()['name']: @@ -563,7 +718,10 @@ class ConfigServer(object): 'snmpwalk -v2c -m {0} -c public localhost {1}' .format( snmp_mib_files, snmp_mib_strings)) self.__logger.info("{}" .format(stdout)) - if 'OID' in stdout: + if stdout is None: + self.__logger.info("No output from snmpwalk") + return False + elif 'OID' in stdout: self.__logger.info("SNMP query failed") return False else: @@ -573,7 +731,9 @@ class ConfigServer(object): 'snmpwalk -v2c -m {0} -c public localhost {1}' .format( snmp_mib_files, snmp_mib_strings)) self.__logger.info("{}" .format(stdout)) - if 'OID' in stdout: + if stdout is None: + self.__logger.info("No output from snmpwalk") + elif 'OID' in stdout: self.__logger.info( "SNMP query failed during second check") self.__logger.info("waiting for 10 sec") @@ -582,7 +742,9 @@ class ConfigServer(object): 'snmpwalk -v2c -m {0} -c public localhost {1}' .format( snmp_mib_files, snmp_mib_strings)) self.__logger.info("{}" .format(stdout)) - if 'OID' in stdout: + if stdout is None: + self.__logger.info("No output from snmpwalk") + elif 'OID' in stdout: self.__logger.info("SNMP query failed again") self.__logger.info("Failing this test case") return False @@ -593,3 +755,126 @@ class ConfigServer(object): return False else: return True + else: + return False + + def check_dma_dummy_included(self, compute, name): + """Check if dummy collectd config by DMA + is included in collectd.conf file. + + Keyword arguments: + compute -- compute node instance + name -- config file name + """ + compute_name = compute.get_name() + nodes = get_apex_nodes() + for node in nodes: + if compute_name == node.get_dict()['name']: + dummy_conf = node.run_cmd('ls /etc/collectd/collectd.conf.d') + if name + '.conf' not in dummy_conf: + self.__logger.error('check conf FAIL') + return False + else: + self.__logger.info('check conf PASS') + fullpath = '/etc/collectd/collectd.conf.d/{}'.format( + name + '.conf') + self.__logger.info('Delete file {}'.format(fullpath)) + node.run_cmd('sudo rm -f ' + fullpath) + return True + self.__logger.error('Some panic, compute not found') + return False + + def create_testvm(self, compute_node, test_name): + nodes = get_apex_nodes() + compute_name = compute_node.get_name() + + controller_node = None + for node in nodes: + if node.is_controller(): + controller_node = node + break + + self.__logger.debug('Creating Test VM on {}' .format(compute_name)) + self.__logger.debug('Create command is executed in {}' .format( + (controller_node.get_dict()['name']))) + + node.put_file(constants.ENV_FILE, 'overcloudrc.v3') + node.put_file(TEST_VM_IMAGE_PATH, TEST_VM_IMAGE) + image = controller_node.run_cmd( + 'source overcloudrc.v3;' + 'openstack image create -f value -c id' + ' --disk-format qcow2 --file {0} {1}' + .format(TEST_VM_IMAGE, test_name)) + flavor = controller_node.run_cmd( + 'source overcloudrc.v3;' + 'openstack flavor create -f value -c id {}' + .format(test_name)) + host = controller_node.run_cmd( + 'source overcloudrc.v3;' + 'openstack hypervisor list -f value -c "Hypervisor Hostname"' + ' | grep "^{}\\."' + .format(compute_name)) + server = controller_node.run_cmd( + 'source overcloudrc.v3;' + 'openstack server create -f value -c id' + ' --image {0} --flavor {1} --availability-zone {2} {3}' + .format(image, flavor, 'nova:' + host, test_name)) + + resources = {"image": image, "flavor": flavor, "server": server} + + if server: + self.__logger.debug('VM created') + self.__logger.debug('VM info: {}'.format(resources)) + + return resources + + def delete_testvm(self, resources): + nodes = get_apex_nodes() + + controller_node = None + for node in nodes: + if node.is_controller(): + controller_node = node + break + + self.__logger.debug('Deleteing Test VM') + self.__logger.debug('VM to be deleted info: {}'.format(resources)) + self.__logger.debug('Delete command is executed in {}' .format( + (controller_node.get_dict()['name']))) + + server = resources.get('server', None) + flavor = resources.get('flavor', None) + image = resources.get('image', None) + if server: + controller_node.run_cmd( + 'source overcloudrc.v3;' + 'openstack server delete {}'.format(server)) + if flavor: + controller_node.run_cmd( + 'source overcloudrc.v3;' + 'openstack flavor delete {}'.format(flavor)) + if image: + controller_node.run_cmd( + 'source overcloudrc.v3;' + 'openstack image delete {}'.format(image)) + + self.__logger.debug('VM and other OpenStack resources deleted') + + def test_dma_infofetch_get_data(self, compute, test_name): + compute_name = compute.get_name() + nodes = get_apex_nodes() + for node in nodes: + if compute_name == node.get_dict()['name']: + stdout = node.run_cmd( + 'redis-cli keys "barometer-dma/vm/*/vminfo"' + ' | while read k; do redis-cli get $k; done' + ' | grep {}'.format(test_name)) + self.__logger.debug('InfoFetch data: {}'.format(stdout)) + if stdout and test_name in stdout: + self.__logger.info('PASS') + return True + else: + self.__logger.info('No test vm info') + + self.__logger.info('FAIL') + return False