baro_tests/config_server.py

   1 # -*- coding: utf-8 -*-
   2 #
   3 # Licensed under the Apache License, Version 2.0 (the "License"); you may
   4 # not use this file except in compliance with the License. You may obtain
   5 # a copy of the License at
   6 #
   7 #      http://www.apache.org/licenses/LICENSE-2.0
   8 #
   9 # Unless required by applicable law or agreed to in writing, software
  10 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  11 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  12 # License for the specific language governing permissions and limitations
  13 # under the License.
  14
  15 """Classes used by collectd.py"""
  16
  17 import paramiko
  18 import time
  19 import os.path
  20 import os
  21 import re
  22 import subprocess
  23 from opnfv.deployment import factory
  24 ID_RSA_PATH = '/root/.ssh/id_rsa'
  25 SSH_KEYS_SCRIPT = '/home/opnfv/barometer/baro_utils/get_ssh_keys.sh'
  26 DEF_PLUGIN_INTERVAL = 10
  27 COLLECTD_CONF = '/etc/collectd.conf'
  28 COLLECTD_CONF_DIR = '/etc/collectd/collectd.conf.d'
  29 NOTIFICATION_FILE = '/var/log/python-notifications.dump'
  30 COLLECTD_NOTIFICATION = '/etc/collectd_notification_dump.py'
  31 APEX_IP = subprocess.check_output("echo $INSTALLER_IP", shell=True)
  32 APEX_USER = 'root'
  33 APEX_USER_STACK = 'stack'
  34 APEX_PKEY = '/root/.ssh/id_rsa'
  35
  36
  37 class Node(object):
  38     """Node configuration class"""
  39     def __init__(self, attrs):
  40         self.__null = attrs[0]
  41         self.__id = attrs[1]
  42         self.__name = attrs[2]
  43         self.__status = attrs[3] if attrs[3] else None
  44         self.__taskState = attrs[4]
  45         self.__pwrState = attrs[5]
  46         self.__ip = re.sub('^[a-z]+=', '', attrs[6])
  47
  48     def get_name(self):
  49         """Get node name"""
  50         return self.__name
  51
  52     def get_id(self):
  53         """Get node ID"""
  54         return self.__id
  55
  56     def get_ip(self):
  57         """Get node IP address"""
  58         return self.__ip
  59
  60     def get_roles(self):
  61         """Get node role"""
  62         return self.__roles
  63
  64
  65 def get_apex_nodes():
  66     handler = factory.Factory.get_handler('apex',
  67                                           APEX_IP,
  68                                           APEX_USER_STACK,
  69                                           APEX_PKEY)
  70     nodes = handler.get_nodes()
  71     return nodes
  72
  73
  74 class ConfigServer(object):
  75     """Class to get env configuration"""
  76     def __init__(self, host, user, logger, priv_key=None):
  77         self.__host = host
  78         self.__user = user
  79         self.__passwd = None
  80         self.__priv_key = priv_key
  81         self.__nodes = list()
  82         self.__logger = logger
  83
  84         self.__private_key_file = ID_RSA_PATH
  85         if not os.path.isfile(self.__private_key_file):
  86             self.__logger.error(
  87                 "Private key file '{}'".format(self.__private_key_file)
  88                 + " not found.")
  89             raise IOError("Private key file '{}' not found.".format(
  90                 self.__private_key_file))
  91
  92         # get list of available nodes
  93         ssh, sftp = self.__open_sftp_session(
  94             self.__host, self.__user, self.__passwd)
  95         attempt = 1
  96         fuel_node_passed = False
  97
  98         while (attempt <= 10) and not fuel_node_passed:
  99             stdin, stdout, stderr = ssh.exec_command(
 100                 "source stackrc; nova list")
 101             stderr_lines = stderr.readlines()
 102             if stderr_lines:
 103                 self.__logger.warning(
 104                     "'fuel node' command failed (try {}):".format(attempt))
 105                 for line in stderr_lines:
 106                     self.__logger.debug(line.strip())
 107             else:
 108                 fuel_node_passed = True
 109                 if attempt > 1:
 110                     self.__logger.info(
 111                         "'fuel node' command passed (try {})".format(attempt))
 112             attempt += 1
 113         if not fuel_node_passed:
 114             self.__logger.error(
 115                 "'fuel node' command failed. This was the last try.")
 116             raise OSError(
 117                 "'fuel node' command failed. This was the last try.")
 118         node_table = stdout.readlines()\
 119
 120         # skip table title and parse table values
 121
 122         for entry in node_table[3:]:
 123             if entry[0] == '+' or entry[0] == '\n':
 124                 print entry
 125                 pass
 126             else:
 127                 self.__nodes.append(
 128                     Node([str(x.strip(' \n')) for x in entry.split('|')]))
 129
 130     def get_controllers(self):
 131         # Get list of controllers
 132         print self.__nodes[0]._Node__ip
 133         return (
 134             [node for node in self.__nodes if 'controller' in node.get_name()])
 135
 136     def get_computes(self):
 137         # Get list of computes
 138         return (
 139             [node for node in self.__nodes if 'compute' in node.get_name()])
 140
 141     def get_nodes(self):
 142         # Get list of nodes
 143         return self.__nodes
 144
 145     def __open_sftp_session(self, host, user, passwd=None):
 146         # Connect to given host.
 147         """Keyword arguments:
 148         host -- host to connect
 149         user -- user to use
 150         passwd -- password to use
 151
 152         Return tuple of SSH and SFTP client instances.
 153         """
 154         # create SSH client
 155         ssh = paramiko.SSHClient()
 156         ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
 157
 158         # try a direct access using password or private key
 159         if not passwd and not self.__priv_key:
 160             # get private key
 161             self.__priv_key = paramiko.RSAKey.from_private_key_file(
 162                 self.__private_key_file)
 163
 164         # connect to the server
 165         ssh.connect(
 166             host, username=user, password=passwd, pkey=self.__priv_key)
 167         sftp = ssh.open_sftp()
 168
 169         # return SFTP client instance
 170         return ssh, sftp
 171
 172     def get_plugin_interval(self, compute, plugin):
 173         """Find the plugin interval in collectd configuration.
 174
 175         Keyword arguments:
 176         compute -- compute node instance
 177         plugin -- plug-in name
 178
 179         If found, return interval value, otherwise the default value"""
 180         default_interval = DEF_PLUGIN_INTERVAL
 181         compute_name = compute.get_name()
 182         nodes = get_apex_nodes()
 183         for node in nodes:
 184             if compute_name == node.get_dict()['name']:
 185                 stdout = node.run_cmd(
 186                     'cat /etc/collectd/collectd.conf.d/{}.conf'.format(plugin))
 187                 for line in stdout.split('\n'):
 188                     if 'Interval' in line:
 189                         # line = line.strip('Interval')
 190                         return 1
 191         return default_interval
 192
 193     def get_plugin_config_values(self, compute, plugin, parameter):
 194         """Get parameter values from collectd config file.
 195
 196         Keyword arguments:
 197         compute -- compute node instance
 198         plugin -- plug-in name
 199         parameter -- plug-in parameter
 200
 201         Return list of found values."""
 202         default_values = []
 203         compute_name = compute.get_name()
 204         nodes = get_apex_nodes()
 205         for node in nodes:
 206             if compute_name == node.get_dict()['name']:
 207                 stdout = node.run_cmd(
 208                     'cat /etc/collectd/collectd.conf.d/{}.conf' .format(plugin))
 209                 for line in stdout.split('\n'):
 210                     if 'Interfaces' in line:
 211                         return line.split(' ', 1)[1]
 212                     elif 'Bridges' in line:
 213                         return line.split(' ', 1)[1]
 214                     elif 'Cores' in line:
 215                         return line.split(' ', 1)[1]
 216                     else:
 217                         pass
 218         return default_values
 219
 220     def execute_command(self, command, host_ip=None, ssh=None):
 221         """Execute command on node and return list of lines of standard output.
 222
 223         Keyword arguments:
 224         command -- command
 225         host_ip -- IP of the node
 226         ssh -- existing open SSH session to use
 227
 228         One of host_ip or ssh must not be None. If both are not None,
 229         existing ssh session is used.
 230         """
 231         if host_ip is None and ssh is None:
 232             raise ValueError('One of host_ip or ssh must not be None.')
 233         if ssh is None:
 234             ssh, sftp = self.__open_sftp_session(host_ip, 'root', 'opnfvapex')
 235         stdin, stdout, stderr = ssh.exec_command(command)
 236         return stdout.readlines()
 237
 238     def get_ovs_interfaces(self, compute):
 239         """Get list of configured OVS interfaces
 240
 241         Keyword arguments:
 242         compute -- compute node instance
 243         """
 244         compute_name = compute.get_name()
 245         nodes = get_apex_nodes()
 246         for node in nodes:
 247             if compute_name == node.get_dict()['name']:
 248                 stdout = node.run_cmd('sudo ovs-vsctl list-br')
 249         return stdout
 250
 251     def is_gnocchi_running(self, controller):
 252         """Check whether Gnocchi is running on controller.
 253
 254         Keyword arguments:
 255         controller -- controller node instance
 256
 257         Return boolean value whether Gnocchi is running.
 258         """
 259         gnocchi_present = False
 260         controller_name = controller.get_name()
 261         nodes = get_apex_nodes()
 262         for node in nodes:
 263             if controller_name == node.get_dict()['name']:
 264                 node.put_file(
 265                     '/home/opnfv/functest/conf/openstack.creds',
 266                     'overcloudrc.v3')
 267                 stdout = node.run_cmd(
 268                     "source overcloudrc.v3;"
 269                     + "openstack catalog list | grep gnocchi")
 270                 if 'gnocchi' in stdout:
 271                     gnocchi_present = True
 272         return gnocchi_present
 273
 274     def is_aodh_running(self, controller):
 275         """Check whether aodh service is running on controller
 276         """
 277         aodh_present = False
 278         controller_name = controller.get_name()
 279         nodes = get_apex_nodes()
 280         for node in nodes:
 281             if controller_name == node.get_dict()['name']:
 282                 node.put_file(
 283                     '/home/opnfv/functest/conf/openstack.creds',
 284                     'overcloudrc.v3')
 285                 stdout = node.run_cmd(
 286                     "source overcloudrc.v3;"
 287                     + "openstack catalog list | grep aodh")
 288                 if 'aodh' in stdout:
 289                     aodh_present = True
 290         return aodh_present
 291
 292     def is_mcelog_installed(self, compute, package):
 293         """Check whether package exists on compute node.
 294
 295         Keyword arguments:
 296         compute -- compute node instance
 297         package -- Linux package to search for
 298
 299         Return boolean value whether package is installed.
 300         """
 301         compute_name = compute.get_name()
 302         nodes = get_apex_nodes()
 303         for node in nodes:
 304             if compute_name == node.get_dict()['name']:
 305                 stdout = node.run_cmd(
 306                     'yum list installed | grep mcelog')
 307                 if 'mcelog' in stdout:
 308                     return 1
 309                 else:
 310                     return 0
 311
 312     def is_libpqos_on_node(self, compute):
 313         """Check whether libpqos is present on compute node"""
 314
 315         compute_name = compute.get_name()
 316         nodes = get_apex_nodes()
 317         for node in nodes:
 318             if compute_name == node.get_dict()['name']:
 319                 stdout = node.run_cmd('ls /usr/local/lib/ | grep libpqos')
 320                 if 'libpqos' in stdout:
 321                     return True
 322         return False
 323
 324     def check_aodh_plugin_included(self, compute):
 325         """Check if aodh plugin is included in collectd.conf file.
 326         If not, try to enable it.
 327
 328         Keyword arguments:
 329         compute -- compute node instance
 330
 331         Return boolean value whether AODH plugin is included
 332         or it's enabling was successful.
 333         """
 334         compute_name = compute.get_name()
 335         nodes = get_apex_nodes()
 336         for node in nodes:
 337             if compute_name == node.get_dict()['name']:
 338                 aodh_conf = node.run_cmd('ls /etc/collectd/collectd.conf.d')
 339                 if 'aodh.conf' not in aodh_conf:
 340                     self.__logger.info(
 341                         "AODH Plugin not included in compute node")
 342                     return False
 343                 else:
 344                     self.__logger.info(
 345                         "AODH plugin present in compute node {}" .format(
 346                             compute_name))
 347                     return True
 348         return True
 349
 350     def check_gnocchi_plugin_included(self, compute):
 351         """Check if gnocchi plugin is included in collectd.conf file.
 352         If not, try to enable it.
 353
 354         Keyword arguments:
 355         compute -- compute node instance
 356
 357         Return boolean value whether gnocchi plugin is included
 358         or it's enabling was successful.
 359         """
 360         compute_name = compute.get_name()
 361         nodes = get_apex_nodes()
 362         for node in nodes:
 363             if compute_name == node.get_dict()['name']:
 364                 gnocchi_conf = node.run_cmd('ls /etc/collectd/collectd.conf.d')
 365                 if 'collectd-ceilometer-plugin.conf' not in gnocchi_conf:
 366                     self.__logger.info("Gnocchi Plugin not included")
 367                     return False
 368                 else:
 369                     self.__logger.info(
 370                         "Gnochi plugin available in compute node {}" .format(
 371                             compute_name))
 372                     return True
 373         return True
 374
 375     def check_snmp_plugin_included(self, compute):
 376         """Check if SNMP plugin is active in compute node.
 377         """
 378         snmp_mib = '/usr/share/snmp/mibs/Intel-Rdt.txt'
 379         snmp_string = 'INTEL-RDT-MIB::intelRdt'
 380         compute_name = compute.get_name()
 381         nodes = get_apex_nodes()
 382         for node in nodes:
 383             if compute_name == node.get_dict()['name']:
 384                 stdout = node.run_cmd(
 385                     'snmpwalk -v2c -m {0} -c public localhost {1}' .format(
 386                         snmp_mib, snmp_string))
 387                 self.__logger.info("snmp output = {}" .format(stdout))
 388                 if 'OID' in stdout:
 389                     return False
 390                 else:
 391                     return True
 392
 393     def enable_plugins(
 394             self, compute, plugins, error_plugins, create_backup=True):
 395         """Enable plugins on compute node
 396
 397         Keyword arguments:
 398         compute -- compute node instance
 399         plugins -- list of plugins to be enabled
 400
 401         Return boolean value indicating whether function was successful.
 402         """
 403         plugins = sorted(plugins)
 404         compute_name = compute.get_name()
 405         nodes = get_apex_nodes()
 406         for node in nodes:
 407             if compute_name == node.get_dict()['name']:
 408                 node.put_file(
 409                     '/usr/local/lib/python2.7/dist-packages/baro_tests/'
 410                     + 'csv.conf', 'csv.conf')
 411                 node.run_cmd(
 412                     'sudo cp csv.conf '
 413                     + '/etc/collectd/collectd.conf.d/csv.conf')
 414         return True
 415
 416     def restart_collectd(self, compute):
 417         """Restart collectd on compute node.
 418
 419         Keyword arguments:
 420         compute -- compute node instance
 421
 422         Retrun tuple with boolean indicating success and list of warnings
 423         received during collectd start.
 424         """
 425         compute_name = compute.get_name()
 426         nodes = get_apex_nodes()
 427
 428         def get_collectd_processes(compute_node):
 429             """Get number of running collectd processes.
 430
 431             Keyword arguments:
 432             ssh_session -- instance of SSH session in which to check
 433                 for processes
 434             """
 435             stdout = compute_node.run_cmd("pgrep collectd")
 436             return len(stdout)
 437
 438         for node in nodes:
 439             if compute_name == node.get_dict()['name']:
 440                 # node.run_cmd('su; "opnfvapex"')
 441                 self.__logger.info('Stopping collectd service...')
 442                 node.run_cmd('sudo systemctl stop collectd')
 443                 time.sleep(10)
 444                 if get_collectd_processes(node):
 445                     self.__logger.error('Collectd is still running...')
 446                     return False, []
 447                 self.__logger.info('Starting collectd service...')
 448                 stdout = node.run_cmd('sudo systemctl start collectd')
 449                 time.sleep(10)
 450                 warning = [
 451                     output.strip() for output in stdout if 'WARN: ' in output]
 452                 if get_collectd_processes(node) == 0:
 453                     self.__logger.error('Collectd is still not running...')
 454                     return False, warning
 455         return True, warning
 456
 457     def test_plugins_with_aodh(
 458             self, compute, plugin_interval, logger,
 459             criteria_list=[]):
 460
 461         metric_id = {}
 462         timestamps1 = {}
 463         timestamps2 = {}
 464         nodes = get_apex_nodes()
 465         for node in nodes:
 466             if node.is_controller():
 467                 self.__logger.info('Getting AODH Alarm list on {}' .format(
 468                     (node.get_dict()['name'])))
 469                 node.put_file(
 470                     '/home/opnfv/functest/conf/openstack.creds',
 471                     'overcloudrc.v3')
 472                 stdout = node.run_cmd(
 473                     "source overcloudrc.v3;"
 474                     + "aodh alarm list | grep {0} | grep {1}"
 475                     .format(criteria_list, compute))
 476                 for line in stdout.splitlines():
 477                     line = line.replace('|', "")
 478                     metric_id = line.split()[0]
 479                     stdout = node.run_cmd(
 480                         'source overcloudrc.v3; aodh alarm show {}' .format(
 481                             metric_id))
 482                     for line in stdout.splitlines()[3: -1]:
 483                         line = line.replace('|', "")
 484                         if line.split()[0] == 'timestamp':
 485                             timestamps1 = line.split()[1]
 486                         else:
 487                             pass
 488                     time.sleep(12)
 489                     stdout = node.run_cmd(
 490                         "source overcloudrc.v3; aodh alarm show {}" .format(
 491                             metric_id))
 492                     for line in stdout.splitlines()[3:-1]:
 493                         line = line.replace('|', "")
 494                         if line.split()[0] == 'timestamp':
 495                             timestamps2 = line.split()[1]
 496                         else:
 497                             pass
 498                     if timestamps1 == timestamps2:
 499                         self.__logger.info(
 500                             "Data not updated after interval of 12 seconds")
 501                         return False
 502                     else:
 503                         self.__logger.info("PASS")
 504                         return True
 505
 506     def test_plugins_with_gnocchi(
 507             self, compute, plugin_interval, logger,
 508             criteria_list=[]):
 509
 510         metric_id = {}
 511         timestamps1 = {}
 512         timestamps2 = {}
 513         nodes = get_apex_nodes()
 514         for node in nodes:
 515             if node.is_controller():
 516                 self.__logger.info('Getting gnocchi metric list on {}' .format(
 517                     (node.get_dict()['name'])))
 518                 node.put_file(
 519                     '/home/opnfv/functest/conf/openstack.creds',
 520                     'overcloudrc.v3')
 521                 stdout = node.run_cmd(
 522                     "source overcloudrc.v3;"
 523                     + "gnocchi metric list | grep {0} | grep {1}"
 524                     .format(criteria_list, compute))
 525                 for line in stdout.splitlines():
 526                     line = line.replace('|', "")
 527                     metric_id = line.split()[0]
 528                     stdout = node.run_cmd(
 529                         'source overcloudrc.v3;gnocchi measures show {}'.format(
 530                             metric_id))
 531                     for line in stdout.splitlines()[3: -1]:
 532                         if line[0] == '+':
 533                             pass
 534                         else:
 535                             timestamps1 = line.replace('|', "")
 536                             timestamps1 = timestamps1.split()[0]
 537                     time.sleep(10)
 538                     stdout = node.run_cmd(
 539                         "source overcloudrc.v3;gnocchi measures show {}".format(
 540                             metric_id))
 541                     for line in stdout.splitlines()[3:-1]:
 542                         if line[0] == '+':
 543                             pass
 544                         else:
 545                             timestamps2 = line.replace('|', "")
 546                             timestamps2 = timestamps2.split()[0]
 547                     if timestamps1 == timestamps2:
 548                         self.__logger.info("Data not updated after 12 seconds")
 549                         return False
 550                     else:
 551                         self.__logger.info("PASS")
 552                         return True
 553
 554     def test_plugins_with_snmp(
 555             self, compute, plugin_interval, logger, plugin, snmp_mib_files=[],
 556             snmp_mib_strings=[], snmp_in_commands=[]):
 557
 558         if plugin == 'intel_rdt':
 559             nodes = get_apex_nodes()
 560             for node in nodes:
 561                 if compute == node.get_dict()['name']:
 562                     stdout = node.run_cmd(
 563                         'snmpwalk -v2c -m {0} -c public localhost {1}' .format(
 564                             snmp_mib_files, snmp_mib_strings))
 565                     self.__logger.info("{}" .format(stdout))
 566                     if 'OID' in stdout:
 567                         self.__logger.info("SNMP query failed")
 568                         return False
 569                     else:
 570                         counter1 = stdout.split()[3]
 571                     time.sleep(10)
 572                     stdout = node.run_cmd(
 573                         'snmpwalk -v2c -m {0} -c public localhost {1}' .format(
 574                             snmp_mib_files, snmp_mib_strings))
 575                     self.__logger.info("{}" .format(stdout))
 576                     if 'OID' in stdout:
 577                         self.__logger.info(
 578                             "SNMP query failed during second check")
 579                         self.__logger.info("waiting for 10 sec")
 580                         time.sleep(10)
 581                     stdout = node.run_cmd(
 582                         'snmpwalk -v2c -m {0} -c public localhost {1}' .format(
 583                             snmp_mib_files, snmp_mib_strings))
 584                     self.__logger.info("{}" .format(stdout))
 585                     if 'OID' in stdout:
 586                         self.__logger.info("SNMP query failed again")
 587                         self.__logger.info("Failing this test case")
 588                         return False
 589                     else:
 590                         counter2 = stdout.split()[3]
 591
 592                     if counter1 == counter2:
 593                         return False
 594                     else:
 595                         return True