baro_tests/config_server.py

   1 # -*- coding: utf-8 -*-
   2 #
   3 # Licensed under the Apache License, Version 2.0 (the "License"); you may
   4 # not use this file except in compliance with the License. You may obtain
   5 # a copy of the License at
   6 #
   7 #      http://www.apache.org/licenses/LICENSE-2.0
   8 #
   9 # Unless required by applicable law or agreed to in writing, software
  10 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  11 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  12 # License for the specific language governing permissions and limitations
  13 # under the License.
  14
  15 """Classes used by collectd.py"""
  16
  17 import paramiko
  18 import time
  19 import os.path
  20 import os
  21 import re
  22 from opnfv.deployment import factory
  23 ID_RSA_PATH = '/root/.ssh/id_rsa'
  24 SSH_KEYS_SCRIPT = '/home/opnfv/barometer/baro_utils/get_ssh_keys.sh'
  25 DEF_PLUGIN_INTERVAL = 10
  26 COLLECTD_CONF = '/etc/collectd.conf'
  27 COLLECTD_CONF_DIR = '/etc/collectd/collectd.conf.d'
  28 NOTIFICATION_FILE = '/var/log/python-notifications.dump'
  29 COLLECTD_NOTIFICATION = '/etc/collectd_notification_dump.py'
  30 APEX_IP = os.getenv("INSTALLER_IP").rstrip('\n')
  31 APEX_USER = 'root'
  32 APEX_USER_STACK = 'stack'
  33 APEX_PKEY = '/root/.ssh/id_rsa'
  34 PATH = os.path.dirname(os.path.realpath(__file__))
  35
  36
  37 class Node(object):
  38     """Node configuration class"""
  39     def __init__(self, attrs):
  40         self.__null = attrs[0]
  41         self.__id = attrs[1]
  42         self.__name = attrs[2]
  43         self.__status = attrs[3] if attrs[3] else None
  44         self.__taskState = attrs[4]
  45         self.__pwrState = attrs[5]
  46         self.__ip = re.sub('^[a-z]+=', '', attrs[6])
  47
  48     def get_name(self):
  49         """Get node name"""
  50         return self.__name
  51
  52     def get_id(self):
  53         """Get node ID"""
  54         return self.__id
  55
  56     def get_ip(self):
  57         """Get node IP address"""
  58         return self.__ip
  59
  60     def get_roles(self):
  61         """Get node role"""
  62         return self.__roles
  63
  64
  65 def get_apex_nodes():
  66     handler = factory.Factory.get_handler('apex',
  67                                           APEX_IP,
  68                                           APEX_USER_STACK,
  69                                           APEX_PKEY)
  70     nodes = handler.get_nodes()
  71     return nodes
  72
  73
  74 class ConfigServer(object):
  75     """Class to get env configuration"""
  76     def __init__(self, host, user, logger, priv_key=None):
  77         self.__host = host
  78         self.__user = user
  79         self.__passwd = None
  80         self.__priv_key = priv_key
  81         self.__nodes = list()
  82         self.__logger = logger
  83
  84         self.__private_key_file = ID_RSA_PATH
  85         if not os.path.isfile(self.__private_key_file):
  86             self.__logger.error(
  87                 "Private key file '{}'".format(self.__private_key_file)
  88                 + " not found.")
  89             raise IOError("Private key file '{}' not found.".format(
  90                 self.__private_key_file))
  91
  92         # get list of available nodes
  93         ssh, sftp = self.__open_sftp_session(
  94             self.__host, self.__user, self.__passwd)
  95         attempt = 1
  96         fuel_node_passed = False
  97
  98         while (attempt <= 10) and not fuel_node_passed:
  99             stdin, stdout, stderr = ssh.exec_command(
 100                 "source stackrc; nova list")
 101             stderr_lines = stderr.readlines()
 102             if stderr_lines:
 103                 self.__logger.warning(
 104                     "'Apex node' command failed (try {}):".format(attempt))
 105                 for line in stderr_lines:
 106                     self.__logger.debug(line.strip())
 107             else:
 108                 fuel_node_passed = True
 109                 if attempt > 1:
 110                     self.__logger.info(
 111                         "'Apex node' command passed (try {})".format(attempt))
 112             attempt += 1
 113         if not fuel_node_passed:
 114             self.__logger.error(
 115                 "'Apex node' command failed. This was the last try.")
 116             raise OSError(
 117                 "'Apex node' command failed. This was the last try.")
 118         node_table = stdout.readlines()\
 119
 120         # skip table title and parse table values
 121
 122         for entry in node_table[3:]:
 123             if entry[0] == '+' or entry[0] == '\n':
 124                 print entry
 125                 pass
 126             else:
 127                 self.__nodes.append(
 128                     Node([str(x.strip(' \n')) for x in entry.split('|')]))
 129
 130     def get_controllers(self):
 131         # Get list of controllers
 132         print self.__nodes[0]._Node__ip
 133         return (
 134             [node for node in self.__nodes if 'controller' in node.get_name()])
 135
 136     def get_computes(self):
 137         # Get list of computes
 138         return (
 139             [node for node in self.__nodes if 'compute' in node.get_name()])
 140
 141     def get_nodes(self):
 142         # Get list of nodes
 143         return self.__nodes
 144
 145     def __open_sftp_session(self, host, user, passwd=None):
 146         # Connect to given host.
 147         """Keyword arguments:
 148         host -- host to connect
 149         user -- user to use
 150         passwd -- password to use
 151
 152         Return tuple of SSH and SFTP client instances.
 153         """
 154         # create SSH client
 155         ssh = paramiko.SSHClient()
 156         ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
 157
 158         # try a direct access using password or private key
 159         if not passwd and not self.__priv_key:
 160             # get private key
 161             self.__priv_key = paramiko.RSAKey.from_private_key_file(
 162                 self.__private_key_file)
 163
 164         # connect to the server
 165         ssh.connect(
 166             host, username=user, password=passwd, pkey=self.__priv_key)
 167         sftp = ssh.open_sftp()
 168
 169         # return SFTP client instance
 170         return ssh, sftp
 171
 172     def get_plugin_interval(self, compute, plugin):
 173         """Find the plugin interval in collectd configuration.
 174
 175         Keyword arguments:
 176         compute -- compute node instance
 177         plugin -- plug-in name
 178
 179         If found, return interval value, otherwise the default value"""
 180         default_interval = DEF_PLUGIN_INTERVAL
 181         compute_name = compute.get_name()
 182         nodes = get_apex_nodes()
 183         for node in nodes:
 184             if compute_name == node.get_dict()['name']:
 185                 stdout = node.run_cmd(
 186                     'cat /etc/collectd/collectd.conf.d/{}.conf'.format(plugin))
 187                 for line in stdout.split('\n'):
 188                     if 'Interval' in line:
 189                         return 1
 190         return default_interval
 191
 192     def get_plugin_config_values(self, compute, plugin, parameter):
 193         """Get parameter values from collectd config file.
 194
 195         Keyword arguments:
 196         compute -- compute node instance
 197         plugin -- plug-in name
 198         parameter -- plug-in parameter
 199
 200         Return list of found values."""
 201         default_values = []
 202         compute_name = compute.get_name()
 203         nodes = get_apex_nodes()
 204         for node in nodes:
 205             if compute_name == node.get_dict()['name']:
 206                 stdout = node.run_cmd(
 207                     'cat /etc/collectd/collectd.conf.d/{}.conf' .format(plugin))
 208                 for line in stdout.split('\n'):
 209                     if 'Interfaces' in line:
 210                         return line.split(' ', 1)[1]
 211                     elif 'Bridges' in line:
 212                         return line.split(' ', 1)[1]
 213                     elif 'Cores' in line:
 214                         return line.split(' ', 1)[1]
 215                     else:
 216                         pass
 217         return default_values
 218
 219     def execute_command(self, command, host_ip=None, ssh=None):
 220         """Execute command on node and return list of lines of standard output.
 221
 222         Keyword arguments:
 223         command -- command
 224         host_ip -- IP of the node
 225         ssh -- existing open SSH session to use
 226
 227         One of host_ip or ssh must not be None. If both are not None,
 228         existing ssh session is used.
 229         """
 230         if host_ip is None and ssh is None:
 231             raise ValueError('One of host_ip or ssh must not be None.')
 232         if ssh is None:
 233             ssh, sftp = self.__open_sftp_session(host_ip, 'root', 'opnfvapex')
 234         stdin, stdout, stderr = ssh.exec_command(command)
 235         return stdout.readlines()
 236
 237     def get_ovs_interfaces(self, compute):
 238         """Get list of configured OVS interfaces
 239
 240         Keyword arguments:
 241         compute -- compute node instance
 242         """
 243         compute_name = compute.get_name()
 244         nodes = get_apex_nodes()
 245         for node in nodes:
 246             if compute_name == node.get_dict()['name']:
 247                 stdout = node.run_cmd('sudo ovs-vsctl list-br')
 248         return stdout
 249
 250     def is_gnocchi_running(self, controller):
 251         """Check whether Gnocchi is running on controller.
 252
 253         Keyword arguments:
 254         controller -- controller node instance
 255
 256         Return boolean value whether Gnocchi is running.
 257         """
 258         gnocchi_present = False
 259         controller_name = controller.get_name()
 260         nodes = get_apex_nodes()
 261         for node in nodes:
 262             if controller_name == node.get_dict()['name']:
 263                 node.put_file(
 264                     '/home/opnfv/functest/conf/openstack.creds',
 265                     'overcloudrc.v3')
 266                 stdout = node.run_cmd(
 267                     "source overcloudrc.v3;"
 268                     + "openstack catalog list | grep gnocchi")
 269                 if 'gnocchi' in stdout:
 270                     gnocchi_present = True
 271         return gnocchi_present
 272
 273     def is_aodh_running(self, controller):
 274         """Check whether aodh service is running on controller
 275         """
 276         aodh_present = False
 277         controller_name = controller.get_name()
 278         nodes = get_apex_nodes()
 279         for node in nodes:
 280             if controller_name == node.get_dict()['name']:
 281                 node.put_file(
 282                     '/home/opnfv/functest/conf/openstack.creds',
 283                     'overcloudrc.v3')
 284                 stdout = node.run_cmd(
 285                     "source overcloudrc.v3;"
 286                     + "openstack catalog list | grep aodh")
 287                 if 'aodh' in stdout:
 288                     aodh_present = True
 289         return aodh_present
 290
 291     def is_mcelog_installed(self, compute, package):
 292         """Check whether package exists on compute node.
 293
 294         Keyword arguments:
 295         compute -- compute node instance
 296         package -- Linux package to search for
 297
 298         Return boolean value whether package is installed.
 299         """
 300         compute_name = compute.get_name()
 301         nodes = get_apex_nodes()
 302         for node in nodes:
 303             if compute_name == node.get_dict()['name']:
 304                 stdout = node.run_cmd(
 305                     'yum list installed | grep mcelog')
 306                 if stdout is None:
 307                     return 0
 308                 elif 'mcelog' in stdout:
 309                     return 1
 310                 else:
 311                     return 0
 312
 313     def is_libpqos_on_node(self, compute):
 314         """Check whether libpqos is present on compute node"""
 315
 316         compute_name = compute.get_name()
 317         nodes = get_apex_nodes()
 318         for node in nodes:
 319             if compute_name == node.get_dict()['name']:
 320                 stdout = node.run_cmd('ls /usr/local/lib/ | grep libpqos')
 321                 if 'libpqos' in stdout:
 322                     return True
 323         return False
 324
 325     def check_aodh_plugin_included(self, compute):
 326         """Check if aodh plugin is included in collectd.conf file.
 327         If not, try to enable it.
 328
 329         Keyword arguments:
 330         compute -- compute node instance
 331
 332         Return boolean value whether AODH plugin is included
 333         or it's enabling was successful.
 334         """
 335         compute_name = compute.get_name()
 336         nodes = get_apex_nodes()
 337         for node in nodes:
 338             if compute_name == node.get_dict()['name']:
 339                 aodh_conf = node.run_cmd('ls /etc/collectd/collectd.conf.d')
 340                 if 'aodh.conf' not in aodh_conf:
 341                     self.__logger.info(
 342                         "AODH Plugin not included in compute node")
 343                     return False
 344                 else:
 345                     self.__logger.info(
 346                         "AODH plugin present in compute node {}" .format(
 347                             compute_name))
 348                     return True
 349         return True
 350
 351     def check_gnocchi_plugin_included(self, compute):
 352         """Check if gnocchi plugin is included in collectd.conf file.
 353         If not, try to enable it.
 354
 355         Keyword arguments:
 356         compute -- compute node instance
 357
 358         Return boolean value whether gnocchi plugin is included
 359         or it's enabling was successful.
 360         """
 361         compute_name = compute.get_name()
 362         nodes = get_apex_nodes()
 363         for node in nodes:
 364             if compute_name == node.get_dict()['name']:
 365                 gnocchi_conf = node.run_cmd('ls /etc/collectd/collectd.conf.d')
 366                 if 'collectd-ceilometer-plugin.conf' not in gnocchi_conf:
 367                     self.__logger.info("Gnocchi Plugin not included")
 368                     return False
 369                 else:
 370                     self.__logger.info(
 371                         "Gnochi plugin available in compute node {}" .format(
 372                             compute_name))
 373                     return True
 374         return True
 375
 376     def check_snmp_plugin_included(self, compute):
 377         """Check if SNMP plugin is active in compute node.
 378         """
 379         snmp_mib = '/usr/share/snmp/mibs/Intel-Rdt.txt'
 380         snmp_string = 'INTEL-RDT-MIB::intelRdt'
 381         compute_name = compute.get_name()
 382         nodes = get_apex_nodes()
 383         for node in nodes:
 384             if compute_name == node.get_dict()['name']:
 385                 stdout = node.run_cmd(
 386                     'snmpwalk -v2c -m {0} -c public localhost {1}' .format(
 387                         snmp_mib, snmp_string))
 388                 self.__logger.info("snmp output = {}" .format(stdout))
 389                 if 'OID' in stdout:
 390                     return False
 391                 else:
 392                     return True
 393
 394     def enable_plugins(
 395             self, compute, plugins, error_plugins, create_backup=True):
 396         """Enable plugins on compute node
 397
 398         Keyword arguments:
 399         compute -- compute node instance
 400         plugins -- list of plugins to be enabled
 401
 402         Return boolean value indicating whether function was successful.
 403         """
 404         plugins = sorted(plugins)
 405         compute_name = compute.get_name()
 406         nodes = get_apex_nodes()
 407         for node in nodes:
 408             if compute_name == node.get_dict()['name']:
 409                 node.put_file(
 410                     'PATH/csv.conf', 'csv.conf')
 411                 node.run_cmd(
 412                     'sudo cp csv.conf '
 413                     + '/etc/collectd/collectd.conf.d/csv.conf')
 414         return True
 415
 416     def restart_collectd(self, compute):
 417         """Restart collectd on compute node.
 418
 419         Keyword arguments:
 420         compute -- compute node instance
 421
 422         Retrun tuple with boolean indicating success and list of warnings
 423         received during collectd start.
 424         """
 425         compute_name = compute.get_name()
 426         nodes = get_apex_nodes()
 427
 428         def get_collectd_processes(compute_node):
 429             """Get number of running collectd processes.
 430
 431             Keyword arguments:
 432             ssh_session -- instance of SSH session in which to check
 433                 for processes
 434             """
 435             stdout = compute_node.run_cmd("pgrep collectd")
 436             return len(stdout)
 437
 438         for node in nodes:
 439             if compute_name == node.get_dict()['name']:
 440                 # node.run_cmd('su; "opnfvapex"')
 441                 self.__logger.info('Stopping collectd service...')
 442                 node.run_cmd('sudo systemctl stop collectd')
 443                 time.sleep(10)
 444                 if get_collectd_processes(node):
 445                     self.__logger.error('Collectd is still running...')
 446                     return False, []
 447                 self.__logger.info('Starting collectd service...')
 448                 stdout = node.run_cmd('sudo systemctl start collectd')
 449                 time.sleep(10)
 450                 warning = [
 451                     output.strip() for output in stdout if 'WARN: ' in output]
 452                 if get_collectd_processes(node) == 0:
 453                     self.__logger.error('Collectd is still not running...')
 454                     return False, warning
 455         return True, warning
 456
 457     def test_plugins_with_aodh(
 458             self, compute, plugin_interval, logger,
 459             criteria_list=[]):
 460
 461         metric_id = {}
 462         timestamps1 = {}
 463         timestamps2 = {}
 464         nodes = get_apex_nodes()
 465         for node in nodes:
 466             if node.is_controller():
 467                 self.__logger.info('Getting AODH Alarm list on {}' .format(
 468                     (node.get_dict()['name'])))
 469                 node.put_file(
 470                     '/home/opnfv/functest/conf/openstack.creds',
 471                     'overcloudrc.v3')
 472                 stdout = node.run_cmd(
 473                     "source overcloudrc.v3;"
 474                     + "aodh alarm list | grep {0} | grep {1}"
 475                     .format(criteria_list, compute))
 476                 for line in stdout.splitlines():
 477                     line = line.replace('|', "")
 478                     metric_id = line.split()[0]
 479                     stdout = node.run_cmd(
 480                         'source overcloudrc.v3; aodh alarm show {}' .format(
 481                             metric_id))
 482                     for line in stdout.splitlines()[3: -1]:
 483                         line = line.replace('|', "")
 484                         if line.split()[0] == 'timestamp':
 485                             timestamps1 = line.split()[1]
 486                         else:
 487                             pass
 488                     time.sleep(12)
 489                     stdout = node.run_cmd(
 490                         "source overcloudrc.v3; aodh alarm show {}" .format(
 491                             metric_id))
 492                     for line in stdout.splitlines()[3:-1]:
 493                         line = line.replace('|', "")
 494                         if line.split()[0] == 'timestamp':
 495                             timestamps2 = line.split()[1]
 496                         else:
 497                             pass
 498                     if timestamps1 == timestamps2:
 499                         self.__logger.info(
 500                             "Data not updated after interval of 12 seconds")
 501                         return False
 502                     else:
 503                         self.__logger.info("PASS")
 504                         return True
 505
 506     def test_plugins_with_gnocchi(
 507             self, compute, plugin_interval, logger,
 508             criteria_list=[]):
 509
 510         metric_id = {}
 511         timestamps1 = {}
 512         timestamps2 = {}
 513         nodes = get_apex_nodes()
 514         for node in nodes:
 515             if node.is_controller():
 516                 self.__logger.info('Getting gnocchi metric list on {}' .format(
 517                     (node.get_dict()['name'])))
 518                 node.put_file(
 519                     '/home/opnfv/functest/conf/openstack.creds',
 520                     'overcloudrc.v3')
 521                 stdout = node.run_cmd(
 522                     "source overcloudrc.v3;"
 523                     + "gnocchi metric list | grep {0} | grep {1}"
 524                     .format(criteria_list, compute))
 525                 for line in stdout.splitlines():
 526                     line = line.replace('|', "")
 527                     metric_id = line.split()[0]
 528                     stdout = node.run_cmd(
 529                         'source overcloudrc.v3;gnocchi measures show {}'.format(
 530                             metric_id))
 531                     for line in stdout.splitlines()[3: -1]:
 532                         if line[0] == '+':
 533                             pass
 534                         else:
 535                             timestamps1 = line.replace('|', "")
 536                             timestamps1 = timestamps1.split()[0]
 537                     time.sleep(10)
 538                     stdout = node.run_cmd(
 539                         "source overcloudrc.v3;gnocchi measures show {}".format(
 540                             metric_id))
 541                     for line in stdout.splitlines()[3:-1]:
 542                         if line[0] == '+':
 543                             pass
 544                         else:
 545                             timestamps2 = line.replace('|', "")
 546                             timestamps2 = timestamps2.split()[0]
 547                     if timestamps1 == timestamps2:
 548                         self.__logger.info("Data not updated after 12 seconds")
 549                         return False
 550                     else:
 551                         self.__logger.info("PASS")
 552                         return True
 553
 554     def test_plugins_with_snmp(
 555             self, compute, plugin_interval, logger, plugin, snmp_mib_files=[],
 556             snmp_mib_strings=[], snmp_in_commands=[]):
 557
 558         if plugin == 'hugepages' or 'intel_rdt' or 'mcelog':
 559             nodes = get_apex_nodes()
 560             for node in nodes:
 561                 if compute == node.get_dict()['name']:
 562                     stdout = node.run_cmd(
 563                         'snmpwalk -v2c -m {0} -c public localhost {1}' .format(
 564                             snmp_mib_files, snmp_mib_strings))
 565                     self.__logger.info("{}" .format(stdout))
 566                     if stdout is None:
 567                         self.__logger.info("No output from snmpwalk")
 568                         return False
 569                     elif 'OID' in stdout:
 570                         self.__logger.info("SNMP query failed")
 571                         return False
 572                     else:
 573                         counter1 = stdout.split()[3]
 574                     time.sleep(10)
 575                     stdout = node.run_cmd(
 576                         'snmpwalk -v2c -m {0} -c public localhost {1}' .format(
 577                             snmp_mib_files, snmp_mib_strings))
 578                     self.__logger.info("{}" .format(stdout))
 579                     if stdout is None:
 580                         self.__logger.info("No output from snmpwalk")
 581                     elif 'OID' in stdout:
 582                         self.__logger.info(
 583                             "SNMP query failed during second check")
 584                         self.__logger.info("waiting for 10 sec")
 585                         time.sleep(10)
 586                     stdout = node.run_cmd(
 587                         'snmpwalk -v2c -m {0} -c public localhost {1}' .format(
 588                             snmp_mib_files, snmp_mib_strings))
 589                     self.__logger.info("{}" .format(stdout))
 590                     if stdout is None:
 591                         self.__logger.info("No output from snmpwalk")
 592                     elif 'OID' in stdout:
 593                         self.__logger.info("SNMP query failed again")
 594                         self.__logger.info("Failing this test case")
 595                         return False
 596                     else:
 597                         counter2 = stdout.split()[3]
 598
 599                     if counter1 == counter2:
 600                         return False
 601                     else:
 602                         return True
 603         else:
 604             return False