baro_tests/config_server.py

   1 # -*- coding: utf-8 -*-
   2 #
   3 # Licensed under the Apache License, Version 2.0 (the "License"); you may
   4 # not use this file except in compliance with the License. You may obtain
   5 # a copy of the License at
   6 #
   7 #      http://www.apache.org/licenses/LICENSE-2.0
   8 #
   9 # Unless required by applicable law or agreed to in writing, software
  10 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  11 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  12 # License for the specific language governing permissions and limitations
  13 # under the License.
  14
  15 """Classes used by collectd.py"""
  16
  17 import paramiko
  18 import time
  19 import os.path
  20 import os
  21 import re
  22 from opnfv.deployment import factory
  23 ID_RSA_PATH = '/root/.ssh/id_rsa'
  24 SSH_KEYS_SCRIPT = '/home/opnfv/barometer/baro_utils/get_ssh_keys.sh'
  25 DEF_PLUGIN_INTERVAL = 10
  26 COLLECTD_CONF = '/etc/collectd.conf'
  27 COLLECTD_CONF_DIR = '/etc/collectd/collectd.conf.d'
  28 NOTIFICATION_FILE = '/var/log/python-notifications.dump'
  29 COLLECTD_NOTIFICATION = '/etc/collectd_notification_dump.py'
  30 APEX_IP = os.getenv("INSTALLER_IP").rstrip('\n')
  31 APEX_USER = 'root'
  32 APEX_USER_STACK = 'stack'
  33 APEX_PKEY = '/root/.ssh/id_rsa'
  34
  35
  36 class Node(object):
  37     """Node configuration class"""
  38     def __init__(self, attrs):
  39         self.__null = attrs[0]
  40         self.__id = attrs[1]
  41         self.__name = attrs[2]
  42         self.__status = attrs[3] if attrs[3] else None
  43         self.__taskState = attrs[4]
  44         self.__pwrState = attrs[5]
  45         self.__ip = re.sub('^[a-z]+=', '', attrs[6])
  46
  47     def get_name(self):
  48         """Get node name"""
  49         return self.__name
  50
  51     def get_id(self):
  52         """Get node ID"""
  53         return self.__id
  54
  55     def get_ip(self):
  56         """Get node IP address"""
  57         return self.__ip
  58
  59     def get_roles(self):
  60         """Get node role"""
  61         return self.__roles
  62
  63
  64 def get_apex_nodes():
  65     handler = factory.Factory.get_handler('apex',
  66                                           APEX_IP,
  67                                           APEX_USER_STACK,
  68                                           APEX_PKEY)
  69     nodes = handler.get_nodes()
  70     return nodes
  71
  72
  73 class ConfigServer(object):
  74     """Class to get env configuration"""
  75     def __init__(self, host, user, logger, priv_key=None):
  76         self.__host = host
  77         self.__user = user
  78         self.__passwd = None
  79         self.__priv_key = priv_key
  80         self.__nodes = list()
  81         self.__logger = logger
  82
  83         self.__private_key_file = ID_RSA_PATH
  84         if not os.path.isfile(self.__private_key_file):
  85             self.__logger.error(
  86                 "Private key file '{}'".format(self.__private_key_file)
  87                 + " not found.")
  88             raise IOError("Private key file '{}' not found.".format(
  89                 self.__private_key_file))
  90
  91         # get list of available nodes
  92         ssh, sftp = self.__open_sftp_session(
  93             self.__host, self.__user, self.__passwd)
  94         attempt = 1
  95         fuel_node_passed = False
  96
  97         while (attempt <= 10) and not fuel_node_passed:
  98             stdin, stdout, stderr = ssh.exec_command(
  99                 "source stackrc; nova list")
 100             stderr_lines = stderr.readlines()
 101             if stderr_lines:
 102                 self.__logger.warning(
 103                     "'Apex node' command failed (try {}):".format(attempt))
 104                 for line in stderr_lines:
 105                     self.__logger.debug(line.strip())
 106             else:
 107                 fuel_node_passed = True
 108                 if attempt > 1:
 109                     self.__logger.info(
 110                         "'Apex node' command passed (try {})".format(attempt))
 111             attempt += 1
 112         if not fuel_node_passed:
 113             self.__logger.error(
 114                 "'Apex node' command failed. This was the last try.")
 115             raise OSError(
 116                 "'Apex node' command failed. This was the last try.")
 117         node_table = stdout.readlines()\
 118
 119         # skip table title and parse table values
 120
 121         for entry in node_table[3:]:
 122             if entry[0] == '+' or entry[0] == '\n':
 123                 print entry
 124                 pass
 125             else:
 126                 self.__nodes.append(
 127                     Node([str(x.strip(' \n')) for x in entry.split('|')]))
 128
 129     def get_controllers(self):
 130         # Get list of controllers
 131         print self.__nodes[0]._Node__ip
 132         return (
 133             [node for node in self.__nodes if 'controller' in node.get_name()])
 134
 135     def get_computes(self):
 136         # Get list of computes
 137         return (
 138             [node for node in self.__nodes if 'compute' in node.get_name()])
 139
 140     def get_nodes(self):
 141         # Get list of nodes
 142         return self.__nodes
 143
 144     def __open_sftp_session(self, host, user, passwd=None):
 145         # Connect to given host.
 146         """Keyword arguments:
 147         host -- host to connect
 148         user -- user to use
 149         passwd -- password to use
 150
 151         Return tuple of SSH and SFTP client instances.
 152         """
 153         # create SSH client
 154         ssh = paramiko.SSHClient()
 155         ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
 156
 157         # try a direct access using password or private key
 158         if not passwd and not self.__priv_key:
 159             # get private key
 160             self.__priv_key = paramiko.RSAKey.from_private_key_file(
 161                 self.__private_key_file)
 162
 163         # connect to the server
 164         ssh.connect(
 165             host, username=user, password=passwd, pkey=self.__priv_key)
 166         sftp = ssh.open_sftp()
 167
 168         # return SFTP client instance
 169         return ssh, sftp
 170
 171     def get_plugin_interval(self, compute, plugin):
 172         """Find the plugin interval in collectd configuration.
 173
 174         Keyword arguments:
 175         compute -- compute node instance
 176         plugin -- plug-in name
 177
 178         If found, return interval value, otherwise the default value"""
 179         default_interval = DEF_PLUGIN_INTERVAL
 180         compute_name = compute.get_name()
 181         nodes = get_apex_nodes()
 182         for node in nodes:
 183             if compute_name == node.get_dict()['name']:
 184                 stdout = node.run_cmd(
 185                     'cat /etc/collectd/collectd.conf.d/{}.conf'.format(plugin))
 186                 if stdout is None:
 187                     return default_interval
 188                 for line in stdout.split('\n'):
 189                     if 'Interval' in line:
 190                         return 1
 191         return default_interval
 192
 193     def get_plugin_config_values(self, compute, plugin, parameter):
 194         """Get parameter values from collectd config file.
 195
 196         Keyword arguments:
 197         compute -- compute node instance
 198         plugin -- plug-in name
 199         parameter -- plug-in parameter
 200
 201         Return list of found values."""
 202         default_values = []
 203         compute_name = compute.get_name()
 204         nodes = get_apex_nodes()
 205         for node in nodes:
 206             if compute_name == node.get_dict()['name']:
 207                 stdout = node.run_cmd(
 208                     'cat /etc/collectd/collectd.conf.d/{}.conf' .format(plugin))
 209                 if stdout is None:
 210                     return default_values
 211                 for line in stdout.split('\n'):
 212                     if 'Interfaces' in line:
 213                         return line.split(' ', 1)[1]
 214                     elif 'Bridges' in line:
 215                         return line.split(' ', 1)[1]
 216                     elif 'Cores' in line:
 217                         return line.split(' ', 1)[1]
 218                     else:
 219                         pass
 220         return default_values
 221
 222     def execute_command(self, command, host_ip=None, ssh=None):
 223         """Execute command on node and return list of lines of standard output.
 224
 225         Keyword arguments:
 226         command -- command
 227         host_ip -- IP of the node
 228         ssh -- existing open SSH session to use
 229
 230         One of host_ip or ssh must not be None. If both are not None,
 231         existing ssh session is used.
 232         """
 233         if host_ip is None and ssh is None:
 234             raise ValueError('One of host_ip or ssh must not be None.')
 235         if ssh is None:
 236             ssh, sftp = self.__open_sftp_session(host_ip, 'root', 'opnfvapex')
 237         stdin, stdout, stderr = ssh.exec_command(command)
 238         return stdout.readlines()
 239
 240     def get_ovs_interfaces(self, compute):
 241         """Get list of configured OVS interfaces
 242
 243         Keyword arguments:
 244         compute -- compute node instance
 245         """
 246         compute_name = compute.get_name()
 247         nodes = get_apex_nodes()
 248         for node in nodes:
 249             if compute_name == node.get_dict()['name']:
 250                 stdout = node.run_cmd('sudo ovs-vsctl list-br')
 251         return stdout
 252
 253     def is_gnocchi_running(self, controller):
 254         """Check whether Gnocchi is running on controller.
 255
 256         Keyword arguments:
 257         controller -- controller node instance
 258
 259         Return boolean value whether Gnocchi is running.
 260         """
 261         gnocchi_present = False
 262         controller_name = controller.get_name()
 263         nodes = get_apex_nodes()
 264         for node in nodes:
 265             if controller_name == node.get_dict()['name']:
 266                 node.put_file(
 267                     '/home/opnfv/functest/conf/openstack.creds',
 268                     'overcloudrc.v3')
 269                 stdout = node.run_cmd(
 270                     "source overcloudrc.v3;"
 271                     + "openstack catalog list | grep gnocchi")
 272                 if 'gnocchi' in stdout:
 273                     gnocchi_present = True
 274         return gnocchi_present
 275
 276     def is_aodh_running(self, controller):
 277         """Check whether aodh service is running on controller
 278         """
 279         aodh_present = False
 280         controller_name = controller.get_name()
 281         nodes = get_apex_nodes()
 282         for node in nodes:
 283             if controller_name == node.get_dict()['name']:
 284                 node.put_file(
 285                     '/home/opnfv/functest/conf/openstack.creds',
 286                     'overcloudrc.v3')
 287                 stdout = node.run_cmd(
 288                     "source overcloudrc.v3;"
 289                     + "openstack catalog list | grep aodh")
 290                 if 'aodh' in stdout:
 291                     aodh_present = True
 292         return aodh_present
 293
 294     def is_mcelog_installed(self, compute, package):
 295         """Check whether package exists on compute node.
 296
 297         Keyword arguments:
 298         compute -- compute node instance
 299         package -- Linux package to search for
 300
 301         Return boolean value whether package is installed.
 302         """
 303         compute_name = compute.get_name()
 304         nodes = get_apex_nodes()
 305         for node in nodes:
 306             if compute_name == node.get_dict()['name']:
 307                 stdout = node.run_cmd(
 308                     'rpm -qa | grep mcelog')
 309                 if stdout is None:
 310                     return 0
 311                 elif 'mcelog' in stdout:
 312                     return 1
 313                 else:
 314                     return 0
 315
 316     def is_libpqos_on_node(self, compute):
 317         """Check whether libpqos is present on compute node"""
 318
 319         compute_name = compute.get_name()
 320         nodes = get_apex_nodes()
 321         for node in nodes:
 322             if compute_name == node.get_dict()['name']:
 323                 stdout = node.run_cmd('ls /usr/local/lib/ | grep libpqos')
 324                 if 'libpqos' in stdout:
 325                     return True
 326         return False
 327
 328     def check_aodh_plugin_included(self, compute):
 329         """Check if aodh plugin is included in collectd.conf file.
 330         If not, try to enable it.
 331
 332         Keyword arguments:
 333         compute -- compute node instance
 334
 335         Return boolean value whether AODH plugin is included
 336         or it's enabling was successful.
 337         """
 338         compute_name = compute.get_name()
 339         nodes = get_apex_nodes()
 340         for node in nodes:
 341             if compute_name == node.get_dict()['name']:
 342                 aodh_conf = node.run_cmd('ls /etc/collectd/collectd.conf.d')
 343                 if 'aodh.conf' not in aodh_conf:
 344                     self.__logger.info(
 345                         "AODH Plugin not included in {}".format(compute_name))
 346                     return False
 347                 else:
 348                     self.__logger.info(
 349                         "AODH plugin present in compute node {}" .format(
 350                             compute_name))
 351                     return True
 352         return True
 353
 354     def check_gnocchi_plugin_included(self, compute):
 355         """Check if gnocchi plugin is included in collectd.conf file.
 356         If not, try to enable it.
 357
 358         Keyword arguments:
 359         compute -- compute node instance
 360
 361         Return boolean value whether gnocchi plugin is included
 362         or it's enabling was successful.
 363         """
 364         compute_name = compute.get_name()
 365         nodes = get_apex_nodes()
 366         for node in nodes:
 367             if compute_name == node.get_dict()['name']:
 368                 gnocchi_conf = node.run_cmd('ls /etc/collectd/collectd.conf.d')
 369                 if 'collectd-ceilometer-plugin.conf' not in gnocchi_conf:
 370                     self.__logger.info(
 371                         "Gnocchi Plugin not included in node {}".format(
 372                             compute_name))
 373                     return False
 374                 else:
 375                     self.__logger.info(
 376                         "Gnochi plugin available in compute node {}" .format(
 377                             compute_name))
 378                     return True
 379         return True
 380
 381     def check_snmp_plugin_included(self, compute):
 382         """Check if SNMP plugin is active in compute node.
 383         """
 384         snmp_mib = '/usr/share/snmp/mibs/Intel-Rdt.txt'
 385         snmp_string = 'INTEL-RDT-MIB::intelRdt'
 386         compute_name = compute.get_name()
 387         nodes = get_apex_nodes()
 388         for node in nodes:
 389             if compute_name == node.get_dict()['name']:
 390                 stdout = node.run_cmd(
 391                     'snmpwalk -v2c -m {0} -c public localhost {1}' .format(
 392                         snmp_mib, snmp_string))
 393                 self.__logger.info("snmp output = {}" .format(stdout))
 394                 if 'OID' in stdout:
 395                     return False
 396                 else:
 397                     return True
 398
 399     def enable_plugins(
 400             self, compute, plugins, error_plugins, create_backup=True):
 401         """Enable plugins on compute node
 402
 403         Keyword arguments:
 404         compute -- compute node instance
 405         plugins -- list of plugins to be enabled
 406
 407         Return boolean value indicating whether function was successful.
 408         """
 409         csv_file = os.path.dirname(os.path.realpath(__file__)) + '/csv.conf'
 410         plugins = sorted(plugins)
 411         compute_name = compute.get_name()
 412         nodes = get_apex_nodes()
 413         for node in nodes:
 414             if compute_name == node.get_dict()['name']:
 415                 node.put_file(csv_file, 'csv.conf')
 416                 node.run_cmd(
 417                     'sudo cp csv.conf '
 418                     + '/etc/collectd/collectd.conf.d/csv.conf')
 419         return True
 420
 421     def restart_collectd(self, compute):
 422         """Restart collectd on compute node.
 423
 424         Keyword arguments:
 425         compute -- compute node instance
 426
 427         Retrun tuple with boolean indicating success and list of warnings
 428         received during collectd start.
 429         """
 430         compute_name = compute.get_name()
 431         nodes = get_apex_nodes()
 432
 433         def get_collectd_processes(compute_node):
 434             """Get number of running collectd processes.
 435
 436             Keyword arguments:
 437             ssh_session -- instance of SSH session in which to check
 438                 for processes
 439             """
 440             stdout = compute_node.run_cmd("pgrep collectd")
 441             return len(stdout)
 442
 443         for node in nodes:
 444             if compute_name == node.get_dict()['name']:
 445                 # node.run_cmd('su; "opnfvapex"')
 446                 self.__logger.info('Stopping collectd service...')
 447                 node.run_cmd('sudo systemctl stop collectd')
 448                 time.sleep(10)
 449                 if get_collectd_processes(node):
 450                     self.__logger.error('Collectd is still running...')
 451                     return False, []
 452                 self.__logger.info('Starting collectd service...')
 453                 stdout = node.run_cmd('sudo systemctl start collectd')
 454                 time.sleep(10)
 455                 warning = [
 456                     output.strip() for output in stdout if 'WARN: ' in output]
 457                 if get_collectd_processes(node) == 0:
 458                     self.__logger.error('Collectd is still not running...')
 459                     return False, warning
 460         return True, warning
 461
 462     def test_plugins_with_aodh(
 463             self, compute, plugin_interval, logger,
 464             criteria_list=[]):
 465
 466         metric_id = {}
 467         timestamps1 = {}
 468         timestamps2 = {}
 469         nodes = get_apex_nodes()
 470         for node in nodes:
 471             if node.is_controller():
 472                 self.__logger.info('Getting AODH Alarm list on {}' .format(
 473                     (node.get_dict()['name'])))
 474                 node.put_file(
 475                     '/home/opnfv/functest/conf/openstack.creds',
 476                     'overcloudrc.v3')
 477                 stdout = node.run_cmd(
 478                     "source overcloudrc.v3;"
 479                     + "aodh alarm list | grep {0} | grep {1}"
 480                     .format(criteria_list, compute))
 481                 if stdout is None:
 482                     self.__logger.info("aodh alarm list was empty")
 483                     return False
 484                 for line in stdout.splitlines():
 485                     line = line.replace('|', "")
 486                     metric_id = line.split()[0]
 487                     stdout = node.run_cmd(
 488                         'source overcloudrc.v3; aodh alarm show {}' .format(
 489                             metric_id))
 490                     if stdout is None:
 491                         self.__logger.info("aodh alarm list was empty")
 492                         return False
 493                     for line in stdout.splitlines()[3: -1]:
 494                         line = line.replace('|', "")
 495                         if line.split()[0] == 'timestamp':
 496                             timestamps1 = line.split()[1]
 497                         else:
 498                             pass
 499                     time.sleep(12)
 500                     stdout = node.run_cmd(
 501                         "source overcloudrc.v3; aodh alarm show {}" .format(
 502                             metric_id))
 503                     if stdout is None:
 504                         self.__logger.info("aodh alarm list was empty")
 505                         return False
 506                     for line in stdout.splitlines()[3:-1]:
 507                         line = line.replace('|', "")
 508                         if line.split()[0] == 'timestamp':
 509                             timestamps2 = line.split()[1]
 510                         else:
 511                             pass
 512                     if timestamps1 == timestamps2:
 513                         self.__logger.info(
 514                             "Data not updated after interval of 12 seconds")
 515                         return False
 516                     else:
 517                         self.__logger.info("PASS")
 518                         return True
 519
 520     def test_plugins_with_gnocchi(
 521             self, compute, plugin_interval, logger,
 522             criteria_list=[]):
 523
 524         metric_id = {}
 525         timestamps1 = {}
 526         timestamps2 = {}
 527         nodes = get_apex_nodes()
 528         for node in nodes:
 529             if node.is_controller():
 530                 self.__logger.info('Getting gnocchi metric list on {}' .format(
 531                     (node.get_dict()['name'])))
 532                 node.put_file(
 533                     '/home/opnfv/functest/conf/openstack.creds',
 534                     'overcloudrc.v3')
 535                 stdout = node.run_cmd(
 536                     "source overcloudrc.v3;"
 537                     + "gnocchi metric list | grep {0} | grep {1}"
 538                     .format(criteria_list, compute))
 539                 if stdout is None:
 540                         self.__logger.info("gnocchi list was empty")
 541                         return False
 542                 for line in stdout.splitlines():
 543                     line = line.replace('|', "")
 544                     metric_id = line.split()[0]
 545                     stdout = node.run_cmd(
 546                         'source overcloudrc.v3;gnocchi measures show {}'.format(
 547                             metric_id))
 548                     if stdout is None:
 549                         self.__logger.info("gnocchi list was empty")
 550                         return False
 551                     for line in stdout.splitlines()[3: -1]:
 552                         if line[0] == '+':
 553                             pass
 554                         else:
 555                             timestamps1 = line.replace('|', "")
 556                             timestamps1 = timestamps1.split()[0]
 557                     time.sleep(10)
 558                     stdout = node.run_cmd(
 559                         "source overcloudrc.v3;gnocchi measures show {}".format(
 560                             metric_id))
 561                     if stdout is None:
 562                         self.__logger.info("gnocchi measures was empty")
 563                         return False
 564                     for line in stdout.splitlines()[3:-1]:
 565                         if line[0] == '+':
 566                             pass
 567                         else:
 568                             timestamps2 = line.replace('|', "")
 569                             timestamps2 = timestamps2.split()[0]
 570                     if timestamps1 == timestamps2:
 571                         self.__logger.info("Data not updated after 12 seconds")
 572                         return False
 573                     else:
 574                         self.__logger.info("PASS")
 575                         return True
 576
 577     def test_plugins_with_snmp(
 578             self, compute, plugin_interval, logger, plugin, snmp_mib_files=[],
 579             snmp_mib_strings=[], snmp_in_commands=[]):
 580
 581         if plugin == 'hugepages' or 'intel_rdt' or 'mcelog':
 582             nodes = get_apex_nodes()
 583             for node in nodes:
 584                 if compute == node.get_dict()['name']:
 585                     stdout = node.run_cmd(
 586                         'snmpwalk -v2c -m {0} -c public localhost {1}' .format(
 587                             snmp_mib_files, snmp_mib_strings))
 588                     self.__logger.info("{}" .format(stdout))
 589                     if stdout is None:
 590                         self.__logger.info("No output from snmpwalk")
 591                         return False
 592                     elif 'OID' in stdout:
 593                         self.__logger.info("SNMP query failed")
 594                         return False
 595                     else:
 596                         counter1 = stdout.split()[3]
 597                     time.sleep(10)
 598                     stdout = node.run_cmd(
 599                         'snmpwalk -v2c -m {0} -c public localhost {1}' .format(
 600                             snmp_mib_files, snmp_mib_strings))
 601                     self.__logger.info("{}" .format(stdout))
 602                     if stdout is None:
 603                         self.__logger.info("No output from snmpwalk")
 604                     elif 'OID' in stdout:
 605                         self.__logger.info(
 606                             "SNMP query failed during second check")
 607                         self.__logger.info("waiting for 10 sec")
 608                         time.sleep(10)
 609                     stdout = node.run_cmd(
 610                         'snmpwalk -v2c -m {0} -c public localhost {1}' .format(
 611                             snmp_mib_files, snmp_mib_strings))
 612                     self.__logger.info("{}" .format(stdout))
 613                     if stdout is None:
 614                         self.__logger.info("No output from snmpwalk")
 615                     elif 'OID' in stdout:
 616                         self.__logger.info("SNMP query failed again")
 617                         self.__logger.info("Failing this test case")
 618                         return False
 619                     else:
 620                         counter2 = stdout.split()[3]
 621
 622                     if counter1 == counter2:
 623                         return False
 624                     else:
 625                         return True
 626         else:
 627             return False