baro_tests/config_server.py

   1 # -*- coding: utf-8 -*-
   2 #
   3 # Licensed under the Apache License, Version 2.0 (the "License"); you may
   4 # not use this file except in compliance with the License. You may obtain
   5 # a copy of the License at
   6 #
   7 #      http://www.apache.org/licenses/LICENSE-2.0
   8 #
   9 # Unless required by applicable law or agreed to in writing, software
  10 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  11 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  12 # License for the specific language governing permissions and limitations
  13 # under the License.
  14
  15 """Classes used by collectd.py"""
  16
  17 import paramiko
  18 import time
  19 import os.path
  20 import os
  21 import re
  22 import subprocess
  23 from opnfv.deployment import factory
  24 ID_RSA_PATH = '/root/.ssh/id_rsa'
  25 SSH_KEYS_SCRIPT = '/home/opnfv/barometer/baro_utils/get_ssh_keys.sh'
  26 DEF_PLUGIN_INTERVAL = 10
  27 COLLECTD_CONF = '/etc/collectd.conf'
  28 COLLECTD_CONF_DIR = '/etc/collectd/collectd.conf.d'
  29 NOTIFICATION_FILE = '/var/log/python-notifications.dump'
  30 COLLECTD_NOTIFICATION = '/etc/collectd_notification_dump.py'
  31 APEX_IP = subprocess.check_output("echo $INSTALLER_IP", shell=True)
  32 APEX_USER = 'root'
  33 APEX_USER_STACK = 'stack'
  34 APEX_PKEY = '/root/.ssh/id_rsa'
  35
  36
  37 class Node(object):
  38     """Node configuration class"""
  39     def __init__(self, attrs):
  40         self.__null = attrs[0]
  41         self.__id = attrs[1]
  42         self.__name = attrs[2]
  43         self.__status = attrs[3] if attrs[3] else None
  44         self.__taskState = attrs[4]
  45         self.__pwrState = attrs[5]
  46         self.__ip = re.sub('^[a-z]+=', '', attrs[6])
  47
  48     def get_name(self):
  49         """Get node name"""
  50         return self.__name
  51
  52     def get_id(self):
  53         """Get node ID"""
  54         return self.__id
  55
  56     def get_ip(self):
  57         """Get node IP address"""
  58         return self.__ip
  59
  60     def get_roles(self):
  61         """Get node role"""
  62         return self.__roles
  63
  64
  65 def get_apex_nodes():
  66     handler = factory.Factory.get_handler('apex',
  67                                           APEX_IP,
  68                                           APEX_USER_STACK,
  69                                           APEX_PKEY)
  70     nodes = handler.get_nodes()
  71     return nodes
  72
  73
  74 class ConfigServer(object):
  75     """Class to get env configuration"""
  76     def __init__(self, host, user, logger, priv_key=None):
  77         self.__host = host
  78         self.__user = user
  79         self.__passwd = None
  80         self.__priv_key = priv_key
  81         self.__nodes = list()
  82         self.__logger = logger
  83
  84         self.__private_key_file = ID_RSA_PATH
  85         if not os.path.isfile(self.__private_key_file):
  86             self.__logger.error(
  87                 "Private key file '{}'".format(self.__private_key_file)
  88                 + " not found.")
  89             raise IOError("Private key file '{}' not found.".format(
  90                 self.__private_key_file))
  91
  92         # get list of available nodes
  93         ssh, sftp = self.__open_sftp_session(
  94             self.__host, self.__user, self.__passwd)
  95         attempt = 1
  96         fuel_node_passed = False
  97
  98         while (attempt <= 10) and not fuel_node_passed:
  99             stdin, stdout, stderr = ssh.exec_command(
 100                 "source stackrc; nova list")
 101             stderr_lines = stderr.readlines()
 102             if stderr_lines:
 103                 self.__logger.warning(
 104                     "'fuel node' command failed (try {}):".format(attempt))
 105                 for line in stderr_lines:
 106                     self.__logger.debug(line.strip())
 107             else:
 108                 fuel_node_passed = True
 109                 if attempt > 1:
 110                     self.__logger.info(
 111                         "'fuel node' command passed (try {})".format(attempt))
 112             attempt += 1
 113         if not fuel_node_passed:
 114             self.__logger.error(
 115                 "'fuel node' command failed. This was the last try.")
 116             raise OSError(
 117                 "'fuel node' command failed. This was the last try.")
 118         node_table = stdout.readlines()\
 119
 120         # skip table title and parse table values
 121
 122         for entry in node_table[3:]:
 123             if entry[0] == '+' or entry[0] == '\n':
 124                 print entry
 125                 pass
 126             else:
 127                 self.__nodes.append(
 128                     Node([str(x.strip(' \n')) for x in entry.split('|')]))
 129
 130     def get_controllers(self):
 131         # Get list of controllers
 132         print self.__nodes[0]._Node__ip
 133         return (
 134             [node for node in self.__nodes if 'controller' in node.get_name()])
 135
 136     def get_computes(self):
 137         # Get list of computes
 138         return (
 139             [node for node in self.__nodes if 'compute' in node.get_name()])
 140
 141     def get_nodes(self):
 142         # Get list of nodes
 143         return self.__nodes
 144
 145     def __open_sftp_session(self, host, user, passwd=None):
 146         # Connect to given host.
 147         """Keyword arguments:
 148         host -- host to connect
 149         user -- user to use
 150         passwd -- password to use
 151
 152         Return tuple of SSH and SFTP client instances.
 153         """
 154         # create SSH client
 155         ssh = paramiko.SSHClient()
 156         ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
 157
 158         # try a direct access using password or private key
 159         if not passwd and not self.__priv_key:
 160             # get private key
 161             self.__priv_key = paramiko.RSAKey.from_private_key_file(
 162                 self.__private_key_file)
 163
 164         # connect to the server
 165         ssh.connect(
 166             host, username=user, password=passwd, pkey=self.__priv_key)
 167         sftp = ssh.open_sftp()
 168
 169         # return SFTP client instance
 170         return ssh, sftp
 171
 172     def get_plugin_interval(self, compute, plugin):
 173         """Find the plugin interval in collectd configuration.
 174
 175         Keyword arguments:
 176         compute -- compute node instance
 177         plugin -- plug-in name
 178
 179         If found, return interval value, otherwise the default value"""
 180         default_interval = DEF_PLUGIN_INTERVAL
 181         compute_name = compute.get_name()
 182         nodes = get_apex_nodes()
 183         for node in nodes:
 184             if compute_name == node.get_dict()['name']:
 185                 stdout = node.run_cmd(
 186                     'cat /etc/collectd/collectd.conf.d/{}.conf'.format(plugin))
 187                 for line in stdout.split('\n'):
 188                     if 'Interval' in line:
 189                         # line = line.strip('Interval')
 190                         return 1
 191         return default_interval
 192
 193     def get_plugin_config_values(self, compute, plugin, parameter):
 194         """Get parameter values from collectd config file.
 195
 196         Keyword arguments:
 197         compute -- compute node instance
 198         plugin -- plug-in name
 199         parameter -- plug-in parameter
 200
 201         Return list of found values."""
 202         default_values = []
 203         compute_name = compute.get_name()
 204         nodes = get_apex_nodes()
 205         for node in nodes:
 206             if compute_name == node.get_dict()['name']:
 207                 stdout = node.run_cmd(
 208                     'cat /etc/collectd/collectd.conf.d/{}.conf' .format(plugin))
 209                 for line in stdout.split('\n'):
 210                     if 'Interfaces' in line:
 211                         return line.split(' ', 1)[1]
 212                     elif 'Bridges' in line:
 213                         return line.split(' ', 1)[1]
 214                     elif 'Cores' in line:
 215                         return line.split(' ', 1)[1]
 216                     else:
 217                         pass
 218         return default_values
 219
 220     def execute_command(self, command, host_ip=None, ssh=None):
 221         """Execute command on node and return list of lines of standard output.
 222
 223         Keyword arguments:
 224         command -- command
 225         host_ip -- IP of the node
 226         ssh -- existing open SSH session to use
 227
 228         One of host_ip or ssh must not be None. If both are not None,
 229         existing ssh session is used.
 230         """
 231         if host_ip is None and ssh is None:
 232             raise ValueError('One of host_ip or ssh must not be None.')
 233         if ssh is None:
 234             ssh, sftp = self.__open_sftp_session(host_ip, 'root', 'opnfvapex')
 235         stdin, stdout, stderr = ssh.exec_command(command)
 236         return stdout.readlines()
 237
 238     def get_ovs_interfaces(self, compute):
 239         """Get list of configured OVS interfaces
 240
 241         Keyword arguments:
 242         compute -- compute node instance
 243         """
 244         compute_name = compute.get_name()
 245         nodes = get_apex_nodes()
 246         for node in nodes:
 247             if compute_name == node.get_dict()['name']:
 248                 stdout = node.run_cmd('sudo ovs-vsctl list-br')
 249         return stdout
 250
 251     def is_gnocchi_running(self, controller):
 252         """Check whether Gnocchi is running on controller.
 253
 254         Keyword arguments:
 255         controller -- controller node instance
 256
 257         Return boolean value whether Gnocchi is running.
 258         """
 259         gnocchi_present = False
 260         controller_name = controller.get_name()
 261         nodes = get_apex_nodes()
 262         for node in nodes:
 263             if controller_name == node.get_dict()['name']:
 264                 node.put_file(
 265                     '/home/opnfv/functest/conf/openstack.creds',
 266                     'overcloudrc.v3')
 267                 stdout = node.run_cmd(
 268                     "source overcloudrc.v3;"
 269                     + "openstack catalog list | grep gnocchi")
 270                 if 'gnocchi' in stdout:
 271                     gnocchi_present = True
 272         return gnocchi_present
 273
 274     def is_aodh_running(self, controller):
 275         """Check whether aodh service is running on controller
 276         """
 277         aodh_present = False
 278         controller_name = controller.get_name()
 279         nodes = get_apex_nodes()
 280         for node in nodes:
 281             if controller_name == node.get_dict()['name']:
 282                 node.put_file(
 283                     '/home/opnfv/functest/conf/openstack.creds',
 284                     'overcloudrc.v3')
 285                 stdout = node.run_cmd(
 286                     "source overcloudrc.v3;"
 287                     + "openstack catalog list | grep aodh")
 288                 if 'aodh' in stdout:
 289                     aodh_present = True
 290         return aodh_present
 291
 292     def is_mcelog_installed(self, compute, package):
 293         """Check whether package exists on compute node.
 294
 295         Keyword arguments:
 296         compute -- compute node instance
 297         package -- Linux package to search for
 298
 299         Return boolean value whether package is installed.
 300         """
 301         compute_name = compute.get_name()
 302         nodes = get_apex_nodes()
 303         for node in nodes:
 304             if compute_name == node.get_dict()['name']:
 305                 stdout = node.run_cmd(
 306                     'yum list installed | grep mcelog')
 307                 if 'mcelog' in stdout:
 308                     return 1
 309                 else:
 310                     return 0
 311
 312     def is_libpqos_on_node(self, compute):
 313         """Check whether libpqos is present on compute node"""
 314
 315         compute_name = compute.get_name()
 316         nodes = get_apex_nodes()
 317         for node in nodes:
 318             if compute_name == node.get_dict()['name']:
 319                 stdout = node.run_cmd('ls /usr/local/lib/ | grep libpqos')
 320                 if 'libpqos' in stdout:
 321                     return True
 322         return False
 323
 324     def check_aodh_plugin_included(self, compute):
 325         """Check if aodh plugin is included in collectd.conf file.
 326         If not, try to enable it.
 327
 328         Keyword arguments:
 329         compute -- compute node instance
 330
 331         Return boolean value whether AODH plugin is included
 332         or it's enabling was successful.
 333         """
 334         compute_name = compute.get_name()
 335         nodes = get_apex_nodes()
 336         for node in nodes:
 337             if compute_name == node.get_dict()['name']:
 338                 aodh_conf = node.run_cmd('ls /etc/collectd/collectd.conf.d')
 339                 if 'aodh.conf' not in aodh_conf:
 340                     self.__logger.info(
 341                         "AODH Plugin not included in compute node")
 342                     return False
 343                 else:
 344                     self.__logger.info(
 345                         "AODH plugin present in compute node {}" .format(
 346                             compute_name))
 347                     return True
 348         return True
 349
 350     def check_gnocchi_plugin_included(self, compute):
 351         """Check if gnocchi plugin is included in collectd.conf file.
 352         If not, try to enable it.
 353
 354         Keyword arguments:
 355         compute -- compute node instance
 356
 357         Return boolean value whether gnocchi plugin is included
 358         or it's enabling was successful.
 359         """
 360         compute_name = compute.get_name()
 361         nodes = get_apex_nodes()
 362         for node in nodes:
 363             if compute_name == node.get_dict()['name']:
 364                 gnocchi_conf = node.run_cmd('ls /etc/collectd/collectd.conf.d')
 365                 if 'collectd-ceilometer-plugin.conf' not in gnocchi_conf:
 366                     self.__logger.info("Gnocchi Plugin not included")
 367                     return False
 368                 else:
 369                     self.__logger.info(
 370                         "Gnochi plugin available in compute node {}" .format(
 371                             compute_name))
 372                     return True
 373         return True
 374
 375     def enable_plugins(
 376             self, compute, plugins, error_plugins, create_backup=True):
 377         """Enable plugins on compute node
 378
 379         Keyword arguments:
 380         compute -- compute node instance
 381         plugins -- list of plugins to be enabled
 382
 383         Return boolean value indicating whether function was successful.
 384         """
 385         plugins = sorted(plugins)
 386         compute_name = compute.get_name()
 387         nodes = get_apex_nodes()
 388         for node in nodes:
 389             if compute_name == node.get_dict()['name']:
 390                 node.put_file(
 391                     '/usr/local/lib/python2.7/dist-packages/baro_tests/'
 392                     + 'csv.conf', 'csv.conf')
 393                 node.run_cmd(
 394                     'sudo cp csv.conf '
 395                     + '/etc/collectd/collectd.conf.d/csv.conf')
 396         return True
 397
 398     def restart_collectd(self, compute):
 399         """Restart collectd on compute node.
 400
 401         Keyword arguments:
 402         compute -- compute node instance
 403
 404         Retrun tuple with boolean indicating success and list of warnings
 405         received during collectd start.
 406         """
 407         compute_name = compute.get_name()
 408         nodes = get_apex_nodes()
 409
 410         def get_collectd_processes(compute_node):
 411             """Get number of running collectd processes.
 412
 413             Keyword arguments:
 414             ssh_session -- instance of SSH session in which to check
 415                 for processes
 416             """
 417             stdout = compute_node.run_cmd("pgrep collectd")
 418             return len(stdout)
 419
 420         for node in nodes:
 421             if compute_name == node.get_dict()['name']:
 422                 # node.run_cmd('su; "opnfvapex"')
 423                 self.__logger.info('Stopping collectd service...')
 424                 node.run_cmd('sudo systemctl stop collectd')
 425                 time.sleep(10)
 426                 if get_collectd_processes(node):
 427                     self.__logger.error('Collectd is still running...')
 428                     return False, []
 429                 self.__logger.info('Starting collectd service...')
 430                 stdout = node.run_cmd('sudo systemctl start collectd')
 431                 time.sleep(10)
 432                 warning = [
 433                     output.strip() for output in stdout if 'WARN: ' in output]
 434                 if get_collectd_processes(node) == 0:
 435                     self.__logger.error('Collectd is still not running...')
 436                     return False, warning
 437         return True, warning
 438
 439     def test_plugins_with_aodh(
 440             self, compute, plugin_interval, logger,
 441             criteria_list=[]):
 442
 443         metric_id = {}
 444         timestamps1 = {}
 445         timestamps2 = {}
 446         nodes = get_apex_nodes()
 447         for node in nodes:
 448             if node.is_controller():
 449                 self.__logger.info('Getting AODH Alarm list on {}' .format(
 450                     (node.get_dict()['name'])))
 451                 node.put_file(
 452                     '/home/opnfv/functest/conf/openstack.creds',
 453                     'overcloudrc.v3')
 454                 stdout = node.run_cmd(
 455                     "source overcloudrc.v3;"
 456                     + "aodh alarm list | grep {0} | grep {1}"
 457                     .format(criteria_list, compute))
 458                 for line in stdout.splitlines():
 459                     line = line.replace('|', "")
 460                     metric_id = line.split()[0]
 461                     stdout = node.run_cmd(
 462                         'source overcloudrc.v3; aodh alarm show {}' .format(
 463                             metric_id))
 464                     for line in stdout.splitlines()[3: -1]:
 465                         line = line.replace('|', "")
 466                         if line.split()[0] == 'timestamp':
 467                             timestamps1 = line.split()[1]
 468                             self.__logger.info("timestamp_before = {}" .format(
 469                                 timestamps1))
 470                         else:
 471                             pass
 472                     time.sleep(12)
 473                     stdout = node.run_cmd(
 474                         "source overcloudrc.v3; aodh alarm show {}" .format(
 475                             metric_id))
 476                     for line in stdout.splitlines()[3:-1]:
 477                         line = line.replace('|', "")
 478                         if line.split()[0] == 'timestamp':
 479                             timestamps2 = line.split()[1]
 480                             self.__logger.info("timestamp_after = {}" .format(
 481                                 timestamps2))
 482                         else:
 483                             pass
 484                     if timestamps1 == timestamps2:
 485                         self.__logger.info(
 486                             "Data not updated after interval of 12 seconds")
 487                         return False
 488                     else:
 489                         self.__logger.info("PASS")
 490                         return True
 491
 492     def test_plugins_with_gnocchi(
 493             self, compute, plugin_interval, logger,
 494             criteria_list=[]):
 495
 496         metric_id = {}
 497         timestamps1 = {}
 498         timestamps2 = {}
 499         nodes = get_apex_nodes()
 500         for node in nodes:
 501             if node.is_controller():
 502                 self.__logger.info('Getting gnocchi metric list on {}' .format(
 503                     (node.get_dict()['name'])))
 504                 node.put_file(
 505                     '/home/opnfv/functest/conf/openstack.creds',
 506                     'overcloudrc.v3')
 507                 stdout = node.run_cmd(
 508                     "source overcloudrc.v3;"
 509                     + "gnocchi metric list | grep {0} | grep {1}"
 510                     .format(criteria_list, compute))
 511                 for line in stdout.splitlines():
 512                     line = line.replace('|', "")
 513                     metric_id = line.split()[0]
 514                     stdout = node.run_cmd(
 515                         'source overcloudrc.v3;gnocchi measures show {}'.format(
 516                             metric_id))
 517                     for line in stdout.splitlines()[3: -1]:
 518                         if line[0] == '+':
 519                             pass
 520                         else:
 521                             timestamps1 = line.replace('|', "")
 522                             timestamps1 = timestamps1.split()[0]
 523                     time.sleep(10)
 524                     stdout = node.run_cmd(
 525                         "source overcloudrc.v3;gnocchi measures show {}".format(
 526                             metric_id))
 527                     for line in stdout.splitlines()[3:-1]:
 528                         if line[0] == '+':
 529                             pass
 530                         else:
 531                             timestamps2 = line.replace('|', "")
 532                             timestamps2 = timestamps2.split()[0]
 533                     if timestamps1 == timestamps2:
 534                         self.__logger.info("Data not updated after 12 seconds")
 535                         return False
 536                     else:
 537                         self.__logger.info("PASS")
 538                         return True