1 # -*- coding: utf-8 -*-
3 # Licensed under the Apache License, Version 2.0 (the "License"); you may
4 # not use this file except in compliance with the License. You may obtain
5 # a copy of the License at
7 # http://www.apache.org/licenses/LICENSE-2.0
9 # Unless required by applicable law or agreed to in writing, software
10 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
11 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
12 # License for the specific language governing permissions and limitations
15 """Classes used by collectd.py"""
23 from opnfv.deployment import factory
24 ID_RSA_PATH = '/root/.ssh/id_rsa'
25 SSH_KEYS_SCRIPT = '/home/opnfv/barometer/baro_utils/get_ssh_keys.sh'
26 DEF_PLUGIN_INTERVAL = 10
27 COLLECTD_CONF = '/etc/collectd.conf'
28 COLLECTD_CONF_DIR = '/etc/collectd/collectd.conf.d'
29 NOTIFICATION_FILE = '/var/log/python-notifications.dump'
30 COLLECTD_NOTIFICATION = '/etc/collectd_notification_dump.py'
31 APEX_IP = subprocess.check_output("echo $INSTALLER_IP", shell=True)
33 APEX_USER_STACK = 'stack'
34 APEX_PKEY = '/root/.ssh/id_rsa'
38 """Node configuration class"""
39 def __init__(self, attrs):
40 self.__null = attrs[0]
42 self.__name = attrs[2]
43 self.__status = attrs[3] if attrs[3] else None
44 self.__taskState = attrs[4]
45 self.__pwrState = attrs[5]
46 self.__ip = re.sub('^[a-z]+=', '', attrs[6])
57 """Get node IP address"""
66 handler = factory.Factory.get_handler('apex',
70 nodes = handler.get_nodes()
74 class ConfigServer(object):
75 """Class to get env configuration"""
76 def __init__(self, host, user, logger, priv_key=None):
80 self.__priv_key = priv_key
82 self.__logger = logger
84 self.__private_key_file = ID_RSA_PATH
85 if not os.path.isfile(self.__private_key_file):
87 "Private key file '{}'".format(self.__private_key_file)
89 raise IOError("Private key file '{}' not found.".format(
90 self.__private_key_file))
92 # get list of available nodes
93 ssh, sftp = self.__open_sftp_session(
94 self.__host, self.__user, self.__passwd)
96 fuel_node_passed = False
98 while (attempt <= 10) and not fuel_node_passed:
99 stdin, stdout, stderr = ssh.exec_command(
100 "source stackrc; nova list")
101 stderr_lines = stderr.readlines()
103 self.__logger.warning(
104 "'fuel node' command failed (try {}):".format(attempt))
105 for line in stderr_lines:
106 self.__logger.debug(line.strip())
108 fuel_node_passed = True
111 "'fuel node' command passed (try {})".format(attempt))
113 if not fuel_node_passed:
115 "'fuel node' command failed. This was the last try.")
117 "'fuel node' command failed. This was the last try.")
118 node_table = stdout.readlines()\
120 # skip table title and parse table values
122 for entry in node_table[3:]:
123 if entry[0] == '+' or entry[0] == '\n':
128 Node([str(x.strip(' \n')) for x in entry.split('|')]))
130 def get_controllers(self):
131 # Get list of controllers
132 print self.__nodes[0]._Node__ip
134 [node for node in self.__nodes if 'controller' in node.get_name()])
136 def get_computes(self):
137 # Get list of computes
139 [node for node in self.__nodes if 'compute' in node.get_name()])
145 def __open_sftp_session(self, host, user, passwd=None):
146 # Connect to given host.
147 """Keyword arguments:
148 host -- host to connect
150 passwd -- password to use
152 Return tuple of SSH and SFTP client instances.
155 ssh = paramiko.SSHClient()
156 ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
158 # try a direct access using password or private key
159 if not passwd and not self.__priv_key:
161 self.__priv_key = paramiko.RSAKey.from_private_key_file(
162 self.__private_key_file)
164 # connect to the server
166 host, username=user, password=passwd, pkey=self.__priv_key)
167 sftp = ssh.open_sftp()
169 # return SFTP client instance
172 def get_plugin_interval(self, compute, plugin):
173 """Find the plugin interval in collectd configuration.
176 compute -- compute node instance
177 plugin -- plug-in name
179 If found, return interval value, otherwise the default value"""
180 default_interval = DEF_PLUGIN_INTERVAL
181 compute_name = compute.get_name()
182 nodes = get_apex_nodes()
184 if compute_name == node.get_dict()['name']:
185 stdout = node.run_cmd(
186 'cat /etc/collectd/collectd.conf.d/{}.conf'.format(plugin))
187 for line in stdout.split('\n'):
188 if 'Interval' in line:
189 # line = line.strip('Interval')
191 return default_interval
193 def get_plugin_config_values(self, compute, plugin, parameter):
194 """Get parameter values from collectd config file.
197 compute -- compute node instance
198 plugin -- plug-in name
199 parameter -- plug-in parameter
201 Return list of found values."""
203 compute_name = compute.get_name()
204 nodes = get_apex_nodes()
206 if compute_name == node.get_dict()['name']:
207 stdout = node.run_cmd(
208 'cat /etc/collectd/collectd.conf.d/{}.conf' .format(plugin))
209 for line in stdout.split('\n'):
210 if 'Interfaces' in line:
211 return line.split(' ', 1)[1]
212 elif 'Bridges' in line:
213 return line.split(' ', 1)[1]
214 elif 'Cores' in line:
215 return line.split(' ', 1)[1]
218 return default_values
220 def execute_command(self, command, host_ip=None, ssh=None):
221 """Execute command on node and return list of lines of standard output.
225 host_ip -- IP of the node
226 ssh -- existing open SSH session to use
228 One of host_ip or ssh must not be None. If both are not None,
229 existing ssh session is used.
231 if host_ip is None and ssh is None:
232 raise ValueError('One of host_ip or ssh must not be None.')
234 ssh, sftp = self.__open_sftp_session(host_ip, 'root', 'opnfvapex')
235 stdin, stdout, stderr = ssh.exec_command(command)
236 return stdout.readlines()
238 def get_ovs_interfaces(self, compute):
239 """Get list of configured OVS interfaces
242 compute -- compute node instance
244 compute_name = compute.get_name()
245 nodes = get_apex_nodes()
247 if compute_name == node.get_dict()['name']:
248 stdout = node.run_cmd('sudo ovs-vsctl list-br')
251 def is_gnocchi_running(self, controller):
252 """Check whether Gnocchi is running on controller.
255 controller -- controller node instance
257 Return boolean value whether Gnocchi is running.
259 gnocchi_present = False
260 lines = self.execute_command(
261 'source overcloudrc.v3;systemctl status openstack-gnocchi-api | '
262 + 'grep running', controller.get_ip())
264 if '(running)' in line:
265 gnocchi_present = True
266 return gnocchi_present
268 def is_aodh_running(self, controller):
269 """Check whether aodh service is running on controller
272 lines = self.execute_command(
273 'source overcloudrc.v3;systemctl openstack-aodh-api | grep running',
276 self.__logger.info("Line = {}" .format(line))
277 if '(running)' in line:
281 def is_installed(self, compute, package):
282 """Check whether package exists on compute node.
285 compute -- compute node instance
286 package -- Linux package to search for
288 Return boolean value whether package is installed.
290 compute_name = compute.get_name()
291 nodes = get_apex_nodes()
293 if compute_name == node.get_dict()['name']:
294 stdout = node.run_cmd(
295 'yum list installed | grep mcelog')
296 if 'mcelog' in stdout:
301 def is_libpqos_on_node(self, compute):
302 """Check whether libpqos is present on compute node"""
304 compute_name = compute.get_name()
305 nodes = get_apex_nodes()
307 if compute_name == node.get_dict()['name']:
308 stdout = node.run_cmd('ls /usr/local/lib/ | grep libpqos')
309 if 'libpqos' in stdout:
313 def check_gnocchi_plugin_included(self, compute):
314 """Check if gnocchi plugin is included in collectd.conf file.
315 If not, try to enable it.
318 compute -- compute node instance
320 Return boolean value whether gnocchi plugin is included
321 or it's enabling was successful.
323 compute_name = compute.get_name()
324 nodes = get_apex_nodes()
326 if compute_name == node.get_dict()['name']:
327 # node.run_cmd('su; "opnfvapex"')
328 gnocchi_conf = node.run_cmd('ls /etc/collectd/collectd.conf.d')
329 if 'collectd-ceilometer-plugin.conf' not in gnocchi_conf:
330 self.__logger.info("Gnocchi Plugin not included")
333 self.__logger.info("Gnochi plugin present")
338 self, compute, plugins, error_plugins, create_backup=True):
339 """Enable plugins on compute node
342 compute -- compute node instance
343 plugins -- list of plugins to be enabled
344 error_plugins -- list of tuples with found errors, new entries
345 may be added there (plugin, error_description, is_critical):
346 plugin -- plug-in name
347 error_decription -- description of the error
348 is_critical -- boolean value indicating whether error
350 create_backup -- boolean value indicating whether backup
353 Return boolean value indicating whether function was successful.
355 plugins = sorted(plugins)
356 compute_name = compute.get_name()
357 nodes = get_apex_nodes()
359 if compute_name == node.get_dict()['name']:
361 '/usr/local/lib/python2.7/dist-packages/baro_tests/'
362 + 'csv.conf', 'csv.conf')
365 + '/etc/collectd/collectd.conf.d/csv.conf')
368 def restore_config(self, compute):
369 """Restore collectd config file from backup on compute node.
372 compute -- compute node instance
374 ssh, sftp = self.__open_sftp_session(
375 compute.get_ip(), 'root', 'opnfvapex')
377 self.__logger.info('Restoring config file from backup...')
378 ssh.exec_command("cp {0} {0}.used".format(COLLECTD_CONF))
379 ssh.exec_command("cp {0}.backup {0}".format(COLLECTD_CONF))
381 def restart_collectd(self, compute):
382 """Restart collectd on compute node.
385 compute -- compute node instance
387 Retrun tuple with boolean indicating success and list of warnings
388 received during collectd start.
390 compute_name = compute.get_name()
391 nodes = get_apex_nodes()
393 def get_collectd_processes(compute_node):
394 """Get number of running collectd processes.
397 ssh_session -- instance of SSH session in which to check
400 stdout = compute_node.run_cmd("pgrep collectd")
404 if compute_name == node.get_dict()['name']:
405 # node.run_cmd('su; "opnfvapex"')
406 self.__logger.info('Stopping collectd service...')
407 node.run_cmd('sudo systemctl stop collectd')
409 if get_collectd_processes(node):
410 self.__logger.error('Collectd is still running...')
412 self.__logger.info('Starting collectd service...')
413 stdout = node.run_cmd('sudo systemctl start collectd')
416 output.strip() for output in stdout if 'WARN: ' in output]
417 if get_collectd_processes(node) == 0:
418 self.__logger.error('Collectd is still not running...')
419 return False, warning
422 def test_gnocchi_is_sending_data(self, controller):
423 """ Checking if Gnocchi is sending metrics to controller"""
427 ssh, sftp = self.__open_sftp_session(
428 controller.get_ip(), 'root', 'opnfvapex')
430 self.__logger.info('Getting gnocchi metric list on{}'.format(
431 controller.get_name()))
432 stdout = self.execute_command(
433 "source overcloudrc.v3;gnocchi metric list | grep if_packets",
436 metric_ids = [r.split('|')[1] for r in stdout]
437 self.__logger.info("Metric ids = {}" .format(metric_ids))
438 for metric_id in metric_ids:
439 metric_id = metric_id.replace("u", "")
440 stdout = self.execute_command(
441 "source overcloudrc.v3;gnocchi measures show {}" .format(
443 self.__logger.info("stdout measures ={}" .format(stdout))
448 self.__logger.info("Line = {}" .format(line))
449 timestamps1 = [line.split('|')[1]]
450 self.__logger.info("Last line timetamp1 = {}" .format(timestamps1))
452 stdout = self.execute_command(
453 "source overcloudrc.v3;gnocchi measures show {}" .format(
459 timestamps2 = [line.split('|')[1]]
460 self.__logger.info("Last line timetamp2 = {}" .format(timestamps2))
461 if timestamps1 == timestamps2:
462 self.__logger.info("False")
466 self.__logger.info("True")
469 def test_plugins_with_aodh(self, controller):
470 """Checking if AODH is sending metrics to controller"""
474 ssh, sftp = self.__open_sftp_session(
475 controller.get_ip(), 'root', 'opnfvapex')
476 self.__logger.info('Getting AODH alarm list on{}'.format(
477 controller.get_name()))
478 stdout = self.execute_command(
479 "source overcloudrc.v3;aodh alarm list | grep mcelog",
482 metric_ids = [r.split('|')[1] for r in stdout]
483 self.__logger.info("Metric ids = {}" .format(metric_ids))
484 for metric_id in metric_ids:
485 metric_id = metric_id.replace("u", "")
486 stdout = self.execute_command(
487 "source overcloudrc.v3;aodh alarm show {}" .format(
489 self.__logger.info("stdout alarms ={}" .format(stdout))
494 self.__logger.info("Line = {}" .format(line))
495 timestamps1 = [line.split('|')[1]]
496 self.__logger.info("Last line timetamp1 = {}" .format(timestamps1))
498 stdout = self.execute_command(
499 "source overcloudrc.v3;aodh alarm show {}" .format(
505 timestamps2 = [line.split('|')[1]]
506 self.__logger.info("Last line timetamp2 = {}" .format(timestamps2))
507 if timestamps1 == timestamps2:
508 self.__logger.info("False")
512 self.__logger.info("True")
515 def test_plugins_with_gnocchi(
516 self, controller, compute_node, plugin_interval, logger,
522 ssh, sftp = self.__open_sftp_session(
523 controller.get_ip(), 'root', 'opnfvapex')
524 self.__logger.info('Getting gnocchi metric list on{}'.format(
525 controller.get_name()))
526 stdout = self.execute_command(
527 "source overcloudrc.v3;gnocchi metric list | grep {0} | grep {1}"
528 .format(compute_node.get_name(), criteria_list), ssh=ssh)
530 metric_ids = [r.split('|')[1] for r in stdout]
531 self.__logger.info("Metric ids = {}" .format(metric_ids))
532 for metric_id in metric_ids:
533 metric_id = metric_id.replace("u", "")
534 stdout = self.execute_command(
535 "source overcloudrc.v3;gnocchi measures show {}" .format(
537 self.__logger.info("stdout measures ={}" .format(stdout))
542 self.__logger.info("Line = {}" .format(line))
543 timestamps1 = [line.split('|')[1]]
544 self.__logger.info("Last line timetamp1 = {}" .format(timestamps1))
546 stdout = self.execute_command(
547 "source overcloudrc.v3;gnocchi measures show {}" .format(
553 timestamps2 = [line.split('|')[1]]
554 self.__logger.info("Last line timetamp2 = {}" .format(timestamps2))
555 if timestamps1 == timestamps2:
556 self.__logger.info("False")
559 self.__logger.info("True")