1 # -*- coding: utf-8 -*-
3 # Licensed under the Apache License, Version 2.0 (the "License"); you may
4 # not use this file except in compliance with the License. You may obtain
5 # a copy of the License at
7 # http://www.apache.org/licenses/LICENSE-2.0
9 # Unless required by applicable law or agreed to in writing, software
10 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
11 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
12 # License for the specific language governing permissions and limitations
15 """Classes used by collectd.py"""
23 from opnfv.deployment import factory
24 ID_RSA_PATH = '/root/.ssh/id_rsa'
25 SSH_KEYS_SCRIPT = '/home/opnfv/barometer/baro_utils/get_ssh_keys.sh'
26 DEF_PLUGIN_INTERVAL = 10
27 COLLECTD_CONF = '/etc/collectd.conf'
28 COLLECTD_CONF_DIR = '/etc/collectd/collectd.conf.d'
29 NOTIFICATION_FILE = '/var/log/python-notifications.dump'
30 COLLECTD_NOTIFICATION = '/etc/collectd_notification_dump.py'
31 APEX_IP = subprocess.check_output("echo $INSTALLER_IP", shell=True)
33 APEX_USER_STACK = 'stack'
34 APEX_PKEY = '/root/.ssh/id_rsa'
38 """Node configuration class"""
39 def __init__(self, attrs):
40 self.__null = attrs[0]
42 self.__name = attrs[2]
43 self.__status = attrs[3] if attrs[3] else None
44 self.__taskState = attrs[4]
45 self.__pwrState = attrs[5]
46 self.__ip = re.sub('^[a-z]+=', '', attrs[6])
57 """Get node IP address"""
66 handler = factory.Factory.get_handler('apex',
70 nodes = handler.get_nodes()
74 class ConfigServer(object):
75 """Class to get env configuration"""
76 def __init__(self, host, user, logger, priv_key=None):
80 self.__priv_key = priv_key
82 self.__logger = logger
84 self.__private_key_file = ID_RSA_PATH
85 if not os.path.isfile(self.__private_key_file):
87 "Private key file '{}'".format(self.__private_key_file)
89 raise IOError("Private key file '{}' not found.".format(
90 self.__private_key_file))
92 # get list of available nodes
93 ssh, sftp = self.__open_sftp_session(
94 self.__host, self.__user, self.__passwd)
96 fuel_node_passed = False
98 while (attempt <= 10) and not fuel_node_passed:
99 stdin, stdout, stderr = ssh.exec_command(
100 "source stackrc; nova list")
101 stderr_lines = stderr.readlines()
103 self.__logger.warning(
104 "'fuel node' command failed (try {}):".format(attempt))
105 for line in stderr_lines:
106 self.__logger.debug(line.strip())
108 fuel_node_passed = True
111 "'fuel node' command passed (try {})".format(attempt))
113 if not fuel_node_passed:
115 "'fuel node' command failed. This was the last try.")
117 "'fuel node' command failed. This was the last try.")
118 node_table = stdout.readlines()\
120 # skip table title and parse table values
122 for entry in node_table[3:]:
123 if entry[0] == '+' or entry[0] == '\n':
128 Node([str(x.strip(' \n')) for x in entry.split('|')]))
130 def get_controllers(self):
131 # Get list of controllers
132 print self.__nodes[0]._Node__ip
134 [node for node in self.__nodes if 'controller' in node.get_name()])
136 def get_computes(self):
137 # Get list of computes
139 [node for node in self.__nodes if 'compute' in node.get_name()])
145 def __open_sftp_session(self, host, user, passwd=None):
146 # Connect to given host.
147 """Keyword arguments:
148 host -- host to connect
150 passwd -- password to use
152 Return tuple of SSH and SFTP client instances.
155 ssh = paramiko.SSHClient()
156 ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
158 # try a direct access using password or private key
159 if not passwd and not self.__priv_key:
161 self.__priv_key = paramiko.RSAKey.from_private_key_file(
162 self.__private_key_file)
164 # connect to the server
166 host, username=user, password=passwd, pkey=self.__priv_key)
167 sftp = ssh.open_sftp()
169 # return SFTP client instance
172 def get_plugin_interval(self, compute, plugin):
173 """Find the plugin interval in collectd configuration.
176 compute -- compute node instance
177 plugin -- plug-in name
179 If found, return interval value, otherwise the default value"""
180 default_interval = DEF_PLUGIN_INTERVAL
181 compute_name = compute.get_name()
182 nodes = get_apex_nodes()
184 if compute_name == node.get_dict()['name']:
185 stdout = node.run_cmd(
186 'cat /etc/collectd/collectd.conf.d/{}.conf'.format(plugin))
187 for line in stdout.split('\n'):
188 if 'Interval' in line:
189 # line = line.strip('Interval')
191 return default_interval
193 def get_plugin_config_values(self, compute, plugin, parameter):
194 """Get parameter values from collectd config file.
197 compute -- compute node instance
198 plugin -- plug-in name
199 parameter -- plug-in parameter
201 Return list of found values."""
203 compute_name = compute.get_name()
204 nodes = get_apex_nodes()
206 if compute_name == node.get_dict()['name']:
207 stdout = node.run_cmd(
208 'cat /etc/collectd/collectd.conf.d/{}.conf' .format(plugin))
209 for line in stdout.split('\n'):
210 if 'Interfaces' in line:
211 return line.split(' ', 1)[1]
212 elif 'Bridges' in line:
213 return line.split(' ', 1)[1]
214 elif 'Cores' in line:
215 return line.split(' ', 1)[1]
218 return default_values
220 def execute_command(self, command, host_ip=None, ssh=None):
221 """Execute command on node and return list of lines of standard output.
225 host_ip -- IP of the node
226 ssh -- existing open SSH session to use
228 One of host_ip or ssh must not be None. If both are not None,
229 existing ssh session is used.
231 if host_ip is None and ssh is None:
232 raise ValueError('One of host_ip or ssh must not be None.')
234 ssh, sftp = self.__open_sftp_session(host_ip, 'root', 'opnfvapex')
235 stdin, stdout, stderr = ssh.exec_command(command)
236 return stdout.readlines()
238 def get_ovs_interfaces(self, compute):
239 """Get list of configured OVS interfaces
242 compute -- compute node instance
244 compute_name = compute.get_name()
245 nodes = get_apex_nodes()
247 if compute_name == node.get_dict()['name']:
248 stdout = node.run_cmd('sudo ovs-vsctl list-br')
251 def is_gnocchi_running(self, controller):
252 """Check whether Gnocchi is running on controller.
255 controller -- controller node instance
257 Return boolean value whether Gnocchi is running.
259 gnocchi_present = False
260 lines = self.execute_command(
261 'source overcloudrc.v3;systemctl status openstack-gnocchi-api | '
262 + 'grep running', controller.get_ip())
264 if '(running)' in line:
265 gnocchi_present = True
266 return gnocchi_present
268 def is_aodh_running(self, controller):
269 """Check whether aodh service is running on controller
272 lines = self.execute_command(
273 'source overcloudrc.v3;systemctl openstack-aodh-api | grep running',
276 self.__logger.info("Line = {}" .format(line))
277 if '(running)' in line:
281 def is_installed(self, compute, package):
282 """Check whether package exists on compute node.
285 compute -- compute node instance
286 package -- Linux package to search for
288 Return boolean value whether package is installed.
290 compute_name = compute.get_name()
291 nodes = get_apex_nodes()
293 if compute_name == node.get_dict()['name']:
294 stdout = node.run_cmd(
295 'yum list installed | grep {}'.format(package))
296 return len(stdout) > 0
298 def is_libpqos_on_node(self, compute):
299 """Check whether libpqos is present on compute node"""
301 compute_name = compute.get_name()
302 nodes = get_apex_nodes()
304 if compute_name == node.get_dict()['name']:
305 stdout = node.run_cmd('ls /usr/local/lib/ | grep libpqos')
306 if 'libpqos' in stdout:
310 def check_gnocchi_plugin_included(self, compute):
311 """Check if gnocchi plugin is included in collectd.conf file.
312 If not, try to enable it.
315 compute -- compute node instance
317 Return boolean value whether gnocchi plugin is included
318 or it's enabling was successful.
320 compute_name = compute.get_name()
321 nodes = get_apex_nodes()
323 if compute_name == node.get_dict()['name']:
324 # node.run_cmd('su; "opnfvapex"')
325 gnocchi_conf = node.run_cmd('ls /etc/collectd/collectd.conf.d')
326 if 'collectd-ceilometer-plugin.conf' not in gnocchi_conf:
327 self.__logger.info("Gnocchi Plugin not included")
330 self.__logger.info("Gnochi plugin present")
335 self, compute, plugins, error_plugins, create_backup=True):
336 """Enable plugins on compute node
339 compute -- compute node instance
340 plugins -- list of plugins to be enabled
341 error_plugins -- list of tuples with found errors, new entries
342 may be added there (plugin, error_description, is_critical):
343 plugin -- plug-in name
344 error_decription -- description of the error
345 is_critical -- boolean value indicating whether error
347 create_backup -- boolean value indicating whether backup
350 Return boolean value indicating whether function was successful.
352 plugins = sorted(plugins)
353 compute_name = compute.get_name()
354 nodes = get_apex_nodes()
356 if compute_name == node.get_dict()['name']:
358 '/usr/local/lib/python2.7/dist-packages/baro_tests/'
359 + 'csv.conf', 'csv.conf')
362 + '/etc/collectd/collectd.conf.d/csv.conf')
365 def restore_config(self, compute):
366 """Restore collectd config file from backup on compute node.
369 compute -- compute node instance
371 ssh, sftp = self.__open_sftp_session(
372 compute.get_ip(), 'root', 'opnfvapex')
374 self.__logger.info('Restoring config file from backup...')
375 ssh.exec_command("cp {0} {0}.used".format(COLLECTD_CONF))
376 ssh.exec_command("cp {0}.backup {0}".format(COLLECTD_CONF))
378 def restart_collectd(self, compute):
379 """Restart collectd on compute node.
382 compute -- compute node instance
384 Retrun tuple with boolean indicating success and list of warnings
385 received during collectd start.
387 compute_name = compute.get_name()
388 nodes = get_apex_nodes()
390 def get_collectd_processes(compute_node):
391 """Get number of running collectd processes.
394 ssh_session -- instance of SSH session in which to check
397 stdout = compute_node.run_cmd("pgrep collectd")
401 if compute_name == node.get_dict()['name']:
402 # node.run_cmd('su; "opnfvapex"')
403 self.__logger.info('Stopping collectd service...')
404 node.run_cmd('sudo systemctl stop collectd')
406 if get_collectd_processes(node):
407 self.__logger.error('Collectd is still running...')
409 self.__logger.info('Starting collectd service...')
410 stdout = node.run_cmd('sudo systemctl start collectd')
413 output.strip() for output in stdout if 'WARN: ' in output]
414 if get_collectd_processes(node) == 0:
415 self.__logger.error('Collectd is still not running...')
416 return False, warning
419 def test_gnocchi_is_sending_data(self, controller):
420 """ Checking if Gnocchi is sending metrics to controller"""
424 ssh, sftp = self.__open_sftp_session(
425 controller.get_ip(), 'root', 'opnfvapex')
427 self.__logger.info('Getting gnocchi metric list on{}'.format(
428 controller.get_name()))
429 stdout = self.execute_command(
430 "source overcloudrc.v3;gnocchi metric list | grep if_packets",
433 metric_ids = [r.split('|')[1] for r in stdout]
434 self.__logger.info("Metric ids = {}" .format(metric_ids))
435 for metric_id in metric_ids:
436 metric_id = metric_id.replace("u", "")
437 stdout = self.execute_command(
438 "source overcloudrc.v3;gnocchi measures show {}" .format(
440 self.__logger.info("stdout measures ={}" .format(stdout))
445 self.__logger.info("Line = {}" .format(line))
446 timestamps1 = [line.split('|')[1]]
447 self.__logger.info("Last line timetamp1 = {}" .format(timestamps1))
449 stdout = self.execute_command(
450 "source overcloudrc.v3;gnocchi measures show {}" .format(
456 timestamps2 = [line.split('|')[1]]
457 self.__logger.info("Last line timetamp2 = {}" .format(timestamps2))
458 if timestamps1 == timestamps2:
459 self.__logger.info("False")
463 self.__logger.info("True")
466 def test_plugins_with_aodh(self, controller):
467 """Checking if AODH is sending metrics to controller"""
471 ssh, sftp = self.__open_sftp_session(
472 controller.get_ip(), 'root', 'opnfvapex')
473 self.__logger.info('Getting AODH alarm list on{}'.format(
474 controller.get_name()))
475 stdout = self.execute_command(
476 "source overcloudrc.v3;aodh alarm list | grep mcelog",
479 metric_ids = [r.split('|')[1] for r in stdout]
480 self.__logger.info("Metric ids = {}" .format(metric_ids))
481 for metric_id in metric_ids:
482 metric_id = metric_id.replace("u", "")
483 stdout = self.execute_command(
484 "source overcloudrc.v3;aodh alarm show {}" .format(
486 self.__logger.info("stdout alarms ={}" .format(stdout))
491 self.__logger.info("Line = {}" .format(line))
492 timestamps1 = [line.split('|')[1]]
493 self.__logger.info("Last line timetamp1 = {}" .format(timestamps1))
495 stdout = self.execute_command(
496 "source overcloudrc.v3;aodh alarm show {}" .format(
502 timestamps2 = [line.split('|')[1]]
503 self.__logger.info("Last line timetamp2 = {}" .format(timestamps2))
504 if timestamps1 == timestamps2:
505 self.__logger.info("False")
509 self.__logger.info("True")
512 def test_plugins_with_gnocchi(
513 self, controller, compute_node, plugin_interval, logger,
519 ssh, sftp = self.__open_sftp_session(
520 controller.get_ip(), 'root', 'opnfvapex')
521 self.__logger.info('Getting gnocchi metric list on{}'.format(
522 controller.get_name()))
523 stdout = self.execute_command(
524 "source overcloudrc.v3;gnocchi metric list | grep {0} | grep {1}"
525 .format(compute_node.get_name(), criteria_list), ssh=ssh)
527 metric_ids = [r.split('|')[1] for r in stdout]
528 self.__logger.info("Metric ids = {}" .format(metric_ids))
529 for metric_id in metric_ids:
530 metric_id = metric_id.replace("u", "")
531 stdout = self.execute_command(
532 "source overcloudrc.v3;gnocchi measures show {}" .format(
534 self.__logger.info("stdout measures ={}" .format(stdout))
539 self.__logger.info("Line = {}" .format(line))
540 timestamps1 = [line.split('|')[1]]
541 self.__logger.info("Last line timetamp1 = {}" .format(timestamps1))
543 stdout = self.execute_command(
544 "source overcloudrc.v3;gnocchi measures show {}" .format(
550 timestamps2 = [line.split('|')[1]]
551 self.__logger.info("Last line timetamp2 = {}" .format(timestamps2))
552 if timestamps1 == timestamps2:
553 self.__logger.info("False")
556 self.__logger.info("True")