1 """Executing test of plugins"""
2 # -*- coding: utf-8 -*-
4 # Licensed under the Apache License, Version 2.0 (the "License"); you may
5 # not use this file except in compliance with the License. You may obtain
6 # a copy of the License at
8 # http://www.apache.org/licenses/LICENSE-2.0
10 # Unless required by applicable law or agreed to in writing, software
11 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
12 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
13 # License for the specific language governing permissions and limitations
17 from keystoneclient.v3 import client
22 from config_server import *
24 from opnfv.deployment import factory
25 from functest.utils import functest_utils
27 CEILOMETER_NAME = 'ceilometer'
28 ID_RSA_SRC = '/root/.ssh/id_rsa'
29 ID_RSA_DST_DIR = '/home/opnfv/.ssh'
30 ID_RSA_DST = ID_RSA_DST_DIR + '/id_rsa'
31 INSTALLER_PARAMS_YAML = pkg_resources.resource_filename(
32 'functest', 'ci/installer_params.yaml')
33 FUEL_IP = functest_utils.get_parameter_from_yaml('fuel.ip', INSTALLER_PARAMS_YAML)
34 FUEL_USER = functest_utils.get_parameter_from_yaml('fuel.user', INSTALLER_PARAMS_YAML)
35 FUEL_PW = functest_utils.get_parameter_from_yaml('fuel.password', INSTALLER_PARAMS_YAML)
38 class KeystoneException(Exception):
39 """Keystone exception class"""
40 def __init__(self, message, exc=None, response=None):
43 message -- error message
48 message += "\nReason: %s" % exc
49 super(KeystoneException, self).__init__(message)
51 self.response = response
55 class InvalidResponse(KeystoneException):
56 """Invalid Keystone exception class"""
57 def __init__(self, exc, response):
63 super(InvalidResponse, self).__init__(
64 "Invalid response", exc, response)
67 class CeilometerClient(object):
68 """Ceilometer Client to authenticate and request meters"""
69 def __init__(self, bc_logger):
72 bc_logger - logger instance
74 self._auth_token = None
75 self._ceilometer_url = None
76 self._meter_list = None
77 self._logger = bc_logger
82 return self._auth_token
84 def get_ceilometer_url(self):
85 """Get Ceilometer URL"""
86 return self._ceilometer_url
88 def get_ceil_metrics(self, criteria=None):
89 """Get Ceilometer metrics for given criteria
92 criteria -- criteria for ceilometer meter list
94 self._request_meters(criteria)
95 return self._meter_list
97 def _auth_server(self):
98 """Request token in authentication server"""
99 self._logger.debug('Connecting to the auth server {}'.format(os.environ['OS_AUTH_URL']))
100 keystone = client.Client(username=os.environ['OS_USERNAME'],
101 password=os.environ['OS_PASSWORD'],
102 tenant_name=os.environ['OS_TENANT_NAME'],
103 auth_url=os.environ['OS_AUTH_URL'])
104 self._auth_token = keystone.auth_token
105 for service in keystone.service_catalog.get_data():
106 if service['name'] == CEILOMETER_NAME:
107 for service_type in service['endpoints']:
108 if service_type['interface'] == 'internal':
109 self._ceilometer_url = service_type['url']
112 if self._ceilometer_url is None:
113 self._logger.warning('Ceilometer is not registered in service catalog')
115 def _request_meters(self, criteria):
116 """Request meter list values from ceilometer
119 criteria -- criteria for ceilometer meter list
122 url = self._ceilometer_url + ('/v2/samples?limit=400')
124 url = self._ceilometer_url + ('/v2/meters/%s?q.field=resource_id&limit=400' % criteria)
125 headers = {'X-Auth-Token': self._auth_token}
126 resp = requests.get(url, headers=headers)
128 resp.raise_for_status()
129 self._meter_list = resp.json()
130 except (KeyError, ValueError, requests.exceptions.HTTPError) as err:
131 raise InvalidResponse(err, resp)
134 class CSVClient(object):
135 """Client to request CSV meters"""
136 def __init__(self, bc_logger, conf):
139 bc_logger - logger instance
140 conf -- ConfigServer instance
142 self._logger = bc_logger
145 def get_csv_metrics(self, compute_node, plugin_subdirectories, meter_categories):
149 compute_node -- compute node instance
150 plugin_subdirectories -- list of subdirectories of plug-in
151 meter_categories -- categories which will be tested
153 Return list of metrics.
155 stdout = self.conf.execute_command("date '+%Y-%m-%d'", compute_node.get_ip())
156 date = stdout[0].strip()
158 for plugin_subdir in plugin_subdirectories:
159 for meter_category in meter_categories:
160 stdout = self.conf.execute_command(
161 "tail -2 /var/lib/collectd/csv/node-"
162 + "{0}.domain.tld/{1}/{2}-{3}".format(
163 compute_node.get_id(), plugin_subdir, meter_category, date),
164 compute_node.get_ip())
165 # Storing last two values
169 'Getting last two CSV entries of meter category '
170 + '{0} in {1} subdir failed'.format(meter_category, plugin_subdir))
172 old_value = int(values[0][0:values[0].index('.')])
173 new_value = int(values[1][0:values[1].index('.')])
174 metrics.append((plugin_subdir, meter_category, old_value, new_value))
179 """Check whether there is global logger available and if not, define one."""
180 if 'logger' not in globals():
182 logger = logger.Logger("barometercollectd").getLogger()
185 def _process_result(compute_node, test, result, results_list):
186 """Print test result and append it to results list.
189 test -- testcase name
190 result -- boolean test result
191 results_list -- results list
194 logger.info('Compute node {0} test case {1} PASSED.'.format(compute_node, test))
196 logger.error('Compute node {0} test case {1} FAILED.'.format(compute_node, test))
197 results_list.append((compute_node, test, result))
200 def _print_label(label):
201 """Print label on the screen
204 label -- label string
206 label = label.strip()
209 label = ' ' + label + ' '
210 length_label = len(label)
211 length1 = (length - length_label) / 2
212 length2 = length - length_label - length1
213 length1 = max(3, length1)
214 length2 = max(3, length2)
215 logger.info(('=' * length1) + label + ('=' * length2))
218 def _print_plugin_label(plugin, node_id):
219 """Print plug-in label.
222 plugin -- plug-in name
225 _print_label('Node {0}: Plug-in {1} Test case execution'.format(node_id, plugin))
228 def _print_final_result_of_plugin(plugin, compute_ids, results, out_plugins, out_plugin):
229 """Print final results of plug-in.
232 plugin -- plug-in name
233 compute_ids -- list of compute node IDs
234 results -- results list
235 out_plugins -- list of out plug-ins
236 out_plugin -- used out plug-in
239 for id in compute_ids:
240 if out_plugins[id] == out_plugin:
241 if (id, plugin, True) in results:
242 print_line += ' PASS |'
243 elif (id, plugin, False) in results and out_plugins[id] == out_plugin:
244 print_line += ' FAIL |'
246 print_line += ' NOT EX |'
247 elif out_plugin == 'Ceilometer':
248 print_line += ' NOT EX |'
250 print_line += ' SKIP |'
254 def print_overall_summary(compute_ids, tested_plugins, results, out_plugins):
255 """Print overall summary table.
258 compute_ids -- list of compute IDs
259 tested_plugins -- list of plug-ins
260 results -- results list
261 out_plugins -- list of used out plug-ins
263 compute_node_names = ['Node-{}'.format(id) for id in compute_ids]
264 all_computes_in_line = ''
265 for compute in compute_node_names:
266 all_computes_in_line = all_computes_in_line + '| ' + compute + (' ' * (7 - len(compute)))
267 line_of_nodes = '| Test ' + all_computes_in_line + '|'
268 logger.info('=' * 70)
269 logger.info('+' + ('-' * ((9 * len(compute_node_names))+16)) + '+')
271 '|' + ' ' * ((9*len(compute_node_names))/2) + ' OVERALL SUMMARY'
272 + ' ' * (9*len(compute_node_names) - (9*len(compute_node_names))/2) + '|')
273 logger.info('+' + ('-' * 16) + '+' + (('-' * 8) + '+') * len(compute_node_names))
274 logger.info(line_of_nodes)
275 logger.info('+' + ('-' * 16) + '+' + (('-' * 8) + '+') * len(compute_node_names))
276 out_plugins_print = ['Ceilometer']
277 if 'CSV' in out_plugins.values():
278 out_plugins_print.append('CSV')
279 for out_plugin in out_plugins_print:
280 output_plugins_line = ''
281 for id in compute_ids:
282 out_plugin_result = '----'
283 if out_plugin == 'Ceilometer':
284 out_plugin_result = 'PASS' if out_plugins[id] == out_plugin else 'FAIL'
285 if out_plugin == 'CSV':
286 if out_plugins[id] == out_plugin:
287 out_plugin_result = \
289 plugin for comp_id, plugin,
290 res in results if comp_id == id and res] else 'FAIL'
292 out_plugin_result = 'SKIP'
293 output_plugins_line += '| ' + out_plugin_result + ' '
295 '| OUT:{}'.format(out_plugin) + (' ' * (11 - len(out_plugin)))
296 + output_plugins_line + '|')
297 for plugin in sorted(tested_plugins.values()):
298 line_plugin = _print_final_result_of_plugin(
299 plugin, compute_ids, results, out_plugins, out_plugin)
300 logger.info('| IN:{}'.format(plugin) + (' ' * (11-len(plugin))) + '|' + line_plugin)
301 logger.info('+' + ('-' * 16) + '+' + (('-' * 8) + '+') * len(compute_node_names))
302 logger.info('=' * 70)
306 test_labels, name, ceilometer_running, compute_node,
307 conf, results, error_plugins):
308 """Execute the testcase.
311 test_labels -- dictionary of plug-in IDs and their display names
312 name -- plug-in ID, key of test_labels dictionary
313 ceilometer_running -- boolean indicating whether Ceilometer is running
314 compute_node -- compute node ID
315 conf -- ConfigServer instance
316 results -- results list
317 error_plugins -- list of tuples with plug-in errors (plugin, error_description, is_critical):
318 plugin -- plug-in ID, key of test_labels dictionary
319 error_decription -- description of the error
320 is_critical -- boolean value indicating whether error is critical
322 ovs_interfaces = conf.get_ovs_interfaces(compute_node)
323 ovs_configured_interfaces = conf.get_plugin_config_values(
324 compute_node, 'ovs_events', 'Interfaces')
325 ovs_existing_configured_int = [
326 interface for interface in ovs_interfaces
327 if interface in ovs_configured_interfaces]
328 plugin_prerequisites = {
329 'mcelog': [(conf.is_installed(compute_node, 'mcelog'), 'mcelog must be installed.')],
331 len(ovs_existing_configured_int) > 0 or len(ovs_interfaces) > 0,
332 'Interfaces must be configured.')]}
333 ceilometer_criteria_lists = {
334 'hugepages': ['hugepages.vmpage_number'],
335 'mcelog': ['mcelog.errors'],
336 'ovs_events': ['ovs_events.gauge']}
337 ceilometer_substr_lists = {
338 'ovs_events': ovs_existing_configured_int if len(ovs_existing_configured_int) > 0 else ovs_interfaces}
341 'hugepages-mm-2048Kb', 'hugepages-node0-2048Kb', 'hugepages-node1-2048Kb',
342 'hugepages-mm-1048576Kb', 'hugepages-node0-1048576Kb', 'hugepages-node1-1048576Kb'],
343 'mcelog': ['mcelog-SOCKET_0_CHANNEL_0_DIMM_any', 'mcelog-SOCKET_0_CHANNEL_any_DIMM_any'],
345 'ovs_events-{}'.format(interface)
346 for interface in (ovs_existing_configured_int if len(ovs_existing_configured_int) > 0 else ovs_interfaces)]}
347 csv_meter_categories = {
348 'hugepages': ['vmpage_number-free', 'vmpage_number-used'],
350 'errors-corrected_memory_errors', 'errors-uncorrected_memory_errors',
351 'errors-corrected_memory_errors_in_24h', 'errors-uncorrected_memory_errors_in_24h'],
352 'ovs_events': ['gauge-link_status']}
354 _print_plugin_label(test_labels[name] if name in test_labels else name, compute_node.get_id())
355 plugin_critical_errors = [
356 error for plugin, error, critical in error_plugins if plugin == name and critical]
357 if plugin_critical_errors:
358 logger.error('Following critical errors occurred:'.format(name))
359 for error in plugin_critical_errors:
360 logger.error(' * ' + error)
361 _process_result(compute_node.get_id(), test_labels[name], False, results)
364 error for plugin, error, critical in error_plugins if plugin == name and not critical]
366 logger.warning('Following non-critical errors occured:')
367 for error in plugin_errors:
368 logger.warning(' * ' + error)
369 failed_prerequisites = []
370 if name in plugin_prerequisites:
371 failed_prerequisites = [
372 prerequisite_name for prerequisite_passed,
373 prerequisite_name in plugin_prerequisites[name] if not prerequisite_passed]
374 if failed_prerequisites:
376 '{} test will not be executed, '.format(name)
377 + 'following prerequisites failed:')
378 for prerequisite in failed_prerequisites:
379 logger.error(' * {}'.format(prerequisite))
381 if ceilometer_running:
382 res = test_ceilometer_node_sends_data(
383 compute_node.get_id(), conf.get_plugin_interval(compute_node, name),
384 logger=logger, client=CeilometerClient(logger),
385 criteria_list=ceilometer_criteria_lists[name],
386 resource_id_substrings=(ceilometer_substr_lists[name]
387 if name in ceilometer_substr_lists else ['']))
389 res = test_csv_handles_plugin_data(
390 compute_node, conf.get_plugin_interval(compute_node, name), name,
391 csv_subdirs[name], csv_meter_categories[name], logger,
392 CSVClient(logger, conf))
393 if res and plugin_errors:
395 'Test works, but will be reported as failure,'
396 + 'because of non-critical errors.')
398 _process_result(compute_node.get_id(), test_labels[name], res, results)
401 def mcelog_install(logger):
402 """Install mcelog on compute nodes.
405 logger - logger instance
407 _print_label('Enabling mcelog on compute nodes')
408 handler = factory.Factory.get_handler('fuel', FUEL_IP, FUEL_USER, installer_pwd='')
409 nodes = handler.get_nodes()
410 openstack_version = handler.get_openstack_version()
411 if openstack_version.find('14.') != 0:
412 logger.info('Mcelog will not be installed,'
413 + ' unsupported Openstack version found ({}).'.format(openstack_version))
416 if node.is_compute():
417 ubuntu_release = node.run_cmd('lsb_release -r')
418 if '16.04' not in ubuntu_release:
419 logger.info('Mcelog will not be enabled'
420 + 'on node-{0}, unsupported Ubuntu release found ({1}).'.format(
421 node.get_dict()['id'], ubuntu_release))
423 logger.info('Checking if mcelog is enabled on node-{}...'.format(
424 node.get_dict()['id']))
425 res = node.run_cmd('ls /root/')
426 if 'mce-inject_df' and 'corrected' in res:
427 logger.info('Mcelog seems to be already installed on node-{}.'.format(
428 node.get_dict()['id']))
429 res = node.run_cmd('modprobe mce-inject')
430 res = node.run_cmd('/root/mce-inject_df < /root/corrected')
432 logger.info('Mcelog will be enabled on node-{}...'.format(
433 node.get_dict()['id']))
434 res = node.put_file('/home/opnfv/repos/barometer/baro_utils/mce-inject_df',
435 '/root/mce-inject_df')
436 res = node.run_cmd('chmod a+x /root/mce-inject_df')
437 res = node.run_cmd('echo "CPU 0 BANK 0" > /root/corrected')
438 res = node.run_cmd('echo "STATUS 0xcc00008000010090" >> /root/corrected')
439 res = node.run_cmd('echo "ADDR 0x0010FFFFFFF" >> /root/corrected')
440 res = node.run_cmd('modprobe mce-inject')
441 res = node.run_cmd('/root/mce-inject_df < /root/corrected')
442 logger.info('Mcelog is installed on all compute nodes')
445 def mcelog_delete(logger):
446 """Uninstall mcelog from compute nodes.
449 logger - logger instance
451 handler = factory.Factory.get_handler('fuel', FUEL_IP, FUEL_USER, installer_pwd='')
452 nodes = handler.get_nodes()
454 if node.is_compute():
455 output = node.run_cmd('ls /root/')
456 if 'mce-inject_df' in output:
457 res = node.run_cmd('rm /root/mce-inject_df')
458 if 'corrected' in output:
459 res = node.run_cmd('rm /root/corrected')
460 res = node.run_cmd('systemctl restart mcelog')
461 logger.info('Mcelog is deleted from all compute nodes')
465 if not os.path.isdir(ID_RSA_DST_DIR):
466 os.makedirs(ID_RSA_DST_DIR)
467 if not os.path.isfile(ID_RSA_DST):
468 logger.info("RSA key file {} doesn't exist, it will be downloaded from installer node.".format(ID_RSA_DST))
469 handler = factory.Factory.get_handler('fuel', FUEL_IP, FUEL_USER, installer_pwd=FUEL_PW)
470 fuel = handler.get_installer_node()
471 fuel.get_file(ID_RSA_SRC, ID_RSA_DST)
473 logger.info("RSA key file {} exists.".format(ID_RSA_DST))
476 def main(bt_logger=None):
477 """Check each compute node sends ceilometer metrics.
480 bt_logger -- logger instance
482 logging.getLogger("paramiko").setLevel(logging.WARNING)
483 logging.getLogger("stevedore").setLevel(logging.WARNING)
484 logging.getLogger("opnfv.deployment.manager").setLevel(logging.WARNING)
485 if bt_logger is None:
491 conf = ConfigServer(FUEL_IP, FUEL_USER, logger)
492 controllers = conf.get_controllers()
493 if len(controllers) == 0:
494 logger.error('No controller nodes found!')
496 computes = conf.get_computes()
497 if len(computes) == 0:
498 logger.error('No compute nodes found!')
501 _print_label('Display of Control and Compute nodes available in the set up')
502 logger.info('controllers: {}'.format([('{0}: {1} ({2})'.format(
503 node.get_id(), node.get_name(), node.get_ip())) for node in controllers]))
504 logger.info('computes: {}'.format([('{0}: {1} ({2})'.format(
505 node.get_id(), node.get_name(), node.get_ip())) for node in computes]))
507 mcelog_install(logger) # installation of mcelog
509 ceilometer_running_on_con = False
510 _print_label('Test Ceilometer on control nodes')
511 for controller in controllers:
512 ceil_client = CeilometerClient(logger)
513 ceil_client.auth_token()
514 ceilometer_running_on_con = (
515 ceilometer_running_on_con or conf.is_ceilometer_running(controller))
516 if ceilometer_running_on_con:
517 logger.info("Ceilometer is running on control node.")
519 logger.error("Ceilometer is not running on control node.")
520 logger.info("CSV will be enabled on compute nodes.")
524 'hugepages': 'Hugepages',
526 'ovs_events': 'OVS events'}
528 for compute_node in computes:
529 node_id = compute_node.get_id()
530 out_plugins[node_id] = 'CSV'
531 compute_ids.append(node_id)
532 # plugins_to_enable = plugin_labels.keys()
533 plugins_to_enable = []
534 _print_label('NODE {}: Test Ceilometer Plug-in'.format(node_id))
535 logger.info('Checking if ceilometer plug-in is included.')
536 if not conf.check_ceil_plugin_included(compute_node):
537 logger.error('Ceilometer plug-in is not included.')
538 logger.info('Testcases on node {} will not be executed'.format(node_id))
540 collectd_restarted, collectd_warnings = conf.restart_collectd(compute_node)
542 logger.info('Sleeping for {} seconds after collectd restart...'.format(sleep_time))
543 time.sleep(sleep_time)
544 if not collectd_restarted:
545 for warning in collectd_warnings:
546 logger.warning(warning)
547 logger.error('Restart of collectd on node {} failed'.format(node_id))
548 logger.info('Testcases on node {} will not be executed'.format(node_id))
550 for warning in collectd_warnings:
551 logger.warning(warning)
552 ceilometer_running = (
553 ceilometer_running_on_con and test_ceilometer_node_sends_data(
554 node_id, 10, logger=logger, client=CeilometerClient(logger)))
555 if ceilometer_running:
556 out_plugins[node_id] = 'Ceilometer'
557 logger.info("Ceilometer is running.")
559 plugins_to_enable.append('csv')
560 out_plugins[node_id] = 'CSV'
561 logger.error("Ceilometer is not running.")
562 logger.info("CSV will be enabled for verification of test plugins.")
563 if plugins_to_enable:
565 'NODE {}: Enabling Test Plug-in '.format(node_id)
566 + 'and Test case execution')
568 if plugins_to_enable and not conf.enable_plugins(
569 compute_node, plugins_to_enable, error_plugins, create_backup=False):
570 logger.error('Failed to test plugins on node {}.'.format(node_id))
571 logger.info('Testcases on node {} will not be executed'.format(node_id))
573 if plugins_to_enable:
574 collectd_restarted, collectd_warnings = conf.restart_collectd(compute_node)
577 'Sleeping for {} seconds after collectd restart...'.format(sleep_time))
578 time.sleep(sleep_time)
579 if plugins_to_enable and not collectd_restarted:
580 for warning in collectd_warnings:
581 logger.warning(warning)
582 logger.error('Restart of collectd on node {} failed'.format(node_id))
583 logger.info('Testcases on node {} will not be executed'.format(node_id))
585 if collectd_warnings:
586 for warning in collectd_warnings:
587 logger.warning(warning)
589 for plugin_name in sorted(plugin_labels.keys()):
591 plugin_labels, plugin_name, ceilometer_running,
592 compute_node, conf, results, error_plugins)
594 _print_label('NODE {}: Restoring config file'.format(node_id))
595 conf.restore_config(compute_node)
597 mcelog_delete(logger) # uninstalling mcelog from compute nodes
599 print_overall_summary(compute_ids, plugin_labels, results, out_plugins)
601 if ((len([res for res in results if not res[2]]) > 0)
602 or (len(results) < len(computes) * len(plugin_labels))):
603 logger.error('Some tests have failed or have not been executed')
608 if __name__ == '__main__':