--- /dev/null
+#! /usr/bin/env python
+import logging
+import argparse
+import shared_utils
+import json
+import urlparse
+
+logger = logging.getLogger('create_kibana_dashboards')
+logger.setLevel(logging.DEBUG)
+file_handler = logging.FileHandler('/var/log/{}.log'.format(__name__))
+file_handler.setFormatter(logging.Formatter('%(asctime)s %(levelname)s: %(message)s'))
+logger.addHandler(file_handler)
+
+_installers = {'fuel', 'apex', 'compass', 'joid'}
+
+# see class VisualizationState for details on format
+_testcases = [
+ ('functest', 'Tempest',
+ [
+ {
+ "metrics": [
+ {
+ "type": "avg",
+ "params": {
+ "field": "details.duration"
+ }
+ }
+ ],
+ "type": "line",
+ "metadata": {
+ "label": "Tempest duration",
+ "test_family": "VIM"
+ }
+ },
+
+ {
+ "metrics": [
+ {
+ "type": "sum",
+ "params": {
+ "field": "details.tests"
+ }
+ },
+ {
+ "type": "sum",
+ "params": {
+ "field": "details.failures"
+ }
+ }
+ ],
+ "type": "histogram",
+ "metadata": {
+ "label": "Tempest nr of tests/failures",
+ "test_family": "VIM"
+ }
+ },
+
+ {
+ "metrics": [
+ {
+ "type": "avg",
+ "params": {
+ "field": "details.success_percentage"
+ }
+ }
+ ],
+ "type": "line",
+ "metadata": {
+ "label": "Tempest success percentage",
+ "test_family": "VIM"
+ }
+ }
+ ]
+ ),
+
+ ('functest', 'Rally',
+ [
+ {
+ "metrics": [
+ {
+ "type": "avg",
+ "params": {
+ "field": "details.duration"
+ }
+ }
+ ],
+ "type": "line",
+ "metadata": {
+ "label": "Rally duration",
+ "test_family": "VIM"
+ }
+ },
+
+ {
+ "metrics": [
+ {
+ "type": "avg",
+ "params": {
+ "field": "details.tests"
+ }
+ }
+ ],
+ "type": "histogram",
+ "metadata": {
+ "label": "Rally nr of tests",
+ "test_family": "VIM"
+ }
+ },
+
+ {
+ "metrics": [
+ {
+ "type": "avg",
+ "params": {
+ "field": "details.success_percentage"
+ }
+ }
+ ],
+ "type": "line",
+ "metadata": {
+ "label": "Rally success percentage",
+ "test_family": "VIM"
+ }
+ }
+ ]
+ ),
+
+ ('functest', 'vPing',
+ [
+ {
+ "metrics": [
+ {
+ "type": "avg",
+ "params": {
+ "field": "details.duration"
+ }
+ }
+ ],
+ "type": "line",
+ "metadata": {
+ "label": "vPing duration",
+ "test_family": "VIM"
+ }
+ }
+ ]
+ ),
+
+ ('functest', 'vPing_userdata',
+ [
+ {
+ "metrics": [
+ {
+ "type": "avg",
+ "params": {
+ "field": "details.duration"
+ }
+ }
+ ],
+ "type": "line",
+ "metadata": {
+ "label": "vPing_userdata duration",
+ "test_family": "VIM"
+ }
+ }
+ ]
+ ),
+
+ ('functest', 'ODL',
+ [
+ {
+ "metrics": [
+ {
+ "type": "sum",
+ "params": {
+ "field": "details.tests"
+ }
+ },
+ {
+ "type": "sum",
+ "params": {
+ "field": "details.failures"
+ }
+ }
+ ],
+ "type": "histogram",
+ "metadata": {
+ "label": "ODL nr of tests/failures",
+ "test_family": "Controller"
+ }
+ },
+
+ {
+ "metrics": [
+ {
+ "type": "avg",
+ "params": {
+ "field": "details.success_percentage"
+ }
+ }
+ ],
+ "type": "line",
+ "metadata": {
+ "label": "ODL success percentage",
+ "test_family": "Controller"
+ }
+ }
+ ]
+ ),
+
+ ('functest', 'ONOS',
+ [
+ {
+ "metrics": [
+ {
+ "type": "avg",
+ "params": {
+ "field": "details.FUNCvirNet.duration"
+ }
+ }
+ ],
+ "type": "line",
+ "metadata": {
+ "label": "ONOS FUNCvirNet duration",
+ "test_family": "Controller"
+ }
+ },
+
+ {
+ "metrics": [
+ {
+ "type": "sum",
+ "params": {
+ "field": "details.FUNCvirNet.tests"
+ }
+ },
+ {
+ "type": "sum",
+ "params": {
+ "field": "details.FUNCvirNet.failures"
+ }
+ }
+ ],
+ "type": "histogram",
+ "metadata": {
+ "label": "ONOS FUNCvirNet nr of tests/failures",
+ "test_family": "Controller"
+ }
+ },
+
+ {
+ "metrics": [
+ {
+ "type": "avg",
+ "params": {
+ "field": "details.FUNCvirNetL3.duration"
+ }
+ }
+ ],
+ "type": "line",
+ "metadata": {
+ "label": "ONOS FUNCvirNetL3 duration",
+ "test_family": "Controller"
+ }
+ },
+
+ {
+ "metrics": [
+ {
+ "type": "sum",
+ "params": {
+ "field": "details.FUNCvirNetL3.tests"
+ }
+ },
+ {
+ "type": "sum",
+ "params": {
+ "field": "details.FUNCvirNetL3.failures"
+ }
+ }
+ ],
+ "type": "histogram",
+ "metadata": {
+ "label": "ONOS FUNCvirNetL3 nr of tests/failures",
+ "test_family": "Controller"
+ }
+ }
+ ]
+ ),
+
+ ('functest', 'vIMS',
+ [
+ {
+ "metrics": [
+ {
+ "type": "sum",
+ "params": {
+ "field": "details.sig_test.tests"
+ }
+ },
+ {
+ "type": "sum",
+ "params": {
+ "field": "details.sig_test.failures"
+ }
+ },
+ {
+ "type": "sum",
+ "params": {
+ "field": "details.sig_test.passed"
+ }
+ },
+ {
+ "type": "sum",
+ "params": {
+ "field": "details.sig_test.skipped"
+ }
+ }
+ ],
+ "type": "histogram",
+ "metadata": {
+ "label": "vIMS nr of tests/failures/passed/skipped",
+ "test_family": "Features"
+ }
+ },
+
+ {
+ "metrics": [
+ {
+ "type": "avg",
+ "params": {
+ "field": "details.vIMS.duration"
+ }
+ },
+ {
+ "type": "avg",
+ "params": {
+ "field": "details.orchestrator.duration"
+ }
+ },
+ {
+ "type": "avg",
+ "params": {
+ "field": "details.sig_test.duration"
+ }
+ }
+ ],
+ "type": "histogram",
+ "metadata": {
+ "label": "vIMS/ochestrator/test duration",
+ "test_family": "Features"
+ }
+ }
+ ]
+ ),
+
+ ('promise', 'promise',
+ [
+ {
+ "metrics": [
+ {
+ "type": "avg",
+ "params": {
+ "field": "details.duration"
+ }
+ }
+ ],
+ "type": "line",
+ "metadata": {
+ "label": "promise duration",
+ "test_family": "Features"
+ }
+ },
+
+ {
+ "metrics": [
+ {
+ "type": "sum",
+ "params": {
+ "field": "details.tests"
+ }
+ },
+ {
+ "type": "sum",
+ "params": {
+ "field": "details.failures"
+ }
+ }
+ ],
+ "type": "histogram",
+ "metadata": {
+ "label": "promise nr of tests/failures",
+ "test_family": "Features"
+ }
+ }
+ ]
+ ),
+
+ ('doctor', 'doctor-notification',
+ [
+ {
+ "metrics": [
+ {
+ "type": "avg",
+ "params": {
+ "field": "details.duration"
+ }
+ }
+ ],
+ "type": "line",
+ "metadata": {
+ "label": "doctor-notification duration",
+ "test_family": "Features"
+ }
+ }
+ ]
+ )
+]
+
+
+class KibanaDashboard(dict):
+ def __init__(self, project_name, case_name, installer, pod, versions, visualization_detail):
+ super(KibanaDashboard, self).__init__()
+ self.project_name = project_name
+ self.case_name = case_name
+ self.installer = installer
+ self.pod = pod
+ self.versions = versions
+ self.visualization_detail = visualization_detail
+ self._visualization_title = None
+ self._kibana_visualizations = []
+ self._kibana_dashboard = None
+ self._create_visualizations()
+ self._create()
+
+ def _create_visualizations(self):
+ for version in self.versions:
+ self._kibana_visualizations.append(KibanaVisualization(self.project_name,
+ self.case_name,
+ self.installer,
+ self.pod,
+ version,
+ self.visualization_detail))
+
+ self._visualization_title = self._kibana_visualizations[0].vis_state_title
+
+ def _publish_visualizations(self):
+ for visualization in self._kibana_visualizations:
+ url = urlparse.urljoin(base_elastic_url, '/.kibana/visualization/{}'.format(visualization.id))
+ logger.debug("publishing visualization '{}'".format(url))
+ shared_utils.publish_json(visualization, es_user, es_passwd, url)
+
+ def _construct_panels(self):
+ size_x = 6
+ size_y = 3
+ max_columns = 7
+ column = 1
+ row = 1
+ panel_index = 1
+ panels_json = []
+ for visualization in self._kibana_visualizations:
+ panels_json.append({
+ "id": visualization.id,
+ "type": 'visualization',
+ "panelIndex": panel_index,
+ "size_x": size_x,
+ "size_y": size_y,
+ "col": column,
+ "row": row
+ })
+ panel_index += 1
+ column += size_x
+ if column > max_columns:
+ column = 1
+ row += size_y
+ return json.dumps(panels_json, separators=(',', ':'))
+
+ def _create(self):
+ self['title'] = '{} {} {} {} {}'.format(self.project_name,
+ self.case_name,
+ self.installer,
+ self._visualization_title,
+ self.pod)
+ self.id = self['title'].replace(' ', '-').replace('/', '-')
+
+ self['hits'] = 0
+ self['description'] = "Kibana dashboard for project_name '{}', case_name '{}', installer '{}', data '{}' and" \
+ " pod '{}'".format(self.project_name,
+ self.case_name,
+ self.installer,
+ self._visualization_title,
+ self.pod)
+ self['panelsJSON'] = self._construct_panels()
+ self['optionsJSON'] = json.dumps({
+ "darkTheme": False
+ },
+ separators=(',', ':'))
+ self['uiStateJSON'] = "{}"
+ self['version'] = 1
+ self['timeRestore'] = False
+ self['kibanaSavedObjectMeta'] = {
+ 'searchSourceJSON': json.dumps({
+ "filter": [
+ {
+ "query": {
+ "query_string": {
+ "query": "*",
+ "analyze_wildcard": True
+ }
+ }
+ }
+ ]
+ },
+ separators=(',', ':'))
+ }
+ self['metadata'] = self.visualization_detail['metadata']
+
+ def _publish(self):
+ url = urlparse.urljoin(base_elastic_url, '/.kibana/dashboard/{}'.format(self.id))
+ logger.debug("publishing dashboard '{}'".format(url))
+ shared_utils.publish_json(self, es_user, es_passwd, url)
+
+ def publish(self):
+ self._publish_visualizations()
+ self._publish()
+
+
+class KibanaSearchSourceJSON(dict):
+ """
+ "filter": [
+ {"match": {"installer": {"query": installer, "type": "phrase"}}},
+ {"match": {"project_name": {"query": project_name, "type": "phrase"}}},
+ {"match": {"case_name": {"query": case_name, "type": "phrase"}}}
+ ]
+ """
+
+ def __init__(self, project_name, case_name, installer, pod, version):
+ super(KibanaSearchSourceJSON, self).__init__()
+ self["filter"] = [
+ {"match": {"project_name": {"query": project_name, "type": "phrase"}}},
+ {"match": {"case_name": {"query": case_name, "type": "phrase"}}},
+ {"match": {"installer": {"query": installer, "type": "phrase"}}},
+ {"match": {"version": {"query": version, "type": "phrase"}}}
+ ]
+ if pod != 'all':
+ self["filter"].append({"match": {"pod_name": {"query": pod, "type": "phrase"}}})
+
+
+class VisualizationState(dict):
+ def __init__(self, input_dict):
+ """
+ dict structure:
+ {
+ "metrics":
+ [
+ {
+ "type": type, # default sum
+ "params": {
+ "field": field # mandatory, no default
+ },
+ {metric2}
+ ],
+ "segments":
+ [
+ {
+ "type": type, # default date_histogram
+ "params": {
+ "field": field # default creation_date
+ },
+ {segment2}
+ ],
+ "type": type, # default area
+ "mode": mode, # default grouped for type 'histogram', stacked for other types
+ "metadata": {
+ "label": "Tempest duration",# mandatory, no default
+ "test_family": "VIM" # mandatory, no default
+ }
+ }
+
+ default modes:
+ type histogram: grouped
+ type area: stacked
+
+ :param input_dict:
+ :return:
+ """
+ super(VisualizationState, self).__init__()
+ metrics = input_dict['metrics']
+ segments = [] if 'segments' not in input_dict else input_dict['segments']
+
+ graph_type = 'area' if 'type' not in input_dict else input_dict['type']
+ self['type'] = graph_type
+
+ if 'mode' not in input_dict:
+ if graph_type == 'histogram':
+ mode = 'grouped'
+ else:
+ # default
+ mode = 'stacked'
+ else:
+ mode = input_dict['mode']
+ self['params'] = {
+ "shareYAxis": True,
+ "addTooltip": True,
+ "addLegend": True,
+ "smoothLines": False,
+ "scale": "linear",
+ "interpolate": "linear",
+ "mode": mode,
+ "times": [],
+ "addTimeMarker": False,
+ "defaultYExtents": False,
+ "setYExtents": False,
+ "yAxis": {}
+ }
+
+ self['aggs'] = []
+
+ i = 1
+ for metric in metrics:
+ self['aggs'].append({
+ "id": str(i),
+ "type": 'sum' if 'type' not in metric else metric['type'],
+ "schema": "metric",
+ "params": {
+ "field": metric['params']['field']
+ }
+ })
+ i += 1
+
+ if len(segments) > 0:
+ for segment in segments:
+ self['aggs'].append({
+ "id": str(i),
+ "type": 'date_histogram' if 'type' not in segment else segment['type'],
+ "schema": "metric",
+ "params": {
+ "field": "creation_date" if ('params' not in segment or 'field' not in segment['params'])
+ else segment['params']['field'],
+ "interval": "auto",
+ "customInterval": "2h",
+ "min_doc_count": 1,
+ "extended_bounds": {}
+ }
+ })
+ i += 1
+ else:
+ self['aggs'].append({
+ "id": str(i),
+ "type": 'date_histogram',
+ "schema": "segment",
+ "params": {
+ "field": "creation_date",
+ "interval": "auto",
+ "customInterval": "2h",
+ "min_doc_count": 1,
+ "extended_bounds": {}
+ }
+ })
+
+ self['listeners'] = {}
+ self['title'] = ' '.join(['{} {}'.format(x['type'], x['params']['field']) for x in self['aggs']
+ if x['schema'] == 'metric'])
+
+
+class KibanaVisualization(dict):
+ def __init__(self, project_name, case_name, installer, pod, version, detail):
+ """
+ We need two things
+ 1. filter created from
+ project_name
+ case_name
+ installer
+ pod
+ version
+ 2. visualization state
+ field for y axis (metric) with type (avg, sum, etc.)
+ field for x axis (segment) with type (date_histogram)
+
+ :return:
+ """
+ super(KibanaVisualization, self).__init__()
+ vis_state = VisualizationState(detail)
+ self.vis_state_title = vis_state['title']
+ self['title'] = '{} {} {} {} {} {}'.format(project_name,
+ case_name,
+ self.vis_state_title,
+ installer,
+ pod,
+ version)
+ self.id = self['title'].replace(' ', '-').replace('/', '-')
+ self['visState'] = json.dumps(vis_state, separators=(',', ':'))
+ self['uiStateJSON'] = "{}"
+ self['description'] = "Kibana visualization for project_name '{}', case_name '{}', data '{}', installer '{}'," \
+ " pod '{}' and version '{}'".format(project_name,
+ case_name,
+ self.vis_state_title,
+ installer,
+ pod,
+ version)
+ self['version'] = 1
+ self['kibanaSavedObjectMeta'] = {"searchSourceJSON": json.dumps(KibanaSearchSourceJSON(project_name,
+ case_name,
+ installer,
+ pod,
+ version),
+ separators=(',', ':'))}
+
+
+def _get_pods_and_versions(project_name, case_name, installer):
+ query_json = json.JSONEncoder().encode({
+ "query": {
+ "bool": {
+ "must": [
+ {"match_all": {}}
+ ],
+ "filter": [
+ {"match": {"installer": {"query": installer, "type": "phrase"}}},
+ {"match": {"project_name": {"query": project_name, "type": "phrase"}}},
+ {"match": {"case_name": {"query": case_name, "type": "phrase"}}}
+ ]
+ }
+ }
+ })
+
+ elastic_data = shared_utils.get_elastic_data(urlparse.urljoin(base_elastic_url, '/test_results/mongo2elastic'),
+ es_user, es_passwd, query_json)
+
+ pods_and_versions = {}
+
+ for data in elastic_data:
+ pod = data['pod_name']
+ if pod in pods_and_versions:
+ pods_and_versions[pod].add(data['version'])
+ else:
+ pods_and_versions[pod] = {data['version']}
+
+ if 'all' in pods_and_versions:
+ pods_and_versions['all'].add(data['version'])
+ else:
+ pods_and_versions['all'] = {data['version']}
+
+ return pods_and_versions
+
+
+def construct_dashboards():
+ """
+ iterate over testcase and installer
+ 1. get available pods for each testcase/installer pair
+ 2. get available version for each testcase/installer/pod tuple
+ 3. construct KibanaInput and append
+
+ :return: list of KibanaDashboards
+ """
+ kibana_dashboards = []
+ for project_name, case_name, visualization_details in _testcases:
+ for installer in _installers:
+ pods_and_versions = _get_pods_and_versions(project_name, case_name, installer)
+ for visualization_detail in visualization_details:
+ for pod, versions in pods_and_versions.iteritems():
+ kibana_dashboards.append(KibanaDashboard(project_name, case_name, installer, pod, versions,
+ visualization_detail))
+ return kibana_dashboards
+
+
+def generate_js_inputs(js_file_path, kibana_url, dashboards):
+ js_dict = {}
+ for dashboard in dashboards:
+ dashboard_meta = dashboard['metadata']
+ test_family = dashboard_meta['test_family']
+ test_label = dashboard_meta['label']
+
+ if test_family not in js_dict:
+ js_dict[test_family] = {}
+
+ js_test_family = js_dict[test_family]
+
+ if test_label not in js_test_family:
+ js_test_family[test_label] = {}
+
+ js_test_label = js_test_family[test_label]
+
+ if dashboard.installer not in js_test_label:
+ js_test_label[dashboard.installer] = {}
+
+ js_installer = js_test_label[dashboard.installer]
+ js_installer[dashboard.pod] = kibana_url + '#/dashboard/' + dashboard.id
+
+ with open(js_file_path, 'w+') as js_file_fdesc:
+ js_file_fdesc.write('var kibana_dashboard_links = ')
+ js_file_fdesc.write(str(js_dict).replace("u'", "'"))
+
+
+if __name__ == '__main__':
+ parser = argparse.ArgumentParser(description='Create Kibana dashboards from data in elasticsearch')
+ parser.add_argument('-e', '--elasticsearch-url', default='http://localhost:9200',
+ help='the url of elasticsearch, defaults to http://localhost:9200')
+ parser.add_argument('-js', '--generate_js_inputs', action='store_true',
+ help='Use this argument to generate javascript inputs for kibana landing page')
+ parser.add_argument('--js_path', default='/usr/share/nginx/html/kibana_dashboards/conf.js',
+ help='Path of javascript file with inputs for kibana landing page')
+ parser.add_argument('-k', '--kibana_url', default='https://testresults.opnfv.org/kibana/app/kibana',
+ help='The url of kibana for javascript inputs')
+
+ parser.add_argument('-u', '--elasticsearch-username',
+ help='the username for elasticsearch')
+
+ parser.add_argument('-p', '--elasticsearch-password',
+ help='the password for elasticsearch')
+
+ args = parser.parse_args()
+ base_elastic_url = args.elasticsearch_url
+ generate_inputs = args.generate_js_inputs
+ input_file_path = args.js_path
+ kibana_url = args.kibana_url
+ es_user = args.elasticsearch_username
+ es_passwd = args.elasticsearch_password
+
+ dashboards = construct_dashboards()
+
+ for kibana_dashboard in dashboards:
+ kibana_dashboard.publish()
+
+ if generate_inputs:
+ generate_js_inputs(input_file_path, kibana_url, dashboards)
--- /dev/null
+#! /usr/bin/env python
+import logging
+import argparse
+import shared_utils
+import json
+import urlparse
+
+logger = logging.getLogger('clear_kibana')
+logger.setLevel(logging.DEBUG)
+file_handler = logging.FileHandler('/var/log/{}.log'.format(__name__))
+file_handler.setFormatter(logging.Formatter('%(asctime)s %(levelname)s: %(message)s'))
+logger.addHandler(file_handler)
+
+
+def delete_all(url, es_user, es_passwd):
+ ids = shared_utils.get_elastic_data(url, es_user, es_passwd, body=None, field='_id')
+ for id in ids:
+ del_url = '/'.join([url, id])
+ shared_utils.delete_request(del_url, es_user, es_passwd)
+
+
+if __name__ == '__main__':
+ parser = argparse.ArgumentParser(description='Delete saved kibana searches, visualizations and dashboards')
+ parser.add_argument('-e', '--elasticsearch-url', default='http://localhost:9200',
+ help='the url of elasticsearch, defaults to http://localhost:9200')
+
+ parser.add_argument('-u', '--elasticsearch-username',
+ help='the username for elasticsearch')
+
+ parser.add_argument('-p', '--elasticsearch-password',
+ help='the password for elasticsearch')
+
+ args = parser.parse_args()
+ base_elastic_url = args.elasticsearch_url
+ es_user = args.elasticsearch_username
+ es_passwd = args.elasticsearch_password
+
+ urls = (urlparse.urljoin(base_elastic_url, '/.kibana/visualization'),
+ urlparse.urljoin(base_elastic_url, '/.kibana/dashboard'),
+ urlparse.urljoin(base_elastic_url, '/.kibana/search'))
+
+ for url in urls:
+ delete_all(url, es_user, es_passwd)
--- /dev/null
+#! /usr/bin/env python
+import logging
+import argparse
+import shared_utils
+import json
+import urlparse
+import uuid
+import os
+import subprocess
+import datetime
+
+logger = logging.getLogger('mongo_to_elasticsearch')
+logger.setLevel(logging.DEBUG)
+file_handler = logging.FileHandler('/var/log/{}.log'.format(__name__))
+file_handler.setFormatter(logging.Formatter('%(asctime)s %(levelname)s: %(message)s'))
+logger.addHandler(file_handler)
+
+
+def _get_dicts_from_list(dict_list, keys):
+ dicts = []
+ for dictionary in dict_list:
+ # iterate over dictionaries in input list
+ if keys == set(dictionary.keys()):
+ # check the dictionary structure
+ dicts.append(dictionary)
+ return dicts
+
+
+def _get_results_from_list_of_dicts(list_of_dict_statuses, dict_indexes, expected_results=None):
+ test_results = {}
+ for test_status in list_of_dict_statuses:
+ status = test_status
+ for index in dict_indexes:
+ status = status[index]
+ if status in test_results:
+ test_results[status] += 1
+ else:
+ test_results[status] = 1
+
+ if expected_results is not None:
+ for expected_result in expected_results:
+ if expected_result not in test_results:
+ test_results[expected_result] = 0
+
+ return test_results
+
+
+def _convert_duration(duration):
+ if (isinstance(duration, str) or isinstance(duration, unicode)) and ':' in duration:
+ hours, minutes, seconds = duration.split(":")
+ int_duration = 3600 * int(hours) + 60 * int(minutes) + float(seconds)
+ else:
+ int_duration = duration
+ return int_duration
+
+
+def modify_functest_tempest(testcase):
+ if modify_default_entry(testcase):
+ testcase_details = testcase['details']
+ testcase_tests = float(testcase_details['tests'])
+ testcase_failures = float(testcase_details['failures'])
+ if testcase_tests != 0:
+ testcase_details['success_percentage'] = 100 * (testcase_tests - testcase_failures) / testcase_tests
+ else:
+ testcase_details['success_percentage'] = 0
+ return True
+ else:
+ return False
+
+
+def modify_functest_vims(testcase):
+ """
+ Structure:
+ details.sig_test.result.[{result}]
+ details.sig_test.duration
+ details.vIMS.duration
+ details.orchestrator.duration
+
+ Find data for these fields
+ -> details.sig_test.duration
+ -> details.sig_test.tests
+ -> details.sig_test.failures
+ -> details.sig_test.passed
+ -> details.sig_test.skipped
+ -> details.vIMS.duration
+ -> details.orchestrator.duration
+ """
+ testcase_details = testcase['details']
+ sig_test_results = _get_dicts_from_list(testcase_details['sig_test']['result'],
+ {'duration', 'result', 'name', 'error'})
+ if len(sig_test_results) < 1:
+ logger.info("No 'result' from 'sig_test' found in vIMS details, skipping")
+ return False
+ else:
+ test_results = _get_results_from_list_of_dicts(sig_test_results, ('result',), ('Passed', 'Skipped', 'Failed'))
+ passed = test_results['Passed']
+ skipped = test_results['Skipped']
+ failures = test_results['Failed']
+ all_tests = passed + skipped + failures
+ testcase['details'] = {
+ 'sig_test': {
+ 'duration': testcase_details['sig_test']['duration'],
+ 'tests': all_tests,
+ 'failures': failures,
+ 'passed': passed,
+ 'skipped': skipped
+ },
+ 'vIMS': {
+ 'duration': testcase_details['vIMS']['duration']
+ },
+ 'orchestrator': {
+ 'duration': testcase_details['orchestrator']['duration']
+ }
+ }
+ return True
+
+
+def modify_functest_onos(testcase):
+ """
+ Structure:
+ details.FUNCvirNet.duration
+ details.FUNCvirNet.status.[{Case result}]
+ details.FUNCvirNetL3.duration
+ details.FUNCvirNetL3.status.[{Case result}]
+
+ Find data for these fields
+ -> details.FUNCvirNet.duration
+ -> details.FUNCvirNet.tests
+ -> details.FUNCvirNet.failures
+ -> details.FUNCvirNetL3.duration
+ -> details.FUNCvirNetL3.tests
+ -> details.FUNCvirNetL3.failures
+ """
+ testcase_details = testcase['details']
+
+ funcvirnet_details = testcase_details['FUNCvirNet']['status']
+ funcvirnet_statuses = _get_dicts_from_list(funcvirnet_details, {'Case result', 'Case name:'})
+
+ funcvirnetl3_details = testcase_details['FUNCvirNetL3']['status']
+ funcvirnetl3_statuses = _get_dicts_from_list(funcvirnetl3_details, {'Case result', 'Case name:'})
+
+ if len(funcvirnet_statuses) < 0:
+ logger.info("No results found in 'FUNCvirNet' part of ONOS results")
+ return False
+ elif len(funcvirnetl3_statuses) < 0:
+ logger.info("No results found in 'FUNCvirNetL3' part of ONOS results")
+ return False
+ else:
+ funcvirnet_results = _get_results_from_list_of_dicts(funcvirnet_statuses,
+ ('Case result',), ('PASS', 'FAIL'))
+ funcvirnetl3_results = _get_results_from_list_of_dicts(funcvirnetl3_statuses,
+ ('Case result',), ('PASS', 'FAIL'))
+
+ funcvirnet_passed = funcvirnet_results['PASS']
+ funcvirnet_failed = funcvirnet_results['FAIL']
+ funcvirnet_all = funcvirnet_passed + funcvirnet_failed
+
+ funcvirnetl3_passed = funcvirnetl3_results['PASS']
+ funcvirnetl3_failed = funcvirnetl3_results['FAIL']
+ funcvirnetl3_all = funcvirnetl3_passed + funcvirnetl3_failed
+
+ testcase_details['FUNCvirNet'] = {
+ 'duration': _convert_duration(testcase_details['FUNCvirNet']['duration']),
+ 'tests': funcvirnet_all,
+ 'failures': funcvirnet_failed
+ }
+
+ testcase_details['FUNCvirNetL3'] = {
+ 'duration': _convert_duration(testcase_details['FUNCvirNetL3']['duration']),
+ 'tests': funcvirnetl3_all,
+ 'failures': funcvirnetl3_failed
+ }
+
+ return True
+
+
+def modify_functest_rally(testcase):
+ """
+ Structure:
+ details.[{summary.duration}]
+ details.[{summary.nb success}]
+ details.[{summary.nb tests}]
+
+ Find data for these fields
+ -> details.duration
+ -> details.tests
+ -> details.success_percentage
+ """
+ summaries = _get_dicts_from_list(testcase['details'], {'summary'})
+
+ if len(summaries) != 1:
+ logger.info("Found zero or more than one 'summaries' in Rally details, skipping")
+ return False
+ else:
+ summary = summaries[0]['summary']
+ testcase['details'] = {
+ 'duration': summary['duration'],
+ 'tests': summary['nb tests'],
+ 'success_percentage': summary['nb success']
+ }
+ return True
+
+
+def modify_functest_odl(testcase):
+ """
+ Structure:
+ details.details.[{test_status.@status}]
+
+ Find data for these fields
+ -> details.tests
+ -> details.failures
+ -> details.success_percentage?
+ """
+ test_statuses = _get_dicts_from_list(testcase['details']['details'], {'test_status', 'test_doc', 'test_name'})
+ if len(test_statuses) < 1:
+ logger.info("No 'test_status' found in ODL details, skipping")
+ return False
+ else:
+ test_results = _get_results_from_list_of_dicts(test_statuses, ('test_status', '@status'), ('PASS', 'FAIL'))
+
+ passed_tests = test_results['PASS']
+ failed_tests = test_results['FAIL']
+ all_tests = passed_tests + failed_tests
+
+ testcase['details'] = {
+ 'tests': all_tests,
+ 'failures': failed_tests,
+ 'success_percentage': 100 * passed_tests / float(all_tests)
+ }
+ return True
+
+
+def modify_default_entry(testcase):
+ """
+ Look for these and leave any of those:
+ details.duration
+ details.tests
+ details.failures
+
+ If none are present, then return False
+ """
+ found = False
+ testcase_details = testcase['details']
+ fields = ['duration', 'tests', 'failures']
+ if isinstance(testcase_details, dict):
+ for key, value in testcase_details.items():
+ if key in fields:
+ found = True
+ if key == 'duration':
+ testcase_details[key] = _convert_duration(value)
+ else:
+ del testcase_details[key]
+
+ return found
+
+
+def _fix_date(date_string):
+ if isinstance(date_string, dict):
+ return date_string['$date']
+ else:
+ return date_string[:-3].replace(' ', 'T') + 'Z'
+
+
+def verify_mongo_entry(testcase):
+ """
+ Mandatory fields:
+ installer
+ pod_name
+ version
+ case_name
+ date
+ project
+ details
+
+ these fields must be present and must NOT be None
+
+ Optional fields:
+ description
+
+ these fields will be preserved if the are NOT None
+ """
+ mandatory_fields = ['installer',
+ 'pod_name',
+ 'version',
+ 'case_name',
+ 'project_name',
+ 'details']
+ mandatory_fields_to_modify = {'creation_date': _fix_date}
+ if '_id' in testcase:
+ mongo_id = testcase['_id']
+ else:
+ mongo_id = None
+ optional_fields = ['description']
+ for key, value in testcase.items():
+ if key in mandatory_fields:
+ if value is None:
+ # empty mandatory field, invalid input
+ logger.info("Skipping testcase with mongo _id '{}' because the testcase was missing value"
+ " for mandatory field '{}'".format(mongo_id, key))
+ return False
+ else:
+ mandatory_fields.remove(key)
+ elif key in mandatory_fields_to_modify:
+ if value is None:
+ # empty mandatory field, invalid input
+ logger.info("Skipping testcase with mongo _id '{}' because the testcase was missing value"
+ " for mandatory field '{}'".format(mongo_id, key))
+ return False
+ else:
+ testcase[key] = mandatory_fields_to_modify[key](value)
+ del mandatory_fields_to_modify[key]
+ elif key in optional_fields:
+ if value is None:
+ # empty optional field, remove
+ del testcase[key]
+ optional_fields.remove(key)
+ else:
+ # unknown field
+ del testcase[key]
+
+ if len(mandatory_fields) > 0:
+ # some mandatory fields are missing
+ logger.info("Skipping testcase with mongo _id '{}' because the testcase was missing"
+ " mandatory field(s) '{}'".format(mongo_id, mandatory_fields))
+ return False
+ else:
+ return True
+
+
+def modify_mongo_entry(testcase):
+ # 1. verify and identify the testcase
+ # 2. if modification is implemented, then use that
+ # 3. if not, try to use default
+ # 4. if 2 or 3 is successful, return True, otherwise return False
+ if verify_mongo_entry(testcase):
+ project = testcase['project_name']
+ case_name = testcase['case_name']
+ if project == 'functest':
+ if case_name == 'Rally':
+ return modify_functest_rally(testcase)
+ elif case_name == 'ODL':
+ return modify_functest_odl(testcase)
+ elif case_name == 'ONOS':
+ return modify_functest_onos(testcase)
+ elif case_name == 'vIMS':
+ return modify_functest_vims(testcase)
+ elif case_name == 'Tempest':
+ return modify_functest_tempest(testcase)
+ return modify_default_entry(testcase)
+ else:
+ return False
+
+
+def publish_mongo_data(output_destination):
+ tmp_filename = 'mongo-{}.log'.format(uuid.uuid4())
+ try:
+ subprocess.check_call(['mongoexport', '--db', 'test_results_collection', '-c', 'test_results', '--out',
+ tmp_filename])
+ with open(tmp_filename) as fobj:
+ for mongo_json_line in fobj:
+ test_result = json.loads(mongo_json_line)
+ if modify_mongo_entry(test_result):
+ shared_utils.publish_json(test_result, output_destination, es_user, es_passwd)
+ finally:
+ if os.path.exists(tmp_filename):
+ os.remove(tmp_filename)
+
+
+def get_mongo_data(days):
+ past_time = datetime.datetime.today() - datetime.timedelta(days=days)
+ mongo_json_lines = subprocess.check_output(['mongoexport', '--db', 'test_results_collection', '-c', 'test_results',
+ '--query', '{{"creation_date":{{$gt:"{}"}}}}'
+ .format(past_time)]).splitlines()
+
+ mongo_data = []
+ for mongo_json_line in mongo_json_lines:
+ test_result = json.loads(mongo_json_line)
+ if modify_mongo_entry(test_result):
+ # if the modification could be applied, append the modified result
+ mongo_data.append(test_result)
+ return mongo_data
+
+
+def publish_difference(mongo_data, elastic_data, output_destination, es_user, es_passwd):
+ for elastic_entry in elastic_data:
+ if elastic_entry in mongo_data:
+ mongo_data.remove(elastic_entry)
+
+ logger.info('number of parsed test results: {}'.format(len(mongo_data)))
+
+ for parsed_test_result in mongo_data:
+ shared_utils.publish_json(parsed_test_result, es_user, es_passwd, output_destination)
+
+
+if __name__ == '__main__':
+ parser = argparse.ArgumentParser(description='Modify and filter mongo json data for elasticsearch')
+ parser.add_argument('-od', '--output-destination',
+ default='elasticsearch',
+ choices=('elasticsearch', 'stdout'),
+ help='defaults to elasticsearch')
+
+ parser.add_argument('-ml', '--merge-latest', default=0, type=int, metavar='N',
+ help='get entries old at most N days from mongodb and'
+ ' parse those that are not already in elasticsearch.'
+ ' If not present, will get everything from mongodb, which is the default')
+
+ parser.add_argument('-e', '--elasticsearch-url', default='http://localhost:9200',
+ help='the url of elasticsearch, defaults to http://localhost:9200')
+
+ parser.add_argument('-u', '--elasticsearch-username',
+ help='the username for elasticsearch')
+
+ parser.add_argument('-p', '--elasticsearch-password',
+ help='the password for elasticsearch')
+
+ parser.add_argument('-m', '--mongodb-url', default='http://localhost:8082',
+ help='the url of mongodb, defaults to http://localhost:8082')
+
+ args = parser.parse_args()
+ base_elastic_url = urlparse.urljoin(args.elasticsearch_url, '/test_results/mongo2elastic')
+ output_destination = args.output_destination
+ days = args.merge_latest
+ es_user = args.elasticsearch_username
+ es_passwd = args.elasticsearch_password
+
+ if output_destination == 'elasticsearch':
+ output_destination = base_elastic_url
+
+ # parsed_test_results will be printed/sent to elasticsearch
+ if days == 0:
+ # TODO get everything from mongo
+ publish_mongo_data(output_destination)
+ elif days > 0:
+ body = '''{{
+ "query" : {{
+ "range" : {{
+ "creation_date" : {{
+ "gte" : "now-{}d"
+ }}
+ }}
+ }}
+}}'''.format(days)
+ elastic_data = shared_utils.get_elastic_data(base_elastic_url, es_user, es_passwd, body)
+ logger.info('number of hits in elasticsearch for now-{}d: {}'.format(days, len(elastic_data)))
+ mongo_data = get_mongo_data(days)
+ publish_difference(mongo_data, elastic_data, output_destination, es_user, es_passwd)
+ else:
+ raise Exception('Update must be non-negative')
--- /dev/null
+import urllib3
+import json
+http = urllib3.PoolManager()
+
+
+def delete_request(url, username, password, body=None):
+ headers = urllib3.util.make_headers(basic_auth=':'.join([username, password]))
+ http.request('DELETE', url, headers=headers, body=body)
+
+
+def publish_json(json_ojb, username, password, output_destination):
+ json_dump = json.dumps(json_ojb)
+ if output_destination == 'stdout':
+ print json_dump
+ else:
+ headers = urllib3.util.make_headers(basic_auth=':'.join([username, password]))
+ http.request('POST', output_destination, headers=headers, body=json_dump)
+
+
+def _get_nr_of_hits(elastic_json):
+ return elastic_json['hits']['total']
+
+
+def get_elastic_data(elastic_url, username, password, body, field='_source'):
+ # 1. get the number of results
+ headers = urllib3.util.make_headers(basic_auth=':'.join([username, password]))
+ elastic_json = json.loads(http.request('GET', elastic_url + '/_search?size=0', headers=headers, body=body).data)
+ nr_of_hits = _get_nr_of_hits(elastic_json)
+
+ # 2. get all results
+ elastic_json = json.loads(http.request('GET', elastic_url + '/_search?size={}'.format(nr_of_hits), headers=headers, body=body).data)
+
+ elastic_data = []
+ for hit in elastic_json['hits']['hits']:
+ elastic_data.append(hit[field])
+ return elastic_data
Code Review / releng.git / commitdiff