8 logger = logging.getLogger('create_kibana_dashboards')
9 logger.setLevel(logging.DEBUG)
10 file_handler = logging.FileHandler('/var/log/{}.log'.format(__name__))
11 file_handler.setFormatter(logging.Formatter('%(asctime)s %(levelname)s: %(message)s'))
12 logger.addHandler(file_handler)
14 _installers = {'fuel', 'apex', 'compass', 'joid'}
16 # see class VisualizationState for details on format
18 ('functest', 'Tempest',
25 "field": "details.duration"
31 "label": "Tempest duration",
41 "field": "details.tests"
47 "field": "details.failures"
53 "label": "Tempest nr of tests/failures",
63 "field": "details.success_percentage"
69 "label": "Tempest success percentage",
83 "field": "details.duration"
89 "label": "Rally duration",
99 "field": "details.tests"
105 "label": "Rally nr of tests",
115 "field": "details.success_percentage"
121 "label": "Rally success percentage",
128 ('functest', 'vPing',
135 "field": "details.duration"
141 "label": "vPing duration",
148 ('functest', 'vPing_userdata',
155 "field": "details.duration"
161 "label": "vPing_userdata duration",
175 "field": "details.tests"
181 "field": "details.failures"
187 "label": "ODL nr of tests/failures",
188 "test_family": "Controller"
197 "field": "details.success_percentage"
203 "label": "ODL success percentage",
204 "test_family": "Controller"
217 "field": "details.FUNCvirNet.duration"
223 "label": "ONOS FUNCvirNet duration",
224 "test_family": "Controller"
233 "field": "details.FUNCvirNet.tests"
239 "field": "details.FUNCvirNet.failures"
245 "label": "ONOS FUNCvirNet nr of tests/failures",
246 "test_family": "Controller"
255 "field": "details.FUNCvirNetL3.duration"
261 "label": "ONOS FUNCvirNetL3 duration",
262 "test_family": "Controller"
271 "field": "details.FUNCvirNetL3.tests"
277 "field": "details.FUNCvirNetL3.failures"
283 "label": "ONOS FUNCvirNetL3 nr of tests/failures",
284 "test_family": "Controller"
297 "field": "details.sig_test.tests"
303 "field": "details.sig_test.failures"
309 "field": "details.sig_test.passed"
315 "field": "details.sig_test.skipped"
321 "label": "vIMS nr of tests/failures/passed/skipped",
322 "test_family": "Features"
331 "field": "details.vIMS.duration"
337 "field": "details.orchestrator.duration"
343 "field": "details.sig_test.duration"
349 "label": "vIMS/ochestrator/test duration",
350 "test_family": "Features"
356 ('promise', 'promise',
363 "field": "details.duration"
369 "label": "promise duration",
370 "test_family": "Features"
379 "field": "details.tests"
385 "field": "details.failures"
391 "label": "promise nr of tests/failures",
392 "test_family": "Features"
398 ('doctor', 'doctor-notification',
405 "field": "details.duration"
411 "label": "doctor-notification duration",
412 "test_family": "Features"
420 class KibanaDashboard(dict):
421 def __init__(self, project_name, case_name, installer, pod, versions, visualization_detail):
422 super(KibanaDashboard, self).__init__()
423 self.project_name = project_name
424 self.case_name = case_name
425 self.installer = installer
427 self.versions = versions
428 self.visualization_detail = visualization_detail
429 self._visualization_title = None
430 self._kibana_visualizations = []
431 self._kibana_dashboard = None
432 self._create_visualizations()
435 def _create_visualizations(self):
436 for version in self.versions:
437 self._kibana_visualizations.append(KibanaVisualization(self.project_name,
442 self.visualization_detail))
444 self._visualization_title = self._kibana_visualizations[0].vis_state_title
446 def _publish_visualizations(self):
447 for visualization in self._kibana_visualizations:
448 url = urlparse.urljoin(base_elastic_url, '/.kibana/visualization/{}'.format(visualization.id))
449 logger.debug("publishing visualization '{}'".format(url))
450 shared_utils.publish_json(visualization, es_user, es_passwd, url)
452 def _construct_panels(self):
460 for visualization in self._kibana_visualizations:
462 "id": visualization.id,
463 "type": 'visualization',
464 "panelIndex": panel_index,
472 if column > max_columns:
475 return json.dumps(panels_json, separators=(',', ':'))
478 self['title'] = '{} {} {} {} {}'.format(self.project_name,
481 self._visualization_title,
483 self.id = self['title'].replace(' ', '-').replace('/', '-')
486 self['description'] = "Kibana dashboard for project_name '{}', case_name '{}', installer '{}', data '{}' and" \
487 " pod '{}'".format(self.project_name,
490 self._visualization_title,
492 self['panelsJSON'] = self._construct_panels()
493 self['optionsJSON'] = json.dumps({
496 separators=(',', ':'))
497 self['uiStateJSON'] = "{}"
499 self['timeRestore'] = False
500 self['kibanaSavedObjectMeta'] = {
501 'searchSourceJSON': json.dumps({
507 "analyze_wildcard": True
513 separators=(',', ':'))
515 self['metadata'] = self.visualization_detail['metadata']
518 url = urlparse.urljoin(base_elastic_url, '/.kibana/dashboard/{}'.format(self.id))
519 logger.debug("publishing dashboard '{}'".format(url))
520 shared_utils.publish_json(self, es_user, es_passwd, url)
523 self._publish_visualizations()
527 class KibanaSearchSourceJSON(dict):
530 {"match": {"installer": {"query": installer, "type": "phrase"}}},
531 {"match": {"project_name": {"query": project_name, "type": "phrase"}}},
532 {"match": {"case_name": {"query": case_name, "type": "phrase"}}}
536 def __init__(self, project_name, case_name, installer, pod, version):
537 super(KibanaSearchSourceJSON, self).__init__()
539 {"match": {"project_name": {"query": project_name, "type": "phrase"}}},
540 {"match": {"case_name": {"query": case_name, "type": "phrase"}}},
541 {"match": {"installer": {"query": installer, "type": "phrase"}}},
542 {"match": {"version": {"query": version, "type": "phrase"}}}
545 self["filter"].append({"match": {"pod_name": {"query": pod, "type": "phrase"}}})
548 class VisualizationState(dict):
549 def __init__(self, input_dict):
556 "type": type, # default sum
558 "field": field # mandatory, no default
565 "type": type, # default date_histogram
567 "field": field # default creation_date
571 "type": type, # default area
572 "mode": mode, # default grouped for type 'histogram', stacked for other types
574 "label": "Tempest duration",# mandatory, no default
575 "test_family": "VIM" # mandatory, no default
580 type histogram: grouped
586 super(VisualizationState, self).__init__()
587 metrics = input_dict['metrics']
588 segments = [] if 'segments' not in input_dict else input_dict['segments']
590 graph_type = 'area' if 'type' not in input_dict else input_dict['type']
591 self['type'] = graph_type
593 if 'mode' not in input_dict:
594 if graph_type == 'histogram':
600 mode = input_dict['mode']
605 "smoothLines": False,
607 "interpolate": "linear",
610 "addTimeMarker": False,
611 "defaultYExtents": False,
612 "setYExtents": False,
619 for metric in metrics:
620 self['aggs'].append({
622 "type": 'sum' if 'type' not in metric else metric['type'],
625 "field": metric['params']['field']
630 if len(segments) > 0:
631 for segment in segments:
632 self['aggs'].append({
634 "type": 'date_histogram' if 'type' not in segment else segment['type'],
637 "field": "creation_date" if ('params' not in segment or 'field' not in segment['params'])
638 else segment['params']['field'],
640 "customInterval": "2h",
642 "extended_bounds": {}
647 self['aggs'].append({
649 "type": 'date_histogram',
652 "field": "creation_date",
654 "customInterval": "2h",
656 "extended_bounds": {}
660 self['listeners'] = {}
661 self['title'] = ' '.join(['{} {}'.format(x['type'], x['params']['field']) for x in self['aggs']
662 if x['schema'] == 'metric'])
665 class KibanaVisualization(dict):
666 def __init__(self, project_name, case_name, installer, pod, version, detail):
669 1. filter created from
675 2. visualization state
676 field for y axis (metric) with type (avg, sum, etc.)
677 field for x axis (segment) with type (date_histogram)
681 super(KibanaVisualization, self).__init__()
682 vis_state = VisualizationState(detail)
683 self.vis_state_title = vis_state['title']
684 self['title'] = '{} {} {} {} {} {}'.format(project_name,
686 self.vis_state_title,
690 self.id = self['title'].replace(' ', '-').replace('/', '-')
691 self['visState'] = json.dumps(vis_state, separators=(',', ':'))
692 self['uiStateJSON'] = "{}"
693 self['description'] = "Kibana visualization for project_name '{}', case_name '{}', data '{}', installer '{}'," \
694 " pod '{}' and version '{}'".format(project_name,
696 self.vis_state_title,
701 self['kibanaSavedObjectMeta'] = {"searchSourceJSON": json.dumps(KibanaSearchSourceJSON(project_name,
706 separators=(',', ':'))}
709 def _get_pods_and_versions(project_name, case_name, installer):
710 query_json = json.JSONEncoder().encode({
717 {"match": {"installer": {"query": installer, "type": "phrase"}}},
718 {"match": {"project_name": {"query": project_name, "type": "phrase"}}},
719 {"match": {"case_name": {"query": case_name, "type": "phrase"}}}
725 elastic_data = shared_utils.get_elastic_data(urlparse.urljoin(base_elastic_url, '/test_results/mongo2elastic'),
726 es_user, es_passwd, query_json)
728 pods_and_versions = {}
730 for data in elastic_data:
731 pod = data['pod_name']
732 if pod in pods_and_versions:
733 pods_and_versions[pod].add(data['version'])
735 pods_and_versions[pod] = {data['version']}
737 if 'all' in pods_and_versions:
738 pods_and_versions['all'].add(data['version'])
740 pods_and_versions['all'] = {data['version']}
742 return pods_and_versions
745 def construct_dashboards():
747 iterate over testcase and installer
748 1. get available pods for each testcase/installer pair
749 2. get available version for each testcase/installer/pod tuple
750 3. construct KibanaInput and append
752 :return: list of KibanaDashboards
754 kibana_dashboards = []
755 for project_name, case_name, visualization_details in _testcases:
756 for installer in _installers:
757 pods_and_versions = _get_pods_and_versions(project_name, case_name, installer)
758 for visualization_detail in visualization_details:
759 for pod, versions in pods_and_versions.iteritems():
760 kibana_dashboards.append(KibanaDashboard(project_name, case_name, installer, pod, versions,
761 visualization_detail))
762 return kibana_dashboards
765 def generate_js_inputs(js_file_path, kibana_url, dashboards):
767 for dashboard in dashboards:
768 dashboard_meta = dashboard['metadata']
769 test_family = dashboard_meta['test_family']
770 test_label = dashboard_meta['label']
772 if test_family not in js_dict:
773 js_dict[test_family] = {}
775 js_test_family = js_dict[test_family]
777 if test_label not in js_test_family:
778 js_test_family[test_label] = {}
780 js_test_label = js_test_family[test_label]
782 if dashboard.installer not in js_test_label:
783 js_test_label[dashboard.installer] = {}
785 js_installer = js_test_label[dashboard.installer]
786 js_installer[dashboard.pod] = kibana_url + '#/dashboard/' + dashboard.id
788 with open(js_file_path, 'w+') as js_file_fdesc:
789 js_file_fdesc.write('var kibana_dashboard_links = ')
790 js_file_fdesc.write(str(js_dict).replace("u'", "'"))
793 if __name__ == '__main__':
794 parser = argparse.ArgumentParser(description='Create Kibana dashboards from data in elasticsearch')
795 parser.add_argument('-e', '--elasticsearch-url', default='http://localhost:9200',
796 help='the url of elasticsearch, defaults to http://localhost:9200')
797 parser.add_argument('-js', '--generate_js_inputs', action='store_true',
798 help='Use this argument to generate javascript inputs for kibana landing page')
799 parser.add_argument('--js_path', default='/usr/share/nginx/html/kibana_dashboards/conf.js',
800 help='Path of javascript file with inputs for kibana landing page')
801 parser.add_argument('-k', '--kibana_url', default='https://testresults.opnfv.org/kibana/app/kibana',
802 help='The url of kibana for javascript inputs')
804 parser.add_argument('-u', '--elasticsearch-username',
805 help='the username for elasticsearch')
807 parser.add_argument('-p', '--elasticsearch-password',
808 help='the password for elasticsearch')
810 args = parser.parse_args()
811 base_elastic_url = args.elasticsearch_url
812 generate_inputs = args.generate_js_inputs
813 input_file_path = args.js_path
814 kibana_url = args.kibana_url
815 es_user = args.elasticsearch_username
816 es_passwd = args.elasticsearch_password
818 dashboards = construct_dashboards()
820 for kibana_dashboard in dashboards:
821 kibana_dashboard.publish()
824 generate_js_inputs(input_file_path, kibana_url, dashboards)