11 logger = logging.getLogger('create_kibana_dashboards')
12 logger.setLevel(logging.DEBUG)
13 file_handler = logging.FileHandler('./{}.log'.format('create_kibana_dashboards'))
14 file_handler.setFormatter(logging.Formatter('%(asctime)s %(levelname)s: %(message)s'))
15 logger.addHandler(file_handler)
17 _installers = {'fuel', 'apex', 'compass', 'joid'}
20 class KibanaDashboard(dict):
21 def __init__(self, project_name, case_name, family, installer, pod, scenarios, visualization):
22 super(KibanaDashboard, self).__init__()
23 self.project_name = project_name
24 self.case_name = case_name
26 self.installer = installer
28 self.scenarios = scenarios
29 self.visualization = visualization
30 self._visualization_title = None
31 self._kibana_visualizations = []
32 self._kibana_dashboard = None
33 self._create_visualizations()
36 def _create_visualizations(self):
37 for scenario in self.scenarios:
38 self._kibana_visualizations.append(KibanaVisualization(self.project_name,
45 self._visualization_title = self._kibana_visualizations[0].vis_state_title
47 def _publish_visualizations(self):
48 for visualization in self._kibana_visualizations:
49 url = urlparse.urljoin(base_elastic_url, '/.kibana/visualization/{}'.format(visualization.id))
50 logger.debug("publishing visualization '{}'".format(url))
51 shared_utils.publish_json(visualization, es_creds, url)
53 def _construct_panels(self):
61 for visualization in self._kibana_visualizations:
63 "id": visualization.id,
64 "type": 'visualization',
65 "panelIndex": panel_index,
73 if column > max_columns:
76 return json.dumps(panels_json, separators=(',', ':'))
79 self['title'] = '{} {} {} {} {}'.format(self.project_name,
82 self._visualization_title,
84 self.id = self['title'].replace(' ', '-').replace('/', '-')
87 self['description'] = "Kibana dashboard for project_name '{}', case_name '{}', installer '{}', data '{}' and" \
88 " pod '{}'".format(self.project_name,
91 self._visualization_title,
93 self['panelsJSON'] = self._construct_panels()
94 self['optionsJSON'] = json.dumps({
97 separators=(',', ':'))
98 self['uiStateJSON'] = "{}"
100 self['timeRestore'] = False
101 self['kibanaSavedObjectMeta'] = {
102 'searchSourceJSON': json.dumps({
108 "analyze_wildcard": True
114 separators=(',', ':'))
117 label = self.case_name
118 if 'label' in self.visualization:
119 label += " %s" % self.visualization.get('label')
120 label += " %s" % self.visualization.get('name')
123 "test_family": self.family
127 url = urlparse.urljoin(base_elastic_url, '/.kibana/dashboard/{}'.format(self.id))
128 logger.debug("publishing dashboard '{}'".format(url))
129 shared_utils.publish_json(self, es_creds, url)
132 self._publish_visualizations()
136 class KibanaSearchSourceJSON(dict):
139 {"match": {"installer": {"query": installer, "type": "phrase"}}},
140 {"match": {"project_name": {"query": project_name, "type": "phrase"}}},
141 {"match": {"case_name": {"query": case_name, "type": "phrase"}}}
145 def __init__(self, project_name, case_name, installer, pod, scenario):
146 super(KibanaSearchSourceJSON, self).__init__()
148 {"match": {"project_name": {"query": project_name, "type": "phrase"}}},
149 {"match": {"case_name": {"query": case_name, "type": "phrase"}}},
150 {"match": {"installer": {"query": installer, "type": "phrase"}}},
151 {"match": {"scenario": {"query": scenario, "type": "phrase"}}}
154 self["filter"].append({"match": {"pod_name": {"query": pod, "type": "phrase"}}})
157 class VisualizationState(dict):
158 def __init__(self, visualization):
159 super(VisualizationState, self).__init__()
160 name = visualization.get('name')
161 fields = visualization.get('fields')
163 if name == 'tests_failures':
166 self['type'] = 'histogram'
168 # duration or success_percentage
171 self['type'] = 'line'
177 "smoothLines": False,
179 "interpolate": "linear",
182 "addTimeMarker": False,
183 "defaultYExtents": False,
184 "setYExtents": False,
192 self['aggs'].append({
197 "field": field.get('field')
202 self['aggs'].append({
204 "type": 'date_histogram',
207 "field": "start_date",
209 "customInterval": "2h",
211 "extended_bounds": {}
215 self['listeners'] = {}
216 self['title'] = ' '.join(['{} {}'.format(x['type'], x['params']['field']) for x in self['aggs']
217 if x['schema'] == 'metric'])
220 class KibanaVisualization(dict):
221 def __init__(self, project_name, case_name, installer, pod, scenario, visualization):
224 1. filter created from
230 2. visualization state
231 field for y axis (metric) with type (avg, sum, etc.)
232 field for x axis (segment) with type (date_histogram)
236 super(KibanaVisualization, self).__init__()
237 vis_state = VisualizationState(visualization)
238 self.vis_state_title = vis_state['title']
239 self['title'] = '{} {} {} {} {} {}'.format(project_name,
241 self.vis_state_title,
245 self.id = self['title'].replace(' ', '-').replace('/', '-')
246 self['visState'] = json.dumps(vis_state, separators=(',', ':'))
247 self['uiStateJSON'] = "{}"
248 self['description'] = "Kibana visualization for project_name '{}', case_name '{}', data '{}', installer '{}'," \
249 " pod '{}' and scenario '{}'".format(project_name,
251 self.vis_state_title,
256 self['kibanaSavedObjectMeta'] = {"searchSourceJSON": json.dumps(KibanaSearchSourceJSON(project_name,
261 separators=(',', ':'))}
264 def _get_pods_and_scenarios(project_name, case_name, installer):
265 query_json = json.JSONEncoder().encode({
272 {"match": {"installer": {"query": installer, "type": "phrase"}}},
273 {"match": {"project_name": {"query": project_name, "type": "phrase"}}},
274 {"match": {"case_name": {"query": case_name, "type": "phrase"}}}
280 elastic_data = shared_utils.get_elastic_data(urlparse.urljoin(base_elastic_url, '/test_results/mongo2elastic'),
281 es_creds, query_json)
283 pods_and_scenarios = {}
285 for data in elastic_data:
286 pod = data['pod_name']
287 if pod in pods_and_scenarios:
288 pods_and_scenarios[pod].add(data['scenario'])
290 pods_and_scenarios[pod] = {data['scenario']}
292 if 'all' in pods_and_scenarios:
293 pods_and_scenarios['all'].add(data['scenario'])
295 pods_and_scenarios['all'] = {data['scenario']}
297 return pods_and_scenarios
300 def construct_dashboards():
302 iterate over testcase and installer
303 1. get available pods for each testcase/installer pair
304 2. get available scenario for each testcase/installer/pod tuple
305 3. construct KibanaInput and append
307 :return: list of KibanaDashboards
309 kibana_dashboards = []
310 for project, case_dicts in conf_utils.testcases_yaml.items():
311 for case in case_dicts:
312 case_name = case.get('name')
313 visualizations = case.get('visualizations')
314 family = case.get('test_family')
315 for installer in _installers:
316 pods_and_scenarios = _get_pods_and_scenarios(project, case_name, installer)
317 for visualization in visualizations:
318 for pod, scenarios in pods_and_scenarios.iteritems():
319 kibana_dashboards.append(KibanaDashboard(project,
326 return kibana_dashboards
329 def generate_js_inputs(js_file_path, kibana_url, dashboards):
331 for dashboard in dashboards:
332 dashboard_meta = dashboard['metadata']
333 test_family = dashboard_meta['test_family']
334 test_label = dashboard_meta['label']
336 if test_family not in js_dict:
337 js_dict[test_family] = {}
339 js_test_family = js_dict[test_family]
341 if test_label not in js_test_family:
342 js_test_family[test_label] = {}
344 js_test_label = js_test_family[test_label]
346 if dashboard.installer not in js_test_label:
347 js_test_label[dashboard.installer] = {}
349 js_installer = js_test_label[dashboard.installer]
350 js_installer[dashboard.pod] = kibana_url + '#/dashboard/' + dashboard.id
352 with open(js_file_path, 'w+') as js_file_fdesc:
353 js_file_fdesc.write('var kibana_dashboard_links = ')
354 js_file_fdesc.write(str(js_dict).replace("u'", "'"))
357 if __name__ == '__main__':
358 parser = argparse.ArgumentParser(description='Create Kibana dashboards from data in elasticsearch')
359 parser.add_argument('-e', '--elasticsearch-url', default='http://localhost:9200',
360 help='the url of elasticsearch, defaults to http://localhost:9200')
362 parser.add_argument('-js', '--generate_js_inputs', action='store_true',
363 help='Use this argument to generate javascript inputs for kibana landing page')
365 parser.add_argument('--js_path', default='/usr/share/nginx/html/kibana_dashboards/conf.js',
366 help='Path of javascript file with inputs for kibana landing page')
368 parser.add_argument('-k', '--kibana_url', default='https://testresults.opnfv.org/kibana/app/kibana',
369 help='The url of kibana for javascript inputs')
371 parser.add_argument('-u', '--elasticsearch-username', default=None,
372 help='The username with password for elasticsearch in format username:password')
374 args = parser.parse_args()
375 base_elastic_url = args.elasticsearch_url
376 generate_inputs = args.generate_js_inputs
377 input_file_path = args.js_path
378 kibana_url = args.kibana_url
379 es_creds = args.elasticsearch_username
381 dashboards = construct_dashboards()
383 for kibana_dashboard in dashboards:
384 kibana_dashboard.publish()
387 generate_js_inputs(input_file_path, kibana_url, dashboards)