9 import testcases_parser
10 from config import APIConfig
12 logger = logger_utils.KibanaDashboardLogger('elastic2kibana').get
14 parser = argparse.ArgumentParser()
15 parser.add_argument("-c", "--config-file",
17 help="Config file location")
19 args = parser.parse_args()
20 CONF = APIConfig().parse(args.config_file)
22 _installers = {'fuel', 'apex', 'compass', 'joid'}
25 class KibanaDashboard(dict):
26 def __init__(self, project_name, case_name, family, installer, pod, scenarios, visualization):
27 super(KibanaDashboard, self).__init__()
28 self.project_name = project_name
29 self.case_name = case_name
31 self.installer = installer
33 self.scenarios = scenarios
34 self.visualization = visualization
35 self._visualization_title = None
36 self._kibana_visualizations = []
37 self._kibana_dashboard = None
38 self._create_visualizations()
41 def _create_visualizations(self):
42 for scenario in self.scenarios:
43 self._kibana_visualizations.append(KibanaVisualization(self.project_name,
50 self._visualization_title = self._kibana_visualizations[0].vis_state_title
52 def _publish_visualizations(self):
53 for visualization in self._kibana_visualizations:
54 url = urlparse.urljoin(base_elastic_url, '/.kibana/visualization/{}'.format(visualization.id))
55 logger.debug("publishing visualization '{}'".format(url))
56 shared_utils.publish_json(visualization, es_creds, url)
58 def _construct_panels(self):
66 for visualization in self._kibana_visualizations:
68 "id": visualization.id,
69 "type": 'visualization',
70 "panelIndex": panel_index,
78 if column > max_columns:
81 return json.dumps(panels_json, separators=(',', ':'))
84 self['title'] = '{} {} {} {} {}'.format(self.project_name,
87 self._visualization_title,
89 self.id = self['title'].replace(' ', '-').replace('/', '-')
92 self['description'] = "Kibana dashboard for project_name '{}', case_name '{}', installer '{}', data '{}' and" \
93 " pod '{}'".format(self.project_name,
96 self._visualization_title,
98 self['panelsJSON'] = self._construct_panels()
99 self['optionsJSON'] = json.dumps({
102 separators=(',', ':'))
103 self['uiStateJSON'] = "{}"
105 self['timeRestore'] = False
106 self['kibanaSavedObjectMeta'] = {
107 'searchSourceJSON': json.dumps({
113 "analyze_wildcard": True
119 separators=(',', ':'))
122 label = self.case_name
123 if 'label' in self.visualization:
124 label += " %s" % self.visualization.get('label')
125 label += " %s" % self.visualization.get('name')
128 "test_family": self.family
132 url = urlparse.urljoin(base_elastic_url, '/.kibana/dashboard/{}'.format(self.id))
133 logger.debug("publishing dashboard '{}'".format(url))
134 shared_utils.publish_json(self, es_creds, url)
137 self._publish_visualizations()
141 class KibanaSearchSourceJSON(dict):
144 {"match": {"installer": {"query": installer, "type": "phrase"}}},
145 {"match": {"project_name": {"query": project_name, "type": "phrase"}}},
146 {"match": {"case_name": {"query": case_name, "type": "phrase"}}}
150 def __init__(self, project_name, case_name, installer, pod, scenario):
151 super(KibanaSearchSourceJSON, self).__init__()
153 {"match": {"project_name": {"query": project_name, "type": "phrase"}}},
154 {"match": {"case_name": {"query": case_name, "type": "phrase"}}},
155 {"match": {"installer": {"query": installer, "type": "phrase"}}},
156 {"match": {"scenario": {"query": scenario, "type": "phrase"}}}
159 self["filter"].append({"match": {"pod_name": {"query": pod, "type": "phrase"}}})
162 class VisualizationState(dict):
163 def __init__(self, visualization):
164 super(VisualizationState, self).__init__()
165 name = visualization.get('name')
166 fields = visualization.get('fields')
168 if name == 'tests_failures':
171 self['type'] = 'histogram'
173 # duration or success_percentage
176 self['type'] = 'line'
182 "smoothLines": False,
184 "interpolate": "linear",
187 "addTimeMarker": False,
188 "defaultYExtents": False,
189 "setYExtents": False,
197 self['aggs'].append({
202 "field": field.get('field')
207 self['aggs'].append({
209 "type": 'date_histogram',
212 "field": "start_date",
214 "customInterval": "2h",
216 "extended_bounds": {}
220 self['listeners'] = {}
221 self['title'] = ' '.join(['{} {}'.format(x['type'], x['params']['field']) for x in self['aggs']
222 if x['schema'] == 'metric'])
225 class KibanaVisualization(dict):
226 def __init__(self, project_name, case_name, installer, pod, scenario, visualization):
229 1. filter created from
235 2. visualization state
236 field for y axis (metric) with type (avg, sum, etc.)
237 field for x axis (segment) with type (date_histogram)
241 super(KibanaVisualization, self).__init__()
242 vis_state = VisualizationState(visualization)
243 self.vis_state_title = vis_state['title']
244 self['title'] = '{} {} {} {} {} {}'.format(project_name,
246 self.vis_state_title,
250 self.id = self['title'].replace(' ', '-').replace('/', '-')
251 self['visState'] = json.dumps(vis_state, separators=(',', ':'))
252 self['uiStateJSON'] = "{}"
253 self['description'] = "Kibana visualization for project_name '{}', case_name '{}', data '{}', installer '{}'," \
254 " pod '{}' and scenario '{}'".format(project_name,
256 self.vis_state_title,
261 self['kibanaSavedObjectMeta'] = {"searchSourceJSON": json.dumps(KibanaSearchSourceJSON(project_name,
266 separators=(',', ':'))}
269 def _get_pods_and_scenarios(project_name, case_name, installer):
270 query_json = json.JSONEncoder().encode({
277 {"match": {"installer": {"query": installer, "type": "phrase"}}},
278 {"match": {"project_name": {"query": project_name, "type": "phrase"}}},
279 {"match": {"case_name": {"query": case_name, "type": "phrase"}}}
285 elastic_data = shared_utils.get_elastic_docs(urlparse.urljoin(base_elastic_url, '/test_results/mongo2elastic'),
286 es_creds, query_json)
288 pods_and_scenarios = {}
290 for data in elastic_data:
291 pod = data['pod_name']
292 if pod in pods_and_scenarios:
293 pods_and_scenarios[pod].add(data['scenario'])
295 pods_and_scenarios[pod] = {data['scenario']}
297 if 'all' in pods_and_scenarios:
298 pods_and_scenarios['all'].add(data['scenario'])
300 pods_and_scenarios['all'] = {data['scenario']}
302 return pods_and_scenarios
305 def construct_dashboards():
307 iterate over testcase and installer
308 1. get available pods for each testcase/installer pair
309 2. get available scenario for each testcase/installer/pod tuple
310 3. construct KibanaInput and append
312 :return: list of KibanaDashboards
314 kibana_dashboards = []
315 for project, case_dicts in testcases_parser.testcases_yaml.items():
316 for case in case_dicts:
317 case_name = case.get('name')
318 visualizations = case.get('visualizations')
319 family = case.get('test_family')
320 for installer in _installers:
321 pods_and_scenarios = _get_pods_and_scenarios(project, case_name, installer)
322 for visualization in visualizations:
323 for pod, scenarios in pods_and_scenarios.iteritems():
324 kibana_dashboards.append(KibanaDashboard(project,
331 return kibana_dashboards
334 def generate_js_inputs(js_file_path, kibana_url, dashboards):
336 for dashboard in dashboards:
337 dashboard_meta = dashboard['metadata']
338 test_family = dashboard_meta['test_family']
339 test_label = dashboard_meta['label']
341 if test_family not in js_dict:
342 js_dict[test_family] = {}
344 js_test_family = js_dict[test_family]
346 if test_label not in js_test_family:
347 js_test_family[test_label] = {}
349 js_test_label = js_test_family[test_label]
351 if dashboard.installer not in js_test_label:
352 js_test_label[dashboard.installer] = {}
354 js_installer = js_test_label[dashboard.installer]
355 js_installer[dashboard.pod] = kibana_url + '#/dashboard/' + dashboard.id
357 with open(js_file_path, 'w+') as js_file_fdesc:
358 js_file_fdesc.write('var kibana_dashboard_links = ')
359 js_file_fdesc.write(str(js_dict).replace("u'", "'"))
362 if __name__ == '__main__':
363 base_elastic_url = CONF.elastic_url
364 generate_inputs = CONF.is_js
365 input_file_path = CONF.js_path
366 kibana_url = CONF.kibana_url
367 es_creds = CONF.elastic_creds
369 dashboards = construct_dashboards()
371 for kibana_dashboard in dashboards:
372 kibana_dashboard.publish()
375 generate_js_inputs(input_file_path, kibana_url, dashboards)