7 from common import logger_utils, elastic_access
8 from conf import testcases
9 from conf.config import APIConfig
11 logger = logger_utils.DashboardLogger('elastic2kibana').get
13 parser = argparse.ArgumentParser()
14 parser.add_argument("-c", "--config-file",
16 help="Config file location")
18 args = parser.parse_args()
19 CONF = APIConfig().parse(args.config_file)
20 base_elastic_url = CONF.elastic_url
21 generate_inputs = CONF.is_js
22 input_file_path = CONF.js_path
23 kibana_url = CONF.kibana_url
24 es_creds = CONF.elastic_creds
26 _installers = {'fuel', 'apex', 'compass', 'joid'}
29 class KibanaDashboard(dict):
30 def __init__(self, project_name, case_name, family, installer, pod, scenarios, visualization):
31 super(KibanaDashboard, self).__init__()
32 self.project_name = project_name
33 self.case_name = case_name
35 self.installer = installer
37 self.scenarios = scenarios
38 self.visualization = visualization
39 self._visualization_title = None
40 self._kibana_visualizations = []
41 self._kibana_dashboard = None
42 self._create_visualizations()
45 def _create_visualizations(self):
46 for scenario in self.scenarios:
47 self._kibana_visualizations.append(KibanaVisualization(self.project_name,
54 self._visualization_title = self._kibana_visualizations[0].vis_state_title
56 def _publish_visualizations(self):
57 for visualization in self._kibana_visualizations:
58 url = urlparse.urljoin(base_elastic_url, '/.kibana/visualization/{}'.format(visualization.id))
59 logger.debug("publishing visualization '{}'".format(url))
60 elastic_access.publish_json(visualization, es_creds, url)
62 def _construct_panels(self):
70 for visualization in self._kibana_visualizations:
72 "id": visualization.id,
73 "type": 'visualization',
74 "panelIndex": panel_index,
82 if column > max_columns:
85 return json.dumps(panels_json, separators=(',', ':'))
88 self['title'] = '{} {} {} {} {}'.format(self.project_name,
91 self._visualization_title,
93 self.id = self['title'].replace(' ', '-').replace('/', '-')
96 self['description'] = "Kibana dashboard for project_name '{}', case_name '{}', installer '{}', data '{}' and" \
97 " pod '{}'".format(self.project_name,
100 self._visualization_title,
102 self['panelsJSON'] = self._construct_panels()
103 self['optionsJSON'] = json.dumps({
106 separators=(',', ':'))
107 self['uiStateJSON'] = "{}"
109 self['timeRestore'] = False
110 self['kibanaSavedObjectMeta'] = {
111 'searchSourceJSON': json.dumps({
117 "analyze_wildcard": True
123 separators=(',', ':'))
126 label = self.case_name
127 if 'label' in self.visualization:
128 label += " %s" % self.visualization.get('label')
129 label += " %s" % self.visualization.get('name')
132 "test_family": self.family
136 url = urlparse.urljoin(base_elastic_url, '/.kibana/dashboard/{}'.format(self.id))
137 logger.debug("publishing dashboard '{}'".format(url))
138 elastic_access.publish_json(self, es_creds, url)
141 self._publish_visualizations()
145 class KibanaSearchSourceJSON(dict):
148 {"match": {"installer": {"query": installer, "type": "phrase"}}},
149 {"match": {"project_name": {"query": project_name, "type": "phrase"}}},
150 {"match": {"case_name": {"query": case_name, "type": "phrase"}}}
154 def __init__(self, project_name, case_name, installer, pod, scenario):
155 super(KibanaSearchSourceJSON, self).__init__()
157 {"match": {"project_name": {"query": project_name, "type": "phrase"}}},
158 {"match": {"case_name": {"query": case_name, "type": "phrase"}}},
159 {"match": {"installer": {"query": installer, "type": "phrase"}}},
160 {"match": {"scenario": {"query": scenario, "type": "phrase"}}}
163 self["filter"].append({"match": {"pod_name": {"query": pod, "type": "phrase"}}})
166 class VisualizationState(dict):
167 def __init__(self, visualization):
168 super(VisualizationState, self).__init__()
169 name = visualization.get('name')
170 fields = visualization.get('fields')
172 if name == 'tests_failures':
175 self['type'] = 'histogram'
177 # duration or success_percentage
180 self['type'] = 'line'
186 "smoothLines": False,
188 "interpolate": "linear",
191 "addTimeMarker": False,
192 "defaultYExtents": False,
193 "setYExtents": False,
201 self['aggs'].append({
206 "field": field.get('field')
211 self['aggs'].append({
213 "type": 'date_histogram',
216 "field": "start_date",
218 "customInterval": "2h",
220 "extended_bounds": {}
224 self['listeners'] = {}
225 self['title'] = ' '.join(['{} {}'.format(x['type'], x['params']['field']) for x in self['aggs']
226 if x['schema'] == 'metric'])
229 class KibanaVisualization(dict):
230 def __init__(self, project_name, case_name, installer, pod, scenario, visualization):
233 1. filter created from
239 2. visualization state
240 field for y axis (metric) with type (avg, sum, etc.)
241 field for x axis (segment) with type (date_histogram)
245 super(KibanaVisualization, self).__init__()
246 vis_state = VisualizationState(visualization)
247 self.vis_state_title = vis_state['title']
248 self['title'] = '{} {} {} {} {} {}'.format(project_name,
250 self.vis_state_title,
254 self.id = self['title'].replace(' ', '-').replace('/', '-')
255 self['visState'] = json.dumps(vis_state, separators=(',', ':'))
256 self['uiStateJSON'] = "{}"
257 self['description'] = "Kibana visualization for project_name '{}', case_name '{}', data '{}', installer '{}'," \
258 " pod '{}' and scenario '{}'".format(project_name,
260 self.vis_state_title,
265 self['kibanaSavedObjectMeta'] = {"searchSourceJSON": json.dumps(KibanaSearchSourceJSON(project_name,
270 separators=(',', ':'))}
273 def _get_pods_and_scenarios(project_name, case_name, installer):
274 query_json = json.JSONEncoder().encode({
281 {"match": {"installer": {"query": installer, "type": "phrase"}}},
282 {"match": {"project_name": {"query": project_name, "type": "phrase"}}},
283 {"match": {"case_name": {"query": case_name, "type": "phrase"}}}
289 elastic_data = elastic_access.get_elastic_docs(urlparse.urljoin(base_elastic_url, '/test_results/mongo2elastic'),
290 es_creds, query_json)
292 pods_and_scenarios = {}
294 for data in elastic_data:
295 pod = data['pod_name']
296 if pod in pods_and_scenarios:
297 pods_and_scenarios[pod].add(data['scenario'])
299 pods_and_scenarios[pod] = {data['scenario']}
301 if 'all' in pods_and_scenarios:
302 pods_and_scenarios['all'].add(data['scenario'])
304 pods_and_scenarios['all'] = {data['scenario']}
306 return pods_and_scenarios
309 def construct_dashboards():
311 iterate over testcase and installer
312 1. get available pods for each testcase/installer pair
313 2. get available scenario for each testcase/installer/pod tuple
314 3. construct KibanaInput and append
316 :return: list of KibanaDashboards
318 kibana_dashboards = []
319 for project, case_dicts in testcases.testcases_yaml.items():
320 for case in case_dicts:
321 case_name = case.get('name')
322 visualizations = case.get('visualizations')
323 family = case.get('test_family')
324 for installer in _installers:
325 pods_and_scenarios = _get_pods_and_scenarios(project, case_name, installer)
326 for visualization in visualizations:
327 for pod, scenarios in pods_and_scenarios.iteritems():
328 kibana_dashboards.append(KibanaDashboard(project,
335 return kibana_dashboards
338 def generate_js_inputs(js_file_path, kibana_url, dashboards):
340 for dashboard in dashboards:
341 dashboard_meta = dashboard['metadata']
342 test_family = dashboard_meta['test_family']
343 test_label = dashboard_meta['label']
345 if test_family not in js_dict:
346 js_dict[test_family] = {}
348 js_test_family = js_dict[test_family]
350 if test_label not in js_test_family:
351 js_test_family[test_label] = {}
353 js_test_label = js_test_family[test_label]
355 if dashboard.installer not in js_test_label:
356 js_test_label[dashboard.installer] = {}
358 js_installer = js_test_label[dashboard.installer]
359 js_installer[dashboard.pod] = kibana_url + '#/dashboard/' + dashboard.id
361 with open(js_file_path, 'w+') as js_file_fdesc:
362 js_file_fdesc.write('var kibana_dashboard_links = ')
363 js_file_fdesc.write(str(js_dict).replace("u'", "'"))
367 dashboards = construct_dashboards()
369 for kibana_dashboard in dashboards:
370 kibana_dashboard.publish()
373 generate_js_inputs(input_file_path, kibana_url, dashboards)