37ce03e52e1e5b5fb85fc26cf4010d76ed11061f
[releng.git] / utils / test / dashboard / dashboard / elastic2kibana / main.py
1 #! /usr/bin/env python
2 import json
3 import urlparse
4
5 import argparse
6 from jinja2 import PackageLoader, Environment
7
8 from common import logger_utils, elastic_access
9 from conf import testcases
10 from conf.config import APIConfig
11
12 logger = logger_utils.DashboardLogger('elastic2kibana').get
13
14 parser = argparse.ArgumentParser()
15 parser.add_argument("-c", "--config-file",
16                     dest='config_file',
17                     help="Config file location")
18
19 args = parser.parse_args()
20 CONF = APIConfig().parse(args.config_file)
21 base_elastic_url = CONF.elastic_url
22 generate_inputs = CONF.is_js
23 input_file_path = CONF.js_path
24 kibana_url = CONF.kibana_url
25 es_creds = CONF.elastic_creds
26
27 _installers = {'fuel', 'apex', 'compass', 'joid'}
28
29
30 class KibanaDashboard(dict):
31     def __init__(self, project_name, case_name, family, installer, pod, scenarios, visualization):
32         super(KibanaDashboard, self).__init__()
33         self.project_name = project_name
34         self.case_name = case_name
35         self.family = family
36         self.installer = installer
37         self.pod = pod
38         self.scenarios = scenarios
39         self.visualization = visualization
40         self._visualization_title = None
41         self._kibana_visualizations = []
42         self._kibana_dashboard = None
43         self._create_visualizations()
44         self._create()
45
46     def _create_visualizations(self):
47         for scenario in self.scenarios:
48             self._kibana_visualizations.append(KibanaVisualization(self.project_name,
49                                                                    self.case_name,
50                                                                    self.installer,
51                                                                    self.pod,
52                                                                    scenario,
53                                                                    self.visualization))
54
55         self._visualization_title = self._kibana_visualizations[0].vis_title
56
57     def _publish_visualizations(self):
58         for visualization in self._kibana_visualizations:
59             url = urlparse.urljoin(base_elastic_url, '/.kibana/visualization/{}'.format(visualization.id))
60             logger.debug("publishing visualization '{}'".format(url))
61             # logger.error("_publish_visualization: %s" % visualization)
62             elastic_access.publish_json(visualization, es_creds, url)
63
64     def _construct_panels(self):
65         size_x = 6
66         size_y = 3
67         max_columns = 7
68         column = 1
69         row = 1
70         panel_index = 1
71         panels_json = []
72         for visualization in self._kibana_visualizations:
73             panels_json.append({
74                 "id": visualization.id,
75                 "type": 'visualization',
76                 "panelIndex": panel_index,
77                 "size_x": size_x,
78                 "size_y": size_y,
79                 "col": column,
80                 "row": row
81             })
82             panel_index += 1
83             column += size_x
84             if column > max_columns:
85                 column = 1
86                 row += size_y
87         return json.dumps(panels_json, separators=(',', ':'))
88
89     def _create(self):
90         self['title'] = '{} {} {} {} {}'.format(self.project_name,
91                                                 self.case_name,
92                                                 self.installer,
93                                                 self._visualization_title,
94                                                 self.pod)
95         self.id = self['title'].replace(' ', '-').replace('/', '-')
96
97         self['hits'] = 0
98         self['description'] = "Kibana dashboard for project_name '{}', case_name '{}', installer '{}', data '{}' and" \
99                               " pod '{}'".format(self.project_name,
100                                                  self.case_name,
101                                                  self.installer,
102                                                  self._visualization_title,
103                                                  self.pod)
104         self['panelsJSON'] = self._construct_panels()
105         self['optionsJSON'] = json.dumps({
106             "darkTheme": False
107         },
108             separators=(',', ':'))
109         self['uiStateJSON'] = "{}"
110         self['scenario'] = 1
111         self['timeRestore'] = False
112         self['kibanaSavedObjectMeta'] = {
113             'searchSourceJSON': json.dumps({
114                 "filter": [
115                     {
116                         "query": {
117                             "query_string": {
118                                 "query": "*",
119                                 "analyze_wildcard": True
120                             }
121                         }
122                     }
123                 ]
124             },
125                 separators=(',', ':'))
126         }
127
128         label = self.case_name
129         if 'label' in self.visualization:
130             label += " %s" % self.visualization.get('label')
131         label += " %s" % self.visualization.get('name')
132         self['metadata'] = {
133             "label": label,
134             "test_family": self.family
135         }
136
137     def _publish(self):
138         url = urlparse.urljoin(base_elastic_url, '/.kibana/dashboard/{}'.format(self.id))
139         logger.debug("publishing dashboard '{}'".format(url))
140         elastic_access.publish_json(self, es_creds, url)
141
142     def publish(self):
143         self._publish_visualizations()
144         self._publish()
145
146
147 class KibanaSearchSourceJSON(dict):
148     """
149     "filter": [
150                     {"match": {"installer": {"query": installer, "type": "phrase"}}},
151                     {"match": {"project_name": {"query": project_name, "type": "phrase"}}},
152                     {"match": {"case_name": {"query": case_name, "type": "phrase"}}}
153                 ]
154     """
155
156     def __init__(self, project_name, case_name, installer, pod, scenario):
157         super(KibanaSearchSourceJSON, self).__init__()
158         self["filter"] = [
159             {"match": {"project_name": {"query": project_name, "type": "phrase"}}},
160             {"match": {"case_name": {"query": case_name, "type": "phrase"}}},
161             {"match": {"installer": {"query": installer, "type": "phrase"}}},
162             {"match": {"scenario": {"query": scenario, "type": "phrase"}}}
163         ]
164         if pod != 'all':
165             self["filter"].append({"match": {"pod_name": {"query": pod, "type": "phrase"}}})
166
167
168 class VisualizationBuilder(object):
169     def __init__(self, visualization):
170         super(VisualizationBuilder, self).__init__()
171         self.visualization = visualization
172
173     def build(self):
174         name = self.visualization.get('name')
175         fields = self.visualization.get('fields')
176
177         aggs = []
178         index = 1
179         for field in fields:
180             aggs.append({
181                 "id": index,
182                 "field": field.get("field")
183             })
184             index += 1
185
186         env = Environment(loader=PackageLoader('elastic2kibana', 'templates'))
187         env.filters['jsonify'] = json.dumps
188         template = env.get_template('{}.json'.format(name))
189         vis = template.render(aggs=aggs)
190         return json.loads(vis)
191
192
193
194 class KibanaVisualization(dict):
195     def __init__(self, project_name, case_name, installer, pod, scenario, visualization):
196         """
197         We need two things
198         1. filter created from
199             project_name
200             case_name
201             installer
202             pod
203             scenario
204         2. visualization state
205             field for y axis (metric) with type (avg, sum, etc.)
206             field for x axis (segment) with type (date_histogram)
207
208         :return:
209         """
210         super(KibanaVisualization, self).__init__()
211         vis = VisualizationBuilder(visualization).build()
212         self.vis_title = vis['title']
213         self['title'] = '{} {} {} {} {} {}'.format(project_name,
214                                                    case_name,
215                                                    self.vis_title,
216                                                    installer,
217                                                    pod,
218                                                    scenario)
219         self.id = self['title'].replace(' ', '-').replace('/', '-')
220         self['visState'] = json.dumps(vis, separators=(',', ':'))
221         self['uiStateJSON'] = "{}"
222         self['description'] = "Kibana visualization for project_name '{}', case_name '{}', metric '{}', installer '{}'," \
223                               " pod '{}' and scenario '{}'".format(project_name,
224                                                                    case_name,
225                                                                    self.vis_title,
226                                                                    installer,
227                                                                    pod,
228                                                                    scenario)
229         self['scenario'] = 1
230         self['kibanaSavedObjectMeta'] = {"searchSourceJSON": json.dumps(KibanaSearchSourceJSON(project_name,
231                                                                                                case_name,
232                                                                                                installer,
233                                                                                                pod,
234                                                                                                scenario),
235                                                                         separators=(',', ':'))}
236
237
238 def _get_pods_and_scenarios(project_name, case_name, installer):
239     query_json = json.JSONEncoder().encode({
240         "query": {
241             "bool": {
242                 "must": [
243                     {"match_all": {}}
244                 ],
245                 "filter": [
246                     {"match": {"installer": {"query": installer, "type": "phrase"}}},
247                     {"match": {"project_name": {"query": project_name, "type": "phrase"}}},
248                     {"match": {"case_name": {"query": case_name, "type": "phrase"}}}
249                 ]
250             }
251         }
252     })
253
254     elastic_data = elastic_access.get_elastic_docs(urlparse.urljoin(base_elastic_url, '/test_results/mongo2elastic'),
255                                                    es_creds, query_json)
256
257     pods_and_scenarios = {}
258
259     for data in elastic_data:
260         pod = data['pod_name']
261         if pod in pods_and_scenarios:
262             pods_and_scenarios[pod].add(data['scenario'])
263         else:
264             pods_and_scenarios[pod] = {data['scenario']}
265
266         if 'all' in pods_and_scenarios:
267             pods_and_scenarios['all'].add(data['scenario'])
268         else:
269             pods_and_scenarios['all'] = {data['scenario']}
270
271     return pods_and_scenarios
272
273
274 def construct_dashboards():
275     """
276     iterate over testcase and installer
277     1. get available pods for each testcase/installer pair
278     2. get available scenario for each testcase/installer/pod tuple
279     3. construct KibanaInput and append
280
281     :return: list of KibanaDashboards
282     """
283     kibana_dashboards = []
284     for project, case_dicts in testcases.testcases_yaml.items():
285         for case in case_dicts:
286             case_name = case.get('name')
287             visualizations = case.get('visualizations')
288             family = case.get('test_family')
289             for installer in _installers:
290                 pods_and_scenarios = _get_pods_and_scenarios(project, case_name, installer)
291                 for visualization in visualizations:
292                     for pod, scenarios in pods_and_scenarios.iteritems():
293                         kibana_dashboards.append(KibanaDashboard(project,
294                                                                  case_name,
295                                                                  family,
296                                                                  installer,
297                                                                  pod,
298                                                                  scenarios,
299                                                                  visualization))
300     return kibana_dashboards
301
302
303 def generate_js_inputs(js_file_path, kibana_url, dashboards):
304     js_dict = {}
305     for dashboard in dashboards:
306         dashboard_meta = dashboard['metadata']
307         test_family = dashboard_meta['test_family']
308         test_label = dashboard_meta['label']
309
310         if test_family not in js_dict:
311             js_dict[test_family] = {}
312
313         js_test_family = js_dict[test_family]
314
315         if test_label not in js_test_family:
316             js_test_family[test_label] = {}
317
318         js_test_label = js_test_family[test_label]
319
320         if dashboard.installer not in js_test_label:
321             js_test_label[dashboard.installer] = {}
322
323         js_installer = js_test_label[dashboard.installer]
324         js_installer[dashboard.pod] = kibana_url + '#/dashboard/' + dashboard.id
325
326     with open(js_file_path, 'w+') as js_file_fdesc:
327         js_file_fdesc.write('var kibana_dashboard_links = ')
328         js_file_fdesc.write(str(js_dict).replace("u'", "'"))
329
330
331 def main():
332     dashboards = construct_dashboards()
333
334     for kibana_dashboard in dashboards:
335         kibana_dashboard.publish()
336
337     if generate_inputs:
338         generate_js_inputs(input_file_path, kibana_url, dashboards)