Merge "prototypes: bifrost: Check we are running the destroy-env.sh as root"
[releng.git] / utils / test / scripts / create_kibana_dashboards.py
1 #! /usr/bin/env python
2 import json
3 import urlparse
4
5 import argparse
6
7 import conf_utils
8 import logger_utils
9 import shared_utils
10
11 logger = logger_utils.KibanaDashboardLogger('elastic2kibana').get
12
13 _installers = {'fuel', 'apex', 'compass', 'joid'}
14
15
16 class KibanaDashboard(dict):
17     def __init__(self, project_name, case_name, family, installer, pod, scenarios, visualization):
18         super(KibanaDashboard, self).__init__()
19         self.project_name = project_name
20         self.case_name = case_name
21         self.family = family
22         self.installer = installer
23         self.pod = pod
24         self.scenarios = scenarios
25         self.visualization = visualization
26         self._visualization_title = None
27         self._kibana_visualizations = []
28         self._kibana_dashboard = None
29         self._create_visualizations()
30         self._create()
31
32     def _create_visualizations(self):
33         for scenario in self.scenarios:
34             self._kibana_visualizations.append(KibanaVisualization(self.project_name,
35                                                                    self.case_name,
36                                                                    self.installer,
37                                                                    self.pod,
38                                                                    scenario,
39                                                                    self.visualization))
40
41         self._visualization_title = self._kibana_visualizations[0].vis_state_title
42
43     def _publish_visualizations(self):
44         for visualization in self._kibana_visualizations:
45             url = urlparse.urljoin(base_elastic_url, '/.kibana/visualization/{}'.format(visualization.id))
46             logger.debug("publishing visualization '{}'".format(url))
47             shared_utils.publish_json(visualization, es_creds, url)
48
49     def _construct_panels(self):
50         size_x = 6
51         size_y = 3
52         max_columns = 7
53         column = 1
54         row = 1
55         panel_index = 1
56         panels_json = []
57         for visualization in self._kibana_visualizations:
58             panels_json.append({
59                 "id": visualization.id,
60                 "type": 'visualization',
61                 "panelIndex": panel_index,
62                 "size_x": size_x,
63                 "size_y": size_y,
64                 "col": column,
65                 "row": row
66             })
67             panel_index += 1
68             column += size_x
69             if column > max_columns:
70                 column = 1
71                 row += size_y
72         return json.dumps(panels_json, separators=(',', ':'))
73
74     def _create(self):
75         self['title'] = '{} {} {} {} {}'.format(self.project_name,
76                                                 self.case_name,
77                                                 self.installer,
78                                                 self._visualization_title,
79                                                 self.pod)
80         self.id = self['title'].replace(' ', '-').replace('/', '-')
81
82         self['hits'] = 0
83         self['description'] = "Kibana dashboard for project_name '{}', case_name '{}', installer '{}', data '{}' and" \
84                               " pod '{}'".format(self.project_name,
85                                                  self.case_name,
86                                                  self.installer,
87                                                  self._visualization_title,
88                                                  self.pod)
89         self['panelsJSON'] = self._construct_panels()
90         self['optionsJSON'] = json.dumps({
91             "darkTheme": False
92         },
93             separators=(',', ':'))
94         self['uiStateJSON'] = "{}"
95         self['scenario'] = 1
96         self['timeRestore'] = False
97         self['kibanaSavedObjectMeta'] = {
98             'searchSourceJSON': json.dumps({
99                 "filter": [
100                     {
101                         "query": {
102                             "query_string": {
103                                 "query": "*",
104                                 "analyze_wildcard": True
105                             }
106                         }
107                     }
108                 ]
109             },
110                 separators=(',', ':'))
111         }
112
113         label = self.case_name
114         if 'label' in self.visualization:
115             label += " %s" % self.visualization.get('label')
116         label += " %s" % self.visualization.get('name')
117         self['metadata'] = {
118             "label": label,
119             "test_family": self.family
120         }
121
122     def _publish(self):
123         url = urlparse.urljoin(base_elastic_url, '/.kibana/dashboard/{}'.format(self.id))
124         logger.debug("publishing dashboard '{}'".format(url))
125         shared_utils.publish_json(self, es_creds, url)
126
127     def publish(self):
128         self._publish_visualizations()
129         self._publish()
130
131
132 class KibanaSearchSourceJSON(dict):
133     """
134     "filter": [
135                     {"match": {"installer": {"query": installer, "type": "phrase"}}},
136                     {"match": {"project_name": {"query": project_name, "type": "phrase"}}},
137                     {"match": {"case_name": {"query": case_name, "type": "phrase"}}}
138                 ]
139     """
140
141     def __init__(self, project_name, case_name, installer, pod, scenario):
142         super(KibanaSearchSourceJSON, self).__init__()
143         self["filter"] = [
144             {"match": {"project_name": {"query": project_name, "type": "phrase"}}},
145             {"match": {"case_name": {"query": case_name, "type": "phrase"}}},
146             {"match": {"installer": {"query": installer, "type": "phrase"}}},
147             {"match": {"scenario": {"query": scenario, "type": "phrase"}}}
148         ]
149         if pod != 'all':
150             self["filter"].append({"match": {"pod_name": {"query": pod, "type": "phrase"}}})
151
152
153 class VisualizationState(dict):
154     def __init__(self, visualization):
155         super(VisualizationState, self).__init__()
156         name = visualization.get('name')
157         fields = visualization.get('fields')
158
159         if name == 'tests_failures':
160             mode = 'grouped'
161             metric_type = 'sum'
162             self['type'] = 'histogram'
163         else:
164             # duration or success_percentage
165             mode = 'stacked'
166             metric_type = 'avg'
167             self['type'] = 'line'
168
169         self['params'] = {
170             "shareYAxis": True,
171             "addTooltip": True,
172             "addLegend": True,
173             "smoothLines": False,
174             "scale": "linear",
175             "interpolate": "linear",
176             "mode": mode,
177             "times": [],
178             "addTimeMarker": False,
179             "defaultYExtents": False,
180             "setYExtents": False,
181             "yAxis": {}
182         }
183
184         self['aggs'] = []
185
186         i = 1
187         for field in fields:
188             self['aggs'].append({
189                 "id": str(i),
190                 "type": metric_type,
191                 "schema": "metric",
192                 "params": {
193                     "field": field.get('field')
194                 }
195             })
196             i += 1
197
198         self['aggs'].append({
199                 "id": str(i),
200                 "type": 'date_histogram',
201                 "schema": "segment",
202                 "params": {
203                     "field": "start_date",
204                     "interval": "auto",
205                     "customInterval": "2h",
206                     "min_doc_count": 1,
207                     "extended_bounds": {}
208                 }
209             })
210
211         self['listeners'] = {}
212         self['title'] = ' '.join(['{} {}'.format(x['type'], x['params']['field']) for x in self['aggs']
213                                   if x['schema'] == 'metric'])
214
215
216 class KibanaVisualization(dict):
217     def __init__(self, project_name, case_name, installer, pod, scenario, visualization):
218         """
219         We need two things
220         1. filter created from
221             project_name
222             case_name
223             installer
224             pod
225             scenario
226         2. visualization state
227             field for y axis (metric) with type (avg, sum, etc.)
228             field for x axis (segment) with type (date_histogram)
229
230         :return:
231         """
232         super(KibanaVisualization, self).__init__()
233         vis_state = VisualizationState(visualization)
234         self.vis_state_title = vis_state['title']
235         self['title'] = '{} {} {} {} {} {}'.format(project_name,
236                                                    case_name,
237                                                    self.vis_state_title,
238                                                    installer,
239                                                    pod,
240                                                    scenario)
241         self.id = self['title'].replace(' ', '-').replace('/', '-')
242         self['visState'] = json.dumps(vis_state, separators=(',', ':'))
243         self['uiStateJSON'] = "{}"
244         self['description'] = "Kibana visualization for project_name '{}', case_name '{}', data '{}', installer '{}'," \
245                               " pod '{}' and scenario '{}'".format(project_name,
246                                                                   case_name,
247                                                                   self.vis_state_title,
248                                                                   installer,
249                                                                   pod,
250                                                                   scenario)
251         self['scenario'] = 1
252         self['kibanaSavedObjectMeta'] = {"searchSourceJSON": json.dumps(KibanaSearchSourceJSON(project_name,
253                                                                                                case_name,
254                                                                                                installer,
255                                                                                                pod,
256                                                                                                scenario),
257                                                                         separators=(',', ':'))}
258
259
260 def _get_pods_and_scenarios(project_name, case_name, installer):
261     query_json = json.JSONEncoder().encode({
262         "query": {
263             "bool": {
264                 "must": [
265                     {"match_all": {}}
266                 ],
267                 "filter": [
268                     {"match": {"installer": {"query": installer, "type": "phrase"}}},
269                     {"match": {"project_name": {"query": project_name, "type": "phrase"}}},
270                     {"match": {"case_name": {"query": case_name, "type": "phrase"}}}
271                 ]
272             }
273         }
274     })
275
276     elastic_data = shared_utils.get_elastic_data(urlparse.urljoin(base_elastic_url, '/test_results/mongo2elastic'),
277                                                  es_creds, query_json)
278
279     pods_and_scenarios = {}
280
281     for data in elastic_data:
282         pod = data['pod_name']
283         if pod in pods_and_scenarios:
284             pods_and_scenarios[pod].add(data['scenario'])
285         else:
286             pods_and_scenarios[pod] = {data['scenario']}
287
288         if 'all' in pods_and_scenarios:
289             pods_and_scenarios['all'].add(data['scenario'])
290         else:
291             pods_and_scenarios['all'] = {data['scenario']}
292
293     return pods_and_scenarios
294
295
296 def construct_dashboards():
297     """
298     iterate over testcase and installer
299     1. get available pods for each testcase/installer pair
300     2. get available scenario for each testcase/installer/pod tuple
301     3. construct KibanaInput and append
302
303     :return: list of KibanaDashboards
304     """
305     kibana_dashboards = []
306     for project, case_dicts in conf_utils.testcases_yaml.items():
307         for case in case_dicts:
308             case_name = case.get('name')
309             visualizations = case.get('visualizations')
310             family = case.get('test_family')
311             for installer in _installers:
312                 pods_and_scenarios = _get_pods_and_scenarios(project, case_name, installer)
313                 for visualization in visualizations:
314                     for pod, scenarios in pods_and_scenarios.iteritems():
315                         kibana_dashboards.append(KibanaDashboard(project,
316                                                                  case_name,
317                                                                  family,
318                                                                  installer,
319                                                                  pod,
320                                                                  scenarios,
321                                                                  visualization))
322     return kibana_dashboards
323
324
325 def generate_js_inputs(js_file_path, kibana_url, dashboards):
326     js_dict = {}
327     for dashboard in dashboards:
328         dashboard_meta = dashboard['metadata']
329         test_family = dashboard_meta['test_family']
330         test_label = dashboard_meta['label']
331
332         if test_family not in js_dict:
333             js_dict[test_family] = {}
334
335         js_test_family = js_dict[test_family]
336
337         if test_label not in js_test_family:
338             js_test_family[test_label] = {}
339
340         js_test_label = js_test_family[test_label]
341
342         if dashboard.installer not in js_test_label:
343             js_test_label[dashboard.installer] = {}
344
345         js_installer = js_test_label[dashboard.installer]
346         js_installer[dashboard.pod] = kibana_url + '#/dashboard/' + dashboard.id
347
348     with open(js_file_path, 'w+') as js_file_fdesc:
349         js_file_fdesc.write('var kibana_dashboard_links = ')
350         js_file_fdesc.write(str(js_dict).replace("u'", "'"))
351
352
353 if __name__ == '__main__':
354     parser = argparse.ArgumentParser(description='Create Kibana dashboards from data in elasticsearch')
355     parser.add_argument('-e', '--elasticsearch-url', default='http://localhost:9200',
356                         help='the url of elasticsearch, defaults to http://localhost:9200')
357
358     parser.add_argument('-js', '--generate_js_inputs', action='store_true',
359                         help='Use this argument to generate javascript inputs for kibana landing page')
360
361     parser.add_argument('--js_path', default='/usr/share/nginx/html/kibana_dashboards/conf.js',
362                         help='Path of javascript file with inputs for kibana landing page')
363
364     parser.add_argument('-k', '--kibana_url', default='https://testresults.opnfv.org/kibana/app/kibana',
365                         help='The url of kibana for javascript inputs')
366
367     parser.add_argument('-u', '--elasticsearch-username', default=None,
368                         help='The username with password for elasticsearch in format username:password')
369
370     args = parser.parse_args()
371     base_elastic_url = args.elasticsearch_url
372     generate_inputs = args.generate_js_inputs
373     input_file_path = args.js_path
374     kibana_url = args.kibana_url
375     es_creds = args.elasticsearch_username
376
377     dashboards = construct_dashboards()
378
379     for kibana_dashboard in dashboards:
380         kibana_dashboard.publish()
381
382     if generate_inputs:
383         generate_js_inputs(input_file_path, kibana_url, dashboards)