utils/test/scripts/create_kibana_dashboards.py

   1 #! /usr/bin/env python
   2 import json
   3 import urlparse
   4
   5 import argparse
   6
   7 import logger_utils
   8 import shared_utils
   9 import testcases_parser
  10 from config import APIConfig
  11
  12 logger = logger_utils.KibanaDashboardLogger('elastic2kibana').get
  13
  14 parser = argparse.ArgumentParser()
  15 parser.add_argument("-c", "--config-file",
  16                     dest='config_file',
  17                     help="Config file location")
  18
  19 args = parser.parse_args()
  20 CONF = APIConfig().parse(args.config_file)
  21
  22 _installers = {'fuel', 'apex', 'compass', 'joid'}
  23
  24
  25 class KibanaDashboard(dict):
  26     def __init__(self, project_name, case_name, family, installer, pod, scenarios, visualization):
  27         super(KibanaDashboard, self).__init__()
  28         self.project_name = project_name
  29         self.case_name = case_name
  30         self.family = family
  31         self.installer = installer
  32         self.pod = pod
  33         self.scenarios = scenarios
  34         self.visualization = visualization
  35         self._visualization_title = None
  36         self._kibana_visualizations = []
  37         self._kibana_dashboard = None
  38         self._create_visualizations()
  39         self._create()
  40
  41     def _create_visualizations(self):
  42         for scenario in self.scenarios:
  43             self._kibana_visualizations.append(KibanaVisualization(self.project_name,
  44                                                                    self.case_name,
  45                                                                    self.installer,
  46                                                                    self.pod,
  47                                                                    scenario,
  48                                                                    self.visualization))
  49
  50         self._visualization_title = self._kibana_visualizations[0].vis_state_title
  51
  52     def _publish_visualizations(self):
  53         for visualization in self._kibana_visualizations:
  54             url = urlparse.urljoin(base_elastic_url, '/.kibana/visualization/{}'.format(visualization.id))
  55             logger.debug("publishing visualization '{}'".format(url))
  56             shared_utils.publish_json(visualization, es_creds, url)
  57
  58     def _construct_panels(self):
  59         size_x = 6
  60         size_y = 3
  61         max_columns = 7
  62         column = 1
  63         row = 1
  64         panel_index = 1
  65         panels_json = []
  66         for visualization in self._kibana_visualizations:
  67             panels_json.append({
  68                 "id": visualization.id,
  69                 "type": 'visualization',
  70                 "panelIndex": panel_index,
  71                 "size_x": size_x,
  72                 "size_y": size_y,
  73                 "col": column,
  74                 "row": row
  75             })
  76             panel_index += 1
  77             column += size_x
  78             if column > max_columns:
  79                 column = 1
  80                 row += size_y
  81         return json.dumps(panels_json, separators=(',', ':'))
  82
  83     def _create(self):
  84         self['title'] = '{} {} {} {} {}'.format(self.project_name,
  85                                                 self.case_name,
  86                                                 self.installer,
  87                                                 self._visualization_title,
  88                                                 self.pod)
  89         self.id = self['title'].replace(' ', '-').replace('/', '-')
  90
  91         self['hits'] = 0
  92         self['description'] = "Kibana dashboard for project_name '{}', case_name '{}', installer '{}', data '{}' and" \
  93                               " pod '{}'".format(self.project_name,
  94                                                  self.case_name,
  95                                                  self.installer,
  96                                                  self._visualization_title,
  97                                                  self.pod)
  98         self['panelsJSON'] = self._construct_panels()
  99         self['optionsJSON'] = json.dumps({
 100             "darkTheme": False
 101         },
 102             separators=(',', ':'))
 103         self['uiStateJSON'] = "{}"
 104         self['scenario'] = 1
 105         self['timeRestore'] = False
 106         self['kibanaSavedObjectMeta'] = {
 107             'searchSourceJSON': json.dumps({
 108                 "filter": [
 109                     {
 110                         "query": {
 111                             "query_string": {
 112                                 "query": "*",
 113                                 "analyze_wildcard": True
 114                             }
 115                         }
 116                     }
 117                 ]
 118             },
 119                 separators=(',', ':'))
 120         }
 121
 122         label = self.case_name
 123         if 'label' in self.visualization:
 124             label += " %s" % self.visualization.get('label')
 125         label += " %s" % self.visualization.get('name')
 126         self['metadata'] = {
 127             "label": label,
 128             "test_family": self.family
 129         }
 130
 131     def _publish(self):
 132         url = urlparse.urljoin(base_elastic_url, '/.kibana/dashboard/{}'.format(self.id))
 133         logger.debug("publishing dashboard '{}'".format(url))
 134         shared_utils.publish_json(self, es_creds, url)
 135
 136     def publish(self):
 137         self._publish_visualizations()
 138         self._publish()
 139
 140
 141 class KibanaSearchSourceJSON(dict):
 142     """
 143     "filter": [
 144                     {"match": {"installer": {"query": installer, "type": "phrase"}}},
 145                     {"match": {"project_name": {"query": project_name, "type": "phrase"}}},
 146                     {"match": {"case_name": {"query": case_name, "type": "phrase"}}}
 147                 ]
 148     """
 149
 150     def __init__(self, project_name, case_name, installer, pod, scenario):
 151         super(KibanaSearchSourceJSON, self).__init__()
 152         self["filter"] = [
 153             {"match": {"project_name": {"query": project_name, "type": "phrase"}}},
 154             {"match": {"case_name": {"query": case_name, "type": "phrase"}}},
 155             {"match": {"installer": {"query": installer, "type": "phrase"}}},
 156             {"match": {"scenario": {"query": scenario, "type": "phrase"}}}
 157         ]
 158         if pod != 'all':
 159             self["filter"].append({"match": {"pod_name": {"query": pod, "type": "phrase"}}})
 160
 161
 162 class VisualizationState(dict):
 163     def __init__(self, visualization):
 164         super(VisualizationState, self).__init__()
 165         name = visualization.get('name')
 166         fields = visualization.get('fields')
 167
 168         if name == 'tests_failures':
 169             mode = 'grouped'
 170             metric_type = 'sum'
 171             self['type'] = 'histogram'
 172         else:
 173             # duration or success_percentage
 174             mode = 'stacked'
 175             metric_type = 'avg'
 176             self['type'] = 'line'
 177
 178         self['params'] = {
 179             "shareYAxis": True,
 180             "addTooltip": True,
 181             "addLegend": True,
 182             "smoothLines": False,
 183             "scale": "linear",
 184             "interpolate": "linear",
 185             "mode": mode,
 186             "times": [],
 187             "addTimeMarker": False,
 188             "defaultYExtents": False,
 189             "setYExtents": False,
 190             "yAxis": {}
 191         }
 192
 193         self['aggs'] = []
 194
 195         i = 1
 196         for field in fields:
 197             self['aggs'].append({
 198                 "id": str(i),
 199                 "type": metric_type,
 200                 "schema": "metric",
 201                 "params": {
 202                     "field": field.get('field')
 203                 }
 204             })
 205             i += 1
 206
 207         self['aggs'].append({
 208                 "id": str(i),
 209                 "type": 'date_histogram',
 210                 "schema": "segment",
 211                 "params": {
 212                     "field": "start_date",
 213                     "interval": "auto",
 214                     "customInterval": "2h",
 215                     "min_doc_count": 1,
 216                     "extended_bounds": {}
 217                 }
 218             })
 219
 220         self['listeners'] = {}
 221         self['title'] = ' '.join(['{} {}'.format(x['type'], x['params']['field']) for x in self['aggs']
 222                                   if x['schema'] == 'metric'])
 223
 224
 225 class KibanaVisualization(dict):
 226     def __init__(self, project_name, case_name, installer, pod, scenario, visualization):
 227         """
 228         We need two things
 229         1. filter created from
 230             project_name
 231             case_name
 232             installer
 233             pod
 234             scenario
 235         2. visualization state
 236             field for y axis (metric) with type (avg, sum, etc.)
 237             field for x axis (segment) with type (date_histogram)
 238
 239         :return:
 240         """
 241         super(KibanaVisualization, self).__init__()
 242         vis_state = VisualizationState(visualization)
 243         self.vis_state_title = vis_state['title']
 244         self['title'] = '{} {} {} {} {} {}'.format(project_name,
 245                                                    case_name,
 246                                                    self.vis_state_title,
 247                                                    installer,
 248                                                    pod,
 249                                                    scenario)
 250         self.id = self['title'].replace(' ', '-').replace('/', '-')
 251         self['visState'] = json.dumps(vis_state, separators=(',', ':'))
 252         self['uiStateJSON'] = "{}"
 253         self['description'] = "Kibana visualization for project_name '{}', case_name '{}', data '{}', installer '{}'," \
 254                               " pod '{}' and scenario '{}'".format(project_name,
 255                                                                   case_name,
 256                                                                   self.vis_state_title,
 257                                                                   installer,
 258                                                                   pod,
 259                                                                   scenario)
 260         self['scenario'] = 1
 261         self['kibanaSavedObjectMeta'] = {"searchSourceJSON": json.dumps(KibanaSearchSourceJSON(project_name,
 262                                                                                                case_name,
 263                                                                                                installer,
 264                                                                                                pod,
 265                                                                                                scenario),
 266                                                                         separators=(',', ':'))}
 267
 268
 269 def _get_pods_and_scenarios(project_name, case_name, installer):
 270     query_json = json.JSONEncoder().encode({
 271         "query": {
 272             "bool": {
 273                 "must": [
 274                     {"match_all": {}}
 275                 ],
 276                 "filter": [
 277                     {"match": {"installer": {"query": installer, "type": "phrase"}}},
 278                     {"match": {"project_name": {"query": project_name, "type": "phrase"}}},
 279                     {"match": {"case_name": {"query": case_name, "type": "phrase"}}}
 280                 ]
 281             }
 282         }
 283     })
 284
 285     elastic_data = shared_utils.get_elastic_docs(urlparse.urljoin(base_elastic_url, '/test_results/mongo2elastic'),
 286                                                  es_creds, query_json)
 287
 288     pods_and_scenarios = {}
 289
 290     for data in elastic_data:
 291         pod = data['pod_name']
 292         if pod in pods_and_scenarios:
 293             pods_and_scenarios[pod].add(data['scenario'])
 294         else:
 295             pods_and_scenarios[pod] = {data['scenario']}
 296
 297         if 'all' in pods_and_scenarios:
 298             pods_and_scenarios['all'].add(data['scenario'])
 299         else:
 300             pods_and_scenarios['all'] = {data['scenario']}
 301
 302     return pods_and_scenarios
 303
 304
 305 def construct_dashboards():
 306     """
 307     iterate over testcase and installer
 308     1. get available pods for each testcase/installer pair
 309     2. get available scenario for each testcase/installer/pod tuple
 310     3. construct KibanaInput and append
 311
 312     :return: list of KibanaDashboards
 313     """
 314     kibana_dashboards = []
 315     for project, case_dicts in testcases_parser.testcases_yaml.items():
 316         for case in case_dicts:
 317             case_name = case.get('name')
 318             visualizations = case.get('visualizations')
 319             family = case.get('test_family')
 320             for installer in _installers:
 321                 pods_and_scenarios = _get_pods_and_scenarios(project, case_name, installer)
 322                 for visualization in visualizations:
 323                     for pod, scenarios in pods_and_scenarios.iteritems():
 324                         kibana_dashboards.append(KibanaDashboard(project,
 325                                                                  case_name,
 326                                                                  family,
 327                                                                  installer,
 328                                                                  pod,
 329                                                                  scenarios,
 330                                                                  visualization))
 331     return kibana_dashboards
 332
 333
 334 def generate_js_inputs(js_file_path, kibana_url, dashboards):
 335     js_dict = {}
 336     for dashboard in dashboards:
 337         dashboard_meta = dashboard['metadata']
 338         test_family = dashboard_meta['test_family']
 339         test_label = dashboard_meta['label']
 340
 341         if test_family not in js_dict:
 342             js_dict[test_family] = {}
 343
 344         js_test_family = js_dict[test_family]
 345
 346         if test_label not in js_test_family:
 347             js_test_family[test_label] = {}
 348
 349         js_test_label = js_test_family[test_label]
 350
 351         if dashboard.installer not in js_test_label:
 352             js_test_label[dashboard.installer] = {}
 353
 354         js_installer = js_test_label[dashboard.installer]
 355         js_installer[dashboard.pod] = kibana_url + '#/dashboard/' + dashboard.id
 356
 357     with open(js_file_path, 'w+') as js_file_fdesc:
 358         js_file_fdesc.write('var kibana_dashboard_links = ')
 359         js_file_fdesc.write(str(js_dict).replace("u'", "'"))
 360
 361
 362 if __name__ == '__main__':
 363     base_elastic_url = CONF.elastic_url
 364     generate_inputs = CONF.is_js
 365     input_file_path = CONF.js_path
 366     kibana_url = CONF.kibana_url
 367     es_creds = CONF.elastic_creds
 368
 369     dashboards = construct_dashboards()
 370
 371     for kibana_dashboard in dashboards:
 372         kibana_dashboard.publish()
 373
 374     if generate_inputs:
 375         generate_js_inputs(input_file_path, kibana_url, dashboards)