utils/test/dashboard/dashboard/elastic2kibana/main.py

   1 #! /usr/bin/env python
   2 import json
   3 import urlparse
   4
   5 import argparse
   6
   7 from common import logger_utils, elastic_access
   8 from conf import testcases
   9 from conf.config import APIConfig
  10
  11 logger = logger_utils.DashboardLogger('elastic2kibana').get
  12
  13 parser = argparse.ArgumentParser()
  14 parser.add_argument("-c", "--config-file",
  15                     dest='config_file',
  16                     help="Config file location")
  17
  18 args = parser.parse_args()
  19 CONF = APIConfig().parse(args.config_file)
  20 base_elastic_url = CONF.elastic_url
  21 generate_inputs = CONF.is_js
  22 input_file_path = CONF.js_path
  23 kibana_url = CONF.kibana_url
  24 es_creds = CONF.elastic_creds
  25
  26 _installers = {'fuel', 'apex', 'compass', 'joid'}
  27
  28
  29 class KibanaDashboard(dict):
  30     def __init__(self, project_name, case_name, family, installer, pod, scenarios, visualization):
  31         super(KibanaDashboard, self).__init__()
  32         self.project_name = project_name
  33         self.case_name = case_name
  34         self.family = family
  35         self.installer = installer
  36         self.pod = pod
  37         self.scenarios = scenarios
  38         self.visualization = visualization
  39         self._visualization_title = None
  40         self._kibana_visualizations = []
  41         self._kibana_dashboard = None
  42         self._create_visualizations()
  43         self._create()
  44
  45     def _create_visualizations(self):
  46         for scenario in self.scenarios:
  47             self._kibana_visualizations.append(KibanaVisualization(self.project_name,
  48                                                                    self.case_name,
  49                                                                    self.installer,
  50                                                                    self.pod,
  51                                                                    scenario,
  52                                                                    self.visualization))
  53
  54         self._visualization_title = self._kibana_visualizations[0].vis_state_title
  55
  56     def _publish_visualizations(self):
  57         for visualization in self._kibana_visualizations:
  58             url = urlparse.urljoin(base_elastic_url, '/.kibana/visualization/{}'.format(visualization.id))
  59             logger.debug("publishing visualization '{}'".format(url))
  60             elastic_access.publish_json(visualization, es_creds, url)
  61
  62     def _construct_panels(self):
  63         size_x = 6
  64         size_y = 3
  65         max_columns = 7
  66         column = 1
  67         row = 1
  68         panel_index = 1
  69         panels_json = []
  70         for visualization in self._kibana_visualizations:
  71             panels_json.append({
  72                 "id": visualization.id,
  73                 "type": 'visualization',
  74                 "panelIndex": panel_index,
  75                 "size_x": size_x,
  76                 "size_y": size_y,
  77                 "col": column,
  78                 "row": row
  79             })
  80             panel_index += 1
  81             column += size_x
  82             if column > max_columns:
  83                 column = 1
  84                 row += size_y
  85         return json.dumps(panels_json, separators=(',', ':'))
  86
  87     def _create(self):
  88         self['title'] = '{} {} {} {} {}'.format(self.project_name,
  89                                                 self.case_name,
  90                                                 self.installer,
  91                                                 self._visualization_title,
  92                                                 self.pod)
  93         self.id = self['title'].replace(' ', '-').replace('/', '-')
  94
  95         self['hits'] = 0
  96         self['description'] = "Kibana dashboard for project_name '{}', case_name '{}', installer '{}', data '{}' and" \
  97                               " pod '{}'".format(self.project_name,
  98                                                  self.case_name,
  99                                                  self.installer,
 100                                                  self._visualization_title,
 101                                                  self.pod)
 102         self['panelsJSON'] = self._construct_panels()
 103         self['optionsJSON'] = json.dumps({
 104             "darkTheme": False
 105         },
 106             separators=(',', ':'))
 107         self['uiStateJSON'] = "{}"
 108         self['scenario'] = 1
 109         self['timeRestore'] = False
 110         self['kibanaSavedObjectMeta'] = {
 111             'searchSourceJSON': json.dumps({
 112                 "filter": [
 113                     {
 114                         "query": {
 115                             "query_string": {
 116                                 "query": "*",
 117                                 "analyze_wildcard": True
 118                             }
 119                         }
 120                     }
 121                 ]
 122             },
 123                 separators=(',', ':'))
 124         }
 125
 126         label = self.case_name
 127         if 'label' in self.visualization:
 128             label += " %s" % self.visualization.get('label')
 129         label += " %s" % self.visualization.get('name')
 130         self['metadata'] = {
 131             "label": label,
 132             "test_family": self.family
 133         }
 134
 135     def _publish(self):
 136         url = urlparse.urljoin(base_elastic_url, '/.kibana/dashboard/{}'.format(self.id))
 137         logger.debug("publishing dashboard '{}'".format(url))
 138         elastic_access.publish_json(self, es_creds, url)
 139
 140     def publish(self):
 141         self._publish_visualizations()
 142         self._publish()
 143
 144
 145 class KibanaSearchSourceJSON(dict):
 146     """
 147     "filter": [
 148                     {"match": {"installer": {"query": installer, "type": "phrase"}}},
 149                     {"match": {"project_name": {"query": project_name, "type": "phrase"}}},
 150                     {"match": {"case_name": {"query": case_name, "type": "phrase"}}}
 151                 ]
 152     """
 153
 154     def __init__(self, project_name, case_name, installer, pod, scenario):
 155         super(KibanaSearchSourceJSON, self).__init__()
 156         self["filter"] = [
 157             {"match": {"project_name": {"query": project_name, "type": "phrase"}}},
 158             {"match": {"case_name": {"query": case_name, "type": "phrase"}}},
 159             {"match": {"installer": {"query": installer, "type": "phrase"}}},
 160             {"match": {"scenario": {"query": scenario, "type": "phrase"}}}
 161         ]
 162         if pod != 'all':
 163             self["filter"].append({"match": {"pod_name": {"query": pod, "type": "phrase"}}})
 164
 165
 166 class VisualizationState(dict):
 167     def __init__(self, visualization):
 168         super(VisualizationState, self).__init__()
 169         name = visualization.get('name')
 170         fields = visualization.get('fields')
 171
 172         if name == 'tests_failures':
 173             mode = 'grouped'
 174             metric_type = 'sum'
 175             self['type'] = 'histogram'
 176         else:
 177             # duration or success_percentage
 178             mode = 'stacked'
 179             metric_type = 'avg'
 180             self['type'] = 'line'
 181
 182         self['params'] = {
 183             "shareYAxis": True,
 184             "addTooltip": True,
 185             "addLegend": True,
 186             "smoothLines": False,
 187             "scale": "linear",
 188             "interpolate": "linear",
 189             "mode": mode,
 190             "times": [],
 191             "addTimeMarker": False,
 192             "defaultYExtents": False,
 193             "setYExtents": False,
 194             "yAxis": {}
 195         }
 196
 197         self['aggs'] = []
 198
 199         i = 1
 200         for field in fields:
 201             self['aggs'].append({
 202                 "id": str(i),
 203                 "type": metric_type,
 204                 "schema": "metric",
 205                 "params": {
 206                     "field": field.get('field')
 207                 }
 208             })
 209             i += 1
 210
 211         self['aggs'].append({
 212                 "id": str(i),
 213                 "type": 'date_histogram',
 214                 "schema": "segment",
 215                 "params": {
 216                     "field": "start_date",
 217                     "interval": "auto",
 218                     "customInterval": "2h",
 219                     "min_doc_count": 1,
 220                     "extended_bounds": {}
 221                 }
 222             })
 223
 224         self['listeners'] = {}
 225         self['title'] = ' '.join(['{} {}'.format(x['type'], x['params']['field']) for x in self['aggs']
 226                                   if x['schema'] == 'metric'])
 227
 228
 229 class KibanaVisualization(dict):
 230     def __init__(self, project_name, case_name, installer, pod, scenario, visualization):
 231         """
 232         We need two things
 233         1. filter created from
 234             project_name
 235             case_name
 236             installer
 237             pod
 238             scenario
 239         2. visualization state
 240             field for y axis (metric) with type (avg, sum, etc.)
 241             field for x axis (segment) with type (date_histogram)
 242
 243         :return:
 244         """
 245         super(KibanaVisualization, self).__init__()
 246         vis_state = VisualizationState(visualization)
 247         self.vis_state_title = vis_state['title']
 248         self['title'] = '{} {} {} {} {} {}'.format(project_name,
 249                                                    case_name,
 250                                                    self.vis_state_title,
 251                                                    installer,
 252                                                    pod,
 253                                                    scenario)
 254         self.id = self['title'].replace(' ', '-').replace('/', '-')
 255         self['visState'] = json.dumps(vis_state, separators=(',', ':'))
 256         self['uiStateJSON'] = "{}"
 257         self['description'] = "Kibana visualization for project_name '{}', case_name '{}', data '{}', installer '{}'," \
 258                               " pod '{}' and scenario '{}'".format(project_name,
 259                                                                   case_name,
 260                                                                   self.vis_state_title,
 261                                                                   installer,
 262                                                                   pod,
 263                                                                   scenario)
 264         self['scenario'] = 1
 265         self['kibanaSavedObjectMeta'] = {"searchSourceJSON": json.dumps(KibanaSearchSourceJSON(project_name,
 266                                                                                                case_name,
 267                                                                                                installer,
 268                                                                                                pod,
 269                                                                                                scenario),
 270                                                                         separators=(',', ':'))}
 271
 272
 273 def _get_pods_and_scenarios(project_name, case_name, installer):
 274     query_json = json.JSONEncoder().encode({
 275         "query": {
 276             "bool": {
 277                 "must": [
 278                     {"match_all": {}}
 279                 ],
 280                 "filter": [
 281                     {"match": {"installer": {"query": installer, "type": "phrase"}}},
 282                     {"match": {"project_name": {"query": project_name, "type": "phrase"}}},
 283                     {"match": {"case_name": {"query": case_name, "type": "phrase"}}}
 284                 ]
 285             }
 286         }
 287     })
 288
 289     elastic_data = elastic_access.get_elastic_docs(urlparse.urljoin(base_elastic_url, '/test_results/mongo2elastic'),
 290                                                    es_creds, query_json)
 291
 292     pods_and_scenarios = {}
 293
 294     for data in elastic_data:
 295         pod = data['pod_name']
 296         if pod in pods_and_scenarios:
 297             pods_and_scenarios[pod].add(data['scenario'])
 298         else:
 299             pods_and_scenarios[pod] = {data['scenario']}
 300
 301         if 'all' in pods_and_scenarios:
 302             pods_and_scenarios['all'].add(data['scenario'])
 303         else:
 304             pods_and_scenarios['all'] = {data['scenario']}
 305
 306     return pods_and_scenarios
 307
 308
 309 def construct_dashboards():
 310     """
 311     iterate over testcase and installer
 312     1. get available pods for each testcase/installer pair
 313     2. get available scenario for each testcase/installer/pod tuple
 314     3. construct KibanaInput and append
 315
 316     :return: list of KibanaDashboards
 317     """
 318     kibana_dashboards = []
 319     for project, case_dicts in testcases.testcases_yaml.items():
 320         for case in case_dicts:
 321             case_name = case.get('name')
 322             visualizations = case.get('visualizations')
 323             family = case.get('test_family')
 324             for installer in _installers:
 325                 pods_and_scenarios = _get_pods_and_scenarios(project, case_name, installer)
 326                 for visualization in visualizations:
 327                     for pod, scenarios in pods_and_scenarios.iteritems():
 328                         kibana_dashboards.append(KibanaDashboard(project,
 329                                                                  case_name,
 330                                                                  family,
 331                                                                  installer,
 332                                                                  pod,
 333                                                                  scenarios,
 334                                                                  visualization))
 335     return kibana_dashboards
 336
 337
 338 def generate_js_inputs(js_file_path, kibana_url, dashboards):
 339     js_dict = {}
 340     for dashboard in dashboards:
 341         dashboard_meta = dashboard['metadata']
 342         test_family = dashboard_meta['test_family']
 343         test_label = dashboard_meta['label']
 344
 345         if test_family not in js_dict:
 346             js_dict[test_family] = {}
 347
 348         js_test_family = js_dict[test_family]
 349
 350         if test_label not in js_test_family:
 351             js_test_family[test_label] = {}
 352
 353         js_test_label = js_test_family[test_label]
 354
 355         if dashboard.installer not in js_test_label:
 356             js_test_label[dashboard.installer] = {}
 357
 358         js_installer = js_test_label[dashboard.installer]
 359         js_installer[dashboard.pod] = kibana_url + '#/dashboard/' + dashboard.id
 360
 361     with open(js_file_path, 'w+') as js_file_fdesc:
 362         js_file_fdesc.write('var kibana_dashboard_links = ')
 363         js_file_fdesc.write(str(js_dict).replace("u'", "'"))
 364
 365
 366 def main():
 367     dashboards = construct_dashboards()
 368
 369     for kibana_dashboard in dashboards:
 370         kibana_dashboard.publish()
 371
 372     if generate_inputs:
 373         generate_js_inputs(input_file_path, kibana_url, dashboards)