utils/test/dashboard/dashboard/elastic2kibana/main.py

   1 #! /usr/bin/env python
   2 import json
   3 import urlparse
   4
   5 import argparse
   6 from jinja2 import PackageLoader, Environment
   7
   8 from common import elastic_access
   9 from common import logger_utils
  10 from conf import testcases
  11 from conf.config import APIConfig
  12
  13 logger = logger_utils.DashboardLogger('elastic2kibana').get
  14
  15 parser = argparse.ArgumentParser()
  16 parser.add_argument("-c", "--config-file",
  17                     dest='config_file',
  18                     help="Config file location")
  19
  20 args = parser.parse_args()
  21 CONF = APIConfig().parse(args.config_file)
  22 base_elastic_url = CONF.elastic_url
  23 generate_inputs = CONF.is_js
  24 input_file_path = CONF.js_path
  25 kibana_url = CONF.kibana_url
  26 es_creds = CONF.elastic_creds
  27
  28 _installers = {'fuel', 'apex', 'compass', 'joid'}
  29
  30 env = Environment(loader=PackageLoader('elastic2kibana', 'templates'))
  31 env.filters['jsonify'] = json.dumps
  32
  33
  34 class KibanaDashboard(dict):
  35     def __init__(self, project_name, case_name, family, installer, pod, scenarios, visualization):
  36         super(KibanaDashboard, self).__init__()
  37         self.project_name = project_name
  38         self.case_name = case_name
  39         self.family = family
  40         self.installer = installer
  41         self.pod = pod
  42         self.scenarios = scenarios
  43         self.visualization = visualization
  44         self._visualization_title = None
  45         self._kibana_visualizations = []
  46         self._kibana_dashboard = None
  47         self._create_visualizations()
  48         self._create()
  49
  50     def _create_visualizations(self):
  51         for scenario in self.scenarios:
  52             self._kibana_visualizations.append(Visualization(self.project_name,
  53                                                              self.case_name,
  54                                                              self.installer,
  55                                                              self.pod,
  56                                                              scenario,
  57                                                              self.visualization))
  58
  59         self._visualization_title = self._kibana_visualizations[0].vis_state_title
  60
  61     def _publish_visualizations(self):
  62         for visualization in self._kibana_visualizations:
  63             url = urlparse.urljoin(base_elastic_url, '/.kibana/visualization/{}'.format(visualization.id))
  64             logger.debug("publishing visualization '{}'".format(url))
  65             # logger.error("_publish_visualization: %s" % visualization)
  66             elastic_access.publish_docs(url, es_creds, visualization)
  67
  68     def _construct_panels(self):
  69         size_x = 6
  70         size_y = 3
  71         max_columns = 7
  72         column = 1
  73         row = 1
  74         panel_index = 1
  75         panels_json = []
  76         for visualization in self._kibana_visualizations:
  77             panels_json.append({
  78                 "id": visualization.id,
  79                 "type": 'visualization',
  80                 "panelIndex": panel_index,
  81                 "size_x": size_x,
  82                 "size_y": size_y,
  83                 "col": column,
  84                 "row": row
  85             })
  86             panel_index += 1
  87             column += size_x
  88             if column > max_columns:
  89                 column = 1
  90                 row += size_y
  91         return json.dumps(panels_json, separators=(',', ':'))
  92
  93     def _create(self):
  94         self['title'] = '{} {} {} {} {}'.format(self.project_name,
  95                                                 self.case_name,
  96                                                 self.installer,
  97                                                 self._visualization_title,
  98                                                 self.pod)
  99         self.id = self['title'].replace(' ', '-').replace('/', '-')
 100
 101         self['hits'] = 0
 102         self['description'] = "Kibana dashboard for project_name '{}', case_name '{}', installer '{}', data '{}' and" \
 103                               " pod '{}'".format(self.project_name,
 104                                                  self.case_name,
 105                                                  self.installer,
 106                                                  self._visualization_title,
 107                                                  self.pod)
 108         self['panelsJSON'] = self._construct_panels()
 109         self['optionsJSON'] = json.dumps({
 110             "darkTheme": False
 111         },
 112             separators=(',', ':'))
 113         self['uiStateJSON'] = "{}"
 114         self['scenario'] = 1
 115         self['timeRestore'] = False
 116         self['kibanaSavedObjectMeta'] = {
 117             'searchSourceJSON': json.dumps({
 118                 "filter": [
 119                     {
 120                         "query": {
 121                             "query_string": {
 122                                 "query": "*",
 123                                 "analyze_wildcard": True
 124                             }
 125                         }
 126                     }
 127                 ]
 128             },
 129                 separators=(',', ':'))
 130         }
 131
 132         label = self.case_name
 133         if 'label' in self.visualization:
 134             label += " %s" % self.visualization.get('label')
 135         label += " %s" % self.visualization.get('name')
 136         self['metadata'] = {
 137             "label": label,
 138             "test_family": self.family
 139         }
 140
 141     def _publish(self):
 142         url = urlparse.urljoin(base_elastic_url, '/.kibana/dashboard/{}'.format(self.id))
 143         logger.debug("publishing dashboard '{}'".format(url))
 144         elastic_access.publish_docs(url, es_creds, self)
 145
 146     def publish(self):
 147         self._publish_visualizations()
 148         self._publish()
 149
 150
 151 class VisStateBuilder(object):
 152     def __init__(self, visualization):
 153         super(VisStateBuilder, self).__init__()
 154         self.visualization = visualization
 155
 156     def build(self):
 157         name = self.visualization.get('name')
 158         fields = self.visualization.get('fields')
 159
 160         aggs = []
 161         index = 1
 162         for field in fields:
 163             aggs.append({
 164                 "id": index,
 165                 "field": field.get("field")
 166             })
 167             index += 1
 168
 169         template = env.get_template('{}.json'.format(name))
 170         vis = template.render(aggs=aggs)
 171         return json.loads(vis)
 172
 173
 174 class Visualization(object):
 175     def __init__(self, project_name, case_name, installer, pod, scenario, visualization):
 176         """
 177         We need two things
 178         1. filter created from
 179             project_name
 180             case_name
 181             installer
 182             pod
 183             scenario
 184         2. visualization state
 185             field for y axis (metric) with type (avg, sum, etc.)
 186             field for x axis (segment) with type (date_histogram)
 187
 188         :return:
 189         """
 190         super(Visualization, self).__init__()
 191         visState = VisStateBuilder(visualization).build()
 192         self.vis_state_title = visState['title']
 193
 194         vis = {
 195             "visState": json.dumps(visState),
 196             "filters": {
 197                 "project_name": project_name,
 198                 "case_name": case_name,
 199                 "installer": installer,
 200                 "metric": self.vis_state_title,
 201                 "pod_name": pod,
 202                 "scenario": scenario
 203             }
 204         }
 205
 206         template = env.get_template('visualization.json')
 207
 208         self.visualization = json.loads(template.render(vis=vis))
 209         self._dumps(['visState', 'description', 'uiStateJSON'])
 210         self._dumps_2deeps('kibanaSavedObjectMeta', 'searchSourceJSON')
 211         self.id = self.visualization['title'].replace(' ', '-').replace('/', '-')
 212
 213     def _dumps(self, items):
 214         for key in items:
 215             self.visualization[key] = json.dumps(self.visualization[key])
 216
 217     def _dumps_2deeps(self, key1, key2):
 218         self.visualization[key1][key2] = json.dumps(self.visualization[key1][key2])
 219
 220
 221 def _get_pods_and_scenarios(project_name, case_name, installer):
 222     query_json = json.JSONEncoder().encode({
 223         "query": {
 224             "bool": {
 225                 "must": [
 226                     {"match_all": {}}
 227                 ],
 228                 "filter": [
 229                     {"match": {"installer": {"query": installer, "type": "phrase"}}},
 230                     {"match": {"project_name": {"query": project_name, "type": "phrase"}}},
 231                     {"match": {"case_name": {"query": case_name, "type": "phrase"}}}
 232                 ]
 233             }
 234         }
 235     })
 236
 237     elastic_data = elastic_access.get_docs(urlparse.urljoin(base_elastic_url, '/test_results/mongo2elastic'),
 238                                                    es_creds, query_json)
 239
 240     pods_and_scenarios = {}
 241
 242     for data in elastic_data:
 243         pod = data['pod_name']
 244         if pod in pods_and_scenarios:
 245             pods_and_scenarios[pod].add(data['scenario'])
 246         else:
 247             pods_and_scenarios[pod] = {data['scenario']}
 248
 249         if 'all' in pods_and_scenarios:
 250             pods_and_scenarios['all'].add(data['scenario'])
 251         else:
 252             pods_and_scenarios['all'] = {data['scenario']}
 253
 254     return pods_and_scenarios
 255
 256
 257 def construct_dashboards():
 258     """
 259     iterate over testcase and installer
 260     1. get available pods for each testcase/installer pair
 261     2. get available scenario for each testcase/installer/pod tuple
 262     3. construct KibanaInput and append
 263
 264     :return: list of KibanaDashboards
 265     """
 266     kibana_dashboards = []
 267     for project, case_dicts in testcases.testcases_yaml.items():
 268         for case in case_dicts:
 269             case_name = case.get('name')
 270             visualizations = case.get('visualizations')
 271             family = case.get('test_family')
 272             for installer in _installers:
 273                 pods_and_scenarios = _get_pods_and_scenarios(project, case_name, installer)
 274                 for visualization in visualizations:
 275                     for pod, scenarios in pods_and_scenarios.iteritems():
 276                         kibana_dashboards.append(KibanaDashboard(project,
 277                                                                  case_name,
 278                                                                  family,
 279                                                                  installer,
 280                                                                  pod,
 281                                                                  scenarios,
 282                                                                  visualization))
 283     return kibana_dashboards
 284
 285
 286 def generate_js_inputs(js_file_path, kibana_url, dashboards):
 287     js_dict = {}
 288     for dashboard in dashboards:
 289         dashboard_meta = dashboard['metadata']
 290         test_family = dashboard_meta['test_family']
 291         test_label = dashboard_meta['label']
 292
 293         if test_family not in js_dict:
 294             js_dict[test_family] = {}
 295
 296         js_test_family = js_dict[test_family]
 297
 298         if test_label not in js_test_family:
 299             js_test_family[test_label] = {}
 300
 301         js_test_label = js_test_family[test_label]
 302
 303         if dashboard.installer not in js_test_label:
 304             js_test_label[dashboard.installer] = {}
 305
 306         js_installer = js_test_label[dashboard.installer]
 307         js_installer[dashboard.pod] = kibana_url + '#/dashboard/' + dashboard.id
 308
 309     with open(js_file_path, 'w+') as js_file_fdesc:
 310         js_file_fdesc.write('var kibana_dashboard_links = ')
 311         js_file_fdesc.write(str(js_dict).replace("u'", "'"))
 312
 313
 314 def main():
 315     dashboards = construct_dashboards()
 316
 317     for kibana_dashboard in dashboards:
 318         kibana_dashboard.publish()
 319
 320     if generate_inputs:
 321         generate_js_inputs(input_file_path, kibana_url, dashboards)