utils/test/dashboard/dashboard/elastic2kibana/main.py

   1 #! /usr/bin/env python
   2 import json
   3 import urlparse
   4
   5 import argparse
   6 from jinja2 import PackageLoader, Environment
   7
   8 from common import elastic_access
   9 from common import logger_utils
  10 from conf import testcases
  11 from conf.config import APIConfig
  12
  13 logger = logger_utils.DashboardLogger('elastic2kibana').get
  14
  15 parser = argparse.ArgumentParser()
  16 parser.add_argument("-c", "--config-file",
  17                     dest='config_file',
  18                     help="Config file location")
  19
  20 args = parser.parse_args()
  21 CONF = APIConfig().parse(args.config_file)
  22 base_elastic_url = CONF.elastic_url
  23 generate_inputs = CONF.is_js
  24 input_file_path = CONF.js_path
  25 kibana_url = CONF.kibana_url
  26 es_creds = CONF.elastic_creds
  27
  28 _installers = {'fuel', 'apex', 'compass', 'joid'}
  29
  30
  31 class KibanaDashboard(dict):
  32     def __init__(self, project_name, case_name, family, installer, pod, scenarios, visualization):
  33         super(KibanaDashboard, self).__init__()
  34         self.project_name = project_name
  35         self.case_name = case_name
  36         self.family = family
  37         self.installer = installer
  38         self.pod = pod
  39         self.scenarios = scenarios
  40         self.visualization = visualization
  41         self._visualization_title = None
  42         self._kibana_visualizations = []
  43         self._kibana_dashboard = None
  44         self._create_visualizations()
  45         self._create()
  46
  47     def _create_visualizations(self):
  48         for scenario in self.scenarios:
  49             self._kibana_visualizations.append(KibanaVisualization(self.project_name,
  50                                                                    self.case_name,
  51                                                                    self.installer,
  52                                                                    self.pod,
  53                                                                    scenario,
  54                                                                    self.visualization))
  55
  56         self._visualization_title = self._kibana_visualizations[0].vis_title
  57
  58     def _publish_visualizations(self):
  59         for visualization in self._kibana_visualizations:
  60             url = urlparse.urljoin(base_elastic_url, '/.kibana/visualization/{}'.format(visualization.id))
  61             logger.debug("publishing visualization '{}'".format(url))
  62             # logger.error("_publish_visualization: %s" % visualization)
  63             elastic_access.publish_docs(visualization, es_creds, url)
  64
  65     def _construct_panels(self):
  66         size_x = 6
  67         size_y = 3
  68         max_columns = 7
  69         column = 1
  70         row = 1
  71         panel_index = 1
  72         panels_json = []
  73         for visualization in self._kibana_visualizations:
  74             panels_json.append({
  75                 "id": visualization.id,
  76                 "type": 'visualization',
  77                 "panelIndex": panel_index,
  78                 "size_x": size_x,
  79                 "size_y": size_y,
  80                 "col": column,
  81                 "row": row
  82             })
  83             panel_index += 1
  84             column += size_x
  85             if column > max_columns:
  86                 column = 1
  87                 row += size_y
  88         return json.dumps(panels_json, separators=(',', ':'))
  89
  90     def _create(self):
  91         self['title'] = '{} {} {} {} {}'.format(self.project_name,
  92                                                 self.case_name,
  93                                                 self.installer,
  94                                                 self._visualization_title,
  95                                                 self.pod)
  96         self.id = self['title'].replace(' ', '-').replace('/', '-')
  97
  98         self['hits'] = 0
  99         self['description'] = "Kibana dashboard for project_name '{}', case_name '{}', installer '{}', data '{}' and" \
 100                               " pod '{}'".format(self.project_name,
 101                                                  self.case_name,
 102                                                  self.installer,
 103                                                  self._visualization_title,
 104                                                  self.pod)
 105         self['panelsJSON'] = self._construct_panels()
 106         self['optionsJSON'] = json.dumps({
 107             "darkTheme": False
 108         },
 109             separators=(',', ':'))
 110         self['uiStateJSON'] = "{}"
 111         self['scenario'] = 1
 112         self['timeRestore'] = False
 113         self['kibanaSavedObjectMeta'] = {
 114             'searchSourceJSON': json.dumps({
 115                 "filter": [
 116                     {
 117                         "query": {
 118                             "query_string": {
 119                                 "query": "*",
 120                                 "analyze_wildcard": True
 121                             }
 122                         }
 123                     }
 124                 ]
 125             },
 126                 separators=(',', ':'))
 127         }
 128
 129         label = self.case_name
 130         if 'label' in self.visualization:
 131             label += " %s" % self.visualization.get('label')
 132         label += " %s" % self.visualization.get('name')
 133         self['metadata'] = {
 134             "label": label,
 135             "test_family": self.family
 136         }
 137
 138     def _publish(self):
 139         url = urlparse.urljoin(base_elastic_url, '/.kibana/dashboard/{}'.format(self.id))
 140         logger.debug("publishing dashboard '{}'".format(url))
 141         elastic_access.publish_docs(self, es_creds, url)
 142
 143     def publish(self):
 144         self._publish_visualizations()
 145         self._publish()
 146
 147
 148 class KibanaSearchSourceJSON(dict):
 149     """
 150     "filter": [
 151                     {"match": {"installer": {"query": installer, "type": "phrase"}}},
 152                     {"match": {"project_name": {"query": project_name, "type": "phrase"}}},
 153                     {"match": {"case_name": {"query": case_name, "type": "phrase"}}}
 154                 ]
 155     """
 156
 157     def __init__(self, project_name, case_name, installer, pod, scenario):
 158         super(KibanaSearchSourceJSON, self).__init__()
 159         self["filter"] = [
 160             {"match": {"project_name": {"query": project_name, "type": "phrase"}}},
 161             {"match": {"case_name": {"query": case_name, "type": "phrase"}}},
 162             {"match": {"installer": {"query": installer, "type": "phrase"}}},
 163             {"match": {"scenario": {"query": scenario, "type": "phrase"}}}
 164         ]
 165         if pod != 'all':
 166             self["filter"].append({"match": {"pod_name": {"query": pod, "type": "phrase"}}})
 167
 168
 169 class VisualizationBuilder(object):
 170     def __init__(self, visualization):
 171         super(VisualizationBuilder, self).__init__()
 172         self.visualization = visualization
 173
 174     def build(self):
 175         name = self.visualization.get('name')
 176         fields = self.visualization.get('fields')
 177
 178         aggs = []
 179         index = 1
 180         for field in fields:
 181             aggs.append({
 182                 "id": index,
 183                 "field": field.get("field")
 184             })
 185             index += 1
 186
 187         env = Environment(loader=PackageLoader('elastic2kibana', 'templates'))
 188         env.filters['jsonify'] = json.dumps
 189         template = env.get_template('{}.json'.format(name))
 190         vis = template.render(aggs=aggs)
 191         return json.loads(vis)
 192
 193
 194
 195 class KibanaVisualization(dict):
 196     def __init__(self, project_name, case_name, installer, pod, scenario, visualization):
 197         """
 198         We need two things
 199         1. filter created from
 200             project_name
 201             case_name
 202             installer
 203             pod
 204             scenario
 205         2. visualization state
 206             field for y axis (metric) with type (avg, sum, etc.)
 207             field for x axis (segment) with type (date_histogram)
 208
 209         :return:
 210         """
 211         super(KibanaVisualization, self).__init__()
 212         vis = VisualizationBuilder(visualization).build()
 213         self.vis_title = vis['title']
 214         self['title'] = '{} {} {} {} {} {}'.format(project_name,
 215                                                    case_name,
 216                                                    self.vis_title,
 217                                                    installer,
 218                                                    pod,
 219                                                    scenario)
 220         self.id = self['title'].replace(' ', '-').replace('/', '-')
 221         self['visState'] = json.dumps(vis, separators=(',', ':'))
 222         self['uiStateJSON'] = "{}"
 223         self['description'] = "Kibana visualization for project_name '{}', case_name '{}', metric '{}', installer '{}'," \
 224                               " pod '{}' and scenario '{}'".format(project_name,
 225                                                                    case_name,
 226                                                                    self.vis_title,
 227                                                                    installer,
 228                                                                    pod,
 229                                                                    scenario)
 230         self['scenario'] = 1
 231         self['kibanaSavedObjectMeta'] = {"searchSourceJSON": json.dumps(KibanaSearchSourceJSON(project_name,
 232                                                                                                case_name,
 233                                                                                                installer,
 234                                                                                                pod,
 235                                                                                                scenario),
 236                                                                         separators=(',', ':'))}
 237
 238
 239 def _get_pods_and_scenarios(project_name, case_name, installer):
 240     query_json = json.JSONEncoder().encode({
 241         "query": {
 242             "bool": {
 243                 "must": [
 244                     {"match_all": {}}
 245                 ],
 246                 "filter": [
 247                     {"match": {"installer": {"query": installer, "type": "phrase"}}},
 248                     {"match": {"project_name": {"query": project_name, "type": "phrase"}}},
 249                     {"match": {"case_name": {"query": case_name, "type": "phrase"}}}
 250                 ]
 251             }
 252         }
 253     })
 254
 255     elastic_data = elastic_access.get_docs(urlparse.urljoin(base_elastic_url, '/test_results/mongo2elastic'),
 256                                                    es_creds, query_json)
 257
 258     pods_and_scenarios = {}
 259
 260     for data in elastic_data:
 261         pod = data['pod_name']
 262         if pod in pods_and_scenarios:
 263             pods_and_scenarios[pod].add(data['scenario'])
 264         else:
 265             pods_and_scenarios[pod] = {data['scenario']}
 266
 267         if 'all' in pods_and_scenarios:
 268             pods_and_scenarios['all'].add(data['scenario'])
 269         else:
 270             pods_and_scenarios['all'] = {data['scenario']}
 271
 272     return pods_and_scenarios
 273
 274
 275 def construct_dashboards():
 276     """
 277     iterate over testcase and installer
 278     1. get available pods for each testcase/installer pair
 279     2. get available scenario for each testcase/installer/pod tuple
 280     3. construct KibanaInput and append
 281
 282     :return: list of KibanaDashboards
 283     """
 284     kibana_dashboards = []
 285     for project, case_dicts in testcases.testcases_yaml.items():
 286         for case in case_dicts:
 287             case_name = case.get('name')
 288             visualizations = case.get('visualizations')
 289             family = case.get('test_family')
 290             for installer in _installers:
 291                 pods_and_scenarios = _get_pods_and_scenarios(project, case_name, installer)
 292                 for visualization in visualizations:
 293                     for pod, scenarios in pods_and_scenarios.iteritems():
 294                         kibana_dashboards.append(KibanaDashboard(project,
 295                                                                  case_name,
 296                                                                  family,
 297                                                                  installer,
 298                                                                  pod,
 299                                                                  scenarios,
 300                                                                  visualization))
 301     return kibana_dashboards
 302
 303
 304 def generate_js_inputs(js_file_path, kibana_url, dashboards):
 305     js_dict = {}
 306     for dashboard in dashboards:
 307         dashboard_meta = dashboard['metadata']
 308         test_family = dashboard_meta['test_family']
 309         test_label = dashboard_meta['label']
 310
 311         if test_family not in js_dict:
 312             js_dict[test_family] = {}
 313
 314         js_test_family = js_dict[test_family]
 315
 316         if test_label not in js_test_family:
 317             js_test_family[test_label] = {}
 318
 319         js_test_label = js_test_family[test_label]
 320
 321         if dashboard.installer not in js_test_label:
 322             js_test_label[dashboard.installer] = {}
 323
 324         js_installer = js_test_label[dashboard.installer]
 325         js_installer[dashboard.pod] = kibana_url + '#/dashboard/' + dashboard.id
 326
 327     with open(js_file_path, 'w+') as js_file_fdesc:
 328         js_file_fdesc.write('var kibana_dashboard_links = ')
 329         js_file_fdesc.write(str(js_dict).replace("u'", "'"))
 330
 331
 332 def main():
 333     dashboards = construct_dashboards()
 334
 335     for kibana_dashboard in dashboards:
 336         kibana_dashboard.publish()
 337
 338     if generate_inputs:
 339         generate_js_inputs(input_file_path, kibana_url, dashboards)