utils/test/dashboard/dashboard/elastic2kibana/main.py

   1 #! /usr/bin/env python
   2 import json
   3 import urlparse
   4
   5 import argparse
   6 from jinja2 import PackageLoader, Environment
   7
   8 from common import elastic_access
   9 from common import logger_utils
  10 from conf import testcases
  11 from conf.config import APIConfig
  12
  13 logger = logger_utils.DashboardLogger('elastic2kibana').get
  14
  15 parser = argparse.ArgumentParser()
  16 parser.add_argument("-c", "--config-file",
  17                     dest='config_file',
  18                     help="Config file location")
  19
  20 args = parser.parse_args()
  21 CONF = APIConfig().parse(args.config_file)
  22 base_elastic_url = CONF.elastic_url
  23 generate_inputs = CONF.is_js
  24 input_file_path = CONF.js_path
  25 kibana_url = CONF.kibana_url
  26 es_creds = CONF.elastic_creds
  27
  28 _installers = {'fuel', 'apex', 'compass', 'joid'}
  29
  30
  31 class KibanaDashboard(dict):
  32     def __init__(self, project_name, case_name, family, installer, pod, scenarios, visualization):
  33         super(KibanaDashboard, self).__init__()
  34         self.project_name = project_name
  35         self.case_name = case_name
  36         self.family = family
  37         self.installer = installer
  38         self.pod = pod
  39         self.scenarios = scenarios
  40         self.visualization = visualization
  41         self._visualization_title = None
  42         self._kibana_visualizations = []
  43         self._kibana_dashboard = None
  44         self._create_visualizations()
  45         self._create()
  46
  47     def _create_visualizations(self):
  48         for scenario in self.scenarios:
  49             self._kibana_visualizations.append(KibanaVisualization(self.project_name,
  50                                                                    self.case_name,
  51                                                                    self.installer,
  52                                                                    self.pod,
  53                                                                    scenario,
  54                                                                    self.visualization))
  55
  56         self._visualization_title = self._kibana_visualizations[0].vis_title
  57
  58     def _publish_visualizations(self):
  59         for visualization in self._kibana_visualizations:
  60             url = urlparse.urljoin(base_elastic_url, '/.kibana/visualization/{}'.format(visualization.id))
  61             logger.debug("publishing visualization '{}'".format(url))
  62             # logger.error("_publish_visualization: %s" % visualization)
  63             elastic_access.publish_docs(url, es_creds, visualization)
  64
  65     def _construct_panels(self):
  66         size_x = 6
  67         size_y = 3
  68         max_columns = 7
  69         column = 1
  70         row = 1
  71         panel_index = 1
  72         panels_json = []
  73         for visualization in self._kibana_visualizations:
  74             panels_json.append({
  75                 "id": visualization.id,
  76                 "type": 'visualization',
  77                 "panelIndex": panel_index,
  78                 "size_x": size_x,
  79                 "size_y": size_y,
  80                 "col": column,
  81                 "row": row
  82             })
  83             panel_index += 1
  84             column += size_x
  85             if column > max_columns:
  86                 column = 1
  87                 row += size_y
  88         return json.dumps(panels_json, separators=(',', ':'))
  89
  90     def _create(self):
  91         self['title'] = '{} {} {} {} {}'.format(self.project_name,
  92                                                 self.case_name,
  93                                                 self.installer,
  94                                                 self._visualization_title,
  95                                                 self.pod)
  96         self.id = self['title'].replace(' ', '-').replace('/', '-')
  97
  98         self['hits'] = 0
  99         self['description'] = "Kibana dashboard for project_name '{}', case_name '{}', installer '{}', data '{}' and" \
 100                               " pod '{}'".format(self.project_name,
 101                                                  self.case_name,
 102                                                  self.installer,
 103                                                  self._visualization_title,
 104                                                  self.pod)
 105         self['panelsJSON'] = self._construct_panels()
 106         self['optionsJSON'] = json.dumps({
 107             "darkTheme": False
 108         },
 109             separators=(',', ':'))
 110         self['uiStateJSON'] = "{}"
 111         self['scenario'] = 1
 112         self['timeRestore'] = False
 113         self['kibanaSavedObjectMeta'] = {
 114             'searchSourceJSON': json.dumps({
 115                 "filter": [
 116                     {
 117                         "query": {
 118                             "query_string": {
 119                                 "query": "*",
 120                                 "analyze_wildcard": True
 121                             }
 122                         }
 123                     }
 124                 ]
 125             },
 126                 separators=(',', ':'))
 127         }
 128
 129         label = self.case_name
 130         if 'label' in self.visualization:
 131             label += " %s" % self.visualization.get('label')
 132         label += " %s" % self.visualization.get('name')
 133         self['metadata'] = {
 134             "label": label,
 135             "test_family": self.family
 136         }
 137
 138     def _publish(self):
 139         url = urlparse.urljoin(base_elastic_url, '/.kibana/dashboard/{}'.format(self.id))
 140         logger.debug("publishing dashboard '{}'".format(url))
 141         elastic_access.publish_docs(url, es_creds, self)
 142
 143     def publish(self):
 144         self._publish_visualizations()
 145         self._publish()
 146
 147
 148 class KibanaSearchSourceJSON(dict):
 149     """
 150     "filter": [
 151                     {"match": {"installer": {"query": installer, "type": "phrase"}}},
 152                     {"match": {"project_name": {"query": project_name, "type": "phrase"}}},
 153                     {"match": {"case_name": {"query": case_name, "type": "phrase"}}}
 154                 ]
 155     """
 156
 157     def __init__(self, project_name, case_name, installer, pod, scenario):
 158         super(KibanaSearchSourceJSON, self).__init__()
 159         self["filter"] = [
 160             {"match": {"project_name": {"query": project_name, "type": "phrase"}}},
 161             {"match": {"case_name": {"query": case_name, "type": "phrase"}}},
 162             {"match": {"installer": {"query": installer, "type": "phrase"}}},
 163             {"match": {"scenario": {"query": scenario, "type": "phrase"}}}
 164         ]
 165         if pod != 'all':
 166             self["filter"].append({"match": {"pod_name": {"query": pod, "type": "phrase"}}})
 167
 168
 169 class VisualizationBuilder(object):
 170     def __init__(self, visualization):
 171         super(VisualizationBuilder, self).__init__()
 172         self.visualization = visualization
 173
 174     def build(self):
 175         name = self.visualization.get('name')
 176         fields = self.visualization.get('fields')
 177
 178         aggs = []
 179         index = 1
 180         for field in fields:
 181             aggs.append({
 182                 "id": index,
 183                 "field": field.get("field")
 184             })
 185             index += 1
 186
 187         env = Environment(loader=PackageLoader('elastic2kibana', 'templates'))
 188         env.filters['jsonify'] = json.dumps
 189         template = env.get_template('{}.json'.format(name))
 190         vis = template.render(aggs=aggs)
 191         return json.loads(vis)
 192
 193
 194 class KibanaVisualization(dict):
 195     def __init__(self, project_name, case_name, installer, pod, scenario, visualization):
 196         """
 197         We need two things
 198         1. filter created from
 199             project_name
 200             case_name
 201             installer
 202             pod
 203             scenario
 204         2. visualization state
 205             field for y axis (metric) with type (avg, sum, etc.)
 206             field for x axis (segment) with type (date_histogram)
 207
 208         :return:
 209         """
 210         super(KibanaVisualization, self).__init__()
 211         vis = VisualizationBuilder(visualization).build()
 212         self.vis_title = vis['title']
 213         self['title'] = '{} {} {} {} {} {}'.format(project_name,
 214                                                    case_name,
 215                                                    self.vis_title,
 216                                                    installer,
 217                                                    pod,
 218                                                    scenario)
 219         self.id = self['title'].replace(' ', '-').replace('/', '-')
 220         self['visState'] = json.dumps(vis, separators=(',', ':'))
 221         self['uiStateJSON'] = "{}"
 222         self['description'] = "Kibana visualization for project_name '{}', case_name '{}', metric '{}', installer '{}'," \
 223                               " pod '{}' and scenario '{}'".format(project_name,
 224                                                                    case_name,
 225                                                                    self.vis_title,
 226                                                                    installer,
 227                                                                    pod,
 228                                                                    scenario)
 229         self['scenario'] = 1
 230         self['kibanaSavedObjectMeta'] = {"searchSourceJSON": json.dumps(KibanaSearchSourceJSON(project_name,
 231                                                                                                case_name,
 232                                                                                                installer,
 233                                                                                                pod,
 234                                                                                                scenario),
 235                                                                         separators=(',', ':'))}
 236
 237
 238 def _get_pods_and_scenarios(project_name, case_name, installer):
 239     query_json = json.JSONEncoder().encode({
 240         "query": {
 241             "bool": {
 242                 "must": [
 243                     {"match_all": {}}
 244                 ],
 245                 "filter": [
 246                     {"match": {"installer": {"query": installer, "type": "phrase"}}},
 247                     {"match": {"project_name": {"query": project_name, "type": "phrase"}}},
 248                     {"match": {"case_name": {"query": case_name, "type": "phrase"}}}
 249                 ]
 250             }
 251         }
 252     })
 253
 254     elastic_data = elastic_access.get_docs(urlparse.urljoin(base_elastic_url, '/test_results/mongo2elastic'),
 255                                                    es_creds, query_json)
 256
 257     pods_and_scenarios = {}
 258
 259     for data in elastic_data:
 260         pod = data['pod_name']
 261         if pod in pods_and_scenarios:
 262             pods_and_scenarios[pod].add(data['scenario'])
 263         else:
 264             pods_and_scenarios[pod] = {data['scenario']}
 265
 266         if 'all' in pods_and_scenarios:
 267             pods_and_scenarios['all'].add(data['scenario'])
 268         else:
 269             pods_and_scenarios['all'] = {data['scenario']}
 270
 271     return pods_and_scenarios
 272
 273
 274 def construct_dashboards():
 275     """
 276     iterate over testcase and installer
 277     1. get available pods for each testcase/installer pair
 278     2. get available scenario for each testcase/installer/pod tuple
 279     3. construct KibanaInput and append
 280
 281     :return: list of KibanaDashboards
 282     """
 283     kibana_dashboards = []
 284     for project, case_dicts in testcases.testcases_yaml.items():
 285         for case in case_dicts:
 286             case_name = case.get('name')
 287             visualizations = case.get('visualizations')
 288             family = case.get('test_family')
 289             for installer in _installers:
 290                 pods_and_scenarios = _get_pods_and_scenarios(project, case_name, installer)
 291                 for visualization in visualizations:
 292                     for pod, scenarios in pods_and_scenarios.iteritems():
 293                         kibana_dashboards.append(KibanaDashboard(project,
 294                                                                  case_name,
 295                                                                  family,
 296                                                                  installer,
 297                                                                  pod,
 298                                                                  scenarios,
 299                                                                  visualization))
 300     return kibana_dashboards
 301
 302
 303 def generate_js_inputs(js_file_path, kibana_url, dashboards):
 304     js_dict = {}
 305     for dashboard in dashboards:
 306         dashboard_meta = dashboard['metadata']
 307         test_family = dashboard_meta['test_family']
 308         test_label = dashboard_meta['label']
 309
 310         if test_family not in js_dict:
 311             js_dict[test_family] = {}
 312
 313         js_test_family = js_dict[test_family]
 314
 315         if test_label not in js_test_family:
 316             js_test_family[test_label] = {}
 317
 318         js_test_label = js_test_family[test_label]
 319
 320         if dashboard.installer not in js_test_label:
 321             js_test_label[dashboard.installer] = {}
 322
 323         js_installer = js_test_label[dashboard.installer]
 324         js_installer[dashboard.pod] = kibana_url + '#/dashboard/' + dashboard.id
 325
 326     with open(js_file_path, 'w+') as js_file_fdesc:
 327         js_file_fdesc.write('var kibana_dashboard_links = ')
 328         js_file_fdesc.write(str(js_dict).replace("u'", "'"))
 329
 330
 331 def main():
 332     dashboards = construct_dashboards()
 333
 334     for kibana_dashboard in dashboards:
 335         kibana_dashboard.publish()
 336
 337     if generate_inputs:
 338         generate_js_inputs(input_file_path, kibana_url, dashboards)