utils/test/dashboard/dashboard/elastic2kibana/main.py

   1 #! /usr/bin/env python
   2 import json
   3 import urlparse
   4
   5 import argparse
   6 from jinja2 import PackageLoader, Environment
   7
   8 from common import elastic_access
   9 from common import logger_utils
  10 from conf import testcases
  11 from conf.config import APIConfig
  12
  13 logger = logger_utils.DashboardLogger('elastic2kibana').get
  14
  15 parser = argparse.ArgumentParser()
  16 parser.add_argument("-c", "--config-file",
  17                     dest='config_file',
  18                     help="Config file location")
  19
  20 args = parser.parse_args()
  21 CONF = APIConfig().parse(args.config_file)
  22 base_elastic_url = CONF.elastic_url
  23 generate_inputs = CONF.is_js
  24 input_file_path = CONF.js_path
  25 kibana_url = CONF.kibana_url
  26 es_creds = CONF.elastic_creds
  27
  28 _installers = {'fuel', 'apex', 'compass', 'joid'}
  29
  30 env = Environment(loader=PackageLoader('elastic2kibana', 'templates'))
  31 env.filters['jsonify'] = json.dumps
  32
  33
  34 def dumps(self, items):
  35     for key in items:
  36         self.visualization[key] = json.dumps(self.visualization[key])
  37
  38
  39 def dumps_2depth(self, key1, key2):
  40     self.visualization[key1][key2] = json.dumps(self.visualization[key1][key2])
  41
  42
  43 class Dashboard(dict):
  44     def __init__(self, project_name, case_name, family, installer, pod, scenarios, visualization):
  45         super(Dashboard, self).__init__()
  46         self.project_name = project_name
  47         self.case_name = case_name
  48         self.family = family
  49         self.installer = installer
  50         self.pod = pod
  51         self.scenarios = scenarios
  52         self.visualization = visualization
  53         self._visualization_title = None
  54         self._kibana_visualizations = []
  55         self._kibana_dashboard = None
  56         self._create_visualizations()
  57         self._create()
  58
  59     def _create_visualizations(self):
  60         for scenario in self.scenarios:
  61             self._kibana_visualizations.append(Visualization(self.project_name,
  62                                                              self.case_name,
  63                                                              self.installer,
  64                                                              self.pod,
  65                                                              scenario,
  66                                                              self.visualization))
  67
  68         self._visualization_title = self._kibana_visualizations[0].vis_state_title
  69
  70     def _publish_visualizations(self):
  71         for visualization in self._kibana_visualizations:
  72             url = urlparse.urljoin(base_elastic_url, '/.kibana/visualization/{}'.format(visualization.id))
  73             logger.debug("publishing visualization '{}'".format(url))
  74             # logger.error("_publish_visualization: %s" % visualization)
  75             elastic_access.publish_docs(url, es_creds, visualization)
  76
  77     def _create(self):
  78         db = {
  79             "query": {
  80                 "project_name": self.project_name,
  81                 "case_name": self.case_name,
  82                 "installer": self.installer,
  83                 "metric": self._visualization_title,
  84                 "pod": self.pod
  85             },
  86             "test_family": self.family,
  87             "ids": [visualization.id for visualization in self._kibana_visualizations]
  88         }
  89         template = env.get_template('dashboard.json')
  90         self.dashboard = json.loads(template.render(db=db))
  91         dumps(self.dashboard, ['description', 'uiStateJSON', 'panelsJSON','optionsJSON'])
  92         dumps_2depth(self.dashboard, 'kibanaSavedObjectMeta', 'searchSourceJSON')
  93         self.id = self.dashboard['title'].replace(' ', '-').replace('/', '-')
  94
  95
  96     def _publish(self):
  97         url = urlparse.urljoin(base_elastic_url, '/.kibana/dashboard/{}'.format(self.id))
  98         logger.debug("publishing dashboard '{}'".format(url))
  99         #logger.error("dashboard: %s" % json.dumps(self.dashboard))
 100         elastic_access.publish_docs(url, es_creds, self.dashboard)
 101
 102     def publish(self):
 103         self._publish_visualizations()
 104         self._publish()
 105
 106
 107 class VisStateBuilder(object):
 108     def __init__(self, visualization):
 109         super(VisStateBuilder, self).__init__()
 110         self.visualization = visualization
 111
 112     def build(self):
 113         name = self.visualization.get('name')
 114         fields = self.visualization.get('fields')
 115
 116         aggs = []
 117         index = 1
 118         for field in fields:
 119             aggs.append({
 120                 "id": index,
 121                 "field": field.get("field")
 122             })
 123             index += 1
 124
 125         template = env.get_template('{}.json'.format(name))
 126         vis = template.render(aggs=aggs)
 127         return json.loads(vis)
 128
 129
 130 class Visualization(object):
 131     def __init__(self, project_name, case_name, installer, pod, scenario, visualization):
 132         """
 133         We need two things
 134         1. filter created from
 135             project_name
 136             case_name
 137             installer
 138             pod
 139             scenario
 140         2. visualization state
 141             field for y axis (metric) with type (avg, sum, etc.)
 142             field for x axis (segment) with type (date_histogram)
 143
 144         :return:
 145         """
 146         super(Visualization, self).__init__()
 147         visState = VisStateBuilder(visualization).build()
 148         self.vis_state_title = visState['title']
 149
 150         vis = {
 151             "visState": json.dumps(visState),
 152             "filters": {
 153                 "project_name": project_name,
 154                 "case_name": case_name,
 155                 "installer": installer,
 156                 "metric": self.vis_state_title,
 157                 "pod_name": pod,
 158                 "scenario": scenario
 159             }
 160         }
 161
 162         template = env.get_template('visualization.json')
 163
 164         self.visualization = json.loads(template.render(vis=vis))
 165         dumps(self.visualization, ['visState', 'description', 'uiStateJSON'])
 166         dumps_2depth(self.visualization, 'kibanaSavedObjectMeta', 'searchSourceJSON')
 167         self.id = self.visualization['title'].replace(' ', '-').replace('/', '-')
 168
 169
 170 def _get_pods_and_scenarios(project_name, case_name, installer):
 171     query_json = json.JSONEncoder().encode({
 172         "query": {
 173             "bool": {
 174                 "must": [
 175                     {"match_all": {}}
 176                 ],
 177                 "filter": [
 178                     {"match": {"installer": {"query": installer, "type": "phrase"}}},
 179                     {"match": {"project_name": {"query": project_name, "type": "phrase"}}},
 180                     {"match": {"case_name": {"query": case_name, "type": "phrase"}}}
 181                 ]
 182             }
 183         }
 184     })
 185
 186     elastic_data = elastic_access.get_docs(urlparse.urljoin(base_elastic_url, '/test_results/mongo2elastic'),
 187                                            es_creds,
 188                                            query_json)
 189
 190     pods_and_scenarios = {}
 191
 192     for data in elastic_data:
 193         pod = data['pod_name']
 194         if pod in pods_and_scenarios:
 195             pods_and_scenarios[pod].add(data['scenario'])
 196         else:
 197             pods_and_scenarios[pod] = {data['scenario']}
 198
 199         if 'all' in pods_and_scenarios:
 200             pods_and_scenarios['all'].add(data['scenario'])
 201         else:
 202             pods_and_scenarios['all'] = {data['scenario']}
 203
 204     return pods_and_scenarios
 205
 206
 207 def construct_dashboards():
 208     """
 209     iterate over testcase and installer
 210     1. get available pods for each testcase/installer pair
 211     2. get available scenario for each testcase/installer/pod tuple
 212     3. construct KibanaInput and append
 213
 214     :return: list of KibanaDashboards
 215     """
 216     kibana_dashboards = []
 217     for project, case_dicts in testcases.testcases_yaml.items():
 218         for case in case_dicts:
 219             case_name = case.get('name')
 220             visualizations = case.get('visualizations')
 221             family = case.get('test_family')
 222             for installer in _installers:
 223                 pods_and_scenarios = _get_pods_and_scenarios(project, case_name, installer)
 224                 for visualization in visualizations:
 225                     for pod, scenarios in pods_and_scenarios.iteritems():
 226                         kibana_dashboards.append(Dashboard(project,
 227                                                            case_name,
 228                                                            family,
 229                                                            installer,
 230                                                            pod,
 231                                                            scenarios,
 232                                                            visualization))
 233     return kibana_dashboards
 234
 235
 236 def generate_js_inputs(js_file_path, kibana_url, dashboards):
 237     js_dict = {}
 238     for dashboard in dashboards:
 239         dashboard_meta = dashboard.dashboard['metadata']
 240         test_family = dashboard_meta['test_family']
 241         test_label = dashboard_meta['label']
 242
 243         if test_family not in js_dict:
 244             js_dict[test_family] = {}
 245
 246         js_test_family = js_dict[test_family]
 247
 248         if test_label not in js_test_family:
 249             js_test_family[test_label] = {}
 250
 251         js_test_label = js_test_family[test_label]
 252
 253         if dashboard.installer not in js_test_label:
 254             js_test_label[dashboard.installer] = {}
 255
 256         js_installer = js_test_label[dashboard.installer]
 257         js_installer[dashboard.pod] = kibana_url + '#/dashboard/' + dashboard.id
 258
 259     with open(js_file_path, 'w+') as js_file_fdesc:
 260         js_file_fdesc.write('var kibana_dashboard_links = ')
 261         js_file_fdesc.write(str(js_dict).replace("u'", "'"))
 262
 263
 264 def main():
 265     dashboards = construct_dashboards()
 266
 267     for kibana_dashboard in dashboards:
 268         kibana_dashboard.publish()
 269
 270     if generate_inputs:
 271         generate_js_inputs(input_file_path, kibana_url, dashboards)