utils/test/scripts/create_kibana_dashboards.py

   1 #! /usr/bin/env python
   2 import json
   3 import urlparse
   4
   5 import argparse
   6
   7 import conf_utils
   8 import logger_utils
   9 import shared_utils
  10
  11 logger = logger_utils.KibanaDashboardLogger('elastic2kibana').get
  12
  13 _installers = {'fuel', 'apex', 'compass', 'joid'}
  14
  15
  16 class KibanaDashboard(dict):
  17     def __init__(self, project_name, case_name, family, installer, pod, scenarios, visualization):
  18         super(KibanaDashboard, self).__init__()
  19         self.project_name = project_name
  20         self.case_name = case_name
  21         self.family = family
  22         self.installer = installer
  23         self.pod = pod
  24         self.scenarios = scenarios
  25         self.visualization = visualization
  26         self._visualization_title = None
  27         self._kibana_visualizations = []
  28         self._kibana_dashboard = None
  29         self._create_visualizations()
  30         self._create()
  31
  32     def _create_visualizations(self):
  33         for scenario in self.scenarios:
  34             self._kibana_visualizations.append(KibanaVisualization(self.project_name,
  35                                                                    self.case_name,
  36                                                                    self.installer,
  37                                                                    self.pod,
  38                                                                    scenario,
  39                                                                    self.visualization))
  40
  41         self._visualization_title = self._kibana_visualizations[0].vis_state_title
  42
  43     def _publish_visualizations(self):
  44         for visualization in self._kibana_visualizations:
  45             url = urlparse.urljoin(base_elastic_url, '/.kibana/visualization/{}'.format(visualization.id))
  46             logger.debug("publishing visualization '{}'".format(url))
  47             shared_utils.publish_json(visualization, es_creds, url)
  48
  49     def _construct_panels(self):
  50         size_x = 6
  51         size_y = 3
  52         max_columns = 7
  53         column = 1
  54         row = 1
  55         panel_index = 1
  56         panels_json = []
  57         for visualization in self._kibana_visualizations:
  58             panels_json.append({
  59                 "id": visualization.id,
  60                 "type": 'visualization',
  61                 "panelIndex": panel_index,
  62                 "size_x": size_x,
  63                 "size_y": size_y,
  64                 "col": column,
  65                 "row": row
  66             })
  67             panel_index += 1
  68             column += size_x
  69             if column > max_columns:
  70                 column = 1
  71                 row += size_y
  72         return json.dumps(panels_json, separators=(',', ':'))
  73
  74     def _create(self):
  75         self['title'] = '{} {} {} {} {}'.format(self.project_name,
  76                                                 self.case_name,
  77                                                 self.installer,
  78                                                 self._visualization_title,
  79                                                 self.pod)
  80         self.id = self['title'].replace(' ', '-').replace('/', '-')
  81
  82         self['hits'] = 0
  83         self['description'] = "Kibana dashboard for project_name '{}', case_name '{}', installer '{}', data '{}' and" \
  84                               " pod '{}'".format(self.project_name,
  85                                                  self.case_name,
  86                                                  self.installer,
  87                                                  self._visualization_title,
  88                                                  self.pod)
  89         self['panelsJSON'] = self._construct_panels()
  90         self['optionsJSON'] = json.dumps({
  91             "darkTheme": False
  92         },
  93             separators=(',', ':'))
  94         self['uiStateJSON'] = "{}"
  95         self['scenario'] = 1
  96         self['timeRestore'] = False
  97         self['kibanaSavedObjectMeta'] = {
  98             'searchSourceJSON': json.dumps({
  99                 "filter": [
 100                     {
 101                         "query": {
 102                             "query_string": {
 103                                 "query": "*",
 104                                 "analyze_wildcard": True
 105                             }
 106                         }
 107                     }
 108                 ]
 109             },
 110                 separators=(',', ':'))
 111         }
 112
 113         label = self.case_name
 114         if 'label' in self.visualization:
 115             label += " %s" % self.visualization.get('label')
 116         label += " %s" % self.visualization.get('name')
 117         self['metadata'] = {
 118             "label": label,
 119             "test_family": self.family
 120         }
 121
 122     def _publish(self):
 123         url = urlparse.urljoin(base_elastic_url, '/.kibana/dashboard/{}'.format(self.id))
 124         logger.debug("publishing dashboard '{}'".format(url))
 125         shared_utils.publish_json(self, es_creds, url)
 126
 127     def publish(self):
 128         self._publish_visualizations()
 129         self._publish()
 130
 131
 132 class KibanaSearchSourceJSON(dict):
 133     """
 134     "filter": [
 135                     {"match": {"installer": {"query": installer, "type": "phrase"}}},
 136                     {"match": {"project_name": {"query": project_name, "type": "phrase"}}},
 137                     {"match": {"case_name": {"query": case_name, "type": "phrase"}}}
 138                 ]
 139     """
 140
 141     def __init__(self, project_name, case_name, installer, pod, scenario):
 142         super(KibanaSearchSourceJSON, self).__init__()
 143         self["filter"] = [
 144             {"match": {"project_name": {"query": project_name, "type": "phrase"}}},
 145             {"match": {"case_name": {"query": case_name, "type": "phrase"}}},
 146             {"match": {"installer": {"query": installer, "type": "phrase"}}},
 147             {"match": {"scenario": {"query": scenario, "type": "phrase"}}}
 148         ]
 149         if pod != 'all':
 150             self["filter"].append({"match": {"pod_name": {"query": pod, "type": "phrase"}}})
 151
 152
 153 class VisualizationState(dict):
 154     def __init__(self, visualization):
 155         super(VisualizationState, self).__init__()
 156         name = visualization.get('name')
 157         fields = visualization.get('fields')
 158
 159         if name == 'tests_failures':
 160             mode = 'grouped'
 161             metric_type = 'sum'
 162             self['type'] = 'histogram'
 163         else:
 164             # duration or success_percentage
 165             mode = 'stacked'
 166             metric_type = 'avg'
 167             self['type'] = 'line'
 168
 169         self['params'] = {
 170             "shareYAxis": True,
 171             "addTooltip": True,
 172             "addLegend": True,
 173             "smoothLines": False,
 174             "scale": "linear",
 175             "interpolate": "linear",
 176             "mode": mode,
 177             "times": [],
 178             "addTimeMarker": False,
 179             "defaultYExtents": False,
 180             "setYExtents": False,
 181             "yAxis": {}
 182         }
 183
 184         self['aggs'] = []
 185
 186         i = 1
 187         for field in fields:
 188             self['aggs'].append({
 189                 "id": str(i),
 190                 "type": metric_type,
 191                 "schema": "metric",
 192                 "params": {
 193                     "field": field.get('field')
 194                 }
 195             })
 196             i += 1
 197
 198         self['aggs'].append({
 199                 "id": str(i),
 200                 "type": 'date_histogram',
 201                 "schema": "segment",
 202                 "params": {
 203                     "field": "start_date",
 204                     "interval": "auto",
 205                     "customInterval": "2h",
 206                     "min_doc_count": 1,
 207                     "extended_bounds": {}
 208                 }
 209             })
 210
 211         self['listeners'] = {}
 212         self['title'] = ' '.join(['{} {}'.format(x['type'], x['params']['field']) for x in self['aggs']
 213                                   if x['schema'] == 'metric'])
 214
 215
 216 class KibanaVisualization(dict):
 217     def __init__(self, project_name, case_name, installer, pod, scenario, visualization):
 218         """
 219         We need two things
 220         1. filter created from
 221             project_name
 222             case_name
 223             installer
 224             pod
 225             scenario
 226         2. visualization state
 227             field for y axis (metric) with type (avg, sum, etc.)
 228             field for x axis (segment) with type (date_histogram)
 229
 230         :return:
 231         """
 232         super(KibanaVisualization, self).__init__()
 233         vis_state = VisualizationState(visualization)
 234         self.vis_state_title = vis_state['title']
 235         self['title'] = '{} {} {} {} {} {}'.format(project_name,
 236                                                    case_name,
 237                                                    self.vis_state_title,
 238                                                    installer,
 239                                                    pod,
 240                                                    scenario)
 241         self.id = self['title'].replace(' ', '-').replace('/', '-')
 242         self['visState'] = json.dumps(vis_state, separators=(',', ':'))
 243         self['uiStateJSON'] = "{}"
 244         self['description'] = "Kibana visualization for project_name '{}', case_name '{}', data '{}', installer '{}'," \
 245                               " pod '{}' and scenario '{}'".format(project_name,
 246                                                                   case_name,
 247                                                                   self.vis_state_title,
 248                                                                   installer,
 249                                                                   pod,
 250                                                                   scenario)
 251         self['scenario'] = 1
 252         self['kibanaSavedObjectMeta'] = {"searchSourceJSON": json.dumps(KibanaSearchSourceJSON(project_name,
 253                                                                                                case_name,
 254                                                                                                installer,
 255                                                                                                pod,
 256                                                                                                scenario),
 257                                                                         separators=(',', ':'))}
 258
 259
 260 def _get_pods_and_scenarios(project_name, case_name, installer):
 261     query_json = json.JSONEncoder().encode({
 262         "query": {
 263             "bool": {
 264                 "must": [
 265                     {"match_all": {}}
 266                 ],
 267                 "filter": [
 268                     {"match": {"installer": {"query": installer, "type": "phrase"}}},
 269                     {"match": {"project_name": {"query": project_name, "type": "phrase"}}},
 270                     {"match": {"case_name": {"query": case_name, "type": "phrase"}}}
 271                 ]
 272             }
 273         }
 274     })
 275
 276     elastic_data = shared_utils.get_elastic_docs(urlparse.urljoin(base_elastic_url, '/test_results/mongo2elastic'),
 277                                                  es_creds, query_json)
 278
 279     pods_and_scenarios = {}
 280
 281     for data in elastic_data:
 282         pod = data['pod_name']
 283         if pod in pods_and_scenarios:
 284             pods_and_scenarios[pod].add(data['scenario'])
 285         else:
 286             pods_and_scenarios[pod] = {data['scenario']}
 287
 288         if 'all' in pods_and_scenarios:
 289             pods_and_scenarios['all'].add(data['scenario'])
 290         else:
 291             pods_and_scenarios['all'] = {data['scenario']}
 292
 293     return pods_and_scenarios
 294
 295
 296 def construct_dashboards():
 297     """
 298     iterate over testcase and installer
 299     1. get available pods for each testcase/installer pair
 300     2. get available scenario for each testcase/installer/pod tuple
 301     3. construct KibanaInput and append
 302
 303     :return: list of KibanaDashboards
 304     """
 305     kibana_dashboards = []
 306     for project, case_dicts in conf_utils.testcases_yaml.items():
 307         for case in case_dicts:
 308             case_name = case.get('name')
 309             visualizations = case.get('visualizations')
 310             family = case.get('test_family')
 311             for installer in _installers:
 312                 pods_and_scenarios = _get_pods_and_scenarios(project, case_name, installer)
 313                 for visualization in visualizations:
 314                     for pod, scenarios in pods_and_scenarios.iteritems():
 315                         kibana_dashboards.append(KibanaDashboard(project,
 316                                                                  case_name,
 317                                                                  family,
 318                                                                  installer,
 319                                                                  pod,
 320                                                                  scenarios,
 321                                                                  visualization))
 322     return kibana_dashboards
 323
 324
 325 def generate_js_inputs(js_file_path, kibana_url, dashboards):
 326     js_dict = {}
 327     for dashboard in dashboards:
 328         dashboard_meta = dashboard['metadata']
 329         test_family = dashboard_meta['test_family']
 330         test_label = dashboard_meta['label']
 331
 332         if test_family not in js_dict:
 333             js_dict[test_family] = {}
 334
 335         js_test_family = js_dict[test_family]
 336
 337         if test_label not in js_test_family:
 338             js_test_family[test_label] = {}
 339
 340         js_test_label = js_test_family[test_label]
 341
 342         if dashboard.installer not in js_test_label:
 343             js_test_label[dashboard.installer] = {}
 344
 345         js_installer = js_test_label[dashboard.installer]
 346         js_installer[dashboard.pod] = kibana_url + '#/dashboard/' + dashboard.id
 347
 348     with open(js_file_path, 'w+') as js_file_fdesc:
 349         js_file_fdesc.write('var kibana_dashboard_links = ')
 350         js_file_fdesc.write(str(js_dict).replace("u'", "'"))
 351
 352
 353 if __name__ == '__main__':
 354     parser = argparse.ArgumentParser(description='Create Kibana dashboards from data in elasticsearch')
 355     parser.add_argument('-e', '--elasticsearch-url', default='http://localhost:9200',
 356                         help='the url of elasticsearch, defaults to http://localhost:9200')
 357
 358     parser.add_argument('-js', '--generate_js_inputs', action='store_true',
 359                         help='Use this argument to generate javascript inputs for kibana landing page')
 360
 361     parser.add_argument('--js_path', default='/usr/share/nginx/html/kibana_dashboards/conf.js',
 362                         help='Path of javascript file with inputs for kibana landing page')
 363
 364     parser.add_argument('-k', '--kibana_url', default='https://testresults.opnfv.org/kibana/app/kibana',
 365                         help='The url of kibana for javascript inputs')
 366
 367     parser.add_argument('-u', '--elasticsearch-username', default=None,
 368                         help='The username with password for elasticsearch in format username:password')
 369
 370     args = parser.parse_args()
 371     base_elastic_url = args.elasticsearch_url
 372     generate_inputs = args.generate_js_inputs
 373     input_file_path = args.js_path
 374     kibana_url = args.kibana_url
 375     es_creds = args.elasticsearch_username
 376
 377     dashboards = construct_dashboards()
 378
 379     for kibana_dashboard in dashboards:
 380         kibana_dashboard.publish()
 381
 382     if generate_inputs:
 383         generate_js_inputs(input_file_path, kibana_url, dashboards)