utils/test/scripts/create_kibana_dashboards.py

   1 #! /usr/bin/env python
   2 import json
   3 import logging
   4 import urlparse
   5
   6 import argparse
   7 import yaml
   8
   9 import shared_utils
  10
  11 logger = logging.getLogger('create_kibana_dashboards')
  12 logger.setLevel(logging.DEBUG)
  13 file_handler = logging.FileHandler('./{}.log'.format('create_kibana_dashboards'))
  14 file_handler.setFormatter(logging.Formatter('%(asctime)s %(levelname)s: %(message)s'))
  15 logger.addHandler(file_handler)
  16
  17 _installers = {'fuel', 'apex', 'compass', 'joid'}
  18
  19
  20 class KibanaDashboard(dict):
  21     def __init__(self, project_name, case_name, family, installer, pod, scenarios, visualization):
  22         super(KibanaDashboard, self).__init__()
  23         self.project_name = project_name
  24         self.case_name = case_name
  25         self.family = family
  26         self.installer = installer
  27         self.pod = pod
  28         self.scenarios = scenarios
  29         self.visualization = visualization
  30         self._visualization_title = None
  31         self._kibana_visualizations = []
  32         self._kibana_dashboard = None
  33         self._create_visualizations()
  34         self._create()
  35
  36     def _create_visualizations(self):
  37         for scenario in self.scenarios:
  38             self._kibana_visualizations.append(KibanaVisualization(self.project_name,
  39                                                                    self.case_name,
  40                                                                    self.installer,
  41                                                                    self.pod,
  42                                                                    scenario,
  43                                                                    self.visualization))
  44
  45         self._visualization_title = self._kibana_visualizations[0].vis_state_title
  46
  47     def _publish_visualizations(self):
  48         for visualization in self._kibana_visualizations:
  49             url = urlparse.urljoin(base_elastic_url, '/.kibana/visualization/{}'.format(visualization.id))
  50             logger.debug("publishing visualization '{}'".format(url))
  51             shared_utils.publish_json(visualization, es_creds, url)
  52
  53     def _construct_panels(self):
  54         size_x = 6
  55         size_y = 3
  56         max_columns = 7
  57         column = 1
  58         row = 1
  59         panel_index = 1
  60         panels_json = []
  61         for visualization in self._kibana_visualizations:
  62             panels_json.append({
  63                 "id": visualization.id,
  64                 "type": 'visualization',
  65                 "panelIndex": panel_index,
  66                 "size_x": size_x,
  67                 "size_y": size_y,
  68                 "col": column,
  69                 "row": row
  70             })
  71             panel_index += 1
  72             column += size_x
  73             if column > max_columns:
  74                 column = 1
  75                 row += size_y
  76         return json.dumps(panels_json, separators=(',', ':'))
  77
  78     def _create(self):
  79         self['title'] = '{} {} {} {} {}'.format(self.project_name,
  80                                                 self.case_name,
  81                                                 self.installer,
  82                                                 self._visualization_title,
  83                                                 self.pod)
  84         self.id = self['title'].replace(' ', '-').replace('/', '-')
  85
  86         self['hits'] = 0
  87         self['description'] = "Kibana dashboard for project_name '{}', case_name '{}', installer '{}', data '{}' and" \
  88                               " pod '{}'".format(self.project_name,
  89                                                  self.case_name,
  90                                                  self.installer,
  91                                                  self._visualization_title,
  92                                                  self.pod)
  93         self['panelsJSON'] = self._construct_panels()
  94         self['optionsJSON'] = json.dumps({
  95             "darkTheme": False
  96         },
  97             separators=(',', ':'))
  98         self['uiStateJSON'] = "{}"
  99         self['scenario'] = 1
 100         self['timeRestore'] = False
 101         self['kibanaSavedObjectMeta'] = {
 102             'searchSourceJSON': json.dumps({
 103                 "filter": [
 104                     {
 105                         "query": {
 106                             "query_string": {
 107                                 "query": "*",
 108                                 "analyze_wildcard": True
 109                             }
 110                         }
 111                     }
 112                 ]
 113             },
 114                 separators=(',', ':'))
 115         }
 116
 117         label = self.case_name
 118         if 'label' in self.visualization:
 119             label += " %s" % self.visualization.get('label')
 120         label += " %s" % self.visualization.get('name')
 121         self['metadata'] = {
 122             "label": label,
 123             "test_family": self.family
 124         }
 125
 126     def _publish(self):
 127         url = urlparse.urljoin(base_elastic_url, '/.kibana/dashboard/{}'.format(self.id))
 128         logger.debug("publishing dashboard '{}'".format(url))
 129         shared_utils.publish_json(self, es_creds, url)
 130
 131     def publish(self):
 132         self._publish_visualizations()
 133         self._publish()
 134
 135
 136 class KibanaSearchSourceJSON(dict):
 137     """
 138     "filter": [
 139                     {"match": {"installer": {"query": installer, "type": "phrase"}}},
 140                     {"match": {"project_name": {"query": project_name, "type": "phrase"}}},
 141                     {"match": {"case_name": {"query": case_name, "type": "phrase"}}}
 142                 ]
 143     """
 144
 145     def __init__(self, project_name, case_name, installer, pod, scenario):
 146         super(KibanaSearchSourceJSON, self).__init__()
 147         self["filter"] = [
 148             {"match": {"project_name": {"query": project_name, "type": "phrase"}}},
 149             {"match": {"case_name": {"query": case_name, "type": "phrase"}}},
 150             {"match": {"installer": {"query": installer, "type": "phrase"}}},
 151             {"match": {"scenario": {"query": scenario, "type": "phrase"}}}
 152         ]
 153         if pod != 'all':
 154             self["filter"].append({"match": {"pod_name": {"query": pod, "type": "phrase"}}})
 155
 156
 157 class VisualizationState(dict):
 158     def __init__(self, visualization):
 159         super(VisualizationState, self).__init__()
 160         name = visualization.get('name')
 161         fields = visualization.get('fields')
 162
 163         if name == 'tests_failures':
 164             mode = 'grouped'
 165             metric_type = 'sum'
 166             self['type'] = 'histogram'
 167         else:
 168             # duration or success_percentage
 169             mode = 'stacked'
 170             metric_type = 'avg'
 171             self['type'] = 'line'
 172
 173         self['params'] = {
 174             "shareYAxis": True,
 175             "addTooltip": True,
 176             "addLegend": True,
 177             "smoothLines": False,
 178             "scale": "linear",
 179             "interpolate": "linear",
 180             "mode": mode,
 181             "times": [],
 182             "addTimeMarker": False,
 183             "defaultYExtents": False,
 184             "setYExtents": False,
 185             "yAxis": {}
 186         }
 187
 188         self['aggs'] = []
 189
 190         i = 1
 191         for field in fields:
 192             self['aggs'].append({
 193                 "id": str(i),
 194                 "type": metric_type,
 195                 "schema": "metric",
 196                 "params": {
 197                     "field": field.get('field')
 198                 }
 199             })
 200             i += 1
 201
 202         self['aggs'].append({
 203                 "id": str(i),
 204                 "type": 'date_histogram',
 205                 "schema": "segment",
 206                 "params": {
 207                     "field": "start_date",
 208                     "interval": "auto",
 209                     "customInterval": "2h",
 210                     "min_doc_count": 1,
 211                     "extended_bounds": {}
 212                 }
 213             })
 214
 215         self['listeners'] = {}
 216         self['title'] = ' '.join(['{} {}'.format(x['type'], x['params']['field']) for x in self['aggs']
 217                                   if x['schema'] == 'metric'])
 218
 219
 220 class KibanaVisualization(dict):
 221     def __init__(self, project_name, case_name, installer, pod, scenario, visualization):
 222         """
 223         We need two things
 224         1. filter created from
 225             project_name
 226             case_name
 227             installer
 228             pod
 229             scenario
 230         2. visualization state
 231             field for y axis (metric) with type (avg, sum, etc.)
 232             field for x axis (segment) with type (date_histogram)
 233
 234         :return:
 235         """
 236         super(KibanaVisualization, self).__init__()
 237         vis_state = VisualizationState(visualization)
 238         self.vis_state_title = vis_state['title']
 239         self['title'] = '{} {} {} {} {} {}'.format(project_name,
 240                                                    case_name,
 241                                                    self.vis_state_title,
 242                                                    installer,
 243                                                    pod,
 244                                                    scenario)
 245         self.id = self['title'].replace(' ', '-').replace('/', '-')
 246         self['visState'] = json.dumps(vis_state, separators=(',', ':'))
 247         self['uiStateJSON'] = "{}"
 248         self['description'] = "Kibana visualization for project_name '{}', case_name '{}', data '{}', installer '{}'," \
 249                               " pod '{}' and scenario '{}'".format(project_name,
 250                                                                   case_name,
 251                                                                   self.vis_state_title,
 252                                                                   installer,
 253                                                                   pod,
 254                                                                   scenario)
 255         self['scenario'] = 1
 256         self['kibanaSavedObjectMeta'] = {"searchSourceJSON": json.dumps(KibanaSearchSourceJSON(project_name,
 257                                                                                                case_name,
 258                                                                                                installer,
 259                                                                                                pod,
 260                                                                                                scenario),
 261                                                                         separators=(',', ':'))}
 262
 263
 264 def _get_pods_and_scenarios(project_name, case_name, installer):
 265     query_json = json.JSONEncoder().encode({
 266         "query": {
 267             "bool": {
 268                 "must": [
 269                     {"match_all": {}}
 270                 ],
 271                 "filter": [
 272                     {"match": {"installer": {"query": installer, "type": "phrase"}}},
 273                     {"match": {"project_name": {"query": project_name, "type": "phrase"}}},
 274                     {"match": {"case_name": {"query": case_name, "type": "phrase"}}}
 275                 ]
 276             }
 277         }
 278     })
 279
 280     elastic_data = shared_utils.get_elastic_data(urlparse.urljoin(base_elastic_url, '/test_results/mongo2elastic'),
 281                                                  es_creds, query_json)
 282
 283     pods_and_scenarios = {}
 284
 285     for data in elastic_data:
 286         pod = data['pod_name']
 287         if pod in pods_and_scenarios:
 288             pods_and_scenarios[pod].add(data['scenario'])
 289         else:
 290             pods_and_scenarios[pod] = {data['scenario']}
 291
 292         if 'all' in pods_and_scenarios:
 293             pods_and_scenarios['all'].add(data['scenario'])
 294         else:
 295             pods_and_scenarios['all'] = {data['scenario']}
 296
 297     return pods_and_scenarios
 298
 299
 300 def construct_dashboards():
 301     """
 302     iterate over testcase and installer
 303     1. get available pods for each testcase/installer pair
 304     2. get available scenario for each testcase/installer/pod tuple
 305     3. construct KibanaInput and append
 306
 307     :return: list of KibanaDashboards
 308     """
 309     kibana_dashboards = []
 310     with open('./testcases.yaml') as f:
 311         testcases_yaml = yaml.safe_load(f)
 312
 313     for project, case_dicts in testcases_yaml.items():
 314         for case in case_dicts:
 315             case_name = case.get('name')
 316             visualizations = case.get('visualizations')
 317             family = case.get('test_family')
 318             for installer in _installers:
 319                 pods_and_scenarios = _get_pods_and_scenarios(project, case_name, installer)
 320                 for visualization in visualizations:
 321                     for pod, scenarios in pods_and_scenarios.iteritems():
 322                         kibana_dashboards.append(KibanaDashboard(project,
 323                                                                  case_name,
 324                                                                  family,
 325                                                                  installer,
 326                                                                  pod,
 327                                                                  scenarios,
 328                                                                  visualization))
 329     return kibana_dashboards
 330
 331
 332 def generate_js_inputs(js_file_path, kibana_url, dashboards):
 333     js_dict = {}
 334     for dashboard in dashboards:
 335         dashboard_meta = dashboard['metadata']
 336         test_family = dashboard_meta['test_family']
 337         test_label = dashboard_meta['label']
 338
 339         if test_family not in js_dict:
 340             js_dict[test_family] = {}
 341
 342         js_test_family = js_dict[test_family]
 343
 344         if test_label not in js_test_family:
 345             js_test_family[test_label] = {}
 346
 347         js_test_label = js_test_family[test_label]
 348
 349         if dashboard.installer not in js_test_label:
 350             js_test_label[dashboard.installer] = {}
 351
 352         js_installer = js_test_label[dashboard.installer]
 353         js_installer[dashboard.pod] = kibana_url + '#/dashboard/' + dashboard.id
 354
 355     with open(js_file_path, 'w+') as js_file_fdesc:
 356         js_file_fdesc.write('var kibana_dashboard_links = ')
 357         js_file_fdesc.write(str(js_dict).replace("u'", "'"))
 358
 359
 360 if __name__ == '__main__':
 361     parser = argparse.ArgumentParser(description='Create Kibana dashboards from data in elasticsearch')
 362     parser.add_argument('-e', '--elasticsearch-url', default='http://localhost:9200',
 363                         help='the url of elasticsearch, defaults to http://localhost:9200')
 364
 365     parser.add_argument('-js', '--generate_js_inputs', action='store_true',
 366                         help='Use this argument to generate javascript inputs for kibana landing page')
 367
 368     parser.add_argument('--js_path', default='/usr/share/nginx/html/kibana_dashboards/conf.js',
 369                         help='Path of javascript file with inputs for kibana landing page')
 370
 371     parser.add_argument('-k', '--kibana_url', default='https://testresults.opnfv.org/kibana/app/kibana',
 372                         help='The url of kibana for javascript inputs')
 373
 374     parser.add_argument('-u', '--elasticsearch-username', default=None,
 375                         help='The username with password for elasticsearch in format username:password')
 376
 377     args = parser.parse_args()
 378     base_elastic_url = args.elasticsearch_url
 379     generate_inputs = args.generate_js_inputs
 380     input_file_path = args.js_path
 381     kibana_url = args.kibana_url
 382     es_creds = args.elasticsearch_username
 383
 384     dashboards = construct_dashboards()
 385
 386     for kibana_dashboard in dashboards:
 387         kibana_dashboard.publish()
 388
 389     if generate_inputs:
 390         generate_js_inputs(input_file_path, kibana_url, dashboards)