utils/test/dashboard/dashboard/elastic2kibana/main.py

   1 #! /usr/bin/env python
   2 import json
   3
   4 import argparse
   5
   6 from dashboard.common import elastic_access
   7 from dashboard.common import logger_utils
   8 from dashboard.conf import config
   9 from dashboard.conf import testcases
  10 from dashboard_assembler import DashboardAssembler
  11 from visualization_assembler import VisualizationAssembler
  12
  13 logger = logger_utils.DashboardLogger('elastic2kibana').get
  14
  15 parser = argparse.ArgumentParser()
  16 parser.add_argument("-c", "--config-file",
  17                     dest='config_file',
  18                     help="Config file location")
  19
  20 args = parser.parse_args()
  21 CONF = config.APIConfig().parse(args.config_file)
  22
  23 _installers = {'fuel', 'apex', 'compass', 'joid'}
  24
  25
  26 class KibanaConstructor(object):
  27     def __init__(self):
  28         super(KibanaConstructor, self).__init__()
  29         self.js_dict = {}
  30
  31     def construct(self):
  32         for project, case_dicts in testcases.testcases_yaml.items():
  33             for case in case_dicts:
  34                 self._construct_by_case(project, case)
  35         return self
  36
  37     def _construct_by_case(self, project, case):
  38         case_name = case.get('name')
  39         vis_ps = case.get('visualizations')
  40         family = case.get('test_family')
  41         for vis_p in vis_ps:
  42             self._construct_by_vis(project, case_name, family, vis_p)
  43
  44     def _construct_by_vis(self, project, case, family, vis_p):
  45         for installer in _installers:
  46             pods_and_scenarios = self._get_pods_and_scenarios(project,
  47                                                               case,
  48                                                               installer)
  49             for pod, scenarios in pods_and_scenarios.iteritems():
  50                 visualizations = self._construct_visualizations(project,
  51                                                                 case,
  52                                                                 installer,
  53                                                                 pod,
  54                                                                 scenarios,
  55                                                                 vis_p,
  56                                                                 CONF.es_url,
  57                                                                 CONF.es_creds)
  58                 dashboard = DashboardAssembler(project,
  59                                                case,
  60                                                family,
  61                                                installer,
  62                                                pod,
  63                                                visualizations,
  64                                                CONF.es_url,
  65                                                CONF.es_creds)
  66                 self._set_js_dict(case,
  67                                   pod,
  68                                   installer,
  69                                   family,
  70                                   vis_p.get('name'),
  71                                   dashboard.id)
  72
  73     @staticmethod
  74     def _construct_visualizations(project,
  75                                   case,
  76                                   installer,
  77                                   pod,
  78                                   scenarios,
  79                                   vis_p,
  80                                   es_url,
  81                                   es_creds):
  82         visualizations = []
  83         for scenario in scenarios:
  84             visualizations.append(VisualizationAssembler(project,
  85                                                          case,
  86                                                          installer,
  87                                                          pod,
  88                                                          scenario,
  89                                                          vis_p,
  90                                                          es_url,
  91                                                          es_creds))
  92         return visualizations
  93
  94     def _set_js_dict(self, case, pod, installer, family, metric, id):
  95         test_label = '{} {}'.format(case, metric)
  96         if family not in self.js_dict:
  97             self.js_dict[family] = {}
  98
  99         js_test_family = self.js_dict[family]
 100
 101         if test_label not in js_test_family:
 102             js_test_family[test_label] = {}
 103
 104         js_test_label = js_test_family[test_label]
 105
 106         if installer not in js_test_label:
 107             js_test_label[installer] = {}
 108
 109         js_installer = js_test_label[installer]
 110         js_installer[pod] = CONF.kibana_url + '#/dashboard/' + id
 111
 112     def config_js(self):
 113         with open(CONF.js_path, 'w+') as conf_js_fdesc:
 114             conf_js_fdesc.write('var kibana_dashboard_links = ')
 115             conf_js_fdesc.write(str(self.js_dict).replace("u'", "'"))
 116
 117     def _get_pods_and_scenarios(self, project, case, installer):
 118         query = json.JSONEncoder().encode({
 119             "query": {
 120                 "bool": {
 121                     "must": [
 122                         {"match_all": {}}
 123                     ],
 124                     "filter": [
 125                         {"match": {"installer": installer}},
 126                         {"match": {"project_name": project}},
 127                         {"match": {"case_name": case}}
 128                     ]
 129                 }
 130             }
 131         })
 132
 133         elastic_data = elastic_access.get_docs(CONF.index_url,
 134                                                CONF.es_creds,
 135                                                query)
 136
 137         pods_and_scenarios = {}
 138
 139         for data in elastic_data:
 140             pod = data['pod_name']
 141             if pod in pods_and_scenarios:
 142                 pods_and_scenarios[pod].add(data['scenario'])
 143             else:
 144                 pods_and_scenarios[pod] = {data['scenario']}
 145
 146             if 'all' in pods_and_scenarios:
 147                 pods_and_scenarios['all'].add(data['scenario'])
 148             else:
 149                 pods_and_scenarios['all'] = {data['scenario']}
 150
 151         return pods_and_scenarios
 152
 153
 154 def main():
 155     KibanaConstructor().construct().config_js()