utils/test/dashboard/dashboard/elastic2kibana/main.py

   1 #! /usr/bin/env python
   2 import json
   3 import urlparse
   4
   5 import argparse
   6
   7 from common import elastic_access
   8 from common import logger_utils
   9 from conf import config
  10 from conf import testcases
  11 from dashboard_assembler import DashboardAssembler
  12 from visualization_assembler import VisualizationAssembler
  13
  14 logger = logger_utils.DashboardLogger('elastic2kibana').get
  15
  16 parser = argparse.ArgumentParser()
  17 parser.add_argument("-c", "--config-file",
  18                     dest='config_file',
  19                     help="Config file location")
  20
  21 args = parser.parse_args()
  22 CONF = config.APIConfig().parse(args.config_file)
  23
  24 _installers = {'fuel', 'apex', 'compass', 'joid'}
  25
  26
  27 class KibanaConstructor(object):
  28     def __init__(self):
  29         super(KibanaConstructor, self).__init__()
  30         self.js_dict = {}
  31
  32     def construct(self):
  33         for project, case_dicts in testcases.testcases_yaml.items():
  34             for case in case_dicts:
  35                 self._construct_by_case(project, case)
  36         return self
  37
  38     def _construct_by_case(self, project, case):
  39         case_name = case.get('name')
  40         vis_ps = case.get('visualizations')
  41         family = case.get('test_family')
  42         for vis_p in vis_ps:
  43             self._construct_by_vis(project, case_name, family, vis_p)
  44
  45     def _construct_by_vis(self, project, case, family, vis_p):
  46         for installer in _installers:
  47             pods_and_scenarios = self._get_pods_and_scenarios(project,
  48                                                               case,
  49                                                               installer)
  50             for pod, scenarios in pods_and_scenarios.iteritems():
  51                 visualizations = self._construct_visualizations(project,
  52                                                                 case,
  53                                                                 installer,
  54                                                                 pod,
  55                                                                 scenarios,
  56                                                                 vis_p,
  57                                                                 CONF.es_url,
  58                                                                 CONF.es_creds)
  59                 dashboard = DashboardAssembler(project,
  60                                                case,
  61                                                family,
  62                                                installer,
  63                                                pod,
  64                                                visualizations,
  65                                                CONF.es_url,
  66                                                CONF.es_creds)
  67                 self._set_js_dict(case,
  68                                   pod,
  69                                   installer,
  70                                   family,
  71                                   vis_p.get('name'),
  72                                   dashboard.id)
  73
  74     @staticmethod
  75     def _construct_visualizations(project,
  76                                   case,
  77                                   installer,
  78                                   pod,
  79                                   scenarios,
  80                                   vis_p,
  81                                   es_url,
  82                                   es_creds):
  83         visualizations = []
  84         for scenario in scenarios:
  85             visualizations.append(VisualizationAssembler(project,
  86                                                          case,
  87                                                          installer,
  88                                                          pod,
  89                                                          scenario,
  90                                                          vis_p,
  91                                                          es_url,
  92                                                          es_creds))
  93         return visualizations
  94
  95     def _set_js_dict(self, case, pod, installer, family, metric, id):
  96         test_label = '{} {}'.format(case, metric)
  97         if family not in self.js_dict:
  98             self.js_dict[family] = {}
  99
 100         js_test_family = self.js_dict[family]
 101
 102         if test_label not in js_test_family:
 103             js_test_family[test_label] = {}
 104
 105         js_test_label = js_test_family[test_label]
 106
 107         if installer not in js_test_label:
 108             js_test_label[installer] = {}
 109
 110         js_installer = js_test_label[installer]
 111         js_installer[pod] = CONF.kibana_url + '#/dashboard/' + id
 112
 113     def config_js(self):
 114         with open(CONF.js_path, 'w+') as conf_js_fdesc:
 115             conf_js_fdesc.write('var kibana_dashboard_links = ')
 116             conf_js_fdesc.write(str(self.js_dict).replace("u'", "'"))
 117
 118     def _get_pods_and_scenarios(self, project, case, installer):
 119         query = json.JSONEncoder().encode({
 120             "query": {
 121                 "bool": {
 122                     "must": [
 123                         {"match_all": {}}
 124                     ],
 125                     "filter": [
 126                         {"match": {"installer": installer}},
 127                         {"match": {"project_name": project}},
 128                         {"match": {"case_name": case}}
 129                     ]
 130                 }
 131             }
 132         })
 133
 134         elastic_data = elastic_access.get_docs(
 135             urlparse.urljoin(CONF.es_url, '/test_results/mongo2elastic'),
 136             CONF.es_creds,
 137             query)
 138
 139         pods_and_scenarios = {}
 140
 141         for data in elastic_data:
 142             pod = data['pod_name']
 143             if pod in pods_and_scenarios:
 144                 pods_and_scenarios[pod].add(data['scenario'])
 145             else:
 146                 pods_and_scenarios[pod] = {data['scenario']}
 147
 148             if 'all' in pods_and_scenarios:
 149                 pods_and_scenarios['all'].add(data['scenario'])
 150             else:
 151                 pods_and_scenarios['all'] = {data['scenario']}
 152
 153         return pods_and_scenarios
 154
 155
 156 def main():
 157     KibanaConstructor().construct().config_js()