utils/test/dashboard/dashboard/elastic2kibana/main.py

   1 #! /usr/bin/env python
   2 import json
   3 import urlparse
   4
   5 import argparse
   6
   7 from common import elastic_access
   8 from common import logger_utils
   9 from conf import config
  10 from conf import testcases
  11 from dashboard_assembler import DashboardAssembler
  12 from visualization_assembler import VisualizationAssembler
  13
  14 logger = logger_utils.DashboardLogger('elastic2kibana').get
  15
  16 parser = argparse.ArgumentParser()
  17 parser.add_argument("-c", "--config-file",
  18                     dest='config_file',
  19                     help="Config file location")
  20
  21 args = parser.parse_args()
  22 CONF = config.APIConfig().parse(args.config_file)
  23
  24 _installers = {'fuel', 'apex', 'compass', 'joid'}
  25
  26
  27 class KibanaConstructor(object):
  28     def __init__(self):
  29         super(KibanaConstructor, self).__init__()
  30         self.js_dict = {}
  31
  32     def construct(self):
  33         for project, case_dicts in testcases.testcases_yaml.items():
  34             for case in case_dicts:
  35                 self._construct_by_case(project, case)
  36         return self
  37
  38     def _construct_by_case(self, project, case):
  39         case_name = case.get('name')
  40         vis_ps = case.get('visualizations')
  41         family = case.get('test_family')
  42         for vis_p in vis_ps:
  43             self._construct_by_vis(project, case_name, family, vis_p)
  44
  45     def _construct_by_vis(self, project, case, family, vis_p):
  46         for installer in _installers:
  47             pods_and_scenarios = self._get_pods_and_scenarios(project,
  48                                                               case,
  49                                                               installer)
  50             for pod, scenarios in pods_and_scenarios.iteritems():
  51                 visualizations = self._construct_visualizations(project,
  52                                                                 case,
  53                                                                 installer,
  54                                                                 pod,
  55                                                                 scenarios,
  56                                                                 vis_p,
  57                                                                 CONF.es_url,
  58                                                                 CONF.es_creds)
  59                 dashboard = DashboardAssembler(project,
  60                                                case,
  61                                                family,
  62                                                installer,
  63                                                pod,
  64                                                visualizations,
  65                                                CONF.es_url,
  66                                                CONF.es_creds)
  67                 if CONF.is_js:
  68                     self._set_js_dict(case,
  69                                       pod,
  70                                       installer,
  71                                       family,
  72                                       vis_p.get('name'),
  73                                       dashboard.id)
  74
  75     @staticmethod
  76     def _construct_visualizations(project,
  77                                   case,
  78                                   installer,
  79                                   pod,
  80                                   scenarios,
  81                                   vis_p,
  82                                   es_url,
  83                                   es_creds):
  84         visualizations = []
  85         for scenario in scenarios:
  86             visualizations.append(VisualizationAssembler(project,
  87                                                          case,
  88                                                          installer,
  89                                                          pod,
  90                                                          scenario,
  91                                                          vis_p,
  92                                                          es_url,
  93                                                          es_creds))
  94         return visualizations
  95
  96     def _set_js_dict(self, case, pod, installer, family, metric, id):
  97         test_label = '{} {}'.format(case, metric)
  98         if family not in self.js_dict:
  99             self.js_dict[family] = {}
 100
 101         js_test_family = self.js_dict[family]
 102
 103         if test_label not in js_test_family:
 104             js_test_family[test_label] = {}
 105
 106         js_test_label = js_test_family[test_label]
 107
 108         if installer not in js_test_label:
 109             js_test_label[installer] = {}
 110
 111         js_installer = js_test_label[installer]
 112         js_installer[pod] = CONF.kibana_url + '#/dashboard/' + id
 113
 114     def config_js(self):
 115         if CONF.is_js:
 116             with open(CONF.js_path, 'w+') as conf_js_fdesc:
 117                 conf_js_fdesc.write('var kibana_dashboard_links = ')
 118                 conf_js_fdesc.write(str(self.js_dict).replace("u'", "'"))
 119
 120     def _get_pods_and_scenarios(self, project, case, installer):
 121         query = json.JSONEncoder().encode({
 122             "query": {
 123                 "bool": {
 124                     "must": [
 125                         {"match_all": {}}
 126                     ],
 127                     "filter": [
 128                         {"match": {"installer": installer}},
 129                         {"match": {"project_name": project}},
 130                         {"match": {"case_name": case}}
 131                     ]
 132                 }
 133             }
 134         })
 135
 136         elastic_data = elastic_access.get_docs(
 137             urlparse.urljoin(CONF.es_url, '/test_results/mongo2elastic'),
 138             CONF.es_creds,
 139             query)
 140
 141         pods_and_scenarios = {}
 142
 143         for data in elastic_data:
 144             pod = data['pod_name']
 145             if pod in pods_and_scenarios:
 146                 pods_and_scenarios[pod].add(data['scenario'])
 147             else:
 148                 pods_and_scenarios[pod] = {data['scenario']}
 149
 150             if 'all' in pods_and_scenarios:
 151                 pods_and_scenarios['all'].add(data['scenario'])
 152             else:
 153                 pods_and_scenarios['all'] = {data['scenario']}
 154
 155         return pods_and_scenarios
 156
 157
 158 def main():
 159     KibanaConstructor().construct().config_js()