6 from jinja2 import PackageLoader, Environment
8 from common import elastic_access
9 from common import logger_utils
10 from conf import testcases
11 from conf.config import APIConfig
13 logger = logger_utils.DashboardLogger('elastic2kibana').get
15 parser = argparse.ArgumentParser()
16 parser.add_argument("-c", "--config-file",
18 help="Config file location")
20 args = parser.parse_args()
21 CONF = APIConfig().parse(args.config_file)
22 base_elastic_url = CONF.elastic_url
23 generate_inputs = CONF.is_js
24 input_file_path = CONF.js_path
25 kibana_url = CONF.kibana_url
26 es_creds = CONF.elastic_creds
28 _installers = {'fuel', 'apex', 'compass', 'joid'}
30 env = Environment(loader=PackageLoader('elastic2kibana', 'templates'))
31 env.filters['jsonify'] = json.dumps
34 def dumps(self, items):
36 self.visualization[key] = json.dumps(self.visualization[key])
39 def dumps_2depth(self, key1, key2):
40 self.visualization[key1][key2] = json.dumps(self.visualization[key1][key2])
43 class Dashboard(dict):
44 def __init__(self, project_name, case_name, family, installer, pod, scenarios, visualization):
45 super(Dashboard, self).__init__()
46 self.project_name = project_name
47 self.case_name = case_name
49 self.installer = installer
51 self.scenarios = scenarios
52 self.visualization = visualization
53 self._visualization_title = None
54 self._kibana_visualizations = []
55 self._kibana_dashboard = None
56 self._create_visualizations()
59 def _create_visualizations(self):
60 for scenario in self.scenarios:
61 self._kibana_visualizations.append(Visualization(self.project_name,
68 self._visualization_title = self._kibana_visualizations[0].vis_state_title
70 def _publish_visualizations(self):
71 for visualization in self._kibana_visualizations:
72 url = urlparse.urljoin(base_elastic_url, '/.kibana/visualization/{}'.format(visualization.id))
73 logger.debug("publishing visualization '{}'".format(url))
74 # logger.error("_publish_visualization: %s" % visualization)
75 elastic_access.publish_docs(url, es_creds, visualization)
80 "project_name": self.project_name,
81 "case_name": self.case_name,
82 "installer": self.installer,
83 "metric": self._visualization_title,
86 "test_family": self.family,
87 "ids": [visualization.id for visualization in self._kibana_visualizations]
89 template = env.get_template('dashboard.json')
90 self.dashboard = json.loads(template.render(db=db))
91 dumps(self.dashboard, ['description', 'uiStateJSON', 'panelsJSON','optionsJSON'])
92 dumps_2depth(self.dashboard, 'kibanaSavedObjectMeta', 'searchSourceJSON')
93 self.id = self.dashboard['title'].replace(' ', '-').replace('/', '-')
97 url = urlparse.urljoin(base_elastic_url, '/.kibana/dashboard/{}'.format(self.id))
98 logger.debug("publishing dashboard '{}'".format(url))
99 #logger.error("dashboard: %s" % json.dumps(self.dashboard))
100 elastic_access.publish_docs(url, es_creds, self.dashboard)
103 self._publish_visualizations()
107 class VisStateBuilder(object):
108 def __init__(self, visualization):
109 super(VisStateBuilder, self).__init__()
110 self.visualization = visualization
113 name = self.visualization.get('name')
114 fields = self.visualization.get('fields')
121 "field": field.get("field")
125 template = env.get_template('{}.json'.format(name))
126 vis = template.render(aggs=aggs)
127 return json.loads(vis)
130 class Visualization(object):
131 def __init__(self, project_name, case_name, installer, pod, scenario, visualization):
134 1. filter created from
140 2. visualization state
141 field for y axis (metric) with type (avg, sum, etc.)
142 field for x axis (segment) with type (date_histogram)
146 super(Visualization, self).__init__()
147 visState = VisStateBuilder(visualization).build()
148 self.vis_state_title = visState['title']
151 "visState": json.dumps(visState),
153 "project_name": project_name,
154 "case_name": case_name,
155 "installer": installer,
156 "metric": self.vis_state_title,
162 template = env.get_template('visualization.json')
164 self.visualization = json.loads(template.render(vis=vis))
165 dumps(self.visualization, ['visState', 'description', 'uiStateJSON'])
166 dumps_2depth(self.visualization, 'kibanaSavedObjectMeta', 'searchSourceJSON')
167 self.id = self.visualization['title'].replace(' ', '-').replace('/', '-')
170 def _get_pods_and_scenarios(project_name, case_name, installer):
171 query_json = json.JSONEncoder().encode({
178 {"match": {"installer": {"query": installer, "type": "phrase"}}},
179 {"match": {"project_name": {"query": project_name, "type": "phrase"}}},
180 {"match": {"case_name": {"query": case_name, "type": "phrase"}}}
186 elastic_data = elastic_access.get_docs(urlparse.urljoin(base_elastic_url, '/test_results/mongo2elastic'),
190 pods_and_scenarios = {}
192 for data in elastic_data:
193 pod = data['pod_name']
194 if pod in pods_and_scenarios:
195 pods_and_scenarios[pod].add(data['scenario'])
197 pods_and_scenarios[pod] = {data['scenario']}
199 if 'all' in pods_and_scenarios:
200 pods_and_scenarios['all'].add(data['scenario'])
202 pods_and_scenarios['all'] = {data['scenario']}
204 return pods_and_scenarios
207 def construct_dashboards():
209 iterate over testcase and installer
210 1. get available pods for each testcase/installer pair
211 2. get available scenario for each testcase/installer/pod tuple
212 3. construct KibanaInput and append
214 :return: list of KibanaDashboards
216 kibana_dashboards = []
217 for project, case_dicts in testcases.testcases_yaml.items():
218 for case in case_dicts:
219 case_name = case.get('name')
220 visualizations = case.get('visualizations')
221 family = case.get('test_family')
222 for installer in _installers:
223 pods_and_scenarios = _get_pods_and_scenarios(project, case_name, installer)
224 for visualization in visualizations:
225 for pod, scenarios in pods_and_scenarios.iteritems():
226 kibana_dashboards.append(Dashboard(project,
233 return kibana_dashboards
236 def generate_js_inputs(js_file_path, kibana_url, dashboards):
238 for dashboard in dashboards:
239 dashboard_meta = dashboard.dashboard['metadata']
240 test_family = dashboard_meta['test_family']
241 test_label = dashboard_meta['label']
243 if test_family not in js_dict:
244 js_dict[test_family] = {}
246 js_test_family = js_dict[test_family]
248 if test_label not in js_test_family:
249 js_test_family[test_label] = {}
251 js_test_label = js_test_family[test_label]
253 if dashboard.installer not in js_test_label:
254 js_test_label[dashboard.installer] = {}
256 js_installer = js_test_label[dashboard.installer]
257 js_installer[dashboard.pod] = kibana_url + '#/dashboard/' + dashboard.id
259 with open(js_file_path, 'w+') as js_file_fdesc:
260 js_file_fdesc.write('var kibana_dashboard_links = ')
261 js_file_fdesc.write(str(js_dict).replace("u'", "'"))
265 dashboards = construct_dashboards()
267 for kibana_dashboard in dashboards:
268 kibana_dashboard.publish()
271 generate_js_inputs(input_file_path, kibana_url, dashboards)