6 from jinja2 import PackageLoader, Environment
8 from common import elastic_access
9 from common import logger_utils
10 from conf import testcases
11 from conf.config import APIConfig
13 logger = logger_utils.DashboardLogger('elastic2kibana').get
15 parser = argparse.ArgumentParser()
16 parser.add_argument("-c", "--config-file",
18 help="Config file location")
20 args = parser.parse_args()
21 CONF = APIConfig().parse(args.config_file)
22 base_elastic_url = CONF.elastic_url
23 generate_inputs = CONF.is_js
24 input_file_path = CONF.js_path
25 kibana_url = CONF.kibana_url
26 es_creds = CONF.elastic_creds
28 _installers = {'fuel', 'apex', 'compass', 'joid'}
30 env = Environment(loader=PackageLoader('elastic2kibana', 'templates'))
31 env.filters['jsonify'] = json.dumps
34 class KibanaDashboard(dict):
35 def __init__(self, project_name, case_name, family, installer, pod, scenarios, visualization):
36 super(KibanaDashboard, self).__init__()
37 self.project_name = project_name
38 self.case_name = case_name
40 self.installer = installer
42 self.scenarios = scenarios
43 self.visualization = visualization
44 self._visualization_title = None
45 self._kibana_visualizations = []
46 self._kibana_dashboard = None
47 self._create_visualizations()
50 def _create_visualizations(self):
51 for scenario in self.scenarios:
52 self._kibana_visualizations.append(Visualization(self.project_name,
59 self._visualization_title = self._kibana_visualizations[0].vis_state_title
61 def _publish_visualizations(self):
62 for visualization in self._kibana_visualizations:
63 url = urlparse.urljoin(base_elastic_url, '/.kibana/visualization/{}'.format(visualization.id))
64 logger.debug("publishing visualization '{}'".format(url))
65 # logger.error("_publish_visualization: %s" % visualization)
66 elastic_access.publish_docs(url, es_creds, visualization)
68 def _construct_panels(self):
76 for visualization in self._kibana_visualizations:
78 "id": visualization.id,
79 "type": 'visualization',
80 "panelIndex": panel_index,
88 if column > max_columns:
91 return json.dumps(panels_json, separators=(',', ':'))
94 self['title'] = '{} {} {} {} {}'.format(self.project_name,
97 self._visualization_title,
99 self.id = self['title'].replace(' ', '-').replace('/', '-')
102 self['description'] = "Kibana dashboard for project_name '{}', case_name '{}', installer '{}', data '{}' and" \
103 " pod '{}'".format(self.project_name,
106 self._visualization_title,
108 self['panelsJSON'] = self._construct_panels()
109 self['optionsJSON'] = json.dumps({
112 separators=(',', ':'))
113 self['uiStateJSON'] = "{}"
115 self['timeRestore'] = False
116 self['kibanaSavedObjectMeta'] = {
117 'searchSourceJSON': json.dumps({
123 "analyze_wildcard": True
129 separators=(',', ':'))
132 label = self.case_name
133 if 'label' in self.visualization:
134 label += " %s" % self.visualization.get('label')
135 label += " %s" % self.visualization.get('name')
138 "test_family": self.family
142 url = urlparse.urljoin(base_elastic_url, '/.kibana/dashboard/{}'.format(self.id))
143 logger.debug("publishing dashboard '{}'".format(url))
144 elastic_access.publish_docs(url, es_creds, self)
147 self._publish_visualizations()
151 class VisStateBuilder(object):
152 def __init__(self, visualization):
153 super(VisStateBuilder, self).__init__()
154 self.visualization = visualization
157 name = self.visualization.get('name')
158 fields = self.visualization.get('fields')
165 "field": field.get("field")
169 template = env.get_template('{}.json'.format(name))
170 vis = template.render(aggs=aggs)
171 return json.loads(vis)
174 class Visualization(object):
175 def __init__(self, project_name, case_name, installer, pod, scenario, visualization):
178 1. filter created from
184 2. visualization state
185 field for y axis (metric) with type (avg, sum, etc.)
186 field for x axis (segment) with type (date_histogram)
190 super(Visualization, self).__init__()
191 visState = VisStateBuilder(visualization).build()
192 self.vis_state_title = visState['title']
195 "visState": json.dumps(visState),
197 "project_name": project_name,
198 "case_name": case_name,
199 "installer": installer,
200 "metric": self.vis_state_title,
206 template = env.get_template('visualization.json')
208 self.visualization = json.loads(template.render(vis=vis))
209 self._dumps(['visState', 'description', 'uiStateJSON'])
210 self._dumps_2deeps('kibanaSavedObjectMeta', 'searchSourceJSON')
211 self.id = self.visualization['title'].replace(' ', '-').replace('/', '-')
213 def _dumps(self, items):
215 self.visualization[key] = json.dumps(self.visualization[key])
217 def _dumps_2deeps(self, key1, key2):
218 self.visualization[key1][key2] = json.dumps(self.visualization[key1][key2])
221 def _get_pods_and_scenarios(project_name, case_name, installer):
222 query_json = json.JSONEncoder().encode({
229 {"match": {"installer": {"query": installer, "type": "phrase"}}},
230 {"match": {"project_name": {"query": project_name, "type": "phrase"}}},
231 {"match": {"case_name": {"query": case_name, "type": "phrase"}}}
237 elastic_data = elastic_access.get_docs(urlparse.urljoin(base_elastic_url, '/test_results/mongo2elastic'),
238 es_creds, query_json)
240 pods_and_scenarios = {}
242 for data in elastic_data:
243 pod = data['pod_name']
244 if pod in pods_and_scenarios:
245 pods_and_scenarios[pod].add(data['scenario'])
247 pods_and_scenarios[pod] = {data['scenario']}
249 if 'all' in pods_and_scenarios:
250 pods_and_scenarios['all'].add(data['scenario'])
252 pods_and_scenarios['all'] = {data['scenario']}
254 return pods_and_scenarios
257 def construct_dashboards():
259 iterate over testcase and installer
260 1. get available pods for each testcase/installer pair
261 2. get available scenario for each testcase/installer/pod tuple
262 3. construct KibanaInput and append
264 :return: list of KibanaDashboards
266 kibana_dashboards = []
267 for project, case_dicts in testcases.testcases_yaml.items():
268 for case in case_dicts:
269 case_name = case.get('name')
270 visualizations = case.get('visualizations')
271 family = case.get('test_family')
272 for installer in _installers:
273 pods_and_scenarios = _get_pods_and_scenarios(project, case_name, installer)
274 for visualization in visualizations:
275 for pod, scenarios in pods_and_scenarios.iteritems():
276 kibana_dashboards.append(KibanaDashboard(project,
283 return kibana_dashboards
286 def generate_js_inputs(js_file_path, kibana_url, dashboards):
288 for dashboard in dashboards:
289 dashboard_meta = dashboard['metadata']
290 test_family = dashboard_meta['test_family']
291 test_label = dashboard_meta['label']
293 if test_family not in js_dict:
294 js_dict[test_family] = {}
296 js_test_family = js_dict[test_family]
298 if test_label not in js_test_family:
299 js_test_family[test_label] = {}
301 js_test_label = js_test_family[test_label]
303 if dashboard.installer not in js_test_label:
304 js_test_label[dashboard.installer] = {}
306 js_installer = js_test_label[dashboard.installer]
307 js_installer[dashboard.pod] = kibana_url + '#/dashboard/' + dashboard.id
309 with open(js_file_path, 'w+') as js_file_fdesc:
310 js_file_fdesc.write('var kibana_dashboard_links = ')
311 js_file_fdesc.write(str(js_dict).replace("u'", "'"))
315 dashboards = construct_dashboards()
317 for kibana_dashboard in dashboards:
318 kibana_dashboard.publish()
321 generate_js_inputs(input_file_path, kibana_url, dashboards)