X-Git-Url: https://gerrit.opnfv.org/gerrit/gitweb?a=blobdiff_plain;f=utils%2Ftest%2Fdashboard%2Fdashboard%2Felastic2kibana%2Fmain.py;h=112d2226e1966278586f145d6715af226f7954f6;hb=e73000209d46461b1196f560bf5c36bade020134;hp=ae5cbe8fa7c3efb5008d401dc05245cc347c9d5b;hpb=a366d65ccaca3c1aa8fdc9e396d708b623dda014;p=releng.git diff --git a/utils/test/dashboard/dashboard/elastic2kibana/main.py b/utils/test/dashboard/dashboard/elastic2kibana/main.py index ae5cbe8fa..112d2226e 100644 --- a/utils/test/dashboard/dashboard/elastic2kibana/main.py +++ b/utils/test/dashboard/dashboard/elastic2kibana/main.py @@ -1,14 +1,14 @@ #! /usr/bin/env python import json -import urlparse import argparse -from jinja2 import PackageLoader, Environment -from common import elastic_access -from common import logger_utils -from conf import testcases -from conf.config import APIConfig +from dashboard.common import elastic_access +from dashboard.common import logger_utils +from dashboard.conf import config +from dashboard.conf import testcases +from dashboard_assembler import DashboardAssembler +from visualization_assembler import VisualizationAssembler logger = logger_utils.DashboardLogger('elastic2kibana').get @@ -18,254 +18,138 @@ parser.add_argument("-c", "--config-file", help="Config file location") args = parser.parse_args() -CONF = APIConfig().parse(args.config_file) -base_elastic_url = CONF.elastic_url -generate_inputs = CONF.is_js -input_file_path = CONF.js_path -kibana_url = CONF.kibana_url -es_creds = CONF.elastic_creds +CONF = config.APIConfig().parse(args.config_file) _installers = {'fuel', 'apex', 'compass', 'joid'} -env = Environment(loader=PackageLoader('elastic2kibana', 'templates')) -env.filters['jsonify'] = json.dumps +class KibanaConstructor(object): + def __init__(self): + super(KibanaConstructor, self).__init__() + self.js_dict = {} + + def construct(self): + for project, case_dicts in testcases.testcases_yaml.items(): + for case in case_dicts: + self._construct_by_case(project, case) + return self + + def _construct_by_case(self, project, case): + case_name = case.get('name') + vis_ps = case.get('visualizations') + family = case.get('test_family') + for vis_p in vis_ps: + self._construct_by_vis(project, case_name, family, vis_p) + + def _construct_by_vis(self, project, case, family, vis_p): + for installer in _installers: + pods_and_scenarios = self._get_pods_and_scenarios(project, + case, + installer) + for pod, scenarios in pods_and_scenarios.iteritems(): + visualizations = self._construct_visualizations(project, + case, + installer, + pod, + scenarios, + vis_p, + CONF.es_url, + CONF.es_creds) + dashboard = DashboardAssembler(project, + case, + family, + installer, + pod, + visualizations, + CONF.es_url, + CONF.es_creds) + self._set_js_dict(case, + pod, + installer, + family, + vis_p.get('name'), + dashboard.id) + + @staticmethod + def _construct_visualizations(project, + case, + installer, + pod, + scenarios, + vis_p, + es_url, + es_creds): + visualizations = [] + for scenario in scenarios: + visualizations.append(VisualizationAssembler(project, + case, + installer, + pod, + scenario, + vis_p, + es_url, + es_creds)) + return visualizations + + def _set_js_dict(self, case, pod, installer, family, metric, id): + test_label = '{} {}'.format(case, metric) + if family not in self.js_dict: + self.js_dict[family] = {} + + js_test_family = self.js_dict[family] -def dumps(self, items): - for key in items: - self.visualization[key] = json.dumps(self.visualization[key]) - - -def dumps_2depth(self, key1, key2): - self.visualization[key1][key2] = json.dumps(self.visualization[key1][key2]) - + if test_label not in js_test_family: + js_test_family[test_label] = {} -class Dashboard(dict): - def __init__(self, project_name, case_name, family, installer, pod, scenarios, visualization): - super(Dashboard, self).__init__() - self.project_name = project_name - self.case_name = case_name - self.family = family - self.installer = installer - self.pod = pod - self.scenarios = scenarios - self.visualization = visualization - self._visualization_title = None - self._kibana_visualizations = [] - self._kibana_dashboard = None - self._create_visualizations() - self._create() + js_test_label = js_test_family[test_label] - def _create_visualizations(self): - for scenario in self.scenarios: - self._kibana_visualizations.append(Visualization(self.project_name, - self.case_name, - self.installer, - self.pod, - scenario, - self.visualization)) + if installer not in js_test_label: + js_test_label[installer] = {} - self._visualization_title = self._kibana_visualizations[0].vis_state_title + js_installer = js_test_label[installer] + js_installer[pod] = CONF.kibana_url + '#/dashboard/' + id - def _publish_visualizations(self): - for visualization in self._kibana_visualizations: - url = urlparse.urljoin(base_elastic_url, '/.kibana/visualization/{}'.format(visualization.id)) - logger.debug("publishing visualization '{}'".format(url)) - # logger.error("_publish_visualization: %s" % visualization) - elastic_access.publish_docs(url, es_creds, visualization) + def config_js(self): + with open(CONF.js_path, 'w+') as conf_js_fdesc: + conf_js_fdesc.write('var kibana_dashboard_links = ') + conf_js_fdesc.write(str(self.js_dict).replace("u'", "'")) - def _create(self): - db = { + def _get_pods_and_scenarios(self, project, case, installer): + query = json.JSONEncoder().encode({ "query": { - "project_name": self.project_name, - "case_name": self.case_name, - "installer": self.installer, - "metric": self._visualization_title, - "pod": self.pod - }, - "test_family": self.family, - "ids": [visualization.id for visualization in self._kibana_visualizations] - } - template = env.get_template('dashboard.json') - self.dashboard = json.loads(template.render(db=db)) - dumps(self.dashboard, ['description', 'uiStateJSON', 'panelsJSON','optionsJSON']) - dumps_2depth(self.dashboard, 'kibanaSavedObjectMeta', 'searchSourceJSON') - self.id = self.dashboard['title'].replace(' ', '-').replace('/', '-') - - - def _publish(self): - url = urlparse.urljoin(base_elastic_url, '/.kibana/dashboard/{}'.format(self.id)) - logger.debug("publishing dashboard '{}'".format(url)) - #logger.error("dashboard: %s" % json.dumps(self.dashboard)) - elastic_access.publish_docs(url, es_creds, self.dashboard) - - def publish(self): - self._publish_visualizations() - self._publish() - - -class VisStateBuilder(object): - def __init__(self, visualization): - super(VisStateBuilder, self).__init__() - self.visualization = visualization - - def build(self): - name = self.visualization.get('name') - fields = self.visualization.get('fields') - - aggs = [] - index = 1 - for field in fields: - aggs.append({ - "id": index, - "field": field.get("field") - }) - index += 1 - - template = env.get_template('{}.json'.format(name)) - vis = template.render(aggs=aggs) - return json.loads(vis) - - -class Visualization(object): - def __init__(self, project_name, case_name, installer, pod, scenario, visualization): - """ - We need two things - 1. filter created from - project_name - case_name - installer - pod - scenario - 2. visualization state - field for y axis (metric) with type (avg, sum, etc.) - field for x axis (segment) with type (date_histogram) - - :return: - """ - super(Visualization, self).__init__() - visState = VisStateBuilder(visualization).build() - self.vis_state_title = visState['title'] - - vis = { - "visState": json.dumps(visState), - "filters": { - "project_name": project_name, - "case_name": case_name, - "installer": installer, - "metric": self.vis_state_title, - "pod_name": pod, - "scenario": scenario - } - } - - template = env.get_template('visualization.json') - - self.visualization = json.loads(template.render(vis=vis)) - dumps(self.visualization, ['visState', 'description', 'uiStateJSON']) - dumps_2depth(self.visualization, 'kibanaSavedObjectMeta', 'searchSourceJSON') - self.id = self.visualization['title'].replace(' ', '-').replace('/', '-') - - -def _get_pods_and_scenarios(project_name, case_name, installer): - query_json = json.JSONEncoder().encode({ - "query": { - "bool": { - "must": [ - {"match_all": {}} - ], - "filter": [ - {"match": {"installer": {"query": installer, "type": "phrase"}}}, - {"match": {"project_name": {"query": project_name, "type": "phrase"}}}, - {"match": {"case_name": {"query": case_name, "type": "phrase"}}} - ] + "bool": { + "must": [ + {"match_all": {}} + ], + "filter": [ + {"match": {"installer": installer}}, + {"match": {"project_name": project}}, + {"match": {"case_name": case}} + ] + } } - } - }) - - elastic_data = elastic_access.get_docs(urlparse.urljoin(base_elastic_url, '/test_results/mongo2elastic'), - es_creds, - query_json) - - pods_and_scenarios = {} - - for data in elastic_data: - pod = data['pod_name'] - if pod in pods_and_scenarios: - pods_and_scenarios[pod].add(data['scenario']) - else: - pods_and_scenarios[pod] = {data['scenario']} - - if 'all' in pods_and_scenarios: - pods_and_scenarios['all'].add(data['scenario']) - else: - pods_and_scenarios['all'] = {data['scenario']} - - return pods_and_scenarios - - -def construct_dashboards(): - """ - iterate over testcase and installer - 1. get available pods for each testcase/installer pair - 2. get available scenario for each testcase/installer/pod tuple - 3. construct KibanaInput and append + }) - :return: list of KibanaDashboards - """ - kibana_dashboards = [] - for project, case_dicts in testcases.testcases_yaml.items(): - for case in case_dicts: - case_name = case.get('name') - visualizations = case.get('visualizations') - family = case.get('test_family') - for installer in _installers: - pods_and_scenarios = _get_pods_and_scenarios(project, case_name, installer) - for visualization in visualizations: - for pod, scenarios in pods_and_scenarios.iteritems(): - kibana_dashboards.append(Dashboard(project, - case_name, - family, - installer, - pod, - scenarios, - visualization)) - return kibana_dashboards + elastic_data = elastic_access.get_docs(CONF.index_url, + CONF.es_creds, + query) + pods_and_scenarios = {} -def generate_js_inputs(js_file_path, kibana_url, dashboards): - js_dict = {} - for dashboard in dashboards: - dashboard_meta = dashboard.dashboard['metadata'] - test_family = dashboard_meta['test_family'] - test_label = dashboard_meta['label'] + for data in elastic_data: + pod = data['pod_name'] + if pod in pods_and_scenarios: + pods_and_scenarios[pod].add(data['scenario']) + else: + pods_and_scenarios[pod] = {data['scenario']} - if test_family not in js_dict: - js_dict[test_family] = {} + if 'all' in pods_and_scenarios: + pods_and_scenarios['all'].add(data['scenario']) + else: + pods_and_scenarios['all'] = {data['scenario']} - js_test_family = js_dict[test_family] - - if test_label not in js_test_family: - js_test_family[test_label] = {} - - js_test_label = js_test_family[test_label] - - if dashboard.installer not in js_test_label: - js_test_label[dashboard.installer] = {} - - js_installer = js_test_label[dashboard.installer] - js_installer[dashboard.pod] = kibana_url + '#/dashboard/' + dashboard.id - - with open(js_file_path, 'w+') as js_file_fdesc: - js_file_fdesc.write('var kibana_dashboard_links = ') - js_file_fdesc.write(str(js_dict).replace("u'", "'")) + return pods_and_scenarios def main(): - dashboards = construct_dashboards() - - for kibana_dashboard in dashboards: - kibana_dashboard.publish() - - if generate_inputs: - generate_js_inputs(input_file_path, kibana_url, dashboards) + KibanaConstructor().construct().config_js()