X-Git-Url: https://gerrit.opnfv.org/gerrit/gitweb?a=blobdiff_plain;f=utils%2Ftest%2Fdashboard%2Fdashboard%2Felastic2kibana%2Fmain.py;h=9ee894279cc8634262bba5cef1cf618af1177f35;hb=f9a4eeb385d42c60af5b68624da8ad2a46fbad3c;hp=f16879b936f26ebcad673a70960372b4f3b33b24;hpb=609bf2f9c9c6885b0a1c6acf292e33d80bf1c3f7;p=releng.git diff --git a/utils/test/dashboard/dashboard/elastic2kibana/main.py b/utils/test/dashboard/dashboard/elastic2kibana/main.py index f16879b93..9ee894279 100644 --- a/utils/test/dashboard/dashboard/elastic2kibana/main.py +++ b/utils/test/dashboard/dashboard/elastic2kibana/main.py @@ -3,12 +3,13 @@ import json import urlparse import argparse -from jinja2 import PackageLoader, Environment from common import elastic_access from common import logger_utils +from conf import config from conf import testcases -from conf.config import APIConfig +from dashboard_assembler import DashboardAssembler +from visualization_assembler import VisualizationAssembler logger = logger_utils.DashboardLogger('elastic2kibana').get @@ -18,304 +19,139 @@ parser.add_argument("-c", "--config-file", help="Config file location") args = parser.parse_args() -CONF = APIConfig().parse(args.config_file) -base_elastic_url = CONF.elastic_url -generate_inputs = CONF.is_js -input_file_path = CONF.js_path -kibana_url = CONF.kibana_url -es_creds = CONF.elastic_creds +CONF = config.APIConfig().parse(args.config_file) _installers = {'fuel', 'apex', 'compass', 'joid'} -env = Environment(loader=PackageLoader('elastic2kibana', 'templates')) -env.filters['jsonify'] = json.dumps - -class KibanaDashboard(dict): - def __init__(self, project_name, case_name, family, installer, pod, scenarios, visualization): - super(KibanaDashboard, self).__init__() - self.project_name = project_name - self.case_name = case_name - self.family = family - self.installer = installer - self.pod = pod - self.scenarios = scenarios - self.visualization = visualization - self._visualization_title = None - self._kibana_visualizations = [] - self._kibana_dashboard = None - self._create_visualizations() - self._create() - - def _create_visualizations(self): - for scenario in self.scenarios: - self._kibana_visualizations.append(Visualization(self.project_name, - self.case_name, - self.installer, - self.pod, - scenario, - self.visualization)) - - self._visualization_title = self._kibana_visualizations[0].vis_state_title - - def _publish_visualizations(self): - for visualization in self._kibana_visualizations: - url = urlparse.urljoin(base_elastic_url, '/.kibana/visualization/{}'.format(visualization.id)) - logger.debug("publishing visualization '{}'".format(url)) - # logger.error("_publish_visualization: %s" % visualization) - elastic_access.publish_docs(url, es_creds, visualization) - - def _construct_panels(self): - size_x = 6 - size_y = 3 - max_columns = 7 - column = 1 - row = 1 - panel_index = 1 - panels_json = [] - for visualization in self._kibana_visualizations: - panels_json.append({ - "id": visualization.id, - "type": 'visualization', - "panelIndex": panel_index, - "size_x": size_x, - "size_y": size_y, - "col": column, - "row": row - }) - panel_index += 1 - column += size_x - if column > max_columns: - column = 1 - row += size_y - return json.dumps(panels_json, separators=(',', ':')) - - def _create(self): - self['title'] = '{} {} {} {} {}'.format(self.project_name, - self.case_name, - self.installer, - self._visualization_title, - self.pod) - self.id = self['title'].replace(' ', '-').replace('/', '-') - - self['hits'] = 0 - self['description'] = "Kibana dashboard for project_name '{}', case_name '{}', installer '{}', data '{}' and" \ - " pod '{}'".format(self.project_name, - self.case_name, - self.installer, - self._visualization_title, - self.pod) - self['panelsJSON'] = self._construct_panels() - self['optionsJSON'] = json.dumps({ - "darkTheme": False - }, - separators=(',', ':')) - self['uiStateJSON'] = "{}" - self['scenario'] = 1 - self['timeRestore'] = False - self['kibanaSavedObjectMeta'] = { - 'searchSourceJSON': json.dumps({ - "filter": [ - { - "query": { - "query_string": { - "query": "*", - "analyze_wildcard": True - } - } - } - ] - }, - separators=(',', ':')) - } - - label = self.case_name - if 'label' in self.visualization: - label += " %s" % self.visualization.get('label') - label += " %s" % self.visualization.get('name') - self['metadata'] = { - "label": label, - "test_family": self.family - } - - def _publish(self): - url = urlparse.urljoin(base_elastic_url, '/.kibana/dashboard/{}'.format(self.id)) - logger.debug("publishing dashboard '{}'".format(url)) - elastic_access.publish_docs(url, es_creds, self) - - def publish(self): - self._publish_visualizations() - self._publish() - - -class VisStateBuilder(object): - def __init__(self, visualization): - super(VisStateBuilder, self).__init__() - self.visualization = visualization - - def build(self): - name = self.visualization.get('name') - fields = self.visualization.get('fields') - - aggs = [] - index = 1 - for field in fields: - aggs.append({ - "id": index, - "field": field.get("field") - }) - index += 1 - - template = env.get_template('{}.json'.format(name)) - vis = template.render(aggs=aggs) - return json.loads(vis) - - -class Visualization(object): - def __init__(self, project_name, case_name, installer, pod, scenario, visualization): - """ - We need two things - 1. filter created from - project_name - case_name - installer - pod - scenario - 2. visualization state - field for y axis (metric) with type (avg, sum, etc.) - field for x axis (segment) with type (date_histogram) - - :return: - """ - super(Visualization, self).__init__() - visState = VisStateBuilder(visualization).build() - self.vis_state_title = visState['title'] - - vis = { - "visState": json.dumps(visState), - "filters": { - "project_name": project_name, - "case_name": case_name, - "installer": installer, - "metric": self.vis_state_title, - "pod_name": pod, - "scenario": scenario - } - } - - template = env.get_template('visualization.json') - - self.visualization = json.loads(template.render(vis=vis)) - self._dumps(['visState', 'description', 'uiStateJSON']) - self._dumps_2deeps('kibanaSavedObjectMeta', 'searchSourceJSON') - self.id = self.visualization['title'].replace(' ', '-').replace('/', '-') - - def _dumps(self, items): - for key in items: - self.visualization[key] = json.dumps(self.visualization[key]) - - def _dumps_2deeps(self, key1, key2): - self.visualization[key1][key2] = json.dumps(self.visualization[key1][key2]) - - -def _get_pods_and_scenarios(project_name, case_name, installer): - query_json = json.JSONEncoder().encode({ - "query": { - "bool": { - "must": [ - {"match_all": {}} - ], - "filter": [ - {"match": {"installer": {"query": installer, "type": "phrase"}}}, - {"match": {"project_name": {"query": project_name, "type": "phrase"}}}, - {"match": {"case_name": {"query": case_name, "type": "phrase"}}} - ] - } - } - }) - - elastic_data = elastic_access.get_docs(urlparse.urljoin(base_elastic_url, '/test_results/mongo2elastic'), - es_creds, query_json) - - pods_and_scenarios = {} - - for data in elastic_data: - pod = data['pod_name'] - if pod in pods_and_scenarios: - pods_and_scenarios[pod].add(data['scenario']) - else: - pods_and_scenarios[pod] = {data['scenario']} - - if 'all' in pods_and_scenarios: - pods_and_scenarios['all'].add(data['scenario']) - else: - pods_and_scenarios['all'] = {data['scenario']} - - return pods_and_scenarios - - -def construct_dashboards(): - """ - iterate over testcase and installer - 1. get available pods for each testcase/installer pair - 2. get available scenario for each testcase/installer/pod tuple - 3. construct KibanaInput and append - - :return: list of KibanaDashboards - """ - kibana_dashboards = [] - for project, case_dicts in testcases.testcases_yaml.items(): - for case in case_dicts: - case_name = case.get('name') - visualizations = case.get('visualizations') - family = case.get('test_family') - for installer in _installers: - pods_and_scenarios = _get_pods_and_scenarios(project, case_name, installer) - for visualization in visualizations: - for pod, scenarios in pods_and_scenarios.iteritems(): - kibana_dashboards.append(KibanaDashboard(project, - case_name, - family, - installer, - pod, - scenarios, - visualization)) - return kibana_dashboards - - -def generate_js_inputs(js_file_path, kibana_url, dashboards): - js_dict = {} - for dashboard in dashboards: - dashboard_meta = dashboard['metadata'] - test_family = dashboard_meta['test_family'] - test_label = dashboard_meta['label'] - - if test_family not in js_dict: - js_dict[test_family] = {} - - js_test_family = js_dict[test_family] +class KibanaConstructor(object): + def __init__(self): + super(KibanaConstructor, self).__init__() + self.js_dict = {} + + def construct(self): + for project, case_dicts in testcases.testcases_yaml.items(): + for case in case_dicts: + self._construct_by_case(project, case) + return self + + def _construct_by_case(self, project, case): + case_name = case.get('name') + vis_ps = case.get('visualizations') + family = case.get('test_family') + for vis_p in vis_ps: + self._construct_by_vis(project, case_name, family, vis_p) + + def _construct_by_vis(self, project, case, family, vis_p): + for installer in _installers: + pods_and_scenarios = self._get_pods_and_scenarios(project, + case, + installer) + for pod, scenarios in pods_and_scenarios.iteritems(): + visualizations = self._construct_visualizations(project, + case, + installer, + pod, + scenarios, + vis_p, + CONF.es_url, + CONF.es_creds) + dashboard = DashboardAssembler(project, + case, + family, + installer, + pod, + visualizations, + CONF.es_url, + CONF.es_creds) + self._set_js_dict(case, + pod, + installer, + family, + vis_p.get('name'), + dashboard.id) + + @staticmethod + def _construct_visualizations(project, + case, + installer, + pod, + scenarios, + vis_p, + es_url, + es_creds): + visualizations = [] + for scenario in scenarios: + visualizations.append(VisualizationAssembler(project, + case, + installer, + pod, + scenario, + vis_p, + es_url, + es_creds)) + return visualizations + + def _set_js_dict(self, case, pod, installer, family, metric, id): + test_label = '{} {}'.format(case, metric) + if family not in self.js_dict: + self.js_dict[family] = {} + + js_test_family = self.js_dict[family] if test_label not in js_test_family: js_test_family[test_label] = {} js_test_label = js_test_family[test_label] - if dashboard.installer not in js_test_label: - js_test_label[dashboard.installer] = {} + if installer not in js_test_label: + js_test_label[installer] = {} + + js_installer = js_test_label[installer] + js_installer[pod] = CONF.kibana_url + '#/dashboard/' + id + + def config_js(self): + with open(CONF.js_path, 'w+') as conf_js_fdesc: + conf_js_fdesc.write('var kibana_dashboard_links = ') + conf_js_fdesc.write(str(self.js_dict).replace("u'", "'")) + + def _get_pods_and_scenarios(self, project, case, installer): + query = json.JSONEncoder().encode({ + "query": { + "bool": { + "must": [ + {"match_all": {}} + ], + "filter": [ + {"match": {"installer": installer}}, + {"match": {"project_name": project}}, + {"match": {"case_name": case}} + ] + } + } + }) - js_installer = js_test_label[dashboard.installer] - js_installer[dashboard.pod] = kibana_url + '#/dashboard/' + dashboard.id + elastic_data = elastic_access.get_docs( + urlparse.urljoin(CONF.es_url, '/test_results/mongo2elastic'), + CONF.es_creds, + query) - with open(js_file_path, 'w+') as js_file_fdesc: - js_file_fdesc.write('var kibana_dashboard_links = ') - js_file_fdesc.write(str(js_dict).replace("u'", "'")) + pods_and_scenarios = {} + for data in elastic_data: + pod = data['pod_name'] + if pod in pods_and_scenarios: + pods_and_scenarios[pod].add(data['scenario']) + else: + pods_and_scenarios[pod] = {data['scenario']} -def main(): - dashboards = construct_dashboards() + if 'all' in pods_and_scenarios: + pods_and_scenarios['all'].add(data['scenario']) + else: + pods_and_scenarios['all'] = {data['scenario']} - for kibana_dashboard in dashboards: - kibana_dashboard.publish() + return pods_and_scenarios - if generate_inputs: - generate_js_inputs(input_file_path, kibana_url, dashboards) + +def main(): + KibanaConstructor().construct().config_js()