Refactoring collector initialization in plan loader

- move ClassProps to module
- collectors should be array in plan definition

Change-Id: I37d85dbc977c91aa1bb81de45b5e1c41660e6d1f
Signed-off-by: Yujun Zhang <zhang.yujunz@zte.com.cn>

diff --git a/qtip/base/constant.py b/qtip/base/constant.py
index 3957204..100ec00 100644 (file)
--- a/qtip/base/constant.py
+++ b/qtip/base/constant.py
@@ -46,29 +46,5 @@ class SpecProp(BaseProp):
     WORKLOADS = 'workloads'
 
 
-class PlanProp(BaseProp):
-    # plan
-    INFO = 'info'
-
-    FACILITY = 'facility'
-    ENGINEER = 'engineer'
-
-    CONFIG = 'config'
-
-    DRIVER = 'driver'
-    COLLECTOR = 'collector'
-    REPORTER = 'reporter'
-
-    QPIS = 'QPIs'
-
-
-class CollectorProp(BaseProp):
-    LOGS = 'logs'
-    FILENAME = 'filename'
-    GREP = 'grep'
-    REGEX = 'regex'
-    CAPTURE = 'capture'
-
-
 class ReporterBaseProp(BaseProp):
     TRANSFORMER = 'transformer'

diff --git a/qtip/collector/base.py b/qtip/collector/base.py
index 2a25455..e7f9756 100644 (file)
--- a/qtip/collector/base.py
+++ b/qtip/collector/base.py
@@ -8,7 +8,20 @@
 ##############################################################################
 
 
+from qtip.base.constant import BaseProp
+
+
 class BaseCollector(object):
     """performance metrics collector"""
     def __init__(self, config):
         self._config = config
+
+
+class CollectorProp(BaseProp):
+    TYPE = 'type'
+    LOGS = 'logs'
+    FILENAME = 'filename'
+    GREP = 'grep'
+    REGEX = 'regex'
+    CAPTURE = 'capture'
+    PATHS = 'path'

diff --git a/qtip/collector/logfile.py b/qtip/collector/logfile.py
index 19780aa..6ed5aaf 100644 (file)
--- a/qtip/collector/logfile.py
+++ b/qtip/collector/logfile.py
@@ -9,16 +9,20 @@
 
 from base import BaseCollector
 
-from qtip.base.constant import CollectorProp as CProp
+from qtip.collector.base import CollectorProp as CProp
 from qtip.loader.file import FileLoader
 
 
 class LogfileCollector(BaseCollector):
     """collect performance metrics from log files"""
 
-    def __init__(self, config, paths=None):
+    TYPE = 'logfile'
+
+    def __init__(self, config, parent=None):
         super(LogfileCollector, self).__init__(config)
+        paths = [config[CProp.PATHS]] if CProp.PATHS in config else ['.']
         self.loader = FileLoader('.', paths)
+        self._parent = parent
 
     def collect(self):
         captured = {}

diff --git a/qtip/loader/module.py b/qtip/loader/module.py
new file mode 100644 (file)
index 0000000..05cb1b7
--- /dev/null
+++ b/qtip/loader/module.py
@@ -0,0 +1,19 @@
+##############################################################################
+# Copyright (c) 2017 ZTE Corp and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+
+from qtip.collector.logfile import LogfileCollector
+
+
+# TODO(yujunz) more elegant way to load module dynamically
+def load_collector(type_name):
+    if type_name == LogfileCollector.TYPE:
+        return LogfileCollector
+    else:
+        raise Exception("Invalid collector type: {}".format(type_name))

diff --git a/qtip/loader/plan.py b/qtip/loader/plan.py
index 9b5546e..0fd9ff5 100644 (file)
--- a/qtip/loader/plan.py
+++ b/qtip/loader/plan.py
@@ -8,8 +8,9 @@
 ##############################################################################
 
 
-from qtip.base.constant import PlanProp
-from qtip.collector.logfile import LogfileCollector
+from qtip.base.constant import BaseProp
+from qtip.collector.base import CollectorProp as CProp
+from qtip.loader.module import load_collector
 from qtip.loader.yaml_file import YamlFileLoader
 from qtip.loader.qpi import QPISpec
 
@@ -24,9 +25,26 @@ class Plan(YamlFileLoader):
     def __init__(self, name, paths=None):
         super(Plan, self).__init__(name, paths)
 
+        _config = self.content[PlanProp.CONFIG]
+
+        self.collectors = [load_collector(c[CProp.TYPE])(c, self)
+                           for c in _config[PlanProp.COLLECTORS]]
+
         self.qpis = [QPISpec(qpi, paths=paths)
                      for qpi in self.content[PlanProp.QPIS]]
-        _config = self.content[PlanProp.CONFIG]
 
-        # TODO(yujunz) create collector by name
-        self.collector = LogfileCollector(_config[PlanProp.COLLECTOR], paths)
+
+class PlanProp(BaseProp):
+    # plan
+    INFO = 'info'
+
+    FACILITY = 'facility'
+    ENGINEER = 'engineer'
+
+    CONFIG = 'config'
+
+    DRIVER = 'driver'
+    COLLECTORS = 'collectors'
+    REPORTER = 'reporter'
+
+    QPIS = 'QPIs'

diff --git a/tests/data/benchmarks/plan/doctor.yaml b/tests/data/benchmarks/plan/doctor.yaml
index 6c95077..f884c60 100644 (file)
--- a/tests/data/benchmarks/plan/doctor.yaml
+++ b/tests/data/benchmarks/plan/doctor.yaml
@@ -5,8 +5,8 @@ info:
   engineer: local
 config:
   driver: sample
-  collector:
-    - name: logfile
+  collectors:
+    - type: logfile
       logs:
         - filename: doctor_consumer.log
           # 2016-12-28 03:16:05,630 consumer.py 26 INFO   doctor consumer notified at 1482894965.63

diff --git a/tests/data/benchmarks/plan/fake-plan.yaml b/tests/data/benchmarks/plan/fake-plan.yaml
index 8887f66..511affd 100644 (file)
--- a/tests/data/benchmarks/plan/fake-plan.yaml
+++ b/tests/data/benchmarks/plan/fake-plan.yaml
@@ -4,7 +4,7 @@ config:
   facility: local
   engineer: local
   driver: sample
-  collector: logfile
+  collectors: []
   reporter: console
 QPIs:
   - fake-qpi.yaml

diff --git a/tests/unit/loader/plan_test.py b/tests/unit/loader/plan_test.py
index 32837f8..81fd0bd 100644 (file)
--- a/tests/unit/loader/plan_test.py
+++ b/tests/unit/loader/plan_test.py
@@ -9,8 +9,9 @@
 
 import pytest
 
-from qtip.base.constant import PlanProp
-from qtip.loader.plan import Plan, QPISpec
+from qtip.loader.plan import Plan
+from qtip.loader.plan import PlanProp
+from qtip.loader.plan import QPISpec
 
 
 def test_init(plan):