Make 'Scenario' classes plugable

This patch makes yardstick.benchmark.scenario.base:Scenario classes
plugable.

A new entry point is added to the setup. This entry point could be
extended in other plugin projects to add new Scenario classes.

E.g.: take a look at [1]. This is a Yardstick plugin example project.
Clone the project and execute, from the project directory:

  $ sudo -EH python setup.py install

This will add a new module to python ('yardstick-new-plugin') and a
new Scenario class.

Now list the scenarios in Yardstick:

  $ yardstick scenario list
    ...
    | SpecCPU2006_for_VM    | Spec CPU2006 benchmark for Virtual Machine |
    | SpecCPU2006           | Spec CPU2006 benchmark                     |
    | Dummy2                | Execute Dummy (v2!) echo                   |
    +-----------------------+--------------------------------------------+

[1] https://github.com/ralonsoh/yardstick_new_plugin

JIRA: YARDSTICK-910

Change-Id: Ib70ee9bf4dc7ff91d1dd6377317b313288e36bff
Signed-off-by: Rodolfo Alonso Hernandez <rodolfo.alonso.hernandez@intel.com>

diff --git a/setup.py b/setup.py
index 7f6571d..881ef92 100755 (executable)
--- a/setup.py
+++ b/setup.py
@@ -53,6 +53,7 @@ setup(
             'yardstick=yardstick.main:main',
             'yardstick-plot=yardstick.plot.plotter:main [plot]'
         ],
+        'yardstick.scenario': []
     },
     scripts=[
         'tools/yardstick-img-modify',

diff --git a/tests/unit/benchmark/scenarios/test_base.py b/tests/unit/benchmark/scenarios/test_base.py
index 78e3429..a95e6bc 100644 (file)
--- a/tests/unit/benchmark/scenarios/test_base.py
+++ b/tests/unit/benchmark/scenarios/test_base.py
@@ -51,3 +51,56 @@ class ScenarioTestCase(unittest.TestCase):
             pass
 
         self.assertEqual(str(None), DummyScenario.get_description())
+
+    def test_get_types(self):
+        scenario_names = set(
+            scenario.__scenario_type__ for scenario in
+            base.Scenario.get_types() if hasattr(scenario,
+                                                 '__scenario_type__'))
+        existing_scenario_class_names = {
+            'Iperf3', 'CACHEstat', 'SpecCPU2006', 'Dummy', 'NSPerf', 'Parser'}
+        self.assertTrue(existing_scenario_class_names.issubset(scenario_names))
+
+    def test_get_cls_existing_scenario(self):
+        scenario_name = 'NSPerf'
+        scenario = base.Scenario.get_cls(scenario_name)
+        self.assertEqual(scenario_name, scenario.__scenario_type__)
+
+    def test_get_cls_non_existing_scenario(self):
+        wrong_scenario_name = 'Non-existing-scenario'
+        with self.assertRaises(RuntimeError) as exc:
+            base.Scenario.get_cls(wrong_scenario_name)
+        self.assertEqual('No such scenario type %s' % wrong_scenario_name,
+                         str(exc.exception))
+
+    def test_get_existing_scenario(self):
+        scenario_name = 'NSPerf'
+        scenario_module = ('yardstick.benchmark.scenarios.networking.'
+                           'vnf_generic.NetworkServiceTestCase')
+        self.assertEqual(scenario_module, base.Scenario.get(scenario_name))
+
+    def test_get_non_existing_scenario(self):
+        wrong_scenario_name = 'Non-existing-scenario'
+        with self.assertRaises(RuntimeError) as exc:
+            base.Scenario.get(wrong_scenario_name)
+        self.assertEqual('No such scenario type %s' % wrong_scenario_name,
+                         str(exc.exception))
+
+
+class IterScenarioClassesTestCase(unittest.TestCase):
+
+    def test_no_scenario_type_defined(self):
+        some_existing_scenario_class_names = [
+            'Iperf3', 'CACHEstat', 'SpecCPU2006', 'Dummy', 'NSPerf', 'Parser']
+        scenario_types = [scenario.__scenario_type__ for scenario
+                          in base._iter_scenario_classes()]
+        for class_name in some_existing_scenario_class_names:
+            self.assertIn(class_name, scenario_types)
+
+    def test_scenario_type_defined(self):
+        some_existing_scenario_class_names = [
+            'Iperf3', 'CACHEstat', 'SpecCPU2006', 'Dummy', 'NSPerf', 'Parser']
+        for class_name in some_existing_scenario_class_names:
+            scenario_class = next(base._iter_scenario_classes(
+                scenario_type=class_name))
+            self.assertEqual(class_name, scenario_class.__scenario_type__)

diff --git a/yardstick/benchmark/scenarios/base.py b/yardstick/benchmark/scenarios/base.py
index 7af8583..10a7288 100644 (file)
--- a/yardstick/benchmark/scenarios/base.py
+++ b/yardstick/benchmark/scenarios/base.py
@@ -16,20 +16,34 @@
 # yardstick comment: this is a modified copy of
 # rally/rally/benchmark/scenarios/base.py
 
-""" Scenario base class
-"""
+from stevedore import extension
 
-from __future__ import absolute_import
 import yardstick.common.utils as utils
 
 
+def _iter_scenario_classes(scenario_type=None):
+    """Generator over all 'Scenario' subclasses
+
+    This function will iterate over all 'Scenario' subclasses defined in this
+    project and will load any class introduced by any installed plugin project,
+    defined in 'entry_points' section, under 'yardstick.scenarios' subsection.
+    """
+    extension.ExtensionManager(namespace='yardstick.scenarios',
+                               invoke_on_load=False)
+    for scenario in utils.itersubclasses(Scenario):
+        if not scenario_type:
+            yield scenario
+        elif getattr(scenario, '__scenario_type__', None) == scenario_type:
+            yield scenario
+
+
 class Scenario(object):
 
     def setup(self):
         """ default impl for scenario setup """
         pass
 
-    def run(self, args):
+    def run(self, *args):
         """ catcher for not implemented run methods in subclasses """
         raise RuntimeError("run method not implemented")
 
@@ -41,16 +55,15 @@ class Scenario(object):
     def get_types():
         """return a list of known runner type (class) names"""
         scenarios = []
-        for scenario in utils.itersubclasses(Scenario):
+        for scenario in _iter_scenario_classes():
             scenarios.append(scenario)
         return scenarios
 
     @staticmethod
     def get_cls(scenario_type):
         """return class of specified type"""
-        for scenario in utils.itersubclasses(Scenario):
-            if scenario_type == scenario.__scenario_type__:
-                return scenario
+        for scenario in _iter_scenario_classes(scenario_type):
+            return scenario
 
         raise RuntimeError("No such scenario type %s" % scenario_type)
 
@@ -58,11 +71,8 @@ class Scenario(object):
     def get(scenario_type):
         """Returns instance of a scenario runner for execution type.
         """
-        for scenario in utils.itersubclasses(Scenario):
-            if scenario_type == scenario.__scenario_type__:
-                return scenario.__module__ + "." + scenario.__name__
-
-        raise RuntimeError("No such scenario type %s" % scenario_type)
+        scenario = Scenario.get_cls(scenario_type)
+        return scenario.__module__ + "." + scenario.__name__
 
     @classmethod
     def get_scenario_type(cls):