Changes Arithmetic runner to accept several step parameters

The Arithmetic runner is modified to accept several named
parameters to iterate over in each scenario. Possible to chose
either to iterate through them in a nested-for-loop fashion
(default) or in a tupled loop fashion (combine i:th element
of each input). Still possible to only provide one parameter
to step/iterate over, as before.

All existing yaml files with Arithmetic types have been modified
to run the tests as before, and to provide same results as before
this change. The only change is in sample file pktgen.yaml to
provide a valid example of how to use this modification.

It is also possible to step backwards, i.e. start > stop, with
a negative step parameter value.

(Noticed that tests are terminated when there is SLA error
problems in "monitor" mode. Should be fixed in another commit)

Change-Id: Ib2ebd24b71dd55d6817cee8e67026a0dd13a9e17
JIRA: -

diff --git a/samples/lmbench.yaml b/samples/lmbench.yaml
index 2b8e990..8baa814 100644 (file)
--- a/samples/lmbench.yaml
+++ b/samples/lmbench.yaml
@@ -9,16 +9,18 @@ scenarios:
   type: Lmbench
   options:
     test_type: "latency"
-    stride: 64
     stop_size: 32
 
   host: demeter.demo
 
   runner:
     type: Arithmetic
-    name: stride
-    stop: 128
-    step: 64
+    iterators:
+    -
+      name: stride
+      start: 64
+      stop: 128
+      step: 64
 
   sla:
     max_latency: 35
@@ -27,16 +29,18 @@ scenarios:
   type: Lmbench
   options:
     test_type: "bandwidth"
-    size: 500
     benchmark: "wr"
 
   host: demeter.demo
 
   runner:
     type: Arithmetic
-    name: size
-    stop: 2000
-    step: 500
+    iterators:
+    -
+      name: size
+      start: 500
+      stop: 2000
+      step: 500
 
   sla:
     min_bandwidth: 10000

diff --git a/samples/ping-serial.yaml b/samples/ping-serial.yaml
index 37ea715..9c492e4 100644 (file)
--- a/samples/ping-serial.yaml
+++ b/samples/ping-serial.yaml
@@ -20,16 +20,17 @@ scenarios:
     action: monitor
 -
   type: Ping
-  options:
-    packetsize: 100
   host: apollo.demo
   target: artemis.demo
   runner:
     type: Arithmetic
-    name: packetsize
     interval: 0.2
-    stop: 6000
-    step: 100
+    iterators:
+    -
+      name: packetsize
+      start: 100
+      stop: 6000
+      step: 100
 
 context:
   name: demo

diff --git a/samples/pktgen.yaml b/samples/pktgen.yaml
index 11d6279..d621cb7 100644 (file)
--- a/samples/pktgen.yaml
+++ b/samples/pktgen.yaml
@@ -1,6 +1,8 @@
 ---
 # Sample benchmark task config file
 # measure network throughput using pktgen
+# with 2 stepping parameters. One stepping
+# in positive and the other in negative direction
 
 schema: "yardstick:task:0.1"
 
@@ -8,8 +10,6 @@ scenarios:
 -
   type: Pktgen
   options:
-    packetsize: 60
-    number_of_ports: 10
     duration: 20
 
   host: demeter.demo
@@ -17,10 +17,20 @@ scenarios:
 
   runner:
     type: Arithmetic
-    name: number_of_ports
-    # run twice with values 10 and 20
-    stop: 20
-    step: 10
+    interval: 2
+    iter_type: nested_for_loops
+    # run with packetsize/number_of_ports: 60,20; 60,10; ... 70,10
+    iterators:
+      -
+       name: packetsize
+       start: 60
+       stop: 70
+       step: 5
+      -
+       name: number_of_ports
+       start: 20
+       stop: 10
+       step: -10
 
   sla:
     max_ppm: 1000

diff --git a/yardstick/benchmark/runners/arithmetic.py b/yardstick/benchmark/runners/arithmetic.py
index 4eab664..b1446e0 100755 (executable)
--- a/yardstick/benchmark/runners/arithmetic.py
+++ b/yardstick/benchmark/runners/arithmetic.py
@@ -7,8 +7,12 @@
 # http://www.apache.org/licenses/LICENSE-2.0
 ##############################################################################
 
-'''A runner that every run arithmetically steps a specified input value to
-the scenario. This just means a step value is added to the previous value.
+'''A runner that every run arithmetically steps specified input value(s) to
+the scenario. This just means step value(s) is added to the previous value(s).
+It is possible to combine several named input values and run with those either
+as nested for loops or combine each i:th index of each "input value list"
+until the end of the shortest list is reached (optimally all lists should be
+defined with the same number of values when using such iter_type).
 '''
 
 import os
@@ -16,6 +20,7 @@ import multiprocessing
 import logging
 import traceback
 import time
+import itertools
 
 from yardstick.benchmark.runners import base
 
@@ -30,16 +35,15 @@ def _worker_process(queue, cls, method_name, scenario_cfg,
     runner_cfg = scenario_cfg['runner']
 
     interval = runner_cfg.get("interval", 1)
-    arg_name = runner_cfg.get('name')
-    stop = runner_cfg.get('stop')
-    step = runner_cfg.get('step')
-    options = scenario_cfg['options']
-    start = options.get(arg_name, 0)
+    if 'options' in scenario_cfg:
+        options = scenario_cfg['options']
+    else:  # options must be instatiated if not present in yaml
+        options = {}
+        scenario_cfg['options'] = options
 
     runner_cfg['runner_id'] = os.getpid()
 
-    LOG.info("worker START, step(%s, %d, %d, %d), class %s",
-             arg_name, start, stop, step, cls)
+    LOG.info("worker START, class %s", cls)
 
     benchmark = cls(scenario_cfg, context_cfg)
     benchmark.setup()
@@ -52,18 +56,39 @@ def _worker_process(queue, cls, method_name, scenario_cfg,
     sla_action = None
     if "sla" in scenario_cfg:
         sla_action = scenario_cfg["sla"].get("action", "assert")
-    margin = 1 if step > 0 else -1
 
-    for value in range(start, stop+margin, step):
+    # To both be able to include the stop value and handle backwards stepping
+    margin = lambda start, stop: -1 if start > stop else 1
+
+    param_iters = \
+        [xrange(d['start'], d['stop'] + margin(d['start'], d['stop']),
+                d['step']) for d in runner_cfg['iterators']]
+    param_names = [d['name'] for d in runner_cfg['iterators']]
+
+    iter_type = runner_cfg.get("iter_type", "nested_for_loops")
+
+    if iter_type == 'nested_for_loops':
+        # Create a complete combination set of all parameter lists
+        loop_iter = itertools.product(*param_iters)
+    elif iter_type == 'tuple_loops':
+        # Combine each i;th index of respective parameter list
+        loop_iter = itertools.izip(*param_iters)
+    else:
+        LOG.warning("iter_type unrecognized: %s", iter_type)
+        raise
+
+    # Populate options and run the requested method for each value combination
+    for comb_values in loop_iter:
 
         if aborted.is_set():
             break
 
-        options[arg_name] = value
-
         LOG.debug("runner=%(runner)s seq=%(sequence)s START" %
                   {"runner": runner_cfg["runner_id"], "sequence": sequence})
 
+        for i, value in enumerate(comb_values):
+            options[param_names[i]] = value
+
         data = {}
         errors = ""
 
@@ -107,29 +132,39 @@ def _worker_process(queue, cls, method_name, scenario_cfg,
 
 
 class ArithmeticRunner(base.Runner):
-    '''Run a scenario arithmetically stepping an input value
+    '''Run a scenario arithmetically stepping input value(s)
 
   Parameters
     interval - time to wait between each scenario invocation
         type:    int
         unit:    seconds
         default: 1 sec
-    name - name of scenario option that will be increased for each invocation
+    iter_type: - Iteration type of input parameter(s): nested_for_loops
+                 or tuple_loops
         type:    string
         unit:    na
-        default: none
-    start - value to use in first invocation of scenario
-        type:    int
-        unit:    na
-        default: none
-    step - value added to start value in next invocation of scenario
-        type:    int
-        unit:    na
-        default: none
-    stop - value indicating end of invocation
-        type:    int
-        unit:    na
-        default: none
+        default: nested_for_loops
+    -
+      name - name of scenario option that will be increased for each invocation
+          type:    string
+          unit:    na
+          default: na
+      start - value to use in first invocation of scenario
+          type:    int
+          unit:    na
+          default: none
+      stop - value indicating end of invocation. Can be set to same
+             value as start for one single value.
+          type:    int
+          unit:    na
+          default: none
+      step - value added to start value in next invocation of scenario.
+             Must not be set to zero. Can be set negative if start > stop
+          type:    int
+          unit:    na
+          default: none
+    -
+      name - and so on......
     '''
 
     __execution_type__ = 'Arithmetic'