Update Sequence runner to use ScenarioOutput class

List of changes:
 - update Sequence runner to use new mechanism for reporting data (kpi)
 - add new rfc2544 example test case that executes the same test with
   different framesizes sequentially

JIRA: YARDSTICK-1592

Change-Id: I731cf1ca40f7232b26ea68f81d22248ebc53eb62
Signed-off-by: Serhiy Pshyk <serhiyx.pshyk@intel.com>

diff --git a/samples/vnf_samples/nsut/agnostic/tc_baremetal_rfc2544_ipv4_ixia_multi_framesize.yaml b/samples/vnf_samples/nsut/agnostic/tc_baremetal_rfc2544_ipv4_ixia_multi_framesize.yaml
new file mode 100644 (file)
index 0000000..dfbff28
--- /dev/null
+++ b/samples/vnf_samples/nsut/agnostic/tc_baremetal_rfc2544_ipv4_ixia_multi_framesize.yaml
@@ -0,0 +1,47 @@
+# Copyright (c) 2019 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+---
+schema: yardstick:task:0.1
+scenarios:
+- type: NSPerf-RFC2544
+  traffic_profile: ../../traffic_profiles/ixia_ipv4_latency.yaml
+  topology: agnostic_vnf_topology_ixia_2ports.yaml
+  nodes:
+    tg__0: tg_0.yardstick
+    vnf__0: vnf_0.yardstick
+  options:
+    framesize:
+    flow:
+      src_ip: [{'tg__0': 'xe0'}]
+      dst_ip: [{'tg__0': 'xe1'}]
+      count: 1
+    traffic_type: 4
+    rfc2544:
+      allowed_drop_rate: 0.0001 - 0.0001
+    vnf__0:
+        []
+  runner:
+    type: Sequence
+    scenario_option_name: framesize
+    sequence:
+    - uplink: {64B: 100}
+      downlink: {64B: 100}
+    - uplink: {512B: 100}
+      downlink: {512B: 100}
+context:
+  type: Node
+  name: yardstick
+  nfvi_type: baremetal
+  file: /etc/yardstick/nodes/pod_ixia.yaml

diff --git a/yardstick/benchmark/runners/sequence.py b/yardstick/benchmark/runners/sequence.py
index 0148a45..58ffddd 100644 (file)
--- a/yardstick/benchmark/runners/sequence.py
+++ b/yardstick/benchmark/runners/sequence.py
@@ -38,8 +38,6 @@ LOG = logging.getLogger(__name__)
 def _worker_process(queue, cls, method_name, scenario_cfg,
                     context_cfg, aborted, output_queue):
 
-    sequence = 1
-
     runner_cfg = scenario_cfg['runner']
 
     interval = runner_cfg.get("interval", 1)
@@ -56,6 +54,7 @@ def _worker_process(queue, cls, method_name, scenario_cfg,
     LOG.info("worker START, sequence_values(%s, %s), class %s",
              arg_name, sequence_values, cls)
 
+    scenario_output = base.ScenarioOutput(queue, sequence=1, errors="")
     benchmark = cls(scenario_cfg, context_cfg)
     benchmark.setup()
     method = getattr(benchmark, method_name)
@@ -68,22 +67,23 @@ def _worker_process(queue, cls, method_name, scenario_cfg,
         options[arg_name] = value
 
         LOG.debug("runner=%(runner)s seq=%(sequence)s START",
-                  {"runner": runner_cfg["runner_id"], "sequence": sequence})
+                  {"runner": runner_cfg["runner_id"],
+                   "sequence": scenario_output.sequence})
 
-        data = {}
-        errors = ""
+        scenario_output.clear()
+        scenario_output.errors = ""
 
         try:
-            result = method(data)
+            result = method(scenario_output)
         except y_exc.SLAValidationError as error:
             # SLA validation failed in scenario, determine what to do now
             if sla_action == "assert":
                 raise
             elif sla_action == "monitor":
                 LOG.warning("SLA validation failed: %s", error.args)
-                errors = error.args
+                scenario_output.errors = error.args
         except Exception as e:  # pylint: disable=broad-except
-            errors = traceback.format_exc()
+            scenario_output.errors = traceback.format_exc()
             LOG.exception(e)
         else:
             if result:
@@ -91,21 +91,16 @@ def _worker_process(queue, cls, method_name, scenario_cfg,
 
         time.sleep(interval)
 
-        benchmark_output = {
-            'timestamp': time.time(),
-            'sequence': sequence,
-            'data': data,
-            'errors': errors
-        }
-
-        queue.put(benchmark_output)
+        if scenario_output:
+            scenario_output.push()
 
         LOG.debug("runner=%(runner)s seq=%(sequence)s END",
-                  {"runner": runner_cfg["runner_id"], "sequence": sequence})
+                  {"runner": runner_cfg["runner_id"],
+                   "sequence": scenario_output.sequence})
 
-        sequence += 1
+        scenario_output.sequence += 1
 
-        if (errors and sla_action is None) or aborted.is_set():
+        if (scenario_output.errors and sla_action is None) or aborted.is_set():
             break
 
     try: