Add functional tests in verify and merge

As Ana said ,"The first functional test should be as simple as a "Hello world",
 it shall be possible to run the "Hello world" test without using OpenStack."
so i just finish functional test framework and do functional test for subcommand
"runner"and"scenario" without using Openstack.

JIRA:YARDSTICK-103

Change-Id: I673ae61f9922536a685d32ae62e5ad5165472f9d
Signed-off-by: kubi <jean.gaoliang@huawei.com>

diff --git a/run_tests.sh b/run_tests.sh
index c8d8e7d..b4ddc94 100755 (executable)
--- a/run_tests.sh
+++ b/run_tests.sh
@@ -53,6 +53,17 @@ run_tests() {
     fi
 }
 
+run_functional_test() {
+
+    mkdir -p .testrepository
+    python -m subunit.run discover tests/functional > .testrepository/subunit.log
+
+    subunit2pyunit < .testrepository/subunit.log
+    EXIT_CODE=$?
+    subunit-stats < .testrepository/subunit.log
+
+    exit $EXIT_CODE
+}
 run_flake8
 run_tests
-
+run_functional_test

diff --git a/setup.py b/setup.py
index f171aaf..75e589e 100755 (executable)
--- a/setup.py
+++ b/setup.py
@@ -29,7 +29,8 @@ setup(
                       "python-novaclient>=2.24.1",
                       "mock>=1.0.1",  # remove with python3
                       "paramiko",
-                      "six"
+                      "six",
+                      "testrepository>=0.0.18"
                       ],
     extras_require={
         'plot': ["matplotlib>=1.4.2"]

diff --git a/tests/functional/__init__.py b/tests/functional/__init__.py
new file mode 100755 (executable)
index 0000000..e69de29

diff --git a/tests/functional/test_cli_runner.py b/tests/functional/test_cli_runner.py
new file mode 100755 (executable)
index 0000000..195b572
--- /dev/null
+++ b/tests/functional/test_cli_runner.py
@@ -0,0 +1,49 @@
+##############################################################################
+# Copyright (c) 2015 Huawei Technologies Co.,Ltd and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+
+import unittest
+
+from tests.functional import utils
+
+
+class RunnerTestCase(unittest.TestCase):
+
+    def setUp(self):
+        super(RunnerTestCase, self).setUp()
+        self.yardstick = utils.Yardstick()
+
+    def test_runner_list(self):
+        res = self.yardstick("runner list")
+
+        self.assertIn("Duration", res)
+        self.assertIn("Arithmetic", res)
+        self.assertIn("Iteration", res)
+        self.assertIn("Sequence", res)
+
+    def test_runner_show_Duration(self):
+        res = self.yardstick("runner show Duration")
+        duration = "duration - amount of time" in res
+        self.assertTrue(duration)
+
+    def test_runner_show_Arithmetic(self):
+        res = self.yardstick("runner show Arithmetic")
+        arithmetic = "Run a scenario arithmetically" in res
+        self.assertTrue(arithmetic)
+
+    def test_runner_show_Iteration(self):
+        res = self.yardstick("runner show Iteration")
+        iteration = "iterations - amount of times" in res
+        self.assertTrue(iteration)
+
+    def test_runner_show_Sequence(self):
+        res = self.yardstick("runner show Sequence")
+        sequence = "sequence - list of values which are executed" in res
+        self.assertTrue(sequence)
+

diff --git a/tests/functional/test_cli_scenario.py b/tests/functional/test_cli_scenario.py
new file mode 100755 (executable)
index 0000000..aad4759
--- /dev/null
+++ b/tests/functional/test_cli_scenario.py
@@ -0,0 +1,61 @@
+##############################################################################
+# Copyright (c) 2015 Huawei Technologies Co.,Ltd and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+
+import unittest
+
+from tests.functional import utils
+
+
+class ScenarioTestCase(unittest.TestCase):
+
+    def setUp(self):
+        super(ScenarioTestCase, self).setUp()
+        self.yardstick = utils.Yardstick()
+
+    def test_scenario_list(self):
+        res = self.yardstick("scenario list")
+
+        self.assertIn("Lmbench", res)
+        self.assertIn("Perf", res)
+        self.assertIn("Fio", res)
+        self.assertIn("Ping", res)
+        self.assertIn("Iperf3", res)
+        self.assertIn("Pktgen", res)
+
+    def test_scenario_show_Lmbench(self):
+        res = self.yardstick("scenario show Lmbench")
+        lmbench = "Execute lmbench memory read latency benchmark in a host" in res
+        self.assertTrue(lmbench)
+
+    def test_scenario_show_Perf(self):
+        res = self.yardstick("scenario show Perf")
+        perf = "Execute perf benchmark in a host" in res
+        self.assertTrue(perf)
+
+    def test_scenario_show_Fio(self):
+        res = self.yardstick("scenario show Fio")
+        fio = "Execute fio benchmark in a host" in res
+        self.assertTrue(fio)
+
+    def test_scenario_show_Ping(self):
+        res = self.yardstick("scenario show Ping")
+        ping = "Execute ping between two hosts" in res
+        self.assertTrue(ping)
+
+    def test_scenario_show_Iperf3(self):
+        res = self.yardstick("scenario show Iperf3")
+        iperf3 = "Execute iperf3 between two hosts" in res
+        self.assertTrue(iperf3)
+
+    def test_scenario_show_Pktgen(self):
+        res = self.yardstick("scenario show Pktgen")
+        pktgen = "Execute pktgen between two hosts" in res
+        self.assertTrue(pktgen)
+

diff --git a/tests/functional/utils.py b/tests/functional/utils.py
new file mode 100755 (executable)
index 0000000..aaaaaac
--- /dev/null
+++ b/tests/functional/utils.py
@@ -0,0 +1,63 @@
+##############################################################################
+# Copyright (c) 2015 Huawei Technologies Co.,Ltd and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+import copy
+import json
+import os
+import shutil
+import subprocess
+
+
+from oslo_utils import encodeutils
+
+
+class Yardstick(object):
+    """Create and represent separate yardstick installation.
+
+    Usage:
+        yardstick = yardstick()
+        output = yardstick("runner list")
+
+    """
+
+    def __init__(self, fake=False):
+
+        self.args = ["yardstick"]
+        self.env = copy.deepcopy(os.environ)
+
+    def __del__(self):
+        pass
+
+    def __call__(self, cmd, getjson=False, report_path=None, raw=False,
+                 suffix=None, extension=None, keep_old=False,
+                 write_report=False):
+        """Call yardstick in the shell
+
+        :param cmd: yardstick command
+        :param getjson: in cases, when yardstick prints JSON, you can catch output
+            deserialized
+        TO DO:
+        :param report_path: if present, yardstick command and its output will be
+            written to file with passed file name
+        :param raw: don't write command itself to report file. Only output
+            will be written
+        """
+
+        if not isinstance(cmd, list):
+            cmd = cmd.split(" ")
+        try:
+            output = encodeutils.safe_decode(subprocess.check_output(
+                self.args + cmd, stderr=subprocess.STDOUT, env=self.env))
+
+            if getjson:
+                return json.loads(output)
+            return output
+        except subprocess.CalledProcessError as e:
+            raise e
+