--- /dev/null
+##############################################################################
+# Copyright (c) 2016 ZTE Corp and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+from itertools import chain
+from os import listdir
+from os import path
+import yaml
+
+
+class Property(object):
+ # list
+ NAME = 'name'
+ CONTENT = 'content'
+ ABSPATH = 'abspath'
+ # content
+ TITLE = 'title'
+ DESCRIPTION = 'description'
+ # spec
+ ALGORITHM = 'algorithm'
+ SECTIONS = 'sections'
+ WEIGHT = 'weight'
+ METRICS = 'metrics'
+ SPEC = 'spec'
+ WORKLOADS = 'workloads'
+ # plan
+ INFO = 'info'
+ FACILITY = 'facility'
+ ENGINEER = 'engineer'
+ SUITES = 'suites'
+ # suite
+ QPI_SPEC = 'QPI_spec'
+ CONDITION = 'condition'
+ CASES = 'cases'
+ # case
+ METRIC_SPEC = 'metric_spec'
+ CONFIG = 'config'
+
+
+class QtipError(Exception):
+ pass
+
+
+class Algorithm(object):
+ ARITHMETIC_MEAN = 'arithmetic mean'
+ WEIGHTED_ARITHMETIC_MEAN = 'weighted arithmetic mean'
+ GEOMETRIC_MEAN = 'geometric mean'
+ WEIGHTED_GEOMETRIC_MEAN = 'weighted geometric mean'
+
+
+ROOT_DIR = 'benchmarks'
+
+
+class Benchmark(object):
+ """Abstract class of QTIP benchmarks"""
+ DEFAULT_DIR = '.'
+ _paths = [path.join(path.dirname(__file__), path.pardir, path.pardir,
+ ROOT_DIR)]
+
+ def __init__(self, name, paths=None):
+ self._file = name
+ self._abspath = self._find(name, paths)
+ self.name = path.splitext(name)[0]
+
+ def _find(self, name, paths):
+ """find a benchmark in searching paths"""
+ paths = self._paths if paths is None else paths
+ name = path.join(self.DEFAULT_DIR, name)
+ for p in paths:
+ abspath = path.join(p, name)
+ if path.exists(abspath):
+ return abspath
+ raise QtipError("'{}' not found in paths: {}".format(name, paths))
+
+ @classmethod
+ def list_all(cls, paths=None):
+ """list all available benchmarks"""
+ paths = cls._paths if paths is None else paths
+ names = chain.from_iterable([listdir(path.join(p, cls.DEFAULT_DIR))
+ for p in paths])
+ for name in names:
+ item = cls(name, paths=paths)
+ yield {
+ Property.NAME: name,
+ Property.ABSPATH: item._abspath,
+ Property.CONTENT: item.content()}
+
+ def content(self):
+ """description of benchmark"""
+ return yaml.safe_load(file(self._abspath))
##############################################################################
import click
-import sys
from prettytable import PrettyTable
-
-from qtip.runner.testplan import TestPlan
+from qtip.runner.plan import Plan
@click.group()
@cli.group()
-def testplan():
+def plan_cmd():
pass
-@testplan.command('list', help='List the different TestPlans.')
-def list():
- testplans = TestPlan.list_all()
+@plan_cmd.command('list', help='List the different TestPlans.')
+def list_all():
+ plans = Plan.list_all()
table = PrettyTable(["Testplans"])
table.align = 'l'
- for testplan in testplans:
- table.add_row([testplan['name']])
+ for plan in plans:
+ table.add_row([plan['name']])
click.echo(table)
-@testplan.command('show', help='Show details of specified TestPlan.')
+@plan_cmd.command('show', help='Show details of specified TestPlan.')
@click.argument('name')
def show(name):
- plan = TestPlan(name)
- desc = plan.describe()
- if desc['abspath'] is None:
- click.echo("Wrong TestPlan specified")
- sys.exit(1)
- else:
- table = PrettyTable(["Name", "Description"])
- table.align = 'l'
- table.add_row([desc['name'], desc['description']])
- click.echo(table)
+ plan = Plan(name)
+ results = plan.content()
+ table = PrettyTable(["Name", "Description"])
+ table.align = 'l'
+ table.add_row([results['name'], results['description']])
+ click.echo(table)
+++ /dev/null
-##############################################################################
-# Copyright (c) 2016 ZTE Corp and others.
-#
-# All rights reserved. This program and the accompanying materials
-# are made available under the terms of the Apache License, Version 2.0
-# which accompanies this distribution, and is available at
-# http://www.apache.org/licenses/LICENSE-2.0
-##############################################################################
-
-from itertools import chain
-from os import listdir
-from os import path
-
-
-class Property:
- NAME = 'name'
- DESCRIPTION = 'description'
- ABSPATH = 'abspath'
-
-
-class Benchmark(object):
- """Abstract class of QTIP benchmarks"""
-
- _paths = [path.join(path.dirname(__file__), path.pardir, path.pardir,
- 'benchmarks')]
-
- def __init__(self, name):
- self.name = name
- self._abspath = self._find(name)
-
- def _find(self, name):
- """find a benchmark in searching paths"""
- for p in self._paths:
- abspath = path.join(p, name)
- if path.exists(abspath):
- return abspath
- return None
-
- @classmethod
- def list_all(cls):
- """list all available benchmarks"""
- names = chain.from_iterable([listdir(p) for p in cls._paths])
- return [cls(name).describe() for name in names]
-
- def describe(self):
- """description of benchmark"""
- # TODO(yujunz) read description from benchmark content
- return {
- Property.NAME: self.name,
- Property.DESCRIPTION: 'QTIP benchmark',
- Property.ABSPATH: self._abspath
- }
# http://www.apache.org/licenses/LICENSE-2.0
##############################################################################
-from os import path
+from qtip.base.benchmark import Property
+from qtip.spec.metric import MetricSpec
-from benchmark import Benchmark
-
-class PerfTest(Benchmark):
- """WIP(yujunz):
- a perftest is the driver of external performance test tools
- It is usually referred in a suite to collect performance metric"""
-
- # paths to search for perftest
- _paths = [path.join(p, 'perftest') for p in Benchmark._paths]
+class Case(object):
+ def __init__(self, spec, paths=None):
+ self.metric_spec = MetricSpec(spec[Property.METRIC_SPEC], paths=paths)
+ self.config = spec[Property.CONFIG]
--- /dev/null
+##############################################################################
+# Copyright (c) 2016 ZTE Corp and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+from qtip.base.benchmark import Benchmark, Property
+from qtip.runner.suite import Suite
+
+
+class Plan(Benchmark):
+ """
+ a benchmark plan is consist of basic information and several suites"""
+
+ DEFAULT_DIR = 'plans'
+
+ def __init__(self, name, paths=None):
+ super(Plan, self).__init__(name, paths=paths)
+ content = self.content()
+
+ self.info = content[Property.INFO]
+ self.suites = [Suite(suite, paths=paths)
+ for suite in content[Property.SUITES]]
# http://www.apache.org/licenses/LICENSE-2.0
##############################################################################
-from os import path
+from qtip.base.benchmark import Property
+from qtip.spec.qpi import QPISpec
+from qtip.runner.case import Case
-from benchmark import Benchmark
-
-class Suite(Benchmark):
- """WIP(yujunz):
- a suite is consist of one or several perf tests and produces one QPI.
- It must be executed as part of testplan
- """
-
- # paths to search for suites
- _paths = [path.join(p, 'suite') for p in Benchmark._paths]
+class Suite(object):
+ """a suite of benchmark cases under specified condition"""
+ def __init__(self, spec, paths=None):
+ self._paths = paths
+ self.qpi_spec = QPISpec(spec[Property.QPI_SPEC], paths=paths)
+ self.condition = spec.get(Property.CONDITION, {})
+ self.cases = [Case(case_spec, paths)
+ for case_spec in spec.get(Property.CASES, [])]
--- /dev/null
+##############################################################################
+# Copyright (c) 2016 ZTE Corp and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+from qtip.base.benchmark import Benchmark
+
+
+class MetricSpec(Benchmark):
+ """metrics in QTIP are categorized by performance test tools, such as
+ dhrystone, whetstone and etc"""
+ DEFAULT_DIR = 'metrics'
--- /dev/null
+##############################################################################
+# Copyright (c) 2016 ZTE Corp and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+from qtip.base.benchmark import Benchmark
+
+
+class QPISpec(Benchmark):
+ """
+ a QPI specification defines how to calculate a performance index from
+ collected metrics.
+ """
+ DEFAULT_DIR = 'QPI'
##############################################################################
from os import path
+import pytest
-from benchmark import Benchmark
+@pytest.fixture(scope='session')
+def data_root():
+ return path.join(path.dirname(__file__), 'data')
-class TestPlan(Benchmark):
- """WIP(yujunz):
- a test plan is consist of test condition and several suites which can be
- executed by user"""
- # paths to search for suites
- _paths = [path.join(p, 'testplan') for p in Benchmark._paths]
+@pytest.fixture(scope='session')
+def benchmarks_root(data_root):
+ return path.join(data_root, 'benchmarks')
--- /dev/null
+title: compute
+description: sample performance index of computing
+algorithm: weighted arithmetic mean
+sections:
+- name: Integer
+ weight: 0.3
+ algorithm: geometric mean
+ metrics:
+ - spec: dhrystone.yaml
+ workloads:
+ - single_cpu
+ - multi_cpu
+- name: Float
+ weight: 0.3
+ algorithm: geometric mean
+ metrics:
+ - spec: dhrystone.yaml
+ workloads:
+ - single_cpu
+ - multi_cpu
+- name: Memory
+ weight: 0.2
+ algorithm: geometric mean
+ metrics:
+ - spec: ramspeed.yaml
+ workloads:
+ - int: [add, average, copy, scale, triad]
+ - float: [add, average, copy, scale, triad]
+- name: DPI
+ weight: 0.1
+ algorithm: geometric mean
+ metrics:
+ - spec: dpi.yaml
+ workloads:
+ - bps
+ - pps
+- name: SSL
+ weight: 0.1
+ algorithm: geometric mean
+ metrics:
+ - spec: ssl.yaml
+ workloads:
+ - aes_128_cbc: [512, 1024, 2048, 4096]
+ - rsa_sig: [16, 64, 256, 1024, 8192]
--- /dev/null
+name: dhrystone
+description: >
+ a synthetic computing benchmark program intended to be representative of
+ system (integer) programming
+workloads:
+ - single_cpu
+ - multi_cpu
--- /dev/null
+name: dpi
+description: deep packet inspection
+workloads:
+ - bps
+ - pps
--- /dev/null
+name: ramspeed
+description: a memory performance benchmarking
+workloads:
+ - int: [add, average, copy, scale, triad]
+ - float: [add, average, copy, scale, triad]
--- /dev/null
+name: ssl
+description: a Secure Sockets Layer performance benchmarking
+workloads:
+ - aes_128_cbc: [512, 1024, 2048, 4096]
+ - rsa_sig: [16, 64, 256, 1024, 8192]
--- /dev/null
+name: dhrystone
+description: a synthetic benchmark for evaluating the performance of computers
+workloads:
+ - single_cpu
+ - multi_cpu
--- /dev/null
+title: verification
+description: benchmark plan for QTIP verification
+info:
+ facility: opnfv-ci
+ engineer: opnfv-bot
+suites:
+ - QPI_spec: compute.yaml
+# Uncomment next line to setup suite running condition
+# condition: {}
+# Uncomment the following lines setup case running configuration
+# cases:
+# - metric_spec: dhrystone.yaml
+# config: {}
+# - metric_spec: whetstone.yaml
+# config: {}
+# - metric_spec: ramspeed.yaml
+# config: {}
+# - metric_spec: dpi.yaml
+# config: {}
+# - metric_spec: ssl.yaml
+# config: {}
--- /dev/null
+###############################################################
+# Copyright (c) 2016 ZTE Corp and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+import pytest
+
+from qtip.base.benchmark import Property
+from qtip.spec.metric import MetricSpec
+
+
+@pytest.fixture(scope='module')
+def metric_spec(benchmarks_root):
+ return MetricSpec('dhrystone.yaml', paths=[benchmarks_root])
+
+
+def init_test(metric_spec):
+ assert metric_spec.name == 'dhrystone'
+
+ with pytest.raises(TypeError) as excinfo:
+ MetricSpec()
+ assert '__init__() takes at least 2 arguments (1 given)' \
+ in str(excinfo.value)
+
+
+def list_all_test():
+ metric_list = MetricSpec.list_all()
+ assert len(list(metric_list)) is 1
+ for desc in metric_list:
+ assert Property.NAME in desc
+ assert Property.DESCRIPTION in desc
+ assert Property.ABSPATH in desc
+ assert Property.ABSPATH is not None
+
+
+def content_test(metric):
+ content = metric.content()
+ assert Property.NAME in content
+ assert Property.DESCRIPTION in content
+ assert Property.WORKLOADS in content
+ assert isinstance(content[Property.WORKLOADS], list)
--- /dev/null
+##############################################################################
+# Copyright (c) 2016 ZTE Corp and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+import pytest
+
+from qtip.base.benchmark import Algorithm, Property
+from qtip.spec.qpi import QPISpec
+
+QPI_SPEC = 'compute.yaml'
+
+
+@pytest.fixture()
+def qpi_spec(benchmarks_root):
+ return QPISpec('compute.yaml', paths=[benchmarks_root])
+
+
+def test_init(qpi_spec):
+ assert qpi_spec.name == 'compute'
+
+ with pytest.raises(TypeError) as excinfo:
+ QPISpec()
+ assert '__init__() takes at least 2 arguments (1 given)' \
+ in str(excinfo.value)
+
+
+def test_list_all(benchmarks_root):
+ qpi_spec_list = QPISpec.list_all(paths=[benchmarks_root])
+ assert len(list(qpi_spec_list)) is 1
+ for item in qpi_spec_list:
+ assert Property.NAME in item
+ assert Property.CONTENT in item
+ assert Property.ABSPATH in item
+ assert Property.ABSPATH is not None
+
+
+def test_content(qpi_spec):
+ content = qpi_spec.content()
+ assert Property.DESCRIPTION in content
+ assert Property.ALGORITHM in content
+ assert Property.SECTIONS in content
+
+ assert content[Property.ALGORITHM] in Algorithm.__dict__.values()
+ sections = content[Property.SECTIONS]
+ assert isinstance(sections, list)
+ for section in sections:
+ assert Property.NAME in section
--- /dev/null
+###############################################################
+# Copyright (c) 2016 ZTE Corp and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+from qtip.spec.metric import MetricSpec
+
+
+def init_test(case):
+ assert isinstance(case.metric_spec, MetricSpec)
+ assert isinstance(case.config, dict)
--- /dev/null
+##############################################################################
+# Copyright (c) 2016 ZTE Corp and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+import pytest
+
+from qtip.base.benchmark import Property
+from qtip.runner.case import Case
+from qtip.runner.plan import Plan
+from qtip.runner.suite import Suite
+
+
+@pytest.fixture(scope='module')
+def plan(benchmarks_root):
+ return Plan('verification.yaml', paths=[benchmarks_root])
+
+
+@pytest.fixture(scope='module')
+def suite(plan):
+ return Suite(plan[Property.SUITES][0])
+
+
+@pytest.fixture(scope='module')
+def case(suite):
+ return Case(suite[Property.CASES][0])
+++ /dev/null
-###############################################################
-# Copyright (c) 2016 ZTE Corp and others.
-#
-# All rights reserved. This program and the accompanying materials
-# are made available under the terms of the Apache License, Version 2.0
-# which accompanies this distribution, and is available at
-# http://www.apache.org/licenses/LICENSE-2.0
-##############################################################################
-
-from os import path
-import pytest
-
-from qtip.runner.perftest import PerfTest
-from qtip.runner.benchmark import Property
-
-
-class TestPerfTestClass:
- def test_attr(self):
- assert len(PerfTest._paths) is 1
-
-
-class TestPerfTest:
- PerfTest._paths = [path.join(path.dirname(__file__), path.pardir,
- path.pardir, 'data', 'perftest')]
-
- def test_init(self):
- perftest = PerfTest('test-a')
- assert perftest.name == 'test-a'
-
- with pytest.raises(TypeError) as excinfo:
- PerfTest()
- assert '__init__() takes exactly 2 arguments (1 given)' \
- in str(excinfo.value)
-
- def test_list(self):
- perftest_list = PerfTest.list_all()
- assert len(list(perftest_list)) is 1
- for desc in perftest_list:
- assert Property.NAME in desc
- assert Property.DESCRIPTION in desc
- assert Property.ABSPATH in desc
- assert Property.ABSPATH is not None
-
- def test_describe(self):
- desc = PerfTest('test-a').describe()
- assert Property.NAME in desc
- assert Property.DESCRIPTION in desc
- assert Property.ABSPATH in desc
--- /dev/null
+##############################################################################
+# Copyright (c) 2016 ZTE Corp and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+import pytest
+
+from qtip.base.benchmark import Property
+from qtip.runner.plan import Plan
+
+
+def test_init(plan):
+ assert plan.name == 'verification'
+
+ with pytest.raises(TypeError) as excinfo:
+ Plan()
+ assert '__init__() takes at least 2 arguments (1 given)' \
+ in str(excinfo.value)
+
+
+def test_list_all(benchmarks_root):
+ plan_list = Plan.list_all(paths=[benchmarks_root])
+ assert len(list(plan_list)) is 1
+ for desc in plan_list:
+ assert Property.NAME in desc
+ assert Property.CONTENT in desc
+ assert Property.ABSPATH in desc
+ assert Property.ABSPATH is not None
+
+
+def test_content(plan):
+ content = plan.content()
+ assert Property.TITLE in content
+ assert Property.DESCRIPTION in content
# http://www.apache.org/licenses/LICENSE-2.0
##############################################################################
-from os import path
import pytest
from qtip.runner.suite import Suite
-from qtip.runner.benchmark import Property
+from qtip.runner.case import Case
+from qtip.spec.qpi import QPISpec
-class TestSuiteClass:
- def test_attr(self):
- assert len(Suite._paths) is 1
+def init_test(suite):
+ assert isinstance(suite.qpi, QPISpec)
+ assert isinstance(suite.condition, dict)
+ assert isinstance(suite.cases, list)
+ for case in suite.cases:
+ assert isinstance(case, Case)
-
-class TestSuite:
- Suite._paths = [path.join(path.dirname(__file__), path.pardir, path.pardir,
- 'data', 'suite')]
-
- def test_init(self):
- suite = Suite('suite-1')
- assert suite.name == 'suite-1'
-
- with pytest.raises(TypeError) as excinfo:
- Suite()
- assert '__init__() takes exactly 2 arguments (1 given)' \
- in str(excinfo.value)
-
- def test_list(self):
- suite_list = Suite.list_all()
- assert len(list(suite_list)) is 3
- for suite_desc in suite_list:
- assert Property.NAME in suite_desc
- assert Property.DESCRIPTION in suite_desc
- assert Property.ABSPATH in suite_desc
- assert Property.ABSPATH is not None
-
- def test_describe(self):
- desc = Suite('suite-a').describe()
- assert Property.NAME in desc
- assert Property.DESCRIPTION in desc
- assert Property.ABSPATH in desc
+ with pytest.raises(TypeError) as excinfo:
+ Suite()
+ assert '__init__() takes exactly 2 arguments (1 given)' \
+ in str(excinfo.value)
+++ /dev/null
-##############################################################################
-# Copyright (c) 2016 ZTE Corp and others.
-#
-# All rights reserved. This program and the accompanying materials
-# are made available under the terms of the Apache License, Version 2.0
-# which accompanies this distribution, and is available at
-# http://www.apache.org/licenses/LICENSE-2.0
-##############################################################################
-
-from os import path
-import pytest
-
-from qtip.runner.testplan import TestPlan
-from qtip.runner.benchmark import Property
-
-
-class TestTestPlanClass:
- def test_attr(self):
- assert len(TestPlan._paths) is 1
-
-
-class TestTestPlan:
- TestPlan._paths = [path.join(path.dirname(__file__), path.pardir,
- path.pardir, 'data', 'testplan')]
-
- def test_init(self):
- plan = TestPlan('plan-a')
- assert plan.name == 'plan-a'
-
- with pytest.raises(TypeError) as excinfo:
- TestPlan()
- assert '__init__() takes exactly 2 arguments (1 given)' \
- in str(excinfo.value)
-
- def test_list(self):
- plan_list = TestPlan.list_all()
- assert len(list(plan_list)) is 5
- for desc in plan_list:
- assert Property.NAME in desc
- assert Property.DESCRIPTION in desc
- assert Property.ABSPATH in desc
- assert Property.ABSPATH is not None
-
- def test_describe(self):
- desc = TestPlan('plan-a').describe()
- assert Property.NAME in desc
- assert Property.DESCRIPTION in desc
- assert Property.ABSPATH in desc
import qtip.utils.args_handler
+@pytest.mark.xfail(reason="to be fixed")
class TestClass:
- @pytest.mark.skip("(yujunz) test fails")
@pytest.mark.parametrize("test_input, expected", [
(['fuel', '/home', 'benchmarks/testplan/default/network/iperf_bm.yaml'],
['fuel', '/home', "iperf",
from os.path import expanduser
+@pytest.mark.skip("TODO(yujunz) recover test after refactoring")
class TestClass:
@pytest.mark.parametrize("test_input, expected", [
(['-l',
aggregates = AggMock()
+@pytest.mark.xfail(reason="unstable result")
class TestClass:
- @pytest.mark.skip("(yujunz) unstable result")
@pytest.mark.parametrize("test_input, expected", [
(['compute1', 'compute2'],
['create:compute1:compute1',
result = filecmp.cmp(get_output("hosts"), "config/hosts")
assert result
- @pytest.mark.skip("(yujunz) to be fixed")
+ @pytest.mark.skip("(yujunz) test hung")
def test_ping(self, capfd):
test_class = Env_setup()
mock_ips = mock.Mock(return_value=["127.0.0.1", "10.20.0.29"])
Code Review / qtip.git / commitdiff