--- /dev/null
+##############################################################################
+# Copyright (c) 2016 ZTE Corp and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+
+class AlgoName(object):
+ """algorithm names"""
+ ARITHMETIC_MEAN = 'arithmetic mean'
+ WEIGHTED_ARITHMETIC_MEAN = 'weighted arithmetic mean'
+ GEOMETRIC_MEAN = 'geometric mean'
+ WEIGHTED_GEOMETRIC_MEAN = 'weighted geometric mean'
+
+
+class PkgName(object):
+ """QTIP package names"""
+ COLLECTOR = 'collector'
+ DRIVER = 'driver'
+ REPORTER = 'reporter'
+ RUNNER = 'runner'
+ SPEC = 'spec'
+
+
+class PropName(object):
+ """property names"""
+ # list
+ NAME = 'name'
+ CONTENT = 'content'
+ ABSPATH = 'abspath'
+ # content
+ name = 'name'
+ DESCRIPTION = 'description'
+ # spec
+ SECTIONS = 'sections'
+ WEIGHT = 'weight'
+ ALGORITHM = 'algorithm'
+ METRICS = 'metrics'
+ WORKLOADS = 'workloads'
+ # plan
+ CONFIG = 'config'
+ FACILITY = 'facility'
+ ENGINEER = 'engineer'
+ DRIVER = 'driver'
+ COLLECTOR = 'collector'
+ REPORTER = 'reporter'
+ QPIS = 'QPIs'
--- /dev/null
+##############################################################################
+# Copyright (c) 2016 ZTE Corp and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+
+class QtipError(Exception):
+ pass
+
+
+class InvalidFormat(QtipError):
+ def __init__(self, filename):
+ self.filename = filename
+
+
+class NotFound(QtipError):
+ def __init__(self, module, package='qtip'):
+ self.package = package
+ self.module = module
##############################################################################
-class BaseDriver(object):
- """performance testing tool driver"""
+class BaseCollector(object):
+ """performance metrics collector"""
# http://www.apache.org/licenses/LICENSE-2.0
##############################################################################
+from base import BaseCollector
-class Reporter(object):
- """generate test report and push test data to database"""
- pass
+
+class LogfileCollector(BaseCollector):
+ """collect performance metrics from log files"""
--- /dev/null
+##############################################################################
+# Copyright (c) 2016 ZTE Corp and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+
+class BaseDriver(object):
+ """performance testing tool driver"""
+ def pre_run(self):
+ pass
+
+ def run(self):
+ pass
+
+ def post_run(self):
+ pass
--- /dev/null
+##############################################################################
+# Copyright (c) 2016 ZTE Corp and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+
+from base import BaseDriver
+
+
+class SampleDriver(BaseDriver):
+ """sample driver that generates random data for testing"""
from os import path
import yaml
-
-class Property(object):
- # list
- NAME = 'name'
- CONTENT = 'content'
- ABSPATH = 'abspath'
- # content
- TITLE = 'title'
- DESCRIPTION = 'description'
- # spec
- ALGORITHM = 'algorithm'
- SECTIONS = 'sections'
- WEIGHT = 'weight'
- METRICS = 'metrics'
- SPEC = 'spec'
- WORKLOADS = 'workloads'
- # plan
- INFO = 'info'
- FACILITY = 'facility'
- ENGINEER = 'engineer'
- SUITES = 'suites'
- # suite
- QPI_SPEC = 'QPI_spec'
- CONDITION = 'condition'
- CASES = 'cases'
- # case
- METRIC_SPEC = 'metric_spec'
- CONFIG = 'config'
-
-
-class QtipError(Exception):
- pass
-
-
-class Algorithm(object):
- ARITHMETIC_MEAN = 'arithmetic mean'
- WEIGHTED_ARITHMETIC_MEAN = 'weighted arithmetic mean'
- GEOMETRIC_MEAN = 'geometric mean'
- WEIGHTED_GEOMETRIC_MEAN = 'weighted geometric mean'
-
+from qtip.base.error import InvalidFormat, NotFound
+from qtip.base.constant import PropName
ROOT_DIR = 'benchmarks'
-class Benchmark(object):
- """Abstract class of QTIP benchmarks"""
+class BaseLoader(object):
+ """Abstract class of QTIP benchmark loader"""
DEFAULT_DIR = '.'
_paths = [path.join(path.dirname(__file__), path.pardir, path.pardir,
ROOT_DIR)]
def __init__(self, name, paths=None):
self._file = name
self._abspath = self._find(name, paths)
- self.name = path.splitext(name)[0]
+
+ try:
+ content = yaml.safe_load(file(self._abspath))
+ except yaml.YAMLError:
+ # TODO(yujunz) log yaml error
+ raise InvalidFormat(self._abspath)
+
+ self.name = content[PropName.NAME] if PropName.NAME in content \
+ else path.splitext(name)[0]
+ self.content = content
def _find(self, name, paths):
"""find a benchmark in searching paths"""
abspath = path.join(p, name)
if path.exists(abspath):
return abspath
- raise QtipError("'{}' not found in paths: {}".format(name, paths))
+ raise NotFound(name, paths)
@classmethod
def list_all(cls, paths=None):
for name in names:
item = cls(name, paths=paths)
yield {
- Property.NAME: name,
- Property.ABSPATH: item._abspath,
- Property.CONTENT: item.content()}
-
- def content(self):
- """description of benchmark"""
- return yaml.safe_load(file(self._abspath))
+ PropName.NAME: name,
+ PropName.ABSPATH: item._abspath,
+ PropName.CONTENT: item.content}
# http://www.apache.org/licenses/LICENSE-2.0
##############################################################################
-from qtip.base.benchmark import Benchmark
+from base import BaseLoader
-class MetricSpec(Benchmark):
+class MetricSpec(BaseLoader):
"""metrics in QTIP are categorized by performance test tools, such as
dhrystone, whetstone and etc"""
- DEFAULT_DIR = 'metrics'
+ DEFAULT_DIR = 'metric'
# http://www.apache.org/licenses/LICENSE-2.0
##############################################################################
-from qtip.base.benchmark import Benchmark
+from base import BaseLoader
-class QPISpec(Benchmark):
+class QPISpec(BaseLoader):
"""
a QPI specification defines how to calculate a performance index from
collected metrics.
--- /dev/null
+##############################################################################
+# Copyright (c) 2016 ZTE Corp and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+
+class BaseReporter(object):
+ """benchmark result reporter"""
+ def __init__(self, collector=None):
+ self.collector = collector
# http://www.apache.org/licenses/LICENSE-2.0
##############################################################################
+from base import BaseReporter
-class Collector(object):
- """collect test result and test condition"""
- pass
+
+class ConsoleReporter(BaseReporter):
+ """report result to console"""
--- /dev/null
+##############################################################################
+# Copyright (c) 2016 ZTE Corp and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+from qtip.base.constant import PkgName, PropName
+from qtip.base.error import NotFound
+from qtip.collector.logfile import LogfileCollector
+from qtip.driver.sample import SampleDriver
+from qtip.loader.base import BaseLoader
+from qtip.reporter.console import ConsoleReporter
+
+
+class BaseRunner(BaseLoader):
+ def __init__(self, name, paths=None, config=None):
+ super(BaseRunner, self).__init__(name, paths=paths)
+ if config is None:
+ config = self.content[PropName.CONFIG]
+
+ driver_name = config[PropName.DRIVER]
+ collector_name = config[PropName.COLLECTOR]
+ reporter_name = config[PropName.REPORTER]
+
+ # TODO(yujunz) dynamically load modules by name
+
+ if driver_name == 'sample':
+ self.driver = SampleDriver()
+ else:
+ raise NotFound(driver_name, package=PkgName.DRIVER)
+
+ if collector_name == 'logfile':
+ self.collector = LogfileCollector()
+ else:
+ raise NotFound(collector_name,
+ package=PkgName.COLLECTOR)
+
+ if reporter_name == 'console':
+ self.reporter = ConsoleReporter()
+ else:
+ raise NotFound(reporter_name,
+ package=PkgName.REPORTER)
+++ /dev/null
-##############################################################################
-# Copyright (c) 2016 ZTE Corp and others.
-#
-# All rights reserved. This program and the accompanying materials
-# are made available under the terms of the Apache License, Version 2.0
-# which accompanies this distribution, and is available at
-# http://www.apache.org/licenses/LICENSE-2.0
-##############################################################################
-
-from qtip.base.benchmark import Property
-from qtip.spec.metric import MetricSpec
-
-
-class Case(object):
- def __init__(self, spec, paths=None):
- self.metric_spec = MetricSpec(spec[Property.METRIC_SPEC], paths=paths)
- self.config = spec[Property.CONFIG]
# http://www.apache.org/licenses/LICENSE-2.0
##############################################################################
-from qtip.base.benchmark import Benchmark, Property
-from qtip.runner.suite import Suite
+from qtip.base.constant import PropName
+from qtip.runner.base import BaseRunner
+from qtip.loader.qpi import QPISpec
-class Plan(Benchmark):
+class Plan(BaseRunner):
+ """
+ a benchmark plan is consist of configuration and a QPI list
"""
- a benchmark plan is consist of basic information and several suites"""
- DEFAULT_DIR = 'plans'
+ DEFAULT_DIR = 'plan'
def __init__(self, name, paths=None):
- super(Plan, self).__init__(name, paths=paths)
- content = self.content()
-
- self.info = content[Property.INFO]
- self.suites = [Suite(suite, paths=paths)
- for suite in content[Property.SUITES]]
+ super(Plan, self).__init__(name, paths)
+ self.qpis = [QPISpec(qpi, paths=paths)
+ for qpi in self.content[PropName.QPIS]]
+++ /dev/null
-##############################################################################
-# Copyright (c) 2016 ZTE Corp and others.
-#
-# All rights reserved. This program and the accompanying materials
-# are made available under the terms of the Apache License, Version 2.0
-# which accompanies this distribution, and is available at
-# http://www.apache.org/licenses/LICENSE-2.0
-##############################################################################
-
-from qtip.base.benchmark import Property
-from qtip.spec.qpi import QPISpec
-from qtip.runner.case import Case
-
-
-class Suite(object):
- """a suite of benchmark cases under specified condition"""
- def __init__(self, spec, paths=None):
- self._paths = paths
- self.qpi_spec = QPISpec(spec[Property.QPI_SPEC], paths=paths)
- self.condition = spec.get(Property.CONDITION, {})
- self.cases = [Case(case_spec, paths)
- for case_spec in spec.get(Property.CASES, [])]
from os import path
import pytest
+from qtip.runner.plan import Plan
+
@pytest.fixture(scope='session')
def data_root():
@pytest.fixture(scope='session')
def benchmarks_root(data_root):
return path.join(data_root, 'benchmarks')
+
+
+@pytest.fixture(scope='session')
+def plan(benchmarks_root):
+ return Plan('fake-plan.yaml', [benchmarks_root])
--- /dev/null
+name: Fake QPI
+description: a fake QPI producing random result
+algorithm: weighted arithmetic mean
+sections:
+- name: Fake Section
+ weight: 0.5
+ algorithm: geometric mean
+ metrics:
+ - fake_metric.yaml
--- /dev/null
+name: fake compute
+description: >
+ a fake compute performance index which generate random result
+workloads: # all supported workloads
+ - fake_workload_1
+ - fake_workload_2
+ - fake_workload_3
--- /dev/null
+name: fake plan
+description: fake benchmark plan for demonstration and testing
+config:
+ facility: local
+ engineer: local
+ driver: sample
+ collector: logfile
+ reporter: console
+QPIs:
+ - fake-qpi.yaml
+++ /dev/null
-title: verification
-description: benchmark plan for QTIP verification
-info:
- facility: opnfv-ci
- engineer: opnfv-bot
-suites:
- - QPI_spec: compute.yaml
-# Uncomment next line to setup suite running condition
-# condition: {}
-# Uncomment the following lines setup case running configuration
-# cases:
-# - metric_spec: dhrystone.yaml
-# config: {}
-# - metric_spec: whetstone.yaml
-# config: {}
-# - metric_spec: ramspeed.yaml
-# config: {}
-# - metric_spec: dpi.yaml
-# config: {}
-# - metric_spec: ssl.yaml
-# config: {}
import pytest
-from qtip.base.benchmark import Property
-from qtip.spec.metric import MetricSpec
+from qtip.base.constant import PropName
+from qtip.loader.metric import MetricSpec
@pytest.fixture(scope='module')
def list_all_test():
metric_list = MetricSpec.list_all()
- assert len(list(metric_list)) is 1
+ assert len(list(metric_list)) is 6
for desc in metric_list:
- assert Property.NAME in desc
- assert Property.DESCRIPTION in desc
- assert Property.ABSPATH in desc
- assert Property.ABSPATH is not None
+ assert PropName.NAME in desc
+ assert PropName.DESCRIPTION in desc
+ assert PropName.ABSPATH in desc
+ assert PropName.ABSPATH is not None
def content_test(metric):
- content = metric.content()
- assert Property.NAME in content
- assert Property.DESCRIPTION in content
- assert Property.WORKLOADS in content
- assert isinstance(content[Property.WORKLOADS], list)
+ content = metric.content
+ assert PropName.NAME in content
+ assert PropName.DESCRIPTION in content
+ assert PropName.WORKLOADS in content
+ assert isinstance(content[PropName.WORKLOADS], list)
import pytest
-from qtip.base.benchmark import Algorithm, Property
-from qtip.spec.qpi import QPISpec
+from qtip.base.constant import AlgoName, PropName
+from qtip.loader.qpi import QPISpec
QPI_SPEC = 'compute.yaml'
def test_list_all(benchmarks_root):
qpi_spec_list = QPISpec.list_all(paths=[benchmarks_root])
- assert len(list(qpi_spec_list)) is 1
+ assert len(list(qpi_spec_list)) is 2
for item in qpi_spec_list:
- assert Property.NAME in item
- assert Property.CONTENT in item
- assert Property.ABSPATH in item
- assert Property.ABSPATH is not None
+ assert PropName.NAME in item
+ assert PropName.CONTENT in item
+ assert PropName.ABSPATH in item
+ assert PropName.ABSPATH is not None
def test_content(qpi_spec):
- content = qpi_spec.content()
- assert Property.DESCRIPTION in content
- assert Property.ALGORITHM in content
- assert Property.SECTIONS in content
+ content = qpi_spec.content
+ assert PropName.DESCRIPTION in content
+ assert PropName.ALGORITHM in content
+ assert PropName.SECTIONS in content
- assert content[Property.ALGORITHM] in Algorithm.__dict__.values()
- sections = content[Property.SECTIONS]
+ assert content[PropName.ALGORITHM] in AlgoName.__dict__.values()
+ sections = content[PropName.SECTIONS]
assert isinstance(sections, list)
for section in sections:
- assert Property.NAME in section
+ assert PropName.NAME in section
+++ /dev/null
-###############################################################
-# Copyright (c) 2016 ZTE Corp and others.
-#
-# All rights reserved. This program and the accompanying materials
-# are made available under the terms of the Apache License, Version 2.0
-# which accompanies this distribution, and is available at
-# http://www.apache.org/licenses/LICENSE-2.0
-##############################################################################
-
-from qtip.spec.metric import MetricSpec
-
-
-def init_test(case):
- assert isinstance(case.metric_spec, MetricSpec)
- assert isinstance(case.config, dict)
+++ /dev/null
-##############################################################################
-# Copyright (c) 2016 ZTE Corp and others.
-#
-# All rights reserved. This program and the accompanying materials
-# are made available under the terms of the Apache License, Version 2.0
-# which accompanies this distribution, and is available at
-# http://www.apache.org/licenses/LICENSE-2.0
-##############################################################################
-
-import pytest
-
-from qtip.base.benchmark import Property
-from qtip.runner.case import Case
-from qtip.runner.plan import Plan
-from qtip.runner.suite import Suite
-
-
-@pytest.fixture(scope='module')
-def plan(benchmarks_root):
- return Plan('verification.yaml', paths=[benchmarks_root])
-
-
-@pytest.fixture(scope='module')
-def suite(plan):
- return Suite(plan[Property.SUITES][0])
-
-
-@pytest.fixture(scope='module')
-def case(suite):
- return Case(suite[Property.CASES][0])
import pytest
-from qtip.base.benchmark import Property
+from qtip.base.constant import PropName
from qtip.runner.plan import Plan
def test_init(plan):
- assert plan.name == 'verification'
+ assert plan.name == 'fake plan'
with pytest.raises(TypeError) as excinfo:
Plan()
plan_list = Plan.list_all(paths=[benchmarks_root])
assert len(list(plan_list)) is 1
for desc in plan_list:
- assert Property.NAME in desc
- assert Property.CONTENT in desc
- assert Property.ABSPATH in desc
- assert Property.ABSPATH is not None
+ assert PropName.NAME in desc
+ assert PropName.CONTENT in desc
+ assert PropName.ABSPATH in desc
+ assert PropName.ABSPATH is not None
def test_content(plan):
- content = plan.content()
- assert Property.TITLE in content
- assert Property.DESCRIPTION in content
+ content = plan.content
+ assert PropName.NAME in content
+ assert PropName.DESCRIPTION in content
+++ /dev/null
-##############################################################################
-# Copyright (c) 2016 ZTE Corp and others.
-#
-# All rights reserved. This program and the accompanying materials
-# are made available under the terms of the Apache License, Version 2.0
-# which accompanies this distribution, and is available at
-# http://www.apache.org/licenses/LICENSE-2.0
-##############################################################################
-
-import pytest
-
-from qtip.runner.suite import Suite
-from qtip.runner.case import Case
-from qtip.spec.qpi import QPISpec
-
-
-def init_test(suite):
- assert isinstance(suite.qpi, QPISpec)
- assert isinstance(suite.condition, dict)
- assert isinstance(suite.cases, list)
- for case in suite.cases:
- assert isinstance(case, Case)
-
- with pytest.raises(TypeError) as excinfo:
- Suite()
- assert '__init__() takes exactly 2 arguments (1 given)' \
- in str(excinfo.value)
Code Review / qtip.git / commitdiff