QPI: compute
description: sample performance index of computing
-algorithm: weighted arithmetic mean
+formula: weighted arithmetic mean
section:
- name: Integer
weight: 0.3
- algorithm: geometric mean
+ formula: geometric mean
perftests:
- name: dhrystone
workloads:
- multi_cpu
- name: Floating
weight: 0.3
- algorithm: geometric mean
+ formula: geometric mean
perftests:
- name: whetstone
workloads:
- multi_cpu
- name: Memory
weight: 0.2
- algorithm: geometric mean
+ formula: geometric mean
perftests:
- name: ramspeed
workloads:
- float: [add, average, copy, scale, triad]
- name: DPI
weight: 0.1
- algorithm: geometric mean
+ formula: geometric mean
perftests:
- name: dpi
workloads:
- pps
- name: SSL
weight: 0.1
- algorithm: geometric mean
+ formula: geometric mean
perftests:
- name: ssl
workloads:
##############################################################################
-class AlgoName(object):
- """algorithm names"""
+class FormulaName(object):
+ """formula names"""
ARITHMETIC_MEAN = 'arithmetic mean'
WEIGHTED_ARITHMETIC_MEAN = 'weighted arithmetic mean'
GEOMETRIC_MEAN = 'geometric mean'
# spec
SECTIONS = 'sections'
WEIGHT = 'weight'
- ALGORITHM = 'algorithm'
+ FORMULA = 'formula'
METRICS = 'metrics'
WORKLOADS = 'workloads'
# plan
def __init__(self, module, package='qtip'):
self.package = package
self.module = module
+
+
+class ToBeDoneError(QtipError):
+ """something still to be done"""
+ def __init__(self, method, module):
+ self.method = method
+ self.module = module
+
+
+def make_tbd(method, module='qtip'):
+ def tbd():
+ raise ToBeDoneError(method, module)
+ return tbd
import click
from prettytable import PrettyTable
-from qtip.runner.plan import Plan
+
+from qtip.loader.plan import Plan
@click.group()
class BaseLoader(object):
"""Abstract class of QTIP benchmark loader"""
- DEFAULT_DIR = '.'
+ RELATIVE_PATH = '.'
_paths = [path.join(path.dirname(__file__), path.pardir, path.pardir,
ROOT_DIR)]
def __init__(self, name, paths=None):
self._file = name
- self._abspath = self._find(name, paths)
+ self._abspath = self._find(name, paths=paths)
try:
content = yaml.safe_load(file(self._abspath))
else path.splitext(name)[0]
self.content = content
- def _find(self, name, paths):
+ def _find(self, name, paths=None):
"""find a benchmark in searching paths"""
paths = self._paths if paths is None else paths
- name = path.join(self.DEFAULT_DIR, name)
for p in paths:
- abspath = path.join(p, name)
+ abspath = path.join(p, self.RELATIVE_PATH, name)
if path.exists(abspath):
return abspath
raise NotFound(name, paths)
def list_all(cls, paths=None):
"""list all available benchmarks"""
paths = cls._paths if paths is None else paths
- names = chain.from_iterable([listdir(path.join(p, cls.DEFAULT_DIR))
+ names = chain.from_iterable([listdir(path.join(p, cls.RELATIVE_PATH))
for p in paths])
for name in names:
item = cls(name, paths=paths)
class MetricSpec(BaseLoader):
"""metrics in QTIP are categorized by performance test tools, such as
dhrystone, whetstone and etc"""
- DEFAULT_DIR = 'metric'
+ RELATIVE_PATH = 'metric'
# http://www.apache.org/licenses/LICENSE-2.0
##############################################################################
+
from qtip.base.constant import PropName
-from qtip.runner.base import BaseRunner
+from qtip.loader.base import BaseLoader
from qtip.loader.qpi import QPISpec
-class Plan(BaseRunner):
+class Plan(BaseLoader):
"""
a benchmark plan is consist of configuration and a QPI list
"""
- DEFAULT_DIR = 'plan'
+ RELATIVE_PATH = 'plan'
def __init__(self, name, paths=None):
super(Plan, self).__init__(name, paths)
+
self.qpis = [QPISpec(qpi, paths=paths)
for qpi in self.content[PropName.QPIS]]
##############################################################################
from base import BaseLoader
+from metric import MetricSpec
+
+from qtip.base.constant import PropName
+from qtip.utils.formula import Formula
class QPISpec(BaseLoader):
a QPI specification defines how to calculate a performance index from
collected metrics.
"""
- DEFAULT_DIR = 'QPI'
+ RELATIVE_PATH = 'QPI'
+
+ def __init__(self, name, paths=None):
+ super(QPISpec, self).__init__(name, paths=paths)
+ content = self.content
+ self.formula = Formula(content[PropName.FORMULA])
+ self.sections = [Section(record, paths=paths)
+ for record in content[PropName.SECTIONS]]
+
+
+class Section(object):
+ def __init__(self, content, paths=None):
+ self.name = content[PropName.NAME]
+ self.weight = content[PropName.WEIGHT]
+ self.formula = Formula(content[PropName.FORMULA])
+ self.metrics = [MetricSpec(record, paths=paths)
+ for record in content[PropName.METRICS]]
+##############################################################################
+# Copyright (c) 2016 ZTE Corp and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+from qtip.base.constant import PkgName, PropName
+from qtip.base.error import NotFound
+from qtip.collector.stdout import StdoutCollector
+from qtip.driver.random import RandomDriver
+from qtip.reporter.console import ConsoleReporter
+
+
+class Runner(object):
+ def __init__(self, spec, config=None):
+ if config is None:
+ config = spec[PropName.CONFIG]
+
+ driver_name = config[PropName.DRIVER]
+ collector_name = config[PropName.COLLECTOR]
+ reporter_name = config[PropName.REPORTER]
+
+ # TODO(yujunz) dynamically load modules by name
+
+ if driver_name == 'random':
+ self.driver = RandomDriver()
+ else:
+ raise NotFound(driver_name, package=PkgName.DRIVER)
+
+ if collector_name == 'stdout':
+ self.collector = StdoutCollector()
+ else:
+ raise NotFound(collector_name,
+ package=PkgName.COLLECTOR)
+
+ if reporter_name == 'console':
+ self.reporter = ConsoleReporter()
+ else:
+ raise NotFound(reporter_name,
+ package=PkgName.REPORTER)
+++ /dev/null
-##############################################################################
-# Copyright (c) 2016 ZTE Corp and others.
-#
-# All rights reserved. This program and the accompanying materials
-# are made available under the terms of the Apache License, Version 2.0
-# which accompanies this distribution, and is available at
-# http://www.apache.org/licenses/LICENSE-2.0
-##############################################################################
-
-from qtip.base.constant import PkgName, PropName
-from qtip.base.error import NotFound
-from qtip.collector.logfile import LogfileCollector
-from qtip.driver.sample import SampleDriver
-from qtip.loader.base import BaseLoader
-from qtip.reporter.console import ConsoleReporter
-
-
-class BaseRunner(BaseLoader):
- def __init__(self, name, paths=None, config=None):
- super(BaseRunner, self).__init__(name, paths=paths)
- if config is None:
- config = self.content[PropName.CONFIG]
-
- driver_name = config[PropName.DRIVER]
- collector_name = config[PropName.COLLECTOR]
- reporter_name = config[PropName.REPORTER]
-
- # TODO(yujunz) dynamically load modules by name
-
- if driver_name == 'sample':
- self.driver = SampleDriver()
- else:
- raise NotFound(driver_name, package=PkgName.DRIVER)
-
- if collector_name == 'logfile':
- self.collector = LogfileCollector()
- else:
- raise NotFound(collector_name,
- package=PkgName.COLLECTOR)
-
- if reporter_name == 'console':
- self.reporter = ConsoleReporter()
- else:
- raise NotFound(reporter_name,
- package=PkgName.REPORTER)
--- /dev/null
+##############################################################################
+# Copyright (c) 2016 ZTE Corp and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+import numpy
+
+from qtip.base.error import make_tbd
+from qtip.base.constant import FormulaName
+
+
+MAPPING = {
+ FormulaName.ARITHMETIC_MEAN: numpy.mean,
+ FormulaName.WEIGHTED_ARITHMETIC_MEAN: numpy.average,
+ # TODO(yujunz) find or implement the method
+ FormulaName.GEOMETRIC_MEAN: make_tbd(FormulaName.GEOMETRIC_MEAN, __name__),
+ # TODO(yujunz) find or implement the method
+ FormulaName.WEIGHTED_GEOMETRIC_MEAN:
+ make_tbd(FormulaName.GEOMETRIC_MEAN, __name__)}
+
+
+class Formula:
+ """calculate a score from give data"""
+ def __init__(self, name):
+ self.calculate = MAPPING[name]
Flask-RESTful==0.3.5
flask-restful-swagger==0.19
ansible==2.1.1.0
+numpy==1.11.3
##############################################################################
from os import path
+
import pytest
-from qtip.runner.plan import Plan
+from qtip.loader.plan import Plan
@pytest.fixture(scope='session')
title: compute
description: sample performance index of computing
-algorithm: weighted arithmetic mean
+formula: weighted arithmetic mean
sections:
- name: Integer
weight: 0.3
- algorithm: geometric mean
+ formula: geometric mean
metrics:
- - spec: dhrystone.yaml
- workloads:
- - single_cpu
- - multi_cpu
+ - dhrystone.yaml
- name: Float
weight: 0.3
- algorithm: geometric mean
+ formula: geometric mean
metrics:
- - spec: dhrystone.yaml
- workloads:
- - single_cpu
- - multi_cpu
+ - whetstone.yaml
- name: Memory
weight: 0.2
- algorithm: geometric mean
+ formula: geometric mean
metrics:
- - spec: ramspeed.yaml
- workloads:
- - int: [add, average, copy, scale, triad]
- - float: [add, average, copy, scale, triad]
+ - ramspeed.yaml
- name: DPI
weight: 0.1
- algorithm: geometric mean
+ formula: geometric mean
metrics:
- - spec: dpi.yaml
- workloads:
- - bps
- - pps
+ - dpi.yaml
- name: SSL
weight: 0.1
- algorithm: geometric mean
+ formula: geometric mean
metrics:
- - spec: ssl.yaml
- workloads:
- - aes_128_cbc: [512, 1024, 2048, 4096]
- - rsa_sig: [16, 64, 256, 1024, 8192]
+ - ssl.yaml
name: Fake QPI
description: a fake QPI producing random result
-algorithm: weighted arithmetic mean
+formula: weighted arithmetic mean
sections:
- name: Fake Section
weight: 0.5
- algorithm: geometric mean
+ formula: geometric mean
metrics:
- - fake_metric.yaml
+ - fake-metric.yaml
name: dhrystone
description: >
- a synthetic computing benchmark program intended to be representative of
- system (integer) programming
+ A synthetic computing benchmark program intended to be representative of
+ system (integer) programming.
+links:
+ - https://en.wikipedia.org/wiki/Dhrystone
workloads:
- single_cpu
- multi_cpu
-name: dhrystone
-description: a synthetic benchmark for evaluating the performance of computers
+name: whetstone
+description: >
+ A synthetic benchmark for evaluating the performance of computers.
+ The Whetstone benchmark primarily measures the floating-point arithmetic
+ performance.
+links:
+ - https://en.wikipedia.org/wiki/Whetstone_(benchmark)
workloads:
- single_cpu
- multi_cpu
import pytest
from qtip.base.constant import PropName
-from qtip.runner.plan import Plan
+from qtip.loader.plan import Plan, QPISpec
def test_init(plan):
assert plan.name == 'fake plan'
+ assert isinstance(plan.content, dict)
+ for qpi in plan.qpis:
+ assert isinstance(qpi, QPISpec)
with pytest.raises(TypeError) as excinfo:
Plan()
content = plan.content
assert PropName.NAME in content
assert PropName.DESCRIPTION in content
+ assert PropName.CONFIG in content
+ assert PropName.QPIS in content
import pytest
-from qtip.base.constant import AlgoName, PropName
+from qtip.base.constant import FormulaName, PropName
from qtip.loader.qpi import QPISpec
QPI_SPEC = 'compute.yaml'
def test_content(qpi_spec):
content = qpi_spec.content
assert PropName.DESCRIPTION in content
- assert PropName.ALGORITHM in content
+ assert PropName.FORMULA in content
assert PropName.SECTIONS in content
- assert content[PropName.ALGORITHM] in AlgoName.__dict__.values()
+ assert content[PropName.FORMULA] in FormulaName.__dict__.values()
sections = content[PropName.SECTIONS]
assert isinstance(sections, list)
for section in sections:
Code Review / qtip.git / commitdiff