[compute]
host4 ansible_host=10.1.0.53 ansible_user=root ansible_ssh_pass=root
host5 ansible_host=10.1.0.54 ansible_user=root ansible_ssh_pass=root
+
+[nodes:children]
+controller
+compute
--- /dev/null
+---
+##############################################################################
+# Copyright (c) 2017 Huawei Technologies Co.,Ltd and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+- name: archive_spec_cpu2006_result
+ archive:
+ path: /usr/cpu2006/result
+ dest: /usr/cpu2006/spec_cpu2006_result.zip
+ become: true
--- /dev/null
+---
+##############################################################################
+# Copyright (c) 2017 Huawei Technologies Co.,Ltd and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+- name: fetch_spec_cpu2006_result
+ fetch:
+ src: /usr/cpu2006/spec_cpu2006_result.zip
+ dest: /tmp/
+ flat: yes
+ become: true
--- /dev/null
+---
+##############################################################################
+# Copyright (c) 2017 Huawei Technologies Co.,Ltd and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+- name: upload SPEC CPU2006 ISO
+ copy:
+ src: /home/opnfv/repos/yardstick/yardstick/resources/cpu2006-1.2.iso
+ dest: ~/cpu2006-1.2.iso
+ become: true
+
+- name: install SPEC CPU2006
+ shell:
+ mount -t iso9660 -o ro,exec ~/cpu2006-1.2.iso /mnt;
+ cd /mnt;
+ ./install.sh -fd /usr/cpu2006;
+ args:
+ executable: /bin/bash
+ become: true
--- /dev/null
+---
+##############################################################################
+# Copyright (c) 2017 Huawei Technologies Co.,Ltd and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+- name: unarchive_spec_cpu2006_result
+ unarchive:
+ src: /tmp/spec_cpu2006_result.zip
+ dest: /tmp/
+ remote_src: True
+ become: true
--- /dev/null
+---
+##############################################################################
+# Copyright (c) 2017 Huawei Technologies Co.,Ltd and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+- name: remove SPEC CPU 2006
+ file:
+ path: /usr/cpu2006
+ state: absent
+ become: true
+
+- name: umount SPEC CPU 2006 ISO
+ shell:
+ umount -l /mnt;
+ args:
+ executable: /bin/bash
+ become: true
+
+- name: remove SPEC CPU 2006 ISO
+ file:
+ path: ~/cpu2006-1.2.iso
+ state: absent
+ become: true
--- /dev/null
+---
+##############################################################################
+# Copyright (c) 2017 Huawei Technologies Co.,Ltd and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+- hosts: spec_target
+ roles:
+ - install_spec_cpu2006
--- /dev/null
+---
+##############################################################################
+# Copyright (c) 2017 Huawei Technologies Co.,Ltd and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+- hosts: spec_target
+ roles:
+ - archive_spec_cpu2006_result
+ - fetch_spec_cpu2006_result
+
+- hosts: localhost
+ roles:
+ - unarchive_spec_cpu2006_result
+
+- hosts: spec_target
+ roles:
+ - uninstall_spec_cpu2006
--- /dev/null
+.. This work is licensed under a Creative Commons Attribution 4.0 International
+.. License.
+.. http://creativecommons.org/licenses/by/4.0
+.. (c) OPNFV, Huawei Technologies Co.,Ltd and others.
+
+*************************************
+Yardstick Test Case Description TC078
+*************************************
+
+.. _spec_cpu2006: https://www.spec.org/cpu2006/
+
++-----------------------------------------------------------------------------+
+|Compute Performance |
+| |
++--------------+--------------------------------------------------------------+
+|test case id | OPNFV_YARDSTICK_TC078_SPEC CPU 2006 |
+| | |
++--------------+--------------------------------------------------------------+
+|metric | compute-intensive performance |
+| | |
++--------------+--------------------------------------------------------------+
+|test purpose | The purpose of TC078 is to evaluate the IaaS compute |
+| | performance by using SPEC CPU 2006 benchmark. The SPEC CPU |
+| | 2006 benchmark has several different ways to measure |
+| | computer performance. One way is to measure how fast the |
+| | computer completes a single task; this is called a speed |
+| | measurement. Another way is to measure how many tasks |
+| | computer can accomplish in a certain amount of time; this is |
+| | called a throughput, capacity or rate measurement. |
+| | |
++--------------+--------------------------------------------------------------+
+|test tool | SPEC CPU 2006 |
+| | |
+| | The SPEC CPU 2006 benchmark is SPEC's industry-standardized, |
+| | CPU-intensive benchmark suite, stressing a system's |
+| | processor, memory subsystem and compiler. This benchmark |
+| | suite includes the SPECint benchmarks and the SPECfp |
+| | benchmarks. The SPECint 2006 benchmark contains 12 different |
+| | enchmark tests and the SPECfp 2006 benchmark contains 19 |
+| | different benchmark tests. |
+| | |
+| | SPEC CPU 2006 is not always part of a Linux distribution. |
+| | SPEC requires that users purchase a license and agree with |
+| | their terms and conditions. For this test case, users must |
+| | manually download cpu2006-1.2.iso from the SPEC website and |
+| | save it under the yardstick/resources folder (e.g. /home/ |
+| | opnfv/repos/yardstick/yardstick/resources/cpu2006-1.2.iso) |
+| | SPEC CPU® 2006 benchmark is available for purchase via the |
+| | SPEC order form (https://www.spec.org/order.html). |
+| | |
++--------------+--------------------------------------------------------------+
+|test | This test case uses SPEC CPU 2006 benchmark to measure |
+|description | compute-intensive performance of hosts. |
+| | |
++--------------+--------------------------------------------------------------+
+|configuration | file: spec_cpu.yaml (in the 'samples' directory) |
+| | |
+| | benchmark_subset is set to int. |
+| | |
+| | SLA is not available in this test case. |
+| | |
++--------------+--------------------------------------------------------------+
+|applicability | Test can be configured with different: |
+| | |
+| | * benchmark_subset - a subset of SPEC CPU2006 benchmarks to |
+| | run; |
+| | * SPECint_benchmark - a SPECint benchmark to run; |
+| | * SPECint_benchmark - a SPECfp benchmark to run; |
+| | * output_format - desired report format; |
+| | * runspec_config - SPEC CPU2006 config file provided to the |
+| | runspec binary; |
+| | * runspec_iterations - the number of benchmark iterations |
+| | to execute. For a reportable run, must be 3; |
+| | * runspec_tune - tuning to use (base, peak, or all). For a |
+| | reportable run, must be either base or all. Reportable |
+| | runs do base first, then (optionally) peak; |
+| | * runspec_size - size of input data to run (test, train, or |
+| | ref). Reportable runs ensure that your binaries can |
+| | produce correct results with the test and train workloads |
+| | |
++--------------+--------------------------------------------------------------+
+|usability | This test case is used for executing SPEC CPU 2006 benchmark |
+| | physical servers. The SPECint 2006 benchmark takes |
+| | approximately 5 hours. |
+| | |
++--------------+--------------------------------------------------------------+
+|references | spec_cpu2006_ |
+| | |
+| | ETSI-NFV-TST001 |
+| | |
++--------------+--------------------------------------------------------------+
+|pre-test | To run and install SPEC CPU2006, the following are required: |
+|conditions | * For SPECint2006: Both C99 and C++98 compilers; |
+| | * For SPECfp2006: All three of C99, C++98 and Fortran-95 |
+| | compilers; |
+| | * At least 8GB of disk space availabile on the system. |
+| | |
++--------------+--------------------------------------------------------------+
+|test sequence | description and expected result |
+| | |
++--------------+--------------------------------------------------------------+
+|step 1 | cpu2006-1.2.iso has been saved under the yardstick/resources |
+| | folder (e.g. /home/opnfv/repos/yardstick/yardstick/resources |
+| | /cpu2006-1.2.iso). Additional, to use your custom runspec |
+| | config file you can save it under the yardstick/resources/ |
+| | files folder and specify the config file name in the |
+| | runspec_config parameter. |
+| | |
++--------------+--------------------------------------------------------------+
+|step 2 | Upload SPEC CPU2006 ISO to the target server and install |
+| | SPEC CPU2006 via ansible. |
+| | |
++--------------+--------------------------------------------------------------+
+|step 3 | Yardstick is connected with the target server by using ssh. |
+| | If custom runspec config file is used, this file is copyied |
+| | from yardstick to the target server via the ssh tunnel. |
+--------------+---------------------------------------------------------------+
+|step 4 | SPEC CPU2006 benchmark is invoked and SPEC CPU 2006 metrics |
+| | are generated. |
+| | |
++--------------+--------------------------------------------------------------+
+|step 5 | Text, HTML, CSV, PDF, and Configuration file outputs for the |
+| | SPEC CPU 2006 metrics are fetch from the server and stored |
+| | under /tmp/result folder. |
+| | |
++--------------+--------------------------------------------------------------+
+|step 6 | uninstall SPEC CPU2006 and remove cpu2006-1.2.iso from the |
+| | target server . |
+| | |
++--------------+--------------------------------------------------------------+
+|test verdict | None. SPEC CPU2006 results are collected and stored. |
+| | |
++--------------+--------------------------------------------------------------+
--- /dev/null
+---
+##############################################################################
+# Copyright (c) 2017 Huawei Technologies Co.,Ltd and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+schema: "yardstick:task:0.1"
+
+{% set file = file or "/etc/yardstick/pod.yaml" %}
+
+scenarios:
+-
+ type: SpecCPU2006
+
+ options:
+ benchmark_subset: int
+ #runspec_tune: all
+ #output_format: all
+ #SPECint_benchmark: perlbench
+ #SPECfp_benchmark: bwaves
+ #runspec_config: yardstick_spec_cpu2006.cfg
+ #runspec_iterations: 1
+ #runspec_tune: base
+ #runspec_size: test
+
+ host: node1.spec_cpu2006
+
+ runner:
+ type: Iteration
+ iterations: 1
+
+context:
+ type: Node
+ name: spec_cpu2006
+ file: {{ file }}
+
+ env:
+ type: ansible
+ setup: spec_cpu2006_install.yaml
+ teardown: spec_cpu2006_uninstall.yaml
--- /dev/null
+#!/usr/bin/env python
+
+##############################################################################
+# Copyright (c) 2017 Huawei Technologies Co.,Ltd and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+# Unittest for yardstick.benchmark.scenarios.compute.spec_cpu.SpecCPU
+
+from __future__ import absolute_import
+
+import unittest
+
+import mock
+
+from yardstick.common import utils
+from yardstick.benchmark.scenarios.compute import spec_cpu
+
+
+@mock.patch('yardstick.benchmark.scenarios.compute.spec_cpu.ssh')
+class SpecCPUTestCase(unittest.TestCase):
+
+ def setUp(self):
+ self.ctx = {
+ 'host': {
+ 'ip': '172.16.0.137',
+ 'user': 'root',
+ 'key_filename': "mykey.key"
+ }
+ }
+
+ self.result = {}
+
+ def test_spec_cpu_successful_setup(self, mock_ssh):
+
+ options = {
+ "SPECint_benchmark": "perlbench",
+ "runspec_tune": "all",
+ "output_format": "all",
+ "runspec_iterations": "1",
+ "runspec_tune": "base",
+ "runspec_size": "test"
+ }
+ args = {"options": options}
+ s = spec_cpu.SpecCPU(args, self.ctx)
+ mock_ssh.SSH.from_node().execute.return_value = (0, '', '')
+
+ s.setup()
+ self.assertIsNotNone(s.client)
+ self.assertTrue(s.setup_done, True)
+
+ def test_spec_cpu_successful__run_no_sla(self, mock_ssh):
+
+ options = {
+ "SPECint_benchmark": "perlbench",
+ "runspec_tune": "all",
+ "output_format": "all"
+ }
+ args = {"options": options}
+ s = spec_cpu.SpecCPU(args, self.ctx)
+
+ sample_output = ''
+ mock_ssh.SSH.from_node().execute.return_value = (0, '', '')
+ s.run(self.result)
+ expected_result = {}
+ self.assertEqual(self.result, expected_result)
+
+ def test_ramspeed_unsuccessful_script_error(self, mock_ssh):
+ options = {
+ "benchmark_subset": "int"
+ }
+ args = {"options": options}
+ s = spec_cpu.SpecCPU(args, self.ctx)
+
+ mock_ssh.SSH.from_node().execute.return_value = (1, '', 'FOOBAR')
+ self.assertRaises(RuntimeError, s.run, self.result)
+
+def main():
+ unittest.main()
+
+if __name__ == '__main__':
+ main()
--- /dev/null
+##############################################################################
+# Copyright (c) 2017 Huawei Technologies Co.,Ltd and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+from __future__ import absolute_import
+
+import logging
+import pkg_resources
+
+import yardstick.ssh as ssh
+from yardstick.benchmark.scenarios import base
+
+LOG = logging.getLogger(__name__)
+
+
+class SpecCPU(base.Scenario):
+ """Spec CPU2006 benchmark
+
+ Parameters
+ benchmark_subset - Specifies a subset of SPEC CPU2006 benchmarks to run
+ type: string
+ unit: na
+ default: na
+
+ SPECint_benchmark - A SPECint benchmark to run
+ type: string
+ unit: na
+ default: na
+
+ SPECint_benchmark - A SPECfp benchmark to run
+ type: string
+ unit: na
+ default: na
+
+ output_format - Desired report format
+ type: string
+ unit: na
+ default: na
+
+ runspec_config - SPEC CPU2006 config file provided to the runspec binary
+ type: string
+ unit: na
+ default: "Example-linux64-amd64-gcc43+.cfg"
+
+ runspec_iterations - The number of benchmark iterations to execute.
+ For a reportable run, must be 3.
+ type: int
+ unit: na
+ default: na
+
+ runspec_tune - Tuning to use (base, peak, or all). For a reportable run, must be either
+ base or all. Reportable runs do base first, then (optionally) peak.
+ type: string
+ unit: na
+ default: na
+
+ runspec_size - Size of input data to run (test, train, or ref). Reportable runs ensure
+ that your binaries can produce correct results with the test and train
+ workloads.
+ type: string
+ unit: na
+ default: na
+ """
+ __scenario_type__ = "SpecCPU2006"
+
+ def __init__(self, scenario_cfg, context_cfg):
+ self.scenario_cfg = scenario_cfg
+ self.context_cfg = context_cfg
+ self.setup_done = False
+ self.options = self.scenario_cfg['options']
+
+ def setup(self):
+ """scenario setup"""
+ host = self.context_cfg['host']
+ LOG.info("user:%s, host:%s", host['user'], host['ip'])
+ self.client = ssh.SSH.from_node(host, defaults={"user": "ubuntu"})
+ self.client.wait(timeout=600)
+
+ if "runspec_config" in self.options:
+ self.runspec_config = self.options["runspec_config"]
+
+ self.runspec_config_file = pkg_resources.resource_filename(
+ "yardstick.resources", 'files/' + self.runspec_config)
+
+ # copy SPEC CPU2006 config file to host if given
+ self.client._put_file_shell(self.runspec_config_file,
+ '/usr/cpu2006/config/yardstick_spec_cpu2006.cfg')
+ else:
+ self.runspec_config = "Example-linux64-amd64-gcc43+.cfg"
+
+ self.setup_done = True
+
+ def run(self, result):
+ """execute the benchmark"""
+
+ if not self.setup_done:
+ self.setup()
+
+ cmd = "cd /usr/cpu2006/ && . ./shrc && runspec --config %s" % self.runspec_config
+ cmd_args = ""
+
+ if "output_format" in self.options:
+ cmd_args += " --output_format %s" % self.options["output_format"]
+
+ if "runspec_tune" in self.options:
+ cmd_args += " --tune %s" % self.options["runspec_tune"]
+
+ benchmark_subset = self.options.get('benchmark_subset', None)
+ specint_benchmark = self.options.get('SPECint_benchmark', None)
+ specfp_benchmark = self.options.get('SPECfp_benchmark', None)
+
+ if benchmark_subset:
+ cmd_args += " %s" % benchmark_subset
+ else:
+ cmd_args += " --noreportable"
+
+ if "runspec_iterations" in self.options:
+ cmd_args += " --iterations %s" % self.options["runspec_iterations"]
+
+ if "runspec_size" in self.options:
+ cmd_args += " --size %s" % self.options["runspec_size"]
+
+ if specint_benchmark:
+ cmd_args += " %s" % specint_benchmark
+
+ if specfp_benchmark:
+ cmd_args += " %s" % specfp_benchmark
+
+ cmd += "%s" % cmd_args
+
+ LOG.debug("Executing command: %s", cmd)
+ status, stdout, stderr = self.client.execute(cmd, timeout=86400)
+ if status:
+ raise RuntimeError(stderr)
+
+ LOG.info('SPEC CPU2006 benchmark completed, please find benchmark reports \
+ at /tmp/result directory')
Code Review / yardstick.git / commitdiff