1 # Copyright (c) 2015 Intel Research and Development Ireland Ltd.
3 # Licensed under the Apache License, Version 2.0 (the "License");
4 # you may not use this file except in compliance with the License.
5 # You may obtain a copy of the License at
7 # http://www.apache.org/licenses/LICENSE-2.0
9 # Unless required by applicable law or agreed to in writing, software
10 # distributed under the License is distributed on an "AS IS" BASIS,
11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 # See the License for the specific language governing permissions and
13 # limitations under the License.
14 from __future__ import absolute_import
19 from experimental_framework import APEX_LAKE_ROOT
20 from experimental_framework.benchmarking_unit import BenchmarkingUnit
21 # from experimental_framework.data_manager import DataManager
22 from experimental_framework.deployment_unit import DeploymentUnit
23 import experimental_framework.common as common
24 from experimental_framework.benchmarks.rfc2544_throughput_benchmark import \
25 RFC2544ThroughputBenchmark
27 __author__ = 'vmriccox'
30 # class DummyDataManager(DataManager):
32 # def __init__(self, experiment_directory):
33 # self.experiment_directory = experiment_directory
34 # self.experiments = dict()
35 # self.new_exp_counter = 0
36 # self.add_bench_counter = 0
37 # self.close_experiment_1_counter = 0
38 # self.close_experiment_2_counter = 0
39 # self.generate_csv_counter = 0
41 # def create_new_experiment(self, experiment_name, get_counter=None):
43 # self.new_exp_counter += 1
45 # return self.new_exp_counter
47 # def add_benchmark(self, experiment_name, benchmark_name, get_counter=None):
49 # self.add_bench_counter += 1
51 # return self.add_bench_counter
53 # def close_experiment(self, experiment, get_counter=None):
55 # return [self.close_experiment_1_counter,
56 # self.close_experiment_2_counter]
57 # if experiment == 'VTC_base_single_vm_wait_1':
58 # self.close_experiment_1_counter += 1
59 # if experiment == 'VTC_base_single_vm_wait_2':
60 # self.close_experiment_2_counter += 1
62 # def generate_result_csv_file(self, get_counter=None):
64 # return self.generate_csv_counter
66 # self.generate_csv_counter += 1
68 # def add_metadata(self, experiment_name, metadata):
71 # def add_configuration(self, experiment_name, configuration):
74 # def add_data_points(self, experiment_name, benchmark_name, result):
78 class Dummy_2544(RFC2544ThroughputBenchmark):
80 def __init__(self, name, params):
83 self.finalize_counter = 0
87 def init(self, get_counter=None):
89 return self.init_counter
91 self.init_counter += 1
93 def finalize(self, get_counter=None):
95 return self.finalize_counter
97 self.finalize_counter += 1
99 def run(self, get_counter=None):
101 return self.run_counter
103 self.run_counter += 1
104 return {'throughput': 10}
107 class DummyDeploymentUnit(DeploymentUnit):
109 def __init__(self, openstack_credentials):
112 def deploy_heat_template(self, template_file, stack_name, parameters,
117 class TestBenchmarkingUnit(unittest.TestCase):
125 @mock.patch('time.time')
126 @mock.patch('experimental_framework.common.get_template_dir')
127 # @mock.patch('experimental_framework.data_manager.DataManager',
128 # side_effect=DummyDataManager)
129 @mock.patch('experimental_framework.deployment_unit.DeploymentUnit')
130 @mock.patch('experimental_framework.benchmarking_unit.heat.'
131 'get_all_heat_templates')
132 def test___init__(self, mock_heat, mock_dep_unit,
134 mock_temp_dir, mock_time):
135 mock_heat.return_value = list()
136 mock_time.return_value = '12345'
137 mock_temp_dir.return_value = 'tests/data/results/'
138 common.TEMPLATE_FILE_EXTENSION = '.ext'
139 common.RESULT_DIR = 'tests/data/results/'
140 heat_template_name = 'name'
141 openstack_credentials = {
145 heat_template_parameters = {
150 benchmarks = ['bench_1', 'bench_2']
151 bu = BenchmarkingUnit(heat_template_name,
152 openstack_credentials,
153 heat_template_parameters,
156 self.assertEqual(bu.required_benchmarks, benchmarks)
157 bu.heat_template_parameters = heat_template_parameters
158 # mock_data_manager.assert_called_once_with('tests/data/results/12345')
159 mock_dep_unit.assert_called_once_with(openstack_credentials)
160 mock_heat.assert_called_once_with('tests/data/results/', '.ext')
162 @mock.patch('experimental_framework.benchmarks.'
163 'rfc2544_throughput_benchmark', side_effect=Dummy_2544)
164 @mock.patch('time.time')
165 @mock.patch('experimental_framework.common.get_template_dir')
166 # @mock.patch('experimental_framework.data_manager.DataManager',
167 # side_effect=DummyDataManager)
168 @mock.patch('experimental_framework.deployment_unit.DeploymentUnit')
169 @mock.patch('experimental_framework.benchmarking_unit.'
170 'heat.get_all_heat_templates')
171 def test_initialize_for_success(self, mock_heat, mock_dep_unit,
174 mock_time, mock_rfc2544):
175 mock_heat.return_value = list()
176 mock_time.return_value = '12345'
177 mock_temp_dir.return_value = 'tests/data/test_templates/'
178 common.TEMPLATE_FILE_EXTENSION = '.yaml'
179 common.RESULT_DIR = 'tests/data/results/'
181 heat_template_name = 'VTC_base_single_vm_wait_'
182 openstack_credentials = {
186 heat_template_parameters = {
194 'rfc2544_throughput_benchmark.RFC2544ThroughputBenchmark',
198 bu = BenchmarkingUnit(heat_template_name,
199 openstack_credentials,
200 heat_template_parameters,
203 self.assertEqual(bu.required_benchmarks, benchmarks)
204 bu.heat_template_parameters = heat_template_parameters
205 bu.template_files = ['VTC_base_single_vm_wait_1.yaml',
206 'VTC_base_single_vm_wait_2.yaml']
208 self.assertTrue(len(bu.benchmarks) == 1)
209 self.assertEqual(bu.benchmarks[0].__class__,
211 # self.assertEqual(bu.data_manager.create_new_experiment('', True), 2)
212 # self.assertEqual(bu.data_manager.add_benchmark('', '', True), 2)
214 @mock.patch('experimental_framework.benchmarks.'
215 'rfc2544_throughput_benchmark', side_effect=Dummy_2544)
216 @mock.patch('time.time')
217 @mock.patch('experimental_framework.common.get_template_dir')
218 # @mock.patch('experimental_framework.data_manager.DataManager',
219 # side_effect=DummyDataManager)
220 @mock.patch('experimental_framework.deployment_unit.DeploymentUnit')
221 @mock.patch('experimental_framework.benchmarking_unit.'
222 'heat.get_all_heat_templates')
223 def test_finalize_for_success(
224 self, mock_heat, mock_dep_unit,
226 mock_temp_dir, mock_time, mock_rfc2544):
227 mock_heat.return_value = list()
228 mock_time.return_value = '12345'
229 mock_temp_dir.return_value = 'tests/data/test_templates/'
230 common.TEMPLATE_FILE_EXTENSION = '.yaml'
231 common.RESULT_DIR = 'tests/data/results/'
233 heat_template_name = 'VTC_base_single_vm_wait_'
234 openstack_credentials = {
238 heat_template_parameters = {
246 'rfc2544_throughput_benchmark.RFC2544ThroughputBenchmark',
250 bu = BenchmarkingUnit(heat_template_name,
251 openstack_credentials,
252 heat_template_parameters,
255 bu.heat_template_parameters = heat_template_parameters
256 bu.template_files = ['VTC_base_single_vm_wait_1.yaml',
257 'VTC_base_single_vm_wait_2.yaml']
259 # self.assertEqual(bu.data_manager.close_experiment('', True), [1, 1])
260 # self.assertEqual(bu.data_manager.generate_result_csv_file(True), 1)
262 @mock.patch('experimental_framework.common.push_data_influxdb')
263 @mock.patch('experimental_framework.common.LOG')
264 @mock.patch('experimental_framework.benchmarks.'
265 'rfc2544_throughput_benchmark', side_effect=Dummy_2544)
266 @mock.patch('time.time')
267 @mock.patch('experimental_framework.common.get_template_dir')
268 # @mock.patch('experimental_framework.data_manager.DataManager',
269 # side_effect=DummyDataManager)
270 @mock.patch('experimental_framework.common.DEPLOYMENT_UNIT')
271 @mock.patch('experimental_framework.deployment_unit.DeploymentUnit')
272 @mock.patch('experimental_framework.benchmarking_unit.'
273 'heat.get_all_heat_templates')
274 def test_run_benchmarks_for_success(self, mock_heat, mock_common_dep_unit,
277 mock_temp_dir, mock_time,
278 mock_rfc2544, mock_log, mock_influx):
279 mock_heat.return_value = list()
280 mock_time.return_value = '12345'
281 mock_temp_dir.return_value = os.path.join(APEX_LAKE_ROOT,
282 'tests/data/test_templates/')
283 common.TEMPLATE_FILE_EXTENSION = '.yaml'
284 common.RESULT_DIR = 'tests/data/results/'
285 common.INFLUXDB_IP = 'InfluxIP'
286 common.INFLUXDB_PORT = '8086'
287 common.INFLUXDB_DB_NAME = 'test_db'
289 heat_template_name = 'VTC_base_single_vm_wait_'
290 openstack_credentials = {
294 heat_template_parameters = {
302 'rfc2544_throughput_benchmark.RFC2544ThroughputBenchmark',
306 bu = BenchmarkingUnit(heat_template_name,
307 openstack_credentials,
308 heat_template_parameters,
311 # bu.data_manager = DummyDataManager('tests/data/results/12345')
312 bu.template_files = ['VTC_base_single_vm_wait_1.yaml',
313 'VTC_base_single_vm_wait_2.yaml']
314 bu.benchmarks = [Dummy_2544('dummy', {'param1': 'val1'})]
316 self.assertEqual(bu.benchmarks[0].init(True), 2)
317 self.assertEqual(bu.benchmarks[0].finalize(True), 2)
318 self.assertEqual(bu.benchmarks[0].run(True), 2)
319 # expected_metric = \
320 # 'throughput,vnic_type=direct,ram=1024,benchmark=dummy,' \
321 # 'vcpus=2,experiment_name=VTC_base_single_vm_wait_2,' \
322 # 'param1=val1 value=10 12345000000000'
323 # mock_influx.assert_called_with(expected_metric)
325 @mock.patch('experimental_framework.common.LOG')
326 @mock.patch('experimental_framework.benchmarks.'
327 'rfc2544_throughput_benchmark', side_effect=Dummy_2544)
328 @mock.patch('time.time')
329 @mock.patch('experimental_framework.common.get_template_dir')
330 # @mock.patch('experimental_framework.data_manager.DataManager',
331 # side_effect=DummyDataManager)
332 @mock.patch('experimental_framework.common.DEPLOYMENT_UNIT')
333 @mock.patch('experimental_framework.deployment_unit.DeploymentUnit')
334 @mock.patch('experimental_framework.benchmarking_unit.'
335 'heat.get_all_heat_templates')
336 def test_run_benchmarks_2_for_success(
337 self, mock_heat, mock_common_dep_unit, mock_dep_unit,
339 mock_temp_dir, mock_time, mock_rfc2544,
341 mock_heat.return_value = list()
342 mock_time.return_value = '12345'
343 mock_temp_dir.return_value = os.path.join(APEX_LAKE_ROOT,
344 'tests/data/test_templates/')
345 common.TEMPLATE_FILE_EXTENSION = '.yaml'
346 common.RESULT_DIR = 'tests/data/results/'
348 heat_template_name = 'VTC_base_single_vm_wait_'
349 openstack_credentials = {
353 heat_template_parameters = {
361 'rfc2544_throughput_benchmark.RFC2544ThroughputBenchmark',
365 bu = BenchmarkingUnit(heat_template_name,
366 openstack_credentials,
367 heat_template_parameters,
370 # bu.data_manager = DummyDataManager('tests/data/results/12345')
371 bu.template_files = ['VTC_base_single_vm_wait_1.yaml',
372 'VTC_base_single_vm_wait_2.yaml']
373 bu.benchmarks = [Dummy_2544('dummy', dict())]
374 common.DEPLOYMENT_UNIT = DummyDeploymentUnit(dict())
376 self.assertEqual(bu.benchmarks[0].init(True), 2)
377 self.assertEqual(bu.benchmarks[0].finalize(True), 0)
378 self.assertEqual(bu.benchmarks[0].run(True), 0)
380 @mock.patch('experimental_framework.common.LOG')
381 @mock.patch('experimental_framework.benchmarks.'
382 'rfc2544_throughput_benchmark', side_effect=Dummy_2544)
383 @mock.patch('time.time')
384 @mock.patch('experimental_framework.common.get_template_dir')
385 # @mock.patch('experimental_framework.data_manager.DataManager',
386 # side_effect=DummyDataManager)
387 @mock.patch('experimental_framework.common.DEPLOYMENT_UNIT')
388 @mock.patch('experimental_framework.deployment_unit.DeploymentUnit')
389 @mock.patch('experimental_framework.benchmarking_unit.'
390 'heat.get_all_heat_templates')
391 def test_get_benchmark_name_for_success(
392 self, mock_heat, mock_common_dep_unit, mock_dep_unit,
394 mock_temp_dir, mock_time, mock_rfc2544,
396 mock_heat.return_value = list()
397 mock_time.return_value = '12345'
398 mock_temp_dir.return_value = 'tests/data/test_templates/'
399 common.TEMPLATE_FILE_EXTENSION = '.yaml'
400 common.RESULT_DIR = 'tests/data/results/'
402 heat_template_name = 'VTC_base_single_vm_wait_'
403 openstack_credentials = {
407 heat_template_parameters = {
415 'rfc2544_throughput_benchmark.RFC2544ThroughputBenchmark',
419 bu = BenchmarkingUnit(heat_template_name,
420 openstack_credentials,
421 heat_template_parameters,
425 expected = 'rfc2544_throughput_benchmark.RFC2544ThroughputBenchmark_0'
426 output = bu.get_benchmark_name(
427 'rfc2544_throughput_benchmark.RFC2544ThroughputBenchmark')
428 self.assertEqual(expected, output)
430 expected = 'rfc2544_throughput_benchmark.RFC2544ThroughputBenchmark_1'
431 output = bu.get_benchmark_name(
432 'rfc2544_throughput_benchmark.RFC2544ThroughputBenchmark')
433 self.assertEqual(expected, output)
435 @mock.patch('experimental_framework.common.LOG')
436 @mock.patch('experimental_framework.benchmarks.'
437 'rfc2544_throughput_benchmark', side_effect=Dummy_2544)
438 @mock.patch('time.time')
439 @mock.patch('experimental_framework.common.get_template_dir')
440 # @mock.patch('experimental_framework.data_manager.DataManager',
441 # side_effect=DummyDataManager)
442 @mock.patch('experimental_framework.common.DEPLOYMENT_UNIT')
443 @mock.patch('experimental_framework.deployment_unit.DeploymentUnit')
444 @mock.patch('experimental_framework.benchmarking_unit.'
445 'heat.get_all_heat_templates')
446 def test_get_required_benchmarks_for_success(
447 self, mock_heat, mock_common_dep_unit, mock_dep_unit,
449 mock_temp_dir, mock_time, mock_rfc2544,
451 mock_heat.return_value = list()
452 mock_time.return_value = '12345'
453 mock_temp_dir.return_value = 'tests/data/test_templates/'
454 common.TEMPLATE_FILE_EXTENSION = '.yaml'
455 common.RESULT_DIR = 'tests/data/results/'
456 openstack_credentials = {
460 heat_template_parameters = {
468 'rfc2544_throughput_benchmark.RFC2544ThroughputBenchmark',
472 bu = BenchmarkingUnit('',
473 openstack_credentials,
474 heat_template_parameters,
478 ['rfc2544_throughput_benchmark.RFC2544ThroughputBenchmark']
479 output = bu.get_required_benchmarks(req_benchs)
480 self.assertEqual(len(req_benchs), 1)
481 self.assertEqual(output[0].__class__, Dummy_2544)