1 # Copyright (c) 2015 Intel Research and Development Ireland Ltd.
3 # Licensed under the Apache License, Version 2.0 (the "License");
4 # you may not use this file except in compliance with the License.
5 # You may obtain a copy of the License at
7 # http://www.apache.org/licenses/LICENSE-2.0
9 # Unless required by applicable law or agreed to in writing, software
10 # distributed under the License is distributed on an "AS IS" BASIS,
11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 # See the License for the specific language governing permissions and
13 # limitations under the License.
15 __author__ = 'vmriccox'
20 from experimental_framework.benchmarking_unit import BenchmarkingUnit
21 from experimental_framework.data_manager import DataManager
22 from experimental_framework.deployment_unit import DeploymentUnit
23 import experimental_framework.common as common
24 from experimental_framework.benchmarks.rfc2544_throughput_benchmark import \
25 RFC2544ThroughputBenchmark
28 class DummyDataManager(DataManager):
30 def __init__(self, experiment_directory):
31 self.experiment_directory = experiment_directory
32 self.experiments = dict()
33 self.new_exp_counter = 0
34 self.add_bench_counter = 0
35 self.close_experiment_1_counter = 0
36 self.close_experiment_2_counter = 0
37 self.generate_csv_counter = 0
39 def create_new_experiment(self, experiment_name, get_counter=None):
41 self.new_exp_counter += 1
43 return self.new_exp_counter
45 def add_benchmark(self, experiment_name, benchmark_name, get_counter=None):
47 self.add_bench_counter += 1
49 return self.add_bench_counter
51 def close_experiment(self, experiment, get_counter=None):
53 return [self.close_experiment_1_counter,
54 self.close_experiment_2_counter]
55 if experiment == 'VTC_base_single_vm_wait_1':
56 self.close_experiment_1_counter += 1
57 if experiment == 'VTC_base_single_vm_wait_2':
58 self.close_experiment_2_counter += 1
60 def generate_result_csv_file(self, get_counter=None):
62 return self.generate_csv_counter
64 self.generate_csv_counter += 1
66 def add_metadata(self, experiment_name, metadata):
69 def add_configuration(self, experiment_name, configuration):
72 def add_data_points(self, experiment_name, benchmark_name, result):
76 class Dummy_2544(RFC2544ThroughputBenchmark):
78 def __init__(self, name, params):
81 self.finalize_counter = 0
85 def init(self, get_counter=None):
87 return self.init_counter
89 self.init_counter += 1
91 def finalize(self, get_counter=None):
93 return self.finalize_counter
95 self.finalize_counter += 1
97 def run(self, get_counter=None):
99 return self.run_counter
101 self.run_counter += 1
102 return {'throughput': 10}
105 class DummyDeploymentUnit(DeploymentUnit):
107 def __init__(self, openstack_credentials):
110 def deploy_heat_template(self, template_file, stack_name, parameters,
115 class TestBenchmarkingUnit(unittest.TestCase):
123 @mock.patch('time.time')
124 @mock.patch('experimental_framework.common.get_template_dir')
125 @mock.patch('experimental_framework.data_manager.DataManager',
126 side_effect=DummyDataManager)
127 @mock.patch('experimental_framework.deployment_unit.DeploymentUnit')
128 @mock.patch('experimental_framework.benchmarking_unit.heat.'
129 'get_all_heat_templates')
130 def test___init__(self, mock_heat, mock_dep_unit, mock_data_manager,
131 mock_temp_dir, mock_time):
132 mock_heat.return_value = list()
133 mock_time.return_value = '12345'
134 mock_temp_dir.return_value = 'tests/data/results/'
135 common.TEMPLATE_FILE_EXTENSION = '.ext'
136 common.RESULT_DIR = 'tests/data/results/'
137 heat_template_name = 'name'
138 openstack_credentials = {
142 heat_template_parameters = {
147 benchmarks = ['bench_1', 'bench_2']
148 bu = BenchmarkingUnit(heat_template_name,
149 openstack_credentials,
150 heat_template_parameters,
153 self.assertEqual(bu.required_benchmarks, benchmarks)
154 bu.heat_template_parameters = heat_template_parameters
155 mock_data_manager.assert_called_once_with('tests/data/results/12345')
156 mock_dep_unit.assert_called_once_with(openstack_credentials)
157 mock_heat.assert_called_once_with('tests/data/results/', '.ext')
159 @mock.patch('experimental_framework.benchmarks.'
160 'rfc2544_throughput_benchmark', side_effect=Dummy_2544)
161 @mock.patch('time.time')
162 @mock.patch('experimental_framework.common.get_template_dir')
163 @mock.patch('experimental_framework.data_manager.DataManager',
164 side_effect=DummyDataManager)
165 @mock.patch('experimental_framework.deployment_unit.DeploymentUnit')
166 @mock.patch('experimental_framework.benchmarking_unit.'
167 'heat.get_all_heat_templates')
168 def test_initialize_for_success(self, mock_heat, mock_dep_unit,
169 mock_data_manager, mock_temp_dir,
170 mock_time, mock_rfc2544):
171 mock_heat.return_value = list()
172 mock_time.return_value = '12345'
173 mock_temp_dir.return_value = 'tests/data/test_templates/'
174 common.TEMPLATE_FILE_EXTENSION = '.yaml'
175 common.RESULT_DIR = 'tests/data/results/'
177 heat_template_name = 'VTC_base_single_vm_wait_'
178 openstack_credentials = {
182 heat_template_parameters = {
190 'rfc2544_throughput_benchmark.RFC2544ThroughputBenchmark',
194 bu = BenchmarkingUnit(heat_template_name,
195 openstack_credentials,
196 heat_template_parameters,
199 self.assertEqual(bu.required_benchmarks, benchmarks)
200 bu.heat_template_parameters = heat_template_parameters
201 bu.template_files = ['VTC_base_single_vm_wait_1.yaml',
202 'VTC_base_single_vm_wait_2.yaml']
204 self.assertTrue(len(bu.benchmarks) == 1)
205 self.assertEqual(bu.benchmarks[0].__class__,
207 self.assertEqual(bu.data_manager.create_new_experiment('', True), 2)
208 self.assertEqual(bu.data_manager.add_benchmark('', '', True), 2)
210 @mock.patch('experimental_framework.benchmarks.'
211 'rfc2544_throughput_benchmark', side_effect=Dummy_2544)
212 @mock.patch('time.time')
213 @mock.patch('experimental_framework.common.get_template_dir')
214 @mock.patch('experimental_framework.data_manager.DataManager',
215 side_effect=DummyDataManager)
216 @mock.patch('experimental_framework.deployment_unit.DeploymentUnit')
217 @mock.patch('experimental_framework.benchmarking_unit.'
218 'heat.get_all_heat_templates')
219 def test_finalize_for_success(
220 self, mock_heat, mock_dep_unit, mock_data_manager, mock_temp_dir,
221 mock_time, mock_rfc2544):
222 mock_heat.return_value = list()
223 mock_time.return_value = '12345'
224 mock_temp_dir.return_value = 'tests/data/test_templates/'
225 common.TEMPLATE_FILE_EXTENSION = '.yaml'
226 common.RESULT_DIR = 'tests/data/results/'
228 heat_template_name = 'VTC_base_single_vm_wait_'
229 openstack_credentials = {
233 heat_template_parameters = {
241 'rfc2544_throughput_benchmark.RFC2544ThroughputBenchmark',
245 bu = BenchmarkingUnit(heat_template_name,
246 openstack_credentials,
247 heat_template_parameters,
250 bu.heat_template_parameters = heat_template_parameters
251 bu.template_files = ['VTC_base_single_vm_wait_1.yaml',
252 'VTC_base_single_vm_wait_2.yaml']
254 # self.assertEqual(bu.data_manager.close_experiment('', True), [1, 1])
255 self.assertEqual(bu.data_manager.generate_result_csv_file(True), 1)
257 @mock.patch('experimental_framework.common.push_data_influxdb')
258 @mock.patch('experimental_framework.common.LOG')
259 @mock.patch('experimental_framework.benchmarks.'
260 'rfc2544_throughput_benchmark', side_effect=Dummy_2544)
261 @mock.patch('time.time')
262 @mock.patch('experimental_framework.common.get_template_dir')
263 @mock.patch('experimental_framework.data_manager.DataManager',
264 side_effect=DummyDataManager)
265 @mock.patch('experimental_framework.common.DEPLOYMENT_UNIT')
266 @mock.patch('experimental_framework.deployment_unit.DeploymentUnit')
267 @mock.patch('experimental_framework.benchmarking_unit.'
268 'heat.get_all_heat_templates')
269 def test_run_benchmarks_for_success(self, mock_heat, mock_common_dep_unit,
270 mock_dep_unit, mock_data_manager,
271 mock_temp_dir, mock_time,
272 mock_rfc2544, mock_log, mock_influx):
273 mock_heat.return_value = list()
274 mock_time.return_value = '12345'
275 mock_temp_dir.return_value = 'tests/data/test_templates/'
276 common.TEMPLATE_FILE_EXTENSION = '.yaml'
277 common.RESULT_DIR = 'tests/data/results/'
278 common.INFLUXDB_IP = 'InfluxIP'
279 common.INFLUXDB_PORT = '8086'
280 common.INFLUXDB_DB_NAME = 'test_db'
282 heat_template_name = 'VTC_base_single_vm_wait_'
283 openstack_credentials = {
287 heat_template_parameters = {
295 'rfc2544_throughput_benchmark.RFC2544ThroughputBenchmark',
299 bu = BenchmarkingUnit(heat_template_name,
300 openstack_credentials,
301 heat_template_parameters,
304 bu.data_manager = DummyDataManager('tests/data/results/12345')
305 bu.template_files = ['VTC_base_single_vm_wait_1.yaml',
306 'VTC_base_single_vm_wait_2.yaml']
307 bu.benchmarks = [Dummy_2544('dummy', {'param1': 'val1'})]
309 self.assertEqual(bu.benchmarks[0].init(True), 2)
310 self.assertEqual(bu.benchmarks[0].finalize(True), 2)
311 self.assertEqual(bu.benchmarks[0].run(True), 2)
312 # expected_metric = \
313 # 'throughput,vnic_type=direct,ram=1024,benchmark=dummy,' \
314 # 'vcpus=2,experiment_name=VTC_base_single_vm_wait_2,' \
315 # 'param1=val1 value=10 12345000000000'
316 # mock_influx.assert_called_with(expected_metric)
318 @mock.patch('experimental_framework.common.LOG')
319 @mock.patch('experimental_framework.benchmarks.'
320 'rfc2544_throughput_benchmark', side_effect=Dummy_2544)
321 @mock.patch('time.time')
322 @mock.patch('experimental_framework.common.get_template_dir')
323 @mock.patch('experimental_framework.data_manager.DataManager',
324 side_effect=DummyDataManager)
325 @mock.patch('experimental_framework.common.DEPLOYMENT_UNIT')
326 @mock.patch('experimental_framework.deployment_unit.DeploymentUnit')
327 @mock.patch('experimental_framework.benchmarking_unit.'
328 'heat.get_all_heat_templates')
329 def test_run_benchmarks_2_for_success(
330 self, mock_heat, mock_common_dep_unit, mock_dep_unit,
331 mock_data_manager, mock_temp_dir, mock_time, mock_rfc2544,
333 mock_heat.return_value = list()
334 mock_time.return_value = '12345'
335 mock_temp_dir.return_value = 'tests/data/test_templates/'
336 common.TEMPLATE_FILE_EXTENSION = '.yaml'
337 common.RESULT_DIR = 'tests/data/results/'
339 heat_template_name = 'VTC_base_single_vm_wait_'
340 openstack_credentials = {
344 heat_template_parameters = {
352 'rfc2544_throughput_benchmark.RFC2544ThroughputBenchmark',
356 bu = BenchmarkingUnit(heat_template_name,
357 openstack_credentials,
358 heat_template_parameters,
361 bu.data_manager = DummyDataManager('tests/data/results/12345')
362 bu.template_files = ['VTC_base_single_vm_wait_1.yaml',
363 'VTC_base_single_vm_wait_2.yaml']
364 bu.benchmarks = [Dummy_2544('dummy', dict())]
365 common.DEPLOYMENT_UNIT = DummyDeploymentUnit(dict())
367 self.assertEqual(bu.benchmarks[0].init(True), 2)
368 self.assertEqual(bu.benchmarks[0].finalize(True), 0)
369 self.assertEqual(bu.benchmarks[0].run(True), 0)
371 @mock.patch('experimental_framework.common.LOG')
372 @mock.patch('experimental_framework.benchmarks.'
373 'rfc2544_throughput_benchmark', side_effect=Dummy_2544)
374 @mock.patch('time.time')
375 @mock.patch('experimental_framework.common.get_template_dir')
376 @mock.patch('experimental_framework.data_manager.DataManager',
377 side_effect=DummyDataManager)
378 @mock.patch('experimental_framework.common.DEPLOYMENT_UNIT')
379 @mock.patch('experimental_framework.deployment_unit.DeploymentUnit')
380 @mock.patch('experimental_framework.benchmarking_unit.'
381 'heat.get_all_heat_templates')
382 def test_get_benchmark_name_for_success(
383 self, mock_heat, mock_common_dep_unit, mock_dep_unit,
384 mock_data_manager, mock_temp_dir, mock_time, mock_rfc2544,
386 mock_heat.return_value = list()
387 mock_time.return_value = '12345'
388 mock_temp_dir.return_value = 'tests/data/test_templates/'
389 common.TEMPLATE_FILE_EXTENSION = '.yaml'
390 common.RESULT_DIR = 'tests/data/results/'
392 heat_template_name = 'VTC_base_single_vm_wait_'
393 openstack_credentials = {
397 heat_template_parameters = {
405 'rfc2544_throughput_benchmark.RFC2544ThroughputBenchmark',
409 bu = BenchmarkingUnit(heat_template_name,
410 openstack_credentials,
411 heat_template_parameters,
415 expected = 'rfc2544_throughput_benchmark.RFC2544ThroughputBenchmark_0'
416 output = bu.get_benchmark_name(
417 'rfc2544_throughput_benchmark.RFC2544ThroughputBenchmark')
418 self.assertEqual(expected, output)
420 expected = 'rfc2544_throughput_benchmark.RFC2544ThroughputBenchmark_1'
421 output = bu.get_benchmark_name(
422 'rfc2544_throughput_benchmark.RFC2544ThroughputBenchmark')
423 self.assertEqual(expected, output)
425 @mock.patch('experimental_framework.common.LOG')
426 @mock.patch('experimental_framework.benchmarks.'
427 'rfc2544_throughput_benchmark', side_effect=Dummy_2544)
428 @mock.patch('time.time')
429 @mock.patch('experimental_framework.common.get_template_dir')
430 @mock.patch('experimental_framework.data_manager.DataManager',
431 side_effect=DummyDataManager)
432 @mock.patch('experimental_framework.common.DEPLOYMENT_UNIT')
433 @mock.patch('experimental_framework.deployment_unit.DeploymentUnit')
434 @mock.patch('experimental_framework.benchmarking_unit.'
435 'heat.get_all_heat_templates')
436 def test_get_required_benchmarks_for_success(
437 self, mock_heat, mock_common_dep_unit, mock_dep_unit,
438 mock_data_manager, mock_temp_dir, mock_time, mock_rfc2544,
440 mock_heat.return_value = list()
441 mock_time.return_value = '12345'
442 mock_temp_dir.return_value = 'tests/data/test_templates/'
443 common.TEMPLATE_FILE_EXTENSION = '.yaml'
444 common.RESULT_DIR = 'tests/data/results/'
445 openstack_credentials = {
449 heat_template_parameters = {
457 'rfc2544_throughput_benchmark.RFC2544ThroughputBenchmark',
461 bu = BenchmarkingUnit('',
462 openstack_credentials,
463 heat_template_parameters,
467 ['rfc2544_throughput_benchmark.RFC2544ThroughputBenchmark']
468 output = bu.get_required_benchmarks(req_benchs)
469 self.assertEqual(len(req_benchs), 1)
470 self.assertEqual(output[0].__class__, Dummy_2544)