1 # Copyright (c) 2015 Intel Research and Development Ireland Ltd.
3 # Licensed under the Apache License, Version 2.0 (the "License");
4 # you may not use this file except in compliance with the License.
5 # You may obtain a copy of the License at
7 # http://www.apache.org/licenses/LICENSE-2.0
9 # Unless required by applicable law or agreed to in writing, software
10 # distributed under the License is distributed on an "AS IS" BASIS,
11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 # See the License for the specific language governing permissions and
13 # limitations under the License.
15 __author__ = 'vmriccox'
20 from experimental_framework.benchmarking_unit import BenchmarkingUnit
21 # from experimental_framework.data_manager import DataManager
22 from experimental_framework.deployment_unit import DeploymentUnit
23 import experimental_framework.common as common
24 from experimental_framework.benchmarks.rfc2544_throughput_benchmark import \
25 RFC2544ThroughputBenchmark
28 # class DummyDataManager(DataManager):
30 # def __init__(self, experiment_directory):
31 # self.experiment_directory = experiment_directory
32 # self.experiments = dict()
33 # self.new_exp_counter = 0
34 # self.add_bench_counter = 0
35 # self.close_experiment_1_counter = 0
36 # self.close_experiment_2_counter = 0
37 # self.generate_csv_counter = 0
39 # def create_new_experiment(self, experiment_name, get_counter=None):
41 # self.new_exp_counter += 1
43 # return self.new_exp_counter
45 # def add_benchmark(self, experiment_name, benchmark_name, get_counter=None):
47 # self.add_bench_counter += 1
49 # return self.add_bench_counter
51 # def close_experiment(self, experiment, get_counter=None):
53 # return [self.close_experiment_1_counter,
54 # self.close_experiment_2_counter]
55 # if experiment == 'VTC_base_single_vm_wait_1':
56 # self.close_experiment_1_counter += 1
57 # if experiment == 'VTC_base_single_vm_wait_2':
58 # self.close_experiment_2_counter += 1
60 # def generate_result_csv_file(self, get_counter=None):
62 # return self.generate_csv_counter
64 # self.generate_csv_counter += 1
66 # def add_metadata(self, experiment_name, metadata):
69 # def add_configuration(self, experiment_name, configuration):
72 # def add_data_points(self, experiment_name, benchmark_name, result):
76 class Dummy_2544(RFC2544ThroughputBenchmark):
78 def __init__(self, name, params):
81 self.finalize_counter = 0
85 def init(self, get_counter=None):
87 return self.init_counter
89 self.init_counter += 1
91 def finalize(self, get_counter=None):
93 return self.finalize_counter
95 self.finalize_counter += 1
97 def run(self, get_counter=None):
99 return self.run_counter
101 self.run_counter += 1
102 return {'throughput': 10}
105 class DummyDeploymentUnit(DeploymentUnit):
107 def __init__(self, openstack_credentials):
110 def deploy_heat_template(self, template_file, stack_name, parameters,
115 class TestBenchmarkingUnit(unittest.TestCase):
123 @mock.patch('time.time')
124 @mock.patch('experimental_framework.common.get_template_dir')
125 # @mock.patch('experimental_framework.data_manager.DataManager',
126 # side_effect=DummyDataManager)
127 @mock.patch('experimental_framework.deployment_unit.DeploymentUnit')
128 @mock.patch('experimental_framework.benchmarking_unit.heat.'
129 'get_all_heat_templates')
130 def test___init__(self, mock_heat, mock_dep_unit,
132 mock_temp_dir, mock_time):
133 mock_heat.return_value = list()
134 mock_time.return_value = '12345'
135 mock_temp_dir.return_value = 'tests/data/results/'
136 common.TEMPLATE_FILE_EXTENSION = '.ext'
137 common.RESULT_DIR = 'tests/data/results/'
138 heat_template_name = 'name'
139 openstack_credentials = {
143 heat_template_parameters = {
148 benchmarks = ['bench_1', 'bench_2']
149 bu = BenchmarkingUnit(heat_template_name,
150 openstack_credentials,
151 heat_template_parameters,
154 self.assertEqual(bu.required_benchmarks, benchmarks)
155 bu.heat_template_parameters = heat_template_parameters
156 # mock_data_manager.assert_called_once_with('tests/data/results/12345')
157 mock_dep_unit.assert_called_once_with(openstack_credentials)
158 mock_heat.assert_called_once_with('tests/data/results/', '.ext')
160 @mock.patch('experimental_framework.benchmarks.'
161 'rfc2544_throughput_benchmark', side_effect=Dummy_2544)
162 @mock.patch('time.time')
163 @mock.patch('experimental_framework.common.get_template_dir')
164 # @mock.patch('experimental_framework.data_manager.DataManager',
165 # side_effect=DummyDataManager)
166 @mock.patch('experimental_framework.deployment_unit.DeploymentUnit')
167 @mock.patch('experimental_framework.benchmarking_unit.'
168 'heat.get_all_heat_templates')
169 def test_initialize_for_success(self, mock_heat, mock_dep_unit,
172 mock_time, mock_rfc2544):
173 mock_heat.return_value = list()
174 mock_time.return_value = '12345'
175 mock_temp_dir.return_value = 'tests/data/test_templates/'
176 common.TEMPLATE_FILE_EXTENSION = '.yaml'
177 common.RESULT_DIR = 'tests/data/results/'
179 heat_template_name = 'VTC_base_single_vm_wait_'
180 openstack_credentials = {
184 heat_template_parameters = {
192 'rfc2544_throughput_benchmark.RFC2544ThroughputBenchmark',
196 bu = BenchmarkingUnit(heat_template_name,
197 openstack_credentials,
198 heat_template_parameters,
201 self.assertEqual(bu.required_benchmarks, benchmarks)
202 bu.heat_template_parameters = heat_template_parameters
203 bu.template_files = ['VTC_base_single_vm_wait_1.yaml',
204 'VTC_base_single_vm_wait_2.yaml']
206 self.assertTrue(len(bu.benchmarks) == 1)
207 self.assertEqual(bu.benchmarks[0].__class__,
209 # self.assertEqual(bu.data_manager.create_new_experiment('', True), 2)
210 # self.assertEqual(bu.data_manager.add_benchmark('', '', True), 2)
212 @mock.patch('experimental_framework.benchmarks.'
213 'rfc2544_throughput_benchmark', side_effect=Dummy_2544)
214 @mock.patch('time.time')
215 @mock.patch('experimental_framework.common.get_template_dir')
216 # @mock.patch('experimental_framework.data_manager.DataManager',
217 # side_effect=DummyDataManager)
218 @mock.patch('experimental_framework.deployment_unit.DeploymentUnit')
219 @mock.patch('experimental_framework.benchmarking_unit.'
220 'heat.get_all_heat_templates')
221 def test_finalize_for_success(
222 self, mock_heat, mock_dep_unit,
224 mock_temp_dir, mock_time, mock_rfc2544):
225 mock_heat.return_value = list()
226 mock_time.return_value = '12345'
227 mock_temp_dir.return_value = 'tests/data/test_templates/'
228 common.TEMPLATE_FILE_EXTENSION = '.yaml'
229 common.RESULT_DIR = 'tests/data/results/'
231 heat_template_name = 'VTC_base_single_vm_wait_'
232 openstack_credentials = {
236 heat_template_parameters = {
244 'rfc2544_throughput_benchmark.RFC2544ThroughputBenchmark',
248 bu = BenchmarkingUnit(heat_template_name,
249 openstack_credentials,
250 heat_template_parameters,
253 bu.heat_template_parameters = heat_template_parameters
254 bu.template_files = ['VTC_base_single_vm_wait_1.yaml',
255 'VTC_base_single_vm_wait_2.yaml']
257 # self.assertEqual(bu.data_manager.close_experiment('', True), [1, 1])
258 # self.assertEqual(bu.data_manager.generate_result_csv_file(True), 1)
260 @mock.patch('experimental_framework.common.push_data_influxdb')
261 @mock.patch('experimental_framework.common.LOG')
262 @mock.patch('experimental_framework.benchmarks.'
263 'rfc2544_throughput_benchmark', side_effect=Dummy_2544)
264 @mock.patch('time.time')
265 @mock.patch('experimental_framework.common.get_template_dir')
266 # @mock.patch('experimental_framework.data_manager.DataManager',
267 # side_effect=DummyDataManager)
268 @mock.patch('experimental_framework.common.DEPLOYMENT_UNIT')
269 @mock.patch('experimental_framework.deployment_unit.DeploymentUnit')
270 @mock.patch('experimental_framework.benchmarking_unit.'
271 'heat.get_all_heat_templates')
272 def test_run_benchmarks_for_success(self, mock_heat, mock_common_dep_unit,
275 mock_temp_dir, mock_time,
276 mock_rfc2544, mock_log, mock_influx):
277 mock_heat.return_value = list()
278 mock_time.return_value = '12345'
279 mock_temp_dir.return_value = 'tests/data/test_templates/'
280 common.TEMPLATE_FILE_EXTENSION = '.yaml'
281 common.RESULT_DIR = 'tests/data/results/'
282 common.INFLUXDB_IP = 'InfluxIP'
283 common.INFLUXDB_PORT = '8086'
284 common.INFLUXDB_DB_NAME = 'test_db'
286 heat_template_name = 'VTC_base_single_vm_wait_'
287 openstack_credentials = {
291 heat_template_parameters = {
299 'rfc2544_throughput_benchmark.RFC2544ThroughputBenchmark',
303 bu = BenchmarkingUnit(heat_template_name,
304 openstack_credentials,
305 heat_template_parameters,
308 # bu.data_manager = DummyDataManager('tests/data/results/12345')
309 bu.template_files = ['VTC_base_single_vm_wait_1.yaml',
310 'VTC_base_single_vm_wait_2.yaml']
311 bu.benchmarks = [Dummy_2544('dummy', {'param1': 'val1'})]
313 self.assertEqual(bu.benchmarks[0].init(True), 2)
314 self.assertEqual(bu.benchmarks[0].finalize(True), 2)
315 self.assertEqual(bu.benchmarks[0].run(True), 2)
316 # expected_metric = \
317 # 'throughput,vnic_type=direct,ram=1024,benchmark=dummy,' \
318 # 'vcpus=2,experiment_name=VTC_base_single_vm_wait_2,' \
319 # 'param1=val1 value=10 12345000000000'
320 # mock_influx.assert_called_with(expected_metric)
322 @mock.patch('experimental_framework.common.LOG')
323 @mock.patch('experimental_framework.benchmarks.'
324 'rfc2544_throughput_benchmark', side_effect=Dummy_2544)
325 @mock.patch('time.time')
326 @mock.patch('experimental_framework.common.get_template_dir')
327 # @mock.patch('experimental_framework.data_manager.DataManager',
328 # side_effect=DummyDataManager)
329 @mock.patch('experimental_framework.common.DEPLOYMENT_UNIT')
330 @mock.patch('experimental_framework.deployment_unit.DeploymentUnit')
331 @mock.patch('experimental_framework.benchmarking_unit.'
332 'heat.get_all_heat_templates')
333 def test_run_benchmarks_2_for_success(
334 self, mock_heat, mock_common_dep_unit, mock_dep_unit,
336 mock_temp_dir, mock_time, mock_rfc2544,
338 mock_heat.return_value = list()
339 mock_time.return_value = '12345'
340 mock_temp_dir.return_value = 'tests/data/test_templates/'
341 common.TEMPLATE_FILE_EXTENSION = '.yaml'
342 common.RESULT_DIR = 'tests/data/results/'
344 heat_template_name = 'VTC_base_single_vm_wait_'
345 openstack_credentials = {
349 heat_template_parameters = {
357 'rfc2544_throughput_benchmark.RFC2544ThroughputBenchmark',
361 bu = BenchmarkingUnit(heat_template_name,
362 openstack_credentials,
363 heat_template_parameters,
366 # bu.data_manager = DummyDataManager('tests/data/results/12345')
367 bu.template_files = ['VTC_base_single_vm_wait_1.yaml',
368 'VTC_base_single_vm_wait_2.yaml']
369 bu.benchmarks = [Dummy_2544('dummy', dict())]
370 common.DEPLOYMENT_UNIT = DummyDeploymentUnit(dict())
372 self.assertEqual(bu.benchmarks[0].init(True), 2)
373 self.assertEqual(bu.benchmarks[0].finalize(True), 0)
374 self.assertEqual(bu.benchmarks[0].run(True), 0)
376 @mock.patch('experimental_framework.common.LOG')
377 @mock.patch('experimental_framework.benchmarks.'
378 'rfc2544_throughput_benchmark', side_effect=Dummy_2544)
379 @mock.patch('time.time')
380 @mock.patch('experimental_framework.common.get_template_dir')
381 # @mock.patch('experimental_framework.data_manager.DataManager',
382 # side_effect=DummyDataManager)
383 @mock.patch('experimental_framework.common.DEPLOYMENT_UNIT')
384 @mock.patch('experimental_framework.deployment_unit.DeploymentUnit')
385 @mock.patch('experimental_framework.benchmarking_unit.'
386 'heat.get_all_heat_templates')
387 def test_get_benchmark_name_for_success(
388 self, mock_heat, mock_common_dep_unit, mock_dep_unit,
390 mock_temp_dir, mock_time, mock_rfc2544,
392 mock_heat.return_value = list()
393 mock_time.return_value = '12345'
394 mock_temp_dir.return_value = 'tests/data/test_templates/'
395 common.TEMPLATE_FILE_EXTENSION = '.yaml'
396 common.RESULT_DIR = 'tests/data/results/'
398 heat_template_name = 'VTC_base_single_vm_wait_'
399 openstack_credentials = {
403 heat_template_parameters = {
411 'rfc2544_throughput_benchmark.RFC2544ThroughputBenchmark',
415 bu = BenchmarkingUnit(heat_template_name,
416 openstack_credentials,
417 heat_template_parameters,
421 expected = 'rfc2544_throughput_benchmark.RFC2544ThroughputBenchmark_0'
422 output = bu.get_benchmark_name(
423 'rfc2544_throughput_benchmark.RFC2544ThroughputBenchmark')
424 self.assertEqual(expected, output)
426 expected = 'rfc2544_throughput_benchmark.RFC2544ThroughputBenchmark_1'
427 output = bu.get_benchmark_name(
428 'rfc2544_throughput_benchmark.RFC2544ThroughputBenchmark')
429 self.assertEqual(expected, output)
431 @mock.patch('experimental_framework.common.LOG')
432 @mock.patch('experimental_framework.benchmarks.'
433 'rfc2544_throughput_benchmark', side_effect=Dummy_2544)
434 @mock.patch('time.time')
435 @mock.patch('experimental_framework.common.get_template_dir')
436 # @mock.patch('experimental_framework.data_manager.DataManager',
437 # side_effect=DummyDataManager)
438 @mock.patch('experimental_framework.common.DEPLOYMENT_UNIT')
439 @mock.patch('experimental_framework.deployment_unit.DeploymentUnit')
440 @mock.patch('experimental_framework.benchmarking_unit.'
441 'heat.get_all_heat_templates')
442 def test_get_required_benchmarks_for_success(
443 self, mock_heat, mock_common_dep_unit, mock_dep_unit,
445 mock_temp_dir, mock_time, mock_rfc2544,
447 mock_heat.return_value = list()
448 mock_time.return_value = '12345'
449 mock_temp_dir.return_value = 'tests/data/test_templates/'
450 common.TEMPLATE_FILE_EXTENSION = '.yaml'
451 common.RESULT_DIR = 'tests/data/results/'
452 openstack_credentials = {
456 heat_template_parameters = {
464 'rfc2544_throughput_benchmark.RFC2544ThroughputBenchmark',
468 bu = BenchmarkingUnit('',
469 openstack_credentials,
470 heat_template_parameters,
474 ['rfc2544_throughput_benchmark.RFC2544ThroughputBenchmark']
475 output = bu.get_required_benchmarks(req_benchs)
476 self.assertEqual(len(req_benchs), 1)
477 self.assertEqual(output[0].__class__, Dummy_2544)