1 # Copyright (c) 2015 Intel Research and Development Ireland Ltd.
3 # Licensed under the Apache License, Version 2.0 (the "License");
4 # you may not use this file except in compliance with the License.
5 # You may obtain a copy of the License at
7 # http://www.apache.org/licenses/LICENSE-2.0
9 # Unless required by applicable law or agreed to in writing, software
10 # distributed under the License is distributed on an "AS IS" BASIS,
11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 # See the License for the specific language governing permissions and
13 # limitations under the License.
17 from experimental_framework.benchmarking_unit import BenchmarkingUnit
18 # from experimental_framework.data_manager import DataManager
19 from experimental_framework.deployment_unit import DeploymentUnit
20 import experimental_framework.common as common
21 from experimental_framework.benchmarks.rfc2544_throughput_benchmark import \
22 RFC2544ThroughputBenchmark
24 __author__ = 'vmriccox'
27 # class DummyDataManager(DataManager):
29 # def __init__(self, experiment_directory):
30 # self.experiment_directory = experiment_directory
31 # self.experiments = dict()
32 # self.new_exp_counter = 0
33 # self.add_bench_counter = 0
34 # self.close_experiment_1_counter = 0
35 # self.close_experiment_2_counter = 0
36 # self.generate_csv_counter = 0
38 # def create_new_experiment(self, experiment_name, get_counter=None):
40 # self.new_exp_counter += 1
42 # return self.new_exp_counter
44 # def add_benchmark(self, experiment_name, benchmark_name, get_counter=None):
46 # self.add_bench_counter += 1
48 # return self.add_bench_counter
50 # def close_experiment(self, experiment, get_counter=None):
52 # return [self.close_experiment_1_counter,
53 # self.close_experiment_2_counter]
54 # if experiment == 'VTC_base_single_vm_wait_1':
55 # self.close_experiment_1_counter += 1
56 # if experiment == 'VTC_base_single_vm_wait_2':
57 # self.close_experiment_2_counter += 1
59 # def generate_result_csv_file(self, get_counter=None):
61 # return self.generate_csv_counter
63 # self.generate_csv_counter += 1
65 # def add_metadata(self, experiment_name, metadata):
68 # def add_configuration(self, experiment_name, configuration):
71 # def add_data_points(self, experiment_name, benchmark_name, result):
75 class Dummy_2544(RFC2544ThroughputBenchmark):
77 def __init__(self, name, params):
80 self.finalize_counter = 0
84 def init(self, get_counter=None):
86 return self.init_counter
88 self.init_counter += 1
90 def finalize(self, get_counter=None):
92 return self.finalize_counter
94 self.finalize_counter += 1
96 def run(self, get_counter=None):
98 return self.run_counter
100 self.run_counter += 1
101 return {'throughput': 10}
104 class DummyDeploymentUnit(DeploymentUnit):
106 def __init__(self, openstack_credentials):
109 def deploy_heat_template(self, template_file, stack_name, parameters,
114 class TestBenchmarkingUnit(unittest.TestCase):
122 @mock.patch('time.time')
123 @mock.patch('experimental_framework.common.get_template_dir')
124 # @mock.patch('experimental_framework.data_manager.DataManager',
125 # side_effect=DummyDataManager)
126 @mock.patch('experimental_framework.deployment_unit.DeploymentUnit')
127 @mock.patch('experimental_framework.benchmarking_unit.heat.'
128 'get_all_heat_templates')
129 def test___init__(self, mock_heat, mock_dep_unit,
131 mock_temp_dir, mock_time):
132 mock_heat.return_value = list()
133 mock_time.return_value = '12345'
134 mock_temp_dir.return_value = 'tests/data/results/'
135 common.TEMPLATE_FILE_EXTENSION = '.ext'
136 common.RESULT_DIR = 'tests/data/results/'
137 heat_template_name = 'name'
138 openstack_credentials = {
142 heat_template_parameters = {
147 benchmarks = ['bench_1', 'bench_2']
148 bu = BenchmarkingUnit(heat_template_name,
149 openstack_credentials,
150 heat_template_parameters,
153 self.assertEqual(bu.required_benchmarks, benchmarks)
154 bu.heat_template_parameters = heat_template_parameters
155 # mock_data_manager.assert_called_once_with('tests/data/results/12345')
156 mock_dep_unit.assert_called_once_with(openstack_credentials)
157 mock_heat.assert_called_once_with('tests/data/results/', '.ext')
159 @mock.patch('experimental_framework.benchmarks.'
160 'rfc2544_throughput_benchmark', side_effect=Dummy_2544)
161 @mock.patch('time.time')
162 @mock.patch('experimental_framework.common.get_template_dir')
163 # @mock.patch('experimental_framework.data_manager.DataManager',
164 # side_effect=DummyDataManager)
165 @mock.patch('experimental_framework.deployment_unit.DeploymentUnit')
166 @mock.patch('experimental_framework.benchmarking_unit.'
167 'heat.get_all_heat_templates')
168 def test_initialize_for_success(self, mock_heat, mock_dep_unit,
171 mock_time, mock_rfc2544):
172 mock_heat.return_value = list()
173 mock_time.return_value = '12345'
174 mock_temp_dir.return_value = 'tests/data/test_templates/'
175 common.TEMPLATE_FILE_EXTENSION = '.yaml'
176 common.RESULT_DIR = 'tests/data/results/'
178 heat_template_name = 'VTC_base_single_vm_wait_'
179 openstack_credentials = {
183 heat_template_parameters = {
191 'rfc2544_throughput_benchmark.RFC2544ThroughputBenchmark',
195 bu = BenchmarkingUnit(heat_template_name,
196 openstack_credentials,
197 heat_template_parameters,
200 self.assertEqual(bu.required_benchmarks, benchmarks)
201 bu.heat_template_parameters = heat_template_parameters
202 bu.template_files = ['VTC_base_single_vm_wait_1.yaml',
203 'VTC_base_single_vm_wait_2.yaml']
205 self.assertTrue(len(bu.benchmarks) == 1)
206 self.assertEqual(bu.benchmarks[0].__class__,
208 # self.assertEqual(bu.data_manager.create_new_experiment('', True), 2)
209 # self.assertEqual(bu.data_manager.add_benchmark('', '', True), 2)
211 @mock.patch('experimental_framework.benchmarks.'
212 'rfc2544_throughput_benchmark', side_effect=Dummy_2544)
213 @mock.patch('time.time')
214 @mock.patch('experimental_framework.common.get_template_dir')
215 # @mock.patch('experimental_framework.data_manager.DataManager',
216 # side_effect=DummyDataManager)
217 @mock.patch('experimental_framework.deployment_unit.DeploymentUnit')
218 @mock.patch('experimental_framework.benchmarking_unit.'
219 'heat.get_all_heat_templates')
220 def test_finalize_for_success(
221 self, mock_heat, mock_dep_unit,
223 mock_temp_dir, mock_time, mock_rfc2544):
224 mock_heat.return_value = list()
225 mock_time.return_value = '12345'
226 mock_temp_dir.return_value = 'tests/data/test_templates/'
227 common.TEMPLATE_FILE_EXTENSION = '.yaml'
228 common.RESULT_DIR = 'tests/data/results/'
230 heat_template_name = 'VTC_base_single_vm_wait_'
231 openstack_credentials = {
235 heat_template_parameters = {
243 'rfc2544_throughput_benchmark.RFC2544ThroughputBenchmark',
247 bu = BenchmarkingUnit(heat_template_name,
248 openstack_credentials,
249 heat_template_parameters,
252 bu.heat_template_parameters = heat_template_parameters
253 bu.template_files = ['VTC_base_single_vm_wait_1.yaml',
254 'VTC_base_single_vm_wait_2.yaml']
256 # self.assertEqual(bu.data_manager.close_experiment('', True), [1, 1])
257 # self.assertEqual(bu.data_manager.generate_result_csv_file(True), 1)
259 @mock.patch('experimental_framework.common.push_data_influxdb')
260 @mock.patch('experimental_framework.common.LOG')
261 @mock.patch('experimental_framework.benchmarks.'
262 'rfc2544_throughput_benchmark', side_effect=Dummy_2544)
263 @mock.patch('time.time')
264 @mock.patch('experimental_framework.common.get_template_dir')
265 # @mock.patch('experimental_framework.data_manager.DataManager',
266 # side_effect=DummyDataManager)
267 @mock.patch('experimental_framework.common.DEPLOYMENT_UNIT')
268 @mock.patch('experimental_framework.deployment_unit.DeploymentUnit')
269 @mock.patch('experimental_framework.benchmarking_unit.'
270 'heat.get_all_heat_templates')
271 def test_run_benchmarks_for_success(self, mock_heat, mock_common_dep_unit,
274 mock_temp_dir, mock_time,
275 mock_rfc2544, mock_log, mock_influx):
276 mock_heat.return_value = list()
277 mock_time.return_value = '12345'
278 mock_temp_dir.return_value = 'tests/data/test_templates/'
279 common.TEMPLATE_FILE_EXTENSION = '.yaml'
280 common.RESULT_DIR = 'tests/data/results/'
281 common.INFLUXDB_IP = 'InfluxIP'
282 common.INFLUXDB_PORT = '8086'
283 common.INFLUXDB_DB_NAME = 'test_db'
285 heat_template_name = 'VTC_base_single_vm_wait_'
286 openstack_credentials = {
290 heat_template_parameters = {
298 'rfc2544_throughput_benchmark.RFC2544ThroughputBenchmark',
302 bu = BenchmarkingUnit(heat_template_name,
303 openstack_credentials,
304 heat_template_parameters,
307 # bu.data_manager = DummyDataManager('tests/data/results/12345')
308 bu.template_files = ['VTC_base_single_vm_wait_1.yaml',
309 'VTC_base_single_vm_wait_2.yaml']
310 bu.benchmarks = [Dummy_2544('dummy', {'param1': 'val1'})]
312 self.assertEqual(bu.benchmarks[0].init(True), 2)
313 self.assertEqual(bu.benchmarks[0].finalize(True), 2)
314 self.assertEqual(bu.benchmarks[0].run(True), 2)
315 # expected_metric = \
316 # 'throughput,vnic_type=direct,ram=1024,benchmark=dummy,' \
317 # 'vcpus=2,experiment_name=VTC_base_single_vm_wait_2,' \
318 # 'param1=val1 value=10 12345000000000'
319 # mock_influx.assert_called_with(expected_metric)
321 @mock.patch('experimental_framework.common.LOG')
322 @mock.patch('experimental_framework.benchmarks.'
323 'rfc2544_throughput_benchmark', side_effect=Dummy_2544)
324 @mock.patch('time.time')
325 @mock.patch('experimental_framework.common.get_template_dir')
326 # @mock.patch('experimental_framework.data_manager.DataManager',
327 # side_effect=DummyDataManager)
328 @mock.patch('experimental_framework.common.DEPLOYMENT_UNIT')
329 @mock.patch('experimental_framework.deployment_unit.DeploymentUnit')
330 @mock.patch('experimental_framework.benchmarking_unit.'
331 'heat.get_all_heat_templates')
332 def test_run_benchmarks_2_for_success(
333 self, mock_heat, mock_common_dep_unit, mock_dep_unit,
335 mock_temp_dir, mock_time, mock_rfc2544,
337 mock_heat.return_value = list()
338 mock_time.return_value = '12345'
339 mock_temp_dir.return_value = 'tests/data/test_templates/'
340 common.TEMPLATE_FILE_EXTENSION = '.yaml'
341 common.RESULT_DIR = 'tests/data/results/'
343 heat_template_name = 'VTC_base_single_vm_wait_'
344 openstack_credentials = {
348 heat_template_parameters = {
356 'rfc2544_throughput_benchmark.RFC2544ThroughputBenchmark',
360 bu = BenchmarkingUnit(heat_template_name,
361 openstack_credentials,
362 heat_template_parameters,
365 # bu.data_manager = DummyDataManager('tests/data/results/12345')
366 bu.template_files = ['VTC_base_single_vm_wait_1.yaml',
367 'VTC_base_single_vm_wait_2.yaml']
368 bu.benchmarks = [Dummy_2544('dummy', dict())]
369 common.DEPLOYMENT_UNIT = DummyDeploymentUnit(dict())
371 self.assertEqual(bu.benchmarks[0].init(True), 2)
372 self.assertEqual(bu.benchmarks[0].finalize(True), 0)
373 self.assertEqual(bu.benchmarks[0].run(True), 0)
375 @mock.patch('experimental_framework.common.LOG')
376 @mock.patch('experimental_framework.benchmarks.'
377 'rfc2544_throughput_benchmark', side_effect=Dummy_2544)
378 @mock.patch('time.time')
379 @mock.patch('experimental_framework.common.get_template_dir')
380 # @mock.patch('experimental_framework.data_manager.DataManager',
381 # side_effect=DummyDataManager)
382 @mock.patch('experimental_framework.common.DEPLOYMENT_UNIT')
383 @mock.patch('experimental_framework.deployment_unit.DeploymentUnit')
384 @mock.patch('experimental_framework.benchmarking_unit.'
385 'heat.get_all_heat_templates')
386 def test_get_benchmark_name_for_success(
387 self, mock_heat, mock_common_dep_unit, mock_dep_unit,
389 mock_temp_dir, mock_time, mock_rfc2544,
391 mock_heat.return_value = list()
392 mock_time.return_value = '12345'
393 mock_temp_dir.return_value = 'tests/data/test_templates/'
394 common.TEMPLATE_FILE_EXTENSION = '.yaml'
395 common.RESULT_DIR = 'tests/data/results/'
397 heat_template_name = 'VTC_base_single_vm_wait_'
398 openstack_credentials = {
402 heat_template_parameters = {
410 'rfc2544_throughput_benchmark.RFC2544ThroughputBenchmark',
414 bu = BenchmarkingUnit(heat_template_name,
415 openstack_credentials,
416 heat_template_parameters,
420 expected = 'rfc2544_throughput_benchmark.RFC2544ThroughputBenchmark_0'
421 output = bu.get_benchmark_name(
422 'rfc2544_throughput_benchmark.RFC2544ThroughputBenchmark')
423 self.assertEqual(expected, output)
425 expected = 'rfc2544_throughput_benchmark.RFC2544ThroughputBenchmark_1'
426 output = bu.get_benchmark_name(
427 'rfc2544_throughput_benchmark.RFC2544ThroughputBenchmark')
428 self.assertEqual(expected, output)
430 @mock.patch('experimental_framework.common.LOG')
431 @mock.patch('experimental_framework.benchmarks.'
432 'rfc2544_throughput_benchmark', side_effect=Dummy_2544)
433 @mock.patch('time.time')
434 @mock.patch('experimental_framework.common.get_template_dir')
435 # @mock.patch('experimental_framework.data_manager.DataManager',
436 # side_effect=DummyDataManager)
437 @mock.patch('experimental_framework.common.DEPLOYMENT_UNIT')
438 @mock.patch('experimental_framework.deployment_unit.DeploymentUnit')
439 @mock.patch('experimental_framework.benchmarking_unit.'
440 'heat.get_all_heat_templates')
441 def test_get_required_benchmarks_for_success(
442 self, mock_heat, mock_common_dep_unit, mock_dep_unit,
444 mock_temp_dir, mock_time, mock_rfc2544,
446 mock_heat.return_value = list()
447 mock_time.return_value = '12345'
448 mock_temp_dir.return_value = 'tests/data/test_templates/'
449 common.TEMPLATE_FILE_EXTENSION = '.yaml'
450 common.RESULT_DIR = 'tests/data/results/'
451 openstack_credentials = {
455 heat_template_parameters = {
463 'rfc2544_throughput_benchmark.RFC2544ThroughputBenchmark',
467 bu = BenchmarkingUnit('',
468 openstack_credentials,
469 heat_template_parameters,
473 ['rfc2544_throughput_benchmark.RFC2544ThroughputBenchmark']
474 output = bu.get_required_benchmarks(req_benchs)
475 self.assertEqual(len(req_benchs), 1)
476 self.assertEqual(output[0].__class__, Dummy_2544)