2 ##############################################################################
3 # Copyright (c) 2017 Huawei Technologies Co.,Ltd and others.
5 # All rights reserved. This program and the accompanying materials
6 # are made available under the terms of the Apache License, Version 2.0
7 # which accompanies this distribution, and is available at
8 # http://www.apache.org/licenses/LICENSE-2.0
9 ##############################################################################
10 '''This file realize the function of run posca multistack storage stress test
11 This file contain several part:
12 First is create a script to realize several threading run'''
14 import utils.logger as log
22 from utils.parser import Parser as conf_parser
23 import utils.env_prepare.quota_prepare as quota_prepare
24 import utils.env_prepare.stack_prepare as stack_prepare
25 import utils.infra_setup.runner.yardstick as yardstick_task
27 import utils.infra_setup.runner.docker_env as docker_env
29 # --------------------------------------------------
30 # logging configuration
31 # --------------------------------------------------
32 LOG = log.Logger(__name__).getLogger()
35 "action": "runTestCase",
40 "testcase": "multistack_storage_bottlenecks"
43 testfile = os.path.basename(__file__)
44 testcase, file_format = os.path.splitext(testfile)
45 cidr = "/home/opnfv/repos/yardstick/samples/storage_bottlenecks.yaml"
48 final_result = Queue.Queue()
51 def env_pre(test_config):
52 test_yardstick = False
53 if "yardstick" in test_config["contexts"].keys():
55 stack_prepare._prepare_env_daemon(test_yardstick)
56 quota_prepare.quota_env_prepare()
57 cmd = ('yardstick env prepare')
58 LOG.info("yardstick environment prepare!")
59 if(test_config["contexts"]['yardstick_envpre']):
60 yardstick_container = docker_env.yardstick_info['container']
61 stdout = docker_env.docker_exec_cmd(yardstick_container, cmd)
65 def testcase_parser(out_file="yardstick.out", **parameter_info):
66 cmd = yardstick_task.yardstick_command_parser(debug=runner_DEBUG,
69 parameter=parameter_info)
73 def do_test(test_config):
74 out_file = ("/tmp/yardstick_" + str(uuid.uuid4()) + ".out")
75 yardstick_container = docker_env.yardstick_info['container']
76 cmd = testcase_parser(out_file=out_file, **test_config)
78 stdout = docker_env.docker_exec_cmd(yardstick_container, cmd)
81 while loop_value < 60:
83 loop_value = loop_value + 1
84 with open(out_file) as f:
86 if data["status"] == 1:
87 LOG.info("yardstick run success")
88 LOG.info("%s" % data["result"]["testcases"])
90 elif data["status"] == 2:
91 LOG.error("yardstick error exit")
94 save_data = final_config_to_result(test_config, data)
98 def config_to_result(num, out_num, during_date):
103 final_data["read_iops"] = 0
104 final_data["read_bw"] = 0
105 final_data["read_lat"] = 0
106 final_data["write_iops"] = 0
107 final_data["write_bw"] = 0
108 final_data["write_lat"] = 0
110 test_result["number_of_stacks"] = float(num)
111 test_result["success_times"] = out_num
112 test_result["success_rate"] = out_num / num
113 test_result["duration_time"] = during_date
114 testdata["data_body"] = test_result
115 testdata["testcase"] = testcase
117 while not final_result.empty():
118 data = final_result.get()
119 final_data["read_iops"] += data["read_iops"]
120 final_data["read_bw"] += data["read_bw"]
121 if final_data["read_lat"] is 0:
122 final_data["read_lat"] = data["read_lat"]
124 final_data["read_lat"] += data["read_lat"]
125 final_data["read_lat"] = final_data["read_lat"]/2
126 final_data["write_iops"] += data["write_iops"]
127 final_data["write_bw"] += data["read_iops"]
128 if final_data["write_lat"] is 0:
129 final_data["write_lat"] = data["write_lat"]
131 final_data["write_lat"] += data["write_lat"]
132 final_data["write_lat"] = final_data["write_lat"]/2
134 testdata["test_value"] = final_data
135 LOG.info("Final testdata is %s" % testdata)
139 def final_config_to_result(test_config, test_result):
140 out_data = test_result["result"]["testcases"]
141 test_data = out_data["storage_bottlenecks"]["tc_data"]
143 testdata["read_iops"] = 0
144 testdata["read_bw"] = 0
145 testdata["read_lat"] = 0
146 testdata["write_iops"] = 0
147 testdata["write_bw"] = 0
148 testdata["write_lat"] = 0
149 print(testdata["read_iops"])
150 for result in test_data:
151 testdata["read_iops"] += result["data"]["read_iops"]
152 testdata["read_bw"] += result["data"]["read_bw"]
153 if testdata["read_lat"] is 0:
154 testdata["read_lat"] = result["data"]["read_lat"]
156 testdata["read_lat"] += result["data"]["read_lat"]
157 testdata["read_lat"] = testdata["read_lat"]/2
158 testdata["write_iops"] += result["data"]["write_iops"]
159 testdata["write_bw"] += result["data"]["write_bw"]
160 if testdata["write_lat"] is 0:
161 testdata["write_lat"] = result["data"]["write_lat"]
163 testdata["write_lat"] += result["data"]["write_lat"]
164 testdata["write_lat"] = testdata["write_lat"]/2
165 final_result.put(testdata)
170 def func_run(con_dic):
171 test_date = do_test(con_dic)
175 def run(test_config):
176 con_dic = test_config["load_manager"]
177 scenarios_conf = con_dic["scenarios"]
179 if test_config["contexts"]["yardstick_ip"] is None:
180 con_dic["contexts"]["yardstick_ip"] =\
181 conf_parser.ip_parser("yardstick_test_ip")
184 LOG.info("yardstick environment prepare done!")
186 stack_num = scenarios_conf["num_stack"]
187 test_num = conf_parser.str_to_list(scenarios_conf["num_thread"])
188 rw = scenarios_conf["rw"]
189 bs = scenarios_conf["bs"]
190 size = scenarios_conf["size"]
191 rwmixwrite = scenarios_conf["rwmixwrite"]
192 numjobs = scenarios_conf["num_jobs"]
193 direct = scenarios_conf["direct"]
194 volume_num = scenarios_conf["volume_num"]
196 for value in test_num:
200 # pool = multiprocessing.Pool(processes=num)
202 LOG.info("begin to run %s thread" % num)
204 starttime = datetime.datetime.now()
206 for i in xrange(0, num):
207 case_config = {"stack_num": int(stack_num),
208 "volume_num": volume_num,
212 "rwmixwrite": rwmixwrite,
215 tmp_thread = threading.Thread(target=func_run, args=(case_config,))
216 threadings.append(tmp_thread)
219 for one_thread in threadings:
222 result.append(q.get())
224 out_num = out_num + float(item)
228 endtime = datetime.datetime.now()
229 LOG.info("%s thread success %d times" % (num, out_num))
230 during_date = (endtime - starttime).seconds
232 data_reply = config_to_result(num, out_num, during_date)
233 conf_parser.result_to_file(data_reply, test_config["out_file"])
235 LOG.info('END POSCA stress multistack storage test')