1 # Copyright 2014: Mirantis Inc.
4 # Licensed under the Apache License, Version 2.0 (the "License"); you may
5 # not use this file except in compliance with the License. You may obtain
6 # a copy of the License at
8 # http://www.apache.org/licenses/LICENSE-2.0
10 # Unless required by applicable law or agreed to in writing, software
11 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
12 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
13 # License for the specific language governing permissions and limitations
16 # yardstick comment: this is a modified copy of
17 # rally/rally/benchmark/runners/base.py
19 from __future__ import absolute_import
22 import multiprocessing
26 from subprocess import CalledProcessError
30 from six.moves.queue import Empty
32 import yardstick.common.utils as utils
33 from yardstick.benchmark.scenarios import base as base_scenario
34 from yardstick.dispatcher.base import Base as DispatcherBase
36 log = logging.getLogger(__name__)
39 def _execute_shell_command(command):
40 """execute shell script with error handling"""
43 output = subprocess.check_output(command, shell=True)
44 except CalledProcessError:
46 output = traceback.format_exc()
47 log.error("exec command '%s' error:\n ", command)
48 log.error(traceback.format_exc())
50 return exitcode, output
53 def _single_action(seconds, command, queue):
54 """entrypoint for the single action process"""
55 log.debug("single action, fires after %d seconds (from now)", seconds)
57 log.debug("single action: executing command: '%s'", command)
58 ret_code, data = _execute_shell_command(command)
60 log.error("single action error! command:%s", command)
61 queue.put({'single-action-data': data})
63 log.debug("single action data: \n%s", data)
64 queue.put({'single-action-data': data})
67 def _periodic_action(interval, command, queue):
68 """entrypoint for the periodic action process"""
69 log.debug("periodic action, fires every: %d seconds", interval)
73 time_spent += interval
74 log.debug("periodic action, executing command: '%s'", command)
75 ret_code, data = _execute_shell_command(command)
77 log.error("periodic action error! command:%s", command)
78 queue.put({'periodic-action-data': data})
80 log.debug("periodic action data: \n%s", data)
81 queue.put({'periodic-action-data': data})
88 def get_cls(runner_type):
89 """return class of specified type"""
90 for runner in utils.itersubclasses(Runner):
91 if runner_type == runner.__execution_type__:
93 raise RuntimeError("No such runner_type %s" % runner_type)
97 """return a list of known runner type (class) names"""
99 for runner in utils.itersubclasses(Runner):
105 """Returns instance of a scenario runner for execution type.
107 return Runner.get_cls(runner_cfg["type"])(runner_cfg)
111 """Release the runner"""
112 if runner in Runner.runners:
113 Runner.runners.remove(runner)
116 def terminate(runner):
117 """Terminate the runner"""
118 if runner.process and runner.process.is_alive():
119 runner.process.terminate()
123 """Terminate all runners (subprocesses)"""
124 log.debug("Terminating all runners", exc_info=True)
126 # release dumper process as some errors before any runner is created
127 if not Runner.runners:
130 for runner in Runner.runners:
131 log.debug("Terminating runner: %s", runner)
133 runner.process.terminate()
134 runner.process.join()
135 if runner.periodic_action_process:
136 log.debug("Terminating periodic action process")
137 runner.periodic_action_process.terminate()
138 runner.periodic_action_process = None
139 Runner.release(runner)
141 def __init__(self, config):
143 self.case_name = None
145 self.periodic_action_process = None
146 self.output_queue = multiprocessing.Queue()
147 self.result_queue = multiprocessing.Queue()
149 self.aborted = multiprocessing.Event()
150 Runner.runners.append(self)
152 def run_post_stop_action(self):
153 """run a potentially configured post-stop action"""
154 if "post-stop-action" in self.config:
155 command = self.config["post-stop-action"]["command"]
156 log.debug("post stop action: command: '%s'", command)
157 ret_code, data = _execute_shell_command(command)
159 log.error("post action error! command:%s", command)
160 self.result_queue.put({'post-stop-action-data': data})
162 log.debug("post-stop data: \n%s", data)
163 self.result_queue.put({'post-stop-action-data': data})
165 def _run_benchmark(self, cls, method_name, scenario_cfg, context_cfg):
166 raise NotImplementedError
168 def run(self, scenario_cfg, context_cfg):
169 scenario_type = scenario_cfg["type"]
170 class_name = base_scenario.Scenario.get(scenario_type)
171 path_split = class_name.split(".")
172 module_path = ".".join(path_split[:-1])
173 module = importlib.import_module(module_path)
174 cls = getattr(module, path_split[-1])
176 self.config['object'] = class_name
177 self.case_name = scenario_cfg['tc']
178 self.task_id = scenario_cfg['task_id']
181 # run a potentially configured pre-start action
182 if "pre-start-action" in self.config:
183 command = self.config["pre-start-action"]["command"]
184 log.debug("pre start action: command: '%s'", command)
185 ret_code, data = _execute_shell_command(command)
187 log.error("pre-start action error! command:%s", command)
188 self.result_queue.put({'pre-start-action-data': data})
190 log.debug("pre-start data: \n%s", data)
191 self.result_queue.put({'pre-start-action-data': data})
193 if "single-shot-action" in self.config:
194 single_action_process = multiprocessing.Process(
195 target=_single_action,
196 name="single-shot-action",
197 args=(self.config["single-shot-action"]["after"],
198 self.config["single-shot-action"]["command"],
200 single_action_process.start()
202 if "periodic-action" in self.config:
203 self.periodic_action_process = multiprocessing.Process(
204 target=_periodic_action,
205 name="periodic-action",
206 args=(self.config["periodic-action"]["interval"],
207 self.config["periodic-action"]["command"],
209 self.periodic_action_process.start()
211 self._run_benchmark(cls, "run", scenario_cfg, context_cfg)
214 """Abort the execution of a scenario"""
217 QUEUE_JOIN_INTERVAL = 5
219 def poll(self, timeout=QUEUE_JOIN_INTERVAL):
220 self.process.join(timeout)
221 return self.process.exitcode
223 def join(self, outputs, result, interval=QUEUE_JOIN_INTERVAL):
224 while self.process.exitcode is None:
225 # drain the queue while we are running otherwise we won't terminate
226 outputs.update(self.get_output())
227 result.extend(self.get_result())
228 self.process.join(interval)
229 # drain after the process has exited
230 outputs.update(self.get_output())
231 result.extend(self.get_result())
233 self.process.terminate()
234 if self.periodic_action_process:
235 self.periodic_action_process.join(1)
236 self.periodic_action_process.terminate()
237 self.periodic_action_process = None
239 self.run_post_stop_action()
240 return self.process.exitcode
242 def get_output(self):
244 while not self.output_queue.empty():
245 log.debug("output_queue size %s", self.output_queue.qsize())
247 result.update(self.output_queue.get(True, 1))
252 def get_result(self):
255 dispatcher = self.config['output_config']['DEFAULT']['dispatcher']
256 output_in_influxdb = 'influxdb' in dispatcher
258 while not self.result_queue.empty():
259 log.debug("result_queue size %s", self.result_queue.qsize())
261 one_record = self.result_queue.get(True, 1)
265 if output_in_influxdb:
266 self._output_to_influxdb(one_record)
268 result.append(one_record)
271 def _output_to_influxdb(self, record):
272 dispatchers = DispatcherBase.get(self.config['output_config'])
273 dispatcher = next((d for d in dispatchers if d.__dispatcher_type__ == 'Influxdb'))
274 dispatcher.upload_one_record(record, self.case_name, '', task_id=self.task_id)