1 # Copyright 2015-2017 Intel Corporation.
3 # Licensed under the Apache License, Version 2.0 (the "License");
4 # you may not use this file except in compliance with the License.
5 # You may obtain a copy of the License at
7 # http://www.apache.org/licenses/LICENSE-2.0
9 # Unless required by applicable law or agreed to in writing, software
10 # distributed under the License is distributed on an "AS IS" BASIS,
11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 # See the License for the specific language governing permissions and
13 # limitations under the License.
15 """Task management helper functions and classes.
28 from conf import settings
29 from tools import systeminfo
35 """Get stdout value for ``subprocess`` calls.
39 if settings.getValue('VERBOSITY') != 'debug':
40 stdout = open(os.devnull, 'wb')
45 def run_task(cmd, logger, msg=None, check_error=False):
46 """Run task, report errors and log overall status.
48 Run given task using ``subprocess.Popen``. Log the commands
49 used and any errors generated. Prints stdout to screen if
50 in verbose mode and returns it regardless. Prints stderr to
53 :param cmd: Exact command to be executed
54 :param logger: Logger to write details to
55 :param msg: Message to be shown to user
56 :param check_error: Throw exception on error
58 :returns: (stdout, stderr)
60 def handle_error(exception):
61 """Handle errors by logging and optionally raising an exception.
64 'Unable to execute %(cmd)s. Exception: %(exception)s',
65 {'cmd': ' '.join(cmd), 'exception': exception})
71 my_encoding = locale.getdefaultlocale()[1]
76 # pylint: disable=too-many-nested-blocks
77 logger.debug('%s%s', CMD_PREFIX, ' '.join(cmd))
79 proc = subprocess.Popen(map(os.path.expanduser, cmd),
80 stdout=subprocess.PIPE,
81 stderr=subprocess.PIPE, bufsize=0)
84 reads = [proc.stdout.fileno(), proc.stderr.fileno()]
85 ret = select.select(reads, [], [])
88 if file_d == proc.stdout.fileno():
90 line = proc.stdout.readline()
93 if settings.getValue('VERBOSITY') == 'debug':
94 sys.stdout.write(line.decode(my_encoding))
96 if file_d == proc.stderr.fileno():
98 line = proc.stderr.readline()
101 sys.stderr.write(line.decode(my_encoding))
104 if proc.poll() is not None:
107 except OSError as ex:
111 ex = subprocess.CalledProcessError(proc.returncode, cmd, stderr)
114 return ('\n'.join(sout.decode(my_encoding).strip() for sout in stdout),
115 ('\n'.join(sout.decode(my_encoding).strip() for sout in stderr)))
117 def update_pids(pid):
118 """update list of running pids, so they can be terminated at the end
120 pids = settings.getValue('_EXECUTED_PIDS')
122 settings.setValue('_EXECUTED_PIDS', pids)
124 def run_background_task(cmd, logger, msg):
125 """Run task in background and log when started.
127 Run given task using ``subprocess.Popen``. Log the command
128 used. Print stdout to screen if in verbose mode. Prints stderr
131 :param cmd: Exact command to be executed
132 :param logger: Logger to write details to
133 :param msg: Message to be shown to user
135 :returns: Process PID
138 logger.debug('%s%s', CMD_PREFIX, ' '.join(cmd))
140 proc = subprocess.Popen(map(os.path.expanduser, cmd), stdout=_get_stdout(), bufsize=0)
142 update_pids(proc.pid)
147 def run_interactive_task(cmd, logger, msg):
148 """Run a task interactively and log when started.
150 Run given task using ``pexpect.spawn``. Log the command used.
151 Performs neither validation of the process - if the process
152 successfully started or is still running - nor killing of the
153 process. The user must do both.
155 :param cmd: Exact command to be executed
156 :param logger: Logger to write details to
157 :param msg: Message to be shown to user
159 :returns: ``pexpect.child`` object
162 logger.debug('%s%s', CMD_PREFIX, cmd)
163 child = pexpect.spawnu(cmd)
165 if settings.getValue('VERBOSITY') == 'debug':
166 child.logfile_read = sys.stdout
170 def terminate_task_subtree(pid, signal='-15', sleep=10, logger=None):
171 """Terminate given process and all its children
173 Function will sent given signal to the process. In case
174 that process will not terminate within given sleep interval
175 and signal was not SIGKILL, then process will be killed by SIGKILL.
176 After that function will check if all children of the process
177 are terminated and if not the same terminating procedure is applied
178 on any living child (only one level of children is considered).
180 :param pid: Process ID to terminate
181 :param signal: Signal to be sent to the process
182 :param sleep: Maximum delay in seconds after signal is sent
183 :param logger: Logger to write details to
186 children = subprocess.check_output("pgrep -P " + str(pid), shell=True).decode().rstrip('\n').split()
187 except subprocess.CalledProcessError:
190 terminate_task(pid, signal, sleep, logger)
192 # just for case children were kept alive
193 for child in children:
194 terminate_task(child, signal, sleep, logger)
196 def terminate_task(pid, signal='-15', sleep=10, logger=None):
197 """Terminate process with given pid
199 Function will sent given signal to the process. In case
200 that process will not terminate within given sleep interval
201 and signal was not SIGKILL, then process will be killed by SIGKILL.
203 :param pid: Process ID to terminate
204 :param signal: Signal to be sent to the process
205 :param sleep: Maximum delay in seconds after signal is sent
206 :param logger: Logger to write details to
208 if systeminfo.pid_isalive(pid):
209 run_task(['sudo', 'kill', signal, str(pid)], logger)
210 logger.debug('Wait for process %s to terminate after signal %s', pid, signal)
211 for dummy in range(sleep):
213 if not systeminfo.pid_isalive(pid):
216 if signal.lstrip('-').upper() not in ('9', 'KILL', 'SIGKILL') and systeminfo.pid_isalive(pid):
217 terminate_task(pid, '-9', sleep, logger)
219 pids = settings.getValue('_EXECUTED_PIDS')
222 settings.setValue('_EXECUTED_PIDS', pids)
224 def terminate_all_tasks(logger):
225 """Terminate all processes executed by vsperf, just for case they were not
226 terminated by standard means.
228 pids = settings.getValue('_EXECUTED_PIDS')
230 logger.debug('Following processes will be terminated: %s', pids)
232 terminate_task_subtree(pid, logger=logger)
233 settings.setValue('_EXECUTED_PIDS', [])
235 class Process(object):
236 """Control an instance of a long-running process.
238 This is basically a context-manager wrapper around the
239 ``run_interactive_task`` function above (with some extra helper
245 _logger = logging.getLogger(__name__)
248 _proc_name = 'unnamed process'
249 _relinquish_thread = None
254 """Start process instance using context manager.
259 def __exit__(self, type_, value, traceback):
260 """Shutdown process instance.
267 """Start process instance.
269 self._start_process()
270 if self._timeout > 0:
271 self._expect_process()
273 def _start_process(self):
274 """Start process instance.
276 cmd = ' '.join(settings.getValue('SHELL_CMD') +
277 ['"%s"' % ' '.join(self._cmd)])
279 self._child = run_interactive_task(cmd, self._logger,
280 'Starting %s...' % self._proc_name)
281 self._child.logfile = open(self._logfile, 'w')
283 def expect(self, msg, timeout=None):
284 """Expect string from process.
286 Expect string and die if not received.
288 :param msg: String to expect.
289 :param timeout: Time to wait for string.
293 self._expect_process(msg, timeout)
295 def _expect_process(self, msg=None, timeout=None):
296 """Expect string from process.
301 timeout = self._timeout
303 # we use exceptions rather than catching conditions in ``expect`` list
304 # as we want to fail catastrophically after handling; there is likely
305 # little we can do from within the scripts to fix issues such as
306 # hugepages not being mounted
308 self._child.expect([msg], timeout=timeout)
309 except pexpect.EOF as exc:
310 self._logger.critical(
311 'An error occurred. Please check the logs (%s) for more'
312 ' information. Exiting...', self._logfile)
314 except pexpect.TIMEOUT as exc:
315 self._logger.critical(
316 'Failed to execute in \'%d\' seconds. Please check the logs'
317 ' (%s) for more information. Exiting...',
318 timeout, self._logfile)
321 except (Exception, KeyboardInterrupt) as exc:
322 self._logger.critical('General exception raised. Exiting...')
326 def kill(self, signal='-15', sleep=10):
327 """Kill process instance if it is alive.
329 :param signal: signal to be sent to the process
330 :param sleep: delay in seconds after signal is sent
332 if self.is_running():
333 terminate_task_subtree(self._child.pid, signal, sleep, self._logger)
335 if self.is_relinquished():
336 self._relinquish_thread.join()
339 'Log available at %s', self._logfile)
341 def is_relinquished(self):
342 """Returns True if process is relinquished.
344 If relinquished the process is no longer controllable and can
347 :returns: True if process is relinquished, else False.
349 return self._relinquish_thread
351 def is_running(self):
352 """Returns True if process is running.
354 :returns: True if process is running, else False
356 return self._child and self._child.isalive()
358 def _affinitize_pid(self, core, pid):
359 """Affinitize a process with ``pid`` to ``core``.
361 :param core: Core to affinitize process to.
362 :param pid: Process ID to affinitize.
366 run_task(['sudo', 'taskset', '-c', '-p', str(core),
370 def affinitize(self, core):
371 """Affinitize process to a specific ``core``.
373 :param core: Core to affinitize process to.
377 self._logger.info('Affinitizing process')
379 if self.is_running():
380 self._affinitize_pid(core, self._child.pid)
382 class ContinueReadPrintLoop(threading.Thread):
383 """Thread to read output from child and log.
385 Taken from: https://github.com/pexpect/pexpect/issues/90
387 def __init__(self, child):
389 threading.Thread.__init__(self)
394 self.child.read_nonblocking()
395 except (pexpect.EOF, pexpect.TIMEOUT):
398 def relinquish(self):
399 """Relinquish control of process.
401 Give up control of application in order to ensure logging
402 continues for the application. After relinquishing control it
403 will no longer be possible to :func:`expect` anything.
405 This works around an issue described here:
407 https://github.com/pexpect/pexpect/issues/90
409 It is hoped that future versions of pexpect will avoid this
412 self._relinquish_thread = self.ContinueReadPrintLoop(self._child)
413 self._relinquish_thread.start()
416 class CustomProcess(Process):
417 """An sample implementation of ``Process``.
419 This is essentially a more detailed version of the
420 ``run_interactive_task`` function that checks for process execution
421 and kills the process (assuming use of the context manager).
423 def __init__(self, cmd, timeout, logfile, expect, name):
424 """Initialise process state.
426 :param cmd: Command to execute.
427 :param timeout: Time to wait for ``expect``.
428 :param logfile: Path to logfile.
429 :param expect: String to expect indicating startup. This is a
430 regex and should be escaped as such.
431 :param name: Name of process to use in logs.
436 self._logfile = logfile
437 self._expect = expect
438 self._proc_name = name
439 self._timeout = timeout