1 # Copyright 2015-2017 Intel Corporation.
3 # Licensed under the Apache License, Version 2.0 (the "License");
4 # you may not use this file except in compliance with the License.
5 # You may obtain a copy of the License at
7 # http://www.apache.org/licenses/LICENSE-2.0
9 # Unless required by applicable law or agreed to in writing, software
10 # distributed under the License is distributed on an "AS IS" BASIS,
11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 # See the License for the specific language governing permissions and
13 # limitations under the License.
15 """Task management helper functions and classes.
28 from conf import settings
29 from tools import systeminfo
35 """Get stdout value for ``subprocess`` calls.
39 if settings.getValue('VERBOSITY') != 'debug':
40 stdout = open(os.devnull, 'wb')
45 def run_task(cmd, logger, msg=None, check_error=False):
46 """Run task, report errors and log overall status.
48 Run given task using ``subprocess.Popen``. Log the commands
49 used and any errors generated. Prints stdout to screen if
50 in verbose mode and returns it regardless. Prints stderr to
53 :param cmd: Exact command to be executed
54 :param logger: Logger to write details to
55 :param msg: Message to be shown to user
56 :param check_error: Throw exception on error
58 :returns: (stdout, stderr)
60 def handle_error(exception):
61 """Handle errors by logging and optionally raising an exception.
64 'Unable to execute %(cmd)s. Exception: %(exception)s',
65 {'cmd': ' '.join(cmd), 'exception': exception})
71 my_encoding = locale.getdefaultlocale()[1]
76 # pylint: disable=too-many-nested-blocks
77 logger.debug('%s%s', CMD_PREFIX, ' '.join(cmd))
79 proc = subprocess.Popen(map(os.path.expanduser, cmd),
80 stdout=subprocess.PIPE,
81 stderr=subprocess.PIPE, bufsize=0)
84 reads = [proc.stdout.fileno(), proc.stderr.fileno()]
85 ret = select.select(reads, [], [])
88 if file_d == proc.stdout.fileno():
90 line = proc.stdout.readline()
93 if settings.getValue('VERBOSITY') == 'debug':
94 sys.stdout.write(line.decode(my_encoding))
96 if file_d == proc.stderr.fileno():
98 line = proc.stderr.readline()
101 sys.stderr.write(line.decode(my_encoding))
104 if proc.poll() is not None:
107 except OSError as ex:
111 ex = subprocess.CalledProcessError(proc.returncode, cmd, stderr)
114 return ('\n'.join(sout.decode(my_encoding).strip() for sout in stdout),
115 ('\n'.join(sout.decode(my_encoding).strip() for sout in stderr)))
117 def run_background_task(cmd, logger, msg):
118 """Run task in background and log when started.
120 Run given task using ``subprocess.Popen``. Log the command
121 used. Print stdout to screen if in verbose mode. Prints stderr
124 :param cmd: Exact command to be executed
125 :param logger: Logger to write details to
126 :param msg: Message to be shown to user
128 :returns: Process PID
131 logger.debug('%s%s', CMD_PREFIX, ' '.join(cmd))
133 proc = subprocess.Popen(map(os.path.expanduser, cmd), stdout=_get_stdout(), bufsize=0)
138 def run_interactive_task(cmd, logger, msg):
139 """Run a task interactively and log when started.
141 Run given task using ``pexpect.spawn``. Log the command used.
142 Performs neither validation of the process - if the process
143 successfully started or is still running - nor killing of the
144 process. The user must do both.
146 :param cmd: Exact command to be executed
147 :param logger: Logger to write details to
148 :param msg: Message to be shown to user
150 :returns: ``pexpect.child`` object
153 logger.debug('%s%s', CMD_PREFIX, cmd)
154 child = pexpect.spawnu(cmd)
156 if settings.getValue('VERBOSITY') == 'debug':
157 child.logfile_read = sys.stdout
161 def terminate_task_subtree(pid, signal='-15', sleep=10, logger=None):
162 """Terminate given process and all its children
164 Function will sent given signal to the process. In case
165 that process will not terminate within given sleep interval
166 and signal was not SIGKILL, then process will be killed by SIGKILL.
167 After that function will check if all children of the process
168 are terminated and if not the same terminating procedure is applied
169 on any living child (only one level of children is considered).
171 :param pid: Process ID to terminate
172 :param signal: Signal to be sent to the process
173 :param sleep: Maximum delay in seconds after signal is sent
174 :param logger: Logger to write details to
177 output = subprocess.check_output("pgrep -P " + str(pid), shell=True).decode().rstrip('\n')
178 except subprocess.CalledProcessError:
181 terminate_task(pid, signal, sleep, logger)
183 # just for case children were kept alive
184 children = output.split('\n')
185 for child in children:
186 terminate_task(child, signal, sleep, logger)
188 def terminate_task(pid, signal='-15', sleep=10, logger=None):
189 """Terminate process with given pid
191 Function will sent given signal to the process. In case
192 that process will not terminate within given sleep interval
193 and signal was not SIGKILL, then process will be killed by SIGKILL.
195 :param pid: Process ID to terminate
196 :param signal: Signal to be sent to the process
197 :param sleep: Maximum delay in seconds after signal is sent
198 :param logger: Logger to write details to
200 if systeminfo.pid_isalive(pid):
201 run_task(['sudo', 'kill', signal, str(pid)], logger)
202 logger.debug('Wait for process %s to terminate after signal %s', pid, signal)
203 for dummy in range(sleep):
205 if not systeminfo.pid_isalive(pid):
208 if signal.lstrip('-').upper() not in ('9', 'KILL', 'SIGKILL') and systeminfo.pid_isalive(pid):
209 terminate_task(pid, '-9', sleep, logger)
211 class Process(object):
212 """Control an instance of a long-running process.
214 This is basically a context-manager wrapper around the
215 ``run_interactive_task`` function above (with some extra helper
221 _logger = logging.getLogger(__name__)
224 _proc_name = 'unnamed process'
225 _relinquish_thread = None
230 """Start process instance using context manager.
235 def __exit__(self, type_, value, traceback):
236 """Shutdown process instance.
243 """Start process instance.
245 self._start_process()
246 if self._timeout > 0:
247 self._expect_process()
249 def _start_process(self):
250 """Start process instance.
252 cmd = ' '.join(settings.getValue('SHELL_CMD') +
253 ['"%s"' % ' '.join(self._cmd)])
255 self._child = run_interactive_task(cmd, self._logger,
256 'Starting %s...' % self._proc_name)
257 self._child.logfile = open(self._logfile, 'w')
259 def expect(self, msg, timeout=None):
260 """Expect string from process.
262 Expect string and die if not received.
264 :param msg: String to expect.
265 :param timeout: Time to wait for string.
269 self._expect_process(msg, timeout)
271 def _expect_process(self, msg=None, timeout=None):
272 """Expect string from process.
277 timeout = self._timeout
279 # we use exceptions rather than catching conditions in ``expect`` list
280 # as we want to fail catastrophically after handling; there is likely
281 # little we can do from within the scripts to fix issues such as
282 # hugepages not being mounted
284 self._child.expect([msg], timeout=timeout)
285 except pexpect.EOF as exc:
286 self._logger.critical(
287 'An error occurred. Please check the logs (%s) for more'
288 ' information. Exiting...', self._logfile)
290 except pexpect.TIMEOUT as exc:
291 self._logger.critical(
292 'Failed to execute in \'%d\' seconds. Please check the logs'
293 ' (%s) for more information. Exiting...',
294 timeout, self._logfile)
297 except (Exception, KeyboardInterrupt) as exc:
298 self._logger.critical('General exception raised. Exiting...')
302 def kill(self, signal='-15', sleep=10):
303 """Kill process instance if it is alive.
305 :param signal: signal to be sent to the process
306 :param sleep: delay in seconds after signal is sent
308 if self.is_running():
309 terminate_task_subtree(self._child.pid, signal, sleep, self._logger)
311 if self.is_relinquished():
312 self._relinquish_thread.join()
315 'Log available at %s', self._logfile)
317 def is_relinquished(self):
318 """Returns True if process is relinquished.
320 If relinquished the process is no longer controllable and can
323 :returns: True if process is relinquished, else False.
325 return self._relinquish_thread
327 def is_running(self):
328 """Returns True if process is running.
330 :returns: True if process is running, else False
332 return self._child and self._child.isalive()
334 def _affinitize_pid(self, core, pid):
335 """Affinitize a process with ``pid`` to ``core``.
337 :param core: Core to affinitize process to.
338 :param pid: Process ID to affinitize.
342 run_task(['sudo', 'taskset', '-c', '-p', str(core),
346 def affinitize(self, core):
347 """Affinitize process to a specific ``core``.
349 :param core: Core to affinitize process to.
353 self._logger.info('Affinitizing process')
355 if self.is_running():
356 self._affinitize_pid(core, self._child.pid)
358 class ContinueReadPrintLoop(threading.Thread):
359 """Thread to read output from child and log.
361 Taken from: https://github.com/pexpect/pexpect/issues/90
363 def __init__(self, child):
365 threading.Thread.__init__(self)
370 self.child.read_nonblocking()
371 except (pexpect.EOF, pexpect.TIMEOUT):
374 def relinquish(self):
375 """Relinquish control of process.
377 Give up control of application in order to ensure logging
378 continues for the application. After relinquishing control it
379 will no longer be possible to :func:`expect` anything.
381 This works around an issue described here:
383 https://github.com/pexpect/pexpect/issues/90
385 It is hoped that future versions of pexpect will avoid this
388 self._relinquish_thread = self.ContinueReadPrintLoop(self._child)
389 self._relinquish_thread.start()
392 class CustomProcess(Process):
393 """An sample implementation of ``Process``.
395 This is essentially a more detailed version of the
396 ``run_interactive_task`` function that checks for process execution
397 and kills the process (assuming use of the context manager).
399 def __init__(self, cmd, timeout, logfile, expect, name):
400 """Initialise process state.
402 :param cmd: Command to execute.
403 :param timeout: Time to wait for ``expect``.
404 :param logfile: Path to logfile.
405 :param expect: String to expect indicating startup. This is a
406 regex and should be escaped as such.
407 :param name: Name of process to use in logs.
412 self._logfile = logfile
413 self._expect = expect
414 self._proc_name = name
415 self._timeout = timeout